Cold-Air-Networks/mirror-llvm-project
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 7

AMDGPU/GlobalISel: Handle G_BITCAST for 16 bit extendedLLTs (#208750)

Browse source 
Handle bitcast between i16 and f16/bf16.
For true16 this was already legal, make it legal in regbanklegalize as well.
For non-true16 widen it using G_ANYEXT to i32 and G_TRUNC to dst.
The "i32 G_ANYEXT f16/bf16" and "f16/bf16 G_TRUNC i32" are already legal,
for example these are generated by common CallLowering argument lowering.
This commit is contained in:
Petar Avramovic 2026-07-20 12:48:21 +02:00 committed by GitHub
parent 195df3d465
commit 91ad9f0ce9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
186 changed files with 24994 additions and 20597 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
View file

@ -3640,6 +3640,26 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
return Legalized;
}
case TargetOpcode::G_BITCAST:
if (WideTy.isVector())
return UnableToLegalize;
Observer.changingInstr(MI);
if (TypeIdx == 0)
widenScalarDst(MI, WideTy, 0, TargetOpcode::G_TRUNC);
else
widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
Observer.changedInstr(MI);
Register Dst = MI.getOperand(0).getReg();
Register Src = MI.getOperand(1).getReg();
if (MRI.getType(Dst) == MRI.getType(Src)) {
Observer.changingAllUsesOfReg(MRI, Dst);
MRI.replaceRegWith(Dst, Src);
Observer.finishedChangingAllUsesOfReg();
MI.eraseFromParent();
}
return Legalized;
}
}

8
llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
View file

@ -293,6 +293,10 @@ static LegalityPredicate elementTypeIsLegal(unsigned TypeIdx) {
};
}
const LLT I16 = LLT::integer(16);
constexpr LLT F16 = LLT::float16();
constexpr LLT BF16 = LLT::bfloat16();
constexpr LLT S1 = LLT::scalar(1);
constexpr LLT S8 = LLT::scalar(8);
constexpr LLT S16 = LLT::scalar(16);
@ -926,6 +930,10 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
getActionDefinitionsBuilder(G_BITCAST)
// Don't worry about the size constraint.
.legalIf(all(isRegisterClassType(ST, 0), isRegisterClassType(ST, 1)))
.widenScalarIf(all(typeInSet(0, {I16, F16, BF16}), isScalar(1)),
changeTo(0, LLT::integer(32)))
.widenScalarIf(all(isScalar(0), typeInSet(1, {I16, F16, BF16})),
changeTo(1, LLT::integer(32)))
.lower();
getActionDefinitionsBuilder(G_CONSTANT)

2
llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
View file

@ -753,6 +753,8 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
.Any({{DivBRC}, {{VgprBRC}, {VgprBRC}}});
addRulesForGOpcs({G_BITCAST})
.Any({{UniS16}, {{Sgpr16}, {Sgpr16}}})
.Any({{DivS16}, {{Vgpr16}, {Vgpr16}}})
.Any({{UniBRC}, {{SgprBRC}, {SgprBRC}}})
.Any({{DivBRC}, {{VgprBRC}, {VgprBRC}}});

2
llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll
View file

@ -731,6 +731,7 @@ define amdgpu_ps i16 @s_ashr_i16_15(i16 inreg %value) {
define amdgpu_ps half @ashr_i16_sv(i16 inreg %value, i16 %amount) {
; GFX6-LABEL: ashr_i16_sv:
; GFX6: ; %bb.0:
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX6-NEXT: s_sext_i32_i16 s0, s0
; GFX6-NEXT: v_ashr_i32_e32 v0, s0, v0
; GFX6-NEXT: ; return to shader part epilog
@ -767,6 +768,7 @@ define amdgpu_ps half @ashr_i16_sv(i16 inreg %value, i16 %amount) {
define amdgpu_ps half @ashr_i16_vs(i16 %value, i16 inreg %amount) {
; GFX6-LABEL: ashr_i16_vs:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
; GFX6-NEXT: v_bfe_i32 v0, v0, 0, 16
; GFX6-NEXT: v_ashrrev_i32_e32 v0, s0, v0
; GFX6-NEXT: ; return to shader part epilog

60
llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_load_flat.ll
View file

@ -121,28 +121,12 @@ define bfloat @atomic_load_flat_monotonic_bf16(ptr %ptr) {
}
define i32 @atomic_load_flat_monotonic_f16_zext_to_i32(ptr %ptr) {
; GFX7-LABEL: atomic_load_flat_monotonic_f16_zext_to_i32:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: flat_load_ushort v0, v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: atomic_load_flat_monotonic_f16_zext_to_i32:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_ushort v0, v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_flat_monotonic_f16_zext_to_i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: flat_load_ushort v0, v[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GCN-LABEL: atomic_load_flat_monotonic_f16_zext_to_i32:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: flat_load_ushort v0, v[0:1] glc
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN-NEXT: s_setpc_b64 s[30:31]
%load = load atomic half, ptr %ptr monotonic, align 2
%cast = bitcast half %load to i16
%ext = zext i16 %cast to i32
@ -150,28 +134,12 @@ define i32 @atomic_load_flat_monotonic_f16_zext_to_i32(ptr %ptr) {
}
define i32 @atomic_load_flat_monotonic_bf16_zext_to_i32(ptr %ptr) {
; GFX7-LABEL: atomic_load_flat_monotonic_bf16_zext_to_i32:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: flat_load_ushort v0, v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: atomic_load_flat_monotonic_bf16_zext_to_i32:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_ushort v0, v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_flat_monotonic_bf16_zext_to_i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: flat_load_ushort v0, v[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GCN-LABEL: atomic_load_flat_monotonic_bf16_zext_to_i32:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: flat_load_ushort v0, v[0:1] glc
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN-NEXT: s_setpc_b64 s[30:31]
%load = load atomic bfloat, ptr %ptr monotonic, align 2
%cast = bitcast bfloat %load to i16
%ext = zext i16 %cast to i32
@ -270,7 +238,7 @@ define <2 x i16> @atomic_load_flat_monotonic_i16_d16_lo_vector_insert(ptr %ptr,
; GFX7-NEXT: flat_load_ushort v0, v[0:1] glc
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: v_or_b32_e32 v0, v1, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: atomic_load_flat_monotonic_i16_d16_lo_vector_insert:
@ -279,7 +247,7 @@ define <2 x i16> @atomic_load_flat_monotonic_i16_d16_lo_vector_insert(ptr %ptr,
; GFX8-NEXT: flat_load_ushort v0, v[0:1] glc
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_flat_monotonic_i16_d16_lo_vector_insert:

22
llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_load_global.ll
View file

@ -367,8 +367,7 @@ define i32 @atomic_load_global_monotonic_f16_zext_to_i32(ptr addrspace(1) %ptr)
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
; GFX6-NEXT: s_mov_b32 s7, 0x100f000
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: buffer_load_ushort v0, v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
@ -385,7 +384,6 @@ define i32 @atomic_load_global_monotonic_f16_zext_to_i32(ptr addrspace(1) %ptr)
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_ushort v0, v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_global_monotonic_f16_zext_to_i32:
@ -393,7 +391,6 @@ define i32 @atomic_load_global_monotonic_f16_zext_to_i32(ptr addrspace(1) %ptr)
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_ushort v0, v[0:1], off glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
%load = load atomic half, ptr addrspace(1) %ptr monotonic, align 2
%cast = bitcast half %load to i16
@ -407,8 +404,7 @@ define i32 @atomic_load_global_monotonic_bf16_zext_to_i32(ptr addrspace(1) %ptr)
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
; GFX6-NEXT: s_mov_b32 s7, 0x100f000
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: buffer_load_ushort v0, v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
@ -425,7 +421,6 @@ define i32 @atomic_load_global_monotonic_bf16_zext_to_i32(ptr addrspace(1) %ptr)
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_ushort v0, v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_global_monotonic_bf16_zext_to_i32:
@ -433,7 +428,6 @@ define i32 @atomic_load_global_monotonic_bf16_zext_to_i32(ptr addrspace(1) %ptr)
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_ushort v0, v[0:1], off glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
%load = load atomic bfloat, ptr addrspace(1) %ptr monotonic, align 2
%cast = bitcast bfloat %load to i16
@ -488,8 +482,7 @@ define <2 x i16> @atomic_load_global_monotonic_i16_d16_hi_vector_insert(ptr addr
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
; GFX6-NEXT: s_mov_b32 s7, 0x100f000
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: buffer_load_ushort v0, v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v2
; GFX6-NEXT: s_waitcnt vmcnt(0)
@ -586,12 +579,11 @@ define <2 x i16> @atomic_load_global_monotonic_i16_d16_lo_vector_insert(ptr addr
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
; GFX6-NEXT: s_mov_b32 s7, 0x100f000
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: buffer_load_ushort v0, v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: v_or_b32_e32 v0, v1, v0
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: atomic_load_global_monotonic_i16_d16_lo_vector_insert:
@ -600,7 +592,7 @@ define <2 x i16> @atomic_load_global_monotonic_i16_d16_lo_vector_insert(ptr addr
; GFX7-NEXT: flat_load_ushort v0, v[0:1] glc
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: v_or_b32_e32 v0, v1, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: atomic_load_global_monotonic_i16_d16_lo_vector_insert:
@ -609,7 +601,7 @@ define <2 x i16> @atomic_load_global_monotonic_i16_d16_lo_vector_insert(ptr addr
; GFX8-NEXT: flat_load_ushort v0, v[0:1] glc
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_global_monotonic_i16_d16_lo_vector_insert:

8
llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_load_local_2.ll
View file

@ -297,7 +297,6 @@ define i32 @atomic_load_local_monotonic_f16_zext_to_i32(ptr addrspace(3) %ptr) {
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: ds_read_u16 v0, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_local_monotonic_f16_zext_to_i32:
@ -305,7 +304,6 @@ define i32 @atomic_load_local_monotonic_f16_zext_to_i32(ptr addrspace(3) %ptr) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: ds_read_u16 v0, v0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
%load = load atomic half, ptr addrspace(3) %ptr monotonic, align 2
%cast = bitcast half %load to i16
@ -328,7 +326,6 @@ define i32 @atomic_load_local_monotonic_bf16_zext_to_i32(ptr addrspace(3) %ptr)
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: ds_read_u16 v0, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_local_monotonic_bf16_zext_to_i32:
@ -336,7 +333,6 @@ define i32 @atomic_load_local_monotonic_bf16_zext_to_i32(ptr addrspace(3) %ptr)
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: ds_read_u16 v0, v0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
%load = load atomic bfloat, ptr addrspace(3) %ptr monotonic, align 2
%cast = bitcast bfloat %load to i16
@ -458,7 +454,7 @@ define <2 x i16> @atomic_load_local_monotonic_i16_d16_lo_vector_insert(ptr addrs
; GFX7-NEXT: ds_read_u16 v0, v0
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: v_or_b32_e32 v0, v1, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: atomic_load_local_monotonic_i16_d16_lo_vector_insert:
@ -468,7 +464,7 @@ define <2 x i16> @atomic_load_local_monotonic_i16_d16_lo_vector_insert(ptr addrs
; GFX8-NEXT: ds_read_u16 v0, v0
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: atomic_load_local_monotonic_i16_d16_lo_vector_insert:

12
llvm/test/CodeGen/AMDGPU/GlobalISel/clamp-fmed3-const-combine.ll
View file

@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-amd-mesa3d < %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.70-amd-mesa3d -mattr=-real-true16 < %s | FileCheck -check-prefix=GFX1170 %s
; RUN: llc -global-isel -mtriple=amdgpu10.10-amd-mesa3d < %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -global-isel -mtriple=amdgpu11.70-amd-mesa3d -mattr=-real-true16 < %s | FileCheck -check-prefix=GFX1170 %s
; RUN: llc -global-isel -mtriple=amdgpu12.00-amd-mesa3d -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-TRUE16 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00-amd-mesa3d -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-FAKE16 %s
; RUN: llc -global-isel -mtriple=amdgpu12.00-amd-mesa3d -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-FAKE16 %s
define float @test_fmed3_f32_known_nnan_ieee_true(float %a) #0 {
; GFX10-LABEL: test_fmed3_f32_known_nnan_ieee_true:
@ -35,13 +35,13 @@ define half @test_fmed3_f16_known_nnan_ieee_false(half %a) #1 {
; GFX10-LABEL: test_fmed3_f16_known_nnan_ieee_false:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_f16_e64 v0, v0, v0 clamp
; GFX10-NEXT: v_mul_f16_e64 v0, v0, 2.0 clamp
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX1170-LABEL: test_fmed3_f16_known_nnan_ieee_false:
; GFX1170: ; %bb.0:
; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1170-NEXT: v_add_f16_e64 v0, v0, v0 clamp
; GFX1170-NEXT: v_mul_f16_e64 v0, v0, 2.0 clamp
; GFX1170-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-TRUE16-LABEL: test_fmed3_f16_known_nnan_ieee_false:
@ -61,7 +61,7 @@ define half @test_fmed3_f16_known_nnan_ieee_false(half %a) #1 {
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: v_add_f16_e64 v0, v0, v0 clamp
; GFX12-FAKE16-NEXT: v_mul_f16_e64 v0, v0, 2.0 clamp
; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%fmul = fmul half %a, 2.0
%fmed = call nnan half @llvm.amdgcn.fmed3.f16(half %fmul, half 0.0, half 1.0)

24
llvm/test/CodeGen/AMDGPU/GlobalISel/clamp-minmax-const-combine.ll
View file

@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-amd-mesa3d < %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.70-amd-mesa3d -mattr=-real-true16 < %s | FileCheck -check-prefix=GFX1170 %s
; RUN: llc -global-isel -mtriple=amdgpu10.10-amd-mesa3d < %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -global-isel -mtriple=amdgpu11.70-amd-mesa3d -mattr=-real-true16 < %s | FileCheck -check-prefix=GFX1170 %s
; RUN: llc -global-isel -mtriple=amdgpu12.00-amd-mesa3d -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-TRUE16 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00-amd-mesa3d -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-FAKE16 %s
; RUN: llc -global-isel -mtriple=amdgpu12.00-amd-mesa3d -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-FAKE16 %s
define float @test_min_max_ValK0_K1_f32(float %a) #0 {
; GFX10-LABEL: test_min_max_ValK0_K1_f32:
@ -65,13 +65,13 @@ define half @test_min_K1max_ValK0_f16(half %a) #2 {
; GFX10-LABEL: test_min_K1max_ValK0_f16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_f16_e64 v0, v0, v0 clamp
; GFX10-NEXT: v_mul_f16_e64 v0, v0, 2.0 clamp
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX1170-LABEL: test_min_K1max_ValK0_f16:
; GFX1170: ; %bb.0:
; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1170-NEXT: v_add_f16_e64 v0, v0, v0 clamp
; GFX1170-NEXT: v_mul_f16_e64 v0, v0, 2.0 clamp
; GFX1170-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-TRUE16-LABEL: test_min_K1max_ValK0_f16:
@ -91,7 +91,7 @@ define half @test_min_K1max_ValK0_f16(half %a) #2 {
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: v_add_f16_e64 v0, v0, v0 clamp
; GFX12-FAKE16-NEXT: v_mul_f16_e64 v0, v0, 2.0 clamp
; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%fmul = fmul half %a, 2.0
%maxnum = call half @llvm.maxnum.f16(half %fmul, half 0.0)
@ -217,17 +217,13 @@ define half @test_max_K0min_ValK1_f16(half %a) #0 {
; GFX10-LABEL: test_max_K0min_ValK1_f16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_f16_e32 v0, v0, v0
; GFX10-NEXT: v_min_f16_e32 v0, 1.0, v0
; GFX10-NEXT: v_max_f16_e32 v0, 0, v0
; GFX10-NEXT: v_mul_f16_e64 v0, v0, 2.0 clamp
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX1170-LABEL: test_max_K0min_ValK1_f16:
; GFX1170: ; %bb.0:
; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1170-NEXT: v_add_f16_e32 v0, v0, v0
; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1170-NEXT: v_minmax_num_f16 v0, v0, 1.0, 0
; GFX1170-NEXT: v_mul_f16_e64 v0, v0, 2.0 clamp
; GFX1170-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-TRUE16-LABEL: test_max_K0min_ValK1_f16:
@ -247,9 +243,7 @@ define half @test_max_K0min_ValK1_f16(half %a) #0 {
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: v_add_f16_e32 v0, v0, v0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_minmax_num_f16 v0, v0, 1.0, 0
; GFX12-FAKE16-NEXT: v_mul_f16_e64 v0, v0, 2.0 clamp
; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%fmul = fmul half %a, 2.0
%minnum = call nnan half @llvm.minnum.f16(half %fmul, half 1.0)

60
llvm/test/CodeGen/AMDGPU/GlobalISel/combine-binop-s64-with-s32-mask.mir
View file

@ -12,9 +12,9 @@ body: |
; CHECK: liveins: $sgpr0_sgpr1, $sgpr2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $sgpr0_sgpr1
; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](s32), [[UV1]](s32)
; CHECK-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[COPY]](s64)
; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](i32), [[UV1]](i32)
; CHECK-NEXT: $sgpr0_sgpr1 = COPY [[MV]](s64)
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
%0:_(s64) = COPY $sgpr0_sgpr1
@ -33,9 +33,9 @@ body: |
; CHECK: liveins: $sgpr0_sgpr1, $sgpr2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $sgpr0_sgpr1
; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](s32), [[UV1]](s32)
; CHECK-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[COPY]](s64)
; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](i32), [[UV1]](i32)
; CHECK-NEXT: $sgpr0_sgpr1 = COPY [[MV]](s64)
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
%0:_(s64) = COPY $sgpr0_sgpr1
@ -54,10 +54,10 @@ body: |
; CHECK: liveins: $sgpr0_sgpr1, $sgpr2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $sgpr0_sgpr1
; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -65536
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[AND]](s32), [[UV1]](s32)
; CHECK-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[COPY]](s64)
; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 -65536
; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[UV]], [[C]]
; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[AND]](i32), [[UV1]](i32)
; CHECK-NEXT: $sgpr0_sgpr1 = COPY [[MV]](s64)
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
%0:_(s64) = COPY $sgpr0_sgpr1
@ -96,8 +96,8 @@ body: |
; CHECK: liveins: $sgpr0_sgpr1, $sgpr2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $sgpr0_sgpr1
; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[TRUNC]](s32)
; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i32) = G_TRUNC [[COPY]](s64)
; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[TRUNC]](i32)
; CHECK-NEXT: $sgpr0_sgpr1 = COPY [[ZEXT]](s64)
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
%0:_(s64) = COPY $sgpr0_sgpr1
@ -116,8 +116,8 @@ body: |
; CHECK: liveins: $sgpr0_sgpr1, $sgpr2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $sgpr0_sgpr1
; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[TRUNC]](s32)
; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i32) = G_TRUNC [[COPY]](s64)
; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[TRUNC]](i32)
; CHECK-NEXT: $sgpr0_sgpr1 = COPY [[ZEXT]](s64)
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
%0:_(s64) = COPY $sgpr0_sgpr1
@ -136,10 +136,10 @@ body: |
; CHECK: liveins: $sgpr0_sgpr1, $sgpr2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $sgpr0_sgpr1
; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[C]]
; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UV]](s32), [[AND]](s32)
; CHECK-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[COPY]](s64)
; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 15
; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[UV1]], [[C]]
; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UV]](i32), [[AND]](i32)
; CHECK-NEXT: $sgpr0_sgpr1 = COPY [[MV]](s64)
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
%0:_(s64) = COPY $sgpr0_sgpr1
@ -202,9 +202,9 @@ body: |
; CHECK: liveins: $sgpr0_sgpr1, $sgpr2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $sgpr0_sgpr1
; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[TRUNC]](s32), [[C]](s32)
; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i32) = G_TRUNC [[COPY]](s64)
; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 -1
; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[TRUNC]](i32), [[C]](i32)
; CHECK-NEXT: $sgpr0_sgpr1 = COPY [[MV]](s64)
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
%0:_(s64) = COPY $sgpr0_sgpr1
@ -223,9 +223,9 @@ body: |
; CHECK: liveins: $sgpr0_sgpr1, $sgpr2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $sgpr0_sgpr1
; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[TRUNC]](s32), [[C]](s32)
; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i32) = G_TRUNC [[COPY]](s64)
; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 -1
; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[TRUNC]](i32), [[C]](i32)
; CHECK-NEXT: $sgpr0_sgpr1 = COPY [[MV]](s64)
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
%0:_(s64) = COPY $sgpr0_sgpr1
@ -244,9 +244,9 @@ body: |
; CHECK: liveins: $sgpr0_sgpr1, $sgpr2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $sgpr0_sgpr1
; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](s32), [[UV1]](s32)
; CHECK-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[COPY]](s64)
; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 -1
; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](i32), [[UV1]](i32)
; CHECK-NEXT: $sgpr0_sgpr1 = COPY [[MV]](s64)
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
%0:_(s64) = COPY $sgpr0_sgpr1
@ -265,9 +265,9 @@ body: |
; CHECK: liveins: $sgpr0_sgpr1, $sgpr2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $sgpr0_sgpr1
; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](s32), [[UV1]](s32)
; CHECK-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[COPY]](s64)
; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 -1
; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](i32), [[UV1]](i32)
; CHECK-NEXT: $sgpr0_sgpr1 = COPY [[MV]](s64)
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
%0:_(s64) = COPY $sgpr0_sgpr1

18
llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-add-ext-fma.ll
View file

@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX9-DENORM %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 -fp-contract=fast < %s | FileCheck -check-prefix=GFX10-CONTRACT %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX10-DENORM %s
; RUN: llc -global-isel -mtriple=amdgpu9.00 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX9-DENORM %s
; RUN: llc -global-isel -mtriple=amdgpu10.10 < %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -global-isel -mtriple=amdgpu10.10 -fp-contract=fast < %s | FileCheck -check-prefix=GFX10-CONTRACT %s
; RUN: llc -global-isel -mtriple=amdgpu10.10 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX10-DENORM %s
; fold (fadd (fma x, y, (fpext (fmul u, v))), z) -> (fma x, y, (fma (fpext u), (fpext v), z))
define amdgpu_vs float @test_f16_f32_add_fma_ext_mul(float %x, float %y, float %z, half %u, half %v) {
@ -45,9 +45,8 @@ define amdgpu_vs float @test_f16_f32_add_fma_ext_mul(float %x, float %y, float %
define amdgpu_vs float @test_f16_f32_add_ext_fma_mul(half %x, half %y, float %z, half %u, half %v) {
; GFX9-DENORM-LABEL: test_f16_f32_add_ext_fma_mul:
; GFX9-DENORM: ; %bb.0: ; %.entry
; GFX9-DENORM-NEXT: v_mul_f16_e32 v3, v3, v4
; GFX9-DENORM-NEXT: v_mac_f16_e32 v3, v0, v1
; GFX9-DENORM-NEXT: v_mad_mix_f32 v0, v3, 1.0, v2 op_sel_hi:[1,1,0]
; GFX9-DENORM-NEXT: v_mad_mix_f32 v2, v3, v4, v2 op_sel_hi:[1,1,0]
; GFX9-DENORM-NEXT: v_mad_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0]
; GFX9-DENORM-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: test_f16_f32_add_ext_fma_mul:
@ -119,9 +118,8 @@ define amdgpu_vs float @test_f16_f32_add_fma_ext_mul_rhs(float %x, float %y, flo
define amdgpu_vs float @test_f16_f32_add_ext_fma_mul_rhs(float %x, half %y, half %z, half %u, half %v) {
; GFX9-DENORM-LABEL: test_f16_f32_add_ext_fma_mul_rhs:
; GFX9-DENORM: ; %bb.0: ; %.entry
; GFX9-DENORM-NEXT: v_mul_f16_e32 v3, v3, v4
; GFX9-DENORM-NEXT: v_mac_f16_e32 v3, v1, v2
; GFX9-DENORM-NEXT: v_mad_mix_f32 v0, v3, 1.0, v0 op_sel_hi:[1,1,0]
; GFX9-DENORM-NEXT: v_mad_mix_f32 v0, v3, v4, v0 op_sel_hi:[1,1,0]
; GFX9-DENORM-NEXT: v_mad_mix_f32 v0, v1, v2, v0 op_sel_hi:[1,1,0]
; GFX9-DENORM-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: test_f16_f32_add_ext_fma_mul_rhs:

12
llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-add-fma-mul.ll
View file

@ -1,12 +1,12 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00 -fp-contract=fast < %s | FileCheck -check-prefix=GFX9-CONTRACT %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX9-DENORM %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 -fp-contract=fast < %s | FileCheck -check-prefix=GFX10-CONTRACT %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX10-DENORM %s
; RUN: llc -global-isel -mtriple=amdgpu9.00 -fp-contract=fast < %s | FileCheck -check-prefix=GFX9-CONTRACT %s
; RUN: llc -global-isel -mtriple=amdgpu9.00 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX9-DENORM %s
; RUN: llc -global-isel -mtriple=amdgpu10.10 -fp-contract=fast < %s | FileCheck -check-prefix=GFX10-CONTRACT %s
; RUN: llc -global-isel -mtriple=amdgpu10.10 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX10-DENORM %s
; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=+real-true16 -fp-contract=fast < %s | FileCheck -check-prefixes=GFX11-CONTRACT,GFX11-CONTRACT-TRUE16 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=-real-true16 -fp-contract=fast < %s | FileCheck -check-prefixes=GFX11-CONTRACT,GFX11-CONTRACT-FAKE16 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=-real-true16 -fp-contract=fast < %s | FileCheck -check-prefixes=GFX11-CONTRACT,GFX11-CONTRACT-FAKE16 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=+real-true16 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefixes=GFX11-DENORM,GFX11-DENORM-TRUE16 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=-real-true16 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefixes=GFX11-DENORM,GFX11-DENORM-FAKE16 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=-real-true16 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefixes=GFX11-DENORM,GFX11-DENORM-FAKE16 %s
; fadd (fma a, b, (fmul c, d)), e --> fma a, b, (fma c, d, e)
; fadd e, (fma a, b, (fmul c, d)) --> fma a, b, (fma c, d, e)

12
llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-add-mul.ll
View file

@ -1,10 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefix=GFX9 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00 -fp-contract=fast < %s | FileCheck -check-prefix=GFX9-CONTRACT %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX9-DENORM %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 -fp-contract=fast < %s | FileCheck -check-prefix=GFX10-CONTRACT %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX10-DENORM %s
; RUN: llc -global-isel -mtriple=amdgpu9.00 < %s | FileCheck -check-prefix=GFX9 %s
; RUN: llc -global-isel -mtriple=amdgpu9.00 -fp-contract=fast < %s | FileCheck -check-prefix=GFX9-CONTRACT %s
; RUN: llc -global-isel -mtriple=amdgpu9.00 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX9-DENORM %s
; RUN: llc -global-isel -mtriple=amdgpu10.10 < %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -global-isel -mtriple=amdgpu10.10 -fp-contract=fast < %s | FileCheck -check-prefix=GFX10-CONTRACT %s
; RUN: llc -global-isel -mtriple=amdgpu10.10 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX10-DENORM %s
define float @test_f32_add_mul(float %x, float %y, float %z) {
; GFX9-LABEL: test_f32_add_mul:

4
llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-sub-ext-mul.ll
View file

@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX9-DENORM %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX10-DENORM %s
; RUN: llc -global-isel -mtriple=amdgpu9.00 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX9-DENORM %s
; RUN: llc -global-isel -mtriple=amdgpu10.10 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX10-DENORM %s
; fold (fsub (fpext (fmul x, y)), z) -> (fma (fpext x), (fpext y), (fneg z))
define amdgpu_vs float @test_f16_to_f32_sub_ext_mul(half %x, half %y, float %z) {

12
llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-sub-ext-neg-mul.ll
View file

@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX9-DENORM %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX10-DENORM %s
; RUN: llc -global-isel -mtriple=amdgpu9.00 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX9-DENORM %s
; RUN: llc -global-isel -mtriple=amdgpu10.10 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX10-DENORM %s
; fold (fsub (fpext (fneg (fmul, x, y))), z) -> (fneg (fma (fpext x), (fpext y), z))
define amdgpu_vs float @test_f16_to_f32_sub_ext_neg_mul(half %x, half %y, float %z) {
@ -45,12 +45,12 @@ entry:
define amdgpu_vs float @test_f16_to_f32_sub_ext_neg_mul2(float %x, half %y, half %z) {
; GFX9-DENORM-LABEL: test_f16_to_f32_sub_ext_neg_mul2:
; GFX9-DENORM: ; %bb.0: ; %entry
; GFX9-DENORM-NEXT: v_mad_mix_f32 v0, v1, v2, v0 op_sel_hi:[1,1,0]
; GFX9-DENORM-NEXT: v_mad_mix_f32 v0, -v1, -v2, v0 op_sel_hi:[1,1,0]
; GFX9-DENORM-NEXT: ; return to shader part epilog
;
; GFX10-DENORM-LABEL: test_f16_to_f32_sub_ext_neg_mul2:
; GFX10-DENORM: ; %bb.0: ; %entry
; GFX10-DENORM-NEXT: v_fma_mix_f32 v0, v1, v2, v0 op_sel_hi:[1,1,0]
; GFX10-DENORM-NEXT: v_fma_mix_f32 v0, -v1, -v2, v0 op_sel_hi:[1,1,0]
; GFX10-DENORM-NEXT: ; return to shader part epilog
entry:
%a = fmul fast half %y, %z
@ -64,12 +64,12 @@ entry:
define amdgpu_vs float @test_f16_to_f32_sub_neg_ext_mul2(float %x, half %y, half %z) {
; GFX9-DENORM-LABEL: test_f16_to_f32_sub_neg_ext_mul2:
; GFX9-DENORM: ; %bb.0: ; %entry
; GFX9-DENORM-NEXT: v_mad_mix_f32 v0, v1, v2, v0 op_sel_hi:[1,1,0]
; GFX9-DENORM-NEXT: v_mad_mix_f32 v0, -v1, -v2, v0 op_sel_hi:[1,1,0]
; GFX9-DENORM-NEXT: ; return to shader part epilog
;
; GFX10-DENORM-LABEL: test_f16_to_f32_sub_neg_ext_mul2:
; GFX10-DENORM: ; %bb.0: ; %entry
; GFX10-DENORM-NEXT: v_fma_mix_f32 v0, v1, v2, v0 op_sel_hi:[1,1,0]
; GFX10-DENORM-NEXT: v_fma_mix_f32 v0, -v1, -v2, v0 op_sel_hi:[1,1,0]
; GFX10-DENORM-NEXT: ; return to shader part epilog
entry:
%a = fmul fast half %y, %z

16
llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-sub-mul.ll
View file

@ -1,14 +1,14 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefix=GFX9 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00 -fp-contract=fast < %s | FileCheck -check-prefix=GFX9-CONTRACT %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX9-DENORM %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 -fp-contract=fast < %s | FileCheck -check-prefix=GFX10-CONTRACT %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX10-DENORM %s
; RUN: llc -global-isel -mtriple=amdgpu9.00 < %s | FileCheck -check-prefix=GFX9 %s
; RUN: llc -global-isel -mtriple=amdgpu9.00 -fp-contract=fast < %s | FileCheck -check-prefix=GFX9-CONTRACT %s
; RUN: llc -global-isel -mtriple=amdgpu9.00 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX9-DENORM %s
; RUN: llc -global-isel -mtriple=amdgpu10.10 < %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -global-isel -mtriple=amdgpu10.10 -fp-contract=fast < %s | FileCheck -check-prefix=GFX10-CONTRACT %s
; RUN: llc -global-isel -mtriple=amdgpu10.10 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX10-DENORM %s
; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=+real-true16 -fp-contract=fast < %s | FileCheck -check-prefixes=GFX11-CONTRACT,GFX11-CONTRACT-TRUE16 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=-real-true16 -fp-contract=fast < %s | FileCheck -check-prefixes=GFX11-CONTRACT,GFX11-CONTRACT-FAKE16 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=-real-true16 -fp-contract=fast < %s | FileCheck -check-prefixes=GFX11-CONTRACT,GFX11-CONTRACT-FAKE16 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=+real-true16 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefixes=GFX11-DENORM,GFX11-DENORM-TRUE16 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=-real-true16 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefixes=GFX11-DENORM,GFX11-DENORM-FAKE16 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=-real-true16 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefixes=GFX11-DENORM,GFX11-DENORM-FAKE16 %s
; fold (fsub (fmul x, y), z) -> (fma x, y, (fneg z))
; fold (fsub x, (fmul y, z)) -> (fma (fneg y), z, x)

16
llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-sub-neg-mul.ll
View file

@ -1,10 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefix=GFX9 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00 -fp-contract=fast < %s | FileCheck -check-prefix=GFX9-CONTRACT %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX9-DENORM %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 -fp-contract=fast < %s | FileCheck -check-prefix=GFX10-CONTRACT %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX10-DENORM %s
; RUN: llc -global-isel -mtriple=amdgpu9.00 < %s | FileCheck -check-prefix=GFX9 %s
; RUN: llc -global-isel -mtriple=amdgpu9.00 -fp-contract=fast < %s | FileCheck -check-prefix=GFX9-CONTRACT %s
; RUN: llc -global-isel -mtriple=amdgpu9.00 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX9-DENORM %s
; RUN: llc -global-isel -mtriple=amdgpu10.10 < %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -global-isel -mtriple=amdgpu10.10 -fp-contract=fast < %s | FileCheck -check-prefix=GFX10-CONTRACT %s
; RUN: llc -global-isel -mtriple=amdgpu10.10 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX10-DENORM %s
; fold (fsub (fneg (fmul, x, y)), z) -> (fma (fneg x), y, (fneg z))
define float @test_f32_sub_ext_neg_mul(float %x, float %y, float %z) {
@ -63,7 +63,7 @@ define half @test_f16_sub_ext_neg_mul(half %x, half %y, half %z) {
; GFX9-CONTRACT-LABEL: test_f16_sub_ext_neg_mul:
; GFX9-CONTRACT: ; %bb.0: ; %entry
; GFX9-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-CONTRACT-NEXT: v_fma_f16 v0, -v0, v1, -v2
; GFX9-CONTRACT-NEXT: v_fma_f16 v0, v0, -v1, -v2
; GFX9-CONTRACT-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-DENORM-LABEL: test_f16_sub_ext_neg_mul:
@ -82,7 +82,7 @@ define half @test_f16_sub_ext_neg_mul(half %x, half %y, half %z) {
; GFX10-CONTRACT-LABEL: test_f16_sub_ext_neg_mul:
; GFX10-CONTRACT: ; %bb.0: ; %entry
; GFX10-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-CONTRACT-NEXT: v_fma_f16 v0, -v0, v1, -v2
; GFX10-CONTRACT-NEXT: v_fma_f16 v0, v0, -v1, -v2
; GFX10-CONTRACT-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-DENORM-LABEL: test_f16_sub_ext_neg_mul:

53
llvm/test/CodeGen/AMDGPU/GlobalISel/fabs.ll
View file

@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX11 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX12 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX11 %s
; RUN: llc -global-isel -mtriple=amdgpu12.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX12 %s
define amdgpu_ps void @v_fabs_f16(half %in, ptr addrspace(1) %out) {
; GCN-LABEL: v_fabs_f16:
@ -13,27 +13,44 @@ define amdgpu_ps void @v_fabs_f16(half %in, ptr addrspace(1) %out) {
ret void
}
define amdgpu_ps void @s_fabs_f16(half inreg %in, ptr addrspace(1) %out) {
; GCN-LABEL: s_fabs_f16:
; GCN: ; %bb.0:
; GCN-NEXT: s_and_b32 s0, s0, 0x7fff
; GCN-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GCN-NEXT: v_mov_b32_e32 v2, s0
; GCN-NEXT: global_store_b16 v[0:1], v2, off
; GCN-NEXT: s_endpgm
; GFX11-LABEL: s_fabs_f16:
; GFX11: ; %bb.0:
; GFX11-NEXT: v_and_b32_e64 v2, 0x7fff, s0
; GFX11-NEXT: global_store_b16 v[0:1], v2, off
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: s_fabs_f16:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_and_b32 s0, s0, 0x7fff
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: v_mov_b32_e32 v2, s0
; GFX12-NEXT: global_store_b16 v[0:1], v2, off
; GFX12-NEXT: s_endpgm
%fabs = call half @llvm.fabs.f16(half %in)
store half %fabs, ptr addrspace(1) %out
ret void
}
define amdgpu_ps void @s_fabs_f16_salu_use(half inreg %in, i32 inreg %val, ptr addrspace(1) %out) {
; GCN-LABEL: s_fabs_f16_salu_use:
; GCN: ; %bb.0:
; GCN-NEXT: s_and_b32 s0, s0, 0x7fff
; GCN-NEXT: s_cmp_eq_u32 s1, 0
; GCN-NEXT: s_cselect_b32 s1, -1, 0
; GCN-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GCN-NEXT: v_cndmask_b32_e64 v2, 0, s0, s1
; GCN-NEXT: global_store_b16 v[0:1], v2, off
; GCN-NEXT: s_endpgm
; GFX11-LABEL: s_fabs_f16_salu_use:
; GFX11: ; %bb.0:
; GFX11-NEXT: v_and_b32_e64 v2, 0x7fff, s0
; GFX11-NEXT: s_cmp_eq_u32 s1, 0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: v_readfirstlane_b32 s0, v2
; GFX11-NEXT: s_cselect_b32 s0, s0, 0
; GFX11-NEXT: v_mov_b32_e32 v2, s0
; GFX11-NEXT: global_store_b16 v[0:1], v2, off
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: s_fabs_f16_salu_use:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_and_b32 s0, s0, 0x7fff
; GFX12-NEXT: s_cmp_eq_u32 s1, 0
; GFX12-NEXT: s_cselect_b32 s0, s0, 0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: v_mov_b32_e32 v2, s0
; GFX12-NEXT: global_store_b16 v[0:1], v2, off
; GFX12-NEXT: s_endpgm
%fabs = call half @llvm.fabs.f16(half %in)
%cond = icmp eq i32 %val, 0
%sel = select i1 %cond, half %fabs, half 0.0

8
llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.ll
View file

@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-FAKE16 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=+real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-TRUE16 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-FAKE16 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00-amd-amdpal -mattr=+real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-TRUE16 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-FAKE16 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -mattr=+real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-TRUE16 %s
; RUN: llc -global-isel -mtriple=amdgpu12.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-FAKE16 %s
; RUN: llc -global-isel -mtriple=amdgpu12.00-amd-amdpal -mattr=+real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-TRUE16 %s
define amdgpu_ps half @fadd_s16_uniform(half inreg %a, half inreg %b) {
; GFX11-FAKE16-LABEL: fadd_s16_uniform:

4
llvm/test/CodeGen/AMDGPU/GlobalISel/fceil.ll
View file

@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgpu8.03 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefix=FIJI %s
; RUN: llc -mtriple=amdgpu12.00 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefix=GFX12 %s
; RUN: llc -mtriple=amdgpu8.03 -global-isel=1 < %s | FileCheck -check-prefix=FIJI %s
; RUN: llc -mtriple=amdgpu12.00 -global-isel=1 < %s | FileCheck -check-prefix=GFX12 %s
define float @fceil_s(float inreg %val) {
; FIJI-LABEL: fceil_s:

252
llvm/test/CodeGen/AMDGPU/GlobalISel/fcmp.ll
View file

@ -1,53 +1,94 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgpu10.10 -global-isel -global-isel-abort=2 < %s | FileCheck %s --check-prefix=GFX10
; RUN: llc -mtriple=amdgpu12.00 -global-isel -global-isel-abort=2 < %s | FileCheck %s --check-prefix=GFX12
; RUN: llc -mtriple=amdgpu10.10 -global-isel < %s | FileCheck %s --check-prefix=GFX10
; RUN: llc -mtriple=amdgpu12.00 -global-isel < %s | FileCheck %s --check-prefix=GFX12
define void @fcmp_f16_uniform(half inreg %a, half inreg %b, ptr %p) {
; GFX10-LABEL: fcmp_f16_uniform:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_cmp_eq_f16_e64 s4, s16, s17
; GFX10-NEXT: v_cmp_gt_f16_e64 s5, s16, s17
; GFX10-NEXT: v_cmp_ge_f16_e64 s6, s16, s17
; GFX10-NEXT: v_cmp_le_f16_e64 s8, s16, s17
; GFX10-NEXT: v_cmp_o_f16_e64 s9, s16, s17
; GFX10-NEXT: v_cmp_nle_f16_e64 s10, s16, s17
; GFX10-NEXT: v_cmp_nge_f16_e64 s11, s16, s17
; GFX10-NEXT: s_and_b32 s5, s5, exec_lo
; GFX10-NEXT: v_cmp_neq_f16_e64 s12, s16, s17
; GFX10-NEXT: s_cselect_b32 s5, 1, 0
; GFX10-NEXT: s_and_b32 s6, s6, exec_lo
; GFX10-NEXT: v_cmp_eq_f16_e64 s4, s16, s17
; GFX10-NEXT: s_cselect_b32 s6, 1, 0
; GFX10-NEXT: s_and_b32 s8, s8, exec_lo
; GFX10-NEXT: s_cselect_b32 s8, 1, 0
; GFX10-NEXT: s_and_b32 s9, s9, exec_lo
; GFX10-NEXT: s_cselect_b32 s9, 1, 0
; GFX10-NEXT: s_and_b32 s10, s10, exec_lo
; GFX10-NEXT: v_cmp_lt_f16_e64 s7, s16, s17
; GFX10-NEXT: s_cselect_b32 s10, 1, 0
; GFX10-NEXT: s_and_b32 s11, s11, exec_lo
; GFX10-NEXT: s_cselect_b32 s11, 1, 0
; GFX10-NEXT: s_and_b32 s12, s12, exec_lo
; GFX10-NEXT: s_cselect_b32 s12, 1, 0
; GFX10-NEXT: v_cmp_le_f16_e64 s8, s16, s17
; GFX10-NEXT: s_cmp_lg_u32 s4, 0
; GFX10-NEXT: v_cmp_lg_f16_e64 s4, s16, s17
; GFX10-NEXT: s_addc_u32 s5, s5, 0
; GFX10-NEXT: s_cmp_lg_u32 s7, 0
; GFX10-NEXT: v_cmp_nlg_f16_e64 s7, s16, s17
; GFX10-NEXT: s_addc_u32 s5, s5, s6
; GFX10-NEXT: s_cmp_lg_u32 s4, 0
; GFX10-NEXT: v_cmp_nlt_f16_e64 s4, s16, s17
; GFX10-NEXT: s_addc_u32 s5, s5, s8
; GFX10-NEXT: s_cmp_lg_u32 s7, 0
; GFX10-NEXT: v_cmp_ngt_f16_e64 s6, s16, s17
; GFX10-NEXT: s_addc_u32 s5, s5, s9
; GFX10-NEXT: s_cmp_lg_u32 s4, 0
; GFX10-NEXT: v_cmp_u_f16_e64 s4, s16, s17
; GFX10-NEXT: s_addc_u32 s5, s5, s10
; GFX10-NEXT: v_cmp_lg_f16_e64 s9, s16, s17
; GFX10-NEXT: s_cselect_b32 s4, 1, 0
; GFX10-NEXT: s_cmp_lg_u32 s5, 0
; GFX10-NEXT: v_cmp_o_f16_e64 s10, s16, s17
; GFX10-NEXT: s_cselect_b32 s5, 1, 0
; GFX10-NEXT: s_cmp_lg_u32 s6, 0
; GFX10-NEXT: s_addc_u32 s5, s5, s11
; GFX10-NEXT: v_cmp_nlg_f16_e64 s11, s16, s17
; GFX10-NEXT: s_cselect_b32 s6, 1, 0
; GFX10-NEXT: s_cmp_lg_u32 s7, 0
; GFX10-NEXT: v_cmp_nle_f16_e64 s12, s16, s17
; GFX10-NEXT: s_cselect_b32 s7, 1, 0
; GFX10-NEXT: s_cmp_lg_u32 s8, 0
; GFX10-NEXT: v_cmp_nlt_f16_e64 s13, s16, s17
; GFX10-NEXT: s_cselect_b32 s8, 1, 0
; GFX10-NEXT: s_cmp_lg_u32 s9, 0
; GFX10-NEXT: v_cmp_nge_f16_e64 s14, s16, s17
; GFX10-NEXT: s_cselect_b32 s9, 1, 0
; GFX10-NEXT: s_cmp_lg_u32 s10, 0
; GFX10-NEXT: v_cmp_ngt_f16_e64 s15, s16, s17
; GFX10-NEXT: s_cselect_b32 s10, 1, 0
; GFX10-NEXT: s_cmp_lg_u32 s11, 0
; GFX10-NEXT: v_cmp_neq_f16_e64 s18, s16, s17
; GFX10-NEXT: s_cselect_b32 s11, 1, 0
; GFX10-NEXT: s_cmp_lg_u32 s12, 0
; GFX10-NEXT: v_cmp_u_f16_e64 s16, s16, s17
; GFX10-NEXT: s_cselect_b32 s12, 1, 0
; GFX10-NEXT: s_cmp_lg_u32 s13, 0
; GFX10-NEXT: s_cselect_b32 s13, 1, 0
; GFX10-NEXT: s_cmp_lg_u32 s14, 0
; GFX10-NEXT: s_cselect_b32 s14, 1, 0
; GFX10-NEXT: s_cmp_lg_u32 s15, 0
; GFX10-NEXT: s_cselect_b32 s15, 1, 0
; GFX10-NEXT: s_cmp_lg_u32 s18, 0
; GFX10-NEXT: s_cselect_b32 s17, 1, 0
; GFX10-NEXT: s_cmp_lg_u32 s16, 0
; GFX10-NEXT: s_cselect_b32 s16, 1, 0
; GFX10-NEXT: s_cmp_lg_u32 s4, 0
; GFX10-NEXT: s_addc_u32 s4, s5, s12
; GFX10-NEXT: s_cselect_b32 s4, 1, 0
; GFX10-NEXT: s_cmp_lg_u32 s5, 0
; GFX10-NEXT: s_cselect_b32 s5, 1, 0
; GFX10-NEXT: s_cmp_lg_u32 s6, 0
; GFX10-NEXT: s_cselect_b32 s6, 1, 0
; GFX10-NEXT: s_cmp_lg_u32 s7, 0
; GFX10-NEXT: s_cselect_b32 s7, 1, 0
; GFX10-NEXT: s_cmp_lg_u32 s8, 0
; GFX10-NEXT: s_cselect_b32 s8, 1, 0
; GFX10-NEXT: s_cmp_lg_u32 s9, 0
; GFX10-NEXT: s_cselect_b32 s9, 1, 0
; GFX10-NEXT: s_cmp_lg_u32 s10, 0
; GFX10-NEXT: s_cselect_b32 s10, 1, 0
; GFX10-NEXT: s_cmp_lg_u32 s11, 0
; GFX10-NEXT: s_cselect_b32 s11, 1, 0
; GFX10-NEXT: s_cmp_lg_u32 s12, 0
; GFX10-NEXT: s_cselect_b32 s12, 1, 0
; GFX10-NEXT: s_cmp_lg_u32 s13, 0
; GFX10-NEXT: s_cselect_b32 s13, 1, 0
; GFX10-NEXT: s_cmp_lg_u32 s14, 0
; GFX10-NEXT: s_cselect_b32 s14, 1, 0
; GFX10-NEXT: s_cmp_lg_u32 s15, 0
; GFX10-NEXT: s_cselect_b32 s15, 1, 0
; GFX10-NEXT: s_cmp_lg_u32 s17, 0
; GFX10-NEXT: s_cselect_b32 s17, 1, 0
; GFX10-NEXT: s_cmp_lg_u32 s16, 0
; GFX10-NEXT: s_cselect_b32 s16, 1, 0
; GFX10-NEXT: s_add_i32 s4, s4, s5
; GFX10-NEXT: s_add_i32 s4, s4, s6
; GFX10-NEXT: s_add_i32 s4, s4, s7
; GFX10-NEXT: s_add_i32 s4, s4, s8
; GFX10-NEXT: s_add_i32 s4, s4, s9
; GFX10-NEXT: s_add_i32 s4, s4, s10
; GFX10-NEXT: s_add_i32 s4, s4, s11
; GFX10-NEXT: s_add_i32 s4, s4, s12
; GFX10-NEXT: s_add_i32 s4, s4, s13
; GFX10-NEXT: s_add_i32 s4, s4, s14
; GFX10-NEXT: s_add_i32 s4, s4, s15
; GFX10-NEXT: s_add_i32 s4, s4, s17
; GFX10-NEXT: s_add_i32 s4, s4, s16
; GFX10-NEXT: v_mov_b32_e32 v2, s4
; GFX10-NEXT: flat_store_dword v[0:1], v2
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
@ -61,54 +102,88 @@ define void @fcmp_f16_uniform(half inreg %a, half inreg %b, ptr %p) {
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_cmp_eq_f16 s0, s1
; GFX12-NEXT: s_cselect_b32 s2, -1, 0
; GFX12-NEXT: s_cmp_lt_f16 s0, s1
; GFX12-NEXT: s_cselect_b32 s3, -1, 0
; GFX12-NEXT: s_cmp_lg_f16 s0, s1
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
; GFX12-NEXT: s_cmp_nlg_f16 s0, s1
; GFX12-NEXT: s_cselect_b32 s5, -1, 0
; GFX12-NEXT: s_cmp_nlt_f16 s0, s1
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
; GFX12-NEXT: s_cmp_ngt_f16 s0, s1
; GFX12-NEXT: s_cselect_b32 s7, -1, 0
; GFX12-NEXT: s_cmp_u_f16 s0, s1
; GFX12-NEXT: s_cselect_b32 s8, -1, 0
; GFX12-NEXT: s_cselect_b32 s2, 1, 0
; GFX12-NEXT: s_cmp_gt_f16 s0, s1
; GFX12-NEXT: s_cselect_b32 s9, 1, 0
; GFX12-NEXT: s_cselect_b32 s3, 1, 0
; GFX12-NEXT: s_cmp_ge_f16 s0, s1
; GFX12-NEXT: s_cselect_b32 s10, 1, 0
; GFX12-NEXT: s_cselect_b32 s4, 1, 0
; GFX12-NEXT: s_cmp_lt_f16 s0, s1
; GFX12-NEXT: s_cselect_b32 s5, 1, 0
; GFX12-NEXT: s_cmp_le_f16 s0, s1
; GFX12-NEXT: s_cselect_b32 s11, 1, 0
; GFX12-NEXT: s_cselect_b32 s6, 1, 0
; GFX12-NEXT: s_cmp_lg_f16 s0, s1
; GFX12-NEXT: s_cselect_b32 s7, 1, 0
; GFX12-NEXT: s_cmp_o_f16 s0, s1
; GFX12-NEXT: s_cselect_b32 s12, 1, 0
; GFX12-NEXT: s_cselect_b32 s8, 1, 0
; GFX12-NEXT: s_cmp_nlg_f16 s0, s1
; GFX12-NEXT: s_cselect_b32 s9, 1, 0
; GFX12-NEXT: s_cmp_nle_f16 s0, s1
; GFX12-NEXT: s_cselect_b32 s13, 1, 0
; GFX12-NEXT: s_cselect_b32 s10, 1, 0
; GFX12-NEXT: s_cmp_nlt_f16 s0, s1
; GFX12-NEXT: s_cselect_b32 s11, 1, 0
; GFX12-NEXT: s_cmp_nge_f16 s0, s1
; GFX12-NEXT: s_cselect_b32 s14, 1, 0
; GFX12-NEXT: s_cselect_b32 s12, 1, 0
; GFX12-NEXT: s_cmp_ngt_f16 s0, s1
; GFX12-NEXT: s_cselect_b32 s13, 1, 0
; GFX12-NEXT: s_cmp_neq_f16 s0, s1
; GFX12-NEXT: s_cselect_b32 s14, 1, 0
; GFX12-NEXT: s_cmp_u_f16 s0, s1
; GFX12-NEXT: s_cselect_b32 s0, 1, 0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_cmp_lg_u32 s2, 0
; GFX12-NEXT: s_add_co_ci_u32 s1, s9, 0
; GFX12-NEXT: s_cselect_b32 s1, 1, 0
; GFX12-NEXT: s_cmp_lg_u32 s3, 0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_add_co_ci_u32 s1, s1, s10
; GFX12-NEXT: s_cselect_b32 s2, 1, 0
; GFX12-NEXT: s_cmp_lg_u32 s4, 0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_add_co_ci_u32 s1, s1, s11
; GFX12-NEXT: s_cselect_b32 s3, 1, 0
; GFX12-NEXT: s_cmp_lg_u32 s5, 0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_add_co_ci_u32 s1, s1, s12
; GFX12-NEXT: s_cselect_b32 s4, 1, 0
; GFX12-NEXT: s_cmp_lg_u32 s6, 0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_add_co_ci_u32 s1, s1, s13
; GFX12-NEXT: s_cselect_b32 s5, 1, 0
; GFX12-NEXT: s_cmp_lg_u32 s7, 0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_add_co_ci_u32 s1, s1, s14
; GFX12-NEXT: s_cselect_b32 s6, 1, 0
; GFX12-NEXT: s_cmp_lg_u32 s8, 0
; GFX12-NEXT: s_cselect_b32 s7, 1, 0
; GFX12-NEXT: s_cmp_lg_u32 s9, 0
; GFX12-NEXT: s_cselect_b32 s8, 1, 0
; GFX12-NEXT: s_cmp_lg_u32 s10, 0
; GFX12-NEXT: s_cselect_b32 s9, 1, 0
; GFX12-NEXT: s_cmp_lg_u32 s11, 0
; GFX12-NEXT: s_cselect_b32 s10, 1, 0
; GFX12-NEXT: s_cmp_lg_u32 s12, 0
; GFX12-NEXT: s_cselect_b32 s11, 1, 0
; GFX12-NEXT: s_cmp_lg_u32 s13, 0
; GFX12-NEXT: s_cselect_b32 s12, 1, 0
; GFX12-NEXT: s_cmp_lg_u32 s14, 0
; GFX12-NEXT: s_cselect_b32 s13, 1, 0
; GFX12-NEXT: s_cmp_lg_u32 s0, 0
; GFX12-NEXT: s_cselect_b32 s0, 1, 0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_add_co_ci_u32 s0, s1, s0
; GFX12-NEXT: s_add_co_i32 s1, s1, s2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_add_co_i32 s1, s1, s3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_add_co_i32 s1, s1, s4
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_add_co_i32 s1, s1, s5
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_add_co_i32 s1, s1, s6
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_add_co_i32 s1, s1, s7
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_add_co_i32 s1, s1, s8
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_add_co_i32 s1, s1, s9
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_add_co_i32 s1, s1, s10
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_add_co_i32 s1, s1, s11
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_add_co_i32 s1, s1, s12
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_add_co_i32 s1, s1, s13
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_add_co_i32 s0, s1, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: v_mov_b32_e32 v2, s0
; GFX12-NEXT: flat_store_b32 v[0:1], v2
@ -163,34 +238,41 @@ define void @fcmp_f16_divergent(half %a, half %b, ptr %p) {
; GFX10-LABEL: fcmp_f16_divergent:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_cmp_gt_f16_e32 vcc_lo, v0, v1
; GFX10-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo
; GFX10-NEXT: v_cmp_ge_f16_e32 vcc_lo, v0, v1
; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc_lo
; GFX10-NEXT: v_cmp_eq_f16_e32 vcc_lo, v0, v1
; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, 0, v4, vcc_lo
; GFX10-NEXT: v_cmp_le_f16_e32 vcc_lo, v0, v1
; GFX10-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo
; GFX10-NEXT: v_cmp_gt_f16_e32 vcc_lo, v0, v1
; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc_lo
; GFX10-NEXT: v_cmp_ge_f16_e32 vcc_lo, v0, v1
; GFX10-NEXT: v_add_nc_u32_e32 v4, v4, v5
; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc_lo
; GFX10-NEXT: v_cmp_lt_f16_e32 vcc_lo, v0, v1
; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, v4, v5, vcc_lo
; GFX10-NEXT: v_cmp_o_f16_e32 vcc_lo, v0, v1
; GFX10-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc_lo
; GFX10-NEXT: v_cmp_le_f16_e32 vcc_lo, v0, v1
; GFX10-NEXT: v_add3_u32 v4, v4, v6, v7
; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc_lo
; GFX10-NEXT: v_cmp_lg_f16_e32 vcc_lo, v0, v1
; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, v4, v6, vcc_lo
; GFX10-NEXT: v_cmp_nle_f16_e32 vcc_lo, v0, v1
; GFX10-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc_lo
; GFX10-NEXT: v_cmp_o_f16_e32 vcc_lo, v0, v1
; GFX10-NEXT: v_add3_u32 v4, v4, v5, v8
; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc_lo
; GFX10-NEXT: v_cmp_nlg_f16_e32 vcc_lo, v0, v1
; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, v4, v5, vcc_lo
; GFX10-NEXT: v_cmp_nge_f16_e32 vcc_lo, v0, v1
; GFX10-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc_lo
; GFX10-NEXT: v_cmp_nle_f16_e32 vcc_lo, v0, v1
; GFX10-NEXT: v_add3_u32 v4, v4, v6, v7
; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc_lo
; GFX10-NEXT: v_cmp_nlt_f16_e32 vcc_lo, v0, v1
; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, v4, v6, vcc_lo
; GFX10-NEXT: v_cmp_neq_f16_e32 vcc_lo, v0, v1
; GFX10-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc_lo
; GFX10-NEXT: v_cmp_nge_f16_e32 vcc_lo, v0, v1
; GFX10-NEXT: v_add3_u32 v4, v4, v5, v8
; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc_lo
; GFX10-NEXT: v_cmp_ngt_f16_e32 vcc_lo, v0, v1
; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, v4, v5, vcc_lo
; GFX10-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc_lo
; GFX10-NEXT: v_cmp_neq_f16_e32 vcc_lo, v0, v1
; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc_lo
; GFX10-NEXT: v_cmp_u_f16_e32 vcc_lo, v0, v1
; GFX10-NEXT: v_add_co_ci_u32_e32 v0, vcc_lo, v4, v6, vcc_lo
; GFX10-NEXT: v_add3_u32 v1, v4, v6, v7
; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
; GFX10-NEXT: v_add3_u32 v0, v1, v5, v0
; GFX10-NEXT: flat_store_dword v[2:3], v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_setpc_b64 s[30:31]

532
llvm/test/CodeGen/AMDGPU/GlobalISel/fdiv.f16.ll
View file

@ -57,22 +57,42 @@ define half @v_fdiv_f16(half %a, half %b) {
; GFX6-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX6-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_fdiv_f16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_cvt_f32_f16_e32 v2, v1
; GFX8-NEXT: v_cvt_f32_f16_e32 v3, v0
; GFX8-NEXT: v_rcp_f32_e32 v4, v2
; GFX8-NEXT: v_mul_f32_e32 v5, v3, v4
; GFX8-NEXT: v_mad_f32 v6, -v2, v5, v3
; GFX8-NEXT: v_mac_f32_e32 v5, v6, v4
; GFX8-NEXT: v_mad_f32 v2, -v2, v5, v3
; GFX8-NEXT: v_mul_f32_e32 v2, v2, v4
; GFX8-NEXT: v_and_b32_e32 v2, 0xff800000, v2
; GFX8-NEXT: v_add_f32_e32 v2, v2, v5
; GFX8-NEXT: v_cvt_f16_f32_e32 v2, v2
; GFX8-NEXT: v_div_fixup_f16 v0, v2, v1, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
; GFX8-IEEE-LABEL: v_fdiv_f16:
; GFX8-IEEE: ; %bb.0:
; GFX8-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-IEEE-NEXT: v_cvt_f32_f16_e32 v2, v1
; GFX8-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v0
; GFX8-IEEE-NEXT: v_rcp_f32_e32 v4, v2
; GFX8-IEEE-NEXT: v_mul_f32_e32 v5, v3, v4
; GFX8-IEEE-NEXT: v_mul_f32_e64 v6, -v2, v5
; GFX8-IEEE-NEXT: v_add_f32_e32 v6, v6, v3
; GFX8-IEEE-NEXT: v_mul_f32_e32 v6, v6, v4
; GFX8-IEEE-NEXT: v_add_f32_e32 v5, v6, v5
; GFX8-IEEE-NEXT: v_mul_f32_e64 v2, -v2, v5
; GFX8-IEEE-NEXT: v_add_f32_e32 v2, v2, v3
; GFX8-IEEE-NEXT: v_mul_f32_e32 v2, v2, v4
; GFX8-IEEE-NEXT: v_and_b32_e32 v2, 0xff800000, v2
; GFX8-IEEE-NEXT: v_add_f32_e32 v2, v2, v5
; GFX8-IEEE-NEXT: v_cvt_f16_f32_e32 v2, v2
; GFX8-IEEE-NEXT: v_div_fixup_f16 v0, v2, v1, v0
; GFX8-IEEE-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-FLUSH-LABEL: v_fdiv_f16:
; GFX8-FLUSH: ; %bb.0:
; GFX8-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v1
; GFX8-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v0
; GFX8-FLUSH-NEXT: v_rcp_f32_e32 v4, v2
; GFX8-FLUSH-NEXT: v_mul_f32_e32 v5, v3, v4
; GFX8-FLUSH-NEXT: v_mad_f32 v6, -v2, v5, v3
; GFX8-FLUSH-NEXT: v_mac_f32_e32 v5, v6, v4
; GFX8-FLUSH-NEXT: v_mad_f32 v2, -v2, v5, v3
; GFX8-FLUSH-NEXT: v_mul_f32_e32 v2, v2, v4
; GFX8-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
; GFX8-FLUSH-NEXT: v_add_f32_e32 v2, v2, v5
; GFX8-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
; GFX8-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v1, v0
; GFX8-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-IEEE-LABEL: v_fdiv_f16:
; GFX9-IEEE: ; %bb.0:
@ -81,9 +101,12 @@ define half @v_fdiv_f16(half %a, half %b) {
; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v0
; GFX9-IEEE-NEXT: v_rcp_f32_e32 v4, v2
; GFX9-IEEE-NEXT: v_mul_f32_e32 v5, v3, v4
; GFX9-IEEE-NEXT: v_mad_f32 v6, -v2, v5, v3
; GFX9-IEEE-NEXT: v_mac_f32_e32 v5, v6, v4
; GFX9-IEEE-NEXT: v_mad_f32 v2, -v2, v5, v3
; GFX9-IEEE-NEXT: v_mul_f32_e64 v6, -v2, v5
; GFX9-IEEE-NEXT: v_add_f32_e32 v6, v6, v3
; GFX9-IEEE-NEXT: v_mul_f32_e32 v6, v6, v4
; GFX9-IEEE-NEXT: v_add_f32_e32 v5, v6, v5
; GFX9-IEEE-NEXT: v_mul_f32_e64 v2, -v2, v5
; GFX9-IEEE-NEXT: v_add_f32_e32 v2, v2, v3
; GFX9-IEEE-NEXT: v_mul_f32_e32 v2, v2, v4
; GFX9-IEEE-NEXT: v_and_b32_e32 v2, 0xff800000, v2
; GFX9-IEEE-NEXT: v_add_f32_e32 v2, v2, v5
@ -107,22 +130,41 @@ define half @v_fdiv_f16(half %a, half %b) {
; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v1, v0
; GFX9-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_fdiv_f16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_cvt_f32_f16_e32 v2, v1
; GFX10-NEXT: v_cvt_f32_f16_e32 v5, v0
; GFX10-NEXT: v_rcp_f32_e32 v3, v2
; GFX10-NEXT: v_fma_mix_f32 v4, v0, v3, neg(0) op_sel_hi:[1,0,0]
; GFX10-NEXT: v_mad_f32 v6, -v2, v4, v5
; GFX10-NEXT: v_mac_f32_e32 v4, v6, v3
; GFX10-NEXT: v_mad_f32 v2, -v2, v4, v5
; GFX10-NEXT: v_mul_f32_e32 v2, v2, v3
; GFX10-NEXT: v_and_b32_e32 v2, 0xff800000, v2
; GFX10-NEXT: v_add_f32_e32 v2, v2, v4
; GFX10-NEXT: v_cvt_f16_f32_e32 v2, v2
; GFX10-NEXT: v_div_fixup_f16 v0, v2, v1, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
; GFX10-IEEE-LABEL: v_fdiv_f16:
; GFX10-IEEE: ; %bb.0:
; GFX10-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-IEEE-NEXT: v_cvt_f32_f16_e32 v2, v1
; GFX10-IEEE-NEXT: v_rcp_f32_e32 v2, v2
; GFX10-IEEE-NEXT: v_fma_mix_f32 v3, v0, v2, neg(0) op_sel_hi:[1,0,0]
; GFX10-IEEE-NEXT: v_fma_mix_f32 v4, -v1, v3, neg(0) op_sel_hi:[1,0,0]
; GFX10-IEEE-NEXT: v_fma_mix_f32 v4, v0, 1.0, v4 op_sel_hi:[1,1,0]
; GFX10-IEEE-NEXT: v_mul_f32_e32 v4, v4, v2
; GFX10-IEEE-NEXT: v_add_f32_e32 v3, v4, v3
; GFX10-IEEE-NEXT: v_fma_mix_f32 v4, -v1, v3, neg(0) op_sel_hi:[1,0,0]
; GFX10-IEEE-NEXT: v_fma_mix_f32 v4, v0, 1.0, v4 op_sel_hi:[1,1,0]
; GFX10-IEEE-NEXT: v_mul_f32_e32 v2, v4, v2
; GFX10-IEEE-NEXT: v_and_b32_e32 v2, 0xff800000, v2
; GFX10-IEEE-NEXT: v_add_f32_e32 v2, v2, v3
; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v2, v2
; GFX10-IEEE-NEXT: v_div_fixup_f16 v0, v2, v1, v0
; GFX10-IEEE-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-FLUSH-LABEL: v_fdiv_f16:
; GFX10-FLUSH: ; %bb.0:
; GFX10-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v1
; GFX10-FLUSH-NEXT: v_cvt_f32_f16_e32 v4, v0
; GFX10-FLUSH-NEXT: v_rcp_f32_e32 v3, v2
; GFX10-FLUSH-NEXT: v_fma_mix_f32 v5, v0, v3, neg(0) op_sel_hi:[1,0,0]
; GFX10-FLUSH-NEXT: v_mad_f32 v6, -v2, v5, v4
; GFX10-FLUSH-NEXT: v_mac_f32_e32 v5, v6, v3
; GFX10-FLUSH-NEXT: v_mad_f32 v2, -v2, v5, v4
; GFX10-FLUSH-NEXT: v_mul_f32_e32 v2, v2, v3
; GFX10-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
; GFX10-FLUSH-NEXT: v_add_f32_e32 v2, v2, v5
; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
; GFX10-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v1, v0
; GFX10-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fdiv_f16:
; GFX11: ; %bb.0:
@ -222,22 +264,42 @@ define half @v_fdiv_f16_ulp25(half %a, half %b) {
; GFX6-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX6-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_fdiv_f16_ulp25:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_cvt_f32_f16_e32 v2, v1
; GFX8-NEXT: v_cvt_f32_f16_e32 v3, v0
; GFX8-NEXT: v_rcp_f32_e32 v4, v2
; GFX8-NEXT: v_mul_f32_e32 v5, v3, v4
; GFX8-NEXT: v_mad_f32 v6, -v2, v5, v3
; GFX8-NEXT: v_mac_f32_e32 v5, v6, v4
; GFX8-NEXT: v_mad_f32 v2, -v2, v5, v3
; GFX8-NEXT: v_mul_f32_e32 v2, v2, v4
; GFX8-NEXT: v_and_b32_e32 v2, 0xff800000, v2
; GFX8-NEXT: v_add_f32_e32 v2, v2, v5
; GFX8-NEXT: v_cvt_f16_f32_e32 v2, v2
; GFX8-NEXT: v_div_fixup_f16 v0, v2, v1, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
; GFX8-IEEE-LABEL: v_fdiv_f16_ulp25:
; GFX8-IEEE: ; %bb.0:
; GFX8-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-IEEE-NEXT: v_cvt_f32_f16_e32 v2, v1
; GFX8-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v0
; GFX8-IEEE-NEXT: v_rcp_f32_e32 v4, v2
; GFX8-IEEE-NEXT: v_mul_f32_e32 v5, v3, v4
; GFX8-IEEE-NEXT: v_mul_f32_e64 v6, -v2, v5
; GFX8-IEEE-NEXT: v_add_f32_e32 v6, v6, v3
; GFX8-IEEE-NEXT: v_mul_f32_e32 v6, v6, v4
; GFX8-IEEE-NEXT: v_add_f32_e32 v5, v6, v5
; GFX8-IEEE-NEXT: v_mul_f32_e64 v2, -v2, v5
; GFX8-IEEE-NEXT: v_add_f32_e32 v2, v2, v3
; GFX8-IEEE-NEXT: v_mul_f32_e32 v2, v2, v4
; GFX8-IEEE-NEXT: v_and_b32_e32 v2, 0xff800000, v2
; GFX8-IEEE-NEXT: v_add_f32_e32 v2, v2, v5
; GFX8-IEEE-NEXT: v_cvt_f16_f32_e32 v2, v2
; GFX8-IEEE-NEXT: v_div_fixup_f16 v0, v2, v1, v0
; GFX8-IEEE-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-FLUSH-LABEL: v_fdiv_f16_ulp25:
; GFX8-FLUSH: ; %bb.0:
; GFX8-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v1
; GFX8-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v0
; GFX8-FLUSH-NEXT: v_rcp_f32_e32 v4, v2
; GFX8-FLUSH-NEXT: v_mul_f32_e32 v5, v3, v4
; GFX8-FLUSH-NEXT: v_mad_f32 v6, -v2, v5, v3
; GFX8-FLUSH-NEXT: v_mac_f32_e32 v5, v6, v4
; GFX8-FLUSH-NEXT: v_mad_f32 v2, -v2, v5, v3
; GFX8-FLUSH-NEXT: v_mul_f32_e32 v2, v2, v4
; GFX8-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
; GFX8-FLUSH-NEXT: v_add_f32_e32 v2, v2, v5
; GFX8-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
; GFX8-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v1, v0
; GFX8-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-IEEE-LABEL: v_fdiv_f16_ulp25:
; GFX9-IEEE: ; %bb.0:
@ -246,9 +308,12 @@ define half @v_fdiv_f16_ulp25(half %a, half %b) {
; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v0
; GFX9-IEEE-NEXT: v_rcp_f32_e32 v4, v2
; GFX9-IEEE-NEXT: v_mul_f32_e32 v5, v3, v4
; GFX9-IEEE-NEXT: v_mad_f32 v6, -v2, v5, v3
; GFX9-IEEE-NEXT: v_mac_f32_e32 v5, v6, v4
; GFX9-IEEE-NEXT: v_mad_f32 v2, -v2, v5, v3
; GFX9-IEEE-NEXT: v_mul_f32_e64 v6, -v2, v5
; GFX9-IEEE-NEXT: v_add_f32_e32 v6, v6, v3
; GFX9-IEEE-NEXT: v_mul_f32_e32 v6, v6, v4
; GFX9-IEEE-NEXT: v_add_f32_e32 v5, v6, v5
; GFX9-IEEE-NEXT: v_mul_f32_e64 v2, -v2, v5
; GFX9-IEEE-NEXT: v_add_f32_e32 v2, v2, v3
; GFX9-IEEE-NEXT: v_mul_f32_e32 v2, v2, v4
; GFX9-IEEE-NEXT: v_and_b32_e32 v2, 0xff800000, v2
; GFX9-IEEE-NEXT: v_add_f32_e32 v2, v2, v5
@ -272,22 +337,41 @@ define half @v_fdiv_f16_ulp25(half %a, half %b) {
; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v1, v0
; GFX9-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_fdiv_f16_ulp25:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_cvt_f32_f16_e32 v2, v1
; GFX10-NEXT: v_cvt_f32_f16_e32 v5, v0
; GFX10-NEXT: v_rcp_f32_e32 v3, v2
; GFX10-NEXT: v_fma_mix_f32 v4, v0, v3, neg(0) op_sel_hi:[1,0,0]
; GFX10-NEXT: v_mad_f32 v6, -v2, v4, v5
; GFX10-NEXT: v_mac_f32_e32 v4, v6, v3
; GFX10-NEXT: v_mad_f32 v2, -v2, v4, v5
; GFX10-NEXT: v_mul_f32_e32 v2, v2, v3
; GFX10-NEXT: v_and_b32_e32 v2, 0xff800000, v2
; GFX10-NEXT: v_add_f32_e32 v2, v2, v4
; GFX10-NEXT: v_cvt_f16_f32_e32 v2, v2
; GFX10-NEXT: v_div_fixup_f16 v0, v2, v1, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
; GFX10-IEEE-LABEL: v_fdiv_f16_ulp25:
; GFX10-IEEE: ; %bb.0:
; GFX10-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-IEEE-NEXT: v_cvt_f32_f16_e32 v2, v1
; GFX10-IEEE-NEXT: v_rcp_f32_e32 v2, v2
; GFX10-IEEE-NEXT: v_fma_mix_f32 v3, v0, v2, neg(0) op_sel_hi:[1,0,0]
; GFX10-IEEE-NEXT: v_fma_mix_f32 v4, -v1, v3, neg(0) op_sel_hi:[1,0,0]
; GFX10-IEEE-NEXT: v_fma_mix_f32 v4, v0, 1.0, v4 op_sel_hi:[1,1,0]
; GFX10-IEEE-NEXT: v_mul_f32_e32 v4, v4, v2
; GFX10-IEEE-NEXT: v_add_f32_e32 v3, v4, v3
; GFX10-IEEE-NEXT: v_fma_mix_f32 v4, -v1, v3, neg(0) op_sel_hi:[1,0,0]
; GFX10-IEEE-NEXT: v_fma_mix_f32 v4, v0, 1.0, v4 op_sel_hi:[1,1,0]
; GFX10-IEEE-NEXT: v_mul_f32_e32 v2, v4, v2
; GFX10-IEEE-NEXT: v_and_b32_e32 v2, 0xff800000, v2
; GFX10-IEEE-NEXT: v_add_f32_e32 v2, v2, v3
; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v2, v2
; GFX10-IEEE-NEXT: v_div_fixup_f16 v0, v2, v1, v0
; GFX10-IEEE-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-FLUSH-LABEL: v_fdiv_f16_ulp25:
; GFX10-FLUSH: ; %bb.0:
; GFX10-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v1
; GFX10-FLUSH-NEXT: v_cvt_f32_f16_e32 v4, v0
; GFX10-FLUSH-NEXT: v_rcp_f32_e32 v3, v2
; GFX10-FLUSH-NEXT: v_fma_mix_f32 v5, v0, v3, neg(0) op_sel_hi:[1,0,0]
; GFX10-FLUSH-NEXT: v_mad_f32 v6, -v2, v5, v4
; GFX10-FLUSH-NEXT: v_mac_f32_e32 v5, v6, v3
; GFX10-FLUSH-NEXT: v_mad_f32 v2, -v2, v5, v4
; GFX10-FLUSH-NEXT: v_mul_f32_e32 v2, v2, v3
; GFX10-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
; GFX10-FLUSH-NEXT: v_add_f32_e32 v2, v2, v5
; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
; GFX10-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v1, v0
; GFX10-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fdiv_f16_ulp25:
; GFX11: ; %bb.0:
@ -2183,7 +2267,9 @@ define amdgpu_ps i16 @s_fdiv_f16(i16 inreg %a.arg, i16 inreg %b.arg) {
; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v1, s1
; GFX6-IEEE-NEXT: v_div_scale_f32 v2, s[0:1], v1, v1, v0
; GFX6-IEEE-NEXT: v_rcp_f32_e32 v3, v2
; GFX6-IEEE-NEXT: v_div_scale_f32 v4, vcc, v0, v1, v0
; GFX6-IEEE-NEXT: v_div_scale_f32 v4, s[0:1], v0, v1, v0
; GFX6-IEEE-NEXT: s_or_b64 s[0:1], s[0:1], s[0:1]
; GFX6-IEEE-NEXT: s_cselect_b64 vcc, exec, 0
; GFX6-IEEE-NEXT: v_fma_f32 v5, -v2, v3, 1.0
; GFX6-IEEE-NEXT: v_fma_f32 v3, v5, v3, v3
; GFX6-IEEE-NEXT: v_mul_f32_e32 v5, v4, v3
@ -2198,11 +2284,12 @@ define amdgpu_ps i16 @s_fdiv_f16(i16 inreg %a.arg, i16 inreg %b.arg) {
;
; GFX6-FLUSH-LABEL: s_fdiv_f16:
; GFX6-FLUSH: ; %bb.0:
; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, s1
; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, s0
; GFX6-FLUSH-NEXT: v_div_scale_f32 v2, s[0:1], v0, v0, v1
; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, s0
; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, s1
; GFX6-FLUSH-NEXT: v_div_scale_f32 v2, s[0:1], v1, v1, v0
; GFX6-FLUSH-NEXT: v_div_scale_f32 v4, s[0:1], v0, v1, v0
; GFX6-FLUSH-NEXT: v_rcp_f32_e32 v3, v2
; GFX6-FLUSH-NEXT: v_div_scale_f32 v4, vcc, v1, v0, v1
; GFX6-FLUSH-NEXT: s_or_b64 s[0:1], s[0:1], s[0:1]
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
; GFX6-FLUSH-NEXT: v_fma_f32 v5, -v2, v3, 1.0
; GFX6-FLUSH-NEXT: v_fma_f32 v3, v5, v3, v3
@ -2211,30 +2298,54 @@ define amdgpu_ps i16 @s_fdiv_f16(i16 inreg %a.arg, i16 inreg %b.arg) {
; GFX6-FLUSH-NEXT: v_fma_f32 v5, v6, v3, v5
; GFX6-FLUSH-NEXT: v_fma_f32 v2, -v2, v5, v4
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
; GFX6-FLUSH-NEXT: s_cselect_b64 vcc, exec, 0
; GFX6-FLUSH-NEXT: v_div_fmas_f32 v2, v2, v3, v5
; GFX6-FLUSH-NEXT: v_div_fixup_f32 v0, v2, v0, v1
; GFX6-FLUSH-NEXT: v_div_fixup_f32 v0, v2, v1, v0
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
; GFX6-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX6-FLUSH-NEXT: v_readfirstlane_b32 s0, v0
; GFX6-FLUSH-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_fdiv_f16:
; GFX8: ; %bb.0:
; GFX8-NEXT: v_cvt_f32_f16_e32 v0, s1
; GFX8-NEXT: v_cvt_f32_f16_e32 v1, s0
; GFX8-NEXT: v_rcp_f32_e32 v2, v0
; GFX8-NEXT: v_mul_f32_e32 v3, v1, v2
; GFX8-NEXT: v_mad_f32 v4, -v0, v3, v1
; GFX8-NEXT: v_mac_f32_e32 v3, v4, v2
; GFX8-NEXT: v_mad_f32 v0, -v0, v3, v1
; GFX8-NEXT: v_mul_f32_e32 v0, v0, v2
; GFX8-NEXT: v_and_b32_e32 v0, 0xff800000, v0
; GFX8-NEXT: v_add_f32_e32 v0, v0, v3
; GFX8-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX8-NEXT: v_mov_b32_e32 v1, s0
; GFX8-NEXT: v_div_fixup_f16 v0, v0, s1, v1
; GFX8-NEXT: v_readfirstlane_b32 s0, v0
; GFX8-NEXT: ; return to shader part epilog
; GFX8-IEEE-LABEL: s_fdiv_f16:
; GFX8-IEEE: ; %bb.0:
; GFX8-IEEE-NEXT: v_cvt_f32_f16_e32 v0, s1
; GFX8-IEEE-NEXT: v_cvt_f32_f16_e32 v1, s0
; GFX8-IEEE-NEXT: v_rcp_f32_e32 v2, v0
; GFX8-IEEE-NEXT: v_mul_f32_e32 v3, v1, v2
; GFX8-IEEE-NEXT: v_mul_f32_e64 v4, -v0, v3
; GFX8-IEEE-NEXT: v_add_f32_e32 v4, v4, v1
; GFX8-IEEE-NEXT: v_mul_f32_e32 v4, v4, v2
; GFX8-IEEE-NEXT: v_add_f32_e32 v3, v4, v3
; GFX8-IEEE-NEXT: v_mul_f32_e64 v0, -v0, v3
; GFX8-IEEE-NEXT: v_add_f32_e32 v0, v0, v1
; GFX8-IEEE-NEXT: v_mul_f32_e32 v0, v0, v2
; GFX8-IEEE-NEXT: v_readfirstlane_b32 s2, v0
; GFX8-IEEE-NEXT: s_and_b32 s2, s2, 0xff800000
; GFX8-IEEE-NEXT: v_add_f32_e32 v0, s2, v3
; GFX8-IEEE-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX8-IEEE-NEXT: v_mov_b32_e32 v1, s1
; GFX8-IEEE-NEXT: v_div_fixup_f16 v0, v0, v1, s0
; GFX8-IEEE-NEXT: v_readfirstlane_b32 s0, v0
; GFX8-IEEE-NEXT: ; return to shader part epilog
;
; GFX8-FLUSH-LABEL: s_fdiv_f16:
; GFX8-FLUSH: ; %bb.0:
; GFX8-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, s1
; GFX8-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, s0
; GFX8-FLUSH-NEXT: v_rcp_f32_e32 v2, v0
; GFX8-FLUSH-NEXT: v_mul_f32_e32 v3, v1, v2
; GFX8-FLUSH-NEXT: v_mad_f32 v4, -v0, v3, v1
; GFX8-FLUSH-NEXT: v_mac_f32_e32 v3, v4, v2
; GFX8-FLUSH-NEXT: v_mad_f32 v0, -v0, v3, v1
; GFX8-FLUSH-NEXT: v_mul_f32_e32 v0, v0, v2
; GFX8-FLUSH-NEXT: v_readfirstlane_b32 s2, v0
; GFX8-FLUSH-NEXT: s_and_b32 s2, s2, 0xff800000
; GFX8-FLUSH-NEXT: v_add_f32_e32 v0, s2, v3
; GFX8-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX8-FLUSH-NEXT: v_mov_b32_e32 v1, s1
; GFX8-FLUSH-NEXT: v_div_fixup_f16 v0, v0, v1, s0
; GFX8-FLUSH-NEXT: v_readfirstlane_b32 s0, v0
; GFX8-FLUSH-NEXT: ; return to shader part epilog
;
; GFX9-IEEE-LABEL: s_fdiv_f16:
; GFX9-IEEE: ; %bb.0:
@ -2242,51 +2353,77 @@ define amdgpu_ps i16 @s_fdiv_f16(i16 inreg %a.arg, i16 inreg %b.arg) {
; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v1, s0
; GFX9-IEEE-NEXT: v_rcp_f32_e32 v2, v0
; GFX9-IEEE-NEXT: v_mul_f32_e32 v3, v1, v2
; GFX9-IEEE-NEXT: v_mad_f32 v4, -v0, v3, v1
; GFX9-IEEE-NEXT: v_mac_f32_e32 v3, v4, v2
; GFX9-IEEE-NEXT: v_mad_f32 v0, -v0, v3, v1
; GFX9-IEEE-NEXT: v_mul_f32_e64 v4, -v0, v3
; GFX9-IEEE-NEXT: v_add_f32_e32 v4, v4, v1
; GFX9-IEEE-NEXT: v_mul_f32_e32 v4, v4, v2
; GFX9-IEEE-NEXT: v_add_f32_e32 v3, v4, v3
; GFX9-IEEE-NEXT: v_mul_f32_e64 v0, -v0, v3
; GFX9-IEEE-NEXT: v_add_f32_e32 v0, v0, v1
; GFX9-IEEE-NEXT: v_mul_f32_e32 v0, v0, v2
; GFX9-IEEE-NEXT: v_and_b32_e32 v0, 0xff800000, v0
; GFX9-IEEE-NEXT: v_add_f32_e32 v0, v0, v3
; GFX9-IEEE-NEXT: v_readfirstlane_b32 s2, v0
; GFX9-IEEE-NEXT: s_and_b32 s2, s2, 0xff800000
; GFX9-IEEE-NEXT: v_add_f32_e32 v0, s2, v3
; GFX9-IEEE-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX9-IEEE-NEXT: v_mov_b32_e32 v1, s0
; GFX9-IEEE-NEXT: v_div_fixup_f16 v0, v0, s1, v1
; GFX9-IEEE-NEXT: v_mov_b32_e32 v1, s1
; GFX9-IEEE-NEXT: v_div_fixup_f16 v0, v0, v1, s0
; GFX9-IEEE-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-IEEE-NEXT: ; return to shader part epilog
;
; GFX9-FLUSH-LABEL: s_fdiv_f16:
; GFX9-FLUSH: ; %bb.0:
; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, s1
; GFX9-FLUSH-NEXT: v_mov_b32_e32 v2, s0
; GFX9-FLUSH-NEXT: v_mov_b32_e32 v1, s1
; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v0, v0
; GFX9-FLUSH-NEXT: v_mad_mix_f32 v1, s0, v0, neg(0) op_sel_hi:[1,0,0]
; GFX9-FLUSH-NEXT: v_mad_mix_f32 v3, -s1, v1, v2 op_sel_hi:[1,0,1]
; GFX9-FLUSH-NEXT: v_mac_f32_e32 v1, v3, v0
; GFX9-FLUSH-NEXT: v_mad_mix_f32 v3, -s1, v1, v2 op_sel_hi:[1,0,1]
; GFX9-FLUSH-NEXT: v_mad_mix_f32 v2, s0, v0, neg(0) op_sel_hi:[1,0,0]
; GFX9-FLUSH-NEXT: v_mad_mix_f32 v3, -v1, v2, s0 op_sel_hi:[1,0,1]
; GFX9-FLUSH-NEXT: v_mac_f32_e32 v2, v3, v0
; GFX9-FLUSH-NEXT: v_mad_mix_f32 v3, -v1, v2, s0 op_sel_hi:[1,0,1]
; GFX9-FLUSH-NEXT: v_mul_f32_e32 v0, v3, v0
; GFX9-FLUSH-NEXT: v_and_b32_e32 v0, 0xff800000, v0
; GFX9-FLUSH-NEXT: v_add_f32_e32 v0, v0, v1
; GFX9-FLUSH-NEXT: v_readfirstlane_b32 s1, v0
; GFX9-FLUSH-NEXT: s_and_b32 s1, s1, 0xff800000
; GFX9-FLUSH-NEXT: v_add_f32_e32 v0, s1, v2
; GFX9-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v0, s1, v2
; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v0, v1, s0
; GFX9-FLUSH-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-FLUSH-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_fdiv_f16:
; GFX10: ; %bb.0:
; GFX10-NEXT: v_cvt_f32_f16_e32 v0, s1
; GFX10-NEXT: v_cvt_f32_f16_e32 v3, s0
; GFX10-NEXT: v_rcp_f32_e32 v1, v0
; GFX10-NEXT: v_fma_mix_f32 v2, s0, v1, neg(0) op_sel_hi:[1,0,0]
; GFX10-NEXT: v_mad_f32 v4, -v0, v2, v3
; GFX10-NEXT: v_mac_f32_e32 v2, v4, v1
; GFX10-NEXT: v_mad_f32 v0, -v0, v2, v3
; GFX10-NEXT: v_mul_f32_e32 v0, v0, v1
; GFX10-NEXT: v_and_b32_e32 v0, 0xff800000, v0
; GFX10-NEXT: v_add_f32_e32 v0, v0, v2
; GFX10-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX10-NEXT: v_div_fixup_f16 v0, v0, s1, s0
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-NEXT: ; return to shader part epilog
; GFX10-IEEE-LABEL: s_fdiv_f16:
; GFX10-IEEE: ; %bb.0:
; GFX10-IEEE-NEXT: v_cvt_f32_f16_e32 v0, s1
; GFX10-IEEE-NEXT: v_rcp_f32_e32 v0, v0
; GFX10-IEEE-NEXT: v_fma_mix_f32 v1, s0, v0, neg(0) op_sel_hi:[1,0,0]
; GFX10-IEEE-NEXT: v_fma_mix_f32 v2, -s1, v1, neg(0) op_sel_hi:[1,0,0]
; GFX10-IEEE-NEXT: v_fma_mix_f32 v2, s0, 1.0, v2 op_sel_hi:[1,1,0]
; GFX10-IEEE-NEXT: v_mul_f32_e32 v2, v2, v0
; GFX10-IEEE-NEXT: v_add_f32_e32 v1, v2, v1
; GFX10-IEEE-NEXT: v_fma_mix_f32 v2, -s1, v1, neg(0) op_sel_hi:[1,0,0]
; GFX10-IEEE-NEXT: v_fma_mix_f32 v2, s0, 1.0, v2 op_sel_hi:[1,1,0]
; GFX10-IEEE-NEXT: v_mul_f32_e32 v0, v2, v0
; GFX10-IEEE-NEXT: v_readfirstlane_b32 s2, v0
; GFX10-IEEE-NEXT: s_and_b32 s2, s2, 0xff800000
; GFX10-IEEE-NEXT: v_add_f32_e32 v0, s2, v1
; GFX10-IEEE-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX10-IEEE-NEXT: v_div_fixup_f16 v0, v0, s1, s0
; GFX10-IEEE-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-IEEE-NEXT: ; return to shader part epilog
;
; GFX10-FLUSH-LABEL: s_fdiv_f16:
; GFX10-FLUSH: ; %bb.0:
; GFX10-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, s1
; GFX10-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, s0
; GFX10-FLUSH-NEXT: v_rcp_f32_e32 v1, v0
; GFX10-FLUSH-NEXT: v_fma_mix_f32 v3, s0, v1, neg(0) op_sel_hi:[1,0,0]
; GFX10-FLUSH-NEXT: v_mad_f32 v4, -v0, v3, v2
; GFX10-FLUSH-NEXT: v_mac_f32_e32 v3, v4, v1
; GFX10-FLUSH-NEXT: v_mad_f32 v0, -v0, v3, v2
; GFX10-FLUSH-NEXT: v_mul_f32_e32 v0, v0, v1
; GFX10-FLUSH-NEXT: v_readfirstlane_b32 s2, v0
; GFX10-FLUSH-NEXT: s_and_b32 s2, s2, 0xff800000
; GFX10-FLUSH-NEXT: v_add_f32_e32 v0, s2, v3
; GFX10-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX10-FLUSH-NEXT: v_div_fixup_f16 v0, v0, s1, s0
; GFX10-FLUSH-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-FLUSH-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_fdiv_f16:
; GFX11: ; %bb.0:
@ -2298,8 +2435,9 @@ define amdgpu_ps i16 @s_fdiv_f16(i16 inreg %a.arg, i16 inreg %b.arg) {
; GFX11-NEXT: v_fmac_f32_e32 v1, v2, v0
; GFX11-NEXT: v_fma_mix_f32 v2, -s1, v1, s0 op_sel_hi:[1,0,1]
; GFX11-NEXT: v_mul_f32_e32 v0, v2, v0
; GFX11-NEXT: v_and_b32_e32 v0, 0xff800000, v0
; GFX11-NEXT: v_add_f32_e32 v0, v0, v1
; GFX11-NEXT: v_readfirstlane_b32 s2, v0
; GFX11-NEXT: s_and_b32 s2, s2, 0xff800000
; GFX11-NEXT: v_add_f32_e32 v0, s2, v1
; GFX11-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX11-NEXT: v_div_fixup_f16 v0, v0, s1, s0
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
@ -2318,7 +2456,9 @@ define amdgpu_ps i16 @s_fdiv_f16_arcp(i16 inreg %a.arg, i16 inreg %b.arg) {
; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v1, s1
; GFX6-IEEE-NEXT: v_div_scale_f32 v2, s[0:1], v1, v1, v0
; GFX6-IEEE-NEXT: v_rcp_f32_e32 v3, v2
; GFX6-IEEE-NEXT: v_div_scale_f32 v4, vcc, v0, v1, v0
; GFX6-IEEE-NEXT: v_div_scale_f32 v4, s[0:1], v0, v1, v0
; GFX6-IEEE-NEXT: s_or_b64 s[0:1], s[0:1], s[0:1]
; GFX6-IEEE-NEXT: s_cselect_b64 vcc, exec, 0
; GFX6-IEEE-NEXT: v_fma_f32 v5, -v2, v3, 1.0
; GFX6-IEEE-NEXT: v_fma_f32 v3, v5, v3, v3
; GFX6-IEEE-NEXT: v_mul_f32_e32 v5, v4, v3
@ -2333,11 +2473,12 @@ define amdgpu_ps i16 @s_fdiv_f16_arcp(i16 inreg %a.arg, i16 inreg %b.arg) {
;
; GFX6-FLUSH-LABEL: s_fdiv_f16_arcp:
; GFX6-FLUSH: ; %bb.0:
; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, s1
; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, s0
; GFX6-FLUSH-NEXT: v_div_scale_f32 v2, s[0:1], v0, v0, v1
; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, s0
; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, s1
; GFX6-FLUSH-NEXT: v_div_scale_f32 v2, s[0:1], v1, v1, v0
; GFX6-FLUSH-NEXT: v_div_scale_f32 v4, s[0:1], v0, v1, v0
; GFX6-FLUSH-NEXT: v_rcp_f32_e32 v3, v2
; GFX6-FLUSH-NEXT: v_div_scale_f32 v4, vcc, v1, v0, v1
; GFX6-FLUSH-NEXT: s_or_b64 s[0:1], s[0:1], s[0:1]
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
; GFX6-FLUSH-NEXT: v_fma_f32 v5, -v2, v3, 1.0
; GFX6-FLUSH-NEXT: v_fma_f32 v3, v5, v3, v3
@ -2346,8 +2487,9 @@ define amdgpu_ps i16 @s_fdiv_f16_arcp(i16 inreg %a.arg, i16 inreg %b.arg) {
; GFX6-FLUSH-NEXT: v_fma_f32 v5, v6, v3, v5
; GFX6-FLUSH-NEXT: v_fma_f32 v2, -v2, v5, v4
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
; GFX6-FLUSH-NEXT: s_cselect_b64 vcc, exec, 0
; GFX6-FLUSH-NEXT: v_div_fmas_f32 v2, v2, v3, v5
; GFX6-FLUSH-NEXT: v_div_fixup_f32 v0, v2, v0, v1
; GFX6-FLUSH-NEXT: v_div_fixup_f32 v0, v2, v1, v0
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
; GFX6-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX6-FLUSH-NEXT: v_readfirstlane_b32 s0, v0
@ -2674,7 +2816,9 @@ define amdgpu_ps i16 @s_rcp_f16(i16 inreg %a.arg) {
; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v0, s0
; GFX6-IEEE-NEXT: v_div_scale_f32 v1, s[0:1], v0, v0, 1.0
; GFX6-IEEE-NEXT: v_rcp_f32_e32 v2, v1
; GFX6-IEEE-NEXT: v_div_scale_f32 v3, vcc, 1.0, v0, 1.0
; GFX6-IEEE-NEXT: v_div_scale_f32 v3, s[0:1], 1.0, v0, 1.0
; GFX6-IEEE-NEXT: s_or_b64 s[0:1], s[0:1], s[0:1]
; GFX6-IEEE-NEXT: s_cselect_b64 vcc, exec, 0
; GFX6-IEEE-NEXT: v_fma_f32 v4, -v1, v2, 1.0
; GFX6-IEEE-NEXT: v_fma_f32 v2, v4, v2, v2
; GFX6-IEEE-NEXT: v_mul_f32_e32 v4, v3, v2
@ -2691,8 +2835,9 @@ define amdgpu_ps i16 @s_rcp_f16(i16 inreg %a.arg) {
; GFX6-FLUSH: ; %bb.0:
; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, s0
; GFX6-FLUSH-NEXT: v_div_scale_f32 v1, s[0:1], v0, v0, 1.0
; GFX6-FLUSH-NEXT: v_div_scale_f32 v3, s[0:1], 1.0, v0, 1.0
; GFX6-FLUSH-NEXT: v_rcp_f32_e32 v2, v1
; GFX6-FLUSH-NEXT: v_div_scale_f32 v3, vcc, 1.0, v0, 1.0
; GFX6-FLUSH-NEXT: s_or_b64 s[0:1], s[0:1], s[0:1]
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
; GFX6-FLUSH-NEXT: v_fma_f32 v4, -v1, v2, 1.0
; GFX6-FLUSH-NEXT: v_fma_f32 v2, v4, v2, v2
@ -2701,6 +2846,7 @@ define amdgpu_ps i16 @s_rcp_f16(i16 inreg %a.arg) {
; GFX6-FLUSH-NEXT: v_fma_f32 v4, v5, v2, v4
; GFX6-FLUSH-NEXT: v_fma_f32 v1, -v1, v4, v3
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
; GFX6-FLUSH-NEXT: s_cselect_b64 vcc, exec, 0
; GFX6-FLUSH-NEXT: v_div_fmas_f32 v1, v1, v2, v4
; GFX6-FLUSH-NEXT: v_div_fixup_f32 v0, v1, v0, 1.0
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
@ -2738,7 +2884,9 @@ define amdgpu_ps i16 @s_neg_rcp_f16(i16 inreg %a.arg) {
; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v0, s0
; GFX6-IEEE-NEXT: v_div_scale_f32 v1, s[0:1], v0, v0, -1.0
; GFX6-IEEE-NEXT: v_rcp_f32_e32 v2, v1
; GFX6-IEEE-NEXT: v_div_scale_f32 v3, vcc, -1.0, v0, -1.0
; GFX6-IEEE-NEXT: v_div_scale_f32 v3, s[0:1], -1.0, v0, -1.0
; GFX6-IEEE-NEXT: s_or_b64 s[0:1], s[0:1], s[0:1]
; GFX6-IEEE-NEXT: s_cselect_b64 vcc, exec, 0
; GFX6-IEEE-NEXT: v_fma_f32 v4, -v1, v2, 1.0
; GFX6-IEEE-NEXT: v_fma_f32 v2, v4, v2, v2
; GFX6-IEEE-NEXT: v_mul_f32_e32 v4, v3, v2
@ -2755,8 +2903,9 @@ define amdgpu_ps i16 @s_neg_rcp_f16(i16 inreg %a.arg) {
; GFX6-FLUSH: ; %bb.0:
; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, s0
; GFX6-FLUSH-NEXT: v_div_scale_f32 v1, s[0:1], v0, v0, -1.0
; GFX6-FLUSH-NEXT: v_div_scale_f32 v3, s[0:1], -1.0, v0, -1.0
; GFX6-FLUSH-NEXT: v_rcp_f32_e32 v2, v1
; GFX6-FLUSH-NEXT: v_div_scale_f32 v3, vcc, -1.0, v0, -1.0
; GFX6-FLUSH-NEXT: s_or_b64 s[0:1], s[0:1], s[0:1]
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
; GFX6-FLUSH-NEXT: v_fma_f32 v4, -v1, v2, 1.0
; GFX6-FLUSH-NEXT: v_fma_f32 v2, v4, v2, v2
@ -2765,6 +2914,7 @@ define amdgpu_ps i16 @s_neg_rcp_f16(i16 inreg %a.arg) {
; GFX6-FLUSH-NEXT: v_fma_f32 v4, v5, v2, v4
; GFX6-FLUSH-NEXT: v_fma_f32 v1, -v1, v4, v3
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
; GFX6-FLUSH-NEXT: s_cselect_b64 vcc, exec, 0
; GFX6-FLUSH-NEXT: v_div_fmas_f32 v1, v1, v2, v4
; GFX6-FLUSH-NEXT: v_div_fixup_f32 v0, v1, v0, -1.0
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
@ -2797,13 +2947,54 @@ define amdgpu_ps i16 @s_neg_rcp_f16(i16 inreg %a.arg) {
}
define amdgpu_ps i16 @s_rsq_f16(i16 inreg %a.arg) {
; GFX6-LABEL: s_rsq_f16:
; GFX6: ; %bb.0:
; GFX6-NEXT: v_cvt_f32_f16_e32 v0, s0
; GFX6-NEXT: v_rsq_f32_e32 v0, v0
; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX6-NEXT: v_readfirstlane_b32 s0, v0
; GFX6-NEXT: ; return to shader part epilog
; GFX6-IEEE-LABEL: s_rsq_f16:
; GFX6-IEEE: ; %bb.0:
; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v0, s0
; GFX6-IEEE-NEXT: v_sqrt_f32_e32 v0, v0
; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX6-IEEE-NEXT: v_div_scale_f32 v1, s[0:1], v0, v0, 1.0
; GFX6-IEEE-NEXT: v_rcp_f32_e32 v2, v1
; GFX6-IEEE-NEXT: v_div_scale_f32 v3, s[0:1], 1.0, v0, 1.0
; GFX6-IEEE-NEXT: s_or_b64 s[0:1], s[0:1], s[0:1]
; GFX6-IEEE-NEXT: s_cselect_b64 vcc, exec, 0
; GFX6-IEEE-NEXT: v_fma_f32 v4, -v1, v2, 1.0
; GFX6-IEEE-NEXT: v_fma_f32 v2, v4, v2, v2
; GFX6-IEEE-NEXT: v_mul_f32_e32 v4, v3, v2
; GFX6-IEEE-NEXT: v_fma_f32 v5, -v1, v4, v3
; GFX6-IEEE-NEXT: v_fma_f32 v4, v5, v2, v4
; GFX6-IEEE-NEXT: v_fma_f32 v1, -v1, v4, v3
; GFX6-IEEE-NEXT: v_div_fmas_f32 v1, v1, v2, v4
; GFX6-IEEE-NEXT: v_div_fixup_f32 v0, v1, v0, 1.0
; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX6-IEEE-NEXT: v_readfirstlane_b32 s0, v0
; GFX6-IEEE-NEXT: ; return to shader part epilog
;
; GFX6-FLUSH-LABEL: s_rsq_f16:
; GFX6-FLUSH: ; %bb.0:
; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, s0
; GFX6-FLUSH-NEXT: v_sqrt_f32_e32 v0, v0
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
; GFX6-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX6-FLUSH-NEXT: v_div_scale_f32 v1, s[0:1], v0, v0, 1.0
; GFX6-FLUSH-NEXT: v_div_scale_f32 v3, s[0:1], 1.0, v0, 1.0
; GFX6-FLUSH-NEXT: v_rcp_f32_e32 v2, v1
; GFX6-FLUSH-NEXT: s_or_b64 s[0:1], s[0:1], s[0:1]
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
; GFX6-FLUSH-NEXT: v_fma_f32 v4, -v1, v2, 1.0
; GFX6-FLUSH-NEXT: v_fma_f32 v2, v4, v2, v2
; GFX6-FLUSH-NEXT: v_mul_f32_e32 v4, v3, v2
; GFX6-FLUSH-NEXT: v_fma_f32 v5, -v1, v4, v3
; GFX6-FLUSH-NEXT: v_fma_f32 v4, v5, v2, v4
; GFX6-FLUSH-NEXT: v_fma_f32 v1, -v1, v4, v3
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
; GFX6-FLUSH-NEXT: s_cselect_b64 vcc, exec, 0
; GFX6-FLUSH-NEXT: v_div_fmas_f32 v1, v1, v2, v4
; GFX6-FLUSH-NEXT: v_div_fixup_f32 v0, v1, v0, 1.0
; GFX6-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX6-FLUSH-NEXT: v_readfirstlane_b32 s0, v0
; GFX6-FLUSH-NEXT: ; return to shader part epilog
;
; GFX89-LABEL: s_rsq_f16:
; GFX89: ; %bb.0:
@ -3074,23 +3265,23 @@ define half @v_neg_rsq_f16(half %a) {
; GFX89-LABEL: v_neg_rsq_f16:
; GFX89: ; %bb.0:
; GFX89-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX89-NEXT: v_rsq_f16_e32 v0, v0
; GFX89-NEXT: v_xor_b32_e32 v0, 0x8000, v0
; GFX89-NEXT: v_sqrt_f16_e32 v0, v0
; GFX89-NEXT: v_rcp_f16_e64 v0, -v0
; GFX89-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_neg_rsq_f16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_rsq_f16_e32 v0, v0
; GFX10-NEXT: v_xor_b32_e32 v0, 0x8000, v0
; GFX10-NEXT: v_sqrt_f16_e32 v0, v0
; GFX10-NEXT: v_rcp_f16_e64 v0, -v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_neg_rsq_f16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_rsq_f16_e32 v0, v0
; GFX11-NEXT: v_sqrt_f16_e32 v0, v0
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11-NEXT: v_xor_b32_e32 v0, 0x8000, v0
; GFX11-NEXT: v_rcp_f16_e64 v0, -v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%sqrt = call contract half @llvm.sqrt.f16(half %a)
%fdiv = fdiv contract half -1.0, %sqrt
@ -3146,23 +3337,26 @@ define { half, half } @v_rsq_f16_multi_use(half %a) {
; GFX89-LABEL: v_rsq_f16_multi_use:
; GFX89: ; %bb.0:
; GFX89-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX89-NEXT: v_sqrt_f16_e32 v0, v0
; GFX89-NEXT: v_rcp_f16_e32 v1, v0
; GFX89-NEXT: v_sqrt_f16_e32 v2, v0
; GFX89-NEXT: v_rsq_f16_e32 v1, v0
; GFX89-NEXT: v_mov_b32_e32 v0, v2
; GFX89-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_rsq_f16_multi_use:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_sqrt_f16_e32 v0, v0
; GFX10-NEXT: v_rcp_f16_e32 v1, v0
; GFX10-NEXT: v_sqrt_f16_e32 v2, v0
; GFX10-NEXT: v_rsq_f16_e32 v1, v0
; GFX10-NEXT: v_mov_b32_e32 v0, v2
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_rsq_f16_multi_use:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_sqrt_f16_e32 v0, v0
; GFX11-NEXT: v_sqrt_f16_e32 v2, v0
; GFX11-NEXT: v_rsq_f16_e32 v1, v0
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11-NEXT: v_rcp_f16_e32 v1, v0
; GFX11-NEXT: v_mov_b32_e32 v0, v2
; GFX11-NEXT: s_setpc_b64 s[30:31]
%sqrt = call contract half @llvm.sqrt.f16(half %a)
%insert.0 = insertvalue { half, half } poison, half %sqrt, 0
@ -3508,23 +3702,23 @@ define half @v_neg_rsq_f16_fabs(half %a) {
; GFX89-LABEL: v_neg_rsq_f16_fabs:
; GFX89: ; %bb.0:
; GFX89-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX89-NEXT: v_rsq_f16_e64 v0, |v0|
; GFX89-NEXT: v_xor_b32_e32 v0, 0x8000, v0
; GFX89-NEXT: v_sqrt_f16_e64 v0, |v0|
; GFX89-NEXT: v_rcp_f16_e64 v0, -v0
; GFX89-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_neg_rsq_f16_fabs:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_rsq_f16_e64 v0, |v0|
; GFX10-NEXT: v_xor_b32_e32 v0, 0x8000, v0
; GFX10-NEXT: v_sqrt_f16_e64 v0, |v0|
; GFX10-NEXT: v_rcp_f16_e64 v0, -v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_neg_rsq_f16_fabs:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_rsq_f16_e64 v0, |v0|
; GFX11-NEXT: v_sqrt_f16_e64 v0, |v0|
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11-NEXT: v_xor_b32_e32 v0, 0x8000, v0
; GFX11-NEXT: v_rcp_f16_e64 v0, -v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%a.fabs = call half @llvm.fabs.f16(half %a)
%sqrt = call contract half @llvm.sqrt.f16(half %a.fabs)
@ -3643,23 +3837,23 @@ define half @v_neg_rsq_f16_arcp(half %a) {
; GFX89-LABEL: v_neg_rsq_f16_arcp:
; GFX89: ; %bb.0:
; GFX89-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX89-NEXT: v_rsq_f16_e32 v0, v0
; GFX89-NEXT: v_xor_b32_e32 v0, 0x8000, v0
; GFX89-NEXT: v_sqrt_f16_e32 v0, v0
; GFX89-NEXT: v_rcp_f16_e64 v0, -v0
; GFX89-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_neg_rsq_f16_arcp:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_rsq_f16_e32 v0, v0
; GFX10-NEXT: v_xor_b32_e32 v0, 0x8000, v0
; GFX10-NEXT: v_sqrt_f16_e32 v0, v0
; GFX10-NEXT: v_rcp_f16_e64 v0, -v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_neg_rsq_f16_arcp:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_rsq_f16_e32 v0, v0
; GFX11-NEXT: v_sqrt_f16_e32 v0, v0
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11-NEXT: v_xor_b32_e32 v0, 0x8000, v0
; GFX11-NEXT: v_rcp_f16_e64 v0, -v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%sqrt = call contract half @llvm.sqrt.f16(half %a)
%fdiv = fdiv contract arcp half -1.0, %sqrt
@ -3991,9 +4185,5 @@ declare <2 x half> @llvm.fabs.v2f16(<2 x half>)
declare <2 x half> @llvm.sqrt.v2f16(<2 x half>)
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GFX10-FLUSH: {{.*}}
; GFX10-IEEE: {{.*}}
; GFX11-FLUSH: {{.*}}
; GFX11-IEEE: {{.*}}
; GFX8-FLUSH: {{.*}}
; GFX8-IEEE: {{.*}}

4
llvm/test/CodeGen/AMDGPU/GlobalISel/ffloor.ll
View file

@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgpu8.03 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefix=FIJI %s
; RUN: llc -mtriple=amdgpu12.00 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefix=GFX12 %s
; RUN: llc -mtriple=amdgpu8.03 -global-isel=1 < %s | FileCheck -check-prefix=FIJI %s
; RUN: llc -mtriple=amdgpu12.00 -global-isel=1 < %s | FileCheck -check-prefix=GFX12 %s
define float @ffloor_s(float inreg %val) {
; FIJI-LABEL: ffloor_s:

6
llvm/test/CodeGen/AMDGPU/GlobalISel/fma.ll
View file

@ -4,9 +4,9 @@
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefix=GFX9 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.0a < %s | FileCheck -check-prefix=GFX90A %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefix=GFX12 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s
; RUN: llc -global-isel -mtriple=amdgpu12.00 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefix=GFX12 %s
define float @v_fma_f32(float %x, float %y, float %z) {
; GFX6-LABEL: v_fma_f32:

18
llvm/test/CodeGen/AMDGPU/GlobalISel/fmad.ll
View file

@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-amd-amdpal --denormal-fp-math=preserve-sign -o - %s | FileCheck -check-prefix=GFX8 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00-amd-amdpal --denormal-fp-math=preserve-sign -o - %s | FileCheck -check-prefix=GFX9 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-amd-amdpal --denormal-fp-math=preserve-sign -o - %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -global-isel -mtriple=amdgpu8.03-amd-amdpal --denormal-fp-math=preserve-sign -o - %s | FileCheck -check-prefix=GFX8 %s
; RUN: llc -global-isel -mtriple=amdgpu9.00-amd-amdpal --denormal-fp-math=preserve-sign -o - %s | FileCheck -check-prefix=GFX9 %s
; RUN: llc -global-isel -mtriple=amdgpu10.10-amd-amdpal --denormal-fp-math=preserve-sign -o - %s | FileCheck -check-prefix=GFX10 %s
define amdgpu_ps float @fmad_s32_uniform(float inreg %a, float inreg %b, float inreg %c) {
; GFX8-LABEL: fmad_s32_uniform:
@ -51,16 +51,16 @@ define amdgpu_ps float @fmad_s32_div(float %a, float %b, float %c) {
define amdgpu_ps half @fmad_s16_uniform(half inreg %a, half inreg %b, half inreg %c) {
; GFX8-LABEL: fmad_s16_uniform:
; GFX8: ; %bb.0:
; GFX8-NEXT: v_mov_b32_e32 v0, s2
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: v_mad_f16 v0, s0, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s1
; GFX8-NEXT: v_mov_b32_e32 v1, s2
; GFX8-NEXT: v_mad_f16 v0, s0, v0, v1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: fmad_s16_uniform:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_mov_b32_e32 v0, s2
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: v_mad_legacy_f16 v0, s0, v1, v0
; GFX9-NEXT: v_mov_b32_e32 v0, s1
; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: v_mad_legacy_f16 v0, s0, v0, v1
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: fmad_s16_uniform:

29
llvm/test/CodeGen/AMDGPU/GlobalISel/fmamix-constant-bus-violation.ll
View file

@ -1,13 +1,16 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.08 %s -o - | FileCheck %s
; RUN: llc -global-isel -mtriple=amdgpu9.08 %s -o - | FileCheck %s
define float @test_fmamix_constant_bus_violation_sss(i32 inreg %val.0, i32 inreg %val.1, i32 inreg %val.2) #0 {
; CHECK-LABEL: test_fmamix_constant_bus_violation_sss:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: v_mov_b32_e32 v0, s17
; CHECK-NEXT: v_mov_b32_e32 v1, s18
; CHECK-NEXT: v_fma_mix_f32 v0, s16, v0, v1 op_sel:[1,1,1] op_sel_hi:[1,1,1]
; CHECK-NEXT: s_lshr_b32 s5, s17, 16
; CHECK-NEXT: s_lshr_b32 s6, s18, 16
; CHECK-NEXT: s_lshr_b32 s4, s16, 16
; CHECK-NEXT: v_mov_b32_e32 v0, s5
; CHECK-NEXT: v_mov_b32_e32 v1, s6
; CHECK-NEXT: v_fma_mix_f32 v0, s4, v0, v1 op_sel_hi:[1,1,1]
; CHECK-NEXT: s_setpc_b64 s[30:31]
%lshr.0 = lshr i32 %val.0, 16
%lshr.1 = lshr i32 %val.1, 16
@ -29,8 +32,10 @@ define float @test_fmamix_constant_bus_violation_ssv(i32 inreg %val.0, i32 inreg
; CHECK-LABEL: test_fmamix_constant_bus_violation_ssv:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: v_mov_b32_e32 v1, s17
; CHECK-NEXT: v_fma_mix_f32 v0, s16, v1, v0 op_sel:[1,1,1] op_sel_hi:[1,1,1]
; CHECK-NEXT: s_lshr_b32 s5, s17, 16
; CHECK-NEXT: s_lshr_b32 s4, s16, 16
; CHECK-NEXT: v_mov_b32_e32 v1, s5
; CHECK-NEXT: v_fma_mix_f32 v0, s4, v1, v0 op_sel:[0,0,1] op_sel_hi:[1,1,1]
; CHECK-NEXT: s_setpc_b64 s[30:31]
%lshr.0 = lshr i32 %val.0, 16
%lshr.1 = lshr i32 %val.1, 16
@ -52,8 +57,10 @@ define float @test_fmamix_constant_bus_violation_svs(i32 inreg %val.0, i32 %val.
; CHECK-LABEL: test_fmamix_constant_bus_violation_svs:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: v_mov_b32_e32 v1, s17
; CHECK-NEXT: v_fma_mix_f32 v0, s16, v0, v1 op_sel:[1,1,1] op_sel_hi:[1,1,1]
; CHECK-NEXT: s_lshr_b32 s5, s17, 16
; CHECK-NEXT: s_lshr_b32 s4, s16, 16
; CHECK-NEXT: v_mov_b32_e32 v1, s5
; CHECK-NEXT: v_fma_mix_f32 v0, s4, v0, v1 op_sel:[0,1,0] op_sel_hi:[1,1,1]
; CHECK-NEXT: s_setpc_b64 s[30:31]
%lshr.0 = lshr i32 %val.0, 16
%lshr.1 = lshr i32 %val.1, 16
@ -75,8 +82,10 @@ define float @test_fmamix_constant_bus_violation_vss(i32 %val.0, i32 inreg %val.
; CHECK-LABEL: test_fmamix_constant_bus_violation_vss:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: v_mov_b32_e32 v1, s17
; CHECK-NEXT: v_fma_mix_f32 v0, v0, s16, v1 op_sel:[1,1,1] op_sel_hi:[1,1,1]
; CHECK-NEXT: s_lshr_b32 s5, s17, 16
; CHECK-NEXT: s_lshr_b32 s4, s16, 16
; CHECK-NEXT: v_mov_b32_e32 v1, s5
; CHECK-NEXT: v_fma_mix_f32 v0, v0, s4, v1 op_sel:[1,0,0] op_sel_hi:[1,1,1]
; CHECK-NEXT: s_setpc_b64 s[30:31]
%lshr.0 = lshr i32 %val.0, 16
%lshr.1 = lshr i32 %val.1, 16

14
llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3-min-max-const-combine.ll
View file

@ -247,14 +247,13 @@ define half @test_max_K0min_ValK1_f16(half %a) #0 {
; GFX10-LABEL: test_max_K0min_ValK1_f16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_min_f16_e32 v0, 4.0, v0
; GFX10-NEXT: v_max_f16_e32 v0, 2.0, v0
; GFX10-NEXT: v_med3_f16 v0, v0, 2.0, 4.0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX1170-LABEL: test_max_K0min_ValK1_f16:
; GFX1170: ; %bb.0:
; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1170-NEXT: v_minmax_num_f16 v0, v0, 4.0, 2.0
; GFX1170-NEXT: v_med3_num_f16 v0, v0, 2.0, 4.0
; GFX1170-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-TRUE16-LABEL: test_max_K0min_ValK1_f16:
@ -274,7 +273,7 @@ define half @test_max_K0min_ValK1_f16(half %a) #0 {
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: v_minmax_num_f16 v0, v0, 4.0, 2.0
; GFX12-FAKE16-NEXT: v_med3_num_f16 v0, v0, 2.0, 4.0
; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%minnum = call nnan half @llvm.minnum.f16(half %a, half 4.0)
%fmed = call nnan half @llvm.maxnum.f16(half 2.0, half %minnum)
@ -292,14 +291,13 @@ define half @test_max_K0min_K1Val_f16(half %a) #1 {
; GFX10-LABEL: test_max_K0min_K1Val_f16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_min_f16_e32 v0, 4.0, v0
; GFX10-NEXT: v_max_f16_e32 v0, 2.0, v0
; GFX10-NEXT: v_med3_f16 v0, v0, 2.0, 4.0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX1170-LABEL: test_max_K0min_K1Val_f16:
; GFX1170: ; %bb.0:
; GFX1170-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1170-NEXT: v_minmax_num_f16 v0, v0, 4.0, 2.0
; GFX1170-NEXT: v_med3_num_f16 v0, v0, 2.0, 4.0
; GFX1170-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-TRUE16-LABEL: test_max_K0min_K1Val_f16:
@ -319,7 +317,7 @@ define half @test_max_K0min_K1Val_f16(half %a) #1 {
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: v_minmax_num_f16 v0, v0, 4.0, 2.0
; GFX12-FAKE16-NEXT: v_med3_num_f16 v0, v0, 2.0, 4.0
; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
%minnum = call nnan half @llvm.minnum.f16(half 4.0, half %a)
%fmed = call nnan half @llvm.maxnum.f16(half 2.0, half %minnum)

10
llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3.ll
View file

@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00 -o - %s | FileCheck -check-prefix=GFX9 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00 -o - %s | FileCheck -check-prefix=GFX12 %s
; RUN: llc -global-isel -mtriple=amdgpu9.00 -o - %s | FileCheck -check-prefix=GFX9 %s
; RUN: llc -global-isel -mtriple=amdgpu12.00 -o - %s | FileCheck -check-prefix=GFX12 %s
define amdgpu_ps half @fmed3_s16_uniform(half inreg %a, half inreg %b, half inreg %c) {
; GFX9-LABEL: fmed3_s16_uniform:
@ -34,7 +34,11 @@ define amdgpu_ps half @fmed3_s16_uniform_salu_use(half inreg %a, half inreg %b,
; GFX12-NEXT: v_mov_b16_e32 v0.l, s2
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_med3_num_f16 v0.l, s0, s1, v0.l
; GFX12-NEXT: v_add_f16_e32 v0.l, s3, v0.l
; GFX12-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-NEXT: s_add_f16 s0, s0, s3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_2)
; GFX12-NEXT: v_mov_b32_e32 v0, s0
; GFX12-NEXT: ; return to shader part epilog
%result = call half @llvm.amdgcn.fmed3.f16(half %a, half %b, half %c)
%add = fadd half %result, %d

2
llvm/test/CodeGen/AMDGPU/GlobalISel/fmin3-fmax3-combine.ll
View file

@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-amd-mesa3d -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -global-isel -mtriple=amdgpu10.10-amd-mesa3d -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -global-isel -mtriple=amdgpu12.00-amd-mesa3d -verify-machineinstrs < %s | FileCheck -check-prefix=GFX12 %s
define float @test_fmin3(float %a, float %b, float %c) {
; GFX10-LABEL: test_fmin3:

8
llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.ll
View file

@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-FAKE16 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=+real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-TRUE16 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-FAKE16 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00-amd-amdpal -mattr=+real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-TRUE16 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-FAKE16 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -mattr=+real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-TRUE16 %s
; RUN: llc -global-isel -mtriple=amdgpu12.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-FAKE16 %s
; RUN: llc -global-isel -mtriple=amdgpu12.00-amd-amdpal -mattr=+real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-TRUE16 %s
define amdgpu_ps half @fmul_s16_uniform(half inreg %a, half inreg %b) {
; GFX11-FAKE16-LABEL: fmul_s16_uniform:

53
llvm/test/CodeGen/AMDGPU/GlobalISel/fneg.ll
View file

@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX11 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX12 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX11 %s
; RUN: llc -global-isel -mtriple=amdgpu12.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX12 %s
define amdgpu_ps void @v_fneg_f16(half %in, ptr addrspace(1) %out) {
; GCN-LABEL: v_fneg_f16:
@ -13,27 +13,44 @@ define amdgpu_ps void @v_fneg_f16(half %in, ptr addrspace(1) %out) {
ret void
}
define amdgpu_ps void @s_fneg_f16(half inreg %in, ptr addrspace(1) %out) {
; GCN-LABEL: s_fneg_f16:
; GCN: ; %bb.0:
; GCN-NEXT: s_xor_b32 s0, s0, 0x8000
; GCN-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GCN-NEXT: v_mov_b32_e32 v2, s0
; GCN-NEXT: global_store_b16 v[0:1], v2, off
; GCN-NEXT: s_endpgm
; GFX11-LABEL: s_fneg_f16:
; GFX11: ; %bb.0:
; GFX11-NEXT: v_xor_b32_e64 v2, 0x8000, s0
; GFX11-NEXT: global_store_b16 v[0:1], v2, off
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: s_fneg_f16:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_xor_b32 s0, s0, 0x8000
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: v_mov_b32_e32 v2, s0
; GFX12-NEXT: global_store_b16 v[0:1], v2, off
; GFX12-NEXT: s_endpgm
%fneg = fneg half %in
store half %fneg, ptr addrspace(1) %out
ret void
}
define amdgpu_ps void @s_fneg_f16_salu_use(half inreg %in, i32 inreg %val, ptr addrspace(1) %out) {
; GCN-LABEL: s_fneg_f16_salu_use:
; GCN: ; %bb.0:
; GCN-NEXT: s_xor_b32 s0, s0, 0x8000
; GCN-NEXT: s_cmp_eq_u32 s1, 0
; GCN-NEXT: s_cselect_b32 s1, -1, 0
; GCN-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GCN-NEXT: v_cndmask_b32_e64 v2, 0, s0, s1
; GCN-NEXT: global_store_b16 v[0:1], v2, off
; GCN-NEXT: s_endpgm
; GFX11-LABEL: s_fneg_f16_salu_use:
; GFX11: ; %bb.0:
; GFX11-NEXT: v_xor_b32_e64 v2, 0x8000, s0
; GFX11-NEXT: s_cmp_eq_u32 s1, 0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: v_readfirstlane_b32 s0, v2
; GFX11-NEXT: s_cselect_b32 s0, s0, 0
; GFX11-NEXT: v_mov_b32_e32 v2, s0
; GFX11-NEXT: global_store_b16 v[0:1], v2, off
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: s_fneg_f16_salu_use:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_xor_b32 s0, s0, 0x8000
; GFX12-NEXT: s_cmp_eq_u32 s1, 0
; GFX12-NEXT: s_cselect_b32 s0, s0, 0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: v_mov_b32_e32 v2, s0
; GFX12-NEXT: global_store_b16 v[0:1], v2, off
; GFX12-NEXT: s_endpgm
%fneg = fneg half %in
%cond = icmp eq i32 %val, 0
%sel = select i1 %cond, half %fneg, half 0.0

36
llvm/test/CodeGen/AMDGPU/GlobalISel/fpow.ll
View file

@ -135,34 +135,34 @@ define half @v_pow_f16(half %x, half %y) {
; GFX8-LABEL: v_pow_f16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX8-NEXT: v_log_f16_e32 v0, v0
; GFX8-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX8-NEXT: v_log_f32_e32 v0, v0
; GFX8-NEXT: v_mul_legacy_f32_e32 v0, v1, v0
; GFX8-NEXT: v_exp_f32_e32 v0, v0
; GFX8-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX8-NEXT: v_mul_legacy_f32_e32 v0, v0, v1
; GFX8-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX8-NEXT: v_exp_f16_e32 v0, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_pow_f16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX9-NEXT: v_log_f16_e32 v0, v0
; GFX9-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX9-NEXT: v_log_f32_e32 v0, v0
; GFX9-NEXT: v_mul_legacy_f32_e32 v0, v1, v0
; GFX9-NEXT: v_exp_f32_e32 v0, v0
; GFX9-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX9-NEXT: v_mul_legacy_f32_e32 v0, v0, v1
; GFX9-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX9-NEXT: v_exp_f16_e32 v0, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_pow_f16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX10-NEXT: v_log_f16_e32 v0, v0
; GFX10-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX10-NEXT: v_log_f32_e32 v0, v0
; GFX10-NEXT: v_mul_legacy_f32_e32 v0, v1, v0
; GFX10-NEXT: v_exp_f32_e32 v0, v0
; GFX10-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX10-NEXT: v_mul_legacy_f32_e32 v0, v0, v1
; GFX10-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX10-NEXT: v_exp_f16_e32 v0, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: v_pow_f16:
@ -182,15 +182,15 @@ define half @v_pow_f16(half %x, half %y) {
; GFX11-FAKE16-LABEL: v_pow_f16:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX11-FAKE16-NEXT: v_log_f16_e32 v0, v0
; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_log_f32_e32 v0, v0
; GFX11-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11-FAKE16-NEXT: v_mul_dx9_zero_f32_e32 v0, v1, v0
; GFX11-FAKE16-NEXT: v_exp_f32_e32 v0, v0
; GFX11-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_mul_dx9_zero_f32_e32 v0, v0, v1
; GFX11-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_exp_f16_e32 v0, v0
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%pow = call half @llvm.pow.f16(half %x, half %y)
ret half %pow

3043
llvm/test/CodeGen/AMDGPU/GlobalISel/frem.ll
View file

File diff suppressed because it is too large Load diff

6355
llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll
View file

File diff suppressed because it is too large Load diff

5982
llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
View file

File diff suppressed because it is too large Load diff

8
llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.ll
View file

@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-FAKE16 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=+real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-TRUE16 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-FAKE16 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00-amd-amdpal -mattr=+real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-TRUE16 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-FAKE16 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -mattr=+real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-TRUE16 %s
; RUN: llc -global-isel -mtriple=amdgpu12.00-amd-amdpal -mattr=-real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-FAKE16 %s
; RUN: llc -global-isel -mtriple=amdgpu12.00-amd-amdpal -mattr=+real-true16 -o - %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-TRUE16 %s
define amdgpu_ps half @fsub_s16_uniform(half inreg %a, half inreg %b) {
; GFX11-FAKE16-LABEL: fsub_s16_uniform:

4
llvm/test/CodeGen/AMDGPU/GlobalISel/intrinsic-trunc.ll
View file

@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgpu8.03 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefix=FIJI %s
; RUN: llc -mtriple=amdgpu12.00 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefix=GFX12 %s
; RUN: llc -mtriple=amdgpu8.03 -global-isel=1 < %s | FileCheck -check-prefix=FIJI %s
; RUN: llc -mtriple=amdgpu12.00 -global-isel=1 < %s | FileCheck -check-prefix=GFX12 %s
define float @intrinsic_trunc_s(float inreg %val) {
; FIJI-LABEL: intrinsic_trunc_s:

18
llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-bitcast.mir
View file

@ -1,6 +1,24 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
# RUN: llc -mtriple=amdgpu8.03-mesa-mesa3d -run-pass=legalizer %s -o - | FileCheck %s
---
name: test_bitcast_i16_to_f16
body: |
bb.0:
liveins: $vgpr0
; CHECK-LABEL: name: test_bitcast_i16_to_f16
; CHECK: liveins: $vgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
; CHECK-NEXT: $vgpr0 = COPY [[COPY]](i32)
%0:_(i32) = COPY $vgpr0
%1:_(i16) = G_TRUNC %0(i32)
%2:_(f16) = G_BITCAST %1(i16)
%3:_(i32) = G_ANYEXT %2(f16)
$vgpr0 = COPY %3(i32)
...
---
name: test_bitcast_s32_to_v2s16
body: |

119
llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fptrunc.mir
View file

@ -1,5 +1,4 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
# UNSUPPORTED: true
# RUN: llc -mtriple=amdgpu8.03-mesa-mesa3d -run-pass=legalizer %s -o - | FileCheck %s
---
@ -11,11 +10,11 @@ body: |
; CHECK-LABEL: name: test_fptrunc_s64_to_s32
; CHECK: liveins: $vgpr0_vgpr1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[FPTRUNC:%[0-9]+]]:_(s32) = G_FPTRUNC [[COPY]](s64)
; CHECK-NEXT: $vgpr0 = COPY [[FPTRUNC]](s32)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s32) = G_FPTRUNC %0
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(f64) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[FPTRUNC:%[0-9]+]]:_(f32) = G_FPTRUNC [[COPY]](f64)
; CHECK-NEXT: $vgpr0 = COPY [[FPTRUNC]](f32)
%0:_(f64) = COPY $vgpr0_vgpr1
%1:_(f32) = G_FPTRUNC %0
$vgpr0 = COPY %1
...
@ -28,12 +27,12 @@ body: |
; CHECK-LABEL: name: test_fptrunc_s32_to_s16
; CHECK: liveins: $vgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
; CHECK-NEXT: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[COPY]](s32)
; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FPTRUNC]](s16)
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(f32) = COPY $vgpr0
; CHECK-NEXT: [[FPTRUNC:%[0-9]+]]:_(f16) = G_FPTRUNC [[COPY]](f32)
; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FPTRUNC]](f16)
; CHECK-NEXT: $vgpr0 = COPY [[ANYEXT]](s32)
%0:_(s32) = COPY $vgpr0
%1:_(s16) = G_FPTRUNC %0
%0:_(f32) = COPY $vgpr0
%1:_(f16) = G_FPTRUNC %0
%2:_(s32) = G_ANYEXT %1
$vgpr0 = COPY %2
...
@ -47,14 +46,14 @@ body: |
; CHECK-LABEL: name: test_fptrunc_v2s64_to_v2s32
; CHECK: liveins: $vgpr0_vgpr1_vgpr2_vgpr3
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
; CHECK-NEXT: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
; CHECK-NEXT: [[FPTRUNC:%[0-9]+]]:_(s32) = G_FPTRUNC [[UV]](s64)
; CHECK-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s32) = G_FPTRUNC [[UV1]](s64)
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FPTRUNC]](s32), [[FPTRUNC1]](s32)
; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
%0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
%1:_(<2 x s32>) = G_FPTRUNC %0
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x f64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
; CHECK-NEXT: [[UV:%[0-9]+]]:_(f64), [[UV1:%[0-9]+]]:_(f64) = G_UNMERGE_VALUES [[COPY]](<2 x f64>)
; CHECK-NEXT: [[FPTRUNC:%[0-9]+]]:_(f32) = G_FPTRUNC [[UV]](f64)
; CHECK-NEXT: [[FPTRUNC1:%[0-9]+]]:_(f32) = G_FPTRUNC [[UV1]](f64)
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[FPTRUNC]](f32), [[FPTRUNC1]](f32)
; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x f32>)
%0:_(<2 x f64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
%1:_(<2 x f32>) = G_FPTRUNC %0
$vgpr0_vgpr1 = COPY %1
...
@ -67,22 +66,22 @@ body: |
; CHECK-LABEL: name: test_fptrunc_v2s32_to_v2s16
; CHECK: liveins: $vgpr0_vgpr1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
; CHECK-NEXT: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[UV]](s32)
; CHECK-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[UV1]](s32)
; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x f32>) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY]](<2 x f32>)
; CHECK-NEXT: [[FPTRUNC:%[0-9]+]]:_(f16) = G_FPTRUNC [[UV]](f32)
; CHECK-NEXT: [[FPTRUNC1:%[0-9]+]]:_(f16) = G_FPTRUNC [[UV1]](f32)
; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](f16)
; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](f16)
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[BITCAST]](<2 x s16>)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[BITCAST]](<2 x f16>)
; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[BITCAST1]](s32), [[LSHR]](s32)
; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
%0:_(<2 x s32>) = COPY $vgpr0_vgpr1
%1:_(<2 x s16>) = G_FPTRUNC %0
%0:_(<2 x f32>) = COPY $vgpr0_vgpr1
%1:_(<2 x f16>) = G_FPTRUNC %0
%2:_(<2 x s32>) = G_ANYEXT %1
$vgpr0_vgpr1 = COPY %2
...
@ -96,8 +95,8 @@ body: |
; CHECK-LABEL: name: test_fptrunc_s64_to_s16
; CHECK: liveins: $vgpr0_vgpr1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(f64) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](f64)
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[UV1]], [[C]](s32)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2047
@ -161,8 +160,8 @@ body: |
; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR4]], [[C20]]
; CHECK-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[AND4]], [[SELECT3]]
; CHECK-NEXT: $vgpr0 = COPY [[OR7]](s32)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s16) = G_FPTRUNC %0
%0:_(f64) = COPY $vgpr0_vgpr1
%1:_(f16) = G_FPTRUNC %0
%2:_(s32) = G_ANYEXT %1
$vgpr0 = COPY %2
...
@ -176,9 +175,9 @@ body: |
; CHECK-LABEL: name: test_fptrunc_v2s64_to_v2s16
; CHECK: liveins: $vgpr0_vgpr1_vgpr2_vgpr3
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
; CHECK-NEXT: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
; CHECK-NEXT: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV]](s64)
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x f64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
; CHECK-NEXT: [[UV:%[0-9]+]]:_(f64), [[UV1:%[0-9]+]]:_(f64) = G_UNMERGE_VALUES [[COPY]](<2 x f64>)
; CHECK-NEXT: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV]](f64)
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[UV3]], [[C]](s32)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2047
@ -241,7 +240,7 @@ body: |
; CHECK-NEXT: [[C20:%[0-9]+]]:_(s32) = G_CONSTANT i32 32768
; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR4]], [[C20]]
; CHECK-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[AND4]], [[SELECT3]]
; CHECK-NEXT: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64)
; CHECK-NEXT: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](f64)
; CHECK-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[UV5]], [[C]](s32)
; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LSHR5]], [[C1]]
; CHECK-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[AND5]], [[C2]]
@ -287,11 +286,11 @@ body: |
; CHECK-NEXT: [[AND10:%[0-9]+]]:_(s32) = G_AND [[OR7]], [[C21]]
; CHECK-NEXT: [[AND11:%[0-9]+]]:_(s32) = G_AND [[OR15]], [[C21]]
; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND11]], [[C19]](s32)
; CHECK-NEXT: [[OR16:%[0-9]+]]:_(s32) = G_OR [[AND10]], [[SHL4]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR16]](s32)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
%0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
%1:_(<2 x s16>) = G_FPTRUNC %0
; CHECK-NEXT: [[OR16:%[0-9]+]]:_(i32) = G_OR [[AND10]], [[SHL4]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR16]](i32)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
%0:_(<2 x f64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
%1:_(<2 x f16>) = G_FPTRUNC %0
$vgpr0 = COPY %1
...
@ -304,8 +303,8 @@ body: |
; CHECK-LABEL: name: test_fptrunc_s64_to_s16_afn
; CHECK: liveins: $vgpr0_vgpr1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(f64) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](f64)
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[UV1]], [[C]](s32)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2047
@ -369,8 +368,8 @@ body: |
; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR4]], [[C20]]
; CHECK-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[AND4]], [[SELECT3]]
; CHECK-NEXT: $vgpr0 = COPY [[OR7]](s32)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s16) = G_FPTRUNC %0
%0:_(f64) = COPY $vgpr0_vgpr1
%1:_(f16) = G_FPTRUNC %0
%2:_(s32) = afn G_ANYEXT %1
$vgpr0 = COPY %2
...
@ -384,20 +383,20 @@ body: |
; CHECK-LABEL: name: test_fptrunc_v2s64_to_v2s16_afn
; CHECK: liveins: $vgpr0_vgpr1_vgpr2_vgpr3
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
; CHECK-NEXT: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
; CHECK-NEXT: [[FPTRUNC:%[0-9]+]]:_(s32) = afn G_FPTRUNC [[UV]](s64)
; CHECK-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = afn G_FPTRUNC [[FPTRUNC]](s32)
; CHECK-NEXT: [[FPTRUNC2:%[0-9]+]]:_(s32) = afn G_FPTRUNC [[UV1]](s64)
; CHECK-NEXT: [[FPTRUNC3:%[0-9]+]]:_(s16) = afn G_FPTRUNC [[FPTRUNC2]](s32)
; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](s16)
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x f64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
; CHECK-NEXT: [[UV:%[0-9]+]]:_(f64), [[UV1:%[0-9]+]]:_(f64) = G_UNMERGE_VALUES [[COPY]](<2 x f64>)
; CHECK-NEXT: [[FPTRUNC:%[0-9]+]]:_(s32) = afn G_FPTRUNC [[UV]](f64)
; CHECK-NEXT: [[FPTRUNC1:%[0-9]+]]:_(f16) = afn G_FPTRUNC [[FPTRUNC]](s32)
; CHECK-NEXT: [[FPTRUNC2:%[0-9]+]]:_(s32) = afn G_FPTRUNC [[UV1]](f64)
; CHECK-NEXT: [[FPTRUNC3:%[0-9]+]]:_(f16) = afn G_FPTRUNC [[FPTRUNC2]](s32)
; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](f16)
; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](f16)
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
%0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
%1:_(<2 x s16>) = afn G_FPTRUNC %0
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
%0:_(<2 x f64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
%1:_(<2 x f16>) = afn G_FPTRUNC %0
$vgpr0 = COPY %1
...

1255
llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.load.2d.d16.ll
View file

File diff suppressed because it is too large Load diff

1652
llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.a16.ll
View file

File diff suppressed because it is too large Load diff

80
llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.g16.a16.ll
View file

@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -global-isel -mtriple=amdgpu10.10-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX11 %s
; RUN: llc -global-isel -mtriple=amdgpu12.00-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX12 %s
@ -23,18 +23,15 @@ define amdgpu_ps <4 x float> @sample_d_1d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[DEF]](f16)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
@ -145,26 +142,20 @@ define amdgpu_ps <4 x float> @sample_d_2d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX10-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(f16) = G_TRUNC [[COPY17]](i32)
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC4]](f16), [[TRUNC5]](f16)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
@ -291,39 +282,30 @@ define amdgpu_ps <4 x float> @sample_d_3d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX10-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(f16) = G_TRUNC [[COPY17]](i32)
; GFX10-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
; GFX10-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
; GFX10-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
; GFX10-NEXT: [[TRUNC6:%[0-9]+]]:_(f16) = G_TRUNC [[COPY18]](i32)
; GFX10-NEXT: [[COPY19:%[0-9]+]]:_(i32) = COPY $vgpr7
; GFX10-NEXT: [[TRUNC7:%[0-9]+]]:_(i16) = G_TRUNC [[COPY19]](i32)
; GFX10-NEXT: [[BITCAST7:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC7]](i16)
; GFX10-NEXT: [[TRUNC7:%[0-9]+]]:_(f16) = G_TRUNC [[COPY19]](i32)
; GFX10-NEXT: [[COPY20:%[0-9]+]]:_(i32) = COPY $vgpr8
; GFX10-NEXT: [[TRUNC8:%[0-9]+]]:_(i16) = G_TRUNC [[COPY20]](i32)
; GFX10-NEXT: [[BITCAST8:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC8]](i16)
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
; GFX10-NEXT: [[TRUNC8:%[0-9]+]]:_(f16) = G_TRUNC [[COPY20]](i32)
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST3]](f16), [[BITCAST4]](f16)
; GFX10-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST5]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BUILD_VECTOR6:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[BITCAST7]](f16)
; GFX10-NEXT: [[BUILD_VECTOR7:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST8]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC3]](f16), [[TRUNC4]](f16)
; GFX10-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC5]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BUILD_VECTOR6:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC6]](f16), [[TRUNC7]](f16)
; GFX10-NEXT: [[BUILD_VECTOR7:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC8]](f16), [[DEF]](f16)
; GFX10-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>), [[BUILD_VECTOR6]](<2 x f16>), [[BUILD_VECTOR7]](<2 x f16>)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.3d), 15, [[CONCAT_VECTORS]](<12 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)

412
llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.g16.ll
View file

@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -global-isel -mtriple=amdgpu10.10-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX11 %s
; RUN: llc -global-isel -mtriple=amdgpu12.00-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX12 %s
@ -23,17 +23,15 @@ define amdgpu_ps <4 x float> @sample_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(f32) = COPY $vgpr2
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@ -139,24 +137,20 @@ define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(f32) = COPY $vgpr4
; GFX10-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST4]](<2 x f16>), [[BITCAST5]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@ -276,35 +270,29 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
; GFX10-NEXT: [[TRUNC4:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX10-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
; GFX10-NEXT: [[TRUNC5:%[0-9]+]]:_(f16) = G_TRUNC [[COPY17]](i32)
; GFX10-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
; GFX10-NEXT: [[COPY19:%[0-9]+]]:_(f32) = COPY $vgpr7
; GFX10-NEXT: [[COPY20:%[0-9]+]]:_(f32) = COPY $vgpr8
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST3]](f16), [[BITCAST4]](f16)
; GFX10-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST5]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
; GFX10-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
; GFX10-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY20]](f32)
; GFX10-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<14 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>), [[BITCAST6]](<2 x f16>), [[BITCAST7]](<2 x f16>), [[BITCAST8]](<2 x f16>)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC3]](f16), [[TRUNC4]](f16)
; GFX10-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC5]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY20]](f32)
; GFX10-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<14 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.3d), 15, [[CONCAT_VECTORS]](<14 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
@ -450,18 +438,16 @@ define amdgpu_ps <4 x float> @sample_c_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(f32) = COPY $vgpr3
; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.1d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@ -572,25 +558,21 @@ define amdgpu_ps <4 x float> @sample_c_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX10-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
; GFX10-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
; GFX10-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.2d), 15, [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST5]](<2 x f16>), [[BITCAST6]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@ -715,19 +697,17 @@ define amdgpu_ps <4 x float> @sample_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(f32) = COPY $vgpr2
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(f32) = COPY $vgpr3
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@ -837,26 +817,22 @@ define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(f32) = COPY $vgpr4
; GFX10-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
; GFX10-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
; GFX10-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST4]](<2 x f16>), [[BITCAST5]](<2 x f16>), [[BITCAST6]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@ -982,20 +958,18 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(f32) = COPY $vgpr3
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(f32) = COPY $vgpr4
; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.1d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@ -1111,27 +1085,23 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX10-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
; GFX10-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
; GFX10-NEXT: [[COPY19:%[0-9]+]]:_(f32) = COPY $vgpr7
; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
; GFX10-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
; GFX10-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
; GFX10-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x f16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST5]](<2 x f16>), [[BITCAST6]](<2 x f16>), [[BITCAST7]](<2 x f16>)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
; GFX10-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x f16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BITCAST3]](<2 x f16>)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.2d), 15, [[CONCAT_VECTORS]](<12 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
@ -1262,17 +1232,15 @@ define amdgpu_ps <4 x float> @sample_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(f32) = COPY $vgpr2
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@ -1378,24 +1346,20 @@ define amdgpu_ps <4 x float> @sample_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(f32) = COPY $vgpr4
; GFX10-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST4]](<2 x f16>), [[BITCAST5]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@ -1516,18 +1480,16 @@ define amdgpu_ps <4 x float> @sample_c_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(f32) = COPY $vgpr3
; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.1d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@ -1638,25 +1600,21 @@ define amdgpu_ps <4 x float> @sample_c_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX10-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
; GFX10-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
; GFX10-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.2d), 15, [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST5]](<2 x f16>), [[BITCAST6]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@ -1781,19 +1739,17 @@ define amdgpu_ps <4 x float> @sample_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(f32) = COPY $vgpr2
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(f32) = COPY $vgpr3
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@ -1903,26 +1859,22 @@ define amdgpu_ps <4 x float> @sample_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX10-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY12]](i32)
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(f32) = COPY $vgpr4
; GFX10-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
; GFX10-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
; GFX10-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST4]](<2 x f16>), [[BITCAST5]](<2 x f16>), [[BITCAST6]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@ -2048,20 +2000,18 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(f32) = COPY $vgpr3
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(f32) = COPY $vgpr4
; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.1d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC1]](f16), [[DEF]](f16)
; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX10-NEXT: $vgpr1 = COPY [[UV1]](f32)
@ -2177,27 +2127,23 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX10-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY13]](i32)
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX10-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
; GFX10-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
; GFX10-NEXT: [[COPY19:%[0-9]+]]:_(f32) = COPY $vgpr7
; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
; GFX10-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
; GFX10-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
; GFX10-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x f16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST5]](<2 x f16>), [[BITCAST6]](<2 x f16>), [[BITCAST7]](<2 x f16>)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
; GFX10-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x f16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BITCAST3]](<2 x f16>)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.2d), 15, [[CONCAT_VECTORS]](<12 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)
@ -2330,28 +2276,24 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX10-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY17]](i32)
; GFX10-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
; GFX10-NEXT: [[COPY19:%[0-9]+]]:_(f32) = COPY $vgpr7
; GFX10-NEXT: [[COPY20:%[0-9]+]]:_(f32) = COPY $vgpr8
; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
; GFX10-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
; GFX10-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
; GFX10-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY20]](f32)
; GFX10-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<14 x f16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x f16>), [[BITCAST5]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST6]](<2 x f16>), [[BITCAST7]](<2 x f16>), [[BITCAST8]](<2 x f16>)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY20]](f32)
; GFX10-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<14 x f16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BITCAST4]](<2 x f16>)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(f32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 4, [[CONCAT_VECTORS]](<14 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (f32), addrspace 8)
; GFX10-NEXT: $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](f32)
; GFX10-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
@ -2476,28 +2418,24 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
; GFX10-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX10-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
; GFX10-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY14]](i32)
; GFX10-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:_(f16) = G_TRUNC [[COPY15]](i32)
; GFX10-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
; GFX10-NEXT: [[TRUNC2:%[0-9]+]]:_(f16) = G_TRUNC [[COPY16]](i32)
; GFX10-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
; GFX10-NEXT: [[TRUNC3:%[0-9]+]]:_(f16) = G_TRUNC [[COPY17]](i32)
; GFX10-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
; GFX10-NEXT: [[COPY19:%[0-9]+]]:_(f32) = COPY $vgpr7
; GFX10-NEXT: [[COPY20:%[0-9]+]]:_(f32) = COPY $vgpr8
; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
; GFX10-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
; GFX10-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
; GFX10-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
; GFX10-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY20]](f32)
; GFX10-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<14 x f16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x f16>), [[BITCAST5]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST6]](<2 x f16>), [[BITCAST7]](<2 x f16>), [[BITCAST8]](<2 x f16>)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
; GFX10-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
; GFX10-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC]](f16), [[TRUNC1]](f16)
; GFX10-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[TRUNC2]](f16), [[TRUNC3]](f16)
; GFX10-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
; GFX10-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
; GFX10-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY20]](f32)
; GFX10-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<14 x f16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BITCAST4]](<2 x f16>)
; GFX10-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 6, [[CONCAT_VECTORS]](<14 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<2 x f32>), addrspace 8)
; GFX10-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x f32>)
; GFX10-NEXT: $vgpr0 = COPY [[UV]](f32)

100
llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.store.2d.d16.ll
View file

@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.02-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=UNPACKED %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.10-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX81 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX9 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -global-isel -mtriple=amdgpu8.02-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=UNPACKED %s
; RUN: llc -global-isel -mtriple=amdgpu8.10-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX81 %s
; RUN: llc -global-isel -mtriple=amdgpu9.00-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX9 %s
; RUN: llc -global-isel -mtriple=amdgpu10.10-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -global-isel -mtriple=amdgpu12.00-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX12 %s
@ -23,10 +23,9 @@ define amdgpu_ps void @image_store_f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t, ha
; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $vgpr2
; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY10]](i32)
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY10]](i32)
; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
; UNPACKED-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[BITCAST]](f16), 1, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable store (f16), addrspace 8)
; UNPACKED-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[TRUNC]](f16), 1, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable store (f16), addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; GFX81-LABEL: name: image_store_f16
@ -45,10 +44,9 @@ define amdgpu_ps void @image_store_f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t, ha
; GFX81-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX81-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
; GFX81-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $vgpr2
; GFX81-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY10]](i32)
; GFX81-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; GFX81-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY10]](i32)
; GFX81-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
; GFX81-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[BITCAST]](f16), 1, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable store (f16), addrspace 8)
; GFX81-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[TRUNC]](f16), 1, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable store (f16), addrspace 8)
; GFX81-NEXT: S_ENDPGM 0
;
; GFX9-LABEL: name: image_store_f16
@ -67,33 +65,11 @@ define amdgpu_ps void @image_store_f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t, ha
; GFX9-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX9-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
; GFX9-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $vgpr2
; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY10]](i32)
; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[COPY10]](i32)
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
; GFX9-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[BITCAST]](f16), 1, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable store (f16), addrspace 8)
; GFX9-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[TRUNC]](f16), 1, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable store (f16), addrspace 8)
; GFX9-NEXT: S_ENDPGM 0
;
; GFX10-LABEL: name: image_store_f16
; GFX10: bb.1 (%ir-block.0):
; GFX10-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2
; GFX10-NEXT: {{ $}}
; GFX10-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX10-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
; GFX10-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
; GFX10-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
; GFX10-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
; GFX10-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
; GFX10-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
; GFX10-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX10-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX10-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
; GFX10-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $vgpr2
; GFX10-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY10]](i32)
; GFX10-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; GFX10-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[BITCAST]](f16), 1, [[COPY8]](i32), [[COPY9]](i32), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable store (f16), addrspace 8)
; GFX10-NEXT: S_ENDPGM 0
;
; GFX12-LABEL: name: image_store_f16
; GFX12: bb.1 (%ir-block.0):
; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2
@ -247,21 +223,13 @@ define amdgpu_ps void @image_store_v3f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t,
; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr2
; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr3
; UNPACKED-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[COPY10]](<2 x f16>), [[COPY11]](<2 x f16>)
; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(f16), [[UV1:%[0-9]+]]:_(f16), [[UV2:%[0-9]+]]:_(f16), [[UV3:%[0-9]+]]:_(f16) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<4 x f16>)
; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<3 x f16>) = G_BUILD_VECTOR [[UV]](f16), [[UV1]](f16), [[UV2]](f16)
; UNPACKED-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
; UNPACKED-NEXT: [[UV4:%[0-9]+]]:_(f16), [[UV5:%[0-9]+]]:_(f16), [[UV6:%[0-9]+]]:_(f16) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<3 x f16>)
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(i16) = G_BITCAST [[UV4]](f16)
; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(i16) = G_BITCAST [[UV5]](f16)
; UNPACKED-NEXT: [[BITCAST2:%[0-9]+]]:_(i16) = G_BITCAST [[UV6]](f16)
; UNPACKED-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<3 x i16>) = G_BUILD_VECTOR [[BITCAST]](i16), [[BITCAST1]](i16), [[BITCAST2]](i16)
; UNPACKED-NEXT: [[UV7:%[0-9]+]]:_(i16), [[UV8:%[0-9]+]]:_(i16), [[UV9:%[0-9]+]]:_(i16) = G_UNMERGE_VALUES [[BUILD_VECTOR3]](<3 x i16>)
; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[UV7]](i16)
; UNPACKED-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[UV8]](i16)
; UNPACKED-NEXT: [[ANYEXT2:%[0-9]+]]:_(i32) = G_ANYEXT [[UV9]](i16)
; UNPACKED-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<3 x i32>) = G_BUILD_VECTOR [[ANYEXT]](i32), [[ANYEXT1]](i32), [[ANYEXT2]](i32)
; UNPACKED-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[BUILD_VECTOR4]](<3 x i32>), 7, [[BUILD_VECTOR2]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable store (<3 x f16>), align 8, addrspace 8)
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY10]](<2 x f16>)
; UNPACKED-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY11]](<2 x f16>)
; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
; UNPACKED-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x i32>) = G_BUILD_VECTOR [[BITCAST]](s32), [[LSHR]](s32), [[BITCAST1]](s32)
; UNPACKED-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[BUILD_VECTOR2]](<3 x i32>), 7, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable store (<3 x f16>), align 8, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; GFX81-LABEL: name: image_store_v3f16
@ -281,20 +249,26 @@ define amdgpu_ps void @image_store_v3f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t,
; GFX81-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
; GFX81-NEXT: [[COPY10:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr2
; GFX81-NEXT: [[COPY11:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr3
; GFX81-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[COPY10]](<2 x f16>), [[COPY11]](<2 x f16>)
; GFX81-NEXT: [[UV:%[0-9]+]]:_(f16), [[UV1:%[0-9]+]]:_(f16), [[UV2:%[0-9]+]]:_(f16), [[UV3:%[0-9]+]]:_(f16) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<4 x f16>)
; GFX81-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<3 x f16>) = G_BUILD_VECTOR [[UV]](f16), [[UV1]](f16), [[UV2]](f16)
; GFX81-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
; GFX81-NEXT: [[UV4:%[0-9]+]]:_(f16), [[UV5:%[0-9]+]]:_(f16), [[UV6:%[0-9]+]]:_(f16) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<3 x f16>)
; GFX81-NEXT: [[BITCAST:%[0-9]+]]:_(i16) = G_BITCAST [[UV4]](f16)
; GFX81-NEXT: [[BITCAST1:%[0-9]+]]:_(i16) = G_BITCAST [[UV5]](f16)
; GFX81-NEXT: [[BITCAST2:%[0-9]+]]:_(i16) = G_BITCAST [[UV6]](f16)
; GFX81-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<3 x i16>) = G_BUILD_VECTOR [[BITCAST]](i16), [[BITCAST1]](i16), [[BITCAST2]](i16)
; GFX81-NEXT: [[UV7:%[0-9]+]]:_(i16), [[UV8:%[0-9]+]]:_(i16), [[UV9:%[0-9]+]]:_(i16) = G_UNMERGE_VALUES [[BUILD_VECTOR3]](<3 x i16>)
; GFX81-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX81-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<6 x i16>) = G_BUILD_VECTOR [[UV7]](i16), [[UV8]](i16), [[UV9]](i16), [[DEF]](i16), [[DEF]](i16), [[DEF]](i16)
; GFX81-NEXT: [[BITCAST3:%[0-9]+]]:_(<3 x i32>) = G_BITCAST [[BUILD_VECTOR4]](<6 x i16>)
; GFX81-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[BITCAST3]](<3 x i32>), 7, [[BUILD_VECTOR2]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable store (<3 x f16>), align 8, addrspace 8)
; GFX81-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY10]](<2 x f16>)
; GFX81-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
; GFX81-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
; GFX81-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY11]](<2 x f16>)
; GFX81-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
; GFX81-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
; GFX81-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
; GFX81-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C]](s32)
; GFX81-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX81-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX81-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
; GFX81-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
; GFX81-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C]](s32)
; GFX81-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; GFX81-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX81-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL1]]
; GFX81-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX81-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x i16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>)
; GFX81-NEXT: [[BITCAST5:%[0-9]+]]:_(<3 x i32>) = G_BITCAST [[CONCAT_VECTORS]](<6 x i16>)
; GFX81-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[BITCAST5]](<3 x i32>), 7, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable store (<3 x f16>), align 8, addrspace 8)
; GFX81-NEXT: S_ENDPGM 0
;
; GFX9-LABEL: name: image_store_v3f16

9
llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.fract.ll
View file

@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00 < %s | FileCheck -check-prefix=GFX12 %s
; RUN: llc -global-isel -mtriple=amdgpu12.00 < %s | FileCheck -check-prefix=GFX12 %s
declare half @llvm.amdgcn.fract.f16(half)
declare float @llvm.amdgcn.fract.f32(float)
@ -9,8 +9,11 @@ define amdgpu_ps half @s_fract_f16(half inreg %src) {
; GFX12-LABEL: s_fract_f16:
; GFX12: ; %bb.0:
; GFX12-NEXT: v_fract_f16_e32 v0.l, s0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_add_f16_e32 v0.l, v0.l, v0.l
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_2)
; GFX12-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-NEXT: s_add_f16 s0, s0, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: v_mov_b32_e32 v0, s0
; GFX12-NEXT: ; return to shader part epilog
%fract = call half @llvm.amdgcn.fract.f16(half %src)
%res = fadd half %fract, %fract

2
llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.frexp.mant.ll
View file

@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefix=GFX9 %s
; RUN: llc -global-isel -mtriple=amdgpu9.00 < %s | FileCheck -check-prefix=GFX9 %s
declare half @llvm.amdgcn.frexp.mant.f16(half)
declare float @llvm.amdgcn.frexp.mant.f32(float)

1441
llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.a16.dim.ll
View file

File diff suppressed because it is too large Load diff

48
llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.cd.g16.ll
View file

@ -1,9 +1,13 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -global-isel -mtriple=amdgpu10.10 < %s | FileCheck -check-prefix=GFX10 %s
define amdgpu_ps <4 x float> @sample_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dsdv, float %s) {
; GFX10-LABEL: sample_cd_1d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX10-NEXT: v_lshl_or_b32 v0, s0, 16, v0
; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX10-NEXT: image_sample_cd_g16 v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@ -15,9 +19,11 @@ main_body:
define amdgpu_ps <4 x float> @sample_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t) {
; GFX10-LABEL: sample_cd_2d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
; GFX10-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
; GFX10-NEXT: image_sample_cd_g16 v[0:3], [v0, v2, v4, v5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX10-NEXT: v_lshl_or_b32 v1, v3, 16, v2
; GFX10-NEXT: image_sample_cd_g16 v[0:3], [v0, v1, v4, v5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
main_body:
@ -28,6 +34,10 @@ main_body:
define amdgpu_ps <4 x float> @sample_c_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %dsdh, half %dsdv, float %s) {
; GFX10-LABEL: sample_c_cd_1d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
; GFX10-NEXT: image_sample_c_cd_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@ -39,9 +49,11 @@ main_body:
define amdgpu_ps <4 x float> @sample_c_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t) {
; GFX10-LABEL: sample_c_cd_2d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: v_perm_b32 v3, v4, v3, 0x5040100
; GFX10-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
; GFX10-NEXT: image_sample_c_cd_g16 v[0:3], [v0, v1, v3, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX10-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX10-NEXT: v_lshl_or_b32 v2, v4, 16, v3
; GFX10-NEXT: image_sample_c_cd_g16 v[0:3], [v0, v1, v2, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
main_body:
@ -52,6 +64,10 @@ main_body:
define amdgpu_ps <4 x float> @sample_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dsdv, float %s, float %clamp) {
; GFX10-LABEL: sample_cd_cl_1d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX10-NEXT: v_lshl_or_b32 v0, s0, 16, v0
; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX10-NEXT: image_sample_cd_cl_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@ -63,9 +79,11 @@ main_body:
define amdgpu_ps <4 x float> @sample_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t, float %clamp) {
; GFX10-LABEL: sample_cd_cl_2d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
; GFX10-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
; GFX10-NEXT: image_sample_cd_cl_g16 v[0:3], [v0, v2, v4, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX10-NEXT: v_lshl_or_b32 v1, v3, 16, v2
; GFX10-NEXT: image_sample_cd_cl_g16 v[0:3], [v0, v1, v4, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
main_body:
@ -76,6 +94,10 @@ main_body:
define amdgpu_ps <4 x float> @sample_c_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %dsdh, half %dsdv, float %s, float %clamp) {
; GFX10-LABEL: sample_c_cd_cl_1d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
; GFX10-NEXT: image_sample_c_cd_cl_g16 v[0:3], v[0:4], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@ -89,8 +111,10 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: v_mov_b32_e32 v8, v2
; GFX10-NEXT: v_mov_b32_e32 v2, v0
; GFX10-NEXT: v_perm_b32 v4, v4, v3, 0x5040100
; GFX10-NEXT: v_perm_b32 v3, v8, v1, 0x5040100
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v1
; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
; GFX10-NEXT: v_lshl_or_b32 v3, v8, 16, v0
; GFX10-NEXT: v_lshl_or_b32 v4, v4, 16, v1
; GFX10-NEXT: image_sample_c_cd_cl_g16 v[0:3], v[2:7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog

236
llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.g16.ll
View file

@ -1,23 +1,35 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 -amdgpu-enable-vopd=0 < %s | FileCheck -check-prefixes=GFX11 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 -amdgpu-enable-vopd=0 < %s | FileCheck -check-prefixes=GFX12 %s
; RUN: llc -global-isel -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 -amdgpu-enable-vopd=0 < %s | FileCheck -check-prefixes=GFX11 %s
; RUN: llc -global-isel -mtriple=amdgpu12.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 -amdgpu-enable-vopd=0 < %s | FileCheck -check-prefixes=GFX12 %s
define amdgpu_ps <4 x float> @sample_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dsdv, float %s) {
; GFX10-LABEL: sample_d_1d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX10-NEXT: v_lshl_or_b32 v0, s0, 16, v0
; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX10-NEXT: image_sample_d_g16 v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: sample_d_1d:
; GFX11: ; %bb.0: ; %main_body
; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-NEXT: v_lshl_or_b32 v0, s0, 16, v0
; GFX11-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX11-NEXT: image_sample_d_g16 v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: sample_d_1d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX12-NEXT: v_lshl_or_b32 v0, s0, 16, v0
; GFX12-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX12-NEXT: image_sample_d_g16 v[0:3], [v0, v1, v2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: ; return to shader part epilog
@ -29,25 +41,31 @@ main_body:
define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t) {
; GFX10-LABEL: sample_d_2d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
; GFX10-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
; GFX10-NEXT: image_sample_d_g16 v[0:3], [v0, v2, v4, v5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX10-NEXT: v_lshl_or_b32 v1, v3, 16, v2
; GFX10-NEXT: image_sample_d_g16 v[0:3], [v0, v1, v4, v5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: sample_d_2d:
; GFX11: ; %bb.0: ; %main_body
; GFX11-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
; GFX11-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
; GFX11-NEXT: image_sample_d_g16 v[0:3], [v0, v2, v4, v5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX11-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-NEXT: v_lshl_or_b32 v1, v3, 16, v2
; GFX11-NEXT: image_sample_d_g16 v[0:3], [v0, v1, v4, v5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: sample_d_2d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
; GFX12-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
; GFX12-NEXT: image_sample_d_g16 v[0:3], [v0, v2, v4, v5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX12-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX12-NEXT: v_lshl_or_b32 v1, v3, 16, v2
; GFX12-NEXT: image_sample_d_g16 v[0:3], [v0, v1, v4, v5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: ; return to shader part epilog
main_body:
@ -58,27 +76,43 @@ main_body:
define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dtdh, half %drdh, half %dsdv, half %dtdv, half %drdv, float %s, float %t, float %r) {
; GFX10-LABEL: sample_d_3d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: v_mov_b32_e32 v9, v3
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: v_perm_b32 v2, v1, v0, 0x5040100
; GFX10-NEXT: v_perm_b32 v4, v4, v9, 0x5040100
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX10-NEXT: v_and_b32_e32 v9, 0xffff, v2
; GFX10-NEXT: v_and_b32_e32 v10, 0xffff, v3
; GFX10-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX10-NEXT: v_lshl_or_b32 v2, v1, 16, v0
; GFX10-NEXT: v_lshl_or_b32 v3, s0, 16, v9
; GFX10-NEXT: v_lshl_or_b32 v4, v4, 16, v10
; GFX10-NEXT: v_lshl_or_b32 v5, s0, 16, v5
; GFX10-NEXT: image_sample_d_g16 v[0:3], v[2:8], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: sample_d_3d:
; GFX11: ; %bb.0: ; %main_body
; GFX11-NEXT: v_perm_b32 v3, v4, v3, 0x5040100
; GFX11-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
; GFX11-NEXT: image_sample_d_g16 v[0:3], [v0, v2, v3, v5, v[6:8]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D
; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX11-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX11-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-NEXT: v_lshl_or_b32 v1, s0, 16, v2
; GFX11-NEXT: v_lshl_or_b32 v2, v4, 16, v3
; GFX11-NEXT: v_lshl_or_b32 v3, s0, 16, v5
; GFX11-NEXT: image_sample_d_g16 v[0:3], [v0, v1, v2, v3, v[6:8]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: sample_d_3d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_perm_b32 v3, v4, v3, 0x5040100
; GFX12-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
; GFX12-NEXT: image_sample_d_g16 v[0:3], [v0, v2, v3, v[5:8]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D
; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX12-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX12-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX12-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX12-NEXT: v_lshl_or_b32 v1, s0, 16, v2
; GFX12-NEXT: v_lshl_or_b32 v2, v4, 16, v3
; GFX12-NEXT: v_lshl_or_b32 v5, s0, 16, v5
; GFX12-NEXT: image_sample_d_g16 v[0:3], [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: ; return to shader part epilog
main_body:
@ -89,18 +123,30 @@ main_body:
define amdgpu_ps <4 x float> @sample_c_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %dsdh, half %dsdv, float %s) {
; GFX10-LABEL: sample_c_d_1d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
; GFX10-NEXT: image_sample_c_d_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: sample_c_d_1d:
; GFX11: ; %bb.0: ; %main_body
; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX11-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX11-NEXT: v_lshl_or_b32 v2, s0, 16, v2
; GFX11-NEXT: image_sample_c_d_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: sample_c_d_1d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX12-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX12-NEXT: v_lshl_or_b32 v2, s0, 16, v2
; GFX12-NEXT: image_sample_c_d_g16 v[0:3], [v0, v1, v2, v3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: ; return to shader part epilog
@ -112,25 +158,31 @@ main_body:
define amdgpu_ps <4 x float> @sample_c_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t) {
; GFX10-LABEL: sample_c_d_2d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: v_perm_b32 v3, v4, v3, 0x5040100
; GFX10-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
; GFX10-NEXT: image_sample_c_d_g16 v[0:3], [v0, v1, v3, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX10-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX10-NEXT: v_lshl_or_b32 v2, v4, 16, v3
; GFX10-NEXT: image_sample_c_d_g16 v[0:3], [v0, v1, v2, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: sample_c_d_2d:
; GFX11: ; %bb.0: ; %main_body
; GFX11-NEXT: v_perm_b32 v3, v4, v3, 0x5040100
; GFX11-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
; GFX11-NEXT: image_sample_c_d_g16 v[0:3], [v0, v1, v3, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX11-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX11-NEXT: v_lshl_or_b32 v2, v4, 16, v3
; GFX11-NEXT: image_sample_c_d_g16 v[0:3], [v0, v1, v2, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: sample_c_d_2d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_perm_b32 v3, v4, v3, 0x5040100
; GFX12-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
; GFX12-NEXT: image_sample_c_d_g16 v[0:3], [v0, v1, v3, v[5:6]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX12-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX12-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX12-NEXT: v_lshl_or_b32 v2, v4, 16, v3
; GFX12-NEXT: image_sample_c_d_g16 v[0:3], [v0, v1, v2, v[5:6]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: ; return to shader part epilog
main_body:
@ -141,18 +193,30 @@ main_body:
define amdgpu_ps <4 x float> @sample_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dsdv, float %s, float %clamp) {
; GFX10-LABEL: sample_d_cl_1d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX10-NEXT: v_lshl_or_b32 v0, s0, 16, v0
; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX10-NEXT: image_sample_d_cl_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: sample_d_cl_1d:
; GFX11: ; %bb.0: ; %main_body
; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-NEXT: v_lshl_or_b32 v0, s0, 16, v0
; GFX11-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX11-NEXT: image_sample_d_cl_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: sample_d_cl_1d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX12-NEXT: v_lshl_or_b32 v0, s0, 16, v0
; GFX12-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX12-NEXT: image_sample_d_cl_g16 v[0:3], [v0, v1, v2, v3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: ; return to shader part epilog
@ -164,25 +228,31 @@ main_body:
define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t, float %clamp) {
; GFX10-LABEL: sample_d_cl_2d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
; GFX10-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
; GFX10-NEXT: image_sample_d_cl_g16 v[0:3], [v0, v2, v4, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX10-NEXT: v_lshl_or_b32 v1, v3, 16, v2
; GFX10-NEXT: image_sample_d_cl_g16 v[0:3], [v0, v1, v4, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: sample_d_cl_2d:
; GFX11: ; %bb.0: ; %main_body
; GFX11-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
; GFX11-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
; GFX11-NEXT: image_sample_d_cl_g16 v[0:3], [v0, v2, v4, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX11-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-NEXT: v_lshl_or_b32 v1, v3, 16, v2
; GFX11-NEXT: image_sample_d_cl_g16 v[0:3], [v0, v1, v4, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: sample_d_cl_2d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
; GFX12-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
; GFX12-NEXT: image_sample_d_cl_g16 v[0:3], [v0, v2, v4, v[5:6]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX12-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX12-NEXT: v_lshl_or_b32 v1, v3, 16, v2
; GFX12-NEXT: image_sample_d_cl_g16 v[0:3], [v0, v1, v4, v[5:6]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: ; return to shader part epilog
main_body:
@ -193,18 +263,30 @@ main_body:
define amdgpu_ps <4 x float> @sample_c_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %dsdh, half %dsdv, float %s, float %clamp) {
; GFX10-LABEL: sample_c_d_cl_1d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX10-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX10-NEXT: v_lshl_or_b32 v2, s0, 16, v2
; GFX10-NEXT: image_sample_c_d_cl_g16 v[0:3], v[0:4], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: sample_c_d_cl_1d:
; GFX11: ; %bb.0: ; %main_body
; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX11-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX11-NEXT: v_lshl_or_b32 v2, s0, 16, v2
; GFX11-NEXT: image_sample_c_d_cl_g16 v[0:3], v[0:4], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: sample_c_d_cl_1d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX12-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX12-NEXT: v_lshl_or_b32 v2, s0, 16, v2
; GFX12-NEXT: image_sample_c_d_cl_g16 v[0:3], [v0, v1, v2, v[3:4]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: ; return to shader part epilog
@ -218,25 +300,31 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: v_mov_b32_e32 v8, v2
; GFX10-NEXT: v_mov_b32_e32 v2, v0
; GFX10-NEXT: v_perm_b32 v4, v4, v3, 0x5040100
; GFX10-NEXT: v_perm_b32 v3, v8, v1, 0x5040100
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v1
; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v3
; GFX10-NEXT: v_lshl_or_b32 v3, v8, 16, v0
; GFX10-NEXT: v_lshl_or_b32 v4, v4, 16, v1
; GFX10-NEXT: image_sample_c_d_cl_g16 v[0:3], v[2:7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: sample_c_d_cl_2d:
; GFX11: ; %bb.0: ; %main_body
; GFX11-NEXT: v_perm_b32 v3, v4, v3, 0x5040100
; GFX11-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
; GFX11-NEXT: image_sample_c_d_cl_g16 v[0:3], [v0, v1, v3, v5, v[6:7]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX11-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX11-NEXT: v_lshl_or_b32 v2, v4, 16, v3
; GFX11-NEXT: image_sample_c_d_cl_g16 v[0:3], [v0, v1, v2, v5, v[6:7]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: sample_c_d_cl_2d:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_perm_b32 v3, v4, v3, 0x5040100
; GFX12-NEXT: v_perm_b32 v1, v2, v1, 0x5040100
; GFX12-NEXT: image_sample_c_d_cl_g16 v[0:3], [v0, v1, v3, v[5:7]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX12-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX12-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX12-NEXT: v_lshl_or_b32 v2, v4, 16, v3
; GFX12-NEXT: image_sample_c_d_cl_g16 v[0:3], [v0, v1, v2, v[5:7]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: ; return to shader part epilog
main_body:
@ -247,28 +335,34 @@ main_body:
define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %zcompare, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t, float %slice) {
; GFX10-LABEL: sample_c_d_o_2darray_V1:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: v_mov_b32_e32 v9, v3
; GFX10-NEXT: v_mov_b32_e32 v10, v2
; GFX10-NEXT: v_mov_b32_e32 v3, v1
; GFX10-NEXT: v_mov_b32_e32 v9, v2
; GFX10-NEXT: v_mov_b32_e32 v10, v3
; GFX10-NEXT: v_mov_b32_e32 v2, v0
; GFX10-NEXT: v_perm_b32 v5, v5, v4, 0x5040100
; GFX10-NEXT: v_perm_b32 v4, v9, v10, 0x5040100
; GFX10-NEXT: v_mov_b32_e32 v3, v1
; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v4
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v9
; GFX10-NEXT: v_lshl_or_b32 v5, v5, 16, v1
; GFX10-NEXT: v_lshl_or_b32 v4, v10, 16, v0
; GFX10-NEXT: image_sample_c_d_o_g16 v0, v[2:8], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: sample_c_d_o_2darray_V1:
; GFX11: ; %bb.0: ; %main_body
; GFX11-NEXT: v_perm_b32 v4, v5, v4, 0x5040100
; GFX11-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
; GFX11-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v4, v[6:8]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY
; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX11-NEXT: v_and_b32_e32 v4, 0xffff, v4
; GFX11-NEXT: v_lshl_or_b32 v2, v3, 16, v2
; GFX11-NEXT: v_lshl_or_b32 v3, v5, 16, v4
; GFX11-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v3, v[6:8]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: sample_c_d_o_2darray_V1:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_perm_b32 v5, v5, v4, 0x5040100
; GFX12-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX12-NEXT: v_and_b32_e32 v4, 0xffff, v4
; GFX12-NEXT: v_lshl_or_b32 v2, v3, 16, v2
; GFX12-NEXT: v_lshl_or_b32 v5, v5, 16, v4
; GFX12-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: ; return to shader part epilog
@ -280,28 +374,34 @@ main_body:
define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %zcompare, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t, float %slice) {
; GFX10-LABEL: sample_c_d_o_2darray_V2:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: v_mov_b32_e32 v9, v3
; GFX10-NEXT: v_mov_b32_e32 v10, v2
; GFX10-NEXT: v_mov_b32_e32 v3, v1
; GFX10-NEXT: v_mov_b32_e32 v9, v2
; GFX10-NEXT: v_mov_b32_e32 v10, v3
; GFX10-NEXT: v_mov_b32_e32 v2, v0
; GFX10-NEXT: v_perm_b32 v5, v5, v4, 0x5040100
; GFX10-NEXT: v_perm_b32 v4, v9, v10, 0x5040100
; GFX10-NEXT: v_mov_b32_e32 v3, v1
; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v4
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v9
; GFX10-NEXT: v_lshl_or_b32 v5, v5, 16, v1
; GFX10-NEXT: v_lshl_or_b32 v4, v10, 16, v0
; GFX10-NEXT: image_sample_c_d_o_g16 v[0:1], v[2:8], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: sample_c_d_o_2darray_V2:
; GFX11: ; %bb.0: ; %main_body
; GFX11-NEXT: v_perm_b32 v4, v5, v4, 0x5040100
; GFX11-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
; GFX11-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v4, v[6:8]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY
; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX11-NEXT: v_and_b32_e32 v4, 0xffff, v4
; GFX11-NEXT: v_lshl_or_b32 v2, v3, 16, v2
; GFX11-NEXT: v_lshl_or_b32 v3, v5, 16, v4
; GFX11-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v3, v[6:8]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: sample_c_d_o_2darray_V2:
; GFX12: ; %bb.0: ; %main_body
; GFX12-NEXT: v_perm_b32 v5, v5, v4, 0x5040100
; GFX12-NEXT: v_perm_b32 v2, v3, v2, 0x5040100
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX12-NEXT: v_and_b32_e32 v4, 0xffff, v4
; GFX12-NEXT: v_lshl_or_b32 v2, v3, 16, v2
; GFX12-NEXT: v_lshl_or_b32 v5, v5, 16, v4
; GFX12-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: ; return to shader part epilog

36
llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.store.2d.d16.ll
View file

@ -9,28 +9,28 @@
define amdgpu_ps void @image_store_f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t, half %data) {
; UNPACKED-LABEL: image_store_f16:
; UNPACKED: ; %bb.0:
; UNPACKED-NEXT: s_mov_b32 s11, s9
; UNPACKED-NEXT: s_mov_b32 s10, s8
; UNPACKED-NEXT: s_mov_b32 s9, s7
; UNPACKED-NEXT: s_mov_b32 s8, s6
; UNPACKED-NEXT: s_mov_b32 s7, s5
; UNPACKED-NEXT: s_mov_b32 s6, s4
; UNPACKED-NEXT: s_mov_b32 s5, s3
; UNPACKED-NEXT: s_mov_b32 s4, s2
; UNPACKED-NEXT: image_store v2, v[0:1], s[4:11] dmask:0x1 unorm d16
; UNPACKED-NEXT: s_mov_b32 s0, s2
; UNPACKED-NEXT: s_mov_b32 s1, s3
; UNPACKED-NEXT: s_mov_b32 s2, s4
; UNPACKED-NEXT: s_mov_b32 s3, s5
; UNPACKED-NEXT: s_mov_b32 s4, s6
; UNPACKED-NEXT: s_mov_b32 s5, s7
; UNPACKED-NEXT: s_mov_b32 s6, s8
; UNPACKED-NEXT: s_mov_b32 s7, s9
; UNPACKED-NEXT: image_store v2, v[0:1], s[0:7] dmask:0x1 unorm d16
; UNPACKED-NEXT: s_endpgm
;
; GFX81-LABEL: image_store_f16:
; GFX81: ; %bb.0:
; GFX81-NEXT: s_mov_b32 s11, s9
; GFX81-NEXT: s_mov_b32 s10, s8
; GFX81-NEXT: s_mov_b32 s9, s7
; GFX81-NEXT: s_mov_b32 s8, s6
; GFX81-NEXT: s_mov_b32 s7, s5
; GFX81-NEXT: s_mov_b32 s6, s4
; GFX81-NEXT: s_mov_b32 s5, s3
; GFX81-NEXT: s_mov_b32 s4, s2
; GFX81-NEXT: image_store v2, v[0:1], s[4:11] dmask:0x1 unorm d16
; GFX81-NEXT: s_mov_b32 s0, s2
; GFX81-NEXT: s_mov_b32 s1, s3
; GFX81-NEXT: s_mov_b32 s2, s4
; GFX81-NEXT: s_mov_b32 s3, s5
; GFX81-NEXT: s_mov_b32 s4, s6
; GFX81-NEXT: s_mov_b32 s5, s7
; GFX81-NEXT: s_mov_b32 s6, s8
; GFX81-NEXT: s_mov_b32 s7, s9
; GFX81-NEXT: image_store v2, v[0:1], s[0:7] dmask:0x1 unorm d16
; GFX81-NEXT: s_endpgm
call void @llvm.amdgcn.image.store.2d.f16.i32(half %data, i32 1, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
ret void

302
llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
View file

@ -1,9 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.30 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX1030 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.13 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX1013 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-TRUE16 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-FAKE16 %s
; RUN: not llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.12 < %s -filetype=null 2>&1 | FileCheck -check-prefix=ERR %s
; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-TRUE16 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-FAKE16 %s
; RUN: not llc -global-isel -mtriple=amdgpu10.12 < %s -filetype=null 2>&1 | FileCheck -check-prefix=ERR %s
; uint4 llvm.amdgcn.image.bvh.intersect.ray.i32.v4f32(uint node_ptr, float ray_extent, float3 ray_origin, float3 ray_dir, float3 ray_inv_dir, uint4 texture_descr)
; uint4 llvm.amdgcn.image.bvh.intersect.ray.i32.v4f16(uint node_ptr, float ray_extent, float3 ray_origin, half3 ray_dir, half3 ray_inv_dir, uint4 texture_descr)
@ -71,9 +71,12 @@ define amdgpu_ps <4 x float> @image_bvh_intersect_ray_a16(i32 %node_ptr, float %
;
; GFX11-FAKE16-LABEL: image_bvh_intersect_ray_a16:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: v_perm_b32 v9, v7, v5, 0x7060302
; GFX11-FAKE16-NEXT: v_perm_b32 v10, v8, v6, 0x5040100
; GFX11-FAKE16-NEXT: v_perm_b32 v8, v7, v5, 0x5040100
; GFX11-FAKE16-NEXT: v_and_b32_e32 v9, 0xffff, v7
; GFX11-FAKE16-NEXT: v_and_b32_e32 v10, 0xffff, v8
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v8, v5, 16, v9
; GFX11-FAKE16-NEXT: v_perm_b32 v9, v5, v7, 0x7060302
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v10, v6, 16, v10
; GFX11-FAKE16-NEXT: image_bvh_intersect_ray v[0:3], [v0, v1, v[2:4], v[8:10]], s[0:3] a16
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: ; return to shader part epilog
@ -137,9 +140,12 @@ define amdgpu_ps <4 x float> @image_bvh64_intersect_ray_a16(i64 %node_ptr, float
;
; GFX11-FAKE16-LABEL: image_bvh64_intersect_ray_a16:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: v_perm_b32 v10, v8, v6, 0x7060302
; GFX11-FAKE16-NEXT: v_perm_b32 v11, v9, v7, 0x5040100
; GFX11-FAKE16-NEXT: v_perm_b32 v9, v8, v6, 0x5040100
; GFX11-FAKE16-NEXT: v_and_b32_e32 v10, 0xffff, v8
; GFX11-FAKE16-NEXT: v_and_b32_e32 v11, 0xffff, v9
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v9, v6, 16, v10
; GFX11-FAKE16-NEXT: v_perm_b32 v10, v6, v8, 0x7060302
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v11, v7, 16, v11
; GFX11-FAKE16-NEXT: image_bvh64_intersect_ray v[0:3], [v[0:1], v2, v[3:5], v[9:11]], s[0:3] a16
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: ; return to shader part epilog
@ -361,33 +367,37 @@ define amdgpu_ps <4 x float> @image_bvh_intersect_ray_a16_vgpr_descr(i32 %node_p
;
; GFX11-FAKE16-LABEL: image_bvh_intersect_ray_a16_vgpr_descr:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v17, v4 :: v_dual_mov_b32 v16, v3
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v15, v2 :: v_dual_mov_b32 v14, v0
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v13, v1
; GFX11-FAKE16-NEXT: v_perm_b32 v19, v7, v5, 0x7060302
; GFX11-FAKE16-NEXT: v_perm_b32 v20, v8, v6, 0x5040100
; GFX11-FAKE16-NEXT: v_perm_b32 v18, v7, v5, 0x5040100
; GFX11-FAKE16-NEXT: s_mov_b32 s4, exec_lo
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_mov_b32 s5, s4
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v16, v0 :: v_dual_mov_b32 v17, v1
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v15, v4 :: v_dual_and_b32 v0, 0xffff, v7
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v8
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v13, v2 :: v_dual_mov_b32 v14, v3
; GFX11-FAKE16-NEXT: s_mov_b32 s1, exec_lo
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v4, v5, 16, v0
; GFX11-FAKE16-NEXT: v_perm_b32 v5, v5, v7, 0x7060302
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v6, v6, 16, v1
; GFX11-FAKE16-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v9
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v10
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v11
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v12
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v9
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v10
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v11
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v12
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[11:12]
; GFX11-FAKE16-NEXT: image_bvh_intersect_ray v[0:3], [v14, v13, v[15:17], v[18:20]], s[0:3] a16
; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr9_vgpr10_vgpr11_vgpr12
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr14
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr13
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr15_vgpr16_vgpr17
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr18_vgpr19_vgpr20
; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[11:12]
; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
; GFX11-FAKE16-NEXT: image_bvh_intersect_ray v[0:3], [v16, v17, v[13:15], v[4:6]], s[4:7] a16
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr9
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr16
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr17
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr13_vgpr14_vgpr15
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr11_vgpr12
; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB7_1
; GFX11-FAKE16-NEXT: ; %bb.2:
; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: ; return to shader part epilog
%v = call <4 x i32> @llvm.amdgcn.image.bvh.intersect.ray.i32.v4f16(i32 %node_ptr, float %ray_extent, <3 x float> %ray_origin, <3 x half> %ray_dir, <3 x half> %ray_inv_dir, <4 x i32> %tdescr)
@ -613,33 +623,38 @@ define amdgpu_ps <4 x float> @image_bvh64_intersect_ray_a16_vgpr_descr(i64 %node
;
; GFX11-FAKE16-LABEL: image_bvh64_intersect_ray_a16_vgpr_descr:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v19, v5 :: v_dual_mov_b32 v18, v4
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v17, v3 :: v_dual_mov_b32 v14, v2
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v16, v1 :: v_dual_mov_b32 v15, v0
; GFX11-FAKE16-NEXT: v_perm_b32 v21, v8, v6, 0x7060302
; GFX11-FAKE16-NEXT: v_perm_b32 v22, v9, v7, 0x5040100
; GFX11-FAKE16-NEXT: v_perm_b32 v20, v8, v6, 0x5040100
; GFX11-FAKE16-NEXT: s_mov_b32 s4, exec_lo
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_mov_b32 s5, s4
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v17, v0 :: v_dual_mov_b32 v18, v1
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v8
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v9
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v19, v2 :: v_dual_mov_b32 v14, v3
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v15, v4 :: v_dual_mov_b32 v16, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v4, v6, 16, v0
; GFX11-FAKE16-NEXT: v_perm_b32 v5, v6, v8, 0x7060302
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v6, v7, 16, v1
; GFX11-FAKE16-NEXT: s_mov_b32 s1, exec_lo
; GFX11-FAKE16-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v10
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v11
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v12
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v13
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v10
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v11
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v12
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v13
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[10:11]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[12:13]
; GFX11-FAKE16-NEXT: image_bvh64_intersect_ray v[0:3], [v[15:16], v14, v[17:19], v[20:22]], s[0:3] a16
; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr10_vgpr11_vgpr12_vgpr13
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr15_vgpr16
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr14
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr17_vgpr18_vgpr19
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr20_vgpr21_vgpr22
; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[10:11]
; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[12:13]
; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s0
; GFX11-FAKE16-NEXT: image_bvh64_intersect_ray v[0:3], [v[17:18], v19, v[14:16], v[4:6]], s[4:7] a16
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr10
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr17_vgpr18
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr19
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr14_vgpr15_vgpr16
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6
; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr12_vgpr13
; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB9_1
; GFX11-FAKE16-NEXT: ; %bb.2:
; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: ; return to shader part epilog
%v = call <4 x i32> @llvm.amdgcn.image.bvh.intersect.ray.i64.v4f16(i64 %node_ptr, float %ray_extent, <3 x float> %ray_origin, <3 x half> %ray_dir, <3 x half> %ray_inv_dir, <4 x i32> %tdescr)
@ -810,29 +825,71 @@ define amdgpu_kernel void @image_bvh_intersect_ray_a16_nsa_reassign(ptr %p_node_
; GFX1013-NEXT: flat_store_dwordx4 v[0:1], v[0:3]
; GFX1013-NEXT: s_endpgm
;
; GFX11-LABEL: image_bvh_intersect_ray_a16_nsa_reassign:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b256 s[0:7], s[4:5], 0x24
; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-NEXT: v_dual_mov_b32 v4, 1.0 :: v_dual_mov_b32 v5, 2.0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v0, s0, s0, v2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
; GFX11-NEXT: v_add_co_u32 v2, s0, s2, v2
; GFX11-NEXT: v_add_co_ci_u32_e64 v3, null, s3, 0, s0
; GFX11-NEXT: flat_load_b32 v6, v[0:1]
; GFX11-NEXT: flat_load_b32 v7, v[2:3]
; GFX11-NEXT: v_mov_b32_e32 v0, 0x46004200
; GFX11-NEXT: v_mov_b32_e32 v1, 0x47004400
; GFX11-NEXT: v_dual_mov_b32 v2, 0x48004500 :: v_dual_mov_b32 v3, 0
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: image_bvh_intersect_ray v[0:3], [v6, v7, v[3:5], v[0:2]], s[4:7] a16
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: flat_store_b128 v[0:1], v[0:3]
; GFX11-NEXT: s_endpgm
; GFX11-TRUE16-LABEL: image_bvh_intersect_ray_a16_nsa_reassign:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_load_b256 s[0:7], s[4:5], 0x24
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-TRUE16-NEXT: s_movk_i32 s8, 0x4600
; GFX11-TRUE16-NEXT: s_movk_i32 s9, 0x4700
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s8, s8, 0x4200
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s9, s9, 0x4400
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s1 :: v_dual_lshlrev_b32 v4, 2, v0
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s3 :: v_dual_mov_b32 v2, s2
; GFX11-TRUE16-NEXT: s_mov_b32 s2, 2.0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
; GFX11-TRUE16-NEXT: s_movk_i32 s3, 0x4800
; GFX11-TRUE16-NEXT: s_mov_b32 s1, 1.0
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s10, s3, 0x4500
; GFX11-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; GFX11-TRUE16-NEXT: flat_load_b32 v6, v[0:1]
; GFX11-TRUE16-NEXT: flat_load_b32 v7, v[2:3]
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v5, s10
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s1 :: v_dual_mov_b32 v2, s2
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s8 :: v_dual_mov_b32 v4, s9
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: image_bvh_intersect_ray v[0:3], [v6, v7, v[0:2], v[3:5]], s[4:7] a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: flat_store_b128 v[0:1], v[0:3]
; GFX11-TRUE16-NEXT: s_endpgm
;
; GFX11-FAKE16-LABEL: image_bvh_intersect_ray_a16_nsa_reassign:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_load_b256 s[0:7], s[4:5], 0x24
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s8, 0x42004600
; GFX11-FAKE16-NEXT: s_mov_b32 s9, 0x44004700
; GFX11-FAKE16-NEXT: s_mov_b32 s10, 0x45004800
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, s10
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s1 :: v_dual_lshlrev_b32 v4, 2, v0
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, s3 :: v_dual_mov_b32 v2, s2
; GFX11-FAKE16-NEXT: s_mov_b32 s2, 2.0
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s0
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 1.0
; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
; GFX11-FAKE16-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; GFX11-FAKE16-NEXT: flat_load_b32 v6, v[0:1]
; GFX11-FAKE16-NEXT: flat_load_b32 v7, v[2:3]
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s8
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s9
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: image_bvh_intersect_ray v[0:3], [v6, v7, v[0:2], v[3:5]], s[4:7] a16
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: flat_store_b128 v[0:1], v[0:3]
; GFX11-FAKE16-NEXT: s_endpgm
%lid = tail call i32 @llvm.amdgcn.workitem.id.x()
%gep_node_ptr = getelementptr inbounds i32, ptr %p_node_ptr, i32 %lid
%node_ptr = load i32, ptr %gep_node_ptr, align 4
@ -1012,31 +1069,70 @@ define amdgpu_kernel void @image_bvh64_intersect_ray_a16_nsa_reassign(ptr %p_ray
; GFX1013-NEXT: flat_store_dwordx4 v[0:1], v[0:3]
; GFX1013-NEXT: s_endpgm
;
; GFX11-LABEL: image_bvh64_intersect_ray_a16_nsa_reassign:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x24
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x34
; GFX11-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX11-NEXT: v_mov_b32_e32 v2, 0x48004500
; GFX11-NEXT: v_mov_b32_e32 v4, 1.0
; GFX11-NEXT: v_mov_b32_e32 v6, 0xb36211c6
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX11-NEXT: v_bfrev_b32_e32 v7, 4.0
; GFX11-NEXT: v_mov_b32_e32 v5, 2.0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v0, s4, s6, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, s7, 0, s4
; GFX11-NEXT: flat_load_b32 v8, v[0:1]
; GFX11-NEXT: v_mov_b32_e32 v0, 0x46004200
; GFX11-NEXT: v_mov_b32_e32 v1, 0x47004400
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: image_bvh64_intersect_ray v[0:3], [v[6:7], v8, v[3:5], v[0:2]], s[0:3] a16
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: flat_store_b128 v[0:1], v[0:3]
; GFX11-NEXT: s_endpgm
; GFX11-TRUE16-LABEL: image_bvh64_intersect_ray_a16_nsa_reassign:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_clause 0x1
; GFX11-TRUE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x24
; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x34
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-TRUE16-NEXT: s_movk_i32 s8, 0x4600
; GFX11-TRUE16-NEXT: s_movk_i32 s9, 0x4700
; GFX11-TRUE16-NEXT: s_mov_b32 s5, 1.0
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s8, s8, 0x4200
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s9, s9, 0x4400
; GFX11-TRUE16-NEXT: v_bfrev_b32_e32 v7, 4.0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s7 :: v_dual_lshlrev_b32 v2, 2, v0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s6
; GFX11-TRUE16-NEXT: s_movk_i32 s7, 0x4800
; GFX11-TRUE16-NEXT: s_mov_b32 s6, 2.0
; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s10, s7, 0x4500
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s8
; GFX11-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, 0xb36211c6 :: v_dual_mov_b32 v5, s10
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, s6
; GFX11-TRUE16-NEXT: flat_load_b32 v8, v[0:1]
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, s9
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: image_bvh64_intersect_ray v[0:3], [v[6:7], v8, v[0:2], v[3:5]], s[0:3] a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: flat_store_b128 v[0:1], v[0:3]
; GFX11-TRUE16-NEXT: s_endpgm
;
; GFX11-FAKE16-LABEL: image_bvh64_intersect_ray_a16_nsa_reassign:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_clause 0x1
; GFX11-FAKE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x24
; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x34
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s5, 1.0
; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s8, 0x42004600
; GFX11-FAKE16-NEXT: s_mov_b32 s9, 0x44004700
; GFX11-FAKE16-NEXT: s_mov_b32 s10, 0x45004800
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v6, 0xb36211c6 :: v_dual_mov_b32 v5, s10
; GFX11-FAKE16-NEXT: v_bfrev_b32_e32 v7, 4.0
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, s9 :: v_dual_mov_b32 v3, s8
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s7 :: v_dual_lshlrev_b32 v2, 2, v0
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s6
; GFX11-FAKE16-NEXT: s_mov_b32 s6, 2.0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s6
; GFX11-FAKE16-NEXT: flat_load_b32 v8, v[0:1]
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: image_bvh64_intersect_ray v[0:3], [v[6:7], v8, v[0:2], v[3:5]], s[0:3] a16
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: flat_store_b128 v[0:1], v[0:3]
; GFX11-FAKE16-NEXT: s_endpgm
%lid = tail call i32 @llvm.amdgcn.workitem.id.x()
%gep_ray = getelementptr inbounds float, ptr %p_ray, i32 %lid
%ray_extent = load float, ptr %gep_ray, align 4

241
llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.tfe.ll
View file

@ -1,12 +1,13 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu6.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX67,GFX6
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu7.01-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX67,GFX7
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-- -stop-after=instruction-select < %s | FileCheck %s -check-prefix=GFX8
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefix=GFX910
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-- -stop-after=instruction-select < %s | FileCheck %s -check-prefix=GFX910
; RUN: llc -global-isel -global-isel-abort=2 -mattr=-real-true16 -mtriple=amdgpu11.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX11
; RUN: llc -global-isel -global-isel-abort=2 -mattr=-real-true16 -mtriple=amdgpu12.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX1200
; RUN: llc -global-isel -global-isel-abort=2 -mattr=-real-true16 -mtriple=amdgpu12.50-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX1250
; UNSUPPORTED: true
; RUN: llc -global-isel -mtriple=amdgpu6.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX67,GFX6
; RUN: llc -global-isel -mtriple=amdgpu7.01-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX67,GFX7
; RUN: llc -global-isel -mtriple=amdgpu8.03-- -stop-after=instruction-select < %s | FileCheck %s -check-prefix=GFX8
; RUN: llc -global-isel -mtriple=amdgpu9.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefix=GFX910
; RUN: llc -global-isel -mtriple=amdgpu10.10-- -stop-after=instruction-select < %s | FileCheck %s -check-prefix=GFX910
; RUN: llc -global-isel -mattr=-real-true16 -mtriple=amdgpu11.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX11
; RUN: llc -global-isel -mattr=-real-true16 -mtriple=amdgpu12.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX1200
; RUN: llc -global-isel -mattr=-real-true16 -mtriple=amdgpu12.50-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX1250
define amdgpu_ps void @raw_buffer_load_i8_tfe(<4 x i32> inreg %rsrc, ptr addrspace(1) %data_addr, ptr addrspace(1) %tfe_addr) {
; GFX67-LABEL: name: raw_buffer_load_i8_tfe
@ -321,150 +322,146 @@ define amdgpu_ps void @raw_buffer_load_f16_tfe(<4 x i32> inreg %rsrc, ptr addrsp
; GFX67: bb.1 (%ir-block.0):
; GFX67-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GFX67-NEXT: {{ $}}
; GFX67-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr0
; GFX67-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr1
; GFX67-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX67-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr3
; GFX67-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX67-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX67-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; GFX67-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](i32), [[COPY5]](i32)
; GFX67-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
; GFX67-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr3
; GFX67-NEXT: [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY6]](i32), [[COPY7]](i32)
; GFX67-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; GFX67-NEXT: [[AMDGPU_BUFFER_LOAD_USHORT_TFE:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_BUFFER_LOAD_USHORT_TFE [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[C]], [[C]], 0, 0, 0 :: (dereferenceable load (f16), align 1, addrspace 8)
; GFX67-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_USHORT_TFE]](<2 x i32>)
; GFX67-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[UV]](i32)
; GFX67-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; GFX67-NEXT: [[ANYEXT:%[0-9]+]]:_(f32) = G_ANYEXT [[BITCAST]](f16)
; GFX67-NEXT: G_STORE [[ANYEXT]](f32), [[MV]](p1) :: (store (f16) into %ir.data_addr, addrspace 1)
; GFX67-NEXT: G_STORE [[UV1]](i32), [[MV1]](p1) :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX67-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
; GFX67-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
; GFX67-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
; GFX67-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
; GFX67-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
; GFX67-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX67-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX67-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
; GFX67-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; GFX67-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
; GFX67-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
; GFX67-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; GFX67-NEXT: [[BUFFER_LOAD_USHORT_TFE_OFFSET:%[0-9]+]]:vreg_64 = BUFFER_LOAD_USHORT_TFE_OFFSET [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
; GFX67-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_OFFSET]].sub0
; GFX67-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_OFFSET]].sub1
; GFX67-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; GFX67-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
; GFX67-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_1]], %subreg.sub0, [[S_MOV_B32_2]], %subreg.sub1
; GFX67-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
; GFX67-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE3]], %subreg.sub2_sub3
; GFX67-NEXT: BUFFER_STORE_SHORT_ADDR64 [[COPY8]], [[REG_SEQUENCE1]], [[REG_SEQUENCE4]], 0, 0, 0, 0, implicit $exec :: (store (f16) into %ir.data_addr, addrspace 1)
; GFX67-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; GFX67-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
; GFX67-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_3]], %subreg.sub0, [[S_MOV_B32_4]], %subreg.sub1
; GFX67-NEXT: [[S_MOV_B64_1:%[0-9]+]]:sreg_64 = S_MOV_B64 0
; GFX67-NEXT: [[REG_SEQUENCE6:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_1]], %subreg.sub0_sub1, [[REG_SEQUENCE5]], %subreg.sub2_sub3
; GFX67-NEXT: BUFFER_STORE_DWORD_ADDR64 [[COPY9]], [[REG_SEQUENCE2]], [[REG_SEQUENCE6]], 0, 0, 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX67-NEXT: S_ENDPGM 0
;
; GFX8-LABEL: name: raw_buffer_load_f16_tfe
; GFX8: bb.1 (%ir-block.0):
; GFX8-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr1
; GFX8-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX8-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr3
; GFX8-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX8-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX8-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; GFX8-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](i32), [[COPY5]](i32)
; GFX8-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
; GFX8-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr3
; GFX8-NEXT: [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY6]](i32), [[COPY7]](i32)
; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; GFX8-NEXT: [[AMDGPU_BUFFER_LOAD_USHORT_TFE:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_BUFFER_LOAD_USHORT_TFE [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[C]], [[C]], 0, 0, 0 :: (dereferenceable load (f16), align 1, addrspace 8)
; GFX8-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_USHORT_TFE]](<2 x i32>)
; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[UV]](i32)
; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; GFX8-NEXT: [[ANYEXT:%[0-9]+]]:_(f32) = G_ANYEXT [[BITCAST]](f16)
; GFX8-NEXT: G_STORE [[ANYEXT]](f32), [[MV]](p1) :: (store (f16) into %ir.data_addr, addrspace 1)
; GFX8-NEXT: G_STORE [[UV1]](i32), [[MV1]](p1) :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX8-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
; GFX8-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
; GFX8-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
; GFX8-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
; GFX8-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX8-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX8-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
; GFX8-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; GFX8-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
; GFX8-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
; GFX8-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; GFX8-NEXT: [[BUFFER_LOAD_USHORT_TFE_OFFSET:%[0-9]+]]:vreg_64 = BUFFER_LOAD_USHORT_TFE_OFFSET [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
; GFX8-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_OFFSET]].sub0
; GFX8-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_OFFSET]].sub1
; GFX8-NEXT: FLAT_STORE_SHORT [[REG_SEQUENCE1]], [[COPY8]], 0, 0, implicit $exec, implicit $flat_scr :: (store (f16) into %ir.data_addr, addrspace 1)
; GFX8-NEXT: FLAT_STORE_DWORD [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec, implicit $flat_scr :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX8-NEXT: S_ENDPGM 0
;
; GFX910-LABEL: name: raw_buffer_load_f16_tfe
; GFX910: bb.1 (%ir-block.0):
; GFX910-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GFX910-NEXT: {{ $}}
; GFX910-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr0
; GFX910-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr1
; GFX910-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX910-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr3
; GFX910-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX910-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX910-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; GFX910-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](i32), [[COPY5]](i32)
; GFX910-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
; GFX910-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr3
; GFX910-NEXT: [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY6]](i32), [[COPY7]](i32)
; GFX910-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; GFX910-NEXT: [[AMDGPU_BUFFER_LOAD_USHORT_TFE:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_BUFFER_LOAD_USHORT_TFE [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[C]], [[C]], 0, 0, 0 :: (dereferenceable load (f16), align 1, addrspace 8)
; GFX910-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_USHORT_TFE]](<2 x i32>)
; GFX910-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[UV]](i32)
; GFX910-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; GFX910-NEXT: [[ANYEXT:%[0-9]+]]:_(f32) = G_ANYEXT [[BITCAST]](f16)
; GFX910-NEXT: G_STORE [[ANYEXT]](f32), [[MV]](p1) :: (store (f16) into %ir.data_addr, addrspace 1)
; GFX910-NEXT: G_STORE [[UV1]](i32), [[MV1]](p1) :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX910-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
; GFX910-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
; GFX910-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
; GFX910-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
; GFX910-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
; GFX910-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX910-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX910-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
; GFX910-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; GFX910-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
; GFX910-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
; GFX910-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; GFX910-NEXT: [[BUFFER_LOAD_USHORT_TFE_OFFSET:%[0-9]+]]:vreg_64 = BUFFER_LOAD_USHORT_TFE_OFFSET [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
; GFX910-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_OFFSET]].sub0
; GFX910-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_OFFSET]].sub1
; GFX910-NEXT: GLOBAL_STORE_SHORT [[REG_SEQUENCE1]], [[COPY8]], 0, 0, implicit $exec :: (store (f16) into %ir.data_addr, addrspace 1)
; GFX910-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX910-NEXT: S_ENDPGM 0
;
; GFX11-LABEL: name: raw_buffer_load_f16_tfe
; GFX11: bb.1 (%ir-block.0):
; GFX11-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr0
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr1
; GFX11-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr3
; GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX11-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX11-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; GFX11-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](i32), [[COPY5]](i32)
; GFX11-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
; GFX11-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr3
; GFX11-NEXT: [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY6]](i32), [[COPY7]](i32)
; GFX11-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; GFX11-NEXT: [[AMDGPU_BUFFER_LOAD_USHORT_TFE:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_BUFFER_LOAD_USHORT_TFE [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[C]], [[C]], 0, 0, 0 :: (dereferenceable load (f16), align 1, addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_USHORT_TFE]](<2 x i32>)
; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[UV]](i32)
; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; GFX11-NEXT: [[ANYEXT:%[0-9]+]]:_(f32) = G_ANYEXT [[BITCAST]](f16)
; GFX11-NEXT: G_STORE [[ANYEXT]](f32), [[MV]](p1) :: (store (f16) into %ir.data_addr, addrspace 1)
; GFX11-NEXT: G_STORE [[UV1]](i32), [[MV1]](p1) :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX11-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
; GFX11-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
; GFX11-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
; GFX11-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
; GFX11-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX11-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX11-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
; GFX11-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; GFX11-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
; GFX11-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
; GFX11-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; GFX11-NEXT: [[BUFFER_LOAD_USHORT_TFE_OFFSET:%[0-9]+]]:vreg_64 = BUFFER_LOAD_USHORT_TFE_OFFSET [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
; GFX11-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_OFFSET]].sub0
; GFX11-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_OFFSET]].sub1
; GFX11-NEXT: GLOBAL_STORE_SHORT [[REG_SEQUENCE1]], [[COPY8]], 0, 0, implicit $exec :: (store (f16) into %ir.data_addr, addrspace 1)
; GFX11-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX11-NEXT: S_ENDPGM 0
;
; GFX1200-LABEL: name: raw_buffer_load_f16_tfe
; GFX1200: bb.1 (%ir-block.0):
; GFX1200-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GFX1200-NEXT: {{ $}}
; GFX1200-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr0
; GFX1200-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr1
; GFX1200-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX1200-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr3
; GFX1200-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX1200-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX1200-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; GFX1200-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](i32), [[COPY5]](i32)
; GFX1200-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
; GFX1200-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr3
; GFX1200-NEXT: [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY6]](i32), [[COPY7]](i32)
; GFX1200-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD_USHORT_TFE:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_BUFFER_LOAD_USHORT_TFE [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[C]], [[C]], 0, 0, 0 :: (dereferenceable load (f16), align 1, addrspace 8)
; GFX1200-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_USHORT_TFE]](<2 x i32>)
; GFX1200-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[UV]](i32)
; GFX1200-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; GFX1200-NEXT: [[ANYEXT:%[0-9]+]]:_(f32) = G_ANYEXT [[BITCAST]](f16)
; GFX1200-NEXT: G_STORE [[ANYEXT]](f32), [[MV]](p1) :: (store (f16) into %ir.data_addr, addrspace 1)
; GFX1200-NEXT: G_STORE [[UV1]](i32), [[MV1]](p1) :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX1200-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
; GFX1200-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
; GFX1200-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
; GFX1200-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
; GFX1200-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
; GFX1200-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX1200-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX1200-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
; GFX1200-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; GFX1200-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
; GFX1200-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
; GFX1200-NEXT: [[BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_64 = BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
; GFX1200-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET]].sub0
; GFX1200-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET]].sub1
; GFX1200-NEXT: GLOBAL_STORE_SHORT [[REG_SEQUENCE1]], [[COPY8]], 0, 0, implicit $exec :: (store (f16) into %ir.data_addr, addrspace 1)
; GFX1200-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX1200-NEXT: S_ENDPGM 0
;
; GFX1250-LABEL: name: raw_buffer_load_f16_tfe
; GFX1250: bb.1 (%ir-block.0):
; GFX1250-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr0
; GFX1250-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr1
; GFX1250-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX1250-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr3
; GFX1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX1250-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX1250-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; GFX1250-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](i32), [[COPY5]](i32)
; GFX1250-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
; GFX1250-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr3
; GFX1250-NEXT: [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY6]](i32), [[COPY7]](i32)
; GFX1250-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD_USHORT_TFE:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_BUFFER_LOAD_USHORT_TFE [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[C]], [[C]], 0, 0, 0 :: (dereferenceable load (f16), align 1, addrspace 8)
; GFX1250-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_USHORT_TFE]](<2 x i32>)
; GFX1250-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[UV]](i32)
; GFX1250-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; GFX1250-NEXT: [[ANYEXT:%[0-9]+]]:_(f32) = G_ANYEXT [[BITCAST]](f16)
; GFX1250-NEXT: G_STORE [[ANYEXT]](f32), [[MV]](p1) :: (store (f16) into %ir.data_addr, addrspace 1)
; GFX1250-NEXT: G_STORE [[UV1]](i32), [[MV1]](p1) :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
; GFX1250-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
; GFX1250-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
; GFX1250-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
; GFX1250-NEXT: [[BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_64_align2 = BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET]].sub0
; GFX1250-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET]].sub1
; GFX1250-NEXT: GLOBAL_STORE_SHORT [[REG_SEQUENCE1]], [[COPY8]], 0, 0, implicit $exec :: (store (f16) into %ir.data_addr, addrspace 1)
; GFX1250-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX1250-NEXT: S_ENDPGM 0
%res = call { half, i32 } @llvm.amdgcn.raw.buffer.load.sl_f16i32s(<4 x i32> %rsrc, i32 0, i32 0, i32 0)
%data = extractvalue { half, i32 } %res, 0

86
llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f16.ll
View file

@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.02-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=UNPACKED %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.10-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=PACKED %s
; RUN: llc -global-isel -mtriple=amdgpu8.10-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=PACKED %s
; RUN: llc -global-isel -mtriple=amdgpu12.00-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=GFX12 %s
define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f16(<4 x i32> inreg %rsrc, half %val, i32 %voffset, i32 inreg %soffset) {
@ -8,38 +8,30 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffse
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY4]](i32)
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; UNPACKED-NEXT: G_AMDGPU_BUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 0, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f16
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY4]](i32)
; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; PACKED-NEXT: G_AMDGPU_BUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 0, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
; PACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
; PACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; PACKED-NEXT: BUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; PACKED-NEXT: S_ENDPGM 0
;
; GFX12-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f16
@ -67,36 +59,28 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__voffset_409
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY4]](i32)
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr6
; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; UNPACKED-NEXT: G_AMDGPU_BUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[C]], [[COPY5]], 4095, 0, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_X_gfx80_OFFSET_exact [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__voffset_4095__sgpr_soffset_f16
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY4]](i32)
; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr6
; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; PACKED-NEXT: G_AMDGPU_BUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[C]], [[COPY5]], 4095, 0, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
; PACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
; PACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; PACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
; PACKED-NEXT: BUFFER_STORE_FORMAT_D16_X_OFFSET_exact [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; PACKED-NEXT: S_ENDPGM 0
;
; GFX12-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__voffset_4095__sgpr_soffset_f16

24
llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.ll
View file

@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=GFX8 %s
; RUN: llc -global-isel -mtriple=amdgpu8.03-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=GFX8 %s
; RUN: llc -global-isel -mtriple=amdgpu12.00-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefixes=GFX12,GFX1200 %s
; RUN: llc -global-isel -mtriple=amdgpu12.50-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefixes=GFX12,GFX1250 %s
; FIXME: Test with SI when argument lowering not broken for f16
@ -947,19 +947,15 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
; GFX8: bb.1 (%ir-block.0):
; GFX8-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
; GFX8-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
; GFX8-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
; GFX8-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX8-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY4]](i32)
; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; GFX8-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; GFX8-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
; GFX8-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; GFX8-NEXT: G_AMDGPU_BUFFER_STORE_SHORT [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 0, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX8-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
; GFX8-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
; GFX8-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
; GFX8-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
; GFX8-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
; GFX8-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX8-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX8-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; GFX8-NEXT: BUFFER_STORE_SHORT_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX8-NEXT: S_ENDPGM 0
;
; GFX12-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f16

4
llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.load.ll
View file

@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
; RUN: llc -global-isel -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck %s
; RUN: llc -global-isel -global-isel -global-isel-abort=2 -mtriple=amdgpu12.50-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefixes=GFX1250 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.50-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefixes=GFX1250 %s
; FIXME: Test with SI when argument lowering not broken for f16
; Natural mapping

92
llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.store.format.f16.ll
View file

@ -7,42 +7,30 @@ define amdgpu_ps void @raw_ptr_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_vo
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
; UNPACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY4]](i32)
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; UNPACKED-NEXT: G_AMDGPU_BUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 0, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_ptr_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f16
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
; PACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY4]](i32)
; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; PACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; PACKED-NEXT: G_AMDGPU_BUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 0, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; PACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
; PACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
; PACKED-NEXT: BUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; PACKED-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.ptr.buffer.store.format.f16(half %val, ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret void
@ -53,40 +41,28 @@ define amdgpu_ps void @raw_ptr_buffer_store_format__sgpr_rsrc__vgpr_val__voffset
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
; UNPACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY4]](i32)
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr6
; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; UNPACKED-NEXT: G_AMDGPU_BUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[C]], [[COPY5]], 4095, 0, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_X_gfx80_OFFSET_exact [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_ptr_buffer_store_format__sgpr_rsrc__vgpr_val__voffset_4095__sgpr_soffset_f16
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
; PACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY4]](i32)
; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr6
; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; PACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; PACKED-NEXT: G_AMDGPU_BUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[C]], [[COPY5]], 4095, 0, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; PACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
; PACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; PACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
; PACKED-NEXT: BUFFER_STORE_FORMAT_D16_X_OFFSET_exact [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; PACKED-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.ptr.buffer.store.format.f16(half %val, ptr addrspace(8) %rsrc, i32 4095, i32 %soffset, i32 0)
ret void

24
llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.store.ll
View file

@ -465,21 +465,15 @@ define amdgpu_ps void @raw_ptr_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__
; CHECK: bb.1 (%ir-block.0):
; CHECK-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
; CHECK-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY4]](i32)
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; CHECK-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; CHECK-NEXT: G_AMDGPU_BUFFER_STORE_SHORT [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 0, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
; CHECK-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
; CHECK-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
; CHECK-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; CHECK-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; CHECK-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
; CHECK-NEXT: BUFFER_STORE_SHORT_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; CHECK-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.ptr.buffer.store.f16(half %val, ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret void

656
llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.tbuffer.store.f16.ll
View file

@ -7,42 +7,30 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
; UNPACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; UNPACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 0, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
; PACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; PACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; PACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 0, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; PACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
; PACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; PACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; PACKED-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.ptr.tbuffer.store.f16(half %val, ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
ret void
@ -92,32 +80,22 @@ define amdgpu_ps void @raw_tbuffer_store_v3f16__sgpr_rsrc__vgpr_voffset__sgpr_so
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr0
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr1
; UNPACKED-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[COPY]](<2 x f16>), [[COPY1]](<2 x f16>)
; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(f16), [[UV1:%[0-9]+]]:_(f16), [[UV2:%[0-9]+]]:_(f16), [[UV3:%[0-9]+]]:_(f16) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<4 x f16>)
; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x f16>) = G_BUILD_VECTOR [[UV]](f16), [[UV1]](f16), [[UV2]](f16)
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr2
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr3
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr4
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr5
; UNPACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32)
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr6
; UNPACKED-NEXT: [[UV4:%[0-9]+]]:_(f16), [[UV5:%[0-9]+]]:_(f16), [[UV6:%[0-9]+]]:_(f16) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<3 x f16>)
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(i16) = G_BITCAST [[UV4]](f16)
; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(i16) = G_BITCAST [[UV5]](f16)
; UNPACKED-NEXT: [[BITCAST2:%[0-9]+]]:_(i16) = G_BITCAST [[UV6]](f16)
; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<3 x i16>) = G_BUILD_VECTOR [[BITCAST]](i16), [[BITCAST1]](i16), [[BITCAST2]](i16)
; UNPACKED-NEXT: [[UV7:%[0-9]+]]:_(i16), [[UV8:%[0-9]+]]:_(i16), [[UV9:%[0-9]+]]:_(i16) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<3 x i16>)
; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[UV7]](i16)
; UNPACKED-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[UV8]](i16)
; UNPACKED-NEXT: [[ANYEXT2:%[0-9]+]]:_(i32) = G_ANYEXT [[UV9]](i16)
; UNPACKED-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x i32>) = G_BUILD_VECTOR [[ANYEXT]](i32), [[ANYEXT1]](i32), [[ANYEXT2]](i32)
; UNPACKED-NEXT: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32), [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
; UNPACKED-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV10]](s32), [[UV11]](s32), [[UV12]](s32), [[UV13]](s32)
; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; UNPACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[BUILD_VECTOR2]](<3 x i32>), [[BUILD_VECTOR3]](<4 x s32>), [[C]](i32), [[COPY6]], [[COPY7]], 0, 78, 0, 0 :: (dereferenceable store (<3 x f16>) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x f16>) = COPY $vgpr0
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32(<2 x f16>) = COPY $vgpr1
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY]](<2 x f16>)
; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST]], [[COPY2]](s32)
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr_32(s32) = COPY [[COPY1]](<2 x f16>)
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32(i32) = COPY $sgpr2
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32(i32) = COPY $sgpr3
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32(i32) = COPY $sgpr4
; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:sreg_32(i32) = COPY $sgpr5
; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr2
; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:sreg_32(i32) = COPY $sgpr6
; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_96(<3 x i32>) = REG_SEQUENCE [[BITCAST]](s32), %subreg.sub0, [[LSHR]](s32), %subreg.sub1, [[COPY3]](s32), %subreg.sub2
; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[COPY4]](i32), %subreg.sub0, [[COPY5]](i32), %subreg.sub1, [[COPY6]](i32), %subreg.sub2, [[COPY7]](i32), %subreg.sub3
; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_XYZ_gfx80_OFFEN_exact [[REG_SEQUENCE]](<3 x i32>), [[COPY8]](i32), [[REG_SEQUENCE1]](<4 x s32>), [[COPY9]](i32), 0, 78, 0, 0, implicit $exec :: (dereferenceable store (<3 x f16>) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_v3f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
@ -189,44 +167,104 @@ define amdgpu_ps void @raw_tbuffer_store_v4f16__sgpr_rsrc__vgpr_voffset__sgpr_so
define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__sgpr_soffset(half %val, ptr addrspace(8) %rsrc, i32 %voffset, i32 inreg %soffset) {
; UNPACKED-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__sgpr_soffset
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: successors: %bb.2(0x80000000)
; UNPACKED-NEXT: liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $vgpr2
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $vgpr3
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr4
; UNPACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr5
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr2
; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; UNPACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 94, 0, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr0
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr4
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr5
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32(i32) = COPY $sgpr2
; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vreg_128(<4 x s32>) = G_BUILD_VECTOR [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32)
; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; UNPACKED-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
; UNPACKED-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.2:
; UNPACKED-NEXT: successors: %bb.3(0x80000000)
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY1]](s32)
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT1:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](i64), [[COPY8]](i64), implicit $exec
; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
; UNPACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_]](s1)
; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY12]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.3:
; UNPACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]](i32), [[COPY5]](i32), [[REG_SEQUENCE]](<4 x s32>), [[COPY6]](i32), 0, 94, 0, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
; UNPACKED-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.4:
; UNPACKED-NEXT: successors: %bb.5(0x80000000)
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: $exec = S_MOV_B64_term [[S_MOV_B64_]]
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.5:
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__sgpr_soffset
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: successors: %bb.2(0x80000000)
; PACKED-NEXT: liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $vgpr2
; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $vgpr3
; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr4
; PACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr5
; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr2
; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; PACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; PACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 94, 0, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; PACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr0
; PACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
; PACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
; PACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
; PACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr4
; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr5
; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32(i32) = COPY $sgpr2
; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vreg_128(<4 x s32>) = G_BUILD_VECTOR [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32)
; PACKED-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
; PACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; PACKED-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
; PACKED-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: bb.2:
; PACKED-NEXT: successors: %bb.3(0x80000000)
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY1]](s32)
; PACKED-NEXT: [[INTRINSIC_CONVERGENT1:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; PACKED-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; PACKED-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
; PACKED-NEXT: [[COPY8:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
; PACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
; PACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
; PACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
; PACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](i64), [[COPY8]](i64), implicit $exec
; PACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
; PACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; PACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_]](s1)
; PACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY12]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: bb.3:
; PACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]](i32), [[COPY5]](i32), [[REG_SEQUENCE]](<4 x s32>), [[COPY6]](i32), 0, 94, 0, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; PACKED-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
; PACKED-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: bb.4:
; PACKED-NEXT: successors: %bb.5(0x80000000)
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: $exec = S_MOV_B64_term [[S_MOV_B64_]]
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: bb.5:
; PACKED-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.ptr.tbuffer.store.f16(half %val, ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 94, i32 0)
ret void
@ -236,44 +274,110 @@ define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__sgpr_soff
define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__vgpr_soffset(half %val, ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset) {
; UNPACKED-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__vgpr_soffset
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: successors: %bb.2(0x80000000)
; UNPACKED-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $vgpr2
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $vgpr3
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr4
; UNPACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr5
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr6
; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; UNPACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 0, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr0
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr4
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr5
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr6
; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vreg_128(<4 x s32>) = G_BUILD_VECTOR [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32)
; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; UNPACKED-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
; UNPACKED-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.2:
; UNPACKED-NEXT: successors: %bb.3(0x80000000)
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY1]](s32)
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT1:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](i64), [[COPY8]](i64), implicit $exec
; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
; UNPACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY6]](i32), implicit $exec
; UNPACKED-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY6]](i32), implicit $exec
; UNPACKED-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY12]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.3:
; UNPACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]](i32), [[COPY5]](i32), [[REG_SEQUENCE]](<4 x s32>), [[V_READFIRSTLANE_B32_]](i32), 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
; UNPACKED-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.4:
; UNPACKED-NEXT: successors: %bb.5(0x80000000)
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: $exec = S_MOV_B64_term [[S_MOV_B64_]]
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.5:
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__vgpr_soffset
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: successors: %bb.2(0x80000000)
; PACKED-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $vgpr2
; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $vgpr3
; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr4
; PACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr5
; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr6
; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; PACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; PACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 0, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; PACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr0
; PACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
; PACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
; PACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
; PACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr4
; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr5
; PACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr6
; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vreg_128(<4 x s32>) = G_BUILD_VECTOR [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32)
; PACKED-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
; PACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; PACKED-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
; PACKED-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: bb.2:
; PACKED-NEXT: successors: %bb.3(0x80000000)
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY1]](s32)
; PACKED-NEXT: [[INTRINSIC_CONVERGENT1:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; PACKED-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; PACKED-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
; PACKED-NEXT: [[COPY8:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
; PACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
; PACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
; PACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
; PACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY10]](i64), [[COPY8]](i64), implicit $exec
; PACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
; PACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; PACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY6]](i32), implicit $exec
; PACKED-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY6]](i32), implicit $exec
; PACKED-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
; PACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
; PACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY12]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: bb.3:
; PACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]](i32), [[COPY5]](i32), [[REG_SEQUENCE]](<4 x s32>), [[V_READFIRSTLANE_B32_]](i32), 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; PACKED-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
; PACKED-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: bb.4:
; PACKED-NEXT: successors: %bb.5(0x80000000)
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: $exec = S_MOV_B64_term [[S_MOV_B64_]]
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: bb.5:
; PACKED-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.ptr.tbuffer.store.f16(half %val, ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
ret void
@ -283,44 +387,112 @@ define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__vgpr_soff
define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__sgpr_voffset__vgpr_soffset(half %val, ptr addrspace(8) %rsrc, i32 inreg %voffset, i32 %soffset) {
; UNPACKED-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__sgpr_voffset__vgpr_soffset
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: successors: %bb.2(0x80000000)
; UNPACKED-NEXT: liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $vgpr2
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $vgpr3
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr4
; UNPACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr2
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr5
; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; UNPACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 0, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr0
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr4
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr5
; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vreg_128(<4 x s32>) = G_BUILD_VECTOR [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32)
; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32(i32) = COPY [[COPY5]](i32)
; UNPACKED-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
; UNPACKED-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.2:
; UNPACKED-NEXT: successors: %bb.3(0x80000000)
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY1]](s32)
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT1:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](i64), [[COPY10]](i64), implicit $exec
; UNPACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY6]](i32), implicit $exec
; UNPACKED-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY6]](i32), implicit $exec
; UNPACKED-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY13]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.3:
; UNPACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]](i32), [[COPY8]](i32), [[REG_SEQUENCE]](<4 x s32>), [[V_READFIRSTLANE_B32_]](i32), 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
; UNPACKED-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.4:
; UNPACKED-NEXT: successors: %bb.5(0x80000000)
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: $exec = S_MOV_B64_term [[S_MOV_B64_]]
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.5:
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__sgpr_voffset__vgpr_soffset
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: successors: %bb.2(0x80000000)
; PACKED-NEXT: liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $vgpr2
; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $vgpr3
; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr4
; PACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr2
; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr5
; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; PACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; PACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 0, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; PACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr0
; PACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
; PACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
; PACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
; PACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr4
; PACKED-NEXT: [[COPY5:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
; PACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr5
; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vreg_128(<4 x s32>) = G_BUILD_VECTOR [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32)
; PACKED-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
; PACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
; PACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32(i32) = COPY [[COPY5]](i32)
; PACKED-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
; PACKED-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: bb.2:
; PACKED-NEXT: successors: %bb.3(0x80000000)
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY1]](s32)
; PACKED-NEXT: [[INTRINSIC_CONVERGENT1:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; PACKED-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; PACKED-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
; PACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
; PACKED-NEXT: [[COPY10:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
; PACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
; PACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
; PACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY11]](i64), [[COPY9]](i64), implicit $exec
; PACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY12]](i64), [[COPY10]](i64), implicit $exec
; PACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; PACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY6]](i32), implicit $exec
; PACKED-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY6]](i32), implicit $exec
; PACKED-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
; PACKED-NEXT: [[COPY13:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
; PACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY13]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: bb.3:
; PACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]](i32), [[COPY8]](i32), [[REG_SEQUENCE]](<4 x s32>), [[V_READFIRSTLANE_B32_]](i32), 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; PACKED-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
; PACKED-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: bb.4:
; PACKED-NEXT: successors: %bb.5(0x80000000)
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: $exec = S_MOV_B64_term [[S_MOV_B64_]]
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: bb.5:
; PACKED-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.ptr.tbuffer.store.f16(half %val, ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
ret void
@ -331,42 +503,30 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
; UNPACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; UNPACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 1, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 1, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
; PACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; PACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; PACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 1, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; PACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
; PACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; PACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 1, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; PACKED-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.ptr.tbuffer.store.f16(half %val, ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 1)
ret void
@ -377,42 +537,30 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
; UNPACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; UNPACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 2, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 2, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
; PACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; PACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; PACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 2, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; PACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
; PACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; PACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 2, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; PACKED-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.ptr.tbuffer.store.f16(half %val, ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 2)
ret void
@ -423,42 +571,30 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
; UNPACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; UNPACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 3, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 3, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc_glc
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
; PACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; PACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; PACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 3, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; PACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
; PACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; PACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 3, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; PACKED-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.ptr.tbuffer.store.f16(half %val, ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 3)
ret void
@ -469,42 +605,30 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
; UNPACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; UNPACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 4, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 4, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_dlc
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
; PACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; PACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; PACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 4, 0 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; PACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
; PACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; PACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 4, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; PACKED-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.ptr.tbuffer.store.f16(half %val, ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 4)
ret void

598
llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.f16.ll
View file

@ -8,38 +8,30 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; UNPACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 0, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; PACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 0, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
; PACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
; PACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; PACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; PACKED-NEXT: S_ENDPGM 0
;
; GFX12-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
@ -121,30 +113,22 @@ define amdgpu_ps void @raw_tbuffer_store_v3f16__sgpr_rsrc__vgpr_voffset__sgpr_so
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr0
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr1
; UNPACKED-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[COPY]](<2 x f16>), [[COPY1]](<2 x f16>)
; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(f16), [[UV1:%[0-9]+]]:_(f16), [[UV2:%[0-9]+]]:_(f16), [[UV3:%[0-9]+]]:_(f16) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<4 x f16>)
; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x f16>) = G_BUILD_VECTOR [[UV]](f16), [[UV1]](f16), [[UV2]](f16)
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr2
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr3
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr4
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr5
; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32)
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr6
; UNPACKED-NEXT: [[UV4:%[0-9]+]]:_(f16), [[UV5:%[0-9]+]]:_(f16), [[UV6:%[0-9]+]]:_(f16) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<3 x f16>)
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(i16) = G_BITCAST [[UV4]](f16)
; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(i16) = G_BITCAST [[UV5]](f16)
; UNPACKED-NEXT: [[BITCAST2:%[0-9]+]]:_(i16) = G_BITCAST [[UV6]](f16)
; UNPACKED-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x i16>) = G_BUILD_VECTOR [[BITCAST]](i16), [[BITCAST1]](i16), [[BITCAST2]](i16)
; UNPACKED-NEXT: [[UV7:%[0-9]+]]:_(i16), [[UV8:%[0-9]+]]:_(i16), [[UV9:%[0-9]+]]:_(i16) = G_UNMERGE_VALUES [[BUILD_VECTOR2]](<3 x i16>)
; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[UV7]](i16)
; UNPACKED-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[UV8]](i16)
; UNPACKED-NEXT: [[ANYEXT2:%[0-9]+]]:_(i32) = G_ANYEXT [[UV9]](i16)
; UNPACKED-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<3 x i32>) = G_BUILD_VECTOR [[ANYEXT]](i32), [[ANYEXT1]](i32), [[ANYEXT2]](i32)
; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; UNPACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[BUILD_VECTOR3]](<3 x i32>), [[BUILD_VECTOR1]](<4 x i32>), [[C]](i32), [[COPY6]], [[COPY7]], 0, 78, 0, 0 :: (dereferenceable store (<3 x f16>), align 1, addrspace 8)
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x f16>) = COPY $vgpr0
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32(<2 x f16>) = COPY $vgpr1
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY]](<2 x f16>)
; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST]], [[COPY2]](s32)
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr_32(s32) = COPY [[COPY1]](<2 x f16>)
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32(i32) = COPY $sgpr2
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32(i32) = COPY $sgpr3
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32(i32) = COPY $sgpr4
; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:sreg_32(i32) = COPY $sgpr5
; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x i32>) = REG_SEQUENCE [[COPY4]](i32), %subreg.sub0, [[COPY5]](i32), %subreg.sub1, [[COPY6]](i32), %subreg.sub2, [[COPY7]](i32), %subreg.sub3
; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr2
; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:sreg_32(i32) = COPY $sgpr6
; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_96(<3 x i32>) = REG_SEQUENCE [[BITCAST]](s32), %subreg.sub0, [[LSHR]](s32), %subreg.sub1, [[COPY3]](s32), %subreg.sub2
; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_XYZ_gfx80_OFFEN_exact [[REG_SEQUENCE1]](<3 x i32>), [[COPY8]](i32), [[REG_SEQUENCE]](<4 x i32>), [[COPY9]](i32), 0, 78, 0, 0, implicit $exec :: (dereferenceable store (<3 x f16>), align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_v3f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
@ -250,40 +234,96 @@ define amdgpu_ps void @raw_tbuffer_store_v4f16__sgpr_rsrc__vgpr_voffset__sgpr_so
define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__sgpr_soffset(half %val, <4 x i32> %rsrc, i32 %voffset, i32 inreg %soffset) {
; UNPACKED-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__sgpr_soffset
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: successors: %bb.2(0x80000000)
; UNPACKED-NEXT: liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $vgpr2
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $vgpr3
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr4
; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr5
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr2
; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; UNPACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 94, 0, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr2
; UNPACKED-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.2:
; UNPACKED-NEXT: successors: %bb.3(0x80000000)
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
; UNPACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.3:
; UNPACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE1]], [[COPY6]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; UNPACKED-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
; UNPACKED-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.4:
; UNPACKED-NEXT: successors: %bb.5(0x80000000)
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: $exec = S_MOV_B64_term [[S_MOV_B64_]]
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.5:
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__sgpr_soffset
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: successors: %bb.2(0x80000000)
; PACKED-NEXT: liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $vgpr2
; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $vgpr3
; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr4
; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr5
; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr2
; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; PACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 94, 0, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
; PACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; PACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; PACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; PACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
; PACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr2
; PACKED-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: bb.2:
; PACKED-NEXT: successors: %bb.3(0x80000000)
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
; PACKED-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
; PACKED-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
; PACKED-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
; PACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
; PACKED-NEXT: [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
; PACKED-NEXT: [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
; PACKED-NEXT: [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
; PACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
; PACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
; PACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
; PACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
; PACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: bb.3:
; PACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE1]], [[COPY6]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; PACKED-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
; PACKED-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: bb.4:
; PACKED-NEXT: successors: %bb.5(0x80000000)
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: $exec = S_MOV_B64_term [[S_MOV_B64_]]
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: bb.5:
; PACKED-NEXT: S_ENDPGM 0
;
; GFX12-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__sgpr_soffset
@ -342,40 +382,102 @@ define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__sgpr_soff
define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__vgpr_soffset(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
; UNPACKED-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__vgpr_soffset
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: successors: %bb.2(0x80000000)
; UNPACKED-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $vgpr2
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $vgpr3
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr4
; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr5
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr6
; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; UNPACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 0, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
; UNPACKED-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.2:
; UNPACKED-NEXT: successors: %bb.3(0x80000000)
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
; UNPACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
; UNPACKED-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
; UNPACKED-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.3:
; UNPACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; UNPACKED-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
; UNPACKED-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.4:
; UNPACKED-NEXT: successors: %bb.5(0x80000000)
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: $exec = S_MOV_B64_term [[S_MOV_B64_]]
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.5:
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__vgpr_soffset
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: successors: %bb.2(0x80000000)
; PACKED-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $vgpr2
; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $vgpr3
; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr4
; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr5
; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr6
; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; PACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 0, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
; PACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; PACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; PACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; PACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
; PACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
; PACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
; PACKED-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: bb.2:
; PACKED-NEXT: successors: %bb.3(0x80000000)
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
; PACKED-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
; PACKED-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
; PACKED-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
; PACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
; PACKED-NEXT: [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
; PACKED-NEXT: [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
; PACKED-NEXT: [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
; PACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
; PACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
; PACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
; PACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
; PACKED-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
; PACKED-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
; PACKED-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
; PACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: bb.3:
; PACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; PACKED-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
; PACKED-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: bb.4:
; PACKED-NEXT: successors: %bb.5(0x80000000)
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: $exec = S_MOV_B64_term [[S_MOV_B64_]]
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: bb.5:
; PACKED-NEXT: S_ENDPGM 0
;
; GFX12-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__vgpr_soffset
@ -437,40 +539,104 @@ define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__vgpr_soff
define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__sgpr_voffset__vgpr_soffset(half %val, <4 x i32> %rsrc, i32 inreg %voffset, i32 %soffset) {
; UNPACKED-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__sgpr_voffset__vgpr_soffset
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: successors: %bb.2(0x80000000)
; UNPACKED-NEXT: liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $vgpr2
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $vgpr3
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr4
; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr2
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr5
; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; UNPACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 0, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr2
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr5
; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
; UNPACKED-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.2:
; UNPACKED-NEXT: successors: %bb.3(0x80000000)
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY11]], [[COPY9]], implicit $exec
; UNPACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
; UNPACKED-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
; UNPACKED-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.3:
; UNPACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY7]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; UNPACKED-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
; UNPACKED-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.4:
; UNPACKED-NEXT: successors: %bb.5(0x80000000)
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: $exec = S_MOV_B64_term [[S_MOV_B64_]]
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.5:
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__sgpr_voffset__vgpr_soffset
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: successors: %bb.2(0x80000000)
; PACKED-NEXT: liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $vgpr2
; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $vgpr3
; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr4
; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr2
; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr5
; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; PACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 0, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
; PACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; PACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; PACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; PACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
; PACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; PACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr2
; PACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr5
; PACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
; PACKED-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: bb.2:
; PACKED-NEXT: successors: %bb.3(0x80000000)
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
; PACKED-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
; PACKED-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
; PACKED-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
; PACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
; PACKED-NEXT: [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
; PACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
; PACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
; PACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
; PACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
; PACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY11]], [[COPY9]], implicit $exec
; PACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
; PACKED-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
; PACKED-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
; PACKED-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
; PACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: bb.3:
; PACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY7]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; PACKED-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
; PACKED-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: bb.4:
; PACKED-NEXT: successors: %bb.5(0x80000000)
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: $exec = S_MOV_B64_term [[S_MOV_B64_]]
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: bb.5:
; PACKED-NEXT: S_ENDPGM 0
;
; GFX12-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__sgpr_voffset__vgpr_soffset
@ -534,38 +700,30 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; UNPACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 1, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 1, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; PACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 1, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
; PACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
; PACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; PACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 1, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; PACKED-NEXT: S_ENDPGM 0
;
; GFX12-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc
@ -593,38 +751,30 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; UNPACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 2, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 2, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; PACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 2, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
; PACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
; PACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; PACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 2, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; PACKED-NEXT: S_ENDPGM 0
;
; GFX12-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc
@ -652,38 +802,30 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; UNPACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 3, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 3, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc_glc
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; PACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 3, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
; PACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
; PACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; PACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 3, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; PACKED-NEXT: S_ENDPGM 0
;
; GFX12-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc_glc
@ -711,38 +853,30 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; UNPACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 4, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; UNPACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 4, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_dlc
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; PACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; PACKED-NEXT: G_AMDGPU_TBUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[COPY5]], [[COPY6]], 0, 78, 4, 0 :: (dereferenceable store (f16), align 1, addrspace 8)
; PACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
; PACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; PACKED-NEXT: TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 4, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; PACKED-NEXT: S_ENDPGM 0
;
; GFX12-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_dlc

48
llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.f16.ll
View file

@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=UNPACKED %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.10-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=PACKED %s
; RUN: llc -global-isel -mtriple=amdgpu8.10-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=PACKED %s
; RUN: llc -global-isel -mtriple=amdgpu12.00-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=GFX12 %s
define amdgpu_ps half @struct_buffer_load_format_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset(<4 x i32> inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) {
@ -462,36 +462,34 @@ define amdgpu_ps half @struct_buffer_load_format_i16__sgpr_rsrc__vgpr_vindex__vg
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
; UNPACKED-NEXT: [[INT:%[0-9]+]]:_(i16) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.struct.buffer.load.format), [[BUILD_VECTOR]](<4 x i32>), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), 0 :: (dereferenceable load (i16), align 1, addrspace 8)
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[INT]](i16)
; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; UNPACKED-NEXT: $vgpr0 = COPY [[ANYEXT]](i32)
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
; UNPACKED-NEXT: [[BUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (i16), align 1, addrspace 8)
; UNPACKED-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN]]
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
; PACKED-LABEL: name: struct_buffer_load_format_i16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
; PACKED-NEXT: [[INT:%[0-9]+]]:_(i16) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.struct.buffer.load.format), [[BUILD_VECTOR]](<4 x i32>), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), 0 :: (dereferenceable load (i16), align 1, addrspace 8)
; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[INT]](i16)
; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; PACKED-NEXT: $vgpr0 = COPY [[ANYEXT]](i32)
; PACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
; PACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; PACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
; PACKED-NEXT: [[BUFFER_LOAD_FORMAT_D16_X_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_X_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (i16), align 1, addrspace 8)
; PACKED-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_D16_X_BOTHEN]]
; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
; GFX12-LABEL: name: struct_buffer_load_format_i16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset

249
llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.tfe.ll
View file

@ -1,12 +1,13 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu6.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX67,GFX6
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu7.01-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX67,GFX7
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-- -stop-after=instruction-select < %s | FileCheck %s -check-prefix=GFX8
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefix=GFX910
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-- -stop-after=instruction-select < %s | FileCheck %s -check-prefix=GFX910
; RUN: llc -global-isel -global-isel-abort=2 -mattr=-real-true16 -mtriple=amdgpu11.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX11
; RUN: llc -global-isel -global-isel-abort=2 -mattr=-real-true16 -mtriple=amdgpu12.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX1200
; RUN: llc -global-isel -global-isel-abort=2 -mattr=-real-true16 -mtriple=amdgpu12.50-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX1250
; UNSUPPORTED: true
; RUN: llc -global-isel -mtriple=amdgpu6.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX67,GFX6
; RUN: llc -global-isel -mtriple=amdgpu7.01-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX67,GFX7
; RUN: llc -global-isel -mtriple=amdgpu8.03-- -stop-after=instruction-select < %s | FileCheck %s -check-prefix=GFX8
; RUN: llc -global-isel -mtriple=amdgpu9.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefix=GFX910
; RUN: llc -global-isel -mtriple=amdgpu10.10-- -stop-after=instruction-select < %s | FileCheck %s -check-prefix=GFX910
; RUN: llc -global-isel -mattr=-real-true16 -mtriple=amdgpu11.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX11
; RUN: llc -global-isel -mattr=-real-true16 -mtriple=amdgpu12.00-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX1200
; RUN: llc -global-isel -mattr=-real-true16 -mtriple=amdgpu12.50-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX1250
define amdgpu_ps void @raw_buffer_load_i8_tfe(<4 x i32> inreg %rsrc, ptr addrspace(1) %data_addr, ptr addrspace(1) %tfe_addr) {
; GFX67-LABEL: name: raw_buffer_load_i8_tfe
@ -337,150 +338,154 @@ define amdgpu_ps void @raw_buffer_load_f16_tfe(<4 x i32> inreg %rsrc, ptr addrsp
; GFX67: bb.1 (%ir-block.0):
; GFX67-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GFX67-NEXT: {{ $}}
; GFX67-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr0
; GFX67-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr1
; GFX67-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX67-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr3
; GFX67-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX67-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX67-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; GFX67-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](i32), [[COPY5]](i32)
; GFX67-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
; GFX67-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr3
; GFX67-NEXT: [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY6]](i32), [[COPY7]](i32)
; GFX67-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; GFX67-NEXT: [[AMDGPU_BUFFER_LOAD_USHORT_TFE:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_BUFFER_LOAD_USHORT_TFE [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[C]], [[C]], 0, 0, -1 :: (dereferenceable load (f16), align 1, addrspace 8)
; GFX67-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_USHORT_TFE]](<2 x i32>)
; GFX67-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[UV]](i32)
; GFX67-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; GFX67-NEXT: [[ANYEXT:%[0-9]+]]:_(f32) = G_ANYEXT [[BITCAST]](f16)
; GFX67-NEXT: G_STORE [[ANYEXT]](f32), [[MV]](p1) :: (store (f16) into %ir.data_addr, addrspace 1)
; GFX67-NEXT: G_STORE [[UV1]](i32), [[MV1]](p1) :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX67-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
; GFX67-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
; GFX67-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
; GFX67-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
; GFX67-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
; GFX67-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX67-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX67-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
; GFX67-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; GFX67-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
; GFX67-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
; GFX67-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; GFX67-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
; GFX67-NEXT: [[BUFFER_LOAD_USHORT_TFE_IDXEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_USHORT_TFE_IDXEN [[COPY8]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
; GFX67-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_IDXEN]].sub0
; GFX67-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_IDXEN]].sub1
; GFX67-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; GFX67-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
; GFX67-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_1]], %subreg.sub0, [[S_MOV_B32_2]], %subreg.sub1
; GFX67-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
; GFX67-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_]], %subreg.sub0_sub1, [[REG_SEQUENCE3]], %subreg.sub2_sub3
; GFX67-NEXT: BUFFER_STORE_SHORT_ADDR64 [[COPY9]], [[REG_SEQUENCE1]], [[REG_SEQUENCE4]], 0, 0, 0, 0, implicit $exec :: (store (f16) into %ir.data_addr, addrspace 1)
; GFX67-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; GFX67-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
; GFX67-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_3]], %subreg.sub0, [[S_MOV_B32_4]], %subreg.sub1
; GFX67-NEXT: [[S_MOV_B64_1:%[0-9]+]]:sreg_64 = S_MOV_B64 0
; GFX67-NEXT: [[REG_SEQUENCE6:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[S_MOV_B64_1]], %subreg.sub0_sub1, [[REG_SEQUENCE5]], %subreg.sub2_sub3
; GFX67-NEXT: BUFFER_STORE_DWORD_ADDR64 [[COPY10]], [[REG_SEQUENCE2]], [[REG_SEQUENCE6]], 0, 0, 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX67-NEXT: S_ENDPGM 0
;
; GFX8-LABEL: name: raw_buffer_load_f16_tfe
; GFX8: bb.1 (%ir-block.0):
; GFX8-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr1
; GFX8-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX8-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr3
; GFX8-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX8-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX8-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; GFX8-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](i32), [[COPY5]](i32)
; GFX8-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
; GFX8-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr3
; GFX8-NEXT: [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY6]](i32), [[COPY7]](i32)
; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; GFX8-NEXT: [[AMDGPU_BUFFER_LOAD_USHORT_TFE:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_BUFFER_LOAD_USHORT_TFE [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[C]], [[C]], 0, 0, -1 :: (dereferenceable load (f16), align 1, addrspace 8)
; GFX8-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_USHORT_TFE]](<2 x i32>)
; GFX8-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[UV]](i32)
; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; GFX8-NEXT: [[ANYEXT:%[0-9]+]]:_(f32) = G_ANYEXT [[BITCAST]](f16)
; GFX8-NEXT: G_STORE [[ANYEXT]](f32), [[MV]](p1) :: (store (f16) into %ir.data_addr, addrspace 1)
; GFX8-NEXT: G_STORE [[UV1]](i32), [[MV1]](p1) :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX8-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
; GFX8-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
; GFX8-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
; GFX8-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
; GFX8-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX8-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX8-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
; GFX8-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; GFX8-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
; GFX8-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
; GFX8-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; GFX8-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
; GFX8-NEXT: [[BUFFER_LOAD_USHORT_TFE_IDXEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_USHORT_TFE_IDXEN [[COPY8]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
; GFX8-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_IDXEN]].sub0
; GFX8-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_IDXEN]].sub1
; GFX8-NEXT: FLAT_STORE_SHORT [[REG_SEQUENCE1]], [[COPY9]], 0, 0, implicit $exec, implicit $flat_scr :: (store (f16) into %ir.data_addr, addrspace 1)
; GFX8-NEXT: FLAT_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec, implicit $flat_scr :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX8-NEXT: S_ENDPGM 0
;
; GFX910-LABEL: name: raw_buffer_load_f16_tfe
; GFX910: bb.1 (%ir-block.0):
; GFX910-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GFX910-NEXT: {{ $}}
; GFX910-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr0
; GFX910-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr1
; GFX910-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX910-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr3
; GFX910-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX910-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX910-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; GFX910-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](i32), [[COPY5]](i32)
; GFX910-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
; GFX910-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr3
; GFX910-NEXT: [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY6]](i32), [[COPY7]](i32)
; GFX910-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; GFX910-NEXT: [[AMDGPU_BUFFER_LOAD_USHORT_TFE:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_BUFFER_LOAD_USHORT_TFE [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[C]], [[C]], 0, 0, -1 :: (dereferenceable load (f16), align 1, addrspace 8)
; GFX910-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_USHORT_TFE]](<2 x i32>)
; GFX910-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[UV]](i32)
; GFX910-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; GFX910-NEXT: [[ANYEXT:%[0-9]+]]:_(f32) = G_ANYEXT [[BITCAST]](f16)
; GFX910-NEXT: G_STORE [[ANYEXT]](f32), [[MV]](p1) :: (store (f16) into %ir.data_addr, addrspace 1)
; GFX910-NEXT: G_STORE [[UV1]](i32), [[MV1]](p1) :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX910-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
; GFX910-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
; GFX910-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
; GFX910-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
; GFX910-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
; GFX910-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX910-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX910-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
; GFX910-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; GFX910-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
; GFX910-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
; GFX910-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; GFX910-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
; GFX910-NEXT: [[BUFFER_LOAD_USHORT_TFE_IDXEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_USHORT_TFE_IDXEN [[COPY8]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
; GFX910-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_IDXEN]].sub0
; GFX910-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_IDXEN]].sub1
; GFX910-NEXT: GLOBAL_STORE_SHORT [[REG_SEQUENCE1]], [[COPY9]], 0, 0, implicit $exec :: (store (f16) into %ir.data_addr, addrspace 1)
; GFX910-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX910-NEXT: S_ENDPGM 0
;
; GFX11-LABEL: name: raw_buffer_load_f16_tfe
; GFX11: bb.1 (%ir-block.0):
; GFX11-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr0
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr1
; GFX11-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr3
; GFX11-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX11-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX11-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; GFX11-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](i32), [[COPY5]](i32)
; GFX11-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
; GFX11-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr3
; GFX11-NEXT: [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY6]](i32), [[COPY7]](i32)
; GFX11-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; GFX11-NEXT: [[AMDGPU_BUFFER_LOAD_USHORT_TFE:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_BUFFER_LOAD_USHORT_TFE [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[C]], [[C]], 0, 0, -1 :: (dereferenceable load (f16), align 1, addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_USHORT_TFE]](<2 x i32>)
; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[UV]](i32)
; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; GFX11-NEXT: [[ANYEXT:%[0-9]+]]:_(f32) = G_ANYEXT [[BITCAST]](f16)
; GFX11-NEXT: G_STORE [[ANYEXT]](f32), [[MV]](p1) :: (store (f16) into %ir.data_addr, addrspace 1)
; GFX11-NEXT: G_STORE [[UV1]](i32), [[MV1]](p1) :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX11-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
; GFX11-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
; GFX11-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
; GFX11-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
; GFX11-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX11-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX11-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
; GFX11-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; GFX11-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
; GFX11-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
; GFX11-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; GFX11-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
; GFX11-NEXT: [[BUFFER_LOAD_USHORT_TFE_IDXEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_USHORT_TFE_IDXEN [[COPY8]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
; GFX11-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_IDXEN]].sub0
; GFX11-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_IDXEN]].sub1
; GFX11-NEXT: GLOBAL_STORE_SHORT [[REG_SEQUENCE1]], [[COPY9]], 0, 0, implicit $exec :: (store (f16) into %ir.data_addr, addrspace 1)
; GFX11-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX11-NEXT: S_ENDPGM 0
;
; GFX1200-LABEL: name: raw_buffer_load_f16_tfe
; GFX1200: bb.1 (%ir-block.0):
; GFX1200-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GFX1200-NEXT: {{ $}}
; GFX1200-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr0
; GFX1200-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr1
; GFX1200-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX1200-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr3
; GFX1200-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX1200-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX1200-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; GFX1200-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](i32), [[COPY5]](i32)
; GFX1200-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
; GFX1200-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr3
; GFX1200-NEXT: [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY6]](i32), [[COPY7]](i32)
; GFX1200-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; GFX1200-NEXT: [[AMDGPU_BUFFER_LOAD_USHORT_TFE:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_BUFFER_LOAD_USHORT_TFE [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[C]], [[C]], 0, 0, -1 :: (dereferenceable load (f16), align 1, addrspace 8)
; GFX1200-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_USHORT_TFE]](<2 x i32>)
; GFX1200-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[UV]](i32)
; GFX1200-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; GFX1200-NEXT: [[ANYEXT:%[0-9]+]]:_(f32) = G_ANYEXT [[BITCAST]](f16)
; GFX1200-NEXT: G_STORE [[ANYEXT]](f32), [[MV]](p1) :: (store (f16) into %ir.data_addr, addrspace 1)
; GFX1200-NEXT: G_STORE [[UV1]](i32), [[MV1]](p1) :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX1200-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
; GFX1200-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
; GFX1200-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
; GFX1200-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
; GFX1200-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
; GFX1200-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX1200-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX1200-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
; GFX1200-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; GFX1200-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
; GFX1200-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
; GFX1200-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; GFX1200-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
; GFX1200-NEXT: [[BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
; GFX1200-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN]].sub0
; GFX1200-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN]].sub1
; GFX1200-NEXT: GLOBAL_STORE_SHORT [[REG_SEQUENCE1]], [[COPY9]], 0, 0, implicit $exec :: (store (f16) into %ir.data_addr, addrspace 1)
; GFX1200-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX1200-NEXT: S_ENDPGM 0
;
; GFX1250-LABEL: name: raw_buffer_load_f16_tfe
; GFX1250: bb.1 (%ir-block.0):
; GFX1250-NEXT: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr0
; GFX1250-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr1
; GFX1250-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX1250-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr3
; GFX1250-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; GFX1250-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX1250-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; GFX1250-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](i32), [[COPY5]](i32)
; GFX1250-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
; GFX1250-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr3
; GFX1250-NEXT: [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY6]](i32), [[COPY7]](i32)
; GFX1250-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; GFX1250-NEXT: [[AMDGPU_BUFFER_LOAD_USHORT_TFE:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_BUFFER_LOAD_USHORT_TFE [[BUILD_VECTOR]](<4 x i32>), [[C]](i32), [[C]], [[C]], 0, 0, -1 :: (dereferenceable load (f16), align 1, addrspace 8)
; GFX1250-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_USHORT_TFE]](<2 x i32>)
; GFX1250-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[UV]](i32)
; GFX1250-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; GFX1250-NEXT: [[ANYEXT:%[0-9]+]]:_(f32) = G_ANYEXT [[BITCAST]](f16)
; GFX1250-NEXT: G_STORE [[ANYEXT]](f32), [[MV]](p1) :: (store (f16) into %ir.data_addr, addrspace 1)
; GFX1250-NEXT: G_STORE [[UV1]](i32), [[MV1]](p1) :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
; GFX1250-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
; GFX1250-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
; GFX1250-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
; GFX1250-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
; GFX1250-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; GFX1250-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
; GFX1250-NEXT: [[BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_64_align2 = BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
; GFX1250-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN]].sub0
; GFX1250-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN]].sub1
; GFX1250-NEXT: GLOBAL_STORE_SHORT [[REG_SEQUENCE1]], [[COPY9]], 0, 0, implicit $exec :: (store (f16) into %ir.data_addr, addrspace 1)
; GFX1250-NEXT: GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (i32) into %ir.tfe_addr, addrspace 1)
; GFX1250-NEXT: S_ENDPGM 0
%res = call { half, i32 } @llvm.amdgcn.struct.buffer.load.sl_f16i32s(<4 x i32> %rsrc, i32 0, i32 0, i32 0, i32 0)
%data = extractvalue { half, i32 } %res, 0

280
llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.store.format.f16.ll
View file

@ -1,45 +1,41 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=UNPACKED %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.10-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=PACKED %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=GFX12 %s
; RUN: llc -global-isel -mtriple=amdgpu8.10-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=PACKED %s
; RUN: llc -global-isel -mtriple=amdgpu12.00-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=GFX12 %s
define amdgpu_ps void @struct_buffer_store_format_f16__vgpr_val__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset(half %val, <4 x i32> inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) {
; UNPACKED-LABEL: name: struct_buffer_store_format_f16__vgpr_val__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr6
; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; UNPACKED-NEXT: G_AMDGPU_BUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[COPY5]](i32), [[COPY6]], [[COPY7]], 0, 0, -1 :: (dereferenceable store (f16), align 1, addrspace 8)
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_X_gfx80_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: struct_buffer_store_format_f16__vgpr_val__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
; PACKED-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr6
; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; PACKED-NEXT: G_AMDGPU_BUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[COPY5]](i32), [[COPY6]], [[COPY7]], 0, 0, -1 :: (dereferenceable store (f16), align 1, addrspace 8)
; PACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
; PACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; PACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; PACKED-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
; PACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
; PACKED-NEXT: BUFFER_STORE_FORMAT_D16_X_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; PACKED-NEXT: S_ENDPGM 0
;
; GFX12-LABEL: name: struct_buffer_store_format_f16__vgpr_val__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
@ -129,30 +125,24 @@ define amdgpu_ps void @struct_buffer_store_format_v3f16__vgpr_val__sgpr_rsrc__vg
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr0
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr1
; UNPACKED-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[COPY]](<2 x f16>), [[COPY1]](<2 x f16>)
; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(f16), [[UV1:%[0-9]+]]:_(f16), [[UV2:%[0-9]+]]:_(f16), [[UV3:%[0-9]+]]:_(f16) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<4 x f16>)
; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x f16>) = G_BUILD_VECTOR [[UV]](f16), [[UV1]](f16), [[UV2]](f16)
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr2
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr3
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr4
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr5
; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32)
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr3
; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $sgpr6
; UNPACKED-NEXT: [[UV4:%[0-9]+]]:_(f16), [[UV5:%[0-9]+]]:_(f16), [[UV6:%[0-9]+]]:_(f16) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<3 x f16>)
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(i16) = G_BITCAST [[UV4]](f16)
; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(i16) = G_BITCAST [[UV5]](f16)
; UNPACKED-NEXT: [[BITCAST2:%[0-9]+]]:_(i16) = G_BITCAST [[UV6]](f16)
; UNPACKED-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x i16>) = G_BUILD_VECTOR [[BITCAST]](i16), [[BITCAST1]](i16), [[BITCAST2]](i16)
; UNPACKED-NEXT: [[UV7:%[0-9]+]]:_(i16), [[UV8:%[0-9]+]]:_(i16), [[UV9:%[0-9]+]]:_(i16) = G_UNMERGE_VALUES [[BUILD_VECTOR2]](<3 x i16>)
; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[UV7]](i16)
; UNPACKED-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[UV8]](i16)
; UNPACKED-NEXT: [[ANYEXT2:%[0-9]+]]:_(i32) = G_ANYEXT [[UV9]](i16)
; UNPACKED-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<3 x i32>) = G_BUILD_VECTOR [[ANYEXT]](i32), [[ANYEXT1]](i32), [[ANYEXT2]](i32)
; UNPACKED-NEXT: G_AMDGPU_BUFFER_STORE_FORMAT_D16 [[BUILD_VECTOR3]](<3 x i32>), [[BUILD_VECTOR1]](<4 x i32>), [[COPY6]](i32), [[COPY7]], [[COPY8]], 0, 0, -1 :: (dereferenceable store (<3 x f16>), align 1, addrspace 8)
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x f16>) = COPY $vgpr0
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32(<2 x f16>) = COPY $vgpr1
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY]](<2 x f16>)
; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST]], [[COPY2]](s32)
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr_32(s32) = COPY [[COPY1]](<2 x f16>)
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32(i32) = COPY $sgpr2
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32(i32) = COPY $sgpr3
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32(i32) = COPY $sgpr4
; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:sreg_32(i32) = COPY $sgpr5
; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x i32>) = REG_SEQUENCE [[COPY4]](i32), %subreg.sub0, [[COPY5]](i32), %subreg.sub1, [[COPY6]](i32), %subreg.sub2, [[COPY7]](i32), %subreg.sub3
; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr2
; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr3
; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_32(i32) = COPY $sgpr6
; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_96(<3 x i32>) = REG_SEQUENCE [[BITCAST]](s32), %subreg.sub0, [[LSHR]](s32), %subreg.sub1, [[COPY3]](s32), %subreg.sub2
; UNPACKED-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64(i64) = REG_SEQUENCE [[COPY8]](i32), %subreg.sub0, [[COPY9]](i32), %subreg.sub1
; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XYZ_gfx80_BOTHEN_exact [[REG_SEQUENCE1]](<3 x i32>), [[REG_SEQUENCE2]](i64), [[REG_SEQUENCE]](<4 x i32>), [[COPY10]](i32), 0, 0, 0, implicit $exec :: (dereferenceable store (<3 x f16>), align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: struct_buffer_store_format_v3f16__vgpr_val__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
@ -267,59 +257,167 @@ define amdgpu_ps void @struct_buffer_store_format_v4f16__vgpr_val__sgpr_rsrc__vg
define amdgpu_ps void @struct_buffer_store_format_f16__sgpr_val__vgpr_rsrc__sgpr_vindex__sgpr_voffset__vgpr_soffset(half inreg %val, <4 x i32> %rsrc, i32 inreg %vindex, i32 inreg %voffset, i32 %soffset) {
; UNPACKED-LABEL: name: struct_buffer_store_format_f16__sgpr_val__vgpr_rsrc__sgpr_vindex__sgpr_voffset__vgpr_soffset
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: successors: %bb.2(0x80000000)
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr0
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $vgpr1
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $vgpr2
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr3
; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr3
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr4
; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr4
; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; UNPACKED-NEXT: G_AMDGPU_BUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[COPY5]](i32), [[COPY6]], [[COPY7]], 0, 0, -1 :: (dereferenceable store (f16), align 1, addrspace 8)
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr3
; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr4
; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
; UNPACKED-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.2:
; UNPACKED-NEXT: successors: %bb.3(0x80000000)
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
; UNPACKED-NEXT: [[COPY14:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY13]], [[COPY11]], implicit $exec
; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY14]], [[COPY12]], implicit $exec
; UNPACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
; UNPACKED-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY7]], implicit $exec
; UNPACKED-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.3:
; UNPACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1
; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_X_gfx80_BOTHEN_exact [[COPY8]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; UNPACKED-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
; UNPACKED-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.4:
; UNPACKED-NEXT: successors: %bb.5(0x80000000)
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: $exec = S_MOV_B64_term [[S_MOV_B64_]]
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.5:
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: struct_buffer_store_format_f16__sgpr_val__vgpr_rsrc__sgpr_vindex__sgpr_voffset__vgpr_soffset
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: successors: %bb.2(0x80000000)
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr0
; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $vgpr1
; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $vgpr2
; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr3
; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr3
; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr4
; PACKED-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr4
; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; PACKED-NEXT: G_AMDGPU_BUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[COPY5]](i32), [[COPY6]], [[COPY7]], 0, 0, -1 :: (dereferenceable store (f16), align 1, addrspace 8)
; PACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
; PACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; PACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; PACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; PACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr3
; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; PACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
; PACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr4
; PACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
; PACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
; PACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
; PACKED-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: bb.2:
; PACKED-NEXT: successors: %bb.3(0x80000000)
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
; PACKED-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
; PACKED-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
; PACKED-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
; PACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
; PACKED-NEXT: [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
; PACKED-NEXT: [[COPY12:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
; PACKED-NEXT: [[COPY13:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
; PACKED-NEXT: [[COPY14:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
; PACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY13]], [[COPY11]], implicit $exec
; PACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY14]], [[COPY12]], implicit $exec
; PACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
; PACKED-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
; PACKED-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY7]], implicit $exec
; PACKED-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
; PACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: bb.3:
; PACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1
; PACKED-NEXT: BUFFER_STORE_FORMAT_D16_X_BOTHEN_exact [[COPY8]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; PACKED-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
; PACKED-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: bb.4:
; PACKED-NEXT: successors: %bb.5(0x80000000)
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: $exec = S_MOV_B64_term [[S_MOV_B64_]]
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: bb.5:
; PACKED-NEXT: S_ENDPGM 0
;
; GFX12-LABEL: name: struct_buffer_store_format_f16__sgpr_val__vgpr_rsrc__sgpr_vindex__sgpr_voffset__vgpr_soffset
; GFX12: bb.1 (%ir-block.0):
; GFX12-NEXT: successors: %bb.2(0x80000000)
; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
; GFX12-NEXT: [[TRUNC:%[0-9]+]]:sgpr(i16) = G_TRUNC [[COPY]](i32)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:sgpr(f16) = G_BITCAST [[TRUNC]](i16)
; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr(i32) = COPY $vgpr1
; GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr(i32) = COPY $vgpr2
; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr3
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
; GFX12-NEXT: [[COPY5:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
; GFX12-NEXT: [[COPY6:%[0-9]+]]:sgpr(i32) = COPY $sgpr4
; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY $vgpr4
; GFX12-NEXT: [[ANYEXT:%[0-9]+]]:sgpr(i32) = G_ANYEXT [[BITCAST]](f16)
; GFX12-NEXT: G_AMDGPU_BUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x i32>), [[COPY5]](i32), [[COPY6]], [[COPY7]], 0, 0, -1 :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr3
; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr4
; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
; GFX12-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
; GFX12-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
; GFX12-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.2:
; GFX12-NEXT: successors: %bb.3(0x80000000)
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
; GFX12-NEXT: [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
; GFX12-NEXT: [[COPY12:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
; GFX12-NEXT: [[COPY13:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
; GFX12-NEXT: [[COPY14:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
; GFX12-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY13]], [[COPY11]], implicit $exec
; GFX12-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY14]], [[COPY12]], implicit $exec
; GFX12-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
; GFX12-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
; GFX12-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY7]], implicit $exec
; GFX12-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
; GFX12-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.3:
; GFX12-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1
; GFX12-NEXT: BUFFER_STORE_FORMAT_D16_X_VBUFFER_BOTHEN_exact [[COPY8]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
; GFX12-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.4:
; GFX12-NEXT: successors: %bb.5(0x80000000)
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.5:
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.struct.buffer.store.format.f16(half %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
ret void

2
llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.atomic.fadd-with-ret.ll
View file

@ -1,4 +1,4 @@
; RUN: llc -global-isel -mtriple=amdgpu9.0a-amd-amdhsa < %s | FileCheck -enable-var-scope -check-prefixes=GFX90A %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.0a-amd-amdhsa < %s | FileCheck -enable-var-scope -check-prefixes=GFX90A %s
; RUN: not llc -global-isel < %s -mtriple=amdgpu9.08 2>&1 | FileCheck %s -check-prefix=GFX908
; GFX908: LLVM ERROR: cannot select: %{{[0-9]+}}:vgpr_32(f32) = G_AMDGPU_BUFFER_ATOMIC_FADD %{{[0-9]+}}:vgpr, %{{[0-9]+}}:sgpr(<4 x s32>), %{{[0-9]+}}:vgpr(i32), %{{[0-9]+}}:vgpr, %{{[0-9]+}}:sgpr, 0, 0, -1 :: (volatile dereferenceable load store (f32) on %ir.rsrc.load, align 1, addrspace 8) (in function: buffer_atomic_add_f32_rtn)

46
llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.format.f16.ll
View file

@ -335,36 +335,34 @@ define amdgpu_ps half @struct_ptr_buffer_load_format_i16__sgpr_rsrc__vgpr_vindex
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
; UNPACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
; UNPACKED-NEXT: [[INT:%[0-9]+]]:_(i16) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.struct.ptr.buffer.load.format), [[MV]](p8), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), 0 :: (dereferenceable load (i16) from %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[INT]](i16)
; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; UNPACKED-NEXT: $vgpr0 = COPY [[ANYEXT]](i32)
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
; UNPACKED-NEXT: [[BUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (i16) from %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN]]
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
; PACKED-LABEL: name: struct_ptr_buffer_load_format_i16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
; PACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr0
; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr6
; PACKED-NEXT: [[INT:%[0-9]+]]:_(i16) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.struct.ptr.buffer.load.format), [[MV]](p8), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), 0 :: (dereferenceable load (i16) from %ir.rsrc, align 1, addrspace 8)
; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[INT]](i16)
; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; PACKED-NEXT: $vgpr0 = COPY [[ANYEXT]](i32)
; PACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
; PACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
; PACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
; PACKED-NEXT: [[BUFFER_LOAD_FORMAT_D16_X_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_X_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (i16) from %ir.rsrc, align 1, addrspace 8)
; PACKED-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_D16_X_BOTHEN]]
; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%val = call i16 @llvm.amdgcn.struct.ptr.buffer.load.format.i16(ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
%fval = bitcast i16 %val to half

228
llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.store.format.f16.ll
View file

@ -7,42 +7,34 @@ define amdgpu_ps void @struct_ptr_buffer_store_format_f16__vgpr_val__sgpr_rsrc__
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
; UNPACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr6
; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
; UNPACKED-NEXT: G_AMDGPU_BUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[COPY5]](i32), [[COPY6]], [[COPY7]], 0, 0, -1 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_X_gfx80_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: struct_ptr_buffer_store_format_f16__vgpr_val__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr2
; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr3
; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr4
; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr5
; PACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $vgpr1
; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
; PACKED-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr6
; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; PACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
; PACKED-NEXT: G_AMDGPU_BUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[COPY5]](i32), [[COPY6]], [[COPY7]], 0, 0, -1 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; PACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
; PACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; PACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; PACKED-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; PACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
; PACKED-NEXT: BUFFER_STORE_FORMAT_D16_X_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; PACKED-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.struct.ptr.buffer.store.format.f16(half %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
ret void
@ -96,32 +88,24 @@ define amdgpu_ps void @struct_ptr_buffer_store_format_v3f16__vgpr_val__sgpr_rsrc
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr0
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(<2 x f16>) = COPY $vgpr1
; UNPACKED-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[COPY]](<2 x f16>), [[COPY1]](<2 x f16>)
; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(f16), [[UV1:%[0-9]+]]:_(f16), [[UV2:%[0-9]+]]:_(f16), [[UV3:%[0-9]+]]:_(f16) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<4 x f16>)
; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x f16>) = G_BUILD_VECTOR [[UV]](f16), [[UV1]](f16), [[UV2]](f16)
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr2
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr3
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr4
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr5
; UNPACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32)
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $vgpr2
; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr3
; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $sgpr6
; UNPACKED-NEXT: [[UV4:%[0-9]+]]:_(f16), [[UV5:%[0-9]+]]:_(f16), [[UV6:%[0-9]+]]:_(f16) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<3 x f16>)
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(i16) = G_BITCAST [[UV4]](f16)
; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(i16) = G_BITCAST [[UV5]](f16)
; UNPACKED-NEXT: [[BITCAST2:%[0-9]+]]:_(i16) = G_BITCAST [[UV6]](f16)
; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<3 x i16>) = G_BUILD_VECTOR [[BITCAST]](i16), [[BITCAST1]](i16), [[BITCAST2]](i16)
; UNPACKED-NEXT: [[UV7:%[0-9]+]]:_(i16), [[UV8:%[0-9]+]]:_(i16), [[UV9:%[0-9]+]]:_(i16) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<3 x i16>)
; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[UV7]](i16)
; UNPACKED-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[UV8]](i16)
; UNPACKED-NEXT: [[ANYEXT2:%[0-9]+]]:_(i32) = G_ANYEXT [[UV9]](i16)
; UNPACKED-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x i32>) = G_BUILD_VECTOR [[ANYEXT]](i32), [[ANYEXT1]](i32), [[ANYEXT2]](i32)
; UNPACKED-NEXT: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32), [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
; UNPACKED-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV10]](s32), [[UV11]](s32), [[UV12]](s32), [[UV13]](s32)
; UNPACKED-NEXT: G_AMDGPU_BUFFER_STORE_FORMAT_D16 [[BUILD_VECTOR2]](<3 x i32>), [[BUILD_VECTOR3]](<4 x s32>), [[COPY6]](i32), [[COPY7]], [[COPY8]], 0, 0, -1 :: (dereferenceable store (<3 x f16>) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x f16>) = COPY $vgpr0
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32(<2 x f16>) = COPY $vgpr1
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(s32) = G_BITCAST [[COPY]](<2 x f16>)
; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
; UNPACKED-NEXT: [[LSHR:%[0-9]+]]:vgpr_32(s32) = G_LSHR [[BITCAST]], [[COPY2]](s32)
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr_32(s32) = COPY [[COPY1]](<2 x f16>)
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32(i32) = COPY $sgpr2
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32(i32) = COPY $sgpr3
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32(i32) = COPY $sgpr4
; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:sreg_32(i32) = COPY $sgpr5
; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr2
; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr3
; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:sreg_32(i32) = COPY $sgpr6
; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_96(<3 x i32>) = REG_SEQUENCE [[BITCAST]](s32), %subreg.sub0, [[LSHR]](s32), %subreg.sub1, [[COPY3]](s32), %subreg.sub2
; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[COPY4]](i32), %subreg.sub0, [[COPY5]](i32), %subreg.sub1, [[COPY6]](i32), %subreg.sub2, [[COPY7]](i32), %subreg.sub3
; UNPACKED-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64(i64) = REG_SEQUENCE [[COPY8]](i32), %subreg.sub0, [[COPY9]](i32), %subreg.sub1
; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_XYZ_gfx80_BOTHEN_exact [[REG_SEQUENCE]](<3 x i32>), [[REG_SEQUENCE2]](i64), [[REG_SEQUENCE1]](<4 x s32>), [[COPY10]](i32), 0, 0, 0, implicit $exec :: (dereferenceable store (<3 x f16>) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: struct_ptr_buffer_store_format_v3f16__vgpr_val__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
@ -198,44 +182,116 @@ define amdgpu_ps void @struct_ptr_buffer_store_format_v4f16__vgpr_val__sgpr_rsrc
define amdgpu_ps void @struct_ptr_buffer_store_format_f16__sgpr_val__vgpr_rsrc__sgpr_vindex__sgpr_voffset__vgpr_soffset(half inreg %val, ptr addrspace(8) %rsrc, i32 inreg %vindex, i32 inreg %voffset, i32 %soffset) {
; UNPACKED-LABEL: name: struct_ptr_buffer_store_format_f16__sgpr_val__vgpr_rsrc__sgpr_vindex__sgpr_voffset__vgpr_soffset
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: successors: %bb.2(0x80000000)
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; UNPACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr0
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $vgpr1
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $vgpr2
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr3
; UNPACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr3
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr4
; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr4
; UNPACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
; UNPACKED-NEXT: G_AMDGPU_BUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[COPY5]](i32), [[COPY6]], [[COPY7]], 0, 0, -1 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sgpr(i32) = COPY $sgpr4
; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr4
; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vreg_128(<4 x s32>) = G_BUILD_VECTOR [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32)
; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32(i32) = COPY [[COPY]](i32)
; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32(i32) = COPY [[COPY5]](i32)
; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr_32(i32) = COPY [[COPY6]](i32)
; UNPACKED-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
; UNPACKED-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.2:
; UNPACKED-NEXT: successors: %bb.3(0x80000000)
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY1]](s32)
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT1:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
; UNPACKED-NEXT: [[COPY14:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](i64), [[COPY11]](i64), implicit $exec
; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY14]](i64), [[COPY12]](i64), implicit $exec
; UNPACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY7]](i32), implicit $exec
; UNPACKED-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY7]](i32), implicit $exec
; UNPACKED-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
; UNPACKED-NEXT: [[COPY15:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY15]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.3:
; UNPACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64(i64) = REG_SEQUENCE [[COPY9]](i32), %subreg.sub0, [[COPY10]](i32), %subreg.sub1
; UNPACKED-NEXT: BUFFER_STORE_FORMAT_D16_X_gfx80_BOTHEN_exact [[COPY8]](i32), [[REG_SEQUENCE1]](i64), [[REG_SEQUENCE]](<4 x s32>), [[V_READFIRSTLANE_B32_]](i32), 0, 0, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
; UNPACKED-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.4:
; UNPACKED-NEXT: successors: %bb.5(0x80000000)
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: $exec = S_MOV_B64_term [[S_MOV_B64_]]
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.5:
; UNPACKED-NEXT: S_ENDPGM 0
;
; PACKED-LABEL: name: struct_ptr_buffer_store_format_f16__sgpr_val__vgpr_rsrc__sgpr_vindex__sgpr_voffset__vgpr_soffset
; PACKED: bb.1 (%ir-block.0):
; PACKED-NEXT: successors: %bb.2(0x80000000)
; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; PACKED-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
; PACKED-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; PACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr0
; PACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $vgpr1
; PACKED-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $vgpr2
; PACKED-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $vgpr3
; PACKED-NEXT: [[MV:%[0-9]+]]:_(p8) = G_MERGE_VALUES [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32)
; PACKED-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr3
; PACKED-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr4
; PACKED-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $vgpr4
; PACKED-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
; PACKED-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](p8)
; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[UV2]](s32), [[UV3]](s32)
; PACKED-NEXT: G_AMDGPU_BUFFER_STORE_FORMAT_D16 [[ANYEXT]](i32), [[BUILD_VECTOR]](<4 x s32>), [[COPY5]](i32), [[COPY6]], [[COPY7]], 0, 0, -1 :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; PACKED-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
; PACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
; PACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
; PACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
; PACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
; PACKED-NEXT: [[COPY5:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
; PACKED-NEXT: [[COPY6:%[0-9]+]]:sgpr(i32) = COPY $sgpr4
; PACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32(i32) = COPY $vgpr4
; PACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vreg_128(<4 x s32>) = G_BUILD_VECTOR [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32)
; PACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32(i32) = COPY [[COPY]](i32)
; PACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32(i32) = COPY [[COPY5]](i32)
; PACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr_32(i32) = COPY [[COPY6]](i32)
; PACKED-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
; PACKED-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: bb.2:
; PACKED-NEXT: successors: %bb.3(0x80000000)
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY1]](s32)
; PACKED-NEXT: [[INTRINSIC_CONVERGENT1:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](s32)
; PACKED-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](s32)
; PACKED-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sreg_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY4]](s32)
; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128(<4 x s32>) = REG_SEQUENCE [[INTRINSIC_CONVERGENT]](s32), %subreg.sub0, [[INTRINSIC_CONVERGENT1]](s32), %subreg.sub1, [[INTRINSIC_CONVERGENT2]](s32), %subreg.sub2, [[INTRINSIC_CONVERGENT3]](s32), %subreg.sub3
; PACKED-NEXT: [[COPY11:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub0_sub1(<4 x s32>)
; PACKED-NEXT: [[COPY12:%[0-9]+]]:vreg_64(i64) = COPY [[BUILD_VECTOR]].sub2_sub3(<4 x s32>)
; PACKED-NEXT: [[COPY13:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub0_sub1(<4 x s32>)
; PACKED-NEXT: [[COPY14:%[0-9]+]]:sreg_64(i64) = COPY [[REG_SEQUENCE]].sub2_sub3(<4 x s32>)
; PACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY13]](i64), [[COPY11]](i64), implicit $exec
; PACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U64_e64 [[COPY14]](i64), [[COPY12]](i64), implicit $exec
; PACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[V_CMP_EQ_U64_e64_]](s1), [[V_CMP_EQ_U64_e64_1]](s1), implicit-def dead $scc
; PACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(i32) = V_READFIRSTLANE_B32 [[COPY7]](i32), implicit $exec
; PACKED-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec(s1) = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](i32), [[COPY7]](i32), implicit $exec
; PACKED-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec(s1) = S_AND_B64 [[S_AND_B64_]](s1), [[V_CMP_EQ_U32_e64_]](s1), implicit-def dead $scc
; PACKED-NEXT: [[COPY15:%[0-9]+]]:sreg_64_xexec(i64) = COPY [[S_AND_B64_1]](s1)
; PACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[COPY15]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: bb.3:
; PACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64(i64) = REG_SEQUENCE [[COPY9]](i32), %subreg.sub0, [[COPY10]](i32), %subreg.sub1
; PACKED-NEXT: BUFFER_STORE_FORMAT_D16_X_BOTHEN_exact [[COPY8]](i32), [[REG_SEQUENCE1]](i64), [[REG_SEQUENCE]](<4 x s32>), [[V_READFIRSTLANE_B32_]](i32), 0, 0, 0, implicit $exec :: (dereferenceable store (f16) into %ir.rsrc, align 1, addrspace 8)
; PACKED-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
; PACKED-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: bb.4:
; PACKED-NEXT: successors: %bb.5(0x80000000)
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: $exec = S_MOV_B64_term [[S_MOV_B64_]]
; PACKED-NEXT: {{ $}}
; PACKED-NEXT: bb.5:
; PACKED-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.struct.ptr.buffer.store.format.f16(half %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
ret void

39
llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.powi.ll
View file

@ -5,16 +5,35 @@
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-mesa-mesa3d -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s
define i16 @v_powi_f16(i16 %l, i32 %r) {
; GFX78-LABEL: v_powi_f16:
; GFX78: ; %bb.0:
; GFX78-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX78-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX78-NEXT: v_cvt_f32_i32_e32 v1, v1
; GFX78-NEXT: v_log_f32_e32 v0, v0
; GFX78-NEXT: v_mul_legacy_f32_e32 v0, v1, v0
; GFX78-NEXT: v_exp_f32_e32 v0, v0
; GFX78-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX78-NEXT: s_setpc_b64 s[30:31]
; GFX7-LABEL: v_powi_f16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX7-NEXT: v_cvt_f32_i32_e32 v1, v1
; GFX7-NEXT: v_mov_b32_e32 v2, 0xc2fc0000
; GFX7-NEXT: v_mov_b32_e32 v3, 0x42800000
; GFX7-NEXT: v_log_f32_e32 v0, v0
; GFX7-NEXT: v_mul_legacy_f32_e32 v0, v0, v1
; GFX7-NEXT: v_cmp_lt_f32_e32 vcc, v0, v2
; GFX7-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
; GFX7-NEXT: v_add_f32_e32 v0, v0, v1
; GFX7-NEXT: v_exp_f32_e32 v0, v0
; GFX7-NEXT: v_not_b32_e32 v1, 63
; GFX7-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
; GFX7-NEXT: v_ldexp_f32_e32 v0, v0, v1
; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_powi_f16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX8-NEXT: v_cvt_f32_i32_e32 v1, v1
; GFX8-NEXT: v_log_f32_e32 v0, v0
; GFX8-NEXT: v_mul_legacy_f32_e32 v0, v1, v0
; GFX8-NEXT: v_exp_f32_e32 v0, v0
; GFX8-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: v_powi_f16:
; GFX11-TRUE16: ; %bb.0:

2
llvm/test/CodeGen/AMDGPU/GlobalISel/lshr.ll
View file

@ -710,6 +710,7 @@ define amdgpu_ps i16 @s_lshr_i16_15(i16 inreg %value) {
define amdgpu_ps half @lshr_i16_sv(i16 inreg %value, i16 %amount) {
; GFX6-LABEL: lshr_i16_sv:
; GFX6: ; %bb.0:
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
; GFX6-NEXT: v_lshr_b32_e32 v0, s0, v0
; GFX6-NEXT: ; return to shader part epilog
@ -736,6 +737,7 @@ define amdgpu_ps half @lshr_i16_sv(i16 inreg %value, i16 %amount) {
define amdgpu_ps half @lshr_i16_vs(i16 %value, i16 inreg %amount) {
; GFX6-LABEL: lshr_i16_vs:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX6-NEXT: v_lshrrev_b32_e32 v0, s0, v0
; GFX6-NEXT: ; return to shader part epilog

29
llvm/test/CodeGen/AMDGPU/GlobalISel/madmix-constant-bus-violation.ll
View file

@ -1,13 +1,16 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00 %s -o - | FileCheck %s
; RUN: llc -global-isel -mtriple=amdgpu9.00 %s -o - | FileCheck %s
define float @test_fmamix_constant_bus_violation_sss(i32 inreg %val.0, i32 inreg %val.1, i32 inreg %val.2) #0 {
; CHECK-LABEL: test_fmamix_constant_bus_violation_sss:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: v_mov_b32_e32 v0, s17
; CHECK-NEXT: v_mov_b32_e32 v1, s18
; CHECK-NEXT: v_mad_mix_f32 v0, s16, v0, v1 op_sel:[1,1,1] op_sel_hi:[1,1,1]
; CHECK-NEXT: s_lshr_b32 s5, s17, 16
; CHECK-NEXT: s_lshr_b32 s6, s18, 16
; CHECK-NEXT: s_lshr_b32 s4, s16, 16
; CHECK-NEXT: v_mov_b32_e32 v0, s5
; CHECK-NEXT: v_mov_b32_e32 v1, s6
; CHECK-NEXT: v_mad_mix_f32 v0, s4, v0, v1 op_sel_hi:[1,1,1]
; CHECK-NEXT: s_setpc_b64 s[30:31]
%lshr.0 = lshr i32 %val.0, 16
%lshr.1 = lshr i32 %val.1, 16
@ -29,8 +32,10 @@ define float @test_fmamix_constant_bus_violation_ssv(i32 inreg %val.0, i32 inreg
; CHECK-LABEL: test_fmamix_constant_bus_violation_ssv:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: v_mov_b32_e32 v1, s17
; CHECK-NEXT: v_mad_mix_f32 v0, s16, v1, v0 op_sel:[1,1,1] op_sel_hi:[1,1,1]
; CHECK-NEXT: s_lshr_b32 s5, s17, 16
; CHECK-NEXT: s_lshr_b32 s4, s16, 16
; CHECK-NEXT: v_mov_b32_e32 v1, s5
; CHECK-NEXT: v_mad_mix_f32 v0, s4, v1, v0 op_sel:[0,0,1] op_sel_hi:[1,1,1]
; CHECK-NEXT: s_setpc_b64 s[30:31]
%lshr.0 = lshr i32 %val.0, 16
%lshr.1 = lshr i32 %val.1, 16
@ -52,8 +57,10 @@ define float @test_fmamix_constant_bus_violation_svs(i32 inreg %val.0, i32 %val.
; CHECK-LABEL: test_fmamix_constant_bus_violation_svs:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: v_mov_b32_e32 v1, s17
; CHECK-NEXT: v_mad_mix_f32 v0, s16, v0, v1 op_sel:[1,1,1] op_sel_hi:[1,1,1]
; CHECK-NEXT: s_lshr_b32 s5, s17, 16
; CHECK-NEXT: s_lshr_b32 s4, s16, 16
; CHECK-NEXT: v_mov_b32_e32 v1, s5
; CHECK-NEXT: v_mad_mix_f32 v0, s4, v0, v1 op_sel:[0,1,0] op_sel_hi:[1,1,1]
; CHECK-NEXT: s_setpc_b64 s[30:31]
%lshr.0 = lshr i32 %val.0, 16
%lshr.1 = lshr i32 %val.1, 16
@ -75,8 +82,10 @@ define float @test_fmamix_constant_bus_violation_vss(i32 %val.0, i32 inreg %val.
; CHECK-LABEL: test_fmamix_constant_bus_violation_vss:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: v_mov_b32_e32 v1, s17
; CHECK-NEXT: v_mad_mix_f32 v0, v0, s16, v1 op_sel:[1,1,1] op_sel_hi:[1,1,1]
; CHECK-NEXT: s_lshr_b32 s5, s17, 16
; CHECK-NEXT: s_lshr_b32 s4, s16, 16
; CHECK-NEXT: v_mov_b32_e32 v1, s5
; CHECK-NEXT: v_mad_mix_f32 v0, v0, s4, v1 op_sel:[1,0,0] op_sel_hi:[1,1,1]
; CHECK-NEXT: s_setpc_b64 s[30:31]
%lshr.0 = lshr i32 %val.0, 16
%lshr.1 = lshr i32 %val.1, 16

59
llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
View file

@ -2597,23 +2597,28 @@ define amdgpu_ps i16 @s_saddsat_i16(i16 inreg %lhs, i16 inreg %rhs) {
define amdgpu_ps half @saddsat_i16_sv(i16 inreg %lhs, i16 %rhs) {
; GFX6-LABEL: saddsat_i16_sv:
; GFX6: ; %bb.0:
; GFX6-NEXT: v_bfe_i32 v0, v0, 0, 16
; GFX6-NEXT: s_sext_i32_i16 s0, s0
; GFX6-NEXT: s_lshl_b32 s0, s0, 16
; GFX6-NEXT: s_min_i32 s2, s0, 0
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: s_max_i32 s1, s0, 0
; GFX6-NEXT: s_sub_i32 s2, 0x80000000, s2
; GFX6-NEXT: s_sub_i32 s1, 0x7fffffff, s1
; GFX6-NEXT: v_max_i32_e32 v0, s2, v0
; GFX6-NEXT: v_min_i32_e32 v0, s1, v0
; GFX6-NEXT: v_add_i32_e32 v0, vcc, s0, v0
; GFX6-NEXT: s_movk_i32 s0, 0x8000
; GFX6-NEXT: v_mov_b32_e32 v1, 0x7fff
; GFX6-NEXT: v_med3_i32 v0, v0, s0, v1
; GFX6-NEXT: v_ashrrev_i32_e32 v0, 16, v0
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: saddsat_i16_sv:
; GFX8: ; %bb.0:
; GFX8-NEXT: v_cmp_gt_i16_e32 vcc, 0, v0
; GFX8-NEXT: s_sext_i32_i16 s1, s0
; GFX8-NEXT: s_max_i32 s2, s1, 0
; GFX8-NEXT: s_min_i32 s1, s1, 0
; GFX8-NEXT: s_sub_i32 s1, 0x8000, s1
; GFX8-NEXT: s_sub_i32 s2, 0x7fff, s2
; GFX8-NEXT: v_max_i16_e32 v0, s1, v0
; GFX8-NEXT: v_min_i16_e32 v0, s2, v0
; GFX8-NEXT: v_add_u16_e32 v0, s0, v0
; GFX8-NEXT: v_cmp_gt_i16_e64 s[0:1], s0, v0
; GFX8-NEXT: v_ashrrev_i16_e32 v1, 15, v0
; GFX8-NEXT: v_xor_b32_e32 v1, 0xffff8000, v1
; GFX8-NEXT: s_xor_b64 vcc, vcc, s[0:1]
; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: saddsat_i16_sv:
@ -2643,25 +2648,27 @@ define amdgpu_ps half @saddsat_i16_sv(i16 inreg %lhs, i16 %rhs) {
define amdgpu_ps half @saddsat_i16_vs(i16 %lhs, i16 inreg %rhs) {
; GFX6-LABEL: saddsat_i16_vs:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_sext_i32_i16 s0, s0
; GFX6-NEXT: v_bfe_i32 v0, v0, 0, 16
; GFX6-NEXT: v_add_i32_e32 v0, vcc, s0, v0
; GFX6-NEXT: s_movk_i32 s0, 0x8000
; GFX6-NEXT: v_mov_b32_e32 v1, 0x7fff
; GFX6-NEXT: v_med3_i32 v0, v0, s0, v1
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: v_min_i32_e32 v2, 0, v0
; GFX6-NEXT: s_lshl_b32 s0, s0, 16
; GFX6-NEXT: v_max_i32_e32 v1, 0, v0
; GFX6-NEXT: v_sub_i32_e32 v2, vcc, 0x80000000, v2
; GFX6-NEXT: v_sub_i32_e32 v1, vcc, 0x7fffffff, v1
; GFX6-NEXT: v_max_i32_e32 v2, s0, v2
; GFX6-NEXT: v_min_i32_e32 v1, v2, v1
; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1
; GFX6-NEXT: v_ashrrev_i32_e32 v0, 16, v0
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: saddsat_i16_vs:
; GFX8: ; %bb.0:
; GFX8-NEXT: v_add_u16_e32 v1, s0, v0
; GFX8-NEXT: s_sext_i32_i16 s0, s0
; GFX8-NEXT: s_cmp_lt_i32 s0, 0
; GFX8-NEXT: v_cmp_lt_i16_e32 vcc, v1, v0
; GFX8-NEXT: s_cselect_b64 s[0:1], -1, 0
; GFX8-NEXT: v_ashrrev_i16_e32 v0, 15, v1
; GFX8-NEXT: v_xor_b32_e32 v0, 0xffff8000, v0
; GFX8-NEXT: s_xor_b64 vcc, s[0:1], vcc
; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
; GFX8-NEXT: v_min_i16_e32 v2, 0, v0
; GFX8-NEXT: v_max_i16_e32 v1, 0, v0
; GFX8-NEXT: v_sub_u16_e32 v2, 0x8000, v2
; GFX8-NEXT: v_sub_u16_e32 v1, 0x7fff, v1
; GFX8-NEXT: v_max_i16_e32 v2, s0, v2
; GFX8-NEXT: v_min_i16_e32 v1, v2, v1
; GFX8-NEXT: v_add_u16_e32 v0, v0, v1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: saddsat_i16_vs:

4
llvm/test/CodeGen/AMDGPU/GlobalISel/select-to-fmin-fmax.ll
View file

@ -5,8 +5,8 @@ define half @test_s16(half %a) #0 {
; GCN-LABEL: test_s16:
; GCN: ; %bb.0: ; %entry
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_cmp_ngt_f16_e32 vcc, 0, v0
; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
; GCN-NEXT: v_cmp_gt_f16_e32 vcc, 0, v0
; GCN-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
; GCN-NEXT: s_setpc_b64 s[30:31]
entry:
%fcmp = fcmp olt half %a, 0.0

2
llvm/test/CodeGen/AMDGPU/GlobalISel/shl-ext-reduce.ll
View file

@ -3,7 +3,7 @@
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-amd-amdpal < %s | FileCheck -check-prefixes=GCN,GFX8 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00-amd-amdpal < %s | FileCheck -check-prefixes=GCN,GFX9 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-amd-amdpal < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-FAKE16 %s
; Test optimization to reduce shifts to narrower sizes.

2
llvm/test/CodeGen/AMDGPU/GlobalISel/shl.ll
View file

@ -650,6 +650,7 @@ define amdgpu_ps i16 @s_shl_i16_15(i16 inreg %value) {
define amdgpu_ps half @shl_i16_sv(i16 inreg %value, i16 %amount) {
; GFX6-LABEL: shl_i16_sv:
; GFX6: ; %bb.0:
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX6-NEXT: v_lshl_b32_e32 v0, s0, v0
; GFX6-NEXT: ; return to shader part epilog
;
@ -675,6 +676,7 @@ define amdgpu_ps half @shl_i16_sv(i16 inreg %value, i16 %amount) {
define amdgpu_ps half @shl_i16_vs(i16 %value, i16 inreg %amount) {
; GFX6-LABEL: shl_i16_vs:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
; GFX6-NEXT: v_lshlrev_b32_e32 v0, s0, v0
; GFX6-NEXT: ; return to shader part epilog
;

4
llvm/test/CodeGen/AMDGPU/GlobalISel/smed3.ll
View file

@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.02-amd-mesa3d < %s | FileCheck -check-prefixes=GFX89,GFX8 %s
; RUN: llc -global-isel -mtriple=amdgpu9.00-amd-mesa3d < %s | FileCheck -check-prefixes=GFX89,GFX9 %s
; RUN: llc -global-isel -mtriple=amdgpu10.10-amd-mesa3d < %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00-amd-mesa3d < %s | FileCheck -check-prefixes=GFX89,GFX9 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-amd-mesa3d < %s | FileCheck -check-prefix=GFX10 %s
define i32 @test_min_max_ValK0_K1_i32(i32 %a) {
; GFX89-LABEL: test_min_max_ValK0_K1_i32:

57
llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
View file

@ -2601,23 +2601,28 @@ define amdgpu_ps i16 @s_ssubsat_i16(i16 inreg %lhs, i16 inreg %rhs) {
define amdgpu_ps half @ssubsat_i16_sv(i16 inreg %lhs, i16 %rhs) {
; GFX6-LABEL: ssubsat_i16_sv:
; GFX6: ; %bb.0:
; GFX6-NEXT: v_bfe_i32 v0, v0, 0, 16
; GFX6-NEXT: s_sext_i32_i16 s0, s0
; GFX6-NEXT: s_lshl_b32 s0, s0, 16
; GFX6-NEXT: s_max_i32 s1, s0, -1
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: s_add_i32 s1, s1, 0x80000001
; GFX6-NEXT: s_min_i32 s2, s0, -1
; GFX6-NEXT: s_add_i32 s2, s2, 0x80000000
; GFX6-NEXT: v_max_i32_e32 v0, s1, v0
; GFX6-NEXT: v_min_i32_e32 v0, s2, v0
; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s0, v0
; GFX6-NEXT: s_movk_i32 s0, 0x8000
; GFX6-NEXT: v_mov_b32_e32 v1, 0x7fff
; GFX6-NEXT: v_med3_i32 v0, v0, s0, v1
; GFX6-NEXT: v_ashrrev_i32_e32 v0, 16, v0
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: ssubsat_i16_sv:
; GFX8: ; %bb.0:
; GFX8-NEXT: v_cmp_lt_i16_e32 vcc, s0, v0
; GFX8-NEXT: s_sext_i32_i16 s1, s0
; GFX8-NEXT: s_max_i32 s2, s1, -1
; GFX8-NEXT: s_add_i32 s2, s2, 0x8001
; GFX8-NEXT: s_min_i32 s1, s1, -1
; GFX8-NEXT: s_add_i32 s1, s1, 0x8000
; GFX8-NEXT: v_max_i16_e32 v0, s2, v0
; GFX8-NEXT: v_min_i16_e32 v0, s1, v0
; GFX8-NEXT: v_sub_u16_e32 v0, s0, v0
; GFX8-NEXT: v_cmp_gt_i16_e64 s[0:1], 0, v0
; GFX8-NEXT: v_ashrrev_i16_e32 v1, 15, v0
; GFX8-NEXT: v_xor_b32_e32 v1, 0xffff8000, v1
; GFX8-NEXT: s_xor_b64 vcc, vcc, s[0:1]
; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: ssubsat_i16_sv:
@ -2647,23 +2652,27 @@ define amdgpu_ps half @ssubsat_i16_sv(i16 inreg %lhs, i16 %rhs) {
define amdgpu_ps half @ssubsat_i16_vs(i16 %lhs, i16 inreg %rhs) {
; GFX6-LABEL: ssubsat_i16_vs:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_sext_i32_i16 s0, s0
; GFX6-NEXT: v_bfe_i32 v0, v0, 0, 16
; GFX6-NEXT: v_subrev_i32_e32 v0, vcc, s0, v0
; GFX6-NEXT: s_movk_i32 s0, 0x8000
; GFX6-NEXT: v_mov_b32_e32 v1, 0x7fff
; GFX6-NEXT: v_med3_i32 v0, v0, s0, v1
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: v_max_i32_e32 v1, -1, v0
; GFX6-NEXT: s_lshl_b32 s0, s0, 16
; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0x80000001, v1
; GFX6-NEXT: v_min_i32_e32 v2, -1, v0
; GFX6-NEXT: v_add_i32_e32 v2, vcc, 0x80000000, v2
; GFX6-NEXT: v_max_i32_e32 v1, s0, v1
; GFX6-NEXT: v_min_i32_e32 v1, v1, v2
; GFX6-NEXT: v_sub_i32_e32 v0, vcc, v0, v1
; GFX6-NEXT: v_ashrrev_i32_e32 v0, 16, v0
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: ssubsat_i16_vs:
; GFX8: ; %bb.0:
; GFX8-NEXT: v_cmp_gt_i16_e32 vcc, s0, v0
; GFX8-NEXT: v_subrev_u16_e32 v0, s0, v0
; GFX8-NEXT: v_cmp_gt_i16_e64 s[0:1], 0, v0
; GFX8-NEXT: v_ashrrev_i16_e32 v1, 15, v0
; GFX8-NEXT: v_xor_b32_e32 v1, 0xffff8000, v1
; GFX8-NEXT: s_xor_b64 vcc, vcc, s[0:1]
; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
; GFX8-NEXT: v_max_i16_e32 v1, -1, v0
; GFX8-NEXT: v_add_u16_e32 v1, 0x8001, v1
; GFX8-NEXT: v_min_i16_e32 v2, -1, v0
; GFX8-NEXT: v_add_u16_e32 v2, 0x8000, v2
; GFX8-NEXT: v_max_i16_e32 v1, s0, v1
; GFX8-NEXT: v_min_i16_e32 v1, v1, v2
; GFX8-NEXT: v_sub_u16_e32 v0, v0, v1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: ssubsat_i16_vs:

2
llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.96.ll
View file

@ -5,7 +5,7 @@
; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal < %s | FileCheck --check-prefix=GFX11 %s
; FIXME:
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu6.00-amd-amdpal < %s | FileCheck --check-prefix=GFX6 %s
; XUN: llc -global-isel -mtriple=amdgpu6.00-amd-amdpal < %s | FileCheck --check-prefix=GFX6 %s
define amdgpu_kernel void @store_lds_v3i32(ptr addrspace(3) %out, <3 x i32> %x) {
; GFX9-LABEL: store_lds_v3i32:

48
llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.f16.ll
View file

@ -41,7 +41,7 @@ define void @v_constained_fma_f16_fpexcept_strict_uni(half inreg %x, half inreg
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b16_e32 v2.l, s2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_fmac_f16_e64 v2.l, s0, s1
; GFX11-NEXT: v_fma_f16 v2.l, s0, s1, v2.l
; GFX11-NEXT: global_store_b16 v[0:1], v2, off
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
@ -83,10 +83,10 @@ define void @v_constained_fma_f16_fpexcept_strict_div(half %x, half %y, half %z,
; GFX942-LABEL: v_constained_fma_f16_fpexcept_strict_div:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v5, v4
; GFX942-NEXT: v_mov_b32_e32 v4, v3
; GFX942-NEXT: v_mov_b32_e32 v6, v3
; GFX942-NEXT: v_mov_b32_e32 v7, v4
; GFX942-NEXT: v_fma_f16 v0, v0, v1, v2
; GFX942-NEXT: global_store_short v[4:5], v0, off
; GFX942-NEXT: global_store_short v[6:7], v0, off
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
@ -463,9 +463,9 @@ define void @v_constained_fma_f16_fpexcept_strict_fneg_uni(half inreg %x, half i
; GFX8-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_uni:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v2, s17
; GFX8-NEXT: v_mov_b32_e32 v3, s18
; GFX8-NEXT: v_fma_f16 v2, s16, v2, -v3
; GFX8-NEXT: v_mov_b32_e32 v2, s16
; GFX8-NEXT: v_mov_b32_e32 v3, s17
; GFX8-NEXT: v_fma_f16 v2, v2, v3, -s18
; GFX8-NEXT: flat_store_short v[0:1], v2
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
@ -473,9 +473,9 @@ define void @v_constained_fma_f16_fpexcept_strict_fneg_uni(half inreg %x, half i
; GFX900-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_uni:
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: v_mov_b32_e32 v2, s17
; GFX900-NEXT: v_mov_b32_e32 v3, s18
; GFX900-NEXT: v_fma_f16 v2, s16, v2, -v3
; GFX900-NEXT: v_mov_b32_e32 v2, s16
; GFX900-NEXT: v_mov_b32_e32 v3, s17
; GFX900-NEXT: v_fma_f16 v2, v2, v3, -s18
; GFX900-NEXT: global_store_short v[0:1], v2, off
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
@ -483,9 +483,9 @@ define void @v_constained_fma_f16_fpexcept_strict_fneg_uni(half inreg %x, half i
; GFX942-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_uni:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v2, s1
; GFX942-NEXT: v_mov_b32_e32 v3, s2
; GFX942-NEXT: v_fma_f16 v2, s0, v2, -v3
; GFX942-NEXT: v_mov_b32_e32 v2, s0
; GFX942-NEXT: v_mov_b32_e32 v3, s1
; GFX942-NEXT: v_fma_f16 v2, v2, v3, -s2
; GFX942-NEXT: global_store_short v[0:1], v2, off
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@ -493,9 +493,9 @@ define void @v_constained_fma_f16_fpexcept_strict_fneg_uni(half inreg %x, half i
; GFX11-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_uni:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b16_e32 v2.l, s2
; GFX11-NEXT: v_mov_b16_e32 v2.l, s1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_fma_f16 v2.l, s0, s1, -v2.l
; GFX11-NEXT: v_fma_f16 v2.l, s0, v2.l, -s2
; GFX11-NEXT: global_store_b16 v[0:1], v2, off
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
@ -540,10 +540,10 @@ define void @v_constained_fma_f16_fpexcept_strict_fneg_div(half %x, half %y, hal
; GFX942-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_div:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v5, v4
; GFX942-NEXT: v_mov_b32_e32 v4, v3
; GFX942-NEXT: v_mov_b32_e32 v6, v3
; GFX942-NEXT: v_mov_b32_e32 v7, v4
; GFX942-NEXT: v_fma_f16 v0, v0, v1, -v2
; GFX942-NEXT: global_store_short v[4:5], v0, off
; GFX942-NEXT: global_store_short v[6:7], v0, off
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
@ -653,10 +653,10 @@ define void @v_constained_fma_f16_fpexcept_strict_fneg_fneg_div(half %x, half %y
; GFX942-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_fneg_div:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v5, v4
; GFX942-NEXT: v_mov_b32_e32 v4, v3
; GFX942-NEXT: v_mov_b32_e32 v6, v3
; GFX942-NEXT: v_mov_b32_e32 v7, v4
; GFX942-NEXT: v_fma_f16 v0, -v0, -v1, v2
; GFX942-NEXT: global_store_short v[4:5], v0, off
; GFX942-NEXT: global_store_short v[6:7], v0, off
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
@ -767,10 +767,10 @@ define void @v_constained_fma_f16_fpexcept_strict_fabs_fabs_div(half %x, half %y
; GFX942-LABEL: v_constained_fma_f16_fpexcept_strict_fabs_fabs_div:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v5, v4
; GFX942-NEXT: v_mov_b32_e32 v4, v3
; GFX942-NEXT: v_mov_b32_e32 v6, v3
; GFX942-NEXT: v_mov_b32_e32 v7, v4
; GFX942-NEXT: v_fma_f16 v0, |v0|, |v1|, v2
; GFX942-NEXT: global_store_short v[4:5], v0, off
; GFX942-NEXT: global_store_short v[6:7], v0, off
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;

8
llvm/test/CodeGen/AMDGPU/GlobalISel/sub.ll
View file

@ -1,10 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu7.01-amd-amdpal < %s | FileCheck -check-prefix=GFX7 %s
; RUN: llc -global-isel -mtriple=amdgpu9.00-amd-amdpal < %s | FileCheck -check-prefix=GFX9 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00-amd-amdpal < %s | FileCheck -check-prefix=GFX9 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-amd-amdpal < %s | FileCheck -check-prefix=GFX8 %s
; RUN: llc -global-isel -mtriple=amdgpu10.10-amd-amdpal < %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefix=GFX11 %s
; RUN: llc -global-isel -mtriple=amdgpu12.00-amd-amdpal -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefix=GFX12 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-amd-amdpal < %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefix=GFX11 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00-amd-amdpal -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefix=GFX12 %s
define i16 @s_sub_i16(i16 inreg %a, i16 inreg %b) {
; GFX7-LABEL: s_sub_i16:

18
llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
View file

@ -1811,10 +1811,12 @@ define amdgpu_ps i16 @s_uaddsat_i16(i16 inreg %lhs, i16 inreg %rhs) {
define amdgpu_ps half @uaddsat_i16_sv(i16 inreg %lhs, i16 %rhs) {
; GFX6-LABEL: uaddsat_i16_sv:
; GFX6: ; %bb.0:
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
; GFX6-NEXT: s_lshl_b32 s0, s0, 16
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: s_not_b32 s1, s0
; GFX6-NEXT: v_min_u32_e32 v0, s1, v0
; GFX6-NEXT: v_add_i32_e32 v0, vcc, s0, v0
; GFX6-NEXT: v_min_u32_e32 v0, 0xffff, v0
; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: uaddsat_i16_sv:
@ -1849,10 +1851,12 @@ define amdgpu_ps half @uaddsat_i16_sv(i16 inreg %lhs, i16 %rhs) {
define amdgpu_ps half @uaddsat_i16_vs(i16 %lhs, i16 inreg %rhs) {
; GFX6-LABEL: uaddsat_i16_vs:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX6-NEXT: v_add_i32_e32 v0, vcc, s0, v0
; GFX6-NEXT: v_min_u32_e32 v0, 0xffff, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: s_lshl_b32 s0, s0, 16
; GFX6-NEXT: v_not_b32_e32 v1, v0
; GFX6-NEXT: v_min_u32_e32 v1, s0, v1
; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1
; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: uaddsat_i16_vs:

18
llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll
View file

@ -1699,10 +1699,11 @@ define amdgpu_ps i16 @s_usubsat_i16(i16 inreg %lhs, i16 inreg %rhs) {
define amdgpu_ps half @usubsat_i16_sv(i16 inreg %lhs, i16 %rhs) {
; GFX6-LABEL: usubsat_i16_sv:
; GFX6: ; %bb.0:
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
; GFX6-NEXT: v_max_u32_e32 v1, s0, v0
; GFX6-NEXT: v_sub_i32_e32 v0, vcc, v1, v0
; GFX6-NEXT: s_lshl_b32 s0, s0, 16
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: v_min_u32_e32 v0, s0, v0
; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s0, v0
; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: usubsat_i16_sv:
@ -1737,10 +1738,11 @@ define amdgpu_ps half @usubsat_i16_sv(i16 inreg %lhs, i16 %rhs) {
define amdgpu_ps half @usubsat_i16_vs(i16 %lhs, i16 inreg %rhs) {
; GFX6-LABEL: usubsat_i16_vs:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX6-NEXT: v_max_u32_e32 v0, s0, v0
; GFX6-NEXT: v_subrev_i32_e32 v0, vcc, s0, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: s_lshl_b32 s0, s0, 16
; GFX6-NEXT: v_min_u32_e32 v1, s0, v0
; GFX6-NEXT: v_sub_i32_e32 v0, vcc, v0, v1
; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: usubsat_i16_vs:

2
llvm/test/CodeGen/AMDGPU/add-max.ll
View file

@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 < %s | FileCheck -check-prefixes=GCN,SDAG %s
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.50 -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GCN,GISEL %s
; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GCN,GISEL %s
define amdgpu_ps float @add_max_u32_vvv(i32 %a, i32 %b, i32 %c) {
; GCN-LABEL: add_max_u32_vvv:

4
llvm/test/CodeGen/AMDGPU/amdgcn-sin-cos-f16-f32.ll
View file

@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter-out "s_setreg_imm32_b32" --filter-out "shader" --version 6
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.50 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,FAKE16 %s
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,REAL16 %s
; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,FAKE16 %s
; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,REAL16 %s
define amdgpu_ps float @v_sin_f32(float %src) #1 {
; GCN-LABEL: v_sin_f32:

482
llvm/test/CodeGen/AMDGPU/atomicrmw_usub_sat.ll
View file

File diff suppressed because it is too large Load diff

4
llvm/test/CodeGen/AMDGPU/bfi_int.ll
View file

@ -2,8 +2,8 @@
; RUN: llc -mtriple=amdgpu6.00 < %s | FileCheck -enable-var-scope -check-prefix=GFX7 %s
; RUN: llc -mtriple=amdgpu8.02 < %s | FileCheck -enable-var-scope -check-prefix=GFX8 %s
; RUN: llc -mtriple=amdgpu10.31 < %s | FileCheck -enable-var-scope -check-prefix=GFX10 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.02 < %s | FileCheck -enable-var-scope -check-prefix=GFX8-GISEL %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.31 < %s | FileCheck -enable-var-scope -check-prefix=GFX10-GISEL %s
; RUN: llc -global-isel -mtriple=amdgpu8.02 < %s | FileCheck -enable-var-scope -check-prefix=GFX8-GISEL %s
; RUN: llc -global-isel -mtriple=amdgpu10.31 < %s | FileCheck -enable-var-scope -check-prefix=GFX10-GISEL %s
; BFI_INT Definition pattern from ISA docs
; (y & x) | (z & ~x)

110
llvm/test/CodeGen/AMDGPU/bitop3.ll
View file

@ -1,9 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel=0 -mtriple=amdgpu9.50-- < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX950,GFX950-SDAG %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.50-- < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX950,GFX950-GISEL %s
; RUN: llc -global-isel -mtriple=amdgpu9.50-- < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX950,GFX950-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-- -mattr=-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX1250,GFX1250-SDAG,GFX1250-SDAG-FAKE16,GFX1250-FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-- -mattr=+real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX1250,GFX1250-SDAG,GFX1250-SDAG-TRUE16,GFX1250-TRUE16 %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.50-- -mattr=-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX1250,GFX1250-GISEL,GFX1250-GISEL-FAKE16,GFX1250-FAKE16 %s
; RUN: llc -global-isel -mtriple=amdgpu12.50-- -mattr=-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX1250,GFX1250-GISEL,GFX1250-GISEL-FAKE16,GFX1250-FAKE16 %s
; RUN: llc -global-isel -mtriple=amdgpu12.50-- -mattr=+real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX1250,GFX1250-GISEL,GFX1250-GISEL-TRUE16,GFX1250-TRUE16 %s
; ========= Single bit functions =========
@ -718,10 +718,16 @@ define amdgpu_ps half @not_and_and_not_and_b16(i16 %a, i16 %b, i16 %c) {
}
define amdgpu_ps half @test_xor3_b16(i16 %a, i16 %b, i16 %c) {
; GFX950-LABEL: test_xor3_b16:
; GFX950: ; %bb.0:
; GFX950-NEXT: v_bitop3_b16 v0, v0, v2, v1 bitop3:0x96
; GFX950-NEXT: ; return to shader part epilog
; GFX950-SDAG-LABEL: test_xor3_b16:
; GFX950-SDAG: ; %bb.0:
; GFX950-SDAG-NEXT: v_bitop3_b16 v0, v0, v2, v1 bitop3:0x96
; GFX950-SDAG-NEXT: ; return to shader part epilog
;
; GFX950-GISEL-LABEL: test_xor3_b16:
; GFX950-GISEL: ; %bb.0:
; GFX950-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
; GFX950-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
; GFX950-GISEL-NEXT: ; return to shader part epilog
;
; GFX1250-SDAG-FAKE16-LABEL: test_xor3_b16:
; GFX1250-SDAG-FAKE16: ; %bb.0:
@ -739,21 +745,13 @@ define amdgpu_ps half @test_xor3_b16(i16 %a, i16 %b, i16 %c) {
; GFX1250-SDAG-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v2.l, v1.l bitop3:0x96
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-FAKE16-LABEL: test_xor3_b16:
; GFX1250-GISEL-FAKE16: ; %bb.0:
; GFX1250-GISEL-FAKE16-NEXT: global_wb
; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: v_bitop3_b16 v0, v0, v2, v1 bitop3:0x96
; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-TRUE16-LABEL: test_xor3_b16:
; GFX1250-GISEL-TRUE16: ; %bb.0:
; GFX1250-GISEL-TRUE16-NEXT: global_wb
; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: v_xor3_b32 v0, v0, v1, v2
; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
; GFX1250-GISEL-LABEL: test_xor3_b16:
; GFX1250-GISEL: ; %bb.0:
; GFX1250-GISEL-NEXT: global_wb
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_xor3_b32 v0, v0, v1, v2
; GFX1250-GISEL-NEXT: ; return to shader part epilog
%xor1 = xor i16 %a, %b
%xor2 = xor i16 %xor1, %c
%ret_cast = bitcast i16 %xor2 to half
@ -761,10 +759,15 @@ define amdgpu_ps half @test_xor3_b16(i16 %a, i16 %b, i16 %c) {
}
define amdgpu_ps half @test_or3_b16(i16 %a, i16 %b, i16 %c) {
; GFX950-LABEL: test_or3_b16:
; GFX950: ; %bb.0:
; GFX950-NEXT: v_bitop3_b16 v0, v0, v2, v1 bitop3:0xfe
; GFX950-NEXT: ; return to shader part epilog
; GFX950-SDAG-LABEL: test_or3_b16:
; GFX950-SDAG: ; %bb.0:
; GFX950-SDAG-NEXT: v_bitop3_b16 v0, v0, v2, v1 bitop3:0xfe
; GFX950-SDAG-NEXT: ; return to shader part epilog
;
; GFX950-GISEL-LABEL: test_or3_b16:
; GFX950-GISEL: ; %bb.0:
; GFX950-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
; GFX950-GISEL-NEXT: ; return to shader part epilog
;
; GFX1250-SDAG-FAKE16-LABEL: test_or3_b16:
; GFX1250-SDAG-FAKE16: ; %bb.0:
@ -782,21 +785,13 @@ define amdgpu_ps half @test_or3_b16(i16 %a, i16 %b, i16 %c) {
; GFX1250-SDAG-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v2.l, v1.l bitop3:0xfe
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-FAKE16-LABEL: test_or3_b16:
; GFX1250-GISEL-FAKE16: ; %bb.0:
; GFX1250-GISEL-FAKE16-NEXT: global_wb
; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: v_bitop3_b16 v0, v0, v2, v1 bitop3:0xfe
; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-TRUE16-LABEL: test_or3_b16:
; GFX1250-GISEL-TRUE16: ; %bb.0:
; GFX1250-GISEL-TRUE16-NEXT: global_wb
; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: v_or3_b32 v0, v0, v1, v2
; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
; GFX1250-GISEL-LABEL: test_or3_b16:
; GFX1250-GISEL: ; %bb.0:
; GFX1250-GISEL-NEXT: global_wb
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
; GFX1250-GISEL-NEXT: ; return to shader part epilog
%or1 = or i16 %a, %b
%or2 = or i16 %or1, %c
%ret_cast = bitcast i16 %or2 to half
@ -804,10 +799,15 @@ define amdgpu_ps half @test_or3_b16(i16 %a, i16 %b, i16 %c) {
}
define amdgpu_ps half @test_and_or_b16(i16 %a, i16 %b, i16 %c) {
; GFX950-LABEL: test_and_or_b16:
; GFX950: ; %bb.0:
; GFX950-NEXT: v_bitop3_b16 v0, v0, v2, v1 bitop3:0xec
; GFX950-NEXT: ; return to shader part epilog
; GFX950-SDAG-LABEL: test_and_or_b16:
; GFX950-SDAG: ; %bb.0:
; GFX950-SDAG-NEXT: v_bitop3_b16 v0, v0, v2, v1 bitop3:0xec
; GFX950-SDAG-NEXT: ; return to shader part epilog
;
; GFX950-GISEL-LABEL: test_and_or_b16:
; GFX950-GISEL: ; %bb.0:
; GFX950-GISEL-NEXT: v_and_or_b32 v0, v0, v1, v2
; GFX950-GISEL-NEXT: ; return to shader part epilog
;
; GFX1250-SDAG-FAKE16-LABEL: test_and_or_b16:
; GFX1250-SDAG-FAKE16: ; %bb.0:
@ -825,21 +825,13 @@ define amdgpu_ps half @test_and_or_b16(i16 %a, i16 %b, i16 %c) {
; GFX1250-SDAG-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v2.l, v1.l bitop3:0xec
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-FAKE16-LABEL: test_and_or_b16:
; GFX1250-GISEL-FAKE16: ; %bb.0:
; GFX1250-GISEL-FAKE16-NEXT: global_wb
; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: v_bitop3_b16 v0, v0, v2, v1 bitop3:0xec
; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-TRUE16-LABEL: test_and_or_b16:
; GFX1250-GISEL-TRUE16: ; %bb.0:
; GFX1250-GISEL-TRUE16-NEXT: global_wb
; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: v_and_or_b32 v0, v0, v1, v2
; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
; GFX1250-GISEL-LABEL: test_and_or_b16:
; GFX1250-GISEL: ; %bb.0:
; GFX1250-GISEL-NEXT: global_wb
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_and_or_b32 v0, v0, v1, v2
; GFX1250-GISEL-NEXT: ; return to shader part epilog
%and1 = and i16 %a, %b
%or1 = or i16 %and1, %c
%ret_cast = bitcast i16 %or1 to half

4
llvm/test/CodeGen/AMDGPU/bitreverse.ll
View file

@ -2,11 +2,11 @@
; RUN: llc < %s -mtriple=amdgpu6.00-- | FileCheck %s --check-prefix=SI
; RUN: llc < %s -mtriple=amdgpu8.02-- -mattr=-flat-for-global | FileCheck %s --check-prefix=FLAT
; RUN: llc < %s -mtriple=amdgpu8.03-- -mattr=-flat-for-global | FileCheck %s --check-prefix=FLAT
; RUN: llc < %s -mtriple=amdgpu8.03-- -global-isel -global-isel-abort=2 | FileCheck %s --check-prefix=GISEL
; RUN: llc < %s -mtriple=amdgpu8.03-- -global-isel | FileCheck %s --check-prefix=GISEL
; RUN: llc < %s -mtriple=amdgpu11.00-- -mattr=+real-true16 -mattr=-flat-for-global | FileCheck %s --check-prefixes=GFX11-FLAT,GFX11-FLAT-TRUE16
; RUN: llc < %s -mtriple=amdgpu11.00-- -mattr=-real-true16 -mattr=-flat-for-global | FileCheck %s --check-prefixes=GFX11-FLAT,GFX11-FLAT-FAKE16
; RUN: llc < %s -mtriple=amdgpu11.00-- -mattr=+real-true16 -global-isel | FileCheck %s --check-prefixes=GFX11-GISEL,GFX11-GISEL-TRUE16
; RUN: llc < %s -mtriple=amdgpu11.00-- -mattr=-real-true16 -global-isel -global-isel-abort=2 | FileCheck %s --check-prefixes=GFX11-GISEL,GFX11-GISEL-FAKE16
; RUN: llc < %s -mtriple=amdgpu11.00-- -mattr=-real-true16 -global-isel | FileCheck %s --check-prefixes=GFX11-GISEL,GFX11-GISEL-FAKE16
declare i32 @llvm.amdgcn.workitem.id.x() #1

2
llvm/test/CodeGen/AMDGPU/buffer-rsrc-ptr-ops.ll
View file

@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00 < %s | FileCheck --check-prefix=GISEL %s
; RUN: llc -global-isel=1 -mtriple=amdgpu9.00 < %s | FileCheck --check-prefix=GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu9.00 < %s | FileCheck --check-prefix=SDAG %s
define amdgpu_kernel void @buffer_ptr_vector_ops(ptr addrspace(1) %somewhere) {

390
llvm/test/CodeGen/AMDGPU/build-vector-packed-partial-undef.ll
View file

@ -2,9 +2,9 @@
; RUN: llc -global-isel=0 -mtriple=amdgpu8.03-amd-amdhsa < %s | FileCheck -enable-var-scope -check-prefixes=GFX8,GFX8-SDAG %s
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.03-amd-amdhsa < %s | FileCheck -enable-var-scope -check-prefixes=GFX8,GFX8-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GFX9-SDAG %s
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=1 -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00-amd-amdhsa -mattr=-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-FAKE16,GFX11-FAKE16-SDAG %s
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdhsa -mattr=-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-FAKE16,GFX11-FAKE16-GISEL %s
; RUN: llc -global-isel=1 -mtriple=amdgpu11.00-amd-amdhsa -mattr=-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-FAKE16,GFX11-FAKE16-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00-amd-amdhsa -mattr=+real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-TRUE16,GFX11-TRUE16-SDAG %s
; RUN: llc -global-isel=1 -mtriple=amdgpu11.00-amd-amdhsa -mattr=+real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-TRUE16,GFX11-TRUE16-GISEL %s
@ -81,23 +81,45 @@ define void @undef_lo_v2f16(half %arg0) {
; GFX8-NEXT: ;;#ASMEND
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: undef_lo_v2f16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: ;;#ASMSTART
; GFX9-NEXT: ; use v0
; GFX9-NEXT: ;;#ASMEND
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-SDAG-LABEL: undef_lo_v2f16:
; GFX9-SDAG: ; %bb.0:
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-SDAG-NEXT: ;;#ASMSTART
; GFX9-SDAG-NEXT: ; use v0
; GFX9-SDAG-NEXT: ;;#ASMEND
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: undef_lo_v2f16:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11-FAKE16-NEXT: ;;#ASMSTART
; GFX11-FAKE16-NEXT: ; use v0
; GFX11-FAKE16-NEXT: ;;#ASMEND
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
; GFX9-GISEL-LABEL: undef_lo_v2f16:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0xffff
; GFX9-GISEL-NEXT: v_and_b32_e32 v1, s4, v1
; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v1
; GFX9-GISEL-NEXT: ;;#ASMSTART
; GFX9-GISEL-NEXT: ; use v0
; GFX9-GISEL-NEXT: ;;#ASMEND
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-SDAG-LABEL: undef_lo_v2f16:
; GFX11-FAKE16-SDAG: ; %bb.0:
; GFX11-FAKE16-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11-FAKE16-SDAG-NEXT: ;;#ASMSTART
; GFX11-FAKE16-SDAG-NEXT: ; use v0
; GFX11-FAKE16-SDAG-NEXT: ;;#ASMEND
; GFX11-FAKE16-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-GISEL-LABEL: undef_lo_v2f16:
; GFX11-FAKE16-GISEL: ; %bb.0:
; GFX11-FAKE16-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-GISEL-NEXT: v_and_b32_e64 v1, 0xffff, s0
; GFX11-FAKE16-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v1
; GFX11-FAKE16-GISEL-NEXT: ;;#ASMSTART
; GFX11-FAKE16-GISEL-NEXT: ; use v0
; GFX11-FAKE16-GISEL-NEXT: ;;#ASMEND
; GFX11-FAKE16-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: undef_lo_v2f16:
; GFX11-TRUE16: ; %bb.0:
@ -124,26 +146,50 @@ define void @undef_lo_op_v2f16(half %arg0) {
; GFX8-NEXT: ;;#ASMEND
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: undef_lo_op_v2f16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: v_pk_add_f16 v0, v0, 1.0 op_sel_hi:[1,0]
; GFX9-NEXT: ;;#ASMSTART
; GFX9-NEXT: ; use v0
; GFX9-NEXT: ;;#ASMEND
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-SDAG-LABEL: undef_lo_op_v2f16:
; GFX9-SDAG: ; %bb.0:
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-SDAG-NEXT: v_pk_add_f16 v0, v0, 1.0 op_sel_hi:[1,0]
; GFX9-SDAG-NEXT: ;;#ASMSTART
; GFX9-SDAG-NEXT: ; use v0
; GFX9-SDAG-NEXT: ;;#ASMEND
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: undef_lo_op_v2f16:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_pk_add_f16 v0, v0, 1.0 op_sel_hi:[1,0]
; GFX11-FAKE16-NEXT: ;;#ASMSTART
; GFX11-FAKE16-NEXT: ; use v0
; GFX11-FAKE16-NEXT: ;;#ASMEND
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
; GFX9-GISEL-LABEL: undef_lo_op_v2f16:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0xffff
; GFX9-GISEL-NEXT: v_and_b32_e32 v1, s4, v1
; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v1
; GFX9-GISEL-NEXT: v_pk_add_f16 v0, v0, 1.0 op_sel_hi:[1,0]
; GFX9-GISEL-NEXT: ;;#ASMSTART
; GFX9-GISEL-NEXT: ; use v0
; GFX9-GISEL-NEXT: ;;#ASMEND
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-SDAG-LABEL: undef_lo_op_v2f16:
; GFX11-FAKE16-SDAG: ; %bb.0:
; GFX11-FAKE16-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11-FAKE16-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-SDAG-NEXT: v_pk_add_f16 v0, v0, 1.0 op_sel_hi:[1,0]
; GFX11-FAKE16-SDAG-NEXT: ;;#ASMSTART
; GFX11-FAKE16-SDAG-NEXT: ; use v0
; GFX11-FAKE16-SDAG-NEXT: ;;#ASMEND
; GFX11-FAKE16-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-GISEL-LABEL: undef_lo_op_v2f16:
; GFX11-FAKE16-GISEL: ; %bb.0:
; GFX11-FAKE16-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-GISEL-NEXT: v_and_b32_e64 v1, 0xffff, s0
; GFX11-FAKE16-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v1
; GFX11-FAKE16-GISEL-NEXT: v_pk_add_f16 v0, v0, 1.0 op_sel_hi:[1,0]
; GFX11-FAKE16-GISEL-NEXT: ;;#ASMSTART
; GFX11-FAKE16-GISEL-NEXT: ; use v0
; GFX11-FAKE16-GISEL-NEXT: ;;#ASMEND
; GFX11-FAKE16-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: undef_lo_op_v2f16:
; GFX11-TRUE16: ; %bb.0:
@ -319,23 +365,45 @@ define void @undef_lo3_v4f16(half %arg0) {
; GFX8-NEXT: ;;#ASMEND
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: undef_lo3_v4f16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-NEXT: ;;#ASMSTART
; GFX9-NEXT: ; use v[0:1]
; GFX9-NEXT: ;;#ASMEND
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-SDAG-LABEL: undef_lo3_v4f16:
; GFX9-SDAG: ; %bb.0:
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX9-SDAG-NEXT: ;;#ASMSTART
; GFX9-SDAG-NEXT: ; use v[0:1]
; GFX9-SDAG-NEXT: ;;#ASMEND
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: undef_lo3_v4f16:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11-FAKE16-NEXT: ;;#ASMSTART
; GFX11-FAKE16-NEXT: ; use v[0:1]
; GFX11-FAKE16-NEXT: ;;#ASMEND
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
; GFX9-GISEL-LABEL: undef_lo3_v4f16:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0xffff
; GFX9-GISEL-NEXT: v_and_b32_e32 v1, s4, v1
; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v1
; GFX9-GISEL-NEXT: ;;#ASMSTART
; GFX9-GISEL-NEXT: ; use v[0:1]
; GFX9-GISEL-NEXT: ;;#ASMEND
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-SDAG-LABEL: undef_lo3_v4f16:
; GFX11-FAKE16-SDAG: ; %bb.0:
; GFX11-FAKE16-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11-FAKE16-SDAG-NEXT: ;;#ASMSTART
; GFX11-FAKE16-SDAG-NEXT: ; use v[0:1]
; GFX11-FAKE16-SDAG-NEXT: ;;#ASMEND
; GFX11-FAKE16-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-GISEL-LABEL: undef_lo3_v4f16:
; GFX11-FAKE16-GISEL: ; %bb.0:
; GFX11-FAKE16-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-GISEL-NEXT: v_and_b32_e64 v1, 0xffff, s0
; GFX11-FAKE16-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v1
; GFX11-FAKE16-GISEL-NEXT: ;;#ASMSTART
; GFX11-FAKE16-GISEL-NEXT: ; use v[0:1]
; GFX11-FAKE16-GISEL-NEXT: ;;#ASMEND
; GFX11-FAKE16-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: undef_lo3_v4f16:
; GFX11-TRUE16: ; %bb.0:
@ -504,29 +572,67 @@ define void @undef_hi_v2i16(i16 %arg0) {
}
define void @undef_hi_v2f16(half %arg0) {
; GFX8-LABEL: undef_hi_v2f16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: ;;#ASMSTART
; GFX8-NEXT: ; use v0
; GFX8-NEXT: ;;#ASMEND
; GFX8-NEXT: s_setpc_b64 s[30:31]
; GFX8-SDAG-LABEL: undef_hi_v2f16:
; GFX8-SDAG: ; %bb.0:
; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-SDAG-NEXT: ;;#ASMSTART
; GFX8-SDAG-NEXT: ; use v0
; GFX8-SDAG-NEXT: ;;#ASMEND
; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: undef_hi_v2f16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: ;;#ASMSTART
; GFX9-NEXT: ; use v0
; GFX9-NEXT: ;;#ASMEND
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX8-GISEL-LABEL: undef_hi_v2f16:
; GFX8-GISEL: ; %bb.0:
; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX8-GISEL-NEXT: ;;#ASMSTART
; GFX8-GISEL-NEXT: ; use v0
; GFX8-GISEL-NEXT: ;;#ASMEND
; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: undef_hi_v2f16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: ;;#ASMSTART
; GFX11-NEXT: ; use v0
; GFX11-NEXT: ;;#ASMEND
; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-SDAG-LABEL: undef_hi_v2f16:
; GFX9-SDAG: ; %bb.0:
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-SDAG-NEXT: ;;#ASMSTART
; GFX9-SDAG-NEXT: ; use v0
; GFX9-SDAG-NEXT: ;;#ASMEND
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-GISEL-LABEL: undef_hi_v2f16:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, s4, 16, v0
; GFX9-GISEL-NEXT: ;;#ASMSTART
; GFX9-GISEL-NEXT: ; use v0
; GFX9-GISEL-NEXT: ;;#ASMEND
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-SDAG-LABEL: undef_hi_v2f16:
; GFX11-FAKE16-SDAG: ; %bb.0:
; GFX11-FAKE16-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-SDAG-NEXT: ;;#ASMSTART
; GFX11-FAKE16-SDAG-NEXT: ; use v0
; GFX11-FAKE16-SDAG-NEXT: ;;#ASMEND
; GFX11-FAKE16-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-GISEL-LABEL: undef_hi_v2f16:
; GFX11-FAKE16-GISEL: ; %bb.0:
; GFX11-FAKE16-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX11-FAKE16-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-GISEL-NEXT: v_lshl_or_b32 v0, s0, 16, v0
; GFX11-FAKE16-GISEL-NEXT: ;;#ASMSTART
; GFX11-FAKE16-GISEL-NEXT: ; use v0
; GFX11-FAKE16-GISEL-NEXT: ;;#ASMEND
; GFX11-FAKE16-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: undef_hi_v2f16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: ;;#ASMSTART
; GFX11-TRUE16-NEXT: ; use v0
; GFX11-TRUE16-NEXT: ;;#ASMEND
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
%undef.hi = insertelement <2 x half> poison, half %arg0, i32 0
call void asm sideeffect "; use $0", "v"(<2 x half> %undef.hi);
ret void
@ -543,23 +649,46 @@ define void @undef_hi_op_v2f16(half %arg0) {
; GFX8-NEXT: ;;#ASMEND
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: undef_hi_op_v2f16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_pk_add_f16 v0, v0, 1.0 op_sel_hi:[1,0]
; GFX9-NEXT: ;;#ASMSTART
; GFX9-NEXT: ; use v0
; GFX9-NEXT: ;;#ASMEND
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-SDAG-LABEL: undef_hi_op_v2f16:
; GFX9-SDAG: ; %bb.0:
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-SDAG-NEXT: v_pk_add_f16 v0, v0, 1.0 op_sel_hi:[1,0]
; GFX9-SDAG-NEXT: ;;#ASMSTART
; GFX9-SDAG-NEXT: ; use v0
; GFX9-SDAG-NEXT: ;;#ASMEND
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: undef_hi_op_v2f16:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_pk_add_f16 v0, v0, 1.0 op_sel_hi:[1,0]
; GFX11-FAKE16-NEXT: ;;#ASMSTART
; GFX11-FAKE16-NEXT: ; use v0
; GFX11-FAKE16-NEXT: ;;#ASMEND
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
; GFX9-GISEL-LABEL: undef_hi_op_v2f16:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, s4, 16, v0
; GFX9-GISEL-NEXT: v_pk_add_f16 v0, v0, 1.0 op_sel_hi:[1,0]
; GFX9-GISEL-NEXT: ;;#ASMSTART
; GFX9-GISEL-NEXT: ; use v0
; GFX9-GISEL-NEXT: ;;#ASMEND
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-SDAG-LABEL: undef_hi_op_v2f16:
; GFX11-FAKE16-SDAG: ; %bb.0:
; GFX11-FAKE16-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-SDAG-NEXT: v_pk_add_f16 v0, v0, 1.0 op_sel_hi:[1,0]
; GFX11-FAKE16-SDAG-NEXT: ;;#ASMSTART
; GFX11-FAKE16-SDAG-NEXT: ; use v0
; GFX11-FAKE16-SDAG-NEXT: ;;#ASMEND
; GFX11-FAKE16-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-GISEL-LABEL: undef_hi_op_v2f16:
; GFX11-FAKE16-GISEL: ; %bb.0:
; GFX11-FAKE16-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX11-FAKE16-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-GISEL-NEXT: v_lshl_or_b32 v0, s0, 16, v0
; GFX11-FAKE16-GISEL-NEXT: v_pk_add_f16 v0, v0, 1.0 op_sel_hi:[1,0]
; GFX11-FAKE16-GISEL-NEXT: ;;#ASMSTART
; GFX11-FAKE16-GISEL-NEXT: ; use v0
; GFX11-FAKE16-GISEL-NEXT: ;;#ASMEND
; GFX11-FAKE16-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-SDAG-LABEL: undef_hi_op_v2f16:
; GFX11-TRUE16-SDAG: ; %bb.0:
@ -730,29 +859,68 @@ define void @undef_hi3_v4i16(i16 %arg0) {
}
define void @undef_hi3_v4f16(half %arg0) {
; GFX8-LABEL: undef_hi3_v4f16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: ;;#ASMSTART
; GFX8-NEXT: ; use v[0:1]
; GFX8-NEXT: ;;#ASMEND
; GFX8-NEXT: s_setpc_b64 s[30:31]
; GFX8-SDAG-LABEL: undef_hi3_v4f16:
; GFX8-SDAG: ; %bb.0:
; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-SDAG-NEXT: ;;#ASMSTART
; GFX8-SDAG-NEXT: ; use v[0:1]
; GFX8-SDAG-NEXT: ;;#ASMEND
; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: undef_hi3_v4f16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: ;;#ASMSTART
; GFX9-NEXT: ; use v[0:1]
; GFX9-NEXT: ;;#ASMEND
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX8-GISEL-LABEL: undef_hi3_v4f16:
; GFX8-GISEL: ; %bb.0:
; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX8-GISEL-NEXT: ;;#ASMSTART
; GFX8-GISEL-NEXT: ; use v[0:1]
; GFX8-GISEL-NEXT: ;;#ASMEND
; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: undef_hi3_v4f16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: ;;#ASMSTART
; GFX11-NEXT: ; use v[0:1]
; GFX11-NEXT: ;;#ASMEND
; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX9-SDAG-LABEL: undef_hi3_v4f16:
; GFX9-SDAG: ; %bb.0:
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-SDAG-NEXT: ;;#ASMSTART
; GFX9-SDAG-NEXT: ; use v[0:1]
; GFX9-SDAG-NEXT: ;;#ASMEND
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-GISEL-LABEL: undef_hi3_v4f16:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, s4, 16, v0
; GFX9-GISEL-NEXT: ;;#ASMSTART
; GFX9-GISEL-NEXT: ; use v[0:1]
; GFX9-GISEL-NEXT: ;;#ASMEND
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-SDAG-LABEL: undef_hi3_v4f16:
; GFX11-FAKE16-SDAG: ; %bb.0:
; GFX11-FAKE16-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-SDAG-NEXT: ;;#ASMSTART
; GFX11-FAKE16-SDAG-NEXT: ; use v[0:1]
; GFX11-FAKE16-SDAG-NEXT: ;;#ASMEND
; GFX11-FAKE16-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-GISEL-LABEL: undef_hi3_v4f16:
; GFX11-FAKE16-GISEL: ; %bb.0:
; GFX11-FAKE16-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX11-FAKE16-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-GISEL-NEXT: v_lshl_or_b32 v0, s0, 16, v0
; GFX11-FAKE16-GISEL-NEXT: ;;#ASMSTART
; GFX11-FAKE16-GISEL-NEXT: ; use v[0:1]
; GFX11-FAKE16-GISEL-NEXT: ;;#ASMEND
; GFX11-FAKE16-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: undef_hi3_v4f16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: ;;#ASMSTART
; GFX11-TRUE16-NEXT: ; use v[0:1]
; GFX11-TRUE16-NEXT: ;;#ASMEND
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
%undef.hi = insertelement <4 x half> poison, half %arg0, i32 0
call void asm sideeffect "; use $0", "v"(<4 x half> %undef.hi);
ret void

4
llvm/test/CodeGen/AMDGPU/call-args-inreg.ll
View file

@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
; RUN: llc -global-isel=0 -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,SDAG %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00-amd-amdhsa < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,SDAG %s
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GISEL %s
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdhsa < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GISEL %s
; RUN: llc -global-isel=1 -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GISEL %s
; RUN: llc -global-isel=1 -mtriple=amdgpu11.00-amd-amdhsa < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GISEL %s
declare hidden void @external_void_func_i8_inreg(i8 inreg) #0
declare hidden void @external_void_func_i16_inreg(i32 inreg) #0

4
llvm/test/CodeGen/AMDGPU/call-return-types.ll
View file

@ -2,9 +2,9 @@
; RUN: llc -global-isel=0 -mtriple=amdgpu7.01-amd-amdhsa < %s | FileCheck -check-prefixes=GCN,GFX7 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu9.00-amd-amdhsa -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GCN,GFX89 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00-amd-amdhsa -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GCN,GFX11 %s
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.03-amd-amdhsa -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GCN,GFX89 %s
; xUN: llc -global-isel=1 -mtriple=amdgpu8.03-amd-amdhsa -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GCN,GFX89 %s
; xUN: llc -global-isel=1 -mtriple=amdgpu7.01-amd-amdhsa < %s | FileCheck -check-prefixes=GCN,GFX7 %s
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00-amd-amdhsa -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GCN,GFX89 %s
; xUN: llc -global-isel=1 -mtriple=amdgpu9.00-amd-amdhsa -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GCN,GFX89 %s
; Ideally, we would also like to test GlobalISel with gfx11 but we are currently blocked on llvm-project#166501.

42
llvm/test/CodeGen/AMDGPU/commute-compares-scalar-float.ll
View file

@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=amdgpu11.50 -amdgpu-sdwa-peephole=0 < %s | FileCheck -check-prefix=SDAG %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.50 -amdgpu-sdwa-peephole=0 < %s | FileCheck -check-prefix=GISEL %s
; RUN: llc -global-isel -mtriple=amdgpu11.50 -amdgpu-sdwa-peephole=0 < %s | FileCheck -check-prefix=GISEL %s
define amdgpu_vs void @fcmp_f32_olt_to_ogt(ptr addrspace(1) inreg %out, float inreg %a) {
; SDAG-LABEL: fcmp_f32_olt_to_ogt:
@ -223,10 +223,11 @@ define amdgpu_vs void @fcmp_f16_olt_to_ogt(ptr addrspace(1) inreg %out, half inr
; GISEL-LABEL: fcmp_f16_olt_to_ogt:
; GISEL: ; %bb.0: ; %entry
; GISEL-NEXT: s_cmp_gt_f16 s2, 0x4000
; GISEL-NEXT: v_mov_b32_e32 v1, 0
; GISEL-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GISEL-NEXT: v_mov_b32_e32 v0, s2
; GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-NEXT: s_endpgm
entry:
%0 = fcmp olt half 2.0, %a
@ -248,10 +249,11 @@ define amdgpu_vs void @fcmp_f16_ogt_to_olt(ptr addrspace(1) inreg %out, half inr
; GISEL-LABEL: fcmp_f16_ogt_to_olt:
; GISEL: ; %bb.0: ; %entry
; GISEL-NEXT: s_cmp_lt_f16 s2, 0x4000
; GISEL-NEXT: v_mov_b32_e32 v1, 0
; GISEL-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GISEL-NEXT: v_mov_b32_e32 v0, s2
; GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-NEXT: s_endpgm
entry:
%0 = fcmp ogt half 2.0, %a
@ -273,10 +275,11 @@ define amdgpu_vs void @fcmp_f16_ole_to_oge(ptr addrspace(1) inreg %out, half inr
; GISEL-LABEL: fcmp_f16_ole_to_oge:
; GISEL: ; %bb.0: ; %entry
; GISEL-NEXT: s_cmp_ge_f16 s2, 0x4000
; GISEL-NEXT: v_mov_b32_e32 v1, 0
; GISEL-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GISEL-NEXT: v_mov_b32_e32 v0, s2
; GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-NEXT: s_endpgm
entry:
%0 = fcmp ole half 2.0, %a
@ -298,10 +301,11 @@ define amdgpu_vs void @fcmp_f16_oge_to_ole(ptr addrspace(1) inreg %out, half inr
; GISEL-LABEL: fcmp_f16_oge_to_ole:
; GISEL: ; %bb.0: ; %entry
; GISEL-NEXT: s_cmp_le_f16 s2, 0x4000
; GISEL-NEXT: v_mov_b32_e32 v1, 0
; GISEL-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GISEL-NEXT: v_mov_b32_e32 v0, s2
; GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-NEXT: s_endpgm
entry:
%0 = fcmp oge half 2.0, %a
@ -323,10 +327,11 @@ define amdgpu_vs void @fcmp_f16_ult_to_ugt(ptr addrspace(1) inreg %out, half inr
; GISEL-LABEL: fcmp_f16_ult_to_ugt:
; GISEL: ; %bb.0: ; %entry
; GISEL-NEXT: s_cmp_nle_f16 s2, 0x4000
; GISEL-NEXT: v_mov_b32_e32 v1, 0
; GISEL-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GISEL-NEXT: v_mov_b32_e32 v0, s2
; GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-NEXT: s_endpgm
entry:
%0 = fcmp ult half 2.0, %a
@ -348,10 +353,11 @@ define amdgpu_vs void @fcmp_f16_ugt_to_ult(ptr addrspace(1) inreg %out, half inr
; GISEL-LABEL: fcmp_f16_ugt_to_ult:
; GISEL: ; %bb.0: ; %entry
; GISEL-NEXT: s_cmp_nge_f16 s2, 0x4000
; GISEL-NEXT: v_mov_b32_e32 v1, 0
; GISEL-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GISEL-NEXT: v_mov_b32_e32 v0, s2
; GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-NEXT: s_endpgm
entry:
%0 = fcmp ugt half 2.0, %a
@ -373,10 +379,11 @@ define amdgpu_vs void @fcmp_ule_to_uge(ptr addrspace(1) inreg %out, half inreg %
; GISEL-LABEL: fcmp_ule_to_uge:
; GISEL: ; %bb.0: ; %entry
; GISEL-NEXT: s_cmp_nlt_f16 s2, 0x4000
; GISEL-NEXT: v_mov_b32_e32 v1, 0
; GISEL-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GISEL-NEXT: v_mov_b32_e32 v0, s2
; GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-NEXT: s_endpgm
entry:
%0 = fcmp ule half 2.0, %a
@ -398,10 +405,11 @@ define amdgpu_vs void @fcmp_uge_to_ule(ptr addrspace(1) inreg %out, half inreg %
; GISEL-LABEL: fcmp_uge_to_ule:
; GISEL: ; %bb.0: ; %entry
; GISEL-NEXT: s_cmp_ngt_f16 s2, 0x4000
; GISEL-NEXT: v_mov_b32_e32 v1, 0
; GISEL-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GISEL-NEXT: v_mov_b32_e32 v0, s2
; GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-NEXT: s_endpgm
entry:
%0 = fcmp uge half 2.0, %a

2
llvm/test/CodeGen/AMDGPU/ctlz.ll
View file

@ -3,7 +3,7 @@
; RUN: llc < %s -mtriple=amdgpu8.02 -mattr=-flat-for-global | FileCheck %s -enable-var-scope --check-prefix=VI
; RUN: llc < %s -mtriple=r600 -mcpu=cypress | FileCheck %s -enable-var-scope --check-prefix=EG
; RUN: llc < %s -mtriple=amdgpu10.10 | FileCheck %s -enable-var-scope --check-prefix=GFX10
; RUN: llc < %s -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 | FileCheck %s -enable-var-scope --check-prefix=GFX10-GISEL
; RUN: llc < %s -global-isel -mtriple=amdgpu10.10 | FileCheck %s -enable-var-scope --check-prefix=GFX10-GISEL
; RUN: llc < %s -mtriple=amdgpu11.00 -mattr=+real-true16 | FileCheck %s -enable-var-scope --check-prefixes=GFX11,GFX11-TRUE16
; RUN: llc < %s -mtriple=amdgpu11.00 -mattr=-real-true16 | FileCheck %s -enable-var-scope --check-prefixes=GFX11,GFX11-FAKE16

2
llvm/test/CodeGen/AMDGPU/ctpop64.ll
View file

@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
; RUN: llc -global-isel=0 -mtriple=amdgpu6.00 < %s | FileCheck -check-prefix=SI %s
; RUN: llc -global-isel=0 -mtriple=amdgpu8.02 -mattr=-flat-for-global < %s | FileCheck -check-prefix=VI %s
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00 < %s | FileCheck -check-prefix=GFX12 %s
; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 < %s | FileCheck -check-prefix=GFX12 %s
declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone

2
llvm/test/CodeGen/AMDGPU/cttz.ll
View file

@ -3,7 +3,7 @@
; RUN: llc < %s -mtriple=amdgpu8.02 -mattr=-flat-for-global | FileCheck %s -enable-var-scope --check-prefix=VI
; RUN: llc < %s -mtriple=r600 -mcpu=cypress | FileCheck %s -enable-var-scope --check-prefix=EG
; RUN: llc < %s -mtriple=amdgpu10.10 | FileCheck %s -enable-var-scope --check-prefix=GFX10
; RUN: llc < %s -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 | FileCheck %s -enable-var-scope --check-prefix=GFX10-GISEL
; RUN: llc < %s -global-isel -mtriple=amdgpu10.10 | FileCheck %s -enable-var-scope --check-prefix=GFX10-GISEL
declare i7 @llvm.cttz.i7(i7, i1) nounwind readnone
declare i8 @llvm.cttz.i8(i8, i1) nounwind readnone

230
llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll
View file

@ -1996,24 +1996,43 @@ define half @fmul_select_f16_test6(half %x, i32 %bool.arg1, i32 %bool.arg2) {
; GFX7-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: fmul_select_f16_test6:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v3, 0x4200
; GFX9-NEXT: v_mov_b32_e32 v4, 0xc800
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
; GFX9-NEXT: v_mul_f16_e32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-SDAG-LABEL: fmul_select_f16_test6:
; GFX9-SDAG: ; %bb.0:
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, 0x4200
; GFX9-SDAG-NEXT: v_mov_b32_e32 v4, 0xc800
; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
; GFX9-SDAG-NEXT: v_mul_f16_e32 v0, v0, v1
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: fmul_select_f16_test6:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v3, 0xc800
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
; GFX10-NEXT: v_cndmask_b32_e32 v1, 0x4200, v3, vcc_lo
; GFX10-NEXT: v_mul_f16_e32 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
; GFX9-GISEL-LABEL: fmul_select_f16_test6:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0xc800
; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, 0x4200
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
; GFX9-GISEL-NEXT: v_mul_f16_e32 v0, v0, v1
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SDAG-LABEL: fmul_select_f16_test6:
; GFX10-SDAG: ; %bb.0:
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: v_mov_b32_e32 v3, 0xc800
; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v1, 0x4200, v3, vcc_lo
; GFX10-SDAG-NEXT: v_mul_f16_e32 v0, v0, v1
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: fmul_select_f16_test6:
; GFX10-GISEL: ; %bb.0:
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, 0x4200
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, v3, 0xc800, vcc_lo
; GFX10-GISEL-NEXT: v_mul_f16_e32 v0, v0, v1
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: fmul_select_f16_test6:
; GFX11-SDAG-TRUE16: ; %bb.0:
@ -2048,10 +2067,10 @@ define half @fmul_select_f16_test6(half %x, i32 %bool.arg1, i32 %bool.arg2) {
; GFX11-GISEL-FAKE16-LABEL: fmul_select_f16_test6:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v3, 0xc800
; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v3, 0x4200
; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v1, 0x4200, v3, vcc_lo
; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, v3, 0xc800, vcc_lo
; GFX11-GISEL-FAKE16-NEXT: v_mul_f16_e32 v0, v0, v1
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%bool = icmp eq i32 %bool.arg1, %bool.arg2
@ -2087,24 +2106,43 @@ define half @fmul_select_f16_test7(half %x, i32 %bool.arg1, i32 %bool.arg2) {
; GFX7-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: fmul_select_f16_test7:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v3, 0xc400
; GFX9-NEXT: v_mov_b32_e32 v4, 0x4800
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
; GFX9-NEXT: v_mul_f16_e32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-SDAG-LABEL: fmul_select_f16_test7:
; GFX9-SDAG: ; %bb.0:
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, 0xc400
; GFX9-SDAG-NEXT: v_mov_b32_e32 v4, 0x4800
; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
; GFX9-SDAG-NEXT: v_mul_f16_e32 v0, v0, v1
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: fmul_select_f16_test7:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v3, 0x4800
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
; GFX10-NEXT: v_cndmask_b32_e32 v1, 0xc400, v3, vcc_lo
; GFX10-NEXT: v_mul_f16_e32 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
; GFX9-GISEL-LABEL: fmul_select_f16_test7:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0x4800
; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, 0xc400
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
; GFX9-GISEL-NEXT: v_mul_f16_e32 v0, v0, v1
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SDAG-LABEL: fmul_select_f16_test7:
; GFX10-SDAG: ; %bb.0:
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: v_mov_b32_e32 v3, 0x4800
; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
; GFX10-SDAG-NEXT: v_cndmask_b32_e32 v1, 0xc400, v3, vcc_lo
; GFX10-SDAG-NEXT: v_mul_f16_e32 v0, v0, v1
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: fmul_select_f16_test7:
; GFX10-GISEL: ; %bb.0:
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, 0xc400
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, v3, 0x4800, vcc_lo
; GFX10-GISEL-NEXT: v_mul_f16_e32 v0, v0, v1
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: fmul_select_f16_test7:
; GFX11-SDAG-TRUE16: ; %bb.0:
@ -2139,10 +2177,10 @@ define half @fmul_select_f16_test7(half %x, i32 %bool.arg1, i32 %bool.arg2) {
; GFX11-GISEL-FAKE16-LABEL: fmul_select_f16_test7:
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v3, 0x4800
; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v3, 0xc400
; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v1, 0xc400, v3, vcc_lo
; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, v3, 0x4800, vcc_lo
; GFX11-GISEL-FAKE16-NEXT: v_mul_f16_e32 v0, v0, v1
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%bool = icmp eq i32 %bool.arg1, %bool.arg2
@ -2324,26 +2362,42 @@ define half @fmul_select_f16_test10_sel_log2val_neg11_pos11(half %x, i32 %bool.a
; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: fmul_select_f16_test10_sel_log2val_neg11_pos11:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
; GFX9-NEXT: v_cndmask_b32_e64 v1, 11, -11, vcc
; GFX9-NEXT: s_movk_i32 s4, 0x8000
; GFX9-NEXT: v_mov_b32_e32 v2, 0x7fff
; GFX9-NEXT: v_med3_i32 v1, v1, s4, v2
; GFX9-NEXT: v_ldexp_f16_e32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-SDAG-LABEL: fmul_select_f16_test10_sel_log2val_neg11_pos11:
; GFX9-SDAG: ; %bb.0:
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v1, 11, -11, vcc
; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x8000
; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0x7fff
; GFX9-SDAG-NEXT: v_med3_i32 v1, v1, s4, v2
; GFX9-SDAG-NEXT: v_ldexp_f16_e32 v0, v0, v1
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: fmul_select_f16_test10_sel_log2val_neg11_pos11:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
; GFX10-NEXT: s_movk_i32 s4, 0x8000
; GFX10-NEXT: v_cndmask_b32_e64 v1, 11, -11, vcc_lo
; GFX10-NEXT: v_med3_i32 v1, v1, s4, 0x7fff
; GFX10-NEXT: v_ldexp_f16_e32 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
; GFX9-GISEL-LABEL: fmul_select_f16_test10_sel_log2val_neg11_pos11:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, 11, -11, vcc
; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v0, v0, v1
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SDAG-LABEL: fmul_select_f16_test10_sel_log2val_neg11_pos11:
; GFX10-SDAG: ; %bb.0:
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
; GFX10-SDAG-NEXT: s_movk_i32 s4, 0x8000
; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v1, 11, -11, vcc_lo
; GFX10-SDAG-NEXT: v_med3_i32 v1, v1, s4, 0x7fff
; GFX10-SDAG-NEXT: v_ldexp_f16_e32 v0, v0, v1
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: fmul_select_f16_test10_sel_log2val_neg11_pos11:
; GFX10-GISEL: ; %bb.0:
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, 11, -11, vcc_lo
; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v0, v1
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: fmul_select_f16_test10_sel_log2val_neg11_pos11:
; GFX11-SDAG-TRUE16: ; %bb.0:
@ -2380,10 +2434,8 @@ define half @fmul_select_f16_test10_sel_log2val_neg11_pos11(half %x, i32 %bool.a
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
; GFX11-GISEL-FAKE16-NEXT: s_movk_i32 s0, 0x8000
; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, 11, -11, vcc_lo
; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v1, v1, s0, 0x7fff
; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v0, v0, v1
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%bool = icmp eq i32 %bool.arg1, %bool.arg2
@ -2403,26 +2455,42 @@ define half @fmul_select_f16_test11_sel_log2val_pos7_neg14(half %x, i32 %bool.ar
; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: fmul_select_f16_test11_sel_log2val_pos7_neg14:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
; GFX9-NEXT: v_cndmask_b32_e64 v1, -14, 7, vcc
; GFX9-NEXT: s_movk_i32 s4, 0x8000
; GFX9-NEXT: v_mov_b32_e32 v2, 0x7fff
; GFX9-NEXT: v_med3_i32 v1, v1, s4, v2
; GFX9-NEXT: v_ldexp_f16_e32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-SDAG-LABEL: fmul_select_f16_test11_sel_log2val_pos7_neg14:
; GFX9-SDAG: ; %bb.0:
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v1, -14, 7, vcc
; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x8000
; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0x7fff
; GFX9-SDAG-NEXT: v_med3_i32 v1, v1, s4, v2
; GFX9-SDAG-NEXT: v_ldexp_f16_e32 v0, v0, v1
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: fmul_select_f16_test11_sel_log2val_pos7_neg14:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
; GFX10-NEXT: s_movk_i32 s4, 0x8000
; GFX10-NEXT: v_cndmask_b32_e64 v1, -14, 7, vcc_lo
; GFX10-NEXT: v_med3_i32 v1, v1, s4, 0x7fff
; GFX10-NEXT: v_ldexp_f16_e32 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
; GFX9-GISEL-LABEL: fmul_select_f16_test11_sel_log2val_pos7_neg14:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, -14, 7, vcc
; GFX9-GISEL-NEXT: v_ldexp_f16_e32 v0, v0, v1
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SDAG-LABEL: fmul_select_f16_test11_sel_log2val_pos7_neg14:
; GFX10-SDAG: ; %bb.0:
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
; GFX10-SDAG-NEXT: s_movk_i32 s4, 0x8000
; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v1, -14, 7, vcc_lo
; GFX10-SDAG-NEXT: v_med3_i32 v1, v1, s4, 0x7fff
; GFX10-SDAG-NEXT: v_ldexp_f16_e32 v0, v0, v1
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: fmul_select_f16_test11_sel_log2val_pos7_neg14:
; GFX10-GISEL: ; %bb.0:
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, -14, 7, vcc_lo
; GFX10-GISEL-NEXT: v_ldexp_f16_e32 v0, v0, v1
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: fmul_select_f16_test11_sel_log2val_pos7_neg14:
; GFX11-SDAG-TRUE16: ; %bb.0:
@ -2459,10 +2527,8 @@ define half @fmul_select_f16_test11_sel_log2val_pos7_neg14(half %x, i32 %bool.ar
; GFX11-GISEL-FAKE16: ; %bb.0:
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
; GFX11-GISEL-FAKE16-NEXT: s_movk_i32 s0, 0x8000
; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, -14, 7, vcc_lo
; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-FAKE16-NEXT: v_med3_i32 v1, v1, s0, 0x7fff
; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-FAKE16-NEXT: v_ldexp_f16_e32 v0, v0, v1
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%bool = icmp eq i32 %bool.arg1, %bool.arg2

6
llvm/test/CodeGen/AMDGPU/diverge-interp-mov-lower.ll
View file

@ -1,9 +1,9 @@
; RUN: llc < %s -global-isel=0 -mtriple=amdgpu8.02--amdpal | FileCheck --check-prefix=GCN %s
; RUN: llc < %s -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.02--amdpal | FileCheck --check-prefix=GCN %s
; RUN: llc < %s -global-isel=1 -mtriple=amdgpu8.02--amdpal | FileCheck --check-prefix=GCN %s
; RUN: llc < %s -global-isel=0 -mtriple=amdgpu8.10--amdpal | FileCheck --check-prefix=GCN %s
; RUN: llc < %s -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.10--amdpal | FileCheck --check-prefix=GCN %s
; RUN: llc < %s -global-isel=1 -mtriple=amdgpu8.10--amdpal | FileCheck --check-prefix=GCN %s
; RUN: llc < %s -global-isel=0 -mtriple=amdgpu9.00--amdpal | FileCheck -check-prefixes=GCN,GFX9 %s
; RUN: llc < %s -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00--amdpal | FileCheck -check-prefixes=GCN,GFX9 %s
; RUN: llc < %s -global-isel=1 -mtriple=amdgpu9.00--amdpal | FileCheck -check-prefixes=GCN,GFX9 %s
; Testing for failures in divergence calculations when divergent intrinsic is lowered during instruction selection

2
llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
View file

@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -global-isel=0 -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -check-prefix=GFX9-SDAG %s
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -check-prefix=GFX9-GISEL %s
; RUN: llc -global-isel=1 -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -check-prefix=GFX9-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00-amd-amdhsa < %s | FileCheck -check-prefix=GFX11-SDAG %s
; RUN: llc -global-isel=1 -mtriple=amdgpu11.00-amd-amdhsa < %s | FileCheck -check-prefix=GFX11-GISEL %s

4
llvm/test/CodeGen/AMDGPU/fadd.f16.ll
View file

@ -2,9 +2,9 @@
; RUN: llc -mtriple=amdgpu6.00 -enable-misched=false < %s | FileCheck -check-prefixes=SI %s
; RUN: llc -mtriple=amdgpu8.03 -mattr=-flat-for-global -enable-misched=false < %s | FileCheck -check-prefixes=VI %s
; RUN: llc -mtriple=amdgpu11.00 -global-isel=0 -mattr=+real-true16,-flat-for-global -enable-misched=false < %s | FileCheck -check-prefixes=GFX11-SDAG %s
; RUN: llc -mtriple=amdgpu11.00 -global-isel=1 -global-isel-abort=2 -mattr=+real-true16,-flat-for-global -enable-misched=false < %s | FileCheck -check-prefixes=GFX11-GISEL %s
; RUN: llc -mtriple=amdgpu11.00 -global-isel=1 -mattr=+real-true16,-flat-for-global -enable-misched=false < %s | FileCheck -check-prefixes=GFX11-GISEL %s
; RUN: llc -mtriple=amdgpu11.00 -global-isel=0 -mattr=-real-true16,-flat-for-global -enable-misched=false < %s | FileCheck -check-prefixes=GFX11-FAKE16-SDAG %s
; RUN: llc -mtriple=amdgpu11.00 -global-isel=1 -global-isel-abort=2 -mattr=-real-true16,-flat-for-global -enable-misched=false < %s | FileCheck -check-prefixes=GFX11-FAKE16-GISEL %s
; RUN: llc -mtriple=amdgpu11.00 -global-isel=1 -mattr=-real-true16,-flat-for-global -enable-misched=false < %s | FileCheck -check-prefixes=GFX11-FAKE16-GISEL %s
define amdgpu_kernel void @fadd_f16(
; SI-LABEL: fadd_f16:

13
llvm/test/CodeGen/AMDGPU/flat-saddr-store.ll
View file

@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-mesa-mesa3d -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-SDAG,GFX1250-SDAG-FAKE16 %s
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.50-mesa-mesa3d -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL,GFX1250-GISEL-FAKE16 %s
; xUN: llc -global-isel=1 -mtriple=amdgpu12.50-mesa-mesa3d -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL,GFX1250-GISEL-FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-mesa-mesa3d -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-SDAG,GFX1250-SDAG-REAL16 %s
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.50-mesa-mesa3d -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL,GFX1250-GISEL-REAL16 %s
@ -1372,6 +1372,11 @@ define amdgpu_ps void @flat_store_saddr_i16_d16hi_zext_vgpr(ptr inreg %sbase, i3
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_store_d16_hi_b16 v0, v1, s[2:3]
; GFX1250-NEXT: s_endpgm
; GFX1250-GISEL-FAKE16-LABEL: flat_store_saddr_i16_d16hi_zext_vgpr:
; GFX1250-GISEL-FAKE16: ; %bb.0:
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: flat_store_d16_hi_b16 v0, v1, s[2:3]
; GFX1250-GISEL-FAKE16-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%data.hi = extractelement <2 x i16> %data, i32 1
@ -1387,6 +1392,11 @@ define amdgpu_ps void @flat_store_saddr_i16_d16hi_zext_vgpr_offset_neg128(ptr in
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_store_d16_hi_b16 v0, v1, s[2:3] offset:-128
; GFX1250-NEXT: s_endpgm
; GFX1250-GISEL-FAKE16-LABEL: flat_store_saddr_i16_d16hi_zext_vgpr_offset_neg128:
; GFX1250-GISEL-FAKE16: ; %bb.0:
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: flat_store_d16_hi_b16 v0, v1, s[2:3] offset:-128
; GFX1250-GISEL-FAKE16-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
@ -1428,7 +1438,6 @@ define amdgpu_ps void @flat_store_saddr_i16_d16hi_trunci8_zext_vgpr_offset_neg12
ret void
}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GFX1250-GISEL-FAKE16: {{.*}}
; GFX1250-GISEL-REAL16: {{.*}}
; GFX1250-SDAG-FAKE16: {{.*}}
; GFX1250-SDAG-REAL16: {{.*}}

144
llvm/test/CodeGen/AMDGPU/float-sopc-vopc.ll
View file

@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=amdgpu11.50 -global-isel=0 < %s | FileCheck -check-prefix=SDAG-GFX1150 %s
; RUN: llc -mtriple=amdgpu11.50 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefix=GISEL-GFX1150 %s
; RUN: llc -mtriple=amdgpu11.50 -global-isel=1 < %s | FileCheck -check-prefix=GISEL-GFX1150 %s
; RUN: llc -mtriple=amdgpu12.00 -global-isel=0 < %s | FileCheck -check-prefix=SDAG-GFX1200 %s
; RUN: llc -mtriple=amdgpu12.00 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefix=GISEL-GFX1200 %s
; RUN: llc -mtriple=amdgpu12.00 -global-isel=1 < %s | FileCheck -check-prefix=GISEL-GFX1200 %s
define amdgpu_vs void @f32_olt(ptr addrspace(1) inreg %out, float inreg %a, float inreg %b) {
; SDAG-GFX1150-LABEL: f32_olt:
@ -647,10 +647,11 @@ define amdgpu_vs void @f16_olt(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1150-LABEL: f16_olt:
; GISEL-GFX1150: ; %bb.0: ; %entry
; GISEL-GFX1150-NEXT: s_cmp_lt_f16 s2, s3
; GISEL-GFX1150-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1150-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-GFX1150-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GISEL-GFX1150-NEXT: global_store_b32 v0, v1, s[0:1]
; GISEL-GFX1150-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX1150-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1150-NEXT: s_endpgm
;
; SDAG-GFX1200-LABEL: f16_olt:
@ -665,10 +666,11 @@ define amdgpu_vs void @f16_olt(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1200-LABEL: f16_olt:
; GISEL-GFX1200: ; %bb.0: ; %entry
; GISEL-GFX1200-NEXT: s_cmp_lt_f16 s2, s3
; GISEL-GFX1200-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1200-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-GFX1200-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GISEL-GFX1200-NEXT: global_store_b32 v0, v1, s[0:1]
; GISEL-GFX1200-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX1200-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1200-NEXT: s_endpgm
entry:
%0 = fcmp olt half %a, %b
@ -690,10 +692,11 @@ define amdgpu_vs void @f16_oeq(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1150-LABEL: f16_oeq:
; GISEL-GFX1150: ; %bb.0: ; %entry
; GISEL-GFX1150-NEXT: s_cmp_eq_f16 s2, s3
; GISEL-GFX1150-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1150-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-GFX1150-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GISEL-GFX1150-NEXT: global_store_b32 v0, v1, s[0:1]
; GISEL-GFX1150-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX1150-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1150-NEXT: s_endpgm
;
; SDAG-GFX1200-LABEL: f16_oeq:
@ -708,10 +711,11 @@ define amdgpu_vs void @f16_oeq(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1200-LABEL: f16_oeq:
; GISEL-GFX1200: ; %bb.0: ; %entry
; GISEL-GFX1200-NEXT: s_cmp_eq_f16 s2, s3
; GISEL-GFX1200-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1200-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-GFX1200-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GISEL-GFX1200-NEXT: global_store_b32 v0, v1, s[0:1]
; GISEL-GFX1200-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX1200-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1200-NEXT: s_endpgm
entry:
%0 = fcmp oeq half %a, %b
@ -733,10 +737,11 @@ define amdgpu_vs void @f16_ole(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1150-LABEL: f16_ole:
; GISEL-GFX1150: ; %bb.0: ; %entry
; GISEL-GFX1150-NEXT: s_cmp_le_f16 s2, s3
; GISEL-GFX1150-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1150-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-GFX1150-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GISEL-GFX1150-NEXT: global_store_b32 v0, v1, s[0:1]
; GISEL-GFX1150-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX1150-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1150-NEXT: s_endpgm
;
; SDAG-GFX1200-LABEL: f16_ole:
@ -751,10 +756,11 @@ define amdgpu_vs void @f16_ole(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1200-LABEL: f16_ole:
; GISEL-GFX1200: ; %bb.0: ; %entry
; GISEL-GFX1200-NEXT: s_cmp_le_f16 s2, s3
; GISEL-GFX1200-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1200-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-GFX1200-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GISEL-GFX1200-NEXT: global_store_b32 v0, v1, s[0:1]
; GISEL-GFX1200-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX1200-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1200-NEXT: s_endpgm
entry:
%0 = fcmp ole half %a, %b
@ -776,10 +782,11 @@ define amdgpu_vs void @f16_ogt(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1150-LABEL: f16_ogt:
; GISEL-GFX1150: ; %bb.0: ; %entry
; GISEL-GFX1150-NEXT: s_cmp_gt_f16 s2, s3
; GISEL-GFX1150-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1150-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-GFX1150-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GISEL-GFX1150-NEXT: global_store_b32 v0, v1, s[0:1]
; GISEL-GFX1150-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX1150-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1150-NEXT: s_endpgm
;
; SDAG-GFX1200-LABEL: f16_ogt:
@ -794,10 +801,11 @@ define amdgpu_vs void @f16_ogt(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1200-LABEL: f16_ogt:
; GISEL-GFX1200: ; %bb.0: ; %entry
; GISEL-GFX1200-NEXT: s_cmp_gt_f16 s2, s3
; GISEL-GFX1200-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1200-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-GFX1200-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GISEL-GFX1200-NEXT: global_store_b32 v0, v1, s[0:1]
; GISEL-GFX1200-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX1200-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1200-NEXT: s_endpgm
entry:
%0 = fcmp ogt half %a, %b
@ -819,10 +827,11 @@ define amdgpu_vs void @f16_one(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1150-LABEL: f16_one:
; GISEL-GFX1150: ; %bb.0: ; %entry
; GISEL-GFX1150-NEXT: s_cmp_lg_f16 s2, s3
; GISEL-GFX1150-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1150-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-GFX1150-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GISEL-GFX1150-NEXT: global_store_b32 v0, v1, s[0:1]
; GISEL-GFX1150-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX1150-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1150-NEXT: s_endpgm
;
; SDAG-GFX1200-LABEL: f16_one:
@ -837,10 +846,11 @@ define amdgpu_vs void @f16_one(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1200-LABEL: f16_one:
; GISEL-GFX1200: ; %bb.0: ; %entry
; GISEL-GFX1200-NEXT: s_cmp_lg_f16 s2, s3
; GISEL-GFX1200-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1200-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-GFX1200-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GISEL-GFX1200-NEXT: global_store_b32 v0, v1, s[0:1]
; GISEL-GFX1200-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX1200-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1200-NEXT: s_endpgm
entry:
%0 = fcmp one half %a, %b
@ -862,10 +872,11 @@ define amdgpu_vs void @f16_oge(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1150-LABEL: f16_oge:
; GISEL-GFX1150: ; %bb.0: ; %entry
; GISEL-GFX1150-NEXT: s_cmp_ge_f16 s2, s3
; GISEL-GFX1150-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1150-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-GFX1150-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GISEL-GFX1150-NEXT: global_store_b32 v0, v1, s[0:1]
; GISEL-GFX1150-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX1150-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1150-NEXT: s_endpgm
;
; SDAG-GFX1200-LABEL: f16_oge:
@ -880,10 +891,11 @@ define amdgpu_vs void @f16_oge(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1200-LABEL: f16_oge:
; GISEL-GFX1200: ; %bb.0: ; %entry
; GISEL-GFX1200-NEXT: s_cmp_ge_f16 s2, s3
; GISEL-GFX1200-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1200-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-GFX1200-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GISEL-GFX1200-NEXT: global_store_b32 v0, v1, s[0:1]
; GISEL-GFX1200-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX1200-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1200-NEXT: s_endpgm
entry:
%0 = fcmp oge half %a, %b
@ -905,10 +917,11 @@ define amdgpu_vs void @f16_ord(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1150-LABEL: f16_ord:
; GISEL-GFX1150: ; %bb.0: ; %entry
; GISEL-GFX1150-NEXT: s_cmp_o_f16 s2, s3
; GISEL-GFX1150-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1150-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-GFX1150-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GISEL-GFX1150-NEXT: global_store_b32 v0, v1, s[0:1]
; GISEL-GFX1150-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX1150-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1150-NEXT: s_endpgm
;
; SDAG-GFX1200-LABEL: f16_ord:
@ -923,10 +936,11 @@ define amdgpu_vs void @f16_ord(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1200-LABEL: f16_ord:
; GISEL-GFX1200: ; %bb.0: ; %entry
; GISEL-GFX1200-NEXT: s_cmp_o_f16 s2, s3
; GISEL-GFX1200-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1200-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-GFX1200-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GISEL-GFX1200-NEXT: global_store_b32 v0, v1, s[0:1]
; GISEL-GFX1200-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX1200-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1200-NEXT: s_endpgm
entry:
%0 = fcmp ord half %a, %b
@ -948,10 +962,11 @@ define amdgpu_vs void @f16_uno(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1150-LABEL: f16_uno:
; GISEL-GFX1150: ; %bb.0: ; %entry
; GISEL-GFX1150-NEXT: s_cmp_u_f16 s2, s3
; GISEL-GFX1150-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1150-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-GFX1150-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GISEL-GFX1150-NEXT: global_store_b32 v0, v1, s[0:1]
; GISEL-GFX1150-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX1150-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1150-NEXT: s_endpgm
;
; SDAG-GFX1200-LABEL: f16_uno:
@ -966,10 +981,11 @@ define amdgpu_vs void @f16_uno(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1200-LABEL: f16_uno:
; GISEL-GFX1200: ; %bb.0: ; %entry
; GISEL-GFX1200-NEXT: s_cmp_u_f16 s2, s3
; GISEL-GFX1200-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1200-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-GFX1200-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GISEL-GFX1200-NEXT: global_store_b32 v0, v1, s[0:1]
; GISEL-GFX1200-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX1200-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1200-NEXT: s_endpgm
entry:
%0 = fcmp uno half %a, %b
@ -991,10 +1007,11 @@ define amdgpu_vs void @f16_ult(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1150-LABEL: f16_ult:
; GISEL-GFX1150: ; %bb.0: ; %entry
; GISEL-GFX1150-NEXT: s_cmp_nge_f16 s2, s3
; GISEL-GFX1150-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1150-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-GFX1150-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GISEL-GFX1150-NEXT: global_store_b32 v0, v1, s[0:1]
; GISEL-GFX1150-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX1150-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1150-NEXT: s_endpgm
;
; SDAG-GFX1200-LABEL: f16_ult:
@ -1009,10 +1026,11 @@ define amdgpu_vs void @f16_ult(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1200-LABEL: f16_ult:
; GISEL-GFX1200: ; %bb.0: ; %entry
; GISEL-GFX1200-NEXT: s_cmp_nge_f16 s2, s3
; GISEL-GFX1200-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1200-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-GFX1200-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GISEL-GFX1200-NEXT: global_store_b32 v0, v1, s[0:1]
; GISEL-GFX1200-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX1200-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1200-NEXT: s_endpgm
entry:
%0 = fcmp ult half %a, %b
@ -1034,10 +1052,11 @@ define amdgpu_vs void @f16_ueq(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1150-LABEL: f16_ueq:
; GISEL-GFX1150: ; %bb.0: ; %entry
; GISEL-GFX1150-NEXT: s_cmp_nlg_f16 s2, s3
; GISEL-GFX1150-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1150-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-GFX1150-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GISEL-GFX1150-NEXT: global_store_b32 v0, v1, s[0:1]
; GISEL-GFX1150-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX1150-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1150-NEXT: s_endpgm
;
; SDAG-GFX1200-LABEL: f16_ueq:
@ -1052,10 +1071,11 @@ define amdgpu_vs void @f16_ueq(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1200-LABEL: f16_ueq:
; GISEL-GFX1200: ; %bb.0: ; %entry
; GISEL-GFX1200-NEXT: s_cmp_nlg_f16 s2, s3
; GISEL-GFX1200-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1200-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-GFX1200-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GISEL-GFX1200-NEXT: global_store_b32 v0, v1, s[0:1]
; GISEL-GFX1200-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX1200-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1200-NEXT: s_endpgm
entry:
%0 = fcmp ueq half %a, %b
@ -1077,10 +1097,11 @@ define amdgpu_vs void @f16_ule(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1150-LABEL: f16_ule:
; GISEL-GFX1150: ; %bb.0: ; %entry
; GISEL-GFX1150-NEXT: s_cmp_ngt_f16 s2, s3
; GISEL-GFX1150-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1150-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-GFX1150-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GISEL-GFX1150-NEXT: global_store_b32 v0, v1, s[0:1]
; GISEL-GFX1150-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX1150-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1150-NEXT: s_endpgm
;
; SDAG-GFX1200-LABEL: f16_ule:
@ -1095,10 +1116,11 @@ define amdgpu_vs void @f16_ule(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1200-LABEL: f16_ule:
; GISEL-GFX1200: ; %bb.0: ; %entry
; GISEL-GFX1200-NEXT: s_cmp_ngt_f16 s2, s3
; GISEL-GFX1200-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1200-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-GFX1200-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GISEL-GFX1200-NEXT: global_store_b32 v0, v1, s[0:1]
; GISEL-GFX1200-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX1200-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1200-NEXT: s_endpgm
entry:
%0 = fcmp ule half %a, %b
@ -1120,10 +1142,11 @@ define amdgpu_vs void @f16_ugt(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1150-LABEL: f16_ugt:
; GISEL-GFX1150: ; %bb.0: ; %entry
; GISEL-GFX1150-NEXT: s_cmp_nle_f16 s2, s3
; GISEL-GFX1150-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1150-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-GFX1150-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GISEL-GFX1150-NEXT: global_store_b32 v0, v1, s[0:1]
; GISEL-GFX1150-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX1150-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1150-NEXT: s_endpgm
;
; SDAG-GFX1200-LABEL: f16_ugt:
@ -1138,10 +1161,11 @@ define amdgpu_vs void @f16_ugt(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1200-LABEL: f16_ugt:
; GISEL-GFX1200: ; %bb.0: ; %entry
; GISEL-GFX1200-NEXT: s_cmp_nle_f16 s2, s3
; GISEL-GFX1200-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1200-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-GFX1200-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GISEL-GFX1200-NEXT: global_store_b32 v0, v1, s[0:1]
; GISEL-GFX1200-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX1200-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1200-NEXT: s_endpgm
entry:
%0 = fcmp ugt half %a, %b
@ -1163,10 +1187,11 @@ define amdgpu_vs void @f16_une(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1150-LABEL: f16_une:
; GISEL-GFX1150: ; %bb.0: ; %entry
; GISEL-GFX1150-NEXT: s_cmp_neq_f16 s2, s3
; GISEL-GFX1150-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1150-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-GFX1150-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GISEL-GFX1150-NEXT: global_store_b32 v0, v1, s[0:1]
; GISEL-GFX1150-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX1150-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1150-NEXT: s_endpgm
;
; SDAG-GFX1200-LABEL: f16_une:
@ -1181,10 +1206,11 @@ define amdgpu_vs void @f16_une(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1200-LABEL: f16_une:
; GISEL-GFX1200: ; %bb.0: ; %entry
; GISEL-GFX1200-NEXT: s_cmp_neq_f16 s2, s3
; GISEL-GFX1200-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1200-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-GFX1200-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GISEL-GFX1200-NEXT: global_store_b32 v0, v1, s[0:1]
; GISEL-GFX1200-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX1200-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1200-NEXT: s_endpgm
entry:
%0 = fcmp une half %a, %b
@ -1206,10 +1232,11 @@ define amdgpu_vs void @f16_uge(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1150-LABEL: f16_uge:
; GISEL-GFX1150: ; %bb.0: ; %entry
; GISEL-GFX1150-NEXT: s_cmp_nlt_f16 s2, s3
; GISEL-GFX1150-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1150-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-GFX1150-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GISEL-GFX1150-NEXT: global_store_b32 v0, v1, s[0:1]
; GISEL-GFX1150-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX1150-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1150-NEXT: s_endpgm
;
; SDAG-GFX1200-LABEL: f16_uge:
@ -1224,10 +1251,11 @@ define amdgpu_vs void @f16_uge(ptr addrspace(1) inreg %out, half inreg %a, half
; GISEL-GFX1200-LABEL: f16_uge:
; GISEL-GFX1200: ; %bb.0: ; %entry
; GISEL-GFX1200-NEXT: s_cmp_nlt_f16 s2, s3
; GISEL-GFX1200-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX1200-NEXT: s_cselect_b32 s2, -1, 0
; GISEL-GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-GFX1200-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GISEL-GFX1200-NEXT: global_store_b32 v0, v1, s[0:1]
; GISEL-GFX1200-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX1200-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX1200-NEXT: s_endpgm
entry:
%0 = fcmp uge half %a, %b

143
llvm/test/CodeGen/AMDGPU/fma.f16.ll
View file

@ -178,12 +178,19 @@ define half @test_fmac(half %x, half %y, half %z) {
; GFX10+ has v_fmaak_f16.
define half @test_fmaak(half %x, half %y, half %z) {
; GFX9-LABEL: test_fmaak:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_movk_i32 s4, 0x4200
; GFX9-NEXT: v_fma_f16 v0, v0, v1, s4
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-SDAG-LABEL: test_fmaak:
; GFX9-SDAG: ; %bb.0:
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x4200
; GFX9-SDAG-NEXT: v_fma_f16 v0, v0, v1, s4
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-GISEL-LABEL: test_fmaak:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0x4200
; GFX9-GISEL-NEXT: v_fma_f16 v0, v0, v1, v2
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: test_fmaak:
; GFX10: ; %bb.0:
@ -260,12 +267,19 @@ define half @test_fmaak(half %x, half %y, half %z) {
; GFX10+ has v_fmamk_f16.
define half @test_fmamk(half %x, half %y, half %z) {
; GFX9-LABEL: test_fmamk:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_movk_i32 s4, 0x4200
; GFX9-NEXT: v_fma_f16 v0, v0, s4, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-SDAG-LABEL: test_fmamk:
; GFX9-SDAG: ; %bb.0:
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x4200
; GFX9-SDAG-NEXT: v_fma_f16 v0, v0, s4, v2
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-GISEL-LABEL: test_fmamk:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0x4200
; GFX9-GISEL-NEXT: v_fma_f16 v0, v0, v1, v2
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: test_fmamk:
; GFX10: ; %bb.0:
@ -342,28 +356,53 @@ define half @test_fmamk(half %x, half %y, half %z) {
; Regression test for a crash caused by D139469.
define i32 @test_D139469_f16(half %arg) {
; GFX9-LABEL: test_D139469_f16:
; GFX9: ; %bb.0: ; %bb
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_movk_i32 s4, 0x291e
; GFX9-NEXT: v_mov_b32_e32 v2, 0x211e
; GFX9-NEXT: v_mul_f16_e32 v1, 0x291e, v0
; GFX9-NEXT: v_fma_f16 v0, v0, s4, v2
; GFX9-NEXT: v_min_f16_e32 v0, v1, v0
; GFX9-NEXT: v_cmp_gt_f16_e32 vcc, 0, v0
; GFX9-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
; GFX9-NEXT: s_setpc_b64 s[30:31]
; GFX9-SDAG-LABEL: test_D139469_f16:
; GFX9-SDAG: ; %bb.0: ; %bb
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x291e
; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0x211e
; GFX9-SDAG-NEXT: v_mul_f16_e32 v1, 0x291e, v0
; GFX9-SDAG-NEXT: v_fma_f16 v0, v0, s4, v2
; GFX9-SDAG-NEXT: v_min_f16_e32 v0, v1, v0
; GFX9-SDAG-NEXT: v_cmp_gt_f16_e32 vcc, 0, v0
; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: test_D139469_f16:
; GFX10: ; %bb.0: ; %bb
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_movk_i32 s4, 0x291e
; GFX10-NEXT: v_mul_f16_e32 v1, 0x291e, v0
; GFX10-NEXT: v_fmaak_f16 v0, s4, v0, 0x211e
; GFX10-NEXT: v_min_f16_e32 v0, v1, v0
; GFX10-NEXT: v_cmp_gt_f16_e32 vcc_lo, 0, v0
; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
; GFX10-NEXT: s_setpc_b64 s[30:31]
; GFX9-GISEL-LABEL: test_D139469_f16:
; GFX9-GISEL: ; %bb.0: ; %bb
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_mul_f16_e32 v2, 0x291e, v0
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0x291e
; GFX9-GISEL-NEXT: v_cmp_gt_f16_e32 vcc, 0, v2
; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0x211e
; GFX9-GISEL-NEXT: v_fma_f16 v0, v0, v1, v2
; GFX9-GISEL-NEXT: v_cmp_gt_f16_e64 s[4:5], 0, v0
; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SDAG-LABEL: test_D139469_f16:
; GFX10-SDAG: ; %bb.0: ; %bb
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: s_movk_i32 s4, 0x291e
; GFX10-SDAG-NEXT: v_mul_f16_e32 v1, 0x291e, v0
; GFX10-SDAG-NEXT: v_fmaak_f16 v0, s4, v0, 0x211e
; GFX10-SDAG-NEXT: v_min_f16_e32 v0, v1, v0
; GFX10-SDAG-NEXT: v_cmp_gt_f16_e32 vcc_lo, 0, v0
; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: test_D139469_f16:
; GFX10-GISEL: ; %bb.0: ; %bb
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0x211e
; GFX10-GISEL-NEXT: v_mul_f16_e32 v2, 0x291e, v0
; GFX10-GISEL-NEXT: v_fmac_f16_e32 v1, 0x291e, v0
; GFX10-GISEL-NEXT: v_cmp_gt_f16_e32 vcc_lo, 0, v2
; GFX10-GISEL-NEXT: v_cmp_gt_f16_e64 s4, 0, v1
; GFX10-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, s4
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_D139469_f16:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
@ -408,13 +447,15 @@ define i32 @test_D139469_f16(half %arg) {
; GFX11-GISEL-FAKE16-LABEL: test_D139469_f16:
; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-FAKE16-NEXT: s_movk_i32 s0, 0x291e
; GFX11-GISEL-FAKE16-NEXT: v_mul_f16_e32 v1, 0x291e, v0
; GFX11-GISEL-FAKE16-NEXT: v_fmaak_f16 v0, s0, v0, 0x211e
; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-FAKE16-NEXT: v_min_f16_e32 v0, v1, v0
; GFX11-GISEL-FAKE16-NEXT: v_cmp_gt_f16_e32 vcc_lo, 0, v0
; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0x211e
; GFX11-GISEL-FAKE16-NEXT: v_mul_f16_e32 v2, 0x291e, v0
; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-GISEL-FAKE16-NEXT: v_fmac_f16_e32 v1, 0x291e, v0
; GFX11-GISEL-FAKE16-NEXT: v_cmp_gt_f16_e32 vcc_lo, 0, v2
; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-GISEL-FAKE16-NEXT: v_cmp_gt_f16_e64 s0, 0, v1
; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_D139469_f16:
@ -480,15 +521,16 @@ define i32 @test_D139469_f16(half %arg) {
; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_movk_i32 s0, 0x291e
; GFX12-GISEL-FAKE16-NEXT: v_mul_f16_e32 v1, 0x291e, v0
; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0x211e
; GFX12-GISEL-FAKE16-NEXT: v_mul_f16_e32 v2, 0x291e, v0
; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-GISEL-FAKE16-NEXT: v_fmac_f16_e32 v1, 0x291e, v0
; GFX12-GISEL-FAKE16-NEXT: v_cmp_gt_f16_e32 vcc_lo, 0, v2
; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-GISEL-FAKE16-NEXT: v_cmp_gt_f16_e64 s0, 0, v1
; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-GISEL-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-FAKE16-NEXT: v_fmaak_f16 v0, s0, v0, 0x211e
; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-GISEL-FAKE16-NEXT: v_min_num_f16_e32 v0, v1, v0
; GFX12-GISEL-FAKE16-NEXT: v_cmp_gt_f16_e32 vcc_lo, 0, v0
; GFX12-GISEL-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
; GFX12-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
bb:
%i = fmul contract half %arg, 0xH291E
@ -698,8 +740,3 @@ bb:
%i5 = zext <2 x i1> %i4 to <2 x i32>
ret <2 x i32> %i5
}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GFX10-GISEL: {{.*}}
; GFX10-SDAG: {{.*}}
; GFX9-GISEL: {{.*}}
; GFX9-SDAG: {{.*}}

Some files were not shown because too many files have changed in this diff Show more