From 36187ce08a65a476f87896174231eea6c8d0e528 Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Fri, 31 Jul 2026 17:38:16 -0400 Subject: [PATCH] [SLP][NFC]Add an extra check for fmul to fmuladd promotion, NFC Reviewers: Pull Request: https://github.com/llvm/llvm-project/pull/213361 --- .../X86/fmuladd-copyable-fmul.ll | 138 ++++++++++++++++++ 1 file changed, 138 insertions(+) diff --git a/llvm/test/Transforms/SLPVectorizer/X86/fmuladd-copyable-fmul.ll b/llvm/test/Transforms/SLPVectorizer/X86/fmuladd-copyable-fmul.ll index 3614f8b1f1a9..28ca0b93a06f 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/fmuladd-copyable-fmul.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/fmuladd-copyable-fmul.ll @@ -1,6 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 ; RUN: opt -passes=slp-vectorizer -slp-threshold=-99999 -S -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s --check-prefixes=ENABLED ; RUN: opt -passes=slp-vectorizer -slp-threshold=-99999 -slp-copyable-elements=false -S -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s --check-prefixes=DISABLED +; RUN: opt -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s --check-prefixes=COST declare float @llvm.fmuladd.f32(float, float, float) @@ -59,6 +60,27 @@ define <4 x float> @buildvec_fmul_absorb(float %p, float %q, float %r, float %s, ; DISABLED-NEXT: [[V31:%.*]] = shufflevector <4 x float> [[TMP19]], <4 x float> [[TMP20]], <4 x i32> ; DISABLED-NEXT: ret <4 x float> [[V31]] ; +; COST-LABEL: define <4 x float> @buildvec_fmul_absorb( +; COST-SAME: float [[P:%.*]], float [[Q:%.*]], float [[R:%.*]], float [[S:%.*]], float [[T:%.*]], float [[U:%.*]], float [[N:%.*]], float [[M:%.*]], float [[E:%.*]], float [[F:%.*]], float [[G:%.*]], float [[H:%.*]], ptr [[SRCB:%.*]]) { +; COST-NEXT: [[ENTRY:.*:]] +; COST-NEXT: [[TMP0:%.*]] = insertelement <2 x float> poison, float [[P]], i64 0 +; COST-NEXT: [[TMP1:%.*]] = insertelement <2 x float> [[TMP0]], float [[R]], i64 1 +; COST-NEXT: [[TMP2:%.*]] = insertelement <2 x float> poison, float [[Q]], i64 0 +; COST-NEXT: [[TMP3:%.*]] = insertelement <2 x float> [[TMP2]], float [[S]], i64 1 +; COST-NEXT: [[TMP4:%.*]] = fmul <2 x float> [[TMP1]], [[TMP3]] +; COST-NEXT: [[X:%.*]] = fmul float [[T]], [[U]] +; COST-NEXT: [[Z:%.*]] = fmul float [[N]], [[M]] +; COST-NEXT: [[C0:%.*]] = fadd float [[E]], [[F]] +; COST-NEXT: [[C1:%.*]] = fsub float [[G]], [[H]] +; COST-NEXT: [[TMP5:%.*]] = load <2 x float>, ptr [[SRCB]], align 4 +; COST-NEXT: [[TMP6:%.*]] = insertelement <2 x float> poison, float [[C0]], i64 0 +; COST-NEXT: [[TMP7:%.*]] = insertelement <2 x float> [[TMP6]], float [[C1]], i64 1 +; COST-NEXT: [[TMP8:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP4]], <2 x float> [[TMP5]], <2 x float> [[TMP7]]) +; COST-NEXT: [[TMP9:%.*]] = shufflevector <2 x float> [[TMP8]], <2 x float> poison, <4 x i32> +; COST-NEXT: [[V2:%.*]] = insertelement <4 x float> [[TMP9]], float [[X]], i32 2 +; COST-NEXT: [[V3:%.*]] = insertelement <4 x float> [[V2]], float [[Z]], i32 3 +; COST-NEXT: ret <4 x float> [[V3]] +; entry: %a0 = fmul float %p, %q %a1 = fmul float %r, %s @@ -146,6 +168,33 @@ define void @storechain_1fmul(ptr %dst, ptr %srcB, float %p, float %q, float %r, ; DISABLED-NEXT: store float [[FMA2]], ptr [[D3]], align 4 ; DISABLED-NEXT: ret void ; +; COST-LABEL: define void @storechain_1fmul( +; COST-SAME: ptr [[DST:%.*]], ptr [[SRCB:%.*]], float [[P:%.*]], float [[Q:%.*]], float [[R:%.*]], float [[S:%.*]], float [[T:%.*]], float [[U:%.*]], float [[N:%.*]], float [[M:%.*]], float [[E:%.*]], float [[F:%.*]], float [[G:%.*]], float [[H:%.*]], float [[I:%.*]], float [[J:%.*]], float [[K:%.*]], float [[L:%.*]]) { +; COST-NEXT: [[ENTRY:.*:]] +; COST-NEXT: [[TMP0:%.*]] = insertelement <2 x float> poison, float [[P]], i64 0 +; COST-NEXT: [[TMP1:%.*]] = insertelement <2 x float> [[TMP0]], float [[R]], i64 1 +; COST-NEXT: [[TMP2:%.*]] = insertelement <2 x float> poison, float [[Q]], i64 0 +; COST-NEXT: [[TMP3:%.*]] = insertelement <2 x float> [[TMP2]], float [[S]], i64 1 +; COST-NEXT: [[TMP4:%.*]] = fmul <2 x float> [[TMP1]], [[TMP3]] +; COST-NEXT: [[A2:%.*]] = fmul float [[I]], [[J]] +; COST-NEXT: [[X:%.*]] = fmul float [[T]], [[U]] +; COST-NEXT: [[C0:%.*]] = fadd float [[E]], [[F]] +; COST-NEXT: [[C1:%.*]] = fsub float [[G]], [[H]] +; COST-NEXT: [[C2:%.*]] = fadd float [[K]], [[L]] +; COST-NEXT: [[GEPB2:%.*]] = getelementptr float, ptr [[SRCB]], i32 2 +; COST-NEXT: [[B2:%.*]] = load float, ptr [[GEPB2]], align 4 +; COST-NEXT: [[FMA2:%.*]] = call float @llvm.fmuladd.f32(float [[A2]], float [[B2]], float [[C2]]) +; COST-NEXT: [[D1:%.*]] = getelementptr float, ptr [[DST]], i32 1 +; COST-NEXT: [[TMP5:%.*]] = load <2 x float>, ptr [[SRCB]], align 4 +; COST-NEXT: [[TMP6:%.*]] = insertelement <2 x float> poison, float [[C0]], i64 0 +; COST-NEXT: [[TMP7:%.*]] = insertelement <2 x float> [[TMP6]], float [[C1]], i64 1 +; COST-NEXT: [[TMP8:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP4]], <2 x float> [[TMP5]], <2 x float> [[TMP7]]) +; COST-NEXT: store float [[X]], ptr [[DST]], align 4 +; COST-NEXT: store <2 x float> [[TMP8]], ptr [[D1]], align 4 +; COST-NEXT: [[D3:%.*]] = getelementptr float, ptr [[DST]], i32 3 +; COST-NEXT: store float [[FMA2]], ptr [[D3]], align 4 +; COST-NEXT: ret void +; entry: %a0 = fmul float %p, %q %a1 = fmul float %r, %s @@ -231,6 +280,28 @@ define <4 x float> @buildvec_fmul_absorb_rev(float %p, float %q, float %r, float ; DISABLED-NEXT: [[V31:%.*]] = shufflevector <4 x float> [[TMP19]], <4 x float> [[TMP20]], <4 x i32> ; DISABLED-NEXT: ret <4 x float> [[V31]] ; +; COST-LABEL: define <4 x float> @buildvec_fmul_absorb_rev( +; COST-SAME: float [[P:%.*]], float [[Q:%.*]], float [[R:%.*]], float [[S:%.*]], float [[T:%.*]], float [[U:%.*]], float [[N:%.*]], float [[M:%.*]], float [[E:%.*]], float [[F:%.*]], float [[G:%.*]], float [[H:%.*]], ptr [[SRCB:%.*]]) { +; COST-NEXT: [[ENTRY:.*:]] +; COST-NEXT: [[TMP0:%.*]] = insertelement <2 x float> poison, float [[P]], i64 0 +; COST-NEXT: [[TMP1:%.*]] = insertelement <2 x float> [[TMP0]], float [[R]], i64 1 +; COST-NEXT: [[TMP2:%.*]] = insertelement <2 x float> poison, float [[Q]], i64 0 +; COST-NEXT: [[TMP3:%.*]] = insertelement <2 x float> [[TMP2]], float [[S]], i64 1 +; COST-NEXT: [[TMP4:%.*]] = fmul <2 x float> [[TMP1]], [[TMP3]] +; COST-NEXT: [[X:%.*]] = fmul float [[T]], [[U]] +; COST-NEXT: [[Z:%.*]] = fmul float [[N]], [[M]] +; COST-NEXT: [[C0:%.*]] = fadd float [[E]], [[F]] +; COST-NEXT: [[C1:%.*]] = fsub float [[G]], [[H]] +; COST-NEXT: [[TMP5:%.*]] = load <2 x float>, ptr [[SRCB]], align 4 +; COST-NEXT: [[TMP6:%.*]] = insertelement <2 x float> poison, float [[C0]], i64 0 +; COST-NEXT: [[TMP7:%.*]] = insertelement <2 x float> [[TMP6]], float [[C1]], i64 1 +; COST-NEXT: [[TMP8:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP4]], <2 x float> [[TMP5]], <2 x float> [[TMP7]]) +; COST-NEXT: [[V0:%.*]] = insertelement <4 x float> poison, float [[X]], i32 0 +; COST-NEXT: [[V1:%.*]] = insertelement <4 x float> [[V0]], float [[Z]], i32 1 +; COST-NEXT: [[TMP9:%.*]] = shufflevector <2 x float> [[TMP8]], <2 x float> poison, <4 x i32> +; COST-NEXT: [[V31:%.*]] = shufflevector <4 x float> [[V1]], <4 x float> [[TMP9]], <4 x i32> +; COST-NEXT: ret <4 x float> [[V31]] +; entry: %a0 = fmul float %p, %q %a1 = fmul float %r, %s @@ -317,6 +388,30 @@ define <4 x float> @buildvec_fmul_no_absorb_multiuse_rev(float %p, float %q, flo ; DISABLED-NEXT: [[V31:%.*]] = shufflevector <4 x float> [[TMP21]], <4 x float> [[TMP22]], <4 x i32> ; DISABLED-NEXT: ret <4 x float> [[V31]] ; +; COST-LABEL: define <4 x float> @buildvec_fmul_no_absorb_multiuse_rev( +; COST-SAME: float [[P:%.*]], float [[Q:%.*]], float [[R:%.*]], float [[S:%.*]], float [[T:%.*]], float [[U:%.*]], float [[N:%.*]], float [[M:%.*]], float [[E:%.*]], float [[F:%.*]], float [[G:%.*]], float [[H:%.*]], ptr [[SRCB:%.*]], ptr [[DST2:%.*]]) { +; COST-NEXT: [[ENTRY:.*:]] +; COST-NEXT: [[TMP0:%.*]] = insertelement <2 x float> poison, float [[P]], i64 0 +; COST-NEXT: [[TMP1:%.*]] = insertelement <2 x float> [[TMP0]], float [[R]], i64 1 +; COST-NEXT: [[TMP2:%.*]] = insertelement <2 x float> poison, float [[Q]], i64 0 +; COST-NEXT: [[TMP3:%.*]] = insertelement <2 x float> [[TMP2]], float [[S]], i64 1 +; COST-NEXT: [[TMP4:%.*]] = fmul <2 x float> [[TMP1]], [[TMP3]] +; COST-NEXT: [[X:%.*]] = fmul float [[T]], [[U]] +; COST-NEXT: [[Z:%.*]] = fmul float [[N]], [[M]] +; COST-NEXT: [[C0:%.*]] = fadd float [[E]], [[F]] +; COST-NEXT: [[C1:%.*]] = fsub float [[G]], [[H]] +; COST-NEXT: [[TMP5:%.*]] = load <2 x float>, ptr [[SRCB]], align 4 +; COST-NEXT: [[TMP6:%.*]] = insertelement <2 x float> poison, float [[C0]], i64 0 +; COST-NEXT: [[TMP7:%.*]] = insertelement <2 x float> [[TMP6]], float [[C1]], i64 1 +; COST-NEXT: [[TMP8:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP4]], <2 x float> [[TMP5]], <2 x float> [[TMP7]]) +; COST-NEXT: store float [[X]], ptr [[DST2]], align 4 +; COST-NEXT: store float [[Z]], ptr [[DST2]], align 4 +; COST-NEXT: [[V0:%.*]] = insertelement <4 x float> poison, float [[X]], i32 0 +; COST-NEXT: [[V1:%.*]] = insertelement <4 x float> [[V0]], float [[Z]], i32 1 +; COST-NEXT: [[TMP9:%.*]] = shufflevector <2 x float> [[TMP8]], <2 x float> poison, <4 x i32> +; COST-NEXT: [[V31:%.*]] = shufflevector <4 x float> [[V1]], <4 x float> [[TMP9]], <4 x i32> +; COST-NEXT: ret <4 x float> [[V31]] +; entry: %a0 = fmul float %p, %q %a1 = fmul float %r, %s @@ -391,6 +486,26 @@ define <4 x float> @buildvec_fmul_absorb_const(float %p, float %q, float %r, flo ; DISABLED-NEXT: [[V31:%.*]] = shufflevector <4 x float> [[TMP17]], <4 x float> [[TMP18]], <4 x i32> ; DISABLED-NEXT: ret <4 x float> [[V31]] ; +; COST-LABEL: define <4 x float> @buildvec_fmul_absorb_const( +; COST-SAME: float [[P:%.*]], float [[Q:%.*]], float [[R:%.*]], float [[S:%.*]], float [[T:%.*]], float [[U:%.*]], float [[E:%.*]], float [[F:%.*]], float [[G:%.*]], float [[H:%.*]], ptr [[SRCB:%.*]]) { +; COST-NEXT: [[ENTRY:.*:]] +; COST-NEXT: [[A0:%.*]] = fmul float [[P]], [[Q]] +; COST-NEXT: [[A1:%.*]] = fmul float [[R]], [[S]] +; COST-NEXT: [[C0:%.*]] = fadd float [[E]], [[F]] +; COST-NEXT: [[C1:%.*]] = fsub float [[G]], [[H]] +; COST-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[SRCB]], align 4 +; COST-NEXT: [[TMP1:%.*]] = insertelement <4 x float> poison, float [[U]], i64 2 +; COST-NEXT: [[TMP2:%.*]] = insertelement <4 x float> [[TMP1]], float [[A0]], i64 0 +; COST-NEXT: [[TMP3:%.*]] = insertelement <4 x float> [[TMP2]], float [[A1]], i64 1 +; COST-NEXT: [[TMP4:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> poison, <4 x i32> +; COST-NEXT: [[TMP5:%.*]] = shufflevector <2 x float> [[TMP0]], <2 x float> poison, <4 x i32> +; COST-NEXT: [[TMP6:%.*]] = shufflevector <4 x float> , <4 x float> [[TMP5]], <4 x i32> +; COST-NEXT: [[TMP7:%.*]] = insertelement <4 x float> , float [[C0]], i64 0 +; COST-NEXT: [[TMP8:%.*]] = insertelement <4 x float> [[TMP7]], float [[C1]], i64 1 +; COST-NEXT: [[TMP9:%.*]] = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> [[TMP4]], <4 x float> [[TMP6]], <4 x float> [[TMP8]]) +; COST-NEXT: [[TMP10:%.*]] = fmul <4 x float> [[TMP9]], +; COST-NEXT: ret <4 x float> [[TMP10]] +; entry: %a0 = fmul float %p, %q %a1 = fmul float %r, %s @@ -473,6 +588,29 @@ define <4 x float> @buildvec_fmul_no_absorb_multiuse(float %p, float %q, float % ; DISABLED-NEXT: [[V31:%.*]] = shufflevector <4 x float> [[TMP21]], <4 x float> [[TMP22]], <4 x i32> ; DISABLED-NEXT: ret <4 x float> [[V31]] ; +; COST-LABEL: define <4 x float> @buildvec_fmul_no_absorb_multiuse( +; COST-SAME: float [[P:%.*]], float [[Q:%.*]], float [[R:%.*]], float [[S:%.*]], float [[T:%.*]], float [[U:%.*]], float [[N:%.*]], float [[M:%.*]], float [[E:%.*]], float [[F:%.*]], float [[G:%.*]], float [[H:%.*]], ptr [[SRCB:%.*]], ptr [[DST2:%.*]]) { +; COST-NEXT: [[ENTRY:.*:]] +; COST-NEXT: [[TMP0:%.*]] = insertelement <2 x float> poison, float [[P]], i64 0 +; COST-NEXT: [[TMP1:%.*]] = insertelement <2 x float> [[TMP0]], float [[R]], i64 1 +; COST-NEXT: [[TMP2:%.*]] = insertelement <2 x float> poison, float [[Q]], i64 0 +; COST-NEXT: [[TMP3:%.*]] = insertelement <2 x float> [[TMP2]], float [[S]], i64 1 +; COST-NEXT: [[TMP4:%.*]] = fmul <2 x float> [[TMP1]], [[TMP3]] +; COST-NEXT: [[X:%.*]] = fmul float [[T]], [[U]] +; COST-NEXT: [[Z:%.*]] = fmul float [[N]], [[M]] +; COST-NEXT: [[C0:%.*]] = fadd float [[E]], [[F]] +; COST-NEXT: [[C1:%.*]] = fsub float [[G]], [[H]] +; COST-NEXT: [[TMP5:%.*]] = load <2 x float>, ptr [[SRCB]], align 4 +; COST-NEXT: [[TMP6:%.*]] = insertelement <2 x float> poison, float [[C0]], i64 0 +; COST-NEXT: [[TMP7:%.*]] = insertelement <2 x float> [[TMP6]], float [[C1]], i64 1 +; COST-NEXT: [[TMP8:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP4]], <2 x float> [[TMP5]], <2 x float> [[TMP7]]) +; COST-NEXT: store float [[X]], ptr [[DST2]], align 4 +; COST-NEXT: store float [[Z]], ptr [[DST2]], align 4 +; COST-NEXT: [[TMP9:%.*]] = shufflevector <2 x float> [[TMP8]], <2 x float> poison, <4 x i32> +; COST-NEXT: [[V2:%.*]] = insertelement <4 x float> [[TMP9]], float [[X]], i32 2 +; COST-NEXT: [[V3:%.*]] = insertelement <4 x float> [[V2]], float [[Z]], i32 3 +; COST-NEXT: ret <4 x float> [[V3]] +; entry: %a0 = fmul float %p, %q %a1 = fmul float %r, %s