Cold-Air-Networks/mirror-llvm-project
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 7

[SCEV] Canonicalise round-up idiom when some bits known (#197126)

Browse source 
Since #174380, instcombine can clear some set bits in the added constant
in expressions like this, when A has some known-clear low order bits.
  (A + 15) & ~15

This transformation is valid, but can make it harder for later passes to
recognise this idiom for rounding up to a power of 2. This is causing
the ARM MVE tail predication pass to fail on loops with a trip count
which is a multiple of a small power of 2.

The fix is to reverse the transformation when building SCEV expressions,
canonicalising to always use the largest valid value for the added
constant.

Alive proofs:
https://alive2.llvm.org/ce/z/hhndoW
https://alive2.llvm.org/ce/z/_JYVat
This commit is contained in:
Oliver Stannard 2026-05-28 14:29:09 +01:00 committed by GitHub
parent ee3709b5db
commit da4e5cc928
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 214 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
llvm/lib/Analysis/ScalarEvolution.cpp
View file

@ -3718,6 +3718,26 @@ const SCEV *ScalarEvolution::getUDivExpr(SCEVUse LHS, SCEVUse RHS) {
}
}
// ((N - M) + (M * A)) / N --> ((N - 1) + (M * A)) / N
// This is an idiom for rounding A up to the next multiple of N, where A
// is aready known to be a multiple of M. In this case, instcombine can
// see that some low bits of the added constant are unused, so can clear
// them, but we want to canonicalise to set the low bits. This makes the
// pattern easier to match, without needing to check for known bits in
// A*M.
const APInt &N = RHSC->getAPInt();
const APInt *NMinusM, *M;
const SCEV *A;
if (match(LHS, m_scev_Add(m_scev_APInt(NMinusM),
m_scev_Mul(m_scev_APInt(M), m_SCEV(A))))) {
if (N.isPowerOf2() && M->isPowerOf2() && M->ult(N) &&
*NMinusM == N - *M) {
return getUDivExpr(
getAddExpr(getConstant(N - 1), getMulExpr(getConstant(*M), A)),
RHS);
}
}
// Fold if both operands are constant.
if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(LHS))
return getConstant(LHSC->getAPInt().udiv(RHSC->getAPInt()));

117
llvm/test/Analysis/ScalarEvolution/rounding-demanded-bits.ll Normal file
View file

@ -0,0 +1,117 @@
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
; RUN: opt < %s "-passes=print<scalar-evolution>" -disable-output 2>&1 | FileCheck %s
; Check that, when constructing a SCEV for rounding a value up to the nearest
; multiple of a constant, we always use that largest possible value for the
; added constant, even if instcombine has cleared some bits in the IR because
; the input is already known to be a multiple of some smaller value.
define i32 @mul_2_round_to_16_a(i32 %val) {
; CHECK-LABEL: 'mul_2_round_to_16_a'
; CHECK-NEXT: Classifying expressions for: @mul_2_round_to_16_a
; CHECK-NEXT: %mul = mul i32 %val, 2
; CHECK-NEXT: --> (2 * %val) U: [0,-1) S: [-2147483648,2147483647)
; CHECK-NEXT: %add = add i32 %mul, 15
; CHECK-NEXT: --> (15 + (2 * %val)) U: [15,14) S: [-2147483633,-2147483634)
; CHECK-NEXT: %round = and i32 %add, -16
; CHECK-NEXT: --> (16 * ((15 + (2 * %val)) /u 16))<nuw> U: [0,-15) S: [-2147483648,2147483633)
; CHECK-NEXT: Determining loop execution counts for: @mul_2_round_to_16_a
;
%mul = mul i32 %val, 2
%add = add i32 %mul, 15
%round = and i32 %add, -16
ret i32 %round
}
define i32 @mul_2_round_to_16_b(i32 %val) {
; CHECK-LABEL: 'mul_2_round_to_16_b'
; CHECK-NEXT: Classifying expressions for: @mul_2_round_to_16_b
; CHECK-NEXT: %mul = mul i32 %val, 2
; CHECK-NEXT: --> (2 * %val) U: [0,-1) S: [-2147483648,2147483647)
; CHECK-NEXT: %add = add i32 %mul, 14
; CHECK-NEXT: --> (14 + (2 * %val)) U: [0,-1) S: [-2147483648,2147483647)
; CHECK-NEXT: %round = and i32 %add, -16
; CHECK-NEXT: --> (16 * ((15 + (2 * %val)) /u 16))<nuw> U: [0,-15) S: [-2147483648,2147483633)
; CHECK-NEXT: Determining loop execution counts for: @mul_2_round_to_16_b
;
%mul = mul i32 %val, 2
%add = add i32 %mul, 14
%round = and i32 %add, -16
ret i32 %round
}
define i32 @mul_2_round_to_16_c(i32 %val) {
; CHECK-LABEL: 'mul_2_round_to_16_c'
; CHECK-NEXT: Classifying expressions for: @mul_2_round_to_16_c
; CHECK-NEXT: %mul = mul i32 %val, 2
; CHECK-NEXT: --> (2 * %val) U: [0,-1) S: [-2147483648,2147483647)
; CHECK-NEXT: %add = add i32 %mul, 14
; CHECK-NEXT: --> (14 + (2 * %val)) U: [0,-1) S: [-2147483648,2147483647)
; CHECK-NEXT: %div = udiv i32 %add, 16
; CHECK-NEXT: --> ((15 + (2 * %val)) /u 16) U: [0,268435456) S: [0,268435456)
; CHECK-NEXT: %round = mul i32 %div, 16
; CHECK-NEXT: --> (16 * ((15 + (2 * %val)) /u 16))<nuw> U: [0,-15) S: [-2147483648,2147483633)
; CHECK-NEXT: Determining loop execution counts for: @mul_2_round_to_16_c
;
%mul = mul i32 %val, 2
%add = add i32 %mul, 14
%div = udiv i32 %add, 16
%round = mul i32 %div, 16
ret i32 %round
}
define i32 @mul_4_round_to_16(i32 %val) {
; CHECK-LABEL: 'mul_4_round_to_16'
; CHECK-NEXT: Classifying expressions for: @mul_4_round_to_16
; CHECK-NEXT: %mul = mul i32 %val, 4
; CHECK-NEXT: --> (4 * %val) U: [0,-3) S: [-2147483648,2147483645)
; CHECK-NEXT: %add = add i32 %mul, 12
; CHECK-NEXT: --> (12 + (4 * %val)) U: [0,-3) S: [-2147483648,2147483645)
; CHECK-NEXT: %round = and i32 %add, -16
; CHECK-NEXT: --> (16 * ((15 + (4 * %val)) /u 16))<nuw> U: [0,-15) S: [-2147483648,2147483633)
; CHECK-NEXT: Determining loop execution counts for: @mul_4_round_to_16
;
%mul = mul i32 %val, 4
%add = add i32 %mul, 12
%round = and i32 %add, -16
ret i32 %round
}
; N is not a power of 2, cannot do transformation
define i32 @invalid1(i32 %val) {
; CHECK-LABEL: 'invalid1'
; CHECK-NEXT: Classifying expressions for: @invalid1
; CHECK-NEXT: %mul = mul i32 %val, 3
; CHECK-NEXT: --> (3 * %val) U: full-set S: full-set
; CHECK-NEXT: %add = add i32 %mul, 7
; CHECK-NEXT: --> (7 + (3 * %val)) U: full-set S: full-set
; CHECK-NEXT: %div = udiv i32 %add, 9
; CHECK-NEXT: --> ((7 + (3 * %val)) /u 9) U: [0,477218589) S: [0,477218589)
; CHECK-NEXT: %round = mul i32 %div, 9
; CHECK-NEXT: --> (9 * ((7 + (3 * %val)) /u 9))<nuw> U: [0,-3) S: [0,-3)
; CHECK-NEXT: Determining loop execution counts for: @invalid1
;
%mul = mul i32 %val, 3
%add = add i32 %mul, 7
%div = udiv i32 %add, 9
%round = mul i32 %div, 9
ret i32 %round
}
; M is greater then N, cannot do transformation
define i32 @invalid2(i32 %val) {
; CHECK-LABEL: 'invalid2'
; CHECK-NEXT: Classifying expressions for: @invalid2
; CHECK-NEXT: %mul = mul i32 %val, 4
; CHECK-NEXT: --> (4 * %val) U: [0,-3) S: [-2147483648,2147483645)
; CHECK-NEXT: %add = add i32 %mul, 1
; CHECK-NEXT: --> (1 + (4 * %val))<nuw><nsw> U: [1,-2) S: [-2147483647,2147483646)
; CHECK-NEXT: %round = and i32 %add, -2
; CHECK-NEXT: --> (2 * ((1 + (4 * %val))<nuw><nsw> /u 2))<nuw> U: [0,-3) S: [-2147483648,2147483647)
; CHECK-NEXT: Determining loop execution counts for: @invalid2
;
%mul = mul i32 %val, 4
%add = add i32 %mul, 1
%round = and i32 %add, -2
ret i32 %round
}

77
llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-known-bits.ll Normal file
View file

@ -0,0 +1,77 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -mtriple=thumbv8.1m.main -mve-tail-predication -tail-predication=enabled -mattr=+mve %s -S -o - | FileCheck %s
; Check that tail predication still triggers correctly when the back-edge taken
; count is known to be a multiple of 2.
; Function Attrs: nofree norecurse nosync nounwind memory(argmem: read)
define dso_local i32 @example(ptr noundef readonly captures(none) %ptr, i32 noundef %n) {
; CHECK-LABEL: define dso_local i32 @example(
; CHECK-SAME: ptr noundef readonly captures(none) [[PTR:%.*]], i32 noundef [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[CMP3:%.*]] = icmp sgt i32 [[N]], 0
; CHECK-NEXT: br i1 [[CMP3]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
; CHECK: [[FOR_BODY_PREHEADER]]:
; CHECK-NEXT: [[MUL:%.*]] = shl i32 [[N]], 1
; CHECK-NEXT: [[N_RND_UP:%.*]] = add i32 [[MUL]], 14
; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -16
; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N_VEC]], -16
; CHECK-NEXT: [[TMP1:%.*]] = lshr i32 [[TMP0]], 4
; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i32 [[TMP1]], 1
; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.start.loop.iterations.i32(i32 [[TMP2]])
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[LSR_IV:%.*]] = phi ptr [ [[SCEVGEP:%.*]], %[[VECTOR_BODY]] ], [ [[PTR]], %[[FOR_BODY_PREHEADER]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[FOR_BODY_PREHEADER]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP4:%.*]] = phi i32 [ [[TMP3]], %[[FOR_BODY_PREHEADER]] ], [ [[TMP12:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP5:%.*]] = phi i32 [ [[MUL]], %[[FOR_BODY_PREHEADER]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP6:%.*]] = call <16 x i1> @llvm.arm.mve.vctp8(i32 [[TMP5]])
; CHECK-NEXT: [[TMP7]] = sub i32 [[TMP5]], 16
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = tail call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[LSR_IV]], <16 x i1> [[TMP6]], <16 x i8> poison)
; CHECK-NEXT: [[TMP8:%.*]] = zext <16 x i8> [[WIDE_MASKED_LOAD]] to <16 x i32>
; CHECK-NEXT: [[TMP9:%.*]] = select <16 x i1> [[TMP6]], <16 x i32> [[TMP8]], <16 x i32> zeroinitializer
; CHECK-NEXT: [[TMP10:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP9]])
; CHECK-NEXT: [[TMP11]] = add i32 [[VEC_PHI]], [[TMP10]]
; CHECK-NEXT: [[SCEVGEP]] = getelementptr i8, ptr [[LSR_IV]], i32 16
; CHECK-NEXT: [[TMP12]] = call i32 @llvm.loop.decrement.reg.i32(i32 [[TMP4]], i32 1)
; CHECK-NEXT: [[TMP13:%.*]] = icmp ne i32 [[TMP12]], 0
; CHECK-NEXT: br i1 [[TMP13]], label %[[VECTOR_BODY]], label %[[FOR_COND_CLEANUP]]
; CHECK: [[FOR_COND_CLEANUP]]:
; CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[TMP11]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: ret i32 [[SUM_0_LCSSA]]
;
entry:
%cmp3 = icmp sgt i32 %n, 0
br i1 %cmp3, label %for.body.preheader, label %for.cond.cleanup
for.body.preheader: ; preds = %entry
%mul = shl nuw i32 %n, 1
%n.rnd.up = add i32 %mul, 14
%n.vec = and i32 %n.rnd.up, -16
%0 = add i32 %n.vec, -16
%1 = lshr i32 %0, 4
%2 = add nuw nsw i32 %1, 1
%3 = call i32 @llvm.start.loop.iterations.i32(i32 %2)
br label %vector.body
vector.body: ; preds = %vector.body, %for.body.preheader
%lsr.iv = phi ptr [ %scevgep, %vector.body ], [ %ptr, %for.body.preheader ]
%index = phi i32 [ 0, %for.body.preheader ], [ %index.next, %vector.body ]
%vec.phi = phi i32 [ 0, %for.body.preheader ], [ %8, %vector.body ]
%4 = phi i32 [ %3, %for.body.preheader ], [ %9, %vector.body ]
%active.lane.mask = tail call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %index, i32 %mul)
%wide.masked.load = tail call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 %lsr.iv, <16 x i1> %active.lane.mask, <16 x i8> poison)
%5 = zext <16 x i8> %wide.masked.load to <16 x i32>
%6 = select <16 x i1> %active.lane.mask, <16 x i32> %5, <16 x i32> zeroinitializer
%7 = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %6)
%8 = add i32 %vec.phi, %7
%index.next = add i32 %index, 16
%scevgep = getelementptr i8, ptr %lsr.iv, i32 16
%9 = call i32 @llvm.loop.decrement.reg.i32(i32 %4, i32 1)
%10 = icmp ne i32 %9, 0
br i1 %10, label %vector.body, label %for.cond.cleanup
for.cond.cleanup: ; preds = %vector.body, %entry
%sum.0.lcssa = phi i32 [ 0, %entry ], [ %8, %vector.body ]
ret i32 %sum.0.lcssa
}