Remove old sync-upstream.yml, superseded by Cold-Air-Networks/mirror-sync

This path transforms `vector.contract(A, B, Acc)` into
`vector.contract(A, B, 0) + Acc` to decouple the contraction computation
from the accumulator update for contractions over the loop.

compiler-rt/lib/builtins/CMakeLists.txt

@@ -288,7 +288,6 @@ if(COMPILER_RT_USE_LIBC_MATH)
   use_libc_builtin(GENERIC_SOURCES    adddf3)
   use_libc_builtin(GENERIC_SOURCES    addsf3)
   use_libc_builtin(GENERIC_TF_SOURCES addtf3)
-  use_libc_builtin(GENERIC_SOURCES    comparesf2)
   use_libc_builtin(GENERIC_SOURCES    divdf3)
   use_libc_builtin(GENERIC_SOURCES    divsf3)
   use_libc_builtin(GENERIC_TF_SOURCES divtf3)

compiler-rt/lib/builtins/comparesf2.cpp

@@ -1,54 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-///
-/// \file
-/// This file implements compiler-rt's float comparison routines
-/// (__lesf2/__gesf2/__unordsf2 and their aliases) on top of LLVM-libc's shared
-/// comparison builtins.
-///
-//===----------------------------------------------------------------------===//
-
-#define SINGLE_PRECISION
-#include "fp_lib.h"
-
-#include "fp_compare_impl.inc"
-#include "fp_libc_config.h"
-#include "shared/builtins/gesf2.h"
-#include "shared/builtins/lesf2.h"
-#include "shared/builtins/unordsf2.h"
-
-extern "C" {
-
-COMPILER_RT_ABI CMP_RESULT __lesf2(fp_t a, fp_t b) {
-  return LIBC_NAMESPACE::shared::lesf2(a, b);
-}
-#if defined(__ELF__)
-COMPILER_RT_ALIAS(__lesf2, __cmpsf2)
-#endif
-COMPILER_RT_ALIAS(__lesf2, __eqsf2)
-COMPILER_RT_ALIAS(__lesf2, __ltsf2)
-COMPILER_RT_ALIAS(__lesf2, __nesf2)
-
-COMPILER_RT_ABI CMP_RESULT __gesf2(fp_t a, fp_t b) {
-  return LIBC_NAMESPACE::shared::gesf2(a, b);
-}
-COMPILER_RT_ALIAS(__gesf2, __gtsf2)
-
-COMPILER_RT_ABI CMP_RESULT __unordsf2(fp_t a, fp_t b) {
-  return LIBC_NAMESPACE::shared::unordsf2(a, b);
-}
-
-#if defined(__ARM_EABI__)
-#if defined(COMPILER_RT_ARMHF_TARGET)
-AEABI_RTABI int __aeabi_fcmpun(fp_t a, fp_t b) { return __unordsf2(a, b); }
-#else
-COMPILER_RT_ALIAS(__unordsf2, __aeabi_fcmpun)
-#endif
-#endif
-
-} // extern "C"

libc/shared/builtins.h

@@ -27,8 +27,6 @@
 #include "builtins/extendsfdf2.h"
 #include "builtins/extendsftf2.h"
 #include "builtins/extendxftf2.h"
-#include "builtins/gesf2.h"
-#include "builtins/lesf2.h"
 #include "builtins/muldf3.h"
 #include "builtins/mulsf3.h"
 #include "builtins/multf3.h"
@@ -41,6 +39,5 @@
 #include "builtins/trunctfdf2.h"
 #include "builtins/trunctfsf2.h"
 #include "builtins/trunctfxf2.h"
-#include "builtins/unordsf2.h"
 
 #endif // LLVM_LIBC_SHARED_BUILTINS_H

libc/shared/builtins/gesf2.h

@@ -1,29 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-///
-/// \file
-/// This header exposes LLVM-libc's __gesf2 implementation as shared::gesf2 so
-/// that it can be reused by compiler-rt's builtins.
-///
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_SHARED_BUILTINS_GESF2_H
-#define LLVM_LIBC_SHARED_BUILTINS_GESF2_H
-
-#include "shared/libc_common.h"
-#include "src/__support/builtins/gesf2.h"
-
-namespace LIBC_NAMESPACE_DECL {
-namespace shared {
-
-using builtins::gesf2;
-
-} // namespace shared
-} // namespace LIBC_NAMESPACE_DECL
-
-#endif // LLVM_LIBC_SHARED_BUILTINS_GESF2_H

libc/shared/builtins/lesf2.h

@@ -1,29 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-///
-/// \file
-/// This header exposes LLVM-libc's __lesf2 implementation as shared::lesf2 so
-/// that it can be reused by compiler-rt's builtins.
-///
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_SHARED_BUILTINS_LESF2_H
-#define LLVM_LIBC_SHARED_BUILTINS_LESF2_H
-
-#include "shared/libc_common.h"
-#include "src/__support/builtins/lesf2.h"
-
-namespace LIBC_NAMESPACE_DECL {
-namespace shared {
-
-using builtins::lesf2;
-
-} // namespace shared
-} // namespace LIBC_NAMESPACE_DECL
-
-#endif // LLVM_LIBC_SHARED_BUILTINS_LESF2_H

libc/shared/builtins/unordsf2.h

@@ -1,29 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-///
-/// \file
-/// This header exposes LLVM-libc's __unordsf2 implementation as
-/// shared::unordsf2 so that it can be reused by compiler-rt's builtins.
-///
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_SHARED_BUILTINS_UNORDSF2_H
-#define LLVM_LIBC_SHARED_BUILTINS_UNORDSF2_H
-
-#include "shared/libc_common.h"
-#include "src/__support/builtins/unordsf2.h"
-
-namespace LIBC_NAMESPACE_DECL {
-namespace shared {
-
-using builtins::unordsf2;
-
-} // namespace shared
-} // namespace LIBC_NAMESPACE_DECL
-
-#endif // LLVM_LIBC_SHARED_BUILTINS_UNORDSF2_H

libc/src/__support/builtins/CMakeLists.txt

@@ -221,39 +221,5 @@ add_header_library(
     truncdfsf2.h
   DEPENDS
     libc.src.__support.builtins.fpconvert_helper
-  cmp_helper
-  HDRS
-    cmp_helper.h
-  DEPENDS
-    libc.src.__support.FPUtil.comparison_operations
-    libc.src.__support.FPUtil.fp_bits
-    libc.src.__support.macros.attributes
-    libc.src.__support.macros.config
-)
-
-add_header_library(
-  lesf2
-  HDRS
-    lesf2.h
-  DEPENDS
-    libc.src.__support.builtins.cmp_helper
-    libc.src.__support.macros.config
-)
-
-add_header_library(
-  gesf2
-  HDRS
-    gesf2.h
-  DEPENDS
-    libc.src.__support.builtins.cmp_helper
-    libc.src.__support.macros.config
-)
-
-add_header_library(
-  unordsf2
-  HDRS
-    unordsf2.h
-  DEPENDS
-    libc.src.__support.builtins.cmp_helper
     libc.src.__support.macros.config
 )

libc/src/__support/builtins/cmp_helper.h

@@ -1,63 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-///
-/// \file
-/// Shared floating-point comparison results for compiler-rt's soft-float
-/// compare builtins.  These mirror compiler-rt's GCC-compatible __le<f>2 /
-/// __ge<f>2 / __unord<f>2 (quiet: they raise no FP exceptions), reusing
-/// LLVM-libc's IEEE comparison predicates, so they can be reused by
-/// compiler-rt's builtins.
-///
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_SRC___SUPPORT_BUILTINS_CMP_HELPER_H
-#define LLVM_LIBC_SRC___SUPPORT_BUILTINS_CMP_HELPER_H
-
-#include "src/__support/FPUtil/FPBits.h"
-#include "src/__support/FPUtil/comparison_operations.h"
-#include "src/__support/macros/attributes.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-namespace builtins {
-
-// GCC soft-float compare, "LE" convention: -1 if a<b, 0 if a==b, 1 if a>b, and
-// 1 when unordered (either operand is NaN).  Quiet -- raises no FP exceptions,
-// matching compiler-rt's __le<f>2 (which __eq/__ne/__lt/__cmp alias).  NaN is
-// handled up front so the ordering predicates are only reached for non-NaN
-// inputs, where they never signal.
-template <typename T> LIBC_INLINE int cmp_le(T a, T b) {
-  using FPBits = fputil::FPBits<T>;
-  if (FPBits(a).is_nan() || FPBits(b).is_nan())
-    return 1; // LE_UNORDERED
-  if (fputil::equals(a, b))
-    return 0;                              // LE_EQUAL
-  return fputil::less_than(a, b) ? -1 : 1; // LE_LESS : LE_GREATER
-}
-
-// Same as cmp_le, but unordered yields -1; matches compiler-rt's __ge<f>2
-// (which __gt aliases).
-template <typename T> LIBC_INLINE int cmp_ge(T a, T b) {
-  using FPBits = fputil::FPBits<T>;
-  if (FPBits(a).is_nan() || FPBits(b).is_nan())
-    return -1; // GE_UNORDERED
-  if (fputil::equals(a, b))
-    return 0;                                 // GE_EQUAL
-  return fputil::greater_than(a, b) ? 1 : -1; // GE_GREATER : GE_LESS
-}
-
-// 1 iff either operand is NaN, else 0; matches compiler-rt's __unord<f>2.
-template <typename T> LIBC_INLINE int cmp_unord(T a, T b) {
-  using FPBits = fputil::FPBits<T>;
-  return FPBits(a).is_nan() || FPBits(b).is_nan();
-}
-
-} // namespace builtins
-} // namespace LIBC_NAMESPACE_DECL
-
-#endif // LLVM_LIBC_SRC___SUPPORT_BUILTINS_CMP_HELPER_H

libc/src/__support/builtins/gesf2.h

@@ -1,30 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-///
-/// \file
-/// This header exposes LLVM-libc's __gesf2 implementation as builtins::gesf2 so
-/// that it can be reused by compiler-rt's builtins.
-///
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_SRC___SUPPORT_BUILTINS_GESF2_H
-#define LLVM_LIBC_SRC___SUPPORT_BUILTINS_GESF2_H
-
-#include "src/__support/builtins/cmp_helper.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-namespace builtins {
-
-// Greater-equal comparison of float; mirrors compiler-rt's __gesf2.
-LIBC_INLINE int gesf2(float a, float b) { return cmp_ge(a, b); }
-
-} // namespace builtins
-} // namespace LIBC_NAMESPACE_DECL
-
-#endif // LLVM_LIBC_SRC___SUPPORT_BUILTINS_GESF2_H

libc/src/__support/builtins/lesf2.h

@@ -1,30 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-///
-/// \file
-/// This header exposes LLVM-libc's __lesf2 implementation as builtins::lesf2 so
-/// that it can be reused by compiler-rt's builtins.
-///
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_SRC___SUPPORT_BUILTINS_LESF2_H
-#define LLVM_LIBC_SRC___SUPPORT_BUILTINS_LESF2_H
-
-#include "src/__support/builtins/cmp_helper.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-namespace builtins {
-
-// Less-equal comparison of float; mirrors compiler-rt's __lesf2.
-LIBC_INLINE int lesf2(float a, float b) { return cmp_le(a, b); }
-
-} // namespace builtins
-} // namespace LIBC_NAMESPACE_DECL
-
-#endif // LLVM_LIBC_SRC___SUPPORT_BUILTINS_LESF2_H

libc/src/__support/builtins/unordsf2.h

@@ -1,30 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-///
-/// \file
-/// This header exposes LLVM-libc's __unordsf2 implementation as
-/// builtins::unordsf2 so that it can be reused by compiler-rt's builtins.
-///
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_SRC___SUPPORT_BUILTINS_UNORDSF2_H
-#define LLVM_LIBC_SRC___SUPPORT_BUILTINS_UNORDSF2_H
-
-#include "src/__support/builtins/cmp_helper.h"
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-namespace builtins {
-
-// Unordered comparison of float; mirrors compiler-rt's __unordsf2.
-LIBC_INLINE int unordsf2(float a, float b) { return cmp_unord(a, b); }
-
-} // namespace builtins
-} // namespace LIBC_NAMESPACE_DECL
-
-#endif // LLVM_LIBC_SRC___SUPPORT_BUILTINS_UNORDSF2_H

libc/src/__support/common.h

@@ -17,6 +17,12 @@
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/properties/architectures.h"
 #include "src/__support/macros/properties/compiler.h"
+#include "src/__support/macros/properties/os.h"
+
+#if defined(__APPLE__) || (defined(LIBC_TARGET_OS_IS_WINDOWS) &&               \
+                           defined(LIBC_TARGET_ARCH_IS_X86_32))
+#define LIBC_TARGET_USES_LEADING_UNDERSCORE
+#endif
 
 #ifndef LLVM_LIBC_FUNCTION_ATTR
 #define LLVM_LIBC_FUNCTION_ATTR
@@ -44,25 +50,24 @@
 
 #define LLVM_LIBC_ATTR(name) EXPAND_THEN_SECOND(LLVM_LIBC_FUNCTION_ATTR_##name)
 
-// At the moment, [[gnu::alias()]] is not supported on MacOS, and it is needed
-// to cleanly export and alias the C++ symbol `LIBC_NAMESPACE::func` with the C
-// symbol `func`.  So for public packaging on MacOS, we will only export the C
-// symbol.  Moreover, a C symbol `func` in macOS is mangled as `_func`.
+// Export both `func` and `LIBC_NAMESPACE::func` using an alias symbol.
+// This does not work on platfors with LIBC_TARGET_USES_LEADING_UNDERSCORE
+// so there this only exports `_func`.
 #if defined(LIBC_COPT_PUBLIC_PACKAGING) && !defined(LIBC_COMPILER_IS_MSVC)
-#ifndef __APPLE__
+#ifndef LIBC_TARGET_USES_LEADING_UNDERSCORE
 #define LLVM_LIBC_FUNCTION_IMPL_4(type, name, arglist, c_alias)                \
   LLVM_LIBC_ATTR(name)                                                         \
   LLVM_LIBC_FUNCTION_ATTR decltype(LIBC_NAMESPACE::name)                       \
       __##name##_impl__ asm(c_alias);                                          \
   decltype(LIBC_NAMESPACE::name) name [[gnu::alias(c_alias)]];                 \
   type __##name##_impl__ arglist
-#else // __APPLE__
+#else // LIBC_TARGET_USES_LEADING_UNDERSCORE
 #define LLVM_LIBC_FUNCTION_IMPL_4(type, name, arglist, c_alias)                \
   LLVM_LIBC_ATTR(name)                                                         \
   LLVM_LIBC_FUNCTION_ATTR decltype(LIBC_NAMESPACE::name) name asm(             \
       "_" c_alias);                                                            \
   type name arglist
-#endif // __APPLE__
+#endif // LIBC_TARGET_USES_LEADING_UNDERSCORE
 
 #else  // LIBC_COPT_PUBLIC_PACKAGING
 #define LLVM_LIBC_FUNCTION_IMPL_4(type, name, arglist, c_alias)                \
@@ -78,25 +83,23 @@
   GET_FIFTH(__VA_ARGS__, LLVM_LIBC_FUNCTION_IMPL_4, LLVM_LIBC_FUNCTION_IMPL_3, \
             GET_NOTHING)(__VA_ARGS__)
 
-// At the moment, [[gnu::alias()]] is not supported on MacOS, and it is needed
-// to cleanly export and alias the C++ symbol `LIBC_NAMESPACE::func` with the C
-// symbol `func`.  So for public packaging on MacOS, we will only export the C
-// symbol.  Moreover, a C symbol `func` in macOS is mangled as `_func`.
+// See comment on LLVM_LIBC_FUNCTION_IMPL_4 for why this checks
+// LIBC_TARGET_USES_LEADING_UNDERSCORE.
 #if defined(LIBC_COPT_PUBLIC_PACKAGING) && !defined(LIBC_COMPILER_IS_MSVC)
-#ifndef __APPLE__
+#ifndef LIBC_TARGET_USES_LEADING_UNDERSCORE
 #define LLVM_LIBC_VARIABLE_IMPL(type, name)                                    \
   LLVM_LIBC_ATTR(name)                                                         \
   extern LLVM_LIBC_VARIABLE_ATTR decltype(LIBC_NAMESPACE::name)                \
       __##name##_impl__ asm(#name);                                            \
   extern decltype(LIBC_NAMESPACE::name) name [[gnu::alias(#name)]];            \
   type __##name##_impl__
-#else // __APPLE__
+#else // LIBC_TARGET_USES_LEADING_UNDERSCORE
 #define LLVM_LIBC_VARIABLE_IMPL(type, name)                                    \
   LLVM_LIBC_ATTR(name)                                                         \
   extern LLVM_LIBC_VARIABLE_ATTR decltype(LIBC_NAMESPACE::name) name asm(      \
       "_" #name);                                                              \
   type name
-#endif // __APPLE__
+#endif // LIBC_TARGET_USES_LEADING_UNDERSCORE
 #else  // LIBC_COPT_PUBLIC_PACKAGING
 #define LLVM_LIBC_VARIABLE_IMPL(type, name) type name
 #endif // LIBC_COPT_PUBLIC_PACKAGING

libc/src/__support/math/expm1.h

@@ -126,13 +126,13 @@ LIBC_INLINE DoubleDouble poly_approx_dd(const DoubleDouble &dx) {
 }
 
 #ifdef DEBUGDEBUG
-std::ostream &operator<<(std::ostream &OS, const DFloat128 &r) {
+LIBC_INLINE std::ostream &operator<<(std::ostream &OS, const DFloat128 &r) {
   OS << (r.sign == Sign::NEG ? "-(" : "(") << r.mantissa.val[0] << " + "
      << r.mantissa.val[1] << " * 2^64) * 2^" << r.exponent << "\n";
   return OS;
 }
 
-std::ostream &operator<<(std::ostream &OS, const DoubleDouble &r) {
+LIBC_INLINE std::ostream &operator<<(std::ostream &OS, const DoubleDouble &r) {
   OS << std::hexfloat << "(" << r.hi << " + " << r.lo << ")"
      << std::defaultfloat << "\n";
   return OS;

libc/src/__support/math/tan.h

@@ -108,8 +108,9 @@ LIBC_INLINE double tan_eval(const DoubleDouble &u, DoubleDouble &result) {
 // Calculation a / b = a * (1/b) for DFloat128.
 // Using the initial approximation of q ~ (1/b), then apply 2 Newton-Raphson
 // iterations, before multiplying by a.
-[[maybe_unused]] DFloat128 newton_raphson_div(const DFloat128 &a, DFloat128 b,
-                                              double q) {
+[[maybe_unused]] LIBC_INLINE DFloat128 newton_raphson_div(const DFloat128 &a,
+                                                          DFloat128 b,
+                                                          double q) {
   DFloat128 q0(q);
   LIBC_BIT_CAST_CONSTEXPR_VAR DFloat128 TWO(2.0);
   b.sign = (b.sign == Sign::POS) ? Sign::NEG : Sign::POS;

libc/test/shared/CMakeLists.txt

@@ -838,8 +838,6 @@ add_fp_unittest(
     libc.src.__support.builtins.extendsfdf2
     libc.src.__support.builtins.extendsftf2
     libc.src.__support.builtins.extendxftf2
-    libc.src.__support.builtins.gesf2
-    libc.src.__support.builtins.lesf2
     libc.src.__support.builtins.muldf3
     libc.src.__support.builtins.mulsf3
     libc.src.__support.builtins.multf3
@@ -852,7 +850,6 @@ add_fp_unittest(
     libc.src.__support.builtins.trunctfdf2
     libc.src.__support.builtins.trunctfsf2
     libc.src.__support.builtins.trunctfxf2
-    libc.src.__support.builtins.unordsf2
 )
 
 add_fp_unittest(

libc/test/shared/shared_builtins_test.cpp

@@ -60,18 +60,3 @@ TEST(LlvmLibcSharedBuiltinsTest, TruncateConversion) {
 #endif // LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80
 #endif // LIBC_TYPES_HAS_FLOAT128
 }
-
-TEST(LlvmLibcSharedBuiltinsTest, SingleCompare) {
-  const float aNaN =
-      LIBC_NAMESPACE::fputil::FPBits<float>::quiet_nan().get_val();
-  EXPECT_EQ(-1, shared::gesf2(1.0f, 2.0f));
-  EXPECT_EQ(0, shared::gesf2(1.0f, 1.0f));
-  EXPECT_EQ(1, shared::gesf2(2.0f, 1.0f));
-  EXPECT_EQ(-1, shared::gesf2(aNaN, 1.0f));
-  EXPECT_EQ(-1, shared::lesf2(1.0f, 2.0f));
-  EXPECT_EQ(0, shared::lesf2(1.0f, 1.0f));
-  EXPECT_EQ(1, shared::lesf2(2.0f, 1.0f));
-  EXPECT_EQ(1, shared::lesf2(aNaN, 1.0f));
-  EXPECT_EQ(0, shared::unordsf2(1.0f, 2.0f));
-  EXPECT_EQ(1, shared::unordsf2(aNaN, 1.0f));
-}

lld/ELF/Arch/SPARCV9.cpp

@@ -166,6 +166,20 @@ void SPARCV9::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels,
       expr = R_GOT_OFF;
       break;
 
+    // Optimize the GOT load to an add of the symbol's GOT-relative address if
+    // applicable. Exclude absolute symbol, which can be arbitrarily far. %l7
+    // points at .got, which must be retained.
+    case R_SPARC_GOTDATA_OP_HIX22:
+    case R_SPARC_GOTDATA_OP_LOX10:
+    case R_SPARC_GOTDATA_OP:
+      if (sym.isPreemptible || isAbsolute(sym)) {
+        expr = R_GOT_OFF;
+      } else {
+        ctx.in.got->hasGotOffRel.store(true, std::memory_order_relaxed);
+        expr = R_GOTREL;
+      }
+      break;
+
     // TLS LE relocations:
     case R_SPARC_TLS_LE_HIX22:
     case R_SPARC_TLS_LE_LOX10:
@@ -322,6 +336,24 @@ void SPARCV9::relocate(uint8_t *loc, const Relocation &rel,
     // T-simm13
     write32be(loc, (read32be(loc) & ~0x00001fff) | (val & 0x000003ff) | 0x1C00);
     break;
+  case R_SPARC_GOTDATA_OP_HIX22: {
+    // V-imm22. sethi encodes the complement of a negative value, which the
+    // paired xor undoes, so the encodable range is 33 bits.
+    checkInt(ctx, loc, val, 33, rel);
+    uint64_t v = int64_t(val) < 0 ? ~val : val;
+    write32be(loc, (read32be(loc) & ~0x003fffff) | ((v >> 10) & 0x003fffff));
+    break;
+  }
+  case R_SPARC_GOTDATA_OP_LOX10:
+    // T-simm13. Only a negative value needs the sign extension bits.
+    write32be(loc, (read32be(loc) & ~0x00001fff) | (val & 0x000003ff) |
+                       (int64_t(val) < 0 ? 0x1c00 : 0));
+    break;
+  case R_SPARC_GOTDATA_OP:
+    // ldx [%rs1 + %rs2], %rd -> add %rs1, %rs2, %rd
+    if (rel.expr == R_GOTREL)
+      write32be(loc, (read32be(loc) & 0x3e07c01f) | 0x80000000);
+    break;
   default:
     llvm_unreachable("unknown relocation");
   }

lld/MachO/InputSection.cpp

@@ -204,16 +204,20 @@ void ConcatInputSection::foldIdentical(ConcatInputSection *copy,
   for (auto &copySym : copy->symbols)
     copySym->identicalCodeFoldingKind = foldKind;
 
-  symbols.insert(symbols.end(), copy->symbols.begin(), copy->symbols.end());
-  copy->symbols.clear();
-
-  // Remove duplicate compact unwind info for symbols at the same address.
-  if (symbols.empty())
+  if (copy->symbols.empty())
     return;
-  for (auto it = symbols.begin() + 1; it != symbols.end(); ++it) {
+  auto *it = copy->symbols.begin();
+  // The first symbol in the merged section (symbols.front()) must keep its
+  // unwind entry. If this section is empty, the copy's first symbol becomes the
+  // new front, so we skip clearing it.
+  if (symbols.empty())
+    ++it;
+  for (; it != copy->symbols.end(); ++it) {
     assert((*it)->value == 0);
     (*it)->originalUnwindEntry = nullptr;
   }
+  symbols.insert(symbols.end(), copy->symbols.begin(), copy->symbols.end());
+  copy->symbols.clear();
 }
 
 void ConcatInputSection::writeTo(uint8_t *buf) {

lld/test/ELF/sparcv9-gotdata.s

@@ -0,0 +1,141 @@
+# REQUIRES: sparc
+## Test R_SPARC_GOTDATA_OP_HIX22, R_SPARC_GOTDATA_OP_LOX10 and R_SPARC_GOTDATA_OP.
+## The GOT load is optimized to an add of the symbol's GOT-relative address
+## unless the symbol is preemptible or absolute.
+
+# RUN: rm -rf %t && split-file %s %t && cd %t
+# RUN: llvm-mc -filetype=obj -triple=sparcv9 a.s -o a.o
+# RUN: llvm-mc -filetype=obj -triple=sparcv9 b.s -o b.o
+# RUN: llvm-mc -filetype=obj -triple=sparcv9 c.s -o c.o
+# RUN: llvm-mc -filetype=obj -triple=sparcv9 ifunc.s -o ifunc.o
+
+# RUN: ld.lld -shared a.o c.o -o a.so
+# RUN: llvm-readelf -S -r a.so | FileCheck %s --check-prefix=RELOC
+# RUN: llvm-objdump -d -j .text --no-show-raw-insn --no-print-imm-hex a.so | FileCheck %s
+
+# RELOC:      [ 8] .got PROGBITS 0000000000200390 {{[0-9a-f]+}} 000020
+# RELOC:      Relocation section '.rela.dyn' {{.*}} contains 1 entries:
+# RELOC:      0000000000200398 {{.*}} R_SPARC_GLOB_DAT {{.*}} foo + 0
+
+## foo is preemptible, so the GOT load stays.
+## .got[1] - _GLOBAL_OFFSET_TABLE_ = 0x200398 - 0x200390 = 8.
+# CHECK-LABEL: <_start>:
+# CHECK-NEXT:   sethi 0, %l1
+# CHECK-NEXT:   xor %l1, 8, %l1
+# CHECK-NEXT:   ldx [%l7+%l1], %l2
+## hid0 - _GLOBAL_OFFSET_TABLE_ = 0x3003b0 - 0x200390 = 0x100020.
+# CHECK-NEXT:   sethi 1024, %g1
+# CHECK-NEXT:   xor %g1, 32, %g1
+# CHECK-NEXT:   add %l7, %g1, %g1
+## hid1 - _GLOBAL_OFFSET_TABLE_ = 0x1002ec - 0x200390 = -0x1000a4. sethi holds
+## the complement, which the sign-extended xor operand undoes.
+# CHECK-NEXT:   sethi 1024, %o2
+# CHECK-NEXT:   xor %o2, -164, %o2
+# CHECK-NEXT:   add %i0, %o2, %i5
+## An undefined symbol and an absolute symbol both have an absolute address, so
+## neither is optimized.
+# CHECK-NEXT:   sethi 0, %l1
+# CHECK-NEXT:   xor %l1, 16, %l1
+# CHECK-NEXT:   ldx [%l7+%l1], %l2
+# CHECK-NEXT:   sethi 0, %g4
+# CHECK-NEXT:   xor %g4, 24, %g4
+# CHECK-NEXT:   ldx [%l7+%g4], %o3
+
+## foo is not preemptible in an executable and is optimized as well. undefweak
+## and absolute keep the GOT load in every mode.
+# RUN: ld.lld a.o c.o -o a
+# RUN: llvm-objdump -d -j .text --no-show-raw-insn --no-print-imm-hex a | FileCheck %s --check-prefix=NOPIE
+# NOPIE-LABEL: <_start>:
+# NOPIE-NEXT:   sethi 1024, %l1
+# NOPIE-NEXT:   xor %l1, -4, %l1
+# NOPIE-NEXT:   add %l7, %l1, %l2
+# NOPIE:        sethi 0, %l1
+# NOPIE-NEXT:   xor %l1, 8, %l1
+# NOPIE-NEXT:   ldx [%l7+%l1], %l2
+# NOPIE-NEXT:   sethi 0, %g4
+# NOPIE-NEXT:   xor %g4, 16, %g4
+# NOPIE-NEXT:   ldx [%l7+%g4], %o3
+
+## .got holds nothing but the header, yet must be kept: %l7 points at it.
+# RUN: ld.lld -shared b.o -o b.so
+# RUN: llvm-readelf -S b.so | FileCheck %s --check-prefix=HEADER
+# HEADER: [ 7] .got PROGBITS 00000000002002b8 {{[0-9a-f]+}} 000008
+
+# RUN: not ld.lld -shared ifunc.o 2>&1 | FileCheck %s --check-prefix=IFUNC --implicit-check-not=error:
+# IFUNC: error: {{.*}}relocation R_SPARC_WDISP19 out of range
+
+## The optimization is decided before addresses are known, so an out-of-range
+## offset is an error rather than a fallback to the GOT load. The offset is
+## reported as signed, not as the complement the sethi encodes. Only hid0, in
+## .data, is out of range; the other symbols stay next to .got.
+# RUN: not ld.lld -shared a.o c.o --section-start=.text=0x10000 --section-start=.got=0x20000 \
+# RUN:   --section-start=.data=0x200000000 2>&1 | FileCheck %s --check-prefix=ERR -DVAL=8589803520 --implicit-check-not=error:
+# RUN: not ld.lld -shared a.o c.o --section-start=.data=0x10000 --section-start=.text=0x200000000 \
+# RUN:   --section-start=.got=0x200020000 2>&1 | FileCheck %s --check-prefix=ERR -DVAL=-8590000128 --implicit-check-not=error:
+# ERR: error: a.o:(.text+0xc): relocation R_SPARC_GOTDATA_OP_HIX22 out of range: [[VAL]] is not in [-4294967296, 4294967295]; references 'hid0'
+
+#--- a.s
+.text
+.globl _start
+_start:
+  sethi %gdop_hix22(foo), %l1
+  xor   %l1, %gdop_lox10(foo), %l1
+  ldx   [%l7 + %l1], %l2, %gdop(foo)
+
+  sethi %gdop_hix22(hid0), %g1
+  xor   %g1, %gdop_lox10(hid0), %g1
+  ldx   [%l7 + %g1], %g1, %gdop(hid0)
+
+## The add keeps rs1, rs2 and rd of the load, and ld is optimized like ldx.
+  sethi %gdop_hix22(hid1), %o2
+  xor   %o2, %gdop_lox10(hid1), %o2
+  ld    [%i0 + %o2], %i5, %gdop(hid1)
+
+  sethi %gdop_hix22(undefweak), %l1
+  xor   %l1, %gdop_lox10(undefweak), %l1
+  ldx   [%l7 + %l1], %l2, %gdop(undefweak)
+
+  sethi %gdop_hix22(absolute), %g4
+  xor   %g4, %gdop_lox10(absolute), %g4
+  ldx   [%l7 + %g4], %o3, %gdop(absolute)
+
+.globl foo, hid0, hid1
+.hidden hid0, hid1
+foo:
+hid1:
+  nop
+
+.weak undefweak
+.hidden undefweak
+
+.data
+hid0:
+  .xword 0
+
+#--- b.s
+.globl _start
+_start:
+  sethi %gdop_hix22(local), %l1
+  xor   %l1, %gdop_lox10(local), %l1
+  ldx   [%l7 + %l1], %l2, %gdop(local)
+local:
+  nop
+
+#--- c.s
+## Separate file to test %gdop_lox10(absolute) relocations in a.s
+.globl absolute
+.hidden absolute
+absolute = 0x1234
+
+#--- ifunc.s
+.globl _start
+_start:
+  sethi %gdop_hix22(ifunc), %l1
+  xor   %l1, %gdop_lox10(ifunc), %l1
+  ldx   [%l7 + %l1], %l2, %gdop(ifunc)
+
+.globl ifunc
+.hidden ifunc
+.type ifunc, @gnu_indirect_function
+ifunc:
+  nop

lld/test/MachO/icf-scale-same-class.s

@@ -0,0 +1,37 @@
+# REQUIRES: x86
+# RUN: rm -rf %t*
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %s -o %t.o
+# RUN: %lld -lSystem --icf=all -o %t %t.o
+# RUN: llvm-objdump --macho --section-headers %t | FileCheck %s --check-prefix=SECT
+# RUN: llvm-objdump --macho --syms %t | FileCheck %s --check-prefix=SYMS
+
+## Every body folds into one, so __text holds a single 6-byte body plus _main
+## rather than the 3 MiB (0x30000d) the unfolded copies occupy.
+# SECT:      Idx Name          Size
+# SECT-NEXT: 0 __text        00000009
+
+## The sentinels bracket the group, so checking that the first and last members
+## of the class share an address covers the whole run of folds.
+# SYMS-DAG: [[#%.16x,F:]] g     F __TEXT,__text _f_first
+# SYMS-DAG: [[#F]]        g     F __TEXT,__text _f_last
+
+.subsections_via_symbols
+.text
+.p2align 2
+
+.globl _f_first
+_f_first:; movl $7, %eax; ret
+
+## 64 Ki identical function landing in one equivalence class.
+.rept 65536
+  .globl _f\+
+  _f\+:; movl $$7, %eax; ret
+.endr
+
+.globl _f_last
+_f_last:; movl $7, %eax; ret
+
+.globl _main
+_main:
+  ret

lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.cpp

@@ -840,22 +840,116 @@ def main(f):
     return ArgInfo(count, varargs)
 )";
 
-Expected<PythonCallable::ArgInfo> PythonCallable::GetArgInfo() const {
-  ArgInfo result = {};
-  if (!IsValid())
-    return nullDeref();
-
+// inspect.signature() is deeply recursive and expensive in C-stack terms;
+// reentrant scripted callbacks dispatched through GetArgInfo() can turn
+// that into a fatal stack overflow instead of a catchable Python
+// RecursionError. GetArgInfo() never calls this itself; callers fall back
+// to it explicitly when they need to handle callables its cheaper,
+// attribute-only approach can't (e.g. builtins).
+Expected<PythonCallable::ArgInfo>
+PythonCallable::GetArgInfoFromInspectSignature(const PythonCallable &callable) {
+  PythonCallable::ArgInfo result = {};
   // no need to synchronize access to this global, we already have the GIL
   static PythonScript get_arg_info(get_arg_info_script);
-  Expected<PythonObject> pyarginfo = get_arg_info(*this);
+  Expected<PythonObject> pyarginfo = get_arg_info(callable);
   if (!pyarginfo)
     return pyarginfo.takeError();
   long long count =
       cantFail(As<long long>(pyarginfo.get().GetAttribute("count")));
   bool has_varargs =
       cantFail(As<bool>(pyarginfo.get().GetAttribute("has_varargs")));
-  result.max_positional_args = has_varargs ? ArgInfo::UNBOUNDED : count;
+  result.max_positional_args =
+      has_varargs ? PythonCallable::ArgInfo::UNBOUNDED : count;
+  return result;
+}
 
+// GetArgInfo()'s branches, top to bottom (`func` is what each branch ends up
+// introspecting; the final step is always func.__code__.co_argcount/co_flags):
+//
+//   callable
+//   |-- has __self__          -> func = __func__           (bound method;
+//   |                             fails for slot wrappers, e.g. (1).__add__)
+//   |-- has __code__ already  -> func = callable            (plain function)
+//   `-- neither
+//       |-- is a class
+//       |   |-- __init__ has __code__ -> func = __init__
+//       |   `-- else: check __new__ too (object.__init__ is lenient about
+//       |       extra args once __new__ is overridden)
+//       |       |-- __new__ has __code__ -> func = __new__
+//       |       `-- else                  -> ArgInfo{0} (object's defaults)
+//       `-- is an instance
+//           `-- func = __call__ (unwrap __func__ if bound)
+//               `-- no __code__ -> error (e.g. a builtin)
+Expected<PythonCallable::ArgInfo> PythonCallable::GetArgInfo() const {
+  if (!IsValid())
+    return nullDeref();
+
+  PythonObject func = *this;
+  bool implicit_first_arg = false;
+  if (HasAttribute("__self__")) {
+    implicit_first_arg = true;
+    Expected<PythonObject> func_or_err = GetAttribute("__func__");
+    if (!func_or_err)
+      return func_or_err.takeError();
+    func = *func_or_err;
+  } else if (!HasAttribute("__code__")) {
+    implicit_first_arg = true;
+    if (PyType_Check(m_py_obj)) {
+      Expected<PythonObject> init_or_err = GetAttribute("__init__");
+      if (!init_or_err)
+        return init_or_err.takeError();
+      func = *init_or_err;
+      if (!func.HasAttribute("__code__")) {
+        // __init__ is still object.__init__. A class may customize
+        // __new__ instead and leave __init__ untouched, which makes
+        // object.__init__ lenient about extra arguments -- so check
+        // __new__ too before concluding there are none.
+        Expected<PythonObject> new_or_err = GetAttribute("__new__");
+        if (!new_or_err)
+          return new_or_err.takeError();
+        func = *new_or_err;
+        if (!func.HasAttribute("__code__"))
+          return ArgInfo{0};
+      }
+    } else {
+      Expected<PythonObject> call_or_err = GetAttribute("__call__");
+      if (!call_or_err)
+        return call_or_err.takeError();
+      func = *call_or_err;
+      if (func.HasAttribute("__self__")) {
+        Expected<PythonObject> inner_or_err = func.GetAttribute("__func__");
+        if (!inner_or_err)
+          return inner_or_err.takeError();
+        func = *inner_or_err;
+      }
+      if (!func.HasAttribute("__code__"))
+        return llvm::createStringError("__call__ has no __code__");
+    }
+  }
+
+  Expected<PythonObject> code_or_err = func.GetAttribute("__code__");
+  if (!code_or_err)
+    return code_or_err.takeError();
+  PythonObject code = *code_or_err;
+
+  Expected<long long> argcount =
+      As<long long>(code.GetAttribute("co_argcount"));
+  if (!argcount)
+    return argcount.takeError();
+  Expected<long long> flags = As<long long>(code.GetAttribute("co_flags"));
+  if (!flags)
+    return flags.takeError();
+
+  ArgInfo result = {};
+  // Mirrors CPython's CO_VARARGS from <code.h>, which isn't reliably
+  // visible across the Python versions/platforms this file builds against.
+  constexpr long long kCoFlagVarArgs = 0x04;
+  if (*flags & kCoFlagVarArgs) {
+    result.max_positional_args = ArgInfo::UNBOUNDED;
+  } else {
+    long long count = *argcount - (implicit_first_arg ? 1 : 0);
+    result.max_positional_args = count > 0 ? static_cast<unsigned>(count) : 0;
+  }
   return result;
 }
 

lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.h

@@ -615,6 +615,15 @@ public:
 
   llvm::Expected<ArgInfo> GetArgInfo() const;
 
+  // Always derives ArgInfo via a Python-level inspect.signature() call,
+  // regardless of whether the callable's argument count/varargs bit could
+  // have been read directly off its data attributes. GetArgInfo() prefers
+  // the cheaper attribute-based path and only falls back to this for
+  // callables it can't introspect that way (e.g. builtins); exposed
+  // separately so that fallback behavior can be tested directly.
+  static llvm::Expected<ArgInfo>
+  GetArgInfoFromInspectSignature(const PythonCallable &callable);
+
   PythonObject operator()();
 
   PythonObject operator()(std::initializer_list<PyObject *> args);

lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp

@@ -1300,6 +1300,14 @@ ScriptInterpreterPythonImpl::GetMaxPositionalArgumentsForCallable(
                                    callable_name.str().c_str());
   }
   llvm::Expected<PythonCallable::ArgInfo> arg_info = pfunc.GetArgInfo();
+  if (!arg_info) {
+    // `-f` may point at a builtin, unlike other GetArgInfo() callers.
+    LLDB_LOG_ERROR(GetLog(LLDBLog::Script), arg_info.takeError(),
+                   "GetArgInfo failed for callable {1}, falling back to "
+                   "inspect.signature: {0}",
+                   callable_name);
+    arg_info = PythonCallable::GetArgInfoFromInspectSignature(pfunc);
+  }
   if (!arg_info)
     return arg_info.takeError();
   return arg_info.get().max_positional_args;

lldb/test/API/commands/register/register_command/TestRegisters.py

@@ -717,7 +717,6 @@ class RegisterCommandsTestCase(TestBase):
         self.expect("register write pc 0", substrs=[err_msg], error=True)
         self.expect("register info pc", substrs=[err_msg], error=True)
 
-    @expectedFailureDarwin(bugnumber="github.com/llvm/llvm-project/issues/213386")
     def test_case_insensitivity(self):
         """
         Register names, their aliases and any generic names like "sp" and "ra"

lldb/unittests/ScriptInterpreter/Python/PythonDataObjectsTests.cpp

@@ -735,6 +735,17 @@ class NewStyle(object):
   def __init__(self, one, two, three):
     pass
 
+class NoConstructorAtAll:
+  pass
+
+class NewOnlyVarArgs:
+  def __new__(cls, *args, **kwargs):
+    return super().__new__(cls)
+
+class NewOnlyFixedArgs:
+  def __new__(cls, a, b):
+    return super().__new__(cls)
+
 )";
     PyObject *o =
         RunString(script, Py_file_input, globals.get(), globals.get());
@@ -782,13 +793,37 @@ class NewStyle(object):
     arginfo = newstyle.get().GetArgInfo();
     ASSERT_THAT_EXPECTED(arginfo, llvm::Succeeded());
     EXPECT_EQ(arginfo.get().max_positional_args, 3u);
+
+    // Neither __init__ nor __new__ overridden: object.__init__ truly takes
+    // no extra arguments here.
+    auto no_ctor = As<PythonCallable>(globals.GetItem("NoConstructorAtAll"));
+    ASSERT_THAT_EXPECTED(no_ctor, llvm::Succeeded());
+    arginfo = no_ctor.get().GetArgInfo();
+    ASSERT_THAT_EXPECTED(arginfo, llvm::Succeeded());
+    EXPECT_EQ(arginfo.get().max_positional_args, 0u);
+
+    // __init__ not overridden, but __new__ is: object.__init__ becomes
+    // lenient about extra arguments once __new__ is overridden, so the
+    // argument count has to come from __new__, not from __init__ alone.
+    auto new_varargs = As<PythonCallable>(globals.GetItem("NewOnlyVarArgs"));
+    ASSERT_THAT_EXPECTED(new_varargs, llvm::Succeeded());
+    arginfo = new_varargs.get().GetArgInfo();
+    ASSERT_THAT_EXPECTED(arginfo, llvm::Succeeded());
+    EXPECT_EQ(arginfo.get().max_positional_args,
+              PythonCallable::ArgInfo::UNBOUNDED);
+
+    auto new_fixed = As<PythonCallable>(globals.GetItem("NewOnlyFixedArgs"));
+    ASSERT_THAT_EXPECTED(new_fixed, llvm::Succeeded());
+    arginfo = new_fixed.get().GetArgInfo();
+    ASSERT_THAT_EXPECTED(arginfo, llvm::Succeeded());
+    EXPECT_EQ(arginfo.get().max_positional_args, 2u);
   }
 
   {
     auto builtins = PythonModule::BuiltinsModule();
     auto hex = As<PythonCallable>(builtins.GetAttribute("hex"));
     ASSERT_THAT_EXPECTED(hex, llvm::Succeeded());
-    auto arginfo = hex.get().GetArgInfo();
+    auto arginfo = PythonCallable::GetArgInfoFromInspectSignature(hex.get());
     ASSERT_THAT_EXPECTED(arginfo, llvm::Succeeded());
     EXPECT_EQ(arginfo.get().max_positional_args, 1u);
   }

llvm/include/llvm/Analysis/ConstantFolding.h

@@ -170,7 +170,8 @@ LLVM_ABI Constant *ConstantFoldCall(const CallBase *Call, Function *F,
                                     bool AllowNonDeterministic = true);
 
 LLVM_ABI Constant *ConstantFoldIntrinsic(Intrinsic::ID ID,
-                                         ArrayRef<Constant *> Ops, Type *Ty);
+                                         ArrayRef<Constant *> Ops, Type *Ty,
+                                         Function *CxtF = nullptr);
 
 /// ConstantFoldLoadThroughBitcast - try to cast constant to destination type
 /// returning null if unsuccessful. Can cast pointer to pointer or pointer to

llvm/include/llvm/Analysis/TargetFolder.h

@@ -193,10 +193,10 @@ public:
 
   Value *FoldIntrinsic(Intrinsic::ID ID, ArrayRef<Value *> Ops, Type *Ty,
                        FastMathFlags FMF = {},
-                       Function *CtxF = nullptr) const override {
+                       Function *CxtF = nullptr) const override {
     if (all_of(Ops, IsaPred<Constant>))
       return ConstantFoldIntrinsic(
-          ID, ArrayRef((Constant *const *)Ops.data(), Ops.size()), Ty);
+          ID, ArrayRef((Constant *const *)Ops.data(), Ops.size()), Ty, CxtF);
     return nullptr;
   }
 

llvm/include/llvm/CodeGen/MachinePipeliner.h

@@ -287,12 +287,6 @@ class LLVM_ABI SwingSchedulerDAG : public ScheduleDAGInstrs {
   unsigned MII = 0;
   /// The maximum initiation interval between iterations for this schedule.
   unsigned MAX_II = 0;
-  /// The resource-constrained minimum initiation interval, i.e. the lower
-  /// bound on MII imposed by the availability of processor resources.
-  unsigned ResMII = 0;
-  /// The recurrence-constrained minimum initiation interval, i.e. the lower
-  /// bound on MII imposed by loop-carried dependence cycles.
-  unsigned RecMII = 0;
   /// Set to true if a valid pipelined schedule is found for the loop.
   bool Scheduled = false;
   MachineLoop &Loop;

llvm/include/llvm/CodeGen/TargetInstrInfo.h

@@ -50,7 +50,6 @@ class MachineLoop;
 class MachineLoopInfo;
 class MachineMemOperand;
 class MachineModuleInfo;
-class MachineOptimizationRemarkEmitter;
 class MachineRegisterInfo;
 class MCAsmInfo;
 class MCInst;
@@ -898,11 +897,8 @@ public:
 
   /// Analyze loop L, which must be a single-basic-block loop, and if the
   /// conditions can be understood enough produce a PipelinerLoopInfo object.
-  /// \p ORE, if non-null, may be used by targets to emit optimization remarks
-  /// explaining why the loop was rejected for pipelining.
-  virtual std::unique_ptr<PipelinerLoopInfo> analyzeLoopForPipelining(
-      MachineBasicBlock *LoopBB,
-      MachineOptimizationRemarkEmitter *ORE = nullptr) const {
+  virtual std::unique_ptr<PipelinerLoopInfo>
+  analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
     return nullptr;
   }
 

llvm/include/llvm/CodeGen/WindowScheduler.h

@@ -109,10 +109,6 @@ public:
 
   bool run();
 
-  /// Returns the Initiation Interval of the best scheduling result found, or
-  /// UINT_MAX if run() has not been called or found no valid schedule.
-  unsigned getBestII() const { return BestII; }
-
 protected:
   /// Two types of ScheduleDAGs are needed, one for creating dependency graphs
   /// only, and the other for list scheduling as determined by the target.

llvm/lib/Analysis/ConstantFolding.cpp

@@ -1730,23 +1730,9 @@ Constant *llvm::ConstantFoldIntegerCast(Constant *C, Type *DestTy,
 //  Constant Folding for Calls
 //
 
-bool llvm::canConstantFoldCallTo(const CallBase *Call, const Function *F) {
-  if (Call->isNoBuiltin())
-    return false;
-  if (Call->getFunctionType() != F->getFunctionType())
-    return false;
-
-  // Allow FP calls (both libcalls and intrinsics) to avoid being folded.
-  // This can be useful for GPU targets or in cross-compilation scenarios
-  // when the exact target FP behaviour is required, and the host compiler's
-  // behaviour may be slightly different from the device's run-time behaviour.
-  if (DisableFPCallFolding && (F->getReturnType()->isFloatingPointTy() ||
-                               any_of(F->args(), [](const Argument &Arg) {
-                                 return Arg.getType()->isFloatingPointTy();
-                               })))
-    return false;
-
-  switch (F->getIntrinsicID()) {
+/// Returns true if the intrinsic can be constant folded, given \p IsStrictFP.
+static bool canConstantFoldIntrinsic(Intrinsic::ID ID, bool IsStrictFP) {
+  switch (ID) {
   // Operations that do not operate floating-point numbers and do not depend on
   // FP environment can be folded even in strictfp functions.
   case Intrinsic::bswap:
@@ -2007,7 +1993,7 @@ bool llvm::canConstantFoldCallTo(const CallBase *Call, const Function *F) {
   case Intrinsic::nvvm_sqrt_rn_d:
   case Intrinsic::nvvm_sqrt_rn_f:
   case Intrinsic::nvvm_sqrt_rn_ftz_f:
-    return !Call->isStrictFP();
+    return !IsStrictFP;
 
   // NVVM add intrinsics with explicit rounding modes
   case Intrinsic::nvvm_add_rm_d:
@@ -2104,8 +2090,35 @@ bool llvm::canConstantFoldCallTo(const CallBase *Call, const Function *F) {
     return true;
   default:
     return false;
-  case Intrinsic::not_intrinsic: break;
   }
+}
+
+/// Given a function's return type and its operands, determine if any of them of
+/// of floating-point type.
+static bool anyTypeContainsFP(Type *RetTy, ArrayRef<Value *> Ops) {
+  return RetTy->isFloatingPointTy() || any_of(Ops, [](Value *V) {
+           return V->getType()->isFloatingPointTy();
+         });
+}
+
+bool llvm::canConstantFoldCallTo(const CallBase *Call, const Function *F) {
+  if (Call->isNoBuiltin())
+    return false;
+  if (Call->getFunctionType() != F->getFunctionType())
+    return false;
+
+  // Allow FP calls (both libcalls and intrinsics) to avoid being folded.
+  // This can be useful for GPU targets or in cross-compilation scenarios
+  // when the exact target FP behaviour is required, and the host compiler's
+  // behaviour may be slightly different from the device's run-time behaviour.
+  if (DisableFPCallFolding &&
+      anyTypeContainsFP(
+          F->getReturnType(),
+          ArrayRef<Value *>((Value *const *)(F->arg_begin()), F->arg_size())))
+    return false;
+
+  if (F->getIntrinsicID() != Intrinsic::not_intrinsic)
+    return canConstantFoldIntrinsic(F->getIntrinsicID(), Call->isStrictFP());
 
   if (!F->hasName() || Call->isStrictFP())
     return false;
@@ -4699,7 +4712,14 @@ ConstantFoldStructCall(StringRef Name, Intrinsic::ID IntrinsicID,
 } // end anonymous namespace
 
 Constant *llvm::ConstantFoldIntrinsic(Intrinsic::ID ID,
-                                      ArrayRef<Constant *> Ops, Type *Ty) {
+                                      ArrayRef<Constant *> Ops, Type *Ty,
+                                      Function *CxtF) {
+  // In the absence of CxtF, assume strictfp conservatively.
+  if (!canConstantFoldIntrinsic(ID, CxtF ? CxtF->isStrictFP() : true) ||
+      (DisableFPCallFolding &&
+       anyTypeContainsFP(
+           Ty, ArrayRef<Value *>((Value *const *)Ops.data(), Ops.size()))))
+    return nullptr;
   return ConstantFoldScalarCall("", ID, Ty, Ops);
 }
 

llvm/lib/Analysis/InstructionSimplify.cpp

@@ -7348,6 +7348,13 @@ Value *llvm::simplifyIntrinsic(Intrinsic::ID IID, Type *ReturnType,
       any_of(Args, IsaPred<PoisonValue>))
     return PoisonValue::get(ReturnType);
 
+  // Defer to ConstantFolding if all args are constants.
+  if (all_of(Args, IsaPred<Constant>))
+    if (Constant *C = ConstantFoldIntrinsic(
+            IID, ArrayRef((Constant *const *)Args.data(), Args.size()),
+            ReturnType, CxtF))
+      return C;
+
   // Most of the intrinsics with no operands have some kind of side effect.
   // Don't simplify.
   if (!NumOperands) {

llvm/lib/CodeGen/MachinePipeliner.cpp

@@ -585,7 +585,7 @@ bool MachinePipeliner::canPipelineLoop(MachineLoop &L) {
 
   LI.LoopInductionVar = nullptr;
   LI.LoopCompare = nullptr;
-  LI.LoopPipelinerInfo = TII->analyzeLoopForPipelining(L.getTopBlock(), ORE);
+  LI.LoopPipelinerInfo = TII->analyzeLoopForPipelining(L.getTopBlock());
   if (!LI.LoopPipelinerInfo) {
     LLVM_DEBUG(dbgs() << "Unable to analyzeLoop, can NOT pipeline Loop\n");
     NumFailLoop++;
@@ -716,23 +716,7 @@ bool MachinePipeliner::runWindowScheduler(MachineLoop &L) {
   Context.RegClassInfo =
       &getAnalysis<MachineRegisterClassInfoWrapperPass>().getRCI();
   WindowScheduler WS(&Context, L);
-  bool Scheduled = WS.run();
-  if (Scheduled) {
-    unsigned II = WS.getBestII();
-    ORE->emit([&]() {
-      return MachineOptimizationRemark(DEBUG_TYPE, "window-schedule",
-                                       L.getStartLoc(), L.getHeader())
-             << "Window scheduled with Initiation Interval: "
-             << ore::NV("II", II);
-    });
-  } else {
-    ORE->emit([&]() {
-      return MachineOptimizationRemarkMissed(DEBUG_TYPE, "window-schedule",
-                                             L.getStartLoc(), L.getHeader())
-             << "Failed to find a valid window schedule";
-    });
-  }
-  return Scheduled;
+  return WS.run();
 }
 
 bool MachinePipeliner::useSwingModuloScheduler() {
@@ -2913,10 +2897,7 @@ bool SwingSchedulerDAG::schedulePipeline(SMSchedule &Schedule) {
              << "Schedule found with Initiation Interval: "
              << ore::NV("II", Schedule.getInitiationInterval())
              << ", MaxStageCount: "
-             << ore::NV("MaxStageCount", Schedule.getMaxStageCount())
-             << ", ResMII: " << ore::NV("ResMII", ResMII)
-             << ", RecMII: " << ore::NV("RecMII", RecMII) << ", Bound: "
-             << ore::NV("Bound", ResMII >= RecMII ? "Resource" : "Recurrence");
+             << ore::NV("MaxStageCount", Schedule.getMaxStageCount());
     });
   } else
     Schedule.reset();

llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp

@@ -54,11 +54,7 @@ void JITLinkerBase::linkPhase1(std::unique_ptr<JITLinkerBase> Self) {
   Ctx->getMemoryManager().allocate(
       Ctx->getJITLinkDylib(), *G,
       [S = std::move(Self)](AllocResult AR) mutable {
-        // FIXME: Once MSVC implements c++17 order of evaluation rules for calls
-        // this can be simplified to
-        //          S->linkPhase2(std::move(S), std::move(AR));
-        auto *TmpSelf = S.get();
-        TmpSelf->linkPhase2(std::move(S), std::move(AR));
+        S->linkPhase2(std::move(S), std::move(AR));
       });
 }
 
@@ -95,10 +91,7 @@ void JITLinkerBase::linkPhase2(std::unique_ptr<JITLinkerBase> Self,
       dbgs() << "No external symbols for " << G->getName()
              << ". Proceeding immediately with link phase 3.\n";
     });
-    // FIXME: Once MSVC implements c++17 order of evaluation rules for calls
-    // this can be simplified. See below.
-    auto &TmpSelf = *Self;
-    TmpSelf.linkPhase3(std::move(Self), AsyncLookupResult());
+    Self->linkPhase3(std::move(Self), AsyncLookupResult());
     return;
   }
 
@@ -108,22 +101,11 @@ void JITLinkerBase::linkPhase2(std::unique_ptr<JITLinkerBase> Self,
            << " (may trigger materialization/linking of other graphs)...\n";
   });
 
-  // We're about to hand off ownership of ourself to the continuation. Grab a
-  // pointer to the context so that we can call it to initiate the lookup.
-  //
-  // FIXME: Once MSVC implements c++17 order of evaluation rules for calls this
-  // can be simplified to:
-  //
-  // Ctx->lookup(std::move(UnresolvedExternals),
-  //             [Self=std::move(Self)](Expected<AsyncLookupResult> Result) {
-  //               Self->linkPhase3(std::move(Self), std::move(Result));
-  //             });
   Ctx->lookup(std::move(ExternalSymbols),
               createLookupContinuation(
                   [S = std::move(Self)](
                       Expected<AsyncLookupResult> LookupResult) mutable {
-                    auto &TmpSelf = *S;
-                    TmpSelf.linkPhase3(std::move(S), std::move(LookupResult));
+                    S->linkPhase3(std::move(S), std::move(LookupResult));
                   }));
 }
 
@@ -171,11 +153,7 @@ void JITLinkerBase::linkPhase3(std::unique_ptr<JITLinkerBase> Self,
   }
 
   Alloc->finalize([S = std::move(Self)](FinalizeResult FR) mutable {
-    // FIXME: Once MSVC implements c++17 order of evaluation rules for calls
-    // this can be simplified to
-    //          S->linkPhase2(std::move(S), std::move(AR));
-    auto *TmpSelf = S.get();
-    TmpSelf->linkPhase4(std::move(S), std::move(FR));
+    S->linkPhase4(std::move(S), std::move(FR));
   });
 }
 

llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h

@@ -116,12 +116,10 @@ public:
     // Ownership of the linker is passed into the linker's doLink function to
     // allow it to be passed on to async continuations.
     //
-    // FIXME: Remove LTmp once we have c++17.
     // C++17 sequencing rules guarantee that function name expressions are
-    // sequenced before arguments, so L->linkPhase1(std::move(L), ...) will be
-    // well formed.
-    auto &LTmp = *L;
-    LTmp.linkPhase1(std::move(L));
+    // sequenced before arguments, so L->linkPhase1(std::move(L)) is well
+    // formed.
+    L->linkPhase1(std::move(L));
   }
 
 private:

llvm/lib/Target/AArch64/AArch64InstrInfo.cpp

@@ -25,7 +25,6 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/AliasAnalysis.h"
-#include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/CodeGen/CFIInstBuilder.h"
 #include "llvm/CodeGen/LivePhysRegs.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
@@ -37,7 +36,6 @@
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineOperand.h"
-#include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/RegisterScavenging.h"
 #include "llvm/CodeGen/StackMaps.h"
@@ -12065,8 +12063,7 @@ static bool getIndVarInfo(Register Reg, const MachineBasicBlock *LoopBB,
 }
 
 std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
-AArch64InstrInfo::analyzeLoopForPipelining(
-    MachineBasicBlock *LoopBB, MachineOptimizationRemarkEmitter *ORE) const {
+AArch64InstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
   // Accept loops that meet the following conditions
   // * The conditional branch is BCC
   // * The compare instruction is ADDS/SUBS/WHILEXX
@@ -12075,66 +12072,24 @@ AArch64InstrInfo::analyzeLoopForPipelining(
   // * The induction variable is incremented/decremented by a single instruction
   // * Does not contain CALL or instructions which have unmodeled side effects
 
-  for (MachineInstr &MI : *LoopBB) {
-    // This instruction may use NZCV, which interferes with the instruction to
-    // be inserted for loop control.
-    if (MI.isCall()) {
-      if (ORE)
-        ORE->emit([&]() {
-          return MachineOptimizationRemarkAnalysis("pipeliner", "analyzeLoop",
-                                                   &MI)
-                 << "loop contains a call and therefore cannot be pipelined";
-        });
+  for (MachineInstr &MI : *LoopBB)
+    if (MI.isCall() || MI.hasUnmodeledSideEffects())
+      // This instruction may use NZCV, which interferes with the instruction to
+      // be inserted for loop control.
       return nullptr;
-    }
-    if (MI.hasUnmodeledSideEffects()) {
-      if (ORE)
-        ORE->emit([&]() {
-          return MachineOptimizationRemarkAnalysis("pipeliner", "analyzeLoop",
-                                                   &MI)
-                 << "loop contains an instruction with unmodeled side effects, "
-                    "and therefore cannot be pipelined";
-        });
-      return nullptr;
-    }
-  }
 
   MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
   SmallVector<MachineOperand, 4> Cond;
-  if (analyzeBranch(*LoopBB, TBB, FBB, Cond)) {
-    if (ORE)
-      ORE->emit([&]() {
-        return MachineOptimizationRemarkAnalysis(
-                   "pipeliner", "analyzeLoop",
-                   LoopBB->findDebugLoc(LoopBB->getFirstTerminator()), LoopBB)
-               << "branch cannot be analyzed";
-      });
+  if (analyzeBranch(*LoopBB, TBB, FBB, Cond))
     return nullptr;
-  }
 
   // Infinite loops are not supported
-  if (TBB == LoopBB && FBB == LoopBB) {
-    if (ORE)
-      ORE->emit([&]() {
-        return MachineOptimizationRemarkAnalysis(
-                   "pipeliner", "analyzeLoop",
-                   LoopBB->findDebugLoc(LoopBB->getFirstTerminator()), LoopBB)
-               << "infinite loops are not supported";
-      });
+  if (TBB == LoopBB && FBB == LoopBB)
     return nullptr;
-  }
 
   // Must be conditional branch
-  if (TBB != LoopBB && FBB == nullptr) {
-    if (ORE)
-      ORE->emit([&]() {
-        return MachineOptimizationRemarkAnalysis(
-                   "pipeliner", "analyzeLoop",
-                   LoopBB->findDebugLoc(LoopBB->getFirstTerminator()), LoopBB)
-               << "loop is not terminated by a conditional branch";
-      });
+  if (TBB != LoopBB && FBB == nullptr)
     return nullptr;
-  }
 
   assert((TBB == LoopBB || FBB == LoopBB) &&
          "The Loop must be a single-basic-block loop");
@@ -12142,16 +12097,8 @@ AArch64InstrInfo::analyzeLoopForPipelining(
   MachineInstr *CondBranch = &*LoopBB->getFirstTerminator();
   const TargetRegisterInfo &TRI = getRegisterInfo();
 
-  if (CondBranch->getOpcode() != AArch64::Bcc) {
-    if (ORE)
-      ORE->emit([&]() {
-        return MachineOptimizationRemarkAnalysis("pipeliner", "analyzeLoop",
-                                                 CondBranch)
-               << "branch opcode not yet supported for pipelining: "
-               << ore::NV("Opcode", getName(CondBranch->getOpcode()));
-      });
+  if (CondBranch->getOpcode() != AArch64::Bcc)
     return nullptr;
-  }
 
   // Normalization for createTripCountGreaterCondition()
   if (TBB == LoopBB)
@@ -12183,13 +12130,6 @@ AArch64InstrInfo::analyzeLoopForPipelining(
           Comp = &MI;
           break;
         }
-        if (ORE)
-          ORE->emit([&]() {
-            return MachineOptimizationRemarkAnalysis("pipeliner", "analyzeLoop",
-                                                     &MI)
-                   << "compare instruction not recognized: "
-                   << ore::NV("Opcode", getName(MI.getOpcode()));
-          });
         return nullptr;
       }
 
@@ -12198,44 +12138,22 @@ AArch64InstrInfo::analyzeLoopForPipelining(
           CompCounterOprNum = 2;
         else if (isDefinedOutside(Comp->getOperand(2).getReg(), LoopBB))
           CompCounterOprNum = 1;
-        else {
-          if (ORE)
-            ORE->emit([&]() {
-              return MachineOptimizationRemarkAnalysis("pipeliner",
-                                                       "analyzeLoop", Comp)
-                     << "neither operand of the compare is loop invariant";
-            });
+        else
           return nullptr;
-        }
       }
       break;
     }
   }
-  if (!Comp) {
-    if (ORE)
-      ORE->emit([&]() {
-        return MachineOptimizationRemarkAnalysis(
-                   "pipeliner", "analyzeLoop",
-                   LoopBB->findDebugLoc(LoopBB->getFirstTerminator()), LoopBB)
-               << "no NZCV-modifying compare instruction found";
-      });
+  if (!Comp)
     return nullptr;
-  }
 
   MachineInstr *Update = nullptr;
   Register Init;
   bool IsUpdatePriorComp;
   unsigned UpdateCounterOprNum;
   if (!getIndVarInfo(Comp->getOperand(CompCounterOprNum).getReg(), LoopBB,
-                     Update, UpdateCounterOprNum, Init, IsUpdatePriorComp)) {
-    if (ORE)
-      ORE->emit([&]() {
-        return MachineOptimizationRemarkAnalysis("pipeliner", "analyzeLoop",
-                                                 Comp)
-               << "loop induction variable pattern not recognized";
-      });
+                     Update, UpdateCounterOprNum, Init, IsUpdatePriorComp))
     return nullptr;
-  }
 
   return std::make_unique<AArch64PipelinerLoopInfo>(
       LoopBB, CondBranch, Comp, CompCounterOprNum, Update, UpdateCounterOprNum,

llvm/lib/Target/AArch64/AArch64InstrInfo.h

@@ -414,9 +414,8 @@ public:
                         const DebugLoc &DL,
                         int *BytesAdded = nullptr) const override;
 
-  std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo> analyzeLoopForPipelining(
-      MachineBasicBlock *LoopBB,
-      MachineOptimizationRemarkEmitter *ORE = nullptr) const override;
+  std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
+  analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override;
 
   bool
   reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const override;

llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp

@@ -6737,8 +6737,7 @@ bool ARMPipelinerLoopInfo::tooMuchRegisterPressure(SwingSchedulerDAG &SSD,
 } // namespace
 
 std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
-ARMBaseInstrInfo::analyzeLoopForPipelining(
-    MachineBasicBlock *LoopBB, MachineOptimizationRemarkEmitter *ORE) const {
+ARMBaseInstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
   MachineBasicBlock::iterator I = LoopBB->getFirstTerminator();
   MachineBasicBlock *Preheader = *LoopBB->pred_begin();
   if (Preheader == LoopBB)

llvm/lib/Target/ARM/ARMBaseInstrInfo.h

@@ -382,9 +382,8 @@ public:
 
   /// Analyze loop L, which must be a single-basic-block loop, and if the
   /// conditions can be understood enough produce a PipelinerLoopInfo object.
-  std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo> analyzeLoopForPipelining(
-      MachineBasicBlock *LoopBB,
-      MachineOptimizationRemarkEmitter *ORE = nullptr) const override;
+  std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
+  analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override;
 
 private:
   /// Returns an unused general-purpose register which can be used for

llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp

@@ -806,8 +806,7 @@ public:
 } // namespace
 
 std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
-HexagonInstrInfo::analyzeLoopForPipelining(
-    MachineBasicBlock *LoopBB, MachineOptimizationRemarkEmitter *ORE) const {
+HexagonInstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
   // We really "analyze" only hardware loops right now.
   MachineBasicBlock::iterator I = LoopBB->getFirstTerminator();
 

llvm/lib/Target/Hexagon/HexagonInstrInfo.h

@@ -139,9 +139,8 @@ public:
 
   /// Analyze loop L, which must be a single-basic-block loop, and if the
   /// conditions can be understood enough produce a PipelinerLoopInfo object.
-  std::unique_ptr<PipelinerLoopInfo> analyzeLoopForPipelining(
-      MachineBasicBlock *LoopBB,
-      MachineOptimizationRemarkEmitter *ORE = nullptr) const override;
+  std::unique_ptr<PipelinerLoopInfo>
+  analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override;
 
   /// Return true if it's profitable to predicate
   /// instructions with accumulated instruction latency of "NumCycles"

llvm/lib/Target/PowerPC/PPCInstrInfo.cpp

@@ -5790,8 +5790,7 @@ public:
 } // namespace
 
 std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
-PPCInstrInfo::analyzeLoopForPipelining(
-    MachineBasicBlock *LoopBB, MachineOptimizationRemarkEmitter *ORE) const {
+PPCInstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
   // We really "analyze" only hardware loops right now.
   MachineBasicBlock::iterator I = LoopBB->getFirstTerminator();
   MachineBasicBlock *Preheader = *LoopBB->pred_begin();

llvm/lib/Target/PowerPC/PPCInstrInfo.h

@@ -789,9 +789,8 @@ public:
 
   /// Analyze loop L, which must be a single-basic-block loop, and if the
   /// conditions can be understood enough produce a PipelinerLoopInfo object.
-  std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo> analyzeLoopForPipelining(
-      MachineBasicBlock *LoopBB,
-      MachineOptimizationRemarkEmitter *ORE = nullptr) const override;
+  std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
+  analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override;
 };
 
 }

llvm/lib/Target/RISCV/RISCVInstrInfo.cpp

@@ -5394,8 +5394,7 @@ public:
 } // namespace
 
 std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
-RISCVInstrInfo::analyzeLoopForPipelining(
-    MachineBasicBlock *LoopBB, MachineOptimizationRemarkEmitter *ORE) const {
+RISCVInstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
   MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
   SmallVector<MachineOperand, 4> Cond;
   if (analyzeBranch(*LoopBB, TBB, FBB, Cond, /*AllowModify=*/false))

llvm/lib/Target/RISCV/RISCVInstrInfo.h

@@ -328,9 +328,8 @@ public:
 
   unsigned getTailDuplicateSize(CodeGenOptLevel OptLevel) const override;
 
-  std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo> analyzeLoopForPipelining(
-      MachineBasicBlock *LoopBB,
-      MachineOptimizationRemarkEmitter *ORE = nullptr) const override;
+  std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
+  analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override;
 
   bool isHighLatencyDef(int Opc) const override;
 

llvm/test/CodeGen/AArch64/sms-analyzeloop-remarks.mir

@@ -1,168 +0,0 @@
-# RUN: llc --verify-machineinstrs -mtriple=aarch64 -o /dev/null %s -run-pass pipeliner \
-# RUN:   -aarch64-enable-pipeliner -pass-remarks-analysis=pipeliner \
-# RUN:   -pass-remarks-missed=pipeliner 2>&1 | FileCheck %s
-
-# Regression test for the analyzeLoopForPipelining() optimization remarks:
-# each rejection point in AArch64InstrInfo::analyzeLoopForPipelining() should
-# report a distinct, specific reason via the "pipeliner"/"analyzeLoop" remark,
-# rather than only the generic "The loop structure is not supported" remark
-# from canPipelineLoop().
-
---- |
-  define dso_local void @call(ptr noalias nocapture noundef writeonly %a, i32 noundef %n) local_unnamed_addr #0 {
-  entry:
-    ret void
-  }
-  declare void @callee()
-
-  define dso_local void @infiniteloop(ptr noalias nocapture noundef writeonly %a, i32 noundef %n) local_unnamed_addr #0 {
-  entry:
-    ret void
-  }
-
-  define dso_local void @unsupportedbranch(ptr noalias nocapture noundef writeonly %a) local_unnamed_addr #0 {
-  entry:
-    ret void
-  }
-
-  define dso_local void @badcompare(ptr noalias nocapture noundef writeonly %a, i32 noundef %n) local_unnamed_addr #0 {
-  entry:
-    ret void
-  }
-
-  define dso_local void @noninvariantcompare(i64 noundef %x, i64 noundef %y) local_unnamed_addr #0 {
-  entry:
-    ret void
-  }
-
-  define dso_local void @badindvar(i64 noundef %x, i64 noundef %y) local_unnamed_addr #0 {
-  entry:
-    ret void
-  }
-...
----
-name:            call
-tracksRegLiveness: true
-body:             |
-  ; A call in the loop body may use NZCV, which interferes with the
-  ; instruction to be inserted for loop control.
-  ; CHECK: loop contains a call and therefore cannot be pipelined
-  ; CHECK: The loop structure is not supported
-  bb.0.entry:
-    liveins: $x0, $w1
-
-    %10:gpr64sp = COPY $x0
-    %11:gpr32common = COPY $w1
-
-  bb.1:
-    %12:gpr64sp = PHI %10, %bb.0, %13, %bb.1
-    %13:gpr64sp = ADDXri %12, 1, 0
-    BL @callee, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp
-    dead $wzr = SUBSWri %11, 1, 0, implicit-def $nzcv
-    Bcc 1, %bb.1, implicit $nzcv
-    B %bb.2
-
-  bb.2:
-...
----
-name:            infiniteloop
-tracksRegLiveness: true
-body:             |
-  ; Both branch targets are the loop itself: nothing exits the loop.
-  ; CHECK: infinite loops are not supported
-  ; CHECK: The loop structure is not supported
-  bb.0.entry:
-    liveins: $x0, $w1
-
-    %10:gpr64sp = COPY $x0
-    %11:gpr32common = COPY $w1
-
-  bb.1:
-    %12:gpr64sp = PHI %10, %bb.0, %13, %bb.1
-    %13:gpr64sp = ADDXri %12, 1, 0
-    dead $wzr = SUBSWri %11, 0, 0, implicit-def $nzcv
-    Bcc 0, %bb.1, implicit $nzcv
-    B %bb.1
-...
----
-name:            unsupportedbranch
-tracksRegLiveness: true
-body:             |
-  ; The loop terminator is an unconditional branch (Bcc is required).
-  ; CHECK: branch opcode not yet supported for pipelining: B
-  ; CHECK: The loop structure is not supported
-  bb.0.entry:
-    liveins: $x0
-
-    %10:gpr64sp = COPY $x0
-
-  bb.1:
-    %12:gpr64sp = PHI %10, %bb.0, %13, %bb.1
-    %13:gpr64sp = ADDXri %12, 1, 0
-    B %bb.1
-...
----
-name:            badcompare
-tracksRegLiveness: true
-body:             |
-  ; The NZCV-modifying instruction is not a recognized compare opcode.
-  ; CHECK: compare instruction not recognized: ANDSWri
-  ; CHECK: The loop structure is not supported
-  bb.0.entry:
-    liveins: $x0, $w1
-
-    %10:gpr64sp = COPY $x0
-    %11:gpr32common = COPY $w1
-
-  bb.1:
-    %12:gpr64sp = PHI %10, %bb.0, %13, %bb.1
-    %13:gpr64sp = ADDXri %12, 1, 0
-    dead $wzr = ANDSWri %11, 1, implicit-def $nzcv
-    Bcc 1, %bb.1, implicit $nzcv
-    B %bb.2
-
-  bb.2:
-...
----
-name:            noninvariantcompare
-tracksRegLiveness: true
-body:             |
-  ; Neither operand of the compare instruction is a loop invariant value.
-  ; CHECK: neither operand of the compare is loop invariant
-  ; CHECK: The loop structure is not supported
-  bb.0.entry:
-    liveins: $x0, $x1
-    %10:gpr64 = COPY $x0
-    %11:gpr64 = COPY $x1
-
-  bb.1:
-    %12:gpr64 = PHI %11, %bb.0, %13, %bb.1
-    %13:gpr64 = ADDXrr %12, %11
-    dead $xzr = SUBSXrr %13, %13, implicit-def $nzcv
-    Bcc 1, %bb.1, implicit $nzcv
-    B %bb.2
-
-  bb.2:
-...
----
-name:            badindvar
-tracksRegLiveness: true
-body:             |
-  ; The value compared against is not updated by a recognized increment
-  ; instruction.
-  ; CHECK: loop induction variable pattern not recognized
-  ; CHECK: The loop structure is not supported
-  bb.0.entry:
-    liveins: $x0, $x1
-    %10:gpr64 = COPY $x0
-    %11:gpr64 = COPY $x1
-
-  bb.1:
-    %12:gpr64 = PHI %11, %bb.0, %13, %bb.1
-    %13:gpr64 = ORRXrr %12, %12
-    dead $xzr = SUBSXrr %12, %10, implicit-def $nzcv
-    Bcc 1, %bb.1, implicit $nzcv
-    B %bb.2
-
-  bb.2:
-...

llvm/test/CodeGen/AArch64/sms-schedule-remark.mir

@@ -1,83 +0,0 @@
-# RUN: llc --verify-machineinstrs -mtriple=aarch64 -o /dev/null %s -run-pass pipeliner \
-# RUN:   -aarch64-enable-pipeliner -pass-remarks-analysis=pipeliner 2>&1 | FileCheck %s
-
-# Regression test for the "schedule" success remark: it should report ResMII,
-# RecMII, and which of the two is the binding constraint on the chosen II, in
-# addition to the Initiation Interval and MaxStageCount that were already
-# reported.
-
-# CHECK: Schedule found with Initiation Interval: 3, MaxStageCount: 2, ResMII: 0, RecMII: 0, Bound: Resource
-
---- |
-  define dso_local void @func(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b, i32 noundef %n) local_unnamed_addr #0 {
-  entry:
-    %cmp6 = icmp sgt i32 %n, 0
-    br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
-
-  for.body.preheader:                               ; preds = %entry
-    %wide.trip.count = zext nneg i32 %n to i64
-    br label %for.body
-
-  for.cond.cleanup:                                 ; preds = %for.body, %entry
-    ret void
-
-  for.body:                                         ; preds = %for.body.preheader, %for.body
-    %lsr.iv11 = phi i64 [ %wide.trip.count, %for.body.preheader ], [ %lsr.iv.next, %for.body ]
-    %lsr.iv9 = phi ptr [ %b, %for.body.preheader ], [ %scevgep10, %for.body ]
-    %lsr.iv = phi ptr [ %a, %for.body.preheader ], [ %scevgep, %for.body ]
-    %0 = load float, ptr %lsr.iv9, align 4
-    %add = fadd float %0, 1.000000e+00
-    store float %add, ptr %lsr.iv, align 4
-    %scevgep = getelementptr i8, ptr %lsr.iv, i64 4
-    %scevgep10 = getelementptr i8, ptr %lsr.iv9, i64 4
-    %lsr.iv.next = add nsw i64 %lsr.iv11, -1
-    %exitcond.not = icmp eq i64 %lsr.iv.next, 0
-    br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
-  }
-
-...
----
-name:            func
-tracksRegLiveness: true
-liveins:
-  - { reg: '$x0', virtual-reg: '%7' }
-  - { reg: '$x1', virtual-reg: '%8' }
-  - { reg: '$w2', virtual-reg: '%9' }
-body:             |
-  bb.0.entry:
-    successors: %bb.1(0x50000000), %bb.2(0x30000000)
-    liveins: $x0, $x1, $w2
-
-    %9:gpr32common = COPY $w2
-    %8:gpr64 = COPY $x1
-    %7:gpr64 = COPY $x0
-    dead $wzr = SUBSWri %9, 1, 0, implicit-def $nzcv
-    Bcc 11, %bb.2, implicit $nzcv
-    B %bb.1
-
-  bb.1.for.body.preheader:
-    %11:gpr32 = ORRWrs $wzr, %9, 0
-    %0:gpr64all = SUBREG_TO_REG killed %11, %subreg.sub_32
-    %14:fpr32 = FMOVSi 112
-    B %bb.3
-
-  bb.2.for.cond.cleanup:
-    RET_ReallyLR
-
-  bb.3.for.body:
-    successors: %bb.2(0x04000000), %bb.3(0x7c000000)
-
-    %1:gpr64sp = PHI %0, %bb.1, %6, %bb.3
-    %2:gpr64sp = PHI %8, %bb.1, %5, %bb.3
-    %3:gpr64sp = PHI %7, %bb.1, %4, %bb.3
-    early-clobber %12:gpr64sp, %13:fpr32 = LDRSpost %2, 4 :: (load (s32) from %ir.lsr.iv9)
-    %15:fpr32 = nofpexcept FADDSrr killed %13, %14, implicit $fpcr
-    early-clobber %16:gpr64sp = STRSpost killed %15, %3, 4 :: (store (s32) into %ir.lsr.iv)
-    %4:gpr64all = COPY %16
-    %5:gpr64all = COPY %12
-    %17:gpr64 = nsw SUBSXri %1, 1, 0, implicit-def $nzcv
-    %6:gpr64all = COPY %17
-    Bcc 0, %bb.2, implicit $nzcv
-    B %bb.3
-
-...

llvm/test/CodeGen/Hexagon/swp-ws-exp.mir

@@ -2,14 +2,9 @@
 # RUN: llc --mtriple=hexagon %s -run-pass=pipeliner -debug-only=pipeliner \
 # RUN: -window-sched=force -filetype=null -verify-machineinstrs 2>&1 \
 # RUN: | FileCheck %s
-# RUN: llc --mtriple=hexagon %s -run-pass=pipeliner \
-# RUN: -window-sched=force -filetype=null -verify-machineinstrs \
-# RUN: -pass-remarks=pipeliner 2>&1 | FileCheck %s --check-prefix=REMARK
 
 # CHECK: Best window offset is {{[0-9]+}} and Best II is {{[0-9]+}}.
 
-# REMARK: remark: {{.*}}Window scheduled with Initiation Interval: {{[0-9]+}}
-
 --- |
   define void @exp_approx_top_six(i32 %N, ptr noalias %x, ptr noalias %y) #0 {
   entry:

llvm/test/CodeGen/Hexagon/swp-ws-fail-0.mir

@@ -9,9 +9,6 @@
 # RUN: llc --mtriple=hexagon %s -run-pass=pipeliner -debug-only=pipeliner \
 # RUN: -window-sched=force -window-region-limit=1 -window-search-ratio=80 \
 # RUN: -filetype=null 2>&1 | FileCheck %s --check-prefix=CHECK-SCHED-NOT-NEEDED
-# RUN: llc --mtriple=hexagon %s -run-pass=pipeliner \
-# RUN: -window-sched=force -filetype=null \
-# RUN: -pass-remarks-missed=pipeliner 2>&1 | FileCheck %s --check-prefix=REMARK
 
 # CHECK-INITIALIZE: There are too few MIs in the window region!
 # CHECK-INITIALIZE: The WindowScheduler failed to initialize!
@@ -19,8 +16,6 @@
 # CHECK-ANALYSE-II: Window scheduling is not needed!
 # CHECK-SCHED-NOT-NEEDED: Window scheduling is not needed!
 
-# REMARK: remark: {{.*}}Failed to find a valid window schedule
-
 ---
 name:            relu
 tracksRegLiveness: true

llvm/test/CodeGen/PowerPC/sms-remark.ll

@@ -15,7 +15,7 @@
 @y = dso_local global [1024 x i32] zeroinitializer, align 4
 
 define dso_local ptr @foo() local_unnamed_addr {
-;ENABLED: Schedule found with Initiation Interval: {{[0-9]+}}, MaxStageCount: {{[0-9]+}}, ResMII: {{[0-9]+}}, RecMII: {{[0-9]+}}, Bound: {{(Recurrence|Resource)}}
+;ENABLED: Schedule found with Initiation Interval
 ;ENABLED: Pipelined succesfully!
 entry:
   %.pre = load i32, ptr @y, align 4

llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll

@@ -253,20 +253,9 @@ define void @test_minbws_for_trunc(i32 %n, ptr noalias %p1, ptr noalias %p2) {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[MUL:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 4, i16 255)
-; CHECK-NEXT:    [[MUL_RESULT:%.*]] = extractvalue { i16, i1 } [[MUL]], 0
-; CHECK-NEXT:    [[MUL_OVERFLOW:%.*]] = extractvalue { i16, i1 } [[MUL]], 1
-; CHECK-NEXT:    [[TMP0:%.*]] = add i16 4, [[MUL_RESULT]]
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i16 [[TMP0]], 4
-; CHECK-NEXT:    [[TMP2:%.*]] = or i1 [[TMP1]], [[MUL_OVERFLOW]]
-; CHECK-NEXT:    [[MUL1:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 32, i64 255)
-; CHECK-NEXT:    [[MUL_RESULT2:%.*]] = extractvalue { i64, i1 } [[MUL1]], 0
-; CHECK-NEXT:    [[MUL_OVERFLOW3:%.*]] = extractvalue { i64, i1 } [[MUL1]], 1
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[P2]], i64 [[MUL_RESULT2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[P2]], i64 8160
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp ult ptr [[TMP3]], [[P2]]
-; CHECK-NEXT:    [[TMP5:%.*]] = or i1 [[TMP4]], [[MUL_OVERFLOW3]]
-; CHECK-NEXT:    [[TMP6:%.*]] = or i1 [[TMP2]], [[TMP5]]
-; CHECK-NEXT:    br i1 [[TMP6]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; CHECK:       [[VECTOR_MEMCHECK]]:
 ; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[P2]], i64 2042
 ; CHECK-NEXT:    [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[P2]], i64 1021

llvm/test/Transforms/LoopVectorize/X86/funclet.ll

@@ -26,10 +26,9 @@ define void @test1() #0 personality ptr @__CxxFrameHandler3 {
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1024
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[TRY_CONT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[TRY_CONT]]:
-; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <16 x double> [[TMP2]], i64 15
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    store double [[TMP4]], ptr @sink, align 8
+; CHECK-NEXT:    store double 1.000000e+00, ptr @sink, align 8
 ; CHECK-NEXT:    catchret from [[TMP1]] to label %[[TRY_CONT1:.*]]
 ; CHECK:       [[TRY_CONT1]]:
 ; CHECK-NEXT:    ret void

llvm/test/Transforms/LoopVectorize/constantfolder.ll

@@ -323,6 +323,41 @@ exit:
   ret void
 }
 
+define void @const_fold_unaryintrinsic(ptr %p) {
+; CHECK-LABEL: define void @const_fold_unaryintrinsic(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr double, ptr [[P]], i64 [[INDEX]]
+; CHECK-NEXT:    store <4 x double> splat (double 1.000000e+00), ptr [[TMP0]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; CHECK-NEXT:    br i1 [[TMP1]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.p.iv = getelementptr double, ptr %p, i64 %iv
+  %res = call double @llvm.canonicalize.f64(double 1.0)
+  store double %res, ptr %gep.p.iv
+  %iv.next = add i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, 100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
 define void @const_fold_widegep(ptr noalias %A, ptr noalias %B, i64 %d) {
 ; CHECK-LABEL: define void @const_fold_widegep(
 ; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[D:%.*]]) {
@@ -334,7 +369,7 @@ define void @const_fold_widegep(ptr noalias %A, ptr noalias %B, i64 %d) {
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; CHECK-NEXT:    br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    store ptr [[A]], ptr [[B]], align 8
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
@@ -432,7 +467,7 @@ define void @unused_live_ins_in_preheader(ptr %dst, ptr %src) {
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw <4 x i8> [[VEC_IND]], splat (i8 4)
 ; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i32 [[INDEX_NEXT]], 12
-; CHECK-NEXT:    br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[EXIT]]:

llvm/test/Transforms/LoopVectorize/hoist-predicated-loads-with-predicated-stores.ll

@@ -932,71 +932,34 @@ exit:
 define void @sink_multiple_store_groups_noalias_via_scev_urem(ptr %dst, ptr %src) {
 ; CHECK-LABEL: define void @sink_multiple_store_groups_noalias_via_scev_urem(
 ; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    br label %[[VECTOR_SCEVCHECK:.*]]
-; CHECK:       [[VECTOR_SCEVCHECK]]:
-; CHECK-NEXT:    [[MUL:%.*]] = call { i7, i1 } @llvm.umul.with.overflow.i7(i7 16, i7 -29)
-; CHECK-NEXT:    [[MUL_OVERFLOW:%.*]] = extractvalue { i7, i1 } [[MUL]], 1
-; CHECK-NEXT:    br i1 [[MUL_OVERFLOW]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
-; CHECK:       [[VECTOR_MEMCHECK]]:
-; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[SRC]], i64 12828
-; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[DST]], i64 12696
-; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP1]]
-; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP]]
-; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
-; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:  [[VECTOR_PH:.*]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE3:.*]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 16>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE3]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = shl i64 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[TMP0]], 16
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr double, ptr [[SRC]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[IV_NEXT:%.*]], %[[PRED_STORE_CONTINUE3:.*]] ]
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr double, ptr [[SRC]], i64 [[TMP1]]
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr [[TMP2]], i64 152
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[TMP3]], i64 152
-; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP4]], align 4, !alias.scope [[META92:![0-9]+]], !noalias [[META95:![0-9]+]]
-; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP5]], align 4, !alias.scope [[META92]], !noalias [[META95]]
-; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x i32> poison, i32 [[TMP6]], i32 0
-; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i32> [[TMP8]], i32 [[TMP7]], i32 1
-; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq <2 x i32> [[TMP9]], zeroinitializer
-; CHECK-NEXT:    [[TMP11:%.*]] = load double, ptr [[TMP2]], align 8, !alias.scope [[META92]], !noalias [[META95]]
-; CHECK-NEXT:    [[TMP12:%.*]] = load double, ptr [[TMP3]], align 8, !alias.scope [[META92]], !noalias [[META95]]
-; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <2 x double> poison, double [[TMP11]], i32 0
-; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <2 x double> [[TMP13]], double [[TMP12]], i32 1
-; CHECK-NEXT:    [[TMP15:%.*]] = urem <2 x i64> [[VEC_IND]], splat (i64 128)
-; CHECK-NEXT:    [[TMP20:%.*]] = fadd <2 x double> [[TMP14]], splat (double 8.000000e+00)
-; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <2 x i64> [[TMP15]], i64 0
-; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP17]]
-; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <2 x i64> [[TMP15]], i64 1
-; CHECK-NEXT:    [[TMP32:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP23]]
-; CHECK-NEXT:    [[TMP18:%.*]] = select <2 x i1> [[TMP10]], <2 x double> [[TMP14]], <2 x double> [[TMP20]]
-; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <2 x double> [[TMP18]], i64 0
-; CHECK-NEXT:    store double [[TMP22]], ptr [[TMP19]], align 8, !alias.scope [[META95]]
-; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <2 x double> [[TMP18]], i64 1
-; CHECK-NEXT:    store double [[TMP34]], ptr [[TMP32]], align 8, !alias.scope [[META95]]
-; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <2 x i1> [[TMP10]], i64 0
+; CHECK-NEXT:    [[C:%.*]] = load i32, ptr [[TMP5]], align 4
+; CHECK-NEXT:    [[TMP24:%.*]] = icmp eq i32 [[C]], 0
+; CHECK-NEXT:    [[V_1:%.*]] = load double, ptr [[TMP3]], align 8
+; CHECK-NEXT:    [[CLAMPED:%.*]] = urem i64 [[TMP1]], 128
 ; CHECK-NEXT:    br i1 [[TMP24]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
 ; CHECK:       [[PRED_STORE_IF]]:
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr double, ptr [[DST]], i64 [[CLAMPED]]
+; CHECK-NEXT:    store double [[V_1]], ptr [[TMP19]], align 8
 ; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr i8, ptr [[TMP19]], i64 16
-; CHECK-NEXT:    store double 1.000000e+01, ptr [[TMP25]], align 8, !alias.scope [[META95]]
-; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE]]
+; CHECK-NEXT:    store double 1.000000e+01, ptr [[TMP25]], align 8
+; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE3]]
 ; CHECK:       [[PRED_STORE_CONTINUE]]:
-; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <2 x i1> [[TMP10]], i64 1
-; CHECK-NEXT:    br i1 [[TMP26]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3]]
-; CHECK:       [[PRED_STORE_IF2]]:
-; CHECK-NEXT:    [[TMP33:%.*]] = getelementptr i8, ptr [[TMP32]], i64 16
-; CHECK-NEXT:    store double 1.000000e+01, ptr [[TMP33]], align 8, !alias.scope [[META95]]
+; CHECK-NEXT:    [[R_1:%.*]] = fadd double [[V_1]], 8.000000e+00
+; CHECK-NEXT:    [[GEP_DST1_ELSE:%.*]] = getelementptr double, ptr [[DST]], i64 [[CLAMPED]]
+; CHECK-NEXT:    store double [[R_1]], ptr [[GEP_DST1_ELSE]], align 8
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE3]]
 ; CHECK:       [[PRED_STORE_CONTINUE3]]:
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 32)
-; CHECK-NEXT:    [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; CHECK-NEXT:    br i1 [[TMP28]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP97:![0-9]+]]
-; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    br [[EXIT:label %.*]]
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[TMP1]], 16
+; CHECK-NEXT:    [[EXIT_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1600
+; CHECK-NEXT:    br i1 [[EXIT_COND]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]]
 ; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -1128,20 +1091,20 @@ define void @test_three_stores_with_different_predicates(ptr %dst, ptr %src, ptr
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE11:.*]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[INDEX]], 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[COND]], i32 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4, !alias.scope [[META99:![0-9]+]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4, !alias.scope [[META92:![0-9]+]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp ule <2 x i32> [[WIDE_LOAD]], splat (i32 11)
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i1> [[TMP3]], i64 0
 ; CHECK-NEXT:    br i1 [[TMP4]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
 ; CHECK:       [[PRED_STORE_IF]]:
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[INDEX]]
-; CHECK-NEXT:    store i32 1, ptr [[TMP5]], align 4, !alias.scope [[META102:![0-9]+]], !noalias [[META99]]
+; CHECK-NEXT:    store i32 1, ptr [[TMP5]], align 4, !alias.scope [[META95:![0-9]+]], !noalias [[META92]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE]]
 ; CHECK:       [[PRED_STORE_CONTINUE]]:
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i1> [[TMP3]], i64 1
 ; CHECK-NEXT:    br i1 [[TMP6]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]]
 ; CHECK:       [[PRED_STORE_IF2]]:
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[TMP1]]
-; CHECK-NEXT:    store i32 1, ptr [[TMP7]], align 4, !alias.scope [[META102]], !noalias [[META99]]
+; CHECK-NEXT:    store i32 1, ptr [[TMP7]], align 4, !alias.scope [[META95]], !noalias [[META92]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE3]]
 ; CHECK:       [[PRED_STORE_CONTINUE3]]:
 ; CHECK-NEXT:    [[TMP10:%.*]] = icmp ule <2 x i32> [[WIDE_LOAD]], splat (i32 10)
@@ -1149,14 +1112,14 @@ define void @test_three_stores_with_different_predicates(ptr %dst, ptr %src, ptr
 ; CHECK-NEXT:    br i1 [[TMP12]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]]
 ; CHECK:       [[PRED_STORE_IF4]]:
 ; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[INDEX]]
-; CHECK-NEXT:    store i32 2, ptr [[TMP13]], align 4, !alias.scope [[META102]], !noalias [[META99]]
+; CHECK-NEXT:    store i32 2, ptr [[TMP13]], align 4, !alias.scope [[META95]], !noalias [[META92]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE5]]
 ; CHECK:       [[PRED_STORE_CONTINUE5]]:
 ; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x i1> [[TMP10]], i64 1
 ; CHECK-NEXT:    br i1 [[TMP14]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]]
 ; CHECK:       [[PRED_STORE_IF6]]:
 ; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[TMP1]]
-; CHECK-NEXT:    store i32 2, ptr [[TMP15]], align 4, !alias.scope [[META102]], !noalias [[META99]]
+; CHECK-NEXT:    store i32 2, ptr [[TMP15]], align 4, !alias.scope [[META95]], !noalias [[META92]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE7]]
 ; CHECK:       [[PRED_STORE_CONTINUE7]]:
 ; CHECK-NEXT:    [[TMP16:%.*]] = icmp ule <2 x i32> [[WIDE_LOAD]], splat (i32 9)
@@ -1164,19 +1127,19 @@ define void @test_three_stores_with_different_predicates(ptr %dst, ptr %src, ptr
 ; CHECK-NEXT:    br i1 [[TMP18]], label %[[PRED_STORE_IF8:.*]], label %[[PRED_STORE_CONTINUE9:.*]]
 ; CHECK:       [[PRED_STORE_IF8]]:
 ; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[INDEX]]
-; CHECK-NEXT:    store i32 3, ptr [[TMP19]], align 4, !alias.scope [[META102]], !noalias [[META99]]
+; CHECK-NEXT:    store i32 3, ptr [[TMP19]], align 4, !alias.scope [[META95]], !noalias [[META92]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE9]]
 ; CHECK:       [[PRED_STORE_CONTINUE9]]:
 ; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <2 x i1> [[TMP16]], i64 1
 ; CHECK-NEXT:    br i1 [[TMP20]], label %[[PRED_STORE_IF10:.*]], label %[[PRED_STORE_CONTINUE11]]
 ; CHECK:       [[PRED_STORE_IF10]]:
 ; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[TMP1]]
-; CHECK-NEXT:    store i32 3, ptr [[TMP21]], align 4, !alias.scope [[META102]], !noalias [[META99]]
+; CHECK-NEXT:    store i32 3, ptr [[TMP21]], align 4, !alias.scope [[META95]], !noalias [[META92]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE11]]
 ; CHECK:       [[PRED_STORE_CONTINUE11]]:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
 ; CHECK-NEXT:    [[TMP22:%.*]] = icmp eq i32 [[INDEX_NEXT]], 100
-; CHECK-NEXT:    br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP104:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP97:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br [[EXIT:label %.*]]
 ; CHECK:       [[SCALAR_PH]]:
@@ -1250,7 +1213,7 @@ define void @stores_never_sunk_past_alising_loads_or_stores(ptr %dst, ptr %src,
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE21:.*]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[INDEX]], 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[COND]], i32 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4, !alias.scope [[META106:![0-9]+]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4, !alias.scope [[META99:![0-9]+]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp ule <2 x i32> [[WIDE_LOAD]], splat (i32 11)
 ; CHECK-NEXT:    [[TMP4:%.*]] = xor <2 x i1> [[TMP3]], splat (i1 true)
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[INDEX]]
@@ -1258,8 +1221,8 @@ define void @stores_never_sunk_past_alising_loads_or_stores(ptr %dst, ptr %src,
 ; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i1> [[TMP4]], i64 0
 ; CHECK-NEXT:    br i1 [[TMP9]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
 ; CHECK:       [[PRED_LOAD_IF]]:
-; CHECK-NEXT:    store i32 99, ptr [[TMP5]], align 4, !alias.scope [[META109:![0-9]+]], !noalias [[META111:![0-9]+]]
-; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP5]], align 4, !alias.scope [[META109]], !noalias [[META111]]
+; CHECK-NEXT:    store i32 99, ptr [[TMP5]], align 4, !alias.scope [[META102:![0-9]+]], !noalias [[META104:![0-9]+]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP5]], align 4, !alias.scope [[META102]], !noalias [[META104]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x i32> poison, i32 [[TMP7]], i64 0
 ; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE]]
 ; CHECK:       [[PRED_LOAD_CONTINUE]]:
@@ -1267,8 +1230,8 @@ define void @stores_never_sunk_past_alising_loads_or_stores(ptr %dst, ptr %src,
 ; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <2 x i1> [[TMP4]], i64 1
 ; CHECK-NEXT:    br i1 [[TMP13]], label %[[PRED_LOAD_IF10:.*]], label %[[PRED_LOAD_CONTINUE11:.*]]
 ; CHECK:       [[PRED_LOAD_IF10]]:
-; CHECK-NEXT:    store i32 99, ptr [[TMP6]], align 4, !alias.scope [[META109]], !noalias [[META111]]
-; CHECK-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP6]], align 4, !alias.scope [[META109]], !noalias [[META111]]
+; CHECK-NEXT:    store i32 99, ptr [[TMP6]], align 4, !alias.scope [[META102]], !noalias [[META104]]
+; CHECK-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP6]], align 4, !alias.scope [[META102]], !noalias [[META104]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <2 x i32> [[TMP10]], i32 [[TMP11]], i64 1
 ; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE11]]
 ; CHECK:       [[PRED_LOAD_CONTINUE11]]:
@@ -1277,30 +1240,30 @@ define void @stores_never_sunk_past_alising_loads_or_stores(ptr %dst, ptr %src,
 ; CHECK-NEXT:    br i1 [[TMP9]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
 ; CHECK:       [[PRED_STORE_IF]]:
 ; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <2 x i32> [[TMP17]], i64 0
-; CHECK-NEXT:    store i32 [[TMP18]], ptr [[TMP5]], align 4, !alias.scope [[META109]], !noalias [[META111]]
+; CHECK-NEXT:    store i32 [[TMP18]], ptr [[TMP5]], align 4, !alias.scope [[META102]], !noalias [[META104]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE]]
 ; CHECK:       [[PRED_STORE_CONTINUE]]:
 ; CHECK-NEXT:    br i1 [[TMP13]], label %[[PRED_STORE_IF12:.*]], label %[[PRED_STORE_CONTINUE13:.*]]
 ; CHECK:       [[PRED_STORE_IF12]]:
 ; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <2 x i32> [[TMP17]], i64 1
-; CHECK-NEXT:    store i32 [[TMP19]], ptr [[TMP6]], align 4, !alias.scope [[META109]], !noalias [[META111]]
+; CHECK-NEXT:    store i32 [[TMP19]], ptr [[TMP6]], align 4, !alias.scope [[META102]], !noalias [[META104]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE13]]
 ; CHECK:       [[PRED_STORE_CONTINUE13]]:
 ; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <2 x i1> [[TMP3]], i64 0
 ; CHECK-NEXT:    br i1 [[TMP22]], label %[[PRED_LOAD_IF14:.*]], label %[[PRED_LOAD_CONTINUE15:.*]]
 ; CHECK:       [[PRED_LOAD_IF14]]:
-; CHECK-NEXT:    [[TMP21:%.*]] = load i32, ptr [[TMP5]], align 4, !alias.scope [[META109]], !noalias [[META111]]
+; CHECK-NEXT:    [[TMP21:%.*]] = load i32, ptr [[TMP5]], align 4, !alias.scope [[META102]], !noalias [[META104]]
 ; CHECK-NEXT:    [[TMP24:%.*]] = insertelement <2 x i32> poison, i32 [[TMP21]], i64 0
-; CHECK-NEXT:    store i32 99, ptr [[TMP5]], align 4, !alias.scope [[META109]], !noalias [[META111]]
+; CHECK-NEXT:    store i32 99, ptr [[TMP5]], align 4, !alias.scope [[META102]], !noalias [[META104]]
 ; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE15]]
 ; CHECK:       [[PRED_LOAD_CONTINUE15]]:
 ; CHECK-NEXT:    [[TMP20:%.*]] = phi <2 x i32> [ poison, %[[PRED_STORE_CONTINUE13]] ], [ [[TMP24]], %[[PRED_LOAD_IF14]] ]
 ; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <2 x i1> [[TMP3]], i64 1
 ; CHECK-NEXT:    br i1 [[TMP27]], label %[[PRED_LOAD_IF16:.*]], label %[[PRED_LOAD_CONTINUE17:.*]]
 ; CHECK:       [[PRED_LOAD_IF16]]:
-; CHECK-NEXT:    [[TMP25:%.*]] = load i32, ptr [[TMP6]], align 4, !alias.scope [[META109]], !noalias [[META111]]
+; CHECK-NEXT:    [[TMP25:%.*]] = load i32, ptr [[TMP6]], align 4, !alias.scope [[META102]], !noalias [[META104]]
 ; CHECK-NEXT:    [[TMP23:%.*]] = insertelement <2 x i32> [[TMP20]], i32 [[TMP25]], i64 1
-; CHECK-NEXT:    store i32 99, ptr [[TMP6]], align 4, !alias.scope [[META109]], !noalias [[META111]]
+; CHECK-NEXT:    store i32 99, ptr [[TMP6]], align 4, !alias.scope [[META102]], !noalias [[META104]]
 ; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE17]]
 ; CHECK:       [[PRED_LOAD_CONTINUE17]]:
 ; CHECK-NEXT:    [[TMP31:%.*]] = phi <2 x i32> [ [[TMP20]], %[[PRED_LOAD_CONTINUE15]] ], [ [[TMP23]], %[[PRED_LOAD_IF16]] ]
@@ -1309,19 +1272,19 @@ define void @stores_never_sunk_past_alising_loads_or_stores(ptr %dst, ptr %src,
 ; CHECK:       [[PRED_STORE_IF18]]:
 ; CHECK-NEXT:    [[TMP34:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[INDEX]]
 ; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <2 x i32> [[TMP32]], i64 0
-; CHECK-NEXT:    store i32 [[TMP28]], ptr [[TMP34]], align 4, !alias.scope [[META113:![0-9]+]], !noalias [[META106]]
+; CHECK-NEXT:    store i32 [[TMP28]], ptr [[TMP34]], align 4, !alias.scope [[META106:![0-9]+]], !noalias [[META99]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE19]]
 ; CHECK:       [[PRED_STORE_CONTINUE19]]:
 ; CHECK-NEXT:    br i1 [[TMP27]], label %[[PRED_STORE_IF20:.*]], label %[[PRED_STORE_CONTINUE21]]
 ; CHECK:       [[PRED_STORE_IF20]]:
 ; CHECK-NEXT:    [[TMP37:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[TMP1]]
 ; CHECK-NEXT:    [[TMP33:%.*]] = extractelement <2 x i32> [[TMP32]], i64 1
-; CHECK-NEXT:    store i32 [[TMP33]], ptr [[TMP37]], align 4, !alias.scope [[META113]], !noalias [[META106]]
+; CHECK-NEXT:    store i32 [[TMP33]], ptr [[TMP37]], align 4, !alias.scope [[META106]], !noalias [[META99]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE21]]
 ; CHECK:       [[PRED_STORE_CONTINUE21]]:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
 ; CHECK-NEXT:    [[TMP39:%.*]] = icmp eq i32 [[INDEX_NEXT]], 100
-; CHECK-NEXT:    br i1 [[TMP39]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP114:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP39]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP107:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br [[EXIT:label %.*]]
 ; CHECK:       [[SCALAR_PH]]:
@@ -1389,16 +1352,16 @@ define void @loads_never_hoisted_past_alising_stores(ptr %dst, ptr %src, ptr %co
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE13:.*]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[INDEX]], 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[COND]], i32 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4, !alias.scope [[META116:![0-9]+]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4, !alias.scope [[META109:![0-9]+]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp ule <2 x i32> [[WIDE_LOAD]], splat (i32 11)
 ; CHECK-NEXT:    [[TMP4:%.*]] = xor <2 x i1> [[TMP3]], splat (i1 true)
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i1> [[TMP4]], i64 0
 ; CHECK-NEXT:    br i1 [[TMP5]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
 ; CHECK:       [[PRED_STORE_IF]]:
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[INDEX]]
-; CHECK-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP6]], align 4, !alias.scope [[META119:![0-9]+]], !noalias [[META121:![0-9]+]]
+; CHECK-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP6]], align 4, !alias.scope [[META112:![0-9]+]], !noalias [[META114:![0-9]+]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x i32> poison, i32 [[TMP9]], i64 0
-; CHECK-NEXT:    store i32 100, ptr [[TMP6]], align 4, !alias.scope [[META119]], !noalias [[META121]]
+; CHECK-NEXT:    store i32 100, ptr [[TMP6]], align 4, !alias.scope [[META112]], !noalias [[META114]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE]]
 ; CHECK:       [[PRED_STORE_CONTINUE]]:
 ; CHECK-NEXT:    [[TMP8:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP7]], %[[PRED_STORE_IF]] ]
@@ -1406,9 +1369,9 @@ define void @loads_never_hoisted_past_alising_stores(ptr %dst, ptr %src, ptr %co
 ; CHECK-NEXT:    br i1 [[TMP10]], label %[[PRED_STORE_IF10:.*]], label %[[PRED_STORE_CONTINUE11:.*]]
 ; CHECK:       [[PRED_STORE_IF10]]:
 ; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[TMP1]]
-; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP11]], align 4, !alias.scope [[META119]], !noalias [[META121]]
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP11]], align 4, !alias.scope [[META112]], !noalias [[META114]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <2 x i32> [[TMP8]], i32 [[TMP13]], i64 1
-; CHECK-NEXT:    store i32 100, ptr [[TMP11]], align 4, !alias.scope [[META119]], !noalias [[META121]]
+; CHECK-NEXT:    store i32 100, ptr [[TMP11]], align 4, !alias.scope [[META112]], !noalias [[META114]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE11]]
 ; CHECK:       [[PRED_STORE_CONTINUE11]]:
 ; CHECK-NEXT:    [[TMP14:%.*]] = phi <2 x i32> [ [[TMP8]], %[[PRED_STORE_CONTINUE]] ], [ [[TMP12]], %[[PRED_STORE_IF10]] ]
@@ -1417,8 +1380,8 @@ define void @loads_never_hoisted_past_alising_stores(ptr %dst, ptr %src, ptr %co
 ; CHECK-NEXT:    br i1 [[TMP16]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
 ; CHECK:       [[PRED_LOAD_IF]]:
 ; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[INDEX]]
-; CHECK-NEXT:    store i32 99, ptr [[TMP17]], align 4, !alias.scope [[META119]], !noalias [[META121]]
-; CHECK-NEXT:    [[TMP20:%.*]] = load i32, ptr [[TMP17]], align 4, !alias.scope [[META119]], !noalias [[META121]]
+; CHECK-NEXT:    store i32 99, ptr [[TMP17]], align 4, !alias.scope [[META112]], !noalias [[META114]]
+; CHECK-NEXT:    [[TMP20:%.*]] = load i32, ptr [[TMP17]], align 4, !alias.scope [[META112]], !noalias [[META114]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = insertelement <2 x i32> poison, i32 [[TMP20]], i64 0
 ; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE]]
 ; CHECK:       [[PRED_LOAD_CONTINUE]]:
@@ -1427,8 +1390,8 @@ define void @loads_never_hoisted_past_alising_stores(ptr %dst, ptr %src, ptr %co
 ; CHECK-NEXT:    br i1 [[TMP21]], label %[[PRED_LOAD_IF12:.*]], label %[[PRED_LOAD_CONTINUE13]]
 ; CHECK:       [[PRED_LOAD_IF12]]:
 ; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[TMP1]]
-; CHECK-NEXT:    store i32 99, ptr [[TMP22]], align 4, !alias.scope [[META119]], !noalias [[META121]]
-; CHECK-NEXT:    [[TMP24:%.*]] = load i32, ptr [[TMP22]], align 4, !alias.scope [[META119]], !noalias [[META121]]
+; CHECK-NEXT:    store i32 99, ptr [[TMP22]], align 4, !alias.scope [[META112]], !noalias [[META114]]
+; CHECK-NEXT:    [[TMP24:%.*]] = load i32, ptr [[TMP22]], align 4, !alias.scope [[META112]], !noalias [[META114]]
 ; CHECK-NEXT:    [[TMP23:%.*]] = insertelement <2 x i32> [[TMP19]], i32 [[TMP24]], i64 1
 ; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE13]]
 ; CHECK:       [[PRED_LOAD_CONTINUE13]]:
@@ -1438,12 +1401,12 @@ define void @loads_never_hoisted_past_alising_stores(ptr %dst, ptr %src, ptr %co
 ; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[TMP1]]
 ; CHECK-NEXT:    [[TMP29:%.*]] = select <2 x i1> [[TMP3]], <2 x i32> [[TMP26]], <2 x i32> [[TMP15]]
 ; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <2 x i32> [[TMP29]], i64 0
-; CHECK-NEXT:    store i32 [[TMP31]], ptr [[TMP27]], align 4, !alias.scope [[META123:![0-9]+]], !noalias [[META116]]
+; CHECK-NEXT:    store i32 [[TMP31]], ptr [[TMP27]], align 4, !alias.scope [[META116:![0-9]+]], !noalias [[META109]]
 ; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <2 x i32> [[TMP29]], i64 1
-; CHECK-NEXT:    store i32 [[TMP30]], ptr [[TMP28]], align 4, !alias.scope [[META123]], !noalias [[META116]]
+; CHECK-NEXT:    store i32 [[TMP30]], ptr [[TMP28]], align 4, !alias.scope [[META116]], !noalias [[META109]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
 ; CHECK-NEXT:    [[TMP32:%.*]] = icmp eq i32 [[INDEX_NEXT]], 100
-; CHECK-NEXT:    br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP124:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP117:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br [[EXIT:label %.*]]
 ; CHECK:       [[SCALAR_PH]]:
@@ -1492,71 +1455,34 @@ exit:
 define void @sink_multiple_store_groups_noalias_via_scev_srem(ptr %dst, ptr %src) {
 ; CHECK-LABEL: define void @sink_multiple_store_groups_noalias_via_scev_srem(
 ; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    br label %[[VECTOR_SCEVCHECK:.*]]
-; CHECK:       [[VECTOR_SCEVCHECK]]:
-; CHECK-NEXT:    [[MUL:%.*]] = call { i7, i1 } @llvm.umul.with.overflow.i7(i7 16, i7 -29)
-; CHECK-NEXT:    [[MUL_OVERFLOW:%.*]] = extractvalue { i7, i1 } [[MUL]], 1
-; CHECK-NEXT:    br i1 [[MUL_OVERFLOW]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
-; CHECK:       [[VECTOR_MEMCHECK]]:
-; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[SRC]], i64 12828
-; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[DST]], i64 12696
-; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP1]]
-; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP]]
-; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
-; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:  [[VECTOR_PH:.*]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE7:.*]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 16>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE7]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = shl i64 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[TMP0]], 16
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr double, ptr [[SRC]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[IV_NEXT:%.*]], %[[PRED_STORE_CONTINUE7:.*]] ]
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr double, ptr [[SRC]], i64 [[TMP1]]
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr [[TMP2]], i64 152
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[TMP3]], i64 152
-; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP4]], align 4, !alias.scope [[META126:![0-9]+]], !noalias [[META129:![0-9]+]]
-; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP5]], align 4, !alias.scope [[META126]], !noalias [[META129]]
-; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x i32> poison, i32 [[TMP6]], i32 0
-; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i32> [[TMP8]], i32 [[TMP7]], i32 1
-; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq <2 x i32> [[TMP9]], zeroinitializer
-; CHECK-NEXT:    [[TMP11:%.*]] = load double, ptr [[TMP2]], align 8, !alias.scope [[META126]], !noalias [[META129]]
-; CHECK-NEXT:    [[TMP12:%.*]] = load double, ptr [[TMP3]], align 8, !alias.scope [[META126]], !noalias [[META129]]
-; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <2 x double> poison, double [[TMP11]], i32 0
-; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <2 x double> [[TMP13]], double [[TMP12]], i32 1
-; CHECK-NEXT:    [[TMP15:%.*]] = srem <2 x i64> [[VEC_IND]], splat (i64 128)
-; CHECK-NEXT:    [[TMP18:%.*]] = fadd <2 x double> [[TMP14]], splat (double 8.000000e+00)
-; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <2 x i64> [[TMP15]], i64 0
-; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP19]]
-; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <2 x i64> [[TMP15]], i64 1
-; CHECK-NEXT:    [[TMP32:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP23]]
-; CHECK-NEXT:    [[TMP17:%.*]] = select <2 x i1> [[TMP10]], <2 x double> [[TMP14]], <2 x double> [[TMP18]]
-; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <2 x double> [[TMP17]], i64 0
-; CHECK-NEXT:    store double [[TMP22]], ptr [[TMP28]], align 8, !alias.scope [[META129]]
-; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <2 x double> [[TMP17]], i64 1
-; CHECK-NEXT:    store double [[TMP25]], ptr [[TMP32]], align 8, !alias.scope [[META129]]
-; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <2 x i1> [[TMP10]], i64 0
+; CHECK-NEXT:    [[C:%.*]] = load i32, ptr [[TMP5]], align 4
+; CHECK-NEXT:    [[TMP26:%.*]] = icmp eq i32 [[C]], 0
+; CHECK-NEXT:    [[V_1:%.*]] = load double, ptr [[TMP3]], align 8
+; CHECK-NEXT:    [[CLAMPED:%.*]] = srem i64 [[TMP1]], 128
 ; CHECK-NEXT:    br i1 [[TMP26]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]]
 ; CHECK:       [[PRED_STORE_IF4]]:
+; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr double, ptr [[DST]], i64 [[CLAMPED]]
+; CHECK-NEXT:    store double [[V_1]], ptr [[TMP28]], align 8
 ; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr i8, ptr [[TMP28]], i64 16
-; CHECK-NEXT:    store double 1.000000e+01, ptr [[TMP29]], align 8, !alias.scope [[META129]]
-; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE5]]
+; CHECK-NEXT:    store double 1.000000e+01, ptr [[TMP29]], align 8
+; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE7]]
 ; CHECK:       [[PRED_STORE_CONTINUE5]]:
-; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <2 x i1> [[TMP10]], i64 1
-; CHECK-NEXT:    br i1 [[TMP30]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7]]
-; CHECK:       [[PRED_STORE_IF6]]:
-; CHECK-NEXT:    [[TMP33:%.*]] = getelementptr i8, ptr [[TMP32]], i64 16
-; CHECK-NEXT:    store double 1.000000e+01, ptr [[TMP33]], align 8, !alias.scope [[META129]]
+; CHECK-NEXT:    [[R_1:%.*]] = fadd double [[V_1]], 8.000000e+00
+; CHECK-NEXT:    [[GEP_DST1_ELSE:%.*]] = getelementptr double, ptr [[DST]], i64 [[CLAMPED]]
+; CHECK-NEXT:    store double [[R_1]], ptr [[GEP_DST1_ELSE]], align 8
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE7]]
 ; CHECK:       [[PRED_STORE_CONTINUE7]]:
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 32)
-; CHECK-NEXT:    [[TMP34:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; CHECK-NEXT:    br i1 [[TMP34]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP131:![0-9]+]]
-; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    br [[EXIT:label %.*]]
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[TMP1]], 16
+; CHECK-NEXT:    [[EXIT_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1600
+; CHECK-NEXT:    br i1 [[EXIT_COND]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]]
 ; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop

llvm/test/Transforms/LoopVectorize/interleaved-accesses-requiring-scev-predicates.ll

@@ -117,47 +117,43 @@ exit:
   ret void
 }
 
-; For %gep, we have the following SCEV: ((4 * (zext i4 {0,+,5}<%loop> to i64))<nuw><nsw> + %x).
-; Note the i4 bit wide AddRec {0,+,5}. It is known to wrap in the loop with trip count 16.
+; For %gep, we an SCEV that is known not to wrap in the loop with the given trip count.
 define void @wrap_predicate_for_interleave_group_wraps_for_known_trip_count(ptr noalias %x, ptr noalias %out) {
 ; CHECK-LABEL: define void @wrap_predicate_for_interleave_group_wraps_for_known_trip_count(
 ; CHECK-SAME: ptr noalias [[X:%.*]], ptr noalias [[OUT:%.*]]) {
 ; CHECK-NEXT:  [[START:.*:]]
-; CHECK-NEXT:    br label %[[VECTOR_SCEVCHECK:.*]]
-; CHECK:       [[VECTOR_SCEVCHECK]]:
-; CHECK-NEXT:    [[MUL:%.*]] = call { i4, i1 } @llvm.umul.with.overflow.i4(i4 5, i4 -1)
-; CHECK-NEXT:    [[MUL_OVERFLOW:%.*]] = extractvalue { i4, i1 } [[MUL]], 1
-; CHECK-NEXT:    br i1 [[MUL_OVERFLOW]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = mul nuw nsw i64 [[INDEX]], 5
-; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 15
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i32, ptr [[X]], i64 [[TMP1]]
-; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <20 x i32>, ptr [[TMP2]], align 4
-; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <20 x i32> [[WIDE_VEC]], <20 x i32> poison, <4 x i32> <i32 0, i32 5, i32 10, i32 15>
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i32, ptr [[OUT]], i64 [[INDEX]]
-; CHECK-NEXT:    store <4 x i32> [[STRIDED_VEC]], ptr [[TMP3]], align 4
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = mul nuw nsw <4 x i64> [[VEC_IND]], splat (i64 5)
+; CHECK-NEXT:    [[TMP1:%.*]] = and <4 x i64> [[TMP0]], splat (i64 9)
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i32, ptr [[X]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i32, ptr [[X]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i32, ptr [[X]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i32, ptr [[X]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP6]], align 4
+; CHECK-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP7]], align 4
+; CHECK-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP8]], align 4
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP9]], align 4
+; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <4 x i32> poison, i32 [[TMP10]], i32 0
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP11]], i32 1
+; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <4 x i32> [[TMP15]], i32 [[TMP12]], i32 2
+; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <4 x i32> [[TMP16]], i32 [[TMP13]], i32 3
+; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds nuw i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT:    store <4 x i32> [[TMP17]], ptr [[TMP18]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 12
-; CHECK-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
+; CHECK-NEXT:    br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    br label %[[SCALAR_PH]]
-; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 12, %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
-; CHECK-NEXT:    br label %[[LOOP:.*]]
-; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT:    [[IV_MUL5:%.*]] = mul nuw nsw i64 [[IV]], 5
-; CHECK-NEXT:    [[IV_MUL5_MASKED:%.*]] = and i64 [[IV_MUL5]], 15
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds nuw i32, ptr [[X]], i64 [[IV_MUL5_MASKED]]
-; CHECK-NEXT:    [[V:%.*]] = load i32, ptr [[GEP]], align 4
-; CHECK-NEXT:    [[OUT_I:%.*]] = getelementptr inbounds nuw i32, ptr [[OUT]], i64 [[IV]]
-; CHECK-NEXT:    store i32 [[V]], ptr [[OUT_I]], align 4
-; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 16
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -168,7 +164,7 @@ loop:
   %iv = phi i64 [ 0, %start ], [ %iv.next, %loop ]
   %iv.next = add nuw nsw i64 %iv, 1
   %iv.mul5 = mul nuw nsw i64 %iv, 5
-  %iv.mul5.masked = and i64 %iv.mul5, 15
+  %iv.mul5.masked = and i64 %iv.mul5, 9
   %gep = getelementptr inbounds nuw i32, ptr %x, i64 %iv.mul5.masked
   %v = load i32, ptr %gep, align 4
   %out.i = getelementptr inbounds nuw i32, ptr %out, i64 %iv
@@ -213,7 +209,7 @@ define void @wrap_predicate_for_interleave_group_unknown_trip_count(ptr noalias
 ; CHECK-NEXT:    store <4 x i32> [[STRIDED_VEC]], ptr [[TMP5]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
@@ -229,7 +225,7 @@ define void @wrap_predicate_for_interleave_group_unknown_trip_count(ptr noalias
 ; CHECK-NEXT:    [[OUT_I:%.*]] = getelementptr inbounds nuw i32, ptr [[OUT]], i64 [[IV]]
 ; CHECK-NEXT:    store i32 [[V]], ptr [[OUT_I]], align 4
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;

llvm/test/tools/llubi/gep-16-bit-addrspace.ll

@@ -0,0 +1,18 @@
+; NOTE: Assertions have been autogenerated by utils/update_llubi_test_checks.py UTC_ARGS: --version 6
+; RUN: llubi --verbose < %s 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-p:16:16"
+
+%s = type { [65536 x i8], i8 }
+
+; Offset of the struct's second field does not fit into a 16 bit pointer.
+define void @main() {
+  %p = alloca i32
+  %a = getelementptr %s, ptr %p, i32 0, i32 1
+  ret void
+}
+; CHECK: Entering function: main
+; CHECK-NEXT:   %p = alloca i32, align 4 => ptr 0x8 [p]
+; CHECK-NEXT:   %a = getelementptr %s, ptr %p, i32 0, i32 1 => ptr 0x8 [p]
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: Exiting function: main

llvm/tools/llubi/lib/Context.cpp

@@ -1054,7 +1054,8 @@ Context::computeGEP(GEPOperator &GEP,
       unsigned ElementIdx = cast<ConstantInt>(V)->getZExtValue();
       const StructLayout *SL = DL.getStructLayout(STy);
       // Element offset is in bytes.
-      ApplyScaledOffset(APInt(IndexBitWidth, SL->getElementOffset(ElementIdx)),
+      ApplyScaledOffset(APInt(IndexBitWidth, SL->getElementOffset(ElementIdx),
+                              /*isSigned=*/false, /*implicitTrunc=*/true),
                         APInt(IndexBitWidth, 1));
       continue;
     }

mlir/include/mlir-c/IR.h

@@ -639,9 +639,46 @@ static inline bool mlirOperationIsNull(MlirOperation op) { return !op.ptr; }
 MLIR_CAPI_EXPORTED bool mlirOperationEqual(MlirOperation op,
                                            MlirOperation other);
 
-/// Compute a hash for the given operation.
+/// Compute a hash for the given operation. Operand and result SSA values are
+/// hashed by identity and locations are significant, so equivalent-but-distinct
+/// operations hash differently; use mlirOperationStructuralHashValue for a hash
+/// that pairs with mlirOperationIsStructurallyEquivalent.
 MLIR_CAPI_EXPORTED size_t mlirOperationHashValue(MlirOperation op);
 
+/// Flags controlling structural operation equivalence and hashing. These mirror
+/// `mlir::OperationEquivalence::Flags` and may be combined with bitwise OR.
+typedef enum MlirOperationEquivalenceFlags {
+  /// No flags: locations, discardable attributes, properties and
+  /// commutativity are all significant.
+  MLIR_OPERATION_EQUIVALENCE_NONE = 0,
+  /// Ignore the locations attached to operations.
+  MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS = 1,
+  /// Ignore the discardable attributes attached to operations.
+  MLIR_OPERATION_EQUIVALENCE_IGNORE_DISCARDABLE_ATTRS = 2,
+  /// Ignore the properties attached to operations.
+  MLIR_OPERATION_EQUIVALENCE_IGNORE_PROPERTIES = 4,
+  /// Ignore commutativity, comparing operands in an order-sensitive way.
+  MLIR_OPERATION_EQUIVALENCE_IGNORE_COMMUTATIVITY = 8,
+} MlirOperationEquivalenceFlags;
+
+/// Checks whether two operations are structurally equivalent, i.e. they have
+/// the same name, attributes, operand and result types, and recursively
+/// equivalent regions. Operand equivalence is tracked structurally while
+/// recursing into regions, so operands defined inside the compared regions need
+/// not be the exact same SSA values; operands defined outside must be. `flags`
+/// is a bitwise OR of MlirOperationEquivalenceFlags values.
+MLIR_CAPI_EXPORTED bool mlirOperationIsStructurallyEquivalent(MlirOperation lhs,
+                                                              MlirOperation rhs,
+                                                              uint32_t flags);
+
+/// Computes a hash for the given operation that pairs with
+/// mlirOperationIsStructurallyEquivalent: two operations that are structurally
+/// equivalent under the same `flags` hash equally. Operands are hashed by
+/// identity, results are not hashed at all, and regions do not participate in
+/// the hash. `flags` is a bitwise OR of MlirOperationEquivalenceFlags values.
+MLIR_CAPI_EXPORTED size_t mlirOperationStructuralHashValue(MlirOperation op,
+                                                           uint32_t flags);
+
 /// Gets the context this operation is associated with
 MLIR_CAPI_EXPORTED MlirContext mlirOperationGetContext(MlirOperation op);
 

mlir/include/mlir/Dialect/X86/TransformOps/X86TransformOps.td

@@ -82,5 +82,15 @@ def ApplyVectorContractToAMXDotProductPatternsOp : Op<Transform_Dialect,
   let assemblyFormat = "attr-dict";
 }
 
+def ApplyMoveAccumulatorForContractLoopPatternsOp : Op<Transform_Dialect,
+    "apply_patterns.x86.move_accumulator_for_contract_loop",
+    [DeclareOpInterfaceMethods<PatternDescriptorOpInterface>]> {
+  let description = [{
+         Rewrites an additive vector contraction by zeroing the accumulator input
+         and materializing the accumulation as a separate add after the contraction.
+  }];
+
+  let assemblyFormat = "attr-dict";
+}
 
 #endif // X86_TRANSFORM_OPS

mlir/include/mlir/Dialect/X86/Transforms.h

@@ -110,6 +110,11 @@ void populateShuffleVectorFMAOpsPatterns(RewritePatternSet &patterns);
 // Int8).
 void populateVectorContractToAMXDotProductPatterns(RewritePatternSet &patterns);
 
+// Rewrites an additive vector contraction by zeroing the accumulator input
+// and materializing the accumulation as a separate add after the contraction.
+void populateMoveAccumulatorForContractLoopPatterns(
+    RewritePatternSet &patterns);
+
 //===----------------------------------------------------------------------===//
 /// Helpers extracted from:
 ///   - clang/lib/Headers/avxintrin.h

mlir/include/mlir/Dialect/X86/Utils/X86Utils.h

@@ -29,6 +29,10 @@ namespace x86 {
 bool isInVnniLayout(Operation *op, llvm::ArrayRef<AffineMap> indexingMaps,
                     std::optional<unsigned> blockingFactor = std::nullopt);
 
+// Recursively follows single-use values through scf.yield operations
+// and returns the first non-yield user result in the contraction chain.
+Value contractionUsersAfterYield(Value v);
+
 // Returns true if two contraction ops form a valid pair for VNNI packing.
 // It verifies that both contractions share the appropriate operand, read from
 // the same source buffer, and use constant indices that differ by 8 or 16.
@@ -60,7 +64,7 @@ LogicalResult shuffleAfterReadLikeOp(PatternRewriter &rewriter, Operation *opA,
 // Shuffles vectors produced by vector.contraction ops into a flat layout
 // before they are written to memory.
 LogicalResult shuffleBeforeWriteLikeOp(PatternRewriter &rewriter,
-                                       Operation *opA, Operation *opB,
+                                       Value contractARes, Value contractBRes,
                                        int64_t nonUnitDimAcc, VectorType accTy);
 
 } // namespace x86

mlir/include/mlir/Dialect/XeGPU/IR/XeGPUAttrs.td

@@ -132,6 +132,27 @@ def XeGPU_FenceScopeAttr:
     let assemblyFormat = "$value";
 }
 
+//===----------------------------------------------------------------------===//
+// XeGPU Lane Shuffle Enums.
+//===----------------------------------------------------------------------===//
+def XeGPU_LaneShufflePack: I32EnumAttrCase<"Pack", 0, "pack">;
+def XeGPU_LaneShuffleUnpack: I32EnumAttrCase<"Unpack", 1, "unpack">;
+def XeGPU_LaneShuffleMode: I32EnumAttr<"LaneShuffleMode",
+      "The direction of the lane shuffle performed by xegpu.lane_shuffle.",
+      [XeGPU_LaneShufflePack, XeGPU_LaneShuffleUnpack]> {
+  let genSpecializedAttr = 0;
+  let cppNamespace = "::mlir::xegpu";
+}
+
+def XeGPU_LaneShuffleModeAttr:
+  EnumAttr<XeGPU_Dialect, XeGPU_LaneShuffleMode, "lane_shuffle_mode"> {
+    let summary = [{Describes the direction of a lane shuffle.
+                    "pack" gathers a lane's elements into consecutive logical
+                    positions. "unpack" scatters them back out, strided by the
+                    subgroup size.}];
+    let assemblyFormat = "$value";
+}
+
 def DistributeLayoutAttr: AttrInterface<"DistributeLayoutAttr"> {
   let cppNamespace = "::mlir::xegpu";
   let description = [{

mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td

@@ -1500,6 +1500,78 @@ def XeGPU_TruncfOp
   let hasVerifier = 1;
 }
 
+def XeGPU_LaneShuffleOp
+    : XeGPU_Op<"lane_shuffle", [Pure, AllTypesMatch<["source", "result"]>]> {
+  let summary = "Re-distributes a subgroup's fragments across its lanes.";
+
+  let description = [{
+    `xegpu.lane_shuffle` is a lane-level operation. The `source` operand is the
+    fragment (a 1D vector) held by one lane, and the `result` is the fragment
+    that lane holds afterwards. It re-distributes the elements across the lanes
+    of the subgroup without changing their type, so `source` and `result` have
+    the same type.
+
+    Let `S` be the subgroup size and `N` the number of elements per lane, so the
+    subgroup collectively holds `S * N` elements. Number these elements by their
+    logical position `0 .. S*N-1`. The `mode` selects which of the two
+    orientations of the `S x N` (lane, element) index grid the operation maps
+    between:
+
+    * `pack`: element `j` of lane `i` holds logical position `j * S + i` before
+      the operation and `i * N + j` after it. A lane's elements end up at
+      consecutive logical positions.
+    * `unpack`: the reverse mapping. A lane's elements end up strided by `S`.
+
+    The two modes are exact inverses, so shuffling one way and then the other
+    yields the original fragments.
+
+    For example, consider `vector<2xi16>` with a subgroup size of 8, denoting a
+    lane's fragment as `<a,b>`. In `pack` mode:
+
+    ```
+    lane:       0      1      2      3      4      5      6      7
+    source:   <0,8>  <1,9>  <2,10> <3,11> <4,12> <5,13> <6,14> <7,15>
+    result:   <0,1>  <2,3>  <4,5>  <6,7>  <8,9> <10,11> <12,13> <14,15>
+    ```
+
+    In general each lane's result elements come from `N` different lanes'
+    `source` fragments -- that is the cross-lane shuffle.
+
+    This operation implements the `xegpu.convert_layout` semantics at the lane
+    level, for the case where the `lane_layout` is the same but the `lane_data`
+    is different.
+
+    The operation assumes a full subgroup (the subgroup size equals the maximum
+    subgroup size) and that all lanes execute the same dynamic instance of the
+    operation; otherwise the behavior is undefined.
+
+    Arguments:
+    - `source`: The 1D vector fragment held by one lane.
+    - `mode`: The direction of the shuffle, either `pack` or `unpack`.
+  }];
+
+  let arguments = (ins XeGPU_LaneShuffleType:$source,
+                       XeGPU_LaneShuffleModeAttr:$mode);
+  let results = (outs XeGPU_LaneShuffleType:$result);
+
+  let extraClassDeclaration = [{
+    VectorType getSourceType() {
+      return llvm::cast<VectorType>(getSource().getType());
+    }
+
+    VectorType getResultType() {
+      return llvm::cast<VectorType>(getResult().getType());
+    }
+  }];
+
+  let assemblyFormat = [{
+    $source $mode attr-dict `:` type($source)
+  }];
+
+  let hasVerifier = 1;
+  let hasFolder = 1;
+}
+
 def XeGPU_DpasMxOp : XeGPU_Op<"dpas_mx", [Pure, AttrSizedOperandSegments,
                                           AnchorLayoutInterface]> {
   let summary = "It performs scaled mma computation";

mlir/include/mlir/Dialect/XeGPU/IR/XeGPUTypes.td

@@ -31,6 +31,9 @@ def XeGPU_VectorOrScalarType
     : AnyTypeOf<[VectorOfRankAndType<[1,2,3,4,5,6,7,8], [XeGPU_ScalarType, Index]>, XeGPU_ScalarType]>;
 def XeGPU_GatherScatterBaseAddrType
     : AnyTypeOf<[MemRefRankOf<[XeGPU_ScalarType], [1]>, XeGPU_PointerType]>;
+// Operand/result type of xegpu.lane_shuffle: a 1D vector of numerical type.
+def XeGPU_LaneShuffleType
+    : FixedVectorOfRankAndType<[1], [XeGPU_ScalarType]>;
 
 // common base class for types in XeGPU dialect
 class XeGPUTypeDef<string name, string typeMnemonic, list<Trait> traits = [],

mlir/lib/CAPI/IR/IR.cpp

@@ -661,6 +661,25 @@ size_t mlirOperationHashValue(MlirOperation op) {
   return OperationEquivalence::computeHash(unwrap(op));
 }
 
+/// Translates the C equivalence flags to mlir::OperationEquivalence::Flags. The
+/// enumerator values mirror each other.
+static OperationEquivalence::Flags unwrapEquivalenceFlags(uint32_t flags) {
+  return static_cast<OperationEquivalence::Flags>(flags);
+}
+
+bool mlirOperationIsStructurallyEquivalent(MlirOperation lhs, MlirOperation rhs,
+                                           uint32_t flags) {
+  return OperationEquivalence::isEquivalentTo(unwrap(lhs), unwrap(rhs),
+                                              unwrapEquivalenceFlags(flags));
+}
+
+size_t mlirOperationStructuralHashValue(MlirOperation op, uint32_t flags) {
+  return OperationEquivalence::computeHash(
+      unwrap(op), /*hashOperands=*/OperationEquivalence::directHashValue,
+      /*hashResults=*/OperationEquivalence::ignoreHashValue,
+      unwrapEquivalenceFlags(flags));
+}
+
 MlirContext mlirOperationGetContext(MlirOperation op) {
   return wrap(unwrap(op)->getContext());
 }

mlir/lib/Dialect/X86/TransformOps/X86TransformOps.cpp

@@ -52,6 +52,11 @@ void mlir::transform::ApplyVectorContractToAMXDotProductPatternsOp::
   x86::populateVectorContractToAMXDotProductPatterns(patterns);
 }
 
+void mlir::transform::ApplyMoveAccumulatorForContractLoopPatternsOp::
+    populatePatterns(RewritePatternSet &patterns) {
+  x86::populateMoveAccumulatorForContractLoopPatterns(patterns);
+}
+
 //===----------------------------------------------------------------------===//
 // Transform op registration
 //===----------------------------------------------------------------------===//

mlir/lib/Dialect/X86/Transforms/CMakeLists.txt

@@ -7,6 +7,7 @@ add_mlir_dialect_library(MLIRX86Transforms
   SinkVectorProducerOps.cpp
   ShuffleVectorFMAOps.cpp
   VectorContractToAMXDotProduct.cpp
+  MoveAccumulatorForContractLoop.cpp
 
   LINK_LIBS PUBLIC
   MLIRArithDialect

mlir/lib/Dialect/X86/Transforms/MoveAccumulatorForContractLoop.cpp

@@ -0,0 +1,118 @@
+//===- MoveAccumulatorForContractLoop.cpp ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/SCF/IR/SCF.h"
+#include "mlir/Dialect/Vector/IR/VectorOps.h"
+#include "mlir/Dialect/Vector/Utils/VectorUtils.h"
+#include "mlir/Dialect/X86/Transforms.h"
+#include "mlir/Dialect/X86/Utils/X86Utils.h"
+#include "mlir/Dialect/X86/X86Dialect.h"
+
+#include "mlir/IR/BuiltinAttributes.h"
+#include "mlir/IR/Dominance.h"
+#include "mlir/IR/PatternMatch.h"
+#include "llvm/Support/Casting.h"
+
+#include "mlir/Pass/Pass.h"
+#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+
+using namespace mlir;
+using namespace mlir::vector;
+using namespace mlir::x86;
+
+namespace {
+// Transforms vector.contract(A, B, Acc) into vector.contract(A, B, 0) + Acc
+// to decouple the contraction computation from the accumulator update.
+struct MoveAccumulatorForContractLoop
+    : public OpRewritePattern<vector::ContractionOp> {
+  using OpRewritePattern<vector::ContractionOp>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(vector::ContractionOp contractOp,
+                                PatternRewriter &rewriter) const override {
+
+    if (contractOp.getKind() != vector::CombiningKind::ADD)
+      return rewriter.notifyMatchFailure(contractOp,
+                                         "Expects add combining kind.");
+
+    Operation *accReadOp =
+        traceToVectorReadLikeParentOperation(contractOp.getAcc());
+
+    Value contractValue = contractionUsersAfterYield(contractOp.getResult());
+
+    if (!contractValue)
+      return rewriter.notifyMatchFailure(
+          contractOp, "Final acc write might have multiple users.");
+
+    Operation *resultUserOp = *contractValue.getUsers().begin();
+
+    if (!accReadOp || !resultUserOp)
+      return rewriter.notifyMatchFailure(
+          contractOp, "Read from acc matrix is not by "
+                      "transfer_read/load/constant_zero or multiple users of "
+                      "contract operation.");
+
+    if (isa<arith::ConstantOp>(accReadOp))
+      return rewriter.notifyMatchFailure(
+          contractOp,
+          "The input acc to contract is already a constant vector.");
+
+    if ((accReadOp->getBlock() == contractOp->getBlock()) ||
+        (resultUserOp->getBlock() == contractOp->getBlock()))
+      return rewriter.notifyMatchFailure(
+          contractOp, "Acc read/write should be in a separate block.");
+
+    // Replace acc of a contraction operation with vector constant.
+    Value accValue = accReadOp->getResult(0);
+    if (!accValue.hasOneUse())
+      return rewriter.notifyMatchFailure(
+          contractOp, "The input accumulator has multiple users.");
+
+    Operation *firstUser = *accValue.getUsers().begin();
+    rewriter.setInsertionPoint(firstUser);
+
+    auto vecTy = dyn_cast<VectorType>(accValue.getType());
+    if (!vecTy)
+      return rewriter.notifyMatchFailure(contractOp, "Expects vector type.");
+
+    Location loc = accReadOp->getLoc();
+    Type elemTy = vecTy.getElementType();
+
+    Value zeroVec = arith::ConstantOp::create(
+        rewriter, loc,
+        DenseElementsAttr::get(vecTy, rewriter.getZeroAttr(elemTy)));
+
+    accValue.replaceAllUsesWith(zeroVec);
+
+    // Adds the initial acc value with contract results before storing to acc
+    // matrix.
+    rewriter.setInsertionPoint(resultUserOp);
+    Location locUser = resultUserOp->getLoc();
+
+    Value addition;
+
+    if (isa<FloatType>(elemTy)) {
+      addition =
+          arith::AddFOp::create(rewriter, locUser, contractValue, accValue);
+    } else if (isa<IntegerType>(elemTy)) {
+      addition =
+          arith::AddIOp::create(rewriter, locUser, contractValue, accValue);
+    } else {
+      llvm_unreachable("expected floating-point or integer element type");
+    }
+
+    resultUserOp->replaceUsesOfWith(contractValue, addition);
+    return success();
+  }
+};
+
+} // namespace
+
+void x86::populateMoveAccumulatorForContractLoopPatterns(
+    RewritePatternSet &patterns) {
+  patterns.add<MoveAccumulatorForContractLoop>(patterns.getContext());
+}

mlir/lib/Dialect/X86/Transforms/VectorContractBF16ToFMA.cpp

@@ -415,25 +415,21 @@ struct VectorContractBF16ToFMA
       Operation *accReadOp1 =
           traceToVectorReadLikeParentOperation(pairContractOp.getAcc());
 
-      // Iterate down to find the users of contact operations until it is store
-      // or transfer_write.
-      Operation *resultWriteOp0 =
-          traceToVectorWriteLikeUserOperation(contractOp.getResult());
-      Operation *resultWriteOp1 =
-          traceToVectorWriteLikeUserOperation(pairContractOp.getResult());
+      if (!isa<arith::ConstantOp>(accReadOp0)) {
+        // Shuffle the accumulators of the contract operations.
+        LogicalResult readShuffle =
+            shuffleAfterReadLikeOp(rewriter, accReadOp0, accReadOp1, contractOp,
+                                   pairContractOp, nonUnitDim, accTy);
 
-      // Shuffle the accumulators of the contract operations.
-      LogicalResult readShuffle =
-          shuffleAfterReadLikeOp(rewriter, accReadOp0, accReadOp1, contractOp,
-                                 pairContractOp, nonUnitDim, accTy);
-
-      if (failed(readShuffle))
-        return rewriter.notifyMatchFailure(
-            contractOp, "Accumulator read is not by transfer_read or load");
+        if (failed(readShuffle))
+          return rewriter.notifyMatchFailure(
+              contractOp, "Accumulator read is not by transfer_read or load");
+      }
 
       // Shuffle the output of contract operations before its use.
       LogicalResult writeShuffle = shuffleBeforeWriteLikeOp(
-          rewriter, resultWriteOp0, resultWriteOp1, nonUnitDim, accTy);
+          rewriter, contractOp.getResult(), pairContractOp.getResult(),
+          nonUnitDim, accTy);
 
       if (failed(writeShuffle))
         return rewriter.notifyMatchFailure(

mlir/lib/Dialect/X86/Transforms/VectorContractToPackedTypeDotProduct.cpp

@@ -325,17 +325,17 @@ struct VectorContractToPackedTypeDotProduct
       }
 
       // If the accumulators are shuffled we get nullptr else the
-      // transfer_read or load operations.
-      Operation *accRead =
-          traceToVectorReadLikeParentOperation(contractOp.getAcc());
+      // transfer_load or store operations.
+      Operation *accWrite =
+          traceToVectorWriteLikeUserOperation(contractOp.getResult());
 
       if (!pairContractOp &&
-          (!isNonUnitDimOperandShuffled(nonUnitDimOperand) || accRead))
+          (!isNonUnitDimOperandShuffled(nonUnitDimOperand) || accWrite))
         return rewriter.notifyMatchFailure(contractOp,
                                            "Could not find a contract pair");
 
       // Validate and shuffle the accumulator
-      if (accRead) {
+      if (accWrite) {
         // Trace back to the load or transfer_read operations of the contract
         // accumulators.
         Operation *accReadOp0 =
@@ -352,8 +352,8 @@ struct VectorContractToPackedTypeDotProduct
 
         if (!accReadOp0 || !accReadOp1)
           return rewriter.notifyMatchFailure(
-              contractOp,
-              "Operands doesn't have load or transfer_read as it's parent op");
+              contractOp, "Operands doesn't have load or transfer_read or "
+                          "dense constant attribute as its parent op");
 
         if (!resultWriteOp0 || !resultWriteOp1)
           return rewriter.notifyMatchFailure(
@@ -373,18 +373,22 @@ struct VectorContractToPackedTypeDotProduct
           return rewriter.notifyMatchFailure(
               contractOp, "The store/write operation of contract operation is "
                           "before the pair contract operation");
-        // Shuffle the accumulators of the contract operations.
-        LogicalResult readShuffle =
-            shuffleAfterReadLikeOp(rewriter, accReadOp0, accReadOp1, contractOp,
-                                   pairContractOp, nonUnitDimValue, accTy);
 
-        if (failed(readShuffle))
-          return rewriter.notifyMatchFailure(
-              contractOp, "Accumulator read is not by transfer_read or load");
+        if (!isa<arith::ConstantOp>(accReadOp0)) {
+          // Shuffle the accumulators of the contract operations.
+          LogicalResult readShuffle = shuffleAfterReadLikeOp(
+              rewriter, accReadOp0, accReadOp1, contractOp, pairContractOp,
+              nonUnitDimValue, accTy);
+
+          if (failed(readShuffle))
+            return rewriter.notifyMatchFailure(
+                contractOp, "Accumulator read is not by transfer_read or load");
+        }
 
         // Shuffle the output of contract operations before it's use.
         LogicalResult writeShuffle = shuffleBeforeWriteLikeOp(
-            rewriter, resultWriteOp0, resultWriteOp1, nonUnitDimValue, accTy);
+            rewriter, contractOp.getResult(), pairContractOp.getResult(),
+            nonUnitDimValue, accTy);
 
         if (failed(writeShuffle))
           return rewriter.notifyMatchFailure(

mlir/lib/Dialect/X86/Utils/X86Utils.cpp

@@ -144,6 +144,25 @@ inline ShuffleMasks getShuffleMasks(int64_t nonUnitDimAcc, bool isInt8Avx2) {
   return {maskLo8, maskHi8};
 }
 
+// Recursively follows single-use values through scf.yield operations
+// and returns the first non-yield user result in the contraction chain.
+Value contractionUsersAfterYield(Value v) {
+  if (!v || v.getNumUses() != 1)
+    return nullptr;
+
+  OpOperand &use = *v.use_begin();
+  Operation *user = use.getOwner();
+
+  if (!isa<scf::YieldOp>(user))
+    return v;
+
+  auto yield = cast<scf::YieldOp>(user);
+  Operation *parent = yield->getParentOp();
+  unsigned idx = use.getOperandNumber();
+
+  return contractionUsersAfterYield(parent->getResult(idx));
+}
+
 // This function walks backward from a value to locate its originating
 // vector read-like operation (`vector.transfer_read` or `vector.load`).
 // It follows simple forwarding through unary ops and across `scf.for`
@@ -155,8 +174,28 @@ Operation *traceToVectorReadLikeParentOperation(Value v) {
   while (true) {
     // Case 1: Value defined by an operation
     if (Operation *defOp = v.getDefiningOp()) {
-      if (isa<vector::TransferReadOp, vector::LoadOp>(defOp))
+      if (isa<vector::TransferReadOp, vector::LoadOp, arith::ConstantOp>(
+              defOp)) {
+
+        if (auto constOp = dyn_cast<arith::ConstantOp>(defOp)) {
+          if (auto denseAttr =
+                  dyn_cast<DenseElementsAttr>(constOp.getValue())) {
+            if (!denseAttr.isSplat())
+              return nullptr;
+
+            Attribute splat = denseAttr.getSplatValue<Attribute>();
+
+            if (auto floatAttr = dyn_cast<FloatAttr>(splat))
+              return floatAttr.getValue().isZero() ? defOp : nullptr;
+
+            if (auto intAttr = dyn_cast<IntegerAttr>(splat))
+              return intAttr.getValue().isZero() ? defOp : nullptr;
+          }
+
+          return nullptr;
+        }
         return defOp;
+      }
 
       return nullptr;
     }
@@ -290,26 +329,23 @@ LogicalResult shuffleAfterReadLikeOp(PatternRewriter &rewriter, Operation *opA,
 // This function shuffles the vectors written by vector.contract operation
 // as a flat layout structure before they are stored.
 LogicalResult shuffleBeforeWriteLikeOp(PatternRewriter &rewriter,
-                                       Operation *opA, Operation *opB,
+                                       Value contractARes, Value contractBRes,
                                        int64_t nonUnitDimAcc,
                                        VectorType accTy) {
-  // Helper to extract vector operand from write-like ops
-  auto getWrittenVector = [](Operation *op) -> Value {
-    if (auto write = dyn_cast<vector::TransferWriteOp>(op))
-      return write.getVector();
-    if (auto store = dyn_cast<vector::StoreOp>(op))
-      return store.getValueToStore();
-    return nullptr;
-  };
 
-  Value vecA = getWrittenVector(opA);
-  Value vecB = getWrittenVector(opB);
+  Value vecA = contractionUsersAfterYield(contractARes);
+  Value vecB = contractionUsersAfterYield(contractBRes);
 
   if (!vecA || !vecB)
     return failure();
 
+  Operation *resultWriteOpA = *vecA.getUsers().begin();
+  Operation *resultWriteOpB = *vecB.getUsers().begin();
+
   // Decide insertion point and location
-  Operation *insertBefore = opA->isBeforeInBlock(opB) ? opA : opB;
+  Operation *insertBefore = resultWriteOpA->isBeforeInBlock(resultWriteOpB)
+                                ? resultWriteOpA
+                                : resultWriteOpB;
 
   rewriter.setInsertionPoint(insertBefore);
   Location loc = insertBefore->getLoc();
@@ -335,10 +371,8 @@ LogicalResult shuffleBeforeWriteLikeOp(PatternRewriter &rewriter,
   auto newVecB = vector::ShapeCastOp::create(rewriter, loc, accTy, shuffledHi);
 
   // Update write operands in place via the rewriter to notify it of changes.
-  rewriter.modifyOpInPlace(opA,
-                           [&]() { opA->setOperand(0, newVecA.getResult()); });
-  rewriter.modifyOpInPlace(opB,
-                           [&]() { opB->setOperand(0, newVecB.getResult()); });
+  resultWriteOpA->replaceUsesOfWith(vecA, newVecA);
+  resultWriteOpB->replaceUsesOfWith(vecB, newVecB);
 
   return success();
 }

mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp

@@ -946,6 +946,29 @@ LogicalResult TruncfOp::verify() {
   return success();
 }
 
+//===----------------------------------------------------------------------===//
+// XeGPU_LaneShuffleOp
+//===----------------------------------------------------------------------===//
+
+LogicalResult LaneShuffleOp::verify() {
+  // With a single element per lane there is nothing to re-distribute, so the
+  // operation would be a no-op.
+  if (getSourceType().getNumElements() < 2)
+    return emitOpError("requires a source vector with at least 2 elements.");
+
+  return success();
+}
+
+OpFoldResult LaneShuffleOp::fold(FoldAdaptor adaptor) {
+  // The two modes are exact inverses, so a pack feeding an unpack (or vice
+  // versa) restores the original fragments.
+  auto producer = getSource().getDefiningOp<LaneShuffleOp>();
+  if (producer && producer.getMode() != getMode())
+    return producer.getSource();
+
+  return {};
+}
+
 //===----------------------------------------------------------------------===//
 // XeGPU_DpasMxOp
 //===----------------------------------------------------------------------===//

mlir/test/CAPI/ir.c

@@ -3034,6 +3034,187 @@ int testReplaceUsesWithIf(MlirContext ctx) {
   return 0;
 }
 
+int testOperationEquivalence(MlirContext ctx) {
+  fprintf(stderr, "@testOperationEquivalence\n");
+  // CHECK-LABEL: @testOperationEquivalence
+
+  mlirContextGetOrLoadDialect(ctx, mlirStringRefCreateFromCString("arith"));
+
+  const char *moduleStr = "func.func @f(%arg0: i32) -> i32 {\n"
+                          "  %0 = arith.constant 42 : i32\n"
+                          "  %1 = arith.constant 42 : i32\n"
+                          "  %2 = arith.constant 7 : i32\n"
+                          "  %3 = arith.subi %0, %2 : i32\n"
+                          "  %4 = arith.subi %0, %2 : i32\n"
+                          "  %5 = arith.subi %2, %0 : i32\n"
+                          "  %6 = arith.subi %0, %2 {dialect.discardable} : "
+                          "i32\n"
+                          "  %7 = arith.addi %0, %2 : i32\n"
+                          "  %8 = arith.addi %2, %0 : i32\n"
+                          "  return %0 : i32\n"
+                          "}\n";
+  MlirModule module =
+      mlirModuleCreateParse(ctx, mlirStringRefCreateFromCString(moduleStr));
+
+  MlirBlock moduleBody = mlirModuleGetBody(module);
+  MlirOperation funcOp = mlirBlockGetFirstOperation(moduleBody);
+  MlirRegion funcRegion = mlirOperationGetRegion(funcOp, 0);
+  MlirBlock funcBody = mlirRegionGetFirstBlock(funcRegion);
+
+  MlirOperation c42a = mlirBlockGetFirstOperation(funcBody);
+  MlirOperation c42b = mlirOperationGetNextInBlock(c42a);
+  MlirOperation c7 = mlirOperationGetNextInBlock(c42b);
+  MlirOperation sub3 = mlirOperationGetNextInBlock(c7);
+  MlirOperation sub4 = mlirOperationGetNextInBlock(sub3);
+  MlirOperation sub5 = mlirOperationGetNextInBlock(sub4);
+  MlirOperation sub6 = mlirOperationGetNextInBlock(sub5);
+  MlirOperation add7 = mlirOperationGetNextInBlock(sub6);
+  MlirOperation add8 = mlirOperationGetNextInBlock(add7);
+
+  // Two identical constants are structurally equivalent when locations are
+  // ignored, even though their result SSA values and source locations differ.
+  assert(mlirOperationIsStructurallyEquivalent(
+      c42a, c42b, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
+
+  // Without ignoring locations they differ, since they sit on distinct lines.
+  assert(!mlirOperationIsStructurallyEquivalent(
+      c42a, c42b, MLIR_OPERATION_EQUIVALENCE_NONE));
+
+  // A constant with a different value is not equivalent.
+  assert(!mlirOperationIsStructurallyEquivalent(
+      c42a, c7, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
+
+  // Equivalence is reflexive.
+  assert(mlirOperationIsStructurallyEquivalent(
+      c42a, c42a, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
+
+  // `%3 = subi %0, %2` and `%4 = subi %0, %2` use the exact same operands, so
+  // they are equivalent.
+  assert(mlirOperationIsStructurallyEquivalent(
+      sub3, sub4, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
+
+  // `%5 = subi %2, %0` is not equivalent to `%3 = subi %0, %2`: `subi` is not
+  // commutative, so operands are compared pairwise, and the very first pair
+  // (`%0` = 42 vs `%2` = 7) already mismatches. See the `addi` pair below for
+  // the commutative path.
+  assert(!mlirOperationIsStructurallyEquivalent(
+      sub3, sub5, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
+
+  // A constant and a subtraction are different operations.
+  assert(!mlirOperationIsStructurallyEquivalent(
+      c42a, sub3, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
+
+  // `%6 = subi %0, %2 {dialect.discardable}` carries a discardable attribute
+  // that `%3 = subi %0, %2` does not, so the two differ unless discardable
+  // attributes are ignored as well. This also exercises OR-ing flags together.
+  assert(!mlirOperationIsStructurallyEquivalent(
+      sub3, sub6, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
+  assert(mlirOperationIsStructurallyEquivalent(
+      sub3, sub6,
+      MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS |
+          MLIR_OPERATION_EQUIVALENCE_IGNORE_DISCARDABLE_ATTRS));
+
+  // `%7 = addi %0, %2` and `%8 = addi %2, %0` have swapped operands, but `addi`
+  // is commutative, so they are equivalent through the commutative path and
+  // only differ once commutativity is explicitly ignored.
+  assert(mlirOperationIsStructurallyEquivalent(
+      add7, add8, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
+  assert(!mlirOperationIsStructurallyEquivalent(
+      add7, add8,
+      MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS |
+          MLIR_OPERATION_EQUIVALENCE_IGNORE_COMMUTATIVITY));
+
+  // The `value` of arith.constant is an inherent attribute held in the
+  // operation's properties, so ignoring properties makes constants with
+  // different values equivalent.
+  assert(mlirOperationIsStructurallyEquivalent(
+      c42a, c7,
+      MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS |
+          MLIR_OPERATION_EQUIVALENCE_IGNORE_PROPERTIES));
+
+  // The structural hash pairs with the equivalence above: operations that are
+  // equivalent under a set of flags hash equally under the same flags.
+  assert(mlirOperationStructuralHashValue(
+             c42a, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS) ==
+         mlirOperationStructuralHashValue(
+             c42b, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
+  assert(mlirOperationStructuralHashValue(
+             sub3, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS) ==
+         mlirOperationStructuralHashValue(
+             sub4, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
+  assert(mlirOperationStructuralHashValue(
+             sub3, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS |
+                       MLIR_OPERATION_EQUIVALENCE_IGNORE_DISCARDABLE_ATTRS) ==
+         mlirOperationStructuralHashValue(
+             sub6, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS |
+                       MLIR_OPERATION_EQUIVALENCE_IGNORE_DISCARDABLE_ATTRS));
+  // Commutative operands are folded into the hash in an order-insensitive way,
+  // matching the commutative equivalence above.
+  assert(mlirOperationStructuralHashValue(
+             add7, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS) ==
+         mlirOperationStructuralHashValue(
+             add8, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
+
+  // The flags are threaded through to the hash: locations and discardable
+  // attributes are hashed unless the corresponding flag is set. Hash
+  // inequality is not part of the contract, but these inputs do differ.
+  assert(
+      mlirOperationStructuralHashValue(c42a, MLIR_OPERATION_EQUIVALENCE_NONE) !=
+      mlirOperationStructuralHashValue(c42b, MLIR_OPERATION_EQUIVALENCE_NONE));
+  assert(mlirOperationStructuralHashValue(
+             sub3, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS) !=
+         mlirOperationStructuralHashValue(
+             sub6, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
+
+  mlirModuleDestroy(module);
+
+  // Operands defined *inside* the compared regions need not be the same SSA
+  // values: recursing into the regions marks the two inner constants
+  // equivalent, and the `addi` operands are then matched through that mapping.
+  const char *regionModuleStr = "func.func @g() {\n"
+                                "  %r0 = scf.execute_region -> i32 {\n"
+                                "    %c = arith.constant 1 : i32\n"
+                                "    %d = arith.addi %c, %c : i32\n"
+                                "    scf.yield %d : i32\n"
+                                "  }\n"
+                                "  %r1 = scf.execute_region -> i32 {\n"
+                                "    %c = arith.constant 1 : i32\n"
+                                "    %d = arith.addi %c, %c : i32\n"
+                                "    scf.yield %d : i32\n"
+                                "  }\n"
+                                "  return\n"
+                                "}\n";
+  MlirModule regionModule = mlirModuleCreateParse(
+      ctx, mlirStringRefCreateFromCString(regionModuleStr));
+  assert(!mlirModuleIsNull(regionModule));
+  MlirBlock regionFuncBody = mlirRegionGetFirstBlock(mlirOperationGetRegion(
+      mlirBlockGetFirstOperation(mlirModuleGetBody(regionModule)), 0));
+  MlirOperation exec0 = mlirBlockGetFirstOperation(regionFuncBody);
+  MlirOperation exec1 = mlirOperationGetNextInBlock(exec0);
+  assert(mlirOperationIsStructurallyEquivalent(
+      exec0, exec1, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
+
+  // The nested `addi` operations compared on their own are *not* equivalent:
+  // the value mapping starts out empty, so operands defined outside the
+  // compared operation must be the exact same SSA values.
+  MlirBlock exec0Body =
+      mlirRegionGetFirstBlock(mlirOperationGetRegion(exec0, 0));
+  MlirBlock exec1Body =
+      mlirRegionGetFirstBlock(mlirOperationGetRegion(exec1, 0));
+  MlirOperation innerAdd0 =
+      mlirOperationGetNextInBlock(mlirBlockGetFirstOperation(exec0Body));
+  MlirOperation innerAdd1 =
+      mlirOperationGetNextInBlock(mlirBlockGetFirstOperation(exec1Body));
+  assert(!mlirOperationIsStructurallyEquivalent(
+      innerAdd0, innerAdd1, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
+
+  mlirModuleDestroy(regionModule);
+
+  // CHECK: testOperationEquivalence: PASSED
+  fprintf(stderr, "testOperationEquivalence: PASSED\n");
+  return 0;
+}
+
 int main(void) {
   MlirContext ctx = mlirContextCreate();
   registerAllUpstreamDialects(ctx);
@@ -3091,6 +3272,8 @@ int main(void) {
     return 20;
   if (testReplaceUsesWithIf(ctx))
     return 21;
+  if (testOperationEquivalence(ctx))
+    return 22;
 
   // CHECK: DESTROY MAIN CONTEXT
   // CHECK: reportResourceDelete: resource_i64_blob

mlir/test/Dialect/X86/move-acc-for-contract-loop.mlir

@@ -0,0 +1,323 @@
+// RUN: mlir-opt %s -transform-interpreter -cse -split-input-file | FileCheck %s
+
+!vecA = vector<1x1x2xbf16>
+!vecB = vector<1x2x16xbf16>
+!vecC = vector<1x16xf32>
+!memrefA = memref<1x1x2xbf16, strided<[2048, 32, 1], offset: ?>>
+!memrefB = memref<1x2x32xbf16, strided<[2048, 64, 1], offset: ?>>
+!memrefC = memref<1x32xf32, strided<[64, 1], offset: ?>>
+#map = affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>
+#map1 = affine_map<(d0, d1, d2, d3) -> (d0, d3, d2)>
+#map2 = affine_map<(d0, d1, d2, d3) -> (d1, d2)>
+
+func.func @brmatmul_acc_mv(%arg0: memref<16x64x32xbf16>, %arg1: memref<16x32x64xbf16>,
+                             %arg2: memref<64x64xf32>) -> memref<64x64xf32> {
+  %0 = ub.poison : f32
+  %1 = ub.poison : bf16
+  %c0 = arith.constant 0 : index
+  %c64 = arith.constant 64 : index
+  %c16 = arith.constant 16 : index
+  %c32 = arith.constant 32 : index
+  %c1 = arith.constant 1 : index
+  %c2 = arith.constant 2 : index
+  scf.for %arg3 = %c0 to %c64 step %c1 {
+    scf.for %arg4 = %c0 to %c64 step %c32 {
+      %subview = memref.subview %arg2[%arg3, %arg4] [1, 32] [1, 1]
+                        : memref<64x64xf32> to !memrefC
+      %2 = vector.transfer_read %subview[%c0, %c0], %0 {in_bounds = [true, true]}
+                        : !memrefC, !vecC
+      %3 = vector.transfer_read %subview[%c0, %c16], %0 {in_bounds = [true, true]}
+                        : !memrefC, !vecC
+
+      %4:2 = scf.for %arg5 = %c0 to %c16 step %c1 iter_args(%arg6 = %2, %arg7 = %3) -> (!vecC, !vecC) {
+        %5:2 = scf.for %arg8 = %c0 to %c32 step %c2 iter_args(%arg9 = %arg6, %arg10 = %arg7) -> (!vecC, !vecC) {
+
+          %subview_0 = memref.subview %arg0[%arg5, %arg3, %arg8] [1, 1, 2] [1, 1, 1]
+                        : memref<16x64x32xbf16> to !memrefA
+          %subview_1 = memref.subview %arg1[%arg5, %arg8, %arg4] [1, 2, 32] [1, 1, 1]
+                        : memref<16x32x64xbf16> to !memrefB
+
+          %6 = vector.transfer_read %subview_0[%c0, %c0, %c0], %1 {in_bounds = [true, true, true]}
+                        : !memrefA, !vecA
+          %7 = vector.transfer_read %subview_1[%c0, %c0, %c0], %1 {in_bounds = [true, true, true]}
+                        : !memrefB, !vecB
+          %8 = vector.transfer_read %subview_1[%c0, %c0, %c16], %1 {in_bounds = [true, true, true]}
+                        : !memrefB, !vecB
+
+          %9 = vector.contract {indexing_maps = [#map, #map1, #map2], iterator_types =
+                        ["reduction", "parallel", "parallel", "reduction"], kind = #vector.kind<add>} %6, %7, %arg9
+                        {unroll_shape = array<i64: 1, 1, 16, 2>} : !vecA, !vecB into !vecC
+          %10 = vector.contract {indexing_maps = [#map, #map1, #map2], iterator_types =
+                        ["reduction", "parallel", "parallel", "reduction"], kind = #vector.kind<add>} %6, %8, %arg10
+                        {unroll_shape = array<i64: 1, 1, 16, 2>} : !vecA, !vecB into !vecC
+
+          scf.yield %9, %10 : !vecC, !vecC
+        }
+        scf.yield %5#0, %5#1 : !vecC, !vecC
+      }
+
+      vector.transfer_write %4#1, %subview[%c0, %c16] {in_bounds = [true, true]}
+                        : !vecC, !memrefC
+      vector.transfer_write %4#0, %subview[%c0, %c0] {in_bounds = [true, true]}
+                        : !vecC, !memrefC
+    }
+  }
+
+  return %arg2 : memref<64x64xf32>
+}
+
+// CHECK-LABEL: @brmatmul_acc_mv
+// CHECK: %[[CST:.*]] = arith.constant dense<0.000000e+00> : vector<1x16xf32>
+// CHECK: scf.for
+// CHECK-NEXT: scf.for
+// CHECK: memref.subview
+// CHECK: %[[TRANSFER_READ_0:.*]] = vector.transfer_read
+// CHECK: %[[TRANSFER_READ_1:.*]] = vector.transfer_read
+// CHECK: scf.for {{.*}} iter_args(%[[VAL_3:.*]] = %[[CST]], %[[VAL_4:.*]] = %[[CST]]) -> (vector<1x16xf32>, vector<1x16xf32>) {
+// CHECK-NEXT: scf.for
+// CHECK: scf.yield
+// CHECK: scf.yield
+// CHECK: %[[ADDF_0:.*]] = arith.addf {{.*}}, %[[TRANSFER_READ_1]] : vector<1x16xf32>
+// CHECK-NEXT: vector.transfer_write %[[ADDF_0:.*]], %subview[%c0, %c16] {{.*}}
+// CHECK: %[[ADDF_1:.*]] = arith.addf {{.*}}, %[[TRANSFER_READ_0]] : vector<1x16xf32>
+// CHECK-NEXT: vector.transfer_write %[[ADDF_1:.*]], %subview[%c0, %c0] {{.*}}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %func = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    transform.apply_patterns to %func {
+      transform.apply_patterns.x86.move_accumulator_for_contract_loop
+    } : !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+!vecA = vector<1x16x16x2xbf16>
+!vecB = vector<1x16x16x2xbf16>
+!vecC = vector<16x16xf32>
+!memrefA = memref<1x32x16x2xbf16>
+!memrefB = memref<1x16x32x2xbf16>
+!memrefC = memref<32x32xf32>
+#map = affine_map<(d0, d4, d1, d2, d3) -> (d0, d1, d3, d4)>
+#map1 = affine_map<(d0, d4, d1, d2, d3) -> (d0, d3, d2, d4)>
+#map2 = affine_map<(d0, d4, d1, d2, d3) -> (d1, d2)>
+func.func @negative_no_loop(
+  %arg0: !memrefA, %arg1: !memrefB, %arg2: !memrefC) -> !memrefC
+{
+  %c0 = arith.constant 0 : index
+  %0 = ub.poison : bf16
+  %32 = ub.poison : f32
+
+  %1 = vector.transfer_read %arg0[%c0, %c0, %c0, %c0], %0 {in_bounds = [true, true, true, true]} :
+        !memrefA, !vecA
+  %2 = vector.transfer_read %arg1[%c0, %c0, %c0, %c0], %0 {in_bounds = [true, true, true, true]} :
+        !memrefB, !vecB
+
+  %3 = vector.transfer_read %arg2[%c0, %c0], %32 {in_bounds = [true, true]} : !memrefC, !vecC
+
+  %4 = vector.contract {
+    indexing_maps = [#map, #map1, #map2],
+    iterator_types = ["reduction", "reduction", "parallel", "parallel", "reduction"],
+    kind = #vector.kind<add>}
+    %1, %2, %3 : !vecA, !vecB into !vecC
+
+  vector.transfer_write %4, %arg2[%c0, %c0] {in_bounds = [true, true]} : !vecC, !memrefC
+
+  return %arg2 : !memrefC
+}
+
+// CHECK-LABEL: @negative_no_loop
+// CHECK-NOT: arith.constant dense<0.000000e+00> {{.*}}
+// CHECK-NOT: arith.addf
+// CHECK-NOT: arith.addf
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %func = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    transform.apply_patterns to %func {
+      transform.apply_patterns.x86.move_accumulator_for_contract_loop
+    } : !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+!vecA = vector<1x1x2xbf16>
+!vecB = vector<1x2x16xbf16>
+!vecC = vector<1x16xf32>
+!memrefA = memref<1x1x2xbf16, strided<[2048, 32, 1], offset: ?>>
+!memrefB = memref<1x2x32xbf16, strided<[2048, 64, 1], offset: ?>>
+!memrefC = memref<1x32xf32, strided<[64, 1], offset: ?>>
+#map = affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>
+#map1 = affine_map<(d0, d1, d2, d3) -> (d0, d3, d2)>
+#map2 = affine_map<(d0, d1, d2, d3) -> (d1, d2)>
+
+func.func @negative_non_zero_cst(%arg0: memref<16x64x32xbf16>, %arg1: memref<16x32x64xbf16>,
+                             %arg2: memref<64x64xf32>) -> memref<64x64xf32> {
+  %cst2 = arith.constant dense<2.000000e+00> : vector<1x16xf32>
+  %cst3 = arith.constant dense<3.000000e+00> : vector<1x16xf32>
+  %0 = ub.poison : f32
+  %1 = ub.poison : bf16
+  %c0 = arith.constant 0 : index
+  %c64 = arith.constant 64 : index
+  %c16 = arith.constant 16 : index
+  %c32 = arith.constant 32 : index
+  %c1 = arith.constant 1 : index
+  %c2 = arith.constant 2 : index
+  scf.for %arg3 = %c0 to %c64 step %c1 {
+    scf.for %arg4 = %c0 to %c64 step %c32 {
+      %subview = memref.subview %arg2[%arg3, %arg4] [1, 32] [1, 1]
+                        : memref<64x64xf32> to !memrefC
+
+      %4:2 = scf.for %arg5 = %c0 to %c16 step %c1 iter_args(%arg6 = %cst2, %arg7 = %cst3) -> (!vecC, !vecC) {
+        %5:2 = scf.for %arg8 = %c0 to %c32 step %c2 iter_args(%arg9 = %arg6, %arg10 = %arg7) -> (!vecC, !vecC) {
+
+          %subview_0 = memref.subview %arg0[%arg5, %arg3, %arg8] [1, 1, 2] [1, 1, 1]
+                        : memref<16x64x32xbf16> to !memrefA
+          %subview_1 = memref.subview %arg1[%arg5, %arg8, %arg4] [1, 2, 32] [1, 1, 1]
+                        : memref<16x32x64xbf16> to !memrefB
+
+          %6 = vector.transfer_read %subview_0[%c0, %c0, %c0], %1 {in_bounds = [true, true, true]}
+                        : !memrefA, !vecA
+          %7 = vector.transfer_read %subview_1[%c0, %c0, %c0], %1 {in_bounds = [true, true, true]}
+                        : !memrefB, !vecB
+          %8 = vector.transfer_read %subview_1[%c0, %c0, %c16], %1 {in_bounds = [true, true, true]}
+                        : !memrefB, !vecB
+
+          %9 = vector.contract {indexing_maps = [#map, #map1, #map2], iterator_types =
+                        ["reduction", "parallel", "parallel", "reduction"], kind = #vector.kind<add>} %6, %7, %arg9
+                        {unroll_shape = array<i64: 1, 1, 16, 2>} : !vecA, !vecB into !vecC
+          %10 = vector.contract {indexing_maps = [#map, #map1, #map2], iterator_types =
+                        ["reduction", "parallel", "parallel", "reduction"], kind = #vector.kind<add>} %6, %8, %arg10
+                        {unroll_shape = array<i64: 1, 1, 16, 2>} : !vecA, !vecB into !vecC
+
+          scf.yield %9, %10 : !vecC, !vecC
+        }
+        scf.yield %5#0, %5#1 : !vecC, !vecC
+      }
+
+      vector.transfer_write %4#1, %subview[%c0, %c16] {in_bounds = [true, true]}
+                        : !vecC, !memrefC
+      vector.transfer_write %4#0, %subview[%c0, %c0] {in_bounds = [true, true]}
+                        : !vecC, !memrefC
+    }
+  }
+
+  return %arg2 : memref<64x64xf32>
+}
+
+// CHECK-LABEL: @negative_non_zero_cst
+// CHECK-NOT: %[[CST:.*]] = arith.constant dense<0.000000e+00> : vector<1x16xf32>
+// CHECK: scf.for
+// CHECK-NEXT: scf.for
+// CHECK: memref.subview
+// CHECK: scf.for 
+// CHECK-NEXT: scf.for
+// CHECK: scf.yield
+// CHECK: scf.yield
+// CHECK-NOT: arith.addf 
+
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %func = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    transform.apply_patterns to %func {
+      transform.apply_patterns.x86.move_accumulator_for_contract_loop
+    } : !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+!vecA = vector<1x1x2xbf16>
+!vecB = vector<1x2x16xbf16>
+!vecC = vector<1x16xf32>
+!memrefA = memref<1x1x2xbf16, strided<[2048, 32, 1], offset: ?>>
+!memrefB = memref<1x2x32xbf16, strided<[2048, 64, 1], offset: ?>>
+!memrefC = memref<1x32xf32, strided<[64, 1], offset: ?>>
+#map = affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>
+#map1 = affine_map<(d0, d1, d2, d3) -> (d0, d3, d2)>
+#map2 = affine_map<(d0, d1, d2, d3) -> (d1, d2)>
+
+func.func @negative_input_acc_many_users(%arg0: memref<16x64x32xbf16>, %arg1: memref<16x32x64xbf16>,
+                             %arg2: memref<64x64xf32>) -> memref<64x64xf32> {
+  %0 = ub.poison : f32
+  %1 = ub.poison : bf16
+  %c0 = arith.constant 0 : index
+  %c64 = arith.constant 64 : index
+  %c16 = arith.constant 16 : index
+  %c32 = arith.constant 32 : index
+  %c1 = arith.constant 1 : index
+  %c2 = arith.constant 2 : index
+  scf.for %arg3 = %c0 to %c64 step %c1 {
+    scf.for %arg4 = %c0 to %c64 step %c32 {
+      %subview = memref.subview %arg2[%arg3, %arg4] [1, 32] [1, 1]
+                        : memref<64x64xf32> to !memrefC
+      %2 = vector.transfer_read %subview[%c0, %c0], %0 {in_bounds = [true, true]}
+                        : !memrefC, !vecC
+      %3 = vector.transfer_read %subview[%c0, %c16], %0 {in_bounds = [true, true]}
+                        : !memrefC, !vecC
+
+      %4:2 = scf.for %arg5 = %c0 to %c16 step %c1 iter_args(%arg6 = %2, %arg7 = %3) -> (!vecC, !vecC) {
+        %5:2 = scf.for %arg8 = %c0 to %c32 step %c2 iter_args(%arg9 = %arg6, %arg10 = %arg7) -> (!vecC, !vecC) {
+
+          %subview_0 = memref.subview %arg0[%arg5, %arg3, %arg8] [1, 1, 2] [1, 1, 1]
+                        : memref<16x64x32xbf16> to !memrefA
+          %subview_1 = memref.subview %arg1[%arg5, %arg8, %arg4] [1, 2, 32] [1, 1, 1]
+                        : memref<16x32x64xbf16> to !memrefB
+
+          %6 = vector.transfer_read %subview_0[%c0, %c0, %c0], %1 {in_bounds = [true, true, true]}
+                        : !memrefA, !vecA
+          %7 = vector.transfer_read %subview_1[%c0, %c0, %c0], %1 {in_bounds = [true, true, true]}
+                        : !memrefB, !vecB
+          %8 = vector.transfer_read %subview_1[%c0, %c0, %c16], %1 {in_bounds = [true, true, true]}
+                        : !memrefB, !vecB
+
+          %9 = vector.contract {indexing_maps = [#map, #map1, #map2], iterator_types =
+                        ["reduction", "parallel", "parallel", "reduction"], kind = #vector.kind<add>} %6, %7, %arg9
+                        {unroll_shape = array<i64: 1, 1, 16, 2>} : !vecA, !vecB into !vecC
+          %10 = vector.contract {indexing_maps = [#map, #map1, #map2], iterator_types =
+                        ["reduction", "parallel", "parallel", "reduction"], kind = #vector.kind<add>} %6, %8, %arg10
+                        {unroll_shape = array<i64: 1, 1, 16, 2>} : !vecA, !vecB into !vecC
+
+          scf.yield %9, %10 : !vecC, !vecC
+        }
+        scf.yield %5#0, %5#1 : !vecC, !vecC
+      }
+
+      %7 = arith.addf %4#1, %2 : vector<1x16xf32>
+      vector.transfer_write %7, %subview[%c0, %c16] {in_bounds = [true, true]}
+                        : !vecC, !memrefC
+      %8 = arith.addf %4#0, %3 : vector<1x16xf32>
+      vector.transfer_write %8, %subview[%c0, %c0] {in_bounds = [true, true]}
+                        : !vecC, !memrefC
+    }
+  }
+
+  return %arg2 : memref<64x64xf32>
+}
+// CHECK-LABEL: @negative_input_acc_many_users
+// CHECK-NOT: %[[CST:.*]] = arith.constant dense<0.000000e+00> : vector<1x16xf32>
+// CHECK: scf.for
+// CHECK-NEXT: scf.for
+// CHECK: memref.subview
+// CHECK: scf.for
+// CHECK-NEXT: scf.for
+// CHECK: scf.yield
+// CHECK: scf.yield
+
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %func = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    transform.apply_patterns to %func {
+      transform.apply_patterns.x86.move_accumulator_for_contract_loop
+    } : !transform.any_op
+    transform.yield
+  }
+}
+

mlir/test/Dialect/X86/vector-contract-bf16-to-fma.mlir

@@ -444,8 +444,6 @@ func.func @matmul_to_fma_flat_layout_loop(%arg0: memref<16x64x32xbf16>, %arg1: m
 }
 
 // CHECK-LABEL: @matmul_to_fma_flat_layout_loop
-// CHECK: vector.shuffle{{.*}}[0, 8, 1, 9, 2, 10, 3, 11] : vector<8xf32>, vector<8xf32>
-// CHECK-NEXT: vector.shuffle{{.*}}[4, 12, 5, 13, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
 // CHECK: scf.for
 // CHECK: scf.for
 // CHECK: x86.avx.bcst_to_f32.packed
@@ -461,6 +459,7 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %func = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     transform.apply_patterns to %func {
+      transform.apply_patterns.x86.move_accumulator_for_contract_loop
       transform.apply_patterns.x86.vector_contract_bf16_to_fma
     } : !transform.any_op
     transform.yield

mlir/test/Dialect/X86/vector-contract-to-packed-type-dotproduct.mlir

@@ -687,8 +687,6 @@ func.func @brmatmul_bf16dp_flat_layout_loop(%arg0: memref<16x64x32xbf16>, %arg1:
 }
 
 // CHECK-LABEL: @brmatmul_bf16dp_flat_layout_loop
-// CHECK: vector.shuffle{{.*}}[0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23] : vector<16xf32>, vector<16xf32>
-// CHECK-NEXT: vector.shuffle{{.*}}[8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31] : vector<16xf32>, vector<16xf32>
 // CHECK: scf.for
 // CHECK: scf.for
 // CHECK: vector.transfer_read {{.*}} vector<32xbf16>
@@ -706,6 +704,7 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %func = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     transform.apply_patterns to %func {
+      transform.apply_patterns.x86.move_accumulator_for_contract_loop
       transform.apply_patterns.x86.vector_contract_to_packed_type_dot_product
     } : !transform.any_op
     transform.yield

mlir/test/Dialect/XeGPU/canonicalize.mlir

@@ -0,0 +1,36 @@
+// RUN: mlir-opt --canonicalize --split-input-file %s | FileCheck %s
+
+// CHECK-LABEL: func.func @fold_lane_shuffle_pack_unpack
+// CHECK-SAME: %[[ARG0:.+]]: vector<2xi16>
+// CHECK-NOT: xegpu.lane_shuffle
+// CHECK: return %[[ARG0]]
+func.func @fold_lane_shuffle_pack_unpack(%a: vector<2xi16>) -> vector<2xi16> {
+  %0 = xegpu.lane_shuffle %a pack : vector<2xi16>
+  %1 = xegpu.lane_shuffle %0 unpack : vector<2xi16>
+  return %1 : vector<2xi16>
+}
+
+// -----
+
+// CHECK-LABEL: func.func @fold_lane_shuffle_unpack_pack
+// CHECK-SAME: %[[ARG0:.+]]: vector<4xf8E5M2>
+// CHECK-NOT: xegpu.lane_shuffle
+// CHECK: return %[[ARG0]]
+func.func @fold_lane_shuffle_unpack_pack(%a: vector<4xf8E5M2>) -> vector<4xf8E5M2> {
+  %0 = xegpu.lane_shuffle %a unpack : vector<4xf8E5M2>
+  %1 = xegpu.lane_shuffle %0 pack : vector<4xf8E5M2>
+  return %1 : vector<4xf8E5M2>
+}
+
+// -----
+
+// Two shuffles in the same direction are not inverses and must not fold.
+
+// CHECK-LABEL: func.func @no_fold_lane_shuffle_pack_pack
+// CHECK: xegpu.lane_shuffle {{.*}} pack
+// CHECK: xegpu.lane_shuffle {{.*}} pack
+func.func @no_fold_lane_shuffle_pack_pack(%a: vector<2xi16>) -> vector<2xi16> {
+  %0 = xegpu.lane_shuffle %a pack : vector<2xi16>
+  %1 = xegpu.lane_shuffle %0 pack : vector<2xi16>
+  return %1 : vector<2xi16>
+}

mlir/test/Dialect/XeGPU/invalid.mlir

@@ -693,6 +693,13 @@ func.func @truncf_invalid_result_size(%a: vector<8x16xf16>) {
   return
 }
 
+// -----
+func.func @lane_shuffle_single_element(%a: vector<1xi32>) {
+  // expected-error@+1 {{op requires a source vector with at least 2 elements}}
+  %1 = xegpu.lane_shuffle %a pack : vector<1xi32>
+  return
+}
+
 // -----
 func.func @dpas_mx_acc_result_type_mismatch(%a : vector<8x16xf8E5M2>, %b: vector<16x16xf8E5M2>, %acc: vector<8x16xbf16>) {
   // expected-error@+1 {{Accumulator type must match result type.}}

mlir/test/Dialect/XeGPU/ops.mlir

@@ -697,6 +697,48 @@ gpu.func @truncf(%a: vector<8x16xf16>) {
   gpu.return
 }
 
+// CHECK-LABEL: gpu.func @lane_shuffle_pack_f4
+gpu.func @lane_shuffle_pack_f4(%a: vector<4xf4E2M1FN>) {
+  // CHECK: %{{.+}} = xegpu.lane_shuffle %{{.+}} pack : vector<4xf4E2M1FN>
+  %1 = xegpu.lane_shuffle %a pack : vector<4xf4E2M1FN>
+  gpu.return
+}
+
+// CHECK-LABEL: gpu.func @lane_shuffle_unpack_f4
+gpu.func @lane_shuffle_unpack_f4(%a: vector<4xf4E2M1FN>) {
+  // CHECK: %{{.+}} = xegpu.lane_shuffle %{{.+}} unpack : vector<4xf4E2M1FN>
+  %1 = xegpu.lane_shuffle %a unpack : vector<4xf4E2M1FN>
+  gpu.return
+}
+
+// CHECK-LABEL: gpu.func @lane_shuffle_pack_f8
+gpu.func @lane_shuffle_pack_f8(%a: vector<4xf8E5M2>) {
+  // CHECK: %{{.+}} = xegpu.lane_shuffle %{{.+}} pack : vector<4xf8E5M2>
+  %1 = xegpu.lane_shuffle %a pack : vector<4xf8E5M2>
+  gpu.return
+}
+
+// CHECK-LABEL: gpu.func @lane_shuffle_unpack_f8
+gpu.func @lane_shuffle_unpack_f8(%a: vector<4xf8E5M2>) {
+  // CHECK: %{{.+}} = xegpu.lane_shuffle %{{.+}} unpack : vector<4xf8E5M2>
+  %1 = xegpu.lane_shuffle %a unpack : vector<4xf8E5M2>
+  gpu.return
+}
+
+// CHECK-LABEL: gpu.func @lane_shuffle_pack_f16
+gpu.func @lane_shuffle_pack_f16(%a: vector<2xf16>) {
+  // CHECK: %{{.+}} = xegpu.lane_shuffle %{{.+}} pack : vector<2xf16>
+  %1 = xegpu.lane_shuffle %a pack : vector<2xf16>
+  gpu.return
+}
+
+// CHECK-LABEL: gpu.func @lane_shuffle_unpack_f16
+gpu.func @lane_shuffle_unpack_f16(%a: vector<2xf16>) {
+  // CHECK: %{{.+}} = xegpu.lane_shuffle %{{.+}} unpack : vector<2xf16>
+  %1 = xegpu.lane_shuffle %a unpack : vector<2xf16>
+  gpu.return
+}
+
 // CHECK-LABEL: gpu.func @dpas_mx
 gpu.func @dpas_mx(%a : vector<8x32xf8E5M2>, %b: vector<32x16xf8E5M2>, %acc: vector<8x16xbf16>, %a_scale: vector<8x1xf8E8M0FNU>, %b_scale: vector<1x16xf8E8M0FNU>) {
   // CHECK: %{{.+}} = xegpu.dpas_mx %{{.+}}, %{{.+}}, %{{.+}} scale_a = %{{.+}} scale_b = %{{.+}} : (vector<8x32xf8E5M2>, vector<32x16xf8E5M2>, vector<8x16xbf16>, vector<8x1xf8E8M0FNU>, vector<1x16xf8E8M0FNU>) -> vector<8x16xbf16>

utils/bazel/llvm-project-overlay/libc/BUILD.bazel

@@ -997,6 +997,7 @@ libc_support_library(
         ":__support_macros_config",
         ":__support_macros_properties_architectures",
         ":__support_macros_properties_compiler",
+        ":__support_macros_properties_os",
         ":hdr_stdint_proxy",
     ],
 )