[compiler-rt][builtins] libc-backed double-float comparison builtins

compiler-rt/lib/builtins/CMakeLists.txt

@@ -288,6 +288,8 @@ if(COMPILER_RT_USE_LIBC_MATH)
   use_libc_builtin(GENERIC_SOURCES    adddf3)
   use_libc_builtin(GENERIC_SOURCES    addsf3)
   use_libc_builtin(GENERIC_TF_SOURCES addtf3)
+  use_libc_builtin(GENERIC_SOURCES    comparedf2)
+  use_libc_builtin(GENERIC_SOURCES    comparesf2)
   use_libc_builtin(GENERIC_SOURCES    divdf3)
   use_libc_builtin(GENERIC_SOURCES    divsf3)
   use_libc_builtin(GENERIC_TF_SOURCES divtf3)

compiler-rt/lib/builtins/comparedf2.cpp

@@ -0,0 +1,54 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file implements compiler-rt's double comparison routines
+/// (__ledf2/__gedf2/__unorddf2 and their aliases) on top of LLVM-libc's shared
+/// comparison builtins.
+///
+//===----------------------------------------------------------------------===//
+
+#define DOUBLE_PRECISION
+#include "fp_lib.h"
+
+#include "fp_compare_impl.inc"
+#include "fp_libc_config.h"
+#include "shared/builtins/gedf2.h"
+#include "shared/builtins/ledf2.h"
+#include "shared/builtins/unorddf2.h"
+
+extern "C" {
+
+COMPILER_RT_ABI CMP_RESULT __ledf2(fp_t a, fp_t b) {
+  return LIBC_NAMESPACE::shared::ledf2(a, b);
+}
+#if defined(__ELF__)
+COMPILER_RT_ALIAS(__ledf2, __cmpdf2)
+#endif
+COMPILER_RT_ALIAS(__ledf2, __eqdf2)
+COMPILER_RT_ALIAS(__ledf2, __ltdf2)
+COMPILER_RT_ALIAS(__ledf2, __nedf2)
+
+COMPILER_RT_ABI CMP_RESULT __gedf2(fp_t a, fp_t b) {
+  return LIBC_NAMESPACE::shared::gedf2(a, b);
+}
+COMPILER_RT_ALIAS(__gedf2, __gtdf2)
+
+COMPILER_RT_ABI CMP_RESULT __unorddf2(fp_t a, fp_t b) {
+  return LIBC_NAMESPACE::shared::unorddf2(a, b);
+}
+
+#if defined(__ARM_EABI__)
+#if defined(COMPILER_RT_ARMHF_TARGET)
+AEABI_RTABI int __aeabi_dcmpun(fp_t a, fp_t b) { return __unorddf2(a, b); }
+#else
+COMPILER_RT_ALIAS(__unorddf2, __aeabi_dcmpun)
+#endif
+#endif
+
+} // extern "C"

compiler-rt/lib/builtins/comparesf2.cpp

@@ -0,0 +1,54 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file implements compiler-rt's float comparison routines
+/// (__lesf2/__gesf2/__unordsf2 and their aliases) on top of LLVM-libc's shared
+/// comparison builtins.
+///
+//===----------------------------------------------------------------------===//
+
+#define SINGLE_PRECISION
+#include "fp_lib.h"
+
+#include "fp_compare_impl.inc"
+#include "fp_libc_config.h"
+#include "shared/builtins/gesf2.h"
+#include "shared/builtins/lesf2.h"
+#include "shared/builtins/unordsf2.h"
+
+extern "C" {
+
+COMPILER_RT_ABI CMP_RESULT __lesf2(fp_t a, fp_t b) {
+  return LIBC_NAMESPACE::shared::lesf2(a, b);
+}
+#if defined(__ELF__)
+COMPILER_RT_ALIAS(__lesf2, __cmpsf2)
+#endif
+COMPILER_RT_ALIAS(__lesf2, __eqsf2)
+COMPILER_RT_ALIAS(__lesf2, __ltsf2)
+COMPILER_RT_ALIAS(__lesf2, __nesf2)
+
+COMPILER_RT_ABI CMP_RESULT __gesf2(fp_t a, fp_t b) {
+  return LIBC_NAMESPACE::shared::gesf2(a, b);
+}
+COMPILER_RT_ALIAS(__gesf2, __gtsf2)
+
+COMPILER_RT_ABI CMP_RESULT __unordsf2(fp_t a, fp_t b) {
+  return LIBC_NAMESPACE::shared::unordsf2(a, b);
+}
+
+#if defined(__ARM_EABI__)
+#if defined(COMPILER_RT_ARMHF_TARGET)
+AEABI_RTABI int __aeabi_fcmpun(fp_t a, fp_t b) { return __unordsf2(a, b); }
+#else
+COMPILER_RT_ALIAS(__unordsf2, __aeabi_fcmpun)
+#endif
+#endif
+
+} // extern "C"

libc/shared/builtins.h

@@ -27,6 +27,10 @@
 #include "builtins/extendsfdf2.h"
 #include "builtins/extendsftf2.h"
 #include "builtins/extendxftf2.h"
+#include "builtins/gedf2.h"
+#include "builtins/gesf2.h"
+#include "builtins/ledf2.h"
+#include "builtins/lesf2.h"
 #include "builtins/muldf3.h"
 #include "builtins/mulsf3.h"
 #include "builtins/multf3.h"
@@ -39,5 +43,7 @@
 #include "builtins/trunctfdf2.h"
 #include "builtins/trunctfsf2.h"
 #include "builtins/trunctfxf2.h"
+#include "builtins/unorddf2.h"
+#include "builtins/unordsf2.h"
 
 #endif // LLVM_LIBC_SHARED_BUILTINS_H

libc/shared/builtins/gedf2.h

@@ -0,0 +1,29 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This header exposes LLVM-libc's __gedf2 implementation as shared::gedf2 so
+/// that it can be reused by compiler-rt's builtins.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SHARED_BUILTINS_GEDF2_H
+#define LLVM_LIBC_SHARED_BUILTINS_GEDF2_H
+
+#include "shared/libc_common.h"
+#include "src/__support/builtins/gedf2.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace shared {
+
+using builtins::gedf2;
+
+} // namespace shared
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SHARED_BUILTINS_GEDF2_H

libc/shared/builtins/gesf2.h

@@ -0,0 +1,29 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This header exposes LLVM-libc's __gesf2 implementation as shared::gesf2 so
+/// that it can be reused by compiler-rt's builtins.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SHARED_BUILTINS_GESF2_H
+#define LLVM_LIBC_SHARED_BUILTINS_GESF2_H
+
+#include "shared/libc_common.h"
+#include "src/__support/builtins/gesf2.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace shared {
+
+using builtins::gesf2;
+
+} // namespace shared
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SHARED_BUILTINS_GESF2_H

libc/shared/builtins/ledf2.h

@@ -0,0 +1,29 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This header exposes LLVM-libc's __ledf2 implementation as shared::ledf2 so
+/// that it can be reused by compiler-rt's builtins.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SHARED_BUILTINS_LEDF2_H
+#define LLVM_LIBC_SHARED_BUILTINS_LEDF2_H
+
+#include "shared/libc_common.h"
+#include "src/__support/builtins/ledf2.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace shared {
+
+using builtins::ledf2;
+
+} // namespace shared
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SHARED_BUILTINS_LEDF2_H

libc/shared/builtins/lesf2.h

@@ -0,0 +1,29 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This header exposes LLVM-libc's __lesf2 implementation as shared::lesf2 so
+/// that it can be reused by compiler-rt's builtins.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SHARED_BUILTINS_LESF2_H
+#define LLVM_LIBC_SHARED_BUILTINS_LESF2_H
+
+#include "shared/libc_common.h"
+#include "src/__support/builtins/lesf2.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace shared {
+
+using builtins::lesf2;
+
+} // namespace shared
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SHARED_BUILTINS_LESF2_H

libc/shared/builtins/unorddf2.h

@@ -0,0 +1,29 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This header exposes LLVM-libc's __unorddf2 implementation as
+/// shared::unorddf2 so that it can be reused by compiler-rt's builtins.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SHARED_BUILTINS_UNORDDF2_H
+#define LLVM_LIBC_SHARED_BUILTINS_UNORDDF2_H
+
+#include "shared/libc_common.h"
+#include "src/__support/builtins/unorddf2.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace shared {
+
+using builtins::unorddf2;
+
+} // namespace shared
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SHARED_BUILTINS_UNORDDF2_H

libc/shared/builtins/unordsf2.h

@@ -0,0 +1,29 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This header exposes LLVM-libc's __unordsf2 implementation as
+/// shared::unordsf2 so that it can be reused by compiler-rt's builtins.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SHARED_BUILTINS_UNORDSF2_H
+#define LLVM_LIBC_SHARED_BUILTINS_UNORDSF2_H
+
+#include "shared/libc_common.h"
+#include "src/__support/builtins/unordsf2.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace shared {
+
+using builtins::unordsf2;
+
+} // namespace shared
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SHARED_BUILTINS_UNORDSF2_H

libc/src/__support/builtins/CMakeLists.txt

@@ -221,5 +221,66 @@ add_header_library(
     truncdfsf2.h
   DEPENDS
     libc.src.__support.builtins.fpconvert_helper
+  cmp_helper
+  HDRS
+    cmp_helper.h
+  DEPENDS
+    libc.src.__support.FPUtil.comparison_operations
+    libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.macros.attributes
+    libc.src.__support.macros.config
+)
+
+add_header_library(
+  lesf2
+  HDRS
+    lesf2.h
+  DEPENDS
+    libc.src.__support.builtins.cmp_helper
+    libc.src.__support.macros.config
+)
+
+add_header_library(
+  gesf2
+  HDRS
+    gesf2.h
+  DEPENDS
+    libc.src.__support.builtins.cmp_helper
+    libc.src.__support.macros.config
+)
+
+add_header_library(
+  unordsf2
+  HDRS
+    unordsf2.h
+  DEPENDS
+    libc.src.__support.builtins.cmp_helper
+    libc.src.__support.macros.config
+)
+
+add_header_library(
+  ledf2
+  HDRS
+    ledf2.h
+  DEPENDS
+    libc.src.__support.builtins.cmp_helper
+    libc.src.__support.macros.config
+)
+
+add_header_library(
+  gedf2
+  HDRS
+    gedf2.h
+  DEPENDS
+    libc.src.__support.builtins.cmp_helper
+    libc.src.__support.macros.config
+)
+
+add_header_library(
+  unorddf2
+  HDRS
+    unorddf2.h
+  DEPENDS
+    libc.src.__support.builtins.cmp_helper
     libc.src.__support.macros.config
 )

libc/src/__support/builtins/cmp_helper.h

@@ -0,0 +1,63 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Shared floating-point comparison results for compiler-rt's soft-float
+/// compare builtins.  These mirror compiler-rt's GCC-compatible __le<f>2 /
+/// __ge<f>2 / __unord<f>2 (quiet: they raise no FP exceptions), reusing
+/// LLVM-libc's IEEE comparison predicates, so they can be reused by
+/// compiler-rt's builtins.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_BUILTINS_CMP_HELPER_H
+#define LLVM_LIBC_SRC___SUPPORT_BUILTINS_CMP_HELPER_H
+
+#include "src/__support/FPUtil/FPBits.h"
+#include "src/__support/FPUtil/comparison_operations.h"
+#include "src/__support/macros/attributes.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace builtins {
+
+// GCC soft-float compare, "LE" convention: -1 if a<b, 0 if a==b, 1 if a>b, and
+// 1 when unordered (either operand is NaN).  Quiet -- raises no FP exceptions,
+// matching compiler-rt's __le<f>2 (which __eq/__ne/__lt/__cmp alias).  NaN is
+// handled up front so the ordering predicates are only reached for non-NaN
+// inputs, where they never signal.
+template <typename T> LIBC_INLINE int cmp_le(T a, T b) {
+  using FPBits = fputil::FPBits<T>;
+  if (FPBits(a).is_nan() || FPBits(b).is_nan())
+    return 1; // LE_UNORDERED
+  if (fputil::equals(a, b))
+    return 0;                              // LE_EQUAL
+  return fputil::less_than(a, b) ? -1 : 1; // LE_LESS : LE_GREATER
+}
+
+// Same as cmp_le, but unordered yields -1; matches compiler-rt's __ge<f>2
+// (which __gt aliases).
+template <typename T> LIBC_INLINE int cmp_ge(T a, T b) {
+  using FPBits = fputil::FPBits<T>;
+  if (FPBits(a).is_nan() || FPBits(b).is_nan())
+    return -1; // GE_UNORDERED
+  if (fputil::equals(a, b))
+    return 0;                                 // GE_EQUAL
+  return fputil::greater_than(a, b) ? 1 : -1; // GE_GREATER : GE_LESS
+}
+
+// 1 iff either operand is NaN, else 0; matches compiler-rt's __unord<f>2.
+template <typename T> LIBC_INLINE int cmp_unord(T a, T b) {
+  using FPBits = fputil::FPBits<T>;
+  return FPBits(a).is_nan() || FPBits(b).is_nan();
+}
+
+} // namespace builtins
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_BUILTINS_CMP_HELPER_H

libc/src/__support/builtins/gedf2.h

@@ -0,0 +1,30 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This header exposes LLVM-libc's __gedf2 implementation as builtins::gedf2 so
+/// that it can be reused by compiler-rt's builtins.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_BUILTINS_GEDF2_H
+#define LLVM_LIBC_SRC___SUPPORT_BUILTINS_GEDF2_H
+
+#include "src/__support/builtins/cmp_helper.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace builtins {
+
+// Greater-equal comparison of double; mirrors compiler-rt's __gedf2.
+LIBC_INLINE int gedf2(double a, double b) { return cmp_ge(a, b); }
+
+} // namespace builtins
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_BUILTINS_GEDF2_H

libc/src/__support/builtins/gesf2.h

@@ -0,0 +1,30 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This header exposes LLVM-libc's __gesf2 implementation as builtins::gesf2 so
+/// that it can be reused by compiler-rt's builtins.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_BUILTINS_GESF2_H
+#define LLVM_LIBC_SRC___SUPPORT_BUILTINS_GESF2_H
+
+#include "src/__support/builtins/cmp_helper.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace builtins {
+
+// Greater-equal comparison of float; mirrors compiler-rt's __gesf2.
+LIBC_INLINE int gesf2(float a, float b) { return cmp_ge(a, b); }
+
+} // namespace builtins
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_BUILTINS_GESF2_H

libc/src/__support/builtins/ledf2.h

@@ -0,0 +1,30 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This header exposes LLVM-libc's __ledf2 implementation as builtins::ledf2 so
+/// that it can be reused by compiler-rt's builtins.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_BUILTINS_LEDF2_H
+#define LLVM_LIBC_SRC___SUPPORT_BUILTINS_LEDF2_H
+
+#include "src/__support/builtins/cmp_helper.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace builtins {
+
+// Less-equal comparison of double; mirrors compiler-rt's __ledf2.
+LIBC_INLINE int ledf2(double a, double b) { return cmp_le(a, b); }
+
+} // namespace builtins
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_BUILTINS_LEDF2_H

libc/src/__support/builtins/lesf2.h

@@ -0,0 +1,30 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This header exposes LLVM-libc's __lesf2 implementation as builtins::lesf2 so
+/// that it can be reused by compiler-rt's builtins.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_BUILTINS_LESF2_H
+#define LLVM_LIBC_SRC___SUPPORT_BUILTINS_LESF2_H
+
+#include "src/__support/builtins/cmp_helper.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace builtins {
+
+// Less-equal comparison of float; mirrors compiler-rt's __lesf2.
+LIBC_INLINE int lesf2(float a, float b) { return cmp_le(a, b); }
+
+} // namespace builtins
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_BUILTINS_LESF2_H

libc/src/__support/builtins/unorddf2.h

@@ -0,0 +1,30 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This header exposes LLVM-libc's __unorddf2 implementation as
+/// builtins::unorddf2 so that it can be reused by compiler-rt's builtins.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_BUILTINS_UNORDDF2_H
+#define LLVM_LIBC_SRC___SUPPORT_BUILTINS_UNORDDF2_H
+
+#include "src/__support/builtins/cmp_helper.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace builtins {
+
+// Unordered comparison of double; mirrors compiler-rt's __unorddf2.
+LIBC_INLINE int unorddf2(double a, double b) { return cmp_unord(a, b); }
+
+} // namespace builtins
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_BUILTINS_UNORDDF2_H

libc/src/__support/builtins/unordsf2.h

@@ -0,0 +1,30 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This header exposes LLVM-libc's __unordsf2 implementation as
+/// builtins::unordsf2 so that it can be reused by compiler-rt's builtins.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_BUILTINS_UNORDSF2_H
+#define LLVM_LIBC_SRC___SUPPORT_BUILTINS_UNORDSF2_H
+
+#include "src/__support/builtins/cmp_helper.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace builtins {
+
+// Unordered comparison of float; mirrors compiler-rt's __unordsf2.
+LIBC_INLINE int unordsf2(float a, float b) { return cmp_unord(a, b); }
+
+} // namespace builtins
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_BUILTINS_UNORDSF2_H

libc/src/__support/common.h

@@ -17,12 +17,6 @@
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/properties/architectures.h"
 #include "src/__support/macros/properties/compiler.h"
-#include "src/__support/macros/properties/os.h"
-
-#if defined(__APPLE__) || (defined(LIBC_TARGET_OS_IS_WINDOWS) &&               \
-                           defined(LIBC_TARGET_ARCH_IS_X86_32))
-#define LIBC_TARGET_USES_LEADING_UNDERSCORE
-#endif
 
 #ifndef LLVM_LIBC_FUNCTION_ATTR
 #define LLVM_LIBC_FUNCTION_ATTR
@@ -50,24 +44,25 @@
 
 #define LLVM_LIBC_ATTR(name) EXPAND_THEN_SECOND(LLVM_LIBC_FUNCTION_ATTR_##name)
 
-// Export both `func` and `LIBC_NAMESPACE::func` using an alias symbol.
-// This does not work on platfors with LIBC_TARGET_USES_LEADING_UNDERSCORE
-// so there this only exports `_func`.
+// At the moment, [[gnu::alias()]] is not supported on MacOS, and it is needed
+// to cleanly export and alias the C++ symbol `LIBC_NAMESPACE::func` with the C
+// symbol `func`.  So for public packaging on MacOS, we will only export the C
+// symbol.  Moreover, a C symbol `func` in macOS is mangled as `_func`.
 #if defined(LIBC_COPT_PUBLIC_PACKAGING) && !defined(LIBC_COMPILER_IS_MSVC)
-#ifndef LIBC_TARGET_USES_LEADING_UNDERSCORE
+#ifndef __APPLE__
 #define LLVM_LIBC_FUNCTION_IMPL_4(type, name, arglist, c_alias)                \
   LLVM_LIBC_ATTR(name)                                                         \
   LLVM_LIBC_FUNCTION_ATTR decltype(LIBC_NAMESPACE::name)                       \
       __##name##_impl__ asm(c_alias);                                          \
   decltype(LIBC_NAMESPACE::name) name [[gnu::alias(c_alias)]];                 \
   type __##name##_impl__ arglist
-#else // LIBC_TARGET_USES_LEADING_UNDERSCORE
+#else // __APPLE__
 #define LLVM_LIBC_FUNCTION_IMPL_4(type, name, arglist, c_alias)                \
   LLVM_LIBC_ATTR(name)                                                         \
   LLVM_LIBC_FUNCTION_ATTR decltype(LIBC_NAMESPACE::name) name asm(             \
       "_" c_alias);                                                            \
   type name arglist
-#endif // LIBC_TARGET_USES_LEADING_UNDERSCORE
+#endif // __APPLE__
 
 #else  // LIBC_COPT_PUBLIC_PACKAGING
 #define LLVM_LIBC_FUNCTION_IMPL_4(type, name, arglist, c_alias)                \
@@ -83,23 +78,25 @@
   GET_FIFTH(__VA_ARGS__, LLVM_LIBC_FUNCTION_IMPL_4, LLVM_LIBC_FUNCTION_IMPL_3, \
             GET_NOTHING)(__VA_ARGS__)
 
-// See comment on LLVM_LIBC_FUNCTION_IMPL_4 for why this checks
-// LIBC_TARGET_USES_LEADING_UNDERSCORE.
+// At the moment, [[gnu::alias()]] is not supported on MacOS, and it is needed
+// to cleanly export and alias the C++ symbol `LIBC_NAMESPACE::func` with the C
+// symbol `func`.  So for public packaging on MacOS, we will only export the C
+// symbol.  Moreover, a C symbol `func` in macOS is mangled as `_func`.
 #if defined(LIBC_COPT_PUBLIC_PACKAGING) && !defined(LIBC_COMPILER_IS_MSVC)
-#ifndef LIBC_TARGET_USES_LEADING_UNDERSCORE
+#ifndef __APPLE__
 #define LLVM_LIBC_VARIABLE_IMPL(type, name)                                    \
   LLVM_LIBC_ATTR(name)                                                         \
   extern LLVM_LIBC_VARIABLE_ATTR decltype(LIBC_NAMESPACE::name)                \
       __##name##_impl__ asm(#name);                                            \
   extern decltype(LIBC_NAMESPACE::name) name [[gnu::alias(#name)]];            \
   type __##name##_impl__
-#else // LIBC_TARGET_USES_LEADING_UNDERSCORE
+#else // __APPLE__
 #define LLVM_LIBC_VARIABLE_IMPL(type, name)                                    \
   LLVM_LIBC_ATTR(name)                                                         \
   extern LLVM_LIBC_VARIABLE_ATTR decltype(LIBC_NAMESPACE::name) name asm(      \
       "_" #name);                                                              \
   type name
-#endif // LIBC_TARGET_USES_LEADING_UNDERSCORE
+#endif // __APPLE__
 #else  // LIBC_COPT_PUBLIC_PACKAGING
 #define LLVM_LIBC_VARIABLE_IMPL(type, name) type name
 #endif // LIBC_COPT_PUBLIC_PACKAGING

libc/src/__support/math/expm1.h

@@ -126,13 +126,13 @@ LIBC_INLINE DoubleDouble poly_approx_dd(const DoubleDouble &dx) {
 }
 
 #ifdef DEBUGDEBUG
-LIBC_INLINE std::ostream &operator<<(std::ostream &OS, const DFloat128 &r) {
+std::ostream &operator<<(std::ostream &OS, const DFloat128 &r) {
   OS << (r.sign == Sign::NEG ? "-(" : "(") << r.mantissa.val[0] << " + "
      << r.mantissa.val[1] << " * 2^64) * 2^" << r.exponent << "\n";
   return OS;
 }
 
-LIBC_INLINE std::ostream &operator<<(std::ostream &OS, const DoubleDouble &r) {
+std::ostream &operator<<(std::ostream &OS, const DoubleDouble &r) {
   OS << std::hexfloat << "(" << r.hi << " + " << r.lo << ")"
      << std::defaultfloat << "\n";
   return OS;

libc/src/__support/math/tan.h

@@ -108,9 +108,8 @@ LIBC_INLINE double tan_eval(const DoubleDouble &u, DoubleDouble &result) {
 // Calculation a / b = a * (1/b) for DFloat128.
 // Using the initial approximation of q ~ (1/b), then apply 2 Newton-Raphson
 // iterations, before multiplying by a.
-[[maybe_unused]] LIBC_INLINE DFloat128 newton_raphson_div(const DFloat128 &a,
-                                                          DFloat128 b,
-                                                          double q) {
+[[maybe_unused]] DFloat128 newton_raphson_div(const DFloat128 &a, DFloat128 b,
+                                              double q) {
   DFloat128 q0(q);
   LIBC_BIT_CAST_CONSTEXPR_VAR DFloat128 TWO(2.0);
   b.sign = (b.sign == Sign::POS) ? Sign::NEG : Sign::POS;

libc/test/shared/CMakeLists.txt

@@ -838,6 +838,10 @@ add_fp_unittest(
     libc.src.__support.builtins.extendsfdf2
     libc.src.__support.builtins.extendsftf2
     libc.src.__support.builtins.extendxftf2
+    libc.src.__support.builtins.gedf2
+    libc.src.__support.builtins.gesf2
+    libc.src.__support.builtins.ledf2
+    libc.src.__support.builtins.lesf2
     libc.src.__support.builtins.muldf3
     libc.src.__support.builtins.mulsf3
     libc.src.__support.builtins.multf3
@@ -850,6 +854,8 @@ add_fp_unittest(
     libc.src.__support.builtins.trunctfdf2
     libc.src.__support.builtins.trunctfsf2
     libc.src.__support.builtins.trunctfxf2
+    libc.src.__support.builtins.unorddf2
+    libc.src.__support.builtins.unordsf2
 )
 
 add_fp_unittest(

libc/test/shared/shared_builtins_test.cpp

@@ -60,3 +60,33 @@ TEST(LlvmLibcSharedBuiltinsTest, TruncateConversion) {
 #endif // LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80
 #endif // LIBC_TYPES_HAS_FLOAT128
 }
+
+TEST(LlvmLibcSharedBuiltinsTest, SingleCompare) {
+  const float aNaN =
+      LIBC_NAMESPACE::fputil::FPBits<float>::quiet_nan().get_val();
+  EXPECT_EQ(-1, shared::gesf2(1.0f, 2.0f));
+  EXPECT_EQ(0, shared::gesf2(1.0f, 1.0f));
+  EXPECT_EQ(1, shared::gesf2(2.0f, 1.0f));
+  EXPECT_EQ(-1, shared::gesf2(aNaN, 1.0f));
+  EXPECT_EQ(-1, shared::lesf2(1.0f, 2.0f));
+  EXPECT_EQ(0, shared::lesf2(1.0f, 1.0f));
+  EXPECT_EQ(1, shared::lesf2(2.0f, 1.0f));
+  EXPECT_EQ(1, shared::lesf2(aNaN, 1.0f));
+  EXPECT_EQ(0, shared::unordsf2(1.0f, 2.0f));
+  EXPECT_EQ(1, shared::unordsf2(aNaN, 1.0f));
+}
+
+TEST_F(LlvmLibcSharedBuiltinsTest, DoubleCompare) {
+  const double aNaN =
+      LIBC_NAMESPACE::fputil::FPBits<double>::quiet_nan().get_val();
+  EXPECT_EQ(-1, shared::gedf2(1.0, 2.0));
+  EXPECT_EQ(0, shared::gedf2(1.0, 1.0));
+  EXPECT_EQ(1, shared::gedf2(2.0, 1.0));
+  EXPECT_EQ(-1, shared::gedf2(aNaN, 1.0));
+  EXPECT_EQ(-1, shared::ledf2(1.0, 2.0));
+  EXPECT_EQ(0, shared::ledf2(1.0, 1.0));
+  EXPECT_EQ(1, shared::ledf2(2.0, 1.0));
+  EXPECT_EQ(1, shared::ledf2(aNaN, 1.0));
+  EXPECT_EQ(0, shared::unorddf2(1.0, 2.0));
+  EXPECT_EQ(1, shared::unorddf2(aNaN, 1.0));
+}

lld/ELF/Arch/SPARCV9.cpp

@@ -166,20 +166,6 @@ void SPARCV9::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels,
       expr = R_GOT_OFF;
       break;
 
-    // Optimize the GOT load to an add of the symbol's GOT-relative address if
-    // applicable. Exclude absolute symbol, which can be arbitrarily far. %l7
-    // points at .got, which must be retained.
-    case R_SPARC_GOTDATA_OP_HIX22:
-    case R_SPARC_GOTDATA_OP_LOX10:
-    case R_SPARC_GOTDATA_OP:
-      if (sym.isPreemptible || isAbsolute(sym)) {
-        expr = R_GOT_OFF;
-      } else {
-        ctx.in.got->hasGotOffRel.store(true, std::memory_order_relaxed);
-        expr = R_GOTREL;
-      }
-      break;
-
     // TLS LE relocations:
     case R_SPARC_TLS_LE_HIX22:
     case R_SPARC_TLS_LE_LOX10:
@@ -336,24 +322,6 @@ void SPARCV9::relocate(uint8_t *loc, const Relocation &rel,
     // T-simm13
     write32be(loc, (read32be(loc) & ~0x00001fff) | (val & 0x000003ff) | 0x1C00);
     break;
-  case R_SPARC_GOTDATA_OP_HIX22: {
-    // V-imm22. sethi encodes the complement of a negative value, which the
-    // paired xor undoes, so the encodable range is 33 bits.
-    checkInt(ctx, loc, val, 33, rel);
-    uint64_t v = int64_t(val) < 0 ? ~val : val;
-    write32be(loc, (read32be(loc) & ~0x003fffff) | ((v >> 10) & 0x003fffff));
-    break;
-  }
-  case R_SPARC_GOTDATA_OP_LOX10:
-    // T-simm13. Only a negative value needs the sign extension bits.
-    write32be(loc, (read32be(loc) & ~0x00001fff) | (val & 0x000003ff) |
-                       (int64_t(val) < 0 ? 0x1c00 : 0));
-    break;
-  case R_SPARC_GOTDATA_OP:
-    // ldx [%rs1 + %rs2], %rd -> add %rs1, %rs2, %rd
-    if (rel.expr == R_GOTREL)
-      write32be(loc, (read32be(loc) & 0x3e07c01f) | 0x80000000);
-    break;
   default:
     llvm_unreachable("unknown relocation");
   }

lld/MachO/InputSection.cpp

@@ -204,20 +204,16 @@ void ConcatInputSection::foldIdentical(ConcatInputSection *copy,
   for (auto &copySym : copy->symbols)
     copySym->identicalCodeFoldingKind = foldKind;
 
-  if (copy->symbols.empty())
-    return;
-  auto *it = copy->symbols.begin();
-  // The first symbol in the merged section (symbols.front()) must keep its
-  // unwind entry. If this section is empty, the copy's first symbol becomes the
-  // new front, so we skip clearing it.
+  symbols.insert(symbols.end(), copy->symbols.begin(), copy->symbols.end());
+  copy->symbols.clear();
+
+  // Remove duplicate compact unwind info for symbols at the same address.
   if (symbols.empty())
-    ++it;
-  for (; it != copy->symbols.end(); ++it) {
+    return;
+  for (auto it = symbols.begin() + 1; it != symbols.end(); ++it) {
     assert((*it)->value == 0);
     (*it)->originalUnwindEntry = nullptr;
   }
-  symbols.insert(symbols.end(), copy->symbols.begin(), copy->symbols.end());
-  copy->symbols.clear();
 }
 
 void ConcatInputSection::writeTo(uint8_t *buf) {

lld/test/ELF/sparcv9-gotdata.s

@@ -1,141 +0,0 @@
-# REQUIRES: sparc
-## Test R_SPARC_GOTDATA_OP_HIX22, R_SPARC_GOTDATA_OP_LOX10 and R_SPARC_GOTDATA_OP.
-## The GOT load is optimized to an add of the symbol's GOT-relative address
-## unless the symbol is preemptible or absolute.
-
-# RUN: rm -rf %t && split-file %s %t && cd %t
-# RUN: llvm-mc -filetype=obj -triple=sparcv9 a.s -o a.o
-# RUN: llvm-mc -filetype=obj -triple=sparcv9 b.s -o b.o
-# RUN: llvm-mc -filetype=obj -triple=sparcv9 c.s -o c.o
-# RUN: llvm-mc -filetype=obj -triple=sparcv9 ifunc.s -o ifunc.o
-
-# RUN: ld.lld -shared a.o c.o -o a.so
-# RUN: llvm-readelf -S -r a.so | FileCheck %s --check-prefix=RELOC
-# RUN: llvm-objdump -d -j .text --no-show-raw-insn --no-print-imm-hex a.so | FileCheck %s
-
-# RELOC:      [ 8] .got PROGBITS 0000000000200390 {{[0-9a-f]+}} 000020
-# RELOC:      Relocation section '.rela.dyn' {{.*}} contains 1 entries:
-# RELOC:      0000000000200398 {{.*}} R_SPARC_GLOB_DAT {{.*}} foo + 0
-
-## foo is preemptible, so the GOT load stays.
-## .got[1] - _GLOBAL_OFFSET_TABLE_ = 0x200398 - 0x200390 = 8.
-# CHECK-LABEL: <_start>:
-# CHECK-NEXT:   sethi 0, %l1
-# CHECK-NEXT:   xor %l1, 8, %l1
-# CHECK-NEXT:   ldx [%l7+%l1], %l2
-## hid0 - _GLOBAL_OFFSET_TABLE_ = 0x3003b0 - 0x200390 = 0x100020.
-# CHECK-NEXT:   sethi 1024, %g1
-# CHECK-NEXT:   xor %g1, 32, %g1
-# CHECK-NEXT:   add %l7, %g1, %g1
-## hid1 - _GLOBAL_OFFSET_TABLE_ = 0x1002ec - 0x200390 = -0x1000a4. sethi holds
-## the complement, which the sign-extended xor operand undoes.
-# CHECK-NEXT:   sethi 1024, %o2
-# CHECK-NEXT:   xor %o2, -164, %o2
-# CHECK-NEXT:   add %i0, %o2, %i5
-## An undefined symbol and an absolute symbol both have an absolute address, so
-## neither is optimized.
-# CHECK-NEXT:   sethi 0, %l1
-# CHECK-NEXT:   xor %l1, 16, %l1
-# CHECK-NEXT:   ldx [%l7+%l1], %l2
-# CHECK-NEXT:   sethi 0, %g4
-# CHECK-NEXT:   xor %g4, 24, %g4
-# CHECK-NEXT:   ldx [%l7+%g4], %o3
-
-## foo is not preemptible in an executable and is optimized as well. undefweak
-## and absolute keep the GOT load in every mode.
-# RUN: ld.lld a.o c.o -o a
-# RUN: llvm-objdump -d -j .text --no-show-raw-insn --no-print-imm-hex a | FileCheck %s --check-prefix=NOPIE
-# NOPIE-LABEL: <_start>:
-# NOPIE-NEXT:   sethi 1024, %l1
-# NOPIE-NEXT:   xor %l1, -4, %l1
-# NOPIE-NEXT:   add %l7, %l1, %l2
-# NOPIE:        sethi 0, %l1
-# NOPIE-NEXT:   xor %l1, 8, %l1
-# NOPIE-NEXT:   ldx [%l7+%l1], %l2
-# NOPIE-NEXT:   sethi 0, %g4
-# NOPIE-NEXT:   xor %g4, 16, %g4
-# NOPIE-NEXT:   ldx [%l7+%g4], %o3
-
-## .got holds nothing but the header, yet must be kept: %l7 points at it.
-# RUN: ld.lld -shared b.o -o b.so
-# RUN: llvm-readelf -S b.so | FileCheck %s --check-prefix=HEADER
-# HEADER: [ 7] .got PROGBITS 00000000002002b8 {{[0-9a-f]+}} 000008
-
-# RUN: not ld.lld -shared ifunc.o 2>&1 | FileCheck %s --check-prefix=IFUNC --implicit-check-not=error:
-# IFUNC: error: {{.*}}relocation R_SPARC_WDISP19 out of range
-
-## The optimization is decided before addresses are known, so an out-of-range
-## offset is an error rather than a fallback to the GOT load. The offset is
-## reported as signed, not as the complement the sethi encodes. Only hid0, in
-## .data, is out of range; the other symbols stay next to .got.
-# RUN: not ld.lld -shared a.o c.o --section-start=.text=0x10000 --section-start=.got=0x20000 \
-# RUN:   --section-start=.data=0x200000000 2>&1 | FileCheck %s --check-prefix=ERR -DVAL=8589803520 --implicit-check-not=error:
-# RUN: not ld.lld -shared a.o c.o --section-start=.data=0x10000 --section-start=.text=0x200000000 \
-# RUN:   --section-start=.got=0x200020000 2>&1 | FileCheck %s --check-prefix=ERR -DVAL=-8590000128 --implicit-check-not=error:
-# ERR: error: a.o:(.text+0xc): relocation R_SPARC_GOTDATA_OP_HIX22 out of range: [[VAL]] is not in [-4294967296, 4294967295]; references 'hid0'
-
-#--- a.s
-.text
-.globl _start
-_start:
-  sethi %gdop_hix22(foo), %l1
-  xor   %l1, %gdop_lox10(foo), %l1
-  ldx   [%l7 + %l1], %l2, %gdop(foo)
-
-  sethi %gdop_hix22(hid0), %g1
-  xor   %g1, %gdop_lox10(hid0), %g1
-  ldx   [%l7 + %g1], %g1, %gdop(hid0)
-
-## The add keeps rs1, rs2 and rd of the load, and ld is optimized like ldx.
-  sethi %gdop_hix22(hid1), %o2
-  xor   %o2, %gdop_lox10(hid1), %o2
-  ld    [%i0 + %o2], %i5, %gdop(hid1)
-
-  sethi %gdop_hix22(undefweak), %l1
-  xor   %l1, %gdop_lox10(undefweak), %l1
-  ldx   [%l7 + %l1], %l2, %gdop(undefweak)
-
-  sethi %gdop_hix22(absolute), %g4
-  xor   %g4, %gdop_lox10(absolute), %g4
-  ldx   [%l7 + %g4], %o3, %gdop(absolute)
-
-.globl foo, hid0, hid1
-.hidden hid0, hid1
-foo:
-hid1:
-  nop
-
-.weak undefweak
-.hidden undefweak
-
-.data
-hid0:
-  .xword 0
-
-#--- b.s
-.globl _start
-_start:
-  sethi %gdop_hix22(local), %l1
-  xor   %l1, %gdop_lox10(local), %l1
-  ldx   [%l7 + %l1], %l2, %gdop(local)
-local:
-  nop
-
-#--- c.s
-## Separate file to test %gdop_lox10(absolute) relocations in a.s
-.globl absolute
-.hidden absolute
-absolute = 0x1234
-
-#--- ifunc.s
-.globl _start
-_start:
-  sethi %gdop_hix22(ifunc), %l1
-  xor   %l1, %gdop_lox10(ifunc), %l1
-  ldx   [%l7 + %l1], %l2, %gdop(ifunc)
-
-.globl ifunc
-.hidden ifunc
-.type ifunc, @gnu_indirect_function
-ifunc:
-  nop

lld/test/MachO/icf-scale-same-class.s

@@ -1,37 +0,0 @@
-# REQUIRES: x86
-# RUN: rm -rf %t*
-
-# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %s -o %t.o
-# RUN: %lld -lSystem --icf=all -o %t %t.o
-# RUN: llvm-objdump --macho --section-headers %t | FileCheck %s --check-prefix=SECT
-# RUN: llvm-objdump --macho --syms %t | FileCheck %s --check-prefix=SYMS
-
-## Every body folds into one, so __text holds a single 6-byte body plus _main
-## rather than the 3 MiB (0x30000d) the unfolded copies occupy.
-# SECT:      Idx Name          Size
-# SECT-NEXT: 0 __text        00000009
-
-## The sentinels bracket the group, so checking that the first and last members
-## of the class share an address covers the whole run of folds.
-# SYMS-DAG: [[#%.16x,F:]] g     F __TEXT,__text _f_first
-# SYMS-DAG: [[#F]]        g     F __TEXT,__text _f_last
-
-.subsections_via_symbols
-.text
-.p2align 2
-
-.globl _f_first
-_f_first:; movl $7, %eax; ret
-
-## 64 Ki identical function landing in one equivalence class.
-.rept 65536
-  .globl _f\+
-  _f\+:; movl $$7, %eax; ret
-.endr
-
-.globl _f_last
-_f_last:; movl $7, %eax; ret
-
-.globl _main
-_main:
-  ret

lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.cpp

@@ -840,116 +840,22 @@ def main(f):
     return ArgInfo(count, varargs)
 )";
 
-// inspect.signature() is deeply recursive and expensive in C-stack terms;
-// reentrant scripted callbacks dispatched through GetArgInfo() can turn
-// that into a fatal stack overflow instead of a catchable Python
-// RecursionError. GetArgInfo() never calls this itself; callers fall back
-// to it explicitly when they need to handle callables its cheaper,
-// attribute-only approach can't (e.g. builtins).
-Expected<PythonCallable::ArgInfo>
-PythonCallable::GetArgInfoFromInspectSignature(const PythonCallable &callable) {
-  PythonCallable::ArgInfo result = {};
+Expected<PythonCallable::ArgInfo> PythonCallable::GetArgInfo() const {
+  ArgInfo result = {};
+  if (!IsValid())
+    return nullDeref();
+
   // no need to synchronize access to this global, we already have the GIL
   static PythonScript get_arg_info(get_arg_info_script);
-  Expected<PythonObject> pyarginfo = get_arg_info(callable);
+  Expected<PythonObject> pyarginfo = get_arg_info(*this);
   if (!pyarginfo)
     return pyarginfo.takeError();
   long long count =
       cantFail(As<long long>(pyarginfo.get().GetAttribute("count")));
   bool has_varargs =
       cantFail(As<bool>(pyarginfo.get().GetAttribute("has_varargs")));
-  result.max_positional_args =
-      has_varargs ? PythonCallable::ArgInfo::UNBOUNDED : count;
-  return result;
-}
+  result.max_positional_args = has_varargs ? ArgInfo::UNBOUNDED : count;
 
-// GetArgInfo()'s branches, top to bottom (`func` is what each branch ends up
-// introspecting; the final step is always func.__code__.co_argcount/co_flags):
-//
-//   callable
-//   |-- has __self__          -> func = __func__           (bound method;
-//   |                             fails for slot wrappers, e.g. (1).__add__)
-//   |-- has __code__ already  -> func = callable            (plain function)
-//   `-- neither
-//       |-- is a class
-//       |   |-- __init__ has __code__ -> func = __init__
-//       |   `-- else: check __new__ too (object.__init__ is lenient about
-//       |       extra args once __new__ is overridden)
-//       |       |-- __new__ has __code__ -> func = __new__
-//       |       `-- else                  -> ArgInfo{0} (object's defaults)
-//       `-- is an instance
-//           `-- func = __call__ (unwrap __func__ if bound)
-//               `-- no __code__ -> error (e.g. a builtin)
-Expected<PythonCallable::ArgInfo> PythonCallable::GetArgInfo() const {
-  if (!IsValid())
-    return nullDeref();
-
-  PythonObject func = *this;
-  bool implicit_first_arg = false;
-  if (HasAttribute("__self__")) {
-    implicit_first_arg = true;
-    Expected<PythonObject> func_or_err = GetAttribute("__func__");
-    if (!func_or_err)
-      return func_or_err.takeError();
-    func = *func_or_err;
-  } else if (!HasAttribute("__code__")) {
-    implicit_first_arg = true;
-    if (PyType_Check(m_py_obj)) {
-      Expected<PythonObject> init_or_err = GetAttribute("__init__");
-      if (!init_or_err)
-        return init_or_err.takeError();
-      func = *init_or_err;
-      if (!func.HasAttribute("__code__")) {
-        // __init__ is still object.__init__. A class may customize
-        // __new__ instead and leave __init__ untouched, which makes
-        // object.__init__ lenient about extra arguments -- so check
-        // __new__ too before concluding there are none.
-        Expected<PythonObject> new_or_err = GetAttribute("__new__");
-        if (!new_or_err)
-          return new_or_err.takeError();
-        func = *new_or_err;
-        if (!func.HasAttribute("__code__"))
-          return ArgInfo{0};
-      }
-    } else {
-      Expected<PythonObject> call_or_err = GetAttribute("__call__");
-      if (!call_or_err)
-        return call_or_err.takeError();
-      func = *call_or_err;
-      if (func.HasAttribute("__self__")) {
-        Expected<PythonObject> inner_or_err = func.GetAttribute("__func__");
-        if (!inner_or_err)
-          return inner_or_err.takeError();
-        func = *inner_or_err;
-      }
-      if (!func.HasAttribute("__code__"))
-        return llvm::createStringError("__call__ has no __code__");
-    }
-  }
-
-  Expected<PythonObject> code_or_err = func.GetAttribute("__code__");
-  if (!code_or_err)
-    return code_or_err.takeError();
-  PythonObject code = *code_or_err;
-
-  Expected<long long> argcount =
-      As<long long>(code.GetAttribute("co_argcount"));
-  if (!argcount)
-    return argcount.takeError();
-  Expected<long long> flags = As<long long>(code.GetAttribute("co_flags"));
-  if (!flags)
-    return flags.takeError();
-
-  ArgInfo result = {};
-  // Mirrors CPython's CO_VARARGS from <code.h>, which isn't reliably
-  // visible across the Python versions/platforms this file builds against.
-  constexpr long long kCoFlagVarArgs = 0x04;
-  if (*flags & kCoFlagVarArgs) {
-    result.max_positional_args = ArgInfo::UNBOUNDED;
-  } else {
-    long long count = *argcount - (implicit_first_arg ? 1 : 0);
-    result.max_positional_args = count > 0 ? static_cast<unsigned>(count) : 0;
-  }
   return result;
 }
 

lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.h

@@ -615,15 +615,6 @@ public:
 
   llvm::Expected<ArgInfo> GetArgInfo() const;
 
-  // Always derives ArgInfo via a Python-level inspect.signature() call,
-  // regardless of whether the callable's argument count/varargs bit could
-  // have been read directly off its data attributes. GetArgInfo() prefers
-  // the cheaper attribute-based path and only falls back to this for
-  // callables it can't introspect that way (e.g. builtins); exposed
-  // separately so that fallback behavior can be tested directly.
-  static llvm::Expected<ArgInfo>
-  GetArgInfoFromInspectSignature(const PythonCallable &callable);
-
   PythonObject operator()();
 
   PythonObject operator()(std::initializer_list<PyObject *> args);

lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp

@@ -1300,14 +1300,6 @@ ScriptInterpreterPythonImpl::GetMaxPositionalArgumentsForCallable(
                                    callable_name.str().c_str());
   }
   llvm::Expected<PythonCallable::ArgInfo> arg_info = pfunc.GetArgInfo();
-  if (!arg_info) {
-    // `-f` may point at a builtin, unlike other GetArgInfo() callers.
-    LLDB_LOG_ERROR(GetLog(LLDBLog::Script), arg_info.takeError(),
-                   "GetArgInfo failed for callable {1}, falling back to "
-                   "inspect.signature: {0}",
-                   callable_name);
-    arg_info = PythonCallable::GetArgInfoFromInspectSignature(pfunc);
-  }
   if (!arg_info)
     return arg_info.takeError();
   return arg_info.get().max_positional_args;

lldb/test/API/commands/register/register_command/TestRegisters.py

@@ -717,6 +717,7 @@ class RegisterCommandsTestCase(TestBase):
         self.expect("register write pc 0", substrs=[err_msg], error=True)
         self.expect("register info pc", substrs=[err_msg], error=True)
 
+    @expectedFailureDarwin(bugnumber="github.com/llvm/llvm-project/issues/213386")
     def test_case_insensitivity(self):
         """
         Register names, their aliases and any generic names like "sp" and "ra"

lldb/unittests/ScriptInterpreter/Python/PythonDataObjectsTests.cpp

@@ -735,17 +735,6 @@ class NewStyle(object):
   def __init__(self, one, two, three):
     pass
 
-class NoConstructorAtAll:
-  pass
-
-class NewOnlyVarArgs:
-  def __new__(cls, *args, **kwargs):
-    return super().__new__(cls)
-
-class NewOnlyFixedArgs:
-  def __new__(cls, a, b):
-    return super().__new__(cls)
-
 )";
     PyObject *o =
         RunString(script, Py_file_input, globals.get(), globals.get());
@@ -793,37 +782,13 @@ class NewOnlyFixedArgs:
     arginfo = newstyle.get().GetArgInfo();
     ASSERT_THAT_EXPECTED(arginfo, llvm::Succeeded());
     EXPECT_EQ(arginfo.get().max_positional_args, 3u);
-
-    // Neither __init__ nor __new__ overridden: object.__init__ truly takes
-    // no extra arguments here.
-    auto no_ctor = As<PythonCallable>(globals.GetItem("NoConstructorAtAll"));
-    ASSERT_THAT_EXPECTED(no_ctor, llvm::Succeeded());
-    arginfo = no_ctor.get().GetArgInfo();
-    ASSERT_THAT_EXPECTED(arginfo, llvm::Succeeded());
-    EXPECT_EQ(arginfo.get().max_positional_args, 0u);
-
-    // __init__ not overridden, but __new__ is: object.__init__ becomes
-    // lenient about extra arguments once __new__ is overridden, so the
-    // argument count has to come from __new__, not from __init__ alone.
-    auto new_varargs = As<PythonCallable>(globals.GetItem("NewOnlyVarArgs"));
-    ASSERT_THAT_EXPECTED(new_varargs, llvm::Succeeded());
-    arginfo = new_varargs.get().GetArgInfo();
-    ASSERT_THAT_EXPECTED(arginfo, llvm::Succeeded());
-    EXPECT_EQ(arginfo.get().max_positional_args,
-              PythonCallable::ArgInfo::UNBOUNDED);
-
-    auto new_fixed = As<PythonCallable>(globals.GetItem("NewOnlyFixedArgs"));
-    ASSERT_THAT_EXPECTED(new_fixed, llvm::Succeeded());
-    arginfo = new_fixed.get().GetArgInfo();
-    ASSERT_THAT_EXPECTED(arginfo, llvm::Succeeded());
-    EXPECT_EQ(arginfo.get().max_positional_args, 2u);
   }
 
   {
     auto builtins = PythonModule::BuiltinsModule();
     auto hex = As<PythonCallable>(builtins.GetAttribute("hex"));
     ASSERT_THAT_EXPECTED(hex, llvm::Succeeded());
-    auto arginfo = PythonCallable::GetArgInfoFromInspectSignature(hex.get());
+    auto arginfo = hex.get().GetArgInfo();
     ASSERT_THAT_EXPECTED(arginfo, llvm::Succeeded());
     EXPECT_EQ(arginfo.get().max_positional_args, 1u);
   }

llvm/include/llvm/Analysis/ConstantFolding.h

@@ -170,8 +170,7 @@ LLVM_ABI Constant *ConstantFoldCall(const CallBase *Call, Function *F,
                                     bool AllowNonDeterministic = true);
 
 LLVM_ABI Constant *ConstantFoldIntrinsic(Intrinsic::ID ID,
-                                         ArrayRef<Constant *> Ops, Type *Ty,
-                                         Function *CxtF = nullptr);
+                                         ArrayRef<Constant *> Ops, Type *Ty);
 
 /// ConstantFoldLoadThroughBitcast - try to cast constant to destination type
 /// returning null if unsuccessful. Can cast pointer to pointer or pointer to

llvm/include/llvm/Analysis/TargetFolder.h

@@ -193,10 +193,10 @@ public:
 
   Value *FoldIntrinsic(Intrinsic::ID ID, ArrayRef<Value *> Ops, Type *Ty,
                        FastMathFlags FMF = {},
-                       Function *CxtF = nullptr) const override {
+                       Function *CtxF = nullptr) const override {
     if (all_of(Ops, IsaPred<Constant>))
       return ConstantFoldIntrinsic(
-          ID, ArrayRef((Constant *const *)Ops.data(), Ops.size()), Ty, CxtF);
+          ID, ArrayRef((Constant *const *)Ops.data(), Ops.size()), Ty);
     return nullptr;
   }
 

llvm/include/llvm/CodeGen/MachinePipeliner.h

@@ -287,6 +287,12 @@ class LLVM_ABI SwingSchedulerDAG : public ScheduleDAGInstrs {
   unsigned MII = 0;
   /// The maximum initiation interval between iterations for this schedule.
   unsigned MAX_II = 0;
+  /// The resource-constrained minimum initiation interval, i.e. the lower
+  /// bound on MII imposed by the availability of processor resources.
+  unsigned ResMII = 0;
+  /// The recurrence-constrained minimum initiation interval, i.e. the lower
+  /// bound on MII imposed by loop-carried dependence cycles.
+  unsigned RecMII = 0;
   /// Set to true if a valid pipelined schedule is found for the loop.
   bool Scheduled = false;
   MachineLoop &Loop;

llvm/include/llvm/CodeGen/TargetInstrInfo.h

@@ -50,6 +50,7 @@ class MachineLoop;
 class MachineLoopInfo;
 class MachineMemOperand;
 class MachineModuleInfo;
+class MachineOptimizationRemarkEmitter;
 class MachineRegisterInfo;
 class MCAsmInfo;
 class MCInst;
@@ -897,8 +898,11 @@ public:
 
   /// Analyze loop L, which must be a single-basic-block loop, and if the
   /// conditions can be understood enough produce a PipelinerLoopInfo object.
-  virtual std::unique_ptr<PipelinerLoopInfo>
-  analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
+  /// \p ORE, if non-null, may be used by targets to emit optimization remarks
+  /// explaining why the loop was rejected for pipelining.
+  virtual std::unique_ptr<PipelinerLoopInfo> analyzeLoopForPipelining(
+      MachineBasicBlock *LoopBB,
+      MachineOptimizationRemarkEmitter *ORE = nullptr) const {
     return nullptr;
   }
 

llvm/include/llvm/CodeGen/WindowScheduler.h

@@ -109,6 +109,10 @@ public:
 
   bool run();
 
+  /// Returns the Initiation Interval of the best scheduling result found, or
+  /// UINT_MAX if run() has not been called or found no valid schedule.
+  unsigned getBestII() const { return BestII; }
+
 protected:
   /// Two types of ScheduleDAGs are needed, one for creating dependency graphs
   /// only, and the other for list scheduling as determined by the target.

llvm/lib/Analysis/ConstantFolding.cpp

@@ -1730,9 +1730,23 @@ Constant *llvm::ConstantFoldIntegerCast(Constant *C, Type *DestTy,
 //  Constant Folding for Calls
 //
 
-/// Returns true if the intrinsic can be constant folded, given \p IsStrictFP.
-static bool canConstantFoldIntrinsic(Intrinsic::ID ID, bool IsStrictFP) {
-  switch (ID) {
+bool llvm::canConstantFoldCallTo(const CallBase *Call, const Function *F) {
+  if (Call->isNoBuiltin())
+    return false;
+  if (Call->getFunctionType() != F->getFunctionType())
+    return false;
+
+  // Allow FP calls (both libcalls and intrinsics) to avoid being folded.
+  // This can be useful for GPU targets or in cross-compilation scenarios
+  // when the exact target FP behaviour is required, and the host compiler's
+  // behaviour may be slightly different from the device's run-time behaviour.
+  if (DisableFPCallFolding && (F->getReturnType()->isFloatingPointTy() ||
+                               any_of(F->args(), [](const Argument &Arg) {
+                                 return Arg.getType()->isFloatingPointTy();
+                               })))
+    return false;
+
+  switch (F->getIntrinsicID()) {
   // Operations that do not operate floating-point numbers and do not depend on
   // FP environment can be folded even in strictfp functions.
   case Intrinsic::bswap:
@@ -1993,7 +2007,7 @@ static bool canConstantFoldIntrinsic(Intrinsic::ID ID, bool IsStrictFP) {
   case Intrinsic::nvvm_sqrt_rn_d:
   case Intrinsic::nvvm_sqrt_rn_f:
   case Intrinsic::nvvm_sqrt_rn_ftz_f:
-    return !IsStrictFP;
+    return !Call->isStrictFP();
 
   // NVVM add intrinsics with explicit rounding modes
   case Intrinsic::nvvm_add_rm_d:
@@ -2090,35 +2104,8 @@ static bool canConstantFoldIntrinsic(Intrinsic::ID ID, bool IsStrictFP) {
     return true;
   default:
     return false;
+  case Intrinsic::not_intrinsic: break;
   }
-}
-
-/// Given a function's return type and its operands, determine if any of them of
-/// of floating-point type.
-static bool anyTypeContainsFP(Type *RetTy, ArrayRef<Value *> Ops) {
-  return RetTy->isFloatingPointTy() || any_of(Ops, [](Value *V) {
-           return V->getType()->isFloatingPointTy();
-         });
-}
-
-bool llvm::canConstantFoldCallTo(const CallBase *Call, const Function *F) {
-  if (Call->isNoBuiltin())
-    return false;
-  if (Call->getFunctionType() != F->getFunctionType())
-    return false;
-
-  // Allow FP calls (both libcalls and intrinsics) to avoid being folded.
-  // This can be useful for GPU targets or in cross-compilation scenarios
-  // when the exact target FP behaviour is required, and the host compiler's
-  // behaviour may be slightly different from the device's run-time behaviour.
-  if (DisableFPCallFolding &&
-      anyTypeContainsFP(
-          F->getReturnType(),
-          ArrayRef<Value *>((Value *const *)(F->arg_begin()), F->arg_size())))
-    return false;
-
-  if (F->getIntrinsicID() != Intrinsic::not_intrinsic)
-    return canConstantFoldIntrinsic(F->getIntrinsicID(), Call->isStrictFP());
 
   if (!F->hasName() || Call->isStrictFP())
     return false;
@@ -4712,14 +4699,7 @@ ConstantFoldStructCall(StringRef Name, Intrinsic::ID IntrinsicID,
 } // end anonymous namespace
 
 Constant *llvm::ConstantFoldIntrinsic(Intrinsic::ID ID,
-                                      ArrayRef<Constant *> Ops, Type *Ty,
-                                      Function *CxtF) {
-  // In the absence of CxtF, assume strictfp conservatively.
-  if (!canConstantFoldIntrinsic(ID, CxtF ? CxtF->isStrictFP() : true) ||
-      (DisableFPCallFolding &&
-       anyTypeContainsFP(
-           Ty, ArrayRef<Value *>((Value *const *)Ops.data(), Ops.size()))))
-    return nullptr;
+                                      ArrayRef<Constant *> Ops, Type *Ty) {
   return ConstantFoldScalarCall("", ID, Ty, Ops);
 }
 

llvm/lib/Analysis/InstructionSimplify.cpp

@@ -7348,13 +7348,6 @@ Value *llvm::simplifyIntrinsic(Intrinsic::ID IID, Type *ReturnType,
       any_of(Args, IsaPred<PoisonValue>))
     return PoisonValue::get(ReturnType);
 
-  // Defer to ConstantFolding if all args are constants.
-  if (all_of(Args, IsaPred<Constant>))
-    if (Constant *C = ConstantFoldIntrinsic(
-            IID, ArrayRef((Constant *const *)Args.data(), Args.size()),
-            ReturnType, CxtF))
-      return C;
-
   // Most of the intrinsics with no operands have some kind of side effect.
   // Don't simplify.
   if (!NumOperands) {

llvm/lib/CodeGen/MachinePipeliner.cpp

@@ -585,7 +585,7 @@ bool MachinePipeliner::canPipelineLoop(MachineLoop &L) {
 
   LI.LoopInductionVar = nullptr;
   LI.LoopCompare = nullptr;
-  LI.LoopPipelinerInfo = TII->analyzeLoopForPipelining(L.getTopBlock());
+  LI.LoopPipelinerInfo = TII->analyzeLoopForPipelining(L.getTopBlock(), ORE);
   if (!LI.LoopPipelinerInfo) {
     LLVM_DEBUG(dbgs() << "Unable to analyzeLoop, can NOT pipeline Loop\n");
     NumFailLoop++;
@@ -716,7 +716,23 @@ bool MachinePipeliner::runWindowScheduler(MachineLoop &L) {
   Context.RegClassInfo =
       &getAnalysis<MachineRegisterClassInfoWrapperPass>().getRCI();
   WindowScheduler WS(&Context, L);
-  return WS.run();
+  bool Scheduled = WS.run();
+  if (Scheduled) {
+    unsigned II = WS.getBestII();
+    ORE->emit([&]() {
+      return MachineOptimizationRemark(DEBUG_TYPE, "window-schedule",
+                                       L.getStartLoc(), L.getHeader())
+             << "Window scheduled with Initiation Interval: "
+             << ore::NV("II", II);
+    });
+  } else {
+    ORE->emit([&]() {
+      return MachineOptimizationRemarkMissed(DEBUG_TYPE, "window-schedule",
+                                             L.getStartLoc(), L.getHeader())
+             << "Failed to find a valid window schedule";
+    });
+  }
+  return Scheduled;
 }
 
 bool MachinePipeliner::useSwingModuloScheduler() {
@@ -2897,7 +2913,10 @@ bool SwingSchedulerDAG::schedulePipeline(SMSchedule &Schedule) {
              << "Schedule found with Initiation Interval: "
              << ore::NV("II", Schedule.getInitiationInterval())
              << ", MaxStageCount: "
-             << ore::NV("MaxStageCount", Schedule.getMaxStageCount());
+             << ore::NV("MaxStageCount", Schedule.getMaxStageCount())
+             << ", ResMII: " << ore::NV("ResMII", ResMII)
+             << ", RecMII: " << ore::NV("RecMII", RecMII) << ", Bound: "
+             << ore::NV("Bound", ResMII >= RecMII ? "Resource" : "Recurrence");
     });
   } else
     Schedule.reset();

llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp

@@ -54,7 +54,11 @@ void JITLinkerBase::linkPhase1(std::unique_ptr<JITLinkerBase> Self) {
   Ctx->getMemoryManager().allocate(
       Ctx->getJITLinkDylib(), *G,
       [S = std::move(Self)](AllocResult AR) mutable {
-        S->linkPhase2(std::move(S), std::move(AR));
+        // FIXME: Once MSVC implements c++17 order of evaluation rules for calls
+        // this can be simplified to
+        //          S->linkPhase2(std::move(S), std::move(AR));
+        auto *TmpSelf = S.get();
+        TmpSelf->linkPhase2(std::move(S), std::move(AR));
       });
 }
 
@@ -91,7 +95,10 @@ void JITLinkerBase::linkPhase2(std::unique_ptr<JITLinkerBase> Self,
       dbgs() << "No external symbols for " << G->getName()
              << ". Proceeding immediately with link phase 3.\n";
     });
-    Self->linkPhase3(std::move(Self), AsyncLookupResult());
+    // FIXME: Once MSVC implements c++17 order of evaluation rules for calls
+    // this can be simplified. See below.
+    auto &TmpSelf = *Self;
+    TmpSelf.linkPhase3(std::move(Self), AsyncLookupResult());
     return;
   }
 
@@ -101,11 +108,22 @@ void JITLinkerBase::linkPhase2(std::unique_ptr<JITLinkerBase> Self,
            << " (may trigger materialization/linking of other graphs)...\n";
   });
 
+  // We're about to hand off ownership of ourself to the continuation. Grab a
+  // pointer to the context so that we can call it to initiate the lookup.
+  //
+  // FIXME: Once MSVC implements c++17 order of evaluation rules for calls this
+  // can be simplified to:
+  //
+  // Ctx->lookup(std::move(UnresolvedExternals),
+  //             [Self=std::move(Self)](Expected<AsyncLookupResult> Result) {
+  //               Self->linkPhase3(std::move(Self), std::move(Result));
+  //             });
   Ctx->lookup(std::move(ExternalSymbols),
               createLookupContinuation(
                   [S = std::move(Self)](
                       Expected<AsyncLookupResult> LookupResult) mutable {
-                    S->linkPhase3(std::move(S), std::move(LookupResult));
+                    auto &TmpSelf = *S;
+                    TmpSelf.linkPhase3(std::move(S), std::move(LookupResult));
                   }));
 }
 
@@ -153,7 +171,11 @@ void JITLinkerBase::linkPhase3(std::unique_ptr<JITLinkerBase> Self,
   }
 
   Alloc->finalize([S = std::move(Self)](FinalizeResult FR) mutable {
-    S->linkPhase4(std::move(S), std::move(FR));
+    // FIXME: Once MSVC implements c++17 order of evaluation rules for calls
+    // this can be simplified to
+    //          S->linkPhase2(std::move(S), std::move(AR));
+    auto *TmpSelf = S.get();
+    TmpSelf->linkPhase4(std::move(S), std::move(FR));
   });
 }
 

llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h

@@ -116,10 +116,12 @@ public:
     // Ownership of the linker is passed into the linker's doLink function to
     // allow it to be passed on to async continuations.
     //
+    // FIXME: Remove LTmp once we have c++17.
     // C++17 sequencing rules guarantee that function name expressions are
-    // sequenced before arguments, so L->linkPhase1(std::move(L)) is well
-    // formed.
-    L->linkPhase1(std::move(L));
+    // sequenced before arguments, so L->linkPhase1(std::move(L), ...) will be
+    // well formed.
+    auto &LTmp = *L;
+    LTmp.linkPhase1(std::move(L));
   }
 
 private:

llvm/lib/Target/AArch64/AArch64InstrInfo.cpp

@@ -25,6 +25,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/CodeGen/CFIInstBuilder.h"
 #include "llvm/CodeGen/LivePhysRegs.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
@@ -36,6 +37,7 @@
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineOperand.h"
+#include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/RegisterScavenging.h"
 #include "llvm/CodeGen/StackMaps.h"
@@ -12063,7 +12065,8 @@ static bool getIndVarInfo(Register Reg, const MachineBasicBlock *LoopBB,
 }
 
 std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
-AArch64InstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
+AArch64InstrInfo::analyzeLoopForPipelining(
+    MachineBasicBlock *LoopBB, MachineOptimizationRemarkEmitter *ORE) const {
   // Accept loops that meet the following conditions
   // * The conditional branch is BCC
   // * The compare instruction is ADDS/SUBS/WHILEXX
@@ -12072,24 +12075,66 @@ AArch64InstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
   // * The induction variable is incremented/decremented by a single instruction
   // * Does not contain CALL or instructions which have unmodeled side effects
 
-  for (MachineInstr &MI : *LoopBB)
-    if (MI.isCall() || MI.hasUnmodeledSideEffects())
-      // This instruction may use NZCV, which interferes with the instruction to
-      // be inserted for loop control.
+  for (MachineInstr &MI : *LoopBB) {
+    // This instruction may use NZCV, which interferes with the instruction to
+    // be inserted for loop control.
+    if (MI.isCall()) {
+      if (ORE)
+        ORE->emit([&]() {
+          return MachineOptimizationRemarkAnalysis("pipeliner", "analyzeLoop",
+                                                   &MI)
+                 << "loop contains a call and therefore cannot be pipelined";
+        });
       return nullptr;
+    }
+    if (MI.hasUnmodeledSideEffects()) {
+      if (ORE)
+        ORE->emit([&]() {
+          return MachineOptimizationRemarkAnalysis("pipeliner", "analyzeLoop",
+                                                   &MI)
+                 << "loop contains an instruction with unmodeled side effects, "
+                    "and therefore cannot be pipelined";
+        });
+      return nullptr;
+    }
+  }
 
   MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
   SmallVector<MachineOperand, 4> Cond;
-  if (analyzeBranch(*LoopBB, TBB, FBB, Cond))
+  if (analyzeBranch(*LoopBB, TBB, FBB, Cond)) {
+    if (ORE)
+      ORE->emit([&]() {
+        return MachineOptimizationRemarkAnalysis(
+                   "pipeliner", "analyzeLoop",
+                   LoopBB->findDebugLoc(LoopBB->getFirstTerminator()), LoopBB)
+               << "branch cannot be analyzed";
+      });
     return nullptr;
+  }
 
   // Infinite loops are not supported
-  if (TBB == LoopBB && FBB == LoopBB)
+  if (TBB == LoopBB && FBB == LoopBB) {
+    if (ORE)
+      ORE->emit([&]() {
+        return MachineOptimizationRemarkAnalysis(
+                   "pipeliner", "analyzeLoop",
+                   LoopBB->findDebugLoc(LoopBB->getFirstTerminator()), LoopBB)
+               << "infinite loops are not supported";
+      });
     return nullptr;
+  }
 
   // Must be conditional branch
-  if (TBB != LoopBB && FBB == nullptr)
+  if (TBB != LoopBB && FBB == nullptr) {
+    if (ORE)
+      ORE->emit([&]() {
+        return MachineOptimizationRemarkAnalysis(
+                   "pipeliner", "analyzeLoop",
+                   LoopBB->findDebugLoc(LoopBB->getFirstTerminator()), LoopBB)
+               << "loop is not terminated by a conditional branch";
+      });
     return nullptr;
+  }
 
   assert((TBB == LoopBB || FBB == LoopBB) &&
          "The Loop must be a single-basic-block loop");
@@ -12097,8 +12142,16 @@ AArch64InstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
   MachineInstr *CondBranch = &*LoopBB->getFirstTerminator();
   const TargetRegisterInfo &TRI = getRegisterInfo();
 
-  if (CondBranch->getOpcode() != AArch64::Bcc)
+  if (CondBranch->getOpcode() != AArch64::Bcc) {
+    if (ORE)
+      ORE->emit([&]() {
+        return MachineOptimizationRemarkAnalysis("pipeliner", "analyzeLoop",
+                                                 CondBranch)
+               << "branch opcode not yet supported for pipelining: "
+               << ore::NV("Opcode", getName(CondBranch->getOpcode()));
+      });
     return nullptr;
+  }
 
   // Normalization for createTripCountGreaterCondition()
   if (TBB == LoopBB)
@@ -12130,6 +12183,13 @@ AArch64InstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
           Comp = &MI;
           break;
         }
+        if (ORE)
+          ORE->emit([&]() {
+            return MachineOptimizationRemarkAnalysis("pipeliner", "analyzeLoop",
+                                                     &MI)
+                   << "compare instruction not recognized: "
+                   << ore::NV("Opcode", getName(MI.getOpcode()));
+          });
         return nullptr;
       }
 
@@ -12138,22 +12198,44 @@ AArch64InstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
           CompCounterOprNum = 2;
         else if (isDefinedOutside(Comp->getOperand(2).getReg(), LoopBB))
           CompCounterOprNum = 1;
-        else
+        else {
+          if (ORE)
+            ORE->emit([&]() {
+              return MachineOptimizationRemarkAnalysis("pipeliner",
+                                                       "analyzeLoop", Comp)
+                     << "neither operand of the compare is loop invariant";
+            });
           return nullptr;
+        }
       }
       break;
     }
   }
-  if (!Comp)
+  if (!Comp) {
+    if (ORE)
+      ORE->emit([&]() {
+        return MachineOptimizationRemarkAnalysis(
+                   "pipeliner", "analyzeLoop",
+                   LoopBB->findDebugLoc(LoopBB->getFirstTerminator()), LoopBB)
+               << "no NZCV-modifying compare instruction found";
+      });
     return nullptr;
+  }
 
   MachineInstr *Update = nullptr;
   Register Init;
   bool IsUpdatePriorComp;
   unsigned UpdateCounterOprNum;
   if (!getIndVarInfo(Comp->getOperand(CompCounterOprNum).getReg(), LoopBB,
-                     Update, UpdateCounterOprNum, Init, IsUpdatePriorComp))
+                     Update, UpdateCounterOprNum, Init, IsUpdatePriorComp)) {
+    if (ORE)
+      ORE->emit([&]() {
+        return MachineOptimizationRemarkAnalysis("pipeliner", "analyzeLoop",
+                                                 Comp)
+               << "loop induction variable pattern not recognized";
+      });
     return nullptr;
+  }
 
   return std::make_unique<AArch64PipelinerLoopInfo>(
       LoopBB, CondBranch, Comp, CompCounterOprNum, Update, UpdateCounterOprNum,

llvm/lib/Target/AArch64/AArch64InstrInfo.h

@@ -414,8 +414,9 @@ public:
                         const DebugLoc &DL,
                         int *BytesAdded = nullptr) const override;
 
-  std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
-  analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override;
+  std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo> analyzeLoopForPipelining(
+      MachineBasicBlock *LoopBB,
+      MachineOptimizationRemarkEmitter *ORE = nullptr) const override;
 
   bool
   reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const override;

llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp

@@ -6737,7 +6737,8 @@ bool ARMPipelinerLoopInfo::tooMuchRegisterPressure(SwingSchedulerDAG &SSD,
 } // namespace
 
 std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
-ARMBaseInstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
+ARMBaseInstrInfo::analyzeLoopForPipelining(
+    MachineBasicBlock *LoopBB, MachineOptimizationRemarkEmitter *ORE) const {
   MachineBasicBlock::iterator I = LoopBB->getFirstTerminator();
   MachineBasicBlock *Preheader = *LoopBB->pred_begin();
   if (Preheader == LoopBB)

llvm/lib/Target/ARM/ARMBaseInstrInfo.h

@@ -382,8 +382,9 @@ public:
 
   /// Analyze loop L, which must be a single-basic-block loop, and if the
   /// conditions can be understood enough produce a PipelinerLoopInfo object.
-  std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
-  analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override;
+  std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo> analyzeLoopForPipelining(
+      MachineBasicBlock *LoopBB,
+      MachineOptimizationRemarkEmitter *ORE = nullptr) const override;
 
 private:
   /// Returns an unused general-purpose register which can be used for

llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp

@@ -806,7 +806,8 @@ public:
 } // namespace
 
 std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
-HexagonInstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
+HexagonInstrInfo::analyzeLoopForPipelining(
+    MachineBasicBlock *LoopBB, MachineOptimizationRemarkEmitter *ORE) const {
   // We really "analyze" only hardware loops right now.
   MachineBasicBlock::iterator I = LoopBB->getFirstTerminator();
 

llvm/lib/Target/Hexagon/HexagonInstrInfo.h

@@ -139,8 +139,9 @@ public:
 
   /// Analyze loop L, which must be a single-basic-block loop, and if the
   /// conditions can be understood enough produce a PipelinerLoopInfo object.
-  std::unique_ptr<PipelinerLoopInfo>
-  analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override;
+  std::unique_ptr<PipelinerLoopInfo> analyzeLoopForPipelining(
+      MachineBasicBlock *LoopBB,
+      MachineOptimizationRemarkEmitter *ORE = nullptr) const override;
 
   /// Return true if it's profitable to predicate
   /// instructions with accumulated instruction latency of "NumCycles"

llvm/lib/Target/PowerPC/PPCInstrInfo.cpp

@@ -5790,7 +5790,8 @@ public:
 } // namespace
 
 std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
-PPCInstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
+PPCInstrInfo::analyzeLoopForPipelining(
+    MachineBasicBlock *LoopBB, MachineOptimizationRemarkEmitter *ORE) const {
   // We really "analyze" only hardware loops right now.
   MachineBasicBlock::iterator I = LoopBB->getFirstTerminator();
   MachineBasicBlock *Preheader = *LoopBB->pred_begin();

llvm/lib/Target/PowerPC/PPCInstrInfo.h

@@ -789,8 +789,9 @@ public:
 
   /// Analyze loop L, which must be a single-basic-block loop, and if the
   /// conditions can be understood enough produce a PipelinerLoopInfo object.
-  std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
-  analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override;
+  std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo> analyzeLoopForPipelining(
+      MachineBasicBlock *LoopBB,
+      MachineOptimizationRemarkEmitter *ORE = nullptr) const override;
 };
 
 }

llvm/lib/Target/RISCV/RISCVInstrInfo.cpp

@@ -5394,7 +5394,8 @@ public:
 } // namespace
 
 std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
-RISCVInstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
+RISCVInstrInfo::analyzeLoopForPipelining(
+    MachineBasicBlock *LoopBB, MachineOptimizationRemarkEmitter *ORE) const {
   MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
   SmallVector<MachineOperand, 4> Cond;
   if (analyzeBranch(*LoopBB, TBB, FBB, Cond, /*AllowModify=*/false))

llvm/lib/Target/RISCV/RISCVInstrInfo.h

@@ -328,8 +328,9 @@ public:
 
   unsigned getTailDuplicateSize(CodeGenOptLevel OptLevel) const override;
 
-  std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
-  analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override;
+  std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo> analyzeLoopForPipelining(
+      MachineBasicBlock *LoopBB,
+      MachineOptimizationRemarkEmitter *ORE = nullptr) const override;
 
   bool isHighLatencyDef(int Opc) const override;
 

llvm/test/CodeGen/AArch64/sms-analyzeloop-remarks.mir

@@ -0,0 +1,168 @@
+# RUN: llc --verify-machineinstrs -mtriple=aarch64 -o /dev/null %s -run-pass pipeliner \
+# RUN:   -aarch64-enable-pipeliner -pass-remarks-analysis=pipeliner \
+# RUN:   -pass-remarks-missed=pipeliner 2>&1 | FileCheck %s
+
+# Regression test for the analyzeLoopForPipelining() optimization remarks:
+# each rejection point in AArch64InstrInfo::analyzeLoopForPipelining() should
+# report a distinct, specific reason via the "pipeliner"/"analyzeLoop" remark,
+# rather than only the generic "The loop structure is not supported" remark
+# from canPipelineLoop().
+
+--- |
+  define dso_local void @call(ptr noalias nocapture noundef writeonly %a, i32 noundef %n) local_unnamed_addr #0 {
+  entry:
+    ret void
+  }
+  declare void @callee()
+
+  define dso_local void @infiniteloop(ptr noalias nocapture noundef writeonly %a, i32 noundef %n) local_unnamed_addr #0 {
+  entry:
+    ret void
+  }
+
+  define dso_local void @unsupportedbranch(ptr noalias nocapture noundef writeonly %a) local_unnamed_addr #0 {
+  entry:
+    ret void
+  }
+
+  define dso_local void @badcompare(ptr noalias nocapture noundef writeonly %a, i32 noundef %n) local_unnamed_addr #0 {
+  entry:
+    ret void
+  }
+
+  define dso_local void @noninvariantcompare(i64 noundef %x, i64 noundef %y) local_unnamed_addr #0 {
+  entry:
+    ret void
+  }
+
+  define dso_local void @badindvar(i64 noundef %x, i64 noundef %y) local_unnamed_addr #0 {
+  entry:
+    ret void
+  }
+...
+---
+name:            call
+tracksRegLiveness: true
+body:             |
+  ; A call in the loop body may use NZCV, which interferes with the
+  ; instruction to be inserted for loop control.
+  ; CHECK: loop contains a call and therefore cannot be pipelined
+  ; CHECK: The loop structure is not supported
+  bb.0.entry:
+    liveins: $x0, $w1
+
+    %10:gpr64sp = COPY $x0
+    %11:gpr32common = COPY $w1
+
+  bb.1:
+    %12:gpr64sp = PHI %10, %bb.0, %13, %bb.1
+    %13:gpr64sp = ADDXri %12, 1, 0
+    BL @callee, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp
+    dead $wzr = SUBSWri %11, 1, 0, implicit-def $nzcv
+    Bcc 1, %bb.1, implicit $nzcv
+    B %bb.2
+
+  bb.2:
+...
+---
+name:            infiniteloop
+tracksRegLiveness: true
+body:             |
+  ; Both branch targets are the loop itself: nothing exits the loop.
+  ; CHECK: infinite loops are not supported
+  ; CHECK: The loop structure is not supported
+  bb.0.entry:
+    liveins: $x0, $w1
+
+    %10:gpr64sp = COPY $x0
+    %11:gpr32common = COPY $w1
+
+  bb.1:
+    %12:gpr64sp = PHI %10, %bb.0, %13, %bb.1
+    %13:gpr64sp = ADDXri %12, 1, 0
+    dead $wzr = SUBSWri %11, 0, 0, implicit-def $nzcv
+    Bcc 0, %bb.1, implicit $nzcv
+    B %bb.1
+...
+---
+name:            unsupportedbranch
+tracksRegLiveness: true
+body:             |
+  ; The loop terminator is an unconditional branch (Bcc is required).
+  ; CHECK: branch opcode not yet supported for pipelining: B
+  ; CHECK: The loop structure is not supported
+  bb.0.entry:
+    liveins: $x0
+
+    %10:gpr64sp = COPY $x0
+
+  bb.1:
+    %12:gpr64sp = PHI %10, %bb.0, %13, %bb.1
+    %13:gpr64sp = ADDXri %12, 1, 0
+    B %bb.1
+...
+---
+name:            badcompare
+tracksRegLiveness: true
+body:             |
+  ; The NZCV-modifying instruction is not a recognized compare opcode.
+  ; CHECK: compare instruction not recognized: ANDSWri
+  ; CHECK: The loop structure is not supported
+  bb.0.entry:
+    liveins: $x0, $w1
+
+    %10:gpr64sp = COPY $x0
+    %11:gpr32common = COPY $w1
+
+  bb.1:
+    %12:gpr64sp = PHI %10, %bb.0, %13, %bb.1
+    %13:gpr64sp = ADDXri %12, 1, 0
+    dead $wzr = ANDSWri %11, 1, implicit-def $nzcv
+    Bcc 1, %bb.1, implicit $nzcv
+    B %bb.2
+
+  bb.2:
+...
+---
+name:            noninvariantcompare
+tracksRegLiveness: true
+body:             |
+  ; Neither operand of the compare instruction is a loop invariant value.
+  ; CHECK: neither operand of the compare is loop invariant
+  ; CHECK: The loop structure is not supported
+  bb.0.entry:
+    liveins: $x0, $x1
+    %10:gpr64 = COPY $x0
+    %11:gpr64 = COPY $x1
+
+  bb.1:
+    %12:gpr64 = PHI %11, %bb.0, %13, %bb.1
+    %13:gpr64 = ADDXrr %12, %11
+    dead $xzr = SUBSXrr %13, %13, implicit-def $nzcv
+    Bcc 1, %bb.1, implicit $nzcv
+    B %bb.2
+
+  bb.2:
+...
+---
+name:            badindvar
+tracksRegLiveness: true
+body:             |
+  ; The value compared against is not updated by a recognized increment
+  ; instruction.
+  ; CHECK: loop induction variable pattern not recognized
+  ; CHECK: The loop structure is not supported
+  bb.0.entry:
+    liveins: $x0, $x1
+    %10:gpr64 = COPY $x0
+    %11:gpr64 = COPY $x1
+
+  bb.1:
+    %12:gpr64 = PHI %11, %bb.0, %13, %bb.1
+    %13:gpr64 = ORRXrr %12, %12
+    dead $xzr = SUBSXrr %12, %10, implicit-def $nzcv
+    Bcc 1, %bb.1, implicit $nzcv
+    B %bb.2
+
+  bb.2:
+...

llvm/test/CodeGen/AArch64/sms-schedule-remark.mir

@@ -0,0 +1,83 @@
+# RUN: llc --verify-machineinstrs -mtriple=aarch64 -o /dev/null %s -run-pass pipeliner \
+# RUN:   -aarch64-enable-pipeliner -pass-remarks-analysis=pipeliner 2>&1 | FileCheck %s
+
+# Regression test for the "schedule" success remark: it should report ResMII,
+# RecMII, and which of the two is the binding constraint on the chosen II, in
+# addition to the Initiation Interval and MaxStageCount that were already
+# reported.
+
+# CHECK: Schedule found with Initiation Interval: 3, MaxStageCount: 2, ResMII: 0, RecMII: 0, Bound: Resource
+
+--- |
+  define dso_local void @func(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b, i32 noundef %n) local_unnamed_addr #0 {
+  entry:
+    %cmp6 = icmp sgt i32 %n, 0
+    br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
+
+  for.body.preheader:                               ; preds = %entry
+    %wide.trip.count = zext nneg i32 %n to i64
+    br label %for.body
+
+  for.cond.cleanup:                                 ; preds = %for.body, %entry
+    ret void
+
+  for.body:                                         ; preds = %for.body.preheader, %for.body
+    %lsr.iv11 = phi i64 [ %wide.trip.count, %for.body.preheader ], [ %lsr.iv.next, %for.body ]
+    %lsr.iv9 = phi ptr [ %b, %for.body.preheader ], [ %scevgep10, %for.body ]
+    %lsr.iv = phi ptr [ %a, %for.body.preheader ], [ %scevgep, %for.body ]
+    %0 = load float, ptr %lsr.iv9, align 4
+    %add = fadd float %0, 1.000000e+00
+    store float %add, ptr %lsr.iv, align 4
+    %scevgep = getelementptr i8, ptr %lsr.iv, i64 4
+    %scevgep10 = getelementptr i8, ptr %lsr.iv9, i64 4
+    %lsr.iv.next = add nsw i64 %lsr.iv11, -1
+    %exitcond.not = icmp eq i64 %lsr.iv.next, 0
+    br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+  }
+
+...
+---
+name:            func
+tracksRegLiveness: true
+liveins:
+  - { reg: '$x0', virtual-reg: '%7' }
+  - { reg: '$x1', virtual-reg: '%8' }
+  - { reg: '$w2', virtual-reg: '%9' }
+body:             |
+  bb.0.entry:
+    successors: %bb.1(0x50000000), %bb.2(0x30000000)
+    liveins: $x0, $x1, $w2
+
+    %9:gpr32common = COPY $w2
+    %8:gpr64 = COPY $x1
+    %7:gpr64 = COPY $x0
+    dead $wzr = SUBSWri %9, 1, 0, implicit-def $nzcv
+    Bcc 11, %bb.2, implicit $nzcv
+    B %bb.1
+
+  bb.1.for.body.preheader:
+    %11:gpr32 = ORRWrs $wzr, %9, 0
+    %0:gpr64all = SUBREG_TO_REG killed %11, %subreg.sub_32
+    %14:fpr32 = FMOVSi 112
+    B %bb.3
+
+  bb.2.for.cond.cleanup:
+    RET_ReallyLR
+
+  bb.3.for.body:
+    successors: %bb.2(0x04000000), %bb.3(0x7c000000)
+
+    %1:gpr64sp = PHI %0, %bb.1, %6, %bb.3
+    %2:gpr64sp = PHI %8, %bb.1, %5, %bb.3
+    %3:gpr64sp = PHI %7, %bb.1, %4, %bb.3
+    early-clobber %12:gpr64sp, %13:fpr32 = LDRSpost %2, 4 :: (load (s32) from %ir.lsr.iv9)
+    %15:fpr32 = nofpexcept FADDSrr killed %13, %14, implicit $fpcr
+    early-clobber %16:gpr64sp = STRSpost killed %15, %3, 4 :: (store (s32) into %ir.lsr.iv)
+    %4:gpr64all = COPY %16
+    %5:gpr64all = COPY %12
+    %17:gpr64 = nsw SUBSXri %1, 1, 0, implicit-def $nzcv
+    %6:gpr64all = COPY %17
+    Bcc 0, %bb.2, implicit $nzcv
+    B %bb.3
+
+...

llvm/test/CodeGen/Hexagon/swp-ws-exp.mir

@@ -2,9 +2,14 @@
 # RUN: llc --mtriple=hexagon %s -run-pass=pipeliner -debug-only=pipeliner \
 # RUN: -window-sched=force -filetype=null -verify-machineinstrs 2>&1 \
 # RUN: | FileCheck %s
+# RUN: llc --mtriple=hexagon %s -run-pass=pipeliner \
+# RUN: -window-sched=force -filetype=null -verify-machineinstrs \
+# RUN: -pass-remarks=pipeliner 2>&1 | FileCheck %s --check-prefix=REMARK
 
 # CHECK: Best window offset is {{[0-9]+}} and Best II is {{[0-9]+}}.
 
+# REMARK: remark: {{.*}}Window scheduled with Initiation Interval: {{[0-9]+}}
+
 --- |
   define void @exp_approx_top_six(i32 %N, ptr noalias %x, ptr noalias %y) #0 {
   entry:

llvm/test/CodeGen/Hexagon/swp-ws-fail-0.mir

@@ -9,6 +9,9 @@
 # RUN: llc --mtriple=hexagon %s -run-pass=pipeliner -debug-only=pipeliner \
 # RUN: -window-sched=force -window-region-limit=1 -window-search-ratio=80 \
 # RUN: -filetype=null 2>&1 | FileCheck %s --check-prefix=CHECK-SCHED-NOT-NEEDED
+# RUN: llc --mtriple=hexagon %s -run-pass=pipeliner \
+# RUN: -window-sched=force -filetype=null \
+# RUN: -pass-remarks-missed=pipeliner 2>&1 | FileCheck %s --check-prefix=REMARK
 
 # CHECK-INITIALIZE: There are too few MIs in the window region!
 # CHECK-INITIALIZE: The WindowScheduler failed to initialize!
@@ -16,6 +19,8 @@
 # CHECK-ANALYSE-II: Window scheduling is not needed!
 # CHECK-SCHED-NOT-NEEDED: Window scheduling is not needed!
 
+# REMARK: remark: {{.*}}Failed to find a valid window schedule
+
 ---
 name:            relu
 tracksRegLiveness: true

llvm/test/CodeGen/PowerPC/sms-remark.ll

@@ -15,7 +15,7 @@
 @y = dso_local global [1024 x i32] zeroinitializer, align 4
 
 define dso_local ptr @foo() local_unnamed_addr {
-;ENABLED: Schedule found with Initiation Interval
+;ENABLED: Schedule found with Initiation Interval: {{[0-9]+}}, MaxStageCount: {{[0-9]+}}, ResMII: {{[0-9]+}}, RecMII: {{[0-9]+}}, Bound: {{(Recurrence|Resource)}}
 ;ENABLED: Pipelined succesfully!
 entry:
   %.pre = load i32, ptr @y, align 4

llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll

@@ -253,9 +253,20 @@ define void @test_minbws_for_trunc(i32 %n, ptr noalias %p1, ptr noalias %p2) {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[P2]], i64 8160
+; CHECK-NEXT:    [[MUL:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 4, i16 255)
+; CHECK-NEXT:    [[MUL_RESULT:%.*]] = extractvalue { i16, i1 } [[MUL]], 0
+; CHECK-NEXT:    [[MUL_OVERFLOW:%.*]] = extractvalue { i16, i1 } [[MUL]], 1
+; CHECK-NEXT:    [[TMP0:%.*]] = add i16 4, [[MUL_RESULT]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i16 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = or i1 [[TMP1]], [[MUL_OVERFLOW]]
+; CHECK-NEXT:    [[MUL1:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 32, i64 255)
+; CHECK-NEXT:    [[MUL_RESULT2:%.*]] = extractvalue { i64, i1 } [[MUL1]], 0
+; CHECK-NEXT:    [[MUL_OVERFLOW3:%.*]] = extractvalue { i64, i1 } [[MUL1]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[P2]], i64 [[MUL_RESULT2]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp ult ptr [[TMP3]], [[P2]]
-; CHECK-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or i1 [[TMP4]], [[MUL_OVERFLOW3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or i1 [[TMP2]], [[TMP5]]
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; CHECK:       [[VECTOR_MEMCHECK]]:
 ; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[P2]], i64 2042
 ; CHECK-NEXT:    [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[P2]], i64 1021

llvm/test/Transforms/LoopVectorize/X86/funclet.ll

@@ -26,9 +26,10 @@ define void @test1() #0 personality ptr @__CxxFrameHandler3 {
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1024
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[TRY_CONT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[TRY_CONT]]:
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <16 x double> [[TMP2]], i64 15
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    store double 1.000000e+00, ptr @sink, align 8
+; CHECK-NEXT:    store double [[TMP4]], ptr @sink, align 8
 ; CHECK-NEXT:    catchret from [[TMP1]] to label %[[TRY_CONT1:.*]]
 ; CHECK:       [[TRY_CONT1]]:
 ; CHECK-NEXT:    ret void

llvm/test/Transforms/LoopVectorize/constantfolder.ll

@@ -323,41 +323,6 @@ exit:
   ret void
 }
 
-define void @const_fold_unaryintrinsic(ptr %p) {
-; CHECK-LABEL: define void @const_fold_unaryintrinsic(
-; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
-; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
-; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr double, ptr [[P]], i64 [[INDEX]]
-; CHECK-NEXT:    store <4 x double> splat (double 1.000000e+00), ptr [[TMP0]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; CHECK-NEXT:    br i1 [[TMP1]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
-; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    br label %[[LOOP:.*]]
-; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    ret void
-;
-entry:
-  br label %loop
-
-loop:
-  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
-  %gep.p.iv = getelementptr double, ptr %p, i64 %iv
-  %res = call double @llvm.canonicalize.f64(double 1.0)
-  store double %res, ptr %gep.p.iv
-  %iv.next = add i64 %iv, 1
-  %ec = icmp eq i64 %iv.next, 100
-  br i1 %ec, label %exit, label %loop
-
-exit:
-  ret void
-}
-
 define void @const_fold_widegep(ptr noalias %A, ptr noalias %B, i64 %d) {
 ; CHECK-LABEL: define void @const_fold_widegep(
 ; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[D:%.*]]) {
@@ -369,7 +334,7 @@ define void @const_fold_widegep(ptr noalias %A, ptr noalias %B, i64 %d) {
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
-; CHECK-NEXT:    br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    store ptr [[A]], ptr [[B]], align 8
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
@@ -467,7 +432,7 @@ define void @unused_live_ins_in_preheader(ptr %dst, ptr %src) {
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw <4 x i8> [[VEC_IND]], splat (i8 4)
 ; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i32 [[INDEX_NEXT]], 12
-; CHECK-NEXT:    br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[EXIT]]:

llvm/test/Transforms/LoopVectorize/hoist-predicated-loads-with-predicated-stores.ll

@@ -932,34 +932,71 @@ exit:
 define void @sink_multiple_store_groups_noalias_via_scev_urem(ptr %dst, ptr %src) {
 ; CHECK-LABEL: define void @sink_multiple_store_groups_noalias_via_scev_urem(
 ; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]]) {
-; CHECK-NEXT:  [[VECTOR_PH:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br label %[[VECTOR_SCEVCHECK:.*]]
+; CHECK:       [[VECTOR_SCEVCHECK]]:
+; CHECK-NEXT:    [[MUL:%.*]] = call { i7, i1 } @llvm.umul.with.overflow.i7(i7 16, i7 -29)
+; CHECK-NEXT:    [[MUL_OVERFLOW:%.*]] = extractvalue { i7, i1 } [[MUL]], 1
+; CHECK-NEXT:    br i1 [[MUL_OVERFLOW]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK:       [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[SRC]], i64 12828
+; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[DST]], i64 12696
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP1]]
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP]]
+; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[TMP1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[IV_NEXT:%.*]], %[[PRED_STORE_CONTINUE3:.*]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE3:.*]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 16>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE3]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = shl i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[TMP0]], 16
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr double, ptr [[SRC]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr double, ptr [[SRC]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr [[TMP2]], i64 152
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[TMP3]], i64 152
-; CHECK-NEXT:    [[C:%.*]] = load i32, ptr [[TMP5]], align 4
-; CHECK-NEXT:    [[TMP24:%.*]] = icmp eq i32 [[C]], 0
-; CHECK-NEXT:    [[V_1:%.*]] = load double, ptr [[TMP3]], align 8
-; CHECK-NEXT:    [[CLAMPED:%.*]] = urem i64 [[TMP1]], 128
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP4]], align 4, !alias.scope [[META92:![0-9]+]], !noalias [[META95:![0-9]+]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP5]], align 4, !alias.scope [[META92]], !noalias [[META95]]
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x i32> poison, i32 [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i32> [[TMP8]], i32 [[TMP7]], i32 1
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq <2 x i32> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = load double, ptr [[TMP2]], align 8, !alias.scope [[META92]], !noalias [[META95]]
+; CHECK-NEXT:    [[TMP12:%.*]] = load double, ptr [[TMP3]], align 8, !alias.scope [[META92]], !noalias [[META95]]
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <2 x double> poison, double [[TMP11]], i32 0
+; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <2 x double> [[TMP13]], double [[TMP12]], i32 1
+; CHECK-NEXT:    [[TMP15:%.*]] = urem <2 x i64> [[VEC_IND]], splat (i64 128)
+; CHECK-NEXT:    [[TMP20:%.*]] = fadd <2 x double> [[TMP14]], splat (double 8.000000e+00)
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <2 x i64> [[TMP15]], i64 0
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP17]]
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <2 x i64> [[TMP15]], i64 1
+; CHECK-NEXT:    [[TMP32:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP23]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <2 x i1> [[TMP10]], <2 x double> [[TMP14]], <2 x double> [[TMP20]]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <2 x double> [[TMP18]], i64 0
+; CHECK-NEXT:    store double [[TMP22]], ptr [[TMP19]], align 8, !alias.scope [[META95]]
+; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <2 x double> [[TMP18]], i64 1
+; CHECK-NEXT:    store double [[TMP34]], ptr [[TMP32]], align 8, !alias.scope [[META95]]
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <2 x i1> [[TMP10]], i64 0
 ; CHECK-NEXT:    br i1 [[TMP24]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
 ; CHECK:       [[PRED_STORE_IF]]:
-; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr double, ptr [[DST]], i64 [[CLAMPED]]
-; CHECK-NEXT:    store double [[V_1]], ptr [[TMP19]], align 8
 ; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr i8, ptr [[TMP19]], i64 16
-; CHECK-NEXT:    store double 1.000000e+01, ptr [[TMP25]], align 8
-; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE3]]
+; CHECK-NEXT:    store double 1.000000e+01, ptr [[TMP25]], align 8, !alias.scope [[META95]]
+; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE]]
 ; CHECK:       [[PRED_STORE_CONTINUE]]:
-; CHECK-NEXT:    [[R_1:%.*]] = fadd double [[V_1]], 8.000000e+00
-; CHECK-NEXT:    [[GEP_DST1_ELSE:%.*]] = getelementptr double, ptr [[DST]], i64 [[CLAMPED]]
-; CHECK-NEXT:    store double [[R_1]], ptr [[GEP_DST1_ELSE]], align 8
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <2 x i1> [[TMP10]], i64 1
+; CHECK-NEXT:    br i1 [[TMP26]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3]]
+; CHECK:       [[PRED_STORE_IF2]]:
+; CHECK-NEXT:    [[TMP33:%.*]] = getelementptr i8, ptr [[TMP32]], i64 16
+; CHECK-NEXT:    store double 1.000000e+01, ptr [[TMP33]], align 8, !alias.scope [[META95]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE3]]
 ; CHECK:       [[PRED_STORE_CONTINUE3]]:
-; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[TMP1]], 16
-; CHECK-NEXT:    [[EXIT_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1600
-; CHECK-NEXT:    br i1 [[EXIT_COND]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 32)
+; CHECK-NEXT:    [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; CHECK-NEXT:    br i1 [[TMP28]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP97:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br [[EXIT:label %.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -1091,20 +1128,20 @@ define void @test_three_stores_with_different_predicates(ptr %dst, ptr %src, ptr
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE11:.*]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[INDEX]], 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[COND]], i32 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4, !alias.scope [[META92:![0-9]+]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4, !alias.scope [[META99:![0-9]+]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp ule <2 x i32> [[WIDE_LOAD]], splat (i32 11)
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i1> [[TMP3]], i64 0
 ; CHECK-NEXT:    br i1 [[TMP4]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
 ; CHECK:       [[PRED_STORE_IF]]:
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[INDEX]]
-; CHECK-NEXT:    store i32 1, ptr [[TMP5]], align 4, !alias.scope [[META95:![0-9]+]], !noalias [[META92]]
+; CHECK-NEXT:    store i32 1, ptr [[TMP5]], align 4, !alias.scope [[META102:![0-9]+]], !noalias [[META99]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE]]
 ; CHECK:       [[PRED_STORE_CONTINUE]]:
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i1> [[TMP3]], i64 1
 ; CHECK-NEXT:    br i1 [[TMP6]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]]
 ; CHECK:       [[PRED_STORE_IF2]]:
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[TMP1]]
-; CHECK-NEXT:    store i32 1, ptr [[TMP7]], align 4, !alias.scope [[META95]], !noalias [[META92]]
+; CHECK-NEXT:    store i32 1, ptr [[TMP7]], align 4, !alias.scope [[META102]], !noalias [[META99]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE3]]
 ; CHECK:       [[PRED_STORE_CONTINUE3]]:
 ; CHECK-NEXT:    [[TMP10:%.*]] = icmp ule <2 x i32> [[WIDE_LOAD]], splat (i32 10)
@@ -1112,14 +1149,14 @@ define void @test_three_stores_with_different_predicates(ptr %dst, ptr %src, ptr
 ; CHECK-NEXT:    br i1 [[TMP12]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]]
 ; CHECK:       [[PRED_STORE_IF4]]:
 ; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[INDEX]]
-; CHECK-NEXT:    store i32 2, ptr [[TMP13]], align 4, !alias.scope [[META95]], !noalias [[META92]]
+; CHECK-NEXT:    store i32 2, ptr [[TMP13]], align 4, !alias.scope [[META102]], !noalias [[META99]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE5]]
 ; CHECK:       [[PRED_STORE_CONTINUE5]]:
 ; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x i1> [[TMP10]], i64 1
 ; CHECK-NEXT:    br i1 [[TMP14]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]]
 ; CHECK:       [[PRED_STORE_IF6]]:
 ; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[TMP1]]
-; CHECK-NEXT:    store i32 2, ptr [[TMP15]], align 4, !alias.scope [[META95]], !noalias [[META92]]
+; CHECK-NEXT:    store i32 2, ptr [[TMP15]], align 4, !alias.scope [[META102]], !noalias [[META99]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE7]]
 ; CHECK:       [[PRED_STORE_CONTINUE7]]:
 ; CHECK-NEXT:    [[TMP16:%.*]] = icmp ule <2 x i32> [[WIDE_LOAD]], splat (i32 9)
@@ -1127,19 +1164,19 @@ define void @test_three_stores_with_different_predicates(ptr %dst, ptr %src, ptr
 ; CHECK-NEXT:    br i1 [[TMP18]], label %[[PRED_STORE_IF8:.*]], label %[[PRED_STORE_CONTINUE9:.*]]
 ; CHECK:       [[PRED_STORE_IF8]]:
 ; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[INDEX]]
-; CHECK-NEXT:    store i32 3, ptr [[TMP19]], align 4, !alias.scope [[META95]], !noalias [[META92]]
+; CHECK-NEXT:    store i32 3, ptr [[TMP19]], align 4, !alias.scope [[META102]], !noalias [[META99]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE9]]
 ; CHECK:       [[PRED_STORE_CONTINUE9]]:
 ; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <2 x i1> [[TMP16]], i64 1
 ; CHECK-NEXT:    br i1 [[TMP20]], label %[[PRED_STORE_IF10:.*]], label %[[PRED_STORE_CONTINUE11]]
 ; CHECK:       [[PRED_STORE_IF10]]:
 ; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[TMP1]]
-; CHECK-NEXT:    store i32 3, ptr [[TMP21]], align 4, !alias.scope [[META95]], !noalias [[META92]]
+; CHECK-NEXT:    store i32 3, ptr [[TMP21]], align 4, !alias.scope [[META102]], !noalias [[META99]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE11]]
 ; CHECK:       [[PRED_STORE_CONTINUE11]]:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
 ; CHECK-NEXT:    [[TMP22:%.*]] = icmp eq i32 [[INDEX_NEXT]], 100
-; CHECK-NEXT:    br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP97:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP104:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br [[EXIT:label %.*]]
 ; CHECK:       [[SCALAR_PH]]:
@@ -1213,7 +1250,7 @@ define void @stores_never_sunk_past_alising_loads_or_stores(ptr %dst, ptr %src,
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE21:.*]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[INDEX]], 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[COND]], i32 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4, !alias.scope [[META99:![0-9]+]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4, !alias.scope [[META106:![0-9]+]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp ule <2 x i32> [[WIDE_LOAD]], splat (i32 11)
 ; CHECK-NEXT:    [[TMP4:%.*]] = xor <2 x i1> [[TMP3]], splat (i1 true)
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[INDEX]]
@@ -1221,8 +1258,8 @@ define void @stores_never_sunk_past_alising_loads_or_stores(ptr %dst, ptr %src,
 ; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i1> [[TMP4]], i64 0
 ; CHECK-NEXT:    br i1 [[TMP9]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
 ; CHECK:       [[PRED_LOAD_IF]]:
-; CHECK-NEXT:    store i32 99, ptr [[TMP5]], align 4, !alias.scope [[META102:![0-9]+]], !noalias [[META104:![0-9]+]]
-; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP5]], align 4, !alias.scope [[META102]], !noalias [[META104]]
+; CHECK-NEXT:    store i32 99, ptr [[TMP5]], align 4, !alias.scope [[META109:![0-9]+]], !noalias [[META111:![0-9]+]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP5]], align 4, !alias.scope [[META109]], !noalias [[META111]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x i32> poison, i32 [[TMP7]], i64 0
 ; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE]]
 ; CHECK:       [[PRED_LOAD_CONTINUE]]:
@@ -1230,8 +1267,8 @@ define void @stores_never_sunk_past_alising_loads_or_stores(ptr %dst, ptr %src,
 ; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <2 x i1> [[TMP4]], i64 1
 ; CHECK-NEXT:    br i1 [[TMP13]], label %[[PRED_LOAD_IF10:.*]], label %[[PRED_LOAD_CONTINUE11:.*]]
 ; CHECK:       [[PRED_LOAD_IF10]]:
-; CHECK-NEXT:    store i32 99, ptr [[TMP6]], align 4, !alias.scope [[META102]], !noalias [[META104]]
-; CHECK-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP6]], align 4, !alias.scope [[META102]], !noalias [[META104]]
+; CHECK-NEXT:    store i32 99, ptr [[TMP6]], align 4, !alias.scope [[META109]], !noalias [[META111]]
+; CHECK-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP6]], align 4, !alias.scope [[META109]], !noalias [[META111]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <2 x i32> [[TMP10]], i32 [[TMP11]], i64 1
 ; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE11]]
 ; CHECK:       [[PRED_LOAD_CONTINUE11]]:
@@ -1240,30 +1277,30 @@ define void @stores_never_sunk_past_alising_loads_or_stores(ptr %dst, ptr %src,
 ; CHECK-NEXT:    br i1 [[TMP9]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
 ; CHECK:       [[PRED_STORE_IF]]:
 ; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <2 x i32> [[TMP17]], i64 0
-; CHECK-NEXT:    store i32 [[TMP18]], ptr [[TMP5]], align 4, !alias.scope [[META102]], !noalias [[META104]]
+; CHECK-NEXT:    store i32 [[TMP18]], ptr [[TMP5]], align 4, !alias.scope [[META109]], !noalias [[META111]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE]]
 ; CHECK:       [[PRED_STORE_CONTINUE]]:
 ; CHECK-NEXT:    br i1 [[TMP13]], label %[[PRED_STORE_IF12:.*]], label %[[PRED_STORE_CONTINUE13:.*]]
 ; CHECK:       [[PRED_STORE_IF12]]:
 ; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <2 x i32> [[TMP17]], i64 1
-; CHECK-NEXT:    store i32 [[TMP19]], ptr [[TMP6]], align 4, !alias.scope [[META102]], !noalias [[META104]]
+; CHECK-NEXT:    store i32 [[TMP19]], ptr [[TMP6]], align 4, !alias.scope [[META109]], !noalias [[META111]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE13]]
 ; CHECK:       [[PRED_STORE_CONTINUE13]]:
 ; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <2 x i1> [[TMP3]], i64 0
 ; CHECK-NEXT:    br i1 [[TMP22]], label %[[PRED_LOAD_IF14:.*]], label %[[PRED_LOAD_CONTINUE15:.*]]
 ; CHECK:       [[PRED_LOAD_IF14]]:
-; CHECK-NEXT:    [[TMP21:%.*]] = load i32, ptr [[TMP5]], align 4, !alias.scope [[META102]], !noalias [[META104]]
+; CHECK-NEXT:    [[TMP21:%.*]] = load i32, ptr [[TMP5]], align 4, !alias.scope [[META109]], !noalias [[META111]]
 ; CHECK-NEXT:    [[TMP24:%.*]] = insertelement <2 x i32> poison, i32 [[TMP21]], i64 0
-; CHECK-NEXT:    store i32 99, ptr [[TMP5]], align 4, !alias.scope [[META102]], !noalias [[META104]]
+; CHECK-NEXT:    store i32 99, ptr [[TMP5]], align 4, !alias.scope [[META109]], !noalias [[META111]]
 ; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE15]]
 ; CHECK:       [[PRED_LOAD_CONTINUE15]]:
 ; CHECK-NEXT:    [[TMP20:%.*]] = phi <2 x i32> [ poison, %[[PRED_STORE_CONTINUE13]] ], [ [[TMP24]], %[[PRED_LOAD_IF14]] ]
 ; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <2 x i1> [[TMP3]], i64 1
 ; CHECK-NEXT:    br i1 [[TMP27]], label %[[PRED_LOAD_IF16:.*]], label %[[PRED_LOAD_CONTINUE17:.*]]
 ; CHECK:       [[PRED_LOAD_IF16]]:
-; CHECK-NEXT:    [[TMP25:%.*]] = load i32, ptr [[TMP6]], align 4, !alias.scope [[META102]], !noalias [[META104]]
+; CHECK-NEXT:    [[TMP25:%.*]] = load i32, ptr [[TMP6]], align 4, !alias.scope [[META109]], !noalias [[META111]]
 ; CHECK-NEXT:    [[TMP23:%.*]] = insertelement <2 x i32> [[TMP20]], i32 [[TMP25]], i64 1
-; CHECK-NEXT:    store i32 99, ptr [[TMP6]], align 4, !alias.scope [[META102]], !noalias [[META104]]
+; CHECK-NEXT:    store i32 99, ptr [[TMP6]], align 4, !alias.scope [[META109]], !noalias [[META111]]
 ; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE17]]
 ; CHECK:       [[PRED_LOAD_CONTINUE17]]:
 ; CHECK-NEXT:    [[TMP31:%.*]] = phi <2 x i32> [ [[TMP20]], %[[PRED_LOAD_CONTINUE15]] ], [ [[TMP23]], %[[PRED_LOAD_IF16]] ]
@@ -1272,19 +1309,19 @@ define void @stores_never_sunk_past_alising_loads_or_stores(ptr %dst, ptr %src,
 ; CHECK:       [[PRED_STORE_IF18]]:
 ; CHECK-NEXT:    [[TMP34:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[INDEX]]
 ; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <2 x i32> [[TMP32]], i64 0
-; CHECK-NEXT:    store i32 [[TMP28]], ptr [[TMP34]], align 4, !alias.scope [[META106:![0-9]+]], !noalias [[META99]]
+; CHECK-NEXT:    store i32 [[TMP28]], ptr [[TMP34]], align 4, !alias.scope [[META113:![0-9]+]], !noalias [[META106]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE19]]
 ; CHECK:       [[PRED_STORE_CONTINUE19]]:
 ; CHECK-NEXT:    br i1 [[TMP27]], label %[[PRED_STORE_IF20:.*]], label %[[PRED_STORE_CONTINUE21]]
 ; CHECK:       [[PRED_STORE_IF20]]:
 ; CHECK-NEXT:    [[TMP37:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[TMP1]]
 ; CHECK-NEXT:    [[TMP33:%.*]] = extractelement <2 x i32> [[TMP32]], i64 1
-; CHECK-NEXT:    store i32 [[TMP33]], ptr [[TMP37]], align 4, !alias.scope [[META106]], !noalias [[META99]]
+; CHECK-NEXT:    store i32 [[TMP33]], ptr [[TMP37]], align 4, !alias.scope [[META113]], !noalias [[META106]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE21]]
 ; CHECK:       [[PRED_STORE_CONTINUE21]]:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
 ; CHECK-NEXT:    [[TMP39:%.*]] = icmp eq i32 [[INDEX_NEXT]], 100
-; CHECK-NEXT:    br i1 [[TMP39]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP107:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP39]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP114:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br [[EXIT:label %.*]]
 ; CHECK:       [[SCALAR_PH]]:
@@ -1352,16 +1389,16 @@ define void @loads_never_hoisted_past_alising_stores(ptr %dst, ptr %src, ptr %co
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE13:.*]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[INDEX]], 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[COND]], i32 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4, !alias.scope [[META109:![0-9]+]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4, !alias.scope [[META116:![0-9]+]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp ule <2 x i32> [[WIDE_LOAD]], splat (i32 11)
 ; CHECK-NEXT:    [[TMP4:%.*]] = xor <2 x i1> [[TMP3]], splat (i1 true)
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i1> [[TMP4]], i64 0
 ; CHECK-NEXT:    br i1 [[TMP5]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
 ; CHECK:       [[PRED_STORE_IF]]:
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[INDEX]]
-; CHECK-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP6]], align 4, !alias.scope [[META112:![0-9]+]], !noalias [[META114:![0-9]+]]
+; CHECK-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP6]], align 4, !alias.scope [[META119:![0-9]+]], !noalias [[META121:![0-9]+]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x i32> poison, i32 [[TMP9]], i64 0
-; CHECK-NEXT:    store i32 100, ptr [[TMP6]], align 4, !alias.scope [[META112]], !noalias [[META114]]
+; CHECK-NEXT:    store i32 100, ptr [[TMP6]], align 4, !alias.scope [[META119]], !noalias [[META121]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE]]
 ; CHECK:       [[PRED_STORE_CONTINUE]]:
 ; CHECK-NEXT:    [[TMP8:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP7]], %[[PRED_STORE_IF]] ]
@@ -1369,9 +1406,9 @@ define void @loads_never_hoisted_past_alising_stores(ptr %dst, ptr %src, ptr %co
 ; CHECK-NEXT:    br i1 [[TMP10]], label %[[PRED_STORE_IF10:.*]], label %[[PRED_STORE_CONTINUE11:.*]]
 ; CHECK:       [[PRED_STORE_IF10]]:
 ; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[TMP1]]
-; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP11]], align 4, !alias.scope [[META112]], !noalias [[META114]]
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP11]], align 4, !alias.scope [[META119]], !noalias [[META121]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <2 x i32> [[TMP8]], i32 [[TMP13]], i64 1
-; CHECK-NEXT:    store i32 100, ptr [[TMP11]], align 4, !alias.scope [[META112]], !noalias [[META114]]
+; CHECK-NEXT:    store i32 100, ptr [[TMP11]], align 4, !alias.scope [[META119]], !noalias [[META121]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE11]]
 ; CHECK:       [[PRED_STORE_CONTINUE11]]:
 ; CHECK-NEXT:    [[TMP14:%.*]] = phi <2 x i32> [ [[TMP8]], %[[PRED_STORE_CONTINUE]] ], [ [[TMP12]], %[[PRED_STORE_IF10]] ]
@@ -1380,8 +1417,8 @@ define void @loads_never_hoisted_past_alising_stores(ptr %dst, ptr %src, ptr %co
 ; CHECK-NEXT:    br i1 [[TMP16]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
 ; CHECK:       [[PRED_LOAD_IF]]:
 ; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[INDEX]]
-; CHECK-NEXT:    store i32 99, ptr [[TMP17]], align 4, !alias.scope [[META112]], !noalias [[META114]]
-; CHECK-NEXT:    [[TMP20:%.*]] = load i32, ptr [[TMP17]], align 4, !alias.scope [[META112]], !noalias [[META114]]
+; CHECK-NEXT:    store i32 99, ptr [[TMP17]], align 4, !alias.scope [[META119]], !noalias [[META121]]
+; CHECK-NEXT:    [[TMP20:%.*]] = load i32, ptr [[TMP17]], align 4, !alias.scope [[META119]], !noalias [[META121]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = insertelement <2 x i32> poison, i32 [[TMP20]], i64 0
 ; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE]]
 ; CHECK:       [[PRED_LOAD_CONTINUE]]:
@@ -1390,8 +1427,8 @@ define void @loads_never_hoisted_past_alising_stores(ptr %dst, ptr %src, ptr %co
 ; CHECK-NEXT:    br i1 [[TMP21]], label %[[PRED_LOAD_IF12:.*]], label %[[PRED_LOAD_CONTINUE13]]
 ; CHECK:       [[PRED_LOAD_IF12]]:
 ; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[TMP1]]
-; CHECK-NEXT:    store i32 99, ptr [[TMP22]], align 4, !alias.scope [[META112]], !noalias [[META114]]
-; CHECK-NEXT:    [[TMP24:%.*]] = load i32, ptr [[TMP22]], align 4, !alias.scope [[META112]], !noalias [[META114]]
+; CHECK-NEXT:    store i32 99, ptr [[TMP22]], align 4, !alias.scope [[META119]], !noalias [[META121]]
+; CHECK-NEXT:    [[TMP24:%.*]] = load i32, ptr [[TMP22]], align 4, !alias.scope [[META119]], !noalias [[META121]]
 ; CHECK-NEXT:    [[TMP23:%.*]] = insertelement <2 x i32> [[TMP19]], i32 [[TMP24]], i64 1
 ; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE13]]
 ; CHECK:       [[PRED_LOAD_CONTINUE13]]:
@@ -1401,12 +1438,12 @@ define void @loads_never_hoisted_past_alising_stores(ptr %dst, ptr %src, ptr %co
 ; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[TMP1]]
 ; CHECK-NEXT:    [[TMP29:%.*]] = select <2 x i1> [[TMP3]], <2 x i32> [[TMP26]], <2 x i32> [[TMP15]]
 ; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <2 x i32> [[TMP29]], i64 0
-; CHECK-NEXT:    store i32 [[TMP31]], ptr [[TMP27]], align 4, !alias.scope [[META116:![0-9]+]], !noalias [[META109]]
+; CHECK-NEXT:    store i32 [[TMP31]], ptr [[TMP27]], align 4, !alias.scope [[META123:![0-9]+]], !noalias [[META116]]
 ; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <2 x i32> [[TMP29]], i64 1
-; CHECK-NEXT:    store i32 [[TMP30]], ptr [[TMP28]], align 4, !alias.scope [[META116]], !noalias [[META109]]
+; CHECK-NEXT:    store i32 [[TMP30]], ptr [[TMP28]], align 4, !alias.scope [[META123]], !noalias [[META116]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
 ; CHECK-NEXT:    [[TMP32:%.*]] = icmp eq i32 [[INDEX_NEXT]], 100
-; CHECK-NEXT:    br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP117:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP124:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br [[EXIT:label %.*]]
 ; CHECK:       [[SCALAR_PH]]:
@@ -1455,34 +1492,71 @@ exit:
 define void @sink_multiple_store_groups_noalias_via_scev_srem(ptr %dst, ptr %src) {
 ; CHECK-LABEL: define void @sink_multiple_store_groups_noalias_via_scev_srem(
 ; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]]) {
-; CHECK-NEXT:  [[VECTOR_PH:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br label %[[VECTOR_SCEVCHECK:.*]]
+; CHECK:       [[VECTOR_SCEVCHECK]]:
+; CHECK-NEXT:    [[MUL:%.*]] = call { i7, i1 } @llvm.umul.with.overflow.i7(i7 16, i7 -29)
+; CHECK-NEXT:    [[MUL_OVERFLOW:%.*]] = extractvalue { i7, i1 } [[MUL]], 1
+; CHECK-NEXT:    br i1 [[MUL_OVERFLOW]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK:       [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[SRC]], i64 12828
+; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[DST]], i64 12696
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP1]]
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP]]
+; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[TMP1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[IV_NEXT:%.*]], %[[PRED_STORE_CONTINUE7:.*]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE7:.*]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 16>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE7]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = shl i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[TMP0]], 16
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr double, ptr [[SRC]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr double, ptr [[SRC]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr [[TMP2]], i64 152
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[TMP3]], i64 152
-; CHECK-NEXT:    [[C:%.*]] = load i32, ptr [[TMP5]], align 4
-; CHECK-NEXT:    [[TMP26:%.*]] = icmp eq i32 [[C]], 0
-; CHECK-NEXT:    [[V_1:%.*]] = load double, ptr [[TMP3]], align 8
-; CHECK-NEXT:    [[CLAMPED:%.*]] = srem i64 [[TMP1]], 128
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP4]], align 4, !alias.scope [[META126:![0-9]+]], !noalias [[META129:![0-9]+]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP5]], align 4, !alias.scope [[META126]], !noalias [[META129]]
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x i32> poison, i32 [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i32> [[TMP8]], i32 [[TMP7]], i32 1
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq <2 x i32> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = load double, ptr [[TMP2]], align 8, !alias.scope [[META126]], !noalias [[META129]]
+; CHECK-NEXT:    [[TMP12:%.*]] = load double, ptr [[TMP3]], align 8, !alias.scope [[META126]], !noalias [[META129]]
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <2 x double> poison, double [[TMP11]], i32 0
+; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <2 x double> [[TMP13]], double [[TMP12]], i32 1
+; CHECK-NEXT:    [[TMP15:%.*]] = srem <2 x i64> [[VEC_IND]], splat (i64 128)
+; CHECK-NEXT:    [[TMP18:%.*]] = fadd <2 x double> [[TMP14]], splat (double 8.000000e+00)
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <2 x i64> [[TMP15]], i64 0
+; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP19]]
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <2 x i64> [[TMP15]], i64 1
+; CHECK-NEXT:    [[TMP32:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP23]]
+; CHECK-NEXT:    [[TMP17:%.*]] = select <2 x i1> [[TMP10]], <2 x double> [[TMP14]], <2 x double> [[TMP18]]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <2 x double> [[TMP17]], i64 0
+; CHECK-NEXT:    store double [[TMP22]], ptr [[TMP28]], align 8, !alias.scope [[META129]]
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <2 x double> [[TMP17]], i64 1
+; CHECK-NEXT:    store double [[TMP25]], ptr [[TMP32]], align 8, !alias.scope [[META129]]
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <2 x i1> [[TMP10]], i64 0
 ; CHECK-NEXT:    br i1 [[TMP26]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]]
 ; CHECK:       [[PRED_STORE_IF4]]:
-; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr double, ptr [[DST]], i64 [[CLAMPED]]
-; CHECK-NEXT:    store double [[V_1]], ptr [[TMP28]], align 8
 ; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr i8, ptr [[TMP28]], i64 16
-; CHECK-NEXT:    store double 1.000000e+01, ptr [[TMP29]], align 8
-; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE7]]
+; CHECK-NEXT:    store double 1.000000e+01, ptr [[TMP29]], align 8, !alias.scope [[META129]]
+; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE5]]
 ; CHECK:       [[PRED_STORE_CONTINUE5]]:
-; CHECK-NEXT:    [[R_1:%.*]] = fadd double [[V_1]], 8.000000e+00
-; CHECK-NEXT:    [[GEP_DST1_ELSE:%.*]] = getelementptr double, ptr [[DST]], i64 [[CLAMPED]]
-; CHECK-NEXT:    store double [[R_1]], ptr [[GEP_DST1_ELSE]], align 8
+; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <2 x i1> [[TMP10]], i64 1
+; CHECK-NEXT:    br i1 [[TMP30]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7]]
+; CHECK:       [[PRED_STORE_IF6]]:
+; CHECK-NEXT:    [[TMP33:%.*]] = getelementptr i8, ptr [[TMP32]], i64 16
+; CHECK-NEXT:    store double 1.000000e+01, ptr [[TMP33]], align 8, !alias.scope [[META129]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE7]]
 ; CHECK:       [[PRED_STORE_CONTINUE7]]:
-; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[TMP1]], 16
-; CHECK-NEXT:    [[EXIT_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1600
-; CHECK-NEXT:    br i1 [[EXIT_COND]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 32)
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; CHECK-NEXT:    br i1 [[TMP34]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP131:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br [[EXIT:label %.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop

llvm/test/Transforms/LoopVectorize/interleaved-accesses-requiring-scev-predicates.ll

@@ -117,43 +117,47 @@ exit:
   ret void
 }
 
-; For %gep, we an SCEV that is known not to wrap in the loop with the given trip count.
+; For %gep, we have the following SCEV: ((4 * (zext i4 {0,+,5}<%loop> to i64))<nuw><nsw> + %x).
+; Note the i4 bit wide AddRec {0,+,5}. It is known to wrap in the loop with trip count 16.
 define void @wrap_predicate_for_interleave_group_wraps_for_known_trip_count(ptr noalias %x, ptr noalias %out) {
 ; CHECK-LABEL: define void @wrap_predicate_for_interleave_group_wraps_for_known_trip_count(
 ; CHECK-SAME: ptr noalias [[X:%.*]], ptr noalias [[OUT:%.*]]) {
 ; CHECK-NEXT:  [[START:.*:]]
-; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:    br label %[[VECTOR_SCEVCHECK:.*]]
+; CHECK:       [[VECTOR_SCEVCHECK]]:
+; CHECK-NEXT:    [[MUL:%.*]] = call { i4, i1 } @llvm.umul.with.overflow.i4(i4 5, i4 -1)
+; CHECK-NEXT:    [[MUL_OVERFLOW:%.*]] = extractvalue { i4, i1 } [[MUL]], 1
+; CHECK-NEXT:    br i1 [[MUL_OVERFLOW]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = mul nuw nsw <4 x i64> [[VEC_IND]], splat (i64 5)
-; CHECK-NEXT:    [[TMP1:%.*]] = and <4 x i64> [[TMP0]], splat (i64 9)
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i32, ptr [[X]], i64 [[TMP2]]
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i32, ptr [[X]], i64 [[TMP3]]
-; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
-; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i32, ptr [[X]], i64 [[TMP4]]
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
-; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i32, ptr [[X]], i64 [[TMP5]]
-; CHECK-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP6]], align 4
-; CHECK-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP7]], align 4
-; CHECK-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP8]], align 4
-; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP9]], align 4
-; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <4 x i32> poison, i32 [[TMP10]], i32 0
-; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP11]], i32 1
-; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <4 x i32> [[TMP15]], i32 [[TMP12]], i32 2
-; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <4 x i32> [[TMP16]], i32 [[TMP13]], i32 3
-; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds nuw i32, ptr [[OUT]], i64 [[INDEX]]
-; CHECK-NEXT:    store <4 x i32> [[TMP17]], ptr [[TMP18]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = mul nuw nsw i64 [[INDEX]], 5
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 15
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i32, ptr [[X]], i64 [[TMP1]]
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <20 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <20 x i32> [[WIDE_VEC]], <20 x i32> poison, <4 x i32> <i32 0, i32 5, i32 10, i32 15>
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT:    store <4 x i32> [[STRIDED_VEC]], ptr [[TMP3]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
-; CHECK-NEXT:    br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 12
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    br label %[[EXIT:.*]]
+; CHECK-NEXT:    br label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 12, %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[IV_MUL5:%.*]] = mul nuw nsw i64 [[IV]], 5
+; CHECK-NEXT:    [[IV_MUL5_MASKED:%.*]] = and i64 [[IV_MUL5]], 15
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds nuw i32, ptr [[X]], i64 [[IV_MUL5_MASKED]]
+; CHECK-NEXT:    [[V:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[OUT_I:%.*]] = getelementptr inbounds nuw i32, ptr [[OUT]], i64 [[IV]]
+; CHECK-NEXT:    store i32 [[V]], ptr [[OUT_I]], align 4
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 16
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -164,7 +168,7 @@ loop:
   %iv = phi i64 [ 0, %start ], [ %iv.next, %loop ]
   %iv.next = add nuw nsw i64 %iv, 1
   %iv.mul5 = mul nuw nsw i64 %iv, 5
-  %iv.mul5.masked = and i64 %iv.mul5, 9
+  %iv.mul5.masked = and i64 %iv.mul5, 15
   %gep = getelementptr inbounds nuw i32, ptr %x, i64 %iv.mul5.masked
   %v = load i32, ptr %gep, align 4
   %out.i = getelementptr inbounds nuw i32, ptr %out, i64 %iv
@@ -209,7 +213,7 @@ define void @wrap_predicate_for_interleave_group_unknown_trip_count(ptr noalias
 ; CHECK-NEXT:    store <4 x i32> [[STRIDED_VEC]], ptr [[TMP5]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
@@ -225,7 +229,7 @@ define void @wrap_predicate_for_interleave_group_unknown_trip_count(ptr noalias
 ; CHECK-NEXT:    [[OUT_I:%.*]] = getelementptr inbounds nuw i32, ptr [[OUT]], i64 [[IV]]
 ; CHECK-NEXT:    store i32 [[V]], ptr [[OUT_I]], align 4
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;

llvm/test/tools/llubi/gep-16-bit-addrspace.ll

@@ -1,18 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llubi_test_checks.py UTC_ARGS: --version 6
-; RUN: llubi --verbose < %s 2>&1 | FileCheck %s
-
-target datalayout = "e-m:e-p:16:16"
-
-%s = type { [65536 x i8], i8 }
-
-; Offset of the struct's second field does not fit into a 16 bit pointer.
-define void @main() {
-  %p = alloca i32
-  %a = getelementptr %s, ptr %p, i32 0, i32 1
-  ret void
-}
-; CHECK: Entering function: main
-; CHECK-NEXT:   %p = alloca i32, align 4 => ptr 0x8 [p]
-; CHECK-NEXT:   %a = getelementptr %s, ptr %p, i32 0, i32 1 => ptr 0x8 [p]
-; CHECK-NEXT:   ret void
-; CHECK-NEXT: Exiting function: main

llvm/tools/llubi/lib/Context.cpp

@@ -1054,8 +1054,7 @@ Context::computeGEP(GEPOperator &GEP,
       unsigned ElementIdx = cast<ConstantInt>(V)->getZExtValue();
       const StructLayout *SL = DL.getStructLayout(STy);
       // Element offset is in bytes.
-      ApplyScaledOffset(APInt(IndexBitWidth, SL->getElementOffset(ElementIdx),
-                              /*isSigned=*/false, /*implicitTrunc=*/true),
+      ApplyScaledOffset(APInt(IndexBitWidth, SL->getElementOffset(ElementIdx)),
                         APInt(IndexBitWidth, 1));
       continue;
     }

mlir/include/mlir-c/IR.h

@@ -639,46 +639,9 @@ static inline bool mlirOperationIsNull(MlirOperation op) { return !op.ptr; }
 MLIR_CAPI_EXPORTED bool mlirOperationEqual(MlirOperation op,
                                            MlirOperation other);
 
-/// Compute a hash for the given operation. Operand and result SSA values are
-/// hashed by identity and locations are significant, so equivalent-but-distinct
-/// operations hash differently; use mlirOperationStructuralHashValue for a hash
-/// that pairs with mlirOperationIsStructurallyEquivalent.
+/// Compute a hash for the given operation.
 MLIR_CAPI_EXPORTED size_t mlirOperationHashValue(MlirOperation op);
 
-/// Flags controlling structural operation equivalence and hashing. These mirror
-/// `mlir::OperationEquivalence::Flags` and may be combined with bitwise OR.
-typedef enum MlirOperationEquivalenceFlags {
-  /// No flags: locations, discardable attributes, properties and
-  /// commutativity are all significant.
-  MLIR_OPERATION_EQUIVALENCE_NONE = 0,
-  /// Ignore the locations attached to operations.
-  MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS = 1,
-  /// Ignore the discardable attributes attached to operations.
-  MLIR_OPERATION_EQUIVALENCE_IGNORE_DISCARDABLE_ATTRS = 2,
-  /// Ignore the properties attached to operations.
-  MLIR_OPERATION_EQUIVALENCE_IGNORE_PROPERTIES = 4,
-  /// Ignore commutativity, comparing operands in an order-sensitive way.
-  MLIR_OPERATION_EQUIVALENCE_IGNORE_COMMUTATIVITY = 8,
-} MlirOperationEquivalenceFlags;
-
-/// Checks whether two operations are structurally equivalent, i.e. they have
-/// the same name, attributes, operand and result types, and recursively
-/// equivalent regions. Operand equivalence is tracked structurally while
-/// recursing into regions, so operands defined inside the compared regions need
-/// not be the exact same SSA values; operands defined outside must be. `flags`
-/// is a bitwise OR of MlirOperationEquivalenceFlags values.
-MLIR_CAPI_EXPORTED bool mlirOperationIsStructurallyEquivalent(MlirOperation lhs,
-                                                              MlirOperation rhs,
-                                                              uint32_t flags);
-
-/// Computes a hash for the given operation that pairs with
-/// mlirOperationIsStructurallyEquivalent: two operations that are structurally
-/// equivalent under the same `flags` hash equally. Operands are hashed by
-/// identity, results are not hashed at all, and regions do not participate in
-/// the hash. `flags` is a bitwise OR of MlirOperationEquivalenceFlags values.
-MLIR_CAPI_EXPORTED size_t mlirOperationStructuralHashValue(MlirOperation op,
-                                                           uint32_t flags);
-
 /// Gets the context this operation is associated with
 MLIR_CAPI_EXPORTED MlirContext mlirOperationGetContext(MlirOperation op);
 

mlir/include/mlir/Dialect/X86/TransformOps/X86TransformOps.td

@@ -82,15 +82,5 @@ def ApplyVectorContractToAMXDotProductPatternsOp : Op<Transform_Dialect,
   let assemblyFormat = "attr-dict";
 }
 
-def ApplyMoveAccumulatorForContractLoopPatternsOp : Op<Transform_Dialect,
-    "apply_patterns.x86.move_accumulator_for_contract_loop",
-    [DeclareOpInterfaceMethods<PatternDescriptorOpInterface>]> {
-  let description = [{
-         Rewrites an additive vector contraction by zeroing the accumulator input
-         and materializing the accumulation as a separate add after the contraction.
-  }];
-
-  let assemblyFormat = "attr-dict";
-}
 
 #endif // X86_TRANSFORM_OPS

mlir/include/mlir/Dialect/X86/Transforms.h

@@ -110,11 +110,6 @@ void populateShuffleVectorFMAOpsPatterns(RewritePatternSet &patterns);
 // Int8).
 void populateVectorContractToAMXDotProductPatterns(RewritePatternSet &patterns);
 
-// Rewrites an additive vector contraction by zeroing the accumulator input
-// and materializing the accumulation as a separate add after the contraction.
-void populateMoveAccumulatorForContractLoopPatterns(
-    RewritePatternSet &patterns);
-
 //===----------------------------------------------------------------------===//
 /// Helpers extracted from:
 ///   - clang/lib/Headers/avxintrin.h

mlir/include/mlir/Dialect/X86/Utils/X86Utils.h

@@ -29,10 +29,6 @@ namespace x86 {
 bool isInVnniLayout(Operation *op, llvm::ArrayRef<AffineMap> indexingMaps,
                     std::optional<unsigned> blockingFactor = std::nullopt);
 
-// Recursively follows single-use values through scf.yield operations
-// and returns the first non-yield user result in the contraction chain.
-Value contractionUsersAfterYield(Value v);
-
 // Returns true if two contraction ops form a valid pair for VNNI packing.
 // It verifies that both contractions share the appropriate operand, read from
 // the same source buffer, and use constant indices that differ by 8 or 16.
@@ -64,7 +60,7 @@ LogicalResult shuffleAfterReadLikeOp(PatternRewriter &rewriter, Operation *opA,
 // Shuffles vectors produced by vector.contraction ops into a flat layout
 // before they are written to memory.
 LogicalResult shuffleBeforeWriteLikeOp(PatternRewriter &rewriter,
-                                       Value contractARes, Value contractBRes,
+                                       Operation *opA, Operation *opB,
                                        int64_t nonUnitDimAcc, VectorType accTy);
 
 } // namespace x86

mlir/include/mlir/Dialect/XeGPU/IR/XeGPUAttrs.td

@@ -132,27 +132,6 @@ def XeGPU_FenceScopeAttr:
     let assemblyFormat = "$value";
 }
 
-//===----------------------------------------------------------------------===//
-// XeGPU Lane Shuffle Enums.
-//===----------------------------------------------------------------------===//
-def XeGPU_LaneShufflePack: I32EnumAttrCase<"Pack", 0, "pack">;
-def XeGPU_LaneShuffleUnpack: I32EnumAttrCase<"Unpack", 1, "unpack">;
-def XeGPU_LaneShuffleMode: I32EnumAttr<"LaneShuffleMode",
-      "The direction of the lane shuffle performed by xegpu.lane_shuffle.",
-      [XeGPU_LaneShufflePack, XeGPU_LaneShuffleUnpack]> {
-  let genSpecializedAttr = 0;
-  let cppNamespace = "::mlir::xegpu";
-}
-
-def XeGPU_LaneShuffleModeAttr:
-  EnumAttr<XeGPU_Dialect, XeGPU_LaneShuffleMode, "lane_shuffle_mode"> {
-    let summary = [{Describes the direction of a lane shuffle.
-                    "pack" gathers a lane's elements into consecutive logical
-                    positions. "unpack" scatters them back out, strided by the
-                    subgroup size.}];
-    let assemblyFormat = "$value";
-}
-
 def DistributeLayoutAttr: AttrInterface<"DistributeLayoutAttr"> {
   let cppNamespace = "::mlir::xegpu";
   let description = [{

mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td

@@ -1500,78 +1500,6 @@ def XeGPU_TruncfOp
   let hasVerifier = 1;
 }
 
-def XeGPU_LaneShuffleOp
-    : XeGPU_Op<"lane_shuffle", [Pure, AllTypesMatch<["source", "result"]>]> {
-  let summary = "Re-distributes a subgroup's fragments across its lanes.";
-
-  let description = [{
-    `xegpu.lane_shuffle` is a lane-level operation. The `source` operand is the
-    fragment (a 1D vector) held by one lane, and the `result` is the fragment
-    that lane holds afterwards. It re-distributes the elements across the lanes
-    of the subgroup without changing their type, so `source` and `result` have
-    the same type.
-
-    Let `S` be the subgroup size and `N` the number of elements per lane, so the
-    subgroup collectively holds `S * N` elements. Number these elements by their
-    logical position `0 .. S*N-1`. The `mode` selects which of the two
-    orientations of the `S x N` (lane, element) index grid the operation maps
-    between:
-
-    * `pack`: element `j` of lane `i` holds logical position `j * S + i` before
-      the operation and `i * N + j` after it. A lane's elements end up at
-      consecutive logical positions.
-    * `unpack`: the reverse mapping. A lane's elements end up strided by `S`.
-
-    The two modes are exact inverses, so shuffling one way and then the other
-    yields the original fragments.
-
-    For example, consider `vector<2xi16>` with a subgroup size of 8, denoting a
-    lane's fragment as `<a,b>`. In `pack` mode:
-
-    ```
-    lane:       0      1      2      3      4      5      6      7
-    source:   <0,8>  <1,9>  <2,10> <3,11> <4,12> <5,13> <6,14> <7,15>
-    result:   <0,1>  <2,3>  <4,5>  <6,7>  <8,9> <10,11> <12,13> <14,15>
-    ```
-
-    In general each lane's result elements come from `N` different lanes'
-    `source` fragments -- that is the cross-lane shuffle.
-
-    This operation implements the `xegpu.convert_layout` semantics at the lane
-    level, for the case where the `lane_layout` is the same but the `lane_data`
-    is different.
-
-    The operation assumes a full subgroup (the subgroup size equals the maximum
-    subgroup size) and that all lanes execute the same dynamic instance of the
-    operation; otherwise the behavior is undefined.
-
-    Arguments:
-    - `source`: The 1D vector fragment held by one lane.
-    - `mode`: The direction of the shuffle, either `pack` or `unpack`.
-  }];
-
-  let arguments = (ins XeGPU_LaneShuffleType:$source,
-                       XeGPU_LaneShuffleModeAttr:$mode);
-  let results = (outs XeGPU_LaneShuffleType:$result);
-
-  let extraClassDeclaration = [{
-    VectorType getSourceType() {
-      return llvm::cast<VectorType>(getSource().getType());
-    }
-
-    VectorType getResultType() {
-      return llvm::cast<VectorType>(getResult().getType());
-    }
-  }];
-
-  let assemblyFormat = [{
-    $source $mode attr-dict `:` type($source)
-  }];
-
-  let hasVerifier = 1;
-  let hasFolder = 1;
-}
-
 def XeGPU_DpasMxOp : XeGPU_Op<"dpas_mx", [Pure, AttrSizedOperandSegments,
                                           AnchorLayoutInterface]> {
   let summary = "It performs scaled mma computation";

mlir/include/mlir/Dialect/XeGPU/IR/XeGPUTypes.td

@@ -31,9 +31,6 @@ def XeGPU_VectorOrScalarType
     : AnyTypeOf<[VectorOfRankAndType<[1,2,3,4,5,6,7,8], [XeGPU_ScalarType, Index]>, XeGPU_ScalarType]>;
 def XeGPU_GatherScatterBaseAddrType
     : AnyTypeOf<[MemRefRankOf<[XeGPU_ScalarType], [1]>, XeGPU_PointerType]>;
-// Operand/result type of xegpu.lane_shuffle: a 1D vector of numerical type.
-def XeGPU_LaneShuffleType
-    : FixedVectorOfRankAndType<[1], [XeGPU_ScalarType]>;
 
 // common base class for types in XeGPU dialect
 class XeGPUTypeDef<string name, string typeMnemonic, list<Trait> traits = [],

mlir/lib/CAPI/IR/IR.cpp

@@ -661,25 +661,6 @@ size_t mlirOperationHashValue(MlirOperation op) {
   return OperationEquivalence::computeHash(unwrap(op));
 }
 
-/// Translates the C equivalence flags to mlir::OperationEquivalence::Flags. The
-/// enumerator values mirror each other.
-static OperationEquivalence::Flags unwrapEquivalenceFlags(uint32_t flags) {
-  return static_cast<OperationEquivalence::Flags>(flags);
-}
-
-bool mlirOperationIsStructurallyEquivalent(MlirOperation lhs, MlirOperation rhs,
-                                           uint32_t flags) {
-  return OperationEquivalence::isEquivalentTo(unwrap(lhs), unwrap(rhs),
-                                              unwrapEquivalenceFlags(flags));
-}
-
-size_t mlirOperationStructuralHashValue(MlirOperation op, uint32_t flags) {
-  return OperationEquivalence::computeHash(
-      unwrap(op), /*hashOperands=*/OperationEquivalence::directHashValue,
-      /*hashResults=*/OperationEquivalence::ignoreHashValue,
-      unwrapEquivalenceFlags(flags));
-}
-
 MlirContext mlirOperationGetContext(MlirOperation op) {
   return wrap(unwrap(op)->getContext());
 }

mlir/lib/Dialect/X86/TransformOps/X86TransformOps.cpp

@@ -52,11 +52,6 @@ void mlir::transform::ApplyVectorContractToAMXDotProductPatternsOp::
   x86::populateVectorContractToAMXDotProductPatterns(patterns);
 }
 
-void mlir::transform::ApplyMoveAccumulatorForContractLoopPatternsOp::
-    populatePatterns(RewritePatternSet &patterns) {
-  x86::populateMoveAccumulatorForContractLoopPatterns(patterns);
-}
-
 //===----------------------------------------------------------------------===//
 // Transform op registration
 //===----------------------------------------------------------------------===//

mlir/lib/Dialect/X86/Transforms/CMakeLists.txt

@@ -7,7 +7,6 @@ add_mlir_dialect_library(MLIRX86Transforms
   SinkVectorProducerOps.cpp
   ShuffleVectorFMAOps.cpp
   VectorContractToAMXDotProduct.cpp
-  MoveAccumulatorForContractLoop.cpp
 
   LINK_LIBS PUBLIC
   MLIRArithDialect

mlir/lib/Dialect/X86/Transforms/MoveAccumulatorForContractLoop.cpp

@@ -1,118 +0,0 @@
-//===- MoveAccumulatorForContractLoop.cpp ---------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "mlir/Dialect/SCF/IR/SCF.h"
-#include "mlir/Dialect/Vector/IR/VectorOps.h"
-#include "mlir/Dialect/Vector/Utils/VectorUtils.h"
-#include "mlir/Dialect/X86/Transforms.h"
-#include "mlir/Dialect/X86/Utils/X86Utils.h"
-#include "mlir/Dialect/X86/X86Dialect.h"
-
-#include "mlir/IR/BuiltinAttributes.h"
-#include "mlir/IR/Dominance.h"
-#include "mlir/IR/PatternMatch.h"
-#include "llvm/Support/Casting.h"
-
-#include "mlir/Pass/Pass.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
-
-using namespace mlir;
-using namespace mlir::vector;
-using namespace mlir::x86;
-
-namespace {
-// Transforms vector.contract(A, B, Acc) into vector.contract(A, B, 0) + Acc
-// to decouple the contraction computation from the accumulator update.
-struct MoveAccumulatorForContractLoop
-    : public OpRewritePattern<vector::ContractionOp> {
-  using OpRewritePattern<vector::ContractionOp>::OpRewritePattern;
-
-  LogicalResult matchAndRewrite(vector::ContractionOp contractOp,
-                                PatternRewriter &rewriter) const override {
-
-    if (contractOp.getKind() != vector::CombiningKind::ADD)
-      return rewriter.notifyMatchFailure(contractOp,
-                                         "Expects add combining kind.");
-
-    Operation *accReadOp =
-        traceToVectorReadLikeParentOperation(contractOp.getAcc());
-
-    Value contractValue = contractionUsersAfterYield(contractOp.getResult());
-
-    if (!contractValue)
-      return rewriter.notifyMatchFailure(
-          contractOp, "Final acc write might have multiple users.");
-
-    Operation *resultUserOp = *contractValue.getUsers().begin();
-
-    if (!accReadOp || !resultUserOp)
-      return rewriter.notifyMatchFailure(
-          contractOp, "Read from acc matrix is not by "
-                      "transfer_read/load/constant_zero or multiple users of "
-                      "contract operation.");
-
-    if (isa<arith::ConstantOp>(accReadOp))
-      return rewriter.notifyMatchFailure(
-          contractOp,
-          "The input acc to contract is already a constant vector.");
-
-    if ((accReadOp->getBlock() == contractOp->getBlock()) ||
-        (resultUserOp->getBlock() == contractOp->getBlock()))
-      return rewriter.notifyMatchFailure(
-          contractOp, "Acc read/write should be in a separate block.");
-
-    // Replace acc of a contraction operation with vector constant.
-    Value accValue = accReadOp->getResult(0);
-    if (!accValue.hasOneUse())
-      return rewriter.notifyMatchFailure(
-          contractOp, "The input accumulator has multiple users.");
-
-    Operation *firstUser = *accValue.getUsers().begin();
-    rewriter.setInsertionPoint(firstUser);
-
-    auto vecTy = dyn_cast<VectorType>(accValue.getType());
-    if (!vecTy)
-      return rewriter.notifyMatchFailure(contractOp, "Expects vector type.");
-
-    Location loc = accReadOp->getLoc();
-    Type elemTy = vecTy.getElementType();
-
-    Value zeroVec = arith::ConstantOp::create(
-        rewriter, loc,
-        DenseElementsAttr::get(vecTy, rewriter.getZeroAttr(elemTy)));
-
-    accValue.replaceAllUsesWith(zeroVec);
-
-    // Adds the initial acc value with contract results before storing to acc
-    // matrix.
-    rewriter.setInsertionPoint(resultUserOp);
-    Location locUser = resultUserOp->getLoc();
-
-    Value addition;
-
-    if (isa<FloatType>(elemTy)) {
-      addition =
-          arith::AddFOp::create(rewriter, locUser, contractValue, accValue);
-    } else if (isa<IntegerType>(elemTy)) {
-      addition =
-          arith::AddIOp::create(rewriter, locUser, contractValue, accValue);
-    } else {
-      llvm_unreachable("expected floating-point or integer element type");
-    }
-
-    resultUserOp->replaceUsesOfWith(contractValue, addition);
-    return success();
-  }
-};
-
-} // namespace
-
-void x86::populateMoveAccumulatorForContractLoopPatterns(
-    RewritePatternSet &patterns) {
-  patterns.add<MoveAccumulatorForContractLoop>(patterns.getContext());
-}

mlir/lib/Dialect/X86/Transforms/VectorContractBF16ToFMA.cpp

@@ -415,21 +415,25 @@ struct VectorContractBF16ToFMA
       Operation *accReadOp1 =
           traceToVectorReadLikeParentOperation(pairContractOp.getAcc());
 
-      if (!isa<arith::ConstantOp>(accReadOp0)) {
-        // Shuffle the accumulators of the contract operations.
-        LogicalResult readShuffle =
-            shuffleAfterReadLikeOp(rewriter, accReadOp0, accReadOp1, contractOp,
-                                   pairContractOp, nonUnitDim, accTy);
+      // Iterate down to find the users of contact operations until it is store
+      // or transfer_write.
+      Operation *resultWriteOp0 =
+          traceToVectorWriteLikeUserOperation(contractOp.getResult());
+      Operation *resultWriteOp1 =
+          traceToVectorWriteLikeUserOperation(pairContractOp.getResult());
 
-        if (failed(readShuffle))
-          return rewriter.notifyMatchFailure(
-              contractOp, "Accumulator read is not by transfer_read or load");
-      }
+      // Shuffle the accumulators of the contract operations.
+      LogicalResult readShuffle =
+          shuffleAfterReadLikeOp(rewriter, accReadOp0, accReadOp1, contractOp,
+                                 pairContractOp, nonUnitDim, accTy);
+
+      if (failed(readShuffle))
+        return rewriter.notifyMatchFailure(
+            contractOp, "Accumulator read is not by transfer_read or load");
 
       // Shuffle the output of contract operations before its use.
       LogicalResult writeShuffle = shuffleBeforeWriteLikeOp(
-          rewriter, contractOp.getResult(), pairContractOp.getResult(),
-          nonUnitDim, accTy);
+          rewriter, resultWriteOp0, resultWriteOp1, nonUnitDim, accTy);
 
       if (failed(writeShuffle))
         return rewriter.notifyMatchFailure(

mlir/lib/Dialect/X86/Transforms/VectorContractToPackedTypeDotProduct.cpp

@@ -325,17 +325,17 @@ struct VectorContractToPackedTypeDotProduct
       }
 
       // If the accumulators are shuffled we get nullptr else the
-      // transfer_load or store operations.
-      Operation *accWrite =
-          traceToVectorWriteLikeUserOperation(contractOp.getResult());
+      // transfer_read or load operations.
+      Operation *accRead =
+          traceToVectorReadLikeParentOperation(contractOp.getAcc());
 
       if (!pairContractOp &&
-          (!isNonUnitDimOperandShuffled(nonUnitDimOperand) || accWrite))
+          (!isNonUnitDimOperandShuffled(nonUnitDimOperand) || accRead))
         return rewriter.notifyMatchFailure(contractOp,
                                            "Could not find a contract pair");
 
       // Validate and shuffle the accumulator
-      if (accWrite) {
+      if (accRead) {
         // Trace back to the load or transfer_read operations of the contract
         // accumulators.
         Operation *accReadOp0 =
@@ -352,8 +352,8 @@ struct VectorContractToPackedTypeDotProduct
 
         if (!accReadOp0 || !accReadOp1)
           return rewriter.notifyMatchFailure(
-              contractOp, "Operands doesn't have load or transfer_read or "
-                          "dense constant attribute as its parent op");
+              contractOp,
+              "Operands doesn't have load or transfer_read as it's parent op");
 
         if (!resultWriteOp0 || !resultWriteOp1)
           return rewriter.notifyMatchFailure(
@@ -373,22 +373,18 @@ struct VectorContractToPackedTypeDotProduct
           return rewriter.notifyMatchFailure(
               contractOp, "The store/write operation of contract operation is "
                           "before the pair contract operation");
+        // Shuffle the accumulators of the contract operations.
+        LogicalResult readShuffle =
+            shuffleAfterReadLikeOp(rewriter, accReadOp0, accReadOp1, contractOp,
+                                   pairContractOp, nonUnitDimValue, accTy);
 
-        if (!isa<arith::ConstantOp>(accReadOp0)) {
-          // Shuffle the accumulators of the contract operations.
-          LogicalResult readShuffle = shuffleAfterReadLikeOp(
-              rewriter, accReadOp0, accReadOp1, contractOp, pairContractOp,
-              nonUnitDimValue, accTy);
-
-          if (failed(readShuffle))
-            return rewriter.notifyMatchFailure(
-                contractOp, "Accumulator read is not by transfer_read or load");
-        }
+        if (failed(readShuffle))
+          return rewriter.notifyMatchFailure(
+              contractOp, "Accumulator read is not by transfer_read or load");
 
         // Shuffle the output of contract operations before it's use.
         LogicalResult writeShuffle = shuffleBeforeWriteLikeOp(
-            rewriter, contractOp.getResult(), pairContractOp.getResult(),
-            nonUnitDimValue, accTy);
+            rewriter, resultWriteOp0, resultWriteOp1, nonUnitDimValue, accTy);
 
         if (failed(writeShuffle))
           return rewriter.notifyMatchFailure(

mlir/lib/Dialect/X86/Utils/X86Utils.cpp

@@ -144,25 +144,6 @@ inline ShuffleMasks getShuffleMasks(int64_t nonUnitDimAcc, bool isInt8Avx2) {
   return {maskLo8, maskHi8};
 }
 
-// Recursively follows single-use values through scf.yield operations
-// and returns the first non-yield user result in the contraction chain.
-Value contractionUsersAfterYield(Value v) {
-  if (!v || v.getNumUses() != 1)
-    return nullptr;
-
-  OpOperand &use = *v.use_begin();
-  Operation *user = use.getOwner();
-
-  if (!isa<scf::YieldOp>(user))
-    return v;
-
-  auto yield = cast<scf::YieldOp>(user);
-  Operation *parent = yield->getParentOp();
-  unsigned idx = use.getOperandNumber();
-
-  return contractionUsersAfterYield(parent->getResult(idx));
-}
-
 // This function walks backward from a value to locate its originating
 // vector read-like operation (`vector.transfer_read` or `vector.load`).
 // It follows simple forwarding through unary ops and across `scf.for`
@@ -174,28 +155,8 @@ Operation *traceToVectorReadLikeParentOperation(Value v) {
   while (true) {
     // Case 1: Value defined by an operation
     if (Operation *defOp = v.getDefiningOp()) {
-      if (isa<vector::TransferReadOp, vector::LoadOp, arith::ConstantOp>(
-              defOp)) {
-
-        if (auto constOp = dyn_cast<arith::ConstantOp>(defOp)) {
-          if (auto denseAttr =
-                  dyn_cast<DenseElementsAttr>(constOp.getValue())) {
-            if (!denseAttr.isSplat())
-              return nullptr;
-
-            Attribute splat = denseAttr.getSplatValue<Attribute>();
-
-            if (auto floatAttr = dyn_cast<FloatAttr>(splat))
-              return floatAttr.getValue().isZero() ? defOp : nullptr;
-
-            if (auto intAttr = dyn_cast<IntegerAttr>(splat))
-              return intAttr.getValue().isZero() ? defOp : nullptr;
-          }
-
-          return nullptr;
-        }
+      if (isa<vector::TransferReadOp, vector::LoadOp>(defOp))
         return defOp;
-      }
 
       return nullptr;
     }
@@ -329,23 +290,26 @@ LogicalResult shuffleAfterReadLikeOp(PatternRewriter &rewriter, Operation *opA,
 // This function shuffles the vectors written by vector.contract operation
 // as a flat layout structure before they are stored.
 LogicalResult shuffleBeforeWriteLikeOp(PatternRewriter &rewriter,
-                                       Value contractARes, Value contractBRes,
+                                       Operation *opA, Operation *opB,
                                        int64_t nonUnitDimAcc,
                                        VectorType accTy) {
+  // Helper to extract vector operand from write-like ops
+  auto getWrittenVector = [](Operation *op) -> Value {
+    if (auto write = dyn_cast<vector::TransferWriteOp>(op))
+      return write.getVector();
+    if (auto store = dyn_cast<vector::StoreOp>(op))
+      return store.getValueToStore();
+    return nullptr;
+  };
 
-  Value vecA = contractionUsersAfterYield(contractARes);
-  Value vecB = contractionUsersAfterYield(contractBRes);
+  Value vecA = getWrittenVector(opA);
+  Value vecB = getWrittenVector(opB);
 
   if (!vecA || !vecB)
     return failure();
 
-  Operation *resultWriteOpA = *vecA.getUsers().begin();
-  Operation *resultWriteOpB = *vecB.getUsers().begin();
-
   // Decide insertion point and location
-  Operation *insertBefore = resultWriteOpA->isBeforeInBlock(resultWriteOpB)
-                                ? resultWriteOpA
-                                : resultWriteOpB;
+  Operation *insertBefore = opA->isBeforeInBlock(opB) ? opA : opB;
 
   rewriter.setInsertionPoint(insertBefore);
   Location loc = insertBefore->getLoc();
@@ -371,8 +335,10 @@ LogicalResult shuffleBeforeWriteLikeOp(PatternRewriter &rewriter,
   auto newVecB = vector::ShapeCastOp::create(rewriter, loc, accTy, shuffledHi);
 
   // Update write operands in place via the rewriter to notify it of changes.
-  resultWriteOpA->replaceUsesOfWith(vecA, newVecA);
-  resultWriteOpB->replaceUsesOfWith(vecB, newVecB);
+  rewriter.modifyOpInPlace(opA,
+                           [&]() { opA->setOperand(0, newVecA.getResult()); });
+  rewriter.modifyOpInPlace(opB,
+                           [&]() { opB->setOperand(0, newVecB.getResult()); });
 
   return success();
 }

mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp

@@ -946,29 +946,6 @@ LogicalResult TruncfOp::verify() {
   return success();
 }
 
-//===----------------------------------------------------------------------===//
-// XeGPU_LaneShuffleOp
-//===----------------------------------------------------------------------===//
-
-LogicalResult LaneShuffleOp::verify() {
-  // With a single element per lane there is nothing to re-distribute, so the
-  // operation would be a no-op.
-  if (getSourceType().getNumElements() < 2)
-    return emitOpError("requires a source vector with at least 2 elements.");
-
-  return success();
-}
-
-OpFoldResult LaneShuffleOp::fold(FoldAdaptor adaptor) {
-  // The two modes are exact inverses, so a pack feeding an unpack (or vice
-  // versa) restores the original fragments.
-  auto producer = getSource().getDefiningOp<LaneShuffleOp>();
-  if (producer && producer.getMode() != getMode())
-    return producer.getSource();
-
-  return {};
-}
-
 //===----------------------------------------------------------------------===//
 // XeGPU_DpasMxOp
 //===----------------------------------------------------------------------===//

mlir/test/CAPI/ir.c

@@ -3034,187 +3034,6 @@ int testReplaceUsesWithIf(MlirContext ctx) {
   return 0;
 }
 
-int testOperationEquivalence(MlirContext ctx) {
-  fprintf(stderr, "@testOperationEquivalence\n");
-  // CHECK-LABEL: @testOperationEquivalence
-
-  mlirContextGetOrLoadDialect(ctx, mlirStringRefCreateFromCString("arith"));
-
-  const char *moduleStr = "func.func @f(%arg0: i32) -> i32 {\n"
-                          "  %0 = arith.constant 42 : i32\n"
-                          "  %1 = arith.constant 42 : i32\n"
-                          "  %2 = arith.constant 7 : i32\n"
-                          "  %3 = arith.subi %0, %2 : i32\n"
-                          "  %4 = arith.subi %0, %2 : i32\n"
-                          "  %5 = arith.subi %2, %0 : i32\n"
-                          "  %6 = arith.subi %0, %2 {dialect.discardable} : "
-                          "i32\n"
-                          "  %7 = arith.addi %0, %2 : i32\n"
-                          "  %8 = arith.addi %2, %0 : i32\n"
-                          "  return %0 : i32\n"
-                          "}\n";
-  MlirModule module =
-      mlirModuleCreateParse(ctx, mlirStringRefCreateFromCString(moduleStr));
-
-  MlirBlock moduleBody = mlirModuleGetBody(module);
-  MlirOperation funcOp = mlirBlockGetFirstOperation(moduleBody);
-  MlirRegion funcRegion = mlirOperationGetRegion(funcOp, 0);
-  MlirBlock funcBody = mlirRegionGetFirstBlock(funcRegion);
-
-  MlirOperation c42a = mlirBlockGetFirstOperation(funcBody);
-  MlirOperation c42b = mlirOperationGetNextInBlock(c42a);
-  MlirOperation c7 = mlirOperationGetNextInBlock(c42b);
-  MlirOperation sub3 = mlirOperationGetNextInBlock(c7);
-  MlirOperation sub4 = mlirOperationGetNextInBlock(sub3);
-  MlirOperation sub5 = mlirOperationGetNextInBlock(sub4);
-  MlirOperation sub6 = mlirOperationGetNextInBlock(sub5);
-  MlirOperation add7 = mlirOperationGetNextInBlock(sub6);
-  MlirOperation add8 = mlirOperationGetNextInBlock(add7);
-
-  // Two identical constants are structurally equivalent when locations are
-  // ignored, even though their result SSA values and source locations differ.
-  assert(mlirOperationIsStructurallyEquivalent(
-      c42a, c42b, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
-
-  // Without ignoring locations they differ, since they sit on distinct lines.
-  assert(!mlirOperationIsStructurallyEquivalent(
-      c42a, c42b, MLIR_OPERATION_EQUIVALENCE_NONE));
-
-  // A constant with a different value is not equivalent.
-  assert(!mlirOperationIsStructurallyEquivalent(
-      c42a, c7, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
-
-  // Equivalence is reflexive.
-  assert(mlirOperationIsStructurallyEquivalent(
-      c42a, c42a, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
-
-  // `%3 = subi %0, %2` and `%4 = subi %0, %2` use the exact same operands, so
-  // they are equivalent.
-  assert(mlirOperationIsStructurallyEquivalent(
-      sub3, sub4, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
-
-  // `%5 = subi %2, %0` is not equivalent to `%3 = subi %0, %2`: `subi` is not
-  // commutative, so operands are compared pairwise, and the very first pair
-  // (`%0` = 42 vs `%2` = 7) already mismatches. See the `addi` pair below for
-  // the commutative path.
-  assert(!mlirOperationIsStructurallyEquivalent(
-      sub3, sub5, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
-
-  // A constant and a subtraction are different operations.
-  assert(!mlirOperationIsStructurallyEquivalent(
-      c42a, sub3, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
-
-  // `%6 = subi %0, %2 {dialect.discardable}` carries a discardable attribute
-  // that `%3 = subi %0, %2` does not, so the two differ unless discardable
-  // attributes are ignored as well. This also exercises OR-ing flags together.
-  assert(!mlirOperationIsStructurallyEquivalent(
-      sub3, sub6, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
-  assert(mlirOperationIsStructurallyEquivalent(
-      sub3, sub6,
-      MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS |
-          MLIR_OPERATION_EQUIVALENCE_IGNORE_DISCARDABLE_ATTRS));
-
-  // `%7 = addi %0, %2` and `%8 = addi %2, %0` have swapped operands, but `addi`
-  // is commutative, so they are equivalent through the commutative path and
-  // only differ once commutativity is explicitly ignored.
-  assert(mlirOperationIsStructurallyEquivalent(
-      add7, add8, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
-  assert(!mlirOperationIsStructurallyEquivalent(
-      add7, add8,
-      MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS |
-          MLIR_OPERATION_EQUIVALENCE_IGNORE_COMMUTATIVITY));
-
-  // The `value` of arith.constant is an inherent attribute held in the
-  // operation's properties, so ignoring properties makes constants with
-  // different values equivalent.
-  assert(mlirOperationIsStructurallyEquivalent(
-      c42a, c7,
-      MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS |
-          MLIR_OPERATION_EQUIVALENCE_IGNORE_PROPERTIES));
-
-  // The structural hash pairs with the equivalence above: operations that are
-  // equivalent under a set of flags hash equally under the same flags.
-  assert(mlirOperationStructuralHashValue(
-             c42a, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS) ==
-         mlirOperationStructuralHashValue(
-             c42b, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
-  assert(mlirOperationStructuralHashValue(
-             sub3, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS) ==
-         mlirOperationStructuralHashValue(
-             sub4, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
-  assert(mlirOperationStructuralHashValue(
-             sub3, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS |
-                       MLIR_OPERATION_EQUIVALENCE_IGNORE_DISCARDABLE_ATTRS) ==
-         mlirOperationStructuralHashValue(
-             sub6, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS |
-                       MLIR_OPERATION_EQUIVALENCE_IGNORE_DISCARDABLE_ATTRS));
-  // Commutative operands are folded into the hash in an order-insensitive way,
-  // matching the commutative equivalence above.
-  assert(mlirOperationStructuralHashValue(
-             add7, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS) ==
-         mlirOperationStructuralHashValue(
-             add8, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
-
-  // The flags are threaded through to the hash: locations and discardable
-  // attributes are hashed unless the corresponding flag is set. Hash
-  // inequality is not part of the contract, but these inputs do differ.
-  assert(
-      mlirOperationStructuralHashValue(c42a, MLIR_OPERATION_EQUIVALENCE_NONE) !=
-      mlirOperationStructuralHashValue(c42b, MLIR_OPERATION_EQUIVALENCE_NONE));
-  assert(mlirOperationStructuralHashValue(
-             sub3, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS) !=
-         mlirOperationStructuralHashValue(
-             sub6, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
-
-  mlirModuleDestroy(module);
-
-  // Operands defined *inside* the compared regions need not be the same SSA
-  // values: recursing into the regions marks the two inner constants
-  // equivalent, and the `addi` operands are then matched through that mapping.
-  const char *regionModuleStr = "func.func @g() {\n"
-                                "  %r0 = scf.execute_region -> i32 {\n"
-                                "    %c = arith.constant 1 : i32\n"
-                                "    %d = arith.addi %c, %c : i32\n"
-                                "    scf.yield %d : i32\n"
-                                "  }\n"
-                                "  %r1 = scf.execute_region -> i32 {\n"
-                                "    %c = arith.constant 1 : i32\n"
-                                "    %d = arith.addi %c, %c : i32\n"
-                                "    scf.yield %d : i32\n"
-                                "  }\n"
-                                "  return\n"
-                                "}\n";
-  MlirModule regionModule = mlirModuleCreateParse(
-      ctx, mlirStringRefCreateFromCString(regionModuleStr));
-  assert(!mlirModuleIsNull(regionModule));
-  MlirBlock regionFuncBody = mlirRegionGetFirstBlock(mlirOperationGetRegion(
-      mlirBlockGetFirstOperation(mlirModuleGetBody(regionModule)), 0));
-  MlirOperation exec0 = mlirBlockGetFirstOperation(regionFuncBody);
-  MlirOperation exec1 = mlirOperationGetNextInBlock(exec0);
-  assert(mlirOperationIsStructurallyEquivalent(
-      exec0, exec1, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
-
-  // The nested `addi` operations compared on their own are *not* equivalent:
-  // the value mapping starts out empty, so operands defined outside the
-  // compared operation must be the exact same SSA values.
-  MlirBlock exec0Body =
-      mlirRegionGetFirstBlock(mlirOperationGetRegion(exec0, 0));
-  MlirBlock exec1Body =
-      mlirRegionGetFirstBlock(mlirOperationGetRegion(exec1, 0));
-  MlirOperation innerAdd0 =
-      mlirOperationGetNextInBlock(mlirBlockGetFirstOperation(exec0Body));
-  MlirOperation innerAdd1 =
-      mlirOperationGetNextInBlock(mlirBlockGetFirstOperation(exec1Body));
-  assert(!mlirOperationIsStructurallyEquivalent(
-      innerAdd0, innerAdd1, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
-
-  mlirModuleDestroy(regionModule);
-
-  // CHECK: testOperationEquivalence: PASSED
-  fprintf(stderr, "testOperationEquivalence: PASSED\n");
-  return 0;
-}
-
 int main(void) {
   MlirContext ctx = mlirContextCreate();
   registerAllUpstreamDialects(ctx);
@@ -3272,8 +3091,6 @@ int main(void) {
     return 20;
   if (testReplaceUsesWithIf(ctx))
     return 21;
-  if (testOperationEquivalence(ctx))
-    return 22;
 
   // CHECK: DESTROY MAIN CONTEXT
   // CHECK: reportResourceDelete: resource_i64_blob

mlir/test/Dialect/X86/move-acc-for-contract-loop.mlir

@@ -1,323 +0,0 @@
-// RUN: mlir-opt %s -transform-interpreter -cse -split-input-file | FileCheck %s
-
-!vecA = vector<1x1x2xbf16>
-!vecB = vector<1x2x16xbf16>
-!vecC = vector<1x16xf32>
-!memrefA = memref<1x1x2xbf16, strided<[2048, 32, 1], offset: ?>>
-!memrefB = memref<1x2x32xbf16, strided<[2048, 64, 1], offset: ?>>
-!memrefC = memref<1x32xf32, strided<[64, 1], offset: ?>>
-#map = affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>
-#map1 = affine_map<(d0, d1, d2, d3) -> (d0, d3, d2)>
-#map2 = affine_map<(d0, d1, d2, d3) -> (d1, d2)>
-
-func.func @brmatmul_acc_mv(%arg0: memref<16x64x32xbf16>, %arg1: memref<16x32x64xbf16>,
-                             %arg2: memref<64x64xf32>) -> memref<64x64xf32> {
-  %0 = ub.poison : f32
-  %1 = ub.poison : bf16
-  %c0 = arith.constant 0 : index
-  %c64 = arith.constant 64 : index
-  %c16 = arith.constant 16 : index
-  %c32 = arith.constant 32 : index
-  %c1 = arith.constant 1 : index
-  %c2 = arith.constant 2 : index
-  scf.for %arg3 = %c0 to %c64 step %c1 {
-    scf.for %arg4 = %c0 to %c64 step %c32 {
-      %subview = memref.subview %arg2[%arg3, %arg4] [1, 32] [1, 1]
-                        : memref<64x64xf32> to !memrefC
-      %2 = vector.transfer_read %subview[%c0, %c0], %0 {in_bounds = [true, true]}
-                        : !memrefC, !vecC
-      %3 = vector.transfer_read %subview[%c0, %c16], %0 {in_bounds = [true, true]}
-                        : !memrefC, !vecC
-
-      %4:2 = scf.for %arg5 = %c0 to %c16 step %c1 iter_args(%arg6 = %2, %arg7 = %3) -> (!vecC, !vecC) {
-        %5:2 = scf.for %arg8 = %c0 to %c32 step %c2 iter_args(%arg9 = %arg6, %arg10 = %arg7) -> (!vecC, !vecC) {
-
-          %subview_0 = memref.subview %arg0[%arg5, %arg3, %arg8] [1, 1, 2] [1, 1, 1]
-                        : memref<16x64x32xbf16> to !memrefA
-          %subview_1 = memref.subview %arg1[%arg5, %arg8, %arg4] [1, 2, 32] [1, 1, 1]
-                        : memref<16x32x64xbf16> to !memrefB
-
-          %6 = vector.transfer_read %subview_0[%c0, %c0, %c0], %1 {in_bounds = [true, true, true]}
-                        : !memrefA, !vecA
-          %7 = vector.transfer_read %subview_1[%c0, %c0, %c0], %1 {in_bounds = [true, true, true]}
-                        : !memrefB, !vecB
-          %8 = vector.transfer_read %subview_1[%c0, %c0, %c16], %1 {in_bounds = [true, true, true]}
-                        : !memrefB, !vecB
-
-          %9 = vector.contract {indexing_maps = [#map, #map1, #map2], iterator_types =
-                        ["reduction", "parallel", "parallel", "reduction"], kind = #vector.kind<add>} %6, %7, %arg9
-                        {unroll_shape = array<i64: 1, 1, 16, 2>} : !vecA, !vecB into !vecC
-          %10 = vector.contract {indexing_maps = [#map, #map1, #map2], iterator_types =
-                        ["reduction", "parallel", "parallel", "reduction"], kind = #vector.kind<add>} %6, %8, %arg10
-                        {unroll_shape = array<i64: 1, 1, 16, 2>} : !vecA, !vecB into !vecC
-
-          scf.yield %9, %10 : !vecC, !vecC
-        }
-        scf.yield %5#0, %5#1 : !vecC, !vecC
-      }
-
-      vector.transfer_write %4#1, %subview[%c0, %c16] {in_bounds = [true, true]}
-                        : !vecC, !memrefC
-      vector.transfer_write %4#0, %subview[%c0, %c0] {in_bounds = [true, true]}
-                        : !vecC, !memrefC
-    }
-  }
-
-  return %arg2 : memref<64x64xf32>
-}
-
-// CHECK-LABEL: @brmatmul_acc_mv
-// CHECK: %[[CST:.*]] = arith.constant dense<0.000000e+00> : vector<1x16xf32>
-// CHECK: scf.for
-// CHECK-NEXT: scf.for
-// CHECK: memref.subview
-// CHECK: %[[TRANSFER_READ_0:.*]] = vector.transfer_read
-// CHECK: %[[TRANSFER_READ_1:.*]] = vector.transfer_read
-// CHECK: scf.for {{.*}} iter_args(%[[VAL_3:.*]] = %[[CST]], %[[VAL_4:.*]] = %[[CST]]) -> (vector<1x16xf32>, vector<1x16xf32>) {
-// CHECK-NEXT: scf.for
-// CHECK: scf.yield
-// CHECK: scf.yield
-// CHECK: %[[ADDF_0:.*]] = arith.addf {{.*}}, %[[TRANSFER_READ_1]] : vector<1x16xf32>
-// CHECK-NEXT: vector.transfer_write %[[ADDF_0:.*]], %subview[%c0, %c16] {{.*}}
-// CHECK: %[[ADDF_1:.*]] = arith.addf {{.*}}, %[[TRANSFER_READ_0]] : vector<1x16xf32>
-// CHECK-NEXT: vector.transfer_write %[[ADDF_1:.*]], %subview[%c0, %c0] {{.*}}
-
-module attributes {transform.with_named_sequence} {
-  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
-    %func = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    transform.apply_patterns to %func {
-      transform.apply_patterns.x86.move_accumulator_for_contract_loop
-    } : !transform.any_op
-    transform.yield
-  }
-}
-
-// -----
-
-!vecA = vector<1x16x16x2xbf16>
-!vecB = vector<1x16x16x2xbf16>
-!vecC = vector<16x16xf32>
-!memrefA = memref<1x32x16x2xbf16>
-!memrefB = memref<1x16x32x2xbf16>
-!memrefC = memref<32x32xf32>
-#map = affine_map<(d0, d4, d1, d2, d3) -> (d0, d1, d3, d4)>
-#map1 = affine_map<(d0, d4, d1, d2, d3) -> (d0, d3, d2, d4)>
-#map2 = affine_map<(d0, d4, d1, d2, d3) -> (d1, d2)>
-func.func @negative_no_loop(
-  %arg0: !memrefA, %arg1: !memrefB, %arg2: !memrefC) -> !memrefC
-{
-  %c0 = arith.constant 0 : index
-  %0 = ub.poison : bf16
-  %32 = ub.poison : f32
-
-  %1 = vector.transfer_read %arg0[%c0, %c0, %c0, %c0], %0 {in_bounds = [true, true, true, true]} :
-        !memrefA, !vecA
-  %2 = vector.transfer_read %arg1[%c0, %c0, %c0, %c0], %0 {in_bounds = [true, true, true, true]} :
-        !memrefB, !vecB
-
-  %3 = vector.transfer_read %arg2[%c0, %c0], %32 {in_bounds = [true, true]} : !memrefC, !vecC
-
-  %4 = vector.contract {
-    indexing_maps = [#map, #map1, #map2],
-    iterator_types = ["reduction", "reduction", "parallel", "parallel", "reduction"],
-    kind = #vector.kind<add>}
-    %1, %2, %3 : !vecA, !vecB into !vecC
-
-  vector.transfer_write %4, %arg2[%c0, %c0] {in_bounds = [true, true]} : !vecC, !memrefC
-
-  return %arg2 : !memrefC
-}
-
-// CHECK-LABEL: @negative_no_loop
-// CHECK-NOT: arith.constant dense<0.000000e+00> {{.*}}
-// CHECK-NOT: arith.addf
-// CHECK-NOT: arith.addf
-
-module attributes {transform.with_named_sequence} {
-  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
-    %func = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    transform.apply_patterns to %func {
-      transform.apply_patterns.x86.move_accumulator_for_contract_loop
-    } : !transform.any_op
-    transform.yield
-  }
-}
-
-// -----
-
-!vecA = vector<1x1x2xbf16>
-!vecB = vector<1x2x16xbf16>
-!vecC = vector<1x16xf32>
-!memrefA = memref<1x1x2xbf16, strided<[2048, 32, 1], offset: ?>>
-!memrefB = memref<1x2x32xbf16, strided<[2048, 64, 1], offset: ?>>
-!memrefC = memref<1x32xf32, strided<[64, 1], offset: ?>>
-#map = affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>
-#map1 = affine_map<(d0, d1, d2, d3) -> (d0, d3, d2)>
-#map2 = affine_map<(d0, d1, d2, d3) -> (d1, d2)>
-
-func.func @negative_non_zero_cst(%arg0: memref<16x64x32xbf16>, %arg1: memref<16x32x64xbf16>,
-                             %arg2: memref<64x64xf32>) -> memref<64x64xf32> {
-  %cst2 = arith.constant dense<2.000000e+00> : vector<1x16xf32>
-  %cst3 = arith.constant dense<3.000000e+00> : vector<1x16xf32>
-  %0 = ub.poison : f32
-  %1 = ub.poison : bf16
-  %c0 = arith.constant 0 : index
-  %c64 = arith.constant 64 : index
-  %c16 = arith.constant 16 : index
-  %c32 = arith.constant 32 : index
-  %c1 = arith.constant 1 : index
-  %c2 = arith.constant 2 : index
-  scf.for %arg3 = %c0 to %c64 step %c1 {
-    scf.for %arg4 = %c0 to %c64 step %c32 {
-      %subview = memref.subview %arg2[%arg3, %arg4] [1, 32] [1, 1]
-                        : memref<64x64xf32> to !memrefC
-
-      %4:2 = scf.for %arg5 = %c0 to %c16 step %c1 iter_args(%arg6 = %cst2, %arg7 = %cst3) -> (!vecC, !vecC) {
-        %5:2 = scf.for %arg8 = %c0 to %c32 step %c2 iter_args(%arg9 = %arg6, %arg10 = %arg7) -> (!vecC, !vecC) {
-
-          %subview_0 = memref.subview %arg0[%arg5, %arg3, %arg8] [1, 1, 2] [1, 1, 1]
-                        : memref<16x64x32xbf16> to !memrefA
-          %subview_1 = memref.subview %arg1[%arg5, %arg8, %arg4] [1, 2, 32] [1, 1, 1]
-                        : memref<16x32x64xbf16> to !memrefB
-
-          %6 = vector.transfer_read %subview_0[%c0, %c0, %c0], %1 {in_bounds = [true, true, true]}
-                        : !memrefA, !vecA
-          %7 = vector.transfer_read %subview_1[%c0, %c0, %c0], %1 {in_bounds = [true, true, true]}
-                        : !memrefB, !vecB
-          %8 = vector.transfer_read %subview_1[%c0, %c0, %c16], %1 {in_bounds = [true, true, true]}
-                        : !memrefB, !vecB
-
-          %9 = vector.contract {indexing_maps = [#map, #map1, #map2], iterator_types =
-                        ["reduction", "parallel", "parallel", "reduction"], kind = #vector.kind<add>} %6, %7, %arg9
-                        {unroll_shape = array<i64: 1, 1, 16, 2>} : !vecA, !vecB into !vecC
-          %10 = vector.contract {indexing_maps = [#map, #map1, #map2], iterator_types =
-                        ["reduction", "parallel", "parallel", "reduction"], kind = #vector.kind<add>} %6, %8, %arg10
-                        {unroll_shape = array<i64: 1, 1, 16, 2>} : !vecA, !vecB into !vecC
-
-          scf.yield %9, %10 : !vecC, !vecC
-        }
-        scf.yield %5#0, %5#1 : !vecC, !vecC
-      }
-
-      vector.transfer_write %4#1, %subview[%c0, %c16] {in_bounds = [true, true]}
-                        : !vecC, !memrefC
-      vector.transfer_write %4#0, %subview[%c0, %c0] {in_bounds = [true, true]}
-                        : !vecC, !memrefC
-    }
-  }
-
-  return %arg2 : memref<64x64xf32>
-}
-
-// CHECK-LABEL: @negative_non_zero_cst
-// CHECK-NOT: %[[CST:.*]] = arith.constant dense<0.000000e+00> : vector<1x16xf32>
-// CHECK: scf.for
-// CHECK-NEXT: scf.for
-// CHECK: memref.subview
-// CHECK: scf.for 
-// CHECK-NEXT: scf.for
-// CHECK: scf.yield
-// CHECK: scf.yield
-// CHECK-NOT: arith.addf 
-
-
-module attributes {transform.with_named_sequence} {
-  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
-    %func = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    transform.apply_patterns to %func {
-      transform.apply_patterns.x86.move_accumulator_for_contract_loop
-    } : !transform.any_op
-    transform.yield
-  }
-}
-
-// -----
-
-!vecA = vector<1x1x2xbf16>
-!vecB = vector<1x2x16xbf16>
-!vecC = vector<1x16xf32>
-!memrefA = memref<1x1x2xbf16, strided<[2048, 32, 1], offset: ?>>
-!memrefB = memref<1x2x32xbf16, strided<[2048, 64, 1], offset: ?>>
-!memrefC = memref<1x32xf32, strided<[64, 1], offset: ?>>
-#map = affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>
-#map1 = affine_map<(d0, d1, d2, d3) -> (d0, d3, d2)>
-#map2 = affine_map<(d0, d1, d2, d3) -> (d1, d2)>
-
-func.func @negative_input_acc_many_users(%arg0: memref<16x64x32xbf16>, %arg1: memref<16x32x64xbf16>,
-                             %arg2: memref<64x64xf32>) -> memref<64x64xf32> {
-  %0 = ub.poison : f32
-  %1 = ub.poison : bf16
-  %c0 = arith.constant 0 : index
-  %c64 = arith.constant 64 : index
-  %c16 = arith.constant 16 : index
-  %c32 = arith.constant 32 : index
-  %c1 = arith.constant 1 : index
-  %c2 = arith.constant 2 : index
-  scf.for %arg3 = %c0 to %c64 step %c1 {
-    scf.for %arg4 = %c0 to %c64 step %c32 {
-      %subview = memref.subview %arg2[%arg3, %arg4] [1, 32] [1, 1]
-                        : memref<64x64xf32> to !memrefC
-      %2 = vector.transfer_read %subview[%c0, %c0], %0 {in_bounds = [true, true]}
-                        : !memrefC, !vecC
-      %3 = vector.transfer_read %subview[%c0, %c16], %0 {in_bounds = [true, true]}
-                        : !memrefC, !vecC
-
-      %4:2 = scf.for %arg5 = %c0 to %c16 step %c1 iter_args(%arg6 = %2, %arg7 = %3) -> (!vecC, !vecC) {
-        %5:2 = scf.for %arg8 = %c0 to %c32 step %c2 iter_args(%arg9 = %arg6, %arg10 = %arg7) -> (!vecC, !vecC) {
-
-          %subview_0 = memref.subview %arg0[%arg5, %arg3, %arg8] [1, 1, 2] [1, 1, 1]
-                        : memref<16x64x32xbf16> to !memrefA
-          %subview_1 = memref.subview %arg1[%arg5, %arg8, %arg4] [1, 2, 32] [1, 1, 1]
-                        : memref<16x32x64xbf16> to !memrefB
-
-          %6 = vector.transfer_read %subview_0[%c0, %c0, %c0], %1 {in_bounds = [true, true, true]}
-                        : !memrefA, !vecA
-          %7 = vector.transfer_read %subview_1[%c0, %c0, %c0], %1 {in_bounds = [true, true, true]}
-                        : !memrefB, !vecB
-          %8 = vector.transfer_read %subview_1[%c0, %c0, %c16], %1 {in_bounds = [true, true, true]}
-                        : !memrefB, !vecB
-
-          %9 = vector.contract {indexing_maps = [#map, #map1, #map2], iterator_types =
-                        ["reduction", "parallel", "parallel", "reduction"], kind = #vector.kind<add>} %6, %7, %arg9
-                        {unroll_shape = array<i64: 1, 1, 16, 2>} : !vecA, !vecB into !vecC
-          %10 = vector.contract {indexing_maps = [#map, #map1, #map2], iterator_types =
-                        ["reduction", "parallel", "parallel", "reduction"], kind = #vector.kind<add>} %6, %8, %arg10
-                        {unroll_shape = array<i64: 1, 1, 16, 2>} : !vecA, !vecB into !vecC
-
-          scf.yield %9, %10 : !vecC, !vecC
-        }
-        scf.yield %5#0, %5#1 : !vecC, !vecC
-      }
-
-      %7 = arith.addf %4#1, %2 : vector<1x16xf32>
-      vector.transfer_write %7, %subview[%c0, %c16] {in_bounds = [true, true]}
-                        : !vecC, !memrefC
-      %8 = arith.addf %4#0, %3 : vector<1x16xf32>
-      vector.transfer_write %8, %subview[%c0, %c0] {in_bounds = [true, true]}
-                        : !vecC, !memrefC
-    }
-  }
-
-  return %arg2 : memref<64x64xf32>
-}
-// CHECK-LABEL: @negative_input_acc_many_users
-// CHECK-NOT: %[[CST:.*]] = arith.constant dense<0.000000e+00> : vector<1x16xf32>
-// CHECK: scf.for
-// CHECK-NEXT: scf.for
-// CHECK: memref.subview
-// CHECK: scf.for
-// CHECK-NEXT: scf.for
-// CHECK: scf.yield
-// CHECK: scf.yield
-
-
-module attributes {transform.with_named_sequence} {
-  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
-    %func = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    transform.apply_patterns to %func {
-      transform.apply_patterns.x86.move_accumulator_for_contract_loop
-    } : !transform.any_op
-    transform.yield
-  }
-}
-

mlir/test/Dialect/X86/vector-contract-bf16-to-fma.mlir

@@ -444,6 +444,8 @@ func.func @matmul_to_fma_flat_layout_loop(%arg0: memref<16x64x32xbf16>, %arg1: m
 }
 
 // CHECK-LABEL: @matmul_to_fma_flat_layout_loop
+// CHECK: vector.shuffle{{.*}}[0, 8, 1, 9, 2, 10, 3, 11] : vector<8xf32>, vector<8xf32>
+// CHECK-NEXT: vector.shuffle{{.*}}[4, 12, 5, 13, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
 // CHECK: scf.for
 // CHECK: scf.for
 // CHECK: x86.avx.bcst_to_f32.packed
@@ -459,7 +461,6 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %func = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     transform.apply_patterns to %func {
-      transform.apply_patterns.x86.move_accumulator_for_contract_loop
       transform.apply_patterns.x86.vector_contract_bf16_to_fma
     } : !transform.any_op
     transform.yield

mlir/test/Dialect/X86/vector-contract-to-packed-type-dotproduct.mlir

@@ -687,6 +687,8 @@ func.func @brmatmul_bf16dp_flat_layout_loop(%arg0: memref<16x64x32xbf16>, %arg1:
 }
 
 // CHECK-LABEL: @brmatmul_bf16dp_flat_layout_loop
+// CHECK: vector.shuffle{{.*}}[0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23] : vector<16xf32>, vector<16xf32>
+// CHECK-NEXT: vector.shuffle{{.*}}[8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31] : vector<16xf32>, vector<16xf32>
 // CHECK: scf.for
 // CHECK: scf.for
 // CHECK: vector.transfer_read {{.*}} vector<32xbf16>
@@ -704,7 +706,6 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %func = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     transform.apply_patterns to %func {
-      transform.apply_patterns.x86.move_accumulator_for_contract_loop
       transform.apply_patterns.x86.vector_contract_to_packed_type_dot_product
     } : !transform.any_op
     transform.yield

mlir/test/Dialect/XeGPU/canonicalize.mlir

@@ -1,36 +0,0 @@
-// RUN: mlir-opt --canonicalize --split-input-file %s | FileCheck %s
-
-// CHECK-LABEL: func.func @fold_lane_shuffle_pack_unpack
-// CHECK-SAME: %[[ARG0:.+]]: vector<2xi16>
-// CHECK-NOT: xegpu.lane_shuffle
-// CHECK: return %[[ARG0]]
-func.func @fold_lane_shuffle_pack_unpack(%a: vector<2xi16>) -> vector<2xi16> {
-  %0 = xegpu.lane_shuffle %a pack : vector<2xi16>
-  %1 = xegpu.lane_shuffle %0 unpack : vector<2xi16>
-  return %1 : vector<2xi16>
-}
-
-// -----
-
-// CHECK-LABEL: func.func @fold_lane_shuffle_unpack_pack
-// CHECK-SAME: %[[ARG0:.+]]: vector<4xf8E5M2>
-// CHECK-NOT: xegpu.lane_shuffle
-// CHECK: return %[[ARG0]]
-func.func @fold_lane_shuffle_unpack_pack(%a: vector<4xf8E5M2>) -> vector<4xf8E5M2> {
-  %0 = xegpu.lane_shuffle %a unpack : vector<4xf8E5M2>
-  %1 = xegpu.lane_shuffle %0 pack : vector<4xf8E5M2>
-  return %1 : vector<4xf8E5M2>
-}
-
-// -----
-
-// Two shuffles in the same direction are not inverses and must not fold.
-
-// CHECK-LABEL: func.func @no_fold_lane_shuffle_pack_pack
-// CHECK: xegpu.lane_shuffle {{.*}} pack
-// CHECK: xegpu.lane_shuffle {{.*}} pack
-func.func @no_fold_lane_shuffle_pack_pack(%a: vector<2xi16>) -> vector<2xi16> {
-  %0 = xegpu.lane_shuffle %a pack : vector<2xi16>
-  %1 = xegpu.lane_shuffle %0 pack : vector<2xi16>
-  return %1 : vector<2xi16>
-}

mlir/test/Dialect/XeGPU/invalid.mlir

@@ -693,13 +693,6 @@ func.func @truncf_invalid_result_size(%a: vector<8x16xf16>) {
   return
 }
 
-// -----
-func.func @lane_shuffle_single_element(%a: vector<1xi32>) {
-  // expected-error@+1 {{op requires a source vector with at least 2 elements}}
-  %1 = xegpu.lane_shuffle %a pack : vector<1xi32>
-  return
-}
-
 // -----
 func.func @dpas_mx_acc_result_type_mismatch(%a : vector<8x16xf8E5M2>, %b: vector<16x16xf8E5M2>, %acc: vector<8x16xbf16>) {
   // expected-error@+1 {{Accumulator type must match result type.}}

mlir/test/Dialect/XeGPU/ops.mlir

@@ -697,48 +697,6 @@ gpu.func @truncf(%a: vector<8x16xf16>) {
   gpu.return
 }
 
-// CHECK-LABEL: gpu.func @lane_shuffle_pack_f4
-gpu.func @lane_shuffle_pack_f4(%a: vector<4xf4E2M1FN>) {
-  // CHECK: %{{.+}} = xegpu.lane_shuffle %{{.+}} pack : vector<4xf4E2M1FN>
-  %1 = xegpu.lane_shuffle %a pack : vector<4xf4E2M1FN>
-  gpu.return
-}
-
-// CHECK-LABEL: gpu.func @lane_shuffle_unpack_f4
-gpu.func @lane_shuffle_unpack_f4(%a: vector<4xf4E2M1FN>) {
-  // CHECK: %{{.+}} = xegpu.lane_shuffle %{{.+}} unpack : vector<4xf4E2M1FN>
-  %1 = xegpu.lane_shuffle %a unpack : vector<4xf4E2M1FN>
-  gpu.return
-}
-
-// CHECK-LABEL: gpu.func @lane_shuffle_pack_f8
-gpu.func @lane_shuffle_pack_f8(%a: vector<4xf8E5M2>) {
-  // CHECK: %{{.+}} = xegpu.lane_shuffle %{{.+}} pack : vector<4xf8E5M2>
-  %1 = xegpu.lane_shuffle %a pack : vector<4xf8E5M2>
-  gpu.return
-}
-
-// CHECK-LABEL: gpu.func @lane_shuffle_unpack_f8
-gpu.func @lane_shuffle_unpack_f8(%a: vector<4xf8E5M2>) {
-  // CHECK: %{{.+}} = xegpu.lane_shuffle %{{.+}} unpack : vector<4xf8E5M2>
-  %1 = xegpu.lane_shuffle %a unpack : vector<4xf8E5M2>
-  gpu.return
-}
-
-// CHECK-LABEL: gpu.func @lane_shuffle_pack_f16
-gpu.func @lane_shuffle_pack_f16(%a: vector<2xf16>) {
-  // CHECK: %{{.+}} = xegpu.lane_shuffle %{{.+}} pack : vector<2xf16>
-  %1 = xegpu.lane_shuffle %a pack : vector<2xf16>
-  gpu.return
-}
-
-// CHECK-LABEL: gpu.func @lane_shuffle_unpack_f16
-gpu.func @lane_shuffle_unpack_f16(%a: vector<2xf16>) {
-  // CHECK: %{{.+}} = xegpu.lane_shuffle %{{.+}} unpack : vector<2xf16>
-  %1 = xegpu.lane_shuffle %a unpack : vector<2xf16>
-  gpu.return
-}
-
 // CHECK-LABEL: gpu.func @dpas_mx
 gpu.func @dpas_mx(%a : vector<8x32xf8E5M2>, %b: vector<32x16xf8E5M2>, %acc: vector<8x16xbf16>, %a_scale: vector<8x1xf8E8M0FNU>, %b_scale: vector<1x16xf8E8M0FNU>) {
   // CHECK: %{{.+}} = xegpu.dpas_mx %{{.+}}, %{{.+}}, %{{.+}} scale_a = %{{.+}} scale_b = %{{.+}} : (vector<8x32xf8E5M2>, vector<32x16xf8E5M2>, vector<8x16xbf16>, vector<8x1xf8E8M0FNU>, vector<1x16xf8E8M0FNU>) -> vector<8x16xbf16>

utils/bazel/llvm-project-overlay/libc/BUILD.bazel

@@ -997,7 +997,6 @@ libc_support_library(
         ":__support_macros_config",
         ":__support_macros_properties_architectures",
         ":__support_macros_properties_compiler",
-        ":__support_macros_properties_os",
         ":hdr_stdint_proxy",
     ],
 )