Remove old sync-upstream.yml, superseded by Cold-Air-Networks/mirror-sync

This path transforms `vector.contract(A, B, Acc)` into
`vector.contract(A, B, 0) + Acc` to decouple the contraction computation
from the accumulator update for contractions over the loop.

lld/ELF/Arch/SPARCV9.cpp

@@ -166,6 +166,20 @@ void SPARCV9::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels,
       expr = R_GOT_OFF;
       break;
 
+    // Optimize the GOT load to an add of the symbol's GOT-relative address if
+    // applicable. Exclude absolute symbol, which can be arbitrarily far. %l7
+    // points at .got, which must be retained.
+    case R_SPARC_GOTDATA_OP_HIX22:
+    case R_SPARC_GOTDATA_OP_LOX10:
+    case R_SPARC_GOTDATA_OP:
+      if (sym.isPreemptible || isAbsolute(sym)) {
+        expr = R_GOT_OFF;
+      } else {
+        ctx.in.got->hasGotOffRel.store(true, std::memory_order_relaxed);
+        expr = R_GOTREL;
+      }
+      break;
+
     // TLS LE relocations:
     case R_SPARC_TLS_LE_HIX22:
     case R_SPARC_TLS_LE_LOX10:
@@ -322,6 +336,24 @@ void SPARCV9::relocate(uint8_t *loc, const Relocation &rel,
     // T-simm13
     write32be(loc, (read32be(loc) & ~0x00001fff) | (val & 0x000003ff) | 0x1C00);
     break;
+  case R_SPARC_GOTDATA_OP_HIX22: {
+    // V-imm22. sethi encodes the complement of a negative value, which the
+    // paired xor undoes, so the encodable range is 33 bits.
+    checkInt(ctx, loc, val, 33, rel);
+    uint64_t v = int64_t(val) < 0 ? ~val : val;
+    write32be(loc, (read32be(loc) & ~0x003fffff) | ((v >> 10) & 0x003fffff));
+    break;
+  }
+  case R_SPARC_GOTDATA_OP_LOX10:
+    // T-simm13. Only a negative value needs the sign extension bits.
+    write32be(loc, (read32be(loc) & ~0x00001fff) | (val & 0x000003ff) |
+                       (int64_t(val) < 0 ? 0x1c00 : 0));
+    break;
+  case R_SPARC_GOTDATA_OP:
+    // ldx [%rs1 + %rs2], %rd -> add %rs1, %rs2, %rd
+    if (rel.expr == R_GOTREL)
+      write32be(loc, (read32be(loc) & 0x3e07c01f) | 0x80000000);
+    break;
   default:
     llvm_unreachable("unknown relocation");
   }

lld/MachO/InputSection.cpp

@@ -204,16 +204,20 @@ void ConcatInputSection::foldIdentical(ConcatInputSection *copy,
   for (auto &copySym : copy->symbols)
     copySym->identicalCodeFoldingKind = foldKind;
 
-  symbols.insert(symbols.end(), copy->symbols.begin(), copy->symbols.end());
-  copy->symbols.clear();
-
-  // Remove duplicate compact unwind info for symbols at the same address.
-  if (symbols.empty())
+  if (copy->symbols.empty())
     return;
-  for (auto it = symbols.begin() + 1; it != symbols.end(); ++it) {
+  auto *it = copy->symbols.begin();
+  // The first symbol in the merged section (symbols.front()) must keep its
+  // unwind entry. If this section is empty, the copy's first symbol becomes the
+  // new front, so we skip clearing it.
+  if (symbols.empty())
+    ++it;
+  for (; it != copy->symbols.end(); ++it) {
     assert((*it)->value == 0);
     (*it)->originalUnwindEntry = nullptr;
   }
+  symbols.insert(symbols.end(), copy->symbols.begin(), copy->symbols.end());
+  copy->symbols.clear();
 }
 
 void ConcatInputSection::writeTo(uint8_t *buf) {

lld/test/ELF/sparcv9-gotdata.s

@@ -0,0 +1,141 @@
+# REQUIRES: sparc
+## Test R_SPARC_GOTDATA_OP_HIX22, R_SPARC_GOTDATA_OP_LOX10 and R_SPARC_GOTDATA_OP.
+## The GOT load is optimized to an add of the symbol's GOT-relative address
+## unless the symbol is preemptible or absolute.
+
+# RUN: rm -rf %t && split-file %s %t && cd %t
+# RUN: llvm-mc -filetype=obj -triple=sparcv9 a.s -o a.o
+# RUN: llvm-mc -filetype=obj -triple=sparcv9 b.s -o b.o
+# RUN: llvm-mc -filetype=obj -triple=sparcv9 c.s -o c.o
+# RUN: llvm-mc -filetype=obj -triple=sparcv9 ifunc.s -o ifunc.o
+
+# RUN: ld.lld -shared a.o c.o -o a.so
+# RUN: llvm-readelf -S -r a.so | FileCheck %s --check-prefix=RELOC
+# RUN: llvm-objdump -d -j .text --no-show-raw-insn --no-print-imm-hex a.so | FileCheck %s
+
+# RELOC:      [ 8] .got PROGBITS 0000000000200390 {{[0-9a-f]+}} 000020
+# RELOC:      Relocation section '.rela.dyn' {{.*}} contains 1 entries:
+# RELOC:      0000000000200398 {{.*}} R_SPARC_GLOB_DAT {{.*}} foo + 0
+
+## foo is preemptible, so the GOT load stays.
+## .got[1] - _GLOBAL_OFFSET_TABLE_ = 0x200398 - 0x200390 = 8.
+# CHECK-LABEL: <_start>:
+# CHECK-NEXT:   sethi 0, %l1
+# CHECK-NEXT:   xor %l1, 8, %l1
+# CHECK-NEXT:   ldx [%l7+%l1], %l2
+## hid0 - _GLOBAL_OFFSET_TABLE_ = 0x3003b0 - 0x200390 = 0x100020.
+# CHECK-NEXT:   sethi 1024, %g1
+# CHECK-NEXT:   xor %g1, 32, %g1
+# CHECK-NEXT:   add %l7, %g1, %g1
+## hid1 - _GLOBAL_OFFSET_TABLE_ = 0x1002ec - 0x200390 = -0x1000a4. sethi holds
+## the complement, which the sign-extended xor operand undoes.
+# CHECK-NEXT:   sethi 1024, %o2
+# CHECK-NEXT:   xor %o2, -164, %o2
+# CHECK-NEXT:   add %i0, %o2, %i5
+## An undefined symbol and an absolute symbol both have an absolute address, so
+## neither is optimized.
+# CHECK-NEXT:   sethi 0, %l1
+# CHECK-NEXT:   xor %l1, 16, %l1
+# CHECK-NEXT:   ldx [%l7+%l1], %l2
+# CHECK-NEXT:   sethi 0, %g4
+# CHECK-NEXT:   xor %g4, 24, %g4
+# CHECK-NEXT:   ldx [%l7+%g4], %o3
+
+## foo is not preemptible in an executable and is optimized as well. undefweak
+## and absolute keep the GOT load in every mode.
+# RUN: ld.lld a.o c.o -o a
+# RUN: llvm-objdump -d -j .text --no-show-raw-insn --no-print-imm-hex a | FileCheck %s --check-prefix=NOPIE
+# NOPIE-LABEL: <_start>:
+# NOPIE-NEXT:   sethi 1024, %l1
+# NOPIE-NEXT:   xor %l1, -4, %l1
+# NOPIE-NEXT:   add %l7, %l1, %l2
+# NOPIE:        sethi 0, %l1
+# NOPIE-NEXT:   xor %l1, 8, %l1
+# NOPIE-NEXT:   ldx [%l7+%l1], %l2
+# NOPIE-NEXT:   sethi 0, %g4
+# NOPIE-NEXT:   xor %g4, 16, %g4
+# NOPIE-NEXT:   ldx [%l7+%g4], %o3
+
+## .got holds nothing but the header, yet must be kept: %l7 points at it.
+# RUN: ld.lld -shared b.o -o b.so
+# RUN: llvm-readelf -S b.so | FileCheck %s --check-prefix=HEADER
+# HEADER: [ 7] .got PROGBITS 00000000002002b8 {{[0-9a-f]+}} 000008
+
+# RUN: not ld.lld -shared ifunc.o 2>&1 | FileCheck %s --check-prefix=IFUNC --implicit-check-not=error:
+# IFUNC: error: {{.*}}relocation R_SPARC_WDISP19 out of range
+
+## The optimization is decided before addresses are known, so an out-of-range
+## offset is an error rather than a fallback to the GOT load. The offset is
+## reported as signed, not as the complement the sethi encodes. Only hid0, in
+## .data, is out of range; the other symbols stay next to .got.
+# RUN: not ld.lld -shared a.o c.o --section-start=.text=0x10000 --section-start=.got=0x20000 \
+# RUN:   --section-start=.data=0x200000000 2>&1 | FileCheck %s --check-prefix=ERR -DVAL=8589803520 --implicit-check-not=error:
+# RUN: not ld.lld -shared a.o c.o --section-start=.data=0x10000 --section-start=.text=0x200000000 \
+# RUN:   --section-start=.got=0x200020000 2>&1 | FileCheck %s --check-prefix=ERR -DVAL=-8590000128 --implicit-check-not=error:
+# ERR: error: a.o:(.text+0xc): relocation R_SPARC_GOTDATA_OP_HIX22 out of range: [[VAL]] is not in [-4294967296, 4294967295]; references 'hid0'
+
+#--- a.s
+.text
+.globl _start
+_start:
+  sethi %gdop_hix22(foo), %l1
+  xor   %l1, %gdop_lox10(foo), %l1
+  ldx   [%l7 + %l1], %l2, %gdop(foo)
+
+  sethi %gdop_hix22(hid0), %g1
+  xor   %g1, %gdop_lox10(hid0), %g1
+  ldx   [%l7 + %g1], %g1, %gdop(hid0)
+
+## The add keeps rs1, rs2 and rd of the load, and ld is optimized like ldx.
+  sethi %gdop_hix22(hid1), %o2
+  xor   %o2, %gdop_lox10(hid1), %o2
+  ld    [%i0 + %o2], %i5, %gdop(hid1)
+
+  sethi %gdop_hix22(undefweak), %l1
+  xor   %l1, %gdop_lox10(undefweak), %l1
+  ldx   [%l7 + %l1], %l2, %gdop(undefweak)
+
+  sethi %gdop_hix22(absolute), %g4
+  xor   %g4, %gdop_lox10(absolute), %g4
+  ldx   [%l7 + %g4], %o3, %gdop(absolute)
+
+.globl foo, hid0, hid1
+.hidden hid0, hid1
+foo:
+hid1:
+  nop
+
+.weak undefweak
+.hidden undefweak
+
+.data
+hid0:
+  .xword 0
+
+#--- b.s
+.globl _start
+_start:
+  sethi %gdop_hix22(local), %l1
+  xor   %l1, %gdop_lox10(local), %l1
+  ldx   [%l7 + %l1], %l2, %gdop(local)
+local:
+  nop
+
+#--- c.s
+## Separate file to test %gdop_lox10(absolute) relocations in a.s
+.globl absolute
+.hidden absolute
+absolute = 0x1234
+
+#--- ifunc.s
+.globl _start
+_start:
+  sethi %gdop_hix22(ifunc), %l1
+  xor   %l1, %gdop_lox10(ifunc), %l1
+  ldx   [%l7 + %l1], %l2, %gdop(ifunc)
+
+.globl ifunc
+.hidden ifunc
+.type ifunc, @gnu_indirect_function
+ifunc:
+  nop

lld/test/MachO/icf-scale-same-class.s

@@ -0,0 +1,37 @@
+# REQUIRES: x86
+# RUN: rm -rf %t*
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %s -o %t.o
+# RUN: %lld -lSystem --icf=all -o %t %t.o
+# RUN: llvm-objdump --macho --section-headers %t | FileCheck %s --check-prefix=SECT
+# RUN: llvm-objdump --macho --syms %t | FileCheck %s --check-prefix=SYMS
+
+## Every body folds into one, so __text holds a single 6-byte body plus _main
+## rather than the 3 MiB (0x30000d) the unfolded copies occupy.
+# SECT:      Idx Name          Size
+# SECT-NEXT: 0 __text        00000009
+
+## The sentinels bracket the group, so checking that the first and last members
+## of the class share an address covers the whole run of folds.
+# SYMS-DAG: [[#%.16x,F:]] g     F __TEXT,__text _f_first
+# SYMS-DAG: [[#F]]        g     F __TEXT,__text _f_last
+
+.subsections_via_symbols
+.text
+.p2align 2
+
+.globl _f_first
+_f_first:; movl $7, %eax; ret
+
+## 64 Ki identical function landing in one equivalence class.
+.rept 65536
+  .globl _f\+
+  _f\+:; movl $$7, %eax; ret
+.endr
+
+.globl _f_last
+_f_last:; movl $7, %eax; ret
+
+.globl _main
+_main:
+  ret

lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.cpp

@@ -840,22 +840,116 @@ def main(f):
     return ArgInfo(count, varargs)
 )";
 
-Expected<PythonCallable::ArgInfo> PythonCallable::GetArgInfo() const {
-  ArgInfo result = {};
-  if (!IsValid())
-    return nullDeref();
-
+// inspect.signature() is deeply recursive and expensive in C-stack terms;
+// reentrant scripted callbacks dispatched through GetArgInfo() can turn
+// that into a fatal stack overflow instead of a catchable Python
+// RecursionError. GetArgInfo() never calls this itself; callers fall back
+// to it explicitly when they need to handle callables its cheaper,
+// attribute-only approach can't (e.g. builtins).
+Expected<PythonCallable::ArgInfo>
+PythonCallable::GetArgInfoFromInspectSignature(const PythonCallable &callable) {
+  PythonCallable::ArgInfo result = {};
   // no need to synchronize access to this global, we already have the GIL
   static PythonScript get_arg_info(get_arg_info_script);
-  Expected<PythonObject> pyarginfo = get_arg_info(*this);
+  Expected<PythonObject> pyarginfo = get_arg_info(callable);
   if (!pyarginfo)
     return pyarginfo.takeError();
   long long count =
       cantFail(As<long long>(pyarginfo.get().GetAttribute("count")));
   bool has_varargs =
       cantFail(As<bool>(pyarginfo.get().GetAttribute("has_varargs")));
-  result.max_positional_args = has_varargs ? ArgInfo::UNBOUNDED : count;
+  result.max_positional_args =
+      has_varargs ? PythonCallable::ArgInfo::UNBOUNDED : count;
+  return result;
+}
 
+// GetArgInfo()'s branches, top to bottom (`func` is what each branch ends up
+// introspecting; the final step is always func.__code__.co_argcount/co_flags):
+//
+//   callable
+//   |-- has __self__          -> func = __func__           (bound method;
+//   |                             fails for slot wrappers, e.g. (1).__add__)
+//   |-- has __code__ already  -> func = callable            (plain function)
+//   `-- neither
+//       |-- is a class
+//       |   |-- __init__ has __code__ -> func = __init__
+//       |   `-- else: check __new__ too (object.__init__ is lenient about
+//       |       extra args once __new__ is overridden)
+//       |       |-- __new__ has __code__ -> func = __new__
+//       |       `-- else                  -> ArgInfo{0} (object's defaults)
+//       `-- is an instance
+//           `-- func = __call__ (unwrap __func__ if bound)
+//               `-- no __code__ -> error (e.g. a builtin)
+Expected<PythonCallable::ArgInfo> PythonCallable::GetArgInfo() const {
+  if (!IsValid())
+    return nullDeref();
+
+  PythonObject func = *this;
+  bool implicit_first_arg = false;
+  if (HasAttribute("__self__")) {
+    implicit_first_arg = true;
+    Expected<PythonObject> func_or_err = GetAttribute("__func__");
+    if (!func_or_err)
+      return func_or_err.takeError();
+    func = *func_or_err;
+  } else if (!HasAttribute("__code__")) {
+    implicit_first_arg = true;
+    if (PyType_Check(m_py_obj)) {
+      Expected<PythonObject> init_or_err = GetAttribute("__init__");
+      if (!init_or_err)
+        return init_or_err.takeError();
+      func = *init_or_err;
+      if (!func.HasAttribute("__code__")) {
+        // __init__ is still object.__init__. A class may customize
+        // __new__ instead and leave __init__ untouched, which makes
+        // object.__init__ lenient about extra arguments -- so check
+        // __new__ too before concluding there are none.
+        Expected<PythonObject> new_or_err = GetAttribute("__new__");
+        if (!new_or_err)
+          return new_or_err.takeError();
+        func = *new_or_err;
+        if (!func.HasAttribute("__code__"))
+          return ArgInfo{0};
+      }
+    } else {
+      Expected<PythonObject> call_or_err = GetAttribute("__call__");
+      if (!call_or_err)
+        return call_or_err.takeError();
+      func = *call_or_err;
+      if (func.HasAttribute("__self__")) {
+        Expected<PythonObject> inner_or_err = func.GetAttribute("__func__");
+        if (!inner_or_err)
+          return inner_or_err.takeError();
+        func = *inner_or_err;
+      }
+      if (!func.HasAttribute("__code__"))
+        return llvm::createStringError("__call__ has no __code__");
+    }
+  }
+
+  Expected<PythonObject> code_or_err = func.GetAttribute("__code__");
+  if (!code_or_err)
+    return code_or_err.takeError();
+  PythonObject code = *code_or_err;
+
+  Expected<long long> argcount =
+      As<long long>(code.GetAttribute("co_argcount"));
+  if (!argcount)
+    return argcount.takeError();
+  Expected<long long> flags = As<long long>(code.GetAttribute("co_flags"));
+  if (!flags)
+    return flags.takeError();
+
+  ArgInfo result = {};
+  // Mirrors CPython's CO_VARARGS from <code.h>, which isn't reliably
+  // visible across the Python versions/platforms this file builds against.
+  constexpr long long kCoFlagVarArgs = 0x04;
+  if (*flags & kCoFlagVarArgs) {
+    result.max_positional_args = ArgInfo::UNBOUNDED;
+  } else {
+    long long count = *argcount - (implicit_first_arg ? 1 : 0);
+    result.max_positional_args = count > 0 ? static_cast<unsigned>(count) : 0;
+  }
   return result;
 }
 

lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.h

@@ -615,6 +615,15 @@ public:
 
   llvm::Expected<ArgInfo> GetArgInfo() const;
 
+  // Always derives ArgInfo via a Python-level inspect.signature() call,
+  // regardless of whether the callable's argument count/varargs bit could
+  // have been read directly off its data attributes. GetArgInfo() prefers
+  // the cheaper attribute-based path and only falls back to this for
+  // callables it can't introspect that way (e.g. builtins); exposed
+  // separately so that fallback behavior can be tested directly.
+  static llvm::Expected<ArgInfo>
+  GetArgInfoFromInspectSignature(const PythonCallable &callable);
+
   PythonObject operator()();
 
   PythonObject operator()(std::initializer_list<PyObject *> args);

lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp

@@ -1300,6 +1300,14 @@ ScriptInterpreterPythonImpl::GetMaxPositionalArgumentsForCallable(
                                    callable_name.str().c_str());
   }
   llvm::Expected<PythonCallable::ArgInfo> arg_info = pfunc.GetArgInfo();
+  if (!arg_info) {
+    // `-f` may point at a builtin, unlike other GetArgInfo() callers.
+    LLDB_LOG_ERROR(GetLog(LLDBLog::Script), arg_info.takeError(),
+                   "GetArgInfo failed for callable {1}, falling back to "
+                   "inspect.signature: {0}",
+                   callable_name);
+    arg_info = PythonCallable::GetArgInfoFromInspectSignature(pfunc);
+  }
   if (!arg_info)
     return arg_info.takeError();
   return arg_info.get().max_positional_args;

lldb/test/API/commands/register/register_command/TestRegisters.py

@@ -717,7 +717,6 @@ class RegisterCommandsTestCase(TestBase):
         self.expect("register write pc 0", substrs=[err_msg], error=True)
         self.expect("register info pc", substrs=[err_msg], error=True)
 
-    @expectedFailureDarwin(bugnumber="github.com/llvm/llvm-project/issues/213386")
     def test_case_insensitivity(self):
         """
         Register names, their aliases and any generic names like "sp" and "ra"

lldb/unittests/ScriptInterpreter/Python/PythonDataObjectsTests.cpp

@@ -735,6 +735,17 @@ class NewStyle(object):
   def __init__(self, one, two, three):
     pass
 
+class NoConstructorAtAll:
+  pass
+
+class NewOnlyVarArgs:
+  def __new__(cls, *args, **kwargs):
+    return super().__new__(cls)
+
+class NewOnlyFixedArgs:
+  def __new__(cls, a, b):
+    return super().__new__(cls)
+
 )";
     PyObject *o =
         RunString(script, Py_file_input, globals.get(), globals.get());
@@ -782,13 +793,37 @@ class NewStyle(object):
     arginfo = newstyle.get().GetArgInfo();
     ASSERT_THAT_EXPECTED(arginfo, llvm::Succeeded());
     EXPECT_EQ(arginfo.get().max_positional_args, 3u);
+
+    // Neither __init__ nor __new__ overridden: object.__init__ truly takes
+    // no extra arguments here.
+    auto no_ctor = As<PythonCallable>(globals.GetItem("NoConstructorAtAll"));
+    ASSERT_THAT_EXPECTED(no_ctor, llvm::Succeeded());
+    arginfo = no_ctor.get().GetArgInfo();
+    ASSERT_THAT_EXPECTED(arginfo, llvm::Succeeded());
+    EXPECT_EQ(arginfo.get().max_positional_args, 0u);
+
+    // __init__ not overridden, but __new__ is: object.__init__ becomes
+    // lenient about extra arguments once __new__ is overridden, so the
+    // argument count has to come from __new__, not from __init__ alone.
+    auto new_varargs = As<PythonCallable>(globals.GetItem("NewOnlyVarArgs"));
+    ASSERT_THAT_EXPECTED(new_varargs, llvm::Succeeded());
+    arginfo = new_varargs.get().GetArgInfo();
+    ASSERT_THAT_EXPECTED(arginfo, llvm::Succeeded());
+    EXPECT_EQ(arginfo.get().max_positional_args,
+              PythonCallable::ArgInfo::UNBOUNDED);
+
+    auto new_fixed = As<PythonCallable>(globals.GetItem("NewOnlyFixedArgs"));
+    ASSERT_THAT_EXPECTED(new_fixed, llvm::Succeeded());
+    arginfo = new_fixed.get().GetArgInfo();
+    ASSERT_THAT_EXPECTED(arginfo, llvm::Succeeded());
+    EXPECT_EQ(arginfo.get().max_positional_args, 2u);
   }
 
   {
     auto builtins = PythonModule::BuiltinsModule();
     auto hex = As<PythonCallable>(builtins.GetAttribute("hex"));
     ASSERT_THAT_EXPECTED(hex, llvm::Succeeded());
-    auto arginfo = hex.get().GetArgInfo();
+    auto arginfo = PythonCallable::GetArgInfoFromInspectSignature(hex.get());
     ASSERT_THAT_EXPECTED(arginfo, llvm::Succeeded());
     EXPECT_EQ(arginfo.get().max_positional_args, 1u);
   }

llvm/include/llvm/CodeGen/MachinePipeliner.h

@@ -287,12 +287,6 @@ class LLVM_ABI SwingSchedulerDAG : public ScheduleDAGInstrs {
   unsigned MII = 0;
   /// The maximum initiation interval between iterations for this schedule.
   unsigned MAX_II = 0;
-  /// The resource-constrained minimum initiation interval, i.e. the lower
-  /// bound on MII imposed by the availability of processor resources.
-  unsigned ResMII = 0;
-  /// The recurrence-constrained minimum initiation interval, i.e. the lower
-  /// bound on MII imposed by loop-carried dependence cycles.
-  unsigned RecMII = 0;
   /// Set to true if a valid pipelined schedule is found for the loop.
   bool Scheduled = false;
   MachineLoop &Loop;

llvm/include/llvm/CodeGen/TargetInstrInfo.h

@@ -50,7 +50,6 @@ class MachineLoop;
 class MachineLoopInfo;
 class MachineMemOperand;
 class MachineModuleInfo;
-class MachineOptimizationRemarkEmitter;
 class MachineRegisterInfo;
 class MCAsmInfo;
 class MCInst;
@@ -898,11 +897,8 @@ public:
 
   /// Analyze loop L, which must be a single-basic-block loop, and if the
   /// conditions can be understood enough produce a PipelinerLoopInfo object.
-  /// \p ORE, if non-null, may be used by targets to emit optimization remarks
-  /// explaining why the loop was rejected for pipelining.
-  virtual std::unique_ptr<PipelinerLoopInfo> analyzeLoopForPipelining(
-      MachineBasicBlock *LoopBB,
-      MachineOptimizationRemarkEmitter *ORE = nullptr) const {
+  virtual std::unique_ptr<PipelinerLoopInfo>
+  analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
     return nullptr;
   }
 

llvm/include/llvm/CodeGen/WindowScheduler.h

@@ -109,10 +109,6 @@ public:
 
   bool run();
 
-  /// Returns the Initiation Interval of the best scheduling result found, or
-  /// UINT_MAX if run() has not been called or found no valid schedule.
-  unsigned getBestII() const { return BestII; }
-
 protected:
   /// Two types of ScheduleDAGs are needed, one for creating dependency graphs
   /// only, and the other for list scheduling as determined by the target.

llvm/lib/CodeGen/MachinePipeliner.cpp

@@ -585,7 +585,7 @@ bool MachinePipeliner::canPipelineLoop(MachineLoop &L) {
 
   LI.LoopInductionVar = nullptr;
   LI.LoopCompare = nullptr;
-  LI.LoopPipelinerInfo = TII->analyzeLoopForPipelining(L.getTopBlock(), ORE);
+  LI.LoopPipelinerInfo = TII->analyzeLoopForPipelining(L.getTopBlock());
   if (!LI.LoopPipelinerInfo) {
     LLVM_DEBUG(dbgs() << "Unable to analyzeLoop, can NOT pipeline Loop\n");
     NumFailLoop++;
@@ -716,23 +716,7 @@ bool MachinePipeliner::runWindowScheduler(MachineLoop &L) {
   Context.RegClassInfo =
       &getAnalysis<MachineRegisterClassInfoWrapperPass>().getRCI();
   WindowScheduler WS(&Context, L);
-  bool Scheduled = WS.run();
-  if (Scheduled) {
-    unsigned II = WS.getBestII();
-    ORE->emit([&]() {
-      return MachineOptimizationRemark(DEBUG_TYPE, "window-schedule",
-                                       L.getStartLoc(), L.getHeader())
-             << "Window scheduled with Initiation Interval: "
-             << ore::NV("II", II);
-    });
-  } else {
-    ORE->emit([&]() {
-      return MachineOptimizationRemarkMissed(DEBUG_TYPE, "window-schedule",
-                                             L.getStartLoc(), L.getHeader())
-             << "Failed to find a valid window schedule";
-    });
-  }
-  return Scheduled;
+  return WS.run();
 }
 
 bool MachinePipeliner::useSwingModuloScheduler() {
@@ -2913,10 +2897,7 @@ bool SwingSchedulerDAG::schedulePipeline(SMSchedule &Schedule) {
              << "Schedule found with Initiation Interval: "
              << ore::NV("II", Schedule.getInitiationInterval())
              << ", MaxStageCount: "
-             << ore::NV("MaxStageCount", Schedule.getMaxStageCount())
-             << ", ResMII: " << ore::NV("ResMII", ResMII)
-             << ", RecMII: " << ore::NV("RecMII", RecMII) << ", Bound: "
-             << ore::NV("Bound", ResMII >= RecMII ? "Resource" : "Recurrence");
+             << ore::NV("MaxStageCount", Schedule.getMaxStageCount());
     });
   } else
     Schedule.reset();

llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp

@@ -54,11 +54,7 @@ void JITLinkerBase::linkPhase1(std::unique_ptr<JITLinkerBase> Self) {
   Ctx->getMemoryManager().allocate(
       Ctx->getJITLinkDylib(), *G,
       [S = std::move(Self)](AllocResult AR) mutable {
-        // FIXME: Once MSVC implements c++17 order of evaluation rules for calls
-        // this can be simplified to
-        //          S->linkPhase2(std::move(S), std::move(AR));
-        auto *TmpSelf = S.get();
-        TmpSelf->linkPhase2(std::move(S), std::move(AR));
+        S->linkPhase2(std::move(S), std::move(AR));
       });
 }
 
@@ -95,10 +91,7 @@ void JITLinkerBase::linkPhase2(std::unique_ptr<JITLinkerBase> Self,
       dbgs() << "No external symbols for " << G->getName()
              << ". Proceeding immediately with link phase 3.\n";
     });
-    // FIXME: Once MSVC implements c++17 order of evaluation rules for calls
-    // this can be simplified. See below.
-    auto &TmpSelf = *Self;
-    TmpSelf.linkPhase3(std::move(Self), AsyncLookupResult());
+    Self->linkPhase3(std::move(Self), AsyncLookupResult());
     return;
   }
 
@@ -108,22 +101,11 @@ void JITLinkerBase::linkPhase2(std::unique_ptr<JITLinkerBase> Self,
            << " (may trigger materialization/linking of other graphs)...\n";
   });
 
-  // We're about to hand off ownership of ourself to the continuation. Grab a
-  // pointer to the context so that we can call it to initiate the lookup.
-  //
-  // FIXME: Once MSVC implements c++17 order of evaluation rules for calls this
-  // can be simplified to:
-  //
-  // Ctx->lookup(std::move(UnresolvedExternals),
-  //             [Self=std::move(Self)](Expected<AsyncLookupResult> Result) {
-  //               Self->linkPhase3(std::move(Self), std::move(Result));
-  //             });
   Ctx->lookup(std::move(ExternalSymbols),
               createLookupContinuation(
                   [S = std::move(Self)](
                       Expected<AsyncLookupResult> LookupResult) mutable {
-                    auto &TmpSelf = *S;
-                    TmpSelf.linkPhase3(std::move(S), std::move(LookupResult));
+                    S->linkPhase3(std::move(S), std::move(LookupResult));
                   }));
 }
 
@@ -171,11 +153,7 @@ void JITLinkerBase::linkPhase3(std::unique_ptr<JITLinkerBase> Self,
   }
 
   Alloc->finalize([S = std::move(Self)](FinalizeResult FR) mutable {
-    // FIXME: Once MSVC implements c++17 order of evaluation rules for calls
-    // this can be simplified to
-    //          S->linkPhase2(std::move(S), std::move(AR));
-    auto *TmpSelf = S.get();
-    TmpSelf->linkPhase4(std::move(S), std::move(FR));
+    S->linkPhase4(std::move(S), std::move(FR));
   });
 }
 

llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h

@@ -116,12 +116,10 @@ public:
     // Ownership of the linker is passed into the linker's doLink function to
     // allow it to be passed on to async continuations.
     //
-    // FIXME: Remove LTmp once we have c++17.
     // C++17 sequencing rules guarantee that function name expressions are
-    // sequenced before arguments, so L->linkPhase1(std::move(L), ...) will be
-    // well formed.
-    auto &LTmp = *L;
-    LTmp.linkPhase1(std::move(L));
+    // sequenced before arguments, so L->linkPhase1(std::move(L)) is well
+    // formed.
+    L->linkPhase1(std::move(L));
   }
 
 private:

llvm/lib/Target/AArch64/AArch64InstrInfo.cpp

@@ -25,7 +25,6 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/AliasAnalysis.h"
-#include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/CodeGen/CFIInstBuilder.h"
 #include "llvm/CodeGen/LivePhysRegs.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
@@ -37,7 +36,6 @@
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineOperand.h"
-#include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/RegisterScavenging.h"
 #include "llvm/CodeGen/StackMaps.h"
@@ -12065,8 +12063,7 @@ static bool getIndVarInfo(Register Reg, const MachineBasicBlock *LoopBB,
 }
 
 std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
-AArch64InstrInfo::analyzeLoopForPipelining(
-    MachineBasicBlock *LoopBB, MachineOptimizationRemarkEmitter *ORE) const {
+AArch64InstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
   // Accept loops that meet the following conditions
   // * The conditional branch is BCC
   // * The compare instruction is ADDS/SUBS/WHILEXX
@@ -12075,66 +12072,24 @@ AArch64InstrInfo::analyzeLoopForPipelining(
   // * The induction variable is incremented/decremented by a single instruction
   // * Does not contain CALL or instructions which have unmodeled side effects
 
-  for (MachineInstr &MI : *LoopBB) {
-    // This instruction may use NZCV, which interferes with the instruction to
-    // be inserted for loop control.
-    if (MI.isCall()) {
-      if (ORE)
-        ORE->emit([&]() {
-          return MachineOptimizationRemarkAnalysis("pipeliner", "analyzeLoop",
-                                                   &MI)
-                 << "loop contains a call and therefore cannot be pipelined";
-        });
+  for (MachineInstr &MI : *LoopBB)
+    if (MI.isCall() || MI.hasUnmodeledSideEffects())
+      // This instruction may use NZCV, which interferes with the instruction to
+      // be inserted for loop control.
       return nullptr;
-    }
-    if (MI.hasUnmodeledSideEffects()) {
-      if (ORE)
-        ORE->emit([&]() {
-          return MachineOptimizationRemarkAnalysis("pipeliner", "analyzeLoop",
-                                                   &MI)
-                 << "loop contains an instruction with unmodeled side effects, "
-                    "and therefore cannot be pipelined";
-        });
-      return nullptr;
-    }
-  }
 
   MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
   SmallVector<MachineOperand, 4> Cond;
-  if (analyzeBranch(*LoopBB, TBB, FBB, Cond)) {
-    if (ORE)
-      ORE->emit([&]() {
-        return MachineOptimizationRemarkAnalysis(
-                   "pipeliner", "analyzeLoop",
-                   LoopBB->findDebugLoc(LoopBB->getFirstTerminator()), LoopBB)
-               << "branch cannot be analyzed";
-      });
+  if (analyzeBranch(*LoopBB, TBB, FBB, Cond))
     return nullptr;
-  }
 
   // Infinite loops are not supported
-  if (TBB == LoopBB && FBB == LoopBB) {
-    if (ORE)
-      ORE->emit([&]() {
-        return MachineOptimizationRemarkAnalysis(
-                   "pipeliner", "analyzeLoop",
-                   LoopBB->findDebugLoc(LoopBB->getFirstTerminator()), LoopBB)
-               << "infinite loops are not supported";
-      });
+  if (TBB == LoopBB && FBB == LoopBB)
     return nullptr;
-  }
 
   // Must be conditional branch
-  if (TBB != LoopBB && FBB == nullptr) {
-    if (ORE)
-      ORE->emit([&]() {
-        return MachineOptimizationRemarkAnalysis(
-                   "pipeliner", "analyzeLoop",
-                   LoopBB->findDebugLoc(LoopBB->getFirstTerminator()), LoopBB)
-               << "loop is not terminated by a conditional branch";
-      });
+  if (TBB != LoopBB && FBB == nullptr)
     return nullptr;
-  }
 
   assert((TBB == LoopBB || FBB == LoopBB) &&
          "The Loop must be a single-basic-block loop");
@@ -12142,16 +12097,8 @@ AArch64InstrInfo::analyzeLoopForPipelining(
   MachineInstr *CondBranch = &*LoopBB->getFirstTerminator();
   const TargetRegisterInfo &TRI = getRegisterInfo();
 
-  if (CondBranch->getOpcode() != AArch64::Bcc) {
-    if (ORE)
-      ORE->emit([&]() {
-        return MachineOptimizationRemarkAnalysis("pipeliner", "analyzeLoop",
-                                                 CondBranch)
-               << "branch opcode not yet supported for pipelining: "
-               << ore::NV("Opcode", getName(CondBranch->getOpcode()));
-      });
+  if (CondBranch->getOpcode() != AArch64::Bcc)
     return nullptr;
-  }
 
   // Normalization for createTripCountGreaterCondition()
   if (TBB == LoopBB)
@@ -12183,13 +12130,6 @@ AArch64InstrInfo::analyzeLoopForPipelining(
           Comp = &MI;
           break;
         }
-        if (ORE)
-          ORE->emit([&]() {
-            return MachineOptimizationRemarkAnalysis("pipeliner", "analyzeLoop",
-                                                     &MI)
-                   << "compare instruction not recognized: "
-                   << ore::NV("Opcode", getName(MI.getOpcode()));
-          });
         return nullptr;
       }
 
@@ -12198,44 +12138,22 @@ AArch64InstrInfo::analyzeLoopForPipelining(
           CompCounterOprNum = 2;
         else if (isDefinedOutside(Comp->getOperand(2).getReg(), LoopBB))
           CompCounterOprNum = 1;
-        else {
-          if (ORE)
-            ORE->emit([&]() {
-              return MachineOptimizationRemarkAnalysis("pipeliner",
-                                                       "analyzeLoop", Comp)
-                     << "neither operand of the compare is loop invariant";
-            });
+        else
           return nullptr;
-        }
       }
       break;
     }
   }
-  if (!Comp) {
-    if (ORE)
-      ORE->emit([&]() {
-        return MachineOptimizationRemarkAnalysis(
-                   "pipeliner", "analyzeLoop",
-                   LoopBB->findDebugLoc(LoopBB->getFirstTerminator()), LoopBB)
-               << "no NZCV-modifying compare instruction found";
-      });
+  if (!Comp)
     return nullptr;
-  }
 
   MachineInstr *Update = nullptr;
   Register Init;
   bool IsUpdatePriorComp;
   unsigned UpdateCounterOprNum;
   if (!getIndVarInfo(Comp->getOperand(CompCounterOprNum).getReg(), LoopBB,
-                     Update, UpdateCounterOprNum, Init, IsUpdatePriorComp)) {
-    if (ORE)
-      ORE->emit([&]() {
-        return MachineOptimizationRemarkAnalysis("pipeliner", "analyzeLoop",
-                                                 Comp)
-               << "loop induction variable pattern not recognized";
-      });
+                     Update, UpdateCounterOprNum, Init, IsUpdatePriorComp))
     return nullptr;
-  }
 
   return std::make_unique<AArch64PipelinerLoopInfo>(
       LoopBB, CondBranch, Comp, CompCounterOprNum, Update, UpdateCounterOprNum,

llvm/lib/Target/AArch64/AArch64InstrInfo.h

@@ -414,9 +414,8 @@ public:
                         const DebugLoc &DL,
                         int *BytesAdded = nullptr) const override;
 
-  std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo> analyzeLoopForPipelining(
-      MachineBasicBlock *LoopBB,
-      MachineOptimizationRemarkEmitter *ORE = nullptr) const override;
+  std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
+  analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override;
 
   bool
   reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const override;

llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp

@@ -6737,8 +6737,7 @@ bool ARMPipelinerLoopInfo::tooMuchRegisterPressure(SwingSchedulerDAG &SSD,
 } // namespace
 
 std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
-ARMBaseInstrInfo::analyzeLoopForPipelining(
-    MachineBasicBlock *LoopBB, MachineOptimizationRemarkEmitter *ORE) const {
+ARMBaseInstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
   MachineBasicBlock::iterator I = LoopBB->getFirstTerminator();
   MachineBasicBlock *Preheader = *LoopBB->pred_begin();
   if (Preheader == LoopBB)

llvm/lib/Target/ARM/ARMBaseInstrInfo.h

@@ -382,9 +382,8 @@ public:
 
   /// Analyze loop L, which must be a single-basic-block loop, and if the
   /// conditions can be understood enough produce a PipelinerLoopInfo object.
-  std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo> analyzeLoopForPipelining(
-      MachineBasicBlock *LoopBB,
-      MachineOptimizationRemarkEmitter *ORE = nullptr) const override;
+  std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
+  analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override;
 
 private:
   /// Returns an unused general-purpose register which can be used for

llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp

@@ -806,8 +806,7 @@ public:
 } // namespace
 
 std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
-HexagonInstrInfo::analyzeLoopForPipelining(
-    MachineBasicBlock *LoopBB, MachineOptimizationRemarkEmitter *ORE) const {
+HexagonInstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
   // We really "analyze" only hardware loops right now.
   MachineBasicBlock::iterator I = LoopBB->getFirstTerminator();
 

llvm/lib/Target/Hexagon/HexagonInstrInfo.h

@@ -139,9 +139,8 @@ public:
 
   /// Analyze loop L, which must be a single-basic-block loop, and if the
   /// conditions can be understood enough produce a PipelinerLoopInfo object.
-  std::unique_ptr<PipelinerLoopInfo> analyzeLoopForPipelining(
-      MachineBasicBlock *LoopBB,
-      MachineOptimizationRemarkEmitter *ORE = nullptr) const override;
+  std::unique_ptr<PipelinerLoopInfo>
+  analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override;
 
   /// Return true if it's profitable to predicate
   /// instructions with accumulated instruction latency of "NumCycles"

llvm/lib/Target/PowerPC/PPCInstrInfo.cpp

@@ -5790,8 +5790,7 @@ public:
 } // namespace
 
 std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
-PPCInstrInfo::analyzeLoopForPipelining(
-    MachineBasicBlock *LoopBB, MachineOptimizationRemarkEmitter *ORE) const {
+PPCInstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
   // We really "analyze" only hardware loops right now.
   MachineBasicBlock::iterator I = LoopBB->getFirstTerminator();
   MachineBasicBlock *Preheader = *LoopBB->pred_begin();

llvm/lib/Target/PowerPC/PPCInstrInfo.h

@@ -789,9 +789,8 @@ public:
 
   /// Analyze loop L, which must be a single-basic-block loop, and if the
   /// conditions can be understood enough produce a PipelinerLoopInfo object.
-  std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo> analyzeLoopForPipelining(
-      MachineBasicBlock *LoopBB,
-      MachineOptimizationRemarkEmitter *ORE = nullptr) const override;
+  std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
+  analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override;
 };
 
 }

llvm/lib/Target/RISCV/RISCVInstrInfo.cpp

@@ -5394,8 +5394,7 @@ public:
 } // namespace
 
 std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
-RISCVInstrInfo::analyzeLoopForPipelining(
-    MachineBasicBlock *LoopBB, MachineOptimizationRemarkEmitter *ORE) const {
+RISCVInstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
   MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
   SmallVector<MachineOperand, 4> Cond;
   if (analyzeBranch(*LoopBB, TBB, FBB, Cond, /*AllowModify=*/false))

llvm/lib/Target/RISCV/RISCVInstrInfo.h

@@ -328,9 +328,8 @@ public:
 
   unsigned getTailDuplicateSize(CodeGenOptLevel OptLevel) const override;
 
-  std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo> analyzeLoopForPipelining(
-      MachineBasicBlock *LoopBB,
-      MachineOptimizationRemarkEmitter *ORE = nullptr) const override;
+  std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
+  analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override;
 
   bool isHighLatencyDef(int Opc) const override;
 

llvm/test/CodeGen/AArch64/sms-analyzeloop-remarks.mir

@@ -1,168 +0,0 @@
-# RUN: llc --verify-machineinstrs -mtriple=aarch64 -o /dev/null %s -run-pass pipeliner \
-# RUN:   -aarch64-enable-pipeliner -pass-remarks-analysis=pipeliner \
-# RUN:   -pass-remarks-missed=pipeliner 2>&1 | FileCheck %s
-
-# Regression test for the analyzeLoopForPipelining() optimization remarks:
-# each rejection point in AArch64InstrInfo::analyzeLoopForPipelining() should
-# report a distinct, specific reason via the "pipeliner"/"analyzeLoop" remark,
-# rather than only the generic "The loop structure is not supported" remark
-# from canPipelineLoop().
-
---- |
-  define dso_local void @call(ptr noalias nocapture noundef writeonly %a, i32 noundef %n) local_unnamed_addr #0 {
-  entry:
-    ret void
-  }
-  declare void @callee()
-
-  define dso_local void @infiniteloop(ptr noalias nocapture noundef writeonly %a, i32 noundef %n) local_unnamed_addr #0 {
-  entry:
-    ret void
-  }
-
-  define dso_local void @unsupportedbranch(ptr noalias nocapture noundef writeonly %a) local_unnamed_addr #0 {
-  entry:
-    ret void
-  }
-
-  define dso_local void @badcompare(ptr noalias nocapture noundef writeonly %a, i32 noundef %n) local_unnamed_addr #0 {
-  entry:
-    ret void
-  }
-
-  define dso_local void @noninvariantcompare(i64 noundef %x, i64 noundef %y) local_unnamed_addr #0 {
-  entry:
-    ret void
-  }
-
-  define dso_local void @badindvar(i64 noundef %x, i64 noundef %y) local_unnamed_addr #0 {
-  entry:
-    ret void
-  }
-...
----
-name:            call
-tracksRegLiveness: true
-body:             |
-  ; A call in the loop body may use NZCV, which interferes with the
-  ; instruction to be inserted for loop control.
-  ; CHECK: loop contains a call and therefore cannot be pipelined
-  ; CHECK: The loop structure is not supported
-  bb.0.entry:
-    liveins: $x0, $w1
-
-    %10:gpr64sp = COPY $x0
-    %11:gpr32common = COPY $w1
-
-  bb.1:
-    %12:gpr64sp = PHI %10, %bb.0, %13, %bb.1
-    %13:gpr64sp = ADDXri %12, 1, 0
-    BL @callee, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp
-    dead $wzr = SUBSWri %11, 1, 0, implicit-def $nzcv
-    Bcc 1, %bb.1, implicit $nzcv
-    B %bb.2
-
-  bb.2:
-...
----
-name:            infiniteloop
-tracksRegLiveness: true
-body:             |
-  ; Both branch targets are the loop itself: nothing exits the loop.
-  ; CHECK: infinite loops are not supported
-  ; CHECK: The loop structure is not supported
-  bb.0.entry:
-    liveins: $x0, $w1
-
-    %10:gpr64sp = COPY $x0
-    %11:gpr32common = COPY $w1
-
-  bb.1:
-    %12:gpr64sp = PHI %10, %bb.0, %13, %bb.1
-    %13:gpr64sp = ADDXri %12, 1, 0
-    dead $wzr = SUBSWri %11, 0, 0, implicit-def $nzcv
-    Bcc 0, %bb.1, implicit $nzcv
-    B %bb.1
-...
----
-name:            unsupportedbranch
-tracksRegLiveness: true
-body:             |
-  ; The loop terminator is an unconditional branch (Bcc is required).
-  ; CHECK: branch opcode not yet supported for pipelining: B
-  ; CHECK: The loop structure is not supported
-  bb.0.entry:
-    liveins: $x0
-
-    %10:gpr64sp = COPY $x0
-
-  bb.1:
-    %12:gpr64sp = PHI %10, %bb.0, %13, %bb.1
-    %13:gpr64sp = ADDXri %12, 1, 0
-    B %bb.1
-...
----
-name:            badcompare
-tracksRegLiveness: true
-body:             |
-  ; The NZCV-modifying instruction is not a recognized compare opcode.
-  ; CHECK: compare instruction not recognized: ANDSWri
-  ; CHECK: The loop structure is not supported
-  bb.0.entry:
-    liveins: $x0, $w1
-
-    %10:gpr64sp = COPY $x0
-    %11:gpr32common = COPY $w1
-
-  bb.1:
-    %12:gpr64sp = PHI %10, %bb.0, %13, %bb.1
-    %13:gpr64sp = ADDXri %12, 1, 0
-    dead $wzr = ANDSWri %11, 1, implicit-def $nzcv
-    Bcc 1, %bb.1, implicit $nzcv
-    B %bb.2
-
-  bb.2:
-...
----
-name:            noninvariantcompare
-tracksRegLiveness: true
-body:             |
-  ; Neither operand of the compare instruction is a loop invariant value.
-  ; CHECK: neither operand of the compare is loop invariant
-  ; CHECK: The loop structure is not supported
-  bb.0.entry:
-    liveins: $x0, $x1
-    %10:gpr64 = COPY $x0
-    %11:gpr64 = COPY $x1
-
-  bb.1:
-    %12:gpr64 = PHI %11, %bb.0, %13, %bb.1
-    %13:gpr64 = ADDXrr %12, %11
-    dead $xzr = SUBSXrr %13, %13, implicit-def $nzcv
-    Bcc 1, %bb.1, implicit $nzcv
-    B %bb.2
-
-  bb.2:
-...
----
-name:            badindvar
-tracksRegLiveness: true
-body:             |
-  ; The value compared against is not updated by a recognized increment
-  ; instruction.
-  ; CHECK: loop induction variable pattern not recognized
-  ; CHECK: The loop structure is not supported
-  bb.0.entry:
-    liveins: $x0, $x1
-    %10:gpr64 = COPY $x0
-    %11:gpr64 = COPY $x1
-
-  bb.1:
-    %12:gpr64 = PHI %11, %bb.0, %13, %bb.1
-    %13:gpr64 = ORRXrr %12, %12
-    dead $xzr = SUBSXrr %12, %10, implicit-def $nzcv
-    Bcc 1, %bb.1, implicit $nzcv
-    B %bb.2
-
-  bb.2:
-...

llvm/test/CodeGen/AArch64/sms-schedule-remark.mir

@@ -1,83 +0,0 @@
-# RUN: llc --verify-machineinstrs -mtriple=aarch64 -o /dev/null %s -run-pass pipeliner \
-# RUN:   -aarch64-enable-pipeliner -pass-remarks-analysis=pipeliner 2>&1 | FileCheck %s
-
-# Regression test for the "schedule" success remark: it should report ResMII,
-# RecMII, and which of the two is the binding constraint on the chosen II, in
-# addition to the Initiation Interval and MaxStageCount that were already
-# reported.
-
-# CHECK: Schedule found with Initiation Interval: 3, MaxStageCount: 2, ResMII: 0, RecMII: 0, Bound: Resource
-
---- |
-  define dso_local void @func(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b, i32 noundef %n) local_unnamed_addr #0 {
-  entry:
-    %cmp6 = icmp sgt i32 %n, 0
-    br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
-
-  for.body.preheader:                               ; preds = %entry
-    %wide.trip.count = zext nneg i32 %n to i64
-    br label %for.body
-
-  for.cond.cleanup:                                 ; preds = %for.body, %entry
-    ret void
-
-  for.body:                                         ; preds = %for.body.preheader, %for.body
-    %lsr.iv11 = phi i64 [ %wide.trip.count, %for.body.preheader ], [ %lsr.iv.next, %for.body ]
-    %lsr.iv9 = phi ptr [ %b, %for.body.preheader ], [ %scevgep10, %for.body ]
-    %lsr.iv = phi ptr [ %a, %for.body.preheader ], [ %scevgep, %for.body ]
-    %0 = load float, ptr %lsr.iv9, align 4
-    %add = fadd float %0, 1.000000e+00
-    store float %add, ptr %lsr.iv, align 4
-    %scevgep = getelementptr i8, ptr %lsr.iv, i64 4
-    %scevgep10 = getelementptr i8, ptr %lsr.iv9, i64 4
-    %lsr.iv.next = add nsw i64 %lsr.iv11, -1
-    %exitcond.not = icmp eq i64 %lsr.iv.next, 0
-    br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
-  }
-
-...
----
-name:            func
-tracksRegLiveness: true
-liveins:
-  - { reg: '$x0', virtual-reg: '%7' }
-  - { reg: '$x1', virtual-reg: '%8' }
-  - { reg: '$w2', virtual-reg: '%9' }
-body:             |
-  bb.0.entry:
-    successors: %bb.1(0x50000000), %bb.2(0x30000000)
-    liveins: $x0, $x1, $w2
-
-    %9:gpr32common = COPY $w2
-    %8:gpr64 = COPY $x1
-    %7:gpr64 = COPY $x0
-    dead $wzr = SUBSWri %9, 1, 0, implicit-def $nzcv
-    Bcc 11, %bb.2, implicit $nzcv
-    B %bb.1
-
-  bb.1.for.body.preheader:
-    %11:gpr32 = ORRWrs $wzr, %9, 0
-    %0:gpr64all = SUBREG_TO_REG killed %11, %subreg.sub_32
-    %14:fpr32 = FMOVSi 112
-    B %bb.3
-
-  bb.2.for.cond.cleanup:
-    RET_ReallyLR
-
-  bb.3.for.body:
-    successors: %bb.2(0x04000000), %bb.3(0x7c000000)
-
-    %1:gpr64sp = PHI %0, %bb.1, %6, %bb.3
-    %2:gpr64sp = PHI %8, %bb.1, %5, %bb.3
-    %3:gpr64sp = PHI %7, %bb.1, %4, %bb.3
-    early-clobber %12:gpr64sp, %13:fpr32 = LDRSpost %2, 4 :: (load (s32) from %ir.lsr.iv9)
-    %15:fpr32 = nofpexcept FADDSrr killed %13, %14, implicit $fpcr
-    early-clobber %16:gpr64sp = STRSpost killed %15, %3, 4 :: (store (s32) into %ir.lsr.iv)
-    %4:gpr64all = COPY %16
-    %5:gpr64all = COPY %12
-    %17:gpr64 = nsw SUBSXri %1, 1, 0, implicit-def $nzcv
-    %6:gpr64all = COPY %17
-    Bcc 0, %bb.2, implicit $nzcv
-    B %bb.3
-
-...

llvm/test/CodeGen/Hexagon/swp-ws-exp.mir

@@ -2,14 +2,9 @@
 # RUN: llc --mtriple=hexagon %s -run-pass=pipeliner -debug-only=pipeliner \
 # RUN: -window-sched=force -filetype=null -verify-machineinstrs 2>&1 \
 # RUN: | FileCheck %s
-# RUN: llc --mtriple=hexagon %s -run-pass=pipeliner \
-# RUN: -window-sched=force -filetype=null -verify-machineinstrs \
-# RUN: -pass-remarks=pipeliner 2>&1 | FileCheck %s --check-prefix=REMARK
 
 # CHECK: Best window offset is {{[0-9]+}} and Best II is {{[0-9]+}}.
 
-# REMARK: remark: {{.*}}Window scheduled with Initiation Interval: {{[0-9]+}}
-
 --- |
   define void @exp_approx_top_six(i32 %N, ptr noalias %x, ptr noalias %y) #0 {
   entry:

llvm/test/CodeGen/Hexagon/swp-ws-fail-0.mir

@@ -9,9 +9,6 @@
 # RUN: llc --mtriple=hexagon %s -run-pass=pipeliner -debug-only=pipeliner \
 # RUN: -window-sched=force -window-region-limit=1 -window-search-ratio=80 \
 # RUN: -filetype=null 2>&1 | FileCheck %s --check-prefix=CHECK-SCHED-NOT-NEEDED
-# RUN: llc --mtriple=hexagon %s -run-pass=pipeliner \
-# RUN: -window-sched=force -filetype=null \
-# RUN: -pass-remarks-missed=pipeliner 2>&1 | FileCheck %s --check-prefix=REMARK
 
 # CHECK-INITIALIZE: There are too few MIs in the window region!
 # CHECK-INITIALIZE: The WindowScheduler failed to initialize!
@@ -19,8 +16,6 @@
 # CHECK-ANALYSE-II: Window scheduling is not needed!
 # CHECK-SCHED-NOT-NEEDED: Window scheduling is not needed!
 
-# REMARK: remark: {{.*}}Failed to find a valid window schedule
-
 ---
 name:            relu
 tracksRegLiveness: true

llvm/test/CodeGen/PowerPC/sms-remark.ll

@@ -15,7 +15,7 @@
 @y = dso_local global [1024 x i32] zeroinitializer, align 4
 
 define dso_local ptr @foo() local_unnamed_addr {
-;ENABLED: Schedule found with Initiation Interval: {{[0-9]+}}, MaxStageCount: {{[0-9]+}}, ResMII: {{[0-9]+}}, RecMII: {{[0-9]+}}, Bound: {{(Recurrence|Resource)}}
+;ENABLED: Schedule found with Initiation Interval
 ;ENABLED: Pipelined succesfully!
 entry:
   %.pre = load i32, ptr @y, align 4

mlir/include/mlir-c/IR.h

@@ -639,9 +639,46 @@ static inline bool mlirOperationIsNull(MlirOperation op) { return !op.ptr; }
 MLIR_CAPI_EXPORTED bool mlirOperationEqual(MlirOperation op,
                                            MlirOperation other);
 
-/// Compute a hash for the given operation.
+/// Compute a hash for the given operation. Operand and result SSA values are
+/// hashed by identity and locations are significant, so equivalent-but-distinct
+/// operations hash differently; use mlirOperationStructuralHashValue for a hash
+/// that pairs with mlirOperationIsStructurallyEquivalent.
 MLIR_CAPI_EXPORTED size_t mlirOperationHashValue(MlirOperation op);
 
+/// Flags controlling structural operation equivalence and hashing. These mirror
+/// `mlir::OperationEquivalence::Flags` and may be combined with bitwise OR.
+typedef enum MlirOperationEquivalenceFlags {
+  /// No flags: locations, discardable attributes, properties and
+  /// commutativity are all significant.
+  MLIR_OPERATION_EQUIVALENCE_NONE = 0,
+  /// Ignore the locations attached to operations.
+  MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS = 1,
+  /// Ignore the discardable attributes attached to operations.
+  MLIR_OPERATION_EQUIVALENCE_IGNORE_DISCARDABLE_ATTRS = 2,
+  /// Ignore the properties attached to operations.
+  MLIR_OPERATION_EQUIVALENCE_IGNORE_PROPERTIES = 4,
+  /// Ignore commutativity, comparing operands in an order-sensitive way.
+  MLIR_OPERATION_EQUIVALENCE_IGNORE_COMMUTATIVITY = 8,
+} MlirOperationEquivalenceFlags;
+
+/// Checks whether two operations are structurally equivalent, i.e. they have
+/// the same name, attributes, operand and result types, and recursively
+/// equivalent regions. Operand equivalence is tracked structurally while
+/// recursing into regions, so operands defined inside the compared regions need
+/// not be the exact same SSA values; operands defined outside must be. `flags`
+/// is a bitwise OR of MlirOperationEquivalenceFlags values.
+MLIR_CAPI_EXPORTED bool mlirOperationIsStructurallyEquivalent(MlirOperation lhs,
+                                                              MlirOperation rhs,
+                                                              uint32_t flags);
+
+/// Computes a hash for the given operation that pairs with
+/// mlirOperationIsStructurallyEquivalent: two operations that are structurally
+/// equivalent under the same `flags` hash equally. Operands are hashed by
+/// identity, results are not hashed at all, and regions do not participate in
+/// the hash. `flags` is a bitwise OR of MlirOperationEquivalenceFlags values.
+MLIR_CAPI_EXPORTED size_t mlirOperationStructuralHashValue(MlirOperation op,
+                                                           uint32_t flags);
+
 /// Gets the context this operation is associated with
 MLIR_CAPI_EXPORTED MlirContext mlirOperationGetContext(MlirOperation op);
 

mlir/include/mlir/Dialect/X86/TransformOps/X86TransformOps.td

@@ -82,5 +82,15 @@ def ApplyVectorContractToAMXDotProductPatternsOp : Op<Transform_Dialect,
   let assemblyFormat = "attr-dict";
 }
 
+def ApplyMoveAccumulatorForContractLoopPatternsOp : Op<Transform_Dialect,
+    "apply_patterns.x86.move_accumulator_for_contract_loop",
+    [DeclareOpInterfaceMethods<PatternDescriptorOpInterface>]> {
+  let description = [{
+         Rewrites an additive vector contraction by zeroing the accumulator input
+         and materializing the accumulation as a separate add after the contraction.
+  }];
+
+  let assemblyFormat = "attr-dict";
+}
 
 #endif // X86_TRANSFORM_OPS

mlir/include/mlir/Dialect/X86/Transforms.h

@@ -110,6 +110,11 @@ void populateShuffleVectorFMAOpsPatterns(RewritePatternSet &patterns);
 // Int8).
 void populateVectorContractToAMXDotProductPatterns(RewritePatternSet &patterns);
 
+// Rewrites an additive vector contraction by zeroing the accumulator input
+// and materializing the accumulation as a separate add after the contraction.
+void populateMoveAccumulatorForContractLoopPatterns(
+    RewritePatternSet &patterns);
+
 //===----------------------------------------------------------------------===//
 /// Helpers extracted from:
 ///   - clang/lib/Headers/avxintrin.h

mlir/include/mlir/Dialect/X86/Utils/X86Utils.h

@@ -29,6 +29,10 @@ namespace x86 {
 bool isInVnniLayout(Operation *op, llvm::ArrayRef<AffineMap> indexingMaps,
                     std::optional<unsigned> blockingFactor = std::nullopt);
 
+// Recursively follows single-use values through scf.yield operations
+// and returns the first non-yield user result in the contraction chain.
+Value contractionUsersAfterYield(Value v);
+
 // Returns true if two contraction ops form a valid pair for VNNI packing.
 // It verifies that both contractions share the appropriate operand, read from
 // the same source buffer, and use constant indices that differ by 8 or 16.
@@ -60,7 +64,7 @@ LogicalResult shuffleAfterReadLikeOp(PatternRewriter &rewriter, Operation *opA,
 // Shuffles vectors produced by vector.contraction ops into a flat layout
 // before they are written to memory.
 LogicalResult shuffleBeforeWriteLikeOp(PatternRewriter &rewriter,
-                                       Operation *opA, Operation *opB,
+                                       Value contractARes, Value contractBRes,
                                        int64_t nonUnitDimAcc, VectorType accTy);
 
 } // namespace x86

mlir/lib/CAPI/IR/IR.cpp

@@ -661,6 +661,25 @@ size_t mlirOperationHashValue(MlirOperation op) {
   return OperationEquivalence::computeHash(unwrap(op));
 }
 
+/// Translates the C equivalence flags to mlir::OperationEquivalence::Flags. The
+/// enumerator values mirror each other.
+static OperationEquivalence::Flags unwrapEquivalenceFlags(uint32_t flags) {
+  return static_cast<OperationEquivalence::Flags>(flags);
+}
+
+bool mlirOperationIsStructurallyEquivalent(MlirOperation lhs, MlirOperation rhs,
+                                           uint32_t flags) {
+  return OperationEquivalence::isEquivalentTo(unwrap(lhs), unwrap(rhs),
+                                              unwrapEquivalenceFlags(flags));
+}
+
+size_t mlirOperationStructuralHashValue(MlirOperation op, uint32_t flags) {
+  return OperationEquivalence::computeHash(
+      unwrap(op), /*hashOperands=*/OperationEquivalence::directHashValue,
+      /*hashResults=*/OperationEquivalence::ignoreHashValue,
+      unwrapEquivalenceFlags(flags));
+}
+
 MlirContext mlirOperationGetContext(MlirOperation op) {
   return wrap(unwrap(op)->getContext());
 }

mlir/lib/Dialect/X86/TransformOps/X86TransformOps.cpp

@@ -52,6 +52,11 @@ void mlir::transform::ApplyVectorContractToAMXDotProductPatternsOp::
   x86::populateVectorContractToAMXDotProductPatterns(patterns);
 }
 
+void mlir::transform::ApplyMoveAccumulatorForContractLoopPatternsOp::
+    populatePatterns(RewritePatternSet &patterns) {
+  x86::populateMoveAccumulatorForContractLoopPatterns(patterns);
+}
+
 //===----------------------------------------------------------------------===//
 // Transform op registration
 //===----------------------------------------------------------------------===//

mlir/lib/Dialect/X86/Transforms/CMakeLists.txt

@@ -7,6 +7,7 @@ add_mlir_dialect_library(MLIRX86Transforms
   SinkVectorProducerOps.cpp
   ShuffleVectorFMAOps.cpp
   VectorContractToAMXDotProduct.cpp
+  MoveAccumulatorForContractLoop.cpp
 
   LINK_LIBS PUBLIC
   MLIRArithDialect

mlir/lib/Dialect/X86/Transforms/MoveAccumulatorForContractLoop.cpp

@@ -0,0 +1,118 @@
+//===- MoveAccumulatorForContractLoop.cpp ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/SCF/IR/SCF.h"
+#include "mlir/Dialect/Vector/IR/VectorOps.h"
+#include "mlir/Dialect/Vector/Utils/VectorUtils.h"
+#include "mlir/Dialect/X86/Transforms.h"
+#include "mlir/Dialect/X86/Utils/X86Utils.h"
+#include "mlir/Dialect/X86/X86Dialect.h"
+
+#include "mlir/IR/BuiltinAttributes.h"
+#include "mlir/IR/Dominance.h"
+#include "mlir/IR/PatternMatch.h"
+#include "llvm/Support/Casting.h"
+
+#include "mlir/Pass/Pass.h"
+#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+
+using namespace mlir;
+using namespace mlir::vector;
+using namespace mlir::x86;
+
+namespace {
+// Transforms vector.contract(A, B, Acc) into vector.contract(A, B, 0) + Acc
+// to decouple the contraction computation from the accumulator update.
+struct MoveAccumulatorForContractLoop
+    : public OpRewritePattern<vector::ContractionOp> {
+  using OpRewritePattern<vector::ContractionOp>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(vector::ContractionOp contractOp,
+                                PatternRewriter &rewriter) const override {
+
+    if (contractOp.getKind() != vector::CombiningKind::ADD)
+      return rewriter.notifyMatchFailure(contractOp,
+                                         "Expects add combining kind.");
+
+    Operation *accReadOp =
+        traceToVectorReadLikeParentOperation(contractOp.getAcc());
+
+    Value contractValue = contractionUsersAfterYield(contractOp.getResult());
+
+    if (!contractValue)
+      return rewriter.notifyMatchFailure(
+          contractOp, "Final acc write might have multiple users.");
+
+    Operation *resultUserOp = *contractValue.getUsers().begin();
+
+    if (!accReadOp || !resultUserOp)
+      return rewriter.notifyMatchFailure(
+          contractOp, "Read from acc matrix is not by "
+                      "transfer_read/load/constant_zero or multiple users of "
+                      "contract operation.");
+
+    if (isa<arith::ConstantOp>(accReadOp))
+      return rewriter.notifyMatchFailure(
+          contractOp,
+          "The input acc to contract is already a constant vector.");
+
+    if ((accReadOp->getBlock() == contractOp->getBlock()) ||
+        (resultUserOp->getBlock() == contractOp->getBlock()))
+      return rewriter.notifyMatchFailure(
+          contractOp, "Acc read/write should be in a separate block.");
+
+    // Replace acc of a contraction operation with vector constant.
+    Value accValue = accReadOp->getResult(0);
+    if (!accValue.hasOneUse())
+      return rewriter.notifyMatchFailure(
+          contractOp, "The input accumulator has multiple users.");
+
+    Operation *firstUser = *accValue.getUsers().begin();
+    rewriter.setInsertionPoint(firstUser);
+
+    auto vecTy = dyn_cast<VectorType>(accValue.getType());
+    if (!vecTy)
+      return rewriter.notifyMatchFailure(contractOp, "Expects vector type.");
+
+    Location loc = accReadOp->getLoc();
+    Type elemTy = vecTy.getElementType();
+
+    Value zeroVec = arith::ConstantOp::create(
+        rewriter, loc,
+        DenseElementsAttr::get(vecTy, rewriter.getZeroAttr(elemTy)));
+
+    accValue.replaceAllUsesWith(zeroVec);
+
+    // Adds the initial acc value with contract results before storing to acc
+    // matrix.
+    rewriter.setInsertionPoint(resultUserOp);
+    Location locUser = resultUserOp->getLoc();
+
+    Value addition;
+
+    if (isa<FloatType>(elemTy)) {
+      addition =
+          arith::AddFOp::create(rewriter, locUser, contractValue, accValue);
+    } else if (isa<IntegerType>(elemTy)) {
+      addition =
+          arith::AddIOp::create(rewriter, locUser, contractValue, accValue);
+    } else {
+      llvm_unreachable("expected floating-point or integer element type");
+    }
+
+    resultUserOp->replaceUsesOfWith(contractValue, addition);
+    return success();
+  }
+};
+
+} // namespace
+
+void x86::populateMoveAccumulatorForContractLoopPatterns(
+    RewritePatternSet &patterns) {
+  patterns.add<MoveAccumulatorForContractLoop>(patterns.getContext());
+}

mlir/lib/Dialect/X86/Transforms/VectorContractBF16ToFMA.cpp

@@ -415,25 +415,21 @@ struct VectorContractBF16ToFMA
       Operation *accReadOp1 =
           traceToVectorReadLikeParentOperation(pairContractOp.getAcc());
 
-      // Iterate down to find the users of contact operations until it is store
-      // or transfer_write.
-      Operation *resultWriteOp0 =
-          traceToVectorWriteLikeUserOperation(contractOp.getResult());
-      Operation *resultWriteOp1 =
-          traceToVectorWriteLikeUserOperation(pairContractOp.getResult());
+      if (!isa<arith::ConstantOp>(accReadOp0)) {
+        // Shuffle the accumulators of the contract operations.
+        LogicalResult readShuffle =
+            shuffleAfterReadLikeOp(rewriter, accReadOp0, accReadOp1, contractOp,
+                                   pairContractOp, nonUnitDim, accTy);
 
-      // Shuffle the accumulators of the contract operations.
-      LogicalResult readShuffle =
-          shuffleAfterReadLikeOp(rewriter, accReadOp0, accReadOp1, contractOp,
-                                 pairContractOp, nonUnitDim, accTy);
-
-      if (failed(readShuffle))
-        return rewriter.notifyMatchFailure(
-            contractOp, "Accumulator read is not by transfer_read or load");
+        if (failed(readShuffle))
+          return rewriter.notifyMatchFailure(
+              contractOp, "Accumulator read is not by transfer_read or load");
+      }
 
       // Shuffle the output of contract operations before its use.
       LogicalResult writeShuffle = shuffleBeforeWriteLikeOp(
-          rewriter, resultWriteOp0, resultWriteOp1, nonUnitDim, accTy);
+          rewriter, contractOp.getResult(), pairContractOp.getResult(),
+          nonUnitDim, accTy);
 
       if (failed(writeShuffle))
         return rewriter.notifyMatchFailure(

mlir/lib/Dialect/X86/Transforms/VectorContractToPackedTypeDotProduct.cpp

@@ -325,17 +325,17 @@ struct VectorContractToPackedTypeDotProduct
       }
 
       // If the accumulators are shuffled we get nullptr else the
-      // transfer_read or load operations.
-      Operation *accRead =
-          traceToVectorReadLikeParentOperation(contractOp.getAcc());
+      // transfer_load or store operations.
+      Operation *accWrite =
+          traceToVectorWriteLikeUserOperation(contractOp.getResult());
 
       if (!pairContractOp &&
-          (!isNonUnitDimOperandShuffled(nonUnitDimOperand) || accRead))
+          (!isNonUnitDimOperandShuffled(nonUnitDimOperand) || accWrite))
         return rewriter.notifyMatchFailure(contractOp,
                                            "Could not find a contract pair");
 
       // Validate and shuffle the accumulator
-      if (accRead) {
+      if (accWrite) {
         // Trace back to the load or transfer_read operations of the contract
         // accumulators.
         Operation *accReadOp0 =
@@ -352,8 +352,8 @@ struct VectorContractToPackedTypeDotProduct
 
         if (!accReadOp0 || !accReadOp1)
           return rewriter.notifyMatchFailure(
-              contractOp,
-              "Operands doesn't have load or transfer_read as it's parent op");
+              contractOp, "Operands doesn't have load or transfer_read or "
+                          "dense constant attribute as its parent op");
 
         if (!resultWriteOp0 || !resultWriteOp1)
           return rewriter.notifyMatchFailure(
@@ -373,18 +373,22 @@ struct VectorContractToPackedTypeDotProduct
           return rewriter.notifyMatchFailure(
               contractOp, "The store/write operation of contract operation is "
                           "before the pair contract operation");
-        // Shuffle the accumulators of the contract operations.
-        LogicalResult readShuffle =
-            shuffleAfterReadLikeOp(rewriter, accReadOp0, accReadOp1, contractOp,
-                                   pairContractOp, nonUnitDimValue, accTy);
 
-        if (failed(readShuffle))
-          return rewriter.notifyMatchFailure(
-              contractOp, "Accumulator read is not by transfer_read or load");
+        if (!isa<arith::ConstantOp>(accReadOp0)) {
+          // Shuffle the accumulators of the contract operations.
+          LogicalResult readShuffle = shuffleAfterReadLikeOp(
+              rewriter, accReadOp0, accReadOp1, contractOp, pairContractOp,
+              nonUnitDimValue, accTy);
+
+          if (failed(readShuffle))
+            return rewriter.notifyMatchFailure(
+                contractOp, "Accumulator read is not by transfer_read or load");
+        }
 
         // Shuffle the output of contract operations before it's use.
         LogicalResult writeShuffle = shuffleBeforeWriteLikeOp(
-            rewriter, resultWriteOp0, resultWriteOp1, nonUnitDimValue, accTy);
+            rewriter, contractOp.getResult(), pairContractOp.getResult(),
+            nonUnitDimValue, accTy);
 
         if (failed(writeShuffle))
           return rewriter.notifyMatchFailure(

mlir/lib/Dialect/X86/Utils/X86Utils.cpp

@@ -144,6 +144,25 @@ inline ShuffleMasks getShuffleMasks(int64_t nonUnitDimAcc, bool isInt8Avx2) {
   return {maskLo8, maskHi8};
 }
 
+// Recursively follows single-use values through scf.yield operations
+// and returns the first non-yield user result in the contraction chain.
+Value contractionUsersAfterYield(Value v) {
+  if (!v || v.getNumUses() != 1)
+    return nullptr;
+
+  OpOperand &use = *v.use_begin();
+  Operation *user = use.getOwner();
+
+  if (!isa<scf::YieldOp>(user))
+    return v;
+
+  auto yield = cast<scf::YieldOp>(user);
+  Operation *parent = yield->getParentOp();
+  unsigned idx = use.getOperandNumber();
+
+  return contractionUsersAfterYield(parent->getResult(idx));
+}
+
 // This function walks backward from a value to locate its originating
 // vector read-like operation (`vector.transfer_read` or `vector.load`).
 // It follows simple forwarding through unary ops and across `scf.for`
@@ -155,8 +174,28 @@ Operation *traceToVectorReadLikeParentOperation(Value v) {
   while (true) {
     // Case 1: Value defined by an operation
     if (Operation *defOp = v.getDefiningOp()) {
-      if (isa<vector::TransferReadOp, vector::LoadOp>(defOp))
+      if (isa<vector::TransferReadOp, vector::LoadOp, arith::ConstantOp>(
+              defOp)) {
+
+        if (auto constOp = dyn_cast<arith::ConstantOp>(defOp)) {
+          if (auto denseAttr =
+                  dyn_cast<DenseElementsAttr>(constOp.getValue())) {
+            if (!denseAttr.isSplat())
+              return nullptr;
+
+            Attribute splat = denseAttr.getSplatValue<Attribute>();
+
+            if (auto floatAttr = dyn_cast<FloatAttr>(splat))
+              return floatAttr.getValue().isZero() ? defOp : nullptr;
+
+            if (auto intAttr = dyn_cast<IntegerAttr>(splat))
+              return intAttr.getValue().isZero() ? defOp : nullptr;
+          }
+
+          return nullptr;
+        }
         return defOp;
+      }
 
       return nullptr;
     }
@@ -290,26 +329,23 @@ LogicalResult shuffleAfterReadLikeOp(PatternRewriter &rewriter, Operation *opA,
 // This function shuffles the vectors written by vector.contract operation
 // as a flat layout structure before they are stored.
 LogicalResult shuffleBeforeWriteLikeOp(PatternRewriter &rewriter,
-                                       Operation *opA, Operation *opB,
+                                       Value contractARes, Value contractBRes,
                                        int64_t nonUnitDimAcc,
                                        VectorType accTy) {
-  // Helper to extract vector operand from write-like ops
-  auto getWrittenVector = [](Operation *op) -> Value {
-    if (auto write = dyn_cast<vector::TransferWriteOp>(op))
-      return write.getVector();
-    if (auto store = dyn_cast<vector::StoreOp>(op))
-      return store.getValueToStore();
-    return nullptr;
-  };
 
-  Value vecA = getWrittenVector(opA);
-  Value vecB = getWrittenVector(opB);
+  Value vecA = contractionUsersAfterYield(contractARes);
+  Value vecB = contractionUsersAfterYield(contractBRes);
 
   if (!vecA || !vecB)
     return failure();
 
+  Operation *resultWriteOpA = *vecA.getUsers().begin();
+  Operation *resultWriteOpB = *vecB.getUsers().begin();
+
   // Decide insertion point and location
-  Operation *insertBefore = opA->isBeforeInBlock(opB) ? opA : opB;
+  Operation *insertBefore = resultWriteOpA->isBeforeInBlock(resultWriteOpB)
+                                ? resultWriteOpA
+                                : resultWriteOpB;
 
   rewriter.setInsertionPoint(insertBefore);
   Location loc = insertBefore->getLoc();
@@ -335,10 +371,8 @@ LogicalResult shuffleBeforeWriteLikeOp(PatternRewriter &rewriter,
   auto newVecB = vector::ShapeCastOp::create(rewriter, loc, accTy, shuffledHi);
 
   // Update write operands in place via the rewriter to notify it of changes.
-  rewriter.modifyOpInPlace(opA,
-                           [&]() { opA->setOperand(0, newVecA.getResult()); });
-  rewriter.modifyOpInPlace(opB,
-                           [&]() { opB->setOperand(0, newVecB.getResult()); });
+  resultWriteOpA->replaceUsesOfWith(vecA, newVecA);
+  resultWriteOpB->replaceUsesOfWith(vecB, newVecB);
 
   return success();
 }

mlir/test/CAPI/ir.c

@@ -3034,6 +3034,187 @@ int testReplaceUsesWithIf(MlirContext ctx) {
   return 0;
 }
 
+int testOperationEquivalence(MlirContext ctx) {
+  fprintf(stderr, "@testOperationEquivalence\n");
+  // CHECK-LABEL: @testOperationEquivalence
+
+  mlirContextGetOrLoadDialect(ctx, mlirStringRefCreateFromCString("arith"));
+
+  const char *moduleStr = "func.func @f(%arg0: i32) -> i32 {\n"
+                          "  %0 = arith.constant 42 : i32\n"
+                          "  %1 = arith.constant 42 : i32\n"
+                          "  %2 = arith.constant 7 : i32\n"
+                          "  %3 = arith.subi %0, %2 : i32\n"
+                          "  %4 = arith.subi %0, %2 : i32\n"
+                          "  %5 = arith.subi %2, %0 : i32\n"
+                          "  %6 = arith.subi %0, %2 {dialect.discardable} : "
+                          "i32\n"
+                          "  %7 = arith.addi %0, %2 : i32\n"
+                          "  %8 = arith.addi %2, %0 : i32\n"
+                          "  return %0 : i32\n"
+                          "}\n";
+  MlirModule module =
+      mlirModuleCreateParse(ctx, mlirStringRefCreateFromCString(moduleStr));
+
+  MlirBlock moduleBody = mlirModuleGetBody(module);
+  MlirOperation funcOp = mlirBlockGetFirstOperation(moduleBody);
+  MlirRegion funcRegion = mlirOperationGetRegion(funcOp, 0);
+  MlirBlock funcBody = mlirRegionGetFirstBlock(funcRegion);
+
+  MlirOperation c42a = mlirBlockGetFirstOperation(funcBody);
+  MlirOperation c42b = mlirOperationGetNextInBlock(c42a);
+  MlirOperation c7 = mlirOperationGetNextInBlock(c42b);
+  MlirOperation sub3 = mlirOperationGetNextInBlock(c7);
+  MlirOperation sub4 = mlirOperationGetNextInBlock(sub3);
+  MlirOperation sub5 = mlirOperationGetNextInBlock(sub4);
+  MlirOperation sub6 = mlirOperationGetNextInBlock(sub5);
+  MlirOperation add7 = mlirOperationGetNextInBlock(sub6);
+  MlirOperation add8 = mlirOperationGetNextInBlock(add7);
+
+  // Two identical constants are structurally equivalent when locations are
+  // ignored, even though their result SSA values and source locations differ.
+  assert(mlirOperationIsStructurallyEquivalent(
+      c42a, c42b, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
+
+  // Without ignoring locations they differ, since they sit on distinct lines.
+  assert(!mlirOperationIsStructurallyEquivalent(
+      c42a, c42b, MLIR_OPERATION_EQUIVALENCE_NONE));
+
+  // A constant with a different value is not equivalent.
+  assert(!mlirOperationIsStructurallyEquivalent(
+      c42a, c7, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
+
+  // Equivalence is reflexive.
+  assert(mlirOperationIsStructurallyEquivalent(
+      c42a, c42a, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
+
+  // `%3 = subi %0, %2` and `%4 = subi %0, %2` use the exact same operands, so
+  // they are equivalent.
+  assert(mlirOperationIsStructurallyEquivalent(
+      sub3, sub4, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
+
+  // `%5 = subi %2, %0` is not equivalent to `%3 = subi %0, %2`: `subi` is not
+  // commutative, so operands are compared pairwise, and the very first pair
+  // (`%0` = 42 vs `%2` = 7) already mismatches. See the `addi` pair below for
+  // the commutative path.
+  assert(!mlirOperationIsStructurallyEquivalent(
+      sub3, sub5, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
+
+  // A constant and a subtraction are different operations.
+  assert(!mlirOperationIsStructurallyEquivalent(
+      c42a, sub3, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
+
+  // `%6 = subi %0, %2 {dialect.discardable}` carries a discardable attribute
+  // that `%3 = subi %0, %2` does not, so the two differ unless discardable
+  // attributes are ignored as well. This also exercises OR-ing flags together.
+  assert(!mlirOperationIsStructurallyEquivalent(
+      sub3, sub6, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
+  assert(mlirOperationIsStructurallyEquivalent(
+      sub3, sub6,
+      MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS |
+          MLIR_OPERATION_EQUIVALENCE_IGNORE_DISCARDABLE_ATTRS));
+
+  // `%7 = addi %0, %2` and `%8 = addi %2, %0` have swapped operands, but `addi`
+  // is commutative, so they are equivalent through the commutative path and
+  // only differ once commutativity is explicitly ignored.
+  assert(mlirOperationIsStructurallyEquivalent(
+      add7, add8, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
+  assert(!mlirOperationIsStructurallyEquivalent(
+      add7, add8,
+      MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS |
+          MLIR_OPERATION_EQUIVALENCE_IGNORE_COMMUTATIVITY));
+
+  // The `value` of arith.constant is an inherent attribute held in the
+  // operation's properties, so ignoring properties makes constants with
+  // different values equivalent.
+  assert(mlirOperationIsStructurallyEquivalent(
+      c42a, c7,
+      MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS |
+          MLIR_OPERATION_EQUIVALENCE_IGNORE_PROPERTIES));
+
+  // The structural hash pairs with the equivalence above: operations that are
+  // equivalent under a set of flags hash equally under the same flags.
+  assert(mlirOperationStructuralHashValue(
+             c42a, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS) ==
+         mlirOperationStructuralHashValue(
+             c42b, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
+  assert(mlirOperationStructuralHashValue(
+             sub3, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS) ==
+         mlirOperationStructuralHashValue(
+             sub4, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
+  assert(mlirOperationStructuralHashValue(
+             sub3, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS |
+                       MLIR_OPERATION_EQUIVALENCE_IGNORE_DISCARDABLE_ATTRS) ==
+         mlirOperationStructuralHashValue(
+             sub6, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS |
+                       MLIR_OPERATION_EQUIVALENCE_IGNORE_DISCARDABLE_ATTRS));
+  // Commutative operands are folded into the hash in an order-insensitive way,
+  // matching the commutative equivalence above.
+  assert(mlirOperationStructuralHashValue(
+             add7, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS) ==
+         mlirOperationStructuralHashValue(
+             add8, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
+
+  // The flags are threaded through to the hash: locations and discardable
+  // attributes are hashed unless the corresponding flag is set. Hash
+  // inequality is not part of the contract, but these inputs do differ.
+  assert(
+      mlirOperationStructuralHashValue(c42a, MLIR_OPERATION_EQUIVALENCE_NONE) !=
+      mlirOperationStructuralHashValue(c42b, MLIR_OPERATION_EQUIVALENCE_NONE));
+  assert(mlirOperationStructuralHashValue(
+             sub3, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS) !=
+         mlirOperationStructuralHashValue(
+             sub6, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
+
+  mlirModuleDestroy(module);
+
+  // Operands defined *inside* the compared regions need not be the same SSA
+  // values: recursing into the regions marks the two inner constants
+  // equivalent, and the `addi` operands are then matched through that mapping.
+  const char *regionModuleStr = "func.func @g() {\n"
+                                "  %r0 = scf.execute_region -> i32 {\n"
+                                "    %c = arith.constant 1 : i32\n"
+                                "    %d = arith.addi %c, %c : i32\n"
+                                "    scf.yield %d : i32\n"
+                                "  }\n"
+                                "  %r1 = scf.execute_region -> i32 {\n"
+                                "    %c = arith.constant 1 : i32\n"
+                                "    %d = arith.addi %c, %c : i32\n"
+                                "    scf.yield %d : i32\n"
+                                "  }\n"
+                                "  return\n"
+                                "}\n";
+  MlirModule regionModule = mlirModuleCreateParse(
+      ctx, mlirStringRefCreateFromCString(regionModuleStr));
+  assert(!mlirModuleIsNull(regionModule));
+  MlirBlock regionFuncBody = mlirRegionGetFirstBlock(mlirOperationGetRegion(
+      mlirBlockGetFirstOperation(mlirModuleGetBody(regionModule)), 0));
+  MlirOperation exec0 = mlirBlockGetFirstOperation(regionFuncBody);
+  MlirOperation exec1 = mlirOperationGetNextInBlock(exec0);
+  assert(mlirOperationIsStructurallyEquivalent(
+      exec0, exec1, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
+
+  // The nested `addi` operations compared on their own are *not* equivalent:
+  // the value mapping starts out empty, so operands defined outside the
+  // compared operation must be the exact same SSA values.
+  MlirBlock exec0Body =
+      mlirRegionGetFirstBlock(mlirOperationGetRegion(exec0, 0));
+  MlirBlock exec1Body =
+      mlirRegionGetFirstBlock(mlirOperationGetRegion(exec1, 0));
+  MlirOperation innerAdd0 =
+      mlirOperationGetNextInBlock(mlirBlockGetFirstOperation(exec0Body));
+  MlirOperation innerAdd1 =
+      mlirOperationGetNextInBlock(mlirBlockGetFirstOperation(exec1Body));
+  assert(!mlirOperationIsStructurallyEquivalent(
+      innerAdd0, innerAdd1, MLIR_OPERATION_EQUIVALENCE_IGNORE_LOCATIONS));
+
+  mlirModuleDestroy(regionModule);
+
+  // CHECK: testOperationEquivalence: PASSED
+  fprintf(stderr, "testOperationEquivalence: PASSED\n");
+  return 0;
+}
+
 int main(void) {
   MlirContext ctx = mlirContextCreate();
   registerAllUpstreamDialects(ctx);
@@ -3091,6 +3272,8 @@ int main(void) {
     return 20;
   if (testReplaceUsesWithIf(ctx))
     return 21;
+  if (testOperationEquivalence(ctx))
+    return 22;
 
   // CHECK: DESTROY MAIN CONTEXT
   // CHECK: reportResourceDelete: resource_i64_blob

mlir/test/Dialect/X86/move-acc-for-contract-loop.mlir

@@ -0,0 +1,323 @@
+// RUN: mlir-opt %s -transform-interpreter -cse -split-input-file | FileCheck %s
+
+!vecA = vector<1x1x2xbf16>
+!vecB = vector<1x2x16xbf16>
+!vecC = vector<1x16xf32>
+!memrefA = memref<1x1x2xbf16, strided<[2048, 32, 1], offset: ?>>
+!memrefB = memref<1x2x32xbf16, strided<[2048, 64, 1], offset: ?>>
+!memrefC = memref<1x32xf32, strided<[64, 1], offset: ?>>
+#map = affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>
+#map1 = affine_map<(d0, d1, d2, d3) -> (d0, d3, d2)>
+#map2 = affine_map<(d0, d1, d2, d3) -> (d1, d2)>
+
+func.func @brmatmul_acc_mv(%arg0: memref<16x64x32xbf16>, %arg1: memref<16x32x64xbf16>,
+                             %arg2: memref<64x64xf32>) -> memref<64x64xf32> {
+  %0 = ub.poison : f32
+  %1 = ub.poison : bf16
+  %c0 = arith.constant 0 : index
+  %c64 = arith.constant 64 : index
+  %c16 = arith.constant 16 : index
+  %c32 = arith.constant 32 : index
+  %c1 = arith.constant 1 : index
+  %c2 = arith.constant 2 : index
+  scf.for %arg3 = %c0 to %c64 step %c1 {
+    scf.for %arg4 = %c0 to %c64 step %c32 {
+      %subview = memref.subview %arg2[%arg3, %arg4] [1, 32] [1, 1]
+                        : memref<64x64xf32> to !memrefC
+      %2 = vector.transfer_read %subview[%c0, %c0], %0 {in_bounds = [true, true]}
+                        : !memrefC, !vecC
+      %3 = vector.transfer_read %subview[%c0, %c16], %0 {in_bounds = [true, true]}
+                        : !memrefC, !vecC
+
+      %4:2 = scf.for %arg5 = %c0 to %c16 step %c1 iter_args(%arg6 = %2, %arg7 = %3) -> (!vecC, !vecC) {
+        %5:2 = scf.for %arg8 = %c0 to %c32 step %c2 iter_args(%arg9 = %arg6, %arg10 = %arg7) -> (!vecC, !vecC) {
+
+          %subview_0 = memref.subview %arg0[%arg5, %arg3, %arg8] [1, 1, 2] [1, 1, 1]
+                        : memref<16x64x32xbf16> to !memrefA
+          %subview_1 = memref.subview %arg1[%arg5, %arg8, %arg4] [1, 2, 32] [1, 1, 1]
+                        : memref<16x32x64xbf16> to !memrefB
+
+          %6 = vector.transfer_read %subview_0[%c0, %c0, %c0], %1 {in_bounds = [true, true, true]}
+                        : !memrefA, !vecA
+          %7 = vector.transfer_read %subview_1[%c0, %c0, %c0], %1 {in_bounds = [true, true, true]}
+                        : !memrefB, !vecB
+          %8 = vector.transfer_read %subview_1[%c0, %c0, %c16], %1 {in_bounds = [true, true, true]}
+                        : !memrefB, !vecB
+
+          %9 = vector.contract {indexing_maps = [#map, #map1, #map2], iterator_types =
+                        ["reduction", "parallel", "parallel", "reduction"], kind = #vector.kind<add>} %6, %7, %arg9
+                        {unroll_shape = array<i64: 1, 1, 16, 2>} : !vecA, !vecB into !vecC
+          %10 = vector.contract {indexing_maps = [#map, #map1, #map2], iterator_types =
+                        ["reduction", "parallel", "parallel", "reduction"], kind = #vector.kind<add>} %6, %8, %arg10
+                        {unroll_shape = array<i64: 1, 1, 16, 2>} : !vecA, !vecB into !vecC
+
+          scf.yield %9, %10 : !vecC, !vecC
+        }
+        scf.yield %5#0, %5#1 : !vecC, !vecC
+      }
+
+      vector.transfer_write %4#1, %subview[%c0, %c16] {in_bounds = [true, true]}
+                        : !vecC, !memrefC
+      vector.transfer_write %4#0, %subview[%c0, %c0] {in_bounds = [true, true]}
+                        : !vecC, !memrefC
+    }
+  }
+
+  return %arg2 : memref<64x64xf32>
+}
+
+// CHECK-LABEL: @brmatmul_acc_mv
+// CHECK: %[[CST:.*]] = arith.constant dense<0.000000e+00> : vector<1x16xf32>
+// CHECK: scf.for
+// CHECK-NEXT: scf.for
+// CHECK: memref.subview
+// CHECK: %[[TRANSFER_READ_0:.*]] = vector.transfer_read
+// CHECK: %[[TRANSFER_READ_1:.*]] = vector.transfer_read
+// CHECK: scf.for {{.*}} iter_args(%[[VAL_3:.*]] = %[[CST]], %[[VAL_4:.*]] = %[[CST]]) -> (vector<1x16xf32>, vector<1x16xf32>) {
+// CHECK-NEXT: scf.for
+// CHECK: scf.yield
+// CHECK: scf.yield
+// CHECK: %[[ADDF_0:.*]] = arith.addf {{.*}}, %[[TRANSFER_READ_1]] : vector<1x16xf32>
+// CHECK-NEXT: vector.transfer_write %[[ADDF_0:.*]], %subview[%c0, %c16] {{.*}}
+// CHECK: %[[ADDF_1:.*]] = arith.addf {{.*}}, %[[TRANSFER_READ_0]] : vector<1x16xf32>
+// CHECK-NEXT: vector.transfer_write %[[ADDF_1:.*]], %subview[%c0, %c0] {{.*}}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %func = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    transform.apply_patterns to %func {
+      transform.apply_patterns.x86.move_accumulator_for_contract_loop
+    } : !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+!vecA = vector<1x16x16x2xbf16>
+!vecB = vector<1x16x16x2xbf16>
+!vecC = vector<16x16xf32>
+!memrefA = memref<1x32x16x2xbf16>
+!memrefB = memref<1x16x32x2xbf16>
+!memrefC = memref<32x32xf32>
+#map = affine_map<(d0, d4, d1, d2, d3) -> (d0, d1, d3, d4)>
+#map1 = affine_map<(d0, d4, d1, d2, d3) -> (d0, d3, d2, d4)>
+#map2 = affine_map<(d0, d4, d1, d2, d3) -> (d1, d2)>
+func.func @negative_no_loop(
+  %arg0: !memrefA, %arg1: !memrefB, %arg2: !memrefC) -> !memrefC
+{
+  %c0 = arith.constant 0 : index
+  %0 = ub.poison : bf16
+  %32 = ub.poison : f32
+
+  %1 = vector.transfer_read %arg0[%c0, %c0, %c0, %c0], %0 {in_bounds = [true, true, true, true]} :
+        !memrefA, !vecA
+  %2 = vector.transfer_read %arg1[%c0, %c0, %c0, %c0], %0 {in_bounds = [true, true, true, true]} :
+        !memrefB, !vecB
+
+  %3 = vector.transfer_read %arg2[%c0, %c0], %32 {in_bounds = [true, true]} : !memrefC, !vecC
+
+  %4 = vector.contract {
+    indexing_maps = [#map, #map1, #map2],
+    iterator_types = ["reduction", "reduction", "parallel", "parallel", "reduction"],
+    kind = #vector.kind<add>}
+    %1, %2, %3 : !vecA, !vecB into !vecC
+
+  vector.transfer_write %4, %arg2[%c0, %c0] {in_bounds = [true, true]} : !vecC, !memrefC
+
+  return %arg2 : !memrefC
+}
+
+// CHECK-LABEL: @negative_no_loop
+// CHECK-NOT: arith.constant dense<0.000000e+00> {{.*}}
+// CHECK-NOT: arith.addf
+// CHECK-NOT: arith.addf
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %func = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    transform.apply_patterns to %func {
+      transform.apply_patterns.x86.move_accumulator_for_contract_loop
+    } : !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+!vecA = vector<1x1x2xbf16>
+!vecB = vector<1x2x16xbf16>
+!vecC = vector<1x16xf32>
+!memrefA = memref<1x1x2xbf16, strided<[2048, 32, 1], offset: ?>>
+!memrefB = memref<1x2x32xbf16, strided<[2048, 64, 1], offset: ?>>
+!memrefC = memref<1x32xf32, strided<[64, 1], offset: ?>>
+#map = affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>
+#map1 = affine_map<(d0, d1, d2, d3) -> (d0, d3, d2)>
+#map2 = affine_map<(d0, d1, d2, d3) -> (d1, d2)>
+
+func.func @negative_non_zero_cst(%arg0: memref<16x64x32xbf16>, %arg1: memref<16x32x64xbf16>,
+                             %arg2: memref<64x64xf32>) -> memref<64x64xf32> {
+  %cst2 = arith.constant dense<2.000000e+00> : vector<1x16xf32>
+  %cst3 = arith.constant dense<3.000000e+00> : vector<1x16xf32>
+  %0 = ub.poison : f32
+  %1 = ub.poison : bf16
+  %c0 = arith.constant 0 : index
+  %c64 = arith.constant 64 : index
+  %c16 = arith.constant 16 : index
+  %c32 = arith.constant 32 : index
+  %c1 = arith.constant 1 : index
+  %c2 = arith.constant 2 : index
+  scf.for %arg3 = %c0 to %c64 step %c1 {
+    scf.for %arg4 = %c0 to %c64 step %c32 {
+      %subview = memref.subview %arg2[%arg3, %arg4] [1, 32] [1, 1]
+                        : memref<64x64xf32> to !memrefC
+
+      %4:2 = scf.for %arg5 = %c0 to %c16 step %c1 iter_args(%arg6 = %cst2, %arg7 = %cst3) -> (!vecC, !vecC) {
+        %5:2 = scf.for %arg8 = %c0 to %c32 step %c2 iter_args(%arg9 = %arg6, %arg10 = %arg7) -> (!vecC, !vecC) {
+
+          %subview_0 = memref.subview %arg0[%arg5, %arg3, %arg8] [1, 1, 2] [1, 1, 1]
+                        : memref<16x64x32xbf16> to !memrefA
+          %subview_1 = memref.subview %arg1[%arg5, %arg8, %arg4] [1, 2, 32] [1, 1, 1]
+                        : memref<16x32x64xbf16> to !memrefB
+
+          %6 = vector.transfer_read %subview_0[%c0, %c0, %c0], %1 {in_bounds = [true, true, true]}
+                        : !memrefA, !vecA
+          %7 = vector.transfer_read %subview_1[%c0, %c0, %c0], %1 {in_bounds = [true, true, true]}
+                        : !memrefB, !vecB
+          %8 = vector.transfer_read %subview_1[%c0, %c0, %c16], %1 {in_bounds = [true, true, true]}
+                        : !memrefB, !vecB
+
+          %9 = vector.contract {indexing_maps = [#map, #map1, #map2], iterator_types =
+                        ["reduction", "parallel", "parallel", "reduction"], kind = #vector.kind<add>} %6, %7, %arg9
+                        {unroll_shape = array<i64: 1, 1, 16, 2>} : !vecA, !vecB into !vecC
+          %10 = vector.contract {indexing_maps = [#map, #map1, #map2], iterator_types =
+                        ["reduction", "parallel", "parallel", "reduction"], kind = #vector.kind<add>} %6, %8, %arg10
+                        {unroll_shape = array<i64: 1, 1, 16, 2>} : !vecA, !vecB into !vecC
+
+          scf.yield %9, %10 : !vecC, !vecC
+        }
+        scf.yield %5#0, %5#1 : !vecC, !vecC
+      }
+
+      vector.transfer_write %4#1, %subview[%c0, %c16] {in_bounds = [true, true]}
+                        : !vecC, !memrefC
+      vector.transfer_write %4#0, %subview[%c0, %c0] {in_bounds = [true, true]}
+                        : !vecC, !memrefC
+    }
+  }
+
+  return %arg2 : memref<64x64xf32>
+}
+
+// CHECK-LABEL: @negative_non_zero_cst
+// CHECK-NOT: %[[CST:.*]] = arith.constant dense<0.000000e+00> : vector<1x16xf32>
+// CHECK: scf.for
+// CHECK-NEXT: scf.for
+// CHECK: memref.subview
+// CHECK: scf.for 
+// CHECK-NEXT: scf.for
+// CHECK: scf.yield
+// CHECK: scf.yield
+// CHECK-NOT: arith.addf 
+
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %func = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    transform.apply_patterns to %func {
+      transform.apply_patterns.x86.move_accumulator_for_contract_loop
+    } : !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+!vecA = vector<1x1x2xbf16>
+!vecB = vector<1x2x16xbf16>
+!vecC = vector<1x16xf32>
+!memrefA = memref<1x1x2xbf16, strided<[2048, 32, 1], offset: ?>>
+!memrefB = memref<1x2x32xbf16, strided<[2048, 64, 1], offset: ?>>
+!memrefC = memref<1x32xf32, strided<[64, 1], offset: ?>>
+#map = affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>
+#map1 = affine_map<(d0, d1, d2, d3) -> (d0, d3, d2)>
+#map2 = affine_map<(d0, d1, d2, d3) -> (d1, d2)>
+
+func.func @negative_input_acc_many_users(%arg0: memref<16x64x32xbf16>, %arg1: memref<16x32x64xbf16>,
+                             %arg2: memref<64x64xf32>) -> memref<64x64xf32> {
+  %0 = ub.poison : f32
+  %1 = ub.poison : bf16
+  %c0 = arith.constant 0 : index
+  %c64 = arith.constant 64 : index
+  %c16 = arith.constant 16 : index
+  %c32 = arith.constant 32 : index
+  %c1 = arith.constant 1 : index
+  %c2 = arith.constant 2 : index
+  scf.for %arg3 = %c0 to %c64 step %c1 {
+    scf.for %arg4 = %c0 to %c64 step %c32 {
+      %subview = memref.subview %arg2[%arg3, %arg4] [1, 32] [1, 1]
+                        : memref<64x64xf32> to !memrefC
+      %2 = vector.transfer_read %subview[%c0, %c0], %0 {in_bounds = [true, true]}
+                        : !memrefC, !vecC
+      %3 = vector.transfer_read %subview[%c0, %c16], %0 {in_bounds = [true, true]}
+                        : !memrefC, !vecC
+
+      %4:2 = scf.for %arg5 = %c0 to %c16 step %c1 iter_args(%arg6 = %2, %arg7 = %3) -> (!vecC, !vecC) {
+        %5:2 = scf.for %arg8 = %c0 to %c32 step %c2 iter_args(%arg9 = %arg6, %arg10 = %arg7) -> (!vecC, !vecC) {
+
+          %subview_0 = memref.subview %arg0[%arg5, %arg3, %arg8] [1, 1, 2] [1, 1, 1]
+                        : memref<16x64x32xbf16> to !memrefA
+          %subview_1 = memref.subview %arg1[%arg5, %arg8, %arg4] [1, 2, 32] [1, 1, 1]
+                        : memref<16x32x64xbf16> to !memrefB
+
+          %6 = vector.transfer_read %subview_0[%c0, %c0, %c0], %1 {in_bounds = [true, true, true]}
+                        : !memrefA, !vecA
+          %7 = vector.transfer_read %subview_1[%c0, %c0, %c0], %1 {in_bounds = [true, true, true]}
+                        : !memrefB, !vecB
+          %8 = vector.transfer_read %subview_1[%c0, %c0, %c16], %1 {in_bounds = [true, true, true]}
+                        : !memrefB, !vecB
+
+          %9 = vector.contract {indexing_maps = [#map, #map1, #map2], iterator_types =
+                        ["reduction", "parallel", "parallel", "reduction"], kind = #vector.kind<add>} %6, %7, %arg9
+                        {unroll_shape = array<i64: 1, 1, 16, 2>} : !vecA, !vecB into !vecC
+          %10 = vector.contract {indexing_maps = [#map, #map1, #map2], iterator_types =
+                        ["reduction", "parallel", "parallel", "reduction"], kind = #vector.kind<add>} %6, %8, %arg10
+                        {unroll_shape = array<i64: 1, 1, 16, 2>} : !vecA, !vecB into !vecC
+
+          scf.yield %9, %10 : !vecC, !vecC
+        }
+        scf.yield %5#0, %5#1 : !vecC, !vecC
+      }
+
+      %7 = arith.addf %4#1, %2 : vector<1x16xf32>
+      vector.transfer_write %7, %subview[%c0, %c16] {in_bounds = [true, true]}
+                        : !vecC, !memrefC
+      %8 = arith.addf %4#0, %3 : vector<1x16xf32>
+      vector.transfer_write %8, %subview[%c0, %c0] {in_bounds = [true, true]}
+                        : !vecC, !memrefC
+    }
+  }
+
+  return %arg2 : memref<64x64xf32>
+}
+// CHECK-LABEL: @negative_input_acc_many_users
+// CHECK-NOT: %[[CST:.*]] = arith.constant dense<0.000000e+00> : vector<1x16xf32>
+// CHECK: scf.for
+// CHECK-NEXT: scf.for
+// CHECK: memref.subview
+// CHECK: scf.for
+// CHECK-NEXT: scf.for
+// CHECK: scf.yield
+// CHECK: scf.yield
+
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %func = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    transform.apply_patterns to %func {
+      transform.apply_patterns.x86.move_accumulator_for_contract_loop
+    } : !transform.any_op
+    transform.yield
+  }
+}
+

mlir/test/Dialect/X86/vector-contract-bf16-to-fma.mlir

@@ -444,8 +444,6 @@ func.func @matmul_to_fma_flat_layout_loop(%arg0: memref<16x64x32xbf16>, %arg1: m
 }
 
 // CHECK-LABEL: @matmul_to_fma_flat_layout_loop
-// CHECK: vector.shuffle{{.*}}[0, 8, 1, 9, 2, 10, 3, 11] : vector<8xf32>, vector<8xf32>
-// CHECK-NEXT: vector.shuffle{{.*}}[4, 12, 5, 13, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32>
 // CHECK: scf.for
 // CHECK: scf.for
 // CHECK: x86.avx.bcst_to_f32.packed
@@ -461,6 +459,7 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %func = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     transform.apply_patterns to %func {
+      transform.apply_patterns.x86.move_accumulator_for_contract_loop
       transform.apply_patterns.x86.vector_contract_bf16_to_fma
     } : !transform.any_op
     transform.yield

mlir/test/Dialect/X86/vector-contract-to-packed-type-dotproduct.mlir

@@ -687,8 +687,6 @@ func.func @brmatmul_bf16dp_flat_layout_loop(%arg0: memref<16x64x32xbf16>, %arg1:
 }
 
 // CHECK-LABEL: @brmatmul_bf16dp_flat_layout_loop
-// CHECK: vector.shuffle{{.*}}[0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23] : vector<16xf32>, vector<16xf32>
-// CHECK-NEXT: vector.shuffle{{.*}}[8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31] : vector<16xf32>, vector<16xf32>
 // CHECK: scf.for
 // CHECK: scf.for
 // CHECK: vector.transfer_read {{.*}} vector<32xbf16>
@@ -706,6 +704,7 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %func = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     transform.apply_patterns to %func {
+      transform.apply_patterns.x86.move_accumulator_for_contract_loop
       transform.apply_patterns.x86.vector_contract_to_packed_type_dot_product
     } : !transform.any_op
     transform.yield

utils/bazel/llvm-project-overlay/libc/BUILD.bazel

@@ -997,6 +997,7 @@ libc_support_library(
         ":__support_macros_config",
         ":__support_macros_properties_architectures",
         ":__support_macros_properties_compiler",
+        ":__support_macros_properties_os",
         ":hdr_stdint_proxy",
     ],
 )