From aaabd24839d3382b5c4944aa5a6a43ffd0ff14cc Mon Sep 17 00:00:00 2001
From: Shilei Tian <i@tianshilei.me>
Date: Sat, 1 Aug 2026 18:20:47 -0400
Subject: [PATCH] [AMDGPU] Support partial and empty WWM pools for SGPR spills

SGPR lane spilling currently treats the WWM VGPR pool as all-or-nothing. This
can fail compilation when the requested pool cannot be formed, even though
scratch spilling or a smaller spillable pool could make progress.

This PR lets ordinary SGPR spills fall back to scratch when the pool is empty
and lets WWM register allocation use a nonempty partial pool. It keeps the
full-pool requirement for strict WWM/WQM and explicit spill-carrier
preallocation.

The no-pool fallback is recorded in SIMachineFunctionInfo so frame lowering can
provide enough emergency scavenging slots. The state is also serialized to
preserve the behavior across MIR round trips.
---
 llvm/lib/Target/AMDGPU/SIFrameLowering.cpp    |  14 +-
 llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp  | 144 ++++++++----
 .../Target/AMDGPU/SIMachineFunctionInfo.cpp   |   2 +
 .../lib/Target/AMDGPU/SIMachineFunctionInfo.h |  13 ++
 .../Target/AMDGPU/SIPreAllocateWWMRegs.cpp    |   9 +-
 llvm/lib/Target/AMDGPU/SIPreAllocateWWMRegs.h |   2 +
 .../schedule-amdgpu-tracker-physreg-crash.ll  |   6 +-
 .../AMDGPU/sgpr-spill-vmem-large-frame.mir    |  36 +++
 .../test/CodeGen/AMDGPU/wwm-regalloc-error.ll |  20 +-
 .../AMDGPU/wwm-regalloc-memory-fallback.ll    | 213 ++++++++++++++++++
 .../AMDGPU/wwm-regalloc-partial-pool.mir      |  50 ++++
 .../wwm-regalloc-preallocation-guard.mir      | 101 +++++++++
 12 files changed, 549 insertions(+), 61 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/wwm-regalloc-memory-fallback.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/wwm-regalloc-partial-pool.mir
 create mode 100644 llvm/test/CodeGen/AMDGPU/wwm-regalloc-preallocation-guard.mir

diff --git a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
index bcd8d0bef062..c1fa828f38bb 100644
--- a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
@@ -1793,10 +1793,16 @@ void SIFrameLowering::processFunctionBeforeFrameFinalized(
     // Add an emergency spill slot
     RS->addScavengingFrameIndex(FuncInfo->getScavengeFI(MFI, *TRI));
 
-    // If we are spilling SGPRs to memory with a large frame, we may need a
-    // second VGPR emergency frame index.
-    if (HaveSGPRToVMemSpill &&
-        allocateScavengingFrameIndexesNearIncomingSP(MF)) {
+    if (HaveSGPRToVMemSpill && FuncInfo->hasNoWWMPoolSGPRSpillFallback()) {
+      // The no-WWM-pool fallback can reach SGPR-to-memory lowering while an
+      // ordinary frame-index scavenge is live. It may then need one slot for
+      // its temporary VGPR and another for recursive address materialization.
+      RS->addScavengingFrameIndex(MFI.CreateSpillStackObject(4, Align(4)));
+      RS->addScavengingFrameIndex(MFI.CreateSpillStackObject(4, Align(4)));
+    } else if (HaveSGPRToVMemSpill &&
+               allocateScavengingFrameIndexesNearIncomingSP(MF)) {
+      // Existing large-frame SGPR-to-memory spills need one additional VGPR
+      // emergency frame index.
       RS->addScavengingFrameIndex(MFI.CreateSpillStackObject(4, Align(4)));
     }
   }
diff --git a/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp b/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
index 8cc7714c6fa2..5bf34ddcb281 100644
--- a/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
+++ b/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
@@ -20,6 +20,7 @@
 #include "GCNSubtarget.h"
 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
 #include "SIMachineFunctionInfo.h"
+#include "SIPreAllocateWWMRegs.h"
 #include "SISpillUtils.h"
 #include "llvm/CodeGen/LiveIntervals.h"
 #include "llvm/CodeGen/MachineCycleAnalysis.h"
@@ -80,7 +81,9 @@ public:
   void updateLaneVGPRDomInstr(
       int FI, MachineBasicBlock *MBB, MachineBasicBlock::iterator InsertPt,
       DenseMap<Register, LaneVGPRInsertPt> &LaneVGPRDomInstr);
-  void determineRegsForWWMAllocation(MachineFunction &MF, BitVector &RegMask);
+  SmallVector<MCRegister> determineRegsForWWMAllocation(MachineFunction &MF);
+  void assignWWMRegs(MachineFunction &MF, ArrayRef<MCRegister> WWMRegCandidates,
+                     bool RequiresFullWWMPool);
 };
 
 class SILowerSGPRSpillsLegacy : public MachineFunctionPass {
@@ -372,42 +375,64 @@ void SILowerSGPRSpills::updateLaneVGPRDomInstr(
   }
 }
 
-void SILowerSGPRSpills::determineRegsForWWMAllocation(MachineFunction &MF,
-                                                      BitVector &RegMask) {
-  // Determine an optimal number of VGPRs for WWM allocation. The complement
-  // list will be available for allocating other VGPR virtual registers.
-  SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
+SmallVector<MCRegister>
+SILowerSGPRSpills::determineRegsForWWMAllocation(MachineFunction &MF) {
+  SmallVector<MCRegister> WWMRegCandidates;
+  if (!MaxNumVGPRsForWwmAllocation)
+    return WWMRegCandidates;
+
   MachineRegisterInfo &MRI = MF.getRegInfo();
   BitVector ReservedRegs = TRI->getReservedRegs(MF);
-  BitVector NonWwmAllocMask(TRI->getNumRegs());
   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+  unsigned MaxNumVGPRs = ST.getMaxNumVectorRegs(MF.getFunction()).first;
 
-  // FIXME: MaxNumVGPRsForWwmAllocation might need to be adjusted in the future
-  // to have a balanced allocation between WWM values and per-thread vector
-  // register operands.
-  unsigned NumRegs = MaxNumVGPRsForWwmAllocation;
-  NumRegs =
-      std::min(static_cast<unsigned>(MFI->getSGPRSpillVGPRs().size()), NumRegs);
-
-  auto [MaxNumVGPRs, MaxNumAGPRs] = ST.getMaxNumVectorRegs(MF.getFunction());
   // Try to use the highest available registers for now. Later after
   // vgpr-regalloc, they can be shifted to the lowest range.
-  unsigned I = 0;
   for (unsigned Reg = AMDGPU::VGPR0 + MaxNumVGPRs - 1;
-       (I < NumRegs) && (Reg >= AMDGPU::VGPR0); --Reg) {
+       WWMRegCandidates.size() < MaxNumVGPRsForWwmAllocation &&
+       Reg >= AMDGPU::VGPR0;
+       --Reg) {
     if (!ReservedRegs.test(Reg) &&
-        !MRI.isPhysRegUsed(Reg, /*SkipRegMaskTest=*/true)) {
-      TRI->markSuperRegs(RegMask, Reg);
-      ++I;
-    }
+        !MRI.isPhysRegUsed(Reg, /*SkipRegMaskTest=*/true))
+      WWMRegCandidates.push_back(Reg);
   }
 
-  if (I != NumRegs) {
+  return WWMRegCandidates;
+}
+
+void SILowerSGPRSpills::assignWWMRegs(MachineFunction &MF,
+                                      ArrayRef<MCRegister> WWMRegCandidates,
+                                      bool RequiresFullWWMPool) {
+  SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
+  if (FuncInfo->getSGPRSpillVGPRs().empty())
+    return;
+
+  BitVector WwmRegMask(TRI->getNumRegs());
+
+  unsigned DesiredPoolSize =
+      std::min(static_cast<unsigned>(FuncInfo->getSGPRSpillVGPRs().size()),
+               static_cast<unsigned>(MaxNumVGPRsForWwmAllocation));
+  unsigned SelectedPoolSize =
+      std::min<unsigned>(DesiredPoolSize, WWMRegCandidates.size());
+  // WWM register candidates are ordered high-to-low, so take the highest
+  // available registers when the desired pool is smaller than the candidate
+  // list.
+  for (MCRegister Reg : WWMRegCandidates.take_front(SelectedPoolSize))
+    TRI->markSuperRegs(WwmRegMask, Reg);
+
+  if (RequiresFullWWMPool && SelectedPoolSize != DesiredPoolSize) {
     // Reserve an arbitrary register and report the error.
-    TRI->markSuperRegs(RegMask, AMDGPU::VGPR0);
+    TRI->markSuperRegs(WwmRegMask, AMDGPU::VGPR0);
     MF.getFunction().getContext().emitError(
         "cannot find enough VGPRs for wwm-regalloc");
   }
+
+  BitVector NonWwmRegMask(WwmRegMask);
+  NonWwmRegMask.flip().clearBitsNotInMask(TRI->getAllVGPRRegMask());
+
+  // The complement set will be the registers for non-wwm (per-thread) vgpr
+  // allocation.
+  FuncInfo->updateNonWWMRegMask(NonWwmRegMask);
 }
 
 bool SILowerSGPRSpillsLegacy::runOnMachineFunction(MachineFunction &MF) {
@@ -465,8 +490,19 @@ bool SILowerSGPRSpills::run(MachineFunction &MF) {
     // To track the IMPLICIT_DEF insertion point for the lane vgprs.
     DenseMap<Register, LaneVGPRInsertPt> LaneVGPRDomInstr;
 
+    // Defer ordinary spills until physical CSR spills have reserved their
+    // lane VGPRs and the WWM allocation pool can be selected.
+    SmallVector<MachineInstr *> OrdinarySGPRSpills;
+    bool HasStrictWWMRegion = false;
+
     for (MachineBasicBlock &MBB : MF) {
       for (MachineInstr &MI : llvm::make_early_inc_range(MBB)) {
+        if (MI.getOpcode() == AMDGPU::ENTER_STRICT_WWM ||
+            MI.getOpcode() == AMDGPU::ENTER_STRICT_WQM) {
+          HasStrictWWMRegion = true;
+          continue;
+        }
+
         if (!TII->isSGPRSpill(MI))
           continue;
 
@@ -500,18 +536,39 @@ bool SILowerSGPRSpills::run(MachineFunction &MF) {
               llvm_unreachable(
                   "failed to spill SGPR to physical VGPR lane when allocated");
           }
-        } else {
-          MachineInstrSpan MIS(&MI, &MBB);
-          if (FuncInfo->allocateSGPRSpillToVGPRLane(MF, FI)) {
-            bool Spilled = TRI->eliminateSGPRToVGPRSpillFrameIndex(
-                MI, FI, nullptr, Indexes, LIS);
-            if (!Spilled)
-              llvm_unreachable(
-                  "failed to spill SGPR to virtual VGPR lane when allocated");
-            SpillFIs.set(FI);
-            updateLaneVGPRDomInstr(FI, &MBB, MIS.begin(), LaneVGPRDomInstr);
-            SpilledToVirtVGPRLanes = true;
-          }
+        } else
+          OrdinarySGPRSpills.push_back(&MI);
+      }
+    }
+
+    // Select candidates once, before ordinary lane lowering creates virtual
+    // VGPRs and changes the number of registers desired for the WWM pool.
+    SmallVector<MCRegister> WWMRegCandidates;
+    // These non-spillable WWM users retain the old all-or-nothing pool policy.
+    const bool RequiresFullWWMPool =
+        HasStrictWWMRegion || isPreallocateSGPRSpillVGPRsEnabled(MF);
+    if (!OrdinarySGPRSpills.empty())
+      WWMRegCandidates = determineRegsForWWMAllocation(MF);
+
+    const bool ShouldLowerOrdinarySpillsToVGPRLanes =
+        RequiresFullWWMPool || !WWMRegCandidates.empty();
+    if (!ShouldLowerOrdinarySpillsToVGPRLanes && !OrdinarySGPRSpills.empty())
+      FuncInfo->setNoWWMPoolSGPRSpillFallback();
+
+    if (ShouldLowerOrdinarySpillsToVGPRLanes) {
+      for (MachineInstr *MI : OrdinarySGPRSpills) {
+        int FI = TII->getNamedOperand(*MI, AMDGPU::OpName::addr)->getIndex();
+        if (FuncInfo->allocateSGPRSpillToVGPRLane(MF, FI)) {
+          MachineBasicBlock *MBB = MI->getParent();
+          MachineInstrSpan MIS(MI, MBB);
+          bool Spilled = TRI->eliminateSGPRToVGPRSpillFrameIndex(
+              *MI, FI, nullptr, Indexes, LIS);
+          if (!Spilled)
+            llvm_unreachable(
+                "failed to spill SGPR to virtual VGPR lane when allocated");
+          SpillFIs.set(FI);
+          updateLaneVGPRDomInstr(FI, MBB, MIS.begin(), LaneVGPRDomInstr);
+          SpilledToVirtVGPRLanes = true;
         }
       }
     }
@@ -538,20 +595,9 @@ bool SILowerSGPRSpills::run(MachineFunction &MF) {
       }
     }
 
-    // Determine the registers for WWM allocation and also compute the register
-    // mask for non-wwm VGPR allocation.
-    if (FuncInfo->getSGPRSpillVGPRs().size()) {
-      BitVector WwmRegMask(TRI->getNumRegs());
-
-      determineRegsForWWMAllocation(MF, WwmRegMask);
-
-      BitVector NonWwmRegMask(WwmRegMask);
-      NonWwmRegMask.flip().clearBitsNotInMask(TRI->getAllVGPRRegMask());
-
-      // The complement set will be the registers for non-wwm (per-thread) vgpr
-      // allocation.
-      FuncInfo->updateNonWWMRegMask(NonWwmRegMask);
-    }
+    // Assign the WWM pool from the pre-selected candidates and compute the
+    // complement mask for per-thread VGPR allocation.
+    assignWWMRegs(MF, WWMRegCandidates, RequiresFullWWMPool);
 
     for (MachineBasicBlock &MBB : MF)
       clearDebugInfoForSpillFIs(MFI, MBB, SpillFIs);
diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
index 59971923e4a5..950019b37a46 100644
--- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
@@ -739,6 +739,7 @@ yaml::SIMachineFunctionInfo::SIMachineFunctionInfo(
       WaveLimiter(MFI.needsWaveLimiter()),
       HasSpilledSGPRs(MFI.hasSpilledSGPRs()),
       HasSpilledVGPRs(MFI.hasSpilledVGPRs()),
+      HasNoWWMPoolSGPRSpillFallback(MFI.hasNoWWMPoolSGPRSpillFallback()),
       NumWaveDispatchSGPRs(MFI.getNumWaveDispatchSGPRs()),
       NumWaveDispatchVGPRs(MFI.getNumWaveDispatchVGPRs()),
       HighBitsOf32BitAddress(MFI.get32BitAddressHighBits()),
@@ -798,6 +799,7 @@ bool SIMachineFunctionInfo::initializeBaseYamlFields(
   WaveLimiter = YamlMFI.WaveLimiter;
   HasSpilledSGPRs = YamlMFI.HasSpilledSGPRs;
   HasSpilledVGPRs = YamlMFI.HasSpilledVGPRs;
+  HasNoWWMPoolSGPRSpillFallback = YamlMFI.HasNoWWMPoolSGPRSpillFallback;
   NumWaveDispatchSGPRs = YamlMFI.NumWaveDispatchSGPRs;
   NumWaveDispatchVGPRs = YamlMFI.NumWaveDispatchVGPRs;
   BytesInStackArgArea = YamlMFI.BytesInStackArgArea;
diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h
index 7374e996837a..2d1b77fc4fb8 100644
--- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h
@@ -272,6 +272,7 @@ struct SIMachineFunctionInfo final : public yaml::MachineFunctionInfo {
   bool WaveLimiter = false;
   bool HasSpilledSGPRs = false;
   bool HasSpilledVGPRs = false;
+  bool HasNoWWMPoolSGPRSpillFallback = false;
   uint16_t NumWaveDispatchSGPRs = 0;
   uint16_t NumWaveDispatchVGPRs = 0;
   uint32_t HighBitsOf32BitAddress = 0;
@@ -334,6 +335,8 @@ template <> struct MappingTraits<SIMachineFunctionInfo> {
     YamlIO.mapOptional("waveLimiter", MFI.WaveLimiter, false);
     YamlIO.mapOptional("hasSpilledSGPRs", MFI.HasSpilledSGPRs, false);
     YamlIO.mapOptional("hasSpilledVGPRs", MFI.HasSpilledVGPRs, false);
+    YamlIO.mapOptional("hasNoWWMPoolSGPRSpillFallback",
+                       MFI.HasNoWWMPoolSGPRSpillFallback, false);
     YamlIO.mapOptional("numWaveDispatchSGPRs", MFI.NumWaveDispatchSGPRs, false);
     YamlIO.mapOptional("numWaveDispatchVGPRs", MFI.NumWaveDispatchVGPRs, false);
     YamlIO.mapOptional("scratchRSrcReg", MFI.ScratchRSrcReg,
@@ -613,6 +616,11 @@ private:
   // frame, so save it here and add it to the RegScavenger later.
   std::optional<int> ScavengeFI;
 
+  // Ordinary SGPR spills fell back to memory because no WWM VGPR pool was
+  // available. This path may require additional nested VGPR scavenging slots
+  // during frame-index elimination.
+  bool HasNoWWMPoolSGPRSpillFallback = false;
+
   // Map each VGPR CSR to the mask needed to save and restore it using block
   // load/store instructions. Only used if the subtarget feature for VGPR block
   // load/store is enabled.
@@ -853,6 +861,11 @@ public:
   int getScavengeFI(MachineFrameInfo &MFI, const SIRegisterInfo &TRI);
   std::optional<int> getOptionalScavengeFI() const { return ScavengeFI; }
 
+  void setNoWWMPoolSGPRSpillFallback() { HasNoWWMPoolSGPRSpillFallback = true; }
+  bool hasNoWWMPoolSGPRSpillFallback() const {
+    return HasNoWWMPoolSGPRSpillFallback;
+  }
+
   unsigned getBytesInStackArgArea() const {
     return BytesInStackArgArea;
   }
diff --git a/llvm/lib/Target/AMDGPU/SIPreAllocateWWMRegs.cpp b/llvm/lib/Target/AMDGPU/SIPreAllocateWWMRegs.cpp
index bf484cef98da..0d9ef77b7056 100644
--- a/llvm/lib/Target/AMDGPU/SIPreAllocateWWMRegs.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPreAllocateWWMRegs.cpp
@@ -35,6 +35,11 @@ static cl::opt<bool>
     EnablePreallocateSGPRSpillVGPRs("amdgpu-prealloc-sgpr-spill-vgprs",
                                     cl::init(false), cl::Hidden);
 
+bool llvm::isPreallocateSGPRSpillVGPRsEnabled(const MachineFunction &MF) {
+  return EnablePreallocateSGPRSpillVGPRs ||
+         MF.getFunction().hasFnAttribute("amdgpu-prealloc-sgpr-spill-vgprs");
+}
+
 namespace {
 
 class SIPreAllocateWWMRegs {
@@ -221,9 +226,7 @@ bool SIPreAllocateWWMRegs::run(MachineFunction &MF) {
   TRI = &TII->getRegisterInfo();
   MRI = &MF.getRegInfo();
 
-  bool PreallocateSGPRSpillVGPRs =
-      EnablePreallocateSGPRSpillVGPRs ||
-      MF.getFunction().hasFnAttribute("amdgpu-prealloc-sgpr-spill-vgprs");
+  bool PreallocateSGPRSpillVGPRs = isPreallocateSGPRSpillVGPRsEnabled(MF);
 
   bool RegsAssigned = false;
 
diff --git a/llvm/lib/Target/AMDGPU/SIPreAllocateWWMRegs.h b/llvm/lib/Target/AMDGPU/SIPreAllocateWWMRegs.h
index 8d3fa2103596..2fcd9f9cba35 100644
--- a/llvm/lib/Target/AMDGPU/SIPreAllocateWWMRegs.h
+++ b/llvm/lib/Target/AMDGPU/SIPreAllocateWWMRegs.h
@@ -13,6 +13,8 @@
 
 namespace llvm {
 
+bool isPreallocateSGPRSpillVGPRsEnabled(const MachineFunction &MF);
+
 class SIPreAllocateWWMRegsPass
     : public RequiredPassInfoMixin<SIPreAllocateWWMRegsPass> {
 public:
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg-crash.ll b/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg-crash.ll
index e9bb77f3db5d..30202943afa8 100644
--- a/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg-crash.ll
+++ b/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg-crash.ll
@@ -16,7 +16,10 @@
                      i64 ; vcc
                      }
 
-; ERR-GCNTRACKERS: ran out of registers during register allocation
+; With the tracker enabled, no separate WWM VGPR is available and the SGPR
+; spill falls back to memory. This case cannot preserve EXEC because SCC is
+; live and no SGPR can be scavenged.
+; ERR-GCNTRACKERS: unhandled SGPR spill to memory
 ; GCN-NOT: ran out of registers during register allocation
 
 ; FIXME: GCN Trackers do not track pressure from PhysRegs, so scheduling is actually worse
@@ -62,4 +65,3 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs() #0
 
 attributes #0 = { nounwind alignstack=64 "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="10,10" "no-realign-stack" }
 attributes #1 = { nounwind alignstack=16 "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="10,10" "no-realign-stack" }
-
diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-spill-vmem-large-frame.mir b/llvm/test/CodeGen/AMDGPU/sgpr-spill-vmem-large-frame.mir
index b23aa0f45a05..ccb963d7fa8a 100644
--- a/llvm/test/CodeGen/AMDGPU/sgpr-spill-vmem-large-frame.mir
+++ b/llvm/test/CodeGen/AMDGPU/sgpr-spill-vmem-large-frame.mir
@@ -4,6 +4,10 @@
 # Check that we allocate 2 emergency stack slots if we're spilling
 # SGPRs to memory and potentially have an offset larger than fits in
 # the addressing mode of the memory instructions.
+#
+# The no-WWM-pool fallback needs 3 emergency slots even for a small frame:
+# one for an ordinary frame-index scavenge, one for the SGPR spill temporary,
+# and one for recursive address materialization.
 
 ---
 name:            test
@@ -49,3 +53,35 @@ body:             |
     renamable $sgpr10 = SI_SPILL_S32_RESTORE %stack.0, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32
     S_SETPC_B64 $sgpr30_sgpr31, implicit $scc
 ...
+---
+name:            fallback_scavenging
+tracksRegLiveness: true
+frameInfo:
+  maxAlignment:    4
+stack:
+  - { id: 0, type: spill-slot, size: 4, alignment: 4, stack-id: sgpr-spill }
+machineFunctionInfo:
+  isEntryFunction: false
+  scratchRSrcReg:  '$sgpr0_sgpr1_sgpr2_sgpr3'
+  stackPtrOffsetReg: '$sgpr32'
+  frameOffsetReg: '$sgpr33'
+  hasSpilledSGPRs: true
+  hasNoWWMPoolSGPRSpillFallback: true
+body:             |
+  bb.0:
+    liveins: $sgpr30_sgpr31, $sgpr10, $sgpr11
+    ; CHECK-LABEL: name: fallback_scavenging
+    ; CHECK: frameInfo:
+    ; CHECK: stackSize: 16
+    ; CHECK: stack:
+    ; CHECK: - { id: 0,
+    ; CHECK: - { id: 1,
+    ; CHECK: - { id: 2,
+    ; CHECK: - { id: 3,
+    ; CHECK: machineFunctionInfo:
+    ; CHECK: hasNoWWMPoolSGPRSpillFallback: true
+    S_CMP_EQ_U32 0, 0, implicit-def $scc
+    SI_SPILL_S32_SAVE killed $sgpr10, %stack.0, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32
+    renamable $sgpr10 = SI_SPILL_S32_RESTORE %stack.0, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32
+    S_SETPC_B64 $sgpr30_sgpr31, implicit $scc
+...
diff --git a/llvm/test/CodeGen/AMDGPU/wwm-regalloc-error.ll b/llvm/test/CodeGen/AMDGPU/wwm-regalloc-error.ll
index 2367af90d955..557c04a9ae38 100644
--- a/llvm/test/CodeGen/AMDGPU/wwm-regalloc-error.ll
+++ b/llvm/test/CodeGen/AMDGPU/wwm-regalloc-error.ll
@@ -1,8 +1,22 @@
-; RUN: not llc -mtriple=amdgpu9.00-amd-amdhsa -stress-regalloc=2 -filetype=null %s 2>&1 | FileCheck %s
+; RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -stress-regalloc=2 -o - %s | FileCheck %s
+; RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -stress-regalloc=2 -stop-after=si-lower-sgpr-spills -o %t.mir %s
+; RUN: FileCheck --check-prefix=MIR %s < %t.mir
+; RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -stress-regalloc=2 -start-after=si-lower-sgpr-spills -o - %t.mir | FileCheck --check-prefix=ROUNDTRIP %s
 
-; A negative test to capture the expected error when the VGPRs are insufficient for wwm-regalloc.
+; All allocatable VGPRs are mentioned by inline assembly. Ordinary SGPR spills
+; must fall back to scratch memory when no separate WWM pool is available.
 
-; CHECK: error: cannot find enough VGPRs for wwm-regalloc
+; CHECK-LABEL: test:
+; CHECK: buffer_store_dword {{.*}} ; 4-byte Folded Spill
+; CHECK: buffer_load_dword {{.*}} ; 4-byte Folded Reload
+; CHECK: s_endpgm
+; CHECK: .amdhsa_private_segment_fixed_size 20
+
+; MIR: hasNoWWMPoolSGPRSpillFallback: true
+
+; ROUNDTRIP-LABEL: test:
+; ROUNDTRIP: s_endpgm
+; ROUNDTRIP: .amdhsa_private_segment_fixed_size 20
 
 define amdgpu_kernel void @test(i32 %in) {
 entry:
diff --git a/llvm/test/CodeGen/AMDGPU/wwm-regalloc-memory-fallback.ll b/llvm/test/CodeGen/AMDGPU/wwm-regalloc-memory-fallback.ll
new file mode 100644
index 000000000000..fa30f46dc83c
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/wwm-regalloc-memory-fallback.ll
@@ -0,0 +1,213 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck %s
+
+; All allocatable VGPRs are clobbered while several large SGPR values remain
+; live. Verify end-to-end that SGPR spills use scratch when no WWM VGPR pool is
+; available, without relying on register-allocation stress options.
+
+define amdgpu_kernel void @repro(i32 %in) {
+; CHECK-LABEL: repro:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_mov_b64 s[70:71], s[2:3]
+; CHECK-NEXT:    s_mov_b64 s[68:69], s[0:1]
+; CHECK-NEXT:    s_load_dword s0, s[8:9], 0x0
+; CHECK-NEXT:    s_add_u32 s68, s68, s17
+; CHECK-NEXT:    s_addc_u32 s69, s69, 0
+; CHECK-NEXT:    s_mov_b64 s[2:3], exec
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    s_mov_b64 exec, 0xffffffff
+; CHECK-NEXT:    buffer_store_dword v0, off, s[68:71], 0 offset:256
+; CHECK-NEXT:    v_writelane_b32 v0, s36, 0
+; CHECK-NEXT:    v_writelane_b32 v0, s37, 1
+; CHECK-NEXT:    v_writelane_b32 v0, s38, 2
+; CHECK-NEXT:    v_writelane_b32 v0, s39, 3
+; CHECK-NEXT:    v_writelane_b32 v0, s40, 4
+; CHECK-NEXT:    v_writelane_b32 v0, s41, 5
+; CHECK-NEXT:    v_writelane_b32 v0, s42, 6
+; CHECK-NEXT:    v_writelane_b32 v0, s43, 7
+; CHECK-NEXT:    v_writelane_b32 v0, s44, 8
+; CHECK-NEXT:    v_writelane_b32 v0, s45, 9
+; CHECK-NEXT:    v_writelane_b32 v0, s46, 10
+; CHECK-NEXT:    v_writelane_b32 v0, s47, 11
+; CHECK-NEXT:    v_writelane_b32 v0, s48, 12
+; CHECK-NEXT:    v_writelane_b32 v0, s49, 13
+; CHECK-NEXT:    v_writelane_b32 v0, s50, 14
+; CHECK-NEXT:    v_writelane_b32 v0, s51, 15
+; CHECK-NEXT:    v_writelane_b32 v0, s52, 16
+; CHECK-NEXT:    v_writelane_b32 v0, s53, 17
+; CHECK-NEXT:    v_writelane_b32 v0, s54, 18
+; CHECK-NEXT:    v_writelane_b32 v0, s55, 19
+; CHECK-NEXT:    v_writelane_b32 v0, s56, 20
+; CHECK-NEXT:    v_writelane_b32 v0, s57, 21
+; CHECK-NEXT:    v_writelane_b32 v0, s58, 22
+; CHECK-NEXT:    v_writelane_b32 v0, s59, 23
+; CHECK-NEXT:    v_writelane_b32 v0, s60, 24
+; CHECK-NEXT:    v_writelane_b32 v0, s61, 25
+; CHECK-NEXT:    v_writelane_b32 v0, s62, 26
+; CHECK-NEXT:    v_writelane_b32 v0, s63, 27
+; CHECK-NEXT:    v_writelane_b32 v0, s64, 28
+; CHECK-NEXT:    v_writelane_b32 v0, s65, 29
+; CHECK-NEXT:    v_writelane_b32 v0, s66, 30
+; CHECK-NEXT:    v_writelane_b32 v0, s67, 31
+; CHECK-NEXT:    buffer_store_dword v0, off, s[68:71], 0 ; 4-byte Folded Spill
+; CHECK-NEXT:    buffer_load_dword v0, off, s[68:71], 0 offset:256
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    s_mov_b64 exec, s[2:3]
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    s_cmp_lg_u32 s0, 0
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    s_cbranch_scc0 .LBB0_2
+; CHECK-NEXT:  ; %bb.1: ; %exit
+; CHECK-NEXT:    s_endpgm
+; CHECK-NEXT:  .LBB0_2: ; %use
+; CHECK-NEXT:    s_mov_b64 s[34:35], exec
+; CHECK-NEXT:    s_mov_b64 exec, 0xffffffff
+; CHECK-NEXT:    buffer_store_dword v0, off, s[68:71], 0 offset:256
+; CHECK-NEXT:    v_writelane_b32 v0, s36, 0
+; CHECK-NEXT:    v_writelane_b32 v0, s37, 1
+; CHECK-NEXT:    v_writelane_b32 v0, s38, 2
+; CHECK-NEXT:    v_writelane_b32 v0, s39, 3
+; CHECK-NEXT:    v_writelane_b32 v0, s40, 4
+; CHECK-NEXT:    v_writelane_b32 v0, s41, 5
+; CHECK-NEXT:    v_writelane_b32 v0, s42, 6
+; CHECK-NEXT:    v_writelane_b32 v0, s43, 7
+; CHECK-NEXT:    v_writelane_b32 v0, s44, 8
+; CHECK-NEXT:    v_writelane_b32 v0, s45, 9
+; CHECK-NEXT:    v_writelane_b32 v0, s46, 10
+; CHECK-NEXT:    v_writelane_b32 v0, s47, 11
+; CHECK-NEXT:    v_writelane_b32 v0, s48, 12
+; CHECK-NEXT:    v_writelane_b32 v0, s49, 13
+; CHECK-NEXT:    v_writelane_b32 v0, s50, 14
+; CHECK-NEXT:    v_writelane_b32 v0, s51, 15
+; CHECK-NEXT:    v_writelane_b32 v0, s52, 16
+; CHECK-NEXT:    v_writelane_b32 v0, s53, 17
+; CHECK-NEXT:    v_writelane_b32 v0, s54, 18
+; CHECK-NEXT:    v_writelane_b32 v0, s55, 19
+; CHECK-NEXT:    v_writelane_b32 v0, s56, 20
+; CHECK-NEXT:    v_writelane_b32 v0, s57, 21
+; CHECK-NEXT:    v_writelane_b32 v0, s58, 22
+; CHECK-NEXT:    v_writelane_b32 v0, s59, 23
+; CHECK-NEXT:    v_writelane_b32 v0, s60, 24
+; CHECK-NEXT:    v_writelane_b32 v0, s61, 25
+; CHECK-NEXT:    v_writelane_b32 v0, s62, 26
+; CHECK-NEXT:    v_writelane_b32 v0, s63, 27
+; CHECK-NEXT:    v_writelane_b32 v0, s64, 28
+; CHECK-NEXT:    v_writelane_b32 v0, s65, 29
+; CHECK-NEXT:    v_writelane_b32 v0, s66, 30
+; CHECK-NEXT:    v_writelane_b32 v0, s67, 31
+; CHECK-NEXT:    buffer_store_dword v0, off, s[68:71], 0 offset:128 ; 4-byte Folded Spill
+; CHECK-NEXT:    buffer_load_dword v0, off, s[68:71], 0 offset:256
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    s_mov_b64 exec, s[34:35]
+; CHECK-NEXT:    s_mov_b64 s[34:35], exec
+; CHECK-NEXT:    s_mov_b64 exec, 0xffffffff
+; CHECK-NEXT:    buffer_store_dword v0, off, s[68:71], 0 offset:256
+; CHECK-NEXT:    buffer_load_dword v0, off, s[68:71], 0 ; 4-byte Folded Reload
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    v_readlane_b32 s36, v0, 0
+; CHECK-NEXT:    v_readlane_b32 s37, v0, 1
+; CHECK-NEXT:    v_readlane_b32 s38, v0, 2
+; CHECK-NEXT:    v_readlane_b32 s39, v0, 3
+; CHECK-NEXT:    v_readlane_b32 s40, v0, 4
+; CHECK-NEXT:    v_readlane_b32 s41, v0, 5
+; CHECK-NEXT:    v_readlane_b32 s42, v0, 6
+; CHECK-NEXT:    v_readlane_b32 s43, v0, 7
+; CHECK-NEXT:    v_readlane_b32 s44, v0, 8
+; CHECK-NEXT:    v_readlane_b32 s45, v0, 9
+; CHECK-NEXT:    v_readlane_b32 s46, v0, 10
+; CHECK-NEXT:    v_readlane_b32 s47, v0, 11
+; CHECK-NEXT:    v_readlane_b32 s48, v0, 12
+; CHECK-NEXT:    v_readlane_b32 s49, v0, 13
+; CHECK-NEXT:    v_readlane_b32 s50, v0, 14
+; CHECK-NEXT:    v_readlane_b32 s51, v0, 15
+; CHECK-NEXT:    v_readlane_b32 s52, v0, 16
+; CHECK-NEXT:    v_readlane_b32 s53, v0, 17
+; CHECK-NEXT:    v_readlane_b32 s54, v0, 18
+; CHECK-NEXT:    v_readlane_b32 s55, v0, 19
+; CHECK-NEXT:    v_readlane_b32 s56, v0, 20
+; CHECK-NEXT:    v_readlane_b32 s57, v0, 21
+; CHECK-NEXT:    v_readlane_b32 s58, v0, 22
+; CHECK-NEXT:    v_readlane_b32 s59, v0, 23
+; CHECK-NEXT:    v_readlane_b32 s60, v0, 24
+; CHECK-NEXT:    v_readlane_b32 s61, v0, 25
+; CHECK-NEXT:    v_readlane_b32 s62, v0, 26
+; CHECK-NEXT:    v_readlane_b32 s63, v0, 27
+; CHECK-NEXT:    v_readlane_b32 s64, v0, 28
+; CHECK-NEXT:    v_readlane_b32 s65, v0, 29
+; CHECK-NEXT:    v_readlane_b32 s66, v0, 30
+; CHECK-NEXT:    v_readlane_b32 s67, v0, 31
+; CHECK-NEXT:    buffer_load_dword v0, off, s[68:71], 0 offset:256
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    s_mov_b64 exec, s[34:35]
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    s_mov_b64 s[34:35], exec
+; CHECK-NEXT:    s_mov_b64 exec, 0xffffffff
+; CHECK-NEXT:    buffer_store_dword v0, off, s[68:71], 0 offset:256
+; CHECK-NEXT:    buffer_load_dword v0, off, s[68:71], 0 offset:128 ; 4-byte Folded Reload
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    v_readlane_b32 s0, v0, 0
+; CHECK-NEXT:    v_readlane_b32 s1, v0, 1
+; CHECK-NEXT:    v_readlane_b32 s2, v0, 2
+; CHECK-NEXT:    v_readlane_b32 s3, v0, 3
+; CHECK-NEXT:    v_readlane_b32 s4, v0, 4
+; CHECK-NEXT:    v_readlane_b32 s5, v0, 5
+; CHECK-NEXT:    v_readlane_b32 s6, v0, 6
+; CHECK-NEXT:    v_readlane_b32 s7, v0, 7
+; CHECK-NEXT:    v_readlane_b32 s8, v0, 8
+; CHECK-NEXT:    v_readlane_b32 s9, v0, 9
+; CHECK-NEXT:    v_readlane_b32 s10, v0, 10
+; CHECK-NEXT:    v_readlane_b32 s11, v0, 11
+; CHECK-NEXT:    v_readlane_b32 s12, v0, 12
+; CHECK-NEXT:    v_readlane_b32 s13, v0, 13
+; CHECK-NEXT:    v_readlane_b32 s14, v0, 14
+; CHECK-NEXT:    v_readlane_b32 s15, v0, 15
+; CHECK-NEXT:    v_readlane_b32 s16, v0, 16
+; CHECK-NEXT:    v_readlane_b32 s17, v0, 17
+; CHECK-NEXT:    v_readlane_b32 s18, v0, 18
+; CHECK-NEXT:    v_readlane_b32 s19, v0, 19
+; CHECK-NEXT:    v_readlane_b32 s20, v0, 20
+; CHECK-NEXT:    v_readlane_b32 s21, v0, 21
+; CHECK-NEXT:    v_readlane_b32 s22, v0, 22
+; CHECK-NEXT:    v_readlane_b32 s23, v0, 23
+; CHECK-NEXT:    v_readlane_b32 s24, v0, 24
+; CHECK-NEXT:    v_readlane_b32 s25, v0, 25
+; CHECK-NEXT:    v_readlane_b32 s26, v0, 26
+; CHECK-NEXT:    v_readlane_b32 s27, v0, 27
+; CHECK-NEXT:    v_readlane_b32 s28, v0, 28
+; CHECK-NEXT:    v_readlane_b32 s29, v0, 29
+; CHECK-NEXT:    v_readlane_b32 s30, v0, 30
+; CHECK-NEXT:    v_readlane_b32 s31, v0, 31
+; CHECK-NEXT:    buffer_load_dword v0, off, s[68:71], 0 offset:256
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    s_mov_b64 exec, s[34:35]
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    s_endpgm
+entry:
+  call void asm sideeffect "", "~{v[0:7]},~{v[8:15]},~{v[16:23]},~{v[24:31]},~{v[32:39]},~{v[40:47]},~{v[48:55]},~{v[56:63]}" ()
+
+  %s0 = call <32 x i32> asm sideeffect "", "=s" ()
+  %s1 = call <32 x i32> asm sideeffect "", "=s" ()
+  %s2 = call <32 x i32> asm sideeffect "", "=s" ()
+
+  %cmp = icmp eq i32 %in, 0
+  br i1 %cmp, label %use, label %exit
+
+use:
+  call void asm sideeffect "", "s"(<32 x i32> %s0)
+  call void asm sideeffect "", "s"(<32 x i32> %s1)
+  call void asm sideeffect "", "s"(<32 x i32> %s2)
+  br label %exit
+
+exit:
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/wwm-regalloc-partial-pool.mir b/llvm/test/CodeGen/AMDGPU/wwm-regalloc-partial-pool.mir
new file mode 100644
index 000000000000..6c79c594771b
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/wwm-regalloc-partial-pool.mir
@@ -0,0 +1,50 @@
+# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -O0 -amdgpu-stress-vgpr=1 -amdgpu-num-vgprs-for-wwm-alloc=2 -start-before=si-lower-sgpr-spills -stop-after=regallocfast,1 -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -O0 -amdgpu-num-vgprs-for-wwm-alloc=0 -start-before=si-lower-sgpr-spills -stop-after=regallocfast,1 -verify-machineinstrs -o - %s | FileCheck --check-prefix=ZERO %s
+
+# CHECK-LABEL: name: partial_pool
+# CHECK-COUNT-2: flags: [ WWM_REG ]
+# CHECK: renamable $vgpr0 = IMPLICIT_DEF
+# CHECK: SI_SPILL_WWM_V32_SAVE $vgpr0
+# CHECK: $vgpr0 = SI_SPILL_WWM_V32_RESTORE
+
+# With an explicitly empty pool, otherwise available VGPRs must remain for the
+# per-thread allocator and the SGPR spill pseudos must remain on the memory path.
+# ZERO-LABEL: name: partial_pool
+# ZERO-NOT: WWM_REG
+# ZERO: SI_SPILL_S1024_SAVE
+# ZERO: SI_SPILL_S32_SAVE
+
+--- |
+  define amdgpu_kernel void @partial_pool() #0 {
+    ret void
+  }
+
+  attributes #0 = { "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" }
+...
+---
+name:            partial_pool
+tracksRegLiveness: true
+frameInfo:
+  maxAlignment:    4
+stack:
+  - { id: 0, type: spill-slot, size: 128, alignment: 4, stack-id: sgpr-spill }
+  - { id: 1, type: spill-slot, size: 128, alignment: 4, stack-id: sgpr-spill }
+  - { id: 2, type: spill-slot, size: 4, alignment: 4, stack-id: sgpr-spill }
+machineFunctionInfo:
+  isEntryFunction: true
+  scratchRSrcReg:  '$sgpr0_sgpr1_sgpr2_sgpr3'
+  stackPtrOffsetReg: '$sgpr32'
+  frameOffsetReg: '$sgpr33'
+  hasSpilledSGPRs: true
+body:             |
+  bb.0:
+    liveins: $sgpr34, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99
+
+    SI_SPILL_S1024_SAVE killed $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, %stack.0, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32
+    SI_SPILL_S1024_SAVE killed $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99, %stack.1, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32
+    SI_SPILL_S32_SAVE killed $sgpr34, %stack.2, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32
+    renamable $sgpr34 = SI_SPILL_S32_RESTORE %stack.2, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32
+    renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.1, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32
+    renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67 = SI_SPILL_S1024_RESTORE %stack.0, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32
+    S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/wwm-regalloc-preallocation-guard.mir b/llvm/test/CodeGen/AMDGPU/wwm-regalloc-preallocation-guard.mir
new file mode 100644
index 000000000000..f75565f4f0eb
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/wwm-regalloc-preallocation-guard.mir
@@ -0,0 +1,101 @@
+# RUN: split-file %s %t
+# RUN: not llc -mtriple=amdgpu9.00-amd-amdhsa -amdgpu-stress-vgpr=1 -amdgpu-num-vgprs-for-wwm-alloc=2 -run-pass=si-lower-sgpr-spills -o /dev/null %t/strict.mir 2>&1 | FileCheck %s
+# RUN: not llc -mtriple=amdgpu9.00-amd-amdhsa -amdgpu-stress-vgpr=1 -amdgpu-num-vgprs-for-wwm-alloc=2 -amdgpu-prealloc-sgpr-spill-vgprs -run-pass=si-lower-sgpr-spills -o /dev/null %t/option.mir 2>&1 | FileCheck %s
+# RUN: not llc -mtriple=amdgpu9.00-amd-amdhsa -amdgpu-stress-vgpr=1 -amdgpu-num-vgprs-for-wwm-alloc=2 -run-pass=si-lower-sgpr-spills -o /dev/null %t/attribute.mir 2>&1 | FileCheck %s
+
+# CHECK: error: cannot find enough VGPRs for wwm-regalloc
+
+#--- strict.mir
+--- |
+  define amdgpu_kernel void @strict() #0 {
+    ret void
+  }
+  attributes #0 = { "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" }
+...
+---
+name:            strict
+tracksRegLiveness: true
+frameInfo:
+  maxAlignment:    4
+stack:
+  - { id: 0, type: spill-slot, size: 128, alignment: 4, stack-id: sgpr-spill }
+  - { id: 1, type: spill-slot, size: 128, alignment: 4, stack-id: sgpr-spill }
+  - { id: 2, type: spill-slot, size: 4, alignment: 4, stack-id: sgpr-spill }
+machineFunctionInfo:
+  isEntryFunction: true
+  scratchRSrcReg:  '$sgpr0_sgpr1_sgpr2_sgpr3'
+  stackPtrOffsetReg: '$sgpr32'
+  frameOffsetReg: '$sgpr33'
+  hasSpilledSGPRs: true
+body:             |
+  bb.0:
+    liveins: $sgpr34, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99
+    %0:sreg_64 = ENTER_STRICT_WWM -1, implicit-def $exec, implicit-def $scc, implicit $exec
+    $exec = EXIT_STRICT_WWM killed %0
+    SI_SPILL_S1024_SAVE killed $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, %stack.0, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32
+    SI_SPILL_S1024_SAVE killed $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99, %stack.1, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32
+    SI_SPILL_S32_SAVE killed $sgpr34, %stack.2, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32
+    S_ENDPGM 0
+...
+
+#--- option.mir
+--- |
+  define amdgpu_kernel void @option() #0 {
+    ret void
+  }
+  attributes #0 = { "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" }
+...
+---
+name:            option
+tracksRegLiveness: true
+frameInfo:
+  maxAlignment:    4
+stack:
+  - { id: 0, type: spill-slot, size: 128, alignment: 4, stack-id: sgpr-spill }
+  - { id: 1, type: spill-slot, size: 128, alignment: 4, stack-id: sgpr-spill }
+  - { id: 2, type: spill-slot, size: 4, alignment: 4, stack-id: sgpr-spill }
+machineFunctionInfo:
+  isEntryFunction: true
+  scratchRSrcReg:  '$sgpr0_sgpr1_sgpr2_sgpr3'
+  stackPtrOffsetReg: '$sgpr32'
+  frameOffsetReg: '$sgpr33'
+  hasSpilledSGPRs: true
+body:             |
+  bb.0:
+    liveins: $sgpr34, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99
+    SI_SPILL_S1024_SAVE killed $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, %stack.0, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32
+    SI_SPILL_S1024_SAVE killed $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99, %stack.1, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32
+    SI_SPILL_S32_SAVE killed $sgpr34, %stack.2, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32
+    S_ENDPGM 0
+...
+
+#--- attribute.mir
+--- |
+  define amdgpu_kernel void @attribute() #0 {
+    ret void
+  }
+  attributes #0 = { "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-prealloc-sgpr-spill-vgprs" }
+...
+---
+name:            attribute
+tracksRegLiveness: true
+frameInfo:
+  maxAlignment:    4
+stack:
+  - { id: 0, type: spill-slot, size: 128, alignment: 4, stack-id: sgpr-spill }
+  - { id: 1, type: spill-slot, size: 128, alignment: 4, stack-id: sgpr-spill }
+  - { id: 2, type: spill-slot, size: 4, alignment: 4, stack-id: sgpr-spill }
+machineFunctionInfo:
+  isEntryFunction: true
+  scratchRSrcReg:  '$sgpr0_sgpr1_sgpr2_sgpr3'
+  stackPtrOffsetReg: '$sgpr32'
+  frameOffsetReg: '$sgpr33'
+  hasSpilledSGPRs: true
+body:             |
+  bb.0:
+    liveins: $sgpr34, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99
+    SI_SPILL_S1024_SAVE killed $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, %stack.0, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32
+    SI_SPILL_S1024_SAVE killed $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99, %stack.1, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32
+    SI_SPILL_S32_SAVE killed $sgpr34, %stack.2, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32
+    S_ENDPGM 0
+...