From 2e81492b8a770ebe927ca2bf0b609b80e0713b66 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Fri, 31 Jul 2026 19:04:36 -0700 Subject: [PATCH] [BFI] Simplify irreducible header discovery. NFC (#213213) IrreducibleGraph is the explicit graph of a region -- a loop, or the whole function -- that BFI searches for irreducible SCCs. It maintains predecessor lists only so findIrreducibleHeaders can ask of each node whether a predecessor lies outside its SCC (an entry), and whether a non-entry in the same SCC reaches it via a backedge (an extra header). Answer both from the successor lists instead; IrrNode then needs only a successor vector. findIrreducibleHeaders is left partitioning the SCC its sole caller is packaging, so fold it into createIrreducibleLoop. Headers and members are sorted, so the changed iteration order does not affect output. Once BFI uses CycleInfo, a region's irreducible SCCs are its non-reducible child cycles, which should let IrreducibleGraph go away entirely. That removal has no predecessor lists to walk and getResolvedNode has no inverse, so it must sweep successors regardless; settling the formulation here leaves it changing only the graph. Aided by Claude Opus 5 --- .../llvm/Analysis/BlockFrequencyInfoImpl.h | 14 +- llvm/lib/Analysis/BlockFrequencyInfoImpl.cpp | 151 ++++++++---------- 2 files changed, 70 insertions(+), 95 deletions(-) diff --git a/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h b/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h index 328db2e99d34..ced61b26f710 100644 --- a/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h +++ b/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h @@ -589,23 +589,23 @@ struct IrreducibleGraph { using BlockNode = BFIBase::BlockNode; struct IrrNode { BlockNode Node; - unsigned NumIn = 0; - std::deque Edges; + SmallVector Succs; IrrNode(const BlockNode &Node) : Node(Node) {} - using iterator = std::deque::const_iterator; + using iterator = SmallVectorImpl::const_iterator; - iterator pred_begin() const { return Edges.begin(); } - iterator succ_begin() const { return Edges.begin() + NumIn; } - iterator pred_end() const { return succ_begin(); } - iterator succ_end() const { return Edges.end(); } + iterator succ_begin() const { return Succs.begin(); } + iterator succ_end() const { return Succs.end(); } }; BlockNode Start; const IrrNode *StartIrr = nullptr; std::vector Nodes; SmallDenseMap Lookup; + /// The position of \p N in \a Nodes, for indexing side tables. + unsigned getIndex(const IrrNode *N) const { return N - Nodes.data(); } + /// Construct an explicit graph containing irreducible control flow. /// /// Construct an explicit graph of the control flow in \c OuterLoop (or the diff --git a/llvm/lib/Analysis/BlockFrequencyInfoImpl.cpp b/llvm/lib/Analysis/BlockFrequencyInfoImpl.cpp index 4680af8f50c2..011435db6678 100644 --- a/llvm/lib/Analysis/BlockFrequencyInfoImpl.cpp +++ b/llvm/lib/Analysis/BlockFrequencyInfoImpl.cpp @@ -660,9 +660,7 @@ void IrreducibleGraph::addEdge(IrrNode &Irr, const BlockNode &Succ, if (L == Lookup.end()) return; IrrNode &SuccIrr = *L->second; - Irr.Edges.push_back(&SuccIrr); - SuccIrr.Edges.push_front(&Irr); - ++SuccIrr.NumIn; + Irr.Succs.push_back(&SuccIrr); } namespace llvm { @@ -679,89 +677,42 @@ template <> struct GraphTraits { } // end namespace llvm -/// Find extra irreducible headers. -/// -/// Find entry blocks and other blocks with backedges, which exist when \c G -/// contains irreducible sub-SCCs. -static void findIrreducibleHeaders( - const BlockFrequencyInfoImplBase &BFI, - const IrreducibleGraph &G, - const std::vector &SCC, - LoopData::NodeList &Headers, LoopData::NodeList &Others) { - // Map from nodes in the SCC to whether it's an entry block. - SmallDenseMap InSCC; - - // InSCC also acts the set of nodes in the graph. Seed it. - for (const auto *I : SCC) - InSCC[I] = false; - - for (auto I = InSCC.begin(), E = InSCC.end(); I != E; ++I) { - auto &Irr = *I->first; - for (const auto *P : make_range(Irr.pred_begin(), Irr.pred_end())) { - if (InSCC.count(P)) - continue; - - // This is an entry block. - I->second = true; - Headers.push_back(Irr.Node); - LLVM_DEBUG(dbgs() << " => entry = " << BFI.getBlockName(Irr.Node) - << "\n"); - break; - } - } - assert(Headers.size() >= 2 && - "Expected irreducible CFG; -loop-info is likely invalid"); - if (Headers.size() == InSCC.size()) { - // Every block is a header. - llvm::sort(Headers); - return; - } - - // Look for extra headers from irreducible sub-SCCs. - for (const auto &I : InSCC) { - // Entry blocks are already headers. - if (I.second) - continue; - - auto &Irr = *I.first; - for (const auto *P : make_range(Irr.pred_begin(), Irr.pred_end())) { - // Skip forward edges. - if (P->Node < Irr.Node) - continue; - - // Skip predecessors from entry blocks. These can have inverted - // ordering. - if (InSCC.lookup(P)) - continue; - - // Store the extra header. - Headers.push_back(Irr.Node); - LLVM_DEBUG(dbgs() << " => extra = " << BFI.getBlockName(Irr.Node) - << "\n"); - break; - } - if (Headers.back() == Irr.Node) - // Added this as a header. - continue; - - // This is not a header. - Others.push_back(Irr.Node); - LLVM_DEBUG(dbgs() << " => other = " << BFI.getBlockName(Irr.Node) << "\n"); - } - llvm::sort(Headers); - llvm::sort(Others); -} - -static void createIrreducibleLoop( - BlockFrequencyInfoImplBase &BFI, const IrreducibleGraph &G, - LoopData *OuterLoop, std::list::iterator Insert, - const std::vector &SCC) { - // Translate the SCC into RPO. +/// Package \c SCC into a loop, headed by the nodes marked in \c IsEntry or +/// \c Extra. +static void +createIrreducibleLoop(BlockFrequencyInfoImplBase &BFI, + const IrreducibleGraph &G, LoopData *OuterLoop, + std::list::iterator Insert, + ArrayRef SCC, + const BitVector &IsEntry, const BitVector &Extra) { LLVM_DEBUG(dbgs() << " - found-scc\n"); LoopData::NodeList Headers; LoopData::NodeList Others; - findIrreducibleHeaders(BFI, G, SCC, Headers, Others); + for (const auto *I : SCC) + if (IsEntry.test(G.getIndex(I))) { + Headers.push_back(I->Node); + LLVM_DEBUG(dbgs() << " => entry = " << BFI.getBlockName(I->Node) + << "\n"); + } + assert(Headers.size() >= 2 && + "Expected irreducible CFG; -loop-info is likely invalid"); + + for (const auto *I : SCC) { + if (IsEntry.test(G.getIndex(I))) + continue; + if (Extra.test(G.getIndex(I))) { + Headers.push_back(I->Node); + LLVM_DEBUG(dbgs() << " => extra = " << BFI.getBlockName(I->Node) + << "\n"); + } else { + Others.push_back(I->Node); + LLVM_DEBUG(dbgs() << " => other = " << BFI.getBlockName(I->Node) + << "\n"); + } + } + llvm::sort(Headers); + llvm::sort(Others); auto Loop = BFI.Loops.emplace(Insert, OuterLoop, Headers.begin(), Headers.end(), Others.begin(), Others.end()); @@ -781,14 +732,38 @@ BlockFrequencyInfoImplBase::analyzeIrreducible( assert((OuterLoop == nullptr) == (Insert == Loops.begin())); auto Prev = OuterLoop ? std::prev(Insert) : Loops.end(); - for (auto I = scc_begin(G); !I.isAtEnd(); ++I) { - if (I->size() < 2) - continue; - - // Translate the SCC into RPO. - createIrreducibleLoop(*this, G, OuterLoop, Insert, *I); + // Number every node's SCC, as the sweeps below compare an edge's two ends. + // Only multi-node SCCs become loops, so keep just their members. + SmallVector SccId(G.Nodes.size(), ~0u); + SmallVector> SCCs; + unsigned Id = 0; + for (auto I = scc_begin(G); !I.isAtEnd(); ++I, ++Id) { + for (const auto *N : *I) + SccId[G.getIndex(N)] = Id; + if (I->size() >= 2) + SCCs.emplace_back(I->begin(), I->end()); } + // A node is an entry if an edge from another SCC reaches it, and an extra + // header if a backedge within its SCC targets it. Backedges from entries + // can have inverted ordering, so they do not make a header. + BitVector IsEntry(G.Nodes.size()); + BitVector Extra(G.Nodes.size()); + for (const auto &U : G.Nodes) + for (const auto *V : U.Succs) + if (SccId[G.getIndex(&U)] != SccId[G.getIndex(V)]) + IsEntry.set(G.getIndex(V)); + for (const auto &U : G.Nodes) { + if (IsEntry.test(G.getIndex(&U))) + continue; + for (const auto *V : U.Succs) + if (SccId[G.getIndex(V)] == SccId[G.getIndex(&U)] && !(U.Node < V->Node)) + Extra.set(G.getIndex(V)); + } + + for (const auto &SCC : SCCs) + createIrreducibleLoop(*this, G, OuterLoop, Insert, SCC, IsEntry, Extra); + if (OuterLoop) return make_range(std::next(Prev), Insert); return make_range(Loops.begin(), Insert);