diff --git a/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h b/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h index 328db2e99d34..ced61b26f710 100644 --- a/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h +++ b/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h @@ -589,23 +589,23 @@ struct IrreducibleGraph { using BlockNode = BFIBase::BlockNode; struct IrrNode { BlockNode Node; - unsigned NumIn = 0; - std::deque Edges; + SmallVector Succs; IrrNode(const BlockNode &Node) : Node(Node) {} - using iterator = std::deque::const_iterator; + using iterator = SmallVectorImpl::const_iterator; - iterator pred_begin() const { return Edges.begin(); } - iterator succ_begin() const { return Edges.begin() + NumIn; } - iterator pred_end() const { return succ_begin(); } - iterator succ_end() const { return Edges.end(); } + iterator succ_begin() const { return Succs.begin(); } + iterator succ_end() const { return Succs.end(); } }; BlockNode Start; const IrrNode *StartIrr = nullptr; std::vector Nodes; SmallDenseMap Lookup; + /// The position of \p N in \a Nodes, for indexing side tables. + unsigned getIndex(const IrrNode *N) const { return N - Nodes.data(); } + /// Construct an explicit graph containing irreducible control flow. /// /// Construct an explicit graph of the control flow in \c OuterLoop (or the diff --git a/llvm/lib/Analysis/BlockFrequencyInfoImpl.cpp b/llvm/lib/Analysis/BlockFrequencyInfoImpl.cpp index 4680af8f50c2..011435db6678 100644 --- a/llvm/lib/Analysis/BlockFrequencyInfoImpl.cpp +++ b/llvm/lib/Analysis/BlockFrequencyInfoImpl.cpp @@ -660,9 +660,7 @@ void IrreducibleGraph::addEdge(IrrNode &Irr, const BlockNode &Succ, if (L == Lookup.end()) return; IrrNode &SuccIrr = *L->second; - Irr.Edges.push_back(&SuccIrr); - SuccIrr.Edges.push_front(&Irr); - ++SuccIrr.NumIn; + Irr.Succs.push_back(&SuccIrr); } namespace llvm { @@ -679,89 +677,42 @@ template <> struct GraphTraits { } // end namespace llvm -/// Find extra irreducible headers. -/// -/// Find entry blocks and other blocks with backedges, which exist when \c G -/// contains irreducible sub-SCCs. -static void findIrreducibleHeaders( - const BlockFrequencyInfoImplBase &BFI, - const IrreducibleGraph &G, - const std::vector &SCC, - LoopData::NodeList &Headers, LoopData::NodeList &Others) { - // Map from nodes in the SCC to whether it's an entry block. - SmallDenseMap InSCC; - - // InSCC also acts the set of nodes in the graph. Seed it. - for (const auto *I : SCC) - InSCC[I] = false; - - for (auto I = InSCC.begin(), E = InSCC.end(); I != E; ++I) { - auto &Irr = *I->first; - for (const auto *P : make_range(Irr.pred_begin(), Irr.pred_end())) { - if (InSCC.count(P)) - continue; - - // This is an entry block. - I->second = true; - Headers.push_back(Irr.Node); - LLVM_DEBUG(dbgs() << " => entry = " << BFI.getBlockName(Irr.Node) - << "\n"); - break; - } - } - assert(Headers.size() >= 2 && - "Expected irreducible CFG; -loop-info is likely invalid"); - if (Headers.size() == InSCC.size()) { - // Every block is a header. - llvm::sort(Headers); - return; - } - - // Look for extra headers from irreducible sub-SCCs. - for (const auto &I : InSCC) { - // Entry blocks are already headers. - if (I.second) - continue; - - auto &Irr = *I.first; - for (const auto *P : make_range(Irr.pred_begin(), Irr.pred_end())) { - // Skip forward edges. - if (P->Node < Irr.Node) - continue; - - // Skip predecessors from entry blocks. These can have inverted - // ordering. - if (InSCC.lookup(P)) - continue; - - // Store the extra header. - Headers.push_back(Irr.Node); - LLVM_DEBUG(dbgs() << " => extra = " << BFI.getBlockName(Irr.Node) - << "\n"); - break; - } - if (Headers.back() == Irr.Node) - // Added this as a header. - continue; - - // This is not a header. - Others.push_back(Irr.Node); - LLVM_DEBUG(dbgs() << " => other = " << BFI.getBlockName(Irr.Node) << "\n"); - } - llvm::sort(Headers); - llvm::sort(Others); -} - -static void createIrreducibleLoop( - BlockFrequencyInfoImplBase &BFI, const IrreducibleGraph &G, - LoopData *OuterLoop, std::list::iterator Insert, - const std::vector &SCC) { - // Translate the SCC into RPO. +/// Package \c SCC into a loop, headed by the nodes marked in \c IsEntry or +/// \c Extra. +static void +createIrreducibleLoop(BlockFrequencyInfoImplBase &BFI, + const IrreducibleGraph &G, LoopData *OuterLoop, + std::list::iterator Insert, + ArrayRef SCC, + const BitVector &IsEntry, const BitVector &Extra) { LLVM_DEBUG(dbgs() << " - found-scc\n"); LoopData::NodeList Headers; LoopData::NodeList Others; - findIrreducibleHeaders(BFI, G, SCC, Headers, Others); + for (const auto *I : SCC) + if (IsEntry.test(G.getIndex(I))) { + Headers.push_back(I->Node); + LLVM_DEBUG(dbgs() << " => entry = " << BFI.getBlockName(I->Node) + << "\n"); + } + assert(Headers.size() >= 2 && + "Expected irreducible CFG; -loop-info is likely invalid"); + + for (const auto *I : SCC) { + if (IsEntry.test(G.getIndex(I))) + continue; + if (Extra.test(G.getIndex(I))) { + Headers.push_back(I->Node); + LLVM_DEBUG(dbgs() << " => extra = " << BFI.getBlockName(I->Node) + << "\n"); + } else { + Others.push_back(I->Node); + LLVM_DEBUG(dbgs() << " => other = " << BFI.getBlockName(I->Node) + << "\n"); + } + } + llvm::sort(Headers); + llvm::sort(Others); auto Loop = BFI.Loops.emplace(Insert, OuterLoop, Headers.begin(), Headers.end(), Others.begin(), Others.end()); @@ -781,14 +732,38 @@ BlockFrequencyInfoImplBase::analyzeIrreducible( assert((OuterLoop == nullptr) == (Insert == Loops.begin())); auto Prev = OuterLoop ? std::prev(Insert) : Loops.end(); - for (auto I = scc_begin(G); !I.isAtEnd(); ++I) { - if (I->size() < 2) - continue; - - // Translate the SCC into RPO. - createIrreducibleLoop(*this, G, OuterLoop, Insert, *I); + // Number every node's SCC, as the sweeps below compare an edge's two ends. + // Only multi-node SCCs become loops, so keep just their members. + SmallVector SccId(G.Nodes.size(), ~0u); + SmallVector> SCCs; + unsigned Id = 0; + for (auto I = scc_begin(G); !I.isAtEnd(); ++I, ++Id) { + for (const auto *N : *I) + SccId[G.getIndex(N)] = Id; + if (I->size() >= 2) + SCCs.emplace_back(I->begin(), I->end()); } + // A node is an entry if an edge from another SCC reaches it, and an extra + // header if a backedge within its SCC targets it. Backedges from entries + // can have inverted ordering, so they do not make a header. + BitVector IsEntry(G.Nodes.size()); + BitVector Extra(G.Nodes.size()); + for (const auto &U : G.Nodes) + for (const auto *V : U.Succs) + if (SccId[G.getIndex(&U)] != SccId[G.getIndex(V)]) + IsEntry.set(G.getIndex(V)); + for (const auto &U : G.Nodes) { + if (IsEntry.test(G.getIndex(&U))) + continue; + for (const auto *V : U.Succs) + if (SccId[G.getIndex(V)] == SccId[G.getIndex(&U)] && !(U.Node < V->Node)) + Extra.set(G.getIndex(V)); + } + + for (const auto &SCC : SCCs) + createIrreducibleLoop(*this, G, OuterLoop, Insert, SCC, IsEntry, Extra); + if (OuterLoop) return make_range(std::next(Prev), Insert); return make_range(Loops.begin(), Insert);