1 //===-- StructurizeCFG.cpp ------------------------------------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 #include "llvm/Transforms/Scalar.h"
11 #include "llvm/ADT/MapVector.h"
12 #include "llvm/ADT/PostOrderIterator.h"
13 #include "llvm/ADT/SCCIterator.h"
14 #include "llvm/Analysis/DivergenceAnalysis.h"
15 #include "llvm/Analysis/LoopInfo.h"
16 #include "llvm/Analysis/RegionInfo.h"
17 #include "llvm/Analysis/RegionIterator.h"
18 #include "llvm/Analysis/RegionPass.h"
19 #include "llvm/IR/Module.h"
20 #include "llvm/IR/PatternMatch.h"
21 #include "llvm/Support/Debug.h"
22 #include "llvm/Support/raw_ostream.h"
23 #include "llvm/Transforms/Utils/SSAUpdater.h"
26 using namespace llvm::PatternMatch;
28 #define DEBUG_TYPE "structurizecfg"
32 // Definition of the complex types used in this pass.
34 typedef std::pair<BasicBlock *, Value *> BBValuePair;
36 typedef SmallVector<RegionNode*, 8> RNVector;
37 typedef SmallVector<BasicBlock*, 8> BBVector;
38 typedef SmallVector<BranchInst*, 8> BranchVector;
39 typedef SmallVector<BBValuePair, 2> BBValueVector;
41 typedef SmallPtrSet<BasicBlock *, 8> BBSet;
43 typedef MapVector<PHINode *, BBValueVector> PhiMap;
44 typedef MapVector<BasicBlock *, BBVector> BB2BBVecMap;
46 typedef DenseMap<DomTreeNode *, unsigned> DTN2UnsignedMap;
47 typedef DenseMap<BasicBlock *, PhiMap> BBPhiMap;
48 typedef DenseMap<BasicBlock *, Value *> BBPredicates;
49 typedef DenseMap<BasicBlock *, BBPredicates> PredMap;
50 typedef DenseMap<BasicBlock *, BasicBlock*> BB2BBMap;
52 // The name for newly created blocks.
54 static const char *const FlowBlockName = "Flow";
56 /// @brief Find the nearest common dominator for multiple BasicBlocks
58 /// Helper class for StructurizeCFG
59 /// TODO: Maybe move into common code
60 class NearestCommonDominator {
63 DTN2UnsignedMap IndexMap;
67 bool ExplicitMentioned;
70 /// \brief Start a new query
71 NearestCommonDominator(DominatorTree *DomTree) {
76 /// \brief Add BB to the resulting dominator
77 void addBlock(BasicBlock *BB, bool Remember = true) {
78 DomTreeNode *Node = DT->getNode(BB);
81 unsigned Numbering = 0;
82 for (;Node;Node = Node->getIDom())
83 IndexMap[Node] = ++Numbering;
86 ExplicitMentioned = Remember;
90 for (;Node;Node = Node->getIDom())
91 if (IndexMap.count(Node))
96 assert(Node && "Dominator tree invalid!");
98 unsigned Numbering = IndexMap[Node];
99 if (Numbering > ResultIndex) {
100 Result = Node->getBlock();
101 ResultIndex = Numbering;
102 ExplicitMentioned = Remember && (Result == BB);
103 } else if (Numbering == ResultIndex) {
104 ExplicitMentioned |= Remember;
108 /// \brief Is "Result" one of the BBs added with "Remember" = True?
109 bool wasResultExplicitMentioned() {
110 return ExplicitMentioned;
113 /// \brief Get the query result
114 BasicBlock *getResult() {
119 /// @brief Transforms the control flow graph on one single entry/exit region
122 /// After the transform all "If"/"Then"/"Else" style control flow looks like
134 /// | | 1 = "If" block, calculates the condition
135 /// 4 | 2 = "Then" subregion, runs if the condition is true
136 /// | / 3 = "Flow" blocks, newly inserted flow blocks, rejoins the flow
137 /// |/ 4 = "Else" optional subregion, runs if the condition is false
138 /// 5 5 = "End" block, also rejoins the control flow
141 /// Control flow is expressed as a branch where the true exit goes into the
142 /// "Then"/"Else" region, while the false exit skips the region
143 /// The condition for the optional "Else" region is expressed as a PHI node.
144 /// The incomming values of the PHI node are true for the "If" edge and false
145 /// for the "Then" edge.
147 /// Additionally to that even complicated loops look like this:
154 /// | / 1 = "Entry" block
155 /// |/ 2 = "Loop" optional subregion, with all exits at "Flow" block
156 /// 3 3 = "Flow" block, with back edge to entry block
160 /// The back edge of the "Flow" block is always on the false side of the branch
161 /// while the true side continues the general flow. So the loop condition
162 /// consist of a network of PHI nodes where the true incoming values expresses
163 /// breaks and the false values expresses continue states.
164 class StructurizeCFG : public RegionPass {
165 bool SkipUniformRegions;
166 DivergenceAnalysis *DA;
169 ConstantInt *BoolTrue;
170 ConstantInt *BoolFalse;
171 UndefValue *BoolUndef;
174 Region *ParentRegion;
182 BBPhiMap DeletedPhis;
183 BB2BBVecMap AddedPhis;
186 BranchVector Conditions;
190 BranchVector LoopConds;
192 RegionNode *PrevNode;
196 void analyzeLoops(RegionNode *N);
198 Value *invert(Value *Condition);
200 Value *buildCondition(BranchInst *Term, unsigned Idx, bool Invert);
202 void gatherPredicates(RegionNode *N);
206 void insertConditions(bool Loops);
208 void delPhiValues(BasicBlock *From, BasicBlock *To);
210 void addPhiValues(BasicBlock *From, BasicBlock *To);
214 void killTerminator(BasicBlock *BB);
216 void changeExit(RegionNode *Node, BasicBlock *NewExit,
217 bool IncludeDominator);
219 BasicBlock *getNextFlow(BasicBlock *Dominator);
221 BasicBlock *needPrefix(bool NeedEmpty);
223 BasicBlock *needPostfix(BasicBlock *Flow, bool ExitUseAllowed);
225 void setPrevNode(BasicBlock *BB);
227 bool dominatesPredicates(BasicBlock *BB, RegionNode *Node);
229 bool isPredictableTrue(RegionNode *Node);
231 void wireFlow(bool ExitUseAllowed, BasicBlock *LoopEnd);
233 void handleLoops(bool ExitUseAllowed, BasicBlock *LoopEnd);
239 bool hasOnlyUniformBranches(const Region *R);
245 RegionPass(ID), SkipUniformRegions(false) {
246 initializeStructurizeCFGPass(*PassRegistry::getPassRegistry());
249 StructurizeCFG(bool SkipUniformRegions) :
250 RegionPass(ID), SkipUniformRegions(SkipUniformRegions) {
251 initializeStructurizeCFGPass(*PassRegistry::getPassRegistry());
254 using Pass::doInitialization;
255 bool doInitialization(Region *R, RGPassManager &RGM) override;
257 bool runOnRegion(Region *R, RGPassManager &RGM) override;
259 const char *getPassName() const override {
260 return "Structurize control flow";
263 void getAnalysisUsage(AnalysisUsage &AU) const override {
264 if (SkipUniformRegions)
265 AU.addRequired<DivergenceAnalysis>();
266 AU.addRequiredID(LowerSwitchID);
267 AU.addRequired<DominatorTreeWrapperPass>();
268 AU.addRequired<LoopInfoWrapperPass>();
269 AU.addPreserved<DominatorTreeWrapperPass>();
270 RegionPass::getAnalysisUsage(AU);
274 } // end anonymous namespace
276 char StructurizeCFG::ID = 0;
278 INITIALIZE_PASS_BEGIN(StructurizeCFG, "structurizecfg", "Structurize the CFG",
280 INITIALIZE_PASS_DEPENDENCY(DivergenceAnalysis)
281 INITIALIZE_PASS_DEPENDENCY(LowerSwitch)
282 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
283 INITIALIZE_PASS_DEPENDENCY(RegionInfoPass)
284 INITIALIZE_PASS_END(StructurizeCFG, "structurizecfg", "Structurize the CFG",
287 /// \brief Initialize the types and constants used in the pass
288 bool StructurizeCFG::doInitialization(Region *R, RGPassManager &RGM) {
289 LLVMContext &Context = R->getEntry()->getContext();
291 Boolean = Type::getInt1Ty(Context);
292 BoolTrue = ConstantInt::getTrue(Context);
293 BoolFalse = ConstantInt::getFalse(Context);
294 BoolUndef = UndefValue::get(Boolean);
299 /// \brief Build up the general order of nodes
300 void StructurizeCFG::orderNodes() {
302 ReversePostOrderTraversal<Region*> RPOT(ParentRegion);
303 TempOrder.append(RPOT.begin(), RPOT.end());
305 std::map<Loop*, unsigned> LoopBlocks;
308 // The reverse post-order traversal of the list gives us an ordering close
309 // to what we want. The only problem with it is that sometimes backedges
310 // for outer loops will be visited before backedges for inner loops.
311 for (RegionNode *RN : TempOrder) {
312 BasicBlock *BB = RN->getEntry();
313 Loop *Loop = LI->getLoopFor(BB);
317 unsigned CurrentLoopDepth = 0;
318 Loop *CurrentLoop = nullptr;
320 for (RNVector::iterator I = TempOrder.begin(), E = TempOrder.end(); I != E; ++I) {
321 BasicBlock *BB = (*I)->getEntry();
322 unsigned LoopDepth = LI->getLoopDepth(BB);
324 if (std::find(Order.begin(), Order.end(), *I) != Order.end())
327 if (LoopDepth < CurrentLoopDepth) {
328 // Make sure we have visited all blocks in this loop before moving back to
331 RNVector::iterator LoopI = I;
332 while (unsigned &BlockCount = LoopBlocks[CurrentLoop]) {
334 BasicBlock *LoopBB = (*LoopI)->getEntry();
335 if (LI->getLoopFor(LoopBB) == CurrentLoop) {
337 Order.push_back(*LoopI);
342 CurrentLoop = LI->getLoopFor(BB);
344 LoopBlocks[CurrentLoop]--;
347 CurrentLoopDepth = LoopDepth;
351 // This pass originally used a post-order traversal and then operated on
352 // the list in reverse. Now that we are using a reverse post-order traversal
353 // rather than re-working the whole pass to operate on the list in order,
354 // we just reverse the list and continue to operate on it in reverse.
355 std::reverse(Order.begin(), Order.end());
358 /// \brief Determine the end of the loops
359 void StructurizeCFG::analyzeLoops(RegionNode *N) {
360 if (N->isSubRegion()) {
361 // Test for exit as back edge
362 BasicBlock *Exit = N->getNodeAs<Region>()->getExit();
363 if (Visited.count(Exit))
364 Loops[Exit] = N->getEntry();
367 // Test for sucessors as back edge
368 BasicBlock *BB = N->getNodeAs<BasicBlock>();
369 BranchInst *Term = cast<BranchInst>(BB->getTerminator());
371 for (BasicBlock *Succ : Term->successors())
372 if (Visited.count(Succ))
377 /// \brief Invert the given condition
378 Value *StructurizeCFG::invert(Value *Condition) {
379 // First: Check if it's a constant
380 if (Constant *C = dyn_cast<Constant>(Condition))
381 return ConstantExpr::getNot(C);
383 // Second: If the condition is already inverted, return the original value
384 if (match(Condition, m_Not(m_Value(Condition))))
387 if (Instruction *Inst = dyn_cast<Instruction>(Condition)) {
388 // Third: Check all the users for an invert
389 BasicBlock *Parent = Inst->getParent();
390 for (User *U : Condition->users())
391 if (Instruction *I = dyn_cast<Instruction>(U))
392 if (I->getParent() == Parent && match(I, m_Not(m_Specific(Condition))))
395 // Last option: Create a new instruction
396 return BinaryOperator::CreateNot(Condition, "", Parent->getTerminator());
399 if (Argument *Arg = dyn_cast<Argument>(Condition)) {
400 BasicBlock &EntryBlock = Arg->getParent()->getEntryBlock();
401 return BinaryOperator::CreateNot(Condition,
402 Arg->getName() + ".inv",
403 EntryBlock.getTerminator());
406 llvm_unreachable("Unhandled condition to invert");
409 /// \brief Build the condition for one edge
410 Value *StructurizeCFG::buildCondition(BranchInst *Term, unsigned Idx,
412 Value *Cond = Invert ? BoolFalse : BoolTrue;
413 if (Term->isConditional()) {
414 Cond = Term->getCondition();
416 if (Idx != (unsigned)Invert)
422 /// \brief Analyze the predecessors of each block and build up predicates
423 void StructurizeCFG::gatherPredicates(RegionNode *N) {
424 RegionInfo *RI = ParentRegion->getRegionInfo();
425 BasicBlock *BB = N->getEntry();
426 BBPredicates &Pred = Predicates[BB];
427 BBPredicates &LPred = LoopPreds[BB];
429 for (pred_iterator PI = pred_begin(BB), PE = pred_end(BB);
432 // Ignore it if it's a branch from outside into our region entry
433 if (!ParentRegion->contains(*PI))
436 Region *R = RI->getRegionFor(*PI);
437 if (R == ParentRegion) {
439 // It's a top level block in our region
440 BranchInst *Term = cast<BranchInst>((*PI)->getTerminator());
441 for (unsigned i = 0, e = Term->getNumSuccessors(); i != e; ++i) {
442 BasicBlock *Succ = Term->getSuccessor(i);
446 if (Visited.count(*PI)) {
447 // Normal forward edge
448 if (Term->isConditional()) {
449 // Try to treat it like an ELSE block
450 BasicBlock *Other = Term->getSuccessor(!i);
451 if (Visited.count(Other) && !Loops.count(Other) &&
452 !Pred.count(Other) && !Pred.count(*PI)) {
454 Pred[Other] = BoolFalse;
455 Pred[*PI] = BoolTrue;
459 Pred[*PI] = buildCondition(Term, i, false);
463 LPred[*PI] = buildCondition(Term, i, true);
469 // It's an exit from a sub region
470 while (R->getParent() != ParentRegion)
473 // Edge from inside a subregion to its entry, ignore it
477 BasicBlock *Entry = R->getEntry();
478 if (Visited.count(Entry))
479 Pred[Entry] = BoolTrue;
481 LPred[Entry] = BoolFalse;
486 /// \brief Collect various loop and predicate infos
487 void StructurizeCFG::collectInfos() {
495 // Reset the visited nodes
498 for (RegionNode *RN : reverse(Order)) {
500 DEBUG(dbgs() << "Visiting: "
501 << (RN->isSubRegion() ? "SubRegion with entry: " : "")
502 << RN->getEntry()->getName() << " Loop Depth: "
503 << LI->getLoopDepth(RN->getEntry()) << "\n");
505 // Analyze all the conditions leading to a node
506 gatherPredicates(RN);
508 // Remember that we've seen this node
509 Visited.insert(RN->getEntry());
511 // Find the last back edges
516 /// \brief Insert the missing branch conditions
517 void StructurizeCFG::insertConditions(bool Loops) {
518 BranchVector &Conds = Loops ? LoopConds : Conditions;
519 Value *Default = Loops ? BoolTrue : BoolFalse;
520 SSAUpdater PhiInserter;
522 for (BranchInst *Term : Conds) {
523 assert(Term->isConditional());
525 BasicBlock *Parent = Term->getParent();
526 BasicBlock *SuccTrue = Term->getSuccessor(0);
527 BasicBlock *SuccFalse = Term->getSuccessor(1);
529 PhiInserter.Initialize(Boolean, "");
530 PhiInserter.AddAvailableValue(&Func->getEntryBlock(), Default);
531 PhiInserter.AddAvailableValue(Loops ? SuccFalse : Parent, Default);
533 BBPredicates &Preds = Loops ? LoopPreds[SuccFalse] : Predicates[SuccTrue];
535 NearestCommonDominator Dominator(DT);
536 Dominator.addBlock(Parent, false);
538 Value *ParentValue = nullptr;
539 for (BBPredicates::iterator PI = Preds.begin(), PE = Preds.end();
542 if (PI->first == Parent) {
543 ParentValue = PI->second;
546 PhiInserter.AddAvailableValue(PI->first, PI->second);
547 Dominator.addBlock(PI->first);
551 Term->setCondition(ParentValue);
553 if (!Dominator.wasResultExplicitMentioned())
554 PhiInserter.AddAvailableValue(Dominator.getResult(), Default);
556 Term->setCondition(PhiInserter.GetValueInMiddleOfBlock(Parent));
561 /// \brief Remove all PHI values coming from "From" into "To" and remember
562 /// them in DeletedPhis
563 void StructurizeCFG::delPhiValues(BasicBlock *From, BasicBlock *To) {
564 PhiMap &Map = DeletedPhis[To];
565 for (BasicBlock::iterator I = To->begin(), E = To->end();
566 I != E && isa<PHINode>(*I);) {
568 PHINode &Phi = cast<PHINode>(*I++);
569 while (Phi.getBasicBlockIndex(From) != -1) {
570 Value *Deleted = Phi.removeIncomingValue(From, false);
571 Map[&Phi].push_back(std::make_pair(From, Deleted));
576 /// \brief Add a dummy PHI value as soon as we knew the new predecessor
577 void StructurizeCFG::addPhiValues(BasicBlock *From, BasicBlock *To) {
578 for (BasicBlock::iterator I = To->begin(), E = To->end();
579 I != E && isa<PHINode>(*I);) {
581 PHINode &Phi = cast<PHINode>(*I++);
582 Value *Undef = UndefValue::get(Phi.getType());
583 Phi.addIncoming(Undef, From);
585 AddedPhis[To].push_back(From);
588 /// \brief Add the real PHI value as soon as everything is set up
589 void StructurizeCFG::setPhiValues() {
591 for (const auto &AddedPhi : AddedPhis) {
593 BasicBlock *To = AddedPhi.first;
594 const BBVector &From = AddedPhi.second;
596 if (!DeletedPhis.count(To))
599 PhiMap &Map = DeletedPhis[To];
600 for (const auto &PI : Map) {
602 PHINode *Phi = PI.first;
603 Value *Undef = UndefValue::get(Phi->getType());
604 Updater.Initialize(Phi->getType(), "");
605 Updater.AddAvailableValue(&Func->getEntryBlock(), Undef);
606 Updater.AddAvailableValue(To, Undef);
608 NearestCommonDominator Dominator(DT);
609 Dominator.addBlock(To, false);
610 for (const auto &VI : PI.second) {
612 Updater.AddAvailableValue(VI.first, VI.second);
613 Dominator.addBlock(VI.first);
616 if (!Dominator.wasResultExplicitMentioned())
617 Updater.AddAvailableValue(Dominator.getResult(), Undef);
619 for (BasicBlock *FI : From) {
621 int Idx = Phi->getBasicBlockIndex(FI);
623 Phi->setIncomingValue(Idx, Updater.GetValueAtEndOfBlock(FI));
627 DeletedPhis.erase(To);
629 assert(DeletedPhis.empty());
632 /// \brief Remove phi values from all successors and then remove the terminator.
633 void StructurizeCFG::killTerminator(BasicBlock *BB) {
634 TerminatorInst *Term = BB->getTerminator();
638 for (succ_iterator SI = succ_begin(BB), SE = succ_end(BB);
641 delPhiValues(BB, *SI);
644 Term->eraseFromParent();
647 /// \brief Let node exit(s) point to NewExit
648 void StructurizeCFG::changeExit(RegionNode *Node, BasicBlock *NewExit,
649 bool IncludeDominator) {
650 if (Node->isSubRegion()) {
651 Region *SubRegion = Node->getNodeAs<Region>();
652 BasicBlock *OldExit = SubRegion->getExit();
653 BasicBlock *Dominator = nullptr;
655 // Find all the edges from the sub region to the exit
656 for (pred_iterator I = pred_begin(OldExit), E = pred_end(OldExit);
659 BasicBlock *BB = *I++;
660 if (!SubRegion->contains(BB))
663 // Modify the edges to point to the new exit
664 delPhiValues(BB, OldExit);
665 BB->getTerminator()->replaceUsesOfWith(OldExit, NewExit);
666 addPhiValues(BB, NewExit);
668 // Find the new dominator (if requested)
669 if (IncludeDominator) {
673 Dominator = DT->findNearestCommonDominator(Dominator, BB);
677 // Change the dominator (if requested)
679 DT->changeImmediateDominator(NewExit, Dominator);
681 // Update the region info
682 SubRegion->replaceExit(NewExit);
685 BasicBlock *BB = Node->getNodeAs<BasicBlock>();
687 BranchInst::Create(NewExit, BB);
688 addPhiValues(BB, NewExit);
689 if (IncludeDominator)
690 DT->changeImmediateDominator(NewExit, BB);
694 /// \brief Create a new flow node and update dominator tree and region info
695 BasicBlock *StructurizeCFG::getNextFlow(BasicBlock *Dominator) {
696 LLVMContext &Context = Func->getContext();
697 BasicBlock *Insert = Order.empty() ? ParentRegion->getExit() :
698 Order.back()->getEntry();
699 BasicBlock *Flow = BasicBlock::Create(Context, FlowBlockName,
701 DT->addNewBlock(Flow, Dominator);
702 ParentRegion->getRegionInfo()->setRegionFor(Flow, ParentRegion);
706 /// \brief Create a new or reuse the previous node as flow node
707 BasicBlock *StructurizeCFG::needPrefix(bool NeedEmpty) {
708 BasicBlock *Entry = PrevNode->getEntry();
710 if (!PrevNode->isSubRegion()) {
711 killTerminator(Entry);
712 if (!NeedEmpty || Entry->getFirstInsertionPt() == Entry->end())
717 // create a new flow node
718 BasicBlock *Flow = getNextFlow(Entry);
721 changeExit(PrevNode, Flow, true);
722 PrevNode = ParentRegion->getBBNode(Flow);
726 /// \brief Returns the region exit if possible, otherwise just a new flow node
727 BasicBlock *StructurizeCFG::needPostfix(BasicBlock *Flow,
728 bool ExitUseAllowed) {
729 if (Order.empty() && ExitUseAllowed) {
730 BasicBlock *Exit = ParentRegion->getExit();
731 DT->changeImmediateDominator(Exit, Flow);
732 addPhiValues(Flow, Exit);
735 return getNextFlow(Flow);
738 /// \brief Set the previous node
739 void StructurizeCFG::setPrevNode(BasicBlock *BB) {
740 PrevNode = ParentRegion->contains(BB) ? ParentRegion->getBBNode(BB)
744 /// \brief Does BB dominate all the predicates of Node ?
745 bool StructurizeCFG::dominatesPredicates(BasicBlock *BB, RegionNode *Node) {
746 BBPredicates &Preds = Predicates[Node->getEntry()];
747 for (BBPredicates::iterator PI = Preds.begin(), PE = Preds.end();
750 if (!DT->dominates(BB, PI->first))
756 /// \brief Can we predict that this node will always be called?
757 bool StructurizeCFG::isPredictableTrue(RegionNode *Node) {
758 BBPredicates &Preds = Predicates[Node->getEntry()];
759 bool Dominated = false;
761 // Regionentry is always true
765 for (BBPredicates::iterator I = Preds.begin(), E = Preds.end();
768 if (I->second != BoolTrue)
771 if (!Dominated && DT->dominates(I->first, PrevNode->getEntry()))
775 // TODO: The dominator check is too strict
779 /// Take one node from the order vector and wire it up
780 void StructurizeCFG::wireFlow(bool ExitUseAllowed,
781 BasicBlock *LoopEnd) {
782 RegionNode *Node = Order.pop_back_val();
783 Visited.insert(Node->getEntry());
785 if (isPredictableTrue(Node)) {
786 // Just a linear flow
788 changeExit(PrevNode, Node->getEntry(), true);
793 // Insert extra prefix node (or reuse last one)
794 BasicBlock *Flow = needPrefix(false);
796 // Insert extra postfix node (or use exit instead)
797 BasicBlock *Entry = Node->getEntry();
798 BasicBlock *Next = needPostfix(Flow, ExitUseAllowed);
800 // let it point to entry and next block
801 Conditions.push_back(BranchInst::Create(Entry, Next, BoolUndef, Flow));
802 addPhiValues(Flow, Entry);
803 DT->changeImmediateDominator(Entry, Flow);
806 while (!Order.empty() && !Visited.count(LoopEnd) &&
807 dominatesPredicates(Entry, Order.back())) {
808 handleLoops(false, LoopEnd);
811 changeExit(PrevNode, Next, false);
816 void StructurizeCFG::handleLoops(bool ExitUseAllowed,
817 BasicBlock *LoopEnd) {
818 RegionNode *Node = Order.back();
819 BasicBlock *LoopStart = Node->getEntry();
821 if (!Loops.count(LoopStart)) {
822 wireFlow(ExitUseAllowed, LoopEnd);
826 if (!isPredictableTrue(Node))
827 LoopStart = needPrefix(true);
829 LoopEnd = Loops[Node->getEntry()];
830 wireFlow(false, LoopEnd);
831 while (!Visited.count(LoopEnd)) {
832 handleLoops(false, LoopEnd);
835 // If the start of the loop is the entry block, we can't branch to it so
836 // insert a new dummy entry block.
837 Function *LoopFunc = LoopStart->getParent();
838 if (LoopStart == &LoopFunc->getEntryBlock()) {
839 LoopStart->setName("entry.orig");
841 BasicBlock *NewEntry =
842 BasicBlock::Create(LoopStart->getContext(),
846 BranchInst::Create(LoopStart, NewEntry);
849 // Create an extra loop end node
850 LoopEnd = needPrefix(false);
851 BasicBlock *Next = needPostfix(LoopEnd, ExitUseAllowed);
852 LoopConds.push_back(BranchInst::Create(Next, LoopStart,
853 BoolUndef, LoopEnd));
854 addPhiValues(LoopEnd, LoopStart);
858 /// After this function control flow looks like it should be, but
859 /// branches and PHI nodes only have undefined conditions.
860 void StructurizeCFG::createFlow() {
861 BasicBlock *Exit = ParentRegion->getExit();
862 bool EntryDominatesExit = DT->dominates(ParentRegion->getEntry(), Exit);
872 while (!Order.empty()) {
873 handleLoops(EntryDominatesExit, nullptr);
877 changeExit(PrevNode, Exit, EntryDominatesExit);
879 assert(EntryDominatesExit);
882 /// Handle a rare case where the disintegrated nodes instructions
883 /// no longer dominate all their uses. Not sure if this is really nessasary
884 void StructurizeCFG::rebuildSSA() {
886 for (auto *BB : ParentRegion->blocks())
887 for (BasicBlock::iterator II = BB->begin(), IE = BB->end();
890 bool Initialized = false;
891 for (auto I = II->use_begin(), E = II->use_end(); I != E;) {
893 Instruction *User = cast<Instruction>(U.getUser());
894 if (User->getParent() == BB) {
897 } else if (PHINode *UserPN = dyn_cast<PHINode>(User)) {
898 if (UserPN->getIncomingBlock(U) == BB)
902 if (DT->dominates(&*II, User))
906 Value *Undef = UndefValue::get(II->getType());
907 Updater.Initialize(II->getType(), "");
908 Updater.AddAvailableValue(&Func->getEntryBlock(), Undef);
909 Updater.AddAvailableValue(BB, &*II);
912 Updater.RewriteUseAfterInsertions(U);
917 bool StructurizeCFG::hasOnlyUniformBranches(const Region *R) {
918 for (const BasicBlock *BB : R->blocks()) {
919 const BranchInst *Br = dyn_cast<BranchInst>(BB->getTerminator());
920 if (!Br || !Br->isConditional())
923 if (!DA->isUniform(Br->getCondition()))
925 DEBUG(dbgs() << "BB: " << BB->getName() << " has uniform terminator\n");
930 /// \brief Run the transformation for each region found
931 bool StructurizeCFG::runOnRegion(Region *R, RGPassManager &RGM) {
932 if (R->isTopLevelRegion())
935 if (SkipUniformRegions) {
936 DA = &getAnalysis<DivergenceAnalysis>();
937 // TODO: We could probably be smarter here with how we handle sub-regions.
938 if (hasOnlyUniformBranches(R)) {
939 DEBUG(dbgs() << "Skipping region with uniform control flow: " << *R << '\n');
941 // Mark all direct child block terminators as having been treated as
942 // uniform. To account for a possible future in which non-uniform
943 // sub-regions are treated more cleverly, indirect children are not
944 // marked as uniform.
945 MDNode *MD = MDNode::get(R->getEntry()->getParent()->getContext(), {});
946 Region::element_iterator E = R->element_end();
947 for (Region::element_iterator I = R->element_begin(); I != E; ++I) {
948 if (I->isSubRegion())
951 if (Instruction *Term = I->getEntry()->getTerminator())
952 Term->setMetadata("structurizecfg.uniform", MD);
959 Func = R->getEntry()->getParent();
962 DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
963 LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
968 insertConditions(false);
969 insertConditions(true);
987 Pass *llvm::createStructurizeCFGPass(bool SkipUniformRegions) {
988 return new StructurizeCFG(SkipUniformRegions);