diff --git a/llvm/include/llvm/Transforms/Scalar/GVN.h b/llvm/include/llvm/Transforms/Scalar/GVN.h index bc0f108ac8260..db74c2066013c 100644 --- a/llvm/include/llvm/Transforms/Scalar/GVN.h +++ b/llvm/include/llvm/Transforms/Scalar/GVN.h @@ -22,6 +22,7 @@ #include "llvm/IR/Dominators.h" #include "llvm/IR/InstrTypes.h" #include "llvm/IR/PassManager.h" +#include "llvm/Analysis/LoopInfo.h" #include "llvm/IR/ValueHandle.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/Compiler.h" @@ -45,6 +46,8 @@ class FunctionPass; class GetElementPtrInst; class ImplicitControlFlowTracking; class LoadInst; +class Loop; +class SelectInst; class LoopInfo; class MemDepResult; class MemoryAccess; @@ -405,6 +408,14 @@ class GVNPass : public PassInfoMixin { void addDeadBlock(BasicBlock *BB); void assignValNumForDeadCode(); void assignBlockRPONumber(Function &F); + + bool recognizeMinFindingSelectPattern(SelectInst *Select); + bool transformMinFindingSelectPattern(Loop *L, Type *LoadType, + BasicBlock *Preheader, BasicBlock *BB, + Value *LHS, Value *RHS, + CmpInst *Comparison, SelectInst *Select, + Value *BasePtr, Value *IndexVal, + Value *OffsetVal); }; /// Create a legacy GVN pass. diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp index 72e1131a54a86..b514210c8cc1c 100644 --- a/llvm/lib/Transforms/Scalar/GVN.cpp +++ b/llvm/lib/Transforms/Scalar/GVN.cpp @@ -34,7 +34,6 @@ #include "llvm/Analysis/InstructionPrecedenceTracking.h" #include "llvm/Analysis/InstructionSimplify.h" #include "llvm/Analysis/Loads.h" -#include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/MemoryBuiltins.h" #include "llvm/Analysis/MemoryDependenceAnalysis.h" #include "llvm/Analysis/MemorySSA.h" @@ -2743,6 +2742,10 @@ bool GVNPass::processInstruction(Instruction *I) { } return Changed; } + if (SelectInst *Select = dyn_cast(I)) { + if (recognizeMinFindingSelectPattern(Select)) + return true; + } // Instructions with void type don't return a value, so there's // no point in trying to find redundancies in them. @@ -3330,6 +3333,235 @@ void GVNPass::assignValNumForDeadCode() { } } +// Hoist the chain of operations for the second load to preheader. +// In this transformation, we hoist the redundant load to the preheader, +// caching the first value of the iteration. This value is used to compare with +// the current value of the iteration and update the minimum value. +// The comparison is done in the loop body using the new select instruction. +// +// *** Before transformation *** +// +// preheader: +// ... +// loop: +// ... +// ... +// %val.first = load , ptr %ptr.first.load, align 4 +// %min.idx.ext = sext i32 %min.idx to i64 +// %ptr..min = getelementptr , ptr %0, i64 %min.idx.ext +// %ptr.second.load = getelementptr i8, ptr %ptr..min, i64 -4 +// %val.current.min = load , ptr %ptr.second.load, align 4 +// ... +// ... +// br i1 %cond, label %loop, label %exit +// +// *** After transformation *** +// +// preheader: +// %min.idx.ext = sext i32 %min.idx.ext to i64 +// %hoist_gep1 = getelementptr , ptr %0, i64 %min.idx.ext +// %hoist_gep2 = getelementptr i8, ptr %hoist_gep1, i64 -4 +// %hoisted_load = load , ptr %hoist_gep2, align 4 +// br label %loop +// +// loop: +// %val.first = load , ptr %ptr.first.load, align 4 +// ... +// (new) %val.current.min = select i1 %cond, %hoisted_load, +// %val.current.min +// ... +// ... +// br i1 %cond, label %loop, label %exit +bool GVNPass::transformMinFindingSelectPattern( + Loop *L, Type *LoadType, BasicBlock *Preheader, BasicBlock *BB, Value *LHS, + Value *LoadVal, CmpInst *Comparison, SelectInst *Select, Value *BasePtr, + Value *IndexVal, Value *OffsetVal) { + + assert(IndexVal && "IndexVal is null"); + AAResults *AA = VN.getAliasAnalysis(); + assert(AA && "AA is null"); + + IRBuilder<> Builder(Preheader->getTerminator()); + Value *InitialMinIndex = + dyn_cast(IndexVal)->getIncomingValueForBlock(Preheader); + + // Insert PHI node at the top of this block. + // This PHI node will be used to memoize the current minimum value so far. + PHINode *KnownMinPhi = PHINode::Create(LoadType, 2, "known_min", BB->begin()); + + // Hoist the load and build the necessary operations. + // 1. hoist_0 = sext i32 1 to i64 + Value *HoistedSExt = + Builder.CreateSExt(InitialMinIndex, Builder.getInt64Ty(), "hoist_sext"); + + // 2. hoist_gep1 = getelementptr float, ptr BasePtr, i64 HoistedSExt + Value *HoistedGEP1 = + Builder.CreateGEP(LoadType, BasePtr, HoistedSExt, "hoist_gep1"); + + // 3. hoist_gep2 = getelementptr i8, ptr HoistedGEP1, i64 OffsetVal + Value *HoistedGEP2 = Builder.CreateGEP(Builder.getInt8Ty(), HoistedGEP1, + OffsetVal, "hoist_gep2"); + + MemoryLocation NewLoc = MemoryLocation( + HoistedGEP2, + LocationSize::precise( + L->getHeader()->getDataLayout().getTypeStoreSize(LoadType))); + // Check if any instruction in the loop clobbers this location. + bool CanHoist = true; + for (BasicBlock *BB : L->blocks()) { + for (Instruction &I : *BB) { + if (I.mayWriteToMemory()) { + // Check if this instruction may clobber our hoisted load. + ModRefInfo MRI = AA->getModRefInfo(&I, NewLoc); + if (isModOrRefSet(MRI)) { + LLVM_DEBUG(dbgs() << "GVN: Cannot hoist - may be clobbered by: " << I + << "\n"); + CanHoist = false; + break; + } + } + } + if (!CanHoist) + break; + } + if (!CanHoist) { + LLVM_DEBUG(dbgs() << "GVN: Cannot hoist - may be clobbered by some " + "instruction in the loop.\n"); + return false; + } + + // 4. hoisted_load = load float, ptr HoistedGEP2 + LoadInst *NewLoad = Builder.CreateLoad(LoadType, HoistedGEP2, "hoisted_load"); + + // Let the new load now take the place of the old load. + LoadVal->replaceAllUsesWith(NewLoad); + dyn_cast(LoadVal)->eraseFromParent(); + + // Comparison should now compare the current value and the newly inserted + // PHI node. + Comparison->setOperand(1, KnownMinPhi); + + // Create new select instruction for selecting the minimum value. + IRBuilder<> SelectBuilder(BB->getTerminator()); + SelectInst *CurrentMinSelect = dyn_cast( + SelectBuilder.CreateSelect(Comparison, LHS, KnownMinPhi, "current_min")); + + // Populate the newly created PHI node + // with (hoisted) NewLoad from the preheader and CurrentMinSelect. + KnownMinPhi->addIncoming(NewLoad, Preheader); + KnownMinPhi->addIncoming(CurrentMinSelect, BB); + + if (MSSAU) { + auto *OrigUse = + MSSAU->getMemorySSA()->getMemoryAccess(dyn_cast(LoadVal)); + if (OrigUse) { + MemoryAccess *DefiningAccess = OrigUse->getDefiningAccess(); + MSSAU->createMemoryAccessInBB(NewLoad, DefiningAccess, Preheader, + MemorySSA::BeforeTerminator); + } + } + LLVM_DEBUG( + dbgs() << "GVN: Transformed the code for minimum finding pattern.\n"); + return true; +} + +// We are looking for the following pattern: +// loop: +// ... +// ... +// %min.idx = phi i32 [ %initial_min_idx, %entry ], [ %min.idx.next, %loop ] +// ... +// %val.first = load , ptr %ptr.first.load, align 4 +// %min.idx.ext = sext i32 %min.idx to i64 +// %ptr..min = getelementptr , ptr %0, i64 %min.idx.ext +// %ptr.second.load = getelementptr i8, ptr %ptr..min, i64 -4 +// %val.current.min = load , ptr %ptr.second.load, align 4 +// %cmp = %val.first, %val.current.min +// ... +// %min.idx.next = select i1 %cmp, ..., i32 %min.idx +// ... +// ... +// br i1 ..., label %loop, ... +bool GVNPass::recognizeMinFindingSelectPattern(SelectInst *Select) { + IRBuilder<> Builder(Select); + Value *BasePtr = nullptr, *IndexVal = nullptr, *OffsetVal = nullptr, + *SExt = nullptr; + BasicBlock *BB = Select->getParent(); + + // If the block is not in a loop, bail out. + Loop *L = LI->getLoopFor(BB); + if (!L) { + LLVM_DEBUG(dbgs() << "GVN: Could not find loop.\n"); + return false; + } + + // If preheader of the loop is not found, bail out. + BasicBlock *Preheader = L->getLoopPreheader(); + if (!Preheader) { + LLVM_DEBUG(dbgs() << "GVN: Could not find loop preheader.\n"); + return false; + } + Value *Condition = Select->getCondition(); + CmpInst *Comparison = dyn_cast(Condition); + if (!Comparison) { + LLVM_DEBUG(dbgs() << "GVN: Condition is not a comparison.\n"); + return false; + } + + // Check if this is less-than comparison. + CmpInst::Predicate Pred = Comparison->getPredicate(); + if (Pred != CmpInst::ICMP_SLT && Pred != CmpInst::ICMP_ULT && + Pred != CmpInst::FCMP_OLT && Pred != CmpInst::FCMP_ULT) { + LLVM_DEBUG(dbgs() << "GVN: Not a less-than comparison, predicate: " << Pred + << "\n"); + return false; + } + + // Check that both operands are loads. + Value *LHS = Comparison->getOperand(0); + Value *RHS = Comparison->getOperand(1); + if (!isa(LHS) || !isa(RHS)) { + LLVM_DEBUG(dbgs() << "GVN: Not both operands are loads.\n"); + return false; + } + + if (!match(RHS, m_Load(m_GEP(m_GEP(m_Value(BasePtr), m_Value(SExt)), + m_Value(OffsetVal))))) { + LLVM_DEBUG(dbgs() << "GVN: Not a required load pattern.\n"); + return false; + } + // Check if the SExt instruction is a sext instruction. + SExtInst *SEInst = dyn_cast(SExt); + if (!SEInst) { + LLVM_DEBUG(dbgs() << "GVN: not a sext instruction.\n"); + return false; + } + // Check if the "To" and "from" type of the sext instruction are i64 and i32 + // respectively. + if (SEInst->getType() != Builder.getInt64Ty() || + SEInst->getOperand(0)->getType() != Builder.getInt32Ty()) { + LLVM_DEBUG( + dbgs() + << "GVN: Not matching the required type for sext instruction.\n"); + return false; + } + + IndexVal = SEInst->getOperand(0); + // Check if the IndexVal is a PHI node. + PHINode *Phi = dyn_cast(IndexVal); + if (!Phi) { + LLVM_DEBUG(dbgs() << "GVN: IndexVal is not a PHI node\n"); + return false; + } + + LLVM_DEBUG(dbgs() << "GVN: Found minimum finding pattern in Block: " + << Select->getParent()->getName() << ".\n"); + + return transformMinFindingSelectPattern(L, dyn_cast(LHS)->getType(), + Preheader, BB, LHS, RHS, Comparison, + Select, BasePtr, IndexVal, OffsetVal); +} + class llvm::gvn::GVNLegacyPass : public FunctionPass { public: static char ID; // Pass identification, replacement for typeid. diff --git a/llvm/test/Transforms/GVN/PRE/gvn-min-pre.ll b/llvm/test/Transforms/GVN/PRE/gvn-min-pre.ll new file mode 100644 index 0000000000000..1435417f80710 --- /dev/null +++ b/llvm/test/Transforms/GVN/PRE/gvn-min-pre.ll @@ -0,0 +1,724 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 +; RUN: opt -passes=gvn -S < %s | FileCheck %s + +; Test the minimum finding pattern. +; The following test case is extracted from rnflow app in Polyhedron benchmark suite. +define void @test_gvn_min_pattern(ptr %0, i32 %initial_min_idx) { +; CHECK-LABEL: define void @test_gvn_min_pattern( +; CHECK-SAME: ptr [[TMP0:%.*]], i32 [[INITIAL_MIN_IDX:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: [[HOIST_SEXT:%.*]] = sext i32 [[INITIAL_MIN_IDX]] to i64 +; CHECK-NEXT: [[HOIST_GEP1:%.*]] = getelementptr float, ptr [[TMP0]], i64 [[HOIST_SEXT]] +; CHECK-NEXT: [[HOIST_GEP2:%.*]] = getelementptr i8, ptr [[HOIST_GEP1]], i64 -4 +; CHECK-NEXT: [[HOISTED_LOAD:%.*]] = load float, ptr [[HOIST_GEP2]], align 4 +; CHECK-NEXT: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK-NEXT: [[KNOWN_MIN:%.*]] = phi float [ [[HOISTED_LOAD]], %[[ENTRY]] ], [ [[CURRENT_MIN:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[INDVARS_IV_I:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT_I:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[LOOP_COUNTER:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[LOOP_COUNTER_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i32 [ [[INITIAL_MIN_IDX]], %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[INDVARS_IV_NEXT_I]] = add nsw i64 [[INDVARS_IV_I]], -1 +; CHECK-NEXT: [[PTR_FLOAT_IV:%.*]] = getelementptr float, ptr [[TMP0]], i64 [[INDVARS_IV_I]] +; CHECK-NEXT: [[PTR_FIRST_LOAD:%.*]] = getelementptr i8, ptr [[PTR_FLOAT_IV]], i64 -8 +; CHECK-NEXT: [[VAL_FIRST:%.*]] = load float, ptr [[PTR_FIRST_LOAD]], align 4 +; CHECK-NEXT: [[MIN_IDX_EXT:%.*]] = sext i32 [[MIN_IDX]] to i64 +; CHECK-NEXT: [[PTR_FLOAT_MIN:%.*]] = getelementptr float, ptr [[TMP0]], i64 [[MIN_IDX_EXT]] +; CHECK-NEXT: [[PTR_SECOND_LOAD:%.*]] = getelementptr i8, ptr [[PTR_FLOAT_MIN]], i64 -4 +; CHECK-NEXT: [[CMP:%.*]] = fcmp contract olt float [[VAL_FIRST]], [[KNOWN_MIN]] +; CHECK-NEXT: [[NEXT_IDX_TRUNC:%.*]] = trunc nsw i64 [[INDVARS_IV_NEXT_I]] to i32 +; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i32 [[NEXT_IDX_TRUNC]], i32 [[MIN_IDX]] +; CHECK-NEXT: [[LOOP_COUNTER_NEXT]] = add nsw i64 [[LOOP_COUNTER]], -1 +; CHECK-NEXT: [[LOOP_CONTINUE:%.*]] = icmp samesign ugt i64 [[LOOP_COUNTER]], 1 +; CHECK-NEXT: [[CURRENT_MIN]] = select i1 [[CMP]], float [[VAL_FIRST]], float [[KNOWN_MIN]] +; CHECK-NEXT: br i1 [[LOOP_CONTINUE]], label %[[LOOP]], label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +entry: + br label %loop + +loop: ; preds = %loop, %entry + %indvars.iv.i = phi i64 [ 1, %entry ], [ %indvars.iv.next.i, %loop ] + %loop.counter = phi i64 [ 0, %entry ], [ %loop.counter.next, %loop ] + %min.idx = phi i32 [ %initial_min_idx, %entry ], [ %min.idx.next, %loop ] + %indvars.iv.next.i = add nsw i64 %indvars.iv.i, -1 + %ptr.float.iv = getelementptr float, ptr %0, i64 %indvars.iv.i + %ptr.first.load = getelementptr i8, ptr %ptr.float.iv, i64 -8 + %val.first = load float, ptr %ptr.first.load, align 4 + %min.idx.ext = sext i32 %min.idx to i64 + %ptr.float.min = getelementptr float, ptr %0, i64 %min.idx.ext + %ptr.second.load = getelementptr i8, ptr %ptr.float.min, i64 -4 + %val.current.min = load float, ptr %ptr.second.load, align 4 + %cmp = fcmp contract olt float %val.first, %val.current.min + %next.idx.trunc = trunc nsw i64 %indvars.iv.next.i to i32 + %min.idx.next = select i1 %cmp, i32 %next.idx.trunc, i32 %min.idx + %loop.counter.next = add nsw i64 %loop.counter, -1 + %loop.continue = icmp samesign ugt i64 %loop.counter, 1 + br i1 %loop.continue, label %loop, label %exit + +exit: + ret void +} + +; Positive test: Minimum finding pattern with i32 loads. +define void @test_gvn_min_pattern_i32(ptr %arr, i32 %initial_min_idx) { +; CHECK-LABEL: define void @test_gvn_min_pattern_i32( +; CHECK-SAME: ptr [[ARR:%.*]], i32 [[INITIAL_MIN_IDX:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: [[HOIST_SEXT:%.*]] = sext i32 [[INITIAL_MIN_IDX]] to i64 +; CHECK-NEXT: [[HOIST_GEP1:%.*]] = getelementptr i32, ptr [[ARR]], i64 [[HOIST_SEXT]] +; CHECK-NEXT: [[HOIST_GEP2:%.*]] = getelementptr i8, ptr [[HOIST_GEP1]], i64 -4 +; CHECK-NEXT: [[HOISTED_LOAD:%.*]] = load i32, ptr [[HOIST_GEP2]], align 4 +; CHECK-NEXT: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK-NEXT: [[KNOWN_MIN:%.*]] = phi i32 [ [[HOISTED_LOAD]], %[[ENTRY]] ], [ [[CURRENT_MIN:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[INDVARS_IV_I:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT_I:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[LOOP_COUNTER:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[LOOP_COUNTER_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i32 [ [[INITIAL_MIN_IDX]], %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[INDVARS_IV_NEXT_I]] = add nsw i64 [[INDVARS_IV_I]], -1 +; CHECK-NEXT: [[PTR_FLOAT_IV:%.*]] = getelementptr i32, ptr [[ARR]], i64 [[INDVARS_IV_I]] +; CHECK-NEXT: [[PTR_FIRST_LOAD:%.*]] = getelementptr i8, ptr [[PTR_FLOAT_IV]], i64 -8 +; CHECK-NEXT: [[VAL_FIRST:%.*]] = load i32, ptr [[PTR_FIRST_LOAD]], align 4 +; CHECK-NEXT: [[MIN_IDX_EXT:%.*]] = sext i32 [[MIN_IDX]] to i64 +; CHECK-NEXT: [[PTR_FLOAT_MIN:%.*]] = getelementptr i32, ptr [[ARR]], i64 [[MIN_IDX_EXT]] +; CHECK-NEXT: [[PTR_SECOND_LOAD:%.*]] = getelementptr i8, ptr [[PTR_FLOAT_MIN]], i64 -4 +; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[VAL_FIRST]], [[KNOWN_MIN]] +; CHECK-NEXT: [[NEXT_IDX_TRUNC:%.*]] = trunc nsw i64 [[INDVARS_IV_NEXT_I]] to i32 +; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i32 [[NEXT_IDX_TRUNC]], i32 [[MIN_IDX]] +; CHECK-NEXT: [[LOOP_COUNTER_NEXT]] = add nsw i64 [[LOOP_COUNTER]], -1 +; CHECK-NEXT: [[LOOP_CONTINUE:%.*]] = icmp samesign ugt i64 [[LOOP_COUNTER]], 1 +; CHECK-NEXT: [[CURRENT_MIN]] = select i1 [[CMP]], i32 [[VAL_FIRST]], i32 [[KNOWN_MIN]] +; CHECK-NEXT: br i1 [[LOOP_CONTINUE]], label %[[LOOP]], label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +entry: + br label %loop + +loop: ; preds = %loop, %entry + %indvars.iv.i = phi i64 [ 1, %entry ], [ %indvars.iv.next.i, %loop ] + %loop.counter = phi i64 [ 0, %entry ], [ %loop.counter.next, %loop ] + %min.idx = phi i32 [ %initial_min_idx, %entry ], [ %min.idx.next, %loop ] + %indvars.iv.next.i = add nsw i64 %indvars.iv.i, -1 + %ptr.i32.iv = getelementptr i32, ptr %arr, i64 %indvars.iv.i + %ptr.first.load = getelementptr i8, ptr %ptr.i32.iv, i64 -8 + %val.first = load i32, ptr %ptr.first.load, align 4 + %min.idx.ext = sext i32 %min.idx to i64 + %ptr.i32.min = getelementptr i32, ptr %arr, i64 %min.idx.ext + %ptr.second.load = getelementptr i8, ptr %ptr.i32.min, i64 -4 + %val.current.min = load i32, ptr %ptr.second.load, align 4 + %cmp = icmp slt i32 %val.first, %val.current.min + %next.idx.trunc = trunc nsw i64 %indvars.iv.next.i to i32 + %min.idx.next = select i1 %cmp, i32 %next.idx.trunc, i32 %min.idx + %loop.counter.next = add nsw i64 %loop.counter, -1 + %loop.continue = icmp samesign ugt i64 %loop.counter, 1 + br i1 %loop.continue, label %loop, label %exit + +exit: + ret void +} + +; Positive test: Minimum finding pattern with i64 loads. +define void @test_gvn_min_pattern_i64(ptr %0, i32 %initial_min_idx) { +; CHECK-LABEL: define void @test_gvn_min_pattern_i64( +; CHECK-SAME: ptr [[TMP0:%.*]], i32 [[INITIAL_MIN_IDX:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: [[HOIST_SEXT:%.*]] = sext i32 [[INITIAL_MIN_IDX]] to i64 +; CHECK-NEXT: [[HOIST_GEP1:%.*]] = getelementptr i64, ptr [[TMP0]], i64 [[HOIST_SEXT]] +; CHECK-NEXT: [[HOIST_GEP2:%.*]] = getelementptr i8, ptr [[HOIST_GEP1]], i64 -4 +; CHECK-NEXT: [[HOISTED_LOAD:%.*]] = load i64, ptr [[HOIST_GEP2]], align 4 +; CHECK-NEXT: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK-NEXT: [[KNOWN_MIN:%.*]] = phi i64 [ [[HOISTED_LOAD]], %[[ENTRY]] ], [ [[CURRENT_MIN:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[INDVARS_IV_I:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT_I:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[LOOP_COUNTER:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[LOOP_COUNTER_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i32 [ [[INITIAL_MIN_IDX]], %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[INDVARS_IV_NEXT_I]] = add nsw i64 [[INDVARS_IV_I]], -1 +; CHECK-NEXT: [[PTR_I64_IV:%.*]] = getelementptr i64, ptr [[TMP0]], i64 [[INDVARS_IV_I]] +; CHECK-NEXT: [[PTR_FIRST_LOAD:%.*]] = getelementptr i8, ptr [[PTR_I64_IV]], i64 -8 +; CHECK-NEXT: [[VAL_FIRST:%.*]] = load i64, ptr [[PTR_FIRST_LOAD]], align 4 +; CHECK-NEXT: [[MIN_IDX_EXT:%.*]] = sext i32 [[MIN_IDX]] to i64 +; CHECK-NEXT: [[PTR_I64_MIN:%.*]] = getelementptr i64, ptr [[TMP0]], i64 [[MIN_IDX_EXT]] +; CHECK-NEXT: [[PTR_SECOND_LOAD:%.*]] = getelementptr i8, ptr [[PTR_I64_MIN]], i64 -4 +; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[VAL_FIRST]], [[KNOWN_MIN]] +; CHECK-NEXT: [[NEXT_IDX_TRUNC:%.*]] = trunc nsw i64 [[INDVARS_IV_NEXT_I]] to i32 +; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i32 [[NEXT_IDX_TRUNC]], i32 [[MIN_IDX]] +; CHECK-NEXT: [[LOOP_COUNTER_NEXT]] = add nsw i64 [[LOOP_COUNTER]], -1 +; CHECK-NEXT: [[LOOP_CONTINUE:%.*]] = icmp samesign ugt i64 [[LOOP_COUNTER]], 1 +; CHECK-NEXT: [[CURRENT_MIN]] = select i1 [[CMP]], i64 [[VAL_FIRST]], i64 [[KNOWN_MIN]] +; CHECK-NEXT: br i1 [[LOOP_CONTINUE]], label %[[LOOP]], label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +entry: + br label %loop + +loop: ; preds = %loop, %entry + %indvars.iv.i = phi i64 [ 1, %entry ], [ %indvars.iv.next.i, %loop ] + %loop.counter = phi i64 [ 0, %entry ], [ %loop.counter.next, %loop ] + %min.idx = phi i32 [ %initial_min_idx, %entry ], [ %min.idx.next, %loop ] + %indvars.iv.next.i = add nsw i64 %indvars.iv.i, -1 + %ptr.i64.iv = getelementptr i64, ptr %0, i64 %indvars.iv.i + %ptr.first.load = getelementptr i8, ptr %ptr.i64.iv, i64 -8 + %val.first = load i64, ptr %ptr.first.load, align 4 + %min.idx.ext = sext i32 %min.idx to i64 + %ptr.i64.min = getelementptr i64, ptr %0, i64 %min.idx.ext + %ptr.second.load = getelementptr i8, ptr %ptr.i64.min, i64 -4 + %val.current.min = load i64, ptr %ptr.second.load, align 4 + %cmp = icmp slt i64 %val.first, %val.current.min + %next.idx.trunc = trunc nsw i64 %indvars.iv.next.i to i32 + %min.idx.next = select i1 %cmp, i32 %next.idx.trunc, i32 %min.idx + %loop.counter.next = add nsw i64 %loop.counter, -1 + %loop.continue = icmp samesign ugt i64 %loop.counter, 1 + br i1 %loop.continue, label %loop, label %exit + +exit: + ret void +} + +; Negative test: Select not in a loop. +define void @test_not_in_loop(ptr %arr) { +; CHECK-LABEL: define void @test_not_in_loop( +; CHECK-SAME: ptr [[ARR:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: [[LOAD1:%.*]] = load float, ptr [[ARR]], align 4 +; CHECK-NEXT: [[GEP2:%.*]] = getelementptr float, ptr [[ARR]], i64 1 +; CHECK-NEXT: [[LOAD2:%.*]] = load float, ptr [[GEP2]], align 4 +; CHECK-NEXT: [[CMP:%.*]] = fcmp olt float [[LOAD1]], [[LOAD2]] +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], float [[LOAD1]], float [[LOAD2]] +; CHECK-NEXT: ret void +; +entry: + %gep1 = getelementptr float, ptr %arr, i32 0 + %load1 = load float, ptr %gep1 + %sext = sext i32 1 to i64 + %gep2 = getelementptr float, ptr %arr, i64 %sext + %gep3 = getelementptr i8, ptr %gep2, i64 0 + %load2 = load float, ptr %gep3 + %cmp = fcmp olt float %load1, %load2 + %sel = select i1 %cmp, float %load1, float %load2 + ret void +} + +; Negative test: Loop without preheader (multiple entry points). +define void @test_no_preheader(ptr %arr, i32 %n, i1 %cond) { +; CHECK-LABEL: define void @test_no_preheader( +; CHECK-SAME: ptr [[ARR:%.*]], i32 [[N:%.*]], i1 [[COND:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: br i1 [[COND]], label %[[LOOP:.*]], label %[[OTHER_ENTRY:.*]] +; CHECK: [[OTHER_ENTRY]]: +; CHECK-NEXT: br label %[[LOOP]] +; CHECK: [[LOOP]]: +; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ 0, %[[OTHER_ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[ARR]], i32 [[I]] +; CHECK-NEXT: [[SEXT:%.*]] = sext i32 [[I]] to i64 +; CHECK-NEXT: [[GEP2:%.*]] = getelementptr float, ptr [[ARR]], i64 [[SEXT]] +; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1 +; CHECK-NEXT: [[EXIT:%.*]] = icmp slt i32 [[I_NEXT]], [[N]] +; CHECK-NEXT: br i1 [[EXIT]], label %[[LOOP]], label %[[EXIT_BLOCK:.*]] +; CHECK: [[EXIT_BLOCK]]: +; CHECK-NEXT: ret void +; +entry: + br i1 %cond, label %loop, label %other_entry + +other_entry: + br label %loop + +loop: + %i = phi i32 [ 0, %entry ], [ 0, %other_entry ], [ %i.next, %loop ] + %gep1 = getelementptr float, ptr %arr, i32 %i + %load1 = load float, ptr %gep1 + %sext = sext i32 %i to i64 + %gep2 = getelementptr float, ptr %arr, i64 %sext + %gep3 = getelementptr i8, ptr %gep2, i64 0 + %load2 = load float, ptr %gep3 + %cmp = fcmp olt float %load1, %load2 + %sel = select i1 %cmp, float %load1, float %load2 + %i.next = add i32 %i, 1 + %exit = icmp slt i32 %i.next, %n + br i1 %exit, label %loop, label %exit_block + +exit_block: + ret void +} + +; Negative test: Condition is not a CmpInst. +define void @test_condition_not_cmp(ptr %arr, i32 %n, i1 %bool) { +; CHECK-LABEL: define void @test_condition_not_cmp( +; CHECK-SAME: ptr [[ARR:%.*]], i32 [[N:%.*]], i1 [[BOOL:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[ARR]], i32 [[I]] +; CHECK-NEXT: [[SEXT:%.*]] = sext i32 [[I]] to i64 +; CHECK-NEXT: [[GEP2:%.*]] = getelementptr float, ptr [[ARR]], i64 [[SEXT]] +; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1 +; CHECK-NEXT: [[EXIT:%.*]] = icmp slt i32 [[I_NEXT]], [[N]] +; CHECK-NEXT: br i1 [[EXIT]], label %[[LOOP]], label %[[EXIT_BLOCK:.*]] +; CHECK: [[EXIT_BLOCK]]: +; CHECK-NEXT: ret void +; +entry: + br label %loop + +loop: + %i = phi i32 [ 0, %entry ], [ %i.next, %loop ] + %gep1 = getelementptr float, ptr %arr, i32 %i + %load1 = load float, ptr %gep1 + %sext = sext i32 %i to i64 + %gep2 = getelementptr float, ptr %arr, i64 %sext + %gep3 = getelementptr i8, ptr %gep2, i64 0 + %load2 = load float, ptr %gep3 + %sel = select i1 %bool, float %load1, float %load2 + %i.next = add i32 %i, 1 + %exit = icmp slt i32 %i.next, %n + br i1 %exit, label %loop, label %exit_block + +exit_block: + ret void +} + +; Negative test: Wrong comparison predicate (>= instead of <). +define void @test_wrong_predicate(ptr %arr, i32 %n) { +; CHECK-LABEL: define void @test_wrong_predicate( +; CHECK-SAME: ptr [[ARR:%.*]], i32 [[N:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[ARR]], i32 [[I]] +; CHECK-NEXT: [[LOAD1:%.*]] = load float, ptr [[GEP1]], align 4 +; CHECK-NEXT: [[SEXT:%.*]] = sext i32 [[I]] to i64 +; CHECK-NEXT: [[GEP2:%.*]] = getelementptr float, ptr [[ARR]], i64 [[SEXT]] +; CHECK-NEXT: [[CMP:%.*]] = fcmp oge float [[LOAD1]], [[LOAD1]] +; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1 +; CHECK-NEXT: [[EXIT:%.*]] = icmp slt i32 [[I_NEXT]], [[N]] +; CHECK-NEXT: br i1 [[EXIT]], label %[[LOOP]], label %[[EXIT_BLOCK:.*]] +; CHECK: [[EXIT_BLOCK]]: +; CHECK-NEXT: ret void +; +entry: + br label %loop + +loop: + %i = phi i32 [ 0, %entry ], [ %i.next, %loop ] + %gep1 = getelementptr float, ptr %arr, i32 %i + %load1 = load float, ptr %gep1 + %sext = sext i32 %i to i64 + %gep2 = getelementptr float, ptr %arr, i64 %sext + %gep3 = getelementptr i8, ptr %gep2, i64 0 + %load2 = load float, ptr %gep3 + %cmp = fcmp oge float %load1, %load2 + %sel = select i1 %cmp, float %load1, float %load2 + %i.next = add i32 %i, 1 + %exit = icmp slt i32 %i.next, %n + br i1 %exit, label %loop, label %exit_block + +exit_block: + ret void +} + +; Negative test: Not both operands are loads (constant RHS). +define void @test_not_both_loads(ptr %arr, i32 %n) { +; CHECK-LABEL: define void @test_not_both_loads( +; CHECK-SAME: ptr [[ARR:%.*]], i32 [[N:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[ARR]], i32 [[I]] +; CHECK-NEXT: [[LOAD1:%.*]] = load float, ptr [[GEP1]], align 4 +; CHECK-NEXT: [[CMP:%.*]] = fcmp olt float [[LOAD1]], 0.000000e+00 +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], float [[LOAD1]], float 0.000000e+00 +; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1 +; CHECK-NEXT: [[EXIT:%.*]] = icmp slt i32 [[I_NEXT]], [[N]] +; CHECK-NEXT: br i1 [[EXIT]], label %[[LOOP]], label %[[EXIT_BLOCK:.*]] +; CHECK: [[EXIT_BLOCK]]: +; CHECK-NEXT: ret void +; +entry: + br label %loop + +loop: + %i = phi i32 [ 0, %entry ], [ %i.next, %loop ] + %gep1 = getelementptr float, ptr %arr, i32 %i + %load1 = load float, ptr %gep1 + %cmp = fcmp olt float %load1, 0.0 + %sel = select i1 %cmp, float %load1, float 0.0 + %i.next = add i32 %i, 1 + %exit = icmp slt i32 %i.next, %n + br i1 %exit, label %loop, label %exit_block + +exit_block: + ret void +} + +; Negative test: Load doesn't match GEP(GEP(...))nested pattern. +define void @test_wrong_gep_pattern(ptr %arr, i32 %n) { +; CHECK-LABEL: define void @test_wrong_gep_pattern( +; CHECK-SAME: ptr [[ARR:%.*]], i32 [[N:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[ARR]], i32 [[I]] +; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1 +; CHECK-NEXT: [[EXIT:%.*]] = icmp slt i32 [[I_NEXT]], [[N]] +; CHECK-NEXT: br i1 [[EXIT]], label %[[LOOP]], label %[[EXIT_BLOCK:.*]] +; CHECK: [[EXIT_BLOCK]]: +; CHECK-NEXT: ret void +; +entry: + br label %loop + +loop: + %i = phi i32 [ 0, %entry ], [ %i.next, %loop ] + %gep1 = getelementptr float, ptr %arr, i32 %i + %load1 = load float, ptr %gep1 + ; Simple GEP, not nested GEP(GEP(...)) + %gep2 = getelementptr float, ptr %arr, i32 %i + %load2 = load float, ptr %gep2 + %cmp = fcmp olt float %load1, %load2 + %sel = select i1 %cmp, float %load1, float %load2 + %i.next = add i32 %i, 1 + %exit = icmp slt i32 %i.next, %n + br i1 %exit, label %loop, label %exit_block + +exit_block: + ret void +} + +; Negative test: ZExt instead of SExt. +define void @test_no_sext(ptr %arr, i32 %n) { +; CHECK-LABEL: define void @test_no_sext( +; CHECK-SAME: ptr [[ARR:%.*]], i32 [[N:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[ARR]], i32 [[I]] +; CHECK-NEXT: [[LOAD1:%.*]] = load float, ptr [[GEP1]], align 4 +; CHECK-NEXT: [[ZEXT:%.*]] = zext i32 [[I]] to i64 +; CHECK-NEXT: [[GEP2:%.*]] = getelementptr float, ptr [[ARR]], i64 [[ZEXT]] +; CHECK-NEXT: [[LOAD2:%.*]] = load float, ptr [[GEP2]], align 4 +; CHECK-NEXT: [[CMP:%.*]] = fcmp olt float [[LOAD1]], [[LOAD2]] +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], float [[LOAD1]], float [[LOAD2]] +; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1 +; CHECK-NEXT: [[EXIT:%.*]] = icmp slt i32 [[I_NEXT]], [[N]] +; CHECK-NEXT: br i1 [[EXIT]], label %[[LOOP]], label %[[EXIT_BLOCK:.*]] +; CHECK: [[EXIT_BLOCK]]: +; CHECK-NEXT: ret void +; +entry: + br label %loop + +loop: + %i = phi i32 [ 0, %entry ], [ %i.next, %loop ] + %gep1 = getelementptr float, ptr %arr, i32 %i + %load1 = load float, ptr %gep1 + %zext = zext i32 %i to i64 + %gep2 = getelementptr float, ptr %arr, i64 %zext + %gep3 = getelementptr i8, ptr %gep2, i64 0 + %load2 = load float, ptr %gep3 + %cmp = fcmp olt float %load1, %load2 + %sel = select i1 %cmp, float %load1, float %load2 + %i.next = add i32 %i, 1 + %exit = icmp slt i32 %i.next, %n + br i1 %exit, label %loop, label %exit_block + +exit_block: + ret void +} + +; Negative test: SExt with wrong types (i16->i32 instead of i32->i64). +define void @test_wrong_sext_types(ptr %arr, i32 %n) { +; CHECK-LABEL: define void @test_wrong_sext_types( +; CHECK-SAME: ptr [[ARR:%.*]], i32 [[N:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[ARR]], i32 [[I]] +; CHECK-NEXT: [[LOAD1:%.*]] = load float, ptr [[GEP1]], align 4 +; CHECK-NEXT: [[I16:%.*]] = trunc i32 [[I]] to i16 +; CHECK-NEXT: [[SEXT:%.*]] = sext i16 [[I16]] to i32 +; CHECK-NEXT: [[GEP2:%.*]] = getelementptr float, ptr [[ARR]], i32 [[SEXT]] +; CHECK-NEXT: [[LOAD2:%.*]] = load float, ptr [[GEP2]], align 4 +; CHECK-NEXT: [[CMP:%.*]] = fcmp olt float [[LOAD1]], [[LOAD2]] +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], float [[LOAD1]], float [[LOAD2]] +; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1 +; CHECK-NEXT: [[EXIT:%.*]] = icmp slt i32 [[I_NEXT]], [[N]] +; CHECK-NEXT: br i1 [[EXIT]], label %[[LOOP]], label %[[EXIT_BLOCK:.*]] +; CHECK: [[EXIT_BLOCK]]: +; CHECK-NEXT: ret void +; +entry: + br label %loop + +loop: + %i = phi i32 [ 0, %entry ], [ %i.next, %loop ] + %gep1 = getelementptr float, ptr %arr, i32 %i + %load1 = load float, ptr %gep1 + %i16 = trunc i32 %i to i16 + %sext = sext i16 %i16 to i32 + %gep2 = getelementptr float, ptr %arr, i32 %sext + %gep3 = getelementptr i8, ptr %gep2, i32 0 + %load2 = load float, ptr %gep3 + %cmp = fcmp olt float %load1, %load2 + %sel = select i1 %cmp, float %load1, float %load2 + %i.next = add i32 %i, 1 + %exit = icmp slt i32 %i.next, %n + br i1 %exit, label %loop, label %exit_block + +exit_block: + ret void +} + +; Negative test: IndexVal is a constant, not a PHI node. +define void @test_index_is_constant(ptr %arr, i32 %n) { +; CHECK-LABEL: define void @test_index_is_constant( +; CHECK-SAME: ptr [[ARR:%.*]], i32 [[N:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: [[GEP2:%.*]] = getelementptr float, ptr [[ARR]], i64 5 +; CHECK-NEXT: [[LOAD2:%.*]] = load float, ptr [[GEP2]], align 4 +; CHECK-NEXT: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[ARR]], i32 [[I]] +; CHECK-NEXT: [[LOAD1:%.*]] = load float, ptr [[GEP1]], align 4 +; CHECK-NEXT: [[CMP:%.*]] = fcmp olt float [[LOAD1]], [[LOAD2]] +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], float [[LOAD1]], float [[LOAD2]] +; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1 +; CHECK-NEXT: [[EXIT:%.*]] = icmp slt i32 [[I_NEXT]], [[N]] +; CHECK-NEXT: br i1 [[EXIT]], label %[[LOOP]], label %[[EXIT_BLOCK:.*]] +; CHECK: [[EXIT_BLOCK]]: +; CHECK-NEXT: ret void +; +entry: + br label %loop + +loop: + %i = phi i32 [ 0, %entry ], [ %i.next, %loop ] + + %gep1 = getelementptr float, ptr %arr, i32 %i + %load1 = load float, ptr %gep1 + ; IndexVal is a constant (5), not a PHI node! + %sext = sext i32 5 to i64 + %gep2 = getelementptr float, ptr %arr, i64 %sext + %gep3 = getelementptr i8, ptr %gep2, i64 0 + %load2 = load float, ptr %gep3 + %cmp = fcmp olt float %load1, %load2 + %sel = select i1 %cmp, float %load1, float %load2 + %i.next = add i32 %i, 1 + %exit = icmp slt i32 %i.next, %n + br i1 %exit, label %loop, label %exit_block + +exit_block: + ret void +} + +declare void @external_function(ptr) + +; Negative test: Initial min index value is not 1. +define void @test_initial_index_not_one(ptr %arr, i32 %n) { +; CHECK-LABEL: define void @test_initial_index_not_one( +; CHECK-SAME: ptr [[ARR:%.*]], i32 [[N:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[ARR]], i32 [[I]] +; CHECK-NEXT: [[LOAD1:%.*]] = load float, ptr [[GEP1]], align 4 +; CHECK-NEXT: [[SEXT:%.*]] = sext i32 [[MIN_IDX]] to i64 +; CHECK-NEXT: [[GEP2:%.*]] = getelementptr float, ptr [[ARR]], i64 [[SEXT]] +; CHECK-NEXT: [[LOAD2:%.*]] = load float, ptr [[GEP2]], align 4 +; CHECK-NEXT: [[CMP:%.*]] = fcmp olt float [[LOAD1]], [[LOAD2]] +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], float [[LOAD1]], float [[LOAD2]] +; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i32 [[I]], i32 [[MIN_IDX]] +; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1 +; CHECK-NEXT: [[EXIT:%.*]] = icmp slt i32 [[I_NEXT]], [[N]] +; CHECK-NEXT: br i1 [[EXIT]], label %[[LOOP]], label %[[EXIT_BLOCK:.*]] +; CHECK: [[EXIT_BLOCK]]: +; CHECK-NEXT: ret void +; +entry: + br label %loop + +loop: + %i = phi i32 [ 0, %entry ], [ %i.next, %loop ] + ; Initial value is 0, not 1! + %min.idx = phi i32 [ 0, %entry ], [ %min.idx.next, %loop ] + + %gep1 = getelementptr float, ptr %arr, i32 %i + %load1 = load float, ptr %gep1 + %sext = sext i32 %min.idx to i64 + %gep2 = getelementptr float, ptr %arr, i64 %sext + %gep3 = getelementptr i8, ptr %gep2, i64 0 + %load2 = load float, ptr %gep3 + %cmp = fcmp olt float %load1, %load2 + %sel = select i1 %cmp, float %load1, float %load2 + %min.idx.next = select i1 %cmp, i32 %i, i32 %min.idx + %i.next = add i32 %i, 1 + %exit = icmp slt i32 %i.next, %n + br i1 %exit, label %loop, label %exit_block + +exit_block: + ret void +} + +; Negative test: Pattern recognized but hoisting not safe due to aliasing store in loop. +define void @test_aliasing_store_in_loop(ptr %arr, i32 %n) { +; CHECK-LABEL: define void @test_aliasing_store_in_loop( +; CHECK-SAME: ptr [[ARR:%.*]], i32 [[N:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i32 [ 1, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[STORE_GEP:%.*]] = getelementptr float, ptr [[ARR]], i32 [[I]] +; CHECK-NEXT: store float 0.000000e+00, ptr [[STORE_GEP]], align 4 +; CHECK-NEXT: [[SEXT:%.*]] = sext i32 [[MIN_IDX]] to i64 +; CHECK-NEXT: [[GEP2:%.*]] = getelementptr float, ptr [[ARR]], i64 [[SEXT]] +; CHECK-NEXT: [[LOAD2:%.*]] = load float, ptr [[GEP2]], align 4 +; CHECK-NEXT: [[CMP:%.*]] = fcmp olt float 0.000000e+00, [[LOAD2]] +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], float 0.000000e+00, float [[LOAD2]] +; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i32 [[I]], i32 [[MIN_IDX]] +; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1 +; CHECK-NEXT: [[EXIT:%.*]] = icmp slt i32 [[I_NEXT]], [[N]] +; CHECK-NEXT: br i1 [[EXIT]], label %[[LOOP]], label %[[EXIT_BLOCK:.*]] +; CHECK: [[EXIT_BLOCK]]: +; CHECK-NEXT: ret void +; +entry: + br label %loop + +loop: + %i = phi i32 [ 0, %entry ], [ %i.next, %loop ] + %min.idx = phi i32 [ 1, %entry ], [ %min.idx.next, %loop ] + + ; This store aliases with the loads, preventing hoisting. + %store.gep = getelementptr float, ptr %arr, i32 %i + store float 0.0, ptr %store.gep + + %gep1 = getelementptr float, ptr %arr, i32 %i + %load1 = load float, ptr %gep1 + %sext = sext i32 %min.idx to i64 + %gep2 = getelementptr float, ptr %arr, i64 %sext + %gep3 = getelementptr i8, ptr %gep2, i64 0 + %load2 = load float, ptr %gep3 + %cmp = fcmp olt float %load1, %load2 + %sel = select i1 %cmp, float %load1, float %load2 + %min.idx.next = select i1 %cmp, i32 %i, i32 %min.idx + %i.next = add i32 %i, 1 + %exit = icmp slt i32 %i.next, %n + br i1 %exit, label %loop, label %exit_block + +exit_block: + ret void +} + +; Negative test: Pattern recognized but BasePtr is not loop invariant. +define void @test_non_invariant_base_ptr(ptr %arr, i32 %n) { +; CHECK-LABEL: define void @test_non_invariant_base_ptr( +; CHECK-SAME: ptr [[ARR:%.*]], i32 [[N:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i32 [ 1, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[BASE_PTR:%.*]] = phi ptr [ [[ARR]], %[[ENTRY]] ], [ [[BASE_PTR_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[BASE_PTR]], i32 [[I]] +; CHECK-NEXT: [[LOAD1:%.*]] = load float, ptr [[GEP1]], align 4 +; CHECK-NEXT: [[SEXT:%.*]] = sext i32 [[MIN_IDX]] to i64 +; CHECK-NEXT: [[GEP2:%.*]] = getelementptr float, ptr [[BASE_PTR]], i64 [[SEXT]] +; CHECK-NEXT: [[LOAD2:%.*]] = load float, ptr [[GEP2]], align 4 +; CHECK-NEXT: [[CMP:%.*]] = fcmp olt float [[LOAD1]], [[LOAD2]] +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], float [[LOAD1]], float [[LOAD2]] +; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i32 [[I]], i32 [[MIN_IDX]] +; CHECK-NEXT: [[BASE_PTR_NEXT]] = getelementptr float, ptr [[BASE_PTR]], i32 1 +; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1 +; CHECK-NEXT: [[EXIT:%.*]] = icmp slt i32 [[I_NEXT]], [[N]] +; CHECK-NEXT: br i1 [[EXIT]], label %[[LOOP]], label %[[EXIT_BLOCK:.*]] +; CHECK: [[EXIT_BLOCK]]: +; CHECK-NEXT: ret void +; +entry: + br label %loop + +loop: + %i = phi i32 [ 0, %entry ], [ %i.next, %loop ] + %min.idx = phi i32 [ 1, %entry ], [ %min.idx.next, %loop ] + ; Base pointer changes each iteration - not loop invariant! + %base.ptr = phi ptr [ %arr, %entry ], [ %base.ptr.next, %loop ] + + %gep1 = getelementptr float, ptr %base.ptr, i32 %i + %load1 = load float, ptr %gep1 + %sext = sext i32 %min.idx to i64 + %gep2 = getelementptr float, ptr %base.ptr, i64 %sext + %gep3 = getelementptr i8, ptr %gep2, i64 0 + %load2 = load float, ptr %gep3 + %cmp = fcmp olt float %load1, %load2 + %sel = select i1 %cmp, float %load1, float %load2 + %min.idx.next = select i1 %cmp, i32 %i, i32 %min.idx + %base.ptr.next = getelementptr float, ptr %base.ptr, i32 1 + %i.next = add i32 %i, 1 + %exit = icmp slt i32 %i.next, %n + br i1 %exit, label %loop, label %exit_block + +exit_block: + ret void +} + +; Negative test: Pattern recognized but may-alias call in loop. +define void @test_may_alias_call_in_loop(ptr %arr, i32 %n) { +; CHECK-LABEL: define void @test_may_alias_call_in_loop( +; CHECK-SAME: ptr [[ARR:%.*]], i32 [[N:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i32 [ 1, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: call void @external_function(ptr [[ARR]]) +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[ARR]], i32 [[I]] +; CHECK-NEXT: [[LOAD1:%.*]] = load float, ptr [[GEP1]], align 4 +; CHECK-NEXT: [[SEXT:%.*]] = sext i32 [[MIN_IDX]] to i64 +; CHECK-NEXT: [[GEP2:%.*]] = getelementptr float, ptr [[ARR]], i64 [[SEXT]] +; CHECK-NEXT: [[LOAD2:%.*]] = load float, ptr [[GEP2]], align 4 +; CHECK-NEXT: [[CMP:%.*]] = fcmp olt float [[LOAD1]], [[LOAD2]] +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], float [[LOAD1]], float [[LOAD2]] +; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i32 [[I]], i32 [[MIN_IDX]] +; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1 +; CHECK-NEXT: [[EXIT:%.*]] = icmp slt i32 [[I_NEXT]], [[N]] +; CHECK-NEXT: br i1 [[EXIT]], label %[[LOOP]], label %[[EXIT_BLOCK:.*]] +; CHECK: [[EXIT_BLOCK]]: +; CHECK-NEXT: ret void +; +entry: + br label %loop + +loop: + %i = phi i32 [ 0, %entry ], [ %i.next, %loop ] + %min.idx = phi i32 [ 1, %entry ], [ %min.idx.next, %loop ] + + ; External function may modify memory. + call void @external_function(ptr %arr) + + %gep1 = getelementptr float, ptr %arr, i32 %i + %load1 = load float, ptr %gep1 + %sext = sext i32 %min.idx to i64 + %gep2 = getelementptr float, ptr %arr, i64 %sext + %gep3 = getelementptr i8, ptr %gep2, i64 0 + %load2 = load float, ptr %gep3 + %cmp = fcmp olt float %load1, %load2 + %sel = select i1 %cmp, float %load1, float %load2 + %min.idx.next = select i1 %cmp, i32 %i, i32 %min.idx + %i.next = add i32 %i, 1 + %exit = icmp slt i32 %i.next, %n + br i1 %exit, label %loop, label %exit_block + +exit_block: + ret void +}