Skip to content

Commit be660f5

Browse files
fhahnfhahn
committed
[VPlan] Truncate/Extend ComputeReductionResult at construction (NFC).
Instead of looking up the narrower reduction type via getRecurrenceType we can generate the needed extend directly at constructiond re-use the truncated value from the loop.
1 parent e29bb9a commit be660f5

File tree

6 files changed

+36
-55
lines changed

6 files changed

+36
-55
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp‎

Lines changed: 29 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -7259,7 +7259,10 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
72597259
// Get the VPInstruction computing the reduction result in the middle block.
72607260
// The first operand may not be from the middle block if it is not connected
72617261
// to the scalar preheader. In that case, there's nothing to fix.
7262-
auto *EpiRedResult = dyn_cast<VPInstruction>(EpiResumePhiR->getOperand(0));
7262+
VPValue *Incoming = EpiResumePhiR->getOperand(0);
7263+
match(Incoming, VPlanPatternMatch::m_ZExtOrSExt(
7264+
VPlanPatternMatch::m_VPValue(Incoming)));
7265+
auto *EpiRedResult = dyn_cast<VPInstruction>(Incoming);
72637266
if (!EpiRedResult ||
72647267
(EpiRedResult->getOpcode() != VPInstruction::ComputeAnyOfResult &&
72657268
EpiRedResult->getOpcode() != VPInstruction::ComputeReductionResult &&
@@ -9195,28 +9198,6 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
91959198
PhiR->setOperand(1, NewExitingVPV);
91969199
}
91979200

9198-
// If the vector reduction can be performed in a smaller type, we truncate
9199-
// then extend the loop exit value to enable InstCombine to evaluate the
9200-
// entire expression in the smaller type.
9201-
if (MinVF.isVector() && PhiTy != RdxDesc.getRecurrenceType() &&
9202-
!RecurrenceDescriptor::isAnyOfRecurrenceKind(
9203-
RdxDesc.getRecurrenceKind())) {
9204-
assert(!PhiR->isInLoop() && "Unexpected truncated inloop reduction!");
9205-
Type *RdxTy = RdxDesc.getRecurrenceType();
9206-
auto *Trunc =
9207-
new VPWidenCastRecipe(Instruction::Trunc, NewExitingVPV, RdxTy);
9208-
auto *Extnd =
9209-
RdxDesc.isSigned()
9210-
? new VPWidenCastRecipe(Instruction::SExt, Trunc, PhiTy)
9211-
: new VPWidenCastRecipe(Instruction::ZExt, Trunc, PhiTy);
9212-
9213-
Trunc->insertAfter(NewExitingVPV->getDefiningRecipe());
9214-
Extnd->insertAfter(Trunc);
9215-
if (PhiR->getOperand(1) == NewExitingVPV)
9216-
PhiR->setOperand(1, Extnd->getVPSingleValue());
9217-
NewExitingVPV = Extnd;
9218-
}
9219-
92209201
// We want code in the middle block to appear to execute on the location of
92219202
// the scalar loop's latch terminator because: (a) it is all compiler
92229203
// generated, (b) these instructions are always executed after evaluating
@@ -9255,6 +9236,31 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
92559236
Builder.createNaryOp(VPInstruction::ComputeReductionResult,
92569237
{PhiR, NewExitingVPV}, Flags, ExitDL);
92579238
}
9239+
// If the vector reduction can be performed in a smaller type, we truncate
9240+
// then extend the loop exit value to enable InstCombine to evaluate the
9241+
// entire expression in the smaller type.
9242+
if (MinVF.isVector() && PhiTy != RdxDesc.getRecurrenceType() &&
9243+
!RecurrenceDescriptor::isAnyOfRecurrenceKind(
9244+
RdxDesc.getRecurrenceKind())) {
9245+
assert(!PhiR->isInLoop() && "Unexpected truncated inloop reduction!");
9246+
Type *RdxTy = RdxDesc.getRecurrenceType();
9247+
auto *Trunc =
9248+
new VPWidenCastRecipe(Instruction::Trunc, NewExitingVPV, RdxTy);
9249+
Instruction::CastOps ExtendOpc =
9250+
RdxDesc.isSigned() ? Instruction::SExt : Instruction::ZExt;
9251+
auto *Extnd = new VPWidenCastRecipe(ExtendOpc, Trunc, PhiTy);
9252+
Trunc->insertAfter(NewExitingVPV->getDefiningRecipe());
9253+
Extnd->insertAfter(Trunc);
9254+
if (PhiR->getOperand(1) == NewExitingVPV)
9255+
PhiR->setOperand(1, Extnd->getVPSingleValue());
9256+
9257+
// Update ComputeReductionResult with the truncated exiting value and
9258+
// extend its result.
9259+
FinalReductionResult->setOperand(1, Trunc);
9260+
FinalReductionResult =
9261+
Builder.createScalarCast(ExtendOpc, FinalReductionResult, PhiTy, {});
9262+
}
9263+
92589264
// Update all users outside the vector region.
92599265
OrigExitingVPV->replaceUsesWithIf(
92609266
FinalReductionResult, [FinalReductionResult](VPUser &User, unsigned) {

‎llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp‎

Lines changed: 0 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -672,7 +672,6 @@ Value *VPInstruction::generate(VPTransformState &State) {
672672
assert(!RecurrenceDescriptor::isFindLastIVRecurrenceKind(RK) &&
673673
"should be handled by ComputeFindLastIVResult");
674674

675-
Type *ResultTy = State.TypeAnalysis.inferScalarType(this);
676675
// The recipe's operands are the reduction phi, followed by one operand for
677676
// each part of the reduction.
678677
unsigned UF = getNumOperands() - 1;
@@ -684,15 +683,6 @@ Value *VPInstruction::generate(VPTransformState &State) {
684683
if (hasFastMathFlags())
685684
Builder.setFastMathFlags(getFastMathFlags());
686685

687-
// If the vector reduction can be performed in a smaller type, we truncate
688-
// then extend the loop exit value to enable InstCombine to evaluate the
689-
// entire expression in the smaller type.
690-
// TODO: Handle this in truncateToMinBW.
691-
if (State.VF.isVector() && ResultTy != RdxDesc.getRecurrenceType()) {
692-
Type *RdxVecTy = VectorType::get(RdxDesc.getRecurrenceType(), State.VF);
693-
for (unsigned Part = 0; Part < UF; ++Part)
694-
RdxParts[Part] = Builder.CreateTrunc(RdxParts[Part], RdxVecTy);
695-
}
696686
// Reduce all of the unrolled parts into a single vector.
697687
Value *ReducedPartRdx = RdxParts[0];
698688
if (PhiR->isOrdered()) {
@@ -717,13 +707,6 @@ Value *VPInstruction::generate(VPTransformState &State) {
717707
// All ops in the reduction inherit fast-math-flags from the recurrence
718708
// descriptor.
719709
ReducedPartRdx = createSimpleReduction(Builder, ReducedPartRdx, RK);
720-
721-
// If the reduction can be performed in a smaller type, we need to extend
722-
// the reduction to the wider type before we branch to the original loop.
723-
if (ResultTy != RdxDesc.getRecurrenceType())
724-
ReducedPartRdx = RdxDesc.isSigned()
725-
? Builder.CreateSExt(ReducedPartRdx, ResultTy)
726-
: Builder.CreateZExt(ReducedPartRdx, ResultTy);
727710
}
728711

729712
return ReducedPartRdx;

‎llvm/test/Transforms/LoopVectorize/X86/cost-model.ll‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1167,8 +1167,7 @@ define i32 @narrowed_reduction(ptr %a, i1 %cmp) #0 {
11671167
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
11681168
; CHECK-NEXT: br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
11691169
; CHECK: middle.block:
1170-
; CHECK-NEXT: [[TMP10:%.*]] = trunc <16 x i32> [[TMP7]] to <16 x i1>
1171-
; CHECK-NEXT: [[TMP20:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP10]])
1170+
; CHECK-NEXT: [[TMP20:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP5]])
11721171
; CHECK-NEXT: [[TMP21:%.*]] = zext i1 [[TMP20]] to i32
11731172
; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[VEC_EPILOG_PH]]
11741173
; CHECK: scalar.ph:

‎llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -208,8 +208,7 @@ define i16 @reduction_or_trunc(ptr noalias nocapture %ptr) {
208208
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
209209
; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
210210
; CHECK: middle.block:
211-
; CHECK-NEXT: [[TMP9:%.*]] = trunc <4 x i32> [[TMP7]] to <4 x i16>
212-
; CHECK-NEXT: [[TMP10:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP9]])
211+
; CHECK-NEXT: [[TMP10:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP6]])
213212
; CHECK-NEXT: [[TMP11:%.*]] = zext i16 [[TMP10]] to i32
214213
; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
215214
; CHECK: vec.epilog.iter.check:
@@ -234,8 +233,7 @@ define i16 @reduction_or_trunc(ptr noalias nocapture %ptr) {
234233
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT4]], 256
235234
; CHECK-NEXT: br i1 [[TMP21]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
236235
; CHECK: vec.epilog.middle.block:
237-
; CHECK-NEXT: [[TMP22:%.*]] = trunc <4 x i32> [[TMP20]] to <4 x i16>
238-
; CHECK-NEXT: [[TMP23:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP22]])
236+
; CHECK-NEXT: [[TMP23:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP19]])
239237
; CHECK-NEXT: [[TMP24:%.*]] = zext i16 [[TMP23]] to i32
240238
; CHECK-NEXT: br i1 true, label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
241239
; CHECK: vec.epilog.scalar.ph:

‎llvm/test/Transforms/LoopVectorize/reduction-small-size.ll‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,7 @@ define i8 @PR34687(i1 %c, i32 %x, i32 %n) {
2525
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
2626
; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
2727
; CHECK: middle.block:
28-
; CHECK-NEXT: [[TMP6:%.*]] = trunc <4 x i32> [[TMP4]] to <4 x i8>
29-
; CHECK-NEXT: [[TMP7:%.*]] = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> [[TMP6]])
28+
; CHECK-NEXT: [[TMP7:%.*]] = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> [[TMP3]])
3029
; CHECK-NEXT: [[TMP8:%.*]] = zext i8 [[TMP7]] to i32
3130
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
3231
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -104,8 +103,7 @@ define i8 @PR34687_no_undef(i1 %c, i32 %x, i32 %n) {
104103
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
105104
; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
106105
; CHECK: middle.block:
107-
; CHECK-NEXT: [[TMP8:%.*]] = trunc <4 x i32> [[TMP6]] to <4 x i8>
108-
; CHECK-NEXT: [[TMP9:%.*]] = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> [[TMP8]])
106+
; CHECK-NEXT: [[TMP9:%.*]] = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> [[TMP5]])
109107
; CHECK-NEXT: [[TMP10:%.*]] = zext i8 [[TMP9]] to i32
110108
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
111109
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -183,8 +181,7 @@ define i32 @PR35734(i32 %x, i32 %y) {
183181
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
184182
; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
185183
; CHECK: middle.block:
186-
; CHECK-NEXT: [[TMP8:%.*]] = trunc <4 x i32> [[TMP6]] to <4 x i1>
187-
; CHECK-NEXT: [[TMP9:%.*]] = call i1 @llvm.vector.reduce.add.v4i1(<4 x i1> [[TMP8]])
184+
; CHECK-NEXT: [[TMP9:%.*]] = call i1 @llvm.vector.reduce.add.v4i1(<4 x i1> [[TMP5]])
188185
; CHECK-NEXT: [[TMP10:%.*]] = sext i1 [[TMP9]] to i32
189186
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC]]
190187
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]

‎llvm/test/Transforms/LoopVectorize/scalable-reduction-inloop.ll‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,9 +43,7 @@ define i8 @reduction_add_trunc(ptr noalias nocapture %A) {
4343
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
4444
; CHECK-NEXT: br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
4545
; CHECK: middle.block:
46-
; CHECK-NEXT: [[TMP37:%.*]] = trunc <vscale x 8 x i32> [[TMP34]] to <vscale x 8 x i8>
47-
; CHECK-NEXT: [[TMP38:%.*]] = trunc <vscale x 8 x i32> [[TMP36]] to <vscale x 8 x i8>
48-
; CHECK-NEXT: [[BIN_RDX:%.*]] = add <vscale x 8 x i8> [[TMP38]], [[TMP37]]
46+
; CHECK-NEXT: [[BIN_RDX:%.*]] = add <vscale x 8 x i8> [[TMP35]], [[TMP33]]
4947
; CHECK-NEXT: [[TMP39:%.*]] = call i8 @llvm.vector.reduce.add.nxv8i8(<vscale x 8 x i8> [[BIN_RDX]])
5048
; CHECK-NEXT: [[TMP40:%.*]] = zext i8 [[TMP39]] to i32
5149
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 256, [[N_VEC]]

0 commit comments

Comments
 (0)