[GlobalISel][Legalizer] Fix legalization of llvm.smul.with.overflow

Previously the code for handling G_SMULO didn't properly check for the signed
multiply overflow, instead treating it the same as the unsigned G_UMULO.

Fixes PR35800.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@321690 91177308-0d34-0410-b5e6-96231b3b80d8

lib/CodeGen/GlobalISel/LegalizerHelper.cpp

@@ -813,7 +813,21 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
 
     unsigned Zero = MRI.createGenericVirtualRegister(Ty);
     MIRBuilder.buildConstant(Zero, 0);
-    MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Zero);
+
+    // For *signed* multiply, overflow is detected by checking:
+    // (hi != (lo >> bitwidth-1))
+    if (Opcode == TargetOpcode::G_SMULH) {
+      unsigned Shifted = MRI.createGenericVirtualRegister(Ty);
+      unsigned ShiftAmt = MRI.createGenericVirtualRegister(Ty);
+      MIRBuilder.buildConstant(ShiftAmt, Ty.getSizeInBits() - 1);
+      MIRBuilder.buildInstr(TargetOpcode::G_ASHR)
+        .addDef(Shifted)
+        .addUse(Res)
+        .addUse(ShiftAmt);
+      MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Shifted);
+    } else {
+      MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Zero);
+    }
     MI.eraseFromParent();
     return Legalized;
   }

test/CodeGen/AArch64/GlobalISel/legalize-mul.mir

@@ -8,7 +8,12 @@
   entry:
     ret void
   }
-  define void @test_mul_overflow() { ret void }
+  define void @test_smul_overflow() {
+    ret void
+  }
+  define void @test_umul_overflow() {
+    ret void
+  }
 ...
 
 ---
@@ -43,18 +48,19 @@ body: |
 
 
 ---
-name:            test_mul_overflow
+name:            test_smul_overflow
 body: |
   bb.0:
     liveins: %x0, %x1, %w2, %w3
 
-    ; CHECK-LABEL: name: test_mul_overflow
+    ; CHECK-LABEL: name: test_smul_overflow
     ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY %x0
     ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY %x1
     ; CHECK: [[MUL:%[0-9]+]]:_(s64) = G_MUL [[COPY]], [[COPY1]]
     ; CHECK: [[SMULH:%[0-9]+]]:_(s64) = G_SMULH [[COPY]], [[COPY1]]
-    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-    ; CHECK: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[SMULH]](s64), [[C]]
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 63
+    ; CHECK: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[MUL]], [[C]]
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[SMULH]](s64), [[ASHR]]
     ; CHECK: %x0 = COPY [[MUL]](s64)
     ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[ICMP]](s32)
     ; CHECK: %w0 = COPY [[COPY2]](s32)
@@ -66,3 +72,29 @@ body: |
     %w0 = COPY %4
 
 ...
+
+
+---
+name:            test_umul_overflow
+body: |
+  bb.0:
+    liveins: %x0, %x1, %w2, %w3
+
+    ; CHECK-LABEL: name: test_umul_overflow
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY %x0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY %x1
+    ; CHECK: [[MUL:%[0-9]+]]:_(s64) = G_MUL [[COPY]], [[COPY1]]
+    ; CHECK: [[UMULH:%[0-9]+]]:_(s64) = G_UMULH [[COPY]], [[COPY1]]
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[UMULH]](s64), [[C]]
+    ; CHECK: %x0 = COPY [[MUL]](s64)
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[ICMP]](s32)
+    ; CHECK: %w0 = COPY [[COPY2]](s32)
+    %0:_(s64) = COPY %x0
+    %1:_(s64) = COPY %x1
+    %2:_(s64), %3:_(s1) = G_UMULO %0, %1
+    %x0 = COPY %2
+    %4:_(s32) = G_ANYEXT %3
+    %w0 = COPY %4
+
+...