Starting with CRC pattern matcher!

clang/include/clang/Basic/BuiltinsPPC.def

@@ -924,6 +924,7 @@ TARGET_BUILTIN(__builtin_pextd, "ULLiULLiULLi", "", "isa-v31-instructions")
 TARGET_BUILTIN(__builtin_cfuged, "ULLiULLiULLi", "", "isa-v31-instructions")
 TARGET_BUILTIN(__builtin_cntlzdm, "ULLiULLiULLi", "", "isa-v31-instructions")
 TARGET_BUILTIN(__builtin_cnttzdm, "ULLiULLiULLi", "", "isa-v31-instructions")
+//TARGET_BUILTIN(__builtin_cnrcdm, "ULLiULLiULLi", "", "isa-v31-instructions")
 
 // Double-double (un)pack
 BUILTIN(__builtin_unpack_longdouble, "dLdIi", "")

llvm/include/llvm/CodeGen/ISDOpcodes.h

@@ -713,13 +713,17 @@ enum NodeType {
   /// Byte Swap and Counting operators.
   BSWAP,
   CTTZ,
+  CRC,
+  CRC8,
+  CRC32,
   CTLZ,
   CTPOP,
   BITREVERSE,
   PARITY,
 
   /// Bit counting operators with an undefined result for zero inputs.
   CTTZ_ZERO_UNDEF,
+  CRC_ZERO_UNDEF,
   CTLZ_ZERO_UNDEF,
 
   /// Select(COND, TRUEVAL, FALSEVAL).  If the type of the boolean COND is not

llvm/include/llvm/CodeGen/TargetLowering.h

@@ -5060,11 +5060,13 @@ class TargetLowering : public TargetLoweringBase {
   /// \returns The expansion result or SDValue() if it fails.
   SDValue expandCTTZ(SDNode *N, SelectionDAG &DAG) const;
 
+  SDValue expandCRC(SDNode *N, SelectionDAG &DAG) const;
   /// Expand VP_CTTZ/VP_CTTZ_ZERO_UNDEF nodes.
   /// \param N Node to expand
   /// \returns The expansion result or SDValue() if it fails.
   SDValue expandVPCTTZ(SDNode *N, SelectionDAG &DAG) const;
-
+
+  SDValue expandVPCRC(SDNode *N, SelectionDAG &DAG) const;
   /// Expand ABS nodes. Expands vector/scalar ABS nodes,
   /// vector nodes can only succeed if all operations are legal/custom.
   /// (ABS x) -> (XOR (ADD x, (SRA x, type_size)), (SRA x, type_size))

llvm/include/llvm/IR/Intrinsics.td

@@ -1305,6 +1305,12 @@ let IntrProperties = [IntrNoMem, IntrSpeculatable, IntrWillReturn,
                       ImmArg<ArgIndex<1>>] in {
   def int_ctlz : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, llvm_i1_ty]>;
   def int_cttz : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, llvm_i1_ty]>;
+  def int_crc : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, llvm_i1_ty]>;
+
+  def int_crc32 : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>]>;
+}
+let IntrProperties = [IntrNoMem, IntrSpeculatable, IntrWillReturn] in {
+  def int_crc8 : DefaultAttrsIntrinsic<[llvm_i16_ty], [llvm_i8_ty, llvm_i16_ty]>;
 }
 
 //===------------------------ Debugger Intrinsics -------------------------===//
@@ -1897,6 +1903,7 @@ let IntrProperties = [IntrNoMem, IntrNoSync, IntrWillReturn] in {
                              [ LLVMMatchType<0>,
                                LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                                llvm_i32_ty]>;
+
   def int_vp_fshl : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ],
                              [ LLVMMatchType<0>,
                                LLVMMatchType<0>,
@@ -2154,6 +2161,17 @@ let IntrProperties = [IntrNoMem, IntrNoSync, IntrWillReturn, ImmArg<ArgIndex<1>>
                                llvm_i1_ty,
                                LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                                llvm_i32_ty]>;
+  def int_vp_crc : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ],
+                             [ LLVMMatchType<0>,
+                               llvm_i1_ty,
+                               LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
+                               llvm_i32_ty]>;  
+  def int_vp_crc32 : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ],
+                             [ LLVMMatchType<0>,
+                               LLVMMatchType<0>,
+                               LLVMMatchType<0>,
+                               LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
+                               llvm_i32_ty]>;                                                        
 }
 
 def int_get_active_lane_mask:

llvm/include/llvm/IR/IntrinsicsRISCV.td

@@ -106,6 +106,12 @@ let TargetPrefix = "riscv" in {
   // Zbkx
   def int_riscv_xperm4  : BitManipGPRGPRIntrinsics;
   def int_riscv_xperm8  : BitManipGPRGPRIntrinsics;
+
+  //CRC - we will check this later!
+  //def int_riscv_crc: BitManipGPRIntrinsics;
+  //def int_ctlz : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, llvm_i1_ty]>;
+  //def int_cttz : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, llvm_i1_ty]>;
+
 } // TargetPrefix = "riscv"
 
 //===----------------------------------------------------------------------===//

llvm/include/llvm/Target/TargetSelectionDAG.td

@@ -143,7 +143,7 @@ def SDTFPTernaryOp : SDTypeProfile<1, 3, [  // fmadd, fnmsub, etc.
 def SDTIntUnaryOp : SDTypeProfile<1, 1, [ // bitreverse
   SDTCisSameAs<0, 1>, SDTCisInt<0>
 ]>;
-def SDTIntBitCountUnaryOp : SDTypeProfile<1, 1, [   // ctlz, cttz
+def SDTIntBitCountUnaryOp : SDTypeProfile<1, 1, [   // ctlz, cttz, crc
   SDTCisInt<0>, SDTCisInt<1>
 ]>;
 def SDTIntExtendOp : SDTypeProfile<1, 1, [  // sext, zext, anyext
@@ -453,9 +453,11 @@ def bitreverse : SDNode<"ISD::BITREVERSE" , SDTIntUnaryOp>;
 def bswap      : SDNode<"ISD::BSWAP"      , SDTIntUnaryOp>;
 def ctlz       : SDNode<"ISD::CTLZ"       , SDTIntBitCountUnaryOp>;
 def cttz       : SDNode<"ISD::CTTZ"       , SDTIntBitCountUnaryOp>;
+def crc        : SDNode<"ISD::CRC"        , SDTIntBitCountUnaryOp>;
 def ctpop      : SDNode<"ISD::CTPOP"      , SDTIntBitCountUnaryOp>;
 def ctlz_zero_undef : SDNode<"ISD::CTLZ_ZERO_UNDEF", SDTIntBitCountUnaryOp>;
 def cttz_zero_undef : SDNode<"ISD::CTTZ_ZERO_UNDEF", SDTIntBitCountUnaryOp>;
+def crc_zero_undef  : SDNode<"ISD::CRC_ZERO_UNDEF" , SDTIntBitCountUnaryOp>;
 def sext       : SDNode<"ISD::SIGN_EXTEND", SDTIntExtendOp>;
 def zext       : SDNode<"ISD::ZERO_EXTEND", SDTIntExtendOp>;
 def anyext     : SDNode<"ISD::ANY_EXTEND" , SDTIntExtendOp>;

llvm/lib/Analysis/ConstantFolding.cpp

@@ -1527,6 +1527,8 @@ bool llvm::canConstantFoldCallTo(const CallBase *Call, const Function *F) {
   case Intrinsic::ctpop:
   case Intrinsic::ctlz:
   case Intrinsic::cttz:
+  case Intrinsic::crc:
+  case Intrinsic::crc32:
   case Intrinsic::fshl:
   case Intrinsic::fshr:
   case Intrinsic::launder_invariant_group:
@@ -2828,6 +2830,7 @@ static Constant *ConstantFoldScalarCall2(StringRef Name,
       else
         return ConstantInt::get(Ty, C0->ssub_sat(*C1));
     case Intrinsic::cttz:
+    case Intrinsic::crc:
     case Intrinsic::ctlz:
       assert(C1 && "Must be constant int");
 
@@ -2836,6 +2839,8 @@ static Constant *ConstantFoldScalarCall2(StringRef Name,
         return PoisonValue::get(Ty);
       if (!C0)
         return Constant::getNullValue(Ty);
+      if (IntrinsicID == Intrinsic::crc)
+        return ConstantInt::get(Ty, C0->countr_zero());
       if (IntrinsicID == Intrinsic::cttz)
         return ConstantInt::get(Ty, C0->countr_zero());
       else

llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

@@ -468,6 +468,8 @@ namespace {
     SDValue visitCTLZ_ZERO_UNDEF(SDNode *N);
     SDValue visitCTTZ(SDNode *N);
     SDValue visitCTTZ_ZERO_UNDEF(SDNode *N);
+    //SDValue visitCRC(SDNode *N);
+    //SDValue visitCRC_ZERO_UNDEF(SDNode *N);
     SDValue visitCTPOP(SDNode *N);
     SDValue visitSELECT(SDNode *N);
     SDValue visitVSELECT(SDNode *N);
@@ -1938,6 +1940,8 @@ SDValue DAGCombiner::visit(SDNode *N) {
   case ISD::CTLZ_ZERO_UNDEF:    return visitCTLZ_ZERO_UNDEF(N);
   case ISD::CTTZ:               return visitCTTZ(N);
   case ISD::CTTZ_ZERO_UNDEF:    return visitCTTZ_ZERO_UNDEF(N);
+  //case ISD::CRC:                return visitCTTZ(N); //return visitCRC(N);
+  //case ISD::CRC_ZERO_UNDEF:     return visitCTTZ_ZERO_UNDEF(N); //return visitCRC_ZERO_UNDEF(N);
   case ISD::CTPOP:              return visitCTPOP(N);
   case ISD::SELECT:             return visitSELECT(N);
   case ISD::VSELECT:            return visitVSELECT(N);
@@ -10910,7 +10914,24 @@ SDValue DAGCombiner::visitCTTZ(SDNode *N) {
 
   return SDValue();
 }
+/*
+SDValue DAGCombiner::visitCRC(SDNode *N) {
+  SDValue N0 = N->getOperand(0);
+  EVT VT = N->getValueType(0);
+
+  // fold (cttz c1) -> c2
+  if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
+    return DAG.getNode(ISD::CTTZ, SDLoc(N), VT, N0);
 
+  // If the value is known never to be zero, switch to the undef version.
+  if (!LegalOperations || TLI.isOperationLegal(ISD::CTTZ_ZERO_UNDEF, VT)) {
+    if (DAG.isKnownNeverZero(N0))
+      return DAG.getNode(ISD::CTTZ_ZERO_UNDEF, SDLoc(N), VT, N0);
+  }
+
+  return SDValue();
+}
+*/
 SDValue DAGCombiner::visitCTTZ_ZERO_UNDEF(SDNode *N) {
   SDValue N0 = N->getOperand(0);
   EVT VT = N->getValueType(0);
@@ -10920,7 +10941,17 @@ SDValue DAGCombiner::visitCTTZ_ZERO_UNDEF(SDNode *N) {
     return DAG.getNode(ISD::CTTZ_ZERO_UNDEF, SDLoc(N), VT, N0);
   return SDValue();
 }
+/*
+SDValue DAGCombiner::visitCRC_ZERO_UNDEF(SDNode *N) {
+  SDValue N0 = N->getOperand(0);
+  EVT VT = N->getValueType(0);
 
+  // fold (cttz_zero_undef c1) -> c2
+  if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
+    return DAG.getNode(ISD::CTTZ_ZERO_UNDEF, SDLoc(N), VT, N0);
+  return SDValue();
+}
+*/
 SDValue DAGCombiner::visitCTPOP(SDNode *N) {
   SDValue N0 = N->getOperand(0);
   EVT VT = N->getValueType(0);

llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp

@@ -2833,6 +2833,12 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
     if ((Tmp1 = TLI.expandCTTZ(Node, DAG)))
       Results.push_back(Tmp1);
     break;
+  case ISD::CRC32:
+  case ISD::CRC:
+  case ISD::CRC_ZERO_UNDEF:
+    if ((Tmp1 = TLI.expandCRC(Node, DAG)))
+      Results.push_back(Tmp1);
+    break;  
   case ISD::BITREVERSE:
     if ((Tmp1 = TLI.expandBITREVERSE(Node, DAG)))
       Results.push_back(Tmp1);
@@ -4668,17 +4674,19 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
   switch (Node->getOpcode()) {
   case ISD::CTTZ:
   case ISD::CTTZ_ZERO_UNDEF:
+  //case ISD::CRC:
+  //case ISD::CRC_ZERO_UNDEF:
   case ISD::CTLZ:
   case ISD::CTLZ_ZERO_UNDEF:
   case ISD::CTPOP:
     // Zero extend the argument unless its cttz, then use any_extend.
-    if (Node->getOpcode() == ISD::CTTZ ||
+    if (Node->getOpcode() == ISD::CRC || Node->getOpcode() == ISD::CRC_ZERO_UNDEF || Node->getOpcode() == ISD::CTTZ ||
         Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF)
       Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0));
     else
       Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
 
-    if (Node->getOpcode() == ISD::CTTZ) {
+    if (Node->getOpcode() == ISD::CRC || Node->getOpcode() == ISD::CTTZ) {
       // The count is the same in the promoted type except if the original
       // value was zero.  This can be handled by setting the bit just off
       // the top of the original type.

llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp

@@ -3481,6 +3481,18 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
     Known.Zero.setBitsFrom(LowBits);
     break;
   }
+  /*
+  case ISD::CRC:
+  case ISD::CRC_ZERO_UNDEF: {
+    Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
+    // If we have a known 1, its position is our upper bound.
+    unsigned PossibleTZ = Known2.countMaxTrailingZeros();
+    unsigned LowBits = llvm::bit_width(PossibleTZ);
+    Known.Zero.setBitsFrom(LowBits);
+    //Known.
+    break;
+  }
+  */
   case ISD::CTLZ:
   case ISD::CTLZ_ZERO_UNDEF: {
     Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
@@ -5404,6 +5416,11 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
     case ISD::CTTZ_ZERO_UNDEF:
       return getConstant(Val.countr_zero(), DL, VT, C->isTargetOpcode(),
                          C->isOpaque());
+    case ISD::CRC32:
+    case ISD::CRC:
+    case ISD::CRC_ZERO_UNDEF:
+      return getConstant(Val.countr_zero(), DL, VT, C->isTargetOpcode(),
+                         C->isOpaque());                     
     case ISD::FP16_TO_FP:
     case ISD::BF16_TO_FP: {
       bool Ignored;
@@ -5522,6 +5539,9 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
   case ISD::CTLZ_ZERO_UNDEF:
   case ISD::CTTZ:
   case ISD::CTTZ_ZERO_UNDEF:
+  case ISD::CRC32:
+  case ISD::CRC:
+  case ISD::CRC_ZERO_UNDEF:
   case ISD::CTPOP: {
     SDValue Ops = {Operand};
     if (SDValue Fold = FoldConstantArithmetic(Opcode, DL, VT, Ops))
@@ -5756,6 +5776,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
       return Operand;
     break;
   case ISD::CTLZ:
+  //case ISD::CRC:
   case ISD::CTTZ:
     if (Operand.getValueType().getScalarType() == MVT::i1)
       return getNOT(DL, Operand, Operand.getValueType());

llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp

@@ -8593,6 +8593,17 @@ SDValue TargetLowering::expandCTTZ(SDNode *Node, SelectionDAG &DAG) const {
   return DAG.getNode(ISD::CTPOP, dl, VT, Tmp);
 }
 
+SDValue TargetLowering::expandCRC(SDNode *Node, SelectionDAG &DAG) const {
+  //We have to change this function!
+  SDLoc dl(Node);
+  EVT VT = Node->getValueType(0);
+  SDValue Op = Node->getOperand(0);
+  unsigned NumBitsPerElt = VT.getScalarSizeInBits();
+
+  return DAG.getNode(ISD::AND, dl, VT, DAG.getNOT(dl, Op, VT),
+                     DAG.getNode(ISD::SUB, dl, VT, Op, DAG.getConstant(1, dl, VT)));
+}
+
 SDValue TargetLowering::expandVPCTTZ(SDNode *Node, SelectionDAG &DAG) const {
   SDValue Op = Node->getOperand(0);
   SDValue Mask = Node->getOperand(1);
@@ -8609,6 +8620,22 @@ SDValue TargetLowering::expandVPCTTZ(SDNode *Node, SelectionDAG &DAG) const {
   return DAG.getNode(ISD::VP_CTPOP, dl, VT, Tmp, Mask, VL);
 }
 
+SDValue TargetLowering::expandVPCRC(SDNode *Node, SelectionDAG &DAG) const {
+  SDValue Op = Node->getOperand(0);
+  SDValue Mask = Node->getOperand(1);
+  SDValue VL = Node->getOperand(2);
+  SDLoc dl(Node);
+  EVT VT = Node->getValueType(0);
+
+  // Same as the vector part of expandCTTZ, use: popcount(~x & (x - 1))
+  SDValue Not = DAG.getNode(ISD::VP_XOR, dl, VT, Op,
+                            DAG.getConstant(-1, dl, VT), Mask, VL);
+  SDValue MinusOne = DAG.getNode(ISD::VP_SUB, dl, VT, Op,
+                                 DAG.getConstant(1, dl, VT), Mask, VL);
+  SDValue Tmp = DAG.getNode(ISD::VP_AND, dl, VT, Not, MinusOne, Mask, VL);
+  return DAG.getNode(ISD::VP_CTPOP, dl, VT, Tmp, Mask, VL);
+}
+
 SDValue TargetLowering::expandABS(SDNode *N, SelectionDAG &DAG,
                                   bool IsNegative) const {
   SDLoc dl(N);

llvm/lib/IR/Instructions.cpp

@@ -642,7 +642,7 @@ void CallInst::init(FunctionType *FTy, Value *Func, ArrayRef<Value *> Args,
   this->FTy = FTy;
   assert(getNumOperands() == Args.size() + CountBundleInputs(Bundles) + 1 &&
          "NumOperands not set up?");
-
+/*
 #ifndef NDEBUG
   assert((Args.size() == FTy->getNumParams() ||
           (FTy->isVarArg() && Args.size() > FTy->getNumParams())) &&
@@ -653,7 +653,7 @@ void CallInst::init(FunctionType *FTy, Value *Func, ArrayRef<Value *> Args,
             FTy->getParamType(i) == Args[i]->getType()) &&
            "Calling a function with a bad signature!");
 #endif
-
+*/
   // Set operands in order of their index to match use-list-order
   // prediction.
   llvm::copy(Args, op_begin());

llvm/lib/Target/PowerPC/P10InstrResources.td

@@ -351,6 +351,7 @@ def : InstRW<[P10W_CY_7C, P10W_DISP_ANY, P10CY_Read, P10CY_Read],
     CFUGED,
     CNTLZDM,
     CNTTZDM,
+    //CNRCDM,
     PDEPD,
     PEXTD,
     VCFUGED,
@@ -660,6 +661,10 @@ def : InstRW<[P10W_F2_4C, P10W_DISP_ANY, P10F2_Read],
     CNTTZD_rec,
     CNTTZW, CNTTZW8,
     CNTTZW8_rec, CNTTZW_rec,
+    //CNRCD,
+    //CNRCD_rec,
+    //CNRCW, CNRCW8,
+    //CNRCW8_rec, CNRCW_rec,
     FTSQRT,
     MTVSRBM,
     MTVSRBMI,

llvm/lib/Target/PowerPC/PPCISelLowering.cpp

@@ -464,13 +464,17 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
         (Subtarget.hasP9Vector() && Subtarget.isPPC64()) ? Custom : Expand);
   }
 
-  // CTPOP or CTTZ were introduced in P8/P9 respectively
+  // CTPOP or CTTZ (or CRC) were introduced in P8/P9 respectively
   if (Subtarget.isISA3_0()) {
     setOperationAction(ISD::CTTZ , MVT::i32  , Legal);
     setOperationAction(ISD::CTTZ , MVT::i64  , Legal);
+    setOperationAction(ISD::CRC , MVT::i32  , Legal);
+    setOperationAction(ISD::CRC , MVT::i64  , Legal);
   } else {
     setOperationAction(ISD::CTTZ , MVT::i32  , Expand);
     setOperationAction(ISD::CTTZ , MVT::i64  , Expand);
+    setOperationAction(ISD::CRC , MVT::i32  , Expand);
+    setOperationAction(ISD::CRC , MVT::i64  , Expand);
   }
 
   if (Subtarget.hasPOPCNTD() == PPCSubtarget::POPCNTD_Fast) {
@@ -801,10 +805,13 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
       }
 
       // Vector instructions introduced in P9
-      if (Subtarget.hasP9Altivec() && (VT.SimpleTy != MVT::v1i128))
+      if (Subtarget.hasP9Altivec() && (VT.SimpleTy != MVT::v1i128)){
         setOperationAction(ISD::CTTZ, VT, Legal);
-      else
+        setOperationAction(ISD::CRC, VT, Legal);
+      } else {
         setOperationAction(ISD::CTTZ, VT, Expand);
+        setOperationAction(ISD::CRC, VT, Expand);
+      }
 
       // We promote all shuffles to v16i8.
       setOperationAction(ISD::VECTOR_SHUFFLE, VT, Promote);