Add ZSTD_HASH_USE_CRC32C to use an optimized hash

.github/workflows/commit.yml

@@ -51,12 +51,16 @@ jobs:
       run: make gnu99build && make clean
     - name: ppc64 build
       run: make ppc64build V=1 && make clean
+    - name: ppc64CRC build
+      run: make ppc64buildCRC V=1 && make clean
     - name: ppc build
       run: make ppcbuild V=1 && make clean
     - name: arm build
       run: make armbuild V=1 && make clean
     - name: aarch64 build
       run: make aarch64build V=1 && make clean
+    - name: aarch64CRC build
+      run: make aarch64buildCRC V=1 && make clean
     - name: test-legacy
       run: make -C tests test-legacy V=1 && make clean
     - name: test-longmatch
@@ -65,40 +69,81 @@ jobs:
       run: make -C lib libzstd-nomt V=1 && make clean
 
   regression-test:
-    runs-on: ubuntu-latest
+    name: regression-${{ matrix.config_name }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - config_name: "default"
+            os: ubuntu-latest
+            baseline: "results.csv"
+            cflags: "-O3"
+
+          - config_name: "x86-crc"
+            os: ubuntu-latest
+            baseline: "results_crc32c.csv"
+            cflags: "-O3 -DZSTD_HASH_USE_CRC32C"
+
+          - config_name: "x86-sse4.2-crc"
+            os: ubuntu-latest
+            baseline: "results_crc32c.csv"
+            cflags: "-O3 -DZSTD_HASH_USE_CRC32C -msse4.2"
+
+          - config_name: "arm-v8.2-crc"
+            os: ubuntu-24.04-arm
+            baseline: "results_crc32c.csv"
+            cflags: "-O3 -DZSTD_HASH_USE_CRC32C -march=armv8.2-a+crc+crypto"
+
     services:
       docker:
         image: fbopensource/zstd-circleci-primary:0.0.1
         options: --entrypoint /bin/bash
+
     env:
       CIRCLE_ARTIFACTS: "/tmp/circleci-artifacts"
+      CFLAGS: ${{ matrix.cflags }}
+
     steps:
     - uses: actions/checkout@v6.0.2
+
     - name: restore_cache
       uses: actions/cache@v5
       with:
-        key: regression-cache-{{ checksum "tests/regression/data.c" }}-v0
+        # Use matrix config_name to keep cache keys unique per architecture/config
+        key: regression-cache-${{ matrix.config_name }}-${{ checksum "tests/regression/data.c" }}-v0
         path: tests/regression/cache
-        restore-keys: regression-cache-{{ checksum "tests/regression/data.c" }}-v0
+
     - name: Install Dependencies
       run: |
         sudo apt-get update
         sudo apt-get install libcurl4-gnutls-dev
+
     - name: Regression Test
       run: |
+        # Build zstd with the matrix-specific CFLAGS
         make -C programs zstd
         make -C tests/regression test
+
         mkdir -p $CIRCLE_ARTIFACTS
+
+        # Run the regression test tool
         ./tests/regression/test                     \
             --cache  tests/regression/cache         \
             --output $CIRCLE_ARTIFACTS/results.csv  \
             --zstd   programs/zstd
-        echo "NOTE: The new results.csv is uploaded as an artifact to this job"
-        echo "      If this fails, go to the Artifacts pane in CircleCI, "
-        echo "      download /tmp/circleci-artifacts/results.csv, and if they "
-        echo "      are still good, copy it into the repo and commit it."
-        echo "> diff tests/regression/results.csv $CIRCLE_ARTIFACTS/results.csv"
-        diff tests/regression/results.csv $CIRCLE_ARTIFACTS/results.csv
+
+        echo "Comparing against baseline: ${{ matrix.baseline }}"
+        # Check if the baseline exists (to avoid failing on the first run of a new config)
+        if [ -f "tests/regression/${{ matrix.baseline }}" ]; then
+            diff tests/regression/${{ matrix.baseline }} $CIRCLE_ARTIFACTS/results.csv
+        else
+            echo "Warning: Baseline tests/regression/${{ matrix.baseline }} not found."
+            echo "If this is a new configuration, please commit the generated results.csv as ${{ matrix.baseline }}."
+        fi
+
     - uses: actions/upload-artifact@v7.0.0
       with:
-        path: "/tmp/circleci-artifacts"
+        # Artifact name includes config for easy identification
+        name: results-${{ matrix.config_name }}
+        path: "$CIRCLE_ARTIFACTS/results.csv"

.github/workflows/dev-long-tests.yml

@@ -42,6 +42,13 @@ jobs:
     - name: make test on macos
       run: make test
 
+  make-test-macos-crc32:
+    runs-on: macos-latest
+    steps:
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
+    - name: make test on macos with CRC32C
+      run: CFLAGS="-DZSTD_HASH_USE_CRC32C" make test
+
   # lasts ~10mn
   make-test-32bit:
     runs-on: ubuntu-latest
@@ -74,6 +81,21 @@ jobs:
     - name: no intrinsics fuzztest
       run: MOREFLAGS="-DZSTD_NO_INTRINSICS" make -C tests fuzztest
 
+  # lasts ~9mn
+  crc-sse42-fuzztest:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
+    - name: CRC32C fuzztest with sse4.2
+      run: MOREFLAGS="-DZSTD_HASH_USE_CRC32C -msse4.2" make -C tests fuzztest
+
+  crc-nosse42-fuzztest:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
+    - name: CRC32C fuzztest without sse4.2
+      run: MOREFLAGS="-DZSTD_HASH_USE_CRC32C" make -C tests fuzztest
+
   # lasts ~8mn
   tsan-zstreamtest:
     runs-on: ubuntu-latest
@@ -209,6 +231,13 @@ jobs:
     - name: clang + ASan + UBSan + Regression Test
       run: CC=clang make -j uasanregressiontest
 
+  clang-asan-ubsan-regressionCRC:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
+    - name: clang + ASan + UBSan + Regression Test + CRC hash
+      run: CC=clang make -j uasanregressiontestCRC
+
   msan-regression:
     runs-on: ubuntu-latest
     steps:

.github/workflows/dev-short-tests.yml

@@ -153,6 +153,16 @@ jobs:
         make libc6install
         CFLAGS="-Werror -mavx2" make -j all
 
+  gcc-make-all-avx2-crc:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
+    - name: Make all, with AVX2 and CRC hash
+      run: |
+        sudo apt-get -qqq update
+        make libc6install
+        CFLAGS="-Werror -mavx2 -DZSTD_HASH_USE_CRC32C" make -j all
+
   gcc-make-all-32bit:
     runs-on: ubuntu-latest
     steps:
@@ -173,6 +183,15 @@ jobs:
         make libc6install
         CPPFLAGS="-DSTATIC_BMI2=1" CFLAGS="-Werror -m32 -mavx2 -mbmi2" make -j all32
 
+  gcc-make-all-32bit-avx2-crc:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
+    - name: Make all, 32bit + AVX2 mode and CRC hash
+      run: |
+        sudo apt-get -qqq update
+        make libc6install
+        CPPFLAGS="-DSTATIC_BMI2=1" CFLAGS="-Werror -m32 -mavx2 -mbmi2 -DZSTD_HASH_USE_CRC32C" make -j all32
 
   gcc-8-make:
     runs-on: ubuntu-latest
@@ -434,8 +453,8 @@ jobs:
         make clean
         LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make -j check
         LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make -j -C tests test-cli-tests
-        CFLAGS="-O3 -march=armv8.2-a+sve2" LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make -j check
-        CFLAGS="-O3 -march=armv8.2-a+sve2" LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make -j -C tests test-cli-tests
+        CFLAGS="-O3 -march=armv8.2-a+sve2+crc -DZSTD_HASH_USE_CRC32C" LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make -j check
+        CFLAGS="-O3 -march=armv8.2-a+sve2+crc -DZSTD_HASH_USE_CRC32C" LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make -j -C tests test-cli-tests
 # This test is only compatible with standard libraries that support BTI (Branch Target Identification).
 # Unfortunately, the standard library provided on Ubuntu 24.04 does not have this feature enabled.
 #        make clean

Makefile

@@ -205,7 +205,7 @@ clangbuild-darwin-fat: clean
 	mv programs/zstd programs/zstd_x64
 	lipo -create programs/zstd_x64 programs/zstd_arm64 -output programs/zstd
 
-.PHONY: gcc5build gcc6build gcc7build clangbuild m32build armbuild aarch64build ppcbuild ppc64build
+.PHONY: gcc5build gcc6build gcc7build clangbuild m32build armbuild aarch64build aarch64buildCRC ppcbuild ppc64build ppc64buildCRC
 gcc5build: clean
 	gcc-5 -v
 	CC=gcc-5 $(MAKE) all MOREFLAGS="-Werror $(MOREFLAGS)"
@@ -232,12 +232,18 @@ armbuild: clean
 aarch64build: clean
 	CC=aarch64-linux-gnu-gcc CFLAGS="-Werror -O0" $(MAKE) allzstd
 
+aarch64buildCRC: clean
+	CC=aarch64-linux-gnu-gcc CFLAGS="-Werror -O0 -DZSTD_HASH_USE_CRC32C -march=armv8.2-a+crc+crypto" $(MAKE) allzstd
+
 ppcbuild: clean
 	CC=powerpc-linux-gnu-gcc CFLAGS="-m32 -Wno-attributes -Werror" $(MAKE) -j allzstd
 
 ppc64build: clean
 	CC=powerpc-linux-gnu-gcc CFLAGS="-m64 -Werror" $(MAKE) -j allzstd
 
+ppc64buildCRC: clean
+	CC=powerpc-linux-gnu-gcc CFLAGS="-m64 -Werror -DZSTD_HASH_USE_CRC32C" $(MAKE) -j allzstd
+
 .PHONY: armfuzz aarch64fuzz ppcfuzz ppc64fuzz
 armfuzz: clean
 	CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" $(MAKE) -C $(TESTDIR) fuzztest
@@ -294,6 +300,9 @@ regressiontest:
 uasanregressiontest:
 	$(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=address,undefined -Werror" CXXFLAGS="-O3 -fsanitize=address,undefined -Werror"
 
+uasanregressiontestCRC:
+	$(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=address,undefined -Werror -DZSTD_HASH_USE_CRC32C" CXXFLAGS="-O3 -fsanitize=address,undefined -Werror"
+
 msanregressiontest:
 	$(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=memory -Werror" CXXFLAGS="-O3 -fsanitize=memory -Werror"
 

lib/compress/zstd_compress.c

@@ -2415,7 +2415,7 @@ static void ZSTD_copyCDictTableIntoCCtx(U32* dst, U32 const* src, size_t tableSi
         size_t i;
         for (i = 0; i < tableSize; i++) {
             U32 const taggedIndex = src[i];
-            U32 const index = taggedIndex >> ZSTD_SHORT_CACHE_TAG_BITS;
+            U32 const index = ZSTD_extractIndex(taggedIndex, ZSTD_SHORT_CACHE_TAG_BITS);
             dst[i] = index;
         }
     } else {