Add ccache support.

Signed-off-by: Puneet Matharu <puneet.matharu@arm.com>

Author: puneetmatharu

.github/workflows/pytorch.yml

@@ -67,6 +67,8 @@ jobs:
             { name: c8g, label: ah-ubuntu_22_04-c8g_8x }
            ]
     runs-on: ${{ matrix.config.label }}
+    env:
+      CCACHE_HOST_DIR: ${{ github.workspace }}/Tool-Solutions/ML-Frameworks/pytorch-aarch64/.ccache
     steps:
       - name: Checkout Tool-Solutions
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -76,11 +78,34 @@ jobs:
       - name: Set up Docker
         uses: docker/setup-docker-action@v4
 
+      - name: Create unique cache key from the year and week (YYYY-WW)
+        id: cache_suffix
+        run: echo "week=$(date -u +%G-%V)" >> "$GITHUB_OUTPUT"
+
+      # Restore cache if available. GitHub automatically evicts cache entries that have not been
+      # accessed for over 7 days. We rotate the cache key weekly; if no cache exists for the 
+      # current week, a cache from a previous week (via the prefix restore key) will be restored 
+      # and then saved under the current week's key at the end of the job. This effectively limits 
+      # the cache to at most two weeks of cache data.
+      - name: Restore ccache cache
+        uses: actions/cache@v5
+        with:
+          path: ${{ env.CCACHE_HOST_DIR }}
+          key: ccache-${{ matrix.config.name }}-${{ steps.cache_suffix.outputs.week }}
+          restore-keys: |
+            ccache-${{ matrix.config.name }}-
+
       - name: Build Tool-Solutions PyTorch
         working-directory: ${{ github.workspace }}/Tool-Solutions/ML-Frameworks/pytorch-aarch64
         run: ${{ github.workspace }}/Tool-Solutions/ML-Frameworks/pytorch-aarch64/build.sh
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          USE_CCACHE: 1
+          CCACHE_HOST_DIR: ${{ env.CCACHE_HOST_DIR }}
+          CCACHE_MAXSIZE: 2G
+
+      - name: Report ccache disk usage
+        run: du -sh "${{ env.CCACHE_HOST_DIR }}" || true
 
       - name: Save image as a artifact
         run: docker save toolsolutions-pytorch:latest -o toolsolutions-pytorch-image-${{ matrix.config.name }}.tar

ML-Frameworks/pytorch-aarch64/CHANGELOG.md

@@ -8,6 +8,8 @@ where `YY` is the year, and `MM` the month of the increment.
 ## [unreleased]
 
 ### Added
+ - Adds PyTorch [PR #170600](https://github.com/pytorch/pytorch/pull/170600), to patch incremental build support.
+ - Adds PyTorch [PR #170062](https://github.com/pytorch/pytorch/pull/170062), to add ccache support to ACL/OpenBLAS and manywheel build script.
 
 ### Changed
 

ML-Frameworks/pytorch-aarch64/build-wheel.sh

@@ -55,6 +55,40 @@ UTILS=/utils
 COMMON_UTILS=/common_utils
 PYTORCH_FINAL_PACKAGE_DIR=/artifacts
 
+# Enable ccache support by default; disable via env var or command line flag
+# NOTE: The intention is to have a project-specific cache directory that we cache build artefacts
+# inside and can be easily wiped. These build artefacts are specific to the manylinux builder
+# container (and thus compilers) that we use to build the torch wheel. As such, you may not want
+# to populate the global ccache cache with them. However, if you wish to do so, simply set
+# CCACHE_HOST_DIR to that directory.
+USE_CCACHE=${USE_CCACHE:-1}
+if [[ "$*" == *--enable-ccache* ]]; then USE_CCACHE=1; fi
+if [[ "$*" == *--disable-ccache* ]]; then USE_CCACHE=0; fi
+CCACHE_HOST_DIR=${CCACHE_HOST_DIR:-"${PWD}/.ccache"}
+CCACHE_ROOT=/.ccache
+CCACHE_MAXSIZE=${CCACHE_MAXSIZE:-}
+
+# If the user wants to use ccache for build caching
+ccache_args=(-e USE_CCACHE="${USE_CCACHE}")
+if [ "${USE_CCACHE}" == "1" ]; then
+    echo "Using ccache for build caching"
+    if [ -d "${CCACHE_HOST_DIR}" ]; then
+        echo "ccache directory exists at ${CCACHE_HOST_DIR}"
+    else
+        echo "Creating ccache directory at ${CCACHE_HOST_DIR}"
+        mkdir -p "${CCACHE_HOST_DIR}"
+    fi
+
+    # ccache-specific docker run args
+    ccache_args+=(
+        -e CCACHE_DIR="${CCACHE_ROOT}"
+        -v "${CCACHE_HOST_DIR}:${CCACHE_ROOT}"
+    )
+    if [ -n "${CCACHE_MAXSIZE}" ]; then
+        ccache_args+=(-e CCACHE_MAXSIZE="${CCACHE_MAXSIZE}")
+    fi
+fi
+
 # Want a CPU build
 DESIRED_CUDA=cpu
 GPU_ARCH_TYPE=cpu-aarch64
@@ -89,13 +123,26 @@ if ! docker container inspect $TORCH_BUILD_CONTAINER >/dev/null 2>&1 ; then
         -e SKIP_ALL_TESTS=1 \
         -e OPENSSL_ROOT_DIR="${OPENSSL_HOST_DIR}" \
         -e CMAKE_INCLUDE_PATH="${OPENSSL_HOST_DIR}/include" \
+        "${ccache_args[@]}" \
         -v "${PYTORCH_HOST_DIR}:${PYTORCH_ROOT}" \
         -v "${PYTORCH_FINAL_PACKAGE_HOST_DIR}:${PYTORCH_FINAL_PACKAGE_DIR}" \
         -v "${PWD}/utils:${UTILS}" \
         -v "${PWD}/../utils:${COMMON_UTILS}" \
         -w / \
         "${IMAGE_NAME}")
 
+    # Provide ccache support
+    if [ "${USE_CCACHE}" == "1" ]; then
+        docker exec "$TORCH_BUILD_CONTAINER" yum install -y ccache || true
+        if [ -n "${CCACHE_MAXSIZE}" ]; then
+            docker exec "$TORCH_BUILD_CONTAINER" ccache --max-size="$CCACHE_MAXSIZE" || true
+        fi
+        docker exec "$TORCH_BUILD_CONTAINER" ccache -z || true
+        docker exec "$TORCH_BUILD_CONTAINER" ccache -o compression=true || true
+        docker exec "$TORCH_BUILD_CONTAINER" ccache -o compression_level=6 || true
+        docker exec "$TORCH_BUILD_CONTAINER" ccache -s || true
+    fi
+
     # Currently changes in these scripts will not be applied without a clean
     # build, which is not ideal for dev work. But we have to balance this with
     # extra time/network traffic when rebuilding many times.
@@ -131,12 +178,25 @@ build_date=$(cd $PYTORCH_HOST_DIR && git log --pretty=format:%cs -1 | tr -d '-')
 version=$(cat $PYTORCH_HOST_DIR/version.txt| tr -d "[:space:]" )
 OVERRIDE_PACKAGE_VERSION="${version%??}.dev${build_date}${TORCH_RELEASE_ID:+"+$TORCH_RELEASE_ID"}"
 
+if [ "${USE_CCACHE}" == "1" ]; then
+    echo "Zero-ing ccache stats."
+    docker exec "$TORCH_BUILD_CONTAINER" ccache -z || true
+fi
+
 docker exec $TORCH_BUILD_CONTAINER bash -lc "
   source /tmp/env &&
   BUILD_TEST=0 \
+  DO_SETUP_PY_CLEAN_BEFORE_BUILD=0 \
+  WIPE_RH_CUDA_AFTER_BUILD=0 \
   OVERRIDE_PACKAGE_VERSION=$OVERRIDE_PACKAGE_VERSION \
   bash ${PYTORCH_ROOT}/.ci/manywheel/build.sh
 "
 
+if [ "${USE_CCACHE}" == "1" ]; then
+    echo "Final ccache stats:"
+    docker exec "$TORCH_BUILD_CONTAINER" ccache -s || true
+fi
+
 # directories generated by the docker container are owned by root, so transfer ownership to user
-docker exec $TORCH_BUILD_CONTAINER chown -R "$(id -u)":"$(id -g)" "${PYTORCH_ROOT}" /artifacts
+docker exec "$TORCH_BUILD_CONTAINER" chown -R "$(id -u)":"$(id -g)" \
+    "${PYTORCH_ROOT}" "${PYTORCH_FINAL_PACKAGE_DIR}" "${CCACHE_ROOT}"

ML-Frameworks/pytorch-aarch64/build.sh

@@ -83,6 +83,12 @@ if ! [[ $* == *--use-existing-sources* ]]; then
     ./get-source.sh
 fi
 
+# Use ccache either via env var or command line flag
+USE_CCACHE=${USE_CCACHE:-1}
+if [[ "$*" == *--enable-ccache* ]]; then USE_CCACHE=1; fi
+if [[ "$*" == *--disable-ccache* ]]; then USE_CCACHE=0; fi
+export USE_CCACHE="${USE_CCACHE}"
+
 ./build-wheel.sh
 
 [[ $* == *--wheel-only* ]] && exit 0

ML-Frameworks/pytorch-aarch64/get-source.sh

@@ -33,10 +33,14 @@ git-shallow-clone https://github.com/pytorch/pytorch.git $PYTORCH_HASH
     cd pytorch
 
     # https://github.com/pytorch/pytorch/pull/167829 - Refactor ACL and OpenBLAS install scripts on AArch64
-    # Note: as part of this patch, setuptools is pinned to ~= 78.1.1 which is not affected by
-    # CVE-2025-47273 and CVE-2024-6345
-    apply-github-patch pytorch/pytorch 69db12b465887df96d27fe2bb93746ac334577f1
-    apply-github-patch pytorch/pytorch 5184c373a8bc77809b6e59361e191d4e78d6a824
+    apply-github-patch pytorch/pytorch f5e7b3ab44b14902f1e44ac138006b04bd9b7728
+
+    # https://github.com/pytorch/pytorch/pull/170062 - Add ccache support to ACL/OpenBLAS and manywheel
+    # build script.
+    apply-github-patch pytorch/pytorch 327b118078869b85d979d9f7eb1038b8a53c8a49
+
+    # https://github.com/pytorch/pytorch/pull/170600 - Gate deletion of clean-up steps in build_common.sh
+    apply-github-patch pytorch/pytorch e368ec2693b8b2b8ba35d0913f1d663ba2fdc804
 
     # FIXME: Temporarily disabled; to be updated in a later PR
     # # https://github.com/pytorch/pytorch/pull/160184 - Draft: separate reqs for manywheel build and pin