Optimizing build cache step (#1395)

While looking for ways to optimize/shave off time from our builds, I was
brought back to [this
ticket](https://app.asana.com/1/90955849329269/inbox/1211294560690016/item/1212009520590921/story/1212029899178754).
This caching step didn't make any sense to me so I delved into it.

Essentially, this step was creating a cache key whose name was a mix
between a hash of our package-lock file and a hash of all of our (70k+)
mdx files (which would take 2-3 min to process). This key would be used
to pull up a cached version of the content we have in ~/.npm and
/.next/cache during the github runner process. The content in those
folders are just downloaded packages, compliled js/ts files, and other
things for nextjs optimization.

If a single mdx file changed, it would just invalidate the cache (even
though the cache was totally good to use), and cause us to run npm
install again, so we never really had the proper benefit of this cache
in the first place.

This PR simplifies and narrows the scope of the files we actually care
about caching the changes for. Included changes in both the
build-preview process and the deploy-udr step.

Author: williamdalessandro

.github/workflows/algolia-prod.yml

@@ -22,7 +22,7 @@ jobs:
           cache: 'npm'
           node-version-file: 'package.json'
 
-      - run: npm i
+      - run: npm ci
       - run: npm run prebuild -- --get-real-file-changed-metadata --build-algolia-index
       - run: npm run algolia
         env:

.github/workflows/build-pr-preview.yml

@@ -2,6 +2,10 @@ name: Build Preview
 run-name: 'Build Preview for "${{ github.event.pull_request.title }}" (#${{ github.event.pull_request.number }})'
 
 on:
+  # 1: Change this to `pull_request` to see changes while testing this file in a PR.
+  # 2: This is because `pull_request_target` only works from the context of the base branch
+  #    https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows#pull_request_target
+  # 3: Change this back to `pull_request_target` before merge for security reasons
   pull_request_target:
     types: [opened, synchronize]
   # Hello Security 👋, we are checking to make sure forked repo PR changed paths are only in content/** inside the job security-check.
@@ -105,20 +109,11 @@ jobs:
           cache: 'npm'
           node-version-file: 'package.json'
 
-      - name: Generate cache manifest
-        run: git ls-files -z -- '*.js' '*.mjs' '*.ts' '*.tsx' '*.mdx' '*.json' | xargs -0 git hash-object > cache-key.txt
-
-      - name: Use cache
+      - name: Cache Next.js build
         uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
-          path: |
-            ~/.npm
-            ${{ github.workspace }}/.next/cache
-          # Generate a new cache whenever packages or source files change.
-          key: ${{ runner.os }}-nextjs-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('cache-key.txt') }}
-          # If source files changed but packages didn't, rebuild from a prior cache.
-          restore-keys: |
-            ${{ runner.os }}-nextjs-${{ hashFiles('**/package-lock.json') }}-
+          path: ${{ github.workspace }}/.next/cache
+          key: ${{ runner.os }}-nextjs-build-${{ hashFiles('package-lock.json', 'next.config.js') }}
 
       - name: Install Vercel CLI
         run: npm i --global vercel@latest
@@ -187,17 +182,6 @@ jobs:
         with:
           repository: hashicorp/dev-portal
           path: ./unified-docs-frontend-preview
-      - name: Use cache
-        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
-        with:
-          path: |
-            ~/.npm
-            ${{ github.workspace }}/unified-docs-frontend-preview/.next/cache
-          # Generate a new cache whenever packages or source files change.
-          key: ${{ runner.os }}-nextjs-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('**/*.js', '**/*.jsx', '**/*.ts', '**/*.tsx', '**/*.mdx') }}
-          # If source files changed but packages didn't, rebuild from a prior cache.
-          restore-keys: |
-            ${{ runner.os }}-nextjs-${{ hashFiles('**/package-lock.json') }}-
 
       - name: Setup Node.js
         uses: actions/setup-node@1d0ff469b7ec7b3cb9d8673fde0c81c44821de2a # v4.2.0
@@ -206,6 +190,12 @@ jobs:
           cache-dependency-path: 'unified-docs-frontend-preview/package-lock.json'
           node-version-file: 'unified-docs-frontend-preview/package.json'
 
+      - name: Cache Next.js build
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
+        with:
+          path: ${{ github.workspace }}/unified-docs-frontend-preview/.next/cache
+          key: ${{ runner.os }}-nextjs-build-devportal-${{ hashFiles('unified-docs-frontend-preview/package-lock.json', 'unified-docs-frontend-preview/next.config.js') }}
+
       - name: Setup Vercel CLI
         run: npm i --global vercel@latest
 

.github/workflows/copy-cloud-docs-for-tfe.yml

@@ -52,7 +52,7 @@ jobs:
       - name: Generate version-metadata for workflow
         working-directory: '${{github.workspace}}/new-docs-pr'
         run: |
-          npm i
+          npm ci
           npm run prebuild -- --only-build-version-metadata
 
       - name: Create the new TFE version folder for RELEASE PR

.github/workflows/deploy-udr.yml

@@ -27,17 +27,11 @@ jobs:
           cache: 'npm'
           node-version-file: 'package.json'
 
-      - name: Use cache
+      - name: Cache Next.js build
         uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
-          path: |
-            ~/.npm
-            ${{ github.workspace }}/.next/cache
-          # Generate a new cache whenever packages or source files change.
-          key: ${{ runner.os }}-nextjs-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('**/*.js', '**/*.mjs', '**/*.ts', '**/*.tsx', '**/*.mdx') }}
-          # If source files changed but packages didn't, rebuild from a prior cache.
-          restore-keys: |
-            ${{ runner.os }}-nextjs-${{ hashFiles('**/package-lock.json') }}-
+          path: ${{ github.workspace }}/.next/cache
+          key: ${{ runner.os }}-nextjs-build-${{ hashFiles('package-lock.json', 'next.config.js') }}
 
       - name: Install Vercel CLI
         run: npm install --global vercel@latest

.github/workflows/sync-docs-for-tfe.yml

@@ -83,7 +83,7 @@ jobs:
       - name: Generate version-metadata for workflow
         working-directory: '${{github.workspace}}/new-docs'
         run: |
-          npm i
+          npm ci
           npm run prebuild -- --only-build-version-metadata
 
       - name: Checkout HCPTF-diff for new docs version DIFF PR

.github/workflows/test-and-lint.yml

@@ -28,7 +28,7 @@ jobs:
           cache: 'npm'
           node-version-file: 'package.json'
 
-      - run: npm i
+      - run: npm ci
       - run: npm run test
 
   lint:
@@ -46,7 +46,7 @@ jobs:
           node-version-file: 'package.json'
 
       - name: Install Dependencies
-        run: npm i
+        run: npm ci
 
       - name: Run lint check
         run: npm run lint

scripts/prebuild/prebuild-arm-linux-binary.gz

@@ -20,8 +20,8 @@ export async function batchPromises(
 	batchName,
 	arrayToBatch,
 	asyncMapFn,
-	{ batchSize = 16, loggingEnabled = true } = {
-		batchSize: 16,
+	{ batchSize = 64, loggingEnabled = true } = {
+		batchSize: 64,
 		loggingEnabled: true,
 	},
 ) {