Feat/hmf reference download

.github/workflows/nf-test.yml

@@ -31,10 +31,13 @@ jobs:
     runs-on: # use self-hosted runners
       - runs-on=${{ github.run_id }}-nf-test-changes
       - runner=4cpu-linux-x64
+      - extras=s3-cache
     outputs:
       shard: ${{ steps.set-shards.outputs.shard }}
       total_shards: ${{ steps.set-shards.outputs.total_shards }}
     steps:
+      - uses: runs-on/action@v2
+
       - name: Clean Workspace # Purge the workspace in case it's running on a self-hosted runner
         run: |
           ls -la ./
@@ -59,14 +62,73 @@ jobs:
           echo ${{ steps.set-shards.outputs.shard }}
           echo ${{ steps.set-shards.outputs.total_shards }}
 
+  download-reference:
+    name: Download HMF reference data
+    needs: [nf-test-changes]
+    if: ${{ needs.nf-test-changes.outputs.total_shards != '0' }}
+    runs-on:
+      - runs-on=${{ github.run_id }}-download-reference
+      - runner=8cpu-linux-x64
+      - volume=160gb
+      - extras=s3-cache
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+        with:
+          fetch-depth: 0
+
+      - uses: runs-on/action@v2
+
+      - name: Restore HMF reference cache
+        id: cache-hmf-ref
+        uses: actions/cache@v4
+        with:
+          path: reference_data/
+          key: hmf-reference-grch38-v25.1-${{ hashFiles('nextflow.config', 'conf/hmf_data.config', 'conf/hmf_genomes.config') }}
+          restore-keys: |
+            hmf-reference-grch38-v25.1-
+
+      - name: Setup Nextflow for reference download
+        if: steps.cache-hmf-ref.outputs.cache-hit != 'true'
+        uses: nf-core/setup-nextflow@v2
+        with:
+          version: "25.04.0"
+          # version: "24.10.5"
+
+      - name: Download HMF reference data from R2
+        if: steps.cache-hmf-ref.outputs.cache-hit != 'true'
+        run: |
+          nextflow run . \
+            -profile docker \
+            --mode prepare_reference \
+            --ref_data_types wgs \
+            --genome GRCh38_hmf \
+            --outdir reference_data/ \
+            -c <(echo "aws.client.anonymous=true")
+
+      - name: Upload Nextflow logfile for debugging purposes
+        if: ${{ always() }}
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: nextflow_logfile.txt
+          path: .nextflow.log*
+          include-hidden-files: true
+
+      - name: Verify reference data
+        run: |
+          echo "Reference data size:"
+          du -sh reference_data/ || echo "reference_data/ not found"
+          echo "Reference data structure:"
+          find reference_data/ -type d -maxdepth 3 || echo "No directories found"
+
   nf-test:
     name: "${{ matrix.profile }} | ${{ matrix.NXF_VER }} | ${{ matrix.shard }}/${{ needs.nf-test-changes.outputs.total_shards }}"
-    needs: [nf-test-changes]
+    needs: [nf-test-changes, download-reference]
     if: ${{ needs.nf-test-changes.outputs.total_shards != '0' }}
     runs-on: # use self-hosted runners
       - runs-on=${{ github.run_id }}-nf-test
-      - runner=4cpu-linux-x64
-      - disk=large
+      - runner=48cpu-linux-x64
+      - volume=300gb
+      - extras=s3-cache
     strategy:
       fail-fast: false
       matrix:
@@ -92,6 +154,16 @@ jobs:
         with:
           fetch-depth: 0
 
+      - uses: runs-on/action@v2
+
+      - name: Restore HMF reference cache
+        uses: actions/cache@v4
+        with:
+          path: reference_data/
+          key: hmf-reference-grch38-v25.1-${{ hashFiles('nextflow.config', 'conf/hmf_data.config', 'conf/hmf_genomes.config') }}
+          restore-keys: |
+            hmf-reference-grch38-v25.1-
+
       - name: Run nf-test
         id: run_nf_test
         uses: ./.github/actions/nf-test
@@ -125,7 +197,10 @@ jobs:
     runs-on: # use self-hosted runners
       - runs-on=${{ github.run_id }}-confirm-pass
       - runner=2cpu-linux-x64
+      - extras=s3-cache
     steps:
+      - uses: runs-on/action@v2
+
       - name: One or more tests failed (excluding latest-everything)
         if: ${{ contains(needs.*.result, 'failure') }}
         run: exit 1

tests/default.nf.test

@@ -5,10 +5,10 @@ nextflow_pipeline {
     tag "pipeline"
     tag "cicd"
 
-    profile "test_stub"
-    options "-stub"
+    profile "test"
+    // options "-stub"
 
-    test("-profile test_stub") {
+    test("-profile test") {
 
         when {
             params {

tests/nextflow.config

@@ -5,3 +5,16 @@
 */
 
 aws.client.anonymous = true // fixes S3 access issues on self-hosted runners
+
+// Override reference data paths to use locally downloaded data in CI
+// The reference_data/ directory is populated by the prepare_reference mode in CI workflow
+// which downloads ~25GB of WiGiTS toolkit data from Hartwig's R2 CDN before running tests
+def referenceDataDir = new File("${projectDir}/../reference_data/2.2.0").exists() ?
+    "${projectDir}/../reference_data/2.2.0" :
+    null
+
+if (referenceDataDir) {
+    params.hmf_genomes_base = "${referenceDataDir}/genomes"
+    // Note: HMF data files are downloaded to reference_data/2.2.0/hmf_data/
+    // The pipeline will automatically find them there via the default config paths
+}