diff --git a/.github/actions/nf-test/action.yml b/.github/actions/nf-test/action.yml index 80accce5..93e49e4e 100644 --- a/.github/actions/nf-test/action.yml +++ b/.github/actions/nf-test/action.yml @@ -61,6 +61,19 @@ runs: env: NFT_WORKDIR: ${{ env.NFT_WORKDIR }} run: | + echo "=== Debug: reference_data structure ===" + if [ -d "reference_data" ]; then + echo "Total size:" + du -sh reference_data 2>/dev/null + echo "Directory structure:" + find reference_data -maxdepth 4 -type d 2>/dev/null + echo "Index directories:" + find reference_data -type d -name "*bwa-mem2*" -o -type d -name "*star_index*" -o -type d -name "*gridss*" 2>/dev/null + else + echo "reference_data does not exist" + fi + echo "=== End debug ===" + nf-test test \ --profile=+${{ inputs.profile }} \ $(if [ -n "${{ inputs.tags }}" ]; then echo "--tag ${{ inputs.tags }}"; fi) \ diff --git a/.github/workflows/nf-test-reusable.yml b/.github/workflows/nf-test-reusable.yml index a3036267..07854b91 100644 --- a/.github/workflows/nf-test-reusable.yml +++ b/.github/workflows/nf-test-reusable.yml @@ -22,10 +22,11 @@ on: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} NFT_VER: "0.9.3" - NFT_WORKDIR: "~" + NFT_WORKDIR: ".nf-test" NXF_ANSI_LOG: false NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity + HMF_REF_VERSION: "grch38-v25.1-v2" jobs: nf-test-changes: @@ -61,14 +62,64 @@ jobs: echo ${{ steps.set-shards.outputs.shard }} echo ${{ steps.set-shards.outputs.total_shards }} + download-reference: + name: Download HMF reference data + needs: [nf-test-changes] + if: ${{ needs.nf-test-changes.outputs.total_shards != '0' }} + runs-on: + - runs-on=${{ github.run_id }}-download-reference + - runner=8cpu-linux-x64 + - volume=300gb + - extras=s3-cache + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + fetch-depth: 0 + + - uses: runs-on/action@v2 + + - name: Restore HMF reference cache + if: ${{ inputs.tags != 'cicd' }} + id: cache-hmf-ref + uses: actions/cache@v4 + with: + path: reference_data/ + key: hmf-reference-${{ env.HMF_REF_VERSION }}-${{ hashFiles('nextflow.config', 'conf/hmf_data.config', 'conf/hmf_genomes.config') }} + + - name: Setup Nextflow for reference download + if: ${{ inputs.tags != 'cicd' && steps.cache-hmf-ref.outputs.cache-hit != 'true' }} + uses: nf-core/setup-nextflow@v2 + with: + version: "25.04.0" + + - name: Download HMF reference data from R2 + if: ${{ inputs.tags != 'cicd' && steps.cache-hmf-ref.outputs.cache-hit != 'true' }} + run: | + nextflow run . \ + -profile docker \ + --mode prepare_reference \ + --ref_data_types wgs,bwamem2_index,star_index \ + --genome GRCh38_hmf \ + --outdir ./ \ + -c <(echo "aws.client.anonymous=true") + + - name: Verify reference data + if: ${{ inputs.tags != 'cicd' }} + run: | + echo "Reference data size:" + du -sh reference_data/ || echo "reference_data/ not found" + echo "Reference data structure:" + find reference_data/ -maxdepth 3 -type d 2>/dev/null || echo "No directories found" + nf-test: name: "${{ inputs.name }} tests | ${{ matrix.profile }} | ${{ matrix.NXF_VER }} | ${{ matrix.shard }}/${{ needs.nf-test-changes.outputs.total_shards }}" - needs: [nf-test-changes] + needs: [nf-test-changes, download-reference] if: ${{ needs.nf-test-changes.outputs.total_shards != '0' }} runs-on: # use self-hosted runners - runs-on=${{ github.run_id }}-nf-test - runner=${{ inputs.runson_instance_type }} - volume=${{ inputs.runson_volume_size }} + - extras=s3-cache strategy: fail-fast: false matrix: @@ -94,6 +145,40 @@ jobs: with: fetch-depth: 0 + - uses: runs-on/action@v2 + + - name: Restore HMF reference cache + if: ${{ inputs.tags != 'cicd' }} + uses: actions/cache@v4 + with: + path: reference_data/ + key: hmf-reference-${{ env.HMF_REF_VERSION }}-${{ hashFiles('nextflow.config', 'conf/hmf_data.config', 'conf/hmf_genomes.config') }} + + - name: Set reference data paths + if: ${{ inputs.tags != 'cicd' }} + run: | + # Find the versioned subdirectory containing the reference FASTA and convert to absolute path + REF_DIR=$(find reference_data -type f -name "GRCh38_masked_exclusions_alts_hlas.fasta" -printf '%h\n' 2>/dev/null | head -1) + if [ -n "$REF_DIR" ]; then + # Convert to absolute path + REF_DIR_ABS=$(cd "$REF_DIR" && pwd) + echo "NXF_REF_DATA_PATH=$REF_DIR_ABS" >> $GITHUB_ENV + echo "Found reference data at: $REF_DIR_ABS" + + # Find index directories using pattern matching (already absolute since REF_DIR_ABS is absolute) + BWAMEM2_INDEX=$(find "$REF_DIR_ABS" -maxdepth 1 -type d -name "bwa-mem2_index*" 2>/dev/null | head -1) + GRIDSS_INDEX=$(find "$REF_DIR_ABS" -maxdepth 1 -type d -name "gridss_index*" 2>/dev/null | head -1) + STAR_INDEX=$(find "$REF_DIR_ABS" -maxdepth 1 -type d -name "star_index*" 2>/dev/null | head -1) + HMF_DATA=$(find "$REF_DIR_ABS" -maxdepth 1 -type d -name "hmf_pipeline_resources*" 2>/dev/null | head -1) + + [ -n "$BWAMEM2_INDEX" ] && echo "NXF_REF_BWAMEM2_INDEX=$BWAMEM2_INDEX" >> $GITHUB_ENV + [ -n "$GRIDSS_INDEX" ] && echo "NXF_REF_GRIDSS_INDEX=$GRIDSS_INDEX" >> $GITHUB_ENV + [ -n "$STAR_INDEX" ] && echo "NXF_REF_STAR_INDEX=$STAR_INDEX" >> $GITHUB_ENV + [ -n "$HMF_DATA" ] && echo "NXF_REF_HMF_DATA=$HMF_DATA" >> $GITHUB_ENV + else + echo "No cached reference data found" + fi + - name: Run nf-test id: run_nf_test uses: ./.github/actions/nf-test diff --git a/tests/cached_refs.config b/tests/cached_refs.config new file mode 100644 index 00000000..2490abf4 --- /dev/null +++ b/tests/cached_refs.config @@ -0,0 +1,19 @@ +/* +======================================================================================== + Nextflow config file for cached reference data paths +======================================================================================== + This config is conditionally included when NXF_REF_DATA_PATH is set. + The path is set by the CI workflow when reference data is cached. +---------------------------------------------------------------------------------------- +*/ + +params { + ref_data_genome_fasta = "${System.getenv('NXF_REF_DATA_PATH')}/GRCh38_masked_exclusions_alts_hlas.fasta" + ref_data_genome_fai = "${System.getenv('NXF_REF_DATA_PATH')}/GRCh38_masked_exclusions_alts_hlas.fasta.fai" + ref_data_genome_dict = "${System.getenv('NXF_REF_DATA_PATH')}/GRCh38_masked_exclusions_alts_hlas.fasta.dict" + ref_data_genome_img = "${System.getenv('NXF_REF_DATA_PATH')}/GRCh38_masked_exclusions_alts_hlas.fasta.img" + ref_data_genome_bwamem2_index = "${System.getenv('NXF_REF_BWAMEM2_INDEX')}/" + ref_data_genome_gridss_index = "${System.getenv('NXF_REF_GRIDSS_INDEX')}/" + ref_data_genome_star_index = "${System.getenv('NXF_REF_STAR_INDEX')}/" + ref_data_hmf_data_path = "${System.getenv('NXF_REF_HMF_DATA')}/" +} diff --git a/tests/default.nf.test.snap b/tests/default.nf.test.snap index 17eeb421..c9120285 100644 --- a/tests/default.nf.test.snap +++ b/tests/default.nf.test.snap @@ -1,4 +1,441 @@ { + "-profile test": { + "content": [ + [ + "pipeline_info", + "pipeline_info/software_versions.yml", + "subject_a", + "subject_a/alignments", + "subject_a/alignments/dna", + "subject_a/alignments/dna/subject_a.normal.jitter_params.tsv", + "subject_a/alignments/dna/subject_a.normal.ms_table.tsv.gz", + "subject_a/alignments/dna/subject_a.normal.redux.bam", + "subject_a/alignments/dna/subject_a.normal.redux.bam.bai", + "subject_a/alignments/dna/subject_a.normal.redux.duplicate_freq.tsv", + "subject_a/alignments/dna/subject_a.tumor.jitter_params.tsv", + "subject_a/alignments/dna/subject_a.tumor.ms_table.tsv.gz", + "subject_a/alignments/dna/subject_a.tumor.redux.bam", + "subject_a/alignments/dna/subject_a.tumor.redux.bam.bai", + "subject_a/alignments/dna/subject_a.tumor.redux.duplicate_freq.tsv", + "subject_a/alignments/rna", + "subject_a/alignments/rna/subject_a.tumor_rna.md.bam", + "subject_a/alignments/rna/subject_a.tumor_rna.md.bam.bai", + "subject_a/alignments/rna/subject_a.tumor_rna.md.metrics", + "subject_a/amber", + "subject_a/amber/amber.version", + "subject_a/amber/subject_a.normal.amber.homozygousregion.tsv", + "subject_a/amber/subject_a.normal.amber.snp.vcf.gz", + "subject_a/amber/subject_a.normal.amber.snp.vcf.gz.tbi", + "subject_a/amber/subject_a.tumor.amber.baf.pcf", + "subject_a/amber/subject_a.tumor.amber.baf.tsv.gz", + "subject_a/amber/subject_a.tumor.amber.contamination.tsv", + "subject_a/amber/subject_a.tumor.amber.contamination.vcf.gz", + "subject_a/amber/subject_a.tumor.amber.contamination.vcf.gz.tbi", + "subject_a/amber/subject_a.tumor.amber.qc", + "subject_a/bamtools", + "subject_a/bamtools/subject_a.normal", + "subject_a/bamtools/subject_a.normal/subject_a.normal.bam_metric.coverage.tsv", + "subject_a/bamtools/subject_a.normal/subject_a.normal.bam_metric.exon_medians.tsv", + "subject_a/bamtools/subject_a.normal/subject_a.normal.bam_metric.flag_counts.tsv", + "subject_a/bamtools/subject_a.normal/subject_a.normal.bam_metric.frag_length.tsv", + "subject_a/bamtools/subject_a.normal/subject_a.normal.bam_metric.gene_coverage.tsv", + "subject_a/bamtools/subject_a.normal/subject_a.normal.bam_metric.partition_stats.tsv", + "subject_a/bamtools/subject_a.normal/subject_a.normal.bam_metric.summary.tsv", + "subject_a/bamtools/subject_a.tumor", + "subject_a/bamtools/subject_a.tumor/subject_a.tumor.bam_metric.coverage.tsv", + "subject_a/bamtools/subject_a.tumor/subject_a.tumor.bam_metric.exon_medians.tsv", + "subject_a/bamtools/subject_a.tumor/subject_a.tumor.bam_metric.flag_counts.tsv", + "subject_a/bamtools/subject_a.tumor/subject_a.tumor.bam_metric.frag_length.tsv", + "subject_a/bamtools/subject_a.tumor/subject_a.tumor.bam_metric.gene_coverage.tsv", + "subject_a/bamtools/subject_a.tumor/subject_a.tumor.bam_metric.partition_stats.tsv", + "subject_a/bamtools/subject_a.tumor/subject_a.tumor.bam_metric.summary.tsv", + "subject_a/chord", + "subject_a/chord/subject_a.tumor.chord.mutation_contexts.tsv", + "subject_a/chord/subject_a.tumor.chord.prediction.tsv", + "subject_a/cider", + "subject_a/cider/subject_a.tumor.cider.alignment_match.tsv.gz", + "subject_a/cider/subject_a.tumor.cider.bam", + "subject_a/cider/subject_a.tumor.cider.layout.gz", + "subject_a/cider/subject_a.tumor.cider.locus_stats.tsv", + "subject_a/cider/subject_a.tumor.cider.vdj.tsv.gz", + "subject_a/cider/subject_a.tumor_rna.cider.alignment_match.tsv.gz", + "subject_a/cider/subject_a.tumor_rna.cider.bam", + "subject_a/cider/subject_a.tumor_rna.cider.layout.gz", + "subject_a/cider/subject_a.tumor_rna.cider.locus_stats.tsv", + "subject_a/cider/subject_a.tumor_rna.cider.vdj.tsv.gz", + "subject_a/cobalt", + "subject_a/cobalt/cobalt.version", + "subject_a/cobalt/subject_a.normal.cobalt.gc.median.tsv", + "subject_a/cobalt/subject_a.normal.cobalt.ratio.median.tsv", + "subject_a/cobalt/subject_a.normal.cobalt.ratio.pcf", + "subject_a/cobalt/subject_a.tumor.cobalt.gc.median.tsv", + "subject_a/cobalt/subject_a.tumor.cobalt.ratio.pcf", + "subject_a/cobalt/subject_a.tumor.cobalt.ratio.tsv.gz", + "subject_a/cuppa", + "subject_a/cuppa/subject_a.tumor.cuppa.pred_summ.tsv", + "subject_a/cuppa/subject_a.tumor.cuppa.vis.png", + "subject_a/cuppa/subject_a.tumor.cuppa.vis_data.tsv", + "subject_a/cuppa/subject_a.tumor.cuppa_data.tsv.gz", + "subject_a/esvee", + "subject_a/esvee/subject_a.normal.esvee.prep.bam", + "subject_a/esvee/subject_a.normal.esvee.prep.bam.bai", + "subject_a/esvee/subject_a.tumor.esvee.alignment.tsv", + "subject_a/esvee/subject_a.tumor.esvee.assembly.tsv", + "subject_a/esvee/subject_a.tumor.esvee.breakend.tsv", + "subject_a/esvee/subject_a.tumor.esvee.germline.vcf.gz", + "subject_a/esvee/subject_a.tumor.esvee.germline.vcf.gz.tbi", + "subject_a/esvee/subject_a.tumor.esvee.phased_assembly.tsv", + "subject_a/esvee/subject_a.tumor.esvee.prep.bam", + "subject_a/esvee/subject_a.tumor.esvee.prep.bam.bai", + "subject_a/esvee/subject_a.tumor.esvee.prep.disc_stats.tsv", + "subject_a/esvee/subject_a.tumor.esvee.prep.fragment_length.tsv", + "subject_a/esvee/subject_a.tumor.esvee.prep.junction.tsv", + "subject_a/esvee/subject_a.tumor.esvee.raw.vcf.gz", + "subject_a/esvee/subject_a.tumor.esvee.raw.vcf.gz.tbi", + "subject_a/esvee/subject_a.tumor.esvee.ref_depth.vcf.gz", + "subject_a/esvee/subject_a.tumor.esvee.ref_depth.vcf.gz.tbi", + "subject_a/esvee/subject_a.tumor.esvee.somatic.vcf.gz", + "subject_a/esvee/subject_a.tumor.esvee.somatic.vcf.gz.tbi", + "subject_a/esvee/subject_a.tumor.esvee.unfiltered.vcf.gz", + "subject_a/esvee/subject_a.tumor.esvee.unfiltered.vcf.gz.tbi", + "subject_a/isofox", + "subject_a/isofox/subject_a.tumor_rna.isf.alt_splice_junc.csv", + "subject_a/isofox/subject_a.tumor_rna.isf.fusions.csv", + "subject_a/isofox/subject_a.tumor_rna.isf.gene_collection.csv", + "subject_a/isofox/subject_a.tumor_rna.isf.gene_data.csv", + "subject_a/isofox/subject_a.tumor_rna.isf.pass_fusions.csv", + "subject_a/isofox/subject_a.tumor_rna.isf.retained_intron.csv", + "subject_a/isofox/subject_a.tumor_rna.isf.summary.csv", + "subject_a/isofox/subject_a.tumor_rna.isf.transcript_data.csv", + "subject_a/lilac", + "subject_a/lilac/subject_a.tumor.lilac.candidates.coverage.tsv", + "subject_a/lilac/subject_a.tumor.lilac.qc.tsv", + "subject_a/lilac/subject_a.tumor.lilac.tsv", + "subject_a/linx", + "subject_a/linx/germline_annotations", + "subject_a/linx/germline_annotations/linx.version", + "subject_a/linx/germline_annotations/subject_a.tumor.linx.germline.breakend.tsv", + "subject_a/linx/germline_annotations/subject_a.tumor.linx.germline.clusters.tsv", + "subject_a/linx/germline_annotations/subject_a.tumor.linx.germline.disruption.tsv", + "subject_a/linx/germline_annotations/subject_a.tumor.linx.germline.driver.catalog.tsv", + "subject_a/linx/germline_annotations/subject_a.tumor.linx.germline.links.tsv", + "subject_a/linx/germline_annotations/subject_a.tumor.linx.germline.svs.tsv", + "subject_a/linx/somatic_annotations", + "subject_a/linx/somatic_annotations/linx.version", + "subject_a/linx/somatic_annotations/subject_a.tumor.linx.breakend.tsv", + "subject_a/linx/somatic_annotations/subject_a.tumor.linx.clusters.tsv", + "subject_a/linx/somatic_annotations/subject_a.tumor.linx.driver.catalog.tsv", + "subject_a/linx/somatic_annotations/subject_a.tumor.linx.drivers.tsv", + "subject_a/linx/somatic_annotations/subject_a.tumor.linx.fusion.tsv", + "subject_a/linx/somatic_annotations/subject_a.tumor.linx.links.tsv", + "subject_a/linx/somatic_annotations/subject_a.tumor.linx.neoepitope.tsv", + "subject_a/linx/somatic_annotations/subject_a.tumor.linx.svs.tsv", + "subject_a/linx/somatic_annotations/subject_a.tumor.linx.vis_copy_number.tsv", + "subject_a/linx/somatic_annotations/subject_a.tumor.linx.vis_fusion.tsv", + "subject_a/linx/somatic_annotations/subject_a.tumor.linx.vis_gene_exon.tsv", + "subject_a/linx/somatic_annotations/subject_a.tumor.linx.vis_protein_domain.tsv", + "subject_a/linx/somatic_annotations/subject_a.tumor.linx.vis_segments.tsv", + "subject_a/linx/somatic_annotations/subject_a.tumor.linx.vis_sv_data.tsv", + "subject_a/linx/somatic_plots", + "subject_a/linx/somatic_plots/all", + "subject_a/linx/somatic_plots/all/subject_a.tumor.chr11.007.png", + "subject_a/linx/somatic_plots/all/subject_a.tumor.chr19.000.png", + "subject_a/linx/somatic_plots/all/subject_a.tumor.chr9.007.png", + "subject_a/linx/somatic_plots/all/subject_a.tumor.cluster-0.resolved_type-SGL_PAIR_INS.sv_count-2.003.png", + "subject_a/linx/somatic_plots/all/subject_a.tumor.cluster-1.resolved_type-RECIP_TRANS.sv_count-2.007.png", + "subject_a/linx/somatic_plots/reportable", + "subject_a/linx/somatic_plots/reportable/subject_a.tumor.cluster-1.resolved_type-RECIP_TRANS.sv_count-2.007.png", + "subject_a/linx/subject_a.tumor_linx.html", + "subject_a/logs", + "subject_a/logs/subject_a.amber.command.err", + "subject_a/logs/subject_a.amber.command.log", + "subject_a/logs/subject_a.amber.command.out", + "subject_a/logs/subject_a.amber.command.run", + "subject_a/logs/subject_a.amber.command.sh", + "subject_a/logs/subject_a.chord.command.err", + "subject_a/logs/subject_a.chord.command.log", + "subject_a/logs/subject_a.chord.command.out", + "subject_a/logs/subject_a.chord.command.run", + "subject_a/logs/subject_a.chord.command.sh", + "subject_a/logs/subject_a.cobalt.command.err", + "subject_a/logs/subject_a.cobalt.command.log", + "subject_a/logs/subject_a.cobalt.command.out", + "subject_a/logs/subject_a.cobalt.command.run", + "subject_a/logs/subject_a.cobalt.command.sh", + "subject_a/logs/subject_a.cuppa.command.err", + "subject_a/logs/subject_a.cuppa.command.log", + "subject_a/logs/subject_a.cuppa.command.out", + "subject_a/logs/subject_a.cuppa.command.run", + "subject_a/logs/subject_a.cuppa.command.sh", + "subject_a/logs/subject_a.esvee.command.err", + "subject_a/logs/subject_a.esvee.command.log", + "subject_a/logs/subject_a.esvee.command.out", + "subject_a/logs/subject_a.esvee.command.run", + "subject_a/logs/subject_a.esvee.command.sh", + "subject_a/logs/subject_a.gatk4_markduplicates.command.log", + "subject_a/logs/subject_a.gatk4_markduplicates.command.sh", + "subject_a/logs/subject_a.isofox.command.err", + "subject_a/logs/subject_a.isofox.command.log", + "subject_a/logs/subject_a.isofox.command.out", + "subject_a/logs/subject_a.isofox.command.run", + "subject_a/logs/subject_a.isofox.command.sh", + "subject_a/logs/subject_a.lilac.command.err", + "subject_a/logs/subject_a.lilac.command.log", + "subject_a/logs/subject_a.lilac.command.out", + "subject_a/logs/subject_a.lilac.command.run", + "subject_a/logs/subject_a.lilac.command.sh", + "subject_a/logs/subject_a.linx_germline.command.err", + "subject_a/logs/subject_a.linx_germline.command.log", + "subject_a/logs/subject_a.linx_germline.command.out", + "subject_a/logs/subject_a.linx_germline.command.run", + "subject_a/logs/subject_a.linx_germline.command.sh", + "subject_a/logs/subject_a.linx_somatic.command.err", + "subject_a/logs/subject_a.linx_somatic.command.log", + "subject_a/logs/subject_a.linx_somatic.command.out", + "subject_a/logs/subject_a.linx_somatic.command.run", + "subject_a/logs/subject_a.linx_somatic.command.sh", + "subject_a/logs/subject_a.linx_visualiser.command.err", + "subject_a/logs/subject_a.linx_visualiser.command.log", + "subject_a/logs/subject_a.linx_visualiser.command.out", + "subject_a/logs/subject_a.linx_visualiser.command.run", + "subject_a/logs/subject_a.linx_visualiser.command.sh", + "subject_a/logs/subject_a.linxreport.command.err", + "subject_a/logs/subject_a.linxreport.command.log", + "subject_a/logs/subject_a.linxreport.command.out", + "subject_a/logs/subject_a.linxreport.command.run", + "subject_a/logs/subject_a.linxreport.command.sh", + "subject_a/logs/subject_a.neo_annotate_fusions.command.err", + "subject_a/logs/subject_a.neo_annotate_fusions.command.log", + "subject_a/logs/subject_a.neo_annotate_fusions.command.out", + "subject_a/logs/subject_a.neo_annotate_fusions.command.run", + "subject_a/logs/subject_a.neo_annotate_fusions.command.sh", + "subject_a/logs/subject_a.neo_finder.command.err", + "subject_a/logs/subject_a.neo_finder.command.log", + "subject_a/logs/subject_a.neo_finder.command.out", + "subject_a/logs/subject_a.neo_finder.command.run", + "subject_a/logs/subject_a.neo_finder.command.sh", + "subject_a/logs/subject_a.neo_scorer.command.err", + "subject_a/logs/subject_a.neo_scorer.command.log", + "subject_a/logs/subject_a.neo_scorer.command.out", + "subject_a/logs/subject_a.neo_scorer.command.run", + "subject_a/logs/subject_a.neo_scorer.command.sh", + "subject_a/logs/subject_a.orange.command.err", + "subject_a/logs/subject_a.orange.command.log", + "subject_a/logs/subject_a.orange.command.out", + "subject_a/logs/subject_a.orange.command.run", + "subject_a/logs/subject_a.orange.command.sh", + "subject_a/logs/subject_a.pave_germline.command.err", + "subject_a/logs/subject_a.pave_germline.command.log", + "subject_a/logs/subject_a.pave_germline.command.out", + "subject_a/logs/subject_a.pave_germline.command.run", + "subject_a/logs/subject_a.pave_germline.command.sh", + "subject_a/logs/subject_a.pave_somatic.command.err", + "subject_a/logs/subject_a.pave_somatic.command.log", + "subject_a/logs/subject_a.pave_somatic.command.out", + "subject_a/logs/subject_a.pave_somatic.command.run", + "subject_a/logs/subject_a.pave_somatic.command.sh", + "subject_a/logs/subject_a.peach.command.err", + "subject_a/logs/subject_a.peach.command.log", + "subject_a/logs/subject_a.peach.command.out", + "subject_a/logs/subject_a.peach.command.run", + "subject_a/logs/subject_a.peach.command.sh", + "subject_a/logs/subject_a.purple.command.err", + "subject_a/logs/subject_a.purple.command.log", + "subject_a/logs/subject_a.purple.command.out", + "subject_a/logs/subject_a.purple.command.run", + "subject_a/logs/subject_a.purple.command.sh", + "subject_a/logs/subject_a.sage_append_germline.command.err", + "subject_a/logs/subject_a.sage_append_germline.command.log", + "subject_a/logs/subject_a.sage_append_germline.command.out", + "subject_a/logs/subject_a.sage_append_germline.command.run", + "subject_a/logs/subject_a.sage_append_germline.command.sh", + "subject_a/logs/subject_a.sage_append_somatic.command.err", + "subject_a/logs/subject_a.sage_append_somatic.command.log", + "subject_a/logs/subject_a.sage_append_somatic.command.out", + "subject_a/logs/subject_a.sage_append_somatic.command.run", + "subject_a/logs/subject_a.sage_append_somatic.command.sh", + "subject_a/logs/subject_a.sage_germline.command.err", + "subject_a/logs/subject_a.sage_germline.command.log", + "subject_a/logs/subject_a.sage_germline.command.out", + "subject_a/logs/subject_a.sage_germline.command.run", + "subject_a/logs/subject_a.sage_germline.command.sh", + "subject_a/logs/subject_a.sage_somatic.command.err", + "subject_a/logs/subject_a.sage_somatic.command.log", + "subject_a/logs/subject_a.sage_somatic.command.out", + "subject_a/logs/subject_a.sage_somatic.command.run", + "subject_a/logs/subject_a.sage_somatic.command.sh", + "subject_a/logs/subject_a.sigs.command.err", + "subject_a/logs/subject_a.sigs.command.log", + "subject_a/logs/subject_a.sigs.command.out", + "subject_a/logs/subject_a.sigs.command.run", + "subject_a/logs/subject_a.sigs.command.sh", + "subject_a/logs/subject_a.teal_pipeline.command.err", + "subject_a/logs/subject_a.teal_pipeline.command.log", + "subject_a/logs/subject_a.teal_pipeline.command.out", + "subject_a/logs/subject_a.teal_pipeline.command.run", + "subject_a/logs/subject_a.teal_pipeline.command.sh", + "subject_a/logs/subject_a.teal_prep.command.err", + "subject_a/logs/subject_a.teal_prep.command.log", + "subject_a/logs/subject_a.teal_prep.command.out", + "subject_a/logs/subject_a.teal_prep.command.run", + "subject_a/logs/subject_a.teal_prep.command.sh", + "subject_a/logs/subject_a.virusbreakend.command.err", + "subject_a/logs/subject_a.virusbreakend.command.log", + "subject_a/logs/subject_a.virusbreakend.command.out", + "subject_a/logs/subject_a.virusbreakend.command.run", + "subject_a/logs/subject_a.virusbreakend.command.sh", + "subject_a/logs/subject_a.virusinterpreter.command.err", + "subject_a/logs/subject_a.virusinterpreter.command.log", + "subject_a/logs/subject_a.virusinterpreter.command.out", + "subject_a/logs/subject_a.virusinterpreter.command.run", + "subject_a/logs/subject_a.virusinterpreter.command.sh", + "subject_a/logs/subject_a_subject_a.normal.bamtools.command.err", + "subject_a/logs/subject_a_subject_a.normal.bamtools.command.log", + "subject_a/logs/subject_a_subject_a.normal.bamtools.command.out", + "subject_a/logs/subject_a_subject_a.normal.bamtools.command.run", + "subject_a/logs/subject_a_subject_a.normal.bamtools.command.sh", + "subject_a/logs/subject_a_subject_a.normal.redux.command.err", + "subject_a/logs/subject_a_subject_a.normal.redux.command.log", + "subject_a/logs/subject_a_subject_a.normal.redux.command.out", + "subject_a/logs/subject_a_subject_a.normal.redux.command.run", + "subject_a/logs/subject_a_subject_a.normal.redux.command.sh", + "subject_a/logs/subject_a_subject_a.tumor.bamtools.command.err", + "subject_a/logs/subject_a_subject_a.tumor.bamtools.command.log", + "subject_a/logs/subject_a_subject_a.tumor.bamtools.command.out", + "subject_a/logs/subject_a_subject_a.tumor.bamtools.command.run", + "subject_a/logs/subject_a_subject_a.tumor.bamtools.command.sh", + "subject_a/logs/subject_a_subject_a.tumor.cider.command.err", + "subject_a/logs/subject_a_subject_a.tumor.cider.command.log", + "subject_a/logs/subject_a_subject_a.tumor.cider.command.out", + "subject_a/logs/subject_a_subject_a.tumor.cider.command.run", + "subject_a/logs/subject_a_subject_a.tumor.cider.command.sh", + "subject_a/logs/subject_a_subject_a.tumor.redux.command.err", + "subject_a/logs/subject_a_subject_a.tumor.redux.command.log", + "subject_a/logs/subject_a_subject_a.tumor.redux.command.out", + "subject_a/logs/subject_a_subject_a.tumor.redux.command.run", + "subject_a/logs/subject_a_subject_a.tumor.redux.command.sh", + "subject_a/logs/subject_a_subject_a.tumor_rna.cider.command.err", + "subject_a/logs/subject_a_subject_a.tumor_rna.cider.command.log", + "subject_a/logs/subject_a_subject_a.tumor_rna.cider.command.out", + "subject_a/logs/subject_a_subject_a.tumor_rna.cider.command.run", + "subject_a/logs/subject_a_subject_a.tumor_rna.cider.command.sh", + "subject_a/neo", + "subject_a/neo/annotated_fusions", + "subject_a/neo/annotated_fusions/subject_a.tumor.isf.neoepitope.tsv", + "subject_a/neo/finder", + "subject_a/neo/finder/subject_a.tumor.neo.neo_data.tsv", + "subject_a/neo/scorer", + "subject_a/neo/scorer/subject_a.tumor.neo.neoepitope.tsv", + "subject_a/neo/scorer/subject_a.tumor.neo.peptide_scores.tsv", + "subject_a/orange", + "subject_a/orange/subject_a.tumor.orange.json", + "subject_a/orange/subject_a.tumor.orange.pdf", + "subject_a/pave", + "subject_a/pave/subject_a.tumor.pave.germline.vcf.gz", + "subject_a/pave/subject_a.tumor.pave.germline.vcf.gz.tbi", + "subject_a/pave/subject_a.tumor.pave.somatic.vcf.gz", + "subject_a/pave/subject_a.tumor.pave.somatic.vcf.gz.tbi", + "subject_a/peach", + "subject_a/peach/subject_a.normal.peach.events.tsv", + "subject_a/peach/subject_a.normal.peach.gene.events.tsv", + "subject_a/peach/subject_a.normal.peach.haplotypes.all.tsv", + "subject_a/peach/subject_a.normal.peach.haplotypes.best.tsv", + "subject_a/peach/subject_a.normal.peach.qc.tsv", + "subject_a/purple", + "subject_a/purple/circos", + "subject_a/purple/circos/gaps.txt", + "subject_a/purple/circos/subject_a.normal.ratio.circos", + "subject_a/purple/circos/subject_a.tumor.baf.circos", + "subject_a/purple/circos/subject_a.tumor.circos.conf", + "subject_a/purple/circos/subject_a.tumor.cnv.circos", + "subject_a/purple/circos/subject_a.tumor.indel.circos", + "subject_a/purple/circos/subject_a.tumor.input.conf", + "subject_a/purple/circos/subject_a.tumor.link.circos", + "subject_a/purple/circos/subject_a.tumor.map.circos", + "subject_a/purple/circos/subject_a.tumor.ratio.circos", + "subject_a/purple/circos/subject_a.tumor.snp.circos", + "subject_a/purple/plot", + "subject_a/purple/plot/subject_a.tumor.circos.png", + "subject_a/purple/plot/subject_a.tumor.copynumber.png", + "subject_a/purple/plot/subject_a.tumor.input.png", + "subject_a/purple/plot/subject_a.tumor.map.png", + "subject_a/purple/plot/subject_a.tumor.purity.range.png", + "subject_a/purple/plot/subject_a.tumor.segment.png", + "subject_a/purple/plot/subject_a.tumor.somatic.clonality.png", + "subject_a/purple/plot/subject_a.tumor.somatic.png", + "subject_a/purple/plot/subject_a.tumor.somatic.rainfall.png", + "subject_a/purple/purple.version", + "subject_a/purple/subject_a.tumor.purple.chromosome_arm.tsv", + "subject_a/purple/subject_a.tumor.purple.cnv.gene.tsv", + "subject_a/purple/subject_a.tumor.purple.cnv.somatic.tsv", + "subject_a/purple/subject_a.tumor.purple.driver.catalog.germline.tsv", + "subject_a/purple/subject_a.tumor.purple.driver.catalog.somatic.tsv", + "subject_a/purple/subject_a.tumor.purple.germline.deletion.tsv", + "subject_a/purple/subject_a.tumor.purple.germline.vcf.gz", + "subject_a/purple/subject_a.tumor.purple.germline.vcf.gz.tbi", + "subject_a/purple/subject_a.tumor.purple.purity.range.tsv", + "subject_a/purple/subject_a.tumor.purple.purity.tsv", + "subject_a/purple/subject_a.tumor.purple.qc", + "subject_a/purple/subject_a.tumor.purple.segment.tsv", + "subject_a/purple/subject_a.tumor.purple.somatic.clonality.tsv", + "subject_a/purple/subject_a.tumor.purple.somatic.hist.tsv", + "subject_a/purple/subject_a.tumor.purple.somatic.vcf.gz", + "subject_a/purple/subject_a.tumor.purple.somatic.vcf.gz.tbi", + "subject_a/purple/subject_a.tumor.purple.sv.germline.vcf.gz", + "subject_a/purple/subject_a.tumor.purple.sv.germline.vcf.gz.tbi", + "subject_a/purple/subject_a.tumor.purple.sv.vcf.gz", + "subject_a/purple/subject_a.tumor.purple.sv.vcf.gz.tbi", + "subject_a/sage", + "subject_a/sage/germline", + "subject_a/sage/germline/subject_a.normal.sage.bqr.png", + "subject_a/sage/germline/subject_a.normal.sage.bqr.tsv", + "subject_a/sage/germline/subject_a.tumor.sage.bqr.png", + "subject_a/sage/germline/subject_a.tumor.sage.bqr.tsv", + "subject_a/sage/germline/subject_a.tumor.sage.germline.vcf.gz", + "subject_a/sage/germline/subject_a.tumor.sage.germline.vcf.gz.tbi", + "subject_a/sage/somatic", + "subject_a/sage/somatic/subject_a.normal.sage.bqr.png", + "subject_a/sage/somatic/subject_a.normal.sage.bqr.tsv", + "subject_a/sage/somatic/subject_a.tumor.sage.bqr.png", + "subject_a/sage/somatic/subject_a.tumor.sage.bqr.tsv", + "subject_a/sage/somatic/subject_a.tumor.sage.somatic.fit_variant.tsv", + "subject_a/sage/somatic/subject_a.tumor.sage.somatic.vcf.gz", + "subject_a/sage/somatic/subject_a.tumor.sage.somatic.vcf.gz.tbi", + "subject_a/sage_append", + "subject_a/sage_append/germline", + "subject_a/sage_append/germline/subject_a.normal.sage.append.frag_lengths.tsv.gz", + "subject_a/sage_append/germline/subject_a.normal.sage.append.vcf.gz", + "subject_a/sage_append/germline/subject_a.normal.sage.append.vcf.gz.tbi", + "subject_a/sage_append/germline/subject_a.tumor_rna.sage.bqr.tsv", + "subject_a/sage_append/somatic", + "subject_a/sage_append/somatic/subject_a.tumor.sage.append.frag_lengths.tsv.gz", + "subject_a/sage_append/somatic/subject_a.tumor.sage.append.vcf.gz", + "subject_a/sage_append/somatic/subject_a.tumor.sage.append.vcf.gz.tbi", + "subject_a/sage_append/somatic/subject_a.tumor_rna.sage.bqr.tsv", + "subject_a/sigs", + "subject_a/sigs/subject_a.tumor.sig.allocation.tsv", + "subject_a/sigs/subject_a.tumor.sig.snv_counts.csv", + "subject_a/teal", + "subject_a/teal/subject_a.normal.teal.telbam.bam", + "subject_a/teal/subject_a.normal.teal.telbam.bam.bai", + "subject_a/teal/subject_a.normal.teal.tellength.tsv", + "subject_a/teal/subject_a.tumor.teal.breakend.tsv.gz", + "subject_a/teal/subject_a.tumor.teal.telbam.bam", + "subject_a/teal/subject_a.tumor.teal.telbam.bam.bai", + "subject_a/teal/subject_a.tumor.teal.tellength.tsv", + "subject_a/virusbreakend", + "subject_a/virusbreakend/subject_a.tumor.virusbreakend.vcf", + "subject_a/virusbreakend/subject_a.tumor.virusbreakend.vcf.summary.tsv", + "subject_a/virusinterpreter", + "subject_a/virusinterpreter/subject_a.tumor.virus.annotated.tsv" + ] + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "25.10.2" + }, + "timestamp": "2026-01-07T19:53:31.782408269" + }, "-profile test_stub": { "content": [ 34, @@ -323,9 +760,9 @@ ] ], "meta": { - "nf-test": "0.9.3", + "nf-test": "0.9.2", "nextflow": "25.10.2" }, - "timestamp": "2025-12-03T04:25:08.751931146" + "timestamp": "2026-01-09T22:31:27.594228294" } } \ No newline at end of file diff --git a/tests/nextflow.config b/tests/nextflow.config index 5d979d45..23e0515d 100644 --- a/tests/nextflow.config +++ b/tests/nextflow.config @@ -4,4 +4,14 @@ ======================================================================================== */ -aws.client.anonymous = true // fixes S3 access issues on self-hosted runners +aws.client.anonymous = true + +// Conditionally include cached reference data paths if NXF_REF_DATA_PATH is set +// Uses immediately-invoked closure pattern for strict syntax compatibility +includeConfig ({ + def refPath = System.getenv('NXF_REF_DATA_PATH') + if (refPath) { + return 'cached_refs.config' + } + return '/dev/null' +}()) diff --git a/tests/profile_test.nf.test b/tests/profile_test.nf.test index a7b03850..cfb08f3b 100644 --- a/tests/profile_test.nf.test +++ b/tests/profile_test.nf.test @@ -20,11 +20,14 @@ nextflow_pipeline { def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}']) // stable_path: All files in ${params.outdir}/ with stable content def stable_path = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore') + // Task count depends on config: when NXF_REF_DATA_PATH is set, reference data is cached (no DECOMP tasks) + def refDataCached = System.getenv('NXF_REF_DATA_PATH') != null + def expectedTaskCount = refDataCached ? 39 : 43 + def actualTaskCount = workflow.trace.succeeded().size() assertAll( - { assert workflow.success}, + { assert workflow.success }, + { assert actualTaskCount == expectedTaskCount : "Expected ${expectedTaskCount} tasks (ref data cached: ${refDataCached}), got ${actualTaskCount}" }, { assert snapshot( - // Number of successful tasks - workflow.trace.succeeded().size(), // All stable path name, with a relative path stable_name, ).match() } diff --git a/tests/profile_test.nf.test.snap b/tests/profile_test.nf.test.snap index 1b476c2d..5474746f 100644 --- a/tests/profile_test.nf.test.snap +++ b/tests/profile_test.nf.test.snap @@ -1,7 +1,6 @@ { "-profile test": { "content": [ - 43, [ "pipeline_info", "pipeline_info/software_versions.yml", @@ -432,9 +431,9 @@ ] ], "meta": { - "nf-test": "0.9.3", + "nf-test": "0.9.2", "nextflow": "25.10.2" }, - "timestamp": "2025-12-03T05:13:51.245667357" + "timestamp": "2026-01-08T12:38:40.441795159" } } \ No newline at end of file