Draft: Update feat/dnase-2.7 with changes from main

.gitignore

@@ -12,7 +12,10 @@ environments/
 **/.nextflow*
 
 **/*html*
-**/*trace.txt*
+**/trace*txt*
+**/report*txt*
+
+**/dag*dot*
 
 
 *.fq
@@ -24,6 +27,7 @@ environments/
 !.gitattributes
 !.gitmodules
 !.pylintrc
+!.pre-commit-config.yaml
 
 work
 genome_build

.pre-commit-config.yaml

@@ -0,0 +1,9 @@
+repos:
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  # Ruff version.
+  rev: v0.5.1
+  hooks:
+    # Run the linter.
+    - id: ruff
+    # Run the formatter.
+    - id: ruff-format

Dockerfile

@@ -65,14 +65,15 @@ RUN apt-get install -y \
     g++ \
     git \
     libbz2-dev \
+    libcurl4-openssl-dev \
     liblzma-dev \
     make \
     ncurses-dev \
     wget \
     zlib1g-dev
-RUN wget --quiet https://github.com/samtools/samtools/releases/download/1.7/samtools-1.7.tar.bz2 \
-      && tar xf samtools-1.7.tar.bz2 \
-      && cd samtools-1.7 \
+RUN wget --quiet https://github.com/samtools/samtools/releases/download/1.12/samtools-1.12.tar.bz2 \
+      && tar xf samtools-1.12.tar.bz2 \
+      && cd samtools-1.12 \
       && make install
 
 #####################
@@ -151,9 +152,8 @@ RUN wget --quiet https://github.com/arq5x/bedtools2/releases/download/v2.25.0/be
 from build-base as build-preseq
 RUN apt-get install -y \
       libgsl-dev
-RUN git clone --recurse-submodules https://github.com/smithlabcode/preseq.git \
+RUN git clone --recurse-submodules -b v2.0.1 --single-branch https://github.com/smithlabcode/preseq.git \
    && cd preseq \
-   && git checkout v2.0.1 \
    && make
 
 
@@ -168,8 +168,9 @@ RUN apt-get install -y \
       bc \
       bowtie \
       build-essential \
-      libboost-dev \
       coreutils \
+      gawk \
+      libboost-dev \
       libgsl-dev \
       littler \
       openjdk-8-jre \

README.md

@@ -19,7 +19,6 @@ Modules
 
 The current modules to load for working the pipeline are:
 
-* `anaquin/2.0.1`
 * `bcl2fastq/1.8.4`
 * `bcl2fastq2/2.15.0.4`
 * `bedops/2.4.35-typical`

containers/fastq/fastq.def

@@ -0,0 +1,46 @@
+Bootstrap: docker
+From: rockylinux:8.6
+
+%arguments
+	FASTQC_VERSION=v0.11.5
+
+%help
+	This container contains the necessary software to run the first stage of
+	the pipeline, beginning from retriving metadata from LIMS, running
+	bcl2fastq and demultiplexing, collating fastq files, running FastQC, and
+	registering output with LIMS.
+
+%files
+	./bcl2fastq /usr/local/bin/bcl2fastq
+	../../scripts/lims/stamlims_api /opt/
+
+%post
+	set -e
+	dnf install -y python39
+	ln -s /usr/bin/python3 /usr/bin/python
+	python -m pip install /opt/stamlims_api/
+	python -m pip install Biopython
+	dnf install -y rsync
+
+	# install fastqc
+	# https://github.com/s-andrews/FastQC/
+	(
+		dnf install -y unzip perl java
+		cd /opt 
+		curl https://www.bioinformatics.babraham.ac.uk/projects/fastqc/fastqc_{{ FASTQC_VERSION }}.zip --output fastqc.zip
+		unzip fastqc.zip
+		rm fastqc.zip
+		chmod +x FastQC/fastqc
+		ln -s /opt/FastQC/fastqc /usr/local/bin/fastqc
+	)
+
+
+%test
+	set -e
+	bcl2fastq --version
+	rsync --version
+	fastqc --version
+	gzip --version
+	python3 --version
+
+# vim: noexpandtab ts=4 sts=4 sw=4

data/flowcells/miniseq/example_SampleSheet.csv

@@ -0,0 +1,127 @@
+[Header],,,,,,,,,,
+IEMFileVersion,4,,,,,,,,,
+Investigator Name,Janghee,,,,,,,,,
+Project Name,2015_09_30,,,,,,,,,
+Experiment Name,,,,,,,,,,
+Date,9/30/15,,,,,,,,,
+Workflow,Resequencing,,,,,,,,,
+Application,TruSeq DNA/RNA,,,,,,,,,
+Assay,TruSeq DNA/RNA,,,,,,,,,
+Description,,,,,,,,,,
+Chemistry,Amplicon,,,,,,,,,
+,,,,,,,,,,
+[Reads],,,,,,,,,,
+150,,,,,,,,,,
+150,,,,,,,,,,
+[Settings],,,,,,,,,,
+OnlyGenerateFASTQ,1,,,,,,,,,
+[Data],,,,,,,,,,
+Sample_ID,Sample_Name,Sample_Plate,Sample_Well,I7_Index_ID,index,I5_Index_ID,index2,Sample_Project,Description,GenomeFolder
+A1-i7-1-N501,,,,i7_1,AACGTGAT,N501,GCGATCTA,,,
+B1-i7-1-N502,,,,i7_1,AACGTGAT,N502,ATAGAGAG,,,
+C1-i7-1-N503,,,,i7_1,AACGTGAT,N503,AGAGGATA,,,
+D1-i7-1-N504,,,,i7_1,AACGTGAT,N504,TCTACTCT,,,
+E1-i7-1-N505,,,,i7_1,AACGTGAT,N505,CTCCTTAC,,,
+F1-i7-1-N506,,,,i7_1,AACGTGAT,N506,TATGCAGT,,,
+G1-i7-1-N507,,,,i7_1,AACGTGAT,N507,TACTCCTT,,,
+H1-i7-1-N508,,,,i7_1,AACGTGAT,N508,AGGCTTAG,,,
+X1-i7-1-N517,,,,i7_1,AACGTGAT,N517,TCTTACGC,,,
+A2-i7-2-N501,,,,i7_2,AAACATCG,N501,GCGATCTA,,,
+B2-i7-2-N502,,,,i7_2,AAACATCG,N502,ATAGAGAG,,,
+C2-i7-2-N503,,,,i7_2,AAACATCG,N503,AGAGGATA,,,
+D2-i7-2-N504,,,,i7_2,AAACATCG,N504,TCTACTCT,,,
+E2-i7-2-N505,,,,i7_2,AAACATCG,N505,CTCCTTAC,,,
+F2-i7-2-N506,,,,i7_2,AAACATCG,N506,TATGCAGT,,,
+G2-i7-2-N507,,,,i7_2,AAACATCG,N507,TACTCCTT,,,
+H2-i7-2-N508,,,,i7_2,AAACATCG,N508,AGGCTTAG,,,
+X2-i7-2-N517,,,,i7_2,AAACATCG,N517,TCTTACGC,,,
+A3-i7-3-N501,,,,i7_3,ATGCCTAA,N501,GCGATCTA,,,
+B3-i7-3-N502,,,,i7_3,ATGCCTAA,N502,ATAGAGAG,,,
+C3-i7-3-N503,,,,i7_3,ATGCCTAA,N503,AGAGGATA,,,
+D3-i7-3-N504,,,,i7_3,ATGCCTAA,N504,TCTACTCT,,,
+E3-i7-3-N505,,,,i7_3,ATGCCTAA,N505,CTCCTTAC,,,
+F3-i7-3-N506,,,,i7_3,ATGCCTAA,N506,TATGCAGT,,,
+G3-i7-3-N507,,,,i7_3,ATGCCTAA,N507,TACTCCTT,,,
+H3-i7-3-N508,,,,i7_3,ATGCCTAA,N508,AGGCTTAG,,,
+X3-i7-3-N517,,,,i7_3,ATGCCTAA,N517,TCTTACGC,,,
+A4-i7-4-N501,,,,i7_4,AGTGGTCA,N501,GCGATCTA,,,
+B4-i7-4-N502,,,,i7_4,AGTGGTCA,N502,ATAGAGAG,,,
+C4-i7-4-N503,,,,i7_4,AGTGGTCA,N503,AGAGGATA,,,
+D4-i7-4-N504,,,,i7_4,AGTGGTCA,N504,TCTACTCT,,,
+E4-i7-4-N505,,,,i7_4,AGTGGTCA,N505,CTCCTTAC,,,
+F4-i7-4-N506,,,,i7_4,AGTGGTCA,N506,TATGCAGT,,,
+G4-i7-4-N507,,,,i7_4,AGTGGTCA,N507,TACTCCTT,,,
+H4-i7-4-N508,,,,i7_4,AGTGGTCA,N508,AGGCTTAG,,,
+X4-i7-4-N517,,,,i7_4,AGTGGTCA,N517,TCTTACGC,,,
+A5-i7-5-N501,,,,i7_5,ACCACTGT,N501,GCGATCTA,,,
+B5-i7-5-N502,,,,i7_5,ACCACTGT,N502,ATAGAGAG,,,
+C5-i7-5-N503,,,,i7_5,ACCACTGT,N503,AGAGGATA,,,
+D5-i7-5-N504,,,,i7_5,ACCACTGT,N504,TCTACTCT,,,
+E5-i7-5-N505,,,,i7_5,ACCACTGT,N505,CTCCTTAC,,,
+F5-i7-5-N506,,,,i7_5,ACCACTGT,N506,TATGCAGT,,,
+G5-i7-5-N507,,,,i7_5,ACCACTGT,N507,TACTCCTT,,,
+H5-i7-5-N508,,,,i7_5,ACCACTGT,N508,AGGCTTAG,,,
+X5-i7-5-N517,,,,i7_5,ACCACTGT,N517,TCTTACGC,,,
+A6-i7-6-N501,,,,i7_6,ACATTGGC,N501,GCGATCTA,,,
+B6-i7-6-N502,,,,i7_6,ACATTGGC,N502,ATAGAGAG,,,
+C6-i7-6-N503,,,,i7_6,ACATTGGC,N503,AGAGGATA,,,
+D6-i7-6-N504,,,,i7_6,ACATTGGC,N504,TCTACTCT,,,
+E6-i7-6-N505,,,,i7_6,ACATTGGC,N505,CTCCTTAC,,,
+F6-i7-6-N506,,,,i7_6,ACATTGGC,N506,TATGCAGT,,,
+G6-i7-6-N507,,,,i7_6,ACATTGGC,N507,TACTCCTT,,,
+H6-i7-6-N508,,,,i7_6,ACATTGGC,N508,AGGCTTAG,,,
+X6-i7-6-N517,,,,i7_6,ACATTGGC,N517,TCTTACGC,,,
+A7-i7-7-N501,,,,i7_7,CAGATCTG,N501,GCGATCTA,,,
+B7-i7-7-N502,,,,i7_7,CAGATCTG,N502,ATAGAGAG,,,
+C7-i7-7-N503,,,,i7_7,CAGATCTG,N503,AGAGGATA,,,
+D7-i7-7-N504,,,,i7_7,CAGATCTG,N504,TCTACTCT,,,
+E7-i7-7-N505,,,,i7_7,CAGATCTG,N505,CTCCTTAC,,,
+F7-i7-7-N506,,,,i7_7,CAGATCTG,N506,TATGCAGT,,,
+G7-i7-7-N507,,,,i7_7,CAGATCTG,N507,TACTCCTT,,,
+H7-i7-7-N508,,,,i7_7,CAGATCTG,N508,AGGCTTAG,,,
+X7-i7-7-N517,,,,i7_7,CAGATCTG,N517,TCTTACGC,,,
+A8-i7-8-N501,,,,i7_8,CATCAAGT,N501,GCGATCTA,,,
+B8-i7-8-N502,,,,i7_8,CATCAAGT,N502,ATAGAGAG,,,
+C8-i7-8-N503,,,,i7_8,CATCAAGT,N503,AGAGGATA,,,
+D8-i7-8-N504,,,,i7_8,CATCAAGT,N504,TCTACTCT,,,
+E8-i7-8-N505,,,,i7_8,CATCAAGT,N505,CTCCTTAC,,,
+F8-i7-8-N506,,,,i7_8,CATCAAGT,N506,TATGCAGT,,,
+G8-i7-8-N507,,,,i7_8,CATCAAGT,N507,TACTCCTT,,,
+H8-i7-8-N508,,,,i7_8,CATCAAGT,N508,AGGCTTAG,,,
+X8-i7-8-N517,,,,i7_8,CATCAAGT,N517,TCTTACGC,,,
+A9-i7-9-N501,,,,i7_9,CGCTGATC,N501,GCGATCTA,,,
+B9-i7-9-N502,,,,i7_9,CGCTGATC,N502,ATAGAGAG,,,
+C9-i7-9-N503,,,,i7_9,CGCTGATC,N503,AGAGGATA,,,
+D9-i7-9-N504,,,,i7_9,CGCTGATC,N504,TCTACTCT,,,
+E9-i7-9-N505,,,,i7_9,CGCTGATC,N505,CTCCTTAC,,,
+F9-i7-9-N506,,,,i7_9,CGCTGATC,N506,TATGCAGT,,,
+G9-i7-9-N507,,,,i7_9,CGCTGATC,N507,TACTCCTT,,,
+H9-i7-9-N508,,,,i7_9,CGCTGATC,N508,AGGCTTAG,,,
+X9-i7-9-N517,,,,i7_9,CGCTGATC,N517,TCTTACGC,,,
+A10-i7-10-N501,,,,i7_10,ACAAGCTA,N501,GCGATCTA,,,
+B10-i7-10-N502,,,,i7_10,ACAAGCTA,N502,ATAGAGAG,,,
+C10-i7-10-N503,,,,i7_10,ACAAGCTA,N503,AGAGGATA,,,
+D10-i7-10-N504,,,,i7_10,ACAAGCTA,N504,TCTACTCT,,,
+E10-i7-10-N505,,,,i7_10,ACAAGCTA,N505,CTCCTTAC,,,
+F10-i7-10-N506,,,,i7_10,ACAAGCTA,N506,TATGCAGT,,,
+G10-i7-10-N507,,,,i7_10,ACAAGCTA,N507,TACTCCTT,,,
+H10-i7-10-N508,,,,i7_10,ACAAGCTA,N508,AGGCTTAG,,,
+X10-i7-10-N517,,,,i7_10,ACAAGCTA,N517,TCTTACGC,,,
+A11-i7-11-N501,,,,i7_11,CTGTAGCC,N501,GCGATCTA,,,
+B11-i7-11-N502,,,,i7_11,CTGTAGCC,N502,ATAGAGAG,,,
+C11-i7-11-N503,,,,i7_11,CTGTAGCC,N503,AGAGGATA,,,
+D11-i7-11-N504,,,,i7_11,CTGTAGCC,N504,TCTACTCT,,,
+E11-i7-11-N505,,,,i7_11,CTGTAGCC,N505,CTCCTTAC,,,
+F11-i7-11-N506,,,,i7_11,CTGTAGCC,N506,TATGCAGT,,,
+G11-i7-11-N507,,,,i7_11,CTGTAGCC,N507,TACTCCTT,,,
+H11-i7-11-N508,,,,i7_11,CTGTAGCC,N508,AGGCTTAG,,,
+X11-i7-11-N517,,,,i7_11,CTGTAGCC,N517,TCTTACGC,,,
+A12-i7-12-N501,,,,i7_12,AGTACAAG,N501,GCGATCTA,,,
+B12-i7-12-N502,,,,i7_12,AGTACAAG,N502,ATAGAGAG,,,
+C12-i7-12-N503,,,,i7_12,AGTACAAG,N503,AGAGGATA,,,
+D12-i7-12-N504,,,,i7_12,AGTACAAG,N504,TCTACTCT,,,
+E12-i7-12-N505,,,,i7_12,AGTACAAG,N505,CTCCTTAC,,,
+F12-i7-12-N506,,,,i7_12,AGTACAAG,N506,TATGCAGT,,,
+G12-i7-12-N507,,,,i7_12,AGTACAAG,N507,TACTCCTT,,,
+H12-i7-12-N508,,,,i7_12,AGTACAAG,N508,AGGCTTAG,,,
+X12-i7-12-N517,,,,i7_12,AGTACAAG,N517,TCTTACGC,,,

modules/adapter_trimming.nf

@@ -6,6 +6,7 @@ process fastp_adapter_trim {
   // 0.23.0 enables deterministic results, which is crucial
   module 'fastp/0.21.0'
   cpus 3
+  scratch false
 
   input:
     tuple path(r1), path(r2), val(adapterP5), val(adapterP7)
@@ -31,8 +32,30 @@ process fastp_adapter_trim {
     """
 }
 
+process agent_adapter_trim {
+  cpus 1
+  scratch false
+  module "jdk/11.0.16"
+
+  input:
+    tuple path(r1), path(r2)
+
+  output:
+    path 'output_R?.fastq.gz', emit: fastq
+
+  script:
+    """
+    agent.sh trim -v2 \
+      -fq1 "${r1}" \
+      -fq2 "${r2}" \
+      -out \$PWD/output
+    """
+}
+
 /// Our custom in-house adapter-trimming script
 process adapter_trim {
+  cpus 3
+  scratch false
   input:
     tuple path(r1), path(r2), val(adapterP5), val(adapterP7)