src/example_variant_calling.nf

+// SPDX-FileCopyrightText: 2022 Laurent Modolo <laurent.modolo@ens-lyon.fr>
+//
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+nextflow.enable.dsl=2
+
+/*
+Testing pipeline for marseq scRNASeq analysis
+*/
+
+include {
+  mapping;
+} from "./nf_modules/bwa/main.nf"
+
+include {
+  sort_bam;
+} from "./nf_modules/samtools/main.nf"
+
+include {
+  germline_cohort_data_variant_calling;
+} from "./nf_modules/gatk4/main.nf" addParams(
+  variant_calling_out: "vcf/",
+)
+
+params.fastq = ""
+params.fasta = ""
+
+channel
+  .fromFilePairs( params.fastq, size: -1)
+  .set { fastq_files }
+channel
+  .fromPath( params.fasta )
+  .map { it -> [it.simpleName, it]}
+  .set { fasta_files }
+
+workflow {
+  mapping(fasta_files, fastq_files)
+  sort_bam(mapping.out.bam)
+  germline_cohort_data_variant_calling(sort_bam.out.bam, fasta_files)
+}

src/fasta_sampler.nf

+// SPDX-FileCopyrightText: 2022 Laurent Modolo <laurent.modolo@ens-lyon.fr>
+//
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+Channel
+  .fromPath( "data/tiny_dataset/fasta/*.fasta" )
+  .set { fasta_file }
+
+process sample_fasta {
+  publishDir "results/sampling/", mode: 'copy'
+
+  input:
+file fasta from fasta_file
+
+  output:
+file "*_sample.fasta" into fasta_sample
+
+  script:
+"""
+head ${fasta} > ${fasta.baseName}_sample.fasta
+"""
+}

src/in2p3.pbs

+#!/bin/sh
+
+# SPDX-FileCopyrightText: 2022 Laurent Modolo <laurent.modolo@ens-lyon.fr>
+#
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+#SBATCH --job-name=nextflow
+
+#SBATCH --ntasks=1
+#SBATCH --output=results/nextflow_%j.log
+#SBATCH --licenses=sps
+
+####################################
+
+# change to your path 
+SCRATCH=/sps/lbmc/lmodolo/nextflow
+
+${SCRATCH}/nextflow run ${SCRATCH}/src/solution_RNASeq.nf -profile ccin2p3 \
+--fastq "${SCRATCH}/data/tiny_dataset/fastq/*_R{1,2}.fastq" \
+--fasta "${SCRATCH}/data/tiny_dataset/fasta/tiny_v2.fasta" \
+--bed "${SCRATCH}/data/tiny_dataset/annot/tiny.bed" \
+-ansi-log \
+-w "${SCRATCH}/work/"
+
+wait

src/install_nextflow.sh

 #!/bin/sh
 
+# SPDX-FileCopyrightText: 2022 Laurent Modolo <laurent.modolo@ens-lyon.fr>
+#
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
 java -version
 curl -s https://get.nextflow.io | bash

src/nextflow.config

+// SPDX-FileCopyrightText: 2022 Laurent Modolo <laurent.modolo@ens-lyon.fr>
+//
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+nextflowVersion = '>=20'
+
+manifest {
+    homePage = 'https://gitbio.ens-lyon.fr/LBMC/nextflow'
+    description = 'pipeline to '
+    mainScript = 'main.nf'
+    version = '0.0.0'
+}
+
+report {
+  enabled = true
+  file = "$baseDir/../results/report.html"
+}
+
+profiles {
+  docker {
+    docker.temp = "auto"
+    docker.enabled = true
+    process {
+      errorStrategy = 'finish'
+      memory = '16GB'
+      withLabel: big_mem_mono_cpus {
+        cpus = 1
+      }
+      withLabel: big_mem_multi_cpus {
+        cpus = 4
+      }
+      withLabel: small_mem_mono_cpus {
+        cpus = 1
+        memory = '2GB'
+      }
+      withLabel: small_mem_multi_cpus {
+        cpus = 4
+        memory = '2GB'
+      }
+    }
+  }
+  podman {
+    charliecloud.enabled = true
+    charliecloud.cacheDir = "./bin/"
+    process {
+      errorStrategy = 'finish'
+      memory = '16GB'
+      withLabel: big_mem_mono_cpus {
+        cpus = 1
+      }
+      withLabel: big_mem_multi_cpus {
+        cpus = 4
+      }
+      withLabel: small_mem_mono_cpus {
+        cpus = 1
+        memory = '2GB'
+      }
+      withLabel: small_mem_multi_cpus {
+        cpus = 4
+        memory = '2GB'
+      }
+    }
+  }
+  singularity {
+    singularity.enabled = true
+    singularity.cacheDir = "./bin/"
+    process {
+      errorStrategy = 'finish'
+      memory = '16GB'
+      withLabel: big_mem_mono_cpus {
+        cpus = 1
+      }
+      withLabel: big_mem_multi_cpus {
+        cpus = 4
+      }
+      withLabel: small_mem_mono_cpus {
+        cpus = 1
+        memory = '2GB'
+      }
+      withLabel: small_mem_multi_cpus {
+        cpus = 4
+        memory = '2GB'
+      }
+    }
+  }
+  psmn {
+    charliecloud.enabled = true
+    charliecloud.cacheDir = "/Xnfs/abc/charliecloud"
+    charliecloud.runOptions = "--bind /scratch:/scratch --bind /Xnfs:/Xnfs --bind /home:/home"
+    charliecloud.readOnlyInputs = true
+    process{
+      errorStrategy = { sleep(Math.pow(2, task.attempt) * 200 as long); return 'retry' }
+      maxRetries = 3
+      executor = "slurm"
+      queue = "Lake"
+      withLabel: big_mem_mono_cpus {
+        cpus = 1
+        memory = "128GB"
+        time = "24h"
+      }
+      withLabel: big_mem_multi_cpus {
+        cpus = 32
+        memory = "192GB"
+        time = "24h"
+      }
+      withLabel: small_mem_mono_cpus {
+        cpus = 1
+        memory = "16GB"
+        time = "24h"
+      }
+      withLabel: small_mem_multi_cpus {
+        cpus = 32
+        memory = "16GB"
+        time = "24h"
+      }
+    }
+  }
+  ccin2p3 {
+    singularity.enabled = true
+    singularity.cacheDir = "$baseDir/../bin/"
+    singularity.runOptions = "--bind /pbs,/sps,/scratch,/tmp"
+    process{
+      maxRetries = 3
+      withLabel: big_mem_mono_cpus {
+        scratch = true
+        stageInMode = "copy"
+        stageOutMode = "rsync"
+        executor = "slurm"
+        clusterOptions = "--licenses=sps"
+        cpus = 1
+        memory = "8GB"
+        queue = "htc"
+      }
+      withLabel: big_mem_multi_cpus {
+        scratch = true
+        stageInMode = "copy"
+        stageOutMode = "rsync"
+        executor = "slurm"
+        clusterOptions = "--licenses=sps"
+        cpus = 1
+        memory = "8GB"
+        queue = "htc"
+      }
+      withLabel: small_mem_mono_cpus {
+        scratch = true
+        stageInMode = "copy"
+        stageOutMode = "rsync"
+        executor = "slurm"
+        clusterOptions = "--licenses=sps"
+        cpus = 1
+        memory = "8GB"
+        queue = "htc"
+      }
+      withLabel: small_mem_multi_cpus {
+        scratch = true
+        stageInMode = "copy"
+        stageOutMode = "rsync"
+        executor = "slurm"
+        clusterOptions = "--licenses=sps"
+        cpus = 1
+        memory = "8GB"
+        queue = "htc"
+      }
+    }
+  }
+}

src/nf_modules/BEDtools/fasta_from_bed.config

-profiles {
-  docker {
-    docker.temp = 'auto'
-    docker.enabled = true
-    process {
-      withName: fasta_from_bed {
-        container = "bedtools:2.25.0"
-      }
-    }
-  }
-  psmn {
-    process{
-      withName: fasta_from_bed {
-        beforeScript = "source /usr/share/lmod/lmod/init/bash; module use ~/privatemodules"
-        module = "BEDtools/2.25.0"
-        executor = "sge"
-        clusterOptions = "-m e -cwd -V"
-        memory = "20GB"
-        time = "12h"
-        queue = 'monointeldeb128'
-      }
-    }
-  }
-}

src/nf_modules/BEDtools/fasta_from_bed.nf

-/*
-* bedtools :
-* Imputs : fasta files
-* Imputs : bed files
-* Output : fasta files
-*/
-/*                      fasta extraction                                     */
-
-params.fasta = "$baseDir/data/fasta/*.fasta"
-params.bed = "$baseDir/data/annot/*.bed"
-
-log.info "fasta file : ${params.fasta}"
-log.info "bed file : ${params.bed}"
-
-Channel
-  .fromPath( params.fasta )
-  .ifEmpty { error "Cannot find any fasta files matching: ${params.fasta}" }
-  .set { fasta_files }
-Channel
-  .fromPath( params.bed )
-  .ifEmpty { error "Cannot find any bed files matching: ${params.bed}" }
-  .set { bed_files }
-
-process fasta_from_bed {
-  tag "${bed.baseName}"
-  cpus 4
-  publishDir "results/fasta/", mode: 'copy'
-
-  input:
-  file fasta from fasta_files
-  file bed from bed_files
-
-  output:
-  file "*_extracted.fasta" into fasta_files_extracted
-
-  script:
-"""
-bedtools getfasta -name \
--fi ${fasta} -bed ${bed} -fo ${bed.baseName}_extracted.fasta
-"""
-}

src/nf_modules/BEDtools/tests.sh

-./nextflow src/nf_modules/BEDtools/fasta_from_bed.nf \
-  -c src/nf_modules/BEDtools/fasta_from_bed.config \
-  -profile docker \
-  --fasta "data/tiny_dataset/fasta/tiny_v2.fasta" \
-  --bed "data/tiny_dataset/annot/tiny.bed" \

src/nf_modules/BWA/indexing.config

-profiles {
-  docker {
-    docker.temp = 'auto'
-    docker.enabled = true
-    process {
-      withName: index_fasta {
-        container = "bwa:0.7.17"
-      }
-    }
-  }
-  psmn {
-    process{
-      withName: index_fasta {
-        beforeScript = "source /usr/share/lmod/lmod/init/bash; module use ~/privatemodules"
-        module = "BWA/0.7.17"
-        executor = "sge"
-        clusterOptions = "-m e -cwd -V"
-        memory = "30GB"
-        time = "12h"
-        queue = 'monointeldeb128'
-      }
-    }
-  }
-}

src/nf_modules/BWA/indexing.nf

-params.fasta = "$baseDir/data/bam/*.fasta"
-
-log.info "fasta files : ${params.fasta}"
-
-Channel
-  .fromPath( params.fasta )
-  .ifEmpty { error "Cannot find any bam files matching: ${params.fasta}" }
-  .map { it -> [(it.baseName =~ /([^\.]*)/)[0][1], it]}
-  .set { fasta_file }
-
-process index_fasta {
-  tag "$fasta_id"
-  cpus 4
-  publishDir "results/mapping/index/", mode: 'copy'
-
-  input:
-    set fasta_id, file(fasta) from fasta_file
-
-  output:
-    set fasta_id, "${fasta.baseName}.*" into index_files
-    file "*_bwa_report.txt" into index_files_report
-
-  script:
-"""
-bwa index -p ${fasta.baseName} ${fasta} \
-&> ${fasta.baseName}_bwa_report.txt
-"""
-}
-

src/nf_modules/BWA/mapping_paired.config

-profiles {
-  docker {
-    docker.temp = 'auto'
-    docker.enabled = true
-    process {
-      withName: mapping_fastq {
-        container = "bwa:0.7.17"
-      }
-    }
-  }
-  psmn {
-    process{
-      withName: mapping_fastq {
-        beforeScript = "source /usr/share/lmod/lmod/init/bash; module use ~/privatemodules"
-        module = "BWA/0.7.17"
-        executor = "sge"
-        clusterOptions = "-m e -cwd -V"
-        memory = "30GB"
-        time = "24h"
-        queue = 'E5-2670deb128A,E5-2670deb128B,E5-2670deb128C,E5-2670deb128D,E5-2670deb128E,E5-2670deb128F'
-        penv = 'openmp16'
-      }
-    }
-  }
-}

src/nf_modules/BWA/mapping_paired.nf

-params.fastq = "$baseDir/data/fastq/*_{1,2}.fastq"
-params.index = "$baseDir/data/index/*.index.*"
-
-log.info "fastq files : ${params.fastq}"
-log.info "index files : ${params.index}"
-
-Channel
-  .fromFilePairs( params.fastq )
-  .ifEmpty { error "Cannot find any fastq files matching: ${params.fastq}" }
-  .set { fastq_files }
-Channel
-  .fromPath( params.index )
-  .ifEmpty { error "Cannot find any index files matching: ${params.index}" }
-  .map { it -> [(it.baseName =~ /([^\.]*)/)[0][1], it]}
-  .groupTuple()
-  .set { index_files }
-
-process mapping_fastq {
-  tag "$reads"
-  cpus 4
-  publishDir "results/mapping/sam/", mode: 'copy'
-
-  input:
-  set pair_id, file(reads) from fastq_files
-  set index_id, file(index) from index_files.collect()
-
-  output:
-  file "${pair_id}.sam" into sam_files
-  file "${pair_id}_bwa_report.txt" into mapping_repport_files
-
-  script:
-"""
-bwa mem -t ${task.cpus} \
-${index_id} ${reads[0]} ${reads[1]} \
--o ${pair_id}.sam &> ${pair_id}_bwa_report.txt
-"""
-}
-

src/nf_modules/BWA/tests.sh

-./nextflow src/nf_modules/BWA/indexing.nf \
-  -c src/nf_modules/BWA/indexing.config \
-  -profile docker \
-  --fasta "data/tiny_dataset/fasta/tiny_v2.fasta"
-
-# ./nextflow src/nf_modules/BWA/mapping_single.nf \
-#   -c src/nf_modules/BWA/mapping_single.config \
-#   -profile docker \
-#   --index "results/mapping/index/tiny_v2.index" \
-#   --fastq "data/tiny_dataset/fastq/tiny*_S.fastq"
-
-./nextflow src/nf_modules/BWA/mapping_paired.nf \
-  -c src/nf_modules/BWA/mapping_paired.config \
-  -profile docker \
-  --index "results/mapping/index/tiny_v2*" \
-  --fastq "data/tiny_dataset/fastq/tiny*_R{1,2}.fastq"
-

src/nf_modules/Bowtie/indexing.config

-profiles {
-  docker {
-    docker.temp = 'auto'
-    docker.enabled = true
-    process {
-      withName: index_fasta {
-        container = "bowtie:1.2.2"
-      }
-    }
-  }
-  psmn {
-    process{
-      withName: index_fasta {
-        beforeScript = "source /usr/share/lmod/lmod/init/bash; module use ~/privatemodules"
-        module = "Bowtie/1.2.2"
-        executor = "sge"
-        clusterOptions = "-m e -cwd -V"
-        memory = "20GB"
-        time = "12h"
-        queue = 'monointeldeb128'
-      }
-    }
-  }
-}

src/nf_modules/Bowtie/indexing.nf

-/*                      fasta indexing                                     */
-
-params.fasta = "$baseDir/data/bam/*.fasta"
-
-log.info "fasta files : ${params.fasta}"
-
-Channel
-  .fromPath( params.fasta )
-  .ifEmpty { error "Cannot find any bam files matching: ${params.fasta}" }
-  .set { fasta_file }
-
-process index_fasta {
-  tag "$fasta.baseName"
-  cpus 4
-  publishDir "results/mapping/index/", mode: 'copy'
-
-  input:
-    file fasta from fasta_file
-
-  output:
-    file "*.index*" into index_files
-    file "*_report.txt" into indexing_report
-
-  script:
-"""
-bowtie-build --threads ${task.cpus} -f ${fasta} ${fasta.baseName}.index &> ${fasta.baseName}_bowtie_report.txt
-
-if grep -q "Error" ${fasta.baseName}_bowtie_report.txt; then
-  exit 1
-fi
-"""
-}
-

src/nf_modules/Bowtie/mapping_paired.config

-profiles {
-  docker {
-    docker.temp = 'auto'
-    docker.enabled = true
-    process {
-      withName: mapping_fastq {
-        container = "bowtie:1.2.2"
-      }
-    }
-  }
-  psmn {
-    process{
-      withName: mapping_fastq {
-        beforeScript = "source /usr/share/lmod/lmod/init/bash; module use ~/privatemodules"
-        module = "Bowtie/1.2.2:SAMtools/1.7"
-        executor = "sge"
-        clusterOptions = "-m e -cwd -V"
-        memory = "30GB"
-        time = "24h"
-        queue = 'E5-2670deb128A,E5-2670deb128B,E5-2670deb128C,E5-2670deb128D,E5-2670deb128E,E5-2670deb128F'
-        penv = 'openmp16'
-      }
-    }
-  }
-}

src/nf_modules/Bowtie/mapping_paired.nf

-/*
-* mapping paired fastq
-*/
-
-params.fastq = "$baseDir/data/fastq/*_{1,2}.fastq"
-params.index = "$baseDir/data/index/*.index.*"
-
-log.info "fastq files : ${params.fastq}"
-log.info "index files : ${params.index}"
-
-Channel
-  .fromFilePairs( params.fastq )
-  .ifEmpty { error "Cannot find any fastq files matching: ${params.fastq}" }
-  .set { fastq_files }
-Channel
-  .fromPath( params.index )
-  .ifEmpty { error "Cannot find any index files matching: ${params.index}" }
-  .set { index_files }
-
-process mapping_fastq {
-  tag "$pair_id"
-  cpus 4
-  publishDir "results/mapping/bams/", mode: 'copy'
-
-  input:
-  set pair_id, file(reads) from fastq_files
-  file index from index_files.collect()
-
-  output:
-  file "*.bam" into bam_files
-  file "*_report.txt" into mapping_report
-
-  script:
-  index_id = index[0]
-  for (index_file in index) {
-  if (index_file =~ /.*\.1\.ebwt/ && !(index_file =~ /.*\.rev\.1\.ebwt/)) {
-        index_id = ( index_file =~ /(.*)\.1\.ebwt/)[0][1]
-    }
-  }
-"""
-# -v specify the max number of missmatch, -k the number of match reported per
-# reads
-bowtie --best -v 3 -k 1 --sam -p ${task.cpus} ${index_id} \
--1 ${reads[0]} -2 ${reads[1]} 2> \
-${pair_id}_bowtie_report.txt | \
-samtools view -Sb - > ${pair_id}.bam
-
-if grep -q "Error" ${pair_id}_bowtie_report.txt; then
-  exit 1
-fi
-"""
-}
-
-

src/nf_modules/Bowtie/mapping_single.config

-profiles {
-  docker {
-    docker.temp = 'auto'
-    docker.enabled = true
-    process {
-      withName: mapping_fastq {
-        container = "bowtie:1.2.2"
-      }
-    }
-  }
-  psmn {
-    process{
-      withName: mapping_fastq {
-        beforeScript = "source /usr/share/lmod/lmod/init/bash; module use ~/privatemodules"
-        module = "Bowtie/1.2.2:SAMtools/1.7"
-        executor = "sge"
-        clusterOptions = "-m e -cwd -V"
-        memory = "30GB"
-        time = "24h"
-        queue = 'E5-2670deb128A,E5-2670deb128B,E5-2670deb128C,E5-2670deb128D,E5-2670deb128E,E5-2670deb128F'
-        penv = 'openmp16'
-      }
-    }
-  }
-}

src/nf_modules/Bowtie/mapping_single.nf

-/*
-* mapping single end fastq
-*/
-
-params.fastq = "$baseDir/data/fastq/*.fastq"
-
-log.info "fastq files : ${params.fastq}"
-log.info "index files : ${params.index}"
-
-Channel
-  .fromPath( params.fastq )
-  .ifEmpty { error "Cannot find any fastq files matching: ${params.fastq}" }
-  .map { it -> [(it.baseName =~ /([^\.]*)/)[0][1], it]}
-  .set { fastq_files }
-Channel
-  .fromPath( params.index )
-  .ifEmpty { error "Cannot find any index files matching: ${params.index}" }
-  .set { index_files }
-
-process mapping_fastq {
-  tag "$file_id"
-  cpus 4
-  publishDir "results/mapping/bams/", mode: 'copy'
-
-  input:
-  set file_id, file(reads) from fastq_files
-  file index from index_files.collect()
-
-  output:
-  set file_id, "*.bam" into bam_files
-  file "*_report.txt" into mapping_report
-
-  script:
-index_id = index[0]
-for (index_file in index) {
-  if (index_file =~ /.*\.1\.ebwt/ && !(index_file =~ /.*\.rev\.1\.ebwt/)) {
-      index_id = ( index_file =~ /(.*)\.1\.ebwt/)[0][1]
-  }
-}
-"""
-bowtie --best -v 3 -k 1 --sam -p ${task.cpus} ${index_id} \
--q ${reads} 2> \
-${file_id}_bowtie_report.txt | \
-samtools view -Sb - > ${file_id}.bam
-
-if grep -q "Error" ${file_id}_bowtie_report.txt; then
-  exit 1
-fi
-"""
-}

src/nf_modules/Bowtie/tests.sh

-./nextflow src/nf_modules/Bowtie/indexing.nf \
-  -c src/nf_modules/Bowtie/indexing.config \
-  -profile docker \
-  --fasta "data/tiny_dataset/fasta/tiny_v2.fasta"
-
-./nextflow src/nf_modules/Bowtie/mapping_single.nf \
-  -c src/nf_modules/Bowtie/mapping_single.config \
-  -profile docker \
-  --index "results/mapping/index/*.ebwt" \
-  --fastq "data/tiny_dataset/fastq/tiny*_S.fastq"
-
-./nextflow src/nf_modules/Bowtie/mapping_paired.nf \
-  -c src/nf_modules/Bowtie/mapping_paired.config \
-  -profile docker \
-  --index "results/mapping/index/*.ebwt" \
-  --fastq "data/tiny_dataset/fastq/tiny*_R{1,2}.fastq"
-