Merge branch 'master' of gitlab_lbmc:pipelines/nextflow

README.md

@@ -76,6 +76,35 @@ for example to run the tests on `Bowtie2` run:
 src/nf_modules/Bowtie2/tests/tests.sh
 ```
 
+## Available tools
+
+| tool | nf module | docker module | sge module |
+|------|:---------:|:-------------:|:----------:|
+BEDtools | ok | ok | ok
+Bowtie | ok | ok | **no**
+Bowtie2 | ok | ok | ok
+canu | ok | ok | ok
+cutadapt | ok | ok | ok
+deepTools | **no** | ok | ok
+FastQC | ok | ok | ok
+file_handle | **no** | ok | ok
+HISAT2 | **no** | ok | **no**
+HTSeq | ok | ok | ok
+Kallisto | ok | ok | ok
+MACS2 | **no** | ok | ok
+MultiQC | ok | ok | ok
+MUSIC | **no** | ok | ok
+picard | **no** | ok | ok
+pigz | **no** | ok | ok
+RSEM | ok | ok | ok
+SAMtools | ok | ok | ok
+SRAtoolkit | ok | ok | ok
+Salmon | **no** | ok | ok
+TopHat | **no** | ok | ok
+Trimmomatic | **no** | ok | ok
+UrQt | ok | ok | ok
+
+
 ## Contributing
 
 Please read [CONTRIBUTING.md](CONTRIBUTING.md) for details on our code of conduct, and the process for submitting pull requests to us.

src/docker_modules/Bowtie/1.2.2/Dockerfile

+FROM ubuntu:18.04
+MAINTAINER Laurent Modolo
+
+ENV BOWTIE_VERSION=1.2.2
+ENV SAMTOOLS_VERSION=1.7
+ENV PACKAGES bowtie=${BOWTIE_VERSION}* \
+    samtools=${SAMTOOLS_VERSION}*
+
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends ${PACKAGES} && \
+    apt-get clean

src/docker_modules/Bowtie/1.2.2/docker_init.sh

+#!/bin/sh
+docker build src/docker_modules/Bowtie/1.2.2 -t 'bowtie:1.2.2'

src/docker_modules/FastQC/0.11.5/Dockerfile

@@ -2,7 +2,8 @@ FROM ubuntu:18.04
 MAINTAINER Laurent Modolo
 
 ENV FASTQC_VERSION=0.11.5
-ENV PACKAGES fastqc=${FASTQC_VERSION}*
+ENV PACKAGES fastqc=${FASTQC_VERSION}* \
+		perl=5.26*
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends ${PACKAGES} && \

src/docker_modules/MACS2/2.1.0/Dockerfile

+FROM ubuntu:18.04
+MAINTAINER Laurent Modolo
+
+ENV MACS_VERSION=2.1.1.20160309
+ENV PACKAGES git=1:2.17* \
+   build-essential=12.4* \
+   python-pip=9.0.1* \
+   ca-certificates=20180409 \
+   python-setuptools=39.0.1* \
+   python-dev=2.7* \
+   python-numpy=1:1.13* \
+   python-wheel=0.30.0* \
+   zlib1g-dev=1:1.2.11*
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends ${PACKAGES} && \
+    apt-get clean
+
+RUN pip install MACS2==${MACS_VERSION}
+

src/docker_modules/MACS2/2.1.0/docker_init.sh

+#!/bin/sh
+docker build src/docker_modules/MACS2/2.1.0 -t 'macs2:2.1.0'

src/docker_modules/MUSIC/6613c53/Dockerfile

+FROM ubuntu:18.04
+MAINTAINER Laurent Modolo
+
+ENV PACKAGES git=1:2.17* \
+   build-essential=12.4* \
+   ca-certificates=20180409 \
+   zlib1g-dev=1:1.2.11*
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends ${PACKAGES} && \
+    apt-get clean
+
+RUN git clone https://github.com/gersteinlab/MUSIC.git && \
+  cd MUSIC && \
+  git checkout ${MUSIC_VERSION} && \
+  make clean && \
+  make  && \
+  cd .. && \
+  mv MUSIC/bin/MUSIC /usr/bin/ && \
+  mv MUSIC/bin/generate_multimappability_signal.csh /usr/bin/ && \
+  mv MUSIC/bin/run_MUSIC.csh /usr/bin/ && \
+  rm -Rf MUSIC
+

src/docker_modules/MUSIC/6613c53/docker_init.sh

+#!/bin/sh
+docker build src/docker_modules/MUSIC/6613c53 -t 'music:6613c53'

src/docker_modules/MultiQC/1.0/Dockerfile

@@ -6,10 +6,16 @@ ENV PACKAGES build-essential=12.4* \
              python3-pip=9.0.1* \
              python3-setuptools=39.0.1* \
              python3-dev=3.6.5* \
-             python3-wheel=0.30.0*
+             python3-wheel=0.30.0* \
+             locales
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends ${PACKAGES} && \
     apt-get clean
 
+RUN locale-gen en_US.UTF-8
+ENV LC_ALL=en_US.utf-8
+ENV LANG=en_US.utf-8
+
 RUN pip3 install multiqc==${MULTIQC_VERSION}
+

src/docker_modules/picard/2.18.11/Dockerfile

+FROM ubuntu:18.04
+MAINTAINER Laurent Modolo
+
+ENV PICARD_VERSION=2.18.11
+ENV PACKAGES default-jre=2:1.1* \
+    curl=7.58.0* \
+    ca-certificates=20180409
+
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends ${PACKAGES} && \
+    apt-get clean
+
+RUN curl -k -L https://github.com/broadinstitute/picard/releases/download/${PICARD_VERSION}/picard.jar -o picard.jar && \
+  mkdir -p /usr/share/java/  && \
+  mv picard.jar /usr/share/java/
+
+COPY PicardCommandLine /usr/bin/
+RUN chmod +x /usr/bin/PicardCommandLine

src/docker_modules/picard/2.18.11/PicardCommandLine

+#!/bin/sh
+set -eu
+PRG="$(basename -- "$0")"
+case "$PRG" in
+picard-tools)
+        echo 1>&2 'Warning: picard-tools is deprecated and should be replaced by PicardCommandLine'
+        ;;
+PicardCommandLine)
+        ;;
+*)
+        set -- "$PRG" "$@"
+        ;;
+esac
+
+exec java ${JAVA_OPTIONS-} -jar /usr/share/java/picard.jar "$@"

src/docker_modules/picard/2.18.11/docker_init.sh

+#!/bin/sh
+docker build src/docker_modules/picard/2.18.11 -t 'picard:2.18.11'

src/nf_modules/Bowtie/bowtie.config

+profiles {
+  docker {
+    docker.temp = 'auto'
+    docker.enabled = true
+    process {
+      $index_fasta {
+        container = "bowtie:1.2.2"
+      }
+      $mapping_fastq {
+        container = "bowtie:1.2.2"
+      }
+    }
+  }
+  sge {
+    process{
+      $index_fasta {
+        beforeScript = "module purge; module load Bowtie/1.2.2"
+      }
+      $mapping_fastq {
+        beforeScript = "module purge; module load SAMtools/1.7; module load Bowtie/1.2.2"
+      }
+    }
+  }
+}

src/nf_modules/Bowtie/bowtie.nf

+/*
+* Bowtie :
+* Imputs : fastq files
+* Imputs : fasta files
+* Output : bam files
+*/
+
+/*                      fasta indexing                                     */
+params.fasta = "$baseDir/data/bam/*.fasta"
+
+log.info "fasta files : ${params.fasta}"
+
+Channel
+  .fromPath( params.fasta )
+  .ifEmpty { error "Cannot find any bam files matching: ${params.fasta}" }
+  .set { fasta_file }
+
+process index_fasta {
+  tag "$fasta.baseName"
+  cpus 4
+  publishDir "results/mapping/index/", mode: 'copy'
+
+  input:
+    file fasta from fasta_file
+
+  output:
+    file "*.index*" into index_files
+
+  script:
+"""
+bowtie-build --threads ${task.cpus} -f ${fasta} ${fasta.baseName}.index &> ${fasta.baseName}_bowtie_report.txt
+
+if grep -q "Error" ${fasta.baseName}_bowtie_report.txt; then
+  exit 1
+fi
+"""
+}
+
+
+
+/*
+* for paired-end data
+*/
+
+params.fastq = "$baseDir/data/fastq/*_{1,2}.fastq"
+params.index = "$baseDir/data/index/*.index.*"
+
+log.info "fastq files : ${params.fastq}"
+log.info "index files : ${params.index}"
+
+Channel
+  .fromFilePairs( params.fastq )
+  .ifEmpty { error "Cannot find any fastq files matching: ${params.fastq}" }
+  .set { fastq_files }
+Channel
+  .fromPath( params.index )
+  .ifEmpty { error "Cannot find any index files matching: ${params.index}" }
+  .set { index_files }
+
+process mapping_fastq {
+  tag "$pair_id"
+  cpus 4
+  publishDir "results/mapping/bams/", mode: 'copy'
+
+  input:
+  set pair_id, file(reads) from fastq_files
+  file index from index_files.toList()
+
+  output:
+  file "*.bam" into bam_files
+
+  script:
+  index_id = index[0]
+  for (index_file in index) {
+    if (index_file =~ /.*\.1\.ebwt/) {
+        index_id = ( index_file =~ /(.*)\.1\.ebwt/)[0][1]
+    }
+  }
+"""
+# -v specify the max number of missmatch, -k the number of match reported per
+# reads
+bowtie --best -v 3 -k 1 --sam -p ${task.cpus} ${index_id} \
+-1 ${reads[0]} -2 ${reads[1]} 2> \
+${pair_id}_bowtie_report.txt | \
+samtools view -Sb - > ${pair_id}.bam
+
+if grep -q "Error" ${pair_id}_bowtie_report.txt; then
+  exit 1
+fi
+"""
+}
+
+
+/*
+* for single-end data
+*/
+params.fastq = "$baseDir/data/fastq/*.fastq"
+
+log.info "fastq files : ${params.fastq}"
+log.info "index files : ${params.index}"
+
+Channel
+  .fromPath( params.fastq )
+  .ifEmpty { error "Cannot find any fastq files matching: ${params.fastq}" }
+  .set { fastq_files }
+Channel
+  .fromPath( params.index )
+  .ifEmpty { error "Cannot find any index files matching: ${params.index}" }
+  .set { index_files }
+
+process mapping_fastq {
+  tag "$reads.baseName"
+  cpus 4
+  publishDir "results/mapping/bams/", mode: 'copy'
+
+  input:
+  file reads from fastq_files
+  file index from index_files.toList()
+
+  output:
+  file "*.bam" into bam_files
+
+  script:
+index_id = index[0]
+for (index_file in index) {
+  if (index_file =~ /.*\.1\.ebwt/) {
+      index_id = ( index_file =~ /(.*)\.1\.ebwt/)[0][1]
+  }
+}
+"""
+bowtie --best -v 3 -k 1 --sam -p ${task.cpus} ${index_id} \
+-U ${reads} 2> \
+${reads.baseName}_bowtie_report.txt | \
+samtools view -Sb - > ${reads.baseName}.bam
+
+if grep -q "Error" ${reads.baseName}_bowtie_report.txt; then
+  exit 1
+fi
+"""
+}

src/nf_modules/Bowtie/tests/index.nf

+params.fasta = "$baseDir/data/bam/*.fasta"
+
+log.info "fasta files : ${params.fasta}"
+
+Channel
+  .fromPath( params.fasta )
+  .ifEmpty { error "Cannot find any bam files matching: ${params.fasta}" }
+  .set { fasta_file }
+
+process index_fasta {
+  tag "$fasta.baseName"
+  cpus 4
+  publishDir "results/mapping/index/", mode: 'copy'
+
+  input:
+    file fasta from fasta_file
+
+  output:
+    file "*.index*" into index_files
+
+  script:
+"""
+bowtie-build --threads ${task.cpus} -f ${fasta} ${fasta.baseName}.index &> ${fasta.baseName}_bowtie_report.txt
+
+if grep -q "Error" ${fasta.baseName}_bowtie_report.txt; then
+  exit 1
+fi
+"""
+}
+

src/nf_modules/Bowtie/tests/mapping_paired.nf

+params.fastq = "$baseDir/data/fastq/*_{1,2}.fastq"
+params.index = "$baseDir/data/index/*.index.*"
+
+log.info "fastq files : ${params.fastq}"
+log.info "index files : ${params.index}"
+
+Channel
+  .fromFilePairs( params.fastq )
+  .ifEmpty { error "Cannot find any fastq files matching: ${params.fastq}" }
+  .set { fastq_files }
+Channel
+  .fromPath( params.index )
+  .ifEmpty { error "Cannot find any index files matching: ${params.index}" }
+  .set { index_files }
+
+process mapping_fastq {
+  tag "$pair_id"
+  cpus 4
+  publishDir "results/mapping/bams/", mode: 'copy'
+
+  input:
+  set pair_id, file(reads) from fastq_files
+  file index from index_files.toList()
+
+  output:
+  file "*.bam" into bam_files
+
+  script:
+  index_id = index[0]
+  for (index_file in index) {
+    if (index_file =~ /.*\.1\.ebwt/) {
+        index_id = ( index_file =~ /(.*)\.1\.ebwt/)[0][1]
+    }
+  }
+"""
+# -v specify the max number of missmatch, -k the number of match reported per
+# reads
+bowtie --best -v 3 -k 1 --sam -p ${task.cpus} ${index_id} \
+-1 ${reads[0]} -2 ${reads[1]} 2> \
+${pair_id}_bowtie_report.txt | \
+samtools view -Sb - > ${pair_id}.bam
+
+if grep -q "Error" ${pair_id}_bowtie_report.txt; then
+  exit 1
+fi
+"""
+}
+
+

src/nf_modules/Bowtie/tests/mapping_single.nf

+params.fastq = "$baseDir/data/fastq/*.fastq"
+
+log.info "fastq files : ${params.fastq}"
+log.info "index files : ${params.index}"
+
+Channel
+  .fromPath( params.fastq )
+  .ifEmpty { error "Cannot find any fastq files matching: ${params.fastq}" }
+  .set { fastq_files }
+Channel
+  .fromPath( params.index )
+  .ifEmpty { error "Cannot find any index files matching: ${params.index}" }
+  .set { index_files }
+
+process mapping_fastq {
+  tag "$reads.baseName"
+  cpus 4
+  publishDir "results/mapping/bams/", mode: 'copy'
+
+  input:
+  file reads from fastq_files
+  file index from index_files.toList()
+
+  output:
+  file "*.bam" into bam_files
+
+  script:
+index_id = index[0]
+for (index_file in index) {
+  if (index_file =~ /.*\.1\.ebwt/) {
+      index_id = ( index_file =~ /(.*)\.1\.ebwt/)[0][1]
+  }
+}
+"""
+bowtie --best -v 3 -k 1 --sam -p ${task.cpus} ${index_id} \
+-U ${reads} 2> \
+${reads.baseName}_bowtie_report.txt | \
+samtools view -Sb - > ${reads.baseName}.bam
+
+if grep -q "Error" ${reads.baseName}_bowtie_report.txt; then
+  exit 1
+fi
+"""
+}

src/nf_modules/Bowtie/tests/tests.sh

+./nextflow src/nf_modules/Bowtie/tests/index.nf \
+  -c src/nf_modules/Bowtie/bowtie.config \
+  -profile docker \
+  --fasta "data/tiny_dataset/fasta/tiny_v2.fasta"
+
+./nextflow src/nf_modules/Bowtie/tests/mapping_single.nf \
+  -c src/nf_modules/Bowtie/bowtie.config \
+  -profile docker \
+  --index "results/mapping/index/*.ebwt" \
+  --fastq "data/tiny_dataset/fastq/tiny*_S.fastq"
+
+./nextflow src/nf_modules/Bowtie/tests/mapping_paired.nf \
+  -c src/nf_modules/Bowtie/bowtie.config \
+  -profile docker \
+  --index "results/mapping/index/*.ebwt" \
+  --fastq "data/tiny_dataset/fastq/tiny*_R{1,2}.fastq"
+

src/nf_modules/Bowtie2/bowtie2.nf

@@ -67,13 +67,20 @@ process mapping_fastq {
 
   output:
   set pair_id, "*.bam" into bam_files
+  file "*_report.txt" into mapping_report
 
   script:
+  index_id = index[0]
+  for (index_file in index) {
+    if (index_file =~ /.*\.1\.bt2/) {
+        index_id = ( index_file =~ /(.*)\.1\.bt2/)[0][1]
+    }
+  }
 """
- bowtie2 --very-sensitive -p ${task.cpus} -x ${index[0].baseName} \
- -1 ${reads[0]} -2 ${reads[1]} 2> \
- ${pair_id}_bowtie2_report.txt | \
- samtools view -Sb - > ${pair_id}.bam
+bowtie2 --very-sensitive -p ${task.cpus} -x ${index_id} \
+-1 ${reads[0]} -2 ${reads[1]} 2> \
+${pair_id}_bowtie2_report.txt | \
+samtools view -Sb - > ${pair_id}.bam
 
 if grep -q "Error" ${pair_id}_bowtie2_report.txt; then
   exit 1
@@ -110,15 +117,22 @@ process mapping_fastq {
 
   output:
   file "*.bam" into bam_files
+  file "*_report.txt" into mapping_report
 
   script:
+  index_id = index[0]
+  for (index_file in index) {
+    if (index_file =~ /.*\.1\.bt2/) {
+        index_id = ( index_file =~ /(.*)\.1\.bt2/)[0][1]
+    }
+  }
 """
-bowtie2 --very_sensitive -p ${task.cpus} -x ${index[0].baseName} \
+bowtie2 --very_sensitive -p ${task.cpus} -x ${index_id} \
 -U ${reads} 2> \
 ${reads.baseName}_bowtie2_report.txt | \
 samtools view -Sb - > ${reads.baseName}.bam
 
-if grep -q "Error" ${fasta.baseName}_bowtie2_report.txt; then
+if grep -q "Error" ${reads.baseName}_bowtie2_report.txt; then
   exit 1
 fi
 """

src/nf_modules/Bowtie2/tests/mapping_paired.nf

@@ -24,13 +24,20 @@ process mapping_fastq {
 
   output:
   set pair_id, "*.bam" into bam_files
+  file "*_report.txt" into mapping_report
 
   script:
+  index_id = index[0]
+  for (index_file in index) {
+    if (index_file =~ /.*\.1\.bt2/) {
+        index_id = ( index_file =~ /(.*)\.1\.bt2/)[0][1]
+    }
+  }
 """
- bowtie2 --very-sensitive -p ${task.cpus} -x ${index[0].baseName} \
- -1 ${reads[0]} -2 ${reads[1]} 2> \
- ${pair_id}_bowtie2_report.txt | \
- samtools view -Sb - > ${pair_id}.bam
+bowtie2 --very-sensitive -p ${task.cpus} -x ${index_id} \
+-1 ${reads[0]} -2 ${reads[1]} 2> \
+${pair_id}_bowtie2_report.txt | \
+samtools view -Sb - > ${pair_id}.bam
 
 if grep -q "Error" ${pair_id}_bowtie2_report.txt; then
   exit 1