Updating forked nextflow

Merge branch 'master' of gitlab_lbmc:pipelines/nextflow

CHANGELOG.md

+# Changelog
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [0.2.6] - 2018-08-23
+### Added
+- Added `src/training_dataset.nf` to build a small training dataset from NGS data
+
+### Changed
+- the structure of `src/nf_modules`: the `tests` folder was removed
+
+## [0.2.5] - 2018-08-22
+### Added
+- This fine changelog
+
+### Changed
+- the structure of `src/nf_modules`: the `tests` folder was removed
+
+
+## [0.2.4] - 2018-08-02
+### Changed
+- add `paired_id` variable in the output of every single-end data processes to match the paired output
+
+
+## [0.2.3] - 2018-07-25
+### Added
+- List of tools available as nextflow, docker or sge module to the `README.md`
+
+
+## [0.2.2] - 2018-07-23
+### Added
+- SRA module from cigogne/nextflow-master 52b510e48daa1fb7
+
+
+## [0.2.1] - 2018-07-23
+### Added
+- List of tools available as nextflow, docker or sge module
+
+
+## [0.2.0] - 2018-06-18
+### Added
+- `doc/TP_computational_biologists.md`
+- Kallisto/0.44.0
+
+### Changed
+- add `paired_id` variable in the output of every paired data processes
+- BEDtools: fixes for fasta handling
+- UrQt: fix git version in Docker
+
+
+## [0.1.2] - 2018-06-18
+### Added
+- `doc/tp_experimental_biologist.md` and Makefile to build the pdf
+- tests files for BEDtools
+
+### Changed
+- Kallisto: various fixes
+- UrQt: improve output and various fixes
+
+### Removed
+- `src/nf_test.config` modules have their own `.config`
+
+
+## [0.1.2] - 2018-06-18
+### Added
+- `doc/tp_experimental_biologist.md` and Makefile to build the pdf
+- tests files for BEDtools
+
+### Changed
+- Kallisto: various fixes
+- UrQt: improve output and various fixes
+
+### Removed
+- `src/nf_test.config` modules have their own `.config`
+
+
+## [0.1.0] - 2018-05-06
+This is the first working version of the repository as a nextflow module repository
+

README.md

@@ -5,7 +5,7 @@ You can fork this repository to build your own pipeline.
 To get the last commits from this repository into your fork use the following commands:
 
 ```sh
-git remote add upstream https://gitlab.biologie.ens-lyon.fr/pipelines/nextflow.git
+git remote add upstream gitlab_lbmc:pipelines/nextflow.git
 git pull upstream master
 ```
 
@@ -35,19 +35,19 @@ To install nextflow on you computer simply run the following command:
 src/install_nextflow.sh
 ```
 
-Then to initialise a given tools run the following command:
+Then to initialize a given tools run the following command:
 
 ```sh
 src/docker_modules/<tool_name>/<tool_version>/docker_init.sh
 ```
 
-for example to initialise `file_handle` version `0.1.1`, run:
+For example to initialize `file_handle` version `0.1.1`, run:
 
 ```sh
 src/docker_modules/file_handle/0.1.1/docker_init.sh
 ```
 
-To initialise all the tools:
+To initialize all the tools:
 ```sh
 find src/docker_modules/ -name "docker_init.sh" | awk '{system($0)}'
 ```
@@ -67,15 +67,44 @@ cd ..
 Then to run the tests for a given tools run the following command:
 
 ```sh
-src/nf_modules/<tool_name>/<tool_version>/tests/tests.sh
+src/nf_modules/<tool_name>/<tool_version>/tests.sh
 ```
 
-for example to run the tests on `Bowtie2` run:
+For example to run the tests on `Bowtie2` run:
 
 ```sh
-src/nf_modules/Bowtie2/tests/tests.sh
+src/nf_modules/Bowtie2/tests.sh
 ```
 
+## Available tools
+
+| tool | nf module | docker module | sge module |
+|------|:---------:|:-------------:|:----------:|
+BEDtools | ok | ok | ok
+Bowtie | ok | ok | **no**
+Bowtie2 | ok | ok | ok
+canu | ok | ok | ok
+cutadapt | ok | ok | ok
+deepTools | **no** | ok | ok
+FastQC | ok | ok | ok
+file_handle | **no** | ok | ok
+HISAT2 | **no** | ok | **no**
+HTSeq | ok | ok | ok
+Kallisto | ok | ok | ok
+MACS2 | **no** | ok | ok
+MultiQC | ok | ok | ok
+MUSIC | ok | ok | ok
+picard | **no** | ok | ok
+pigz | **no** | ok | ok
+RSEM | ok | ok | ok
+SAMtools | ok | ok | ok
+SRAtoolkit | ok | ok | ok
+Salmon | **no** | ok | ok
+TopHat | **no** | ok | ok
+Trimmomatic | **no** | ok | ok
+UrQt | ok | ok | ok
+
+
 ## Contributing
 
 Please read [CONTRIBUTING.md](CONTRIBUTING.md) for details on our code of conduct, and the process for submitting pull requests to us.

src/Rnaseq.nf

-params.fastq = "$baseDir/data/fastq/*_{1,2}.fastq"
-params.fasta = "$baseDir/data/fasta/*.fasta"
-params.bed = "$baseDir/data/annot/*.bed"
-
-log.info "fasta file : ${params.fasta}"
-log.info "bed file : ${params.bed}"
-log.info "fastq files : ${params.fastq}"
-
-Channel
-  .fromPath( params.fasta )
-  .ifEmpty { error "Cannot find any fasta files matching: ${params.fasta}" }
-  .set { fasta_files }
-
-Channel
-  .fromPath( params.bed )
-  .ifEmpty { error "Cannot find any bed files matching: ${params.bed}" }
-  .set { bed_files }
-
-Channel
-  .fromFilePairs( params.fastq )
-  .ifEmpty { error "Cannot find any fastq files matching: ${params.fastq}" }
-  .set { fastq_files }
-
-process adaptor_removal {
-  tag "$pair_id"
-  publishDir "results/fastq/adaptor_removal/", mode: 'copy'
-
-  input:
-  set pair_id, file(reads) from fastq_files
-
-  output:
-  file "*_cut_R{1,2}.fastq.gz" into fastq_files_cut
-
-  script:
-  """
-  cutadapt -a AGATCGGAAGAG -g CTCTTCCGATCT -A AGATCGGAAGAG -G CTCTTCCGATCT \
-  -o ${pair_id}_cut_R1.fastq.gz -p ${pair_id}_cut_R2.fastq.gz \
-  ${reads[0]} ${reads[1]} > ${pair_id}_report.txt
-  """
-}
-
-process trimming {
-  tag "${reads}"
-  cpus 4
-  publishDir "results/fastq/trimming/", mode: 'copy'
-
-  input:
-  file reads from fastq_files_cut
-
-  output:
-  file "*_trim_R{1,2}.fastq.gz" into fastq_files_trim
-
-  script:
-"""
-UrQt --t 20 --m ${task.cpus} --gz \
---in ${reads[0]} --inpair ${reads[1]} \
---out ${reads[0].baseName}_trim_R1.fastq.gz --outpair ${reads[1].baseName}_trim_R2.fastq.gz \
-> ${reads[0].baseName}_trimming_report.txt
-"""
-}
-
-process fasta_from_bed {
-  tag "${bed.baseName}"
-  cpus 4
-  publishDir "results/fasta/", mode: 'copy'
-
-  input:
-  file fasta from fasta_files
-  file bed from bed_files
-
-  output:
-  file "*_extracted.fasta" into fasta_files_extracted
-
-  script:
-"""
-bedtools getfasta -name \
--fi ${fasta} -bed ${bed} -fo ${bed.baseName}_extracted.fasta
-"""
-}
-
-process index_fasta {
-  tag "$fasta.baseName"
-  publishDir "results/mapping/index/", mode: 'copy'
-
-  input:
-    file fasta from fasta_files_extracted
-
-  output:
-    file "*.index*" into index_files
-
-  script:
-"""
-kallisto index -k 31 --make-unique -i ${fasta.baseName}.index ${fasta} \
-> ${fasta.baseName}_kallisto_report.txt
-"""
-}
-
-
-process mapping_fastq {
-  tag "$reads"
-  cpus 4
-  publishDir "results/mapping/quantification/", mode: 'copy'
-
-  input:
-  file reads from fastq_files_trim
-  file index from index_files
-
-  output:
-  file "*" into counts_files
-
-  script:
-"""
-mkdir ${reads[0].baseName}
-kallisto quant -i ${index} -t ${task.cpus} \
---bias --bootstrap-samples 100 -o ${reads[0].baseName} \
-${reads[0]} ${reads[1]} &> ${reads[0].baseName}_kallisto_report.txt
-"""
-}
-
-
-
-

src/docker_modules/Bowtie/1.2.2/Dockerfile

+FROM ubuntu:18.04
+MAINTAINER Laurent Modolo
+
+ENV BOWTIE_VERSION=1.2.2
+ENV SAMTOOLS_VERSION=1.7
+ENV PACKAGES bowtie=${BOWTIE_VERSION}* \
+    samtools=${SAMTOOLS_VERSION}*
+
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends ${PACKAGES} && \
+    apt-get clean

src/docker_modules/Bowtie/1.2.2/docker_init.sh

+#!/bin/sh
+docker build src/docker_modules/Bowtie/1.2.2 -t 'bowtie:1.2.2'

src/docker_modules/Bowtie2/2.3.4.1/Dockerfile

@@ -4,8 +4,8 @@ MAINTAINER Laurent Modolo
 ENV BOWTIE2_VERSION=2.3.4.1
 ENV SAMTOOLS_VERSION=1.7
 ENV PACKAGES bowtie2=${BOWTIE2_VERSION}* \
-    samtools=${SAMTOOLS_VERSION}*
-
+    samtools=${SAMTOOLS_VERSION}* \
+    perl=5.26.1*
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends ${PACKAGES} && \

src/docker_modules/FastQC/0.11.5/Dockerfile

@@ -2,7 +2,8 @@ FROM ubuntu:18.04
 MAINTAINER Laurent Modolo
 
 ENV FASTQC_VERSION=0.11.5
-ENV PACKAGES fastqc=${FASTQC_VERSION}*
+ENV PACKAGES fastqc=${FASTQC_VERSION}* \
+		perl=5.26*
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends ${PACKAGES} && \

src/docker_modules/HTSeq/0.8.0/Dockerfile

@@ -14,4 +14,5 @@ RUN apt-get update && \
     apt-get clean
 
 RUN pip3 install numpy==1.14.3
+RUN pip3 install pysam==0.15.0
 RUN pip3 install HTSeq==${HTSEQ_VERSION}

src/docker_modules/MACS2/2.1.0/Dockerfile

+FROM ubuntu:18.04
+MAINTAINER Laurent Modolo
+
+ENV MACS_VERSION=2.1.1.20160309
+ENV PACKAGES git=1:2.17* \
+   build-essential=12.4* \
+   python-pip=9.0.1* \
+   ca-certificates=20180409 \
+   python-setuptools=39.0.1* \
+   python-dev=2.7* \
+   python-numpy=1:1.13* \
+   python-wheel=0.30.0* \
+   zlib1g-dev=1:1.2.11*
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends ${PACKAGES} && \
+    apt-get clean
+
+RUN pip install MACS2==${MACS_VERSION}
+

src/docker_modules/MACS2/2.1.0/docker_init.sh

+#!/bin/sh
+docker build src/docker_modules/MACS2/2.1.0 -t 'macs2:2.1.0'

src/docker_modules/MUSIC/6613c53/Dockerfile

+FROM samtools:1.7
+MAINTAINER Laurent Modolo
+
+ENV PACKAGES git=1:2.17* \
+   build-essential=12.4* \
+   ca-certificates=20180409 \
+   zlib1g-dev=1:1.2.11*
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends ${PACKAGES} && \
+    apt-get clean
+
+RUN git clone https://github.com/gersteinlab/MUSIC.git && \
+  cd MUSIC && \
+  git checkout ${MUSIC_VERSION} && \
+  make clean && \
+  make  && \
+  cd .. && \
+  mv MUSIC/bin/MUSIC /usr/bin/ && \
+  mv MUSIC/bin/generate_multimappability_signal.csh /usr/bin/ && \
+  mv MUSIC/bin/run_MUSIC.csh /usr/bin/ && \
+  rm -Rf MUSIC
+
+RUN chmod +x /usr/bin/*
+

src/docker_modules/MUSIC/6613c53/docker_init.sh

+#!/bin/sh
+docker build src/docker_modules/MUSIC/6613c53 -t 'music:6613c53'

src/docker_modules/MultiQC/1.0/Dockerfile

@@ -6,10 +6,16 @@ ENV PACKAGES build-essential=12.4* \
              python3-pip=9.0.1* \
              python3-setuptools=39.0.1* \
              python3-dev=3.6.5* \
-             python3-wheel=0.30.0*
+             python3-wheel=0.30.0* \
+             locales
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends ${PACKAGES} && \
     apt-get clean
 
+RUN locale-gen en_US.UTF-8
+ENV LC_ALL=en_US.utf-8
+ENV LANG=en_US.utf-8
+
 RUN pip3 install multiqc==${MULTIQC_VERSION}
+

src/docker_modules/RSEM/1.3.0/Dockerfile

@@ -4,7 +4,7 @@ MAINTAINER Laurent Modolo
 ENV RSEM_VERSION=1.3.0
 ENV BOWTIE2_VERSION=2.3.4.1
 ENV SAMTOOLS_VERSION=1.7
-ENV PACKAGES git=1:2.17.0* \
+ENV PACKAGES git=1:2.17* \
    build-essential=12.4* \
    ca-certificates=20180409 \
    zlib1g-dev=1:1.2.11* \

src/docker_modules/picard/2.18.11/Dockerfile

+FROM ubuntu:18.04
+MAINTAINER Laurent Modolo
+
+ENV PICARD_VERSION=2.18.11
+ENV PACKAGES default-jre=2:1.1* \
+    curl=7.58.0* \
+    ca-certificates=20180409
+
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends ${PACKAGES} && \
+    apt-get clean
+
+RUN curl -k -L https://github.com/broadinstitute/picard/releases/download/${PICARD_VERSION}/picard.jar -o picard.jar && \
+  mkdir -p /usr/share/java/  && \
+  mv picard.jar /usr/share/java/
+
+COPY PicardCommandLine /usr/bin/
+RUN chmod +x /usr/bin/PicardCommandLine

src/docker_modules/picard/2.18.11/PicardCommandLine

+#!/bin/sh
+set -eu
+PRG="$(basename -- "$0")"
+case "$PRG" in
+picard-tools)
+        echo 1>&2 'Warning: picard-tools is deprecated and should be replaced by PicardCommandLine'
+        ;;
+PicardCommandLine)
+        ;;
+*)
+        set -- "$PRG" "$@"
+        ;;
+esac
+
+exec java ${JAVA_OPTIONS-} -jar /usr/share/java/picard.jar "$@"

src/docker_modules/picard/2.18.11/docker_init.sh

+#!/bin/sh
+docker build src/docker_modules/picard/2.18.11 -t 'picard:2.18.11'

src/fasta_sampler.nf

-Channel
-  .fromPath( "data/tiny_dataset/fasta/*.fasta" )
-  .set { fasta_file }
-
-process sample_fasta {
-  publishDir "results/sampling/", mode: 'copy'
-
-  input:
-file fasta from fasta_file
-
-  output:
-file "*_sample.fasta" into fasta_sample
-
-  script:
-"""
-head ${fasta} > ${fasta.baseName}_sample.fasta
-"""
-}