Merge pull request #32 from nf-core/dev

Dev

Author: apeltzer

bin/scrape_software_versions.py

@@ -14,6 +14,8 @@
     'BWA': ['v_bwa.txt', r"Version: (\S+)"],
     'Qualimap': ['v_qualimap.txt', r"QualiMap v.(\S+)"],
     'GATK': ['v_gatk.txt', r"Version:([\d\.]+)"],
+    'bamUtil' : ['v_bamutil.txt', r"Version: ([\d\.]+)"],
+    'fastP': ['v_fastp.txt', r"([\d\.]+)"],
 }
 results = OrderedDict()
 results['nf-core/eager'] = '<span style="color:#999999;\">N/A</span>'
@@ -26,6 +28,8 @@
 results['BWA'] = '<span style="color:#999999;\">N/A</span>'
 results['Qualimap'] = '<span style="color:#999999;\">N/A</span>'
 results['GATK'] = '<span style="color:#999999;\">N/A</span>'
+results['bamUtil'] = '<span style="color:#999999;\">N/A</span>'
+results['fastP'] = '<span style="color:#999999;\">N/A</span>'
 
 # Search each file using its regex
 for k, v in regexes.items():

conf/base.config

@@ -22,6 +22,18 @@ process {
   maxErrors = '-1'
 
   // Process-specific resource requirements (others leave at default, e.g. Fastqc)
+  withName:get_software_versions {
+    memory = { check_max( 2.GB, 'memory' ) }
+    cache = false
+    //errorStrategy = 'ignore'
+  }
+  withName:workflow_summary_mqc {
+    memory = { check_max( 2.GB, 'memory' ) }
+    cache = false
+    executor = 'local'
+    errorStrategy = 'ignore'
+}
+
   withName:bwa {
     memory = { check_max( 16.GB * task.attempt, 'memory' ) }
     cpus = { check_max(8 * task.attempt, 'cpus') }

environment.yml

@@ -9,7 +9,7 @@ dependencies:
   - adapterremoval=2.2.2
   - adapterremovalfixprefix=0.0.4
   - bwa=0.7.17
-  - picard=2.18.11
+  - picard=2.18.14
   - samtools=1.9
   - dedup=0.12.3
   - angsd=0.921
@@ -23,4 +23,6 @@ dependencies:
   - r-markdown=0.8
   - sequencetools=1.2.2
   - preseq=2.0.3
-  #Missing Schmutzi
+  - fastp=0.19.4
+  - bamutil=1.0.14
+  #Missing Schmutzi,snpAD

main.nf

@@ -75,6 +75,10 @@ params.skip_damage_calculation = false
 params.skip_qualimap = false
 params.skip_deduplication = false
 
+//Complexity filtering reads
+params.complexity_filter = false
+params.complexity_filter_poly_g_min = 10
+
 //Read clipping and merging parameters
 params.clip_forward_adaptor = "AGATCGGAAGAGCACACGTCTGAACTCCAGTCAC"
 params.clip_reverse_adaptor = "AGATCGGAAGAGCGTCGTGTAGGGAAAGAGTGTA"
@@ -135,23 +139,23 @@ if(params.readPaths){
             .map { row -> [ row[0], [file(row[1][0])]] }
             .ifEmpty { exit 1, "params.readPaths was empty - no input files supplied" }
             .dump(tag:'input')
-            .into { read_files_fastqc; read_files_trimming }
+            .into { ch_read_files_fastqc; ch_read_files_trimming; ch_read_files_complexity_filtering }
 
     } else {
         Channel
             .from(params.readPaths)
             .map { row -> [ row[0], [file(row[1][0]), file(row[1][1])]] }
             .ifEmpty { exit 1, "params.readPaths was empty - no input files supplied" }
             .dump(tag:'input')
-            .into { ch_read_files_clip; ch_read_files_fastqc }
+            .into { ch_read_files_clip; ch_read_files_fastqc; ch_read_files_complexity_filtering }
 
     }
 } else {
     Channel
         .fromFilePairs( params.reads, size: params.singleEnd ? 1 : 2 )
         .ifEmpty { exit 1, "Cannot find any reads matching: ${params.reads}\nNB: Path needs to be enclosed in quotes!\nNB: Path requires at least one * wildcard!\nIf this is single-end data, please specify --singleEnd on the command line." }
         .dump(tag:'input')
-        .into { ch_read_files_clip; ch_read_files_fastqc }
+        .into { ch_read_files_clip; ch_read_files_fastqc; ch_read_files_complexity_filtering }
 
 }
 
@@ -222,17 +226,20 @@ process get_software_versions {
     """
     echo $workflow.manifest.version &> v_pipeline.txt
     echo $workflow.nextflow.version &> v_nextflow.txt
-    fastqc --version &> v_fastqc.txt
-    multiqc --version &> v_multiqc.txt
-    echo \$(bwa 2>&1) &> v_bwa.txt
-    samtools --version &> v_samtools.txt
-    echo \$(AdapterRemoval -version  2>&1) &> v_adapterremoval.txt
-    picard MarkDuplicates --version &> v_markduplicates.txt  || true
-    echo \$(dedup -v 2>&1) &> v_dedup.txt
-    preseq &> v_preseq.txt
-    gatk BaseRecalibrator --version &> v_gatk.txt
-    qualimap --version &> v_qualimap.txt
-    vcf2genome &> v_vcf2genome.txt
+    fastqc --version &> v_fastqc.txt 2>&1 || true
+    multiqc --version &> v_multiqc.txt 2>&1 || true
+    bwa &> v_bwa.txt 2>&1 || true
+    samtools --version &> v_samtools.txt 2>&1 || true
+    AdapterRemoval -version  &> v_adapterremoval.txt 2>&1 || true
+    picard MarkDuplicates --version &> v_markduplicates.txt  2>&1 || true
+    dedup -v &> v_dedup.txt 2>&1 || true
+    preseq &> v_preseq.txt 2>&1 || true
+    gatk BaseRecalibrator --version 2>&1 | grep Version: > v_gatk.txt 2>&1 || true
+    vcf2genome &> v_vcf2genome.txt 2>&1 || true
+    fastp --version &> v_fastp.txt 2>&1 || true
+    bam --version &> v_bamutil.txt 2>&1 || true
+    qualimap --version &> v_qualimap.txt 2>&1 || true
+    
     scrape_software_versions.py &> software_versions_mqc.yaml
     """
 }
@@ -322,6 +329,38 @@ process fastqc {
     """
 }
 
+
+/* STEP 2.0 - FastP
+* Optional poly-G complexity filtering step before read merging/adapter clipping etc
+* Note: Clipping, Merging, Quality Trimning are turned off here - we leave this to adapter removal itself!
+*/
+
+process fastp {
+    tag "$name"
+    publishDir "${params.outdir}/01-FastP", mode: 'copy'
+
+    when: params.complexity_filter
+
+    input:
+    set val(name), file(reads) from ch_read_files_complexity_filtering
+
+    output:
+    set val(name), file("*pG.fq.gz") into (ch_clipped_reads_complexity_filtered, ch_debug_complexity_filtering)
+    file("*.json") into ch_fastp_for_multiqc
+
+    script:
+    if(${params.singleEnd}){
+    """
+    fastp -in1 ${reads[0]} -out1 "${reads[0].baseName}.pG.fq.gz" -A -g --poly_g_min_lin "${params.complexity_filter_poly_g_min}" -Q -L -w ${task.cpus} -json "${reads[0].baseName}"_fastp.json 
+    """
+    } else {
+    """
+    fastp -in1 ${reads[0]} -in2 ${reads[1]} -out1 "${reads[0].baseName}.pG.fq.gz" -out2 "${reads[1].baseName}.pG.fq.gz" -A -g --poly_g_min_lin "${params.complexity_filter_poly_g_min}" -Q -L -w ${task.cpus} -json "$read.baseName}"_fastp.json
+    """
+    }
+}
+
+
 /*
  * STEP 2 - Adapter Clipping / Read Merging
  */
@@ -332,7 +371,7 @@ process adapter_removal {
     publishDir "${params.outdir}/02-Merging", mode: 'copy'
 
     input:
-    set val(name), file(reads) from ch_read_files_clip
+    set val(name), file(reads) from ( params.complexity_filter ? ch_clipped_reads_complexity_filtered : ch_read_files_clip )
 
     output:
     file "*.combined*.gz" into ch_clipped_reads
@@ -594,6 +633,8 @@ process multiqc {
     file ('qualimap/*') from ch_qualimap_results.collect().ifEmpty([])
     file ('markdup/*') from ch_markdup_results_for_multiqc.collect().ifEmpty([])
     file ('dedup/*') from ch_dedup_results_for_multiqc.collect().ifEmpty([])
+    file ('fastp/*') from ch_fastp_for_multiqc.collect().ifEmpty([])
+
     file workflow_summary from create_workflow_summary(summary)
 
     output: