Merge pull request #321 from jfy133/dev

Resource requirement and abstraction process fixes

Author: apeltzer

CHANGELOG.md

@@ -22,7 +22,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 * [#302](https://github.com/nf-core/eager/issues/302) - Added mitochondrial to nuclear ratio calculation
 * [#302](https://github.com/nf-core/eager/issues/302) - Added VCF2Genome for concensus sequence generation
 * Fancy new logo from [ZandraFagernas](https://github.com/ZandraFagernas)
-* [#286](https://github.com/nf-core/eager/issues/286) Adds pipeline-specific profiles (loaded from nf-core configs)
+* [#286](https://github.com/nf-core/eager/issues/286) - Adds pipeline-specific profiles (loaded from nf-core configs)
+* [#310](https://github.com/nf-core/eager/issues/310) - Generalises base.config
 
 ### `Fixed`
 

conf/base.config

@@ -11,79 +11,85 @@
 
 process {
   cpus = { check_max( 1 * task.attempt, 'cpus' ) }
-  memory = { check_max( 8.GB * task.attempt, 'memory' ) }
+  memory = { check_max( 4.GB * task.attempt, 'memory' ) }
   time = { check_max( 2.h * task.attempt, 'time' ) }
 
   errorStrategy = { task.exitStatus in [143,137,104,134,139] ? 'retry' : 'finish' }
-  maxRetries = 1
+  maxRetries = 3
   maxErrors = '-1'
 
-  // Process-specific resource requirements (others leave at default, e.g. Fastqc)
-  withName:get_software_versions {
-    memory = { check_max( 2.GB, 'memory' ) }
-    cache = false
+  // Generic resource requirements - s(ingle)c(ore)/m(ulti)c(ore)
+
+  withLabel:'sc_tiny'{
+      cpus = { check_max( 1, 'cpus' ) }
+      memory = { check_max( 1.GB * task.attempt, 'memory' ) }
+      time = { check_max( 2.h * task.attempt, 'time' ) }
   }
-  withName:convertBam {
-    cpus = { check_max(8 * task.attempt, 'cpus') }
+
+  withLabel:'sc_small'{
+      cpus = { check_max( 1, 'cpus' ) }
+      memory = { check_max( 4.GB * task.attempt, 'memory' ) }
+      time = { check_max( 2.h * task.attempt, 'time' ) }
   }
-  withName:makeSeqDict {
-    memory = { check_max( 16.GB * task.attempt, 'memory' ) }
+
+  withLabel:'sc_medium'{
+      cpus = { check_max( 1, 'cpus' ) }
+      memory = { check_max( 8.GB * task.attempt, 'memory' ) }
+      time = { check_max( 2.h * task.attempt, 'time' ) }
   }
-  withname:makeBWAIndex {
-    time = params.large_ref ? '12.h' : { check_max(8.h * task.attempt, 'time') }
+
+  withLabel:'mc_small'{
+      cpus = { check_max( 2, 'cpus' ) }
+      memory = { check_max( 4.GB * task.attempt, 'memory' ) }
+      time = { check_max( 2.h * task.attempt, 'time' ) }
   }
-  withName:bwa {
-    memory = { check_max( 16.GB * task.attempt, 'memory' ) }
-    cpus = { check_max(8 * task.attempt, 'cpus') }
-    time = { check_max(8.h * task.attempt, 'time') }
+
+  withLabel:'mc_medium' {
+      cpus = { check_max( 4, 'cpus' ) }
+      memory = { check_max( 8.GB * task.attempt, 'memory' ) }
+      time = { check_max( 2.h * task.attempt, 'time' ) }
   }
-  withName:bwamem{
-    memory = { check_max( 16.GB * task.attempt, 'memory' ) }
-    cpus = { check_max(8 * task.attempt, 'cpus') }
-    time = { check_max(8.h * task.attempt, 'time') }
+
+  withLabel:'mc_large'{
+      cpus = { check_max( 8, 'cpus' ) }
+      memory = { check_max( 16.GB * task.attempt, 'memory' ) }
+      time = { check_max( 2.h * task.attempt, 'time' ) }
   }
-  withName:qualimap{
-    cpus = { check_max(8 * task.attempt, 'cpus') }
-    errorStrategy = 'ignore'
+
+  withLabel:'mc_huge'{
+      cpus = { check_max( 32, 'cpus' ) }
+      memory = { check_max( 256.GB * task.attempt, 'memory' ) }
+      time = { check_max( 2.h * task.attempt, 'time' ) }
   }
-  withName:bam_trim{
-    cpus = { check_max(4 * task.attempt, 'cpus') }
+
+  // Process-specific resource requirements (others leave at default, e.g. Fastqc)
+  withName:get_software_versions {
+    memory = { check_max( 2.GB, 'memory' ) }
+    cache = false
+  }
+
+  withName:strip_input_fastq {
+     time = { check_max( 4.h * task.attempt, 'time' ) }
   }
-  withName:markDup{
-    cpus = { check_max(8 * task.attempt, 'cpus') }
+
+  withName:qualimap{
+    errorStrategy = 'ignore'
   }
+
   withName:preseq {
     errorStrategy = 'ignore'
   }
-  withName: fastqc {
-    errorStrategy = { task.exitStatus in [143,137] ? 'retry' : 'ignore' }
+
+  // Add 141 ignore due to unclean pipe closing by pmdtools https://github.com/pontussk/PMDtools/issues/7
+  withName: pmdtools {
+    errorStrategy = { task.exitStatus in [141] ? 'ignore' : 'retry' }
   }
+
   withName: multiqc {
     errorStrategy = { task.exitStatus in [143,137] ? 'retry' : 'ignore' }
   }
-  withName: damageprofiler {
-    time = params.large_ref ? { check_max(8.h * task.attempt, 'time') } : { check_max(2.h * task.attempt, 'time')}
-  }
-  withName: strip_input_fastq {
-    cpus = { check_max(8 * task.attempt, 'cpus') }
-    memory = { check_max( 8.GB * task.attempt, 'memory' ) }
-  }
-  withName: malt {
-    memory = { check_max( 128.GB * task.attempt, 'memory' ) }
-    cpus = { check_max(16 * task.attempt, 'cpus') }
-    time = { check_max(2.h * task.attempt, 'time') }
-  }
-  withName: maltextract {
-    memory = { check_max( 8.GB * task.attempt, 'memory' ) }
-    cpus = { check_max(4 * task.attempt, 'cpus') }
-  }
-  withName: vcf2genome {
-    memory = { check_max( 4.GB * task.attempt, 'memory' ) }
-    cpus =  1
-  }
 }
 
-
 params {
   // Defaults only, expecting to be overwritten
   max_memory = 128.GB

environment.yml

@@ -12,7 +12,7 @@ dependencies:
   - bioconda::picard=2.21.4
   - bioconda::samtools=1.9
   - bioconda::dedup=0.12.5
-  - bioconda::angsd=0.931
+  - bioconda::angsd=0.921
   - bioconda::circularmapper=1.93.4
   - bioconda::gatk4=4.1.4.1
   - bioconda::qualimap=2.2.2d