Resource requirement and abstraction process fixes

CHANGELOG.md

@@ -22,7 +22,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 * [#302](https://github.com/nf-core/eager/issues/302) - Added mitochondrial to nuclear ratio calculation
 * [#302](https://github.com/nf-core/eager/issues/302) - Added VCF2Genome for concensus sequence generation
 * Fancy new logo from [ZandraFagernas](https://github.com/ZandraFagernas)
-* [#286](https://github.com/nf-core/eager/issues/286) Adds pipeline-specific profiles (loaded from nf-core configs)
+* [#286](https://github.com/nf-core/eager/issues/286) - Adds pipeline-specific profiles (loaded from nf-core configs)
+* [#310](https://github.com/nf-core/eager/issues/310) - Generalises base.config
 
 ### `Fixed`
 

conf/base.config

@@ -11,79 +11,85 @@
 
 process {
   cpus = { check_max( 1 * task.attempt, 'cpus' ) }
-  memory = { check_max( 8.GB * task.attempt, 'memory' ) }
+  memory = { check_max( 4.GB * task.attempt, 'memory' ) }
   time = { check_max( 2.h * task.attempt, 'time' ) }
 
   errorStrategy = { task.exitStatus in [143,137,104,134,139] ? 'retry' : 'finish' }
-  maxRetries = 1
+  maxRetries = 3
   maxErrors = '-1'
 
-  // Process-specific resource requirements (others leave at default, e.g. Fastqc)
-  withName:get_software_versions {
-    memory = { check_max( 2.GB, 'memory' ) }
-    cache = false
+  // Generic resource requirements - s(ingle)c(ore)/m(ulti)c(ore)
+
+  withLabel:'sc_tiny'{
+      cpus = { check_max( 1, 'cpus' ) }
+      memory = { check_max( 1.GB * task.attempt, 'memory' ) }
+      time = { check_max( 2.h * task.attempt, 'time' ) }
   }
-  withName:convertBam {
-    cpus = { check_max(8 * task.attempt, 'cpus') }
+
+  withLabel:'sc_small'{
+      cpus = { check_max( 1, 'cpus' ) }
+      memory = { check_max( 4.GB * task.attempt, 'memory' ) }
+      time = { check_max( 2.h * task.attempt, 'time' ) }
   }
-  withName:makeSeqDict {
-    memory = { check_max( 16.GB * task.attempt, 'memory' ) }
+
+  withLabel:'sc_medium'{
+      cpus = { check_max( 1, 'cpus' ) }
+      memory = { check_max( 8.GB * task.attempt, 'memory' ) }
+      time = { check_max( 2.h * task.attempt, 'time' ) }
   }
-  withname:makeBWAIndex {
-    time = params.large_ref ? '12.h' : { check_max(8.h * task.attempt, 'time') }
+
+  withLabel:'mc_small'{
+      cpus = { check_max( 2, 'cpus' ) }
+      memory = { check_max( 4.GB * task.attempt, 'memory' ) }
+      time = { check_max( 2.h * task.attempt, 'time' ) }
   }
-  withName:bwa {
-    memory = { check_max( 16.GB * task.attempt, 'memory' ) }
-    cpus = { check_max(8 * task.attempt, 'cpus') }
-    time = { check_max(8.h * task.attempt, 'time') }
+
+  withLabel:'mc_medium' {
+      cpus = { check_max( 4, 'cpus' ) }
+      memory = { check_max( 8.GB * task.attempt, 'memory' ) }
+      time = { check_max( 2.h * task.attempt, 'time' ) }
   }
-  withName:bwamem{
-    memory = { check_max( 16.GB * task.attempt, 'memory' ) }
-    cpus = { check_max(8 * task.attempt, 'cpus') }
-    time = { check_max(8.h * task.attempt, 'time') }
+
+  withLabel:'mc_large'{
+      cpus = { check_max( 8, 'cpus' ) }
+      memory = { check_max( 16.GB * task.attempt, 'memory' ) }
+      time = { check_max( 2.h * task.attempt, 'time' ) }
   }
-  withName:qualimap{
-    cpus = { check_max(8 * task.attempt, 'cpus') }
-    errorStrategy = 'ignore'
+
+  withLabel:'mc_huge'{
+      cpus = { check_max( 32, 'cpus' ) }
+      memory = { check_max( 256.GB * task.attempt, 'memory' ) }
+      time = { check_max( 2.h * task.attempt, 'time' ) }
   }
-  withName:bam_trim{
-    cpus = { check_max(4 * task.attempt, 'cpus') }
+
+  // Process-specific resource requirements (others leave at default, e.g. Fastqc)
+  withName:get_software_versions {
+    memory = { check_max( 2.GB, 'memory' ) }
+    cache = false
+  }
+
+  withName:strip_input_fastq {
+     time = { check_max( 4.h * task.attempt, 'time' ) }
   }
-  withName:markDup{
-    cpus = { check_max(8 * task.attempt, 'cpus') }
+
+  withName:qualimap{
+    errorStrategy = 'ignore'
   }
+
   withName:preseq {
     errorStrategy = 'ignore'
   }
-  withName: fastqc {
-    errorStrategy = { task.exitStatus in [143,137] ? 'retry' : 'ignore' }
+
+  // Add 141 ignore due to unclean pipe closing by pmdtools https://github.com/pontussk/PMDtools/issues/7
+  withName: pmdtools {
+    errorStrategy = { task.exitStatus in [141] ? 'ignore' : 'retry' }
   }
+
   withName: multiqc {
     errorStrategy = { task.exitStatus in [143,137] ? 'retry' : 'ignore' }
   }
-  withName: damageprofiler {
-    time = params.large_ref ? { check_max(8.h * task.attempt, 'time') } : { check_max(2.h * task.attempt, 'time')}
-  }
-  withName: strip_input_fastq {
-    cpus = { check_max(8 * task.attempt, 'cpus') }
-    memory = { check_max( 8.GB * task.attempt, 'memory' ) }
-  }
-  withName: malt {
-    memory = { check_max( 128.GB * task.attempt, 'memory' ) }
-    cpus = { check_max(16 * task.attempt, 'cpus') }
-    time = { check_max(2.h * task.attempt, 'time') }
-  }
-  withName: maltextract {
-    memory = { check_max( 8.GB * task.attempt, 'memory' ) }
-    cpus = { check_max(4 * task.attempt, 'cpus') }
-  }
-  withName: vcf2genome {
-    memory = { check_max( 4.GB * task.attempt, 'memory' ) }
-    cpus =  1
-  }
 }
 
-
 params {
   // Defaults only, expecting to be overwritten
   max_memory = 128.GB

docs/usage.md

@@ -114,7 +114,7 @@ Use this parameter to choose a configuration profile. Profiles can give configur
 For more details on how to set up your own private profile, please see [installation](../configuration/adding_your_own.md).
 
 **Basic profiles**
-These are basic profiles which primarily define where you derive the pipeline's software packages from. These are typically the profiles you would use if you are running the pipeline on your **own PC*- (vs. a HPC cluster - see below).
+These are basic profiles which primarily define where you derive the pipeline's software packages from. These are typically the profiles you would use if you are running the pipeline on your **own PC** (vs. a HPC cluster - see below).
 
 - `awsbatch`
   - A generic configuration profile to be used with AWS Batch.
@@ -274,7 +274,7 @@ The output directory where the results will be saved.
 
 #### `-w / -work-dir`
 
-The output directory where _intermediate_ files will be saved. It is **highly recommended*- that this is the same path as `--outdir`, otherwise you may 'lose' your intermediate files if you need to re-run a pipeline. By default, if this flag is not given, the intermediate files will be saved in a `work/` and `.nextflow/` directory from wherever you have run EAGER from.
+The output directory where _intermediate_ files will be saved. It is **highly recommended** that this is the same path as `--outdir`, otherwise you may 'lose' your intermediate files if you need to re-run a pipeline. By default, if this flag is not given, the intermediate files will be saved in a `work/` and `.nextflow/` directory from wherever you have run EAGER from.
 
 ### Optional Reference Options
 

environment.yml

@@ -12,7 +12,7 @@ dependencies:
   - bioconda::picard=2.21.4
   - bioconda::samtools=1.9
   - bioconda::dedup=0.12.5
-  - bioconda::angsd=0.931
+  - bioconda::angsd=0.921
   - bioconda::circularmapper=1.93.4
   - bioconda::gatk4=4.1.4.1
   - bioconda::qualimap=2.2.2d