Merge pull request #548 from nf-core/timelimit-mods

apeltzer · web-flow · commit 47c016b122f5 · 2020-09-14T11:05:43.000+02:00
Timelimit mods
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -33,6 +33,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 * Added a `.dockstore.yml` config file for automatic workflow registration with [dockstore.org](https://dockstore.org/)
 * Updated template to nf-core/tools 1.10.2
 * [#544](https://github.com/nf-core/eager/pull/544) Add script to perform bam filtering on fragment length
+* [#456](https://github.com/nf-core/eager/pull/546) Bumps the base (default) runtime of all processes to 4 hours, and set shorter timelimits for test profiles (1 hour)
 
 ### `Fixed`
 
diff --git a/conf/base.config b/conf/base.config
@@ -12,7 +12,7 @@
 process {
   cpus = { check_max( 1 * task.attempt, 'cpus' ) }
   memory = { check_max( 7.GB * task.attempt, 'memory' ) }
-  time = { check_max( 4.h * task.attempt, 'time' ) }
+  time = { check_max( 24.h * task.attempt, 'time' ) }
 
   errorStrategy = { task.exitStatus in [143,137,104,134,139] ? 'retry' : 'finish' }
   maxRetries = 3
@@ -23,43 +23,43 @@ process {
   withLabel:'sc_tiny'{
       cpus = { check_max( 1, 'cpus' ) }
       memory = { check_max( 1.GB * task.attempt, 'memory' ) }
-      time = { check_max( 2.h * task.attempt, 'time' ) }
+      time = { check_max( 4.h * task.attempt, 'time' ) }
   }
 
   withLabel:'sc_small'{
       cpus = { check_max( 1, 'cpus' ) }
       memory = { check_max( 4.GB * task.attempt, 'memory' ) }
-      time = { check_max( 2.h * task.attempt, 'time' ) }
+      time = { check_max( 4.h * task.attempt, 'time' ) }
   }
 
   withLabel:'sc_medium'{
       cpus = { check_max( 1, 'cpus' ) }
       memory = { check_max( 8.GB * task.attempt, 'memory' ) }
-      time = { check_max( 2.h * task.attempt, 'time' ) }
+      time = { check_max( 4.h * task.attempt, 'time' ) }
   }
 
   withLabel:'mc_small'{
-      cpus = { check_max( 2, 'cpus' ) }
+      cpus = { check_max( 2 * task.attempt, 'cpus' ) }
       memory = { check_max( 4.GB * task.attempt, 'memory' ) }
-      time = { check_max( 2.h * task.attempt, 'time' ) }
+      time = { check_max( 4.h * task.attempt, 'time' ) }
   }
 
   withLabel:'mc_medium' {
-      cpus = { check_max( 4, 'cpus' ) }
+      cpus = { check_max( 4 * task.attempt, 'cpus' ) }
       memory = { check_max( 8.GB * task.attempt, 'memory' ) }
-      time = { check_max( 2.h * task.attempt, 'time' ) }
+      time = { check_max( 4.h * task.attempt, 'time' ) }
   }
 
   withLabel:'mc_large'{
-      cpus = { check_max( 8, 'cpus' ) }
+      cpus = { check_max( 8 * task.attempt, 'cpus' ) }
       memory = { check_max( 16.GB * task.attempt, 'memory' ) }
-      time = { check_max( 2.h * task.attempt, 'time' ) }
+      time = { check_max( 4.h * task.attempt, 'time' ) }
   }
 
   withLabel:'mc_huge'{
-      cpus = { check_max( 32, 'cpus' ) }
+      cpus = { check_max( 32 * task.attempt, 'cpus' ) }
       memory = { check_max( 256.GB * task.attempt, 'memory' ) }
-      time = { check_max( 2.h * task.attempt, 'time' ) }
+      time = { check_max( 4.h * task.attempt, 'time' ) }
   }
 
   // Process-specific resource requirements (others leave at default, e.g. Fastqc)
diff --git a/conf/benchmarking_pathogenscreening.config b/conf/benchmarking_pathogenscreening.config
@@ -8,9 +8,8 @@
  */
 
 params {
-  config_profile_name = 'AWS Megatest Pathogen Screening profile'
-  config_profile_description = "An AWS 'megatest' for testing complex sequencing aDNA data for human genome mapping and pathogen screening" 
-  name = "nf-core_eager_awsmegatest_pathogenscreening"
+  config_profile_name = 'nf-core/eager benchmarking - pathogen screening profile'
+  config_profile_description = "A 'fullsized' benchmarking profile for deep screening pathogen aDNA data" 
 
   //Input data
   input = 'https://github.com/nf-core/test-datasets/raw/eager/testdata/Benchmarking/benchmarking_pathogenscreening.tsv'
diff --git a/conf/test.config b/conf/test.config
@@ -20,3 +20,49 @@ params {
   // Genome references
   fasta = 'https://raw.githubusercontent.com/nf-core/test-datasets/eager/reference/Mammoth/Mammoth_MT_Krause.fasta'
 }
+
+process {
+
+  withLabel:'sc_tiny'{
+      cpus = { check_max( 1, 'cpus' ) }
+      memory = { check_max( 1.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
+  withLabel:'sc_small'{
+      cpus = { check_max( 1, 'cpus' ) }
+      memory = { check_max( 4.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
+  withLabel:'sc_medium'{
+      cpus = { check_max( 1, 'cpus' ) }
+      memory = { check_max( 8.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
+  withLabel:'mc_small'{
+      cpus = { check_max( 2 * task.attempt, 'cpus' ) }
+      memory = { check_max( 4.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
+  withLabel:'mc_medium' {
+      cpus = { check_max( 4 * task.attempt, 'cpus' ) }
+      memory = { check_max( 8.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
+  withLabel:'mc_large'{
+      cpus = { check_max( 8 * task.attempt, 'cpus' ) }
+      memory = { check_max( 16.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
+  withLabel:'mc_huge'{
+      cpus = { check_max( 32 * task.attempt, 'cpus' ) }
+      memory = { check_max( 256.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
+}
diff --git a/conf/test_full.config b/conf/test_full.config
@@ -31,6 +31,7 @@ params {
 }
 
 process {
+   
    withName:'adapter_removal'{
       cpus = { check_max( 8, 'cpus' ) }
       memory = { check_max( 16.GB * task.attempt, 'memory' ) }
diff --git a/conf/test_tsv.config b/conf/test_tsv.config
@@ -20,3 +20,49 @@ params {
   // Genome references
   fasta = 'https://raw.githubusercontent.com/nf-core/test-datasets/eager/reference/Mammoth/Mammoth_MT_Krause.fasta'
 }
+
+process {
+
+  withLabel:'sc_tiny'{
+      cpus = { check_max( 1, 'cpus' ) }
+      memory = { check_max( 1.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
+  withLabel:'sc_small'{
+      cpus = { check_max( 1, 'cpus' ) }
+      memory = { check_max( 4.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
+  withLabel:'sc_medium'{
+      cpus = { check_max( 1, 'cpus' ) }
+      memory = { check_max( 8.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
+  withLabel:'mc_small'{
+      cpus = { check_max( 2 * task.attempt, 'cpus' ) }
+      memory = { check_max( 4.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
+  withLabel:'mc_medium' {
+      cpus = { check_max( 4 * task.attempt, 'cpus' ) }
+      memory = { check_max( 8.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
+  withLabel:'mc_large'{
+      cpus = { check_max( 8 * task.attempt, 'cpus' ) }
+      memory = { check_max( 16.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
+  withLabel:'mc_huge'{
+      cpus = { check_max( 32 * task.attempt, 'cpus' ) }
+      memory = { check_max( 256.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
+}
diff --git a/conf/test_tsv_bam.config b/conf/test_tsv_bam.config
@@ -19,4 +19,50 @@ params {
   input = 'https://raw.githubusercontent.com/nf-core/test-datasets/eager/testdata/Mammoth/mammoth_design_bam.tsv'
   // Genome references
   fasta = 'https://raw.githubusercontent.com/nf-core/test-datasets/eager/reference/Mammoth/Mammoth_MT_Krause.fasta'
+}
+
+process {
+
+  withLabel:'sc_tiny'{
+      cpus = { check_max( 1, 'cpus' ) }
+      memory = { check_max( 1.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
+  withLabel:'sc_small'{
+      cpus = { check_max( 1, 'cpus' ) }
+      memory = { check_max( 4.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
+  withLabel:'sc_medium'{
+      cpus = { check_max( 1, 'cpus' ) }
+      memory = { check_max( 8.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
+  withLabel:'mc_small'{
+      cpus = { check_max( 2 * task.attempt, 'cpus' ) }
+      memory = { check_max( 4.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
+  withLabel:'mc_medium' {
+      cpus = { check_max( 4 * task.attempt, 'cpus' ) }
+      memory = { check_max( 8.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
+  withLabel:'mc_large'{
+      cpus = { check_max( 8 * task.attempt, 'cpus' ) }
+      memory = { check_max( 16.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
+  withLabel:'mc_huge'{
+      cpus = { check_max( 32 * task.attempt, 'cpus' ) }
+      memory = { check_max( 256.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
 }
diff --git a/conf/test_tsv_complex.config b/conf/test_tsv_complex.config
@@ -20,3 +20,49 @@ params {
   // Genome references
   fasta = 'https://raw.githubusercontent.com/nf-core/test-datasets/eager/reference/Mammoth/Mammoth_MT_Krause.fasta'
 }
+
+process {
+
+  withLabel:'sc_tiny'{
+      cpus = { check_max( 1, 'cpus' ) }
+      memory = { check_max( 1.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
+  withLabel:'sc_small'{
+      cpus = { check_max( 1, 'cpus' ) }
+      memory = { check_max( 4.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
+  withLabel:'sc_medium'{
+      cpus = { check_max( 1, 'cpus' ) }
+      memory = { check_max( 8.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
+  withLabel:'mc_small'{
+      cpus = { check_max( 2 * task.attempt, 'cpus' ) }
+      memory = { check_max( 4.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
+  withLabel:'mc_medium' {
+      cpus = { check_max( 4 * task.attempt, 'cpus' ) }
+      memory = { check_max( 8.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
+  withLabel:'mc_large'{
+      cpus = { check_max( 8 * task.attempt, 'cpus' ) }
+      memory = { check_max( 16.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
+  withLabel:'mc_huge'{
+      cpus = { check_max( 32 * task.attempt, 'cpus' ) }
+      memory = { check_max( 256.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
+}
diff --git a/conf/test_tsv_fna.config b/conf/test_tsv_fna.config
@@ -19,4 +19,50 @@ params {
   input = 'https://raw.githubusercontent.com/nf-core/test-datasets/eager/testdata/Mammoth/mammoth_design_fastq.tsv'
   // Genome references
   fasta = 'https://raw.githubusercontent.com/nf-core/test-datasets/eager/reference/Mammoth/Mammoth_MT_Krause.fna'
+}
+
+process {
+
+  withLabel:'sc_tiny'{
+      cpus = { check_max( 1, 'cpus' ) }
+      memory = { check_max( 1.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
+  withLabel:'sc_small'{
+      cpus = { check_max( 1, 'cpus' ) }
+      memory = { check_max( 4.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
+  withLabel:'sc_medium'{
+      cpus = { check_max( 1, 'cpus' ) }
+      memory = { check_max( 8.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
+  withLabel:'mc_small'{
+      cpus = { check_max( 2 * task.attempt, 'cpus' ) }
+      memory = { check_max( 4.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
+  withLabel:'mc_medium' {
+      cpus = { check_max( 4 * task.attempt, 'cpus' ) }
+      memory = { check_max( 8.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
+  withLabel:'mc_large'{
+      cpus = { check_max( 8 * task.attempt, 'cpus' ) }
+      memory = { check_max( 16.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
+  withLabel:'mc_huge'{
+      cpus = { check_max( 32 * task.attempt, 'cpus' ) }
+      memory = { check_max( 256.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
 }
diff --git a/conf/test_tsv_humanbam.config b/conf/test_tsv_humanbam.config
@@ -24,3 +24,49 @@ params {
   pileupcaller_bedfile = 'https://raw.githubusercontent.com/nf-core/test-datasets/eager/reference/Human/1240K.pos.list_hs37d5.0based.bed.gz'
   pileupcaller_snpfile = 'https://raw.githubusercontent.com/nf-core/test-datasets/eager/reference/Human/1240K_covered_in_JK2067_downsampled_s0.1.numeric_chromosomes.snp'
 }
+
+process {
+
+  withLabel:'sc_tiny'{
+      cpus = { check_max( 1, 'cpus' ) }
+      memory = { check_max( 1.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
+  withLabel:'sc_small'{
+      cpus = { check_max( 1, 'cpus' ) }
+      memory = { check_max( 4.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
+  withLabel:'sc_medium'{
+      cpus = { check_max( 1, 'cpus' ) }
+      memory = { check_max( 8.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
+  withLabel:'mc_small'{
+      cpus = { check_max( 2 * task.attempt, 'cpus' ) }
+      memory = { check_max( 4.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
+  withLabel:'mc_medium' {
+      cpus = { check_max( 4 * task.attempt, 'cpus' ) }
+      memory = { check_max( 8.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
+  withLabel:'mc_large'{
+      cpus = { check_max( 8 * task.attempt, 'cpus' ) }
+      memory = { check_max( 16.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
+  withLabel:'mc_huge'{
+      cpus = { check_max( 32 * task.attempt, 'cpus' ) }
+      memory = { check_max( 256.GB * task.attempt, 'memory' ) }
+      time = { check_max( 1.h * task.attempt, 'time' ) }
+  }
+
+}
diff --git a/conf/test_tsv_kraken.config b/conf/test_tsv_kraken.config
diff --git a/conf/test_tsv_pretrim.config b/conf/test_tsv_pretrim.config

Original file line number	Diff line number	Diff line change
`@@ -31,6 +31,7 @@ params {`
`31`	`31`	`}`
`32`	`32`
`33`	`33`	`process {`
	`34`	`+`
`34`	`35`	`withName:'adapter_removal'{`
`35`	`36`	`cpus = { check_max( 8, 'cpus' ) }`
`36`	`37`	`memory = { check_max( 16.GB * task.attempt, 'memory' ) }`