diff --git a/CHANGELOG.md b/CHANGELOG.md index 48446b612..475230a20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. * Added a `.dockstore.yml` config file for automatic workflow registration with [dockstore.org](https://dockstore.org/) * Updated template to nf-core/tools 1.10.2 * [#544](https://github.com/nf-core/eager/pull/544) Add script to perform bam filtering on fragment length +* [#456](https://github.com/nf-core/eager/pull/546) Bumps the base (default) runtime of all processes to 4 hours, and set shorter timelimits for test profiles (1 hour) ### `Fixed` diff --git a/conf/base.config b/conf/base.config index 50f24a906..e7e51328d 100644 --- a/conf/base.config +++ b/conf/base.config @@ -12,7 +12,7 @@ process { cpus = { check_max( 1 * task.attempt, 'cpus' ) } memory = { check_max( 7.GB * task.attempt, 'memory' ) } - time = { check_max( 4.h * task.attempt, 'time' ) } + time = { check_max( 24.h * task.attempt, 'time' ) } errorStrategy = { task.exitStatus in [143,137,104,134,139] ? 'retry' : 'finish' } maxRetries = 3 @@ -23,43 +23,43 @@ process { withLabel:'sc_tiny'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 1.GB * task.attempt, 'memory' ) } - time = { check_max( 2.h * task.attempt, 'time' ) } + time = { check_max( 4.h * task.attempt, 'time' ) } } withLabel:'sc_small'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 4.GB * task.attempt, 'memory' ) } - time = { check_max( 2.h * task.attempt, 'time' ) } + time = { check_max( 4.h * task.attempt, 'time' ) } } withLabel:'sc_medium'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 8.GB * task.attempt, 'memory' ) } - time = { check_max( 2.h * task.attempt, 'time' ) } + time = { check_max( 4.h * task.attempt, 'time' ) } } withLabel:'mc_small'{ - cpus = { check_max( 2, 'cpus' ) } + cpus = { check_max( 2 * task.attempt, 'cpus' ) } memory = { check_max( 4.GB * task.attempt, 'memory' ) } - time = { check_max( 2.h * task.attempt, 'time' ) } + time = { check_max( 4.h * task.attempt, 'time' ) } } withLabel:'mc_medium' { - cpus = { check_max( 4, 'cpus' ) } + cpus = { check_max( 4 * task.attempt, 'cpus' ) } memory = { check_max( 8.GB * task.attempt, 'memory' ) } - time = { check_max( 2.h * task.attempt, 'time' ) } + time = { check_max( 4.h * task.attempt, 'time' ) } } withLabel:'mc_large'{ - cpus = { check_max( 8, 'cpus' ) } + cpus = { check_max( 8 * task.attempt, 'cpus' ) } memory = { check_max( 16.GB * task.attempt, 'memory' ) } - time = { check_max( 2.h * task.attempt, 'time' ) } + time = { check_max( 4.h * task.attempt, 'time' ) } } withLabel:'mc_huge'{ - cpus = { check_max( 32, 'cpus' ) } + cpus = { check_max( 32 * task.attempt, 'cpus' ) } memory = { check_max( 256.GB * task.attempt, 'memory' ) } - time = { check_max( 2.h * task.attempt, 'time' ) } + time = { check_max( 4.h * task.attempt, 'time' ) } } // Process-specific resource requirements (others leave at default, e.g. Fastqc) diff --git a/conf/benchmarking_pathogenscreening.config b/conf/benchmarking_pathogenscreening.config index fddf24bf2..7f53fd3be 100644 --- a/conf/benchmarking_pathogenscreening.config +++ b/conf/benchmarking_pathogenscreening.config @@ -8,9 +8,8 @@ */ params { - config_profile_name = 'AWS Megatest Pathogen Screening profile' - config_profile_description = "An AWS 'megatest' for testing complex sequencing aDNA data for human genome mapping and pathogen screening" - name = "nf-core_eager_awsmegatest_pathogenscreening" + config_profile_name = 'nf-core/eager benchmarking - pathogen screening profile' + config_profile_description = "A 'fullsized' benchmarking profile for deep screening pathogen aDNA data" //Input data input = 'https://github.com/nf-core/test-datasets/raw/eager/testdata/Benchmarking/benchmarking_pathogenscreening.tsv' diff --git a/conf/test.config b/conf/test.config index 5a0e29297..815afbf28 100644 --- a/conf/test.config +++ b/conf/test.config @@ -20,3 +20,49 @@ params { // Genome references fasta = 'https://raw.githubusercontent.com/nf-core/test-datasets/eager/reference/Mammoth/Mammoth_MT_Krause.fasta' } + +process { + + withLabel:'sc_tiny'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 1.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'sc_small'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 4.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'sc_medium'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'mc_small'{ + cpus = { check_max( 2 * task.attempt, 'cpus' ) } + memory = { check_max( 4.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'mc_medium' { + cpus = { check_max( 4 * task.attempt, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'mc_large'{ + cpus = { check_max( 8 * task.attempt, 'cpus' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'mc_huge'{ + cpus = { check_max( 32 * task.attempt, 'cpus' ) } + memory = { check_max( 256.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + +} diff --git a/conf/test_full.config b/conf/test_full.config index e395a5558..175e593ae 100644 --- a/conf/test_full.config +++ b/conf/test_full.config @@ -31,6 +31,7 @@ params { } process { + withName:'adapter_removal'{ cpus = { check_max( 8, 'cpus' ) } memory = { check_max( 16.GB * task.attempt, 'memory' ) } diff --git a/conf/test_tsv.config b/conf/test_tsv.config index 6212deea7..f865f14f7 100644 --- a/conf/test_tsv.config +++ b/conf/test_tsv.config @@ -20,3 +20,49 @@ params { // Genome references fasta = 'https://raw.githubusercontent.com/nf-core/test-datasets/eager/reference/Mammoth/Mammoth_MT_Krause.fasta' } + +process { + + withLabel:'sc_tiny'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 1.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'sc_small'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 4.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'sc_medium'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'mc_small'{ + cpus = { check_max( 2 * task.attempt, 'cpus' ) } + memory = { check_max( 4.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'mc_medium' { + cpus = { check_max( 4 * task.attempt, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'mc_large'{ + cpus = { check_max( 8 * task.attempt, 'cpus' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'mc_huge'{ + cpus = { check_max( 32 * task.attempt, 'cpus' ) } + memory = { check_max( 256.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + +} \ No newline at end of file diff --git a/conf/test_tsv_bam.config b/conf/test_tsv_bam.config index 605febb80..883ada8c4 100644 --- a/conf/test_tsv_bam.config +++ b/conf/test_tsv_bam.config @@ -19,4 +19,50 @@ params { input = 'https://raw.githubusercontent.com/nf-core/test-datasets/eager/testdata/Mammoth/mammoth_design_bam.tsv' // Genome references fasta = 'https://raw.githubusercontent.com/nf-core/test-datasets/eager/reference/Mammoth/Mammoth_MT_Krause.fasta' +} + +process { + + withLabel:'sc_tiny'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 1.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'sc_small'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 4.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'sc_medium'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'mc_small'{ + cpus = { check_max( 2 * task.attempt, 'cpus' ) } + memory = { check_max( 4.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'mc_medium' { + cpus = { check_max( 4 * task.attempt, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'mc_large'{ + cpus = { check_max( 8 * task.attempt, 'cpus' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'mc_huge'{ + cpus = { check_max( 32 * task.attempt, 'cpus' ) } + memory = { check_max( 256.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + } \ No newline at end of file diff --git a/conf/test_tsv_complex.config b/conf/test_tsv_complex.config index 9b718f437..6de1a85b6 100644 --- a/conf/test_tsv_complex.config +++ b/conf/test_tsv_complex.config @@ -20,3 +20,49 @@ params { // Genome references fasta = 'https://raw.githubusercontent.com/nf-core/test-datasets/eager/reference/Mammoth/Mammoth_MT_Krause.fasta' } + +process { + + withLabel:'sc_tiny'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 1.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'sc_small'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 4.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'sc_medium'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'mc_small'{ + cpus = { check_max( 2 * task.attempt, 'cpus' ) } + memory = { check_max( 4.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'mc_medium' { + cpus = { check_max( 4 * task.attempt, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'mc_large'{ + cpus = { check_max( 8 * task.attempt, 'cpus' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'mc_huge'{ + cpus = { check_max( 32 * task.attempt, 'cpus' ) } + memory = { check_max( 256.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + +} \ No newline at end of file diff --git a/conf/test_tsv_fna.config b/conf/test_tsv_fna.config index 6c74d0d7e..63fed01f9 100644 --- a/conf/test_tsv_fna.config +++ b/conf/test_tsv_fna.config @@ -19,4 +19,50 @@ params { input = 'https://raw.githubusercontent.com/nf-core/test-datasets/eager/testdata/Mammoth/mammoth_design_fastq.tsv' // Genome references fasta = 'https://raw.githubusercontent.com/nf-core/test-datasets/eager/reference/Mammoth/Mammoth_MT_Krause.fna' +} + +process { + + withLabel:'sc_tiny'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 1.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'sc_small'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 4.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'sc_medium'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'mc_small'{ + cpus = { check_max( 2 * task.attempt, 'cpus' ) } + memory = { check_max( 4.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'mc_medium' { + cpus = { check_max( 4 * task.attempt, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'mc_large'{ + cpus = { check_max( 8 * task.attempt, 'cpus' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'mc_huge'{ + cpus = { check_max( 32 * task.attempt, 'cpus' ) } + memory = { check_max( 256.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + } \ No newline at end of file diff --git a/conf/test_tsv_humanbam.config b/conf/test_tsv_humanbam.config index 3a02a95a6..0c7bd4aa1 100644 --- a/conf/test_tsv_humanbam.config +++ b/conf/test_tsv_humanbam.config @@ -24,3 +24,49 @@ params { pileupcaller_bedfile = 'https://raw.githubusercontent.com/nf-core/test-datasets/eager/reference/Human/1240K.pos.list_hs37d5.0based.bed.gz' pileupcaller_snpfile = 'https://raw.githubusercontent.com/nf-core/test-datasets/eager/reference/Human/1240K_covered_in_JK2067_downsampled_s0.1.numeric_chromosomes.snp' } + +process { + + withLabel:'sc_tiny'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 1.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'sc_small'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 4.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'sc_medium'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'mc_small'{ + cpus = { check_max( 2 * task.attempt, 'cpus' ) } + memory = { check_max( 4.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'mc_medium' { + cpus = { check_max( 4 * task.attempt, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'mc_large'{ + cpus = { check_max( 8 * task.attempt, 'cpus' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'mc_huge'{ + cpus = { check_max( 32 * task.attempt, 'cpus' ) } + memory = { check_max( 256.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + +} \ No newline at end of file diff --git a/conf/test_tsv_kraken.config b/conf/test_tsv_kraken.config index dbc648563..8b7c724b1 100644 --- a/conf/test_tsv_kraken.config +++ b/conf/test_tsv_kraken.config @@ -22,4 +22,50 @@ params { // Genome references fasta = 'https://raw.githubusercontent.com/nf-core/test-datasets/eager/reference/Mammoth/Mammoth_MT_Krause.fasta' database = 'https://github.com/nf-core/test-datasets/raw/eager/databases/kraken/eager_test.tar.gz' +} + +process { + + withLabel:'sc_tiny'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 1.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'sc_small'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 4.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'sc_medium'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'mc_small'{ + cpus = { check_max( 2 * task.attempt, 'cpus' ) } + memory = { check_max( 4.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'mc_medium' { + cpus = { check_max( 4 * task.attempt, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'mc_large'{ + cpus = { check_max( 8 * task.attempt, 'cpus' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'mc_huge'{ + cpus = { check_max( 32 * task.attempt, 'cpus' ) } + memory = { check_max( 256.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + } \ No newline at end of file diff --git a/conf/test_tsv_pretrim.config b/conf/test_tsv_pretrim.config index be4bcc946..9893f2131 100644 --- a/conf/test_tsv_pretrim.config +++ b/conf/test_tsv_pretrim.config @@ -19,4 +19,50 @@ params { input = 'https://raw.githubusercontent.com/nf-core/test-datasets/eager/testdata/Mammoth/mammoth_design_fastq_pretrim.tsv' // Genome references fasta = 'https://raw.githubusercontent.com/nf-core/test-datasets/eager/reference/Mammoth/Mammoth_MT_Krause.fasta' +} + +process { + + withLabel:'sc_tiny'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 1.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'sc_small'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 4.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'sc_medium'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'mc_small'{ + cpus = { check_max( 2 * task.attempt, 'cpus' ) } + memory = { check_max( 4.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'mc_medium' { + cpus = { check_max( 4 * task.attempt, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'mc_large'{ + cpus = { check_max( 8 * task.attempt, 'cpus' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + + withLabel:'mc_huge'{ + cpus = { check_max( 32 * task.attempt, 'cpus' ) } + memory = { check_max( 256.GB * task.attempt, 'memory' ) } + time = { check_max( 1.h * task.attempt, 'time' ) } + } + } \ No newline at end of file