Skip to content

Commit bd75f9c

Browse files
jfy133jfy133
authored
Merge pull request #753 from nf-core/cpu-fix
Fix specified CPUs in piped commands
2 parents 2828014 + 49956c1 commit bd75f9c

2 files changed

Lines changed: 40 additions & 36 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
1616
- Updated template to nf-core/tools 1.14
1717
- [#688](https://github.com/nf-core/eager/issues/688) - Clarified the pipeline is not just for humans and microbes, but also plants and animals, and also for modern DNA
1818
- [#751](https://github.com/nf-core/eager/pull/751) - Added missing label to mtnucratio
19-
- General code cleanup and standarisation of parameters with no default setting
19+
- General code cleanup and standardisation of parameters with no default setting
20+
- [#750](https://github.com/nf-core/eager/issues/750) - Fixed piped commands requesting the same number of CPUs at each command step
2021

2122
### `Dependencies`
2223

‎main.nf‎

Lines changed: 38 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -791,7 +791,7 @@ process adapter_removal {
791791
mv *.settings output/
792792
793793
## Add R_ and L_ for unmerged reads for DeDup compatibility
794-
AdapterRemovalFixPrefix -Xmx${task.memory.toGiga()}g output/${base}.pe.combined.tmp.fq.gz | pigz -p ${task.cpus} > output/${base}.pe.combined.fq.gz
794+
AdapterRemovalFixPrefix -Xmx${task.memory.toGiga()}g output/${base}.pe.combined.tmp.fq.gz | pigz -p ${task.cpus - 1} > output/${base}.pe.combined.fq.gz
795795
"""
796796
//PE mode, collapse and trim, outputting all reads, preserving 5p
797797
} else if (seqtype == 'PE' && !params.skip_collapse && !params.skip_trim && !params.mergedonly && params.preserve5p) {
@@ -805,7 +805,7 @@ process adapter_removal {
805805
mv *.settings output/
806806
807807
## Add R_ and L_ for unmerged reads for DeDup compatibility
808-
AdapterRemovalFixPrefix -Xmx${task.memory.toGiga()}g output/${base}.pe.combined.tmp.fq.gz | pigz -p ${task.cpus} > output/${base}.pe.combined.fq.gz
808+
AdapterRemovalFixPrefix -Xmx${task.memory.toGiga()}g output/${base}.pe.combined.tmp.fq.gz | pigz -p ${task.cpus - 1} > output/${base}.pe.combined.fq.gz
809809
"""
810810
// PE mode, collapse and trim but only output collapsed reads
811811
} else if ( seqtype == 'PE' && !params.skip_collapse && !params.skip_trim && params.mergedonly && !params.preserve5p ) {
@@ -816,7 +816,7 @@ process adapter_removal {
816816
cat *.collapsed.gz *.collapsed.truncated.gz > output/${base}.pe.combined.tmp.fq.gz
817817
818818
## Add R_ and L_ for unmerged reads for DeDup compatibility
819-
AdapterRemovalFixPrefix -Xmx${task.memory.toGiga()}g output/${base}.pe.combined.tmp.fq.gz | pigz -p ${task.cpus} > output/${base}.pe.combined.fq.gz
819+
AdapterRemovalFixPrefix -Xmx${task.memory.toGiga()}g output/${base}.pe.combined.tmp.fq.gz | pigz -p ${task.cpus - 1} > output/${base}.pe.combined.fq.gz
820820
821821
mv *.settings output/
822822
"""
@@ -829,7 +829,7 @@ process adapter_removal {
829829
cat *.collapsed.gz > output/${base}.pe.combined.tmp.fq.gz
830830
831831
## Add R_ and L_ for unmerged reads for DeDup compatibility
832-
AdapterRemovalFixPrefix -Xmx${task.memory.toGiga()}g output/${base}.pe.combined.tmp.fq.gz | pigz -p ${task.cpus} > output/${base}.pe.combined.fq.gz
832+
AdapterRemovalFixPrefix -Xmx${task.memory.toGiga()}g output/${base}.pe.combined.tmp.fq.gz | pigz -p ${task.cpus - 1} > output/${base}.pe.combined.fq.gz
833833
834834
mv *.settings output/
835835
"""
@@ -843,7 +843,7 @@ process adapter_removal {
843843
cat *.collapsed.gz *.pair1.truncated.gz *.pair2.truncated.gz > output/${base}.pe.combined.tmp.fq.gz
844844
845845
## Add R_ and L_ for unmerged reads for DeDup compatibility
846-
AdapterRemovalFixPrefix -Xmx${task.memory.toGiga()}g output/${base}.pe.combined.tmp.fq.gz | pigz -p ${task.cpus} > output/${base}.pe.combined.fq.gz
846+
AdapterRemovalFixPrefix -Xmx${task.memory.toGiga()}g output/${base}.pe.combined.tmp.fq.gz | pigz -p ${task.cpus - 1} > output/${base}.pe.combined.fq.gz
847847
848848
mv *.settings output/
849849
"""
@@ -857,7 +857,7 @@ process adapter_removal {
857857
cat *.collapsed.gz > output/${base}.pe.combined.tmp.fq.gz
858858
859859
## Add R_ and L_ for unmerged reads for DeDup compatibility
860-
AdapterRemovalFixPrefix -Xmx${task.memory.toGiga()}g output/${base}.pe.combined.tmp.fq.gz | pigz -p ${task.cpus} > output/${base}.pe.combined.fq.gz
860+
AdapterRemovalFixPrefix -Xmx${task.memory.toGiga()}g output/${base}.pe.combined.tmp.fq.gz | pigz -p ${task.cpus - 1} > output/${base}.pe.combined.fq.gz
861861
862862
mv *.settings output/
863863
"""
@@ -1162,14 +1162,14 @@ process bwa {
11621162
"""
11631163
bwa aln -t ${task.cpus} $fasta ${r1} -n ${params.bwaalnn} -l ${params.bwaalnl} -k ${params.bwaalnk} -o ${params.bwaalno} -f ${libraryid}.r1.sai
11641164
bwa aln -t ${task.cpus} $fasta ${r2} -n ${params.bwaalnn} -l ${params.bwaalnl} -k ${params.bwaalnk} -o ${params.bwaalno} -f ${libraryid}.r2.sai
1165-
bwa sampe -r "@RG\\tID:ILLUMINA-${libraryid}\\tSM:${libraryid}\\tPL:illumina\\tPU:ILLUMINA-${libraryid}-${seqtype}" $fasta ${libraryid}.r1.sai ${libraryid}.r2.sai ${r1} ${r2} | samtools sort -@ ${task.cpus} -O bam - > ${libraryid}_"${seqtype}".mapped.bam
1165+
bwa sampe -r "@RG\\tID:ILLUMINA-${libraryid}\\tSM:${libraryid}\\tPL:illumina\\tPU:ILLUMINA-${libraryid}-${seqtype}" $fasta ${libraryid}.r1.sai ${libraryid}.r2.sai ${r1} ${r2} | samtools sort -@ ${task.cpus - 1} -O bam - > ${libraryid}_"${seqtype}".mapped.bam
11661166
samtools index "${libraryid}"_"${seqtype}".mapped.bam ${size}
11671167
"""
11681168
} else {
11691169
//PE collapsed, or SE data
11701170
"""
11711171
bwa aln -t ${task.cpus} ${fasta} ${r1} -n ${params.bwaalnn} -l ${params.bwaalnl} -k ${params.bwaalnk} -o ${params.bwaalno} -f ${libraryid}.sai
1172-
bwa samse -r "@RG\\tID:ILLUMINA-${libraryid}\\tSM:${libraryid}\\tPL:illumina\\tPU:ILLUMINA-${libraryid}-${seqtype}" $fasta ${libraryid}.sai $r1 | samtools sort -@ ${task.cpus} -O bam - > "${libraryid}"_"${seqtype}".mapped.bam
1172+
bwa samse -r "@RG\\tID:ILLUMINA-${libraryid}\\tSM:${libraryid}\\tPL:illumina\\tPU:ILLUMINA-${libraryid}-${seqtype}" $fasta ${libraryid}.sai $r1 | samtools sort -@ ${task.cpus - 1} -O bam - > "${libraryid}"_"${seqtype}".mapped.bam
11731173
samtools index "${libraryid}"_"${seqtype}".mapped.bam ${size}
11741174
"""
11751175
}
@@ -1194,17 +1194,18 @@ process bwamem {
11941194
params.mapper == 'bwamem'
11951195

11961196
script:
1197+
def split_cpus = Math.floor(task.cpus/2)
11971198
def fasta = "${index}/${fasta_base}"
11981199
def size = params.large_ref ? '-c' : ''
11991200

12001201
if (!params.single_end && params.skip_collapse){
12011202
"""
1202-
bwa mem -t ${task.cpus} $fasta $r1 $r2 -R "@RG\\tID:ILLUMINA-${libraryid}\\tSM:${libraryid}\\tPL:illumina\\tPU:ILLUMINA-${libraryid}-${seqtype}" | samtools sort -@ ${task.cpus} -O bam - > "${libraryid}"_"${seqtype}".mapped.bam
1203+
bwa mem -t ${split_cpus} $fasta $r1 $r2 -R "@RG\\tID:ILLUMINA-${libraryid}\\tSM:${libraryid}\\tPL:illumina\\tPU:ILLUMINA-${libraryid}-${seqtype}" | samtools sort -@ ${split_cpus} -O bam - > "${libraryid}"_"${seqtype}".mapped.bam
12031204
samtools index ${size} -@ ${task.cpus} "${libraryid}".mapped.bam
12041205
"""
12051206
} else {
12061207
"""
1207-
bwa mem -t ${task.cpus} $fasta $r1 -R "@RG\\tID:ILLUMINA-${libraryid}\\tSM:${libraryid}\\tPL:illumina\\tPU:ILLUMINA-${libraryid}-${seqtype}" | samtools sort -@ ${task.cpus} -O bam - > "${libraryid}"_"${seqtype}".mapped.bam
1208+
bwa mem -t ${split_cpus} $fasta $r1 -R "@RG\\tID:ILLUMINA-${libraryid}\\tSM:${libraryid}\\tPL:illumina\\tPU:ILLUMINA-${libraryid}-${seqtype}" | samtools sort -@ ${split_cpus} -O bam - > "${libraryid}"_"${seqtype}".mapped.bam
12081209
samtools index -@ ${task.cpus} "${libraryid}"_"${seqtype}".mapped.bam ${size}
12091210
"""
12101211
}
@@ -1302,6 +1303,7 @@ process bowtie2 {
13021303
params.mapper == 'bowtie2'
13031304

13041305
script:
1306+
def split_cpus = Math.floor(task.cpus/2)
13051307
def size = params.large_ref ? '-c' : ''
13061308
def fasta = "${index}/${fasta_base}"
13071309
def trim5 = params.bt2_trim5 != 0 ? "--trim5 ${params.bt2_trim5}" : ""
@@ -1345,13 +1347,13 @@ process bowtie2 {
13451347
//PE data without merging, PE data without any AR applied
13461348
if ( seqtype == 'PE' && ( params.skip_collapse || params.skip_adapterremoval ) ){
13471349
"""
1348-
bowtie2 -x ${fasta} -1 ${r1} -2 ${r2} -p ${task.cpus} ${sensitivity} ${bt2n} ${bt2l} ${trim5} ${trim3} --maxins ${params.bt2_maxins} --rg-id ILLUMINA-${libraryid} --rg SM:${libraryid} --rg PL:illumina --rg PU:ILLUMINA-${libraryid}-${seqtype} 2> "${libraryid}"_bt2.log | samtools sort -@ ${task.cpus} -O bam > "${libraryid}"_"${seqtype}".mapped.bam
1350+
bowtie2 -x ${fasta} -1 ${r1} -2 ${r2} -p ${split_cpus} ${sensitivity} ${bt2n} ${bt2l} ${trim5} ${trim3} --maxins ${params.bt2_maxins} --rg-id ILLUMINA-${libraryid} --rg SM:${libraryid} --rg PL:illumina --rg PU:ILLUMINA-${libraryid}-${seqtype} 2> "${libraryid}"_bt2.log | samtools sort -@ ${split_cpus} -O bam > "${libraryid}"_"${seqtype}".mapped.bam
13491351
samtools index "${libraryid}"_"${seqtype}".mapped.bam ${size}
13501352
"""
13511353
} else {
13521354
//PE collapsed, or SE data
13531355
"""
1354-
bowtie2 -x ${fasta} -U ${r1} -p ${task.cpus} ${sensitivity} ${bt2n} ${bt2l} ${trim5} ${trim3} --rg-id ILLUMINA-${libraryid} --rg SM:${libraryid} --rg PL:illumina --rg PU:ILLUMINA-${libraryid}-${seqtype} 2> "${libraryid}"_bt2.log | samtools sort -@ ${task.cpus} -O bam > "${libraryid}"_"${seqtype}".mapped.bam
1356+
bowtie2 -x ${fasta} -U ${r1} -p ${split_cpus} ${sensitivity} ${bt2n} ${bt2l} ${trim5} ${trim3} --rg-id ILLUMINA-${libraryid} --rg SM:${libraryid} --rg PL:illumina --rg PU:ILLUMINA-${libraryid}-${seqtype} 2> "${libraryid}"_bt2.log | samtools sort -@ ${split_cpus} -O bam > "${libraryid}"_"${seqtype}".mapped.bam
13551357
samtools index "${libraryid}"_"${seqtype}".mapped.bam ${size}
13561358
"""
13571359
}
@@ -1540,6 +1542,7 @@ process samtools_filter {
15401542

15411543
// Using shell block rather than script because we are playing with awk
15421544
shell:
1545+
15431546
size = !{params.large_ref} ? '-c' : ''
15441547

15451548
// Unmapped/MAPQ Filtering WITHOUT min-length filtering
@@ -1555,28 +1558,28 @@ process samtools_filter {
15551558
'''
15561559
} else if ( "${params.bam_unmapped_type}" == "bam" && params.bam_filter_minreadlength == 0 ){
15571560
'''
1558-
samtools view -h !{bam} | samtools view - -@ !{task.cpus} -f4 -o !{libraryid}.unmapped.bam
1559-
samtools view -h !{bam} | samtools view - -@ !{task.cpus} -F4 -q !{params.bam_mapping_quality_threshold} -o !{libraryid}.filtered.bam
1561+
samtools view -h !{bam} -@ !{task.cpus} -f4 -o !{libraryid}.unmapped.bam
1562+
samtools view -h !{bam} -@ !{task.cpus} -F4 -q !{params.bam_mapping_quality_threshold} -o !{libraryid}.filtered.bam
15601563
samtools index !{libraryid}.filtered.bam !{size}
15611564
'''
15621565
} else if ( "${params.bam_unmapped_type}" == "fastq" && params.bam_filter_minreadlength == 0 ){
15631566
'''
1564-
samtools view -h !{bam} | samtools view - -@ !{task.cpus} -f4 -o !{libraryid}.unmapped.bam
1565-
samtools view -h !{bam} | samtools view - -@ !{task.cpus} -F4 -q !{params.bam_mapping_quality_threshold} -o !{libraryid}.filtered.bam
1567+
samtools view -h !{bam} -@ !{task.cpus} -f4 -o !{libraryid}.unmapped.bam
1568+
samtools view -h !{bam} -@ !{task.cpus} -F4 -q !{params.bam_mapping_quality_threshold} -o !{libraryid}.filtered.bam
15661569
samtools index !{libraryid}.filtered.bam !{size}
15671570
15681571
## FASTQ
1569-
samtools fastq -tn !{libraryid}.unmapped.bam | pigz -p !{task.cpus} > !{libraryid}.unmapped.fastq.gz
1572+
samtools fastq -tn !{libraryid}.unmapped.bam | pigz -p !{task.cpus - 1} > !{libraryid}.unmapped.fastq.gz
15701573
rm !{libraryid}.unmapped.bam
15711574
'''
15721575
} else if ( "${params.bam_unmapped_type}" == "both" && params.bam_filter_minreadlength == 0 ){
15731576
'''
1574-
samtools view -h !{bam} | samtools view - -@ !{task.cpus} -f4 -o !{libraryid}.unmapped.bam
1575-
samtools view -h !{bam} | samtools view - -@ !{task.cpus} -F4 -q !{params.bam_mapping_quality_threshold} -o !{libraryid}.filtered.bam
1577+
samtools view -h !{bam} -@ !{task.cpus} -f4 -o !{libraryid}.unmapped.bam
1578+
samtools view -h !{bam} -@ !{task.cpus} -F4 -q !{params.bam_mapping_quality_threshold} -o !{libraryid}.filtered.bam
15761579
samtools index !{libraryid}.filtered.bam !{size}
15771580
15781581
## FASTQ
1579-
samtools fastq -tn !{libraryid}.unmapped.bam | pigz -p !{task.cpus} > !{libraryid}.unmapped.fastq.gz
1582+
samtools fastq -tn !{libraryid}.unmapped.bam | pigz -p !{task.cpus -1} > !{libraryid}.unmapped.fastq.gz
15801583
'''
15811584
// Unmapped/MAPQ Filtering WITH min-length filtering
15821585
} else if ( "${params.bam_unmapped_type}" == "keep" && params.bam_filter_minreadlength != 0 ) {
@@ -1593,26 +1596,26 @@ process samtools_filter {
15931596
'''
15941597
} else if ( "${params.bam_unmapped_type}" == "bam" && params.bam_filter_minreadlength != 0 ){
15951598
'''
1596-
samtools view -h !{bam} | samtools view - -@ !{task.cpus} -f4 -o !{libraryid}.unmapped.bam
1597-
samtools view -h !{bam} | samtools view - -@ !{task.cpus} -F4 -q !{params.bam_mapping_quality_threshold} -o tmp_mapped.bam
1599+
samtools view -h !{bam} -@ !{task.cpus} -f4 -o !{libraryid}.unmapped.bam
1600+
samtools view -h !{bam} -@ !{task.cpus} -F4 -q !{params.bam_mapping_quality_threshold} -o tmp_mapped.bam
15981601
filter_bam_fragment_length.py -a -l !{params.bam_filter_minreadlength} -o !{libraryid} tmp_mapped.bam
15991602
samtools index !{libraryid}.filtered.bam !{size}
16001603
'''
16011604
} else if ( "${params.bam_unmapped_type}" == "fastq" && params.bam_filter_minreadlength != 0 ){
16021605
'''
1603-
samtools view -h !{bam} | samtools view - -@ !{task.cpus} -f4 -o !{libraryid}.unmapped.bam
1604-
samtools view -h !{bam} | samtools view - -@ !{task.cpus} -F4 -q !{params.bam_mapping_quality_threshold} -o tmp_mapped.bam
1606+
samtools view -h !{bam} -@ !{task.cpus} -f4 -o !{libraryid}.unmapped.bam
1607+
samtools view -h !{bam} -@ !{task.cpus} -F4 -q !{params.bam_mapping_quality_threshold} -o tmp_mapped.bam
16051608
filter_bam_fragment_length.py -a -l !{params.bam_filter_minreadlength} -o !{libraryid} tmp_mapped.bam
16061609
samtools index !{libraryid}.filtered.bam !{size}
16071610
16081611
## FASTQ
1609-
samtools fastq -tn !{libraryid}.unmapped.bam | pigz -p !{task.cpus} > !{libraryid}.unmapped.fastq.gz
1612+
samtools fastq -tn !{libraryid}.unmapped.bam | pigz -p !{task.cpus - 1} > !{libraryid}.unmapped.fastq.gz
16101613
rm !{libraryid}.unmapped.bam
16111614
'''
16121615
} else if ( "${params.bam_unmapped_type}" == "both" && params.bam_filter_minreadlength != 0 ){
16131616
'''
1614-
samtools view -h !{bam} | samtools view - -@ !{task.cpus} -f4 -o !{libraryid}.unmapped.bam
1615-
samtools view -h !{bam} | samtools view - -@ !{task.cpus} -F4 -q !{params.bam_mapping_quality_threshold} -o tmp_mapped.bam
1617+
samtools view -h !{bam} -@ !{task.cpus} -f4 -o !{libraryid}.unmapped.bam
1618+
samtools view -h !{bam} -@ !{task.cpus} -F4 -q !{params.bam_mapping_quality_threshold} -o tmp_mapped.bam
16161619
filter_bam_fragment_length.py -a -l !{params.bam_filter_minreadlength} -o !{libraryid} tmp_mapped.bam
16171620
samtools index !{libraryid}.filtered.bam !{size}
16181621
@@ -1936,8 +1939,8 @@ process bedtools {
19361939

19371940
script:
19381941
"""
1939-
bedtools coverage -nonamecheck -a ${anno_file} -b $bam | pigz -p ${task.cpus} > "${bam.baseName}".breadth.gz
1940-
bedtools coverage -nonamecheck -a ${anno_file} -b $bam -mean | pigz -p ${task.cpus} > "${bam.baseName}".depth.gz
1942+
bedtools coverage -nonamecheck -a ${anno_file} -b $bam | pigz -p ${task.cpus - 1} > "${bam.baseName}".breadth.gz
1943+
bedtools coverage -nonamecheck -a ${anno_file} -b $bam -mean | pigz -p ${task.cpus - 1} > "${bam.baseName}".depth.gz
19411944
"""
19421945
}
19431946

@@ -2006,7 +2009,7 @@ process mapdamage_rescaling {
20062009
// Optionally perform further aDNA evaluation or filtering for just reads with damage etc.
20072010

20082011
process pmdtools {
2009-
label 'mc_small'
2012+
label 'mc_medium'
20102013
tag "${libraryid}"
20112014
publishDir "${params.outdir}/pmdtools", mode: params.publish_dir_mode
20122015

@@ -2032,15 +2035,15 @@ process pmdtools {
20322035
def size = params.large_ref ? '-c' : ''
20332036
def platypus = params.pmdtools_platypus ? '--platypus' : ''
20342037
"""
2038+
## Note all `samtools view` steps ARE required for some reason
20352039
#Run Filtering step
2036-
samtools calmd -b ${bam} ${fasta} | samtools view -h - | pmdtools --threshold ${params.pmdtools_threshold} ${treatment} ${snpcap} --header | samtools view -@ ${task.cpus} -Sb - > "${libraryid}".pmd.bam
2040+
samtools calmd ${bam} ${fasta} | pmdtools --threshold ${params.pmdtools_threshold} ${treatment} ${snpcap} --header | samtools view -Sb - > "${libraryid}".pmd.bam
20372041
20382042
#Run Calc Range step
20392043
## To allow early shut off of pipe: https://github.com/nextflow-io/nextflow/issues/1564
20402044
trap 'if [[ \$? == 141 ]]; then echo "Shutting samtools early due to -n parameter" && samtools index ${libraryid}.pmd.bam ${size}; exit 0; fi' EXIT
2041-
samtools calmd -b ${bam} ${fasta} | samtools view -h - | pmdtools --deamination ${platypus} --range ${params.pmdtools_range} ${treatment} ${snpcap} -n ${params.pmdtools_max_reads} > "${libraryid}".cpg.range."${params.pmdtools_range}".txt
2045+
samtools calmd ${bam} ${fasta} | pmdtools --deamination ${platypus} --range ${params.pmdtools_range} ${treatment} ${snpcap} -n ${params.pmdtools_max_reads} > "${libraryid}".cpg.range."${params.pmdtools_range}".txt
20422046
2043-
echo "Running indexing"
20442047
samtools index ${libraryid}.pmd.bam ${size}
20452048
"""
20462049
}
@@ -2573,7 +2576,7 @@ process sexdeterrmine_prep {
25732576

25742577
// As we collect all files for a single sex_deterrmine run, we DO NOT use the normal input/output tuple
25752578
process sexdeterrmine {
2576-
label 'sc_small'
2579+
label 'mc_small'
25772580
publishDir "${params.outdir}/sex_determination", mode: params.publish_dir_mode
25782581

25792582
input:
@@ -2909,7 +2912,7 @@ process output_documentation {
29092912
*/
29102913

29112914
process get_software_versions {
2912-
label 'sc_tiny'
2915+
label 'mc_small'
29132916
publishDir "${params.outdir}/pipeline_info", mode: params.publish_dir_mode,
29142917
saveAs: { filename ->
29152918
if (filename.indexOf(".csv") > 0) filename

0 commit comments

Comments
 (0)
⚡