Skip to content

Commit 631b0dc

Browse files
maxulyssemaxulysse
committed
Add new feature to get software versions for MultiQC
1 parent 9dec134 commit 631b0dc

12 files changed

Lines changed: 517 additions & 218 deletions

‎annotate.nf‎

Lines changed: 54 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ tools = params.tools ? params.tools.split(',').collect{it.trim().toLowerCase()}
5757
annotateTools = params.annotateTools ? params.annotateTools.split(',').collect{it.trim().toLowerCase()} : []
5858
annotateVCF = params.annotateVCF ? params.annotateVCF.split(',').collect{it.trim()} : []
5959

60-
directoryMap = defineDirectoryMap()
60+
directoryMap = SarekUtils.defineDirectoryMap(params.outDir)
6161
toolList = defineToolList()
6262

6363
if (!SarekUtils.checkParameterList(tools,toolList)) exit 1, 'Unknown tool(s), see --help for more information'
@@ -102,7 +102,7 @@ if (annotateVCF == []) {
102102

103103
vcfNotToAnnotate.close()
104104

105-
(vcfForBCFtools, vcfForSnpeff, vcfForVep) = vcfToAnnotate.into(3)
105+
(vcfForBCFtools, vcfForVCFtools, vcfForSnpeff, vcfForVep) = vcfToAnnotate.into(4)
106106

107107
process RunBcftoolsStats {
108108
tag {vcf}
@@ -117,17 +117,35 @@ process RunBcftoolsStats {
117117

118118
when: !params.noReports
119119

120-
script:
121-
"""
122-
bcftools stats ${vcf} > ${vcf.baseName}.bcf.tools.stats.out
123-
"""
120+
script: QC.bcftools(vcf)
124121
}
125122

126123
if (params.verbose) bcfReport = bcfReport.view {
127124
"BCFTools stats report:\n\
128125
File : [${it.fileName}]"
129126
}
130127

128+
process RunVcftools {
129+
tag {vcf}
130+
131+
publishDir directoryMap.vcftools, mode: 'link'
132+
133+
input:
134+
set variantCaller, file(vcf) from vcfForVCFtools
135+
136+
output:
137+
file ("${vcf.baseName}.*") into vcfReport
138+
139+
when: !params.noReports
140+
141+
script: QC.vcftools(vcf)
142+
}
143+
144+
if (params.verbose) vcfReport = vcfReport.view {
145+
"VCFTools stats report:\n\
146+
File : [${it.fileName}]"
147+
}
148+
131149
process RunSnpeff {
132150
tag {vcf}
133151

@@ -208,6 +226,34 @@ if (params.verbose) vepReport = vepReport.view {
208226
Files : ${it.fileName}"
209227
}
210228

229+
process GetVersionBCFtools {
230+
publishDir directoryMap.version, mode: 'link'
231+
output: file("v_*.txt")
232+
when: !params.noReports
233+
script: QC.getVersionBCFtools()
234+
}
235+
236+
process GetVersionSnpEFF {
237+
publishDir directoryMap.version, mode: 'link'
238+
output: file("v_*.txt")
239+
when: 'snpeff' in tools || 'merge' in tools
240+
script: QC.getVersionSnpEFF()
241+
}
242+
243+
process GetVersionVCFtools {
244+
publishDir directoryMap.version, mode: 'link'
245+
output: file("v_*.txt")
246+
when: !params.noReports
247+
script: QC.getVersionVCFtools()
248+
}
249+
250+
process GetVersionVEP {
251+
publishDir directoryMap.version, mode: 'link'
252+
output: file("v_*.txt")
253+
when: 'vep' in tools || 'merge' in tools
254+
script: QC.getVersionVEP()
255+
}
256+
211257
/*
212258
================================================================================
213259
= F U N C T I O N S =
@@ -219,26 +265,11 @@ def checkUppmaxProject() {
219265
return !(workflow.profile == 'slurm' && !params.project)
220266
}
221267

222-
def defineDirectoryMap() {
223-
return [
224-
'haplotypecaller' : "${params.outDir}/VariantCalling/HaplotypeCaller",
225-
'manta' : "${params.outDir}/VariantCalling/Manta",
226-
'mutect1' : "${params.outDir}/VariantCalling/MuTect1",
227-
'mutect2' : "${params.outDir}/VariantCalling/MuTect2",
228-
'strelka' : "${params.outDir}/VariantCalling/Strelka",
229-
'strelkabp' : "${params.outDir}/VariantCalling/StrelkaBP",
230-
'bcftoolsStats' : "${params.outDir}/Reports/BCFToolsStats",
231-
'snpeffReports' : "${params.outDir}/Reports/SnpEff",
232-
'snpeff' : "${params.outDir}/Annotation/SnpEff",
233-
'vep' : "${params.outDir}/Annotation/VEP"
234-
]
235-
}
236-
237268
def defineToolList() {
238269
return [
270+
'merge',
239271
'snpeff',
240-
'vep',
241-
'merge'
272+
'vep'
242273
]
243274
}
244275

‎bin/scrape_tool_versions.py‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
#!/usr/bin/env python
2+
from __future__ import print_function
3+
from collections import OrderedDict
4+
import re
5+
6+
regexes = {
7+
'AlleleCount': ['v_allelecount.txt', r"(\S+)"],
8+
'ASCAT': ['v_ascat.txt', r"(\d\.\d+)"],
9+
'bcftools': ['v_bcftools.txt', r"bcftools (\S+)"],
10+
'BWA': ['v_bwa.txt', r"Version: (\S+)"],
11+
'FastQC': ['v_fastqc.txt', r"FastQC v(\S+)"],
12+
'GATK': ['v_gatk.txt', r"GATK version(\S+)"],
13+
'htslib': ['v_samtools.txt', r"htslib (\S+)"],
14+
'Manta': ['v_manta.txt', r"([0-9.]+)"],
15+
'MultiQC': ['v_multiqc.txt', r"multiqc, version (\S+)"],
16+
'Nextflow': ['v_nextflow.txt', r"(\S+)"],
17+
'FreeBayes': ['v_freebayes.txt', r"version: v(\d\.\d\.\d+)"],
18+
'Picard': ['v_picard.txt', r"Picard version:(\d\.\d\.\d+)"],
19+
'Qualimap': ['v_qualimap.txt', r"QualiMap v.(\S+)"],
20+
'R': ['v_r.txt', r"R version (\S+)"],
21+
'samtools': ['v_samtools.txt', r"samtools (\S+)"],
22+
'Sarek': ['v_sarek.txt', r"(\S+)"],
23+
'SnpEff': ['v_snpeff.txt', r"version SnpEff (\S+)"],
24+
'Strelka': ['v_strelka.txt', r"([0-9.]+)"],
25+
'vcftools': ['v_vcftools.txt', r"([0-9.]+)"],
26+
'VEP': ['v_vep.txt', r"ensembl-vep : (\S+)"],
27+
}
28+
results = OrderedDict()
29+
results['Sarek'] = '<span style="color:#999999;\">N/A</span>'
30+
results['Nextflow'] = '<span style="color:#999999;\">N/A</span>'
31+
results['BWA'] = '<span style="color:#999999;\">N/A</span>'
32+
results['samtools'] = '<span style="color:#999999;\">N/A</span>'
33+
results['htslib'] = '<span style="color:#999999;\">N/A</span>'
34+
results['GATK'] = '<span style="color:#999999;\">N/A</span>'
35+
results['Picard'] = '<span style="color:#999999;\">N/A</span>'
36+
results['Manta'] = '<span style="color:#999999;\">N/A</span>'
37+
results['Strelka'] = '<span style="color:#999999;\">N/A</span>'
38+
results['FreeBayes'] = '<span style="color:#999999;\">N/A</span>'
39+
results['AlleleCount'] = '<span style="color:#999999;\">N/A</span>'
40+
results['R'] = '<span style="color:#999999;\">N/A</span>'
41+
results['ASCAT'] = '<span style="color:#999999;\">N/A</span>'
42+
results['SnpEff'] = '<span style="color:#999999;\">N/A</span>'
43+
results['VEP'] = '<span style="color:#999999;\">N/A</span>'
44+
results['FastQC'] = '<span style="color:#999999;\">N/A</span>'
45+
results['Qualimap'] = '<span style="color:#999999;\">N/A</span>'
46+
results['bcftools'] = '<span style="color:#999999;\">N/A</span>'
47+
results['vcftools'] = '<span style="color:#999999;\">N/A</span>'
48+
results['MultiQC'] = '<span style="color:#999999;\">N/A</span>'
49+
50+
# Search each file using its regex
51+
for k, v in regexes.items():
52+
try:
53+
with open(v[0]) as x:
54+
versions = x.read()
55+
match = re.search(v[1], versions)
56+
if match:
57+
results[k] = "v {}".format(match.group(1))
58+
except Exception as FileNotFoundError:
59+
print("No such file:", v[0])
60+
61+
# Remove empty keys (defining them above ensures correct order)
62+
for k in ['Sarek', 'Nextflow', 'BWA', 'samtools', 'htslib', 'GATK', 'Picard', 'Manta', 'Strelka', 'FreeBayes', 'AlleleCount', 'R', 'ASCAT', 'SnpEff', 'VEP', 'FastQC', 'Qualimap', 'bcftools', 'vcftools', 'MultiQC']:
63+
if results[k] == '<span style="color:#999999;\">N/A</span>':
64+
del(results[k])
65+
66+
# Dump to YAML
67+
print ('''
68+
id: 'Sarek'
69+
order: -1000
70+
section_href: 'https://github.com/SciLifeLab/Sarek'
71+
plot_type: 'html'
72+
description: 'tool versions are collected at run time from output.'
73+
data: |
74+
<dl class="dl-horizontal" style="margin-bottom:0;">
75+
''')
76+
for k,v in results.items():
77+
print(" <dt>{}</dt><dd>{}</dd>".format(k,v))
78+
print (" </dl>")

‎buildReferences.nf‎

Lines changed: 10 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -98,27 +98,24 @@ if (params.verbose) ch_decompressedFiles = ch_decompressedFiles.view {
9898
}
9999

100100
ch_fastaFile = Channel.create()
101-
ch_otherFiles = Channel.create()
102-
ch_vcfFiles = Channel.create()
101+
ch_fastaForBWA = Channel.create()
102+
ch_fastaForPicard = Channel.create()
103+
ch_fastaForSAMTools = Channel.create()
104+
ch_otherFile = Channel.create()
105+
ch_vcfFile = Channel.create()
103106

104107
ch_decompressedFiles
105-
.choice(ch_fastaFile, ch_vcfFiles, ch_otherFiles) {
108+
.choice(ch_fastaFile, ch_vcfFile, ch_otherFile) {
106109
it =~ ".fasta" ? 0 :
107110
it =~ ".vcf" ? 1 : 2}
108111

109-
(ch_fastaFile, ch_fastaFileToKeep) = ch_fastaFile.into(2)
110-
(ch_vcfFiles, ch_vcfFilesToKeep) = ch_vcfFiles.into(2)
112+
(ch_fastaForBWA, ch_fastaForPicard, ch_fastaForSAMTools, ch_fastaFileToKeep) = ch_fastaFile.into(4)
113+
(ch_vcfFile, ch_vcfFileToKeep) = ch_vcfFile.into(2)
111114

112115
ch_notCompressedfiles
113-
.mix(ch_otherFiles, ch_fastaFileToKeep, ch_vcfFilesToKeep)
116+
.mix(ch_fastaFileToKeep, ch_vcfFileToKeep, ch_otherFile)
114117
.collectFile(storeDir: params.outDir)
115118

116-
ch_fastaForBWA = Channel.create()
117-
ch_fastaForPicard = Channel.create()
118-
ch_fastaForSAMTools = Channel.create()
119-
120-
ch_fastaFile.into(ch_fastaForBWA,ch_fastaForPicard,ch_fastaForSAMTools)
121-
122119
process BuildBWAindexes {
123120
tag {f_reference}
124121

@@ -193,7 +190,7 @@ process BuildVCFIndex {
193190
publishDir params.outDir, mode: 'link'
194191

195192
input:
196-
file(f_reference) from ch_vcfFiles
193+
file(f_reference) from ch_vcfFile
197194

198195
output:
199196
file("${f_reference}.idx") into ch_vcfIndex

‎configuration/containers.config‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,21 @@ process {
1414
$BuildVCFIndex.container = "${params.repository}/igvtools:${params.tag}"
1515
$ConcatVCF.container = "${params.repository}/sarek:${params.tag}"
1616
$CreateRecalibrationTable.container = "${params.repository}/gatk:${params.tag}"
17+
$GetVersionAll.container = "${params.repository}/qctools:${params.tag}"
18+
$GetVersionAlleleCount.container = "${params.repository}/runallelecount:${params.tag}"
19+
$GetVersionASCAT.container = "${params.repository}/r-base:${params.tag}"
20+
$GetVersionBamQC.container = "${params.repository}/qctools:${params.tag}"
21+
$GetVersionBCFtools.container = "${params.repository}/sarek:${params.tag}"
22+
$GetVersionBWAsamtools.container = "${params.repository}/sarek:${params.tag}"
23+
$GetVersionFastQC.container = "${params.repository}/qctools:${params.tag}"
24+
$GetVersionFreeBayes.container = "${params.repository}/freebayes:${params.tag}"
25+
$GetVersionGATK.container = "${params.repository}/gatk:${params.tag}"
26+
$GetVersionManta.container = "${params.repository}/sarek:${params.tag}"
27+
$GetVersionPicard.container = "${params.repository}/picard:${params.tag}"
28+
$GetVersionSnpeff.container = {params.genome == 'GRCh38' ? "${params.repository}/snpeffgrch38:${params.tag}" : "${params.repository}/snpeffgrch37:${params.tag}"}
29+
$GetVersionStrelka.container = "${params.repository}/sarek:${params.tag}"
30+
$GetVersionVCFtools.container = "${params.repository}/qctools:${params.tag}"
31+
$GetVersionVEP.container = {params.genome == 'GRCh38' ? "${params.repository}/vepgrch38:${params.tag}" : "${params.repository}/vepgrch37:${params.tag}"}
1732
$IndelRealigner.container = "${params.repository}/gatk:${params.tag}"
1833
$MapReads.container = "${params.repository}/sarek:${params.tag}"
1934
$MarkDuplicates.container = "${params.repository}/picard:${params.tag}"

‎configuration/singularity-path.config‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,21 @@ process {
1919
$BuildVCFIndex.container = "${params.containerPath}/igvtools-${params.tag}.img"
2020
$ConcatVCF.container = "${params.containerPath}/sarek-${params.tag}.img"
2121
$CreateRecalibrationTable.container = "${params.containerPath}/gatk-${params.tag}.img"
22-
$GenerateMultiQCconfig.container = "${params.containerPath}/qctools-${params.tag}.img"
22+
$GetVersionAll.container = "${params.containerPath}/qctools-${params.tag}.img"
23+
$GetVersionAlleleCount.container = "${params.containerPath}/runallelecount-${params.tag}.img"
24+
$GetVersionASCAT.container = "${params.containerPath}/r-base-${params.tag}.img"
25+
$GetVersionBamQC.container = "${params.containerPath}/qctools-${params.tag}.img"
26+
$GetVersionBCFtools.container = "${params.containerPath}/sarek-${params.tag}.img"
27+
$GetVersionBWAsamtools.container = "${params.containerPath}/sarek-${params.tag}.img"
28+
$GetVersionFastQC.container = "${params.containerPath}/qctools-${params.tag}.img"
29+
$GetVersionFreeBayes.container = "${params.containerPath}/freebayes-${params.tag}.img"
30+
$GetVersionGATK.container = "${params.containerPath}/gatk-${params.tag}.img"
31+
$GetVersionManta.container = "${params.containerPath}/sarek-${params.tag}.img"
32+
$GetVersionPicard.container = "${params.containerPath}/picard-${params.tag}.img"
33+
$GetVersionSnpeff.container = {params.genome == 'GRCh38' ? "${params.containerPath}/snpeffgrch38-${params.tag}.img" : "${params.containerPath}/snpeffgrch37-${params.tag}.img"}
34+
$GetVersionStrelka.container = "${params.containerPath}/sarek-${params.tag}.img"
35+
$GetVersionVCFtools.container = "${params.containerPath}/qctools-${params.tag}.img"
36+
$GetVersionVEP.container = {params.genome == 'GRCh38' ? "${params.containerPath}/vepgrch38-${params.tag}.img" : "${params.containerPath}/vepgrch37-${params.tag}.img"}
2337
$IndelRealigner.container = "${params.containerPath}/gatk-${params.tag}.img"
2438
$MapReads.container = "${params.containerPath}/sarek-${params.tag}.img"
2539
$MarkDuplicates.container = "${params.containerPath}/picard-${params.tag}.img"

0 commit comments

Comments
 (0)
⚡