Merge pull request #671 from MaxUlysse/Publish

add publishDirMode param and docs

Author: maxulysse

CHANGELOG.md

@@ -7,9 +7,14 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+### `Added`
+
+-   [#671](https://github.com/SciLifeLab/Sarek/pull/671) - New `publishDirMode` param and docs
+
 ### `Changed`
 
 -   [#663](https://github.com/SciLifeLab/Sarek/pull/663) - Update `do_release.sh` script
+-   [#671](https://github.com/SciLifeLab/Sarek/pull/671) - publishDir modes are now params
 
 ### `Fixed`
 

annotate.nf

@@ -103,7 +103,7 @@ vcfForVep = vcfForVep.map {
 process RunBcftoolsStats {
   tag {vcf}
 
-  publishDir directoryMap.bcftoolsStats, mode: 'link'
+  publishDir directoryMap.bcftoolsStats, mode: params.publishDirMode
 
   input:
     set variantCaller, file(vcf) from vcfForBCFtools
@@ -124,7 +124,7 @@ if (params.verbose) bcfReport = bcfReport.view {
 process RunVcftools {
   tag {vcf}
 
-  publishDir directoryMap.vcftools, mode: 'link'
+  publishDir directoryMap.vcftools, mode: params.publishDirMode
 
   input:
     set variantCaller, file(vcf) from vcfForVCFtools
@@ -145,7 +145,7 @@ if (params.verbose) vcfReport = vcfReport.view {
 process RunSnpeff {
   tag {"${variantCaller} - ${vcf}"}
 
-  publishDir params.outDir, mode: 'link', saveAs: {
+  publishDir params.outDir, mode: params.publishDirMode, saveAs: {
     if (it == "${vcf.simpleName}_snpEff.csv") "${directoryMap.snpeffReports}/${it}"
     else if (it == "${vcf.simpleName}_snpEff.ann.vcf") null
     else "${directoryMap.snpeff}/${it}"
@@ -198,7 +198,7 @@ if('merge' in tools) {
 process RunVEP {
   tag {"${variantCaller} - ${vcf}"}
 
-  publishDir params.outDir, mode: 'link', saveAs: {
+  publishDir params.outDir, mode: params.publishDirMode, saveAs: {
     if (it == "${vcf.simpleName}_VEP.summary.html") "${directoryMap.vep}/${it}"
     else null
   }
@@ -245,7 +245,7 @@ vcfToCompress = snpeffVCF.mix(vepVCF)
 process CompressVCF {
   tag {"${annotator} - ${vcf}"}
 
-  publishDir "${directoryMap."$finalannotator"}", mode: 'link'
+  publishDir "${directoryMap."$finalannotator"}", mode: params.publishDirMode
 
   input:
     set annotator, variantCaller, file(vcf) from vcfToCompress
@@ -268,14 +268,14 @@ if (params.verbose) vcfCompressedoutput = vcfCompressedoutput.view {
 }
 
 process GetVersionSnpeff {
-  publishDir directoryMap.version, mode: 'link'
+  publishDir directoryMap.version, mode: params.publishDirMode
   output: file("v_*.txt")
   when: 'snpeff' in tools || 'merge' in tools
   script: QC.getVersionSnpEFF()
 }
 
 process GetVersionVEP {
-  publishDir directoryMap.version, mode: 'link'
+  publishDir directoryMap.version, mode: params.publishDirMode
   output: file("v_*.txt")
   when: 'vep' in tools || 'merge' in tools
   script: QC.getVersionVEP()

buildContainers.nf

@@ -86,7 +86,7 @@ if (params.verbose) containersBuilt = containersBuilt.view {
 process PullSingularityContainers {
   tag {"${params.repository}/${container}:${params.tag}"}
 
-  publishDir "${params.containerPath}", mode: 'move'
+  publishDir "${params.containerPath}", mode: params.publishDirMode
 
   input:
     val container from singularityContainers

buildReferences.nf

@@ -103,7 +103,7 @@ ch_notCompressedfiles
 process BuildBWAindexes {
   tag {f_reference}
 
-  publishDir params.outDir, mode: 'link'
+  publishDir params.outDir, mode: params.publishDirMode
 
   input:
     file(f_reference) from ch_fastaForBWA
@@ -125,7 +125,7 @@ if (params.verbose) bwaIndexes.flatten().view {
 process BuildReferenceIndex {
   tag {f_reference}
 
-  publishDir params.outDir, mode: 'link'
+  publishDir params.outDir, mode: params.publishDirMode
 
   input:
     file(f_reference) from ch_fastaReference
@@ -149,7 +149,7 @@ if (params.verbose) ch_referenceIndex.view {
 process BuildSAMToolsIndex {
   tag {f_reference}
 
-  publishDir params.outDir, mode: 'link'
+  publishDir params.outDir, mode: params.publishDirMode
 
   input:
     file(f_reference) from ch_fastaForSAMTools
@@ -170,7 +170,7 @@ if (params.verbose) ch_samtoolsIndex.view {
 process BuildVCFIndex {
   tag {f_reference}
 
-  publishDir params.outDir, mode: 'link'
+  publishDir params.outDir, mode: params.publishDirMode
 
   input:
     file(f_reference) from ch_vcfFile

conf/base.config

@@ -27,6 +27,7 @@ params {
   nucleotidesPerSecond = 1000.0 // To estimate interval size by default
   onlyQC = false // All process will be run and not only the QC tools
   outDir = "${PWD}" // Path to output directory
+  publishDirMode = 'link' // publishDir mode is 'link' by default
   push = false // Don't push container to DockerHub
   repository = wf_repository // DockerHub containers repository
   singularity = false // Don't use singularity to build buildContainers.nf

docs/PARAMETERS.md

@@ -21,6 +21,21 @@ Choose an output directory
 Specify a project number ID on a UPPMAX cluster.
 (optional if not on such a cluster)
 
+## --publishDirMode
+
+Specify wich mode `publishDir` directive need to follow, in Sarek link is the default mode.
+
+From [Nextflow documentation](https://www.nextflow.io/docs/latest/process.html#publishdir):
+
+| Mode         | Description |
+|--------------|-------------|
+| copy         | Copies the output files into the published directory |
+| copyNoFollow | Copies the output files into the published directory without following symlinks ie. copies the links themselves |
+| link         | Creates a hard link in the published directory for each process output file (default) |
+| move         | Moves the output files into the published directory. Note: this is only supposed to be used for a terminating process i.e. a process whose output is not consumed by any other downstream process |
+| rellink      | Creates a relative symbolic link in the published directory for each process output file |
+| symlink      | Creates an absolute symbolic link in the published directory for each process output file |
+
 ### --sample `file.tsv`
 
 Use the given TSV file as sample (cf [TSV documentation](TSV.md)).

lib/SarekUtils.groovy

@@ -71,6 +71,8 @@ class SarekUtils {
       'outDir',
       'params',
       'project',
+      'publish-dir-mode',
+      'publishDirMode',
       'push',
       'ref-dir',
       'refDir',

main.nf

@@ -123,7 +123,7 @@ if (params.verbose) bamFiles = bamFiles.view {
 process RunFastQC {
   tag {idPatient + "-" + idRun}
 
-  publishDir "${directoryMap.fastQC}/${idRun}", mode: 'link'
+  publishDir "${directoryMap.fastQC}/${idRun}", mode: params.publishDirMode
 
   input:
     set idPatient, status, idSample, idRun, file(fastqFile1), file(fastqFile2) from fastqFilesforFastQC
@@ -226,7 +226,7 @@ if (params.verbose) mergedBam = mergedBam.view {
 process MarkDuplicates {
   tag {idPatient + "-" + idSample}
 
-  publishDir params.outDir, mode: 'link',
+  publishDir params.outDir, mode: params.publishDirMode,
     saveAs: {
       if (it == "${bam}.metrics") "${directoryMap.markDuplicatesQC}/${it}"
       else "${directoryMap.duplicateMarked}/${it}"
@@ -283,7 +283,7 @@ if (params.verbose) duplicateMarkedBams = duplicateMarkedBams.view {
 process CreateRecalibrationTable {
   tag {idPatient + "-" + idSample}
 
-  publishDir directoryMap.duplicateMarked, mode: 'link', overwrite: false
+  publishDir directoryMap.duplicateMarked, mode: params.publishDirMode, overwrite: false
 
   input:
     set idPatient, status, idSample, file(bam), file(bai) from mdBam // realignedBam
@@ -350,7 +350,7 @@ recalTables = recalTables.map { [it[0]] + it[2..-1] } // remove status
 process RecalibrateBam {
   tag {idPatient + "-" + idSample}
 
-  publishDir directoryMap.recalibrated, mode: 'link'
+  publishDir directoryMap.recalibrated, mode: params.publishDirMode
 
   input:
     set idPatient, status, idSample, file(bam), file(bai), file(recalibrationReport) from recalibrationTable
@@ -398,7 +398,7 @@ if (params.verbose) recalibratedBam = recalibratedBam.view {
 process RunSamtoolsStats {
   tag {idPatient + "-" + idSample}
 
-  publishDir directoryMap.samtoolsStats, mode: 'link'
+  publishDir directoryMap.samtoolsStats, mode: params.publishDirMode
 
   input:
     set idPatient, status, idSample, file(bam), file(bai) from bamForSamToolsStats
@@ -419,7 +419,7 @@ if (params.verbose) samtoolsStatsReport = samtoolsStatsReport.view {
 process RunBamQC {
   tag {idPatient + "-" + idSample}
 
-  publishDir directoryMap.bamQC, mode: 'link'
+  publishDir directoryMap.bamQC, mode: params.publishDirMode
 
   input:
     set idPatient, status, idSample, file(bam), file(bai) from bamForBamQC

runMultiQC.nf

@@ -47,7 +47,7 @@ directoryMap = SarekUtils.defineDirectoryMap(params.outDir)
 startMessage()
 
 process GetVersionAll {
-  publishDir directoryMap.multiQC, mode: 'link'
+  publishDir directoryMap.multiQC, mode: params.publishDirMode
 
   input:
     file(versions) from Channel.fromPath("${directoryMap.version}/*").collect().ifEmpty(file ("empty"))
@@ -94,7 +94,7 @@ reportsForMultiQC = Channel.empty()
   ).collect()
 
 process RunMultiQC {
-  publishDir directoryMap.multiQC, mode: 'link'
+  publishDir directoryMap.multiQC, mode: params.publishDirMode
 
   input:
     file (multiqcConfig) from createMultiQCconfig()

somaticVC.nf

@@ -113,7 +113,7 @@ if (params.verbose) recalibratedBam = recalibratedBam.view {
 process RunSamtoolsStats {
   tag {idPatient + "-" + idSample}
 
-  publishDir directoryMap.samtoolsStats, mode: 'link'
+  publishDir directoryMap.samtoolsStats, mode: params.publishDirMode
 
   input:
     set idPatient, status, idSample, file(bam), file(bai) from bamForSamToolsStats
@@ -134,7 +134,7 @@ if (params.verbose) samtoolsStatsReport = samtoolsStatsReport.view {
 process RunBamQC {
   tag {idPatient + "-" + idSample}
 
-  publishDir directoryMap.bamQC, mode: 'link'
+  publishDir directoryMap.bamQC, mode: params.publishDirMode
 
   input:
     set idPatient, status, idSample, file(bam), file(bai) from bamForBamQC
@@ -357,7 +357,7 @@ if (params.verbose) vcfsToMerge = vcfsToMerge.view {
 process ConcatVCF {
   tag {variantCaller + "_" + idSampleTumor + "_vs_" + idSampleNormal}
 
-  publishDir "${directoryMap."$variantCaller"}", mode: 'link'
+  publishDir "${directoryMap."$variantCaller"}", mode: params.publishDirMode
 
   input:
     set variantCaller, idPatient, idSampleNormal, idSampleTumor, file(vcFiles) from vcfsToMerge
@@ -392,7 +392,7 @@ if (params.verbose) vcfConcatenated = vcfConcatenated.view {
 process RunStrelka {
   tag {idSampleTumor + "_vs_" + idSampleNormal}
 
-  publishDir directoryMap.strelka, mode: 'link'
+  publishDir directoryMap.strelka, mode: params.publishDirMode
 
   input:
     set idPatient, idSampleNormal, file(bamNormal), file(baiNormal), idSampleTumor, file(bamTumor), file(baiTumor) from bamsForStrelka
@@ -449,7 +449,7 @@ if (params.verbose) strelkaOutput = strelkaOutput.view {
 process RunManta {
   tag {idSampleTumor + "_vs_" + idSampleNormal}
 
-  publishDir directoryMap.manta, mode: 'link'
+  publishDir directoryMap.manta, mode: params.publishDirMode
 
   input:
     set idPatient, idSampleNormal, file(bamNormal), file(baiNormal), idSampleTumor, file(bamTumor), file(baiTumor) from bamsForManta
@@ -503,7 +503,7 @@ if (params.verbose) mantaOutput = mantaOutput.view {
 process RunSingleManta {
   tag {idSample + " - Tumor-Only"}
 
-  publishDir directoryMap.manta, mode: 'link'
+  publishDir directoryMap.manta, mode: params.publishDirMode
 
   input:
     set idPatient, status, idSample, file(bam), file(bai) from bamsForSingleManta
@@ -562,7 +562,7 @@ bamsForStrelkaBP = bamsForStrelkaBP.map {
 process RunStrelkaBP {
   tag {idSampleTumor + "_vs_" + idSampleNormal}
 
-  publishDir directoryMap.strelkabp, mode: 'link'
+  publishDir directoryMap.strelkabp, mode: params.publishDirMode
 
   input:
     set idPatient, idSampleNormal, file(bamNormal), file(baiNormal), idSampleTumor, file(bamTumor), file(baiTumor), file(mantaCSI), file(mantaCSIi) from bamsForStrelkaBP
@@ -654,7 +654,7 @@ alleleCountOutput = alleleCountOutput.map {
 process RunConvertAlleleCounts {
   tag {idSampleTumor + "_vs_" + idSampleNormal}
 
-  publishDir directoryMap.ascat, mode: 'link'
+  publishDir directoryMap.ascat, mode: params.publishDirMode
 
   input:
     set idPatient, idSampleNormal, idSampleTumor, file(alleleCountNormal), file(alleleCountTumor) from alleleCountOutput
@@ -676,7 +676,7 @@ process RunConvertAlleleCounts {
 process RunAscat {
   tag {idSampleTumor + "_vs_" + idSampleNormal}
 
-  publishDir directoryMap.ascat, mode: 'link'
+  publishDir directoryMap.ascat, mode: params.publishDirMode
 
   input:
     set idPatient, idSampleNormal, idSampleTumor, file(bafNormal), file(logrNormal), file(bafTumor), file(logrTumor) from convertAlleleCountsOutput
@@ -734,7 +734,7 @@ vcfForQC = Channel.empty().mix(
 process RunBcftoolsStats {
   tag {vcf}
 
-  publishDir directoryMap.bcftoolsStats, mode: 'link'
+  publishDir directoryMap.bcftoolsStats, mode: params.publishDirMode
 
   input:
     set variantCaller, file(vcf) from vcfForBCFtools
@@ -757,7 +757,7 @@ bcfReport.close()
 process RunVcftools {
   tag {vcf}
 
-  publishDir directoryMap.vcftools, mode: 'link'
+  publishDir directoryMap.vcftools, mode: params.publishDirMode
 
   input:
     set variantCaller, file(vcf) from vcfForVCFtools
@@ -778,7 +778,7 @@ if (params.verbose) vcfReport = vcfReport.view {
 vcfReport.close()
 
 process GetVersionAlleleCount {
-  publishDir directoryMap.version, mode: 'link'
+  publishDir directoryMap.version, mode: params.publishDirMode
   output: file("v_*.txt")
   when: 'ascat' in tools && !params.onlyQC
 
@@ -789,7 +789,7 @@ process GetVersionAlleleCount {
 }
 
 process GetVersionASCAT {
-  publishDir directoryMap.version, mode: 'link'
+  publishDir directoryMap.version, mode: params.publishDirMode
   output: file("v_*.txt")
   when: 'ascat' in tools && !params.onlyQC