Merge pull request #696 from KochTobi/AWSQueueParam

AWS Batch queue parameter

Author: maxulysse

annotate.nf

@@ -42,6 +42,13 @@ if (params.help) exit 0, helpMessage()
 if (!SarekUtils.isAllowedParams(params)) exit 1, "params unknown, see --help for more information"
 if (!checkUppmaxProject()) exit 1, "No UPPMAX project ID found! Use --project <UPPMAX Project ID>"
 
+// Check for awsbatch profile configuration
+// make sure queue is defined
+if (workflow.profile == 'awsbatch') {
+    if(!params.awsqueue) exit 1, "Provide the job queue for aws batch!"
+}
+
+
 tools = params.tools ? params.tools.split(',').collect{it.trim().toLowerCase()} : []
 annotateTools = params.annotateTools ? params.annotateTools.split(',').collect{it.trim().toLowerCase()} : []
 annotateVCF = params.annotateVCF ? params.annotateVCF.split(',').collect{it.trim()} : []

buildContainers.nf

@@ -38,6 +38,12 @@ if (params.help) exit 0, helpMessage()
 if (!SarekUtils.isAllowedParams(params)) exit 1, "params unknown, see --help for more information"
 if (!checkUppmaxProject()) exit 1, "No UPPMAX project ID found! Use --project <UPPMAX Project ID>"
 
+// Check for awsbatch profile configuration
+// make sure queue is defined
+if (workflow.profile == 'awsbatch') {
+    if(!params.awsqueue) exit 1, "Provide the job queue for aws batch!"
+}
+
 // Define containers to handle (build/push or pull)
 containersList = defineContainersList()
 containers = params.containers.split(',').collect {it.trim()}

buildReferences.nf

@@ -40,6 +40,12 @@ if (params.help) exit 0, helpMessage()
 if (!SarekUtils.isAllowedParams(params)) exit 1, "params unknown, see --help for more information"
 if (!checkUppmaxProject()) exit 1, "No UPPMAX project ID found! Use --project <UPPMAX Project ID>"
 
+// Check for awsbatch profile configuration
+// make sure queue is defined
+if (workflow.profile == 'awsbatch') {
+    if(!params.awsqueue) exit 1, "Provide the job queue for aws batch!"
+}
+
 ch_referencesFiles = Channel.fromPath("${params.refDir}/*")
 
 /*

conf/aws-batch.config

@@ -12,16 +12,18 @@ params {
   publishDirMode = 'copy'
 }
 
-executor.name = 'awsbatch'
-executor.awscli = '/home/ec2-user/miniconda/bin/aws'
+executor {
+  name = 'awsbatch'
+  awscli = '/home/ec2-user/miniconda/bin/aws'
+}
 
 process {
-  executor = 'awsbatch'
-  queue = 'Sarek-queue'
+  queue = params.awsqueue
 
   errorStrategy = {task.exitStatus == 143 ? 'retry' : 'terminate'}
   maxErrors = '-1'
-  maxRetries = 2
+  maxRetries = 4
   cpus = 2
-  memory = 7.GB
+  memory = 8.GB
+
 }

conf/base.config

@@ -38,6 +38,7 @@ params {
   targetBED = false // no targets by default
   test = false // Not testing by default
   verbose = false // Enable for more verbose information
+  awsqueue = false // Queue has to be provided when using awsbatch executor
 }
 
 process {
@@ -70,4 +71,3 @@ trace { // Turning on trace tracking by default
   fields = 'process,task_id,hash,name,attempt,status,exit,realtime,%cpu,vmem,rss,submit,start,complete,duration,realtime,rchar,wchar'
   file = "${params.outDir}/Reports/Sarek_trace.txt"
 }
-

docs/PARAMETERS.md

@@ -58,6 +58,10 @@ Possible values are:
 `--tools` option is case insensitive to avoid easy introduction of errors when choosing tools.
 So you can write `--tools mutect2,ascat` or `--tools MuTect2,ASCAT` without worrying about case sensitivity.
 
+### --awsqueue `BatchQueueName`
+
+Only required if you use the awsbatch profile. This parameter specifies the queue for which jobs are submitted in AWS Batch.
+
 ### --verbose
 
 Display more information about files being processed.

germlineVC.nf

@@ -43,6 +43,12 @@ if (params.help) exit 0, helpMessage()
 if (!SarekUtils.isAllowedParams(params)) exit 1, "params unknown, see --help for more information"
 if (!checkUppmaxProject()) exit 1, "No UPPMAX project ID found! Use --project <UPPMAX Project ID>"
 
+// Check for awsbatch profile configuration
+// make sure queue is defined
+if (workflow.profile == 'awsbatch') {
+    if(!params.awsqueue) exit 1, "Provide the job queue for aws batch!"
+}
+
 tools = params.tools ? params.tools.split(',').collect{it.trim().toLowerCase()} : []
 
 directoryMap = SarekUtils.defineDirectoryMap(params.outDir)

lib/SarekUtils.groovy

@@ -35,6 +35,7 @@ class SarekUtils {
       'annotate-VCF',
       'annotateTools',
       'annotateVCF',
+      'awsqueue',
       'build',
       'call-name',
       'callName',

main.nf

@@ -40,6 +40,13 @@ kate: syntax groovy; space-indent on; indent-width 2;
 ================================================================================
 */
 
+// Check for awsbatch profile configuration
+// make sure queue is defined
+if (workflow.profile == 'awsbatch') {
+    if(!params.awsqueue) exit 1, "Provide the job queue for aws batch!"
+}
+
+
 if (params.help) exit 0, helpMessage()
 if (!SarekUtils.isAllowedParams(params)) exit 1, "params unknown, see --help for more information"
 if (!checkUppmaxProject()) exit 1, "No UPPMAX project ID found! Use --project <UPPMAX Project ID>"

runMultiQC.nf

@@ -37,6 +37,12 @@ if (params.help) exit 0, helpMessage()
 if (!SarekUtils.isAllowedParams(params)) exit 1, "params unknown, see --help for more information"
 if (!checkUppmaxProject()) exit 1, "No UPPMAX project ID found! Use --project <UPPMAX Project ID>"
 
+// Check for awsbatch profile configuration
+// make sure queue is defined
+if (workflow.profile == 'awsbatch') {
+    if(!params.awsqueue) exit 1, "Provide the job queue for aws batch!"
+}
+
 directoryMap = SarekUtils.defineDirectoryMap(params.outDir)
 /*
 ================================================================================