From 45ab4873b243acf49a678b26cd73cfef46a19296 Mon Sep 17 00:00:00 2001 From: kevinmenden Date: Wed, 17 Feb 2021 15:52:20 +0100 Subject: [PATCH 01/13] including boilerplate libraries --- .../lib/Completion.groovy | 128 +++++++++ .../lib/Headers.groovy | 43 +++ .../lib/NfcoreSchema.groovy | 219 +++++++++++++++ .../{{cookiecutter.name_noslash}}/main.nf | 250 ++---------------- 4 files changed, 413 insertions(+), 227 deletions(-) create mode 100644 nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/Completion.groovy create mode 100644 nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/Headers.groovy diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/Completion.groovy b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/Completion.groovy new file mode 100644 index 0000000000..b786ed2af3 --- /dev/null +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/Completion.groovy @@ -0,0 +1,128 @@ +/* + * Functions to be run on completion of pipeline + */ + +class Completion { + static void email(workflow, params, summary_params, projectDir, log, multiqc_report=[]) { + + // Set up the e-mail variables + def subject = "[$workflow.manifest.name] Successful: $workflow.runName" + if (!workflow.success) { + subject = "[$workflow.manifest.name] FAILED: $workflow.runName" + } + + def summary = [:] + for (group in summary_params.keySet()) { + summary << summary_params[group] + } + + def misc_fields = [:] + misc_fields['Date Started'] = workflow.start + misc_fields['Date Completed'] = workflow.complete + misc_fields['Pipeline script file path'] = workflow.scriptFile + misc_fields['Pipeline script hash ID'] = workflow.scriptId + if (workflow.repository) misc_fields['Pipeline repository Git URL'] = workflow.repository + if (workflow.commitId) misc_fields['Pipeline repository Git Commit'] = workflow.commitId + if (workflow.revision) misc_fields['Pipeline Git branch/tag'] = workflow.revision + misc_fields['Nextflow Version'] = workflow.nextflow.version + misc_fields['Nextflow Build'] = workflow.nextflow.build + misc_fields['Nextflow Compile Timestamp'] = workflow.nextflow.timestamp + + def email_fields = [:] + email_fields['version'] = workflow.manifest.version + email_fields['runName'] = workflow.runName + email_fields['success'] = workflow.success + email_fields['dateComplete'] = workflow.complete + email_fields['duration'] = workflow.duration + email_fields['exitStatus'] = workflow.exitStatus + email_fields['errorMessage'] = (workflow.errorMessage ?: 'None') + email_fields['errorReport'] = (workflow.errorReport ?: 'None') + email_fields['commandLine'] = workflow.commandLine + email_fields['projectDir'] = workflow.projectDir + email_fields['summary'] = summary << misc_fields + + // On success try attach the multiqc report + def mqc_report = null + try { + if (workflow.success && !params.skip_multiqc) { + mqc_report = multiqc_report.getVal() + if (mqc_report.getClass() == ArrayList && mqc_report.size() >= 1) { + if (mqc_report.size() > 1) { + log.warn "[$workflow.manifest.name] Found multiple reports from process 'MULTIQC', will use only one" + } + mqc_report = mqc_report[0] + } + } + } catch (all) { + log.warn "[$workflow.manifest.name] Could not attach MultiQC report to summary email" + } + + // Check if we are only sending emails on failure + def email_address = params.email + if (!params.email && params.email_on_fail && !workflow.success) { + email_address = params.email_on_fail + } + + // Render the TXT template + def engine = new groovy.text.GStringTemplateEngine() + def tf = new File("$projectDir/assets/email_template.txt") + def txt_template = engine.createTemplate(tf).make(email_fields) + def email_txt = txt_template.toString() + + // Render the HTML template + def hf = new File("$projectDir/assets/email_template.html") + def html_template = engine.createTemplate(hf).make(email_fields) + def email_html = html_template.toString() + + // Render the sendmail template + def max_multiqc_email_size = params.max_multiqc_email_size as nextflow.util.MemoryUnit + def smail_fields = [ email: email_address, subject: subject, email_txt: email_txt, email_html: email_html, projectDir: "$projectDir", mqcFile: mqc_report, mqcMaxSize: max_multiqc_email_size.toBytes()] + def sf = new File("$projectDir/assets/sendmail_template.txt") + def sendmail_template = engine.createTemplate(sf).make(smail_fields) + def sendmail_html = sendmail_template.toString() + + // Send the HTML e-mail + Map colors = Headers.log_colours(params.monochrome_logs) + if (email_address) { + try { + if (params.plaintext_email) { throw GroovyException('Send plaintext e-mail, not HTML') } + // Try to send HTML e-mail using sendmail + [ 'sendmail', '-t' ].execute() << sendmail_html + log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Sent summary e-mail to $email_address (sendmail)-" + } catch (all) { + // Catch failures and try with plaintext + def mail_cmd = [ 'mail', '-s', subject, '--content-type=text/html', email_address ] + if ( mqc_report.size() <= max_multiqc_email_size.toBytes() ) { + mail_cmd += [ '-A', mqc_report ] + } + mail_cmd.execute() << email_html + log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Sent summary e-mail to $email_address (mail)-" + } + } + + // Write summary e-mail HTML to a file + def output_d = new File("${params.outdir}/pipeline_info/") + if (!output_d.exists()) { + output_d.mkdirs() + } + def output_hf = new File(output_d, "pipeline_report.html") + output_hf.withWriter { w -> w << email_html } + def output_tf = new File(output_d, "pipeline_report.txt") + output_tf.withWriter { w -> w << email_txt } + } + + static void summary(workflow, params, log) { + Map colors = Headers.log_colours(params.monochrome_logs) + + if (workflow.success) { + if (workflow.stats.ignoredCount == 0) { + log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Pipeline completed successfully${colors.reset}-" + } else { + log.info "-${colors.purple}[$workflow.manifest.name]${colors.red} Pipeline completed successfully, but with errored process(es) ${colors.reset}-" + } + } else { + //Checks.hostname(workflow, params, log) + log.info "-${colors.purple}[$workflow.manifest.name]${colors.red} Pipeline completed with errors${colors.reset}-" + } + } +} \ No newline at end of file diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/Headers.groovy b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/Headers.groovy new file mode 100644 index 0000000000..15d1d38800 --- /dev/null +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/Headers.groovy @@ -0,0 +1,43 @@ +/* + * This file holds several functions used to render the nf-core ANSI header. + */ + +class Headers { + + private static Map log_colours(Boolean monochrome_logs) { + Map colorcodes = [:] + colorcodes['reset'] = monochrome_logs ? '' : "\033[0m" + colorcodes['dim'] = monochrome_logs ? '' : "\033[2m" + colorcodes['black'] = monochrome_logs ? '' : "\033[0;30m" + colorcodes['green'] = monochrome_logs ? '' : "\033[0;32m" + colorcodes['yellow'] = monochrome_logs ? '' : "\033[0;33m" + colorcodes['yellow_bold'] = monochrome_logs ? '' : "\033[1;93m" + colorcodes['blue'] = monochrome_logs ? '' : "\033[0;34m" + colorcodes['purple'] = monochrome_logs ? '' : "\033[0;35m" + colorcodes['cyan'] = monochrome_logs ? '' : "\033[0;36m" + colorcodes['white'] = monochrome_logs ? '' : "\033[0;37m" + colorcodes['red'] = monochrome_logs ? '' : "\033[1;91m" + return colorcodes + } + + static String dashed_line(monochrome_logs) { + Map colors = log_colours(monochrome_logs) + return "-${colors.dim}----------------------------------------------------${colors.reset}-" + } + + static String nf_core(workflow, monochrome_logs) { + Map colors = log_colours(monochrome_logs) + String.format( + """\n + ${dashed_line(monochrome_logs)} + ${colors.green},--.${colors.black}/${colors.green},-.${colors.reset} + ${colors.blue} ___ __ __ __ ___ ${colors.green}/,-._.--~\'${colors.reset} + ${colors.blue} |\\ | |__ __ / ` / \\ |__) |__ ${colors.yellow}} {${colors.reset} + ${colors.blue} | \\| | \\__, \\__/ | \\ |___ ${colors.green}\\`-._,-`-,${colors.reset} + ${colors.green}`._,._,\'${colors.reset} + ${colors.purple} ${workflow.manifest.name} v${workflow.manifest.version}${colors.reset} + ${dashed_line(monochrome_logs)} + """.stripIndent() + ) + } +} diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/NfcoreSchema.groovy b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/NfcoreSchema.groovy index 174e5c54ac..d602e69cfc 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/NfcoreSchema.groovy +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/NfcoreSchema.groovy @@ -205,4 +205,223 @@ class NfcoreSchema { return new_params } + /* + * This method tries to read a JSON params file + */ + private static LinkedHashMap params_load(String json_schema) { + def params_map = new LinkedHashMap() + try { + params_map = params_read(json_schema) + } catch (Exception e) { + println "Could not read parameters settings from JSON. $e" + params_map = new LinkedHashMap() + } + return params_map + } + + /* + Method to actually read in JSON file using Groovy. + Group (as Key), values are all parameters + - Parameter1 as Key, Description as Value + - Parameter2 as Key, Description as Value + .... + Group + - + */ + private static LinkedHashMap params_read(String json_schema) throws Exception { + def json = new File(json_schema).text + def Map json_params = (Map) new JsonSlurper().parseText(json).get('definitions') + /* Tree looks like this in nf-core schema + * definitions <- this is what the first get('definitions') gets us + group 1 + title + description + properties + parameter 1 + type + description + parameter 2 + type + description + group 2 + title + description + properties + parameter 1 + type + description + */ + def params_map = new LinkedHashMap() + json_params.each { key, val -> + def Map group = json_params."$key".properties // Gets the property object of the group + def title = json_params."$key".title + def sub_params = new LinkedHashMap() + group.each { innerkey, value -> + sub_params.put(innerkey, value) + } + params_map.put(title, sub_params) + } + return params_map + } + + /* + * Get maximum number of characters across all parameter names + */ + private static Integer params_max_chars(params_map) { + Integer max_chars = 0 + for (group in params_map.keySet()) { + def group_params = params_map.get(group) // This gets the parameters of that particular group + for (param in group_params.keySet()) { + if (param.size() > max_chars) { + max_chars = param.size() + } + } + } + return max_chars + } + + /* + * Beautify parameters for --help + */ + private static String params_help(workflow, params, json_schema, command) { + String output = "" + output += "Typical pipeline command:\n\n" + output += " ${command}\n\n" + def params_map = params_load(json_schema) + def max_chars = params_max_chars(params_map) + 1 + for (group in params_map.keySet()) { + output += group + "\n" + def group_params = params_map.get(group) // This gets the parameters of that particular group + for (param in group_params.keySet()) { + def type = "[" + group_params.get(param).type + "]" + def description = group_params.get(param).description + output += " \u001B[1m--" + param.padRight(max_chars) + "\u001B[1m" + type.padRight(10) + description + "\n" + } + output += "\n" + } + output += Headers.dashed_line(params.monochrome_logs) + output += "\n\n" + Headers.dashed_line(params.monochrome_logs) + return output + } + + /* + * Groovy Map summarising parameters/workflow options used by the pipeline + */ + private static LinkedHashMap params_summary_map(workflow, params, json_schema) { + // Get a selection of core Nextflow workflow options + def Map workflow_summary = [:] + if (workflow.revision) { + workflow_summary['revision'] = workflow.revision + } + workflow_summary['runName'] = workflow.runName + if (workflow.containerEngine) { + workflow_summary['containerEngine'] = "$workflow.containerEngine" + } + if (workflow.container) { + workflow_summary['container'] = "$workflow.container" + } + workflow_summary['launchDir'] = workflow.launchDir + workflow_summary['workDir'] = workflow.workDir + workflow_summary['projectDir'] = workflow.projectDir + workflow_summary['userName'] = workflow.userName + workflow_summary['profile'] = workflow.profile + workflow_summary['configFiles'] = workflow.configFiles.join(', ') + + // Get pipeline parameters defined in JSON Schema + def Map params_summary = [:] + def blacklist = ['hostnames'] + def params_map = params_load(json_schema) + for (group in params_map.keySet()) { + def sub_params = new LinkedHashMap() + def group_params = params_map.get(group) // This gets the parameters of that particular group + for (param in group_params.keySet()) { + if (params.containsKey(param) && !blacklist.contains(param)) { + def params_value = params.get(param) + def schema_value = group_params.get(param).default + def param_type = group_params.get(param).type + if (schema_value == null) { + if (param_type == 'boolean') { + schema_value = false + } + if (param_type == 'string') { + schema_value = '' + } + if (param_type == 'integer') { + schema_value = 0 + } + } else { + if (param_type == 'string') { + if (schema_value.contains('$projectDir') || schema_value.contains('${projectDir}')) { + def sub_string = schema_value.replace('\$projectDir','') + sub_string = sub_string.replace('\${projectDir}','') + if (params_value.contains(sub_string)) { + schema_value = params_value + } + } + if (schema_value.contains('$params.outdir') || schema_value.contains('${params.outdir}')) { + def sub_string = schema_value.replace('\$params.outdir','') + sub_string = sub_string.replace('\${params.outdir}','') + if ("${params.outdir}${sub_string}" == params_value) { + schema_value = params_value + } + } + } + } + + if (params_value != schema_value) { + sub_params.put("$param", params_value) + } + } + } + params_summary.put(group, sub_params) + } + return [ 'Core Nextflow options' : workflow_summary ] << params_summary + } + + /* + * Beautify parameters for summary and return as string + */ + private static String params_summary_log(workflow, params, json_schema) { + String output = "" + def params_map = params_summary_map(workflow, params, json_schema) + def max_chars = params_max_chars(params_map) + for (group in params_map.keySet()) { + def group_params = params_map.get(group) // This gets the parameters of that particular group + if (group_params) { + output += group + "\n" + for (param in group_params.keySet()) { + output += " \u001B[1m" + param.padRight(max_chars) + ": \u001B[1m" + group_params.get(param) + "\n" + } + output += "\n" + } + } + output += Headers.dashed_line(params.monochrome_logs) + output += "\n\n" + Headers.dashed_line(params.monochrome_logs) + return output + } + + static String params_summary_multiqc(workflow, summary) { + String summary_section = '' + for (group in summary.keySet()) { + def group_params = summary.get(group) // This gets the parameters of that particular group + if (group_params) { + summary_section += "

$group

\n" + summary_section += "
\n" + for (param in group_params.keySet()) { + summary_section += "
$param
${group_params.get(param) ?: 'N/A'}
\n" + } + summary_section += "
\n" + } + } + + String yaml_file_text = "id: '${workflow.manifest.name.replace('/','-')}-summary'\n" + yaml_file_text += "description: ' - this information is collected when the pipeline is started.'\n" + yaml_file_text += "section_name: '${workflow.manifest.name} Workflow Summary'\n" + yaml_file_text += "section_href: 'https://github.com/${workflow.manifest.name}'\n" + yaml_file_text += "plot_type: 'html'\n" + yaml_file_text += "data: |\n" + yaml_file_text += "${summary_section}" + return yaml_file_text + } + } diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf index 818e4f8fd1..8c18522b87 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf @@ -9,64 +9,29 @@ ---------------------------------------------------------------------------------------- */ -log.info nfcoreHeader() +log.info Headers.nf_core(workflow, params.monochrome_logs) -def helpMessage() { - // TODO nf-core: Add to this help message with new command line parameters - log.info""" - - Usage: - - The typical command for running the pipeline is as follows: - - nextflow run {{ cookiecutter.name }} --input '*_R{1,2}.fastq.gz' -profile docker - - Mandatory arguments: - --input [file] Path to input data (must be surrounded with quotes) - -profile [str] Configuration profile to use. Can use multiple (comma separated) - Available: conda, docker, singularity, test, awsbatch, and more - - Options: - --genome [str] Name of iGenomes reference - --single_end [bool] Specifies that the input is single-end reads - - References If not specified in the configuration file or you wish to overwrite any of the references - --fasta [file] Path to fasta reference - - Other options: - --outdir [file] The output directory where the results will be saved - --publish_dir_mode [str] Mode for publishing results in the output directory. Available: symlink, rellink, link, copy, copyNoFollow, move (Default: copy) - --email [email] Set this parameter to your e-mail address to get a summary e-mail with details of the run sent to you when the workflow exits - --email_on_fail [email] Same as --email, except only send mail if the workflow is not successful - --max_multiqc_email_size [str] Threshold size for MultiQC report to be attached in notification email. If file generated by pipeline exceeds the threshold, it will not be attached (Default: 25MB) - -name [str] Name for the pipeline run. If not specified, Nextflow will automatically generate a random mnemonic - - AWSBatch options: - --awsqueue [str] The AWSBatch JobQueue that needs to be set when running on AWSBatch - --awsregion [str] The AWS Region for your AWS Batch job to run on - --awscli [str] Path to the AWS CLI tool - """.stripIndent() -} - -// Show help message +//////////////////////////////////////////////////// +/* -- PRINT HELP -- */ +////////////////////////////////////////////////////+ +def json_schema = "$baseDir/nextflow_schema.json" if (params.help) { - helpMessage() + def command = "nextflow run {{ cookiecutter.name }} --input '*_R{1,2}.fastq.gz' -profile docker" + log.info NfcoreSchema.params_help(workflow, params, json_schema, command) exit 0 } //////////////////////////////////////////////////// /* -- VALIDATE PARAMETERS -- */ ////////////////////////////////////////////////////+ -def json_schema = "$baseDir/nextflow_schema.json" def unexpectedParams = [] if (params.validate_params) { unexpectedParams = NfcoreSchema.validateParameters(params, json_schema, log) } -//////////////////////////////////////////////////// -/* - * SET UP CONFIGURATION VARIABLES - */ +//////////////////////////////////////////////////// +/* -- Collect configuration parameters -- */ +//////////////////////////////////////////////////// // Check if genome exists in the config file if (params.genomes && params.genome && !params.genomes.containsKey(params.genome)) { @@ -125,57 +90,18 @@ if (params.input_paths) { .into { ch_read_files_fastqc; ch_read_files_trimming } } -// Header log info -def summary = [:] -if (workflow.revision) summary['Pipeline Release'] = workflow.revision -summary['Run Name'] = workflow.runName -// TODO nf-core: Report custom parameters here -summary['Input'] = params.input -summary['Fasta Ref'] = params.fasta -summary['Data Type'] = params.single_end ? 'Single-End' : 'Paired-End' -summary['Max Resources'] = "$params.max_memory memory, $params.max_cpus cpus, $params.max_time time per job" -if (workflow.containerEngine) summary['Container'] = "$workflow.containerEngine - $workflow.container" -summary['Output dir'] = params.outdir -summary['Launch dir'] = workflow.launchDir -summary['Working dir'] = workflow.workDir -summary['Script dir'] = workflow.projectDir -summary['User'] = workflow.userName -if (workflow.profile.contains('awsbatch')) { - summary['AWS Region'] = params.awsregion - summary['AWS Queue'] = params.awsqueue - summary['AWS CLI'] = params.awscli -} -summary['Config Profile'] = workflow.profile -if (params.config_profile_description) summary['Config Profile Description'] = params.config_profile_description -if (params.config_profile_contact) summary['Config Profile Contact'] = params.config_profile_contact -if (params.config_profile_url) summary['Config Profile URL'] = params.config_profile_url -summary['Config Files'] = workflow.configFiles.join(', ') -if (params.email || params.email_on_fail) { - summary['E-mail Address'] = params.email - summary['E-mail on failure'] = params.email_on_fail - summary['MultiQC maxsize'] = params.max_multiqc_email_size -} -log.info summary.collect { k,v -> "${k.padRight(18)}: $v" }.join("\n") -log.info "-\033[2m--------------------------------------------------\033[0m-" + +//////////////////////////////////////////////////// +/* -- PRINT PARAMETER SUMMARY -- */ +//////////////////////////////////////////////////// + +def summary_params = NfcoreSchema.params_summary_map(workflow, params, json_schema) +log.info NfcoreSchema.params_summary_log(workflow, params, json_schema) + // Check the hostnames against configured profiles checkHostname() -Channel.from(summary.collect{ [it.key, it.value] }) - .map { k,v -> "
$k
${v ?: 'N/A'}
" } - .reduce { a, b -> return [a, b].join("\n ") } - .map { x -> """ - id: '{{ cookiecutter.name_noslash }}-summary' - description: " - this information is collected when the pipeline is started." - section_name: '{{ cookiecutter.name }} Workflow Summary' - section_href: 'https://github.com/{{ cookiecutter.name }}' - plot_type: 'html' - data: | -
- $x -
- """.stripIndent() } - .set { ch_workflow_summary } /* * Parse software version numbers @@ -225,6 +151,9 @@ process fastqc { """ } +workflow_summary = NfcoreSchema.params_summary_multiqc(workflow, summary_params) +ch_workflow_summary = Channel.value(workflow_summary) + /* * STEP 2 - MultiQC */ @@ -281,119 +210,8 @@ process output_documentation { * Completion e-mail notification */ workflow.onComplete { - - // Set up the e-mail variables - def subject = "[{{ cookiecutter.name }}] Successful: $workflow.runName" - if (!workflow.success) { - subject = "[{{ cookiecutter.name }}] FAILED: $workflow.runName" - } - def email_fields = [:] - email_fields['version'] = workflow.manifest.version - email_fields['runName'] = workflow.runName - email_fields['success'] = workflow.success - email_fields['dateComplete'] = workflow.complete - email_fields['duration'] = workflow.duration - email_fields['exitStatus'] = workflow.exitStatus - email_fields['errorMessage'] = (workflow.errorMessage ?: 'None') - email_fields['errorReport'] = (workflow.errorReport ?: 'None') - email_fields['commandLine'] = workflow.commandLine - email_fields['projectDir'] = workflow.projectDir - email_fields['summary'] = summary - email_fields['summary']['Date Started'] = workflow.start - email_fields['summary']['Date Completed'] = workflow.complete - email_fields['summary']['Pipeline script file path'] = workflow.scriptFile - email_fields['summary']['Pipeline script hash ID'] = workflow.scriptId - if (workflow.repository) email_fields['summary']['Pipeline repository Git URL'] = workflow.repository - if (workflow.commitId) email_fields['summary']['Pipeline repository Git Commit'] = workflow.commitId - if (workflow.revision) email_fields['summary']['Pipeline Git branch/tag'] = workflow.revision - email_fields['summary']['Nextflow Version'] = workflow.nextflow.version - email_fields['summary']['Nextflow Build'] = workflow.nextflow.build - email_fields['summary']['Nextflow Compile Timestamp'] = workflow.nextflow.timestamp - - // TODO nf-core: If not using MultiQC, strip out this code (including params.max_multiqc_email_size) - // On success try attach the multiqc report - def mqc_report = null - try { - if (workflow.success) { - mqc_report = ch_multiqc_report.getVal() - if (mqc_report.getClass() == ArrayList) { - log.warn "[{{ cookiecutter.name }}] Found multiple reports from process 'multiqc', will use only one" - mqc_report = mqc_report[0] - } - } - } catch (all) { - log.warn "[{{ cookiecutter.name }}] Could not attach MultiQC report to summary email" - } - - // Check if we are only sending emails on failure - email_address = params.email - if (!params.email && params.email_on_fail && !workflow.success) { - email_address = params.email_on_fail - } - - // Render the TXT template - def engine = new groovy.text.GStringTemplateEngine() - def tf = new File("$projectDir/assets/email_template.txt") - def txt_template = engine.createTemplate(tf).make(email_fields) - def email_txt = txt_template.toString() - - // Render the HTML template - def hf = new File("$projectDir/assets/email_template.html") - def html_template = engine.createTemplate(hf).make(email_fields) - def email_html = html_template.toString() - - // Render the sendmail template - def smail_fields = [ email: email_address, subject: subject, email_txt: email_txt, email_html: email_html, projectDir: "$projectDir", mqcFile: mqc_report, mqcMaxSize: params.max_multiqc_email_size.toBytes() ] - def sf = new File("$projectDir/assets/sendmail_template.txt") - def sendmail_template = engine.createTemplate(sf).make(smail_fields) - def sendmail_html = sendmail_template.toString() - - // Send the HTML e-mail - if (email_address) { - try { - if (params.plaintext_email) { throw GroovyException('Send plaintext e-mail, not HTML') } - // Try to send HTML e-mail using sendmail - [ 'sendmail', '-t' ].execute() << sendmail_html - log.info "[{{ cookiecutter.name }}] Sent summary e-mail to $email_address (sendmail)" - } catch (all) { - // Catch failures and try with plaintext - def mail_cmd = [ 'mail', '-s', subject, '--content-type=text/html', email_address ] - if ( mqc_report.size() <= params.max_multiqc_email_size.toBytes() ) { - mail_cmd += [ '-A', mqc_report ] - } - mail_cmd.execute() << email_html - log.info "[{{ cookiecutter.name }}] Sent summary e-mail to $email_address (mail)" - } - } - - // Write summary e-mail HTML to a file - def output_d = new File("${params.outdir}/pipeline_info/") - if (!output_d.exists()) { - output_d.mkdirs() - } - def output_hf = new File(output_d, "pipeline_report.html") - output_hf.withWriter { w -> w << email_html } - def output_tf = new File(output_d, "pipeline_report.txt") - output_tf.withWriter { w -> w << email_txt } - - c_green = params.monochrome_logs ? '' : "\033[0;32m"; - c_purple = params.monochrome_logs ? '' : "\033[0;35m"; - c_red = params.monochrome_logs ? '' : "\033[0;31m"; - c_reset = params.monochrome_logs ? '' : "\033[0m"; - - if (workflow.stats.ignoredCount > 0 && workflow.success) { - log.info "-${c_purple}Warning, pipeline completed, but with errored process(es) ${c_reset}-" - log.info "-${c_red}Number of ignored errored process(es) : ${workflow.stats.ignoredCount} ${c_reset}-" - log.info "-${c_green}Number of successfully ran process(es) : ${workflow.stats.succeedCount} ${c_reset}-" - } - - if (workflow.success) { - log.info "-${c_purple}[{{ cookiecutter.name }}]${c_green} Pipeline completed successfully${c_reset}-" - } else { - checkHostname() - log.info "-${c_purple}[{{ cookiecutter.name }}]${c_red} Pipeline completed with errors${c_reset}-" - } - + Completion.email(workflow, params, summary_params, projectDir, log, multiqc_report) + Completion.summary(workflow, params, log) } workflow.onError { @@ -403,28 +221,6 @@ workflow.onError { } } -def nfcoreHeader() { - // Log colors ANSI codes - c_black = params.monochrome_logs ? '' : "\033[0;30m"; - c_blue = params.monochrome_logs ? '' : "\033[0;34m"; - c_cyan = params.monochrome_logs ? '' : "\033[0;36m"; - c_dim = params.monochrome_logs ? '' : "\033[2m"; - c_green = params.monochrome_logs ? '' : "\033[0;32m"; - c_purple = params.monochrome_logs ? '' : "\033[0;35m"; - c_reset = params.monochrome_logs ? '' : "\033[0m"; - c_white = params.monochrome_logs ? '' : "\033[0;37m"; - c_yellow = params.monochrome_logs ? '' : "\033[0;33m"; - - return """ -${c_dim}--------------------------------------------------${c_reset}- - ${c_green},--.${c_black}/${c_green},-.${c_reset} - ${c_blue} ___ __ __ __ ___ ${c_green}/,-._.--~\'${c_reset} - ${c_blue} |\\ | |__ __ / ` / \\ |__) |__ ${c_yellow}} {${c_reset} - ${c_blue} | \\| | \\__, \\__/ | \\ |___ ${c_green}\\`-._,-`-,${c_reset} - ${c_green}`._,._,\'${c_reset} - ${c_purple} {{ cookiecutter.name }} v${workflow.manifest.version}${c_reset} - -${c_dim}--------------------------------------------------${c_reset}- - """.stripIndent() -} def checkHostname() { def c_reset = params.monochrome_logs ? '' : "\033[0m" From 2fd2dba2d7f77c863f2de872c1b4a7fe579a4aa4 Mon Sep 17 00:00:00 2001 From: kevinmenden Date: Wed, 17 Feb 2021 16:57:32 +0100 Subject: [PATCH 02/13] made schema independent from headers --- .../lib/NfcoreSchema.groovy | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/NfcoreSchema.groovy b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/NfcoreSchema.groovy index d602e69cfc..bb28012fe9 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/NfcoreSchema.groovy +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/NfcoreSchema.groovy @@ -219,6 +219,11 @@ class NfcoreSchema { return params_map } + static String dashed_line(monochrome_logs) { + Map colors = log_colours(monochrome_logs) + return "-${colors.dim}----------------------------------------------------${colors.reset}-" + } + /* Method to actually read in JSON file using Groovy. Group (as Key), values are all parameters @@ -299,8 +304,8 @@ class NfcoreSchema { } output += "\n" } - output += Headers.dashed_line(params.monochrome_logs) - output += "\n\n" + Headers.dashed_line(params.monochrome_logs) + output += dashed_line(params.monochrome_logs) + output += "\n\n" + dashed_line(params.monochrome_logs) return output } @@ -395,8 +400,8 @@ class NfcoreSchema { output += "\n" } } - output += Headers.dashed_line(params.monochrome_logs) - output += "\n\n" + Headers.dashed_line(params.monochrome_logs) + output += dashed_line(params.monochrome_logs) + output += "\n\n" + dashed_line(params.monochrome_logs) return output } From dbda27efb73327d7b0cf58f959d0d09cc2414946 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Thu, 18 Feb 2021 09:34:28 +0100 Subject: [PATCH 03/13] Update nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/NfcoreSchema.groovy Co-authored-by: Phil Ewels --- .../{{cookiecutter.name_noslash}}/lib/NfcoreSchema.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/NfcoreSchema.groovy b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/NfcoreSchema.groovy index bb28012fe9..0f4b3f96a6 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/NfcoreSchema.groovy +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/NfcoreSchema.groovy @@ -205,7 +205,7 @@ class NfcoreSchema { return new_params } - /* + /* * This method tries to read a JSON params file */ private static LinkedHashMap params_load(String json_schema) { From f2bf623e134596fa3d35a151604f49cb4349bfc5 Mon Sep 17 00:00:00 2001 From: kevinmenden Date: Thu, 18 Feb 2021 12:21:46 +0100 Subject: [PATCH 04/13] removed completion --- .../lib/Completion.groovy | 128 ----------- .../lib/NfcoreSchema.groovy | 75 ++++--- .../{{cookiecutter.name_noslash}}/main.nf | 205 +++++++++++++++--- 3 files changed, 223 insertions(+), 185 deletions(-) delete mode 100644 nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/Completion.groovy diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/Completion.groovy b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/Completion.groovy deleted file mode 100644 index b786ed2af3..0000000000 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/Completion.groovy +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Functions to be run on completion of pipeline - */ - -class Completion { - static void email(workflow, params, summary_params, projectDir, log, multiqc_report=[]) { - - // Set up the e-mail variables - def subject = "[$workflow.manifest.name] Successful: $workflow.runName" - if (!workflow.success) { - subject = "[$workflow.manifest.name] FAILED: $workflow.runName" - } - - def summary = [:] - for (group in summary_params.keySet()) { - summary << summary_params[group] - } - - def misc_fields = [:] - misc_fields['Date Started'] = workflow.start - misc_fields['Date Completed'] = workflow.complete - misc_fields['Pipeline script file path'] = workflow.scriptFile - misc_fields['Pipeline script hash ID'] = workflow.scriptId - if (workflow.repository) misc_fields['Pipeline repository Git URL'] = workflow.repository - if (workflow.commitId) misc_fields['Pipeline repository Git Commit'] = workflow.commitId - if (workflow.revision) misc_fields['Pipeline Git branch/tag'] = workflow.revision - misc_fields['Nextflow Version'] = workflow.nextflow.version - misc_fields['Nextflow Build'] = workflow.nextflow.build - misc_fields['Nextflow Compile Timestamp'] = workflow.nextflow.timestamp - - def email_fields = [:] - email_fields['version'] = workflow.manifest.version - email_fields['runName'] = workflow.runName - email_fields['success'] = workflow.success - email_fields['dateComplete'] = workflow.complete - email_fields['duration'] = workflow.duration - email_fields['exitStatus'] = workflow.exitStatus - email_fields['errorMessage'] = (workflow.errorMessage ?: 'None') - email_fields['errorReport'] = (workflow.errorReport ?: 'None') - email_fields['commandLine'] = workflow.commandLine - email_fields['projectDir'] = workflow.projectDir - email_fields['summary'] = summary << misc_fields - - // On success try attach the multiqc report - def mqc_report = null - try { - if (workflow.success && !params.skip_multiqc) { - mqc_report = multiqc_report.getVal() - if (mqc_report.getClass() == ArrayList && mqc_report.size() >= 1) { - if (mqc_report.size() > 1) { - log.warn "[$workflow.manifest.name] Found multiple reports from process 'MULTIQC', will use only one" - } - mqc_report = mqc_report[0] - } - } - } catch (all) { - log.warn "[$workflow.manifest.name] Could not attach MultiQC report to summary email" - } - - // Check if we are only sending emails on failure - def email_address = params.email - if (!params.email && params.email_on_fail && !workflow.success) { - email_address = params.email_on_fail - } - - // Render the TXT template - def engine = new groovy.text.GStringTemplateEngine() - def tf = new File("$projectDir/assets/email_template.txt") - def txt_template = engine.createTemplate(tf).make(email_fields) - def email_txt = txt_template.toString() - - // Render the HTML template - def hf = new File("$projectDir/assets/email_template.html") - def html_template = engine.createTemplate(hf).make(email_fields) - def email_html = html_template.toString() - - // Render the sendmail template - def max_multiqc_email_size = params.max_multiqc_email_size as nextflow.util.MemoryUnit - def smail_fields = [ email: email_address, subject: subject, email_txt: email_txt, email_html: email_html, projectDir: "$projectDir", mqcFile: mqc_report, mqcMaxSize: max_multiqc_email_size.toBytes()] - def sf = new File("$projectDir/assets/sendmail_template.txt") - def sendmail_template = engine.createTemplate(sf).make(smail_fields) - def sendmail_html = sendmail_template.toString() - - // Send the HTML e-mail - Map colors = Headers.log_colours(params.monochrome_logs) - if (email_address) { - try { - if (params.plaintext_email) { throw GroovyException('Send plaintext e-mail, not HTML') } - // Try to send HTML e-mail using sendmail - [ 'sendmail', '-t' ].execute() << sendmail_html - log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Sent summary e-mail to $email_address (sendmail)-" - } catch (all) { - // Catch failures and try with plaintext - def mail_cmd = [ 'mail', '-s', subject, '--content-type=text/html', email_address ] - if ( mqc_report.size() <= max_multiqc_email_size.toBytes() ) { - mail_cmd += [ '-A', mqc_report ] - } - mail_cmd.execute() << email_html - log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Sent summary e-mail to $email_address (mail)-" - } - } - - // Write summary e-mail HTML to a file - def output_d = new File("${params.outdir}/pipeline_info/") - if (!output_d.exists()) { - output_d.mkdirs() - } - def output_hf = new File(output_d, "pipeline_report.html") - output_hf.withWriter { w -> w << email_html } - def output_tf = new File(output_d, "pipeline_report.txt") - output_tf.withWriter { w -> w << email_txt } - } - - static void summary(workflow, params, log) { - Map colors = Headers.log_colours(params.monochrome_logs) - - if (workflow.success) { - if (workflow.stats.ignoredCount == 0) { - log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Pipeline completed successfully${colors.reset}-" - } else { - log.info "-${colors.purple}[$workflow.manifest.name]${colors.red} Pipeline completed successfully, but with errored process(es) ${colors.reset}-" - } - } else { - //Checks.hostname(workflow, params, log) - log.info "-${colors.purple}[$workflow.manifest.name]${colors.red} Pipeline completed with errors${colors.reset}-" - } - } -} \ No newline at end of file diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/NfcoreSchema.groovy b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/NfcoreSchema.groovy index bb28012fe9..390fef5d92 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/NfcoreSchema.groovy +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/NfcoreSchema.groovy @@ -12,6 +12,7 @@ import groovy.json.JsonSlurper import groovy.json.JsonBuilder class NfcoreSchema { + /* * Function to loop over all parameters defined in schema and check * whether the given paremeters adhere to the specificiations @@ -134,23 +135,23 @@ class NfcoreSchema { try { schema.validate(paramsJSON) } catch (ValidationException e) { - println "" + println '' log.error 'ERROR: Validation of pipeline parameters failed!' JSONObject exceptionJSON = e.toJSON() printExceptions(exceptionJSON, paramsJSON, log) - if (unexpectedParams.size() > 0){ - println "" + if (unexpectedParams.size() > 0) { + println '' def warn_msg = 'Found unexpected parameters:' - for (unexpectedParam in unexpectedParams){ + for (unexpectedParam in unexpectedParams) { warn_msg = warn_msg + "\n* --${unexpectedParam}: ${paramsJSON[unexpectedParam].toString()}" } log.warn warn_msg } - println "" + println '' has_error = true } - if(has_error){ + if (has_error) { System.exit(1) } @@ -163,11 +164,11 @@ class NfcoreSchema { if (causingExceptions.length() == 0) { def m = exJSON['message'] =~ /required key \[([^\]]+)\] not found/ // Missing required param - if(m.matches()){ + if (m.matches()) { log.error "* Missing required parameter: --${m[0][1]}" } // Other base-level error - else if(exJSON['pointerToViolation'] == '#'){ + else if (exJSON['pointerToViolation'] == '#') { log.error "* ${exJSON['message']}" } // Error with specific param @@ -219,6 +220,22 @@ class NfcoreSchema { return params_map } + private static Map log_colours(Boolean monochrome_logs) { + Map colorcodes = [:] + colorcodes['reset'] = monochrome_logs ? '' : "\033[0m" + colorcodes['dim'] = monochrome_logs ? '' : "\033[2m" + colorcodes['black'] = monochrome_logs ? '' : "\033[0;30m" + colorcodes['green'] = monochrome_logs ? '' : "\033[0;32m" + colorcodes['yellow'] = monochrome_logs ? '' : "\033[0;33m" + colorcodes['yellow_bold'] = monochrome_logs ? '' : "\033[1;93m" + colorcodes['blue'] = monochrome_logs ? '' : "\033[0;34m" + colorcodes['purple'] = monochrome_logs ? '' : "\033[0;35m" + colorcodes['cyan'] = monochrome_logs ? '' : "\033[0;36m" + colorcodes['white'] = monochrome_logs ? '' : "\033[0;37m" + colorcodes['red'] = monochrome_logs ? '' : "\033[1;91m" + return colorcodes + } + static String dashed_line(monochrome_logs) { Map colors = log_colours(monochrome_logs) return "-${colors.dim}----------------------------------------------------${colors.reset}-" @@ -289,23 +306,23 @@ class NfcoreSchema { * Beautify parameters for --help */ private static String params_help(workflow, params, json_schema, command) { - String output = "" - output += "Typical pipeline command:\n\n" + String output = '' + output += 'Typical pipeline command:\n\n' output += " ${command}\n\n" def params_map = params_load(json_schema) def max_chars = params_max_chars(params_map) + 1 for (group in params_map.keySet()) { - output += group + "\n" + output += group + '\n' def group_params = params_map.get(group) // This gets the parameters of that particular group for (param in group_params.keySet()) { - def type = "[" + group_params.get(param).type + "]" + def type = '[' + group_params.get(param).type + ']' def description = group_params.get(param).description - output += " \u001B[1m--" + param.padRight(max_chars) + "\u001B[1m" + type.padRight(10) + description + "\n" + output += " \u001B[1m--" + param.padRight(max_chars) + "\u001B[1m" + type.padRight(10) + description + '\n' } - output += "\n" + output += '\n' } output += dashed_line(params.monochrome_logs) - output += "\n\n" + dashed_line(params.monochrome_logs) + output += '\n\n' + dashed_line(params.monochrome_logs) return output } @@ -314,7 +331,7 @@ class NfcoreSchema { */ private static LinkedHashMap params_summary_map(workflow, params, json_schema) { // Get a selection of core Nextflow workflow options - def Map workflow_summary = [:] + def Map workflow_summary = [:] if (workflow.revision) { workflow_summary['revision'] = workflow.revision } @@ -331,7 +348,7 @@ class NfcoreSchema { workflow_summary['userName'] = workflow.userName workflow_summary['profile'] = workflow.profile workflow_summary['configFiles'] = workflow.configFiles.join(', ') - + // Get pipeline parameters defined in JSON Schema def Map params_summary = [:] def blacklist = ['hostnames'] @@ -357,15 +374,15 @@ class NfcoreSchema { } else { if (param_type == 'string') { if (schema_value.contains('$projectDir') || schema_value.contains('${projectDir}')) { - def sub_string = schema_value.replace('\$projectDir','') - sub_string = sub_string.replace('\${projectDir}','') + def sub_string = schema_value.replace('\$projectDir', '') + sub_string = sub_string.replace('\${projectDir}', '') if (params_value.contains(sub_string)) { schema_value = params_value } } if (schema_value.contains('$params.outdir') || schema_value.contains('${params.outdir}')) { - def sub_string = schema_value.replace('\$params.outdir','') - sub_string = sub_string.replace('\${params.outdir}','') + def sub_string = schema_value.replace('\$params.outdir', '') + sub_string = sub_string.replace('\${params.outdir}', '') if ("${params.outdir}${sub_string}" == params_value) { schema_value = params_value } @@ -387,21 +404,21 @@ class NfcoreSchema { * Beautify parameters for summary and return as string */ private static String params_summary_log(workflow, params, json_schema) { - String output = "" + String output = '' def params_map = params_summary_map(workflow, params, json_schema) def max_chars = params_max_chars(params_map) for (group in params_map.keySet()) { def group_params = params_map.get(group) // This gets the parameters of that particular group if (group_params) { - output += group + "\n" + output += group + '\n' for (param in group_params.keySet()) { - output += " \u001B[1m" + param.padRight(max_chars) + ": \u001B[1m" + group_params.get(param) + "\n" + output += " \u001B[1m" + param.padRight(max_chars) + ": \u001B[1m" + group_params.get(param) + '\n' } - output += "\n" + output += '\n' } } output += dashed_line(params.monochrome_logs) - output += "\n\n" + dashed_line(params.monochrome_logs) + output += '\n\n' + dashed_line(params.monochrome_logs) return output } @@ -415,16 +432,16 @@ class NfcoreSchema { for (param in group_params.keySet()) { summary_section += "
$param
${group_params.get(param) ?: 'N/A'}
\n" } - summary_section += " \n" + summary_section += ' \n' } } - String yaml_file_text = "id: '${workflow.manifest.name.replace('/','-')}-summary'\n" + String yaml_file_text = "id: '${workflow.manifest.name.replace('/', '-')}-summary'\n" yaml_file_text += "description: ' - this information is collected when the pipeline is started.'\n" yaml_file_text += "section_name: '${workflow.manifest.name} Workflow Summary'\n" yaml_file_text += "section_href: 'https://github.com/${workflow.manifest.name}'\n" yaml_file_text += "plot_type: 'html'\n" - yaml_file_text += "data: |\n" + yaml_file_text += 'data: |\n' yaml_file_text += "${summary_section}" return yaml_file_text } diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf index 8c18522b87..b309879e8d 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf @@ -35,7 +35,7 @@ if (params.validate_params) { // Check if genome exists in the config file if (params.genomes && params.genome && !params.genomes.containsKey(params.genome)) { - exit 1, "The provided genome '${params.genome}' is not available in the iGenomes file. Currently the available genomes are ${params.genomes.keySet().join(", ")}" + exit 1, "The provided genome '${params.genome}' is not available in the iGenomes file. Currently the available genomes are ${params.genomes.keySet().join(', ')}" } // TODO nf-core: Add any reference files that are needed @@ -52,12 +52,12 @@ if (params.fasta) { ch_fasta = file(params.fasta, checkIfExists: true) } // Check AWS batch settings if (workflow.profile.contains('awsbatch')) { // AWSBatch sanity checking - if (!params.awsqueue || !params.awsregion) exit 1, "Specify correct --awsqueue and --awsregion parameters on AWSBatch!" + if (!params.awsqueue || !params.awsregion) exit 1, 'Specify correct --awsqueue and --awsregion parameters on AWSBatch!' // Check outdir paths to be S3 buckets if running on AWSBatch // related: https://github.com/nextflow-io/nextflow/issues/813 - if (!params.outdir.startsWith('s3:')) exit 1, "Outdir not on S3 - specify S3 Bucket to run on AWSBatch!" + if (!params.outdir.startsWith('s3:')) exit 1, 'Outdir not on S3 - specify S3 Bucket to run on AWSBatch!' // Prevent trace files to be stored on S3 since S3 does not support rolling files. - if (params.tracedir.startsWith('s3:')) exit 1, "Specify a local tracedir or run without trace! S3 cannot be used for tracefiles." + if (params.tracedir.startsWith('s3:')) exit 1, 'Specify a local tracedir or run without trace! S3 cannot be used for tracefiles.' } // Stage config files @@ -74,13 +74,13 @@ if (params.input_paths) { Channel .from(params.input_paths) .map { row -> [ row[0], [ file(row[1][0], checkIfExists: true) ] ] } - .ifEmpty { exit 1, "params.input_paths was empty - no input files supplied" } + .ifEmpty { exit 1, 'params.input_paths was empty - no input files supplied' } .into { ch_read_files_fastqc; ch_read_files_trimming } } else { Channel .from(params.input_paths) .map { row -> [ row[0], [ file(row[1][0], checkIfExists: true), file(row[1][1], checkIfExists: true) ] ] } - .ifEmpty { exit 1, "params.input_paths was empty - no input files supplied" } + .ifEmpty { exit 1, 'params.input_paths was empty - no input files supplied' } .into { ch_read_files_fastqc; ch_read_files_trimming } } } else { @@ -90,7 +90,6 @@ if (params.input_paths) { .into { ch_read_files_fastqc; ch_read_files_trimming } } - //////////////////////////////////////////////////// /* -- PRINT PARAMETER SUMMARY -- */ //////////////////////////////////////////////////// @@ -98,10 +97,55 @@ if (params.input_paths) { def summary_params = NfcoreSchema.params_summary_map(workflow, params, json_schema) log.info NfcoreSchema.params_summary_log(workflow, params, json_schema) +// Header log info +def summary = [:] +if (workflow.revision) summary['Pipeline Release'] = workflow.revision +summary['Run Name'] = workflow.runName +// TODO nf-core: Report custom parameters here +summary['Input'] = params.input +summary['Fasta Ref'] = params.fasta +summary['Data Type'] = params.single_end ? 'Single-End' : 'Paired-End' +summary['Max Resources'] = "$params.max_memory memory, $params.max_cpus cpus, $params.max_time time per job" +if (workflow.containerEngine) summary['Container'] = "$workflow.containerEngine - $workflow.container" +summary['Output dir'] = params.outdir +summary['Launch dir'] = workflow.launchDir +summary['Working dir'] = workflow.workDir +summary['Script dir'] = workflow.projectDir +summary['User'] = workflow.userName +if (workflow.profile.contains('awsbatch')) { + summary['AWS Region'] = params.awsregion + summary['AWS Queue'] = params.awsqueue + summary['AWS CLI'] = params.awscli +} +summary['Config Profile'] = workflow.profile +if (params.config_profile_description) summary['Config Profile Description'] = params.config_profile_description +if (params.config_profile_contact) summary['Config Profile Contact'] = params.config_profile_contact +if (params.config_profile_url) summary['Config Profile URL'] = params.config_profile_url +summary['Config Files'] = workflow.configFiles.join(', ') +if (params.email || params.email_on_fail) { + summary['E-mail Address'] = params.email + summary['E-mail on failure'] = params.email_on_fail + summary['MultiQC maxsize'] = params.max_multiqc_email_size +} // Check the hostnames against configured profiles checkHostname() +Channel.from(summary.collect { [it.key, it.value] }) + .map { k, v -> "
$k
${v ?: 'N/A'}
" } + .reduce { a, b -> return [a, b].join('\n ') } + .map { x -> ''' + id: '{{ cookiecutter.name_noslash }}-summary' + description: " - this information is collected when the pipeline is started." + section_name: '{{ cookiecutter.name }} Workflow Summary' + section_href: 'https://github.com/{{ cookiecutter.name }}' + plot_type: 'html' + data: | +
+ $x +
+ '''.stripIndent() } + .set { ch_workflow_summary } /* * Parse software version numbers @@ -109,13 +153,13 @@ checkHostname() process get_software_versions { publishDir "${params.outdir}/pipeline_info", mode: params.publish_dir_mode, saveAs: { filename -> - if (filename.indexOf(".csv") > 0) filename + if (filename.indexOf('.csv') > 0) filename else null - } + } output: file 'software_versions_mqc.yaml' into ch_software_versions_yaml - file "software_versions.csv" + file 'software_versions.csv' script: // TODO nf-core: Get all tools to print their version number here @@ -136,14 +180,14 @@ process fastqc { label 'process_medium' publishDir "${params.outdir}/fastqc", mode: params.publish_dir_mode, saveAs: { filename -> - filename.indexOf(".zip") > 0 ? "zips/$filename" : "$filename" - } + filename.indexOf('.zip') > 0 ? "zips/$filename" : "$filename" + } input: set val(name), file(reads) from ch_read_files_fastqc output: - file "*_fastqc.{zip,html}" into ch_fastqc_results + file '*_fastqc.{zip,html}' into ch_fastqc_results script: """ @@ -151,9 +195,6 @@ process fastqc { """ } -workflow_summary = NfcoreSchema.params_summary_multiqc(workflow, summary_params) -ch_workflow_summary = Channel.value(workflow_summary) - /* * STEP 2 - MultiQC */ @@ -166,19 +207,19 @@ process multiqc { // TODO nf-core: Add in log files from your new processes for MultiQC to find! file ('fastqc/*') from ch_fastqc_results.collect().ifEmpty([]) file ('software_versions/*') from ch_software_versions_yaml.collect() - file workflow_summary from ch_workflow_summary.collectFile(name: "workflow_summary_mqc.yaml") + file workflow_summary from ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml') output: - file "*multiqc_report.html" into ch_multiqc_report - file "*_data" - file "multiqc_plots" + file '*multiqc_report.html' into ch_multiqc_report + file '*_data' + file 'multiqc_plots' script: rtitle = '' rfilename = '' if (!(workflow.runName ==~ /[a-z]+_[a-z]+/)) { rtitle = "--title \"${workflow.runName}\"" - rfilename = "--filename " + workflow.runName.replaceAll('\\W','_').replaceAll('_+','_') + "_multiqc_report" + rfilename = '--filename ' + workflow.runName.replaceAll('\\W', '_').replaceAll('_+', '_') + '_multiqc_report' } custom_config_file = params.multiqc_config ? "--config $mqc_custom_config" : '' // TODO nf-core: Specify which MultiQC modules to use with -m for a faster run time @@ -198,7 +239,7 @@ process output_documentation { file images from ch_output_docs_images output: - file "results_description.html" + file 'results_description.html' script: """ @@ -210,8 +251,117 @@ process output_documentation { * Completion e-mail notification */ workflow.onComplete { - Completion.email(workflow, params, summary_params, projectDir, log, multiqc_report) - Completion.summary(workflow, params, log) + // Set up the e-mail variables + def subject = "[{{ cookiecutter.name }}] Successful: $workflow.runName" + if (!workflow.success) { + subject = "[{{ cookiecutter.name }}] FAILED: $workflow.runName" + } + def email_fields = [:] + email_fields['version'] = workflow.manifest.version + email_fields['runName'] = workflow.runName + email_fields['success'] = workflow.success + email_fields['dateComplete'] = workflow.complete + email_fields['duration'] = workflow.duration + email_fields['exitStatus'] = workflow.exitStatus + email_fields['errorMessage'] = (workflow.errorMessage ?: 'None') + email_fields['errorReport'] = (workflow.errorReport ?: 'None') + email_fields['commandLine'] = workflow.commandLine + email_fields['projectDir'] = workflow.projectDir + email_fields['summary'] = summary + email_fields['summary']['Date Started'] = workflow.start + email_fields['summary']['Date Completed'] = workflow.complete + email_fields['summary']['Pipeline script file path'] = workflow.scriptFile + email_fields['summary']['Pipeline script hash ID'] = workflow.scriptId + if (workflow.repository) email_fields['summary']['Pipeline repository Git URL'] = workflow.repository + if (workflow.commitId) email_fields['summary']['Pipeline repository Git Commit'] = workflow.commitId + if (workflow.revision) email_fields['summary']['Pipeline Git branch/tag'] = workflow.revision + email_fields['summary']['Nextflow Version'] = workflow.nextflow.version + email_fields['summary']['Nextflow Build'] = workflow.nextflow.build + email_fields['summary']['Nextflow Compile Timestamp'] = workflow.nextflow.timestamp + + // TODO nf-core: If not using MultiQC, strip out this code (including params.max_multiqc_email_size) + // On success try attach the multiqc report + def mqc_report = null + try { + if (workflow.success) { + mqc_report = ch_multiqc_report.getVal() + if (mqc_report.getClass() == ArrayList) { + log.warn "[{{ cookiecutter.name }}] Found multiple reports from process 'multiqc', will use only one" + mqc_report = mqc_report[0] + } + } + } catch (all) { + log.warn '[{{ cookiecutter.name }}] Could not attach MultiQC report to summary email' + } + + // Check if we are only sending emails on failure + email_address = params.email + if (!params.email && params.email_on_fail && !workflow.success) { + email_address = params.email_on_fail + } + + // Render the TXT template + def engine = new groovy.text.GStringTemplateEngine() + def tf = new File("$projectDir/assets/email_template.txt") + def txt_template = engine.createTemplate(tf).make(email_fields) + def email_txt = txt_template.toString() + + // Render the HTML template + def hf = new File("$projectDir/assets/email_template.html") + def html_template = engine.createTemplate(hf).make(email_fields) + def email_html = html_template.toString() + + // Render the sendmail template + def smail_fields = [ email: email_address, subject: subject, email_txt: email_txt, email_html: email_html, projectDir: "$projectDir", mqcFile: mqc_report, mqcMaxSize: params.max_multiqc_email_size.toBytes() ] + def sf = new File("$projectDir/assets/sendmail_template.txt") + def sendmail_template = engine.createTemplate(sf).make(smail_fields) + def sendmail_html = sendmail_template.toString() + + // Send the HTML e-mail + if (email_address) { + try { + if (params.plaintext_email) { throw GroovyException('Send plaintext e-mail, not HTML') } + // Try to send HTML e-mail using sendmail + [ 'sendmail', '-t' ].execute() << sendmail_html + log.info "[{{ cookiecutter.name }}] Sent summary e-mail to $email_address (sendmail)" + } catch (all) { + // Catch failures and try with plaintext + def mail_cmd = [ 'mail', '-s', subject, '--content-type=text/html', email_address ] + if ( mqc_report.size() <= params.max_multiqc_email_size.toBytes() ) { + mail_cmd += [ '-A', mqc_report ] + } + mail_cmd.execute() << email_html + log.info "[{{ cookiecutter.name }}] Sent summary e-mail to $email_address (mail)" + } + } + + // Write summary e-mail HTML to a file + def output_d = new File("${params.outdir}/pipeline_info/") + if (!output_d.exists()) { + output_d.mkdirs() + } + def output_hf = new File(output_d, 'pipeline_report.html') + output_hf.withWriter { w -> w << email_html } + def output_tf = new File(output_d, 'pipeline_report.txt') + output_tf.withWriter { w -> w << email_txt } + + c_green = params.monochrome_logs ? '' : "\033[0;32m"; + c_purple = params.monochrome_logs ? '' : "\033[0;35m"; + c_red = params.monochrome_logs ? '' : "\033[0;31m"; + c_reset = params.monochrome_logs ? '' : "\033[0m" + + if (workflow.stats.ignoredCount > 0 && workflow.success) { + log.info "-${c_purple}Warning, pipeline completed, but with errored process(es) ${c_reset}-" + log.info "-${c_red}Number of ignored errored process(es) : ${workflow.stats.ignoredCount} ${c_reset}-" + log.info "-${c_green}Number of successfully ran process(es) : ${workflow.stats.succeedCount} ${c_reset}-" + } + + if (workflow.success) { + log.info "-${c_purple}[{{ cookiecutter.name }}]${c_green} Pipeline completed successfully${c_reset}-" + } else { + checkHostname() + log.info "-${c_purple}[{{ cookiecutter.name }}]${c_red} Pipeline completed with errors${c_reset}-" + } } workflow.onError { @@ -221,22 +371,21 @@ workflow.onError { } } - def checkHostname() { def c_reset = params.monochrome_logs ? '' : "\033[0m" def c_white = params.monochrome_logs ? '' : "\033[0;37m" def c_red = params.monochrome_logs ? '' : "\033[1;91m" def c_yellow_bold = params.monochrome_logs ? '' : "\033[1;93m" if (params.hostnames) { - def hostname = "hostname".execute().text.trim() + def hostname = 'hostname'.execute().text.trim() params.hostnames.each { prof, hnames -> hnames.each { hname -> if (hostname.contains(hname) && !workflow.profile.contains(prof)) { - log.error "====================================================\n" + + log.error '====================================================\n' + " ${c_red}WARNING!${c_reset} You are running with `-profile $workflow.profile`\n" + " but your machine hostname is ${c_white}'$hostname'${c_reset}\n" + " ${c_yellow_bold}It's highly recommended that you use `-profile $prof${c_reset}`\n" + - "============================================================" + '============================================================' } } } From 24bf5ad458a28766f2e52480cd7d1b1b81c00afd Mon Sep 17 00:00:00 2001 From: kevinmenden Date: Thu, 18 Feb 2021 13:23:38 +0100 Subject: [PATCH 05/13] final fixes --- .../{{cookiecutter.name_noslash}}/main.nf | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf index b309879e8d..a6c8d1ffb9 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf @@ -93,8 +93,6 @@ if (params.input_paths) { //////////////////////////////////////////////////// /* -- PRINT PARAMETER SUMMARY -- */ //////////////////////////////////////////////////// - -def summary_params = NfcoreSchema.params_summary_map(workflow, params, json_schema) log.info NfcoreSchema.params_summary_log(workflow, params, json_schema) // Header log info @@ -131,10 +129,10 @@ if (params.email || params.email_on_fail) { // Check the hostnames against configured profiles checkHostname() -Channel.from(summary.collect { [it.key, it.value] }) - .map { k, v -> "
$k
${v ?: 'N/A'}
" } - .reduce { a, b -> return [a, b].join('\n ') } - .map { x -> ''' +Channel.from(summary.collect{ [it.key, it.value] }) + .map { k,v -> "
$k
${v ?: 'N/A'}
" } + .reduce { a, b -> return [a, b].join("\n ") } + .map { x -> """ id: '{{ cookiecutter.name_noslash }}-summary' description: " - this information is collected when the pipeline is started." section_name: '{{ cookiecutter.name }} Workflow Summary' @@ -144,7 +142,7 @@ Channel.from(summary.collect { [it.key, it.value] })
$x
- '''.stripIndent() } + """.stripIndent() } .set { ch_workflow_summary } /* @@ -207,19 +205,19 @@ process multiqc { // TODO nf-core: Add in log files from your new processes for MultiQC to find! file ('fastqc/*') from ch_fastqc_results.collect().ifEmpty([]) file ('software_versions/*') from ch_software_versions_yaml.collect() - file workflow_summary from ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml') + file workflow_summary from ch_workflow_summary.collectFile(name: "workflow_summary_mqc.yaml") output: - file '*multiqc_report.html' into ch_multiqc_report - file '*_data' - file 'multiqc_plots' + file "*multiqc_report.html" into ch_multiqc_report + file "*_data" + file "multiqc_plots" script: rtitle = '' rfilename = '' if (!(workflow.runName ==~ /[a-z]+_[a-z]+/)) { rtitle = "--title \"${workflow.runName}\"" - rfilename = '--filename ' + workflow.runName.replaceAll('\\W', '_').replaceAll('_+', '_') + '_multiqc_report' + rfilename = "--filename " + workflow.runName.replaceAll('\\W','_').replaceAll('_+','_') + "_multiqc_report" } custom_config_file = params.multiqc_config ? "--config $mqc_custom_config" : '' // TODO nf-core: Specify which MultiQC modules to use with -m for a faster run time @@ -251,6 +249,7 @@ process output_documentation { * Completion e-mail notification */ workflow.onComplete { + // Set up the e-mail variables def subject = "[{{ cookiecutter.name }}] Successful: $workflow.runName" if (!workflow.success) { @@ -291,7 +290,7 @@ workflow.onComplete { } } } catch (all) { - log.warn '[{{ cookiecutter.name }}] Could not attach MultiQC report to summary email' + log.warn "[{{ cookiecutter.name }}] Could not attach MultiQC report to summary email" } // Check if we are only sending emails on failure @@ -328,7 +327,7 @@ workflow.onComplete { // Catch failures and try with plaintext def mail_cmd = [ 'mail', '-s', subject, '--content-type=text/html', email_address ] if ( mqc_report.size() <= params.max_multiqc_email_size.toBytes() ) { - mail_cmd += [ '-A', mqc_report ] + mail_cmd += [ '-A', mqc_report ] } mail_cmd.execute() << email_html log.info "[{{ cookiecutter.name }}] Sent summary e-mail to $email_address (mail)" @@ -340,15 +339,15 @@ workflow.onComplete { if (!output_d.exists()) { output_d.mkdirs() } - def output_hf = new File(output_d, 'pipeline_report.html') + def output_hf = new File(output_d, "pipeline_report.html") output_hf.withWriter { w -> w << email_html } - def output_tf = new File(output_d, 'pipeline_report.txt') + def output_tf = new File(output_d, "pipeline_report.txt") output_tf.withWriter { w -> w << email_txt } c_green = params.monochrome_logs ? '' : "\033[0;32m"; c_purple = params.monochrome_logs ? '' : "\033[0;35m"; c_red = params.monochrome_logs ? '' : "\033[0;31m"; - c_reset = params.monochrome_logs ? '' : "\033[0m" + c_reset = params.monochrome_logs ? '' : "\033[0m"; if (workflow.stats.ignoredCount > 0 && workflow.success) { log.info "-${c_purple}Warning, pipeline completed, but with errored process(es) ${c_reset}-" @@ -362,6 +361,7 @@ workflow.onComplete { checkHostname() log.info "-${c_purple}[{{ cookiecutter.name }}]${c_red} Pipeline completed with errors${c_reset}-" } + } workflow.onError { From 9db29050565567672c2c019d63c26e86d8198c7c Mon Sep 17 00:00:00 2001 From: kevinmenden Date: Thu, 18 Feb 2021 13:25:53 +0100 Subject: [PATCH 06/13] removed multiqc function from nfcoreschema --- .../lib/NfcoreSchema.groovy | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/NfcoreSchema.groovy b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/NfcoreSchema.groovy index 6c8f7457ff..e73be4c12d 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/NfcoreSchema.groovy +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/NfcoreSchema.groovy @@ -422,28 +422,4 @@ class NfcoreSchema { return output } - static String params_summary_multiqc(workflow, summary) { - String summary_section = '' - for (group in summary.keySet()) { - def group_params = summary.get(group) // This gets the parameters of that particular group - if (group_params) { - summary_section += "

$group

\n" - summary_section += "
\n" - for (param in group_params.keySet()) { - summary_section += "
$param
${group_params.get(param) ?: 'N/A'}
\n" - } - summary_section += '
\n' - } - } - - String yaml_file_text = "id: '${workflow.manifest.name.replace('/', '-')}-summary'\n" - yaml_file_text += "description: ' - this information is collected when the pipeline is started.'\n" - yaml_file_text += "section_name: '${workflow.manifest.name} Workflow Summary'\n" - yaml_file_text += "section_href: 'https://github.com/${workflow.manifest.name}'\n" - yaml_file_text += "plot_type: 'html'\n" - yaml_file_text += 'data: |\n' - yaml_file_text += "${summary_section}" - return yaml_file_text - } - } From 7ce3224fa7aca146dc073fbb9de6826cb7c5922e Mon Sep 17 00:00:00 2001 From: kevinmenden Date: Thu, 18 Feb 2021 14:00:22 +0100 Subject: [PATCH 07/13] resources as string --- .../{{cookiecutter.name_noslash}}/nextflow.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config index 5f0f2e7c35..fe4c9c063a 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config @@ -37,9 +37,9 @@ params { schema_ignore_params = 'genomes' // Defaults only, expecting to be overwritten - max_memory = 128.GB + max_memory = "128.GB" max_cpus = 16 - max_time = 240.h + max_time = "240.h" } From 65d4fc256390faae3fbd4bb47ad6c82823037a2c Mon Sep 17 00:00:00 2001 From: kevinmenden Date: Thu, 18 Feb 2021 14:09:26 +0100 Subject: [PATCH 08/13] adjusted resource pattern --- .../{{cookiecutter.name_noslash}}/nextflow.config | 4 ++-- .../{{cookiecutter.name_noslash}}/nextflow_schema.json | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config index fe4c9c063a..5f0f2e7c35 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config @@ -37,9 +37,9 @@ params { schema_ignore_params = 'genomes' // Defaults only, expecting to be overwritten - max_memory = "128.GB" + max_memory = 128.GB max_cpus = 16 - max_time = "240.h" + max_time = 240.h } diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow_schema.json b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow_schema.json index d8156953ee..909d42c4ab 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow_schema.json +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow_schema.json @@ -174,7 +174,7 @@ "description": "Maximum amount of memory that can be requested for any single job.", "default": "128.GB", "fa_icon": "fas fa-memory", - "pattern": "^[\\d\\.]+\\.(K|M|G|T)?B$", + "pattern": "^[\\d\\.]+\\s*.(K|M|G|T)?B$", "hidden": true, "help_text": "Use to set an upper-limit for the memory requirement for each process. Should be a string in the format integer-unit e.g. `--max_memory '8.GB'`" }, @@ -183,7 +183,7 @@ "description": "Maximum amount of time that can be requested for any single job.", "default": "240.h", "fa_icon": "far fa-clock", - "pattern": "^[\\d\\.]+\\.(s|m|h|d)$", + "pattern": "^[\\d\\.]+\\.*(s|m|h|d)$", "hidden": true, "help_text": "Use to set an upper-limit for the time requirement for each process. Should be a string in the format integer-unit e.g. `--max_time '2.h'`" } @@ -256,4 +256,4 @@ "$ref": "#/definitions/institutional_config_options" } ] -} +} \ No newline at end of file From 6531dd615df44b3d2b351b397ec1a51de99d67d6 Mon Sep 17 00:00:00 2001 From: kevinmenden Date: Thu, 25 Feb 2021 15:30:00 +0100 Subject: [PATCH 09/13] add default value to help message --- .../{{cookiecutter.name_noslash}}/lib/NfcoreSchema.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/NfcoreSchema.groovy b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/NfcoreSchema.groovy index e73be4c12d..7076d6d638 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/NfcoreSchema.groovy +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/NfcoreSchema.groovy @@ -317,7 +317,8 @@ class NfcoreSchema { for (param in group_params.keySet()) { def type = '[' + group_params.get(param).type + ']' def description = group_params.get(param).description - output += " \u001B[1m--" + param.padRight(max_chars) + "\u001B[1m" + type.padRight(10) + description + '\n' + def defaultValue = group_params.get(param).default ? " [default: " + group_params.get(param).default.toString() + "]" : '' + output += " \u001B[1m--" + param.padRight(max_chars) + "\u001B[1m" + type.padRight(10) + description + defaultValue + '\n' } output += '\n' } From 8e719d6c5e63f9187e114dbab1a5b81a5c4b8f83 Mon Sep 17 00:00:00 2001 From: kevinmenden Date: Fri, 26 Feb 2021 10:45:38 +0100 Subject: [PATCH 10/13] download file and create template --- nf_core/__main__.py | 10 ++++++ nf_core/modules.py | 80 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 2fc4580885..086e578750 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -416,6 +416,16 @@ def remove(ctx, pipeline_dir, tool): mods.remove(tool) +@modules.command(help_priority=5) +@click.pass_context +@click.argument("directory", type=click.Path(exists=True), required=True, metavar="") +@click.argument("tool", type=str, required=True, metavar="") +@click.argument("subtool", type=str, required=False, metavar="") +def create(ctx, directory, tool, subtool=None): + mods = nf_core.modules.PipelineModules() + mods.create(directory=directory, tool=tool, subtool=subtool) + + @modules.command(help_priority=5) @click.pass_context def check(ctx): diff --git a/nf_core/modules.py b/nf_core/modules.py index 4d9270e629..89326fcd90 100644 --- a/nf_core/modules.py +++ b/nf_core/modules.py @@ -231,3 +231,83 @@ def has_valid_pipeline(self): if not os.path.exists(main_nf) and not os.path.exists(nf_config): log.error("Could not find a main.nf or nextfow.config file in: {}".format(self.pipeline_dir)) return False + + def create(self, directory, tool, subtool=None): + """ + Create a new module from the template + """ + + # Check whether the given directory is a nf-core pipeline or a clone + # of nf-core modules + self.repo_type = self.get_repo_type(directory) + + # Create template for new module in nf-core + if self.repo_type == "pipeline": + # Create the (sub)tool name + if subtool: + tool_name = tool + "_" + subtool + ".nf" + else: + tool_name = tool + ".nf" + module_file = os.path.join(directory, "modules", "local", "process", tool_name) + # Check whether module file already exists + if os.path.exists(module_file): + log.error(f"Module file {module_file} already exists!") + sys.exit(1) + + # Dowload template + template_copy = self.download_template() + + # Replace TOOL and SUBTOOL with correct names + + # Create directories (if necessary) and the module .nf file + os.makedirs(os.path.join(directory, "modules", "local", "process"), exist_ok=True) + with open(module_file, "w") as fh: + fh.write(template_copy) + + log.info("Module successfully created.") + + + + # Create template for new module in an nf-core pipeline + if self.repo_type == "modules": + print("hello") + # Dowload the template and create the necessary files and dctinoaries + + + + def get_repo_type(self, directory): + """ + Determine whether this is a pipeline repository or a clone of + nf-core/modules + """ + # Verify that the pipeline dir exists + if dir is None or not os.path.exists(directory): + log.error("Could not find directory: {}".format(directory)) + sys.exit(1) + + # Determine repository type + if os.path.exists(os.path.join(directory, "main.nf")): + return "pipeline" + elif os.path.exists(os.path.join(directory, "software")): + return "modules" + else: + log.error("Could not determine repository type of {}".format(directory)) + sys.exit(1) + + def download_template(self): + """ Download the module template """ + + url = "https://raw.githubusercontent.com/nf-core/modules/master/software/TOOL/SUBTOOL/main.nf" + r = requests.get(url=url) + + if r.status_code != 200: + log.error("Could not download the demplate") + sys.exit(1) + else: + try: + template_copy = r.content.decode("ascii") + except UnicodeDecodeError as e: + log.error(f"Could not decode template file from {url}: {e}") + sys.exit(1) + + return template_copy \ No newline at end of file From a2424c72246ed706ba10f0ecd807b26d7b57473c Mon Sep 17 00:00:00 2001 From: kevinmenden Date: Fri, 26 Feb 2021 10:51:14 +0100 Subject: [PATCH 11/13] evert "download file and create template" This reverts commit 8e719d6c5e63f9187e114dbab1a5b81a5c4b8f83. --- nf_core/__main__.py | 10 ------ nf_core/modules.py | 80 --------------------------------------------- 2 files changed, 90 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 086e578750..2fc4580885 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -416,16 +416,6 @@ def remove(ctx, pipeline_dir, tool): mods.remove(tool) -@modules.command(help_priority=5) -@click.pass_context -@click.argument("directory", type=click.Path(exists=True), required=True, metavar="") -@click.argument("tool", type=str, required=True, metavar="") -@click.argument("subtool", type=str, required=False, metavar="") -def create(ctx, directory, tool, subtool=None): - mods = nf_core.modules.PipelineModules() - mods.create(directory=directory, tool=tool, subtool=subtool) - - @modules.command(help_priority=5) @click.pass_context def check(ctx): diff --git a/nf_core/modules.py b/nf_core/modules.py index 89326fcd90..4d9270e629 100644 --- a/nf_core/modules.py +++ b/nf_core/modules.py @@ -231,83 +231,3 @@ def has_valid_pipeline(self): if not os.path.exists(main_nf) and not os.path.exists(nf_config): log.error("Could not find a main.nf or nextfow.config file in: {}".format(self.pipeline_dir)) return False - - def create(self, directory, tool, subtool=None): - """ - Create a new module from the template - """ - - # Check whether the given directory is a nf-core pipeline or a clone - # of nf-core modules - self.repo_type = self.get_repo_type(directory) - - # Create template for new module in nf-core - if self.repo_type == "pipeline": - # Create the (sub)tool name - if subtool: - tool_name = tool + "_" + subtool + ".nf" - else: - tool_name = tool + ".nf" - module_file = os.path.join(directory, "modules", "local", "process", tool_name) - # Check whether module file already exists - if os.path.exists(module_file): - log.error(f"Module file {module_file} already exists!") - sys.exit(1) - - # Dowload template - template_copy = self.download_template() - - # Replace TOOL and SUBTOOL with correct names - - # Create directories (if necessary) and the module .nf file - os.makedirs(os.path.join(directory, "modules", "local", "process"), exist_ok=True) - with open(module_file, "w") as fh: - fh.write(template_copy) - - log.info("Module successfully created.") - - - - # Create template for new module in an nf-core pipeline - if self.repo_type == "modules": - print("hello") - # Dowload the template and create the necessary files and dctinoaries - - - - def get_repo_type(self, directory): - """ - Determine whether this is a pipeline repository or a clone of - nf-core/modules - """ - # Verify that the pipeline dir exists - if dir is None or not os.path.exists(directory): - log.error("Could not find directory: {}".format(directory)) - sys.exit(1) - - # Determine repository type - if os.path.exists(os.path.join(directory, "main.nf")): - return "pipeline" - elif os.path.exists(os.path.join(directory, "software")): - return "modules" - else: - log.error("Could not determine repository type of {}".format(directory)) - sys.exit(1) - - def download_template(self): - """ Download the module template """ - - url = "https://raw.githubusercontent.com/nf-core/modules/master/software/TOOL/SUBTOOL/main.nf" - r = requests.get(url=url) - - if r.status_code != 200: - log.error("Could not download the demplate") - sys.exit(1) - else: - try: - template_copy = r.content.decode("ascii") - except UnicodeDecodeError as e: - log.error(f"Could not decode template file from {url}: {e}") - sys.exit(1) - - return template_copy \ No newline at end of file From 21b5587a7cf9aa9ecaf08e6532368ec5b77570db Mon Sep 17 00:00:00 2001 From: kevinmenden Date: Mon, 1 Mar 2021 15:23:26 +0100 Subject: [PATCH 12/13] added info to parameter summary message --- .../{{cookiecutter.name_noslash}}/lib/NfcoreSchema.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/NfcoreSchema.groovy b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/NfcoreSchema.groovy index 7076d6d638..bf307c6795 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/NfcoreSchema.groovy +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/NfcoreSchema.groovy @@ -418,6 +418,7 @@ class NfcoreSchema { output += '\n' } } + output += "[Only displaying parameters that differ from pipeline default]\n" output += dashed_line(params.monochrome_logs) output += '\n\n' + dashed_line(params.monochrome_logs) return output From 98f7b66caecfc7fbb0bd00af74528d3f228663c3 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 11 Mar 2021 17:17:58 +0100 Subject: [PATCH 13/13] Always show unexepected params. Add log explaining how to hide warning. --- .../lib/NfcoreSchema.groovy | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/NfcoreSchema.groovy b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/NfcoreSchema.groovy index bf307c6795..abec9df1ab 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/NfcoreSchema.groovy +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/lib/NfcoreSchema.groovy @@ -139,18 +139,26 @@ class NfcoreSchema { log.error 'ERROR: Validation of pipeline parameters failed!' JSONObject exceptionJSON = e.toJSON() printExceptions(exceptionJSON, paramsJSON, log) - if (unexpectedParams.size() > 0) { - println '' - def warn_msg = 'Found unexpected parameters:' - for (unexpectedParam in unexpectedParams) { - warn_msg = warn_msg + "\n* --${unexpectedParam}: ${paramsJSON[unexpectedParam].toString()}" - } - log.warn warn_msg - } println '' has_error = true } + // Check for unexpected parameters + // Getting this message a lot for parameters that you *do* expect? + // You can make a csv list of expected params not in the schema with 'params.schema_ignore_params' + // for example, in your institutional config + if (unexpectedParams.size() > 0) { + Map colors = log_colours(params.monochrome_logs) + println '' + def warn_msg = 'Found unexpected parameters:' + for (unexpectedParam in unexpectedParams) { + warn_msg = warn_msg + "\n* --${unexpectedParam}: ${paramsJSON[unexpectedParam].toString()}" + } + log.warn warn_msg + log.info "- ${colors.dim}(Hide this message with 'params.schema_ignore_params')${colors.reset} -" + println '' + } + if (has_error) { System.exit(1) } @@ -317,7 +325,7 @@ class NfcoreSchema { for (param in group_params.keySet()) { def type = '[' + group_params.get(param).type + ']' def description = group_params.get(param).description - def defaultValue = group_params.get(param).default ? " [default: " + group_params.get(param).default.toString() + "]" : '' + def defaultValue = group_params.get(param).default ? " [default: " + group_params.get(param).default.toString() + "]" : '' output += " \u001B[1m--" + param.padRight(max_chars) + "\u001B[1m" + type.padRight(10) + description + defaultValue + '\n' } output += '\n'