Merge pull request #556 from nf-core/olgabotpatch1-sync

Get latest dev version to check no flaky bam filtering

Author: jfy133

.github/.dockstore.yml

@@ -0,0 +1,5 @@
+# Dockstore config version, not pipeline version
+version: 1.2
+workflows:
+  - subclass: nfl
+    primaryDescriptorPath: /nextflow.config

.github/CONTRIBUTING.md

@@ -47,12 +47,12 @@ These tests are run both with the latest available version of `Nextflow` and als
 
 ## Patch
 
-: warning: Only in the unlikely and regretful event of a release happening with a bug.
+:warning: Only in the unlikely and regretful event of a release happening with a bug.
 
 * On your own fork, make a new branch `patch` based on `upstream/master`.
 * Fix the bug, and bump version (X.Y.Z+1).
 * A PR should be made on `master` from patch to directly this particular bug.
 
 ## Getting help
 
-For further information/help, please consult the [nf-core/eager documentation](https://nf-co.re/nf-core/eager/docs) and don't hesitate to get in touch on the nf-core Slack [#eager](https://nfcore.slack.com/channels/eager) channel ([join our Slack here](https://nf-co.re/join/slack)).
+For further information/help, please consult the [nf-core/eager documentation](https://nf-co.re/eager/docs) and don't hesitate to get in touch on the nf-core Slack [#eager](https://nfcore.slack.com/channels/eager) channel ([join our Slack here](https://nf-co.re/join/slack)).

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,18 +1,12 @@
----
-name: Bug report
-about: Create a report to help us improve
-title: ''
-labels: ''
-assignees: ''
-
----
-
 # nf-core/eager bug report
-
+<!--
 Hi there!
 
 Thanks for telling us about a problem with the pipeline.
 
+Please delete this text and anything that's not relevant from the template below:
+-->
+
 ## Check Documentation
 
 Have you checked in the following places for your error?:
@@ -21,7 +15,7 @@ Have you checked in the following places for your error?:
 - [ ] [Troubleshooting](https://nf-co.re/usage/troubleshooting)
       (for nf-core specific information)
 
-Please also check the the corresponding branch documentation on github, if not 
+Please also check the the corresponding branch documentation on github, if not
 testing the latest release.
 
 ## Describe the bug

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,32 +1,26 @@
----
-name: Feature request
-about: Suggest an idea for this project
-title: ''
-labels: ''
-assignees: ''
-
----
-
 # nf-core/eager feature request
+<!--
 
 Hi there!
 
 Thanks for suggesting a new feature for the pipeline!
+Please delete this text and anything that's not relevant from the template below:
+-->
 
 ## Is your feature request related to a problem? Please describe
 
-A clear and concise description of what the problem is.
+<!-- A clear and concise description of what the problem is. -->
 
-Ex. I'm always frustrated when [...]
+<!-- e.g. [I'm always frustrated when ...] -->
 
 ## Describe the solution you'd like
 
-A clear and concise description of what you want to happen.
+<!-- A clear and concise description of what you want to happen. -->
 
 ## Describe alternatives you've considered
 
-A clear and concise description of any alternative solutions or features you've considered.
+<!-- A clear and concise description of any alternative solutions or features you've considered. -->
 
 ## Additional context
 
-Add any other context about the feature request here.
+<!-- Add any other context about the feature request here. -->

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,9 +1,11 @@
 # nf-core/eager pull request
+<!--
 
 Many thanks for contributing to nf-core/eager!
 
 Please fill in the appropriate checklist below (delete whatever is not relevant).
 These are the most common things requested on pull requests (PRs).
+-->
 
 ## PR checklist
 

.github/workflows/awsfulltest.yml

@@ -1,15 +1,15 @@
-name: nf-core AWS full test
-# This workflow is triggered on push to the master branch and on published releases.
-# It runs the -profile 'test_full' on AWS batch
+
+name: nf-core AWS full size tests
+# This workflow is triggered on push to the master branch.
 
 on:
   release:
     types: [published]
 
 jobs:
   run-awstest:
-    if: github.repository == 'nf-core/eager'
     name: Run AWS full tests
+    if: github.repository == 'nf-core/eager'
     runs-on: ubuntu-latest
     steps:
       - name: Setup Miniconda
@@ -20,20 +20,20 @@ jobs:
       - name: Install awscli
         run: conda install -c conda-forge awscli
       - name: Start AWS batch job
-        # Add full size test data (but still relatively small datasets for few samples) 
+        # Add full size test data (but still relatively small datasets for few samples)
         # on the `test_full.config` test runs with only one set of parameters
         # Then specify `-profile test_full` instead of `-profile test` on the AWS batch command
         env:
-          AWS_ACCESS_KEY_ID: ${{secrets.AWS_ACCESS_KEY_ID}}
-          AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_SECRET_ACCESS_KEY}}
-          TOWER_ACCESS_TOKEN: ${{secrets.AWS_TOWER_TOKEN}}
-          AWS_JOB_DEFINITION: ${{secrets.AWS_JOB_DEFINITION}}
-          AWS_JOB_QUEUE: ${{secrets.AWS_JOB_QUEUE}}
-          AWS_S3_BUCKET: ${{secrets.AWS_S3_BUCKET}}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          TOWER_ACCESS_TOKEN: ${{ secrets.AWS_TOWER_TOKEN }}
+          AWS_JOB_DEFINITION: ${{ secrets.AWS_JOB_DEFINITION }}
+          AWS_JOB_QUEUE: ${{ secrets.AWS_JOB_QUEUE }}
+          AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
         run: |
           aws batch submit-job \
-          --region eu-west-1 \
-          --job-name nf-core-eager \
-          --job-queue $AWS_JOB_QUEUE \
-          --job-definition $AWS_JOB_DEFINITION \
-          --container-overrides '{"command": ["nf-core/eager", "-r '"${GITHUB_SHA}"' -profile test_full --outdir s3://'"${AWS_S3_BUCKET}"'/eager/results-'"${GITHUB_SHA}"' -w s3://'"${AWS_S3_BUCKET}"'/eager/work-'"${GITHUB_SHA}"' -with-tower"], "environment": [{"name": "TOWER_ACCESS_TOKEN", "value": "'"$TOWER_ACCESS_TOKEN"'"}]}'
+            --region eu-west-1 \
+            --job-name nf-core-eager \
+            --job-queue $AWS_JOB_QUEUE \
+            --job-definition $AWS_JOB_DEFINITION \
+            --container-overrides '{"command": ["nf-core/eager", "-r '"${GITHUB_SHA}"' -profile awsfulltest --outdir s3://'"${AWS_S3_BUCKET}"'/eager/results-'"${GITHUB_SHA}"' -w s3://'"${AWS_S3_BUCKET}"'/eager/work-'"${GITHUB_SHA}"' -with-tower"], "environment": [{"name": "TOWER_ACCESS_TOKEN", "value": "'"$TOWER_ACCESS_TOKEN"'"}]}'

.github/workflows/awstest.yml

@@ -1,6 +1,6 @@
 name: nf-core AWS test
-# This workflow is triggered on push to the master branch and on published releases.
-# It runs the -profile 'test_tsv' on AWS batch
+# This workflow is triggered on push to the master branch.
+# It runs the -profile 'test' on AWS batch
 
 on:
   push:
@@ -9,8 +9,8 @@ on:
 
 jobs:
   run-awstest:
+    name: Run AWS tests
     if: github.repository == 'nf-core/eager'
-    name: Run AWS test
     runs-on: ubuntu-latest
     steps:
       - name: Setup Miniconda
@@ -21,17 +21,19 @@ jobs:
       - name: Install awscli
         run: conda install -c conda-forge awscli
       - name: Start AWS batch job
+        # For example: adding multiple test runs with different parameters
+        # Remember that you can parallelise this by using strategy.matrix
         env:
-          AWS_ACCESS_KEY_ID: ${{secrets.AWS_ACCESS_KEY_ID}}
-          AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_SECRET_ACCESS_KEY}}
-          TOWER_ACCESS_TOKEN: ${{secrets.AWS_TOWER_TOKEN}}
-          AWS_JOB_DEFINITION: ${{secrets.AWS_JOB_DEFINITION}}
-          AWS_JOB_QUEUE: ${{secrets.AWS_JOB_QUEUE}}
-          AWS_S3_BUCKET: ${{secrets.AWS_S3_BUCKET}}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          TOWER_ACCESS_TOKEN: ${{ secrets.AWS_TOWER_TOKEN }}
+          AWS_JOB_DEFINITION: ${{ secrets.AWS_JOB_DEFINITION }}
+          AWS_JOB_QUEUE: ${{ secrets.AWS_JOB_QUEUE }}
+          AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
         run: |
           aws batch submit-job \
           --region eu-west-1 \
           --job-name nf-core-eager \
           --job-queue $AWS_JOB_QUEUE \
           --job-definition $AWS_JOB_DEFINITION \
-          --container-overrides '{"command": ["nf-core/eager", "-r '"${GITHUB_SHA}"' -profile test_tsv --outdir s3://'"${AWS_S3_BUCKET}"'/eager/results-'"${GITHUB_SHA}"' -w s3://'"${AWS_S3_BUCKET}"'/eager/work-'"${GITHUB_SHA}"' -with-tower"], "environment": [{"name": "TOWER_ACCESS_TOKEN", "value": "'"$TOWER_ACCESS_TOKEN"'"}]}'
+          --container-overrides '{"command": ["nf-core/eager", "-r '"${GITHUB_SHA}"' -profile test_tsv_complex --outdir s3://'"${AWS_S3_BUCKET}"'/eager/results-'"${GITHUB_SHA}"' -w s3://'"${AWS_S3_BUCKET}"'/eager/work-'"${GITHUB_SHA}"' -with-tower"], "environment": [{"name": "TOWER_ACCESS_TOKEN", "value": "'"$TOWER_ACCESS_TOKEN"'"}]}'eager/work-'"${GITHUB_SHA}"' -with-tower"], "environment": [{"name": "TOWER_ACCESS_TOKEN", "value": "'"$TOWER_ACCESS_TOKEN"'"}]}'

.github/workflows/branch.yml

@@ -3,14 +3,35 @@ name: nf-core branch protection
 # It fails when someone tries to make a PR against the nf-core `master` branch instead of `dev`
 on:
   pull_request:
-    branches:
-    - master
+    branches: [master]
 
 jobs:
   test:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     steps:
-      # PRs are only ok if coming from an nf-core `dev` branch or a fork `patch` branch
+      # PRs to the nf-core repo master branch are only ok if coming from the nf-core repo `dev` or any `patch` branches
       - name: Check PRs
+        if: github.repository == 'nf-core/eager'
         run: |
-          { [[ $(git remote get-url origin) == *nf-core/eager ]] && [[ ${GITHUB_HEAD_REF} = "dev" ]]; } || [[ ${GITHUB_HEAD_REF} == "patch" ]]
+          { [[ ${{github.event.pull_request.head.repo.full_name}} == nf-core/eager ]] && [[ $GITHUB_HEAD_REF = "dev" ]]; } || [[ $GITHUB_HEAD_REF == "patch" ]]
+
+
+      # If the above check failed, post a comment on the PR explaining the failure
+      # NOTE - this doesn't currently work if the PR is coming from a fork, due to limitations in GitHub actions secrets
+      - name: Post PR comment
+        if: failure()
+        uses: mshick/add-pr-comment@v1
+        with:
+          message: |
+            Hi @${{ github.event.pull_request.user.login }},
+
+            It looks like this pull-request is has been made against the ${{github.event.pull_request.head.repo.full_name}} `master` branch.
+            The `master` branch on nf-core repositories should always contain code from the latest release.
+            Because of this, PRs to `master` are only allowed if they come from the ${{github.event.pull_request.head.repo.full_name}} `dev` branch.
+
+            You do not need to close this PR, you can change the target branch to `dev` by clicking the _"Edit"_ button at the top of this page.
+
+            Thanks again for your contribution!
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          allow-repeats: false
+

.github/workflows/ci.yml

@@ -1,20 +1,22 @@
-name: nf-core eager CI
-#This workflow is triggered on pushes and PRs to the repository.
-on: [push, pull_request]  
+name: nf-core CI
+# This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors
+on:
+  push:
+    branches:
+      - dev
+  pull_request:
+  release:
+    types: [published]
 
 jobs:
-  conda_build:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v1
-      - name: Try Creating Conda env
-        run: |
-          conda env create --prefix nf-core-eager --file environment.yml
   test:
+    name: Run workflow tests
+    # Only run on push if this is the nf-core dev branch (merged PRs)
+    if: ${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/eager') }}
+    runs-on: ubuntu-latest
     env:
       NXF_VER: ${{ matrix.nxf_ver }}
       NXF_ANSI_LOG: false
-    runs-on: ubuntu-latest
     strategy:
       matrix:
         # Nextflow versions: check pipeline minimum and current latest
@@ -64,7 +66,7 @@ jobs:
           nextflow run ${GITHUB_WORKSPACE} -profile test_tsv,docker --save_reference
       - name: REFERENCE Basic workflow, with supplied indices
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -profile test_tsv,docker --bwa_index 'results/reference_genome/bwa_index/BWAIndex/Mammoth_MT_Krause.fasta' --fasta_index 'https://github.com/nf-core/test-datasets/blob/eager/reference/Mammoth/Mammoth_MT_Krause.fasta.fai' 
+          nextflow run ${GITHUB_WORKSPACE} -profile test_tsv,docker --bwa_index 'results/reference_genome/bwa_index/BWAIndex/' --fasta_index 'https://github.com/nf-core/test-datasets/blob/eager/reference/Mammoth/Mammoth_MT_Krause.fasta.fai' 
       - name: REFERENCE Run the basic pipeline with FastA reference with `fna` extension
         run: |
           nextflow run ${GITHUB_WORKSPACE} -profile test_tsv_fna,docker
@@ -103,7 +105,7 @@ jobs:
           nextflow run ${GITHUB_WORKSPACE} -profile test_tsv,docker --mapper 'bowtie2' --bt2_alignmode 'local' --bt2_sensitivity 'sensitive' --bt2n 1 --bt2l 16 --bt2_trim5 1 --bt2_trim3 1
       - name: HOST REMOVAL_FASTQ Run the basic pipeline with output unmapped reads as fastq
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -profile test_tsv,docker --hostremoval_input_fastq
+          nextflow run ${GITHUB_WORKSPACE} -profile test_tsv_complex,docker --hostremoval_input_fastq
       - name: BAM_FILTERING Run basic mapping pipeline with mapping quality filtering, and unmapped export
         run: |
           nextflow run ${GITHUB_WORKSPACE} -profile test_tsv,docker --run_bam_filtering --bam_mapping_quality_threshold 37  --bam_unmapped_type 'fastq'
@@ -152,7 +154,7 @@ jobs:
          nextflow run ${GITHUB_WORKSPACE} -profile test_tsv,docker --run_genotyping --run_trim_bam --genotyping_source 'trimmed' --gatk_ug_jar '/home/runner/work/eager/eager/jars/gatk_3_5/GenomeAnalysisTK.jar' --genotyping_tool 'ug' --gatk_out_mode 'EMIT_ALL_SITES' --gatk_ug_genotype_model 'SNP'
       - name: BAM_INPUT Run the basic pipeline with the bam input profile, skip AdapterRemoval as no convertBam
         run: |
-         nextflow run ${GITHUB_WORKSPACE} -profile test_tsv_bam,docker --skip_adapterremoval --run_convertinputbam
+         nextflow run ${GITHUB_WORKSPACE} -profile test_tsv_bam,docker --skip_adapterremoval
       - name: BAM_INPUT Run the basic pipeline with the bam input profile, convert to FASTQ for adapterremoval test and downstream
         run: |
           nextflow run ${GITHUB_WORKSPACE} -profile test_tsv_bam,docker --run_convertinputbam
@@ -163,7 +165,7 @@ jobs:
           for i in index0.idx ref.db ref.idx ref.inf table0.db table0.idx taxonomy.idx taxonomy.map taxonomy.tre; do wget https://github.com/nf-core/test-datasets/raw/eager/databases/malt/"$i" -P databases/malt/; done
       - name: METAGENOMIC Run the basic pipeline but with unmapped reads going into MALT
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -profile test_tsv,docker --run_bam_filtering  --bam_unmapped_type 'fastq' --run_metagenomic_screening --database "/home/runner/work/eager/eager/databases/malt/"
+          nextflow run ${GITHUB_WORKSPACE} -profile test_tsv,docker --run_bam_filtering  --bam_unmapped_type 'fastq' --run_metagenomic_screening --metagenomic_tool 'malt' --database "/home/runner/work/eager/eager/databases/malt/"
       - name: MALTEXTRACT Download resource files
         run: |
             mkdir -p databases/maltextract

.github/workflows/linting.yml

@@ -33,7 +33,10 @@ jobs:
   nf-core:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+
+      - name: Check out pipeline code
+        uses: actions/checkout@v2
+
       - name: Install Nextflow
         run: |
           wget -qO- get.nextflow.io | bash
@@ -46,5 +49,18 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install nf-core
+
       - name: Run nf-core lint
-        run: nf-core lint ${GITHUB_WORKSPACE}
+        env:
+          GITHUB_COMMENTS_URL: ${{ github.event.pull_request.comments_url }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_PR_COMMIT: ${{ github.event.pull_request.head.sha }}
+        run: nf-core -l lint_log.txt lint ${GITHUB_WORKSPACE}
+
+      - name: Upload linting log file artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: linting-log-file
+          path: lint_log.txt
+