Remove unused scripts and improve CI

.github/workflows/ci.yml

@@ -21,4 +21,110 @@ jobs:
           docker tag nfcore/sarek:dev nfcore/sarek:dev
       - name: Run test
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -profile test,docker
+          nextflow run ${GITHUB_WORKSPACE} -profile test,docker
+  annotation:
+    runs-on: ubuntu-18.04
+    strategy:
+      matrix:
+        annotator: [snpeff]
+        specie: [GRCh37]
+    steps:
+      - uses: actions/checkout@v1
+      - name: Install Nextflow
+        run: |
+          wget -qO- get.nextflow.io | bash
+          sudo mv nextflow /usr/local/bin/
+        env:
+          NXF_VER: '19.10.0'
+      - name: Download and tag images
+        run: |
+          docker pull nfcore/sarek:dev
+          docker tag nfcore/sarek:dev nfcore/sarek:dev
+          docker pull nfcore/sarek${{ matrix.annotator }}:dev.${{ matrix.specie }}
+          docker tag nfcore/sarek${{ matrix.annotator }}:dev.${{ matrix.specie }} nfcore/sarek${{ matrix.annotator }}:dev.${{ matrix.specie }}
+      - name: Run annotation test
+        run: |
+          nextflow run . -profile test_annotation,docker --verbose --tools ${{ matrix.annotator }}
+  germline:
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v1
+      - name: Install Nextflow
+        run: |
+          wget -qO- get.nextflow.io | bash
+          sudo mv nextflow /usr/local/bin/
+        env:
+          NXF_VER: '19.10.0'
+      - name: Download and tag image
+        run: |
+          docker pull nfcore/sarek:dev
+          docker tag nfcore/sarek:dev nfcore/sarek:dev
+      - name: Get test data
+        run: |
+          git clone --single-branch --branch sarek https://github.com/nf-core/test-datasets.git data
+      - name: Run germline test
+        run: |
+          nextflow run . -profile test,docker --input data/testdata/tiny/normal
+          nextflow run . -profile test,docker --input=false --step recalibrate -resume
+          nextflow run . -profile test,docker --input=false --step variantCalling
+  minimal:
+    runs-on: ubuntu-18.04
+    strategy:
+      matrix:
+        genome: [smallerGRCh37, minimalGRCh37]
+        intervals: [--no_intervals, '']
+    steps:
+      - uses: actions/checkout@v1
+      - name: Install Nextflow
+        run: |
+          wget -qO- get.nextflow.io | bash
+          sudo mv nextflow /usr/local/bin/
+        env:
+          NXF_VER: '19.10.0'
+      - name: Download and tag image
+        run: |
+          docker pull nfcore/sarek:dev
+          docker tag nfcore/sarek:dev nfcore/sarek:dev
+      - name: Run test for minimal genomes
+        run: |
+          nextflow run . -profile test,docker --skipQC all --verbose --genome ${{ matrix.genome }} ${{ matrix.intervals }} --tools Manta,mpileup,Strelka
+  profile:
+    runs-on: ubuntu-18.04
+    strategy:
+      matrix:
+        profile: [test_splitfastq, test_targeted]
+    steps:
+      - uses: actions/checkout@v1
+      - name: Install Nextflow
+        run: |
+          wget -qO- get.nextflow.io | bash
+          sudo mv nextflow /usr/local/bin/
+        env:
+          NXF_VER: '19.10.0'
+      - name: Download and tag image
+        run: |
+          docker pull nfcore/sarek:dev
+          docker tag nfcore/sarek:dev nfcore/sarek:dev
+      - name: Run targeted and splitfastq tests
+        run: |
+          nextflow run . -profile ${{ matrix.profile }},docker --verbose
+  tools:
+    runs-on: ubuntu-18.04
+    strategy:
+      matrix:
+        tool: [Haplotypecaller, Freebayes, Manta, mpileup, Mutect2, Strelka]
+    steps:
+      - uses: actions/checkout@v1
+      - name: Install Nextflow
+        run: |
+          wget -qO- get.nextflow.io | bash
+          sudo mv nextflow /usr/local/bin/
+        env:
+          NXF_VER: '19.10.0'
+      - name: Download and tag image
+        run: |
+          docker pull nfcore/sarek:dev
+          docker tag nfcore/sarek:dev nfcore/sarek:dev
+      - name: Run vriant calling test on specific tools
+        run: |
+          nextflow run . -profile test_tool,docker --verbose --tools ${{ matrix.tool }}

CHANGELOG.md

@@ -35,10 +35,14 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a
 - [#60](https://github.com/nf-core/sarek/pull/60) - Update documentation
 - [#71](https://github.com/nf-core/sarek/pull/71) - Update `README`
 - [#71](https://github.com/nf-core/sarek/pull/71) - Update `CHANGELOG`
+- [#74](https://github.com/nf-core/sarek/pull/74) - Update docs
+- [#74](https://github.com/nf-core/sarek/pull/74) - Improve CI tests (both Jenkins and GitHub actions tests)
+- [#74](https://github.com/nf-core/sarek/pull/74) - Move all ci from `ci-extra.yml` to `ci.yml`
 
 ### `Removed`
 
 - [#46](https://github.com/nf-core/sarek/pull/46) - Remove mention of old `build.nf` script which was included in `main.nf`
+- [#74](https://github.com/nf-core/sarek/pull/74) - Remove `download_image.sh` and `run_tests.sh` scripts
 
 ### `Fixed`
 

Jenkinsfile

@@ -8,35 +8,46 @@ pipeline {
     stages {
         stage('Docker setup') {
             steps {
-                 sh "./scripts/download_image.sh -n docker -t ALL --source-version dev --target-version dev -g smallGRCh37"
+                 sh "docker pull nfcore/sarek:dev"
+                 sh "docker tag nfcore/sarek:dev nfcore/sarek:dev"
+                 sh "docker pull nfcore/sareksnpeff:dev.GRCh37"
+                 sh "docker tag nfcore/sareksnpeff:dev.GRCh37 nfcore/sareksnpeff:dev.GRCh37"
+                 sh "docker pull nfcore/sarekvep:dev.GRCh37"
+                 sh "docker tag nfcore/sarekvep:dev.GRCh37 nfcore/sarekvep:dev.GRCh37"
+            }
+        }
+        stage('Annotation') {
+            steps {
+                sh "nextflow run . -profile test_annotation,kraken --verbose --tools snpeff,vep,merge"
             }
         }
         stage('Germline') {
             steps {
                 sh "rm -rf data/"
                 sh "git clone --single-branch --branch sarek https://github.com/nf-core/test-datasets.git data"
-                sh "./scripts/run_tests.sh --profile kraken --test GERMLINE --no-reports"
+                sh "nextflow run . -profile test,kraken --input data/testdata/tiny/normal"
+                sh "nextflow run . -profile test,kraken --input=false --step recalibrate -resume"
+                sh "nextflow run . -profile test,kraken --input=false --step variantCalling"
                 sh "rm -rf data/"
             }
         }
-        stage('Somatic') {
+        stage('Minimal') {
             steps {
-                sh "./scripts/run_tests.sh --profile kraken --test SOMATIC --no-reports"
+                sh "nextflow run . -profile test,kraken --skipQC all --verbose --genome smallerGRCh37 --no_intervals --tools Manta,mpileup,Strelka"
+                sh "nextflow run . -profile test,kraken --skipQC all --verbose --genome smallerGRCh37 --tools Manta,mpileup,Strelka"
+                sh "nextflow run . -profile test,kraken --skipQC all --verbose --genome minimalGRCh37 --no_intervals --tools Manta,mpileup,Strelka"
+                sh "nextflow run . -profile test,kraken --skipQC all --verbose --genome minimalGRCh37 --tools Manta,mpileup,Strelka"
             }
         }
-        stage('Targeted') {
-            steps {
-                sh "./scripts/run_tests.sh --profile kraken --test TARGETED --no-reports"
-            }
-        }
-        stage('Annotation') {
+        stage('Profile') {
             steps {
-                sh "./scripts/run_tests.sh --profile kraken --test ANNOTATEBOTH --no-reports"
+                sh "nextflow run . -profile test_splitfastq,kraken --verbose"
+                sh "nextflow run . -profile test_targeted,kraken --verbose"
             }
         }
-        stage('Multiple') {
+        stage('Tools') {
             steps {
-                sh "./scripts/run_tests.sh --profile kraken --test MULTIPLE"
+                sh "nextflow run . -profile test_tool,kraken --verbose --tools Haplotypecaller,Freebayes,Manta,mpileup,Mutect2,Strelka"
             }
         }
     }

README.md

@@ -7,7 +7,6 @@
 [![DOI](https://zenodo.org/badge/184289291.svg)](https://zenodo.org/badge/latestdoi/184289291)
 
 [![GitHub Actions CI status](https://github.com/nf-core/sarek/workflows/nf-core%20CI/badge.svg)](https://github.com/nf-core/sarek/actions?query=workflow%3A%22sarek+CI%22)
-[![GitHub Actions extra-CI status](https://github.com/nf-core/sarek/workflows/nf-core%20extra%20CI/badge.svg)](https://github.com/nf-core/sarek/actions?query=workflow%3A%22sarek+extra+CI%22)
 [![GitHub Actions Linting status](https://github.com/nf-core/sarek/workflows/nf-core%20linting/badge.svg)](https://github.com/nf-core/sarek/actions?query=workflow%3A%22sarek+linting%22)
 [![CircleCi build status](https://img.shields.io/circleci/project/github/nf-core/sarek?logo=circleci)](https://circleci.com/gh/nf-core/sarek/)
 

docs/annotation.md

@@ -36,8 +36,8 @@ The cache will only be used when `--annotation_cache` and cache directories are
 Example:
 
 ```bash
-nextflow run nf-core/sarek/main.nf --tools snpEff --step annotate --sample file.vcf.gz --snpEff_cache /Path/To/snpEffCache --annotation_cache
-nextflow run nf-core/sarek/main.nf --tools VEP --step annotate --sample file.vcf.gz --vep_cache /Path/To/vepCache --annotation_cache
+nextflow run nf-core/sarek --tools snpEff --step annotate --sample file.vcf.gz --snpEff_cache /Path/To/snpEffCache --annotation_cache
+nextflow run nf-core/sarek --tools VEP --step annotate --sample file.vcf.gz --vep_cache /Path/To/vepCache --annotation_cache
 ```
 
 ## Using VEP CADD plugin
@@ -51,7 +51,7 @@ To enable the use of the VEP CADD plugin:
 Example:
 
 ```bash
-nextflow run nf-core/sarek/main.nf --step annotate --tools VEP --sample file.vcf.gz --cadd_cache \
+nextflow run nf-core/sarek --step annotate --tools VEP --sample file.vcf.gz --cadd_cache \
     --cadd_InDels /PathToCADD/InDels.tsv.gz \
     --cadd_InDels_tbi /PathToCADD/InDels.tsv.gz.tbi \
     --cadd_WG_SNVs /PathToCADD/whole_genome_SNVs.tsv.gz \
@@ -76,5 +76,5 @@ To enable the use of the VEP GeneSplicer plugin:
 Example:
 
 ```bash
-nextflow run nf-core/sarek/main.nf --step annotate --tools VEP --sample file.vcf.gz --genesplicer
+nextflow run nf-core/sarek --step annotate --tools VEP --sample file.vcf.gz --genesplicer
 ```