Merge pull request #17 from nf-core/dev

jfy133 · web-flow · commit f6d4c136c2da · 2019-11-16T21:20:56.000+01:00
Dev
diff --git a/.github/workflows/branch.yml b/.github/workflows/branch.yml
@@ -7,7 +7,7 @@ on:
 
 jobs:
   test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-18.04
     steps:
       # PRs are only ok if coming from an nf-core dev branch
       - name: Check PRs
@@ -24,4 +24,4 @@ jobs:
           echo "GITHUB_REF: ${GITHUB_REF}"
           echo "GITHUB_HEAD_REF: ${GITHUB_HEAD_REF}"
           echo "GITHUB_BASE_REF: ${GITHUB_BASE_REF}"
-          [ ${GITHUB_ACTOR} = "nf-core" ] && [ ${GITHUB_HEAD_REF} = "dev" ]
+          [ ${GITHUB_ACTOR} = "nf-core" ] && [ ${GITHUB_HEAD_REF} = "dev" ]
diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml
@@ -4,7 +4,7 @@ on: [push, pull_request]
 
 jobs:
   Markdown:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-18.04
     steps:
       - uses: actions/checkout@v1
       - uses: actions/setup-node@v1
@@ -37,4 +37,4 @@ jobs:
           pip install nf-core
       - name: Run nf-core lint
         run: |
-          nf-core lint ${GITHUB_WORKSPACE}
+          nf-core lint ${GITHUB_WORKSPACE}
diff --git a/.github/workflows/nf-core_eager.yml b/.github/workflows/nf-core_eager.yml
@@ -3,10 +3,17 @@ name: nf-core eager CI
 on: [push, pull_request]  
 
 jobs:
+  conda_build:
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v1
+      - name: Try Creating Conda env
+        run: |
+          conda env create --prefix nf-core-eager-2.1.0dev-d95a13feb408cc17e95d38f16a81010d --file environment.yml
   github_actions_ci:
     runs-on: ubuntu-latest
     env:
-      TOWER_ACCESS_TOKEN: 1c1f493bc2703472d6f1b9f6fb9e9d117abab7b1
+      TOWER_ACCESS_TOKEN: ${{ secrets.TOWER_ACCESS_TOKEN }}
       NXF_ANSI_LOG: 0
     strategy:
       matrix:
@@ -21,74 +28,78 @@ jobs:
         shell: bash
         run: echo "::set-env name=RUN_NAME::`echo ${GITHUB_REPOSITORY//\//_}`-`echo ${GITHUB_HEAD_REF//\//@} | rev | cut -f1 -d@ | rev`-${{ github.event_name }}-`echo ${GITHUB_SHA} | cut -c1-6`"
         id: extract_branch
+      - name: Determine tower usage
+        shell: bash
+        run: echo "::set-env name=TOWER::`[ -z "$TOWER_ACCESS_TOKEN" ] && echo '' || echo '-with-tower'`" 
+        id: tower_usage
       - name:  BASIC Run the basic pipeline with the test profileBasic workflow, PE/SE, bwa aln
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -with-tower -name "$RUN_NAME-basic" -profile test,docker ${{ matrix.endedness }} --saveReference
+          nextflow run ${GITHUB_WORKSPACE} "$TOWER" -name "$RUN_NAME-basic" -profile test,docker ${{ matrix.endedness }} --saveReference
       - name: REFERENCE Basic workflow, with supplied indices
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -with-tower -name "$RUN_NAME-preindex_ref" -profile test,docker ${{ matrix.endedness }} --bwa_index results/reference_genome/bwa_index/BWAIndex/Mammoth_MT_Krause.fasta 
+          nextflow run ${GITHUB_WORKSPACE} "$TOWER" -name "$RUN_NAME-preindex_ref" -profile test,docker ${{ matrix.endedness }} --bwa_index results/reference_genome/bwa_index/BWAIndex/Mammoth_MT_Krause.fasta 
       - name: REFERENCE Run the basic pipeline with FastA reference with `fna` extension
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -with-tower -name "$RUN_NAME-fna_ref" -profile test_fna,docker --pairedEnd
+          nextflow run ${GITHUB_WORKSPACE} "$TOWER" -name "$RUN_NAME-fna_ref" -profile test_fna,docker --pairedEnd
       - name: REFERENCE Test with zipped reference input
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -with-tower -name "$RUN_NAME-gz_ref" -profile test,docker --pairedEnd --fasta 'https://github.com/jfy133/test-datasets/raw/eager/reference/Mammoth/Mammoth_MT_Krause.fasta.gz'
+          nextflow run ${GITHUB_WORKSPACE} "$TOWER" -name "$RUN_NAME-gz_ref" -profile test,docker --pairedEnd --fasta 'https://github.com/jfy133/test-datasets/raw/eager/reference/Mammoth/Mammoth_MT_Krause.fasta.gz'
       - name: FASTP Test fastp complexity filtering
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -with-tower -name "$RUN_NAME-fastp" -profile test,docker --pairedEnd --complexity_filter
+          nextflow run ${GITHUB_WORKSPACE} "$TOWER" -name "$RUN_NAME-fastp" -profile test,docker --pairedEnd --complexity_filter
       - name: ADAPTERREMOVAL Test skip pairedEnd collapsing
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -with-tower -name "$RUN_NAME-skip_collapse" -profile test,docker --pairedEnd --skip_collapse
+          nextflow run ${GITHUB_WORKSPACE} "$TOWER" -name "$RUN_NAME-skip_collapse" -profile test,docker --pairedEnd --skip_collapse
       - name: ADAPTERREMOVAL Test pairedEnd collapsing but no trimming
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -with-tower -name "$RUN_NAME-pretrim" -profile test_pretrim,docker --pairedEnd --skip_trim
+          nextflow run ${GITHUB_WORKSPACE} "$TOWER" -name "$RUN_NAME-pretrim" -profile test_pretrim,docker --pairedEnd --skip_trim
       - name: ADAPTERREMOVAL Run the basic pipeline with paired end data without adapterRemoval
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -with-tower -name "$RUN_NAME-skip_adapterremoval" -profile test,docker --pairedEnd --skip_adapterremoval
+          nextflow run ${GITHUB_WORKSPACE} "$TOWER" -name "$RUN_NAME-skip_adapterremoval" -profile test,docker --pairedEnd --skip_adapterremoval
       - name: ADAPTERREMOVAL Run the basic pipeline with preserve5p end option
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -with-tower -name "$RUN_NAME-preserve5p" -profile test,docker --pairedEnd --preserve5p
+          nextflow run ${GITHUB_WORKSPACE} "$TOWER" -name "$RUN_NAME-preserve5p" -profile test,docker --pairedEnd --preserve5p
       - name: ADAPTERREMOVAL Run the basic pipeline with merged only option
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -with-tower -name "$RUN_NAME-mergedonly" -profile test,docker --pairedEnd --mergedonly
+          nextflow run ${GITHUB_WORKSPACE} "$TOWER" -name "$RUN_NAME-mergedonly" -profile test,docker --pairedEnd --mergedonly
       - name: ADAPTERREMOVAL Run the basic pipeline with preserve5p end and merged reads only options
         run: |
-         nextflow run ${GITHUB_WORKSPACE} -with-tower -name "$RUN_NAME-preserve5p_mergedonly" -profile test,docker --pairedEnd --preserve5p --mergedonly
+         nextflow run ${GITHUB_WORKSPACE} "$TOWER" -name "$RUN_NAME-preserve5p_mergedonly" -profile test,docker --pairedEnd --preserve5p --mergedonly
       - name: MAPPER_CIRCULARMAPPER Test running with CircularMapper
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -with-tower -name "$RUN_NAME-circularmapper" -profile test,docker --pairedEnd --mapper 'circularmapper' --circulartarget 'NC_007596.2'
+          nextflow run ${GITHUB_WORKSPACE} "$TOWER" -name "$RUN_NAME-circularmapper" -profile test,docker --pairedEnd --mapper 'circularmapper' --circulartarget 'NC_007596.2'
       - name: MAPPER_BWAMEM Test running with BWA Mem
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -with-tower -name "$RUN_NAME-bwa_mem" -profile test,docker --pairedEnd --mapper 'bwamem'
+          nextflow run ${GITHUB_WORKSPACE} "$TOWER" -name "$RUN_NAME-bwa_mem" -profile test,docker --pairedEnd --mapper 'bwamem'
       - name: STRIP_FASTQ Run the basic pipeline with output unmapped reads as fastq
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -with-tower -name "$RUN_NAME-stripfastq" -profile test,docker --pairedEnd --strip_input_fastq
+          nextflow run ${GITHUB_WORKSPACE} "$TOWER" -name "$RUN_NAME-stripfastq" -profile test,docker --pairedEnd --strip_input_fastq
       - name: BAM_FILTERING Run basic mapping pipeline with mapping quality filtering, and unmapped export
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -with-tower -name "$RUN_NAME-unmapped_export" -profile test,docker --pairedEnd --run_bam_filtering --bam_mapping_quality_threshold 37 --bam_discard_umapped --bam_unmapped_type 'fastq'
+          nextflow run ${GITHUB_WORKSPACE} "$TOWER" -name "$RUN_NAME-unmapped_export" -profile test,docker --pairedEnd --run_bam_filtering --bam_mapping_quality_threshold 37 --bam_discard_umapped --bam_unmapped_type 'fastq'
       - name: GENOTYPING_HC Test running GATK HaplotypeCaller
         run: |
-         nextflow run ${GITHUB_WORKSPACE} -with-tower -name "$RUN_NAME-haplotypercaller" -profile test_fna,docker --pairedEnd  --dedupper 'dedup' --run_genotyping --genotyping_tool 'hc' --gatk_out_mode 'EMIT_ALL_SITES' --gatk_hc_emitrefconf 'BP_RESOLUTION'
+         nextflow run ${GITHUB_WORKSPACE} "$TOWER" -name "$RUN_NAME-haplotypercaller" -profile test_fna,docker --pairedEnd  --dedupper 'dedup' --run_genotyping --genotyping_tool 'hc' --gatk_out_mode 'EMIT_ALL_SITES' --gatk_hc_emitrefconf 'BP_RESOLUTION'
       - name: GENOTYPING_FB Test running FreeBayes
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -with-tower -name "$RUN_NAME-freebayes" -profile test,docker --pairedEnd  --dedupper 'dedup' --run_genotyping --genotyping_tool 'freebayes'
+          nextflow run ${GITHUB_WORKSPACE} "$TOWER" -name "$RUN_NAME-freebayes" -profile test,docker --pairedEnd  --dedupper 'dedup' --run_genotyping --genotyping_tool 'freebayes'
       - name: SKIPPING Test checking all skip steps work i.e. input bam, skipping straight to genotyping
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -with-tower -name "$RUN_NAME-skipping_logic" -profile test_bam,docker --bam --singleEnd --skip_fastqc --skip_adapterremoval --skip_mapping --skip_deduplication --skip_qualimap --skip_preseq --skip_damage_calculation --run_genotyping --genotyping_tool 'freebayes'
+          nextflow run ${GITHUB_WORKSPACE} "$TOWER" -name "$RUN_NAME-skipping_logic" -profile test_bam,docker --bam --singleEnd --skip_fastqc --skip_adapterremoval --skip_mapping --skip_deduplication --skip_qualimap --skip_preseq --skip_damage_calculation --run_genotyping --genotyping_tool 'freebayes'
       - name: TRIM_BAM/PMD/GENOTYPING_UG/MULTIVCFANALYZER Test running PMDTools, TrimBam, GATK UnifiedGenotyper and MultiVCFAnalyzer
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -with-tower -name "$RUN_NAME-pmd_trimbam_unifiedgenotyper_multivcfanalyzer" -profile test,docker --pairedEnd  --dedupper 'dedup' --run_trim_bam --run_pmdtools --run_genotyping --genotyping_source 'trimmed' --genotyping_tool 'ug' --gatk_out_mode 'EMIT_ALL_SITES' --gatk_ug_genotype_model 'SNP' --run_multivcfanalyzer
+          nextflow run ${GITHUB_WORKSPACE} "$TOWER" -name "$RUN_NAME-pmd_trimbam_unifiedgenotyper_multivcfanalyzer" -profile test,docker --pairedEnd  --dedupper 'dedup' --run_trim_bam --run_pmdtools --run_genotyping --genotyping_source 'trimmed' --genotyping_tool 'ug' --gatk_out_mode 'EMIT_ALL_SITES' --gatk_ug_genotype_model 'SNP' --run_multivcfanalyzer
       - name: GENOTYPING_UG/PMD/MULTIVCFANALYZER Test running GATK UnifiedGenotyper and MultiVCFAnalyzer, additional VCFS
         run: |
-         nextflow run ${GITHUB_WORKSPACE} -with-tower -name "$RUN_NAME-multivcfanalyzer_additionalvcfs" -profile test,docker --pairedEnd  --dedupper 'dedup' --run_genotyping --genotyping_tool 'ug' --gatk_out_mode 'EMIT_ALL_SITES' --gatk_ug_genotype_model 'SNP' --run_multivcfanalyzer --additional_vcf_files 'https://raw.githubusercontent.com/jfy133/test-datasets/eager/testdata/Mammoth/vcf/JK2772_CATCAGTGAGTAGA_L008_R1_001.fastq.gz.tengrand.fq.combined.fq.mapped_rmdup.bam.unifiedgenotyper.vcf.gz' --write_allele_frequencies
+         nextflow run ${GITHUB_WORKSPACE} "$TOWER" -name "$RUN_NAME-multivcfanalyzer_additionalvcfs" -profile test,docker --pairedEnd  --dedupper 'dedup' --run_genotyping --genotyping_tool 'ug' --gatk_out_mode 'EMIT_ALL_SITES' --gatk_ug_genotype_model 'SNP' --run_multivcfanalyzer --additional_vcf_files 'https://raw.githubusercontent.com/jfy133/test-datasets/eager/testdata/Mammoth/vcf/JK2772_CATCAGTGAGTAGA_L008_R1_001.fastq.gz.tengrand.fq.combined.fq.mapped_rmdup.bam.unifiedgenotyper.vcf.gz' --write_allele_frequencies
       - name: BAM_INPUT Run the basic pipeline with the bam input profile, skip AdapterRemoval as no convertBam
         run: |
-         nextflow run ${GITHUB_WORKSPACE} -with-tower -name "$RUN_NAME-baminput_noConvertBam" -profile test_bam,docker --bam --skip_adapterremoval --run_convertbam
+         nextflow run ${GITHUB_WORKSPACE} "$TOWER" -name "$RUN_NAME-baminput_noConvertBam" -profile test_bam,docker --bam --skip_adapterremoval --run_convertbam
       - name: BAM_INPUT Run the basic pipeline with the bam input profile, convert to FASTQ for adapterremoval test and downstream
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -with-tower -name "$RUN_NAME-baminput_convertbam_basic" -profile test_bam,docker --bam --run_convertbam
+          nextflow run ${GITHUB_WORKSPACE} "$TOWER" -name "$RUN_NAME-baminput_convertbam_basic" -profile test_bam,docker --bam --run_convertbam
       #- name: [DISABLED UNTIL BED FILE AVAILABLE - 2h RUN TIME WITHOUT] - SEXDETERMINAION Run the basic pipeline with the bam input profile, but don't convert BAM, skip everything but sex determination
-         #nextflow run ${GITHUB_WORKSPACE} -with-tower -profile test_humanbam,docker --bam --skip_fastqc --skip_adapterremoval --skip_mapping --skip_deduplication --skip_qualimap --singleEnd --run_sexdeterrmine
+         #nextflow run ${GITHUB_WORKSPACE} "$TOWER" -profile test_humanbam,docker --bam --skip_fastqc --skip_adapterremoval --skip_mapping --skip_deduplication --skip_qualimap --singleEnd --run_sexdeterrmine
       #- name: [DISABLED UNTIL SMALL HUMAN REFERENCE AVALIABLE - REQUIRES HUMAN FASTA] - NUCLEAR INPUT
-          #nextflow run ${GITHUB_WORKSPACE} -with-tower -profile test_humanbam,docker --bam --skip_fastqc --skip_adapterremoval --skip_mapping --skip_deduplication --skip_qualimap --singleEnd --run_nuclearcontamination
+          #nextflow run ${GITHUB_WORKSPACE} "$TOWER" -profile test_humanbam,docker --bam --skip_fastqc --skip_adapterremoval --skip_mapping --skip_deduplication --skip_qualimap --singleEnd --run_nuclearcontamination
 
diff --git a/bin/print_x_contamination.py b/bin/print_x_contamination.py
@@ -26,13 +26,16 @@
                 err_mom1=fields[4].split(":")[1]
                 ml1=fields[5].split(":")[1]
                 err_ml1=fields[6].split(":")[1]
+                ## Sometimes angsd fails to run method 2, and the error is printed directly after the SE for ML. When that happens, exclude the first word in the error from the output. (Method 2 data will be shown as NA)
+                if err_ml1.endswith("contamination"):
+                    err_ml1 = err_ml1[:-13]
             elif line.strip()[0:7] == "Method2" and line.strip()[9:16] == 'new_llh':
                 mom2=fields[3].split(":")[1]
                 err_mom2=fields[4].split(":")[1]
                 ml2=fields[5].split(":")[1]
                 err_ml2=fields[6].split(":")[1]
-        data[Ind]={ "Method1_MOM_estimate" : mom1, "Method1_MOM_SE" : err_mom1, "Method1_ML_estimate" : ml1, "Method1_ML_SE" : err_ml1, "Method2_MOM_estimate" : mom2, "Method2_MOM_SE" : err_mom2, "Method2_ML_estimate" : ml2, "Method2_ML_SE" : err_ml2 }
-        print (Ind, mom1, err_mom1, ml1, err_ml1, mom2, err_mom2, ml2, err_ml2, sep="\t", file=output)
+        data[Ind]={ "Number_of_SNPs" : nSNPs, "Method1_MOM_estimate" : mom1, "Method1_MOM_SE" : err_mom1, "Method1_ML_estimate" : ml1, "Method1_ML_SE" : err_ml1, "Method2_MOM_estimate" : mom2, "Method2_MOM_SE" : err_mom2, "Method2_ML_estimate" : ml2, "Method2_ML_SE" : err_ml2 }
+        print (Ind, nSNPs, mom1, err_mom1, ml1, err_ml1, mom2, err_mom2, ml2, err_ml2, sep="\t", file=output)
 
 with open('nuclear_contamination.json', 'w') as outfile:
     json.dump(data, outfile)
diff --git a/environment.yml b/environment.yml
@@ -4,7 +4,7 @@ channels:
   - bioconda
   - conda-forge
 dependencies:
-  - anaconda::openjdk=8.0.152
+  - conda-forge::openjdk=8.0.144
   - bioconda::fastqc=0.11.8
   - bioconda::adapterremoval=2.3.1
   - bioconda::adapterremovalfixprefix=0.0.5
