Skip to content

Commit 1c37caa

Browse files
aidaanvaaidaanva
authored
Merge branch 'dev' into mva
2 parents a0a5c37 + c7dc65b commit 1c37caa

26 files changed

Lines changed: 2200 additions & 762 deletions

‎.github/workflows/ci.yml‎

Lines changed: 28 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,13 @@
1-
name: nf-core CI
21
# This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors
2+
name: nf-core CI
33
on:
44
push:
55
branches:
6-
- dev
6+
- "dev"
77
pull_request:
8+
branches:
9+
- "dev"
10+
- "master"
811
release:
912
types: [published]
1013
workflow_dispatch:
@@ -15,16 +18,31 @@ env:
1518
NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity
1619

1720
concurrency:
18-
group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}"
21+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
1922
cancel-in-progress: true
2023

2124
jobs:
25+
define_nxf_versions:
26+
name: Choose nextflow versions to test against depending on target branch
27+
runs-on: ubuntu-latest
28+
outputs:
29+
matrix: ${{ steps.nxf_versions.outputs.matrix }}
30+
steps:
31+
- id: nxf_versions
32+
run: |
33+
if [[ "${{ github.event_name }}" == "pull_request" && "${{ github.base_ref }}" == "dev" && "${{ matrix.NXF_VER }}" != "latest-everything" ]]; then
34+
echo matrix='["latest-everything"]' | tee -a $GITHUB_OUTPUT
35+
else
36+
echo matrix='["latest-everything", "23.10.0"]' | tee -a $GITHUB_OUTPUT
37+
fi
38+
2239
test:
2340
name: "Run pipeline with test data (${{ matrix.NXF_VER }} | ${{ matrix.test_name }} | ${{ matrix.profile }})"
2441
# Only run on push if this is the nf-core dev branch (merged PRs)
2542
if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/eager') }}"
2643
runs-on: ubuntu-latest
2744
strategy:
45+
fail-fast: false
2846
matrix:
2947
NXF_VER:
3048
- "24.04.2"
@@ -35,6 +53,12 @@ jobs:
3553
- "singularity"
3654
test_name:
3755
- "test"
56+
- "test_minimal"
57+
- "test_modern"
58+
- "test_microbial"
59+
- "test_shortdna"
60+
- "test_humanpopgen"
61+
- "test_long"
3862
isMaster:
3963
- ${{ github.base_ref == 'master' }}
4064
# Exclude conda and singularity on dev
@@ -43,16 +67,6 @@ jobs:
4367
profile: "conda"
4468
- isMaster: false
4569
profile: "singularity"
46-
PARAMS:
47-
- " --preprocessing_tool fastp --preprocessing_adapterlist 'https://github.com/nf-core/test-datasets/raw/modules/data/delete_me/fastp/adapters.fasta'"
48-
- " --preprocessing_tool adapterremoval --preprocessing_adapterlist 'https://github.com/nf-core/test-datasets/raw/modules/data/delete_me/adapterremoval/adapterremoval_adapterlist.txt' --sequencing_qc_tool falco --run_genotyping --genotyping_tool 'freebayes' --genotyping_source 'raw'"
49-
- " --mapping_tool bwamem --run_mapdamage_rescaling --run_pmd_filtering --run_trim_bam --run_genotyping --genotyping_tool 'ug' --genotyping_source 'trimmed'"
50-
- " --mapping_tool bowtie2 --damagecalculation_tool mapdamage --damagecalculation_mapdamage_downsample 100 --run_genotyping --genotyping_tool 'hc' --genotyping_source 'raw'"
51-
- " --mapping_tool mapad"
52-
- " --mapping_tool circularmapper --skip_preprocessing --convert_inputbam --fasta_circular_target 'NC_007596.2' --fasta_circularmapper_elongationfactor 500"
53-
- "_humanbam --run_mtnucratio --run_contamination_estimation_angsd --snpcapture_bed 'https://raw.githubusercontent.com/nf-core/test-datasets/eager/reference/Human/1240K.pos.list_hs37d5.0based.bed.gz' --run_genotyping --genotyping_tool 'pileupcaller' --genotyping_source 'raw'"
54-
- "_humanbam --run_sexdeterrmine --run_genotyping --genotyping_tool 'angsd' --genotyping_source 'raw'"
55-
- "_multiref" ## TODO add damage manipulation here instead once it goes multiref
5670
steps:
5771
- name: Check out pipeline code
5872
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
@@ -95,4 +109,4 @@ jobs:
95109
- name: "Run pipeline with test data ${{ matrix.NXF_VER }} | ${{ matrix.test_name }} | ${{ matrix.profile }}"
96110
continue-on-error: ${{ matrix.NXF_VER == 'latest-everything' }}
97111
run: |
98-
nextflow run ${GITHUB_WORKSPACE} -profile ${{matrix.profile}},${{ matrix.test_name }}${{ matrix.PARAMS }} --outdir ./results
112+
nextflow run ${GITHUB_WORKSPACE} -profile ${{matrix.profile}},${{ matrix.test_name }} --outdir ./results

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,3 +8,4 @@ testing*
88
*.pyc
99
null/
1010
.nf-test*
11+

‎conf/modules.config‎

Lines changed: 378 additions & 472 deletions
Large diffs are not rendered by default.

‎conf/test.config‎

Lines changed: 33 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -19,30 +19,48 @@ process {
1919
}
2020

2121
params {
22-
config_profile_name = 'Test profile'
23-
config_profile_description = 'Minimal test dataset to check pipeline function'
22+
config_profile_name = 'Test profile'
23+
config_profile_description = 'Minimal test dataset to check pipeline function'
2424

2525
// Input data
26-
// TODO nf-core: Specify the paths to your test data on nf-core/test-datasets
27-
// TODO nf-core: Give any required params for the test so that command line flags are not needed
28-
input = params.pipelines_testdata_base_path + 'eager/testdata/Mammoth/samplesheet_v3.tsv'
26+
input = params.pipelines_testdata_base_path + 'eager/testdata/Mammoth/samplesheet_v3.tsv'
2927

3028
// Genome references
31-
fasta = params.pipelines_testdata_base_path + 'eager/reference/Mammoth/Mammoth_MT_Krause.fasta'
29+
fasta = params.pipelines_testdata_base_path + 'eager/reference/Mammoth/Mammoth_MT_Krause.fasta'
30+
31+
// Preprocessing
32+
preprocessing_tool = 'adapterremoval'
3233

3334
// Sharding FASTQ
34-
run_fastq_sharding = true
35-
fastq_shard_size = 5000
35+
run_fastq_sharding = true
36+
fastq_shard_size = 5000
37+
38+
// Mapping
39+
mapping_tool = 'bwaaln'
40+
skip_qualimap = false
3641

3742
// BAM filtering
38-
run_bamfiltering = true
39-
bamfiltering_minreadlength = 30
40-
bamfiltering_mappingquality = 37
43+
run_bamfiltering = true
44+
bamfiltering_minreadlength = 30
45+
bamfiltering_mappingquality = 37
46+
deduplication_tool = 'markduplicates'
47+
bamfiltering_savefilteredbams = true
48+
49+
// PreSeq
50+
mapstats_preseq_mode = 'c_curve'
51+
52+
// Damage calculation
53+
damagecalculation_tool = 'damageprofiler'
54+
skip_qualimap = false
4155

42-
// Map Stats
43-
run_bedtools_coverage = true
44-
mapstats_bedtools_featurefile = params.pipelines_testdata_base_path + 'eager/reference/Mammoth/Mammoth_MT_Krause.gff3'
56+
// Genotyping
57+
run_genotyping = true
58+
genotyping_source = 'raw'
59+
genotyping_tool = 'ug'
4560

4661
// Metagenomic screening
47-
run_metagenomics = false
62+
run_metagenomics = true
63+
metagenomics_profiling_tool = 'metaphlan'
64+
metagenomics_profiling_database = params.pipelines_testdata_base_path + 'eager/databases/metaphlan/metaphlan4_database.tar.gz'
65+
metagenomics_run_postprocessing = true
4866
}

‎conf/test_humanbam.config‎

Lines changed: 0 additions & 55 deletions
This file was deleted.

‎conf/test_humanpopgen.config‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
/*
2+
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
3+
Nextflow config file for running minimal tests
4+
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
5+
Defines input files and everything required to run a fast and simple pipeline test.
6+
7+
Use as follows:
8+
nextflow run nf-core/eager -profile test_humanpopgen,<docker/singularity> --outdir <OUTDIR>
9+
10+
----------------------------------------------------------------------------------------
11+
*/
12+
13+
process {
14+
resourceLimits = [
15+
cpus: 4,
16+
memory: '15.GB',
17+
time: '1.h',
18+
]
19+
20+
// To avoid pipeline failure due to not having X reads and to not have overcrowded datasets in the test
21+
withName: ANGSD_CONTAMINATION {
22+
errorStrategy = { task.exitStatus in [134] ? 'ignore' : 'finish' }
23+
}
24+
}
25+
26+
params {
27+
config_profile_name = 'Test human popgen profile'
28+
config_profile_description = 'Minimal test dataset to check pipeline function'
29+
30+
// Input data
31+
input = params.pipelines_testdata_base_path + 'eager/testdata/Human/human_design_bam_eager3.tsv'
32+
33+
// Genome references
34+
fasta = params.pipelines_testdata_base_path + 'eager/reference/Human/hs37d5_chr21-MT.fa.gz'
35+
snpcature_bed = params.pipelines_testdata_base_path + 'eager/reference/Human/1240K.pos.list_hs37d5.0based.bed.gz'
36+
37+
// Mapping
38+
mapping_tool = 'bowtie2'
39+
convert_inputbam = true
40+
41+
// BAM filtering
42+
run_bamfiltering = true
43+
bamfiltering_minreadlength = 30
44+
bamfiltering_mappingquality = 37
45+
46+
// Damage
47+
damagecalculation_tool = 'mapdamage'
48+
damagecalculation_mapdamage_downsample = 10000
49+
run_trim_bam = true
50+
51+
// Contamination
52+
run_mtnucratio = true
53+
run_contamination_estimation_angsd = true
54+
55+
// Genotyping
56+
genotyping_tool = 'pileupcaller'
57+
run_genotyping = true
58+
genotyping_source = 'trimmed'
59+
genotyping_pileupcaller_bedfile = params.pipelines_testdata_base_path + 'eager/reference/Human/1240K.pos.list_hs37d5.0based.bed.gz'
60+
genotyping_pileupcaller_snpfile = params.pipelines_testdata_base_path + 'eager/reference/Human/1240K_covered_in_JK2067_downsampled_s0.1.numeric_chromosomes.snp'
61+
62+
//Sex Determination
63+
run_sexdeterrmine = true
64+
sexdeterrmine_bedfile = params.pipelines_testdata_base_path + 'eager/reference/Human/1240K.pos.list_hs37d5.0based.bed.gz'
65+
66+
// Metagenomics
67+
run_metagenomics = true
68+
metagenomics_complexity_tool = 'prinseq'
69+
metagenomics_profiling_tool = 'kraken2'
70+
metagenomics_profiling_database = params.pipelines_testdata_base_path + 'eager/databases/kraken/eager_test.tar.gz'
71+
metagenomics_run_postprocessing = true
72+
}

‎conf/test_kraken2.config‎

Lines changed: 0 additions & 35 deletions
This file was deleted.

‎conf/test_krakenuniq.config‎

Lines changed: 0 additions & 36 deletions
This file was deleted.

‎conf/test_long.config‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
/*
2+
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
3+
Nextflow config file for running minimal tests
4+
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
5+
Defines input files and everything required to run a fast and simple pipeline test.
6+
7+
Use as follows:
8+
nextflow run nf-core/eager -profile test_long,<docker/singularity> --outdir <OUTDIR>
9+
10+
----------------------------------------------------------------------------------------
11+
*/
12+
13+
process {
14+
resourceLimits = [
15+
cpus: 4,
16+
memory: '15.GB',
17+
time: '3.h',
18+
]
19+
}
20+
21+
includeConfig 'test_minimal.config'
22+
23+
params {
24+
config_profile_name = 'Test long profile'
25+
config_profile_description = 'Minimal test dataset to check pipeline function'
26+
27+
run_mapdamage_rescaling = true
28+
29+
// Genotypying
30+
run_genotyping = true
31+
genotyping_source = 'rescaled'
32+
genotyping_tool = 'freebayes'
33+
}

0 commit comments

Comments
 (0)
⚡