Replace all reference of strip with host removal (or equivalent)

Author: jfy133

.github/workflows/ci.yml

@@ -101,9 +101,9 @@ jobs:
       - name: MAPPER_BT2 Test running with BowTie2
         run: |
           nextflow run ${GITHUB_WORKSPACE} -profile test_tsv,docker --mapper 'bowtie2' --bt2_alignmode 'local' --bt2_sensitivity 'sensitive' --bt2n 1 --bt2l 16 --bt2_trim5 1 --bt2_trim3 1
-      - name: STRIP_FASTQ Run the basic pipeline with output unmapped reads as fastq
+      - name: HOST REMOVAL_FASTQ Run the basic pipeline with output unmapped reads as fastq
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -profile test_tsv,docker --strip_input_fastq
+          nextflow run ${GITHUB_WORKSPACE} -profile test_tsv,docker --hostremoval_input_fastq
       - name: BAM_FILTERING Run basic mapping pipeline with mapping quality filtering, and unmapped export
         run: |
           nextflow run ${GITHUB_WORKSPACE} -profile test_tsv,docker --run_bam_filtering --bam_mapping_quality_threshold 37  --bam_unmapped_type 'fastq'

CHANGELOG.md

@@ -51,6 +51,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 * [#516](https://github.com/nf-core/eager/issues/516) - Made bedtools not report out of memory exit code when warning of inconsistant FASTA/Bed entry names
 * [#504](https://github.com/nf-core/eager/issues/504) - Removed uninformative sexdeterrmine-snps plot from MultiQC report.
 * Nuclear contamination is now reported with the correct library names.
+* [#531](https://github.com/nf-core/eager/pull/531) - Renamed FASTQ stripping with host removal
 
 ### `Dependencies`
 

bin/extract_map_reads.py

@@ -36,8 +36,8 @@ def _get_args():
     parser.add_argument(
         '-m',
         dest='mode',
-        default='strip',
-        help='Read removal mode: remove reads (strip) or replace sequence by N (replace)'
+        default='remove',
+        help='Read removal mode: remove reads (remove) or replace sequence by N (replace)'
     )
     parser.add_argument(
         '-p',
@@ -179,27 +179,27 @@ def difference(list1, list2):
     return(fqd)
 
 
-def write_fq(fq_dict, fname, write_mode, strip_mode, proc):
+def write_fq(fq_dict, fname, write_mode, remove_mode, proc):
     """Write to fastq file
     Args:
         fq_dict(dict): fq_dict with unmapped read names as keys,
             unmapped/mapped (u|m), seq, and quality as values in a list
         fname(string) Path to output fastq file
         write_mode (str): 'rb' or 'r'
-        strip_mode (str): strip (remove read) or replace (replace read sequence) by Ns
+        remove_mode (str): remove (remove read) or replace (replace read sequence) by Ns
         proc(int) number of processes
     """
     fq_dict_keys = list(fq_dict.keys())
     if write_mode == 'wb':
         with xopen(fname, mode='wb', threads=proc) as fw:
             for fq_dict_key in fq_dict_keys:
                 wstring = ""
-                if strip_mode == 'strip':
+                if remove_mode == 'remove':
                     if fq_dict[fq_dict_key][0] == 'u':
                         wstring += f"@{fq_dict_key+fq_dict[fq_dict_key][1]}\n"
                         for i in fq_dict[fq_dict_key][2:]:
                             wstring += f"{i}\n"
-                elif strip_mode == 'replace':
+                elif remove_mode == 'replace':
                     # if unmapped, write all the read lines
                     if fq_dict[fq_dict_key][0] == 'u':
                         wstring += f"@{fq_dict_key+fq_dict[fq_dict_key][1]}\n"
@@ -217,12 +217,12 @@ def write_fq(fq_dict, fname, write_mode, strip_mode, proc):
         with open(fname, 'w') as fw:
             for fq_dict_key in fq_dict_keys:
                 wstring = ""
-                if strip_mode == 'strip':
+                if remove_mode == 'remove':
                     if fq_dict[fq_dict_key][0] == 'u':
                         wstring += f"@{fq_dict_key+fq_dict[fq_dict_key][1]}\n"
                         for i in fq_dict[fq_dict_key][2:]:
                             wstring += f"{i}\n"
-                elif strip_mode == 'replace':
+                elif remove_mode == 'replace':
                     # if unmapped, write all the read lines
                     if fq_dict[fq_dict_key][0] == 'u':
                         wstring += f"@{fq_dict_key+fq_dict[fq_dict_key][1]}\n"
@@ -238,8 +238,8 @@ def write_fq(fq_dict, fname, write_mode, strip_mode, proc):
                 fw.write(wstring)
 
 
-def check_strip_mode(mode):
-    if mode.lower() not in ['replace', 'strip']:
+def check_remove_mode(mode):
+    if mode.lower() not in ['replace', 'remove']:
         print(f"Mode must be {' or '.join(mode)}")
     return(mode.lower())
 
@@ -257,7 +257,7 @@ def check_strip_mode(mode):
     else:
         write_mode = "w"
 
-    strip_mode = check_strip_mode(MODE)
+    remove_mode = check_remove_mode(MODE)
     BAMFILE = pysam.AlignmentFile(BAM, 'r')
 
    # FORWARD OR SE FILE
@@ -270,7 +270,7 @@ def check_strip_mode(mode):
     # print(fq_dict_fwd)
     print(f"- Writing forward fq to {out_fwd}")
     write_fq(fq_dict=fq_dict_fwd, fname=out_fwd,
-             write_mode=write_mode, strip_mode=strip_mode, proc=PROC)
+             write_mode=write_mode, remove_mode=remove_mode, proc=PROC)
 
     # REVERSE FILE
     if IN_REV:
@@ -284,4 +284,4 @@ def check_strip_mode(mode):
         fq_dict_rev = get_mapped_reads(fqd_rev, mapped_reads)
         print(f"- Writing reverse fq to {out_rev}")
         write_fq(fq_dict=fq_dict_rev, fname=out_rev,
-                 write_mode=write_mode, strip_mode=strip_mode, proc=PROC)
+                 write_mode=write_mode, remove_mode=remove_mode, proc=PROC)

docs/usage.md

@@ -10,30 +10,189 @@
   - [Running the pipeline](#running-the-pipeline)
     - [Updating the pipeline](#updating-the-pipeline)
     - [Mandatory Arguments](#mandatory-arguments)
+      - [`-profile`](#-profile)
+      - [`--input`](#--input)
+        - [Direct Input Method](#direct-input-method)
+        - [TSV Input Method](#tsv-input-method)
+      - [`--bam`](#--bam)
+      - [`--single_stranded`](#--single_stranded)
+      - [`--colour_chemistry`](#--colour_chemistry)
+      - [`--fasta`](#--fasta)
+      - [`--genome` (using iGenomes)](#--genome-using-igenomes)
     - [Output Directories](#output-directories)
+      - [`--outdir`](#--outdir)
+      - [`-w / -work-dir`](#-w---work-dir)
     - [Optional Reference Options](#optional-reference-options)
+      - [`--large_ref`](#--large_ref)
+      - [`--save_reference`](#--save_reference)
+      - [`--bwa_index`](#--bwa_index)
+      - [`--seq_dict`](#--seq_dict)
+      - [`--fasta_index`](#--fasta_index)
     - [Other run specific parameters](#other-run-specific-parameters)
+      - [`-r`](#-r)
+      - [`--max_memory`](#--max_memory)
+      - [`--max_time`](#--max_time)
+      - [`--max_cpus`](#--max_cpus)
+      - [`--email`](#--email)
+      - [`--plaintext_email`](#--plaintext_email)
+      - [`-name`](#-name)
+      - [`-resume`](#-resume)
+      - [`-c`](#-c)
+      - [`--monochrome_logs`](#--monochrome_logs)
+      - [`--multiqc_config`](#--multiqc_config)
+      - [`--custom_config_version`](#--custom_config_version)
   - [Adjustable parameters for nf-core/eager](#adjustable-parameters-for-nf-coreeager)
     - [Step skipping parameters](#step-skipping-parameters)
+      - [`--skip_fastqc`](#--skip_fastqc)
+      - [`--skip_adapterremoval`](#--skip_adapterremoval)
+      - [`--skip_preseq`](#--skip_preseq)
+      - [`--skip_deduplication`](#--skip_deduplication)
+      - [`--skip_damage_calculation`](#--skip_damage_calculation)
+      - [`--skip_qualimap`](#--skip_qualimap)
     - [BAM Conversion Options](#bam-conversion-options)
+      - [`--run_convertinputbam`](#--run_convertinputbam)
     - [Complexity Filtering Options](#complexity-filtering-options)
+      - [`--complexity_filter_poly_g`](#--complexity_filter_poly_g)
+      - [`--complexity_filter_poly_g_min`](#--complexity_filter_poly_g_min)
     - [Adapter Clipping and Merging Options](#adapter-clipping-and-merging-options)
+      - [`--clip_forward_adaptor`](#--clip_forward_adaptor)
+      - [`--clip_reverse_adaptor`](#--clip_reverse_adaptor)
+      - [`--clip_readlength`](#--clip_readlength)
+      - [`--clip_min_read_quality`](#--clip_min_read_quality)
+      - [`--clip_min_adap_overlap`](#--clip_min_adap_overlap)
+      - [`--skip_collapse`](#--skip_collapse)
+      - [`--skip_trim`](#--skip_trim)
+      - [`--preserve5p`](#--preserve5p)
+      - [`--mergedonly`](#--mergedonly)
     - [Read Mapping Parameters](#read-mapping-parameters)
-    - [Mapped Reads Stripping](#mapped-reads-stripping)
+      - [`--mapper`](#--mapper)
+      - [BWA (default)](#bwa-default)
+        - [`--bwaalnn`](#--bwaalnn)
+        - [`--bwaalnk`](#--bwaalnk)
+        - [`--bwaalnl`](#--bwaalnl)
+      - [CircularMapper](#circularmapper)
+        - [`--circularextension`](#--circularextension)
+        - [`--circulartarget`](#--circulartarget)
+        - [`--circularfilter`](#--circularfilter)
+      - [Bowtie2](#bowtie2)
+        - [`--bt2_alignmode`](#--bt2_alignmode)
+        - [`--bt2_sensitivity`](#--bt2_sensitivity)
+        - [`--bt2n`](#--bt2n)
+        - [`--bt2l`](#--bt2l)
+        - [`-bt2_trim5`](#-bt2_trim5)
+        - [`-bt2_trim3`](#-bt2_trim3)
+    - [Mapped Reads Host Removal](#mapped-reads-host-removal)
+      - [`--hostremoval_input_fastq`](#--hostremoval_input_fastq)
+      - [`--hostremoval_mode`](#--hostremoval_mode)
     - [Read Filtering and Conversion Parameters](#read-filtering-and-conversion-parameters)
+      - [`--run_bam_filtering`](#--run_bam_filtering)
+      - [`--bam_unmapped_type`](#--bam_unmapped_type)
+      - [`--bam_mapping_quality_threshold`](#--bam_mapping_quality_threshold)
+      - [`bam_filter_minreadlength`](#bam_filter_minreadlength)
     - [Read DeDuplication Parameters](#read-deduplication-parameters)
+      - [`--dedupper`](#--dedupper)
+      - [`--dedup_all_merged`](#--dedup_all_merged)
     - [Library Complexity Estimation Parameters](#library-complexity-estimation-parameters)
+      - [`--preseq_step_size`](#--preseq_step_size)
     - [DNA Damage Assessment Parameters](#dna-damage-assessment-parameters)
+      - [`--udg_type`](#--udg_type)
+      - [`--damageprofiler_length`](#--damageprofiler_length)
+      - [`--damageprofiler_threshold`](#--damageprofiler_threshold)
+      - [`--damageprofiler_yaxis`](#--damageprofiler_yaxis)
+      - [`--run_pmdtools`](#--run_pmdtools)
+      - [`--pmdtools_range`](#--pmdtools_range)
+      - [`--pmdtools_threshold`](#--pmdtools_threshold)
+      - [`--pmdtools_reference_mask`](#--pmdtools_reference_mask)
+      - [`--pmdtools_max_reads`](#--pmdtools_max_reads)
     - [BAM Trimming Parameters](#bam-trimming-parameters)
+      - [`--run_trim_bam`](#--run_trim_bam)
+      - [`--bamutils_clip_half_udg_left` / `--bamutils_clip_half_udg_right`](#--bamutils_clip_half_udg_left----bamutils_clip_half_udg_right)
+      - [`--bamutils_clip_none_udg_left` / `--bamutils_clip_none_udg_right`](#--bamutils_clip_none_udg_left----bamutils_clip_none_udg_right)
+      - [`--bamutils_softclip`](#--bamutils_softclip)
     - [Captured Library Parameters](#captured-library-parameters)
+      - [`--snpcapture` false](#--snpcapture-false)
+      - [`--bedfile`](#--bedfile)
     - [Feature Annotation Statistics](#feature-annotation-statistics)
+      - [`--run_bedtools_coverage`](#--run_bedtools_coverage)
+      - [`--anno_file`](#--anno_file)
     - [Genotyping Parameters](#genotyping-parameters)
+      - [`--run_genotyping`](#--run_genotyping)
+      - [`--genotyping_tool`](#--genotyping_tool)
+      - [`--genotyping_source`](#--genotyping_source)
+      - [`--gatk_ug_jar`](#--gatk_ug_jar)
+      - [`--gatk_call_conf`](#--gatk_call_conf)
+      - [`--gatk_ploidy`](#--gatk_ploidy)
+      - [`--gatk_dbsnp`](#--gatk_dbsnp)
+      - [`--gatk_ug_out_mode`](#--gatk_ug_out_mode)
+      - [`--gatk_hc_out_mode`](#--gatk_hc_out_mode)
+      - [`--gatk_ug_genotype_model`](#--gatk_ug_genotype_model)
+      - [`--gatk_hc_emitrefconf`](#--gatk_hc_emitrefconf)
+      - [`--gatk_ug_keep_realign_bam`](#--gatk_ug_keep_realign_bam)
+      - [`--gatk_downsample`](#--gatk_downsample)
+      - [`--gatk_ug_gatk_ug_defaultbasequalities`](#--gatk_ug_gatk_ug_defaultbasequalities)
+      - [`--freebayes_C`](#--freebayes_c)
+      - [`--freebayes_g`](#--freebayes_g)
+      - [`--freebayes_p`](#--freebayes_p)
+      - [`--pileupcaller_bedfile`](#--pileupcaller_bedfile)
+      - [`--pileupcaller_snpfile`](#--pileupcaller_snpfile)
+      - [`--pileupcaller_method`](#--pileupcaller_method)
+      - [`--angsd_glmodel`](#--angsd_glmodel)
+      - [`--angsd_glformat`](#--angsd_glformat)
+      - [`--angsd_createfasta`](#--angsd_createfasta)
+      - [`--angsd_fastamethod`](#--angsd_fastamethod)
+      - [`--pileupcaller_transitions_mode`](#--pileupcaller_transitions_mode)
     - [Consensus Sequence Generation](#consensus-sequence-generation)
+      - [`--run_vcf2genome`](#--run_vcf2genome)
+      - [`--vcf2genome_outfile`](#--vcf2genome_outfile)
+      - [`--vcf2genome_header`](#--vcf2genome_header)
+      - [`--vcf2genome_minc`](#--vcf2genome_minc)
+      - [`--vcf2genome_minq`](#--vcf2genome_minq)
+      - [`--vcf2genome_minfreq`](#--vcf2genome_minfreq)
     - [Mitochondrial to Nuclear Ratio](#mitochondrial-to-nuclear-ratio)
+      - [`--run_mtnucratio`](#--run_mtnucratio)
+      - [`--mtnucratio_header`](#--mtnucratio_header)
     - [SNP Table Generation](#snp-table-generation)
+      - [`--run_multivcfanalyzer`](#--run_multivcfanalyzer)
+      - [`--write_allele_frequencies`](#--write_allele_frequencies)
+      - [`--min_genotype_quality`](#--min_genotype_quality)
+      - [`--min_base_coverage`](#--min_base_coverage)
+      - [`--min_allele_freq_hom`](#--min_allele_freq_hom)
+      - [`--min_allele_freq_het`](#--min_allele_freq_het)
+      - [`--additional_vcf_files`](#--additional_vcf_files)
+      - [`--reference_gff_annotations`](#--reference_gff_annotations)
+      - [`--reference_gff_exclude`](#--reference_gff_exclude)
+      - [`--snp_eff_results`](#--snp_eff_results)
     - [Human Sex Determination](#human-sex-determination)
+      - [`--run_sexdeterrmine`](#--run_sexdeterrmine)
+      - [`--sexdeterrmine_bedfile`](#--sexdeterrmine_bedfile)
     - [Human Nuclear Contamination](#human-nuclear-contamination)
+      - [`--run_nuclear_contamination`](#--run_nuclear_contamination)
+      - [`--contamination_chrom_name`](#--contamination_chrom_name)
     - [Metagenomic Screening](#metagenomic-screening)
+      - [`--run_metagenomic_screening`](#--run_metagenomic_screening)
+      - [`--metagenomic_tool`](#--metagenomic_tool)
+      - [`--metagenomic_min_support_reads`](#--metagenomic_min_support_reads)
+      - [`--database`](#--database)
+      - [`--percent_identity`](#--percent_identity)
+      - [`--malt_mode`](#--malt_mode)
+      - [`--malt_alignment_mode`](#--malt_alignment_mode)
+      - [`--malt_top_percent`](#--malt_top_percent)
+      - [`--malt_min_support_mode`](#--malt_min_support_mode)
+      - [`--malt_min_support_percent`](#--malt_min_support_percent)
+      - [`--malt_max_queries`](#--malt_max_queries)
+      - [`--malt_memory_mode`](#--malt_memory_mode)
+      - [`--run_maltextract`](#--run_maltextract)
+      - [`--maltextract_taxon_list`](#--maltextract_taxon_list)
+      - [`--maltextract_ncbifiles`](#--maltextract_ncbifiles)
+      - [`--maltextract_filter`](#--maltextract_filter)
+      - [`--maltextract_toppercent`](#--maltextract_toppercent)
+      - [`--maltextract_destackingoff`](#--maltextract_destackingoff)
+      - [`--maltextract_downsamplingoff`](#--maltextract_downsamplingoff)
+      - [`--maltextract_duplicateremovaloff`](#--maltextract_duplicateremovaloff)
+      - [`--maltextract_matches`](#--maltextract_matches)
+      - [`--maltextract_megansummary`](#--maltextract_megansummary)
+      - [`--maltextract_percentidentity`](#--maltextract_percentidentity)
+      - [`maltextract_topalignment`](#maltextract_topalignment)
   - [Clean up](#clean-up)
 
 ## General Nextflow info
@@ -669,23 +828,23 @@ Number of bases to trim of 5' (left) end of read prior alignment. Maybe useful w
 
 Number of bases to trim of 3' (right) end of read prior alignment. Maybe useful when left-over sequencing artefacts of in-line barcodes present Default: 0.
 
-### Mapped Reads Stripping
+### Mapped Reads Host Removal
 
 These parameters are used for removing mapped reads from the original input FASTQ files, usually in the context of uploading the original FASTQ files to a public read archive (NCBI SRA/EBI ENA).
 
 These flags will produce FASTQ files almost identical to your input files, except that reads with the same read ID as one found in the mapped bam file, are either removed or 'masked' (every base replaced with Ns).
 
 This functionality allows you to provide other researchers who wish to re-use your data to apply their own adapter removal/read merging procedures, while maintaining anonyminity for sample donors - for example with microbiome research.
 
-If using TSV input, stripping is performed library, i.e. after lane merging.
+If using TSV input, mapped read removal is performed library, i.e. after lane merging.
 
-#### `--strip_input_fastq`
+#### `--hostremoval_input_fastq`
 
 Create pre-Adapter Removal FASTQ files without reads that mapped to reference (e.g. for public upload of privacy sensitive non-host data)
 
-#### `--strip_mode`
+#### `--hostremoval_mode`
 
-Read removal mode. Strip mapped reads completely (`'strip'`) or just replace mapped reads sequence by N (`'replace'`)
+Read removal mode. Completely remove mapped reads from the file(s) (`'remove'`) or just replace mapped reads sequence by N (`'replace'`)
 
 ### Read Filtering and Conversion Parameters