Merge pull request nf-core#606 from MaxUlysse/submodule

add test data as a submodule

Author: maxulysse

.gitignore

@@ -2,7 +2,6 @@ Annotation/
 Preprocessing/
 References/
 Reports/
-Sarek-data/
 VariantCalling/
 work/
 .*swp

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "Sarek-data"]
+	path = Sarek-data
+	url = https://github.com/SciLifeLab/Sarek-data.git

Sarek-data

@@ -20,6 +20,11 @@ All Reference files are already stored in `bianca`.
 
 # Clone the repository
 > git clone https://github.com/SciLifeLab/Sarek.git
+
+# If you want to include the test data, you should use --recursive
+> git clone --recursive https://github.com/SciLifeLab/Sarek.git
+
+# Go to the newly created directory
 > cd Sarek
 
 # It is also possible to checkout a specific version using
@@ -28,6 +33,20 @@ All Reference files are already stored in `bianca`.
 # Use our script to make an archive to send to bianca
 > ./scripts/makeSnapshot.sh
 
+# Or you can also include the test data in this archive using git-archive-all
+# Install pip
+> curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
+> python get-pip.py
+
+# If it fails due to permission, you could consider using
+> python get-pip.py --user
+
+# Install git-archive-all using pip
+> pip install git-archive-all
+# If you used --user before, you might want to do that here too
+> pip install git-archive-all --user
+> ./scripts/makeSnapshot.sh --include-test-data
+
 # You will get this message in your terminal
 Wrote Sarek-[snapID].tar.gz
 

doc/INSTALL_BIANCA.md

@@ -7,16 +7,13 @@ One simple bash script is available, which will pull the Sarek-data repository a
 
 Such tests are used in our Continuous Integration with Travis. You can perform the same tests to familiarize yourself with the workflow.
 
-### Testing with Singularity
+## Testing with Singularity
 For testing with Docker, just replace `singularity` with `docker` in every occurence.
 ```bash
-# Dowload Sarek
-git clone https://github.com/SciLifeLab/Sarek Sarek-test
+# Dowload Sarek and the test data
+git clone --recursive https://github.com/SciLifeLab/Sarek Sarek-test
 cd Sarek-test
 
-# Dowload Sarek test data
-git clone https://github.com/SciLifeLab/Sarek-data
-
 # Build the references for the test data
 nextflow run buildReferences.nf --outDir References/smallGRCh37 \
   --refDir Sarek-data/reference --genome smallGRCh37 --tag latest \
@@ -77,6 +74,13 @@ nextflow run annotate.nf --tools snpEFF,VEP \
 nextflow run runMultiQC.nf -profile singularity
 ```
 
+## Testing on a secure cluster
+On a secure cluster as bianca, with no internet access, you will need to download and transfer Sarek and the test data first.
+
+Follow the [installation guide for `bianca`](https://github.com/SciLifeLab/Sarek/blob/master/doc/INSTALL_BIANCA.md).
+
+And then start the test at the `Build the references for the test data` step.
+
 ## Usage
 
 Four optional arguments are supported:

doc/TESTS.md

@@ -208,12 +208,12 @@ class SarekUtils {
 
   // Sarek ascii art
   static def sarek_ascii() {
-    println "    ____        _____                _     "
-    println "  .' _  `.     / ____|              | |    "
-    println " /  |\\`-_ \\   | (___   __ _ _ __ ___| | __ "
-    println "|   | \\  `-|   \\___ \\ / _` | '__/ __| |/ / "
-    println " \\ |   \\  /    ____) | (_| | | |  __|   <  "
-    println "  `|____\\'    |_____/ \\__,_|_|  \\___|_|\\_\\ "
+    println "    ____        _____               _    "
+    println "  .' _  `.     / ____|             | |   "
+    println " /  |\\`-_ \\   | (___  ___  _ __ __ | | __"
+    println "|   | \\  `-|   \\___ \\/__ \\| ´__/ _\\| |/ /"
+    println " \\ |   \\  /    ____) | __ | | |  __|   < "
+    println "  `|____\\'    |_____/\\____|_|  \\__/|_|\\_\\"
   }
 
 }

lib/SarekUtils.groovy

@@ -1,5 +1,30 @@
 #!/bin/bash
 set -euo pipefail
+
+TEST=false
 NAME=Sarek-$(git describe --tags)
-git archive HEAD --prefix=$NAME/ | gzip > $NAME.tar.gz
+
+while [[ $# -gt 0 ]]
+do
+  key="$1"
+  case $key in
+    -i|-t|--include-test-data)
+    TEST=true
+    shift # past argument
+    ;;
+    *) # unknown option
+    shift # past argument
+    ;;
+  esac
+done
+
+if [[ $TEST == false ]]
+then
+  echo "Archiving Sarek without test data"
+  git archive HEAD --prefix=$NAME/ | gzip > $NAME.tar.gz
+else
+  echo "Archiving Sarek with test data"
+  git-archive-all --prefix=$NAME/ --force-submodules $NAME.tar.gz
+fi
+
 echo "Wrote $NAME.tar.gz"

scripts/makeSnapshot.sh

@@ -62,10 +62,10 @@ function clean_repo() {
 # Build references only for smallGRCh37
 if [[ $GENOME == smallGRCh37 ]] && [[ $TEST != BUILDCONTAINERS ]] && [[ BUILD ]]
 then
-  if [[ ! -d Sarek-data ]]
+  if [[ -z "$(ls -A Sarek-data)" ]]
   then
-    echo "$(tput setaf 1)Cloning Sarek-data repository$(tput sgr0)"
-    git clone https://github.com/SciLifeLab/Sarek-data.git
+    git submodule init
+    git submodule update
   fi
   if [[ ! -d References ]]
   then