feat: support for testing qtl reheat idempotence

- add `--submit` option to `qtl reheat`, to submit slurm jobs
  right away, or not
- add `util/run-here.sh` to run a list of jobs on the local interactive
  node; for development testing only, not for chefs

Author: c-dilks

bin/qtl-reheat

@@ -16,6 +16,10 @@ declare -A REHEAT_METHODS=(
 
 # default options
 dataset=train
+declare -A modes
+for key in submit; do
+  modes[$key]=false
+done
 
 # usage
 sep="================================================================"
@@ -37,6 +41,8 @@ $(for key in "${!REHEAT_METHODS[@]}"; do printf "%24s %-11s %-s\n" "" "$key" "${
   OPTIONAL OPTIONS:
      -d [DATASET]      unique name for this dataset
                        default: $dataset
+     --submit          submit the slurm jobs, rather than just
+                       printing the \`sbatch\` command
   """ >&2
 }
 if [ $# -lt 1 ]; then
@@ -48,7 +54,7 @@ fi
 inputDir=""
 outputDir=""
 cmd=""
-while getopts "i:o:c:d:h" opt; do
+while getopts "i:o:c:d:h-:" opt; do
   case $opt in
     i) inputDir=$OPTARG ;;
     o) outputDir=$OPTARG ;;
@@ -61,6 +67,12 @@ while getopts "i:o:c:d:h" opt; do
       usage
       exit 101
       ;;
+    -)
+      for key in "${!modes[@]}"; do
+        [ "$key" == "$OPTARG" ] && modes[$OPTARG]=true && break
+      done
+      [ -z "${modes[$OPTARG]-}" ] && printError "unknown option --$OPTARG" && exit 100
+      ;;
     *) exit 100 ;;
   esac
 done
@@ -159,6 +171,12 @@ EOF
 echo """
 SLURM SCRIPT: $slurmScript
 JOB LIST:     $jobList
-Now submitting!
 """
-sbatch $slurmScript
+if ${modes['submit']}; then
+  echo "Now submitting!"
+  sbatch $slurmScript
+else
+  echo """Run this command to submit:
+  sbatch $slurmScript
+  """
+fi

util/run-here.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+set -euo pipefail
+offset=0
+usage() {
+  echo """
+  Run jobs from a slurm submission script's job list on
+  the current interactive node; be nice and keep NUM_JOBS
+  low, since this is only meant for rapid testing
+
+  USAGE: $0 [JOB_LIST] [LOG_DIR] [NUM_JOBS] [OFFSET]
+    JOB_LIST  file with job scripts, one per line
+    LOG_DIR   directory for log files (clobbered)
+    NUM_JOBS  the number of parallel jobs to run
+    OFFSET    offset of the first job; NUM_JOBS
+              consecutive lines will be used
+              default: $offset
+    """
+}
+
+if [ $# -lt 3 ]; then
+  usage
+  exit 2
+fi
+job_list=$1
+log_dir=$2
+num_jobs=$3
+[ $# -ge 4 ] && offset=$4
+
+if [ ! -f "$job_list" ]; then
+  echo "ERROR: File '$job_list' not found"
+  exit 1
+fi
+
+if [ $num_jobs -gt 16 ]; then
+  echo "ERROR: too many jobs!"
+  exit 1
+fi
+
+mkdir -p $log_dir
+
+i=0
+echo "SUBMITTING:"
+tail -n +$((offset + 1)) $job_list | head -n $num_jobs | while IFS= read -r cmd; do
+  echo "JOB $i: $cmd"
+  # $cmd > $log_dir/job.$i.out 2> $log_dir/job.$i.err &
+  i=$((i+1))
+done
+echo """
+JOBS SUBMITTED.
+- They are running in the backround
+- Monitor with \`htop -u $(whoami)\`
+- Logs written to \`$log_dir\`
+"""