check_GenPipes.sh

#!/usr/bin/env bash

THIS_SCRIPT=$(basename "$0")

usage() {
  echo "script usage: $THIS_SCRIPT -h [-c cluster] [-j genpipes_json] [-r readset_file] [-l job_list]"
  echo "Usage:"
  echo " -h                               Display this help message."
  echo " -c <cluster>                     Cluster name on which the check is expected /!\ MANDATORY /!\."
  echo " -j <genpipes_json>               json file generated by GenPipes /!\ MANDATORY /!\."
  echo " -r <readset_file>                Readset File used to submit GenPipes /!\ MANDATORY /!\."
  echo " -l <job_list>                    job_list file generated by GenPipes /!\ MANDATORY /!\."
  exit 1
  }

genpipes_tagging() {
  echo "-> Tagging GenPipes json..."
  module load mugqic/python/3.11.1
  # shellcheck disable=SC2086
  $MOH_path/moh_automation/moh_automation_main/genpipes_deliverables_metrics_tagging.py -i $1 -o ${1/.json/_tagged.json}
  module unload mugqic/python/3.11.1
}

genpipes_ingesting() {
  echo "-> Ingesting GenPipes json..."
  # shellcheck disable=SC1091,SC2086
  source $MOH_path/project_tracking_cli/venv/bin/activate
  # shellcheck disable=SC2086
  pt-cli ingest genpipes --input-json $1
  deactivate
}

genpipes_transfer() {
  genpipes_run=$(basename $(dirname "$readset_file"))
  transfer_log=$(dirname "$readset_file")/transfer.log
  echo "-> Transferring GenPipes run $genpipes_run..."
  {
    if [ -z "$3" ]; then
      # If $3 (protocol) is empty, omit the -t option
      nohup "$MOH_path/moh_automation/moh_automation_main/transfer_GenPipes.sh" -r "$1" -p "$2" 2>&1 &
    else
      # If $3 (protocol) is not empty, include the -t option
      nohup "$MOH_path/moh_automation/moh_automation_main/transfer_GenPipes.sh" -r "$1" -p "$2" -t "$3" 2>&1 &
    fi
    echo -n "PID: "
    echo $!
    echo "LOG: "
  } >> "$transfer_log"
  echo "-> To follow transfer status see $transfer_log"
}

while getopts 'hc:j:r:l:' OPTION; do
  case "$OPTION" in
  c)
    cluster="$OPTARG"
      if [[ $cluster == abacus ]]; then
        MOH_path="/lb/project/mugqic/projects/MOH"
        if [ -z "${MUGQIC_INSTALL_HOME_DEV:-}" ]; then
          export MUGQIC_INSTALL_HOME_DEV=/lb/project/mugqic/analyste_dev
        fi
        if [ -z "${MUGQIC_INSTALL_HOME_PRIVATE:-}" ]; then
          export MUGQIC_INSTALL_HOME_PRIVATE=/lb/project/mugqic/analyste_private
        fi
      elif [[ $cluster == rorqual ]]; then
        MOH_path="/project/6007512/C3G/projects/MOH_PROCESSING"
        if [ -z "${MUGQIC_INSTALL_HOME_DEV:-}" ]; then
          export MUGQIC_INSTALL_HOME_DEV=/project/6007512/C3G/analyste_dev
        fi
      elif [[ $cluster == cardinal ]]; then
        MOH_path="/project/60007/MOH"
        if [ -z "${MUGQIC_INSTALL_HOME_DEV:-}" ]; then
          export MUGQIC_INSTALL_HOME_DEV=/project/60007/analyste_dev
        fi
      elif [[ $cluster == narval ]]; then
        MOH_path="/lustre06/project/6084703/C3G/projects/MOH"
        if [ -z "${MUGQIC_INSTALL_HOME_DEV:-}" ]; then
          export MUGQIC_INSTALL_HOME_DEV=/lustre06/project/6007512/C3G/analyste_dev
        fi
      elif [[ $cluster == fir ]]; then
        MOH_path="/project/6007512/C3G/projects/MOH"
        if [ -z "${MUGQIC_INSTALL_HOME_DEV:-}" ]; then
          export MUGQIC_INSTALL_HOME_DEV=/project/6007512/C3G/analyste_dev
        fi
      else
        echo -e "ERROR: Invalid cluster: '$cluster'. It has to be either 'abacus', 'rorqual', 'narval', 'fir', or 'cardinal'\n"
        usage
      fi
    ;;
  j)
    genpipes_json=$(readlink -f "$OPTARG")
    ;;
  r)
    readset_file="$OPTARG"
    ;;
  l)
    job_list="$OPTARG"
    ;;
  h)
    usage
    ;;
  ?)
    usage
    ;;
  esac
done

# mandatory arguments
if [ ! "$cluster" ] || [ ! "$readset_file" ] || [ ! "$genpipes_json" ] || [ ! "$job_list" ]; then
  echo -e "ERROR: Missing mandatory arguments -c and/or -r and/or -j and/or -l.\n"
  usage
fi

if [ -z "${MUGQIC_INSTALL_HOME:-}" ]; then
  export MUGQIC_INSTALL_HOME=/cvmfs/soft.mugqic/CentOS6
fi

if [ -z "${PORTAL_OUTPUT_DIR:-}" ]; then
  export PORTAL_OUTPUT_DIR=$MUGQIC_INSTALL_HOME_DEV/portal_out_dir
fi

module avail 2>&1 | grep -m 1 -q "mugqic"; greprc=$?
if ! [[ $greprc -eq 0 ]]; then
  module use "$MUGQIC_INSTALL_HOME/modulefiles" "$MUGQIC_INSTALL_HOME_DEV/modulefiles"
fi

if [ -z "${JOB_MAIL:-}" ]; then
  export JOB_MAIL=c3g-processing@fakeemail.ca
fi

# Check if json file is empty
if [ ! -s "$genpipes_json" ]; then
  echo "ERROR: GenPipes json file is empty Cf. $genpipes_json"
  exit 1
fi

operation_cmd_line=$(jq '.operation_cmd_line' "$genpipes_json")
pipeline=$(echo "$operation_cmd_line" | cut -d' ' -f1 | rev | cut -d'/' -f2 | rev)
# if pipeline is rnaseq_light, protocol is empty
if [[ $pipeline == "rnaseq_light" ]]; then
  protocol=""
else
  protocol=$(echo "$genpipes_json" | cut -d'.' -f2 |  cut -d'_' -f1)
fi

# MAIN folder location
MOH_MAIN="$MOH_path/MAIN"

genpipes_submission_folder=$(dirname "$readset_file")

echo "-> Checking $genpipes_submission_folder..."
if [[ $cluster == rorqual ]] || [[ $cluster == narval ]] || [[ $cluster == fir ]] || [[ $cluster == cardinal ]] ; then
  log_report_file="${job_list}.tsv"
  # shellcheck disable=SC2046,SC2086
  $MOH_MAIN/genpipes_moh/genpipes/utils/log_report.py $(readlink -f $job_list) --tsv $log_report_file 2>&1
  status=$(awk -F'\t' 'NR>1 {print $5"\n"$6"\n"$7}' "$log_report_file" | sort | uniq)
  chmod 660 "$log_report_file"
  # FIRST check if still running and skipping
  if [[ $status =~ (^|[[:space:]])"RUNNING"([[:space:]]|$) ]] || [[ $status =~ (^|[[:space:]])"PENDING"([[:space:]]|$) ]]; then
    # Let's skip and wait
    echo "INFO: Job(s) still running Cf. $log_report_file"
  # SECOND check if failed or timeout
  elif [[ $status =~ (^|[[:space:]])"FAILED"([[:space:]]|$) ]] || [[ $status =~ (^|[[:space:]])"TIMEOUT"([[:space:]]|$) ]] || [[ $status =~ (^|[[:space:]])"OUT_OF_MEMORY"([[:space:]]|$) ]]; then
    echo "WARNING: FAILED and/or TIMEOUT found in $job_list Cf. $log_report_file"
    # Let's tag GenPipes + Ingest GenPipes
    genpipes_tagging "$genpipes_json"
    genpipes_ingesting "${genpipes_json/.json/_tagged.json}"
    touch "${genpipes_submission_folder}.checked"
    chmod 660 "${genpipes_submission_folder}.checked"
  # THIRD check if completed and cancelled for instance when cancelled by a user
  elif [[ $status =~ (^|[[:space:]])"COMPLETED"([[:space:]]|$) ]] && [[ $status =~ (^|[[:space:]])"CANCELLED"([[:space:]]|$) ]]; then
    echo "WARNING: It seems to have been cancelled by a user"
    # Let's tag GenPipes + Ingest GenPipes
    genpipes_tagging "$genpipes_json"
    genpipes_ingesting "${genpipes_json/.json/_tagged.json}"
    touch "${genpipes_submission_folder}.checked"
    chmod 660 "${genpipes_submission_folder}.checked"
  # FOURTH check if success or completed
  elif [[ $status =~ (^|[[:space:]])"COMPLETED"([[:space:]]|$) ]]; then
    echo "SUCCESS: All jobs completed"
    # Let's tag GenPipes + Ingest GenPipes
    genpipes_tagging "$genpipes_json"
    genpipes_ingesting "${genpipes_json/.json/_tagged.json}"
    touch "${genpipes_submission_folder}.checked"
    chmod 660 "${genpipes_submission_folder}.checked"
  # FIFTH check if cancelled
  elif [[ $status =~ (^|[[:space:]])"CANCELLED"([[:space:]]|$) ]]; then
    echo "INFO: All jobs cancelled Cf. $log_report_file"
    touch "${genpipes_submission_folder}.checked"
    chmod 660 "${genpipes_submission_folder}.checked"
  # FIFTH check if unknown status
  else
    echo "ERROR: Unknown status Cf. $log_report_file"
  fi
elif [[ $cluster == abacus ]]; then
  log_report_file="${job_list}.txt"
  # shellcheck disable=SC2086
  $MOH_MAIN/genpipes_moh/genpipes/utils/log_report.pl $job_list > $log_report_file
  status=$(grep -v "^#" "$log_report_file" | awk -F'\t' '{print $5}' | sort | uniq)
  chmod 660 "${job_list}.txt"
  # FIRST check if still running and skipping
  if [[ $status =~ (^|[[:space:]])"ACTIVE"([[:space:]]|$) ]] || [[ $status =~ (^|[[:space:]])"QUEUED"([[:space:]]|$) ]]; then
    # Let's skip and wait
    echo "INFO: Job(s) still running Cf. $log_report_file"
  # SECOND check if failed or timeout
  elif [[ $status =~ (^|[[:space:]])"FAILED"([[:space:]]|$) ]]; then
    echo "WARNING: FAILED and/or TIMEOUT found in $job_list Cf. $log_report_file"
    # Let's tag GenPipes + Ingest GenPipes
    genpipes_tagging "$genpipes_json"
    genpipes_ingesting "${genpipes_json/.json/_tagged.json}"
    touch "${genpipes_submission_folder}.checked"
    chmod 660 "${genpipes_submission_folder}.checked"
  # THIRD check if completed and cancelled for instance when cancelled by a user
  elif [[ $status =~ (^|[[:space:]])"SUCCESS"([[:space:]]|$) ]] && [[ $status =~ (^|[[:space:]])"CANCELLED"([[:space:]]|$) ]]; then
    echo "WARNING: It seems to have been cancelled by a user"
    # Let's tag GenPipes + Ingest GenPipes
    genpipes_tagging "$genpipes_json"
    genpipes_ingesting "${genpipes_json/.json/_tagged.json}"
    touch "${genpipes_submission_folder}.checked"
    chmod 660 "${genpipes_submission_folder}.checked"
  # FOURTH check if success or completed
  elif [[ $status =~ (^|[[:space:]])"SUCCESS"([[:space:]]|$) ]]; then
    echo "SUCCESS: All jobs completed"
    # Let's tag GenPipes + Ingest GenPipes
    genpipes_tagging "$genpipes_json"
    genpipes_ingesting "${genpipes_json/.json/_tagged.json}"
    touch "${genpipes_submission_folder}.checked"
    chmod 660 "${genpipes_submission_folder}.checked"
  # FIFTH check if cancelled
  elif [[ $status =~ (^|[[:space:]])"CANCELLED"([[:space:]]|$) ]]; then
    echo "INFO: All jobs cancelled Cf. $log_report_file"
    touch "${genpipes_submission_folder}.checked"
    chmod 660 "${genpipes_submission_folder}.checked"
  # FIFTH check if unknown status
  else
    echo "ERROR: Unknown status Cf. $log_report_file"
  fi
fi