hotfix: remove bad timeline HIPO files and allow Step 2 to finish (#299)

Author: c-dilks

bin/hipo-check.sh

@@ -4,17 +4,33 @@ set -e
 set -u
 source $(dirname $0)/environ.sh
 
+# default options
+rm_bad=false
+hipoFiles=()
+
 # arguments
 if [ $# -lt 1 ]; then
-  printError "no HIPO files specified for $(basename $0)"
   echo """
-  USAGE: $0 [HIPO_FILE(S)]...
+  USAGE: $0 [OPTIONS]... [HIPO_FILE(S)]...
 
   Checks each [HIPO_FILE] for corruption, etc.
+
+  OPTIONS
+    --rm-bad           delete (rm) bad HIPO files
   """ >&2
   exit 101
 fi
-hipoFiles=$@
+for arg in "$@"; do
+  if [[ $arg =~ ^- ]]; then
+    case $arg in
+      --rm-bad) rm_bad=true ;;
+      *) printError "unknown option $arg" && exit 100 ;;
+    esac
+  else
+    hipoFiles+=($arg)
+  fi
+done
+[ ${#hipoFiles[@]} -lt 1 ] && printError "no HIPO files specified for $(basename $0)" && exit 100
 
 # minimum file size for a valid HIPO file
 # - seems to be 192 bytes, but setting the threshold slightly higher may be safer
@@ -56,5 +72,12 @@ if [ ${#badFiles[@]} -gt 0 ]; then
   for badFile in ${badFiles[@]}; do
     echo "         - $badFile" >&2
   done
+  # remove bad HIPO files
+  if $rm_bad; then
+    printError "These HIPO files will now be REMOVED!"
+    for badFile in ${badFiles[@]}; do
+      rm -v $badFile >&2
+    done
+  fi
   exit 100
 fi

bin/run-detectors-timelines.sh

@@ -242,7 +242,6 @@ if ${modes['focus-all']} || ${modes['focus-timelines']}; then
       echo "PREMATURE EXIT, since --debug option was used"
       exit
     else
-      #sleep 1 
       java $TIMELINE_JAVA_OPTS $run_detectors_script $timelineObj $inputDir > $logFile.out 2> $logFile.err || touch $logFile.fail &
       job_ids+=($!)
       job_names+=($timelineObj)
@@ -279,9 +278,13 @@ if ${modes['focus-all']} || ${modes['focus-timelines']}; then
     esac
   done
 
-  # check timelines
+  # check timelines; remove and complain about any bad ones
+  echo ">>> running hipo-check on timeline HIPO files..."
   outputFiles=$(find . -name "*.hipo")
-  [ -n "$outputFiles" ] && $TIMELINESRC/bin/hipo-check.sh $outputFiles
+  if [ -n "$outputFiles" ]; then
+    logFile=$logDir/hipo-check
+    $TIMELINESRC/bin/hipo-check.sh --rm-bad $outputFiles > $logFile.out 2> $logFile.err || touch $logFile.fail
+  fi
 
   # remove any empty directories
   echo ">>> removing any empty directories..."
@@ -326,31 +329,44 @@ $logDir/*.err
 
 # exit nonzero if any jobs exitted nonzero
 failedJobs=($(find $logDir -name "*.fail" | xargs -I{} basename {} .fail))
+somethingFailed=false
 if [ ${#failedJobs[@]} -gt 0 ]; then
   for failedJob in ${failedJobs[@]}; do
     echo $sep
     printError "job '$failedJob' returned non-zero exit code; error log dump:"
     cat $logDir/$failedJob.err
+    if [ "$failedJob" = "hipo-check" ]; then
+      printWarning "These HIPO files are TIMELINE files; if this '$failedJob' job is the ONLY failed job, you may proceed with timeline deployment, but these failed timelines will not be deployed."
+    fi
   done
   if [ -z "$singleTimeline" -a ${modes['focus-qa']} = false ]; then
     echo $sep
     echo "To re-run only the failed timelines, for debugging, try one of the following commands:"
     for failedJob in ${failedJobs[@]}; do
       if [ "$failedJob" = "qa" ]; then
         echo "  $0 $@ --focus-qa"
+      elif [ "$failedJob" = "hipo-check" ]; then
+        echo "  $0 $@ --focus-timelines -t [BAD_TIMELINE]"
+        echo "  where [BAD_TIMELINE] is any timeline that failed 'hipo-check'"
       else
         echo "  $0 $@ --focus-timelines -t $failedJob"
       fi
     done
   fi
-  exit 100
+  somethingFailed=true
 else
   echo "All jobs exitted normally"
 fi
 
 # grep for suspicious things in error logs
 errPattern="error:|exception:|warning"
-echo """To look for any quieter errors, running \`grep -iE '$errPattern'\` on *.err files:
+echo """Now scanning for any quieter errors, by running \`grep -iE '$errPattern'\` on *.err files:
 $sep"""
 grep -iE --color "$errPattern" $logDir/*.err || echo "Good news: grep found no errors, but you still may want to take a look yourself..."
 echo $sep
+
+# exit nonzero if something failed
+if $somethingFailed; then
+  printWarning "At least one job had issues; look above or in the log files to see what's wrong."
+  exit 100
+fi