refactor: detector QA cut code should not be standalone

The code in `qa-detectors` just applies detector QA cut lines and
duplicates timelines filtered by the QA cuts. This code was developed
independently, and makes assumptions of the contents of HIPO timeline
files, such as graph names. This is not ideal, and `applyBounds.groovy`
is especially difficult to maintain.

Instead, apply QA cuts _directly to_ each timeline object, which is much
more flexible. Later on, we could apply these new QA methods to physics
timelines too.

Author: c-dilks

bin/qtl-analysis

@@ -255,7 +255,6 @@ fi
 [ -z "$outputDir" ] && outputDir=$(realpath $(pwd -P)/outfiles/$dataset) || outputDir=$(realpath $outputDir)
 
 # set subdirectories
-finalDirPreQA=$outputDir/timeline_web_preQA
 finalDir=$outputDir/timeline_web
 logDir=$outputDir/log
 
@@ -267,7 +266,6 @@ $sep
 INPUT_DIR       = $inputDir
 DATASET_NAME    = $dataset
 OUTPUT_DIR      = $outputDir
-FINAL_DIR_PREQA = $finalDirPreQA
 FINAL_DIR       = $finalDir
 LOG_DIR         = $logDir
 NUM_THREADS     = $numThreads
@@ -303,9 +301,6 @@ detDirs=(
 
 # cleanup output directories
 if $enableAna; then
-  if [ -d $finalDirPreQA ]; then
-    rm -rv $finalDirPreQA
-  fi
   if [ -d $logDir ]; then
     for fail in $(find $logDir -name "*.fail"); do
       rm $fail
@@ -314,7 +309,7 @@ if $enableAna; then
 fi
 
 # make output directories
-mkdir -p $logDir $finalDirPreQA $finalDir
+mkdir -p $logDir $finalDir
 
 
 ##################################################################################
@@ -353,7 +348,7 @@ if $enableAna; then
   }
 
   # change working directory to output directory
-  pushd $finalDirPreQA
+  pushd $finalDir
 
   # make detector subdirectories
   for detDir in ${detDirs[@]}; do
@@ -426,23 +421,6 @@ if $enableAna; then
   popd
 fi
 
-######################################
-# run QA
-######################################
-
-if $enableAna; then
-  # first, copy the timelines to the final timeline directory
-  echo ">>> copy timelines to final directory..."
-  cp -rL $finalDirPreQA/* $finalDir/
-  # then add the QA lines
-  echo ">>> add QA lines..."
-  logFile=$logDir/qa
-  $TIMELINESRC/libexec/run-groovy-timeline.sh $TIMELINESRC/qa-detectors/util/applyBounds.groovy $finalDirPreQA $finalDir > $logFile.out 2> $logFile.err || touch $logFile.fail
-  outputFiles=$(find $finalDir -name "*_QA.hipo")
-  [ -n "$outputFiles" ] && $TIMELINESRC/libexec/hipo-check.sh $outputFiles
-fi
-
-
 
 ######################################
 # error checking

src/main/java/org/jlab/clas/timeline/analysis/ec/ec_Sampl.groovy

@@ -3,6 +3,7 @@ import java.util.concurrent.ConcurrentHashMap
 import org.jlab.groot.data.TDirectory
 import org.jlab.groot.data.GraphErrors
 import org.jlab.clas.timeline.fitter.ECFitter
+import org.jlab.clas.timeline.util.QA
 
 class ec_Sampl {
 
@@ -35,7 +36,7 @@ def write() {
 
   TDirectory out = new TDirectory()
   out.mkdir('/timelines')
-  (0..<6).each{ sec->
+  def grtlList = (0..<6).collect{ sec->
     def grtl = new GraphErrors('sec'+(sec+1))
     grtl.setTitle("ECAL Sampling Fraction per sector")
     grtl.setTitleY("ECAL Sampling Fraction per sector")
@@ -50,10 +51,10 @@ def write() {
       out.addDataSet(it.flist[sec])
       grtl.addPoint(it.run, it.Sampling[sec], 0, 0)
     }
-    out.cd('/timelines')
-    out.addDataSet(grtl)
+    grtl
   }
-
-  out.writeFile('ec_Sampling.hipo')
+  out.cd('/timelines')
+  QA.cutGraphs(grtlList, lb: 0.23, ub: 0.26, out: out)
+  out.writeFile('ec_Sampling_QA.hipo')
 }
 }

src/main/java/org/jlab/clas/timeline/util/QA.groovy

@@ -0,0 +1,81 @@
+package org.jlab.clas.timeline.util
+import org.jlab.groot.data.GraphErrors
+import org.jlab.groot.data.TDirectory
+
+class QA {
+
+  /// @returns a single horizontal cut line
+  /// @param value the y-axis value of this line
+  /// @param the color of this line
+  static GraphErrors makeCutLine(double value, String color=null) {
+    if(value==null) return null;
+    line_name = [
+      'plotLine',
+      'horizontal',
+      value,
+      color ?: 'black',
+    ].join(':');
+    new GraphErrors(line_name);
+  }
+
+  /// result of `cutGraphs`
+  class CutGraphResult {
+    /// graphs which include only the "bad" points (points outside QA cuts)
+    public GraphErrors[] bad_graphs;
+    /// cut lines
+    public GraphErrors[] cut_lines;
+  }
+
+  /// @param input_graphs a list of graphs to cut
+  /// @param args.lb lower QA bound (default: no bound)
+  /// @param args.ub upper QA bound (default: no bound)
+  /// @param args.lb_color color of lower bound line
+  /// @param args.ub_color color of upper bound line
+  /// @param args.out TDirectory for adding graphs and lines, if defined
+  /// @returns `CutGraphResult`
+  static CutGraphResult cutGraphs(Map args, GraphErrors[] input_graphs) {
+    def result = new CutGraphResult();
+    // make lines
+    result.cut_lines = [
+      [args.lb, args.lb_color],
+      [args.ub, args.ub_color],
+    ]
+      .collect{makeCutLine(*it)}
+      .findAll{it != null};
+    // define QA cut
+    def reqs = []
+    if(args.lb != null)
+      reqs << {val -> val >= args.lb };
+    if(args.ub != null)
+      reqs << {val -> val <= args.ub };
+    def checkPoint = { val ->
+      allow = true;
+      reqs.each{ req -> allow &= req(val) };
+      pass;
+    };
+    // apply cuts
+    result.bad_graphs = input_graphs.collect{ input_graph ->
+      def bad_graph = new GraphErrors();
+      bad_graph.setName(input_graph.getName() + "__bad");
+      bad_graph.setTitle(input_graph.getTitle());
+      bad_graph.setTitleX(input_graph.getTitleX());
+      bad_graph.setTitleY(input_graph.getTitleY());
+      // loop over points, checking the cuts
+      input_graph.getDataSize(0).times{ i ->
+        def val_x = input_graph.getDataX(i);
+        def val_y = input_graph.getDataY(i);
+        if(!checkPoint(val_y))
+          bad_graph.addPoint(val_x, val_y);
+      }
+      bad_graph;
+    }
+    // write output
+    if(args.out != null) {
+      result.cut_lines.each{args.out.addDataSet(it)};
+      input_graphs.each{args.out.addDataSet(it)};
+      result.bad_graphs.each{args.out.addDataSet(it)};
+    }
+    result;
+  }
+
+}