refactor!: don't make chefs pick a dataset name

be more consistent with workflow step: require only one output-type
argument (`-p`)

Author: c-dilks

bin/qtl-analysis

@@ -4,16 +4,19 @@ set -e
 set -u
 source $(dirname $0)/../libexec/environ.sh
 
+##################################################################################
+
 # timeline webserver directory
 WEBURL="https://clas12mon.jlab.org"
-WEBDIR=/u/group/clas/www/clas12mon/html/hipo
+WEBDIR=/group/clas/www/clas12mon/html/hipo
+
+##################################################################################
 
 # default options
 match="^"
 inputDir=""
 publishDir=""
 publishNote=""
-publishUrl=""
 metaSettings=""
 dataset=""
 outputDir=""
@@ -25,6 +28,8 @@ for key in list just-pub just-ana skip-mya debug overwrite custom-pub help; do
   modes[$key]=false
 done
 
+##################################################################################
+
 # usage
 sep="================================================================"
 usageTerse() {
@@ -36,21 +41,15 @@ usageTerse() {
 
   CHEF OPTIONS: most of these are required for chef's production
 
-    -d [DATASET_NAME]   required unique dataset name, defined by the user
-
-    -i [INPUT_DIR]      input directory
-                        - chefs: use the output 'hist' directory, which
-                          contains subdirectories for each run
-                        - default = ./outfiles/[DATASET_NAME]
+    -i [INPUT_DIR]      input directory; use the output workflow
+                        directory, which has subdirectories for each run
 
-    -p [PUBLISH_DIR]    publish timeline results to
-                        $WEBURL/[PUBLISH_DIR]/[DATASET_NAME]/tlsummary
-                        - chefs: usually your run group (e.g., 'rgb'), or a
-                          subdirectory within (e.g. 'rgb/pass0')
-                        - required (or see expert options with '--help')
+    -p [PUBLISH_DIR]    publish timeline results to URL
+                        $WEBURL/[PUBLISH_DIR]/tlsummary
+                        (i.e., $WEBDIR/[PUBLISH_DIR])
 
-    -n [NOTE]           additional note that will be shown on the clas12mon webpage
-                        (i.e., the README file); surround your note in quotes
+    -n [NOTE]           additional note that will be shown on the clas12mon
+                        webpage; surround it in quotes
                         - default: ''
 
     -s [SETTINGS]       apply run-group or dataset specific settings; one of:
@@ -59,13 +58,19 @@ $(find $TIMELINESRC/data/metadata -name "*.json" -exec basename {} .json \; | se
   """>&2
 }
 usageVerbose() {
-  echo """  EXPERT OPTIONS: these are generally for developers, not chefs
+  echo """  DEVELOPER OPTIONS: these are generally for developers, not chefs
+
+    -d [DATASET_NAME]   unique dataset name, defined by the user
+                        - you may have used it in 'qtl histogram'
+                        - default: directory basename of [PUBLISH_DIR]
+
+    -i [INPUT_DIR]      input directory
+                        - the default is such that users only ever have
+                          to use '-d [DATASET]' to refer to a local dataset
+                        - default: ./outfiles/[DATASET_NAME]
 
     -o [OUTPUT_DIR]     output directory for the analyzed timelines
                         - default = ./outfiles/[DATASET_NAME]
-                          - chefs: if you use this default, it may be convenient
-                            to make a symbolic link named 'outfiles' to a
-                            directory on '/volatile'
 
     -j [NUM_THREADS]    number of parallel threads to run
                         - default = $numThreads
@@ -98,7 +103,7 @@ usageVerbose() {
 }
 if [ $# -eq 0 ]; then
   usageTerse
-  echo "  For more (expert) options, run with '--help'" >&2
+  echo "  For more options, run with '--help'" >&2
   exit 101
 fi
 
@@ -133,6 +138,29 @@ if ${modes['help']}; then
   exit 101
 fi
 
+##################################################################################
+
+# main executable for detector timelines
+run_analysis_script="org.jlab.clas.timeline.analysis.run_analysis"
+
+# build list of timelines
+if ${modes['skip-mya']}; then
+  timelineList=$(java $TIMELINE_JAVA_OPTS $run_analysis_script --timelines | grep -vE '^epics_' | sort | grep $match)
+else
+  timelineList=$(java $TIMELINE_JAVA_OPTS $run_analysis_script --timelines | sort | grep $match)
+fi
+
+# list detector timelines, if requested
+if ${modes['list']}; then
+  echo $sep
+  echo "LIST OF TIMELINES"
+  echo $sep
+  echo $timelineList | sed 's; ;\n;g'
+  exit $?
+fi
+
+##################################################################################
+
 # set flow control booleans
 ## handle `--just-ana` and `--just-pub`
 enableAna=false
@@ -158,37 +186,10 @@ if ${modes['debug']}; then
   enablePub=false
 fi
 
-# get main executable for detector timelines
-run_analysis_script="org.jlab.clas.timeline.analysis.run_analysis"
-
-# build list of timelines
-if ${modes['skip-mya']}; then
-  timelineList=$(java $TIMELINE_JAVA_OPTS $run_analysis_script --timelines | grep -vE '^epics_' | sort | grep $match)
-else
-  timelineList=$(java $TIMELINE_JAVA_OPTS $run_analysis_script --timelines | sort | grep $match)
-fi
-
-# list detector timelines, if requested
-if ${modes['list']}; then
-  echo $sep
-  echo "LIST OF TIMELINES"
-  echo $sep
-  echo $timelineList | sed 's; ;\n;g'
-  exit $?
-fi
-
-# set input and output directories
-[ -z "$dataset" ] && printError "data set not specified, use option '-d'" && exit 100
-[ -z "$inputDir" ] && inputDir=$(pwd -P)/outfiles/$dataset/timeline_detectors
-[ ! -d $inputDir ] && printError "input directory $inputDir does not exist" && exit 100
-[ -z "$outputDir" ] && outputDir=$(realpath $(pwd -P)/outfiles/$dataset) || outputDir=$(realpath $outputDir)
-
-# set subdirectories
-finalDirPreQA=$outputDir/timeline_web_preQA
-finalDir=$outputDir/timeline_web
-logDir=$outputDir/log
+##################################################################################
 
 # check publishing options
+publishUrl=""
 if $enablePub; then
   # check publishDir arg
   if [ -z "$publishDir" ]; then
@@ -198,9 +199,9 @@ if $enablePub; then
   fi
   # set full path to publish dir (unless --custom-pub)
   if ! ${modes['custom-pub']}; then
-    publishUrl=$WEBURL/$publishDir/$dataset/tlsummary
+    publishUrl=$WEBURL/$publishDir/tlsummary
     [ "$publishDir" = "." ] && printError "publishing directory argument cannot be '.'" && exit 100
-    publishDir=$WEBDIR/$publishDir/$dataset
+    publishDir=$WEBDIR/$publishDir
   else
     publishUrl="option '--custom-pub' was used, therefore timeline URL is UNKNOWN"
   fi
@@ -236,6 +237,27 @@ if $enablePub; then
   publishNote="${timestamp}${publishNote}; timeline-code v$version"
 fi
 
+# set the dataset name, if not already set
+if [ -z "$dataset" ]; then
+  if [ -n "$publishDir" ]; then
+    dataset=$(basename $publishDir)
+    echo "setting dataset name from the publishing directory to '$dataset'"
+  else
+    printError "neither dataset name (-d) nor publishing directory (-p) are set; need at least one..."
+    exit 100
+  fi
+fi
+
+# set input and output directories
+[ -z "$inputDir" ] && inputDir=$(pwd -P)/outfiles/$dataset/timeline_detectors
+[ ! -d $inputDir ] && printError "input directory $inputDir does not exist" && exit 100
+[ -z "$outputDir" ] && outputDir=$(realpath $(pwd -P)/outfiles/$dataset) || outputDir=$(realpath $outputDir)
+
+# set subdirectories
+finalDirPreQA=$outputDir/timeline_web_preQA
+finalDir=$outputDir/timeline_web
+logDir=$outputDir/log
+
 # print settings
 echo """
 Settings:
@@ -291,6 +313,11 @@ fi
 mkdir -p $logDir $finalDirPreQA $finalDir
 
 
+##################################################################################
+##################################################################################
+##################################################################################
+
+
 ######################################
 # produce detector timelines
 ######################################

bin/qtl-physics

@@ -164,7 +164,7 @@ customPubArg=''
 if ${modes['custom-pub']}; then
   customPubArg=--custom-pub
 fi
-$TIMELINESRC/qtl analysis \
+$TIMELINESRC/bin/qtl analysis \
   -d $dataset \
   -i $inputDir \
   -o $outputDir \

doc/chef_guide.md

@@ -4,39 +4,53 @@ The timeline code is provided on `ifarm` via
 ```bash
 module load timeline
 ```
+The main script for running timelines is `qtl`:
+```bash
+qtl --help
+qtl --version
+```
 
 Please report any issues to the software maintainers, such as warnings or error messages.
 
-## :green_circle: Step 1: The workflow
+## General Timelines
 
-Use the `qtl` model as part of your usual cooking workflow;
-see [the Chefs' documentation wiki](https://clasweb.jlab.org/wiki/index.php/CLAS12_Chef_Documentation).
+### :green_circle: Step 1: Fill Timeline Histograms
 
-Output files will appear in your chosen output directory, within `hist/detectors/`.
+Use the "qtl" model as part of your usual cooking workflow; see [the Chefs' documentation wiki](https://clasweb.jlab.org/wiki/index.php/CLAS12_Chef_Documentation).
 
-> [!NOTE]
-> If you want physics timelines, for now you'll need to use `qtl histogram --focus-physics`; we will eventually include this in the workflow!
+Output files will appear in your chosen output directory, within `hist/detectors/`.
 
-## :green_circle: Step 2: Make the timelines
+### :green_circle: Step 2: Analyze Histograms and Make the Timelines
 
+Print the usage guide:
+```bash
+qtl analysis
+```
+In general,
 ```bash
-qtl analysis -d $dataset -i $out_dir/hist/detectors -p $publish_dir
+qtl analysis -i $out_dir/hist/detectors -p $publish_dir
 ```
-- `$dataset` is a unique name for this cook, _e.g._, `rga_v1.23`
+where
 - `$out_dir` is your output directory from **Step 1**
-- `$publish_dir` is the publishing directory
-  - example: `rgb/pass0`
-  - note: the full publishing path will be `/group/clas/www/clas12mon/html/hipo/${publish_dir}/${dataset}`
+- `$publish_dir` is the publishing directory to [`clas12mon`](https://clas12mon.jlab.org/)
+  - the full publishing path will be `/group/clas/www/clas12mon/html/hipo/${publish_dir}`
+  - the URL will be `https://clas12mon.jlab.org/${publish_dir}/tlsummary
 
-A URL will be printed upon success, and a link will appear in [`clas12mon`](https://clas12mon.jlab.org/) in your run group's area momentarily.
+Additional options may be _needed_ for your specific dataset, so check the usage guide (run `qtl analysis`).
 
-> [!IMPORTANT]
-> Run `qtl analysis` with no further arguments for additional usage; some options may be _needed_ for your run group!
+A URL will be printed upon success, and a link will appear in [`clas12mon`](https://clas12mon.jlab.org/) in your run group's area momentarily.
 
 > [!TIP]
-> Step 2 generates analyzed timeline files and logging information to a separate directory.
-> - You can control that directory with the `-o` option, or just use the default.
-> - If using the default, consider making a symbolic link named `outfiles` pointing to somewhere on `/volatile`, since once timelines are published successfully, you don't really need these output files
+> `qtl analysis` produces temporary files, by default in a subdirectory of `./outfiles/`. Consider making a symbolic link named `outfiles` pointing to somewhere on `/volatile`.
+
+## Physics QA timelines
+
+The physics timelines will eventually be combined with the detector timelines; until then, they are run separately:
+
+- **Step 1:** Either:
+  - Use `qtl histogram` instead of the workflow, with the option `--focus-physics`; this will run on SLURM directly (rather than through SWIF)
+  - Use the `--physics` option with the workflow "qtl" model
+- **Step 2:** Run `qtl physics` instead of `qtl analysis`; its options are similar
 
 ---
 

util/dump-timelines.groovy

@@ -14,7 +14,7 @@ if(args.length>1) outFilePrefix = args[1]
 def inSpec = args[0]
 def inFiles = []
 if(inSpec ==~ /^https.*clas12mon.jlab.org.*timeline.*/) {
-  def inDir = inSpec.replaceAll(/^.*jlab.org/, "/u/group/clas/www/clas12mon/html/hipo")
+  def inDir = inSpec.replaceAll(/^.*jlab.org/, "/group/clas/www/clas12mon/html/hipo")
   inDir = "/"+inDir.tokenize('/')[0..-2].join("/")
   def inDirObj = new File(inDir)
   inDirObj.traverse(type: groovy.io.FileType.FILES, nameFilter: ~/.*\.hipo/) {