feat: --fast-ls

Author: c-dilks

bin/qtl-histogram

@@ -17,7 +17,7 @@ SLURM_LOG=/farm_out/%u/%x-%A_%a
 # default options
 dataset=test_v0
 declare -A modes
-for key in findhipo rundir eachdir flatdir single series submit check-cache swifjob focus-detectors focus-physics help; do
+for key in findhipo rundir eachdir flatdir single series submit check-cache fast-ls swifjob focus-detectors focus-physics help; do
   modes[$key]=false
 done
 outputDir=""
@@ -81,8 +81,9 @@ usageVerbose() {
                       directory tree, e.g., runs grouped by target
 
        --rundir       assume each specified [RUN_DIRECTORY] contains
-                      subdirectories named as just run numbers; it is not
-                      recommended to use wildcards for this option
+                      subdirectories named as just run numbers;
+                      - it is not recommended to use wildcards for this option
+                      - see also \`--fast-ls\` option
                       **this is the DEFAULT option**
 
        --eachdir      assume each specified [RUN_DIRECTORY] is a single
@@ -121,6 +122,13 @@ usageVerbose() {
 
        --focus-physics     run histogramming for physics QA timelines
 
+     *** MISCELLANEOUS OPTIONS
+
+       --fast-ls           use \`jasmine list-files\` to get lists of files, rather
+                           than commands such as \`find\` or \`ls\`
+                           - useful for options such as \`--rundir\` for DST files
+                           - WARNING: assumes you already tested \`--check-cache\`
+
   $sep
 
   EXAMPLES:
@@ -367,7 +375,18 @@ for rdir in ${rdirs[@]}; do
     realpath $rdir > $inputListFile
   else
     [[ "$(realpath $rdir)" =~ /mss/ ]] && swifPrefix="mss:" || swifPrefix="file:"
-    realpath $rdir/*.hipo | sed "s;^;$swifPrefix;" > $inputListFile
+    if ${modes['fast-ls']}; then
+      rdir_mss=$(realpath $rdir | sed 's;^.*/cache/;/mss/;')
+      rdir_cache=$(realpath $rdir)
+      # fast-ls mode will use `jasmine list-files`, but requires the stub files on `mss` to exist; if a cook is ongoing,
+      # it's possible the `/mss` directory won't be populated yet, although files are already on `/cache`; for that case,
+      # fall back to "slow" ls (i.e., the command that runs when not using `--fast-ls`)
+      jasmine list-files -display json -under $rdir_mss | jq -r '.[].mss' | xargs -n1 basename | sed "s;^;${swifPrefix}${rdir_cache}/;" > $inputListFile \
+        || { printWarning "falling back to slow ls..."; realpath $rdir/*.hipo | sed "s;^;$swifPrefix;" > $inputListFile; }
+    else
+      echo "      (use \`--fast-ls\` if this seems too slow)"
+      realpath $rdir/*.hipo | sed "s;^;$swifPrefix;" > $inputListFile
+    fi
   fi
 
   # generate job scripts

qadb/notes/rge_sp24.md

@@ -21,16 +21,16 @@ The DSTs are found in:
 > Check any run-dependent settings in `qa-physics/monitorRead.groovy`, such as beam energy.
 
 > [!NOTE]
-> Using `--rundir` here rather than `--flatdir`, since we're using a directory of DST files.
+> Using `--rundir` and `--fast-ls` here rather than `--flatdir`, since we're using a directory of DST files.
 
 check cache:
 ```bash
-bin/qtl histogram -d rge_sp24 --check-cache --rundir --focus-physics $(ls -d /cache/clas12/rg-e/production/spring2024/pass1/*/*/dst/recon)
+bin/qtl histogram -d rge_sp24 --check-cache --rundir --fast-ls --focus-physics $(ls -d /cache/clas12/rg-e/production/spring2024/pass1/*/*/dst/recon)
 ```
 
 run:
 ```bash
-bin/qtl histogram -d rge_sp24 --rundir --focus-physics $(ls -d /cache/clas12/rg-e/production/spring2024/pass1/*/*/dst/recon)
+bin/qtl histogram -d rge_sp24 --rundir --fast-ls --focus-physics $(ls -d /cache/clas12/rg-e/production/spring2024/pass1/*/*/dst/recon)
 ```
 
 ## Double check that we have all the runs