From 3cf1571c87fd1f45e73b569b34b0019c8ab1f5f8 Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Tue, 29 Jul 2025 18:13:51 -0400 Subject: [PATCH 01/12] feat: RG-K Fall 18 QA configuration and notes --- qadb/notes/rgk_fa18.md | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 qadb/notes/rgk_fa18.md diff --git a/qadb/notes/rgk_fa18.md b/qadb/notes/rgk_fa18.md new file mode 100644 index 000000000..8a4273277 --- /dev/null +++ b/qadb/notes/rgk_fa18.md @@ -0,0 +1,39 @@ +# Run Group K, Fall 2018, Pass 2 + +## Run monitoring + +> [!IMPORTANT] +> Check any run-dependent settings in `qa-physics/monitorRead.groovy`, such as beam energy. + +We will use SIDIS skims from two datasets: one for a 6.5 GeV beam, and another for a 7.5 GeV beam; we will keep them separate: +- `rgk_fa18_6.5GeV`: from `/cache/clas12/rg-k/production/recon/fall2018/torus+1/6535MeV/pass2/v0/dst/train/skim1` +- `rgk_fa18_7.5GeV`: from `/cache/clas12/rg-k/production/recon/fall2018/torus+1/7546MeV/pass2/v0/dst/train/skim1` + +- **6.5 GeV:** +```bash +qtl histogram --check-cache -d rgk_fa18_6.5GeV --rundir --focus-physics +qtl histogram -d rgk_fa18_6.5GeV --rundir --focus-physics +``` + +- **7.5 GeV:** +```bash +qtl histogram --check-cache -d rgk_fa18_7.5GeV --rundir --focus-physics +qtl histogram -d rgk_fa18_7.5GeV --rundir --focus-physics +``` + +## Double check that we have all the runs + +> [!IMPORTANT] +> In case any runs disappeared from `/cache` while running monitoring, be sure to cross check the output +> runs with those from `/mss` + +## Make timelines + +Make the timelines, separately for each of the 2 datasets: +```bash +bin/qtl physics -d rgk_fa18_6.5GeV -p rgk/pass2/qa/fa18_6.5GeV +bin/qtl physics -d rgk_fa18_7.5GeV -p rgk/pass2/qa/fa18_7.5GeV +``` + +> [!WARNING] +> You may not have permission to publish to this directory; if not, publish to another place and ask the chef to copy the files From a71443bc23a805d12209ebe8a04d5f9bb30b3864 Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Mon, 4 Aug 2025 21:35:47 -0400 Subject: [PATCH 02/12] fix: paths --- qadb/notes/rgk_fa18.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/qadb/notes/rgk_fa18.md b/qadb/notes/rgk_fa18.md index 8a4273277..88f430b52 100644 --- a/qadb/notes/rgk_fa18.md +++ b/qadb/notes/rgk_fa18.md @@ -11,14 +11,14 @@ We will use SIDIS skims from two datasets: one for a 6.5 GeV beam, and another f - **6.5 GeV:** ```bash -qtl histogram --check-cache -d rgk_fa18_6.5GeV --rundir --focus-physics -qtl histogram -d rgk_fa18_6.5GeV --rundir --focus-physics +qtl histogram --check-cache -d rgk_fa18_6.5GeV --rundir --focus-physics /cache/clas12/rg-k/production/recon/fall2018/torus+1/6535MeV/pass2/v0/dst/train/skim1 +qtl histogram -d rgk_fa18_6.5GeV --rundir --focus-physics /cache/clas12/rg-k/production/recon/fall2018/torus+1/6535MeV/pass2/v0/dst/train/skim1 ``` - **7.5 GeV:** ```bash -qtl histogram --check-cache -d rgk_fa18_7.5GeV --rundir --focus-physics -qtl histogram -d rgk_fa18_7.5GeV --rundir --focus-physics +qtl histogram --check-cache -d rgk_fa18_7.5GeV --rundir --focus-physics /cache/clas12/rg-k/production/recon/fall2018/torus+1/7546MeV/pass2/v0/dst/train/skim1 +qtl histogram -d rgk_fa18_7.5GeV --rundir --focus-physics /cache/clas12/rg-k/production/recon/fall2018/torus+1/7546MeV/pass2/v0/dst/train/skim1 ``` ## Double check that we have all the runs From 0ad2f6847649bcd95799343d9d021c4828d27317 Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Mon, 4 Aug 2025 21:36:32 -0400 Subject: [PATCH 03/12] fix: flatdir --- qadb/notes/rgk_fa18.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/qadb/notes/rgk_fa18.md b/qadb/notes/rgk_fa18.md index 88f430b52..42e683e55 100644 --- a/qadb/notes/rgk_fa18.md +++ b/qadb/notes/rgk_fa18.md @@ -11,14 +11,14 @@ We will use SIDIS skims from two datasets: one for a 6.5 GeV beam, and another f - **6.5 GeV:** ```bash -qtl histogram --check-cache -d rgk_fa18_6.5GeV --rundir --focus-physics /cache/clas12/rg-k/production/recon/fall2018/torus+1/6535MeV/pass2/v0/dst/train/skim1 -qtl histogram -d rgk_fa18_6.5GeV --rundir --focus-physics /cache/clas12/rg-k/production/recon/fall2018/torus+1/6535MeV/pass2/v0/dst/train/skim1 +qtl histogram --check-cache -d rgk_fa18_6.5GeV --flatdir --focus-physics /cache/clas12/rg-k/production/recon/fall2018/torus+1/6535MeV/pass2/v0/dst/train/skim1 +qtl histogram -d rgk_fa18_6.5GeV --flatdir --focus-physics /cache/clas12/rg-k/production/recon/fall2018/torus+1/6535MeV/pass2/v0/dst/train/skim1 ``` - **7.5 GeV:** ```bash -qtl histogram --check-cache -d rgk_fa18_7.5GeV --rundir --focus-physics /cache/clas12/rg-k/production/recon/fall2018/torus+1/7546MeV/pass2/v0/dst/train/skim1 -qtl histogram -d rgk_fa18_7.5GeV --rundir --focus-physics /cache/clas12/rg-k/production/recon/fall2018/torus+1/7546MeV/pass2/v0/dst/train/skim1 +qtl histogram --check-cache -d rgk_fa18_7.5GeV --flatdir --focus-physics /cache/clas12/rg-k/production/recon/fall2018/torus+1/7546MeV/pass2/v0/dst/train/skim1 +qtl histogram -d rgk_fa18_7.5GeV --flatdir --focus-physics /cache/clas12/rg-k/production/recon/fall2018/torus+1/7546MeV/pass2/v0/dst/train/skim1 ``` ## Double check that we have all the runs From a2ba699c13aea8d6e98b8bb4143255f04c7979b2 Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Tue, 26 Aug 2025 16:46:59 -0400 Subject: [PATCH 04/12] fix: loudly warn instead of error --- qa-physics/monitorRead.groovy | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/qa-physics/monitorRead.groovy b/qa-physics/monitorRead.groovy index 5a9d31d94..fdb4c3643 100644 --- a/qa-physics/monitorRead.groovy +++ b/qa-physics/monitorRead.groovy @@ -527,8 +527,9 @@ defineTimeBins = { // in its own closure, so giant data structures are garbage c tag1eventNumList = tag1events.sort(false){it[0]}.collect{it[0]} // check that we would get the same result, if we instead sorted by timestamp if(FCmode!=3 && tag1eventNumList != tag1events.sort(false){it[1]}.collect{it[0]}) { - System.err.println "ERROR: sorting tag1 events by event number is DIFFERENT than sorting by timestamp" - System.exit(100) + System.err.println "WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING" + System.err.println "WARNING: sorting tag1 events by event number is DIFFERENT than sorting by timestamp" + System.err.println "WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING" } // define the time bin boundaries: first, some sorting and transformations From 8a8e3f76f2525718cdf64018732fb6cd4aa516df Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Wed, 27 Aug 2025 11:15:32 -0400 Subject: [PATCH 05/12] fix: rename epochs files correctly --- qadb/epochs/{epochs.rgk_6.txt => epochs.rgk_fa18_6.5GeV.txt} | 0 qadb/epochs/{epochs.rgk_7.txt => epochs.rgk_fa18_7.5GeV.txt} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename qadb/epochs/{epochs.rgk_6.txt => epochs.rgk_fa18_6.5GeV.txt} (100%) rename qadb/epochs/{epochs.rgk_7.txt => epochs.rgk_fa18_7.5GeV.txt} (100%) diff --git a/qadb/epochs/epochs.rgk_6.txt b/qadb/epochs/epochs.rgk_fa18_6.5GeV.txt similarity index 100% rename from qadb/epochs/epochs.rgk_6.txt rename to qadb/epochs/epochs.rgk_fa18_6.5GeV.txt diff --git a/qadb/epochs/epochs.rgk_7.txt b/qadb/epochs/epochs.rgk_fa18_7.5GeV.txt similarity index 100% rename from qadb/epochs/epochs.rgk_7.txt rename to qadb/epochs/epochs.rgk_fa18_7.5GeV.txt From ccf71e184c3a0e73803aa5e3946a94a4a960f624 Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Wed, 27 Aug 2025 11:21:53 -0400 Subject: [PATCH 06/12] feat: cutdefs files --- qadb/cutdefs/rgk_fa18_6.5GeV.yaml | 12 ++++++++++++ qadb/cutdefs/rgk_fa18_7.5GeV.yaml | 1 + 2 files changed, 13 insertions(+) create mode 100644 qadb/cutdefs/rgk_fa18_6.5GeV.yaml create mode 120000 qadb/cutdefs/rgk_fa18_7.5GeV.yaml diff --git a/qadb/cutdefs/rgk_fa18_6.5GeV.yaml b/qadb/cutdefs/rgk_fa18_6.5GeV.yaml new file mode 100644 index 000000000..340462978 --- /dev/null +++ b/qadb/cutdefs/rgk_fa18_6.5GeV.yaml @@ -0,0 +1,12 @@ +OutlierFD: + IQR_cut_factor: 4.0 +OutlierFT: + IQR_cut_factor: 4.0 +LowLiveTime: + min_live_time: 0.9 +ChargeHigh: + IQR_cut_factor: 4.0 +PossiblyNoBeam: + max_num_events: 40000 + max_num_electrons: 100 + max_FC_charge: 20 diff --git a/qadb/cutdefs/rgk_fa18_7.5GeV.yaml b/qadb/cutdefs/rgk_fa18_7.5GeV.yaml new file mode 120000 index 000000000..bd27b6ca2 --- /dev/null +++ b/qadb/cutdefs/rgk_fa18_7.5GeV.yaml @@ -0,0 +1 @@ +rgk_fa18_6.5GeV.yaml \ No newline at end of file From f4ebf2e04cd9ec5bb3f0fe921d6d7abb7fd31ca6 Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Wed, 27 Aug 2025 11:34:54 -0400 Subject: [PATCH 07/12] doc: handling 5863 --- qadb/notes/rgk_fa18.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/qadb/notes/rgk_fa18.md b/qadb/notes/rgk_fa18.md index 42e683e55..c3fbe6534 100644 --- a/qadb/notes/rgk_fa18.md +++ b/qadb/notes/rgk_fa18.md @@ -29,6 +29,13 @@ qtl histogram -d rgk_fa18_7.5GeV --flatdir --focus-physics /cache/clas12/rg-k/pr ## Make timelines +> [!IMPORTANT] +> Pass 2 run 5863 was mistakenly cooked into the 6.5 GeV, but it has beam energy 7.5 GeV. To make sure its QA +> is performed in the correct epoch, move its step-1 output files to the correct output dataset directory: +> ```bash +> mv -v outfiles/rgk_fa18_6.5GeV/timeline_physics/5863 outfiles/rgk_fa18_7.5GeV/timeline_physics/ +> ``` + Make the timelines, separately for each of the 2 datasets: ```bash bin/qtl physics -d rgk_fa18_6.5GeV -p rgk/pass2/qa/fa18_6.5GeV From 9715c4b4cce1bd22b2a30e1a97c7e51627ec72b6 Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Mon, 16 Mar 2026 16:28:22 -0400 Subject: [PATCH 08/12] doc: prepare for reheat --- qadb/notes/rga_sp18.md | 2 +- qadb/notes/rgk_fa18.md | 51 +++++++++++++++++++++++++++++++----------- 2 files changed, 39 insertions(+), 14 deletions(-) diff --git a/qadb/notes/rga_sp18.md b/qadb/notes/rga_sp18.md index f192b1ce5..b1e663306 100644 --- a/qadb/notes/rga_sp18.md +++ b/qadb/notes/rga_sp18.md @@ -95,7 +95,7 @@ bin/qtl reheat -m rollover -c 13.3.0 -d rga_sp18_6.4GeV_inbending_lo_nSidis -o for f in /cache/clas12/rg-a/production/recon/spring2018/6.42gev/torus+1/pass1/dst/train/nSidis/*.hipo; do bin/qtl xcharge -m charge -i $f -o test_charge -s original; done # after reheat -for f in ~/v/reheat/rga_sp18_6.4GeV_outbending_nSidis/*.hipo; do bin/qtl xcharge -m charge -i $f -o test_charge -s original; done +for f in ~/v/reheat/rga_sp18_6.4GeV_outbending_nSidis/*.hipo; do bin/qtl xcharge -m charge -i $f -o test_charge -s reheated; done ``` then see files in `test_charge/`. diff --git a/qadb/notes/rgk_fa18.md b/qadb/notes/rgk_fa18.md index c3fbe6534..3168bdfd6 100644 --- a/qadb/notes/rgk_fa18.md +++ b/qadb/notes/rgk_fa18.md @@ -1,37 +1,62 @@ # Run Group K, Fall 2018, Pass 2 -## Run monitoring - -> [!IMPORTANT] -> Check any run-dependent settings in `qa-physics/monitorRead.groovy`, such as beam energy. - We will use SIDIS skims from two datasets: one for a 6.5 GeV beam, and another for a 7.5 GeV beam; we will keep them separate: - `rgk_fa18_6.5GeV`: from `/cache/clas12/rg-k/production/recon/fall2018/torus+1/6535MeV/pass2/v0/dst/train/skim1` - `rgk_fa18_7.5GeV`: from `/cache/clas12/rg-k/production/recon/fall2018/torus+1/7546MeV/pass2/v0/dst/train/skim1` -- **6.5 GeV:** +Before anything, cross check the train and DST run lists: ```bash -qtl histogram --check-cache -d rgk_fa18_6.5GeV --flatdir --focus-physics /cache/clas12/rg-k/production/recon/fall2018/torus+1/6535MeV/pass2/v0/dst/train/skim1 -qtl histogram -d rgk_fa18_6.5GeV --flatdir --focus-physics /cache/clas12/rg-k/production/recon/fall2018/torus+1/6535MeV/pass2/v0/dst/train/skim1 +bin/qtl xtrain /mss/clas12/rg-k/production/recon/fall2018/torus+1/6535MeV/pass2/v0/dst/train/skim1 /mss/clas12/rg-k/production/recon/fall2018/torus+1/6535MeV/pass2/v0/dst/recon +bin/qtl xtrain /mss/clas12/rg-k/production/recon/fall2018/torus+1/7546MeV/pass2/v0/dst/train/skim1 /mss/clas12/rg-k/production/recon/fall2018/torus+1/7546MeV/pass2/v0/dst/recon ``` -- **7.5 GeV:** +## Reheat + +> [!WARNING] +> The FC charge from the Pass 2 data files is incorrect, since they were cooked with `recharge` set to `false`; therefore, we need to "reheat" the data. +> See `rga_sp18.md` for some more details. + +1. make sure all data are on `/cache`; re-cache them if necessary: ```bash +qtl histogram --check-cache -d rgk_fa18_6.5GeV --flatdir --focus-physics /cache/clas12/rg-k/production/recon/fall2018/torus+1/6535MeV/pass2/v0/dst/train/skim1 qtl histogram --check-cache -d rgk_fa18_7.5GeV --flatdir --focus-physics /cache/clas12/rg-k/production/recon/fall2018/torus+1/7546MeV/pass2/v0/dst/train/skim1 -qtl histogram -d rgk_fa18_7.5GeV --flatdir --focus-physics /cache/clas12/rg-k/production/recon/fall2018/torus+1/7546MeV/pass2/v0/dst/train/skim1 +``` +2. run reheat: +```bash +bin/qtl reheat -m rollover -c 13.3.0 rgk_fa18_6.5GeV -o /volatile/clas12/users/$LOGNAME/reheat/rgk_fa18_6.5GeV -i /cache/clas12/rg-k/production/recon/fall2018/torus+1/6535MeV/pass2/v0/dst/train/skim1 +bin/qtl reheat -m rollover -c 13.3.0 rgk_fa18_7.5GeV -o /volatile/clas12/users/$LOGNAME/reheat/rgk_fa18_7.5GeV -i /cache/clas12/rg-k/production/recon/fall2018/torus+1/7546MeV/pass2/v0/dst/train/skim1 +``` +3. check the results: +```bash +# before +for f in /cache/clas12/rg-k/production/recon/fall2018/torus+1/6535MeV/pass2/v0/dst/train/skim1/*.hipo; do bin/qtl xcharge -m charge -i $f -o test_charge_rgk_6.5GeV -s original; done +for f in /cache/clas12/rg-k/production/recon/fall2018/torus+1/7546MeV/pass2/v0/dst/train/skim1/*.hipo; do bin/qtl xcharge -m charge -i $f -o test_charge_rgk_7.5GeV -s original; done +# after +for f in /volatile/clas12/users/$LOGNAME/reheat/rgk_fa18_6.5GeV/*.hipo; do bin/qtl xcharge -m charge -i $f -o test_charge_rgk_6.5GeV -s reheated; done +for f in /volatile/clas12/users/$LOGNAME/reheat/rgk_fa18_7.5GeV/*.hipo; do bin/qtl xcharge -m charge -i $f -o test_charge_rgk_7.5GeV -s reheated; done +``` + + +## Run monitoring + +> [!IMPORTANT] +> Check any run-dependent settings in `qa-physics/monitorRead.groovy`, such as beam energy. + +```bash +qtl histogram -d rgk_fa18_6.5GeV --flatdir --focus-physics /volatile/clas12/users/$LOGNAME/reheat/rgk_fa18_6.5GeV +qtl histogram -d rgk_fa18_7.5GeV --flatdir --focus-physics /volatile/clas12/users/$LOGNAME/reheat/rgk_fa18_7.5GeV ``` ## Double check that we have all the runs > [!IMPORTANT] -> In case any runs disappeared from `/cache` while running monitoring, be sure to cross check the output -> runs with those from `/mss` +> In case any runs disappeared from `/cache` while running monitoring or reheating, be sure to cross check the output runs with those from `/mss` ## Make timelines > [!IMPORTANT] > Pass 2 run 5863 was mistakenly cooked into the 6.5 GeV, but it has beam energy 7.5 GeV. To make sure its QA -> is performed in the correct epoch, move its step-1 output files to the correct output dataset directory: +> is performed in the correct epoch, move its `outfiles/` files to the correct output dataset directory: > ```bash > mv -v outfiles/rgk_fa18_6.5GeV/timeline_physics/5863 outfiles/rgk_fa18_7.5GeV/timeline_physics/ > ``` From a930c0d9f23f35f61d6ce2fa0dfdc0fdcaf3bd08 Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Mon, 16 Mar 2026 18:14:41 -0400 Subject: [PATCH 09/12] doc: add reheating to the checklist --- doc/qa.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/doc/qa.md b/doc/qa.md index f08093694..110796595 100644 --- a/doc/qa.md +++ b/doc/qa.md @@ -70,6 +70,16 @@ If you are performing a manual QA as part of a cross check, skip to the next sec - run `qtl histogram --check-cache` again, when done, in case additional files were auto-removed from `/cache` during your `jcache` run +
+- [ ] reheat the data, if necessary + +- see [RG-A Spring 2018](/qadb/notes/rga_sp18.md) and [RG-K Fall 2018](/qadb/notes/rgk_fa18.md) for details +- basically: + - use `qtl xcharge` to check the charge + - if reheating is needed, use `qtl reheat` + - use `qtl xcharge` afterward to check the reheated charge +
+
- [ ] verify run-dependent settings are correct for these data From e9b3e0c526cd2667321ad3b59a7a5bf5680c1a89 Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Mon, 16 Mar 2026 18:26:19 -0400 Subject: [PATCH 10/12] feat: `qtl xtrain` for multiple train dirs --- bin/qtl-xtrain | 120 ++++++++++++++++++++++++++++--------------------- 1 file changed, 69 insertions(+), 51 deletions(-) diff --git a/bin/qtl-xtrain b/bin/qtl-xtrain index 11fb755aa..d18b57b1f 100755 --- a/bin/qtl-xtrain +++ b/bin/qtl-xtrain @@ -7,67 +7,85 @@ unless ARGV.length == 2 Verify that a directory of a train's skim files has the same list of run numbers as a directory of DST-file run directories. - USAGE: qtl xtrain [TRAIN_DIR] [DST_DIR] + USAGE: qtl xtrain [TRAIN_DIR(S)] [DST_DIR] Both directories must be on /mss + + Use globs to specify multiple TRAIN_DIRS (surround in quotes) """ exit 2 end -train_dir, dst_dir = ARGV -# function to get a set of run numbers from one of the argument dirs -def get_runnums(path, type) - runnums = Set.new - raise "#{type} dir `#{path}` is not on /mss" unless path.match? /^\/mss\// - raise "#{type} dir `#{path}` does not exist" unless Dir.exist? path +train_dirs, dst_dir = ARGV +num_failures = 0 - # get list of files/directories within - files = [] - case type - when :train - files = Dir.glob File.join(path, '*.hipo') - when :DST - files = Dir.glob File.join(path, '*/') - else - raise 'bad type' - end - raise "no #{type} files found in #{type} dir `#{path}`" if files.empty? +# loop over train directories glob +Dir.glob(train_dirs).each do |train_dir| + + # function to get a set of run numbers from one of the argument dirs + def get_runnums(path, type) + runnums = Set.new + raise "#{type} dir `#{path}` is not on /mss" unless path.match? /^\/mss\// + raise "#{type} dir `#{path}` does not exist" unless Dir.exist? path - # extract their run numbers - files.each do |file| - nums = File.basename(file).scan(/\d+/).map &:to_i - raise "failed to get run number from #{type} object `#{file}`" unless nums.length == 1 - runnums << nums[0] + # get list of files/directories within + files = [] + case type + when :train + files = Dir.glob File.join(path, '*.hipo') + when :DST + files = Dir.glob File.join(path, '*/') + else + raise 'bad type' + end + raise "no #{type} files found in #{type} dir `#{path}`" if files.empty? + + # extract their run numbers + files.each do |file| + nums = File.basename(file).scan(/\d+/).map &:to_i + raise "failed to get run number from #{type} object `#{file}`" if nums.empty? + runnums << nums[-1] + end + raise "failed to get run numbers from #{type} dir `#{path}`" if runnums.empty? + runnums end - raise "failed to get run numbers from #{type} dir `#{path}`" if runnums.empty? - runnums -end -# get runnum lists -train_runs = get_runnums train_dir, :train -dst_runs = get_runnums dst_dir, :DST -puts """---------------------------------------------------------------------------------- -train dir run list: -#{train_runs} -DST dir run list: -#{dst_runs} -----------------------------------------------------------------------------------""" + # get runnum lists + train_runs = get_runnums train_dir, :train + dst_runs = get_runnums dst_dir, :DST + puts """ + ================================================================================== + train dir: #{train_dir} + run list: #{train_runs} + ----- + DST dir: #{dst_dir} + run list: #{dst_runs} + ==================================================================================""" -# compare runnum sets -only_in_trains = train_runs - dst_runs -only_in_dsts = dst_runs - train_runs + # compare runnum sets + only_in_trains = train_runs - dst_runs + only_in_dsts = dst_runs - train_runs -# return results -code = 0 -unless only_in_trains.empty? - $stderr.puts "ERROR: there are runs with skim files, but no corresponding DST-file directories:" - $stderr.puts only_in_trains - code = 1 -end -unless only_in_dsts.empty? - $stderr.puts "ERROR: there are runs with DST-file directories, but no corresponding skim files:" - $stderr.puts only_in_dsts - code = 1 + # print results + $stdout.flush + $stderr.flush + code = 0 + unless only_in_trains.empty? + $stderr.puts "ERROR: there are runs with skim files, but no corresponding DST-file directories:" + $stderr.puts only_in_trains + code = 1 + end + unless only_in_dsts.empty? + $stderr.puts "ERROR: there are runs with DST-file directories, but no corresponding skim files:" + $stderr.puts only_in_dsts + code = 1 + end + + # handle exit code + if code == 0 + puts "All good" + else + num_failures += 1 + end end -puts "All good" if code == 0 -exit code +exit num_failures > 0 ? 1 : 0 From 10d7655d1a7d2a39711cfd905b3ca80944c14985 Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Mon, 16 Mar 2026 18:32:29 -0400 Subject: [PATCH 11/12] fix: filter by dir --- bin/qtl-xtrain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/qtl-xtrain b/bin/qtl-xtrain index d18b57b1f..920d03cbd 100755 --- a/bin/qtl-xtrain +++ b/bin/qtl-xtrain @@ -20,7 +20,7 @@ train_dirs, dst_dir = ARGV num_failures = 0 # loop over train directories glob -Dir.glob(train_dirs).each do |train_dir| +Dir.glob(train_dirs).select{|d|File.directory? d}.each do |train_dir| # function to get a set of run numbers from one of the argument dirs def get_runnums(path, type) From 7f9b76a4658bf61cd6055cc5541dcc54a7703a77 Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Tue, 17 Mar 2026 10:55:49 -0400 Subject: [PATCH 12/12] fix: allow `xtrain` to use `/cache` --- bin/qtl-xtrain | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/qtl-xtrain b/bin/qtl-xtrain index 920d03cbd..aee80d452 100755 --- a/bin/qtl-xtrain +++ b/bin/qtl-xtrain @@ -9,7 +9,8 @@ unless ARGV.length == 2 USAGE: qtl xtrain [TRAIN_DIR(S)] [DST_DIR] - Both directories must be on /mss + Both directories must be on /mss or /cache; the /mss dir is checked + (even if the argument is /cache) Use globs to specify multiple TRAIN_DIRS (surround in quotes) """ @@ -25,7 +26,7 @@ Dir.glob(train_dirs).select{|d|File.directory? d}.each do |train_dir| # function to get a set of run numbers from one of the argument dirs def get_runnums(path, type) runnums = Set.new - raise "#{type} dir `#{path}` is not on /mss" unless path.match? /^\/mss\// + path = path.sub /^\/cache\//, '/mss/' if path.match? /^\/cache\// raise "#{type} dir `#{path}` does not exist" unless Dir.exist? path # get list of files/directories within