diff --git a/bin/qtl-xtrain b/bin/qtl-xtrain index 11fb755aa..aee80d452 100755 --- a/bin/qtl-xtrain +++ b/bin/qtl-xtrain @@ -7,67 +7,86 @@ unless ARGV.length == 2 Verify that a directory of a train's skim files has the same list of run numbers as a directory of DST-file run directories. - USAGE: qtl xtrain [TRAIN_DIR] [DST_DIR] + USAGE: qtl xtrain [TRAIN_DIR(S)] [DST_DIR] - Both directories must be on /mss + Both directories must be on /mss or /cache; the /mss dir is checked + (even if the argument is /cache) + + Use globs to specify multiple TRAIN_DIRS (surround in quotes) """ exit 2 end -train_dir, dst_dir = ARGV -# function to get a set of run numbers from one of the argument dirs -def get_runnums(path, type) - runnums = Set.new - raise "#{type} dir `#{path}` is not on /mss" unless path.match? /^\/mss\// - raise "#{type} dir `#{path}` does not exist" unless Dir.exist? path +train_dirs, dst_dir = ARGV +num_failures = 0 - # get list of files/directories within - files = [] - case type - when :train - files = Dir.glob File.join(path, '*.hipo') - when :DST - files = Dir.glob File.join(path, '*/') - else - raise 'bad type' - end - raise "no #{type} files found in #{type} dir `#{path}`" if files.empty? +# loop over train directories glob +Dir.glob(train_dirs).select{|d|File.directory? d}.each do |train_dir| + + # function to get a set of run numbers from one of the argument dirs + def get_runnums(path, type) + runnums = Set.new + path = path.sub /^\/cache\//, '/mss/' if path.match? /^\/cache\// + raise "#{type} dir `#{path}` does not exist" unless Dir.exist? path - # extract their run numbers - files.each do |file| - nums = File.basename(file).scan(/\d+/).map &:to_i - raise "failed to get run number from #{type} object `#{file}`" unless nums.length == 1 - runnums << nums[0] + # get list of files/directories within + files = [] + case type + when :train + files = Dir.glob File.join(path, '*.hipo') + when :DST + files = Dir.glob File.join(path, '*/') + else + raise 'bad type' + end + raise "no #{type} files found in #{type} dir `#{path}`" if files.empty? + + # extract their run numbers + files.each do |file| + nums = File.basename(file).scan(/\d+/).map &:to_i + raise "failed to get run number from #{type} object `#{file}`" if nums.empty? + runnums << nums[-1] + end + raise "failed to get run numbers from #{type} dir `#{path}`" if runnums.empty? + runnums end - raise "failed to get run numbers from #{type} dir `#{path}`" if runnums.empty? - runnums -end -# get runnum lists -train_runs = get_runnums train_dir, :train -dst_runs = get_runnums dst_dir, :DST -puts """---------------------------------------------------------------------------------- -train dir run list: -#{train_runs} -DST dir run list: -#{dst_runs} -----------------------------------------------------------------------------------""" + # get runnum lists + train_runs = get_runnums train_dir, :train + dst_runs = get_runnums dst_dir, :DST + puts """ + ================================================================================== + train dir: #{train_dir} + run list: #{train_runs} + ----- + DST dir: #{dst_dir} + run list: #{dst_runs} + ==================================================================================""" -# compare runnum sets -only_in_trains = train_runs - dst_runs -only_in_dsts = dst_runs - train_runs + # compare runnum sets + only_in_trains = train_runs - dst_runs + only_in_dsts = dst_runs - train_runs -# return results -code = 0 -unless only_in_trains.empty? - $stderr.puts "ERROR: there are runs with skim files, but no corresponding DST-file directories:" - $stderr.puts only_in_trains - code = 1 -end -unless only_in_dsts.empty? - $stderr.puts "ERROR: there are runs with DST-file directories, but no corresponding skim files:" - $stderr.puts only_in_dsts - code = 1 + # print results + $stdout.flush + $stderr.flush + code = 0 + unless only_in_trains.empty? + $stderr.puts "ERROR: there are runs with skim files, but no corresponding DST-file directories:" + $stderr.puts only_in_trains + code = 1 + end + unless only_in_dsts.empty? + $stderr.puts "ERROR: there are runs with DST-file directories, but no corresponding skim files:" + $stderr.puts only_in_dsts + code = 1 + end + + # handle exit code + if code == 0 + puts "All good" + else + num_failures += 1 + end end -puts "All good" if code == 0 -exit code +exit num_failures > 0 ? 1 : 0 diff --git a/qadb/cutdefs/rgk_fa18_6.5GeV.yaml b/qadb/cutdefs/rgk_fa18_6.5GeV.yaml new file mode 100644 index 000000000..340462978 --- /dev/null +++ b/qadb/cutdefs/rgk_fa18_6.5GeV.yaml @@ -0,0 +1,12 @@ +OutlierFD: + IQR_cut_factor: 4.0 +OutlierFT: + IQR_cut_factor: 4.0 +LowLiveTime: + min_live_time: 0.9 +ChargeHigh: + IQR_cut_factor: 4.0 +PossiblyNoBeam: + max_num_events: 40000 + max_num_electrons: 100 + max_FC_charge: 20 diff --git a/qadb/cutdefs/rgk_fa18_7.5GeV.yaml b/qadb/cutdefs/rgk_fa18_7.5GeV.yaml new file mode 120000 index 000000000..bd27b6ca2 --- /dev/null +++ b/qadb/cutdefs/rgk_fa18_7.5GeV.yaml @@ -0,0 +1 @@ +rgk_fa18_6.5GeV.yaml \ No newline at end of file diff --git a/qadb/epochs/epochs.rgk_6.txt b/qadb/epochs/epochs.rgk_fa18_6.5GeV.txt similarity index 100% rename from qadb/epochs/epochs.rgk_6.txt rename to qadb/epochs/epochs.rgk_fa18_6.5GeV.txt diff --git a/qadb/epochs/epochs.rgk_7.txt b/qadb/epochs/epochs.rgk_fa18_7.5GeV.txt similarity index 100% rename from qadb/epochs/epochs.rgk_7.txt rename to qadb/epochs/epochs.rgk_fa18_7.5GeV.txt diff --git a/qadb/notes/rga_sp18.md b/qadb/notes/rga_sp18.md index f192b1ce5..b1e663306 100644 --- a/qadb/notes/rga_sp18.md +++ b/qadb/notes/rga_sp18.md @@ -95,7 +95,7 @@ bin/qtl reheat -m rollover -c 13.3.0 -d rga_sp18_6.4GeV_inbending_lo_nSidis -o for f in /cache/clas12/rg-a/production/recon/spring2018/6.42gev/torus+1/pass1/dst/train/nSidis/*.hipo; do bin/qtl xcharge -m charge -i $f -o test_charge -s original; done # after reheat -for f in ~/v/reheat/rga_sp18_6.4GeV_outbending_nSidis/*.hipo; do bin/qtl xcharge -m charge -i $f -o test_charge -s original; done +for f in ~/v/reheat/rga_sp18_6.4GeV_outbending_nSidis/*.hipo; do bin/qtl xcharge -m charge -i $f -o test_charge -s reheated; done ``` then see files in `test_charge/`. diff --git a/qadb/notes/rgk_fa18.md b/qadb/notes/rgk_fa18.md new file mode 100644 index 000000000..3168bdfd6 --- /dev/null +++ b/qadb/notes/rgk_fa18.md @@ -0,0 +1,71 @@ +# Run Group K, Fall 2018, Pass 2 + +We will use SIDIS skims from two datasets: one for a 6.5 GeV beam, and another for a 7.5 GeV beam; we will keep them separate: +- `rgk_fa18_6.5GeV`: from `/cache/clas12/rg-k/production/recon/fall2018/torus+1/6535MeV/pass2/v0/dst/train/skim1` +- `rgk_fa18_7.5GeV`: from `/cache/clas12/rg-k/production/recon/fall2018/torus+1/7546MeV/pass2/v0/dst/train/skim1` + +Before anything, cross check the train and DST run lists: +```bash +bin/qtl xtrain /mss/clas12/rg-k/production/recon/fall2018/torus+1/6535MeV/pass2/v0/dst/train/skim1 /mss/clas12/rg-k/production/recon/fall2018/torus+1/6535MeV/pass2/v0/dst/recon +bin/qtl xtrain /mss/clas12/rg-k/production/recon/fall2018/torus+1/7546MeV/pass2/v0/dst/train/skim1 /mss/clas12/rg-k/production/recon/fall2018/torus+1/7546MeV/pass2/v0/dst/recon +``` + +## Reheat + +> [!WARNING] +> The FC charge from the Pass 2 data files is incorrect, since they were cooked with `recharge` set to `false`; therefore, we need to "reheat" the data. +> See `rga_sp18.md` for some more details. + +1. make sure all data are on `/cache`; re-cache them if necessary: +```bash +qtl histogram --check-cache -d rgk_fa18_6.5GeV --flatdir --focus-physics /cache/clas12/rg-k/production/recon/fall2018/torus+1/6535MeV/pass2/v0/dst/train/skim1 +qtl histogram --check-cache -d rgk_fa18_7.5GeV --flatdir --focus-physics /cache/clas12/rg-k/production/recon/fall2018/torus+1/7546MeV/pass2/v0/dst/train/skim1 +``` +2. run reheat: +```bash +bin/qtl reheat -m rollover -c 13.3.0 rgk_fa18_6.5GeV -o /volatile/clas12/users/$LOGNAME/reheat/rgk_fa18_6.5GeV -i /cache/clas12/rg-k/production/recon/fall2018/torus+1/6535MeV/pass2/v0/dst/train/skim1 +bin/qtl reheat -m rollover -c 13.3.0 rgk_fa18_7.5GeV -o /volatile/clas12/users/$LOGNAME/reheat/rgk_fa18_7.5GeV -i /cache/clas12/rg-k/production/recon/fall2018/torus+1/7546MeV/pass2/v0/dst/train/skim1 +``` +3. check the results: +```bash +# before +for f in /cache/clas12/rg-k/production/recon/fall2018/torus+1/6535MeV/pass2/v0/dst/train/skim1/*.hipo; do bin/qtl xcharge -m charge -i $f -o test_charge_rgk_6.5GeV -s original; done +for f in /cache/clas12/rg-k/production/recon/fall2018/torus+1/7546MeV/pass2/v0/dst/train/skim1/*.hipo; do bin/qtl xcharge -m charge -i $f -o test_charge_rgk_7.5GeV -s original; done +# after +for f in /volatile/clas12/users/$LOGNAME/reheat/rgk_fa18_6.5GeV/*.hipo; do bin/qtl xcharge -m charge -i $f -o test_charge_rgk_6.5GeV -s reheated; done +for f in /volatile/clas12/users/$LOGNAME/reheat/rgk_fa18_7.5GeV/*.hipo; do bin/qtl xcharge -m charge -i $f -o test_charge_rgk_7.5GeV -s reheated; done +``` + + +## Run monitoring + +> [!IMPORTANT] +> Check any run-dependent settings in `qa-physics/monitorRead.groovy`, such as beam energy. + +```bash +qtl histogram -d rgk_fa18_6.5GeV --flatdir --focus-physics /volatile/clas12/users/$LOGNAME/reheat/rgk_fa18_6.5GeV +qtl histogram -d rgk_fa18_7.5GeV --flatdir --focus-physics /volatile/clas12/users/$LOGNAME/reheat/rgk_fa18_7.5GeV +``` + +## Double check that we have all the runs + +> [!IMPORTANT] +> In case any runs disappeared from `/cache` while running monitoring or reheating, be sure to cross check the output runs with those from `/mss` + +## Make timelines + +> [!IMPORTANT] +> Pass 2 run 5863 was mistakenly cooked into the 6.5 GeV, but it has beam energy 7.5 GeV. To make sure its QA +> is performed in the correct epoch, move its `outfiles/` files to the correct output dataset directory: +> ```bash +> mv -v outfiles/rgk_fa18_6.5GeV/timeline_physics/5863 outfiles/rgk_fa18_7.5GeV/timeline_physics/ +> ``` + +Make the timelines, separately for each of the 2 datasets: +```bash +bin/qtl physics -d rgk_fa18_6.5GeV -p rgk/pass2/qa/fa18_6.5GeV +bin/qtl physics -d rgk_fa18_7.5GeV -p rgk/pass2/qa/fa18_7.5GeV +``` + +> [!WARNING] +> You may not have permission to publish to this directory; if not, publish to another place and ask the chef to copy the files