Skip to content

Commit 7ec9437

Browse files
singhd789singhd789
authored
Merge pull request #229 from singhd789/216.refinediag-preanalysis-tests
Add refinediag preanalysis tests
2 parents b80c7f1 + 7c6c9bc commit 7ec9437

11 files changed

Lines changed: 207 additions & 23 deletions

File tree

‎.github/workflows/test_cloud_runner.yml‎

Lines changed: 107 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -74,10 +74,6 @@ jobs:
7474
7575
singularity exec --writable-tmpfs ${bind_paths} ${container_path} ${runscript_path}
7676
77-
# Can potentially do this to get list of tasks
78-
#task_list=$(cylc list test_pp__ptest__ttest)
79-
#task1=$(echo $task_list | cut -d ' ' -f1) ....
80-
8177
- name: Upload workflow-run log files
8278
uses: actions/upload-artifact@v4
8379
with:
@@ -166,6 +162,106 @@ jobs:
166162
exit 2
167163
fi
168164
165+
- name: Print stage-history-refined successes or failures
166+
run: |
167+
set +e
168+
# Successes
169+
success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/stage-history-refined/0[1-3]/job.status)
170+
exit_status_1=$?
171+
# Failures
172+
failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/stage-history-refined/0[1-3]/job.status)
173+
exit_status_2=$?
174+
if [ $exit_status_1 -eq 0 ]; then
175+
echo $success | sed 's/ /\n/g' #split string by spaces
176+
elif [ $exit_status_1 -eq 1 ]; then
177+
echo "No succeeded tasks found"
178+
else
179+
echo "WARNING: error with grep execution"
180+
exit 2
181+
fi
182+
if [ $exit_status_2 -eq 0 ]; then
183+
echo "Failures found"
184+
echo $failure | sed 's/ /\n/g' #split string by spaces
185+
exit 1 #exit with error
186+
elif [ $exit_status_2 -eq 1 ]; then
187+
echo "No task failures found."
188+
else
189+
echo "WARNING: error with grep execution"
190+
exit 2
191+
fi
192+
193+
- name: Print refine-diag successes or failures
194+
run: |
195+
set +e
196+
# Successes
197+
success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/refineDiag-*/0[1-3]/job.status)
198+
exit_status_1=$?
199+
200+
# Failures
201+
failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/refineDiag-*/0[1-3]/job.status)
202+
exit_status_2=$?
203+
204+
if [ $exit_status_1 -eq 0 ]; then
205+
echo $success | sed 's/ /\n/g' #split string by spaces
206+
207+
# Check if refine-diag task executed correctly
208+
# history files copied into another history file
209+
# ........
210+
211+
elif [ $exit_status_1 -eq 1 ]; then
212+
echo "No succeeded tasks found"
213+
else
214+
echo "WARNING: error with grep execution"
215+
exit 2
216+
fi
217+
218+
if [ $exit_status_2 -eq 0 ]; then
219+
echo "Failures found"
220+
echo $failure | sed 's/ /\n/g' #split string by spaces
221+
exit 1 #exit with error
222+
elif [ $exit_status_2 -eq 1 ]; then
223+
echo "No task failures found."
224+
else
225+
echo "WARNING: error with grep execution"
226+
exit 2
227+
fi
228+
229+
- name: Print pre-analysis successes or failures
230+
run: |
231+
set +e
232+
# Successes
233+
success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/preAnalysis-*/0[1-3]/job.status)
234+
exit_status_1=$?
235+
# Failures
236+
failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/preAnalysis-*/0[1-3]/job.status)
237+
exit_status_2=$?
238+
if [ $exit_status_1 -eq 0 ]; then
239+
echo $success | sed 's/ /\n/g' #split string by spaces
240+
# Check if preanalysis task executed correctly: Check database exists
241+
if [ -s "/contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/db" ]; then
242+
echo "Pre-analysis task produced non-empty db file"
243+
exit 0
244+
else
245+
echo "Pre-analysis task did not produce expected, non-empty db file"
246+
exit 1
247+
fi
248+
elif [ $exit_status_1 -eq 1 ]; then
249+
echo "No succeeded tasks found"
250+
else
251+
echo "WARNING: error with grep execution"
252+
exit 2
253+
fi
254+
if [ $exit_status_2 -eq 0 ]; then
255+
echo "Failures found"
256+
echo $failure | sed 's/ /\n/g' #split string by spaces
257+
exit 1 #exit with error
258+
elif [ $exit_status_2 -eq 1 ]; then
259+
echo "No task failures found."
260+
else
261+
echo "WARNING: error with grep execution"
262+
exit 2
263+
fi
264+
169265
- name: Print regrid-xy successes or failures
170266
run: |
171267
set +e
@@ -446,6 +542,13 @@ jobs:
446542
exit 2
447543
fi
448544
545+
## I think this will work (without cylc clean) because the
546+
## OUTPUT_DIR is bind mounted from the host
547+
## This is to retain clean workspace on cloud and to not waste
548+
## allocation
549+
- name: Clean up output folder
550+
run: rm -rf /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest
551+
449552
## Keep in here for now as a just in case reference ##
450553
# - name: Print any succeeded tasks
451554
# run: |

‎flow.cylc‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -465,6 +465,7 @@
465465
env
466466
set -x
467467
mkdir -p $work $tempCache $refineDiagDir
468+
hsmget -v -t -a $histDir -p {{ PTMP_DIR }}/$histDir -w $work $hsmdate/\*
468469
cd $work/$hsmdate
469470
ls
470471
"""
@@ -500,19 +501,23 @@
500501
fi
501502
if [[ $refinedCount > 0 ]]; then
502503
for file in $(ls -1 *nc); do
503-
list_ncvars.csh -st01234 $file |& tee $CYLC_WORKFLOW_SHARE_DIR/refineDiag.log
504+
# list the variables names in the netcdf file
505+
ncdump -h $file |
506+
grep -E "float|double" |
507+
awk '{print $2}' |
508+
cut -d '(' -f1 |& tee $CYLC_WORKFLOW_SHARE_DIR/refineDiag.log
504509
done
505510
else
506511
echo ERROR: RefineDiag script did not create any NetCDF files as it was expected to do
507512
exit 1
508513
fi
509514
if [[ -f {{ REFINED_HISTORY_DIR }}/$oname.nc.tar ]]; then
515+
hsmget -v -t -a {{ REFINED_HISTORY_DIR }} -p {{ PTMP_DIR }}/{{ REFINED_HISTORY_DIR }} \
516+
-w $TMPDIR/modify_refineDiag $hsmdate/\*
510517
echo "the contents of {{ PTMP_DIR }}/{{ REFINED_HISTORY_DIR }} is..."
511518
ls {{ PTMP_DIR }}/{{ REFINED_HISTORY_DIR }}
512519
echo "the contents of {{ PTMP_DIR }}/{{ REFINED_HISTORY_DIR }}/$oname.nc is..."
513520
ls {{ PTMP_DIR }}/{{ REFINED_HISTORY_DIR }}/$oname.nc
514-
hsmget -v -t -a {{ REFINED_HISTORY_DIR }} -p {{ PTMP_DIR }}/{{ REFINED_HISTORY_DIR }} \
515-
-w $TMPDIR/modify_refineDiag $hsmdate/\*
516521
mv -f * $TMPDIR/modify_refineDiag
517522
mv -f $TMPDIR/modify_refineDiag/* .
518523
rm -rf $TMPDIR/modify_refineDiag

‎for_gh_runner/dummy_preanalysis.csh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/bin/csh
2+
3+
echo "Pre-analysis script is running!"

‎for_gh_runner/dummy_refinediag.csh‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#!/bin/csh
2+
3+
## RefineDiag scripts reads raw history files and generates new, refined history files
4+
# Script copies test history file to another history file
5+
set input_dir = `pwd`
6+
7+
if ( $?refineDiagDir ) then
8+
set output_dir = $refineDiagDir
9+
else
10+
echo "ERROR: refineDiagDir environment variable is not set."
11+
exit 1
12+
endif
13+
14+
if ( -d "$input_dir" ) then
15+
#ls -aF
16+
echo "Input directory found: $input_dir"
17+
echo "good to go"
18+
foreach INFILE (*atmos_month*.nc)
19+
set OUTFILE = ${INFILE:r}_refined.${INFILE:e}
20+
echo "OUTFILE $OUTFILE"
21+
cp "$INFILE" "$output_dir/$OUTFILE"
22+
end
23+
else
24+
echo "Error: Input directory $input_dir does not exist."
25+
exit 1
26+
endif

‎for_gh_runner/run_pp_locally.sh‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,13 @@ if [ -d $cylc_src_dir_name ]; then
5858
sleep 10s
5959
fi
6060

61+
## More cleaning neede for refineDiag output
62+
if [ -d /work/$USER/fre-wf-local-testing/refined_history ]; then
63+
echo -e "Refine Diag scripts previously run, removing ..."
64+
rm -rf "/work/$USER/fre-wf-local-testing/refined_history"
65+
fi
66+
67+
6168
# if you want this, comment on https://github.com/NOAA-GFDL/fre-cli/issues/673
6269
#echo ""
6370
#echo ""

‎for_gh_runner/runscript.sh‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -75,13 +75,20 @@ check_exit_status () {
7575
fre_pp_steps () {
7676
set -x
7777

78+
## Clean previous experiment
7879
echo "experiment cleaning, if it was previously installed"
7980
if [ -d /mnt/cylc-run/${name} ]; then
8081
echo -e "\n${name} previously installed"
8182
echo " Removing ${name}..."
8283
cylc clean ${name}
8384
fi
8485

86+
## More cleaning needed for refineDiag output
87+
if [ -d /mnt/$USER/refined_history ]; then
88+
echo -e "Refine Diag scripts previously run, removing ..."
89+
rm -rf /mnt/$USER/refined_history
90+
fi
91+
8592
## Checkout
8693
echo -e "\nCreating $name directory in ${HOME}/cylc-src/${name} ..."
8794
rm -rf /mnt/cylc-src/${name}
@@ -94,13 +101,13 @@ fre_pp_steps () {
94101
#Not sure if needed because if no global.cylc found, cylc uses default, which utilizes background jobs anyway ...
95102
#export CYLC_CONF_PATH=/mnt/cylc-src/${name}/generic-global-config/
96103

97-
## Configure the rose-suite and rose-app files for the workflow
98-
echo -e "\nRunning fre pp configure-yaml, combining separate yaml configs into one, then writing rose-suite/app config files ..."
104+
## Configure the rose-suite file for the workflow
105+
echo -e "\nRunning fre pp configure-yaml, combining separate yaml configs into one, then writing rose-suite config file ..."
99106
fre -vv pp configure-yaml -e ${expname} -p ${plat} -t ${targ} -y ${yamlfile}
100107
check_exit_status "CONFIGURE-YAML"
101108

102109
## Validate the configuration files
103-
echo -e "\nRunning fre pp validate, validating rose-suite/app config files ..."
110+
echo -e "\nRunning fre pp validate, validating rose-suite config file ..."
104111
fre -vv pp validate -e ${expname} -p ${plat} -t ${targ}
105112
check_exit_status "VALIDATE"
106113

‎for_gh_runner/yaml_workflow/local_settings.yaml‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,36 @@
11
directories:
22
## to test staging data from archive, use the below
33
# history_dir: "/archive/inl/test_cloud_runner_history_files"
4+
#
45
# to have a faster turn-around (does not test archive data staging), use the below
5-
history_dir: "/work/inl/test_cloud_runner_history_files"
6+
history_dir: "/work/inl/test_cloud_runner_history_files"
7+
refined_history_dir: "/work/$USER/fre-wf-local-testing/refined_history"
68
pp_dir: !join [/work/$USER/test_cloud_runner_scratch_dir/, *FRE_STEM, /, pp]
79
# analysis_dir: !join [/nbhome/$USER/, *FRE_STEM, /, *name]
810
ptmp_dir: "/xtmp/$USER/ptmp"
911

1012
postprocess:
1113
settings:
1214
history_segment: "P1Y"
13-
# site: "ppan"
14-
site: "ppan_test"
15+
site: "ppan"
16+
# site: "ppan_test"
1517
pp_chunks: [*PP_CHUNK96]
1618
pp_start: *EXP_AMIP_START
1719
pp_stop: *EXP_AMIP_END
1820
pp_grid_spec: *GRID_SPEC96_LOCAL
1921
switches:
2022
clean_work: True
2123
do_atmos_plevel_masking: True
24+
25+
# to be first-step-user-script
26+
preanalysis:
27+
vitals:
28+
script: "/work/d4s/fre-wf-local-testing/dummy_preanalysis.csh"
29+
inputs: ['atmos_month', 'aerosol_month'] # not used currently
30+
do_preanalysis: True
31+
refinediag:
32+
ocean:
33+
script: "/work/d4s/fre-wf-local-testing/dummy_refinediag.csh"
34+
inputs: ['ocean_monthly', 'ocean_bgc'] # not used currently
35+
outputs: ['ocean_refined'] # not used currently
36+
do_refinediag: True

‎for_gh_runner/yaml_workflow/settings.yaml‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
# directories shared across tools
22
directories:
3-
history_dir: "/mnt/history"
4-
pp_dir: "/mnt/pp"
3+
history_dir: "/mnt/history"
4+
refined_history_dir: "/mnt/$USER/refined_history"
5+
pp_dir: "/mnt/pp"
56
# analysis_dir: !join [/nbhome/$USER/, *FRE_STEM, /, *name]
6-
ptmp_dir: "/mnt/ptmp"
7+
ptmp_dir: "/mnt/ptmp"
78

89
postprocess:
910
settings:
@@ -16,3 +17,20 @@ postprocess:
1617
switches:
1718
clean_work: True
1819
do_atmos_plevel_masking: True
20+
21+
# to be first-step-user-scripts
22+
preanalysis:
23+
vitals:
24+
script: "/contrib-efs/fre-workflows-runners/actions-runner-1/_work/fre-workflows/fre-workflows/for_gh_runner/dummy_preanalysis.csh"
25+
inputs: ['atmos_month', 'aerosol_month'] #not used currently
26+
# to be user-script_on
27+
do_preanalysis: True
28+
29+
# to be refinediag-user-scripts
30+
refinediag:
31+
ocean:
32+
script: "/contrib-efs/fre-workflows-runners/actions-runner-1/_work/fre-workflows/fre-workflows/for_gh_runner/dummy_refinediag.csh"
33+
inputs: ['ocean_monthly', 'ocean_bgc'] # not used currently
34+
outputs: ['ocean_refined'] # not used currently
35+
# to be user_script_on
36+
do_refinediag: True

‎site/gaea.cylc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
COPY_TOOL=gcp
2323

2424
[[STAGE-HISTORY]]
25-
pre-script = module load hsm/1.3.0
25+
pre-script = module load hsm/1.4.0
2626

2727
[[MAKE-TIMESERIES]]
2828
pre-script = module load cdo

‎site/ppan.cylc‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
#module load conda
99
#conda activate user-fre-cli
1010
11-
which fre
11+
#which fre
1212
module load gcp/2.3
1313
module list
1414
"""
@@ -24,15 +24,15 @@
2424

2525
[[STAGE-HISTORY]]
2626
pre-script = """
27-
module load gcp/2.3 hsm/1.3.0
27+
module load gcp/2.3 hsm/1.4.0
2828
"""
2929

3030
{% if DO_REFINEDIAG or DO_PREANALYSIS %}
3131
[[PRE-ANALYSIS]]
3232
pre-script = """
3333
env
3434
set -x
35-
module load hsm/1.3.0
35+
module load hsm/1.4.0
3636
module load gcp
3737
# needed only for list_ncvars.
3838
module load fre-nctools/2024.05.01

0 commit comments

Comments
 (0)
⚡