Skip to content

revision of diagnostic testing #1260

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
painter1 merged 5 commits into from
May 14, 2015
Merged

revision of diagnostic testing #1260

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
a0d6a7f
Part of the reorg of diagnostics testing. Also adding tests for plot …
mcenerney1 May 1, 2015
4033481
Added several tests.
mcenerney1 May 1, 2015
08616f8
Deleted the varopts in order to match baseline.
mcenerney1 May 1, 2015
26e1072
new set of test data
doutriaux1 May 4, 2015
4c044d1
Changed the directory for obs data to obs_data_5.6
mcenerney1 May 4, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 10 additions & 7 deletions CMake/cdat_modules/uvcmetrics_external.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,20 +1,23 @@
# create an external project to install MyProxyClient,
# and configure and build it
configure_file(${cdat_CMAKE_SOURCE_DIR}/cdat_modules_extra/fetch_uvcmetrics_testdata.py.in
${cdat_CMAKE_BINARY_DIR}/fetch_uvcmetrics_testdata.py
@ONLY)
if (CDAT_DOWNLOAD_UVCMETRICS_TESTDATA)
configure_file(${cdat_CMAKE_SOURCE_DIR}/cdat_modules_extra/fetch_uvcmetrics_testdata.py.in
${cdat_CMAKE_BINARY_DIR}/fetch_uvcmetrics_testdata.py
@ONLY)

execute_process(
COMMAND ${PYTHON_EXECUTABLE} ${cdat_CMAKE_BINARY_DIR}/fetch_uvcmetrics_testdata.py
execute_process(
COMMAND ${PYTHON_EXECUTABLE} ${cdat_CMAKE_BINARY_DIR}/fetch_uvcmetrics_testdata.py
WORKING_DIRECTORY ${cdat_SOURCE_DIR}
RESULT_VARIABLE res
OUTPUT_VARIABLE ver
)

if (NOT ${res} EQUAL 0)
message("[INFO] Failed to fetch test data for uvcmetrics, tests will fail")
if (NOT ${res} EQUAL 0)
message("[INFO] Failed to fetch test data for uvcmetrics, tests will fail")
endif()
endif()


set(GIT_CMD_STR GIT_REPOSITORY "${UVCMETRICS_SOURCE}")
set(GIT_TAG GIT_TAG "${UVCMETRICS_BRANCH}")
set(nm UVCMETRICS)
Expand Down
71 changes: 71 additions & 0 deletions CMake/cdat_modules_extra/uvcmetrics_test_data_md5s.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,33 @@ cffdc3aab308d233c956720d80671b95 acme_hires_land/c_t_t341f02.F1850p.clm2.h0.185
e6931415ab36579fff13f4933a6bf1f5 acme_hires_land/c_t_t341f02.F1850p.clm2.h0.1855-10.nc
526fbd9987a6d5faf927106bf048aa2b acme_hires_land/c_t_t341f02.F1850p.clm2.h0.1855-11.nc
839301c709e5a7b3eb271e75782979af acme_hires_land/c_t_t341f02.F1850p.clm2.h0.1855-12.nc
6d86e5edd0a92931226ac99d13167980 acme_lores_atm_climo/F1850.g37_bench_ANN_climo.nc
f3c5c5a4d91d6e3e0cc4d2df362b0503 acme_lores_atm_climo/F1850.g37_bench_DJF_climo.nc
06e0cc5f50cd7f2c1e2f30c4c4278b47 acme_lores_cam_climo/F1850.g37_bench_ANN_climo.nc
712d887975121e81168ab03a535cadba acme_lores_cam_climo/F1850.g37_bench_APR_climo.nc
e2e5505205c326eea69574226d881359 acme_lores_cam_climo/F1850.g37_bench_DJF_climo.nc
d36b0d4f7fb27c3897668131bdec05aa acme_lores_cam_climo/F1850.g37_bench_JAN_climo.nc
39342297493a616eb8988ef0a3a9c988 acme_lores_cam_climo/F1850.g37_bench_JJA_climo.nc
0a8cbf9b41f2cc752800a584f6356cbd acme_lores_cam_climo/F1850.g37_bench_JUL_climo.nc
7b2da1926acf2c0f9ffad80497775bb6 acme_lores_cam_climo/F1850.g37_bench_MAM_climo.nc
7df286b070640d0074c556560edc6a73 acme_lores_cam_climo/F1850.g37_bench_SON_climo.nc
77d7b6de33467bdebe1a05700f03cae7 acme_lores_clm_climo/ANN_climo.nc
a075f9d88b0b29b9f6a706f56bc628fa acme_lores_clm_climo/APR_climo.nc
f0694a365f88bef9f2ae34169afcd99b acme_lores_clm_climo/AUG_climo.nc
3928a8108bed42d5035bb9e9ef06a227 acme_lores_clm_climo/DEC_climo.nc
5cd00312d791f34b1d33ca336d874473 acme_lores_clm_climo/DJF_climo.nc
5e2849739943108c549c6724c6927ccd acme_lores_clm_climo/FEB_climo.nc
4badd8e20c7e45d8156f0677416d4f85 acme_lores_clm_climo/JAN_climo.nc
bcc44d7f27938f1f21cf3c34d29dfe0d acme_lores_clm_climo/JJA_climo.nc
c11b441acebdf5e7dac696485abd31b8 acme_lores_clm_climo/JUL_climo.nc
1740586484d8e59b18bf97d89658cd97 acme_lores_clm_climo/JUN_climo.nc
6aca924e7541a42f37c189934912d4bb acme_lores_clm_climo/MAM_climo.nc
16c8c8d84c30d2f72b1bafd7929841a5 acme_lores_clm_climo/MAR_climo.nc
eb483652fc0b0b069761659262d1d111 acme_lores_clm_climo/MAY_climo.nc
e3e52b82e64357c50fe42aed7e0ba56c acme_lores_clm_climo/NOV_climo.nc
8969b2045cd430d03cebaccb91995f3d acme_lores_clm_climo/OCT_climo.nc
4a1d44b3ab16645aef032006be8b4af3 acme_lores_clm_climo/SEP_climo.nc
f57a1c82229d2985894ef643e0392135 acme_lores_clm_climo/SON_climo.nc
2a40dbd588429cbefb6317fc48076bb9 acme_lores_land/c_t_F1850.g37_bench.clm2.h0.0001-01.nc
176fbe665aa0ea9ee3ba63d2df780537 acme_lores_land/c_t_F1850.g37_bench.clm2.h0.0001-02.nc
cc857575c3b7e81520be03a20fd5fc4c acme_lores_land/c_t_F1850.g37_bench.clm2.h0.0001-03.nc
Expand All @@ -83,6 +110,18 @@ ee65c00602bc7e0de884e09be4b2bb1d acme_lores_land/c_t_F1850.g37_bench.clm2.h0.00
1909f013d84b298eeff19b5250f61daa acme_lores_land/c_t_F1850.g37_bench.clm2.h0.0001-10.nc
4b96d62be06f31b8be94388ce59dbeb7 acme_lores_land/c_t_F1850.g37_bench.clm2.h0.0001-11.nc
486218898744c21420a24ab36121520d acme_lores_land/c_t_F1850.g37_bench.clm2.h0.0001-12.nc
950360fe5f5334d3026ba44850c539a6 cam35_data/cam3_5_01_climo.nc
fc869f4e9c79960f2f0766905379c4c3 cam35_data/cam3_5_02_climo.nc
c11b0b3283b726318d84edc8ad042714 cam35_data/cam3_5_03_climo.nc
4d1bfc12f358026addd34d47eca1b52c cam35_data/cam3_5_04_climo.nc
da9d0149d3e81d7bdae96076e07daf39 cam35_data/cam3_5_05_climo.nc
a5526dbaac0a0da52ca04bc5b9c71c56 cam35_data/cam3_5_06_climo.nc
00662c2eafcb297cf8aabf8c51456d0b cam35_data/cam3_5_07_climo.nc
ba72017189a80edd5181d639ae6204e9 cam35_data/cam3_5_08_climo.nc
b23c87bbf00d39b0966e3a6d072c0abc cam35_data/cam3_5_09_climo.nc
4f5d4e732e97c163f63ed1430858c5e3 cam35_data/cam3_5_10_climo.nc
6abc0b942e43cf5fbadbead8ea2aac26 cam35_data/cam3_5_11_climo.nc
c9ecb1cbabcc60196263f0a8b488d1e1 cam35_data/cam3_5_12_climo.nc
35aec5315a3d656e4d2a6d519e17ed56 cam_output/c_t_b30.009.cam2.h0.0600-01.nc
a46fa6ad6c34abbb4b5ddf185641975a cam_output/c_t_b30.009.cam2.h0.0600-02.nc
2de348a80721a9680f04bce71c368d0e cam_output/c_t_b30.009.cam2.h0.0600-03.nc
Expand All @@ -107,6 +146,21 @@ d8e3dd773ac3be993ba8801e5623b4d0 cam_output/c_t_b30.009.cam2.h0.0601-09.nc
b674e343d51edf8524714fe5f9b4ee3e cam_output/c_t_b30.009.cam2.h0.0601-10.nc
a142e0dfba3f7ba99df8eb688804660c cam_output/c_t_b30.009.cam2.h0.0601-11.nc
c65663cfcbf1881ee62bf8fda3e083d1 cam_output/c_t_b30.009.cam2.h0.0601-12.nc
e241fc465279e7126e0e59789d9baedf obs/NCEP_01_climo.nc
cd1f8016b4f575c4b2a08a69c78b041a obs/NCEP_02_climo.nc
fae4c3bfa51707a9329b274e8de3633e obs/NCEP_03_climo.nc
49d418806a382eb17ae1c7cfa5295355 obs/NCEP_04_climo.nc
97fa9532455053987f1a92645c42ef71 obs/NCEP_05_climo.nc
078fece9cf0a1730ee13a18211cefa05 obs/NCEP_06_climo.nc
039a8dd4c98b2e2332699e750f72e2b2 obs/NCEP_07_climo.nc
4bc14d3447ff3af8c0fec1a19c7cd7b3 obs/NCEP_08_climo.nc
ef45a99e527f5a36b4a145d9919ac628 obs/NCEP_09_climo.nc
6673a7bcbf1476015dad7b5106a4213f obs/NCEP_10_climo.nc
9e86a777517ad6f4b392f7d63d8e98f7 obs/NCEP_11_climo.nc
5f4ec5821d1ebb9e5a73c9a46666291a obs/NCEP_12_climo.nc
578dcbfb4979cd3cbee2bde42a52d5c7 obs/NCEP_ANN_climo.nc
78c01194a72dc3da7b25c1ce402dfe7b obs/NCEP_DJF_climo.nc
dcd392831c5c0628fde4f92e2f704c18 obs/NCEP_JJA_climo.nc
185a376e3e6403191d42dbef55b72928 obs_atmos/c_CRU_ANN_climo.nc
9c754380f93e4305c5ed40b67d7282e5 obs_atmos/c_CRU_DJF_climo.nc
a8b02bd2ea54d089db13005e7a9b4999 obs_atmos/c_CRU_JJA_climo.nc
Expand All @@ -125,3 +179,20 @@ c1c5580c10e6017d7a1b4c844f4bee95 obs_atmos/c_t_NCEP_10_climo.nc
53a07928fd5bb8282e3b00707c30d352 obs_atmos/c_t_NCEP_ANN_climo.nc
07fbdfe7c5ac96dca4d5b30cf0ffca4d obs_atmos/c_t_NCEP_DJF_climo.nc
bba7b95da836594ba56eccc5cc735953 obs_atmos/c_t_NCEP_JJA_climo.nc
25da719f4a94f073b344d463ef46dd5c obs_data_5.6/ERS_01_climo.nc
82938151479416212514ea92f5c8944d obs_data_5.6/ERS_02_climo.nc
4474e171bc3ed010bc4cf85f2156331c obs_data_5.6/ERS_03_climo.nc
5928149aaa7e20e8e021051e4c1cf8af obs_data_5.6/ERS_04_climo.nc
8ba71cabf16409ec359250137313e1fc obs_data_5.6/ERS_05_climo.nc
7173b6c6ad21ebba3faae364bb0e2abd obs_data_5.6/ERS_06_climo.nc
4a4dce6ec29ff746e6ca438a1144e2f9 obs_data_5.6/ERS_07_climo.nc
89b82d69760e786d4c5cd6007e67ad8e obs_data_5.6/ERS_08_climo.nc
703d8a3c2bca30d721db74e4a9607991 obs_data_5.6/ERS_09_climo.nc
6be5b6eaacbd4bfee413b0432a3822bd obs_data_5.6/ERS_10_climo.nc
3aab5e306b45952d4bc538cf09733d36 obs_data_5.6/ERS_11_climo.nc
b7d52d062f54e6c28b73c1630866eb8f obs_data_5.6/ERS_12_climo.nc
257874570e3aeeda6cbd55accf60f6c9 obs_data_5.6/ERS_ANN_climo.nc
d7fc6bbb9a2dfdb0fa44d7835f94a3d4 obs_data_5.6/ERS_DJF_climo.nc
3cce9af23687f27d3b134f60039ebdce obs_data_5.6/ERS_JJA_climo.nc
aaedba911f145e711d05b6430e13ce4e obs_data_5.6/ERS_MAM_climo.nc
e40f05dfec15f145e9623290d5142705 obs_data_5.6/ERS_SON_climo.nc
5 changes: 4 additions & 1 deletion CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -284,6 +284,7 @@ option(CDAT_BUILD_GUI "Builds GUI-based dependencies (Vistrails, ParaView, VisIt
option(CDAT_BUILD_GRAPHICS "Build graphics-based dependencies (vcs, pyqt, Vistrails, ParaView, VisIt, R, etc.) " ON)
option(CDAT_BUILD_ESGF "Alias for CDAT_BUILD_LEAN" OFF)
option(CDAT_BUILD_UVCMETRICSPKG "Builds uvcmetrics package " ON)
option(CDAT_DOWNLOAD_UVCMETRICS_TESTDATA "Download test data uvcmetrics package " OFF)

# If ESGF option is on then our build mode is LEAN.
if (CDAT_BUILD_ESGF)
Expand Down Expand Up @@ -538,7 +539,9 @@ include(udunits2_pkg)
include(uuid_pkg)
# IF we build the UVCDAT Metrics package
if (CDAT_BUILD_UVCMETRICSPKG)
set(UVCMETRICS_TEST_DATA_DIRECTORY $ENV{HOME}/uvcmetrics_test_data CACHE PATH "DIR FOR UVCMETRICS TEST DATA" )
if (CDAT_DOWNLOAD_UVCMETRICS_TESTDATA)
set(UVCMETRICS_TEST_DATA_DIRECTORY $ENV{HOME}/uvcmetrics_test_data CACHE PATH "DIR FOR UVCMETRICS TEST DATA" )
endif()
include(uvcmetrics_pkg)
endif()
include(vistrails_pkg)
Expand Down
115 changes: 80 additions & 35 deletions testing/metrics/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,37 +1,82 @@
set(BASELINE_DIR "${UVCDAT_GIT_TESTDATA_DIR}/baselines/")
if (CDAT_DOWNLOAD_UVCMETRICS_TESTDATA)
cdat_add_test("diags_test_01"
"${PYTHON_EXECUTABLE}"
${cdat_SOURCE_DIR}/testing/metrics/diagtest01.py
${UVCMETRICS_TEST_DATA_DIRECTORY}
--baseline=${BASELINE_DIR}/metrics/ )

cdat_add_test("diags_test_01"
"${PYTHON_EXECUTABLE}"
${cdat_SOURCE_DIR}/testing/metrics/diagtest01.py
${UVCMETRICS_TEST_DATA_DIRECTORY}
--baseline=${BASELINE_DIR}/metrics/ )

#cdat_add_test("diags_test_02"
# "${PYTHON_EXECUTABLE}"
# ${cdat_SOURCE_DIR}/testing/metrics/diagtest02.py
# --datadir=${UVCMETRICS_TEST_DATA_DIRECTORY}
# --baseline=${BASELINE_DIR}/metrics/ )

cdat_add_test("diags_test_03"
"${PYTHON_EXECUTABLE}"
${cdat_SOURCE_DIR}/testing/metrics/diagtest03.py
--datadir=${UVCMETRICS_TEST_DATA_DIRECTORY}
--baseline=${BASELINE_DIR}/metrics/ )

cdat_add_test("diags_test_04"
"${PYTHON_EXECUTABLE}"
${cdat_SOURCE_DIR}/testing/metrics/diagtest04.py
--datadir=${UVCMETRICS_TEST_DATA_DIRECTORY}/
--baseline=${BASELINE_DIR}/metrics/ )

cdat_add_test("diags_test_41"
"${PYTHON_EXECUTABLE}"
${cdat_SOURCE_DIR}/testing/metrics/diagtest41.py
--datadir=${UVCMETRICS_TEST_DATA_DIRECTORY}/
--baseline=${BASELINE_DIR}/metrics/ )

#cdat_add_test("diags_test_05"
# "${PYTHON_EXECUTABLE}"
# ${cdat_SOURCE_DIR}/testing/metrics/diagtest05.py
# --datadir=${UVCMETRICS_TEST_DATA_DIRECTORY}/
# --baseline=${BASELINE_DIR}/metrics/ )
cdat_add_test("diags_test_02"
"${PYTHON_EXECUTABLE}"
${cdat_SOURCE_DIR}/testing/metrics/diagtest02.py
--datadir=${UVCMETRICS_TEST_DATA_DIRECTORY}
--baseline=${BASELINE_DIR}/metrics/ )

cdat_add_test("diags_test_03"
"${PYTHON_EXECUTABLE}"
${cdat_SOURCE_DIR}/testing/metrics/diagtest03.py
--datadir=${UVCMETRICS_TEST_DATA_DIRECTORY}
--baseline=${BASELINE_DIR}/metrics/ )

cdat_add_test("diags_test_04"
"${PYTHON_EXECUTABLE}"
${cdat_SOURCE_DIR}/testing/metrics/diagtest04.py
--datadir=${UVCMETRICS_TEST_DATA_DIRECTORY}/
--baseline=${BASELINE_DIR}/metrics/ )

cdat_add_test("diags_test_41"
"${PYTHON_EXECUTABLE}"
${cdat_SOURCE_DIR}/testing/metrics/diagtest41.py
--datadir=${UVCMETRICS_TEST_DATA_DIRECTORY}/
--baseline=${BASELINE_DIR}/metrics/ )

cdat_add_test("diags_test_05"
"${PYTHON_EXECUTABLE}"
${cdat_SOURCE_DIR}/testing/metrics/diagtest05.py
--datadir=${UVCMETRICS_TEST_DATA_DIRECTORY}/
--baseline=${BASELINE_DIR}/metrics/ )

cdat_add_test("diags_test_06"
"${PYTHON_EXECUTABLE}"
${cdat_SOURCE_DIR}/testing/metrics/diagtest06.py
--datadir=${UVCMETRICS_TEST_DATA_DIRECTORY}/
--baseline=${BASELINE_DIR}/metrics/ )

cdat_add_test("diags_test_07"
"${PYTHON_EXECUTABLE}"
${cdat_SOURCE_DIR}/testing/metrics/diagtest07.py
--datadir=${UVCMETRICS_TEST_DATA_DIRECTORY}/
--baseline=${BASELINE_DIR}/metrics/ )

cdat_add_test("diags_test_08"
"${PYTHON_EXECUTABLE}"
${cdat_SOURCE_DIR}/testing/metrics/diagtest08.py
--datadir=${UVCMETRICS_TEST_DATA_DIRECTORY}/
--baseline=${BASELINE_DIR}/metrics/ )

cdat_add_test("diags_test_09"
"${PYTHON_EXECUTABLE}"
${cdat_SOURCE_DIR}/testing/metrics/diagtest09.py
--datadir=${UVCMETRICS_TEST_DATA_DIRECTORY}/
--baseline=${BASELINE_DIR}/metrics/ )

cdat_add_test("diags_test_10"
"${PYTHON_EXECUTABLE}"
${cdat_SOURCE_DIR}/testing/metrics/diagtest10.py
--datadir=${UVCMETRICS_TEST_DATA_DIRECTORY}/
--baseline=${BASELINE_DIR}/metrics/ )

cdat_add_test("diags_test_13"
"${PYTHON_EXECUTABLE}"
${cdat_SOURCE_DIR}/testing/metrics/diagtest13.py
--datadir=${UVCMETRICS_TEST_DATA_DIRECTORY}/
--baseline=${BASELINE_DIR}/metrics/ )

cdat_add_test("diags_test_15"
"${PYTHON_EXECUTABLE}"
${cdat_SOURCE_DIR}/testing/metrics/diagtest15.py
--datadir=${UVCMETRICS_TEST_DATA_DIRECTORY}/
--baseline=${BASELINE_DIR}/metrics/ )
else()
message("[INFO] You did not download UVCMETRICS Test data not test will be run")
endif()
30 changes: 30 additions & 0 deletions testing/metrics/README
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
Each test is named diags_test_xx; for example xx = 09. Each test refers to a python script such as diagtest09.py.
These scripts setup the inputs that will be executed in the DiagTest method execute.
To create a new one, copy one that works into a new file: for example diagtest10.py. This file needs to be modified
in order for it to execute properly. Modify the inputs plotset, filterid, obsid, etc in such a way that the plot set would
run to completion without fatal errors. In the casel plot set 10, filterid = 'f_contains', obsid = 'NCEP', etc.

In uvcdat/testing/metrics run
./diagtest10.py --datadir ~/uvcmetrics_test_data/ --baseline ~/uvcdat-testdata/baselines/metrics/ --keep True
This will create a string and execute the string; note the script should run from the command line. It would look like
diags --outputdir /var/folders/r0/bt1s3__x5yjfntcc1_xt8qd8000696/T/tmpoOrv7K/ --model path=~/uvcmetrics_test_data/cam_output,climos=no
--obs path=~/uvcmetrics_test_data/obs_atmos,filter="f_contains('NCEP')",climos=yes --package AMWG --set 10 --var T --seasons ANN
When complete it displays the temp directory where the output of the plot set is put. Without --keep, this
directory is deleted. It is of the form
output directory is = /var/folders/r0/bt1s3__x5yjfntcc1_xt8qd8000696/T/tmpFBJhqQ/
Move to this temp directory and delete those files that start with figure or end with xml; they are unnecessary for the test.
In ~/uvcdat-testdata/baselines/metrics create a directory plotset10 and copy the files from /var/folders/r0/bt1s3__x5yjfntcc1_xt8qd8000696/T/tmpFBJhqQ/
into plotset10.

In the script plotset10 copy the name of the combined png into imagefilename and the nc file names as keys to the dictionary ncfiles.
The values of these latter are obtained by doing ncdump -h on the file, such as ncdump -h "CLISCCP_ANN_Global_(1)_None.nc"

Once complete rerun diagtest10 without the keep flag:
./diagtest10.py --datadir ~/uvcmetrics_test_data/ --baseline ~/uvcdat-testdata/baselines/metrics/
This just simply test to see if you get the same answers that you got when you generated the output files.
There should be no errors once complete.

Next change CMakeLists.txt and add the new test; it has the same form as the others. Run cmake . in the build directory.
This will pick up the new CMakeLists.txt. Also in the build directory move to uvcdat-testdata/baselines/metrics and create
a subdirectory plotset10. Move into this new directory and copy the files from ~/uvcdat-testdata/baselines/metrics/plotset10
into this plotset10. Then in the build directory run ctest -V -R diags_test_15. There should be no errors and the test passes.
Loading
⚡