Merge pull request #237 from ceblanton/236.combine-statics

Two combine-static updates

Author: ceblanton

app/combine-statics/bin/combine-statics

@@ -11,6 +11,7 @@ echo "    input dir: $inputDir"
 echo "    output dir: $outputDir"
 echo Utilities:
 type cdo
+type ncatted
 
 # Verify input directory exists and is a directory
 if [[ ! -d $inputDir ]]; then
@@ -28,16 +29,21 @@ cd $inputDir
 
 for comp in $(ls); do
     pushd $comp/P0Y/P0Y || continue
-
     mkdir -p $outputDir/$comp
-    cdo -O merge *.nc $outputDir/$comp/$comp.static.nc
-
-#   #check
-#    if [ -z "$( ls $outputDir/$comp)" ]; then
-#        echo "empty"
-#    else
-#        echo "$(ls $outputDir/$comp)"
-#    fi
+
+    # Remove the coordinates attribute (https://github.com/NOAA-GFDL/FMS/issues/1859)
+    for file in *.nc; do
+        ncatted -a coordinates,,d,, $file
+    done
+
+    # Tiled and non-tiled output
+    if ls *.tile1.nc; then
+        for tile in tile1 tile2 tile3 tile4 tile5 tile6; do
+            cdo -O merge *.$tile.nc $outputDir/$comp/$comp.static.$tile.nc
+        done
+    else
+        cdo -O merge *.nc $outputDir/$comp/$comp.static.nc
+    fi
 
     popd
 done

app/combine-statics/tests/test-data/atmos_cubedsphere_1.tile1.cdl

@@ -0,0 +1,17 @@
+netcdf atmos_cubedsphere_1.tile1 {
+dimensions:
+        grid_xt = 2 ;
+        grid_yt = 2 ;
+variables:
+        float orog(grid_yt, grid_xt) ;
+                orog:long_name = "surface geopotential height" ;
+                orog:units = "m" ;
+                orog:missing_value = 1.e+20f ;
+                orog:_FillValue = 1.e+20f ;
+        float grid_xt(grid_xt) ;
+        float grid_yt(grid_yt) ;
+data:
+  grid_xt = 0, 1 ;
+  grid_yt = 0, 1 ;
+  orog = 1, 2, 3, 4 ;
+}

app/combine-statics/tests/test-data/atmos_cubedsphere_1.tile2.cdl

@@ -0,0 +1,17 @@
+netcdf atmos_cubedsphere_1.tile2 {
+dimensions:
+        grid_xt = 2 ;
+        grid_yt = 2 ;
+variables:
+        float orog(grid_yt, grid_xt) ;
+                orog:long_name = "surface geopotential height" ;
+                orog:units = "m" ;
+                orog:missing_value = 1.e+20f ;
+                orog:_FillValue = 1.e+20f ;
+        float grid_xt(grid_xt) ;
+        float grid_yt(grid_yt) ;
+data:
+  grid_xt = 0, 1 ;
+  grid_yt = 0, 1 ;
+  orog = 5, 6, 7, 8 ;
+}

app/combine-statics/tests/test-data/atmos_cubedsphere_1.tile3.cdl

@@ -0,0 +1,17 @@
+netcdf atmos_cubedsphere_1.tile3 {
+dimensions:
+        grid_xt = 2 ;
+        grid_yt = 2 ;
+variables:
+        float orog(grid_yt, grid_xt) ;
+                orog:long_name = "surface geopotential height" ;
+                orog:units = "m" ;
+                orog:missing_value = 1.e+20f ;
+                orog:_FillValue = 1.e+20f ;
+        float grid_xt(grid_xt) ;
+        float grid_yt(grid_yt) ;
+data:
+  grid_xt = 0, 1 ;
+  grid_yt = 0, 1 ;
+  orog = 9, 10, 11, 12 ;
+}

app/combine-statics/tests/test-data/atmos_cubedsphere_1.tile4.cdl

@@ -0,0 +1,17 @@
+netcdf atmos_cubedsphere_1.tile4 {
+dimensions:
+        grid_xt = 2 ;
+        grid_yt = 2 ;
+variables:
+        float orog(grid_yt, grid_xt) ;
+                orog:long_name = "surface geopotential height" ;
+                orog:units = "m" ;
+                orog:missing_value = 1.e+20f ;
+                orog:_FillValue = 1.e+20f ;
+        float grid_xt(grid_xt) ;
+        float grid_yt(grid_yt) ;
+data:
+  grid_xt = 0, 1 ;
+  grid_yt = 0, 1 ;
+  orog = 13, 14, 15, 16 ;
+}

app/combine-statics/tests/test-data/atmos_cubedsphere_1.tile5.cdl

@@ -0,0 +1,17 @@
+netcdf atmos_cubedsphere_1.tile5 {
+dimensions:
+        grid_xt = 2 ;
+        grid_yt = 2 ;
+variables:
+        float orog(grid_yt, grid_xt) ;
+                orog:long_name = "surface geopotential height" ;
+                orog:units = "m" ;
+                orog:missing_value = 1.e+20f ;
+                orog:_FillValue = 1.e+20f ;
+        float grid_xt(grid_xt) ;
+        float grid_yt(grid_yt) ;
+data:
+  grid_xt = 0, 1 ;
+  grid_yt = 0, 1 ;
+  orog = 17, 18, 19, 20 ;
+}

app/combine-statics/tests/test-data/atmos_cubedsphere_1.tile6.cdl

@@ -0,0 +1,17 @@
+netcdf atmos_cubedsphere_1.tile6 {
+dimensions:
+        grid_xt = 2 ;
+        grid_yt = 2 ;
+variables:
+        float orog(grid_yt, grid_xt) ;
+                orog:long_name = "surface geopotential height" ;
+                orog:units = "m" ;
+                orog:missing_value = 1.e+20f ;
+                orog:_FillValue = 1.e+20f ;
+        float grid_xt(grid_xt) ;
+        float grid_yt(grid_yt) ;
+data:
+  grid_xt = 0, 1 ;
+  grid_yt = 0, 1 ;
+  orog = 21, 22, 23, 24 ;
+}

app/combine-statics/tests/test_combine_statics.py

@@ -27,6 +27,11 @@
                        "atmos_static_scalar_2.ak.nc",
                        "atmos_static_scalar_3.ck.nc"]
 
+# Tiled (cubed-sphere) component
+TILED_COMP_NAME = "atmos_cubedsphere"
+TILED_CDLFILES = [f"atmos_cubedsphere_1.tile{t}.cdl" for t in range(1, 7)]
+TILED_NCFILES  = [f"atmos_cubedsphere_1.tile{t}.nc"  for t in range(1, 7)]
+
 # Input/Output directories
 # OutDir for generating netcdf files; used as input for combine-statics task
 NCGEN_OUT = f"{TEST_DIR}/test-output/ncgen-output"
@@ -196,6 +201,71 @@ def test_combine_statics_output_data(load_ncfile):
         assert (val == expected_data).all()
         #print(f"{v} has values: {val}")
 
+#### TILED (CUBED-SPHERE) TEST CASE ####
+# NOTE: The tiled component directory is created inside test_ncgen_tiled_nc_files
+# rather than at module level. An empty component directory would cause
+# `cdo merge *.nc` to fail when the non-tiled test_combine_statics runs first.
+
+def test_tiled_cdl_files_exist():
+    """
+    Test for the existence of tiled CDL test files (one per cube face)
+    """
+    assert all(Path(f"{DATA_DIR}/{f}").exists() for f in TILED_CDLFILES)
+
+def test_ncgen_tiled_nc_files():
+    """
+    Generate tiled netCDF files from CDL files; test they exist in the
+    correct input location for combine-statics
+    """
+    Path(f"{NCGEN_OUT}/{TILED_COMP_NAME}/P0Y/P0Y").mkdir(parents=True, exist_ok=True)
+    for cdl, nc in zip(TILED_CDLFILES, TILED_NCFILES):
+        output = f"{NCGEN_OUT}/{TILED_COMP_NAME}/P0Y/P0Y/{nc}"
+        ex = ["ncgen", "-k", "64-bit offset",
+              "-o", output,
+              f"{DATA_DIR}/{cdl}"]
+        sp = subprocess.run(ex, check=True)
+        assert all([sp.returncode == 0,
+                    Path(output).exists()])
+
+def test_combine_statics_tiled(monkeypatch):
+    """
+    Test that combine-statics produces one output file per tile for a
+    tiled (cubed-sphere) component
+    """
+    monkeypatch.setenv("inputDir", NCGEN_OUT)
+    monkeypatch.setenv("outputDir", COMBINE_STATICS_OUT)
+
+    script = f"{COMBINE_STATICS_DIR}/bin/combine-statics"
+    sp = subprocess.run(script, check=True)
+
+    assert sp.returncode == 0
+    for t in range(1, 7):
+        assert Path(f"{COMBINE_STATICS_OUT}/{TILED_COMP_NAME}/{TILED_COMP_NAME}.static.tile{t}.nc").is_file()
+
+@pytest.fixture(name="load_tiled_ncfile")
+def load_tiled_dataset():
+    """
+    Load tile1 output file using xarray for content validation
+    """
+    outfile = f"{TILED_COMP_NAME}.static.tile1.nc"
+    sf = xr.open_dataset(f"{COMBINE_STATICS_OUT}/{TILED_COMP_NAME}/{outfile}")
+    yield sf
+    sf.close()
+
+def test_combine_statics_tiled_output_variables(load_tiled_ncfile):
+    """
+    Test that tiled output file contains the expected variable
+    """
+    assert "orog" in load_tiled_ncfile.data_vars
+
+def test_combine_statics_tiled_output_data(load_tiled_ncfile):
+    """
+    Test that tile1 output contains the expected data values
+    """
+    expected = np.array([[1, 2], [3, 4]], dtype=np.float32)
+    assert (load_tiled_ncfile["orog"].values == expected).all()
+
+
 #####assert only one lat and lon exists...........
 # TO-DO: Having issues trying to generate an expected failure when
 #        adding a static netcdf file that should fail at cdo merge

site/gaea.cylc

@@ -43,5 +43,5 @@
 
 {% if DO_REGRID_STATIC or DO_NATIVE_STATIC %}
     [[combine-statics]]
-        pre-script = module load cdo
+        pre-script = module load cdo nco
 {% endif %}