extract bounding box from NetCDF/HDF5 files and insert into the geospatial metadata block

doc/release-notes/9331-extract-bounding-box.md

@@ -0,0 +1 @@
+An attempt will be made to extract a geospatial bounding box (west, south, east, north) from NetCDF and HDF5 files and then insert these values into the geospatial metadata block, if enabled.

doc/sphinx-guides/source/user/dataset-management.rst

@@ -350,6 +350,8 @@ For NetCDF and HDF5 files, an attempt will be made to extract metadata in NcML_
 
 .. _NcML: https://docs.unidata.ucar.edu/netcdf-java/current/userguide/ncml_overview.html
 
+In addition, an attempt will be made to extract a geospatial bounding box (west, south, east, north) from NetCDF and HDF5 files and then insert these values into the geospatial metadata block, if enabled.
+
 Compressed Files
 ----------------
 

src/main/java/edu/harvard/iq/dataverse/DatasetFieldConstant.java

@@ -112,8 +112,8 @@ public class DatasetFieldConstant implements java.io.Serializable  {
     public final static String geographicUnit="geographicUnit";
     public final static String westLongitude="westLongitude";
     public final static String eastLongitude="eastLongitude";
-    public final static String northLatitude="northLongitude"; //Changed to match DB - incorrectly entered into DB
-    public final static String southLatitude="southLongitude"; //Incorrect in DB
+    public final static String northLatitude="northLongitude"; //Changed to match DB - incorrectly entered into DB: https://github.com/IQSS/dataverse/issues/5645
+    public final static String southLatitude="southLongitude"; //Incorrect in DB: https://github.com/IQSS/dataverse/issues/5645
     public final static String unitOfAnalysis="unitOfAnalysis";
     public final static String universe="universe";
     public final static String kindOfData="kindOfData";

src/main/java/edu/harvard/iq/dataverse/ingest/metadataextraction/impl/plugins/netcdf/NetcdfFileMetadataExtractor.java

@@ -0,0 +1,168 @@
+package edu.harvard.iq.dataverse.ingest.metadataextraction.impl.plugins.netcdf;
+
+import edu.harvard.iq.dataverse.DatasetFieldConstant;
+import edu.harvard.iq.dataverse.ingest.metadataextraction.FileMetadataExtractor;
+import edu.harvard.iq.dataverse.ingest.metadataextraction.FileMetadataIngest;
+import edu.harvard.iq.dataverse.ingest.metadataextraction.spi.FileMetadataExtractorSpi;
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.logging.Logger;
+import ucar.ma2.DataType;
+import ucar.nc2.Attribute;
+import ucar.nc2.NetcdfFile;
+import ucar.nc2.NetcdfFiles;
+
+public class NetcdfFileMetadataExtractor extends FileMetadataExtractor {
+
+    private static final Logger logger = Logger.getLogger(NetcdfFileMetadataExtractor.class.getCanonicalName());
+
+    public static final String WEST_LONGITUDE_KEY = "geospatial_lon_min";
+    public static final String EAST_LONGITUDE_KEY = "geospatial_lon_max";
+    public static final String NORTH_LATITUDE_KEY = "geospatial_lat_max";
+    public static final String SOUTH_LATITUDE_KEY = "geospatial_lat_min";
+
+    private static final String GEOSPATIAL_BLOCK_NAME = "geospatial";
+    private static final String WEST_LONGITUDE = DatasetFieldConstant.westLongitude;
+    private static final String EAST_LONGITUDE = DatasetFieldConstant.eastLongitude;
+    private static final String NORTH_LATITUDE = DatasetFieldConstant.northLatitude;
+    private static final String SOUTH_LATITUDE = DatasetFieldConstant.southLatitude;
+
+    public NetcdfFileMetadataExtractor(FileMetadataExtractorSpi originatingProvider) {
+        super(originatingProvider);
+    }
+
+    public NetcdfFileMetadataExtractor() {
+        super(null);
+    }
+
+    @Override
+    public FileMetadataIngest ingest(BufferedInputStream stream) throws IOException {
+        throw new UnsupportedOperationException("Not supported yet.");
+    }
+
+    public FileMetadataIngest ingestFile(File file) throws IOException {
+        FileMetadataIngest fileMetadataIngest = new FileMetadataIngest();
+        fileMetadataIngest.setMetadataBlockName(GEOSPATIAL_BLOCK_NAME);
+
+        Map<String, String> geoFields = parseGeospatial(getNetcdfFile(file));
+        WestAndEastLongitude welong = getStandardLongitude(new WestAndEastLongitude(geoFields.get(WEST_LONGITUDE), geoFields.get(EAST_LONGITUDE)));
+        String westLongitudeFinal = welong != null ? welong.getWestLongitude() : null;
+        String eastLongitudeFinal = welong != null ? welong.getEastLongitude() : null;
+        String northLatitudeFinal = geoFields.get(NORTH_LATITUDE);
+        String southLatitudeFinal = geoFields.get(SOUTH_LATITUDE);
+
+        logger.info(getLineStringsUrl(westLongitudeFinal, southLatitudeFinal, eastLongitudeFinal, northLatitudeFinal));
+
+        Map<String, Set<String>> metadataMap = new HashMap<>();
+        metadataMap.put(WEST_LONGITUDE, new HashSet<>());
+        metadataMap.get(WEST_LONGITUDE).add(westLongitudeFinal);
+        metadataMap.put(EAST_LONGITUDE, new HashSet<>());
+        metadataMap.get(EAST_LONGITUDE).add(eastLongitudeFinal);
+        metadataMap.put(NORTH_LATITUDE, new HashSet<>());
+        metadataMap.get(NORTH_LATITUDE).add(northLatitudeFinal);
+        metadataMap.put(SOUTH_LATITUDE, new HashSet<>());
+        metadataMap.get(SOUTH_LATITUDE).add(southLatitudeFinal);
+        fileMetadataIngest.setMetadataMap(metadataMap);
+        return fileMetadataIngest;
+    }
+
+    public NetcdfFile getNetcdfFile(File file) throws IOException {
+        /**
+         * <attribute name="geospatial_lat_min" value="25.066666666666666" />
+         * south
+         * <attribute name="geospatial_lat_max" value="49.40000000000000" />
+         * north
+         * <attribute name="geospatial_lon_min" value="-124.7666666333333" />
+         * west
+         * <attribute name="geospatial_lon_max" value="-67.058333300000015" />
+         * east
+         * <attribute name="geospatial_lon_resolution" value="0.041666666666666" />
+         * <attribute name="geospatial_lat_resolution" value="0.041666666666666" />
+         * <attribute name="geospatial_lat_units" value="decimal_degrees north" />
+         * <attribute name="geospatial_lon_units" value="decimal_degrees east" />
+         */
+        return NetcdfFiles.open(file.getAbsolutePath());
+    }
+
+    private Map<String, String> parseGeospatial(NetcdfFile netcdfFile) {
+        Map<String, String> geoFields = new HashMap<>();
+
+        Attribute westLongitude = netcdfFile.findGlobalAttribute(WEST_LONGITUDE_KEY);
+        Attribute eastLongitude = netcdfFile.findGlobalAttribute(EAST_LONGITUDE_KEY);
+        Attribute northLatitude = netcdfFile.findGlobalAttribute(NORTH_LATITUDE_KEY);
+        Attribute southLatitude = netcdfFile.findGlobalAttribute(SOUTH_LATITUDE_KEY);
+
+        geoFields.put(DatasetFieldConstant.westLongitude, getValue(westLongitude));
+        geoFields.put(DatasetFieldConstant.eastLongitude, getValue(eastLongitude));
+        geoFields.put(DatasetFieldConstant.northLatitude, getValue(northLatitude));
+        geoFields.put(DatasetFieldConstant.southLatitude, getValue(southLatitude));
+
+        logger.info(getLineStringsUrl(
+                geoFields.get(DatasetFieldConstant.westLongitude),
+                geoFields.get(DatasetFieldConstant.southLatitude),
+                geoFields.get(DatasetFieldConstant.eastLongitude),
+                geoFields.get(DatasetFieldConstant.northLatitude)));
+
+        return geoFields;
+    }
+
+    // We store strings in the database.
+    private String getValue(Attribute attribute) {
+        if (attribute == null) {
+            return null;
+        }
+        DataType dataType = attribute.getDataType();
+        if (dataType.isString()) {
+            return attribute.getStringValue();
+        } else if (dataType.isNumeric()) {
+            return attribute.getNumericValue().toString();
+        } else {
+            return null;
+        }
+    }
+
+    // Convert to standard -180 to 180 range by subtracting 360
+    // if either longitude is greater than 180.
+    //       west     south      east      north
+    //     343.68,     41.8,   353.78,     49.62 becomes
+    //     -16.320007, 41.8,    -6.220001, 49.62 instead
+    // "If one of them is > 180, the domain is 0:360.
+    // If one of them is <0, the domain is -180:180.
+    // If both are between 0 and 180, the answer is indeterminate."
+    // https://github.com/cf-convention/cf-conventions/issues/435#issuecomment-1505614364
+    public WestAndEastLongitude getStandardLongitude(WestAndEastLongitude westAndEastLongitude) {
+        if (westAndEastLongitude == null) {
+            return null;
+        }
+        if (westAndEastLongitude.getWestLongitude() == null || westAndEastLongitude.getEastLongitude() == null) {
+            return null;
+        }
+        float eastAsFloat;
+        float westAsFloat;
+        try {
+            westAsFloat = Float.valueOf(westAndEastLongitude.getWestLongitude());
+            eastAsFloat = Float.valueOf(westAndEastLongitude.getEastLongitude());
+        } catch (NumberFormatException ex) {
+            return null;
+        }
+        if (westAsFloat > 180 || eastAsFloat > 180) {
+            Float westStandard = westAsFloat - 360;
+            Float eastStandard = eastAsFloat - 360;
+            WestAndEastLongitude updatedWeLong = new WestAndEastLongitude(westStandard.toString(), eastStandard.toString());
+            return updatedWeLong;
+        }
+        return westAndEastLongitude;
+    }
+
+    // Generates a handy link to see what the bounding box looks like on a map
+    private String getLineStringsUrl(String west, String south, String east, String north) {
+        // BBOX (Left (LON) ,Bottom (LAT), Right (LON), Top (LAT), comma separated, with or without decimal point):
+        return "https://linestrings.com/bbox/#" + west + "," + south + "," + east + "," + north;
+    }
+
+}

src/main/java/edu/harvard/iq/dataverse/ingest/metadataextraction/impl/plugins/netcdf/WestAndEastLongitude.java

@@ -0,0 +1,52 @@
+package edu.harvard.iq.dataverse.ingest.metadataextraction.impl.plugins.netcdf;
+
+import java.util.Objects;
+
+public class WestAndEastLongitude {
+
+    private final String westLongitude;
+    private final String eastLongitude;
+
+    public WestAndEastLongitude(String westLongitude, String eastLongitude) {
+        this.westLongitude = westLongitude;
+        this.eastLongitude = eastLongitude;
+    }
+
+    public String getWestLongitude() {
+        return westLongitude;
+    }
+
+    public String getEastLongitude() {
+        return eastLongitude;
+    }
+
+    @Override
+    public String toString() {
+        return "WestAndEastLongitude{" + "westLongitude=" + westLongitude + ", eastLongitude=" + eastLongitude + '}';
+    }
+
+    @Override
+    public int hashCode() {
+        int hash = 3;
+        return hash;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null) {
+            return false;
+        }
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+        final WestAndEastLongitude other = (WestAndEastLongitude) obj;
+        if (!Objects.equals(this.westLongitude, other.westLongitude)) {
+            return false;
+        }
+        return Objects.equals(this.eastLongitude, other.eastLongitude);
+    }
+
+}

src/test/java/edu/harvard/iq/dataverse/api/NetcdfIT.java

@@ -7,6 +7,7 @@
 import java.io.IOException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import javax.json.Json;
 import static javax.ws.rs.core.Response.Status.CREATED;
 import static javax.ws.rs.core.Response.Status.FORBIDDEN;
 import static javax.ws.rs.core.Response.Status.NOT_FOUND;
@@ -179,4 +180,48 @@ public void testNmclFromNetcdfErrorChecking() throws IOException {
 
     }
 
+    @Test
+    public void testExtraBoundingBoxFromNetcdf() throws IOException {
+        Response createUser = UtilIT.createRandomUser();
+        createUser.then().assertThat().statusCode(OK.getStatusCode());
+        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
+        String username = UtilIT.getUsernameFromResponse(createUser);
+
+        Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
+        createDataverseResponse.prettyPrint();
+        createDataverseResponse.then().assertThat()
+                .statusCode(CREATED.getStatusCode());
+
+        String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
+
+        Response setMetadataBlocks = UtilIT.setMetadataBlocks(dataverseAlias, Json.createArrayBuilder().add("citation").add("geospatial"), apiToken);
+        setMetadataBlocks.prettyPrint();
+        setMetadataBlocks.then().assertThat().statusCode(OK.getStatusCode());
+
+        Response createDataset = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
+        createDataset.prettyPrint();
+        createDataset.then().assertThat()
+                .statusCode(CREATED.getStatusCode());
+
+        Integer datasetId = UtilIT.getDatasetIdFromResponse(createDataset);
+        String datasetPid = UtilIT.getDatasetPersistentIdFromResponse(createDataset);
+
+        // From https://www.ncei.noaa.gov/data/international-comprehensive-ocean-atmosphere/v3/archive/nrt/ICOADS_R3.0.0_1662-10.nc
+        // via https://data.noaa.gov/onestop/collections/details/9bd5c743-0684-4e70-817a-ed977117f80c?f=temporalResolution:1%20Minute%20-%20%3C%201%20Hour;dataFormats:NETCDF
+        String pathToFile = "src/test/resources/netcdf/ICOADS_R3.0.0_1662-10.nc";
+
+        Response uploadFile = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile, apiToken);
+        uploadFile.prettyPrint();
+        uploadFile.then().assertThat().statusCode(OK.getStatusCode());
+
+        Response getJson = UtilIT.nativeGet(datasetId, apiToken);
+        getJson.prettyPrint();
+        getJson.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data.latestVersion.metadataBlocks.geospatial.fields[0].value[0].westLongitude.value", equalTo("-16.320007"))
+                .body("data.latestVersion.metadataBlocks.geospatial.fields[0].value[0].eastLongitude.value", equalTo("-6.220001"))
+                .body("data.latestVersion.metadataBlocks.geospatial.fields[0].value[0].northLongitude.value", equalTo("49.62"))
+                .body("data.latestVersion.metadataBlocks.geospatial.fields[0].value[0].southLongitude.value", equalTo("41.8"));
+    }
+
 }

src/test/java/edu/harvard/iq/dataverse/ingest/metadataextraction/impl/plugins/netcdf/NetcdfFileMetadataExtractorTest.java

@@ -0,0 +1,62 @@
+package edu.harvard.iq.dataverse.ingest.metadataextraction.impl.plugins.netcdf;
+
+import edu.harvard.iq.dataverse.DatasetFieldConstant;
+import edu.harvard.iq.dataverse.ingest.metadataextraction.FileMetadataIngest;
+import java.io.File;
+import java.util.Map;
+import java.util.Set;
+import static org.junit.Assert.assertNull;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import org.junit.jupiter.api.Test;
+
+public class NetcdfFileMetadataExtractorTest {
+
+    /**
+     * Expect some lat/long values (geospatial bounding box) with longtude
+     * values that have been transformed from a domain of 0 to 360 to a domain
+     * of -180 to 180.
+     */
+    @Test
+    public void testExtractLatLong() throws Exception {
+        String pathAndFile = "src/test/resources/netcdf/ICOADS_R3.0.0_1662-10.nc";
+        File file = new File(pathAndFile);
+        NetcdfFileMetadataExtractor instance = new NetcdfFileMetadataExtractor();
+        FileMetadataIngest netcdfMetadata = instance.ingestFile(file);
+        Map<String, Set<String>> map = netcdfMetadata.getMetadataMap();
+        assertEquals("-16.320007", map.get(DatasetFieldConstant.westLongitude).toArray()[0]);
+        assertEquals("-6.220001", map.get(DatasetFieldConstant.eastLongitude).toArray()[0]);
+        assertEquals("41.8", map.get(DatasetFieldConstant.southLatitude).toArray()[0]);
+        assertEquals("49.62", map.get(DatasetFieldConstant.northLatitude).toArray()[0]);
+    }
+
+    /**
+     * The NetCDF file under test doesn't have values for latitude/longitude
+     * (geospatial bounding box).
+     */
+    @Test
+    public void testExtractNoLatLong() throws Exception {
+        String pathAndFile = "src/test/resources/netcdf/madis-raob";
+        File file = new File(pathAndFile);
+        NetcdfFileMetadataExtractor instance = new NetcdfFileMetadataExtractor();
+        FileMetadataIngest netcdfMetadata = null;
+        netcdfMetadata = instance.ingestFile(file);
+        Map<String, Set<String>> map = netcdfMetadata.getMetadataMap();
+        assertNull(map.get(DatasetFieldConstant.westLongitude).toArray()[0]);
+        assertNull(map.get(DatasetFieldConstant.eastLongitude).toArray()[0]);
+        assertNull(map.get(DatasetFieldConstant.southLatitude).toArray()[0]);
+        assertNull(map.get(DatasetFieldConstant.northLatitude).toArray()[0]);
+    }
+
+    @Test
+    public void testStandardLongitude() {
+        NetcdfFileMetadataExtractor extractor = new NetcdfFileMetadataExtractor();
+        // Both are over 180. Change.
+        assertEquals(new WestAndEastLongitude("-16.320007", "-6.220001"), extractor.getStandardLongitude(new WestAndEastLongitude("343.68", "353.78")));
+        // One is over 180. Change
+        assertEquals(new WestAndEastLongitude("-260.0", "-179.0"), extractor.getStandardLongitude(new WestAndEastLongitude("100", "181")));
+        // Both are under 180. No change.
+        assertEquals(new WestAndEastLongitude("25", "35"), extractor.getStandardLongitude(new WestAndEastLongitude("25", "35")));
+        assertEquals(null, extractor.getStandardLongitude(new WestAndEastLongitude("foo", "bar")));
+    }
+
+}