diff --git a/doc/release-notes/9331-extract-bounding-box.md b/doc/release-notes/9331-extract-bounding-box.md new file mode 100644 index 00000000000..c4ff83e40c0 --- /dev/null +++ b/doc/release-notes/9331-extract-bounding-box.md @@ -0,0 +1 @@ +An attempt will be made to extract a geospatial bounding box (west, south, east, north) from NetCDF and HDF5 files and then insert these values into the geospatial metadata block, if enabled. diff --git a/doc/sphinx-guides/source/user/dataset-management.rst b/doc/sphinx-guides/source/user/dataset-management.rst index 3fc208fea33..9223768b49f 100755 --- a/doc/sphinx-guides/source/user/dataset-management.rst +++ b/doc/sphinx-guides/source/user/dataset-management.rst @@ -346,10 +346,34 @@ A map will be shown as a preview of GeoJSON files when the previewer has been en NetCDF and HDF5 --------------- +NcML +~~~~ + For NetCDF and HDF5 files, an attempt will be made to extract metadata in NcML_ (XML) format and save it as an auxiliary file. (See also :doc:`/developers/aux-file-support` in the Developer Guide.) A previewer for these NcML files is available (see :ref:`file-previews`). .. _NcML: https://docs.unidata.ucar.edu/netcdf-java/current/userguide/ncml_overview.html +Geospatial Bounding Box +~~~~~~~~~~~~~~~~~~~~~~~ + +An attempt will be made to extract a geospatial bounding box (west, south, east, north) from NetCDF and HDF5 files and then insert these values into the geospatial metadata block, if enabled. + +This is the mapping that is used: + +- geospatial_lon_min: West Longitude +- geospatial_lon_max: East Longitude +- geospatial_lat_max: North Latitude +- geospatial_lat_min: South Latitude + +Please note the following rules regarding these fields: + +- West Longitude and East Longitude are expected to be in the range of -180 and 180. (When using :ref:`geospatial-search`, you should use this range for longitude.) +- If West Longitude and East Longitude are both over 180 (outside the expected -180:180 range), 360 will be subtracted to shift the values from the 0:360 range to the expected -180:180 range. +- If either West Longitude or East Longitude are less than zero but the other longitude is greater than 180 (which would imply an indeterminate domain, a lack of clarity of if the domain is -180:180 or 0:360), metadata will be not be extracted. +- If the bounding box was successfully populated, the subsequent removal of the NetCDF or HDF5 file from the dataset does not automatically remove the bounding box from the dataset metadata. You must remove the bounding box manually, if desired. + +If the bounding box was successfully populated, :ref:`geospatial-search` should be able to find it. + Compressed Files ---------------- diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldConstant.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldConstant.java index 6d26c0cba58..e57a2a1538d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldConstant.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldConstant.java @@ -112,8 +112,8 @@ public class DatasetFieldConstant implements java.io.Serializable { public final static String geographicUnit="geographicUnit"; public final static String westLongitude="westLongitude"; public final static String eastLongitude="eastLongitude"; - public final static String northLatitude="northLongitude"; //Changed to match DB - incorrectly entered into DB - public final static String southLatitude="southLongitude"; //Incorrect in DB + public final static String northLatitude="northLongitude"; //Changed to match DB - incorrectly entered into DB: https://github.com/IQSS/dataverse/issues/5645 + public final static String southLatitude="southLongitude"; //Incorrect in DB: https://github.com/IQSS/dataverse/issues/5645 public final static String unitOfAnalysis="unitOfAnalysis"; public final static String universe="universe"; public final static String kindOfData="kindOfData"; diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java index fd850ac1b9d..7cdfda8d082 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java @@ -53,6 +53,7 @@ import edu.harvard.iq.dataverse.ingest.metadataextraction.FileMetadataExtractor; import edu.harvard.iq.dataverse.ingest.metadataextraction.FileMetadataIngest; import edu.harvard.iq.dataverse.ingest.metadataextraction.impl.plugins.fits.FITSFileMetadataExtractor; +import edu.harvard.iq.dataverse.ingest.metadataextraction.impl.plugins.netcdf.NetcdfFileMetadataExtractor; import edu.harvard.iq.dataverse.ingest.tabulardata.TabularDataFileReader; import edu.harvard.iq.dataverse.ingest.tabulardata.TabularDataIngest; import edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.dta.DTAFileReader; @@ -341,6 +342,7 @@ public List saveAndAddFilesToDataset(DatasetVersion version, String fileName = fileMetadata.getLabel(); boolean metadataExtracted = false; + boolean metadataExtractedFromNetcdf = false; if (tabIngest && FileUtil.canIngestAsTabular(dataFile)) { /* * Note that we don't try to ingest the file right away - instead we mark it as @@ -367,7 +369,20 @@ public List saveAndAddFilesToDataset(DatasetVersion version, } else { logger.fine("Failed to extract indexable metadata from file " + fileName); } - } else if (FileUtil.MIME_TYPE_INGESTED_FILE.equals(dataFile.getContentType())) { + } else if (fileMetadataExtractableFromNetcdf(dataFile, tempLocationPath)) { + try { + logger.fine("trying to extract metadata from netcdf"); + metadataExtractedFromNetcdf = extractMetadataFromNetcdf(tempFileLocation, dataFile, version); + } catch (IOException ex) { + logger.fine("could not extract metadata from netcdf: " + ex); + } + if (metadataExtractedFromNetcdf) { + logger.fine("Successfully extracted indexable metadata from netcdf file " + fileName); + } else { + logger.fine("Failed to extract indexable metadata from netcdf file " + fileName); + } + + } else if (FileUtil.MIME_TYPE_INGESTED_FILE.equals(dataFile.getContentType())) { // Make sure no *uningested* tab-delimited files are saved with the type "text/tab-separated-values"! // "text/tsv" should be used instead: dataFile.setContentType(FileUtil.MIME_TYPE_TSV); @@ -1166,7 +1181,58 @@ public boolean fileMetadataExtractable(DataFile dataFile) { } return false; } - + + // Inspired by fileMetadataExtractable, above + public boolean fileMetadataExtractableFromNetcdf(DataFile dataFile, Path tempLocationPath) { + logger.fine("fileMetadataExtractableFromNetcdf dataFileIn: " + dataFile + ". tempLocationPath: " + tempLocationPath); + boolean extractable = false; + String dataFileLocation = null; + if (tempLocationPath != null) { + // This file was just uploaded and hasn't been saved to S3 or local storage. + dataFileLocation = tempLocationPath.toString(); + } else { + // This file is already on S3 or local storage. + File tempFile = null; + File localFile; + StorageIO storageIO; + try { + storageIO = dataFile.getStorageIO(); + storageIO.open(); + if (storageIO.isLocalFile()) { + localFile = storageIO.getFileSystemPath().toFile(); + dataFileLocation = localFile.getAbsolutePath(); + logger.info("fileMetadataExtractable2: file is local. Path: " + dataFileLocation); + } else { + // Need to create a temporary local file: + tempFile = File.createTempFile("tempFileExtractMetadataNcml", ".tmp"); + try ( ReadableByteChannel targetFileChannel = (ReadableByteChannel) storageIO.getReadChannel(); FileChannel tempFileChannel = new FileOutputStream(tempFile).getChannel();) { + tempFileChannel.transferFrom(targetFileChannel, 0, storageIO.getSize()); + } + dataFileLocation = tempFile.getAbsolutePath(); + logger.info("fileMetadataExtractable2: file is on S3. Downloaded and saved to temp path: " + dataFileLocation); + } + } catch (IOException ex) { + logger.info("fileMetadataExtractable2, could not use storageIO for data file id " + dataFile.getId() + ". Exception: " + ex); + } + } + if (dataFileLocation != null) { + try ( NetcdfFile netcdfFile = NetcdfFiles.open(dataFileLocation)) { + logger.info("fileMetadataExtractable2: trying to open " + dataFileLocation); + if (netcdfFile != null) { + logger.info("fileMetadataExtractable2: returning true"); + extractable = true; + } else { + logger.info("NetcdfFiles.open() could not open file id " + dataFile.getId() + " (null returned)."); + } + } catch (IOException ex) { + logger.info("NetcdfFiles.open() could not open file id " + dataFile.getId() + ". Exception caught: " + ex); + } + } else { + logger.info("dataFileLocation is null for file id " + dataFile.getId() + ". Can't extract NcML."); + } + return extractable; + } + /* * extractMetadata: * framework for extracting metadata from uploaded files. The results will @@ -1218,6 +1284,63 @@ public boolean extractMetadata(String tempFileLocation, DataFile dataFile, Datas return ingestSuccessful; } + /** + * Try to extract bounding box (west, south, east, north). + * + * Inspired by extractMetadata(). Consider merging the methods. Note that + * unlike extractMetadata(), we are not calling processFileLevelMetadata(). + * + * Also consider merging with extractMetadataNcml() but while NcML should be + * extractable from all files that the NetCDF Java library can open only + * some NetCDF files will have a bounding box. + * + * Note that if we ever create an API endpoint for this method for files + * that are already persisted to disk or S3, we will need to use something + * like getExistingFile() from extractMetadataNcml() to pull the file down + * from S3 to a temporary file location on local disk so that it can + * (ultimately) be opened by the NetcdfFiles.open() method, which only + * operates on local files (not an input stream). What we have now is not a + * problem for S3 because the files are saved locally before the are + * uploaded to S3. It's during this time that the files are local that this + * method is run. + */ + public boolean extractMetadataFromNetcdf(String tempFileLocation, DataFile dataFile, DatasetVersion editVersion) throws IOException { + boolean ingestSuccessful = false; + + InputStream tempFileInputStream = null; + if (tempFileLocation == null) { + StorageIO sio = dataFile.getStorageIO(); + sio.open(DataAccessOption.READ_ACCESS); + tempFileInputStream = sio.getInputStream(); + } else { + try { + tempFileInputStream = new FileInputStream(new File(tempFileLocation)); + } catch (FileNotFoundException notfoundEx) { + throw new IOException("Could not open temp file " + tempFileLocation); + } + } + + // Locate metadata extraction plugin for the file format by looking + // it up with the Ingest Service Provider Registry: + NetcdfFileMetadataExtractor extractorPlugin = new NetcdfFileMetadataExtractor(); + logger.fine("creating file from " + tempFileLocation); + File file = new File(tempFileLocation); + FileMetadataIngest extractedMetadata = extractorPlugin.ingestFile(file); + Map> extractedMetadataMap = extractedMetadata.getMetadataMap(); + + if (extractedMetadataMap != null) { + logger.fine("Ingest Service: Processing extracted metadata from netcdf;"); + if (extractedMetadata.getMetadataBlockName() != null) { + logger.fine("Ingest Service: This metadata from netcdf belongs to the " + extractedMetadata.getMetadataBlockName() + " metadata block."); + processDatasetMetadata(extractedMetadata, editVersion); + } + } + + ingestSuccessful = true; + + return ingestSuccessful; + } + /** * @param dataFile The DataFile from which to attempt NcML extraction * (NetCDF or HDF5 format) @@ -1531,72 +1654,71 @@ private void processDatasetMetadata(FileMetadataIngest fileMetadataIngest, Datas // create a new compound field value and its child // DatasetFieldCompoundValue compoundDsfv = new DatasetFieldCompoundValue(); - int nonEmptyFields = 0; + int nonEmptyFields = 0; for (DatasetFieldType cdsft : dsft.getChildDatasetFieldTypes()) { String dsfName = cdsft.getName(); - if (fileMetadataMap.get(dsfName) != null && !fileMetadataMap.get(dsfName).isEmpty()) { - logger.fine("Ingest Service: found extracted metadata for field " + dsfName + ", part of the compound field "+dsft.getName()); - + if (fileMetadataMap.get(dsfName) != null && !fileMetadataMap.get(dsfName).isEmpty()) { + logger.fine("Ingest Service: found extracted metadata for field " + dsfName + ", part of the compound field " + dsft.getName()); + if (cdsft.isPrimitive()) { // probably an unnecessary check - child fields - // of compound fields are always primitive... - // but maybe it'll change in the future. + // of compound fields are always primitive... + // but maybe it'll change in the future. if (!cdsft.isControlledVocabulary()) { // TODO: can we have controlled vocabulary // sub-fields inside compound fields? - + DatasetField childDsf = new DatasetField(); childDsf.setDatasetFieldType(cdsft); - + DatasetFieldValue newDsfv = new DatasetFieldValue(childDsf); - newDsfv.setValue((String)fileMetadataMap.get(dsfName).toArray()[0]); + newDsfv.setValue((String) fileMetadataMap.get(dsfName).toArray()[0]); childDsf.getDatasetFieldValues().add(newDsfv); - + childDsf.setParentDatasetFieldCompoundValue(compoundDsfv); compoundDsfv.getChildDatasetFields().add(childDsf); - + nonEmptyFields++; - } - } + } + } } } - + if (nonEmptyFields > 0) { - // let's go through this dataset's fields and find the - // actual parent for this sub-field: + // let's go through this dataset's fields and find the + // actual parent for this sub-field: for (DatasetField dsf : editVersion.getFlatDatasetFields()) { if (dsf.getDatasetFieldType().equals(dsft)) { - + // Now let's check that the dataset version doesn't already have - // this compound value - we are only interested in aggregating - // unique values. Note that we need to compare compound values - // as sets! -- i.e. all the sub fields in 2 compound fields - // must match in order for these 2 compounds to be recognized + // this compound value - we are only interested in aggregating + // unique values. Note that we need to compare compound values + // as sets! -- i.e. all the sub fields in 2 compound fields + // must match in order for these 2 compounds to be recognized // as "the same": - - boolean alreadyExists = false; + boolean alreadyExists = false; for (DatasetFieldCompoundValue dsfcv : dsf.getDatasetFieldCompoundValues()) { - int matches = 0; + int matches = 0; for (DatasetField cdsf : dsfcv.getChildDatasetFields()) { String cdsfName = cdsf.getDatasetFieldType().getName(); String cdsfValue = cdsf.getDatasetFieldValues().get(0).getValue(); if (cdsfValue != null && !cdsfValue.equals("")) { - String extractedValue = (String)fileMetadataMap.get(cdsfName).toArray()[0]; - logger.fine("values: existing: "+cdsfValue+", extracted: "+extractedValue); + String extractedValue = (String) fileMetadataMap.get(cdsfName).toArray()[0]; + logger.fine("values: existing: " + cdsfValue + ", extracted: " + extractedValue); if (cdsfValue.equals(extractedValue)) { matches++; } } } if (matches == nonEmptyFields) { - alreadyExists = true; + alreadyExists = true; break; } } - + if (!alreadyExists) { - // save this compound value, by attaching it to the + // save this compound value, by attaching it to the // version for proper cascading: compoundDsfv.setParentDatasetField(dsf); dsf.getDatasetFieldCompoundValues().add(compoundDsfv); diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/metadataextraction/impl/plugins/netcdf/NetcdfFileMetadataExtractor.java b/src/main/java/edu/harvard/iq/dataverse/ingest/metadataextraction/impl/plugins/netcdf/NetcdfFileMetadataExtractor.java new file mode 100644 index 00000000000..66f0c25f3d7 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/metadataextraction/impl/plugins/netcdf/NetcdfFileMetadataExtractor.java @@ -0,0 +1,192 @@ +package edu.harvard.iq.dataverse.ingest.metadataextraction.impl.plugins.netcdf; + +import edu.harvard.iq.dataverse.DatasetFieldConstant; +import edu.harvard.iq.dataverse.ingest.metadataextraction.FileMetadataExtractor; +import edu.harvard.iq.dataverse.ingest.metadataextraction.FileMetadataIngest; +import edu.harvard.iq.dataverse.ingest.metadataextraction.spi.FileMetadataExtractorSpi; +import java.io.BufferedInputStream; +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.logging.Logger; +import ucar.ma2.DataType; +import ucar.nc2.Attribute; +import ucar.nc2.NetcdfFile; +import ucar.nc2.NetcdfFiles; + +public class NetcdfFileMetadataExtractor extends FileMetadataExtractor { + + private static final Logger logger = Logger.getLogger(NetcdfFileMetadataExtractor.class.getCanonicalName()); + + public static final String WEST_LONGITUDE_KEY = "geospatial_lon_min"; + public static final String EAST_LONGITUDE_KEY = "geospatial_lon_max"; + public static final String NORTH_LATITUDE_KEY = "geospatial_lat_max"; + public static final String SOUTH_LATITUDE_KEY = "geospatial_lat_min"; + + private static final String GEOSPATIAL_BLOCK_NAME = "geospatial"; + private static final String WEST_LONGITUDE = DatasetFieldConstant.westLongitude; + private static final String EAST_LONGITUDE = DatasetFieldConstant.eastLongitude; + private static final String NORTH_LATITUDE = DatasetFieldConstant.northLatitude; + private static final String SOUTH_LATITUDE = DatasetFieldConstant.southLatitude; + + public NetcdfFileMetadataExtractor(FileMetadataExtractorSpi originatingProvider) { + super(originatingProvider); + } + + public NetcdfFileMetadataExtractor() { + super(null); + } + + @Override + public FileMetadataIngest ingest(BufferedInputStream stream) throws IOException { + throw new UnsupportedOperationException("Not supported yet."); + } + + public FileMetadataIngest ingestFile(File file) throws IOException { + FileMetadataIngest fileMetadataIngest = new FileMetadataIngest(); + fileMetadataIngest.setMetadataBlockName(GEOSPATIAL_BLOCK_NAME); + + Map geoFields = parseGeospatial(getNetcdfFile(file)); + WestAndEastLongitude welong = getStandardLongitude(new WestAndEastLongitude(geoFields.get(WEST_LONGITUDE), geoFields.get(EAST_LONGITUDE))); + String westLongitudeFinal = welong != null ? welong.getWestLongitude() : null; + String eastLongitudeFinal = welong != null ? welong.getEastLongitude() : null; + String northLatitudeFinal = geoFields.get(NORTH_LATITUDE); + String southLatitudeFinal = geoFields.get(SOUTH_LATITUDE); + + logger.info(getLineStringsUrl(westLongitudeFinal, southLatitudeFinal, eastLongitudeFinal, northLatitudeFinal)); + + Map> metadataMap = new HashMap<>(); + metadataMap.put(WEST_LONGITUDE, new HashSet<>()); + metadataMap.get(WEST_LONGITUDE).add(westLongitudeFinal); + metadataMap.put(EAST_LONGITUDE, new HashSet<>()); + metadataMap.get(EAST_LONGITUDE).add(eastLongitudeFinal); + metadataMap.put(NORTH_LATITUDE, new HashSet<>()); + metadataMap.get(NORTH_LATITUDE).add(northLatitudeFinal); + metadataMap.put(SOUTH_LATITUDE, new HashSet<>()); + metadataMap.get(SOUTH_LATITUDE).add(southLatitudeFinal); + fileMetadataIngest.setMetadataMap(metadataMap); + return fileMetadataIngest; + } + + public NetcdfFile getNetcdfFile(File file) throws IOException { + /** + * + * south + * + * north + * + * west + * + * east + * + * + * + * + */ + return NetcdfFiles.open(file.getAbsolutePath()); + } + + private Map parseGeospatial(NetcdfFile netcdfFile) { + Map geoFields = new HashMap<>(); + + Attribute westLongitude = netcdfFile.findGlobalAttribute(WEST_LONGITUDE_KEY); + Attribute eastLongitude = netcdfFile.findGlobalAttribute(EAST_LONGITUDE_KEY); + Attribute northLatitude = netcdfFile.findGlobalAttribute(NORTH_LATITUDE_KEY); + Attribute southLatitude = netcdfFile.findGlobalAttribute(SOUTH_LATITUDE_KEY); + + geoFields.put(DatasetFieldConstant.westLongitude, getValue(westLongitude)); + geoFields.put(DatasetFieldConstant.eastLongitude, getValue(eastLongitude)); + geoFields.put(DatasetFieldConstant.northLatitude, getValue(northLatitude)); + geoFields.put(DatasetFieldConstant.southLatitude, getValue(southLatitude)); + + logger.info(getLineStringsUrl( + geoFields.get(DatasetFieldConstant.westLongitude), + geoFields.get(DatasetFieldConstant.southLatitude), + geoFields.get(DatasetFieldConstant.eastLongitude), + geoFields.get(DatasetFieldConstant.northLatitude))); + + return geoFields; + } + + // We store strings in the database. + private String getValue(Attribute attribute) { + if (attribute == null) { + return null; + } + DataType dataType = attribute.getDataType(); + if (dataType.isString()) { + return attribute.getStringValue(); + } else if (dataType.isNumeric()) { + return attribute.getNumericValue().toString(); + } else { + return null; + } + } + + // Convert to standard -180 to 180 range by subtracting 360 + // if both longitudea are greater than 180. For example: + // west south east north + // 343.68, 41.8, 353.78, 49.62 becomes + // -16.320007, 41.8, -6.220001, 49.62 instead + // "If one of them is > 180, the domain is 0:360. + // If one of them is <0, the domain is -180:180. + // If both are between 0 and 180, the answer is indeterminate." + // https://github.com/cf-convention/cf-conventions/issues/435#issuecomment-1505614364 + // Solr only wants -180 to 180. It will throw an error for values outside this range. + public WestAndEastLongitude getStandardLongitude(WestAndEastLongitude westAndEastLongitude) { + if (westAndEastLongitude == null) { + return null; + } + if (westAndEastLongitude.getWestLongitude() == null || westAndEastLongitude.getEastLongitude() == null) { + return null; + } + float eastAsFloat; + float westAsFloat; + try { + westAsFloat = Float.valueOf(westAndEastLongitude.getWestLongitude()); + eastAsFloat = Float.valueOf(westAndEastLongitude.getEastLongitude()); + } catch (NumberFormatException ex) { + return null; + } + // "If one of them is > 180, the domain is 0:360" + if (westAsFloat > 180 && eastAsFloat > 180) { + Float westStandard = westAsFloat - 360; + Float eastStandard = eastAsFloat - 360; + WestAndEastLongitude updatedWeLong = new WestAndEastLongitude(westStandard.toString(), eastStandard.toString()); + return updatedWeLong; + } + // "If one of them is <0, the domain is -180:180." + // 180:180 is what Solr wants. Return it. + if (westAsFloat < 0 || eastAsFloat < 0) { + // BUT! Don't return it if the values + // are so low to be out of range! + // Something must be wrong with the data. + if (westAsFloat < -180 || eastAsFloat < -180) { + return null; + } + if (westAsFloat > 180 || eastAsFloat > 180) { + // Not in the proper range of -80:180 + return null; + } + return westAndEastLongitude; + } + if ((westAsFloat > 180 || eastAsFloat > 180) && (westAsFloat < 180 || eastAsFloat < 180)) { + // One value is over 180 and the other is under 180. + // We don't know if we should subtract 360 or not. + // Return null to prevent inserting a potentially + // incorrect bounding box. + return null; + } + return westAndEastLongitude; + } + + // Generates a handy link to see what the bounding box looks like on a map + private String getLineStringsUrl(String west, String south, String east, String north) { + // BBOX (Left (LON) ,Bottom (LAT), Right (LON), Top (LAT), comma separated, with or without decimal point): + return "https://linestrings.com/bbox/#" + west + "," + south + "," + east + "," + north; + } + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/metadataextraction/impl/plugins/netcdf/WestAndEastLongitude.java b/src/main/java/edu/harvard/iq/dataverse/ingest/metadataextraction/impl/plugins/netcdf/WestAndEastLongitude.java new file mode 100644 index 00000000000..02a984f3424 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/metadataextraction/impl/plugins/netcdf/WestAndEastLongitude.java @@ -0,0 +1,52 @@ +package edu.harvard.iq.dataverse.ingest.metadataextraction.impl.plugins.netcdf; + +import java.util.Objects; + +public class WestAndEastLongitude { + + private final String westLongitude; + private final String eastLongitude; + + public WestAndEastLongitude(String westLongitude, String eastLongitude) { + this.westLongitude = westLongitude; + this.eastLongitude = eastLongitude; + } + + public String getWestLongitude() { + return westLongitude; + } + + public String getEastLongitude() { + return eastLongitude; + } + + @Override + public String toString() { + return "WestAndEastLongitude{" + "westLongitude=" + westLongitude + ", eastLongitude=" + eastLongitude + '}'; + } + + @Override + public int hashCode() { + int hash = 3; + return hash; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + final WestAndEastLongitude other = (WestAndEastLongitude) obj; + if (!Objects.equals(this.westLongitude, other.westLongitude)) { + return false; + } + return Objects.equals(this.eastLongitude, other.eastLongitude); + } + +} diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FitsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FitsIT.java new file mode 100644 index 00000000000..b154205ce2d --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/api/FitsIT.java @@ -0,0 +1,78 @@ +package edu.harvard.iq.dataverse.api; + +import com.jayway.restassured.RestAssured; +import static com.jayway.restassured.path.json.JsonPath.with; +import com.jayway.restassured.response.Response; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import javax.json.Json; +import javax.json.JsonObject; +import static javax.ws.rs.core.Response.Status.CREATED; +import static javax.ws.rs.core.Response.Status.OK; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.junit.Assert.assertTrue; +import org.junit.BeforeClass; +import org.junit.Test; + +public class FitsIT { + + @BeforeClass + public static void setUp() { + RestAssured.baseURI = UtilIT.getRestAssuredBaseUri(); + } + + @Test + public void testAstroFieldsFromFits() throws IOException { + Response createUser = UtilIT.createRandomUser(); + createUser.then().assertThat().statusCode(OK.getStatusCode()); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + String username = UtilIT.getUsernameFromResponse(createUser); + + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + createDataverseResponse.prettyPrint(); + createDataverseResponse.then().assertThat() + .statusCode(CREATED.getStatusCode()); + + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + Response setMetadataBlocks = UtilIT.setMetadataBlocks(dataverseAlias, Json.createArrayBuilder().add("citation").add("astrophysics"), apiToken); + setMetadataBlocks.prettyPrint(); + setMetadataBlocks.then().assertThat().statusCode(OK.getStatusCode()); + + Response createDataset = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + createDataset.prettyPrint(); + createDataset.then().assertThat() + .statusCode(CREATED.getStatusCode()); + + Integer datasetId = UtilIT.getDatasetIdFromResponse(createDataset); + String datasetPid = UtilIT.getDatasetPersistentIdFromResponse(createDataset); + + // "FOS 2 x 2064 primary array spectrum containing the flux and wavelength arrays, plus a small table extension" + // from https://fits.gsfc.nasa.gov/fits_samples.html + String pathToFile = "src/test/resources/fits/FOSy19g0309t_c2f.fits"; + + Response uploadFile = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile, apiToken); + uploadFile.prettyPrint(); + uploadFile.then().assertThat().statusCode(OK.getStatusCode()); + + Response getJson = UtilIT.nativeGet(datasetId, apiToken); + getJson.prettyPrint(); + getJson.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.latestVersion.metadataBlocks.astrophysics.fields[0].value[0]", equalTo("Image")); + + // a bit more precise than the check for "Image" above (but annoyingly fiddly) + List astroTypeFromNativeGet = with(getJson.body().asString()).param("astroType", "astroType") + .getJsonObject("data.latestVersion.metadataBlocks.astrophysics.fields.findAll { fields -> fields.typeName == astroType }"); + Map firstAstroTypeFromNativeGet = astroTypeFromNativeGet.get(0); + assertTrue(firstAstroTypeFromNativeGet.toString().contains("Image")); + + List coverageTemportalFromNativeGet = with(getJson.body().asString()).param("coverageTemporal", "coverage.Temporal") + .getJsonObject("data.latestVersion.metadataBlocks.astrophysics.fields.findAll { fields -> fields.typeName == coverageTemporal }"); + Map firstcoverageTemporalFromNativeGet = coverageTemportalFromNativeGet.get(0); + assertTrue(firstcoverageTemporalFromNativeGet.toString().contains("1993")); + + } + +} diff --git a/src/test/java/edu/harvard/iq/dataverse/api/NetcdfIT.java b/src/test/java/edu/harvard/iq/dataverse/api/NetcdfIT.java index 9716e7aca13..89ae1b9202e 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/NetcdfIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/NetcdfIT.java @@ -7,6 +7,7 @@ import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; +import javax.json.Json; import static javax.ws.rs.core.Response.Status.CREATED; import static javax.ws.rs.core.Response.Status.FORBIDDEN; import static javax.ws.rs.core.Response.Status.NOT_FOUND; @@ -179,4 +180,48 @@ public void testNmclFromNetcdfErrorChecking() throws IOException { } + @Test + public void testExtraBoundingBoxFromNetcdf() throws IOException { + Response createUser = UtilIT.createRandomUser(); + createUser.then().assertThat().statusCode(OK.getStatusCode()); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + String username = UtilIT.getUsernameFromResponse(createUser); + + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + createDataverseResponse.prettyPrint(); + createDataverseResponse.then().assertThat() + .statusCode(CREATED.getStatusCode()); + + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + Response setMetadataBlocks = UtilIT.setMetadataBlocks(dataverseAlias, Json.createArrayBuilder().add("citation").add("geospatial"), apiToken); + setMetadataBlocks.prettyPrint(); + setMetadataBlocks.then().assertThat().statusCode(OK.getStatusCode()); + + Response createDataset = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + createDataset.prettyPrint(); + createDataset.then().assertThat() + .statusCode(CREATED.getStatusCode()); + + Integer datasetId = UtilIT.getDatasetIdFromResponse(createDataset); + String datasetPid = UtilIT.getDatasetPersistentIdFromResponse(createDataset); + + // From https://www.ncei.noaa.gov/data/international-comprehensive-ocean-atmosphere/v3/archive/nrt/ICOADS_R3.0.0_1662-10.nc + // via https://data.noaa.gov/onestop/collections/details/9bd5c743-0684-4e70-817a-ed977117f80c?f=temporalResolution:1%20Minute%20-%20%3C%201%20Hour;dataFormats:NETCDF + String pathToFile = "src/test/resources/netcdf/ICOADS_R3.0.0_1662-10.nc"; + + Response uploadFile = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile, apiToken); + uploadFile.prettyPrint(); + uploadFile.then().assertThat().statusCode(OK.getStatusCode()); + + Response getJson = UtilIT.nativeGet(datasetId, apiToken); + getJson.prettyPrint(); + getJson.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.latestVersion.metadataBlocks.geospatial.fields[0].value[0].westLongitude.value", equalTo("-16.320007")) + .body("data.latestVersion.metadataBlocks.geospatial.fields[0].value[0].eastLongitude.value", equalTo("-6.220001")) + .body("data.latestVersion.metadataBlocks.geospatial.fields[0].value[0].northLongitude.value", equalTo("49.62")) + .body("data.latestVersion.metadataBlocks.geospatial.fields[0].value[0].southLongitude.value", equalTo("41.8")); + } + } diff --git a/src/test/java/edu/harvard/iq/dataverse/ingest/metadataextraction/impl/plugins/netcdf/NetcdfFileMetadataExtractorTest.java b/src/test/java/edu/harvard/iq/dataverse/ingest/metadataextraction/impl/plugins/netcdf/NetcdfFileMetadataExtractorTest.java new file mode 100644 index 00000000000..203fc96e70a --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/ingest/metadataextraction/impl/plugins/netcdf/NetcdfFileMetadataExtractorTest.java @@ -0,0 +1,82 @@ +package edu.harvard.iq.dataverse.ingest.metadataextraction.impl.plugins.netcdf; + +import edu.harvard.iq.dataverse.DatasetFieldConstant; +import edu.harvard.iq.dataverse.ingest.metadataextraction.FileMetadataIngest; +import java.io.File; +import java.util.Map; +import java.util.Set; +import static org.junit.Assert.assertNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import org.junit.jupiter.api.Test; + +public class NetcdfFileMetadataExtractorTest { + + /** + * Expect some lat/long values (geospatial bounding box) with longtude + * values that have been transformed from a domain of 0 to 360 to a domain + * of -180 to 180. + */ + @Test + public void testExtractLatLong() throws Exception { + String pathAndFile = "src/test/resources/netcdf/ICOADS_R3.0.0_1662-10.nc"; + File file = new File(pathAndFile); + NetcdfFileMetadataExtractor instance = new NetcdfFileMetadataExtractor(); + FileMetadataIngest netcdfMetadata = instance.ingestFile(file); + Map> map = netcdfMetadata.getMetadataMap(); + assertEquals("-16.320007", map.get(DatasetFieldConstant.westLongitude).toArray()[0]); + assertEquals("-6.220001", map.get(DatasetFieldConstant.eastLongitude).toArray()[0]); + assertEquals("41.8", map.get(DatasetFieldConstant.southLatitude).toArray()[0]); + assertEquals("49.62", map.get(DatasetFieldConstant.northLatitude).toArray()[0]); + } + + /** + * The NetCDF file under test doesn't have values for latitude/longitude + * (geospatial bounding box). + */ + @Test + public void testExtractNoLatLong() throws Exception { + String pathAndFile = "src/test/resources/netcdf/madis-raob"; + File file = new File(pathAndFile); + NetcdfFileMetadataExtractor instance = new NetcdfFileMetadataExtractor(); + FileMetadataIngest netcdfMetadata = null; + netcdfMetadata = instance.ingestFile(file); + Map> map = netcdfMetadata.getMetadataMap(); + assertNull(map.get(DatasetFieldConstant.westLongitude).toArray()[0]); + assertNull(map.get(DatasetFieldConstant.eastLongitude).toArray()[0]); + assertNull(map.get(DatasetFieldConstant.southLatitude).toArray()[0]); + assertNull(map.get(DatasetFieldConstant.northLatitude).toArray()[0]); + } + + @Test + public void testStandardLongitude() { + NetcdfFileMetadataExtractor extractor = new NetcdfFileMetadataExtractor(); + + // Both are over 180. Subtract 360 from both. + // before: https://linestrings.com/bbox/#343.68,41.8,353.78,49.62 + // after: https://linestrings.com/bbox/#-16.320007,41.8,-6.220001,49.62 + assertEquals(new WestAndEastLongitude("-16.320007", "-6.220001"), extractor.getStandardLongitude(new WestAndEastLongitude("343.68", "353.78"))); + + // "If one of them is <0, the domain is -180:180." No change. https://linestrings.com/bbox/#-10,20,100,40 + assertEquals(new WestAndEastLongitude("-10", "100"), extractor.getStandardLongitude(new WestAndEastLongitude("-10", "100"))); + + // Both are negative. No change. https://linestrings.com/bbox/#-124.7666666333333,25.066666666666666,-67.058333300000015,49.40000000000000 + assertEquals(new WestAndEastLongitude("-124.7666666333333", "-67.058333300000015"), extractor.getStandardLongitude(new WestAndEastLongitude("-124.7666666333333", "-67.058333300000015"))); + + // Both between 0 and 180. Leave it alone. No change. https://linestrings.com/bbox/#25,20,35,40 + assertEquals(new WestAndEastLongitude("25", "35"), extractor.getStandardLongitude(new WestAndEastLongitude("25", "35"))); + + // When only one value is over 180 we can't know if we should subtract 360 from both. + // Expect null. Don't insert potentially incorrect data into the database. https://linestrings.com/bbox/#100,20,181,40 + assertEquals(null, extractor.getStandardLongitude(new WestAndEastLongitude("100", "181"))); + + // "If one of them is <0, the domain is -180:180." No change. https://linestrings.com/bbox/#-10,20,181,40 + assertEquals(null, extractor.getStandardLongitude(new WestAndEastLongitude("-10", "181"))); + + // Both values are less than -180 and out of range. Expect null. No database insert https://linestrings.com/bbox/#999,20,-888,40 + assertEquals(null, extractor.getStandardLongitude(new WestAndEastLongitude("-999", "-888"))); + + // Garbage in, garbage out. You can't bass "foo" and "bar" as longitudes + assertEquals(null, extractor.getStandardLongitude(new WestAndEastLongitude("foo", "bar"))); + } + +} diff --git a/src/test/resources/fits/FOSy19g0309t_c2f.fits b/src/test/resources/fits/FOSy19g0309t_c2f.fits new file mode 100644 index 00000000000..bc76e4066ef --- /dev/null +++ b/src/test/resources/fits/FOSy19g0309t_c2f.fits @@ -0,0 +1,42 @@ +SIMPLE = T / Standard FITS format BITPIX = -32 / 32 bit IEEE floating point numbers NAXIS = 2 / Number of axes NAXIS1 = 2064 / NAXIS2 = 2 / EXTEND = T / There may be standard extensions OPSIZE = 832 / PSIZE of original image ORIGIN = 'ST-DADS ' / Institution that originated the FITS file FITSDATE= '12/07/94' / Date FITS file was created FILENAME= 'y19g0309t_cvt.c2h' / Original GEIS header file name with _cvt ODATTYPE= 'FLOATING' / Original datatype SDASMGNU= 2 / GCOUNT of original image DADSFILE= 'Y19G0309T.C2F' / DADSCLAS= 'CAL ' / DADSDATE= '12-JUL-1994 02:44:39' / CRVAL1 = 1.0000000000000 / CRPIX1 = 1.0000000000000 / CD1_1 = 1.0000000000000 / DATAMIN = 0.00000000000000 / DATAMAX = 2.7387550387959E-15 / RA_APER = 182.63573015260 / DEC_APER= 39.405888372580 / FILLCNT = 0 / ERRCNT = 0 / FPKTTIME= 49099.133531036 / LPKTTIME= 49099.133541164 / CTYPE1 = 'PIXEL ' / APER_POS= 'SINGLE ' / PASS_DIR= 0 / YPOS = -1516.0000000000 / YTYPE = 'OBJ ' / EXPOSURE= 31.249689102173 / X_OFFSET= 0.00000000000000 / Y_OFFSET= 0.00000000000000 / / GROUP PARAMETERS: OSS / GROUP PARAMETERS: PODPS / FOS DATA DESCRIPTOR KEYWORDS INSTRUME= 'FOS ' / instrument in use ROOTNAME= 'Y19G0309T ' / rootname of the observation set FILETYPE= 'ERR ' / file type BUNIT = 'ERGS/CM**2/S/A' / brightness units / GENERIC CONVERSION KEYWORDS HEADER = T / science header line exists TRAILER = F / reject array exists YWRDSLIN= 516 / science words per packet YLINSFRM= 5 / packets per frame / CALIBRATION FLAGS AND INDICATORS GRNDMODE= 'SPECTROSCOPY ' / ground software mode DETECTOR= 'AMBER ' / detector in use: amber, blue APER_ID = 'B-2 ' / aperture id POLAR_ID= 'C ' / polarizer id POLANG = 0.0000000E+00 / initial angular position of polarizer FGWA_ID = 'H57 ' / FGWA id FCHNL = 0 / first channel NCHNLS = 512 / number of channels OVERSCAN= 5 / overscan number NXSTEPS = 4 / number of x steps YFGIMPEN= T / onboard GIMP correction enabled (T/F) YFGIMPER= 'NO ' / error in onboard GIMP correction (YES/NO) / CALIBRATION REFERENCE FILES AND TABLES DEFDDTBL= F / UDL disabled diode table used BACHFILE= 'yref$b3m1128fy.r0h' / background header file FL1HFILE= 'yref$baf13103y.r1h' / first flat-field header file FL2HFILE= 'yref$n/a ' / second flat-field header file IV1HFILE= 'yref$c3u13412y.r2h' / first inverse sensitivity header file IV2HFILE= 'yref$n/a ' / second inverse sensitivity header file RETHFILE= 'yref$n/a ' / waveplate retardation header file DDTHFILE= 'yref$c861559ay.r4h' / disabled diode table header file DQ1HFILE= 'yref$b2f1301qy.r5h' / first data quality initialization header file DQ2HFILE= 'yref$n/a ' / second data quality initialization header file CCG2 = 'mtab$a3d1145ly.cmg' / paired pulse correction parameters CCS0 = 'ytab$a3d1145dy.cy0' / aperture parameters CCS1 = 'ytab$aaj0732ay.cy1' / aperture position parameters CCS2 = 'ytab$a3d1145fy.cy2' / sky emission line regions CCS3 = 'ytab$a3d1145gy.cy3' / big and sky filter widths and prism X0 CCS4 = 'ytab$b9d1019my.cy4' / polarimetry parameters CCS5 = 'ytab$a3d1145jy.cy5' / sky shifts CCS6 = 'ytab$bck10546y.cy6' / wavelength coefficients CCS7 = 'ytab$ba910502y.cy7' / GIMP correction scale factores CCS8 = 'ytab$ba31407ly.cy8' / predicted background count rates / CALIBRATION SWITCHES CNT_CORR= 'COMPLETE' / count to count rate conversion OFF_CORR= 'OMIT ' / GIMP correction PPC_CORR= 'COMPLETE' / paired pulse correction BAC_CORR= 'COMPLETE' / background subtraction GMF_CORR= 'COMPLETE' / scale reference background FLT_CORR= 'COMPLETE' / flat-fielding SKY_CORR= 'COMPLETE' / sky subtraction WAV_CORR= 'COMPLETE' / wavelength scale generation FLX_CORR= 'COMPLETE' / flux scale generation ERR_CORR= 'COMPLETE' / propagated error computation MOD_CORR= 'OMIT ' / ground software mode dependent reductions / PATTERN KEYWORDS INTS = 2 / number of integrations YBASE = -1516 / y base YRANGE = 0 / y range YSTEPS = 1 / number of y steps YSPACE = 0.0000000E+00 / yrange * 32 / ysteps SLICES = 1 / number of time slices NPAT = 12 / number of patterns per readout NREAD = 2 / number of readouts per memory clear NMCLEARS= 1 / number of memory clears per acquisition YSTEP1 = 'OBJ ' / first ystep data type: OBJ, SKY, BCK, NUL YSTEP2 = 'NUL ' / second ystep data type: OBJ, SKY, BCK, NUL YSTEP3 = 'NUL ' / third ystep data type: OBJ, SKY, BCK, NUL XBASE = 0 / X-deflection base XPITCH = 1521 / X-deflection pitch between diode YPITCH = 1834 / Y-deflection pitch / CALIBRATION KEYWORDS LIVETIME= 33333 / accumulator open time (unit=7.8125 microsec) DEADTIME= 1280 / accumulator close time (unit=7.8125 microsec) MAXCLK = 0 / maximum clock count PA_APER = 0.2462417E+03 / position ang of aperture used with target (deg)NOISELM = 65535 / burst noise rejection limit OFFS_TAB= 'n/a ' / GIMP offsets (post-pipeline processing only) MINWAVE = 4569.102 / minimum wavelength (angstroms) MAXWAVE = 6817.517 / maximum wavelength (angstroms) / STATISTICAL KEYWORDS DATE = '22/04/93 ' / date this file was written (dd/mm/yy) PKTFMT = 96 / packet format code PODPSFF = '0 ' / 0=(no podps fill), 1=(podps fill present) STDCFFF = '0 ' / 0=(no st dcf fill), 1=(st dcf fill present) STDCFFP = '0000 ' / st dcf fill pattern (hex) / APERTURE POSITION RA_APER1= 0.1826357301526E+03 / right ascension of the aperture (deg) DECAPER1= 0.3940588837258E+02 / declination of the aperture (deg) / EXPOSURE INFORMATION EQUINOX = 'J2000 ' / equinox of the celestial coordinate system SUNANGLE= 0.1225114E+03 / angle between sun and V1 axis (deg) MOONANGL= 0.1191039E+03 / angle between moon and V1 axis (deg) SUN_ALT = 0.4515910E+02 / altitude of the sun above Earth's limb (deg) FGSLOCK = 'COARSE ' / commanded FGS lock (FINE,COARSE,GYROS,UNKNOWN) DATE-OBS= '22/04/93 ' / UT date of start of observation (dd/mm/yy) TIME-OBS= '03:12:17 ' / UT time of start of observation (hh:mm:ss) EXPSTART= 0.4909913202874E+05 / exposure start time (Modified Julian Date) EXPEND = 0.4909913505303E+05 / exposure end time (Modified Julian Date) EXPTIME = 0.2499975E+03 / exposure duration (seconds)--calculated EXPFLAG = 'NORMAL ' / Exposure interruption indicator / TARGET & PROPOSAL ID TARGNAME= 'NGC4151-CLOUD2 ' / proposer's target name RA_TARG = 0.1826357301526E+03 / right ascension of the target (deg) (J2000) DEC_TARG= 0.3940588837258E+02 / declination of the target (deg) (J2000) PROPOSID= 4220 / PEP proposal identifier PEP_EXPO= '174.0000000 ' / PEP exposure identifier including sequence LINENUM = '174.000 ' / PEP proposal line number SEQLINE = ' ' / PEP line number of defined sequence SEQNAME = ' ' / PEP define/use sequence name END '3C&1f&U&"1&a9&$&+#&&&cm&&3&P&C<&Zq&Lw&<&b&%&&N6&.5Z&2&ex&&x&|~&;&}i&:S&&W&?&+M!&I_&cI&;&^&U&Sd&^j&E2&c&r-.&mK&& &&RW&z&$& u&~-&yR&y&z&Mv&i7&{y&\&&p/&t9&Ep$&d*&TS&Xi:&zI&7&qU@&q.&U&i&[3W&Oq.&n1y&Z`&&_&OM&hy&!&N٢&Sv&SH&*vy&M׏&#_v&<H&A&%N&TV&N&C\4&a&M!&-&J&J&N&&k&D3& &2&-&>&2&,5& %%&%&2 &34&&Ї& +ȱ&?2%z%&& & &%& 8&AO&\%j[&A&V:&*I&)&K.m&&&A&&F"&8Y&~o%k %b&&I%%&% &&%2&7&&=8%&%z&ɬ&#&&%NL%ۓK&.;&Qo&&et&:&` &#!& %&71&jL&4$&=&& q&@d&}&/&=O& F&+&%1& G/&+&6&;&N&%&@&+}&6F%a@&2j&2%ʫ%n-&e&l&7& ~&& b&<%%9%z& +%ԘT&%v%#I%&&\&&Re%]&<%&5&9%%&&&&*& p%f&\&&Ø&*&+k6&e & &/#&= +& O&F&0& r&A?&+&g&&f;S&=a&4!&\3%-&f&0&j%Q%!k&c&4&´%k%5&&4% |&%%&N&-=3%}%&+Q%̸&"&%&1f& +&&7%؋%ܻ&%麄% &%=%̶&&q%&n%ڗ&/&%1& +Cj&&&R%&4%&hV%%up&&>&%E%:&& %5%:&}&`4&%5& +&&& +&n%t&.& *%`&(o&& "%a&+ %=s&%{/%J&Z{%%e%Ԛ:%&^8& L &%F%%0%%%J& (.&Y&$|%<&@%&ܮ&s& '_%&%&%%%%>/%g%q&%[%{<&q&r& +/%<%:%%K& +#& %&%%H%D&&&Õ&A&& +t&i&8&&]I%%& P&o<& +^&&!%&¿%R%q%U%bD%Q&/%fd&^%b&)>%ޮ%ũY%!%E%żJ%%{b%߫&W&%PT%Яc%-%%U%%%Σ +%&9]&@&&& %>&&t&&%g&,2%|& &"5e%%G& cz%%K%o%i%&&L%O4%a%%n% +%V%!%%%֣&T%&-&x%د%]%o*& % Q%h%\%b& &+&'6%Z&%s%&G,%%N&ܡ%5%hr& ;%& &&8l%}@%?&%&d%^ & L[%@_%n%%dE%I%%缶&~_%&%5G%&&d%\%)&H&:%|&V_% %%&o&%)&-%I&~%%Ė&iC%%m%%c% :& ȍ%`%-&2&#R%& ~q%l%!%5&%R%Φ%%-F& @&%Ͼ&& & & ՞%%Z&m& A%_H&%%&We%٩%%~F&ij%3%f8%v:&%%{%%E&oN%%3&&&Q%-h&Б&%&I&+&*1%%:%]&Z'%&5pi%-%jm%s!& %ޞ%&G%ԝ&%+%o%%%c%U%5B%.%Ԏ%ϸF&g%;?&& =j& ҍ&1&%Թ%%\%J%C& >&% % &le%&_& _%%&F%%&y5&& +%:a%%V&+m&K&F& y3%&1&9]U%%߅&t & a&D%%jg%eQ%ɪ&Jy%U#%P&Z&%@&y&%|b%%%ѝ& I%r&2%(%4%J%!%Ҡ{%%%۟%9& %d%w%& i%IJx%%%A&&H{%թ%^%Y%8%%\%Z%72%%I%%֔ %a%ۭ%e%ޣ%y%$%=%5y%=%Ȥ&K%;w%Wm%&2&&%ན%!%1%v%v%%2%0H%B%S%w%%9%%%׈T%K%5H%S}%h%S5%+%h%d%+%Xo%ּ%L%%a%Υ%Ƙ7%%D%C%I% %䀮%%=%J%%%Jo%O%%[&Md%b%%iI%d%q%%0%;%!$%& %S%"%&F%գ%S&& &&%,%p%޳%0&38%}%4%j%h%?%އr%&%lT%%R%&I%:%x%y%%м%,%t%!%%ׂa%r%%%͋%Ǯ%%%%˚%%$%%(N%G%3%̾% %j%%>%%{% %I%ߠM%=%:,%з5%&%%%Η%LJ%V%e%+u%6&%̋%\%V%q_%ԫ%%=%%Ϧ%%%%޺j&<%y%ʄ%"%f%d %Q%(%%Ǩ%|%U% %Ϯ%%|%%]%[%'%ӧ%Ӣ%8%Ux%%%t%%Ӗr%j%*%%ܭ%˼%%%˖:%%,%%)%y%F%|%e%%юA%%<&#%ޓH%%%L%%%%%M%%)%Ķ5%!%[%%=&%CW%hN% %Œ%Z%E%%ӂ%+%%o%%h%%3%!^%%|%M%oQ%*4%>%7;%uu%x%)W%ъ%[&p%F%?%TJ%U3%꽡%%m%%W%.&%e%:%i%$H%|%靇%U%%޸u%&%%%\%%%)G%n%ٙZ%xS%ԟ%ž%%%f%u%%S%%w%%5[% %Z%%ڈ%%W%s1%%5%g%އq%%%%,%Q%ޝ%m%%)%w%V%Ѓ\%՘%*%%Ǻ%.%%,0%%Ŭ%u%3%UZ%5%b%%#%i%%Gv%%ļ%%{%f% %%S%%G%ń%%;%Rb%̌^%%"%똛%5 %ؘ%S%%ԅ%f]%%%p&%u%%2m%I%%>%%\%߻%y%k%?%%l%%%%Խ%%5[%%%N]%;%Ɣ%%ƗA%&ч%`%%\%M%u%\%& +x&%%~%ֵ%Һ%y%ø@%O%a%ȴ{%,%E%%% %s%%/%&%h%T%#%1%h%٦%%%c%%%%qk%C%w%%%/%%sE%bA%p%X_%ĞU%&%I%X%`%B%m%4%%٦%`%љ%t/% %0%/P%bM%%G%իj%%H& &&J%$%1j%ڦ|%v%v%!%?%C%%P%c-%V{%0%D%ь%V]%X%%\%%#%.%%{%@%:B%%ˤ%%%%S%C$%x%h%}%C%%{%ư%%%%d%ș%)%%/%e%^%% %%%+=%t%%Κf%%$%ݝ%%%%<%%%%*%%%a%-%V%}%W%h%%KT%o%R%A%4q%%&&&%@& ّ&;%{&5%wZ&&%I&S& $& &&3!&NM&Tx&v~h&.&&?&&' +'/ '&%&& a&>%%5%%%%k&&&&%}&p%Jj%%&݅&G&u&~%^%%Ĭ%%|%n%f&)S%ѣ%Z%%\& +(%=%]%1&;&e%ս%S%w%!%%<&%%?Z%(%L%%%%6x%%zA%%w%e%|G%%ݜ%᧽%`%y%}%]%L%P%% %%{%%%%h2%%͝% X% L%'%ʺ%+%S%%% & %@&&|%%Y%r%Ǎ%i%E%>%@z%n% %S%q%P0%Xu%Ss%%N%!&!%ۼ!&%%x%I%%%{%ˈ%%1%%T%%V%x%h%9%\& q%ގq%;8%wK%&6&n&X&&FC&1&!'%%%mV%"%z%\&& =s%p%sZ%%wr%N%y'%%%ڸr%-%䨔%D`%ܠ'%_%ph&hW%Fs%%y%%%޺%%%%A%W%Bo%ʀI% %J%%%v1%%%Ժ%ꭳ& %-4%ůW%W%%Y%!=&~%b%䖜%X%ռ%sy%̘e%?|%*%%ɥf%5%r%?8%'%%G%%̊-%%ċ% %e%F%%%~%R%_%ߋ%o%C%w%i%6%%V%%%RO%4!%׉e%%\%*%%C:%W%%%&%i%Ml%o%R%&z[&&&6K&-ռ&P&iO&V&P&P&)&N&&5Ր%0J& +&@&k&Q&cL&D&S3& v&1&2h&&R&V& +z&&&Љ%^+&2%x& o&y&%\%&'%I&@%k,&& +%&(&=&3&s&d&a&%!%[v%^%!W%\%D%m%%ƅK&~%& c%4% }%%%%ڄ&%l%ʫ%z%v%˭)%e3%r%2%%;%:%Ǽ% %%ˋS%[%{%c%>"%%%e=%^%ܥ%%o%%Ԕ%1% %<%%!%l%\%Z%h%%%%H&4%E&`%HM%&& |T%©&+&a&D7&P&&)D& m%Q1%F%Q%%j%ͼ,%7%k%Dh%I%%f%%i%%q%L%%%S-%AZ%p%o8%[%%d%T=%5%%%%.%%%[% %+%m%ֲ% %'o%=%C%(%-%%%%[%Ұ%i%%% %hk%%)%%|%A%A6% %%,%/E%n%Y% %C%(W%G2%b%g7%%%6j%S%%E6%%%C%6%>%%%c%-%%HG%c% +&%l%%7%E%f%u%%%|%,^%Ӱ%y%G{%%%ǟ"%ÚE%t%z%9%%ڐ%Ƈ%C%%c%%&%>E%H%*%<%&%}%r%b}%%%І;%%N% %(%%%*%Ο%%=%%_%Ha%f%s%З%%ib%&%%%E%5%%%%%ɴ%|%|%a%%.%>%f\%X}%%%%3%~%b:%%](%|%%M;%a%%9s%%:%ǘ%% U%ӣ%O%%%%%%Y%.%٠*%%%BF%P%%9*%q%% %˟%t%$%%%%%R% +%%%%b%%%%C%%ʅV%ҍ%l%bH%>8%Ĩ%n%%%Δ%k%Q%%C%#%%d%%%U%y%%%[%%%c%v%|%ME%m%%<%2W%%<%*d%q %%V%1%%% +%%á%%j%%%%CR%Pi%c%%%%x%z%T%)%%-% %Š%%%%n%_%9J%%U`%~%%%>%%]<%2%[%.%z%،%ɱ%ǃ~%'%E%%t%R%r%ˠ% %m%%n%N%к3%l%׳%e%%sv%%%TQ%%%޹f%F%"%L%Z%\p%̟[%%%ף%%Ł%s^%H%x%\%(J%j%T%% %y%%ч%6a%X %r% J%W%tJ%Η%*%Ʉ%dT%%%I%x%%%ȑ7%%%q%z%{x%v%*T%v%Y%֘%%R%f%ty%+%Ο%I%I$%߁%?%Zz%83%u% %%ա%2%7%`%7% %8% %%%ʀj%ǟ%%Z!%%%e%%!G%с%W%%5%%t%_%>%%%%%%%O=%5%h%n%%%!%%%0L%^F%=%I%"%%%%]-%C+%%*~%OG%J%2%H%P%6%%:%%N;%+%%%%Ƃ%%l%`W%p%_%D%U%%n%%%K%%E%X%Z)%% %@)% j%7%%>% U%>%u%=% %%%%&8%%8%%%%%m>%%N0%*%g%5%~ %h%{i%3%%*%L% %Z %B% -%LC%5w% +%ܧ%ճ%% %C%e%>%U%R%k%%=%y%#%%%\%_;%}A%"%[%9%%]%mw%% +%%?R%B`%9%%+%V%~%V%5%,U%%%T%e%Zc%%%%<%%<1%o%o%>%*!%\%%%%e%G%i%%Z%j%%Aq%S%%%% %%&%Q%u%%^c%S%Mk%%]%9%x%%%v%%%s=%% %D%Ay%%ތ%%t%Kk%˥%W%N%%]%b %%Z %e%%`%!%[>%Jm%%%Z%(%%u%V%&%d%%2%% %4%3,%%3%\%%U%}%%Z +%y %$%Š%j|%N%vz%%}%;%.%9%H@%ib%f%4r%<%fF%%t-%%3%%%%,q%)@%hڕ%lI%?%% 1%Fs%^%9% %%pn%~%%a%}%S%5%%d%.%(,%%%)%%K%D%X%[%Q%U%%}6%'5%]%)%- %7%%N%۞%A%1g%{%%B%}:%Ш%%eR%%D%C%%|%%g%M%%a%n%A%X%%=% %v%&s%%tJ%@R%vu% '%%%%'B% 2%#%%%S%K%ǭ%%m%m%;9%%rB%d#%w%՛q%J%㌶& &#P&1&U?&E&YS&z&{&l-&'' kF&?&b&&&sR&KH&-&r&L%v<%%-%ɩ%<%F%w%%Q%Rv%%@%R%%E`%%(% %3I%%%%%%%%R%F%%B&V&&)i3&T״&G&&&9&Q&/&% 9%o)%N% %%%TZ%%Lf%2%u%m%ԟ%%u%%Y%I%o%%7v%-%%a%S%o%]%c%:%_%H%% % +%.%4%%-%4%%%%%Ō% ^%%V% "%B%%%>%9%f%q%%% %}%_%u%T%%>%%ı%%&%Ŋ#%%%Q%N%ۮ%%P%%%?%%Z%̓%L& m&`&&&Z&#%qA%[%°\%%¥%%C%n%i3%%}%ðL%%Pm%J%r8%B%ÑM%`%%Á%%:%5%D%s%&%y%b%%%)[%W%w%%]%D% %%cT%%Y%%w$%%%͠%ظ%4p%1%X%~%%K%sm%%J%Z%%[%%R%X%}%%u%N%1%%%%%YK%%%&%f7%%?%a̅%@%L7%%%%ű%Ӝ%v%/%`%%0%/%%0 %c%q%v %T3%%%%A8%8%%n%%U%=%%8q%%%B%n%%%N%$ %%+%%W[%%%%r%oh%%%%%%%B%2% % %%%%%%%_%6%e'%b%W%Mb%^%;%Tr%^%%?%#%/%%mL%%h%^?%[%K)%Ŏ%%b'%ދ%%%:%%;%V%r%+%Z%b%T%&%%ռ%z%߳%f%q%%(%X%6%Q%^%%%G%%Y%W%`c%I%%%%Fr%_%%%>%\%^%1 %B% %8%]t%> %0%v%%%%/%C%%%o%%%%{%%B.%o%'%р%%%%%E%R%%;% %}H%%>1%̠%%%|%%8%f%t#%A%%]%=%%%>%%*%M%%k;%_%%O%$+%#$%R%g%@o%ˇ%?&&?4&&&4*&y#&$5XTENSION= 'TABLE ' / Table extension BITPIX = 8 / Printable ASCII characters NAXIS = 2 / Simple 2-D matrix NAXIS1 = 336 / Number of characters per row NAXIS2 = 2 / GCOUNT of original file PCOUNT = 0 / No random parameter GCOUNT = 1 / Only one group TFIELDS = 19 / PCOUNT of original file EXTNAME = 'y19g0309t.c2h.tab' / GEIS header file name with .tab TTYPE1 = 'CRVAL1 ' / CRVAL1 = 'pixel number' / TFORM1 = 'D25.16 ' / TBCOL1 = 1 / TTYPE2 = 'CRPIX1 ' / CRPIX1 = 'pixel number of reference pixel' / TFORM2 = 'E15.7 ' / TBCOL2 = 29 / TTYPE3 = 'CD1_1 ' / CD1_1 = 'pixel increment' / TFORM3 = 'E15.7 ' / TBCOL3 = 45 / TTYPE4 = 'DATAMIN ' / DATAMIN = 'the minimum value of the data' / TFORM4 = 'E15.7 ' / TBCOL4 = 61 / TTYPE5 = 'DATAMAX ' / DATAMAX = 'the maximum value of the data' / TFORM5 = 'E15.7 ' / TBCOL5 = 77 / TTYPE6 = 'RA_APER ' / RA_APER = 'right ascension of aperture (deg)' / TFORM6 = 'D25.16 ' / TBCOL6 = 93 / TTYPE7 = 'DEC_APER' / DEC_APER= 'declination of aperture (deg)' / TFORM7 = 'D25.16 ' / TBCOL7 = 121 / TTYPE8 = 'FILLCNT ' / FILLCNT = 'number of segments containing fill' / TFORM8 = 'I11 ' / TBCOL8 = 149 / TTYPE9 = 'ERRCNT ' / ERRCNT = 'the error count of the data' / TFORM9 = 'I11 ' / TBCOL9 = 161 / TTYPE10 = 'FPKTTIME' / FPKTTIME= 'the time of the first packet' / TFORM10 = 'D25.16 ' / TBCOL10 = 173 / TTYPE11 = 'LPKTTIME' / LPKTTIME= 'the time of the last packet' / TFORM11 = 'D25.16 ' / TBCOL11 = 201 / TTYPE12 = 'CTYPE1 ' / CTYPE1 = 'the first coordinate type' / TFORM12 = 'A8 ' / TBCOL12 = 229 / TTYPE13 = 'APER_POS' / APER_POS= 'aperture used' / TFORM13 = 'A8 ' / TBCOL13 = 241 / TTYPE14 = 'PASS_DIR' / PASS_DIR= 'polarization pass direction' / TFORM14 = 'I11 ' / TBCOL14 = 253 / TTYPE15 = 'YPOS ' / YPOS = 'y-position on photocathode' / TFORM15 = 'E15.7 ' / TBCOL15 = 265 / TTYPE16 = 'YTYPE ' / YTYPE = 'observation type: OBJ, SKY, BCK' / TFORM16 = 'A4 ' / TBCOL16 = 281 / TTYPE17 = 'EXPOSURE' / EXPOSURE= 'exposure time per pixel (seconds)' / TFORM17 = 'E15.7 ' / TBCOL17 = 289 / TTYPE18 = 'X_OFFSET' / X_OFFSET= 'x_offset for GIMP correction (diodes)' / TFORM18 = 'E15.7 ' / TBCOL18 = 305 / TTYPE19 = 'Y_OFFSET' / Y_OFFSET= 'y_offset for GIMP correction (defl.units)' / TFORM19 = 'E15.7 ' / TBCOL19 = 321 / END 1.0000000000000000E+00 1.0000000E+00 1.0000000E+00 0.0000000E+00 2.7387550E-15 1.8263573015259999E+02 3.9405888372579994E+01 0 0 4.9099133531036357E+04 4.9099133541163668E+04 PIXEL SINGLE 0 -1.5160000E+03 OBJ 3.1249689E+01 0.0000000E+00 0.0000000E+00 1.0000000000000000E+00 1.0000000E+00 1.0000000E+00 0.0000000E+00 1.9348280E-15 1.8263573015259999E+02 3.9405888372579994E+01 0 0 4.9099135042899798E+04 4.9099135053027116E+04 PIXEL SINGLE 0 -1.5160000E+03 OBJ 6.2499371E+01 0.0000000E+00 0.0000000E+00 \ No newline at end of file diff --git a/src/test/resources/netcdf/ICOADS_R3.0.0_1662-10.nc b/src/test/resources/netcdf/ICOADS_R3.0.0_1662-10.nc new file mode 100644 index 00000000000..30117b7b455 Binary files /dev/null and b/src/test/resources/netcdf/ICOADS_R3.0.0_1662-10.nc differ diff --git a/tests/integration-tests.txt b/tests/integration-tests.txt index 158393791f2..9c955416361 100644 --- a/tests/integration-tests.txt +++ b/tests/integration-tests.txt @@ -1 +1 @@ -DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,HarvestingClientsIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT,LinkIT,DeleteUsersIT,DeactivateUsersIT,AuxiliaryFilesIT,InvalidCharactersIT,LicensesIT,NotificationsIT,BagIT,MetadataBlocksIT,NetcdfIT,SignpostingIT +DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,HarvestingClientsIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT,LinkIT,DeleteUsersIT,DeactivateUsersIT,AuxiliaryFilesIT,InvalidCharactersIT,LicensesIT,NotificationsIT,BagIT,MetadataBlocksIT,NetcdfIT,SignpostingIT,FitsIT