Skip to content

Commit 189c774

Browse files
pdurbinpdurbin
committed
restore original "extract from compound" code #9331
It seems to be working! \o/
1 parent 1fb2798 commit 189c774

2 files changed

Lines changed: 72 additions & 58 deletions

File tree

‎src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java‎

Lines changed: 67 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -36,13 +36,11 @@
3636
import edu.harvard.iq.dataverse.DatasetFieldType;
3737
import edu.harvard.iq.dataverse.DatasetFieldValue;
3838
import edu.harvard.iq.dataverse.DatasetFieldCompoundValue;
39-
import edu.harvard.iq.dataverse.DatasetFieldConstant;
4039
import edu.harvard.iq.dataverse.DatasetLock;
4140
import edu.harvard.iq.dataverse.DatasetVersion;
4241
import edu.harvard.iq.dataverse.DvObject;
4342
import edu.harvard.iq.dataverse.FileMetadata;
4443
import edu.harvard.iq.dataverse.MetadataBlock;
45-
import edu.harvard.iq.dataverse.MetadataBlockServiceBean;
4644
import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
4745
import edu.harvard.iq.dataverse.dataaccess.DataAccess;
4846
import edu.harvard.iq.dataverse.dataaccess.DataAccessOption;
@@ -71,11 +69,7 @@
7169
import edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.sav.SAVFileReaderSpi;
7270
import edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.por.PORFileReader;
7371
import edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.por.PORFileReaderSpi;
74-
import edu.harvard.iq.dataverse.license.LicenseServiceBean;
75-
import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
7672
import edu.harvard.iq.dataverse.util.*;
77-
import edu.harvard.iq.dataverse.util.json.JsonParseException;
78-
import edu.harvard.iq.dataverse.util.json.JsonParser;
7973

8074
import org.apache.commons.io.IOUtils;
8175
//import edu.harvard.iq.dvn.unf.*;
@@ -124,8 +118,6 @@
124118
import javax.jms.QueueSession;
125119
import javax.jms.Message;
126120
import javax.faces.application.FacesMessage;
127-
import javax.json.Json;
128-
import javax.json.JsonObject;
129121
import javax.ws.rs.core.MediaType;
130122
import ucar.nc2.NetcdfFile;
131123
import ucar.nc2.NetcdfFiles;
@@ -153,12 +145,6 @@ public class IngestServiceBean {
153145
AuxiliaryFileServiceBean auxiliaryFileService;
154146
@EJB
155147
SystemConfig systemConfig;
156-
@EJB
157-
MetadataBlockServiceBean metadataBlockService;
158-
@EJB
159-
SettingsServiceBean settingsService;
160-
@EJB
161-
LicenseServiceBean licenseService;
162148

163149
@Resource(lookup = "java:app/jms/queue/ingest")
164150
Queue queue;
@@ -1667,53 +1653,76 @@ private void processDatasetMetadata(FileMetadataIngest fileMetadataIngest, Datas
16671653
// of the child values in the map of extracted values, we'll
16681654
// create a new compound field value and its child
16691655
//
1670-
if (dsft.getName().equals(DatasetFieldConstant.geographicBoundingBox)) {
1671-
for (DatasetFieldType cdsft : dsft.getChildDatasetFieldTypes()) {
1672-
if (cdsft.getName().equals(DatasetFieldConstant.westLongitude)) {
1673-
1674-
String westLongitude = (String) fileMetadataMap.get(DatasetFieldConstant.westLongitude).toArray()[0];
1675-
String eastLongitude = (String) fileMetadataMap.get(DatasetFieldConstant.eastLongitude).toArray()[0];
1676-
String northLatitude = (String) fileMetadataMap.get(DatasetFieldConstant.northLatitude).toArray()[0];
1677-
String southLatitude = (String) fileMetadataMap.get(DatasetFieldConstant.southLatitude).toArray()[0];
1678-
1679-
if (westLongitude == null || eastLongitude == null || northLatitude == null || southLatitude == null) {
1680-
// only populate the bounding box if we have all four values (north, south, east, west)
1681-
break;
1656+
DatasetFieldCompoundValue compoundDsfv = new DatasetFieldCompoundValue();
1657+
int nonEmptyFields = 0;
1658+
for (DatasetFieldType cdsft : dsft.getChildDatasetFieldTypes()) {
1659+
String dsfName = cdsft.getName();
1660+
if (fileMetadataMap.get(dsfName) != null && !fileMetadataMap.get(dsfName).isEmpty()) {
1661+
logger.fine("Ingest Service: found extracted metadata for field " + dsfName + ", part of the compound field " + dsft.getName());
1662+
1663+
if (cdsft.isPrimitive()) {
1664+
// probably an unnecessary check - child fields
1665+
// of compound fields are always primitive...
1666+
// but maybe it'll change in the future.
1667+
if (!cdsft.isControlledVocabulary()) {
1668+
// TODO: can we have controlled vocabulary
1669+
// sub-fields inside compound fields?
1670+
1671+
DatasetField childDsf = new DatasetField();
1672+
childDsf.setDatasetFieldType(cdsft);
1673+
1674+
DatasetFieldValue newDsfv = new DatasetFieldValue(childDsf);
1675+
newDsfv.setValue((String) fileMetadataMap.get(dsfName).toArray()[0]);
1676+
childDsf.getDatasetFieldValues().add(newDsfv);
1677+
1678+
childDsf.setParentDatasetFieldCompoundValue(compoundDsfv);
1679+
compoundDsfv.getChildDatasetFields().add(childDsf);
1680+
1681+
nonEmptyFields++;
1682+
}
1683+
}
1684+
}
1685+
}
1686+
1687+
if (nonEmptyFields > 0) {
1688+
// let's go through this dataset's fields and find the
1689+
// actual parent for this sub-field:
1690+
for (DatasetField dsf : editVersion.getFlatDatasetFields()) {
1691+
if (dsf.getDatasetFieldType().equals(dsft)) {
1692+
1693+
// Now let's check that the dataset version doesn't already have
1694+
// this compound value - we are only interested in aggregating
1695+
// unique values. Note that we need to compare compound values
1696+
// as sets! -- i.e. all the sub fields in 2 compound fields
1697+
// must match in order for these 2 compounds to be recognized
1698+
// as "the same":
1699+
boolean alreadyExists = false;
1700+
for (DatasetFieldCompoundValue dsfcv : dsf.getDatasetFieldCompoundValues()) {
1701+
int matches = 0;
1702+
1703+
for (DatasetField cdsf : dsfcv.getChildDatasetFields()) {
1704+
String cdsfName = cdsf.getDatasetFieldType().getName();
1705+
String cdsfValue = cdsf.getDatasetFieldValues().get(0).getValue();
1706+
if (cdsfValue != null && !cdsfValue.equals("")) {
1707+
String extractedValue = (String) fileMetadataMap.get(cdsfName).toArray()[0];
1708+
logger.fine("values: existing: " + cdsfValue + ", extracted: " + extractedValue);
1709+
if (cdsfValue.equals(extractedValue)) {
1710+
matches++;
1711+
}
1712+
}
1713+
}
1714+
if (matches == nonEmptyFields) {
1715+
alreadyExists = true;
1716+
break;
1717+
}
16821718
}
16831719

1684-
JsonObject json
1685-
= Json.createObjectBuilder()
1686-
.add("typeName", DatasetFieldConstant.geographicBoundingBox)
1687-
.add("value", Json.createArrayBuilder()
1688-
.add(Json.createObjectBuilder()
1689-
.add(DatasetFieldConstant.westLongitude, Json.createObjectBuilder()
1690-
.add("typeName", DatasetFieldConstant.westLongitude)
1691-
.add("value", westLongitude)
1692-
)
1693-
.add(DatasetFieldConstant.eastLongitude, Json.createObjectBuilder()
1694-
.add("typeName", DatasetFieldConstant.eastLongitude)
1695-
.add("value", eastLongitude)
1696-
)
1697-
.add(DatasetFieldConstant.northLatitude, Json.createObjectBuilder()
1698-
.add("typeName", DatasetFieldConstant.northLatitude)
1699-
.add("value", northLatitude)
1700-
)
1701-
.add(DatasetFieldConstant.southLatitude, Json.createObjectBuilder()
1702-
.add("typeName", DatasetFieldConstant.southLatitude)
1703-
.add("value", southLatitude)
1704-
)
1705-
)
1706-
)
1707-
.build();
1708-
1709-
JsonParser jsonParser = new JsonParser(fieldService, metadataBlockService, settingsService, licenseService);
1710-
DatasetField fieldToAdd = null;
1711-
try {
1712-
fieldToAdd = jsonParser.parseField(json, Boolean.FALSE);
1713-
} catch (JsonParseException ex) {
1714-
logger.info("unable to add " + json + ". Exception: " + ex);
1720+
if (!alreadyExists) {
1721+
// save this compound value, by attaching it to the
1722+
// version for proper cascading:
1723+
compoundDsfv.setParentDatasetField(dsf);
1724+
dsf.getDatasetFieldCompoundValues().add(compoundDsfv);
17151725
}
1716-
editVersion.getDatasetFields().add(fieldToAdd);
17171726
}
17181727
}
17191728
}

‎src/test/java/edu/harvard/iq/dataverse/api/FitsIT.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,11 @@ public void testAstroFieldsFromFits() throws IOException {
6868
Map firstAstroTypeFromNativeGet = astroTypeFromNativeGet.get(0);
6969
assertTrue(firstAstroTypeFromNativeGet.toString().contains("Image"));
7070

71+
List<JsonObject> coverageTemportalFromNativeGet = with(getJson.body().asString()).param("coverageTemporal", "coverage.Temporal")
72+
.getJsonObject("data.latestVersion.metadataBlocks.astrophysics.fields.findAll { fields -> fields.typeName == coverageTemporal }");
73+
Map firstcoverageTemporalFromNativeGet = coverageTemportalFromNativeGet.get(0);
74+
assertTrue(firstcoverageTemporalFromNativeGet.toString().contains("1993"));
75+
7176
}
7277

7378
}

0 commit comments

Comments
 (0)
⚡