From 264d64808e3d71f7584a9d3b0941c9f4892224ee Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 4 Nov 2024 12:47:55 -0500 Subject: [PATCH 01/17] allow links between dataset types and metadata blocks #10519 --- doc/release-notes/10519-dataset-types.md | 10 + doc/sphinx-guides/source/api/native-api.rst | 29 +++ .../iq/dataverse/DatasetFieldServiceBean.java | 31 ++- .../harvard/iq/dataverse/api/Datasets.java | 61 ++++- .../harvard/iq/dataverse/api/Dataverses.java | 10 +- .../iq/dataverse/dataset/DatasetType.java | 27 ++- .../impl/ListMetadataBlocksCommand.java | 21 +- ...pdateDatasetTypeLinksToMetadataBlocks.java | 37 +++ .../iq/dataverse/util/json/JsonPrinter.java | 14 +- .../iq/dataverse/api/DatasetTypesIT.java | 228 ++++++++++++++++++ .../iq/dataverse/api/MetadataBlocksIT.java | 3 + .../edu/harvard/iq/dataverse/api/UtilIT.java | 22 +- 12 files changed, 466 insertions(+), 27 deletions(-) create mode 100644 doc/release-notes/10519-dataset-types.md create mode 100644 src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetTypeLinksToMetadataBlocks.java diff --git a/doc/release-notes/10519-dataset-types.md b/doc/release-notes/10519-dataset-types.md new file mode 100644 index 00000000000..e4200e1a3fc --- /dev/null +++ b/doc/release-notes/10519-dataset-types.md @@ -0,0 +1,10 @@ +## Dataset Types can be linked to Metadata Blocks + +Metadata blocks (e.g. "CodeMeta") can now be linked to dataset types (e.g. "software") using new superuser APIs. + +This will have the following effects for the APIs used by the new Dataverse UI ( https://github.com/IQSS/dataverse-frontend ): + +- The list of fields shown when creating a dataset will include fields marked as "displayoncreate" (in the tsv/database) for metadata blocks (e.g. "CodeMeta") that are linked to the dataset type (e.g. "software") that is passed to the API. +- The metadata blocks shown when editing a dataset will include metadata blocks (e.g. "CodeMeta") that are linked to the dataset type (e.g. "software") that is passed to the API. + +For more information, see the guides and #10519. diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 6254742eebb..54325a711e1 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -546,6 +546,9 @@ This endpoint supports the following optional query parameters: - ``returnDatasetFieldTypes``: Whether or not to return the dataset field types present in each metadata block. If not set, the default value is false. - ``onlyDisplayedOnCreate``: Whether or not to return only the metadata blocks that are displayed on dataset creation. If ``returnDatasetFieldTypes`` is true, only the dataset field types shown on dataset creation will be returned within each metadata block. If not set, the default value is false. +- ``datasetType``: Whether or not to return additional fields from metadata blocks that are linked with a particular dataset type. + +are displayed on dataset creation. If ``returnDatasetFieldTypes`` is true, only the dataset field types shown on dataset creation will be returned within each metadata block. If not set, the default value is false. An example using the optional query parameters is presented below: @@ -3193,6 +3196,32 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -H "Content-Type: application/json" "https://demo.dataverse.org/api/datasets/datasetTypes" -X POST -d '{"name": "software"}' +.. _api-link-dataset-type: + +Link Dataset Type with Metadata Blocks +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This API endpoint is superuser only. + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export TYPE=software + export JSON='["codeMeta20"]' + + curl -H "X-Dataverse-key:$API_TOKEN" -H "Content-Type: application/json" "$SERVER_URL/api/datasets/datasetTypes/$TYPE" -X POST -d $JSON + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -H "Content-Type: application/json" "https://demo.dataverse.org/api/datasets/datasetTypes/software" -X POST -d '["codeMeta20"]' + +To update the blocks that are link, send an array with those blocks. + +To remove all links to blocks, send an empty array. + .. _api-delete-dataset-type: Delete Dataset Type diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java index ded7c83de62..b49d179386e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java @@ -1,5 +1,6 @@ package edu.harvard.iq.dataverse; +import edu.harvard.iq.dataverse.dataset.DatasetType; import java.io.IOException; import java.io.StringReader; import java.net.URI; @@ -52,6 +53,7 @@ import org.apache.http.protocol.HttpContext; import org.apache.http.util.EntityUtils; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import java.util.stream.Stream; /** * @@ -890,9 +892,9 @@ public List findAllDisplayedOnCreateInMetadataBlock(MetadataBl return typedQuery.getResultList(); } - public List findAllInMetadataBlockAndDataverse(MetadataBlock metadataBlock, Dataverse dataverse, boolean onlyDisplayedOnCreate) { + public List findAllInMetadataBlockAndDataverse(MetadataBlock metadataBlock, Dataverse dataverse, boolean onlyDisplayedOnCreate, DatasetType datasetType) { if (!dataverse.isMetadataBlockRoot() && dataverse.getOwner() != null) { - return findAllInMetadataBlockAndDataverse(metadataBlock, dataverse.getOwner(), onlyDisplayedOnCreate); + return findAllInMetadataBlockAndDataverse(metadataBlock, dataverse.getOwner(), onlyDisplayedOnCreate, datasetType); } CriteriaBuilder criteriaBuilder = em.getCriteriaBuilder(); @@ -959,7 +961,30 @@ public List findAllInMetadataBlockAndDataverse(MetadataBlock m criteriaQuery.select(datasetFieldTypeRoot).distinct(true); - return em.createQuery(criteriaQuery).getResultList(); + List orig = em.createQuery(criteriaQuery).getResultList(); + List extraFromDatasetTypes = new ArrayList<>(); + + if (datasetType != null) { + // TODO: instead of looping here, try to add datasetType + // to the criteria query above. + for (MetadataBlock mdb : datasetType.getMetadataBlocks()) { + if (mdb.equals(metadataBlock)) { + for (DatasetFieldType datasetFieldType : metadataBlock.getDatasetFieldTypes()) { + if (onlyDisplayedOnCreate) { + if (datasetFieldType.isDisplayOnCreate()) { + extraFromDatasetTypes.add(datasetFieldType); + } else { + logger.fine("skipping because " + datasetFieldType.getName() + " is not 'display on create'"); + } + } else { + extraFromDatasetTypes.add(datasetFieldType); + } + } + } + } + } + + return Stream.concat(orig.stream(), extraFromDatasetTypes.stream()).toList(); } private Predicate buildRequiredInDataversePredicate(CriteriaBuilder criteriaBuilder, Root datasetFieldTypeRoot) { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 369a22fe8d7..ea58a670627 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -5109,14 +5109,10 @@ public Response resetPidGenerator(@Context ContainerRequestContext crc, @PathPar @Path("datasetTypes") public Response getDatasetTypes() { JsonArrayBuilder jab = Json.createArrayBuilder(); - List datasetTypes = datasetTypeSvc.listAll(); - for (DatasetType datasetType : datasetTypes) { - JsonObjectBuilder job = Json.createObjectBuilder(); - job.add("id", datasetType.getId()); - job.add("name", datasetType.getName()); - jab.add(job); - } - return ok(jab.build()); + for (DatasetType datasetType : datasetTypeSvc.listAll()) { + jab.add(datasetType.toJson()); + } + return ok(jab); } @GET @@ -5231,4 +5227,53 @@ public Response deleteDatasetType(@Context ContainerRequestContext crc, @PathPar } } + @AuthRequired + @PUT + @Path("datasetTypes/{idOrName}") + public Response updateDatasetTypeLinksWithMetadataBlocks(@Context ContainerRequestContext crc, @PathParam("idOrName") String idOrName, String jsonBody) { + DatasetType datasetType = null; + if (StringUtils.isNumeric(idOrName)) { + try { + long id = Long.parseLong(idOrName); + datasetType = datasetTypeSvc.getById(id); + } catch (NumberFormatException ex) { + return error(NOT_FOUND, "Could not find a dataset type with id " + idOrName); + } + } else { + datasetType = datasetTypeSvc.getByName(idOrName); + } + JsonArrayBuilder datasetTypesBefore = Json.createArrayBuilder(); + for (MetadataBlock metadataBlock : datasetType.getMetadataBlocks()) { + datasetTypesBefore.add(metadataBlock.getName()); + } + JsonArrayBuilder datasetTypesAfter = Json.createArrayBuilder(); + List metadataBlocksToSave = new ArrayList<>(); + if (jsonBody != null && !jsonBody.isEmpty()) { + JsonArray json = JsonUtil.getJsonArray(jsonBody); + for (JsonString jsonValue : json.getValuesAs(JsonString.class)) { + String name = jsonValue.getString(); + System.out.println("name: " + name); + MetadataBlock metadataBlock = metadataBlockSvc.findByName(name); + if (metadataBlock != null) { + metadataBlocksToSave.add(metadataBlock); + datasetTypesAfter.add(name); + } else { + String availableBlocks = metadataBlockSvc.listMetadataBlocks().stream().map(MetadataBlock::getName).collect(Collectors.joining(", ")); + return badRequest("Metadata block not found: " + name + ". Available metadata blocks: " + availableBlocks); + } + } + } + try { + execCommand(new UpdateDatasetTypeLinksToMetadataBlocks(createDataverseRequest(getRequestUser(crc)), datasetType, metadataBlocksToSave)); + return ok(Json.createObjectBuilder() + .add("linkedMetadataBlocks", Json.createObjectBuilder() + .add("before", datasetTypesBefore) + .add("after", datasetTypesAfter)) + ); + + } catch (WrappedResponse ex) { + return ex.getResponse(); + } + } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index f05bba8830e..8010079ef86 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -15,6 +15,7 @@ import edu.harvard.iq.dataverse.authorization.groups.impl.explicit.ExplicitGroupServiceBean; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.authorization.users.User; +import edu.harvard.iq.dataverse.dataset.DatasetType; import edu.harvard.iq.dataverse.dataverse.DataverseUtil; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.impl.*; @@ -801,17 +802,20 @@ public Response deleteDataverseLinkingDataverse(@Context ContainerRequestContext public Response listMetadataBlocks(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf, @QueryParam("onlyDisplayedOnCreate") boolean onlyDisplayedOnCreate, - @QueryParam("returnDatasetFieldTypes") boolean returnDatasetFieldTypes) { + @QueryParam("returnDatasetFieldTypes") boolean returnDatasetFieldTypes, + @QueryParam("datasetType") String datasetTypeIn) { try { Dataverse dataverse = findDataverseOrDie(dvIdtf); + DatasetType datasetType = datasetTypeSvc.getByName(datasetTypeIn); final List metadataBlocks = execCommand( new ListMetadataBlocksCommand( createDataverseRequest(getRequestUser(crc)), dataverse, - onlyDisplayedOnCreate + onlyDisplayedOnCreate, + datasetType ) ); - return ok(json(metadataBlocks, returnDatasetFieldTypes, onlyDisplayedOnCreate, dataverse)); + return ok(json(metadataBlocks, returnDatasetFieldTypes, onlyDisplayedOnCreate, dataverse, datasetType)); } catch (WrappedResponse we) { return we.getResponse(); } diff --git a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetType.java b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetType.java index 78bf232e1a6..727703852eb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetType.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetType.java @@ -1,17 +1,23 @@ package edu.harvard.iq.dataverse.dataset; +import edu.harvard.iq.dataverse.MetadataBlock; import jakarta.json.Json; +import jakarta.json.JsonArrayBuilder; import jakarta.json.JsonObjectBuilder; +import jakarta.persistence.CascadeType; import jakarta.persistence.Column; import jakarta.persistence.Entity; import jakarta.persistence.GeneratedValue; import jakarta.persistence.GenerationType; import jakarta.persistence.Id; +import jakarta.persistence.ManyToMany; import jakarta.persistence.NamedQueries; import jakarta.persistence.NamedQuery; import jakarta.persistence.Table; import jakarta.persistence.UniqueConstraint; import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; @NamedQueries({ @NamedQuery(name = "DatasetType.findAll", @@ -42,6 +48,12 @@ public class DatasetType implements Serializable { @Column(nullable = false) private String name; + /** + * The metadata blocks this dataset type is linked to. + */ + @ManyToMany(cascade = {CascadeType.MERGE}) + private List metadataBlocks = new ArrayList<>(); + public DatasetType() { } @@ -61,10 +73,23 @@ public void setName(String name) { this.name = name; } + public List getMetadataBlocks() { + return metadataBlocks; + } + + public void setMetadataBlocks(List metadataBlocks) { + this.metadataBlocks = metadataBlocks; + } + public JsonObjectBuilder toJson() { + JsonArrayBuilder linkedMetadataBlocks = Json.createArrayBuilder(); + for (MetadataBlock metadataBlock : this.getMetadataBlocks()) { + linkedMetadataBlocks.add(metadataBlock.getName()); + } return Json.createObjectBuilder() .add("id", getId()) - .add("name", getName()); + .add("name", getName()) + .add("linkedMetadataBlocks", linkedMetadataBlocks); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListMetadataBlocksCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListMetadataBlocksCommand.java index 8275533ced2..e79d36de07d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListMetadataBlocksCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListMetadataBlocksCommand.java @@ -3,15 +3,18 @@ import edu.harvard.iq.dataverse.Dataverse; import edu.harvard.iq.dataverse.MetadataBlock; import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.dataset.DatasetType; import edu.harvard.iq.dataverse.engine.command.AbstractCommand; import edu.harvard.iq.dataverse.engine.command.CommandContext; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Stream; /** * Lists the metadata blocks of a {@link Dataverse}. @@ -23,11 +26,13 @@ public class ListMetadataBlocksCommand extends AbstractCommand execute(CommandContext ctxt) throws CommandException if (onlyDisplayedOnCreate) { return listMetadataBlocksDisplayedOnCreate(ctxt, dataverse); } - return dataverse.getMetadataBlocks(); + List orig = dataverse.getMetadataBlocks(); + List extraFromDatasetTypes = new ArrayList<>(); + if (datasetType != null) { + extraFromDatasetTypes = datasetType.getMetadataBlocks(); + } + return Stream.concat(orig.stream(), extraFromDatasetTypes.stream()).toList(); } private List listMetadataBlocksDisplayedOnCreate(CommandContext ctxt, Dataverse dataverse) { if (dataverse.isMetadataBlockRoot() || dataverse.getOwner() == null) { - return ctxt.metadataBlocks().listMetadataBlocksDisplayedOnCreate(dataverse); + List orig = ctxt.metadataBlocks().listMetadataBlocksDisplayedOnCreate(dataverse); + List extraFromDatasetTypes = new ArrayList<>(); + if (datasetType != null) { + extraFromDatasetTypes = datasetType.getMetadataBlocks(); + } + return Stream.concat(orig.stream(), extraFromDatasetTypes.stream()).toList(); } return listMetadataBlocksDisplayedOnCreate(ctxt, dataverse.getOwner()); } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetTypeLinksToMetadataBlocks.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetTypeLinksToMetadataBlocks.java new file mode 100644 index 00000000000..476935facd7 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetTypeLinksToMetadataBlocks.java @@ -0,0 +1,37 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.DvObject; +import edu.harvard.iq.dataverse.MetadataBlock; +import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import edu.harvard.iq.dataverse.dataset.DatasetType; +import edu.harvard.iq.dataverse.engine.command.AbstractVoidCommand; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; +import edu.harvard.iq.dataverse.engine.command.exception.PermissionException; +import java.util.List; + +@RequiredPermissions({}) +public class UpdateDatasetTypeLinksToMetadataBlocks extends AbstractVoidCommand { + + final DatasetType datasetType; + List metadataBlocks; + + public UpdateDatasetTypeLinksToMetadataBlocks(DataverseRequest dataverseRequest, DatasetType datasetType, List metadataBlocks) { + super(dataverseRequest, (DvObject) null); + this.datasetType = datasetType; + this.metadataBlocks = metadataBlocks; + } + + @Override + protected void executeImpl(CommandContext ctxt) throws CommandException { + if (!(getUser() instanceof AuthenticatedUser) || !getUser().isSuperuser()) { + throw new PermissionException("Update dataset type links to metadata block command can only be called by superusers.", + this, null, null); + } + datasetType.setMetadataBlocks(metadataBlocks); + ctxt.em().merge(datasetType); + } + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index 1bdee48b14d..3509972855a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -596,13 +596,14 @@ public static JsonObjectBuilder json(MetadataBlock block, List fie } public static JsonArrayBuilder json(List metadataBlocks, boolean returnDatasetFieldTypes, boolean printOnlyDisplayedOnCreateDatasetFieldTypes) { - return json(metadataBlocks, returnDatasetFieldTypes, printOnlyDisplayedOnCreateDatasetFieldTypes, null); + return json(metadataBlocks, returnDatasetFieldTypes, printOnlyDisplayedOnCreateDatasetFieldTypes, null, null); } - public static JsonArrayBuilder json(List metadataBlocks, boolean returnDatasetFieldTypes, boolean printOnlyDisplayedOnCreateDatasetFieldTypes, Dataverse ownerDataverse) { + // TODO: consider renaming "ownerDataverse" to just "dataverse" + public static JsonArrayBuilder json(List metadataBlocks, boolean returnDatasetFieldTypes, boolean printOnlyDisplayedOnCreateDatasetFieldTypes, Dataverse ownerDataverse, DatasetType datasetType) { JsonArrayBuilder arrayBuilder = Json.createArrayBuilder(); for (MetadataBlock metadataBlock : metadataBlocks) { - arrayBuilder.add(returnDatasetFieldTypes ? json(metadataBlock, printOnlyDisplayedOnCreateDatasetFieldTypes, ownerDataverse) : brief.json(metadataBlock)); + arrayBuilder.add(returnDatasetFieldTypes ? json(metadataBlock, printOnlyDisplayedOnCreateDatasetFieldTypes, ownerDataverse, datasetType) : brief.json(metadataBlock)); } return arrayBuilder; } @@ -630,10 +631,11 @@ public static JsonObject json(DatasetField dfv) { } public static JsonObjectBuilder json(MetadataBlock metadataBlock) { - return json(metadataBlock, false, null); + return json(metadataBlock, false, null, null); } - public static JsonObjectBuilder json(MetadataBlock metadataBlock, boolean printOnlyDisplayedOnCreateDatasetFieldTypes, Dataverse ownerDataverse) { + // TODO: consider renaming "ownerDataverse" to just "dataverse" + public static JsonObjectBuilder json(MetadataBlock metadataBlock, boolean printOnlyDisplayedOnCreateDatasetFieldTypes, Dataverse ownerDataverse, DatasetType datasetType) { JsonObjectBuilder jsonObjectBuilder = jsonObjectBuilder() .add("id", metadataBlock.getId()) .add("name", metadataBlock.getName()) @@ -644,7 +646,7 @@ public static JsonObjectBuilder json(MetadataBlock metadataBlock, boolean printO if (ownerDataverse != null) { datasetFieldTypes = new TreeSet<>(datasetFieldService.findAllInMetadataBlockAndDataverse( - metadataBlock, ownerDataverse, printOnlyDisplayedOnCreateDatasetFieldTypes)); + metadataBlock, ownerDataverse, printOnlyDisplayedOnCreateDatasetFieldTypes, datasetType)); } else { datasetFieldTypes = printOnlyDisplayedOnCreateDatasetFieldTypes ? new TreeSet<>(datasetFieldService.findAllDisplayedOnCreateInMetadataBlock(metadataBlock)) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java index a0b9f5325d0..dc61455fc7f 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java @@ -9,11 +9,16 @@ import static jakarta.ws.rs.core.Response.Status.CREATED; import static jakarta.ws.rs.core.Response.Status.FORBIDDEN; import static jakarta.ws.rs.core.Response.Status.OK; +import java.io.IOException; +import java.nio.file.Paths; import java.util.UUID; import org.hamcrest.CoreMatchers; import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.nullValue; import static org.junit.jupiter.api.Assertions.assertEquals; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; public class DatasetTypesIT { @@ -265,4 +270,227 @@ public void testAddAndDeleteDatasetType() { } + @Test + public void testUpdateDatasetTypeLinksWithMetadataBlocks() { + Response createUser = UtilIT.createRandomUser(); + createUser.then().assertThat().statusCode(OK.getStatusCode()); + String username = UtilIT.getUsernameFromResponse(createUser); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + UtilIT.setSuperuserStatus(username, true).then().assertThat().statusCode(OK.getStatusCode()); + + System.out.println("listing root collection blocks with display on create: only citation"); + Response listBlocks = UtilIT.listMetadataBlocks(":root", true, false, apiToken); + listBlocks.prettyPrint(); + listBlocks.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data[0].name", is("citation")) + .body("data[1].name", nullValue()); + + System.out.println("listing root collection blocks without display on create: only citation"); + listBlocks = UtilIT.listMetadataBlocks(":root", false, false, apiToken); + listBlocks.prettyPrint(); + listBlocks.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data[0].name", is("citation")) + .body("data[1].name", nullValue()); + + //Avoid all-numeric names (which are not allowed) + String randomName = "zzz" + UUID.randomUUID().toString().substring(0, 8); + String jsonIn = Json.createObjectBuilder().add("name", randomName).build().toString(); + + System.out.println("adding type with name " + randomName); + Response typeAdded = UtilIT.addDatasetType(jsonIn, apiToken); + typeAdded.prettyPrint(); + typeAdded.then().assertThat().statusCode(OK.getStatusCode()); + + Long typeId = JsonPath.from(typeAdded.getBody().asString()).getLong("data.id"); + + System.out.println("id of type: " + typeId); + Response getTypeById = UtilIT.getDatasetType(typeId.toString()); + getTypeById.prettyPrint(); + getTypeById.then().assertThat().statusCode(OK.getStatusCode()); + + String updateToTheseTypes = Json.createArrayBuilder() + .add(randomName) + .build().toString(); + + String metadataBlockToLink = """ + ["geospatial"] +"""; + + Response linkDatasetType1ToGeospatial = UtilIT.updateDatasetTypeLinksWithMetadataBlocks(randomName, metadataBlockToLink, apiToken); + linkDatasetType1ToGeospatial.prettyPrint(); + linkDatasetType1ToGeospatial.then().assertThat(). + statusCode(OK.getStatusCode()) + .body("data.linkedMetadataBlocks.after[0]", CoreMatchers.is("geospatial")); + + getTypeById = UtilIT.getDatasetType(typeId.toString()); + getTypeById.prettyPrint(); + getTypeById.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.linkedMetadataBlocks[0]", CoreMatchers.is("geospatial")); + + System.out.println("listing root collection blocks with display on create"); + listBlocks = UtilIT.listMetadataBlocks(":root", true, false, randomName, apiToken); + listBlocks.prettyPrint(); + listBlocks.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data[0].name", is("citation")) + .body("data[1].name", is("geospatial")) + .body("data[2].name", nullValue()); + + System.out.println("listing root collection blocks without display on create"); + listBlocks = UtilIT.listMetadataBlocks(":root", false, false, randomName, apiToken); + listBlocks.prettyPrint(); + listBlocks.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data[0].name", is("citation")) + .body("data[1].name", is("geospatial")) + .body("data[2].name", nullValue()); + + Response createDataverse = UtilIT.createRandomDataverse(apiToken); + createDataverse.then().assertThat().statusCode(CREATED.getStatusCode()); + + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverse); + Integer dataverseId = UtilIT.getDataverseIdFromResponse(createDataverse); + + UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken).then().assertThat().statusCode(OK.getStatusCode()); + + System.out.println("listing " + dataverseAlias + " collection blocks with display on create using dataset type " + randomName); + listBlocks = UtilIT.listMetadataBlocks(dataverseAlias, true, false, randomName, apiToken); + listBlocks.prettyPrint(); + listBlocks.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data[0].name", is("citation")) + .body("data[1].name", is("geospatial")) + .body("data[2].name", nullValue()); + + System.out.println("listing " + dataverseAlias + " collection blocks without display on create using dataset type " + randomName); + listBlocks = UtilIT.listMetadataBlocks(dataverseAlias, false, false, randomName, apiToken); + listBlocks.prettyPrint(); + listBlocks.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data[0].name", is("citation")) + .body("data[1].name", is("geospatial")) + .body("data[2].name", nullValue()); + + // We send an empty array to mean "delete or clear all" + String emptyJsonArray = "[]"; + Response removeDatasetTypeLinks = UtilIT.updateDatasetTypeLinksWithMetadataBlocks(randomName, emptyJsonArray, apiToken); + removeDatasetTypeLinks.prettyPrint(); + removeDatasetTypeLinks.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.linkedMetadataBlocks.after[0]", CoreMatchers.nullValue()); + + listBlocks = UtilIT.listMetadataBlocks(dataverseAlias, true, false, randomName, apiToken); + listBlocks.prettyPrint(); + listBlocks.then().assertThat() + .body("data[0].name", is("citation")); + } + + /** + * This test is disabled because loads the "codeMeta20" metadata block. + * + * Loading any additional block causes this test to fail: + * MetadataBlocksIT#testListMetadataBlocks + */ + @Disabled + @Test + public void testLinkSoftwareToCodemeta() { + Response listBlocksAvailable = UtilIT.listMetadataBlocks(false, false); + listBlocksAvailable.then().assertThat().statusCode(OK.getStatusCode()); + String blocksAsString = JsonPath.from(listBlocksAvailable.getBody().asString()).getString("data"); + System.out.println("blocks: " + blocksAsString); + if (!blocksAsString.contains("codeMeta20")) { + System.out.println("CodeMeta hasn't been added. Adding it..."); + byte[] codemetaTsv = null; + try { + codemetaTsv = java.nio.file.Files.readAllBytes(Paths.get("scripts/api/data/metadatablocks/codemeta.tsv")); + } catch (IOException e) { + } + UtilIT.loadMetadataBlock("", codemetaTsv); + } else { + System.out.println("CodeMeta has already been added."); + } + + Response createUser = UtilIT.createRandomUser(); + createUser.then().assertThat().statusCode(OK.getStatusCode()); + String username = UtilIT.getUsernameFromResponse(createUser); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + UtilIT.setSuperuserStatus(username, true).then().assertThat().statusCode(OK.getStatusCode()); + + String metadataBlockLink = """ + ["codeMeta20"] +//"""; + + String datasetType = "software"; + Response linkSoftwareToCodemeta = UtilIT.updateDatasetTypeLinksWithMetadataBlocks(datasetType, metadataBlockLink, apiToken); + linkSoftwareToCodemeta.prettyPrint(); + linkSoftwareToCodemeta.then().assertThat(). + statusCode(OK.getStatusCode()) + .body("data.linkedMetadataBlocks.after[0]", CoreMatchers.is("codeMeta20")); + + Response createDataverse = UtilIT.createRandomDataverse(apiToken); + createDataverse.then().assertThat().statusCode(CREATED.getStatusCode()); + + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverse); + Integer dataverseId = UtilIT.getDataverseIdFromResponse(createDataverse); + + UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken).then().assertThat().statusCode(OK.getStatusCode()); + + Response listBlocks = null; + System.out.println("listing root collection blocks with display on create using dataset type " + datasetType); + listBlocks = UtilIT.listMetadataBlocks(":root", true, true, datasetType, apiToken); + listBlocks.prettyPrint(); + listBlocks.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data[0].name", is("citation")) + .body("data[1].name", is("codeMeta20")) + .body("data[2].name", nullValue()) + .body("data[0].fields.title.displayOnCreate", equalTo(true)) + .body("data[1].fields.codeVersion.displayOnCreate", equalTo(true)); + + System.out.println("listing root collection blocks with all fields (not display on create) using dataset type " + datasetType); + listBlocks = UtilIT.listMetadataBlocks(":root", false, true, datasetType, apiToken); + listBlocks.prettyPrint(); + listBlocks.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data[0].name", is("citation")) + .body("data[1].name", is("codeMeta20")) + .body("data[2].name", nullValue()) + .body("data[0].fields.title.displayOnCreate", equalTo(true)) + .body("data[0].fields.subtitle.displayOnCreate", equalTo(false)) + .body("data[1].fields.codeVersion.displayOnCreate", equalTo(true)) + .body("data[1].fields.issueTracker.displayOnCreate", equalTo(false)); + + System.out.println("listing " + dataverseAlias + " collection blocks with display on create using dataset type " + datasetType); + listBlocks = UtilIT.listMetadataBlocks(dataverseAlias, true, true, datasetType, apiToken); + listBlocks.prettyPrint(); + listBlocks.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data[0].name", is("citation")) + .body("data[1].name", is("codeMeta20")) + .body("data[2].name", nullValue()) + .body("data[0].fields.title.displayOnCreate", equalTo(true)) + // subtitle is hidden because it is not "display on create" + .body("data[0].fields.subtitle", nullValue()) + .body("data[1].fields.codeVersion.displayOnCreate", equalTo(true)) + // issueTracker is hidden because it is not "display on create" + .body("data[1].fields.issueTracker", nullValue()); + + System.out.println("listing " + dataverseAlias + " collection blocks with all fields (not display on create) using dataset type " + datasetType); + listBlocks = UtilIT.listMetadataBlocks(dataverseAlias, false, true, datasetType, apiToken); + listBlocks.prettyPrint(); + listBlocks.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data[0].name", is("citation")) + .body("data[1].name", is("codeMeta20")) + .body("data[2].name", nullValue()) + .body("data[0].fields.title.displayOnCreate", equalTo(true)) + .body("data[0].fields.subtitle.displayOnCreate", equalTo(false)) + .body("data[1].fields.codeVersion.displayOnCreate", equalTo(true)) + .body("data[1].fields.issueTracker.displayOnCreate", equalTo(false)); + + } + } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/MetadataBlocksIT.java b/src/test/java/edu/harvard/iq/dataverse/api/MetadataBlocksIT.java index 6e7061961f0..072cb09050c 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/MetadataBlocksIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/MetadataBlocksIT.java @@ -25,6 +25,9 @@ public static void setUpClass() { void testListMetadataBlocks() { // No optional params enabled Response listMetadataBlocksResponse = UtilIT.listMetadataBlocks(false, false); + // TODO: consider re-writing this test to allow additional metadata blocks + // to be added by other tests. We'd like to load the "codeMeta20" block in + // DatasetTypesIT#testLinkSoftwareToCodemeta but it causes this test to break. int expectedDefaultNumberOfMetadataBlocks = 6; listMetadataBlocksResponse.then().assertThat() .statusCode(OK.getStatusCode()) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 502f1ecb0a8..01b9f6423c9 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -753,11 +753,18 @@ static Response setMetadataBlocks(String dataverseAlias, JsonArrayBuilder blocks } static Response listMetadataBlocks(String dataverseAlias, boolean onlyDisplayedOnCreate, boolean returnDatasetFieldTypes, String apiToken) { - return given() + return listMetadataBlocks(dataverseAlias, onlyDisplayedOnCreate, returnDatasetFieldTypes, null, apiToken); + } + + static Response listMetadataBlocks(String dataverseAlias, boolean onlyDisplayedOnCreate, boolean returnDatasetFieldTypes, String datasetType, String apiToken) { + RequestSpecification requestSpecification = given() .header(API_TOKEN_HTTP_HEADER, apiToken) .queryParam("onlyDisplayedOnCreate", onlyDisplayedOnCreate) - .queryParam("returnDatasetFieldTypes", returnDatasetFieldTypes) - .get("/api/dataverses/" + dataverseAlias + "/metadatablocks"); + .queryParam("returnDatasetFieldTypes", returnDatasetFieldTypes); + if (datasetType != null) { + requestSpecification.queryParam("datasetType", datasetType); + } + return requestSpecification.get("/api/dataverses/" + dataverseAlias + "/metadatablocks"); } static Response listMetadataBlocks(boolean onlyDisplayedOnCreate, boolean returnDatasetFieldTypes) { @@ -4241,4 +4248,13 @@ static Response deleteDatasetTypes(long doomed, String apiToken) { .delete("/api/datasets/datasetTypes/" + doomed); } + static Response updateDatasetTypeLinksWithMetadataBlocks(String idOrName, String jsonArrayOfMetadataBlocks, String apiToken) { + return given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .body(jsonArrayOfMetadataBlocks) + // Do we need to send content type = json? + .contentType(ContentType.JSON) + .put("/api/datasets/datasetTypes/" + idOrName); + } + } From b6fb92b05d3d71c533c504f2b1ce1399118419bd Mon Sep 17 00:00:00 2001 From: GPortas Date: Tue, 5 Nov 2024 17:11:38 +0000 Subject: [PATCH 02/17] Changed: using JPA criteria instead of code looping for DatasetType query --- .../iq/dataverse/DatasetFieldServiceBean.java | 96 ++++++++++++------- .../iq/dataverse/api/DatasetTypesIT.java | 20 ++++ 2 files changed, 82 insertions(+), 34 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java index b49d179386e..a9752bc0596 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java @@ -53,7 +53,6 @@ import org.apache.http.protocol.HttpContext; import org.apache.http.util.EntityUtils; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; -import java.util.stream.Stream; /** * @@ -873,7 +872,7 @@ public List findAllDisplayedOnCreateInMetadataBlock(MetadataBl Root metadataBlockRoot = criteriaQuery.from(MetadataBlock.class); Root datasetFieldTypeRoot = criteriaQuery.from(DatasetFieldType.class); - Predicate requiredInDataversePredicate = buildRequiredInDataversePredicate(criteriaBuilder, datasetFieldTypeRoot); + Predicate fieldRequiredInTheInstallation = buildFieldRequiredInTheInstallationPredicate(criteriaBuilder, datasetFieldTypeRoot); criteriaQuery.where( criteriaBuilder.and( @@ -881,7 +880,7 @@ public List findAllDisplayedOnCreateInMetadataBlock(MetadataBl datasetFieldTypeRoot.in(metadataBlockRoot.get("datasetFieldTypes")), criteriaBuilder.or( criteriaBuilder.isTrue(datasetFieldTypeRoot.get("displayOnCreate")), - requiredInDataversePredicate + fieldRequiredInTheInstallation ) ) ); @@ -902,6 +901,29 @@ public List findAllInMetadataBlockAndDataverse(MetadataBlock m Root metadataBlockRoot = criteriaQuery.from(MetadataBlock.class); Root datasetFieldTypeRoot = criteriaQuery.from(DatasetFieldType.class); + + // Build the main predicate to include fields that belong to the specified dataverse and metadataBlock and match the onlyDisplayedOnCreate value. + Predicate fieldPresentInDataverse = buildFieldPresentInDataversePredicate(dataverse, onlyDisplayedOnCreate, criteriaQuery, criteriaBuilder, datasetFieldTypeRoot, metadataBlockRoot); + + // Build an additional predicate to include fields from the datasetType, if the datasetType is specified and contains the given metadataBlock. + Predicate fieldPresentInDatasetType = buildFieldPresentInDatasetTypePredicate(datasetType, criteriaQuery, criteriaBuilder, datasetFieldTypeRoot, metadataBlockRoot, onlyDisplayedOnCreate); + + // Build the final WHERE clause by combining all the predicates. + criteriaQuery.where( + criteriaBuilder.equal(metadataBlockRoot.get("id"), metadataBlock.getId()), // Match the MetadataBlock ID. + datasetFieldTypeRoot.in(metadataBlockRoot.get("datasetFieldTypes")), // Ensure the DatasetFieldType is part of the MetadataBlock. + criteriaBuilder.or( + fieldPresentInDataverse, + fieldPresentInDatasetType + ) + ); + + criteriaQuery.select(datasetFieldTypeRoot).distinct(true); + + return em.createQuery(criteriaQuery).getResultList(); + } + + private Predicate buildFieldPresentInDataversePredicate(Dataverse dataverse, boolean onlyDisplayedOnCreate, CriteriaQuery criteriaQuery, CriteriaBuilder criteriaBuilder, Root datasetFieldTypeRoot, Root metadataBlockRoot) { Root dataverseRoot = criteriaQuery.from(Dataverse.class); // Join Dataverse with DataverseFieldTypeInputLevel on the "dataverseFieldTypeInputLevels" attribute, using a LEFT JOIN. @@ -932,7 +954,7 @@ public List findAllInMetadataBlockAndDataverse(MetadataBlock m Predicate hasNoInputLevelPredicate = criteriaBuilder.not(criteriaBuilder.exists(subquery)); // Define a predicate to include the required fields in Dataverse. - Predicate requiredInDataversePredicate = buildRequiredInDataversePredicate(criteriaBuilder, datasetFieldTypeRoot); + Predicate fieldRequiredInTheInstallation = buildFieldRequiredInTheInstallationPredicate(criteriaBuilder, datasetFieldTypeRoot); // Define a predicate for displaying DatasetFieldTypes on create. // If onlyDisplayedOnCreate is true, include fields that: @@ -943,51 +965,57 @@ public List findAllInMetadataBlockAndDataverse(MetadataBlock m ? criteriaBuilder.or( criteriaBuilder.or( criteriaBuilder.isTrue(datasetFieldTypeRoot.get("displayOnCreate")), - requiredInDataversePredicate + fieldRequiredInTheInstallation ), requiredAsInputLevelPredicate ) : criteriaBuilder.conjunction(); - // Build the final WHERE clause by combining all the predicates. - criteriaQuery.where( + // Combine all the predicates. + return criteriaBuilder.and( criteriaBuilder.equal(dataverseRoot.get("id"), dataverse.getId()), // Match the Dataverse ID. - criteriaBuilder.equal(metadataBlockRoot.get("id"), metadataBlock.getId()), // Match the MetadataBlock ID. metadataBlockRoot.in(dataverseRoot.get("metadataBlocks")), // Ensure the MetadataBlock is part of the Dataverse. - datasetFieldTypeRoot.in(metadataBlockRoot.get("datasetFieldTypes")), // Ensure the DatasetFieldType is part of the MetadataBlock. criteriaBuilder.or(includedAsInputLevelPredicate, hasNoInputLevelPredicate), // Include DatasetFieldTypes based on the input level predicates. displayedOnCreatePredicate // Apply the display-on-create filter if necessary. ); + } - criteriaQuery.select(datasetFieldTypeRoot).distinct(true); - - List orig = em.createQuery(criteriaQuery).getResultList(); - List extraFromDatasetTypes = new ArrayList<>(); - + private Predicate buildFieldPresentInDatasetTypePredicate(DatasetType datasetType, + CriteriaQuery criteriaQuery, + CriteriaBuilder criteriaBuilder, + Root datasetFieldTypeRoot, + Root metadataBlockRoot, + boolean onlyDisplayedOnCreate) { + Predicate datasetTypePredicate = criteriaBuilder.isFalse(criteriaBuilder.literal(true)); // Initialize datasetTypePredicate to always false by default if (datasetType != null) { - // TODO: instead of looping here, try to add datasetType - // to the criteria query above. - for (MetadataBlock mdb : datasetType.getMetadataBlocks()) { - if (mdb.equals(metadataBlock)) { - for (DatasetFieldType datasetFieldType : metadataBlock.getDatasetFieldTypes()) { - if (onlyDisplayedOnCreate) { - if (datasetFieldType.isDisplayOnCreate()) { - extraFromDatasetTypes.add(datasetFieldType); - } else { - logger.fine("skipping because " + datasetFieldType.getName() + " is not 'display on create'"); - } - } else { - extraFromDatasetTypes.add(datasetFieldType); - } - } - } - } + // Create a subquery to check for the presence of the specified metadataBlock within the datasetType + Subquery datasetTypeSubquery = criteriaQuery.subquery(Long.class); + Root datasetTypeRoot = criteriaQuery.from(DatasetType.class); + + // Define a predicate for displaying DatasetFieldTypes on create. + // If onlyDisplayedOnCreate is true, include fields that are either marked as displayed on create OR marked as required. + // Otherwise, use an always-true predicate (conjunction). + Predicate displayedOnCreatePredicate = onlyDisplayedOnCreate ? + criteriaBuilder.or( + criteriaBuilder.isTrue(datasetFieldTypeRoot.get("displayOnCreate")), + buildFieldRequiredInTheInstallationPredicate(criteriaBuilder, datasetFieldTypeRoot) + ) + : criteriaBuilder.conjunction(); + + datasetTypeSubquery.select(criteriaBuilder.literal(1L)) + .where( + criteriaBuilder.equal(datasetTypeRoot.get("id"), datasetType.getId()), // Match the DatasetType ID. + metadataBlockRoot.in(datasetTypeRoot.get("metadataBlocks")), // Ensure the metadataBlock is included in the datasetType's list of metadata blocks. + displayedOnCreatePredicate + ); + + // Now set the datasetTypePredicate to true if the subquery finds a matching metadataBlock + datasetTypePredicate = criteriaBuilder.exists(datasetTypeSubquery); } - - return Stream.concat(orig.stream(), extraFromDatasetTypes.stream()).toList(); + return datasetTypePredicate; } - private Predicate buildRequiredInDataversePredicate(CriteriaBuilder criteriaBuilder, Root datasetFieldTypeRoot) { + private Predicate buildFieldRequiredInTheInstallationPredicate(CriteriaBuilder criteriaBuilder, Root datasetFieldTypeRoot) { // Predicate to check if the current DatasetFieldType is required. Predicate isRequired = criteriaBuilder.isTrue(datasetFieldTypeRoot.get("required")); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java index dc61455fc7f..aa552c3ed79 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java @@ -374,6 +374,26 @@ public void testUpdateDatasetTypeLinksWithMetadataBlocks() { .body("data[1].name", is("geospatial")) .body("data[2].name", nullValue()); + System.out.println("listing " + dataverseAlias + " collection blocks and inner dataset field types, without display on create and return dataset field types set to true using dataset type " + randomName); + listBlocks = UtilIT.listMetadataBlocks(dataverseAlias, false, true, randomName, apiToken); + listBlocks.prettyPrint(); + listBlocks.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data[0].name", is("citation")) + .body("data[1].name", is("geospatial")) + .body("data[0].fields.size()", is(80)) + .body("data[1].fields.size()", is(11)); + + System.out.println("listing " + dataverseAlias + " collection blocks and inner dataset field types, with display on create and return dataset field types set to true using dataset type " + randomName); + listBlocks = UtilIT.listMetadataBlocks(dataverseAlias, true, true, randomName, apiToken); + listBlocks.prettyPrint(); + listBlocks.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data[0].name", is("citation")) + .body("data[1].name", is("geospatial")) + .body("data[0].fields.size()", is(28)) + .body("data[1].fields.size()", is(0)); // There are no fields required or with displayOnCreate=true in geospatial.tsv + // We send an empty array to mean "delete or clear all" String emptyJsonArray = "[]"; Response removeDatasetTypeLinks = UtilIT.updateDatasetTypeLinksWithMetadataBlocks(randomName, emptyJsonArray, apiToken); From 399fe8d2e827ef1308b023cfc4eabc22dc0b974e Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 5 Nov 2024 16:35:47 -0500 Subject: [PATCH 03/17] add missing "Command" suffix #10519 --- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 2 +- ...ava => UpdateDatasetTypeLinksToMetadataBlocksCommand.java} | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) rename src/main/java/edu/harvard/iq/dataverse/engine/command/impl/{UpdateDatasetTypeLinksToMetadataBlocks.java => UpdateDatasetTypeLinksToMetadataBlocksCommand.java} (85%) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index ea58a670627..b3506a07482 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -5264,7 +5264,7 @@ public Response updateDatasetTypeLinksWithMetadataBlocks(@Context ContainerReque } } try { - execCommand(new UpdateDatasetTypeLinksToMetadataBlocks(createDataverseRequest(getRequestUser(crc)), datasetType, metadataBlocksToSave)); + execCommand(new UpdateDatasetTypeLinksToMetadataBlocksCommand(createDataverseRequest(getRequestUser(crc)), datasetType, metadataBlocksToSave)); return ok(Json.createObjectBuilder() .add("linkedMetadataBlocks", Json.createObjectBuilder() .add("before", datasetTypesBefore) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetTypeLinksToMetadataBlocks.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetTypeLinksToMetadataBlocksCommand.java similarity index 85% rename from src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetTypeLinksToMetadataBlocks.java rename to src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetTypeLinksToMetadataBlocksCommand.java index 476935facd7..57b6da3f90c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetTypeLinksToMetadataBlocks.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetTypeLinksToMetadataBlocksCommand.java @@ -13,12 +13,12 @@ import java.util.List; @RequiredPermissions({}) -public class UpdateDatasetTypeLinksToMetadataBlocks extends AbstractVoidCommand { +public class UpdateDatasetTypeLinksToMetadataBlocksCommand extends AbstractVoidCommand { final DatasetType datasetType; List metadataBlocks; - public UpdateDatasetTypeLinksToMetadataBlocks(DataverseRequest dataverseRequest, DatasetType datasetType, List metadataBlocks) { + public UpdateDatasetTypeLinksToMetadataBlocksCommand(DataverseRequest dataverseRequest, DatasetType datasetType, List metadataBlocks) { super(dataverseRequest, (DvObject) null); this.datasetType = datasetType; this.metadataBlocks = metadataBlocks; From be83ada2d23f4bb560d95ff382283cd566b6fcb0 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 5 Nov 2024 16:37:51 -0500 Subject: [PATCH 04/17] remove debug line #10519 --- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index b3506a07482..80ee827b67a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -5252,7 +5252,6 @@ public Response updateDatasetTypeLinksWithMetadataBlocks(@Context ContainerReque JsonArray json = JsonUtil.getJsonArray(jsonBody); for (JsonString jsonValue : json.getValuesAs(JsonString.class)) { String name = jsonValue.getString(); - System.out.println("name: " + name); MetadataBlock metadataBlock = metadataBlockSvc.findByName(name); if (metadataBlock != null) { metadataBlocksToSave.add(metadataBlock); From a7eee45dff7b1cbc8cb63eebc7d27ac38d6ce55d Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 5 Nov 2024 16:50:30 -0500 Subject: [PATCH 05/17] improve docs #10519 --- doc/sphinx-guides/source/api/native-api.rst | 44 ++++++++++++------- .../source/user/dataset-management.rst | 6 ++- 2 files changed, 31 insertions(+), 19 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 54325a711e1..64a0f6aec2d 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -519,6 +519,8 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/dataverses/root/assignments/6" +.. _list-metadata-blocks-for-a-collection: + List Metadata Blocks Defined on a Dataverse Collection ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -566,6 +568,8 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/dataverses/root/metadatablocks?returnDatasetFieldTypes=true&onlyDisplayedOnCreate=true" +.. _define-metadata-blocks-for-a-dataverse-collection: + Define Metadata Blocks for a Dataverse Collection ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -591,6 +595,8 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST -H "Content-type:application/json" --upload-file define-metadatablocks.json "https://demo.dataverse.org/api/dataverses/root/metadatablocks" +An alternative to defining metadata blocks at a collection level is to create and use a dataset type. See :ref:`api-link-dataset-type`. + Determine if a Dataverse Collection Inherits Its Metadata Blocks from Its Parent ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -3196,52 +3202,56 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -H "Content-Type: application/json" "https://demo.dataverse.org/api/datasets/datasetTypes" -X POST -d '{"name": "software"}' -.. _api-link-dataset-type: +.. _api-delete-dataset-type: -Link Dataset Type with Metadata Blocks -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Delete Dataset Type +^^^^^^^^^^^^^^^^^^^ -This API endpoint is superuser only. +Superuser only. .. code-block:: bash export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx export SERVER_URL=https://demo.dataverse.org - export TYPE=software - export JSON='["codeMeta20"]' + export TYPE_ID=3 - curl -H "X-Dataverse-key:$API_TOKEN" -H "Content-Type: application/json" "$SERVER_URL/api/datasets/datasetTypes/$TYPE" -X POST -d $JSON + curl -H "X-Dataverse-key:$API_TOKEN" -X DELETE "$SERVER_URL/api/datasets/datasetTypes/$TYPE_ID" The fully expanded example above (without environment variables) looks like this: .. code-block:: bash - curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -H "Content-Type: application/json" "https://demo.dataverse.org/api/datasets/datasetTypes/software" -X POST -d '["codeMeta20"]' + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/datasets/datasetTypes/3" -To update the blocks that are link, send an array with those blocks. +.. _api-link-dataset-type: -To remove all links to blocks, send an empty array. +Link Dataset Type with Metadata Blocks +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. _api-delete-dataset-type: +Linking a dataset type with one or more metadata blocks results in additional fields from those blocks appearing in the output from the :ref:`list-metadata-blocks-for-a-collection` API endpoint. The new frontend for Dataverse (https://github.com/IQSS/dataverse-frontend) uses the JSON output from this API endpoint to construct the page that users see when creating or editing a dataset. Once the frontend has been updated to pass in the dataset type (https://github.com/IQSS/dataverse-client-javascript/issues/210), specifying a dataset type in this way can be an alternative way to display additional metadata fields than the traditional method, which is to enabled a metadata block at the collection level (see :ref:`define-metadata-blocks-for-a-dataverse-collection`). -Delete Dataset Type -^^^^^^^^^^^^^^^^^^^ +For example, a superuser could create a type called "software" and link it to the "CodeMeta" metadata block (this example is below). Then, once the new front end allows it, the user can specify that they want to create a dataset of type software and see the additional metadata fields from the CodeMeta block when creating or editing their dataset. -Superuser only. +This API endpoint is superuser only. .. code-block:: bash export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx export SERVER_URL=https://demo.dataverse.org - export TYPE_ID=3 + export TYPE=software + export JSON='["codeMeta20"]' - curl -H "X-Dataverse-key:$API_TOKEN" -X DELETE "$SERVER_URL/api/datasets/datasetTypes/$TYPE_ID" + curl -H "X-Dataverse-key:$API_TOKEN" -H "Content-Type: application/json" "$SERVER_URL/api/datasets/datasetTypes/$TYPE" -X POST -d $JSON The fully expanded example above (without environment variables) looks like this: .. code-block:: bash - curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/datasets/datasetTypes/3" + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -H "Content-Type: application/json" "https://demo.dataverse.org/api/datasets/datasetTypes/software" -X POST -d '["codeMeta20"]' + +To update the blocks that are linked, send an array with those blocks. + +To remove all links to blocks, send an empty array. Files ----- diff --git a/doc/sphinx-guides/source/user/dataset-management.rst b/doc/sphinx-guides/source/user/dataset-management.rst index 2e5d84748a8..a2a542cf315 100755 --- a/doc/sphinx-guides/source/user/dataset-management.rst +++ b/doc/sphinx-guides/source/user/dataset-management.rst @@ -790,13 +790,15 @@ If you deaccession the most recently published version of the dataset but not al Dataset Types ============= +.. note:: Development of the dataset types feature is ongoing. Please see https://github.com/IQSS/dataverse-pm/issues/307 for details. + Out of the box, all datasets have a dataset type of "dataset". Superusers can add additional types such as "software" or "workflow" using the :ref:`api-add-dataset-type` API endpoint. Once more than one type appears in search results, a facet called "Dataset Type" will appear allowing you to filter down to a certain type. If your installation is configured to use DataCite as a persistent ID (PID) provider, the appropriate type ("Dataset", "Software", "Workflow") will be sent to DataCite when the dataset is published for those three types. -Currently, the dataset type can only be specified via API and only when the dataset is created. For details, see the following sections of the API guide: +Currently, specifying a type for a dataset can only be done via API and only when the dataset is created. The type can't currently be changed afterward. For details, see the following sections of the API guide: - :ref:`api-create-dataset-with-type` (Native API) - :ref:`api-semantic-create-dataset-with-type` (Semantic API) @@ -804,7 +806,7 @@ Currently, the dataset type can only be specified via API and only when the data Dataset types can be listed, added, or deleted via API. See :ref:`api-dataset-types` in the API Guide for more. -Development of the dataset types feature is ongoing. Please see https://github.com/IQSS/dataverse/issues/10489 for details. +Dataset types can be linked with metadata blocks to make fields from those blocks available when datasets of that type are created or edited. See :ref:`api-link-dataset-type` for details. .. |image1| image:: ./img/DatasetDiagram.png :class: img-responsive From 4c45fc0a9732ef77c83c54a117cd705b7b31f333 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 5 Nov 2024 17:04:18 -0500 Subject: [PATCH 06/17] rename "ownerDataverse" to "dataverse", remove TODOs #10519 --- .../iq/dataverse/util/json/JsonPrinter.java | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index 3509972855a..156e5885978 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -599,11 +599,10 @@ public static JsonArrayBuilder json(List metadataBlocks, boolean return json(metadataBlocks, returnDatasetFieldTypes, printOnlyDisplayedOnCreateDatasetFieldTypes, null, null); } - // TODO: consider renaming "ownerDataverse" to just "dataverse" - public static JsonArrayBuilder json(List metadataBlocks, boolean returnDatasetFieldTypes, boolean printOnlyDisplayedOnCreateDatasetFieldTypes, Dataverse ownerDataverse, DatasetType datasetType) { + public static JsonArrayBuilder json(List metadataBlocks, boolean returnDatasetFieldTypes, boolean printOnlyDisplayedOnCreateDatasetFieldTypes, Dataverse dataverse, DatasetType datasetType) { JsonArrayBuilder arrayBuilder = Json.createArrayBuilder(); for (MetadataBlock metadataBlock : metadataBlocks) { - arrayBuilder.add(returnDatasetFieldTypes ? json(metadataBlock, printOnlyDisplayedOnCreateDatasetFieldTypes, ownerDataverse, datasetType) : brief.json(metadataBlock)); + arrayBuilder.add(returnDatasetFieldTypes ? json(metadataBlock, printOnlyDisplayedOnCreateDatasetFieldTypes, dataverse, datasetType) : brief.json(metadataBlock)); } return arrayBuilder; } @@ -634,8 +633,7 @@ public static JsonObjectBuilder json(MetadataBlock metadataBlock) { return json(metadataBlock, false, null, null); } - // TODO: consider renaming "ownerDataverse" to just "dataverse" - public static JsonObjectBuilder json(MetadataBlock metadataBlock, boolean printOnlyDisplayedOnCreateDatasetFieldTypes, Dataverse ownerDataverse, DatasetType datasetType) { + public static JsonObjectBuilder json(MetadataBlock metadataBlock, boolean printOnlyDisplayedOnCreateDatasetFieldTypes, Dataverse dataverse, DatasetType datasetType) { JsonObjectBuilder jsonObjectBuilder = jsonObjectBuilder() .add("id", metadataBlock.getId()) .add("name", metadataBlock.getName()) @@ -644,9 +642,9 @@ public static JsonObjectBuilder json(MetadataBlock metadataBlock, boolean printO Set datasetFieldTypes; - if (ownerDataverse != null) { + if (dataverse != null) { datasetFieldTypes = new TreeSet<>(datasetFieldService.findAllInMetadataBlockAndDataverse( - metadataBlock, ownerDataverse, printOnlyDisplayedOnCreateDatasetFieldTypes, datasetType)); + metadataBlock, dataverse, printOnlyDisplayedOnCreateDatasetFieldTypes, datasetType)); } else { datasetFieldTypes = printOnlyDisplayedOnCreateDatasetFieldTypes ? new TreeSet<>(datasetFieldService.findAllDisplayedOnCreateInMetadataBlock(metadataBlock)) @@ -655,7 +653,7 @@ public static JsonObjectBuilder json(MetadataBlock metadataBlock, boolean printO JsonObjectBuilder fieldsBuilder = Json.createObjectBuilder(); for (DatasetFieldType datasetFieldType : datasetFieldTypes) { - fieldsBuilder.add(datasetFieldType.getName(), json(datasetFieldType, ownerDataverse)); + fieldsBuilder.add(datasetFieldType.getName(), json(datasetFieldType, dataverse)); } jsonObjectBuilder.add("fields", fieldsBuilder); From 4f77a3b040e160cbbbfc3f7bd7bbec2ec63c65b5 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 6 Nov 2024 09:02:40 -0500 Subject: [PATCH 07/17] enable codeMeta20 test, disable size=6 test #10519 --- .../edu/harvard/iq/dataverse/api/DatasetTypesIT.java | 8 -------- .../harvard/iq/dataverse/api/MetadataBlocksIT.java | 11 ++++++++--- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java index aa552c3ed79..6e211884a9d 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java @@ -18,7 +18,6 @@ import static org.hamcrest.CoreMatchers.nullValue; import static org.junit.jupiter.api.Assertions.assertEquals; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; public class DatasetTypesIT { @@ -408,13 +407,6 @@ public void testUpdateDatasetTypeLinksWithMetadataBlocks() { .body("data[0].name", is("citation")); } - /** - * This test is disabled because loads the "codeMeta20" metadata block. - * - * Loading any additional block causes this test to fail: - * MetadataBlocksIT#testListMetadataBlocks - */ - @Disabled @Test public void testLinkSoftwareToCodemeta() { Response listBlocksAvailable = UtilIT.listMetadataBlocks(false, false); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/MetadataBlocksIT.java b/src/test/java/edu/harvard/iq/dataverse/api/MetadataBlocksIT.java index 072cb09050c..c20008a4bf0 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/MetadataBlocksIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/MetadataBlocksIT.java @@ -13,6 +13,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assumptions.assumeFalse; import static org.junit.jupiter.api.Assumptions.assumeTrue; +import org.junit.jupiter.api.Disabled; public class MetadataBlocksIT { @@ -21,13 +22,17 @@ public static void setUpClass() { RestAssured.baseURI = UtilIT.getRestAssuredBaseUri(); } + /** + * TODO: re-enable this test after deciding how to allow additional metadata + * blocks to be added by other tests. We load the "codeMeta20" block in + * DatasetTypesIT#testLinkSoftwareToCodemeta but it causes this test to + * break. It asserts there are only 6 blocks, for example. + */ + @Disabled @Test void testListMetadataBlocks() { // No optional params enabled Response listMetadataBlocksResponse = UtilIT.listMetadataBlocks(false, false); - // TODO: consider re-writing this test to allow additional metadata blocks - // to be added by other tests. We'd like to load the "codeMeta20" block in - // DatasetTypesIT#testLinkSoftwareToCodemeta but it causes this test to break. int expectedDefaultNumberOfMetadataBlocks = 6; listMetadataBlocksResponse.then().assertThat() .statusCode(OK.getStatusCode()) From e7652dbc4ee743a906b961189cc4e497555f0598 Mon Sep 17 00:00:00 2001 From: GPortas Date: Thu, 7 Nov 2024 08:46:06 +0000 Subject: [PATCH 08/17] Fixed: removed distinct modifier in JPA query to avoid ignoring results --- .../java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java index a9752bc0596..210cf383378 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java @@ -918,7 +918,7 @@ public List findAllInMetadataBlockAndDataverse(MetadataBlock m ) ); - criteriaQuery.select(datasetFieldTypeRoot).distinct(true); + criteriaQuery.select(datasetFieldTypeRoot); return em.createQuery(criteriaQuery).getResultList(); } From 5afad8e6ac390461d42603f56efa583e70d87421 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 8 Nov 2024 15:23:09 -0500 Subject: [PATCH 09/17] load codemeta metadata block in docker dev persona #10519 It's used in API tests. --- doc/release-notes/10519-dataset-types.md | 2 ++ docker-compose-dev.yml | 2 ++ .../scripts/bootstrap/dev/init.sh | 9 +++++++++ .../iq/dataverse/api/DatasetTypesIT.java | 18 ------------------ .../iq/dataverse/api/MetadataBlocksIT.java | 13 +++---------- 5 files changed, 16 insertions(+), 28 deletions(-) diff --git a/doc/release-notes/10519-dataset-types.md b/doc/release-notes/10519-dataset-types.md index e4200e1a3fc..1e1ce08ced3 100644 --- a/doc/release-notes/10519-dataset-types.md +++ b/doc/release-notes/10519-dataset-types.md @@ -7,4 +7,6 @@ This will have the following effects for the APIs used by the new Dataverse UI ( - The list of fields shown when creating a dataset will include fields marked as "displayoncreate" (in the tsv/database) for metadata blocks (e.g. "CodeMeta") that are linked to the dataset type (e.g. "software") that is passed to the API. - The metadata blocks shown when editing a dataset will include metadata blocks (e.g. "CodeMeta") that are linked to the dataset type (e.g. "software") that is passed to the API. +The CodeMeta metadata block is now available in the Dockerized development environment. + For more information, see the guides and #10519. diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 384b70b7a7b..b178344c200 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -89,6 +89,8 @@ services: - dev networks: - dataverse + volumes: + - ./docker-dev-volumes/solr/data:/var/solr dev_dv_initializer: container_name: "dev_dv_initializer" diff --git a/modules/container-configbaker/scripts/bootstrap/dev/init.sh b/modules/container-configbaker/scripts/bootstrap/dev/init.sh index f8770436652..f70cc099e2d 100644 --- a/modules/container-configbaker/scripts/bootstrap/dev/init.sh +++ b/modules/container-configbaker/scripts/bootstrap/dev/init.sh @@ -17,6 +17,15 @@ export API_TOKEN # ${ENV_OUT} comes from bootstrap.sh and will expose the saved information back to the host if enabled. echo "API_TOKEN=${API_TOKEN}" >> "${ENV_OUT}" +echo "Loading CodeMeta metadata block (needed for API tests)..." +curl "${DATAVERSE_URL}/api/admin/datasetfield/load" -X POST --data-binary @/scripts/bootstrap/base/data/metadatablocks/codemeta.tsv -H "Content-type: text/tab-separated-values" + +echo "Fetching Solr schema from Dataverse and running update-fields.sh..." +curl "${DATAVERSE_URL}/api/admin/index/solr/schema" | /scripts/update-fields.sh /var/solr/data/collection1/conf/schema.xml + +echo "Reloading Solr..." +curl "http://solr:8983/solr/admin/cores?action=RELOAD&core=collection1" + echo "Publishing root dataverse..." curl -H "X-Dataverse-key:$API_TOKEN" -X POST "${DATAVERSE_URL}/api/dataverses/:root/actions/:publish" diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java index 6e211884a9d..6cb5220500f 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java @@ -9,8 +9,6 @@ import static jakarta.ws.rs.core.Response.Status.CREATED; import static jakarta.ws.rs.core.Response.Status.FORBIDDEN; import static jakarta.ws.rs.core.Response.Status.OK; -import java.io.IOException; -import java.nio.file.Paths; import java.util.UUID; import org.hamcrest.CoreMatchers; import static org.hamcrest.CoreMatchers.equalTo; @@ -409,22 +407,6 @@ public void testUpdateDatasetTypeLinksWithMetadataBlocks() { @Test public void testLinkSoftwareToCodemeta() { - Response listBlocksAvailable = UtilIT.listMetadataBlocks(false, false); - listBlocksAvailable.then().assertThat().statusCode(OK.getStatusCode()); - String blocksAsString = JsonPath.from(listBlocksAvailable.getBody().asString()).getString("data"); - System.out.println("blocks: " + blocksAsString); - if (!blocksAsString.contains("codeMeta20")) { - System.out.println("CodeMeta hasn't been added. Adding it..."); - byte[] codemetaTsv = null; - try { - codemetaTsv = java.nio.file.Files.readAllBytes(Paths.get("scripts/api/data/metadatablocks/codemeta.tsv")); - } catch (IOException e) { - } - UtilIT.loadMetadataBlock("", codemetaTsv); - } else { - System.out.println("CodeMeta has already been added."); - } - Response createUser = UtilIT.createRandomUser(); createUser.then().assertThat().statusCode(OK.getStatusCode()); String username = UtilIT.getUsernameFromResponse(createUser); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/MetadataBlocksIT.java b/src/test/java/edu/harvard/iq/dataverse/api/MetadataBlocksIT.java index c20008a4bf0..40502708e2f 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/MetadataBlocksIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/MetadataBlocksIT.java @@ -13,7 +13,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assumptions.assumeFalse; import static org.junit.jupiter.api.Assumptions.assumeTrue; -import org.junit.jupiter.api.Disabled; public class MetadataBlocksIT { @@ -22,18 +21,11 @@ public static void setUpClass() { RestAssured.baseURI = UtilIT.getRestAssuredBaseUri(); } - /** - * TODO: re-enable this test after deciding how to allow additional metadata - * blocks to be added by other tests. We load the "codeMeta20" block in - * DatasetTypesIT#testLinkSoftwareToCodemeta but it causes this test to - * break. It asserts there are only 6 blocks, for example. - */ - @Disabled @Test void testListMetadataBlocks() { // No optional params enabled Response listMetadataBlocksResponse = UtilIT.listMetadataBlocks(false, false); - int expectedDefaultNumberOfMetadataBlocks = 6; + int expectedDefaultNumberOfMetadataBlocks = 7; listMetadataBlocksResponse.then().assertThat() .statusCode(OK.getStatusCode()) .body("data[0].fields", equalTo(null)) @@ -41,11 +33,12 @@ void testListMetadataBlocks() { // onlyDisplayedOnCreate=true listMetadataBlocksResponse = UtilIT.listMetadataBlocks(true, false); - int expectedOnlyDisplayedOnCreateNumberOfMetadataBlocks = 1; + int expectedOnlyDisplayedOnCreateNumberOfMetadataBlocks = 2; listMetadataBlocksResponse.then().assertThat() .statusCode(OK.getStatusCode()) .body("data[0].fields", equalTo(null)) .body("data[0].displayName", equalTo("Citation Metadata")) + .body("data[1].displayName", equalTo("Software Metadata (CodeMeta v2.0)")) .body("data.size()", equalTo(expectedOnlyDisplayedOnCreateNumberOfMetadataBlocks)); // returnDatasetFieldTypes=true From a05473784cd57d3a396a2f8e059277f1100f5f15 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Tue, 21 Jan 2025 11:37:57 -0500 Subject: [PATCH 10/17] fix merge conflicts --- .../edu/harvard/iq/dataverse/util/json/JsonPrinter.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index 0a9e87c69a6..dee115d15c2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -643,7 +643,7 @@ public static JsonObjectBuilder json(MetadataBlock metadataBlock) { return json(metadataBlock, false, null, null); } - public static JsonObjectBuilder json(MetadataBlock metadataBlock, boolean printOnlyDisplayedOnCreateDatasetFieldTypes, Dataverse ownerDataverse, DatasetType datasetType) { + public static JsonObjectBuilder json(MetadataBlock metadataBlock, boolean printOnlyDisplayedOnCreateDatasetFieldTypes, Dataverse dataverse, DatasetType datasetType) { JsonObjectBuilder jsonObjectBuilder = jsonObjectBuilder() .add("id", metadataBlock.getId()) .add("name", metadataBlock.getName()) @@ -652,9 +652,9 @@ public static JsonObjectBuilder json(MetadataBlock metadataBlock, boolean printO List datasetFieldTypesList; - if (ownerDataverse != null) { + if (dataverse != null) { datasetFieldTypesList = datasetFieldService.findAllInMetadataBlockAndDataverse( - metadataBlock, ownerDataverse, printOnlyDisplayedOnCreateDatasetFieldTypes, datasetType); + metadataBlock, dataverse, printOnlyDisplayedOnCreateDatasetFieldTypes, datasetType); } else { datasetFieldTypesList = printOnlyDisplayedOnCreateDatasetFieldTypes ? datasetFieldService.findAllDisplayedOnCreateInMetadataBlock(metadataBlock) @@ -665,7 +665,7 @@ public static JsonObjectBuilder json(MetadataBlock metadataBlock, boolean printO JsonObjectBuilder fieldsBuilder = Json.createObjectBuilder(); for (DatasetFieldType datasetFieldType : datasetFieldTypes) { - fieldsBuilder.add(datasetFieldType.getName(), json(datasetFieldType, ownerDataverse)); + fieldsBuilder.add(datasetFieldType.getName(), json(datasetFieldType, dataverse)); } jsonObjectBuilder.add("fields", fieldsBuilder); From faf1d37e6e16b1363a8a6e1d780e5a8f292753d3 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 23 Jan 2025 14:22:43 -0500 Subject: [PATCH 11/17] fix JsonPrinter and failing tests #10519 JsonPrinter was edited in #10764 and #11066 which were merged five and two weeks ago, respectively. This caused merge conflicts which should be resolved correctly now. In this commit, we are starting with JsonPrinter as of the most recent "develop" branch commit (7fdb21a) and adding in our DatasetType changes. PR #10764 resulted in the number of metadata fields being reduced from 80 to 35 and this commit make the same change in an assertion. --- .../iq/dataverse/util/json/JsonPrinter.java | 37 +++++++++++++++---- .../iq/dataverse/api/DatasetTypesIT.java | 6 +-- 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index dee115d15c2..7929021d2b8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -56,6 +56,7 @@ import jakarta.ejb.Singleton; import jakarta.json.JsonArray; import jakarta.json.JsonObject; +import java.util.function.Predicate; /** * Convert objects to Json. @@ -609,10 +610,10 @@ public static JsonArrayBuilder json(List metadataBlocks, boolean return json(metadataBlocks, returnDatasetFieldTypes, printOnlyDisplayedOnCreateDatasetFieldTypes, null, null); } - public static JsonArrayBuilder json(List metadataBlocks, boolean returnDatasetFieldTypes, boolean printOnlyDisplayedOnCreateDatasetFieldTypes, Dataverse dataverse, DatasetType datasetType) { + public static JsonArrayBuilder json(List metadataBlocks, boolean returnDatasetFieldTypes, boolean printOnlyDisplayedOnCreateDatasetFieldTypes, Dataverse ownerDataverse, DatasetType datasetType) { JsonArrayBuilder arrayBuilder = Json.createArrayBuilder(); for (MetadataBlock metadataBlock : metadataBlocks) { - arrayBuilder.add(returnDatasetFieldTypes ? json(metadataBlock, printOnlyDisplayedOnCreateDatasetFieldTypes, dataverse, datasetType) : brief.json(metadataBlock)); + arrayBuilder.add(returnDatasetFieldTypes ? json(metadataBlock, printOnlyDisplayedOnCreateDatasetFieldTypes, ownerDataverse, datasetType) : brief.json(metadataBlock)); } return arrayBuilder; } @@ -643,7 +644,7 @@ public static JsonObjectBuilder json(MetadataBlock metadataBlock) { return json(metadataBlock, false, null, null); } - public static JsonObjectBuilder json(MetadataBlock metadataBlock, boolean printOnlyDisplayedOnCreateDatasetFieldTypes, Dataverse dataverse, DatasetType datasetType) { + public static JsonObjectBuilder json(MetadataBlock metadataBlock, boolean printOnlyDisplayedOnCreateDatasetFieldTypes, Dataverse ownerDataverse, DatasetType datasetType) { JsonObjectBuilder jsonObjectBuilder = jsonObjectBuilder() .add("id", metadataBlock.getId()) .add("name", metadataBlock.getName()) @@ -652,9 +653,9 @@ public static JsonObjectBuilder json(MetadataBlock metadataBlock, boolean printO List datasetFieldTypesList; - if (dataverse != null) { + if (ownerDataverse != null) { datasetFieldTypesList = datasetFieldService.findAllInMetadataBlockAndDataverse( - metadataBlock, dataverse, printOnlyDisplayedOnCreateDatasetFieldTypes, datasetType); + metadataBlock, ownerDataverse, printOnlyDisplayedOnCreateDatasetFieldTypes, datasetType); } else { datasetFieldTypesList = printOnlyDisplayedOnCreateDatasetFieldTypes ? datasetFieldService.findAllDisplayedOnCreateInMetadataBlock(metadataBlock) @@ -664,10 +665,30 @@ public static JsonObjectBuilder json(MetadataBlock metadataBlock, boolean printO Set datasetFieldTypes = filterOutDuplicateDatasetFieldTypes(datasetFieldTypesList); JsonObjectBuilder fieldsBuilder = Json.createObjectBuilder(); - for (DatasetFieldType datasetFieldType : datasetFieldTypes) { - fieldsBuilder.add(datasetFieldType.getName(), json(datasetFieldType, dataverse)); + + Predicate isNoChild = element -> element.isChild() == false; + List childLessList = metadataBlock.getDatasetFieldTypes().stream().filter(isNoChild).toList(); + Set datasetFieldTypesNoChildSorted = new TreeSet<>(childLessList); + + for (DatasetFieldType datasetFieldType : datasetFieldTypesNoChildSorted) { + + Long datasetFieldTypeId = datasetFieldType.getId(); + boolean requiredAsInputLevelInOwnerDataverse = ownerDataverse != null && ownerDataverse.isDatasetFieldTypeRequiredAsInputLevel(datasetFieldTypeId); + boolean includedAsInputLevelInOwnerDataverse = ownerDataverse != null && ownerDataverse.isDatasetFieldTypeIncludedAsInputLevel(datasetFieldTypeId); + boolean isNotInputLevelInOwnerDataverse = ownerDataverse != null && !ownerDataverse.isDatasetFieldTypeInInputLevels(datasetFieldTypeId); + + DatasetFieldType parentDatasetFieldType = datasetFieldType.getParentDatasetFieldType(); + boolean isRequired = parentDatasetFieldType == null ? datasetFieldType.isRequired() : parentDatasetFieldType.isRequired(); + + boolean displayCondition = printOnlyDisplayedOnCreateDatasetFieldTypes + ? (datasetFieldType.isDisplayOnCreate() || isRequired || requiredAsInputLevelInOwnerDataverse) + : ownerDataverse == null || includedAsInputLevelInOwnerDataverse || isNotInputLevelInOwnerDataverse; + + if (displayCondition) { + fieldsBuilder.add(datasetFieldType.getName(), json(datasetFieldType, ownerDataverse)); + } } - + jsonObjectBuilder.add("fields", fieldsBuilder); return jsonObjectBuilder; } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java index 6cb5220500f..83730263978 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java @@ -378,8 +378,8 @@ public void testUpdateDatasetTypeLinksWithMetadataBlocks() { .statusCode(OK.getStatusCode()) .body("data[0].name", is("citation")) .body("data[1].name", is("geospatial")) - .body("data[0].fields.size()", is(80)) - .body("data[1].fields.size()", is(11)); + .body("data[0].fields.size()", is(35)) + .body("data[1].fields.size()", is(3)); System.out.println("listing " + dataverseAlias + " collection blocks and inner dataset field types, with display on create and return dataset field types set to true using dataset type " + randomName); listBlocks = UtilIT.listMetadataBlocks(dataverseAlias, true, true, randomName, apiToken); @@ -388,7 +388,7 @@ public void testUpdateDatasetTypeLinksWithMetadataBlocks() { .statusCode(OK.getStatusCode()) .body("data[0].name", is("citation")) .body("data[1].name", is("geospatial")) - .body("data[0].fields.size()", is(28)) + .body("data[0].fields.size()", is(10)) .body("data[1].fields.size()", is(0)); // There are no fields required or with displayOnCreate=true in geospatial.tsv // We send an empty array to mean "delete or clear all" From 2773c2e757410369b6336a31c647c94ca5f9157c Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 23 Jan 2025 14:23:29 -0500 Subject: [PATCH 12/17] cleanup #10519 --- .../java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java | 4 ---- src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java | 1 - 2 files changed, 5 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java index 83730263978..c225ea3085a 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java @@ -307,10 +307,6 @@ public void testUpdateDatasetTypeLinksWithMetadataBlocks() { getTypeById.prettyPrint(); getTypeById.then().assertThat().statusCode(OK.getStatusCode()); - String updateToTheseTypes = Json.createArrayBuilder() - .add(randomName) - .build().toString(); - String metadataBlockToLink = """ ["geospatial"] """; diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index c90595685c3..baef1b223d9 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -4364,7 +4364,6 @@ static Response getDatasetType(String idOrName) { } static Response addDatasetType(String jsonIn, String apiToken) { - System.out.println("called addDatasetType..."); return given() .header(API_TOKEN_HTTP_HEADER, apiToken) .body(jsonIn) From e52052a3771314fd382710c6872ce30e7e13beb5 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 23 Jan 2025 14:35:52 -0500 Subject: [PATCH 13/17] remove unnecessary sending of content type #10519 --- src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index baef1b223d9..55baa965314 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -4435,8 +4435,6 @@ static Response updateDatasetTypeLinksWithMetadataBlocks(String idOrName, String return given() .header(API_TOKEN_HTTP_HEADER, apiToken) .body(jsonArrayOfMetadataBlocks) - // Do we need to send content type = json? - .contentType(ContentType.JSON) .put("/api/datasets/datasetTypes/" + idOrName); } } From 57e0c712321de3248260b63be275276f0d18b955 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 23 Jan 2025 14:55:19 -0500 Subject: [PATCH 14/17] clarify docs #10519 --- doc/sphinx-guides/source/api/native-api.rst | 9 ++++----- doc/sphinx-guides/source/user/dataset-management.rst | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 48014d7068d..3042afbf2fa 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -569,9 +569,7 @@ This endpoint supports the following optional query parameters: - ``returnDatasetFieldTypes``: Whether or not to return the dataset field types present in each metadata block. If not set, the default value is false. - ``onlyDisplayedOnCreate``: Whether or not to return only the metadata blocks that are displayed on dataset creation. If ``returnDatasetFieldTypes`` is true, only the dataset field types shown on dataset creation will be returned within each metadata block. If not set, the default value is false. -- ``datasetType``: Whether or not to return additional fields from metadata blocks that are linked with a particular dataset type. - -are displayed on dataset creation. If ``returnDatasetFieldTypes`` is true, only the dataset field types shown on dataset creation will be returned within each metadata block. If not set, the default value is false. +- ``datasetType``: Optionally return additional fields from metadata blocks that are linked with a particular dataset type (see :ref:`dataset-types` in the User Guide). Pass a single dataset type as a string. For a list of dataset types you can pass, see :ref:`api-list-dataset-types`. An example using the optional query parameters is presented below: @@ -580,14 +578,15 @@ An example using the optional query parameters is presented below: export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx export SERVER_URL=https://demo.dataverse.org export ID=root + export DATASET_TYPE=software - curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/dataverses/$ID/metadatablocks?returnDatasetFieldTypes=true&onlyDisplayedOnCreate=true" + curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/dataverses/$ID/metadatablocks?returnDatasetFieldTypes=true&onlyDisplayedOnCreate=true&datasetType=$DATASET_TYPE" The fully expanded example above (without environment variables) looks like this: .. code-block:: bash - curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/dataverses/root/metadatablocks?returnDatasetFieldTypes=true&onlyDisplayedOnCreate=true" + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/dataverses/root/metadatablocks?returnDatasetFieldTypes=true&onlyDisplayedOnCreate=true&datasetType=software" .. _define-metadata-blocks-for-a-dataverse-collection: diff --git a/doc/sphinx-guides/source/user/dataset-management.rst b/doc/sphinx-guides/source/user/dataset-management.rst index 0cd57c8da32..04b09321490 100755 --- a/doc/sphinx-guides/source/user/dataset-management.rst +++ b/doc/sphinx-guides/source/user/dataset-management.rst @@ -808,7 +808,7 @@ Currently, specifying a type for a dataset can only be done via API and only whe Dataset types can be listed, added, or deleted via API. See :ref:`api-dataset-types` in the API Guide for more. -Dataset types can be linked with metadata blocks to make fields from those blocks available when datasets of that type are created or edited. See :ref:`api-link-dataset-type` for details. +Dataset types can be linked with metadata blocks to make fields from those blocks available when datasets of that type are created or edited. See :ref:`api-link-dataset-type` and :ref:`list-metadata-blocks-for-a-collection` for details. .. |image1| image:: ./img/DatasetDiagram.png :class: img-responsive From 9b2fb8d7338e7aec6c91e53efd6e40c896ec08b9 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 24 Jan 2025 11:20:37 -0500 Subject: [PATCH 15/17] stop loading Codemeta in Docker, add setDisplayOnCreate API #10519 In Docker, we were testing with the Codemeta block. This made for a nice real-world story of creating a "software" dataset type and using it with the Codemeta block. The Codemeta block also has the advantage of having some fields that are set to displayOnCreate, which helped us make assertions that the code is working properly. It's the only non-citation block with fields set to displayOnCreate. However, Jenkins doesn't have the Codemeta block, meaning that tests are failing. Also, we aren't ready to promote the Codemeta block to be shipped with Dataverse because a new version is out: https://github.com/IQSS/dataverse/issues/10859 So, we are switching from Codemeta to the Astrophysics block. We create an "instrument" dataset type. Like all non-citation blocks (except for Codemeta), there are no fields that are set to displayOnCreate=true. Therefore, we added an API for this. --- doc/release-notes/10519-dataset-types.md | 4 +- doc/sphinx-guides/source/api/native-api.rst | 21 ++++++ .../scripts/bootstrap/dev/init.sh | 9 --- .../dataverse/api/DatasetFieldServiceApi.java | 16 +++++ .../iq/dataverse/api/DatasetTypesIT.java | 68 +++++++++++-------- .../iq/dataverse/api/MetadataBlocksIT.java | 5 +- .../edu/harvard/iq/dataverse/api/UtilIT.java | 7 ++ 7 files changed, 86 insertions(+), 44 deletions(-) diff --git a/doc/release-notes/10519-dataset-types.md b/doc/release-notes/10519-dataset-types.md index 1e1ce08ced3..99cf79a796f 100644 --- a/doc/release-notes/10519-dataset-types.md +++ b/doc/release-notes/10519-dataset-types.md @@ -7,6 +7,6 @@ This will have the following effects for the APIs used by the new Dataverse UI ( - The list of fields shown when creating a dataset will include fields marked as "displayoncreate" (in the tsv/database) for metadata blocks (e.g. "CodeMeta") that are linked to the dataset type (e.g. "software") that is passed to the API. - The metadata blocks shown when editing a dataset will include metadata blocks (e.g. "CodeMeta") that are linked to the dataset type (e.g. "software") that is passed to the API. -The CodeMeta metadata block is now available in the Dockerized development environment. +Mostly in order to write automated tests for the above, a [displayOnCreate](https://dataverse-guide--11001.org.readthedocs.build/en/11001/api/native-api.html#set-displayoncreate-for-a-dataset-field) API endpoint has been added. -For more information, see the guides and #10519. +For more information, see the guides ([overview](https://dataverse-guide--11001.org.readthedocs.build/en/11001/user/dataset-management.html#dataset-types), [new APIs](https://dataverse-guide--11001.org.readthedocs.build/en/11001/api/native-api.html#link-dataset-type-with-metadata-blocks)), #10519 and #11001. diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 3042afbf2fa..f28187b965c 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -5090,6 +5090,27 @@ The fully expanded example above (without environment variables) looks like this curl "https://demo.dataverse.org/api/datasetfields/facetables" +.. _setDisplayOnCreate: + +Set displayOnCreate for a Dataset Field +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Set displayOnCreate for a dataset field. See also :doc:`/admin/metadatacustomization` in the Admin Guide. + +.. code-block:: bash + + export SERVER_URL=http://localhost:8080 + export FIELD=subtitle + export BOOLEAN=true + + curl -X POST "$SERVER_URL/api/admin/datasetfield/setDisplayOnCreate?datasetFieldType=$FIELD&setDisplayOnCreate=$BOOLEAN" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -X POST "http://localhost:8080/api/admin/datasetfield/setDisplayOnCreate?datasetFieldType=studyAssayCellType&setDisplayOnCreate=true" + .. _Notifications: Notifications diff --git a/modules/container-configbaker/scripts/bootstrap/dev/init.sh b/modules/container-configbaker/scripts/bootstrap/dev/init.sh index f70cc099e2d..f8770436652 100644 --- a/modules/container-configbaker/scripts/bootstrap/dev/init.sh +++ b/modules/container-configbaker/scripts/bootstrap/dev/init.sh @@ -17,15 +17,6 @@ export API_TOKEN # ${ENV_OUT} comes from bootstrap.sh and will expose the saved information back to the host if enabled. echo "API_TOKEN=${API_TOKEN}" >> "${ENV_OUT}" -echo "Loading CodeMeta metadata block (needed for API tests)..." -curl "${DATAVERSE_URL}/api/admin/datasetfield/load" -X POST --data-binary @/scripts/bootstrap/base/data/metadatablocks/codemeta.tsv -H "Content-type: text/tab-separated-values" - -echo "Fetching Solr schema from Dataverse and running update-fields.sh..." -curl "${DATAVERSE_URL}/api/admin/index/solr/schema" | /scripts/update-fields.sh /var/solr/data/collection1/conf/schema.xml - -echo "Reloading Solr..." -curl "http://solr:8983/solr/admin/cores?action=RELOAD&core=collection1" - echo "Publishing root dataverse..." curl -H "X-Dataverse-key:$API_TOKEN" -X POST "${DATAVERSE_URL}/api/dataverses/:root/actions/:publish" diff --git a/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java b/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java index 907295ad848..cbb0f4ffcfd 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java @@ -42,6 +42,7 @@ import java.util.logging.Logger; import jakarta.persistence.NoResultException; import jakarta.persistence.TypedQuery; +import jakarta.ws.rs.QueryParam; import jakarta.ws.rs.core.Response.Status; import java.io.BufferedInputStream; @@ -545,4 +546,19 @@ public static String getDataverseLangDirectory() { return dataverseLangDirectory; } + /** + * Set setDisplayOnCreate for a DatasetFieldType. + */ + @POST + @Path("/setDisplayOnCreate") + public Response setDisplayOnCreate(@QueryParam("datasetFieldType") String datasetFieldTypeIn, @QueryParam("setDisplayOnCreate") boolean setDisplayOnCreateIn) { + DatasetFieldType dft = datasetFieldService.findByName(datasetFieldTypeIn); + if (dft == null) { + return error(Status.NOT_FOUND, "Cound not find a DatasetFieldType by looking up " + datasetFieldTypeIn); + } + dft.setDisplayOnCreate(setDisplayOnCreateIn); + DatasetFieldType saved = datasetFieldService.save(dft); + return ok("DisplayOnCreate for DatasetFieldType " + saved.getName() + " is now " + saved.isDisplayOnCreate()); + } + } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java index c225ea3085a..7c73498dead 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java @@ -20,28 +20,32 @@ public class DatasetTypesIT { + final static String INSTRUMENT = "instrument"; + @BeforeAll public static void setUpClass() { RestAssured.baseURI = UtilIT.getRestAssuredBaseUri(); - Response getSoftwareType = UtilIT.getDatasetType(DatasetType.DATASET_TYPE_SOFTWARE); - getSoftwareType.prettyPrint(); - - String typeFound = JsonPath.from(getSoftwareType.getBody().asString()).getString("data.name"); - System.out.println("type found: " + typeFound); - if (DatasetType.DATASET_TYPE_SOFTWARE.equals(typeFound)) { - return; - } - - System.out.println("The \"software\" type wasn't found. Create it."); Response createUser = UtilIT.createRandomUser(); createUser.then().assertThat().statusCode(OK.getStatusCode()); String username = UtilIT.getUsernameFromResponse(createUser); String apiToken = UtilIT.getApiTokenFromResponse(createUser); UtilIT.setSuperuserStatus(username, true).then().assertThat().statusCode(OK.getStatusCode()); - String jsonIn = Json.createObjectBuilder().add("name", DatasetType.DATASET_TYPE_SOFTWARE).build().toString(); + ensureDatasetTypeIsPresent(DatasetType.DATASET_TYPE_SOFTWARE, apiToken); + ensureDatasetTypeIsPresent(INSTRUMENT, apiToken); + } + private static void ensureDatasetTypeIsPresent(String datasetType, String apiToken) { + Response getDatasetType = UtilIT.getDatasetType(datasetType); + getDatasetType.prettyPrint(); + String typeFound = JsonPath.from(getDatasetType.getBody().asString()).getString("data.name"); + System.out.println("type found: " + typeFound); + if (datasetType.equals(typeFound)) { + return; + } + System.out.println("The " + datasetType + "type wasn't found. Create it."); + String jsonIn = Json.createObjectBuilder().add("name", datasetType).build().toString(); Response typeAdded = UtilIT.addDatasetType(jsonIn, apiToken); typeAdded.prettyPrint(); typeAdded.then().assertThat().statusCode(OK.getStatusCode()); @@ -402,7 +406,7 @@ public void testUpdateDatasetTypeLinksWithMetadataBlocks() { } @Test - public void testLinkSoftwareToCodemeta() { + public void testLinkInstrumentToAstro() { Response createUser = UtilIT.createRandomUser(); createUser.then().assertThat().statusCode(OK.getStatusCode()); String username = UtilIT.getUsernameFromResponse(createUser); @@ -410,15 +414,15 @@ public void testLinkSoftwareToCodemeta() { UtilIT.setSuperuserStatus(username, true).then().assertThat().statusCode(OK.getStatusCode()); String metadataBlockLink = """ - ["codeMeta20"] + ["astrophysics"] //"""; - String datasetType = "software"; - Response linkSoftwareToCodemeta = UtilIT.updateDatasetTypeLinksWithMetadataBlocks(datasetType, metadataBlockLink, apiToken); - linkSoftwareToCodemeta.prettyPrint(); - linkSoftwareToCodemeta.then().assertThat(). + String datasetType = "instrument"; + Response linkInstrumentToAstro = UtilIT.updateDatasetTypeLinksWithMetadataBlocks(datasetType, metadataBlockLink, apiToken); + linkInstrumentToAstro.prettyPrint(); + linkInstrumentToAstro.then().assertThat(). statusCode(OK.getStatusCode()) - .body("data.linkedMetadataBlocks.after[0]", CoreMatchers.is("codeMeta20")); + .body("data.linkedMetadataBlocks.after[0]", CoreMatchers.is("astrophysics")); Response createDataverse = UtilIT.createRandomDataverse(apiToken); createDataverse.then().assertThat().statusCode(CREATED.getStatusCode()); @@ -428,6 +432,10 @@ public void testLinkSoftwareToCodemeta() { UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken).then().assertThat().statusCode(OK.getStatusCode()); + // displayOnCreate will only be true for fields that are set this way in the database. + // We set it here so we can make assertions below. + UtilIT.setDisplayOnCreate("astroInstrument", true); + Response listBlocks = null; System.out.println("listing root collection blocks with display on create using dataset type " + datasetType); listBlocks = UtilIT.listMetadataBlocks(":root", true, true, datasetType, apiToken); @@ -435,10 +443,10 @@ public void testLinkSoftwareToCodemeta() { listBlocks.then().assertThat() .statusCode(OK.getStatusCode()) .body("data[0].name", is("citation")) - .body("data[1].name", is("codeMeta20")) + .body("data[1].name", is("astrophysics")) .body("data[2].name", nullValue()) .body("data[0].fields.title.displayOnCreate", equalTo(true)) - .body("data[1].fields.codeVersion.displayOnCreate", equalTo(true)); + .body("data[1].fields.astroInstrument.displayOnCreate", equalTo(true)); System.out.println("listing root collection blocks with all fields (not display on create) using dataset type " + datasetType); listBlocks = UtilIT.listMetadataBlocks(":root", false, true, datasetType, apiToken); @@ -446,12 +454,12 @@ public void testLinkSoftwareToCodemeta() { listBlocks.then().assertThat() .statusCode(OK.getStatusCode()) .body("data[0].name", is("citation")) - .body("data[1].name", is("codeMeta20")) + .body("data[1].name", is("astrophysics")) .body("data[2].name", nullValue()) .body("data[0].fields.title.displayOnCreate", equalTo(true)) .body("data[0].fields.subtitle.displayOnCreate", equalTo(false)) - .body("data[1].fields.codeVersion.displayOnCreate", equalTo(true)) - .body("data[1].fields.issueTracker.displayOnCreate", equalTo(false)); + .body("data[1].fields.astroInstrument.displayOnCreate", equalTo(true)) + .body("data[1].fields.astroObject.displayOnCreate", equalTo(false)); System.out.println("listing " + dataverseAlias + " collection blocks with display on create using dataset type " + datasetType); listBlocks = UtilIT.listMetadataBlocks(dataverseAlias, true, true, datasetType, apiToken); @@ -459,14 +467,14 @@ public void testLinkSoftwareToCodemeta() { listBlocks.then().assertThat() .statusCode(OK.getStatusCode()) .body("data[0].name", is("citation")) - .body("data[1].name", is("codeMeta20")) + .body("data[1].name", is("astrophysics")) .body("data[2].name", nullValue()) .body("data[0].fields.title.displayOnCreate", equalTo(true)) // subtitle is hidden because it is not "display on create" .body("data[0].fields.subtitle", nullValue()) - .body("data[1].fields.codeVersion.displayOnCreate", equalTo(true)) - // issueTracker is hidden because it is not "display on create" - .body("data[1].fields.issueTracker", nullValue()); + .body("data[1].fields.astroInstrument.displayOnCreate", equalTo(true)) + // astroObject is hidden because it is not "display on create" + .body("data[1].fields.astroObject", nullValue()); System.out.println("listing " + dataverseAlias + " collection blocks with all fields (not display on create) using dataset type " + datasetType); listBlocks = UtilIT.listMetadataBlocks(dataverseAlias, false, true, datasetType, apiToken); @@ -474,12 +482,12 @@ public void testLinkSoftwareToCodemeta() { listBlocks.then().assertThat() .statusCode(OK.getStatusCode()) .body("data[0].name", is("citation")) - .body("data[1].name", is("codeMeta20")) + .body("data[1].name", is("astrophysics")) .body("data[2].name", nullValue()) .body("data[0].fields.title.displayOnCreate", equalTo(true)) .body("data[0].fields.subtitle.displayOnCreate", equalTo(false)) - .body("data[1].fields.codeVersion.displayOnCreate", equalTo(true)) - .body("data[1].fields.issueTracker.displayOnCreate", equalTo(false)); + .body("data[1].fields.astroInstrument.displayOnCreate", equalTo(true)) + .body("data[1].fields.astroObject.displayOnCreate", equalTo(false)); } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/MetadataBlocksIT.java b/src/test/java/edu/harvard/iq/dataverse/api/MetadataBlocksIT.java index 28cf6228a59..316ac579de4 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/MetadataBlocksIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/MetadataBlocksIT.java @@ -27,7 +27,7 @@ public static void setUpClass() { void testListMetadataBlocks() { // No optional params enabled Response listMetadataBlocksResponse = UtilIT.listMetadataBlocks(false, false); - int expectedDefaultNumberOfMetadataBlocks = 7; + int expectedDefaultNumberOfMetadataBlocks = 6; listMetadataBlocksResponse.then().assertThat() .statusCode(OK.getStatusCode()) .body("data[0].fields", equalTo(null)) @@ -35,12 +35,11 @@ void testListMetadataBlocks() { // onlyDisplayedOnCreate=true listMetadataBlocksResponse = UtilIT.listMetadataBlocks(true, false); - int expectedOnlyDisplayedOnCreateNumberOfMetadataBlocks = 2; + int expectedOnlyDisplayedOnCreateNumberOfMetadataBlocks = 1; listMetadataBlocksResponse.then().assertThat() .statusCode(OK.getStatusCode()) .body("data[0].fields", equalTo(null)) .body("data[0].displayName", equalTo("Citation Metadata")) - .body("data[1].displayName", equalTo("Software Metadata (CodeMeta v2.0)")) .body("data.size()", equalTo(expectedOnlyDisplayedOnCreateNumberOfMetadataBlocks)); // returnDatasetFieldTypes=true diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 55baa965314..142b9cd8371 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -828,6 +828,13 @@ static Response getMetadataBlock(String block) { .get("/api/metadatablocks/" + block); } + static Response setDisplayOnCreate(String datasetFieldType, boolean setDisplayOnCreate) { + return given() + .queryParam("datasetFieldType", datasetFieldType) + .queryParam("setDisplayOnCreate", setDisplayOnCreate) + .post("/api/admin/datasetfield/setDisplayOnCreate"); + } + static private String getDatasetXml(String title, String author, String description) { String nullLicense = null; String nullRights = null; From 907383e0515795aed0366eb1d829b94cda0ddda6 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 5 Feb 2025 11:34:25 -0500 Subject: [PATCH 16/17] doc tweaks #11001 Co-authored-by: Omer Fahim --- doc/sphinx-guides/source/api/native-api.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 625e50011ad..5c5f453d51c 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -3486,11 +3486,11 @@ The fully expanded example above (without environment variables) looks like this Link Dataset Type with Metadata Blocks ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Linking a dataset type with one or more metadata blocks results in additional fields from those blocks appearing in the output from the :ref:`list-metadata-blocks-for-a-collection` API endpoint. The new frontend for Dataverse (https://github.com/IQSS/dataverse-frontend) uses the JSON output from this API endpoint to construct the page that users see when creating or editing a dataset. Once the frontend has been updated to pass in the dataset type (https://github.com/IQSS/dataverse-client-javascript/issues/210), specifying a dataset type in this way can be an alternative way to display additional metadata fields than the traditional method, which is to enabled a metadata block at the collection level (see :ref:`define-metadata-blocks-for-a-dataverse-collection`). +Linking a dataset type with one or more metadata blocks results in additional fields from those blocks appearing in the output from the :ref:`list-metadata-blocks-for-a-collection` API endpoint. The new frontend for Dataverse (https://github.com/IQSS/dataverse-frontend) uses the JSON output from this API endpoint to construct the page that users see when creating or editing a dataset. Once the frontend has been updated to pass in the dataset type (https://github.com/IQSS/dataverse-client-javascript/issues/210), specifying a dataset type in this way can be an alternative way to display additional metadata fields than the traditional method, which is to enable a metadata block at the collection level (see :ref:`define-metadata-blocks-for-a-dataverse-collection`). -For example, a superuser could create a type called "software" and link it to the "CodeMeta" metadata block (this example is below). Then, once the new front end allows it, the user can specify that they want to create a dataset of type software and see the additional metadata fields from the CodeMeta block when creating or editing their dataset. +For example, a superuser could create a type called "software" and link it to the "CodeMeta" metadata block (this example is below). Then, once the new frontend allows it, the user can specify that they want to create a dataset of type software and see the additional metadata fields from the CodeMeta block when creating or editing their dataset. -This API endpoint is superuser only. +This API endpoint is for superusers only. .. code-block:: bash From ce03b309cadd656af559a99c7ab32ca379440d52 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 5 Feb 2025 12:14:48 -0500 Subject: [PATCH 17/17] fix typo in docs (PUT vs. POST) #10519 --- doc/sphinx-guides/source/api/native-api.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 5c5f453d51c..717633442c6 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -3499,13 +3499,13 @@ This API endpoint is for superusers only. export TYPE=software export JSON='["codeMeta20"]' - curl -H "X-Dataverse-key:$API_TOKEN" -H "Content-Type: application/json" "$SERVER_URL/api/datasets/datasetTypes/$TYPE" -X POST -d $JSON + curl -H "X-Dataverse-key:$API_TOKEN" -H "Content-Type: application/json" "$SERVER_URL/api/datasets/datasetTypes/$TYPE" -X PUT -d $JSON The fully expanded example above (without environment variables) looks like this: .. code-block:: bash - curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -H "Content-Type: application/json" "https://demo.dataverse.org/api/datasets/datasetTypes/software" -X POST -d '["codeMeta20"]' + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -H "Content-Type: application/json" "https://demo.dataverse.org/api/datasets/datasetTypes/software" -X PUT -d '["codeMeta20"]' To update the blocks that are linked, send an array with those blocks.