stop loading Codemeta in Docker, add setDisplayOnCreate API #10519

In Docker, we were testing with the Codemeta block. This made for a nice
real-world story of creating a "software" dataset type and using it with
the Codemeta block.

The Codemeta block also has the advantage of having some fields that are
set to displayOnCreate, which helped us make assertions that the code is
working properly. It's the only non-citation block with fields set to
displayOnCreate.

However, Jenkins doesn't have the Codemeta block, meaning that tests are
failing. Also, we aren't ready to promote the Codemeta block to be
shipped with Dataverse because a new version is out:
#10859

So, we are switching from Codemeta to the Astrophysics block. We create
an "instrument" dataset type. Like all non-citation blocks (except for
Codemeta), there are no fields that are set to displayOnCreate=true.
Therefore, we added an API for this.

Author: pdurbin

doc/release-notes/10519-dataset-types.md

@@ -7,6 +7,6 @@ This will have the following effects for the APIs used by the new Dataverse UI (
 - The list of fields shown when creating a dataset will include fields marked as "displayoncreate" (in the tsv/database) for metadata blocks (e.g. "CodeMeta") that are linked to the dataset type (e.g. "software") that is passed to the API.
 - The metadata blocks shown when editing a dataset will include metadata blocks (e.g. "CodeMeta") that are linked to the dataset type (e.g. "software") that is passed to the API.
 
-The CodeMeta metadata block is now available in the Dockerized development environment.
+Mostly in order to write automated tests for the above, a [displayOnCreate](https://dataverse-guide--11001.org.readthedocs.build/en/11001/api/native-api.html#set-displayoncreate-for-a-dataset-field) API endpoint has been added.
 
-For more information, see the guides and #10519.
+For more information, see the guides ([overview](https://dataverse-guide--11001.org.readthedocs.build/en/11001/user/dataset-management.html#dataset-types), [new APIs](https://dataverse-guide--11001.org.readthedocs.build/en/11001/api/native-api.html#link-dataset-type-with-metadata-blocks)), #10519 and #11001.

doc/sphinx-guides/source/api/native-api.rst

@@ -5090,6 +5090,27 @@ The fully expanded example above (without environment variables) looks like this
 
   curl "https://demo.dataverse.org/api/datasetfields/facetables"
 
+.. _setDisplayOnCreate:
+
+Set displayOnCreate for a Dataset Field
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Set displayOnCreate for a dataset field. See also :doc:`/admin/metadatacustomization` in the Admin Guide.
+
+.. code-block:: bash
+
+  export SERVER_URL=http://localhost:8080
+  export FIELD=subtitle
+  export BOOLEAN=true
+
+  curl -X POST "$SERVER_URL/api/admin/datasetfield/setDisplayOnCreate?datasetFieldType=$FIELD&setDisplayOnCreate=$BOOLEAN"
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+  curl -X POST "http://localhost:8080/api/admin/datasetfield/setDisplayOnCreate?datasetFieldType=studyAssayCellType&setDisplayOnCreate=true"
+
 .. _Notifications:
 
 Notifications

modules/container-configbaker/scripts/bootstrap/dev/init.sh

@@ -17,15 +17,6 @@ export API_TOKEN
 # ${ENV_OUT} comes from bootstrap.sh and will expose the saved information back to the host if enabled.
 echo "API_TOKEN=${API_TOKEN}" >> "${ENV_OUT}"
 
-echo "Loading CodeMeta metadata block (needed for API tests)..."
-curl "${DATAVERSE_URL}/api/admin/datasetfield/load" -X POST --data-binary @/scripts/bootstrap/base/data/metadatablocks/codemeta.tsv -H "Content-type: text/tab-separated-values"
-
-echo "Fetching Solr schema from Dataverse and running update-fields.sh..."
-curl "${DATAVERSE_URL}/api/admin/index/solr/schema" | /scripts/update-fields.sh /var/solr/data/collection1/conf/schema.xml
-
-echo "Reloading Solr..."
-curl "http://solr:8983/solr/admin/cores?action=RELOAD&core=collection1"
-
 echo "Publishing root dataverse..."
 curl -H "X-Dataverse-key:$API_TOKEN" -X POST "${DATAVERSE_URL}/api/dataverses/:root/actions/:publish"
 

src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java

@@ -42,6 +42,7 @@
 import java.util.logging.Logger;
 import jakarta.persistence.NoResultException;
 import jakarta.persistence.TypedQuery;
+import jakarta.ws.rs.QueryParam;
 import jakarta.ws.rs.core.Response.Status;
 
 import java.io.BufferedInputStream;
@@ -545,4 +546,19 @@ public static String getDataverseLangDirectory() {
         return dataverseLangDirectory;
     }
 
+    /**
+     * Set setDisplayOnCreate for a DatasetFieldType.
+     */
+    @POST
+    @Path("/setDisplayOnCreate")
+    public Response setDisplayOnCreate(@QueryParam("datasetFieldType") String datasetFieldTypeIn, @QueryParam("setDisplayOnCreate") boolean setDisplayOnCreateIn) {
+        DatasetFieldType dft = datasetFieldService.findByName(datasetFieldTypeIn);
+        if (dft == null) {
+            return error(Status.NOT_FOUND, "Cound not find a DatasetFieldType by looking up " + datasetFieldTypeIn);
+        }
+        dft.setDisplayOnCreate(setDisplayOnCreateIn);
+        DatasetFieldType saved = datasetFieldService.save(dft);
+        return ok("DisplayOnCreate for DatasetFieldType " + saved.getName() + " is now " + saved.isDisplayOnCreate());
+    }
+
 }

src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java

@@ -20,28 +20,32 @@
 
 public class DatasetTypesIT {
 
+    final static String INSTRUMENT = "instrument";
+
     @BeforeAll
     public static void setUpClass() {
         RestAssured.baseURI = UtilIT.getRestAssuredBaseUri();
 
-        Response getSoftwareType = UtilIT.getDatasetType(DatasetType.DATASET_TYPE_SOFTWARE);
-        getSoftwareType.prettyPrint();
-
-        String typeFound = JsonPath.from(getSoftwareType.getBody().asString()).getString("data.name");
-        System.out.println("type found: " + typeFound);
-        if (DatasetType.DATASET_TYPE_SOFTWARE.equals(typeFound)) {
-            return;
-        }
-
-        System.out.println("The \"software\" type wasn't found. Create it.");
         Response createUser = UtilIT.createRandomUser();
         createUser.then().assertThat().statusCode(OK.getStatusCode());
         String username = UtilIT.getUsernameFromResponse(createUser);
         String apiToken = UtilIT.getApiTokenFromResponse(createUser);
         UtilIT.setSuperuserStatus(username, true).then().assertThat().statusCode(OK.getStatusCode());
 
-        String jsonIn = Json.createObjectBuilder().add("name", DatasetType.DATASET_TYPE_SOFTWARE).build().toString();
+        ensureDatasetTypeIsPresent(DatasetType.DATASET_TYPE_SOFTWARE, apiToken);
+        ensureDatasetTypeIsPresent(INSTRUMENT, apiToken);
+    }
 
+    private static void ensureDatasetTypeIsPresent(String datasetType, String apiToken) {
+        Response getDatasetType = UtilIT.getDatasetType(datasetType);
+        getDatasetType.prettyPrint();
+        String typeFound = JsonPath.from(getDatasetType.getBody().asString()).getString("data.name");
+        System.out.println("type found: " + typeFound);
+        if (datasetType.equals(typeFound)) {
+            return;
+        }
+        System.out.println("The " + datasetType + "type wasn't found. Create it.");
+        String jsonIn = Json.createObjectBuilder().add("name", datasetType).build().toString();
         Response typeAdded = UtilIT.addDatasetType(jsonIn, apiToken);
         typeAdded.prettyPrint();
         typeAdded.then().assertThat().statusCode(OK.getStatusCode());
@@ -402,23 +406,23 @@ public void testUpdateDatasetTypeLinksWithMetadataBlocks() {
     }
 
     @Test
-    public void testLinkSoftwareToCodemeta() {
+    public void testLinkInstrumentToAstro() {
         Response createUser = UtilIT.createRandomUser();
         createUser.then().assertThat().statusCode(OK.getStatusCode());
         String username = UtilIT.getUsernameFromResponse(createUser);
         String apiToken = UtilIT.getApiTokenFromResponse(createUser);
         UtilIT.setSuperuserStatus(username, true).then().assertThat().statusCode(OK.getStatusCode());
 
         String metadataBlockLink = """
-            ["codeMeta20"]
+            ["astrophysics"]
 //""";
 
-        String datasetType = "software";
-        Response linkSoftwareToCodemeta = UtilIT.updateDatasetTypeLinksWithMetadataBlocks(datasetType, metadataBlockLink, apiToken);
-        linkSoftwareToCodemeta.prettyPrint();
-        linkSoftwareToCodemeta.then().assertThat().
+        String datasetType = "instrument";
+        Response linkInstrumentToAstro = UtilIT.updateDatasetTypeLinksWithMetadataBlocks(datasetType, metadataBlockLink, apiToken);
+        linkInstrumentToAstro.prettyPrint();
+        linkInstrumentToAstro.then().assertThat().
                 statusCode(OK.getStatusCode())
-                .body("data.linkedMetadataBlocks.after[0]", CoreMatchers.is("codeMeta20"));
+                .body("data.linkedMetadataBlocks.after[0]", CoreMatchers.is("astrophysics"));
 
         Response createDataverse = UtilIT.createRandomDataverse(apiToken);
         createDataverse.then().assertThat().statusCode(CREATED.getStatusCode());
@@ -428,58 +432,62 @@ public void testLinkSoftwareToCodemeta() {
 
         UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken).then().assertThat().statusCode(OK.getStatusCode());
 
+        // displayOnCreate will only be true for fields that are set this way in the database.
+        // We set it here so we can make assertions below.
+        UtilIT.setDisplayOnCreate("astroInstrument", true);
+
         Response listBlocks = null;
         System.out.println("listing root collection blocks with display on create using dataset type " + datasetType);
         listBlocks = UtilIT.listMetadataBlocks(":root", true, true, datasetType, apiToken);
         listBlocks.prettyPrint();
         listBlocks.then().assertThat()
                 .statusCode(OK.getStatusCode())
                 .body("data[0].name", is("citation"))
-                .body("data[1].name", is("codeMeta20"))
+                .body("data[1].name", is("astrophysics"))
                 .body("data[2].name", nullValue())
                 .body("data[0].fields.title.displayOnCreate", equalTo(true))
-                .body("data[1].fields.codeVersion.displayOnCreate", equalTo(true));
+                .body("data[1].fields.astroInstrument.displayOnCreate", equalTo(true));
 
         System.out.println("listing root collection blocks with all fields (not display on create) using dataset type " + datasetType);
         listBlocks = UtilIT.listMetadataBlocks(":root", false, true, datasetType, apiToken);
         listBlocks.prettyPrint();
         listBlocks.then().assertThat()
                 .statusCode(OK.getStatusCode())
                 .body("data[0].name", is("citation"))
-                .body("data[1].name", is("codeMeta20"))
+                .body("data[1].name", is("astrophysics"))
                 .body("data[2].name", nullValue())
                 .body("data[0].fields.title.displayOnCreate", equalTo(true))
                 .body("data[0].fields.subtitle.displayOnCreate", equalTo(false))
-                .body("data[1].fields.codeVersion.displayOnCreate", equalTo(true))
-                .body("data[1].fields.issueTracker.displayOnCreate", equalTo(false));
+                .body("data[1].fields.astroInstrument.displayOnCreate", equalTo(true))
+                .body("data[1].fields.astroObject.displayOnCreate", equalTo(false));
 
         System.out.println("listing " + dataverseAlias + " collection blocks with display on create using dataset type " + datasetType);
         listBlocks = UtilIT.listMetadataBlocks(dataverseAlias, true, true, datasetType, apiToken);
         listBlocks.prettyPrint();
         listBlocks.then().assertThat()
                 .statusCode(OK.getStatusCode())
                 .body("data[0].name", is("citation"))
-                .body("data[1].name", is("codeMeta20"))
+                .body("data[1].name", is("astrophysics"))
                 .body("data[2].name", nullValue())
                 .body("data[0].fields.title.displayOnCreate", equalTo(true))
                 // subtitle is hidden because it is not "display on create"
                 .body("data[0].fields.subtitle", nullValue())
-                .body("data[1].fields.codeVersion.displayOnCreate", equalTo(true))
-                // issueTracker is hidden because it is not "display on create"
-                .body("data[1].fields.issueTracker", nullValue());
+                .body("data[1].fields.astroInstrument.displayOnCreate", equalTo(true))
+                // astroObject is hidden because it is not "display on create"
+                .body("data[1].fields.astroObject", nullValue());
 
         System.out.println("listing " + dataverseAlias + " collection blocks with all fields (not display on create) using dataset type " + datasetType);
         listBlocks = UtilIT.listMetadataBlocks(dataverseAlias, false, true, datasetType, apiToken);
         listBlocks.prettyPrint();
         listBlocks.then().assertThat()
                 .statusCode(OK.getStatusCode())
                 .body("data[0].name", is("citation"))
-                .body("data[1].name", is("codeMeta20"))
+                .body("data[1].name", is("astrophysics"))
                 .body("data[2].name", nullValue())
                 .body("data[0].fields.title.displayOnCreate", equalTo(true))
                 .body("data[0].fields.subtitle.displayOnCreate", equalTo(false))
-                .body("data[1].fields.codeVersion.displayOnCreate", equalTo(true))
-                .body("data[1].fields.issueTracker.displayOnCreate", equalTo(false));
+                .body("data[1].fields.astroInstrument.displayOnCreate", equalTo(true))
+                .body("data[1].fields.astroObject.displayOnCreate", equalTo(false));
 
     }
 

src/test/java/edu/harvard/iq/dataverse/api/MetadataBlocksIT.java

@@ -27,20 +27,19 @@ public static void setUpClass() {
     void testListMetadataBlocks() {
         // No optional params enabled
         Response listMetadataBlocksResponse = UtilIT.listMetadataBlocks(false, false);
-        int expectedDefaultNumberOfMetadataBlocks = 7;
+        int expectedDefaultNumberOfMetadataBlocks = 6;
         listMetadataBlocksResponse.then().assertThat()
                 .statusCode(OK.getStatusCode())
                 .body("data[0].fields", equalTo(null))
                 .body("data.size()", equalTo(expectedDefaultNumberOfMetadataBlocks));
 
         // onlyDisplayedOnCreate=true
         listMetadataBlocksResponse = UtilIT.listMetadataBlocks(true, false);
-        int expectedOnlyDisplayedOnCreateNumberOfMetadataBlocks = 2;
+        int expectedOnlyDisplayedOnCreateNumberOfMetadataBlocks = 1;
         listMetadataBlocksResponse.then().assertThat()
                 .statusCode(OK.getStatusCode())
                 .body("data[0].fields", equalTo(null))
                 .body("data[0].displayName", equalTo("Citation Metadata"))
-                .body("data[1].displayName", equalTo("Software Metadata (CodeMeta v2.0)"))
                 .body("data.size()", equalTo(expectedOnlyDisplayedOnCreateNumberOfMetadataBlocks));
 
         // returnDatasetFieldTypes=true

src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java

@@ -828,6 +828,13 @@ static Response getMetadataBlock(String block) {
                 .get("/api/metadatablocks/" + block);
     }
 
+    static Response setDisplayOnCreate(String datasetFieldType, boolean setDisplayOnCreate) {
+        return given()
+                .queryParam("datasetFieldType", datasetFieldType)
+                .queryParam("setDisplayOnCreate", setDisplayOnCreate)
+                .post("/api/admin/datasetfield/setDisplayOnCreate");
+    }
+
     static private String getDatasetXml(String title, String author, String description) {
         String nullLicense = null;
         String nullRights = null;