Merge branch 'develop' into 11912-edit-template-api

Author: sekmiller

doc/release-notes/12014-croissant-1.1.md

@@ -0,0 +1,25 @@
+### Croissant 1.1 (Summary Statistics)
+
+The Croissant metadata export format has been updated from version 1.0 to 1.1.
+
+Summary statistics (mean, min, max, etc.) are now included for tabular files that were successfully ingested.
+
+You can download an example Croissant file from the [Supported Metadata Export Formats](https://dataverse-guide--12214.org.readthedocs.build/en/12214/user/dataset-management.html#supported-metadata-export-formats) section of the guides.
+
+Minor backward-incompatible changes were made, which are noted below.
+
+See #12014 and #12214
+
+## Backward Incompatible Changes
+
+Generally speaking, see the [API Changelog](https://guides.dataverse.org/en/latest/api/changelog.html) for a list of backward-incompatible API changes.
+
+Minor changes in the `croissant` format are noted in the [API changelog](https://dataverse-guide--12214.org.readthedocs.build/en/12214/api/changelog.html).
+
+## Upgrade Instructions
+
+1. Re-export metadata export formats
+
+   We re-export because the Croissant format was updated.
+
+   `curl http://localhost:8080/api/admin/metadata/reExportAll`

doc/release-notes/12267-does-not-save-guestbook-response.md

@@ -0,0 +1,4 @@
+## BUG
+Fixes 2 bugs
+1. missing "gbrids" in the signed url query parameter list will no longer include "&gbrids=" without a value
+2. For SPA, when a user attempting to download files with a guestbook response has no api token but is authenticated by bearer token, a temporary api token will be generated with an expiration of 1 minute which is used for signing and decoding the signed url.

doc/release-notes/12268-direct-upload-support-legacy-lowercase-dois.md

@@ -0,0 +1 @@
+Fixed a problem with S3 direct upload to datasets which using lower- or mixed-case PID authority/identifier in the database and to datasets using an alternative identifier for file storage.

doc/sphinx-guides/source/api/changelog.rst

@@ -7,6 +7,11 @@ This API changelog is experimental and we would love feedback on its usefulness.
     :local:
     :depth: 1
 
+v6.11
+-----
+
+- The Croissant :ref:`metadata export format <metadata-export-formats>` has been updated from version 1.0 to 1.1, which is reflected in the ``conformsTo`` property. ``@vocab`` and ``sc`` properties now use "http" as `recommended <https://github.com/mlcommons/croissant/pull/929#pullrequestreview-3079137662>`_. The unused ``wd`` property has been dropped.
+
 v6.10
 -----
 - The following GET APIs will now return ``400`` if a required Guestbook Response is not supplied. A Guestbook Response can be passed to these APIs in the JSON body using a POST call. See the notes under :ref:`basic-file-access` and :ref:`download-by-dataset-by-version` for details.

doc/sphinx-guides/source/user/dataset-management.rst

@@ -28,7 +28,7 @@ Supported Metadata Export Formats
 
 Once a dataset has been published, its metadata can be exported in a variety of other metadata standards and formats, which help make datasets more :doc:`discoverable </admin/discoverability>` and usable in other systems, such as other data repositories. On each dataset page's metadata tab, the following exports are available:
 
-- Croissant
+- Croissant (example: :download:`max-croissant.json <../../../../src/test/resources/croissant/max/expected/max-croissant.json>`)
 - Dublin Core
 - DDI (Data Documentation Initiative Codebook 2.5)
 - DDI HTML Codebook (A more human-readable, HTML version of the DDI Codebook 2.5 metadata export)

src/main/java/edu/harvard/iq/dataverse/Dataset.java

@@ -662,6 +662,10 @@ public String getIdentifierForFileStorage(){
         return retVal;         
     }
 
+    public String getGlobalIdForFileStorageAsString() {
+        return getProtocolForFileStorage() + ":" + getAuthorityForFileStorage() + "/" + getIdentifierForFileStorage();
+    }
+
     public String getNextMajorVersionString() {
         // Never need to get the next major version for harvested studies.
         if (isHarvested()) {

src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java

@@ -1706,7 +1706,7 @@ public void requestDirectUploadUrls() {
         String storageIdentifier = null;
         try {
             storageIdentifier = FileUtil.getStorageIdentifierFromLocation(s3io.getStorageLocation());
-            urls = s3io.generateTemporaryS3UploadUrls(dataset.getGlobalId().asString(), storageIdentifier, fileSize);
+            urls = s3io.generateTemporaryS3UploadUrls(dataset.getGlobalIdForFileStorageAsString(), storageIdentifier, fileSize);
 
         } catch (IOException io) {
             logger.warning(io.getMessage());

src/main/java/edu/harvard/iq/dataverse/api/Access.java

@@ -8,6 +8,7 @@
 
 import edu.harvard.iq.dataverse.*;
 import edu.harvard.iq.dataverse.api.auth.AuthRequired;
+import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean;
 import edu.harvard.iq.dataverse.authorization.DataverseRole;
 import edu.harvard.iq.dataverse.authorization.Permission;
 import edu.harvard.iq.dataverse.authorization.RoleAssignee;
@@ -422,7 +423,7 @@ private Response processDatafileWithGuestbookResponse(ContainerRequestContext cr
 
         // Handle Guestbook Responses
         String displayName = "";
-        String gbrids = "";
+        String gbrids = null;
         Long datasetId = null;
         try {
             // since all files must be in the same Dataset we can generate a Guestbook Response once and just replace the DataFile for each file in the list
@@ -487,8 +488,15 @@ private Response returnSignedUrl(ContainerRequestContext crc, UriInfo uriInfo, U
         if (user != null && user instanceof AuthenticatedUser) {
             AuthenticatedUser requestor = (AuthenticatedUser) user;
             userIdentifier = requestor.getUserIdentifier();
+            // Find the latest token: Use for signing
+            // Could be null if no token was generated: Generate one to be used for signing (expire in 1 minute to match timeout in signedUrl)
+            // Could be expired: The user was already authenticated (possible by bearer token). Only used for signing so we don't care
             ApiToken apiToken = authSvc.findApiTokenByUser(requestor);
-            if (apiToken != null && !apiToken.isExpired() && !apiToken.isDisabled()) {
+            if (apiToken == null) {
+                logger.fine("Generating temporary API token for user " + userIdentifier);
+                apiToken = authSvc.generateApiTokenForUser(requestor, AuthenticationServiceBean.INTERVAL.MINUTES, 1);
+            }
+            if (apiToken != null) {
                 key = apiToken.getTokenString();
             }
         } else {
@@ -499,7 +507,9 @@ private Response returnSignedUrl(ContainerRequestContext crc, UriInfo uriInfo, U
 
         UriBuilder builder = UriBuilder.fromUri(uriInfo.getRequestUri());
         builder.replaceQueryParam("gbrecs", true);
-        builder.replaceQueryParam("gbrids", gbrids);
+        if (gbrids != null && !gbrids.isEmpty()) {
+            builder.replaceQueryParam("gbrids", gbrids);
+        }
         builder.replaceQueryParam("persistentId", null); // remove this as a parm and add the id to the path
         crc.setProperty("gbrids", gbrids);
         String baseUrlEncoded = builder.build().toString();

src/main/java/edu/harvard/iq/dataverse/api/Datasets.java

@@ -2798,7 +2798,7 @@ public Response getMPUploadUrls(@Context ContainerRequestContext crc, @PathParam
             String storageIdentifier = null;
             try {
                 storageIdentifier = FileUtil.getStorageIdentifierFromLocation(s3io.getStorageLocation());
-                response = s3io.generateTemporaryS3UploadUrls(dataset.getGlobalId().asString(), storageIdentifier, fileSize);
+                response = s3io.generateTemporaryS3UploadUrls(dataset.getGlobalIdForFileStorageAsString(), storageIdentifier, fileSize);
 
             } catch (IOException io) {
                 logger.warning(io.getMessage());