IQSS/7068 Reserve File Pids

doc/release-notes/7068-reserve-file-pids.md

@@ -0,0 +1,9 @@
+## Release Highlights
+
+### Pre-Publish File DOI Reservation with DataCite
+
+Dataverse installations using DataCite will be able to reserve the persistent identifiers for files with DataCite ahead of publishing time. This allows the file DOI to be reserved earlier in the data sharing process and makes the step of publishing datasets simpler and less error-prone. Note that reserving file DOIs can slow uploads with large numbers of files so administrators may need to adjust timeouts (specifically any Apache "``ProxyPass / ajp://localhost:8009/ timeout=``" setting in the recommended Dataverse configuration).
+
+## Major Use Cases
+
+- Users will have DOIs reserved for their files upon dataset create instead of at publish time. (Issue #7068, PR #7334)

doc/sphinx-guides/source/api/native-api.rst

@@ -4889,7 +4889,7 @@ The fully expanded example above (without environment variables) looks like this
 Reserve a PID
 ~~~~~~~~~~~~~
 
-Reserved a PID for a dataset. A superuser API token is required.
+Reserve a PID for a dataset if not yet registered, and, if FilePIDs are enabled, reserve any file PIDs that are not yet registered. A superuser API token is required.
 
 .. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of export below.
 

src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java

@@ -1238,14 +1238,6 @@ public List<Long> selectFilesWithMissingOriginalSizes() {
     }
 
 
-    /**
-     * Check that a identifier entered by the user is unique (not currently used
-     * for any other study in this Dataverse Network). Also check for duplicate
-     * in the remote PID service if needed
-     * @param datafileId
-     * @param storageLocation
-     * @return  {@code true} iff the global identifier is unique.
-     */
     public void finalizeFileDelete(Long dataFileId, String storageLocation) throws IOException {
         // Verify that the DataFile no longer exists: 
         if (find(dataFileId) != null) {

src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractDatasetCommand.java

@@ -1,5 +1,6 @@
 package edu.harvard.iq.dataverse.engine.command.impl;
 
+import edu.harvard.iq.dataverse.DataFile;
 import edu.harvard.iq.dataverse.Dataset;
 import edu.harvard.iq.dataverse.DatasetField;
 import edu.harvard.iq.dataverse.DatasetVersion;
@@ -15,9 +16,11 @@
 import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException;
 import edu.harvard.iq.dataverse.pidproviders.PidProvider;
 import edu.harvard.iq.dataverse.pidproviders.PidUtil;
+import edu.harvard.iq.dataverse.pidproviders.doi.fake.FakeDOIProvider;
 import edu.harvard.iq.dataverse.util.BundleUtil;
 
 import java.sql.Timestamp;
+import java.util.Arrays;
 import java.util.Date;
 import java.util.Set;
 import java.util.logging.Level;
@@ -167,13 +170,12 @@ protected void registerExternalIdentifier(Dataset theDataset, CommandContext ctx
                             } while (pidProvider.alreadyRegistered(theDataset) && attempts <= FOOLPROOF_RETRIAL_ATTEMPTS_LIMIT);
                         }
                         if(!retry) {
-                            logger.warning("Reserving PID for: "  + getDataset().getId() + " during publication failed.");
-                            throw new IllegalCommandException(BundleUtil.getStringFromBundle("publishDatasetCommand.pidNotReserved"), this);
+                            logger.warning("Reserving PID for: "  + getDataset().getId() + " failed.");
+                            throw new CommandExecutionException(BundleUtil.getStringFromBundle("abstractDatasetCommand.pidNotReserved", Arrays.asList(theDataset.getIdentifier())), this);
                         }
                         if(attempts > FOOLPROOF_RETRIAL_ATTEMPTS_LIMIT) {
                             //Didn't work - we existed the loop with too many tries
-                            throw new CommandExecutionException("This dataset may not be published because its identifier is already in use by another dataset; "
-                                + "gave up after " + attempts + " attempts. Current (last requested) identifier: " + theDataset.getIdentifier(), this);
+                            throw new CommandExecutionException(BundleUtil.getStringFromBundle("abstractDatasetCommand.pidReservationRetryExceeded", Arrays.asList(Integer.toString(attempts), theDataset.getIdentifier())), this);
                         }
                     }
                     // Invariant: Dataset identifier does not exist in the remote registry
@@ -186,6 +188,9 @@ protected void registerExternalIdentifier(Dataset theDataset, CommandContext ctx
                     }
 
                 } catch (Throwable e) {
+                    if (e instanceof CommandException) {
+                        throw (CommandException) e;
+                    }
                     throw new CommandException(BundleUtil.getStringFromBundle("dataset.publish.error", pidProvider.getProviderInformation()), this);
                 }
             } else {
@@ -215,6 +220,73 @@ protected Timestamp getTimestamp() {
         return timestamp;
     }
 
+    protected void registerFilePidsIfNeeded(Dataset theDataset, CommandContext ctxt, boolean b) throws CommandException {
+        // Register file PIDs if needed
+        PidProvider pidGenerator = ctxt.dvObjects().getEffectivePidGenerator(getDataset());
+        boolean shouldRegister = !pidGenerator.registerWhenPublished() &&
+                ctxt.systemConfig().isFilePIDsEnabledForCollection(getDataset().getOwner()) &&
+                pidGenerator.canCreatePidsLike(getDataset().getGlobalId());
+        if (shouldRegister) {
+            for (DataFile dataFile : theDataset.getFiles()) {
+                logger.fine(dataFile.getId() + " is registered?: " + dataFile.isIdentifierRegistered());
+                if (!dataFile.isIdentifierRegistered()) {
+                    // pre-register a persistent id
+                    registerFileExternalIdentifier(dataFile, pidGenerator, ctxt, true);
+                }
+            }
+        }
+    }
+
+    private void registerFileExternalIdentifier(DataFile dataFile, PidProvider pidProvider, CommandContext ctxt, boolean retry) throws CommandException {
+
+        if (!dataFile.isIdentifierRegistered()) {
+
+            if (pidProvider instanceof FakeDOIProvider) {
+                retry = false; // No reason to allow a retry with the FakeProvider (even if it allows
+                               // pre-registration someday), so set false for efficiency
+            }
+            try {
+                if (pidProvider.alreadyRegistered(dataFile)) {
+                    int attempts = 0;
+                    if (retry) {
+                        do {
+                            pidProvider.generatePid(dataFile);
+                            logger.log(Level.INFO, "Attempting to register external identifier for datafile {0} (trying: {1}).",
+                                    new Object[] { dataFile.getId(), dataFile.getIdentifier() });
+                            attempts++;
+                        } while (pidProvider.alreadyRegistered(dataFile) && attempts <= FOOLPROOF_RETRIAL_ATTEMPTS_LIMIT);
+                    }
+                    if (!retry) {
+                        logger.warning("Reserving File PID for: " + getDataset().getId() + ", fileId: " + dataFile.getId() + ", during publication failed.");
+                        throw new CommandExecutionException(BundleUtil.getStringFromBundle("abstractDatasetCommand.filePidNotReserved", Arrays.asList(getDataset().getIdentifier())), this);
+                    }
+                    if (attempts > FOOLPROOF_RETRIAL_ATTEMPTS_LIMIT) {
+                        // Didn't work - we existed the loop with too many tries
+                        throw new CommandExecutionException("This dataset may not be published because its identifier is already in use by another dataset; "
+                                + "gave up after " + attempts + " attempts. Current (last requested) identifier: " + dataFile.getIdentifier(), this);
+                    }
+                }
+                // Invariant: DataFile identifier does not exist in the remote registry
+                try {
+                    pidProvider.createIdentifier(dataFile);
+                    dataFile.setGlobalIdCreateTime(getTimestamp());
+                    dataFile.setIdentifierRegistered(true);
+                } catch (Throwable ex) {
+                    logger.info("Call to globalIdServiceBean.createIdentifier failed: " + ex);
+                }
+
+            } catch (Throwable e) {
+                if (e instanceof CommandException) {
+                    throw (CommandException) e;
+                }
+                throw new CommandException(BundleUtil.getStringFromBundle("file.register.error", pidProvider.getProviderInformation()), this);
+            }
+        } else {
+            throw new IllegalCommandException("This datafile may not have a PID because its id registry service is not supported.", this);
+        }
+
+    }
+
     protected void checkSystemMetadataKeyIfNeeded(DatasetVersion newVersion, DatasetVersion persistedVersion) throws IllegalCommandException {
         Set<MetadataBlock> changedMDBs = DatasetVersionDifference.getBlocksWithChanges(newVersion, persistedVersion);
         for (MetadataBlock mdb : changedMDBs) {