IQSS/7068 Reserve File Pids

src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java

@@ -1463,7 +1463,7 @@ public List<Long> selectFilesWithMissingOriginalSizes() {
 
     public String generateDataFileIdentifier(DataFile datafile, GlobalIdServiceBean idServiceBean) {
         String doiIdentifierType = settingsService.getValueForKey(SettingsServiceBean.Key.IdentifierGenerationStyle, "randomString");
-        String doiDataFileFormat = settingsService.getValueForKey(SettingsServiceBean.Key.DataFilePIDFormat, "DEPENDENT");
+        String doiDataFileFormat = settingsService.getValueForKey(SettingsServiceBean.Key.DataFilePIDFormat, SystemConfig.DataFilePIDFormat.DEPENDENT.toString());
 
         String prepend = "";
         if (doiDataFileFormat.equals(SystemConfig.DataFilePIDFormat.DEPENDENT.toString())){

src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractDatasetCommand.java

@@ -1,5 +1,6 @@
 package edu.harvard.iq.dataverse.engine.command.impl;
 
+import edu.harvard.iq.dataverse.DataFile;
 import edu.harvard.iq.dataverse.Dataset;
 import edu.harvard.iq.dataverse.DatasetField;
 import edu.harvard.iq.dataverse.DatasetVersion;
@@ -13,6 +14,8 @@
 import edu.harvard.iq.dataverse.engine.command.exception.CommandExecutionException;
 import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException;
 import edu.harvard.iq.dataverse.util.BundleUtil;
+import edu.harvard.iq.dataverse.util.SystemConfig;
+
 import java.sql.Timestamp;
 import java.util.Date;
 import java.util.Iterator;
@@ -23,6 +26,7 @@
 import javax.validation.ConstraintViolation;
 import edu.harvard.iq.dataverse.GlobalIdServiceBean;
 import edu.harvard.iq.dataverse.pidproviders.FakePidProviderServiceBean;
+import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 
 /**
  *
@@ -35,7 +39,7 @@
 public abstract class AbstractDatasetCommand<T> extends AbstractCommand<T> {
 
     private static final Logger logger = Logger.getLogger(AbstractDatasetCommand.class.getName());
-    private static final int FOOLPROOF_RETRIAL_ATTEMPTS_LIMIT = 2 ^ 8;
+    protected static final int FOOLPROOF_RETRIAL_ATTEMPTS_LIMIT = 2 ^ 8;
     private Dataset dataset;
     private final Timestamp timestamp = new Timestamp(new Date().getTime());
 
@@ -221,4 +225,81 @@ public void setDataset(Dataset dataset) {
     protected Timestamp getTimestamp() {
         return timestamp;
     }
+
+    protected void registerFilePidsIfNeeded(Dataset theDataset, CommandContext ctxt, boolean b) throws CommandException {
+     // Register file PIDs if needed
+        String protocol = theDataset.getProtocol();
+        String authority = theDataset.getAuthority();
+        GlobalIdServiceBean idServiceBean = GlobalIdServiceBean.getBean(protocol, ctxt);
+        String currentGlobalIdProtocol = ctxt.settings().getValueForKey(SettingsServiceBean.Key.Protocol, "");
+        String currentGlobalAuthority = ctxt.settings().getValueForKey(SettingsServiceBean.Key.Authority, "");
+        String dataFilePIDFormat = ctxt.settings().getValueForKey(SettingsServiceBean.Key.DataFilePIDFormat, SystemConfig.DataFilePIDFormat.DEPENDENT.toString());
+        boolean shouldRegister = ctxt.systemConfig().isFilePIDsEnabled()                                  // We use file PIDs
+                && !idServiceBean.registerWhenPublished()                                                 // The provider can pre-register
+                &&((currentGlobalIdProtocol.equals(protocol) && currentGlobalAuthority.equals(authority)) // the dataset PID is a protocol/authority Dataverse can create new PIDs in
+                        || dataFilePIDFormat.equals(SystemConfig.DataFilePIDFormat.INDEPENDENT.toString()));                                      // or the files can use a different protocol/authority
+        logger.fine("IsFilePIDsEnabled: " + ctxt.systemConfig().isFilePIDsEnabled());
+        logger.fine("RegWhenPub: " +  !idServiceBean.registerWhenPublished());
+        logger.fine("OK provider: " + ((currentGlobalIdProtocol.equals(protocol) && currentGlobalAuthority.equals(authority)) // the dataset PID is a protocol/authority Dataverse can create new PIDs in
+                        || dataFilePIDFormat.equals(SystemConfig.DataFilePIDFormat.INDEPENDENT.toString())));  
+        logger.fine("Should register: " + shouldRegister);
+        if (shouldRegister) {
+            for (DataFile dataFile : theDataset.getFiles()) {
+                logger.fine(dataFile.getId() + " is registered?: " + dataFile.isIdentifierRegistered());
+                if (!dataFile.isIdentifierRegistered()) {
+                    // pre-register a persistent id
+                    registerFileExternalIdentifier(dataFile, idServiceBean, ctxt, true);
+                }
+            }
+        }
+    }
+
+    private void registerFileExternalIdentifier(DataFile dataFile, GlobalIdServiceBean globalIdServiceBean, CommandContext ctxt, boolean retry) throws CommandException {
+
+        if (!dataFile.isIdentifierRegistered()) {
+            if (globalIdServiceBean != null) {
+                if (globalIdServiceBean instanceof FakePidProviderServiceBean) {
+                    retry = false; // No reason to allow a retry with the FakeProvider (even if it allows
+                                   // pre-registration someday), so set false for efficiency
+                }
+                try {
+                    if (globalIdServiceBean.alreadyExists(dataFile)) {
+                        int attempts = 0;
+                        if (retry) {
+                            do {
+                                dataFile.setIdentifier(ctxt.files().generateDataFileIdentifier(dataFile, globalIdServiceBean));
+                                logger.log(Level.INFO, "Attempting to register external identifier for datafile {0} (trying: {1}).",
+                                        new Object[] { dataFile.getId(), dataFile.getIdentifier() });
+                                attempts++;
+                            } while (globalIdServiceBean.alreadyExists(dataFile) && attempts <= FOOLPROOF_RETRIAL_ATTEMPTS_LIMIT);
+                        }
+                        if (!retry) {
+                            logger.warning("Reserving File PID for: " + getDataset().getId() + ", fileId: " + dataFile.getId() + ", during publication failed.");
+                            throw new IllegalCommandException(BundleUtil.getStringFromBundle("publishDatasetCommand.filePidNotReserved"), this);
+                        }
+                        if (attempts > FOOLPROOF_RETRIAL_ATTEMPTS_LIMIT) {
+                            // Didn't work - we existed the loop with too many tries
+                            throw new CommandExecutionException("This dataset may not be published because its identifier is already in use by another dataset; "
+                                    + "gave up after " + attempts + " attempts. Current (last requested) identifier: " + dataFile.getIdentifier(), this);
+                        }
+                    }
+                    // Invariant: DataFile identifier does not exist in the remote registry
+                    try {
+                        globalIdServiceBean.createIdentifier(dataFile);
+                        dataFile.setGlobalIdCreateTime(getTimestamp());
+                        dataFile.setIdentifierRegistered(true);
+                    } catch (Throwable ex) {
+                        logger.info("Call to globalIdServiceBean.createIdentifier failed: " + ex);
+                    }
+
+                } catch (Throwable e) {
+                    throw new CommandException(BundleUtil.getStringFromBundle("file.register.error", globalIdServiceBean.getProviderInformation()), this);
+                }
+            } else {
+                throw new IllegalCommandException("This datafile may not have a PID because its id registry service is not supported.", this);
+            }
+
+        }
+
+    }
 }

src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java

@@ -32,6 +32,8 @@
 import edu.harvard.iq.dataverse.batch.util.LoggingUtil;
 import edu.harvard.iq.dataverse.engine.command.Command;
 import edu.harvard.iq.dataverse.util.FileUtil;
+import edu.harvard.iq.dataverse.util.SystemConfig;
+
 import java.util.concurrent.Future;
 import org.apache.solr.client.solrj.SolrServerException;
 
@@ -82,16 +84,16 @@ public Dataset execute(CommandContext ctxt) throws CommandException {
         }
 
 		/*
-		 * Try to register the dataset identifier. For PID providers that have registerWhenPublished == false (all except the FAKE provider at present)
+		 * Try to register the dataset identifier, and file pids if enabled and needed. For PID providers that have registerWhenPublished == false (all except the FAKE provider at present)
 		 * the registerExternalIdentifier command will make one try to create the identifier if needed (e.g. if reserving at dataset creation wasn't done/failed).
 		 * For registerWhenPublished == true providers, if a PID conflict is found, the call will retry with new PIDs. 
 		 */
         if ( theDataset.getGlobalIdCreateTime() == null ) {
             try {
                 // This can potentially throw a CommandException, so let's make 
                 // sure we exit cleanly:
-
-            	registerExternalIdentifier(theDataset, ctxt, false);
+                registerExternalIdentifier(theDataset, ctxt, false);
+                registerFilePidsIfNeeded(theDataset, ctxt, false);
             } catch (CommandException comEx) {
                 logger.warning("Failed to reserve the identifier "+theDataset.getGlobalId().asString()+"; notifying the user(s), unlocking the dataset");
                 // Send failure notification to the user: 
@@ -328,7 +330,7 @@ private void publicizeExternalIdentifier(Dataset dataset, CommandContext ctxt) t
             try {
                 String currentGlobalIdProtocol = ctxt.settings().getValueForKey(SettingsServiceBean.Key.Protocol, "");
                 String currentGlobalAuthority = ctxt.settings().getValueForKey(SettingsServiceBean.Key.Authority, "");
-                String dataFilePIDFormat = ctxt.settings().getValueForKey(SettingsServiceBean.Key.DataFilePIDFormat, "DEPENDENT");
+                String dataFilePIDFormat = ctxt.settings().getValueForKey(SettingsServiceBean.Key.DataFilePIDFormat, SystemConfig.DataFilePIDFormat.DEPENDENT.toString());
                 boolean isFilePIDsEnabled = ctxt.systemConfig().isFilePIDsEnabled();
                 // We will skip trying to register the global identifiers for datafiles 
                 // if "dependent" file-level identifiers are requested, AND the naming 
@@ -340,7 +342,7 @@ private void publicizeExternalIdentifier(Dataset dataset, CommandContext ctxt) t
                 // Additionaly in 4.9.3 we have added a system variable to disable 
                 // registering file PIDs on the installation level.
                 if (((currentGlobalIdProtocol.equals(protocol) && currentGlobalAuthority.equals(authority))
-                        || dataFilePIDFormat.equals("INDEPENDENT"))
+                        || dataFilePIDFormat.equals(SystemConfig.DataFilePIDFormat.INDEPENDENT.toString()))
                         && isFilePIDsEnabled
                         && dataset.getLatestVersion().getMinorVersionNumber() != null
                         && dataset.getLatestVersion().getMinorVersionNumber().equals((long) 0)) {

src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java

@@ -14,6 +14,7 @@
 import edu.harvard.iq.dataverse.privateurl.PrivateUrl;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import edu.harvard.iq.dataverse.util.BundleUtil;
+import edu.harvard.iq.dataverse.util.SystemConfig;
 import edu.harvard.iq.dataverse.workflow.Workflow;
 import edu.harvard.iq.dataverse.workflow.WorkflowContext.TriggerType;
 import java.util.Date;
@@ -109,9 +110,9 @@ public PublishDatasetResult execute(CommandContext ctxt) throws CommandException
             // registering file PIDs on the installation level.
             String currentGlobalIdProtocol = ctxt.settings().getValueForKey(SettingsServiceBean.Key.Protocol, "");
             String currentGlobalAuthority= ctxt.settings().getValueForKey(SettingsServiceBean.Key.Authority, "");
-            String dataFilePIDFormat = ctxt.settings().getValueForKey(SettingsServiceBean.Key.DataFilePIDFormat, "DEPENDENT");
+            String dataFilePIDFormat = ctxt.settings().getValueForKey(SettingsServiceBean.Key.DataFilePIDFormat, SystemConfig.DataFilePIDFormat.DEPENDENT.toString());
             boolean registerGlobalIdsForFiles = 
-                    (currentGlobalIdProtocol.equals(theDataset.getProtocol()) || dataFilePIDFormat.equals("INDEPENDENT")) 
+                    (currentGlobalIdProtocol.equals(theDataset.getProtocol()) || dataFilePIDFormat.equals(SystemConfig.DataFilePIDFormat.INDEPENDENT.toString())) 
                     && ctxt.systemConfig().isFilePIDsEnabled();
 
             if ( registerGlobalIdsForFiles ){

src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java

@@ -9,6 +9,7 @@
 import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
 import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
 import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException;
+
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
@@ -130,7 +131,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException {
             		throw e;
             	}
             }
-
+            //Set creator and create date for files if needed
             for (DataFile dataFile : theDataset.getFiles()) {
                 if (dataFile.getCreateDate() == null) {
                     dataFile.setCreateDate(getTimestamp());
@@ -163,6 +164,9 @@ public Dataset execute(CommandContext ctxt) throws CommandException {
                     recalculateUNF = true;
                 }
             }
+
+            theDataset.getEditVersion().setLastUpdateTime(getTimestamp());
+
             // we have to merge to update the database but not flush because
             // we don't want to create two draft versions!
             // Dataset tempDataset = ctxt.em().merge(theDataset);
@@ -181,7 +185,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException {
                     ctxt.engine().submit(new DeleteDataFileCommand(fmd.getDataFile(), getRequest()));
                     theDataset.getFiles().remove(fmd.getDataFile());
                     theDataset.getEditVersion().getFileMetadatas().remove(fmd);
-                    // added this check to handle issue where you could not deleter a file that
+                    // added this check to handle issue where you could not delete a file that
                     // shared a category with a new file
                     // the relation ship does not seem to cascade, yet somehow it was trying to
                     // merge the filemetadata
@@ -197,11 +201,13 @@ public Dataset execute(CommandContext ctxt) throws CommandException {
                 }
             }
 
+            registerFilePidsIfNeeded(theDataset, ctxt, true);
+
             if (recalculateUNF) {
                 ctxt.ingest().recalculateDatasetVersionUNF(theDataset.getEditVersion());
             }
 
-            theDataset.getEditVersion().setLastUpdateTime(getTimestamp());
+
             theDataset.setModificationTime(getTimestamp());
 
             savedDataset = ctxt.em().merge(theDataset);
@@ -222,10 +228,9 @@ public Dataset execute(CommandContext ctxt) throws CommandException {
                 ctxt.datasets().removeDatasetLocks(theDataset, DatasetLock.Reason.EditInProgress);
             }
         }
-
         return savedDataset; 
     }
-    
+
     @Override
     public boolean onSuccess(CommandContext ctxt, Object r) {
 

src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDvObjectPIDMetadataCommand.java

@@ -13,6 +13,8 @@
 import edu.harvard.iq.dataverse.engine.command.exception.PermissionException;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import edu.harvard.iq.dataverse.util.BundleUtil;
+import edu.harvard.iq.dataverse.util.SystemConfig;
+
 import java.sql.Timestamp;
 import java.util.Collections;
 import java.util.Date;
@@ -56,7 +58,7 @@ protected void executeImpl(CommandContext ctxt) throws CommandException {
                 // When updating, we want to traverse through files even if the dataset itself
                 // didn't need updating.
                 String currentGlobalIdProtocol = ctxt.settings().getValueForKey(SettingsServiceBean.Key.Protocol, "");
-                String dataFilePIDFormat = ctxt.settings().getValueForKey(SettingsServiceBean.Key.DataFilePIDFormat, "DEPENDENT");
+                String dataFilePIDFormat = ctxt.settings().getValueForKey(SettingsServiceBean.Key.DataFilePIDFormat, SystemConfig.DataFilePIDFormat.DEPENDENT.toString());
                 boolean isFilePIDsEnabled = ctxt.systemConfig().isFilePIDsEnabled();
                 // We will skip trying to update the global identifiers for datafiles if they
                 // aren't being used.
@@ -68,7 +70,7 @@ protected void executeImpl(CommandContext ctxt) throws CommandException {
                     if (isFilePIDsEnabled && // using file PIDs and
                             (!(df.getIdentifier() == null || df.getIdentifier().isEmpty()) || // identifier exists, or
                                     currentGlobalIdProtocol.equals(protocol) || // right protocol to create dependent DOIs, or
-                                    dataFilePIDFormat.equals("INDEPENDENT"))// or independent. TODO(pm) - check authority too
+                                    dataFilePIDFormat.equals(SystemConfig.DataFilePIDFormat.INDEPENDENT.toString()))// or independent. TODO(pm) - check authority too
                     ) {
                         doiRetString = idServiceBean.publicizeIdentifier(df);
                         if (doiRetString) {

src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java

@@ -1014,8 +1014,8 @@ public Integer getUploadMethodCount(){
     }
     public boolean isDataFilePIDSequentialDependent(){
         String doiIdentifierType = settingsService.getValueForKey(SettingsServiceBean.Key.IdentifierGenerationStyle, "randomString");
-        String doiDataFileFormat = settingsService.getValueForKey(SettingsServiceBean.Key.DataFilePIDFormat, "DEPENDENT");
-        if (doiIdentifierType.equals("sequentialNumber") && doiDataFileFormat.equals("DEPENDENT")){
+        String doiDataFileFormat = settingsService.getValueForKey(SettingsServiceBean.Key.DataFilePIDFormat, DataFilePIDFormat.DEPENDENT.toString());
+        if (doiIdentifierType.equals("sequentialNumber") && doiDataFileFormat.equals(DataFilePIDFormat.DEPENDENT.toString())){
             return true;
         }
         return false;

src/main/java/propertyFiles/Bundle.properties

@@ -2571,6 +2571,8 @@ pids.datacite.errors.DoiOnly=Only doi: is supported.
 
 #PublishDatasetCommand
 publishDatasetCommand.pidNotReserved=Cannot publish dataset because its persistent identifier has not been reserved.
+publishDatasetCommand.pidNotReserved=Cannot publish dataset because a persistent identifier for one or more of its files has not been reserved.
+
 
 # APIs
 api.errors.invalidApiToken=Invalid API token.