reuse stream supplier, update archivers to send oversized files

Author: qqmyers

src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DuraCloudSubmitToArchiveCommand.java

@@ -7,16 +7,24 @@
 import edu.harvard.iq.dataverse.authorization.users.ApiToken;
 import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
 import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
+import edu.harvard.iq.dataverse.util.bagit.BagGenerator;
+import edu.harvard.iq.dataverse.util.bagit.OREMap;
+
 import static edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key.DuraCloudContext;
 import static edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key.DuraCloudHost;
 import static edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key.DuraCloudPort;
 import edu.harvard.iq.dataverse.workflow.step.Failure;
 import edu.harvard.iq.dataverse.workflow.step.WorkflowStepResult;
 
+import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.PipedInputStream;
 import java.io.PipedOutputStream;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.security.DigestInputStream;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
@@ -96,6 +104,8 @@ public WorkflowStepResult performArchiveSubmission(DatasetVersion dv, ApiToken t
                 statusObject.add(DatasetVersion.ARCHIVAL_STATUS, DatasetVersion.ARCHIVAL_STATUS_FAILURE);
                 statusObject.add(DatasetVersion.ARCHIVAL_STATUS_MESSAGE, "Bag not transferred");
 
+                Path tempBagFile = null;
+                
                 try {
                     /*
                      * If there is a failure in creating a space, it is likely that a prior version
@@ -161,20 +171,38 @@ public void run() {
                         // Add BagIt ZIP file
                         // Although DuraCloud uses SHA-256 internally, it's API uses MD5 to verify the
                         // transfer
+                        Path bagFile = null;
+                        
 
-                        messageDigest = MessageDigest.getInstance("MD5");
-                        try (PipedInputStream in = new PipedInputStream(100000);
-                                DigestInputStream digestInputStream2 = new DigestInputStream(in, messageDigest)) {
-                            Thread bagThread = startBagThread(dv, in, digestInputStream2, dataciteXml, token);
-                            checksum = store.addContent(spaceName, fileName, digestInputStream2, -1l, null, null, null);
-                            bagThread.join();
-                            if (success) {
-                                logger.fine("Content: " + fileName + " added with checksum: " + checksum);
-                                localchecksum = Hex.encodeHexString(digestInputStream2.getMessageDigest().digest());
+                        tempBagFile = Files.createTempFile("dataverse-bag-", ".zip");
+                        logger.fine("Creating bag in temporary file: " + tempBagFile.toString());
+                        
+                        BagGenerator bagger = new BagGenerator(new OREMap(dv, false), dataciteXml);
+                        bagger.setAuthenticationKey(token.getTokenString());
+                        // Generate bag to temporary file using the provided ore JsonObject
+                        try (FileOutputStream fos = new FileOutputStream(tempBagFile.toFile())) {
+                            if (!bagger.generateBag(fos)) {
+                                throw new IOException("Bag generation failed");
                             }
-                            if (!success || !checksum.equals(localchecksum)) {
+                        }
+
+                        // Store BagIt file
+                        long bagSize = Files.size(tempBagFile);
+                        logger.fine("Bag created successfully, size: " + bagSize + " bytes");
+
+                            // Now upload the bag file
+                           messageDigest = MessageDigest.getInstance("MD5");
+                            try (InputStream is = Files.newInputStream(bagFile);
+                                    DigestInputStream bagDigestInputStream = new DigestInputStream(is, messageDigest)) {
+                                checksum = store.addContent(spaceName, fileName, bagDigestInputStream, bagFile.toFile().length(), "application/zip", null, null);
+                                localchecksum = Hex.encodeHexString(bagDigestInputStream.getMessageDigest().digest());
+                            
+                            if (checksum != null && checksum.equals(localchecksum)) {
+                                logger.fine("Content: " + fileName + " added with checksum: " + checksum);
+                                success = true;
+                            } else {
                                 logger.severe("Failure on " + fileName);
-                                logger.severe(success ? checksum + " not equal to " + localchecksum : "failed to transfer to DuraCloud");
+                                logger.severe(checksum + " not equal to " + localchecksum);
                                 try {
                                     store.deleteContent(spaceName, fileName);
                                     store.deleteContent(spaceName, baseFileName + "_datacite.xml");
@@ -185,9 +213,6 @@ public void run() {
                                         "DuraCloud Submission Failure: incomplete archive transfer");
                             }
                         }
-
-                        logger.fine("DuraCloud Submission step: Content Transferred");
-
                         // Document the location of dataset archival copy location (actually the URL
                         // where you can
                         // view it as an admin)
@@ -223,8 +248,20 @@ public void run() {
                     return new Failure("Unable to create DuraCloud space with name: " + baseFileName, mesg);
                 } catch (NoSuchAlgorithmException e) {
                     logger.severe("MD5 MessageDigest not available!");
+                } catch (Exception e) {
+                    logger.warning(e.getLocalizedMessage());
+                    e.printStackTrace();
+                    return new Failure("Error in transferring file to DuraCloud",
+                            "DuraCloud Submission Failure: internal error");
                 }
                 finally {
+                    if (tempBagFile != null) {
+                        try {
+                            Files.deleteIfExists(tempBagFile);
+                        } catch (IOException e) {
+                            logger.warning("Failed to delete temporary bag file: " + tempBagFile + " : " + e.getMessage());
+                        }
+                    }
                     dv.setArchivalCopyLocation(statusObject.build().toString());
                 }
             } else {