4813 allow duplicate files

doc/release-notes/4813-allow-duplicate-files.md

@@ -0,0 +1 @@
+We should note that duplicate files are now allowed, and installations may want to contact people now that this is available.

src/main/java/edu/harvard/iq/dataverse/DataFile.java

@@ -254,6 +254,22 @@ public boolean isDeleted() {
     public void setDeleted(boolean deleted) {
         this.deleted = deleted;
     }
+
+    /*
+    For use during file upload so that the user may delete 
+    files that have already been uploaded to the current dataset version
+    */
+
+    @Transient
+    private boolean markedAsDuplicate;
+
+    public boolean isMarkedAsDuplicate() {
+        return markedAsDuplicate;
+    }
+
+    public void setMarkedAsDuplicate(boolean markedAsDuplicate) {
+        this.markedAsDuplicate = markedAsDuplicate;
+    }
 
     /**
      * All constructors should use this method

src/main/java/edu/harvard/iq/dataverse/FileMetadata.java

@@ -461,6 +461,22 @@ public void setSelected(boolean selected) {
         this.selected = selected;
     }
 
+    /*
+    For use during file upload so that the user may delete 
+    files that have already been uploaded to the current dataset version
+    */
+
+    @Transient
+    private boolean markedAsDuplicate;
+
+    public boolean isMarkedAsDuplicate() {
+        return markedAsDuplicate;
+    }
+
+    public void setMarkedAsDuplicate(boolean markedAsDuplicate) {
+        this.markedAsDuplicate = markedAsDuplicate;
+    }
+
     @Transient
     private boolean restrictedUI;
 

src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java

@@ -670,6 +670,15 @@ protected Response ok( String msg ) {
             .type(MediaType.APPLICATION_JSON)
             .build();
     }
+
+    protected Response ok( String msg, JsonObjectBuilder bld  ) {
+        return Response.ok().entity(Json.createObjectBuilder()
+            .add("status", STATUS_OK)
+            .add("message", Json.createObjectBuilder().add("message",msg))     
+            .add("data", bld).build())      
+            .type(MediaType.APPLICATION_JSON)
+            .build();
+    }
 
     protected Response ok( boolean value ) {
         return Response.ok().entity(Json.createObjectBuilder()

src/main/java/edu/harvard/iq/dataverse/api/Datasets.java

@@ -1658,7 +1658,13 @@ public Response addFileToDataset(@PathParam("id") String idSupplied,
                  * user. Human readable.
                  */
                 logger.fine("successMsg: " + successMsg);
-                return ok(addFileHelper.getSuccessResultAsJsonObjectBuilder());
+                String duplicateWarning = addFileHelper.getDuplicateFileWarning();
+                if (duplicateWarning != null && !duplicateWarning.isEmpty()) {
+                    return ok(addFileHelper.getDuplicateFileWarning(), addFileHelper.getSuccessResultAsJsonObjectBuilder());
+                } else {
+                    return ok(addFileHelper.getSuccessResultAsJsonObjectBuilder());
+                }
+
                 //"Look at that!  You added a file! (hey hey, it may have worked)");
             } catch (NoFilesException ex) {
                 Logger.getLogger(Files.class.getName()).log(Level.SEVERE, null, ex);

src/main/java/edu/harvard/iq/dataverse/api/Files.java

@@ -378,9 +378,16 @@ public Response updateFileMetadata(@FormDataParam("jsonData") String jsonData,
                     // on *new* datafiles, that haven't been saved in the database yet;
                     // but it should never be the case in the context of this API) 
                     // -- L.A. Mar. 2020
+                    //SEK 5/2020 - we can't use checksum because it
+                    //is no longer guaranteed to be unique.
+                    /*
                     if(daf.getChecksumType().equals(df.getChecksumType())
                         && daf.getChecksumValue().equals(df.getChecksumValue())) {
                         upFmd = testFmd;
+                    }*/
+                    if(daf.equals(df)){
+                       upFmd = testFmd;
+                       break;
                     }
                 }
 
@@ -410,7 +417,7 @@ public Response updateFileMetadata(@FormDataParam("jsonData") String jsonData,
 
             } catch (Exception e) {
                 logger.log(Level.WARNING, "Dataset publication finalization: exception while exporting:{0}", e);
-                return error(Response.Status.INTERNAL_SERVER_ERROR, "Error adding metadata to DataFile" + e);
+                return error(Response.Status.INTERNAL_SERVER_ERROR, "Error adding metadata to DataFile: " + e);
             }
 
         } catch (WrappedResponse wr) {

src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java

@@ -153,6 +153,25 @@ public class AddReplaceFileHelper{
     private boolean contentTypeWarningFound;
     private String contentTypeWarningString;
 
+    private boolean duplicateFileWarningFound;
+    private String duplicateFileWarningString;
+
+    public boolean isDuplicateFileWarningFound() {
+        return duplicateFileWarningFound;
+    }
+
+    public void setDuplicateFileWarningFound(boolean duplicateFileWarningFound) {
+        this.duplicateFileWarningFound = duplicateFileWarningFound;
+    }
+
+    public String getDuplicateFileWarningString() {
+        return duplicateFileWarningString;
+    }
+
+    public void setDuplicateFileWarningString(String duplicateFileWarningString) {
+        this.duplicateFileWarningString = duplicateFileWarningString;
+    }
+
     public void resetFileHelper(){
 
         initErrorHandling();
@@ -761,6 +780,17 @@ private void addError(Response.Status badHttpResponse, String errMsg){
 
     }
 
+    private void addErrorWarning(String errMsg){
+        if (errMsg == null){
+            throw new NullPointerException("errMsg cannot be null");
+        }
+
+        logger.severe(errMsg);
+        this.setDuplicateFileWarning(errMsg);
+        this.errorMessages.add(errMsg);
+
+    }
+
 
     private void addErrorSevere(String errMsg){
 
@@ -1179,20 +1209,18 @@ private boolean step_040_auto_checkForDuplicates(){
 
             // -----------------------------------------------------------
             // (2) Check for duplicates
+            // Only a warning now
             // -----------------------------------------------------------     
             if (isFileReplaceOperation() && Objects.equals(df.getChecksumValue(), fileToReplace.getChecksumValue())){
-                this.addErrorSevere(getBundleErr("replace.new_file_same_as_replacement"));                                
-                break;
+                this.addErrorWarning(getBundleErr("replace.new_file_same_as_replacement")); 
+                this.duplicateFileWarningFound = true;
+                this.duplicateFileWarningString = getBundleErr("replace.new_file_same_as_replacement");
+              //  break;
             } else if (DuplicateFileChecker.isDuplicateOriginalWay(workingVersion, df.getFileMetadata())){
                 String dupeName = df.getFileMetadata().getLabel();
-                //removeUnSavedFilesFromWorkingVersion();
-                //removeLinkedFileFromDataset(dataset, df);
-                //abandonOperationRemoveAllNewFilesFromDataset();
-                this.addErrorSevere(getBundleErr("duplicate_file") + " " + dupeName);   
-                //return false;
-            } else {
-                finalFileList.add(df);
-            }
+                this.addErrorWarning(getBundleErr("warning.duplicate_file") + " " + dupeName + " " + getBundleErr("duplicate_file.continue")); 
+            }             
+            finalFileList.add(df);
         }
 
         if (this.hasError()){
@@ -1913,6 +1941,16 @@ public String getContentTypeWarningString(){
         return contentTypeWarningString;
     }
 
+    private String duplicateFileWarning;
+
+    public String getDuplicateFileWarning() {
+        return duplicateFileWarning;
+    }
+
+    public void setDuplicateFileWarning(String duplicateFileWarning) {
+        this.duplicateFileWarning = duplicateFileWarning;
+    }
+
 } // end class
   /*
     DatasetPage sequence:

src/main/java/edu/harvard/iq/dataverse/datasetutility/DuplicateFileChecker.java

@@ -150,18 +150,21 @@ public static boolean isDuplicateOriginalWay(DatasetVersion workingVersion, File
         List<FileMetadata> wvCopy = new ArrayList<>(workingVersion.getFileMetadatas());
         Iterator<FileMetadata> fmIt = wvCopy.iterator();
 
-        while (fmIt.hasNext()) {
+        while (fmIt.hasNext()) {            
             FileMetadata fm = fmIt.next();
-            String currentCheckSum = fm.getDataFile().getChecksumValue();
+            String currentCheckSum = fm.getDataFile().getChecksumValue();            
             if (currentCheckSum != null) {
+                if (currentCheckSum.equals(selectedCheckSum)) return true;
+                /*
                 if (checkSumMap.get(currentCheckSum) != null) {
                     checkSumMap.put(currentCheckSum, checkSumMap.get(currentCheckSum).intValue() + 1);
                 } else {
                     checkSumMap.put(currentCheckSum, 1);
-                }
+                }*/
             }
         }
-        return checkSumMap.get(selectedCheckSum) != null; // && checkSumMap.get(selectedCheckSum).intValue() > 1;
+        return false;
+       // return checkSumMap.get(selectedCheckSum) != null; // && checkSumMap.get(selectedCheckSum).intValue() > 1;
 
     }
 

src/main/java/edu/harvard/iq/dataverse/datasetutility/FileReplacePageHelper.java

@@ -199,6 +199,10 @@ public List<FileMetadata> getNewFileMetadatasBeforeSave(){
 
     }
 
+    public AddReplaceFileHelper getAddReplaceFileHelper(){
+        return replaceFileHelper;
+    }
+
     /** 
      * 
      * Show file upload component if Phase 1 hasn't happened yet

src/main/java/edu/harvard/iq/dataverse/ingest/IngestUtil.java

@@ -116,7 +116,22 @@ public static String duplicateFilenameCheck(FileMetadata fileMetadata, Set<Strin
      * @return true if there is a conflict, false otherwise.
      */
     public static boolean conflictsWithExistingFilenames(String label, String directoryLabel, List<FileMetadata> fileMetadatas, DataFile dataFile) {
-        List<String> filePathsAndNames = getPathsAndFileNames(fileMetadatas);
+
+        //Remove from the metadatas list the name of the datafile in question
+        List <FileMetadata> copy = new ArrayList<>();
+        for (FileMetadata fm: fileMetadatas){
+            copy.add(fm);
+        }
+        Iterator<FileMetadata> fmIt = copy.iterator();
+
+        while (fmIt.hasNext()) {
+            FileMetadata fm = fmIt.next();
+            if (fm.getDataFile().equals(dataFile)) {
+                fmIt.remove();
+            }
+        }
+
+        List<String> filePathsAndNames = getPathsAndFileNames(copy);
         if (label != null || directoryLabel != null) {
             String path = "";
             if (directoryLabel != null) {

src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java

@@ -1836,5 +1836,29 @@ public static void deleteTempFile(DataFile dataFile, Dataset dataset, IngestServ
     		dataFile.setOwner(null);
     	}
     }
+
+    public static boolean isFileAlreadyUploaded(DataFile dataFile, Map checksumMapNew, Map fileAlreadyExists) {
+        if (checksumMapNew == null) {
+            checksumMapNew = new HashMap<>();
+        }
+
+        if (fileAlreadyExists == null) {
+            fileAlreadyExists = new HashMap<>();
+        }
+
+        String chksum = dataFile.getChecksumValue();
+
+        if (chksum == null) {
+            return false;
+        }
+
+        if (checksumMapNew.get(chksum) != null) {
+            fileAlreadyExists.put(dataFile, checksumMapNew.get(chksum));
+            return true;
+        }
+
+        checksumMapNew.put(chksum, dataFile);
+        return false;
+    }
 
 }

src/main/java/propertyFiles/Bundle.properties

@@ -1886,15 +1886,17 @@ file.addreplace.error.no_edit_dataset_permission=You do not have permission to e
 file.addreplace.error.filename_undetermined=The file name cannot be determined.
 file.addreplace.error.file_content_type_undetermined=The file content type cannot be determined.
 file.addreplace.error.file_upload_failed=The file upload failed.
-file.addreplace.error.duplicate_file=This file already exists in the dataset. 
+file.addreplace.error.warning.duplicate_file=Warning! This file already exists in the dataset. 
+file.addreplace.error.duplicate_file.continue=You may delete if it was not intentional.
 file.addreplace.error.existing_file_to_replace_id_is_null=The ID of the existing file to replace must be provided.
 file.addreplace.error.existing_file_to_replace_not_found_by_id=Replacement file not found. There was no file found for ID: {0}
 file.addreplace.error.existing_file_to_replace_is_null=The file to replace cannot be null.
 file.addreplace.error.existing_file_to_replace_not_in_dataset=The file to replace does not belong to this dataset.
 file.addreplace.error.existing_file_not_in_latest_published_version=You cannot replace a file that is not in the most recently published dataset. (The file is unpublished or was deleted from a previous version.)
 file.addreplace.content_type.header=File Type Different
+file.addreplace.already_exists.header=Duplicate Files Uploaded
 file.addreplace.error.replace.new_file_has_different_content_type=The original file ({0}) and replacement file ({1}) are different file types.
-file.addreplace.error.replace.new_file_same_as_replacement=You cannot replace a file with the exact same file.
+file.addreplace.error.replace.new_file_same_as_replacement=Warning! This is the same file as the one being replaced.
 file.addreplace.error.unpublished_file_cannot_be_replaced=You cannot replace an unpublished file. Please delete it instead of replacing it.
 file.addreplace.error.ingest_create_file_err=There was an error when trying to add the new file.
 file.addreplace.error.initial_file_list_empty=An error occurred and the new file was not added.

src/main/webapp/editFilesFragment.xhtml

@@ -527,7 +527,7 @@
 
     <!-- Popups -->
     <p:dialog id="fileTypeDifferentPopup" styleClass="smallPopUp" header="#{bundle['file.addreplace.content_type.header']}" widgetVar="fileTypeDifferentPopup" modal="true">
-        <p class="text-warning"><span class="glyphicon glyphicon-warning-sign"/> #{EditDatafilesPage.warningMessageForPopUp}</p>
+        <p class="text-warning"><span class="glyphicon glyphicon-warning-sign"/> #{EditDatafilesPage.warningMessageForFileTypeDifferentPopUp}</p>
         <div class="button-block">
             <p:commandButton styleClass="btn btn-default" value="#{bundle['file.delete']}" onclick="PF('fileTypeDifferentPopup').hide()" oncomplete="uploadWidgetDropMsg();"
                              action="#{EditDatafilesPage.deleteFiles()}"
@@ -537,6 +537,18 @@
             </button>
         </div>
     </p:dialog>
+    <p:dialog id="fileAlreadyExistsPopup" styleClass="smallPopUp" header="#{bundle['file.addreplace.already_exists.header']}" widgetVar="fileAlreadyExistsPopup" modal="true">
+        <p class="text-warning"><span class="glyphicon glyphicon-warning-sign"/> #{EditDatafilesPage.warningMessageForAlreadyExistsPopUp}</p>
+        <div class="button-block">
+            <p:commandButton styleClass="btn btn-default" value="#{bundle['file.delete']}" onclick="PF('fileAlreadyExistsPopup').hide()" oncomplete="uploadWidgetDropMsg();"
+                             action="#{EditDatafilesPage.deleteMarkedAsDuplicateFiles()}"
+                             update=":#{p:resolveClientId('datasetForm:filesTable', view)},:messagePanel,:#{p:resolveClientId('datasetForm:fileUpload', view)},uploadMessage"/>
+            <button class="btn btn-default" onclick="PF('fileAlreadyExistsPopup').hide();" type="button">
+                #{bundle.continue}
+            </button>
+        </div>
+    </p:dialog>
+
     <p:dialog id="selectFilesForDeleteFragment" styleClass="smallPopUp"
               header="#{bundle['dataset.noSelectedFiles.header']}" widgetVar="selectFilesForDeleteFragment" modal="true">
         <p class="text-danger"><span class="glyphicon glyphicon-exclamation-sign"/> #{bundle['dataset.noSelectedFilesForDelete']}</p>