Fix for ConcurrentMod Exception seen in DatasetsIT test (#12128)

* Add synchronization, avoid sharing list across fmds, copy prov

* make sure fmd is associated with correct version

* make test use new method

Author: qqmyers

src/main/java/edu/harvard/iq/dataverse/Dataset.java

@@ -384,21 +384,7 @@ private DatasetVersion createNewDatasetVersion(Template template, FileMetadata f
             in a pre-save validation SEK 12/6/2021
             */
             for (FileMetadata fm : latestVersion.getFileMetadatas()) {
-                FileMetadata newFm = new FileMetadata();
-                // TODO: 
-                // the "category" will be removed, shortly. 
-                // (replaced by multiple, tag-like categories of 
-                // type DataFileCategory) -- L.A. beta 10
-                //newFm.setCategory(fm.getCategory());
-                // yep, these are the new categories:
-                newFm.setCategories(fm.getCategories());
-                newFm.setDescription(fm.getDescription());
-                newFm.setLabel(fm.getLabel());
-                newFm.setDirectoryLabel(fm.getDirectoryLabel());
-                newFm.setRestricted(fm.isRestricted());
-                newFm.setDataFile(fm.getDataFile());
-                newFm.setDatasetVersion(dsv);
-                newFm.setProvFreeForm(fm.getProvFreeForm());
+                FileMetadata newFm = fm.createCopyInVersion(dsv);
                 newFm.setInPriorVersion(true);
 
                 //fmVarMet would be updated in DCT
@@ -410,8 +396,6 @@ private DatasetVersion createNewDatasetVersion(Template template, FileMetadata f
                         newFm.copyVarGroups(fm.getVarGroups());
                     }
                 }
-                
-                dsv.getFileMetadatas().add(newFm);
             }
 
             if (latestVersion.getTermsOfUseAndAccess()!= null){

src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java

@@ -706,23 +706,7 @@ public DatasetVersion cloneDatasetVersion(){
             */
 
             for (FileMetadata fm : this.getFileMetadatas()) {
-                FileMetadata newFm = new FileMetadata();
-                // TODO: 
-                // the "category" will be removed, shortly. 
-                // (replaced by multiple, tag-like categories of 
-                // type DataFileCategory) -- L.A. beta 10
-                //newFm.setCategory(fm.getCategory());
-                // yep, these are the new categories:
-                newFm.setCategories(fm.getCategories());
-                newFm.setDescription(fm.getDescription());
-                newFm.setLabel(fm.getLabel());
-                newFm.setDirectoryLabel(fm.getDirectoryLabel());
-                newFm.setRestricted(fm.isRestricted());
-                newFm.setDataFile(fm.getDataFile());
-                newFm.setDatasetVersion(dsv);
-                newFm.setProvFreeForm(fm.getProvFreeForm());
-                
-                dsv.getFileMetadatas().add(newFm);
+                fm.createCopyInVersion(dsv);
             }
 
             if (this.getTermsOfUseAndAccess()!= null){

src/main/java/edu/harvard/iq/dataverse/FileMetadata.java

@@ -154,21 +154,21 @@ public class FileMetadata implements Serializable {
     private Collection<VariableMetadata> variableMetadatas;
 
     /**
-     * Creates a copy of {@code this}, with identical business logic fields.
-     * E.g., {@link #label} would be duplicated; {@link #version} will not.
+     * Creates a copy of {@code this}, with identical business logic fields, making the bi-drectional connections to the specified version.
      * 
-     * @return A copy of {@code this}, except for the DB-related data.
+     * @return A copy of {@code this}
      */
-    public FileMetadata createCopy() {
+    public FileMetadata createCopyInVersion(DatasetVersion dsv) {
         FileMetadata fmd = new FileMetadata();
         fmd.setCategories(new LinkedList<>(getCategories()) );
         fmd.setDataFile( getDataFile() );
-        fmd.setDatasetVersion( getDatasetVersion() );
+        fmd.setDatasetVersion( dsv );
         fmd.setDescription( getDescription() );
         fmd.setLabel( getLabel() );
         fmd.setRestricted( isRestricted() );
         fmd.setDirectoryLabel(getDirectoryLabel());
-        
+        fmd.setProvFreeForm(getProvFreeForm());
+        dsv.getFileMetadatas().add(fmd);
         return fmd;
     }
 
@@ -245,38 +245,26 @@ public void setVarGroups(List<VarGroup> varGroups) {
 
     public List<DataFileCategory> getCategories() {
         if (fileCategories != null) {
-            /*
-             * fileCategories can sometimes be an
-             * org.eclipse.persistence.indirection.IndirectList When that happens, the
-             * comparator in the Collections.sort below is not called, possibly due to
-             * https://bugs.eclipse.org/bugs/show_bug.cgi?id=446236 which is Java 1.8+
-             * specific Converting to an ArrayList solves the problem, but the longer term
-             * solution may be in avoiding the IndirectList or moving to a new version of
-             * the jar it is in.
-             */
-            if (!(fileCategories instanceof ArrayList)) {
-                List<DataFileCategory> newDFCs = new ArrayList<DataFileCategory>();
-                for (DataFileCategory fdc : fileCategories) {
-                    newDFCs.add(fdc);
+            synchronized (this) {
+                if (!(fileCategories instanceof ArrayList)) {
+                    fileCategories = new ArrayList<>(fileCategories);
                 }
-                setCategories(newDFCs);
+                Collections.sort(fileCategories, FileMetadata.compareByNameWithSortCategories);
             }
-            Collections.sort(fileCategories, FileMetadata.compareByNameWithSortCategories);
         }
         return fileCategories;
     }
-    
-    public void setCategories(List<DataFileCategory> fileCategories) {
+
+    public synchronized void setCategories(List<DataFileCategory> fileCategories) {
         this.fileCategories = fileCategories; 
     }
-    
-    public void addCategory(DataFileCategory category) {
+
+    public synchronized void addCategory(DataFileCategory category) {
         if (fileCategories == null) {
             fileCategories = new ArrayList<>();
         }
         fileCategories.add(category);
     }
-
     /**
      * Retrieve categories 
      * @return 

src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDatasetVersionCommand.java

@@ -61,13 +61,11 @@ public DatasetVersion execute(CommandContext ctxt) throws CommandException {
 
         registerExternalVocabValuesIfAny(ctxt, newVersion);
 
-        List<FileMetadata> newVersionMetadatum = new ArrayList<>(latest.getFileMetadatas().size());
-        for ( FileMetadata fmd : latest.getFileMetadatas() ) {
-            FileMetadata fmdCopy = fmd.createCopy();
-            fmdCopy.setDatasetVersion(newVersion);
-            newVersionMetadatum.add( fmdCopy );
+        List<FileMetadata> latestVersionMetadata = latest.getFileMetadatas();
+        newVersion.setFileMetadatas(new ArrayList<>(latest.getFileMetadatas().size()));
+        for ( FileMetadata fmd : latestVersionMetadata ) {
+            fmd.createCopyInVersion(newVersion);
         }
-        newVersion.setFileMetadatas(newVersionMetadatum);
 
         //moving prepare Dataset here
         //because it includes validation and we need the validation

src/test/java/edu/harvard/iq/dataverse/DatasetVersionDifferenceTest.java

@@ -76,7 +76,8 @@ public void testDifferencing() {
         datasetVersion2.setVersionState(DatasetVersion.VersionState.DRAFT);
         datasetVersion2.setTermsOfUseAndAccess(new TermsOfUseAndAccess());
         datasetVersion2.getTermsOfUseAndAccess().setLicense(license);
-
+        datasetVersion.setFileMetadatas(new ArrayList<>());
+        
         // Published version's two files
         DataFile dataFile = new DataFile();
         dataFile.setId(1L);
@@ -88,19 +89,17 @@ public void testDifferencing() {
 
         FileMetadata fileMetadata2 = createFileMetadata(20L, datasetVersion, dataFile2, "file2.txt");
 
+        List<FileMetadata> fileMetadatas = new ArrayList<>(Arrays.asList(fileMetadata1, fileMetadata2));
+        datasetVersion.setFileMetadatas(fileMetadatas);
+        
         // Draft version - same two files with one label change
-        FileMetadata fileMetadata3 = fileMetadata1.createCopy();
+        FileMetadata fileMetadata3 = fileMetadata1.createCopyInVersion(datasetVersion2);
         fileMetadata3.setId(30L);
 
-        FileMetadata fileMetadata4 = fileMetadata2.createCopy();
+        FileMetadata fileMetadata4 = fileMetadata2.createCopyInVersion(datasetVersion2);
         fileMetadata4.setLabel("file3.txt");
         fileMetadata4.setId(40L);
 
-        List<FileMetadata> fileMetadatas = new ArrayList<>(Arrays.asList(fileMetadata1, fileMetadata2));
-        datasetVersion.setFileMetadatas(fileMetadatas);
-        List<FileMetadata> fileMetadatas2 = new ArrayList<>(Arrays.asList(fileMetadata3, fileMetadata4));
-        datasetVersion2.setFileMetadatas(fileMetadatas2);
-
         SimpleDateFormat dateFmt = new SimpleDateFormat("yyyyMMdd");
         Date publicationDate;
         try {