Merge branch 'develop' into 11447-mydata-retrieve-empty-result-set

Author: stevenwinship

doc/release-notes/12258-publish-submit-contains-files.md

@@ -0,0 +1,2 @@
+## Bug: Publish and Submit for review must contain files
+When `requireFilesToPublishDataset` is set on a Dataverse a Dataset must contain files to be published or submitted for review. This fix make sure the dataset version contains files, and not just the dataset. It also adds this check to the `Submit for Review` functionality.

doc/release-notes/7956-show_original_file_format.md

@@ -0,0 +1,2 @@
+This release changes the display behavior of the file, dataset and collection ("dataverse") pages for tabular files. They will show the original file name and information, and will allow editing of the original file name. The ingested *.tab version is still available from the download menu. 
+ 

doc/sphinx-guides/source/api/changelog.rst

@@ -12,6 +12,10 @@ v6.11
 
 - The GET /api/mydata/retrieve, if the search returns no data, now includes the "data" block with 0 results. The message that was returned in "error_message" will be returned in "message" and the "success" will be `true`. All other errors will continue to reply with "success":false and the error message in "error_message".
 
+- The following API will now return ``403`` if the ``requireFilesToPublishDataset`` flag is set and the dataset version contains 0 files.
+
+  - **/api/datasets/{Id}/submitForReview**
+
 - The Croissant :ref:`metadata export format <metadata-export-formats>` has been updated from version 1.0 to 1.1, which is reflected in the ``conformsTo`` property. The unused ``wd`` property has been dropped.
 
 v6.10

doc/sphinx-guides/source/api/native-api.rst

@@ -1299,7 +1299,7 @@ The following attributes are supported:
 * ``description`` Description
 * ``affiliation`` Affiliation
 * ``filePIDsEnabled`` ("true" or "false") Restricted to use by superusers and only when the :ref:`:AllowEnablingFilePIDsPerCollection <:AllowEnablingFilePIDsPerCollection>` setting is true. Enables or disables registration of file-level PIDs in datasets within the collection (overriding the instance-wide setting).
-* ``requireFilesToPublishDataset`` ("true" or "false") Restricted to use by superusers. Defines if Dataset needs files in order to be published.  If not set the determination will be made through inheritance by checking the owners of this collection. Publishing by a superusers will not be blocked.
+* ``requireFilesToPublishDataset`` ("true" or "false") Restricted to use by superusers. Defines if Dataset version needs files in order to be published or submitted for review.  If not set the determination will be made through inheritance by checking the owners of this collection. Publishing by a superusers will not be blocked.
 * ``allowedDatasetTypes`` Restricted to use by superusers. By default "dataset" is implied. Pass a comma-separated list of dataset types (e.g. "dataset,software"). You cannot unset this attribute so if you want to delete a dataset type, set ``allowedDatasetTypes`` to a dataset type you won't be deleting. See also :ref:`dataset-types`.
 
 See also :ref:`update-dataverse-api`.

src/main/java/edu/harvard/iq/dataverse/DataCitation.java

@@ -141,7 +141,7 @@ public DataCitation(FileMetadata fm, boolean direct) {
         getCommonValuesFrom(dsv);
 
         // file Title for direct File citation
-        fileTitle = fm.getLabel();
+        fileTitle = fm.getLabelForOriginal();
         DataFile df = fm.getDataFile();
 
         // File description

src/main/java/edu/harvard/iq/dataverse/DataFile.java

@@ -547,6 +547,11 @@ public Long getOriginalFileSize() {
         return null;
     }
 
+    public String getFriendlyOriginalFileSize() {
+        Long size = (getOriginalFileSize()==null) ? filesize : getOriginalFileSize();
+        return getFriendlySize(size);
+    }
+    
     public String getOriginalFileName() {
         if (isTabularData()) {
             DataTable dataTable = getDataTable();
@@ -559,7 +564,7 @@ public String getOriginalFileName() {
     }
 
 
-    private String getDerivedOriginalFileName() {
+    public String getDerivedOriginalFileName() {
         FileMetadata fm = getFileMetadata();
         String filename = fm.getLabel();
         String originalExtension = FileUtil.generateOriginalExtension(getOriginalFileFormat());
@@ -709,8 +714,12 @@ public void setFilesize(long filesize) {
      * @return 
      */
     public String getFriendlySize() {
-        if (filesize != null) {
-            return FileSizeChecker.bytesToHumanReadable(filesize);
+        return getFriendlySize(filesize);
+    }
+    
+    private String getFriendlySize(Long size) {
+        if (size != null) {
+            return FileSizeChecker.bytesToHumanReadable(size);
         } else {
             return BundleUtil.getStringFromBundle("file.sizeNotAvailable");
         }

src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java

@@ -835,6 +835,18 @@ public String getReminderString(Dataset dataset, boolean canPublishDataset, bool
 
         String reminderString;
 
+        if (dataset.getOwner().getEffectiveRequiresFilesToPublishDataset()) {
+            List<FileMetadata> files = dataset.getLatestVersion().getFileMetadatas();
+            if (files.size() < 1) {
+                if (canPublishDataset) {
+                    reminderString = BundleUtil.getStringFromBundle("dataset.mayNotPublish.FilesRequired");
+                } else {
+                    reminderString = BundleUtil.getStringFromBundle("dataset.mayNotSubmitForReview.FilesRequired");
+                }
+                return reminderString;
+            }
+        }
+
         if (canPublishDataset) {
             reminderString = BundleUtil.getStringFromBundle("dataset.message.publish.warning");
         } else {

src/main/java/edu/harvard/iq/dataverse/FileLabelValidator.java

@@ -0,0 +1,34 @@
+package edu.harvard.iq.dataverse;
+
+import jakarta.validation.ConstraintValidator;
+import jakarta.validation.ConstraintValidatorContext;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class FileLabelValidator implements ConstraintValidator<ValidateDataFileLabel, String> {
+
+    @Override
+    public void initialize(ValidateDataFileLabel constraintAnnotation) {
+        
+    }
+
+    @Override
+    public boolean isValid(String value, ConstraintValidatorContext context) {
+        return isFileLabelValid(value, context);
+        
+    }
+    
+    public static boolean isFileLabelValid(String value, ConstraintValidatorContext context) {
+
+        if (value == null || value.isEmpty()) {
+            return true;
+        }
+        String validCharacters = "^[^:<>;#/\"\\*\\|\\?\\\\]*$";
+        Pattern p = Pattern.compile(validCharacters);
+        Matcher m = p.matcher(value);
+        return m.matches();
+
+    }
+    
+}

src/main/java/edu/harvard/iq/dataverse/FileMetadata.java

@@ -39,6 +39,7 @@
 import jakarta.persistence.NamedNativeQuery;
 import jakarta.persistence.OneToMany;
 import jakarta.persistence.OrderBy;
+import jakarta.persistence.PostLoad;
 import jakarta.persistence.SqlResultSetMapping;
 import jakarta.persistence.Table;
 import jakarta.persistence.Transient;
@@ -153,6 +154,18 @@ public class FileMetadata implements Serializable {
     @OneToMany (mappedBy="fileMetadata", cascade={ CascadeType.REMOVE, CascadeType.MERGE,CascadeType.PERSIST})
     private Collection<VariableMetadata> variableMetadatas;
 
+    // A transient field is needed for JSF UI - validation errors on label do not get routed directly to the input for labelNoExtension, causing rollback.
+    // With a separate transient field kept in sync with label, the validation can be done on the labelNoExtension field, which avoids the issue and allows proper validation.
+    @Transient
+    @ValidateDataFileLabel(message = "{filename.illegalCharacters}")
+    String labelNoExtension;
+    
+    // Initialize the labelNoExtension from label after loading the entity
+    @PostLoad
+    public void postLoad() {
+        getLabelNoExtension();
+    }
+
     /**
      * Creates a copy of {@code this}, with identical business logic fields, making the bi-drectional connections to the specified version.
      * 
@@ -173,13 +186,46 @@ public FileMetadata createCopyInVersion(DatasetVersion dsv) {
     }
 
     public String getLabel() {
+        getLabelNoExtension();
         return label;
     }
 
     public void setLabel(String label) {
         this.label = label;
+        getLabelNoExtension();
+    }
+
+
+    public String getLabelNoExtension() {
+        int last = label.lastIndexOf(".");
+        labelNoExtension = (last == -1) ? label : label.substring(0, last);
+        return labelNoExtension;
+    }
+
+    public String getOriginalExtension() {
+        String origFilename = getLabelForOriginal();
+        int last = origFilename.lastIndexOf(".");
+        return (last == -1) ? "" : origFilename.substring(last);
     }
 
+    public void setLabelNoExtension(String name) {
+        labelNoExtension = name;
+        int last = this.label.lastIndexOf(".");
+        if (last == -1) {
+            this.label = name;
+        } else {
+            this.label = name + this.label.substring(last);
+        }
+    }
+    
+    public String getLabelForOriginal() {
+        if(dataFile.isTabularData()) {
+            return dataFile.getDerivedOriginalFileName();
+        } else {
+            return label;
+        }
+    }
+    
     public FileMetadata() {
         variableMetadatas = new ArrayList<VariableMetadata>();
         varGroups = new ArrayList<VarGroup>();

src/main/java/edu/harvard/iq/dataverse/PermissionsWrapper.java

@@ -12,6 +12,7 @@
 import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
 import edu.harvard.iq.dataverse.engine.command.impl.*;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.logging.Logger;
 import jakarta.ejb.EJB;
@@ -253,9 +254,38 @@ public boolean canIssueDeleteDatasetCommand(DvObject dvo){
 
     // PUBLISH DATASET
     public boolean canIssuePublishDatasetCommand(DvObject dvo){
+        User u = session.getUser();
+        if (u != null && u.isSuperuser()) {
+            return true;
+        }
+        // Return false if dataset has 0 files and user want to 'publish' or 'submit for review' and 'publish dataset requires files' flag is set
+        if (dvo.isInstanceofDataset()) {
+            Dataverse dv =((Dataset)dvo).getOwner();
+            if (dv != null) {
+                List<FileMetadata> metadataList = ((Dataset) dvo).getLatestVersion().getFileMetadatas();
+                if (metadataList.size() == 0 && dv.getEffectiveRequiresFilesToPublishDataset()) {
+                    return false;
+                }
+            }
+        }
         return canIssueCommand(dvo, PublishDatasetCommand.class);
     }
-    
+
+    // SUBMIT DATASET FOR REVIEW
+    public boolean canIssueSubmitDatasetForReviewCommand(DvObject dvo){
+        // Return false if dataset has 0 files and user want to 'publish' or 'submit for review' and 'publish dataset requires files' flag is set
+        if (dvo.isInstanceofDataset()) {
+            Dataverse dv =((Dataset)dvo).getOwner();
+            if (dv != null) {
+                List<FileMetadata> metadataList = ((Dataset) dvo).getLatestVersion().getFileMetadatas();
+                if (metadataList.size() == 0 && dv.getEffectiveRequiresFilesToPublishDataset()) {
+                    return false;
+                }
+            }
+        }
+        return canIssueCommand(dvo, SubmitDatasetForReviewCommand.class);
+    }
+
     // For the dataverse_header fragment (and therefore, most of the pages),
     // we need to know if authenticated users can add dataverses and datasets to the
     // root collection. For the "Add Data" menu further in the search include fragment