Merge branch 'develop' into 11912-edit-template-api

Author: sekmiller

doc/release-notes/7956-show_original_file_format.md

@@ -0,0 +1,2 @@
+This release changes the display behavior of the file, dataset and collection ("dataverse") pages for tabular files. They will show the original file name and information, and will allow editing of the original file name. The ingested *.tab version is still available from the download menu. 
+ 

doc/sphinx-guides/source/api/changelog.rst

@@ -10,7 +10,7 @@ This API changelog is experimental and we would love feedback on its usefulness.
 v6.11
 -----
 
-- The Croissant :ref:`metadata export format <metadata-export-formats>` has been updated from version 1.0 to 1.1, which is reflected in the ``conformsTo`` property. ``@vocab`` and ``sc`` properties now use "http" as `recommended <https://github.com/mlcommons/croissant/pull/929#pullrequestreview-3079137662>`_. The unused ``wd`` property has been dropped.
+- The Croissant :ref:`metadata export format <metadata-export-formats>` has been updated from version 1.0 to 1.1, which is reflected in the ``conformsTo`` property. The unused ``wd`` property has been dropped.
 
 v6.10
 -----

src/main/java/edu/harvard/iq/dataverse/DataCitation.java

@@ -141,7 +141,7 @@ public DataCitation(FileMetadata fm, boolean direct) {
         getCommonValuesFrom(dsv);
 
         // file Title for direct File citation
-        fileTitle = fm.getLabel();
+        fileTitle = fm.getLabelForOriginal();
         DataFile df = fm.getDataFile();
 
         // File description

src/main/java/edu/harvard/iq/dataverse/DataFile.java

@@ -547,6 +547,11 @@ public Long getOriginalFileSize() {
         return null;
     }
 
+    public String getFriendlyOriginalFileSize() {
+        Long size = (getOriginalFileSize()==null) ? filesize : getOriginalFileSize();
+        return getFriendlySize(size);
+    }
+    
     public String getOriginalFileName() {
         if (isTabularData()) {
             DataTable dataTable = getDataTable();
@@ -559,7 +564,7 @@ public String getOriginalFileName() {
     }
 
 
-    private String getDerivedOriginalFileName() {
+    public String getDerivedOriginalFileName() {
         FileMetadata fm = getFileMetadata();
         String filename = fm.getLabel();
         String originalExtension = FileUtil.generateOriginalExtension(getOriginalFileFormat());
@@ -709,8 +714,12 @@ public void setFilesize(long filesize) {
      * @return 
      */
     public String getFriendlySize() {
-        if (filesize != null) {
-            return FileSizeChecker.bytesToHumanReadable(filesize);
+        return getFriendlySize(filesize);
+    }
+    
+    private String getFriendlySize(Long size) {
+        if (size != null) {
+            return FileSizeChecker.bytesToHumanReadable(size);
         } else {
             return BundleUtil.getStringFromBundle("file.sizeNotAvailable");
         }

src/main/java/edu/harvard/iq/dataverse/FileLabelValidator.java

@@ -0,0 +1,34 @@
+package edu.harvard.iq.dataverse;
+
+import jakarta.validation.ConstraintValidator;
+import jakarta.validation.ConstraintValidatorContext;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class FileLabelValidator implements ConstraintValidator<ValidateDataFileLabel, String> {
+
+    @Override
+    public void initialize(ValidateDataFileLabel constraintAnnotation) {
+        
+    }
+
+    @Override
+    public boolean isValid(String value, ConstraintValidatorContext context) {
+        return isFileLabelValid(value, context);
+        
+    }
+    
+    public static boolean isFileLabelValid(String value, ConstraintValidatorContext context) {
+
+        if (value == null || value.isEmpty()) {
+            return true;
+        }
+        String validCharacters = "^[^:<>;#/\"\\*\\|\\?\\\\]*$";
+        Pattern p = Pattern.compile(validCharacters);
+        Matcher m = p.matcher(value);
+        return m.matches();
+
+    }
+    
+}

src/main/java/edu/harvard/iq/dataverse/FileMetadata.java

@@ -39,6 +39,7 @@
 import jakarta.persistence.NamedNativeQuery;
 import jakarta.persistence.OneToMany;
 import jakarta.persistence.OrderBy;
+import jakarta.persistence.PostLoad;
 import jakarta.persistence.SqlResultSetMapping;
 import jakarta.persistence.Table;
 import jakarta.persistence.Transient;
@@ -153,6 +154,18 @@ public class FileMetadata implements Serializable {
     @OneToMany (mappedBy="fileMetadata", cascade={ CascadeType.REMOVE, CascadeType.MERGE,CascadeType.PERSIST})
     private Collection<VariableMetadata> variableMetadatas;
 
+    // A transient field is needed for JSF UI - validation errors on label do not get routed directly to the input for labelNoExtension, causing rollback.
+    // With a separate transient field kept in sync with label, the validation can be done on the labelNoExtension field, which avoids the issue and allows proper validation.
+    @Transient
+    @ValidateDataFileLabel(message = "{filename.illegalCharacters}")
+    String labelNoExtension;
+    
+    // Initialize the labelNoExtension from label after loading the entity
+    @PostLoad
+    public void postLoad() {
+        getLabelNoExtension();
+    }
+
     /**
      * Creates a copy of {@code this}, with identical business logic fields, making the bi-drectional connections to the specified version.
      * 
@@ -173,13 +186,46 @@ public FileMetadata createCopyInVersion(DatasetVersion dsv) {
     }
 
     public String getLabel() {
+        getLabelNoExtension();
         return label;
     }
 
     public void setLabel(String label) {
         this.label = label;
+        getLabelNoExtension();
+    }
+
+
+    public String getLabelNoExtension() {
+        int last = label.lastIndexOf(".");
+        labelNoExtension = (last == -1) ? label : label.substring(0, last);
+        return labelNoExtension;
+    }
+
+    public String getOriginalExtension() {
+        String origFilename = getLabelForOriginal();
+        int last = origFilename.lastIndexOf(".");
+        return (last == -1) ? "" : origFilename.substring(last);
     }
 
+    public void setLabelNoExtension(String name) {
+        labelNoExtension = name;
+        int last = this.label.lastIndexOf(".");
+        if (last == -1) {
+            this.label = name;
+        } else {
+            this.label = name + this.label.substring(last);
+        }
+    }
+    
+    public String getLabelForOriginal() {
+        if(dataFile.isTabularData()) {
+            return dataFile.getDerivedOriginalFileName();
+        } else {
+            return label;
+        }
+    }
+    
     public FileMetadata() {
         variableMetadatas = new ArrayList<VariableMetadata>();
         varGroups = new ArrayList<VarGroup>();

src/main/java/edu/harvard/iq/dataverse/ValidateDataFileLabel.java

@@ -0,0 +1,25 @@
+package edu.harvard.iq.dataverse;
+
+import jakarta.validation.Constraint;
+import jakarta.validation.Payload;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+
+import static java.lang.annotation.ElementType.FIELD;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+@Target({FIELD})
+@Retention(RUNTIME)
+@Constraint(validatedBy = {FileLabelValidator.class})
+@Documented
+public @interface ValidateDataFileLabel {
+    
+    String message() default "Failed Validation for Validate Data File Label";
+
+    Class<?>[] groups() default {};
+
+    Class<? extends Payload>[] payload() default {};
+    
+}

src/main/java/edu/harvard/iq/dataverse/export/croissant/CroissantExportUtil.java

@@ -18,6 +18,7 @@
 import java.util.Map;
 import org.apache.commons.text.StringEscapeUtils;
 
+// Validate with src/test/resources/croissant/validate.sh
 public class CroissantExportUtil {
 
     public static void exportDataset(
@@ -31,7 +32,7 @@ public static void exportDataset(
             {
                 "@context": {
                     "@language": "en",
-                    "@vocab": "http://schema.org/",
+                    "@vocab": "https://schema.org/",
                     "citeAs": "cr:citeAs",
                     "column": "cr:column",
                     "conformsTo": "dct:conformsTo",
@@ -70,7 +71,7 @@ public static void exportDataset(
                     "repeated": "cr:repeated",
                     "replace": "cr:replace",
                     "samplingRate": "cr:samplingRate",
-                    "sc": "http://schema.org/",
+                    "sc": "https://schema.org/",
                     "separator": "cr:separator",
                     "source": "cr:source",
                     "subField": "cr:subField",

src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java

@@ -1611,7 +1611,7 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set<Long
 
                     String filenameCompleteFinal = "";
                     if (fileMetadata != null) {
-                        String filenameComplete = fileMetadata.getLabel();
+                        String filenameComplete = fileMetadata.getLabelForOriginal();
                         if (filenameComplete != null) {
                             String filenameWithoutExtension = "";
                             // String extension = "";

src/main/webapp/dataset.xhtml

@@ -2034,7 +2034,8 @@
                         <ui:fragment rendered="#{valid and settingsWrapper.isHasPublishDatasetDisclaimerText()}">
                             <div class="form-group">
                                 <p:selectBooleanCheckbox class="text-danger" id="publishDisclaimer"
-                                                         itemLabel="#{settingsWrapper.getPublishDatasetDisclaimerText()}"
+                                                         itemLabel="#{MarkupChecker:sanitizeAdvancedHTML(settingsWrapper.getPublishDatasetDisclaimerText())}"
+                                                         escape="false"
                                                          value="#{DatasetPage.publishDisclaimerAcknowledged}">
                                     <p:ajax event="change" update="releaseDatasetButton"/>
                                 </p:selectBooleanCheckbox>