Merge branch 'develop' into 3776-public-only

Author: mheppler

doc/sphinx-guides/source/conf.py

@@ -64,9 +64,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '4.7'
+version = '4.7.1'
 # The full version, including alpha/beta/rc tags.
-release = '4.7'
+release = '4.7.1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

doc/sphinx-guides/source/index.rst

@@ -3,10 +3,10 @@
    You can adapt this file completely to your liking, but it should at least
    contain the root `toctree` directive.
 
-Dataverse 4.7 Guides
-====================
+Dataverse 4.7.1 Guides
+======================
 
-These guides are for the most recent version of Dataverse. For the guides for **version 4.6.2** please go `here <http://guides.dataverse.org/en/4.6.2/>`_.
+These guides are for the most recent version of Dataverse. For the guides for **version 4.7** please go `here <http://guides.dataverse.org/en/4.7/>`_.
 
 .. toctree::
   :glob:

pom.xml

@@ -4,7 +4,7 @@
 
     <groupId>edu.harvard.iq</groupId>
     <artifactId>dataverse</artifactId>
-    <version>4.7</version>
+    <version>4.7.1</version>
     <packaging>war</packaging>
 
     <name>dataverse</name>

scripts/database/upgrades/upgrade_v4.7_to_v4.7.1.sql

@@ -4,4 +4,32 @@
 ALTER TABLE authenticateduser ADD COLUMN createdtime TIMESTAMP NOT NULL DEFAULT '01-01-2000 00:00:00';
 ALTER TABLE authenticateduser ADD COLUMN lastlogintime TIMESTAMP DEFAULT NULL;
 ALTER TABLE authenticateduser ADD COLUMN lastapiusetime TIMESTAMP DEFAULT NULL;
-ALTER TABLE authenticateduser DROP COLUMN modificationtime;
+ALTER TABLE authenticateduser DROP COLUMN modificationtime;
+
+-- Removing authenticated builtin users who do not exist in the builtin table because they were created through faulty validation
+-- creates view containing authentication ids that you will be deleting
+CREATE TEMP VIEW useridstodelete AS (SELECT DISTINCT a.id FROM authenticateduserlookup al, authenticateduser a WHERE al.authenticateduser_id = a.id AND al.authenticationproviderid = 'builtin'  AND a.useridentifier NOT IN (SELECT username FROM builtinuser));
+-- commands to remove the users from the appropriate tables
+DELETE FROM confirmemaildata WHERE authenticateduser_id IN (SELECT * FROM useridstodelete);
+DELETE FROM usernotification WHERE user_id IN (SELECT * FROM useridstodelete);
+DELETE FROM guestbookresponse WHERE authenticateduser_id IN (SELECT * FROM useridstodelete);
+DELETE FROM authenticateduserlookup WHERE authenticateduser_id IN (SELECT * FROM useridstodelete);
+DELETE FROM authenticateduser WHERE id NOT IN (SELECT authenticateduser_id FROM authenticateduserlookup);
+
+/* 
+Add validationFormat to DatasetFieldType to 
+ */
+ALTER TABLE datasetfieldtype
+ADD COLUMN validationFormat character varying(255);
+
+/*
+for testing display format 
+This adds a display format that links out to an outside site. The format of the #VALUE is
+four characters alpha numeric (3fki works) 
+
+update datasetfieldtype 
+set displayformat = '<a target="_blank" href="http://www.rcsb.org/pdb/explore/explore.do?structureId=#VALUE">PDB (RCSB) #VALUE</a>',
+fieldType= 'TEXT'
+where id = xxx;
+
+*/

src/main/java/Bundle.properties

@@ -184,6 +184,8 @@ user.signup.otherLogInOptions.tip= <a href="/loginpage.xhtml" title="Dataverse L
 user.username.illegal.tip=Between 2-60 characters, and can use "a-z", "0-9", "_" for your username.
 user.username=Username
 user.username.taken=This username is already taken.
+user.username.invalid=This username contains an invalid character or is outside the length requirement (2-60 characters). 
+user.username.valid=Create a valid username of 2 to 60 characters in length containing letters (a-Z), numbers (0-9), dashes (-), underscores (_), and periods (.).
 user.noPasswd=No Password
 user.currentPasswd=Current Password
 user.currentPasswd.tip=Please enter the current password for this account.
@@ -206,7 +208,7 @@ user.acccountterms.iagree=I have read and accept the Dataverse General Terms of
 user.createBtn=Create Account
 user.updatePassword.welcome=Welcome to Dataverse {0}, {1}
 user.updatePassword.warning=With the release of our new Dataverse 4.0 upgrade, the password requirements and General Terms of Use have updated. As this is the first time you are using Dataverse since the update, you need to create a new password and agree to the new General Terms of Use.
-user.updatePassword.password=Create a password that is minimum six characters long and uses at least one letter or number.
+user.updatePassword.password=Create a password that is minimum six characters long and uses at least one letter and number.
 authenticationProvidersAvailable.tip={0}There are no active authentication providers{1}If you are a system administrator, please enable one using the API.{2}If you are not a system administrator, please contact the one for your institution.
 
 #loginpage.xhtml

src/main/java/edu/harvard/iq/dataverse/DatasetField.java

@@ -274,7 +274,7 @@ public List<String> getValues() {
         List returnList = new ArrayList();
         if (!datasetFieldValues.isEmpty()) {
             for (DatasetFieldValue dsfv : datasetFieldValues) {
-                returnList.add(dsfv.getValue());
+                returnList.add(dsfv.getDisplayValue());
             }
         } else {
             for (ControlledVocabularyValue cvv : controlledVocabularyValues) {

src/main/java/edu/harvard/iq/dataverse/DatasetFieldCompoundValue.java

@@ -5,6 +5,7 @@
  */
 package edu.harvard.iq.dataverse;
 
+import edu.harvard.iq.dataverse.util.MarkupChecker;
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Comparator;
@@ -143,15 +144,18 @@ public Map<DatasetField,String> getDisplayValueMap() {
                 if (StringUtils.isBlank(format)) {
                     format = "#VALUE";
                 }
-
+                String sanitizedValue = childDatasetField.getDatasetFieldType().isSanitizeHtml() ? MarkupChecker.sanitizeBasicHTML(childDatasetField.getValue()) :  childDatasetField.getValue();
+                if (!childDatasetField.getDatasetFieldType().isSanitizeHtml() && childDatasetField.getDatasetFieldType().isEscapeOutputText()){
+                    sanitizedValue = MarkupChecker.stripAllTags(sanitizedValue);
+                }
                 // replace the special values in the format (note: we replace #VALUE last since we don't
                 // want any issues if the value itself has #NAME in it)
                 String displayValue = format
                         .replace("#NAME", childDatasetField.getDatasetFieldType().getTitle())
                         //todo: this should be handled in more generic way for any other text that can then be internationalized
                         // if we need to use replaceAll for regexp, then make sure to use: java.util.regex.Matcher.quoteReplacement(<target string>)
                         .replace("#EMAIL", ResourceBundle.getBundle("Bundle").getString("dataset.email.hiddenMessage"))
-                        .replace("#VALUE", childDatasetField.getValue());
+                        .replace("#VALUE",  sanitizedValue );
 
                 fieldMap.put(childDatasetField,displayValue);
             }

src/main/java/edu/harvard/iq/dataverse/DatasetFieldType.java

@@ -46,10 +46,7 @@ public Long getId() {
     public void setId(Long id) {
         this.id = id;
     }
-    
-    public String getIdString(){
-        return id.toString();
-    }
+
 
     /**
      * The internal, DDI-like name, no spaces, etc.
@@ -83,6 +80,8 @@ public String getIdString(){
      * A watermark to be displayed in the UI.
      */
     private String watermark;
+    
+    private String validationFormat;
 
     @OneToMany(mappedBy = "datasetFieldType")
     private Set<DataverseFacet> dataverseFacets;
@@ -164,8 +163,23 @@ public void setDisplayFormat(String displayFormat) {
         this.displayFormat = displayFormat;
     }
 
+    public Boolean isSanitizeHtml(){
+        if (this.fieldType.equals(FieldType.URL)){
+            return true;
+        }
+        return this.fieldType.equals(FieldType.TEXTBOX);
+    }
+    
+    public Boolean isEscapeOutputText(){
+        if (this.fieldType.equals(FieldType.URL)){
+            return false;
+        }
+        if (this.fieldType.equals(FieldType.TEXTBOX)){
+            return false;
+        }
+        return !(this.fieldType.equals(FieldType.TEXT) &&  this.displayFormat != null &&this.displayFormat.contains("<a"));
+    }
 
-
     public String getName() {
         return name;
     }
@@ -239,6 +253,14 @@ public boolean isFacetable() {
     public void setFacetable(boolean facetable) {
         this.facetable = facetable;
     }
+    
+    public String getValidationFormat() {
+        return validationFormat;
+    }
+
+    public void setValidationFormat(String validationFormat) {
+        this.validationFormat = validationFormat;
+    }
 
     /**
      * Determines whether this field type is displayed in the form when creating

src/main/java/edu/harvard/iq/dataverse/DatasetFieldValidator.java

@@ -31,7 +31,12 @@ public boolean isValid(DatasetField value, ConstraintValidatorContext context) {
         }
         if (((dsfType.isPrimitive() && dsfType.isRequired())  || (dsfType.isPrimitive() && value.isRequired())) 
                 && StringUtils.isBlank(value.getValue())) {
-            context.buildConstraintViolationWithTemplate(dsfType.getDisplayName() + " is required.").addConstraintViolation();
+            try{
+                context.buildConstraintViolationWithTemplate(dsfType.getDisplayName() + " is required.").addConstraintViolation();
+            } catch (NullPointerException npe){
+                //if there's no context for the error we can't put it anywhere....
+            }
+
             return false;
         }
         return true;

src/main/java/edu/harvard/iq/dataverse/DatasetFieldValue.java

@@ -6,8 +6,10 @@
 
 package edu.harvard.iq.dataverse;
 
+import edu.harvard.iq.dataverse.util.MarkupChecker;
 import java.io.Serializable;
 import java.util.Comparator;
+import java.util.ResourceBundle;
 import javax.persistence.Column;
 import javax.persistence.Entity;
 import javax.persistence.GeneratedValue;
@@ -18,6 +20,7 @@
 import javax.persistence.ManyToOne;
 import javax.persistence.Table;
 import javax.persistence.Transient;
+import org.apache.commons.lang.StringUtils;
 
 /**
  *
@@ -85,6 +88,31 @@ public String getValueForEdit() {
     public void setValueForEdit(String value) {
         this.value = value;
     }
+    
+    public String getDisplayValue() {
+        String retVal = "";
+        if (!StringUtils.isBlank(this.getValue()) && !DatasetField.NA_VALUE.equals(this.getValue())) {
+            String format = this.datasetField.getDatasetFieldType().getDisplayFormat();
+            if (StringUtils.isBlank(format)) {
+                format = "#VALUE";
+            }           
+            String sanitizedValue = !this.datasetField.getDatasetFieldType().isSanitizeHtml() ? this.getValue() :  MarkupChecker.sanitizeBasicHTML(this.getValue());    
+            
+                if (!this.datasetField.getDatasetFieldType().isSanitizeHtml() && this.datasetField.getDatasetFieldType().isEscapeOutputText()){
+                    sanitizedValue = MarkupChecker.stripAllTags(sanitizedValue);
+                }
+            
+            // replace the special values in the format (note: we replace #VALUE last since we don't
+            // want any issues if the value itself has #NAME in it)
+            String displayValue = format
+                    .replace("#NAME",  this.datasetField.getDatasetFieldType().getTitle() == null ? "" : this.datasetField.getDatasetFieldType().getTitle())
+                    .replace("#EMAIL", ResourceBundle.getBundle("Bundle").getString("dataset.email.hiddenMessage"))
+                    .replace("#VALUE", sanitizedValue);
+            retVal = displayValue;
+        }
+
+        return retVal;
+    }
 
     public int getDisplayOrder() {
         return displayOrder;