Merge branch 'IQSS:develop' into codemeta_structure

Author: doigl

doc/release-notes/11041-datatype-for-vtt.md

@@ -0,0 +1,43 @@
+### video subtitles (vtt files)
+
+The `IQSS/dataverse` PR sets the content type for new(!) files with extension `vtt` to `text/vtt`
+what is presented as "_Web Video Text Tracks_". The PR also enables full text indexing for these files,
+if [configured](https://guides.dataverse.org/en/latest/installation/config.html#solrfulltextindexing).
+
+The `gdcc/dataverse-previewer` PRs provide a new version of the video previewer. 
+The new previewer version presents `vtt` files as subtitles for videos,
+the naming convention is `<video-basename>.<language-tag>.vtt`.
+The previewer does not rely on the content type.
+A proper content type may hint users to ask permission for the subtitles together with a video.
+
+Existing files with extension `vtt` will keep content type `application/octet-stream` presented as "_Unknown_".
+The following query shows the number of files per extension with an "_Unknown_" content type:
+
+      SELECT substring(m.label from (length(label) - strpos(reverse(m.label), '.') + 2)) AS extension, COUNT(*) as count
+      FROM datafile f LEFT JOIN filemetadata m ON f.id = m.datafile_id
+      WHERE f.contenttype = 'application/octet-stream'
+      GROUP BY extension;
+
+If `vtt` does not appear in the result, you are done. 
+Otherwise, you may want to update the content type for existing files and reindex those datasets.
+
+First figure out which datasets would need [reindexing](https://guides.dataverse.org/en/latest/admin/solr-search-index.html#manual-reindexing):
+
+      select distinct
+        o.protocol, o.authority, o.identifier,
+        v.versionnumber, v.minorversionnumber, v.versionstate
+        from      datafile       f
+        left join filemetadata   m on f.id = m.datafile_id
+        left join datasetversion v on v.id = m.datasetversion_id
+        left join dvobject       o on o.id = v.dataset_id
+        WHERE contenttype = 'application/octet-stream' 
+        AND 'vtt' = substring(m.label from (length(label) - strpos(reverse(m.label), '.') + 2))
+        ;
+
+Then update the content type for the files:
+
+      UPDATE datafile SET contenttype = 'text/vtt' WHERE id IN (
+        SELECT datafile_id FROM filemetadata m
+        WHERE contenttype = 'application/octet-stream' 
+        AND 'vtt' = substring(m.label from (length(label) - strpos(reverse(m.label), '.') + 2))
+      );

doc/release-notes/11147-string-field-type.md

@@ -0,0 +1,5 @@
+The "string" type has been added as a new field type for metadata fields.
+
+In contrast to "text" fields, "string" fields are stored and indexed exactly as provided, without any text analysis or transformations.
+
+This field type is suitable for fields like IDs (e.g. ORCIDs) or enums, where exact matches are required when searching.

doc/release-notes/11292-update-tabular-tags-with-replace.md

@@ -0,0 +1,5 @@
+### Tabular Tags can now be replaced
+
+Previously the API POST /files/{id}/metadata/tabularTags could only add new tags to the tabular tags list. Now with the query parameter ?replace=true the list of tags will be replaced.
+
+See also [the guides](https://dataverse-guide--11359.org.readthedocs.build/en/11359/api/native-api.html#updating-file-tabular-tags), #11292, and #11359.

doc/sphinx-guides/source/admin/metadatacustomization.rst

@@ -144,6 +144,7 @@ Each of the three main sections own sets of properties:
 |                           |                                                        |                                                          | \• email              |
 |                           |                                                        |                                                          | \• text               |
 |                           |                                                        |                                                          | \• textbox            |
+|                           |                                                        |                                                          | \• string             |
 |                           |                                                        |                                                          | \• url                |
 |                           |                                                        |                                                          | \• int                |
 |                           |                                                        |                                                          | \• float              |
@@ -315,6 +316,12 @@ FieldType definitions
 |               | section of the Dataset + File      |
 |               | Management page in the User Guide. |
 +---------------+------------------------------------+
+| string        | Any text may be entered into this  |
+|               | field. The value is stored and     |
+|               | indexed exactly as provided,       |
+|               | without any text analysis or       |
+|               | transformations.                   |
++---------------+------------------------------------+
 | url           | If not empty, field must contain   |
 |               | a valid URL.                       |
 +---------------+------------------------------------+

doc/sphinx-guides/source/api/native-api.rst

@@ -4669,6 +4669,8 @@ Updating File Tabular Tags
 
 Updates the tabular tags for an existing tabular file where ``ID`` is the database id of the file to update or ``PERSISTENT_ID`` is the persistent id (DOI or Handle) of the file. Requires a ``jsonString`` expressing the tabular tag names.
 
+The list of "tabularTags" will be added to the existing list unless the optional ``replace=true`` query parameter is included. The inclusion of this parameter will cause the pre-existing tags to be deleted and the "tabularTags" to be added. Sending an empty list will remove all of the pre-existing tags.
+
 The JSON representation of tabular tags (``tags.json``) looks like this::
 
   {
@@ -4698,6 +4700,9 @@ The fully expanded example above (without environment variables) looks like this
   curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \
     "http://demo.dataverse.org/api/files/24/metadata/tabularTags" \
     -H "Content-type:application/json" --upload-file tags.json
+  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \
+    "http://demo.dataverse.org/api/files/24/metadata/tabularTags?replace=true" \
+    -H "Content-type:application/json" --upload-file tags.json
 
 A curl example using a ``PERSISTENT_ID``
 
@@ -4711,6 +4716,9 @@ A curl example using a ``PERSISTENT_ID``
   curl -H "X-Dataverse-key:$API_TOKEN" -X POST \
     "$SERVER_URL/api/files/:persistentId/metadata/tabularTags?persistentId=$PERSISTENT_ID" \
     -H "Content-type:application/json" --upload-file $FILE_PATH
+  curl -H "X-Dataverse-key:$API_TOKEN" -X POST \
+    "$SERVER_URL/api/files/:persistentId/metadata/tabularTags?persistentId=$PERSISTENT_ID&replace=true" \
+    -H "Content-type:application/json" --upload-file $FILE_PATH
 
 The fully expanded example above (without environment variables) looks like this:
 
@@ -4719,6 +4727,9 @@ The fully expanded example above (without environment variables) looks like this
   curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \
     "https://demo.dataverse.org/api/files/:persistentId/metadata/tabularTags?persistentId=doi:10.5072/FK2/AAA000" \
     -H "Content-type:application/json" --upload-file tags.json
+  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \
+    "https://demo.dataverse.org/api/files/:persistentId/metadata/tabularTags?persistentId=doi:10.5072/FK2/AAA000&replace=true" \
+    -H "Content-type:application/json" --upload-file tags.json
 
 Note that the specified tabular tags must be valid. The supported tags are:
 

src/main/java/edu/harvard/iq/dataverse/DatasetFieldType.java

@@ -36,8 +36,8 @@ public class DatasetFieldType implements Serializable, Comparable<DatasetFieldTy
      * The set of possible metatypes of the field. Used for validation and layout.
      */
     public enum FieldType {
-        TEXT, TEXTBOX, DATE, EMAIL, URL, FLOAT, INT, NONE
-    };    
+        TEXT, TEXTBOX, STRING, DATE, EMAIL, URL, FLOAT, INT, NONE
+    };
 
     @Id
     @GeneratedValue(strategy = GenerationType.IDENTITY)
@@ -558,6 +558,8 @@ public SolrField getSolrField() {
                 solrType = SolrField.SolrType.INTEGER;
             } else if (fieldType.equals(FieldType.FLOAT)) {
                 solrType = SolrField.SolrType.FLOAT;
+            } else if (fieldType.equals(FieldType.STRING)) {
+                solrType = SolrField.SolrType.STRING;
             }
 
             Boolean anyParentAllowsMultiplesBoolean = false;

src/main/java/edu/harvard/iq/dataverse/api/Files.java

@@ -1,5 +1,6 @@
 package edu.harvard.iq.dataverse.api;
 
+import com.google.api.client.util.Lists;
 import com.google.gson.Gson;
 import com.google.gson.JsonObject;
 import edu.harvard.iq.dataverse.*;
@@ -947,7 +948,7 @@ public Response setFileCategories(@Context ContainerRequestContext crc, @PathPar
     @AuthRequired
     @Path("{id}/metadata/tabularTags")
     @Produces(MediaType.APPLICATION_JSON)
-    public Response setFileTabularTags(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId, String jsonBody) {
+    public Response setFileTabularTags(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId, String jsonBody, @QueryParam("replace") boolean replaceData) {
         return response(req -> {
             DataFile dataFile = execCommand(new GetDataFileCommand(req, findDataFileOrDie(dataFileId)));
             if (!dataFile.isTabularData()) {
@@ -957,6 +958,9 @@ public Response setFileTabularTags(@Context ContainerRequestContext crc, @PathPa
             try (StringReader stringReader = new StringReader(jsonBody)) {
                 jsonObject = Json.createReader(stringReader).readObject();
                 JsonArray requestedTabularTagsJson = jsonObject.getJsonArray("tabularTags");
+                if (replaceData) {
+                    dataFile.setTags(Lists.newArrayList());
+                }
                 for (JsonValue jsonValue : requestedTabularTagsJson) {
                     JsonString jsonString = (JsonString) jsonValue;
                     try {

src/main/java/propertyFiles/MimeTypeDetectionByFileExtension.properties

@@ -16,6 +16,7 @@ mat=application/matlab-mat
 md=text/markdown
 mp3=audio/mp3
 m4a=audio/mp4
+vtt=text/vtt
 nii=image/nii
 nc=application/netcdf
 ods=application/vnd.oasis.opendocument.spreadsheet

src/main/java/propertyFiles/MimeTypeDisplay.properties

@@ -217,6 +217,7 @@ video/x-m4v=MPEG-4 Video
 video/ogg=OGG Video
 video/quicktime=Quicktime Video
 video/webm=WebM Video
+text/vtt=Web Video Text Tracks
 # Network Data
 text/xml-graphml=GraphML Network Data
 # 3D Data

src/main/java/propertyFiles/MimeTypeFacets.properties

@@ -30,6 +30,7 @@ text/richtext=Text
 text/turtle=Text
 application/xml=Text
 text/xml=Text
+text/vtt=Text
 # Code
 text/x-c=Code
 text/x-c++src=Code