File Metadata Update - Empty values clear fields

doc/release-notes/11392-edit-file-metadata-empty-values.md

@@ -0,0 +1,7 @@
+### Edit File Metadata empty values should clear data
+
+Previously the API POST /files/{id}/metadata would ignore fields with empty values. Now the API updates the fields with the empty values essentially clearing the data. Missing fields will still be ignored.
+
+An optional query parameter (sourceInternalVersionTimestamp) was added to ensure the metadata update doesn't overwrite stale data.
+
+See also [the guides](https://dataverse-guide--11359.org.readthedocs.build/en/11359/api/native-api.html#updating-file-metadata), #11392, and #11359.

doc/sphinx-guides/source/api/changelog.rst

@@ -11,6 +11,7 @@ v6.7
 ----
 
 - An undocumented :doc:`search` parameter called "show_my_data" has been removed. It was never exercised by tests and is believed to be unused. API users should use the :ref:`api-mydata` API instead.
+- For POST /api/files/{id}/metadata passing an empty string (“description”:””)  or array (“categories”:[]) will no longer be ignored. Empty fields will now clear out the values in the file's metadata. To ignore the fields simply do not include them in the Json string.
 
 v6.6
 ----

doc/sphinx-guides/source/api/native-api.rst

@@ -4612,6 +4612,8 @@ Updating File Metadata
 
 Updates the file metadata for an existing file where ``ID`` is the database id of the file to update or ``PERSISTENT_ID`` is the persistent id (DOI or Handle) of the file. Requires a ``jsonString`` expressing the new metadata. No metadata from the previous version of this file will be persisted, so if you want to update a specific field first get the json with the above command and alter the fields you want.
 
+Optional Parameter for verifying that the Dataset Version being edited is the latest version can be added &sourceInternalVersionTimestamp=datetime(in format: "yyyy-MM-dd'T'HH:mm:ss'Z'"). This is to prevent stale data from being edited. The value for sourceInternalVersionTimestamp comes from ``lastUpdateTime`` in the response to get $SERVER_URL/api/files/$ID API call
+
 A curl example using an ``ID``
 
 .. code-block:: bash
@@ -4639,18 +4641,19 @@ A curl example using a ``PERSISTENT_ID``
   export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
   export SERVER_URL=https://demo.dataverse.org
   export PERSISTENT_ID=doi:10.5072/FK2/AAA000
+  export UPDATE_TIME=2025-04-25T13:58:28Z
 
   curl -H "X-Dataverse-key:$API_TOKEN" -X POST \
     -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"dataFileTags":["Survey"],"restrict":false}' \
-    "$SERVER_URL/api/files/:persistentId/metadata?persistentId=$PERSISTENT_ID"
+    "$SERVER_URL/api/files/:persistentId/metadata?persistentId=$PERSISTENT_ID&sourceInternalVersionTimestamp=$UPDATE_TIME"
 
 The fully expanded example above (without environment variables) looks like this:
 
 .. code-block:: bash
 
   curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \
     -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"dataFileTags":["Survey"],"restrict":false}' \
-    "https://demo.dataverse.org/api/files/:persistentId/metadata?persistentId=doi:10.5072/FK2/AAA000"
+    "https://demo.dataverse.org/api/files/:persistentId/metadata?persistentId=doi:10.5072/FK2/AAA000&sourceInternalVersionTimestamp=2025-04-25T13:58:28Z"
 
 Note: To update the 'tabularTags' property of file metadata, use the 'dataFileTags' key when making API requests. This property is used to update the 'tabularTags' of the file metadata.
 

src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java

@@ -27,6 +27,7 @@
 import edu.harvard.iq.dataverse.search.savedsearch.SavedSearchServiceBean;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import edu.harvard.iq.dataverse.util.BundleUtil;
+import edu.harvard.iq.dataverse.util.DateUtil;
 import edu.harvard.iq.dataverse.util.FileUtil;
 import edu.harvard.iq.dataverse.util.SystemConfig;
 import edu.harvard.iq.dataverse.util.json.JsonParser;
@@ -51,6 +52,7 @@
 
 import java.io.InputStream;
 import java.net.URI;
+import java.time.Instant;
 import java.util.*;
 import java.util.concurrent.Callable;
 import java.util.logging.Level;
@@ -451,6 +453,21 @@ protected void validateInternalVersionNumberIsNotOutdated(Dataset dataset, int i
         }
     }
 
+    protected void validateInternalTimestampIsNotOutdated(DataFile dataFile, String sourceInternalVersionTimestamp) throws WrappedResponse {
+        Date date = sourceInternalVersionTimestamp != null ? DateUtil.parseDate(sourceInternalVersionTimestamp, "yyyy-MM-dd'T'HH:mm:ss'Z'") : null;
+        if (date == null) {
+            throw new WrappedResponse(
+                    badRequest(BundleUtil.getStringFromBundle("jsonparser.error.parsing.date", Collections.singletonList(sourceInternalVersionTimestamp)))
+            );
+        }
+        Instant instant = date.toInstant();
+        if (dataFile.getFileMetadata().getDatasetVersion().getLastUpdateTime().toInstant().getEpochSecond() != instant.getEpochSecond()) {
+            throw new WrappedResponse(
+                    badRequest(BundleUtil.getStringFromBundle("abstractApiBean.error.datafileInternalVersionTimestampIsOutdated", Collections.singletonList(sourceInternalVersionTimestamp)))
+            );
+        }
+    }
+
     protected DataFile findDataFileOrDie(String id) throws WrappedResponse {
         DataFile datafile;
         if (id.equals(PERSISTENT_ID_KEY)) {

src/main/java/edu/harvard/iq/dataverse/api/Files.java

@@ -410,8 +410,8 @@ public Response deleteFileInDataset(@Context ContainerRequestContext crc, @PathP
     @AuthRequired
     @Path("{id}/metadata")
     public Response updateFileMetadata(@Context ContainerRequestContext crc, @FormDataParam("jsonData") String jsonData,
-                    @PathParam("id") String fileIdOrPersistentId
-        ) throws DataFileTagException, CommandException {
+                    @PathParam("id") String fileIdOrPersistentId, @QueryParam("sourceInternalVersionTimestamp") String sourceInternalVersionTimestamp
+        ) throws CommandException {
 
         FileMetadata upFmd = null;
 
@@ -429,6 +429,13 @@ public Response updateFileMetadata(@Context ContainerRequestContext crc, @FormDa
                 return error(BAD_REQUEST, "Error attempting get the requested data file.");
             }
 
+            if (sourceInternalVersionTimestamp != null) {
+                try {
+                    validateInternalTimestampIsNotOutdated(df, sourceInternalVersionTimestamp);
+                } catch (WrappedResponse wr) {
+                    return wr.getResponse();
+                }
+            }
 
             //You shouldn't be trying to edit a datafile that has been replaced
             List<Long> result = em.createNamedQuery("DataFile.findDataFileThatReplacedId", Long.class)

src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java

@@ -194,46 +194,28 @@ public boolean getTabIngest() {
         return this.tabIngest;
     }
 
-    public boolean hasCategories(){
-        if ((categories == null)||(this.categories.isEmpty())){
-            return false;
-        }
-        return true;
+    public boolean hasCategories() {
+        return categories != null;
     }
 
-    public boolean hasFileDataTags(){
-        if ((dataFileTags == null)||(this.dataFileTags.isEmpty())){
-            return false;
-        }
-        return true;
+    public boolean hasFileDataTags() {
+        return dataFileTags != null;
     }
 
     public boolean hasDescription(){
-        if ((description == null)||(this.description.isEmpty())){
-            return false;
-        }
-        return true;
+        return description != null;
     }
 
-    public boolean hasDirectoryLabel(){
-        if ((directoryLabel == null)||(this.directoryLabel.isEmpty())){
-            return false;
-        }
-        return true;
+    public boolean hasDirectoryLabel() {
+        return directoryLabel != null;
     }
 
-    public boolean hasLabel(){
-        if ((label == null)||(this.label.isEmpty())){
-            return false;
-        }
-        return true;
+    public boolean hasLabel() {
+        return label != null;
     }
 
-    public boolean hasProvFreeform(){
-        if ((provFreeForm == null)||(this.provFreeForm.isEmpty())){
-            return false;
-        }
-        return true;
+    public boolean hasProvFreeform() {
+        return provFreeForm != null;
     }
 
 	public boolean hasStorageIdentifier() {
@@ -245,15 +227,15 @@ public String getStorageIdentifier() {
 	}
 
 	public boolean hasFileName() {
-		return ((fileName!=null)&&(!fileName.isEmpty()));
+		return fileName != null;
 	}
 
 	public String getFileName() {
 		return fileName;
 	}
 
 	public boolean hasMimetype() {
-		return ((mimeType!=null)&&(!mimeType.isEmpty()));
+		return mimeType != null;
 	}
 
 	public String getMimeType() {
@@ -266,7 +248,7 @@ public void setCheckSum(String checkSum, ChecksumType type) {
 	}
 
 	public boolean hasCheckSum() {
-		return ((checkSumValue!=null)&&(!checkSumValue.isEmpty()));
+		return checkSumValue != null;
 	}
 
 	public String getCheckSum() {
@@ -294,15 +276,10 @@ public void setFileSize(long fileSize) {
      *  @param tags
      */
     public void setCategories(List<String> newCategories) {
-
         if (newCategories != null) {
             newCategories = Util.removeDuplicatesNullsEmptyStrings(newCategories);
-            if (newCategories.isEmpty()) {
-                newCategories = null;
-            }
+            this.categories = newCategories;
         }
-
-        this.categories = newCategories;
     }
 
     /**
@@ -495,27 +472,20 @@ private void addFileDataTags(List<String> potentialTags) throws DataFileTagExcep
         }
 
         potentialTags = Util.removeDuplicatesNullsEmptyStrings(potentialTags);
-
-        if (potentialTags.isEmpty()){
-            return;
-        }
-
+
          // Make a new list
-        this.dataFileTags = new ArrayList<>();
+        List<String> newList = new ArrayList<>();
 
         // Add valid potential tags to the list
         for (String tagToCheck : potentialTags){
             if (DataFileTag.isDataFileTag(tagToCheck)){
-                this.dataFileTags.add(tagToCheck);
+                newList.add(tagToCheck);
             }else{                    
                 String errMsg = BundleUtil.getStringFromBundle("file.addreplace.error.invalid_datafile_tag");
                 throw new DataFileTagException(errMsg + " [" + tagToCheck + "]. Please use one of the following: " + DataFileTag.getListofLabelsAsString());
             }
         }
-         // Shouldn't happen....
-         if (dataFileTags.isEmpty()){
-            dataFileTags = null;
-        }
+        this.dataFileTags = newList;
     }
 
     private void msg(String s){

src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java

@@ -873,7 +873,8 @@ public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata, boo
                 .add("tabularData", df.isTabularData())
                 .add("tabularTags", getTabularFileTags(df))
                 .add("creationDate", df.getCreateDateFormattedYYYYMMDD())
-                .add("publicationDate",  df.getPublicationDateFormattedYYYYMMDD());
+                .add("publicationDate",  df.getPublicationDateFormattedYYYYMMDD())
+                .add("lastUpdateTime", format(fileMetadata.getDatasetVersion().getLastUpdateTime()));
         Dataset dfOwner = df.getOwner();
         if (dfOwner != null) {
             builder.add("fileAccessRequest", dfOwner.isFileAccessRequest());

src/main/java/propertyFiles/Bundle.properties

@@ -3205,3 +3205,4 @@ updateDatasetFieldsCommand.api.processDatasetUpdate.parseError=Error parsing dat
 
 #AbstractApiBean.java
 abstractApiBean.error.datasetInternalVersionNumberIsOutdated=Dataset internal version number {0} is outdated
+abstractApiBean.error.datafileInternalVersionTimestampIsOutdated=File Metadata internal version timestamp {0} is outdated