Switching to the DataCite REST API for retrieving registration metadata (#12270)

* A quick implementation of using DataCite REST API for retrieving
DOI metadata from DataCite. This is to address an apparent issue
with UTF8 characters when relying on the MDS API used traditionally.
(#12070)

* Removing the old-style constructors that are no longer needed.
Removing the old-style constructors that are no longer needed. #12070

* whitespaces #12070

* a release note for #12070

* some last touches. #12070

* cosmetic - cleaning up a dev. comment #12070

* cosmetic^2 - fixes a typo in a cosmetic comment fix #12070

Author: landreev

doc/release-notes/12070-datacite-getmetadata.md

@@ -0,0 +1,11 @@
+Incremental improvements have been made to the process of registering dataset metadata with DataCite. If your instance is using DataCite, please make sure you have a valid DataCite REST API url configured, since it is now required.
+
+The JVM option(s) in question are `dataverse.pid.*.datacite.rest-api-url` if the recommended, new-style pid configuration is used, or `doi.dataciterestapiurlstring` if the legacy settings are in place. In the latter case however, this may be a good occasion to switch to the new configuration setup.
+
+For instances using registered DataCite authorities in production the url should be:
+
+```<jvm-options>-Ddataverse.pid.<providername>.datacite.rest-api-url=https://api.datacite.org</jvm-options>```
+
+Or, for test and development instances:
+
+```<jvm-options>-Ddataverse.pid.<providername>.datacite.rest-api-url=https://api.test.datacite.org</jvm-options>```

src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DOIDataCiteRegisterService.java

@@ -42,8 +42,8 @@ public class DOIDataCiteRegisterService {
     //A singleton since it, and the httpClient in it can be reused.
     private DataCiteRESTfullClient client=null;
 
-    public DOIDataCiteRegisterService(String url, String username, String password) {
-            client = new DataCiteRESTfullClient(url, username, password);
+    public DOIDataCiteRegisterService(String url, String restApiUrl, String username, String password) {
+            client = new DataCiteRESTfullClient(url, restApiUrl, username, password);
     }
 
     /**
@@ -80,13 +80,14 @@ public String registerIdentifier(String identifier, Map<String, String> metadata
 
     public String reRegisterIdentifier(String identifier, Map<String, String> metadata, DvObject dvObject) throws IOException {
         String retString = "";
-        String numericIdentifier = identifier.substring(identifier.indexOf(":") + 1);
+        // "bare identifier" is the canonical pid with the "doi:" prefix stripped
+        String bareIdentifier = identifier.substring(identifier.indexOf(":") + 1);
         String xmlMetadata = getMetadataFromDvObject(identifier, metadata, dvObject);
         String target = metadata.get("_target");
         String currentMetadata = null;
         boolean hasDifferences = false;
         try {
-            currentMetadata = client.getMetadata(numericIdentifier);
+            currentMetadata = client.getMetadata(bareIdentifier);
             Diff myDiff = DiffBuilder.compare(xmlMetadata).withTest(currentMetadata).ignoreWhitespace().checkForSimilar()
                     .build();
             hasDifferences = myDiff.hasDifferences();
@@ -96,7 +97,7 @@ public String reRegisterIdentifier(String identifier, Map<String, String> metada
                 }
             }
         } catch (RuntimeException e) {
-            logger.log(Level.INFO, "DOI " + numericIdentifier + " not registered with DataCite, registering now.");
+            logger.log(Level.INFO, "DOI " + bareIdentifier + " not registered with DataCite, registering now.");
             hasDifferences = true;
         }
 
@@ -106,13 +107,13 @@ public String reRegisterIdentifier(String identifier, Map<String, String> metada
         String currentUrl = null;
         try {
             //May get a 204 if the DOI is still draft
-            currentUrl = client.getUrl(numericIdentifier);
+            currentUrl = client.getUrl(bareIdentifier);
         } catch (RuntimeException ex) {
-            logger.fine("Error getting Url for " + numericIdentifier + ": " + ex.getMessage());
+            logger.fine("Error getting Url for " + bareIdentifier + ": " + ex.getMessage());
         }
         if (!target.equals(currentUrl)) {
             logger.info("Updating target URL to " +  target);
-            client.postUrl(numericIdentifier, target);
+            client.postUrl(bareIdentifier, target);
             retString = retString + "url:\\r" + target;
 
         }

src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteDOIProvider.java

@@ -59,7 +59,7 @@ public DataCiteDOIProvider(String id, String label, String providerAuthority, St
         this.apiUrl = apiUrl;
         this.username = username;
         this.password = password;
-        doiDataCiteRegisterService = new DOIDataCiteRegisterService(mdsUrl, username, password);
+        doiDataCiteRegisterService = new DOIDataCiteRegisterService(mdsUrl, apiUrl, username, password);
     }
 
     @Override
@@ -349,7 +349,7 @@ public boolean updateIdentifier(DvObject dvObject) {
                 logger.info(identifier + "updated: " + updated );
                 return true;
             } else {
-                logger.info("No updated needed for " + identifier);
+                logger.info("No update needed for " + identifier);
                 return false; //No update needed
             }
         } catch (Exception e) {

src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteRESTfullClient.java

@@ -8,6 +8,7 @@
 
 import java.io.Closeable;
 import java.io.IOException;
+import java.util.Base64;
 
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -26,10 +27,9 @@
 import org.apache.http.impl.client.BasicCredentialsProvider;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClients;
-
-
-
 import org.apache.http.util.EntityUtils;
+import edu.harvard.iq.dataverse.util.json.JsonUtil;
+import jakarta.json.JsonObject;
 
 /**
  * DataCiteRESTfullClient
@@ -46,12 +46,14 @@ public class DataCiteRESTfullClient implements Closeable {
     private static final long RETRY_DELAY_MS = 10000; // 10 seconds
 
     private String url;
+    private String restApiUrl;
     private CloseableHttpClient httpClient;
     private HttpClientContext context;
     private String encoding = "utf-8";
-    
-    public DataCiteRESTfullClient(String url, String username, String password) {
+
+    public DataCiteRESTfullClient(String url, String restApiUrl, String username, String password) {
         this.url = url;
+        this.restApiUrl = restApiUrl;
         context = HttpClientContext.create();
         CredentialsProvider credsProvider = new BasicCredentialsProvider();
         credsProvider.setCredentials(new AuthScope(null, -1), new UsernamePasswordCredentials(username, password));
@@ -192,6 +194,13 @@ public String postUrl(String doi, String url) throws IOException {
      * @return
      */
     public String getMetadata(String doi) {
+        // Try obtaining the metadata using the new, REST API:
+        try {
+            return getMetadataViaRestApi(doi);
+        } catch (RuntimeException rex) {
+            logger.warning("Failed to getMetadata via REST API for doi " + doi +", falling back to MDS");
+        }
+        
         HttpGet httpGet = new HttpGet(this.url + "/metadata/" + doi);
         httpGet.setHeader("Accept", "application/xml");
         try {
@@ -209,6 +218,58 @@ public String getMetadata(String doi) {
         }
     }
 
+    /**
+     * getMetadataViaRestApi
+     * obtains registration metadata utilizing REST API instead of MDS. 
+     * This solves some known issues, see #12070 for details. 
+     *
+     * @param doi
+     * @return
+     */
+    public String getMetadataViaRestApi(String doi) {
+        HttpGet httpGet = new HttpGet(this.restApiUrl + "/dois/" + doi);
+
+        try {
+            HttpResponse response = executeWithRetry(httpGet, "getMetadataViaRestApi");
+            String restApiRawData = EntityUtils.toString(response.getEntity(), encoding);
+            
+            logger.fine("REST API raw data: " + restApiRawData);
+            
+            if (response.getStatusLine().getStatusCode() != 200) {
+                String errMsg = "getMetadataViaRestApi, Response: " + response.getStatusLine().getStatusCode() + ", " + restApiRawData;
+                logger.log(Level.SEVERE, errMsg);
+                throw new RuntimeException(errMsg);
+            }
+
+            JsonObject restApiJson = JsonUtil.getJsonObject(restApiRawData);
+            String xmlEncoded = null; 
+            
+            JsonObject restApiJsonData = restApiJson.getJsonObject("data");
+            if (restApiJsonData != null) {
+                JsonObject restApiJsonAttributes = restApiJsonData.getJsonObject("attributes");
+                if (restApiJsonAttributes != null) {
+                    xmlEncoded = restApiJsonAttributes.getString("xml");
+                }
+            }
+            logger.fine("encoded XML entry: " + xmlEncoded);
+            
+            String metadata = null; // what we want to return, registration metadata in the XML format
+
+            if (xmlEncoded != null) {
+                // Stripping any newlines below may be unnecessary - it is likely
+                // always returned as a continuous string; but shouldn't hurt 
+                // either. 
+                metadata = new String(Base64.getDecoder().decode(xmlEncoded.replaceAll("[\\r\\n]", "")), encoding);
+            }
+            
+            logger.fine("decoded XML metadata: " + metadata);
+            return metadata;
+        } catch (IOException ioe) {
+            logger.log(Level.SEVERE, "IOException in getMetadataViaRestApi", ioe);
+            throw new RuntimeException("IOException in getMetadataViaRestAPi", ioe);
+        }
+    }
+    
     /**
      * testDOIExists
      *
@@ -270,9 +331,16 @@ public String inactiveDataset(String doi) {
         }
     }
 
+    /**
+     * The main() method can be used to test the functionality on the command
+     * line outside of Dataverse.
+     * Un-comment out and modify the code below as needed.
+     * @param args
+     * @throws Exception
+     */
     public static void main(String[] args) throws Exception {
         String doi = "10.5072/DVN/274533";
-        DataCiteRESTfullClient client = new DataCiteRESTfullClient("https://mds.test.datacite.org", "DATACITE_TEST_USERNAME", "DATACITE_TEST_PASSWORD");
+        DataCiteRESTfullClient client = new DataCiteRESTfullClient("https://mds.test.datacite.org", "https://api.test.datacite.org", "DATACITE_TEST_USERNAME", "DATACITE_TEST_PASSWORD");
 //		System.out.println(client.getUrl(doi));
 //		System.out.println(client.getMetadata(doi));
 //        System.out.println(client.postMetadata(readAndClose("C:/Users/luopc/Desktop/datacite.xml", "utf-8")));