Switching to the DataCite REST API for retrieving registration metadata

doc/release-notes/12070-datacite-getmetadata.md

@@ -0,0 +1,11 @@
+Incremental improvements have been made to the process of registering dataset metadata with DataCite. If your instance is using DataCite, please make sure you have a valid DataCite REST API url configured, since it is now required.
+
+The JVM option(s) in question are `dataverse.pid.*.datacite.rest-api-url` if the recommended, new-style pid configuration is used, or `doi.dataciterestapiurlstring` if the legacy settings are in place. In the latter case however, this may be a good occasion to switch to the new configuration setup.
+
+For instances using registered DataCite authorities in production the url should be:
+
+```<jvm-options>-Ddataverse.pid.<providername>.datacite.rest-api-url=https://api.datacite.org</jvm-options>```
+
+Or, for test and development instances:
+
+```<jvm-options>-Ddataverse.pid.<providername>.datacite.rest-api-url=https://api.test.datacite.org</jvm-options>```

src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DOIDataCiteRegisterService.java

@@ -42,8 +42,8 @@ public class DOIDataCiteRegisterService {
     //A singleton since it, and the httpClient in it can be reused.
     private DataCiteRESTfullClient client=null;
 
-    public DOIDataCiteRegisterService(String url, String username, String password) {
-            client = new DataCiteRESTfullClient(url, username, password);
+    public DOIDataCiteRegisterService(String url, String restApiUrl, String username, String password) {
+            client = new DataCiteRESTfullClient(url, restApiUrl, username, password);
     }
 
     /**
@@ -80,13 +80,14 @@ public String registerIdentifier(String identifier, Map<String, String> metadata
 
     public String reRegisterIdentifier(String identifier, Map<String, String> metadata, DvObject dvObject) throws IOException {
         String retString = "";
-        String numericIdentifier = identifier.substring(identifier.indexOf(":") + 1);
+        // "bare identifier" is the canonical pid with the "doi:" prefix stripped
+        String bareIdentifier = identifier.substring(identifier.indexOf(":") + 1);
         String xmlMetadata = getMetadataFromDvObject(identifier, metadata, dvObject);
         String target = metadata.get("_target");
         String currentMetadata = null;
         boolean hasDifferences = false;
         try {
-            currentMetadata = client.getMetadata(numericIdentifier);
+            currentMetadata = client.getMetadata(bareIdentifier);
             Diff myDiff = DiffBuilder.compare(xmlMetadata).withTest(currentMetadata).ignoreWhitespace().checkForSimilar()
                     .build();
             hasDifferences = myDiff.hasDifferences();
@@ -96,7 +97,7 @@ public String reRegisterIdentifier(String identifier, Map<String, String> metada
                 }
             }
         } catch (RuntimeException e) {
-            logger.log(Level.INFO, "DOI " + numericIdentifier + " not registered with DataCite, registering now.");
+            logger.log(Level.INFO, "DOI " + bareIdentifier + " not registered with DataCite, registering now.");
             hasDifferences = true;
         }
 
@@ -106,13 +107,13 @@ public String reRegisterIdentifier(String identifier, Map<String, String> metada
         String currentUrl = null;
         try {
             //May get a 204 if the DOI is still draft
-            currentUrl = client.getUrl(numericIdentifier);
+            currentUrl = client.getUrl(bareIdentifier);
         } catch (RuntimeException ex) {
-            logger.fine("Error getting Url for " + numericIdentifier + ": " + ex.getMessage());
+            logger.fine("Error getting Url for " + bareIdentifier + ": " + ex.getMessage());
         }
         if (!target.equals(currentUrl)) {
             logger.info("Updating target URL to " +  target);
-            client.postUrl(numericIdentifier, target);
+            client.postUrl(bareIdentifier, target);
             retString = retString + "url:\\r" + target;
 
         }

src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteDOIProvider.java

@@ -59,7 +59,7 @@ public DataCiteDOIProvider(String id, String label, String providerAuthority, St
         this.apiUrl = apiUrl;
         this.username = username;
         this.password = password;
-        doiDataCiteRegisterService = new DOIDataCiteRegisterService(mdsUrl, username, password);
+        doiDataCiteRegisterService = new DOIDataCiteRegisterService(mdsUrl, apiUrl, username, password);
     }
 
     @Override
@@ -349,7 +349,7 @@ public boolean updateIdentifier(DvObject dvObject) {
                 logger.info(identifier + "updated: " + updated );
                 return true;
             } else {
-                logger.info("No updated needed for " + identifier);
+                logger.info("No update needed for " + identifier);
                 return false; //No update needed
             }
         } catch (Exception e) {

src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteRESTfullClient.java

@@ -8,6 +8,7 @@
 
 import java.io.Closeable;
 import java.io.IOException;
+import java.util.Base64;
 
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -26,10 +27,9 @@
 import org.apache.http.impl.client.BasicCredentialsProvider;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClients;
-
-
-
 import org.apache.http.util.EntityUtils;
+import edu.harvard.iq.dataverse.util.json.JsonUtil;
+import jakarta.json.JsonObject;
 
 /**
  * DataCiteRESTfullClient
@@ -46,12 +46,14 @@ public class DataCiteRESTfullClient implements Closeable {
     private static final long RETRY_DELAY_MS = 10000; // 10 seconds
 
     private String url;
+    private String restApiUrl;
     private CloseableHttpClient httpClient;
     private HttpClientContext context;
     private String encoding = "utf-8";
-    
-    public DataCiteRESTfullClient(String url, String username, String password) {
+
+    public DataCiteRESTfullClient(String url, String restApiUrl, String username, String password) {
         this.url = url;
+        this.restApiUrl = restApiUrl;
         context = HttpClientContext.create();
         CredentialsProvider credsProvider = new BasicCredentialsProvider();
         credsProvider.setCredentials(new AuthScope(null, -1), new UsernamePasswordCredentials(username, password));
@@ -192,6 +194,13 @@ public String postUrl(String doi, String url) throws IOException {
      * @return
      */
     public String getMetadata(String doi) {
+        // Try obtaining the metadata using the new, REST API:
+        try {
+            return getMetadataViaRestApi(doi);
+        } catch (RuntimeException rex) {
+            logger.warning("Failed to getMetadata via REST API for doi " + doi +", falling back to MDS");
+        }
+
         HttpGet httpGet = new HttpGet(this.url + "/metadata/" + doi);
         httpGet.setHeader("Accept", "application/xml");
         try {
@@ -209,6 +218,57 @@ public String getMetadata(String doi) {
         }
     }
 
+    /**
+     * getMetadataViaRestApi
+     * a temporary/dev. version of the method utilizing REST API instead of MDS
+     *
+     * @param doi
+     * @return
+     */
+    public String getMetadataViaRestApi(String doi) {
+        HttpGet httpGet = new HttpGet(this.restApiUrl + "/dois/" + doi);
+
+        try {
+            HttpResponse response = executeWithRetry(httpGet, "getMetadataViaRestApi");
+            String restApiRawData = EntityUtils.toString(response.getEntity(), encoding);
+
+            logger.fine("REST API raw data: " + restApiRawData);
+
+            if (response.getStatusLine().getStatusCode() != 200) {
+                String errMsg = "getMetadataViaRestApi, Response: " + response.getStatusLine().getStatusCode() + ", " + restApiRawData;
+                logger.log(Level.SEVERE, errMsg);
+                throw new RuntimeException(errMsg);
+            }
+
+            JsonObject restApiJson = JsonUtil.getJsonObject(restApiRawData);
+            String xmlEncoded = null; 
+
+            JsonObject restApiJsonData = restApiJson.getJsonObject("data");
+            if (restApiJsonData != null) {
+                JsonObject restApiJsonAttributes = restApiJsonData.getJsonObject("attributes");
+                if (restApiJsonAttributes != null) {
+                    xmlEncoded = restApiJsonAttributes.getString("xml");
+                }
+            }
+            logger.fine("encoded XML entry: " + xmlEncoded);
+
+            String metadata = null; // what we want to return, registration metadata in the XML format
+
+            if (xmlEncoded != null) {
+                // Stripping any newlines below may be unnecessary - it is likely
+                // always returned as a continuous string; but shouldn't hurt 
+                // either. 
+                metadata = new String(Base64.getDecoder().decode(xmlEncoded.replaceAll("[\\r\\n]", "")), encoding);
+            }
+
+            logger.fine("decoded XML metadata: " + metadata);
+            return metadata;
+        } catch (IOException ioe) {
+            logger.log(Level.SEVERE, "IOException in getMetadataViaRestApi", ioe);
+            throw new RuntimeException("IOException in getMetadataViaRestAPi", ioe);
+        }
+    }
+
     /**
      * testDOIExists
      *
@@ -270,9 +330,16 @@ public String inactiveDataset(String doi) {
         }
     }
 
+    /**
+     * The main() method can be used to test the functionality on the command
+     * line outside of Dataverse.
+     * Un-comment out and modify the code below as needed.
+     * @param args
+     * @throws Exception
+     */
     public static void main(String[] args) throws Exception {
         String doi = "10.5072/DVN/274533";
-        DataCiteRESTfullClient client = new DataCiteRESTfullClient("https://mds.test.datacite.org", "DATACITE_TEST_USERNAME", "DATACITE_TEST_PASSWORD");
+        DataCiteRESTfullClient client = new DataCiteRESTfullClient("https://mds.test.datacite.org", "https://api.test.datacite.org", "DATACITE_TEST_USERNAME", "DATACITE_TEST_PASSWORD");
 //		System.out.println(client.getUrl(doi));
 //		System.out.println(client.getMetadata(doi));
 //        System.out.println(client.postMetadata(readAndClose("C:/Users/luopc/Desktop/datacite.xml", "utf-8")));