add "requirements" and "auxFilesExist" to external tools IQSS#9153

The use case is an external tool that operates on aux files pulled out
of NetCDF/HDF5 files.

Author: pdurbin

doc/sphinx-guides/source/_static/installation/files/root/external-tools/auxFileTool.json

@@ -0,0 +1,26 @@
+{
+  "displayName": "AuxFileViewer",
+  "description": "Show an auxiliary file from a dataset file.",
+  "toolName": "auxPreviewer",
+  "scope": "file",
+  "types": [
+    "preview"
+  ],
+  "toolUrl": "https://example.com/AuxFileViewer.html",
+  "toolParameters": {
+    "queryParameters": [
+      {
+        "fileid": "{fileId}"
+      }
+    ]
+  },
+  "requirements": {
+    "auxFilesExist": [
+      {
+        "formatTag": "myFormatTag",
+        "formatVersion": "0.1"
+      }
+    ]
+  },
+  "contentType": "application/foobar"
+}

doc/sphinx-guides/source/api/external-tools.rst

@@ -53,15 +53,21 @@ External tools must be expressed in an external tool manifest file, a specific J
 Examples of Manifests
 +++++++++++++++++++++
 
-Let's look at two examples of external tool manifests (one at the file level and one at the dataset level) before we dive into how they work.
+Let's look at a few examples of external tool manifests (both at the file level and at the dataset level) before we dive into how they work.
+
+.. _tools-for-files:
 
 External Tools for Files
 ^^^^^^^^^^^^^^^^^^^^^^^^
 
-:download:`fabulousFileTool.json <../_static/installation/files/root/external-tools/fabulousFileTool.json>` is a file level both an "explore" tool and a "preview" tool that operates on tabular files:
+:download:`fabulousFileTool.json <../_static/installation/files/root/external-tools/fabulousFileTool.json>` is a file level (both an "explore" tool and a "preview" tool) that operates on tabular files:
 
 .. literalinclude:: ../_static/installation/files/root/external-tools/fabulousFileTool.json
 
+:download:`auxFileTool.json <../_static/installation/files/root/external-tools/auxFileTool.json>` is a file level preview tool that operates on auxiliary files associated with a data file (note the "requirements" section):
+
+.. literalinclude:: ../_static/installation/files/root/external-tools/auxFileTool.json
+
 External Tools for Datasets
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -113,6 +119,10 @@ Terminology
     allowedApiCalls httpMethod   Which HTTP method the specified callback uses such as ``GET`` or ``POST``.
 
     allowedApiCalls timeOut      For non-public datasets and datafiles, how many minutes the signed URLs given to the tool should be valid for. Must be an integer.
+
+    requirements                 **Resources your tool needs to function.** For now, the only requirement you can specify is that one or more auxiliary files exist (see auxFilesExist in the :ref:`tools-for-files` example). Currently, requirements only apply to preview tools. If the requirements are not met, the preview tool is not shown.
+
+    auxFilesExist                **An array containing formatTag and formatVersion pairs** for each auxiliary file that your tool needs to download to function properly. For example, a required aux file could have a ``formatTag`` of "NcML" and a ``formatVersion`` of "1.0". See also :doc:`/developers/aux-file-support`.
 
     toolName                     A **name** of an external tool that is used to differentiate between external tools and also used in bundle.properties for localization in the Dataverse installation web interface. For example, the toolName for Data Explorer is ``explorer``. For the Data Curation Tool the toolName is ``dct``. This is an optional parameter in the manifest JSON file.   
     ===========================  ==========

src/main/java/edu/harvard/iq/dataverse/DatasetPage.java

@@ -5490,7 +5490,7 @@ public List<ExternalTool> getCachedToolsForDataFile(Long fileId, ExternalTool.Ty
             return cachedTools;
         }
         DataFile dataFile = datafileService.find(fileId);
-        cachedTools = ExternalToolServiceBean.findExternalToolsByFile(externalTools, dataFile);
+        cachedTools = externalToolService.findExternalToolsByFile(externalTools, dataFile);
         cachedToolsByFileId.put(fileId, cachedTools); //add to map so we don't have to do the lifting again
         return cachedTools;
     }

src/main/java/edu/harvard/iq/dataverse/FilePage.java

@@ -39,6 +39,7 @@
 import edu.harvard.iq.dataverse.util.JsfHelper;
 import static edu.harvard.iq.dataverse.util.JsfHelper.JH;
 import edu.harvard.iq.dataverse.util.SystemConfig;
+import edu.harvard.iq.dataverse.util.json.JsonUtil;
 import java.io.IOException;
 import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
@@ -57,6 +58,9 @@
 import javax.faces.view.ViewScoped;
 import javax.inject.Inject;
 import javax.inject.Named;
+import javax.json.JsonArray;
+import javax.json.JsonObject;
+import javax.json.JsonValue;
 import javax.validation.ConstraintViolation;
 
 import org.primefaces.PrimeFaces;
@@ -125,6 +129,8 @@ public class FilePage implements java.io.Serializable {
     ExternalToolServiceBean externalToolService;
     @EJB
     PrivateUrlServiceBean privateUrlService;
+    @EJB
+    AuxiliaryFileServiceBean auxiliaryFileService;
 
     @Inject
     DataverseRequestServiceBean dvRequestService;
@@ -285,8 +291,15 @@ public void setDatasetVersionId(Long datasetVersionId) {
         this.datasetVersionId = datasetVersionId;
     }
 
+    // findPreviewTools would be a better name
     private List<ExternalTool> sortExternalTools(){
-        List<ExternalTool> retList = externalToolService.findFileToolsByTypeAndContentType(ExternalTool.Type.PREVIEW, file.getContentType());
+        List<ExternalTool> retList = new ArrayList<>();
+        List<ExternalTool> previewTools = externalToolService.findFileToolsByTypeAndContentType(ExternalTool.Type.PREVIEW, file.getContentType());
+        for (ExternalTool previewTool : previewTools) {
+            if (externalToolService.meetsRequirements(previewTool, file)) {
+                retList.add(previewTool);
+            }
+        }
         Collections.sort(retList, CompareExternalToolName);
         return retList;
     }

src/main/java/edu/harvard/iq/dataverse/api/TestApi.java

@@ -63,7 +63,9 @@ public Response getExternalToolsForFile(@PathParam("id") String idSupplied, @Que
                 ApiToken apiToken = externalToolService.getApiToken(getRequestApiKey());
                 ExternalToolHandler externalToolHandler = new ExternalToolHandler(tool, dataFile, apiToken, dataFile.getFileMetadata(), null);
                 JsonObjectBuilder toolToJson = externalToolService.getToolAsJsonWithQueryParameters(externalToolHandler);
-                tools.add(toolToJson);
+                if (externalToolService.meetsRequirements(tool, dataFile)) {
+                    tools.add(toolToJson);
+                }
             }
             return ok(tools);
         } catch (WrappedResponse wr) {

src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java

@@ -39,6 +39,7 @@ public class ExternalTool implements Serializable {
     public static final String CONTENT_TYPE = "contentType";
     public static final String TOOL_NAME = "toolName";
     public static final String ALLOWED_API_CALLS = "allowedApiCalls";
+    public static final String REQUIREMENTS = "requirements";
 
     @Id
     @GeneratedValue(strategy = GenerationType.IDENTITY)
@@ -103,6 +104,15 @@ public class ExternalTool implements Serializable {
     @Column(nullable = true, columnDefinition = "TEXT")
     private String allowedApiCalls;
 
+    /**
+     * When non-null, the tool has indicated that it has certain requirements
+     * that must be met before it should be shown to the user. This
+     * functionality was added for tools that operate on aux files rather than
+     * data files so "auxFilesExist" is one of the possible values.
+     */
+    @Column(nullable = true, columnDefinition = "TEXT")
+    private String requirements;
+
     /**
      * This default constructor is only here to prevent this error at
      * deployment:
@@ -118,10 +128,10 @@ public ExternalTool() {
     }
 
     public ExternalTool(String displayName, String toolName, String description, List<ExternalToolType> externalToolTypes, Scope scope, String toolUrl, String toolParameters, String contentType) {
-       this(displayName, toolName, description, externalToolTypes, scope, toolUrl, toolParameters, contentType, null);
+       this(displayName, toolName, description, externalToolTypes, scope, toolUrl, toolParameters, contentType, null, null);
     }
 
-    public ExternalTool(String displayName, String toolName, String description, List<ExternalToolType> externalToolTypes, Scope scope, String toolUrl, String toolParameters, String contentType, String allowedApiCalls) {
+    public ExternalTool(String displayName, String toolName, String description, List<ExternalToolType> externalToolTypes, Scope scope, String toolUrl, String toolParameters, String contentType, String allowedApiCalls, String requirements) {
         this.displayName = displayName;
         this.toolName = toolName;
         this.description = description;
@@ -131,6 +141,7 @@ public ExternalTool(String displayName, String toolName, String description, Lis
         this.toolParameters = toolParameters;
         this.contentType = contentType;
         this.allowedApiCalls = allowedApiCalls;
+        this.requirements = requirements;
     }
 
     public enum Type {
@@ -326,5 +337,12 @@ public void setAllowedApiCalls(String allowedApiCalls) {
         this.allowedApiCalls = allowedApiCalls;
     }
 
+    public String getRequirements() {
+        return requirements;
+    }
+
+    public void setRequirements(String requirements) {
+        this.requirements = requirements;
+    }
 
 }

src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java

@@ -1,5 +1,7 @@
 package edu.harvard.iq.dataverse.externaltools;
 
+import edu.harvard.iq.dataverse.AuxiliaryFile;
+import edu.harvard.iq.dataverse.AuxiliaryFileServiceBean;
 import edu.harvard.iq.dataverse.DataFile;
 import edu.harvard.iq.dataverse.DataFileServiceBean;
 import edu.harvard.iq.dataverse.authorization.users.ApiToken;
@@ -30,6 +32,8 @@
 import static edu.harvard.iq.dataverse.externaltools.ExternalTool.*;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
+import javax.ejb.EJB;
+import javax.json.JsonValue;
 
 @Stateless
 @Named
@@ -40,6 +44,9 @@ public class ExternalToolServiceBean {
     @PersistenceContext(unitName = "VDCNet-ejbPU")
     private EntityManager em;
 
+    @EJB
+    AuxiliaryFileServiceBean auxiliaryFileService;
+
     public List<ExternalTool> findAll() {
         TypedQuery<ExternalTool> typedQuery = em.createQuery("SELECT OBJECT(o) FROM ExternalTool AS o ORDER BY o.id", ExternalTool.class);
         return typedQuery.getResultList();
@@ -133,20 +140,45 @@ public ExternalTool save(ExternalTool externalTool) {
      * file supports The list of tools is passed in so it doesn't hit the
      * database each time
      */
-    public static List<ExternalTool> findExternalToolsByFile(List<ExternalTool> allExternalTools, DataFile file) {
+    public List<ExternalTool> findExternalToolsByFile(List<ExternalTool> allExternalTools, DataFile file) {
         List<ExternalTool> externalTools = new ArrayList<>();
         //Map tabular data to it's mimetype (the isTabularData() check assures that this code works the same as before, but it may need to change if tabular data is split into subtypes with differing mimetypes)
         final String contentType = file.isTabularData() ? DataFileServiceBean.MIME_TYPE_TSV_ALT : file.getContentType();
         allExternalTools.forEach((externalTool) -> {
-            //Match tool and file type 
-            if (contentType.equals(externalTool.getContentType())) {
+            //Match tool and file type, then check requirements
+            if (contentType.equals(externalTool.getContentType()) && meetsRequirements(externalTool, file)) {
                 externalTools.add(externalTool);
             }
         });
 
         return externalTools;
     }
 
+    public boolean meetsRequirements(ExternalTool externalTool, DataFile dataFile) {
+        String requirements = externalTool.getRequirements();
+        if (requirements == null) {
+            logger.fine("Data file id" + dataFile.getId() + ": no requirements for tool id " + externalTool.getId());
+            return true;
+        }
+        boolean meetsRequirements = true;
+        JsonObject requirementsObj = JsonUtil.getJsonObject(requirements);
+        JsonArray auxFilesExist = requirementsObj.getJsonArray("auxFilesExist");
+        for (JsonValue jsonValue : auxFilesExist) {
+            String formatTag = jsonValue.asJsonObject().getString("formatTag");
+            String formatVersion = jsonValue.asJsonObject().getString("formatVersion");
+            AuxiliaryFile auxFile = auxiliaryFileService.lookupAuxiliaryFile(dataFile, formatTag, formatVersion);
+            if (auxFile == null) {
+                logger.fine("Data file id" + dataFile.getId() + ": cannot find required aux file. formatTag=" + formatTag + ". formatVersion=" + formatVersion);
+                meetsRequirements = false;
+                break;
+            } else {
+                logger.fine("Data file id" + dataFile.getId() + ": found required aux file. formatTag=" + formatTag + ". formatVersion=" + formatVersion);
+                meetsRequirements = true;
+            }
+        }
+        return meetsRequirements;
+    }
+
     public static ExternalTool parseAddExternalToolManifest(String manifest) {
 
         if (manifest == null || manifest.isEmpty()) {
@@ -170,6 +202,7 @@ public static ExternalTool parseAddExternalToolManifest(String manifest) {
         JsonObject toolParametersObj = jsonObject.getJsonObject(TOOL_PARAMETERS);
         JsonArray queryParams = toolParametersObj.getJsonArray("queryParameters");
         JsonArray allowedApiCallsArray = jsonObject.getJsonArray(ALLOWED_API_CALLS);
+        JsonObject requirementsObj = jsonObject.getJsonObject(REQUIREMENTS);
 
         boolean allRequiredReservedWordsFound = false;
         if (scope.equals(Scope.FILE)) {
@@ -227,8 +260,12 @@ public static ExternalTool parseAddExternalToolManifest(String manifest) {
         if(allowedApiCallsArray !=null) {
             allowedApiCalls = allowedApiCallsArray.toString();
         }
+        String requirements = null;
+        if (requirementsObj != null) {
+            requirements = requirementsObj.toString();
+        }
 
-        return new ExternalTool(displayName, toolName, description, externalToolTypes, scope, toolUrl, toolParameters, contentType, allowedApiCalls);
+        return new ExternalTool(displayName, toolName, description, externalToolTypes, scope, toolUrl, toolParameters, contentType, allowedApiCalls, requirements);
     }
 
     private static String getRequiredTopLevelField(JsonObject jsonObject, String key) {

src/main/resources/db/migration/V5.13.0.3__9153-extract-metadata.sql

@@ -0,0 +1 @@
+ALTER TABLE externaltool ADD COLUMN IF NOT EXISTS requirements TEXT;