Merge pull request #11494 from GlobalDataverseCommunityConsortium/indexing_fixes

Full-text Indexing Fixes

Author: ofahimIQSS

src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java

@@ -240,10 +240,10 @@ public long retrieveSizeFromMedia() {
     public InputStream getInputStream() throws IOException {
         //Currently only supported when using an S3 store with the Globus S3Connector.
         //ToDo: Support when using a managed Globus endpoint that supports http access
-        if(StorageIO.isDataverseAccessible(endpoint)) {
+        if(isDataverseAccessible(driverId)) {
             return baseStore.getInputStream();
         } else {
-            throw new IOException("Not implemented");
+            return null;
         }
     }
 
@@ -504,5 +504,10 @@ public void saveInputStream(InputStream inputStream, Long filesize) throws IOExc
         throw new UnsupportedDataAccessOperationException(
                 this.getClass().getName() + ": savePath() not implemented in this storage driver.");
     }
+    
+    @Override
+    public boolean isDataverseAccessible() {
+        return isDataverseAccessible(this.driverId);
+    }
 
 }

src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java

@@ -671,6 +671,10 @@ protected String getConfigParam(String parameterName, String defaultValue) {
         return getConfigParamForDriver(this.driverId, parameterName, defaultValue);
     }
 
+    public boolean isDataverseAccessible() {
+        return true;
+    }
+    
     protected static String getConfigParamForDriver(String driverId, String parameterName) {
         return getConfigParamForDriver(driverId, parameterName, null);
     }
@@ -681,5 +685,4 @@ protected static String getConfigParamForDriver(String driverId, String paramete
     public static String getNewIdentifier(String driverId) {
         return driverId + DataAccess.SEPARATOR + FileUtil.generateStorageIdentifier();
     }
-
 }

src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java

@@ -1459,6 +1459,7 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set<Long
             String datasetPersistentURL = dataset.getPersistentURL();
             boolean isHarvested = dataset.isHarvested();
             long startTime = System.currentTimeMillis();
+            LocalDate now = LocalDate.now();
             fileMetadatas.stream().forEach(fileMetadata -> {
                 DataFile datafile = fileMetadata.getDataFile();
                 Embargo emb = datafile.getEmbargo();
@@ -1499,8 +1500,10 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set<Long
                     datafileSolrInputDocument.addField(SearchFields.PERSISTENT_URL, datasetPersistentURL);
                     datafileSolrInputDocument.addField(SearchFields.TYPE, "files");
                     datafileSolrInputDocument.addField(SearchFields.CATEGORY_OF_DATAVERSE, dvIndexableCategoryName);
+                    boolean embargoed = false;
                     if (end != null) {
                         datafileSolrInputDocument.addField(SearchFields.EMBARGO_END_DATE, end.toEpochDay());
+                        embargoed = end.isAfter(now);
                     }
                     if (start != null) {
                         datafileSolrInputDocument.addField(SearchFields.RETENTION_END_DATE, start.toEpochDay());
@@ -1511,14 +1514,15 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set<Long
                         if (!isHarvested && !datafile.isRestricted()
                                 && !datafile.isFilePackage()
                                 && fileSize != 0 && fileSize <= maxSize
-                                && datafile.getRetention() == null) {
+                                && datafile.getRetention() == null
+                                && !embargoed) {
                             StorageIO<DataFile> accessObject = null;
                             InputStream instream = null;
                             ContentHandler textHandler = null;
                             try {
                                 accessObject = DataAccess.getStorageIO(datafile,
                                         new DataAccessRequest());
-                                if (accessObject != null) {
+                                if (accessObject != null && accessObject.isDataverseAccessible()) {
                                     accessObject.open();
                                     // If the size is >max, we don't use the stream. However, for S3, the stream is
                                     // currently opened in the call above (see