Merge remote-tracking branch 'IQSS/develop' into Arch4-fix_isArchivable_caching

Author: qqmyers

doc/release-notes/11606-hide-spa-oidc-providers-from-jsf-login-screen.md

@@ -0,0 +1,7 @@
+This release fixes a bug where the value of the dataverse.auth.oidc.enabled setting, available when Provisioning an authentication provider via JVM options (see ref: https://guides.dataverse.org/en/latest/installation/oidc.html#provision-via-jvm-options) was not being not being propagated to the current Dataverse user interface (where enabled=false providers are not displayed for login/registration) or represented in the GET api/admin/authenticationProviders API call.
+
+A new JVM setting ('dataverse.auth.oidc.hidden-jsf') was added to hide an enabled OIDC Provider from the JSF UI. 
+
+For Dataverse instances deploying both the current JSF UI and the new SPA UI, this fix allows the OIDC Keycloak provider configured for the SPA to be hidden in the JSF UI (useful in cases where it would duplicate other configured providers).
+
+Note: The API to create a new Auth Provider can only be used to create a provider for both JSF and SPA. Use JVM / MicroProfile config setting to create SPA only providers.

doc/release-notes/12082-permission-indexing-improvements.md

@@ -0,0 +1,6 @@
+Changes in v6.9 that significantly improved re-indexing performance and lowered memory use in situations
+such as when a user's role on the root collection were changed, also slowed reindexing of individual
+datasets ater editing and publication.
+
+This release restores/improves the individual dataset reindexing performance while retaining the
+benefits of the earlier update. 

doc/sphinx-guides/source/api/native-api.rst

@@ -7654,6 +7654,8 @@ Add new authentication provider. The POST data is in JSON format, similar to the
 
   POST http://$SERVER/api/admin/authenticationProviders
 
+.. note:: This endpoint will create providers for both JSF and SPA. Use :ref:`jvm-options` / *MicroProfile Config* if you need to create SPA only providers.
+
 Show Authentication Provider
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

doc/sphinx-guides/source/installation/oidc.rst

@@ -151,6 +151,10 @@ The following options are available:
     - Enable or disable provisioning the provider via MicroProfile.
     - N
     - ``false``
+  * - ``dataverse.auth.oidc.hidden-jsf``
+    - Show or Hide the provider from the JSF UI via MicroProfile.
+    - N
+    - ``false``
   * - ``dataverse.auth.oidc.client-id``
     - The client-id of the application to identify it at your provider.
     - Y
@@ -187,4 +191,4 @@ The following options are available:
     - Tune the maximum age, in seconds, of all OIDC providers' verifier cache entries. Default is 5 minutes, equivalent to lifetime
       of many OIDC access tokens.
     - N
-    - 300
+    - 300

docker-compose-dev.yml

@@ -22,6 +22,7 @@ services:
       DATAVERSE_MAIL_SYSTEM_EMAIL: "dataverse@localhost"
       DATAVERSE_MAIL_MTA_HOST: "smtp"
       DATAVERSE_AUTH_OIDC_ENABLED: "1"
+      DATAVERSE_AUTH_OIDC_HIDDEN_JSF: "1"
       DATAVERSE_AUTH_OIDC_CLIENT_ID: test
       DATAVERSE_AUTH_OIDC_CLIENT_SECRET: 94XHrfNRwXsjqTqApRrwWmhDLDHpIYV8
       DATAVERSE_AUTH_OIDC_AUTH_SERVER_URL: http://keycloak.mydomain.com:8090/realms/test

src/main/java/edu/harvard/iq/dataverse/LoginPage.java

@@ -143,7 +143,7 @@ public List<AuthenticationProviderDisplayInfo> listAuthenticationProviders() {
         Collections.sort(idps, Comparator.comparing(AuthenticationProvider::getOrder).thenComparing(AuthenticationProvider::getId));
 
         for (AuthenticationProvider idp : idps) {
-            if (idp != null) {
+            if (idp != null && !idp.isHidden()) {
                 infos.add(idp.getInfo());
             }
         }

src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationProvider.java

@@ -18,7 +18,7 @@
  *                "authenticationProvider.name." + "ship" ->
  *            (c)  Bundle.properties entry: "authenticationProvider.name.shib=Shibboleth"
  *          
- * {@code AuthenticationPrvider}s are normally registered at startup in {@link AuthenticationServiceBean#startup()}.
+ * {@code AuthenticationProvider}s are normally registered at startup in {@link AuthenticationServiceBean#startup()}.
  * 
  * @author michael
  */
@@ -33,9 +33,13 @@ public interface AuthenticationProvider {
     default boolean isUserInfoUpdateAllowed() { return false; };
     default boolean isUserDeletionAllowed() { return false; };
     default boolean isOAuthProvider() { return false; };
-    
-    
-    
+    default boolean isEnabled() {
+        return true;
+    };
+    default boolean isHidden() {
+        return false;
+    };
+
     /**
      * Some providers (e.g organizational ones) provide verified email addresses.
      * @return {@code true} if we can treat email addresses coming from this provider as verified, {@code false} otherwise.

src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/AbstractOAuth2AuthenticationProvider.java

@@ -93,6 +93,8 @@ public String toString() {
     protected String clientSecret;
     protected String baseUserEndpoint;
     protected String redirectUrl;
+    protected boolean enabled = true;
+    protected boolean hidden = false; // Special flag to hide this provider in JSF UI
 
     /**
      * List of scopes to be requested for authorization at identity provider.
@@ -272,6 +274,21 @@ public String getSubTitle() {
 
     public String getSpacedScope() { return String.join(" ", getScope()); }
 
+    public void setEnabled(boolean enabled) {
+        this.enabled = enabled;
+    }
+
+    public boolean isEnabled() {
+        return enabled;
+    }
+
+    public void setHidden(boolean hidden) {
+        this.hidden = hidden;
+    }
+    public boolean isHidden() {
+        return hidden;
+    }
+
     @Override
     public int hashCode() {
         int hash = 7;

src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactory.java

@@ -49,6 +49,7 @@ public AuthenticationProvider buildProvider( AuthenticationProviderRow aRow ) th
         oidc.setId(aRow.getId());
         oidc.setTitle(aRow.getTitle());
         oidc.setSubTitle(aRow.getSubtitle());
+        oidc.setEnabled(aRow.isEnabled());
 
         return oidc;
     }
@@ -70,6 +71,8 @@ public static AuthenticationProvider buildFromSettings() throws AuthorizationSet
         oidc.setId("oidc-mpconfig");
         oidc.setTitle(JvmSettings.OIDC_TITLE.lookupOptional().orElse("OpenID Connect"));
         oidc.setSubTitle(JvmSettings.OIDC_SUBTITLE.lookupOptional().orElse("OpenID Connect"));
+        oidc.setEnabled(JvmSettings.OIDC_ENABLED.lookupOptional(Boolean.class).orElse(true));
+        oidc.setHidden(JvmSettings.OIDC_HIDDEN_JSF.lookupOptional(Boolean.class).orElse(false));
 
         return oidc;
     }

src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java

@@ -1,6 +1,7 @@
 package edu.harvard.iq.dataverse.search;
 
 import edu.harvard.iq.dataverse.DataFile;
+import edu.harvard.iq.dataverse.DataFileServiceBean;
 import edu.harvard.iq.dataverse.Dataset;
 import edu.harvard.iq.dataverse.DatasetServiceBean;
 import edu.harvard.iq.dataverse.DatasetVersion;
@@ -53,6 +54,8 @@ public class SolrIndexServiceBean {
     @EJB
     SearchPermissionsServiceBean searchPermissionsService;
     @EJB
+    DataFileServiceBean dataFileService;
+    @EJB
     DataverseServiceBean dataverseService;
     @EJB
     DatasetServiceBean datasetService;
@@ -365,8 +368,30 @@ public IndexResponse indexPermissionsOnSelfAndChildren(DvObject definitionPoint)
             indexPermissionsForOneDvObject(definitionPoint);
             numObjects++;
 
-            // Process the dataset's files in a new transaction
-            self.indexDatasetFilesInNewTransaction(dataset.getId(), counter, fileQueryMin);
+            /**
+             * Prepare the data needed for the new transaction. For performance reasons, indexDatasetFilesInNewTransaction does not merge the dataset or versions into the new transaction (we only read info, there
+             * are no changes to write). However, there are two ways the code here is used. In one case, indexing content and permissions, the versions and fileMetadatas in them are already loaded. In the other
+             * case, indexing permissions only, the fileMetadatas are not yet loaded, and we may need them, but only if there are fewer than fileQueryMin. For each version that will get reindexed (at most two of
+             * them), the code below does a lightweight query to see how many fileMetadatas exist in it and, if it is equal to or below fileQueryMin, calls getFileMetadatas().size() to assure they are loaded
+             * (before we pass the version into a new transaction where it will be detached and fileMetadatas can't be loaded). Calling getFileMetadas.size() should be lightweight when the fileMetadatas are
+             * loaded (first case) and done only when needed for the second case.
+             * 
+             **/
+            Map<DatasetVersion.VersionState, Boolean> desiredCards = searchPermissionsService.getDesiredCards(dataset);
+            List<DatasetVersion> versionsToIndex = new ArrayList<>();
+            for (DatasetVersion version : versionsToReIndexPermissionsFor(dataset)) {
+                if (desiredCards.get(version.getVersionState())) {
+                    int fileCount = dataFileService.findCountByDatasetVersionId(version.getId()).intValue();
+                    if (fileCount <= fileQueryMin) {
+                        // IMPORTANT: This triggers the loading of fileMetadatas within the current transaction
+                        version.getFileMetadatas().size();
+                    }
+                    versionsToIndex.add(version);
+                }
+            }
+
+            // Process the dataset's files in a new transaction, passing the pre-loaded data
+            self.indexDatasetFilesInNewTransaction(versionsToIndex, counter, fileQueryMin);
         } else {
             // For other types (like files), just index in a new transaction
             indexPermissionsForOneDvObject(definitionPoint);
@@ -399,15 +424,11 @@ public void indexDatasetBatchInNewTransaction(List<Long> datasetIds, final int[]
     }
 
     @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
-    public void indexDatasetFilesInNewTransaction(Long datasetId, final int[] fileCounter, int fileQueryMin) {
-        Dataset dataset = datasetService.find(datasetId);
-        if (dataset != null) {
-            Map<DatasetVersion.VersionState, Boolean> desiredCards = searchPermissionsService.getDesiredCards(dataset);
-            for (DatasetVersion version : versionsToReIndexPermissionsFor(dataset)) {
-                if (desiredCards.get(version.getVersionState())) {
-                    processDatasetVersionFiles(version, fileCounter, fileQueryMin);
-                }
-            }
+    public void indexDatasetFilesInNewTransaction(List<DatasetVersion> versions, final int[] fileCounter, int fileQueryMin) {
+        for (DatasetVersion version : versions) {
+            // The version object is detached, but its fileMetadatas collection is already loaded.
+            // We only need its ID and state, which are available.
+            processDatasetVersionFiles(version, fileCounter, fileQueryMin);
         }
     }
 
@@ -421,22 +442,24 @@ private void processDatasetVersionFiles(DatasetVersion version,
         // Process files in batches of 100
         int batchSize = 100;
 
-        if (version.getFileMetadatas().size() > fileQueryMin) {
+        if (dataFileService.findCountByDatasetVersionId(version.getId()).intValue() > fileQueryMin) {
             // For large datasets, use a more efficient SQL query
-            Stream<DataFileProxy> fileStream = getDataFileInfoForPermissionIndexing(version.getId());
+            try (Stream<DataFileProxy> fileStream = getDataFileInfoForPermissionIndexing(version.getId())) {
 
-            // Process files in batches to avoid memory issues
-            fileStream.forEach(fileInfo -> {
-                filesToReindexAsBatch.add(fileInfo);
-                fileCounter[0]++;
+                // Process files in batches to avoid memory issues
+                fileStream.forEach(fileInfo -> {
+                    filesToReindexAsBatch.add(fileInfo);
+                    fileCounter[0]++;
 
-                if (filesToReindexAsBatch.size() >= batchSize) {
-                    reindexFilesInBatches(filesToReindexAsBatch, cachedPerms, versionId, solrIdEnd);
-                    filesToReindexAsBatch.clear();
-                }
-            });
+                    if (filesToReindexAsBatch.size() >= batchSize) {
+                        reindexFilesInBatches(filesToReindexAsBatch, cachedPerms, versionId, solrIdEnd);
+                        filesToReindexAsBatch.clear();
+                    }
+                });
+            }
         } else {
             // For smaller datasets, process files directly
+            // We only call getFileMetadatas() in the case where we know they have already been loaded
             for (FileMetadata fmd : version.getFileMetadatas()) {
                 DataFileProxy fileProxy = new DataFileProxy(fmd);
                 filesToReindexAsBatch.add(fileProxy);