Merge remote-tracking branch 'IQSS/develop' into DANS-external_exporters

Author: qqmyers

.github/workflows/container_app_pr.yml

@@ -58,36 +58,39 @@ jobs:
                   echo "IMAGE_TAG=$(echo "${{ github.event.client_payload.pull_request.head.ref }}" | tr '\\/_:&+,;#*' '-')" >> $GITHUB_ENV
 
             # Necessary to split as otherwise the submodules are not available (deploy skips install)
-            - name: Build app container image with local architecture and submodules (profile will skip tests)
+            - name: Build app and configbaker container image with local architecture and submodules (profile will skip tests)
               run: >
                   mvn -B -f modules/dataverse-parent
                   -P ct -pl edu.harvard.iq:dataverse -am
                   install
-            - name: Deploy multi-arch application container image
+            - name: Deploy multi-arch application and configbaker container image
               run: >
                   mvn 
                   -Dapp.image.tag=${{ env.IMAGE_TAG }} -Dbase.image.tag=${{ env.BASE_IMAGE_TAG }}
-                  ${{ env.REGISTRY }} -Ddocker.platforms=${{ env.PLATFORMS }}
-                  -P ct deploy
+                  -Ddocker.registry=ghcr.io -Ddocker.platforms=${{ env.PLATFORMS }}
+                  -Pct deploy
 
             - uses: marocchino/sticky-pull-request-comment@v2
               with:
-                  header: app-registry-push
+                  header: registry-push
                   hide_and_recreate: true
                   hide_classify: "OUTDATED"
                   number: ${{ github.event.client_payload.pull_request.number }}
                   message: |
-                      :package: Pushed preview application image as
+                      :package: Pushed preview images as
                       ```
                       ghcr.io/gdcc/dataverse:${{ env.IMAGE_TAG }}
                       ```
-                      :ship: [See on GHCR](https://github.com/orgs/gdcc/packages/container/package/dataverse). Use by referencing with full name as printed above, mind the registry name.
+                      ```
+                      ghcr.io/gdcc/configbaker:${{ env.IMAGE_TAG }}
+                      ```
+                      :ship: [See on GHCR](https://github.com/orgs/gdcc/packages/container). Use by referencing with full name as printed above, mind the registry name.
 
             # Leave a note when things have gone sideways
             - uses: peter-evans/create-or-update-comment@v3
               if: ${{ failure() }}
               with:
                   issue-number: ${{ github.event.client_payload.pull_request.number }}
                   body: >
-                      :package: Could not push preview image :disappointed:.
-                      See [log](https://github.com/IQSS/dataverse/actions/runs/${{ github.run_id }}) for details.
+                      :package: Could not push preview images :disappointed:.
+                      See [log](https://github.com/IQSS/dataverse/actions/runs/${{ github.run_id }}) for details.

.github/workflows/container_app_push.yml

@@ -11,6 +11,7 @@ on:
             - master
         paths:
             - 'src/main/docker/**'
+            - 'modules/container-configbaker/**'
             - '.github/workflows/container_app_push.yml'
 
 env:
@@ -42,7 +43,7 @@ jobs:
                   distribution: temurin
                   cache: maven
 
-            - name: Build app container image with local architecture and submodules (profile will skip tests)
+            - name: Build app and configbaker container image with local architecture and submodules (profile will skip tests)
               run: >
                   mvn -B -f modules/dataverse-parent
                   -P ct -pl edu.harvard.iq:dataverse -am
@@ -52,7 +53,7 @@ jobs:
 
     hub-description:
         needs: build
-        name: Push image description to Docker Hub
+        name: Push image descriptions to Docker Hub
         # Run this when triggered via push or schedule as reused workflow from base / maven unit tests.
         # Excluding PRs here means we will have no trouble with secrets access. Also avoid runs in forks.
         if: ${{ github.event_name != 'pull_request' && github.ref_name == 'develop' && github.repository_owner == 'IQSS' }}
@@ -66,6 +67,13 @@ jobs:
                   repository: gdcc/dataverse
                   short-description: "Dataverse Application Container Image providing the executable"
                   readme-filepath: ./src/main/docker/README.md
+            - uses: peter-evans/dockerhub-description@v3
+              with:
+                  username: ${{ secrets.DOCKERHUB_USERNAME }}
+                  password: ${{ secrets.DOCKERHUB_TOKEN }}
+                  repository: gdcc/configbaker
+                  short-description: "Dataverse Config Baker Container Image providing setup tooling and more"
+                  readme-filepath: ./modules/container-configbaker/README.md
 
     # Note: Accessing, pushing tags etc. to DockerHub or GHCR will only succeed in upstream because secrets.
     # We check for them here and subsequent jobs can rely on this to decide if they shall run.
@@ -130,12 +138,12 @@ jobs:
                   echo "REGISTRY='-Ddocker.registry=ghcr.io'" >> $GITHUB_ENV
 
             # Necessary to split as otherwise the submodules are not available (deploy skips install)
-            - name: Build app container image with local architecture and submodules (profile will skip tests)
+            - name: Build app and configbaker container image with local architecture and submodules (profile will skip tests)
               run: >
                   mvn -B -f modules/dataverse-parent
                   -P ct -pl edu.harvard.iq:dataverse -am
                   install
-            - name: Deploy multi-arch application container image
+            - name: Deploy multi-arch application and configbaker container image
               run: >
                   mvn 
                   -Dapp.image.tag=${{ env.IMAGE_TAG }} -Dbase.image.tag=${{ env.BASE_IMAGE_TAG }}
@@ -145,12 +153,15 @@ jobs:
             - uses: marocchino/sticky-pull-request-comment@v2
               if: ${{ github.event_name == 'pull_request' }}
               with:
-                  header: app-registry-push
+                  header: registry-push
                   hide_and_recreate: true
                   hide_classify: "OUTDATED"
                   message: |
-                      :package: Pushed preview application image as
+                      :package: Pushed preview images as
                       ```
                       ghcr.io/gdcc/dataverse:${{ env.IMAGE_TAG }}
                       ```
-                      :ship: [See on GHCR](https://github.com/orgs/gdcc/packages/container/package/dataverse). Use by referencing with full name as printed above, mind the registry name.
+                      ```
+                      ghcr.io/gdcc/configbaker:${{ env.IMAGE_TAG }}
+                      ```
+                      :ship: [See on GHCR](https://github.com/orgs/gdcc/packages/container). Use by referencing with full name as printed above, mind the registry name.

.github/workflows/shellcheck.yml

@@ -1,24 +1,47 @@
 name: "Shellcheck"
 on:
     push:
+        branches:
+            - develop
         paths:
-            - conf/solr/**
-            - modules/container-base/**
+            - conf/solr/**/.sh
+            - modules/container-base/**/*.sh
+            - modules/container-configbaker/**/*.sh
     pull_request:
+        branches:
+            - develop
         paths:
-            - conf/solr/**
-            - modules/container-base/**
+            - conf/solr/**/*.sh
+            - modules/container-base/**/*.sh
+            - modules/container-configbaker/**/*.sh
 jobs:
     shellcheck:
         name: Shellcheck
         runs-on: ubuntu-latest
+        permissions:
+            pull-requests: write
         steps:
-            - uses: actions/checkout@v2
+            - uses: actions/checkout@v3
             - name: shellcheck
               uses: reviewdog/action-shellcheck@v1
               with:
                   github_token: ${{ secrets.github_token }}
                   reporter: github-pr-review # Change reporter.
                   fail_on_error: true
                   # Container base image uses dumb-init shebang, so nail to using bash
-                  shellcheck_flags: "--shell=bash --external-sources"
+                  shellcheck_flags: "--shell=bash --external-sources"
+                  # Exclude old scripts
+                  exclude: |
+                      */.git/*
+                      conf/docker-aio/*
+                      doc/*
+                      downloads/*
+                      scripts/database/*
+                      scripts/globalid/*
+                      scripts/icons/*
+                      scripts/installer/*
+                      scripts/issues/*
+                      scripts/r/*
+                      scripts/tests/*
+                      scripts/vagrant/*
+                      tests/*

conf/solr/8.11.1/update-fields.sh

@@ -2,6 +2,8 @@
 
 set -euo pipefail
 
+# [INFO]: Update a prepared Solr schema.xml for Dataverse with a given list of metadata fields
+
 #### #### #### #### #### #### #### #### #### #### #### #### #### #### #### #### #### ####
 # This script will
 # 1. take a file (or read it from STDIN) with all <field> and <copyField> definitions

doc/release-notes/6542-mdc-legacy-counts.md

@@ -0,0 +1,3 @@
+###For installations using MDC (Make Data Count), it is now possible to display both the MDC metrics and the legacy access counts, generated before MDC was enabled.
+
+This is enabled via the new setting `:MDCStartDate` that specifies the cutoff date. If a dataset has any legacy access counts collected prior to that date, those numbers will be displayed in addition to the any MDC numbers recorded since then.

doc/release-notes/8889-filepids-in-collections.md

@@ -0,0 +1,3 @@
+It is now possible to configure registering PIDs for files in individual collections.
+
+For example, registration of PIDs for files can be enabled in a specific collection when it is disabled instance-wide. Or it can be disabled in specific collections where it is enabled by default. See the [:FilePIDsEnabled](https://guides.dataverse.org/en/latest/installation/config.html#filepidsenabled) section of the Configuration guide for details.

doc/release-notes/9431-checksum-alg-in-direct-uploads.md

@@ -0,0 +1,4 @@
+Direct upload via the Dataverse UI will now support any algorithm configured via the :FileFixityChecksumAlgorithm setting.
+External apps using the direct upload API can now query Dataverse to discover which algorithm should be used.
+
+Sites that have been using an algorithm other than MD5 and direct upload and/or dvwebloader may want to use the /api/admin/updateHashValues call (see https://guides.dataverse.org/en/latest/installation/config.html?highlight=updatehashvalues#filefixitychecksumalgorithm) to replace any MD5 hashes on existing files.

doc/release-notes/9480-h5web.md

@@ -0,0 +1 @@
+A file previewer called H5Web is now available for exploring and visualizing NetCDF and HDF5 files.

doc/release-notes/9558-async-indexing.md

@@ -0,0 +1,3 @@
+Performance improvements, especially for large datasets containing thousands of files.
+Uploading files one by one to the dataset is much faster now, allowing uploading thousands of files in an acceptable timeframe. Not only uploading a file, but all edit operations on datasets containing many files, got faster.
+Performance tweaks include indexing of the datasets in the background and optimizations in the amount of the indexing operations needed. Furthermore, updates to the dateset no longer wait for ingesting to finish. Ingesting was already running in the background, but it took a lock, preventing updating the dataset and degrading performance for datasets containing many files.

doc/release-notes/9573-configbaker.md

@@ -0,0 +1 @@
+A container has been added called "configbaker" that configures Dataverse while running in containers. This allows developers to spin up Dataverse with a single command.