Merge branch 'develop' into 9331-extract-bounding-box2 #9331

Author: pdurbin

.github/workflows/container_app_push.yml

@@ -0,0 +1,102 @@
+---
+name: Application Container Image
+
+on:
+    # We are deliberately *not* running on push events here to avoid double runs.
+    # Instead, push events will trigger from the base image and maven unit tests via workflow_call.
+    workflow_call:
+    pull_request:
+        branches:
+            - develop
+            - master
+        paths:
+            - 'src/main/docker/**'
+            - '.github/workflows/container_app_push.yml'
+
+env:
+    IMAGE_TAG: unstable
+    BASE_IMAGE_TAG: unstable
+    REGISTRY: "" # Empty means default to Docker Hub
+    PLATFORMS: "linux/amd64,linux/arm64"
+
+jobs:
+    build:
+        name: Build & deploy
+        runs-on: ubuntu-latest
+        permissions:
+            contents: read
+            packages: write
+            pull-requests: write
+        # Only run in upstream repo - avoid unnecessary runs in forks
+        if: ${{ github.repository_owner == 'IQSS' }}
+
+        steps:
+            - name: Checkout repository
+              uses: actions/checkout@v3
+
+            - name: Set up JDK 11
+              uses: actions/setup-java@v3
+              with:
+                  java-version: "11"
+                  distribution: 'adopt'
+            - name: Cache Maven packages
+              uses: actions/cache@v3
+              with:
+                  path: ~/.m2
+                  key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+                  restore-keys: ${{ runner.os }}-m2
+
+            - name: Build app container image with local architecture
+              run: mvn -Pct package
+
+            # Note: Accessing, pushing tags etc. to DockerHub or GHCR will only succeed in upstream because secrets.
+
+            # Run this when triggered via push or schedule as reused workflow from base / maven unit tests
+            - if: ${{ github.event_name != 'pull_request' && github.ref_name == 'develop' }}
+              name: Push description to DockerHub
+              uses: peter-evans/dockerhub-description@v3
+              with:
+                  username: ${{ secrets.DOCKERHUB_USERNAME }}
+                  password: ${{ secrets.DOCKERHUB_TOKEN }}
+                  repository: gdcc/dataverse
+                  short-description: "Dataverse Application Container Image providing the executable"
+                  readme-filepath: ./src/main/docker/README.md
+
+            - if: ${{ github.event_name != 'pull_request' }}
+              name: Log in to Docker Hub registry
+              uses: docker/login-action@v2
+              with:
+                  username: ${{ secrets.DOCKERHUB_USERNAME }}
+                  password: ${{ secrets.DOCKERHUB_TOKEN }}
+            - if: ${{ github.event_name == 'pull_request' }}
+              name: Login to Github Container Registry
+              uses: docker/login-action@v2
+              with:
+                  registry: ghcr.io
+                  username: ${{ secrets.GHCR_USERNAME }}
+                  password: ${{ secrets.GHCR_TOKEN }}
+
+            - name: Set up QEMU for multi-arch builds
+              uses: docker/setup-qemu-action@v2
+
+            - name: Re-set image tag based on branch (if master)
+              if: ${{ github.ref_name == 'master' }}
+              run: |
+                  echo "IMAGE_TAG=alpha" >> $GITHUB_ENV
+                  echo "BASE_IMAGE_TAG=alpha" >> $GITHUB_ENV
+            - name: Re-set image tag and container registry when on PR
+              if: ${{ github.event_name == 'pull_request' }}
+              run: |
+                  echo "IMAGE_TAG=$(echo "$GITHUB_HEAD_REF" | tr '\\/_:&+,;#*' '-')" >> $GITHUB_ENV
+                  echo "REGISTRY='-Ddocker.registry=ghcr.io'" >> $GITHUB_ENV
+
+            - name: Deploy multi-arch application container image
+              run: mvn -Pct deploy -Dapp.image.tag=${{ env.IMAGE_TAG }} -Dbase.image.tag=${{ env.BASE_IMAGE_TAG }} ${{ env.REGISTRY }} -Ddocker.platforms=${{ env.PLATFORMS }}
+
+            - uses: marocchino/sticky-pull-request-comment@v2
+              if: ${{ github.event_name == 'pull_request' }}
+              with:
+                  header: app-registry-push
+                  message: |
+                      Pushed preview application image as [`ghcr.io/gdcc/dataverse:${{ env.IMAGE_TAG }}`](https://github.com/orgs/gdcc/packages/container/package/dataverse).
+                      Use it by referencing it with its full name as printed above.

.github/workflows/container_base_push.yml

@@ -1,5 +1,5 @@
 ---
-name: Container Base Module
+name: Base Container Image
 
 on:
     push:
@@ -18,9 +18,12 @@ on:
             - 'modules/container-base/**'
             - 'modules/dataverse-parent/pom.xml'
             - '.github/workflows/container_base_push.yml'
+    schedule:
+        - cron: '23 3 * * 0' # Run for 'develop' every Sunday at 03:23 UTC
 
 env:
     IMAGE_TAG: unstable
+    PLATFORMS: linux/amd64,linux/arm64
 
 jobs:
     build:
@@ -79,7 +82,18 @@ jobs:
               uses: docker/setup-qemu-action@v2
             - name: Re-set image tag based on branch
               if: ${{ github.ref_name == 'master' }}
-              run: echo "IMAGE_TAG=stable"
+              run: echo "IMAGE_TAG=alpha" >> $GITHUB_ENV
             - if: ${{ github.event_name != 'pull_request' }}
               name: Deploy multi-arch base container image to Docker Hub
-              run: mvn -f modules/container-base -Pct deploy -Dbase.image.tag=${{ env.IMAGE_TAG }}
+              run: mvn -f modules/container-base -Pct deploy -Dbase.image.tag=${{ env.IMAGE_TAG }} -Ddocker.platforms=${{ env.PLATFORMS }}
+    push-app-img:
+        name: "Rebase & Publish App Image"
+        permissions:
+            contents: read
+            packages: write
+            pull-requests: write
+        needs: build
+        # We do not release a new base image for pull requests, so do not trigger.
+        if: ${{ github.event_name != 'pull_request' }}
+        uses: ./.github/workflows/container_app_push.yml
+        secrets: inherit

.github/workflows/maven_unit_test.yml

@@ -51,4 +51,13 @@ jobs:
             env:
                 CI_NAME: github
                 COVERALLS_SECRET: ${{ secrets.GITHUB_TOKEN }}
-            run: mvn -V -B jacoco:report coveralls:report -DrepoToken=${COVERALLS_SECRET} -DpullRequest=${{ github.event.number }}
+            run: mvn -V -B jacoco:report coveralls:report -DrepoToken=${COVERALLS_SECRET} -DpullRequest=${{ github.event.number }}
+    push-app-img:
+        name: Publish App Image
+        permissions:
+            contents: read
+            packages: write
+            pull-requests: write
+        needs: unittest
+        uses: ./.github/workflows/container_app_push.yml
+        secrets: inherit

doc/release-notes/8092-timestamp-of-data-access-request.md

@@ -0,0 +1,3 @@
+A date column has been added to the restricted file access request overview, indicating when the earliest request by that user was made.
+
+An issue was fixed where where the request list was not updated when a request was approved or rejected.

doc/release-notes/9150-improved-external-vocab-supprt.md

@@ -0,0 +1 @@
+It is now possible to write external vocabulary scripts that target a single child field in a metadata block. Example scripts are now available at https://github.com/gdcc/dataverse-external-vocab-support that can be configured to support lookup from the Research Orgnaization Registry (ROR) for the Author Affiliation Field and for the CrossRef Funding Registry (Fundreg) in the Funding Information/Agency field, both in the standard Citation metadata block. Application if these scripts to other fields, and the development of other scripts targetting child fields are now possible.

doc/sphinx-guides/source/container/base-image.rst

@@ -31,7 +31,7 @@ upstream branches:
 
 - The ``unstable`` tag corresponds to the ``develop`` branch, where pull requests are merged.
   (`Dockerfile <https://github.com/IQSS/dataverse/tree/develop/modules/container-base/src/main/docker/Dockerfile>`__)
-- The ``stable`` tag corresponds to the ``master`` branch, where releases are cut from.
+- The ``alpha`` tag corresponds to the ``master`` branch, where releases are cut from.
   (`Dockerfile <https://github.com/IQSS/dataverse/tree/master/modules/container-base/src/main/docker/Dockerfile>`__)
 
 
@@ -108,13 +108,19 @@ AMD64 (Windows/Linux/...) and ARM64 (Apple M1/M2), by using `Maven Docker Plugin
 Building the image via ``mvn -Pct package`` or ``mvn -Pct install`` as above will only build for the architecture of
 the Docker machine's CPU.
 
-Only ``mvn -Pct deploy`` will trigger building on all enabled architectures.
-Yet, to enable building with non-native code on your build machine, you will need to setup a cross-platform builder.
+Only ``mvn -Pct deploy`` will trigger building on all enabled architectures (and will try to push the images to a
+registry, which is Docker Hub by default).
+
+You can specify which architectures you would like to build for and include by them as a comma separated list:
+``mvn -Pct deploy -Ddocker.platforms="linux/amd64,linux/arm64"``. The shown configuration is the default and may be omitted.
+
+Yet, to enable building with non-native code on your build machine, you will need to setup a cross-platform builder!
 
 On Linux, you should install `qemu-user-static <https://github.com/multiarch/qemu-user-static>`__ (preferably via
 your package management) on the host and run ``docker run --rm --privileged multiarch/qemu-user-static --reset -p yes``
 to enable that builder. The Docker plugin will setup everything else for you.
 
+The upstream CI workflows publish images supporting AMD64 and ARM64 (see e.g. tag details on Docker Hub)
 
 .. _base-tunables:
 

modules/container-base/README.md

@@ -7,7 +7,9 @@ Adding basic functionality like executing scripts at container boot, monitoring,
 at this layer. Application images building from this very base focus on adding deployable Dataverse code and 
 actual scripts.
 
-*Note:* Currently, there is no application image. Please watch https://github.com/IQSS/dataverse/issues/8934
+There is a community based [application image](https://hub.docker.com/r/gdcc/dataverse) 
+([docs](https://guides.dataverse.org/en/latest/container/app-image.html)), but you may create your own or even reuse
+this image for other purposes than the Dataverse application.
 
 ## Quick Reference
 
@@ -23,8 +25,8 @@ provides in-depth information about content, building, tuning and so on for this
 **Where to get help and ask questions:**
 
 IQSS will not offer support on how to deploy or run it. Please reach out to the community for help on using it.
-You can join the Community Chat on Matrix at https://chat.dataverse.org or the Community Slack at
-https://dataversecommunity.slack.com to ask for help and guidance.
+You can join the Community Chat on Matrix at https://chat.dataverse.org and https://groups.google.com/g/dataverse-community
+to ask for help and guidance.
 
 ## Supported Image Tags
 
@@ -34,7 +36,7 @@ happens there (again, by the community). Community-supported image tags are base
 
 - The `unstable` tag corresponds to the `develop` branch, where pull requests are merged.
   ([`Dockerfile`](https://github.com/IQSS/dataverse/tree/develop/modules/container-base/src/main/docker/Dockerfile))
-- The `stable` tag corresponds to the `master` branch, where releases are cut from.
+- The `alpha` tag corresponds to the `master` branch, where releases are cut from.
   ([`Dockerfile`](https://github.com/IQSS/dataverse/tree/master/modules/container-base/src/main/docker/Dockerfile))
 
 Within the main repository, you may find the base image files at `<git root>/modules/container-base`.

modules/container-base/pom.xml

@@ -44,6 +44,7 @@
                 <java.image>eclipse-temurin:${target.java.version}-jre</java.image>
                 <base.image.uid>1000</base.image.uid>
                 <base.image.gid>1000</base.image.gid>
+                <docker.platforms>linux/amd64,linux/arm64</docker.platforms>
             </properties>
 
             <build>
@@ -94,8 +95,8 @@
                                     <build>
                                         <buildx>
                                             <platforms>
-                                                <platform>linux/arm64</platform>
-                                                <platform>linux/amd64</platform>
+                                                <!-- Will be empty by default, deactivating buildx -->
+                                                <platform>${docker.platforms}</platform>
                                             </platforms>
                                             <dockerStateDir>${project.build.directory}/buildx-state</dockerStateDir>
                                         </buildx>

modules/container-base/src/main/docker/scripts/init_1_generate_deploy_commands.sh

@@ -61,5 +61,6 @@ find "$DEPLOY_DIR" -mindepth 1 -maxdepth 1 -name "*.rar" -print0 \
   | while IFS= read -r -d '' file; do deploy "$file"; done
 
 # Then every other WAR, EAR, JAR or directory
-find "$DEPLOY_DIR" -mindepth 1 -maxdepth 1 ! -name "*.rar" -a -name "*.war" -o -name "*.ear" -o -name "*.jar" -o -type d -print0 \
-  | while IFS= read -r -d '' file; do deploy "$file"; done
+find "$DEPLOY_DIR" -mindepth 1 -maxdepth 1 \
+  \( ! -name "*.rar" -a -name "*.war" -o -name "*.ear" -o -name "*.jar" -o -type d \) \
+  -print0 | while IFS= read -r -d '' file; do deploy "$file"; done

modules/dataverse-parent/pom.xml

@@ -189,7 +189,7 @@
         <maven-checkstyle-plugin.version>3.1.2</maven-checkstyle-plugin.version>
 
         <!-- Container related -->
-        <fabric8-dmp.version>0.42.0</fabric8-dmp.version>
+        <fabric8-dmp.version>0.42.1</fabric8-dmp.version>
     </properties>
 
     <pluginRepositories>
@@ -345,8 +345,9 @@
                 <!--
                     Payara 5.2022.3 has problems with postboot deployment scripts.
                     Fixed in this release, see https://github.com/payara/Payara/pull/5991
+                    Payara 5.2022.4 has security issues.
                 -->
-                <payara.version>5.2022.4</payara.version>
+                <payara.version>5.2022.5</payara.version>
             </properties>
 
             <build>