Merge pull request #11477 from gdcc/10618-app-image-tags

Change CI workflows to release tagged container images for Dataverse

Author: ofahimIQSS

.github/CODEOWNERS

@@ -0,0 +1,8 @@
+
+# Any container related stuff should be assigned to / reviewed by Oliver and/or Phil
+modules/container-configbaker/** @poikilotherm @pdurbin
+modules/container-base/** @poikilotherm @pdurbin
+src/main/docker/** @poikilotherm @pdurbin
+docker-compose-dev.yml @poikilotherm @pdurbin
+.github/workflows/scripts/containers** @poikilotherm @pdurbin
+.github/workflows/container_* @poikilotherm @pdurbin

.github/workflows/container_app_pr.yml

@@ -1,15 +1,15 @@
 ---
 name: Preview Application Container Image
 
+# TODO: merge this workflow into the existing container_app_push.yaml flow - there's not much difference!
+
 on:
     # We only run the push commands if we are asked to by an issue comment with the correct command.
     # This workflow is always taken from the default branch and runs in repo context with access to secrets.
     repository_dispatch:
         types: [ push-image-command ]
 
 env:
-    IMAGE_TAG: unstable
-    BASE_IMAGE_TAG: unstable
     PLATFORMS: "linux/amd64,linux/arm64"
 
 jobs:
@@ -65,10 +65,9 @@ jobs:
                   install
             - name: Deploy multi-arch application and configbaker container image
               run: >
-                  mvn 
-                  -Dapp.image.tag=${{ env.IMAGE_TAG }} -Dbase.image.tag=${{ env.BASE_IMAGE_TAG }}
+                  mvn -Pct deploy 
+                  -Dapp.image.tag=${{ env.IMAGE_TAG }}
                   -Ddocker.registry=ghcr.io -Ddocker.platforms=${{ env.PLATFORMS }}
-                  -Pct deploy
 
             - uses: marocchino/sticky-pull-request-comment@v2
               with:

.github/workflows/container_app_push.yml

@@ -24,7 +24,6 @@ env:
     IMAGE_TAG: unstable
     REGISTRY: "" # Empty means default to Docker Hub
     PLATFORMS: "linux/amd64,linux/arm64"
-    MASTER_BRANCH_TAG: alpha
 
 jobs:
     build:
@@ -60,30 +59,6 @@ jobs:
 
             # TODO: add smoke / integration testing here (add "-Pct -DskipIntegrationTests=false")
 
-    hub-description:
-        needs: build
-        name: Push image descriptions to Docker Hub
-        # Run this when triggered via push or schedule as reused workflow from base / maven unit tests.
-        # Excluding PRs here means we will have no trouble with secrets access. Also avoid runs in forks.
-        if: ${{ github.event_name != 'pull_request' && github.ref_name == 'develop' && github.repository_owner == 'IQSS' }}
-        runs-on: ubuntu-latest
-        steps:
-            - uses: actions/checkout@v4
-            - uses: peter-evans/dockerhub-description@v4
-              with:
-                  username: ${{ secrets.DOCKERHUB_USERNAME }}
-                  password: ${{ secrets.DOCKERHUB_TOKEN }}
-                  repository: gdcc/dataverse
-                  short-description: "Dataverse Application Container Image providing the executable"
-                  readme-filepath: ./src/main/docker/README.md
-            - uses: peter-evans/dockerhub-description@v4
-              with:
-                  username: ${{ secrets.DOCKERHUB_USERNAME }}
-                  password: ${{ secrets.DOCKERHUB_TOKEN }}
-                  repository: gdcc/configbaker
-                  short-description: "Dataverse Config Baker Container Image providing setup tooling and more"
-                  readme-filepath: ./modules/container-configbaker/README.md
-
     # Note: Accessing, pushing tags etc. to DockerHub or GHCR will only succeed in upstream because secrets.
     # We check for them here and subsequent jobs can rely on this to decide if they shall run.
     check-secrets:
@@ -107,13 +82,13 @@ jobs:
         needs: check-secrets
         name: "Package & Publish"
         runs-on: ubuntu-latest
-        # Only run this job if we have access to secrets. This is true for events like push/schedule which run in
+        # Only run this job if we have access to secrets. This is true for events like push/schedule which run in the
         # context of the main repo, but for PRs only true if coming from the main repo! Forks have no secret access.
         #
         # Note: The team's decision was to not auto-deploy an image on any git push where no PR exists (yet).
-        #       Accordingly, only run for push events on branches develop and master.
+        #       Accordingly, only run for push events on the 'develop' branch.
         if: needs.check-secrets.outputs.available == 'true' &&
-            ( github.event_name != 'push' || ( github.event_name == 'push' && contains(fromJSON('["develop", "master"]'), github.ref_name)))
+            ( github.event_name != 'push' || ( github.event_name == 'push' && github.ref_name == 'develop' ))
         steps:
             - name: Checkout and Setup Maven
               uses: IQSS/dataverse/.github/actions/setup-maven@develop
@@ -141,16 +116,15 @@ jobs:
             - name: Set up QEMU for multi-arch builds
               uses: docker/setup-qemu-action@v3
 
-            - name: Re-set image tag based on branch (if master)
-              if: ${{ github.ref_name == 'master' }}
+            - name: Add rolling image tag when pushing to develop
+              if: ${{ github.event_name == 'push' && github.ref_name == 'develop' }}
               run: |
-                  echo "IMAGE_TAG=${{ env.MASTER_BRANCH_TAG }}" >> $GITHUB_ENV
-                  echo "BASE_IMAGE_TAG=${{ env.MASTER_BRANCH_TAG }}" >> $GITHUB_ENV
+                  echo "ADDITIONAL_TAGS=-Ddocker.tags.upcoming=$( mvn initialize help:evaluate -Pct -Dexpression=app.image.tag -Dapp.image.tag='${app.image.version}-${base.image.flavor}' -q -DforceStdout )" | tee -a "$GITHUB_ENV"
             - name: Re-set image tag and container registry when on PR
               if: ${{ github.event_name == 'pull_request' }}
               run: |
-                  echo "IMAGE_TAG=$(echo "$GITHUB_HEAD_REF" | tr '\\/_:&+,;#*' '-')" >> $GITHUB_ENV
-                  echo "REGISTRY='-Ddocker.registry=ghcr.io'" >> $GITHUB_ENV
+                  echo "IMAGE_TAG=$(echo "$GITHUB_HEAD_REF" | tr '\\/_:&+,;#*' '-')" | tee -a "$GITHUB_ENV"
+                  echo "REGISTRY='-Ddocker.registry=ghcr.io'" | tee -a "$GITHUB_ENV"
 
             # Necessary to split as otherwise the submodules are not available (deploy skips install)
             - name: Build app and configbaker container image with local architecture and submodules (profile will skip tests)
@@ -162,7 +136,7 @@ jobs:
             - name: Deploy multi-arch application and configbaker container image
               run: >
                   mvn 
-                  -Dapp.image.tag=${{ env.IMAGE_TAG }}
+                  -Dapp.image.tag=${{ env.IMAGE_TAG }} ${{ env.ADDITIONAL_TAGS }}
                   $( [[ -n "${{ inputs.base-image-ref }}" ]] && echo "-Dbase.image=${{ inputs.base-image-ref }}" )
                   ${{ env.REGISTRY }} -Ddocker.platforms=${{ env.PLATFORMS }}
                   -P ct deploy

.github/workflows/container_base_push.yml

@@ -1,10 +1,8 @@
 ---
-name: Container Images Releasing
+name: Base Container Image
 
 on:
     push:
-        tags:
-            - 'v[6-9].**'
         branches:
             - 'develop'
         # "Path filters are not evaluated for pushes of tags" https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#onpushpull_requestpull_request_targetpathspaths-ignore
@@ -33,7 +31,7 @@ jobs:
         # Only run in upstream repo - avoid unnecessary runs in forks
         if: ${{ github.repository_owner == 'IQSS' }}
         outputs:
-          base-image-ref: ${{ steps.finalize.outputs.base-image-ref }}
+          base-image-ref: ${{ steps.determine-name.outputs.full-ref }}
 
         steps:
             - name: Checkout and Setup Maven
@@ -52,12 +50,10 @@ jobs:
             # In case this is a push to develop, we care about buildtime.
             # Configure a remote ARM64 build host in addition to the local AMD64 in two steps.
             - name: Setup SSH agent
-              if: ${{ github.event_name != 'schedule' }}
               uses: webfactory/ssh-agent@v0.9.1
               with:
                   ssh-private-key: ${{ secrets.BUILDER_ARM64_SSH_PRIVATE_KEY }}
             - name: Provide the known hosts key and the builder config
-              if: ${{ github.event_name != 'schedule' }}
               run: |
                   echo "${{ secrets.BUILDER_ARM64_SSH_HOST_KEY }}" > ~/.ssh/known_hosts
                   mkdir -p modules/container-base/target/buildx-state/buildx/instances
@@ -81,40 +77,25 @@ jobs:
 
             # Determine the base image name we are going to use from here on
             - name: Determine base image name
+              id: determine-name
               run: |
-                if [[ "${{ github.ref_name }}" = "${{ env.DEVELOPMENT_BRANCH }}" ]]; then
-                  echo "BASE_IMAGE=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout )" | tee -a "${GITHUB_ENV}"
-                  echo "BASE_IMAGE_UPCOMING=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout )" | tee -a "${GITHUB_ENV}"
-                else
-                  echo "BASE_IMAGE=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout )" | tee -a "${GITHUB_ENV}"
-                fi
-            - name: Calculate revision number for immutable tag (on release branches only)
-              if: ${{ github.ref_name != env.DEVELOPMENT_BRANCH }}
-              id: revision-tag
-              uses: ./.github/actions/get-image-revision
-              with:
-                image-ref: ${{ env.BASE_IMAGE }}
-                tag-options-prefix: "-Dbase.image.tag.suffix='' -Ddocker.tags.revision="
+                BASE_IMAGE=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout )
+                BASE_IMAGE_UPCOMING=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout )
+                
+                echo "BASE_IMAGE=${BASE_IMAGE}" | tee -a "${GITHUB_ENV}"
+                echo "BASE_IMAGE_UPCOMING=${BASE_IMAGE_UPCOMING}" | tee -a "${GITHUB_ENV}"
+                echo "full-ref=${BASE_IMAGE_UPCOMING}" | tee -a "$GITHUB_OUTPUT"
+
             - name: Configure update of "latest" tag for development branch
               id: develop-tag
-              if: ${{ github.ref_name == env.DEVELOPMENT_BRANCH }}
               run: |
                 echo "tag-options=-Ddocker.tags.develop=unstable -Ddocker.tags.upcoming=${BASE_IMAGE_UPCOMING#*:}" | tee -a "${GITHUB_OUTPUT}"
 
             - name: Deploy multi-arch base container image to Docker Hub
               id: build
               run: |
                 mvn -f modules/container-base -Pct deploy -Ddocker.noCache -Ddocker.platforms=${{ env.PLATFORMS }} \
-                  -Ddocker.imagePropertyConfiguration=override ${{ steps.develop-tag.outputs.tag-options }} ${{ steps.revision-tag.outputs.tag-options }}
-
-            - name: Determine appropriate base image ref for app image
-              id: finalize
-              run: |
-                if [[ "${{ github.ref_name }}" = "${{ env.DEVELOPMENT_BRANCH }}" ]]; then
-                  echo "base-image-ref=${BASE_IMAGE_UPCOMING}" | tee -a "$GITHUB_OUTPUT"
-                else
-                  echo "base-image-ref=gdcc/base:${{ steps.revision-tag.outputs.revision-tag }}" | tee -a "$GITHUB_OUTPUT"
-                fi
+                  -Ddocker.imagePropertyConfiguration=override ${{ steps.develop-tag.outputs.tag-options }}
 
     push-app-img:
         name: "Rebase & Publish App Image"