diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000000..5c9ad7581f8 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,8 @@ + +# Any container related stuff should be assigned to / reviewed by Oliver and/or Phil +modules/container-configbaker/** @poikilotherm @pdurbin +modules/container-base/** @poikilotherm @pdurbin +src/main/docker/** @poikilotherm @pdurbin +docker-compose-dev.yml @poikilotherm @pdurbin +.github/workflows/scripts/containers** @poikilotherm @pdurbin +.github/workflows/container_* @poikilotherm @pdurbin diff --git a/.github/workflows/container_app_pr.yml b/.github/workflows/container_app_pr.yml index 4a06cb567b0..a4c52805156 100644 --- a/.github/workflows/container_app_pr.yml +++ b/.github/workflows/container_app_pr.yml @@ -1,6 +1,8 @@ --- name: Preview Application Container Image +# TODO: merge this workflow into the existing container_app_push.yaml flow - there's not much difference! + on: # We only run the push commands if we are asked to by an issue comment with the correct command. # This workflow is always taken from the default branch and runs in repo context with access to secrets. @@ -8,8 +10,6 @@ on: types: [ push-image-command ] env: - IMAGE_TAG: unstable - BASE_IMAGE_TAG: unstable PLATFORMS: "linux/amd64,linux/arm64" jobs: @@ -65,10 +65,9 @@ jobs: install - name: Deploy multi-arch application and configbaker container image run: > - mvn - -Dapp.image.tag=${{ env.IMAGE_TAG }} -Dbase.image.tag=${{ env.BASE_IMAGE_TAG }} + mvn -Pct deploy + -Dapp.image.tag=${{ env.IMAGE_TAG }} -Ddocker.registry=ghcr.io -Ddocker.platforms=${{ env.PLATFORMS }} - -Pct deploy - uses: marocchino/sticky-pull-request-comment@v2 with: diff --git a/.github/workflows/container_app_push.yml b/.github/workflows/container_app_push.yml index 71ffffb5f48..0472ab97dee 100644 --- a/.github/workflows/container_app_push.yml +++ b/.github/workflows/container_app_push.yml @@ -24,7 +24,6 @@ env: IMAGE_TAG: unstable REGISTRY: "" # Empty means default to Docker Hub PLATFORMS: "linux/amd64,linux/arm64" - MASTER_BRANCH_TAG: alpha jobs: build: @@ -60,30 +59,6 @@ jobs: # TODO: add smoke / integration testing here (add "-Pct -DskipIntegrationTests=false") - hub-description: - needs: build - name: Push image descriptions to Docker Hub - # Run this when triggered via push or schedule as reused workflow from base / maven unit tests. - # Excluding PRs here means we will have no trouble with secrets access. Also avoid runs in forks. - if: ${{ github.event_name != 'pull_request' && github.ref_name == 'develop' && github.repository_owner == 'IQSS' }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: peter-evans/dockerhub-description@v4 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - repository: gdcc/dataverse - short-description: "Dataverse Application Container Image providing the executable" - readme-filepath: ./src/main/docker/README.md - - uses: peter-evans/dockerhub-description@v4 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - repository: gdcc/configbaker - short-description: "Dataverse Config Baker Container Image providing setup tooling and more" - readme-filepath: ./modules/container-configbaker/README.md - # Note: Accessing, pushing tags etc. to DockerHub or GHCR will only succeed in upstream because secrets. # We check for them here and subsequent jobs can rely on this to decide if they shall run. check-secrets: @@ -107,13 +82,13 @@ jobs: needs: check-secrets name: "Package & Publish" runs-on: ubuntu-latest - # Only run this job if we have access to secrets. This is true for events like push/schedule which run in + # Only run this job if we have access to secrets. This is true for events like push/schedule which run in the # context of the main repo, but for PRs only true if coming from the main repo! Forks have no secret access. # # Note: The team's decision was to not auto-deploy an image on any git push where no PR exists (yet). - # Accordingly, only run for push events on branches develop and master. + # Accordingly, only run for push events on the 'develop' branch. if: needs.check-secrets.outputs.available == 'true' && - ( github.event_name != 'push' || ( github.event_name == 'push' && contains(fromJSON('["develop", "master"]'), github.ref_name))) + ( github.event_name != 'push' || ( github.event_name == 'push' && github.ref_name == 'develop' )) steps: - name: Checkout and Setup Maven uses: IQSS/dataverse/.github/actions/setup-maven@develop @@ -141,16 +116,15 @@ jobs: - name: Set up QEMU for multi-arch builds uses: docker/setup-qemu-action@v3 - - name: Re-set image tag based on branch (if master) - if: ${{ github.ref_name == 'master' }} + - name: Add rolling image tag when pushing to develop + if: ${{ github.event_name == 'push' && github.ref_name == 'develop' }} run: | - echo "IMAGE_TAG=${{ env.MASTER_BRANCH_TAG }}" >> $GITHUB_ENV - echo "BASE_IMAGE_TAG=${{ env.MASTER_BRANCH_TAG }}" >> $GITHUB_ENV + echo "ADDITIONAL_TAGS=-Ddocker.tags.upcoming=$( mvn initialize help:evaluate -Pct -Dexpression=app.image.tag -Dapp.image.tag='${app.image.version}-${base.image.flavor}' -q -DforceStdout )" | tee -a "$GITHUB_ENV" - name: Re-set image tag and container registry when on PR if: ${{ github.event_name == 'pull_request' }} run: | - echo "IMAGE_TAG=$(echo "$GITHUB_HEAD_REF" | tr '\\/_:&+,;#*' '-')" >> $GITHUB_ENV - echo "REGISTRY='-Ddocker.registry=ghcr.io'" >> $GITHUB_ENV + echo "IMAGE_TAG=$(echo "$GITHUB_HEAD_REF" | tr '\\/_:&+,;#*' '-')" | tee -a "$GITHUB_ENV" + echo "REGISTRY='-Ddocker.registry=ghcr.io'" | tee -a "$GITHUB_ENV" # Necessary to split as otherwise the submodules are not available (deploy skips install) - name: Build app and configbaker container image with local architecture and submodules (profile will skip tests) @@ -162,7 +136,7 @@ jobs: - name: Deploy multi-arch application and configbaker container image run: > mvn - -Dapp.image.tag=${{ env.IMAGE_TAG }} + -Dapp.image.tag=${{ env.IMAGE_TAG }} ${{ env.ADDITIONAL_TAGS }} $( [[ -n "${{ inputs.base-image-ref }}" ]] && echo "-Dbase.image=${{ inputs.base-image-ref }}" ) ${{ env.REGISTRY }} -Ddocker.platforms=${{ env.PLATFORMS }} -P ct deploy diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 3830fd2f99f..3b375e13864 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -1,10 +1,8 @@ --- -name: Container Images Releasing +name: Base Container Image on: push: - tags: - - 'v[6-9].**' branches: - 'develop' # "Path filters are not evaluated for pushes of tags" https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#onpushpull_requestpull_request_targetpathspaths-ignore @@ -33,7 +31,7 @@ jobs: # Only run in upstream repo - avoid unnecessary runs in forks if: ${{ github.repository_owner == 'IQSS' }} outputs: - base-image-ref: ${{ steps.finalize.outputs.base-image-ref }} + base-image-ref: ${{ steps.determine-name.outputs.full-ref }} steps: - name: Checkout and Setup Maven @@ -52,12 +50,10 @@ jobs: # In case this is a push to develop, we care about buildtime. # Configure a remote ARM64 build host in addition to the local AMD64 in two steps. - name: Setup SSH agent - if: ${{ github.event_name != 'schedule' }} uses: webfactory/ssh-agent@v0.9.1 with: ssh-private-key: ${{ secrets.BUILDER_ARM64_SSH_PRIVATE_KEY }} - name: Provide the known hosts key and the builder config - if: ${{ github.event_name != 'schedule' }} run: | echo "${{ secrets.BUILDER_ARM64_SSH_HOST_KEY }}" > ~/.ssh/known_hosts mkdir -p modules/container-base/target/buildx-state/buildx/instances @@ -81,23 +77,17 @@ jobs: # Determine the base image name we are going to use from here on - name: Determine base image name + id: determine-name run: | - if [[ "${{ github.ref_name }}" = "${{ env.DEVELOPMENT_BRANCH }}" ]]; then - echo "BASE_IMAGE=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout )" | tee -a "${GITHUB_ENV}" - echo "BASE_IMAGE_UPCOMING=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout )" | tee -a "${GITHUB_ENV}" - else - echo "BASE_IMAGE=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout )" | tee -a "${GITHUB_ENV}" - fi - - name: Calculate revision number for immutable tag (on release branches only) - if: ${{ github.ref_name != env.DEVELOPMENT_BRANCH }} - id: revision-tag - uses: ./.github/actions/get-image-revision - with: - image-ref: ${{ env.BASE_IMAGE }} - tag-options-prefix: "-Dbase.image.tag.suffix='' -Ddocker.tags.revision=" + BASE_IMAGE=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout ) + BASE_IMAGE_UPCOMING=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout ) + + echo "BASE_IMAGE=${BASE_IMAGE}" | tee -a "${GITHUB_ENV}" + echo "BASE_IMAGE_UPCOMING=${BASE_IMAGE_UPCOMING}" | tee -a "${GITHUB_ENV}" + echo "full-ref=${BASE_IMAGE_UPCOMING}" | tee -a "$GITHUB_OUTPUT" + - name: Configure update of "latest" tag for development branch id: develop-tag - if: ${{ github.ref_name == env.DEVELOPMENT_BRANCH }} run: | echo "tag-options=-Ddocker.tags.develop=unstable -Ddocker.tags.upcoming=${BASE_IMAGE_UPCOMING#*:}" | tee -a "${GITHUB_OUTPUT}" @@ -105,16 +95,7 @@ jobs: id: build run: | mvn -f modules/container-base -Pct deploy -Ddocker.noCache -Ddocker.platforms=${{ env.PLATFORMS }} \ - -Ddocker.imagePropertyConfiguration=override ${{ steps.develop-tag.outputs.tag-options }} ${{ steps.revision-tag.outputs.tag-options }} - - - name: Determine appropriate base image ref for app image - id: finalize - run: | - if [[ "${{ github.ref_name }}" = "${{ env.DEVELOPMENT_BRANCH }}" ]]; then - echo "base-image-ref=${BASE_IMAGE_UPCOMING}" | tee -a "$GITHUB_OUTPUT" - else - echo "base-image-ref=gdcc/base:${{ steps.revision-tag.outputs.revision-tag }}" | tee -a "$GITHUB_OUTPUT" - fi + -Ddocker.imagePropertyConfiguration=override ${{ steps.develop-tag.outputs.tag-options }} push-app-img: name: "Rebase & Publish App Image" diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index 986fe25cdf5..8abe33bdefc 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -12,15 +12,46 @@ on: required: false default: false description: "Build and deploy even if no newer Java images or package updates are found." + dry_run: + type: boolean + required: false + default: false + description: "Run in dry-run mode (no builds, verify logic)" + damp_run: + type: boolean + required: false + default: false + description: "Run in damp-run mode (build but don't push)" schedule: - cron: '23 3 * * 0' # Run for 'develop' every Sunday at 03:23 UTC + release: + types: [published] env: PLATFORMS: linux/amd64,linux/arm64 NUM_PAST_RELEASES: 3 jobs: - build: + discover: + name: Discover supported releases + runs-on: ubuntu-latest + permissions: + contents: read + packages: read + outputs: + branches: ${{ steps.discover.outputs.branches }} + develop-branch: ${{ steps.discover.outputs.develop-branch }} + steps: + - name: Discover maintained releases + id: discover + run: | + DEVELOPMENT_BRANCH=$( curl -f -sS https://api.github.com/repos/${{ github.repository }} | jq -r '.default_branch' ) + echo "develop-branch=$DEVELOPMENT_BRANCH" | tee -a "${GITHUB_OUTPUT}" + + SUPPORTED_BRANCHES=$( curl -f -sS https://api.github.com/repos/IQSS/dataverse/releases | jq -r " .[0:${{ env.NUM_PAST_RELEASES }}] | .[].tag_name, \"${DEVELOPMENT_BRANCH}\" " | tr "\n" " " ) + echo "branches=$SUPPORTED_BRANCHES" | tee -a "${GITHUB_OUTPUT}" + + base-image: name: Base Image Matrix Build runs-on: ubuntu-latest permissions: @@ -28,9 +59,16 @@ jobs: packages: read # Only run in upstream repo - avoid unnecessary runs in forks if: ${{ github.repository_owner == 'IQSS' }} + needs: + - discover outputs: + # This is a JSON map with keys of branch names (supported releases & develop) and values containing an array of known image tags for the branch + # Example: {"v6.6": ["latest", "6.6-noble", "6.6-noble-r1"], "v6.5": ["6.5-noble", "6.5-noble-r5"], "v6.4": ["6.4-noble", "6.4-noble-r12"], "develop": ["unstable", "6.7-noble", "6.7-noble-p6.2025.3-j17"]} supported_tag_matrix: ${{ steps.execute.outputs.supported_tag_matrix }} - rebuilt_base_images: ${{ steps.execute.outputs.rebuilt_base_images }} + + # This is a JSON list containing a flattened map of branch names and the latest non-rolling tag + # Example: [ "v6.6=gdcc/base:6.6-noble-r1", "v6.5=gdcc/base:6.5-noble-r5", "v6.4=gdcc/base:6.4-noble-r12", "develop=gdcc/base:6.7-noble-p6.2025.3-j17" ] + rebuilt_images: ${{ steps.execute.outputs.rebuilt_images }} steps: - name: Checkout and Setup Maven @@ -50,40 +88,104 @@ jobs: with: platforms: ${{ env.PLATFORMS }} - # Discover the releases we want to maintain - - name: Discover maintained releases - id: discover - run: | - echo "FORCE_BUILD=$( [[ "${{ inputs.force_build }}" = "true" ]] && echo 1 || echo 0 )" | tee -a "$GITHUB_ENV" - DEVELOPMENT_BRANCH=$( curl -f -sS https://api.github.com/repos/${{ github.repository }} | jq -r '.default_branch' ) - echo "DEVELOPMENT_BRANCH=$DEVELOPMENT_BRANCH" | tee -a "$GITHUB_ENV" - echo "branches=$( curl -f -sS https://api.github.com/repos/IQSS/dataverse/releases | jq -r " .[0:${{ env.NUM_PAST_RELEASES }}] | .[].tag_name, \"${DEVELOPMENT_BRANCH}\" " | tr "\n" " " )" | tee -a "${GITHUB_OUTPUT}" + # Execute matrix build for the discovered branches + - name: Execute build matrix script + id: execute + run: > + FORCE_BUILD=$( [[ "${{ inputs.force_build }}" = "true" ]] && echo 1 || echo 0 ) + DRY_RUN=$( [[ "${{ inputs.dry_run }}" = "true" ]] && echo 1 || echo 0 ) + DAMP_RUN=$( [[ "${{ inputs.damp_run }}" = "true" ]] && echo 1 || echo 0 ) + DEVELOPMENT_BRANCH=${{ needs.discover.outputs.develop-branch }} + .github/workflows/scripts/containers/maintain-base.sh ${{ needs.discover.outputs.branches }} + + application-image: + name: "Application Image Matrix Build" + runs-on: ubuntu-latest + permissions: + contents: read + packages: read + needs: + - discover + - base-image + # Only run in upstream repo - avoid unnecessary runs in forks. + # TODO: If we add a push trigger later, we might want to prepend "always() &&" to ignore the status of the base job. Needs further investigation. + if: ${{ github.repository_owner == 'IQSS' }} + outputs: + supported_tag_matrix: ${{ steps.execute.outputs.supported_tag_matrix }} + rebuilt_images: ${{ steps.execute.outputs.rebuilt_images }} + steps: + - name: Checkout and Setup Maven + uses: IQSS/dataverse/.github/actions/setup-maven@develop + with: + pom-paths: ./pom.xml + + # Note: Accessing, pushing tags etc. to DockerHub will only succeed in upstream and + # on events in context of upstream because secrets. PRs run in context of forks by default! + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Set up QEMU for multi-arch builds + uses: docker/setup-qemu-action@v3 + with: + platforms: ${{ env.PLATFORMS }} # Execute matrix build for the discovered branches - name: Execute build matrix script id: execute - run: | - .github/workflows/scripts/maintenance-job.sh ${{ steps.discover.outputs.branches }} - - # TODO: Use the needs.build.outputs.rebuilt_base_images with fromJSON() to create a matrix job. - # Must be a single rank matrix (vector), the branch and base image tag information ships as "branch=tag" string - # Will be part of working on #10618, app image versioned tags. - #push-app-img: - # name: "Rebase & Publish App Image" - # permissions: - # contents: read - # packages: write - # pull-requests: write - # secrets: inherit - # needs: - # - build - # strategy: - # fail-fast: false - # matrix: - # branch: ${{ fromJson(needs.discover.outputs.branches) }} - # uses: ./.github/workflows/container_app_push.yml - # with: - # branch: ${{ matrix.branch }} + run: > + FORCE_BUILD=$( [[ "${{ inputs.force_build }}" = "true" ]] && echo 1 || echo 0 ) + DRY_RUN=$( [[ "${{ inputs.dry_run }}" = "true" ]] && echo 1 || echo 0 ) + DAMP_RUN=$( [[ "${{ inputs.damp_run }}" = "true" ]] && echo 1 || echo 0 ) + DEVELOPMENT_BRANCH=${{ needs.discover.outputs.develop-branch }} + .github/workflows/scripts/containers/maintain-application.sh ${{ needs.discover.outputs.branches }} + + configbaker-image: + name: "ConfigBaker Image Matrix Build" + runs-on: ubuntu-latest + permissions: + contents: read + packages: read + needs: + - discover + # Only run in upstream repo - avoid unnecessary runs in forks. + # TODO: If we add a push trigger later, we might want to prepend "always() &&" to ignore the status of the base job. Needs further investigation. + if: ${{ github.repository_owner == 'IQSS' }} + outputs: + supported_tag_matrix: ${{ steps.execute.outputs.supported_tag_matrix }} + rebuilt_images: ${{ steps.execute.outputs.rebuilt_images }} + steps: + - name: Checkout and Setup Maven + uses: IQSS/dataverse/.github/actions/setup-maven@develop + with: + pom-paths: ./pom.xml + + # Note: Accessing, pushing tags etc. to DockerHub will only succeed in upstream and + # on events in context of upstream because secrets. PRs run in context of forks by default! + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Set up QEMU for multi-arch builds + uses: docker/setup-qemu-action@v3 + with: + platforms: ${{ env.PLATFORMS }} + - name: Setup Trivy binary for vulnerability scanning + uses: aquasecurity/setup-trivy@v0.2.3 + with: + version: v0.63.0 + + # Execute matrix build for the discovered branches + - name: Execute build matrix script + id: execute + run: > + FORCE_BUILD=$( [[ "${{ inputs.force_build }}" = "true" ]] && echo 1 || echo 0 ) + DRY_RUN=$( [[ "${{ inputs.dry_run }}" = "true" ]] && echo 1 || echo 0 ) + DAMP_RUN=$( [[ "${{ inputs.damp_run }}" = "true" ]] && echo 1 || echo 0 ) + DEVELOPMENT_BRANCH=${{ needs.discover.outputs.develop-branch }} + .github/workflows/scripts/containers/maintain-configbaker.sh ${{ needs.discover.outputs.branches }} hub-description: name: Push description to DockerHub @@ -91,29 +193,85 @@ jobs: permissions: contents: read packages: read - needs: build + needs: + - base-image + - application-image + - configbaker-image steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Render README - id: render + + ### BASE IMAGE + - name: Render README for base image + if: toJSON(needs.base-image.outputs.rebuilt_images) != '[]' run: | - TAGS_JSON='${{ needs.build.outputs.supported_tag_matrix }}' + TAGS_JSON='${{ needs.base-image.outputs.supported_tag_matrix }}' echo "$TAGS_JSON" | jq -r 'keys | sort | reverse | .[]' | while IFS= read -r branch; do echo \ "- \`$( echo "$TAGS_JSON" | jq --arg v "$branch" -r '.[$v] | join("`, `")' )\`" \ "([Dockerfile](https://github.com/IQSS/dataverse/blob/${branch}/modules/container-base/src/main/docker/Dockerfile)," \ "[Patches](https://github.com/IQSS/dataverse/blob/develop/modules/container-base/src/backports/${branch}))" \ - | tee -a "${GITHUB_WORKSPACE}/tags.md" + | tee -a "${GITHUB_WORKSPACE}/tags-base.md" done - sed -i -e "/<\!-- TAG BLOCK HERE -->/r ${GITHUB_WORKSPACE}/tags.md" "./modules/container-base/README.md" - - - name: Push description to DockerHub + sed -i -e "/<\!-- TAG BLOCK HERE -->/r ${GITHUB_WORKSPACE}/tags-base.md" "./modules/container-base/README.md" + cat "./modules/container-base/README.md" + - name: Push description to DockerHub for base image + if: ${{ ! inputs.dry_run && ! inputs.damp_run && toJSON(needs.base-image.outputs.rebuilt_images) != '[]' }} uses: peter-evans/dockerhub-description@v4 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} repository: gdcc/base short-description: "Dataverse Base Container image providing Payara application server and optimized configuration" - readme-filepath: ./modules/container-base/README.md \ No newline at end of file + readme-filepath: ./modules/container-base/README.md + + ### APPLICATION IMAGE + - name: Render README for application image + if: toJSON(needs.application-image.outputs.rebuilt_images) != '[]' + run: | + TAGS_JSON='${{ needs.application-image.outputs.supported_tag_matrix }}' + echo "$TAGS_JSON" | jq -r 'keys | sort | reverse | .[]' | + while IFS= read -r branch; do + echo \ + "- \`$( echo "$TAGS_JSON" | jq --arg v "$branch" -r '.[$v] | join("`, `")' )\`" \ + "([Dockerfile](https://github.com/IQSS/dataverse/blob/${branch}/src/main/docker/Dockerfile)," \ + "[Patches](https://github.com/IQSS/dataverse/blob/develop/src/backports/${branch}))" \ + | tee -a "${GITHUB_WORKSPACE}/tags-app.md" + done + sed -i -e "/<\!-- TAG BLOCK HERE -->/r ${GITHUB_WORKSPACE}/tags-app.md" "./src/main/docker/README.md" + cat "./src/main/docker/README.md" + - name: Push description to DockerHub for application image + if: ${{ ! inputs.dry_run && ! inputs.damp_run && toJSON(needs.application-image.outputs.rebuilt_images) != '[]' }} + uses: peter-evans/dockerhub-description@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + repository: gdcc/dataverse + short-description: "Dataverse Application Container Image providing the executable" + readme-filepath: ./src/main/docker/README.md + + ### CONFIGBAKER IMAGE + - name: Render README for config baker image + if: toJSON(needs.configbaker-image.outputs.rebuilt_images) != '[]' + run: | + TAGS_JSON='${{ needs.configbaker-image.outputs.supported_tag_matrix }}' + echo "$TAGS_JSON" | jq -r 'keys | sort | reverse | .[]' | + while IFS= read -r branch; do + echo \ + "- \`$( echo "$TAGS_JSON" | jq --arg v "$branch" -r '.[$v] | join("`, `")' )\`" \ + "([Dockerfile](https://github.com/IQSS/dataverse/blob/${branch}/modules/container-configbaker/Dockerfile)," \ + "[Patches](https://github.com/IQSS/dataverse/blob/develop/modules/container-configbaker/backports/${branch}))" \ + | tee -a "${GITHUB_WORKSPACE}/tags-config.md" + done + sed -i -e "/<\!-- TAG BLOCK HERE -->/r ${GITHUB_WORKSPACE}/tags-config.md" "./modules/container-configbaker/README.md" + cat "./modules/container-configbaker/README.md" + - name: Push description to DockerHub for config baker image + if: ${{ ! inputs.dry_run && ! inputs.damp_run && toJSON(needs.configbaker-image.outputs.rebuilt_images) != '[]' }} + uses: peter-evans/dockerhub-description@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + repository: gdcc/base + short-description: "Dataverse Config Baker Container Image providing setup tooling and more" + readme-filepath: ./modules/container-configbaker/README.md diff --git a/.github/workflows/guides_build_sphinx.yml b/.github/workflows/guides_build_sphinx.yml index fa3a876c418..a3b5882626c 100644 --- a/.github/workflows/guides_build_sphinx.yml +++ b/.github/workflows/guides_build_sphinx.yml @@ -11,6 +11,18 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: uncch-rdmc/sphinx-action@master + - id: lookup + run: | + echo "sphinx_version=$(grep Sphinx== ./doc/sphinx-guides/requirements.txt | tr -s "=" | cut -f 2 -d=)" | tee -a "${GITHUB_OUTPUT}" + - run: | + sudo apt-get update -q + sudo apt-get install -qqy --no-install-recommends graphviz + - uses: sphinx-notes/pages@v3 with: - docs-folder: "doc/sphinx-guides/" + documentation_path: ./doc/sphinx-guides/source + requirements_path: ./doc/sphinx-guides/requirements.txt + sphinx_version: ${{ steps.lookup.outputs.sphinx_version }} + sphinx_build_options: "-W" + cache: false + publish: false + diff --git a/.github/workflows/maven_unit_test.yml b/.github/workflows/maven_unit_test.yml index 45180ea7aec..efefdaee02a 100644 --- a/.github/workflows/maven_unit_test.yml +++ b/.github/workflows/maven_unit_test.yml @@ -2,6 +2,11 @@ name: Maven Tests on: push: + # Only run for development and feature branches. Don't waste CPU cycles testing + # master when the PR to update it from develop already ran these tests. + branches: + - '*' + - '!master' paths: - "**.java" - "**.sql" diff --git a/.github/workflows/scripts/containers/maintain-application.sh b/.github/workflows/scripts/containers/maintain-application.sh new file mode 100755 index 00000000000..b68c2a53d96 --- /dev/null +++ b/.github/workflows/scripts/containers/maintain-application.sh @@ -0,0 +1,202 @@ +#!/bin/bash + +# A matrix-like job to maintain a number of releases as well as the latest snap of Dataverse. + +# PREREQUISITES: +# - You have Java, Maven, QEMU and Docker all setup and ready to go +# - You obviously checked out the develop branch, otherwise you'd not be executing this script +# - You added all the branch names you want to run maintenance for as arguments +# Optional, but recommended: +# - You added a DEVELOPMENT_BRANCH env var to your runner/job env with the name of the development branch +# - You added a FORCE_BUILD=0|1 env var to indicate if the base image build should be forced +# - You added a PLATFORMS env var with all the target platforms you want to build for +# Optional: +# - Use DRY_RUN=1 env var to skip actually building, but see how the tag lookups play out +# - Use DAMP_RUN=1 env var to skip pushing images, but build them + +# NOTE: +# This script is a culmination of Github Action steps into a single script. +# The reason to put all of this in here is due to the complexity of the Github Action and the limitation of the +# matrix support in Github actions, where outputs cannot be aggregated or otherwise used further. + +set -euo pipefail + +# Get all the inputs +# If not within a runner, just print to stdout (duplicating the output in case of tee usage, but that's ok for testing) +GITHUB_OUTPUT=${GITHUB_OUTPUT:-"/proc/self/fd/1"} +GITHUB_ENV=${GITHUB_ENV:-"/proc/self/fd/1"} +GITHUB_WORKSPACE=${GITHUB_WORKSPACE:-"$(pwd)"} +GITHUB_SERVER_URL=${GITHUB_SERVER_URL:-"https://github.com"} +GITHUB_REPOSITORY=${GITHUB_REPOSITORY:-"IQSS/dataverse"} + +MAINTENANCE_WORKSPACE="${GITHUB_WORKSPACE}/maintenance-job" + +DEVELOPMENT_BRANCH="${DEVELOPMENT_BRANCH:-"develop"}" +FORCE_BUILD="${FORCE_BUILD:-"0"}" +DRY_RUN="${DRY_RUN:-"0"}" +DAMP_RUN="${DAMP_RUN:-"0"}" +PLATFORMS="${PLATFORMS:-"linux/amd64,linux/arm64"}" + +# Setup and validation +if [[ -z "$*" ]]; then + >&2 echo "You must give a list of branch names as arguments" + exit 1; +fi + +if (( DRY_RUN + DAMP_RUN > 1 )); then + >&2 echo "You must either use DRY_RUN=1 or DAMP_RUN=1, but not both" + exit 1; +fi + +source "$( dirname "$0" )/utils.sh" + +# Delete old stuff if present +rm -rf "$MAINTENANCE_WORKSPACE" +mkdir -p "$MAINTENANCE_WORKSPACE" + +# Store the image tags we maintain in this array (same order as branches array!) +# This list will be used to build the support matrix within the Docker Hub image description +SUPPORTED_ROLLING_TAGS=() +# Store the tags of application images we are actually rebuilding +# Takes the from "branch-name=app-image-ref" +REBUILT_APP_IMAGES=() + +for BRANCH in "$@"; do + echo "::group::Running maintenance for $BRANCH" + + # 0. Determine if this is a development branch and the most current release + IS_DEV=0 + if [[ "$BRANCH" = "$DEVELOPMENT_BRANCH" ]]; then + IS_DEV=1 + fi + IS_CURRENT_RELEASE=0 + if [[ "$BRANCH" = $( curl -f -sS "https://api.github.com/repos/$GITHUB_REPOSITORY/releases" | jq -r '.[0].tag_name' ) ]]; then + IS_CURRENT_RELEASE=1 + fi + + # 1. Let's get the maintained sources + git clone -c advice.detachedHead=false --depth 1 --branch "$BRANCH" "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" "$MAINTENANCE_WORKSPACE/$BRANCH" + # Switch context + cd "$MAINTENANCE_WORKSPACE/$BRANCH" + + # 2. Now let's apply the patches (we have them checked out in $GITHUB_WORKSPACE, not necessarily in this local checkout) + echo "Checking for patches..." + if [[ -d ${GITHUB_WORKSPACE}/src/backports/$BRANCH ]]; then + echo "Applying patches now." + find "${GITHUB_WORKSPACE}/src/backports/$BRANCH" -type f -name '*.patch' -print0 | xargs -0 -n1 patch -p1 -l -s -i + fi + + # 3a. Determine the base image ref (/:) + BASE_IMAGE_REF="" + # For the dev branch we want to full flexi stack tag, to detect stack upgrades requiring new build + if (( IS_DEV )); then + BASE_IMAGE_REF=$( mvn initialize help:evaluate -Pct -f . -Dexpression=base.image -q -DforceStdout ) + else + # Frist, get the rolling tag of the base image + ROLLING_BASE_REF=$( mvn initialize help:evaluate -Pct -f . -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout ) + # Now, we want to build any release branch application images on top of a _fixed_ tag, so let's fetch the newest fixed tag + CURRENT_REV=$( current_revision "$ROLLING_BASE_REF" ) + BASE_IMAGE_REF="$ROLLING_BASE_REF-r$CURRENT_REV" + fi + echo "Determined BASE_IMAGE_REF=$BASE_IMAGE_REF from Maven" + + # 3b. Determine the app image ref (/:) + APP_IMAGE_REF="" + if (( IS_DEV )); then + # Results in the rolling tag for the dev branch + APP_IMAGE_REF=$( mvn initialize help:evaluate -Pct -f . -Dexpression=app.image -q -DforceStdout ) + else + # Results in the rolling tag for the release branch (the fixed tag will be determined from this rolling tag) + # shellcheck disable=SC2016 + APP_IMAGE_REF=$( mvn initialize help:evaluate -Pct -f . -Dexpression=app.image -Dapp.image.tag='${app.image.version}-${base.image.flavor}' -q -DforceStdout ) + fi + echo "Determined APP_IMAGE_REF=$APP_IMAGE_REF from Maven" + + # 4. Check for Base image updates + NEWER_BASE_IMAGE=0 + if check_newer_parent "$BASE_IMAGE_REF" "$APP_IMAGE_REF"; then + NEWER_BASE_IMAGE=1 + fi + + # 5. Get current immutable revision tag if not on the dev branch + REV=$( current_revision "$APP_IMAGE_REF" ) + CURRENT_REV_TAG="${APP_IMAGE_REF#*:}-r$REV" + NEXT_REV_TAG="${APP_IMAGE_REF#*:}-r$(( REV + 1 ))" + + # 6. Let's put together what tags we want added to this build run + TAG_OPTIONS="" + if ! (( IS_DEV )); then + TAG_OPTIONS="-Dapp.image=$APP_IMAGE_REF -Ddocker.tags.revision=$NEXT_REV_TAG" + # In case of the current release, add the "latest" tag as well. + if (( IS_CURRENT_RELEASE )); then + TAG_OPTIONS="$TAG_OPTIONS -Ddocker.tags.latest=latest" + fi + else + # shellcheck disable=SC2016 + UPCOMING_TAG=$( mvn initialize help:evaluate -Pct -f . -Dexpression=app.image.tag -Dapp.image.tag='${app.image.version}-${base.image.flavor}' -q -DforceStdout ) + TAG_OPTIONS="-Ddocker.tags.upcoming=$UPCOMING_TAG" + + # For the dev branch we only have rolling tags and can add them now already + SUPPORTED_ROLLING_TAGS+=("[\"unstable\", \"$UPCOMING_TAG\"]") + fi + echo "Determined these additional Maven tag options: $TAG_OPTIONS" + + # 8. Let's build the base image if necessary + NEWER_IMAGE=0 + if (( NEWER_BASE_IMAGE + FORCE_BUILD > 0 )); then + if ! (( DRY_RUN )); then + # Build the application image, but skip the configbaker image (that's a different job)! + # shellcheck disable=SC2046 + mvn -Pct -f . deploy -Ddocker.noCache -Ddocker.platforms="${PLATFORMS}" \ + -Dconf.skipBuild -Dbase.image="${BASE_IMAGE_REF}" \ + -Ddocker.imagePropertyConfiguration=override $TAG_OPTIONS \ + $( if (( DAMP_RUN )); then echo "-Ddocker.skip.push -Ddocker.skip.tag"; fi ) + else + echo "Skipping Maven build as requested by DRY_RUN=1" + fi + NEWER_IMAGE=1 + # Save the information about the immutable or rolling tag we just built + if ! (( IS_DEV )); then + REBUILT_APP_IMAGES+=("$BRANCH=${APP_IMAGE_REF%:*}:$NEXT_REV_TAG") + else + REBUILT_APP_IMAGES+=("$BRANCH=$APP_IMAGE_REF") + fi + else + echo "No rebuild necessary, we're done here." + fi + + # 9. Add list of rolling and immutable tags for release builds + if ! (( IS_DEV )); then + RELEASE_TAGS_LIST="[" + if (( IS_CURRENT_RELEASE )); then + RELEASE_TAGS_LIST+="\"latest\", " + fi + RELEASE_TAGS_LIST+="\"${APP_IMAGE_REF#*:}\", " + if (( NEWER_IMAGE )); then + RELEASE_TAGS_LIST+="\"$NEXT_REV_TAG\"]" + else + RELEASE_TAGS_LIST+="\"$CURRENT_REV_TAG\"]" + fi + SUPPORTED_ROLLING_TAGS+=("${RELEASE_TAGS_LIST}") + fi + + echo "::endgroup::" +done + +# Built the output which images have actually been rebuilt as JSON +REBUILT_IMAGES="[" +for IMAGE in "${REBUILT_APP_IMAGES[@]}"; do + REBUILT_IMAGES+=" \"$IMAGE\" " +done +REBUILT_IMAGES+="]" +echo "rebuilt_images=${REBUILT_IMAGES// /, }" | tee -a "${GITHUB_OUTPUT}" + +# Built the supported rolling tags matrix as JSON +SUPPORTED_TAGS="{" +for (( i=0; i < ${#SUPPORTED_ROLLING_TAGS[@]} ; i++ )); do + j=$((i+1)) + SUPPORTED_TAGS+="\"${!j}\": ${SUPPORTED_ROLLING_TAGS[$i]}" + (( i < ${#SUPPORTED_ROLLING_TAGS[@]}-1 )) && SUPPORTED_TAGS+=", " +done +SUPPORTED_TAGS+="}" +echo "supported_tag_matrix=$SUPPORTED_TAGS" | tee -a "$GITHUB_OUTPUT" diff --git a/.github/workflows/scripts/maintenance-job.sh b/.github/workflows/scripts/containers/maintain-base.sh similarity index 89% rename from .github/workflows/scripts/maintenance-job.sh rename to .github/workflows/scripts/containers/maintain-base.sh index 370988b9812..5b9ae738b98 100755 --- a/.github/workflows/scripts/maintenance-job.sh +++ b/.github/workflows/scripts/containers/maintain-base.sh @@ -10,6 +10,9 @@ # - You added a DEVELOPMENT_BRANCH env var to your runner/job env with the name of the development branch # - You added a FORCE_BUILD=0|1 env var to indicate if the base image build should be forced # - You added a PLATFORMS env var with all the target platforms you want to build for +# Optional: +# - Use DRY_RUN=1 env var to skip actually building, but see how the tag lookups play out +# - Use DAMP_RUN=1 env var to skip pushing images, but build them # NOTE: # This script is a culmination of Github Action steps into a single script. @@ -30,6 +33,8 @@ MAINTENANCE_WORKSPACE="${GITHUB_WORKSPACE}/maintenance-job" DEVELOPMENT_BRANCH="${DEVELOPMENT_BRANCH:-"develop"}" FORCE_BUILD="${FORCE_BUILD:-"0"}" +DRY_RUN="${DRY_RUN:-"0"}" +DAMP_RUN="${DAMP_RUN:-"0"}" PLATFORMS="${PLATFORMS:-"linux/amd64,linux/arm64"}" # Setup and validation @@ -38,6 +43,11 @@ if [[ -z "$*" ]]; then exit 1; fi +if (( DRY_RUN + DAMP_RUN > 1 )); then + >&2 echo "You must either use DRY_RUN=1 or DAMP_RUN=1, but not both" + exit 1; +fi + source "$( dirname "$0" )/utils.sh" # Delete old stuff if present @@ -73,7 +83,7 @@ for BRANCH in "$@"; do echo "Checking for patches..." if [[ -d ${GITHUB_WORKSPACE}/modules/container-base/src/backports/$BRANCH ]]; then echo "Applying patches now." - find "${GITHUB_WORKSPACE}/modules/container-base/src/backports/$BRANCH" -type f -name '*.patch' -print0 | xargs -0 -n1 patch -p1 -s -i + find "${GITHUB_WORKSPACE}/modules/container-base/src/backports/$BRANCH" -type f -name '*.patch' -print0 | xargs -0 -n1 patch -p1 -l -s -i fi # 3. Determine the base image ref (/:) @@ -130,8 +140,14 @@ for BRANCH in "$@"; do # 8. Let's build the base image if necessary NEWER_IMAGE=0 if (( NEWER_JAVA_IMAGE + NEWER_PKGS + FORCE_BUILD > 0 )); then - mvn -Pct -f modules/container-base deploy -Ddocker.noCache -Ddocker.platforms="${PLATFORMS}" \ - -Ddocker.imagePropertyConfiguration=override $TAG_OPTIONS + if ! (( DRY_RUN )); then + # shellcheck disable=SC2046 + mvn -Pct -f modules/container-base deploy -Ddocker.noCache -Ddocker.platforms="${PLATFORMS}" \ + -Ddocker.imagePropertyConfiguration=override $TAG_OPTIONS \ + $( if (( DAMP_RUN )); then echo "-Ddocker.skip.push -Ddocker.skip.tag"; fi ) + else + echo "Skipping Maven build as requested by DRY_RUN=1" + fi NEWER_IMAGE=1 # Save the information about the immutable or rolling tag we just built if ! (( IS_DEV )); then @@ -167,7 +183,7 @@ for IMAGE in "${REBUILT_BASE_IMAGES[@]}"; do REBUILT_IMAGES+=" \"$IMAGE\" " done REBUILT_IMAGES+="]" -echo "rebuilt_base_images=${REBUILT_IMAGES// /, }" | tee -a "${GITHUB_OUTPUT}" +echo "rebuilt_images=${REBUILT_IMAGES// /, }" | tee -a "${GITHUB_OUTPUT}" # Built the supported rolling tags matrix as JSON SUPPORTED_TAGS="{" diff --git a/.github/workflows/scripts/containers/maintain-configbaker.sh b/.github/workflows/scripts/containers/maintain-configbaker.sh new file mode 100755 index 00000000000..0b5b60b459c --- /dev/null +++ b/.github/workflows/scripts/containers/maintain-configbaker.sh @@ -0,0 +1,199 @@ +#!/bin/bash + +# A matrix-like job to maintain a number of releases as well as the latest snap of Dataverse. + +# PREREQUISITES: +# - You have Java, Maven, QEMU and Docker all setup and ready to go +# - You obviously checked out the develop branch, otherwise you'd not be executing this script +# - You added all the branch names you want to run maintenance for as arguments +# Optional, but recommended: +# - You added a DEVELOPMENT_BRANCH env var to your runner/job env with the name of the development branch +# - You added a FORCE_BUILD=0|1 env var to indicate if the base image build should be forced +# - You added a PLATFORMS env var with all the target platforms you want to build for +# Optional: +# - Use DRY_RUN=1 env var to skip actually building, but see how the tag lookups play out +# - Use DAMP_RUN=1 env var to skip pushing images, but build them + +# NOTE: +# This script is a culmination of Github Action steps into a single script. +# The reason to put all of this in here is due to the complexity of the Github Action and the limitation of the +# matrix support in Github actions, where outputs cannot be aggregated or otherwise used further. + +set -euo pipefail + +# Get all the inputs +# If not within a runner, just print to stdout (duplicating the output in case of tee usage, but that's ok for testing) +GITHUB_OUTPUT=${GITHUB_OUTPUT:-"/proc/self/fd/1"} +GITHUB_ENV=${GITHUB_ENV:-"/proc/self/fd/1"} +GITHUB_WORKSPACE=${GITHUB_WORKSPACE:-"$(pwd)"} +GITHUB_SERVER_URL=${GITHUB_SERVER_URL:-"https://github.com"} +GITHUB_REPOSITORY=${GITHUB_REPOSITORY:-"IQSS/dataverse"} + +MAINTENANCE_WORKSPACE="${GITHUB_WORKSPACE}/maintenance-job" + +DEVELOPMENT_BRANCH="${DEVELOPMENT_BRANCH:-"develop"}" +FORCE_BUILD="${FORCE_BUILD:-"0"}" +DRY_RUN="${DRY_RUN:-"0"}" +DAMP_RUN="${DAMP_RUN:-"0"}" +PLATFORMS="${PLATFORMS:-"linux/amd64,linux/arm64"}" + +# Setup and validation +if [[ -z "$*" ]]; then + >&2 echo "You must give a list of branch names as arguments" + exit 1; +fi + +if (( DRY_RUN + DAMP_RUN > 1 )); then + >&2 echo "You must either use DRY_RUN=1 or DAMP_RUN=1, but not both" + exit 1; +fi + +source "$( dirname "$0" )/utils.sh" + +# Delete old stuff if present +rm -rf "$MAINTENANCE_WORKSPACE" +mkdir -p "$MAINTENANCE_WORKSPACE" + +# Store the image tags we maintain in this array (same order as branches array!) +# This list will be used to build the support matrix within the Docker Hub image description +SUPPORTED_ROLLING_TAGS=() +# Store the tags of config baker images we are actually rebuilding +# Takes the from "branch-name=config-image-ref" +REBUILT_CONFIG_IMAGES=() + +for BRANCH in "$@"; do + echo "::group::Running maintenance for $BRANCH" + + # 0. Determine if this is a development branch and the most current release + IS_DEV=0 + if [[ "$BRANCH" = "$DEVELOPMENT_BRANCH" ]]; then + IS_DEV=1 + fi + IS_CURRENT_RELEASE=0 + if [[ "$BRANCH" = $( curl -f -sS "https://api.github.com/repos/$GITHUB_REPOSITORY/releases" | jq -r '.[0].tag_name' ) ]]; then + IS_CURRENT_RELEASE=1 + fi + + # 1. Let's get the maintained sources + git clone -c advice.detachedHead=false --depth=1 --branch "$BRANCH" "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" "$MAINTENANCE_WORKSPACE/$BRANCH" + # Switch context + cd "$MAINTENANCE_WORKSPACE/$BRANCH" + + # 2. Now let's apply the patches (we have them checked out in $GITHUB_WORKSPACE, not necessarily in this local checkout) + echo "Checking for patches..." + if [[ -d ${GITHUB_WORKSPACE}/modules/container-configbaker/backports/$BRANCH ]]; then + echo "Applying patches now." + find "${GITHUB_WORKSPACE}/modules/container-configbaker/backports/$BRANCH" -type f -name '*.patch' -print0 | xargs -0 -n1 patch -p1 -l -s -i + fi + + # 3a. Determine the base image ref (/:) + BASE_IMAGE_REF=$( mvn initialize help:evaluate -Pct -f . -Dexpression=conf.image.base -q -DforceStdout ) + echo "Determined BASE_IMAGE_REF=$BASE_IMAGE_REF from Maven" + + # 3b. Determine the configbaker image ref (/:) + CONFIG_IMAGE_REF="" + if (( IS_DEV )); then + # Results in the rolling tag for the dev branch + CONFIG_IMAGE_REF=$( mvn initialize help:evaluate -Pct -f . -Dexpression=conf.image -q -DforceStdout ) + else + # Results in the rolling tag for the release branch (the fixed tag will be determined from this rolling tag) + # shellcheck disable=SC2016 + CONFIG_IMAGE_REF=$( mvn initialize help:evaluate -Pct -f . -Dexpression=conf.image -Dconf.image.tag='${app.image.version}-${conf.image.flavor}' -q -DforceStdout ) + fi + echo "Determined CONFIG_IMAGE_REF=$CONFIG_IMAGE_REF from Maven" + + # 4a. Check for Base image updates + NEWER_BASE_IMAGE=0 + if check_newer_parent "$BASE_IMAGE_REF" "$CONFIG_IMAGE_REF"; then + NEWER_BASE_IMAGE=1 + fi + + # 4b. Check for vulnerabilities in packages fixable by updating + FIXES_AVAILABLE=0 + if ! (( NEWER_BASE_IMAGE )) && check_trivy_fixes_for_os "$CONFIG_IMAGE_REF"; then + FIXES_AVAILABLE=1 + fi + + # 5. Get current immutable revision tag if not on the dev branch + REV=$( current_revision "$CONFIG_IMAGE_REF" ) + CURRENT_REV_TAG="${CONFIG_IMAGE_REF#*:}-r$REV" + NEXT_REV_TAG="${CONFIG_IMAGE_REF#*:}-r$(( REV + 1 ))" + + # 6. Let's put together what tags we want added to this build run + TAG_OPTIONS="" + if ! (( IS_DEV )); then + TAG_OPTIONS="-Dconf.image=$CONFIG_IMAGE_REF -Ddocker.tags.revision=$NEXT_REV_TAG" + # In case of the current release, add the "latest" tag as well. + if (( IS_CURRENT_RELEASE )); then + TAG_OPTIONS="$TAG_OPTIONS -Ddocker.tags.latest=latest" + fi + else + # shellcheck disable=SC2016 + UPCOMING_TAG=$( mvn initialize help:evaluate -Pct -f . -Dexpression=conf.image.tag -Dconf.image.tag='${app.image.version}-${conf.image.flavor}' -q -DforceStdout ) + TAG_OPTIONS="-Ddocker.tags.upcoming=$UPCOMING_TAG" + + # For the dev branch we only have rolling tags and can add them now already + SUPPORTED_ROLLING_TAGS+=("[\"unstable\", \"$UPCOMING_TAG\"]") + fi + echo "Determined these additional Maven tag options: $TAG_OPTIONS" + + # 8. Let's build the base image if necessary + NEWER_IMAGE=0 + if (( NEWER_BASE_IMAGE + FIXES_AVAILABLE + FORCE_BUILD > 0 )); then + if ! (( DRY_RUN )); then + # Build the application image, but skip the configbaker image (that's a different job)! + # shellcheck disable=SC2046 + mvn -Pct -f . deploy -Ddocker.noCache -Ddocker.platforms="${PLATFORMS}" \ + -Dapp.skipBuild -Dconf.image.base="${BASE_IMAGE_REF}" \ + -Dmaven.main.skip -Dmaven.test.skip -Dmaven.war.skip \ + -Ddocker.imagePropertyConfiguration=override $TAG_OPTIONS \ + $( if (( DAMP_RUN )); then echo "-Ddocker.skip.push -Ddocker.skip.tag"; fi ) + else + echo "Skipping Maven build as requested by DRY_RUN=1" + fi + NEWER_IMAGE=1 + # Save the information about the immutable or rolling tag we just built + if ! (( IS_DEV )); then + REBUILT_CONFIG_IMAGES+=("$BRANCH=${CONFIG_IMAGE_REF%:*}:$NEXT_REV_TAG") + else + REBUILT_CONFIG_IMAGES+=("$BRANCH=$CONFIG_IMAGE_REF") + fi + else + echo "No rebuild necessary, we're done here." + fi + + # 9. Add list of rolling and immutable tags for release builds + if ! (( IS_DEV )); then + RELEASE_TAGS_LIST="[" + if (( IS_CURRENT_RELEASE )); then + RELEASE_TAGS_LIST+="\"latest\", " + fi + RELEASE_TAGS_LIST+="\"${CONFIG_IMAGE_REF#*:}\", " + if (( NEWER_IMAGE )); then + RELEASE_TAGS_LIST+="\"$NEXT_REV_TAG\"]" + else + RELEASE_TAGS_LIST+="\"$CURRENT_REV_TAG\"]" + fi + SUPPORTED_ROLLING_TAGS+=("${RELEASE_TAGS_LIST}") + fi + + echo "::endgroup::" +done + +# Built the output which images have actually been rebuilt as JSON +REBUILT_IMAGES="[" +for IMAGE in "${REBUILT_CONFIG_IMAGES[@]}"; do + REBUILT_IMAGES+=" \"$IMAGE\" " +done +REBUILT_IMAGES+="]" +echo "rebuilt_images=${REBUILT_IMAGES// /, }" | tee -a "${GITHUB_OUTPUT}" + +# Built the supported rolling tags matrix as JSON +SUPPORTED_TAGS="{" +for (( i=0; i < ${#SUPPORTED_ROLLING_TAGS[@]} ; i++ )); do + j=$((i+1)) + SUPPORTED_TAGS+="\"${!j}\": ${SUPPORTED_ROLLING_TAGS[$i]}" + (( i < ${#SUPPORTED_ROLLING_TAGS[@]}-1 )) && SUPPORTED_TAGS+=", " +done +SUPPORTED_TAGS+="}" +echo "supported_tag_matrix=$SUPPORTED_TAGS" | tee -a "$GITHUB_OUTPUT" diff --git a/.github/workflows/scripts/utils.sh b/.github/workflows/scripts/containers/utils.sh similarity index 81% rename from .github/workflows/scripts/utils.sh rename to .github/workflows/scripts/containers/utils.sh index 987b58d8bb5..a61df5467fe 100644 --- a/.github/workflows/scripts/utils.sh +++ b/.github/workflows/scripts/containers/utils.sh @@ -2,6 +2,10 @@ set -euo pipefail +function is_bin_in_path { + builtin type -P "$1" &> /dev/null +} + function check_newer_parent() { PARENT_IMAGE="$1" # Get namespace, default to "library" if not found @@ -9,7 +13,8 @@ function check_newer_parent() { if [[ "$PARENT_IMAGE_NS" = "${PARENT_IMAGE}" ]]; then PARENT_IMAGE_NS="library" fi - PARENT_IMAGE_REPO="${PARENT_IMAGE%:*}" + PARENT_IMAGE_REPO_CUT_NS="${PARENT_IMAGE#*/}" + PARENT_IMAGE_REPO="${PARENT_IMAGE_REPO_CUT_NS%:*}" PARENT_IMAGE_TAG="${PARENT_IMAGE#*:}" PARENT_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${PARENT_IMAGE_NS}/repositories/${PARENT_IMAGE_REPO}/tags/${PARENT_IMAGE_TAG}" | jq -r .last_updated )" @@ -60,6 +65,27 @@ function check_newer_pkgs() { } +function check_trivy_fixes_for_os() { + IMAGE_REF="$1" + if [[ -z "$IMAGE_REF" ]]; then + echo "You must give an image reference as argument to check_trivy_fixes_for_os" + exit 1 + fi + is_bin_in_path trivy || { echo "Trivy Scanner not installed" 1>&2; exit 1; } + JSON_REPORT=$(mktemp) + + trivy image --ignore-unfixed --scanners vuln --disable-telemetry --pkg-types os -f json "$IMAGE_REF" -o "$JSON_REPORT" + + HAS_FIXES=$( jq -r '.Results[] | select(has("Vulnerabilities") and .Vulnerabilities != null and (.Vulnerabilities | length > 100)) | .Vulnerabilities | length > 0' "$JSON_REPORT") + if [[ "true" = "$HAS_FIXES" ]]; then + echo "Trivy Scan showed fixes to known vulnerabilities by updating packages exist for image $IMAGE_REF" + return 0 + else + echo "Trivy Scan showed no fixes to known vulnerabilities by updating packages exist for image $IMAGE_REF" + return 1 + fi +} + function current_revision() { IMAGE="$1" IMAGE_NS_REPO="${IMAGE%:*}" diff --git a/doc/release-notes/10618-app-image-tags.md b/doc/release-notes/10618-app-image-tags.md new file mode 100644 index 00000000000..84d73dc036e --- /dev/null +++ b/doc/release-notes/10618-app-image-tags.md @@ -0,0 +1,23 @@ +### Ability to pin to a specific Dataverse version in Docker (and more) + +Container image management has been enhanced to provide better support for multiple Dataverse releases and improved maintenance workflows. + +**Versioned Image Tags**: Application ("dataverse") and Config Baker [images on Docker Hub](https://hub.docker.com/u/gdcc) now have versioned tags, supporting the latest three Dataverse software releases. This enables users to pin to specific versions (e.g. 6.7), providing better stability for production deployments. Previously, the "alpha" tag could be used, but it was always overwritten by the latest release. Now, you can choose the 6.7 tag, for example, to stay on that version. Please note that the "alpha" tag should no longer be used and will likely be deleted. The equivalent is the new "latest" tag. + +**Backport Support**: Application and Config Baker image builds now support including code backports for past releases, enabling the delivery of security fixes and critical updates to older (supported) versions. + +**Enhanced Documentation**: Container image [documentation](https://dataverse-guide--11477.org.readthedocs.build/en/11477/container/index.html) has been updated to reflect the new versioning scheme and maintenance processes. + +**Config Baker Base Image Change**: The Config Baker image has been migrated from Alpine to Ubuntu as its base operating system, aligning with other container images in the project for consistency and better compatibility. The past releases have not been migrated, only future releases (6.7+) will use Ubuntu. + +**Workflow Responsibility Split**: GitHub Actions workflows for containers have been reorganized with a clear separation of concerns: + +- `container_maintenance.yml` handles all release-time and maintenance activities +- Other workflows focus solely on preview images for development merges and pull requests + +These improvements provide more robust container image lifecycle management, better security update delivery, and clearer operational procedures for both development and production environments. +See also the [Container Guide](https://dataverse-guide--11477.org.readthedocs.build/en/11477/container/index.html), #10618, and #11477. + +## Notes for documentation writers + +Sphinx has been upgraded to 7.4.0 and new dependencies have been added, including semver. Please re-run the `pip install -r requirements.txt` setup [step](https://guides.dataverse.org/en/6.7/contributor/documentation.html#installing-sphinx) to upgrade your environment. Otherwise you might see an error like `ModuleNotFoundError: No module named 'semver'`. diff --git a/doc/sphinx-guides/requirements.txt b/doc/sphinx-guides/requirements.txt index 8eadb843cff..9c74ed75f6d 100755 --- a/doc/sphinx-guides/requirements.txt +++ b/doc/sphinx-guides/requirements.txt @@ -1,4 +1,4 @@ -Sphinx==7.2.6 +Sphinx==7.4.0 # inline icons sphinx-icon==0.1.2 @@ -11,3 +11,6 @@ sphinx-tabs==3.4.5 # jQuery sphinxcontrib-jquery + +Sphinx-Substitution-Extensions==2025.1.2 +semver>=3,<4 \ No newline at end of file diff --git a/doc/sphinx-guides/source/conf.py b/doc/sphinx-guides/source/conf.py index 26e71672f2e..eeba70dacde 100755 --- a/doc/sphinx-guides/source/conf.py +++ b/doc/sphinx-guides/source/conf.py @@ -15,6 +15,7 @@ import sys import os from datetime import datetime +import semver sys.path.insert(0, os.path.abspath('../../')) import sphinx_bootstrap_theme @@ -45,6 +46,7 @@ 'sphinxcontrib.jquery', 'myst_parser', 'sphinx_tabs.tabs', + 'sphinx_substitution_extensions', ] # Add any paths that contain templates here, relative to this directory. @@ -70,7 +72,7 @@ # The short X.Y version. version = '6.6' # The full version, including alpha/beta/rc tags. -release = '6.6' +release = version # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -437,4 +439,6 @@ rst_prolog = """ .. |toctitle| replace:: Contents: .. |anotherSub| replace:: Yes, there can be multiple. -""" +.. |version| replace:: %s +.. |nextVersion| replace:: %s +""" % (version, "%s.%s" % semver.Version.parse(version, optional_minor_and_patch=True).bump_minor().to_tuple()[0:2] ) diff --git a/doc/sphinx-guides/source/container/app-image.rst b/doc/sphinx-guides/source/container/app-image.rst index e1f5517629d..afffeae1c0b 100644 --- a/doc/sphinx-guides/source/container/app-image.rst +++ b/doc/sphinx-guides/source/container/app-image.rst @@ -1,5 +1,5 @@ -Dataverse Application Image -=========================== +Application Image +================= The application image is a layer on top of the base image and contains the Dataverse software. @@ -22,20 +22,72 @@ IQSS will not offer you support how to deploy or run it, please reach out to the You might be interested in taking a look at :doc:`../developers/containers`, linking you to some (community-based) efforts. -.. _supported-image-tags-app: +.. _app-image-supported-tags: Supported Image Tags ++++++++++++++++++++ This image is sourced from the main upstream code `repository of the Dataverse software `_. Development and maintenance of the `image's code `_ -happens there (again, by the community). Community-supported image tags are based on the two most important -upstream branches: +happens there (again, by the community). + +All supported images receive scheduled maintenance, executed every Sunday. +New revisions are kept to a minimum, usually created when some dependency needs (security) updates. +For the application images it correlates mostly to the :doc:`base image ` receiving updates. + +Our tagging is inspired by `Bitnami `_ and we offer two categories of tags: + +- rolling: images change over time +- immutable: images are fixed and never change + +In the tags below you'll see the term "flavor". This refers to flavor of Linux the container is built on. We use Ubuntu as the basis for our images and, for the time being, the only operating system flavors we use and support are ``noble`` (6.4+) and ``jammy`` (pre-6.4). + +You can find all the tags at https://hub.docker.com/r/gdcc/dataverse/tags + +Tags for Production Use +^^^^^^^^^^^^^^^^^^^^^^^ + +The images of the three latest releases of Dataverse will receive updates such as security patches for the underlying operating system. +Content will be fairly stable as disruptive changes like Payara or Java upgrades will be handled in a new major or minor upgrade to Dataverse (a new ``.`` tag). +Expect disruptive changes in case of high risk security threats. + +- | **Latest** + | Definition: ``latest`` + | Summary: Rolling tag, always pointing to the latest revision of the most current Dataverse release. In Dataverse 6.6 and lower, the equivalent was the ``alpha`` tag, which has been deleted. +- | **Rolling Production** + | Definition: ``.-`` + | Example: :substitution-code:`|version|-noble` + | Summary: Rolling tag, pointing to the latest revision of an immutable production image for released versions of Dataverse. +- | **Immutable Production** + | Definition: ``.--r`` + | Example: :substitution-code:`|version|-noble-r1` + | Summary: An **immutable tag** where the revision is incremented for rebuilds of the image. + | This image should be especially attractive if you want explict control over when your images are updated. + +Tags for Development Use +^^^^^^^^^^^^^^^^^^^^^^^^ + +All of the tags below are strongly recommended only for development purposes due to their fast-changing nature. +In addition to updates due to PR merges, the most recent tags undergo scheduled maintenance to ensure timely security fixes. +When a development cycle of Dataverse finishes (see :doc:`/developers/making-releases`), maintenance ceases for any of the tags below carrying version numbers. +For now, stale images will be kept on Docker Hub indefinitely. + +- | **Unstable** + | Definition: ``unstable`` + | Summary: Rolling tag, tracking the ``develop`` branch (see also :ref:`develop-branch`). (`Dockerfile `__) + | Please expect abrupt changes like new Payara or Java versions as well as OS updates or flavor switches when using this tag. +- | **Upcoming** + | Definition: ``.-`` + | Example: :substitution-code:`|nextVersion|-noble` + | Summary: Rolling tag, equivalent to ``unstable`` for current development cycle. + Will roll over to the rolling production tag after a Dataverse release. + | Discussion: Perhaps you are eager to starting testing features of an upcoming version (e.g. |nextVersion|) in a staging environment. You select the :substitution-code:`|nextVersion|-noble` tag (as opposed to ``unstable``) because you want to stay on |nextVersion| rather switching to the version **after that** when a release is made (which would happen if you had selected the ``unstable`` tag). Also, when the next release comes out (|nextVersion| in this example), you would stay on the :substitution-code:`|nextVersion|-noble` tag, which is the same tag that someone would use who wants the final release of |nextVersion|. (See "Rolling Production", above.) + +**NOTE**: In these tags for development usage, the version number will always be 1 minor version ahead of existing Dataverse releases. +Example: Assume Dataverse ``6.x`` is released, ``6.(x+1)`` is underway. +The rolling tag in use during the cycle will be ``6.(x+1)-FFF`` and ``6.(x+1)-FFF-p6.202P.P-jJJ``. +See also: :doc:`/developers/making-releases`. -- The ``unstable`` tag corresponds to the ``develop`` branch, where pull requests are merged. - (`Dockerfile `__) -- The ``alpha`` tag corresponds to the ``master`` branch, where releases are cut from. - (`Dockerfile `__) Image Contents ++++++++++++++ diff --git a/doc/sphinx-guides/source/container/base-image.rst b/doc/sphinx-guides/source/container/base-image.rst index a0852a5465f..d279135dac4 100644 --- a/doc/sphinx-guides/source/container/base-image.rst +++ b/doc/sphinx-guides/source/container/base-image.rst @@ -1,5 +1,5 @@ -Application Base Image -====================== +Base Image +========== The base image contains Payara and other dependencies that the Dataverse software runs on. It is the foundation for the :doc:`app-image`. Note that some dependencies, such as PostgreSQL and Solr, run in their own containers and are not part of the base image. @@ -21,7 +21,7 @@ IQSS will not offer you support how to deploy or run it, please reach out to the You might be interested in taking a look at :doc:`../developers/containers`, linking you to some (community-based) efforts. -.. _base-supported-image-tags: +.. _base-image-supported-tags: Supported Image Tags ++++++++++++++++++++ @@ -30,6 +30,10 @@ This image is sourced from the main upstream code `repository of the Dataverse s Development and maintenance of the `image's code `_ happens there (again, by the community). +All supported images receive scheduled maintenance, executed every Sunday. +New revisions are kept to a minimum, usually created when some dependency needs (security) updates. +(Examples: JRE patch releases, ImageMagick fixes, etc.) + Our tagging is inspired by `Bitnami `_ and we offer two categories of tags: - rolling: images change over time @@ -51,11 +55,11 @@ Expect disruptive changes in case of high risk security threats. | Summary: Rolling tag, always pointing to the latest revision of the most current Dataverse release. - | **Rolling Production** | Definition: ``.-`` - | Example: ``6.4-noble`` + | Example: :substitution-code:`|version|-noble` | Summary: Rolling tag, pointing to the latest revision of an immutable production image for released versions of Dataverse. - | **Immutable Production** | Definition: ``.--r`` - | Example: ``6.4-noble-r1`` + | Example: :substitution-code:`|version|-noble-r1` | Summary: An **immutable tag** where the revision is incremented for rebuilds of the image. | This image should be especially attractive if you want explict control over when your images are updated. @@ -73,12 +77,12 @@ For now, stale images will be kept on Docker Hub indefinitely. | Please expect abrupt changes like new Payara or Java versions as well as OS updates or flavor switches when using this tag. - | **Upcoming** | Definition: ``.-`` - | Example: ``6.5-noble`` + | Example: :substitution-code:`|nextVersion|-noble` | Summary: Rolling tag, equivalent to ``unstable`` for current development cycle. Will roll over to the rolling production tag after a Dataverse release. - | **Flexible Stack** | Definition: ``.--p-j`` - | Example: ``6.5-noble-p6.2024.6-j17`` + | Example: :substitution-code:`|nextVersion|-noble-p6.2025.3-j17` | Summary: Rolling tag during a development cycle of the Dataverse software (`Dockerfile `__). **NOTE**: In these tags for development usage, the version number will always be 1 minor version ahead of existing Dataverse releases. diff --git a/doc/sphinx-guides/source/container/configbaker-image.rst b/doc/sphinx-guides/source/container/configbaker-image.rst index 09e431eb547..4cc64fb150e 100644 --- a/doc/sphinx-guides/source/container/configbaker-image.rst +++ b/doc/sphinx-guides/source/container/configbaker-image.rst @@ -14,19 +14,70 @@ To see the Config Baker help screen: ``docker run -it --rm gdcc/configbaker:unstable`` +.. _config-image-supported-tags: + Supported Image Tags ++++++++++++++++++++ This image is sourced from the main upstream code `repository of the Dataverse software `_. Development and maintenance of the `image's code `_ -happens there (again, by the community). Community-supported image tags are based on the two most important -upstream branches: - -- The ``unstable`` tag corresponds to the ``develop`` branch, where pull requests are merged. - (`Dockerfile `__) -- The ``alpha`` tag corresponds to the ``master`` branch, where releases are cut from. - (`Dockerfile `__) - +happens there (again, by the community). + +All supported images receive scheduled maintenance, executed every Sunday. +New revisions are kept to a minimum, usually created when some dependency needs (security) updates. +The `Trivy `_ scanner is used to check for fixed vulnerabilities and rebuilds are issued if such a fix is detected. + +Our tagging is inspired by `Bitnami `_ and we offer two categories of tags: + +- rolling: images change over time +- immutable: images are fixed and never change + +In the tags below you'll see the term "flavor". This refers to flavor of Linux the container is built on. We use Ubuntu as the basis for our images and, for the time being, the only operating system flavors we use and support are ``noble`` (6.7+) and ``alpine`` (pre-6.7). + +You can find all the tags at https://hub.docker.com/r/gdcc/configbaker/tags + +Tags for Production Use +^^^^^^^^^^^^^^^^^^^^^^^ + +The images of the three latest releases of the Dataverse project will receive updates such as security patches for the underlying operating system. +Content will be fairly stable as disruptive changes like Payara or Java upgrades will be handled in a new major or minor upgrade to Dataverse (a new ``.`` tag). +Expect disruptive changes in case of high risk security threats. + +- | **Latest** + | Definition: ``latest`` + | Summary: Rolling tag, always pointing to the latest revision of the most current Dataverse release. +- | **Rolling Production** + | Definition: ``.-`` + | Example: :substitution-code:`|version|-noble` + | Summary: Rolling tag, pointing to the latest revision of an immutable production image for released versions of Dataverse. +- | **Immutable Production** + | Definition: ``.--r`` + | Example: :substitution-code:`|version|-noble-r1` + | Summary: An **immutable tag** where the revision is incremented for rebuilds of the image. + | This image should be especially attractive if you want explict control over when your images are updated. + +Tags for Development Use +^^^^^^^^^^^^^^^^^^^^^^^^ + +All of the tags below are strongly recommended only for development purposes due to their fast-changing nature. +In addition to updates due to PR merges, the most recent tags undergo scheduled maintenance to ensure timely security fixes. +When a development cycle of Dataverse finishes, maintenance ceases for any tags carrying version numbers. +For now, stale images will be kept on Docker Hub indefinitely. + +- | **Unstable** + | Definition: ``unstable`` + | Summary: Rolling tag, tracking the ``develop`` branch (see also :ref:`develop-branch`). (`Dockerfile `__) + | Please expect abrupt changes like new Payara or Java versions as well as OS updates or flavor switches when using this tag. +- | **Upcoming** + | Definition: ``.-`` + | Example: :substitution-code:`|nextVersion|-noble` + | Summary: Rolling tag, equivalent to ``unstable`` for current development cycle. + Will roll over to the rolling production tag after a Dataverse release. + +**NOTE**: In these tags for development usage, the version number will always be 1 minor version ahead of existing Dataverse releases. +Example: Assume Dataverse ``6.x`` is released, ``6.(x+1)`` is underway. +The rolling tag in use during the cycle will be ``6.(x+1)-FFF`` and ``6.(x+1)-FFF-p6.202P.P-jJJ``. +See also: :doc:`/developers/making-releases`. Image Contents diff --git a/doc/sphinx-guides/source/container/running/demo.rst b/doc/sphinx-guides/source/container/running/demo.rst index 18fbc6c808d..d4afee8a18a 100644 --- a/doc/sphinx-guides/source/container/running/demo.rst +++ b/doc/sphinx-guides/source/container/running/demo.rst @@ -61,7 +61,7 @@ Edit the ``compose.yml`` file and look for the following section. bootstrap: container_name: "bootstrap" - image: gdcc/configbaker:alpha + image: gdcc/configbaker:latest restart: "no" environment: - TIMEOUT=3m @@ -192,6 +192,8 @@ PID Providers Dataverse supports multiple Persistent ID (PID) providers. The ``compose.yml`` file uses the Permalink PID provider. Follow :ref:`pids-configuration` to reconfigure as needed. +.. _file-previewers-ct: + File Previewers +++++++++++++++ @@ -291,7 +293,10 @@ Additional containers are used in development (see :doc:`../dev-usage`), but for Tags and Versions +++++++++++++++++ -The compose file references a tag called "alpha", which corresponds to the latest released version of Dataverse. This means that if a release of Dataverse comes out while you are demo'ing or evaluating, the version of Dataverse you are using could change if you do a ``docker pull``. We are aware that there is a desire for tags that correspond to versions to ensure consistency. You are welcome to join `the discussion `_ and otherwise get in touch (see :ref:`helping-containers`). For more on tags, see :ref:`supported-image-tags-app`. +The compose file references a tag called "latest", which corresponds to the latest released version of Dataverse. +This means that if a release of Dataverse comes out while you are demo'ing or evaluating, the version of Dataverse you are using could change if you do a ``docker pull``. +Feel free to change it to a specific version to avoid this. +For more on available tags, see supported tags section for :ref:`Application ` and :ref:`Config Baker ` images. Once Dataverse is running, you can check which version you have through the normal methods: diff --git a/doc/sphinx-guides/source/container/running/production.rst b/doc/sphinx-guides/source/container/running/production.rst index 3294db8ec1b..4fe16447d7e 100644 --- a/doc/sphinx-guides/source/container/running/production.rst +++ b/doc/sphinx-guides/source/container/running/production.rst @@ -7,18 +7,12 @@ Production (Future) Status ------ -The images described in this guide are not yet recommended for production usage, but we think we are close. We'd like to make the following improvements: +The images described in this guide are not yet recommended for production usage, but we think we are close. (Tagged releases are done; see the "supported image tags" section for :ref:`Application ` and :ref:`Config Baker ` images.) For now, please see :doc:`demo`. -- Tagged releases - - - Currently, you have the choice between "alpha" images that change under your feet every time a new version of Dataverse is released or "unstable" images that track the "develop" branch, which is updated frequently. Instead, we'd like to offer images like 6.4, 6.5, etc. We are tracking this work at https://github.com/IQSS/dataverse/issues/10478 and there is some preliminary code at https://github.com/IQSS/dataverse/tree/10478-version-base-img . You are welcome to join the following discussions: - - - https://dataverse.zulipchat.com/#narrow/stream/375812-containers/topic/change.20version.20scheme.20base.20image.3F/near/405636949 - - https://dataverse.zulipchat.com/#narrow/stream/375812-containers/topic/tagging.20images.20with.20versions/near/366600747 +We'd like to make the following improvements: - More docs on setting up additional features - - How to set up previewers. See https://github.com/IQSS/dataverse/issues/10506 - How to set up Rserve. - Go through all the features in docs and check what needs to be done differently with containers diff --git a/doc/sphinx-guides/source/developers/making-releases.rst b/doc/sphinx-guides/source/developers/making-releases.rst index ec3f6589395..01bd729a62f 100755 --- a/doc/sphinx-guides/source/developers/making-releases.rst +++ b/doc/sphinx-guides/source/developers/making-releases.rst @@ -135,11 +135,11 @@ Make the following changes in the release branch. Increment the version number to the milestone (e.g. 5.10.1) in the following two files: - modules/dataverse-parent/pom.xml -> ```` -> ```` (e.g. `pom.xml commit `_) -- doc/sphinx-guides/source/conf.py (two places, e.g. `conf.py commit `_) +- doc/sphinx-guides/source/conf.py -Add the version being released to the lists in the following file: +In the following ``versions.rst`` file: -- doc/sphinx-guides/source/versions.rst (e.g. `versions.rst commit `_) +- doc/sphinx-guides/source/versions.rst - Below the ``- |version|`` bullet (``|version|`` comes from the ``conf.py`` file you just edited), add a bullet for what is soon to be the previous release. Return to the parent pom and make the following change, which is necessary for proper tagging of images: @@ -174,15 +174,13 @@ Check for merged pull requests that have no milestone by going to https://github (Optional) Test Docker Images ----------------------------- -After the "master" branch has been updated and the GitHub Action to build and push Docker images has run (see `PR #9776 `_), go to https://hub.docker.com/u/gdcc and make sure the "alpha" tag for the following images has been updated: +After the "master" branch has been updated and the GitHub Action to build and push Docker images has run (see `PR #9776 `_), go to https://hub.docker.com/u/gdcc and make sure the "latest" tag for the following images has been updated: - https://hub.docker.com/r/gdcc/base - https://hub.docker.com/r/gdcc/dataverse - https://hub.docker.com/r/gdcc/configbaker -To test these images against our API test suite, go to the "alpha" workflow at https://github.com/gdcc/api-test-runner/actions/workflows/alpha.yml and run it. - -Don't be surprised if there are failures. The test runner is a work in progress! Additional dependencies or settings may have been added to the "develop" workflow. Copy them over and try again. +TODO: Get https://github.com/gdcc/api-test-runner working. .. _build-guides: @@ -315,7 +313,7 @@ Create a new branch (any name is fine but ``prepare-next-iteration`` is suggeste Create a pull request and put it through code review, like usual. Give it a milestone of the next release, the one **after** the one we're working on. Once the pull request has been approved, merge it. It should the the first PR merged of the next release. -For more background, see :ref:`base-supported-image-tags`. For an example, see https://github.com/IQSS/dataverse/pull/10896 +For more background, see :ref:`base-image-supported-tags`. For an example, see https://github.com/IQSS/dataverse/pull/10896 Lift the Code Freeze and Encourage Developers to Update Their Branches ---------------------------------------------------------------------- diff --git a/doc/sphinx-guides/source/versions.rst b/doc/sphinx-guides/source/versions.rst index cd19837dff1..e877efde83a 100755 --- a/doc/sphinx-guides/source/versions.rst +++ b/doc/sphinx-guides/source/versions.rst @@ -7,7 +7,7 @@ Dataverse Software Documentation Versions This list provides a way to refer to the documentation for previous and future versions of the Dataverse Software. In order to learn more about the updates delivered from one version to another, visit the `Releases `__ page in our GitHub repo. - pre-release `HTML (not final!) `__ and `PDF (experimental!) `__ built from the :doc:`develop ` branch :doc:`(how to contribute!) ` -- 6.6 +- |version| - `6.5 `__ - `6.4 `__ - `6.3 `__ diff --git a/docker/compose/demo/compose.yml b/docker/compose/demo/compose.yml index ed421cc49b7..5f3ce59055c 100644 --- a/docker/compose/demo/compose.yml +++ b/docker/compose/demo/compose.yml @@ -5,7 +5,7 @@ services: dataverse: container_name: "dataverse" hostname: dataverse - image: gdcc/dataverse:alpha + image: gdcc/dataverse:latest restart: on-failure user: payara environment: @@ -50,7 +50,7 @@ services: bootstrap: container_name: "bootstrap" - image: gdcc/configbaker:alpha + image: gdcc/configbaker:latest restart: "no" environment: - TIMEOUT=3m @@ -65,7 +65,7 @@ services: dv_initializer: container_name: "dv_initializer" - image: gdcc/configbaker:alpha + image: gdcc/configbaker:latest restart: "no" command: - sh @@ -122,7 +122,7 @@ services: solr_initializer: container_name: "solr_initializer" - image: gdcc/configbaker:alpha + image: gdcc/configbaker:latest restart: "no" command: - sh diff --git a/modules/container-base/README.md b/modules/container-base/README.md index 0598d709eac..f6854482073 100644 --- a/modules/container-base/README.md +++ b/modules/container-base/README.md @@ -32,7 +32,6 @@ to ask for help and guidance. This image is sourced within the main upstream code [repository of the Dataverse software](https://github.com/IQSS/dataverse). Development and maintenance of the [image's code](https://github.com/IQSS/dataverse/tree/develop/modules/container-base) happens there (again, by the community). -Community-supported image tags are based on the two most important branches: Our tagging is inspired by [Bitnami](https://docs.vmware.com/en/VMware-Tanzu-Application-Catalog/services/tutorials/GUID-understand-rolling-tags-containers-index.html). For more detailed information about our tagging policy, please read about our [base image tags](https://guides.dataverse.org/en/latest/container/base-image.html#supported-image-tags) in the Dataverse Containers Guide. diff --git a/modules/container-configbaker/Dockerfile b/modules/container-configbaker/Dockerfile index 351425a17ba..6f4e0c066bc 100644 --- a/modules/container-configbaker/Dockerfile +++ b/modules/container-configbaker/Dockerfile @@ -6,14 +6,13 @@ # This build arg must be given or build will fail ARG SOLR_VERSION +ARG BASE_IMAGE="ubuntu:noble" + # We simply have this intermediate stage here without any activity to copy the default configset over FROM solr:${SOLR_VERSION} AS solr -# Let's build us a baker -# WARNING: -# Do not upgrade the tag to :3 or :3.19 until https://pkgs.alpinelinux.org/package/v3.19/main/x86_64/c-ares is at v1.26.0+! -# See https://github.com/IQSS/dataverse/issues/10413 for more information. -FROM alpine:3.18 +# Let's build ourselves a baker +FROM ${BASE_IMAGE} ENV SCRIPT_DIR="/scripts" \ SECRETS_DIR="/secrets" \ @@ -21,13 +20,53 @@ ENV SCRIPT_DIR="/scripts" \ ENV PATH="${PATH}:${SCRIPT_DIR}" \ BOOTSTRAP_DIR="${SCRIPT_DIR}/bootstrap" -ARG APK_PACKAGES="curl bind-tools netcat-openbsd jq bash dumb-init wait4x ed postgresql-client aws-cli" +ARG PKGS="curl dnsutils dumb-init ed jq netcat-openbsd postgresql-client" +# renovate: datasource=github-releases depName=wait4x/wait4x +ARG WAIT4X_VERSION="v3.2.0" +# renovate: datasource=pypi depName=awscli +ARG AWSCLI_VERSION="1.40.15" +ARG PYTHON_PKGS="awscli==${AWSCLI_VERSION}" + +# Auto-populated by BuildKit / buildx +ARG TARGETARCH +SHELL ["/bin/bash", "-eu", "-c"] RUN true && \ - # Install necessary software and tools - apk add --no-cache ${APK_PACKAGES} && \ # Make our working directories - mkdir -p ${SCRIPT_DIR} ${SECRETS_DIR} ${SOLR_TEMPLATE} + mkdir -p ${SCRIPT_DIR} ${SECRETS_DIR} ${SOLR_TEMPLATE} && \ + + # Install packages + apt-get update -q && \ + apt-get install -qqy --no-install-recommends ${PKGS} && \ + + # Workaround to install Python and pipx 1.5+ on Ubuntu 24.04 LTS: first install Python and pipx 1.4 \ + # Adapted from https://github.com/pypa/pipx/issues/1481#issuecomment-2593124603 + apt -qqy --no-install-recommends install python3 python3-venv pipx && \ + # Now install 1.5+ in ~/.local/bin/ + pipx install pipx && \ + # Remove 1.4 again + apt purge -qqy --autoremove pipx && \ + # Install 1.5+ in /usr/local/bin/pipx + ~/.local/bin/pipx install --global pipx && \ + # Remove the virtual env install of pipx + rm -rf "~/.local" && \ + + # Cleanup apt cache + rm -rf "/var/lib/apt/lists/*" + +# New step (and shell) as this is a different manager and we need pipx around as command +RUN true && \ + # Install things not available as packages \ + ARCH="${TARGETARCH:-$(uname -m | sed -e 's/x86_64/amd64/' -e 's/\(arm\)\(64\)\?.*/\1\2/' -e 's/aarch64$/arm64/')}" && \ + + # 1. wait4x \ + curl -sSfL -o /usr/bin/wait4x.tar.gz "https://github.com/wait4x/wait4x/releases/download/${WAIT4X_VERSION}/wait4x-linux-${ARCH}.tar.gz" && \ + curl -sSfL -o /tmp/w4x-checksum "https://github.com/wait4x/wait4x/releases/download/${WAIT4X_VERSION}/wait4x-linux-${ARCH}.tar.gz.sha256sum" && \ + echo "$(cat /tmp/w4x-checksum | cut -f1 -d" ") /usr/bin/wait4x.tar.gz" | sha256sum -c - && \ + tar -xzf /usr/bin/wait4x.tar.gz -C /usr/bin && chmod +x /usr/bin/wait4x && \ + + # 2. Python packages + pipx install --global ${PYTHON_PKGS} # Get in the scripts COPY maven/scripts maven/solr/update-fields.sh ${SCRIPT_DIR}/ @@ -48,14 +87,17 @@ ENTRYPOINT ["/usr/bin/dumb-init", "--"] # By default run a script that will print a help message and terminate CMD ["help.sh"] +# Workaround for fabric8io/docker-maven-plugin#1865 +ARG APP_IMAGE_VERSION LABEL org.opencontainers.image.created="@git.build.time@" \ org.opencontainers.image.authors="Research Data Management at FZJ " \ org.opencontainers.image.url="https://guides.dataverse.org/en/latest/container/" \ org.opencontainers.image.documentation="https://guides.dataverse.org/en/latest/container/" \ org.opencontainers.image.source="https://github.com/IQSS/dataverse/tree/develop/modules/container-configbaker" \ - org.opencontainers.image.version="@project.version@" \ + org.opencontainers.image.version="$APP_IMAGE_VERSION" \ org.opencontainers.image.revision="@git.commit.id.abbrev@" \ org.opencontainers.image.vendor="Global Dataverse Community Consortium" \ org.opencontainers.image.licenses="Apache-2.0" \ org.opencontainers.image.title="Dataverse Config Baker Image" \ - org.opencontainers.image.description="This container image configures Dataverse and provides other tooling" + org.opencontainers.image.description="This container image configures Dataverse and provides other tooling" \ + org.opencontainers.image.base.name="$BASE_IMAGE" diff --git a/modules/container-configbaker/README.md b/modules/container-configbaker/README.md index 17b6f985798..75862ee0809 100644 --- a/modules/container-configbaker/README.md +++ b/modules/container-configbaker/README.md @@ -25,13 +25,25 @@ to ask for help and guidance. ## Supported Image Tags This image is sourced within the main upstream code [repository of the Dataverse software](https://github.com/IQSS/dataverse). -Development and maintenance of the [image's code](https://github.com/IQSS/dataverse/tree/develop/modules/container-configbaker) -happens there (again, by the community). Community-supported image tags are based on the two most important branches: +Development and maintenance of the [image's code](https://github.com/IQSS/dataverse/tree/develop/modules/container-configbaker) happens there (again, by the community). -- The `unstable` tag corresponds to the `develop` branch, where pull requests are merged. - ([`Dockerfile`](https://github.com/IQSS/dataverse/tree/develop/modules/container-configbaker/src/main/docker/Dockerfile)) -- The `alpha` tag corresponds to the `master` branch, where releases are cut from. - ([`Dockerfile`](https://github.com/IQSS/dataverse/tree/master/modules/container-configbaker/src/main/docker/Dockerfile)) +Our tagging is inspired by [Bitnami](https://docs.vmware.com/en/VMware-Tanzu-Application-Catalog/services/tutorials/GUID-understand-rolling-tags-containers-index.html). +For more detailed information about our tagging policy, please read about our [config baker image tags](https://guides.dataverse.org/en/latest/container/configbaker-image.html#supported-image-tags) in the Dataverse Containers Guide. + +For ease of use, here is a list of images that are currently maintained. + + + +All of them are rolling tags, except those ending with `-r`, which are the most recent immutable tags. +The `unstable` tags are the current development branch snapshot. +We strongly recommend using only immutable tags for production use cases. + +Within the main repository, you may find the image's files at `/modules/container-configbaker`. +This Maven module uses the [Maven Docker Plugin](https://dmp.fabric8.io) to build and ship the image. +You may use, extend, or alter this image to your liking and/or host in some different registry if you want to. + +**Supported architectures:** This image is created as a "multi-arch image", supporting the most common architectures +Dataverse usually runs on: AMD64 (Windows/Linux/...) and ARM64 (Apple M1/M2). ## License diff --git a/modules/container-configbaker/backports/v6.4/001-parent-pom.xml.patch b/modules/container-configbaker/backports/v6.4/001-parent-pom.xml.patch new file mode 100644 index 00000000000..9b2b9dd4b5c --- /dev/null +++ b/modules/container-configbaker/backports/v6.4/001-parent-pom.xml.patch @@ -0,0 +1,10 @@ +--- a/modules/dataverse-parent/pom.xml ++++ b/modules/dataverse-parent/pom.xml +@@ -455,6 +455,7 @@ + --> + + ${revision} ++ ${base.image.version} + + + 17 + + gdcc/dataverse:${app.image.tag} ++ + unstable ++ false + false ++ + gdcc/base:${base.image.tag} + + noble + +- ${base.image.version}-${base.image.flavor}-p${payara.version}-j${target.java.version} ++ ${base.image.version}-${base.image.flavor}${base.image.tag.suffix} ++ ++ -p${payara.version}-j${target.java.version} ++ + gdcc/configbaker:${conf.image.tag} ++ + ${app.image.tag} +- ++ alpine ++ alpine:3.18 ++ false ++ + +- ++ + + ${app.image} + ${postgresql.server.version} +@@ -1088,7 +1102,7 @@ + dataverse + ${app.skipDeploy} + +- ++ + + + +@@ -1106,7 +1120,7 @@ + + + +- ++ + + + io.fabric8 +@@ -1119,6 +1133,7 @@ + dev_dataverse + ${app.image} + ++ ${app.skipBuild} + + + ${docker.platforms} +@@ -1133,21 +1148,22 @@ + assembly.xml + + +- ++ + + +- ++ + + compose + ${project.basedir} + docker-compose-dev.yml + + +- ++ + + dev_bootstrap + ${conf.image} + ++ ${conf.skipBuild} + + + ${docker.platforms} +@@ -1155,6 +1171,7 @@ + + ${project.basedir}/modules/container-configbaker/Dockerfile + ++ ${conf.image.base} + ${SOLR_VERSION} + + @ +@@ -1162,7 +1179,7 @@ + ${project.basedir}/modules/container-configbaker/assembly.xml + + +- ++ + + + ${revision} ++ ${base.image.version} + + + 17 + + gdcc/dataverse:${app.image.tag} ++ + unstable ++ false + false ++ + gdcc/base:${base.image.tag} + + noble + +- ${base.image.version}-${base.image.flavor}-p${payara.version}-j${target.java.version} ++ ${base.image.version}-${base.image.flavor}${base.image.tag.suffix} ++ ++ -p${payara.version}-j${target.java.version} ++ + gdcc/configbaker:${conf.image.tag} ++ + ${app.image.tag} +- ++ alpine ++ alpine:3.18 ++ false ++ + +- ++ + + ${app.image} + ${postgresql.server.version} +@@ -1088,7 +1102,7 @@ + dataverse + ${app.skipDeploy} + +- ++ + + + +@@ -1106,7 +1120,7 @@ + + + +- ++ + + + io.fabric8 +@@ -1119,6 +1133,7 @@ + dev_dataverse + ${app.image} + ++ ${app.skipBuild} + + + ${docker.platforms} +@@ -1133,21 +1148,22 @@ + assembly.xml + + +- ++ + + +- ++ + + compose + ${project.basedir} + docker-compose-dev.yml + + +- ++ + + dev_bootstrap + ${conf.image} + ++ ${conf.skipBuild} + + + ${docker.platforms} +@@ -1155,6 +1171,7 @@ + + ${project.basedir}/modules/container-configbaker/Dockerfile + ++ ${conf.image.base} + ${SOLR_VERSION} + + @ +@@ -1162,7 +1179,7 @@ + ${project.basedir}/modules/container-configbaker/assembly.xml + + +- ++ + + + ${revision} ++ ${base.image.version} + + + 17 + + gdcc/dataverse:${app.image.tag} ++ + unstable ++ false + false ++ + gdcc/base:${base.image.tag} + + noble + +- ${base.image.version}-${base.image.flavor}-p${payara.version}-j${target.java.version} ++ ${base.image.version}-${base.image.flavor}${base.image.tag.suffix} ++ ++ -p${payara.version}-j${target.java.version} ++ + gdcc/configbaker:${conf.image.tag} ++ + ${app.image.tag} +- ++ alpine ++ alpine:3.18 ++ false ++ + +- ++ + + ${app.image} + ${postgresql.server.version} +@@ -1088,7 +1102,7 @@ + dataverse + ${app.skipDeploy} + +- ++ + + + +@@ -1106,7 +1120,7 @@ + + + +- ++ + + + io.fabric8 +@@ -1119,6 +1133,7 @@ + dev_dataverse + ${app.image} + ++ ${app.skipBuild} + + + ${docker.platforms} +@@ -1133,21 +1148,22 @@ + assembly.xml + + +- ++ + + +- ++ + + compose + ${project.basedir} + docker-compose-dev.yml + + +- ++ + + dev_bootstrap + ${conf.image} + ++ ${conf.skipBuild} + + + ${docker.platforms} +@@ -1155,6 +1171,7 @@ + + ${project.basedir}/modules/container-configbaker/Dockerfile + ++ ${conf.image.base} + ${SOLR_VERSION} + + @ +@@ -1162,7 +1179,7 @@ + ${project.basedir}/modules/container-configbaker/assembly.xml + + +- ++ + - 0.45.0 + 0.46.0 @@ -455,6 +455,8 @@ --> ${parsedVersion.majorVersion}.${parsedVersion.nextMinorVersion} + + ${base.image.version} diff --git a/pom.xml b/pom.xml index 2fed88cb473..f310fee8698 100644 --- a/pom.xml +++ b/pom.xml @@ -1072,15 +1072,29 @@ 17 gdcc/dataverse:${app.image.tag} + unstable + false false + gdcc/base:${base.image.tag} noble - ${base.image.version}-${base.image.flavor}-p${payara.version}-j${target.java.version} + ${base.image.version}-${base.image.flavor}${base.image.tag.suffix} + + -p${payara.version}-j${target.java.version} + gdcc/configbaker:${conf.image.tag} + ${app.image.tag} + noble + ubuntu:${conf.image.flavor} + false @@ -1122,6 +1136,7 @@ dev_dataverse ${app.image} + ${app.skipBuild} ${docker.platforms} @@ -1130,6 +1145,8 @@ Dockerfile ${base.image} + + ${app.image.version} @ @@ -1151,6 +1168,7 @@ dev_bootstrap ${conf.image} + ${conf.skipBuild} ${docker.platforms} @@ -1158,7 +1176,10 @@ ${project.basedir}/modules/container-configbaker/Dockerfile + ${conf.image.base} ${SOLR_VERSION} + + ${app.image.version} @ diff --git a/renovate.json5 b/renovate.json5 new file mode 100644 index 00000000000..c7bc73cef49 --- /dev/null +++ b/renovate.json5 @@ -0,0 +1,61 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + // We don't use this for now, as we want to avoid the noise - we'll only selectively activate updates + // "extends": [ + // "config:recommended", + // ] + + // We want a nice overview of dependencies under renovate's supervision + dependencyDashboard: true, + // See .github/CODEOWNERS + assigneesFromCodeOwners: true, + // Do not separate patch and minor upgrades into separate PRs for the same dependency + separateMinorPatch: false, + // Use nicer semantic commits style for messages + semanticCommits: "enabled", + + // Don't jump the gun in case something goes awry in upstream releases + minimumReleaseAge: "3 days", + // Only have this number of PRs open at any time. We will further limit this by using grouping for packages + prConcurrentLimit: 5, + // By default, create PRs only on Sundays to avoid noise during the week (sufficient for scheduled maintenance) + schedule: ["* * * * 0"], + + // Only include certain paths we actually want Renovate to take care of. + includePaths: [ + "modules/container-*/**", + ], + + packageRules: [ + { + description: "Disable Maven Manager", + matchManagers: "maven", + enabled: false + }, + { + description: "Disable Dockerfile Manager", + matchManagers: "dockerfile", + enabled: false + }, + { + description: "Group package updates for the Config Baker Container Image", + matchFileNames: ["modules/container-configbaker/**/*[dD]ockerfile"], + groupName: "Config Baker Container Packages" + }, + ], + + customManagers: [ + { + customType: "regex", + description: "Update _VERSION variables in Dockerfiles", + managerFilePatterns: [ + "/(^|/|\\.)Dockerfile$/", + "/(^|/)Dockerfile\\.[^/]*$/" + ], + matchStrings: [ + "# renovate: datasource=(?[a-zA-Z0-9-._]+?) depName=(?[^\\s]+?)(?: (lookupName|packageName)=(?[^\\s]+?))?(?: versioning=(?[^\\s]+?))?(?: extractVersion=(?[^\\s]+?))?(?: registryUrl=(?[^\\s]+?))?\\s(?:ENV|ARG)\\s+[A-Za-z0-9_]+?_VERSION[ =][\"']?(?.+?)[\"']?\\s" + ], + versioningTemplate: '{{#if versioning}}{{{versioning}}}{{else}}semver{{/if}}' + } + ] +} \ No newline at end of file diff --git a/src/backports/README.md b/src/backports/README.md new file mode 100644 index 00000000000..420e1dbd466 --- /dev/null +++ b/src/backports/README.md @@ -0,0 +1,81 @@ +# Backports Directory + +This directory contains patch files for backporting changes across different Dataverse releases. +These changes can be features, build enhancements, security patches, and more. + +## Directory Structure + +The backports directory is organized by "release version" (or "release tag") folders. +Each folder name corresponds to a [Dataverse release](https://github.com/IQSS/dataverse/releases) and [its associated Git tag](https://github.com/IQSS/dataverse/tags): + +``` +src/backports/ +├── v6.4/ # Patches for Dataverse 6.4 +├── v6.5/ # Patches for Dataverse 6.5 +├── v6.6/ # Patches for Dataverse 6.6 +└── ... +``` + +Each version folder contains numbered patch files that modify specific components. + +For example: +- `001-parent-pom.xml.patch` -> Modifications to the parent POM configuration +- `002-pom.xml.patch` -> Changes to the main POM file +- Additional patches as needed. + +## Intended Usage + +The patch files in each release folder are designed to be applied in numerical order. + +Currently, they are primarily used by the container maintenance workflows (see `.github/workflows/container_maintenance.yml`) to ensure maintenance script compatibility across different Dataverse versions. +See also the "three releases back" support promise at https://guides.dataverse.org/en/latest/container/app-image.html#tags-for-production-use. + +*Note: The backport patches mechanism is by no means limited to usage in a container context.* +*They can be applied manually or from some other automated release process to backports changes to older releases.* + +## Creating Patch Files + +To create a new patch file using `git diff`, follow these steps: + +### Example: Creating a POM patch + +1. **Make your changes** to the target file(s) on the appropriate branch (usually your feature branch). +2. **Head to the root** of the Git repository. +3. **Generate the patch** using `git diff`: + ```bash + # Create a patch for changes to pom.xml, comparing with the pom.xml contained in the v6.5 tag: + git --no-pager diff v6.5 pom.xml > src/backports/v6.5/002-pom.xml.patch + + # Or create a patch for multiple files: + git --no-pager diff v6.5 modules/dataverse-parent/pom.xml pom.xml > src/backports/v6.5/003-multi-pom.patch + + # Create a patch from staged changes + git diff --cached > src/backports/v6.5/004-staged-changes.patch + ``` + +4. **Review the patch** to ensure it contains only the intended changes: + ```bash + cat src/backports/v6.5/002-pom.xml.patch + ``` + +5. **Repeat for other tags** as necessary. + +### Patch Naming Convention + +Use the following naming pattern: +- `001-` prefix with three-digit numbering for ordering +- Descriptive name indicating what is being patched +- `.patch` file extension + +Examples: +- `001-parent-pom.xml.patch` +- `002-pom.xml.patch` +- `003-dockerfile-updates.patch` + +## Integration with CI/CD + +These patches are automatically applied during the container maintenance workflows to ensure that older release versions can be built with updated dependencies and configurations while maintaining compatibility. +The patches support the multi-version container image strategy that builds and maintains Docker images for the current development branch plus the last three released versions. +See also the "three releases back" support promise at https://guides.dataverse.org/en/latest/container/app-image.html#tags-for-production-use. + +In the future, other automations may pick up the patches to release updated WAR files or similar. \ No newline at end of file diff --git a/src/backports/v6.4/001-parent-pom.xml.patch b/src/backports/v6.4/001-parent-pom.xml.patch new file mode 100644 index 00000000000..b862eae0fe5 --- /dev/null +++ b/src/backports/v6.4/001-parent-pom.xml.patch @@ -0,0 +1,10 @@ +--- a/modules/dataverse-parent/pom.xml ++++ b/modules/dataverse-parent/pom.xml +@@ -448,6 +448,7 @@ + --> + + ${revision} ++ ${base.image.version} + + + gdcc/dataverse:${app.image.tag} + unstable ++ false + false + gdcc/base:${base.image.tag} + + noble + +- ${base.image.version}-${base.image.flavor}-p${payara.version}-j${target.java.version} ++ ${base.image.version}-${base.image.flavor}${base.image.tag.suffix} ++ ++ -p${payara.version}-j${target.java.version} + gdcc/configbaker:${conf.image.tag} + ${app.image.tag} ++ false + + + +@@ -1046,6 +1050,7 @@ + dev_dataverse + ${app.image} + ++ ${app.skipBuild} + + + ${docker.platforms} +@@ -1075,6 +1080,7 @@ + dev_bootstrap + ${conf.image} + ++ ${conf.skipBuild} + + + ${docker.platforms} diff --git a/src/backports/v6.5/001-parent-pom.xml.patch b/src/backports/v6.5/001-parent-pom.xml.patch new file mode 100644 index 00000000000..b862eae0fe5 --- /dev/null +++ b/src/backports/v6.5/001-parent-pom.xml.patch @@ -0,0 +1,10 @@ +--- a/modules/dataverse-parent/pom.xml ++++ b/modules/dataverse-parent/pom.xml +@@ -448,6 +448,7 @@ + --> + + ${revision} ++ ${base.image.version} + + + gdcc/dataverse:${app.image.tag} + unstable ++ false + false + gdcc/base:${base.image.tag} + + noble + +- ${base.image.version}-${base.image.flavor}-p${payara.version}-j${target.java.version} ++ ${base.image.version}-${base.image.flavor}${base.image.tag.suffix} ++ ++ -p${payara.version}-j${target.java.version} + gdcc/configbaker:${conf.image.tag} + ${app.image.tag} ++ false + + + +@@ -1046,6 +1050,7 @@ + dev_dataverse + ${app.image} + ++ ${app.skipBuild} + + + ${docker.platforms} +@@ -1075,6 +1080,7 @@ + dev_bootstrap + ${conf.image} + ++ ${conf.skipBuild} + + + ${docker.platforms} diff --git a/src/backports/v6.6/001-parent-pom.xml.patch b/src/backports/v6.6/001-parent-pom.xml.patch new file mode 100644 index 00000000000..9b2b9dd4b5c --- /dev/null +++ b/src/backports/v6.6/001-parent-pom.xml.patch @@ -0,0 +1,10 @@ +--- a/modules/dataverse-parent/pom.xml ++++ b/modules/dataverse-parent/pom.xml +@@ -455,6 +455,7 @@ + --> + + ${revision} ++ ${base.image.version} + + + gdcc/dataverse:${app.image.tag} + unstable ++ false + false + gdcc/base:${base.image.tag} + + noble + +- ${base.image.version}-${base.image.flavor}-p${payara.version}-j${target.java.version} ++ ${base.image.version}-${base.image.flavor}${base.image.tag.suffix} ++ ++ -p${payara.version}-j${target.java.version} + gdcc/configbaker:${conf.image.tag} + ${app.image.tag} ++ false + + + +@@ -1046,6 +1050,7 @@ + dev_dataverse + ${app.image} + ++ ${app.skipBuild} + + + ${docker.platforms} +@@ -1075,6 +1080,7 @@ + dev_bootstrap + ${conf.image} + ++ ${conf.skipBuild} + + + ${docker.platforms} diff --git a/src/main/docker/Dockerfile b/src/main/docker/Dockerfile index ed670294873..beb1de53cd5 100644 --- a/src/main/docker/Dockerfile +++ b/src/main/docker/Dockerfile @@ -46,16 +46,19 @@ RUN ln -s "${DEPLOY_DIR}/dataverse/supplements/jhove.conf" "${PAYARA_DIR}/glassf ln -s "${DEPLOY_DIR}/dataverse/supplements/jhoveConfig.xsd" "${PAYARA_DIR}/glassfish/domains/${DOMAIN_NAME}/config/jhoveConfig.xsd" && \ sed -i "${PAYARA_DIR}/glassfish/domains/${DOMAIN_NAME}/config/jhove.conf" -e "s:/usr/local/payara./glassfish/domains/domain1:${PAYARA_DIR}/glassfish/domains/${DOMAIN_NAME}:g" +# Workaround for fabric8io/docker-maven-plugin#1865 +ARG APP_IMAGE_VERSION LABEL org.opencontainers.image.created="@git.build.time@" \ org.opencontainers.image.authors="Research Data Management at FZJ " \ org.opencontainers.image.url="https://guides.dataverse.org/en/latest/container/" \ org.opencontainers.image.documentation="https://guides.dataverse.org/en/latest/container/" \ org.opencontainers.image.source="https://github.com/IQSS/dataverse" \ - org.opencontainers.image.version="@project.version@" \ + org.opencontainers.image.version="$APP_IMAGE_VERSION" \ org.opencontainers.image.revision="@git.commit.id.abbrev@" \ org.opencontainers.image.vendor="Global Dataverse Community Consortium" \ org.opencontainers.image.licenses="Apache-2.0" \ org.opencontainers.image.title="Dataverse Application Image" \ org.opencontainers.image.description="This container image provides the research data repository software Dataverse in a box." \ + org.opencontainers.image.base.name="$BASE_IMAGE" \ org.dataverse.deps.postgresql.version="@postgresql.server.version@" \ org.dataverse.deps.solr.version="@solr.version@" \ No newline at end of file diff --git a/src/main/docker/README.md b/src/main/docker/README.md index 06e2769ed6e..48416c196ca 100644 --- a/src/main/docker/README.md +++ b/src/main/docker/README.md @@ -30,13 +30,18 @@ to ask for help and guidance. ## Supported Image Tags This image is sourced within the main upstream code [repository of the Dataverse software](https://github.com/IQSS/dataverse). -Development and maintenance of the [image's code](https://github.com/IQSS/dataverse/tree/develop/src/main/docker) -happens there (again, by the community). Community-supported image tags are based on the two most important branches: +Development and maintenance of the [image's code](https://github.com/IQSS/dataverse/tree/develop/src/main/docker) happens there (again, by the community). -- The `unstable` tag corresponds to the `develop` branch, where pull requests are merged. - ([`Dockerfile`](https://github.com/IQSS/dataverse/tree/develop/src/main/docker/Dockerfile)) -- The `alpha` tag corresponds to the `master` branch, where releases are cut from. - ([`Dockerfile`](https://github.com/IQSS/dataverse/tree/master/src/main/docker/Dockerfile)) +Our tagging is inspired by [Bitnami](https://docs.vmware.com/en/VMware-Tanzu-Application-Catalog/services/tutorials/GUID-understand-rolling-tags-containers-index.html). +For more detailed information about our tagging policy, please read about our [application image tags](https://guides.dataverse.org/en/latest/container/app-image.html#supported-image-tags) in the Dataverse Containers Guide. + +For ease of use, here is a list of images that are currently maintained. + + + +All of them are rolling tags, except those ending with `-r`, which are the most recent immutable tags. +The `unstable` tags are the current development branch snapshot. +We strongly recommend using only immutable tags for production use cases. Within the main repository, you may find the application image files at `/src/main/docker`. This Maven module uses the [Maven Docker Plugin](https://dmp.fabric8.io) to build and ship the image.