|
| 1 | +#!/bin/bash |
| 2 | + |
| 3 | +# A matrix-like job to maintain a number of releases as well as the latest snap of Dataverse. |
| 4 | + |
| 5 | +# PREREQUISITES: |
| 6 | +# - You have Java, Maven, QEMU and Docker all setup and ready to go |
| 7 | +# - You obviously checked out the develop branch, otherwise you'd not be executing this script |
| 8 | +# - You added all the branch names you want to run maintenance for as arguments |
| 9 | +# Optional, but recommended: |
| 10 | +# - You added a DEVELOPMENT_BRANCH env var to your runner/job env with the name of the development branch |
| 11 | +# - You added a FORCE_BUILD=0|1 env var to indicate if the base image build should be forced |
| 12 | +# - You added a PLATFORMS env var with all the target platforms you want to build for |
| 13 | +# Optional: |
| 14 | +# - Use DRY_RUN=1 env var to skip actually building, but see how the tag lookups play out |
| 15 | +# - Use DAMP_RUN=1 env var to skip pushing images, but build them |
| 16 | + |
| 17 | +# NOTE: |
| 18 | +# This script is a culmination of Github Action steps into a single script. |
| 19 | +# The reason to put all of this in here is due to the complexity of the Github Action and the limitation of the |
| 20 | +# matrix support in Github actions, where outputs cannot be aggregated or otherwise used further. |
| 21 | + |
| 22 | +set -euo pipefail |
| 23 | + |
| 24 | +# Get all the inputs |
| 25 | +# If not within a runner, just print to stdout (duplicating the output in case of tee usage, but that's ok for testing) |
| 26 | +GITHUB_OUTPUT=${GITHUB_OUTPUT:-"/proc/self/fd/1"} |
| 27 | +GITHUB_ENV=${GITHUB_ENV:-"/proc/self/fd/1"} |
| 28 | +GITHUB_WORKSPACE=${GITHUB_WORKSPACE:-"$(pwd)"} |
| 29 | +GITHUB_SERVER_URL=${GITHUB_SERVER_URL:-"https://github.com"} |
| 30 | +GITHUB_REPOSITORY=${GITHUB_REPOSITORY:-"IQSS/dataverse"} |
| 31 | + |
| 32 | +MAINTENANCE_WORKSPACE="${GITHUB_WORKSPACE}/maintenance-job" |
| 33 | + |
| 34 | +DEVELOPMENT_BRANCH="${DEVELOPMENT_BRANCH:-"develop"}" |
| 35 | +FORCE_BUILD="${FORCE_BUILD:-"0"}" |
| 36 | +DRY_RUN="${DRY_RUN:-"0"}" |
| 37 | +DAMP_RUN="${DAMP_RUN:-"0"}" |
| 38 | +PLATFORMS="${PLATFORMS:-"linux/amd64,linux/arm64"}" |
| 39 | + |
| 40 | +# Setup and validation |
| 41 | +if [[ -z "$*" ]]; then |
| 42 | + >&2 echo "You must give a list of branch names as arguments" |
| 43 | + exit 1; |
| 44 | +fi |
| 45 | + |
| 46 | +if (( DRY_RUN + DAMP_RUN > 1 )); then |
| 47 | + >&2 echo "You must either use DRY_RUN=1 or DAMP_RUN=1, but not both" |
| 48 | + exit 1; |
| 49 | +fi |
| 50 | + |
| 51 | +source "$( dirname "$0" )/utils.sh" |
| 52 | + |
| 53 | +# Delete old stuff if present |
| 54 | +rm -rf "$MAINTENANCE_WORKSPACE" |
| 55 | +mkdir -p "$MAINTENANCE_WORKSPACE" |
| 56 | + |
| 57 | +# Store the image tags we maintain in this array (same order as branches array!) |
| 58 | +# This list will be used to build the support matrix within the Docker Hub image description |
| 59 | +SUPPORTED_ROLLING_TAGS=() |
| 60 | +# Store the tags of config baker images we are actually rebuilding |
| 61 | +# Takes the from "branch-name=config-image-ref" |
| 62 | +REBUILT_CONFIG_IMAGES=() |
| 63 | + |
| 64 | +for BRANCH in "$@"; do |
| 65 | + echo "::group::Running maintenance for $BRANCH" |
| 66 | + |
| 67 | + # 0. Determine if this is a development branch and the most current release |
| 68 | + IS_DEV=0 |
| 69 | + if [[ "$BRANCH" = "$DEVELOPMENT_BRANCH" ]]; then |
| 70 | + IS_DEV=1 |
| 71 | + fi |
| 72 | + IS_CURRENT_RELEASE=0 |
| 73 | + if [[ "$BRANCH" = $( curl -f -sS "https://api.github.com/repos/$GITHUB_REPOSITORY/releases" | jq -r '.[0].tag_name' ) ]]; then |
| 74 | + IS_CURRENT_RELEASE=1 |
| 75 | + fi |
| 76 | + |
| 77 | + # 1. Let's get the maintained sources |
| 78 | + git clone -c advice.detachedHead=false --depth=1 --branch "$BRANCH" "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" "$MAINTENANCE_WORKSPACE/$BRANCH" |
| 79 | + # Switch context |
| 80 | + cd "$MAINTENANCE_WORKSPACE/$BRANCH" |
| 81 | + |
| 82 | + # 2. Now let's apply the patches (we have them checked out in $GITHUB_WORKSPACE, not necessarily in this local checkout) |
| 83 | + echo "Checking for patches..." |
| 84 | + if [[ -d ${GITHUB_WORKSPACE}/modules/container-configbaker/backports/$BRANCH ]]; then |
| 85 | + echo "Applying patches now." |
| 86 | + find "${GITHUB_WORKSPACE}/modules/container-configbaker/backports/$BRANCH" -type f -name '*.patch' -print0 | xargs -0 -n1 patch -p1 -l -s -i |
| 87 | + fi |
| 88 | + |
| 89 | + # 3a. Determine the base image ref (<namespace>/<repo>:<tag>) |
| 90 | + BASE_IMAGE_REF=$( mvn initialize help:evaluate -Pct -f . -Dexpression=conf.image.base -q -DforceStdout ) |
| 91 | + echo "Determined BASE_IMAGE_REF=$BASE_IMAGE_REF from Maven" |
| 92 | + |
| 93 | + # 3b. Determine the configbaker image ref (<namespace>/<repo>:<tag>) |
| 94 | + CONFIG_IMAGE_REF="" |
| 95 | + if (( IS_DEV )); then |
| 96 | + # Results in the rolling tag for the dev branch |
| 97 | + CONFIG_IMAGE_REF=$( mvn initialize help:evaluate -Pct -f . -Dexpression=conf.image -q -DforceStdout ) |
| 98 | + else |
| 99 | + # Results in the rolling tag for the release branch (the fixed tag will be determined from this rolling tag) |
| 100 | + # shellcheck disable=SC2016 |
| 101 | + CONFIG_IMAGE_REF=$( mvn initialize help:evaluate -Pct -f . -Dexpression=conf.image -Dconf.image.tag='${app.image.version}-${conf.image.flavor}' -q -DforceStdout ) |
| 102 | + fi |
| 103 | + echo "Determined CONFIG_IMAGE_REF=$CONFIG_IMAGE_REF from Maven" |
| 104 | + |
| 105 | + # 4a. Check for Base image updates |
| 106 | + NEWER_BASE_IMAGE=0 |
| 107 | + if check_newer_parent "$BASE_IMAGE_REF" "$CONFIG_IMAGE_REF"; then |
| 108 | + NEWER_BASE_IMAGE=1 |
| 109 | + fi |
| 110 | + |
| 111 | + # 4b. Check for vulnerabilities in packages fixable by updating |
| 112 | + FIXES_AVAILABLE=0 |
| 113 | + if ! (( NEWER_BASE_IMAGE )) && check_trivy_fixes_for_os "$CONFIG_IMAGE_REF"; then |
| 114 | + FIXES_AVAILABLE=1 |
| 115 | + fi |
| 116 | + |
| 117 | + # 5. Get current immutable revision tag if not on the dev branch |
| 118 | + REV=$( current_revision "$CONFIG_IMAGE_REF" ) |
| 119 | + CURRENT_REV_TAG="${CONFIG_IMAGE_REF#*:}-r$REV" |
| 120 | + NEXT_REV_TAG="${CONFIG_IMAGE_REF#*:}-r$(( REV + 1 ))" |
| 121 | + |
| 122 | + # 6. Let's put together what tags we want added to this build run |
| 123 | + TAG_OPTIONS="" |
| 124 | + if ! (( IS_DEV )); then |
| 125 | + TAG_OPTIONS="-Dconf.image=$CONFIG_IMAGE_REF -Ddocker.tags.revision=$NEXT_REV_TAG" |
| 126 | + # In case of the current release, add the "latest" tag as well. |
| 127 | + if (( IS_CURRENT_RELEASE )); then |
| 128 | + TAG_OPTIONS="$TAG_OPTIONS -Ddocker.tags.latest=latest" |
| 129 | + fi |
| 130 | + else |
| 131 | + # shellcheck disable=SC2016 |
| 132 | + UPCOMING_TAG=$( mvn initialize help:evaluate -Pct -f . -Dexpression=conf.image.tag -Dconf.image.tag='${app.image.version}-${conf.image.flavor}' -q -DforceStdout ) |
| 133 | + TAG_OPTIONS="-Ddocker.tags.upcoming=$UPCOMING_TAG" |
| 134 | + |
| 135 | + # For the dev branch we only have rolling tags and can add them now already |
| 136 | + SUPPORTED_ROLLING_TAGS+=("[\"unstable\", \"$UPCOMING_TAG\"]") |
| 137 | + fi |
| 138 | + echo "Determined these additional Maven tag options: $TAG_OPTIONS" |
| 139 | + |
| 140 | + # 8. Let's build the base image if necessary |
| 141 | + NEWER_IMAGE=0 |
| 142 | + if (( NEWER_BASE_IMAGE + FIXES_AVAILABLE + FORCE_BUILD > 0 )); then |
| 143 | + if ! (( DRY_RUN )); then |
| 144 | + # Build the application image, but skip the configbaker image (that's a different job)! |
| 145 | + # shellcheck disable=SC2046 |
| 146 | + mvn -Pct -f . deploy -Ddocker.noCache -Ddocker.platforms="${PLATFORMS}" \ |
| 147 | + -Dapp.skipBuild -Dconf.image.base="${BASE_IMAGE_REF}" \ |
| 148 | + -Dmaven.main.skip -Dmaven.test.skip -Dmaven.war.skip \ |
| 149 | + -Ddocker.imagePropertyConfiguration=override $TAG_OPTIONS \ |
| 150 | + $( if (( DAMP_RUN )); then echo "-Ddocker.skip.push -Ddocker.skip.tag"; fi ) |
| 151 | + else |
| 152 | + echo "Skipping Maven build as requested by DRY_RUN=1" |
| 153 | + fi |
| 154 | + NEWER_IMAGE=1 |
| 155 | + # Save the information about the immutable or rolling tag we just built |
| 156 | + if ! (( IS_DEV )); then |
| 157 | + REBUILT_CONFIG_IMAGES+=("$BRANCH=${CONFIG_IMAGE_REF%:*}:$NEXT_REV_TAG") |
| 158 | + else |
| 159 | + REBUILT_CONFIG_IMAGES+=("$BRANCH=$CONFIG_IMAGE_REF") |
| 160 | + fi |
| 161 | + else |
| 162 | + echo "No rebuild necessary, we're done here." |
| 163 | + fi |
| 164 | + |
| 165 | + # 9. Add list of rolling and immutable tags for release builds |
| 166 | + if ! (( IS_DEV )); then |
| 167 | + RELEASE_TAGS_LIST="[" |
| 168 | + if (( IS_CURRENT_RELEASE )); then |
| 169 | + RELEASE_TAGS_LIST+="\"latest\", " |
| 170 | + fi |
| 171 | + RELEASE_TAGS_LIST+="\"${CONFIG_IMAGE_REF#*:}\", " |
| 172 | + if (( NEWER_IMAGE )); then |
| 173 | + RELEASE_TAGS_LIST+="\"$NEXT_REV_TAG\"]" |
| 174 | + else |
| 175 | + RELEASE_TAGS_LIST+="\"$CURRENT_REV_TAG\"]" |
| 176 | + fi |
| 177 | + SUPPORTED_ROLLING_TAGS+=("${RELEASE_TAGS_LIST}") |
| 178 | + fi |
| 179 | + |
| 180 | + echo "::endgroup::" |
| 181 | +done |
| 182 | + |
| 183 | +# Built the output which images have actually been rebuilt as JSON |
| 184 | +REBUILT_IMAGES="[" |
| 185 | +for IMAGE in "${REBUILT_CONFIG_IMAGES[@]}"; do |
| 186 | + REBUILT_IMAGES+=" \"$IMAGE\" " |
| 187 | +done |
| 188 | +REBUILT_IMAGES+="]" |
| 189 | +echo "rebuilt_images=${REBUILT_IMAGES// /, }" | tee -a "${GITHUB_OUTPUT}" |
| 190 | + |
| 191 | +# Built the supported rolling tags matrix as JSON |
| 192 | +SUPPORTED_TAGS="{" |
| 193 | +for (( i=0; i < ${#SUPPORTED_ROLLING_TAGS[@]} ; i++ )); do |
| 194 | + j=$((i+1)) |
| 195 | + SUPPORTED_TAGS+="\"${!j}\": ${SUPPORTED_ROLLING_TAGS[$i]}" |
| 196 | + (( i < ${#SUPPORTED_ROLLING_TAGS[@]}-1 )) && SUPPORTED_TAGS+=", " |
| 197 | +done |
| 198 | +SUPPORTED_TAGS+="}" |
| 199 | +echo "supported_tag_matrix=$SUPPORTED_TAGS" | tee -a "$GITHUB_OUTPUT" |
0 commit comments