build-test.yml

# Copyright 2024-2025 New Vector Ltd
#
# SPDX-License-Identifier: AGPL-3.0-only

name: Helm Chart Building tests
on:
  pull_request:
  push:
    branches:
    - main
  workflow_dispatch:

jobs:
  # We build from source and commit all generated file changes so that we can see the impact in PRs
  # We want to ensure that the commit of built changes does happen, so fail if building creates changes
  # If this gets problematic we change to not committing the built schemas/values to git
  no-changes-after-building:
    runs-on: ubuntu-latest
    steps:
    - name: Checkout
      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4

    - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4  # v2
      with:
        python-version: "3.x"
        poetry-version: "1.8.5"

    - name: Load poetry path
      run: |
        echo "$(poetry env info -p)/bin" >> "${GITHUB_PATH}"

    - name: Build and check for changes
      run: |
        git config --global --add safe.directory "$GITHUB_WORKSPACE"

        # Ensure all JSON files are consistently formatted
        for file in $(git ls-files | grep -E "json$"); do yq -iP --indent 2 -o json '.' "$file"; done

        # Rebuild the charts & test values files with from the current source
        scripts/assemble_ci_values_files_from_fragments.sh
        version=$(yq '.version' charts/matrix-stack/Chart.yaml)
        scripts/assemble_helm_charts_from_fragments.sh
        scripts/set_chart_version.sh "$version"

        git diff --exit-code

  helm-lint:
    runs-on: ubuntu-latest
    steps:
    - name: Checkout
      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4

    - name: Set up chart-testing
      uses: helm/chart-testing-action@0d28d3144d3a25ea2cc349d6e59901c4ff469b3b  # v2.7.0

    - name: Run chart-testing (lint)
      run: |
        git config --global --add safe.directory "$GITHUB_WORKSPACE"
        scripts/ct-lint.sh --config ct.yaml

    - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4  # v2
      with:
        poetry-version: "1.8.5"
        python-version: "3.x"

    - name: Load poetry path
      run: |
        echo "$(poetry env info -p)/bin" >> "${GITHUB_PATH}"

    - name: Set up Kubeconform
      uses: bmuschko/setup-kubeconform@5ccaecbbf012bcb1eeeab66e649db64a477ade8f  # v1

    - name: Run kubeconform
      run: |
        for values in charts/matrix-stack/ci/*values.yaml; do
          echo "Testing matrix-stack with $values";
          helm template \
              -n ess-ci \
              -a monitoring.coreos.com/v1/ServiceMonitor \
              -f "$values" charts/matrix-stack \
            | kubeconform \
              -schema-location default \
              -schema-location 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json' \
              -strict \
              -summary
        done

    - name: Run checkov
      run: |
        checkov --version
        for checkov_values in charts/matrix-stack/ci/*checkov*values.yaml; do
          scripts/checkov.sh "$checkov_values"
        done

  template-dyff:
    runs-on: ubuntu-latest
    permissions:
      contents: read
      pull-requests: write # required to post a comment to a pull request
    steps:
    - name: Checkout PR
      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4
      with:
        fetch-depth: 0

    # helm template doesn't reliably order manifests within the same kind, so use yq to do it for us
    - name: Generate manifests for PR
      id: generate-manifests
      run: |
        mkdir -p "$RUNNER_TEMP/new"
        for values in charts/matrix-stack/ci/*values.yaml; do
          echo "Generating new templates with $values";
          mkdir -p "$RUNNER_TEMP/new/$(basename "$values" ".yaml")"
          helm template \
              -n ess-ci \
              -a monitoring.coreos.com/v1/ServiceMonitor \
              -f "$values" charts/matrix-stack | \
          yq ea '[.] | .[] | splitDoc' | \
          yq -s "\"$RUNNER_TEMP/new/$(basename "$values" ".yaml")/\""' + ([.kind, .metadata.name] | join("-") | downcase) + ".yaml"'
        done
        echo "output_dir=$RUNNER_TEMP/new" | tee -a "$GITHUB_OUTPUT"

    # We want the most recent common ancestor between the target & PR branches rather than the target branch itself
    # There could have been more commits to the target branch since the PR branch was created and we don't want to see
    # those changes in the dyff, only what this branch is doing.
    - name: Determine most recent common ancestor of target and PR branches
      id: merge-base
      run: |
        echo "merge-base=$(git merge-base ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }})" | tee -a "$GITHUB_OUTPUT"

    - name: Checkout target
      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4
      with:
        ref: ${{ steps.merge-base.outputs.merge-base }}

    - name: Generate manifests for base
      run: |
        mkdir -p "$RUNNER_TEMP/old"
        for values in charts/matrix-stack/ci/*values.yaml; do
          echo "Generating old templates with $values";
          mkdir -p "$RUNNER_TEMP/old/$(basename "$values" ".yaml")"
          helm template \
              -n ess-ci \
              -a monitoring.coreos.com/v1/ServiceMonitor \
              -f "$values" charts/matrix-stack | \
          yq ea '[.] | .[] | splitDoc' | \
          yq -s "\"$RUNNER_TEMP/old/$(basename "$values" ".yaml")/\""' + ([.kind, .metadata.name] | join("-") | downcase) + ".yaml"'
        done

    - name: Install dyff with asdf
      uses: asdf-vm/actions/install@1902764435ca0dd2f3388eea723a4f92a4eb8302  # v4
      with:
        tool_versions: |
          dyff 1.10.1

    - name: Upload new manifests
      id: upload-new
      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4
      with:
        name: new-manifests
        path: ${{ steps.generate-manifests.outputs.output_dir }}
        retention-days: 1

    - name: dyff old and new manifests
      id: dyff
      shell: bash
      env:
        ARTIFACT_URL: ${{ steps.upload-new.outputs.artifact-url }}
      run: |
        echo "output_dir=$RUNNER_TEMP" | tee -a "$GITHUB_OUTPUT"
        values_directories=$(find "$RUNNER_TEMP/old" "$RUNNER_TEMP/new" -maxdepth 1 -type d | sed -E 's|'"$RUNNER_TEMP"'/(old\|new)||' | sed -E 's|^/||' | sort | uniq)
        header="# dyff of changes in rendered templates of CI manifests\n\n"
        comment_body=""
        while read -r values_dir; do
          if [ -z "$values_dir" ]; then
            continue
          fi

          templates_files=$(find "$RUNNER_TEMP/old" "$RUNNER_TEMP/new" -maxdepth 2 -name '*.yaml' | grep "$values_dir" | sed -E 's|'"$RUNNER_TEMP"'/(old\|new)/||' | sort | uniq)
          comment_templates_body=""

          while read -r templates_file; do
            current_file="$(basename "$templates_file")"
            if [[ "$current_file" == ".yaml" ]] && [ ! -s "$template_file" ]; then
              continue
            fi

            if [ ! -f "$RUNNER_TEMP/old/$templates_file" ]; then
              api_version=$(yq '.apiVersion' "$RUNNER_TEMP/new/$templates_file")
              kind=$(yq '.kind' "$RUNNER_TEMP/new/$templates_file")
              name=$(yq '.metadata.name' "$RUNNER_TEMP/new/$templates_file")
              namespace=$(yq '.metadata.namespace' "$RUNNER_TEMP/new/$templates_file")
              metadata=$(yq '.metadata' "$RUNNER_TEMP/new/$templates_file")
              comment_templates_body+="@@ $current_file @@\n"
              comment_templates_body+="# $api_version/$kind/$namespace/$name\n"
              comment_templates_body+="! + one file added - the full content of the file is available in ${ARTIFACT_URL}\n"
              comment_templates_body+="+ apiVersion: $api_version\n"
              comment_templates_body+="+ kind: $kind\n"
              comment_templates_body+="+ metadata:\n"
              while IFS= read -r line; do
                comment_templates_body+="+   $line\n"
              done <<< "$metadata"
              comment_templates_body+="\n\n"
              continue
            fi

            if [ ! -f "$RUNNER_TEMP/new/$templates_file" ]; then
              api_version=$(yq '.apiVersion' "$RUNNER_TEMP/old/$templates_file" )
              kind=$(yq '.kind' "$RUNNER_TEMP/old/$templates_file")
              name=$(yq '.metadata.name' "$RUNNER_TEMP/old/$templates_file")
              namespace=$(yq '.metadata.namespace' "$RUNNER_TEMP/old/$templates_file")
              metadata=$(yq '.metadata' "$RUNNER_TEMP/old/$templates_file")
              comment_templates_body+="@@ $current_file @@\n"
              comment_templates_body+="# $api_version/$kind/$namespace/$name\n"
              comment_templates_body+="! - one file removed\n"
              comment_templates_body+="- apiVersion: $api_version\n"
              comment_templates_body+="- kind: $kind\n"
              comment_templates_body+="- metadata:\n"
              while IFS= read -r line; do
                comment_templates_body+="-   $line\n"
              done <<< "$metadata"
              comment_templates_body+="\n\n"
              continue
            fi

            exit_code=0
            dyff_detail=$(dyff between --set-exit-code --omit-header --output=github "$RUNNER_TEMP/old/$templates_file" "$RUNNER_TEMP/new/$templates_file" 2>&1) || exit_code=$?
            if [ $exit_code -ne 0 ]; then
              if [[ "$dyff_detail" == *"failed to compare input files"* ]]; then
                echo "failed with file $templates_file"
                exit 1
              fi

              api_version=$(yq '.apiVersion' "$RUNNER_TEMP/new/$templates_file")
              kind=$(yq '.kind' "$RUNNER_TEMP/new/$templates_file")
              name=$(yq '.metadata.name' "$RUNNER_TEMP/new/$templates_file")
              namespace=$(yq '.metadata.namespace' "$RUNNER_TEMP/new/$templates_file")
              resource_metadata="# $api_version/$kind/$namespace/$name"
              comment_templates_body+=$(sed -e "1d" -e "/^@@/a$resource_metadata" <<< "$dyff_detail")
              comment_templates_body+="\n\n\n"
            fi
          done <<< "$templates_files"

          if [[ -n "$comment_templates_body" ]]; then
              comment_body+="<details><summary><b>$values_dir.yaml</b></summary>\n"
              comment_body+='\n```diff\n'
              comment_body+="$comment_templates_body"
              comment_body+='```\n'
              comment_body+="\n</details>\n"
          fi

        done <<< "$values_directories"

        if [ -z "$comment_body" ]; then
          comment_body="No changes in rendered templates"
        fi

        echo -e "$header$comment_body" | tee "$RUNNER_TEMP/dyff-output.md"

    - name: Upload generated manifests
      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4
      with:
        name: dyff-templates
        path: ${{ steps.dyff.outputs.output_dir }}
        retention-days: 1

    - name: Find dyff comment
      if: github.event.pull_request.number != ''
      uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e  # v3
      id: find-dyff-comment
      with:
        issue-number: ${{ github.event.pull_request.number }}
        comment-author: 'github-actions[bot]'
        body-includes: 'dyff of changes in rendered templates'

    - name: Create or update comment
      if: github.event.pull_request.number != ''
      uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043  # v4
      with:
        comment-id: ${{ steps.find-dyff-comment.outputs.comment-id }}
        issue-number: ${{ github.event.pull_request.number }}
        body-path: ${{ runner.temp }}/dyff-output.md
        edit-mode: replace