pytest.yml

# Copyright 2024-2025 New Vector Ltd
#
# SPDX-License-Identifier: AGPL-3.0-only

name: Run PyTest tests - Integration and Manifests

on:
  pull_request:
  push:
    branches:
    - main
  workflow_dispatch:

permissions:
  contents: read
  packages: read

env:
  DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
  DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
  GHCR_USERNAME: ${{ github.actor }}
  GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}

jobs:
  pytest-setup:
    runs-on: ubuntu-latest
    outputs:
      envFiles: ${{ steps.data.outputs.envFiles }}
      manifestTests: ${{ steps.data.outputs.manifestTests }}
      upgradeFrom: ${{ steps.data.outputs.upgradeFrom }}
    steps:
    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4
      with:
        fetch-depth: 0
        tags: true
    - id: data
      run: |
        echo "envFiles=$(find tests/integration/env -name '*.rc' | sed 's|tests/integration/env/||' | sed 's/.rc$//' | jq -ncR '{envFiles: [inputs]}')" | tee -a "$GITHUB_OUTPUT"
        echo "manifestTests=$(find tests/manifests -name 'test_*.py' | sed 's|tests/manifests/||' | sed 's/.py$//' | jq -ncR '{manifestTests: [inputs]}')" | tee -a "$GITHUB_OUTPUT"
        LATEST_TAG=$(git describe --abbrev=0 --tags --match '[0-9]*.[0-9]*.[0-9]*')
        echo "upgradeFrom=$LATEST_TAG" | tee -a "$GITHUB_OUTPUT"

  pytest-integration:
    runs-on: ubuntu-latest
    needs: pytest-setup
    strategy:
      fail-fast: false
      matrix:
        test-component: ${{ fromJSON(needs.pytest-setup.outputs.envFiles).envFiles }}
        test-from-ref:
        - "${{ github.event.pull_request.head.sha }}"
        - "${{ needs.pytest-setup.outputs.upgradeFrom }}"
    env:
      MATRIX_TEST_COMPONENT: ${{ matrix.test-component }}
      MATRIX_TEST_FROM_REF: ${{ matrix.test-from-ref }}
    steps:
    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4
      with:
        fetch-depth: 0
        fetch-tags: true

    - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4  # v2
      with:
        poetry-version: "1.8.5"
        python-version: "3.x"

    - name: Load poetry path
      run: |
        echo "$(poetry env info -p)/bin" >> "${GITHUB_PATH}"

    - name: Login to GHCR
      uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772  # v3
      with:
        registry: ghcr.io
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}

    - name: Login to Dockerhub
      uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772  # v3
      with:
        registry: docker.io
        username: ${{ secrets.DOCKERHUB_USERNAME }}
        password: ${{ secrets.DOCKERHUB_TOKEN }}

    - uses: engineerd/setup-kind@ecfad61750951586a9ef973db567df1d28671bdc  # v0.6.2
      with:
        version: "v0.29.0"
        name: "ess-helm"
        skipClusterCreation: "true"
        skipClusterLogsExport: "true"

    - uses: azure/setup-helm@b9e51907a09c216f16ebe8536097933489208112  # v4.3.0

    - name: Checkout values files from previous ref
      run: |
        git checkout --no-overlay -f "${MATRIX_TEST_FROM_REF}" -- charts/matrix-stack tests/integration/env/
      if: ${{ matrix.test-from-ref != github.event.pull_request.head.sha }}

    - name: Deploy with pytest once
      run: |
        if [ -f "tests/integration/env/${MATRIX_TEST_COMPONENT}.rc" ]; then
          # shellcheck source=/dev/null
          . "tests/integration/env/${MATRIX_TEST_COMPONENT}.rc"
          PYTEST_KEEP_CLUSTER=1 poetry run pytest -vv tests/integration
        else
          echo "The integration test does not exist in previous ref, skipping..."
        fi

    - name: Checkout values files back
      run: |
        git checkout --no-overlay -f HEAD -- charts/matrix-stack tests/integration/env/

    - name: On upgrade, Synapse can restart, expect 429 to occur
      run: |
        export PYTEST_EXPECTED_HTTP_STATUS_CODES="429"
      if: ${{ matrix.test-from-ref != github.event.pull_request.head.sha }}

    - name: Test with pytest (upgrade or idempotent setup)
      run: |
        # shellcheck source=/dev/null
        . "tests/integration/env/${MATRIX_TEST_COMPONENT}.rc"
        PYTEST_KEEP_CLUSTER=1 poetry run pytest -vv tests/integration
      # syn2mas is a special case that cannot run twice by design
      if: ${{ matrix.test-component != 'matrix-authentication-service-syn2mas' }}

    - name: Export logs
      if: ${{ failure() }}
      shell: bash
      run: |
        kind export logs --name ess-helm ./ess-helm-logs
        kind export kubeconfig --name ess-helm
        ns=$(kubectl --context kind-ess-helm get ns -l app.kubernetes.io/managed-by=pytest  -o jsonpath='{.items[].metadata.name}')
        resources=("pods" "deployments" "statefulsets" "services" "configmaps" "secrets" "ingresses" "persistentvolumes" "persistentvolumeclaims" "endpoints")
        for i in "${resources[@]}"; do
          kubectl --context kind-ess-helm get "$i" -n "$ns" > "./ess-helm-logs/$i.txt"
          echo "----" >> "./ess-helm-logs/$i.txt"
          kubectl --context kind-ess-helm get "$i" -o yaml -n "$ns" >> "./ess-helm-logs/$i.txt"
        done
        kubectl --context kind-ess-helm get events --sort-by=.metadata.creationTimestamp -n "$ns" > ./ess-helm-logs/events.txt
        kind delete cluster --name ess-helm

    - name: Upload logs
      if: ${{ failure() }}
      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4
      with:
        name: ess-helm-logs-${{ matrix.test-component }}
        path: ess-helm-logs
        retention-days: 1

  pytest-manifests:
    runs-on: ubuntu-latest
    needs: pytest-setup
    strategy:
      fail-fast: false
      matrix:
        manifest-test: ${{ fromJSON(needs.pytest-setup.outputs.manifestTests).manifestTests }}
    steps:
    - name: Checkout
      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4

    - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4  # v2
      with:
        poetry-version: "1.8.5"
        python-version: "3.x"

    - name: Load poetry path
      run: |
        echo "$(poetry env info -p)/bin" >> "${GITHUB_PATH}"

    - name: Run our manifest test
      run: poetry run pytest -vv tests/manifests/${{ matrix.manifest-test }}.py