ci-multi-server-tests.yml

name: Multi-Server CI Tests

on:
  push:
    branches-ignore:
      - coverity_scan
      - run-fuzzer**
      - debug-fuzzer-**
  pull_request:
    paths:
      - '.github/workflows/ci-multi-server-tests.yml'
      - 'src/tests/multi-server/**'
  workflow_dispatch:

jobs:
  pre-test:
    runs-on: ubuntu-latest
    outputs:
      #selfhosted: 0
      selfhosted: ${{ (github.repository_owner == 'FreeRADIUS') && '1' || '0' }}
    steps:
      - run: echo "Pre-test job; checking if using self-hosted runners"

  # Job only runs on GitHub-hosted runners
  run-multi-server-tests:
    needs: pre-test
    runs-on: ubuntu-24.04
    if: ${{ needs.pre-test.outputs.selfhosted != '1' }}

    env:
      MULTI_SERVER_ENV_DOCKER_BUILD_OS: ubuntu24
      MULTI_SERVER_TEST_LOG: build/tests/multi-server/freeradius-multi-server-test-runtime-logs/multi-server-test.log
      MULTI_SERVER_TEST_LISTENER_LOG: build/tests/multi-server/freeradius-multi-server-test-runtime-logs/multi-server-test-linelog-msg-output.log

    steps:
      # Checkout, but defer pulling LFS objects until we've restored the cache
      - uses: actions/checkout@v4
        with:
          lfs: false

      - name: Package manager performance improvements
        run: |
          sudo sh -c 'echo force-unsafe-io > /etc/dpkg/dpkg.cfg.d/02speedup'
          echo 'man-db man-db/auto-update boolean false' | sudo debconf-set-selections
          sudo dpkg-reconfigure man-db
          sudo sed -i 's/^update_initramfs=.*/update_initramfs=no/' /etc/initramfs-tools/update-initramfs.conf

      - name: NetworkRADIUS signing key
        shell: bash
        run: |
          sudo install -d -o root -g root -m 0755 /etc/apt/keyrings
          curl -s 'https://packages.inkbridgenetworks.com/pgp/packages.networkradius.com.asc' | sudo tee /etc/apt/keyrings/packages.networkradius.com.asc > /dev/null

      - name: Set up NetworkRADIUS extras repository
        shell: bash
        run: |
          DIST=$(lsb_release -is | tr '[:upper:]' '[:lower:]')
          RELEASE=$(lsb_release -cs)
          sudo /bin/sh -c "echo \"deb [arch=amd64 signed-by=/etc/apt/keyrings/packages.networkradius.com.asc] http://packages.networkradius.com/extras/${DIST}/${RELEASE} ${RELEASE} main\" \
            > /etc/apt/sources.list.d/networkradius-extras.list"
          sudo apt-get update

      # Remove pre-installed package which conflicts with dependency installation
      - name: Remove package conflicts
        run: |
          sudo apt-get remove -y libhashkit2

      # Installing build dependencies requires a ubuntu-24.04 runner
      - name: Install build dependencies
        run: |
          sudo apt-get install -y --no-install-recommends build-essential devscripts equivs quilt
          debian/rules debian/control
          sudo mk-build-deps -irt"apt-get -y --no-install-recommends" debian/control
          sudo mk-build-deps -irt"apt-get -y --no-install-recommends" scripts/ci/extra-packages.debian.control

      - name: Build Docker image for multi-server test environment
        run: |
          ./configure
          make docker.${MULTI_SERVER_ENV_DOCKER_BUILD_OS}.build

          # List all images
          docker images --all

          # Tag freeradius build image using using a non-OS specific name to be used with the multi-server docker compose environment.
          docker tag freeradius4/${MULTI_SERVER_ENV_DOCKER_BUILD_OS}:latest freeradius-build:latest

      - name: Run multi-server tests
        run: |
          if ! docker images --format '{{.Repository}}:{{.Tag}}' | grep -q "^freeradius-build:latest$"; then
            echo "Error: freeradius-build:latest Docker image not found and required for multi-server test environment."
            exit 1
          fi

          make -f src/tests/multi-server/all.mk

      - name: Verify test results
        shell: bash
        run: |
          set -euo pipefail

          echo "${MULTI_SERVER_TEST_LOG}:"
          cat "${MULTI_SERVER_TEST_LOG}"

          if grep -Eq "\(Failures:[^0-9]*[1-9][0-9]*\)" "${MULTI_SERVER_TEST_LOG}"; then
            echo "TEST FAILED: Found > 0 failures in log"

            # Display the test's linelog message file
            echo "${MULTI_SERVER_TEST_LISTENER_LOG}:"
            cat "${MULTI_SERVER_TEST_LISTENER_LOG}"
            exit 1
          fi

          # Verify that all of the expected linelog results were matched
          # during the test(s).
          #
          # Checking to see that the <actual> value matches <expected> value in lines like:
          #
          #   Matched: <actual> / <expected> (Failures: 0)
          #
          if awk '
            BEGIN { found=0; n1=0; n2=0; line="" }
            index($0,"Matched:") {
              orig=$0
              s=$0
              gsub(/[[:space:]]/, "", s)
              sub(/^.*Matched:/, "", s)
              sub(/\(.*/, "", s)
              n = split(s, parts, "/")
              if (n == 2) {
                a = parts[1] + 0
                b = parts[2] + 0
                if (a > 0 && a == b) { found=1; n1=a; n2=b; line=orig }
              }
            }
            END {
              if (found) { printf "MATCH: %d / %d\nMatched line: %s\n", n1, n2, line; exit 0 }
              exit 1
            }
          ' "${MULTI_SERVER_TEST_LOG}"
          then
            : # PASS
          else
            echo "TEST FAILED: Detected issues with the test's linelog message logs"
            # Display the test's linelog message file
            echo "${MULTI_SERVER_TEST_LISTENER_LOG}:"
            cat "${MULTI_SERVER_TEST_LISTENER_LOG}"
            exit 1
          fi

  # Job only runs on self-hosted runners
  run-multi-server-tests-selfhosted:
    needs: pre-test
    runs-on: self-hosted
    if: ${{ needs.pre-test.outputs.selfhosted == '1' }}

    services:
      dind:
        image: docker:dind
        options: --privileged
        env:
          DOCKER_TLS_CERTDIR: ""
          #  Bypass the squid proxy for internal registry access.
          NO_PROXY: "*.networkradius.com,127.0.0.1"
        #  Mount the host's internal CA so dind trusts
        #  docker.internal.networkradius.com for image pulls.
        #
        #  Share the runner's workspace with dind so that docker
        #  compose bind-mounts (radiusd.conf, env-setup.sh, listener
        #  dirs, etc.) resolve to real files inside the dind daemon.
        #
        #  github.workspace is the HOST path to the workspace.
        #  The runner mounts it into the job container at a
        #  different path (/__w/...), so we use a fixed mount
        #  point (/workspace) that both containers agree on.
        volumes:
          - /usr/local/share/ca-certificates/networkradius.com.crt:/etc/docker/certs.d/docker.internal.networkradius.com/ca.crt:ro
          - ${{ github.workspace }}:/workspace

    env:
      MULTI_SERVER_ENV_DOCKER_BUILD_OS: ubuntu24
      MULTI_SERVER_TEST_LOG: build/tests/multi-server/freeradius-multi-server-test-runtime-logs/multi-server-test.log
      MULTI_SERVER_TEST_LISTENER_LOG: build/tests/multi-server/freeradius-multi-server-test-runtime-logs/multi-server-test-linelog-msg-output.log

    container:
      image: docker.internal.networkradius.com/self-hosted
      #  "privileged" is needed for Samba install
      #  "memory-swap -1" enables full use of host swap and may help
      #    with containers randomly quitting with "The operation was
      #    canceled"
      options: >-
        --privileged
        --memory-swap -1
      env:
        DOCKER_HOST: tcp://dind:2375
        NO_PROXY: dind
      #  Shared workspace — see dind volumes comment above.
      volumes:
        - ${{ github.workspace }}:/workspace

    defaults:
      run:
        working-directory: /workspace

    steps:

      - name: Install extra packages
        run: |
          apt-get update && apt-get install -y --no-install-recommends docker.io docker-buildx docker-compose-v2 python3-venv

      # Checkout, but defer pulling LFS objects until we've restored the cache
      - uses: actions/checkout@v4
        with:
          lfs: false

      #  Authenticate to the internal registry via the dind daemon.
      #  The host Docker daemon is logged in via the runner's
      #  job-started hook, but dind is a separate daemon with no
      #  auth config.
      - name: Login to internal Docker registry
        env:
          DOCKER_USERNAME: ${{ secrets.DOCKER_REPO_USERNAME }}
          DOCKER_PASSWORD: ${{ secrets.DOCKER_REPO_PASSWORD }}
        run: |
          echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin https://docker.internal.networkradius.com/

      - name: Build Docker image from source
        run: |
          make docker.${MULTI_SERVER_ENV_DOCKER_BUILD_OS}.build
          docker tag freeradius4/${MULTI_SERVER_ENV_DOCKER_BUILD_OS}:latest freeradius-build:latest
          docker images --all

      - name: Run multi-server tests
        run: |
          if ! docker images --format '{{.Repository}}:{{.Tag}}' | grep -q "^freeradius-build:latest$"; then
            echo "Error: freeradius-build:latest Docker image not found and required for multi-server test environment."
            exit 1
          fi

          make -f src/tests/multi-server/all.mk

      - name: Verify test results
        shell: bash
        run: |
          set -euo pipefail

          echo "${MULTI_SERVER_TEST_LOG}:"
          cat "${MULTI_SERVER_TEST_LOG}"

          if grep -Eq "\(Failures:[^0-9]*[1-9][0-9]*\)" "${MULTI_SERVER_TEST_LOG}"; then
            echo "TEST FAILED: Found > 0 failures in log"

            # Display the test's linelog message file
            echo "${MULTI_SERVER_TEST_LISTENER_LOG}:"
            cat "${MULTI_SERVER_TEST_LISTENER_LOG}"
            exit 1
          fi

          # Verify that all of the expected linelog results were matched
          # during the test(s).
          #
          # Checking to see that the <actual> value matches <expected> value in lines like:
          #
          #   Matched: <actual> / <expected> (Failures: 0)
          #
          if awk '
            BEGIN { found=0; n1=0; n2=0; line="" }
            index($0,"Matched:") {
              orig=$0
              s=$0
              gsub(/[[:space:]]/, "", s)
              sub(/^.*Matched:/, "", s)
              sub(/\(.*/, "", s)
              n = split(s, parts, "/")
              if (n == 2) {
                a = parts[1] + 0
                b = parts[2] + 0
                if (a > 0 && a == b) { found=1; n1=a; n2=b; line=orig }
              }
            }
            END {
              if (found) { printf "MATCH: %d / %d\nMatched line: %s\n", n1, n2, line; exit 0 }
              exit 1
            }
          ' "${MULTI_SERVER_TEST_LOG}"
          then
            : # PASS
          else
            echo "TEST FAILED: Detected issues with the test's linelog message logs"
            # Display the test's linelog message file
            echo "${MULTI_SERVER_TEST_LISTENER_LOG}:"
            cat "${MULTI_SERVER_TEST_LISTENER_LOG}"
            exit 1
          fi

      #
      #  If the CI has failed and the branch is multi-server-baseline-tests then we start a tmate
      #  session to provide interactive shell access to the session.
      #
      #  The SSH rendezvous point will be emitted continuously in the job output,
      #  which will look something like:
      #
      #      SSH: ssh VfuX8SrNuU5pGPMyZcz7TpJTa@sfo2.tmate.io
      #
      #  For example:
      #
      #      git push origin multi-server-baseline-tests --force
      #
      #  Look at the job output in: https://github.com/FreeRADIUS/freeradius-server/actions
      #
      #      ssh VfuX8SrNuU5pGPMyZcz7TpJTa@sfo2.tmate.io
      #
      #  Access requires that you have the private key corresponding to the
      #  public key of the GitHub user that initiated the job.
      #
      - name: "Debug: Start tmate"
        uses: mxschmitt/action-tmate@v3
        with:
          limit-access-to-actor: true
        if: ${{ github.ref == 'refs/heads/multi-server-baseline-tests' && failure() }}