Initial baseline multi-server tests for load-gen -> 5 homeserver, and load-gen -> 1 proxy -> 2 home server

.github/workflows/ci-multi-server-tests.yml

@@ -0,0 +1,319 @@
+name: Multi-Server CI Tests
+
+on:
+  push:
+    branches-ignore:
+      - coverity_scan
+      - run-fuzzer**
+      - debug-fuzzer-**
+  pull_request:
+    paths:
+      - '.github/workflows/ci-multi-server-tests.yml'
+      - 'src/tests/multi-server/**'
+  workflow_dispatch:
+
+jobs:
+  pre-test:
+    runs-on: ubuntu-latest
+    outputs:
+      #selfhosted: 0
+      selfhosted: ${{ (github.repository_owner == 'FreeRADIUS') && '1' || '0' }}
+    steps:
+      - run: echo "Pre-test job; checking if using self-hosted runners"
+
+  # Job only runs on GitHub-hosted runners
+  run-multi-server-tests:
+    needs: pre-test
+    runs-on: ubuntu-24.04
+    if: ${{ needs.pre-test.outputs.selfhosted != '1' }}
+
+    env:
+      MULTI_SERVER_ENV_DOCKER_BUILD_OS: ubuntu24
+      MULTI_SERVER_TEST_LOG: build/tests/multi-server/freeradius-multi-server-test-runtime-logs/multi-server-test.log
+      MULTI_SERVER_TEST_LISTENER_LOG: build/tests/multi-server/freeradius-multi-server-test-runtime-logs/multi-server-test-linelog-msg-output.log
+
+    steps:
+      # Checkout, but defer pulling LFS objects until we've restored the cache
+      - uses: actions/checkout@v4
+        with:
+          lfs: false
+
+      - name: Package manager performance improvements
+        run: |
+          sudo sh -c 'echo force-unsafe-io > /etc/dpkg/dpkg.cfg.d/02speedup'
+          echo 'man-db man-db/auto-update boolean false' | sudo debconf-set-selections
+          sudo dpkg-reconfigure man-db
+          sudo sed -i 's/^update_initramfs=.*/update_initramfs=no/' /etc/initramfs-tools/update-initramfs.conf
+
+      - name: NetworkRADIUS signing key
+        shell: bash
+        run: |
+          sudo install -d -o root -g root -m 0755 /etc/apt/keyrings
+          curl -s 'https://packages.inkbridgenetworks.com/pgp/packages.networkradius.com.asc' | sudo tee /etc/apt/keyrings/packages.networkradius.com.asc > /dev/null
+
+      - name: Set up NetworkRADIUS extras repository
+        shell: bash
+        run: |
+          DIST=$(lsb_release -is | tr '[:upper:]' '[:lower:]')
+          RELEASE=$(lsb_release -cs)
+          sudo /bin/sh -c "echo \"deb [arch=amd64 signed-by=/etc/apt/keyrings/packages.networkradius.com.asc] http://packages.networkradius.com/extras/${DIST}/${RELEASE} ${RELEASE} main\" \
+            > /etc/apt/sources.list.d/networkradius-extras.list"
+          sudo apt-get update
+
+      # Remove pre-installed package which conflicts with dependency installation
+      - name: Remove package conflicts
+        run: |
+          sudo apt-get remove -y libhashkit2
+
+      # Installing build dependencies requires a ubuntu-24.04 runner
+      - name: Install build dependencies
+        run: |
+          sudo apt-get install -y --no-install-recommends build-essential devscripts equivs quilt
+          debian/rules debian/control
+          sudo mk-build-deps -irt"apt-get -y --no-install-recommends" debian/control
+          sudo mk-build-deps -irt"apt-get -y --no-install-recommends" scripts/ci/extra-packages.debian.control
+
+      - name: Build Docker image for multi-server test environment
+        run: |
+          ./configure
+          make docker.${MULTI_SERVER_ENV_DOCKER_BUILD_OS}.build
+
+          # List all images
+          docker images --all
+
+          # Tag freeradius build image using using a non-OS specific name to be used with the multi-server docker compose environment.
+          docker tag freeradius4/${MULTI_SERVER_ENV_DOCKER_BUILD_OS}:latest freeradius-build:latest
+
+      - name: Run multi-server tests
+        run: |
+          if ! docker images --format '{{.Repository}}:{{.Tag}}' | grep -q "^freeradius-build:latest$"; then
+            echo "Error: freeradius-build:latest Docker image not found and required for multi-server test environment."
+            exit 1
+          fi
+
+          make -f src/tests/multi-server/all.mk
+
+      - name: Verify test results
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          echo "${MULTI_SERVER_TEST_LOG}:"
+          cat "${MULTI_SERVER_TEST_LOG}"
+
+          if grep -Eq "\(Failures:[^0-9]*[1-9][0-9]*\)" "${MULTI_SERVER_TEST_LOG}"; then
+            echo "TEST FAILED: Found > 0 failures in log"
+
+            # Display the test's linelog message file
+            echo "${MULTI_SERVER_TEST_LISTENER_LOG}:"
+            cat "${MULTI_SERVER_TEST_LISTENER_LOG}"
+            exit 1
+          fi
+
+          # Verify that all of the expected linelog results were matched
+          # during the test(s).
+          #
+          # Checking to see that the <actual> value matches <expected> value in lines like:
+          #
+          #   Matched: <actual> / <expected> (Failures: 0)
+          #
+          if awk '
+            BEGIN { found=0; n1=0; n2=0; line="" }
+            index($0,"Matched:") {
+              orig=$0
+              s=$0
+              gsub(/[[:space:]]/, "", s)
+              sub(/^.*Matched:/, "", s)
+              sub(/\(.*/, "", s)
+              n = split(s, parts, "/")
+              if (n == 2) {
+                a = parts[1] + 0
+                b = parts[2] + 0
+                if (a > 0 && a == b) { found=1; n1=a; n2=b; line=orig }
+              }
+            }
+            END {
+              if (found) { printf "MATCH: %d / %d\nMatched line: %s\n", n1, n2, line; exit 0 }
+              exit 1
+            }
+          ' "${MULTI_SERVER_TEST_LOG}"
+          then
+            : # PASS
+          else
+            echo "TEST FAILED: Detected issues with the test's linelog message logs"
+            # Display the test's linelog message file
+            echo "${MULTI_SERVER_TEST_LISTENER_LOG}:"
+            cat "${MULTI_SERVER_TEST_LISTENER_LOG}"
+            exit 1
+          fi
+
+  # Job only runs on self-hosted runners
+  run-multi-server-tests-selfhosted:
+    needs: pre-test
+    runs-on: self-hosted
+    if: ${{ needs.pre-test.outputs.selfhosted == '1' }}
+
+    services:
+      dind:
+        image: docker:dind
+        options: --privileged
+        env:
+          DOCKER_TLS_CERTDIR: ""
+          #  Bypass the squid proxy for internal registry access.
+          NO_PROXY: "*.networkradius.com,127.0.0.1"
+        #  Mount the host's internal CA so dind trusts
+        #  docker.internal.networkradius.com for image pulls.
+        #
+        #  Share the runner's workspace with dind so that docker
+        #  compose bind-mounts (radiusd.conf, env-setup.sh, listener
+        #  dirs, etc.) resolve to real files inside the dind daemon.
+        #
+        #  github.workspace is the HOST path to the workspace.
+        #  The runner mounts it into the job container at a
+        #  different path (/__w/...), so we use a fixed mount
+        #  point (/workspace) that both containers agree on.
+        volumes:
+          - /usr/local/share/ca-certificates/networkradius.com.crt:/etc/docker/certs.d/docker.internal.networkradius.com/ca.crt:ro
+          - ${{ github.workspace }}:/workspace
+
+    env:
+      MULTI_SERVER_ENV_DOCKER_BUILD_OS: ubuntu24
+      MULTI_SERVER_TEST_LOG: build/tests/multi-server/freeradius-multi-server-test-runtime-logs/multi-server-test.log
+      MULTI_SERVER_TEST_LISTENER_LOG: build/tests/multi-server/freeradius-multi-server-test-runtime-logs/multi-server-test-linelog-msg-output.log
+
+    container:
+      image: docker.internal.networkradius.com/self-hosted
+      #  "privileged" is needed for Samba install
+      #  "memory-swap -1" enables full use of host swap and may help
+      #    with containers randomly quitting with "The operation was
+      #    canceled"
+      options: >-
+        --privileged
+        --memory-swap -1
+      env:
+        DOCKER_HOST: tcp://dind:2375
+        NO_PROXY: dind
+      #  Shared workspace — see dind volumes comment above.
+      volumes:
+        - ${{ github.workspace }}:/workspace
+
+    defaults:
+      run:
+        working-directory: /workspace
+
+    steps:
+
+      - name: Install extra packages
+        run: |
+          apt-get update && apt-get install -y --no-install-recommends docker.io docker-buildx docker-compose-v2 python3-venv
+
+      # Checkout, but defer pulling LFS objects until we've restored the cache
+      - uses: actions/checkout@v4
+        with:
+          lfs: false
+
+      #  Authenticate to the internal registry via the dind daemon.
+      #  The host Docker daemon is logged in via the runner's
+      #  job-started hook, but dind is a separate daemon with no
+      #  auth config.
+      - name: Login to internal Docker registry
+        env:
+          DOCKER_USERNAME: ${{ secrets.DOCKER_REPO_USERNAME }}
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_REPO_PASSWORD }}
+        run: |
+          echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin https://docker.internal.networkradius.com/
+
+      - name: Build Docker image from source
+        run: |
+          make docker.${MULTI_SERVER_ENV_DOCKER_BUILD_OS}.build
+          docker tag freeradius4/${MULTI_SERVER_ENV_DOCKER_BUILD_OS}:latest freeradius-build:latest
+          docker images --all
+
+      - name: Run multi-server tests
+        run: |
+          if ! docker images --format '{{.Repository}}:{{.Tag}}' | grep -q "^freeradius-build:latest$"; then
+            echo "Error: freeradius-build:latest Docker image not found and required for multi-server test environment."
+            exit 1
+          fi
+
+          make -f src/tests/multi-server/all.mk
+
+      - name: Verify test results
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          echo "${MULTI_SERVER_TEST_LOG}:"
+          cat "${MULTI_SERVER_TEST_LOG}"
+
+          if grep -Eq "\(Failures:[^0-9]*[1-9][0-9]*\)" "${MULTI_SERVER_TEST_LOG}"; then
+            echo "TEST FAILED: Found > 0 failures in log"
+
+            # Display the test's linelog message file
+            echo "${MULTI_SERVER_TEST_LISTENER_LOG}:"
+            cat "${MULTI_SERVER_TEST_LISTENER_LOG}"
+            exit 1
+          fi
+
+          # Verify that all of the expected linelog results were matched
+          # during the test(s).
+          #
+          # Checking to see that the <actual> value matches <expected> value in lines like:
+          #
+          #   Matched: <actual> / <expected> (Failures: 0)
+          #
+          if awk '
+            BEGIN { found=0; n1=0; n2=0; line="" }
+            index($0,"Matched:") {
+              orig=$0
+              s=$0
+              gsub(/[[:space:]]/, "", s)
+              sub(/^.*Matched:/, "", s)
+              sub(/\(.*/, "", s)
+              n = split(s, parts, "/")
+              if (n == 2) {
+                a = parts[1] + 0
+                b = parts[2] + 0
+                if (a > 0 && a == b) { found=1; n1=a; n2=b; line=orig }
+              }
+            }
+            END {
+              if (found) { printf "MATCH: %d / %d\nMatched line: %s\n", n1, n2, line; exit 0 }
+              exit 1
+            }
+          ' "${MULTI_SERVER_TEST_LOG}"
+          then
+            : # PASS
+          else
+            echo "TEST FAILED: Detected issues with the test's linelog message logs"
+            # Display the test's linelog message file
+            echo "${MULTI_SERVER_TEST_LISTENER_LOG}:"
+            cat "${MULTI_SERVER_TEST_LISTENER_LOG}"
+            exit 1
+          fi
+
+      #
+      #  If the CI has failed and the branch is multi-server-baseline-tests then we start a tmate
+      #  session to provide interactive shell access to the session.
+      #
+      #  The SSH rendezvous point will be emitted continuously in the job output,
+      #  which will look something like:
+      #
+      #      SSH: ssh VfuX8SrNuU5pGPMyZcz7TpJTa@sfo2.tmate.io
+      #
+      #  For example:
+      #
+      #      git push origin multi-server-baseline-tests --force
+      #
+      #  Look at the job output in: https://github.com/FreeRADIUS/freeradius-server/actions
+      #
+      #      ssh VfuX8SrNuU5pGPMyZcz7TpJTa@sfo2.tmate.io
+      #
+      #  Access requires that you have the private key corresponding to the
+      #  public key of the GitHub user that initiated the job.
+      #
+      - name: "Debug: Start tmate"
+        uses: mxschmitt/action-tmate@v3
+        with:
+          limit-access-to-actor: true
+        if: ${{ github.ref == 'refs/heads/multi-server-baseline-tests' && failure() }}

src/tests/all.mk

@@ -102,6 +102,10 @@ endif
 $(BUILD_DIR)/tests:
 	${Q}mkdir -p $@
 
+.PHONY: test.multi-server.%
+test.multi-server.%:
+	$(MAKE) -f src/tests/multi-server/all.mk $*
+
 ######################################################################
 #
 #  Generic rules to set up the tests