Merge branch 'loadgen-5hs-tests' into multi-server-tests

marc-casavant · marc-casavant · commit 6ef31e89f4df · 2026-02-11T09:52:23.000-05:00
diff --git a/.github/workflows/ci-multi-server-tests.yml b/.github/workflows/ci-multi-server-tests.yml
@@ -0,0 +1,147 @@
+name: Multi-Server CI Tests
+
+on:
+  push:
+    branches: [ "loadgen-5hs-tests" ]
+
+  pull_request:
+    branches: [ "loadgen-5hs-tests" ]
+    paths:
+      - '.github/workflows/ci-multi-server-tests.yml'
+      - 'src/tests/multi-server/**'
+  workflow_dispatch:
+
+jobs:
+  pre-test:
+    runs-on: ubuntu-latest
+    outputs:
+      selfhosted: 0
+      #selfhosted: ${{ github.repository_owner == 'FreeRADIUS' && '1' || '0' }}
+    steps:
+      - run: echo "Pre-test job; checking if using self-hosted runners"
+
+  test-5hs-autoaccept:
+    needs: pre-test
+    runs-on: ${{ needs.pre-test.outputs.selfhosted == '1' && 'self-hosted' || 'ubuntu-24.04' }}
+
+    env:
+      MULTI_SERVER_ENV_DOCKER_BUILD_OS: ubuntu24
+      MULTI_SERVER_TEST_LOG: build/tests/multi-server/freeradius-multi-server/multi_server_test.log
+
+    steps:
+      # Checkout, but defer pulling LFS objects until we've restored the cache
+      - uses: actions/checkout@v4
+        with:
+          lfs: false
+
+      - name: Package manager performance improvements
+        run: |
+          sudo sh -c 'echo force-unsafe-io > /etc/dpkg/dpkg.cfg.d/02speedup'
+          echo 'man-db man-db/auto-update boolean false' | sudo debconf-set-selections
+          sudo dpkg-reconfigure man-db
+          sudo sed -i 's/^update_initramfs=.*/update_initramfs=no/' /etc/initramfs-tools/update-initramfs.conf
+
+      - name: NetworkRADIUS signing key
+        shell: bash
+        run: |
+          sudo install -d -o root -g root -m 0755 /etc/apt/keyrings
+          curl -s 'https://packages.inkbridgenetworks.com/pgp/packages.networkradius.com.asc' | sudo tee /etc/apt/keyrings/packages.networkradius.com.asc > /dev/null
+
+      - name: Set up NetworkRADIUS extras repository
+        shell: bash
+        run: |
+          DIST=$(lsb_release -is | tr '[:upper:]' '[:lower:]')
+          RELEASE=$(lsb_release -cs)
+          sudo /bin/sh -c "echo \"deb [arch=amd64 signed-by=/etc/apt/keyrings/packages.networkradius.com.asc] http://packages.networkradius.com/extras/${DIST}/${RELEASE} ${RELEASE} main\" \
+            > /etc/apt/sources.list.d/networkradius-extras.list"
+          sudo apt-get update
+
+      # Remove pre-installed package which conflicts with dependency installation
+      - name: Remove package conflicts
+        run: |
+          sudo apt-get remove -y libhashkit2
+
+      # Installing build dependencies requires a ubuntu-24.04 runner
+      - name: Install build dependencies
+        run: |
+          sudo apt-get install -y --no-install-recommends build-essential devscripts equivs quilt
+          debian/rules debian/control
+          sudo mk-build-deps -irt"apt-get -y --no-install-recommends" debian/control
+          sudo mk-build-deps -irt"apt-get -y --no-install-recommends" scripts/ci/extra-packages.debian.control
+
+      - name: Build Docker image for multi-server test environment
+        if: ${{ needs.pre-test.outputs.selfhosted != '1' }}
+        run: |
+          ./configure
+          make docker.${MULTI_SERVER_ENV_DOCKER_BUILD_OS}.build
+
+          # List all images
+          docker images --all
+
+          # Tag freeradius build image using using a non-OS specific name to be used with the multi-server docker compose environment.
+          docker tag freeradius4/${MULTI_SERVER_ENV_DOCKER_BUILD_OS}:latest freeradius-build:latest
+
+      - name: Get pre-built Docker image for self-hosted runner test
+        if: ${{ needs.pre-test.outputs.selfhosted == '1' }}
+        run: |
+          docker pull docker.internal.networkradius.com/self-hosted-${MULTI_SERVER_ENV_DOCKER_BUILD_OS}:latest
+
+          # Tag freeradius build image using using a non-OS specific name to be used with the multi-server docker compose environment.
+          docker tag docker.internal.networkradius.com/self-hosted-${MULTI_SERVER_ENV_DOCKER_BUILD_OS}:latest freeradius-build:latest
+
+      - name: Run test-5hs-autoaccept test
+        if: ${{ needs.pre-test.outputs.selfhosted != '1' }}
+        run: |
+          if ! docker images --format '{{.Repository}}:{{.Tag}}' | grep -q "^freeradius-build:latest$"; then
+            echo "Error: freeradius-build:latest Docker image not found and required for multi-server test environment."
+            exit 1
+          fi
+          make -f src/tests/multi-server/all.mk test-5hs-autoaccept
+
+      - name: Verify test results
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          echo "============ ${MULTI_SERVER_TEST_LOG} ============"
+          cat "${MULTI_SERVER_TEST_LOG}"
+
+          if grep -q '\[Failed\]' "${MULTI_SERVER_TEST_LOG}"; then
+            echo "TEST FAILED: Found [Failed] in log"
+            grep '\[Failed\]' "${MULTI_SERVER_TEST_LOG}"
+            exit 1
+          fi
+
+          if grep -Eq '\(Failures:[[:space:]]*[1-9][0-9]*[[:space:]]*\)' "${MULTI_SERVER_TEST_LOG}"; then
+            echo "TEST FAILED: Found Failures > 0 in log"
+            grep -E '\(Failures:[[:space:]]*[1-9][0-9]*[[:space:]]*\)' "${MULTI_SERVER_TEST_LOG}"
+            exit 1
+          fi
+
+          log="${MULTI_SERVER_TEST_LOG}"
+          if awk '
+            BEGIN { found=0; n1=0; n2=0; line="" }
+            index($0,"Matched:") {
+              orig=$0
+              s=$0
+              gsub(/[[:space:]]/, "", s)
+              sub(/^.*Matched:/, "", s)
+              sub(/\(.*/, "", s)
+              n = split(s, parts, "/")
+              if (n == 2) {
+                a = parts[1] + 0
+                b = parts[2] + 0
+                if (a > 0 && a == b) { found=1; n1=a; n2=b; line=orig }
+              }
+            }
+            END {
+              if (found) { printf "MATCH: %d / %d\nMatched line: %s\n", n1, n2, line; exit 0 }
+              exit 1
+            }
+          ' "$log"
+          then
+            : # PASS
+          else
+            echo "TEST FAILED: No Matched line found with equal non-zero counts"
+            exit 1
+          fi
diff --git a/src/tests/multi-server/all.mk b/src/tests/multi-server/all.mk
@@ -26,22 +26,20 @@
 # of the test-framework.
 #
 
-# Find ENV compose file by stripping trailing "-suffix" chunks until a match exists.
-# Returns: environments/docker-compose/env-<base-without-test->.yml
-define FIND_ENV_COMPOSE
+# Find ENV compose template by stripping trailing "-suffix" chunks until a match exists.
+# Returns: environments/docker-compose/env-<base-without-test->.yml.j2
+define FIND_ENV_COMPOSE_J2
 $(strip $(shell \
 	name='$(1)'; base="$$name"; \
 	while :; do \
 		env="environments/docker-compose/env-$${base#test-}.yml"; \
 		envj2="$$env.j2"; \
 		if [ -f "$(MULTI_SERVER_TESTS_BASE_DIR_ABS_PATH)$$envj2" ]; then \
-			printf '%s' "$$env"; exit 0; \
-		elif [ -f "$(MULTI_SERVER_TESTS_BASE_DIR_ABS_PATH)$$env" ]; then \
-			printf '%s' "$$env"; exit 0; \
+			printf '%s' "$$envj2"; exit 0; \
 		fi; \
 		newbase="$${base%-*}"; \
 		if [ "$$newbase" = "$$base" ]; then \
-			echo "ERROR: No matching env compose file for $(1) (tried $$env(.j2) and shorter prefixes)" 1>&2; \
+			echo "ERROR: No matching env compose template for $(1) (expected $$env.j2 and shorter prefixes)" 1>&2; \
 			exit 1; \
 		fi; \
 		base="$$newbase"; \
@@ -54,49 +52,66 @@ define MAKE_TEST_TARGET
 $(1): TEST_NAME     := $(1)
 $(1): TEST_FILENAME := $$(TEST_NAME).yml
 
-# Compute compose path by finding the longest matching base env file
-$(1): ENV_COMPOSE_PATH := $$(call FIND_ENV_COMPOSE,$$(TEST_NAME))
+# Compose template (must exist)
+$(1): ENV_COMPOSE_TEMPLATE_PATH := $$(call FIND_ENV_COMPOSE_J2,$$(TEST_NAME))
+
+# Output compose file (strip .j2)
+$(1): ENV_COMPOSE_PATH := $$(patsubst %.j2,%,$$(ENV_COMPOSE_TEMPLATE_PATH))
+
+# Stem: environments/docker-compose/env-5hs-autoaccept.yml -> 5hs-autoaccept
+$(1): ENV_STEM := $$(patsubst environments/docker-compose/env-%.yml,%,$$(ENV_COMPOSE_PATH))
+
+# Vars aligned to stem
+$(1): VARS_FILE_REL ?= environments/jinja-vars/env-$$(ENV_STEM).vars.yml
 
 $(1): clone
 	@echo "MULTI_SERVER_BUILD_DIR_REL_PATH=$(MULTI_SERVER_BUILD_DIR_REL_PATH)"
 	@echo "MULTI_SERVER_BUILD_DIR_ABS_PATH=$(MULTI_SERVER_BUILD_DIR_ABS_PATH)"
 	@mkdir -p "$(MULTI_SERVER_BUILD_DIR_REL_PATH)/freeradius-listener-logs/$$(TEST_NAME)"
-	@cd "$(FRAMEWORK_REPO_DIR)" && \
-		git pull && \
-		\
-		$(MAKE) configure && \
-		. ".venv/bin/activate" && \
+	@bash -lc 'set -euo pipefail; \
+		echo "INFO: entering framework repo: $(FRAMEWORK_REPO_DIR)"; \
+		cd "$(FRAMEWORK_REPO_DIR)"; \
+		git pull; \
+		$(MAKE) configure; \
+		. ".venv/bin/activate"; \
 		\
 		DATA_PATH="$(MULTI_SERVER_TESTS_BASE_DIR_ABS_PATH)environments/configs"; \
 		LISTENER_DIR="$(MULTI_SERVER_BUILD_DIR_ABS_PATH)/freeradius-listener-logs/$$(TEST_NAME)"; \
+		INCLUDE_PATH_ABS="$(MULTI_SERVER_TESTS_BASE_DIR_ABS_PATH)"; \
+		VARS_FILE_ABS="$(MULTI_SERVER_TESTS_BASE_DIR_ABS_PATH)$$(VARS_FILE_REL)"; \
+		ENV_COMPOSE_TEMPLATE_ABS="$(MULTI_SERVER_TESTS_BASE_DIR_ABS_PATH)$$(ENV_COMPOSE_TEMPLATE_PATH)"; \
+		ENV_COMPOSE_ABS="$(MULTI_SERVER_TESTS_BASE_DIR_ABS_PATH)$$(ENV_COMPOSE_PATH)"; \
+		TEST_ABS="$(MULTI_SERVER_TESTS_BASE_DIR_ABS_PATH)$$(TEST_FILENAME)"; \
 		\
-		echo "INFO: TEST_FILENAME=$$(TEST_FILENAME)"; \
 		echo "INFO: TEST_NAME=$$(TEST_NAME)"; \
+		echo "INFO: TEST_FILENAME=$$(TEST_FILENAME)"; \
+		echo "INFO: TEST_ABS=$$TEST_ABS"; \
+		echo "INFO: ENV_COMPOSE_TEMPLATE_PATH=$$(ENV_COMPOSE_TEMPLATE_PATH)"; \
 		echo "INFO: ENV_COMPOSE_PATH=$$(ENV_COMPOSE_PATH)"; \
-		echo "INFO: MULTI_SERVER_TESTS_BASE_DIR_ABS_PATH=$(MULTI_SERVER_TESTS_BASE_DIR_ABS_PATH)"; \
-		echo "INFO: DATA_PATH=$$$$DATA_PATH"; \
-		echo "INFO: MULTI_SERVER_BUILD_DIR_REL_PATH=$(MULTI_SERVER_BUILD_DIR_REL_PATH)"; \
+		echo "INFO: ENV_STEM=$$(ENV_STEM)"; \
+		echo "INFO: VARS_FILE_REL=$$(VARS_FILE_REL)"; \
+		echo "INFO: VARS_FILE_ABS=$$$$VARS_FILE_ABS"; \
+		echo "INFO: ENV_COMPOSE_TEMPLATE_ABS=$$$$ENV_COMPOSE_TEMPLATE_ABS"; \
+		echo "INFO: ENV_COMPOSE_ABS=$$$$ENV_COMPOSE_ABS"; \
 		echo "INFO: LISTENER_DIR=$$$$LISTENER_DIR"; \
 		\
-		CMD="python3 src/config_builder.py --listener_type file --aux $(MULTI_SERVER_TESTS_BASE_DIR_ABS_PATH)environments/configs/freeradius/homeserver/radiusd.conf.j2 --includepath $(MULTI_SERVER_TESTS_BASE_DIR_ABS_PATH)"; \
-		echo "INFO: CMD = $$$$CMD"; \
-		bash -c "$$$$CMD"; \
+		test -f "$$$$VARS_FILE_ABS" || { echo "ERROR: Missing vars file: $$$$VARS_FILE_ABS" >&2; exit 1; }; \
+		test -f "$$$$ENV_COMPOSE_TEMPLATE_ABS" || { echo "ERROR: Missing compose template: $$$$ENV_COMPOSE_TEMPLATE_ABS" >&2; exit 1; }; \
+		test -f "$$$$TEST_ABS" || { echo "ERROR: Missing test file: $$$$TEST_ABS" >&2; exit 1; }; \
 		\
-		CMD="python3 src/config_builder.py --listener_type file --aux $(MULTI_SERVER_TESTS_BASE_DIR_ABS_PATH)environments/configs/freeradius/load-generator/radiusd.conf.j2 --includepath $(MULTI_SERVER_TESTS_BASE_DIR_ABS_PATH)"; \
-		echo "INFO: CMD = $$$$CMD"; \
-		bash -c "$$$$CMD"; \
+		echo "INFO: Rendering homeserver radiusd.conf.j2"; \
+		python3 src/config_builder.py --vars-file "$$$$VARS_FILE_ABS" --aux-file "$(MULTI_SERVER_TESTS_BASE_DIR_ABS_PATH)environments/configs/freeradius/homeserver/radiusd.conf.j2" --include-path "$$$$INCLUDE_PATH_ABS"; \
 		\
-		CMD="python3 src/config_builder.py --listener_type file --aux $(MULTI_SERVER_TESTS_BASE_DIR_ABS_PATH)environments/docker-compose/env-5hs-autoaccept.yml.j2 --includepath $(MULTI_SERVER_TESTS_BASE_DIR_ABS_PATH)"; \
-		echo "INFO: CMD = $$$$CMD"; \
-		bash -c "$$$$CMD"; \
+		echo "INFO: Rendering load-generator radiusd.conf.j2"; \
+		python3 src/config_builder.py --vars-file "$$$$VARS_FILE_ABS" --aux-file "$(MULTI_SERVER_TESTS_BASE_DIR_ABS_PATH)environments/configs/freeradius/load-generator/radiusd.conf.j2" --include-path "$$$$INCLUDE_PATH_ABS"; \
 		\
-		test -f "$(MULTI_SERVER_TESTS_BASE_DIR_ABS_PATH)$$(ENV_COMPOSE_PATH)" || { \
-			echo "ERROR: Missing compose file: $(MULTI_SERVER_TESTS_BASE_DIR_ABS_PATH)$$(ENV_COMPOSE_PATH)"; \
-			exit 1; \
-		} && \
-		CMD="DATA_PATH=$(MULTI_SERVER_TESTS_BASE_DIR_ABS_PATH)environments/configs make test-framework -- -x -v --compose $(MULTI_SERVER_TESTS_BASE_DIR_ABS_PATH)$$(ENV_COMPOSE_PATH) --test $(MULTI_SERVER_TESTS_BASE_DIR_ABS_PATH)$$(TEST_FILENAME) --use-files --listener-dir $$$$LISTENER_DIR"; \
-		echo "INFO: CMD = $$$$CMD"; \
-		bash -c "$$$$CMD"
+		echo "INFO: Rendering docker-compose env from template"; \
+		python3 src/config_builder.py --vars-file "$$$$VARS_FILE_ABS" --aux-file "$$$$ENV_COMPOSE_TEMPLATE_ABS" --include-path "$$$$INCLUDE_PATH_ABS"; \
+		\
+		test -f "$$$$ENV_COMPOSE_ABS" || { echo "ERROR: Compose file was not generated: $$$$ENV_COMPOSE_ABS" >&2; exit 1; }; \
+		\
+		echo "INFO: Running test-framework"; \
+		DATA_PATH="$(MULTI_SERVER_TESTS_BASE_DIR_ABS_PATH)environments/configs" make test-framework -- -x -v --compose "$$$$ENV_COMPOSE_ABS" --test "$$$$TEST_ABS" --use-files --listener-dir "$$$$LISTENER_DIR"'
 endef
 
 # Set directory name where all.mk is located. Help with relative paths
diff --git a/src/tests/multi-server/environments/configs/freeradius/homeserver/radiusd.conf.j2 b/src/tests/multi-server/environments/configs/freeradius/homeserver/radiusd.conf.j2
@@ -38,9 +38,9 @@ modules {
 	{% include "environments/configs/freeradius/common/mods-available/linelog" %}
 
 	# Test-framework specific linelog module configuration based on OS environment
-	{% if test.listener_type == "unix" %}
+	{% if listener_type == "unix" %}
         {% include "environments/configs/freeradius/common/mods-available/linelog_socket" %}
-	{% elif test.listener_type == "file" %}
+	{% elif listener_type == "file" %}
         {% include "environments/configs/freeradius/common/mods-available/linelog_file" %}
 	{% endif %}
 
diff --git a/src/tests/multi-server/environments/configs/freeradius/load-generator/radiusd.conf.j2 b/src/tests/multi-server/environments/configs/freeradius/load-generator/radiusd.conf.j2
@@ -30,17 +30,17 @@ modules {
 	{% include "environments/configs/freeradius/common/mods-available/linelog" %}
 
 	# Test-framework specific linelog module configuration based on OS environment
-	{% if test.listener_type == "unix" %}
+	{% if listener_type == "unix" %}
         {% include "environments/configs/freeradius/common/mods-available/linelog_socket" %}
-	{% elif test.listener_type == "file" %}
+	{% elif listener_type == "file" %}
         {% include "environments/configs/freeradius/common/mods-available/linelog_file" %}
 	{% endif %}
 
 	# Common stats module configuration
 	{% include "environments/configs/freeradius/common/mods-available/stats" %}
 
 	# radius module instance configuration
-	{% for i in range(1, 6) %}
+	{% for i in range(1, (num_of_dst_servers + 1)) %}
 	radius radius-module-dst-server{{ i }} {
 		$template radius-module-dst-server-tmpl
 		udp {
@@ -183,7 +183,7 @@ server load-generator {
 	authenticate load-generator-proxy {
 		# Use redundant-load-balance to distribute requests to multiple home servers
 		redundant-load-balance {
-			{% for i in range(1, 6) %}
+			{% for i in range(1, (num_of_dst_servers + 1)) %}
 			radius-module-dst-server{{ i }}
 			{% endfor %}
 		}
diff --git a/src/tests/multi-server/environments/docker-compose/env-5hs-autoaccept.yml.j2 b/src/tests/multi-server/environments/docker-compose/env-5hs-autoaccept.yml.j2
@@ -17,9 +17,9 @@ x-common-config: &id001
   - NET_ADMIN
   - SYS_PTRACE
 services:
-{% for i in range(1,6) %}
+{% for i in range(1, (num_of_dst_servers + 1)) %}
   homeserver{{ i }}:
-    image: fr-build-ubuntu22:latest
+    image: freeradius-build:latest
     volumes:
     - ${DATA_PATH}/freeradius/homeserver/radiusd.conf:/etc/raddb/radiusd.conf
     - ${DATA_PATH}/freeradius/env-setup.sh:/tmp/env-setup.sh
@@ -41,14 +41,14 @@ services:
     <<: *id001
 {% endfor %}
   load-generator:
-    image: fr-build-ubuntu22:latest
+    image: freeradius-build:latest
     ports:
       # Expose RADIUS ports for load generator to allow us to access the server from outside docker
       - "1812:1812/udp"
       - "1813:1813/udp"
       - "1820:1820/udp"
     depends_on:
-{% for i in range(1,6) %}
+{% for i in range(1, (num_of_dst_servers + 1)) %}
       - homeserver{{ i }}
 {% endfor %}
     volumes:
diff --git a/src/tests/multi-server/environments/jinja-vars/env-5hs-autoaccept.vars.yml b/src/tests/multi-server/environments/jinja-vars/env-5hs-autoaccept.vars.yml
@@ -0,0 +1,3 @@
+num_of_dst_servers: 5
+dst_server_name: homeserver
+listener_type: file

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+num_of_dst_servers: 5`
	`2`	`+dst_server_name: homeserver`
	`3`	`+listener_type: file`