From 297e903e44d8c9ce52105f77422d7c546583afd5 Mon Sep 17 00:00:00 2001 From: Terry Burton Date: Thu, 30 Apr 2026 19:05:19 +0100 Subject: [PATCH 1/2] CI: Summarise test results; capture failure logs; keep going on failure --- .github/actions/ci-tests/action.yml | 38 ++++++- scripts/ci/ci-summary.sh | 103 +++++++++++++++++++ scripts/ci/package-test.mk | 1 + src/tests/all.mk | 2 + src/tests/auth/all.mk | 3 + src/tests/bin/all.mk | 2 + src/tests/config/all.mk | 3 + src/tests/detail/all.mk | 3 + src/tests/dict/all.mk | 6 ++ src/tests/digest/all.mk | 2 + src/tests/eapol_test/all.mk | 2 + src/tests/keywords/all.mk | 3 + src/tests/ldap_sync/active_directory/all.mk | 5 +- src/tests/ldap_sync/persistent_search/all.mk | 5 +- src/tests/ldap_sync/rfc4533/all.mk | 5 +- src/tests/modules/all.mk | 3 + src/tests/multi-server/all.mk | 2 + src/tests/process/all.mk | 2 + src/tests/radclient/all.mk | 6 +- src/tests/radmin/all.mk | 5 +- src/tests/radsniff/all.mk | 5 + src/tests/tacacs/all.mk | 5 +- src/tests/test_record.mk | 12 +++ src/tests/trie/test.mk | 18 +++- src/tests/unit/all.mk | 9 +- src/tests/vmps/all.mk | 5 +- src/tests/xlat/all.mk | 2 + 27 files changed, 243 insertions(+), 14 deletions(-) create mode 100755 scripts/ci/ci-summary.sh create mode 100644 src/tests/test_record.mk diff --git a/.github/actions/ci-tests/action.yml b/.github/actions/ci-tests/action.yml index b1d2d04c68f63..adb8541426dd0 100644 --- a/.github/actions/ci-tests/action.yml +++ b/.github/actions/ci-tests/action.yml @@ -237,10 +237,13 @@ runs: run: | [ -d /opt/openssl ] && export PATH=/opt/openssl/bin:$PATH - make -j `nproc` test.keywords test.unit test.auth test.digest - make test.modules.redis test.modules.redis_ippool - make -j `nproc` test.modules - make ci-test + # Activate per-test result recording for the summary table. + mkdir -p build/tests && touch build/tests/results.tsv + + make -j `nproc` -k test.keywords test.unit test.auth test.digest || : + make -k test.modules.redis test.modules.redis_ippool || : + make -j `nproc` -k test.modules || : + make -k ci-test env: USE_DOCKER: ${{ inputs.use_docker }} @@ -269,6 +272,33 @@ runs: KAFKA_TEST_SERVER_PORT: ${{ inputs.kafka_test_server_port }} CACHE_MEMCACHED_TEST_SERVER: ${{ inputs.memcached_test_server }} + - name: Test summary + if: ${{ always() }} + shell: bash + run: | + ./scripts/ci/ci-summary.sh build/tests/results.tsv >> "$GITHUB_STEP_SUMMARY" || : + + - name: Stage failed test logs + if: ${{ failure() }} + shell: bash + run: | + mkdir -p build/tests/failed + [ -s build/tests/results.tsv ] && cp build/tests/results.tsv build/tests/failed/ + awk -F'\t' '$3=="FAIL"{print $1"\t"$2"\t"$4}' build/tests/results.tsv 2>/dev/null | \ + while IFS=$(printf '\t') read -r cat name logf; do + [ -n "$logf" ] && [ -f "$logf" ] || continue + mkdir -p "build/tests/failed/$cat" + cp "$logf" "build/tests/failed/$cat/$name.log" + done + + - name: Upload failed test logs + if: ${{ failure() }} + uses: actions/upload-artifact@v6 + with: + name: ci-test-logs-${{ inputs.use_docker == 'true' && 'docker' || 'host' }} + path: build/tests/failed + if-no-files-found: ignore + # Restore ucf - name: Restore ucf if: ${{ inputs.use_docker != 'true' }} diff --git a/scripts/ci/ci-summary.sh b/scripts/ci/ci-summary.sh new file mode 100755 index 0000000000000..be35fd586e08b --- /dev/null +++ b/scripts/ci/ci-summary.sh @@ -0,0 +1,103 @@ +#!/bin/sh +# +# scripts/ci/ci-summary.sh +# +# Wrapped awk script that reads the test record file and emits a markdown +# summary to stdout, intended for piping into $GITHUB_STEP_SUMMARY. +# + +set -eu + +file=${1:-${CI_TEST_RECORD_FILE:-build/tests/results.tsv}} + +if [ ! -s "$file" ]; then + echo "## Test results" + echo + echo "_No results recorded._" + exit 0 +fi + +awk -F'\t' ' +{ + key = $1 "\t" $2 + cat[key] = $1 + name[key] = $2 + status[key] = $3 + logf[key] = $4 + seen[key] = NR # last-wins +} +END { + # Per-category counts + for (k in seen) { + c = cat[k] + cats[c] = 1 + if (status[k] == "PASS") { cat_pass[c]++; total_pass++ } + else if (status[k] == "FAIL") { cat_fail[c]++; total_fail++ } + total++ + } + + # Header + printf("## Test results\n\n") + if (total_fail > 0) { + printf("**%d / %d passed** (%d failed)\n\n", total_pass, total, total_fail) + } else { + printf("**%d / %d passed**\n\n", total_pass, total) + } + + # Failure logs as collapsible blocks + if (total_fail > 0) { + printf("### Failure logs (last 20 lines)\n\n") + for (k in seen) { + if (status[k] != "FAIL") continue + printf("
%s/%s — %s\n\n", + cat[k], name[k], (logf[k] != "" ? logf[k] : "(no log captured)")) + if (logf[k] != "") { + printf("```\n") + n = 0 + while ((getline line < logf[k]) > 0) buf[++n % 20] = line + close(logf[k]) + start = (n < 20) ? 1 : n - 19 + for (j = start; j <= n; j++) print buf[j % 20] + delete buf + printf("```\n") + } else { + printf("_No log file recorded._\n") + } + printf("\n
\n\n") + } + printf("---\n\n") + } + + # Per-category sections. Categories with failures float to the top + # via a single insertion sort; otherwise alphabetical. + n = 0 + for (c in cats) ordered[++n] = c + for (i = 2; i <= n; i++) { + cur = ordered[i]; j = i - 1 + while (j >= 1) { + a = ordered[j]; b = cur + af = (cat_fail[a] ? 0 : 1); bf = (cat_fail[b] ? 0 : 1) + if (af < bf || (af == bf && a < b)) break + ordered[j+1] = ordered[j]; j-- + } + ordered[j+1] = cur + } + + for (i = 1; i <= n; i++) { + c = ordered[i] + cp = cat_pass[c] + 0; cf = cat_fail[c] + 0 + ct = cp + cf + if (cf > 0) printf("### %s (%d/%d, %d failed)\n\n", c, cp, ct, cf) + else printf("### %s (%d/%d)\n\n", c, cp, ct) + printf("| Test | Status |\n|---|---|\n") + # FAIL rows first, then PASS, each in test-execution order. + for (k in seen) if (cat[k] == c && status[k] == "FAIL") emit(k) + for (k in seen) if (cat[k] == c && status[k] == "PASS") emit(k) + printf("\n") + } +} + +function emit(k) { + printf("| `%s` | %s |\n", name[k], status[k]) +} +' "$file" diff --git a/scripts/ci/package-test.mk b/scripts/ci/package-test.mk index 86c0701a50486..cc086e4e58871 100644 --- a/scripts/ci/package-test.mk +++ b/scripts/ci/package-test.mk @@ -52,4 +52,5 @@ POST_INSTALL_RADIUSD_BIN_ARG:=RADIUSD_BIN=$(RADIUSD_BIN) .PHONY: package-test package-test: test.eap +include $(top_srcdir)/src/tests/test_record.mk include $(DIR)/all.mk diff --git a/src/tests/all.mk b/src/tests/all.mk index 8eedaeea21f90..9fa38fdfed10e 100644 --- a/src/tests/all.mk +++ b/src/tests/all.mk @@ -6,6 +6,8 @@ PORT := $(if $(PORT),$(PORT),12340) SECRET := $(if $(SECRET),$(SECRET),testing123) DICT_PATH := $(top_srcdir)/share/dictionary +include $(top_srcdir)/src/tests/test_record.mk + # # We need the 'git-lfs' installed to fetch some binary files. # diff --git a/src/tests/auth/all.mk b/src/tests/auth/all.mk index 838a1c8569d9d..5fd622bb786c2 100644 --- a/src/tests/auth/all.mk +++ b/src/tests/auth/all.mk @@ -100,6 +100,7 @@ $(OUTPUT)/%: $(DIR)/% $(OUTPUT)/%.attrs $(TEST_BIN_DIR)/unit_test_module | $(AUT echo "# $@.log"; \ echo "TESTDIR=$(notdir $@) $(TEST_BIN)/unit_test_module -D share/dictionary -d src/tests/auth/ -i \"$@.attrs\" -f \"$@.attrs\" -r \"$@\" -xxx > \"$@.log\" 2>&1"; \ rm -f $(BUILD_DIR)/tests/test.auth; \ + $(call test_record,auth,$(notdir $@),FAIL,$@.log); \ exit 1; \ fi; \ FOUND=$$(grep ^$< $@.log | head -1 | sed 's/:.*//;s/.*\[//;s/\].*//'); \ @@ -109,8 +110,10 @@ $(OUTPUT)/%: $(DIR)/% $(OUTPUT)/%.attrs $(TEST_BIN_DIR)/unit_test_module | $(AUT echo "# $@.log"; \ echo "TESTDIR=$(notdir $@) $(TEST_BIN)/unit_test_module -D share/dictionary -d src/tests/auth/ -i \"$@.attrs\" -f \"$@.attrs\" -r \"$@\" -xxx > \"$@.log\" 2>&1"; \ rm -f $(BUILD_DIR)/tests/test.auth; \ + $(call test_record,auth,$(notdir $@),FAIL,$@.log); \ exit 1; \ else \ touch "$@"; \ fi \ fi + @$(call test_record,auth,$(notdir $@),PASS,$@.log) diff --git a/src/tests/bin/all.mk b/src/tests/bin/all.mk index a585c8c8346e2..bb438e1a9dae7 100644 --- a/src/tests/bin/all.mk +++ b/src/tests/bin/all.mk @@ -57,8 +57,10 @@ $(BUILD_DIR)/tests/bin/%: $(BUILD_DIR)/bin/local/% echo LOG in $@.log; \ cat $@.log; \ echo $(TEST_BIN)/$(notdir $<) $($(notdir $@).ARGS); \ + $(call test_record,bin,$(notdir $@),FAIL,$@.log); \ exit 1; \ fi + @$(call test_record,bin,$(notdir $@),PASS,$@.log) ${Q}touch $@ # diff --git a/src/tests/config/all.mk b/src/tests/config/all.mk index 2a34f9df28981..3bf23f76a1046 100644 --- a/src/tests/config/all.mk +++ b/src/tests/config/all.mk @@ -67,6 +67,7 @@ $(OUTPUT)/%: $(DIR)/% $(TEST_BIN_DIR)/unit_test_module | $(CONFIG_LIBS) echo "# $@.log"; \ echo $(CMD); \ rm -f $(BUILD_DIR)/tests/test.config; \ + $(call test_record,config,$(notdir $@),FAIL,$@.log); \ exit 1; \ fi; \ FOUND=$$(grep 'Error : src/tests/config/' $@.log | egrep -v -- '-->' | head -1 | sed 's/]:.*//;s/.*\[//;s/\].*//'); \ @@ -76,9 +77,11 @@ $(OUTPUT)/%: $(DIR)/% $(TEST_BIN_DIR)/unit_test_module | $(CONFIG_LIBS) echo "# $@.log"; \ echo $(CMD); \ rm -f $(BUILD_DIR)/tests/test.config; \ + $(call test_record,config,$(notdir $@),FAIL,$@.log); \ exit 1; \ fi; \ fi + @$(call test_record,config,$(notdir $@),PASS,$@.log) touch "$@" $(TEST): diff --git a/src/tests/detail/all.mk b/src/tests/detail/all.mk index 9057fd154f788..ee854480f7ae8 100644 --- a/src/tests/detail/all.mk +++ b/src/tests/detail/all.mk @@ -23,13 +23,16 @@ $(OUTPUT)/%: $(DIR)/% $(addprefix ${BUILD_DIR}/lib/,proto_detail.la proto_detail ${Q}if ! $(TEST_BIN)/radiusd -d $(DIR)/config -D ${top_srcdir}/share/dictionary -X > $@.log; then \ tail $@.log; \ echo "cp $< $(dir $@)/detail.txt; $(TEST_BIN)/radiusd -d $(DIR)/config -D ${top_srcdir}/share/dictionary -X "; \ + $(call test_record,detail,$(notdir $@),FAIL,$@.log); \ exit 1; \ fi ${Q}if [ ! -e $(dir $@)/processed ] ; then \ tail $@.log; \ echo "Processing $< failed to produce expected output $(dir $@)/processed"; \ + $(call test_record,detail,$(notdir $@),FAIL,$@.log); \ exit 1; \ fi + @$(call test_record,detail,$(notdir $@),PASS,$@.log) ${Q}touch $@ .NO_PARALLEL: $(TEST) diff --git a/src/tests/dict/all.mk b/src/tests/dict/all.mk index 6fcee8b5524d8..bb877cb9e6cdf 100644 --- a/src/tests/dict/all.mk +++ b/src/tests/dict/all.mk @@ -25,8 +25,10 @@ $(OUTPUT)/%.dict: $(DIR)/%.dict $(TEST_BIN_DIR)/unit_test_attribute cat "$@.log"; \ echo "# $@.log"; \ echo "$(TEST_BIN)/unit_test_attribute -D $(top_srcdir)/share/dictionary -o "$@" -xx '$@.txt'"; \ + $(call test_record,dict,$(notdir $@),FAIL,$@.log); \ exit 1; \ fi + @$(call test_record,dict,$(notdir $@),PASS,$@.log) # And the actual script to run each test. # @@ -43,8 +45,10 @@ $(OUTPUT)/%.dict: $(DIR)/%.dict $(TEST_BIN_DIR)/unit_test_attribute cat "$@.log"; \ echo "# $@.log"; \ echo "$(TEST_BIN)/unit_test_attribute -D $(top_srcdir)/share/dictionary -o "$@" -xx '$@.txt'"; \ + $(call test_record,dict,$(notdir $@),FAIL,$@.log); \ exit 1; \ fi + @$(call test_record,dict,$(notdir $@),PASS,$@.log) # # Tests which are supposed to fail. @@ -65,6 +69,7 @@ $(OUTPUT)/%.error: $(DIR)/%.error $(TEST_BIN_DIR)/unit_test_attribute cat "$@.log"; \ echo "# $@.log"; \ echo "$(TEST_BIN)/unit_test_attribute -D $(top_srcdir)/share/dictionary -xx '$@.txt'"; \ + $(call test_record,dict,$(notdir $@),FAIL,$@.log); \ exit 1; \ fi ${Q}sed 's,${top_srcdir}/,,g' < "$@.log" > "$@.out" @@ -72,3 +77,4 @@ $(OUTPUT)/%.error: $(DIR)/%.error $(TEST_BIN_DIR)/unit_test_attribute echo "diff $@.out $(subst .error,,$<).out"; \ echo "FAILED"; \ fi + @$(call test_record,dict,$(notdir $@),PASS,$@.log) diff --git a/src/tests/digest/all.mk b/src/tests/digest/all.mk index d30597751e038..557046a45a416 100644 --- a/src/tests/digest/all.mk +++ b/src/tests/digest/all.mk @@ -48,8 +48,10 @@ $(OUTPUT)/%: $(DIR)/% | $(TEST).radiusd_kill $(TEST).radiusd_start $(MAKE) --no-print-directory test.digest.radiusd_kill; \ echo "RADIUSD: $(RADIUSD_RUN)"; \ echo "RADCLIENT: $(TEST_BIN)/radclient -f $@.request -xF -d src/tests/digest/config -D share/dictionary 127.0.0.1:$(digest_port) auth $(SECRET)"; \ + $(call test_record,digest,$(TARGET)_$${_num},FAIL,$@.out); \ exit 1; \ fi; \ + $(call test_record,digest,$(TARGET)_$${_num},PASS,$@.out); \ touch $@; \ done diff --git a/src/tests/eapol_test/all.mk b/src/tests/eapol_test/all.mk index e75e1872e0a61..8873488539a6b 100644 --- a/src/tests/eapol_test/all.mk +++ b/src/tests/eapol_test/all.mk @@ -134,9 +134,11 @@ $(OUTPUT)/%.ok: $(DIR)/%.conf $(if $(POST_INSTALL_MAKEFILE_ARG),,$(BUILD_DIR)/li echo " log is in $(OUT)"; \ rm -f $(BUILD_DIR)/tests/test.eap; \ $(MAKE) $(POST_INSTALL_MAKEFILE_ARG) --no-print-directory test.$(METHOD).radiusd_kill; \ + $(call test_record,eap,$(METHOD),FAIL,$(EAPOL_TEST_LOG)); \ exit 1;\ fi ${Q}$(MAKE) $(POST_INSTALL_MAKEFILE_ARG) --no-print-directory test.$(METHOD).radiusd_stop + ${Q}$(call test_record,eap,$(METHOD),PASS,$(EAPOL_TEST_LOG)) ${Q}touch $@ $(TEST): $(EAPOL_OK_FILES) diff --git a/src/tests/keywords/all.mk b/src/tests/keywords/all.mk index 0504fcbca44d2..ea80efff968eb 100644 --- a/src/tests/keywords/all.mk +++ b/src/tests/keywords/all.mk @@ -146,6 +146,7 @@ $(OUTPUT)/%: $(DIR)/% $(TEST_BIN_DIR)/unit_test_module | $(KEYWORD_RADDB) $(KEYW echo "# $@.log"; \ echo $(CMD); \ rm -f $(BUILD_DIR)/tests/test.keywords; \ + $(call test_record,keywords,$(notdir $@),FAIL,$@.log); \ exit 1; \ fi; \ FOUND=$$(grep 'Error : src/tests/keywords/' $@.log | egrep -v -- '-->' | head -1 | sed 's/]:.*//;s/.*\[//;s/\].*//'); \ @@ -155,11 +156,13 @@ $(OUTPUT)/%: $(DIR)/% $(TEST_BIN_DIR)/unit_test_module | $(KEYWORD_RADDB) $(KEYW echo "# $@.log"; \ echo $(CMD); \ rm -f $(BUILD_DIR)/tests/test.keywords; \ + $(call test_record,keywords,$(notdir $@),FAIL,$@.log); \ exit 1; \ else \ touch "$@"; \ fi \ fi + @$(call test_record,keywords,$(notdir $@),PASS,$@.log) $(TEST): @touch $(BUILD_DIR)/tests/$@ diff --git a/src/tests/ldap_sync/active_directory/all.mk b/src/tests/ldap_sync/active_directory/all.mk index 8eb20b4dd896f..67ac1804c82cf 100644 --- a/src/tests/ldap_sync/active_directory/all.mk +++ b/src/tests/ldap_sync/active_directory/all.mk @@ -32,7 +32,7 @@ $(OUTPUT)/%: $(DIR)/% | $(TEST).trigger_clear $(TEST).radiusd_kill $(TEST).radiu $(eval OUT_DIR := $(BUILD_DIR)/tests/ldap_sync/active_directory) ${Q}echo "LDAPSYNC-TEST active_directory $(TARGET)" - ${Q}[ -f $(dir $@)/radiusd.pid ] || exit 1 + ${Q}if ! [ -f $(dir $@)/radiusd.pid ]; then $(call test_record,ldap_sync,active_directory/$(TARGET),FAIL,$(OUT_DIR)/radiusd.log); exit 1; fi ${Q}rm -f $(OUT_DIR)/$(OUT).out # Wait for the sync to start before applying changes @@ -60,6 +60,7 @@ $(OUTPUT)/%: $(DIR)/% | $(TEST).trigger_clear $(TEST).radiusd_kill $(TEST).radiu cat $(OUT_DIR)/radiusd.log; \ echo "LDAP_SYNC FAILED $(TARGET) - expected output file not produced"; \ rm -rf $(BUILD_DIR)/tests/test.ldap_sync/active_directory; \ + $(call test_record,ldap_sync,active_directory/$(TARGET),FAIL,$(OUT_DIR)/radiusd.log); \ exit 1; \ fi ${Q}mv $(OUT_DIR)/$(OUT).out $(FOUND) @@ -69,8 +70,10 @@ $(OUTPUT)/%: $(DIR)/% | $(TEST).trigger_clear $(TEST).radiusd_kill $(TEST).radiu cat $(OUT_DIR)/radiusd.log; \ echo "LDAP_SYNC FAILED $(TARGET)"; \ rm -rf $(BUILD_DIR)/tests/test.ldap_sync/active_directory; \ + $(call test_record,ldap_sync,active_directory/$(TARGET),FAIL,$(FOUND)); \ exit 1; \ fi + @$(call test_record,ldap_sync,active_directory/$(TARGET),PASS,$(FOUND)) ${Q}touch $@ $(TEST): diff --git a/src/tests/ldap_sync/persistent_search/all.mk b/src/tests/ldap_sync/persistent_search/all.mk index 954dbd383c981..6c9a076439086 100644 --- a/src/tests/ldap_sync/persistent_search/all.mk +++ b/src/tests/ldap_sync/persistent_search/all.mk @@ -28,7 +28,7 @@ $(OUTPUT)/%: $(DIR)/% | $(TEST).trigger_clear $(TEST).radiusd_kill $(TEST).radiu $(eval OUT := $(shell grep "#.*OUT:" $< | cut -f2 -d ':')) ${Q}echo "LDAPSYNC-TEST persistent_search $(TARGET)" - ${Q}[ -f $(dir $@)/radiusd.pid ] || exit 1 + ${Q}if ! [ -f $(dir $@)/radiusd.pid ]; then $(call test_record,ldap_sync,persistent_search/$(TARGET),FAIL,$(OUT_DIR)/radiusd.log); exit 1; fi ${Q}rm -f $(OUT_DIR)/$(OUT).out # Wait for the sync to start before applying changes @@ -56,6 +56,7 @@ $(OUTPUT)/%: $(DIR)/% | $(TEST).trigger_clear $(TEST).radiusd_kill $(TEST).radiu cat $(OUT_DIR)/radiusd.log; \ echo "LDAP_SYNC FAILED $(TARGET) - expected output file not produced"; \ rm -rf $(BUILD_DIR)/tests/test.ldap_sync/persistent_search; \ + $(call test_record,ldap_sync,persistent_search/$(TARGET),FAIL,$(OUT_DIR)/radiusd.log); \ exit 1; \ fi ${Q}mv $(OUT_DIR)/$(OUT).out $(FOUND) @@ -65,8 +66,10 @@ $(OUTPUT)/%: $(DIR)/% | $(TEST).trigger_clear $(TEST).radiusd_kill $(TEST).radiu cat $(OUT_DIR)/radiusd.log; \ echo "LDAP_SYNC FAILED $(TARGET)"; \ rm -rf $(BUILD_DIR)/tests/test.ldap_sync/persistent_search; \ + $(call test_record,ldap_sync,persistent_search/$(TARGET),FAIL,$(FOUND)); \ exit 1; \ fi + @$(call test_record,ldap_sync,persistent_search/$(TARGET),PASS,$(FOUND)) ${Q}touch $@ $(TEST): diff --git a/src/tests/ldap_sync/rfc4533/all.mk b/src/tests/ldap_sync/rfc4533/all.mk index d34e047e30b99..7ace5e2b4c86e 100644 --- a/src/tests/ldap_sync/rfc4533/all.mk +++ b/src/tests/ldap_sync/rfc4533/all.mk @@ -29,7 +29,7 @@ $(OUTPUT)/%: $(DIR)/% | $(TEST).trigger_clear $(TEST).radiusd_kill $(TEST).radiu $(eval OUT := $(shell grep "#.*OUT:" $< | cut -f2 -d ':')) ${Q}echo "LDAPSYNC-TEST rfc4533 $(TARGET)" - ${Q}[ -f $(dir $@)/radiusd.pid ] || exit 1 + ${Q}if ! [ -f $(dir $@)/radiusd.pid ]; then $(call test_record,ldap_sync,rfc4533/$(TARGET),FAIL,$(OUT_DIR)/radiusd.log); exit 1; fi ${Q}rm -f $(OUT_DIR)/$(OUT).out # Wait for the sync to start before applying changes @@ -57,6 +57,7 @@ $(OUTPUT)/%: $(DIR)/% | $(TEST).trigger_clear $(TEST).radiusd_kill $(TEST).radiu cat $(OUT_DIR)/radiusd.log; \ echo "LDAP_SYNC FAILED $(TARGET) - expected output file not produced"; \ rm -rf $(BUILD_DIR)/tests/test.ldap_sync/rfc4533; \ + $(call test_record,ldap_sync,rfc4533/$(TARGET),FAIL,$(OUT_DIR)/radiusd.log); \ exit 1; \ fi ${Q}mv $(OUT_DIR)/$(OUT).out $(FOUND) @@ -65,8 +66,10 @@ $(OUTPUT)/%: $(DIR)/% | $(TEST).trigger_clear $(TEST).radiusd_kill $(TEST).radiu cat $(OUT_DIR)/radiusd.log; \ echo "LDAP_SYNC FAILED $(TARGET)"; \ rm -rf $(BUILD_DIR)/tests/test.ldap_sync/rfc4533; \ + $(call test_record,ldap_sync,rfc4533/$(TARGET),FAIL,$(FOUND)); \ exit 1; \ fi + @$(call test_record,ldap_sync,rfc4533/$(TARGET),PASS,$(FOUND)) ${Q}touch $@ $(TEST): diff --git a/src/tests/modules/all.mk b/src/tests/modules/all.mk index b99f6667d3516..22b0d8436b34e 100644 --- a/src/tests/modules/all.mk +++ b/src/tests/modules/all.mk @@ -166,6 +166,7 @@ $(OUTPUT)/%: $(DIR)/%.unlang $(TEST_BIN_DIR)/unit_test_module | build.raddb cat "$@.log"; \ echo "# $@.log"; \ echo "MODULE_TEST_DIR=$(dir $<) MODULE_TEST_UNLANG=$< OUTPUT_DIR=$(dir $@) $(TEST_BIN)/unit_test_module -D share/dictionary -d src/tests/modules/ -i \"$@.attrs\" -f \"$@.attrs\" -r \"$@\" -xx"; \ + $(call test_record,modules,$(lastword $(subst /, ,$(dir $@)))/$(basename $(notdir $@)),FAIL,$@.log); \ exit 1; \ fi; \ FOUND=$$(grep -E 'Error : $<' $@.log | head -1 | sed 's/.*\[//;s/\].*//'); \ @@ -174,11 +175,13 @@ $(OUTPUT)/%: $(DIR)/%.unlang $(TEST_BIN_DIR)/unit_test_module | build.raddb cat "$@.log"; \ echo "# $@.log"; \ echo "MODULE_TEST_DIR=$(dir $<) MODULE_TEST_UNLANG=$< OUTPUT_DIR=$(dir $@) $(TEST_BIN)/unit_test_module -D share/dictionary -d src/tests/modules/ -i \"$@.attrs\" -f \"$@.attrs\" -r \"$@\" -xx"; \ + $(call test_record,modules,$(lastword $(subst /, ,$(dir $@)))/$(basename $(notdir $@)),FAIL,$@.log); \ exit 1; \ else \ touch "$@"; \ fi \ fi + @$(call test_record,modules,$(lastword $(subst /, ,$(dir $@)))/$(basename $(notdir $@)),PASS,$@.log) # # Allow running individual tests. diff --git a/src/tests/multi-server/all.mk b/src/tests/multi-server/all.mk index 50f9006b13711..c6c3ab271126b 100644 --- a/src/tests/multi-server/all.mk +++ b/src/tests/multi-server/all.mk @@ -161,8 +161,10 @@ test.multi-server.${1}.${2}: $$(TEST_MULTI_SERVER_RENDERED.${1}.${2}) esac; \ tail -200 "$$$$f"; \ done; \ + $$(call test_record,multi-server,${1}.${2},FAIL,${4}/logs/stderr.log); \ exit 1; \ } + ${Q}$$(call test_record,multi-server,${1}.${2},PASS,${4}/logs/stdout.log) endef # diff --git a/src/tests/process/all.mk b/src/tests/process/all.mk index d1f187e713d8f..b9c125ce282fa 100644 --- a/src/tests/process/all.mk +++ b/src/tests/process/all.mk @@ -102,8 +102,10 @@ $(OUTPUT)/%: $(DIR)/% $(PROCESS_DICT) $(TEST_BIN_DIR)/unit_test_module $(DIR)/un cat $@.log; \ echo "# $@.log"; \ echo $(CMD); \ + $(call test_record,process,$(PROTOCOL_NAME)/$(notdir $@),FAIL,$@.log); \ exit 1; \ fi + @$(call test_record,process,$(PROTOCOL_NAME)/$(notdir $@),PASS,$@.log) $(TEST): $(Q)touch $(BUILD_DIR)/tests/$@ diff --git a/src/tests/radclient/all.mk b/src/tests/radclient/all.mk index dc9fd27a917b8..4ffcf47e50080 100644 --- a/src/tests/radclient/all.mk +++ b/src/tests/radclient/all.mk @@ -54,7 +54,7 @@ $(OUTPUT)/%: $(DIR)/% $(BUILD_DIR)/bin/local/$(RADCLIENT) $(BUILD_DIR)/lib/local $(eval RADCLIENT_CLIENT_PORT := $(shell echo $$(($(RADCLIENT_CLIENT_PORT)+1)))) ${Q}echo "RADCLIENT-TEST INPUT=$(TARGET) ARGV=\"$(ARGV)\"" - ${Q}[ -f $(dir $@)/radiusd.pid ] || exit 1 + ${Q}if ! [ -f $(dir $@)/radiusd.pid ]; then $(call test_record,radclient,$(notdir $@),FAIL,); exit 1; fi ${Q}if ! $(TEST_BIN)/$(RADCLIENT) $(ARGV) -C $(RADCLIENT_CLIENT_PORT) -f $< -d src/tests/radclient/config -D share/dictionary 127.0.0.1:$(radclient_port) $(TYPE) $(SECRET) 1> $(FOUND) 2>&1; then \ if [ "$(IGNORE_ERROR)" != "1" ]; then \ echo "FAILED"; \ @@ -63,6 +63,7 @@ $(OUTPUT)/%: $(DIR)/% $(BUILD_DIR)/bin/local/$(RADCLIENT) $(BUILD_DIR)/lib/local $(MAKE) --no-print-directory test.radclient.radiusd_kill; \ echo "RADIUSD: $(RADIUSD_RUN)"; \ echo "RADCLIENT: $(TEST_BIN)/$(RADCLIENT) $(ARGV) -C $(RADCLIENT_CLIENT_PORT) -f $< -xF -d src/tests/radclient/config -D share/dictionary 127.0.0.1:$(radclient_port) $(TYPE) $(SECRET)"; \ + $(call test_record,radclient,$(notdir $@),FAIL,$(FOUND)); \ exit 1; \ fi; \ fi @@ -99,6 +100,7 @@ $(OUTPUT)/%: $(DIR)/% $(BUILD_DIR)/bin/local/$(RADCLIENT) $(BUILD_DIR)/lib/local diff -I 'Sent' -I 'Received' $(EXPECTED) $(FOUND); \ rm -f $(BUILD_DIR)/tests/test.radclient; \ $(MAKE) --no-print-directory test.radclient.radiusd_kill; \ + $(call test_record,radclient,$(notdir $@),FAIL,$(FOUND)); \ exit 1; \ elif [ -e "$(CMD_TEST)" ] && ! $(SHELL) $(CMD_TEST); then \ echo "RADCLIENT FAILED $@"; \ @@ -108,8 +110,10 @@ $(OUTPUT)/%: $(DIR)/% $(BUILD_DIR)/bin/local/$(RADCLIENT) $(BUILD_DIR)/lib/local echo "If you did some update on the radclient code, please be sure to update the unit tests."; \ rm -f $(BUILD_DIR)/tests/test.radclient; \ $(MAKE) --no-print-directory test.radclient.radiusd_kill; \ + $(call test_record,radclient,$(notdir $@),FAIL,$(FOUND)); \ exit 1; \ fi + @$(call test_record,radclient,$(notdir $@),PASS,$(FOUND)) ${Q}touch $@ .NO_PARALLEL: $(TEST) diff --git a/src/tests/radmin/all.mk b/src/tests/radmin/all.mk index c41506773b473..3f701323a4c0b 100644 --- a/src/tests/radmin/all.mk +++ b/src/tests/radmin/all.mk @@ -54,7 +54,7 @@ $(OUTPUT)/depends.mk: $(addprefix $(DIR)/,$(FILES)) | $(OUTPUT) # $(OUTPUT)/%: $(DIR)/% | $(TEST).radiusd_kill $(TEST).radiusd_start @echo "RADMIN-TEST $(notdir $@)" - ${Q} [ -f $(dir $@)/radiusd.pid ] || exit 1 + ${Q} if ! [ -f $(dir $@)/radiusd.pid ]; then $(call test_record,radmin,$(notdir $@),FAIL,); exit 1; fi $(eval EXPECTED := $(patsubst %.txt,%.out,$<)) $(eval FOUND := $(patsubst %.txt,%.out,$@)) $(eval TARGET := $(patsubst %.txt,%,$(notdir $@)$(E))) @@ -67,6 +67,7 @@ $(OUTPUT)/%: $(DIR)/% | $(TEST).radiusd_kill $(TEST).radiusd_start echo "RADMIN : $(TEST_BIN)/radmin -q -f $(RADMIN_SOCKET_FILE) < $< > $(FOUND)"; \ rm -f $(BUILD_DIR)/tests/test.radmin; \ $(MAKE) --no-print-directory test.radmin.radiusd_kill; \ + $(call test_record,radmin,$(notdir $@),FAIL,$(FOUND)); \ exit 1; \ fi; \ sed -i.bak -e '$${/Executing: /d;}' $(FOUND); \ @@ -80,8 +81,10 @@ $(OUTPUT)/%: $(DIR)/% | $(TEST).radiusd_kill $(TEST).radiusd_start diff $(EXPECTED) $(FOUND); \ rm -f $(BUILD_DIR)/tests/test.radmin; \ $(MAKE) --no-print-directory test.radmin.radiusd_kill; \ + $(call test_record,radmin,$(notdir $@),FAIL,$(FOUND)); \ exit 1; \ else \ + $(call test_record,radmin,$(notdir $@),PASS,$(FOUND)); \ touch $@;\ fi diff --git a/src/tests/radsniff/all.mk b/src/tests/radsniff/all.mk index 516941c9a1219..b7f4feee9915f 100644 --- a/src/tests/radsniff/all.mk +++ b/src/tests/radsniff/all.mk @@ -56,6 +56,7 @@ $(OUTPUT)/%.txt: $(DIR)/%.txt $(TEST_BIN_DIR)/radsniff $(PCAP_IN) cat $(FOUND); \ echo "RADSNIFF: TZ='UTC' $(TEST_BIN)/radsniff $(ARGV) -I $(PCAP_IN) -D share/dictionary" -xx; \ rm -f $@; \ + $(call test_record,radsniff,$(notdir $@),FAIL,$(FOUND)); \ exit 1; \ fi ${Q}if [ -e "$(EXPECTED)" ]; then \ @@ -70,6 +71,7 @@ $(OUTPUT)/%.txt: $(DIR)/%.txt $(TEST_BIN_DIR)/radsniff $(PCAP_IN) diff $(EXPECTED).simple $(FOUND); \ echo "diff $(EXPECTED).simple $(FOUND)"; \ rm -f $@; \ + $(call test_record,radsniff,$(notdir $@),FAIL,$(FOUND)); \ exit 1; \ fi; \ elif [ -e "$(CMD_TEST)" ] && ! $(SHELL) $(CMD_TEST); then \ @@ -78,11 +80,14 @@ $(OUTPUT)/%.txt: $(DIR)/%.txt $(TEST_BIN_DIR)/radsniff $(PCAP_IN) echo "ERROR: The script $(CMD_TEST) can't validate the content of $(FOUND)"; \ echo "If radsniff has been modified, please update the unit tests."; \ rm -f $@; \ + $(call test_record,radsniff,$(notdir $@),FAIL,$(FOUND)); \ exit 1; \ else \ echo "ERROR! We should have at least one .txt or .cmd test"; \ + $(call test_record,radsniff,$(notdir $@),FAIL,$(FOUND)); \ exit 1; \ fi ${Q}rm -f ${EXPECTED}.simple + @$(call test_record,radsniff,$(notdir $@),PASS,$(FOUND)) ${Q}touch $@ endif diff --git a/src/tests/tacacs/all.mk b/src/tests/tacacs/all.mk index 0776489e7d519..0de404151ff12 100644 --- a/src/tests/tacacs/all.mk +++ b/src/tests/tacacs/all.mk @@ -53,7 +53,7 @@ $(OUTPUT)/%: $(DIR)/% $(BUILD_DIR)/lib/libfreeradius-tacacs.la $(BUILD_DIR)/lib/ $(eval FOUND := $(patsubst %.txt,%.out,$@)) $(eval ARGV := $(shell grep "#.*ARGV:" $< | cut -f2 -d ':')) ${Q}echo "TACACS-TEST INPUT=$(TARGET) TACACS_ARGV=\"$(ARGV)\"" - ${Q}[ -f $(dir $@)/radiusd.pid ] || exit 1 + ${Q}if ! [ -f $(dir $@)/radiusd.pid ]; then $(call test_record,tacacs,$(notdir $@),FAIL,); exit 1; fi ${Q}if ! $(TACCLIENT) --return-0-if-failed -v -k $(SECRET) -p $(tacacs_port) -H localhost -r 192.168.69.1 -P pegapilha/0 --timeout 2 $(ARGV) 1> $(FOUND) 2>&1; then \ echo "FAILED"; \ cat $(FOUND); \ @@ -61,6 +61,7 @@ $(OUTPUT)/%: $(DIR)/% $(BUILD_DIR)/lib/libfreeradius-tacacs.la $(BUILD_DIR)/lib/ $(MAKE) --no-print-directory test.tacacs.radiusd_kill; \ echo "RADIUSD: $(RADIUSD_RUN)"; \ echo "TACCLIENT: $(TACCLIENT) --return-0-if-failed -v -k $(SECRET) -p $(tacacs_port) -H localhost -r 192.168.69.1 -P pegapilha/0 --timeout 2 $(ARGV)"; \ + $(call test_record,tacacs,$(notdir $@),FAIL,$(FOUND)); \ exit 1; \ fi # @@ -78,8 +79,10 @@ $(OUTPUT)/%: $(DIR)/% $(BUILD_DIR)/lib/libfreeradius-tacacs.la $(BUILD_DIR)/lib/ diff $(EXPECTED) $(FOUND); \ rm -f $(BUILD_DIR)/tests/test.tacacs; \ $(MAKE) --no-print-directory test.tacacs.radiusd_kill; \ + $(call test_record,tacacs,$(notdir $@),FAIL,$(FOUND)); \ exit 1; \ fi + @$(call test_record,tacacs,$(notdir $@),PASS,$(FOUND)) ${Q}touch $@ $(TEST): diff --git a/src/tests/test_record.mk b/src/tests/test_record.mk new file mode 100644 index 0000000000000..dfd750c7fdf50 --- /dev/null +++ b/src/tests/test_record.mk @@ -0,0 +1,12 @@ +# +# Per-test result recorder. Invoked from individual test recipes as: +# +# $(call test_record,,,,) +# +# Activated by touching $(BUILD_DIR)/tests/results.tsv +# + +BUILD_DIR ?= $(top_builddir)/build +RESULTS_TSV ?= $(BUILD_DIR)/tests/results.tsv + +test_record = [ -e $(RESULTS_TSV) ] && printf '%s\t%s\t%s\t%s\n' '$(1)' '$(2)' '$(3)' '$(4)' >> $(RESULTS_TSV) || true diff --git a/src/tests/trie/test.mk b/src/tests/trie/test.mk index b553452a334c0..4cfbbed986303 100644 --- a/src/tests/trie/test.mk +++ b/src/tests/trie/test.mk @@ -12,11 +12,25 @@ $(BUILD_DIR)/tests/trie: $(BUILD_DIR)/tests/trie/trie-%: $(DIR)/% $(TEST_BIN_DIR)/trie | $(BUILD_DIR)/tests/trie @echo TRIE-TEST $(notdir $@) - @$(TEST_BIN)/trie $^ > $@ + @if $(TEST_BIN)/trie $^ > $@ 2> $@.log; then \ + $(call test_record,trie,$(notdir $@),PASS,$@.log); \ + else \ + cat $@.log; \ + rm -f $@; \ + $(call test_record,trie,$(notdir $@),FAIL,$@.log); \ + exit 1; \ + fi $(BUILD_DIR)/tests/trie/nopc-%: $(DIR)/% $(TEST_BIN_DIR)/nopc | $(BUILD_DIR)/tests/trie @echo TRIE-NO-PC-TEST $(notdir $@) - @$(TEST_BIN)/nopc $^ > $@ + @if $(TEST_BIN)/nopc $^ > $@ 2> $@.log; then \ + $(call test_record,trie,$(notdir $@),PASS,$@.log); \ + else \ + cat $@.log; \ + rm -f $@; \ + $(call test_record,trie,$(notdir $@),FAIL,$@.log); \ + exit 1; \ + fi # # Get all of the unit test output files diff --git a/src/tests/unit/all.mk b/src/tests/unit/all.mk index b9281c62ceda2..58b23ae20c561 100644 --- a/src/tests/unit/all.mk +++ b/src/tests/unit/all.mk @@ -95,7 +95,14 @@ $(filter $(BUILD_DIR)/tests/unit/purify/%,$(FILES.$(TEST))): PURIFY=-p $(OUTPUT)/%: $(DIR)/% $(TEST_BIN_DIR)/unit_test_attribute $(eval DIR:=${top_srcdir}/src/tests/unit) $(eval export UNIT_TEST_ATTRIBUTE:=TZ=GMT $(TEST_BIN_NO_TIMEOUT)/unit_test_attribute $(PURIFY) -F ./src/tests/fuzzer-corpus -D ./share/dictionary -d $(DIR) -o \"$@\" $<) - ${Q}$(TEST_BIN)/unit_test_attribute $(PURIFY) $(REWRITE_FLAGS) -F ./src/tests/fuzzer-corpus -D ./share/dictionary -d $(DIR) -o "$@" $< + ${Q}mkdir -p $(dir $@) + ${Q}if $(TEST_BIN)/unit_test_attribute $(PURIFY) $(REWRITE_FLAGS) -F ./src/tests/fuzzer-corpus -D ./share/dictionary -d $(DIR) -o "$@" $< > $@.log 2>&1; then \ + $(call test_record,unit,$(patsubst $(BUILD_DIR)/tests/unit/%,%,$@),PASS,$@.log); \ + else \ + cat $@.log; \ + $(call test_record,unit,$(patsubst $(BUILD_DIR)/tests/unit/%,%,$@),FAIL,$@.log); \ + exit 1; \ + fi $(TEST): @touch $(BUILD_DIR)/tests/$@ diff --git a/src/tests/vmps/all.mk b/src/tests/vmps/all.mk index e8ac8a4a61f10..204ddca6adbf8 100644 --- a/src/tests/vmps/all.mk +++ b/src/tests/vmps/all.mk @@ -34,7 +34,7 @@ $(OUTPUT)/%: $(DIR)/% | $(TEST).radiusd_kill $(TEST).radiusd_start $(eval FOUND := $(patsubst %.txt,%.out,$@)) $(eval ARGV := $(shell grep "#.*ARGV:" $< | cut -f2 -d ':')) ${Q}echo "VMPS-TEST INPUT=$(TARGET) VMPS_ARGV=\"$(ARGV)\"" - ${Q}[ -f $(dir $@)/radiusd.pid ] || exit 1 + ${Q}if ! [ -f $(dir $@)/radiusd.pid ]; then $(call test_record,vmps,$(notdir $@),FAIL,); exit 1; fi ${Q}if ! $(VQCLI) -s 127.0.0.1 -p $(vmps_port) $(ARGV) 1> $(FOUND) 2>&1; then \ echo "FAILED"; \ cat $(FOUND); \ @@ -42,6 +42,7 @@ $(OUTPUT)/%: $(DIR)/% | $(TEST).radiusd_kill $(TEST).radiusd_start $(MAKE) --no-print-directory test.vmps.radiusd_kill; \ echo "RADIUSD: $(RADIUSD_RUN)"; \ echo "VQCLI: $(VQCLI) -s 127.0.0.1 -p $(vmps_port) $(ARGV)"; \ + $(call test_record,vmps,$(notdir $@),FAIL,$(FOUND)); \ exit 1; \ fi # @@ -59,8 +60,10 @@ $(OUTPUT)/%: $(DIR)/% | $(TEST).radiusd_kill $(TEST).radiusd_start diff $(EXPECTED) $(FOUND); \ rm -f $(BUILD_DIR)/tests/test.vmps; \ $(MAKE) --no-print-directory test.vmps.radiusd_kill; \ + $(call test_record,vmps,$(notdir $@),FAIL,$(FOUND)); \ exit 1; \ fi + @$(call test_record,vmps,$(notdir $@),PASS,$(FOUND)) ${Q}touch $@ $(TEST): diff --git a/src/tests/xlat/all.mk b/src/tests/xlat/all.mk index d54a933979338..c65621b6d269b 100644 --- a/src/tests/xlat/all.mk +++ b/src/tests/xlat/all.mk @@ -26,5 +26,7 @@ $(OUTPUT)/%: $(DIR)/% $(TEST_BIN_DIR)/unit_test_module $(DIR)/packet | build.rad cat $@.log; \ echo "./$(TEST_BIN)/unit_test_module -D share/dictionary -d src/tests/xlat/ -r \"$@\" -i $(dir $<)/packet -I \"$<\" -xx "; \ rm -f $(BUILD_DIR)/tests/test.xlat; \ + $(call test_record,xlat,$(notdir $@),FAIL,$@.log); \ exit 1; \ fi + @$(call test_record,xlat,$(notdir $@),PASS,$@.log) From 196a3921569e6d1335c4f075a66e71537d48bd97 Mon Sep 17 00:00:00 2001 From: Terry Burton Date: Fri, 1 May 2026 12:53:09 +0100 Subject: [PATCH 2/2] CI: Capture reproducer, add to summary; open failures; collapse passes --- .github/actions/ci-tests/action.yml | 2 ++ scripts/ci/ci-summary.sh | 33 ++++++++++++++++++++++------- src/tests/auth/all.mk | 6 +++--- src/tests/bin/all.mk | 5 +++-- src/tests/config/all.mk | 3 +-- src/tests/detail/all.mk | 5 +++-- src/tests/dict/all.mk | 15 +++++++------ src/tests/digest/all.mk | 5 +++-- src/tests/eapol_test/all.mk | 5 ++++- src/tests/keywords/all.mk | 3 +-- src/tests/modules/all.mk | 6 +++--- src/tests/multi-server/all.mk | 1 + src/tests/process/all.mk | 2 +- src/tests/radclient/all.mk | 5 +++-- src/tests/radmin/all.mk | 3 ++- src/tests/radsniff/all.mk | 4 +++- src/tests/tacacs/all.mk | 5 +++-- src/tests/trie/test.mk | 8 +++++-- src/tests/unit/all.mk | 4 +++- src/tests/vmps/all.mk | 5 +++-- src/tests/xlat/all.mk | 5 +++-- 21 files changed, 85 insertions(+), 45 deletions(-) diff --git a/.github/actions/ci-tests/action.yml b/.github/actions/ci-tests/action.yml index adb8541426dd0..06e1d41654a07 100644 --- a/.github/actions/ci-tests/action.yml +++ b/.github/actions/ci-tests/action.yml @@ -289,6 +289,8 @@ runs: [ -n "$logf" ] && [ -f "$logf" ] || continue mkdir -p "build/tests/failed/$cat" cp "$logf" "build/tests/failed/$cat/$name.log" + cmdf=${logf%.*}.cmd + [ -f "$cmdf" ] && cp "$cmdf" "build/tests/failed/$cat/$name.cmd" done - name: Upload failed test logs diff --git a/scripts/ci/ci-summary.sh b/scripts/ci/ci-summary.sh index be35fd586e08b..8cab12ba7b5ab 100755 --- a/scripts/ci/ci-summary.sh +++ b/scripts/ci/ci-summary.sh @@ -44,18 +44,27 @@ END { printf("**%d / %d passed**\n\n", total_pass, total) } - # Failure logs as collapsible blocks + # Failure logs as expanded-by-default blocks if (total_fail > 0) { printf("### Failure logs (last 20 lines)\n\n") for (k in seen) { if (status[k] != "FAIL") continue - printf("
%s/%s — %s\n\n", + printf("
%s/%s — %s\n\n", cat[k], name[k], (logf[k] != "" ? logf[k] : "(no log captured)")) if (logf[k] != "") { - printf("```\n") + # Reproducer comes from a side-car .cmd file the recipe wrote + # alongside the log (last extension replaced with .cmd). + cmdf = logf[k]; sub(/\.[^.]+$/, ".cmd", cmdf) + if ((getline reproducer < cmdf) > 0) { + close(cmdf) + printf("\nReproduce locally:\n```\n%s\n```\n\n", reproducer) + } + printf("Full log: `build/tests/failed/%s/%s.log` in the uploaded artifact.\n\n", cat[k], name[k]) + # Embed last 20 lines of the log. n = 0 while ((getline line < logf[k]) > 0) buf[++n % 20] = line close(logf[k]) + printf("```\n") start = (n < 20) ? 1 : n - 19 for (j = start; j <= n; j++) print buf[j % 20] delete buf @@ -89,11 +98,19 @@ END { ct = cp + cf if (cf > 0) printf("### %s (%d/%d, %d failed)\n\n", c, cp, ct, cf) else printf("### %s (%d/%d)\n\n", c, cp, ct) - printf("| Test | Status |\n|---|---|\n") - # FAIL rows first, then PASS, each in test-execution order. - for (k in seen) if (cat[k] == c && status[k] == "FAIL") emit(k) - for (k in seen) if (cat[k] == c && status[k] == "PASS") emit(k) - printf("\n") + # FAIL rows visible at the top. PASS rows hidden in a collapsed + # block so reviewers do not have to scroll past walls of green. + if (cf > 0) { + printf("| Test | Status |\n|---|---|\n") + for (k in seen) if (cat[k] == c && status[k] == "FAIL") emit(k) + printf("\n") + } + if (cp > 0) { + printf("
%d passing test%s\n\n", cp, (cp == 1 ? "" : "s")) + printf("| Test | Status |\n|---|---|\n") + for (k in seen) if (cat[k] == c && status[k] == "PASS") emit(k) + printf("\n
\n\n") + } } } diff --git a/src/tests/auth/all.mk b/src/tests/auth/all.mk index 5fd622bb786c2..d0bda6a289532 100644 --- a/src/tests/auth/all.mk +++ b/src/tests/auth/all.mk @@ -93,12 +93,13 @@ AUTH_LIBS := $(addsuffix .la,$(addprefix rlm_,$(AUTH_MODULES))) # ERROR line in the input. # $(OUTPUT)/%: $(DIR)/% $(OUTPUT)/%.attrs $(TEST_BIN_DIR)/unit_test_module | $(AUTH_RADDB) $(AUTH_LIBS) build.raddb + $(eval CMD := TESTDIR=$(notdir $@) $(TEST_BIN)/unit_test_module -D share/dictionary -d src/tests/auth/ -i "$@.attrs" -f "$@.attrs" -r "$@" -xx) @echo "AUTH-TEST $(notdir $@)" - ${Q}if ! TESTDIR=$(notdir $@) $(TEST_BIN)/unit_test_module -D share/dictionary -d src/tests/auth/ -i "$@.attrs" -f "$@.attrs" -r "$@" -xx > "$@.log" 2>&1 || ! test -f "$@"; then \ + @printf '%s\n' '$(CMD)' > $@.cmd + ${Q}if ! $(CMD) > "$@.log" 2>&1 || ! test -f "$@"; then \ if ! grep ERROR $< 2>&1 > /dev/null; then \ cat $@.log; \ echo "# $@.log"; \ - echo "TESTDIR=$(notdir $@) $(TEST_BIN)/unit_test_module -D share/dictionary -d src/tests/auth/ -i \"$@.attrs\" -f \"$@.attrs\" -r \"$@\" -xxx > \"$@.log\" 2>&1"; \ rm -f $(BUILD_DIR)/tests/test.auth; \ $(call test_record,auth,$(notdir $@),FAIL,$@.log); \ exit 1; \ @@ -108,7 +109,6 @@ $(OUTPUT)/%: $(DIR)/% $(OUTPUT)/%.attrs $(TEST_BIN_DIR)/unit_test_module | $(AUT if [ "$$EXPECTED" != "$$FOUND" ]; then \ cat $@.log; \ echo "# $@.log"; \ - echo "TESTDIR=$(notdir $@) $(TEST_BIN)/unit_test_module -D share/dictionary -d src/tests/auth/ -i \"$@.attrs\" -f \"$@.attrs\" -r \"$@\" -xxx > \"$@.log\" 2>&1"; \ rm -f $(BUILD_DIR)/tests/test.auth; \ $(call test_record,auth,$(notdir $@),FAIL,$@.log); \ exit 1; \ diff --git a/src/tests/bin/all.mk b/src/tests/bin/all.mk index bb438e1a9dae7..f1d65d257c6ad 100644 --- a/src/tests/bin/all.mk +++ b/src/tests/bin/all.mk @@ -52,11 +52,12 @@ unit_test_module.ARGS = -h # that we're running # $(BUILD_DIR)/tests/bin/%: $(BUILD_DIR)/bin/local/% + $(eval CMD := $(TEST_BIN)/$(notdir $<) $($(notdir $@).ARGS)) @echo "BIN-TEST $(notdir $@)" - ${Q}if ! $(TEST_BIN)/$(notdir $<) $($(notdir $@).ARGS) > $@.log 2>&1; then \ + @printf '%s\n' '$(CMD)' > $@.cmd + ${Q}if ! $(CMD) > $@.log 2>&1; then \ echo LOG in $@.log; \ cat $@.log; \ - echo $(TEST_BIN)/$(notdir $<) $($(notdir $@).ARGS); \ $(call test_record,bin,$(notdir $@),FAIL,$@.log); \ exit 1; \ fi diff --git a/src/tests/config/all.mk b/src/tests/config/all.mk index 3bf23f76a1046..4e488e40c83a4 100644 --- a/src/tests/config/all.mk +++ b/src/tests/config/all.mk @@ -61,11 +61,11 @@ $(OUTPUT)/%: $(DIR)/% $(TEST_BIN_DIR)/unit_test_module | $(CONFIG_LIBS) @mkdir -p $(dir $@) $(eval CMD:=CONFIG_FILE=$(notdir $@) $(TEST_BIN)/unit_test_module $(UNIT_TEST_CONFIG_ARGS.$(subst -,_,$(notdir $@))) -D share/dictionary -d src/tests/config/ -xxC ) @echo "CONFIG-TEST $(notdir $@)" + @printf '%s\n' '$(CMD)' > $@.cmd ${Q}if ! $(CMD) > "$@.log" 2>&1; then \ if ! grep ERROR $< 2>&1 > /dev/null; then \ cat $@.log; \ echo "# $@.log"; \ - echo $(CMD); \ rm -f $(BUILD_DIR)/tests/test.config; \ $(call test_record,config,$(notdir $@),FAIL,$@.log); \ exit 1; \ @@ -75,7 +75,6 @@ $(OUTPUT)/%: $(DIR)/% $(TEST_BIN_DIR)/unit_test_module | $(CONFIG_LIBS) if [ "$$EXPECTED" != "$$FOUND" ]; then \ cat $@.log; \ echo "# $@.log"; \ - echo $(CMD); \ rm -f $(BUILD_DIR)/tests/test.config; \ $(call test_record,config,$(notdir $@),FAIL,$@.log); \ exit 1; \ diff --git a/src/tests/detail/all.mk b/src/tests/detail/all.mk index ee854480f7ae8..c36d879c397ec 100644 --- a/src/tests/detail/all.mk +++ b/src/tests/detail/all.mk @@ -20,9 +20,10 @@ $(OUTPUT)/%: $(DIR)/% $(addprefix ${BUILD_DIR}/lib/,proto_detail.la proto_detail ${Q}echo "DETAIL $(notdir $<)" ${Q}cp $< $(dir $@)/detail.txt ${Q}rm -f $(dir $@)/processed - ${Q}if ! $(TEST_BIN)/radiusd -d $(DIR)/config -D ${top_srcdir}/share/dictionary -X > $@.log; then \ + $(eval CMD := $(TEST_BIN)/radiusd -d $(DIR)/config -D ${top_srcdir}/share/dictionary -X) + @printf 'cp %s %s/detail.txt && %s\n' '$<' '$(dir $@)' '$(CMD)' > $@.cmd + ${Q}if ! $(CMD) > $@.log; then \ tail $@.log; \ - echo "cp $< $(dir $@)/detail.txt; $(TEST_BIN)/radiusd -d $(DIR)/config -D ${top_srcdir}/share/dictionary -X "; \ $(call test_record,detail,$(notdir $@),FAIL,$@.log); \ exit 1; \ fi diff --git a/src/tests/dict/all.mk b/src/tests/dict/all.mk index bb877cb9e6cdf..b69319c6d3c19 100644 --- a/src/tests/dict/all.mk +++ b/src/tests/dict/all.mk @@ -16,15 +16,16 @@ $(eval $(call TEST_BOOTSTRAP)) # "foo_dir" directory, and copy "foo" into "foo_dir/dictionary" # $(OUTPUT)/%.dict: $(DIR)/%.dict $(TEST_BIN_DIR)/unit_test_attribute + $(eval CMD := $(TEST_BIN)/unit_test_attribute -D $(top_srcdir)/share/dictionary -o "$@" "$@.txt") @echo "DICT-TEST $(notdir $@)" @cp $< $@ @cp src/tests/dict/base.txt $@.txt @echo "load-dictionary $(top_srcdir)/$<" >> $@.txt - ${Q}if ! $(TEST_BIN)/unit_test_attribute -D $(top_srcdir)/share/dictionary -o "$@" "$@.txt" > "$@.log" 2>&1 || ! test -f "$@"; then \ + @printf '%s\n' '$(CMD)' > $@.cmd + ${Q}if ! $(CMD) > "$@.log" 2>&1 || ! test -f "$@"; then \ rm -f $@; \ cat "$@.log"; \ echo "# $@.log"; \ - echo "$(TEST_BIN)/unit_test_attribute -D $(top_srcdir)/share/dictionary -o "$@" -xx '$@.txt'"; \ $(call test_record,dict,$(notdir $@),FAIL,$@.log); \ exit 1; \ fi @@ -36,15 +37,16 @@ $(OUTPUT)/%.dict: $(DIR)/%.dict $(TEST_BIN_DIR)/unit_test_attribute # "foo_dir" directory, and copy "foo" into "foo_dir/dictionary" # $(OUTPUT)/%.dict: $(DIR)/%.dict $(TEST_BIN_DIR)/unit_test_attribute + $(eval CMD := $(TEST_BIN)/unit_test_attribute -D $(top_srcdir)/share/dictionary -o "$@" "$@.txt") @echo "DICT-TEST $(notdir $@)" @cp $< $@ @cp src/tests/dict/base.txt $@.txt @echo "load-dictionary $(top_srcdir)/$<" >> $@.txt - ${Q}if ! $(TEST_BIN)/unit_test_attribute -D $(top_srcdir)/share/dictionary -o "$@" "$@.txt" > "$@.log" 2>&1 || ! test -f "$@"; then \ + @printf '%s\n' '$(CMD)' > $@.cmd + ${Q}if ! $(CMD) > "$@.log" 2>&1 || ! test -f "$@"; then \ rm -f $@; \ cat "$@.log"; \ echo "# $@.log"; \ - echo "$(TEST_BIN)/unit_test_attribute -D $(top_srcdir)/share/dictionary -o "$@" -xx '$@.txt'"; \ $(call test_record,dict,$(notdir $@),FAIL,$@.log); \ exit 1; \ fi @@ -60,15 +62,16 @@ $(OUTPUT)/%.dict: $(DIR)/%.dict $(TEST_BIN_DIR)/unit_test_attribute # See if the current output and the expected output are the same. # $(OUTPUT)/%.error: $(DIR)/%.error $(TEST_BIN_DIR)/unit_test_attribute + $(eval CMD := $(TEST_BIN)/unit_test_attribute -D $(top_srcdir)/share/dictionary "$@.txt") @echo "DICT-TEST $(notdir $@)" @cp $< $@ @cp src/tests/dict/base.txt $@.txt @echo "load-dictionary $(top_srcdir)/$<" >> $@.txt - ${Q}if $(TEST_BIN)/unit_test_attribute -D $(top_srcdir)/share/dictionary "$@.txt" > "$@.log" 2>&1 ; then \ + @printf '%s\n' '$(CMD)' > $@.cmd + ${Q}if $(CMD) > "$@.log" 2>&1 ; then \ rm -f $@; \ cat "$@.log"; \ echo "# $@.log"; \ - echo "$(TEST_BIN)/unit_test_attribute -D $(top_srcdir)/share/dictionary -xx '$@.txt'"; \ $(call test_record,dict,$(notdir $@),FAIL,$@.log); \ exit 1; \ fi diff --git a/src/tests/digest/all.mk b/src/tests/digest/all.mk index 557046a45a416..c3020794ca8de 100644 --- a/src/tests/digest/all.mk +++ b/src/tests/digest/all.mk @@ -35,19 +35,20 @@ $(eval $(call RADIUSD_SERVICE,digest,$(OUTPUT))) $(OUTPUT)/%: $(DIR)/% | $(TEST).radiusd_kill $(TEST).radiusd_start ${Q} [ -f $(dir $@)/radiusd.pid ] || exit 1 $(eval TARGET := $(patsubst %.txt,%,$(notdir $@)$(E))) + $(eval CMD := $(TEST_BIN)/radclient -f $@.request -xF -d src/tests/digest/config -D share/dictionary 127.0.0.1:$(digest_port) auth $(SECRET)) ${Q}for _num in $$(sed '/^#.*TESTS/!d; s/.*TESTS//g' $<); do \ echo "DIGEST-TEST $(TARGET)_$${_num}"; \ cp -f $< $@.request; \ echo "Vendor-Specific.Test.Test-Name = \"$(TARGET)\"" >> $@.request; \ echo "Vendor-Specific.Test.Test-Number = \"$${_num}\"" >> $@.request; \ - if ! $(TEST_BIN)/radclient -f $@.request -xF -d src/tests/digest/config -D share/dictionary 127.0.0.1:$(digest_port) auth $(SECRET) > $@.out; then \ + printf '%s\n' '$(CMD)' > $@.cmd; \ + if ! $(CMD) > $@.out; then \ cat $(DIGEST_RADIUS_LOG); \ echo "FAILED $(TARGET)"; \ cat $@.out; \ rm -f $(BUILD_DIR)/tests/test.digest; \ $(MAKE) --no-print-directory test.digest.radiusd_kill; \ echo "RADIUSD: $(RADIUSD_RUN)"; \ - echo "RADCLIENT: $(TEST_BIN)/radclient -f $@.request -xF -d src/tests/digest/config -D share/dictionary 127.0.0.1:$(digest_port) auth $(SECRET)"; \ $(call test_record,digest,$(TARGET)_$${_num},FAIL,$@.out); \ exit 1; \ fi; \ diff --git a/src/tests/eapol_test/all.mk b/src/tests/eapol_test/all.mk index 8873488539a6b..5dd854043dac2 100644 --- a/src/tests/eapol_test/all.mk +++ b/src/tests/eapol_test/all.mk @@ -115,14 +115,17 @@ test.eap.check: $(OUTPUT) $(GENERATED_CERT_FILES) # $(OUTPUT)/%.ok: $(DIR)/%.conf $(if $(POST_INSTALL_MAKEFILE_ARG),,$(BUILD_DIR)/lib/libfreeradius-server.la $(BUILD_DIR)/lib/libfreeradius-util.la) | $(GENERATED_CERT_FILES) $(eval EAPOL_TEST_LOG := $(patsubst %.ok,%.log,$@)) + $(eval EAPOL_TEST_CMD_FILE := $(patsubst %.ok,%.cmd,$@)) $(eval METHOD := $(notdir $(subst _,-,$(patsubst %.conf,%,$<)))) $(eval KEY := $(shell grep key_mgmt=NONE $< | sed 's/key_mgmt=NONE/-n/')) $(eval RADIUS_LOG := $(dir $@)/test.$(METHOD)/radiusd.log) $(eval TEST_PORT := $($(METHOD)_port)) + $(eval CMD := $(EAPOL_TEST) -t $(EAPOL_TEST_TIMEOUT) -c $< -p $(TEST_PORT) -s $(SECRET) $(KEY)) @echo "EAPOL-TEST $(METHOD)" + @printf '%s\n' '$(CMD)' > $(EAPOL_TEST_CMD_FILE) ${Q}$(MAKE) $(POST_INSTALL_MAKEFILE_ARG) --no-print-directory test.$(METHOD).radiusd_kill ${Q}$(MAKE) $(POST_INSTALL_MAKEFILE_ARG) --no-print-directory test.$(METHOD).radiusd_start $(POST_INSTALL_RADIUSD_BIN_ARG) - ${Q}if ! $(EAPOL_TEST) -t $(EAPOL_TEST_TIMEOUT) -c $< -p $(TEST_PORT) -s $(SECRET) $(KEY) > $(EAPOL_TEST_LOG) 2>&1; then \ + ${Q}if ! $(CMD) > $(EAPOL_TEST_LOG) 2>&1; then \ echo "Last entries in supplicant log ($(EAPOL_TEST_LOG)):"; \ tail -n 40 "$(EAPOL_TEST_LOG)"; \ echo "--------------------------------------------------"; \ diff --git a/src/tests/keywords/all.mk b/src/tests/keywords/all.mk index ea80efff968eb..5f51e29c42eae 100644 --- a/src/tests/keywords/all.mk +++ b/src/tests/keywords/all.mk @@ -140,11 +140,11 @@ export IF_ENV_CONDITION := !test || (test != 'if-env') $(OUTPUT)/%: $(DIR)/% $(TEST_BIN_DIR)/unit_test_module | $(KEYWORD_RADDB) $(KEYWORD_LIBS) build.raddb rlm_test.la rlm_csv.la rlm_unpack.la $(eval CMD:=KEYWORD=$(notdir $@) $(TEST_BIN)/unit_test_module $(UNIT_TEST_KEYWORD_ARGS.$(subst -,_,$(notdir $@))) -D share/dictionary -d src/tests/keywords/ -i "$@.attrs" -f "$@.attrs" -r "$@" -xx ) @echo "KEYWORD-TEST $(notdir $@)" + @printf '%s\n' '$(CMD)' > $@.cmd ${Q}if ! $(CMD) > "$@.log" 2>&1 || ! test -f "$@"; then \ if ! grep ERROR $< 2>&1 > /dev/null; then \ cat $@.log; \ echo "# $@.log"; \ - echo $(CMD); \ rm -f $(BUILD_DIR)/tests/test.keywords; \ $(call test_record,keywords,$(notdir $@),FAIL,$@.log); \ exit 1; \ @@ -154,7 +154,6 @@ $(OUTPUT)/%: $(DIR)/% $(TEST_BIN_DIR)/unit_test_module | $(KEYWORD_RADDB) $(KEYW if [ "$$EXPECTED" != "$$FOUND" ]; then \ cat $@.log; \ echo "# $@.log"; \ - echo $(CMD); \ rm -f $(BUILD_DIR)/tests/test.keywords; \ $(call test_record,keywords,$(notdir $@),FAIL,$@.log); \ exit 1; \ diff --git a/src/tests/modules/all.mk b/src/tests/modules/all.mk index 22b0d8436b34e..d642881cc5729 100644 --- a/src/tests/modules/all.mk +++ b/src/tests/modules/all.mk @@ -154,10 +154,12 @@ $(foreach x, $(filter cache_%,$(FILES)), $(eval $$(OUTPUT.$(TEST))/$x: $(BUILD_D $(OUTPUT)/%: TEST=$(lastword $(subst /, ,$(dir $@))) $(basename $(notdir $@)) $(OUTPUT)/%: $(DIR)/%.unlang $(TEST_BIN_DIR)/unit_test_module | build.raddb + $(eval CMD := MODULE_TEST_DIR=$(dir $<) MODULE_TEST_UNLANG=$< TEST="$(TEST)" OUTPUT_DIR=$(dir $@) $(TEST_BIN)/unit_test_module -D share/dictionary -d src/tests/modules/ -i "$@.attrs" -f "$@.attrs" -r "$@" -xxx) @echo "MODULE-TEST $(TEST)" ${Q}mkdir -p $(dir $@) ${Q}cp $(if $(wildcard $(basename $<).attrs),$(basename $<).attrs,src/tests/modules/default-input.attrs) $@.attrs - ${Q}if ! MODULE_TEST_DIR=$(dir $<) MODULE_TEST_UNLANG=$< TEST="$(TEST)" OUTPUT_DIR=$(dir $@) $(TEST_BIN)/unit_test_module -D share/dictionary -d src/tests/modules/ -i "$@.attrs" -f "$@.attrs" -r "$@" -xxx > "$@.log" 2>&1 || ! test -f "$@"; then \ + @printf '%s\n' '$(CMD)' > $@.cmd + ${Q}if ! $(CMD) > "$@.log" 2>&1 || ! test -f "$@"; then \ if ! grep ERROR $< 2>&1 > /dev/null; then \ if grep 'LeakSanitizer has encountered a fatal error' $@.log 2>&1 > /dev/null; then \ echo "MODULE-TEST $(TEST) - ignoring LeakSanitizer fatal error."; \ @@ -165,7 +167,6 @@ $(OUTPUT)/%: $(DIR)/%.unlang $(TEST_BIN_DIR)/unit_test_module | build.raddb fi; \ cat "$@.log"; \ echo "# $@.log"; \ - echo "MODULE_TEST_DIR=$(dir $<) MODULE_TEST_UNLANG=$< OUTPUT_DIR=$(dir $@) $(TEST_BIN)/unit_test_module -D share/dictionary -d src/tests/modules/ -i \"$@.attrs\" -f \"$@.attrs\" -r \"$@\" -xx"; \ $(call test_record,modules,$(lastword $(subst /, ,$(dir $@)))/$(basename $(notdir $@)),FAIL,$@.log); \ exit 1; \ fi; \ @@ -174,7 +175,6 @@ $(OUTPUT)/%: $(DIR)/%.unlang $(TEST_BIN_DIR)/unit_test_module | build.raddb if [ "$$EXPECTED" != "$$FOUND" ]; then \ cat "$@.log"; \ echo "# $@.log"; \ - echo "MODULE_TEST_DIR=$(dir $<) MODULE_TEST_UNLANG=$< OUTPUT_DIR=$(dir $@) $(TEST_BIN)/unit_test_module -D share/dictionary -d src/tests/modules/ -i \"$@.attrs\" -f \"$@.attrs\" -r \"$@\" -xx"; \ $(call test_record,modules,$(lastword $(subst /, ,$(dir $@)))/$(basename $(notdir $@)),FAIL,$@.log); \ exit 1; \ else \ diff --git a/src/tests/multi-server/all.mk b/src/tests/multi-server/all.mk index c6c3ab271126b..28252d9ab38fc 100644 --- a/src/tests/multi-server/all.mk +++ b/src/tests/multi-server/all.mk @@ -144,6 +144,7 @@ test.multi-server.${1}.${2}: $$(TEST_MULTI_SERVER_RENDERED.${1}.${2}) $$(eval CMD := cd $(TEST_MULTI_SERVER_FRAMEWORK_DIR) && . .venv/bin/activate && DATA_PATH="${4}" python3 -m src.multi_server_test $(TEST_MULTI_SERVER_FLAGS) --project-name "${1}-${2}" --compose "${4}/environment.yml" --test "${4}/template.yml" --use-files --listener-dir "${4}/listener" --log-dir "${4}/logs" --output "${4}/logs/result.log") ${Q}mkdir -p "${4}/logs" "${4}/listener" ${Q}echo "MULTI-SERVER-TEST test.multi-server.${1}.${2}" + @printf '%s\n' '$$(CMD)' > "${4}/logs/stderr.cmd" ${Q}$$(CMD) > "${4}/logs/stdout.log" 2> "${4}/logs/stderr.log" || \ { \ echo "FAILED: test.multi-server.${1}.${2}"; \ diff --git a/src/tests/process/all.mk b/src/tests/process/all.mk index b9c125ce282fa..2f34941de1120 100644 --- a/src/tests/process/all.mk +++ b/src/tests/process/all.mk @@ -98,10 +98,10 @@ $(OUTPUT)/%: $(DIR)/% $(PROCESS_DICT) $(TEST_BIN_DIR)/unit_test_module $(DIR)/un $(eval export TEST_PORT:=$(TEST_PORT)) @echo PROCESS-TEST $(PROTOCOL_NAME) $(notdir $@) $(Q)mkdir -p $(dir $@) + @printf '%s\n' '$(CMD)' > $@.cmd $(Q)if ! $(CMD) > "$@.log" 2>&1 || ! test -f "$@"; then \ cat $@.log; \ echo "# $@.log"; \ - echo $(CMD); \ $(call test_record,process,$(PROTOCOL_NAME)/$(notdir $@),FAIL,$@.log); \ exit 1; \ fi diff --git a/src/tests/radclient/all.mk b/src/tests/radclient/all.mk index 4ffcf47e50080..73e4a3871422d 100644 --- a/src/tests/radclient/all.mk +++ b/src/tests/radclient/all.mk @@ -53,16 +53,17 @@ $(OUTPUT)/%: $(DIR)/% $(BUILD_DIR)/bin/local/$(RADCLIENT) $(BUILD_DIR)/lib/local $(eval IGNORE_ERROR := $(shell grep -q "#.*IGNORE_ERROR:.*1" $< && echo 1 || echo 0)) $(eval RADCLIENT_CLIENT_PORT := $(shell echo $$(($(RADCLIENT_CLIENT_PORT)+1)))) + $(eval CMD := $(TEST_BIN)/$(RADCLIENT) $(ARGV) -C $(RADCLIENT_CLIENT_PORT) -f $< -d src/tests/radclient/config -D share/dictionary 127.0.0.1:$(radclient_port) $(TYPE) $(SECRET)) ${Q}echo "RADCLIENT-TEST INPUT=$(TARGET) ARGV=\"$(ARGV)\"" + @printf '%s\n' '$(CMD)' > $(basename $(FOUND)).cmd ${Q}if ! [ -f $(dir $@)/radiusd.pid ]; then $(call test_record,radclient,$(notdir $@),FAIL,); exit 1; fi - ${Q}if ! $(TEST_BIN)/$(RADCLIENT) $(ARGV) -C $(RADCLIENT_CLIENT_PORT) -f $< -d src/tests/radclient/config -D share/dictionary 127.0.0.1:$(radclient_port) $(TYPE) $(SECRET) 1> $(FOUND) 2>&1; then \ + ${Q}if ! $(CMD) 1> $(FOUND) 2>&1; then \ if [ "$(IGNORE_ERROR)" != "1" ]; then \ echo "FAILED"; \ cat $(FOUND); \ rm -f $(BUILD_DIR)/tests/test.radclient; \ $(MAKE) --no-print-directory test.radclient.radiusd_kill; \ echo "RADIUSD: $(RADIUSD_RUN)"; \ - echo "RADCLIENT: $(TEST_BIN)/$(RADCLIENT) $(ARGV) -C $(RADCLIENT_CLIENT_PORT) -f $< -xF -d src/tests/radclient/config -D share/dictionary 127.0.0.1:$(radclient_port) $(TYPE) $(SECRET)"; \ $(call test_record,radclient,$(notdir $@),FAIL,$(FOUND)); \ exit 1; \ fi; \ diff --git a/src/tests/radmin/all.mk b/src/tests/radmin/all.mk index 3f701323a4c0b..d37dd213022b1 100644 --- a/src/tests/radmin/all.mk +++ b/src/tests/radmin/all.mk @@ -58,13 +58,14 @@ $(OUTPUT)/%: $(DIR)/% | $(TEST).radiusd_kill $(TEST).radiusd_start $(eval EXPECTED := $(patsubst %.txt,%.out,$<)) $(eval FOUND := $(patsubst %.txt,%.out,$@)) $(eval TARGET := $(patsubst %.txt,%,$(notdir $@)$(E))) + $(eval CMD := $(TEST_BIN)/radmin -q -f $(RADMIN_SOCKET_FILE) < $< > $(FOUND)) + @printf '%s\n' '$(CMD)' > $(basename $(FOUND)).cmd ${Q}if ! $(TEST_BIN)/radmin -q -f $(RADMIN_SOCKET_FILE) < $< > $(FOUND) 2>&1; then\ echo "--------------------------------------------------"; \ tail -n 20 "$(RADMIN_RADIUS_LOG)"; \ echo "Last entries in server log ($(RADMIN_RADIUS_LOG)):"; \ echo "--------------------------------------------------"; \ echo "RADIUSD: $(RADIUSD_RUN)"; \ - echo "RADMIN : $(TEST_BIN)/radmin -q -f $(RADMIN_SOCKET_FILE) < $< > $(FOUND)"; \ rm -f $(BUILD_DIR)/tests/test.radmin; \ $(MAKE) --no-print-directory test.radmin.radiusd_kill; \ $(call test_record,radmin,$(notdir $@),FAIL,$(FOUND)); \ diff --git a/src/tests/radsniff/all.mk b/src/tests/radsniff/all.mk index b7f4feee9915f..de308cd8a4b1b 100644 --- a/src/tests/radsniff/all.mk +++ b/src/tests/radsniff/all.mk @@ -47,11 +47,13 @@ $(OUTPUT)/%.txt: $(DIR)/%.txt $(TEST_BIN_DIR)/radsniff $(PCAP_IN) $(eval EXPECTED := $<) $(eval ARGV := $(shell grep "^#.*ARGV:" $< | cut -f2 -d ':')) + $(eval CMD := TZ='UTC' $(TEST_BIN)/radsniff $(ARGV) -I $(PCAP_IN) -D share/dictionary) ${Q}echo "RADSNIFF-TEST INPUT=$(TARGET) ARGV=\"$(ARGV)\"" + @printf '%s\n' '$(CMD)' > $(basename $(FOUND)).cmd # # We need that 'TZ=UTC ...' to libpcap pass the same timestamp in anywhere. # - ${Q}if ! TZ='UTC' $(TEST_BIN)/radsniff $(ARGV) -I $(PCAP_IN) -D share/dictionary 1> $(FOUND); then \ + ${Q}if ! $(CMD) 1> $(FOUND); then \ echo "FAILED"; \ cat $(FOUND); \ echo "RADSNIFF: TZ='UTC' $(TEST_BIN)/radsniff $(ARGV) -I $(PCAP_IN) -D share/dictionary" -xx; \ diff --git a/src/tests/tacacs/all.mk b/src/tests/tacacs/all.mk index 0de404151ff12..393cd91acdfae 100644 --- a/src/tests/tacacs/all.mk +++ b/src/tests/tacacs/all.mk @@ -52,15 +52,16 @@ $(OUTPUT)/%: $(DIR)/% $(BUILD_DIR)/lib/libfreeradius-tacacs.la $(BUILD_DIR)/lib/ $(eval EXPECTED := $(patsubst %.txt,%.out,$<)) $(eval FOUND := $(patsubst %.txt,%.out,$@)) $(eval ARGV := $(shell grep "#.*ARGV:" $< | cut -f2 -d ':')) + $(eval CMD := $(TACCLIENT) --return-0-if-failed -v -k $(SECRET) -p $(tacacs_port) -H localhost -r 192.168.69.1 -P pegapilha/0 --timeout 2 $(ARGV)) ${Q}echo "TACACS-TEST INPUT=$(TARGET) TACACS_ARGV=\"$(ARGV)\"" + @printf '%s\n' '$(CMD)' > $(basename $(FOUND)).cmd ${Q}if ! [ -f $(dir $@)/radiusd.pid ]; then $(call test_record,tacacs,$(notdir $@),FAIL,); exit 1; fi - ${Q}if ! $(TACCLIENT) --return-0-if-failed -v -k $(SECRET) -p $(tacacs_port) -H localhost -r 192.168.69.1 -P pegapilha/0 --timeout 2 $(ARGV) 1> $(FOUND) 2>&1; then \ + ${Q}if ! $(CMD) 1> $(FOUND) 2>&1; then \ echo "FAILED"; \ cat $(FOUND); \ rm -f $(BUILD_DIR)/tests/test.tacacs; \ $(MAKE) --no-print-directory test.tacacs.radiusd_kill; \ echo "RADIUSD: $(RADIUSD_RUN)"; \ - echo "TACCLIENT: $(TACCLIENT) --return-0-if-failed -v -k $(SECRET) -p $(tacacs_port) -H localhost -r 192.168.69.1 -P pegapilha/0 --timeout 2 $(ARGV)"; \ $(call test_record,tacacs,$(notdir $@),FAIL,$(FOUND)); \ exit 1; \ fi diff --git a/src/tests/trie/test.mk b/src/tests/trie/test.mk index 4cfbbed986303..acc6fdfdfabc9 100644 --- a/src/tests/trie/test.mk +++ b/src/tests/trie/test.mk @@ -11,8 +11,10 @@ $(BUILD_DIR)/tests/trie: ${Q}mkdir -p $@ $(BUILD_DIR)/tests/trie/trie-%: $(DIR)/% $(TEST_BIN_DIR)/trie | $(BUILD_DIR)/tests/trie + $(eval CMD := $(TEST_BIN)/trie $^) @echo TRIE-TEST $(notdir $@) - @if $(TEST_BIN)/trie $^ > $@ 2> $@.log; then \ + @printf '%s\n' '$(CMD)' > $@.cmd + @if $(CMD) > $@ 2> $@.log; then \ $(call test_record,trie,$(notdir $@),PASS,$@.log); \ else \ cat $@.log; \ @@ -22,8 +24,10 @@ $(BUILD_DIR)/tests/trie/trie-%: $(DIR)/% $(TEST_BIN_DIR)/trie | $(BUILD_DIR)/tes fi $(BUILD_DIR)/tests/trie/nopc-%: $(DIR)/% $(TEST_BIN_DIR)/nopc | $(BUILD_DIR)/tests/trie + $(eval CMD := $(TEST_BIN)/nopc $^) @echo TRIE-NO-PC-TEST $(notdir $@) - @if $(TEST_BIN)/nopc $^ > $@ 2> $@.log; then \ + @printf '%s\n' '$(CMD)' > $@.cmd + @if $(CMD) > $@ 2> $@.log; then \ $(call test_record,trie,$(notdir $@),PASS,$@.log); \ else \ cat $@.log; \ diff --git a/src/tests/unit/all.mk b/src/tests/unit/all.mk index 58b23ae20c561..746032ea89a04 100644 --- a/src/tests/unit/all.mk +++ b/src/tests/unit/all.mk @@ -94,9 +94,11 @@ $(filter $(BUILD_DIR)/tests/unit/purify/%,$(FILES.$(TEST))): PURIFY=-p # $(OUTPUT)/%: $(DIR)/% $(TEST_BIN_DIR)/unit_test_attribute $(eval DIR:=${top_srcdir}/src/tests/unit) + $(eval CMD := $(TEST_BIN)/unit_test_attribute $(PURIFY) $(REWRITE_FLAGS) -F ./src/tests/fuzzer-corpus -D ./share/dictionary -d $(DIR) -o "$@" $<) $(eval export UNIT_TEST_ATTRIBUTE:=TZ=GMT $(TEST_BIN_NO_TIMEOUT)/unit_test_attribute $(PURIFY) -F ./src/tests/fuzzer-corpus -D ./share/dictionary -d $(DIR) -o \"$@\" $<) ${Q}mkdir -p $(dir $@) - ${Q}if $(TEST_BIN)/unit_test_attribute $(PURIFY) $(REWRITE_FLAGS) -F ./src/tests/fuzzer-corpus -D ./share/dictionary -d $(DIR) -o "$@" $< > $@.log 2>&1; then \ + @printf '%s\n' '$(CMD)' > $@.cmd + ${Q}if $(CMD) > $@.log 2>&1; then \ $(call test_record,unit,$(patsubst $(BUILD_DIR)/tests/unit/%,%,$@),PASS,$@.log); \ else \ cat $@.log; \ diff --git a/src/tests/vmps/all.mk b/src/tests/vmps/all.mk index 204ddca6adbf8..0dfddaef7969d 100644 --- a/src/tests/vmps/all.mk +++ b/src/tests/vmps/all.mk @@ -33,15 +33,16 @@ $(OUTPUT)/%: $(DIR)/% | $(TEST).radiusd_kill $(TEST).radiusd_start $(eval EXPECTED := $(patsubst %.txt,%.out,$<)) $(eval FOUND := $(patsubst %.txt,%.out,$@)) $(eval ARGV := $(shell grep "#.*ARGV:" $< | cut -f2 -d ':')) + $(eval CMD := $(VQCLI) -s 127.0.0.1 -p $(vmps_port) $(ARGV)) ${Q}echo "VMPS-TEST INPUT=$(TARGET) VMPS_ARGV=\"$(ARGV)\"" + @printf '%s\n' '$(CMD)' > $(basename $(FOUND)).cmd ${Q}if ! [ -f $(dir $@)/radiusd.pid ]; then $(call test_record,vmps,$(notdir $@),FAIL,); exit 1; fi - ${Q}if ! $(VQCLI) -s 127.0.0.1 -p $(vmps_port) $(ARGV) 1> $(FOUND) 2>&1; then \ + ${Q}if ! $(CMD) 1> $(FOUND) 2>&1; then \ echo "FAILED"; \ cat $(FOUND); \ rm -f $(BUILD_DIR)/tests/test.vmps; \ $(MAKE) --no-print-directory test.vmps.radiusd_kill; \ echo "RADIUSD: $(RADIUSD_RUN)"; \ - echo "VQCLI: $(VQCLI) -s 127.0.0.1 -p $(vmps_port) $(ARGV)"; \ $(call test_record,vmps,$(notdir $@),FAIL,$(FOUND)); \ exit 1; \ fi diff --git a/src/tests/xlat/all.mk b/src/tests/xlat/all.mk index c65621b6d269b..a9be0488c5fae 100644 --- a/src/tests/xlat/all.mk +++ b/src/tests/xlat/all.mk @@ -21,10 +21,11 @@ $(eval $(call TEST_BOOTSTRAP)) # And the actual script to run each test. # $(OUTPUT)/%: $(DIR)/% $(TEST_BIN_DIR)/unit_test_module $(DIR)/packet | build.raddb + $(eval CMD := $(TEST_BIN)/unit_test_module -D share/dictionary -d src/tests/xlat/ -r "$@" -i $(dir $<)/packet -I "$<" -xx) @echo "XLAT-TEST $(notdir $@)" - ${Q}if ! $(TEST_BIN)/unit_test_module -D share/dictionary -d src/tests/xlat/ -r "$@" -i $(dir $<)/packet -I "$<" -xx > "$@.log" 2>&1 || ! test -f "$@"; then \ + @printf '%s\n' '$(CMD)' > $@.cmd + ${Q}if ! $(CMD) > "$@.log" 2>&1 || ! test -f "$@"; then \ cat $@.log; \ - echo "./$(TEST_BIN)/unit_test_module -D share/dictionary -d src/tests/xlat/ -r \"$@\" -i $(dir $<)/packet -I \"$<\" -xx "; \ rm -f $(BUILD_DIR)/tests/test.xlat; \ $(call test_record,xlat,$(notdir $@),FAIL,$@.log); \ exit 1; \