diff --git a/.github/actions/ci-tests/action.yml b/.github/actions/ci-tests/action.yml
index b1d2d04c68f6..06e1d41654a0 100644
--- a/.github/actions/ci-tests/action.yml
+++ b/.github/actions/ci-tests/action.yml
@@ -237,10 +237,13 @@ runs:
run: |
[ -d /opt/openssl ] && export PATH=/opt/openssl/bin:$PATH
- make -j `nproc` test.keywords test.unit test.auth test.digest
- make test.modules.redis test.modules.redis_ippool
- make -j `nproc` test.modules
- make ci-test
+ # Activate per-test result recording for the summary table.
+ mkdir -p build/tests && touch build/tests/results.tsv
+
+ make -j `nproc` -k test.keywords test.unit test.auth test.digest || :
+ make -k test.modules.redis test.modules.redis_ippool || :
+ make -j `nproc` -k test.modules || :
+ make -k ci-test
env:
USE_DOCKER: ${{ inputs.use_docker }}
@@ -269,6 +272,35 @@ runs:
KAFKA_TEST_SERVER_PORT: ${{ inputs.kafka_test_server_port }}
CACHE_MEMCACHED_TEST_SERVER: ${{ inputs.memcached_test_server }}
+ - name: Test summary
+ if: ${{ always() }}
+ shell: bash
+ run: |
+ ./scripts/ci/ci-summary.sh build/tests/results.tsv >> "$GITHUB_STEP_SUMMARY" || :
+
+ - name: Stage failed test logs
+ if: ${{ failure() }}
+ shell: bash
+ run: |
+ mkdir -p build/tests/failed
+ [ -s build/tests/results.tsv ] && cp build/tests/results.tsv build/tests/failed/
+ awk -F'\t' '$3=="FAIL"{print $1"\t"$2"\t"$4}' build/tests/results.tsv 2>/dev/null | \
+ while IFS=$(printf '\t') read -r cat name logf; do
+ [ -n "$logf" ] && [ -f "$logf" ] || continue
+ mkdir -p "build/tests/failed/$cat"
+ cp "$logf" "build/tests/failed/$cat/$name.log"
+ cmdf=${logf%.*}.cmd
+ [ -f "$cmdf" ] && cp "$cmdf" "build/tests/failed/$cat/$name.cmd"
+ done
+
+ - name: Upload failed test logs
+ if: ${{ failure() }}
+ uses: actions/upload-artifact@v6
+ with:
+ name: ci-test-logs-${{ inputs.use_docker == 'true' && 'docker' || 'host' }}
+ path: build/tests/failed
+ if-no-files-found: ignore
+
# Restore ucf
- name: Restore ucf
if: ${{ inputs.use_docker != 'true' }}
diff --git a/scripts/ci/ci-summary.sh b/scripts/ci/ci-summary.sh
new file mode 100755
index 000000000000..8cab12ba7b5a
--- /dev/null
+++ b/scripts/ci/ci-summary.sh
@@ -0,0 +1,120 @@
+#!/bin/sh
+#
+# scripts/ci/ci-summary.sh
+#
+# Wrapped awk script that reads the test record file and emits a markdown
+# summary to stdout, intended for piping into $GITHUB_STEP_SUMMARY.
+#
+
+set -eu
+
+file=${1:-${CI_TEST_RECORD_FILE:-build/tests/results.tsv}}
+
+if [ ! -s "$file" ]; then
+ echo "## Test results"
+ echo
+ echo "_No results recorded._"
+ exit 0
+fi
+
+awk -F'\t' '
+{
+ key = $1 "\t" $2
+ cat[key] = $1
+ name[key] = $2
+ status[key] = $3
+ logf[key] = $4
+ seen[key] = NR # last-wins
+}
+END {
+ # Per-category counts
+ for (k in seen) {
+ c = cat[k]
+ cats[c] = 1
+ if (status[k] == "PASS") { cat_pass[c]++; total_pass++ }
+ else if (status[k] == "FAIL") { cat_fail[c]++; total_fail++ }
+ total++
+ }
+
+ # Header
+ printf("## Test results\n\n")
+ if (total_fail > 0) {
+ printf("**%d / %d passed** (%d failed)\n\n", total_pass, total, total_fail)
+ } else {
+ printf("**%d / %d passed**\n\n", total_pass, total)
+ }
+
+ # Failure logs as expanded-by-default blocks
+ if (total_fail > 0) {
+ printf("### Failure logs (last 20 lines)\n\n")
+ for (k in seen) {
+ if (status[k] != "FAIL") continue
+ printf("%s/%s — %s
\n\n",
+ cat[k], name[k], (logf[k] != "" ? logf[k] : "(no log captured)"))
+ if (logf[k] != "") {
+ # Reproducer comes from a side-car .cmd file the recipe wrote
+ # alongside the log (last extension replaced with .cmd).
+ cmdf = logf[k]; sub(/\.[^.]+$/, ".cmd", cmdf)
+ if ((getline reproducer < cmdf) > 0) {
+ close(cmdf)
+ printf("\nReproduce locally:\n```\n%s\n```\n\n", reproducer)
+ }
+ printf("Full log: `build/tests/failed/%s/%s.log` in the uploaded artifact.\n\n", cat[k], name[k])
+ # Embed last 20 lines of the log.
+ n = 0
+ while ((getline line < logf[k]) > 0) buf[++n % 20] = line
+ close(logf[k])
+ printf("```\n")
+ start = (n < 20) ? 1 : n - 19
+ for (j = start; j <= n; j++) print buf[j % 20]
+ delete buf
+ printf("```\n")
+ } else {
+ printf("_No log file recorded._\n")
+ }
+ printf("\n \n\n")
+ }
+ printf("---\n\n")
+ }
+
+ # Per-category sections. Categories with failures float to the top
+ # via a single insertion sort; otherwise alphabetical.
+ n = 0
+ for (c in cats) ordered[++n] = c
+ for (i = 2; i <= n; i++) {
+ cur = ordered[i]; j = i - 1
+ while (j >= 1) {
+ a = ordered[j]; b = cur
+ af = (cat_fail[a] ? 0 : 1); bf = (cat_fail[b] ? 0 : 1)
+ if (af < bf || (af == bf && a < b)) break
+ ordered[j+1] = ordered[j]; j--
+ }
+ ordered[j+1] = cur
+ }
+
+ for (i = 1; i <= n; i++) {
+ c = ordered[i]
+ cp = cat_pass[c] + 0; cf = cat_fail[c] + 0
+ ct = cp + cf
+ if (cf > 0) printf("### %s (%d/%d, %d failed)\n\n", c, cp, ct, cf)
+ else printf("### %s (%d/%d)\n\n", c, cp, ct)
+ # FAIL rows visible at the top. PASS rows hidden in a collapsed
+ # block so reviewers do not have to scroll past walls of green.
+ if (cf > 0) {
+ printf("| Test | Status |\n|---|---|\n")
+ for (k in seen) if (cat[k] == c && status[k] == "FAIL") emit(k)
+ printf("\n")
+ }
+ if (cp > 0) {
+ printf("%d passing test%s
\n\n", cp, (cp == 1 ? "" : "s"))
+ printf("| Test | Status |\n|---|---|\n")
+ for (k in seen) if (cat[k] == c && status[k] == "PASS") emit(k)
+ printf("\n \n\n")
+ }
+ }
+}
+
+function emit(k) {
+ printf("| `%s` | %s |\n", name[k], status[k])
+}
+' "$file"
diff --git a/scripts/ci/package-test.mk b/scripts/ci/package-test.mk
index 86c0701a5048..cc086e4e5887 100644
--- a/scripts/ci/package-test.mk
+++ b/scripts/ci/package-test.mk
@@ -52,4 +52,5 @@ POST_INSTALL_RADIUSD_BIN_ARG:=RADIUSD_BIN=$(RADIUSD_BIN)
.PHONY: package-test
package-test: test.eap
+include $(top_srcdir)/src/tests/test_record.mk
include $(DIR)/all.mk
diff --git a/src/tests/all.mk b/src/tests/all.mk
index 8eedaeea21f9..9fa38fdfed10 100644
--- a/src/tests/all.mk
+++ b/src/tests/all.mk
@@ -6,6 +6,8 @@ PORT := $(if $(PORT),$(PORT),12340)
SECRET := $(if $(SECRET),$(SECRET),testing123)
DICT_PATH := $(top_srcdir)/share/dictionary
+include $(top_srcdir)/src/tests/test_record.mk
+
#
# We need the 'git-lfs' installed to fetch some binary files.
#
diff --git a/src/tests/auth/all.mk b/src/tests/auth/all.mk
index 838a1c8569d9..d0bda6a28953 100644
--- a/src/tests/auth/all.mk
+++ b/src/tests/auth/all.mk
@@ -93,13 +93,15 @@ AUTH_LIBS := $(addsuffix .la,$(addprefix rlm_,$(AUTH_MODULES)))
# ERROR line in the input.
#
$(OUTPUT)/%: $(DIR)/% $(OUTPUT)/%.attrs $(TEST_BIN_DIR)/unit_test_module | $(AUTH_RADDB) $(AUTH_LIBS) build.raddb
+ $(eval CMD := TESTDIR=$(notdir $@) $(TEST_BIN)/unit_test_module -D share/dictionary -d src/tests/auth/ -i "$@.attrs" -f "$@.attrs" -r "$@" -xx)
@echo "AUTH-TEST $(notdir $@)"
- ${Q}if ! TESTDIR=$(notdir $@) $(TEST_BIN)/unit_test_module -D share/dictionary -d src/tests/auth/ -i "$@.attrs" -f "$@.attrs" -r "$@" -xx > "$@.log" 2>&1 || ! test -f "$@"; then \
+ @printf '%s\n' '$(CMD)' > $@.cmd
+ ${Q}if ! $(CMD) > "$@.log" 2>&1 || ! test -f "$@"; then \
if ! grep ERROR $< 2>&1 > /dev/null; then \
cat $@.log; \
echo "# $@.log"; \
- echo "TESTDIR=$(notdir $@) $(TEST_BIN)/unit_test_module -D share/dictionary -d src/tests/auth/ -i \"$@.attrs\" -f \"$@.attrs\" -r \"$@\" -xxx > \"$@.log\" 2>&1"; \
rm -f $(BUILD_DIR)/tests/test.auth; \
+ $(call test_record,auth,$(notdir $@),FAIL,$@.log); \
exit 1; \
fi; \
FOUND=$$(grep ^$< $@.log | head -1 | sed 's/:.*//;s/.*\[//;s/\].*//'); \
@@ -107,10 +109,11 @@ $(OUTPUT)/%: $(DIR)/% $(OUTPUT)/%.attrs $(TEST_BIN_DIR)/unit_test_module | $(AUT
if [ "$$EXPECTED" != "$$FOUND" ]; then \
cat $@.log; \
echo "# $@.log"; \
- echo "TESTDIR=$(notdir $@) $(TEST_BIN)/unit_test_module -D share/dictionary -d src/tests/auth/ -i \"$@.attrs\" -f \"$@.attrs\" -r \"$@\" -xxx > \"$@.log\" 2>&1"; \
rm -f $(BUILD_DIR)/tests/test.auth; \
+ $(call test_record,auth,$(notdir $@),FAIL,$@.log); \
exit 1; \
else \
touch "$@"; \
fi \
fi
+ @$(call test_record,auth,$(notdir $@),PASS,$@.log)
diff --git a/src/tests/bin/all.mk b/src/tests/bin/all.mk
index a585c8c8346e..f1d65d257c6a 100644
--- a/src/tests/bin/all.mk
+++ b/src/tests/bin/all.mk
@@ -52,13 +52,16 @@ unit_test_module.ARGS = -h
# that we're running
#
$(BUILD_DIR)/tests/bin/%: $(BUILD_DIR)/bin/local/%
+ $(eval CMD := $(TEST_BIN)/$(notdir $<) $($(notdir $@).ARGS))
@echo "BIN-TEST $(notdir $@)"
- ${Q}if ! $(TEST_BIN)/$(notdir $<) $($(notdir $@).ARGS) > $@.log 2>&1; then \
+ @printf '%s\n' '$(CMD)' > $@.cmd
+ ${Q}if ! $(CMD) > $@.log 2>&1; then \
echo LOG in $@.log; \
cat $@.log; \
- echo $(TEST_BIN)/$(notdir $<) $($(notdir $@).ARGS); \
+ $(call test_record,bin,$(notdir $@),FAIL,$@.log); \
exit 1; \
fi
+ @$(call test_record,bin,$(notdir $@),PASS,$@.log)
${Q}touch $@
#
diff --git a/src/tests/config/all.mk b/src/tests/config/all.mk
index 2a34f9df2898..4e488e40c83a 100644
--- a/src/tests/config/all.mk
+++ b/src/tests/config/all.mk
@@ -61,12 +61,13 @@ $(OUTPUT)/%: $(DIR)/% $(TEST_BIN_DIR)/unit_test_module | $(CONFIG_LIBS)
@mkdir -p $(dir $@)
$(eval CMD:=CONFIG_FILE=$(notdir $@) $(TEST_BIN)/unit_test_module $(UNIT_TEST_CONFIG_ARGS.$(subst -,_,$(notdir $@))) -D share/dictionary -d src/tests/config/ -xxC )
@echo "CONFIG-TEST $(notdir $@)"
+ @printf '%s\n' '$(CMD)' > $@.cmd
${Q}if ! $(CMD) > "$@.log" 2>&1; then \
if ! grep ERROR $< 2>&1 > /dev/null; then \
cat $@.log; \
echo "# $@.log"; \
- echo $(CMD); \
rm -f $(BUILD_DIR)/tests/test.config; \
+ $(call test_record,config,$(notdir $@),FAIL,$@.log); \
exit 1; \
fi; \
FOUND=$$(grep 'Error : src/tests/config/' $@.log | egrep -v -- '-->' | head -1 | sed 's/]:.*//;s/.*\[//;s/\].*//'); \
@@ -74,11 +75,12 @@ $(OUTPUT)/%: $(DIR)/% $(TEST_BIN_DIR)/unit_test_module | $(CONFIG_LIBS)
if [ "$$EXPECTED" != "$$FOUND" ]; then \
cat $@.log; \
echo "# $@.log"; \
- echo $(CMD); \
rm -f $(BUILD_DIR)/tests/test.config; \
+ $(call test_record,config,$(notdir $@),FAIL,$@.log); \
exit 1; \
fi; \
fi
+ @$(call test_record,config,$(notdir $@),PASS,$@.log)
touch "$@"
$(TEST):
diff --git a/src/tests/detail/all.mk b/src/tests/detail/all.mk
index 9057fd154f78..c36d879c397e 100644
--- a/src/tests/detail/all.mk
+++ b/src/tests/detail/all.mk
@@ -20,16 +20,20 @@ $(OUTPUT)/%: $(DIR)/% $(addprefix ${BUILD_DIR}/lib/,proto_detail.la proto_detail
${Q}echo "DETAIL $(notdir $<)"
${Q}cp $< $(dir $@)/detail.txt
${Q}rm -f $(dir $@)/processed
- ${Q}if ! $(TEST_BIN)/radiusd -d $(DIR)/config -D ${top_srcdir}/share/dictionary -X > $@.log; then \
+ $(eval CMD := $(TEST_BIN)/radiusd -d $(DIR)/config -D ${top_srcdir}/share/dictionary -X)
+ @printf 'cp %s %s/detail.txt && %s\n' '$<' '$(dir $@)' '$(CMD)' > $@.cmd
+ ${Q}if ! $(CMD) > $@.log; then \
tail $@.log; \
- echo "cp $< $(dir $@)/detail.txt; $(TEST_BIN)/radiusd -d $(DIR)/config -D ${top_srcdir}/share/dictionary -X "; \
+ $(call test_record,detail,$(notdir $@),FAIL,$@.log); \
exit 1; \
fi
${Q}if [ ! -e $(dir $@)/processed ] ; then \
tail $@.log; \
echo "Processing $< failed to produce expected output $(dir $@)/processed"; \
+ $(call test_record,detail,$(notdir $@),FAIL,$@.log); \
exit 1; \
fi
+ @$(call test_record,detail,$(notdir $@),PASS,$@.log)
${Q}touch $@
.NO_PARALLEL: $(TEST)
diff --git a/src/tests/dict/all.mk b/src/tests/dict/all.mk
index 6fcee8b5524d..b69319c6d3c1 100644
--- a/src/tests/dict/all.mk
+++ b/src/tests/dict/all.mk
@@ -16,17 +16,20 @@ $(eval $(call TEST_BOOTSTRAP))
# "foo_dir" directory, and copy "foo" into "foo_dir/dictionary"
#
$(OUTPUT)/%.dict: $(DIR)/%.dict $(TEST_BIN_DIR)/unit_test_attribute
+ $(eval CMD := $(TEST_BIN)/unit_test_attribute -D $(top_srcdir)/share/dictionary -o "$@" "$@.txt")
@echo "DICT-TEST $(notdir $@)"
@cp $< $@
@cp src/tests/dict/base.txt $@.txt
@echo "load-dictionary $(top_srcdir)/$<" >> $@.txt
- ${Q}if ! $(TEST_BIN)/unit_test_attribute -D $(top_srcdir)/share/dictionary -o "$@" "$@.txt" > "$@.log" 2>&1 || ! test -f "$@"; then \
+ @printf '%s\n' '$(CMD)' > $@.cmd
+ ${Q}if ! $(CMD) > "$@.log" 2>&1 || ! test -f "$@"; then \
rm -f $@; \
cat "$@.log"; \
echo "# $@.log"; \
- echo "$(TEST_BIN)/unit_test_attribute -D $(top_srcdir)/share/dictionary -o "$@" -xx '$@.txt'"; \
+ $(call test_record,dict,$(notdir $@),FAIL,$@.log); \
exit 1; \
fi
+ @$(call test_record,dict,$(notdir $@),PASS,$@.log)
# And the actual script to run each test.
#
@@ -34,17 +37,20 @@ $(OUTPUT)/%.dict: $(DIR)/%.dict $(TEST_BIN_DIR)/unit_test_attribute
# "foo_dir" directory, and copy "foo" into "foo_dir/dictionary"
#
$(OUTPUT)/%.dict: $(DIR)/%.dict $(TEST_BIN_DIR)/unit_test_attribute
+ $(eval CMD := $(TEST_BIN)/unit_test_attribute -D $(top_srcdir)/share/dictionary -o "$@" "$@.txt")
@echo "DICT-TEST $(notdir $@)"
@cp $< $@
@cp src/tests/dict/base.txt $@.txt
@echo "load-dictionary $(top_srcdir)/$<" >> $@.txt
- ${Q}if ! $(TEST_BIN)/unit_test_attribute -D $(top_srcdir)/share/dictionary -o "$@" "$@.txt" > "$@.log" 2>&1 || ! test -f "$@"; then \
+ @printf '%s\n' '$(CMD)' > $@.cmd
+ ${Q}if ! $(CMD) > "$@.log" 2>&1 || ! test -f "$@"; then \
rm -f $@; \
cat "$@.log"; \
echo "# $@.log"; \
- echo "$(TEST_BIN)/unit_test_attribute -D $(top_srcdir)/share/dictionary -o "$@" -xx '$@.txt'"; \
+ $(call test_record,dict,$(notdir $@),FAIL,$@.log); \
exit 1; \
fi
+ @$(call test_record,dict,$(notdir $@),PASS,$@.log)
#
# Tests which are supposed to fail.
@@ -56,15 +62,17 @@ $(OUTPUT)/%.dict: $(DIR)/%.dict $(TEST_BIN_DIR)/unit_test_attribute
# See if the current output and the expected output are the same.
#
$(OUTPUT)/%.error: $(DIR)/%.error $(TEST_BIN_DIR)/unit_test_attribute
+ $(eval CMD := $(TEST_BIN)/unit_test_attribute -D $(top_srcdir)/share/dictionary "$@.txt")
@echo "DICT-TEST $(notdir $@)"
@cp $< $@
@cp src/tests/dict/base.txt $@.txt
@echo "load-dictionary $(top_srcdir)/$<" >> $@.txt
- ${Q}if $(TEST_BIN)/unit_test_attribute -D $(top_srcdir)/share/dictionary "$@.txt" > "$@.log" 2>&1 ; then \
+ @printf '%s\n' '$(CMD)' > $@.cmd
+ ${Q}if $(CMD) > "$@.log" 2>&1 ; then \
rm -f $@; \
cat "$@.log"; \
echo "# $@.log"; \
- echo "$(TEST_BIN)/unit_test_attribute -D $(top_srcdir)/share/dictionary -xx '$@.txt'"; \
+ $(call test_record,dict,$(notdir $@),FAIL,$@.log); \
exit 1; \
fi
${Q}sed 's,${top_srcdir}/,,g' < "$@.log" > "$@.out"
@@ -72,3 +80,4 @@ $(OUTPUT)/%.error: $(DIR)/%.error $(TEST_BIN_DIR)/unit_test_attribute
echo "diff $@.out $(subst .error,,$<).out"; \
echo "FAILED"; \
fi
+ @$(call test_record,dict,$(notdir $@),PASS,$@.log)
diff --git a/src/tests/digest/all.mk b/src/tests/digest/all.mk
index d30597751e03..c3020794ca8d 100644
--- a/src/tests/digest/all.mk
+++ b/src/tests/digest/all.mk
@@ -35,21 +35,24 @@ $(eval $(call RADIUSD_SERVICE,digest,$(OUTPUT)))
$(OUTPUT)/%: $(DIR)/% | $(TEST).radiusd_kill $(TEST).radiusd_start
${Q} [ -f $(dir $@)/radiusd.pid ] || exit 1
$(eval TARGET := $(patsubst %.txt,%,$(notdir $@)$(E)))
+ $(eval CMD := $(TEST_BIN)/radclient -f $@.request -xF -d src/tests/digest/config -D share/dictionary 127.0.0.1:$(digest_port) auth $(SECRET))
${Q}for _num in $$(sed '/^#.*TESTS/!d; s/.*TESTS//g' $<); do \
echo "DIGEST-TEST $(TARGET)_$${_num}"; \
cp -f $< $@.request; \
echo "Vendor-Specific.Test.Test-Name = \"$(TARGET)\"" >> $@.request; \
echo "Vendor-Specific.Test.Test-Number = \"$${_num}\"" >> $@.request; \
- if ! $(TEST_BIN)/radclient -f $@.request -xF -d src/tests/digest/config -D share/dictionary 127.0.0.1:$(digest_port) auth $(SECRET) > $@.out; then \
+ printf '%s\n' '$(CMD)' > $@.cmd; \
+ if ! $(CMD) > $@.out; then \
cat $(DIGEST_RADIUS_LOG); \
echo "FAILED $(TARGET)"; \
cat $@.out; \
rm -f $(BUILD_DIR)/tests/test.digest; \
$(MAKE) --no-print-directory test.digest.radiusd_kill; \
echo "RADIUSD: $(RADIUSD_RUN)"; \
- echo "RADCLIENT: $(TEST_BIN)/radclient -f $@.request -xF -d src/tests/digest/config -D share/dictionary 127.0.0.1:$(digest_port) auth $(SECRET)"; \
+ $(call test_record,digest,$(TARGET)_$${_num},FAIL,$@.out); \
exit 1; \
fi; \
+ $(call test_record,digest,$(TARGET)_$${_num},PASS,$@.out); \
touch $@; \
done
diff --git a/src/tests/eapol_test/all.mk b/src/tests/eapol_test/all.mk
index e75e1872e0a6..5dd854043dac 100644
--- a/src/tests/eapol_test/all.mk
+++ b/src/tests/eapol_test/all.mk
@@ -115,14 +115,17 @@ test.eap.check: $(OUTPUT) $(GENERATED_CERT_FILES)
#
$(OUTPUT)/%.ok: $(DIR)/%.conf $(if $(POST_INSTALL_MAKEFILE_ARG),,$(BUILD_DIR)/lib/libfreeradius-server.la $(BUILD_DIR)/lib/libfreeradius-util.la) | $(GENERATED_CERT_FILES)
$(eval EAPOL_TEST_LOG := $(patsubst %.ok,%.log,$@))
+ $(eval EAPOL_TEST_CMD_FILE := $(patsubst %.ok,%.cmd,$@))
$(eval METHOD := $(notdir $(subst _,-,$(patsubst %.conf,%,$<))))
$(eval KEY := $(shell grep key_mgmt=NONE $< | sed 's/key_mgmt=NONE/-n/'))
$(eval RADIUS_LOG := $(dir $@)/test.$(METHOD)/radiusd.log)
$(eval TEST_PORT := $($(METHOD)_port))
+ $(eval CMD := $(EAPOL_TEST) -t $(EAPOL_TEST_TIMEOUT) -c $< -p $(TEST_PORT) -s $(SECRET) $(KEY))
@echo "EAPOL-TEST $(METHOD)"
+ @printf '%s\n' '$(CMD)' > $(EAPOL_TEST_CMD_FILE)
${Q}$(MAKE) $(POST_INSTALL_MAKEFILE_ARG) --no-print-directory test.$(METHOD).radiusd_kill
${Q}$(MAKE) $(POST_INSTALL_MAKEFILE_ARG) --no-print-directory test.$(METHOD).radiusd_start $(POST_INSTALL_RADIUSD_BIN_ARG)
- ${Q}if ! $(EAPOL_TEST) -t $(EAPOL_TEST_TIMEOUT) -c $< -p $(TEST_PORT) -s $(SECRET) $(KEY) > $(EAPOL_TEST_LOG) 2>&1; then \
+ ${Q}if ! $(CMD) > $(EAPOL_TEST_LOG) 2>&1; then \
echo "Last entries in supplicant log ($(EAPOL_TEST_LOG)):"; \
tail -n 40 "$(EAPOL_TEST_LOG)"; \
echo "--------------------------------------------------"; \
@@ -134,9 +137,11 @@ $(OUTPUT)/%.ok: $(DIR)/%.conf $(if $(POST_INSTALL_MAKEFILE_ARG),,$(BUILD_DIR)/li
echo " log is in $(OUT)"; \
rm -f $(BUILD_DIR)/tests/test.eap; \
$(MAKE) $(POST_INSTALL_MAKEFILE_ARG) --no-print-directory test.$(METHOD).radiusd_kill; \
+ $(call test_record,eap,$(METHOD),FAIL,$(EAPOL_TEST_LOG)); \
exit 1;\
fi
${Q}$(MAKE) $(POST_INSTALL_MAKEFILE_ARG) --no-print-directory test.$(METHOD).radiusd_stop
+ ${Q}$(call test_record,eap,$(METHOD),PASS,$(EAPOL_TEST_LOG))
${Q}touch $@
$(TEST): $(EAPOL_OK_FILES)
diff --git a/src/tests/keywords/all.mk b/src/tests/keywords/all.mk
index 0504fcbca44d..5f51e29c42ea 100644
--- a/src/tests/keywords/all.mk
+++ b/src/tests/keywords/all.mk
@@ -140,12 +140,13 @@ export IF_ENV_CONDITION := !test || (test != 'if-env')
$(OUTPUT)/%: $(DIR)/% $(TEST_BIN_DIR)/unit_test_module | $(KEYWORD_RADDB) $(KEYWORD_LIBS) build.raddb rlm_test.la rlm_csv.la rlm_unpack.la
$(eval CMD:=KEYWORD=$(notdir $@) $(TEST_BIN)/unit_test_module $(UNIT_TEST_KEYWORD_ARGS.$(subst -,_,$(notdir $@))) -D share/dictionary -d src/tests/keywords/ -i "$@.attrs" -f "$@.attrs" -r "$@" -xx )
@echo "KEYWORD-TEST $(notdir $@)"
+ @printf '%s\n' '$(CMD)' > $@.cmd
${Q}if ! $(CMD) > "$@.log" 2>&1 || ! test -f "$@"; then \
if ! grep ERROR $< 2>&1 > /dev/null; then \
cat $@.log; \
echo "# $@.log"; \
- echo $(CMD); \
rm -f $(BUILD_DIR)/tests/test.keywords; \
+ $(call test_record,keywords,$(notdir $@),FAIL,$@.log); \
exit 1; \
fi; \
FOUND=$$(grep 'Error : src/tests/keywords/' $@.log | egrep -v -- '-->' | head -1 | sed 's/]:.*//;s/.*\[//;s/\].*//'); \
@@ -153,13 +154,14 @@ $(OUTPUT)/%: $(DIR)/% $(TEST_BIN_DIR)/unit_test_module | $(KEYWORD_RADDB) $(KEYW
if [ "$$EXPECTED" != "$$FOUND" ]; then \
cat $@.log; \
echo "# $@.log"; \
- echo $(CMD); \
rm -f $(BUILD_DIR)/tests/test.keywords; \
+ $(call test_record,keywords,$(notdir $@),FAIL,$@.log); \
exit 1; \
else \
touch "$@"; \
fi \
fi
+ @$(call test_record,keywords,$(notdir $@),PASS,$@.log)
$(TEST):
@touch $(BUILD_DIR)/tests/$@
diff --git a/src/tests/ldap_sync/active_directory/all.mk b/src/tests/ldap_sync/active_directory/all.mk
index 8eb20b4dd896..67ac1804c82c 100644
--- a/src/tests/ldap_sync/active_directory/all.mk
+++ b/src/tests/ldap_sync/active_directory/all.mk
@@ -32,7 +32,7 @@ $(OUTPUT)/%: $(DIR)/% | $(TEST).trigger_clear $(TEST).radiusd_kill $(TEST).radiu
$(eval OUT_DIR := $(BUILD_DIR)/tests/ldap_sync/active_directory)
${Q}echo "LDAPSYNC-TEST active_directory $(TARGET)"
- ${Q}[ -f $(dir $@)/radiusd.pid ] || exit 1
+ ${Q}if ! [ -f $(dir $@)/radiusd.pid ]; then $(call test_record,ldap_sync,active_directory/$(TARGET),FAIL,$(OUT_DIR)/radiusd.log); exit 1; fi
${Q}rm -f $(OUT_DIR)/$(OUT).out
# Wait for the sync to start before applying changes
@@ -60,6 +60,7 @@ $(OUTPUT)/%: $(DIR)/% | $(TEST).trigger_clear $(TEST).radiusd_kill $(TEST).radiu
cat $(OUT_DIR)/radiusd.log; \
echo "LDAP_SYNC FAILED $(TARGET) - expected output file not produced"; \
rm -rf $(BUILD_DIR)/tests/test.ldap_sync/active_directory; \
+ $(call test_record,ldap_sync,active_directory/$(TARGET),FAIL,$(OUT_DIR)/radiusd.log); \
exit 1; \
fi
${Q}mv $(OUT_DIR)/$(OUT).out $(FOUND)
@@ -69,8 +70,10 @@ $(OUTPUT)/%: $(DIR)/% | $(TEST).trigger_clear $(TEST).radiusd_kill $(TEST).radiu
cat $(OUT_DIR)/radiusd.log; \
echo "LDAP_SYNC FAILED $(TARGET)"; \
rm -rf $(BUILD_DIR)/tests/test.ldap_sync/active_directory; \
+ $(call test_record,ldap_sync,active_directory/$(TARGET),FAIL,$(FOUND)); \
exit 1; \
fi
+ @$(call test_record,ldap_sync,active_directory/$(TARGET),PASS,$(FOUND))
${Q}touch $@
$(TEST):
diff --git a/src/tests/ldap_sync/persistent_search/all.mk b/src/tests/ldap_sync/persistent_search/all.mk
index 954dbd383c98..6c9a07643908 100644
--- a/src/tests/ldap_sync/persistent_search/all.mk
+++ b/src/tests/ldap_sync/persistent_search/all.mk
@@ -28,7 +28,7 @@ $(OUTPUT)/%: $(DIR)/% | $(TEST).trigger_clear $(TEST).radiusd_kill $(TEST).radiu
$(eval OUT := $(shell grep "#.*OUT:" $< | cut -f2 -d ':'))
${Q}echo "LDAPSYNC-TEST persistent_search $(TARGET)"
- ${Q}[ -f $(dir $@)/radiusd.pid ] || exit 1
+ ${Q}if ! [ -f $(dir $@)/radiusd.pid ]; then $(call test_record,ldap_sync,persistent_search/$(TARGET),FAIL,$(OUT_DIR)/radiusd.log); exit 1; fi
${Q}rm -f $(OUT_DIR)/$(OUT).out
# Wait for the sync to start before applying changes
@@ -56,6 +56,7 @@ $(OUTPUT)/%: $(DIR)/% | $(TEST).trigger_clear $(TEST).radiusd_kill $(TEST).radiu
cat $(OUT_DIR)/radiusd.log; \
echo "LDAP_SYNC FAILED $(TARGET) - expected output file not produced"; \
rm -rf $(BUILD_DIR)/tests/test.ldap_sync/persistent_search; \
+ $(call test_record,ldap_sync,persistent_search/$(TARGET),FAIL,$(OUT_DIR)/radiusd.log); \
exit 1; \
fi
${Q}mv $(OUT_DIR)/$(OUT).out $(FOUND)
@@ -65,8 +66,10 @@ $(OUTPUT)/%: $(DIR)/% | $(TEST).trigger_clear $(TEST).radiusd_kill $(TEST).radiu
cat $(OUT_DIR)/radiusd.log; \
echo "LDAP_SYNC FAILED $(TARGET)"; \
rm -rf $(BUILD_DIR)/tests/test.ldap_sync/persistent_search; \
+ $(call test_record,ldap_sync,persistent_search/$(TARGET),FAIL,$(FOUND)); \
exit 1; \
fi
+ @$(call test_record,ldap_sync,persistent_search/$(TARGET),PASS,$(FOUND))
${Q}touch $@
$(TEST):
diff --git a/src/tests/ldap_sync/rfc4533/all.mk b/src/tests/ldap_sync/rfc4533/all.mk
index d34e047e30b9..7ace5e2b4c86 100644
--- a/src/tests/ldap_sync/rfc4533/all.mk
+++ b/src/tests/ldap_sync/rfc4533/all.mk
@@ -29,7 +29,7 @@ $(OUTPUT)/%: $(DIR)/% | $(TEST).trigger_clear $(TEST).radiusd_kill $(TEST).radiu
$(eval OUT := $(shell grep "#.*OUT:" $< | cut -f2 -d ':'))
${Q}echo "LDAPSYNC-TEST rfc4533 $(TARGET)"
- ${Q}[ -f $(dir $@)/radiusd.pid ] || exit 1
+ ${Q}if ! [ -f $(dir $@)/radiusd.pid ]; then $(call test_record,ldap_sync,rfc4533/$(TARGET),FAIL,$(OUT_DIR)/radiusd.log); exit 1; fi
${Q}rm -f $(OUT_DIR)/$(OUT).out
# Wait for the sync to start before applying changes
@@ -57,6 +57,7 @@ $(OUTPUT)/%: $(DIR)/% | $(TEST).trigger_clear $(TEST).radiusd_kill $(TEST).radiu
cat $(OUT_DIR)/radiusd.log; \
echo "LDAP_SYNC FAILED $(TARGET) - expected output file not produced"; \
rm -rf $(BUILD_DIR)/tests/test.ldap_sync/rfc4533; \
+ $(call test_record,ldap_sync,rfc4533/$(TARGET),FAIL,$(OUT_DIR)/radiusd.log); \
exit 1; \
fi
${Q}mv $(OUT_DIR)/$(OUT).out $(FOUND)
@@ -65,8 +66,10 @@ $(OUTPUT)/%: $(DIR)/% | $(TEST).trigger_clear $(TEST).radiusd_kill $(TEST).radiu
cat $(OUT_DIR)/radiusd.log; \
echo "LDAP_SYNC FAILED $(TARGET)"; \
rm -rf $(BUILD_DIR)/tests/test.ldap_sync/rfc4533; \
+ $(call test_record,ldap_sync,rfc4533/$(TARGET),FAIL,$(FOUND)); \
exit 1; \
fi
+ @$(call test_record,ldap_sync,rfc4533/$(TARGET),PASS,$(FOUND))
${Q}touch $@
$(TEST):
diff --git a/src/tests/modules/all.mk b/src/tests/modules/all.mk
index b99f6667d351..d642881cc572 100644
--- a/src/tests/modules/all.mk
+++ b/src/tests/modules/all.mk
@@ -154,10 +154,12 @@ $(foreach x, $(filter cache_%,$(FILES)), $(eval $$(OUTPUT.$(TEST))/$x: $(BUILD_D
$(OUTPUT)/%: TEST=$(lastword $(subst /, ,$(dir $@))) $(basename $(notdir $@))
$(OUTPUT)/%: $(DIR)/%.unlang $(TEST_BIN_DIR)/unit_test_module | build.raddb
+ $(eval CMD := MODULE_TEST_DIR=$(dir $<) MODULE_TEST_UNLANG=$< TEST="$(TEST)" OUTPUT_DIR=$(dir $@) $(TEST_BIN)/unit_test_module -D share/dictionary -d src/tests/modules/ -i "$@.attrs" -f "$@.attrs" -r "$@" -xxx)
@echo "MODULE-TEST $(TEST)"
${Q}mkdir -p $(dir $@)
${Q}cp $(if $(wildcard $(basename $<).attrs),$(basename $<).attrs,src/tests/modules/default-input.attrs) $@.attrs
- ${Q}if ! MODULE_TEST_DIR=$(dir $<) MODULE_TEST_UNLANG=$< TEST="$(TEST)" OUTPUT_DIR=$(dir $@) $(TEST_BIN)/unit_test_module -D share/dictionary -d src/tests/modules/ -i "$@.attrs" -f "$@.attrs" -r "$@" -xxx > "$@.log" 2>&1 || ! test -f "$@"; then \
+ @printf '%s\n' '$(CMD)' > $@.cmd
+ ${Q}if ! $(CMD) > "$@.log" 2>&1 || ! test -f "$@"; then \
if ! grep ERROR $< 2>&1 > /dev/null; then \
if grep 'LeakSanitizer has encountered a fatal error' $@.log 2>&1 > /dev/null; then \
echo "MODULE-TEST $(TEST) - ignoring LeakSanitizer fatal error."; \
@@ -165,7 +167,7 @@ $(OUTPUT)/%: $(DIR)/%.unlang $(TEST_BIN_DIR)/unit_test_module | build.raddb
fi; \
cat "$@.log"; \
echo "# $@.log"; \
- echo "MODULE_TEST_DIR=$(dir $<) MODULE_TEST_UNLANG=$< OUTPUT_DIR=$(dir $@) $(TEST_BIN)/unit_test_module -D share/dictionary -d src/tests/modules/ -i \"$@.attrs\" -f \"$@.attrs\" -r \"$@\" -xx"; \
+ $(call test_record,modules,$(lastword $(subst /, ,$(dir $@)))/$(basename $(notdir $@)),FAIL,$@.log); \
exit 1; \
fi; \
FOUND=$$(grep -E 'Error : $<' $@.log | head -1 | sed 's/.*\[//;s/\].*//'); \
@@ -173,12 +175,13 @@ $(OUTPUT)/%: $(DIR)/%.unlang $(TEST_BIN_DIR)/unit_test_module | build.raddb
if [ "$$EXPECTED" != "$$FOUND" ]; then \
cat "$@.log"; \
echo "# $@.log"; \
- echo "MODULE_TEST_DIR=$(dir $<) MODULE_TEST_UNLANG=$< OUTPUT_DIR=$(dir $@) $(TEST_BIN)/unit_test_module -D share/dictionary -d src/tests/modules/ -i \"$@.attrs\" -f \"$@.attrs\" -r \"$@\" -xx"; \
+ $(call test_record,modules,$(lastword $(subst /, ,$(dir $@)))/$(basename $(notdir $@)),FAIL,$@.log); \
exit 1; \
else \
touch "$@"; \
fi \
fi
+ @$(call test_record,modules,$(lastword $(subst /, ,$(dir $@)))/$(basename $(notdir $@)),PASS,$@.log)
#
# Allow running individual tests.
diff --git a/src/tests/multi-server/all.mk b/src/tests/multi-server/all.mk
index 50f9006b1371..28252d9ab38f 100644
--- a/src/tests/multi-server/all.mk
+++ b/src/tests/multi-server/all.mk
@@ -144,6 +144,7 @@ test.multi-server.${1}.${2}: $$(TEST_MULTI_SERVER_RENDERED.${1}.${2})
$$(eval CMD := cd $(TEST_MULTI_SERVER_FRAMEWORK_DIR) && . .venv/bin/activate && DATA_PATH="${4}" python3 -m src.multi_server_test $(TEST_MULTI_SERVER_FLAGS) --project-name "${1}-${2}" --compose "${4}/environment.yml" --test "${4}/template.yml" --use-files --listener-dir "${4}/listener" --log-dir "${4}/logs" --output "${4}/logs/result.log")
${Q}mkdir -p "${4}/logs" "${4}/listener"
${Q}echo "MULTI-SERVER-TEST test.multi-server.${1}.${2}"
+ @printf '%s\n' '$$(CMD)' > "${4}/logs/stderr.cmd"
${Q}$$(CMD) > "${4}/logs/stdout.log" 2> "${4}/logs/stderr.log" || \
{ \
echo "FAILED: test.multi-server.${1}.${2}"; \
@@ -161,8 +162,10 @@ test.multi-server.${1}.${2}: $$(TEST_MULTI_SERVER_RENDERED.${1}.${2})
esac; \
tail -200 "$$$$f"; \
done; \
+ $$(call test_record,multi-server,${1}.${2},FAIL,${4}/logs/stderr.log); \
exit 1; \
}
+ ${Q}$$(call test_record,multi-server,${1}.${2},PASS,${4}/logs/stdout.log)
endef
#
diff --git a/src/tests/process/all.mk b/src/tests/process/all.mk
index d1f187e713d8..2f34941de112 100644
--- a/src/tests/process/all.mk
+++ b/src/tests/process/all.mk
@@ -98,12 +98,14 @@ $(OUTPUT)/%: $(DIR)/% $(PROCESS_DICT) $(TEST_BIN_DIR)/unit_test_module $(DIR)/un
$(eval export TEST_PORT:=$(TEST_PORT))
@echo PROCESS-TEST $(PROTOCOL_NAME) $(notdir $@)
$(Q)mkdir -p $(dir $@)
+ @printf '%s\n' '$(CMD)' > $@.cmd
$(Q)if ! $(CMD) > "$@.log" 2>&1 || ! test -f "$@"; then \
cat $@.log; \
echo "# $@.log"; \
- echo $(CMD); \
+ $(call test_record,process,$(PROTOCOL_NAME)/$(notdir $@),FAIL,$@.log); \
exit 1; \
fi
+ @$(call test_record,process,$(PROTOCOL_NAME)/$(notdir $@),PASS,$@.log)
$(TEST):
$(Q)touch $(BUILD_DIR)/tests/$@
diff --git a/src/tests/radclient/all.mk b/src/tests/radclient/all.mk
index dc9fd27a917b..73e4a3871422 100644
--- a/src/tests/radclient/all.mk
+++ b/src/tests/radclient/all.mk
@@ -53,16 +53,18 @@ $(OUTPUT)/%: $(DIR)/% $(BUILD_DIR)/bin/local/$(RADCLIENT) $(BUILD_DIR)/lib/local
$(eval IGNORE_ERROR := $(shell grep -q "#.*IGNORE_ERROR:.*1" $< && echo 1 || echo 0))
$(eval RADCLIENT_CLIENT_PORT := $(shell echo $$(($(RADCLIENT_CLIENT_PORT)+1))))
+ $(eval CMD := $(TEST_BIN)/$(RADCLIENT) $(ARGV) -C $(RADCLIENT_CLIENT_PORT) -f $< -d src/tests/radclient/config -D share/dictionary 127.0.0.1:$(radclient_port) $(TYPE) $(SECRET))
${Q}echo "RADCLIENT-TEST INPUT=$(TARGET) ARGV=\"$(ARGV)\""
- ${Q}[ -f $(dir $@)/radiusd.pid ] || exit 1
- ${Q}if ! $(TEST_BIN)/$(RADCLIENT) $(ARGV) -C $(RADCLIENT_CLIENT_PORT) -f $< -d src/tests/radclient/config -D share/dictionary 127.0.0.1:$(radclient_port) $(TYPE) $(SECRET) 1> $(FOUND) 2>&1; then \
+ @printf '%s\n' '$(CMD)' > $(basename $(FOUND)).cmd
+ ${Q}if ! [ -f $(dir $@)/radiusd.pid ]; then $(call test_record,radclient,$(notdir $@),FAIL,); exit 1; fi
+ ${Q}if ! $(CMD) 1> $(FOUND) 2>&1; then \
if [ "$(IGNORE_ERROR)" != "1" ]; then \
echo "FAILED"; \
cat $(FOUND); \
rm -f $(BUILD_DIR)/tests/test.radclient; \
$(MAKE) --no-print-directory test.radclient.radiusd_kill; \
echo "RADIUSD: $(RADIUSD_RUN)"; \
- echo "RADCLIENT: $(TEST_BIN)/$(RADCLIENT) $(ARGV) -C $(RADCLIENT_CLIENT_PORT) -f $< -xF -d src/tests/radclient/config -D share/dictionary 127.0.0.1:$(radclient_port) $(TYPE) $(SECRET)"; \
+ $(call test_record,radclient,$(notdir $@),FAIL,$(FOUND)); \
exit 1; \
fi; \
fi
@@ -99,6 +101,7 @@ $(OUTPUT)/%: $(DIR)/% $(BUILD_DIR)/bin/local/$(RADCLIENT) $(BUILD_DIR)/lib/local
diff -I 'Sent' -I 'Received' $(EXPECTED) $(FOUND); \
rm -f $(BUILD_DIR)/tests/test.radclient; \
$(MAKE) --no-print-directory test.radclient.radiusd_kill; \
+ $(call test_record,radclient,$(notdir $@),FAIL,$(FOUND)); \
exit 1; \
elif [ -e "$(CMD_TEST)" ] && ! $(SHELL) $(CMD_TEST); then \
echo "RADCLIENT FAILED $@"; \
@@ -108,8 +111,10 @@ $(OUTPUT)/%: $(DIR)/% $(BUILD_DIR)/bin/local/$(RADCLIENT) $(BUILD_DIR)/lib/local
echo "If you did some update on the radclient code, please be sure to update the unit tests."; \
rm -f $(BUILD_DIR)/tests/test.radclient; \
$(MAKE) --no-print-directory test.radclient.radiusd_kill; \
+ $(call test_record,radclient,$(notdir $@),FAIL,$(FOUND)); \
exit 1; \
fi
+ @$(call test_record,radclient,$(notdir $@),PASS,$(FOUND))
${Q}touch $@
.NO_PARALLEL: $(TEST)
diff --git a/src/tests/radmin/all.mk b/src/tests/radmin/all.mk
index c41506773b47..d37dd213022b 100644
--- a/src/tests/radmin/all.mk
+++ b/src/tests/radmin/all.mk
@@ -54,19 +54,21 @@ $(OUTPUT)/depends.mk: $(addprefix $(DIR)/,$(FILES)) | $(OUTPUT)
#
$(OUTPUT)/%: $(DIR)/% | $(TEST).radiusd_kill $(TEST).radiusd_start
@echo "RADMIN-TEST $(notdir $@)"
- ${Q} [ -f $(dir $@)/radiusd.pid ] || exit 1
+ ${Q} if ! [ -f $(dir $@)/radiusd.pid ]; then $(call test_record,radmin,$(notdir $@),FAIL,); exit 1; fi
$(eval EXPECTED := $(patsubst %.txt,%.out,$<))
$(eval FOUND := $(patsubst %.txt,%.out,$@))
$(eval TARGET := $(patsubst %.txt,%,$(notdir $@)$(E)))
+ $(eval CMD := $(TEST_BIN)/radmin -q -f $(RADMIN_SOCKET_FILE) < $< > $(FOUND))
+ @printf '%s\n' '$(CMD)' > $(basename $(FOUND)).cmd
${Q}if ! $(TEST_BIN)/radmin -q -f $(RADMIN_SOCKET_FILE) < $< > $(FOUND) 2>&1; then\
echo "--------------------------------------------------"; \
tail -n 20 "$(RADMIN_RADIUS_LOG)"; \
echo "Last entries in server log ($(RADMIN_RADIUS_LOG)):"; \
echo "--------------------------------------------------"; \
echo "RADIUSD: $(RADIUSD_RUN)"; \
- echo "RADMIN : $(TEST_BIN)/radmin -q -f $(RADMIN_SOCKET_FILE) < $< > $(FOUND)"; \
rm -f $(BUILD_DIR)/tests/test.radmin; \
$(MAKE) --no-print-directory test.radmin.radiusd_kill; \
+ $(call test_record,radmin,$(notdir $@),FAIL,$(FOUND)); \
exit 1; \
fi; \
sed -i.bak -e '$${/Executing: /d;}' $(FOUND); \
@@ -80,8 +82,10 @@ $(OUTPUT)/%: $(DIR)/% | $(TEST).radiusd_kill $(TEST).radiusd_start
diff $(EXPECTED) $(FOUND); \
rm -f $(BUILD_DIR)/tests/test.radmin; \
$(MAKE) --no-print-directory test.radmin.radiusd_kill; \
+ $(call test_record,radmin,$(notdir $@),FAIL,$(FOUND)); \
exit 1; \
else \
+ $(call test_record,radmin,$(notdir $@),PASS,$(FOUND)); \
touch $@;\
fi
diff --git a/src/tests/radsniff/all.mk b/src/tests/radsniff/all.mk
index 516941c9a121..de308cd8a4b1 100644
--- a/src/tests/radsniff/all.mk
+++ b/src/tests/radsniff/all.mk
@@ -47,15 +47,18 @@ $(OUTPUT)/%.txt: $(DIR)/%.txt $(TEST_BIN_DIR)/radsniff $(PCAP_IN)
$(eval EXPECTED := $<)
$(eval ARGV := $(shell grep "^#.*ARGV:" $< | cut -f2 -d ':'))
+ $(eval CMD := TZ='UTC' $(TEST_BIN)/radsniff $(ARGV) -I $(PCAP_IN) -D share/dictionary)
${Q}echo "RADSNIFF-TEST INPUT=$(TARGET) ARGV=\"$(ARGV)\""
+ @printf '%s\n' '$(CMD)' > $(basename $(FOUND)).cmd
#
# We need that 'TZ=UTC ...' to libpcap pass the same timestamp in anywhere.
#
- ${Q}if ! TZ='UTC' $(TEST_BIN)/radsniff $(ARGV) -I $(PCAP_IN) -D share/dictionary 1> $(FOUND); then \
+ ${Q}if ! $(CMD) 1> $(FOUND); then \
echo "FAILED"; \
cat $(FOUND); \
echo "RADSNIFF: TZ='UTC' $(TEST_BIN)/radsniff $(ARGV) -I $(PCAP_IN) -D share/dictionary" -xx; \
rm -f $@; \
+ $(call test_record,radsniff,$(notdir $@),FAIL,$(FOUND)); \
exit 1; \
fi
${Q}if [ -e "$(EXPECTED)" ]; then \
@@ -70,6 +73,7 @@ $(OUTPUT)/%.txt: $(DIR)/%.txt $(TEST_BIN_DIR)/radsniff $(PCAP_IN)
diff $(EXPECTED).simple $(FOUND); \
echo "diff $(EXPECTED).simple $(FOUND)"; \
rm -f $@; \
+ $(call test_record,radsniff,$(notdir $@),FAIL,$(FOUND)); \
exit 1; \
fi; \
elif [ -e "$(CMD_TEST)" ] && ! $(SHELL) $(CMD_TEST); then \
@@ -78,11 +82,14 @@ $(OUTPUT)/%.txt: $(DIR)/%.txt $(TEST_BIN_DIR)/radsniff $(PCAP_IN)
echo "ERROR: The script $(CMD_TEST) can't validate the content of $(FOUND)"; \
echo "If radsniff has been modified, please update the unit tests."; \
rm -f $@; \
+ $(call test_record,radsniff,$(notdir $@),FAIL,$(FOUND)); \
exit 1; \
else \
echo "ERROR! We should have at least one .txt or .cmd test"; \
+ $(call test_record,radsniff,$(notdir $@),FAIL,$(FOUND)); \
exit 1; \
fi
${Q}rm -f ${EXPECTED}.simple
+ @$(call test_record,radsniff,$(notdir $@),PASS,$(FOUND))
${Q}touch $@
endif
diff --git a/src/tests/tacacs/all.mk b/src/tests/tacacs/all.mk
index 0776489e7d51..393cd91acdfa 100644
--- a/src/tests/tacacs/all.mk
+++ b/src/tests/tacacs/all.mk
@@ -52,15 +52,17 @@ $(OUTPUT)/%: $(DIR)/% $(BUILD_DIR)/lib/libfreeradius-tacacs.la $(BUILD_DIR)/lib/
$(eval EXPECTED := $(patsubst %.txt,%.out,$<))
$(eval FOUND := $(patsubst %.txt,%.out,$@))
$(eval ARGV := $(shell grep "#.*ARGV:" $< | cut -f2 -d ':'))
+ $(eval CMD := $(TACCLIENT) --return-0-if-failed -v -k $(SECRET) -p $(tacacs_port) -H localhost -r 192.168.69.1 -P pegapilha/0 --timeout 2 $(ARGV))
${Q}echo "TACACS-TEST INPUT=$(TARGET) TACACS_ARGV=\"$(ARGV)\""
- ${Q}[ -f $(dir $@)/radiusd.pid ] || exit 1
- ${Q}if ! $(TACCLIENT) --return-0-if-failed -v -k $(SECRET) -p $(tacacs_port) -H localhost -r 192.168.69.1 -P pegapilha/0 --timeout 2 $(ARGV) 1> $(FOUND) 2>&1; then \
+ @printf '%s\n' '$(CMD)' > $(basename $(FOUND)).cmd
+ ${Q}if ! [ -f $(dir $@)/radiusd.pid ]; then $(call test_record,tacacs,$(notdir $@),FAIL,); exit 1; fi
+ ${Q}if ! $(CMD) 1> $(FOUND) 2>&1; then \
echo "FAILED"; \
cat $(FOUND); \
rm -f $(BUILD_DIR)/tests/test.tacacs; \
$(MAKE) --no-print-directory test.tacacs.radiusd_kill; \
echo "RADIUSD: $(RADIUSD_RUN)"; \
- echo "TACCLIENT: $(TACCLIENT) --return-0-if-failed -v -k $(SECRET) -p $(tacacs_port) -H localhost -r 192.168.69.1 -P pegapilha/0 --timeout 2 $(ARGV)"; \
+ $(call test_record,tacacs,$(notdir $@),FAIL,$(FOUND)); \
exit 1; \
fi
#
@@ -78,8 +80,10 @@ $(OUTPUT)/%: $(DIR)/% $(BUILD_DIR)/lib/libfreeradius-tacacs.la $(BUILD_DIR)/lib/
diff $(EXPECTED) $(FOUND); \
rm -f $(BUILD_DIR)/tests/test.tacacs; \
$(MAKE) --no-print-directory test.tacacs.radiusd_kill; \
+ $(call test_record,tacacs,$(notdir $@),FAIL,$(FOUND)); \
exit 1; \
fi
+ @$(call test_record,tacacs,$(notdir $@),PASS,$(FOUND))
${Q}touch $@
$(TEST):
diff --git a/src/tests/test_record.mk b/src/tests/test_record.mk
new file mode 100644
index 000000000000..dfd750c7fdf5
--- /dev/null
+++ b/src/tests/test_record.mk
@@ -0,0 +1,12 @@
+#
+# Per-test result recorder. Invoked from individual test recipes as:
+#
+# $(call test_record,,,,)
+#
+# Activated by touching $(BUILD_DIR)/tests/results.tsv
+#
+
+BUILD_DIR ?= $(top_builddir)/build
+RESULTS_TSV ?= $(BUILD_DIR)/tests/results.tsv
+
+test_record = [ -e $(RESULTS_TSV) ] && printf '%s\t%s\t%s\t%s\n' '$(1)' '$(2)' '$(3)' '$(4)' >> $(RESULTS_TSV) || true
diff --git a/src/tests/trie/test.mk b/src/tests/trie/test.mk
index b553452a334c..acc6fdfdfabc 100644
--- a/src/tests/trie/test.mk
+++ b/src/tests/trie/test.mk
@@ -11,12 +11,30 @@ $(BUILD_DIR)/tests/trie:
${Q}mkdir -p $@
$(BUILD_DIR)/tests/trie/trie-%: $(DIR)/% $(TEST_BIN_DIR)/trie | $(BUILD_DIR)/tests/trie
+ $(eval CMD := $(TEST_BIN)/trie $^)
@echo TRIE-TEST $(notdir $@)
- @$(TEST_BIN)/trie $^ > $@
+ @printf '%s\n' '$(CMD)' > $@.cmd
+ @if $(CMD) > $@ 2> $@.log; then \
+ $(call test_record,trie,$(notdir $@),PASS,$@.log); \
+ else \
+ cat $@.log; \
+ rm -f $@; \
+ $(call test_record,trie,$(notdir $@),FAIL,$@.log); \
+ exit 1; \
+ fi
$(BUILD_DIR)/tests/trie/nopc-%: $(DIR)/% $(TEST_BIN_DIR)/nopc | $(BUILD_DIR)/tests/trie
+ $(eval CMD := $(TEST_BIN)/nopc $^)
@echo TRIE-NO-PC-TEST $(notdir $@)
- @$(TEST_BIN)/nopc $^ > $@
+ @printf '%s\n' '$(CMD)' > $@.cmd
+ @if $(CMD) > $@ 2> $@.log; then \
+ $(call test_record,trie,$(notdir $@),PASS,$@.log); \
+ else \
+ cat $@.log; \
+ rm -f $@; \
+ $(call test_record,trie,$(notdir $@),FAIL,$@.log); \
+ exit 1; \
+ fi
#
# Get all of the unit test output files
diff --git a/src/tests/unit/all.mk b/src/tests/unit/all.mk
index b9281c62ceda..746032ea89a0 100644
--- a/src/tests/unit/all.mk
+++ b/src/tests/unit/all.mk
@@ -94,8 +94,17 @@ $(filter $(BUILD_DIR)/tests/unit/purify/%,$(FILES.$(TEST))): PURIFY=-p
#
$(OUTPUT)/%: $(DIR)/% $(TEST_BIN_DIR)/unit_test_attribute
$(eval DIR:=${top_srcdir}/src/tests/unit)
+ $(eval CMD := $(TEST_BIN)/unit_test_attribute $(PURIFY) $(REWRITE_FLAGS) -F ./src/tests/fuzzer-corpus -D ./share/dictionary -d $(DIR) -o "$@" $<)
$(eval export UNIT_TEST_ATTRIBUTE:=TZ=GMT $(TEST_BIN_NO_TIMEOUT)/unit_test_attribute $(PURIFY) -F ./src/tests/fuzzer-corpus -D ./share/dictionary -d $(DIR) -o \"$@\" $<)
- ${Q}$(TEST_BIN)/unit_test_attribute $(PURIFY) $(REWRITE_FLAGS) -F ./src/tests/fuzzer-corpus -D ./share/dictionary -d $(DIR) -o "$@" $<
+ ${Q}mkdir -p $(dir $@)
+ @printf '%s\n' '$(CMD)' > $@.cmd
+ ${Q}if $(CMD) > $@.log 2>&1; then \
+ $(call test_record,unit,$(patsubst $(BUILD_DIR)/tests/unit/%,%,$@),PASS,$@.log); \
+ else \
+ cat $@.log; \
+ $(call test_record,unit,$(patsubst $(BUILD_DIR)/tests/unit/%,%,$@),FAIL,$@.log); \
+ exit 1; \
+ fi
$(TEST):
@touch $(BUILD_DIR)/tests/$@
diff --git a/src/tests/vmps/all.mk b/src/tests/vmps/all.mk
index e8ac8a4a61f1..0dfddaef7969 100644
--- a/src/tests/vmps/all.mk
+++ b/src/tests/vmps/all.mk
@@ -33,15 +33,17 @@ $(OUTPUT)/%: $(DIR)/% | $(TEST).radiusd_kill $(TEST).radiusd_start
$(eval EXPECTED := $(patsubst %.txt,%.out,$<))
$(eval FOUND := $(patsubst %.txt,%.out,$@))
$(eval ARGV := $(shell grep "#.*ARGV:" $< | cut -f2 -d ':'))
+ $(eval CMD := $(VQCLI) -s 127.0.0.1 -p $(vmps_port) $(ARGV))
${Q}echo "VMPS-TEST INPUT=$(TARGET) VMPS_ARGV=\"$(ARGV)\""
- ${Q}[ -f $(dir $@)/radiusd.pid ] || exit 1
- ${Q}if ! $(VQCLI) -s 127.0.0.1 -p $(vmps_port) $(ARGV) 1> $(FOUND) 2>&1; then \
+ @printf '%s\n' '$(CMD)' > $(basename $(FOUND)).cmd
+ ${Q}if ! [ -f $(dir $@)/radiusd.pid ]; then $(call test_record,vmps,$(notdir $@),FAIL,); exit 1; fi
+ ${Q}if ! $(CMD) 1> $(FOUND) 2>&1; then \
echo "FAILED"; \
cat $(FOUND); \
rm -f $(BUILD_DIR)/tests/test.vmps; \
$(MAKE) --no-print-directory test.vmps.radiusd_kill; \
echo "RADIUSD: $(RADIUSD_RUN)"; \
- echo "VQCLI: $(VQCLI) -s 127.0.0.1 -p $(vmps_port) $(ARGV)"; \
+ $(call test_record,vmps,$(notdir $@),FAIL,$(FOUND)); \
exit 1; \
fi
#
@@ -59,8 +61,10 @@ $(OUTPUT)/%: $(DIR)/% | $(TEST).radiusd_kill $(TEST).radiusd_start
diff $(EXPECTED) $(FOUND); \
rm -f $(BUILD_DIR)/tests/test.vmps; \
$(MAKE) --no-print-directory test.vmps.radiusd_kill; \
+ $(call test_record,vmps,$(notdir $@),FAIL,$(FOUND)); \
exit 1; \
fi
+ @$(call test_record,vmps,$(notdir $@),PASS,$(FOUND))
${Q}touch $@
$(TEST):
diff --git a/src/tests/xlat/all.mk b/src/tests/xlat/all.mk
index d54a93397933..a9be0488c5fa 100644
--- a/src/tests/xlat/all.mk
+++ b/src/tests/xlat/all.mk
@@ -21,10 +21,13 @@ $(eval $(call TEST_BOOTSTRAP))
# And the actual script to run each test.
#
$(OUTPUT)/%: $(DIR)/% $(TEST_BIN_DIR)/unit_test_module $(DIR)/packet | build.raddb
+ $(eval CMD := $(TEST_BIN)/unit_test_module -D share/dictionary -d src/tests/xlat/ -r "$@" -i $(dir $<)/packet -I "$<" -xx)
@echo "XLAT-TEST $(notdir $@)"
- ${Q}if ! $(TEST_BIN)/unit_test_module -D share/dictionary -d src/tests/xlat/ -r "$@" -i $(dir $<)/packet -I "$<" -xx > "$@.log" 2>&1 || ! test -f "$@"; then \
+ @printf '%s\n' '$(CMD)' > $@.cmd
+ ${Q}if ! $(CMD) > "$@.log" 2>&1 || ! test -f "$@"; then \
cat $@.log; \
- echo "./$(TEST_BIN)/unit_test_module -D share/dictionary -d src/tests/xlat/ -r \"$@\" -i $(dir $<)/packet -I \"$<\" -xx "; \
rm -f $(BUILD_DIR)/tests/test.xlat; \
+ $(call test_record,xlat,$(notdir $@),FAIL,$@.log); \
exit 1; \
fi
+ @$(call test_record,xlat,$(notdir $@),PASS,$@.log)