CI: Summarise test results; capture failure logs; keep going on failure

.github/actions/ci-tests/action.yml

@@ -237,10 +237,13 @@ runs:
       run: |
         [ -d /opt/openssl ] && export PATH=/opt/openssl/bin:$PATH
 
-        make -j `nproc` test.keywords test.unit test.auth test.digest
-        make test.modules.redis test.modules.redis_ippool
-        make -j `nproc` test.modules
-        make ci-test
+        # Activate per-test result recording for the summary table.
+        mkdir -p build/tests && touch build/tests/results.tsv
+
+        make -j `nproc` -k test.keywords test.unit test.auth test.digest || :
+        make -k test.modules.redis test.modules.redis_ippool || :
+        make -j `nproc` -k test.modules || :
+        make -k ci-test
 
       env:
         USE_DOCKER: ${{ inputs.use_docker }}
@@ -269,6 +272,33 @@ runs:
         KAFKA_TEST_SERVER_PORT: ${{ inputs.kafka_test_server_port }}
         CACHE_MEMCACHED_TEST_SERVER: ${{ inputs.memcached_test_server }}
 
+    - name: Test summary
+      if: ${{ always() }}
+      shell: bash
+      run: |
+        ./scripts/ci/ci-summary.sh build/tests/results.tsv >> "$GITHUB_STEP_SUMMARY" || :
+
+    - name: Stage failed test logs
+      if: ${{ failure() }}
+      shell: bash
+      run: |
+        mkdir -p build/tests/failed
+        [ -s build/tests/results.tsv ] && cp build/tests/results.tsv build/tests/failed/
+        awk -F'\t' '$3=="FAIL"{print $1"\t"$2"\t"$4}' build/tests/results.tsv 2>/dev/null | \
+        while IFS=$(printf '\t') read -r cat name logf; do
+          [ -n "$logf" ] && [ -f "$logf" ] || continue
+          mkdir -p "build/tests/failed/$cat"
+          cp "$logf" "build/tests/failed/$cat/$name.log"
+        done
+
+    - name: Upload failed test logs
+      if: ${{ failure() }}
+      uses: actions/upload-artifact@v6
+      with:
+        name: ci-test-logs-${{ inputs.use_docker == 'true' && 'docker' || 'host' }}
+        path: build/tests/failed
+        if-no-files-found: ignore
+
     # Restore ucf
     - name: Restore ucf
       if: ${{ inputs.use_docker != 'true' }}

scripts/ci/ci-summary.sh

@@ -0,0 +1,103 @@
+#!/bin/sh
+#
+# scripts/ci/ci-summary.sh
+#
+# Wrapped awk script that reads the test record file and emits a markdown
+# summary to stdout, intended for piping into $GITHUB_STEP_SUMMARY.
+#
+
+set -eu
+
+file=${1:-${CI_TEST_RECORD_FILE:-build/tests/results.tsv}}
+
+if [ ! -s "$file" ]; then
+    echo "## Test results"
+    echo
+    echo "_No results recorded._"
+    exit 0
+fi
+
+awk -F'\t' '
+{
+    key = $1 "\t" $2
+    cat[key] = $1
+    name[key] = $2
+    status[key] = $3
+    logf[key] = $4
+    seen[key] = NR  # last-wins
+}
+END {
+    # Per-category counts
+    for (k in seen) {
+        c = cat[k]
+        cats[c] = 1
+        if (status[k] == "PASS") { cat_pass[c]++; total_pass++ }
+        else if (status[k] == "FAIL") { cat_fail[c]++; total_fail++ }
+        total++
+    }
+
+    # Header
+    printf("## Test results\n\n")
+    if (total_fail > 0) {
+        printf("**%d / %d passed** (%d failed)\n\n", total_pass, total, total_fail)
+    } else {
+        printf("**%d / %d passed**\n\n", total_pass, total)
+    }
+
+    # Failure logs as collapsible blocks
+    if (total_fail > 0) {
+        printf("### Failure logs (last 20 lines)\n\n")
+        for (k in seen) {
+            if (status[k] != "FAIL") continue
+            printf("<details><summary><code>%s/%s</code> &mdash; <code>%s</code></summary>\n\n",
+                   cat[k], name[k], (logf[k] != "" ? logf[k] : "(no log captured)"))
+            if (logf[k] != "") {
+                printf("```\n")
+                n = 0
+                while ((getline line < logf[k]) > 0) buf[++n % 20] = line
+                close(logf[k])
+                start = (n < 20) ? 1 : n - 19
+                for (j = start; j <= n; j++) print buf[j % 20]
+                delete buf
+                printf("```\n")
+            } else {
+                printf("_No log file recorded._\n")
+            }
+            printf("\n</details>\n\n")
+        }
+        printf("---\n\n")
+    }
+
+    # Per-category sections.  Categories with failures float to the top
+    # via a single insertion sort; otherwise alphabetical.
+    n = 0
+    for (c in cats) ordered[++n] = c
+    for (i = 2; i <= n; i++) {
+        cur = ordered[i]; j = i - 1
+        while (j >= 1) {
+            a = ordered[j]; b = cur
+            af = (cat_fail[a] ? 0 : 1); bf = (cat_fail[b] ? 0 : 1)
+            if (af < bf || (af == bf && a < b)) break
+            ordered[j+1] = ordered[j]; j--
+        }
+        ordered[j+1] = cur
+    }
+
+    for (i = 1; i <= n; i++) {
+        c = ordered[i]
+        cp = cat_pass[c] + 0; cf = cat_fail[c] + 0
+        ct = cp + cf
+        if (cf > 0)  printf("### %s (%d/%d, %d failed)\n\n", c, cp, ct, cf)
+        else         printf("### %s (%d/%d)\n\n", c, cp, ct)
+        printf("| Test | Status |\n|---|---|\n")
+        # FAIL rows first, then PASS, each in test-execution order.
+        for (k in seen) if (cat[k] == c && status[k] == "FAIL") emit(k)
+        for (k in seen) if (cat[k] == c && status[k] == "PASS") emit(k)
+        printf("\n")
+    }
+}
+
+function emit(k) {
+    printf("| `%s` | %s |\n", name[k], status[k])
+}
+' "$file"

scripts/ci/package-test.mk

@@ -52,4 +52,5 @@ POST_INSTALL_RADIUSD_BIN_ARG:=RADIUSD_BIN=$(RADIUSD_BIN)
 .PHONY: package-test
 package-test: test.eap
 
+include $(top_srcdir)/src/tests/test_record.mk
 include $(DIR)/all.mk

src/tests/all.mk

@@ -6,6 +6,8 @@ PORT := $(if $(PORT),$(PORT),12340)
 SECRET := $(if $(SECRET),$(SECRET),testing123)
 DICT_PATH := $(top_srcdir)/share/dictionary
 
+include $(top_srcdir)/src/tests/test_record.mk
+
 #
 #	We need the 'git-lfs' installed to fetch some binary files.
 #

src/tests/auth/all.mk

@@ -100,6 +100,7 @@ $(OUTPUT)/%: $(DIR)/% $(OUTPUT)/%.attrs $(TEST_BIN_DIR)/unit_test_module | $(AUT
 			echo "# $@.log"; \
 			echo "TESTDIR=$(notdir $@) $(TEST_BIN)/unit_test_module -D share/dictionary -d src/tests/auth/ -i \"$@.attrs\" -f \"$@.attrs\" -r \"$@\" -xxx > \"$@.log\" 2>&1"; \
 			rm -f $(BUILD_DIR)/tests/test.auth; \
+			$(call test_record,auth,$(notdir $@),FAIL,$@.log); \
 			exit 1; \
 		fi; \
 		FOUND=$$(grep ^$< $@.log | head -1 | sed 's/:.*//;s/.*\[//;s/\].*//'); \
@@ -109,8 +110,10 @@ $(OUTPUT)/%: $(DIR)/% $(OUTPUT)/%.attrs $(TEST_BIN_DIR)/unit_test_module | $(AUT
 			echo "# $@.log"; \
 			echo "TESTDIR=$(notdir $@) $(TEST_BIN)/unit_test_module -D share/dictionary -d src/tests/auth/ -i \"$@.attrs\" -f \"$@.attrs\" -r \"$@\" -xxx > \"$@.log\" 2>&1"; \
 			rm -f $(BUILD_DIR)/tests/test.auth; \
+			$(call test_record,auth,$(notdir $@),FAIL,$@.log); \
 			exit 1; \
 		else \
 			touch "$@"; \
 		fi \
 	fi
+	@$(call test_record,auth,$(notdir $@),PASS,$@.log)

src/tests/bin/all.mk

@@ -57,8 +57,10 @@ $(BUILD_DIR)/tests/bin/%: $(BUILD_DIR)/bin/local/%
 		echo LOG in $@.log; \
 		cat $@.log; \
 		echo $(TEST_BIN)/$(notdir $<) $($(notdir $@).ARGS); \
+		$(call test_record,bin,$(notdir $@),FAIL,$@.log); \
 		exit 1; \
 	fi
+	@$(call test_record,bin,$(notdir $@),PASS,$@.log)
 	${Q}touch $@
 
 #

src/tests/config/all.mk

@@ -67,6 +67,7 @@ $(OUTPUT)/%: $(DIR)/% $(TEST_BIN_DIR)/unit_test_module | $(CONFIG_LIBS)
 			echo "# $@.log"; \
 			echo $(CMD); \
 			rm -f $(BUILD_DIR)/tests/test.config; \
+			$(call test_record,config,$(notdir $@),FAIL,$@.log); \
 			exit 1; \
 		fi; \
 		FOUND=$$(grep 'Error : src/tests/config/' $@.log | egrep -v -- '-->' | head -1 | sed 's/]:.*//;s/.*\[//;s/\].*//'); \
@@ -76,9 +77,11 @@ $(OUTPUT)/%: $(DIR)/% $(TEST_BIN_DIR)/unit_test_module | $(CONFIG_LIBS)
 			echo "# $@.log"; \
 			echo $(CMD); \
 			rm -f $(BUILD_DIR)/tests/test.config; \
+			$(call test_record,config,$(notdir $@),FAIL,$@.log); \
 			exit 1; \
 		fi; \
 	fi
+	@$(call test_record,config,$(notdir $@),PASS,$@.log)
 	touch "$@"
 
 $(TEST):

src/tests/detail/all.mk

@@ -23,13 +23,16 @@ $(OUTPUT)/%: $(DIR)/% $(addprefix ${BUILD_DIR}/lib/,proto_detail.la proto_detail
 	${Q}if ! $(TEST_BIN)/radiusd -d $(DIR)/config -D ${top_srcdir}/share/dictionary -X > $@.log; then \
 		tail $@.log; \
 		echo "cp $< $(dir $@)/detail.txt; $(TEST_BIN)/radiusd -d $(DIR)/config -D ${top_srcdir}/share/dictionary -X "; \
+		$(call test_record,detail,$(notdir $@),FAIL,$@.log); \
 		exit 1; \
 	fi
 	${Q}if [ ! -e $(dir $@)/processed ] ; then \
 		tail $@.log; \
 		echo "Processing $< failed to produce expected output $(dir $@)/processed"; \
+		$(call test_record,detail,$(notdir $@),FAIL,$@.log); \
 		exit 1; \
 	fi
+	@$(call test_record,detail,$(notdir $@),PASS,$@.log)
 	${Q}touch $@
 
 .NO_PARALLEL: $(TEST)

src/tests/dict/all.mk

@@ -25,8 +25,10 @@ $(OUTPUT)/%.dict: $(DIR)/%.dict $(TEST_BIN_DIR)/unit_test_attribute
 		cat "$@.log"; \
 		echo "# $@.log"; \
 		echo "$(TEST_BIN)/unit_test_attribute -D $(top_srcdir)/share/dictionary -o "$@" -xx '$@.txt'"; \
+		$(call test_record,dict,$(notdir $@),FAIL,$@.log); \
 		exit 1; \
 	fi
+	@$(call test_record,dict,$(notdir $@),PASS,$@.log)
 
 #  And the actual script to run each test.
 #
@@ -43,8 +45,10 @@ $(OUTPUT)/%.dict: $(DIR)/%.dict $(TEST_BIN_DIR)/unit_test_attribute
 		cat "$@.log"; \
 		echo "# $@.log"; \
 		echo "$(TEST_BIN)/unit_test_attribute -D $(top_srcdir)/share/dictionary -o "$@" -xx '$@.txt'"; \
+		$(call test_record,dict,$(notdir $@),FAIL,$@.log); \
 		exit 1; \
 	fi
+	@$(call test_record,dict,$(notdir $@),PASS,$@.log)
 
 #
 #  Tests which are supposed to fail.
@@ -65,10 +69,12 @@ $(OUTPUT)/%.error: $(DIR)/%.error $(TEST_BIN_DIR)/unit_test_attribute
 		cat "$@.log"; \
 		echo "# $@.log"; \
 		echo "$(TEST_BIN)/unit_test_attribute -D $(top_srcdir)/share/dictionary -xx '$@.txt'"; \
+		$(call test_record,dict,$(notdir $@),FAIL,$@.log); \
 		exit 1; \
 	fi
 	${Q}sed 's,${top_srcdir}/,,g' < "$@.log" > "$@.out"
 	${Q}if ! diff "$@.out" "$(subst .error,,$<).out" ; then \
 		echo "diff $@.out $(subst .error,,$<).out"; \
 		echo "FAILED"; \
 	fi
+	@$(call test_record,dict,$(notdir $@),PASS,$@.log)

src/tests/digest/all.mk

@@ -48,8 +48,10 @@ $(OUTPUT)/%: $(DIR)/% | $(TEST).radiusd_kill $(TEST).radiusd_start
 			$(MAKE) --no-print-directory test.digest.radiusd_kill; \
 			echo "RADIUSD:   $(RADIUSD_RUN)";		\
 			echo "RADCLIENT: $(TEST_BIN)/radclient -f $@.request -xF -d src/tests/digest/config -D share/dictionary 127.0.0.1:$(digest_port) auth $(SECRET)"; \
+			$(call test_record,digest,$(TARGET)_$${_num},FAIL,$@.out); \
 			exit 1;						\
 		fi;							\
+		$(call test_record,digest,$(TARGET)_$${_num},PASS,$@.out); \
 		touch $@;						\
 	done
 

src/tests/eapol_test/all.mk

@@ -134,9 +134,11 @@ $(OUTPUT)/%.ok: $(DIR)/%.conf $(if $(POST_INSTALL_MAKEFILE_ARG),,$(BUILD_DIR)/li
 		echo "           log is in $(OUT)"; \
 		rm -f $(BUILD_DIR)/tests/test.eap;                                      \
 		$(MAKE) $(POST_INSTALL_MAKEFILE_ARG) --no-print-directory test.$(METHOD).radiusd_kill;			\
+		$(call test_record,eap,$(METHOD),FAIL,$(EAPOL_TEST_LOG)); \
 		exit 1;\
 	fi
 	${Q}$(MAKE) $(POST_INSTALL_MAKEFILE_ARG) --no-print-directory test.$(METHOD).radiusd_stop
+	${Q}$(call test_record,eap,$(METHOD),PASS,$(EAPOL_TEST_LOG))
 	${Q}touch $@
 
 $(TEST): $(EAPOL_OK_FILES)

src/tests/keywords/all.mk

@@ -146,6 +146,7 @@ $(OUTPUT)/%: $(DIR)/% $(TEST_BIN_DIR)/unit_test_module | $(KEYWORD_RADDB) $(KEYW
 			echo "# $@.log"; \
 			echo $(CMD); \
 			rm -f $(BUILD_DIR)/tests/test.keywords; \
+			$(call test_record,keywords,$(notdir $@),FAIL,$@.log); \
 			exit 1; \
 		fi; \
 		FOUND=$$(grep 'Error : src/tests/keywords/' $@.log | egrep -v -- '-->' | head -1 | sed 's/]:.*//;s/.*\[//;s/\].*//'); \
@@ -155,11 +156,13 @@ $(OUTPUT)/%: $(DIR)/% $(TEST_BIN_DIR)/unit_test_module | $(KEYWORD_RADDB) $(KEYW
 			echo "# $@.log"; \
 			echo $(CMD); \
 			rm -f $(BUILD_DIR)/tests/test.keywords; \
+			$(call test_record,keywords,$(notdir $@),FAIL,$@.log); \
 			exit 1; \
 		else \
 			touch "$@"; \
 		fi \
 	fi
+	@$(call test_record,keywords,$(notdir $@),PASS,$@.log)
 
 $(TEST):
 	@touch $(BUILD_DIR)/tests/$@

src/tests/ldap_sync/active_directory/all.mk

@@ -32,7 +32,7 @@ $(OUTPUT)/%: $(DIR)/% | $(TEST).trigger_clear $(TEST).radiusd_kill $(TEST).radiu
 	$(eval OUT_DIR  := $(BUILD_DIR)/tests/ldap_sync/active_directory)
 
 	${Q}echo "LDAPSYNC-TEST active_directory $(TARGET)"
-	${Q}[ -f $(dir $@)/radiusd.pid ] || exit 1
+	${Q}if ! [ -f $(dir $@)/radiusd.pid ]; then $(call test_record,ldap_sync,active_directory/$(TARGET),FAIL,$(OUT_DIR)/radiusd.log); exit 1; fi
 	${Q}rm -f $(OUT_DIR)/$(OUT).out
 
 #	Wait for the sync to start before applying changes
@@ -60,6 +60,7 @@ $(OUTPUT)/%: $(DIR)/% | $(TEST).trigger_clear $(TEST).radiusd_kill $(TEST).radiu
 		cat $(OUT_DIR)/radiusd.log;					\
 		echo "LDAP_SYNC FAILED $(TARGET) - expected output file not produced";	\
 		rm -rf $(BUILD_DIR)/tests/test.ldap_sync/active_directory;	\
+		$(call test_record,ldap_sync,active_directory/$(TARGET),FAIL,$(OUT_DIR)/radiusd.log); \
 		exit 1;								\
 	fi
 	${Q}mv $(OUT_DIR)/$(OUT).out $(FOUND)
@@ -69,8 +70,10 @@ $(OUTPUT)/%: $(DIR)/% | $(TEST).trigger_clear $(TEST).radiusd_kill $(TEST).radiu
 		cat $(OUT_DIR)/radiusd.log;					\
 		echo "LDAP_SYNC FAILED $(TARGET)";				\
 		rm -rf $(BUILD_DIR)/tests/test.ldap_sync/active_directory;	\
+		$(call test_record,ldap_sync,active_directory/$(TARGET),FAIL,$(FOUND)); \
 		exit 1;								\
 	fi
+	@$(call test_record,ldap_sync,active_directory/$(TARGET),PASS,$(FOUND))
 	${Q}touch $@
 
 $(TEST):

src/tests/ldap_sync/persistent_search/all.mk

@@ -28,7 +28,7 @@ $(OUTPUT)/%: $(DIR)/% | $(TEST).trigger_clear $(TEST).radiusd_kill $(TEST).radiu
 	$(eval OUT      := $(shell grep "#.*OUT:" $< | cut -f2 -d ':'))
 
 	${Q}echo "LDAPSYNC-TEST persistent_search $(TARGET)"
-	${Q}[ -f $(dir $@)/radiusd.pid ] || exit 1
+	${Q}if ! [ -f $(dir $@)/radiusd.pid ]; then $(call test_record,ldap_sync,persistent_search/$(TARGET),FAIL,$(OUT_DIR)/radiusd.log); exit 1; fi
 	${Q}rm -f $(OUT_DIR)/$(OUT).out
 
 #	Wait for the sync to start before applying changes
@@ -56,6 +56,7 @@ $(OUTPUT)/%: $(DIR)/% | $(TEST).trigger_clear $(TEST).radiusd_kill $(TEST).radiu
 		cat $(OUT_DIR)/radiusd.log;					\
 		echo "LDAP_SYNC FAILED $(TARGET) - expected output file not produced";	\
 		rm -rf $(BUILD_DIR)/tests/test.ldap_sync/persistent_search;	\
+		$(call test_record,ldap_sync,persistent_search/$(TARGET),FAIL,$(OUT_DIR)/radiusd.log); \
 		exit 1;								\
 	fi
 	${Q}mv $(OUT_DIR)/$(OUT).out $(FOUND)
@@ -65,8 +66,10 @@ $(OUTPUT)/%: $(DIR)/% | $(TEST).trigger_clear $(TEST).radiusd_kill $(TEST).radiu
 		cat $(OUT_DIR)/radiusd.log;					\
 		echo "LDAP_SYNC FAILED $(TARGET)";				\
 		rm -rf $(BUILD_DIR)/tests/test.ldap_sync/persistent_search;	\
+		$(call test_record,ldap_sync,persistent_search/$(TARGET),FAIL,$(FOUND)); \
 		exit 1;								\
 	fi
+	@$(call test_record,ldap_sync,persistent_search/$(TARGET),PASS,$(FOUND))
 	${Q}touch $@
 
 $(TEST):