Skip to content

Commit af7661f

Browse files
committed
CI: Summarise test results; capture failure logs; keep going on failure
We skip ordering when recording test results (for example in CI) since that's only really useful for ensuring stable output for local testing.
1 parent 094d722 commit af7661f

11 files changed

Lines changed: 184 additions & 4 deletions

File tree

.github/workflows/ci.yml

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -318,7 +318,35 @@ jobs:
318318
openssl dhparam -out ./raddb/certs/dh -2 512 || \
319319
true
320320
321-
make ci-test
321+
# Activate per-test result recording for the summary table.
322+
mkdir -p build/tests && touch build/tests/results.tsv
323+
324+
make -k ci-test
325+
326+
- name: Test summary
327+
if: ${{ always() }}
328+
run: |
329+
./scripts/ci/ci-summary.sh build/tests/results.tsv >> "$GITHUB_STEP_SUMMARY" || :
330+
331+
- name: Stage failed test logs
332+
if: ${{ failure() }}
333+
run: |
334+
mkdir -p build/tests/failed
335+
[ -s build/tests/results.tsv ] && cp build/tests/results.tsv build/tests/failed/
336+
awk -F'\t' '$3=="FAIL"{print $1"\t"$2"\t"$4}' build/tests/results.tsv 2>/dev/null | \
337+
while IFS=$(printf '\t') read -r cat name logf; do
338+
[ -n "$logf" ] && [ -f "$logf" ] || continue
339+
mkdir -p "build/tests/failed/$cat"
340+
cp "$logf" "build/tests/failed/$cat/$name.log"
341+
done
342+
343+
- name: Upload failed test logs
344+
if: ${{ failure() }}
345+
uses: actions/upload-artifact@v6
346+
with:
347+
name: ci-test-logs-${{ matrix.os.name }}-${{ matrix.env.NAME }}
348+
path: build/tests/failed
349+
if-no-files-found: ignore
322350

323351
- name: Show debug logs on failure
324352
if: ${{ failure() }}

scripts/ci/ci-summary.sh

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
#!/bin/sh
2+
#
3+
# scripts/ci/ci-summary.sh
4+
#
5+
# Wrapped awk script that reads the test record file and emits a markdown
6+
# summary to stdout, intended for piping into $GITHUB_STEP_SUMMARY.
7+
#
8+
9+
set -eu
10+
11+
file=${1:-${CI_TEST_RECORD_FILE:-build/tests/results.tsv}}
12+
13+
if [ ! -s "$file" ]; then
14+
echo "## Test results"
15+
echo
16+
echo "_No results recorded._"
17+
exit 0
18+
fi
19+
20+
awk -F'\t' '
21+
{
22+
key = $1 "\t" $2
23+
cat[key] = $1
24+
name[key] = $2
25+
status[key] = $3
26+
logf[key] = $4
27+
seen[key] = NR # last-wins
28+
}
29+
END {
30+
# Per-category counts
31+
for (k in seen) {
32+
c = cat[k]
33+
cats[c] = 1
34+
if (status[k] == "PASS") { cat_pass[c]++; total_pass++ }
35+
else if (status[k] == "FAIL") { cat_fail[c]++; total_fail++ }
36+
total++
37+
}
38+
39+
# Header
40+
printf("## Test results\n\n")
41+
if (total_fail > 0) {
42+
printf("**%d / %d passed** (%d failed)\n\n", total_pass, total, total_fail)
43+
} else {
44+
printf("**%d / %d passed**\n\n", total_pass, total)
45+
}
46+
47+
# Failure logs as collapsible blocks
48+
if (total_fail > 0) {
49+
printf("### Failure logs (last 20 lines)\n\n")
50+
for (k in seen) {
51+
if (status[k] != "FAIL") continue
52+
printf("<details><summary><code>%s/%s</code> &mdash; <code>%s</code></summary>\n\n",
53+
cat[k], name[k], (logf[k] != "" ? logf[k] : "(no log captured)"))
54+
if (logf[k] != "") {
55+
printf("```\n")
56+
n = 0
57+
while ((getline line < logf[k]) > 0) buf[++n % 20] = line
58+
close(logf[k])
59+
start = (n < 20) ? 1 : n - 19
60+
for (j = start; j <= n; j++) print buf[j % 20]
61+
delete buf
62+
printf("```\n")
63+
} else {
64+
printf("_No log file recorded._\n")
65+
}
66+
printf("\n</details>\n\n")
67+
}
68+
printf("---\n\n")
69+
}
70+
71+
# Per-category sections. Categories with failures float to the top
72+
# via a single insertion sort; otherwise alphabetical.
73+
n = 0
74+
for (c in cats) ordered[++n] = c
75+
for (i = 2; i <= n; i++) {
76+
cur = ordered[i]; j = i - 1
77+
while (j >= 1) {
78+
a = ordered[j]; b = cur
79+
af = (cat_fail[a] ? 0 : 1); bf = (cat_fail[b] ? 0 : 1)
80+
if (af < bf || (af == bf && a < b)) break
81+
ordered[j+1] = ordered[j]; j--
82+
}
83+
ordered[j+1] = cur
84+
}
85+
86+
for (i = 1; i <= n; i++) {
87+
c = ordered[i]
88+
cp = cat_pass[c] + 0; cf = cat_fail[c] + 0
89+
ct = cp + cf
90+
if (cf > 0) printf("### %s (%d/%d, %d failed)\n\n", c, cp, ct, cf)
91+
else printf("### %s (%d/%d)\n\n", c, cp, ct)
92+
printf("| Test | Status |\n|---|---|\n")
93+
# FAIL rows first, then PASS, each in test-execution order.
94+
for (k in seen) if (cat[k] == c && status[k] == "FAIL") emit(k)
95+
for (k in seen) if (cat[k] == c && status[k] == "PASS") emit(k)
96+
printf("\n")
97+
}
98+
}
99+
100+
function emit(k) {
101+
printf("| `%s` | %s |\n", name[k], status[k])
102+
}
103+
' "$file"

src/tests/Makefile

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
##
88
#
99
include ../../Make.inc
10+
include $(top_srcdir)/src/tests/test_record.mk
1011

1112
BUILD_PATH := $(top_builddir)/build
1213

@@ -239,9 +240,11 @@ $(BUILD_PATH)/tests/eap/%.ok: $(top_builddir)/src/tests/%.conf | radiusd.kill $(
239240
tail -10 $(LOG); \
240241
echo "===================="; \
241242
$(MAKE) radiusd.kill; \
243+
$(call test_record,eap,$(notdir $(patsubst %.conf,%,$<)),FAIL,$(LOG)); \
242244
exit 1; \
243245
fi
244246
@echo
247+
@$(call test_record,eap,$(notdir $(patsubst %.conf,%,$<)),PASS,$(LOG))
245248
@touch $@
246249

247250
#
@@ -271,21 +274,25 @@ $(BUILD_PATH)/tests/eap/${1}-${2}.ok: $(BUILD_PATH)/tests/eap/${1}-${2}.conf
271274
tail -10 $$(LOG); \
272275
echo "===================="; \
273276
$(MAKE) radiusd.kill; \
277+
$$(call test_record,eap,$$(notdir $$(patsubst %.ok,%,$$@)),FAIL,$$(LOG)); \
274278
exit 1; \
275279
elif ! grep -q '^SSL: Using TLS version TLSv${2}$$$$' $$(patsubst %.ok,%,$$@).log; then \
276280
echo " - " FAILED - not using TLS version ${2}; \
277281
echo ">>> cmd -" $$(CMD) -r 1; \
278282
echo ">>> log -" $$(LOG); \
279283
$(MAKE) radiusd.kill; \
284+
$$(call test_record,eap,$$(notdir $$(patsubst %.ok,%,$$@)),FAIL,$$(LOG)); \
280285
exit 1; \
281286
elif ! grep -q '^OpenSSL: Handshake finished - resumed=1$$$$' $$(patsubst %.ok,%,$$@).log; then \
282287
echo " - " FAILED - did not use resumption; \
283288
echo ">>> cmd -" $$(CMD) -r 1; \
284289
echo ">>> log -" $$(LOG); \
285290
$(MAKE) radiusd.kill; \
291+
$$(call test_record,eap,$$(notdir $$(patsubst %.ok,%,$$@)),FAIL,$$(LOG)); \
286292
exit 1; \
287293
fi
288294
@echo
295+
@$$(call test_record,eap,$$(notdir $$(patsubst %.ok,%,$$@)),PASS,$$(LOG))
289296
@touch $$@
290297

291298
# EAP-FAST doesn't do TLS 1.3

src/tests/all.mk

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ PORT := 12340
33
SECRET := testing123
44
DICT_PATH := $(top_srcdir)/share
55

6+
include $(top_srcdir)/src/tests/test_record.mk
7+
68
#
79
# Include all of the autoconf definitions into the Make variable space
810
#

src/tests/auth/all.mk

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ $(BUILD_DIR)/tests/auth/%: $(DIR)/% $(BUILD_DIR)/tests/auth/%.attrs $(TESTBINDIR
8989
cat $@.log; \
9090
echo "# $@.log"; \
9191
echo "TESTDIR=$(notdir $@) $(TESTBIN)/unittest -D share -d src/tests/auth/ -i $@.attrs -f $@.attrs -xxx > $@.log 2>&1"; \
92+
$(call test_record,auth,$(notdir $@),FAIL,$@.log); \
9293
exit 1; \
9394
fi; \
9495
FOUND=$$(grep ^$< $@.log | head -1 | sed 's/:.*//;s/.*\[//;s/\].*//'); \
@@ -97,9 +98,11 @@ $(BUILD_DIR)/tests/auth/%: $(DIR)/% $(BUILD_DIR)/tests/auth/%.attrs $(TESTBINDIR
9798
cat $@.log; \
9899
echo "# $@.log"; \
99100
echo "TESTDIR=$(notdir $@) $(TESTBIN)/unittest -D share -d src/tests/auth/ -i $@.attrs -f $@.attrs -xxx > $@.log 2>&1"; \
101+
$(call test_record,auth,$(notdir $@),FAIL,$@.log); \
100102
exit 1; \
101103
fi \
102104
fi
105+
@$(call test_record,auth,$(notdir $@),PASS,$@.log)
103106
@touch $@
104107

105108
#
@@ -112,7 +115,9 @@ TESTS.AUTH_FILES := $(addprefix $(BUILD_DIR)/tests/auth/,$(AUTH_FILES))
112115
#
113116
tests.auth: $(TESTS.AUTH_FILES)
114117

118+
ifeq "$(RECORDING)" ""
115119
$(TESTS.AUTH_FILES): $(TESTS.KEYWORDS_FILES)
120+
endif
116121

117122
.PHONY: clean.tests.auth
118123
clean.tests.auth:

src/tests/keywords/all.mk

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@ $(BUILD_DIR)/tests/keywords/%: ${DIR}/% $(BUILD_DIR)/tests/keywords/%.attrs $(TE
9393
cat $@.log; \
9494
echo "# $@.log"; \
9595
echo KEYWORD=$(notdir $@) $(TESTBIN)/unittest -D share -d src/tests/keywords/ -i $@.attrs -f $@.attrs -xx; \
96+
$(call test_record,keywords,$(notdir $@),FAIL,$@.log); \
9697
exit 1; \
9798
fi; \
9899
FOUND=$$(grep ^$< $@.log | head -1 | sed 's/:.*//;s/.*\[//;s/\].*//'); \
@@ -101,9 +102,11 @@ $(BUILD_DIR)/tests/keywords/%: ${DIR}/% $(BUILD_DIR)/tests/keywords/%.attrs $(TE
101102
cat $@.log; \
102103
echo "# $@.log"; \
103104
echo KEYWORD=$(notdir $@) $(TESTBIN)/unittest -D share -d src/tests/keywords/ -i $@.attrs -f $@.attrs -xx; \
105+
$(call test_record,keywords,$(notdir $@),FAIL,$@.log); \
104106
exit 1; \
105107
fi \
106108
fi
109+
@$(call test_record,keywords,$(notdir $@),PASS,$@.log)
107110
@touch $@
108111

109112
#
@@ -116,7 +119,9 @@ TESTS.KEYWORDS_FILES := $(addprefix $(BUILD_DIR)/tests/keywords/,$(KEYWORD_FILES
116119
#
117120
tests.keywords: $(TESTS.KEYWORDS_FILES)
118121

122+
ifeq "$(RECORDING)" ""
119123
$(TESTS.KEYWORDS_FILES): $(TESTS.XLAT_FILES) $(TESTS.MAP_FILES)
124+
endif
120125

121126
.PHONY: clean.tests.keywords
122127
clean.tests.keywords:

src/tests/modules/test.mk

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ $(BUILD_DIR)/tests/modules/%: src/tests/modules/%.unlang $(BUILD_DIR)/tests/modu
5757
cat $@.log; \
5858
echo "# $@.log"; \
5959
echo MODULE_TEST_DIR=$(dir $<) MODULE_TEST_UNLANG=$< $(TESTBIN)/unittest -D share -d src/tests/modules/ -i $@.attrs -f $@.attrs -xx; \
60+
$(call test_record,modules,$(lastword $(subst /, ,$(dir $@)))/$(basename $(notdir $@)),FAIL,$@.log); \
6061
exit 1; \
6162
fi; \
6263
FOUND=$$(grep ^$< $@.log | head -1 | sed 's/:.*//;s/.*\[//;s/\].*//'); \
@@ -65,9 +66,11 @@ $(BUILD_DIR)/tests/modules/%: src/tests/modules/%.unlang $(BUILD_DIR)/tests/modu
6566
cat $@.log; \
6667
echo "# $@.log"; \
6768
echo MODULE_TEST_DIR=$(dir $<) MODULE_TEST_UNLANG=$< $(TESTBIN)/unittest -D share -d src/tests/modules/ -i $@.attrs -f $@.attrs -xx; \
69+
$(call test_record,modules,$(lastword $(subst /, ,$(dir $@)))/$(basename $(notdir $@)),FAIL,$@.log); \
6870
exit 1; \
6971
fi \
7072
fi
73+
@$(call test_record,modules,$(lastword $(subst /, ,$(dir $@)))/$(basename $(notdir $@)),PASS,$@.log)
7174
@touch $@
7275

7376
#
@@ -140,7 +143,9 @@ $(foreach x,$(MODULE_ATTRS_NEEDS),$(eval $(call MODULE_COPY_ATTR,$(subst src/,,$
140143
$(foreach x,$(MODULE_UNLANG),$(eval $(call MODULE_FILE_TARGET,$(patsubst %.unlang,%,$(subst src/,,$x)))))
141144
$(foreach x,$(MODULE_TESTS),$(eval $(call MODULE_TEST_TARGET,$x)))
142145

146+
ifeq "$(RECORDING)" ""
143147
$(TESTS.MODULES_FILES): $(TESTS.AUTH_FILES)
148+
endif
144149

145150
.PHONY: clean.modules.test
146151
clean.modules.test:

src/tests/sql_nas_table/all.mk

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ sql_nas_table_bootstrap:
5252
$(OUTPUT)/%: $(DIR)/% | $(TEST).radiusd_kill sql_nas_table_bootstrap $(TEST).radiusd_start
5353
${Q}echo "SQL_NASTABLE-TEST"
5454
${Q}mkdir -p $(dir $@)
55-
${Q}[ -f $(dir $@)/radiusd.pid ] || exit 1
55+
${Q}if ! [ -f $(dir $@)/radiusd.pid ]; then $(call test_record,sql_nas_table,$(notdir $@),FAIL,); exit 1; fi
5656
${Q}if ! $(TESTBIN)/radclient $(ARGV) -xf src/tests/sql_nas_table/auth.txt -D share/ 127.0.0.1:$(PORT) auth $(SECRET) 1> $(SQL_NASTABLE_BUILD_DIR)/radclient.log 2>&1; then \
5757
echo "FAILED"; \
5858
rm -f $(BUILD_DIR)/tests/test.sql_nas_table; \
@@ -62,9 +62,10 @@ $(OUTPUT)/%: $(DIR)/% | $(TEST).radiusd_kill sql_nas_table_bootstrap $(TEST).rad
6262
echo ==============================; \
6363
echo "RADIUSD: $(RADIUSD_RUN)"; \
6464
echo "SQL_NASTABLE: $(TESTBIN)/radclient $(ARGV) -f $< -xF -d src/tests/sql_nas_table/config -D share/ 127.0.0.1:$(PORT) auth $(SECRET)"; \
65+
$(call test_record,sql_nas_table,$(notdir $@),FAIL,$(SQL_NASTABLE_BUILD_DIR)/radclient.log); \
6566
exit 1; \
6667
fi
67-
68+
${Q}$(call test_record,sql_nas_table,$(notdir $@),PASS,$(SQL_NASTABLE_BUILD_DIR)/radclient.log)
6869
${Q}touch $@
6970

7071
$(TEST):

src/tests/test_record.mk

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#
2+
# Per-test result recorder. Invoked from individual test recipes as:
3+
#
4+
# $(call test_record,<category>,<name>,<status>,<logfile>)
5+
#
6+
# Activated by touching $(BUILD_DIR)/tests/results.tsv
7+
#
8+
9+
BUILD_DIR ?= $(top_builddir)/build
10+
RESULTS_TSV ?= $(BUILD_DIR)/tests/results.tsv
11+
12+
# When the TSV exists, recording is on AND categories run independently
13+
# (the inter-category file deps are skipped) so make -k can collect every
14+
# test result instead of stopping at the first failed phase.
15+
RECORDING := $(wildcard $(RESULTS_TSV))
16+
17+
test_record = [ -e $(RESULTS_TSV) ] && printf '%s\t%s\t%s\t%s\n' '$(1)' '$(2)' '$(3)' '$(4)' >> $(RESULTS_TSV) || true

src/tests/unit/all.mk

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,10 +36,13 @@ $(BUILD_DIR)/share/dictionary: $(top_srcdir)/share/dictionary $(top_srcdir)/shar
3636
#
3737
$(BUILD_DIR)/tests/unit/%: $(DIR)/% $(BUILD_DIR)/bin/radattr $(TESTBINDIR)/radattr $(BUILD_DIR)/share/dictionary | $(BUILD_DIR)/tests/unit
3838
@echo UNIT-TEST $(notdir $@)
39-
@if ! $(TESTBIN)/radattr -D $(BUILD_DIR)/share $<; then \
39+
@if ! $(TESTBIN)/radattr -D $(BUILD_DIR)/share $< > $@.log 2>&1; then \
40+
cat $@.log; \
4041
echo "$(TESTBIN)/radattr -D $(BUILD_DIR)/share $<"; \
42+
$(call test_record,unit,$(notdir $@),FAIL,$@.log); \
4143
exit 1; \
4244
fi
45+
@$(call test_record,unit,$(notdir $@),PASS,$@.log)
4346
@touch $@
4447

4548
#

0 commit comments

Comments
 (0)