Skip to content

Commit c358fc4

Browse files
terryburtonalandekok
authored andcommitted
CI: Summarise test results; capture failure logs; keep going on failure
We skip ordering when recording test results (for example in CI) since that's only really useful for ensuring stable output for local testing.
1 parent f58e9d3 commit c358fc4

11 files changed

Lines changed: 184 additions & 4 deletions

File tree

.github/workflows/ci.yml

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -372,7 +372,35 @@ jobs:
372372
openssl dhparam -out ./raddb/certs/dh -2 512 || \
373373
true
374374
375-
make ci-test
375+
# Activate per-test result recording for the summary table.
376+
mkdir -p build/tests && touch build/tests/results.tsv
377+
378+
make -k ci-test
379+
380+
- name: Test summary
381+
if: ${{ always() }}
382+
run: |
383+
./scripts/ci/ci-summary.sh build/tests/results.tsv >> "$GITHUB_STEP_SUMMARY" || :
384+
385+
- name: Stage failed test logs
386+
if: ${{ failure() }}
387+
run: |
388+
mkdir -p build/tests/failed
389+
[ -s build/tests/results.tsv ] && cp build/tests/results.tsv build/tests/failed/
390+
awk -F'\t' '$3=="FAIL"{print $1"\t"$2"\t"$4}' build/tests/results.tsv 2>/dev/null | \
391+
while IFS=$(printf '\t') read -r cat name logf; do
392+
[ -n "$logf" ] && [ -f "$logf" ] || continue
393+
mkdir -p "build/tests/failed/$cat"
394+
cp "$logf" "build/tests/failed/$cat/$name.log"
395+
done
396+
397+
- name: Upload failed test logs
398+
if: ${{ failure() }}
399+
uses: actions/upload-artifact@v6
400+
with:
401+
name: ci-test-logs-${{ matrix.os.name }}-${{ matrix.env.NAME }}
402+
path: build/tests/failed
403+
if-no-files-found: ignore
376404

377405
- name: Show debug logs on failure
378406
if: ${{ failure() }}

scripts/ci/ci-summary.sh

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
#!/bin/sh
2+
#
3+
# scripts/ci/ci-summary.sh
4+
#
5+
# Wrapped awk script that reads the test record file and emits a markdown
6+
# summary to stdout, intended for piping into $GITHUB_STEP_SUMMARY.
7+
#
8+
9+
set -eu
10+
11+
file=${1:-${CI_TEST_RECORD_FILE:-build/tests/results.tsv}}
12+
13+
if [ ! -s "$file" ]; then
14+
echo "## Test results"
15+
echo
16+
echo "_No results recorded._"
17+
exit 0
18+
fi
19+
20+
awk -F'\t' '
21+
{
22+
key = $1 "\t" $2
23+
cat[key] = $1
24+
name[key] = $2
25+
status[key] = $3
26+
logf[key] = $4
27+
seen[key] = NR # last-wins
28+
}
29+
END {
30+
# Per-category counts
31+
for (k in seen) {
32+
c = cat[k]
33+
cats[c] = 1
34+
if (status[k] == "PASS") { cat_pass[c]++; total_pass++ }
35+
else if (status[k] == "FAIL") { cat_fail[c]++; total_fail++ }
36+
total++
37+
}
38+
39+
# Header
40+
printf("## Test results\n\n")
41+
if (total_fail > 0) {
42+
printf("**%d / %d passed** (%d failed)\n\n", total_pass, total, total_fail)
43+
} else {
44+
printf("**%d / %d passed**\n\n", total_pass, total)
45+
}
46+
47+
# Failure logs as collapsible blocks
48+
if (total_fail > 0) {
49+
printf("### Failure logs (last 20 lines)\n\n")
50+
for (k in seen) {
51+
if (status[k] != "FAIL") continue
52+
printf("<details><summary><code>%s/%s</code> &mdash; <code>%s</code></summary>\n\n",
53+
cat[k], name[k], (logf[k] != "" ? logf[k] : "(no log captured)"))
54+
if (logf[k] != "") {
55+
printf("```\n")
56+
n = 0
57+
while ((getline line < logf[k]) > 0) buf[++n % 20] = line
58+
close(logf[k])
59+
start = (n < 20) ? 1 : n - 19
60+
for (j = start; j <= n; j++) print buf[j % 20]
61+
delete buf
62+
printf("```\n")
63+
} else {
64+
printf("_No log file recorded._\n")
65+
}
66+
printf("\n</details>\n\n")
67+
}
68+
printf("---\n\n")
69+
}
70+
71+
# Per-category sections. Categories with failures float to the top
72+
# via a single insertion sort; otherwise alphabetical.
73+
n = 0
74+
for (c in cats) ordered[++n] = c
75+
for (i = 2; i <= n; i++) {
76+
cur = ordered[i]; j = i - 1
77+
while (j >= 1) {
78+
a = ordered[j]; b = cur
79+
af = (cat_fail[a] ? 0 : 1); bf = (cat_fail[b] ? 0 : 1)
80+
if (af < bf || (af == bf && a < b)) break
81+
ordered[j+1] = ordered[j]; j--
82+
}
83+
ordered[j+1] = cur
84+
}
85+
86+
for (i = 1; i <= n; i++) {
87+
c = ordered[i]
88+
cp = cat_pass[c] + 0; cf = cat_fail[c] + 0
89+
ct = cp + cf
90+
if (cf > 0) printf("### %s (%d/%d, %d failed)\n\n", c, cp, ct, cf)
91+
else printf("### %s (%d/%d)\n\n", c, cp, ct)
92+
printf("| Test | Status |\n|---|---|\n")
93+
# FAIL rows first, then PASS, each in test-execution order.
94+
for (k in seen) if (cat[k] == c && status[k] == "FAIL") emit(k)
95+
for (k in seen) if (cat[k] == c && status[k] == "PASS") emit(k)
96+
printf("\n")
97+
}
98+
}
99+
100+
function emit(k) {
101+
printf("| `%s` | %s |\n", name[k], status[k])
102+
}
103+
' "$file"

src/tests/Makefile

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
##
88
#
99
include ../../Make.inc
10+
include $(top_srcdir)/src/tests/test_record.mk
1011

1112
#
1213
# You can watch what it's doing by:
@@ -268,9 +269,11 @@ $(BUILD_PATH)/tests/eap/%.ok: $(top_builddir)/src/tests/%.conf | radiusd.kill $(
268269
tail -10 $(LOG); \
269270
echo "===================="; \
270271
$(MAKE) radiusd.kill; \
272+
$(call test_record,eap,$(notdir $(patsubst %.conf,%,$<)),FAIL,$(LOG)); \
271273
exit 1; \
272274
fi
273275
@echo
276+
${Q}$(call test_record,eap,$(notdir $(patsubst %.conf,%,$<)),PASS,$(LOG))
274277
${Q}touch $@
275278

276279
#
@@ -300,21 +303,25 @@ $(BUILD_PATH)/tests/eap/${1}-${2}.ok: $(BUILD_PATH)/tests/eap/${1}-${2}.conf
300303
tail -10 $$(LOG); \
301304
echo "===================="; \
302305
$(MAKE) radiusd.kill; \
306+
$$(call test_record,eap,$$(notdir $$(patsubst %.ok,%,$$@)),FAIL,$$(LOG)); \
303307
exit 1; \
304308
elif ! grep -q '^SSL: Using TLS version TLSv${2}$$$$' $$(patsubst %.ok,%,$$@).log; then \
305309
echo " - " FAILED - not using TLS version ${2}; \
306310
echo ">>> cmd -" $$(CMD) -r 1; \
307311
echo ">>> log -" $$(LOG); \
308312
$(MAKE) radiusd.kill; \
313+
$$(call test_record,eap,$$(notdir $$(patsubst %.ok,%,$$@)),FAIL,$$(LOG)); \
309314
exit 1; \
310315
elif ! grep -q '^OpenSSL: Handshake finished - resumed=1$$$$' $$(patsubst %.ok,%,$$@).log; then \
311316
echo " - " FAILED - did not use resumption; \
312317
echo ">>> cmd -" $$(CMD) -r 1; \
313318
echo ">>> log -" $$(LOG); \
314319
$(MAKE) radiusd.kill; \
320+
$$(call test_record,eap,$$(notdir $$(patsubst %.ok,%,$$@)),FAIL,$$(LOG)); \
315321
exit 1; \
316322
fi
317323
@echo
324+
${Q}$$(call test_record,eap,$$(notdir $$(patsubst %.ok,%,$$@)),PASS,$$(LOG))
318325
${Q}touch $$@
319326

320327
# EAP-FAST doesn't do TLS 1.3

src/tests/all.mk

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ PORT := 12340
33
SECRET := testing123
44
DICT_PATH := $(top_srcdir)/share
55

6+
include $(top_srcdir)/src/tests/test_record.mk
7+
68
#
79
# Pull all of the autoconf stuff into here.
810
#

src/tests/auth/all.mk

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ $(BUILD_DIR)/tests/auth/%: $(DIR)/% $(BUILD_DIR)/tests/auth/%.attrs $(TESTBINDIR
8989
cat $@.log; \
9090
echo "# $@.log"; \
9191
echo "TESTDIR=$(notdir $@) $(TESTBIN)/unittest -D share -d src/tests/auth/ -i $@.attrs -f $@.attrs -xxx > $@.log 2>&1"; \
92+
$(call test_record,auth,$(notdir $@),FAIL,$@.log); \
9293
exit 1; \
9394
fi; \
9495
FOUND=$$(grep ^$< $@.log | head -1 | sed 's/:.*//;s/.*\[//;s/\].*//'); \
@@ -97,9 +98,11 @@ $(BUILD_DIR)/tests/auth/%: $(DIR)/% $(BUILD_DIR)/tests/auth/%.attrs $(TESTBINDIR
9798
cat $@.log; \
9899
echo "# $@.log"; \
99100
echo "TESTDIR=$(notdir $@) $(TESTBIN)/unittest -D share -d src/tests/auth/ -i $@.attrs -f $@.attrs -xxx > $@.log 2>&1"; \
101+
$(call test_record,auth,$(notdir $@),FAIL,$@.log); \
100102
exit 1; \
101103
fi \
102104
fi
105+
@$(call test_record,auth,$(notdir $@),PASS,$@.log)
103106
@touch $@
104107

105108
#
@@ -112,7 +115,9 @@ TESTS.AUTH_FILES := $(addprefix $(BUILD_DIR)/tests/auth/,$(AUTH_FILES))
112115
#
113116
tests.auth: $(TESTS.AUTH_FILES)
114117

118+
ifeq "$(RECORDING)" ""
115119
$(TESTS.AUTH_FILES): $(TESTS.KEYWORDS_FILES)
120+
endif
116121

117122
.PHONY: clean.tests.auth
118123
clean.tests.auth:

src/tests/keywords/all.mk

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,7 @@ $(BUILD_DIR)/tests/keywords/%: ${DIR}/% $(BUILD_DIR)/tests/keywords/%.attrs $(TE
102102
cat $@.log; \
103103
echo "# $@.log"; \
104104
echo KEYWORD=$(notdir $@) $(TESTBIN)/unittest -D share -d src/tests/keywords/ -i $@.attrs -f $@.attrs -xx; \
105+
$(call test_record,keywords,$(notdir $@),FAIL,$@.log); \
105106
exit 1; \
106107
fi; \
107108
FOUND=$$(grep ^$< $@.log | head -1 | sed 's/:.*//;s/.*\[//;s/\].*//'); \
@@ -110,9 +111,11 @@ $(BUILD_DIR)/tests/keywords/%: ${DIR}/% $(BUILD_DIR)/tests/keywords/%.attrs $(TE
110111
cat $@.log; \
111112
echo "# $@.log"; \
112113
echo KEYWORD=$(notdir $@) $(TESTBIN)/unittest -D share -d src/tests/keywords/ -i $@.attrs -f $@.attrs -xx; \
114+
$(call test_record,keywords,$(notdir $@),FAIL,$@.log); \
113115
exit 1; \
114116
fi \
115117
fi
118+
@$(call test_record,keywords,$(notdir $@),PASS,$@.log)
116119
@touch $@
117120

118121
#
@@ -125,7 +128,9 @@ TESTS.KEYWORDS_FILES := $(addprefix $(BUILD_DIR)/tests/keywords/,$(KEYWORD_FILES
125128
#
126129
tests.keywords: $(TESTS.KEYWORDS_FILES)
127130

131+
ifeq "$(RECORDING)" ""
128132
$(TESTS.KEYWORDS_FILES): $(TESTS.XLAT_FILES) $(TESTS.MAP_FILES)
133+
endif
129134

130135
.PHONY: clean.tests.keywords
131136
clean.tests.keywords:

src/tests/modules/test.mk

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ $(BUILD_DIR)/tests/modules/%: src/tests/modules/%.unlang $(BUILD_DIR)/tests/modu
5757
cat $@.log; \
5858
echo "# $@.log"; \
5959
echo LANG=C TZ=UTC MODULE_TEST_DIR=$(dir $<) MODULE_TEST_UNLANG=$< $(TESTBIN)/unittest -D share -d src/tests/modules/ -i $@.attrs -f $@.attrs -xx; \
60+
$(call test_record,modules,$(lastword $(subst /, ,$(dir $@)))/$(basename $(notdir $@)),FAIL,$@.log); \
6061
exit 1; \
6162
fi; \
6263
FOUND=$$(grep ^$< $@.log | head -1 | sed 's/:.*//;s/.*\[//;s/\].*//'); \
@@ -65,9 +66,11 @@ $(BUILD_DIR)/tests/modules/%: src/tests/modules/%.unlang $(BUILD_DIR)/tests/modu
6566
cat $@.log; \
6667
echo "# $@.log"; \
6768
echo LANG=C TZ=UTC MODULE_TEST_DIR=$(dir $<) MODULE_TEST_UNLANG=$< $(TESTBIN)/unittest -D share -d src/tests/modules/ -i $@.attrs -f $@.attrs -xx; \
69+
$(call test_record,modules,$(lastword $(subst /, ,$(dir $@)))/$(basename $(notdir $@)),FAIL,$@.log); \
6870
exit 1; \
6971
fi \
7072
fi
73+
@$(call test_record,modules,$(lastword $(subst /, ,$(dir $@)))/$(basename $(notdir $@)),PASS,$@.log)
7174
@touch $@
7275

7376
#
@@ -140,7 +143,9 @@ $(foreach x,$(MODULE_ATTRS_NEEDS),$(eval $(call MODULE_COPY_ATTR,$(subst src/,,$
140143
$(foreach x,$(MODULE_UNLANG),$(eval $(call MODULE_FILE_TARGET,$(patsubst %.unlang,%,$(subst src/,,$x)))))
141144
$(foreach x,$(MODULE_TESTS),$(eval $(call MODULE_TEST_TARGET,$x)))
142145

146+
ifeq "$(RECORDING)" ""
143147
$(TESTS.MODULES_FILES): $(TESTS.AUTH_FILES)
148+
endif
144149

145150
.PHONY: clean.modules.test
146151
clean.modules.test:

src/tests/sql_nas_table/all.mk

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ sql_nas_table_bootstrap:
5252
$(OUTPUT)/%: $(DIR)/% | $(TEST).radiusd_kill sql_nas_table_bootstrap $(TEST).radiusd_start
5353
${Q}echo "SQL_NASTABLE-TEST"
5454
${Q}mkdir -p $(dir $@)
55-
${Q}[ -f $(dir $@)/radiusd.pid ] || exit 1
55+
${Q}if ! [ -f $(dir $@)/radiusd.pid ]; then $(call test_record,sql_nas_table,$(notdir $@),FAIL,); exit 1; fi
5656
${Q}if ! $(TESTBIN)/radclient $(ARGV) -xf src/tests/sql_nas_table/auth.txt -D share/ 127.0.0.1:$(PORT) auth $(SECRET) 1> $(SQL_NASTABLE_BUILD_DIR)/radclient.log 2>&1; then \
5757
echo "FAILED"; \
5858
rm -f $(BUILD_DIR)/tests/test.sql_nas_table; \
@@ -62,9 +62,10 @@ $(OUTPUT)/%: $(DIR)/% | $(TEST).radiusd_kill sql_nas_table_bootstrap $(TEST).rad
6262
echo ==============================; \
6363
echo "RADIUSD: $(RADIUSD_RUN)"; \
6464
echo "SQL_NASTABLE: $(TESTBIN)/radclient $(ARGV) -f $< -xF -d src/tests/sql_nas_table/config -D share/ 127.0.0.1:$(PORT) auth $(SECRET)"; \
65+
$(call test_record,sql_nas_table,$(notdir $@),FAIL,$(SQL_NASTABLE_BUILD_DIR)/radclient.log); \
6566
exit 1; \
6667
fi
67-
68+
${Q}$(call test_record,sql_nas_table,$(notdir $@),PASS,$(SQL_NASTABLE_BUILD_DIR)/radclient.log)
6869
${Q}touch $@
6970

7071
$(TEST):

src/tests/test_record.mk

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#
2+
# Per-test result recorder. Invoked from individual test recipes as:
3+
#
4+
# $(call test_record,<category>,<name>,<status>,<logfile>)
5+
#
6+
# Activated by touching $(BUILD_DIR)/tests/results.tsv
7+
#
8+
9+
BUILD_DIR ?= $(top_builddir)/build
10+
RESULTS_TSV ?= $(BUILD_DIR)/tests/results.tsv
11+
12+
# When the TSV exists, recording is on AND categories run independently
13+
# (the inter-category file deps are skipped) so make -k can collect every
14+
# test result instead of stopping at the first failed phase.
15+
RECORDING := $(wildcard $(RESULTS_TSV))
16+
17+
test_record = [ -e $(RESULTS_TSV) ] && printf '%s\t%s\t%s\t%s\n' '$(1)' '$(2)' '$(3)' '$(4)' >> $(RESULTS_TSV) || true

src/tests/unit/all.mk

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,10 +36,13 @@ $(BUILD_DIR)/share/dictionary: $(top_srcdir)/share/dictionary $(top_srcdir)/shar
3636
#
3737
$(BUILD_DIR)/tests/unit/%: $(DIR)/% $(BUILD_DIR)/bin/radattr $(TESTBINDIR)/radattr $(BUILD_DIR)/share/dictionary | $(BUILD_DIR)/tests/unit
3838
@echo UNIT-TEST $(notdir $@)
39-
@if ! $(TESTBIN)/radattr -D $(BUILD_DIR)/share $<; then \
39+
@if ! $(TESTBIN)/radattr -D $(BUILD_DIR)/share $< > $@.log 2>&1; then \
40+
cat $@.log; \
4041
echo "$(TESTBIN)/radattr -D $(BUILD_DIR)/share $<"; \
42+
$(call test_record,unit,$(notdir $@),FAIL,$@.log); \
4143
exit 1; \
4244
fi
45+
@$(call test_record,unit,$(notdir $@),PASS,$@.log)
4346
@touch $@
4447

4548
#

0 commit comments

Comments
 (0)