Skip to content

Commit 297e903

Browse files
committed
CI: Summarise test results; capture failure logs; keep going on failure
1 parent cd0d7b9 commit 297e903

27 files changed

Lines changed: 243 additions & 14 deletions

File tree

.github/actions/ci-tests/action.yml

Lines changed: 34 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -237,10 +237,13 @@ runs:
237237
run: |
238238
[ -d /opt/openssl ] && export PATH=/opt/openssl/bin:$PATH
239239
240-
make -j `nproc` test.keywords test.unit test.auth test.digest
241-
make test.modules.redis test.modules.redis_ippool
242-
make -j `nproc` test.modules
243-
make ci-test
240+
# Activate per-test result recording for the summary table.
241+
mkdir -p build/tests && touch build/tests/results.tsv
242+
243+
make -j `nproc` -k test.keywords test.unit test.auth test.digest || :
244+
make -k test.modules.redis test.modules.redis_ippool || :
245+
make -j `nproc` -k test.modules || :
246+
make -k ci-test
244247
245248
env:
246249
USE_DOCKER: ${{ inputs.use_docker }}
@@ -269,6 +272,33 @@ runs:
269272
KAFKA_TEST_SERVER_PORT: ${{ inputs.kafka_test_server_port }}
270273
CACHE_MEMCACHED_TEST_SERVER: ${{ inputs.memcached_test_server }}
271274

275+
- name: Test summary
276+
if: ${{ always() }}
277+
shell: bash
278+
run: |
279+
./scripts/ci/ci-summary.sh build/tests/results.tsv >> "$GITHUB_STEP_SUMMARY" || :
280+
281+
- name: Stage failed test logs
282+
if: ${{ failure() }}
283+
shell: bash
284+
run: |
285+
mkdir -p build/tests/failed
286+
[ -s build/tests/results.tsv ] && cp build/tests/results.tsv build/tests/failed/
287+
awk -F'\t' '$3=="FAIL"{print $1"\t"$2"\t"$4}' build/tests/results.tsv 2>/dev/null | \
288+
while IFS=$(printf '\t') read -r cat name logf; do
289+
[ -n "$logf" ] && [ -f "$logf" ] || continue
290+
mkdir -p "build/tests/failed/$cat"
291+
cp "$logf" "build/tests/failed/$cat/$name.log"
292+
done
293+
294+
- name: Upload failed test logs
295+
if: ${{ failure() }}
296+
uses: actions/upload-artifact@v6
297+
with:
298+
name: ci-test-logs-${{ inputs.use_docker == 'true' && 'docker' || 'host' }}
299+
path: build/tests/failed
300+
if-no-files-found: ignore
301+
272302
# Restore ucf
273303
- name: Restore ucf
274304
if: ${{ inputs.use_docker != 'true' }}

scripts/ci/ci-summary.sh

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
#!/bin/sh
2+
#
3+
# scripts/ci/ci-summary.sh
4+
#
5+
# Wrapped awk script that reads the test record file and emits a markdown
6+
# summary to stdout, intended for piping into $GITHUB_STEP_SUMMARY.
7+
#
8+
9+
set -eu
10+
11+
file=${1:-${CI_TEST_RECORD_FILE:-build/tests/results.tsv}}
12+
13+
if [ ! -s "$file" ]; then
14+
echo "## Test results"
15+
echo
16+
echo "_No results recorded._"
17+
exit 0
18+
fi
19+
20+
awk -F'\t' '
21+
{
22+
key = $1 "\t" $2
23+
cat[key] = $1
24+
name[key] = $2
25+
status[key] = $3
26+
logf[key] = $4
27+
seen[key] = NR # last-wins
28+
}
29+
END {
30+
# Per-category counts
31+
for (k in seen) {
32+
c = cat[k]
33+
cats[c] = 1
34+
if (status[k] == "PASS") { cat_pass[c]++; total_pass++ }
35+
else if (status[k] == "FAIL") { cat_fail[c]++; total_fail++ }
36+
total++
37+
}
38+
39+
# Header
40+
printf("## Test results\n\n")
41+
if (total_fail > 0) {
42+
printf("**%d / %d passed** (%d failed)\n\n", total_pass, total, total_fail)
43+
} else {
44+
printf("**%d / %d passed**\n\n", total_pass, total)
45+
}
46+
47+
# Failure logs as collapsible blocks
48+
if (total_fail > 0) {
49+
printf("### Failure logs (last 20 lines)\n\n")
50+
for (k in seen) {
51+
if (status[k] != "FAIL") continue
52+
printf("<details><summary><code>%s/%s</code> &mdash; <code>%s</code></summary>\n\n",
53+
cat[k], name[k], (logf[k] != "" ? logf[k] : "(no log captured)"))
54+
if (logf[k] != "") {
55+
printf("```\n")
56+
n = 0
57+
while ((getline line < logf[k]) > 0) buf[++n % 20] = line
58+
close(logf[k])
59+
start = (n < 20) ? 1 : n - 19
60+
for (j = start; j <= n; j++) print buf[j % 20]
61+
delete buf
62+
printf("```\n")
63+
} else {
64+
printf("_No log file recorded._\n")
65+
}
66+
printf("\n</details>\n\n")
67+
}
68+
printf("---\n\n")
69+
}
70+
71+
# Per-category sections. Categories with failures float to the top
72+
# via a single insertion sort; otherwise alphabetical.
73+
n = 0
74+
for (c in cats) ordered[++n] = c
75+
for (i = 2; i <= n; i++) {
76+
cur = ordered[i]; j = i - 1
77+
while (j >= 1) {
78+
a = ordered[j]; b = cur
79+
af = (cat_fail[a] ? 0 : 1); bf = (cat_fail[b] ? 0 : 1)
80+
if (af < bf || (af == bf && a < b)) break
81+
ordered[j+1] = ordered[j]; j--
82+
}
83+
ordered[j+1] = cur
84+
}
85+
86+
for (i = 1; i <= n; i++) {
87+
c = ordered[i]
88+
cp = cat_pass[c] + 0; cf = cat_fail[c] + 0
89+
ct = cp + cf
90+
if (cf > 0) printf("### %s (%d/%d, %d failed)\n\n", c, cp, ct, cf)
91+
else printf("### %s (%d/%d)\n\n", c, cp, ct)
92+
printf("| Test | Status |\n|---|---|\n")
93+
# FAIL rows first, then PASS, each in test-execution order.
94+
for (k in seen) if (cat[k] == c && status[k] == "FAIL") emit(k)
95+
for (k in seen) if (cat[k] == c && status[k] == "PASS") emit(k)
96+
printf("\n")
97+
}
98+
}
99+
100+
function emit(k) {
101+
printf("| `%s` | %s |\n", name[k], status[k])
102+
}
103+
' "$file"

scripts/ci/package-test.mk

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,4 +52,5 @@ POST_INSTALL_RADIUSD_BIN_ARG:=RADIUSD_BIN=$(RADIUSD_BIN)
5252
.PHONY: package-test
5353
package-test: test.eap
5454

55+
include $(top_srcdir)/src/tests/test_record.mk
5556
include $(DIR)/all.mk

src/tests/all.mk

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ PORT := $(if $(PORT),$(PORT),12340)
66
SECRET := $(if $(SECRET),$(SECRET),testing123)
77
DICT_PATH := $(top_srcdir)/share/dictionary
88

9+
include $(top_srcdir)/src/tests/test_record.mk
10+
911
#
1012
# We need the 'git-lfs' installed to fetch some binary files.
1113
#

src/tests/auth/all.mk

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@ $(OUTPUT)/%: $(DIR)/% $(OUTPUT)/%.attrs $(TEST_BIN_DIR)/unit_test_module | $(AUT
100100
echo "# $@.log"; \
101101
echo "TESTDIR=$(notdir $@) $(TEST_BIN)/unit_test_module -D share/dictionary -d src/tests/auth/ -i \"$@.attrs\" -f \"$@.attrs\" -r \"$@\" -xxx > \"$@.log\" 2>&1"; \
102102
rm -f $(BUILD_DIR)/tests/test.auth; \
103+
$(call test_record,auth,$(notdir $@),FAIL,$@.log); \
103104
exit 1; \
104105
fi; \
105106
FOUND=$$(grep ^$< $@.log | head -1 | sed 's/:.*//;s/.*\[//;s/\].*//'); \
@@ -109,8 +110,10 @@ $(OUTPUT)/%: $(DIR)/% $(OUTPUT)/%.attrs $(TEST_BIN_DIR)/unit_test_module | $(AUT
109110
echo "# $@.log"; \
110111
echo "TESTDIR=$(notdir $@) $(TEST_BIN)/unit_test_module -D share/dictionary -d src/tests/auth/ -i \"$@.attrs\" -f \"$@.attrs\" -r \"$@\" -xxx > \"$@.log\" 2>&1"; \
111112
rm -f $(BUILD_DIR)/tests/test.auth; \
113+
$(call test_record,auth,$(notdir $@),FAIL,$@.log); \
112114
exit 1; \
113115
else \
114116
touch "$@"; \
115117
fi \
116118
fi
119+
@$(call test_record,auth,$(notdir $@),PASS,$@.log)

src/tests/bin/all.mk

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,8 +57,10 @@ $(BUILD_DIR)/tests/bin/%: $(BUILD_DIR)/bin/local/%
5757
echo LOG in $@.log; \
5858
cat $@.log; \
5959
echo $(TEST_BIN)/$(notdir $<) $($(notdir $@).ARGS); \
60+
$(call test_record,bin,$(notdir $@),FAIL,$@.log); \
6061
exit 1; \
6162
fi
63+
@$(call test_record,bin,$(notdir $@),PASS,$@.log)
6264
${Q}touch $@
6365

6466
#

src/tests/config/all.mk

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ $(OUTPUT)/%: $(DIR)/% $(TEST_BIN_DIR)/unit_test_module | $(CONFIG_LIBS)
6767
echo "# $@.log"; \
6868
echo $(CMD); \
6969
rm -f $(BUILD_DIR)/tests/test.config; \
70+
$(call test_record,config,$(notdir $@),FAIL,$@.log); \
7071
exit 1; \
7172
fi; \
7273
FOUND=$$(grep 'Error : src/tests/config/' $@.log | egrep -v -- '-->' | head -1 | sed 's/]:.*//;s/.*\[//;s/\].*//'); \
@@ -76,9 +77,11 @@ $(OUTPUT)/%: $(DIR)/% $(TEST_BIN_DIR)/unit_test_module | $(CONFIG_LIBS)
7677
echo "# $@.log"; \
7778
echo $(CMD); \
7879
rm -f $(BUILD_DIR)/tests/test.config; \
80+
$(call test_record,config,$(notdir $@),FAIL,$@.log); \
7981
exit 1; \
8082
fi; \
8183
fi
84+
@$(call test_record,config,$(notdir $@),PASS,$@.log)
8285
touch "$@"
8386

8487
$(TEST):

src/tests/detail/all.mk

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,16 @@ $(OUTPUT)/%: $(DIR)/% $(addprefix ${BUILD_DIR}/lib/,proto_detail.la proto_detail
2323
${Q}if ! $(TEST_BIN)/radiusd -d $(DIR)/config -D ${top_srcdir}/share/dictionary -X > $@.log; then \
2424
tail $@.log; \
2525
echo "cp $< $(dir $@)/detail.txt; $(TEST_BIN)/radiusd -d $(DIR)/config -D ${top_srcdir}/share/dictionary -X "; \
26+
$(call test_record,detail,$(notdir $@),FAIL,$@.log); \
2627
exit 1; \
2728
fi
2829
${Q}if [ ! -e $(dir $@)/processed ] ; then \
2930
tail $@.log; \
3031
echo "Processing $< failed to produce expected output $(dir $@)/processed"; \
32+
$(call test_record,detail,$(notdir $@),FAIL,$@.log); \
3133
exit 1; \
3234
fi
35+
@$(call test_record,detail,$(notdir $@),PASS,$@.log)
3336
${Q}touch $@
3437

3538
.NO_PARALLEL: $(TEST)

src/tests/dict/all.mk

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,10 @@ $(OUTPUT)/%.dict: $(DIR)/%.dict $(TEST_BIN_DIR)/unit_test_attribute
2525
cat "$@.log"; \
2626
echo "# $@.log"; \
2727
echo "$(TEST_BIN)/unit_test_attribute -D $(top_srcdir)/share/dictionary -o "$@" -xx '$@.txt'"; \
28+
$(call test_record,dict,$(notdir $@),FAIL,$@.log); \
2829
exit 1; \
2930
fi
31+
@$(call test_record,dict,$(notdir $@),PASS,$@.log)
3032

3133
# And the actual script to run each test.
3234
#
@@ -43,8 +45,10 @@ $(OUTPUT)/%.dict: $(DIR)/%.dict $(TEST_BIN_DIR)/unit_test_attribute
4345
cat "$@.log"; \
4446
echo "# $@.log"; \
4547
echo "$(TEST_BIN)/unit_test_attribute -D $(top_srcdir)/share/dictionary -o "$@" -xx '$@.txt'"; \
48+
$(call test_record,dict,$(notdir $@),FAIL,$@.log); \
4649
exit 1; \
4750
fi
51+
@$(call test_record,dict,$(notdir $@),PASS,$@.log)
4852

4953
#
5054
# Tests which are supposed to fail.
@@ -65,10 +69,12 @@ $(OUTPUT)/%.error: $(DIR)/%.error $(TEST_BIN_DIR)/unit_test_attribute
6569
cat "$@.log"; \
6670
echo "# $@.log"; \
6771
echo "$(TEST_BIN)/unit_test_attribute -D $(top_srcdir)/share/dictionary -xx '$@.txt'"; \
72+
$(call test_record,dict,$(notdir $@),FAIL,$@.log); \
6873
exit 1; \
6974
fi
7075
${Q}sed 's,${top_srcdir}/,,g' < "$@.log" > "$@.out"
7176
${Q}if ! diff "$@.out" "$(subst .error,,$<).out" ; then \
7277
echo "diff $@.out $(subst .error,,$<).out"; \
7378
echo "FAILED"; \
7479
fi
80+
@$(call test_record,dict,$(notdir $@),PASS,$@.log)

src/tests/digest/all.mk

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,10 @@ $(OUTPUT)/%: $(DIR)/% | $(TEST).radiusd_kill $(TEST).radiusd_start
4848
$(MAKE) --no-print-directory test.digest.radiusd_kill; \
4949
echo "RADIUSD: $(RADIUSD_RUN)"; \
5050
echo "RADCLIENT: $(TEST_BIN)/radclient -f $@.request -xF -d src/tests/digest/config -D share/dictionary 127.0.0.1:$(digest_port) auth $(SECRET)"; \
51+
$(call test_record,digest,$(TARGET)_$${_num},FAIL,$@.out); \
5152
exit 1; \
5253
fi; \
54+
$(call test_record,digest,$(TARGET)_$${_num},PASS,$@.out); \
5355
touch $@; \
5456
done
5557

0 commit comments

Comments
 (0)