Skip to content

Commit f3d4655

Browse files
djmdjmdjmdjm
committed
upstream: basic benchmarking support for the unit test framework enable
with "make UNITTEST_BENCHMARK=yes" ok dtucker@ OpenBSD-Regress-ID: 7f16a2e247f860897ca46ff87bccbe6002a32564
1 parent 609fe2c commit f3d4655

File tree

26 files changed

+580
-105
lines changed

26 files changed

+580
-105
lines changed

‎Makefile.in‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -782,6 +782,13 @@ unit: regress-unit-binaries
782782
OBJ="$(BUILDDIR)/regress" \
783783
$@ && echo $@ tests passed
784784

785+
unit-bench: regress-unit-binaries
786+
cd $(srcdir)/regress || exit $$?; \
787+
$(MAKE) \
788+
.CURDIR="$(abs_top_srcdir)/regress" \
789+
.OBJDIR="$(BUILDDIR)/regress" \
790+
OBJ="$(BUILDDIR)/regress" $@
791+
785792
TEST_SSH_SSHD="$(BUILDDIR)/sshd"
786793

787794
interop-tests t-exec file-tests extra-tests: regress-prep regress-binaries $(TARGETS)

‎regress/Makefile‎

Lines changed: 19 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -292,26 +292,33 @@ t-extra: ${EXTRA_TESTS:=.sh}
292292
interop: ${INTEROP_TARGETS}
293293

294294
# Unit tests, built by top-level Makefile
295-
unit:
295+
unit unit-bench:
296296
set -e ; if test -z "${SKIP_UNIT}" ; then \
297297
V="" ; \
298298
test "x${USE_VALGRIND}" = "x" || \
299299
V=${.CURDIR}/valgrind-unit.sh ; \
300-
$$V ${.OBJDIR}/unittests/sshbuf/test_sshbuf ; \
300+
ARGS=""; \
301+
test "x$@" = "xunit-bench" && ARGS="-b"; \
302+
test "x${UNITTEST_FAST}" = "x" || ARGS="$$ARGS -f"; \
303+
test "x${UNITTEST_SLOW}" = "x" || ARGS="$$ARGS -F"; \
304+
test "x${UNITTEST_VERBOSE}" = "x" || ARGS="$$ARGS -v"; \
305+
test "x${UNITTEST_BENCH_DETAIL}" = "x" || ARGS="$$ARGS -B"; \
306+
test "x${UNITTEST_BENCH_ONLY}" = "x" || ARGS="$$ARGS -O ${UNITTEST_BENCH_ONLY}"; \
307+
$$V ${.OBJDIR}/unittests/sshbuf/test_sshbuf $${ARGS}; \
301308
$$V ${.OBJDIR}/unittests/sshkey/test_sshkey \
302-
-d ${.CURDIR}/unittests/sshkey/testdata ; \
309+
-d ${.CURDIR}/unittests/sshkey/testdata $${ARGS}; \
303310
$$V ${.OBJDIR}/unittests/sshsig/test_sshsig \
304-
-d ${.CURDIR}/unittests/sshsig/testdata ; \
311+
-d ${.CURDIR}/unittests/sshsig/testdata $${ARGS}; \
305312
$$V ${.OBJDIR}/unittests/authopt/test_authopt \
306-
-d ${.CURDIR}/unittests/authopt/testdata ; \
307-
$$V ${.OBJDIR}/unittests/bitmap/test_bitmap ; \
308-
$$V ${.OBJDIR}/unittests/conversion/test_conversion ; \
309-
$$V ${.OBJDIR}/unittests/kex/test_kex ; \
313+
-d ${.CURDIR}/unittests/authopt/testdata $${ARGS}; \
314+
$$V ${.OBJDIR}/unittests/bitmap/test_bitmap $${ARGS}; \
315+
$$V ${.OBJDIR}/unittests/conversion/test_conversion $${ARGS}; \
316+
$$V ${.OBJDIR}/unittests/kex/test_kex $${ARGS}; \
310317
$$V ${.OBJDIR}/unittests/hostkeys/test_hostkeys \
311-
-d ${.CURDIR}/unittests/hostkeys/testdata ; \
312-
$$V ${.OBJDIR}/unittests/match/test_match ; \
313-
$$V ${.OBJDIR}/unittests/misc/test_misc ; \
318+
-d ${.CURDIR}/unittests/hostkeys/testdata $${ARGS}; \
319+
$$V ${.OBJDIR}/unittests/match/test_match $${ARGS}; \
320+
$$V ${.OBJDIR}/unittests/misc/test_misc $${ARGS}; \
314321
if test "x${TEST_SSH_UTF8}" = "xyes" ; then \
315-
$$V ${.OBJDIR}/unittests/utf8/test_utf8 ; \
322+
$$V ${.OBJDIR}/unittests/utf8/test_utf8 $${ARGS}; \
316323
fi \
317324
fi

‎regress/unittests/Makefile.inc‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# $OpenBSD: Makefile.inc,v 1.16 2024/01/11 01:45:58 djm Exp $
1+
# $OpenBSD: Makefile.inc,v 1.17 2025/04/15 04:00:42 djm Exp $
22

33
.include <bsd.own.mk>
44
.include <bsd.obj.mk>
@@ -7,6 +7,9 @@
77
UNITTEST_FAST?= no # Skip slow tests (e.g. less intensive fuzzing).
88
UNITTEST_SLOW?= no # Include slower tests (e.g. more intensive fuzzing).
99
UNITTEST_VERBOSE?= no # Verbose test output (inc. per-test names).
10+
UNITTEST_BENCHMARK?= no # Run unit tests in benchmarking mode.
11+
UNITTEST_BENCH_DETAIL?=no # Detailed benchmark statistics.
12+
UNITTEST_BENCH_ONLY?= # Run only these benchmarks
1013

1114
MALLOC_OPTIONS?= CFGJRSUX
1215
TEST_ENV?= MALLOC_OPTIONS=${MALLOC_OPTIONS}
@@ -69,8 +72,8 @@ DPADD+=${.CURDIR}/../test_helper/libtest_helper.a
6972

7073
.PATH: ${.CURDIR}/${SSHREL}
7174

72-
LDADD+= -lutil
73-
DPADD+= ${LIBUTIL}
75+
LDADD+= -lutil -lm
76+
DPADD+= ${LIBUTIL} ${LIBM}
7477

7578
.if (${OPENSSL:L} == "yes")
7679
LDADD+= -lcrypto
@@ -82,11 +85,21 @@ DPADD+= ${LIBFIDO2} ${LIBCBOR} ${LIBUSBHID}
8285

8386
UNITTEST_ARGS?=
8487

85-
.if (${UNITTEST_VERBOSE:L} != "no")
88+
.if (${UNITTEST_VERBOSE:L:R} != "no")
8689
UNITTEST_ARGS+= -v
8790
.endif
88-
.if (${UNITTEST_FAST:L} != "no")
91+
.if (${UNITTEST_FAST:L:R} != "no")
8992
UNITTEST_ARGS+= -f
90-
.elif (${UNITTEST_SLOW:L} != "no")
93+
.elif (${UNITTEST_SLOW:L:R} != "no")
9194
UNITTEST_ARGS+= -F
9295
.endif
96+
97+
.if (${UNITTEST_BENCHMARK:L:R} != "no")
98+
UNITTEST_ARGS+= -b
99+
.endif
100+
.if (${UNITTEST_BENCH_DETAIL:L:R} != "no")
101+
UNITTEST_ARGS+= -B
102+
.endif
103+
.if (${UNITTEST_BENCH_ONLY:L} != "")
104+
UNITTEST_ARGS+= -O "${UNITTEST_BENCH_ONLY}"
105+
.endif

‎regress/unittests/authopt/Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# $OpenBSD: Makefile,v 1.7 2023/01/15 23:35:10 djm Exp $
1+
# $OpenBSD: Makefile,v 1.8 2025/04/15 04:00:42 djm Exp $
22

33
PROG=test_authopt
44
SRCS=tests.c
@@ -22,6 +22,6 @@ SRCS+=utf8.c
2222
REGRESS_TARGETS=run-regress-${PROG}
2323

2424
run-regress-${PROG}: ${PROG}
25-
env ${TEST_ENV} ./${PROG} -d ${.CURDIR}/testdata
25+
env ${TEST_ENV} ./${PROG} ${UNITTEST_ARGS} -d ${.CURDIR}/testdata
2626

2727
.include <bsd.regress.mk>

‎regress/unittests/authopt/tests.c‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
/* $OpenBSD: tests.c,v 1.3 2021/12/14 21:25:27 deraadt Exp $ */
1+
/* $OpenBSD: tests.c,v 1.4 2025/04/15 04:00:42 djm Exp $ */
22

33
/*
44
* Regress test for keys options functions.
@@ -576,3 +576,9 @@ tests(void)
576576
test_cert_parse();
577577
test_merge();
578578
}
579+
580+
void
581+
benchmarks(void)
582+
{
583+
printf("no benchmarks\n");
584+
}

‎regress/unittests/bitmap/Makefile‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,15 @@
1-
# $OpenBSD: Makefile,v 1.4 2017/12/21 00:41:22 djm Exp $
1+
# $OpenBSD: Makefile,v 1.5 2025/04/15 04:00:42 djm Exp $
22

33
PROG=test_bitmap
44
SRCS=tests.c
55

66
# From usr.sbin/ssh
7-
SRCS+=bitmap.c atomicio.c
7+
SRCS+=bitmap.c atomicio.c misc.c xmalloc.c fatal.c log.c cleanup.c match.c
8+
SRCS+=sshbuf.c sshbuf-getput-basic.c sshbuf-misc.c ssherr.c addr.c addrmatch.c
89

910
REGRESS_TARGETS=run-regress-${PROG}
1011

1112
run-regress-${PROG}: ${PROG}
12-
env ${TEST_ENV} ./${PROG}
13+
env ${TEST_ENV} ./${PROG} ${UNITTEST_ARGS}
1314

1415
.include <bsd.regress.mk>

‎regress/unittests/bitmap/tests.c‎

Lines changed: 20 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
/* $OpenBSD: tests.c,v 1.2 2021/12/14 21:25:27 deraadt Exp $ */
1+
/* $OpenBSD: tests.c,v 1.3 2025/04/15 04:00:42 djm Exp $ */
22
/*
33
* Regress test for bitmap.h bitmap API
44
*
@@ -23,7 +23,7 @@
2323

2424
#include "bitmap.h"
2525

26-
#define NTESTS 131
26+
#define DEFAULT_NTESTS 131
2727

2828
void
2929
tests(void)
@@ -32,10 +32,15 @@ tests(void)
3232
struct bitmap *b;
3333
BIGNUM *bn;
3434
size_t len;
35-
int i, j, k, n;
35+
int i, j, k, n, ntests = DEFAULT_NTESTS;
3636
u_char bbuf[1024], bnbuf[1024];
3737
int r;
3838

39+
if (test_is_fast())
40+
ntests /= 4;
41+
else if (test_is_slow())
42+
ntests *= 2;
43+
3944
TEST_START("bitmap_new");
4045
b = bitmap_new();
4146
ASSERT_PTR_NE(b, NULL);
@@ -44,9 +49,9 @@ tests(void)
4449
TEST_DONE();
4550

4651
TEST_START("bitmap_set_bit / bitmap_test_bit");
47-
for (i = -1; i < NTESTS; i++) {
48-
for (j = -1; j < NTESTS; j++) {
49-
for (k = -1; k < NTESTS; k++) {
52+
for (i = -1; i < ntests; i++) {
53+
for (j = -1; j < ntests; j++) {
54+
for (k = -1; k < ntests; k++) {
5055
bitmap_zero(b);
5156
BN_clear(bn);
5257

@@ -67,7 +72,7 @@ tests(void)
6772

6873
/* Check perfect match between bitmap and bn */
6974
test_subtest_info("match %d/%d/%d", i, j, k);
70-
for (n = 0; n < NTESTS; n++) {
75+
for (n = 0; n < ntests; n++) {
7176
ASSERT_INT_EQ(BN_is_bit_set(bn, n),
7277
bitmap_test_bit(b, n));
7378
}
@@ -99,15 +104,15 @@ tests(void)
99104
bitmap_zero(b);
100105
ASSERT_INT_EQ(bitmap_from_string(b, bnbuf,
101106
len), 0);
102-
for (n = 0; n < NTESTS; n++) {
107+
for (n = 0; n < ntests; n++) {
103108
ASSERT_INT_EQ(BN_is_bit_set(bn, n),
104109
bitmap_test_bit(b, n));
105110
}
106111

107112
/* Test clearing bits */
108113
test_subtest_info("clear %d/%d/%d",
109114
i, j, k);
110-
for (n = 0; n < NTESTS; n++) {
115+
for (n = 0; n < ntests; n++) {
111116
ASSERT_INT_EQ(bitmap_set_bit(b, n), 0);
112117
ASSERT_INT_EQ(BN_set_bit(bn, n), 1);
113118
}
@@ -123,7 +128,7 @@ tests(void)
123128
bitmap_clear_bit(b, k);
124129
BN_clear_bit(bn, k);
125130
}
126-
for (n = 0; n < NTESTS; n++) {
131+
for (n = 0; n < ntests; n++) {
127132
ASSERT_INT_EQ(BN_is_bit_set(bn, n),
128133
bitmap_test_bit(b, n));
129134
}
@@ -135,4 +140,9 @@ tests(void)
135140
TEST_DONE();
136141
#endif
137142
}
143+
void
144+
benchmarks(void)
145+
{
146+
printf("no benchmarks\n");
147+
}
138148

‎regress/unittests/conversion/Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# $OpenBSD: Makefile,v 1.4 2021/01/09 12:24:30 dtucker Exp $
1+
# $OpenBSD: Makefile,v 1.5 2025/04/15 04:00:42 djm Exp $
22

33
PROG=test_conversion
44
SRCS=tests.c
@@ -11,6 +11,6 @@ SRCS+=match.c addr.c addrmatch.c
1111
REGRESS_TARGETS=run-regress-${PROG}
1212

1313
run-regress-${PROG}: ${PROG}
14-
env ${TEST_ENV} ./${PROG}
14+
env ${TEST_ENV} ./${PROG} ${UNITTEST_ARGS}
1515

1616
.include <bsd.regress.mk>

‎regress/unittests/conversion/tests.c‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
/* $OpenBSD: tests.c,v 1.4 2021/12/14 21:25:27 deraadt Exp $ */
1+
/* $OpenBSD: tests.c,v 1.5 2025/04/15 04:00:42 djm Exp $ */
22
/*
33
* Regress test for conversions
44
*
@@ -50,3 +50,9 @@ tests(void)
5050
ASSERT_INT_EQ(convtime("1000000000000000000000w"), -1);
5151
TEST_DONE();
5252
}
53+
54+
void
55+
benchmarks(void)
56+
{
57+
printf("no benchmarks\n");
58+
}

‎regress/unittests/hostkeys/Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# $OpenBSD: Makefile,v 1.10 2023/01/15 23:35:10 djm Exp $
1+
# $OpenBSD: Makefile,v 1.11 2025/04/15 04:00:42 djm Exp $
22

33
PROG=test_hostkeys
44
SRCS=tests.c test_iterate.c
@@ -20,6 +20,6 @@ SRCS+=utf8.c
2020
REGRESS_TARGETS=run-regress-${PROG}
2121

2222
run-regress-${PROG}: ${PROG}
23-
env ${TEST_ENV} ./${PROG} -d ${.CURDIR}/testdata
23+
env ${TEST_ENV} ./${PROG} ${UNITTEST_ARGS} -d ${.CURDIR}/testdata
2424

2525
.include <bsd.regress.mk>

0 commit comments

Comments
 (0)
⚡