From 42d46e57ec9718c1090e43db75f433d0841af525 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Wed, 24 Apr 2019 16:12:30 -0700 Subject: selftests: Extract single-test shell logic from lib.mk In order to improve the reusability of the kselftest test running logic, this extracts the single-test logic from lib.mk into kselftest/runner.sh which lib.mk can call directly. No changes in output. As part of the change, this moves the "summary" Makefile logic around to set a new "logfile" output. This will be used again in the future "emit_tests" target as well. Signed-off-by: Kees Cook Signed-off-by: Shuah Khan --- tools/testing/selftests/lib.mk | 37 +++++++------------------------------ 1 file changed, 7 insertions(+), 30 deletions(-) (limited to 'tools/testing/selftests/lib.mk') diff --git a/tools/testing/selftests/lib.mk b/tools/testing/selftests/lib.mk index 5979fdc4f36c..9d2b3c303bfa 100644 --- a/tools/testing/selftests/lib.mk +++ b/tools/testing/selftests/lib.mk @@ -14,6 +14,7 @@ ifeq (0,$(MAKELEVEL)) endif endif endif +selfdir = $(realpath $(dir $(filter %/lib.mk,$(MAKEFILE_LIST)))) # The following are built by lib.mk common compile rules. # TEST_CUSTOM_PROGS should be used by tests that require @@ -65,43 +66,19 @@ all: $(TEST_GEN_PROGS) $(TEST_GEN_PROGS_EXTENDED) $(TEST_GEN_FILES) endif .ONESHELL: -define RUN_TEST_PRINT_RESULT - TEST_HDR_MSG="selftests: "`basename $$PWD`:" $$BASENAME_TEST"; \ - echo $$TEST_HDR_MSG; \ - echo "========================================"; \ - if [ ! -x $$TEST ]; then \ - echo "$$TEST_HDR_MSG: Warning: file $$BASENAME_TEST is not executable, correct this.";\ - echo "not ok 1..$$test_num $$TEST_HDR_MSG [FAIL]"; \ - else \ - cd `dirname $$TEST` > /dev/null; \ - if [ "X$(summary)" != "X" ]; then \ - (./$$BASENAME_TEST > /tmp/$$BASENAME_TEST 2>&1 && \ - echo "ok 1..$$test_num $$TEST_HDR_MSG [PASS]") || \ - (if [ $$? -eq $$skip ]; then \ - echo "not ok 1..$$test_num $$TEST_HDR_MSG [SKIP]"; \ - else echo "not ok 1..$$test_num $$TEST_HDR_MSG [FAIL]"; \ - fi;) \ - else \ - (./$$BASENAME_TEST && \ - echo "ok 1..$$test_num $$TEST_HDR_MSG [PASS]") || \ - (if [ $$? -eq $$skip ]; then \ - echo "not ok 1..$$test_num $$TEST_HDR_MSG [SKIP]"; \ - else echo "not ok 1..$$test_num $$TEST_HDR_MSG [FAIL]"; \ - fi;) \ - fi; \ - cd - > /dev/null; \ - fi; -endef - define RUN_TESTS @export KSFT_TAP_LEVEL=`echo 1`; \ test_num=`echo 0`; \ - skip=`echo 4`; \ + . $(selfdir)/kselftest/runner.sh; \ echo "TAP version 13"; \ for TEST in $(1); do \ BASENAME_TEST=`basename $$TEST`; \ test_num=`echo $$test_num+1 | bc`; \ - $(call RUN_TEST_PRINT_RESULT,$(TEST),$(BASENAME_TEST),$(test_num),$(skip)) \ + if [ "X$(summary)" != "X" ]; then \ + logfile="/tmp/$$BASENAME_TEST"; \ + cat /dev/null > "$$logfile"; \ + fi; \ + run_one "$$BASENAME_TEST" "$$test_num"; \ done; endef -- cgit v1.2.3 From d4e59a536f505c6760ba0187e451daa62a2df703 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Wed, 24 Apr 2019 16:12:31 -0700 Subject: selftests: Use runner.sh for emit targets This reuses the new runner.sh for the emit targets instead of manually running each test via run_kselftest.sh. Signed-off-by: Kees Cook Signed-off-by: Shuah Khan --- tools/testing/selftests/Makefile | 11 +++++------ tools/testing/selftests/lib.mk | 15 ++------------- 2 files changed, 7 insertions(+), 19 deletions(-) (limited to 'tools/testing/selftests/lib.mk') diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index f2ebf8cf4686..c5f9f736cdbd 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -176,7 +176,8 @@ ALL_SCRIPT := $(INSTALL_PATH)/run_kselftest.sh install: ifdef INSTALL_PATH @# Ask all targets to install their files - mkdir -p $(INSTALL_PATH) + mkdir -p $(INSTALL_PATH)/kselftest + install -m 744 kselftest/runner.sh $(INSTALL_PATH)/kselftest/ @for TARGET in $(TARGETS); do \ BUILD_TARGET=$$BUILD/$$TARGET; \ make OUTPUT=$$BUILD_TARGET -C $$TARGET INSTALL_PATH=$(INSTALL_PATH)/$$TARGET install; \ @@ -186,15 +187,13 @@ ifdef INSTALL_PATH echo "#!/bin/sh" > $(ALL_SCRIPT) echo "BASE_DIR=\$$(realpath \$$(dirname \$$0))" >> $(ALL_SCRIPT) echo "cd \$$BASE_DIR" >> $(ALL_SCRIPT) + echo ". ./kselftest/runner.sh" >> $(ALL_SCRIPT) echo "ROOT=\$$PWD" >> $(ALL_SCRIPT) echo "if [ \"\$$1\" = \"--summary\" ]; then" >> $(ALL_SCRIPT) - echo " OUTPUT=\$$BASE_DIR/output.log" >> $(ALL_SCRIPT) - echo " cat /dev/null > \$$OUTPUT" >> $(ALL_SCRIPT) - echo "else" >> $(ALL_SCRIPT) - echo " OUTPUT=/dev/stdout" >> $(ALL_SCRIPT) + echo " logfile=\$$BASE_DIR/output.log" >> $(ALL_SCRIPT) + echo " cat /dev/null > \$$logfile" >> $(ALL_SCRIPT) echo "fi" >> $(ALL_SCRIPT) echo "export KSFT_TAP_LEVEL=1" >> $(ALL_SCRIPT) - echo "export skip=4" >> $(ALL_SCRIPT) for TARGET in $(TARGETS); do \ BUILD_TARGET=$$BUILD/$$TARGET; \ diff --git a/tools/testing/selftests/lib.mk b/tools/testing/selftests/lib.mk index 9d2b3c303bfa..6b2d026a94ea 100644 --- a/tools/testing/selftests/lib.mk +++ b/tools/testing/selftests/lib.mk @@ -116,24 +116,13 @@ else $(error Error: set INSTALL_PATH to use install) endif -define EMIT_TESTS +emit_tests: @test_num=`echo 0`; \ for TEST in $(TEST_GEN_PROGS) $(TEST_CUSTOM_PROGS) $(TEST_PROGS); do \ BASENAME_TEST=`basename $$TEST`; \ test_num=`echo $$test_num+1 | bc`; \ - TEST_HDR_MSG="selftests: "`basename $$PWD`:" $$BASENAME_TEST"; \ - echo "echo $$TEST_HDR_MSG"; \ - if [ ! -x $$TEST ]; then \ - echo "echo \"$$TEST_HDR_MSG: Warning: file $$BASENAME_TEST is not executable, correct this.\""; \ - echo "echo \"not ok 1..$$test_num $$TEST_HDR_MSG [FAIL]\""; \ - else - echo "(./$$BASENAME_TEST >> \$$OUTPUT 2>&1 && echo \"ok 1..$$test_num $$TEST_HDR_MSG [PASS]\") || (if [ \$$? -eq \$$skip ]; then echo \"not ok 1..$$test_num $$TEST_HDR_MSG [SKIP]\"; else echo \"not ok 1..$$test_num $$TEST_HDR_MSG [FAIL]\"; fi;)"; \ - fi; \ + echo "run_one \"$$BASENAME_TEST\" \"$$test_num\""; \ done; -endef - -emit_tests: - $(EMIT_TESTS) # define if isn't already. It is undefined in make O= case. ifeq ($(RM),) -- cgit v1.2.3 From bf66078235ca27062f5924ed6901f40becc4a1a4 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Wed, 24 Apr 2019 16:12:32 -0700 Subject: selftests: Extract logic for multiple test runs This moves the logic for running multiple tests into a single "run_many" function of runner.sh. Both "run_tests" and "emit_tests" are modified to use it. Summary handling is now controlled by the "per_test_logging" shell flag. Signed-off-by: Kees Cook Signed-off-by: Shuah Khan --- tools/testing/selftests/Makefile | 6 ++---- tools/testing/selftests/kselftest/runner.sh | 25 ++++++++++++++++++++++--- tools/testing/selftests/lib.mk | 25 ++++++++----------------- 3 files changed, 32 insertions(+), 24 deletions(-) (limited to 'tools/testing/selftests/lib.mk') diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index c5f9f736cdbd..4ac1d1c7ce5b 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -193,16 +193,14 @@ ifdef INSTALL_PATH echo " logfile=\$$BASE_DIR/output.log" >> $(ALL_SCRIPT) echo " cat /dev/null > \$$logfile" >> $(ALL_SCRIPT) echo "fi" >> $(ALL_SCRIPT) - echo "export KSFT_TAP_LEVEL=1" >> $(ALL_SCRIPT) for TARGET in $(TARGETS); do \ BUILD_TARGET=$$BUILD/$$TARGET; \ - echo "echo ; echo TAP version 13" >> $(ALL_SCRIPT); \ - echo "echo Running tests in $$TARGET" >> $(ALL_SCRIPT); \ - echo "echo ========================================" >> $(ALL_SCRIPT); \ echo "[ -w /dev/kmsg ] && echo \"kselftest: Running tests in $$TARGET\" >> /dev/kmsg" >> $(ALL_SCRIPT); \ echo "cd $$TARGET" >> $(ALL_SCRIPT); \ + echo -n "run_many" >> $(ALL_SCRIPT); \ make -s --no-print-directory OUTPUT=$$BUILD_TARGET -C $$TARGET emit_tests >> $(ALL_SCRIPT); \ + echo "" >> $(ALL_SCRIPT); \ echo "cd \$$ROOT" >> $(ALL_SCRIPT); \ done; diff --git a/tools/testing/selftests/kselftest/runner.sh b/tools/testing/selftests/kselftest/runner.sh index e1117d703887..f12b0a631273 100644 --- a/tools/testing/selftests/kselftest/runner.sh +++ b/tools/testing/selftests/kselftest/runner.sh @@ -2,17 +2,20 @@ # SPDX-License-Identifier: GPL-2.0 # # Runs a set of tests in a given subdirectory. +export KSFT_TAP_LEVEL=1 export skip_rc=4 export logfile=/dev/stdout +export per_test_logging= run_one() { - TEST="$1" - NUM="$2" + DIR="$1" + TEST="$2" + NUM="$3" BASENAME_TEST=$(basename $TEST) - TEST_HDR_MSG="selftests: "`basename $PWD`:" $BASENAME_TEST" + TEST_HDR_MSG="selftests: $DIR: $BASENAME_TEST" echo "$TEST_HDR_MSG" echo "========================================" if [ ! -x "$TEST" ]; then @@ -30,3 +33,19 @@ run_one() cd - >/dev/null fi } + +run_many() +{ + echo "TAP version 13" + DIR=$(basename "$PWD") + test_num=0 + for TEST in "$@"; do + BASENAME_TEST=$(basename $TEST) + test_num=$(( test_num + 1 )) + if [ -n "$per_test_logging" ]; then + logfile="/tmp/$BASENAME_TEST" + cat /dev/null > "$logfile" + fi + run_one "$DIR" "$TEST" "$test_num" + done +} diff --git a/tools/testing/selftests/lib.mk b/tools/testing/selftests/lib.mk index 6b2d026a94ea..28b8ffedfdf1 100644 --- a/tools/testing/selftests/lib.mk +++ b/tools/testing/selftests/lib.mk @@ -67,19 +67,11 @@ endif .ONESHELL: define RUN_TESTS - @export KSFT_TAP_LEVEL=`echo 1`; \ - test_num=`echo 0`; \ - . $(selfdir)/kselftest/runner.sh; \ - echo "TAP version 13"; \ - for TEST in $(1); do \ - BASENAME_TEST=`basename $$TEST`; \ - test_num=`echo $$test_num+1 | bc`; \ - if [ "X$(summary)" != "X" ]; then \ - logfile="/tmp/$$BASENAME_TEST"; \ - cat /dev/null > "$$logfile"; \ - fi; \ - run_one "$$BASENAME_TEST" "$$test_num"; \ - done; + @. $(selfdir)/kselftest/runner.sh; \ + if [ "X$(summary)" != "X" ]; then \ + per_test_logging=1; \ + fi; \ + run_many $(1) endef run_tests: all @@ -117,12 +109,11 @@ else endif emit_tests: - @test_num=`echo 0`; \ for TEST in $(TEST_GEN_PROGS) $(TEST_CUSTOM_PROGS) $(TEST_PROGS); do \ BASENAME_TEST=`basename $$TEST`; \ - test_num=`echo $$test_num+1 | bc`; \ - echo "run_one \"$$BASENAME_TEST\" \"$$test_num\""; \ - done; + echo " \\"; \ + echo -n " \"$$BASENAME_TEST\""; \ + done; \ # define if isn't already. It is undefined in make O= case. ifeq ($(RM),) -- cgit v1.2.3 From 5c069b6dedef1fab5420ca8658ed7f9ee4d26007 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Wed, 24 Apr 2019 16:12:35 -0700 Subject: selftests: Move test output to diagnostic lines This changes the selftest output so that each test's output is prefixed with "# " as a TAP "diagnostic line". This creates a bit of a kernel-specific TAP dialect where the diagnostics precede the results. The TAP spec isn't entirely clear about this, though, so I think it's the correct solution so as to keep interactive runs making sense. If the output _followed_ the result line in the spec-suggested YAML form, each test would dump all of its output at once instead of as it went, making debugging harder. This does, however, solve the recursive TAP output problem, as sub-tests will simply be prefixed by "# ". Parsing sub-tests becomes a simple problem of just removing the first two characters of a given top-level test's diagnostic output, and parsing the results. Note that the shell construct needed to both get an exit code from the first command in a pipe and still filter the pipe (to add the "# " prefix) uses a POSIX solution rather than the bash "pipefail" option which is not supported by dash. Since some test environments may have a very minimal set of utilities available, the new prefixing code will fall back to doing line-at-a-time prefixing if perl and/or stdbuf are not available. Signed-off-by: Kees Cook Signed-off-by: Shuah Khan --- tools/testing/selftests/Makefile | 1 + tools/testing/selftests/kselftest.h | 2 +- tools/testing/selftests/kselftest/prefix.pl | 23 ++++++++++++++++++ tools/testing/selftests/kselftest/runner.sh | 37 +++++++++++++++++++++++++---- tools/testing/selftests/lib.mk | 3 ++- 5 files changed, 60 insertions(+), 6 deletions(-) create mode 100755 tools/testing/selftests/kselftest/prefix.pl (limited to 'tools/testing/selftests/lib.mk') diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index 4ac1d1c7ce5b..64699f59b95f 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -178,6 +178,7 @@ ifdef INSTALL_PATH @# Ask all targets to install their files mkdir -p $(INSTALL_PATH)/kselftest install -m 744 kselftest/runner.sh $(INSTALL_PATH)/kselftest/ + install -m 744 kselftest/prefix.pl $(INSTALL_PATH)/kselftest/ @for TARGET in $(TARGETS); do \ BUILD_TARGET=$$BUILD/$$TARGET; \ make OUTPUT=$$BUILD_TARGET -C $$TARGET INSTALL_PATH=$(INSTALL_PATH)/$$TARGET install; \ diff --git a/tools/testing/selftests/kselftest.h b/tools/testing/selftests/kselftest.h index 9f4147a6fdbc..7f078e79a9fa 100644 --- a/tools/testing/selftests/kselftest.h +++ b/tools/testing/selftests/kselftest.h @@ -63,7 +63,7 @@ static inline void ksft_print_header(void) static inline void ksft_print_cnts(void) { - printf("Pass %d Fail %d Xfail %d Xpass %d Skip %d Error %d\n", + printf("# Pass %d Fail %d Xfail %d Xpass %d Skip %d Error %d\n", ksft_cnt.ksft_pass, ksft_cnt.ksft_fail, ksft_cnt.ksft_xfail, ksft_cnt.ksft_xpass, ksft_cnt.ksft_xskip, ksft_cnt.ksft_error); diff --git a/tools/testing/selftests/kselftest/prefix.pl b/tools/testing/selftests/kselftest/prefix.pl new file mode 100755 index 000000000000..ec7e48118183 --- /dev/null +++ b/tools/testing/selftests/kselftest/prefix.pl @@ -0,0 +1,23 @@ +#!/usr/bin/perl +# SPDX-License-Identifier: GPL-2.0 +# Prefix all lines with "# ", unbuffered. Command being piped in may need +# to have unbuffering forced with "stdbuf -i0 -o0 -e0 $cmd". +use strict; + +binmode STDIN; +binmode STDOUT; + +STDOUT->autoflush(1); + +my $needed = 1; +while (1) { + my $char; + my $bytes = sysread(STDIN, $char, 1); + exit 0 if ($bytes == 0); + if ($needed) { + print "# "; + $needed = 0; + } + print $char; + $needed = 1 if ($char eq "\n"); +} diff --git a/tools/testing/selftests/kselftest/runner.sh b/tools/testing/selftests/kselftest/runner.sh index a66fb64e61e9..b9f74e5a2ee5 100644 --- a/tools/testing/selftests/kselftest/runner.sh +++ b/tools/testing/selftests/kselftest/runner.sh @@ -7,6 +7,34 @@ export skip_rc=4 export logfile=/dev/stdout export per_test_logging= +# There isn't a shell-agnostic way to find the path of a sourced file, +# so we must rely on BASE_DIR being set to find other tools. +if [ -z "$BASE_DIR" ]; then + echo "Error: BASE_DIR must be set before sourcing." >&2 + exit 1 +fi + +# If Perl is unavailable, we must fall back to line-at-a-time prefixing +# with sed instead of unbuffered output. +tap_prefix() +{ + if [ ! -x /usr/bin/perl ]; then + sed -e 's/^/# /' + else + "$BASE_DIR"/kselftest/prefix.pl + fi +} + +# If stdbuf is unavailable, we must fall back to line-at-a-time piping. +tap_unbuffer() +{ + if ! which stdbuf >/dev/null ; then + "$@" + else + stdbuf -i0 -o0 -e0 "$@" + fi +} + run_one() { DIR="$1" @@ -16,10 +44,9 @@ run_one() BASENAME_TEST=$(basename $TEST) TEST_HDR_MSG="selftests: $DIR: $BASENAME_TEST" - echo "$TEST_HDR_MSG" - echo "========================================" + echo "# $TEST_HDR_MSG" if [ ! -x "$TEST" ]; then - echo -n "$TEST_HDR_MSG: Warning: file $TEST is " + echo -n "# Warning: file $TEST is " if [ ! -e "$TEST" ]; then echo "missing!" else @@ -28,7 +55,9 @@ run_one() echo "not ok $test_num $TEST_HDR_MSG" else cd `dirname $TEST` > /dev/null - (./$BASENAME_TEST >> "$logfile" 2>&1 && + (((((tap_unbuffer ./$BASENAME_TEST 2>&1; echo $? >&3) | + tap_prefix >&4) 3>&1) | + (read xs; exit $xs)) 4>>"$logfile" && echo "ok $test_num $TEST_HDR_MSG") || (if [ $? -eq $skip_rc ]; then \ echo "not ok $test_num $TEST_HDR_MSG # SKIP" diff --git a/tools/testing/selftests/lib.mk b/tools/testing/selftests/lib.mk index 28b8ffedfdf1..098dd0065fb1 100644 --- a/tools/testing/selftests/lib.mk +++ b/tools/testing/selftests/lib.mk @@ -67,7 +67,8 @@ endif .ONESHELL: define RUN_TESTS - @. $(selfdir)/kselftest/runner.sh; \ + @BASE_DIR="$(selfdir)"; \ + . $(selfdir)/kselftest/runner.sh; \ if [ "X$(summary)" != "X" ]; then \ per_test_logging=1; \ fi; \ -- cgit v1.2.3 From 27d79a2b2bf0dbec1cc3d8ea269db4d5a0dac2f3 Mon Sep 17 00:00:00 2001 From: Shuah Khan Date: Fri, 10 May 2019 19:38:39 -0600 Subject: selftests: fix bpf build/test workflow regression when KBUILD_OUTPUT is set commit 8ce72dc32578 ("selftests: fix headers_install circular dependency") broke bpf build/test workflow. When KBUILD_OUTPUT is set, bpf objects end up in KBUILD_OUTPUT build directory instead of in ../selftests/bpf. The following bpf workflow breaks when it can't find the test_verifier: cd tools/testing/selftests/bpf; make; ./test_verifier; Fix it to set OUTPUT only when it is undefined in lib.mk. It didn't need to be set in the first place. Fixes: 8ce72dc32578 ("selftests: fix headers_install circular dependency") Reported-by: Alexei Starovoitov Signed-off-by: Shuah Khan Acked-by: Alexei Starovoitov Signed-off-by: Shuah Khan --- tools/testing/selftests/lib.mk | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) (limited to 'tools/testing/selftests/lib.mk') diff --git a/tools/testing/selftests/lib.mk b/tools/testing/selftests/lib.mk index 098dd0065fb1..077337195783 100644 --- a/tools/testing/selftests/lib.mk +++ b/tools/testing/selftests/lib.mk @@ -3,15 +3,9 @@ CC := $(CROSS_COMPILE)gcc ifeq (0,$(MAKELEVEL)) - ifneq ($(O),) - OUTPUT := $(O) - else - ifneq ($(KBUILD_OUTPUT),) - OUTPUT := $(KBUILD_OUTPUT) - else - OUTPUT := $(shell pwd) - DEFAULT_INSTALL_HDR_PATH := 1 - endif + ifeq ($(OUTPUT),) + OUTPUT := $(shell pwd) + DEFAULT_INSTALL_HDR_PATH := 1 endif endif selfdir = $(realpath $(dir $(filter %/lib.mk,$(MAKEFILE_LIST)))) -- cgit v1.2.3