diff options
Diffstat (limited to 'tools/testing/kunit')
25 files changed, 681 insertions, 79 deletions
diff --git a/tools/testing/kunit/configs/all_tests.config b/tools/testing/kunit/configs/all_tests.config index b0049be00c70..bccc2c77196d 100644 --- a/tools/testing/kunit/configs/all_tests.config +++ b/tools/testing/kunit/configs/all_tests.config @@ -20,6 +20,7 @@ CONFIG_VFAT_FS=y CONFIG_PCI=y CONFIG_USB4=y +CONFIG_I2C=y CONFIG_NET=y CONFIG_MCTP=y @@ -41,6 +42,14 @@ CONFIG_DAMON_PADDR=y CONFIG_REGMAP_BUILD=y +CONFIG_AUDIT=y + +CONFIG_CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT=y + +CONFIG_PRIME_NUMBERS=y + +CONFIG_CRC_ENABLE_ALL_FOR_KUNIT=y + CONFIG_SECURITY=y CONFIG_SECURITY_APPARMOR=y CONFIG_SECURITY_LANDLOCK=y @@ -49,3 +58,4 @@ CONFIG_SOUND=y CONFIG_SND=y CONFIG_SND_SOC=y CONFIG_SND_SOC_TOPOLOGY_BUILD=y +CONFIG_SND_SOC_CS35L56_I2C=y diff --git a/tools/testing/kunit/configs/arch_uml.config b/tools/testing/kunit/configs/arch_uml.config index 54ad8972681a..28edf816aa70 100644 --- a/tools/testing/kunit/configs/arch_uml.config +++ b/tools/testing/kunit/configs/arch_uml.config @@ -1,8 +1,7 @@ # Config options which are added to UML builds by default -# Enable virtio/pci, as a lot of tests require it. -CONFIG_VIRTIO_UML=y -CONFIG_UML_PCI_OVER_VIRTIO=y +# Enable pci, as a lot of tests require it. +CONFIG_KUNIT_UML_PCI=y # Enable FORTIFY_SOURCE for wider checking. CONFIG_FORTIFY_SOURCE=y diff --git a/tools/testing/kunit/kunit-completion.sh b/tools/testing/kunit/kunit-completion.sh new file mode 100644 index 000000000000..f053e7b5d265 --- /dev/null +++ b/tools/testing/kunit/kunit-completion.sh @@ -0,0 +1,34 @@ +# SPDX-License-Identifier: GPL-2.0 +# bash completion support for KUnit + +_kunit_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + +_kunit() +{ + local cur prev words cword + _init_completion || return + + local script="${_kunit_dir}/kunit.py" + + if [[ $cword -eq 1 && "$cur" != -* ]]; then + local cmds=$(${script} --list-cmds 2>/dev/null) + COMPREPLY=($(compgen -W "${cmds}" -- "$cur")) + return 0 + fi + + if [[ "$cur" == -* ]]; then + if [[ -n "${words[1]}" && "${words[1]}" != -* ]]; then + local opts=$(${script} ${words[1]} --list-opts 2>/dev/null) + COMPREPLY=($(compgen -W "${opts}" -- "$cur")) + return 0 + else + local opts=$(${script} --list-opts 2>/dev/null) + COMPREPLY=($(compgen -W "${opts}" -- "$cur")) + return 0 + fi + fi +} + +complete -o default -F _kunit kunit.py +complete -o default -F _kunit kunit +complete -o default -F _kunit ./tools/testing/kunit/kunit.py diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py index 7f9ae55fd6d5..ac3f7159e67f 100755 --- a/tools/testing/kunit/kunit.py +++ b/tools/testing/kunit/kunit.py @@ -21,6 +21,7 @@ from enum import Enum, auto from typing import Iterable, List, Optional, Sequence, Tuple import kunit_json +import kunit_junit import kunit_kernel import kunit_parser from kunit_printer import stdout, null_printer @@ -49,6 +50,7 @@ class KunitBuildRequest(KunitConfigRequest): class KunitParseRequest: raw_output: Optional[str] json: Optional[str] + junit: Optional[str] summary: bool failed: bool @@ -63,6 +65,7 @@ class KunitExecRequest(KunitParseRequest): run_isolated: Optional[str] list_tests: bool list_tests_attr: bool + list_suites: bool @dataclass class KunitRequest(KunitExecRequest, KunitBuildRequest): @@ -168,6 +171,12 @@ def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) - for line in attr_output: print(line.rstrip()) return KunitResult(status=KunitStatus.SUCCESS, elapsed_time=0.0) + if request.list_suites: + tests = _list_tests(linux, request) + output = _suites_from_test_list(tests) + for line in output: + print(line.rstrip()) + return KunitResult(status=KunitStatus.SUCCESS, elapsed_time=0.0) if request.run_isolated: tests = _list_tests(linux, request) if request.run_isolated == 'test': @@ -228,7 +237,7 @@ def parse_tests(request: KunitParseRequest, metadata: kunit_json.Metadata, input fake_test.counts.passed = 1 output: Iterable[str] = input_data - if request.raw_output == 'all': + if request.raw_output == 'all' or request.raw_output == 'full': pass elif request.raw_output == 'kunit': output = kunit_parser.extract_tap_lines(output) @@ -261,6 +270,13 @@ def parse_tests(request: KunitParseRequest, metadata: kunit_json.Metadata, input stdout.print_with_timestamp("Test results stored in %s" % os.path.abspath(request.json)) + if request.junit: + if request.junit == 'stdout': + kunit_junit.print_junit_result(test=test) + else: + kunit_junit.write_junit_result(test=test,filename=request.junit) + stdout.print_with_timestamp(f"Test results stored in {os.path.abspath(request.junit)}") + if test.status != kunit_parser.TestStatus.SUCCESS: return KunitResult(KunitStatus.TEST_FAILURE, parse_time), test @@ -302,6 +318,7 @@ def run_tests(linux: kunit_kernel.LinuxSourceTree, # So we hackily automatically rewrite --json => --json=stdout pseudo_bool_flag_defaults = { '--json': 'stdout', + '--junit': 'stdout', '--raw_output': 'kunit', } def massage_argv(argv: Sequence[str]) -> Sequence[str]: @@ -323,11 +340,27 @@ def get_default_jobs() -> int: return ncpu raise RuntimeError("os.cpu_count() returned None") +def get_default_build_dir() -> str: + if 'KBUILD_OUTPUT' in os.environ: + return os.path.join(os.environ['KBUILD_OUTPUT'], '.kunit') + return '.kunit' + +def add_completion_opts(parser: argparse.ArgumentParser) -> None: + parser.add_argument('--list-opts', + help=argparse.SUPPRESS, + action='store_true') + +def add_root_opts(parser: argparse.ArgumentParser) -> None: + parser.add_argument('--list-cmds', + help=argparse.SUPPRESS, + action='store_true') + add_completion_opts(parser) + def add_common_opts(parser: argparse.ArgumentParser) -> None: parser.add_argument('--build_dir', help='As in the make command, it specifies the build ' 'directory.', - type=str, default='.kunit', metavar='DIR') + type=str, default=get_default_build_dir(), metavar='DIR') parser.add_argument('--make_options', help='X=Y make option, can be repeated.', action='append', metavar='X=Y') @@ -374,6 +407,8 @@ def add_common_opts(parser: argparse.ArgumentParser) -> None: help='Additional QEMU arguments, e.g. "-smp 8"', action='append', metavar='') + add_completion_opts(parser) + def add_build_opts(parser: argparse.ArgumentParser) -> None: parser.add_argument('--jobs', help='As in the make command, "Specifies the number of ' @@ -420,17 +455,25 @@ def add_exec_opts(parser: argparse.ArgumentParser) -> None: parser.add_argument('--list_tests_attr', help='If set, list all tests and test ' 'attributes.', action='store_true') + parser.add_argument('--list_suites', help='If set, list all suites that will be ' + 'run.', + action='store_true') def add_parse_opts(parser: argparse.ArgumentParser) -> None: parser.add_argument('--raw_output', help='If set don\'t parse output from kernel. ' 'By default, filters to just KUnit output. Use ' '--raw_output=all to show everything', - type=str, nargs='?', const='all', default=None, choices=['all', 'kunit']) + type=str, nargs='?', const='all', default=None, choices=['all', 'full', 'kunit']) parser.add_argument('--json', nargs='?', help='Prints parsed test results as JSON to stdout or a file if ' 'a filename is specified. Does nothing if --raw_output is set.', type=str, const='stdout', default=None, metavar='FILE') + parser.add_argument('--junit', + nargs='?', + help='Prints parsed test results as JUnit XML to stdout or a file if ' + 'a filename is specified. Does nothing if --raw_output is set.', + type=str, const='stdout', default=None, metavar='FILE') parser.add_argument('--summary', help='Prints only the summary line for parsed test results.' 'Does nothing if --raw_output is set.', @@ -474,6 +517,7 @@ def run_handler(cli_args: argparse.Namespace) -> None: jobs=cli_args.jobs, raw_output=cli_args.raw_output, json=cli_args.json, + junit=cli_args.junit, summary=cli_args.summary, failed=cli_args.failed, timeout=cli_args.timeout, @@ -483,7 +527,8 @@ def run_handler(cli_args: argparse.Namespace) -> None: kernel_args=cli_args.kernel_args, run_isolated=cli_args.run_isolated, list_tests=cli_args.list_tests, - list_tests_attr=cli_args.list_tests_attr) + list_tests_attr=cli_args.list_tests_attr, + list_suites=cli_args.list_suites) result = run_tests(linux, request) if result.status != KunitStatus.SUCCESS: sys.exit(1) @@ -523,6 +568,7 @@ def exec_handler(cli_args: argparse.Namespace) -> None: exec_request = KunitExecRequest(raw_output=cli_args.raw_output, build_dir=cli_args.build_dir, json=cli_args.json, + junit=cli_args.junit, summary=cli_args.summary, failed=cli_args.failed, timeout=cli_args.timeout, @@ -532,7 +578,8 @@ def exec_handler(cli_args: argparse.Namespace) -> None: kernel_args=cli_args.kernel_args, run_isolated=cli_args.run_isolated, list_tests=cli_args.list_tests, - list_tests_attr=cli_args.list_tests_attr) + list_tests_attr=cli_args.list_tests_attr, + list_suites=cli_args.list_suites) result = exec_tests(linux, exec_request) stdout.print_with_timestamp(( 'Elapsed time: %.3fs\n') % (result.elapsed_time)) @@ -550,7 +597,9 @@ def parse_handler(cli_args: argparse.Namespace) -> None: # We know nothing about how the result was created! metadata = kunit_json.Metadata() request = KunitParseRequest(raw_output=cli_args.raw_output, - json=cli_args.json, summary=cli_args.summary, + json=cli_args.json, + junit=cli_args.junit, + summary=cli_args.summary, failed=cli_args.failed) result, _ = parse_tests(request, metadata, kunit_output) if result.status != KunitStatus.SUCCESS: @@ -569,6 +618,7 @@ subcommand_handlers_map = { def main(argv: Sequence[str]) -> None: parser = argparse.ArgumentParser( description='Helps writing and running KUnit tests.') + add_root_opts(parser) subparser = parser.add_subparsers(dest='subcommand') # The 'run' command will config, build, exec, and parse in one go. @@ -603,12 +653,28 @@ def main(argv: Sequence[str]) -> None: parse_parser.add_argument('file', help='Specifies the file to read results from.', type=str, nargs='?', metavar='input_file') + add_completion_opts(parse_parser) cli_args = parser.parse_args(massage_argv(argv)) if get_kernel_root_path(): os.chdir(get_kernel_root_path()) + if cli_args.list_cmds: + print(" ".join(subparser.choices.keys())) + return + + if cli_args.list_opts: + target_parser = subparser.choices.get(cli_args.subcommand) + if not target_parser: + target_parser = parser + + # Accessing private attribute _option_string_actions to get + # the list of options. This is not a public API, but argparse + # does not provide a way to inspect options programmatically. + print(' '.join(target_parser._option_string_actions.keys())) + return + subcomand_handler = subcommand_handlers_map.get(cli_args.subcommand, None) if subcomand_handler is None: diff --git a/tools/testing/kunit/kunit_json.py b/tools/testing/kunit/kunit_json.py index 10ff65689dd8..80fa4e354a17 100644 --- a/tools/testing/kunit/kunit_json.py +++ b/tools/testing/kunit/kunit_json.py @@ -39,10 +39,20 @@ def _get_group_json(test: Test, common_fields: JsonObj) -> JsonObj: status = _status_map.get(subtest.status, "FAIL") test_cases.append({"name": subtest.name, "status": status}) + test_counts = test.counts + counts_json = { + "tests": test_counts.total(), + "passed": test_counts.passed, + "failed": test_counts.failed, + "crashed": test_counts.crashed, + "skipped": test_counts.skipped, + "errors": test_counts.errors, + } test_group = { "name": test.name, "sub_groups": sub_groups, "test_cases": test_cases, + "misc": counts_json } test_group.update(common_fields) return test_group diff --git a/tools/testing/kunit/kunit_junit.py b/tools/testing/kunit/kunit_junit.py new file mode 100644 index 000000000000..3622070358e7 --- /dev/null +++ b/tools/testing/kunit/kunit_junit.py @@ -0,0 +1,61 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Generates JUnit XML files from KUnit test results +# +# Copyright (C) 2026, Google LLC and David Gow. + +from xml.sax.saxutils import quoteattr, XMLGenerator +import xml.etree.ElementTree as ET +from kunit_parser import Test, TestStatus +from typing import Optional + +# Get a string representing a tes suite (including subtests) in JUnit XML +def get_test_suite(test: Test, parent: Optional[ET.Element]) -> ET.Element: + suite_attrs = { + 'name': test.name, + 'tests': str(test.counts.total()), + 'failures': str(test.counts.failed), + 'skipped': str(test.counts.skipped), + 'errors': str(test.counts.crashed + test.counts.errors), + } + + if parent is not None: + test_suite_element = ET.SubElement(parent, 'testsuite', suite_attrs) + else: + test_suite_element = ET.Element('testsuite', suite_attrs) + + for subtest in test.subtests: + if subtest.subtests: + get_test_suite(subtest, test_suite_element) + continue + test_case_element = ET.SubElement(test_suite_element, 'testcase', {'name': subtest.name}) + if subtest.status == TestStatus.FAILURE: + ET.SubElement(test_case_element, 'failure', {}).text = 'Test Failed' + elif subtest.status == TestStatus.SKIPPED: + ET.SubElement(test_case_element, 'skipped', {}).text = subtest.skip_reason + elif subtest.status == TestStatus.TEST_CRASHED: + ET.SubElement(test_case_element, 'error', {}).text = 'Test Crashed' + + if subtest.log: + ET.SubElement(test_case_element, 'system-out', {}).text = "\n".join(subtest.log) + + return test_suite_element + +# Get a string for an entire XML file for the test structure starting at test +def get_junit_result(test: Test) -> str: + root_element = get_test_suite(test, None) + ET.indent(root_element) + return ET.tostring(root_element, encoding="unicode", xml_declaration=True) + +# Print a JUnit result to stdout. +def print_junit_result(test: Test) -> None: + root_element = get_test_suite(test, None) + ET.indent(root_element) + ET.dump(root_element) + +# Write an entire XML file for the test structure starting at test +def write_junit_result(test: Test, filename: str) -> None: + root_element = get_test_suite(test, None) + ET.indent(root_element) + root_et = ET.ElementTree(root_element) + root_et.write(filename, encoding='utf-8', xml_declaration=True) diff --git a/tools/testing/kunit/kunit_kernel.py b/tools/testing/kunit/kunit_kernel.py index d30f90eae9a4..58557c47d85f 100644 --- a/tools/testing/kunit/kunit_kernel.py +++ b/tools/testing/kunit/kunit_kernel.py @@ -14,8 +14,9 @@ import os import shlex import shutil import signal +import sys import threading -from typing import Iterator, List, Optional, Tuple +from typing import Iterator, List, Optional, Tuple, Any from types import FrameType import kunit_config @@ -72,8 +73,8 @@ class LinuxSourceTreeOperations: raise ConfigError(e.output.decode()) def make(self, jobs: int, build_dir: str, make_options: Optional[List[str]]) -> None: - command = ['make', 'all', 'compile_commands.json', 'ARCH=' + self._linux_arch, - 'O=' + build_dir, '--jobs=' + str(jobs)] + command = ['make', 'all', 'compile_commands.json', 'scripts_gdb', + 'ARCH=' + self._linux_arch, 'O=' + build_dir, '--jobs=' + str(jobs)] if make_options: command.extend(make_options) if self._cross_compile: @@ -201,6 +202,13 @@ def _default_qemu_config_path(arch: str) -> str: return config_path options = [f[:-3] for f in os.listdir(QEMU_CONFIGS_DIR) if f.endswith('.py')] + + if arch == 'help': + print('um') + for option in options: + print(option) + sys.exit() + raise ConfigError(arch + ' is not a valid arch, options are ' + str(sorted(options))) def _get_qemu_ops(config_path: str, @@ -210,7 +218,7 @@ def _get_qemu_ops(config_path: str, # exists (I learned this through experimentation and could not find it # anywhere in the Python documentation). # - # Bascially, we completely ignore the actual file location of the config + # Basically, we completely ignore the actual file location of the config # we are loading and just tell Python that the module lives in the # QEMU_CONFIGS_DIR for import purposes regardless of where it actually # exists as a file. @@ -257,6 +265,7 @@ class LinuxSourceTree: if kconfig_add: kconfig = kunit_config.parse_from_string('\n'.join(kconfig_add)) self._kconfig.merge_in_entries(kconfig) + self._process : Optional[subprocess.Popen[Any]] = None def arch(self) -> str: return self._arch @@ -337,9 +346,17 @@ class LinuxSourceTree: return False return self.validate_config(build_dir) + def _restore_terminal_if_tty(self) -> None: + # stty requires a controlling terminal; skip headless runs. + if sys.stdin is None or not sys.stdin.isatty(): + return + subprocess.call(['stty', 'sane']) + def run_kernel(self, args: Optional[List[str]]=None, build_dir: str='', filter_glob: str='', filter: str='', filter_action: Optional[str]=None, timeout: Optional[int]=None) -> Iterator[str]: - if not args: - args = [] + # Copy to avoid mutating the caller-supplied list. exec_tests() reuses + # the same args across repeated run_kernel() calls (e.g. --run_isolated), + # so appending to the original would accumulate stale flags on each call. + args = list(args) if args else [] if filter_glob: args.append('kunit.filter_glob=' + filter_glob) if filter: @@ -348,36 +365,45 @@ class LinuxSourceTree: args.append('kunit.filter_action=' + filter_action) args.append('kunit.enable=1') - process = self._ops.start(args, build_dir) - assert process.stdout is not None # tell mypy it's set + self._process = self._ops.start(args, build_dir) + assert self._process is not None # tell mypy it's set + assert self._process.stdout is not None # tell mypy it's set # Enforce the timeout in a background thread. def _wait_proc() -> None: try: - process.wait(timeout=timeout) + if self._process: + self._process.wait(timeout=timeout) except Exception as e: print(e) - process.terminate() - process.wait() + if self._process: + self._process.terminate() + self._process.wait() waiter = threading.Thread(target=_wait_proc) waiter.start() output = open(get_outfile_path(build_dir), 'w') try: # Tee the output to the file and to our caller in real time. - for line in process.stdout: + for line in self._process.stdout: output.write(line) yield line # This runs even if our caller doesn't consume every line. finally: # Flush any leftover output to the file - output.write(process.stdout.read()) + if self._process: + if self._process.stdout: + output.write(self._process.stdout.read()) + self._process.stdout.close() + self._process = None output.close() - process.stdout.close() waiter.join() - subprocess.call(['stty', 'sane']) + self._restore_terminal_if_tty() def signal_handler(self, unused_sig: int, unused_frame: Optional[FrameType]) -> None: logging.error('Build interruption occurred. Cleaning console.') - subprocess.call(['stty', 'sane']) + if self._process: + self._process.terminate() + self._process.wait() + self._restore_terminal_if_tty() diff --git a/tools/testing/kunit/kunit_parser.py b/tools/testing/kunit/kunit_parser.py index 29fc27e8949b..d722874bc660 100644 --- a/tools/testing/kunit/kunit_parser.py +++ b/tools/testing/kunit/kunit_parser.py @@ -17,7 +17,7 @@ import textwrap from enum import Enum, auto from typing import Iterable, Iterator, List, Optional, Tuple -from kunit_printer import Printer, stdout +from kunit_printer import Printer class Test: """ @@ -44,11 +44,12 @@ class Test: self.subtests = [] # type: List[Test] self.log = [] # type: List[str] self.counts = TestCounts() + self.skip_reason = '' def __str__(self) -> str: """Returns string representation of a Test class object.""" return (f'Test({self.status}, {self.name}, {self.expected_count}, ' - f'{self.subtests}, {self.log}, {self.counts})') + f'{self.subtests}, {self.log}, {self.counts}, {self.skip_reason})') def __repr__(self) -> str: """Returns string representation of a Test class object.""" @@ -57,7 +58,7 @@ class Test: def add_error(self, printer: Printer, error_message: str) -> None: """Records an error that occurred while parsing this test.""" self.counts.errors += 1 - printer.print_with_timestamp(stdout.red('[ERROR]') + f' Test: {self.name}: {error_message}') + printer.print_with_timestamp(printer.red('[ERROR]') + f' Test: {self.name}: {error_message}') def ok_status(self) -> bool: """Returns true if the status was ok, i.e. passed or skipped.""" @@ -268,7 +269,7 @@ def check_version(version_num: int, accepted_versions: List[int], if version_num < min(accepted_versions): test.add_error(printer, f'{version_type} version lower than expected!') elif version_num > max(accepted_versions): - test.add_error(printer, f'{version_type} version higer than expected!') + test.add_error(printer, f'{version_type} version higher than expected!') def parse_ktap_header(lines: LineStream, test: Test, printer: Printer) -> bool: """ @@ -352,9 +353,9 @@ def parse_test_plan(lines: LineStream, test: Test) -> bool: lines.pop() return True -TEST_RESULT = re.compile(r'^\s*(ok|not ok) ([0-9]+) (- )?([^#]*)( # .*)?$') +TEST_RESULT = re.compile(r'^\s*(ok|not ok) ([0-9]+) ?(:?- )?([^#]*)( # .*)?$') -TEST_RESULT_SKIP = re.compile(r'^\s*(ok|not ok) ([0-9]+) (- )?(.*) # SKIP(.*)$') +TEST_RESULT_SKIP = re.compile(r'^\s*(ok|not ok) ([0-9]+) ?(:?- )?(.*) # SKIP ?(.*)$') def peek_test_name_match(lines: LineStream, test: Test) -> bool: """ @@ -379,6 +380,8 @@ def peek_test_name_match(lines: LineStream, test: Test) -> bool: if not match: return False name = match.group(4) + if not name: + return False return name == test.name def parse_test_result(lines: LineStream, test: Test, @@ -429,6 +432,7 @@ def parse_test_result(lines: LineStream, test: Test, status = match.group(1) if skip_match: test.status = TestStatus.SKIPPED + test.skip_reason = skip_match.group(5) or '' elif status == 'ok': test.status = TestStatus.SUCCESS else: @@ -537,12 +541,15 @@ def format_test_result(test: Test, printer: Printer) -> str: if test.status == TestStatus.SUCCESS: return printer.green('[PASSED] ') + test.name if test.status == TestStatus.SKIPPED: - return printer.yellow('[SKIPPED] ') + test.name + skip_message = printer.yellow('[SKIPPED] ') + test.name + if test.skip_reason != '': + skip_message += printer.yellow(' (' + test.skip_reason + ')') + return skip_message if test.status == TestStatus.NO_TESTS: return printer.yellow('[NO TESTS RUN] ') + test.name if test.status == TestStatus.TEST_CRASHED: print_log(test.log, printer) - return stdout.red('[CRASHED] ') + test.name + return printer.red('[CRASHED] ') + test.name print_log(test.log, printer) return printer.red('[FAILED] ') + test.name @@ -649,11 +656,11 @@ def print_summary_line(test: Test, printer: Printer) -> None: printer - Printer object to output results """ if test.status == TestStatus.SUCCESS: - color = stdout.green + color = printer.green elif test.status in (TestStatus.SKIPPED, TestStatus.NO_TESTS): - color = stdout.yellow + color = printer.yellow else: - color = stdout.red + color = printer.red printer.print_with_timestamp(color(f'Testing complete. {test.counts}')) # Summarize failures that might have gone off-screen since we had a lot @@ -687,6 +694,9 @@ def bubble_up_test_results(test: Test) -> None: elif test.counts.get_status() == TestStatus.TEST_CRASHED: test.status = TestStatus.TEST_CRASHED + if status == TestStatus.FAILURE and test.counts.get_status() == TestStatus.SUCCESS: + counts.add_status(status) + def parse_test(lines: LineStream, expected_num: int, log: List[str], is_subtest: bool, printer: Printer) -> Test: """ Finds next test to parse in LineStream, creates new Test object, @@ -759,7 +769,7 @@ def parse_test(lines: LineStream, expected_num: int, log: List[str], is_subtest: # If parsing the main/top-level test, parse KTAP version line and # test plan test.name = "main" - ktap_line = parse_ktap_header(lines, test, printer) + parse_ktap_header(lines, test, printer) test.log.extend(parse_diagnostic(lines)) parse_test_plan(lines, test) parent_test = True @@ -768,13 +778,12 @@ def parse_test(lines: LineStream, expected_num: int, log: List[str], is_subtest: # the KTAP version line and/or subtest header line ktap_line = parse_ktap_header(lines, test, printer) subtest_line = parse_test_header(lines, test) + test.log.extend(parse_diagnostic(lines)) + parse_test_plan(lines, test) parent_test = (ktap_line or subtest_line) if parent_test: - # If KTAP version line and/or subtest header is found, attempt - # to parse test plan and print test header - test.log.extend(parse_diagnostic(lines)) - parse_test_plan(lines, test) print_test_header(test, printer) + expected_count = test.expected_count subtests = [] test_num = 1 @@ -810,6 +819,10 @@ def parse_test(lines: LineStream, expected_num: int, log: List[str], is_subtest: test.log.extend(parse_diagnostic(lines)) if test.name != "" and not peek_test_name_match(lines, test): test.add_error(printer, 'missing subtest result line!') + elif not lines: + print_log(test.log, printer) + test.status = TestStatus.NO_TESTS + test.add_error(printer, 'No more test results!') else: parse_test_result(lines, test, expected_num, printer) @@ -849,7 +862,8 @@ def parse_run_tests(kernel_output: Iterable[str], printer: Printer) -> Test: test = Test() if not lines: test.name = '<missing>' - test.add_error(printer, 'Could not find any KTAP output. Did any KUnit tests run?') + test.add_error(printer, 'Could not find any KTAP output. Did any KUnit tests run?\n' + + 'Try running with the --raw_output=all option to see any log messages.') test.status = TestStatus.FAILURE_TO_PARSE_TESTS else: test = parse_test(lines, 0, [], False, printer) diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py index 0bcb0cc002f8..da88c3a1651d 100755 --- a/tools/testing/kunit/kunit_tool_test.py +++ b/tools/testing/kunit/kunit_tool_test.py @@ -11,17 +11,20 @@ from unittest import mock import tempfile, shutil # Handling test_tmpdir +import io import itertools import json import os import signal import subprocess +import sys from typing import Iterable import kunit_config import kunit_parser import kunit_kernel import kunit_json +import kunit_junit import kunit from kunit_printer import stdout @@ -36,7 +39,7 @@ def setUpModule(): def tearDownModule(): shutil.rmtree(test_tmpdir) -def test_data_path(path): +def _test_data_path(path): return os.path.join(abs_test_data_dir, path) class KconfigTest(unittest.TestCase): @@ -52,7 +55,7 @@ class KconfigTest(unittest.TestCase): self.assertFalse(kconfig1.is_subset_of(kconfig0)) def test_read_from_file(self): - kconfig_path = test_data_path('test_read_from_file.kconfig') + kconfig_path = _test_data_path('test_read_from_file.kconfig') kconfig = kunit_config.parse_file(kconfig_path) @@ -98,7 +101,7 @@ class KUnitParserTest(unittest.TestCase): raise AssertionError(f'"{needle}" not found in {list(backup)}!') def test_output_isolated_correctly(self): - log_path = test_data_path('test_output_isolated_correctly.log') + log_path = _test_data_path('test_output_isolated_correctly.log') with open(log_path) as file: result = kunit_parser.extract_tap_lines(file.readlines()) self.assertContains('TAP version 14', result) @@ -109,7 +112,7 @@ class KUnitParserTest(unittest.TestCase): self.assertContains('ok 1 - example', result) def test_output_with_prefix_isolated_correctly(self): - log_path = test_data_path('test_pound_sign.log') + log_path = _test_data_path('test_pound_sign.log') with open(log_path) as file: result = kunit_parser.extract_tap_lines(file.readlines()) self.assertContains('TAP version 14', result) @@ -138,35 +141,46 @@ class KUnitParserTest(unittest.TestCase): self.assertContains('ok 3 - string-stream-test', result) def test_parse_successful_test_log(self): - all_passed_log = test_data_path('test_is_test_passed-all_passed.log') + all_passed_log = _test_data_path('test_is_test_passed-all_passed.log') with open(all_passed_log) as file: result = kunit_parser.parse_run_tests(file.readlines(), stdout) self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status) self.assertEqual(result.counts.errors, 0) def test_parse_successful_nested_tests_log(self): - all_passed_log = test_data_path('test_is_test_passed-all_passed_nested.log') + all_passed_log = _test_data_path('test_is_test_passed-all_passed_nested.log') with open(all_passed_log) as file: result = kunit_parser.parse_run_tests(file.readlines(), stdout) self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status) self.assertEqual(result.counts.errors, 0) def test_kselftest_nested(self): - kselftest_log = test_data_path('test_is_test_passed-kselftest.log') + kselftest_log = _test_data_path('test_is_test_passed-kselftest.log') with open(kselftest_log) as file: result = kunit_parser.parse_run_tests(file.readlines(), stdout) self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status) self.assertEqual(result.counts.errors, 0) def test_parse_failed_test_log(self): - failed_log = test_data_path('test_is_test_passed-failure.log') + failed_log = _test_data_path('test_is_test_passed-failure.log') with open(failed_log) as file: result = kunit_parser.parse_run_tests(file.readlines(), stdout) self.assertEqual(kunit_parser.TestStatus.FAILURE, result.status) self.assertEqual(result.counts.errors, 0) + def test_parse_failed_nested_tests_log(self): + nested_log = _test_data_path('test_is_test_passed-failure-nested.log') + with open(nested_log) as file: + result = kunit_parser.parse_run_tests(file.readlines(), stdout) + self.assertEqual(kunit_parser.TestStatus.FAILURE, result.status) + self.assertEqual(result.counts.failed, 2) + self.assertEqual(kunit_parser.TestStatus.FAILURE, result.subtests[0].status) + self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.subtests[0].subtests[0].status) + self.assertEqual(kunit_parser.TestStatus.FAILURE, result.subtests[1].status) + self.assertEqual(kunit_parser.TestStatus.FAILURE, result.subtests[1].subtests[0].status) + def test_no_header(self): - empty_log = test_data_path('test_is_test_passed-no_tests_run_no_header.log') + empty_log = _test_data_path('test_is_test_passed-no_tests_run_no_header.log') with open(empty_log) as file: result = kunit_parser.parse_run_tests( kunit_parser.extract_tap_lines(file.readlines()), stdout) @@ -175,7 +189,7 @@ class KUnitParserTest(unittest.TestCase): self.assertEqual(result.counts.errors, 1) def test_missing_test_plan(self): - missing_plan_log = test_data_path('test_is_test_passed-' + missing_plan_log = _test_data_path('test_is_test_passed-' 'missing_plan.log') with open(missing_plan_log) as file: result = kunit_parser.parse_run_tests( @@ -186,7 +200,7 @@ class KUnitParserTest(unittest.TestCase): self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status) def test_no_tests(self): - header_log = test_data_path('test_is_test_passed-no_tests_run_with_header.log') + header_log = _test_data_path('test_is_test_passed-no_tests_run_with_header.log') with open(header_log) as file: result = kunit_parser.parse_run_tests( kunit_parser.extract_tap_lines(file.readlines()), stdout) @@ -195,7 +209,7 @@ class KUnitParserTest(unittest.TestCase): self.assertEqual(result.counts.errors, 1) def test_no_tests_no_plan(self): - no_plan_log = test_data_path('test_is_test_passed-no_tests_no_plan.log') + no_plan_log = _test_data_path('test_is_test_passed-no_tests_no_plan.log') with open(no_plan_log) as file: result = kunit_parser.parse_run_tests( kunit_parser.extract_tap_lines(file.readlines()), stdout) @@ -207,7 +221,7 @@ class KUnitParserTest(unittest.TestCase): def test_no_kunit_output(self): - crash_log = test_data_path('test_insufficient_memory.log') + crash_log = _test_data_path('test_insufficient_memory.log') print_mock = mock.patch('kunit_printer.Printer.print').start() with open(crash_log) as file: result = kunit_parser.parse_run_tests( @@ -218,16 +232,33 @@ class KUnitParserTest(unittest.TestCase): self.assertEqual(result.counts.errors, 1) def test_skipped_test(self): - skipped_log = test_data_path('test_skip_tests.log') + skipped_log = _test_data_path('test_skip_tests.log') with open(skipped_log) as file: result = kunit_parser.parse_run_tests(file.readlines(), stdout) + # The test result is skipped, and the skip reason is valid + self.assertEqual(kunit_parser.TestStatus.SKIPPED, result.subtests[1].subtests[1].status) + self.assertEqual("this test should be skipped", result.subtests[1].subtests[1].skip_reason) + # A skipped test does not fail the whole suite. self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status) self.assertEqual(result.counts, kunit_parser.TestCounts(passed=4, skipped=1)) + def test_skipped_reason_parse(self): + skipped_log = _test_data_path('test_skip_all_tests.log') + with open(skipped_log) as file: + result = kunit_parser.parse_run_tests(file.readlines(), stdout) + + # The first test is skipped, with the correct reaons + self.assertEqual(kunit_parser.TestStatus.SKIPPED, result.subtests[0].subtests[0].status) + self.assertEqual("all tests skipped", result.subtests[0].subtests[0].skip_reason) + + # The first suite is skipped, with no reason + self.assertEqual(kunit_parser.TestStatus.SKIPPED, result.subtests[0].status) + self.assertEqual("", result.subtests[0].skip_reason) + def test_skipped_all_tests(self): - skipped_log = test_data_path('test_skip_all_tests.log') + skipped_log = _test_data_path('test_skip_all_tests.log') with open(skipped_log) as file: result = kunit_parser.parse_run_tests(file.readlines(), stdout) @@ -235,7 +266,7 @@ class KUnitParserTest(unittest.TestCase): self.assertEqual(result.counts, kunit_parser.TestCounts(skipped=5)) def test_ignores_hyphen(self): - hyphen_log = test_data_path('test_strip_hyphen.log') + hyphen_log = _test_data_path('test_strip_hyphen.log') with open(hyphen_log) as file: result = kunit_parser.parse_run_tests(file.readlines(), stdout) @@ -249,7 +280,7 @@ class KUnitParserTest(unittest.TestCase): result.subtests[1].name) def test_ignores_prefix_printk_time(self): - prefix_log = test_data_path('test_config_printk_time.log') + prefix_log = _test_data_path('test_config_printk_time.log') with open(prefix_log) as file: result = kunit_parser.parse_run_tests(file.readlines(), stdout) self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status) @@ -257,7 +288,7 @@ class KUnitParserTest(unittest.TestCase): self.assertEqual(result.counts.errors, 0) def test_ignores_multiple_prefixes(self): - prefix_log = test_data_path('test_multiple_prefixes.log') + prefix_log = _test_data_path('test_multiple_prefixes.log') with open(prefix_log) as file: result = kunit_parser.parse_run_tests(file.readlines(), stdout) self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status) @@ -265,7 +296,7 @@ class KUnitParserTest(unittest.TestCase): self.assertEqual(result.counts.errors, 0) def test_prefix_mixed_kernel_output(self): - mixed_prefix_log = test_data_path('test_interrupted_tap_output.log') + mixed_prefix_log = _test_data_path('test_interrupted_tap_output.log') with open(mixed_prefix_log) as file: result = kunit_parser.parse_run_tests(file.readlines(), stdout) self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status) @@ -273,7 +304,7 @@ class KUnitParserTest(unittest.TestCase): self.assertEqual(result.counts.errors, 0) def test_prefix_poundsign(self): - pound_log = test_data_path('test_pound_sign.log') + pound_log = _test_data_path('test_pound_sign.log') with open(pound_log) as file: result = kunit_parser.parse_run_tests(file.readlines(), stdout) self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status) @@ -281,7 +312,7 @@ class KUnitParserTest(unittest.TestCase): self.assertEqual(result.counts.errors, 0) def test_kernel_panic_end(self): - panic_log = test_data_path('test_kernel_panic_interrupt.log') + panic_log = _test_data_path('test_kernel_panic_interrupt.log') with open(panic_log) as file: result = kunit_parser.parse_run_tests(file.readlines(), stdout) self.assertEqual(kunit_parser.TestStatus.TEST_CRASHED, result.status) @@ -289,7 +320,7 @@ class KUnitParserTest(unittest.TestCase): self.assertGreaterEqual(result.counts.errors, 1) def test_pound_no_prefix(self): - pound_log = test_data_path('test_pound_no_prefix.log') + pound_log = _test_data_path('test_pound_no_prefix.log') with open(pound_log) as file: result = kunit_parser.parse_run_tests(file.readlines(), stdout) self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status) @@ -318,7 +349,7 @@ class KUnitParserTest(unittest.TestCase): 'Failures: all_failed_suite, some_failed_suite.test2') def test_ktap_format(self): - ktap_log = test_data_path('test_parse_ktap_output.log') + ktap_log = _test_data_path('test_parse_ktap_output.log') with open(ktap_log) as file: result = kunit_parser.parse_run_tests(file.readlines(), stdout) self.assertEqual(result.counts, kunit_parser.TestCounts(passed=3)) @@ -327,13 +358,13 @@ class KUnitParserTest(unittest.TestCase): self.assertEqual('case_2', result.subtests[0].subtests[1].name) def test_parse_subtest_header(self): - ktap_log = test_data_path('test_parse_subtest_header.log') + ktap_log = _test_data_path('test_parse_subtest_header.log') with open(ktap_log) as file: kunit_parser.parse_run_tests(file.readlines(), stdout) self.print_mock.assert_any_call(StrContains('suite (1 subtest)')) def test_parse_attributes(self): - ktap_log = test_data_path('test_parse_attributes.log') + ktap_log = _test_data_path('test_parse_attributes.log') with open(ktap_log) as file: result = kunit_parser.parse_run_tests(file.readlines(), stdout) @@ -363,6 +394,17 @@ class KUnitParserTest(unittest.TestCase): self.print_mock.assert_any_call(StrContains(' Indented more.')) self.noPrintCallContains('not ok 1 test1') + def test_parse_late_test_plan(self): + output = """ + TAP version 13 + ok 4 test4 + 1..4 + """ + result = kunit_parser.parse_run_tests(output.splitlines(), stdout) + # Missing test results after test plan should alert a suspected test crash. + self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status) + self.assertEqual(result.counts, kunit_parser.TestCounts(passed=1, errors=2)) + def line_stream_from_strs(strs: Iterable[str]) -> kunit_parser.LineStream: return kunit_parser.LineStream(enumerate(strs, start=1)) @@ -455,7 +497,8 @@ class LinuxSourceTreeTest(unittest.TestCase): want_kconfig = kunit_config.Kconfig() want_kconfig.add_entry('NOT_REAL', 'y') - tree = kunit_kernel.LinuxSourceTree('', kconfig_add=['CONFIG_NOT_REAL=y']) + tree = kunit_kernel.LinuxSourceTree('', kunitconfig_paths=[os.devnull], + kconfig_add=['CONFIG_NOT_REAL=y']) self.assertTrue(want_kconfig.is_subset_of(tree._kconfig), msg=tree._kconfig) def test_invalid_arch(self): @@ -467,7 +510,7 @@ class LinuxSourceTreeTest(unittest.TestCase): return subprocess.Popen(['echo "hi\nbye"'], shell=True, text=True, stdout=subprocess.PIPE) with tempfile.TemporaryDirectory('') as build_dir: - tree = kunit_kernel.LinuxSourceTree(build_dir) + tree = kunit_kernel.LinuxSourceTree(build_dir, kunitconfig_paths=[os.devnull]) mock.patch.object(tree._ops, 'start', side_effect=fake_start).start() with self.assertRaises(ValueError): @@ -478,6 +521,74 @@ class LinuxSourceTreeTest(unittest.TestCase): with open(kunit_kernel.get_outfile_path(build_dir), 'rt') as outfile: self.assertEqual(outfile.read(), 'hi\nbye\n', msg='Missing some output') + def test_run_kernel_args_not_mutated(self): + """Verify run_kernel() copies args so callers can reuse them.""" + start_calls = [] + + def fake_start(start_args, unused_build_dir): + start_calls.append(list(start_args)) + return subprocess.Popen(['printf', 'KTAP version 1\n'], + text=True, stdout=subprocess.PIPE) + + with tempfile.TemporaryDirectory('') as build_dir: + tree = kunit_kernel.LinuxSourceTree(build_dir, + kunitconfig_paths=[os.devnull]) + with mock.patch.object(tree._ops, 'start', side_effect=fake_start), \ + mock.patch.object(kunit_kernel.subprocess, 'call'): + kernel_args = ['mem=1G'] + for _ in tree.run_kernel(args=kernel_args, build_dir=build_dir, + filter_glob='suite.test1'): + pass + for _ in tree.run_kernel(args=kernel_args, build_dir=build_dir, + filter_glob='suite.test2'): + pass + self.assertEqual(kernel_args, ['mem=1G'], + 'run_kernel() should not modify caller args') + self.assertIn('kunit.filter_glob=suite.test1', start_calls[0]) + self.assertIn('kunit.filter_glob=suite.test2', start_calls[1]) + + def test_run_kernel_skips_terminal_reset_without_tty(self): + def fake_start(unused_args, unused_build_dir): + return subprocess.Popen(['printf', 'KTAP version 1\n'], + text=True, stdout=subprocess.PIPE) + + non_tty_stdin = mock.Mock() + non_tty_stdin.isatty.return_value = False + + with tempfile.TemporaryDirectory('') as build_dir: + tree = kunit_kernel.LinuxSourceTree(build_dir, kunitconfig_paths=[os.devnull]) + with mock.patch.object(tree._ops, 'start', side_effect=fake_start), \ + mock.patch.object(kunit_kernel.sys, 'stdin', non_tty_stdin), \ + mock.patch.object(kunit_kernel.subprocess, 'call') as mock_call: + for _ in tree.run_kernel(build_dir=build_dir): + pass + + mock_call.assert_not_called() + + def test_signal_handler_skips_terminal_reset_without_tty(self): + non_tty_stdin = mock.Mock() + non_tty_stdin.isatty.return_value = False + tree = kunit_kernel.LinuxSourceTree('', kunitconfig_paths=[os.devnull]) + + with mock.patch.object(kunit_kernel.sys, 'stdin', non_tty_stdin), \ + mock.patch.object(kunit_kernel.subprocess, 'call') as mock_call, \ + mock.patch.object(kunit_kernel.logging, 'error') as mock_error: + tree.signal_handler(signal.SIGINT, None) + mock_error.assert_called_once() + mock_call.assert_not_called() + + def test_signal_handler_resets_terminal_with_tty(self): + tty_stdin = mock.Mock() + tty_stdin.isatty.return_value = True + tree = kunit_kernel.LinuxSourceTree('', kunitconfig_paths=[os.devnull]) + + with mock.patch.object(kunit_kernel.sys, 'stdin', tty_stdin), \ + mock.patch.object(kunit_kernel.subprocess, 'call') as mock_call, \ + mock.patch.object(kunit_kernel.logging, 'error') as mock_error: + tree.signal_handler(signal.SIGINT, None) + mock_error.assert_called_once() + mock_call.assert_called_once_with(['stty', 'sane']) + def test_build_reconfig_no_config(self): with tempfile.TemporaryDirectory('') as build_dir: with open(kunit_kernel.get_kunitconfig_path(build_dir), 'w') as f: @@ -544,7 +655,7 @@ class KUnitJsonTest(unittest.TestCase): self.addCleanup(mock.patch.stopall) def _json_for(self, log_file): - with open(test_data_path(log_file)) as file: + with open(_test_data_path(log_file)) as file: test_result = kunit_parser.parse_run_tests(file, stdout) json_obj = kunit_json.get_json_result( test=test_result, @@ -583,13 +694,46 @@ class StrContains(str): def __eq__(self, other): return self in other +class KUnitJUnitTest(unittest.TestCase): + def setUp(self): + self.print_mock = mock.patch('kunit_printer.Printer.print').start() + self.addCleanup(mock.patch.stopall) + + def _junit_string(self, log_file): + with open(_test_data_path(log_file)) as file: + test_result = kunit_parser.parse_run_tests(file, stdout) + junit_string = kunit_junit.get_junit_result( + test=test_result) + print(junit_string) + return junit_string + + def test_failed_test_junit(self): + result = self._junit_string('test_is_test_passed-failure.log') + self.assertTrue("<failure>" in result) + + def test_skipped_test_junit(self): + result = self._junit_string('test_skip_tests.log') + self.assertTrue("<skipped>" in result) + self.assertTrue("skipped=\"1\"" in result) + + def test_crashed_test_junit(self): + result = self._junit_string('test_kernel_panic_interrupt.log') + self.assertTrue("<error>" in result); + + def test_no_tests_junit(self): + result = self._junit_string('test_is_test_passed-no_tests_run_with_header.log') + self.assertTrue("tests=\"0\"" in result) + self.assertFalse("testcase" in result) + + class KUnitMainTest(unittest.TestCase): def setUp(self): - path = test_data_path('test_is_test_passed-all_passed.log') + path = _test_data_path('test_is_test_passed-all_passed.log') with open(path) as file: all_passed_log = file.readlines() self.print_mock = mock.patch('kunit_printer.Printer.print').start() + mock.patch.dict(os.environ, clear=True).start() self.addCleanup(mock.patch.stopall) self.mock_linux_init = mock.patch.object(kunit_kernel, 'LinuxSourceTree').start() @@ -712,6 +856,24 @@ class KUnitMainTest(unittest.TestCase): args=None, build_dir=build_dir, filter_glob='', filter='', filter_action=None, timeout=300) self.print_mock.assert_any_call(StrContains('Testing complete.')) + @mock.patch.dict(os.environ, {'KBUILD_OUTPUT': '/tmp'}) + def test_run_builddir_from_env(self): + build_dir = '/tmp/.kunit' + kunit.main(['run']) + self.assertEqual(self.linux_source_mock.build_reconfig.call_count, 1) + self.linux_source_mock.run_kernel.assert_called_once_with( + args=None, build_dir=build_dir, filter_glob='', filter='', filter_action=None, timeout=300) + self.print_mock.assert_any_call(StrContains('Testing complete.')) + + @mock.patch.dict(os.environ, {'KBUILD_OUTPUT': '/tmp'}) + def test_run_builddir_override(self): + build_dir = '.kunit' + kunit.main(['run', '--build_dir=.kunit']) + self.assertEqual(self.linux_source_mock.build_reconfig.call_count, 1) + self.linux_source_mock.run_kernel.assert_called_once_with( + args=None, build_dir=build_dir, filter_glob='', filter='', filter_action=None, timeout=300) + self.print_mock.assert_any_call(StrContains('Testing complete.')) + def test_config_builddir(self): build_dir = '.kunit' kunit.main(['config', '--build_dir', build_dir]) @@ -811,7 +973,7 @@ class KUnitMainTest(unittest.TestCase): self.linux_source_mock.run_kernel.return_value = ['TAP version 14', 'init: random output'] + want got = kunit._list_tests(self.linux_source_mock, - kunit.KunitExecRequest(None, None, False, False, '.kunit', 300, 'suite*', '', None, None, 'suite', False, False)) + kunit.KunitExecRequest(None, None, None, False, False, '.kunit', 300, 'suite*', '', None, None, 'suite', False, False, False)) self.assertEqual(got, want) # Should respect the user's filter glob when listing tests. self.linux_source_mock.run_kernel.assert_called_once_with( @@ -824,7 +986,7 @@ class KUnitMainTest(unittest.TestCase): # Should respect the user's filter glob when listing tests. mock_tests.assert_called_once_with(mock.ANY, - kunit.KunitExecRequest(None, None, False, False, '.kunit', 300, 'suite*.test*', '', None, None, 'suite', False, False)) + kunit.KunitExecRequest(None, None, None, False, False, '.kunit', 300, 'suite*.test*', '', None, None, 'suite', False, False, False)) self.linux_source_mock.run_kernel.assert_has_calls([ mock.call(args=None, build_dir='.kunit', filter_glob='suite.test*', filter='', filter_action=None, timeout=300), mock.call(args=None, build_dir='.kunit', filter_glob='suite2.test*', filter='', filter_action=None, timeout=300), @@ -837,12 +999,41 @@ class KUnitMainTest(unittest.TestCase): # Should respect the user's filter glob when listing tests. mock_tests.assert_called_once_with(mock.ANY, - kunit.KunitExecRequest(None, None, False, False, '.kunit', 300, 'suite*', '', None, None, 'test', False, False)) + kunit.KunitExecRequest(None, None, None, False, False, '.kunit', 300, 'suite*', '', None, None, 'test', False, False, False)) self.linux_source_mock.run_kernel.assert_has_calls([ mock.call(args=None, build_dir='.kunit', filter_glob='suite.test1', filter='', filter_action=None, timeout=300), mock.call(args=None, build_dir='.kunit', filter_glob='suite.test2', filter='', filter_action=None, timeout=300), mock.call(args=None, build_dir='.kunit', filter_glob='suite2.test1', filter='', filter_action=None, timeout=300), ]) + @mock.patch.object(kunit, '_list_tests') + @mock.patch.object(sys, 'stdout', new_callable=io.StringIO) + def test_list_suites(self, mock_stdout, mock_tests): + mock_tests.return_value = ['suite.test1', 'suite.test2', 'suite2.test1'] + kunit.main(['run', '--list_suites']) + + want = ['suite', 'suite2'] + output = mock_stdout.getvalue().split() + self.assertEqual(output, want) + + @mock.patch.object(sys, 'stdout', new_callable=io.StringIO) + def test_list_cmds(self, mock_stdout): + kunit.main(['--list-cmds']) + output = mock_stdout.getvalue() + output_cmds = sorted(output.split()) + expected_cmds = sorted(['build', 'config', 'exec', 'parse', 'run']) + self.assertEqual(output_cmds, expected_cmds) + + @mock.patch.object(sys, 'stdout', new_callable=io.StringIO) + def test_run_list_opts(self, mock_stdout): + kunit.main(['run', '--list-opts']) + output = mock_stdout.getvalue() + output_cmds = set(output.split()) + self.assertIn('--help', output_cmds) + self.assertIn('--kunitconfig', output_cmds) + self.assertIn('--jobs', output_cmds) + self.assertIn('--kernel_args', output_cmds) + self.assertIn('--raw_output', output_cmds) + if __name__ == '__main__': unittest.main() diff --git a/tools/testing/kunit/qemu_configs/armeb.py b/tools/testing/kunit/qemu_configs/armeb.py new file mode 100644 index 000000000000..86d326651490 --- /dev/null +++ b/tools/testing/kunit/qemu_configs/armeb.py @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: GPL-2.0 + +from ..qemu_config import QemuArchParams + +QEMU_ARCH = QemuArchParams(linux_arch='arm', + kconfig=''' +CONFIG_CPU_BIG_ENDIAN=y +CONFIG_ARCH_VIRT=y +CONFIG_SERIAL_AMBA_PL010=y +CONFIG_SERIAL_AMBA_PL010_CONSOLE=y +CONFIG_SERIAL_AMBA_PL011=y +CONFIG_SERIAL_AMBA_PL011_CONSOLE=y''', + qemu_arch='arm', + kernel_path='arch/arm/boot/zImage', + kernel_command_line='console=ttyAMA0', + extra_qemu_params=['-machine', 'virt']) diff --git a/tools/testing/kunit/qemu_configs/mips.py b/tools/testing/kunit/qemu_configs/mips.py new file mode 100644 index 000000000000..8899ac157b30 --- /dev/null +++ b/tools/testing/kunit/qemu_configs/mips.py @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: GPL-2.0 + +from ..qemu_config import QemuArchParams + +QEMU_ARCH = QemuArchParams(linux_arch='mips', + kconfig=''' +CONFIG_32BIT=y +CONFIG_CPU_BIG_ENDIAN=y +CONFIG_MIPS_MALTA=y +CONFIG_SERIAL_8250=y +CONFIG_SERIAL_8250_CONSOLE=y +CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_SYSCON=y +''', + qemu_arch='mips', + kernel_path='vmlinuz', + kernel_command_line='console=ttyS0', + extra_qemu_params=['-M', 'malta']) diff --git a/tools/testing/kunit/qemu_configs/mips64.py b/tools/testing/kunit/qemu_configs/mips64.py new file mode 100644 index 000000000000..1478aed05b94 --- /dev/null +++ b/tools/testing/kunit/qemu_configs/mips64.py @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: GPL-2.0 + +from ..qemu_config import QemuArchParams + +QEMU_ARCH = QemuArchParams(linux_arch='mips', + kconfig=''' +CONFIG_CPU_MIPS64_R2=y +CONFIG_64BIT=y +CONFIG_CPU_BIG_ENDIAN=y +CONFIG_MIPS_MALTA=y +CONFIG_SERIAL_8250=y +CONFIG_SERIAL_8250_CONSOLE=y +CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_SYSCON=y +''', + qemu_arch='mips64', + kernel_path='vmlinuz', + kernel_command_line='console=ttyS0', + extra_qemu_params=['-M', 'malta', '-cpu', '5KEc']) diff --git a/tools/testing/kunit/qemu_configs/mips64el.py b/tools/testing/kunit/qemu_configs/mips64el.py new file mode 100644 index 000000000000..300c711d7a82 --- /dev/null +++ b/tools/testing/kunit/qemu_configs/mips64el.py @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: GPL-2.0 + +from ..qemu_config import QemuArchParams + +QEMU_ARCH = QemuArchParams(linux_arch='mips', + kconfig=''' +CONFIG_CPU_MIPS64_R2=y +CONFIG_64BIT=y +CONFIG_CPU_LITTLE_ENDIAN=y +CONFIG_MIPS_MALTA=y +CONFIG_SERIAL_8250=y +CONFIG_SERIAL_8250_CONSOLE=y +CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_SYSCON=y +''', + qemu_arch='mips64el', + kernel_path='vmlinuz', + kernel_command_line='console=ttyS0', + extra_qemu_params=['-M', 'malta', '-cpu', '5KEc']) diff --git a/tools/testing/kunit/qemu_configs/mipsel.py b/tools/testing/kunit/qemu_configs/mipsel.py new file mode 100644 index 000000000000..3d3543315b45 --- /dev/null +++ b/tools/testing/kunit/qemu_configs/mipsel.py @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: GPL-2.0 + +from ..qemu_config import QemuArchParams + +QEMU_ARCH = QemuArchParams(linux_arch='mips', + kconfig=''' +CONFIG_32BIT=y +CONFIG_CPU_LITTLE_ENDIAN=y +CONFIG_MIPS_MALTA=y +CONFIG_SERIAL_8250=y +CONFIG_SERIAL_8250_CONSOLE=y +CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_SYSCON=y +''', + qemu_arch='mipsel', + kernel_path='vmlinuz', + kernel_command_line='console=ttyS0', + extra_qemu_params=['-M', 'malta']) diff --git a/tools/testing/kunit/qemu_configs/or1k.py b/tools/testing/kunit/qemu_configs/or1k.py new file mode 100644 index 000000000000..dfbbad0f9076 --- /dev/null +++ b/tools/testing/kunit/qemu_configs/or1k.py @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: GPL-2.0-only +from ..qemu_config import QemuArchParams + +QEMU_ARCH = QemuArchParams(linux_arch='openrisc', + kconfig=''' +CONFIG_SERIAL_8250=y +CONFIG_SERIAL_8250_CONSOLE=y +CONFIG_SERIAL_OF_PLATFORM=y +CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_SYSCON=y +''', + qemu_arch='or1k', + kernel_path='vmlinux', + kernel_command_line='console=ttyS0', + extra_qemu_params=[ + '-machine', 'virt', + '-m', '512', + ]) diff --git a/tools/testing/kunit/qemu_configs/powerpc.py b/tools/testing/kunit/qemu_configs/powerpc.py index 7ec38d4131f7..5b4c895d5d5a 100644 --- a/tools/testing/kunit/qemu_configs/powerpc.py +++ b/tools/testing/kunit/qemu_configs/powerpc.py @@ -3,6 +3,7 @@ from ..qemu_config import QemuArchParams QEMU_ARCH = QemuArchParams(linux_arch='powerpc', kconfig=''' CONFIG_PPC64=y +CONFIG_CPU_BIG_ENDIAN=y CONFIG_SERIAL_8250=y CONFIG_SERIAL_8250_CONSOLE=y CONFIG_HVC_CONSOLE=y''', diff --git a/tools/testing/kunit/qemu_configs/powerpc32.py b/tools/testing/kunit/qemu_configs/powerpc32.py new file mode 100644 index 000000000000..88bd60dbb948 --- /dev/null +++ b/tools/testing/kunit/qemu_configs/powerpc32.py @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: GPL-2.0 + +from ..qemu_config import QemuArchParams + +QEMU_ARCH = QemuArchParams(linux_arch='powerpc', + kconfig=''' +CONFIG_PPC32=y +CONFIG_CPU_BIG_ENDIAN=y +CONFIG_ADB_CUDA=y +CONFIG_SERIAL_PMACZILOG=y +CONFIG_SERIAL_PMACZILOG_TTYS=y +CONFIG_SERIAL_PMACZILOG_CONSOLE=y +''', + qemu_arch='ppc', + kernel_path='vmlinux', + kernel_command_line='console=ttyS0', + extra_qemu_params=['-M', 'g3beige', '-cpu', 'max']) diff --git a/tools/testing/kunit/qemu_configs/powerpcle.py b/tools/testing/kunit/qemu_configs/powerpcle.py new file mode 100644 index 000000000000..7ddee8af4bd7 --- /dev/null +++ b/tools/testing/kunit/qemu_configs/powerpcle.py @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: GPL-2.0 + +from ..qemu_config import QemuArchParams + +QEMU_ARCH = QemuArchParams(linux_arch='powerpc', + kconfig=''' +CONFIG_PPC64=y +CONFIG_CPU_LITTLE_ENDIAN=y +CONFIG_HVC_CONSOLE=y +''', + qemu_arch='ppc64', + kernel_path='vmlinux', + kernel_command_line='console=ttyS0', + extra_qemu_params=['-M', 'pseries', '-cpu', 'power8']) diff --git a/tools/testing/kunit/qemu_configs/riscv32.py b/tools/testing/kunit/qemu_configs/riscv32.py new file mode 100644 index 000000000000..b79ba0ae30f8 --- /dev/null +++ b/tools/testing/kunit/qemu_configs/riscv32.py @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: GPL-2.0 + +from ..qemu_config import QemuArchParams + +QEMU_ARCH = QemuArchParams(linux_arch='riscv', + kconfig=''' +CONFIG_NONPORTABLE=y +CONFIG_ARCH_RV32I=y +CONFIG_ARCH_VIRT=y +CONFIG_SERIAL_8250=y +CONFIG_SERIAL_8250_CONSOLE=y +CONFIG_SERIAL_OF_PLATFORM=y +''', + qemu_arch='riscv32', + kernel_path='arch/riscv/boot/Image', + kernel_command_line='console=ttyS0', + extra_qemu_params=['-machine', 'virt']) diff --git a/tools/testing/kunit/qemu_configs/sh.py b/tools/testing/kunit/qemu_configs/sh.py index 78a474a5b95f..f00cb89fdef6 100644 --- a/tools/testing/kunit/qemu_configs/sh.py +++ b/tools/testing/kunit/qemu_configs/sh.py @@ -7,7 +7,9 @@ CONFIG_CPU_SUBTYPE_SH7751R=y CONFIG_MEMORY_START=0x0c000000 CONFIG_SH_RTS7751R2D=y CONFIG_RTS7751R2D_PLUS=y -CONFIG_SERIAL_SH_SCI=y''', +CONFIG_SERIAL_SH_SCI=y +CONFIG_CMDLINE_EXTEND=y +''', qemu_arch='sh4', kernel_path='arch/sh/boot/zImage', kernel_command_line='console=ttySC1', diff --git a/tools/testing/kunit/qemu_configs/sparc.py b/tools/testing/kunit/qemu_configs/sparc.py index e975c4331a7c..2019550a1b69 100644 --- a/tools/testing/kunit/qemu_configs/sparc.py +++ b/tools/testing/kunit/qemu_configs/sparc.py @@ -2,8 +2,11 @@ from ..qemu_config import QemuArchParams QEMU_ARCH = QemuArchParams(linux_arch='sparc', kconfig=''' -CONFIG_SERIAL_8250=y -CONFIG_SERIAL_8250_CONSOLE=y''', +CONFIG_KUNIT_FAULT_TEST=n +CONFIG_SPARC32=y +CONFIG_SERIAL_SUNZILOG=y +CONFIG_SERIAL_SUNZILOG_CONSOLE=y +''', qemu_arch='sparc', kernel_path='arch/sparc/boot/zImage', kernel_command_line='console=ttyS0 mem=256M', diff --git a/tools/testing/kunit/qemu_configs/sparc64.py b/tools/testing/kunit/qemu_configs/sparc64.py new file mode 100644 index 000000000000..53d4e5a8c972 --- /dev/null +++ b/tools/testing/kunit/qemu_configs/sparc64.py @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: GPL-2.0 + +from ..qemu_config import QemuArchParams + +QEMU_ARCH = QemuArchParams(linux_arch='sparc', + kconfig=''' +CONFIG_64BIT=y +CONFIG_SPARC64=y +CONFIG_PCI=y +CONFIG_SERIAL_SUNSU=y +CONFIG_SERIAL_SUNSU_CONSOLE=y +''', + qemu_arch='sparc64', + kernel_path='arch/sparc/boot/image', + kernel_command_line='console=ttyS0 kunit_shutdown=poweroff', + extra_qemu_params=[]) diff --git a/tools/testing/kunit/qemu_configs/x86_64.py b/tools/testing/kunit/qemu_configs/x86_64.py index dc7949076863..4a6bf4e048f5 100644 --- a/tools/testing/kunit/qemu_configs/x86_64.py +++ b/tools/testing/kunit/qemu_configs/x86_64.py @@ -7,4 +7,6 @@ CONFIG_SERIAL_8250_CONSOLE=y''', qemu_arch='x86_64', kernel_path='arch/x86/boot/bzImage', kernel_command_line='console=ttyS0', - extra_qemu_params=[]) + # qboot is faster than SeaBIOS and doesn't mess up + # the terminal. + extra_qemu_params=['-bios', 'qboot.rom']) diff --git a/tools/testing/kunit/test_data/test_is_test_passed-failure-nested.log b/tools/testing/kunit/test_data/test_is_test_passed-failure-nested.log new file mode 100644 index 000000000000..5498dfd0b0db --- /dev/null +++ b/tools/testing/kunit/test_data/test_is_test_passed-failure-nested.log @@ -0,0 +1,10 @@ +KTAP version 1 +1..2 + KTAP version 1 + 1..1 + ok 1 test 1 +not ok 1 subtest 1 + KTAP version 1 + 1..1 + not ok 1 subsubtest 1 +not ok 2 subtest 2 diff --git a/tools/testing/kunit/test_data/test_is_test_passed-kselftest.log b/tools/testing/kunit/test_data/test_is_test_passed-kselftest.log index 65d3f27feaf2..30d9ef18bcec 100644 --- a/tools/testing/kunit/test_data/test_is_test_passed-kselftest.log +++ b/tools/testing/kunit/test_data/test_is_test_passed-kselftest.log @@ -1,5 +1,5 @@ TAP version 13 -1..2 +1..3 # selftests: membarrier: membarrier_test_single_thread # TAP version 13 # 1..2 @@ -12,3 +12,4 @@ ok 1 selftests: membarrier: membarrier_test_single_thread # ok 1 sys_membarrier available # ok 2 sys membarrier invalid command test: command = -1, flags = 0, errno = 22. Failed as expected ok 2 selftests: membarrier: membarrier_test_multi_thread +ok 3 # SKIP selftests: membarrier: membarrier_test_multi_thread |