diff options
| author | Linus Torvalds <torvalds@linux-foundation.org> | 2026-06-16 16:33:57 +0530 |
|---|---|---|
| committer | Linus Torvalds <torvalds@linux-foundation.org> | 2026-06-16 16:33:57 +0530 |
| commit | 42eb3a5ef6bc56192bf450c79a3f274e081f8131 (patch) | |
| tree | cd5e440cd913b4005909eafdd613acad5807783d /tools | |
| parent | b1cbabe84ca1381a004fb91ee1791a1a53bce44e (diff) | |
| parent | 29afed142d64e181749214072315c976f8510bd7 (diff) | |
| download | lwn-42eb3a5ef6bc56192bf450c79a3f274e081f8131.tar.gz lwn-42eb3a5ef6bc56192bf450c79a3f274e081f8131.zip | |
Merge tag 'linux_kselftest-kunit-7.2-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest
Pull kunit updates from Shuah Khan:
"Fixes to tool and kunit core and new features to both to support JUnit
XML (primitive) and backtrace suppression API:
- Core support for suppressing warning backtraces
- Parse and print the reason tests are skipped
- Add (primitive) support for outputting JUnit XML
- Don't write to stdout when it should be disabled
- Add backtrace suppression self-tests
- Suppress intentional warning backtraces in scaling unit tests
- Add documentation for warning backtrace suppression API
- Fix spelling mistakes in comments and messages
- gen_compile_commands: Ignore libgcc.a
- qemu_configs: Add or1k / openrisc configuration"
* tag 'linux_kselftest-kunit-7.2-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest:
kunit:tool: Don't write to stdout when it should be disabled
kunit: tool: Add (primitive) support for outputting JUnit XML
kunit: tool: Parse and print the reason tests are skipped
kunit: Add documentation for warning backtrace suppression API
drm: Suppress intentional warning backtraces in scaling unit tests
kunit: Add backtrace suppression self-tests
bug/kunit: Core support for suppressing warning backtraces
kunit: Fix spelling mistakes in comments and messages
kunit: qemu_configs: Add or1k / openrisc configuration
gen_compile_commands: Ignore libgcc.a
Diffstat (limited to 'tools')
| -rwxr-xr-x | tools/testing/kunit/kunit.py | 21 | ||||
| -rw-r--r-- | tools/testing/kunit/kunit_junit.py | 61 | ||||
| -rw-r--r-- | tools/testing/kunit/kunit_kernel.py | 2 | ||||
| -rw-r--r-- | tools/testing/kunit/kunit_parser.py | 29 | ||||
| -rwxr-xr-x | tools/testing/kunit/kunit_tool_test.py | 56 | ||||
| -rw-r--r-- | tools/testing/kunit/qemu_configs/or1k.py | 18 |
6 files changed, 170 insertions, 17 deletions
diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py index 742f5c555666..ac3f7159e67f 100755 --- a/tools/testing/kunit/kunit.py +++ b/tools/testing/kunit/kunit.py @@ -21,6 +21,7 @@ from enum import Enum, auto from typing import Iterable, List, Optional, Sequence, Tuple import kunit_json +import kunit_junit import kunit_kernel import kunit_parser from kunit_printer import stdout, null_printer @@ -49,6 +50,7 @@ class KunitBuildRequest(KunitConfigRequest): class KunitParseRequest: raw_output: Optional[str] json: Optional[str] + junit: Optional[str] summary: bool failed: bool @@ -268,6 +270,13 @@ def parse_tests(request: KunitParseRequest, metadata: kunit_json.Metadata, input stdout.print_with_timestamp("Test results stored in %s" % os.path.abspath(request.json)) + if request.junit: + if request.junit == 'stdout': + kunit_junit.print_junit_result(test=test) + else: + kunit_junit.write_junit_result(test=test,filename=request.junit) + stdout.print_with_timestamp(f"Test results stored in {os.path.abspath(request.junit)}") + if test.status != kunit_parser.TestStatus.SUCCESS: return KunitResult(KunitStatus.TEST_FAILURE, parse_time), test @@ -309,6 +318,7 @@ def run_tests(linux: kunit_kernel.LinuxSourceTree, # So we hackily automatically rewrite --json => --json=stdout pseudo_bool_flag_defaults = { '--json': 'stdout', + '--junit': 'stdout', '--raw_output': 'kunit', } def massage_argv(argv: Sequence[str]) -> Sequence[str]: @@ -459,6 +469,11 @@ def add_parse_opts(parser: argparse.ArgumentParser) -> None: help='Prints parsed test results as JSON to stdout or a file if ' 'a filename is specified. Does nothing if --raw_output is set.', type=str, const='stdout', default=None, metavar='FILE') + parser.add_argument('--junit', + nargs='?', + help='Prints parsed test results as JUnit XML to stdout or a file if ' + 'a filename is specified. Does nothing if --raw_output is set.', + type=str, const='stdout', default=None, metavar='FILE') parser.add_argument('--summary', help='Prints only the summary line for parsed test results.' 'Does nothing if --raw_output is set.', @@ -502,6 +517,7 @@ def run_handler(cli_args: argparse.Namespace) -> None: jobs=cli_args.jobs, raw_output=cli_args.raw_output, json=cli_args.json, + junit=cli_args.junit, summary=cli_args.summary, failed=cli_args.failed, timeout=cli_args.timeout, @@ -552,6 +568,7 @@ def exec_handler(cli_args: argparse.Namespace) -> None: exec_request = KunitExecRequest(raw_output=cli_args.raw_output, build_dir=cli_args.build_dir, json=cli_args.json, + junit=cli_args.junit, summary=cli_args.summary, failed=cli_args.failed, timeout=cli_args.timeout, @@ -580,7 +597,9 @@ def parse_handler(cli_args: argparse.Namespace) -> None: # We know nothing about how the result was created! metadata = kunit_json.Metadata() request = KunitParseRequest(raw_output=cli_args.raw_output, - json=cli_args.json, summary=cli_args.summary, + json=cli_args.json, + junit=cli_args.junit, + summary=cli_args.summary, failed=cli_args.failed) result, _ = parse_tests(request, metadata, kunit_output) if result.status != KunitStatus.SUCCESS: diff --git a/tools/testing/kunit/kunit_junit.py b/tools/testing/kunit/kunit_junit.py new file mode 100644 index 000000000000..3622070358e7 --- /dev/null +++ b/tools/testing/kunit/kunit_junit.py @@ -0,0 +1,61 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Generates JUnit XML files from KUnit test results +# +# Copyright (C) 2026, Google LLC and David Gow. + +from xml.sax.saxutils import quoteattr, XMLGenerator +import xml.etree.ElementTree as ET +from kunit_parser import Test, TestStatus +from typing import Optional + +# Get a string representing a tes suite (including subtests) in JUnit XML +def get_test_suite(test: Test, parent: Optional[ET.Element]) -> ET.Element: + suite_attrs = { + 'name': test.name, + 'tests': str(test.counts.total()), + 'failures': str(test.counts.failed), + 'skipped': str(test.counts.skipped), + 'errors': str(test.counts.crashed + test.counts.errors), + } + + if parent is not None: + test_suite_element = ET.SubElement(parent, 'testsuite', suite_attrs) + else: + test_suite_element = ET.Element('testsuite', suite_attrs) + + for subtest in test.subtests: + if subtest.subtests: + get_test_suite(subtest, test_suite_element) + continue + test_case_element = ET.SubElement(test_suite_element, 'testcase', {'name': subtest.name}) + if subtest.status == TestStatus.FAILURE: + ET.SubElement(test_case_element, 'failure', {}).text = 'Test Failed' + elif subtest.status == TestStatus.SKIPPED: + ET.SubElement(test_case_element, 'skipped', {}).text = subtest.skip_reason + elif subtest.status == TestStatus.TEST_CRASHED: + ET.SubElement(test_case_element, 'error', {}).text = 'Test Crashed' + + if subtest.log: + ET.SubElement(test_case_element, 'system-out', {}).text = "\n".join(subtest.log) + + return test_suite_element + +# Get a string for an entire XML file for the test structure starting at test +def get_junit_result(test: Test) -> str: + root_element = get_test_suite(test, None) + ET.indent(root_element) + return ET.tostring(root_element, encoding="unicode", xml_declaration=True) + +# Print a JUnit result to stdout. +def print_junit_result(test: Test) -> None: + root_element = get_test_suite(test, None) + ET.indent(root_element) + ET.dump(root_element) + +# Write an entire XML file for the test structure starting at test +def write_junit_result(test: Test, filename: str) -> None: + root_element = get_test_suite(test, None) + ET.indent(root_element) + root_et = ET.ElementTree(root_element) + root_et.write(filename, encoding='utf-8', xml_declaration=True) diff --git a/tools/testing/kunit/kunit_kernel.py b/tools/testing/kunit/kunit_kernel.py index 2869fcb199ff..58557c47d85f 100644 --- a/tools/testing/kunit/kunit_kernel.py +++ b/tools/testing/kunit/kunit_kernel.py @@ -218,7 +218,7 @@ def _get_qemu_ops(config_path: str, # exists (I learned this through experimentation and could not find it # anywhere in the Python documentation). # - # Bascially, we completely ignore the actual file location of the config + # Basically, we completely ignore the actual file location of the config # we are loading and just tell Python that the module lives in the # QEMU_CONFIGS_DIR for import purposes regardless of where it actually # exists as a file. diff --git a/tools/testing/kunit/kunit_parser.py b/tools/testing/kunit/kunit_parser.py index 1c61a0ed740d..d722874bc660 100644 --- a/tools/testing/kunit/kunit_parser.py +++ b/tools/testing/kunit/kunit_parser.py @@ -17,7 +17,7 @@ import textwrap from enum import Enum, auto from typing import Iterable, Iterator, List, Optional, Tuple -from kunit_printer import Printer, stdout +from kunit_printer import Printer class Test: """ @@ -44,11 +44,12 @@ class Test: self.subtests = [] # type: List[Test] self.log = [] # type: List[str] self.counts = TestCounts() + self.skip_reason = '' def __str__(self) -> str: """Returns string representation of a Test class object.""" return (f'Test({self.status}, {self.name}, {self.expected_count}, ' - f'{self.subtests}, {self.log}, {self.counts})') + f'{self.subtests}, {self.log}, {self.counts}, {self.skip_reason})') def __repr__(self) -> str: """Returns string representation of a Test class object.""" @@ -57,7 +58,7 @@ class Test: def add_error(self, printer: Printer, error_message: str) -> None: """Records an error that occurred while parsing this test.""" self.counts.errors += 1 - printer.print_with_timestamp(stdout.red('[ERROR]') + f' Test: {self.name}: {error_message}') + printer.print_with_timestamp(printer.red('[ERROR]') + f' Test: {self.name}: {error_message}') def ok_status(self) -> bool: """Returns true if the status was ok, i.e. passed or skipped.""" @@ -268,7 +269,7 @@ def check_version(version_num: int, accepted_versions: List[int], if version_num < min(accepted_versions): test.add_error(printer, f'{version_type} version lower than expected!') elif version_num > max(accepted_versions): - test.add_error(printer, f'{version_type} version higer than expected!') + test.add_error(printer, f'{version_type} version higher than expected!') def parse_ktap_header(lines: LineStream, test: Test, printer: Printer) -> bool: """ @@ -352,9 +353,9 @@ def parse_test_plan(lines: LineStream, test: Test) -> bool: lines.pop() return True -TEST_RESULT = re.compile(r'^\s*(ok|not ok) ([0-9]+) ?(- )?([^#]*)( # .*)?$') +TEST_RESULT = re.compile(r'^\s*(ok|not ok) ([0-9]+) ?(:?- )?([^#]*)( # .*)?$') -TEST_RESULT_SKIP = re.compile(r'^\s*(ok|not ok) ([0-9]+) ?(- )?(.*) # SKIP ?(.*)$') +TEST_RESULT_SKIP = re.compile(r'^\s*(ok|not ok) ([0-9]+) ?(:?- )?(.*) # SKIP ?(.*)$') def peek_test_name_match(lines: LineStream, test: Test) -> bool: """ @@ -418,7 +419,7 @@ def parse_test_result(lines: LineStream, test: Test, # Set name of test object if skip_match: - test.name = skip_match.group(4) or skip_match.group(5) + test.name = skip_match.group(4) else: test.name = match.group(4) @@ -431,6 +432,7 @@ def parse_test_result(lines: LineStream, test: Test, status = match.group(1) if skip_match: test.status = TestStatus.SKIPPED + test.skip_reason = skip_match.group(5) or '' elif status == 'ok': test.status = TestStatus.SUCCESS else: @@ -539,12 +541,15 @@ def format_test_result(test: Test, printer: Printer) -> str: if test.status == TestStatus.SUCCESS: return printer.green('[PASSED] ') + test.name if test.status == TestStatus.SKIPPED: - return printer.yellow('[SKIPPED] ') + test.name + skip_message = printer.yellow('[SKIPPED] ') + test.name + if test.skip_reason != '': + skip_message += printer.yellow(' (' + test.skip_reason + ')') + return skip_message if test.status == TestStatus.NO_TESTS: return printer.yellow('[NO TESTS RUN] ') + test.name if test.status == TestStatus.TEST_CRASHED: print_log(test.log, printer) - return stdout.red('[CRASHED] ') + test.name + return printer.red('[CRASHED] ') + test.name print_log(test.log, printer) return printer.red('[FAILED] ') + test.name @@ -651,11 +656,11 @@ def print_summary_line(test: Test, printer: Printer) -> None: printer - Printer object to output results """ if test.status == TestStatus.SUCCESS: - color = stdout.green + color = printer.green elif test.status in (TestStatus.SKIPPED, TestStatus.NO_TESTS): - color = stdout.yellow + color = printer.yellow else: - color = stdout.red + color = printer.red printer.print_with_timestamp(color(f'Testing complete. {test.counts}')) # Summarize failures that might have gone off-screen since we had a lot diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py index 267c33cecf87..da88c3a1651d 100755 --- a/tools/testing/kunit/kunit_tool_test.py +++ b/tools/testing/kunit/kunit_tool_test.py @@ -24,6 +24,7 @@ import kunit_config import kunit_parser import kunit_kernel import kunit_json +import kunit_junit import kunit from kunit_printer import stdout @@ -235,10 +236,27 @@ class KUnitParserTest(unittest.TestCase): with open(skipped_log) as file: result = kunit_parser.parse_run_tests(file.readlines(), stdout) + # The test result is skipped, and the skip reason is valid + self.assertEqual(kunit_parser.TestStatus.SKIPPED, result.subtests[1].subtests[1].status) + self.assertEqual("this test should be skipped", result.subtests[1].subtests[1].skip_reason) + # A skipped test does not fail the whole suite. self.assertEqual(kunit_parser.TestStatus.SUCCESS, result.status) self.assertEqual(result.counts, kunit_parser.TestCounts(passed=4, skipped=1)) + def test_skipped_reason_parse(self): + skipped_log = _test_data_path('test_skip_all_tests.log') + with open(skipped_log) as file: + result = kunit_parser.parse_run_tests(file.readlines(), stdout) + + # The first test is skipped, with the correct reaons + self.assertEqual(kunit_parser.TestStatus.SKIPPED, result.subtests[0].subtests[0].status) + self.assertEqual("all tests skipped", result.subtests[0].subtests[0].skip_reason) + + # The first suite is skipped, with no reason + self.assertEqual(kunit_parser.TestStatus.SKIPPED, result.subtests[0].status) + self.assertEqual("", result.subtests[0].skip_reason) + def test_skipped_all_tests(self): skipped_log = _test_data_path('test_skip_all_tests.log') with open(skipped_log) as file: @@ -676,6 +694,38 @@ class StrContains(str): def __eq__(self, other): return self in other +class KUnitJUnitTest(unittest.TestCase): + def setUp(self): + self.print_mock = mock.patch('kunit_printer.Printer.print').start() + self.addCleanup(mock.patch.stopall) + + def _junit_string(self, log_file): + with open(_test_data_path(log_file)) as file: + test_result = kunit_parser.parse_run_tests(file, stdout) + junit_string = kunit_junit.get_junit_result( + test=test_result) + print(junit_string) + return junit_string + + def test_failed_test_junit(self): + result = self._junit_string('test_is_test_passed-failure.log') + self.assertTrue("<failure>" in result) + + def test_skipped_test_junit(self): + result = self._junit_string('test_skip_tests.log') + self.assertTrue("<skipped>" in result) + self.assertTrue("skipped=\"1\"" in result) + + def test_crashed_test_junit(self): + result = self._junit_string('test_kernel_panic_interrupt.log') + self.assertTrue("<error>" in result); + + def test_no_tests_junit(self): + result = self._junit_string('test_is_test_passed-no_tests_run_with_header.log') + self.assertTrue("tests=\"0\"" in result) + self.assertFalse("testcase" in result) + + class KUnitMainTest(unittest.TestCase): def setUp(self): path = _test_data_path('test_is_test_passed-all_passed.log') @@ -923,7 +973,7 @@ class KUnitMainTest(unittest.TestCase): self.linux_source_mock.run_kernel.return_value = ['TAP version 14', 'init: random output'] + want got = kunit._list_tests(self.linux_source_mock, - kunit.KunitExecRequest(None, None, False, False, '.kunit', 300, 'suite*', '', None, None, 'suite', False, False, False)) + kunit.KunitExecRequest(None, None, None, False, False, '.kunit', 300, 'suite*', '', None, None, 'suite', False, False, False)) self.assertEqual(got, want) # Should respect the user's filter glob when listing tests. self.linux_source_mock.run_kernel.assert_called_once_with( @@ -936,7 +986,7 @@ class KUnitMainTest(unittest.TestCase): # Should respect the user's filter glob when listing tests. mock_tests.assert_called_once_with(mock.ANY, - kunit.KunitExecRequest(None, None, False, False, '.kunit', 300, 'suite*.test*', '', None, None, 'suite', False, False, False)) + kunit.KunitExecRequest(None, None, None, False, False, '.kunit', 300, 'suite*.test*', '', None, None, 'suite', False, False, False)) self.linux_source_mock.run_kernel.assert_has_calls([ mock.call(args=None, build_dir='.kunit', filter_glob='suite.test*', filter='', filter_action=None, timeout=300), mock.call(args=None, build_dir='.kunit', filter_glob='suite2.test*', filter='', filter_action=None, timeout=300), @@ -949,7 +999,7 @@ class KUnitMainTest(unittest.TestCase): # Should respect the user's filter glob when listing tests. mock_tests.assert_called_once_with(mock.ANY, - kunit.KunitExecRequest(None, None, False, False, '.kunit', 300, 'suite*', '', None, None, 'test', False, False, False)) + kunit.KunitExecRequest(None, None, None, False, False, '.kunit', 300, 'suite*', '', None, None, 'test', False, False, False)) self.linux_source_mock.run_kernel.assert_has_calls([ mock.call(args=None, build_dir='.kunit', filter_glob='suite.test1', filter='', filter_action=None, timeout=300), mock.call(args=None, build_dir='.kunit', filter_glob='suite.test2', filter='', filter_action=None, timeout=300), diff --git a/tools/testing/kunit/qemu_configs/or1k.py b/tools/testing/kunit/qemu_configs/or1k.py new file mode 100644 index 000000000000..dfbbad0f9076 --- /dev/null +++ b/tools/testing/kunit/qemu_configs/or1k.py @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: GPL-2.0-only +from ..qemu_config import QemuArchParams + +QEMU_ARCH = QemuArchParams(linux_arch='openrisc', + kconfig=''' +CONFIG_SERIAL_8250=y +CONFIG_SERIAL_8250_CONSOLE=y +CONFIG_SERIAL_OF_PLATFORM=y +CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_SYSCON=y +''', + qemu_arch='or1k', + kernel_path='vmlinux', + kernel_command_line='console=ttyS0', + extra_qemu_params=[ + '-machine', 'virt', + '-m', '512', + ]) |
