From 2b144a30a407d29b7e6d24549f5316175115e788 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Mon, 2 Mar 2026 17:40:44 +0100 Subject: docs: kdoc_re: add support for groups() Add an equivalent to re groups() method. This is useful on debug messages. Signed-off-by: Mauro Carvalho Chehab Acked-by: Randy Dunlap Tested-by: Randy Dunlap Reviewed-by: Aleksandr Loktionov Signed-off-by: Jonathan Corbet Message-ID: <20d1a9c77200e28cc2ff1d6122635c43f8ba6a71.1772469446.git.mchehab+huawei@kernel.org> --- tools/lib/python/kdoc/kdoc_re.py | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'tools/lib/python/kdoc') diff --git a/tools/lib/python/kdoc/kdoc_re.py b/tools/lib/python/kdoc/kdoc_re.py index 0bf9e01cdc57..774dd747ecb0 100644 --- a/tools/lib/python/kdoc/kdoc_re.py +++ b/tools/lib/python/kdoc/kdoc_re.py @@ -106,6 +106,13 @@ class KernRe: return self.last_match.group(num) + def groups(self): + """ + Returns the group results of the last match + """ + + return self.last_match.groups() + class NestedMatch: """ -- cgit v1.2.3 From 8eb49357ffa229c9b65a002f655c1280dc09769a Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Mon, 2 Mar 2026 17:40:45 +0100 Subject: docs: kdoc_re: don't go past the end of a line The logic which checks if the line ends with ";" is currently broken: it may try to read past the buffer. Fix it by checking before trying to access line[pos]. Signed-off-by: Mauro Carvalho Chehab Acked-by: Randy Dunlap Tested-by: Randy Dunlap Reviewed-by: Aleksandr Loktionov Signed-off-by: Jonathan Corbet Message-ID: --- tools/lib/python/kdoc/kdoc_re.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tools/lib/python/kdoc') diff --git a/tools/lib/python/kdoc/kdoc_re.py b/tools/lib/python/kdoc/kdoc_re.py index 774dd747ecb0..6c44fcce0415 100644 --- a/tools/lib/python/kdoc/kdoc_re.py +++ b/tools/lib/python/kdoc/kdoc_re.py @@ -269,7 +269,7 @@ class NestedMatch: out += new_sub # Drop end ';' if any - if line[pos] == ';': + if pos < len(line) and line[pos] == ';': pos += 1 cur_pos = pos -- cgit v1.2.3 From 77e6e17e9fc4cb4e59ad97de5453bb6f963a5fd4 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Mon, 2 Mar 2026 17:40:46 +0100 Subject: docs: kdoc_parser: move var transformers to the beginning Just like functions and structs had their transform variables placed at the beginning, move variable transforms to there as well. No functional changes. Signed-off-by: Mauro Carvalho Chehab Acked-by: Randy Dunlap Tested-by: Randy Dunlap Reviewed-by: Aleksandr Loktionov Signed-off-by: Jonathan Corbet Message-ID: <491b290252a308f381f88353a3bbe9e2bd1f6a62.1772469446.git.mchehab+huawei@kernel.org> --- tools/lib/python/kdoc/kdoc_parser.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) (limited to 'tools/lib/python/kdoc') diff --git a/tools/lib/python/kdoc/kdoc_parser.py b/tools/lib/python/kdoc/kdoc_parser.py index ca00695b47b3..68a5aea9175d 100644 --- a/tools/lib/python/kdoc/kdoc_parser.py +++ b/tools/lib/python/kdoc/kdoc_parser.py @@ -192,6 +192,18 @@ function_xforms = [ (KernRe(r"__attribute__\s*\(\((?:[\w\s]+(?:\([^)]*\))?\s*,?)+\)\)\s+"), ""), ] +# +# Transforms for variable prototypes +# +var_xforms = [ + (KernRe(r"__read_mostly"), ""), + (KernRe(r"__ro_after_init"), ""), + (KernRe(r"(?://.*)$"), ""), + (KernRe(r"(?:/\*.*\*/)"), ""), + (KernRe(r";$"), ""), + (KernRe(r"=.*"), ""), +] + # # Ancillary functions # @@ -972,15 +984,6 @@ class KernelDoc: ] OPTIONAL_VAR_ATTR = "^(?:" + "|".join(VAR_ATTRIBS) + ")?" - sub_prefixes = [ - (KernRe(r"__read_mostly"), ""), - (KernRe(r"__ro_after_init"), ""), - (KernRe(r"(?://.*)$"), ""), - (KernRe(r"(?:/\*.*\*/)"), ""), - (KernRe(r";$"), ""), - (KernRe(r"=.*"), ""), - ] - # # Store the full prototype before modifying it # @@ -1004,7 +1007,7 @@ class KernelDoc: # Drop comments and macros to have a pure C prototype # if not declaration_name: - for r, sub in sub_prefixes: + for r, sub in var_xforms: proto = r.sub(sub, proto) proto = proto.rstrip() -- cgit v1.2.3 From cca1bbdd72f72a3cf86d90fd6f326fd709ae931f Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Mon, 2 Mar 2026 17:40:47 +0100 Subject: docs: kdoc_parser: don't mangle with function defines Mangling with #defines is not nice, as we may end removing the macro names, preventing several macros from being properly documented. Also, on defines, we have something like: #define foo(a1, a2, a3, ...) \ /* some real implementation */ The prototype part (first line on this example) won't contain any macros, so no need to apply any regexes on it. With that, move the apply_transforms() logic to ensure that it will be called only on functions. Signed-off-by: Mauro Carvalho Chehab Acked-by: Randy Dunlap Tested-by: Randy Dunlap Reviewed-by: Aleksandr Loktionov Signed-off-by: Jonathan Corbet Message-ID: <8f9854c8ca1c794b6a3fe418f7adbc32aa68b432.1772469446.git.mchehab+huawei@kernel.org> --- tools/lib/python/kdoc/kdoc_parser.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'tools/lib/python/kdoc') diff --git a/tools/lib/python/kdoc/kdoc_parser.py b/tools/lib/python/kdoc/kdoc_parser.py index 68a5aea9175d..9643ffb7584a 100644 --- a/tools/lib/python/kdoc/kdoc_parser.py +++ b/tools/lib/python/kdoc/kdoc_parser.py @@ -163,7 +163,7 @@ struct_nested_prefixes = [ # # Transforms for function prototypes # -function_xforms = [ +function_xforms = [ (KernRe(r"^static +"), ""), (KernRe(r"^extern +"), ""), (KernRe(r"^asmlinkage +"), ""), @@ -1066,10 +1066,7 @@ class KernelDoc: found = func_macro = False return_type = '' decl_type = 'function' - # - # Apply the initial transformations. - # - prototype = apply_transforms(function_xforms, prototype) + # # If we have a macro, remove the "#define" at the front. # @@ -1088,6 +1085,11 @@ class KernelDoc: declaration_name = r.group(1) func_macro = True found = True + else: + # + # Apply the initial transformations. + # + prototype = apply_transforms(function_xforms, prototype) # Yes, this truly is vile. We are looking for: # 1. Return type (may be nothing if we're looking at a macro) -- cgit v1.2.3 From 4fd349f03dc51bc2f9cd2ea9f6309b0bc2b848ca Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Mon, 2 Mar 2026 17:40:48 +0100 Subject: docs: kdoc_parser: fix variable regexes to work with size_t The regular expressions meant to pick variable types are too naive: they forgot that the type word may contain underlines. It also means that we need to change the regex which detects var attributes to handle "const". Co-developed-by: Randy Dunlap Signed-off-by: Mauro Carvalho Chehab Acked-by: Randy Dunlap Tested-by: Randy Dunlap Reviewed-by: Aleksandr Loktionov Signed-off-by: Jonathan Corbet Message-ID: <8230715239929cf9d475ab81ca1df7de65d82d06.1772469446.git.mchehab+huawei@kernel.org> --- tools/lib/python/kdoc/kdoc_parser.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'tools/lib/python/kdoc') diff --git a/tools/lib/python/kdoc/kdoc_parser.py b/tools/lib/python/kdoc/kdoc_parser.py index 9643ffb7584a..9c9443281c40 100644 --- a/tools/lib/python/kdoc/kdoc_parser.py +++ b/tools/lib/python/kdoc/kdoc_parser.py @@ -981,8 +981,9 @@ class KernelDoc: """ VAR_ATTRIBS = [ "extern", + "const", ] - OPTIONAL_VAR_ATTR = "^(?:" + "|".join(VAR_ATTRIBS) + ")?" + OPTIONAL_VAR_ATTR = r"^(?:\b(?:" +"|".join(VAR_ATTRIBS) +r")\b\s*)*" # # Store the full prototype before modifying it @@ -1018,14 +1019,14 @@ class KernelDoc: default_val = None - r= KernRe(OPTIONAL_VAR_ATTR + r"\w.*\s+(?:\*+)?([\w_]+)\s*[\d\]\[]*\s*(=.*)?") + r= KernRe(OPTIONAL_VAR_ATTR + r"[\w_]*\s+(?:\*+)?([\w_]+)\s*[\d\]\[]*\s*(=.*)?") if r.match(proto): if not declaration_name: declaration_name = r.group(1) default_val = r.group(2) else: - r= KernRe(OPTIONAL_VAR_ATTR + r"(?:\w.*)?\s+(?:\*+)?(?:[\w_]+)\s*[\d\]\[]*\s*(=.*)?") + r= KernRe(OPTIONAL_VAR_ATTR + r"(?:[\w_]*)?\s+(?:\*+)?(?:[\w_]+)\s*[\d\]\[]*\s*(=.*)?") if r.match(proto): default_val = r.group(1) -- cgit v1.2.3 From 9bbf22b87d866fa1e6a1f9f6376d2ef458b6dcc7 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Mon, 2 Mar 2026 17:40:49 +0100 Subject: docs: kdoc_parser: fix the default_value logic for variables The indentation is wrong for the second regex, which causes problems on variables with defaults. Signed-off-by: Mauro Carvalho Chehab Acked-by: Randy Dunlap Tested-by: Randy Dunlap Reviewed-by: Aleksandr Loktionov Signed-off-by: Jonathan Corbet Message-ID: <681f18338abd6ae33cb9c15d72bb31a1cba75a9a.1772469446.git.mchehab+huawei@kernel.org> --- tools/lib/python/kdoc/kdoc_parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'tools/lib/python/kdoc') diff --git a/tools/lib/python/kdoc/kdoc_parser.py b/tools/lib/python/kdoc/kdoc_parser.py index 9c9443281c40..4bf55244870f 100644 --- a/tools/lib/python/kdoc/kdoc_parser.py +++ b/tools/lib/python/kdoc/kdoc_parser.py @@ -1027,9 +1027,9 @@ class KernelDoc: default_val = r.group(2) else: r= KernRe(OPTIONAL_VAR_ATTR + r"(?:[\w_]*)?\s+(?:\*+)?(?:[\w_]+)\s*[\d\]\[]*\s*(=.*)?") - if r.match(proto): - default_val = r.group(1) + if r.match(proto): + default_val = r.group(1) if not declaration_name: self.emit_msg(ln,f"{proto}: can't parse variable") return -- cgit v1.2.3 From b7dc635459ad5b00f2d482406dbdca3291622ce2 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Mon, 2 Mar 2026 17:40:50 +0100 Subject: docs: kdoc_parser: don't exclude defaults from prototype If we do that, the defaults won't be parsed. Signed-off-by: Mauro Carvalho Chehab Acked-by: Randy Dunlap Tested-by: Randy Dunlap Reviewed-by: Aleksandr Loktionov Signed-off-by: Jonathan Corbet Message-ID: --- tools/lib/python/kdoc/kdoc_parser.py | 1 - 1 file changed, 1 deletion(-) (limited to 'tools/lib/python/kdoc') diff --git a/tools/lib/python/kdoc/kdoc_parser.py b/tools/lib/python/kdoc/kdoc_parser.py index 4bf55244870f..39ff27d421eb 100644 --- a/tools/lib/python/kdoc/kdoc_parser.py +++ b/tools/lib/python/kdoc/kdoc_parser.py @@ -201,7 +201,6 @@ var_xforms = [ (KernRe(r"(?://.*)$"), ""), (KernRe(r"(?:/\*.*\*/)"), ""), (KernRe(r";$"), ""), - (KernRe(r"=.*"), ""), ] # -- cgit v1.2.3 From 6d9c2e9575b8630e17571a77eef8ade84a2a6344 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Mon, 2 Mar 2026 17:40:51 +0100 Subject: docs: kdoc_parser: fix parser to support multi-word types The regular expression currently expects a single word for the type, but it may be something like "struct foo". Add support for it. Signed-off-by: Mauro Carvalho Chehab Acked-by: Randy Dunlap Tested-by: Randy Dunlap Reviewed-by: Aleksandr Loktionov Signed-off-by: Jonathan Corbet Message-ID: <544c73a9e670b6fef1828bf4f2ba0de7d29d8675.1772469446.git.mchehab+huawei@kernel.org> --- tools/lib/python/kdoc/kdoc_parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'tools/lib/python/kdoc') diff --git a/tools/lib/python/kdoc/kdoc_parser.py b/tools/lib/python/kdoc/kdoc_parser.py index 39ff27d421eb..22a820d33dc8 100644 --- a/tools/lib/python/kdoc/kdoc_parser.py +++ b/tools/lib/python/kdoc/kdoc_parser.py @@ -1018,14 +1018,14 @@ class KernelDoc: default_val = None - r= KernRe(OPTIONAL_VAR_ATTR + r"[\w_]*\s+(?:\*+)?([\w_]+)\s*[\d\]\[]*\s*(=.*)?") + r= KernRe(OPTIONAL_VAR_ATTR + r"\s*[\w_\s]*\s+(?:\*+)?([\w_]+)\s*[\d\]\[]*\s*(=.*)?") if r.match(proto): if not declaration_name: declaration_name = r.group(1) default_val = r.group(2) else: - r= KernRe(OPTIONAL_VAR_ATTR + r"(?:[\w_]*)?\s+(?:\*+)?(?:[\w_]+)\s*[\d\]\[]*\s*(=.*)?") + r= KernRe(OPTIONAL_VAR_ATTR + r"(?:[\w_\s]*)?\s+(?:\*+)?(?:[\w_]+)\s*[\d\]\[]*\s*(=.*)?") if r.match(proto): default_val = r.group(1) -- cgit v1.2.3 From 9bff5121fe22fdd0bb5bd6f744e136ec20bf7b95 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Mon, 2 Mar 2026 17:40:52 +0100 Subject: docs: kdoc_parser: add support for LIST_HEAD Convert LIST_HEAD into struct list_head when handling its prototype. Signed-off-by: Mauro Carvalho Chehab Acked-by: Randy Dunlap Tested-by: Randy Dunlap Reviewed-by: Aleksandr Loktionov Signed-off-by: Jonathan Corbet Message-ID: <8bdfa6ba6002b0a73a83660f0ce7b40e30124552.1772469446.git.mchehab+huawei@kernel.org> --- tools/lib/python/kdoc/kdoc_parser.py | 1 + 1 file changed, 1 insertion(+) (limited to 'tools/lib/python/kdoc') diff --git a/tools/lib/python/kdoc/kdoc_parser.py b/tools/lib/python/kdoc/kdoc_parser.py index 22a820d33dc8..1df869061bf3 100644 --- a/tools/lib/python/kdoc/kdoc_parser.py +++ b/tools/lib/python/kdoc/kdoc_parser.py @@ -198,6 +198,7 @@ function_xforms = [ var_xforms = [ (KernRe(r"__read_mostly"), ""), (KernRe(r"__ro_after_init"), ""), + (KernRe(r"LIST_HEAD\(([\w_]+)\)"), r"struct list_head \1"), (KernRe(r"(?://.*)$"), ""), (KernRe(r"(?:/\*.*\*/)"), ""), (KernRe(r";$"), ""), -- cgit v1.2.3 From 97d4e70bc2c6f75911a9a5e1a75f2de13fde9b6b Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Mon, 2 Mar 2026 17:40:53 +0100 Subject: docs: kdoc_parser: handle struct member macro VIRTIO_DECLARE_FEATURES(name) Parse the macro VIRTIO_DECLARE_FEATURES(name) and expand it to its definition. These prevents one build warning: WARNING: include/linux/virtio.h:188 struct member 'VIRTIO_DECLARE_FEATURES(features' not described in 'virtio_device' Signed-off-by: Randy Dunlap Signed-off-by: Mauro Carvalho Chehab Reviewed-by: Aleksandr Loktionov Signed-off-by: Jonathan Corbet Message-ID: <6f62e1f1210e74906fa50f4e937f66f54813661b.1772469446.git.mchehab+huawei@kernel.org> --- tools/lib/python/kdoc/kdoc_parser.py | 1 + 1 file changed, 1 insertion(+) (limited to 'tools/lib/python/kdoc') diff --git a/tools/lib/python/kdoc/kdoc_parser.py b/tools/lib/python/kdoc/kdoc_parser.py index 1df869061bf3..917e4528bfbf 100644 --- a/tools/lib/python/kdoc/kdoc_parser.py +++ b/tools/lib/python/kdoc/kdoc_parser.py @@ -150,6 +150,7 @@ struct_xforms = [ struct_args_pattern + r'\)', re.S), r'\1 \2[]'), (KernRe(r'DEFINE_DMA_UNMAP_ADDR\s*\(' + struct_args_pattern + r'\)', re.S), r'dma_addr_t \1'), (KernRe(r'DEFINE_DMA_UNMAP_LEN\s*\(' + struct_args_pattern + r'\)', re.S), r'__u32 \1'), + (KernRe(r'VIRTIO_DECLARE_FEATURES\(([\w_]+)\)'), r'union { u64 \1; u64 \1_array[VIRTIO_FEATURES_U64S]; }'), ] # # Regexes here are guaranteed to have the end delimiter matching -- cgit v1.2.3 From 95a9429cc6d31371575793ab7beb94bf3e7a2f92 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Mon, 2 Mar 2026 17:40:54 +0100 Subject: docs: kdoc_re: better show KernRe() at documentation the __repr__() function is used by autodoc to document macro initialization. Add a better representation for them. Signed-off-by: Mauro Carvalho Chehab Reviewed-by: Aleksandr Loktionov Signed-off-by: Jonathan Corbet Message-ID: <80d27732368c14125c1b76048a70d8b4aee527ef.1772469446.git.mchehab+huawei@kernel.org> --- tools/lib/python/kdoc/kdoc_re.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) (limited to 'tools/lib/python/kdoc') diff --git a/tools/lib/python/kdoc/kdoc_re.py b/tools/lib/python/kdoc/kdoc_re.py index 6c44fcce0415..664c04c8cc9f 100644 --- a/tools/lib/python/kdoc/kdoc_re.py +++ b/tools/lib/python/kdoc/kdoc_re.py @@ -52,7 +52,28 @@ class KernRe: return self.regex.pattern def __repr__(self): - return f're.compile("{self.regex.pattern}")' + """ + Returns a displayable version of the class init. + """ + + flag_map = { + re.IGNORECASE: "re.I", + re.MULTILINE: "re.M", + re.DOTALL: "re.S", + re.VERBOSE: "re.X", + } + + flags = [] + for flag, name in flag_map.items(): + if self.regex.flags & flag: + flags.append(name) + + flags_name = " | ".join(flags) + + if flags_name: + return f'KernRe("{self.regex.pattern}", {flags_name})' + else: + return f'KernRe("{self.regex.pattern}")' def __add__(self, other): """ -- cgit v1.2.3 From d842057c4a205084fb3036122c7426963f04e826 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Mon, 2 Mar 2026 17:40:55 +0100 Subject: docs: kdoc_parser: move transform lists to a separate file Over the time, most of the changes at kernel-doc are related to maintaining a list of transforms to convert macros into pure C code. Place such transforms on a separate module, to cleanup the parser module. There is an advantage on that: QEMU also uses our own kernel-doc, but the xforms list there is different. By placing it on a separate module, we can minimize the differences and make it easier to keep QEMU in sync with Kernel upstream. No functional changes. Signed-off-by: Mauro Carvalho Chehab Reviewed-by: Aleksandr Loktionov Signed-off-by: Jonathan Corbet Message-ID: --- Documentation/tools/kdoc_parser.rst | 8 ++ tools/lib/python/kdoc/kdoc_files.py | 3 +- tools/lib/python/kdoc/kdoc_parser.py | 145 ++------------------------------ tools/lib/python/kdoc/xforms_lists.py | 153 ++++++++++++++++++++++++++++++++++ 4 files changed, 168 insertions(+), 141 deletions(-) create mode 100644 tools/lib/python/kdoc/xforms_lists.py (limited to 'tools/lib/python/kdoc') diff --git a/Documentation/tools/kdoc_parser.rst b/Documentation/tools/kdoc_parser.rst index 03ee54a1b1cc..55b202173195 100644 --- a/Documentation/tools/kdoc_parser.rst +++ b/Documentation/tools/kdoc_parser.rst @@ -4,6 +4,14 @@ Kernel-doc parser stage ======================= +C replacement rules used by the parser +====================================== + +.. automodule:: lib.python.kdoc.xforms_lists + :members: + :show-inheritance: + :undoc-members: + File handler classes ==================== diff --git a/tools/lib/python/kdoc/kdoc_files.py b/tools/lib/python/kdoc/kdoc_files.py index 022487ea2cc6..33618c6abec2 100644 --- a/tools/lib/python/kdoc/kdoc_files.py +++ b/tools/lib/python/kdoc/kdoc_files.py @@ -15,6 +15,7 @@ import os import re from kdoc.kdoc_parser import KernelDoc +from kdoc.xforms_lists import CTransforms from kdoc.kdoc_output import OutputFormat @@ -117,7 +118,7 @@ class KernelFiles(): if fname in self.files: return - doc = KernelDoc(self.config, fname) + doc = KernelDoc(self.config, fname, CTransforms()) export_table, entries = doc.parse_kdoc() self.export_table[fname] = export_table diff --git a/tools/lib/python/kdoc/kdoc_parser.py b/tools/lib/python/kdoc/kdoc_parser.py index 917e4528bfbf..d7daf658e9d2 100644 --- a/tools/lib/python/kdoc/kdoc_parser.py +++ b/tools/lib/python/kdoc/kdoc_parser.py @@ -69,89 +69,6 @@ doc_begin_func = KernRe(str(doc_com) + # initial " * ' r'(?:[-:].*)?$', # description (not captured) cache = False) -# -# Here begins a long set of transformations to turn structure member prefixes -# and macro invocations into something we can parse and generate kdoc for. -# -struct_args_pattern = r'([^,)]+)' - -struct_xforms = [ - # Strip attributes - (KernRe(r"__attribute__\s*\(\([a-z0-9,_\*\s\(\)]*\)\)", flags=re.I | re.S, cache=False), ' '), - (KernRe(r'\s*__aligned\s*\([^;]*\)', re.S), ' '), - (KernRe(r'\s*__counted_by\s*\([^;]*\)', re.S), ' '), - (KernRe(r'\s*__counted_by_(le|be)\s*\([^;]*\)', re.S), ' '), - (KernRe(r'\s*__packed\s*', re.S), ' '), - (KernRe(r'\s*CRYPTO_MINALIGN_ATTR', re.S), ' '), - (KernRe(r'\s*__private', re.S), ' '), - (KernRe(r'\s*__rcu', re.S), ' '), - (KernRe(r'\s*____cacheline_aligned_in_smp', re.S), ' '), - (KernRe(r'\s*____cacheline_aligned', re.S), ' '), - (KernRe(r'\s*__cacheline_group_(begin|end)\([^\)]+\);'), ''), - # - # Unwrap struct_group macros based on this definition: - # __struct_group(TAG, NAME, ATTRS, MEMBERS...) - # which has variants like: struct_group(NAME, MEMBERS...) - # Only MEMBERS arguments require documentation. - # - # Parsing them happens on two steps: - # - # 1. drop struct group arguments that aren't at MEMBERS, - # storing them as STRUCT_GROUP(MEMBERS) - # - # 2. remove STRUCT_GROUP() ancillary macro. - # - # The original logic used to remove STRUCT_GROUP() using an - # advanced regex: - # - # \bSTRUCT_GROUP(\(((?:(?>[^)(]+)|(?1))*)\))[^;]*; - # - # with two patterns that are incompatible with - # Python re module, as it has: - # - # - a recursive pattern: (?1) - # - an atomic grouping: (?>...) - # - # I tried a simpler version: but it didn't work either: - # \bSTRUCT_GROUP\(([^\)]+)\)[^;]*; - # - # As it doesn't properly match the end parenthesis on some cases. - # - # So, a better solution was crafted: there's now a NestedMatch - # class that ensures that delimiters after a search are properly - # matched. So, the implementation to drop STRUCT_GROUP() will be - # handled in separate. - # - (KernRe(r'\bstruct_group\s*\(([^,]*,)', re.S), r'STRUCT_GROUP('), - (KernRe(r'\bstruct_group_attr\s*\(([^,]*,){2}', re.S), r'STRUCT_GROUP('), - (KernRe(r'\bstruct_group_tagged\s*\(([^,]*),([^,]*),', re.S), r'struct \1 \2; STRUCT_GROUP('), - (KernRe(r'\b__struct_group\s*\(([^,]*,){3}', re.S), r'STRUCT_GROUP('), - # - # Replace macros - # - # TODO: use NestedMatch for FOO($1, $2, ...) matches - # - # it is better to also move those to the NestedMatch logic, - # to ensure that parentheses will be properly matched. - # - (KernRe(r'__ETHTOOL_DECLARE_LINK_MODE_MASK\s*\(([^\)]+)\)', re.S), - r'DECLARE_BITMAP(\1, __ETHTOOL_LINK_MODE_MASK_NBITS)'), - (KernRe(r'DECLARE_PHY_INTERFACE_MASK\s*\(([^\)]+)\)', re.S), - r'DECLARE_BITMAP(\1, PHY_INTERFACE_MODE_MAX)'), - (KernRe(r'DECLARE_BITMAP\s*\(' + struct_args_pattern + r',\s*' + struct_args_pattern + r'\)', - re.S), r'unsigned long \1[BITS_TO_LONGS(\2)]'), - (KernRe(r'DECLARE_HASHTABLE\s*\(' + struct_args_pattern + r',\s*' + struct_args_pattern + r'\)', - re.S), r'unsigned long \1[1 << ((\2) - 1)]'), - (KernRe(r'DECLARE_KFIFO\s*\(' + struct_args_pattern + r',\s*' + struct_args_pattern + - r',\s*' + struct_args_pattern + r'\)', re.S), r'\2 *\1'), - (KernRe(r'DECLARE_KFIFO_PTR\s*\(' + struct_args_pattern + r',\s*' + - struct_args_pattern + r'\)', re.S), r'\2 *\1'), - (KernRe(r'(?:__)?DECLARE_FLEX_ARRAY\s*\(' + struct_args_pattern + r',\s*' + - struct_args_pattern + r'\)', re.S), r'\1 \2[]'), - (KernRe(r'DEFINE_DMA_UNMAP_ADDR\s*\(' + struct_args_pattern + r'\)', re.S), r'dma_addr_t \1'), - (KernRe(r'DEFINE_DMA_UNMAP_LEN\s*\(' + struct_args_pattern + r'\)', re.S), r'__u32 \1'), - (KernRe(r'VIRTIO_DECLARE_FEATURES\(([\w_]+)\)'), r'union { u64 \1; u64 \1_array[VIRTIO_FEATURES_U64S]; }'), -] # # Regexes here are guaranteed to have the end delimiter matching # the start delimiter. Yet, right now, only one replace group @@ -161,62 +78,10 @@ struct_nested_prefixes = [ (re.compile(r'\bSTRUCT_GROUP\('), r'\1'), ] -# -# Transforms for function prototypes -# -function_xforms = [ - (KernRe(r"^static +"), ""), - (KernRe(r"^extern +"), ""), - (KernRe(r"^asmlinkage +"), ""), - (KernRe(r"^inline +"), ""), - (KernRe(r"^__inline__ +"), ""), - (KernRe(r"^__inline +"), ""), - (KernRe(r"^__always_inline +"), ""), - (KernRe(r"^noinline +"), ""), - (KernRe(r"^__FORTIFY_INLINE +"), ""), - (KernRe(r"__init +"), ""), - (KernRe(r"__init_or_module +"), ""), - (KernRe(r"__exit +"), ""), - (KernRe(r"__deprecated +"), ""), - (KernRe(r"__flatten +"), ""), - (KernRe(r"__meminit +"), ""), - (KernRe(r"__must_check +"), ""), - (KernRe(r"__weak +"), ""), - (KernRe(r"__sched +"), ""), - (KernRe(r"_noprof"), ""), - (KernRe(r"__always_unused *"), ""), - (KernRe(r"__printf\s*\(\s*\d*\s*,\s*\d*\s*\) +"), ""), - (KernRe(r"__(?:re)?alloc_size\s*\(\s*\d+\s*(?:,\s*\d+\s*)?\) +"), ""), - (KernRe(r"__diagnose_as\s*\(\s*\S+\s*(?:,\s*\d+\s*)*\) +"), ""), - (KernRe(r"DECL_BUCKET_PARAMS\s*\(\s*(\S+)\s*,\s*(\S+)\s*\)"), r"\1, \2"), - (KernRe(r"__attribute_const__ +"), ""), - (KernRe(r"__attribute__\s*\(\((?:[\w\s]+(?:\([^)]*\))?\s*,?)+\)\)\s+"), ""), -] - -# -# Transforms for variable prototypes -# -var_xforms = [ - (KernRe(r"__read_mostly"), ""), - (KernRe(r"__ro_after_init"), ""), - (KernRe(r"LIST_HEAD\(([\w_]+)\)"), r"struct list_head \1"), - (KernRe(r"(?://.*)$"), ""), - (KernRe(r"(?:/\*.*\*/)"), ""), - (KernRe(r";$"), ""), -] - # # Ancillary functions # -def apply_transforms(xforms, text): - """ - Apply a set of transforms to a block of text. - """ - for search, subst in xforms: - text = search.sub(subst, text) - return text - multi_space = KernRe(r'\s\s+') def trim_whitespace(s): """ @@ -395,11 +260,12 @@ class KernelDoc: #: String to write when a parameter is not described. undescribed = "-- undescribed --" - def __init__(self, config, fname): + def __init__(self, config, fname, xforms): """Initialize internal variables""" self.fname = fname self.config = config + self.xforms = xforms # Initial state for the state machines self.state = state.NORMAL @@ -883,7 +749,7 @@ class KernelDoc: # Go through the list of members applying all of our transformations. # members = trim_private_members(members) - members = apply_transforms(struct_xforms, members) + members = self.xforms.apply("struct", members) nested = NestedMatch() for search, sub in struct_nested_prefixes: @@ -1009,8 +875,7 @@ class KernelDoc: # Drop comments and macros to have a pure C prototype # if not declaration_name: - for r, sub in var_xforms: - proto = r.sub(sub, proto) + proto = self.xforms.apply("var", proto) proto = proto.rstrip() @@ -1091,7 +956,7 @@ class KernelDoc: # # Apply the initial transformations. # - prototype = apply_transforms(function_xforms, prototype) + prototype = self.xforms.apply("func", prototype) # Yes, this truly is vile. We are looking for: # 1. Return type (may be nothing if we're looking at a macro) diff --git a/tools/lib/python/kdoc/xforms_lists.py b/tools/lib/python/kdoc/xforms_lists.py new file mode 100644 index 000000000000..e6e0302e5dd0 --- /dev/null +++ b/tools/lib/python/kdoc/xforms_lists.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +# Copyright(c) 2026: Mauro Carvalho Chehab . + +import re + +from kdoc.kdoc_re import KernRe + +struct_args_pattern = r'([^,)]+)' + +class CTransforms: + """ + Data class containing a long set of transformations to turn + structure member prefixes, and macro invocations and variables + into something we can parse and generate kdoc for. + """ + + #: Transforms for structs and unions. + struct_xforms = [ + # Strip attributes + (KernRe(r"__attribute__\s*\(\([a-z0-9,_\*\s\(\)]*\)\)", flags=re.I | re.S, cache=False), ' '), + (KernRe(r'\s*__aligned\s*\([^;]*\)', re.S), ' '), + (KernRe(r'\s*__counted_by\s*\([^;]*\)', re.S), ' '), + (KernRe(r'\s*__counted_by_(le|be)\s*\([^;]*\)', re.S), ' '), + (KernRe(r'\s*__packed\s*', re.S), ' '), + (KernRe(r'\s*CRYPTO_MINALIGN_ATTR', re.S), ' '), + (KernRe(r'\s*__private', re.S), ' '), + (KernRe(r'\s*__rcu', re.S), ' '), + (KernRe(r'\s*____cacheline_aligned_in_smp', re.S), ' '), + (KernRe(r'\s*____cacheline_aligned', re.S), ' '), + (KernRe(r'\s*__cacheline_group_(begin|end)\([^\)]+\);'), ''), + # + # Unwrap struct_group macros based on this definition: + # __struct_group(TAG, NAME, ATTRS, MEMBERS...) + # which has variants like: struct_group(NAME, MEMBERS...) + # Only MEMBERS arguments require documentation. + # + # Parsing them happens on two steps: + # + # 1. drop struct group arguments that aren't at MEMBERS, + # storing them as STRUCT_GROUP(MEMBERS) + # + # 2. remove STRUCT_GROUP() ancillary macro. + # + # The original logic used to remove STRUCT_GROUP() using an + # advanced regex: + # + # \bSTRUCT_GROUP(\(((?:(?>[^)(]+)|(?1))*)\))[^;]*; + # + # with two patterns that are incompatible with + # Python re module, as it has: + # + # - a recursive pattern: (?1) + # - an atomic grouping: (?>...) + # + # I tried a simpler version: but it didn't work either: + # \bSTRUCT_GROUP\(([^\)]+)\)[^;]*; + # + # As it doesn't properly match the end parenthesis on some cases. + # + # So, a better solution was crafted: there's now a NestedMatch + # class that ensures that delimiters after a search are properly + # matched. So, the implementation to drop STRUCT_GROUP() will be + # handled in separate. + # + (KernRe(r'\bstruct_group\s*\(([^,]*,)', re.S), r'STRUCT_GROUP('), + (KernRe(r'\bstruct_group_attr\s*\(([^,]*,){2}', re.S), r'STRUCT_GROUP('), + (KernRe(r'\bstruct_group_tagged\s*\(([^,]*),([^,]*),', re.S), r'struct \1 \2; STRUCT_GROUP('), + (KernRe(r'\b__struct_group\s*\(([^,]*,){3}', re.S), r'STRUCT_GROUP('), + # + # Replace macros + # + # TODO: use NestedMatch for FOO($1, $2, ...) matches + # + # it is better to also move those to the NestedMatch logic, + # to ensure that parentheses will be properly matched. + # + (KernRe(r'__ETHTOOL_DECLARE_LINK_MODE_MASK\s*\(([^\)]+)\)', re.S), + r'DECLARE_BITMAP(\1, __ETHTOOL_LINK_MODE_MASK_NBITS)'), + (KernRe(r'DECLARE_PHY_INTERFACE_MASK\s*\(([^\)]+)\)', re.S), + r'DECLARE_BITMAP(\1, PHY_INTERFACE_MODE_MAX)'), + (KernRe(r'DECLARE_BITMAP\s*\(' + struct_args_pattern + r',\s*' + struct_args_pattern + r'\)', + re.S), r'unsigned long \1[BITS_TO_LONGS(\2)]'), + (KernRe(r'DECLARE_HASHTABLE\s*\(' + struct_args_pattern + r',\s*' + struct_args_pattern + r'\)', + re.S), r'unsigned long \1[1 << ((\2) - 1)]'), + (KernRe(r'DECLARE_KFIFO\s*\(' + struct_args_pattern + r',\s*' + struct_args_pattern + + r',\s*' + struct_args_pattern + r'\)', re.S), r'\2 *\1'), + (KernRe(r'DECLARE_KFIFO_PTR\s*\(' + struct_args_pattern + r',\s*' + + struct_args_pattern + r'\)', re.S), r'\2 *\1'), + (KernRe(r'(?:__)?DECLARE_FLEX_ARRAY\s*\(' + struct_args_pattern + r',\s*' + + struct_args_pattern + r'\)', re.S), r'\1 \2[]'), + (KernRe(r'DEFINE_DMA_UNMAP_ADDR\s*\(' + struct_args_pattern + r'\)', re.S), r'dma_addr_t \1'), + (KernRe(r'DEFINE_DMA_UNMAP_LEN\s*\(' + struct_args_pattern + r'\)', re.S), r'__u32 \1'), + (KernRe(r'VIRTIO_DECLARE_FEATURES\(([\w_]+)\)'), r'union { u64 \1; u64 \1_array[VIRTIO_FEATURES_U64S]; }'), + ] + + #: Transforms for function prototypes. + function_xforms = [ + (KernRe(r"^static +"), ""), + (KernRe(r"^extern +"), ""), + (KernRe(r"^asmlinkage +"), ""), + (KernRe(r"^inline +"), ""), + (KernRe(r"^__inline__ +"), ""), + (KernRe(r"^__inline +"), ""), + (KernRe(r"^__always_inline +"), ""), + (KernRe(r"^noinline +"), ""), + (KernRe(r"^__FORTIFY_INLINE +"), ""), + (KernRe(r"__init +"), ""), + (KernRe(r"__init_or_module +"), ""), + (KernRe(r"__exit +"), ""), + (KernRe(r"__deprecated +"), ""), + (KernRe(r"__flatten +"), ""), + (KernRe(r"__meminit +"), ""), + (KernRe(r"__must_check +"), ""), + (KernRe(r"__weak +"), ""), + (KernRe(r"__sched +"), ""), + (KernRe(r"_noprof"), ""), + (KernRe(r"__always_unused *"), ""), + (KernRe(r"__printf\s*\(\s*\d*\s*,\s*\d*\s*\) +"), ""), + (KernRe(r"__(?:re)?alloc_size\s*\(\s*\d+\s*(?:,\s*\d+\s*)?\) +"), ""), + (KernRe(r"__diagnose_as\s*\(\s*\S+\s*(?:,\s*\d+\s*)*\) +"), ""), + (KernRe(r"DECL_BUCKET_PARAMS\s*\(\s*(\S+)\s*,\s*(\S+)\s*\)"), r"\1, \2"), + (KernRe(r"__attribute_const__ +"), ""), + (KernRe(r"__attribute__\s*\(\((?:[\w\s]+(?:\([^)]*\))?\s*,?)+\)\)\s+"), ""), + ] + + #: Transforms for variable prototypes. + var_xforms = [ + (KernRe(r"__read_mostly"), ""), + (KernRe(r"__ro_after_init"), ""), + (KernRe(r"LIST_HEAD\(([\w_]+)\)"), r"struct list_head \1"), + (KernRe(r"(?://.*)$"), ""), + (KernRe(r"(?:/\*.*\*/)"), ""), + (KernRe(r";$"), ""), + ] + + #: Transforms main dictionary used at apply_transforms(). + xforms = { + "struct": struct_xforms, + "func": function_xforms, + "var": var_xforms, + } + + def apply(self, xforms_type, text): + """ + Apply a set of transforms to a block of text. + """ + if xforms_type not in self.xforms: + return text + + for search, subst in self.xforms[xforms_type]: + text = search.sub(subst, text) + return text -- cgit v1.2.3 From 4ff59bdd93f0e80b5014977502d082c778f96304 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Mon, 2 Mar 2026 17:40:56 +0100 Subject: docs: xforms_lists: ignore context analysis and lock attributes Drop context analysis and lock (tracking) attributes to avoid kernel-doc warnings. There are now lots of warnings like these: Documentation/core-api/kref:328: ../include/linux/kref.h:72: WARNING: Invalid C declaration: Expected end of definition. [error at 96] int kref_put_mutex (struct kref *kref, void (*release)(struct kref *kref), struct mutex *mutex) __cond_acquires(true# mutex) ------------------------------------------------------------------------------------------------^ Documentation/core-api/kref:328: ../include/linux/kref.h:94: WARNING: Invalid C declaration: Expected end of definition. [error at 92] int kref_put_lock (struct kref *kref, void (*release)(struct kref *kref), spinlock_t *lock) __cond_acquires(true# lock) --------------------------------------------------------------------------------------------^ The regex is suggested by Mauro; mine was too greedy. Thanks. Updated context analysis and lock macros list provided by PeterZ. Thanks. [mchehab: modified to be applied after xforms_lists split] Reported-by: Stephen Rothwell Closes: https://lore.kernel.org/all/20260107161548.45530e1c@canb.auug.org.au/ Signed-off-by: Randy Dunlap Reviewed-by: Mauro Carvalho Chehab Reviewed-by: Aleksandr Loktionov Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Message-ID: <3c7fdfc364a8920f92530b47bdbf4bb29a40371f.1772469446.git.mchehab+huawei@kernel.org> --- tools/lib/python/kdoc/kdoc_parser.py | 10 ++++++++++ tools/lib/python/kdoc/xforms_lists.py | 5 +++++ 2 files changed, 15 insertions(+) (limited to 'tools/lib/python/kdoc') diff --git a/tools/lib/python/kdoc/kdoc_parser.py b/tools/lib/python/kdoc/kdoc_parser.py index d7daf658e9d2..503a18212747 100644 --- a/tools/lib/python/kdoc/kdoc_parser.py +++ b/tools/lib/python/kdoc/kdoc_parser.py @@ -75,6 +75,16 @@ doc_begin_func = KernRe(str(doc_com) + # initial " * ' # is allowed. # struct_nested_prefixes = [ + (re.compile(r"__cond_acquires\s*\("), ""), + (re.compile(r"__cond_releases\s*\("), ""), + (re.compile(r"__acquires\s*\("), ""), + (re.compile(r"__releases\s*\("), ""), + (re.compile(r"__must_hold\s*\("), ""), + (re.compile(r"__must_not_hold\s*\("), ""), + (re.compile(r"__must_hold_shared\s*\("), ""), + (re.compile(r"__cond_acquires_shared\s*\("), ""), + (re.compile(r"__acquires_shared\s*\("), ""), + (re.compile(r"__releases_shared\s*\("), ""), (re.compile(r'\bSTRUCT_GROUP\('), r'\1'), ] diff --git a/tools/lib/python/kdoc/xforms_lists.py b/tools/lib/python/kdoc/xforms_lists.py index e6e0302e5dd0..1bda7c4634c3 100644 --- a/tools/lib/python/kdoc/xforms_lists.py +++ b/tools/lib/python/kdoc/xforms_lists.py @@ -22,6 +22,8 @@ class CTransforms: (KernRe(r'\s*__aligned\s*\([^;]*\)', re.S), ' '), (KernRe(r'\s*__counted_by\s*\([^;]*\)', re.S), ' '), (KernRe(r'\s*__counted_by_(le|be)\s*\([^;]*\)', re.S), ' '), + (KernRe(r'\s*__guarded_by\s*\([^\)]*\)', re.S), ' '), + (KernRe(r'\s*__pt_guarded_by\s*\([^\)]*\)', re.S), ' '), (KernRe(r'\s*__packed\s*', re.S), ' '), (KernRe(r'\s*CRYPTO_MINALIGN_ATTR', re.S), ' '), (KernRe(r'\s*__private', re.S), ' '), @@ -120,6 +122,7 @@ class CTransforms: (KernRe(r"__(?:re)?alloc_size\s*\(\s*\d+\s*(?:,\s*\d+\s*)?\) +"), ""), (KernRe(r"__diagnose_as\s*\(\s*\S+\s*(?:,\s*\d+\s*)*\) +"), ""), (KernRe(r"DECL_BUCKET_PARAMS\s*\(\s*(\S+)\s*,\s*(\S+)\s*\)"), r"\1, \2"), + (KernRe(r"__no_context_analysis\s*"), ""), (KernRe(r"__attribute_const__ +"), ""), (KernRe(r"__attribute__\s*\(\((?:[\w\s]+(?:\([^)]*\))?\s*,?)+\)\)\s+"), ""), ] @@ -128,6 +131,8 @@ class CTransforms: var_xforms = [ (KernRe(r"__read_mostly"), ""), (KernRe(r"__ro_after_init"), ""), + (KernRe(r'\s*__guarded_by\s*\([^\)]*\)', re.S), ""), + (KernRe(r'\s*__pt_guarded_by\s*\([^\)]*\)', re.S), ""), (KernRe(r"LIST_HEAD\(([\w_]+)\)"), r"struct list_head \1"), (KernRe(r"(?://.*)$"), ""), (KernRe(r"(?:/\*.*\*/)"), ""), -- cgit v1.2.3 From 134468b0e2043efec4bd25dc6bcef238358a8111 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Mon, 2 Mar 2026 17:40:57 +0100 Subject: docs: kdoc_re: handle strings and escape chars on NextMatch The logic inside NestedMatch currently doesn't consider that function arguments may have chars and strings, which may eventually contain delimiters. Add logic to handle strings and escape characters on them. Signed-off-by: Mauro Carvalho Chehab Reviewed-by: Aleksandr Loktionov Signed-off-by: Jonathan Corbet Message-ID: --- tools/lib/python/kdoc/kdoc_re.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'tools/lib/python/kdoc') diff --git a/tools/lib/python/kdoc/kdoc_re.py b/tools/lib/python/kdoc/kdoc_re.py index 664c04c8cc9f..0a7f12616f9f 100644 --- a/tools/lib/python/kdoc/kdoc_re.py +++ b/tools/lib/python/kdoc/kdoc_re.py @@ -216,6 +216,8 @@ class NestedMatch: for match_re in regex.finditer(line): start = match_re.start() offset = match_re.end() + string_char = None + escape = False d = line[offset - 1] if d not in self.DELIMITER_PAIRS: @@ -229,6 +231,22 @@ class NestedMatch: d = line[pos] + if escape: + escape = False + continue + + if string_char: + if d == '\\': + escape = True + elif d == string_char: + string_char = None + + continue + + if d in ('"', "'"): + string_char = d + continue + if d in self.DELIMITER_PAIRS: end = self.DELIMITER_PAIRS[d] -- cgit v1.2.3 From 962bdc440df58008e0319d6cbe08c4ca1193c112 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Mon, 2 Mar 2026 17:40:58 +0100 Subject: docs: kdoc_re: don't recompile NestedMatch regex every time Store delimiters and its regex-compiled version as const vars. Signed-off-by: Mauro Carvalho Chehab Reviewed-by: Aleksandr Loktionov Signed-off-by: Jonathan Corbet Message-ID: <0cf2b72d4785aa8b727188b56688ff442d1c65ce.1772469446.git.mchehab+huawei@kernel.org> --- tools/lib/python/kdoc/kdoc_re.py | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) (limited to 'tools/lib/python/kdoc') diff --git a/tools/lib/python/kdoc/kdoc_re.py b/tools/lib/python/kdoc/kdoc_re.py index 0a7f12616f9f..00afa5bccd6d 100644 --- a/tools/lib/python/kdoc/kdoc_re.py +++ b/tools/lib/python/kdoc/kdoc_re.py @@ -99,6 +99,13 @@ class KernRe: self.last_match = self.regex.search(string) return self.last_match + def finditer(self, string): + """ + Alias to re.finditer. + """ + + return self.regex.finditer(string) + def findall(self, string): """ Alias to re.findall. @@ -134,6 +141,16 @@ class KernRe: return self.last_match.groups() +#: Nested delimited pairs (brackets and parenthesis) +DELIMITER_PAIRS = { + '{': '}', + '(': ')', + '[': ']', +} + +#: compiled delimiters +RE_DELIM = KernRe(r'[\{\}\[\]\(\)]') + class NestedMatch: """ @@ -183,14 +200,6 @@ class NestedMatch: # # FOO(arg1, arg2, arg3) - DELIMITER_PAIRS = { - '{': '}', - '(': ')', - '[': ']', - } - - RE_DELIM = re.compile(r'[\{\}\[\]\(\)]') - def _search(self, regex, line): """ Finds paired blocks for a regex that ends with a delimiter. @@ -220,13 +229,13 @@ class NestedMatch: escape = False d = line[offset - 1] - if d not in self.DELIMITER_PAIRS: + if d not in DELIMITER_PAIRS: continue - end = self.DELIMITER_PAIRS[d] + end = DELIMITER_PAIRS[d] stack.append(end) - for match in self.RE_DELIM.finditer(line[offset:]): + for match in RE_DELIM.finditer(line[offset:]): pos = match.start() + offset d = line[pos] @@ -247,8 +256,8 @@ class NestedMatch: string_char = d continue - if d in self.DELIMITER_PAIRS: - end = self.DELIMITER_PAIRS[d] + if d in DELIMITER_PAIRS: + end = DELIMITER_PAIRS[d] stack.append(end) continue -- cgit v1.2.3 From 34503b5fd10d8c7f1b1f4fecb6aae826fcf79424 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Mon, 2 Mar 2026 17:40:59 +0100 Subject: docs: kdoc_re: Change NestedMath args replacement to \0 Future patches will allow parsing each argument instead of the hole set. Prepare for it by changing the replace all args from \1 to \0. No functional changes. Signed-off-by: Mauro Carvalho Chehab Reviewed-by: Aleksandr Loktionov Signed-off-by: Jonathan Corbet Message-ID: <46e383118be9d9e432e3814fe819ebb12261d7b4.1772469446.git.mchehab+huawei@kernel.org> --- tools/lib/python/kdoc/kdoc_parser.py | 2 +- tools/lib/python/kdoc/kdoc_re.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'tools/lib/python/kdoc') diff --git a/tools/lib/python/kdoc/kdoc_parser.py b/tools/lib/python/kdoc/kdoc_parser.py index 503a18212747..0f90c16cb51a 100644 --- a/tools/lib/python/kdoc/kdoc_parser.py +++ b/tools/lib/python/kdoc/kdoc_parser.py @@ -85,7 +85,7 @@ struct_nested_prefixes = [ (re.compile(r"__cond_acquires_shared\s*\("), ""), (re.compile(r"__acquires_shared\s*\("), ""), (re.compile(r"__releases_shared\s*\("), ""), - (re.compile(r'\bSTRUCT_GROUP\('), r'\1'), + (re.compile(r'\bSTRUCT_GROUP\('), r'\0'), ] # diff --git a/tools/lib/python/kdoc/kdoc_re.py b/tools/lib/python/kdoc/kdoc_re.py index 00afa5bccd6d..ea4f6f3d9e42 100644 --- a/tools/lib/python/kdoc/kdoc_re.py +++ b/tools/lib/python/kdoc/kdoc_re.py @@ -188,7 +188,7 @@ class NestedMatch: # except that the content inside the match group is delimiter-aligned. # # The content inside parentheses is converted into a single replace - # group (e.g. r`\1'). + # group (e.g. r`\0'). # # It would be nice to change such definition to support multiple # match groups, allowing a regex equivalent to: @@ -291,7 +291,7 @@ class NestedMatch: if the sub argument contains:: - r'\1' + r'\0' it will work just like re: it places there the matched paired data with the delimiter stripped. @@ -310,9 +310,9 @@ class NestedMatch: # Value, ignoring start/end delimiters value = line[end:pos - 1] - # replaces \1 at the sub string, if \1 is used there + # replaces \0 at the sub string, if \0 is used there new_sub = sub - new_sub = new_sub.replace(r'\1', value) + new_sub = new_sub.replace(r'\0', value) out += new_sub -- cgit v1.2.3 From fc44c0a0b2a72f2e9331063a311a548634ae18af Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Mon, 2 Mar 2026 17:41:00 +0100 Subject: docs: kdoc_re: make NestedMatch use KernRe Instead of using re_compile, let's create the class with the regex and use KernRe to keep it cached. Signed-off-by: Mauro Carvalho Chehab Reviewed-by: Aleksandr Loktionov Signed-off-by: Jonathan Corbet Message-ID: --- tools/lib/python/kdoc/kdoc_parser.py | 25 ++++++++++++------------- tools/lib/python/kdoc/kdoc_re.py | 24 +++++++++++++++++------- 2 files changed, 29 insertions(+), 20 deletions(-) (limited to 'tools/lib/python/kdoc') diff --git a/tools/lib/python/kdoc/kdoc_parser.py b/tools/lib/python/kdoc/kdoc_parser.py index 0f90c16cb51a..cd9857906a2b 100644 --- a/tools/lib/python/kdoc/kdoc_parser.py +++ b/tools/lib/python/kdoc/kdoc_parser.py @@ -75,17 +75,17 @@ doc_begin_func = KernRe(str(doc_com) + # initial " * ' # is allowed. # struct_nested_prefixes = [ - (re.compile(r"__cond_acquires\s*\("), ""), - (re.compile(r"__cond_releases\s*\("), ""), - (re.compile(r"__acquires\s*\("), ""), - (re.compile(r"__releases\s*\("), ""), - (re.compile(r"__must_hold\s*\("), ""), - (re.compile(r"__must_not_hold\s*\("), ""), - (re.compile(r"__must_hold_shared\s*\("), ""), - (re.compile(r"__cond_acquires_shared\s*\("), ""), - (re.compile(r"__acquires_shared\s*\("), ""), - (re.compile(r"__releases_shared\s*\("), ""), - (re.compile(r'\bSTRUCT_GROUP\('), r'\0'), + (NestedMatch(r"__cond_acquires\s*\("), ""), + (NestedMatch(r"__cond_releases\s*\("), ""), + (NestedMatch(r"__acquires\s*\("), ""), + (NestedMatch(r"__releases\s*\("), ""), + (NestedMatch(r"__must_hold\s*\("), ""), + (NestedMatch(r"__must_not_hold\s*\("), ""), + (NestedMatch(r"__must_hold_shared\s*\("), ""), + (NestedMatch(r"__cond_acquires_shared\s*\("), ""), + (NestedMatch(r"__acquires_shared\s*\("), ""), + (NestedMatch(r"__releases_shared\s*\("), ""), + (NestedMatch(r'\bSTRUCT_GROUP\('), r'\0'), ] # @@ -761,9 +761,8 @@ class KernelDoc: members = trim_private_members(members) members = self.xforms.apply("struct", members) - nested = NestedMatch() for search, sub in struct_nested_prefixes: - members = nested.sub(search, sub, members) + members = search.sub(search, sub, members) # # Deal with embedded struct and union members, and drop enums entirely. # diff --git a/tools/lib/python/kdoc/kdoc_re.py b/tools/lib/python/kdoc/kdoc_re.py index ea4f6f3d9e42..085b89a4547c 100644 --- a/tools/lib/python/kdoc/kdoc_re.py +++ b/tools/lib/python/kdoc/kdoc_re.py @@ -200,7 +200,10 @@ class NestedMatch: # # FOO(arg1, arg2, arg3) - def _search(self, regex, line): + def __init__(self, regex): + self.regex = KernRe(regex) + + def _search(self, line): """ Finds paired blocks for a regex that ends with a delimiter. @@ -222,7 +225,7 @@ class NestedMatch: stack = [] - for match_re in regex.finditer(line): + for match_re in self.regex.finditer(line): start = match_re.start() offset = match_re.end() string_char = None @@ -270,7 +273,7 @@ class NestedMatch: yield start, offset, pos + 1 break - def search(self, regex, line): + def search(self, line): """ This is similar to re.search: @@ -278,12 +281,12 @@ class NestedMatch: returning occurrences only if all delimiters are paired. """ - for t in self._search(regex, line): + for t in self._search(line): yield line[t[0]:t[2]] - def sub(self, regex, sub, line, count=0): - r""" + def sub(self, sub, line, count=0): + """ This is similar to re.sub: It matches a regex that it is followed by a delimiter, @@ -304,7 +307,7 @@ class NestedMatch: cur_pos = 0 n = 0 - for start, end, pos in self._search(regex, line): + for start, end, pos in self._search(line): out += line[cur_pos:start] # Value, ignoring start/end delimiters @@ -331,3 +334,10 @@ class NestedMatch: out += line[cur_pos:l] return out + + def __repr__(self): + """ + Returns a displayable version of the class init. + """ + + return f'NestedMatch("{self.regex.regex.pattern}")' -- cgit v1.2.3 From 85c2a51357f720fabfb6fa8d2551d87a94e797cb Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Mon, 2 Mar 2026 17:41:01 +0100 Subject: docs: kdoc_parser: move nested match transforms to xforms_lists.py As NestedMatch now has a sub method and a declaration close to what KernRe does, we can move the rules to xforms_lists and simplify kdoc_parser a little bit. No functional changes. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Message-ID: <762ce2a58ff024c1b0b6f6a6e05020d1415b8308.1772469446.git.mchehab+huawei@kernel.org> --- tools/lib/python/kdoc/kdoc_parser.py | 21 --------------------- tools/lib/python/kdoc/xforms_lists.py | 14 +++++++++++++- 2 files changed, 13 insertions(+), 22 deletions(-) (limited to 'tools/lib/python/kdoc') diff --git a/tools/lib/python/kdoc/kdoc_parser.py b/tools/lib/python/kdoc/kdoc_parser.py index cd9857906a2b..edf70ba139a5 100644 --- a/tools/lib/python/kdoc/kdoc_parser.py +++ b/tools/lib/python/kdoc/kdoc_parser.py @@ -69,25 +69,6 @@ doc_begin_func = KernRe(str(doc_com) + # initial " * ' r'(?:[-:].*)?$', # description (not captured) cache = False) -# -# Regexes here are guaranteed to have the end delimiter matching -# the start delimiter. Yet, right now, only one replace group -# is allowed. -# -struct_nested_prefixes = [ - (NestedMatch(r"__cond_acquires\s*\("), ""), - (NestedMatch(r"__cond_releases\s*\("), ""), - (NestedMatch(r"__acquires\s*\("), ""), - (NestedMatch(r"__releases\s*\("), ""), - (NestedMatch(r"__must_hold\s*\("), ""), - (NestedMatch(r"__must_not_hold\s*\("), ""), - (NestedMatch(r"__must_hold_shared\s*\("), ""), - (NestedMatch(r"__cond_acquires_shared\s*\("), ""), - (NestedMatch(r"__acquires_shared\s*\("), ""), - (NestedMatch(r"__releases_shared\s*\("), ""), - (NestedMatch(r'\bSTRUCT_GROUP\('), r'\0'), -] - # # Ancillary functions # @@ -761,8 +742,6 @@ class KernelDoc: members = trim_private_members(members) members = self.xforms.apply("struct", members) - for search, sub in struct_nested_prefixes: - members = search.sub(search, sub, members) # # Deal with embedded struct and union members, and drop enums entirely. # diff --git a/tools/lib/python/kdoc/xforms_lists.py b/tools/lib/python/kdoc/xforms_lists.py index 1bda7c4634c3..c07cbe1e6349 100644 --- a/tools/lib/python/kdoc/xforms_lists.py +++ b/tools/lib/python/kdoc/xforms_lists.py @@ -4,7 +4,7 @@ import re -from kdoc.kdoc_re import KernRe +from kdoc.kdoc_re import KernRe, NestedMatch struct_args_pattern = r'([^,)]+)' @@ -94,6 +94,18 @@ class CTransforms: (KernRe(r'DEFINE_DMA_UNMAP_ADDR\s*\(' + struct_args_pattern + r'\)', re.S), r'dma_addr_t \1'), (KernRe(r'DEFINE_DMA_UNMAP_LEN\s*\(' + struct_args_pattern + r'\)', re.S), r'__u32 \1'), (KernRe(r'VIRTIO_DECLARE_FEATURES\(([\w_]+)\)'), r'union { u64 \1; u64 \1_array[VIRTIO_FEATURES_U64S]; }'), + + (NestedMatch(r"__cond_acquires\s*\("), ""), + (NestedMatch(r"__cond_releases\s*\("), ""), + (NestedMatch(r"__acquires\s*\("), ""), + (NestedMatch(r"__releases\s*\("), ""), + (NestedMatch(r"__must_hold\s*\("), ""), + (NestedMatch(r"__must_not_hold\s*\("), ""), + (NestedMatch(r"__must_hold_shared\s*\("), ""), + (NestedMatch(r"__cond_acquires_shared\s*\("), ""), + (NestedMatch(r"__acquires_shared\s*\("), ""), + (NestedMatch(r"__releases_shared\s*\("), ""), + (NestedMatch(r'\bSTRUCT_GROUP\('), r'\0'), ] #: Transforms for function prototypes. -- cgit v1.2.3