From a9e4e50519e99e5be2d69fa4eebec6a4848ae201 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Fri, 8 May 2026 10:45:16 -0700 Subject: locking/rtmutex: Annotate API and implementation Enable context analysis for struct rt_mutex and annotate all functions that accept a struct rt_mutex pointer. In the __rt_mutex_lock_common() callers, instead of adding the __no_context_analysis annotation, emit a runtime warning if the __rt_mutex_lock_common() return value is not zero and add an __acquire() statement. Signed-off-by: Bart Van Assche Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260508174520.1416285-1-bvanassche@acm.org --- include/linux/rtmutex.h | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rtmutex.h b/include/linux/rtmutex.h index 78e7e588817c..9e1f012f89db 100644 --- a/include/linux/rtmutex.h +++ b/include/linux/rtmutex.h @@ -56,6 +56,8 @@ static inline struct task_struct *rt_mutex_owner(struct rt_mutex_base *lock) #endif extern void rt_mutex_base_init(struct rt_mutex_base *rtb); +context_lock_struct(rt_mutex); + /** * The rt_mutex structure * @@ -108,8 +110,10 @@ do { \ extern void __rt_mutex_init(struct rt_mutex *lock, const char *name, struct lock_class_key *key); #ifdef CONFIG_DEBUG_LOCK_ALLOC -extern void rt_mutex_lock_nested(struct rt_mutex *lock, unsigned int subclass); -extern void _rt_mutex_lock_nest_lock(struct rt_mutex *lock, struct lockdep_map *nest_lock); +extern void rt_mutex_lock_nested(struct rt_mutex *lock, unsigned int subclass) + __acquires(lock); +extern void _rt_mutex_lock_nest_lock(struct rt_mutex *lock, struct lockdep_map *nest_lock) + __acquires(lock); #define rt_mutex_lock(lock) rt_mutex_lock_nested(lock, 0) #define rt_mutex_lock_nest_lock(lock, nest_lock) \ do { \ @@ -118,15 +122,19 @@ extern void _rt_mutex_lock_nest_lock(struct rt_mutex *lock, struct lockdep_map * } while (0) #else -extern void rt_mutex_lock(struct rt_mutex *lock); +extern void rt_mutex_lock(struct rt_mutex *lock) __acquires(lock); #define rt_mutex_lock_nested(lock, subclass) rt_mutex_lock(lock) #define rt_mutex_lock_nest_lock(lock, nest_lock) rt_mutex_lock(lock) #endif -extern int rt_mutex_lock_interruptible(struct rt_mutex *lock); -extern int rt_mutex_lock_killable(struct rt_mutex *lock); -extern int rt_mutex_trylock(struct rt_mutex *lock); +extern int rt_mutex_lock_interruptible(struct rt_mutex *lock) + __cond_acquires(0, lock); +extern int rt_mutex_lock_killable(struct rt_mutex *lock) + __cond_acquires(0, lock); +extern int rt_mutex_trylock(struct rt_mutex *lock) + __cond_acquires(true, lock); -extern void rt_mutex_unlock(struct rt_mutex *lock); +extern void rt_mutex_unlock(struct rt_mutex *lock) + __releases(lock); #endif -- cgit v1.2.3 From f45c5c4adb27540d43302155e2fbaeca6c3c6d03 Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 15 May 2026 14:43:31 +0200 Subject: compiler-context-analysis: Bump required Clang version to 23 Clang 23 introduces several major improvements: 1. Support for multiple arguments in the `guarded_by` and `pt_guarded_by` attributes [1]. This allows defining variables protected by multiple context locks, where read access requires holding at least one lock (shared or exclusive), and write access requires holding all of them exclusively. 2. Function pointer support [2]. We can now add attributes to function pointers just like we do on normal functions. 3. A fix to use arrays of locks [3]. Each index is now correctly treated as a separate lock instance. 4. A fix for implicit member access in attributes [4]. This allows to use __guarded_by(&foo->lock) correctly. Overall that makes it worthwhile bumping the compiler version instead of trying to make both Clang 22 and later work while supporting these new features. Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Nathan Chancellor Reviewed-by: Bart Van Assche Link: https://github.com/llvm/llvm-project/pull/186838 [1] Link: https://github.com/llvm/llvm-project/pull/191187 [2] Link: https://github.com/llvm/llvm-project/pull/148551 [3] Link: https://github.com/llvm/llvm-project/pull/194457 [4] Link: https://patch.msgid.link/20260515124426.2227783-1-elver@google.com --- Documentation/dev-tools/context-analysis.rst | 2 +- include/linux/compiler-context-analysis.h | 30 ++++++++++++++++++++-------- lib/Kconfig.debug | 4 ++-- lib/test_context-analysis.c | 24 ++++++++++++++++++++++ 4 files changed, 49 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/Documentation/dev-tools/context-analysis.rst b/Documentation/dev-tools/context-analysis.rst index 54d9ee28de98..8e71e1e75b5b 100644 --- a/Documentation/dev-tools/context-analysis.rst +++ b/Documentation/dev-tools/context-analysis.rst @@ -17,7 +17,7 @@ features. To enable for Clang, configure the kernel with:: CONFIG_WARN_CONTEXT_ANALYSIS=y -The feature requires Clang 22 or later. +The feature requires Clang 23 or later. The analysis is *opt-in by default*, and requires declaring which modules and subsystems should be analyzed in the respective `Makefile`:: diff --git a/include/linux/compiler-context-analysis.h b/include/linux/compiler-context-analysis.h index a9317571e6af..8302ebc2ea8c 100644 --- a/include/linux/compiler-context-analysis.h +++ b/include/linux/compiler-context-analysis.h @@ -39,12 +39,14 @@ # define __assumes_shared_ctx_lock(...) __attribute__((assert_shared_capability(__VA_ARGS__))) /** - * __guarded_by - struct member and globals attribute, declares variable - * only accessible within active context + * __guarded_by() - struct member and globals attribute, declares variable + * only accessible within active context + * @...: context lock instance pointer(s) * * Declares that the struct member or global variable is only accessible within - * the context entered by the given context lock. Read operations on the data - * require shared access, while write operations require exclusive access. + * the context entered by the given context lock(s). Read operations on the data + * require shared access to at least one of the context locks, while write + * operations require exclusive access to all listed context locks. * * .. code-block:: c * @@ -52,17 +54,24 @@ * spinlock_t lock; * long counter __guarded_by(&lock); * }; + * + * struct some_state { + * spinlock_t lock1, lock2; + * long counter __guarded_by(&lock1, &lock2); + * }; */ # define __guarded_by(...) __attribute__((guarded_by(__VA_ARGS__))) /** - * __pt_guarded_by - struct member and globals attribute, declares pointed-to - * data only accessible within active context + * __pt_guarded_by() - struct member and globals attribute, declares pointed-to + * data only accessible within active context + * @...: context lock instance pointer(s) * * Declares that the data pointed to by the struct member pointer or global * pointer is only accessible within the context entered by the given context - * lock. Read operations on the data require shared access, while write - * operations require exclusive access. + * lock(s). Read operations on the data require shared access to at least one + * of the context locks, while write operations require exclusive access to all + * listed context locks. * * .. code-block:: c * @@ -70,6 +79,11 @@ * spinlock_t lock; * long *counter __pt_guarded_by(&lock); * }; + * + * struct some_state { + * spinlock_t lock1, lock2; + * long *counter __pt_guarded_by(&lock1, &lock2); + * }; */ # define __pt_guarded_by(...) __attribute__((pt_guarded_by(__VA_ARGS__))) diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 8ff5adcfe1e0..b0f3028a213d 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -630,7 +630,7 @@ config DEBUG_FORCE_WEAK_PER_CPU config WARN_CONTEXT_ANALYSIS bool "Compiler context-analysis warnings" - depends on CC_IS_CLANG && CLANG_VERSION >= 220100 + depends on CC_IS_CLANG && CLANG_VERSION >= 230000 # Branch profiling re-defines "if", which messes with the compiler's # ability to analyze __cond_acquires(..), resulting in false positives. depends on !TRACE_BRANCH_PROFILING @@ -641,7 +641,7 @@ config WARN_CONTEXT_ANALYSIS and releasing user-definable "context locks". Clang's name of the feature is "Thread Safety Analysis". Requires - Clang 22.1.0 or later. + Clang 23 or later. Produces warnings by default. Select CONFIG_WERROR if you wish to turn these warnings into errors. diff --git a/lib/test_context-analysis.c b/lib/test_context-analysis.c index 06b4a6a028e0..316f4dfcda65 100644 --- a/lib/test_context-analysis.c +++ b/lib/test_context-analysis.c @@ -159,6 +159,10 @@ TEST_SPINLOCK_COMMON(read_lock, struct test_mutex_data { struct mutex mtx; int counter __guarded_by(&mtx); + + struct mutex mtx2; + int anyread __guarded_by(&mtx, &mtx2); + int *anyptr __pt_guarded_by(&mtx, &mtx2); }; static void __used test_mutex_init(struct test_mutex_data *d) @@ -219,6 +223,26 @@ static void __used test_mutex_cond_guard(struct test_mutex_data *d) } } +static void __used test_mutex_multiguard(struct test_mutex_data *d) +{ + mutex_lock(&d->mtx); + (void)d->anyread; + (void)*d->anyptr; + mutex_unlock(&d->mtx); + + mutex_lock(&d->mtx2); + (void)d->anyread; + (void)*d->anyptr; + mutex_unlock(&d->mtx2); + + mutex_lock(&d->mtx); + mutex_lock(&d->mtx2); + d->anyread++; + (*d->anyptr)++; + mutex_unlock(&d->mtx2); + mutex_unlock(&d->mtx); +} + struct test_seqlock_data { seqlock_t sl; int counter __guarded_by(&sl); -- cgit v1.2.3 From 88331c4ec23a28c1006ec532fa64763d4c695e90 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Tue, 19 May 2026 13:03:15 +0200 Subject: seqlock: Allow UBSAN_ALIGNMENT to fail optimizing With gcc-15 and gcc-16 with UBSAN_ALIGNMENT enabled the compiler fails to inline and optimize __scoped_seqlock_bug() away on s390: s390x-16.1.0-ld: kernel/sched/build_policy.o: in function `__scoped_seqlock_next': /.../seqlock.h:1286:(.text+0x22030): undefined reference to `__scoped_seqlock_bug' Fix this by adding UBSAN_ALIGNMENT to the list of config options where a not inlined empty __scoped_seqlock_bug() is allowed. Closes: https://lore.kernel.org/r/20260515092057.810542-1-arnd@kernel.org/ Reported-by: Arnd Bergmann Signed-off-by: Heiko Carstens Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260519110315.1385307-1-hca@linux.ibm.com --- include/linux/seqlock.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/seqlock.h b/include/linux/seqlock.h index 5a40252b8334..f865491c4f2c 100644 --- a/include/linux/seqlock.h +++ b/include/linux/seqlock.h @@ -1259,14 +1259,15 @@ static __always_inline void __scoped_seqlock_cleanup(struct ss_tmp *sst) extern void __scoped_seqlock_invalid_target(void); -#if (defined(CONFIG_CC_IS_GCC) && CONFIG_GCC_VERSION < 90000) || defined(CONFIG_KASAN) +#if (defined(CONFIG_CC_IS_GCC) && CONFIG_GCC_VERSION < 90000) || \ + defined(CONFIG_KASAN) || defined(CONFIG_UBSAN_ALIGNMENT) /* * For some reason some GCC-8 architectures (nios2, alpha) have trouble * determining that the ss_done state is impossible in __scoped_seqlock_next() * below. * - * Similarly KASAN is known to confuse compilers enough to break this. But we - * don't care about code quality for KASAN builds anyway. + * Similarly KASAN and UBSAN_ALIGNMENT are known to confuse compilers enough + * to break this. But we don't care about code quality for such builds anyway. */ static inline void __scoped_seqlock_bug(void) { } #else -- cgit v1.2.3 From 22302af28d3f7c3ca38536978c80db51d2a9e283 Mon Sep 17 00:00:00 2001 From: Dmitry Ilvokhin Date: Tue, 2 Jun 2026 07:12:52 +0000 Subject: cleanup: Annotate guard constructors with nonnull Add __nonnull_args() to unconditional guard constructors so the compiler warns when NULL is statically known to be passed: - DEFINE_GUARD(): re-declare the constructor with __nonnull_args(). - __DEFINE_LOCK_GUARD_1(): annotate the constructor directly. DEFINE_LOCK_GUARD_0() needs no annotation: its constructor takes no pointer arguments (.lock is hardcoded to (void *)1). Define the __nonnull_args() macro in compiler_attributes.h, following the existing convention for attribute wrappers. Deliberately not named '__nonnull', to avoid clashing with glibc's __nonnull() when kernel and userspace headers are combined (User Mode Linux for example). Signed-off-by: Dmitry Ilvokhin Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/85fee12eec20abfcf711443518e8f0caec982a86.1780064327.git.d@ilvokhin.com --- include/linux/cleanup.h | 4 +++- include/linux/compiler_attributes.h | 9 +++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/cleanup.h b/include/linux/cleanup.h index ea95ca4bc11c..4e60d519713c 100644 --- a/include/linux/cleanup.h +++ b/include/linux/cleanup.h @@ -397,6 +397,7 @@ static __maybe_unused const bool class_##_name##_is_conditional = _is_cond __DEFINE_GUARD_LOCK_PTR(_name, _T) #define DEFINE_GUARD(_name, _type, _lock, _unlock) \ + static __always_inline __nonnull_args() _type class_##_name##_constructor(_type _T); \ DEFINE_CLASS(_name, _type, if (_T) { _unlock; }, ({ _lock; _T; }), _type _T); \ DEFINE_CLASS_IS_GUARD(_name) @@ -497,7 +498,8 @@ static __always_inline void class_##_name##_destructor(class_##_name##_t *_T) \ __DEFINE_GUARD_LOCK_PTR(_name, &_T->lock) #define __DEFINE_LOCK_GUARD_1(_name, _type, ...) \ -static __always_inline class_##_name##_t class_##_name##_constructor(_type *l) \ +static __always_inline __nonnull_args() \ +class_##_name##_t class_##_name##_constructor(_type *l) \ __no_context_analysis \ { \ class_##_name##_t _t = { .lock = l }, *_T = &_t; \ diff --git a/include/linux/compiler_attributes.h b/include/linux/compiler_attributes.h index c16d4199bf92..cffe09387ea6 100644 --- a/include/linux/compiler_attributes.h +++ b/include/linux/compiler_attributes.h @@ -230,6 +230,15 @@ */ #define noinline __attribute__((__noinline__)) +/* + * Note: deliberately not named '__nonnull', to avoid clashing with glibc's + * __nonnull() when kernel and userspace headers are combined. + * + * gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Attributes.html#index-nonnull + * clang: https://clang.llvm.org/docs/AttributeReference.html#nonnull + */ +#define __nonnull_args(x...) __attribute__((__nonnull__(x))) + /* * Optional: only supported since gcc >= 8 * Optional: not supported by clang -- cgit v1.2.3 From a13ab9dd6eb2a89f14b466c3884730ea7969253f Mon Sep 17 00:00:00 2001 From: Dmitry Ilvokhin Date: Tue, 2 Jun 2026 07:12:53 +0000 Subject: cleanup: Remove NULL check from unconditional guards The unconditional guard destructors check whether the lock pointer is NULL before unlocking. This check is dead code because unconditional guards guarantee a non-NULL lock pointer at destructor time. DEFINE_GUARD() runs the lock operation unconditionally in the constructor. If the pointer were NULL, the lock operation (e.g. mutex_lock(NULL)) would crash before the constructor returns. The destructor never runs with a NULL pointer. All DEFINE_GUARD() users dereference the pointer in their lock. Verified by auditing every instance found by: git grep -n -A 1 'DEFINE_GUARD('. The only exception is xe_pm_runtime_release_only, whose constructor is a noop, but it has no callers. __DEFINE_UNLOCK_GUARD() has only a few usages outside of include/linux/cleanup.h: tty_port_tty (NULL-checks in its tty_kref_put() call), irqdesc_lock (fixed earlier) and two guards in kernel/sched/sched.h (dereference the pointer unconditionally in their lock constructors). DEFINE_LOCK_GUARD_1() sets .lock from its argument and runs the lock operation in the constructor. Same reasoning applies. All DEFINE_LOCK_GUARD_1() users dereference the pointer in their lock. Also, verified by auditing every match of: git grep -n 'DEFINE_LOCK_GUARD_1('. DEFINE_LOCK_GUARD_0() hardcodes .lock = (void *)1 in the constructor, so it is never NULL by construction. Conditional (_try) variants: DEFINE_GUARD_COND() and DEFINE_LOCK_GUARD_1_COND() use EXTEND_CLASS_COND(), whose wrapper destructor returns early when the lock was not acquired, before reaching the base destructor since commit 2deccd5c862a ("cleanup: Optimize guards"): if (_cond) return; class_##_name##_destructor(_T); As compiled by GCC-11 with defconfig on top of the locking/core: Total: Before=23889980, After=23834334, chg -0.23% Signed-off-by: Dmitry Ilvokhin Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/0503a089389b2270c478a873e095cf0a4ff26d24.1780064327.git.d@ilvokhin.com --- include/linux/cleanup.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/cleanup.h b/include/linux/cleanup.h index 4e60d519713c..65416938e318 100644 --- a/include/linux/cleanup.h +++ b/include/linux/cleanup.h @@ -398,7 +398,7 @@ static __maybe_unused const bool class_##_name##_is_conditional = _is_cond #define DEFINE_GUARD(_name, _type, _lock, _unlock) \ static __always_inline __nonnull_args() _type class_##_name##_constructor(_type _T); \ - DEFINE_CLASS(_name, _type, if (_T) { _unlock; }, ({ _lock; _T; }), _type _T); \ + DEFINE_CLASS(_name, _type, _unlock, ({ _lock; _T; }), _type _T); \ DEFINE_CLASS_IS_GUARD(_name) #define DEFINE_GUARD_COND_4(_name, _ext, _lock, _cond) \ @@ -492,7 +492,7 @@ typedef struct { \ static __always_inline void class_##_name##_destructor(class_##_name##_t *_T) \ __no_context_analysis \ { \ - if (_T->lock) { _unlock; } \ + _unlock; \ } \ \ __DEFINE_GUARD_LOCK_PTR(_name, &_T->lock) -- cgit v1.2.3 From c06cd66387da92e6cdac44e16c7b5ef9219c53ac Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 2 Jun 2026 11:09:21 +0200 Subject: percpu: Sanitize __percpu_qual include hell Slapping __percpu_qual into the next available header is sloppy at best. It's required by __percpu which is defined in compiler_types.h and that is meant to be included without requiring a boatload of other headers so that a struct or function declaration can contain a __percpu qualifier w/o further prerequisites. This implicit dependency on linux/percpu.h makes that impossible and causes a major problem when trying to separate headers. Create asm/percpu_types.h and move it there. Include that from compiler_types.h and the whole recursion problem goes away. Fix up UM so it uses the generic header and includes it in the UM_HOST build, which pulls in compiler_types.h. The USER_CFLAGS fix was suggested by Richard. Signed-off-by: Thomas Gleixner Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260602090535.254874125@kernel.org --- arch/um/Makefile | 3 ++- arch/um/include/asm/Kbuild | 1 + arch/x86/include/asm/percpu.h | 5 ----- arch/x86/include/asm/percpu_types.h | 17 +++++++++++++++++ include/asm-generic/Kbuild | 1 + include/asm-generic/percpu_types.h | 19 +++++++++++++++++++ include/linux/compiler_types.h | 3 +++ include/linux/percpu.h | 9 +++++---- 8 files changed, 48 insertions(+), 10 deletions(-) create mode 100644 arch/x86/include/asm/percpu_types.h create mode 100644 include/asm-generic/percpu_types.h (limited to 'include/linux') diff --git a/arch/um/Makefile b/arch/um/Makefile index 721b652ffb65..937639edc295 100644 --- a/arch/um/Makefile +++ b/arch/um/Makefile @@ -71,7 +71,8 @@ USER_CFLAGS = $(patsubst $(KERNEL_DEFINES),,$(patsubst -I%,,$(KBUILD_CFLAGS))) \ -D_FILE_OFFSET_BITS=64 -idirafter $(srctree)/include \ -idirafter $(objtree)/include -D__KERNEL__ -D__UM_HOST__ \ -include $(srctree)/include/linux/compiler-version.h \ - -include $(srctree)/include/linux/kconfig.h + -include $(srctree)/include/linux/kconfig.h \ + -idirafter $(ARCH_DIR)/include/generated #This will adjust *FLAGS accordingly to the platform. include $(srctree)/$(ARCH_DIR)/Makefile-os-Linux diff --git a/arch/um/include/asm/Kbuild b/arch/um/include/asm/Kbuild index 1b9b82bbe322..e91ba12b7ffc 100644 --- a/arch/um/include/asm/Kbuild +++ b/arch/um/include/asm/Kbuild @@ -16,6 +16,7 @@ generic-y += module.h generic-y += module.lds.h generic-y += parport.h generic-y += percpu.h +generic-y += percpu_types.h generic-y += preempt.h generic-y += runtime-const.h generic-y += softirq_stack.h diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h index 409981468cba..cef9a4ca9841 100644 --- a/arch/x86/include/asm/percpu.h +++ b/arch/x86/include/asm/percpu.h @@ -40,12 +40,10 @@ #endif #define __percpu_prefix -#define __percpu_seg_override CONCATENATE(__seg_, __percpu_seg) #else /* !CONFIG_CC_HAS_NAMED_AS: */ #define __percpu_prefix __force_percpu_prefix -#define __percpu_seg_override #endif /* CONFIG_CC_HAS_NAMED_AS */ @@ -82,7 +80,6 @@ #define __force_percpu_prefix #define __percpu_prefix -#define __percpu_seg_override #define PER_CPU_VAR(var) (var)__percpu_rel @@ -92,8 +89,6 @@ # define __my_cpu_type(var) typeof(var) # define __my_cpu_ptr(ptr) (ptr) # define __my_cpu_var(var) (var) - -# define __percpu_qual __percpu_seg_override #else # define __my_cpu_type(var) typeof(var) __percpu_seg_override # define __my_cpu_ptr(ptr) (__my_cpu_type(*(ptr))*)(__force uintptr_t)(ptr) diff --git a/arch/x86/include/asm/percpu_types.h b/arch/x86/include/asm/percpu_types.h new file mode 100644 index 000000000000..0aa3e47a3643 --- /dev/null +++ b/arch/x86/include/asm/percpu_types.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_PERCPU_TYPES_H +#define _ASM_X86_PERCPU_TYPES_H + +#if defined(CONFIG_SMP) && defined(CONFIG_CC_HAS_NAMED_AS) +#define __percpu_seg_override CONCATENATE(__seg_, __percpu_seg) +#else /* !CONFIG_CC_HAS_NAMED_AS: */ +#define __percpu_seg_override +#endif + +#if defined(CONFIG_USE_X86_SEG_SUPPORT) && defined(USE_TYPEOF_UNQUAL) +#define __percpu_qual __percpu_seg_override +#endif + +#include + +#endif diff --git a/include/asm-generic/Kbuild b/include/asm-generic/Kbuild index 2c53a1e0b760..15df9dcb42a5 100644 --- a/include/asm-generic/Kbuild +++ b/include/asm-generic/Kbuild @@ -44,6 +44,7 @@ mandatory-y += module.lds.h mandatory-y += msi.h mandatory-y += pci.h mandatory-y += percpu.h +mandatory-y += percpu_types.h mandatory-y += pgalloc.h mandatory-y += preempt.h mandatory-y += rqspinlock.h diff --git a/include/asm-generic/percpu_types.h b/include/asm-generic/percpu_types.h new file mode 100644 index 000000000000..a095cea7fa20 --- /dev/null +++ b/include/asm-generic/percpu_types.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_GENERIC_PERCPU_TYPES_H_ +#define _ASM_GENERIC_PERCPU_TYPES_H_ + +#ifndef __ASSEMBLER__ +/* + * __percpu_qual is the qualifier for the percpu named address space. + * + * Most architectures use generic named address space for percpu variables but + * some architectures define percpu variables in different named address space. + * E.g. on x86, percpu variable may be declared as being relative to the %fs or + * %gs segments using __seg_fs or __seg_gs named address space qualifier. + */ +#ifndef __percpu_qual +# define __percpu_qual +#endif + +#endif /* __ASSEMBLER__ */ +#endif /* _ASM_GENERIC_PERCPU_TYPES_H_ */ diff --git a/include/linux/compiler_types.h b/include/linux/compiler_types.h index e8fd77593b68..7ad37adda1dd 100644 --- a/include/linux/compiler_types.h +++ b/include/linux/compiler_types.h @@ -634,6 +634,9 @@ struct ftrace_likely_data { #else #define __unqual_scalar_typeof(x) __typeof_unqual__(x) #endif + +#include + #endif /* !__ASSEMBLY__ */ /* diff --git a/include/linux/percpu.h b/include/linux/percpu.h index 85bf8dd9f087..2f5a889aa50d 100644 --- a/include/linux/percpu.h +++ b/include/linux/percpu.h @@ -3,13 +3,14 @@ #define __LINUX_PERCPU_H #include +#include +#include +#include #include -#include -#include #include -#include -#include +#include #include +#include #include -- cgit v1.2.3 From c1ffc9c6e4f8a13dd68e97920c9a24d095c6e41a Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 2 Jun 2026 11:09:25 +0200 Subject: futex: Move futex task related data into a struct MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Having all these members in task_struct along with the required #ifdeffery is annoying, does not allow efficient initializing of the data with memset() and makes extending it tedious. Move it into a data structure and fix up all usage sites. Signed-off-by: Thomas Gleixner Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Mathieu Desnoyers Reviewed-by: André Almeida Link: https://patch.msgid.link/20260602090535.308220888@kernel.org --- Documentation/locking/robust-futexes.rst | 8 ++--- include/linux/futex.h | 12 +++---- include/linux/futex_types.h | 36 +++++++++++++++++++ include/linux/sched.h | 16 +++------ kernel/exit.c | 4 +-- kernel/futex/core.c | 59 ++++++++++++++++---------------- kernel/futex/pi.c | 26 +++++++------- kernel/futex/syscalls.c | 23 +++++-------- 8 files changed, 101 insertions(+), 83 deletions(-) create mode 100644 include/linux/futex_types.h (limited to 'include/linux') diff --git a/Documentation/locking/robust-futexes.rst b/Documentation/locking/robust-futexes.rst index 6361fb01c9c1..1423f53ea2f4 100644 --- a/Documentation/locking/robust-futexes.rst +++ b/Documentation/locking/robust-futexes.rst @@ -94,7 +94,7 @@ time, the kernel checks this user-space list: are there any robust futex locks to be cleaned up? In the common case, at do_exit() time, there is no list registered, so -the cost of robust futexes is just a simple current->robust_list != NULL +the cost of robust futexes is just a current->futex.robust_list != NULL comparison. If the thread has registered a list, then normally the list is empty. If the thread/process crashed or terminated in some incorrect way then the list might be non-empty: in this case the kernel carefully @@ -178,9 +178,9 @@ one to query the registered list pointer:: size_t __user *len_ptr); List registration is very fast: the pointer is simply stored in -current->robust_list. [Note that in the future, if robust futexes become -widespread, we could extend sys_clone() to register a robust-list head -for new threads, without the need of another syscall.] +current->futex.robust_list. [Note that in the future, if robust futexes +become widespread, we could extend sys_clone() to register a robust-list +head for new threads, without the need of another syscall.] So there is virtually zero overhead for tasks not using robust futexes, and even for robust futex users, there is only one extra syscall per diff --git a/include/linux/futex.h b/include/linux/futex.h index 9e9750f04980..563e8dd67179 100644 --- a/include/linux/futex.h +++ b/include/linux/futex.h @@ -64,14 +64,10 @@ enum { static inline void futex_init_task(struct task_struct *tsk) { - tsk->robust_list = NULL; -#ifdef CONFIG_COMPAT - tsk->compat_robust_list = NULL; -#endif - INIT_LIST_HEAD(&tsk->pi_state_list); - tsk->pi_state_cache = NULL; - tsk->futex_state = FUTEX_STATE_OK; - mutex_init(&tsk->futex_exit_mutex); + memset(&tsk->futex, 0, sizeof(tsk->futex)); + INIT_LIST_HEAD(&tsk->futex.pi_state_list); + tsk->futex.state = FUTEX_STATE_OK; + mutex_init(&tsk->futex.exit_mutex); } void futex_exit_recursive(struct task_struct *tsk); diff --git a/include/linux/futex_types.h b/include/linux/futex_types.h new file mode 100644 index 000000000000..9c6c0dc4148d --- /dev/null +++ b/include/linux/futex_types.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_FUTEX_TYPES_H +#define _LINUX_FUTEX_TYPES_H + +#ifdef CONFIG_FUTEX +#include +#include + +struct compat_robust_list_head; +struct futex_pi_state; +struct robust_list_head; + +/** + * struct futex_sched_data - Futex related per task data + * @robust_list: User space registered robust list pointer + * @compat_robust_list: User space registered robust list pointer for compat tasks + * @pi_state_list: List head for Priority Inheritance (PI) state management + * @pi_state_cache: Pointer to cache one PI state object per task + * @exit_mutex: Mutex for serializing exit + * @state: Futex handling state to handle exit races correctly + */ +struct futex_sched_data { + struct robust_list_head __user *robust_list; +#ifdef CONFIG_COMPAT + struct compat_robust_list_head __user *compat_robust_list; +#endif + struct list_head pi_state_list; + struct futex_pi_state *pi_state_cache; + struct mutex exit_mutex; + unsigned int state; +}; +#else +struct futex_sched_data { }; +#endif /* !CONFIG_FUTEX */ + +#endif /* _LINUX_FUTEX_TYPES_H */ diff --git a/include/linux/sched.h b/include/linux/sched.h index 368c7b4d7cb5..c88fc10e9c38 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -64,7 +65,6 @@ struct bpf_net_context; struct capture_control; struct cfs_rq; struct fs_struct; -struct futex_pi_state; struct io_context; struct io_uring_task; struct mempolicy; @@ -76,7 +76,6 @@ struct pid_namespace; struct pipe_inode_info; struct rcu_node; struct reclaim_state; -struct robust_list_head; struct root_domain; struct rq; struct sched_attr; @@ -1331,16 +1330,9 @@ struct task_struct { u32 closid; u32 rmid; #endif -#ifdef CONFIG_FUTEX - struct robust_list_head __user *robust_list; -#ifdef CONFIG_COMPAT - struct compat_robust_list_head __user *compat_robust_list; -#endif - struct list_head pi_state_list; - struct futex_pi_state *pi_state_cache; - struct mutex futex_exit_mutex; - unsigned int futex_state; -#endif + + struct futex_sched_data futex; + #ifdef CONFIG_PERF_EVENTS u8 perf_recursion[PERF_NR_CONTEXTS]; struct perf_event_context *perf_event_ctxp; diff --git a/kernel/exit.c b/kernel/exit.c index 25e9cb6de7e7..1b4e55b60bc2 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -988,8 +988,8 @@ void __noreturn do_exit(long code) proc_exit_connector(tsk); mpol_put_task_policy(tsk); #ifdef CONFIG_FUTEX - if (unlikely(current->pi_state_cache)) - kfree(current->pi_state_cache); + if (unlikely(current->futex.pi_state_cache)) + kfree(current->futex.pi_state_cache); #endif /* * Make sure we are holding no locks: diff --git a/kernel/futex/core.c b/kernel/futex/core.c index ff2a4fb2993f..e7d33d2771ec 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -32,18 +32,19 @@ * "But they come in a choice of three flavours!" */ #include -#include -#include #include -#include +#include #include -#include +#include #include -#include -#include -#include #include #include +#include +#include +#include +#include +#include +#include #include "futex.h" #include "../locking/rtmutex_common.h" @@ -829,7 +830,7 @@ void wait_for_owner_exiting(int ret, struct task_struct *exiting) if (WARN_ON_ONCE(ret == -EBUSY && !exiting)) return; - mutex_lock(&exiting->futex_exit_mutex); + mutex_lock(&exiting->futex.exit_mutex); /* * No point in doing state checking here. If the waiter got here * while the task was in exec()->exec_futex_release() then it can @@ -838,7 +839,7 @@ void wait_for_owner_exiting(int ret, struct task_struct *exiting) * already. Highly unlikely and not a problem. Just one more round * through the futex maze. */ - mutex_unlock(&exiting->futex_exit_mutex); + mutex_unlock(&exiting->futex.exit_mutex); put_task_struct(exiting); } @@ -1047,7 +1048,7 @@ retry: * * In both cases the following conditions are met: * - * 1) task->robust_list->list_op_pending != NULL + * 1) task->futex.robust_list->list_op_pending != NULL * @pending_op == true * 2) The owner part of user space futex value == 0 * 3) Regular futex: @pi == false @@ -1152,7 +1153,7 @@ static inline int fetch_robust_entry(struct robust_list __user **entry, */ static void exit_robust_list(struct task_struct *curr) { - struct robust_list_head __user *head = curr->robust_list; + struct robust_list_head __user *head = curr->futex.robust_list; struct robust_list __user *entry, *next_entry, *pending; unsigned int limit = ROBUST_LIST_LIMIT, pi, pip; unsigned int next_pi; @@ -1246,7 +1247,7 @@ compat_fetch_robust_entry(compat_uptr_t *uentry, struct robust_list __user **ent */ static void compat_exit_robust_list(struct task_struct *curr) { - struct compat_robust_list_head __user *head = curr->compat_robust_list; + struct compat_robust_list_head __user *head = curr->futex.compat_robust_list; struct robust_list __user *entry, *next_entry, *pending; unsigned int limit = ROBUST_LIST_LIMIT, pi, pip; unsigned int next_pi; @@ -1322,7 +1323,7 @@ static void compat_exit_robust_list(struct task_struct *curr) */ static void exit_pi_state_list(struct task_struct *curr) { - struct list_head *next, *head = &curr->pi_state_list; + struct list_head *next, *head = &curr->futex.pi_state_list; struct futex_pi_state *pi_state; union futex_key key = FUTEX_KEY_INIT; @@ -1406,19 +1407,19 @@ static inline void exit_pi_state_list(struct task_struct *curr) { } static void futex_cleanup(struct task_struct *tsk) { - if (unlikely(tsk->robust_list)) { + if (unlikely(tsk->futex.robust_list)) { exit_robust_list(tsk); - tsk->robust_list = NULL; + tsk->futex.robust_list = NULL; } #ifdef CONFIG_COMPAT - if (unlikely(tsk->compat_robust_list)) { + if (unlikely(tsk->futex.compat_robust_list)) { compat_exit_robust_list(tsk); - tsk->compat_robust_list = NULL; + tsk->futex.compat_robust_list = NULL; } #endif - if (unlikely(!list_empty(&tsk->pi_state_list))) + if (unlikely(!list_empty(&tsk->futex.pi_state_list))) exit_pi_state_list(tsk); } @@ -1442,23 +1443,23 @@ static void futex_cleanup(struct task_struct *tsk) void futex_exit_recursive(struct task_struct *tsk) { /* If the state is FUTEX_STATE_EXITING then futex_exit_mutex is held */ - if (tsk->futex_state == FUTEX_STATE_EXITING) { - __assume_ctx_lock(&tsk->futex_exit_mutex); - mutex_unlock(&tsk->futex_exit_mutex); + if (tsk->futex.state == FUTEX_STATE_EXITING) { + __assume_ctx_lock(&tsk->futex.exit_mutex); + mutex_unlock(&tsk->futex.exit_mutex); } - tsk->futex_state = FUTEX_STATE_DEAD; + tsk->futex.state = FUTEX_STATE_DEAD; } static void futex_cleanup_begin(struct task_struct *tsk) - __acquires(&tsk->futex_exit_mutex) + __acquires(&tsk->futex.exit_mutex) { /* * Prevent various race issues against a concurrent incoming waiter * including live locks by forcing the waiter to block on - * tsk->futex_exit_mutex when it observes FUTEX_STATE_EXITING in + * tsk->futex.exit_mutex when it observes FUTEX_STATE_EXITING in * attach_to_pi_owner(). */ - mutex_lock(&tsk->futex_exit_mutex); + mutex_lock(&tsk->futex.exit_mutex); /* * Switch the state to FUTEX_STATE_EXITING under tsk->pi_lock. @@ -1472,23 +1473,23 @@ static void futex_cleanup_begin(struct task_struct *tsk) * be observed in exit_pi_state_list(). */ raw_spin_lock_irq(&tsk->pi_lock); - tsk->futex_state = FUTEX_STATE_EXITING; + tsk->futex.state = FUTEX_STATE_EXITING; raw_spin_unlock_irq(&tsk->pi_lock); } static void futex_cleanup_end(struct task_struct *tsk, int state) - __releases(&tsk->futex_exit_mutex) + __releases(&tsk->futex.exit_mutex) { /* * Lockless store. The only side effect is that an observer might * take another loop until it becomes visible. */ - tsk->futex_state = state; + tsk->futex.state = state; /* * Drop the exit protection. This unblocks waiters which observed * FUTEX_STATE_EXITING to reevaluate the state. */ - mutex_unlock(&tsk->futex_exit_mutex); + mutex_unlock(&tsk->futex.exit_mutex); } void futex_exec_release(struct task_struct *tsk) diff --git a/kernel/futex/pi.c b/kernel/futex/pi.c index 643199fdbe62..e037a97fe277 100644 --- a/kernel/futex/pi.c +++ b/kernel/futex/pi.c @@ -14,7 +14,7 @@ int refill_pi_state_cache(void) { struct futex_pi_state *pi_state; - if (likely(current->pi_state_cache)) + if (likely(current->futex.pi_state_cache)) return 0; pi_state = kzalloc_obj(*pi_state); @@ -28,17 +28,17 @@ int refill_pi_state_cache(void) refcount_set(&pi_state->refcount, 1); pi_state->key = FUTEX_KEY_INIT; - current->pi_state_cache = pi_state; + current->futex.pi_state_cache = pi_state; return 0; } static struct futex_pi_state *alloc_pi_state(void) { - struct futex_pi_state *pi_state = current->pi_state_cache; + struct futex_pi_state *pi_state = current->futex.pi_state_cache; WARN_ON(!pi_state); - current->pi_state_cache = NULL; + current->futex.pi_state_cache = NULL; return pi_state; } @@ -60,7 +60,7 @@ static void pi_state_update_owner(struct futex_pi_state *pi_state, if (new_owner) { raw_spin_lock(&new_owner->pi_lock); WARN_ON(!list_empty(&pi_state->list)); - list_add(&pi_state->list, &new_owner->pi_state_list); + list_add(&pi_state->list, &new_owner->futex.pi_state_list); pi_state->owner = new_owner; raw_spin_unlock(&new_owner->pi_lock); } @@ -96,7 +96,7 @@ void put_pi_state(struct futex_pi_state *pi_state) raw_spin_unlock_irqrestore(&pi_state->pi_mutex.wait_lock, flags); } - if (current->pi_state_cache) { + if (current->futex.pi_state_cache) { kfree(pi_state); } else { /* @@ -106,7 +106,7 @@ void put_pi_state(struct futex_pi_state *pi_state) */ pi_state->owner = NULL; refcount_set(&pi_state->refcount, 1); - current->pi_state_cache = pi_state; + current->futex.pi_state_cache = pi_state; } } @@ -179,7 +179,7 @@ void put_pi_state(struct futex_pi_state *pi_state) * * p->pi_lock: * - * p->pi_state_list -> pi_state->list, relation + * p->futex.pi_state_list -> pi_state->list, relation * pi_mutex->owner -> pi_state->owner, relation * * pi_state->refcount: @@ -327,7 +327,7 @@ static int handle_exit_race(u32 __user *uaddr, u32 uval, * If the futex exit state is not yet FUTEX_STATE_DEAD, tell the * caller that the alleged owner is busy. */ - if (tsk && tsk->futex_state != FUTEX_STATE_DEAD) + if (tsk && tsk->futex.state != FUTEX_STATE_DEAD) return -EBUSY; /* @@ -346,8 +346,8 @@ static int handle_exit_race(u32 __user *uaddr, u32 uval, * *uaddr = 0xC0000000; tsk = get_task(PID); * } if (!tsk->flags & PF_EXITING) { * ... attach(); - * tsk->futex_state = } else { - * FUTEX_STATE_DEAD; if (tsk->futex_state != + * tsk->futex.state = } else { + * FUTEX_STATE_DEAD; if (tsk->futex.state != * FUTEX_STATE_DEAD) * return -EAGAIN; * return -ESRCH; <--- FAIL @@ -396,7 +396,7 @@ static void __attach_to_pi_owner(struct task_struct *p, union futex_key *key, pi_state->key = *key; WARN_ON(!list_empty(&pi_state->list)); - list_add(&pi_state->list, &p->pi_state_list); + list_add(&pi_state->list, &p->futex.pi_state_list); /* * Assignment without holding pi_state->pi_mutex.wait_lock is safe * because there is no concurrency as the object is not published yet. @@ -440,7 +440,7 @@ static int attach_to_pi_owner(u32 __user *uaddr, u32 uval, union futex_key *key, * in futex_exit_release(), we do this protected by p->pi_lock: */ raw_spin_lock_irq(&p->pi_lock); - if (unlikely(p->futex_state != FUTEX_STATE_OK)) { + if (unlikely(p->futex.state != FUTEX_STATE_OK)) { /* * The task is on the way out. When the futex state is * FUTEX_STATE_DEAD, we know that the task has finished diff --git a/kernel/futex/syscalls.c b/kernel/futex/syscalls.c index 77ad9691f6a6..8944ff4930d7 100644 --- a/kernel/futex/syscalls.c +++ b/kernel/futex/syscalls.c @@ -25,17 +25,13 @@ * @head: pointer to the list-head * @len: length of the list-head, as userspace expects */ -SYSCALL_DEFINE2(set_robust_list, struct robust_list_head __user *, head, - size_t, len) +SYSCALL_DEFINE2(set_robust_list, struct robust_list_head __user *, head, size_t, len) { - /* - * The kernel knows only one size for now: - */ + /* The kernel knows only one size for now. */ if (unlikely(len != sizeof(*head))) return -EINVAL; - current->robust_list = head; - + current->futex.robust_list = head; return 0; } @@ -43,9 +39,9 @@ static inline void __user *futex_task_robust_list(struct task_struct *p, bool co { #ifdef CONFIG_COMPAT if (compat) - return p->compat_robust_list; + return p->futex.compat_robust_list; #endif - return p->robust_list; + return p->futex.robust_list; } static void __user *futex_get_robust_list_common(int pid, bool compat) @@ -475,15 +471,13 @@ SYSCALL_DEFINE4(futex_requeue, } #ifdef CONFIG_COMPAT -COMPAT_SYSCALL_DEFINE2(set_robust_list, - struct compat_robust_list_head __user *, head, - compat_size_t, len) +COMPAT_SYSCALL_DEFINE2(set_robust_list, struct compat_robust_list_head __user *, head, + compat_size_t, len) { if (unlikely(len != sizeof(*head))) return -EINVAL; - current->compat_robust_list = head; - + current->futex.compat_robust_list = head; return 0; } @@ -523,4 +517,3 @@ SYSCALL_DEFINE6(futex_time32, u32 __user *, uaddr, int, op, u32, val, return do_futex(uaddr, op, val, tp, uaddr2, (unsigned long)utime, val3); } #endif /* CONFIG_COMPAT_32BIT_TIME */ - -- cgit v1.2.3 From d7b3f52c861f54ba2fff15696d3798277fb4c19f Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 2 Jun 2026 11:09:29 +0200 Subject: futex: Make futex_mm_init() void Nothing fails there. Mop up the leftovers of the early version of this, which did an allocation. While at it clean up the stubs and the #ifdef comments to make the header file readable. Signed-off-by: Thomas Gleixner Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260602090535.356789395@kernel.org --- include/linux/futex.h | 28 +++++++++++----------------- kernel/fork.c | 8 ++------ kernel/futex/core.c | 3 +-- 3 files changed, 14 insertions(+), 25 deletions(-) (limited to 'include/linux') diff --git a/include/linux/futex.h b/include/linux/futex.h index 563e8dd67179..9e6218c2be66 100644 --- a/include/linux/futex.h +++ b/include/linux/futex.h @@ -81,22 +81,20 @@ int futex_hash_prctl(unsigned long arg2, unsigned long arg3, unsigned long arg4) #ifdef CONFIG_FUTEX_PRIVATE_HASH int futex_hash_allocate_default(void); void futex_hash_free(struct mm_struct *mm); -int futex_mm_init(struct mm_struct *mm); - -#else /* !CONFIG_FUTEX_PRIVATE_HASH */ +void futex_mm_init(struct mm_struct *mm); +#else /* CONFIG_FUTEX_PRIVATE_HASH */ static inline int futex_hash_allocate_default(void) { return 0; } static inline int futex_hash_free(struct mm_struct *mm) { return 0; } -static inline int futex_mm_init(struct mm_struct *mm) { return 0; } -#endif /* CONFIG_FUTEX_PRIVATE_HASH */ +static inline void futex_mm_init(struct mm_struct *mm) { } +#endif /* !CONFIG_FUTEX_PRIVATE_HASH */ -#else /* !CONFIG_FUTEX */ +#else /* CONFIG_FUTEX */ static inline void futex_init_task(struct task_struct *tsk) { } static inline void futex_exit_recursive(struct task_struct *tsk) { } static inline void futex_exit_release(struct task_struct *tsk) { } static inline void futex_exec_release(struct task_struct *tsk) { } -static inline long do_futex(u32 __user *uaddr, int op, u32 val, - ktime_t *timeout, u32 __user *uaddr2, - u32 val2, u32 val3) +static inline long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, + u32 __user *uaddr2, u32 val2, u32 val3) { return -EINVAL; } @@ -104,13 +102,9 @@ static inline int futex_hash_prctl(unsigned long arg2, unsigned long arg3, unsig { return -EINVAL; } -static inline int futex_hash_allocate_default(void) -{ - return 0; -} +static inline int futex_hash_allocate_default(void) { return 0; } static inline int futex_hash_free(struct mm_struct *mm) { return 0; } -static inline int futex_mm_init(struct mm_struct *mm) { return 0; } - -#endif +static inline void futex_mm_init(struct mm_struct *mm) { } +#endif /* !CONFIG_FUTEX */ -#endif +#endif /* _LINUX_FUTEX_H */ diff --git a/kernel/fork.c b/kernel/fork.c index 5f3fdfdb14c7..bb490d97c222 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1101,6 +1101,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, #endif mm_init_uprobes_state(mm); hugetlb_count_init(mm); + futex_mm_init(mm); mm_flags_clear_all(mm); if (current->mm) { @@ -1113,11 +1114,8 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, mm->def_flags = 0; } - if (futex_mm_init(mm)) - goto fail_mm_init; - if (mm_alloc_pgd(mm)) - goto fail_nopgd; + goto fail_mm_init; if (mm_alloc_id(mm)) goto fail_noid; @@ -1144,8 +1142,6 @@ fail_nocontext: mm_free_id(mm); fail_noid: mm_free_pgd(mm); -fail_nopgd: - futex_hash_free(mm); fail_mm_init: free_mm(mm); return NULL; diff --git a/kernel/futex/core.c b/kernel/futex/core.c index e7d33d2771ec..ec23de4912b3 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -1720,7 +1720,7 @@ static bool futex_ref_is_dead(struct futex_private_hash *fph) return atomic_long_read(&mm->futex_atomic) == 0; } -int futex_mm_init(struct mm_struct *mm) +void futex_mm_init(struct mm_struct *mm) { mutex_init(&mm->futex_hash_lock); RCU_INIT_POINTER(mm->futex_phash, NULL); @@ -1729,7 +1729,6 @@ int futex_mm_init(struct mm_struct *mm) mm->futex_ref = NULL; atomic_long_set(&mm->futex_atomic, 0); mm->futex_batches = get_state_synchronize_rcu(); - return 0; } void futex_hash_free(struct mm_struct *mm) -- cgit v1.2.3 From 1f7f4816b9b05e5110bc1c8a05c3c478e2dae11b Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 2 Jun 2026 11:09:34 +0200 Subject: futex: Move futex related mm_struct data into a struct Having all these members in mm_struct along with the required #ifdeffery is annoying, does not allow efficient initializing of the data with memset() and makes extending it tedious. Move it into a data structure and fix up all usage sites. The extra struct for the private hash is intentional to make integration of other conditional mechanisms easier in terms of initialization and separation. Signed-off-by: Thomas Gleixner Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260602090535.407756793@kernel.org --- include/linux/futex_types.h | 36 +++++++++++- include/linux/mm_types.h | 12 +--- kernel/futex/core.c | 133 ++++++++++++++++++++------------------------ 3 files changed, 98 insertions(+), 83 deletions(-) (limited to 'include/linux') diff --git a/include/linux/futex_types.h b/include/linux/futex_types.h index 9c6c0dc4148d..d41557d2790d 100644 --- a/include/linux/futex_types.h +++ b/include/linux/futex_types.h @@ -3,6 +3,7 @@ #define _LINUX_FUTEX_TYPES_H #ifdef CONFIG_FUTEX +#include #include #include @@ -29,8 +30,41 @@ struct futex_sched_data { struct mutex exit_mutex; unsigned int state; }; -#else + +#ifdef CONFIG_FUTEX_PRIVATE_HASH +/** + * struct futex_mm_phash - Futex private hash related per MM data + * @lock: Mutex to protect the private hash operations + * @hash: RCU managed pointer to the private hash + * @hash_new: Pointer to a newly allocated private hash + * @batches: Batch state for RCU synchronization + * @rcu: RCU head for call_rcu() + * @atomic: Aggregate value for @hash_ref + * @ref: Per CPU reference counter for a private hash + */ +struct futex_mm_phash { + struct mutex lock; + struct futex_private_hash __rcu *hash; + struct futex_private_hash *hash_new; + unsigned long batches; + struct rcu_head rcu; + atomic_long_t atomic; + unsigned int __percpu *ref; +}; +#else /* CONFIG_FUTEX_ROBUST_UNLOCK */ +struct futex_mm_phash { }; +#endif /* !CONFIG_FUTEX_ROBUST_UNLOCK */ + +/** + * struct futex_mm_data - Futex related per MM data + * @phash: Futex private hash related data + */ +struct futex_mm_data { + struct futex_mm_phash phash; +}; +#else /* CONFIG_FUTEX */ struct futex_sched_data { }; +struct futex_mm_data { }; #endif /* !CONFIG_FUTEX */ #endif /* _LINUX_FUTEX_TYPES_H */ diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index a308e2c23b82..1d0c8d8875aa 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -1270,16 +1271,7 @@ struct mm_struct { */ seqcount_t mm_lock_seq; #endif -#ifdef CONFIG_FUTEX_PRIVATE_HASH - struct mutex futex_hash_lock; - struct futex_private_hash __rcu *futex_phash; - struct futex_private_hash *futex_phash_new; - /* futex-ref */ - unsigned long futex_batches; - struct rcu_head futex_rcu; - atomic_long_t futex_atomic; - unsigned int __percpu *futex_ref; -#endif + struct futex_mm_data futex; unsigned long hiwater_rss; /* High-watermark of RSS usage */ unsigned long hiwater_vm; /* High-water virtual memory usage */ diff --git a/kernel/futex/core.c b/kernel/futex/core.c index ec23de4912b3..79456b0ac1c6 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -188,13 +188,13 @@ __futex_hash_private(union futex_key *key, struct futex_private_hash *fph) return NULL; if (!fph) - fph = rcu_dereference(key->private.mm->futex_phash); + fph = rcu_dereference(key->private.mm->futex.phash.hash); if (!fph || !fph->hash_mask) return NULL; - hash = jhash2((void *)&key->private.address, - sizeof(key->private.address) / 4, + hash = jhash2((void *)&key->private.address, sizeof(key->private.address) / 4, key->both.offset); + return &fph->queues[hash & fph->hash_mask]; } @@ -233,18 +233,17 @@ static void futex_rehash_private(struct futex_private_hash *old, } } -static bool __futex_pivot_hash(struct mm_struct *mm, - struct futex_private_hash *new) +static bool __futex_pivot_hash(struct mm_struct *mm, struct futex_private_hash *new) { + struct futex_mm_phash *mmph = &mm->futex.phash; struct futex_private_hash *fph; - WARN_ON_ONCE(mm->futex_phash_new); + WARN_ON_ONCE(mmph->hash_new); - fph = rcu_dereference_protected(mm->futex_phash, - lockdep_is_held(&mm->futex_hash_lock)); + fph = rcu_dereference_protected(mmph->hash, lockdep_is_held(&mmph->lock)); if (fph) { if (!futex_ref_is_dead(fph)) { - mm->futex_phash_new = new; + mmph->hash_new = new; return false; } @@ -252,8 +251,8 @@ static bool __futex_pivot_hash(struct mm_struct *mm, } new->state = FR_PERCPU; scoped_guard(rcu) { - mm->futex_batches = get_state_synchronize_rcu(); - rcu_assign_pointer(mm->futex_phash, new); + mmph->batches = get_state_synchronize_rcu(); + rcu_assign_pointer(mmph->hash, new); } kvfree_rcu(fph, rcu); return true; @@ -261,12 +260,12 @@ static bool __futex_pivot_hash(struct mm_struct *mm, static void futex_pivot_hash(struct mm_struct *mm) { - scoped_guard(mutex, &mm->futex_hash_lock) { + scoped_guard(mutex, &mm->futex.phash.lock) { struct futex_private_hash *fph; - fph = mm->futex_phash_new; + fph = mm->futex.phash.hash_new; if (fph) { - mm->futex_phash_new = NULL; + mm->futex.phash.hash_new = NULL; __futex_pivot_hash(mm, fph); } } @@ -289,7 +288,7 @@ again: scoped_guard(rcu) { struct futex_private_hash *fph; - fph = rcu_dereference(mm->futex_phash); + fph = rcu_dereference(mm->futex.phash.hash); if (!fph) return NULL; @@ -412,8 +411,7 @@ static int futex_mpol(struct mm_struct *mm, unsigned long addr) * private hash) is returned if existing. Otherwise a hash bucket from the * global hash is returned. */ -static struct futex_hash_bucket * -__futex_hash(union futex_key *key, struct futex_private_hash *fph) +static struct futex_hash_bucket *__futex_hash(union futex_key *key, struct futex_private_hash *fph) { int node = key->both.node; u32 hash; @@ -426,8 +424,7 @@ __futex_hash(union futex_key *key, struct futex_private_hash *fph) return hb; } - hash = jhash2((u32 *)key, - offsetof(typeof(*key), both.offset) / sizeof(u32), + hash = jhash2((u32 *)key, offsetof(typeof(*key), both.offset) / sizeof(u32), key->both.offset); if (node == FUTEX_NO_NODE) { @@ -442,8 +439,7 @@ __futex_hash(union futex_key *key, struct futex_private_hash *fph) */ node = (hash >> futex_hashshift) % nr_node_ids; if (!node_possible(node)) { - node = find_next_bit_wrap(node_possible_map.bits, - nr_node_ids, node); + node = find_next_bit_wrap(node_possible_map.bits, nr_node_ids, node); } } @@ -460,9 +456,8 @@ __futex_hash(union futex_key *key, struct futex_private_hash *fph) * Return: Initialized hrtimer_sleeper structure or NULL if no timeout * value given */ -struct hrtimer_sleeper * -futex_setup_timer(ktime_t *time, struct hrtimer_sleeper *timeout, - int flags, u64 range_ns) +struct hrtimer_sleeper *futex_setup_timer(ktime_t *time, struct hrtimer_sleeper *timeout, + int flags, u64 range_ns) { if (!time) return NULL; @@ -1554,17 +1549,17 @@ static void __futex_ref_atomic_begin(struct futex_private_hash *fph) * otherwise it would be impossible for it to have reported success * from futex_ref_is_dead(). */ - WARN_ON_ONCE(atomic_long_read(&mm->futex_atomic) != 0); + WARN_ON_ONCE(atomic_long_read(&mm->futex.phash.atomic) != 0); /* * Set the atomic to the bias value such that futex_ref_{get,put}() * will never observe 0. Will be fixed up in __futex_ref_atomic_end() * when folding in the percpu count. */ - atomic_long_set(&mm->futex_atomic, LONG_MAX); + atomic_long_set(&mm->futex.phash.atomic, LONG_MAX); smp_store_release(&fph->state, FR_ATOMIC); - call_rcu_hurry(&mm->futex_rcu, futex_ref_rcu); + call_rcu_hurry(&mm->futex.phash.rcu, futex_ref_rcu); } static void __futex_ref_atomic_end(struct futex_private_hash *fph) @@ -1585,7 +1580,7 @@ static void __futex_ref_atomic_end(struct futex_private_hash *fph) * Therefore the per-cpu counter is now stable, sum and reset. */ for_each_possible_cpu(cpu) { - unsigned int *ptr = per_cpu_ptr(mm->futex_ref, cpu); + unsigned int *ptr = per_cpu_ptr(mm->futex.phash.ref, cpu); count += *ptr; *ptr = 0; } @@ -1593,7 +1588,7 @@ static void __futex_ref_atomic_end(struct futex_private_hash *fph) /* * Re-init for the next cycle. */ - this_cpu_inc(*mm->futex_ref); /* 0 -> 1 */ + this_cpu_inc(*mm->futex.phash.ref); /* 0 -> 1 */ /* * Add actual count, subtract bias and initial refcount. @@ -1601,7 +1596,7 @@ static void __futex_ref_atomic_end(struct futex_private_hash *fph) * The moment this atomic operation happens, futex_ref_is_dead() can * become true. */ - ret = atomic_long_add_return(count - LONG_MAX - 1, &mm->futex_atomic); + ret = atomic_long_add_return(count - LONG_MAX - 1, &mm->futex.phash.atomic); if (!ret) wake_up_var(mm); @@ -1611,8 +1606,8 @@ static void __futex_ref_atomic_end(struct futex_private_hash *fph) static void futex_ref_rcu(struct rcu_head *head) { - struct mm_struct *mm = container_of(head, struct mm_struct, futex_rcu); - struct futex_private_hash *fph = rcu_dereference_raw(mm->futex_phash); + struct mm_struct *mm = container_of(head, struct mm_struct, futex.phash.rcu); + struct futex_private_hash *fph = rcu_dereference_raw(mm->futex.phash.hash); if (fph->state == FR_PERCPU) { /* @@ -1641,7 +1636,7 @@ static void futex_ref_drop(struct futex_private_hash *fph) /* * Can only transition the current fph; */ - WARN_ON_ONCE(rcu_dereference_raw(mm->futex_phash) != fph); + WARN_ON_ONCE(rcu_dereference_raw(mm->futex.phash.hash) != fph); /* * We enqueue at least one RCU callback. Ensure mm stays if the task * exits before the transition is completed. @@ -1652,9 +1647,9 @@ static void futex_ref_drop(struct futex_private_hash *fph) * In order to avoid the following scenario: * * futex_hash() __futex_pivot_hash() - * guard(rcu); guard(mm->futex_hash_lock); - * fph = mm->futex_phash; - * rcu_assign_pointer(&mm->futex_phash, new); + * guard(rcu); guard(mm->futex.phash.lock); + * fph = mm->futex.phash.hash; + * rcu_assign_pointer(&mm->futex.phash.hash, new); * futex_hash_allocate() * futex_ref_drop() * fph->state = FR_ATOMIC; @@ -1669,7 +1664,7 @@ static void futex_ref_drop(struct futex_private_hash *fph) * There must be at least one full grace-period between publishing a * new fph and trying to replace it. */ - if (poll_state_synchronize_rcu(mm->futex_batches)) { + if (poll_state_synchronize_rcu(mm->futex.phash.batches)) { /* * There was a grace-period, we can begin now. */ @@ -1677,7 +1672,7 @@ static void futex_ref_drop(struct futex_private_hash *fph) return; } - call_rcu_hurry(&mm->futex_rcu, futex_ref_rcu); + call_rcu_hurry(&mm->futex.phash.rcu, futex_ref_rcu); } static bool futex_ref_get(struct futex_private_hash *fph) @@ -1687,11 +1682,11 @@ static bool futex_ref_get(struct futex_private_hash *fph) guard(preempt)(); if (READ_ONCE(fph->state) == FR_PERCPU) { - __this_cpu_inc(*mm->futex_ref); + __this_cpu_inc(*mm->futex.phash.ref); return true; } - return atomic_long_inc_not_zero(&mm->futex_atomic); + return atomic_long_inc_not_zero(&mm->futex.phash.atomic); } static bool futex_ref_put(struct futex_private_hash *fph) @@ -1701,11 +1696,11 @@ static bool futex_ref_put(struct futex_private_hash *fph) guard(preempt)(); if (READ_ONCE(fph->state) == FR_PERCPU) { - __this_cpu_dec(*mm->futex_ref); + __this_cpu_dec(*mm->futex.phash.ref); return false; } - return atomic_long_dec_and_test(&mm->futex_atomic); + return atomic_long_dec_and_test(&mm->futex.phash.atomic); } static bool futex_ref_is_dead(struct futex_private_hash *fph) @@ -1717,27 +1712,23 @@ static bool futex_ref_is_dead(struct futex_private_hash *fph) if (smp_load_acquire(&fph->state) == FR_PERCPU) return false; - return atomic_long_read(&mm->futex_atomic) == 0; + return atomic_long_read(&mm->futex.phash.atomic) == 0; } void futex_mm_init(struct mm_struct *mm) { - mutex_init(&mm->futex_hash_lock); - RCU_INIT_POINTER(mm->futex_phash, NULL); - mm->futex_phash_new = NULL; - /* futex-ref */ - mm->futex_ref = NULL; - atomic_long_set(&mm->futex_atomic, 0); - mm->futex_batches = get_state_synchronize_rcu(); + memset(&mm->futex, 0, sizeof(mm->futex)); + mutex_init(&mm->futex.phash.lock); + mm->futex.phash.batches = get_state_synchronize_rcu(); } void futex_hash_free(struct mm_struct *mm) { struct futex_private_hash *fph; - free_percpu(mm->futex_ref); - kvfree(mm->futex_phash_new); - fph = rcu_dereference_raw(mm->futex_phash); + free_percpu(mm->futex.phash.ref); + kvfree(mm->futex.phash.hash_new); + fph = rcu_dereference_raw(mm->futex.phash.hash); if (fph) kvfree(fph); } @@ -1748,10 +1739,10 @@ static bool futex_pivot_pending(struct mm_struct *mm) guard(rcu)(); - if (!mm->futex_phash_new) + if (!mm->futex.phash.hash_new) return true; - fph = rcu_dereference(mm->futex_phash); + fph = rcu_dereference(mm->futex.phash.hash); return futex_ref_is_dead(fph); } @@ -1793,7 +1784,7 @@ static int futex_hash_allocate(unsigned int hash_slots, unsigned int flags) * Once we've disabled the global hash there is no way back. */ scoped_guard(rcu) { - fph = rcu_dereference(mm->futex_phash); + fph = rcu_dereference(mm->futex.phash.hash); if (fph && !fph->hash_mask) { if (custom) return -EBUSY; @@ -1801,15 +1792,15 @@ static int futex_hash_allocate(unsigned int hash_slots, unsigned int flags) } } - if (!mm->futex_ref) { + if (!mm->futex.phash.ref) { /* * This will always be allocated by the first thread and * therefore requires no locking. */ - mm->futex_ref = alloc_percpu(unsigned int); - if (!mm->futex_ref) + mm->futex.phash.ref = alloc_percpu(unsigned int); + if (!mm->futex.phash.ref) return -ENOMEM; - this_cpu_inc(*mm->futex_ref); /* 0 -> 1 */ + this_cpu_inc(*mm->futex.phash.ref); /* 0 -> 1 */ } fph = kvzalloc(struct_size(fph, queues, hash_slots), @@ -1832,14 +1823,14 @@ again: wait_var_event(mm, futex_pivot_pending(mm)); } - scoped_guard(mutex, &mm->futex_hash_lock) { + scoped_guard(mutex, &mm->futex.phash.lock) { struct futex_private_hash *free __free(kvfree) = NULL; struct futex_private_hash *cur, *new; - cur = rcu_dereference_protected(mm->futex_phash, - lockdep_is_held(&mm->futex_hash_lock)); - new = mm->futex_phash_new; - mm->futex_phash_new = NULL; + cur = rcu_dereference_protected(mm->futex.phash.hash, + lockdep_is_held(&mm->futex.phash.lock)); + new = mm->futex.phash.hash_new; + mm->futex.phash.hash_new = NULL; if (fph) { if (cur && !cur->hash_mask) { @@ -1849,7 +1840,7 @@ again: * the second one returns here. */ free = fph; - mm->futex_phash_new = new; + mm->futex.phash.hash_new = new; return -EBUSY; } if (cur && !new) { @@ -1879,7 +1870,7 @@ again: if (new) { /* - * Will set mm->futex_phash_new on failure; + * Will set mm->futex.phash.new_hash on failure; * futex_private_hash_get() will try again. */ if (!__futex_pivot_hash(mm, new) && custom) @@ -1898,11 +1889,9 @@ int futex_hash_allocate_default(void) return 0; scoped_guard(rcu) { - threads = min_t(unsigned int, - get_nr_threads(current), - num_online_cpus()); + threads = min_t(unsigned int, get_nr_threads(current), num_online_cpus()); - fph = rcu_dereference(current->mm->futex_phash); + fph = rcu_dereference(current->mm->futex.phash.hash); if (fph) { if (fph->custom) return 0; @@ -1929,7 +1918,7 @@ static int futex_hash_get_slots(void) struct futex_private_hash *fph; guard(rcu)(); - fph = rcu_dereference(current->mm->futex_phash); + fph = rcu_dereference(current->mm->futex.phash.hash); if (fph && fph->hash_mask) return fph->hash_mask + 1; return 0; -- cgit v1.2.3 From 6149fc36c09b91050b62e8e68a91027df8df7345 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 2 Jun 2026 11:09:42 +0200 Subject: uaccess: Provide unsafe_atomic_store_release_user() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The upcoming support for unlocking robust futexes in the kernel requires store release semantics. Syscalls do not imply memory ordering on all architectures so the unlock operation requires a barrier. This barrier can be avoided when stores imply release like on x86. Provide a generic version with a smp_mb() before the unsafe_put_user(), which can be overridden by architectures. Provide also a ARCH_MEMORY_ORDER_TSO Kconfig option, which can be selected by architectures with Total Store Order (TSO), where store implies release, so that the smp_mb() in the generic implementation can be avoided. If that is set a barrier() is used instead of smp_mb(), which is not required for the use case at hand, but makes it future proof for other usage to prevent the compiler from reordering. Signed-off-by: Thomas Gleixner Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: André Almeida Link: https://patch.msgid.link/20260602090535.513181528@kernel.org --- arch/Kconfig | 4 ++++ include/linux/uaccess.h | 11 +++++++++++ 2 files changed, 15 insertions(+) (limited to 'include/linux') diff --git a/arch/Kconfig b/arch/Kconfig index e86880045158..83d362fca38c 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -403,6 +403,10 @@ config ARCH_32BIT_OFF_T config ARCH_32BIT_USTAT_F_TINODE bool +# Selected by architectures with Total Store Order (TSO) +config ARCH_MEMORY_ORDER_TSO + bool + config HAVE_ASM_MODVERSIONS bool help diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h index 56328601218c..c6bd20065a5d 100644 --- a/include/linux/uaccess.h +++ b/include/linux/uaccess.h @@ -649,6 +649,17 @@ static inline void user_access_restore(unsigned long flags) { } #define user_read_access_end user_access_end #endif +#ifndef unsafe_atomic_store_release_user +# define unsafe_atomic_store_release_user(val, uptr, elbl) \ + do { \ + if (!IS_ENABLED(CONFIG_ARCH_MEMORY_ORDER_TSO)) \ + smp_mb(); \ + else \ + barrier(); \ + unsafe_put_user(val, uptr, elbl); \ + } while (0) +#endif + /* Define RW variant so the below _mode macro expansion works */ #define masked_user_rw_access_begin(u) masked_user_access_begin(u) #define user_rw_access_begin(u, s) user_access_begin(u, s) -- cgit v1.2.3 From 042df0c1d48609a85580dcbaff498c95ced20a5f Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 2 Jun 2026 11:09:59 +0200 Subject: futex: Add robust futex unlock IP range MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There will be a VDSO function to unlock robust futexes in user space. The unlock sequence is racy vs. clearing the list_pending_op pointer in the tasks robust list head. To plug this race the kernel needs to know the instruction window. As the VDSO is per MM the addresses are stored in mm_struct::futex. Architectures which implement support for this have to update these addresses when the VDSO is (re)mapped and indicate the pending op pointer size which is matching the IP. Arguably this could be resolved by chasing mm->context->vdso->image, but that's architecture specific and requires to touch quite some cache lines. Having it in mm::futex reduces the cache line impact and avoids having yet another set of architecture specific functionality. To support multi size robust list applications (gaming) this provides two ranges when COMPAT is enabled. Signed-off-by: Thomas Gleixner Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: André Almeida Link: https://patch.msgid.link/20260602090535.718926819@kernel.org --- include/linux/futex.h | 21 ++++++++++++++++++--- include/linux/futex_types.h | 28 +++++++++++++++++++++++++++ init/Kconfig | 6 ++++++ kernel/futex/core.c | 46 ++++++++++++++++++++++++++++++++++++--------- 4 files changed, 89 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/include/linux/futex.h b/include/linux/futex.h index 9e6218c2be66..cb2a182547dd 100644 --- a/include/linux/futex.h +++ b/include/linux/futex.h @@ -81,11 +81,9 @@ int futex_hash_prctl(unsigned long arg2, unsigned long arg3, unsigned long arg4) #ifdef CONFIG_FUTEX_PRIVATE_HASH int futex_hash_allocate_default(void); void futex_hash_free(struct mm_struct *mm); -void futex_mm_init(struct mm_struct *mm); #else /* CONFIG_FUTEX_PRIVATE_HASH */ static inline int futex_hash_allocate_default(void) { return 0; } static inline int futex_hash_free(struct mm_struct *mm) { return 0; } -static inline void futex_mm_init(struct mm_struct *mm) { } #endif /* !CONFIG_FUTEX_PRIVATE_HASH */ #else /* CONFIG_FUTEX */ @@ -104,7 +102,24 @@ static inline int futex_hash_prctl(unsigned long arg2, unsigned long arg3, unsig } static inline int futex_hash_allocate_default(void) { return 0; } static inline int futex_hash_free(struct mm_struct *mm) { return 0; } -static inline void futex_mm_init(struct mm_struct *mm) { } #endif /* !CONFIG_FUTEX */ +#ifdef CONFIG_FUTEX_ROBUST_UNLOCK +void futex_reset_cs_ranges(struct futex_mm_data *fd); + +static inline void futex_set_vdso_cs_range(struct futex_mm_data *fd, unsigned int idx, + unsigned long start, unsigned long end, bool sz32) +{ + fd->unlock.cs_ranges[idx].start_ip = start; + fd->unlock.cs_ranges[idx].len = end - start; + fd->unlock.cs_ranges[idx].pop_size32 = sz32; +} +#endif /* CONFIG_FUTEX_ROBUST_UNLOCK */ + +#if defined(CONFIG_FUTEX_PRIVATE_HASH) || defined(CONFIG_FUTEX_ROBUST_UNLOCK) +void futex_mm_init(struct mm_struct *mm); +#else +static inline void futex_mm_init(struct mm_struct *mm) { } +#endif + #endif /* _LINUX_FUTEX_H */ diff --git a/include/linux/futex_types.h b/include/linux/futex_types.h index d41557d2790d..d320c0571f0c 100644 --- a/include/linux/futex_types.h +++ b/include/linux/futex_types.h @@ -55,12 +55,40 @@ struct futex_mm_phash { struct futex_mm_phash { }; #endif /* !CONFIG_FUTEX_ROBUST_UNLOCK */ +#ifdef CONFIG_FUTEX_ROBUST_UNLOCK +/** + * struct futex_unlock_cs_range - Range for the VDSO unlock critical section + * @start_ip: The start IP of the robust futex unlock critical section (inclusive) + * @len: The length of the robust futex unlock critical section + * @pop_size32: Pending OP pointer size indicator. 0 == 64-bit, 1 == 32-bit + */ +struct futex_unlock_cs_range { + unsigned long start_ip; + unsigned int len; + unsigned int pop_size32; +}; + +#define FUTEX_ROBUST_MAX_CS_RANGES (1 + IS_ENABLED(CONFIG_COMPAT)) + +/** + * struct futex_unlock_cs_ranges - Futex unlock VSDO critical sections + * @cs_ranges: Array of critical section ranges + */ +struct futex_unlock_cs_ranges { + struct futex_unlock_cs_range cs_ranges[FUTEX_ROBUST_MAX_CS_RANGES]; +}; +#else /* CONFIG_FUTEX_ROBUST_UNLOCK */ +struct futex_unlock_cs_ranges { }; +#endif /* !CONFIG_FUTEX_ROBUST_UNLOCK */ + /** * struct futex_mm_data - Futex related per MM data * @phash: Futex private hash related data + * @unlock: Futex unlock VDSO critical sections */ struct futex_mm_data { struct futex_mm_phash phash; + struct futex_unlock_cs_ranges unlock; }; #else /* CONFIG_FUTEX */ struct futex_sched_data { }; diff --git a/init/Kconfig b/init/Kconfig index 2937c4d308ae..165b08e336a8 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -1842,6 +1842,12 @@ config FUTEX_MPOL depends on FUTEX && NUMA default y +config HAVE_FUTEX_ROBUST_UNLOCK + bool + +config FUTEX_ROBUST_UNLOCK + def_bool FUTEX && HAVE_GENERIC_VDSO && GENERIC_IRQ_ENTRY && RSEQ && HAVE_FUTEX_ROBUST_UNLOCK + config EPOLL bool "Enable eventpoll support" if EXPERT default y diff --git a/kernel/futex/core.c b/kernel/futex/core.c index 77ccb7787c34..aad6e501fabd 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -1761,11 +1761,11 @@ static bool futex_ref_is_dead(struct futex_private_hash *fph) return atomic_long_read(&mm->futex.phash.atomic) == 0; } -void futex_mm_init(struct mm_struct *mm) +static void futex_hash_init_mm(struct futex_mm_data *fd) { - memset(&mm->futex, 0, sizeof(mm->futex)); - mutex_init(&mm->futex.phash.lock); - mm->futex.phash.batches = get_state_synchronize_rcu(); + memset(&fd->phash, 0, sizeof(fd->phash)); + mutex_init(&fd->phash.lock); + fd->phash.batches = get_state_synchronize_rcu(); } void futex_hash_free(struct mm_struct *mm) @@ -1969,19 +1969,47 @@ static int futex_hash_get_slots(void) return fph->hash_mask + 1; return 0; } +#else /* CONFIG_FUTEX_PRIVATE_HASH */ +static inline int futex_hash_allocate(unsigned int hslots, unsigned int flags) { return -EINVAL; } +static inline int futex_hash_get_slots(void) { return 0; } +static inline void futex_hash_init_mm(struct futex_mm_data *fd) { } +#endif /* !CONFIG_FUTEX_PRIVATE_HASH */ -#else +#ifdef CONFIG_FUTEX_ROBUST_UNLOCK +static void futex_invalidate_cs_ranges(struct futex_mm_data *fd) +{ + /* + * Invalidate start_ip so that the quick check fails for ip >= start_ip + * if VDSO is not mapped or the second slot is not available for compat + * tasks as they use VDSO32 which does not provide the 64-bit pointer + * variant. + */ + for (int i = 0; i < FUTEX_ROBUST_MAX_CS_RANGES; i++) + fd->unlock.cs_ranges[i].start_ip = ~0UL; +} -static int futex_hash_allocate(unsigned int hash_slots, unsigned int flags) +void futex_reset_cs_ranges(struct futex_mm_data *fd) { - return -EINVAL; + memset(fd->unlock.cs_ranges, 0, sizeof(fd->unlock.cs_ranges)); + futex_invalidate_cs_ranges(fd); } -static int futex_hash_get_slots(void) +static void futex_robust_unlock_init_mm(struct futex_mm_data *fd) { - return 0; + /* mm_dup() preserves the range, mm_alloc() clears it */ + if (!fd->unlock.cs_ranges[0].start_ip) + futex_invalidate_cs_ranges(fd); } +#else /* CONFIG_FUTEX_ROBUST_UNLOCK */ +static inline void futex_robust_unlock_init_mm(struct futex_mm_data *fd) { } +#endif /* !CONFIG_FUTEX_ROBUST_UNLOCK */ +#if defined(CONFIG_FUTEX_PRIVATE_HASH) || defined(CONFIG_FUTEX_ROBUST_UNLOCK) +void futex_mm_init(struct mm_struct *mm) +{ + futex_hash_init_mm(&mm->futex); + futex_robust_unlock_init_mm(&mm->futex); +} #endif int futex_hash_prctl(unsigned long arg2, unsigned long arg3, unsigned long arg4) -- cgit v1.2.3 From 7010c39d8fc5063af69ee63f905e592e046f8e5d Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 2 Jun 2026 11:10:04 +0200 Subject: futex: Provide infrastructure to plug the non contended robust futex unlock race MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the FUTEX_ROBUST_UNLOCK mechanism is used for unlocking (PI-)futexes, then the unlock sequence in user space looks like this: 1) robust_list_set_op_pending(mutex); 2) robust_list_remove(mutex); lval = gettid(); 3) if (atomic_try_cmpxchg(&mutex->lock, lval, 0)) 4) robust_list_clear_op_pending(); else 5) sys_futex(OP | FUTEX_ROBUST_UNLOCK, ....); That still leaves a minimal race window between #3 and #4 where the mutex could be acquired by some other task, which observes that it is the last user and: 1) unmaps the mutex memory 2) maps a different file, which ends up covering the same address When then the original task exits before reaching #5 then the kernel robust list handling observes the pending op entry and tries to fix up user space. In case that the newly mapped data contains the TID of the exiting thread at the address of the mutex/futex the kernel will set the owner died bit in that memory and therefore corrupt unrelated data. On X86 this boils down to this simplified assembly sequence: mov %esi,%eax // Load TID into EAX xor %ecx,%ecx // Set ECX to 0 #3 lock cmpxchg %ecx,(%rdi) // Try the TID -> 0 transition .Lstart: jnz .Lend #4 movq %rcx,(%rdx) // Clear list_op_pending .Lend: If the cmpxchg() succeeds and the task is interrupted before it can clear list_op_pending in the robust list head (#4) and the task crashes in a signal handler or gets killed then it ends up in do_exit() and subsequently in the robust list handling, which then might run into the unmap/map issue described above. This is only relevant when user space was interrupted and a signal is pending. The fix-up has to be done before signal delivery is attempted because: 1) The signal might be fatal so get_signal() ends up in do_exit() 2) The signal handler might crash or the task is killed before returning from the handler. At that point the instruction pointer in pt_regs is not longer the instruction pointer of the initially interrupted unlock sequence. The right place to handle this is in __exit_to_user_mode_loop() before invoking arch_do_signal_or_restart() as this covers obviously both scenarios. As this is only relevant when the task was interrupted in user space, this is tied to RSEQ and the generic entry code as RSEQ keeps track of user space interrupts unconditionally even if the task does not have a RSEQ region installed. That makes the decision very lightweight: if (current->rseq.user_irq && within(regs, csr->unlock_ip_range)) futex_fixup_robust_unlock(regs, csr); futex_fixup_robust_unlock() then invokes a architecture specific function to return the pending op pointer or NULL. The function evaluates the register content to decide whether the pending ops pointer in the robust list head needs to be cleared. Assuming the above unlock sequence, then on x86 this decision is the trivial evaluation of the zero flag: return regs->eflags & X86_EFLAGS_ZF ? regs->dx : NULL; Other architectures might need to do more complex evaluations due to LLSC, but the approach is valid in general. The size of the pointer is determined from the matching range struct, which covers both 32-bit and 64-bit builds including COMPAT. The unlock sequence is going to be placed in the VDSO so that the kernel can keep everything synchronized, especially the register usage. The resulting code sequence for user space is: if (__vdso_futex_robust_list$SZ_try_unlock(lock, tid, &pending_op) != tid) err = sys_futex($OP | FUTEX_ROBUST_UNLOCK,....); Both the VDSO unlock and the kernel side unlock ensure that the pending_op pointer is always cleared when the lock becomes unlocked. Signed-off-by: Thomas Gleixner Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: André Almeida Link: https://patch.msgid.link/20260602090535.773669210@kernel.org --- include/linux/futex.h | 39 +++++++++++++++++++++++++++++++++++++- include/vdso/futex.h | 52 +++++++++++++++++++++++++++++++++++++++++++++++++++ kernel/entry/common.c | 9 ++++++--- kernel/futex/core.c | 18 ++++++++++++++++++ 4 files changed, 114 insertions(+), 4 deletions(-) create mode 100644 include/vdso/futex.h (limited to 'include/linux') diff --git a/include/linux/futex.h b/include/linux/futex.h index cb2a182547dd..51f4ccdc9092 100644 --- a/include/linux/futex.h +++ b/include/linux/futex.h @@ -105,7 +105,41 @@ static inline int futex_hash_free(struct mm_struct *mm) { return 0; } #endif /* !CONFIG_FUTEX */ #ifdef CONFIG_FUTEX_ROBUST_UNLOCK +#include + void futex_reset_cs_ranges(struct futex_mm_data *fd); +void __futex_fixup_robust_unlock(struct pt_regs *regs, struct futex_unlock_cs_range *csr); + +static inline bool futex_within_robust_unlock(struct pt_regs *regs, + struct futex_unlock_cs_range *csr) +{ + unsigned long ip = instruction_pointer(regs); + + return ip >= csr->start_ip && ip < csr->start_ip + csr->len; +} + +static inline void futex_fixup_robust_unlock(struct pt_regs *regs) +{ + struct futex_unlock_cs_range *csr; + + /* + * Avoid dereferencing current->mm if not returning from interrupt. + * current->rseq.event is going to be used subsequently, so bringing the + * cache line in is not a big deal. + */ + if (!current->rseq.event.user_irq) + return; + + csr = current->mm->futex.unlock.cs_ranges; + + /* The loop is optimized out for !COMPAT */ + for (int r = 0; r < FUTEX_ROBUST_MAX_CS_RANGES; r++, csr++) { + if (unlikely(futex_within_robust_unlock(regs, csr))) { + __futex_fixup_robust_unlock(regs, csr); + return; + } + } +} static inline void futex_set_vdso_cs_range(struct futex_mm_data *fd, unsigned int idx, unsigned long start, unsigned long end, bool sz32) @@ -114,7 +148,10 @@ static inline void futex_set_vdso_cs_range(struct futex_mm_data *fd, unsigned in fd->unlock.cs_ranges[idx].len = end - start; fd->unlock.cs_ranges[idx].pop_size32 = sz32; } -#endif /* CONFIG_FUTEX_ROBUST_UNLOCK */ +#else /* CONFIG_FUTEX_ROBUST_UNLOCK */ +static inline void futex_fixup_robust_unlock(struct pt_regs *regs) { } +#endif /* !CONFIG_FUTEX_ROBUST_UNLOCK */ + #if defined(CONFIG_FUTEX_PRIVATE_HASH) || defined(CONFIG_FUTEX_ROBUST_UNLOCK) void futex_mm_init(struct mm_struct *mm); diff --git a/include/vdso/futex.h b/include/vdso/futex.h new file mode 100644 index 000000000000..3cd175eefe64 --- /dev/null +++ b/include/vdso/futex.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _VDSO_FUTEX_H +#define _VDSO_FUTEX_H + +#include + +/** + * __vdso_futex_robust_list64_try_unlock - Try to unlock an uncontended robust futex + * with a 64-bit pending op pointer + * @lock: Pointer to the futex lock object + * @tid: The TID of the calling task + * @pop: Pointer to the task's robust_list_head::list_pending_op + * + * Return: The content of *@lock. On success this is the same as @tid. + * + * The function implements: + * if (atomic_try_cmpxchg(lock, &tid, 0)) + * *op = NULL; + * return tid; + * + * There is a race between a successful unlock and clearing the pending op + * pointer in the robust list head. If the calling task is interrupted in the + * race window and has to handle a (fatal) signal on return to user space then + * the kernel handles the clearing of @pending_op before attempting to deliver + * the signal. That ensures that a task cannot exit with a potentially invalid + * pending op pointer. + * + * User space uses it in the following way: + * + * if (__vdso_futex_robust_list64_try_unlock(lock, tid, &pending_op) != tid) + * err = sys_futex($OP | FUTEX_ROBUST_UNLOCK,....); + * + * If the unlock attempt fails due to the FUTEX_WAITERS bit set in the lock, + * then the syscall does the unlock, clears the pending op pointer and wakes the + * requested number of waiters. + */ +__u32 __vdso_futex_robust_list64_try_unlock(__u32 *lock, __u32 tid, __u64 *pop); + +/** + * __vdso_futex_robust_list32_try_unlock - Try to unlock an uncontended robust futex + * with a 32-bit pending op pointer + * @lock: Pointer to the futex lock object + * @tid: The TID of the calling task + * @pop: Pointer to the task's robust_list_head::list_pending_op + * + * Return: The content of *@lock. On success this is the same as @tid. + * + * Same as __vdso_futex_robust_list64_try_unlock() just with a 32-bit @pop pointer. + */ +__u32 __vdso_futex_robust_list32_try_unlock(__u32 *lock, __u32 tid, __u32 *pop); + +#endif diff --git a/kernel/entry/common.c b/kernel/entry/common.c index 19d2244a9fef..e3d381fd3d25 100644 --- a/kernel/entry/common.c +++ b/kernel/entry/common.c @@ -1,11 +1,12 @@ // SPDX-License-Identifier: GPL-2.0 -#include -#include +#include #include +#include #include #include #include +#include #include /* Workaround to allow gradual conversion of architecture code */ @@ -60,8 +61,10 @@ static __always_inline unsigned long __exit_to_user_mode_loop(struct pt_regs *re if (ti_work & _TIF_PATCH_PENDING) klp_update_patch_state(current); - if (ti_work & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) + if (ti_work & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) { + futex_fixup_robust_unlock(regs); arch_do_signal_or_restart(regs); + } if (ti_work & _TIF_NOTIFY_RESUME) resume_user_mode_work(regs); diff --git a/kernel/futex/core.c b/kernel/futex/core.c index aad6e501fabd..6ea4a97796a1 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -46,6 +46,8 @@ #include #include +#include + #include "futex.h" #include "../locking/rtmutex_common.h" @@ -1446,6 +1448,22 @@ bool futex_robust_list_clear_pending(void __user *pop, unsigned int flags) return robust_list_clear_pending(pop); } +#ifdef CONFIG_FUTEX_ROBUST_UNLOCK +void __futex_fixup_robust_unlock(struct pt_regs *regs, struct futex_unlock_cs_range *csr) +{ + /* + * arch_futex_robust_unlock_get_pop() returns the list pending op pointer from + * @regs if the try_cmpxchg() succeeded. + */ + void __user *pop = arch_futex_robust_unlock_get_pop(regs); + + if (!pop) + return; + + futex_robust_list_clear_pending(pop, csr->pop_size32 ? FLAGS_ROBUST_LIST32 : 0); +} +#endif /* CONFIG_FUTEX_ROBUST_UNLOCK */ + static void futex_cleanup(struct task_struct *tsk) { if (unlikely(tsk->futex.robust_list)) { -- cgit v1.2.3 From a40e0f8eadd44d7b0f856b54c876aea1b93415f4 Mon Sep 17 00:00:00 2001 From: Dmitry Ilvokhin Date: Fri, 5 Jun 2026 03:06:22 -0700 Subject: cleanup: Specify nonnull argument index The guard constructors were annotated with an empty __nonnull_args(), relying on __nonnull__() marking every pointer parameter as non-NULL. Sparse cannot parse the empty argument list. Both constructors take the lock pointer as their first parameter, so specify the index explicitly: __nonnull_args(1). Reported-by: Dan Carpenter Closes: https://lore.kernel.org/all/aiJi0WcYE8FZt-jO@stanley.mountain/ Signed-off-by: Dmitry Ilvokhin Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/aiKpH3cLBEj3TF2Q@shell.ilvokhin.com --- include/linux/cleanup.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/cleanup.h b/include/linux/cleanup.h index 65416938e318..b1b5698cbf1b 100644 --- a/include/linux/cleanup.h +++ b/include/linux/cleanup.h @@ -397,7 +397,7 @@ static __maybe_unused const bool class_##_name##_is_conditional = _is_cond __DEFINE_GUARD_LOCK_PTR(_name, _T) #define DEFINE_GUARD(_name, _type, _lock, _unlock) \ - static __always_inline __nonnull_args() _type class_##_name##_constructor(_type _T); \ + static __always_inline __nonnull_args(1) _type class_##_name##_constructor(_type _T); \ DEFINE_CLASS(_name, _type, _unlock, ({ _lock; _T; }), _type _T); \ DEFINE_CLASS_IS_GUARD(_name) @@ -498,7 +498,7 @@ static __always_inline void class_##_name##_destructor(class_##_name##_t *_T) \ __DEFINE_GUARD_LOCK_PTR(_name, &_T->lock) #define __DEFINE_LOCK_GUARD_1(_name, _type, ...) \ -static __always_inline __nonnull_args() \ +static __always_inline __nonnull_args(1) \ class_##_name##_t class_##_name##_constructor(_type *l) \ __no_context_analysis \ { \ -- cgit v1.2.3 From 7cfa62cf9432df67b1c95a59984a03a3bc023f98 Mon Sep 17 00:00:00 2001 From: Dmitry Ilvokhin Date: Thu, 4 Jun 2026 07:15:06 +0000 Subject: locking/percpu-rwsem: Extract __percpu_up_read() Move the percpu_up_read() slowpath out of the inline function into a new __percpu_up_read() to avoid binary size increase from adding a tracepoint to an inlined function. Signed-off-by: Dmitry Ilvokhin Signed-off-by: Peter Zijlstra (Intel) Acked-by: Usama Arif Link: https://patch.msgid.link/3dd2a1b9ab4f469e1892766cb63f41d6b0f53d29.1780506267.git.d@ilvokhin.com --- include/linux/percpu-rwsem.h | 15 +++------------ kernel/locking/percpu-rwsem.c | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/include/linux/percpu-rwsem.h b/include/linux/percpu-rwsem.h index c8cb010d655e..39d5bf8e6562 100644 --- a/include/linux/percpu-rwsem.h +++ b/include/linux/percpu-rwsem.h @@ -107,6 +107,8 @@ static inline bool percpu_down_read_trylock(struct percpu_rw_semaphore *sem) return ret; } +extern void __percpu_up_read(struct percpu_rw_semaphore *sem); + static inline void percpu_up_read(struct percpu_rw_semaphore *sem) { rwsem_release(&sem->dep_map, _RET_IP_); @@ -118,18 +120,7 @@ static inline void percpu_up_read(struct percpu_rw_semaphore *sem) if (likely(rcu_sync_is_idle(&sem->rss))) { this_cpu_dec(*sem->read_count); } else { - /* - * slowpath; reader will only ever wake a single blocked - * writer. - */ - smp_mb(); /* B matches C */ - /* - * In other words, if they see our decrement (presumably to - * aggregate zero, as that is the only time it matters) they - * will also see our critical section. - */ - this_cpu_dec(*sem->read_count); - rcuwait_wake_up(&sem->writer); + __percpu_up_read(sem); } preempt_enable(); } diff --git a/kernel/locking/percpu-rwsem.c b/kernel/locking/percpu-rwsem.c index ef234469baac..f3ee7a0d6047 100644 --- a/kernel/locking/percpu-rwsem.c +++ b/kernel/locking/percpu-rwsem.c @@ -288,3 +288,21 @@ void percpu_up_write(struct percpu_rw_semaphore *sem) rcu_sync_exit(&sem->rss); } EXPORT_SYMBOL_GPL(percpu_up_write); + +void __percpu_up_read(struct percpu_rw_semaphore *sem) +{ + lockdep_assert_preemption_disabled(); + /* + * slowpath; reader will only ever wake a single blocked + * writer. + */ + smp_mb(); /* B matches C */ + /* + * In other words, if they see our decrement (presumably to + * aggregate zero, as that is the only time it matters) they + * will also see our critical section. + */ + this_cpu_dec(*sem->read_count); + rcuwait_wake_up(&sem->writer); +} +EXPORT_SYMBOL_GPL(__percpu_up_read); -- cgit v1.2.3