From c4d6d7829817f762dfdce829ffd0c14ea3bad7fe Mon Sep 17 00:00:00 2001 From: Harry Yoo Date: Tue, 10 Feb 2026 13:46:41 +0900 Subject: mm/slab: allow freeing kmalloc_nolock()'d objects using kfree[_rcu]() Slab objects that are allocated with kmalloc_nolock() must be freed using kfree_nolock() because only a subset of alloc hooks are called, since kmalloc_nolock() can't spin on a lock during allocation. This imposes a limitation: such objects cannot be freed with kfree_rcu(), forcing users to work around this limitation by calling call_rcu() with a callback that frees the object using kfree_nolock(). Remove this limitation by teaching kmemleak to gracefully ignore cases when kmemleak_free() or kmemleak_ignore() is called without a prior kmemleak_alloc(). Unlike kmemleak, kfence already handles this case, because, due to its design, only a subset of allocations are served from kfence. With this change, kfree() and kfree_rcu() can be used to free objects that are allocated using kmalloc_nolock(). Suggested-by: Alexei Starovoitov Acked-by: Alexei Starovoitov Reviewed-by: Catalin Marinas Signed-off-by: Harry Yoo Link: https://patch.msgid.link/20260210044642.139482-2-harry.yoo@oracle.com Signed-off-by: Vlastimil Babka --- include/linux/rcupdate.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index c5b30054cd01..72ba681360ad 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -1076,8 +1076,8 @@ static inline void rcu_read_unlock_migrate(void) * either fall back to use of call_rcu() or rearrange the structure to * position the rcu_head structure into the first 4096 bytes. * - * The object to be freed can be allocated either by kmalloc() or - * kmem_cache_alloc(). + * The object to be freed can be allocated either by kmalloc(), + * kmalloc_nolock(), or kmem_cache_alloc(). * * Note that the allowable offset might decrease in the future. * -- cgit v1.2.3 From 27125df9a5d3b4cfd03bce3a8ec405a368cc9aae Mon Sep 17 00:00:00 2001 From: Harry Yoo Date: Tue, 10 Feb 2026 13:46:42 +0900 Subject: mm/slab: drop the OBJEXTS_NOSPIN_ALLOC flag from enum objext_flags OBJEXTS_NOSPIN_ALLOC was used to remember whether a slabobj_ext vector was allocated via kmalloc_nolock(), so that free_slab_obj_exts() could call kfree_nolock() instead of kfree(). Now that kfree() supports freeing kmalloc_nolock() objects, this flag is no longer needed. Instead, pass the allow_spin parameter down to free_slab_obj_exts() to determine whether kfree_nolock() or kfree() should be called in the free path, and free one bit in enum objext_flags. Acked-by: Alexei Starovoitov Signed-off-by: Harry Yoo Reviewed-by: Hao Li Link: https://patch.msgid.link/20260210044642.139482-3-harry.yoo@oracle.com Signed-off-by: Vlastimil Babka --- include/linux/memcontrol.h | 3 +-- mm/slub.c | 18 ++++++++---------- 2 files changed, 9 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 0651865a4564..bb789ec4a2a2 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -359,8 +359,7 @@ enum objext_flags { * MEMCG_DATA_OBJEXTS. */ OBJEXTS_ALLOC_FAIL = __OBJEXTS_ALLOC_FAIL, - /* slabobj_ext vector allocated with kmalloc_nolock() */ - OBJEXTS_NOSPIN_ALLOC = __FIRST_OBJEXT_FLAG, + __OBJEXTS_FLAG_UNUSED = __FIRST_OBJEXT_FLAG, /* the next bit after the last actual flag */ __NR_OBJEXTS_FLAGS = (__FIRST_OBJEXT_FLAG << 1), }; diff --git a/mm/slub.c b/mm/slub.c index 3f64a6b94571..18c30872d196 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -2190,8 +2190,6 @@ int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, virt_to_slab(vec)->slab_cache == s); new_exts = (unsigned long)vec; - if (unlikely(!allow_spin)) - new_exts |= OBJEXTS_NOSPIN_ALLOC; #ifdef CONFIG_MEMCG new_exts |= MEMCG_DATA_OBJEXTS; #endif @@ -2229,7 +2227,7 @@ retry: return 0; } -static inline void free_slab_obj_exts(struct slab *slab) +static inline void free_slab_obj_exts(struct slab *slab, bool allow_spin) { struct slabobj_ext *obj_exts; @@ -2257,10 +2255,10 @@ static inline void free_slab_obj_exts(struct slab *slab) * the extension for obj_exts is expected to be NULL. */ mark_objexts_empty(obj_exts); - if (unlikely(READ_ONCE(slab->obj_exts) & OBJEXTS_NOSPIN_ALLOC)) - kfree_nolock(obj_exts); - else + if (allow_spin) kfree(obj_exts); + else + kfree_nolock(obj_exts); slab->obj_exts = 0; } @@ -2324,7 +2322,7 @@ static int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, return 0; } -static inline void free_slab_obj_exts(struct slab *slab) +static inline void free_slab_obj_exts(struct slab *slab, bool allow_spin) { } @@ -3404,14 +3402,14 @@ static __always_inline void account_slab(struct slab *slab, int order, } static __always_inline void unaccount_slab(struct slab *slab, int order, - struct kmem_cache *s) + struct kmem_cache *s, bool allow_spin) { /* * The slab object extensions should now be freed regardless of * whether mem_alloc_profiling_enabled() or not because profiling * might have been disabled after slab->obj_exts got allocated. */ - free_slab_obj_exts(slab); + free_slab_obj_exts(slab, allow_spin); mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s), -(PAGE_SIZE << order)); @@ -3515,7 +3513,7 @@ static void __free_slab(struct kmem_cache *s, struct slab *slab, bool allow_spin page->mapping = NULL; __ClearPageSlab(page); mm_account_reclaimed_pages(pages); - unaccount_slab(slab, order, s); + unaccount_slab(slab, order, s, allow_spin); if (allow_spin) free_frozen_pages(page, order); else -- cgit v1.2.3