From 28c75fbfec8f024db1278194918e5f6eda4c570f Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Wed, 11 Feb 2026 14:32:19 -0800 Subject: perf/core: Pass GFP flags to attach_task_ctx_data() This is a preparation for the next change to reduce the computational complexity in the global context data handling for LBR callstacks. Signed-off-by: Namhyung Kim Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260211223222.3119790-2-namhyung@kernel.org --- kernel/events/core.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/events/core.c b/kernel/events/core.c index ac70d68217b6..90b0c9324cbf 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -5370,15 +5370,15 @@ static void unaccount_freq_event(void) static struct perf_ctx_data * -alloc_perf_ctx_data(struct kmem_cache *ctx_cache, bool global) +alloc_perf_ctx_data(struct kmem_cache *ctx_cache, bool global, gfp_t gfp_flags) { struct perf_ctx_data *cd; - cd = kzalloc_obj(*cd); + cd = kzalloc_obj(*cd, gfp_flags); if (!cd) return NULL; - cd->data = kmem_cache_zalloc(ctx_cache, GFP_KERNEL); + cd->data = kmem_cache_zalloc(ctx_cache, gfp_flags); if (!cd->data) { kfree(cd); return NULL; @@ -5412,11 +5412,11 @@ static inline void perf_free_ctx_data_rcu(struct perf_ctx_data *cd) static int attach_task_ctx_data(struct task_struct *task, struct kmem_cache *ctx_cache, - bool global) + bool global, gfp_t gfp_flags) { struct perf_ctx_data *cd, *old = NULL; - cd = alloc_perf_ctx_data(ctx_cache, global); + cd = alloc_perf_ctx_data(ctx_cache, global, gfp_flags); if (!cd) return -ENOMEM; @@ -5499,7 +5499,7 @@ again: return 0; alloc: - ret = attach_task_ctx_data(p, ctx_cache, true); + ret = attach_task_ctx_data(p, ctx_cache, true, GFP_KERNEL); put_task_struct(p); if (ret) { __detach_global_ctx_data(); @@ -5519,7 +5519,7 @@ attach_perf_ctx_data(struct perf_event *event) return -ENOMEM; if (task) - return attach_task_ctx_data(task, ctx_cache, false); + return attach_task_ctx_data(task, ctx_cache, false, GFP_KERNEL); ret = attach_global_ctx_data(ctx_cache); if (ret) @@ -9240,7 +9240,7 @@ perf_event_alloc_task_data(struct task_struct *child, return; attach: - attach_task_ctx_data(child, ctx_cache, true); + attach_task_ctx_data(child, ctx_cache, true, GFP_KERNEL); } void perf_event_fork(struct task_struct *task) -- cgit v1.2.3 From bec2ee2390c95ed0c44494340464e69e79802e4a Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Wed, 11 Feb 2026 14:32:20 -0800 Subject: perf/core: Try to allocate task_ctx_data quickly The attach_global_ctx_data() has O(N^2) algorithm to allocate the context data for each thread. This caused perfomance problems on large systems with O(100k) threads. Because kmalloc(GFP_KERNEL) can go sleep it cannot be called under the RCU lock. So let's try with GFP_NOWAIT first so that it can proceed in normal cases. Signed-off-by: Namhyung Kim Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260211223222.3119790-3-namhyung@kernel.org --- kernel/events/core.c | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'kernel') diff --git a/kernel/events/core.c b/kernel/events/core.c index 90b0c9324cbf..d357714fc02e 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -5489,6 +5489,12 @@ again: cd = NULL; } if (!cd) { + /* + * Try to allocate context quickly before + * traversing the whole thread list again. + */ + if (!attach_task_ctx_data(p, ctx_cache, true, GFP_NOWAIT)) + continue; get_task_struct(p); goto alloc; } -- cgit v1.2.3 From da45c8d5f051434a3c68397e66ae2d3b3c97cdec Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Wed, 11 Feb 2026 14:32:21 -0800 Subject: perf/core: Simplify __detach_global_ctx_data() Like in the attach_global_ctx_data() it has a O(N^2) loop to delete task context data for each thread. But perf_free_ctx_data_rcu() can be called under RCU read lock, so just calls it directly rather than iterating the whole thread list again. Signed-off-by: Namhyung Kim Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260211223222.3119790-4-namhyung@kernel.org --- kernel/events/core.c | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) (limited to 'kernel') diff --git a/kernel/events/core.c b/kernel/events/core.c index d357714fc02e..5eeae8636996 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -5560,22 +5560,15 @@ static void __detach_global_ctx_data(void) struct task_struct *g, *p; struct perf_ctx_data *cd; -again: scoped_guard (rcu) { for_each_process_thread(g, p) { cd = rcu_dereference(p->perf_ctx_data); - if (!cd || !cd->global) - continue; - cd->global = 0; - get_task_struct(p); - goto detach; + if (cd && cd->global) { + cd->global = 0; + detach_task_ctx_data(p); + } } } - return; -detach: - detach_task_ctx_data(p); - put_task_struct(p); - goto again; } static void detach_global_ctx_data(void) -- cgit v1.2.3 From 5a84b600050c5f16b8bba25dd0e7aea845880407 Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Fri, 27 Feb 2026 06:27:44 +0000 Subject: perf/events: Replace READ_ONCE() with standard pgtable accessors Replace raw READ_ONCE() dereferences of pgtable entries with corresponding standard page table accessors pxdp_get() in perf_get_pgtable_size(). These accessors default to READ_ONCE() on platforms that don't override them. So there is no functional change on such platforms. However arm64 platform is being extended to support 128 bit page tables via a new architecture feature i.e FEAT_D128 in which case READ_ONCE() will not provide required single copy atomic access for 128 bit page table entries. Although pxdp_get() accessors can later be overridden on arm64 platform to extend required single copy atomicity support on 128 bit entries. Signed-off-by: Anshuman Khandual Signed-off-by: Peter Zijlstra (Intel) Acked-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260227062744.2215491-1-anshuman.khandual@arm.com --- kernel/events/core.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/events/core.c b/kernel/events/core.c index 5eeae8636996..95d7a3e57268 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -8421,7 +8421,7 @@ static u64 perf_get_pgtable_size(struct mm_struct *mm, unsigned long addr) pte_t *ptep, pte; pgdp = pgd_offset(mm, addr); - pgd = READ_ONCE(*pgdp); + pgd = pgdp_get(pgdp); if (pgd_none(pgd)) return 0; @@ -8429,7 +8429,7 @@ static u64 perf_get_pgtable_size(struct mm_struct *mm, unsigned long addr) return pgd_leaf_size(pgd); p4dp = p4d_offset_lockless(pgdp, pgd, addr); - p4d = READ_ONCE(*p4dp); + p4d = p4dp_get(p4dp); if (!p4d_present(p4d)) return 0; @@ -8437,7 +8437,7 @@ static u64 perf_get_pgtable_size(struct mm_struct *mm, unsigned long addr) return p4d_leaf_size(p4d); pudp = pud_offset_lockless(p4dp, p4d, addr); - pud = READ_ONCE(*pudp); + pud = pudp_get(pudp); if (!pud_present(pud)) return 0; -- cgit v1.2.3