From aa61e8b4fb2c9c5315228d0c830b5b2ae4df5cca Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 7 May 2026 09:57:15 -0700 Subject: rcutorture: Fully test lazy RCU Currently, rcutorture bypasses lazy RCU by using call_rcu_hurry(). This works, avoiding the dreaded rtort_pipe_count WARN(), but fails to fully test lazy RCU. The rtort_pipe_count WARN() splats because lazy RCU could delay the start of an RCU grace period for a full stutter period, which defaults to only three seconds. This commit therefore reverts the call_rcu_hurry() instances back to call_rcu(), but, in kernels built with CONFIG_RCU_LAZY=y, queues a workqueue handler just before the call to stutter_wait() in rcu_torture_writer(). This workqueue handler invokes rcu_barrier(), which motivates any lingering lazy callbacks, thus avoiding the splat. Reported-by: Saravana Kannan Signed-off-by: Paul E. McKenney Signed-off-by: Uladzislau Rezki (Sony) --- kernel/rcu/rcutorture.c | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) (limited to 'kernel/rcu') diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 5f2848b828dc..882a158ada7b 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -572,7 +572,7 @@ static unsigned long rcu_no_completed(void) static void rcu_torture_deferred_free(struct rcu_torture *p) { - call_rcu_hurry(&p->rtort_rcu, rcu_torture_cb); + call_rcu(&p->rtort_rcu, rcu_torture_cb); } static void rcu_sync_torture_init(void) @@ -619,7 +619,7 @@ static struct rcu_torture_ops rcu_ops = { .poll_gp_state_exp = poll_state_synchronize_rcu, .cond_sync_exp = cond_synchronize_rcu_expedited, .cond_sync_exp_full = cond_synchronize_rcu_expedited_full, - .call = call_rcu_hurry, + .call = call_rcu, .cb_barrier = rcu_barrier, .fqs = rcu_force_quiescent_state, .gp_kthread_dbg = show_rcu_gp_kthreads, @@ -1145,7 +1145,7 @@ static void rcu_tasks_torture_deferred_free(struct rcu_torture *p) static void synchronize_rcu_mult_test(void) { - synchronize_rcu_mult(call_rcu_tasks, call_rcu_hurry); + synchronize_rcu_mult(call_rcu_tasks, call_rcu); } static struct rcu_torture_ops tasks_ops = { @@ -1631,6 +1631,17 @@ static void do_rtws_sync(struct torture_random_state *trsp, void (*sync)(void)) cpus_read_unlock(); } +/* + * Do an rcu_barrier() to motivate lazy callbacks during a stutter + * pause. Without this, we can get false-positives rtort_pipe_count + * splats. + */ +static void rcu_torture_writer_work(struct work_struct *work) +{ + if (cur_ops->cb_barrier) + cur_ops->cb_barrier(); +} + /* * RCU torture writer kthread. Repeatedly substitutes a new structure * for that pointed to by rcu_torture_current, freeing the old structure @@ -1651,6 +1662,7 @@ rcu_torture_writer(void *arg) int i; int idx; unsigned long j; + struct work_struct lazy_work; int oldnice = task_nice(current); struct rcu_gp_oldstate *rgo = NULL; int rgo_size = 0; @@ -1703,6 +1715,9 @@ rcu_torture_writer(void *arg) pr_alert("%s" TORTURE_FLAG " Waited %lu jiffies for boot to complete.\n", torture_type, jiffies - j); + if (IS_ENABLED(CONFIG_RCU_LAZY)) + INIT_WORK_ONSTACK(&lazy_work, rcu_torture_writer_work); + do { rcu_torture_writer_state = RTWS_FIXED_DELAY; torture_hrtimeout_us(500, 1000, &rand); @@ -1895,6 +1910,8 @@ rcu_torture_writer(void *arg) !rcu_gp_is_normal(); } rcu_torture_writer_state = RTWS_STUTTER; + if (IS_ENABLED(CONFIG_RCU_LAZY)) + queue_work(system_percpu_wq, &lazy_work); stutter_waited = stutter_wait("rcu_torture_writer"); if (stutter_waited && !atomic_read(&rcu_fwd_cb_nodelay) && @@ -1925,6 +1942,12 @@ rcu_torture_writer(void *arg) pr_alert("%s" TORTURE_FLAG " Dynamic grace-period expediting was disabled.\n", torture_type); + + if (IS_ENABLED(CONFIG_RCU_LAZY)) { + cancel_work_sync(&lazy_work); + destroy_work_on_stack(&lazy_work); + } + kfree(ulo); kfree(rgo); rcu_torture_writer_state = RTWS_STOPPING; -- cgit v1.2.3 From 5f136351edd37d779e45704d573f5b6d6cab1e6e Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 8 May 2026 10:43:51 -0700 Subject: rcu: Simplify rcu_do_batch() by applying clamp() This commit replaces a nested ?: sequence with clamp(). This does not reduce the number of lines of code, but it does simplify the line that it modifies. Reviewed-by: Frederic Weisbecker Signed-off-by: Paul E. McKenney Signed-off-by: Uladzislau Rezki (Sony) --- kernel/rcu/tree.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel/rcu') diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 55df6d37145e..e46a5124c3eb 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -2584,7 +2584,7 @@ static void rcu_do_batch(struct rcu_data *rdp) const long npj = NSEC_PER_SEC / HZ; long rrn = READ_ONCE(rcu_resched_ns); - rrn = rrn < NSEC_PER_MSEC ? NSEC_PER_MSEC : rrn > NSEC_PER_SEC ? NSEC_PER_SEC : rrn; + rrn = clamp(rrn, NSEC_PER_MSEC, NSEC_PER_SEC); tlimit = local_clock() + rrn; jlimit = jiffies + (rrn + npj + 1) / npj; jlimit_check = true; -- cgit v1.2.3 From d9b4d36b8c8fb6006e7b62dd0ddc218c87e8196a Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 8 May 2026 10:43:52 -0700 Subject: rcu: Simplify param_set_next_fqs_jiffies() by applying clamp_val() This commit replaces a nested ?: sequence with clamp_val(). This does not reduce the number of lines of code, but it does simplify the line that it modifies. Reviewed-by: Frederic Weisbecker Signed-off-by: Paul E. McKenney Signed-off-by: Uladzislau Rezki (Sony) --- kernel/rcu/tree.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel/rcu') diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index e46a5124c3eb..09f0cef5014c 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -492,7 +492,7 @@ static int param_set_next_fqs_jiffies(const char *val, const struct kernel_param int ret = kstrtoul(val, 0, &j); if (!ret) { - WRITE_ONCE(*(ulong *)kp->arg, (j > HZ) ? HZ : (j ?: 1)); + WRITE_ONCE(*(ulong *)kp->arg, clamp_val(j, 1, HZ)); adjust_jiffies_till_sched_qs(); } return ret; -- cgit v1.2.3 From eceb65975256d7f7a5e5a5a2f4b7865eb32eaaf7 Mon Sep 17 00:00:00 2001 From: "Uladzislau Rezki (Sony)" Date: Wed, 11 Mar 2026 19:58:11 +0100 Subject: rcu: Latch normal synchronize_rcu() path on flood Currently, rcu_normal_wake_from_gp is only enabled by default on small systems(<= 16 CPUs) or when a user explicitly set it enabled. Introduce an adaptive latching mechanism: * Track the number of in-flight synchronize_rcu() requests using a new rcu_sr_normal_count counter; * If the count reaches/exceeds RCU_SR_NORMAL_LATCH_THR(64), it sets the rcu_sr_normal_latched, reverting new requests onto the scaled wait_rcu_gp() path; * The latch is cleared only when the pending requests are fully drained(nr == 0); * Enables rcu_normal_wake_from_gp by default for all systems, relying on this dynamic throttling instead of static CPU limits. Testing(synthetic flood workload): * Kernel version: 6.19.0-rc6 * Number of CPUs: 1536 * 60K concurrent synchronize_rcu() calls Perf(cycles, system-wide): total cycles: 932020263832 rcu_sr_normal_add_req(): 2650282811 cycles(~0.28%) Perf report excerpt: 0.01% 0.01% sync_test/... [k] rcu_sr_normal_add_req Measured overhead of rcu_sr_normal_add_req() remained ~0.28% of total CPU cycles in this synthetic stress test. Reviewed-by: Frederic Weisbecker Tested-by: Samir M Suggested-by: Joel Fernandes Signed-off-by: Uladzislau Rezki (Sony) --- Documentation/admin-guide/kernel-parameters.txt | 10 ++--- kernel/rcu/tree.c | 52 ++++++++++++++++++------- 2 files changed, 44 insertions(+), 18 deletions(-) (limited to 'kernel/rcu') diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 4d0f545fb3ec..d5db2e85d551 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -5862,13 +5862,13 @@ Kernel parameters use a call_rcu[_hurry]() path. Please note, this is for a normal grace period. - How to enable it: + How to disable it: - echo 1 > /sys/module/rcutree/parameters/rcu_normal_wake_from_gp - or pass a boot parameter "rcutree.rcu_normal_wake_from_gp=1" + echo 0 > /sys/module/rcutree/parameters/rcu_normal_wake_from_gp + or pass a boot parameter "rcutree.rcu_normal_wake_from_gp=0" - Default is 1 if num_possible_cpus() <= 16 and it is not explicitly - disabled by the boot parameter passing 0. + Default is 1 if it is not explicitly disabled by the boot parameter + passing 0. rcuscale.gp_async= [KNL] Measure performance of asynchronous diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 09f0cef5014c..afb9e7db8f78 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -1632,17 +1632,21 @@ static void rcu_sr_put_wait_head(struct llist_node *node) atomic_set_release(&sr_wn->inuse, 0); } -/* Enable rcu_normal_wake_from_gp automatically on small systems. */ -#define WAKE_FROM_GP_CPU_THRESHOLD 16 - -static int rcu_normal_wake_from_gp = -1; +static int rcu_normal_wake_from_gp = 1; module_param(rcu_normal_wake_from_gp, int, 0644); static struct workqueue_struct *sync_wq; +#define RCU_SR_NORMAL_LATCH_THR 64 + +/* Number of in-flight synchronize_rcu() calls queued on srs_next. */ +static atomic_long_t rcu_sr_normal_count; +static int rcu_sr_normal_latched; /* 0/1 */ + static void rcu_sr_normal_complete(struct llist_node *node) { struct rcu_synchronize *rs = container_of( (struct rcu_head *) node, struct rcu_synchronize, head); + long nr; WARN_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && !poll_state_synchronize_rcu_full(&rs->oldstate), @@ -1650,6 +1654,15 @@ static void rcu_sr_normal_complete(struct llist_node *node) /* Finally. */ complete(&rs->completion); + nr = atomic_long_dec_return(&rcu_sr_normal_count); + WARN_ON_ONCE(nr < 0); + + /* + * Unlatch: switch back to normal path when fully + * drained and if it has been latched. + */ + if (nr == 0) + (void)cmpxchg_relaxed(&rcu_sr_normal_latched, 1, 0); } static void rcu_sr_normal_gp_cleanup_work(struct work_struct *work) @@ -1795,6 +1808,24 @@ static bool rcu_sr_normal_gp_init(void) static void rcu_sr_normal_add_req(struct rcu_synchronize *rs) { + /* + * Increment before publish to avoid a complete + * vs enqueue race on latch. + */ + long nr = atomic_long_inc_return(&rcu_sr_normal_count); + + /* + * Latch when threshold is reached. Checking for an exact match + * restricts cmpxchg() to a single context. + * + * This latch is intentionally relaxed and best-effort. Concurrent + * set/clear can race and temporarily lose the latch, which is OK + * because it only selects between the fast and fallback paths. + */ + if (nr == RCU_SR_NORMAL_LATCH_THR) + (void)cmpxchg_relaxed(&rcu_sr_normal_latched, 0, 1); + + /* Publish for the GP kthread/worker. */ llist_add((struct llist_node *) &rs->head, &rcu_state.srs_next); } @@ -3278,14 +3309,15 @@ static void synchronize_rcu_normal(void) { struct rcu_synchronize rs; + init_rcu_head_on_stack(&rs.head); trace_rcu_sr_normal(rcu_state.name, &rs.head, TPS("request")); - if (READ_ONCE(rcu_normal_wake_from_gp) < 1) { + if (READ_ONCE(rcu_normal_wake_from_gp) < 1 || + READ_ONCE(rcu_sr_normal_latched)) { wait_rcu_gp(call_rcu_hurry); goto trace_complete_out; } - init_rcu_head_on_stack(&rs.head); init_completion(&rs.completion); /* @@ -3302,10 +3334,10 @@ static void synchronize_rcu_normal(void) /* Now we can wait. */ wait_for_completion(&rs.completion); - destroy_rcu_head_on_stack(&rs.head); trace_complete_out: trace_rcu_sr_normal(rcu_state.name, &rs.head, TPS("complete")); + destroy_rcu_head_on_stack(&rs.head); } /** @@ -4904,12 +4936,6 @@ void __init rcu_init(void) sync_wq = alloc_workqueue("sync_wq", WQ_MEM_RECLAIM | WQ_UNBOUND, 0); WARN_ON(!sync_wq); - /* Respect if explicitly disabled via a boot parameter. */ - if (rcu_normal_wake_from_gp < 0) { - if (num_possible_cpus() <= WAKE_FROM_GP_CPU_THRESHOLD) - rcu_normal_wake_from_gp = 1; - } - /* Fill in default value for rcutree.qovld boot parameter. */ /* -After- the rcu_node ->lock fields are initialized! */ if (qovld < 0) -- cgit v1.2.3 From 42c5468f9cdc0c892fec3c0916b3ac5b670775af Mon Sep 17 00:00:00 2001 From: Zqiang Date: Thu, 23 Apr 2026 19:19:30 +0800 Subject: rcu-tasks: Fix possible boot-time tests failed for the call_rcu_tasks() The following scenarios will cause the call_rcu_tasks() boot-time tests failed: CPU0 CPU1 rcu_init_tasks_generic() ->rcu_tasks_initiate_self_tests() ->call_rcu_tasks_trace(&tests[1].rh, test_rcu_tasks_callback) ->call_rcu_tasks_generic() ->havekthread = smp_load_acquire(&rtp->kthread_ptr) "The havekthread is false" .... rcu_tasks_kthread() ->smp_store_release(&rtp->kthread_ptr, current) ->rcu_tasks_one_gp() ->rcuwait_wait_event() ->rcu_tasks_need_gpcb() ->for (cpu = 0; cpu < dequeue_limit; cpu++) ->rcu_segcblist_n_cbs(&rtpcp->cblist) == 0 ->schedule() ->raw_spin_trylock_rcu_node() ->needwake = (func == wakeme_after_rcu) || (rcu_segcblist_n_cbs(&rtpcp->cblist) == rcu_task_lazy_lim) "the rcu_task_lazy_lim default value is 32, and the func pointer is test_rcu_tasks_callback, lead to needwake is false." ->if (havekthread && !needwake && !timer_pending(&rtpcp->lazy_timer)) "the havekthread is false, will not enter here." .... "the needwake is false lead to rtp_irq_work can not queue, even if the rtp->kthread_ptr already exists at this point." ->if (needwake && READ_ONCE(rtp->kthread_ptr)) ->irq_work_queue(&rtpcp->rtp_irq_work) For the above scenarios, if the call_rcu_tasks() is not called again afterward, the rcu_tasks_kthread will not have a chance to be wakeup, the test_rcu_tasks_callback() will never be called, the boot-time tests failed can happen, this commit therefore check havekthread variable, if it's false and the rtpcp->cblist is empty, set needwake variable is true, if the rtp->kthread_ptr exist, the rtpcp->rtp_irq_work can be queued to wakeup rcu_tasks_kthread. Signed-off-by: Zqiang Signed-off-by: Uladzislau Rezki (Sony) --- kernel/rcu/tasks.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'kernel/rcu') diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 48f0d803c8e2..f4da5fad70f5 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -373,7 +373,8 @@ static void call_rcu_tasks_generic(struct rcu_head *rhp, rcu_callback_t func, // Queuing callbacks before initialization not yet supported. if (WARN_ON_ONCE(!rcu_segcblist_is_enabled(&rtpcp->cblist))) rcu_segcblist_init(&rtpcp->cblist); - needwake = (func == wakeme_after_rcu) || + needwake = (!havekthread && rcu_segcblist_empty(&rtpcp->cblist)) || + (func == wakeme_after_rcu) || (rcu_segcblist_n_cbs(&rtpcp->cblist) == rcu_task_lazy_lim); if (havekthread && !needwake && !timer_pending(&rtpcp->lazy_timer)) { if (rtp->lazy_jiffies) -- cgit v1.2.3 From 002668809b068c528838c1ab1ff46c87bdbb095d Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 19 May 2026 21:01:28 +0200 Subject: rcu/nocb: reduce stack usage in nocb_gp_wait() When CONFIG_UBSAN_ALIGNMENT is enabled, the stack usage of nocb_gp_wait() grows above typical warning limits: In file included from kernel/rcu/tree.c:4930: kernel/rcu/tree_nocb.h: In function 'rcu_nocb_gp_kthread': kernel/rcu/tree_nocb.h:866:1: error: the frame size of 1968 bytes is larger than 1280 bytes [-Werror=frame-larger-than=] Apparently, the problem is passing rcu_data from a 'void *' pointer, which gcc assumes may be misaligned. When the function is not inlined into rcu_nocb_gp_kthread(), that is no longer visible to gcc. Add a 'noinline_for_stack' annotation that leads to skipping a lot of the alignment sanitizer checks and keeps the stack usage 60% lower here. Reviewed-by: Kunwu Chan Reviewed-by: Frederic Weisbecker Reviewed-by: Paul E. McKenney Signed-off-by: Arnd Bergmann Signed-off-by: Uladzislau Rezki (Sony) --- kernel/rcu/tree_nocb.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel/rcu') diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h index 1047b30cd46b..373b877cf171 100644 --- a/kernel/rcu/tree_nocb.h +++ b/kernel/rcu/tree_nocb.h @@ -655,7 +655,7 @@ static void nocb_gp_sleep(struct rcu_data *my_rdp, int cpu) * No-CBs GP kthreads come here to wait for additional callbacks to show up * or for grace periods to end. */ -static void nocb_gp_wait(struct rcu_data *my_rdp) +static noinline_for_stack void nocb_gp_wait(struct rcu_data *my_rdp) { bool bypass = false; int __maybe_unused cpu = my_rdp->cpu; -- cgit v1.2.3 From 080b5c6d95034e46f5ed1abe98c06218a1386aef Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Fri, 8 May 2026 15:16:35 +0200 Subject: sched/cputime: Remove superfluous and error prone kcpustat_field() parameter The first parameter to kcpustat_field() is a pointer to the cpu kcpustat to be fetched from. This parameter is error prone because a copy to a kcpustat could be passed by accident instead of the original one. Also the kcpustat structure can already be retrieved with the help of the mandatory CPU argument. Remove the needless parameter. Signed-off-by: Frederic Weisbecker Signed-off-by: Thomas Gleixner Tested-by: Shrikanth Hegde Reviewed-by: Shrikanth Hegde Link: https://patch.msgid.link/20260508131647.43868-4-frederic@kernel.org --- drivers/cpufreq/cpufreq_governor.c | 6 +++--- drivers/macintosh/rack-meter.c | 2 +- include/linux/kernel_stat.h | 8 +++----- kernel/rcu/tree.c | 9 +++------ kernel/rcu/tree_stall.h | 7 +++---- kernel/sched/cputime.c | 5 ++--- 6 files changed, 15 insertions(+), 22 deletions(-) (limited to 'kernel/rcu') diff --git a/drivers/cpufreq/cpufreq_governor.c b/drivers/cpufreq/cpufreq_governor.c index 86f35e451914..3c4a1f9af3ae 100644 --- a/drivers/cpufreq/cpufreq_governor.c +++ b/drivers/cpufreq/cpufreq_governor.c @@ -105,7 +105,7 @@ void gov_update_cpu_data(struct dbs_data *dbs_data) j_cdbs->prev_cpu_idle = get_cpu_idle_time(j, &j_cdbs->prev_update_time, dbs_data->io_is_busy); if (dbs_data->ignore_nice_load) - j_cdbs->prev_cpu_nice = kcpustat_field(&kcpustat_cpu(j), CPUTIME_NICE, j); + j_cdbs->prev_cpu_nice = kcpustat_field(CPUTIME_NICE, j); } } } @@ -165,7 +165,7 @@ unsigned int dbs_update(struct cpufreq_policy *policy) j_cdbs->prev_cpu_idle = cur_idle_time; if (ignore_nice) { - u64 cur_nice = kcpustat_field(&kcpustat_cpu(j), CPUTIME_NICE, j); + u64 cur_nice = kcpustat_field(CPUTIME_NICE, j); idle_time += div_u64(cur_nice - j_cdbs->prev_cpu_nice, NSEC_PER_USEC); j_cdbs->prev_cpu_nice = cur_nice; @@ -539,7 +539,7 @@ int cpufreq_dbs_governor_start(struct cpufreq_policy *policy) j_cdbs->prev_load = 0; if (ignore_nice) - j_cdbs->prev_cpu_nice = kcpustat_field(&kcpustat_cpu(j), CPUTIME_NICE, j); + j_cdbs->prev_cpu_nice = kcpustat_field(CPUTIME_NICE, j); } gov->start(policy); diff --git a/drivers/macintosh/rack-meter.c b/drivers/macintosh/rack-meter.c index 8a1e2c08b096..26cb93191ede 100644 --- a/drivers/macintosh/rack-meter.c +++ b/drivers/macintosh/rack-meter.c @@ -87,7 +87,7 @@ static inline u64 get_cpu_idle_time(unsigned int cpu) kcpustat->cpustat[CPUTIME_IOWAIT]; if (rackmeter_ignore_nice) - retval += kcpustat_field(kcpustat, CPUTIME_NICE, cpu); + retval += kcpustat_field(CPUTIME_NICE, cpu); return retval; } diff --git a/include/linux/kernel_stat.h b/include/linux/kernel_stat.h index b97ce2df376f..dd020ecaf67b 100644 --- a/include/linux/kernel_stat.h +++ b/include/linux/kernel_stat.h @@ -100,14 +100,12 @@ static inline unsigned long kstat_cpu_irqs_sum(unsigned int cpu) } #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN -extern u64 kcpustat_field(struct kernel_cpustat *kcpustat, - enum cpu_usage_stat usage, int cpu); +extern u64 kcpustat_field(enum cpu_usage_stat usage, int cpu); extern void kcpustat_cpu_fetch(struct kernel_cpustat *dst, int cpu); #else -static inline u64 kcpustat_field(struct kernel_cpustat *kcpustat, - enum cpu_usage_stat usage, int cpu) +static inline u64 kcpustat_field(enum cpu_usage_stat usage, int cpu) { - return kcpustat->cpustat[usage]; + return kcpustat_cpu(cpu).cpustat[usage]; } static inline void kcpustat_cpu_fetch(struct kernel_cpustat *dst, int cpu) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 55df6d37145e..3cbf79bee976 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -969,14 +969,11 @@ static int rcu_watching_snap_recheck(struct rcu_data *rdp) if (rcu_cpu_stall_cputime && rdp->snap_record.gp_seq != rdp->gp_seq) { int cpu = rdp->cpu; struct rcu_snap_record *rsrp; - struct kernel_cpustat *kcsp; - - kcsp = &kcpustat_cpu(cpu); rsrp = &rdp->snap_record; - rsrp->cputime_irq = kcpustat_field(kcsp, CPUTIME_IRQ, cpu); - rsrp->cputime_softirq = kcpustat_field(kcsp, CPUTIME_SOFTIRQ, cpu); - rsrp->cputime_system = kcpustat_field(kcsp, CPUTIME_SYSTEM, cpu); + rsrp->cputime_irq = kcpustat_field(CPUTIME_IRQ, cpu); + rsrp->cputime_softirq = kcpustat_field(CPUTIME_SOFTIRQ, cpu); + rsrp->cputime_system = kcpustat_field(CPUTIME_SYSTEM, cpu); rsrp->nr_hardirqs = kstat_cpu_irqs_sum(cpu) + arch_irq_stat_cpu(cpu); rsrp->nr_softirqs = kstat_cpu_softirqs_sum(cpu); rsrp->nr_csw = nr_context_switches_cpu(cpu); diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h index b67532cb8770..cf7ae51cba40 100644 --- a/kernel/rcu/tree_stall.h +++ b/kernel/rcu/tree_stall.h @@ -479,7 +479,6 @@ static void print_cpu_stat_info(int cpu) { struct rcu_snap_record rsr, *rsrp; struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); - struct kernel_cpustat *kcsp = &kcpustat_cpu(cpu); if (!rcu_cpu_stall_cputime) return; @@ -488,9 +487,9 @@ static void print_cpu_stat_info(int cpu) if (rsrp->gp_seq != rdp->gp_seq) return; - rsr.cputime_irq = kcpustat_field(kcsp, CPUTIME_IRQ, cpu); - rsr.cputime_softirq = kcpustat_field(kcsp, CPUTIME_SOFTIRQ, cpu); - rsr.cputime_system = kcpustat_field(kcsp, CPUTIME_SYSTEM, cpu); + rsr.cputime_irq = kcpustat_field(CPUTIME_IRQ, cpu); + rsr.cputime_softirq = kcpustat_field(CPUTIME_SOFTIRQ, cpu); + rsr.cputime_system = kcpustat_field(CPUTIME_SYSTEM, cpu); pr_err("\t hardirqs softirqs csw/system\n"); pr_err("\t number: %8lld %10d %12lld\n", diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index fbf31db0d2f3..caaaf0a04ced 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -975,10 +975,9 @@ static int kcpustat_field_vtime(u64 *cpustat, return 0; } -u64 kcpustat_field(struct kernel_cpustat *kcpustat, - enum cpu_usage_stat usage, int cpu) +u64 kcpustat_field(enum cpu_usage_stat usage, int cpu) { - u64 *cpustat = kcpustat->cpustat; + u64 *cpustat = kcpustat_cpu(cpu).cpustat; u64 val = cpustat[usage]; struct rq *rq; int err; -- cgit v1.2.3