From 88e0b3bb9930605ad7e5d3ddac7d9875325bebf0 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Mon, 3 Aug 2026 20:17:27 +0100 Subject: sched: dynamic: Simplify {cond,might}_resched() PREEMPT_DYNAMIC is now limited to the FULL and LAZY preemption models. In either model, both cond_resched() and might_resched() are always disabled and do nothing. Remove the unnecessary code for these when PREEMPT_DYNAMIC is selected. Signed-off-by: Mark Rutland Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Shrikanth Hegde Reviewed-by: Jinjie Ruan Tested-by: Mete Durlu Tested-by: Shrikanth Hegde Tested-by: Jinjie Ruan Link: https://patch.msgid.link/20260803191731.3244294-3-mark.rutland@arm.com --- include/linux/kernel.h | 20 -------------------- include/linux/sched.h | 31 +++---------------------------- 2 files changed, 3 insertions(+), 48 deletions(-) (limited to 'include/linux') diff --git a/include/linux/kernel.h b/include/linux/kernel.h index 24414c79e59a..4b11d1dc0a67 100644 --- a/include/linux/kernel.h +++ b/include/linux/kernel.h @@ -43,30 +43,10 @@ struct completion; struct user; #ifdef CONFIG_PREEMPT_VOLUNTARY_BUILD - extern int __cond_resched(void); # define might_resched() __cond_resched() - -#elif defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) - -extern int __cond_resched(void); - -DECLARE_STATIC_CALL(might_resched, __cond_resched); - -static __always_inline void might_resched(void) -{ - static_call_mod(might_resched)(); -} - -#elif defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) - -extern int dynamic_might_resched(void); -# define might_resched() dynamic_might_resched() - #else - # define might_resched() do { } while (0) - #endif /* CONFIG_PREEMPT_* */ #ifdef CONFIG_DEBUG_ATOMIC_SLEEP diff --git a/include/linux/sched.h b/include/linux/sched.h index 8b3d47a325cc..d7cc77181ef9 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -2134,44 +2134,19 @@ static inline void set_need_resched_current(void) * value indicates whether a reschedule was done in fact. * cond_resched_lock() will drop the spinlock before scheduling, */ -#if !defined(CONFIG_PREEMPTION) || defined(CONFIG_PREEMPT_DYNAMIC) +#if !defined(CONFIG_PREEMPTION) extern int __cond_resched(void); -#if defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) - -DECLARE_STATIC_CALL(cond_resched, __cond_resched); - -static __always_inline int _cond_resched(void) -{ - return static_call_mod(cond_resched)(); -} - -#elif defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) - -extern int dynamic_cond_resched(void); - -static __always_inline int _cond_resched(void) -{ - return dynamic_cond_resched(); -} - -#else /* !CONFIG_PREEMPTION */ - static inline int _cond_resched(void) { return __cond_resched(); } - -#endif /* PREEMPT_DYNAMIC && CONFIG_HAVE_PREEMPT_DYNAMIC_CALL */ - -#else /* CONFIG_PREEMPTION && !CONFIG_PREEMPT_DYNAMIC */ - +#else static inline int _cond_resched(void) { return 0; } - -#endif /* !CONFIG_PREEMPTION || CONFIG_PREEMPT_DYNAMIC */ +#endif #define cond_resched() ({ \ __might_resched(__FILE__, __LINE__, 0); \ -- cgit v1.2.3 From aa4178f63847f86e3b27b9e53ee8ccbd6ed6bf8c Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Mon, 3 Aug 2026 20:17:29 +0100 Subject: sched: dynamic: Simplify irqentry_exit_cond_resched() PREEMPT_DYNAMIC is now limited to the FULL and LAZY preemption models. In either model, irqentry_exit_cond_resched() is always called and never disabled. Remove the unnecessary code for this when PREEMPT_DYNAMIC is selected. Signed-off-by: Mark Rutland Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Shrikanth Hegde Reviewed-by: Jinjie Ruan Tested-by: Mete Durlu Tested-by: Shrikanth Hegde Tested-by: Jinjie Ruan Link: https://patch.msgid.link/20260803191731.3244294-5-mark.rutland@arm.com --- include/linux/irq-entry-common.h | 17 +---------------- kernel/entry/common.c | 17 ++--------------- kernel/sched/core.c | 7 ------- 3 files changed, 3 insertions(+), 38 deletions(-) (limited to 'include/linux') diff --git a/include/linux/irq-entry-common.h b/include/linux/irq-entry-common.h index 0bb6c03481fa..2be273bb68f0 100644 --- a/include/linux/irq-entry-common.h +++ b/include/linux/irq-entry-common.h @@ -346,22 +346,7 @@ typedef struct irqentry_state { * * Conditional reschedule with additional sanity checks. */ -void raw_irqentry_exit_cond_resched(void); - -#ifdef CONFIG_PREEMPT_DYNAMIC -#if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) -#define irqentry_exit_cond_resched_dynamic_enabled raw_irqentry_exit_cond_resched -#define irqentry_exit_cond_resched_dynamic_disabled NULL -DECLARE_STATIC_CALL(irqentry_exit_cond_resched, raw_irqentry_exit_cond_resched); -#define irqentry_exit_cond_resched() static_call(irqentry_exit_cond_resched)() -#elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) -DECLARE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched); -void dynamic_irqentry_exit_cond_resched(void); -#define irqentry_exit_cond_resched() dynamic_irqentry_exit_cond_resched() -#endif -#else /* CONFIG_PREEMPT_DYNAMIC */ -#define irqentry_exit_cond_resched() raw_irqentry_exit_cond_resched() -#endif /* CONFIG_PREEMPT_DYNAMIC */ +void irqentry_exit_cond_resched(void); /** * irqentry_enter_from_kernel_mode - Establish state before invoking the irq handler diff --git a/kernel/entry/common.c b/kernel/entry/common.c index e3d381fd3d25..e234b04373fe 100644 --- a/kernel/entry/common.c +++ b/kernel/entry/common.c @@ -123,7 +123,7 @@ noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs) /** * arch_irqentry_exit_need_resched - Architecture specific need resched function * - * Invoked from raw_irqentry_exit_cond_resched() to check if resched is needed. + * Invoked from irqentry_exit_cond_resched() to check if resched is needed. * Defaults return true. * * The main purpose is to permit arch to avoid preemption of a task from an IRQ. @@ -134,7 +134,7 @@ static inline bool arch_irqentry_exit_need_resched(void); static inline bool arch_irqentry_exit_need_resched(void) { return true; } #endif -void raw_irqentry_exit_cond_resched(void) +void irqentry_exit_cond_resched(void) { if (!preempt_count()) { /* Sanity check RCU and thread stack */ @@ -145,19 +145,6 @@ void raw_irqentry_exit_cond_resched(void) preempt_schedule_irq(); } } -#ifdef CONFIG_PREEMPT_DYNAMIC -#if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) -DEFINE_STATIC_CALL(irqentry_exit_cond_resched, raw_irqentry_exit_cond_resched); -#elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) -DEFINE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched); -void dynamic_irqentry_exit_cond_resched(void) -{ - if (!static_branch_unlikely(&sk_dynamic_irqentry_exit_cond_resched)) - return; - raw_irqentry_exit_cond_resched(); -} -#endif -#endif noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t state) { diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 844f37465d62..0e9022f2d05d 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -7849,9 +7849,6 @@ EXPORT_SYMBOL(__cond_resched_rwlock_write); # endif /* - * SC:irqentry_exit_cond_resched - * - * * NONE: * (unselectable) * @@ -7859,11 +7856,9 @@ EXPORT_SYMBOL(__cond_resched_rwlock_write); * (unselectable) * * FULL: - * irqentry_exit_cond_resched <- irqentry_exit_cond_resched * dynamic_preempt_lazy <- false * * LAZY: - * irqentry_exit_cond_resched <- irqentry_exit_cond_resched * dynamic_preempt_lazy <- true */ @@ -7907,14 +7902,12 @@ static void __sched_dynamic_update(int mode) { switch (mode) { case preempt_dynamic_full: - preempt_dynamic_enable(irqentry_exit_cond_resched); preempt_dynamic_key_disable(preempt_lazy); if (mode != preempt_dynamic_mode) pr_info("Dynamic Preempt: full\n"); break; case preempt_dynamic_lazy: - preempt_dynamic_enable(irqentry_exit_cond_resched); preempt_dynamic_key_enable(preempt_lazy); if (mode != preempt_dynamic_mode) pr_info("Dynamic Preempt: lazy\n"); -- cgit v1.2.3 From 9650ce11f2e377310332d87a902dc4df3f59b905 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Mon, 3 Aug 2026 20:17:31 +0100 Subject: sched: dynamic: Simplify preempt model accessors PREEMPT_DYNAMIC is now limited to the FULL and LAZY preemption models. When CONFIG_PREEMPT_DYNAMIC=y, the NONE and VOLUNTARY models cannot be used, and neither CONFIG_PREEMPT_NONE nor CONFIG_PREEMPT_VOLUNTARY can be selected. Simplify the preempt model accessors accordingly: remove the out-of-line implementations of preempt_model_none() and preempt_model_voluntary(), and remove the preempt_dynamic_{none,voluntary} values. There are no current users of preempt_model_voluntary(), but I've kept it around for consistency, with every scheduler model having its own preempt_model_*() accessor. Suggested-by: Shrikanth Hegde Signed-off-by: Mark Rutland Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Jinjie Ruan Reviewed-by: Shrikanth Hegde Tested-by: Mete Durlu Tested-by: Shrikanth Hegde Tested-by: Jinjie Ruan Link: https://patch.msgid.link/20260803191731.3244294-7-mark.rutland@arm.com --- include/linux/preempt.h | 20 ++++++++++---------- kernel/sched/core.c | 4 ---- 2 files changed, 10 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/include/linux/preempt.h b/include/linux/preempt.h index 8299657f0f86..40310675a7cc 100644 --- a/include/linux/preempt.h +++ b/include/linux/preempt.h @@ -502,21 +502,11 @@ DEFINE_LOCK_GUARD_0(preempt_notrace, preempt_disable_notrace(), preempt_enable_n #ifdef CONFIG_PREEMPT_DYNAMIC -extern bool preempt_model_none(void); -extern bool preempt_model_voluntary(void); extern bool preempt_model_full(void); extern bool preempt_model_lazy(void); #else -static inline bool preempt_model_none(void) -{ - return IS_ENABLED(CONFIG_PREEMPT_NONE); -} -static inline bool preempt_model_voluntary(void) -{ - return IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY); -} static inline bool preempt_model_full(void) { return IS_ENABLED(CONFIG_PREEMPT); @@ -529,6 +519,16 @@ static inline bool preempt_model_lazy(void) #endif +static inline bool preempt_model_none(void) +{ + return IS_ENABLED(CONFIG_PREEMPT_NONE); +} + +static inline bool preempt_model_voluntary(void) +{ + return IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY); +} + static inline bool preempt_model_rt(void) { return IS_ENABLED(CONFIG_PREEMPT_RT); diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 05923200ef40..c9dd6f4d42a4 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -7864,8 +7864,6 @@ EXPORT_SYMBOL(__cond_resched_rwlock_write); enum { preempt_dynamic_undefined = -1, - preempt_dynamic_none, - preempt_dynamic_voluntary, preempt_dynamic_full, preempt_dynamic_lazy, }; @@ -7950,8 +7948,6 @@ static void __init preempt_dynamic_init(void) } \ EXPORT_SYMBOL_GPL(preempt_model_##mode) -PREEMPT_MODEL_ACCESSOR(none); -PREEMPT_MODEL_ACCESSOR(voluntary); PREEMPT_MODEL_ACCESSOR(full); PREEMPT_MODEL_ACCESSOR(lazy); -- cgit v1.2.3 From a5576ebce9201e2abc820348a494e666de8efe64 Mon Sep 17 00:00:00 2001 From: Hongyan Xia Date: Wed, 19 Aug 2026 08:12:09 +0000 Subject: sched: Convert paravirt_steal to new static key APIs paravirt_steal_rq_enabled and paravirt_steal_enabled use raw static_key APIs which are now deprecated. Use the new API instead. No functional change. Signed-off-by: Hongyan Xia Signed-off-by: Peter Zijlstra (Intel) Acked-by: Juergen Gross Link: https://patch.msgid.link/20260819081207.12150-1-hongyan.xia@transsion.com --- arch/arm64/kernel/paravirt.c | 4 ++-- arch/loongarch/kernel/paravirt.c | 4 ++-- arch/powerpc/platforms/pseries/setup.c | 4 ++-- arch/riscv/kernel/paravirt.c | 4 ++-- arch/x86/kernel/cpu/vmware.c | 4 ++-- arch/x86/kernel/kvm.c | 4 ++-- drivers/xen/time.c | 4 ++-- include/linux/sched/cputime.h | 6 +++--- kernel/sched/core.c | 4 ++-- kernel/sched/cputime.c | 4 ++-- 10 files changed, 21 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/arch/arm64/kernel/paravirt.c b/arch/arm64/kernel/paravirt.c index 572efb96b23f..30bf61d031eb 100644 --- a/arch/arm64/kernel/paravirt.c +++ b/arch/arm64/kernel/paravirt.c @@ -157,9 +157,9 @@ int __init pv_time_init(void) static_call_update(pv_steal_clock, para_steal_clock); - static_key_slow_inc(¶virt_steal_enabled); + static_branch_inc(¶virt_steal_enabled); if (steal_acc) - static_key_slow_inc(¶virt_steal_rq_enabled); + static_branch_inc(¶virt_steal_rq_enabled); pr_info("using stolen time PV\n"); diff --git a/arch/loongarch/kernel/paravirt.c b/arch/loongarch/kernel/paravirt.c index 10821cce554c..e8965a3f8082 100644 --- a/arch/loongarch/kernel/paravirt.c +++ b/arch/loongarch/kernel/paravirt.c @@ -308,10 +308,10 @@ int __init pv_time_init(void) static_call_update(pv_steal_clock, paravt_steal_clock); - static_key_slow_inc(¶virt_steal_enabled); + static_branch_inc(¶virt_steal_enabled); #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING if (steal_acc) - static_key_slow_inc(¶virt_steal_rq_enabled); + static_branch_inc(¶virt_steal_rq_enabled); #endif if (static_key_enabled(&virt_preempt_key)) diff --git a/arch/powerpc/platforms/pseries/setup.c b/arch/powerpc/platforms/pseries/setup.c index f29e7547c995..c181df4dec70 100644 --- a/arch/powerpc/platforms/pseries/setup.c +++ b/arch/powerpc/platforms/pseries/setup.c @@ -876,9 +876,9 @@ static void __init pSeries_setup_arch(void) static_branch_enable(&shared_processor); pv_spinlocks_init(); #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING - static_key_slow_inc(¶virt_steal_enabled); + static_branch_inc(¶virt_steal_enabled); if (steal_acc) - static_key_slow_inc(¶virt_steal_rq_enabled); + static_branch_inc(¶virt_steal_rq_enabled); #endif } diff --git a/arch/riscv/kernel/paravirt.c b/arch/riscv/kernel/paravirt.c index 5f56be79cd06..9c13a6f1ea2a 100644 --- a/arch/riscv/kernel/paravirt.c +++ b/arch/riscv/kernel/paravirt.c @@ -116,9 +116,9 @@ int __init pv_time_init(void) static_call_update(pv_steal_clock, pv_time_steal_clock); - static_key_slow_inc(¶virt_steal_enabled); + static_branch_inc(¶virt_steal_enabled); if (steal_acc) - static_key_slow_inc(¶virt_steal_rq_enabled); + static_branch_inc(¶virt_steal_rq_enabled); pr_info("Computing paravirt steal-time\n"); diff --git a/arch/x86/kernel/cpu/vmware.c b/arch/x86/kernel/cpu/vmware.c index 34b73573b108..f7ab9e7902cf 100644 --- a/arch/x86/kernel/cpu/vmware.c +++ b/arch/x86/kernel/cpu/vmware.c @@ -328,9 +328,9 @@ static int vmware_cpu_down_prepare(unsigned int cpu) static __init int activate_jump_labels(void) { if (has_steal_clock) { - static_key_slow_inc(¶virt_steal_enabled); + static_branch_inc(¶virt_steal_enabled); if (steal_acc) - static_key_slow_inc(¶virt_steal_rq_enabled); + static_branch_inc(¶virt_steal_rq_enabled); } return 0; diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index 6b0a5861ccb8..2ce01fdb8b4b 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -1054,9 +1054,9 @@ const __initconst struct hypervisor_x86 x86_hyper_kvm = { static __init int activate_jump_labels(void) { if (has_steal_clock) { - static_key_slow_inc(¶virt_steal_enabled); + static_branch_inc(¶virt_steal_enabled); if (steal_acc) - static_key_slow_inc(¶virt_steal_rq_enabled); + static_branch_inc(¶virt_steal_rq_enabled); } return 0; diff --git a/drivers/xen/time.c b/drivers/xen/time.c index a2be0a4d45b0..a02d48a2aa68 100644 --- a/drivers/xen/time.c +++ b/drivers/xen/time.c @@ -169,7 +169,7 @@ void __init xen_time_setup_guest(void) static_call_update(pv_steal_clock, xen_steal_clock); - static_key_slow_inc(¶virt_steal_enabled); + static_branch_inc(¶virt_steal_enabled); if (xen_runstate_remote) - static_key_slow_inc(¶virt_steal_rq_enabled); + static_branch_inc(¶virt_steal_rq_enabled); } diff --git a/include/linux/sched/cputime.h b/include/linux/sched/cputime.h index e90efaf6d26e..694126411dfe 100644 --- a/include/linux/sched/cputime.h +++ b/include/linux/sched/cputime.h @@ -182,9 +182,9 @@ extern unsigned long long task_sched_runtime(struct task_struct *task); #ifdef CONFIG_PARAVIRT -struct static_key; -extern struct static_key paravirt_steal_enabled; -extern struct static_key paravirt_steal_rq_enabled; +#include +DECLARE_STATIC_KEY_FALSE(paravirt_steal_enabled); +DECLARE_STATIC_KEY_FALSE(paravirt_steal_rq_enabled); #ifdef CONFIG_HAVE_PV_STEAL_CLOCK_GEN u64 dummy_steal_clock(int cpu); diff --git a/kernel/sched/core.c b/kernel/sched/core.c index c9dd6f4d42a4..4e9776681d0e 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -805,7 +805,7 @@ struct rq *_task_rq_lock(struct task_struct *p, struct rq_flags *rf) /* Use CONFIG_PARAVIRT as this will avoid more #ifdef in arch code. */ #ifdef CONFIG_PARAVIRT -struct static_key paravirt_steal_rq_enabled; +DEFINE_STATIC_KEY_FALSE(paravirt_steal_rq_enabled); #endif static void update_rq_clock_task(struct rq *rq, s64 delta) @@ -844,7 +844,7 @@ static void update_rq_clock_task(struct rq *rq, s64 delta) } #endif #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING - if (static_key_false((¶virt_steal_rq_enabled))) { + if (static_branch_unlikely(¶virt_steal_rq_enabled)) { u64 prev_steal; steal = prev_steal = paravirt_steal_clock(cpu_of(rq)); diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index 06bddaa738e5..f16970ca81d0 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -255,7 +255,7 @@ void __account_forceidle_time(struct task_struct *p, u64 delta) * occasion account more time than the calling functions think elapsed. */ #ifdef CONFIG_PARAVIRT -struct static_key paravirt_steal_enabled; +DEFINE_STATIC_KEY_FALSE(paravirt_steal_enabled); #ifdef CONFIG_HAVE_PV_STEAL_CLOCK_GEN static u64 native_steal_clock(int cpu) @@ -270,7 +270,7 @@ DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock); static __always_inline u64 steal_account_process_time(u64 maxtime) { #ifdef CONFIG_PARAVIRT - if (static_key_false(¶virt_steal_enabled)) { + if (static_branch_unlikely(¶virt_steal_enabled)) { u64 steal; steal = paravirt_steal_clock(smp_processor_id()); -- cgit v1.2.3 From a8d0854a76a876ea474e17322acb680716a09da2 Mon Sep 17 00:00:00 2001 From: Shrikanth Hegde Date: Mon, 28 Sep 2026 11:07:16 +0530 Subject: sched/cputime: Add kcpustat_field_total helper Provide a new helper function which sums up a given type of cpustat over a specified cpumask. This allows the caller's code to be simpler and avoids duplication. For example, subsequent patch in the steal governor use this exact same pattern when calculating steal time. Number of cpus can be derived from cpumask_weight() where necessary. Suggested-by: Yury Norov Signed-off-by: Shrikanth Hegde Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Yury Norov Reviewed-by: Mete Durlu Acked-by: Frederic Weisbecker Link: https://patch.msgid.link/20260928053728.797539-2-sshegde@linux.ibm.com --- arch/s390/kernel/hiperdispatch.c | 10 +++------- fs/proc/uptime.c | 6 +----- include/linux/kernel_stat.h | 11 +++++++++++ 3 files changed, 15 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/arch/s390/kernel/hiperdispatch.c b/arch/s390/kernel/hiperdispatch.c index 217206522266..c21496f0a141 100644 --- a/arch/s390/kernel/hiperdispatch.c +++ b/arch/s390/kernel/hiperdispatch.c @@ -207,16 +207,12 @@ static unsigned long hd_calculate_steal_percentage(void) { unsigned long time_delta, steal_delta, steal, percentage; static ktime_t prev; - int cpus, cpu; + int cpus; ktime_t now; - cpus = 0; - steal = 0; percentage = 0; - for_each_cpu(cpu, &hd_vmvl_cpumask) { - steal += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL]; - cpus++; - } + steal = kcpustat_field_total(CPUTIME_STEAL, &hd_vmvl_cpumask); + cpus = cpumask_weight(&hd_vmvl_cpumask); /* * If there is no vertical medium and low CPUs steal time * is 0 as vertical high CPUs shouldn't experience steal time. diff --git a/fs/proc/uptime.c b/fs/proc/uptime.c index 433aa947cd57..53143c66cbe1 100644 --- a/fs/proc/uptime.c +++ b/fs/proc/uptime.c @@ -15,12 +15,8 @@ static int uptime_proc_show(struct seq_file *m, void *v) struct timespec64 idle; u64 idle_nsec; u32 rem; - int i; - - idle_nsec = 0; - for_each_possible_cpu(i) - idle_nsec += kcpustat_field(CPUTIME_IDLE, i); + idle_nsec = kcpustat_field_total(CPUTIME_IDLE, cpu_possible_mask); ktime_get_boottime_ts64(&uptime); timens_add_boottime(&uptime); diff --git a/include/linux/kernel_stat.h b/include/linux/kernel_stat.h index 9ca6c2259dfe..c1e85550bf12 100644 --- a/include/linux/kernel_stat.h +++ b/include/linux/kernel_stat.h @@ -196,6 +196,17 @@ static inline void kcpustat_cpu_fetch(struct kernel_cpustat *dst, int cpu) } #endif /* !CONFIG_VIRT_CPU_ACCOUNTING_GEN */ +static inline u64 kcpustat_field_total(enum cpu_usage_stat usage, const struct cpumask *cpus) +{ + u64 total = 0; + int cpu; + + for_each_cpu(cpu, cpus) + total += kcpustat_field(usage, cpu); + + return total; +} + extern void account_user_time(struct task_struct *, u64); extern void account_guest_time(struct task_struct *, u64); extern void account_system_time(struct task_struct *, int, u64); -- cgit v1.2.3 From cfb463b7172dca2f29b2221b71b54890f268e3fb Mon Sep 17 00:00:00 2001 From: Shrikanth Hegde Date: Mon, 28 Sep 2026 11:07:17 +0530 Subject: cpumask: Introduce cpumask_intersects_and Introduce bitmap_intersects_and() to determine whether the intersection of three bitmaps is non-empty. Unlike cpumask_first_and_and(), this returns immediately when an intersecting word is found and does not calculate the first matching bit. Add cpumask_intersects_and() as the corresponding cpumask wrapper. A subsequent patch uses the helper to determine whether a task has a CPU that is present in its affinity mask, the preferred CPU mask, and task possible CPU mask. Suggested-by: Yury Norov Signed-off-by: Shrikanth Hegde Reviwed-by: Yury Norov Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Yury Norov Link: https://patch.msgid.link/20260928053728.797539-3-sshegde@linux.ibm.com --- include/linux/bitmap.h | 14 ++++++++++++++ include/linux/cpumask.h | 18 ++++++++++++++++++ lib/bitmap.c | 17 +++++++++++++++++ 3 files changed, 49 insertions(+) (limited to 'include/linux') diff --git a/include/linux/bitmap.h b/include/linux/bitmap.h index 7df1573a409c..adafbcf2016b 100644 --- a/include/linux/bitmap.h +++ b/include/linux/bitmap.h @@ -52,6 +52,7 @@ struct device; * bitmap_complement(dst, src, nbits) *dst = ~(*src) * bitmap_equal(src1, src2, nbits) Are *src1 and *src2 equal? * bitmap_intersects(src1, src2, nbits) Do *src1 and *src2 overlap? + * bitmap_intersects_and(src1, src2, src3, nbits) Do *src1, *src2 and *src3 overlap? * bitmap_subset(src1, src2, nbits) Is *src1 a subset of *src2? * bitmap_empty(src, nbits) Are all bits zero in *src? * bitmap_full(src, nbits) Are all bits set in *src? @@ -181,6 +182,9 @@ void __bitmap_replace(unsigned long *dst, const unsigned long *mask, unsigned int nbits); bool __bitmap_intersects(const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int nbits); +bool __bitmap_intersects_and(const unsigned long *bitmap1, + const unsigned long *bitmap2, + const unsigned long *bitmap3, unsigned int nbits); bool __bitmap_subset(const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int nbits); unsigned int __bitmap_weight(const unsigned long *bitmap, unsigned int nbits); @@ -445,6 +449,16 @@ bool bitmap_intersects(const unsigned long *src1, const unsigned long *src2, uns return __bitmap_intersects(src1, src2, nbits); } +static __always_inline +bool bitmap_intersects_and(const unsigned long *src1, const unsigned long *src2, + const unsigned long *src3, unsigned int nbits) +{ + if (small_const_nbits(nbits)) + return ((*src1 & *src2 & *src3) & BITMAP_LAST_WORD_MASK(nbits)) != 0; + else + return __bitmap_intersects_and(src1, src2, src3, nbits); +} + static __always_inline bool bitmap_subset(const unsigned long *src1, const unsigned long *src2, unsigned int nbits) { diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index 4c8bb6953107..7c8f16797f94 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h @@ -824,6 +824,24 @@ bool cpumask_intersects(const struct cpumask *src1p, const struct cpumask *src2p small_cpumask_bits); } +/** + * cpumask_intersects_and - (*src1p & *src2p & *src3p) != 0 + * @src1p: the first input + * @src2p: the second input + * @src3p: the third input + * + * Return: true if AND of the three cpumasks is non-empty, + * otherwise false + */ +static __always_inline +bool cpumask_intersects_and(const struct cpumask *src1p, + const struct cpumask *src2p, + const struct cpumask *src3p) +{ + return bitmap_intersects_and(cpumask_bits(src1p), cpumask_bits(src2p), + cpumask_bits(src3p), small_cpumask_bits); +} + /** * cpumask_subset - (*src1p & ~*src2p) == 0 * @src1p: the first input diff --git a/lib/bitmap.c b/lib/bitmap.c index ed685127a107..d1cb8a507c60 100644 --- a/lib/bitmap.c +++ b/lib/bitmap.c @@ -308,6 +308,23 @@ bool __bitmap_intersects(const unsigned long *bitmap1, } EXPORT_SYMBOL(__bitmap_intersects); +bool __bitmap_intersects_and(const unsigned long *bitmap1, + const unsigned long *bitmap2, + const unsigned long *bitmap3, unsigned int bits) +{ + unsigned int k, lim = bits / BITS_PER_LONG; + + for (k = 0; k < lim; ++k) + if (bitmap1[k] & bitmap2[k] & bitmap3[k]) + return true; + + if (bits % BITS_PER_LONG) + if ((bitmap1[k] & bitmap2[k] & bitmap3[k]) & BITMAP_LAST_WORD_MASK(bits)) + return true; + return false; +} +EXPORT_SYMBOL(__bitmap_intersects_and); + bool __bitmap_subset(const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int bits) { -- cgit v1.2.3 From 518b32bd5bb373aa334e4b73d359689071d50b5c Mon Sep 17 00:00:00 2001 From: Shrikanth Hegde Date: Mon, 28 Sep 2026 11:07:19 +0530 Subject: cpumask: Introduce cpu_preferred_mask MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Provide the preferred CPU infrastructure. Define get/set macros which could be used to get/set CPU state as preferred. CONFIG_PREFERRED_CPU will be selected by the driver which handles steal time values. It is going to set/clear preferred CPU state. This driver will be called steal_governor and it is introduced in subsequent patches. It periodically computes the steal ratio and decides on preferred CPU state. A CPU is set to preferred when it becomes active. Later it may be marked as non-preferred depending on steal ratio by the steal_governor. Always maintain design construct of preferred is subset of active. i.e. preferred ⊆ active ⊆ online ⊆ present ⊆ possible With CONFIG_PREFERRED_CPU=n, ensure set_cpu_preferred is a nop and get method returns the active state in that case. Signed-off-by: Shrikanth Hegde Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260928053728.797539-5-sshegde@linux.ibm.com --- include/linux/cpumask.h | 24 ++++++++++++++++++++++++ kernel/Kconfig.preempt | 4 ++++ kernel/cpu.c | 6 ++++++ kernel/sched/core.c | 5 +++++ 4 files changed, 39 insertions(+) (limited to 'include/linux') diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index 7c8f16797f94..bf89bb3f30f6 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h @@ -121,12 +121,20 @@ extern struct cpumask __cpu_enabled_mask; extern struct cpumask __cpu_present_mask; extern struct cpumask __cpu_active_mask; extern struct cpumask __cpu_dying_mask; + +#ifdef CONFIG_PREFERRED_CPU +extern struct cpumask __cpu_preferred_mask; +#else +#define __cpu_preferred_mask __cpu_active_mask +#endif + #define cpu_possible_mask ((const struct cpumask *)&__cpu_possible_mask) #define cpu_online_mask ((const struct cpumask *)&__cpu_online_mask) #define cpu_enabled_mask ((const struct cpumask *)&__cpu_enabled_mask) #define cpu_present_mask ((const struct cpumask *)&__cpu_present_mask) #define cpu_active_mask ((const struct cpumask *)&__cpu_active_mask) #define cpu_dying_mask ((const struct cpumask *)&__cpu_dying_mask) +#define cpu_preferred_mask ((const struct cpumask *)&__cpu_preferred_mask) extern atomic_t __num_online_cpus; extern unsigned int __num_possible_cpus; @@ -1181,6 +1189,12 @@ void init_cpu_possible(const struct cpumask *src); #define set_cpu_active(cpu, active) assign_cpu((cpu), &__cpu_active_mask, (active)) #define set_cpu_dying(cpu, dying) assign_cpu((cpu), &__cpu_dying_mask, (dying)) +#ifdef CONFIG_PREFERRED_CPU +#define set_cpu_preferred(cpu, preferred) assign_cpu((cpu), &__cpu_preferred_mask, (preferred)) +#else +#define set_cpu_preferred(cpu, preferred) do { } while (0) +#endif + void set_cpu_online(unsigned int cpu, bool online); void set_cpu_possible(unsigned int cpu, bool possible); @@ -1275,6 +1289,11 @@ static __always_inline bool cpu_dying(unsigned int cpu) return cpumask_test_cpu(cpu, cpu_dying_mask); } +static __always_inline bool cpu_preferred(unsigned int cpu) +{ + return cpumask_test_cpu(cpu, cpu_preferred_mask); +} + #else #define num_online_cpus() 1U @@ -1313,6 +1332,11 @@ static __always_inline bool cpu_dying(unsigned int cpu) return false; } +static __always_inline bool cpu_preferred(unsigned int cpu) +{ + return cpu == 0; +} + #endif /* NR_CPUS > 1 */ #define cpu_is_offline(cpu) unlikely(!cpu_online(cpu)) diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt index 985aea617cfe..edc067a0c422 100644 --- a/kernel/Kconfig.preempt +++ b/kernel/Kconfig.preempt @@ -194,3 +194,7 @@ config SCHED_CLASS_EXT For more information: Documentation/scheduler/sched-ext.rst https://github.com/sched-ext/scx + +config PREFERRED_CPU + bool + depends on SMP && PARAVIRT diff --git a/kernel/cpu.c b/kernel/cpu.c index b3c8553d7bd6..376d297a6292 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -3103,6 +3103,11 @@ EXPORT_SYMBOL(__cpu_dying_mask); atomic_t __num_online_cpus __read_mostly; EXPORT_SYMBOL(__num_online_cpus); +#ifdef CONFIG_PREFERRED_CPU +struct cpumask __cpu_preferred_mask __read_mostly; +EXPORT_SYMBOL_GPL(__cpu_preferred_mask); +#endif + void init_cpu_present(const struct cpumask *src) { cpumask_copy(&__cpu_present_mask, src); @@ -3160,6 +3165,7 @@ void __init boot_cpu_init(void) /* Mark the boot cpu "present", "online" etc for SMP and UP case */ set_cpu_online(cpu, true); set_cpu_active(cpu, true); + set_cpu_preferred(cpu, true); set_cpu_present(cpu, true); set_cpu_possible(cpu, true); diff --git a/kernel/sched/core.c b/kernel/sched/core.c index acc89310d488..d4edee72d793 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -8632,6 +8632,9 @@ int sched_cpu_activate(unsigned int cpu) */ sched_set_rq_online(rq, cpu); + /* preferred is subset of active and follows its state */ + set_cpu_preferred(cpu, true); + return 0; } @@ -8645,6 +8648,8 @@ int sched_cpu_deactivate(unsigned int cpu) if (ret) return ret; + set_cpu_preferred(cpu, false); + /* * Remove CPU from nohz.idle_cpus_mask to prevent participating in * load balancing when not active -- cgit v1.2.3 From 68957caaa9c0e127bc87fee64e45be843f6c5c91 Mon Sep 17 00:00:00 2001 From: Shrikanth Hegde Date: Mon, 28 Sep 2026 11:07:24 +0530 Subject: sched/debug: Add migration stats due to non preferred CPUs Add a new per-task stat, - nr_migrations_cpu_non_preferred: number of push migrations while the CPU is non-preferred. Since this new stat is per-task, it changes only /proc//sched. It doesn't update /proc/schedstat. Hence increasing the schedstat version is not necessary. Signed-off-by: Shrikanth Hegde Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260928053728.797539-10-sshegde@linux.ibm.com --- include/linux/sched.h | 1 + kernel/sched/core.c | 9 +++++++-- kernel/sched/debug.c | 1 + 3 files changed, 9 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sched.h b/include/linux/sched.h index d7cc77181ef9..48639b0c75d0 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -554,6 +554,7 @@ struct sched_statistics { u64 nr_failed_migrations_running; u64 nr_failed_migrations_hot; u64 nr_forced_migrations; + u64 nr_migrations_cpu_non_preferred; u64 nr_wakeups; u64 nr_wakeups_sync; diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 0e40055c2367..0bb86a43a592 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -11288,8 +11288,13 @@ static int sched_non_preferred_cpu_push_stop(void *arg) update_rq_clock(rq); context_unsafe_alias(rq); - if (task_rq(p) == rq && task_on_rq_queued(p)) - rq = __migrate_task(rq, &rf, p, cpu); + if (task_rq(p) == rq && task_on_rq_queued(p)) { + struct rq *dest_rq = __migrate_task(rq, &rf, p, cpu); + + if (rq != dest_rq) + schedstat_inc(p->stats.nr_migrations_cpu_non_preferred); + rq = dest_rq; + } rq_unlock(rq, &rf); } diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index fda10b3fa249..e6a3b516c703 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -1440,6 +1440,7 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, P_SCHEDSTAT(nr_failed_migrations_running); P_SCHEDSTAT(nr_failed_migrations_hot); P_SCHEDSTAT(nr_forced_migrations); + P_SCHEDSTAT(nr_migrations_cpu_non_preferred); P_SCHEDSTAT(nr_wakeups); P_SCHEDSTAT(nr_wakeups_sync); P_SCHEDSTAT(nr_wakeups_migrate); -- cgit v1.2.3 From 53bc5c556b82a3ca32b62ba349c4511f91526b9f Mon Sep 17 00:00:00 2001 From: Sechang Lim Date: Tue, 30 Jun 2026 08:47:37 +0000 Subject: sched: Set TIF_NEED_RESCHED before calling __trace_set_need_resched() set_tsk_need_resched() tests TIF_NEED_RESCHED, calls __trace_set_need_resched() if the flag is clear, then sets it via set_tsk_thread_flag(). A BPF raw_tp program attached to sched_set_need_resched executes synchronously inside __bpf_trace_run(). On return, __bpf_trace_run() drops the RCU lock with rcu_read_unlock_migrate(), which on the preempt-or-BH-disabled path calls set_need_resched_current() -> set_tsk_need_resched() again. set_tsk_thread_flag() follows the tracepoint call, so every re-entrant frame sees TIF_NEED_RESCHED clear and calls __trace_set_need_resched() again: BUG: TASK stack guard page was hit at ffffc9001224ff98 Oops: stack guard page: 0000 [#1] SMP KASAN PTI RIP: 0010:__bpf_trace_sched_set_need_resched_tp+0x1c/0x190 Call Trace: trace_sched_set_need_resched_tp+0x110/0x130 set_tsk_need_resched include/linux/sched.h:2076 set_need_resched_current include/linux/sched.h:2094 rcu_read_unlock_special+0x43a/0x440 __rcu_read_unlock+0x9e/0x120 rcu_read_unlock_migrate+0xa9/0x240 __bpf_trace_run+0x131/0x180 bpf_trace_run3+0x333/0x430 __bpf_trace_sched_set_need_resched_tp+0x13a/0x190 trace_sched_set_need_resched_tp+0x110/0x130 set_tsk_need_resched include/linux/sched.h:2076 ... __resched_curr() has the same ordering, firing the tracepoint before setting the flag via set_ti_thread_flag() or set_nr_and_not_polling(). Fix it for consistency. Replace the separate test_tsk_thread_flag() + set_tsk_thread_flag() pair in set_tsk_need_resched() with test_and_set_tsk_thread_flag(). In __resched_curr(), move the tracepoint call after the flag is set in each path. Fixes: adcc3bfa8806 ("sched: Adapt sched tracepoints for RV task model") Signed-off-by: Sechang Lim Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Andrea Righi Acked-by: Gabriele Monaco Link: https://patch.msgid.link/20260630084750.2792851-1-rhkrqnwk98@gmail.com --- include/linux/sched.h | 5 ++--- kernel/sched/core.c | 7 +++++-- 2 files changed, 7 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sched.h b/include/linux/sched.h index 48639b0c75d0..7b91cae4a59c 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -2105,10 +2105,9 @@ static inline int test_tsk_thread_flag(struct task_struct *tsk, int flag) static inline void set_tsk_need_resched(struct task_struct *tsk) { - if (tracepoint_enabled(sched_set_need_resched_tp) && - !test_tsk_thread_flag(tsk, TIF_NEED_RESCHED)) + if (!test_and_set_tsk_thread_flag(tsk, TIF_NEED_RESCHED) && + tracepoint_enabled(sched_set_need_resched_tp)) __trace_set_need_resched(tsk, TIF_NEED_RESCHED); - set_tsk_thread_flag(tsk,TIF_NEED_RESCHED); } static inline void clear_tsk_need_resched(struct task_struct *tsk) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 23677d76f9d2..71d3c948e14a 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -1196,6 +1196,7 @@ static void __resched_curr(struct rq *rq, int tif) { struct task_struct *curr = rq->curr; struct thread_info *cti = task_thread_info(curr); + bool need_ipi; int cpu; lockdep_assert_rq_held(rq); @@ -1212,15 +1213,17 @@ static void __resched_curr(struct rq *rq, int tif) cpu = cpu_of(rq); - trace_sched_set_need_resched_tp(curr, cpu, tif); if (cpu == smp_processor_id()) { set_ti_thread_flag(cti, tif); if (tif == TIF_NEED_RESCHED) set_preempt_need_resched(); + trace_sched_set_need_resched_tp(curr, cpu, tif); return; } - if (set_nr_and_not_polling(cti, tif)) { + need_ipi = set_nr_and_not_polling(cti, tif); + trace_sched_set_need_resched_tp(curr, cpu, tif); + if (need_ipi) { if (tif == TIF_NEED_RESCHED) smp_send_reschedule(cpu); } else { -- cgit v1.2.3 From 40dcc9bdbef3d93a516c8ee32cb1b50eddeeebcc Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Fri, 11 Sep 2026 16:38:13 +0200 Subject: irq_work: Flush lazy work CPU down on PREEMPT_RT PREEMPT_RT invokes IRQ_WORK_LAZY callbacks and callbacks which are not explicitly marked IRQ_WORK_HARD_IRQ from thread context. If a CPU gets shutdown, the irq_work is flushed during smpcfd_dying_cpu() which is invoked on the target CPU with disabled interrupts. The callbacks enqueued on lazy_list on PREEMPT_RT are not flushed because they require thread context. The callbacks remain and get processed once the CPU gets back online. per-CPU irq_work items continue to work for enqueues on their local-CPU, only the work on the "offline" CPU get stuck. "Single" irq_work are worse because they remain "claimed" and can not be used again. One "symptom" is the late printk() in CPU down path which enqueues the irq_work for printing but it never gets scheduled. Further prints don't print to the console because the irq_work is "pending" of the offline CPU. Another problem is the canceling/ flushing of the irq_work which is stuck. Since the irq_work will not continue, the task waiting for its completion will block waiting. While looking through the users of irq_work, it does not matter if the callback is enqueued on another CPU. This makes it possible to invoke the callbacks from a remote CPU by smpcfd_dead_cpu(). This is invoked from the control CPU with enabled interrupts shortly after the hotplugged CPU is shutdown so the llist can be accessed safely. Add irq_work_run_cpu() which invokes the irq_work callbacks of the specified dead CPU. Invoke it from smpcfd_dead_cpu(). Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260911143815.997254-3-bigeasy@linutronix.de --- include/linux/irq_work.h | 2 ++ kernel/irq_work.c | 11 +++++++++++ kernel/smp.c | 1 + 3 files changed, 14 insertions(+) (limited to 'include/linux') diff --git a/include/linux/irq_work.h b/include/linux/irq_work.h index c5afd053ae32..7184e1273965 100644 --- a/include/linux/irq_work.h +++ b/include/linux/irq_work.h @@ -59,6 +59,7 @@ void irq_work_sync(struct irq_work *work); #include void irq_work_run(void); +void irq_work_run_cpu(unsigned int cpu); bool irq_work_needs_cpu(void); void irq_work_single(void *arg); @@ -67,6 +68,7 @@ void arch_irq_work_raise(void); #else static inline bool irq_work_needs_cpu(void) { return false; } static inline void irq_work_run(void) { } +static inline void irq_work_run_cpu(unsigned int cpu) { } static inline void irq_work_single(void *arg) { } #endif diff --git a/kernel/irq_work.c b/kernel/irq_work.c index 9f19c05c9962..73eabcbdcd50 100644 --- a/kernel/irq_work.c +++ b/kernel/irq_work.c @@ -263,6 +263,17 @@ void irq_work_run(void) } EXPORT_SYMBOL_GPL(irq_work_run); +void irq_work_run_cpu(unsigned int cpu) +{ + if (WARN_ON_ONCE(!cpumask_test_cpu(cpu, cpu_dying_mask))) + return; + + if (!IS_ENABLED(CONFIG_PREEMPT_RT)) + return; + + irq_work_run_list(per_cpu_ptr(&lazy_list, cpu)); +} + void irq_work_tick(void) { struct llist_head *raised = this_cpu_ptr(&raised_list); diff --git a/kernel/smp.c b/kernel/smp.c index b696bcc60c08..8f530a094c30 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -87,6 +87,7 @@ int smpcfd_dead_cpu(unsigned int cpu) free_cpumask_var(cfd->cpumask); free_cpumask_var(cfd->cpumask_ipi); + irq_work_run_cpu(cpu); return 0; } -- cgit v1.2.3