From 37cca059804ee35782a1cc9d7b087e5d3c855eae Mon Sep 17 00:00:00 2001 From: Christian Loehle Date: Fri, 19 Jun 2026 16:59:20 +0100 Subject: sched_ext: Remove deprecated scx_bpf_cpu_rq() scx_bpf_cpu_rq() exposes rq pointers without requiring the rq lock and has emitted a deprecation warning since commit 5c48d88fe004 ("sched_ext: deprecation warn for scx_bpf_cpu_rq()"). The supported replacements cover the intended uses: scx_bpf_locked_rq() for locked rq access and scx_bpf_cpu_curr() for remote curr lookup. Remove the kfunc, its BTF registrations, the deprecation warning state, and the BPF-side prototype and compat fallback. Signed-off-by: Christian Loehle Reviewed-by: Andrea Righi Reviewed-by: Hongyan Xia Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 30 ------------------------------ kernel/sched/ext/internal.h | 1 - 2 files changed, 31 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index aecbb021d6d7..f08ef32b9bd6 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -10109,34 +10109,6 @@ __bpf_kfunc s32 scx_bpf_task_cid(const struct task_struct *p) return tbl[task_cpu(p)]; } -/** - * scx_bpf_cpu_rq - Fetch the rq of a CPU - * @cpu: CPU of the rq - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - */ -__bpf_kfunc struct rq *scx_bpf_cpu_rq(s32 cpu, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return NULL; - - if (!scx_cpu_valid(sch, cpu, NULL)) - return NULL; - - if (!sch->warned_deprecated_rq) { - printk_deferred(KERN_WARNING "sched_ext: %s() is deprecated; " - "use scx_bpf_locked_rq() when holding rq lock " - "or scx_bpf_cpu_curr() to read remote curr safely.\n", __func__); - sch->warned_deprecated_rq = true; - } - - return cpu_rq(cpu); -} - /** * scx_bpf_locked_rq - Return the rq currently locked by SCX * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs @@ -10427,7 +10399,6 @@ BTF_ID_FLAGS(func, scx_bpf_put_cpumask, KF_RELEASE) BTF_ID_FLAGS(func, scx_bpf_task_running, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_task_cpu, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_task_cid, KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_cpu_rq, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_locked_rq, KF_IMPLICIT_ARGS | KF_RET_NULL) BTF_ID_FLAGS(func, scx_bpf_cpu_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED) BTF_ID_FLAGS(func, scx_bpf_cid_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED) @@ -10462,7 +10433,6 @@ static const struct btf_kfunc_id_set scx_kfunc_set_any = { BTF_KFUNCS_START(scx_kfunc_ids_cpu_only) BTF_ID_FLAGS(func, scx_bpf_kick_cpu, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_task_cpu, KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_cpu_rq, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_cpu_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED) BTF_ID_FLAGS(func, scx_bpf_cpu_node, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_cpuperf_cap, KF_IMPLICIT_ARGS) diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 673059fa9d72..f4ba67799b0f 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1171,7 +1171,6 @@ struct scx_sched { * but it doesn't really matter. */ bool warned_zero_slice:1; - bool warned_deprecated_rq:1; bool warned_unassoc_progs:1; struct list_head all; -- cgit v1.2.3 From da428d572e07bb9dd2d076297ef5700f6483aafd Mon Sep 17 00:00:00 2001 From: Cheng-Yang Chou Date: Wed, 10 Jun 2026 23:26:55 +0800 Subject: sched_ext: Fix exit_cpu accuracy for lockup paths handle_lockup() uses raw_smp_processor_id() for exit_cpu, which is wrong for two paths: - scx_hardlockup_irq_workfn() has the hung CPU in a local variable but irq_work may run elsewhere. Pass the local cpu explicitly. - scx_rcu_cpu_stall() records the detector CPU rather than the stalled one. Pass -1 for now. The next patch fixes this properly. Signed-off-by: Cheng-Yang Chou Reviewed-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 15 +++++++++------ kernel/sched/ext/internal.h | 2 -- 2 files changed, 9 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index f08ef32b9bd6..7044a591e4c5 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -5092,6 +5092,7 @@ bool scx_allow_ttwu_queue(const struct task_struct *p) /** * handle_lockup - sched_ext common lockup handler + * @exit_cpu: CPU to record in exit_info. Pass the stalled/hung CPU, not current. * @fmt: format string * * Called on system stall or lockup condition and initiates abort of sched_ext @@ -5101,7 +5102,7 @@ bool scx_allow_ttwu_queue(const struct task_struct *p) * resolve the lockup. %false if sched_ext is not enabled or abort was already * initiated by someone else. */ -static __printf(1, 2) bool handle_lockup(const char *fmt, ...) +static __printf(2, 3) bool handle_lockup(int exit_cpu, const char *fmt, ...) { struct scx_sched *sch; va_list args; @@ -5117,7 +5118,7 @@ static __printf(1, 2) bool handle_lockup(const char *fmt, ...) case SCX_ENABLING: case SCX_ENABLED: va_start(args, fmt); - ret = scx_verror(sch, fmt, args); + ret = scx_vexit(sch, SCX_EXIT_ERROR, 0, exit_cpu, fmt, args); va_end(args); return ret; default: @@ -5139,7 +5140,7 @@ static __printf(1, 2) bool handle_lockup(const char *fmt, ...) */ bool scx_rcu_cpu_stall(void) { - return handle_lockup("RCU CPU stall detected!"); + return handle_lockup(-1, "RCU CPU stall detected!"); } /** @@ -5154,11 +5155,13 @@ bool scx_rcu_cpu_stall(void) */ void scx_softlockup(u32 dur_s) { - if (!handle_lockup("soft lockup - CPU %d stuck for %us", smp_processor_id(), dur_s)) + int cpu = smp_processor_id(); + + if (!handle_lockup(cpu, "soft lockup - CPU %d stuck for %us", cpu, dur_s)) return; printk_deferred(KERN_ERR "sched_ext: Soft lockup - CPU %d stuck for %us, disabling BPF scheduler\n", - smp_processor_id(), dur_s); + cpu, dur_s); } /* @@ -5173,7 +5176,7 @@ static void scx_hardlockup_irq_workfn(struct irq_work *work) { int cpu = atomic_xchg(&scx_hardlockup_cpu, -1); - if (cpu >= 0 && handle_lockup("hard lockup - CPU %d", cpu)) + if (cpu >= 0 && handle_lockup(cpu, "hard lockup - CPU %d", cpu)) printk_deferred(KERN_ERR "sched_ext: Hard lockup - CPU %d, disabling BPF scheduler\n", cpu); } diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index f4ba67799b0f..65eceefcf5e2 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1543,8 +1543,6 @@ __printf(5, 6) bool __scx_exit(struct scx_sched *sch, enum scx_exit_kind kind, __scx_exit(sch, kind, exit_code, raw_smp_processor_id(), fmt, ##args) #define scx_error(sch, fmt, args...) \ scx_exit((sch), SCX_EXIT_ERROR, 0, fmt, ##args) -#define scx_verror(sch, fmt, args) \ - scx_vexit((sch), SCX_EXIT_ERROR, 0, raw_smp_processor_id(), fmt, args) /* * Return the rq currently locked from an scx callback, or NULL if no rq is -- cgit v1.2.3 From 252891d03dfb239fb76a78ab06b0e5a8719e0f86 Mon Sep 17 00:00:00 2001 From: Cheng-Yang Chou Date: Wed, 10 Jun 2026 23:26:56 +0800 Subject: sched_ext, rcu: Upgrade RCU stall paths to report cpumask of stalled CPUs scx_rcu_cpu_stall() previously recorded the detector CPU rather than the stalled one, and the expedited grace period path had no stalled CPU to report at all. Thread a cpumask through panic_on_rcu_stall() and scx_rcu_cpu_stall() to capture all stalled CPUs. Report cpumask_first() as exit_cpu and the full CPU list in the exit message. Task-only stalls yield exit_cpu = -1. Store the stall mask in scx_sched rather than scx_exit_info, keeping the BPF-visible struct unchanged. scx_dump_state() reads sch->stall_cpus directly and dumps all stalled CPUs first to avoid losing them to truncation. Signed-off-by: Cheng-Yang Chou Reviewed-by: Paul E. McKenney Reviewed-by: Andrea Righi Signed-off-by: Tejun Heo --- include/linux/sched/ext.h | 4 +-- kernel/rcu/tree.c | 3 ++ kernel/rcu/tree_exp.h | 5 +++- kernel/rcu/tree_stall.h | 13 +++++--- kernel/sched/ext/ext.c | 73 +++++++++++++++++++++++++++++++++++++++------ kernel/sched/ext/internal.h | 1 + 6 files changed, 83 insertions(+), 16 deletions(-) (limited to 'kernel') diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index 20b2343aa344..75cb8b119fb7 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -263,7 +263,7 @@ void sched_ext_dead(struct task_struct *p); void print_scx_info(const char *log_lvl, struct task_struct *p); void scx_softlockup(u32 dur_s); bool scx_hardlockup(int cpu); -bool scx_rcu_cpu_stall(void); +bool scx_rcu_cpu_stall(const struct cpumask *stalled_mask); #else /* !CONFIG_SCHED_CLASS_EXT */ @@ -271,7 +271,7 @@ static inline void sched_ext_dead(struct task_struct *p) {} static inline void print_scx_info(const char *log_lvl, struct task_struct *p) {} static inline void scx_softlockup(u32 dur_s) {} static inline bool scx_hardlockup(int cpu) { return false; } -static inline bool scx_rcu_cpu_stall(void) { return false; } +static inline bool scx_rcu_cpu_stall(const struct cpumask *stalled_mask) { return false; } #endif /* CONFIG_SCHED_CLASS_EXT */ diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 03a43d3d2616..415583c35f8c 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -4900,6 +4900,9 @@ static void __init rcu_dump_rcu_node_tree(void) struct workqueue_struct *rcu_gp_wq; +static struct cpumask rcu_stall_cpumask; +static struct cpumask rcu_exp_stall_cpumask; + void __init rcu_init(void) { int cpu = smp_processor_id(); diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h index 82cada459e5d..46b6907f1b09 100644 --- a/kernel/rcu/tree_exp.h +++ b/kernel/rcu/tree_exp.h @@ -578,6 +578,7 @@ static void synchronize_rcu_expedited_stall(unsigned long jiffies_start, unsigne if (!(READ_ONCE(rnp->expmask) & mask)) continue; ndetected++; + cpumask_set_cpu(cpu, &rcu_exp_stall_cpumask); rdp = per_cpu_ptr(&rcu_data, cpu); pr_cont(" %d-%c%c%c%c", cpu, "O."[!!cpu_online(cpu)], @@ -665,6 +666,8 @@ static void synchronize_rcu_expedited_wait(void) if (rcu_stall_is_suppressed()) continue; + cpumask_clear(&rcu_exp_stall_cpumask); + nbcon_cpu_emergency_enter(); j = jiffies; @@ -675,7 +678,7 @@ static void synchronize_rcu_expedited_wait(void) nbcon_cpu_emergency_exit(); - panic_on_rcu_stall(); + panic_on_rcu_stall(&rcu_exp_stall_cpumask); } } diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h index cf7ae51cba40..ebf381936eb1 100644 --- a/kernel/rcu/tree_stall.h +++ b/kernel/rcu/tree_stall.h @@ -159,7 +159,7 @@ static int __init check_cpu_stall_init(void) early_initcall(check_cpu_stall_init); /* If so specified via sysctl, panic, yielding cleaner stall-warning output. */ -static void panic_on_rcu_stall(void) +static void panic_on_rcu_stall(const struct cpumask *stalled_mask) { static int cpu_stall; @@ -167,7 +167,7 @@ static void panic_on_rcu_stall(void) * Attempt to kick out the BPF scheduler if it's installed and defer * the panic to give the system a chance to recover. */ - if (scx_rcu_cpu_stall()) + if (scx_rcu_cpu_stall(stalled_mask)) return; if (++cpu_stall < sysctl_max_rcu_stall_to_panic) @@ -644,6 +644,8 @@ static void print_other_cpu_stall(unsigned long gp_seq, unsigned long gps) if (rcu_stall_is_suppressed()) return; + cpumask_clear(&rcu_stall_cpumask); + nbcon_cpu_emergency_enter(); /* @@ -659,6 +661,7 @@ static void print_other_cpu_stall(unsigned long gp_seq, unsigned long gps) for_each_leaf_node_possible_cpu(rnp, cpu) if (rnp->qsmask & leaf_node_cpu_bit(rnp, cpu)) { print_cpu_stall_info(cpu); + cpumask_set_cpu(cpu, &rcu_stall_cpumask); ndetected++; } } @@ -700,7 +703,7 @@ static void print_other_cpu_stall(unsigned long gp_seq, unsigned long gps) nbcon_cpu_emergency_exit(); - panic_on_rcu_stall(); + panic_on_rcu_stall(&rcu_stall_cpumask); rcu_force_quiescent_state(); /* Kick them all. */ } @@ -753,7 +756,9 @@ static void print_cpu_stall(unsigned long gp_seq, unsigned long gps) nbcon_cpu_emergency_exit(); - panic_on_rcu_stall(); + cpumask_clear(&rcu_stall_cpumask); + cpumask_set_cpu(smp_processor_id(), &rcu_stall_cpumask); + panic_on_rcu_stall(&rcu_stall_cpumask); /* * Attempt to revive the RCU machinery by forcing a context switch. diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 7044a591e4c5..9c9cb9d08bca 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -4853,6 +4853,8 @@ static const struct attribute_group scx_global_attr_group = { static void free_pnode(struct scx_sched_pnode *pnode); static void free_exit_info(struct scx_exit_info *ei); +static const char *scx_exit_reason(enum scx_exit_kind kind); +static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind); static s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch) { @@ -4909,6 +4911,7 @@ static void scx_sched_free_rcu_work(struct work_struct *work) timer_shutdown_sync(&sch->bypass_lb_timer); free_cpumask_var(sch->bypass_lb_donee_cpumask); free_cpumask_var(sch->bypass_lb_resched_cpumask); + free_cpumask_var(sch->stall_cpus); #ifdef CONFIG_EXT_SUB_SCHED kfree(sch->cgrp_path); @@ -5138,9 +5141,46 @@ static __printf(2, 3) bool handle_lockup(int exit_cpu, const char *fmt, ...) * resolve the reported RCU stall. %false if sched_ext is not enabled or someone * else already initiated abort. */ -bool scx_rcu_cpu_stall(void) +bool scx_rcu_cpu_stall(const struct cpumask *stalled_mask) { - return handle_lockup(-1, "RCU CPU stall detected!"); + struct scx_sched *sch; + struct scx_exit_info *ei; + int exit_cpu; + + guard(rcu)(); + + sch = rcu_dereference(scx_root); + if (unlikely(!sch)) + return false; + + switch (scx_enable_state()) { + case SCX_ENABLING: + case SCX_ENABLED: + break; + default: + return false; + } + + exit_cpu = cpumask_empty(stalled_mask) ? -1 : (int)cpumask_first(stalled_mask); + ei = sch->exit_info; + + guard(preempt)(); + + if (!scx_claim_exit(sch, SCX_EXIT_ERROR)) + return false; + +#ifdef CONFIG_STACKTRACE + ei->bt_len = stack_trace_save(ei->bt, SCX_EXIT_BT_LEN, 1); +#endif + scnprintf(ei->msg, SCX_EXIT_MSG_LEN, "RCU CPU stall on CPUs (%*pbl)", + cpumask_pr_args(stalled_mask)); + ei->kind = SCX_EXIT_ERROR; + ei->reason = scx_exit_reason(SCX_EXIT_ERROR); + ei->exit_cpu = exit_cpu; + cpumask_copy(sch->stall_cpus, stalled_mask); + + irq_work_queue(&sch->disable_irq_work); + return true; } /** @@ -6587,14 +6627,23 @@ static void scx_dump_state(struct scx_sched *sch, struct scx_exit_info *ei, dump_line(&s, "----------"); /* - * Dump the exit CPU first so it isn't lost to dump truncation, then - * walk the rest in order, skipping the one already dumped. + * Dump stalled CPUs first so they aren't lost to dump truncation, then + * walk the rest in order. Fall back to exit_cpu if no stall mask set. */ - if (ei->exit_cpu >= 0) - scx_dump_cpu(sch, &s, &dctx, ei->exit_cpu, dump_all_tasks); - for_each_possible_cpu(cpu) { - if (cpu != ei->exit_cpu) + if (!cpumask_empty(sch->stall_cpus)) { + for_each_cpu(cpu, sch->stall_cpus) scx_dump_cpu(sch, &s, &dctx, cpu, dump_all_tasks); + for_each_possible_cpu(cpu) { + if (!cpumask_test_cpu(cpu, sch->stall_cpus)) + scx_dump_cpu(sch, &s, &dctx, cpu, dump_all_tasks); + } + } else { + if (ei->exit_cpu >= 0) + scx_dump_cpu(sch, &s, &dctx, ei->exit_cpu, dump_all_tasks); + for_each_possible_cpu(cpu) { + if (cpu != ei->exit_cpu) + scx_dump_cpu(sch, &s, &dctx, cpu, dump_all_tasks); + } } dump_newline(&s); @@ -6831,6 +6880,10 @@ static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, ret = -ENOMEM; goto err_free_lb_cpumask; } + if (!zalloc_cpumask_var(&sch->stall_cpus, GFP_KERNEL)) { + ret = -ENOMEM; + goto err_free_lb_resched_cpumask; + } /* * Copy ops through the right union view. For cid-form the source is * struct sched_ext_ops_cid which lacks the trailing cpu_acquire/ @@ -6914,8 +6967,10 @@ static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, #ifdef CONFIG_EXT_SUB_SCHED err_free_lb_resched: RCU_INIT_POINTER(ops->priv, NULL); - free_cpumask_var(sch->bypass_lb_resched_cpumask); + free_cpumask_var(sch->stall_cpus); #endif +err_free_lb_resched_cpumask: + free_cpumask_var(sch->bypass_lb_resched_cpumask); err_free_lb_cpumask: free_cpumask_var(sch->bypass_lb_donee_cpumask); err_stop_helper: diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 65eceefcf5e2..75522a5f28f4 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1205,6 +1205,7 @@ struct scx_sched { struct timer_list bypass_lb_timer; cpumask_var_t bypass_lb_donee_cpumask; cpumask_var_t bypass_lb_resched_cpumask; + cpumask_var_t stall_cpus; struct rcu_work rcu_work; /* all ancestors including self */ -- cgit v1.2.3 From e2c60e15d0803d58d757dbfa5db6e528571248a5 Mon Sep 17 00:00:00 2001 From: Cheng-Yang Chou Date: Thu, 25 Jun 2026 23:13:11 +0800 Subject: sched_ext: Convert scx_arena_grow() to use scx_arena_to_kaddr() scx_arena_grow() was left open-coded when 2e05f2fd0dd7 ("sched_ext: Add scx_arena_to_kaddr() / scx_kaddr_to_arena()") introduced the helper. Replace the manual bpf_arena_map_kern_vm_start() fetch and #ifdef guard with scx_arena_to_kaddr(sch, p). Signed-off-by: Cheng-Yang Chou Signed-off-by: Tejun Heo --- kernel/sched/ext/arena.c | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/arena.c b/kernel/sched/ext/arena.c index 5783694ec21d..b0e6a0be4913 100644 --- a/kernel/sched/ext/arena.c +++ b/kernel/sched/ext/arena.c @@ -70,8 +70,6 @@ void scx_arena_pool_destroy(struct scx_sched *sch) */ static int scx_arena_grow(struct scx_sched *sch, u32 page_cnt) { - u64 kern_vm_start; - u32 uaddr32; void *p; int ret; @@ -83,15 +81,8 @@ static int scx_arena_grow(struct scx_sched *sch, u32 page_cnt) if (!p) return -ENOMEM; - uaddr32 = (u32)(unsigned long)p; - /* arena.o, which defines these, is built only on MMU && 64BIT */ -#if defined(CONFIG_MMU) && defined(CONFIG_64BIT) - kern_vm_start = bpf_arena_map_kern_vm_start(sch->arena_map); -#else - kern_vm_start = 0; -#endif - - ret = gen_pool_add(sch->arena_pool, kern_vm_start + uaddr32, + ret = gen_pool_add(sch->arena_pool, + (unsigned long)scx_arena_to_kaddr(sch, p), page_cnt * PAGE_SIZE, NUMA_NO_NODE); if (ret) { bpf_arena_free_pages_non_sleepable(sch->arena_map, p, page_cnt); -- cgit v1.2.3 From b164760d7a648451f6621f11e3c1d3125f00305c Mon Sep 17 00:00:00 2001 From: Cheng-Yang Chou Date: Tue, 30 Jun 2026 22:06:23 +0800 Subject: sched_ext: Replace open-coded event lists with SCX_EVENTS_LIST Three sites enumerate every scx_event_stats field by name: scx_read_events(), scx_attr_events_show(), and scx_dump_state(). Adding a new SCX_EV_* event requires three separate manual edits. Missing any one silently drops counts or omits the field from diagnostic output with no compile-time indication. SCX_EVENTS_LIST(SCX_EVENT) in internal.h becomes the single authoritative list of event names, so all three sites stay in sync. No functional changes. Signed-off-by: Cheng-Yang Chou Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 61 ++++++++------------------------------------- kernel/sched/ext/internal.h | 15 +++++++++++ 2 files changed, 25 insertions(+), 51 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 9c9cb9d08bca..0c51f9826a77 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1004,16 +1004,6 @@ static struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter) trace_sched_ext_event(#name, cnt); \ } while(0) -/** - * scx_agg_event - Aggregate an event counter 'kind' from 'src_e' to 'dst_e' - * @dst_e: destination event stats - * @src_e: source event stats - * @kind: a kind of event to be aggregated - */ -#define scx_agg_event(dst_e, src_e, kind) do { \ - (dst_e)->kind += READ_ONCE((src_e)->kind); \ -} while(0) - /** * scx_dump_event - Dump an event 'kind' in 'events' to 's' * @s: output seq_buf @@ -4989,19 +4979,9 @@ static ssize_t scx_attr_events_show(struct kobject *kobj, int at = 0; scx_read_events(sch, &events); - at += scx_attr_event_show(buf, at, &events, SCX_EV_SELECT_CPU_FALLBACK); - at += scx_attr_event_show(buf, at, &events, SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE); - at += scx_attr_event_show(buf, at, &events, SCX_EV_DISPATCH_KEEP_LAST); - at += scx_attr_event_show(buf, at, &events, SCX_EV_ENQ_SKIP_EXITING); - at += scx_attr_event_show(buf, at, &events, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED); - at += scx_attr_event_show(buf, at, &events, SCX_EV_REENQ_IMMED); - at += scx_attr_event_show(buf, at, &events, SCX_EV_REENQ_LOCAL_REPEAT); - at += scx_attr_event_show(buf, at, &events, SCX_EV_REFILL_SLICE_DFL); - at += scx_attr_event_show(buf, at, &events, SCX_EV_BYPASS_DURATION); - at += scx_attr_event_show(buf, at, &events, SCX_EV_BYPASS_DISPATCH); - at += scx_attr_event_show(buf, at, &events, SCX_EV_BYPASS_ACTIVATE); - at += scx_attr_event_show(buf, at, &events, SCX_EV_INSERT_NOT_OWNED); - at += scx_attr_event_show(buf, at, &events, SCX_EV_SUB_BYPASS_DISPATCH); +#define SCX_EVENT(name) at += scx_attr_event_show(buf, at, &events, name) + SCX_EVENTS_LIST(SCX_EVENT); +#undef SCX_EVENT return at; } SCX_ATTR(events); @@ -6651,19 +6631,9 @@ static void scx_dump_state(struct scx_sched *sch, struct scx_exit_info *ei, dump_line(&s, "--------------"); scx_read_events(sch, &events); - scx_dump_event(s, &events, SCX_EV_SELECT_CPU_FALLBACK); - scx_dump_event(s, &events, SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE); - scx_dump_event(s, &events, SCX_EV_DISPATCH_KEEP_LAST); - scx_dump_event(s, &events, SCX_EV_ENQ_SKIP_EXITING); - scx_dump_event(s, &events, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED); - scx_dump_event(s, &events, SCX_EV_REENQ_IMMED); - scx_dump_event(s, &events, SCX_EV_REENQ_LOCAL_REPEAT); - scx_dump_event(s, &events, SCX_EV_REFILL_SLICE_DFL); - scx_dump_event(s, &events, SCX_EV_BYPASS_DURATION); - scx_dump_event(s, &events, SCX_EV_BYPASS_DISPATCH); - scx_dump_event(s, &events, SCX_EV_BYPASS_ACTIVATE); - scx_dump_event(s, &events, SCX_EV_INSERT_NOT_OWNED); - scx_dump_event(s, &events, SCX_EV_SUB_BYPASS_DISPATCH); +#define SCX_EVENT(name) scx_dump_event(s, &events, name) + SCX_EVENTS_LIST(SCX_EVENT); +#undef SCX_EVENT if (seq_buf_has_overflowed(&s) && dump_len >= sizeof(trunc_marker)) memcpy(ei->dump + dump_len - sizeof(trunc_marker), @@ -10333,26 +10303,15 @@ __bpf_kfunc u64 scx_bpf_now(void) static void scx_read_events(struct scx_sched *sch, struct scx_event_stats *events) { - struct scx_event_stats *e_cpu; int cpu; /* Aggregate per-CPU event counters into @events. */ memset(events, 0, sizeof(*events)); for_each_possible_cpu(cpu) { - e_cpu = &per_cpu_ptr(sch->pcpu, cpu)->event_stats; - scx_agg_event(events, e_cpu, SCX_EV_SELECT_CPU_FALLBACK); - scx_agg_event(events, e_cpu, SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE); - scx_agg_event(events, e_cpu, SCX_EV_DISPATCH_KEEP_LAST); - scx_agg_event(events, e_cpu, SCX_EV_ENQ_SKIP_EXITING); - scx_agg_event(events, e_cpu, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED); - scx_agg_event(events, e_cpu, SCX_EV_REENQ_IMMED); - scx_agg_event(events, e_cpu, SCX_EV_REENQ_LOCAL_REPEAT); - scx_agg_event(events, e_cpu, SCX_EV_REFILL_SLICE_DFL); - scx_agg_event(events, e_cpu, SCX_EV_BYPASS_DURATION); - scx_agg_event(events, e_cpu, SCX_EV_BYPASS_DISPATCH); - scx_agg_event(events, e_cpu, SCX_EV_BYPASS_ACTIVATE); - scx_agg_event(events, e_cpu, SCX_EV_INSERT_NOT_OWNED); - scx_agg_event(events, e_cpu, SCX_EV_SUB_BYPASS_DISPATCH); + struct scx_event_stats *e_cpu = &per_cpu_ptr(sch->pcpu, cpu)->event_stats; +#define SCX_EVENT(name) events->name += READ_ONCE(e_cpu->name) + SCX_EVENTS_LIST(SCX_EVENT); +#undef SCX_EVENT } } diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 75522a5f28f4..0256931a379a 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1051,6 +1051,21 @@ struct scx_event_stats { s64 SCX_EV_SUB_BYPASS_DISPATCH; }; +#define SCX_EVENTS_LIST(SCX_EVENT) \ + SCX_EVENT(SCX_EV_SELECT_CPU_FALLBACK); \ + SCX_EVENT(SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE); \ + SCX_EVENT(SCX_EV_DISPATCH_KEEP_LAST); \ + SCX_EVENT(SCX_EV_ENQ_SKIP_EXITING); \ + SCX_EVENT(SCX_EV_ENQ_SKIP_MIGRATION_DISABLED); \ + SCX_EVENT(SCX_EV_REENQ_IMMED); \ + SCX_EVENT(SCX_EV_REENQ_LOCAL_REPEAT); \ + SCX_EVENT(SCX_EV_REFILL_SLICE_DFL); \ + SCX_EVENT(SCX_EV_BYPASS_DURATION); \ + SCX_EVENT(SCX_EV_BYPASS_DISPATCH); \ + SCX_EVENT(SCX_EV_BYPASS_ACTIVATE); \ + SCX_EVENT(SCX_EV_INSERT_NOT_OWNED); \ + SCX_EVENT(SCX_EV_SUB_BYPASS_DISPATCH) + struct scx_sched; enum scx_sched_pcpu_flags { -- cgit v1.2.3 From 5df6a4506d060fcc3b875437cea8d77b5950325d Mon Sep 17 00:00:00 2001 From: Cheng-Yang Chou Date: Wed, 1 Jul 2026 01:05:35 +0800 Subject: sched_ext: Parenthesize local SCX_EVENT defines to fix checkpatch error Fix checkpatch "Macros with complex values should be enclosed in parentheses" error. Suggested-by: Andrea Righi Signed-off-by: Cheng-Yang Chou Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 0c51f9826a77..a9d23d9f5947 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -4979,7 +4979,7 @@ static ssize_t scx_attr_events_show(struct kobject *kobj, int at = 0; scx_read_events(sch, &events); -#define SCX_EVENT(name) at += scx_attr_event_show(buf, at, &events, name) +#define SCX_EVENT(name) (at += scx_attr_event_show(buf, at, &events, name)) SCX_EVENTS_LIST(SCX_EVENT); #undef SCX_EVENT return at; @@ -10309,7 +10309,7 @@ static void scx_read_events(struct scx_sched *sch, struct scx_event_stats *event memset(events, 0, sizeof(*events)); for_each_possible_cpu(cpu) { struct scx_event_stats *e_cpu = &per_cpu_ptr(sch->pcpu, cpu)->event_stats; -#define SCX_EVENT(name) events->name += READ_ONCE(e_cpu->name) +#define SCX_EVENT(name) (events->name += READ_ONCE(e_cpu->name)) SCX_EVENTS_LIST(SCX_EVENT); #undef SCX_EVENT } -- cgit v1.2.3 From 26e5a408b7a7fead67395a81acebd84090559ed6 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 1 Jul 2026 10:28:48 -1000 Subject: sched_ext: Prefix file-local ext.c helpers exposed by the sub.c split A later change moves the sub-scheduler implementation out of ext.c into its own file, from where it calls a number of file-local ext.c helpers. Give those helpers the scx_ prefix that cross-file sched_ext symbols carry, ahead of the move so the mechanical rename stays out of the code-motion patch. No functional change. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 192 ++++++++++++++++++++++---------------------- kernel/sched/ext/internal.h | 2 +- 2 files changed, 97 insertions(+), 97 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 4e0cd08a6a2e..56e6a13fd0f8 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -369,7 +369,7 @@ static const struct sched_class *scx_setscheduler_class(struct task_struct *p) return __setscheduler_class(p->policy, p->prio); } -static struct scx_dispatch_q *bypass_dsq(struct scx_sched *sch, s32 cpu) +static struct scx_dispatch_q *scx_bypass_dsq(struct scx_sched *sch, s32 cpu) { return &per_cpu_ptr(sch->pcpu, cpu)->bypass_dsq; } @@ -392,11 +392,11 @@ static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, s32 c sch = scx_parent(sch); #endif /* CONFIG_EXT_SUB_SCHED */ - return bypass_dsq(sch, cpu); + return scx_bypass_dsq(sch, cpu); } /** - * bypass_dsp_enabled - Check if bypass dispatch path is enabled + * scx_bypass_dsp_enabled - Check if bypass dispatch path is enabled * @sch: scheduler to check * * When a descendant scheduler enters bypass mode, bypassed tasks are scheduled @@ -408,9 +408,9 @@ static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, s32 c * * This function checks bypass_dsp_enable_depth which is managed separately from * bypass_depth to enable this decoupling. See enable_bypass_dsp() and - * disable_bypass_dsp(). + * scx_disable_bypass_dsp(). */ -static bool bypass_dsp_enabled(struct scx_sched *sch) +static bool scx_bypass_dsp_enabled(struct scx_sched *sch) { return unlikely(atomic_read(&sch->bypass_dsp_enable_depth)); } @@ -1079,7 +1079,7 @@ bool scx_cpu_valid(struct scx_sched *sch, s32 cpu, const char *where) } /** - * ops_sanitize_err - Sanitize a -errno value + * scx_ops_sanitize_err - Sanitize a -errno value * @sch: scx_sched to error out on error * @ops_name: operation to blame on failure * @err: -errno value to sanitize @@ -1091,7 +1091,7 @@ bool scx_cpu_valid(struct scx_sched *sch, s32 cpu, const char *where) * value fails IS_ERR() test after being encoded with ERR_PTR() and then is * handled as a pointer. */ -static int ops_sanitize_err(struct scx_sched *sch, const char *ops_name, s32 err) +static int scx_ops_sanitize_err(struct scx_sched *sch, const char *ops_name, s32 err) { if (err < 0 && err >= -MAX_ERRNO) return err; @@ -1251,7 +1251,7 @@ static void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq schedule_deferred(rq); } -static void schedule_reenq_local(struct rq *rq, u64 reenq_flags) +static void scx_schedule_reenq_local(struct rq *rq, u64 reenq_flags) { struct scx_sched *root = rcu_dereference_sched(scx_root); @@ -1347,8 +1347,8 @@ static void dsq_inc_nr(struct scx_dispatch_q *dsq, struct task_struct *p, u64 en * to the CPU or dequeued. In both cases, the only way @p can go back to * the BPF sched is through enqueueing. If being inserted into a local * DSQ with IMMED, persist the state until the next enqueueing event in - * do_enqueue_task() so that we can maintain IMMED protection through - * e.g. SAVE/RESTORE cycles and slice extensions. + * scx_do_enqueue_task() so that we can maintain IMMED protection + * through e.g. SAVE/RESTORE cycles and slice extensions. */ if (enq_flags & SCX_ENQ_IMMED) { if (unlikely(dsq->id != SCX_DSQ_LOCAL)) { @@ -1371,7 +1371,7 @@ static void dsq_inc_nr(struct scx_dispatch_q *dsq, struct task_struct *p, u64 en * done yet, @p can't go on the CPU immediately. Re-enqueue. */ if (unlikely(dsq->nr > 1 || !rq_is_open(rq, enq_flags))) - schedule_reenq_local(rq, 0); + scx_schedule_reenq_local(rq, 0); } } @@ -1488,9 +1488,9 @@ static void local_dsq_post_enq(struct scx_sched *sch, struct scx_dispatch_q *dsq } } -static void dispatch_enqueue(struct scx_sched *sch, struct rq *rq, - struct scx_dispatch_q *dsq, struct task_struct *p, - u64 enq_flags) +static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq, + struct scx_dispatch_q *dsq, struct task_struct *p, + u64 enq_flags) { bool is_local = dsq->id == SCX_DSQ_LOCAL; @@ -1638,7 +1638,7 @@ static void task_unlink_from_dsq(struct task_struct *p, } } -static void dispatch_dequeue(struct rq *rq, struct task_struct *p) +static void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p) { struct scx_dispatch_q *dsq = p->scx.dsq; bool is_local = dsq == &rq->scx.local_dsq; @@ -1692,8 +1692,8 @@ static void dispatch_dequeue(struct rq *rq, struct task_struct *p) } /* - * Abbreviated version of dispatch_dequeue() that can be used when both @p's rq - * and dsq are locked. + * Abbreviated version of scx_dispatch_dequeue() that can be used when both + * @p's rq and dsq are locked. */ static void dispatch_dequeue_locked(struct task_struct *p, struct scx_dispatch_q *dsq) @@ -1774,10 +1774,10 @@ static void mark_direct_dispatch(struct scx_sched *sch, * - direct_dispatch(): cleared on the synchronous enqueue path, deferred * dispatch keeps the state until consumed * - process_ddsp_deferred_locals(): cleared after consuming deferred state, - * - do_enqueue_task(): cleared on enqueue fallbacks where the dispatch + * - scx_do_enqueue_task(): cleared on enqueue fallbacks where the dispatch * verdict is ignored (local/global/bypass) - * - dequeue_task_scx(): cleared after dispatch_dequeue(), covering deferred - * cancellation and holding_cpu races + * - dequeue_task_scx(): cleared after scx_dispatch_dequeue(), covering + * deferred cancellation and holding_cpu races * - scx_disable_task(): cleared for queued wakeup tasks, which are excluded by * the scx_bypass() loop, so that stale state is not reused by a subsequent * scheduler instance @@ -1838,7 +1838,7 @@ static void direct_dispatch(struct scx_sched *sch, struct task_struct *p, ddsp_enq_flags = p->scx.ddsp_enq_flags; clear_direct_dispatch(p); - dispatch_enqueue(sch, rq, dsq, p, ddsp_enq_flags | SCX_ENQ_CLEAR_OPSS); + scx_dispatch_enqueue(sch, rq, dsq, p, ddsp_enq_flags | SCX_ENQ_CLEAR_OPSS); } static bool scx_rq_online(struct rq *rq) @@ -1853,8 +1853,8 @@ static bool scx_rq_online(struct rq *rq) return likely((rq->scx.flags & SCX_RQ_ONLINE) && cpu_active(cpu_of(rq))); } -static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, - int sticky_cpu) +static void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, + int sticky_cpu) { struct scx_sched *sch = scx_task_sched(p); struct task_struct **ddsp_taskp; @@ -1941,7 +1941,7 @@ direct: direct_dispatch(sch, p, enq_flags); return; local_norefill: - dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, enq_flags); + scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, enq_flags); return; local: dsq = &rq->scx.local_dsq; @@ -1962,7 +1962,7 @@ enqueue: touch_core_sched(rq, p); refill_task_slice_dfl(sch, p); clear_direct_dispatch(p); - dispatch_enqueue(sch, rq, dsq, p, enq_flags); + scx_dispatch_enqueue(sch, rq, dsq, p, enq_flags); } static bool task_runnable(const struct task_struct *p) @@ -2031,7 +2031,7 @@ static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_ if (rq->scx.nr_running == 1) dl_server_start(&rq->ext_server); - do_enqueue_task(rq, p, enq_flags, sticky_cpu); + scx_do_enqueue_task(rq, p, enq_flags, sticky_cpu); if (sticky_cpu >= 0) p->scx.sticky_cpu = -1; @@ -2167,7 +2167,7 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_ rq->scx.nr_running--; sub_nr_running(rq, 1); - dispatch_dequeue(rq, p); + scx_dispatch_dequeue(rq, p); clear_direct_dispatch(p); return true; } @@ -2215,7 +2215,7 @@ static void wakeup_preempt_scx(struct rq *rq, struct task_struct *p, int wake_fl * - A higher-priority wakes up while SCX dispatch is in progress. */ if (rq->scx.nr_immed) - schedule_reenq_local(rq, 0); + scx_schedule_reenq_local(rq, 0); } static void move_local_task_to_local_dsq(struct scx_sched *sch, @@ -2380,7 +2380,7 @@ static bool task_can_run_on_remote_rq(struct scx_sched *sch, * values afterwards, as this operation can't be preempted or recurse, the * holding_cpu can never become this CPU again before we're done. Thus, we can * tell whether we lost to dequeue by testing whether the holding_cpu still - * points to this CPU. See dispatch_dequeue() for the counterpart. + * points to this CPU. See scx_dispatch_dequeue() for the counterpart. * * On return, @dsq is unlocked and @src_rq is locked. Returns %true if @p is * still valid. %false if lost to dequeue. @@ -2485,14 +2485,14 @@ static struct rq *move_task_between_dsqs(struct scx_sched *sch, dispatch_dequeue_locked(p, src_dsq); raw_spin_unlock(&src_dsq->lock); - dispatch_enqueue(sch, dst_rq, dst_dsq, p, enq_flags); + scx_dispatch_enqueue(sch, dst_rq, dst_dsq, p, enq_flags); } return dst_rq; } -static bool consume_dispatch_q(struct scx_sched *sch, struct rq *rq, - struct scx_dispatch_q *dsq, u64 enq_flags) +static bool scx_consume_dispatch_q(struct scx_sched *sch, struct rq *rq, + struct scx_dispatch_q *dsq, u64 enq_flags) { struct task_struct *p; retry: @@ -2538,11 +2538,11 @@ retry: return false; } -static bool consume_global_dsq(struct scx_sched *sch, struct rq *rq) +static bool scx_consume_global_dsq(struct scx_sched *sch, struct rq *rq) { int node = cpu_to_node(cpu_of(rq)); - return consume_dispatch_q(sch, rq, &sch->pnode[node]->global_dsq, 0); + return scx_consume_dispatch_q(sch, rq, &sch->pnode[node]->global_dsq, 0); } /** @@ -2575,8 +2575,8 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, * If dispatching to @rq that @p is already on, no lock dancing needed. */ if (rq == src_rq && rq == dst_rq) { - dispatch_enqueue(sch, rq, dst_dsq, p, - enq_flags | SCX_ENQ_CLEAR_OPSS); + scx_dispatch_enqueue(sch, rq, dst_dsq, p, + enq_flags | SCX_ENQ_CLEAR_OPSS); return; } @@ -2614,13 +2614,13 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, */ if (src_rq == dst_rq) { p->scx.holding_cpu = -1; - dispatch_enqueue(sch, dst_rq, &dst_rq->scx.local_dsq, p, - enq_flags); + scx_dispatch_enqueue(sch, dst_rq, &dst_rq->scx.local_dsq, p, + enq_flags); } else if (unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) { p->scx.holding_cpu = -1; fallback = true; - dispatch_enqueue(sch, src_rq, find_global_dsq(sch, task_cpu(p)), - p, enq_flags | SCX_ENQ_GDSQ_FALLBACK); + scx_dispatch_enqueue(sch, src_rq, find_global_dsq(sch, task_cpu(p)), + p, enq_flags | SCX_ENQ_GDSQ_FALLBACK); } else { move_remote_task_to_local_dsq(p, enq_flags, src_rq, dst_rq); @@ -2708,10 +2708,10 @@ retry: goto retry; case SCX_OPSS_QUEUEING: /* - * do_enqueue_task() is in the process of transferring the task - * to the BPF scheduler while holding @p's rq lock. As we aren't - * holding any kernel or BPF resource that the enqueue path may - * depend upon, it's safe to wait. + * scx_do_enqueue_task() is in the process of transferring the + * task to the BPF scheduler while holding @p's rq lock. As we + * aren't holding any kernel or BPF resource that the enqueue + * path may depend upon, it's safe to wait. */ wait_ops_state(p, opss); goto retry; @@ -2724,10 +2724,10 @@ retry: if (dsq->id == SCX_DSQ_LOCAL) dispatch_to_local_dsq(sch, rq, dsq, p, enq_flags); else - dispatch_enqueue(sch, rq, dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); + scx_dispatch_enqueue(sch, rq, dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); } -static void flush_dispatch_buf(struct scx_sched *sch, struct rq *rq) +static void scx_flush_dispatch_buf(struct scx_sched *sch, struct rq *rq) { struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; u32 u; @@ -2771,13 +2771,13 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, bool prev_on_sch = (prev->sched_class == &ext_sched_class) && scx_task_on_sched(sch, prev); - if (consume_global_dsq(sch, rq)) + if (scx_consume_global_dsq(sch, rq)) return true; - if (bypass_dsp_enabled(sch)) { + if (scx_bypass_dsp_enabled(sch)) { /* if @sch is bypassing, only the bypass DSQs are active */ if (scx_bypassing(sch, cpu)) - return consume_dispatch_q(sch, rq, bypass_dsq(sch, cpu), 0); + return scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0); #ifdef CONFIG_EXT_SUB_SCHED /* @@ -2795,7 +2795,7 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); if (!(pcpu->bypass_host_seq++ % SCX_BYPASS_HOST_NTH) && - consume_dispatch_q(sch, rq, bypass_dsq(sch, cpu), 0)) { + scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0)) { __scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1); return true; } @@ -2808,8 +2808,8 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, dspc->rq = rq; /* - * The dispatch loop. Because flush_dispatch_buf() may drop the rq lock, - * the local DSQ might still end up empty after a successful + * The dispatch loop. Because scx_flush_dispatch_buf() may drop the rq + * lock, the local DSQ might still end up empty after a successful * ops.dispatch(). If the local DSQ is empty even after ops.dispatch() * produced some tasks, retry. The BPF scheduler may depend on this * looping behavior to simplify its implementation. @@ -2828,7 +2828,7 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, rq->scx.sub_dispatch_prev = NULL; } - flush_dispatch_buf(sch, rq); + scx_flush_dispatch_buf(sch, rq); if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice) { rq->scx.flags |= SCX_RQ_BAL_KEEP; @@ -2836,7 +2836,7 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, } if (rq->scx.local_dsq.nr) return true; - if (consume_global_dsq(sch, rq)) + if (scx_consume_global_dsq(sch, rq)) return true; /* @@ -2859,8 +2859,8 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, * queued. Without this fallback, bypassed tasks could stall if the host * scheduler's ops.dispatch() doesn't yield any tasks. */ - if (bypass_dsp_enabled(sch)) - return consume_dispatch_q(sch, rq, bypass_dsq(sch, cpu), 0); + if (scx_bypass_dsp_enabled(sch)) + return scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0); return false; } @@ -2939,7 +2939,7 @@ has_tasks: * between the IMMED queueing and the subsequent scheduling event. */ if (unlikely(rq->scx.local_dsq.nr > 1 && rq->scx.nr_immed)) - schedule_reenq_local(rq, 0); + scx_schedule_reenq_local(rq, 0); rq->scx.flags &= ~SCX_RQ_IN_BALANCE; return true; @@ -2955,7 +2955,7 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first) * dispatched. Call ops_dequeue() to notify the BPF scheduler. */ ops_dequeue(rq, p, SCX_DEQ_CORE_SCHED_EXEC); - dispatch_dequeue(rq, p); + scx_dispatch_dequeue(rq, p); } p->se.exec_start = rq_clock_task(rq); @@ -3067,10 +3067,10 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, if (p->scx.slice && !scx_bypassing(sch, cpu_of(rq))) { if (p->scx.flags & SCX_TASK_IMMED) { p->scx.flags |= SCX_TASK_REENQ_PREEMPTED; - do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); + scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; } else { - dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, SCX_ENQ_HEAD); + scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, SCX_ENQ_HEAD); } goto switch_class; } @@ -3088,9 +3088,9 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, if (next && sched_class_above(&ext_sched_class, next->sched_class)) { WARN_ON_ONCE(sched_cpu_cookie_match(rq, p) && !(sch->ops.flags & SCX_OPS_ENQ_LAST)); - do_enqueue_task(rq, p, SCX_ENQ_LAST, -1); + scx_do_enqueue_task(rq, p, SCX_ENQ_LAST, -1); } else { - do_enqueue_task(rq, p, 0, -1); + scx_do_enqueue_task(rq, p, 0, -1); } } @@ -3562,7 +3562,7 @@ static int __scx_init_task(struct scx_sched *sch, struct task_struct *p, bool fo ret = SCX_CALL_OP_RET(sch, init_task, NULL, p, &args); if (unlikely(ret)) { - ret = ops_sanitize_err(sch, "init_task", ret); + ret = scx_ops_sanitize_err(sch, "init_task", ret); return ret; } } @@ -4107,7 +4107,7 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) if (!local_task_should_reenq(p, &reenq_flags, &reason)) continue; - dispatch_dequeue(rq, p); + scx_dispatch_dequeue(rq, p); if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; @@ -4119,7 +4119,7 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) { list_del_init(&p->scx.dsq_list.node); - do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); + scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; nr_enqueued++; @@ -4234,7 +4234,7 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; p->scx.flags |= reason; - do_enqueue_task(task_rq, p, SCX_ENQ_REENQ, -1); + scx_do_enqueue_task(task_rq, p, SCX_ENQ_REENQ, -1); p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; @@ -4354,7 +4354,7 @@ int scx_tg_online(struct task_group *tg) ret = SCX_CALL_OP_RET(sch, cgroup_init, NULL, tg->css.cgroup, &args); if (ret) - ret = ops_sanitize_err(sch, "cgroup_init", ret); + ret = scx_ops_sanitize_err(sch, "cgroup_init", ret); } if (ret == 0) tg->scx.flags |= SCX_TG_ONLINE | SCX_TG_INITED; @@ -4422,7 +4422,7 @@ err: p->scx.cgrp_moving_from = NULL; } - return ops_sanitize_err(sch, "cgroup_prep_move", ret); + return scx_ops_sanitize_err(sch, "cgroup_prep_move", ret); } void scx_cgroup_move_task(struct task_struct *p) @@ -4700,7 +4700,7 @@ static void destroy_dsq(struct scx_sched *sch, u64 dsq_id) goto out_unlock_dsq; /* - * Mark dead by invalidating ->id to prevent dispatch_enqueue() from + * Mark dead by invalidating ->id to prevent scx_dispatch_enqueue() from * queueing more tasks. As this function can be called from anywhere, * freeing is bounced through an irq work to avoid nesting RCU * operations inside scheduler locks. @@ -4928,7 +4928,7 @@ static void scx_sched_free_rcu_work(struct work_struct *work) */ WARN_ON_ONCE(!list_empty(&pcpu->deferred_reenq_local.node)); - exit_dsq(bypass_dsq(sch, cpu)); + exit_dsq(scx_bypass_dsq(sch, cpu)); } free_percpu(sch->pcpu); @@ -5239,7 +5239,7 @@ static u32 bypass_lb_cpu(struct scx_sched *sch, s32 donor, u32 nr_donor_target, u32 nr_donee_target) { struct rq *donor_rq = cpu_rq(donor); - struct scx_dispatch_q *donor_dsq = bypass_dsq(sch, donor); + struct scx_dispatch_q *donor_dsq = scx_bypass_dsq(sch, donor); struct task_struct *p, *n; struct scx_dsq_list_node cursor = INIT_DSQ_LIST_CURSOR(cursor, donor_dsq, 0); s32 delta = READ_ONCE(donor_dsq->nr) - nr_donor_target; @@ -5287,7 +5287,7 @@ resume: if (donee >= nr_cpu_ids) continue; - donee_dsq = bypass_dsq(sch, donee); + donee_dsq = scx_bypass_dsq(sch, donee); /* * $p's rq is not locked but $p's DSQ lock protects its @@ -5308,7 +5308,7 @@ resume: * between bypass DSQs. */ dispatch_dequeue_locked(p, donor_dsq); - dispatch_enqueue(sch, cpu_rq(donee), donee_dsq, p, SCX_ENQ_NESTED); + scx_dispatch_enqueue(sch, cpu_rq(donee), donee_dsq, p, SCX_ENQ_NESTED); /* * $donee might have been idle and need to be woken up. No need @@ -5351,7 +5351,7 @@ static void bypass_lb_node(struct scx_sched *sch, int node) /* count the target tasks and CPUs */ for_each_cpu_and(cpu, cpu_online_mask, node_mask) { - u32 nr = READ_ONCE(bypass_dsq(sch, cpu)->nr); + u32 nr = READ_ONCE(scx_bypass_dsq(sch, cpu)->nr); nr_tasks += nr; nr_cpus++; @@ -5373,7 +5373,7 @@ static void bypass_lb_node(struct scx_sched *sch, int node) cpumask_clear(donee_mask); for_each_cpu_and(cpu, cpu_online_mask, node_mask) { - if (READ_ONCE(bypass_dsq(sch, cpu)->nr) < nr_target) + if (READ_ONCE(scx_bypass_dsq(sch, cpu)->nr) < nr_target) cpumask_set_cpu(cpu, donee_mask); } @@ -5384,7 +5384,7 @@ static void bypass_lb_node(struct scx_sched *sch, int node) break; if (cpumask_test_cpu(cpu, donee_mask)) continue; - if (READ_ONCE(bypass_dsq(sch, cpu)->nr) <= nr_donor_target) + if (READ_ONCE(scx_bypass_dsq(sch, cpu)->nr) <= nr_donor_target) continue; nr_balanced += bypass_lb_cpu(sch, cpu, donee_mask, resched_mask, @@ -5395,7 +5395,7 @@ static void bypass_lb_node(struct scx_sched *sch, int node) resched_cpu(cpu); for_each_cpu_and(cpu, cpu_online_mask, node_mask) { - u32 nr = READ_ONCE(bypass_dsq(sch, cpu)->nr); + u32 nr = READ_ONCE(scx_bypass_dsq(sch, cpu)->nr); after_min = min(nr, after_min); after_max = max(nr, after_max); @@ -5421,7 +5421,7 @@ static void scx_bypass_lb_timerfn(struct timer_list *timer) int node; u32 intv_us; - if (!bypass_dsp_enabled(sch)) + if (!scx_bypass_dsp_enabled(sch)) return; for_each_node_with_cpus(node) @@ -5487,9 +5487,9 @@ static void enable_bypass_dsp(struct scx_sched *sch) * dispatch enabled while a descendant is bypassing, which is all that's * required. * - * bypass_dsp_enabled() test is used to determine whether to enter the - * bypass dispatch handling path from both bypassing and hosting scheds. - * Bump enable depth on both @sch and bypass dispatch host. + * scx_bypass_dsp_enabled() test is used to determine whether to enter + * the bypass dispatch handling path from both bypassing and hosting + * scheds. Bump enable depth on both @sch and bypass dispatch host. */ ret = atomic_inc_return(&sch->bypass_dsp_enable_depth); WARN_ON_ONCE(ret <= 0); @@ -5509,7 +5509,7 @@ static void enable_bypass_dsp(struct scx_sched *sch) } /* may be called without holding scx_bypass_lock */ -static void disable_bypass_dsp(struct scx_sched *sch) +static void scx_disable_bypass_dsp(struct scx_sched *sch) { s32 ret; @@ -5654,7 +5654,7 @@ static void scx_bypass(struct scx_sched *sch, bool bypass) /* disarming must come after moving all tasks out of the bypass DSQs */ if (!bypass) - disable_bypass_dsp(sch); + scx_disable_bypass_dsp(sch); unlock: raw_spin_unlock_irqrestore(&scx_bypass_lock, flags); } @@ -6003,7 +6003,7 @@ static void scx_sub_disable(struct scx_sched *sch) * DSQs for us. */ synchronize_rcu_expedited(); - disable_bypass_dsp(sch); + scx_disable_bypass_dsp(sch); scx_unlink_sched(sch); @@ -6810,7 +6810,7 @@ static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, } for_each_possible_cpu(cpu) { - ret = init_dsq(bypass_dsq(sch, cpu), SCX_DSQ_BYPASS, sch); + ret = init_dsq(scx_bypass_dsq(sch, cpu), SCX_DSQ_BYPASS, sch); if (ret) { bypass_fail_cpu = cpu; goto err_free_pcpu; @@ -6963,7 +6963,7 @@ err_free_pcpu: for_each_possible_cpu(cpu) { if (cpu == bypass_fail_cpu) break; - exit_dsq(bypass_dsq(sch, cpu)); + exit_dsq(scx_bypass_dsq(sch, cpu)); } free_percpu(sch->pcpu); err_free_pnode: @@ -7007,7 +7007,7 @@ static int check_hotplug_seq(struct scx_sched *sch, return 0; } -static int validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops) +static int scx_validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops) { /* * It doesn't make sense to specify the SCX_OPS_ENQ_LAST flag if the @@ -7170,7 +7170,7 @@ static void scx_root_enable_workfn(struct kthread_work *work) if (sch->ops.init) { ret = SCX_CALL_OP_RET(sch, init, NULL); if (ret) { - ret = ops_sanitize_err(sch, "init", ret); + ret = scx_ops_sanitize_err(sch, "init", ret); cpus_read_unlock(); scx_error(sch, "ops.init() failed (%d)", ret); goto err_disable; @@ -7203,7 +7203,7 @@ static void scx_root_enable_workfn(struct kthread_work *work) cpus_read_unlock(); - ret = validate_ops(sch, ops); + ret = scx_validate_ops(sch, ops); if (ret) goto err_disable; @@ -7545,7 +7545,7 @@ static void scx_sub_enable_workfn(struct kthread_work *work) if (sch->ops.init) { ret = SCX_CALL_OP_RET(sch, init, NULL); if (ret) { - ret = ops_sanitize_err(sch, "init", ret); + ret = scx_ops_sanitize_err(sch, "init", ret); scx_error(sch, "ops.init() failed (%d)", ret); goto err_disable; } @@ -7560,7 +7560,7 @@ static void scx_sub_enable_workfn(struct kthread_work *work) if (ret) goto err_disable; - if (validate_ops(sch, ops)) + if (scx_validate_ops(sch, ops)) goto err_disable; struct scx_sub_attach_args sub_attach_args = { @@ -7571,7 +7571,7 @@ static void scx_sub_enable_workfn(struct kthread_work *work) ret = SCX_CALL_OP_RET(parent, sub_attach, NULL, &sub_attach_args); if (ret) { - ret = ops_sanitize_err(sch, "sub_attach", ret); + ret = scx_ops_sanitize_err(sch, "sub_attach", ret); scx_error(sch, "parent rejected (%d)", ret); goto err_disable; } @@ -8830,7 +8830,7 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, /* * If the BPF scheduler keeps calling this function repeatedly, it can - * cause similar live-lock conditions as consume_dispatch_q(). + * cause similar live-lock conditions as scx_consume_dispatch_q(). */ if (unlikely(READ_ONCE(sch->aborting))) return false; @@ -8991,7 +8991,7 @@ __bpf_kfunc bool scx_bpf_dsq_move_to_local___v2(u64 dsq_id, u64 enq_flags, dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; - flush_dispatch_buf(sch, dspc->rq); + scx_flush_dispatch_buf(sch, dspc->rq); dsq = find_user_dsq(sch, dsq_id); if (unlikely(!dsq)) { @@ -8999,7 +8999,7 @@ __bpf_kfunc bool scx_bpf_dsq_move_to_local___v2(u64 dsq_id, u64 enq_flags, return false; } - if (consume_dispatch_q(sch, dspc->rq, dsq, enq_flags)) { + if (scx_consume_dispatch_q(sch, dspc->rq, dsq, enq_flags)) { /* * A successfully consumed task can be dequeued before it starts * running while the CPU is trying to migrate other dispatched @@ -10683,7 +10683,7 @@ static int __init scx_init(void) /* @priv tail must align since both share the same data block */ CID_OFFSET_MATCH(priv, priv); /* - * cid-form must end exactly at @priv - validate_ops() skips + * cid-form must end exactly at @priv - scx_validate_ops() skips * cpu_acquire/cpu_release for cid-form because reading those fields * past the BPF allocation would be UB. */ diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 0256931a379a..743980dc60b0 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1172,7 +1172,7 @@ struct scx_sched { u64 bypass_timestamp; s32 bypass_depth; - /* bypass dispatch path enable state, see bypass_dsp_enabled() */ + /* bypass dispatch path enable state, see scx_bypass_dsp_enabled() */ unsigned long bypass_dsp_claim; atomic_t bypass_dsp_enable_depth; -- cgit v1.2.3 From a56469087c8065f90833762450be04a600c47e9b Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 1 Jul 2026 10:28:48 -1000 Subject: sched_ext: Expose the ext.c internals used by the sub.c split The sub-scheduler implementation is about to move into its own sub.c, from where it calls a set of ext.c helpers and shares a few ext.c globals. Make those reachable across the new file boundary ahead of the move. No functional change. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 116 +++++++++++++++----------------------------- kernel/sched/ext/internal.h | 80 ++++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 77 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 56e6a13fd0f8..bdbc66466962 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -20,7 +20,7 @@ #include "arena.h" #include "idle.h" -static DEFINE_RAW_SPINLOCK(scx_sched_lock); +DEFINE_RAW_SPINLOCK(scx_sched_lock); /* * NOTE: sched_ext is in the process of growing multiple scheduler support and @@ -39,14 +39,14 @@ struct scx_sched __rcu *scx_root; static LIST_HEAD(scx_sched_all); #ifdef CONFIG_EXT_SUB_SCHED -static const struct rhashtable_params scx_sched_hash_params = { +const struct rhashtable_params scx_sched_hash_params = { .key_len = sizeof_field(struct scx_sched, ops.sub_cgroup_id), .key_offset = offsetof(struct scx_sched, ops.sub_cgroup_id), .head_offset = offsetof(struct scx_sched, hash_node), .insecure_elasticity = true, /* inserted under scx_sched_lock */ }; -static struct rhashtable scx_sched_hash; +struct rhashtable scx_sched_hash; #endif /* see SCX_OPS_TID_TO_TASK */ @@ -68,9 +68,9 @@ static DEFINE_RAW_SPINLOCK(scx_tasks_lock); static LIST_HEAD(scx_tasks); /* ops enable/disable */ -static DEFINE_MUTEX(scx_enable_mutex); +DEFINE_MUTEX(scx_enable_mutex); DEFINE_STATIC_KEY_FALSE(__scx_enabled); -DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem); +DEFINE_PERCPU_RWSEM(scx_fork_rwsem); static atomic_t scx_enable_state_var = ATOMIC_INIT(SCX_DISABLED); static DEFINE_RAW_SPINLOCK(scx_bypass_lock); static bool scx_init_task_enabled; @@ -101,7 +101,7 @@ static atomic64_t scx_tid_cursor = ATOMIC64_INIT(1); * tasks for the sub-sched being enabled. Use a global variable instead of a * per-task field as all enables are serialized. */ -static struct scx_sched *scx_enabling_sub_sched; +struct scx_sched *scx_enabling_sub_sched; #else #define scx_enabling_sub_sched (struct scx_sched *)NULL #endif /* CONFIG_EXT_SUB_SCHED */ @@ -242,7 +242,6 @@ MODULE_PARM_DESC(bypass_lb_intv_us, "bypass load balance interval in microsecond static void run_deferred(struct rq *rq); static bool task_dead_and_done(struct task_struct *p); -static void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags); static void scx_disable(struct scx_sched *sch, enum scx_exit_kind kind); __printf(5, 6) bool __scx_exit(struct scx_sched *sch, @@ -676,12 +675,12 @@ struct bpf_iter_scx_dsq { } __attribute__((aligned(8))); -static u32 scx_get_task_state(const struct task_struct *p) +u32 scx_get_task_state(const struct task_struct *p) { return p->scx.flags & SCX_TASK_STATE_MASK; } -static void scx_set_task_state(struct task_struct *p, u32 state) +void scx_set_task_state(struct task_struct *p, u32 state) { u32 prev_state = scx_get_task_state(p); bool warn = false; @@ -721,23 +720,6 @@ static void scx_set_task_state(struct task_struct *p, u32 state) p->scx.flags |= state; } -/* - * SCX task iterator. - */ -struct scx_task_iter { - struct sched_ext_entity cursor; - struct task_struct *locked_task; - struct rq *rq; - struct rq_flags rf; - u32 cnt; - bool list_locked; -#ifdef CONFIG_EXT_SUB_SCHED - struct cgroup *cgrp; - struct cgroup_subsys_state *css_pos; - struct css_task_iter css_iter; -#endif -}; - /** * scx_task_iter_start - Lock scx_tasks_lock and start a task iteration * @iter: iterator to init @@ -766,7 +748,7 @@ struct scx_task_iter { * All tasks which existed when the iteration started are guaranteed to be * visited as long as they are not dead. */ -static void scx_task_iter_start(struct scx_task_iter *iter, struct cgroup *cgrp) +void scx_task_iter_start(struct scx_task_iter *iter, struct cgroup *cgrp) { memset(iter, 0, sizeof(*iter)); @@ -805,7 +787,7 @@ static void __scx_task_iter_rq_unlock(struct scx_task_iter *iter) * This function can be safely called anytime during an iteration. The next * iterator operation will automatically restore the necessary locking. */ -static void scx_task_iter_unlock(struct scx_task_iter *iter) +void scx_task_iter_unlock(struct scx_task_iter *iter) { __scx_task_iter_rq_unlock(iter); if (iter->list_locked) { @@ -848,7 +830,7 @@ static void scx_task_iter_relock(struct scx_task_iter *iter, * which is released on return. If the iterator holds a task's rq lock, that rq * lock is also released. See scx_task_iter_start() for details. */ -static void scx_task_iter_stop(struct scx_task_iter *iter) +void scx_task_iter_stop(struct scx_task_iter *iter) { #ifdef CONFIG_EXT_SUB_SCHED if (iter->cgrp) { @@ -923,7 +905,7 @@ static struct task_struct *scx_task_iter_next(struct scx_task_iter *iter) * whether they would like to filter out dead tasks. See scx_task_iter_start() * for details. */ -static struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter) +struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter) { struct task_struct *p; @@ -1186,8 +1168,8 @@ static void schedule_deferred_locked(struct rq *rq) schedule_deferred(rq); } -static void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, - u64 reenq_flags, struct rq *locked_rq) +void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, + u64 reenq_flags, struct rq *locked_rq) { struct rq *rq; @@ -1841,7 +1823,7 @@ static void direct_dispatch(struct scx_sched *sch, struct task_struct *p, scx_dispatch_enqueue(sch, rq, dsq, p, ddsp_enq_flags | SCX_ENQ_CLEAR_OPSS); } -static bool scx_rq_online(struct rq *rq) +bool scx_rq_online(struct rq *rq) { /* * Test both cpu_active() and %SCX_RQ_ONLINE. %SCX_RQ_ONLINE indicates @@ -2491,8 +2473,8 @@ static struct rq *move_task_between_dsqs(struct scx_sched *sch, return dst_rq; } -static bool scx_consume_dispatch_q(struct scx_sched *sch, struct rq *rq, - struct scx_dispatch_q *dsq, u64 enq_flags) +bool scx_consume_dispatch_q(struct scx_sched *sch, struct rq *rq, + struct scx_dispatch_q *dsq, u64 enq_flags) { struct task_struct *p; retry: @@ -2538,7 +2520,7 @@ retry: return false; } -static bool scx_consume_global_dsq(struct scx_sched *sch, struct rq *rq) +bool scx_consume_global_dsq(struct scx_sched *sch, struct rq *rq) { int node = cpu_to_node(cpu_of(rq)); @@ -2727,7 +2709,7 @@ retry: scx_dispatch_enqueue(sch, rq, dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); } -static void scx_flush_dispatch_buf(struct scx_sched *sch, struct rq *rq) +void scx_flush_dispatch_buf(struct scx_sched *sch, struct rq *rq) { struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; u32 u; @@ -3548,7 +3530,7 @@ static struct cgroup *tg_cgrp(struct task_group *tg) #endif /* CONFIG_EXT_GROUP_SCHED */ -static int __scx_init_task(struct scx_sched *sch, struct task_struct *p, bool fork) +int __scx_init_task(struct scx_sched *sch, struct task_struct *p, bool fork) { int ret; @@ -3631,7 +3613,7 @@ static void __scx_enable_task(struct scx_sched *sch, struct task_struct *p) SCX_CALL_OP_TASK(sch, set_weight, rq, p, p->scx.weight); } -static void scx_enable_task(struct scx_sched *sch, struct task_struct *p) +void scx_enable_task(struct scx_sched *sch, struct task_struct *p) { __scx_enable_task(sch, p); scx_set_task_state(p, SCX_TASK_ENABLED); @@ -3665,8 +3647,7 @@ static void scx_disable_task(struct scx_sched *sch, struct task_struct *p) WARN_ON_ONCE(p->scx.flags & SCX_TASK_IN_CUSTODY); } -static void __scx_disable_and_exit_task(struct scx_sched *sch, - struct task_struct *p) +void __scx_disable_and_exit_task(struct scx_sched *sch, struct task_struct *p) { struct scx_exit_task_args args = { .cancelled = false, @@ -3700,7 +3681,7 @@ static void __scx_disable_and_exit_task(struct scx_sched *sch, * ran. The task state has not been transitioned, so this mirrors the * SCX_TASK_INIT branch in __scx_disable_and_exit_task(). */ -static void scx_sub_init_cancel_task(struct scx_sched *sch, struct task_struct *p) +void scx_sub_init_cancel_task(struct scx_sched *sch, struct task_struct *p) { struct scx_exit_task_args args = { .cancelled = true }; @@ -3711,8 +3692,7 @@ static void scx_sub_init_cancel_task(struct scx_sched *sch, struct task_struct * SCX_CALL_OP_TASK(sch, exit_task, task_rq(p), p, &args); } -static void scx_disable_and_exit_task(struct scx_sched *sch, - struct task_struct *p) +void scx_disable_and_exit_task(struct scx_sched *sch, struct task_struct *p) { __scx_disable_and_exit_task(sch, p); @@ -4525,7 +4505,7 @@ static struct cgroup *root_cgroup(void) return &cgrp_dfl_root.cgrp; } -static void scx_cgroup_lock(void) +void scx_cgroup_lock(void) { #ifdef CONFIG_EXT_GROUP_SCHED percpu_down_write(&scx_cgroup_ops_rwsem); @@ -4533,7 +4513,7 @@ static void scx_cgroup_lock(void) cgroup_lock(); } -static void scx_cgroup_unlock(void) +void scx_cgroup_unlock(void) { cgroup_unlock(); #ifdef CONFIG_EXT_GROUP_SCHED @@ -4851,7 +4831,7 @@ static void free_exit_info(struct scx_exit_info *ei); static const char *scx_exit_reason(enum scx_exit_kind kind); static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind); -static s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch) +s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch) { size_t size = struct_size_t(struct scx_cmask, bits, SCX_CMASK_NR_WORDS(num_possible_cpus())); @@ -5509,7 +5489,7 @@ static void enable_bypass_dsp(struct scx_sched *sch) } /* may be called without holding scx_bypass_lock */ -static void scx_disable_bypass_dsp(struct scx_sched *sch) +void scx_disable_bypass_dsp(struct scx_sched *sch) { s32 ret; @@ -5557,7 +5537,7 @@ static void scx_disable_bypass_dsp(struct scx_sched *sch) * * - scx_prio_less() reverts to the default core_sched_at order. */ -static void scx_bypass(struct scx_sched *sch, bool bypass) +void scx_bypass(struct scx_sched *sch, bool bypass) { struct scx_sched *pos; unsigned long flags; @@ -5746,7 +5726,7 @@ static void refresh_watchdog(void) cancel_delayed_work_sync(&scx_watchdog_work); } -static s32 scx_link_sched(struct scx_sched *sch) +s32 scx_link_sched(struct scx_sched *sch) { const char *err_msg = ""; s32 ret = 0; @@ -5795,7 +5775,7 @@ static s32 scx_link_sched(struct scx_sched *sch) return 0; } -static void scx_unlink_sched(struct scx_sched *sch) +void scx_unlink_sched(struct scx_sched *sch) { scoped_guard(raw_spinlock_irq, &scx_sched_lock) { #ifdef CONFIG_EXT_SUB_SCHED @@ -5816,13 +5796,13 @@ static void scx_unlink_sched(struct scx_sched *sch) * @sch. Once @sch becomes empty during disable, there's no point in dumping it. * This prevents calling dump ops on a dead sch. */ -static void scx_disable_dump(struct scx_sched *sch) +void scx_disable_dump(struct scx_sched *sch) { guard(raw_spinlock_irqsave)(&scx_dump_lock); sch->dump_disabled = true; } -static void scx_log_sched_disable(struct scx_sched *sch) +void scx_log_sched_disable(struct scx_sched *sch) { struct scx_exit_info *ei = sch->exit_info; const char *type = scx_parent(sch) ? "sub-scheduler" : "scheduler"; @@ -6285,7 +6265,7 @@ static void scx_disable(struct scx_sched *sch, enum scx_exit_kind kind) * as a noop. Syncing the irq_work first is required to guarantee the * kthread work has been queued before waiting for it. */ -static void scx_flush_disable_work(struct scx_sched *sch) +void scx_flush_disable_work(struct scx_sched *sch) { int kind; @@ -6739,31 +6719,13 @@ static struct scx_sched_pnode *alloc_pnode(struct scx_sched *sch, int node) return pnode; } -/* - * scx_enable() is offloaded to a dedicated system-wide RT kthread to avoid - * starvation. During the READY -> ENABLED task switching loop, the calling - * thread's sched_class gets switched from fair to ext. As fair has higher - * priority than ext, the calling thread can be indefinitely starved under - * fair-class saturation, leading to a system hang. - */ -struct scx_enable_cmd { - struct kthread_work work; - union { - struct sched_ext_ops *ops; - struct sched_ext_ops_cid *ops_cid; - }; - bool is_cid_type; - struct bpf_map *arena_map; /* arena ref to transfer to sch */ - int ret; -}; - /* * Allocate and initialize a new scx_sched. @cgrp's reference is always * consumed whether the function succeeds or fails. */ -static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, - struct cgroup *cgrp, - struct scx_sched *parent) +struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, + struct cgroup *cgrp, + struct scx_sched *parent) { struct sched_ext_ops *ops = cmd->ops; struct scx_sched *sch; @@ -7007,7 +6969,7 @@ static int check_hotplug_seq(struct scx_sched *sch, return 0; } -static int scx_validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops) +int scx_validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops) { /* * It doesn't make sense to specify the SCX_OPS_ENQ_LAST flag if the @@ -9342,7 +9304,7 @@ __bpf_kfunc bool scx_bpf_task_set_dsq_vtime(struct task_struct *p, u64 vtime, return true; } -static void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags) +void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags) { struct rq *this_rq; unsigned long irq_flags; diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 743980dc60b0..c4a910d2ca91 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1535,6 +1535,41 @@ enum scx_ops_state { #define SCX_OPSS_STATE_MASK ((1LU << SCX_OPSS_QSEQ_SHIFT) - 1) #define SCX_OPSS_QSEQ_MASK (~SCX_OPSS_STATE_MASK) +/* + * SCX task iterator. + */ +struct scx_task_iter { + struct sched_ext_entity cursor; + struct task_struct *locked_task; + struct rq *rq; + struct rq_flags rf; + u32 cnt; + bool list_locked; +#ifdef CONFIG_EXT_SUB_SCHED + struct cgroup *cgrp; + struct cgroup_subsys_state *css_pos; + struct css_task_iter css_iter; +#endif +}; + +/* + * scx_enable() is offloaded to a dedicated system-wide RT kthread to avoid + * starvation. During the READY -> ENABLED task switching loop, the calling + * thread's sched_class gets switched from fair to ext. As fair has higher + * priority than ext, the calling thread can be indefinitely starved under + * fair-class saturation, leading to a system hang. + */ +struct scx_enable_cmd { + struct kthread_work work; + union { + struct sched_ext_ops *ops; + struct sched_ext_ops_cid *ops_cid; + }; + bool is_cid_type; + struct bpf_map *arena_map; /* arena ref to transfer to sch */ + int ret; +}; + extern struct scx_sched __rcu *scx_root; DECLARE_PER_CPU(struct rq *, scx_locked_rq_state); @@ -1555,6 +1590,51 @@ __printf(5, 0) bool scx_vexit(struct scx_sched *sch, enum scx_exit_kind kind, __printf(5, 6) bool __scx_exit(struct scx_sched *sch, enum scx_exit_kind kind, s64 exit_code, s32 exit_cpu, const char *fmt, ...); +u32 scx_get_task_state(const struct task_struct *p); +void scx_set_task_state(struct task_struct *p, u32 state); +void scx_task_iter_start(struct scx_task_iter *iter, struct cgroup *cgrp); +void scx_task_iter_unlock(struct scx_task_iter *iter); +void scx_task_iter_stop(struct scx_task_iter *iter); +struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter); +bool scx_consume_dispatch_q(struct scx_sched *sch, struct rq *rq, + struct scx_dispatch_q *dsq, u64 enq_flags); +bool scx_consume_global_dsq(struct scx_sched *sch, struct rq *rq); +bool scx_rq_online(struct rq *rq); +void scx_flush_dispatch_buf(struct scx_sched *sch, struct rq *rq); +void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags); +void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, + u64 reenq_flags, struct rq *locked_rq); +int __scx_init_task(struct scx_sched *sch, struct task_struct *p, bool fork); +void scx_enable_task(struct scx_sched *sch, struct task_struct *p); +void __scx_disable_and_exit_task(struct scx_sched *sch, struct task_struct *p); +void scx_sub_init_cancel_task(struct scx_sched *sch, struct task_struct *p); +void scx_disable_and_exit_task(struct scx_sched *sch, struct task_struct *p); +#if defined(CONFIG_EXT_GROUP_SCHED) || defined(CONFIG_EXT_SUB_SCHED) +void scx_cgroup_lock(void); +void scx_cgroup_unlock(void); +#endif +s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch); +void scx_disable_bypass_dsp(struct scx_sched *sch); +void scx_bypass(struct scx_sched *sch, bool bypass); +s32 scx_link_sched(struct scx_sched *sch); +void scx_unlink_sched(struct scx_sched *sch); +void scx_disable_dump(struct scx_sched *sch); +void scx_log_sched_disable(struct scx_sched *sch); +void scx_flush_disable_work(struct scx_sched *sch); +struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, + struct cgroup *cgrp, + struct scx_sched *parent); +int scx_validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops); + +extern raw_spinlock_t scx_sched_lock; +extern struct mutex scx_enable_mutex; +extern struct percpu_rw_semaphore scx_fork_rwsem; +#ifdef CONFIG_EXT_SUB_SCHED +extern const struct rhashtable_params scx_sched_hash_params; +extern struct rhashtable scx_sched_hash; +extern struct scx_sched *scx_enabling_sub_sched; +#endif + #define scx_exit(sch, kind, exit_code, fmt, args...) \ __scx_exit(sch, kind, exit_code, raw_smp_processor_id(), fmt, ##args) #define scx_error(sch, fmt, args...) \ -- cgit v1.2.3 From 305f86e8a730a850bf6fd3fb88a6ae7386e0d5df Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 1 Jul 2026 10:28:48 -1000 Subject: sched_ext: Inline small ext.c helpers shared across the sub.c split The following trivial helpers in ext.c are called from both ext.c and the sub-scheduler code. Define them as static inline in internal.h. - scx_bypass_dsq() - scx_bypass_dsp_enabled() - scx_ops_sanitize_err() - scx_schedule_reenq_local() No functional change. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 57 --------------------------------------------- kernel/sched/ext/internal.h | 57 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 57 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index bdbc66466962..d1ef79c1038d 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -368,11 +368,6 @@ static const struct sched_class *scx_setscheduler_class(struct task_struct *p) return __setscheduler_class(p->policy, p->prio); } -static struct scx_dispatch_q *scx_bypass_dsq(struct scx_sched *sch, s32 cpu) -{ - return &per_cpu_ptr(sch->pcpu, cpu)->bypass_dsq; -} - static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, s32 cpu) { #ifdef CONFIG_EXT_SUB_SCHED @@ -394,26 +389,6 @@ static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, s32 c return scx_bypass_dsq(sch, cpu); } -/** - * scx_bypass_dsp_enabled - Check if bypass dispatch path is enabled - * @sch: scheduler to check - * - * When a descendant scheduler enters bypass mode, bypassed tasks are scheduled - * by the nearest non-bypassing ancestor, or the root scheduler if all ancestors - * are bypassing. In the former case, the ancestor is not itself bypassing but - * its bypass DSQs will be populated with bypassed tasks from descendants. Thus, - * the ancestor's bypass dispatch path must be active even though its own - * bypass_depth remains zero. - * - * This function checks bypass_dsp_enable_depth which is managed separately from - * bypass_depth to enable this decoupling. See enable_bypass_dsp() and - * scx_disable_bypass_dsp(). - */ -static bool scx_bypass_dsp_enabled(struct scx_sched *sch) -{ - return unlikely(atomic_read(&sch->bypass_dsp_enable_depth)); -} - /** * rq_is_open - Is the rq available for immediate execution of an SCX task? * @rq: rq to test @@ -1060,28 +1035,6 @@ bool scx_cpu_valid(struct scx_sched *sch, s32 cpu, const char *where) } } -/** - * scx_ops_sanitize_err - Sanitize a -errno value - * @sch: scx_sched to error out on error - * @ops_name: operation to blame on failure - * @err: -errno value to sanitize - * - * Verify @err is a valid -errno. If not, trigger scx_error() and return - * -%EPROTO. This is necessary because returning a rogue -errno up the chain can - * cause misbehaviors. For an example, a large negative return from - * ops.init_task() triggers an oops when passed up the call chain because the - * value fails IS_ERR() test after being encoded with ERR_PTR() and then is - * handled as a pointer. - */ -static int scx_ops_sanitize_err(struct scx_sched *sch, const char *ops_name, s32 err) -{ - if (err < 0 && err >= -MAX_ERRNO) - return err; - - scx_error(sch, "ops.%s() returned an invalid errno %d", ops_name, err); - return -EPROTO; -} - static void deferred_bal_cb_workfn(struct rq *rq) { run_deferred(rq); @@ -1233,16 +1186,6 @@ void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, schedule_deferred(rq); } -static void scx_schedule_reenq_local(struct rq *rq, u64 reenq_flags) -{ - struct scx_sched *root = rcu_dereference_sched(scx_root); - - if (WARN_ON_ONCE(!root)) - return; - - schedule_dsq_reenq(root, &rq->scx.local_dsq, reenq_flags, rq); -} - /** * touch_core_sched - Update timestamp used for core-sched task ordering * @rq: rq to read clock from, must be locked diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index c4a910d2ca91..c3b97ea4ae79 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1640,6 +1640,63 @@ extern struct scx_sched *scx_enabling_sub_sched; #define scx_error(sch, fmt, args...) \ scx_exit((sch), SCX_EXIT_ERROR, 0, fmt, ##args) +static inline struct scx_dispatch_q *scx_bypass_dsq(struct scx_sched *sch, s32 cpu) +{ + return &per_cpu_ptr(sch->pcpu, cpu)->bypass_dsq; +} + +/** + * scx_bypass_dsp_enabled - Check if bypass dispatch path is enabled + * @sch: scheduler to check + * + * When a descendant scheduler enters bypass mode, bypassed tasks are scheduled + * by the nearest non-bypassing ancestor, or the root scheduler if all ancestors + * are bypassing. In the former case, the ancestor is not itself bypassing but + * its bypass DSQs will be populated with bypassed tasks from descendants. Thus, + * the ancestor's bypass dispatch path must be active even though its own + * bypass_depth remains zero. + * + * This function checks bypass_dsp_enable_depth which is managed separately from + * bypass_depth to enable this decoupling. See enable_bypass_dsp() and + * scx_disable_bypass_dsp(). + */ +static inline bool scx_bypass_dsp_enabled(struct scx_sched *sch) +{ + return unlikely(atomic_read(&sch->bypass_dsp_enable_depth)); +} + +/** + * scx_ops_sanitize_err - Sanitize a -errno value + * @sch: scx_sched to error out on error + * @ops_name: operation to blame on failure + * @err: -errno value to sanitize + * + * Verify @err is a valid -errno. If not, trigger scx_error() and return + * -%EPROTO. This is necessary because returning a rogue -errno up the chain can + * cause misbehaviors. For an example, a large negative return from + * ops.init_task() triggers an oops when passed up the call chain because the + * value fails IS_ERR() test after being encoded with ERR_PTR() and then is + * handled as a pointer. + */ +static inline int scx_ops_sanitize_err(struct scx_sched *sch, const char *ops_name, s32 err) +{ + if (err < 0 && err >= -MAX_ERRNO) + return err; + + scx_error(sch, "ops.%s() returned an invalid errno %d", ops_name, err); + return -EPROTO; +} + +static inline void scx_schedule_reenq_local(struct rq *rq, u64 reenq_flags) +{ + struct scx_sched *root = rcu_dereference_sched(scx_root); + + if (WARN_ON_ONCE(!root)) + return; + + schedule_dsq_reenq(root, &rq->scx.local_dsq, reenq_flags, rq); +} + /* * Return the rq currently locked from an scx callback, or NULL if no rq is * locked. -- cgit v1.2.3 From daf8e166ba59ddf1dacb080228cb42e5d1de30ae Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 1 Jul 2026 10:28:48 -1000 Subject: sched_ext: Split sub-scheduler implementation into sub.c The sub-scheduler implementation has grown and will continue to expand. Move the sub-scheduler functions from ext.c into a new kernel/sched/ext/sub.c. sub.h holds the prototypes and the !CONFIG_EXT_SUB_SCHED no-op stubs. scx_dispatch_sched() is shared: balance_one() in ext.c and the scx_bpf_sub_dispatch() kfunc in sub.c both call it, and the latter re-enters it as sub-scheduler dispatch nests. It moves into sub.h as a static __always_inline so both callers keep it inlined and per-level stack stays bounded across the recursion. The event macros it uses move to internal.h. No functional change. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/build_policy.c | 2 + kernel/sched/ext/ext.c | 811 +------------------------------------------- kernel/sched/ext/internal.h | 28 ++ kernel/sched/ext/sub.c | 668 ++++++++++++++++++++++++++++++++++++ kernel/sched/ext/sub.h | 161 +++++++++ 5 files changed, 860 insertions(+), 810 deletions(-) create mode 100644 kernel/sched/ext/sub.c create mode 100644 kernel/sched/ext/sub.h (limited to 'kernel') diff --git a/kernel/sched/build_policy.c b/kernel/sched/build_policy.c index d74b54f81992..01dc7bf89af8 100644 --- a/kernel/sched/build_policy.c +++ b/kernel/sched/build_policy.c @@ -66,10 +66,12 @@ # include "ext/cid.h" # include "ext/arena.h" # include "ext/idle.h" +# include "ext/sub.h" # include "ext/ext.c" # include "ext/cid.c" # include "ext/arena.c" # include "ext/idle.c" +# include "ext/sub.c" #endif #include "syscalls.c" diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index d1ef79c1038d..1a0ec985da77 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -19,6 +19,7 @@ #include "cid.h" #include "arena.h" #include "idle.h" +#include "sub.h" DEFINE_RAW_SPINLOCK(scx_sched_lock); @@ -271,58 +272,6 @@ static bool u32_before(u32 a, u32 b) return (s32)(a - b) < 0; } -#ifdef CONFIG_EXT_SUB_SCHED -/** - * scx_next_descendant_pre - find the next descendant for pre-order walk - * @pos: the current position (%NULL to initiate traversal) - * @root: sched whose descendants to walk - * - * To be used by scx_for_each_descendant_pre(). Find the next descendant to - * visit for pre-order traversal of @root's descendants. @root is included in - * the iteration and the first node to be visited. - */ -static struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, - struct scx_sched *root) -{ - struct scx_sched *next; - - lockdep_assert(lockdep_is_held(&scx_enable_mutex) || - lockdep_is_held(&scx_sched_lock)); - - /* if first iteration, visit @root */ - if (!pos) - return root; - - /* visit the first child if exists */ - next = list_first_entry_or_null(&pos->children, struct scx_sched, sibling); - if (next) - return next; - - /* no child, visit my or the closest ancestor's next sibling */ - while (pos != root) { - if (!list_is_last(&pos->sibling, &scx_parent(pos)->children)) - return list_next_entry(pos, sibling); - pos = scx_parent(pos); - } - - return NULL; -} - -static struct scx_sched *scx_find_sub_sched(u64 cgroup_id) -{ - return rhashtable_lookup(&scx_sched_hash, &cgroup_id, - scx_sched_hash_params); -} - -static void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) -{ - rcu_assign_pointer(p->scx.sched, sch); -} -#else /* CONFIG_EXT_SUB_SCHED */ -static inline struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; } -static inline void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) {} -#endif /* CONFIG_EXT_SUB_SCHED */ - /** * scx_is_descendant - Test whether sched is a descendant * @sch: sched to test @@ -337,19 +286,6 @@ static bool scx_is_descendant(struct scx_sched *sch, struct scx_sched *ancestor) return sch->ancestors[ancestor->level] == ancestor; } -/** - * scx_for_each_descendant_pre - pre-order walk of a sched's descendants - * @pos: iteration cursor - * @root: sched to walk the descendants of - * - * Walk @root's descendants. @root is included in the iteration and the first - * node to be visited. Must be called with either scx_enable_mutex or - * scx_sched_lock held. - */ -#define scx_for_each_descendant_pre(pos, root) \ - for ((pos) = scx_next_descendant_pre(NULL, (root)); (pos); \ - (pos) = scx_next_descendant_pre((pos), (root))) - static struct scx_dispatch_q *find_global_dsq(struct scx_sched *sch, s32 cpu) { return &sch->pnode[cpu_to_node(cpu)]->global_dsq; @@ -935,32 +871,6 @@ struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter) return NULL; } -/** - * scx_add_event - Increase an event counter for 'name' by 'cnt' - * @sch: scx_sched to account events for - * @name: an event name defined in struct scx_event_stats - * @cnt: the number of the event occurred - * - * This can be used when preemption is not disabled. - */ -#define scx_add_event(sch, name, cnt) do { \ - this_cpu_add((sch)->pcpu->event_stats.name, (cnt)); \ - trace_sched_ext_event(#name, (cnt)); \ -} while(0) - -/** - * __scx_add_event - Increase an event counter for 'name' by 'cnt' - * @sch: scx_sched to account events for - * @name: an event name defined in struct scx_event_stats - * @cnt: the number of the event occurred - * - * This should be used only when preemption is disabled. - */ -#define __scx_add_event(sch, name, cnt) do { \ - __this_cpu_add((sch)->pcpu->event_stats.name, (cnt)); \ - trace_sched_ext_event(#name, cnt); \ -} while(0) - /** * scx_dump_event - Dump an event 'kind' in 'events' to 's' * @s: output seq_buf @@ -2681,115 +2591,6 @@ static inline void maybe_queue_balance_callback(struct rq *rq) rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING; } -/* - * One user of this function is scx_bpf_dispatch() which can be called - * recursively as sub-sched dispatches nest. Always inline to reduce stack usage - * from the call frame. - */ -static __always_inline bool -scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, - struct task_struct *prev, bool nested) -{ - struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; - int nr_loops = SCX_DSP_MAX_LOOPS; - s32 cpu = cpu_of(rq); - bool prev_on_sch = (prev->sched_class == &ext_sched_class) && - scx_task_on_sched(sch, prev); - - if (scx_consume_global_dsq(sch, rq)) - return true; - - if (scx_bypass_dsp_enabled(sch)) { - /* if @sch is bypassing, only the bypass DSQs are active */ - if (scx_bypassing(sch, cpu)) - return scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0); - -#ifdef CONFIG_EXT_SUB_SCHED - /* - * If @sch isn't bypassing but its children are, @sch is - * responsible for making forward progress for both its own - * tasks that aren't bypassing and the bypassing descendants' - * tasks. The following implements a simple built-in behavior - - * let each CPU try to run the bypass DSQ every Nth time. - * - * Later, if necessary, we can add an ops flag to suppress the - * auto-consumption and a kfunc to consume the bypass DSQ and, - * so that the BPF scheduler can fully control scheduling of - * bypassed tasks. - */ - struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); - - if (!(pcpu->bypass_host_seq++ % SCX_BYPASS_HOST_NTH) && - scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0)) { - __scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1); - return true; - } -#endif /* CONFIG_EXT_SUB_SCHED */ - } - - if (unlikely(!SCX_HAS_OP(sch, dispatch)) || !scx_rq_online(rq)) - return false; - - dspc->rq = rq; - - /* - * The dispatch loop. Because scx_flush_dispatch_buf() may drop the rq - * lock, the local DSQ might still end up empty after a successful - * ops.dispatch(). If the local DSQ is empty even after ops.dispatch() - * produced some tasks, retry. The BPF scheduler may depend on this - * looping behavior to simplify its implementation. - */ - do { - dspc->nr_tasks = 0; - - if (nested) { - SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu), - prev_on_sch ? prev : NULL); - } else { - /* stash @prev so that nested invocations can access it */ - rq->scx.sub_dispatch_prev = prev; - SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu), - prev_on_sch ? prev : NULL); - rq->scx.sub_dispatch_prev = NULL; - } - - scx_flush_dispatch_buf(sch, rq); - - if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice) { - rq->scx.flags |= SCX_RQ_BAL_KEEP; - return true; - } - if (rq->scx.local_dsq.nr) - return true; - if (scx_consume_global_dsq(sch, rq)) - return true; - - /* - * ops.dispatch() can trap us in this loop by repeatedly - * dispatching ineligible tasks. Break out once in a while to - * allow the watchdog to run. As IRQ can't be enabled in - * balance(), we want to complete this scheduling cycle and then - * start a new one. IOW, we want to call resched_curr() on the - * next, most likely idle, task, not the current one. Use - * __scx_bpf_kick_cpu() for deferred kicking. - */ - if (unlikely(!--nr_loops)) { - scx_kick_cpu(sch, cpu, 0); - break; - } - } while (dspc->nr_tasks); - - /* - * Prevent the CPU from going idle while bypassed descendants have tasks - * queued. Without this fallback, bypassed tasks could stall if the host - * scheduler's ops.dispatch() doesn't yield any tasks. - */ - if (scx_bypass_dsp_enabled(sch)) - return scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0); - - return false; -} - static int balance_one(struct rq *rq, struct task_struct *prev) { struct scx_sched *sch = scx_root; @@ -4469,26 +4270,6 @@ static inline void scx_cgroup_lock(void) {} static inline void scx_cgroup_unlock(void) {} #endif /* CONFIG_EXT_GROUP_SCHED || CONFIG_EXT_SUB_SCHED */ -#ifdef CONFIG_EXT_SUB_SCHED -static struct cgroup *sch_cgroup(struct scx_sched *sch) -{ - return sch->cgrp; -} - -/* for each descendant of @cgrp including self, set ->scx_sched to @sch */ -static void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch) -{ - struct cgroup *pos; - struct cgroup_subsys_state *css; - - cgroup_for_each_live_descendant_pre(pos, css, cgrp) - rcu_assign_pointer(pos->scx_sched, sch); -} -#else /* CONFIG_EXT_SUB_SCHED */ -static inline struct cgroup *sch_cgroup(struct scx_sched *sch) { return NULL; } -static inline void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch) {} -#endif /* CONFIG_EXT_SUB_SCHED */ - /* * Omitted operations: * @@ -5765,202 +5546,6 @@ void scx_log_sched_disable(struct scx_sched *sch) } } -#ifdef CONFIG_EXT_SUB_SCHED -static DECLARE_WAIT_QUEUE_HEAD(scx_unlink_waitq); - -static void drain_descendants(struct scx_sched *sch) -{ - /* - * Child scheds that finished the critical part of disabling will take - * themselves off @sch->children. Wait for it to drain. As propagation - * is recursive, empty @sch->children means that all proper descendant - * scheds reached unlinking stage. - */ - wait_event(scx_unlink_waitq, list_empty(&sch->children)); -} - -static void scx_fail_parent(struct scx_sched *sch, - struct task_struct *failed, s32 fail_code) -{ - struct scx_sched *parent = scx_parent(sch); - struct scx_task_iter sti; - struct task_struct *p; - - scx_error(parent, "ops.init_task() failed (%d) for %s[%d] while disabling a sub-scheduler", - fail_code, failed->comm, failed->pid); - - /* - * Once $parent is bypassed, it's safe to put SCX_TASK_NONE tasks into - * it. This may cause downstream failures on the BPF side but $parent is - * dying anyway. - */ - scx_bypass(parent, true); - - scx_task_iter_start(&sti, sch->cgrp); - while ((p = scx_task_iter_next_locked(&sti))) { - if (scx_task_on_sched(parent, p)) - continue; - - scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { - scx_disable_and_exit_task(sch, p); - scx_set_task_sched(p, parent); - } - } - scx_task_iter_stop(&sti); -} - -static void scx_sub_disable(struct scx_sched *sch) -{ - struct scx_sched *parent = scx_parent(sch); - struct scx_task_iter sti; - struct task_struct *p; - int ret; - - /* - * Guarantee forward progress and wait for descendants to be disabled. - * To limit disruptions, $parent is not bypassed. Tasks are fully - * prepped and then inserted back into $parent. - */ - scx_bypass(sch, true); - drain_descendants(sch); - - /* - * Here, every runnable task is guaranteed to make forward progress and - * we can safely use blocking synchronization constructs. Actually - * disable ops. - */ - mutex_lock(&scx_enable_mutex); - percpu_down_write(&scx_fork_rwsem); - scx_cgroup_lock(); - - set_cgroup_sched(sch_cgroup(sch), parent); - - scx_task_iter_start(&sti, sch->cgrp); - while ((p = scx_task_iter_next_locked(&sti))) { - struct rq *rq; - struct rq_flags rf; - - /* filter out duplicate visits */ - if (scx_task_on_sched(parent, p)) - continue; - - /* - * By the time control reaches here, all descendant schedulers - * should already have been disabled. - */ - WARN_ON_ONCE(!scx_task_on_sched(sch, p)); - - /* - * @p is pinned by the iter: css_task_iter_next() takes a - * reference and holds it until the next iter_next() call, so - * @p->usage is guaranteed > 0. - */ - get_task_struct(p); - - scx_task_iter_unlock(&sti); - - /* - * $p is READY or ENABLED on @sch. Initialize for $parent, - * disable and exit from @sch, and then switch over to $parent. - * - * If a task fails to initialize for $parent, the only available - * action is disabling $parent too. While this allows disabling - * of a child sched to cause the parent scheduler to fail, the - * failure can only originate from ops.init_task() of the - * parent. A child can't directly affect the parent through its - * own failures. - */ - ret = __scx_init_task(parent, p, false); - if (ret) { - scx_fail_parent(sch, p, ret); - put_task_struct(p); - break; - } - - rq = task_rq_lock(p, &rf); - - if (scx_get_task_state(p) == SCX_TASK_DEAD) { - /* - * sched_ext_dead() raced us between __scx_init_task() - * and this rq lock and ran exit_task() on @sch (the - * sched @p was on at that point), not on $parent. - * $parent's just-completed init is owed an exit_task() - * and we issue it here. - */ - scx_sub_init_cancel_task(parent, p); - task_rq_unlock(rq, p, &rf); - put_task_struct(p); - continue; - } - - scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { - /* - * $p is initialized for $parent and still attached to - * @sch. Disable and exit for @sch, switch over to - * $parent, override the state to READY to account for - * $p having already been initialized, and then enable. - */ - scx_disable_and_exit_task(sch, p); - scx_set_task_state(p, SCX_TASK_INIT_BEGIN); - scx_set_task_state(p, SCX_TASK_INIT); - scx_set_task_sched(p, parent); - scx_set_task_state(p, SCX_TASK_READY); - scx_enable_task(parent, p); - } - - task_rq_unlock(rq, p, &rf); - put_task_struct(p); - } - scx_task_iter_stop(&sti); - - scx_disable_dump(sch); - - scx_cgroup_unlock(); - percpu_up_write(&scx_fork_rwsem); - - /* - * All tasks are moved off of @sch but there may still be on-going - * operations (e.g. ops.select_cpu()). Drain them by flushing RCU. Use - * the expedited version as ancestors may be waiting in bypass mode. - * Also, tell the parent that there is no need to keep running bypass - * DSQs for us. - */ - synchronize_rcu_expedited(); - scx_disable_bypass_dsp(sch); - - scx_unlink_sched(sch); - - mutex_unlock(&scx_enable_mutex); - - /* - * @sch is now unlinked from the parent's children list. Notify and call - * ops.sub_detach/exit(). Note that ops.sub_detach/exit() must be called - * after unlinking and releasing all locks. See scx_claim_exit(). - */ - wake_up_all(&scx_unlink_waitq); - - if (parent->ops.sub_detach && sch->sub_attached) { - struct scx_sub_detach_args sub_detach_args = { - .ops = &sch->ops, - .cgroup_path = sch->cgrp_path, - }; - SCX_CALL_OP(parent, sub_detach, NULL, - &sub_detach_args); - } - - scx_log_sched_disable(sch); - - if (sch->ops.exit) - SCX_CALL_OP(sch, exit, NULL, sch->exit_info); - if (sch->sub_kset) - kobject_del(&sch->sub_kset->kobj); - kobject_del(&sch->kobj); -} -#else /* CONFIG_EXT_SUB_SCHED */ -static inline void drain_descendants(struct scx_sched *sch) { } -static inline void scx_sub_disable(struct scx_sched *sch) { } -#endif /* CONFIG_EXT_SUB_SCHED */ - static void scx_root_disable(struct scx_sched *sch) { struct scx_task_iter sti; @@ -7350,347 +6935,6 @@ err_disable: cmd->ret = 0; } -#ifdef CONFIG_EXT_SUB_SCHED -/* verify that a scheduler can be attached to @cgrp and return the parent */ -static struct scx_sched *find_parent_sched(struct cgroup *cgrp) -{ - struct scx_sched *parent = cgrp->scx_sched; - struct scx_sched *pos; - - lockdep_assert_held(&scx_sched_lock); - - /* can't attach twice to the same cgroup */ - if (parent->cgrp == cgrp) - return ERR_PTR(-EBUSY); - - /* does $parent allow sub-scheds? */ - if (!parent->ops.sub_attach) - return ERR_PTR(-EOPNOTSUPP); - - /* can't insert between $parent and its exiting children */ - list_for_each_entry(pos, &parent->children, sibling) - if (cgroup_is_descendant(pos->cgrp, cgrp)) - return ERR_PTR(-EBUSY); - - return parent; -} - -static bool assert_task_ready_or_enabled(struct task_struct *p) -{ - u32 state = scx_get_task_state(p); - - switch (state) { - case SCX_TASK_READY: - case SCX_TASK_ENABLED: - return true; - default: - WARN_ONCE(true, "sched_ext: Invalid task state %d for %s[%d] during enabling sub sched", - state, p->comm, p->pid); - return false; - } -} - -static void scx_sub_enable_workfn(struct kthread_work *work) -{ - struct scx_enable_cmd *cmd = container_of(work, struct scx_enable_cmd, work); - struct sched_ext_ops *ops = cmd->ops; - struct cgroup *cgrp; - struct scx_sched *parent, *sch; - struct scx_task_iter sti; - struct task_struct *p; - s32 i, ret; - - mutex_lock(&scx_enable_mutex); - - if (!scx_enabled()) { - ret = -ENODEV; - goto out_unlock; - } - - /* See scx_root_enable_workfn() for the @ops->priv check. */ - if (rcu_access_pointer(ops->priv)) { - ret = -EBUSY; - goto out_unlock; - } - - cgrp = cgroup_get_from_id(ops->sub_cgroup_id); - if (IS_ERR(cgrp)) { - ret = PTR_ERR(cgrp); - goto out_unlock; - } - - raw_spin_lock_irq(&scx_sched_lock); - parent = find_parent_sched(cgrp); - if (IS_ERR(parent)) { - raw_spin_unlock_irq(&scx_sched_lock); - ret = PTR_ERR(parent); - goto out_put_cgrp; - } - kobject_get(&parent->kobj); - raw_spin_unlock_irq(&scx_sched_lock); - - /* scx_alloc_and_add_sched() consumes @cgrp whether it succeeds or not */ - sch = scx_alloc_and_add_sched(cmd, cgrp, parent); - kobject_put(&parent->kobj); - if (IS_ERR(sch)) { - ret = PTR_ERR(sch); - goto out_unlock; - } - - ret = scx_link_sched(sch); - if (ret) - goto err_disable; - - if (sch->level >= SCX_SUB_MAX_DEPTH) { - scx_error(sch, "max nesting depth %d violated", - SCX_SUB_MAX_DEPTH); - goto err_disable; - } - - if (sch->ops.init) { - ret = SCX_CALL_OP_RET(sch, init, NULL); - if (ret) { - ret = scx_ops_sanitize_err(sch, "init", ret); - scx_error(sch, "ops.init() failed (%d)", ret); - goto err_disable; - } - sch->exit_info->flags |= SCX_EFLAG_INITIALIZED; - } - - ret = scx_arena_pool_init(sch); - if (ret) - goto err_disable; - - ret = scx_set_cmask_scratch_alloc(sch); - if (ret) - goto err_disable; - - if (scx_validate_ops(sch, ops)) - goto err_disable; - - struct scx_sub_attach_args sub_attach_args = { - .ops = &sch->ops, - .cgroup_path = sch->cgrp_path, - }; - - ret = SCX_CALL_OP_RET(parent, sub_attach, NULL, - &sub_attach_args); - if (ret) { - ret = scx_ops_sanitize_err(sch, "sub_attach", ret); - scx_error(sch, "parent rejected (%d)", ret); - goto err_disable; - } - sch->sub_attached = true; - - scx_bypass(sch, true); - - for (i = SCX_OPI_BEGIN; i < SCX_OPI_END; i++) - if (((void (**)(void))ops)[i]) - set_bit(i, sch->has_op); - - percpu_down_write(&scx_fork_rwsem); - scx_cgroup_lock(); - - /* - * Set cgroup->scx_sched's and check CSS_ONLINE. Either we see - * !CSS_ONLINE or scx_cgroup_lifetime_notify() sees and shoots us down. - */ - set_cgroup_sched(sch_cgroup(sch), sch); - if (!(cgrp->self.flags & CSS_ONLINE)) { - scx_error(sch, "cgroup is not online"); - goto err_unlock_and_disable; - } - - /* - * Initialize tasks for the new child $sch without exiting them for - * $parent so that the tasks can always be reverted back to $parent - * sched on child init failure. - */ - WARN_ON_ONCE(scx_enabling_sub_sched); - scx_enabling_sub_sched = sch; - - scx_task_iter_start(&sti, sch->cgrp); - while ((p = scx_task_iter_next_locked(&sti))) { - struct rq *rq; - struct rq_flags rf; - - /* - * Task iteration may visit the same task twice when racing - * against exiting. Use %SCX_TASK_SUB_INIT to mark tasks which - * finished __scx_init_task() and skip if set. - * - * A task may exit and get freed between __scx_init_task() - * completion and scx_enable_task(). In such cases, - * scx_disable_and_exit_task() must exit the task for both the - * parent and child scheds. - */ - if (p->scx.flags & SCX_TASK_SUB_INIT) - continue; - - /* @p is pinned by the iter; see scx_sub_disable() */ - get_task_struct(p); - - if (!assert_task_ready_or_enabled(p)) { - ret = -EINVAL; - goto abort; - } - - scx_task_iter_unlock(&sti); - - /* - * As $p is still on $parent, it can't be transitioned to INIT. - * Let's worry about task state later. Use __scx_init_task(). - */ - ret = __scx_init_task(sch, p, false); - if (ret) - goto abort; - - rq = task_rq_lock(p, &rf); - - if (scx_get_task_state(p) == SCX_TASK_DEAD) { - /* - * sched_ext_dead() raced us between __scx_init_task() - * and this rq lock and ran exit_task() on $parent (the - * sched @p was on at that point), not on @sch. @sch's - * just-completed init is owed an exit_task() and we - * issue it here. - */ - scx_sub_init_cancel_task(sch, p); - task_rq_unlock(rq, p, &rf); - put_task_struct(p); - continue; - } - - p->scx.flags |= SCX_TASK_SUB_INIT; - task_rq_unlock(rq, p, &rf); - - put_task_struct(p); - } - scx_task_iter_stop(&sti); - - /* - * All tasks are prepped. Disable/exit tasks for $parent and enable for - * the new @sch. - */ - scx_task_iter_start(&sti, sch->cgrp); - while ((p = scx_task_iter_next_locked(&sti))) { - /* - * Use clearing of %SCX_TASK_SUB_INIT to detect and skip - * duplicate iterations. - */ - if (!(p->scx.flags & SCX_TASK_SUB_INIT)) - continue; - - scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { - /* - * $p must be either READY or ENABLED. If ENABLED, - * __scx_disabled_and_exit_task() first disables and - * makes it READY. However, after exiting $p, it will - * leave $p as READY. - */ - assert_task_ready_or_enabled(p); - __scx_disable_and_exit_task(parent, p); - - /* - * $p is now only initialized for @sch and READY, which - * is what we want. Assign it to @sch and enable. - */ - scx_set_task_sched(p, sch); - scx_enable_task(sch, p); - - p->scx.flags &= ~SCX_TASK_SUB_INIT; - } - } - scx_task_iter_stop(&sti); - - scx_enabling_sub_sched = NULL; - - scx_cgroup_unlock(); - percpu_up_write(&scx_fork_rwsem); - - scx_bypass(sch, false); - - pr_info("sched_ext: BPF sub-scheduler \"%s\" enabled\n", sch->ops.name); - kobject_uevent(&sch->kobj, KOBJ_ADD); - ret = 0; - goto out_unlock; - -out_put_cgrp: - cgroup_put(cgrp); -out_unlock: - mutex_unlock(&scx_enable_mutex); - cmd->ret = ret; - return; - -abort: - put_task_struct(p); - scx_task_iter_stop(&sti); - - /* - * Undo __scx_init_task() for tasks we marked. scx_enable_task() never - * ran for @sch on them, so calling scx_disable_task() here would invoke - * ops.disable() without a matching ops.enable(). scx_enabling_sub_sched - * must stay set until SUB_INIT is cleared from every marked task - - * scx_disable_and_exit_task() reads it when a task exits concurrently. - */ - scx_task_iter_start(&sti, sch->cgrp); - while ((p = scx_task_iter_next_locked(&sti))) { - if (p->scx.flags & SCX_TASK_SUB_INIT) { - scx_sub_init_cancel_task(sch, p); - p->scx.flags &= ~SCX_TASK_SUB_INIT; - } - } - scx_task_iter_stop(&sti); - scx_enabling_sub_sched = NULL; -err_unlock_and_disable: - /* we'll soon enter disable path, keep bypass on */ - scx_cgroup_unlock(); - percpu_up_write(&scx_fork_rwsem); -err_disable: - mutex_unlock(&scx_enable_mutex); - scx_flush_disable_work(sch); - cmd->ret = 0; -} - -static s32 scx_cgroup_lifetime_notify(struct notifier_block *nb, - unsigned long action, void *data) -{ - struct cgroup *cgrp = data; - struct cgroup *parent = cgroup_parent(cgrp); - - if (!cgroup_on_dfl(cgrp)) - return NOTIFY_OK; - - switch (action) { - case CGROUP_LIFETIME_ONLINE: - /* inherit ->scx_sched from $parent */ - if (parent) - rcu_assign_pointer(cgrp->scx_sched, parent->scx_sched); - break; - case CGROUP_LIFETIME_OFFLINE: - /* if there is a sched attached, shoot it down */ - if (cgrp->scx_sched && cgrp->scx_sched->cgrp == cgrp) - scx_exit(cgrp->scx_sched, SCX_EXIT_UNREG_KERN, - SCX_ECODE_RSN_CGROUP_OFFLINE, - "cgroup %llu going offline", cgroup_id(cgrp)); - break; - } - - return NOTIFY_OK; -} - -static struct notifier_block scx_cgroup_lifetime_nb = { - .notifier_call = scx_cgroup_lifetime_notify, -}; - -static s32 __init scx_cgroup_lifetime_notifier_init(void) -{ - return blocking_notifier_chain_register(&cgroup_lifetime_notifier, - &scx_cgroup_lifetime_nb); -} -core_initcall(scx_cgroup_lifetime_notifier_init); -#endif /* CONFIG_EXT_SUB_SCHED */ - static s32 scx_enable(struct scx_enable_cmd *cmd, struct bpf_link *link) { static struct kthread_worker *helper; @@ -7837,20 +7081,6 @@ static int bpf_scx_init_member(const struct btf_type *t, return 0; } -#ifdef CONFIG_EXT_SUB_SCHED -static void scx_pstack_recursion_on_dispatch(struct bpf_prog *prog) -{ - struct scx_sched *sch; - - guard(rcu)(); - sch = scx_prog_sched(prog->aux); - if (unlikely(!sch)) - return; - - scx_error(sch, "dispatch recursion detected"); -} -#endif /* CONFIG_EXT_SUB_SCHED */ - static int bpf_scx_check_member(const struct btf_type *t, const struct btf_member *member, const struct bpf_prog *prog) @@ -9021,45 +8251,6 @@ __bpf_kfunc bool scx_bpf_dsq_move_vtime(struct bpf_iter_scx_dsq *it__iter, p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ); } -#ifdef CONFIG_EXT_SUB_SCHED -/** - * scx_bpf_sub_dispatch - Trigger dispatching on a child scheduler - * @cgroup_id: cgroup ID of the child scheduler to dispatch - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Allows a parent scheduler to trigger dispatching on one of its direct - * child schedulers. The child scheduler runs its dispatch operation to - * move tasks from dispatch queues to the local runqueue. - * - * Returns: true on success, false if cgroup_id is invalid, not a direct - * child, or caller lacks dispatch permission. - */ -__bpf_kfunc bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux *aux) -{ - struct rq *this_rq = this_rq(); - struct scx_sched *parent, *child; - - guard(rcu)(); - parent = scx_prog_sched(aux); - if (unlikely(!parent)) - return false; - - child = scx_find_sub_sched(cgroup_id); - - if (unlikely(!child)) - return false; - - if (unlikely(scx_parent(child) != parent)) { - scx_error(parent, "trying to dispatch a distant sub-sched on cgroup %llu", - cgroup_id); - return false; - } - - return scx_dispatch_sched(child, this_rq, this_rq->scx.sub_dispatch_prev, - true); -} -#endif /* CONFIG_EXT_SUB_SCHED */ - __bpf_kfunc_end_defs(); BTF_KFUNCS_START(scx_kfunc_ids_dispatch) diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index c3b97ea4ae79..f9fe7c6ebc4b 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -11,6 +11,34 @@ #include "../sched.h" #include "types.h" +#include + +/** + * scx_add_event - Increase an event counter for 'name' by 'cnt' + * @sch: scx_sched to account events for + * @name: an event name defined in struct scx_event_stats + * @cnt: the number of the event occurred + * + * This can be used when preemption is not disabled. + */ +#define scx_add_event(sch, name, cnt) do { \ + this_cpu_add((sch)->pcpu->event_stats.name, (cnt)); \ + trace_sched_ext_event(#name, (cnt)); \ +} while(0) + +/** + * __scx_add_event - Increase an event counter for 'name' by 'cnt' + * @sch: scx_sched to account events for + * @name: an event name defined in struct scx_event_stats + * @cnt: the number of the event occurred + * + * This should be used only when preemption is disabled. + */ +#define __scx_add_event(sch, name, cnt) do { \ + __this_cpu_add((sch)->pcpu->event_stats.name, (cnt)); \ + trace_sched_ext_event(#name, cnt); \ +} while(0) + #define SCX_OP_IDX(op) (offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void))) #define SCX_MOFF_IDX(moff) ((moff) / sizeof(void (*)(void))) diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c new file mode 100644 index 000000000000..050420427273 --- /dev/null +++ b/kernel/sched/ext/sub.c @@ -0,0 +1,668 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst + * + * Sub-scheduler hierarchy support. + * + * A sub-scheduler is an scx_sched attached to a cgroup subtree under another + * scx_sched. This file holds the sub-scheduler implementation: the scheduler + * tree walk, capability delegation, per-shard cap state and its sync, and the + * sub-scheduler enable/disable paths. The core dispatch/enqueue machinery it + * builds on lives in ext.c. + * + * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2026 Tejun Heo + */ +#include +#include "internal.h" +#include "cid.h" +#include "arena.h" +#include "sub.h" + +#ifdef CONFIG_EXT_SUB_SCHED + +/** + * scx_next_descendant_pre - find the next descendant for pre-order walk + * @pos: the current position (%NULL to initiate traversal) + * @root: sched whose descendants to walk + * + * To be used by scx_for_each_descendant_pre(). Find the next descendant to + * visit for pre-order traversal of @root's descendants. @root is included in + * the iteration and the first node to be visited. + */ +struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) +{ + struct scx_sched *next; + + lockdep_assert(lockdep_is_held(&scx_enable_mutex) || + lockdep_is_held(&scx_sched_lock)); + + /* if first iteration, visit @root */ + if (!pos) + return root; + + /* visit the first child if exists */ + next = list_first_entry_or_null(&pos->children, struct scx_sched, sibling); + if (next) + return next; + + /* no child, visit my or the closest ancestor's next sibling */ + while (pos != root) { + if (!list_is_last(&pos->sibling, &scx_parent(pos)->children)) + return list_next_entry(pos, sibling); + pos = scx_parent(pos); + } + + return NULL; +} + +static struct scx_sched *scx_find_sub_sched(u64 cgroup_id) +{ + return rhashtable_lookup(&scx_sched_hash, &cgroup_id, + scx_sched_hash_params); +} + +void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) +{ + rcu_assign_pointer(p->scx.sched, sch); +} + +struct cgroup *sch_cgroup(struct scx_sched *sch) +{ + return sch->cgrp; +} + +/* for each descendant of @cgrp including self, set ->scx_sched to @sch */ +void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch) +{ + struct cgroup *pos; + struct cgroup_subsys_state *css; + + cgroup_for_each_live_descendant_pre(pos, css, cgrp) + rcu_assign_pointer(pos->scx_sched, sch); +} + +static DECLARE_WAIT_QUEUE_HEAD(scx_unlink_waitq); + +void drain_descendants(struct scx_sched *sch) +{ + /* + * Child scheds that finished the critical part of disabling will take + * themselves off @sch->children. Wait for it to drain. As propagation + * is recursive, empty @sch->children means that all proper descendant + * scheds reached unlinking stage. + */ + wait_event(scx_unlink_waitq, list_empty(&sch->children)); +} + +static void scx_fail_parent(struct scx_sched *sch, + struct task_struct *failed, s32 fail_code) +{ + struct scx_sched *parent = scx_parent(sch); + struct scx_task_iter sti; + struct task_struct *p; + + scx_error(parent, "ops.init_task() failed (%d) for %s[%d] while disabling a sub-scheduler", + fail_code, failed->comm, failed->pid); + + /* + * Once $parent is bypassed, it's safe to put SCX_TASK_NONE tasks into + * it. This may cause downstream failures on the BPF side but $parent is + * dying anyway. + */ + scx_bypass(parent, true); + + scx_task_iter_start(&sti, sch->cgrp); + while ((p = scx_task_iter_next_locked(&sti))) { + if (scx_task_on_sched(parent, p)) + continue; + + scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { + scx_disable_and_exit_task(sch, p); + scx_set_task_sched(p, parent); + } + } + scx_task_iter_stop(&sti); +} + +void scx_sub_disable(struct scx_sched *sch) +{ + struct scx_sched *parent = scx_parent(sch); + struct scx_task_iter sti; + struct task_struct *p; + int ret; + + /* + * Guarantee forward progress and wait for descendants to be disabled. + * To limit disruptions, $parent is not bypassed. Tasks are fully + * prepped and then inserted back into $parent. + */ + scx_bypass(sch, true); + drain_descendants(sch); + + /* + * Here, every runnable task is guaranteed to make forward progress and + * we can safely use blocking synchronization constructs. Actually + * disable ops. + */ + mutex_lock(&scx_enable_mutex); + percpu_down_write(&scx_fork_rwsem); + scx_cgroup_lock(); + + set_cgroup_sched(sch_cgroup(sch), parent); + + scx_task_iter_start(&sti, sch->cgrp); + while ((p = scx_task_iter_next_locked(&sti))) { + struct rq *rq; + struct rq_flags rf; + + /* filter out duplicate visits */ + if (scx_task_on_sched(parent, p)) + continue; + + /* + * By the time control reaches here, all descendant schedulers + * should already have been disabled. + */ + WARN_ON_ONCE(!scx_task_on_sched(sch, p)); + + /* + * @p is pinned by the iter: css_task_iter_next() takes a + * reference and holds it until the next iter_next() call, so + * @p->usage is guaranteed > 0. + */ + get_task_struct(p); + + scx_task_iter_unlock(&sti); + + /* + * $p is READY or ENABLED on @sch. Initialize for $parent, + * disable and exit from @sch, and then switch over to $parent. + * + * If a task fails to initialize for $parent, the only available + * action is disabling $parent too. While this allows disabling + * of a child sched to cause the parent scheduler to fail, the + * failure can only originate from ops.init_task() of the + * parent. A child can't directly affect the parent through its + * own failures. + */ + ret = __scx_init_task(parent, p, false); + if (ret) { + scx_fail_parent(sch, p, ret); + put_task_struct(p); + break; + } + + rq = task_rq_lock(p, &rf); + + if (scx_get_task_state(p) == SCX_TASK_DEAD) { + /* + * sched_ext_dead() raced us between __scx_init_task() + * and this rq lock and ran exit_task() on @sch (the + * sched @p was on at that point), not on $parent. + * $parent's just-completed init is owed an exit_task() + * and we issue it here. + */ + scx_sub_init_cancel_task(parent, p); + task_rq_unlock(rq, p, &rf); + put_task_struct(p); + continue; + } + + scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { + /* + * $p is initialized for $parent and still attached to + * @sch. Disable and exit for @sch, switch over to + * $parent, override the state to READY to account for + * $p having already been initialized, and then enable. + */ + scx_disable_and_exit_task(sch, p); + scx_set_task_state(p, SCX_TASK_INIT_BEGIN); + scx_set_task_state(p, SCX_TASK_INIT); + scx_set_task_sched(p, parent); + scx_set_task_state(p, SCX_TASK_READY); + scx_enable_task(parent, p); + } + + task_rq_unlock(rq, p, &rf); + put_task_struct(p); + } + scx_task_iter_stop(&sti); + + scx_disable_dump(sch); + + scx_cgroup_unlock(); + percpu_up_write(&scx_fork_rwsem); + + /* + * All tasks are moved off of @sch but there may still be on-going + * operations (e.g. ops.select_cpu()). Drain them by flushing RCU. Use + * the expedited version as ancestors may be waiting in bypass mode. + * Also, tell the parent that there is no need to keep running bypass + * DSQs for us. + */ + synchronize_rcu_expedited(); + scx_disable_bypass_dsp(sch); + + scx_unlink_sched(sch); + + mutex_unlock(&scx_enable_mutex); + + /* + * @sch is now unlinked from the parent's children list. Notify and call + * ops.sub_detach/exit(). Note that ops.sub_detach/exit() must be called + * after unlinking and releasing all locks. See scx_claim_exit(). + */ + wake_up_all(&scx_unlink_waitq); + + if (parent->ops.sub_detach && sch->sub_attached) { + struct scx_sub_detach_args sub_detach_args = { + .ops = &sch->ops, + .cgroup_path = sch->cgrp_path, + }; + SCX_CALL_OP(parent, sub_detach, NULL, + &sub_detach_args); + } + + scx_log_sched_disable(sch); + + if (sch->ops.exit) + SCX_CALL_OP(sch, exit, NULL, sch->exit_info); + if (sch->sub_kset) + kobject_del(&sch->sub_kset->kobj); + kobject_del(&sch->kobj); +} + +/* verify that a scheduler can be attached to @cgrp and return the parent */ +static struct scx_sched *find_parent_sched(struct cgroup *cgrp) +{ + struct scx_sched *parent = cgrp->scx_sched; + struct scx_sched *pos; + + lockdep_assert_held(&scx_sched_lock); + + /* can't attach twice to the same cgroup */ + if (parent->cgrp == cgrp) + return ERR_PTR(-EBUSY); + + /* does $parent allow sub-scheds? */ + if (!parent->ops.sub_attach) + return ERR_PTR(-EOPNOTSUPP); + + /* can't insert between $parent and its exiting children */ + list_for_each_entry(pos, &parent->children, sibling) + if (cgroup_is_descendant(pos->cgrp, cgrp)) + return ERR_PTR(-EBUSY); + + return parent; +} + +static bool assert_task_ready_or_enabled(struct task_struct *p) +{ + u32 state = scx_get_task_state(p); + + switch (state) { + case SCX_TASK_READY: + case SCX_TASK_ENABLED: + return true; + default: + WARN_ONCE(true, "sched_ext: Invalid task state %d for %s[%d] during enabling sub sched", + state, p->comm, p->pid); + return false; + } +} + +void scx_sub_enable_workfn(struct kthread_work *work) +{ + struct scx_enable_cmd *cmd = container_of(work, struct scx_enable_cmd, work); + struct sched_ext_ops *ops = cmd->ops; + struct cgroup *cgrp; + struct scx_sched *parent, *sch; + struct scx_task_iter sti; + struct task_struct *p; + s32 i, ret; + + mutex_lock(&scx_enable_mutex); + + if (!scx_enabled()) { + ret = -ENODEV; + goto out_unlock; + } + + /* See scx_root_enable_workfn() for the @ops->priv check. */ + if (rcu_access_pointer(ops->priv)) { + ret = -EBUSY; + goto out_unlock; + } + + cgrp = cgroup_get_from_id(ops->sub_cgroup_id); + if (IS_ERR(cgrp)) { + ret = PTR_ERR(cgrp); + goto out_unlock; + } + + raw_spin_lock_irq(&scx_sched_lock); + parent = find_parent_sched(cgrp); + if (IS_ERR(parent)) { + raw_spin_unlock_irq(&scx_sched_lock); + ret = PTR_ERR(parent); + goto out_put_cgrp; + } + kobject_get(&parent->kobj); + raw_spin_unlock_irq(&scx_sched_lock); + + /* scx_alloc_and_add_sched() consumes @cgrp whether it succeeds or not */ + sch = scx_alloc_and_add_sched(cmd, cgrp, parent); + kobject_put(&parent->kobj); + if (IS_ERR(sch)) { + ret = PTR_ERR(sch); + goto out_unlock; + } + + ret = scx_link_sched(sch); + if (ret) + goto err_disable; + + if (sch->level >= SCX_SUB_MAX_DEPTH) { + scx_error(sch, "max nesting depth %d violated", + SCX_SUB_MAX_DEPTH); + goto err_disable; + } + + if (sch->ops.init) { + ret = SCX_CALL_OP_RET(sch, init, NULL); + if (ret) { + ret = scx_ops_sanitize_err(sch, "init", ret); + scx_error(sch, "ops.init() failed (%d)", ret); + goto err_disable; + } + sch->exit_info->flags |= SCX_EFLAG_INITIALIZED; + } + + ret = scx_arena_pool_init(sch); + if (ret) + goto err_disable; + + ret = scx_set_cmask_scratch_alloc(sch); + if (ret) + goto err_disable; + + if (scx_validate_ops(sch, ops)) + goto err_disable; + + struct scx_sub_attach_args sub_attach_args = { + .ops = &sch->ops, + .cgroup_path = sch->cgrp_path, + }; + + ret = SCX_CALL_OP_RET(parent, sub_attach, NULL, + &sub_attach_args); + if (ret) { + ret = scx_ops_sanitize_err(sch, "sub_attach", ret); + scx_error(sch, "parent rejected (%d)", ret); + goto err_disable; + } + sch->sub_attached = true; + + scx_bypass(sch, true); + + for (i = SCX_OPI_BEGIN; i < SCX_OPI_END; i++) + if (((void (**)(void))ops)[i]) + set_bit(i, sch->has_op); + + percpu_down_write(&scx_fork_rwsem); + scx_cgroup_lock(); + + /* + * Set cgroup->scx_sched's and check CSS_ONLINE. Either we see + * !CSS_ONLINE or scx_cgroup_lifetime_notify() sees and shoots us down. + */ + set_cgroup_sched(sch_cgroup(sch), sch); + if (!(cgrp->self.flags & CSS_ONLINE)) { + scx_error(sch, "cgroup is not online"); + goto err_unlock_and_disable; + } + + /* + * Initialize tasks for the new child $sch without exiting them for + * $parent so that the tasks can always be reverted back to $parent + * sched on child init failure. + */ + WARN_ON_ONCE(scx_enabling_sub_sched); + scx_enabling_sub_sched = sch; + + scx_task_iter_start(&sti, sch->cgrp); + while ((p = scx_task_iter_next_locked(&sti))) { + struct rq *rq; + struct rq_flags rf; + + /* + * Task iteration may visit the same task twice when racing + * against exiting. Use %SCX_TASK_SUB_INIT to mark tasks which + * finished __scx_init_task() and skip if set. + * + * A task may exit and get freed between __scx_init_task() + * completion and scx_enable_task(). In such cases, + * scx_disable_and_exit_task() must exit the task for both the + * parent and child scheds. + */ + if (p->scx.flags & SCX_TASK_SUB_INIT) + continue; + + /* @p is pinned by the iter; see scx_sub_disable() */ + get_task_struct(p); + + if (!assert_task_ready_or_enabled(p)) { + ret = -EINVAL; + goto abort; + } + + scx_task_iter_unlock(&sti); + + /* + * As $p is still on $parent, it can't be transitioned to INIT. + * Let's worry about task state later. Use __scx_init_task(). + */ + ret = __scx_init_task(sch, p, false); + if (ret) + goto abort; + + rq = task_rq_lock(p, &rf); + + if (scx_get_task_state(p) == SCX_TASK_DEAD) { + /* + * sched_ext_dead() raced us between __scx_init_task() + * and this rq lock and ran exit_task() on $parent (the + * sched @p was on at that point), not on @sch. @sch's + * just-completed init is owed an exit_task() and we + * issue it here. + */ + scx_sub_init_cancel_task(sch, p); + task_rq_unlock(rq, p, &rf); + put_task_struct(p); + continue; + } + + p->scx.flags |= SCX_TASK_SUB_INIT; + task_rq_unlock(rq, p, &rf); + + put_task_struct(p); + } + scx_task_iter_stop(&sti); + + /* + * All tasks are prepped. Disable/exit tasks for $parent and enable for + * the new @sch. + */ + scx_task_iter_start(&sti, sch->cgrp); + while ((p = scx_task_iter_next_locked(&sti))) { + /* + * Use clearing of %SCX_TASK_SUB_INIT to detect and skip + * duplicate iterations. + */ + if (!(p->scx.flags & SCX_TASK_SUB_INIT)) + continue; + + scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { + /* + * $p must be either READY or ENABLED. If ENABLED, + * __scx_disabled_and_exit_task() first disables and + * makes it READY. However, after exiting $p, it will + * leave $p as READY. + */ + assert_task_ready_or_enabled(p); + __scx_disable_and_exit_task(parent, p); + + /* + * $p is now only initialized for @sch and READY, which + * is what we want. Assign it to @sch and enable. + */ + scx_set_task_sched(p, sch); + scx_enable_task(sch, p); + + p->scx.flags &= ~SCX_TASK_SUB_INIT; + } + } + scx_task_iter_stop(&sti); + + scx_enabling_sub_sched = NULL; + + scx_cgroup_unlock(); + percpu_up_write(&scx_fork_rwsem); + + scx_bypass(sch, false); + + pr_info("sched_ext: BPF sub-scheduler \"%s\" enabled\n", sch->ops.name); + kobject_uevent(&sch->kobj, KOBJ_ADD); + ret = 0; + goto out_unlock; + +out_put_cgrp: + cgroup_put(cgrp); +out_unlock: + mutex_unlock(&scx_enable_mutex); + cmd->ret = ret; + return; + +abort: + put_task_struct(p); + scx_task_iter_stop(&sti); + + /* + * Undo __scx_init_task() for tasks we marked. scx_enable_task() never + * ran for @sch on them, so calling scx_disable_task() here would invoke + * ops.disable() without a matching ops.enable(). scx_enabling_sub_sched + * must stay set until SUB_INIT is cleared from every marked task - + * scx_disable_and_exit_task() reads it when a task exits concurrently. + */ + scx_task_iter_start(&sti, sch->cgrp); + while ((p = scx_task_iter_next_locked(&sti))) { + if (p->scx.flags & SCX_TASK_SUB_INIT) { + scx_sub_init_cancel_task(sch, p); + p->scx.flags &= ~SCX_TASK_SUB_INIT; + } + } + scx_task_iter_stop(&sti); + scx_enabling_sub_sched = NULL; +err_unlock_and_disable: + /* we'll soon enter disable path, keep bypass on */ + scx_cgroup_unlock(); + percpu_up_write(&scx_fork_rwsem); +err_disable: + mutex_unlock(&scx_enable_mutex); + scx_flush_disable_work(sch); + cmd->ret = 0; +} + +static s32 scx_cgroup_lifetime_notify(struct notifier_block *nb, + unsigned long action, void *data) +{ + struct cgroup *cgrp = data; + struct cgroup *parent = cgroup_parent(cgrp); + + if (!cgroup_on_dfl(cgrp)) + return NOTIFY_OK; + + switch (action) { + case CGROUP_LIFETIME_ONLINE: + /* inherit ->scx_sched from $parent */ + if (parent) + rcu_assign_pointer(cgrp->scx_sched, parent->scx_sched); + break; + case CGROUP_LIFETIME_OFFLINE: + /* if there is a sched attached, shoot it down */ + if (cgrp->scx_sched && cgrp->scx_sched->cgrp == cgrp) + scx_exit(cgrp->scx_sched, SCX_EXIT_UNREG_KERN, + SCX_ECODE_RSN_CGROUP_OFFLINE, + "cgroup %llu going offline", cgroup_id(cgrp)); + break; + } + + return NOTIFY_OK; +} + +static struct notifier_block scx_cgroup_lifetime_nb = { + .notifier_call = scx_cgroup_lifetime_notify, +}; + +static s32 __init scx_cgroup_lifetime_notifier_init(void) +{ + return blocking_notifier_chain_register(&cgroup_lifetime_notifier, + &scx_cgroup_lifetime_nb); +} +core_initcall(scx_cgroup_lifetime_notifier_init); + +void scx_pstack_recursion_on_dispatch(struct bpf_prog *prog) +{ + struct scx_sched *sch; + + guard(rcu)(); + sch = scx_prog_sched(prog->aux); + if (unlikely(!sch)) + return; + + scx_error(sch, "dispatch recursion detected"); +} + +__bpf_kfunc_start_defs(); + +/** + * scx_bpf_sub_dispatch - Trigger dispatching on a child scheduler + * @cgroup_id: cgroup ID of the child scheduler to dispatch + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Allows a parent scheduler to trigger dispatching on one of its direct + * child schedulers. The child scheduler runs its dispatch operation to + * move tasks from dispatch queues to the local runqueue. + * + * Returns: true on success, false if cgroup_id is invalid, not a direct + * child, or caller lacks dispatch permission. + */ +__bpf_kfunc bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux *aux) +{ + struct rq *this_rq = this_rq(); + struct scx_sched *parent, *child; + + guard(rcu)(); + parent = scx_prog_sched(aux); + if (unlikely(!parent)) + return false; + + child = scx_find_sub_sched(cgroup_id); + + if (unlikely(!child)) + return false; + + if (unlikely(scx_parent(child) != parent)) { + scx_error(parent, "trying to dispatch a distant sub-sched on cgroup %llu", + cgroup_id); + return false; + } + + return scx_dispatch_sched(child, this_rq, this_rq->scx.sub_dispatch_prev, + true); +} + +__bpf_kfunc_end_defs(); + +#endif /* CONFIG_EXT_SUB_SCHED */ diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h new file mode 100644 index 000000000000..460a9fd196dc --- /dev/null +++ b/kernel/sched/ext/sub.h @@ -0,0 +1,161 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst + * + * Sub-scheduler hierarchy support. + * + * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2026 Tejun Heo + */ +#ifndef _KERNEL_SCHED_EXT_SUB_H +#define _KERNEL_SCHED_EXT_SUB_H + +#include "internal.h" +#include "cid.h" + +#ifdef CONFIG_EXT_SUB_SCHED + +struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root); +void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch); +struct cgroup *sch_cgroup(struct scx_sched *sch); +void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch); +void scx_pstack_recursion_on_dispatch(struct bpf_prog *prog); +void drain_descendants(struct scx_sched *sch); +void scx_sub_disable(struct scx_sched *sch); +void scx_sub_enable_workfn(struct kthread_work *work); +bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux *aux); + +#else /* CONFIG_EXT_SUB_SCHED */ + +static inline struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; } +static inline void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) {} +static inline struct cgroup *sch_cgroup(struct scx_sched *sch) { return NULL; } +static inline void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch) {} +static inline void drain_descendants(struct scx_sched *sch) { } +static inline void scx_sub_disable(struct scx_sched *sch) { } + +#endif /* CONFIG_EXT_SUB_SCHED */ + +/** + * scx_for_each_descendant_pre - pre-order walk of a sched's descendants + * @pos: iteration cursor + * @root: sched to walk the descendants of + * + * Walk @root's descendants. @root is included in the iteration and the first + * node to be visited. Must be called with either scx_enable_mutex or + * scx_sched_lock held. + */ +#define scx_for_each_descendant_pre(pos, root) \ + for ((pos) = scx_next_descendant_pre(NULL, (root)); (pos); \ + (pos) = scx_next_descendant_pre((pos), (root))) + +/* + * One user of this function is scx_bpf_dispatch() which can be called + * recursively as sub-sched dispatches nest. Always inline to reduce stack usage + * from the call frame. + */ +static __always_inline bool +scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, + struct task_struct *prev, bool nested) +{ + struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; + int nr_loops = SCX_DSP_MAX_LOOPS; + s32 cpu = cpu_of(rq); + bool prev_on_sch = (prev->sched_class == &ext_sched_class) && + scx_task_on_sched(sch, prev); + + if (scx_consume_global_dsq(sch, rq)) + return true; + + if (scx_bypass_dsp_enabled(sch)) { + /* if @sch is bypassing, only the bypass DSQs are active */ + if (scx_bypassing(sch, cpu)) + return scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0); + +#ifdef CONFIG_EXT_SUB_SCHED + /* + * If @sch isn't bypassing but its children are, @sch is + * responsible for making forward progress for both its own + * tasks that aren't bypassing and the bypassing descendants' + * tasks. The following implements a simple built-in behavior - + * let each CPU try to run the bypass DSQ every Nth time. + * + * Later, if necessary, we can add an ops flag to suppress the + * auto-consumption and a kfunc to consume the bypass DSQ and, + * so that the BPF scheduler can fully control scheduling of + * bypassed tasks. + */ + struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); + + if (!(pcpu->bypass_host_seq++ % SCX_BYPASS_HOST_NTH) && + scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0)) { + __scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1); + return true; + } +#endif /* CONFIG_EXT_SUB_SCHED */ + } + + if (unlikely(!SCX_HAS_OP(sch, dispatch)) || !scx_rq_online(rq)) + return false; + + dspc->rq = rq; + + /* + * The dispatch loop. Because scx_flush_dispatch_buf() may drop the rq + * lock, the local DSQ might still end up empty after a successful + * ops.dispatch(). If the local DSQ is empty even after ops.dispatch() + * produced some tasks, retry. The BPF scheduler may depend on this + * looping behavior to simplify its implementation. + */ + do { + dspc->nr_tasks = 0; + + if (nested) { + SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu), + prev_on_sch ? prev : NULL); + } else { + /* stash @prev so that nested invocations can access it */ + rq->scx.sub_dispatch_prev = prev; + SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu), + prev_on_sch ? prev : NULL); + rq->scx.sub_dispatch_prev = NULL; + } + + scx_flush_dispatch_buf(sch, rq); + + if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice) { + rq->scx.flags |= SCX_RQ_BAL_KEEP; + return true; + } + if (rq->scx.local_dsq.nr) + return true; + if (scx_consume_global_dsq(sch, rq)) + return true; + + /* + * ops.dispatch() can trap us in this loop by repeatedly + * dispatching ineligible tasks. Break out once in a while to + * allow the watchdog to run. As IRQ can't be enabled in + * balance(), we want to complete this scheduling cycle and then + * start a new one. IOW, we want to call resched_curr() on the + * next, most likely idle, task, not the current one. Use + * __scx_bpf_kick_cpu() for deferred kicking. + */ + if (unlikely(!--nr_loops)) { + scx_kick_cpu(sch, cpu, 0); + break; + } + } while (dspc->nr_tasks); + + /* + * Prevent the CPU from going idle while bypassed descendants have tasks + * queued. Without this fallback, bypassed tasks could stall if the host + * scheduler's ops.dispatch() doesn't yield any tasks. + */ + if (scx_bypass_dsp_enabled(sch)) + return scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0); + + return false; +} + +#endif /* _KERNEL_SCHED_EXT_SUB_H */ -- cgit v1.2.3 From f3c6919207ebade88fe40377b6a8b69294b4d35f Mon Sep 17 00:00:00 2001 From: Liang Luo Date: Thu, 9 Jul 2026 16:02:05 +0800 Subject: sched_ext: Fix typo in scx_bpf_dsq_insert() comment The comment for scx_bpf_dsq_insert() references "@dsp_id" in the description body, but the parameter is named "@dsq_id" in both the parameter list and the function signature. Signed-off-by: Liang Luo Acked-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 1a0ec985da77..29fa9f30aaaa 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -7764,7 +7764,7 @@ __bpf_kfunc_start_defs(); * When called from ops.select_cpu() or ops.enqueue(), it's for direct dispatch * and @p must match the task being enqueued. * - * When called from ops.select_cpu(), @enq_flags and @dsp_id are stored, and @p + * When called from ops.select_cpu(), @enq_flags and @dsq_id are stored, and @p * will be directly inserted into the corresponding dispatch queue after * ops.select_cpu() returns. If @p is inserted into SCX_DSQ_LOCAL, it will be * inserted into the local DSQ of the CPU returned by ops.select_cpu(). -- cgit v1.2.3 From 05e72aeaaa059aceb662a936007a95ea1912560b Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Thu, 9 Jul 2026 12:06:10 -1000 Subject: sched_ext: Use READ_ONCE/WRITE_ONCE in cmask word ops and drop _RACY variants The cmask ops can operate on BPF-arena cmasks which BPF programs may read and write concurrently. The _RACY op variants existed to make such lockless reads explicit but this turned out to be too restrictive. Mark the word accesses in all the two-cmask ops with READ_ONCE/WRITE_ONCE instead and drop the _RACY variants. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/cid.c | 55 +++++++++----------------------------------------- kernel/sched/ext/cid.h | 2 -- 2 files changed, 10 insertions(+), 47 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/cid.c b/kernel/sched/ext/cid.c index af83084ec740..f31113b080e9 100644 --- a/kernel/sched/ext/cid.c +++ b/kernel/sched/ext/cid.c @@ -395,17 +395,15 @@ __bpf_kfunc s32 scx_bpf_cpu_to_cid(s32 cpu, const struct bpf_prog_aux *aux) * bits outside stay untouched. In particular, scx_cmask_copy() does NOT zero * @dst bits that lie outside @src's range. * - * The _RACY variants are otherwise identical to their non-racy counterpart but - * read @src word-by-word via data_race(). Memory ordering with concurrent - * writers is the caller's responsibility. + * Word accesses use READ_ONCE/WRITE_ONCE so a caller may read @src + * locklessly. Memory ordering against concurrent writers is the caller's + * responsibility. */ enum cmask_op2 { /* mutating */ CMASK_OP2_AND, CMASK_OP2_OR, - CMASK_OP2_OR_RACY, CMASK_OP2_COPY, - CMASK_OP2_COPY_RACY, CMASK_OP2_ANDNOT, /* predicates - short-circuit when the per-word result is true */ CMASK_OP2_SUBSET, @@ -422,28 +420,22 @@ static __always_inline bool cmask_word_op2(u64 *av, const u64 *bp, u64 mask, { switch (op) { case CMASK_OP2_AND: - *av &= ~mask | *bp; + WRITE_ONCE(*av, *av & (~mask | READ_ONCE(*bp))); return false; case CMASK_OP2_OR: - *av |= *bp & mask; - return false; - case CMASK_OP2_OR_RACY: - *av |= data_race(*bp) & mask; + WRITE_ONCE(*av, *av | (READ_ONCE(*bp) & mask)); return false; case CMASK_OP2_COPY: - *av = (*av & ~mask) | (*bp & mask); - return false; - case CMASK_OP2_COPY_RACY: - *av = (*av & ~mask) | (data_race(*bp) & mask); + WRITE_ONCE(*av, (*av & ~mask) | (READ_ONCE(*bp) & mask)); return false; case CMASK_OP2_ANDNOT: - *av &= ~(*bp & mask); + WRITE_ONCE(*av, *av & ~(READ_ONCE(*bp) & mask)); return false; case CMASK_OP2_SUBSET: /* stop on the first bit in @sub not set in @super */ - return (*bp & ~*av) & mask; + return (READ_ONCE(*bp) & ~READ_ONCE(*av)) & mask; case CMASK_OP2_INTERSECTS: - return (*av & *bp) & mask; + return (READ_ONCE(*av) & READ_ONCE(*bp)) & mask; } unreachable(); } @@ -504,7 +496,7 @@ static __always_inline bool cmask_word_op1(const u64 *ap, u64 mask, { switch (op) { case CMASK_OP1_ANY_SET: - return *ap & mask; + return READ_ONCE(*ap) & mask; } unreachable(); } @@ -556,39 +548,12 @@ void scx_cmask_or(struct scx_cmask *dst, const struct scx_cmask *src) src->bits, src->base, src->nr_cids, CMASK_OP2_OR); } -/** - * scx_cmask_or_racy - OR @src into @dst, reading @src without locking - * - * @src is read word-by-word through data_race(). Same per-bit independence - * rationale as scx_cmask_copy_racy(). Memory ordering with writers is the - * caller's responsibility. - */ -void scx_cmask_or_racy(struct scx_cmask *dst, const struct scx_cmask *src) -{ - cmask_walk_op2(dst->bits, dst->base, dst->nr_cids, - src->bits, src->base, src->nr_cids, CMASK_OP2_OR_RACY); -} - void scx_cmask_copy(struct scx_cmask *dst, const struct scx_cmask *src) { cmask_walk_op2(dst->bits, dst->base, dst->nr_cids, src->bits, src->base, src->nr_cids, CMASK_OP2_COPY); } -/** - * scx_cmask_copy_racy - Snapshot @src into @dst without locking - * - * @src is read word-by-word through data_race(). Head/tail masking matches - * scx_cmask_copy(). Each bit in a cmask is independent, so partial updates - * just leave some bits fresher than others. Memory ordering with writers is - * the caller's responsibility. - */ -void scx_cmask_copy_racy(struct scx_cmask *dst, const struct scx_cmask *src) -{ - cmask_walk_op2(dst->bits, dst->base, dst->nr_cids, - src->bits, src->base, src->nr_cids, CMASK_OP2_COPY_RACY); -} - void scx_cmask_andnot(struct scx_cmask *dst, const struct scx_cmask *src) { cmask_walk_op2(dst->bits, dst->base, dst->nr_cids, diff --git a/kernel/sched/ext/cid.h b/kernel/sched/ext/cid.h index 9c4f4b907f12..54b10df32fd5 100644 --- a/kernel/sched/ext/cid.h +++ b/kernel/sched/ext/cid.h @@ -57,9 +57,7 @@ void scx_cmask_clear(struct scx_cmask *m); void scx_cmask_fill(struct scx_cmask *m); void scx_cmask_and(struct scx_cmask *dst, const struct scx_cmask *src); void scx_cmask_or(struct scx_cmask *dst, const struct scx_cmask *src); -void scx_cmask_or_racy(struct scx_cmask *dst, const struct scx_cmask *src); void scx_cmask_copy(struct scx_cmask *dst, const struct scx_cmask *src); -void scx_cmask_copy_racy(struct scx_cmask *dst, const struct scx_cmask *src); void scx_cmask_andnot(struct scx_cmask *dst, const struct scx_cmask *src); bool scx_cmask_subset(const struct scx_cmask *sub, const struct scx_cmask *super); bool scx_cmask_intersects(const struct scx_cmask *a, const struct scx_cmask *b); -- cgit v1.2.3 From 8da6d37d1dda3842249c7ea2b84daf88d52da557 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Thu, 9 Jul 2026 12:06:10 -1000 Subject: sched_ext: Add SCX_CALL_CID_OP_TASK() for cid-form op dispatch The cid-form ops overlay their cpu-form siblings at the same struct slot. Ops whose signature matches the sibling are invoked through the cpu-form call sites unchanged, but set_cmask() takes an arena cmask address rather than a cpumask, so scx_call_op_set_cpumask() calls ops_cid.set_cmask() directly and hand-rolled the kf_tasks[] and locked_rq bracket that SCX_CALL_OP_TASK() provides. The hand-rolled bracket reset locked_rq to NULL on exit instead of restoring the saved value, so a nested call would clobber the outer op's locked-rq tracking. Parameterize the dispatch macros by the ops-table member and add SCX_CALL_CID_OP_TASK(), which routes through sch->ops_cid. Convert scx_call_op_set_cpumask() to it and drop the hand-rolled bracket. The only behavioral change is that locked_rq is now saved and restored like every other op call site. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 14 +++----------- kernel/sched/ext/internal.h | 23 ++++++++++++++++++++--- 2 files changed, 23 insertions(+), 14 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 032f4f908796..806e8ae1cf33 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -422,11 +422,6 @@ static inline void scx_call_op_set_cpumask(struct scx_sched *sch, struct rq *rq, struct task_struct *task, const struct cpumask *cpumask) { - WARN_ON_ONCE(current->scx.kf_tasks[0]); - current->scx.kf_tasks[0] = task; - if (rq) - update_locked_rq(rq); - if (scx_is_cid_type()) { struct scx_cmask *kern_va = *this_cpu_ptr(sch->set_cmask_scratch); /* @@ -435,14 +430,11 @@ static inline void scx_call_op_set_cpumask(struct scx_sched *sch, struct rq *rq, * the sole user of the scratch area. */ scx_cpumask_to_cmask(cpumask, kern_va); - sch->ops_cid.set_cmask(task, scx_kaddr_to_arena(sch, kern_va)); + SCX_CALL_CID_OP_TASK(sch, set_cmask, rq, task, + scx_kaddr_to_arena(sch, kern_va)); } else { - sch->ops.set_cpumask(task, cpumask); + SCX_CALL_OP_TASK(sch, set_cpumask, rq, task, cpumask); } - - if (rq) - update_locked_rq(NULL); - current->scx.kf_tasks[0] = NULL; } enum scx_dsq_iter_flags { diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index f9fe7c6ebc4b..5ca44ad88786 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1751,8 +1751,11 @@ static inline void update_locked_rq(struct rq *rq) /* * SCX ops can recurse via scx_bpf_sub_dispatch() - the inner call must not * clobber the outer's scx_locked_rq_state. Save it on entry, restore on exit. + * + * @ops is the ops table to dispatch through: ops for the cpu form, ops_cid + * for the cid form. */ -#define SCX_CALL_OP(sch, op, locked_rq, args...) \ +#define __SCX_CALL_OP(sch, ops, op, locked_rq, args...) \ do { \ struct rq *__prev_locked_rq; \ \ @@ -1765,6 +1768,9 @@ do { \ update_locked_rq(__prev_locked_rq); \ } while (0) +#define SCX_CALL_OP(sch, op, locked_rq, args...) \ + __SCX_CALL_OP(sch, ops, op, locked_rq, ##args) + #define SCX_CALL_OP_RET(sch, op, locked_rq, args...) \ ({ \ struct rq *__prev_locked_rq; \ @@ -1796,14 +1802,25 @@ do { \ * WARN_ON_ONCE() in each macro catches a re-entry of any of the three variants * while a previous one is still in progress. */ -#define SCX_CALL_OP_TASK(sch, op, locked_rq, task, args...) \ +#define __SCX_CALL_OP_TASK(sch, ops, op, locked_rq, task, args...) \ do { \ WARN_ON_ONCE(current->scx.kf_tasks[0]); \ current->scx.kf_tasks[0] = task; \ - SCX_CALL_OP((sch), op, locked_rq, task, ##args); \ + __SCX_CALL_OP((sch), ops, op, locked_rq, task, ##args); \ current->scx.kf_tasks[0] = NULL; \ } while (0) +#define SCX_CALL_OP_TASK(sch, op, locked_rq, task, args...) \ + __SCX_CALL_OP_TASK(sch, ops, op, locked_rq, task, ##args) + +/* + * Dispatch a task op through the cid-form ops_cid table. Only set_cmask() needs + * this: it takes an arena cmask address instead of a cpumask, so it cannot be + * invoked via its cpu-form set_cpumask() slot. + */ +#define SCX_CALL_CID_OP_TASK(sch, op, locked_rq, task, args...) \ + __SCX_CALL_OP_TASK(sch, ops_cid, op, locked_rq, task, ##args) + #define SCX_CALL_OP_TASK_RET(sch, op, locked_rq, task, args...) \ ({ \ __typeof__((sch)->ops.op(task, ##args)) __ret; \ -- cgit v1.2.3 From 915feb4119e09292f0b0a7392120a7995dc3114a Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Thu, 9 Jul 2026 12:06:11 -1000 Subject: sched_ext: Rename extra_enq_flags to remote_activate_enq_flags scx_rq.extra_enq_flags carries scx-specific enqueue flags across the activate_task() boundary during a cross-rq task move in move_remote_task_to_local_dsq(). Rename it to remote_activate_enq_flags to name that role, and fix the stale comment that referenced the old move_task_to_local_dsq() name. No functional change. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 10 +++++----- kernel/sched/sched.h | 3 ++- 2 files changed, 7 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 806e8ae1cf33..dbf2f52388df 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1836,7 +1836,7 @@ static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_ { struct scx_sched *sch = scx_task_sched(p); int sticky_cpu = p->scx.sticky_cpu; - u64 enq_flags = core_enq_flags | rq->scx.extra_enq_flags; + u64 enq_flags = core_enq_flags | rq->scx.remote_activate_enq_flags; if (enq_flags & ENQUEUE_WAKEUP) rq->scx.flags |= SCX_RQ_IN_WAKEUP; @@ -2108,13 +2108,13 @@ static void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags, /* * We want to pass scx-specific enq_flags but activate_task() will * truncate the upper 32 bit. As we own @rq, we can pass them through - * @rq->scx.extra_enq_flags instead. + * @rq->scx.remote_activate_enq_flags instead. */ WARN_ON_ONCE(!cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr)); - WARN_ON_ONCE(dst_rq->scx.extra_enq_flags); - dst_rq->scx.extra_enq_flags = enq_flags; + WARN_ON_ONCE(dst_rq->scx.remote_activate_enq_flags); + dst_rq->scx.remote_activate_enq_flags = enq_flags; activate_task(dst_rq, p, 0); - dst_rq->scx.extra_enq_flags = 0; + dst_rq->scx.remote_activate_enq_flags = 0; } /* diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 56acf502ba26..80b72d934ff3 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -797,7 +797,8 @@ struct scx_rq { struct list_head runnable_list; /* runnable tasks on this rq */ struct list_head ddsp_deferred_locals; /* deferred ddsps from enq */ unsigned long ops_qseq; - u64 extra_enq_flags; /* see move_task_to_local_dsq() */ + /* see move_remote_task_to_local_dsq() */ + u64 remote_activate_enq_flags; u32 nr_running; u32 cpuperf_target; /* [0, SCHED_CAPACITY_SCALE] */ bool in_select_cpu; -- cgit v1.2.3 From 31645fb113af646792ebb1e51eceebb9a04fdea1 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Thu, 9 Jul 2026 12:06:11 -1000 Subject: sched_ext: Reject direct slice and dsq_vtime writes for cid-form schedulers Direct writes to p->scx.slice and p->scx.dsq_vtime bypass scx_bpf_task_set_slice/dsq_vtime() and the authority checks they carry. Those checks exist for sub-schedulers, which attach only through the cid-form struct_ops, so the direct writes only need to be closed there. Give sched_ext_ops_cid its own verifier ops that reject the two fields. cid-form is a new interface with no legacy users, so there is no compatibility to keep. The cpu-form keeps direct writes, and the deprecation warning they carried is dropped. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 44 ++++++++++++++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 12 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index dbf2f52388df..fedda8f9ebe3 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -7026,6 +7026,21 @@ static bool bpf_scx_is_valid_access(int off, int size, return btf_ctx_access(off, size, type, prog, info); } +/* common to both forms: only scx.disallow is writable */ +static int bpf_scx_btf_struct_access_common(const struct bpf_reg_state *reg, + int off, int size) +{ + const struct btf_type *t; + + t = btf_type_by_id(reg->btf, reg->btf_id); + if (t == task_struct_type && + off >= offsetof(struct task_struct, scx.disallow) && + off + size <= offsetofend(struct task_struct, scx.disallow)) + return SCALAR_VALUE; + + return -EACCES; +} + static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log, const struct bpf_reg_state *reg, int off, int size) @@ -7034,23 +7049,22 @@ static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log, t = btf_type_by_id(reg->btf, reg->btf_id); if (t == task_struct_type) { - /* - * COMPAT: Will be removed in v6.23. - */ if ((off >= offsetof(struct task_struct, scx.slice) && off + size <= offsetofend(struct task_struct, scx.slice)) || (off >= offsetof(struct task_struct, scx.dsq_vtime) && - off + size <= offsetofend(struct task_struct, scx.dsq_vtime))) { - pr_warn_ratelimited("sched_ext: Writing directly to p->scx.slice/dsq_vtime is deprecated, use scx_bpf_task_set_slice/dsq_vtime()\n"); - return SCALAR_VALUE; - } - - if (off >= offsetof(struct task_struct, scx.disallow) && - off + size <= offsetofend(struct task_struct, scx.disallow)) + off + size <= offsetofend(struct task_struct, scx.dsq_vtime))) return SCALAR_VALUE; } - return -EACCES; + return bpf_scx_btf_struct_access_common(reg, off, size); +} + +/* cid-form rejects direct slice and dsq_vtime writes in favor of the kfuncs */ +static int bpf_scx_cid_btf_struct_access(struct bpf_verifier_log *log, + const struct bpf_reg_state *reg, int off, + int size) +{ + return bpf_scx_btf_struct_access_common(reg, off, size); } static const struct bpf_verifier_ops bpf_scx_verifier_ops = { @@ -7059,6 +7073,12 @@ static const struct bpf_verifier_ops bpf_scx_verifier_ops = { .btf_struct_access = bpf_scx_btf_struct_access, }; +static const struct bpf_verifier_ops bpf_scx_cid_verifier_ops = { + .get_func_proto = bpf_base_func_proto, + .is_valid_access = bpf_scx_is_valid_access, + .btf_struct_access = bpf_scx_cid_btf_struct_access, +}; + static int bpf_scx_init_member(const struct btf_type *t, const struct btf_member *member, void *kdata, const void *udata) @@ -7399,7 +7419,7 @@ static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = { * verified to match by the BUILD_BUG_ON checks in scx_init(). */ static struct bpf_struct_ops bpf_sched_ext_ops_cid = { - .verifier_ops = &bpf_scx_verifier_ops, + .verifier_ops = &bpf_scx_cid_verifier_ops, .reg = bpf_scx_reg_cid, .unreg = bpf_scx_unreg, .check_member = bpf_scx_check_member, -- cgit v1.2.3 From b38332be61a8a76cf77586fb769849257ff217ae Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Thu, 9 Jul 2026 12:06:11 -1000 Subject: sched_ext: Make scx_bpf_kick_cid() return void scx_bpf_kick_cid() returned an error code, but the value conveys nothing actionable and no caller consumes it. The kick is asynchronous, so a successful return only means it was queued. An invalid @cid is already reported through scx_error() by scx_cid_to_cpu(), and a missing scheduler leaves nothing to kick. Make scx_bpf_kick_cid() return void to match scx_bpf_kick_cpu(). The cid-form kfuncs are not in practical use yet, so the ABI change is safe. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 11 +++++------ tools/sched_ext/include/scx/common.bpf.h | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index fedda8f9ebe3..80cc7f68a3cd 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -8560,10 +8560,10 @@ __bpf_kfunc void scx_bpf_kick_cpu(s32 cpu, u64 flags, const struct bpf_prog_aux * @flags: %SCX_KICK_* flags * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * - * cid-addressed equivalent of scx_bpf_kick_cpu(). Return 0 on success, - * -errno otherwise. + * cid-addressed equivalent of scx_bpf_kick_cpu(). An invalid @cid aborts the + * scheduler via scx_cid_to_cpu(). */ -__bpf_kfunc s32 scx_bpf_kick_cid(s32 cid, u64 flags, const struct bpf_prog_aux *aux) +__bpf_kfunc void scx_bpf_kick_cid(s32 cid, u64 flags, const struct bpf_prog_aux *aux) { struct scx_sched *sch; s32 cpu; @@ -8571,12 +8571,11 @@ __bpf_kfunc s32 scx_bpf_kick_cid(s32 cid, u64 flags, const struct bpf_prog_aux * guard(rcu)(); sch = scx_prog_sched(aux); if (unlikely(!sch)) - return -ENODEV; + return; cpu = scx_cid_to_cpu(sch, cid); if (cpu < 0) - return cpu; + return; scx_kick_cpu(sch, cpu, flags); - return 0; } /** diff --git a/tools/sched_ext/include/scx/common.bpf.h b/tools/sched_ext/include/scx/common.bpf.h index bd51986c4c42..e7b3ba491c5e 100644 --- a/tools/sched_ext/include/scx/common.bpf.h +++ b/tools/sched_ext/include/scx/common.bpf.h @@ -104,7 +104,7 @@ void scx_bpf_events(struct scx_event_stats *events, size_t events__sz) __ksym __ s32 scx_bpf_cpu_to_cid(s32 cpu) __ksym __weak; s32 scx_bpf_cid_to_cpu(s32 cid) __ksym __weak; void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out) __ksym __weak; -s32 scx_bpf_kick_cid(s32 cid, u64 flags) __ksym __weak; +void scx_bpf_kick_cid(s32 cid, u64 flags) __ksym __weak; s32 scx_bpf_task_cid(const struct task_struct *p) __ksym __weak; s32 scx_bpf_this_cid(void) __ksym __weak; struct task_struct *scx_bpf_cid_curr(s32 cid) __ksym __weak; -- cgit v1.2.3 From 3d1519011e395ea96c7fbc5ced35b49cfe60d93e Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Thu, 9 Jul 2026 07:17:08 +0200 Subject: sched_ext: Keep rq tracking accurate in the consume path consume_remote_task() drops this_rq before unlinking a remote task from its DSQ and locking src_rq. When called from ops.dispatch(), scx_locked_rq() keeps pointing to this_rq across the lock dance. The subsequent switch_rq_lock(src_rq, this_rq) cannot update the tracking because its guard does not match, leaving the tracking stale while src_rq is held. Keep this_rq locked until the task has been unlinked and the DSQ lock released, then use switch_rq_lock() to switch directly to src_rq. Use the same helper to restore this_rq after losing the dequeue race. The successful path already switches back through move_remote_task_to_local_dsq(), so scx_locked_rq() now follows the actually held rq throughout the consume path. Suggested-by: Tejun Heo Link: https://lore.kernel.org/all/455e701bca66bdecde530d225f4dba0a@kernel.org Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 80cc7f68a3cd..5241b55a58ec 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -2195,13 +2195,14 @@ static bool task_can_run_on_remote_rq(struct scx_sched *sch, } /** - * unlink_dsq_and_lock_src_rq() - Unlink task from its DSQ and lock its task_rq + * unlink_dsq_and_switch_rq_lock() - Unlink task and switch to its rq lock * @p: target task * @dsq: locked DSQ @p is currently on + * @locked_rq: currently locked rq * @src_rq: rq @p is currently on, stable with @dsq locked * - * Called with @dsq locked but no rq's locked. We want to move @p to a different - * DSQ, including any local DSQ, but are not locking @src_rq. Locking @src_rq is + * Called with @dsq and @locked_rq locked. We want to move @p to a different DSQ, + * including any local DSQ, but are not locking @src_rq. Locking @src_rq is * required when transferring into a local DSQ. Even when transferring into a * non-local DSQ, it's better to use the same mechanism to protect against * dequeues and maintain the invariant that @p->scx.dsq can only change while @@ -2223,20 +2224,22 @@ static bool task_can_run_on_remote_rq(struct scx_sched *sch, * On return, @dsq is unlocked and @src_rq is locked. Returns %true if @p is * still valid. %false if lost to dequeue. */ -static bool unlink_dsq_and_lock_src_rq(struct task_struct *p, - struct scx_dispatch_q *dsq, - struct rq *src_rq) +static bool unlink_dsq_and_switch_rq_lock(struct task_struct *p, + struct scx_dispatch_q *dsq, + struct rq *locked_rq, + struct rq *src_rq) { s32 cpu = raw_smp_processor_id(); lockdep_assert_held(&dsq->lock); + lockdep_assert_rq_held(locked_rq); WARN_ON_ONCE(p->scx.holding_cpu >= 0); task_unlink_from_dsq(p, dsq); p->scx.holding_cpu = cpu; raw_spin_unlock(&dsq->lock); - raw_spin_rq_lock(src_rq); + switch_rq_lock(locked_rq, src_rq); /* task_rq couldn't have changed if we're still the holding cpu */ return likely(p->scx.holding_cpu == cpu) && @@ -2247,14 +2250,11 @@ static bool consume_remote_task(struct rq *this_rq, struct task_struct *p, u64 enq_flags, struct scx_dispatch_q *dsq, struct rq *src_rq) { - raw_spin_rq_unlock(this_rq); - - if (unlink_dsq_and_lock_src_rq(p, dsq, src_rq)) { + if (unlink_dsq_and_switch_rq_lock(p, dsq, this_rq, src_rq)) { move_remote_task_to_local_dsq(p, enq_flags, src_rq, this_rq); return true; } else { - raw_spin_rq_unlock(src_rq); - raw_spin_rq_lock(this_rq); + switch_rq_lock(src_rq, this_rq); return false; } } @@ -2427,7 +2427,7 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, * As DISPATCHING guarantees that @p is wholly ours, we can pretend that * we're moving from a DSQ and use the same mechanism - mark the task * under transfer with holding_cpu, release DISPATCHING and then follow - * the same protocol. See unlink_dsq_and_lock_src_rq(). + * the same protocol. See unlink_dsq_and_switch_rq_lock(). */ p->scx.holding_cpu = raw_smp_processor_id(); -- cgit v1.2.3 From cbcda14b12fbc8c89546bfe4568df9b984238687 Mon Sep 17 00:00:00 2001 From: Pat Somaru Date: Fri, 10 Jul 2026 01:59:11 -0400 Subject: sched_ext: Add tracepoint for scheduler exit sched_ext schedulers have state in BPF programs and kernel. scx_dump provides kernel state and BPF program state on error, but this is static in what it can provide. Add a sched_ext_exit tracepoint in scx_claim_exit() so that BPF programs can dynamically inspect scheduler specific state at the moment of exit. Pass the exiting scx_sched so attached programs can read its state, and, since exits propagate through a hierarchy of sub-schedulers, identify which scheduler each event belongs to. Signed-off-by: Pat Somaru Signed-off-by: Tejun Heo --- include/trace/events/sched_ext.h | 28 ++++++++++++++++++++++++++++ kernel/sched/ext/ext.c | 2 ++ kernel/sched/ext/sub.h | 6 ++++++ 3 files changed, 36 insertions(+) (limited to 'kernel') diff --git a/include/trace/events/sched_ext.h b/include/trace/events/sched_ext.h index d1bf5acd59c5..1e54f9564ef3 100644 --- a/include/trace/events/sched_ext.h +++ b/include/trace/events/sched_ext.h @@ -84,6 +84,34 @@ TRACE_EVENT(sched_ext_bypass_lb, ) ); +TRACE_EVENT(sched_ext_exit, + + TP_PROTO(struct scx_sched *sch, __u32 kind), + + TP_ARGS(sch, kind), + + TP_STRUCT__entry( + __string( name, sch->ops.name ) + __field( __s32, level ) + __field( __u64, sub_cgroup_id ) + __string( cgrp_path, sch_cgrp_path(sch) ) + __field( __u32, kind ) + ), + + TP_fast_assign( + __assign_str(name); + __entry->level = sch->level; + __entry->sub_cgroup_id = sch->ops.sub_cgroup_id; + __assign_str(cgrp_path); + __entry->kind = kind; + ), + + TP_printk("sched %s level %d sub_cgroup_id %llu cgrp_path %s kind %u", + __get_str(name), __entry->level, __entry->sub_cgroup_id, + __get_str(cgrp_path), __entry->kind + ) +); + #endif /* _TRACE_SCHED_EXT_H */ /* This part must be outside protection */ diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 5241b55a58ec..9aeb378fa894 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -5753,6 +5753,8 @@ static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind) */ WRITE_ONCE(sch->aborting, true); + trace_sched_ext_exit(sch, kind); + /* * Propagate exits to descendants immediately. Each has a dedicated * helper kthread and can run in parallel. While most of disabling is diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index 460a9fd196dc..9b5ac07e5e76 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -25,11 +25,17 @@ void scx_sub_disable(struct scx_sched *sch); void scx_sub_enable_workfn(struct kthread_work *work); bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux *aux); +static inline const char *sch_cgrp_path(struct scx_sched *sch) +{ + return sch->cgrp_path; +} + #else /* CONFIG_EXT_SUB_SCHED */ static inline struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; } static inline void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) {} static inline struct cgroup *sch_cgroup(struct scx_sched *sch) { return NULL; } +static inline const char *sch_cgrp_path(struct scx_sched *sch) { return "/"; } static inline void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch) {} static inline void drain_descendants(struct scx_sched *sch) { } static inline void scx_sub_disable(struct scx_sched *sch) { } -- cgit v1.2.3 From 6c646d053f61133f3185b0904a12f6b38a3465d4 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 13 Jul 2026 22:18:42 -1000 Subject: sched_ext: Assert per-task ops run on the task's owner A per-task op must be dispatched on the scheduler that owns the task. SCX_CALL_OP_TASK() and its _RET twin take @sch explicitly, and a caller that passes the wrong scheduler would silently run the op on it. Add a WARN_ON_ONCE() that @sch matches the task's owner so such a mismatch is caught rather than hidden. Two sites legitimately target a scheduler other than the task's owner: cgroup_move() runs on the root sched, and scx_sub_init_cancel_task() fires exit_task() on a task not yet associated with @sch. Both switch to the inner __SCX_CALL_OP_TASK(), which dispatches on the explicit @sch without the assert. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 20 +++++++++++++------- kernel/sched/ext/internal.h | 10 +++++++++- 2 files changed, 22 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 9aeb378fa894..b4cc097def72 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1436,8 +1436,12 @@ static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq, local_dsq_post_enq(sch, dsq, p, enq_flags); } else { /* - * Task on global/bypass DSQ: leave custody, task on - * non-terminal DSQ: enter custody. + * Global and bypass DSQs are terminal - the task leaves the + * scheduler's custody, so ops.dequeue() fires here. It can run + * without @p's rq lock (finish_dispatch() passes the dispatch + * rq); that's safe because dequeue_task_scx() waits on + * SCX_OPSS_DISPATCHING (see the ops_state note above) and so + * can't race it. A non-terminal DSQ keeps the task in custody. */ if (dsq->id == SCX_DSQ_GLOBAL || dsq->id == SCX_DSQ_BYPASS) call_task_dequeue(sch, rq, p, 0); @@ -3446,8 +3450,9 @@ void scx_sub_init_cancel_task(struct scx_sched *sch, struct task_struct *p) lockdep_assert_held(&p->pi_lock); lockdep_assert_rq_held(task_rq(p)); + /* @p was never associated with @sch, dispatch on the explicit @sch */ if (SCX_HAS_OP(sch, exit_task)) - SCX_CALL_OP_TASK(sch, exit_task, task_rq(p), p, &args); + __SCX_CALL_OP_TASK(sch, ops, exit_task, task_rq(p), p, &args); } void scx_disable_and_exit_task(struct scx_sched *sch, struct task_struct *p) @@ -4184,12 +4189,13 @@ void scx_cgroup_move_task(struct task_struct *p) * cgroup changes. Migration keys off css rather than cgroup identity, * so it can hand an unchanged-cgroup task here with cgrp_moving_from * NULL. Nothing to report to the BPF scheduler then, so skip it and - * keep prep_move and move paired. + * keep prep_move and move paired. Cgroup ops run on the root sched, + * dispatch on the explicit @sch. */ if (SCX_HAS_OP(sch, cgroup_move) && p->scx.cgrp_moving_from) - SCX_CALL_OP_TASK(sch, cgroup_move, task_rq(p), - p, p->scx.cgrp_moving_from, - tg_cgrp(task_group(p))); + __SCX_CALL_OP_TASK(sch, ops, cgroup_move, task_rq(p), + p, p->scx.cgrp_moving_from, + tg_cgrp(task_group(p))); p->scx.cgrp_moving_from = NULL; } diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 5ca44ad88786..8aabd92fae3b 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1810,8 +1810,15 @@ do { \ current->scx.kf_tasks[0] = NULL; \ } while (0) +/* + * A per-task op runs on @task's owner - WARN if @sch isn't it. Sites that must + * target a different scheduler call __SCX_CALL_OP_TASK() directly. + */ #define SCX_CALL_OP_TASK(sch, op, locked_rq, task, args...) \ - __SCX_CALL_OP_TASK(sch, ops, op, locked_rq, task, ##args) +do { \ + WARN_ON_ONCE((sch) != scx_task_sched_rcu(task)); \ + __SCX_CALL_OP_TASK((sch), ops, op, locked_rq, task, ##args); \ +} while (0) /* * Dispatch a task op through the cid-form ops_cid table. Only set_cmask() needs @@ -1824,6 +1831,7 @@ do { \ #define SCX_CALL_OP_TASK_RET(sch, op, locked_rq, task, args...) \ ({ \ __typeof__((sch)->ops.op(task, ##args)) __ret; \ + WARN_ON_ONCE((sch) != scx_task_sched_rcu(task)); \ WARN_ON_ONCE(current->scx.kf_tasks[0]); \ current->scx.kf_tasks[0] = task; \ __ret = SCX_CALL_OP_RET((sch), op, locked_rq, task, ##args); \ -- cgit v1.2.3 From 8b75c71f23e528e8662f48e7e8635cc1d32c2fdd Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 13 Jul 2026 22:18:42 -1000 Subject: sched_ext: Make the kick machinery per-sched The kick machinery kept its targets in rq->scx shared by every sched on the cpu. A preempt kick carried no record of which scheduler requested it. A later patch needs preempt kicks scoped to the requesting scheduler so a sub-scheduler can preempt only tasks in its own subtree. Move the kick masks into the per-sched per-cpu scx_sched_pcpu and have scx_kick_cpu() link the sched onto a per-cpu list (rq->scx.sched_pcpus_to_kick). The cpu's single kick irq_work walks that list and kicks each sched's targets on its behalf, so a kick stays attributed to its scheduler. The SCX_KICK_WAIT sync set (cpus_to_sync, the kick_sync snapshot and the balance-callback trigger) stays in rq->scx: the waiter is the cpu, not the scheduler, and its only writers, the kick irq_work and the wait balance callback, are cpu-local. On disable, free_kick_syncs() flushes each cpu's pending kick irq_work before clearing @ksyncs, so a late kick unlinks its to_kick_node instead of early-returning on a NULL @ksyncs and leaving the node linked at free. v3: Flush the kick irq_work in free_kick_syncs() before clearing @ksyncs. (sashiko AI) v2: Warn once per sched on scx_bpf_kick_cpu() from NMI. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 121 ++++++++++++++++++++++++++++++++------------ kernel/sched/ext/internal.h | 14 +++++ kernel/sched/sched.h | 6 +-- 3 files changed, 106 insertions(+), 35 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index b4cc097def72..ab0b6f15e318 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -4661,6 +4661,17 @@ static void scx_sched_free_rcu_work(struct work_struct *work) */ WARN_ON_ONCE(!list_empty(&pcpu->deferred_reenq_local.node)); + /* + * Bypass blocks new kicks. Flush the kick irq_work so this + * pcpu's to_kick_node is off the list before it is freed. + */ + irq_work_sync(&cpu_rq(cpu)->scx.kick_cpus_irq_work); + WARN_ON_ONCE(!list_empty(&pcpu->to_kick_node)); + free_cpumask_var(pcpu->cpus_to_kick); + free_cpumask_var(pcpu->cpus_to_kick_if_idle); + free_cpumask_var(pcpu->cpus_to_preempt); + free_cpumask_var(pcpu->cpus_to_wait); + exit_dsq(scx_bypass_dsq(sch, cpu)); } @@ -5453,6 +5464,8 @@ static void free_kick_syncs(void) struct scx_kick_syncs __rcu **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu); struct scx_kick_syncs *to_free; + /* flush the pending kick before freeing @ksyncs */ + irq_work_sync(&cpu_rq(cpu)->scx.kick_cpus_irq_work); to_free = rcu_replace_pointer(*ksyncs, NULL, true); if (to_free) kvfree_rcu(to_free, rcu); @@ -6006,6 +6019,7 @@ static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s, bool dump_all_tasks) { struct rq *rq = cpu_rq(cpu); + struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); struct rq_flags rf; struct task_struct *p; struct seq_buf ns; @@ -6038,18 +6052,18 @@ static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s, dump_line(&ns, " curr=%s[%d] class=%ps", rq->curr->comm, rq->curr->pid, rq->curr->sched_class); - if (!cpumask_empty(rq->scx.cpus_to_kick)) + if (!cpumask_empty(pcpu->cpus_to_kick)) dump_line(&ns, " cpus_to_kick : %*pb", - cpumask_pr_args(rq->scx.cpus_to_kick)); - if (!cpumask_empty(rq->scx.cpus_to_kick_if_idle)) + cpumask_pr_args(pcpu->cpus_to_kick)); + if (!cpumask_empty(pcpu->cpus_to_kick_if_idle)) dump_line(&ns, " idle_to_kick : %*pb", - cpumask_pr_args(rq->scx.cpus_to_kick_if_idle)); - if (!cpumask_empty(rq->scx.cpus_to_preempt)) + cpumask_pr_args(pcpu->cpus_to_kick_if_idle)); + if (!cpumask_empty(pcpu->cpus_to_preempt)) dump_line(&ns, " cpus_to_preempt: %*pb", - cpumask_pr_args(rq->scx.cpus_to_preempt)); - if (!cpumask_empty(rq->scx.cpus_to_wait)) + cpumask_pr_args(pcpu->cpus_to_preempt)); + if (!cpumask_empty(pcpu->cpus_to_wait)) dump_line(&ns, " cpus_to_wait : %*pb", - cpumask_pr_args(rq->scx.cpus_to_wait)); + cpumask_pr_args(pcpu->cpus_to_wait)); if (!cpumask_empty(rq->scx.cpus_to_sync)) dump_line(&ns, " cpus_to_sync : %*pb", cpumask_pr_args(rq->scx.cpus_to_sync)); @@ -6341,8 +6355,17 @@ struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, for_each_possible_cpu(cpu) { struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); + node = cpu_to_node(cpu); pcpu->sch = sch; INIT_LIST_HEAD(&pcpu->deferred_reenq_local.node); + INIT_LIST_HEAD(&pcpu->to_kick_node); + if (!zalloc_cpumask_var_node(&pcpu->cpus_to_kick, GFP_KERNEL, node) || + !zalloc_cpumask_var_node(&pcpu->cpus_to_kick_if_idle, GFP_KERNEL, node) || + !zalloc_cpumask_var_node(&pcpu->cpus_to_preempt, GFP_KERNEL, node) || + !zalloc_cpumask_var_node(&pcpu->cpus_to_wait, GFP_KERNEL, node)) { + ret = -ENOMEM; + goto err_free_pcpu; + } } sch->helper = kthread_run_worker(0, "sched_ext_helper"); @@ -6487,6 +6510,14 @@ err_free_lb_cpumask: err_stop_helper: kthread_destroy_worker(sch->helper); err_free_pcpu: + for_each_possible_cpu(cpu) { + struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); + + free_cpumask_var(pcpu->cpus_to_kick); + free_cpumask_var(pcpu->cpus_to_kick_if_idle); + free_cpumask_var(pcpu->cpus_to_preempt); + free_cpumask_var(pcpu->cpus_to_wait); + } for_each_possible_cpu(cpu) { if (cpu == bypass_fail_cpu) break; @@ -7500,7 +7531,8 @@ static bool can_skip_idle_kick(struct rq *rq) return !is_idle_task(rq->curr) && !(rq->scx.flags & SCX_RQ_IN_BALANCE); } -static bool kick_one_cpu(s32 cpu, struct rq *this_rq, unsigned long *ksyncs) +static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_rq, + unsigned long *ksyncs) { struct rq *rq = cpu_rq(cpu); struct scx_rq *this_scx = &this_rq->scx; @@ -7519,25 +7551,25 @@ static bool kick_one_cpu(s32 cpu, struct rq *this_rq, unsigned long *ksyncs) */ if ((cpu_online(cpu) || cpu == cpu_of(this_rq)) && !sched_class_above(cur_class, &ext_sched_class)) { - if (cpumask_test_cpu(cpu, this_scx->cpus_to_preempt)) { + if (cpumask_test_cpu(cpu, pcpu->cpus_to_preempt)) { if (cur_class == &ext_sched_class) rq->curr->scx.slice = 0; - cpumask_clear_cpu(cpu, this_scx->cpus_to_preempt); + cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt); } - if (cpumask_test_cpu(cpu, this_scx->cpus_to_wait)) { + if (cpumask_test_cpu(cpu, pcpu->cpus_to_wait)) { if (cur_class == &ext_sched_class) { cpumask_set_cpu(cpu, this_scx->cpus_to_sync); ksyncs[cpu] = rq->scx.kick_sync; should_wait = true; } - cpumask_clear_cpu(cpu, this_scx->cpus_to_wait); + cpumask_clear_cpu(cpu, pcpu->cpus_to_wait); } resched_curr(rq); } else { - cpumask_clear_cpu(cpu, this_scx->cpus_to_preempt); - cpumask_clear_cpu(cpu, this_scx->cpus_to_wait); + cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt); + cpumask_clear_cpu(cpu, pcpu->cpus_to_wait); } raw_spin_rq_unlock_irqrestore(rq, flags); @@ -7564,6 +7596,7 @@ static void kick_cpus_irq_workfn(struct irq_work *irq_work) struct rq *this_rq = this_rq(); struct scx_rq *this_scx = &this_rq->scx; struct scx_kick_syncs __rcu *ksyncs_pcpu = __this_cpu_read(scx_kick_syncs); + struct scx_sched_pcpu *pcpu, *tmp; bool should_wait = false; unsigned long *ksyncs; s32 cpu; @@ -7574,15 +7607,24 @@ static void kick_cpus_irq_workfn(struct irq_work *irq_work) ksyncs = rcu_dereference_bh(ksyncs_pcpu)->syncs; - for_each_cpu(cpu, this_scx->cpus_to_kick) { - should_wait |= kick_one_cpu(cpu, this_rq, ksyncs); - cpumask_clear_cpu(cpu, this_scx->cpus_to_kick); - cpumask_clear_cpu(cpu, this_scx->cpus_to_kick_if_idle); - } + /* + * Walk scheds with pending kicks on this cpu. scx_kick_cpu() adds to + * the list under local_irq_save() and only this irq_work consumes it. + * A plain list without locking is sufficient. + */ + list_for_each_entry_safe(pcpu, tmp, &this_scx->sched_pcpus_to_kick, to_kick_node) { + list_del_init(&pcpu->to_kick_node); + + for_each_cpu(cpu, pcpu->cpus_to_kick) { + should_wait |= kick_one_cpu(cpu, pcpu, this_rq, ksyncs); + cpumask_clear_cpu(cpu, pcpu->cpus_to_kick); + cpumask_clear_cpu(cpu, pcpu->cpus_to_kick_if_idle); + } - for_each_cpu(cpu, this_scx->cpus_to_kick_if_idle) { - kick_one_cpu_if_idle(cpu, this_rq); - cpumask_clear_cpu(cpu, this_scx->cpus_to_kick_if_idle); + for_each_cpu(cpu, pcpu->cpus_to_kick_if_idle) { + kick_one_cpu_if_idle(cpu, this_rq); + cpumask_clear_cpu(cpu, pcpu->cpus_to_kick_if_idle); + } } /* @@ -7707,11 +7749,8 @@ void __init init_sched_ext_class(void) INIT_LIST_HEAD(&rq->scx.runnable_list); INIT_LIST_HEAD(&rq->scx.ddsp_deferred_locals); - BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_kick, GFP_KERNEL, n)); - BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_kick_if_idle, GFP_KERNEL, n)); - BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_preempt, GFP_KERNEL, n)); - BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_wait, GFP_KERNEL, n)); BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_sync, GFP_KERNEL, n)); + INIT_LIST_HEAD(&rq->scx.sched_pcpus_to_kick); raw_spin_lock_init(&rq->scx.deferred_reenq_lock); INIT_LIST_HEAD(&rq->scx.deferred_reenq_locals); INIT_LIST_HEAD(&rq->scx.deferred_reenq_users); @@ -8493,12 +8532,27 @@ __bpf_kfunc bool scx_bpf_task_set_dsq_vtime(struct task_struct *p, u64 vtime, void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags) { + struct scx_sched_pcpu *pcpu; struct rq *this_rq; unsigned long irq_flags; + /* + * The per-cpu kick list is guarded only by local_irq_save(), which does + * not mask NMIs, so kicking from NMI could corrupt it and is unsupported. + */ + if (unlikely(in_nmi())) { + if (!sch->warned_nmi_kick) { + sch->warned_nmi_kick = true; + pr_warn("sched_ext: %s: scx_bpf_kick_cpu() from NMI ignored\n", + sch->ops.name); + } + return; + } + local_irq_save(irq_flags); this_rq = this_rq(); + pcpu = this_cpu_ptr(sch->pcpu); /* * While bypassing for PM ops, IRQ handling may not be online which can @@ -8512,6 +8566,9 @@ void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags) * Actual kicking is bounced to kick_cpus_irq_workfn() to avoid nesting * rq locks. We can probably be smarter and avoid bouncing if called * from ops which don't hold a rq lock. + * + * The kick masks are owned by @sch->pcpu, so that a preempt kick can be + * attributed to @sch. */ if (flags & SCX_KICK_IDLE) { struct rq *target_rq = cpu_rq(cpu); @@ -8526,16 +8583,18 @@ void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags) } raw_spin_rq_unlock(target_rq); } - cpumask_set_cpu(cpu, this_rq->scx.cpus_to_kick_if_idle); + cpumask_set_cpu(cpu, pcpu->cpus_to_kick_if_idle); } else { - cpumask_set_cpu(cpu, this_rq->scx.cpus_to_kick); + cpumask_set_cpu(cpu, pcpu->cpus_to_kick); if (flags & SCX_KICK_PREEMPT) - cpumask_set_cpu(cpu, this_rq->scx.cpus_to_preempt); + cpumask_set_cpu(cpu, pcpu->cpus_to_preempt); if (flags & SCX_KICK_WAIT) - cpumask_set_cpu(cpu, this_rq->scx.cpus_to_wait); + cpumask_set_cpu(cpu, pcpu->cpus_to_wait); } + if (list_empty(&pcpu->to_kick_node)) + list_add_tail(&pcpu->to_kick_node, &this_rq->scx.sched_pcpus_to_kick); irq_work_queue(&this_rq->scx.kick_cpus_irq_work); out: local_irq_restore(irq_flags); diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 8aabd92fae3b..2777e50ef6f6 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1126,6 +1126,19 @@ struct scx_sched_pcpu { struct scx_sched *sch; u64 flags; /* protected by rq lock */ + /* + * Kick state owned by this cpu for this sched. scx_kick_cpu() records + * targets here and links @to_kick_node onto the cpu's + * rq->scx.sched_pcpus_to_kick. The cpu's single kick irq_work walks + * that list and kicks each sched's targets on its behalf. Per-sched so + * a kick stays attributed to its scheduler. + */ + cpumask_var_t cpus_to_kick; + cpumask_var_t cpus_to_kick_if_idle; + cpumask_var_t cpus_to_preempt; + cpumask_var_t cpus_to_wait; + struct list_head to_kick_node; + /* * The event counters are in a per-CPU variable to minimize the * accounting overhead. A system-wide view on the event counter is @@ -1215,6 +1228,7 @@ struct scx_sched { */ bool warned_zero_slice:1; bool warned_unassoc_progs:1; + bool warned_nmi_kick:1; struct list_head all; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 80b72d934ff3..9e41a23ef49e 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -806,14 +806,12 @@ struct scx_rq { u32 flags; u32 nr_immed; /* ENQ_IMMED tasks on local_dsq */ u64 clock; /* current per-rq clock -- see scx_bpf_now() */ - cpumask_var_t cpus_to_kick; - cpumask_var_t cpus_to_kick_if_idle; - cpumask_var_t cpus_to_preempt; - cpumask_var_t cpus_to_wait; cpumask_var_t cpus_to_sync; bool kick_sync_pending; unsigned long kick_sync; + struct list_head sched_pcpus_to_kick; /* see kick_cpus_irq_workfn() */ + struct task_struct *sub_dispatch_prev; raw_spinlock_t deferred_reenq_lock; -- cgit v1.2.3 From 874fdc0e86e4d32539026c8e9b3999c0523c96d4 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 13 Jul 2026 22:18:42 -1000 Subject: sched_ext: Add ops.init_cids() to finalize the cid layout before init A cid-form scheduler that calls scx_bpf_cid_override() to install a custom cid layout can only do so from ops.init(). Enable-path setup that depends on the cid layout thus has to run after ops.init(), and ops.init() itself can't use anything derived from the final layout, which turned out to be too restrictive. Add an ops.init_cids() callback dedicated to finalizing the cid layout. It runs before the rest of the enable-path setup, so the final layout is in effect for everything that follows including ops.init(), which now runs after the arena pool and cmask scratch allocations. scx_bpf_cid_override() is restricted to ops.init_cids() at load time. It sits in a kfunc set gated by SCX_KF_ALLOW_INIT_CIDS, a flag set only on the init_cids op, so the verifier rejects a call from any other context. The runtime root-only check is dropped as ops.init_cids() only runs during root enable. The qmap demo moves its override into a dedicated qmap_init_cids() and, while at it, introduces an enum for the cid override modes instead of hard-coded integers. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/cid.c | 25 ++++++++++-------------- kernel/sched/ext/cid.h | 2 +- kernel/sched/ext/ext.c | 41 +++++++++++++++++++++++++++++---------- kernel/sched/ext/internal.h | 16 ++++++++++++--- tools/sched_ext/scx_qmap.bpf.c | 44 ++++++++++++++++++++++++++---------------- tools/sched_ext/scx_qmap.c | 12 ++++++------ tools/sched_ext/scx_qmap.h | 8 ++++++++ 7 files changed, 96 insertions(+), 52 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/cid.c b/kernel/sched/ext/cid.c index f31113b080e9..b0ec4d76f855 100644 --- a/kernel/sched/ext/cid.c +++ b/kernel/sched/ext/cid.c @@ -13,10 +13,10 @@ /* * cid tables. * - * Pointers are published once on first enable and never revoked. The default - * mapping is populated before ops.init() runs; scx_bpf_cid_override() commits - * before it returns. As long as the BPF scheduler only uses the tables from - * those points onward, it sees a consistent view. + * Pointers are allocated on first enable and never freed. During root enable, + * the default mapping is populated and then ops.init_cids() is called which can + * use scx_bpf_cid_override() to change the mapping. The mapping stays stable + * until the root is disabled. */ s16 *scx_cid_to_cpu_tbl; s16 *scx_cpu_to_cid_tbl; @@ -282,7 +282,7 @@ __bpf_kfunc_start_defs(); * @cpu_to_cid__sz: must be nr_cpu_ids * sizeof(s32) bytes * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * - * May only be called from ops.init() of the root scheduler. Replace the + * May only be called from ops.init_cids() of the root scheduler. Replace the * topology-probed cid mapping with the caller-provided one. Each possible cpu * must map to a unique cid in [0, num_possible_cpus()). Topo info is cleared. * On invalid input, trigger scx_error() to abort the scheduler. @@ -309,11 +309,6 @@ __bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid, u32 cpu_to_cid__sz, return; } - if (scx_parent(sch)) { - scx_error(sch, "scx_bpf_cid_override() only allowed from root sched"); - return; - } - if (cpu_to_cid__sz != nr_cpu_ids * sizeof(s32)) { scx_error(sch, "scx_bpf_cid_override: expected %zu bytes, got %u", nr_cpu_ids * sizeof(s32), cpu_to_cid__sz); @@ -645,13 +640,13 @@ __bpf_kfunc void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out__uninit, __bpf_kfunc_end_defs(); -BTF_KFUNCS_START(scx_kfunc_ids_init) +BTF_KFUNCS_START(scx_kfunc_ids_init_cids) BTF_ID_FLAGS(func, scx_bpf_cid_override, KF_IMPLICIT_ARGS | KF_SLEEPABLE) -BTF_KFUNCS_END(scx_kfunc_ids_init) +BTF_KFUNCS_END(scx_kfunc_ids_init_cids) -static const struct btf_kfunc_id_set scx_kfunc_set_init = { +static const struct btf_kfunc_id_set scx_kfunc_set_init_cids = { .owner = THIS_MODULE, - .set = &scx_kfunc_ids_init, + .set = &scx_kfunc_ids_init_cids, .filter = scx_kfunc_context_filter, }; @@ -668,7 +663,7 @@ static const struct btf_kfunc_id_set scx_kfunc_set_cid = { int scx_cid_kfunc_init(void) { - return register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_init) ?: + return register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_init_cids) ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_cid) ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &scx_kfunc_set_cid) ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &scx_kfunc_set_cid); diff --git a/kernel/sched/ext/cid.h b/kernel/sched/ext/cid.h index 54b10df32fd5..cd0d4b9f1088 100644 --- a/kernel/sched/ext/cid.h +++ b/kernel/sched/ext/cid.h @@ -51,7 +51,7 @@ struct scx_sched; extern s16 *scx_cid_to_cpu_tbl; extern s16 *scx_cpu_to_cid_tbl; extern struct scx_cid_topo *scx_cid_topo; -extern struct btf_id_set8 scx_kfunc_ids_init; +extern struct btf_id_set8 scx_kfunc_ids_init_cids; void scx_cmask_clear(struct scx_cmask *m); void scx_cmask_fill(struct scx_cmask *m); diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index ab0b6f15e318..434bec61642d 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -6725,15 +6725,19 @@ static void scx_root_enable_workfn(struct kthread_work *work) scx_idle_enable(ops); - if (sch->ops.init) { - ret = SCX_CALL_OP_RET(sch, init, NULL); + /* + * A cid-form scheduler finalizes its cid layout in ops.init_cids(), + * which may call scx_bpf_cid_override(). Run it before ops.init() so + * the final layout is in effect. + */ + if (sch->is_cid_type && sch->ops_cid.init_cids) { + ret = SCX_CALL_OP_RET(sch, init_cids, NULL); if (ret) { - ret = scx_ops_sanitize_err(sch, "init", ret); + ret = scx_ops_sanitize_err(sch, "init_cids", ret); cpus_read_unlock(); - scx_error(sch, "ops.init() failed (%d)", ret); + scx_error(sch, "ops.init_cids() failed (%d)", ret); goto err_disable; } - sch->exit_info->flags |= SCX_EFLAG_INITIALIZED; } ret = scx_arena_pool_init(sch); @@ -6748,6 +6752,17 @@ static void scx_root_enable_workfn(struct kthread_work *work) goto err_disable; } + if (sch->ops.init) { + ret = SCX_CALL_OP_RET(sch, init, NULL); + if (ret) { + ret = scx_ops_sanitize_err(sch, "init", ret); + cpus_read_unlock(); + scx_error(sch, "ops.init() failed (%d)", ret); + goto err_disable; + } + sch->exit_info->flags |= SCX_EFLAG_INITIALIZED; + } + for (i = SCX_OPI_CPU_HOTPLUG_BEGIN; i < SCX_OPI_CPU_HOTPLUG_END; i++) if (((void (**)(void))ops)[i]) set_bit(i, sch->has_op); @@ -7184,6 +7199,7 @@ static int bpf_scx_check_member(const struct btf_type *t, #endif case offsetof(struct sched_ext_ops, cpu_online): case offsetof(struct sched_ext_ops, cpu_offline): + case offsetof(struct sched_ext_ops, init_cids): case offsetof(struct sched_ext_ops, init): case offsetof(struct sched_ext_ops, exit): case offsetof(struct sched_ext_ops, sub_attach): @@ -7342,6 +7358,7 @@ static s32 sched_ext_ops__sub_attach(struct scx_sub_attach_args *args) { return static void sched_ext_ops__sub_detach(struct scx_sub_detach_args *args) {} static void sched_ext_ops__cpu_online(s32 cpu) {} static void sched_ext_ops__cpu_offline(s32 cpu) {} +static s32 sched_ext_ops__init_cids(void) { return -EINVAL; } static s32 sched_ext_ops__init(void) { return -EINVAL; } static void sched_ext_ops__exit(struct scx_exit_info *info) {} static void sched_ext_ops__dump(struct scx_dump_ctx *ctx) {} @@ -7383,6 +7400,7 @@ static struct sched_ext_ops __bpf_ops_sched_ext_ops = { .sub_detach = sched_ext_ops__sub_detach, .cpu_online = sched_ext_ops__cpu_online, .cpu_offline = sched_ext_ops__cpu_offline, + .init_cids = sched_ext_ops__init_cids, .init = sched_ext_ops__init, .exit = sched_ext_ops__exit, .dump = sched_ext_ops__dump, @@ -7445,6 +7463,7 @@ static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = { .sub_detach = sched_ext_ops__sub_detach, .cid_online = sched_ext_ops__cpu_online, .cid_offline = sched_ext_ops__cpu_offline, + .init_cids = sched_ext_ops__init_cids, .init = sched_ext_ops__init, .exit = sched_ext_ops__exit, .dump = sched_ext_ops__dump, @@ -9703,7 +9722,7 @@ BTF_KFUNCS_END(scx_kfunc_ids_cpu_only) */ enum scx_kf_allow_flags { SCX_KF_ALLOW_UNLOCKED = 1 << 0, - SCX_KF_ALLOW_INIT = 1 << 1, + SCX_KF_ALLOW_INIT_CIDS = 1 << 1, SCX_KF_ALLOW_CPU_RELEASE = 1 << 2, SCX_KF_ALLOW_DISPATCH = 1 << 3, SCX_KF_ALLOW_ENQUEUE = 1 << 4, @@ -9735,7 +9754,8 @@ static const u32 scx_kf_allow_flags[] = { [SCX_OP_IDX(sub_detach)] = SCX_KF_ALLOW_UNLOCKED, [SCX_OP_IDX(cpu_online)] = SCX_KF_ALLOW_UNLOCKED, [SCX_OP_IDX(cpu_offline)] = SCX_KF_ALLOW_UNLOCKED, - [SCX_OP_IDX(init)] = SCX_KF_ALLOW_UNLOCKED | SCX_KF_ALLOW_INIT, + [SCX_OP_IDX(init_cids)] = SCX_KF_ALLOW_UNLOCKED | SCX_KF_ALLOW_INIT_CIDS, + [SCX_OP_IDX(init)] = SCX_KF_ALLOW_UNLOCKED, [SCX_OP_IDX(exit)] = SCX_KF_ALLOW_UNLOCKED, }; @@ -9750,7 +9770,7 @@ static const u32 scx_kf_allow_flags[] = { int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id) { bool in_unlocked = btf_id_set8_contains(&scx_kfunc_ids_unlocked, kfunc_id); - bool in_init = btf_id_set8_contains(&scx_kfunc_ids_init, kfunc_id); + bool in_init_cids = btf_id_set8_contains(&scx_kfunc_ids_init_cids, kfunc_id); bool in_select_cpu = btf_id_set8_contains(&scx_kfunc_ids_select_cpu, kfunc_id); bool in_enqueue = btf_id_set8_contains(&scx_kfunc_ids_enqueue_dispatch, kfunc_id); bool in_dispatch = btf_id_set8_contains(&scx_kfunc_ids_dispatch, kfunc_id); @@ -9761,7 +9781,7 @@ int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id) u32 moff, flags; /* Not an SCX kfunc - allow. */ - if (!(in_unlocked || in_init || in_select_cpu || in_enqueue || in_dispatch || + if (!(in_unlocked || in_init_cids || in_select_cpu || in_enqueue || in_dispatch || in_cpu_release || in_idle || in_any)) return 0; @@ -9813,7 +9833,7 @@ int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id) if ((flags & SCX_KF_ALLOW_UNLOCKED) && in_unlocked) return 0; - if ((flags & SCX_KF_ALLOW_INIT) && in_init) + if ((flags & SCX_KF_ALLOW_INIT_CIDS) && in_init_cids) return 0; if ((flags & SCX_KF_ALLOW_CPU_RELEASE) && in_cpu_release) return 0; @@ -9869,6 +9889,7 @@ static int __init scx_init(void) CID_OFFSET_MATCH(dump_task, dump_task); CID_OFFSET_MATCH(sub_attach, sub_attach); CID_OFFSET_MATCH(sub_detach, sub_detach); + CID_OFFSET_MATCH(init_cids, init_cids); CID_OFFSET_MATCH(init, init); CID_OFFSET_MATCH(exit, exit); #ifdef CONFIG_EXT_GROUP_SCHED diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 2777e50ef6f6..902244dc27ff 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -780,9 +780,18 @@ struct sched_ext_ops { void (*cpu_offline)(s32 cpu); /* - * All CPU hotplug ops must come before ops.init(). + * All CPU hotplug ops must come before ops.init_cids(). */ + /** + * @init_cids: Finalize the cid layout (cid-form only) + * + * Runs after the default cid layout is built, before ops.init(). A + * cid-form scheduler may call scx_bpf_cid_override() here for a custom + * layout. Ignored for cpu-form schedulers. + */ + s32 (*init_cids)(void); + /** * @init: Initialize the BPF scheduler */ @@ -958,6 +967,7 @@ struct sched_ext_ops_cid { void (*sub_detach)(struct scx_sub_detach_args *args); void (*cid_online)(s32 cid); void (*cid_offline)(s32 cid); + s32 (*init_cids)(void); s32 (*init)(void); void (*exit)(struct scx_exit_info *info); @@ -982,8 +992,8 @@ enum scx_opi { SCX_OPI_NORMAL_BEGIN = 0, SCX_OPI_NORMAL_END = SCX_OP_IDX(cpu_online), SCX_OPI_CPU_HOTPLUG_BEGIN = SCX_OP_IDX(cpu_online), - SCX_OPI_CPU_HOTPLUG_END = SCX_OP_IDX(init), - SCX_OPI_END = SCX_OP_IDX(init), + SCX_OPI_CPU_HOTPLUG_END = SCX_OP_IDX(init_cids), + SCX_OPI_END = SCX_OP_IDX(init_cids), }; /* diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c index fd9a82a67627..2df7c53992dc 100644 --- a/tools/sched_ext/scx_qmap.bpf.c +++ b/tools/sched_ext/scx_qmap.bpf.c @@ -54,13 +54,9 @@ const volatile u32 max_tasks; /* * Optional cid-override test harness. When cid_override_mode is non-zero, - * qmap_init() calls scx_bpf_cid_override() with the caller-supplied - * cpu_to_cid array to exercise the kfunc's acceptance and error paths. - * - * 0 = disabled - * 1 = valid reverse mapping - * 2 = invalid: duplicate cid assignment - * 3 = invalid: out-of-range cid + * qmap_init_cids() calls scx_bpf_cid_override() with the caller-supplied + * cpu_to_cid array to exercise the kfunc's acceptance and error paths. See enum + * qmap_cid_override for the modes. */ const volatile u32 cid_override_mode; /* @@ -1067,6 +1063,29 @@ static int lowpri_timerfn(void *map, int *key, struct bpf_timer *timer) return 0; } +/* + * Custom cid layout for the cid-override test. On invalid input the kfunc + * scx_error()s and aborts the scheduler. + */ +s32 BPF_STRUCT_OPS_SLEEPABLE(qmap_init_cids) +{ + u32 nr_cpu_ids = scx_bpf_nr_cpu_ids(); + + if (!cid_override_mode) + return 0; + + /* bound the count so the verifier accepts cpu_to_cid's mem/len pair */ + if (nr_cpu_ids > SCX_QMAP_MAX_CPUS) { + scx_bpf_error("nr_cpu_ids=%u exceeds SCX_QMAP_MAX_CPUS=%d", + nr_cpu_ids, SCX_QMAP_MAX_CPUS); + return -EINVAL; + } + + scx_bpf_cid_override((const s32 *)cid_override_cpu_to_cid, + nr_cpu_ids * sizeof(s32)); + return 0; +} + s32 BPF_STRUCT_OPS_SLEEPABLE(qmap_init) { u8 __arena *slab; @@ -1089,16 +1108,6 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(qmap_init) return -EINVAL; } - /* - * cid-override test hook. Must run before anything that reads the - * cid space (scx_bpf_nr_cids, cmask_init, etc.). On invalid input, - * the kfunc calls scx_error() which aborts the scheduler. - */ - if (cid_override_mode) { - scx_bpf_cid_override((const s32 *)cid_override_cpu_to_cid, - nr_cpu_ids * sizeof(s32)); - } - /* * Allocate the task_ctx slab in arena and thread the entire slab onto * the free list. max_tasks is set by userspace before load. Each entry @@ -1235,6 +1244,7 @@ SCX_OPS_CID_DEFINE(qmap_ops, .cgroup_set_bandwidth = (void *)qmap_cgroup_set_bandwidth, .sub_attach = (void *)qmap_sub_attach, .sub_detach = (void *)qmap_sub_detach, + .init_cids = (void *)qmap_init_cids, .init = (void *)qmap_init, .exit = (void *)qmap_exit, .timeout_ms = 5000U, diff --git a/tools/sched_ext/scx_qmap.c b/tools/sched_ext/scx_qmap.c index f1eaebcab5dc..c0b5cab579d6 100644 --- a/tools/sched_ext/scx_qmap.c +++ b/tools/sched_ext/scx_qmap.c @@ -157,11 +157,11 @@ restart: u32 mode, i; if (!strcmp(optarg, "shuffle")) - mode = 1; + mode = QMAP_CID_OVR_SHUFFLE; else if (!strcmp(optarg, "bad-dup")) - mode = 2; + mode = QMAP_CID_OVR_BAD_DUP; else if (!strcmp(optarg, "bad-range")) - mode = 3; + mode = QMAP_CID_OVR_BAD_RANGE; else { fprintf(stderr, "unknown cid-override mode '%s'\n", optarg); return 1; @@ -170,14 +170,14 @@ restart: /* shuffle: reversed cpu_to_cid, bad-dup: dup cid 0, bad-range: identity */ for (i = 0; i < nr_cpus; i++) { - if (mode == 1) + if (mode == QMAP_CID_OVR_SHUFFLE) skel->bss->cid_override_cpu_to_cid[i] = nr_cpus - 1 - i; else skel->bss->cid_override_cpu_to_cid[i] = i; } - if (mode == 2 && nr_cpus >= 2) + if (mode == QMAP_CID_OVR_BAD_DUP && nr_cpus >= 2) skel->bss->cid_override_cpu_to_cid[1] = 0; - if (mode == 3) + if (mode == QMAP_CID_OVR_BAD_RANGE) skel->bss->cid_override_cpu_to_cid[0] = (s32)nr_cpus; break; } diff --git a/tools/sched_ext/scx_qmap.h b/tools/sched_ext/scx_qmap.h index 808237540f5c..3bcc3579839d 100644 --- a/tools/sched_ext/scx_qmap.h +++ b/tools/sched_ext/scx_qmap.h @@ -27,6 +27,14 @@ */ #define SCX_QMAP_MAX_CPUS 1024 +/* -C cid-override test modes. Selects cid_override_mode in scx_qmap.bpf.c. */ +enum qmap_cid_override { + QMAP_CID_OVR_OFF = 0, /* disabled */ + QMAP_CID_OVR_SHUFFLE = 1, /* valid reversed cpu->cid mapping */ + QMAP_CID_OVR_BAD_DUP = 2, /* invalid: duplicate cid assignment */ + QMAP_CID_OVR_BAD_RANGE = 3, /* invalid: out-of-range cid */ +}; + struct cpu_ctx { u64 dsp_idx; /* dispatch index */ u64 dsp_cnt; /* remaining count */ -- cgit v1.2.3 From e2c841b91bf960f5dd57aa9dd49b14a1d30f1a56 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 13 Jul 2026 22:18:42 -1000 Subject: sched_ext: Add CID sharding Sub-sched operations need a scalable locking / work domain smaller than the whole cid space. Carve the cid space into topology-respecting shards: each shard is a contiguous cid range that stays within one LLC, and LLCs larger than the per-shard cap (default 24 cids, configurable via ops.cid_shard_size) split into enough shards to fit. A hard cap of SCX_CID_SHARD_MAX_CPUS prevents pathological sizes under custom configurations. No-topo cids pack into their own shards so every cid has a shard assignment. Also build scx_cid_shard_ranges[] for O(1) shard-to-cid-range lookup and scx_shard_node[] so callers can size or place work by NUMA without walking cids. Auto-built shards inherit their LLC's node. No-topo shards carry NUMA_NO_NODE. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/cid.c | 132 +++++++++++++++++++++++++++++++++++++++++--- kernel/sched/ext/cid.h | 4 ++ kernel/sched/ext/ext.c | 4 ++ kernel/sched/ext/internal.h | 13 +++++ kernel/sched/ext/types.h | 35 ++++++++++-- 5 files changed, 176 insertions(+), 12 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/cid.c b/kernel/sched/ext/cid.c index b0ec4d76f855..f3c54f4223b4 100644 --- a/kernel/sched/ext/cid.c +++ b/kernel/sched/ext/cid.c @@ -18,13 +18,17 @@ * use scx_bpf_cid_override() to change the mapping. The mapping stays stable * until the root is disabled. */ +u32 scx_nr_cid_shards; s16 *scx_cid_to_cpu_tbl; s16 *scx_cpu_to_cid_tbl; +s32 *scx_cid_to_shard; +s32 *scx_shard_node; +struct scx_cid_shard *scx_cid_shard_ranges; struct scx_cid_topo *scx_cid_topo; #define SCX_CID_TOPO_NEG (struct scx_cid_topo) { \ .core_cid = -1, .core_idx = -1, .llc_cid = -1, .llc_idx = -1, \ - .node_cid = -1, .node_idx = -1, \ + .node_cid = -1, .node_idx = -1, .shard_cid = -1, .shard_idx = -1, \ } /* @@ -43,11 +47,40 @@ static const struct cpumask *cpu_llc_mask(int cpu, struct cpumask *fallbacks) return &ci->info_list[ci->num_leaves - 1].shared_cpu_map; } +/* + * Compute per-LLC shard layout. Each shard holds at most @shard_size cids, and + * in any case no more than SCX_CID_SHARD_MAX_CPUS. Cores are spread as evenly + * as possible across shards so cpu count is balanced: the first *@nr_large_p + * shards get (*@cores_per_shard_p + 1) cores, the rest get *@cores_per_shard_p. + */ +static void calc_shard_layout(const struct cpumask *llc_cpus, u32 shard_size, + u32 *cores_per_shard_p, u32 *nr_large_p) +{ + u32 nr_cores = 0, nr_cpus = 0, nr_shards; + int cpu; + + for_each_cpu(cpu, llc_cpus) { + nr_cpus++; + if (cpumask_first(topology_sibling_cpumask(cpu)) == cpu) + nr_cores++; + } + + nr_shards = max_t(u32, 1, DIV_ROUND_UP(nr_cpus, shard_size)); + nr_shards = max_t(u32, nr_shards, + DIV_ROUND_UP(nr_cpus, SCX_CID_SHARD_MAX_CPUS)); + + *cores_per_shard_p = nr_cores / nr_shards; + *nr_large_p = nr_cores % nr_shards; +} + /* Allocate the cid tables once on first enable; never freed. */ static s32 scx_cid_arrays_alloc(void) { u32 npossible = num_possible_cpus(); s16 *cid_to_cpu, *cpu_to_cid; + s32 *cid_to_shard; + s32 *shard_node; + struct scx_cid_shard *cid_shard_ranges; struct scx_cid_topo *cid_topo; if (scx_cid_to_cpu_tbl) @@ -55,17 +88,27 @@ static s32 scx_cid_arrays_alloc(void) cid_to_cpu = kzalloc_objs(*scx_cid_to_cpu_tbl, npossible, GFP_KERNEL); cpu_to_cid = kzalloc_objs(*scx_cpu_to_cid_tbl, nr_cpu_ids, GFP_KERNEL); + cid_to_shard = kzalloc_objs(*scx_cid_to_shard, npossible, GFP_KERNEL); + shard_node = kmalloc_objs(*scx_shard_node, npossible, GFP_KERNEL); + cid_shard_ranges = kzalloc_objs(*scx_cid_shard_ranges, npossible, GFP_KERNEL); cid_topo = kmalloc_objs(*scx_cid_topo, npossible, GFP_KERNEL); - if (!cid_to_cpu || !cpu_to_cid || !cid_topo) { + if (!cid_to_cpu || !cpu_to_cid || !cid_to_shard || !shard_node || + !cid_shard_ranges || !cid_topo) { kfree(cid_to_cpu); kfree(cpu_to_cid); + kfree(cid_to_shard); + kfree(shard_node); + kfree(cid_shard_ranges); kfree(cid_topo); return -ENOMEM; } WRITE_ONCE(scx_cid_to_cpu_tbl, cid_to_cpu); WRITE_ONCE(scx_cpu_to_cid_tbl, cpu_to_cid); + WRITE_ONCE(scx_cid_to_shard, cid_to_shard); + WRITE_ONCE(scx_shard_node, shard_node); + WRITE_ONCE(scx_cid_shard_ranges, cid_shard_ranges); WRITE_ONCE(scx_cid_topo, cid_topo); return 0; } @@ -90,17 +133,29 @@ s32 scx_cid_init(struct scx_sched *sch) cpumask_var_t online_no_topo __free(free_cpumask_var) = CPUMASK_VAR_NULL; u32 next_cid = 0; s32 next_node_idx = 0, next_llc_idx = 0, next_core_idx = 0; - s32 cpu, ret; + s32 next_shard_idx = 0; + u32 shard_size, max_cids; + u32 notopo_in_shard; + s32 notopo_shard_cid, notopo_shard_idx; + s32 cpu, cid, si, ret; /* CMASK_MAX_WORDS in cid.bpf.h covers NR_CPUS up to 8192 */ BUILD_BUG_ON(NR_CPUS > 8192); lockdep_assert_cpus_held(); + shard_size = sch->ops.cid_shard_size ?: SCX_CID_SHARD_SIZE_DFL; + max_cids = min_t(u32, shard_size, SCX_CID_SHARD_MAX_CPUS); + ret = scx_cid_arrays_alloc(); if (ret) return ret; + /* clear shard ranges and reset shard_node for repopulate */ + memset(scx_cid_shard_ranges, 0, num_possible_cpus() * sizeof(*scx_cid_shard_ranges)); + for (si = 0; si < num_possible_cpus(); si++) + scx_shard_node[si] = NUMA_NO_NODE; + if (!zalloc_cpumask_var(&to_walk, GFP_KERNEL) || !zalloc_cpumask_var(&node_scratch, GFP_KERNEL) || !zalloc_cpumask_var(&llc_scratch, GFP_KERNEL) || @@ -142,29 +197,60 @@ s32 scx_cid_init(struct scx_sched *sch) const struct cpumask *llc_mask = cpu_llc_mask(ncpu, llc_fallback); s32 llc_cid = next_cid; s32 llc_idx = next_llc_idx++; + u32 cores_per_shard, nr_large; + u32 shard_local = 0, cores_in_shard = 0, cids_in_shard = 0; + s32 shard_cid, shard_idx; /* llc_scratch = node_scratch & this llc */ cpumask_and(llc_scratch, node_scratch, llc_mask); if (WARN_ON_ONCE(!cpumask_test_cpu(ncpu, llc_scratch))) return -EINVAL; + calc_shard_layout(llc_scratch, shard_size, &cores_per_shard, &nr_large); + shard_cid = next_cid; + shard_idx = next_shard_idx++; + scx_shard_node[shard_idx] = nid; + while (!cpumask_empty(llc_scratch)) { s32 lcpu = cpumask_first(llc_scratch); const struct cpumask *sib = topology_sibling_cpumask(lcpu); s32 core_cid = next_cid; s32 core_idx = next_core_idx++; s32 ccpu; + u32 max_cores, cids_in_core; /* core_scratch = llc_scratch & this core */ cpumask_and(core_scratch, llc_scratch, sib); if (WARN_ON_ONCE(!cpumask_test_cpu(lcpu, core_scratch))) return -EINVAL; + /* + * Advance to a new shard when either core or + * cid count reaches max. The latter bounds + * shard sizes under uneven SMT. Never start an + * empty shard. + */ + cids_in_core = cpumask_weight(core_scratch); + max_cores = cores_per_shard + (shard_local < nr_large ? 1 : 0); + if (cores_in_shard && + (cores_in_shard >= max_cores || + cids_in_shard + cids_in_core > max_cids)) { + shard_local++; + cores_in_shard = 0; + cids_in_shard = 0; + shard_cid = next_cid; + shard_idx = next_shard_idx++; + scx_shard_node[shard_idx] = nid; + } + cores_in_shard++; + cids_in_shard += cids_in_core; + for_each_cpu(ccpu, core_scratch) { s32 cid = next_cid++; scx_cid_to_cpu_tbl[cid] = ccpu; scx_cpu_to_cid_tbl[ccpu] = cid; + scx_cid_to_shard[cid] = shard_idx; scx_cid_topo[cid] = (struct scx_cid_topo){ .core_cid = core_cid, .core_idx = core_idx, @@ -172,6 +258,8 @@ s32 scx_cid_init(struct scx_sched *sch) .llc_idx = llc_idx, .node_cid = node_cid, .node_idx = node_idx, + .shard_cid = shard_cid, + .shard_idx = shard_idx, }; cpumask_clear_cpu(ccpu, llc_scratch); @@ -184,12 +272,17 @@ s32 scx_cid_init(struct scx_sched *sch) /* * No-topo section: any possible cpu without a cid - normally just the - * not-online ones. Collect any currently-online cpus that land here in - * @online_no_topo so we can warn about them at the end. + * not-online ones. Pack into shards of up to min(@shard_size, + * SCX_CID_SHARD_MAX_CPUS) cids so that every cid has a valid shard + * assignment and the hard cap holds even with a large @shard_size. + * Collect any currently-online cpus that land here in @online_no_topo + * so we can warn about them at the end. */ - for_each_cpu(cpu, cpu_possible_mask) { - s32 cid; + notopo_in_shard = min_t(u32, shard_size, SCX_CID_SHARD_MAX_CPUS); + notopo_shard_cid = -1; + notopo_shard_idx = -1; + for_each_cpu(cpu, cpu_possible_mask) { if (__scx_cpu_to_cid(cpu) != -1) continue; if (cpu_online(cpu)) @@ -198,7 +291,18 @@ s32 scx_cid_init(struct scx_sched *sch) cid = next_cid++; scx_cid_to_cpu_tbl[cid] = cpu; scx_cpu_to_cid_tbl[cpu] = cid; + + if (notopo_in_shard >= min_t(u32, shard_size, SCX_CID_SHARD_MAX_CPUS)) { + notopo_shard_cid = cid; + notopo_shard_idx = next_shard_idx++; + notopo_in_shard = 0; + } + notopo_in_shard++; + + scx_cid_to_shard[cid] = notopo_shard_idx; scx_cid_topo[cid] = SCX_CID_TOPO_NEG; + scx_cid_topo[cid].shard_cid = notopo_shard_cid; + scx_cid_topo[cid].shard_idx = notopo_shard_idx; } if (!cpumask_empty(llc_fallback)) @@ -208,6 +312,20 @@ s32 scx_cid_init(struct scx_sched *sch) pr_warn("scx_cid: online cpus with no usable topology: %*pbl\n", cpumask_pr_args(online_no_topo)); + /* + * Fill cid_shard_ranges[] from cid_to_shard[]. Shards are contiguous + * cid ranges by construction: base_cid is the first cid landing in a + * shard, nr_cids is the count. + */ + for (cid = 0; cid < next_cid; cid++) { + s32 sidx = scx_cid_to_shard[cid]; + + if (scx_cid_shard_ranges[sidx].nr_cids == 0) + scx_cid_shard_ranges[sidx].base_cid = cid; + scx_cid_shard_ranges[sidx].nr_cids++; + } + + scx_nr_cid_shards = next_shard_idx; return 0; } diff --git a/kernel/sched/ext/cid.h b/kernel/sched/ext/cid.h index cd0d4b9f1088..cdc18a7a48f5 100644 --- a/kernel/sched/ext/cid.h +++ b/kernel/sched/ext/cid.h @@ -48,8 +48,12 @@ struct scx_sched; * See the comment above the table definitions in cid.c for the * memory-ordering and visibility contract. */ +extern u32 scx_nr_cid_shards; extern s16 *scx_cid_to_cpu_tbl; extern s16 *scx_cpu_to_cid_tbl; +extern s32 *scx_cid_to_shard; +extern s32 *scx_shard_node; +extern struct scx_cid_shard *scx_cid_shard_ranges; extern struct scx_cid_topo *scx_cid_topo; extern struct btf_id_set8 scx_kfunc_ids_init_cids; diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 434bec61642d..b3e26a46f821 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -7174,6 +7174,9 @@ static int bpf_scx_init_member(const struct btf_type *t, case offsetof(struct sched_ext_ops, hotplug_seq): ops->hotplug_seq = *(u64 *)(udata + moff); return 1; + case offsetof(struct sched_ext_ops, cid_shard_size): + ops->cid_shard_size = *(u32 *)(udata + moff); + return 1; #ifdef CONFIG_EXT_SUB_SCHED case offsetof(struct sched_ext_ops, sub_cgroup_id): ops->sub_cgroup_id = *(u64 *)(udata + moff); @@ -9867,6 +9870,7 @@ static int __init scx_init(void) CID_OFFSET_MATCH(timeout_ms, timeout_ms); CID_OFFSET_MATCH(exit_dump_len, exit_dump_len); CID_OFFSET_MATCH(hotplug_seq, hotplug_seq); + CID_OFFSET_MATCH(cid_shard_size, cid_shard_size); CID_OFFSET_MATCH(sub_cgroup_id, sub_cgroup_id); /* shared callbacks: the union view requires byte-for-byte offset match */ CID_OFFSET_MATCH(enqueue, enqueue); diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 902244dc27ff..6560a0fa3efa 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -845,6 +845,18 @@ struct sched_ext_ops { */ u64 hotplug_seq; + /** + * @cid_shard_size: Target number of CIDs per shard + * + * Shards are contiguous CID ranges used as operation and locking + * domains for sub-scheduling. Each LLC is divided into ceil(nr_cpus / + * @cid_shard_size) shards, then cores are distributed across them + * evenly. If one core has more logical CPUs than @cid_shard_size, its + * shard will become larger than @cid_shard_size. Values above + * SCX_CID_SHARD_MAX_CPUS are capped. 0 means use the default (24). + */ + u32 cid_shard_size; + /** * @cgroup_id: When >1, attach the scheduler as a sub-scheduler on the * specified cgroup. @@ -977,6 +989,7 @@ struct sched_ext_ops_cid { u32 timeout_ms; u32 exit_dump_len; u64 hotplug_seq; + u32 cid_shard_size; u64 sub_cgroup_id; char name[SCX_OPS_NAME_LEN]; diff --git a/kernel/sched/ext/types.h b/kernel/sched/ext/types.h index bc74eafd43f1..b31d12931999 100644 --- a/kernel/sched/ext/types.h +++ b/kernel/sched/ext/types.h @@ -47,11 +47,16 @@ enum scx_consts { }; /* - * Per-cid topology info. For each topology level (core, LLC, node), records - * the first cid in the unit and its global index. Global indices are - * consecutive integers assigned in cid-walk order, so e.g. core_idx ranges - * over [0, nr_cores_at_init) with no gaps. No-topo cids have all fields set - * to -1. + * Per-cid topology info. For each topology level (core, LLC, node) and shard, + * records the first cid in the unit and its global index. Global indices are + * consecutive integers assigned in cid-walk order, so e.g. core_idx ranges over + * [0, nr_cores_at_init) with no gaps. No-topo cids have core/LLC/node fields + * set to -1 but always have valid shard assignments. + * + * Shards are contiguous CID ranges used as scalable locking/work domains for + * sub-scheduler operations. By default each LLC becomes one shard, split into + * smaller shards if the LLC exceeds the target size. No-topo cids are packed + * into their own max-sized shards. * * @core_cid: first cid of this cid's core (smt-sibling group) * @core_idx: global index of that core, in [0, nr_cores_at_init) @@ -59,6 +64,8 @@ enum scx_consts { * @llc_idx: global index of that LLC, in [0, nr_llcs_at_init) * @node_cid: first cid of this cid's NUMA node * @node_idx: global index of that node, in [0, nr_nodes_at_init) + * @shard_cid: first cid of this cid's shard + * @shard_idx: global index of that shard, in [0, scx_nr_cid_shards) */ struct scx_cid_topo { s32 core_cid; @@ -67,6 +74,24 @@ struct scx_cid_topo { s32 llc_idx; s32 node_cid; s32 node_idx; + s32 shard_cid; + s32 shard_idx; +}; + +enum scx_cid_consts { + SCX_CID_SHARD_SIZE_DFL = 24, + SCX_CID_SHARD_MAX_CPUS = 512, +}; + +/* + * Per-shard metadata for O(1) shard->cid-range lookup. + * + * @base_cid: first cid of the shard + * @nr_cids: number of cids in the shard + */ +struct scx_cid_shard { + s32 base_cid; + s32 nr_cids; }; /* -- cgit v1.2.3 From 30067643bce0d02cbeb512416372e93272e674f3 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 13 Jul 2026 22:18:42 -1000 Subject: sched_ext: Add shard boundaries to scx_bpf_cid_override() An overridden cid mapping invalidates the auto-generated shard layout, so the override call has to provide both. Extend scx_bpf_cid_override() with a shard_start[] array that lists the first cid of each shard (starting at 0, strictly increasing, last shard implicitly extends to num_possible_cpus()). A scheduler that wants only custom shards with the auto-generated cid mapping can read the current mapping and pass it back unchanged. Overridden shards can span NUMA nodes, so scx_shard_node[] is rebuilt by majority count: each shard is assigned to the node that owns the most cpus in it. v2: Snapshot the caller's cpu_to_cid/shard_start arrays before validating. (sashiko AI) Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/cid.c | 154 +++++++++++++++++++++++++++---- tools/sched_ext/include/scx/compat.bpf.h | 11 ++- tools/sched_ext/scx_qmap.bpf.c | 16 ++-- tools/sched_ext/scx_qmap.c | 34 ++++++- tools/sched_ext/scx_qmap.h | 1 + 5 files changed, 187 insertions(+), 29 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/cid.c b/kernel/sched/ext/cid.c index f3c54f4223b4..6a22bead4c60 100644 --- a/kernel/sched/ext/cid.c +++ b/kernel/sched/ext/cid.c @@ -392,29 +392,66 @@ void scx_cpumask_to_cmask(const struct cpumask *src, struct scx_cmask *dst) } } +/* + * Return the index of the largest entry in @counts, or NUMA_NO_NODE if all + * entries are zero. Ties resolve to the lowest index. + */ +static s32 pick_max_node(const u32 *counts, u32 n) +{ + s32 best = NUMA_NO_NODE; + u32 best_count = 0, i; + + for (i = 0; i < n; i++) { + if (counts[i] > best_count) { + best_count = counts[i]; + best = i; + } + } + return best; +} + __bpf_kfunc_start_defs(); /** - * scx_bpf_cid_override - Install an explicit cpu->cid mapping - * @cpu_to_cid: array of nr_cpu_ids s32 entries (cid for each cpu) - * @cpu_to_cid__sz: must be nr_cpu_ids * sizeof(s32) bytes + * scx_bpf_cid_override - Install an explicit cpu->cid mapping with shard info + * @cpu_to_cid_src: array of nr_cpu_ids s32 entries (cid for each cpu) + * @cpu_to_cid_src__sz: must be nr_cpu_ids * sizeof(s32) bytes + * @shard_start_src: array of first-cid-of-each-shard, strictly increasing from 0 + * @shard_start_src__sz: nr_shards * sizeof(s32) bytes * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * * May only be called from ops.init_cids() of the root scheduler. Replace the - * topology-probed cid mapping with the caller-provided one. Each possible cpu - * must map to a unique cid in [0, num_possible_cpus()). Topo info is cleared. - * On invalid input, trigger scx_error() to abort the scheduler. + * topology-probed cid mapping and shard layout with caller-provided ones. Each + * possible cpu must map to a unique cid in [0, num_possible_cpus()). The shard + * starts must be strictly increasing with the first entry 0 and all values < + * num_possible_cpus(). The last shard extends to num_possible_cpus() and no + * shard may span more than SCX_CID_SHARD_MAX_CPUS cids. Topo info + * (core/LLC/node) is cleared and the shard layout is set from the input. On + * invalid input, abort the scheduler. */ -__bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid, u32 cpu_to_cid__sz, - const struct bpf_prog_aux *aux) +__bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid_src, u32 cpu_to_cid_src__sz, + const s32 *shard_start_src, u32 shard_start_src__sz, + const struct bpf_prog_aux *aux) { cpumask_var_t seen __free(free_cpumask_var) = CPUMASK_VAR_NULL; + u32 *node_counts __free(kfree) = NULL; + s32 *cpu_to_cid __free(kfree) = NULL; + s32 *shard_start __free(kfree) = NULL; + u32 npossible = num_possible_cpus(); struct scx_sched *sch; + u32 nr_shards; bool alloced; - s32 cpu, cid; + s32 cpu, cid, si; - /* GFP_KERNEL alloc must happen before the rcu read section */ + /* + * GFP_KERNEL allocs must happen before the rcu read section. Snapshot + * the BPF-supplied arrays so a concurrent map mutation can't change + * them between validation and use. + */ alloced = zalloc_cpumask_var(&seen, GFP_KERNEL); + node_counts = kcalloc(nr_node_ids, sizeof(*node_counts), GFP_KERNEL); + cpu_to_cid = kmemdup(cpu_to_cid_src, cpu_to_cid_src__sz, GFP_KERNEL); + shard_start = kmemdup(shard_start_src, shard_start_src__sz, GFP_KERNEL); guard(rcu)(); @@ -422,17 +459,57 @@ __bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid, u32 cpu_to_cid__sz, if (unlikely(!sch)) return; - if (!alloced) { - scx_error(sch, "scx_bpf_cid_override: failed to allocate cpumask"); + if (!alloced || !node_counts || !cpu_to_cid || !shard_start) { + scx_error(sch, "scx_bpf_cid_override: allocation failed"); return; } - if (cpu_to_cid__sz != nr_cpu_ids * sizeof(s32)) { - scx_error(sch, "scx_bpf_cid_override: expected %zu bytes, got %u", - nr_cpu_ids * sizeof(s32), cpu_to_cid__sz); + if (cpu_to_cid_src__sz != nr_cpu_ids * sizeof(s32)) { + scx_error(sch, "scx_bpf_cid_override: cpu_to_cid expected %zu bytes, got %u", + nr_cpu_ids * sizeof(s32), cpu_to_cid_src__sz); + return; + } + + if (!shard_start_src__sz || shard_start_src__sz % sizeof(s32)) { + scx_error(sch, "scx_bpf_cid_override: invalid shard_start size %u", + shard_start_src__sz); + return; + } + + nr_shards = shard_start_src__sz / sizeof(s32); + + /* validate shard_start[]: starts at 0, strictly increasing, in range */ + if (shard_start[0] != 0) { + scx_error(sch, "scx_bpf_cid_override: shard_start[0] must be 0, got %d", + shard_start[0]); + return; + } + for (si = 1; si < nr_shards; si++) { + if (shard_start[si] <= shard_start[si - 1]) { + scx_error(sch, "scx_bpf_cid_override: shard_start not increasing at [%d]", + si); + return; + } + if (shard_start[si] >= npossible) { + scx_error(sch, "scx_bpf_cid_override: shard_start[%d]=%d >= %u", + si, shard_start[si], npossible); + return; + } + if (shard_start[si] - shard_start[si - 1] > SCX_CID_SHARD_MAX_CPUS) { + scx_error(sch, "scx_bpf_cid_override: shard[%d] span %d exceeds max %d", + si - 1, shard_start[si] - shard_start[si - 1], + SCX_CID_SHARD_MAX_CPUS); + return; + } + } + if (npossible - shard_start[nr_shards - 1] > SCX_CID_SHARD_MAX_CPUS) { + scx_error(sch, "scx_bpf_cid_override: shard[%d] span %d exceeds max %d", + nr_shards - 1, npossible - shard_start[nr_shards - 1], + SCX_CID_SHARD_MAX_CPUS); return; } + /* Validate first so that invalid input leaves globals untouched. */ for_each_possible_cpu(cpu) { s32 c = cpu_to_cid[cpu]; @@ -442,13 +519,56 @@ __bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid, u32 cpu_to_cid__sz, scx_error(sch, "cid %d assigned to multiple cpus", c); return; } + } + + for_each_possible_cpu(cpu) { + s32 c = cpu_to_cid[cpu]; + scx_cpu_to_cid_tbl[cpu] = c; scx_cid_to_cpu_tbl[c] = cpu; } - /* Invalidate stale topo info - the override carries no topology. */ - for (cid = 0; cid < num_possible_cpus(); cid++) + /* + * Derive scx_shard_node[] by majority count: an overridden shard may + * span NUMA nodes, so assign each to the node that owns the most cpus. + */ + for (si = 0; si < nr_shards; si++) { + u32 end = (si + 1 < nr_shards) ? shard_start[si + 1] : npossible; + + memset(node_counts, 0, nr_node_ids * sizeof(*node_counts)); + for (cid = shard_start[si]; cid < end; cid++) { + s32 node = cpu_to_node(scx_cid_to_cpu_tbl[cid]); + + if (numa_valid_node(node)) + node_counts[node]++; + } + scx_shard_node[si] = pick_max_node(node_counts, nr_node_ids); + } + + /* + * Invalidate stale topo info and install shard layout from + * @shard_start. Walk shards to derive shard_cid/shard_idx for each cid. + */ + si = 0; + for (cid = 0; cid < npossible; cid++) { + if (si + 1 < nr_shards && cid >= shard_start[si + 1]) + si++; + scx_cid_to_shard[cid] = si; scx_cid_topo[cid] = SCX_CID_TOPO_NEG; + scx_cid_topo[cid].shard_cid = shard_start[si]; + scx_cid_topo[cid].shard_idx = si; + } + + /* Rebuild scx_cid_shard_ranges[] for the new layout. */ + memset(scx_cid_shard_ranges, 0, npossible * sizeof(*scx_cid_shard_ranges)); + for (si = 0; si < nr_shards; si++) { + u32 end = (si + 1 < nr_shards) ? shard_start[si + 1] : npossible; + + scx_cid_shard_ranges[si].base_cid = shard_start[si]; + scx_cid_shard_ranges[si].nr_cids = end - shard_start[si]; + } + + scx_nr_cid_shards = nr_shards; } /** diff --git a/tools/sched_ext/include/scx/compat.bpf.h b/tools/sched_ext/include/scx/compat.bpf.h index 133058578668..cf469d5ff9ca 100644 --- a/tools/sched_ext/include/scx/compat.bpf.h +++ b/tools/sched_ext/include/scx/compat.bpf.h @@ -122,15 +122,18 @@ static inline bool scx_bpf_sub_dispatch(u64 cgroup_id) } /* - * v7.2: scx_bpf_cid_override() for explicit cpu->cid mapping. Ignore if + * v7.3: scx_bpf_cid_override() for explicit cid and shard mapping. Ignore if * missing. */ -void scx_bpf_cid_override___compat(const s32 *cpu_to_cid, u32 cpu_to_cid__sz) __ksym __weak; +void scx_bpf_cid_override___compat(const s32 *cpu_to_cid, u32 cpu_to_cid__sz, + const s32 *shard_start, u32 shard_start__sz) __ksym __weak; -static inline void scx_bpf_cid_override(const s32 *cpu_to_cid, u32 cpu_to_cid__sz) +static inline void scx_bpf_cid_override(const s32 *cpu_to_cid, u32 cpu_to_cid__sz, + const s32 *shard_start, u32 shard_start__sz) { if (bpf_ksym_exists(scx_bpf_cid_override___compat)) - return scx_bpf_cid_override___compat(cpu_to_cid, cpu_to_cid__sz); + scx_bpf_cid_override___compat(cpu_to_cid, cpu_to_cid__sz, + shard_start, shard_start__sz); } /** diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c index 2df7c53992dc..f6cfe63425d3 100644 --- a/tools/sched_ext/scx_qmap.bpf.c +++ b/tools/sched_ext/scx_qmap.bpf.c @@ -54,18 +54,20 @@ const volatile u32 max_tasks; /* * Optional cid-override test harness. When cid_override_mode is non-zero, - * qmap_init_cids() calls scx_bpf_cid_override() with the caller-supplied - * cpu_to_cid array to exercise the kfunc's acceptance and error paths. See enum + * qmap_init_cids() calls scx_bpf_cid_override() with the caller-supplied arrays + * to exercise the kfunc's acceptance and error paths. See enum * qmap_cid_override for the modes. */ const volatile u32 cid_override_mode; +const volatile u32 cid_override_nr_shards; /* - * Array lives in bss (writable) because scx_bpf_cid_override()'s BPF - * verifier signature treats its len-paired pointer as read/write - rodata + * Arrays live in bss (writable) because scx_bpf_cid_override()'s BPF + * verifier signature treats its len-paired pointers as read/write - rodata * fails verification with "write into map forbidden". Userspace populates - * it before SCX_OPS_LOAD, same as rodata, and nothing writes it after. + * them before SCX_OPS_LOAD, same as rodata, and nothing writes them after. */ s32 cid_override_cpu_to_cid[SCX_QMAP_MAX_CPUS]; +s32 cid_override_shard_start[SCX_QMAP_MAX_CPUS]; UEI_DEFINE(uei); @@ -1082,7 +1084,9 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(qmap_init_cids) } scx_bpf_cid_override((const s32 *)cid_override_cpu_to_cid, - nr_cpu_ids * sizeof(s32)); + nr_cpu_ids * sizeof(s32), + (const s32 *)cid_override_shard_start, + cid_override_nr_shards * sizeof(s32)); return 0; } diff --git a/tools/sched_ext/scx_qmap.c b/tools/sched_ext/scx_qmap.c index c0b5cab579d6..9124183bffec 100644 --- a/tools/sched_ext/scx_qmap.c +++ b/tools/sched_ext/scx_qmap.c @@ -43,7 +43,7 @@ const char help_fmt[] = " -p Switch only tasks on SCHED_EXT policy instead of all\n" " -I Turn on SCX_OPS_ALWAYS_ENQ_IMMED\n" " -F COUNT IMMED stress: force every COUNT'th enqueue to a busy local DSQ (use with -I)\n" -" -C MODE cid-override test (shuffle|bad-dup|bad-range)\n" +" -C MODE cid-override test (shuffle|bad-dup|bad-range|bad-mono)\n" " -v Print libbpf debug messages\n" " -h Display this help and exit\n"; @@ -155,6 +155,7 @@ restart: case 'C': { u32 nr_cpus = libbpf_num_possible_cpus(); u32 mode, i; + s32 shard_sz = 4; if (!strcmp(optarg, "shuffle")) mode = QMAP_CID_OVR_SHUFFLE; @@ -162,13 +163,15 @@ restart: mode = QMAP_CID_OVR_BAD_DUP; else if (!strcmp(optarg, "bad-range")) mode = QMAP_CID_OVR_BAD_RANGE; + else if (!strcmp(optarg, "bad-mono")) + mode = QMAP_CID_OVR_BAD_MONO; else { fprintf(stderr, "unknown cid-override mode '%s'\n", optarg); return 1; } skel->rodata->cid_override_mode = mode; - /* shuffle: reversed cpu_to_cid, bad-dup: dup cid 0, bad-range: identity */ + /* shuffle: reversed cpu_to_cid; others: identity */ for (i = 0; i < nr_cpus; i++) { if (mode == QMAP_CID_OVR_SHUFFLE) skel->bss->cid_override_cpu_to_cid[i] = nr_cpus - 1 - i; @@ -179,6 +182,33 @@ restart: skel->bss->cid_override_cpu_to_cid[1] = 0; if (mode == QMAP_CID_OVR_BAD_RANGE) skel->bss->cid_override_cpu_to_cid[0] = (s32)nr_cpus; + + /* + * bad-mono needs >= 3 shards to build a 0-based but + * non-monotonic shard_start. Shrink the shard size so + * the test runs on any machine with >= 3 cpus. + */ + if (mode == QMAP_CID_OVR_BAD_MONO) { + if (nr_cpus < 3) { + fprintf(stderr, "bad-mono needs >= 3 cpus (have %u)\n", + nr_cpus); + return 1; + } + shard_sz = nr_cpus / 3; + } + + /* shards of shard_sz each */ + skel->rodata->cid_override_nr_shards = (nr_cpus + shard_sz - 1) / shard_sz; + for (i = 0; i < skel->rodata->cid_override_nr_shards; i++) + skel->bss->cid_override_shard_start[i] = i * shard_sz; + + if (mode == QMAP_CID_OVR_BAD_MONO) { + /* swap [1] and [2] to break monotonicity */ + s32 tmp = skel->bss->cid_override_shard_start[1]; + skel->bss->cid_override_shard_start[1] = + skel->bss->cid_override_shard_start[2]; + skel->bss->cid_override_shard_start[2] = tmp; + } break; } case 'v': diff --git a/tools/sched_ext/scx_qmap.h b/tools/sched_ext/scx_qmap.h index 3bcc3579839d..6c3ea1fc74ed 100644 --- a/tools/sched_ext/scx_qmap.h +++ b/tools/sched_ext/scx_qmap.h @@ -33,6 +33,7 @@ enum qmap_cid_override { QMAP_CID_OVR_SHUFFLE = 1, /* valid reversed cpu->cid mapping */ QMAP_CID_OVR_BAD_DUP = 2, /* invalid: duplicate cid assignment */ QMAP_CID_OVR_BAD_RANGE = 3, /* invalid: out-of-range cid */ + QMAP_CID_OVR_BAD_MONO = 4, /* invalid: non-monotonic shard_start */ }; struct cpu_ctx { -- cgit v1.2.3 From 80e6adaa3578b41724dfd4aeaf742aca5d8a7220 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 13 Jul 2026 22:18:42 -1000 Subject: sched_ext: Defer scx_sched kobj sysfs add into the enable workfns Split kobject_init_and_add() in scx_alloc_and_add_sched(): only kobject_init() runs there. A new scx_sched_sysfs_add() helper does kobject_add() (and creates sub_kset when the scheduler implements ops.sub_attach), called by both enable workfns once @sch is linked and its sysfs-visible state is initialized. Prep so a future caps attribute can rely on @sch being fully built by the time it's sysfs-visible. Add early enough that a stall later in enable still leaves sysfs inspectable. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 73 ++++++++++++++++++++++++++++----------------- kernel/sched/ext/internal.h | 1 + kernel/sched/ext/sub.c | 8 ++++- 3 files changed, 53 insertions(+), 29 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index b3e26a46f821..b70170c3de18 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -5735,7 +5735,9 @@ static void scx_root_disable(struct scx_sched *sch) if (sch->sub_kset) kobject_del(&sch->sub_kset->kobj); #endif - kobject_del(&sch->kobj); + /* not added if enable failed before scx_sched_sysfs_add() */ + if (sch->kobj.state_in_sysfs) + kobject_del(&sch->kobj); free_kick_syncs(); @@ -6454,36 +6456,15 @@ struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, * disable. Released in scx_sched_free_rcu_work(). */ kobject_get(&parent->kobj); - ret = kobject_init_and_add(&sch->kobj, &scx_ktype, - &parent->sub_kset->kobj, - "sub-%llu", cgroup_id(cgrp)); - } else { - ret = kobject_init_and_add(&sch->kobj, &scx_ktype, NULL, "root"); - } - - if (ret < 0) { - RCU_INIT_POINTER(ops->priv, NULL); - kobject_put(&sch->kobj); - return ERR_PTR(ret); - } - - if (ops->sub_attach) { - sch->sub_kset = kset_create_and_add("sub", NULL, &sch->kobj); - if (!sch->sub_kset) { - RCU_INIT_POINTER(ops->priv, NULL); - kobject_put(&sch->kobj); - return ERR_PTR(-ENOMEM); - } - } -#else /* CONFIG_EXT_SUB_SCHED */ - ret = kobject_init_and_add(&sch->kobj, &scx_ktype, NULL, "root"); - if (ret < 0) { - RCU_INIT_POINTER(ops->priv, NULL); - kobject_put(&sch->kobj); - return ERR_PTR(ret); } #endif /* CONFIG_EXT_SUB_SCHED */ + /* + * Init the kobj but don't add to sysfs yet. The enable path calls + * scx_sched_sysfs_add() once @sch's sysfs-visible state is initialized. + */ + kobject_init(&sch->kobj, &scx_ktype); + /* * Consume the arena_map ref bpf_scx_reg_cid() took. Defer to here so * earlier failure paths leave cmd->arena_map set and bpf_scx_reg_cid @@ -6541,6 +6522,36 @@ err_put_cgrp: return ERR_PTR(ret); } +/* + * Add @sch's kobject to sysfs, and create its sub_kset if the scheduler + * implements ops.sub_attach. Called by the enable workfns once @sch's + * sysfs-visible state is initialized. + */ +int scx_sched_sysfs_add(struct scx_sched *sch) +{ +#ifdef CONFIG_EXT_SUB_SCHED + struct scx_sched *parent = scx_parent(sch); + int ret; + + if (parent) + ret = kobject_add(&sch->kobj, &parent->sub_kset->kobj, + "sub-%llu", cgroup_id(sch_cgroup(sch))); + else + ret = kobject_add(&sch->kobj, NULL, "root"); + if (ret < 0) + return ret; + + if (sch->ops.sub_attach) { + sch->sub_kset = kset_create_and_add("sub", NULL, &sch->kobj); + if (!sch->sub_kset) + return -ENOMEM; + } + return 0; +#else + return kobject_add(&sch->kobj, NULL, "root"); +#endif +} + static int check_hotplug_seq(struct scx_sched *sch, const struct sched_ext_ops *ops) { @@ -6763,6 +6774,12 @@ static void scx_root_enable_workfn(struct kthread_work *work) sch->exit_info->flags |= SCX_EFLAG_INITIALIZED; } + ret = scx_sched_sysfs_add(sch); + if (ret) { + cpus_read_unlock(); + goto err_disable; + } + for (i = SCX_OPI_CPU_HOTPLUG_BEGIN; i < SCX_OPI_CPU_HOTPLUG_END; i++) if (((void (**)(void))ops)[i]) set_bit(i, sch->has_op); diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 6560a0fa3efa..160ff79faedc 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1690,6 +1690,7 @@ struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, struct cgroup *cgrp, struct scx_sched *parent); int scx_validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops); +int scx_sched_sysfs_add(struct scx_sched *sch); extern raw_spinlock_t scx_sched_lock; extern struct mutex scx_enable_mutex; diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index ce76ae141e0a..9855a9a4e709 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -270,7 +270,9 @@ void scx_sub_disable(struct scx_sched *sch) SCX_CALL_OP(sch, exit, NULL, sch->exit_info); if (sch->sub_kset) kobject_del(&sch->sub_kset->kobj); - kobject_del(&sch->kobj); + /* not added if enable failed before scx_sched_sysfs_add() */ + if (sch->kobj.state_in_sysfs) + kobject_del(&sch->kobj); } /* verify that a scheduler can be attached to @cgrp and return the parent */ @@ -363,6 +365,10 @@ void scx_sub_enable_workfn(struct kthread_work *work) if (ret) goto err_disable; + ret = scx_sched_sysfs_add(sch); + if (ret) + goto err_disable; + if (sch->level >= SCX_SUB_MAX_DEPTH) { scx_error(sch, "max nesting depth %d violated", SCX_SUB_MAX_DEPTH); -- cgit v1.2.3 From 8dba3bbd63e39c97a4d45aea6e95e00b5d14bc8d Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 13 Jul 2026 22:18:42 -1000 Subject: sched_ext: Add per-shard scx_sched storage scaffolding Add struct scx_pshard and sch->pshard[] indexed by shard_idx, each entry allocated on its shard's NUMA node from scx_shard_node[si]. The struct starts empty (one dummy field). Follow-up patches will grow it as shard-local state lands. Only cid-type schedulers with an arena pool get pshards. Allocation happens after ops.init_cids() returns so any scx_bpf_cid_override() it issues has finalized scx_nr_cid_shards and scx_shard_node[]. sch->nr_pshards records the array size for the async RCU free path, which may run after a later scheduler's scx_cid_init() has rewritten the global. v3: Build and publish pshard[] fully-formed here rather than a later patch. v2: Free the partially-allocated pshard array on alloc failure. (sashiko AI) Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 8 +++++++ kernel/sched/ext/internal.h | 18 +++++++++++++++ kernel/sched/ext/sub.c | 54 +++++++++++++++++++++++++++++++++++++++++++++ kernel/sched/ext/sub.h | 4 ++++ 4 files changed, 84 insertions(+) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index b70170c3de18..7a4e007e4ed7 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -4681,6 +4681,8 @@ static void scx_sched_free_rcu_work(struct work_struct *work) free_pnode(sch->pnode[node]); kfree(sch->pnode); + scx_free_pshards(sch); + rhashtable_walk_enter(&sch->dsq_hash, &rht_iter); do { rhashtable_walk_start(&rht_iter); @@ -6763,6 +6765,12 @@ static void scx_root_enable_workfn(struct kthread_work *work) goto err_disable; } + ret = scx_alloc_pshards(sch); + if (ret) { + cpus_read_unlock(); + goto err_disable; + } + if (sch->ops.init) { ret = SCX_CALL_OP_RET(sch, init, NULL); if (ret) { diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 160ff79faedc..5b18c4192c62 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1183,6 +1183,12 @@ struct scx_sched_pnode { struct scx_dispatch_q global_dsq; }; +#ifdef CONFIG_EXT_SUB_SCHED +struct scx_pshard { + int _dummy; /* until the first real field lands */ +}; +#endif + struct scx_sched { /* * cpu-form and cid-form ops share field offsets up to .priv (verified @@ -1230,6 +1236,9 @@ struct scx_sched { */ struct rhashtable dsq_hash; struct scx_sched_pnode **pnode; +#ifdef CONFIG_EXT_SUB_SCHED + struct scx_pshard **pshard; /* indexed by shard_idx */ +#endif struct scx_sched_pcpu __percpu *pcpu; u64 slice_dfl; @@ -1245,6 +1254,15 @@ struct scx_sched { u32 dsp_max_batch; s32 level; +#ifdef CONFIG_EXT_SUB_SCHED + /* + * pshard[] size captured at enable for the async RCU free path - + * scx_nr_cid_shards may be rewritten by a later scx_cid_init() before + * free runs. While sch is active, use the global. + */ + u32 nr_pshards; +#endif + /* * Updates to the following warned bitfields can race causing RMW issues * but it doesn't really matter. diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 9855a9a4e709..3adec9343e46 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -82,6 +82,60 @@ void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch) rcu_assign_pointer(pos->scx_sched, sch); } +static void free_pshard(struct scx_pshard *pshard) +{ + kfree(pshard); +} + +void scx_free_pshards(struct scx_sched *sch) +{ + s32 si; + + if (!sch->pshard) + return; + for (si = 0; si < sch->nr_pshards; si++) + free_pshard(sch->pshard[si]); + kfree(sch->pshard); +} + +static struct scx_pshard *alloc_pshard(struct scx_sched *sch, s32 shard_idx, s32 node) +{ + return kzalloc_node(sizeof(struct scx_pshard), GFP_KERNEL, node); +} + +s32 scx_alloc_pshards(struct scx_sched *sch) +{ + struct scx_pshard **pshard; + s32 si; + + if (!sch->is_cid_type || !sch->arena_pool) + return 0; + + pshard = kzalloc_objs(pshard[0], scx_nr_cid_shards, GFP_KERNEL); + if (!pshard) + return -ENOMEM; + + for (si = 0; si < scx_nr_cid_shards; si++) { + pshard[si] = alloc_pshard(sch, si, scx_shard_node[si]); + if (!pshard[si]) { + while (--si >= 0) + free_pshard(pshard[si]); + kfree(pshard); + return -ENOMEM; + } + } + + sch->nr_pshards = scx_nr_cid_shards; + /* + * Publish only after every entry is built so a reader observing + * @sch->pshard never sees a partially-filled array. Pair the store + * with a barrier and READ_ONCE() on the read side. + */ + smp_wmb(); + WRITE_ONCE(sch->pshard, pshard); + return 0; +} + static DECLARE_WAIT_QUEUE_HEAD(scx_unlink_waitq); void drain_descendants(struct scx_sched *sch) diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index 9b5ac07e5e76..c06d0bfafbd8 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -24,6 +24,8 @@ void drain_descendants(struct scx_sched *sch); void scx_sub_disable(struct scx_sched *sch); void scx_sub_enable_workfn(struct kthread_work *work); bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux *aux); +void scx_free_pshards(struct scx_sched *sch); +s32 scx_alloc_pshards(struct scx_sched *sch); static inline const char *sch_cgrp_path(struct scx_sched *sch) { @@ -39,6 +41,8 @@ static inline const char *sch_cgrp_path(struct scx_sched *sch) { return "/"; } static inline void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch) {} static inline void drain_descendants(struct scx_sched *sch) { } static inline void scx_sub_disable(struct scx_sched *sch) { } +static inline void scx_free_pshards(struct scx_sched *sch) {} +static inline s32 scx_alloc_pshards(struct scx_sched *sch) { return 0; } #endif /* CONFIG_EXT_SUB_SCHED */ -- cgit v1.2.3 From cc7c254c8fd71818b4bd2fbf3dddcd2ebc79e678 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 13 Jul 2026 22:18:42 -1000 Subject: sched_ext: Add scx_cmask_ref for validated arena cmask access kfuncs taking struct scx_cmask * from BPF arena memory have two problems. The pointer can be any value the BPF prog hands in, and the header (@base, @nr_cids, @alloc_words) can be mutated by the prog concurrently with kernel access. Add scx_cmask_ref, a validated handle. _init() normalizes the input pointer into the arena's kern_vm range via scx_arena_to_kaddr() and snapshots the header, rejecting a range outside the machine or a nr_cids whose words exceed the declared @alloc_words. Downstream sizing uses the snapshot, not the live header. _shard() reads slices while _or() and _copy() write back, all bounded by the snapshot. No callers yet. struct scx_cmask's bits[] carried __counted_by(alloc_words), so UBSAN_BOUNDS and FORTIFY_SOURCE bound accesses to the array. That bound is read from @alloc_words at the access. For an arena cmask @alloc_words is BPF-writable. A prog that sets it larger than the real allocation makes the check pass on a genuine overrun, so the annotation catches nothing, and it only runs under those debug configs. Drop it - _init() validates @alloc_words explicitly, and kernel-owned cmasks set it themselves. v2: Validate @alloc_words in _init(), drop __counted_by. (Andrea, sashiko AI) Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/cid.c | 133 +++++++++++++++++++++++++++++++++++++++++++++++ kernel/sched/ext/cid.h | 7 +++ kernel/sched/ext/types.h | 39 +++++++++++++- 3 files changed, 178 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched/ext/cid.c b/kernel/sched/ext/cid.c index 6a22bead4c60..02053a4828e3 100644 --- a/kernel/sched/ext/cid.c +++ b/kernel/sched/ext/cid.c @@ -641,6 +641,12 @@ enum cmask_op2 { /* predicates - short-circuit when the per-word result is true */ CMASK_OP2_SUBSET, CMASK_OP2_INTERSECTS, + /* + * @a is a BPF-arena cmask. Words on @a use READ_ONCE/WRITE_ONCE since + * BPF may read/write concurrently. See scx_cmask_ref_or() / _copy(). + */ + CMASK_OP2_REF_OR, + CMASK_OP2_REF_COPY, }; static __always_inline bool cmask_op2_is_pred(const enum cmask_op2 op) @@ -669,6 +675,12 @@ static __always_inline bool cmask_word_op2(u64 *av, const u64 *bp, u64 mask, return (READ_ONCE(*bp) & ~READ_ONCE(*av)) & mask; case CMASK_OP2_INTERSECTS: return (READ_ONCE(*av) & READ_ONCE(*bp)) & mask; + case CMASK_OP2_REF_OR: + WRITE_ONCE(*av, READ_ONCE(*av) | (READ_ONCE(*bp) & mask)); + return false; + case CMASK_OP2_REF_COPY: + WRITE_ONCE(*av, (READ_ONCE(*av) & ~mask) | (READ_ONCE(*bp) & mask)); + return false; } unreachable(); } @@ -899,6 +911,127 @@ static const struct btf_kfunc_id_set scx_kfunc_set_cid = { .set = &scx_kfunc_ids_cid, }; +/** + * scx_cmask_ref_init - Bind a scx_cmask_ref to a BPF-arena cmask + * @sch: scheduler whose arena hosts @src + * @src: BPF-supplied cmask pointer + * @ref: output ref + * + * Snapshot @src's @base, @nr_cids and @alloc_words. The snapshot is necessary + * because BPF may mutate the live header asynchronously. + * + * Return 0 on success, -EINVAL if the range is out of bounds or @alloc_words + * doesn't cover it. + */ +int scx_cmask_ref_init(struct scx_sched *sch, const struct scx_cmask *src, + struct scx_cmask_ref *ref) +{ + struct scx_cmask *kern_src = scx_arena_to_kaddr(sch, src); + u32 base, nr_cids, alloc_words, npossible = num_possible_cpus(); + + base = READ_ONCE(kern_src->base); + nr_cids = READ_ONCE(kern_src->nr_cids); + alloc_words = READ_ONCE(kern_src->alloc_words); + + if (unlikely(base >= npossible || nr_cids > npossible - base || + SCX_CMASK_NR_WORDS(nr_cids) > alloc_words)) + return -EINVAL; + + ref->sch = sch; + ref->src = kern_src; + ref->base = base; + ref->nr_cids = nr_cids; + + ref->shard_first = scx_cid_to_shard[base]; + if (likely(nr_cids)) + ref->shard_end = scx_cid_to_shard[base + nr_cids - 1] + 1; + else + ref->shard_end = ref->shard_first; + + return 0; +} + +/** + * scx_cmask_ref_shard - Read one shard from @ref into @out + * @ref: validated ref + * @shard_idx: target shard, in [@ref->shard_first, @ref->shard_end) + * @out: output cmask whose @out->alloc_words must hold the shard + * + * Set @out to the intersection of @ref's range with @shard_idx's cid range, + * with bits[] read from @ref->src via READ_ONCE. Empty intersection sets + * @out->nr_cids to 0. scx_error()s on @ref's sched if @out can't hold the + * shard. + */ +void scx_cmask_ref_shard(const struct scx_cmask_ref *ref, s32 shard_idx, + struct scx_cmask *out) +{ + const struct scx_cid_shard *shard = &scx_cid_shard_ranges[shard_idx]; + u32 shard_base = shard->base_cid; + u32 shard_end = shard_base + shard->nr_cids; + u32 isect_base, isect_end, nr_words, src_off, wi; + u64 head_mask, tail_mask; + + isect_base = max(ref->base, shard_base); + isect_end = min(ref->base + ref->nr_cids, shard_end); + + if (isect_base >= isect_end) { + out->base = shard_base; + out->nr_cids = 0; + return; + } + + nr_words = ((isect_end - 1) / 64) - (isect_base / 64) + 1; + if (nr_words > out->alloc_words) { + scx_error(ref->sch, "scx_cmask_ref_shard: out alloc_words=%u < %u for shard %d", + out->alloc_words, nr_words, shard_idx); + out->base = shard_base; + out->nr_cids = 0; + return; + } + + out->base = isect_base; + out->nr_cids = isect_end - isect_base; + src_off = (isect_base / 64) - (ref->base / 64); + + for (wi = 0; wi < nr_words; wi++) + out->bits[wi] = READ_ONCE(ref->src->bits[src_off + wi]); + + head_mask = GENMASK_U64(63, isect_base & 63); + out->bits[0] &= head_mask; + tail_mask = GENMASK_U64((isect_end - 1) & 63, 0); + out->bits[nr_words - 1] &= tail_mask; +} + +/** + * scx_cmask_ref_or - OR @src into the arena cmask referenced by @ref + * @ref: validated ref + * @src: stable kernel cmask + * + * Bits inside the intersection of @ref's snapshotted range with @src's range + * are OR'd into @ref->src and bits outside are left unchanged. Stores on + * @ref->src use WRITE_ONCE since BPF may read/write concurrently. + */ +void scx_cmask_ref_or(const struct scx_cmask_ref *ref, const struct scx_cmask *src) +{ + cmask_walk_op2(ref->src->bits, ref->base, ref->nr_cids, + src->bits, src->base, src->nr_cids, CMASK_OP2_REF_OR); +} + +/** + * scx_cmask_ref_copy - Copy @src into the arena cmask referenced by @ref + * @ref: validated ref + * @src: stable kernel cmask + * + * Bits inside the intersection of @ref's snapshotted range with @src's range + * take @src's values and bits outside are left unchanged. Stores on @ref->src + * use WRITE_ONCE since BPF may read/write concurrently. + */ +void scx_cmask_ref_copy(const struct scx_cmask_ref *ref, const struct scx_cmask *src) +{ + cmask_walk_op2(ref->src->bits, ref->base, ref->nr_cids, + src->bits, src->base, src->nr_cids, CMASK_OP2_REF_COPY); +} + int scx_cid_kfunc_init(void) { return register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_init_cids) ?: diff --git a/kernel/sched/ext/cid.h b/kernel/sched/ext/cid.h index cdc18a7a48f5..70d97acd0ac4 100644 --- a/kernel/sched/ext/cid.h +++ b/kernel/sched/ext/cid.h @@ -293,4 +293,11 @@ static inline s32 scx_cpu_ret(struct scx_sched *sch, s32 cpu_or_cid) return scx_cid_to_cpu(sch, cpu_or_cid); } +int scx_cmask_ref_init(struct scx_sched *sch, const struct scx_cmask *src, + struct scx_cmask_ref *ref); +void scx_cmask_ref_shard(const struct scx_cmask_ref *ref, s32 shard_idx, + struct scx_cmask *out); +void scx_cmask_ref_or(const struct scx_cmask_ref *ref, const struct scx_cmask *src); +void scx_cmask_ref_copy(const struct scx_cmask_ref *ref, const struct scx_cmask *src); + #endif /* _KERNEL_SCHED_EXT_CID_H */ diff --git a/kernel/sched/ext/types.h b/kernel/sched/ext/types.h index b31d12931999..a1a05820725e 100644 --- a/kernel/sched/ext/types.h +++ b/kernel/sched/ext/types.h @@ -116,7 +116,7 @@ struct scx_cmask { u32 base; u32 nr_cids; u32 alloc_words; - u64 bits[] __counted_by(alloc_words); + u64 bits[]; }; /* @@ -172,4 +172,41 @@ struct scx_cmask { #define SCX_CMASK_DEFINE_SHARD(NAME, BASE, NR_CIDS) \ __SCX_CMASK_DEFINE(NAME, BASE, NR_CIDS, SCX_CID_SHARD_MAX_CPUS) +/* + * scx_cmask_ref: validated reference to a BPF-arena cmask. + * + * scx_cmask_ref_init() normalizes the pointer into the arena and snapshots + * @base/@nr_cids. The snapshot is what downstream code uses for sizing - the + * live header can be mutated concurrently by BPF. + * + * scx_cmask_ref_shard() reads one shard into a cmask. scx_cmask_ref_or() and + * scx_cmask_ref_copy() write back into the referenced arena cmask, bounded by + * the snapshot. + * + * Typical input use: + * + * struct scx_cmask_ref ref; + * SCX_CMASK_DEFINE(shard, 0, SCX_CID_SHARD_MAX_CPUS); + * s32 idx, ret; + * + * ret = scx_cmask_ref_init(sch, src, &ref); + * if (ret < 0) + * return ret; + * + * for (idx = ref.shard_first; idx < ref.shard_end; idx++) { + * scx_cmask_ref_shard(&ref, idx, shard); + * if (!shard->nr_cids) + * continue; + * ... use idx and shard ... + * } + */ +struct scx_cmask_ref { + struct scx_sched *sch; + struct scx_cmask *src; + u32 base; + u32 nr_cids; + s32 shard_first; + s32 shard_end; +}; + #endif /* _KERNEL_SCHED_EXT_TYPES_H */ -- cgit v1.2.3 From 33ffb56e852cdb352d9ffeecfc2c7ffe0945c815 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 13 Jul 2026 22:18:42 -1000 Subject: sched_ext: Build the set_cmask scratch from trusted geometry scx_call_op_set_cpumask() builds a per-cpu cmask in the set_cmask scratch, which lives in BPF-writable arena. A scheduler can corrupt the scratch's inline header (base, nr_cids, alloc_words) from another cpu, so sizing and indexing the write from it risks an out-of-bounds write. Drive the build from kernel-known geometry instead. scx_cmask_ref_init_kern() imposes base and nr_cids rather than reading them, and scx_cmask_ref_from_cpumask() fills the scratch from the ref. Neither reads the header back. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/cid.c | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++ kernel/sched/ext/cid.h | 4 +++ kernel/sched/ext/ext.c | 11 +++++--- 3 files changed, 81 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/cid.c b/kernel/sched/ext/cid.c index 02053a4828e3..9dfd242be34f 100644 --- a/kernel/sched/ext/cid.c +++ b/kernel/sched/ext/cid.c @@ -951,6 +951,38 @@ int scx_cmask_ref_init(struct scx_sched *sch, const struct scx_cmask *src, return 0; } +/** + * scx_cmask_ref_init_kern - Bind a scx_cmask_ref to a kernel-owned cmask + * @sch: scheduler the cmask belongs to + * @m: kernel address of the target cmask, storage sized for @nr_cids at @base + * @base: first cid of the active range + * @nr_cids: active range length + * @ref: output ref + * + * Like scx_cmask_ref_init() but the geometry is supplied by the caller, not + * read from @m's header, so a concurrent BPF write to the header can't steer + * later sizing or offsets. Rewrite the header from the trusted geometry and + * bind @ref to it. + */ +void scx_cmask_ref_init_kern(struct scx_sched *sch, struct scx_cmask *m, + u32 base, u32 nr_cids, struct scx_cmask_ref *ref) +{ + WRITE_ONCE(m->base, base); + WRITE_ONCE(m->nr_cids, nr_cids); + WRITE_ONCE(m->alloc_words, SCX_CMASK_NR_WORDS(nr_cids)); + + ref->sch = sch; + ref->src = m; + ref->base = base; + ref->nr_cids = nr_cids; + + ref->shard_first = scx_cid_to_shard[base]; + if (likely(nr_cids)) + ref->shard_end = scx_cid_to_shard[base + nr_cids - 1] + 1; + else + ref->shard_end = ref->shard_first; +} + /** * scx_cmask_ref_shard - Read one shard from @ref into @out * @ref: validated ref @@ -1032,6 +1064,44 @@ void scx_cmask_ref_copy(const struct scx_cmask_ref *ref, const struct scx_cmask src->bits, src->base, src->nr_cids, CMASK_OP2_REF_COPY); } +/** + * scx_cmask_ref_from_cpumask - Populate @ref's arena cmask from a cpumask + * @ref: kern-bound ref, see scx_cmask_ref_init_kern() + * @cpumask: cpus to translate into cids + * + * Write @ref's active range one word at a time, setting each cid's bit when + * its cpu is in @cpumask. Offsets and length come from @ref's trusted geometry + * and stores use WRITE_ONCE since BPF may read concurrently, so the arena + * header is never read. + */ +void scx_cmask_ref_from_cpumask(const struct scx_cmask_ref *ref, + const struct cpumask *cpumask) +{ + struct scx_cmask *m = ref->src; + u32 base = ref->base, nr_cids = ref->nr_cids; + u32 wi, nr_words; + + if (!nr_cids) + return; + + nr_words = (base + nr_cids - 1) / 64 - base / 64 + 1; + for (wi = 0; wi < nr_words; wi++) { + u32 word_first_cid = (base / 64 + wi) * 64; + u64 word = 0; + u32 bit; + + for (bit = 0; bit < 64; bit++) { + u32 cid = word_first_cid + bit; + + if (cid < base || cid >= base + nr_cids) + continue; + if (cpumask_test_cpu(__scx_cid_to_cpu(cid), cpumask)) + word |= BIT_U64(bit); + } + WRITE_ONCE(m->bits[wi], word); + } +} + int scx_cid_kfunc_init(void) { return register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_init_cids) ?: diff --git a/kernel/sched/ext/cid.h b/kernel/sched/ext/cid.h index 70d97acd0ac4..b36a1a28eac8 100644 --- a/kernel/sched/ext/cid.h +++ b/kernel/sched/ext/cid.h @@ -295,8 +295,12 @@ static inline s32 scx_cpu_ret(struct scx_sched *sch, s32 cpu_or_cid) int scx_cmask_ref_init(struct scx_sched *sch, const struct scx_cmask *src, struct scx_cmask_ref *ref); +void scx_cmask_ref_init_kern(struct scx_sched *sch, struct scx_cmask *m, + u32 base, u32 nr_cids, struct scx_cmask_ref *ref); void scx_cmask_ref_shard(const struct scx_cmask_ref *ref, s32 shard_idx, struct scx_cmask *out); +void scx_cmask_ref_from_cpumask(const struct scx_cmask_ref *ref, + const struct cpumask *cpumask); void scx_cmask_ref_or(const struct scx_cmask_ref *ref, const struct scx_cmask *src); void scx_cmask_ref_copy(const struct scx_cmask_ref *ref, const struct scx_cmask *src); diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 7a4e007e4ed7..396245de01f7 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -424,12 +424,15 @@ static inline void scx_call_op_set_cpumask(struct scx_sched *sch, struct rq *rq, { if (scx_is_cid_type()) { struct scx_cmask *kern_va = *this_cpu_ptr(sch->set_cmask_scratch); + struct scx_cmask_ref ref; + /* - * Build the per-CPU arena cmask and hand BPF its arena address. - * Caller holds the rq lock with IRQs disabled, which makes us - * the sole user of the scratch area. + * Build the per-cpu arena cmask from kernel geometry via @ref, + * never reading its BPF-writable header, and hand BPF the arena + * address. The rq lock makes this cpu the sole kernel writer. */ - scx_cpumask_to_cmask(cpumask, kern_va); + scx_cmask_ref_init_kern(sch, kern_va, 0, num_possible_cpus(), &ref); + scx_cmask_ref_from_cpumask(&ref, cpumask); SCX_CALL_CID_OP_TASK(sch, set_cmask, rq, task, scx_kaddr_to_arena(sch, kern_va)); } else { -- cgit v1.2.3 From 70f8b1785327f233d667d8fc0751a7d2ec231597 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 13 Jul 2026 22:18:43 -1000 Subject: sched_ext: RCU-protect the sub-sched tree's children/sibling lists Future kfuncs need to walk descendants without scx_sched_lock. Make the walker RCU-safe so that they can. A sub-sched's fields are initialized before it is linked, so a walk that observes a linked node also observes its setup. In-place changes after linking carry their own ordering. Switch the children/sibling list ops to RCU and expand the descendant walker to accept rcu_read_lock as a valid read-side context. Walkers that mutate keep scx_sched_lock. A sub-sched can be linked while an ancestor is bypassing, after the bypass walk that propagates the depth has passed its parent. Bypass state is a per-cpu flag plus a depth count and can't be established atomically at link time, so refuse to link under a bypassing ancestor. Take scx_bypass_lock across linking to check the parent's bypass state coherently. v3: Reject linking under a bypassing ancestor instead of inheriting bypass_depth. (sashiko AI) v2: Inherit bypass_depth before publishing @sch on the RCU sibling list. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 18 +++++++++++++++--- kernel/sched/ext/sub.c | 11 +++++++---- kernel/sched/ext/sub.h | 4 ++-- 3 files changed, 24 insertions(+), 9 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 396245de01f7..6f63d29e0a9c 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -5502,7 +5502,8 @@ s32 scx_link_sched(struct scx_sched *sch) const char *err_msg = ""; s32 ret = 0; - scoped_guard(raw_spinlock_irq, &scx_sched_lock) { + scoped_guard(raw_spinlock_irqsave, &scx_bypass_lock) /* for the parent bypass check */ + scoped_guard(raw_spinlock, &scx_sched_lock) { #ifdef CONFIG_EXT_SUB_SCHED struct scx_sched *parent = scx_parent(sch); @@ -5519,6 +5520,17 @@ s32 scx_link_sched(struct scx_sched *sch) break; } + /* + * Bypass state is spread across per-cpu flags and a + * depth count, so inheriting it is tricky and has no + * valid use case. Refuse it. + */ + if (READ_ONCE(parent->bypass_depth)) { + err_msg = "parent bypassing"; + ret = -EBUSY; + break; + } + ret = rhashtable_lookup_insert_fast(&scx_sched_hash, &sch->hash_node, scx_sched_hash_params); if (ret) { @@ -5526,7 +5538,7 @@ s32 scx_link_sched(struct scx_sched *sch) break; } - list_add_tail(&sch->sibling, &parent->children); + list_add_tail_rcu(&sch->sibling, &parent->children); } #endif /* CONFIG_EXT_SUB_SCHED */ @@ -5553,7 +5565,7 @@ void scx_unlink_sched(struct scx_sched *sch) if (scx_parent(sch)) { rhashtable_remove_fast(&scx_sched_hash, &sch->hash_node, scx_sched_hash_params); - list_del_init(&sch->sibling); + list_del_rcu(&sch->sibling); } #endif /* CONFIG_EXT_SUB_SCHED */ list_del_rcu(&sch->all); diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 3adec9343e46..5fe2f79064dc 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -35,21 +35,24 @@ struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sche struct scx_sched *next; lockdep_assert(lockdep_is_held(&scx_enable_mutex) || - lockdep_is_held(&scx_sched_lock)); + lockdep_is_held(&scx_sched_lock) || + rcu_read_lock_any_held()); /* if first iteration, visit @root */ if (!pos) return root; /* visit the first child if exists */ - next = list_first_entry_or_null(&pos->children, struct scx_sched, sibling); + next = list_first_or_null_rcu(&pos->children, struct scx_sched, sibling); if (next) return next; /* no child, visit my or the closest ancestor's next sibling */ while (pos != root) { - if (!list_is_last(&pos->sibling, &scx_parent(pos)->children)) - return list_next_entry(pos, sibling); + next = list_next_or_null_rcu(&scx_parent(pos)->children, &pos->sibling, + struct scx_sched, sibling); + if (next) + return next; pos = scx_parent(pos); } diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index c06d0bfafbd8..45f3952b5d81 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -52,8 +52,8 @@ static inline s32 scx_alloc_pshards(struct scx_sched *sch) { return 0; } * @root: sched to walk the descendants of * * Walk @root's descendants. @root is included in the iteration and the first - * node to be visited. Must be called with either scx_enable_mutex or - * scx_sched_lock held. + * node to be visited. Must be called with scx_enable_mutex, scx_sched_lock, or + * RCU read lock. */ #define scx_for_each_descendant_pre(pos, root) \ for ((pos) = scx_next_descendant_pre(NULL, (root)); (pos); \ -- cgit v1.2.3 From bbda59d85341d6d4d957596233646c84d9d9a451 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 13 Jul 2026 22:18:43 -1000 Subject: sched_ext: Add scx_skip_subtree_pre() Factor the sibling/ancestor portion of scx_next_descendant_pre() out as scx_skip_subtree_pre(), a pre-order walk primitive that skips @pos's subtree, and call it from scx_next_descendant_pre(). Same locking rules as the existing primitive. Used in a follow-up to fast-skip subtrees that have nothing to do during a descendant walk. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/sub.c | 37 ++++++++++++++++++++++++++++--------- kernel/sched/ext/sub.h | 2 ++ 2 files changed, 30 insertions(+), 9 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 5fe2f79064dc..17ed0d28f383 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -21,6 +21,33 @@ #ifdef CONFIG_EXT_SUB_SCHED +/** + * scx_skip_subtree_pre - Skip @pos's subtree in a pre-order walk + * @pos: current position + * @root: walk root + * + * In a walk started by scx_next_descendant_pre(), continue past @pos's subtree: + * return @pos's next sibling, or the closest ancestor's next sibling, or NULL + * if @pos's subtree is the last under @root. Same locking rules. + */ +struct scx_sched *scx_skip_subtree_pre(struct scx_sched *pos, struct scx_sched *root) +{ + struct scx_sched *next; + + lockdep_assert(lockdep_is_held(&scx_enable_mutex) || + lockdep_is_held(&scx_sched_lock) || + rcu_read_lock_any_held()); + + while (pos != root) { + next = list_next_or_null_rcu(&scx_parent(pos)->children, &pos->sibling, + struct scx_sched, sibling); + if (next) + return next; + pos = scx_parent(pos); + } + return NULL; +} + /** * scx_next_descendant_pre - find the next descendant for pre-order walk * @pos: the current position (%NULL to initiate traversal) @@ -48,15 +75,7 @@ struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sche return next; /* no child, visit my or the closest ancestor's next sibling */ - while (pos != root) { - next = list_next_or_null_rcu(&scx_parent(pos)->children, &pos->sibling, - struct scx_sched, sibling); - if (next) - return next; - pos = scx_parent(pos); - } - - return NULL; + return scx_skip_subtree_pre(pos, root); } static struct scx_sched *scx_find_sub_sched(u64 cgroup_id) diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index 45f3952b5d81..2688087a0654 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -15,6 +15,7 @@ #ifdef CONFIG_EXT_SUB_SCHED +struct scx_sched *scx_skip_subtree_pre(struct scx_sched *pos, struct scx_sched *root); struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root); void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch); struct cgroup *sch_cgroup(struct scx_sched *sch); @@ -35,6 +36,7 @@ static inline const char *sch_cgrp_path(struct scx_sched *sch) #else /* CONFIG_EXT_SUB_SCHED */ static inline struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; } +static inline struct scx_sched *scx_skip_subtree_pre(struct scx_sched *pos, struct scx_sched *root) { return NULL; } static inline void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) {} static inline struct cgroup *sch_cgroup(struct scx_sched *sch) { return NULL; } static inline const char *sch_cgrp_path(struct scx_sched *sch) { return "/"; } -- cgit v1.2.3 From 81507f148e9f98be3a75b3c26979f3b4f7c8001e Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 13 Jul 2026 22:18:43 -1000 Subject: sched_ext: Stop resolving a disabled scheduler's programs A scheduler's BPF programs can outlive it. A timer it armed or a tracing program it loaded can fire after ops.exit() has run, before the programs are unloaded, and scx_prog_sched() still resolves the program to its scheduler through ops->priv. Harmless while kfuncs touch only lifetime-stable state, but a hazard once a kfunc reads global state a newly loaded scheduler can change underneath it. Add scx_sched->dead, set right after ops.exit() and drained with synchronize_rcu(). It follows exit() rather than preceding it so exit()'s own kfunc calls still resolve to @sch. scx_prog_sched() returns NULL for a dead scheduler, so every kfunc's existing !sch bail rejects it at one choke point. v2: Check dead in the CONFIG_EXT_SUB_SCHED=n scx_prog_sched() too. (sashiko AI) Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 8 ++++++++ kernel/sched/ext/internal.h | 19 +++++++++++++++---- kernel/sched/ext/sub.c | 9 +++++++++ 3 files changed, 32 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 6f63d29e0a9c..44231474e2da 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -5733,6 +5733,14 @@ static void scx_root_disable(struct scx_sched *sch) if (sch->ops.exit) SCX_CALL_OP(sch, exit, NULL, sch->exit_info); + /* + * @sch's non-ops programs such as timers and tracers can fire after + * ops.exit(). Now that exit is complete, stop scx_prog_sched() from + * resolving to @sch and drain in-flight resolvers. + */ + WRITE_ONCE(sch->dead, true); + synchronize_rcu(); + scx_unlink_sched(sch); /* diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 5b18c4192c62..4452aac89b14 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1202,6 +1202,7 @@ struct scx_sched { struct sched_ext_ops_cid ops_cid; }; bool is_cid_type; /* true if registered via bpf_sched_ext_ops_cid */ + bool dead; /* set after ops.exit(), gates scx_prog_sched() */ /* * Arena map auto-discovered from member progs at struct_ops attach. @@ -1976,14 +1977,20 @@ static inline bool scx_task_on_sched(struct scx_sched *sch, static inline struct scx_sched *scx_prog_sched(const struct bpf_prog_aux *aux) { struct sched_ext_ops *ops; - struct scx_sched *root; + struct scx_sched *sch, *root; ops = bpf_prog_get_assoc_struct_ops(aux); - if (likely(ops)) - return rcu_dereference_all(ops->priv); + if (likely(ops)) { + sch = rcu_dereference_all(ops->priv); + if (sch && unlikely(READ_ONCE(sch->dead))) + return NULL; + return sch; + } root = rcu_dereference_all(scx_root); if (root) { + if (unlikely(READ_ONCE(root->dead))) + return NULL; /* * COMPAT-v6.19: Schedulers built before sub-sched support was * introduced may have unassociated non-struct_ops programs. @@ -2035,7 +2042,11 @@ static inline bool scx_task_on_sched(struct scx_sched *sch, static inline struct scx_sched *scx_prog_sched(const struct bpf_prog_aux *aux) { - return rcu_dereference_all(scx_root); + struct scx_sched *root = rcu_dereference_all(scx_root); + + if (root && unlikely(READ_ONCE(root->dead))) + return NULL; + return root; } static inline struct scx_sched *scx_parent(struct scx_sched *sch) { return NULL; } diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 17ed0d28f383..017225ad492f 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -344,6 +344,15 @@ void scx_sub_disable(struct scx_sched *sch) if (sch->ops.exit) SCX_CALL_OP(sch, exit, NULL, sch->exit_info); + + /* + * @sch's non-ops programs such as timers and tracers can fire after + * ops.exit(). Now that exit is complete, stop scx_prog_sched() from + * resolving to @sch and drain in-flight resolvers. + */ + WRITE_ONCE(sch->dead, true); + synchronize_rcu(); + if (sch->sub_kset) kobject_del(&sch->sub_kset->kobj); /* not added if enable failed before scx_sched_sysfs_add() */ -- cgit v1.2.3 From 86094b95efcf747c80930a2a675c5b14505a19b1 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 13 Jul 2026 22:18:43 -1000 Subject: sched_ext: Add per-shard cap delegation for sub-schedulers Caps are per-cid permissions parents delegate to direct children via scx_bpf_sub_grant() / scx_bpf_sub_revoke(). A child's cap set is always a subset of its parent's. Sub-scheds check their caps locally, and cross-sched communication is needed only when the delegation set itself changes. Caps will be used to implement sub-sched scheduling on the enqueue path. Picking a cid for a task at a leaf depends on which cids the leaf is allowed to use, and resolving that programmatically on every enqueue would mean a cross-sched round-trip call chain, possibly retrying if the request can't be granted as-is. The dispatch path is different - it runs as top-down recursion via scx_bpf_sub_dispatch(). Locking is per shard. cid space is split into shards, and each sub-sched has its own pshard->lock for each shard. Operations are broken up on shard boundaries. Different shards never contend. Shards are expected to be topology-aligned and likely to serve as the locality unit when cids are allocated to schedulers, so per-shard lock granularity scales naturally with the allocation pattern. This patch adds the framework with a single dummy cap. Real caps land in later patches. The enable path is reordered for pshards. scx_arena_pool_init() moves ahead of scx_link_sched() so the pshards are allocated before the sched becomes reachable - scx_alloc_pshards() skips allocation when the arena pool isn't initialized. - scx_bpf_sub_grant(): Per-cid all-or-nothing grant to direct child. - scx_bpf_sub_revoke(): Clear caps on @cmask across @child and its subtree. - scx_bpf_sub_caps(): Lockless snapshot of caps on a cid range. /sys/kernel/sched_ext/SCHED/caps shows the caps each scheduler currently holds. v4: Move the pshard[] full build/publish and the err_disable scx_error() recording to earlier patches. (sashiko AI) v3: Build pshard[] fully before publishing it, read it with READ_ONCE. (sashiko AI) v2: Validate ops before scx_link_sched() publishes the sub. (sashiko AI) Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 77 ++++++- kernel/sched/ext/internal.h | 56 ++++- kernel/sched/ext/sub.c | 342 ++++++++++++++++++++++++++++++- kernel/sched/ext/sub.h | 2 + tools/sched_ext/include/scx/common.bpf.h | 6 + 5 files changed, 468 insertions(+), 15 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 44231474e2da..3cc7e5939a71 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -4742,9 +4742,52 @@ static ssize_t scx_attr_events_show(struct kobject *kobj, } SCX_ATTR(events); +#ifdef CONFIG_EXT_SUB_SCHED +static const char *scx_cap_names[__SCX_NR_CAPS] = { + [__SCX_CAP_DUMMY] = "dummy", +}; + +static ssize_t scx_attr_caps_show(struct kobject *kobj, + struct kobj_attribute *ka, char *buf) +{ + struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj); + u32 npossible = num_possible_cpus(); + struct scx_cmask *agg __free(kfree) = + kzalloc(struct_size(agg, bits, SCX_CMASK_NR_WORDS(npossible)), GFP_KERNEL); + unsigned long *agg_bm __free(bitmap) = bitmap_zalloc(npossible, GFP_KERNEL); + ssize_t count = 0; + s32 cap, si; + + if (!agg || !agg_bm) + return -ENOMEM; + + for (cap = 0; cap < __SCX_NR_CAPS; cap++) { + SCX_CMASK_DEFINE(snap, 0, SCX_CID_SHARD_MAX_CPUS); + + scx_cmask_init(agg, 0, npossible); + for (si = 0; si < sch->nr_pshards; si++) { + struct scx_cmask *cm = &sch->pshard[si]->caps[cap].cmask; + + scx_cmask_reframe(snap, cm->base, cm->nr_cids); + scx_cmask_copy(snap, cm); + scx_cmask_or(agg, snap); + } + /* %*pbl takes unsigned long bitmap layout, convert from u64 */ + bitmap_from_arr64(agg_bm, agg->bits, npossible); + count += sysfs_emit_at(buf, count, "%s: %*pbl\n", + scx_cap_names[cap], npossible, agg_bm); + } + return count; +} +SCX_ATTR(caps); +#endif /* CONFIG_EXT_SUB_SCHED */ + static struct attribute *scx_sched_attrs[] = { &scx_attr_ops.attr, &scx_attr_events.attr, +#ifdef CONFIG_EXT_SUB_SCHED + &scx_attr_caps.attr, +#endif NULL, }; ATTRIBUTE_GROUPS(scx_sched); @@ -6763,8 +6806,8 @@ static void scx_root_enable_workfn(struct kthread_work *work) /* * A cid-form scheduler finalizes its cid layout in ops.init_cids(), - * which may call scx_bpf_cid_override(). Run it before ops.init() so - * the final layout is in effect. + * which may call scx_bpf_cid_override(). Run it before the caps and + * shard state are built so the final layout is in effect. */ if (sch->is_cid_type && sch->ops_cid.init_cids) { ret = SCX_CALL_OP_RET(sch, init_cids, NULL); @@ -6794,6 +6837,9 @@ static void scx_root_enable_workfn(struct kthread_work *work) goto err_disable; } + scx_init_root_caps(sch); + + /* the cid caps and shards are live now, so ops.init() can query them */ if (sch->ops.init) { ret = SCX_CALL_OP_RET(sch, init, NULL); if (ret) { @@ -7475,7 +7521,7 @@ static struct bpf_struct_ops bpf_sched_ext_ops = { /* * cid-form cfi stubs. Stubs whose signatures match the cpu-form (param types - * identical, only param names differ across structs) are reused; only + * identical, only param names differ across structs) are reused. Only * set_cmask needs a fresh stub since the second argument type differs. */ static void sched_ext_ops_cid__set_cmask(struct task_struct *p, @@ -9672,6 +9718,28 @@ out: } #endif /* CONFIG_CGROUP_SCHED */ +#ifndef CONFIG_EXT_SUB_SCHED +__bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, + const struct scx_cmask *cmask__ign, + struct scx_cmask *denied_out__ign, + const struct bpf_prog_aux *aux) +{ + return -EOPNOTSUPP; +} + +__bpf_kfunc void scx_bpf_sub_revoke(u64 cgroup_id, u64 caps, + const struct scx_cmask *cmask__ign, + const struct bpf_prog_aux *aux) +{ +} + +__bpf_kfunc s32 scx_bpf_sub_caps(u64 cgroup_id, u64 caps, struct scx_cmask *out__ign, + const struct bpf_prog_aux *aux) +{ + return -EOPNOTSUPP; +} +#endif /* !CONFIG_EXT_SUB_SCHED */ + __bpf_kfunc_end_defs(); BTF_KFUNCS_START(scx_kfunc_ids_any) @@ -9716,6 +9784,9 @@ BTF_ID_FLAGS(func, scx_bpf_events) #ifdef CONFIG_CGROUP_SCHED BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_IMPLICIT_ARGS | KF_RCU | KF_ACQUIRE) #endif +BTF_ID_FLAGS(func, scx_bpf_sub_grant, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_sub_revoke, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_sub_caps, KF_IMPLICIT_ARGS) BTF_KFUNCS_END(scx_kfunc_ids_any) static const struct btf_kfunc_id_set scx_kfunc_set_any = { diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 4452aac89b14..1686aa823932 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -786,9 +786,9 @@ struct sched_ext_ops { /** * @init_cids: Finalize the cid layout (cid-form only) * - * Runs after the default cid layout is built, before ops.init(). A - * cid-form scheduler may call scx_bpf_cid_override() here for a custom - * layout. Ignored for cpu-form schedulers. + * Runs after the default cid layout is built, before caps and shards + * are finalized. A cid-form scheduler may call scx_bpf_cid_override() + * here for a custom layout. Ignored for cpu-form schedulers. */ s32 (*init_cids)(void); @@ -1183,9 +1183,57 @@ struct scx_sched_pnode { struct scx_dispatch_q global_dsq; }; +/* + * Sub-sched capability delegation. + * + * Caps are per-cid permissions parents delegate to direct children via + * scx_bpf_sub_grant() / scx_bpf_sub_revoke(). A child's cap set is always a + * subset of its parent's. A sub-sched checks its caps locally, and cross-sched + * communication is needed only when the delegation set itself changes. + * + * Caps are used to implement sub-sched scheduling on the enqueue path. Picking + * a cid for a task at a leaf depends on which cids the leaf is allowed to use. + * Resolving that programmatically on every enqueue would mean a cross-sched + * round-trip call chain, possibly retrying if the request can't be granted + * as-is. + * + * The dispatch path is different - it runs as top-down recursion via + * scx_bpf_sub_dispatch(): a sched's dispatch op invokes a child's dispatch op + * on the local rq, and the subtree dispatches in a single pass. + * + * Locking is per shard. cid space is split into shards, and each sub-sched has + * its own pshard->lock for each shard. Operations are broken up on shard + * boundaries. Different shards never contend. Shards are expected to be + * topology-aligned and likely to serve as the locality unit when cids are + * allocated to schedulers, so per-shard lock granularity scales naturally with + * the allocation pattern. + */ +enum scx_cap_flags { + __SCX_CAP_DUMMY = 0, + + __SCX_NR_CAPS, + __SCX_CAP_ALL = BIT_U64(__SCX_NR_CAPS) - 1, + + SCX_CAP_DUMMY = BIT_U64(__SCX_CAP_DUMMY), +}; + #ifdef CONFIG_EXT_SUB_SCHED +/* iterate set bits in a u64 cap mask */ +#define scx_for_each_cap_bit(cap_bit, caps) \ + for (u64 __caps = (caps); \ + __caps && ((cap_bit) = __ffs64(__caps), true); \ + __caps &= __caps - 1) + struct scx_pshard { - int _dummy; /* until the first real field lands */ + raw_spinlock_t lock; /* serializes caps */ + struct scx_sched *sch; /* backpointer */ + /* + * Per-cap cmask, inline via TRAILING_OVERLAP so cmask.bits[] overlaps + * the trailing _bits[] storage. Access as &caps[i].cmask. + */ + TRAILING_OVERLAP(struct scx_cmask, cmask, bits, + u64 _bits[SCX_CMASK_NR_WORDS(SCX_CID_SHARD_MAX_CPUS)]; + ) caps[__SCX_NR_CAPS]; }; #endif diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 017225ad492f..638b3744854b 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -122,7 +122,21 @@ void scx_free_pshards(struct scx_sched *sch) static struct scx_pshard *alloc_pshard(struct scx_sched *sch, s32 shard_idx, s32 node) { - return kzalloc_node(sizeof(struct scx_pshard), GFP_KERNEL, node); + const struct scx_cid_shard *shard = &scx_cid_shard_ranges[shard_idx]; + struct scx_pshard *pshard; + s32 i; + + pshard = kzalloc_node(sizeof(*pshard), GFP_KERNEL, node); + if (!pshard) + return NULL; + + raw_spin_lock_init(&pshard->lock); + pshard->sch = sch; + + for (i = 0; i < __SCX_NR_CAPS; i++) + scx_cmask_init(&pshard->caps[i].cmask, shard->base_cid, shard->nr_cids); + + return pshard; } s32 scx_alloc_pshards(struct scx_sched *sch) @@ -158,6 +172,22 @@ s32 scx_alloc_pshards(struct scx_sched *sch) return 0; } +/* + * Seed the root's caps fully. Root owns all cids on all caps at enable time. + * Children acquire caps via scx_bpf_sub_grant(). + */ +void scx_init_root_caps(struct scx_sched *sch) +{ + s32 si, i; + + for (si = 0; si < sch->nr_pshards; si++) { + struct scx_pshard *ps = sch->pshard[si]; + + for (i = 0; i < __SCX_NR_CAPS; i++) + scx_cmask_fill(&ps->caps[i].cmask); + } +} + static DECLARE_WAIT_QUEUE_HEAD(scx_unlink_waitq); void drain_descendants(struct scx_sched *sch) @@ -446,6 +476,31 @@ void scx_sub_enable_workfn(struct kthread_work *work) goto out_unlock; } + /* + * Validate before scx_link_sched() publishes @sch, so an invalid sub + * never becomes visible with an unallocated pshard. + */ + ret = scx_validate_ops(sch, ops); + if (ret) + goto err_disable; + + /* + * Allocate pshard[] before scx_link_sched() publishes @sch into the + * parent's RCU children list. A concurrent revoke walking the tree + * would otherwise dereference sch->pshard[si] while it's still NULL. + * Unlike the root path, the cid shard layout is stable at this point. + * + * scx_alloc_pshards() skips allocation when @sch's arena pool isn't + * initialized, so scx_arena_pool_init() must run first. + */ + ret = scx_arena_pool_init(sch); + if (ret) + goto err_disable; + + ret = scx_alloc_pshards(sch); + if (ret) + goto err_disable; + ret = scx_link_sched(sch); if (ret) goto err_disable; @@ -470,17 +525,10 @@ void scx_sub_enable_workfn(struct kthread_work *work) sch->exit_info->flags |= SCX_EFLAG_INITIALIZED; } - ret = scx_arena_pool_init(sch); - if (ret) - goto err_disable; - ret = scx_set_cmask_scratch_alloc(sch); if (ret) goto err_disable; - if (scx_validate_ops(sch, ops)) - goto err_disable; - struct scx_sub_attach_args sub_attach_args = { .ops = &sch->ops, .cgroup_path = sch->cgrp_path, @@ -760,6 +808,284 @@ __bpf_kfunc bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux * true); } +/* Validate common inputs. On success, *parent_out and *child_out are set. */ +static s32 sub_cap_preamble(u64 cgroup_id, u64 caps, const struct bpf_prog_aux *aux, + struct scx_sched **parent_out, struct scx_sched **child_out) +{ + struct scx_sched *parent, *child; + + parent = scx_prog_sched(aux); + if (unlikely(!parent)) + return -ENODEV; + + if (!scx_is_cid_type()) { + scx_error(parent, "sub-cap kfuncs require a cid-form scheduler"); + return -EOPNOTSUPP; + } + + child = scx_find_sub_sched(cgroup_id); + if (unlikely(!child)) + return -ENODEV; + + if (unlikely(scx_parent(child) != parent)) { + scx_error(parent, "%s: sub-%llu is not a direct child", + parent->cgrp_path, cgroup_id); + return -EINVAL; + } + + if (unlikely(caps & ~__SCX_CAP_ALL)) { + scx_error(parent, "invalid caps 0x%llx", caps); + return -EINVAL; + } + + *parent_out = parent; + *child_out = child; + return 0; +} + +/** + * scx_bpf_sub_grant - Grant @caps on @cmask__ign's cids to a direct child + * @cgroup_id: cgroup id of the direct child sub-sched + * @caps: bitmask of SCX_CAP_* to grant + * @cmask__ign: cid cmask to grant @caps on (arena pointer) + * @denied_out__ign: optional arena cmask accumulating refused cids + * @aux: implicit BPF argument + * + * A cid in @cmask__ign is granted to the child only if the parent holds every + * requested cap on it. Refused cids are OR'd into @denied_out__ign when + * provided. Refusals outside @denied_out__ign's range are not recorded. + * + * All-or-nothing keeps the caller-visible result binary per cid, so + * @denied_out__ign is one mask to interpret rather than a per-cap matrix. + * + * Return 0 on full success, -EPERM if any cid was refused, or a negative + * errno on other failures. + */ +__bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, + const struct scx_cmask *cmask__ign, + struct scx_cmask *denied_out__ign, + const struct bpf_prog_aux *aux) +{ + struct scx_cmask_ref ref, denied_ref; + struct scx_sched *parent, *child; + bool any_denied = false; + s32 si, ret; + + guard(irqsave)(); + + ret = sub_cap_preamble(cgroup_id, caps, aux, &parent, &child); + if (ret) + return ret; + + ret = scx_cmask_ref_init(parent, cmask__ign, &ref); + if (ret) { + scx_error(parent, "invalid cmask (%d)", ret); + return ret; + } + + if (denied_out__ign) { + ret = scx_cmask_ref_init(parent, denied_out__ign, &denied_ref); + if (ret) { + scx_error(parent, "invalid denied_out (%d)", ret); + return ret; + } + } + + /* apply the grant one shard at a time */ + for (si = ref.shard_first; si < ref.shard_end; si++) { + SCX_CMASK_DEFINE_SHARD(slice, 0, SCX_CID_SHARD_MAX_CPUS); + struct scx_pshard *pps = parent->pshard[si]; + struct scx_pshard *cps = child->pshard[si]; + u32 cap_bit; + + scx_cmask_ref_shard(&ref, si, slice); + if (scx_cmask_empty(slice)) + continue; + + SCX_CMASK_DEFINE_SHARD(granted_cids, slice->base, slice->nr_cids); + scx_cmask_copy(granted_cids, slice); + + scoped_guard (raw_spinlock, &pps->lock) { + guard(raw_spinlock_nested)(&cps->lock); + + /* + * Narrow granted_cids to cids the parent holds every + * requested cap on. All-or-nothing per cid. + */ + scx_for_each_cap_bit(cap_bit, caps) + scx_cmask_and(granted_cids, &pps->caps[cap_bit].cmask); + + /* fold granted_cids into the child per requested cap */ + scx_for_each_cap_bit(cap_bit, caps) + scx_cmask_or(&cps->caps[cap_bit].cmask, granted_cids); + } + + /* record cids that didn't make it through into @denied_out */ + if (!scx_cmask_subset(slice, granted_cids)) { + any_denied = true; + if (denied_out__ign) { + SCX_CMASK_DEFINE_SHARD(denied, slice->base, slice->nr_cids); + + scx_cmask_copy(denied, slice); + scx_cmask_andnot(denied, granted_cids); + scx_cmask_ref_or(&denied_ref, denied); + } + } + } + return any_denied ? -EPERM : 0; +} + +/** + * scx_bpf_sub_revoke - Revoke @caps on @cmask__ign's cids from @child + * @cgroup_id: cgroup id of the direct child sub-sched + * @caps: bitmask of SCX_CAP_* to revoke + * @cmask__ign: cid cmask to revoke @caps on (arena pointer) + * @aux: implicit BPF argument + * + * Clear @caps bits on @cmask__ign from the child named by @cgroup_id and all + * its descendants. The origin parent's pshard lock is held across the subtree + * walk so a concurrent grant from the origin parent observes the revoked + * state. + */ +__bpf_kfunc void scx_bpf_sub_revoke(u64 cgroup_id, u64 caps, + const struct scx_cmask *cmask__ign, + const struct bpf_prog_aux *aux) +{ + struct scx_cmask_ref ref; + struct scx_sched *parent, *child, *pos; + s32 si, ret; + + guard(irqsave)(); + + if (sub_cap_preamble(cgroup_id, caps, aux, &parent, &child)) + return; + + ret = scx_cmask_ref_init(parent, cmask__ign, &ref); + if (ret) { + scx_error(parent, "invalid cmask (%d)", ret); + return; + } + + /* per-shard, walk child's subtree and clear @caps */ + for (si = ref.shard_first; si < ref.shard_end; si++) { + SCX_CMASK_DEFINE_SHARD(slice, 0, SCX_CID_SHARD_MAX_CPUS); + + scx_cmask_ref_shard(&ref, si, slice); + if (scx_cmask_empty(slice)) + continue; + + /* + * Pre-order with subtree skip: a descendant that cleared + * nothing means no descendant of it can hold @caps on these + * cids either. + */ + guard(raw_spinlock)(&parent->pshard[si]->lock); + pos = scx_next_descendant_pre(NULL, child); + while (pos) { + struct scx_pshard *ps = pos->pshard[si]; + u64 revoked_caps = 0; + u32 cap_bit; + + scoped_guard (raw_spinlock_nested, &ps->lock) { + scx_for_each_cap_bit(cap_bit, caps) { + struct scx_cmask *cm = &ps->caps[cap_bit].cmask; + + if (!scx_cmask_intersects(cm, slice)) + continue; + scx_cmask_andnot(cm, slice); + revoked_caps |= BIT_U64(cap_bit); + } + } + + if (revoked_caps) + pos = scx_next_descendant_pre(pos, child); + else + pos = scx_skip_subtree_pre(pos, child); + } + } +} + +/** + * scx_bpf_sub_caps - Read self's or a direct child's cap cmasks + * @cgroup_id: 0 for self, or a direct child's cgroup id + * @caps: one or more SCX_CAP_* bits + * @out__ign: arena cmask to receive the union of @caps within its range + * @aux: implicit BPF argument + * + * Read the cap cmasks granted on each cid for self (@cgroup_id 0) or a direct + * child - the literal granted set. A sched can read only itself or a direct + * child. + * + * Return 0, -ENODEV if @cgroup_id names no direct child, or -EINVAL on bad + * inputs. + */ +__bpf_kfunc s32 scx_bpf_sub_caps(u64 cgroup_id, u64 caps, struct scx_cmask *out__ign, + const struct bpf_prog_aux *aux) +{ + struct scx_cmask_ref ref; + struct scx_sched *sch, *target; + struct scx_pshard **pshard; + s32 si, ret; + + guard(irqsave)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return -ENODEV; + + if (!scx_is_cid_type()) { + scx_error(sch, "sub-cap kfuncs require a cid-form scheduler"); + return -EOPNOTSUPP; + } + + if (unlikely(caps & ~__SCX_CAP_ALL)) { + scx_error(sch, "invalid caps 0x%llx", caps); + return -EINVAL; + } + + /* @cgroup_id 0 reads self, otherwise a direct child */ + if (cgroup_id) { + target = scx_find_sub_sched(cgroup_id); + if (unlikely(!target)) + return -ENODEV; + if (unlikely(scx_parent(target) != sch)) { + scx_error(sch, "%s: sub-%llu is not a direct child", + sch->cgrp_path, cgroup_id); + return -EINVAL; + } + } else { + target = sch; + } + + /* + * The target's caps storage may not be set up yet (e.g. a self-read + * during ops.init_cids()). Pairs with the publish in + * scx_alloc_pshards(): a non-NULL pshard has every element set. + */ + pshard = READ_ONCE(target->pshard); + if (unlikely(!pshard)) { + scx_error(sch, "scx_bpf_sub_caps() called before caps storage is initialized"); + return -ENODEV; + } + + ret = scx_cmask_ref_init(sch, out__ign, &ref); + if (ret) { + scx_error(sch, "invalid out (%d)", ret); + return ret; + } + + for (si = ref.shard_first; si < ref.shard_end; si++) { + const struct scx_cid_shard *shard = &scx_cid_shard_ranges[si]; + SCX_CMASK_DEFINE_SHARD(local_out, shard->base_cid, shard->nr_cids); + u32 cap_bit; + + scx_for_each_cap_bit(cap_bit, caps) + scx_cmask_or(local_out, &pshard[si]->caps[cap_bit].cmask); + scx_cmask_ref_copy(&ref, local_out); + } + return 0; +} + __bpf_kfunc_end_defs(); #endif /* CONFIG_EXT_SUB_SCHED */ diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index 2688087a0654..c8fd1a349ae6 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -27,6 +27,7 @@ void scx_sub_enable_workfn(struct kthread_work *work); bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux *aux); void scx_free_pshards(struct scx_sched *sch); s32 scx_alloc_pshards(struct scx_sched *sch); +void scx_init_root_caps(struct scx_sched *sch); static inline const char *sch_cgrp_path(struct scx_sched *sch) { @@ -45,6 +46,7 @@ static inline void drain_descendants(struct scx_sched *sch) { } static inline void scx_sub_disable(struct scx_sched *sch) { } static inline void scx_free_pshards(struct scx_sched *sch) {} static inline s32 scx_alloc_pshards(struct scx_sched *sch) { return 0; } +static inline void scx_init_root_caps(struct scx_sched *sch) {} #endif /* CONFIG_EXT_SUB_SCHED */ diff --git a/tools/sched_ext/include/scx/common.bpf.h b/tools/sched_ext/include/scx/common.bpf.h index e7b3ba491c5e..09c21602b2ed 100644 --- a/tools/sched_ext/include/scx/common.bpf.h +++ b/tools/sched_ext/include/scx/common.bpf.h @@ -114,6 +114,12 @@ u32 scx_bpf_cidperf_cap(s32 cid) __ksym __weak; u32 scx_bpf_cidperf_cur(s32 cid) __ksym __weak; void scx_bpf_cidperf_set(s32 cid, u32 perf) __ksym __weak; +/* sub-scheduler cap control, scx_bpf_sub_caps() cgroup_id 0 == self */ +s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, const struct scx_cmask *cmask, + struct scx_cmask *denied) __ksym __weak; +void scx_bpf_sub_revoke(u64 cgroup_id, u64 caps, const struct scx_cmask *cmask) __ksym __weak; +s32 scx_bpf_sub_caps(u64 cgroup_id, u64 caps, struct scx_cmask *out) __ksym __weak; + /* * Use the following as @it__iter when calling scx_bpf_dsq_move[_vtime]() from * within bpf_for_each() loops. -- cgit v1.2.3 From 5f2a9a4c2e6c827eafa8491764125934c5466a84 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 13 Jul 2026 22:18:43 -1000 Subject: sched_ext: Add coalescing sub_caps_updated() notifier for sub-schedulers Wire up ops_cid.sub_caps_updated() to notify sub-scheds of cap changes. Three constraints shape the design: 1. Static memory. Deliveries use a fixed-size buffer, both for runtime efficiency and so notifications can't be lost under memory pressure. 2. High-frequency updates. Grant/revoke can mutate caps in bursts, and the notifier path must absorb that without amplifying it. 3. Recursive grant/revoke from the callback. A child receiving a notification can call grant/revoke on its own children, which can cascade recursively down its subtree. (1) and (2) lead to coalescing into a fixed payload. Each delivery carries a single (cmask, caps) pair covering every change since the previous one. Direction (set vs cleared) isn't encoded as it doesn't fit in the fixed-size summary. The callback queries scx_bpf_sub_caps() for current state. Only one delivery is in flight per shard. Further changes fold into the same buffer and ship as the next callback, so a shard's callbacks fire in order. (3) leads to deferred delivery. Events accumulate during grant/revoke and are delivered after the shard lock is released. v2: - Request a private stack for ops.sub_caps_updated(). (sashiko AI) - Build cmask_arena_out via scx_cmask_ref, not by re-reading its header. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 13 +++- kernel/sched/ext/internal.h | 71 ++++++++++++++++++ kernel/sched/ext/sub.c | 178 ++++++++++++++++++++++++++++++++++++++++++-- kernel/sched/ext/sub.h | 1 + 4 files changed, 254 insertions(+), 9 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 3cc7e5939a71..7599ba9c23ea 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -7320,6 +7320,11 @@ static int bpf_scx_check_member(const struct btf_type *t, case offsetof(struct sched_ext_ops, dispatch): prog->aux->priv_stack_requested = true; prog->aux->recursion_detected = scx_pstack_recursion_on_dispatch; + break; + case offsetof(struct sched_ext_ops, sub_caps_updated): + prog->aux->priv_stack_requested = true; + prog->aux->recursion_detected = scx_pstack_recursion_on_caps_updated; + break; } #endif /* CONFIG_EXT_SUB_SCHED */ @@ -7521,11 +7526,13 @@ static struct bpf_struct_ops bpf_sched_ext_ops = { /* * cid-form cfi stubs. Stubs whose signatures match the cpu-form (param types - * identical, only param names differ across structs) are reused. Only - * set_cmask needs a fresh stub since the second argument type differs. + * identical, only param names differ across structs) are reused. Some need + * fresh stubs, set_cmask due to an argument type difference and the sub-sched + * notifiers because no cpu-form stub exists to reuse. */ static void sched_ext_ops_cid__set_cmask(struct task_struct *p, const struct scx_cmask *cmask) {} +static void sched_ext_ops__sub_caps_updated(const struct scx_cmask *cmask, u64 caps) {} static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = { .select_cid = sched_ext_ops__select_cpu, @@ -7558,6 +7565,7 @@ static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = { #endif .sub_attach = sched_ext_ops__sub_attach, .sub_detach = sched_ext_ops__sub_detach, + .sub_caps_updated = sched_ext_ops__sub_caps_updated, .cid_online = sched_ext_ops__cpu_online, .cid_offline = sched_ext_ops__cpu_offline, .init_cids = sched_ext_ops__init_cids, @@ -10012,6 +10020,7 @@ static int __init scx_init(void) CID_OFFSET_MATCH(dump_task, dump_task); CID_OFFSET_MATCH(sub_attach, sub_attach); CID_OFFSET_MATCH(sub_detach, sub_detach); + CID_OFFSET_MATCH(sub_caps_updated, sub_caps_updated); CID_OFFSET_MATCH(init_cids, init_cids); CID_OFFSET_MATCH(init, init); CID_OFFSET_MATCH(exit, exit); diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 1686aa823932..2bbe93eb86fa 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -757,6 +757,25 @@ struct sched_ext_ops { */ void (*sub_detach)(struct scx_sub_detach_args *args); + /** + * @sub_caps_updated: Caps on this sub-sched's shard changed + * @cmask: cids whose caps changed (cmask->base identifies the shard) + * @caps: SCX_CAP_* that changed + * + * Invoked after grant or revoke modifies caps on a shard. There can be + * only one in-flight invocation per shard. @cmask and @caps coalesce + * all changes since the last delivery. Direction (set vs cleared) isn't + * encoded. Query current state with scx_bpf_sub_caps(). + * + * Delivered asynchronously after the change is recorded, and may run + * before it takes effect on any given cpu. Use it to track which caps + * the sub-sched holds and propagate to its own children, not to decide + * if a task can run on a cpu now. + * + * May call scx_bpf_sub_grant() / scx_bpf_sub_revoke() on children. + */ + void (*sub_caps_updated)(const struct scx_cmask *cmask, u64 caps); + /* * All online ops must come before ops.cpu_online(). */ @@ -977,6 +996,7 @@ struct sched_ext_ops_cid { #endif /* CONFIG_EXT_GROUP_SCHED */ s32 (*sub_attach)(struct scx_sub_attach_args *args); void (*sub_detach)(struct scx_sub_detach_args *args); + void (*sub_caps_updated)(const struct scx_cmask *cmask, u64 caps); void (*cid_online)(s32 cid); void (*cid_offline)(s32 cid); s32 (*init_cids)(void); @@ -1224,9 +1244,51 @@ enum scx_cap_flags { __caps && ((cap_bit) = __ffs64(__caps), true); \ __caps &= __caps - 1) +/* + * Sub-cap update notifier. + * + * ops_cid.sub_caps_updated() notifies sub-scheds when their cap state changes + * so they can refresh internal state without polling scx_bpf_sub_caps() per + * enqueue. + * + * Three constraints shape the design: + * + * 1. Static memory. Deliveries use a fixed-size buffer, both for runtime + * efficiency and so notifications can't be lost under memory pressure. + * + * 2. High-frequency updates. Grant/revoke can mutate caps in bursts, and the + * notifier path must absorb that without amplifying it. + * + * 3. Recursive grant/revoke from the callback. A child receiving a + * notification can call grant/revoke on its own children, which can + * cascade recursively down its subtree. + * + * (1) and (2) lead to coalescing into a fixed payload. Each delivery carries a + * single (cmask, caps) pair covering every change since the previous one. + * Direction (set vs cleared) isn't encoded as it doesn't fit in the fixed-size + * summary. The callback queries scx_bpf_sub_caps() for current state. Only one + * delivery is in flight per shard. Further changes fold into the same buffer + * and ship as the next callback, so a shard's callbacks fire in order. + * + * (3) leads to deferred delivery. Events accumulate during grant/revoke and are + * delivered after the shard lock is released. + */ +struct scx_caps_updated { + raw_spinlock_t lock; + u64 caps; + struct scx_cmask *cmask_arena_out; + struct list_head node_in_flight; + /* Kernel-side accumulator. Access as &cu->cmask. */ + TRAILING_OVERLAP(struct scx_cmask, cmask, bits, + u64 _bits[SCX_CMASK_NR_WORDS(SCX_CID_SHARD_MAX_CPUS)]; + ); +}; + struct scx_pshard { raw_spinlock_t lock; /* serializes caps */ struct scx_sched *sch; /* backpointer */ + struct scx_caps_updated caps_updated; + /* * Per-cap cmask, inline via TRAILING_OVERLAP so cmask.bits[] overlaps * the trailing _bits[] storage. Access as &caps[i].cmask. @@ -1234,6 +1296,15 @@ struct scx_pshard { TRAILING_OVERLAP(struct scx_cmask, cmask, bits, u64 _bits[SCX_CMASK_NR_WORDS(SCX_CID_SHARD_MAX_CPUS)]; ) caps[__SCX_NR_CAPS]; + + /* + * Shard geometry captured at alloc. cmask_arena_out's own header is + * bpf-writable and the live shard range can change before the + * rcu-deferred free, so re-init and size cmask_arena_out from these + * trusted copies instead. + */ + u32 base; + u32 nr_cids; }; #endif diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 638b3744854b..07972f2972f3 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -106,6 +106,15 @@ void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch) static void free_pshard(struct scx_pshard *pshard) { + struct scx_caps_updated *cu; + + if (!pshard) + return; + cu = &pshard->caps_updated; + if (cu->cmask_arena_out) + scx_arena_free(pshard->sch, cu->cmask_arena_out, + struct_size_t(struct scx_cmask, bits, + SCX_CMASK_NR_WORDS(pshard->nr_cids))); kfree(pshard); } @@ -123,7 +132,10 @@ void scx_free_pshards(struct scx_sched *sch) static struct scx_pshard *alloc_pshard(struct scx_sched *sch, s32 shard_idx, s32 node) { const struct scx_cid_shard *shard = &scx_cid_shard_ranges[shard_idx]; + size_t cmask_size = struct_size_t(struct scx_cmask, bits, + SCX_CMASK_NR_WORDS(shard->nr_cids)); struct scx_pshard *pshard; + struct scx_caps_updated *cu; s32 i; pshard = kzalloc_node(sizeof(*pshard), GFP_KERNEL, node); @@ -132,10 +144,25 @@ static struct scx_pshard *alloc_pshard(struct scx_sched *sch, s32 shard_idx, s32 raw_spin_lock_init(&pshard->lock); pshard->sch = sch; + pshard->base = shard->base_cid; + pshard->nr_cids = shard->nr_cids; for (i = 0; i < __SCX_NR_CAPS; i++) scx_cmask_init(&pshard->caps[i].cmask, shard->base_cid, shard->nr_cids); + cu = &pshard->caps_updated; + raw_spin_lock_init(&cu->lock); + INIT_LIST_HEAD(&cu->node_in_flight); + __scx_cmask_init(&cu->cmask, shard->base_cid, shard->nr_cids, SCX_CID_SHARD_MAX_CPUS); + + cu->cmask_arena_out = scx_arena_alloc(sch, cmask_size); + if (!cu->cmask_arena_out) { + free_pshard(pshard); + return NULL; + } + + scx_cmask_init(cu->cmask_arena_out, shard->base_cid, shard->nr_cids); + return pshard; } @@ -188,6 +215,88 @@ void scx_init_root_caps(struct scx_sched *sch) } } +/* record a caps change, see struct scx_caps_updated */ +static void caps_updated_record(struct scx_pshard *ps, const struct scx_cmask *cids, u64 caps, + struct list_head *to_deliver) +{ + struct scx_caps_updated *cu = &ps->caps_updated; + + guard(raw_spinlock)(&cu->lock); + scx_cmask_or(&cu->cmask, cids); + cu->caps |= caps; + if (list_empty(&cu->node_in_flight)) + list_add_tail(&cu->node_in_flight, to_deliver); +} + +/* deliver queued caps_updated callbacks, see struct scx_caps_updated */ +static void caps_updated_deliver(struct list_head *to_deliver) +{ + struct scx_caps_updated *cu, *tmp; + + list_for_each_entry_safe(cu, tmp, to_deliver, node_in_flight) { + struct scx_pshard *ps = container_of(cu, struct scx_pshard, caps_updated); + struct scx_sched *sch = ps->sch; + + while (true) { + u64 caps = 0; + + /* + * During enable, has_op is set after ops.sub_attach(), + * so !has_op means the op is absent or the sched isn't + * live yet - e.g. caps grant from ops.sub_attach(). + * Either way don't consume - leave for + * scx_sub_seed_caps() to deliver once live. + */ + scoped_guard (raw_spinlock, &cu->lock) { + if (cu->caps && SCX_HAS_OP(sch, sub_caps_updated) && + likely(!READ_ONCE(sch->aborting))) { + struct scx_cmask_ref ref; + + caps = cu->caps; + scx_cmask_ref_init_kern(sch, cu->cmask_arena_out, + ps->base, ps->nr_cids, &ref); + scx_cmask_ref_copy(&ref, &cu->cmask); + scx_cmask_clear(&cu->cmask); + cu->caps = 0; + } else { + list_del_init(&cu->node_in_flight); + } + } + if (!caps) + break; + + /* caps != 0 only when deliverable (has_op, above) */ + SCX_CALL_OP(sch, sub_caps_updated, NULL, + scx_kaddr_to_arena(sch, cu->cmask_arena_out), + caps); + } + } +} + +/* + * Deliver caps owed to @sch that couldn't be delivered earlier (e.g. a grant + * taken during its sub_attach(), before has_op was set). Called once @sch is + * enabled. + */ +static void scx_sub_seed_caps(struct scx_sched *sch) +{ + LIST_HEAD(to_deliver); + s32 si; + + guard(irqsave)(); + + for (si = 0; si < sch->nr_pshards; si++) { + struct scx_pshard *ps = sch->pshard[si]; + struct scx_caps_updated *cu = &ps->caps_updated; + + scoped_guard (raw_spinlock, &cu->lock) { + if (cu->caps && list_empty(&cu->node_in_flight)) + list_add_tail(&cu->node_in_flight, &to_deliver); + } + } + caps_updated_deliver(&to_deliver); +} + static DECLARE_WAIT_QUEUE_HEAD(scx_unlink_waitq); void drain_descendants(struct scx_sched *sch) @@ -671,6 +780,9 @@ void scx_sub_enable_workfn(struct kthread_work *work) scx_bypass(sch, false); + /* @sch is enabled; deliver any caps owed since its sub_attach() */ + scx_sub_seed_caps(sch); + pr_info("sched_ext: BPF sub-scheduler \"%s\" enabled\n", sch->ops.name); kobject_uevent(&sch->kobj, KOBJ_ADD); ret = 0; @@ -757,7 +869,7 @@ static s32 __init scx_cgroup_lifetime_notifier_init(void) } core_initcall(scx_cgroup_lifetime_notifier_init); -void scx_pstack_recursion_on_dispatch(struct bpf_prog *prog) +static void scx_pstack_recursion(struct bpf_prog *prog, const char *op) { struct scx_sched *sch; @@ -766,7 +878,17 @@ void scx_pstack_recursion_on_dispatch(struct bpf_prog *prog) if (unlikely(!sch)) return; - scx_error(sch, "dispatch recursion detected"); + scx_error(sch, "%s recursion detected", op); +} + +void scx_pstack_recursion_on_dispatch(struct bpf_prog *prog) +{ + scx_pstack_recursion(prog, "dispatch"); +} + +void scx_pstack_recursion_on_caps_updated(struct bpf_prog *prog) +{ + scx_pstack_recursion(prog, "sub_caps_updated"); } __bpf_kfunc_start_defs(); @@ -869,6 +991,7 @@ __bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, struct scx_cmask_ref ref, denied_ref; struct scx_sched *parent, *child; bool any_denied = false; + LIST_HEAD(to_deliver); s32 si, ret; guard(irqsave)(); @@ -896,6 +1019,7 @@ __bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, SCX_CMASK_DEFINE_SHARD(slice, 0, SCX_CID_SHARD_MAX_CPUS); struct scx_pshard *pps = parent->pshard[si]; struct scx_pshard *cps = child->pshard[si]; + u64 granted_caps = 0; u32 cap_bit; scx_cmask_ref_shard(&ref, si, slice); @@ -903,6 +1027,9 @@ __bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, continue; SCX_CMASK_DEFINE_SHARD(granted_cids, slice->base, slice->nr_cids); + SCX_CMASK_DEFINE_SHARD(changed_cids, slice->base, slice->nr_cids); + SCX_CMASK_DEFINE_SHARD(delta, slice->base, slice->nr_cids); + scx_cmask_copy(granted_cids, slice); scoped_guard (raw_spinlock, &pps->lock) { @@ -915,9 +1042,26 @@ __bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, scx_for_each_cap_bit(cap_bit, caps) scx_cmask_and(granted_cids, &pps->caps[cap_bit].cmask); - /* fold granted_cids into the child per requested cap */ - scx_for_each_cap_bit(cap_bit, caps) - scx_cmask_or(&cps->caps[cap_bit].cmask, granted_cids); + /* + * For each requested cap, fold the newly-set cids into + * the child and accumulate the delta. + */ + scx_for_each_cap_bit(cap_bit, caps) { + struct scx_cmask *ccm = &cps->caps[cap_bit].cmask; + + scx_cmask_copy(delta, granted_cids); + scx_cmask_andnot(delta, ccm); + if (scx_cmask_empty(delta)) + continue; + + scx_cmask_or(ccm, delta); + scx_cmask_or(changed_cids, delta); + granted_caps |= BIT_U64(cap_bit); + } + + if (granted_caps) + caps_updated_record(cps, changed_cids, granted_caps, + &to_deliver); } /* record cids that didn't make it through into @denied_out */ @@ -932,6 +1076,9 @@ __bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, } } } + + caps_updated_deliver(&to_deliver); + return any_denied ? -EPERM : 0; } @@ -953,6 +1100,7 @@ __bpf_kfunc void scx_bpf_sub_revoke(u64 cgroup_id, u64 caps, { struct scx_cmask_ref ref; struct scx_sched *parent, *child, *pos; + LIST_HEAD(to_deliver); s32 si, ret; guard(irqsave)(); @@ -983,18 +1131,32 @@ __bpf_kfunc void scx_bpf_sub_revoke(u64 cgroup_id, u64 caps, pos = scx_next_descendant_pre(NULL, child); while (pos) { struct scx_pshard *ps = pos->pshard[si]; + SCX_CMASK_DEFINE_SHARD(changed_cids, slice->base, slice->nr_cids); + SCX_CMASK_DEFINE_SHARD(delta, slice->base, slice->nr_cids); u64 revoked_caps = 0; u32 cap_bit; scoped_guard (raw_spinlock_nested, &ps->lock) { + /* + * For each cap, clear lost cids and accumulate + * the per-cap diff for notification. + */ scx_for_each_cap_bit(cap_bit, caps) { struct scx_cmask *cm = &ps->caps[cap_bit].cmask; - if (!scx_cmask_intersects(cm, slice)) + scx_cmask_copy(delta, cm); + scx_cmask_and(delta, slice); + if (scx_cmask_empty(delta)) continue; - scx_cmask_andnot(cm, slice); + + scx_cmask_andnot(cm, delta); + scx_cmask_or(changed_cids, delta); revoked_caps |= BIT_U64(cap_bit); } + + if (revoked_caps) + caps_updated_record(ps, changed_cids, revoked_caps, + &to_deliver); } if (revoked_caps) @@ -1003,6 +1165,8 @@ __bpf_kfunc void scx_bpf_sub_revoke(u64 cgroup_id, u64 caps, pos = scx_skip_subtree_pre(pos, child); } } + + caps_updated_deliver(&to_deliver); } /** diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index c8fd1a349ae6..822118b91edc 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -21,6 +21,7 @@ void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch); struct cgroup *sch_cgroup(struct scx_sched *sch); void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch); void scx_pstack_recursion_on_dispatch(struct bpf_prog *prog); +void scx_pstack_recursion_on_caps_updated(struct bpf_prog *prog); void drain_descendants(struct scx_sched *sch); void scx_sub_disable(struct scx_sched *sch); void scx_sub_enable_workfn(struct kthread_work *work); -- cgit v1.2.3 From 56fdc35b7471639bae66f2ce3885e59a4543b9dd Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 13 Jul 2026 22:18:43 -1000 Subject: sched_ext: Maintain per-cpu effective cap copies for single-read checks Checking a sched's caps on a cid would need to test several cap bits against caps[] to account for implied caps. Also, caps[] modifications aren't synchronized against scheduling operations on each cpu, which can lead to awkward race conditions. Collect them per cpu instead. caps[] under pshard->lock stays the target configuration. scx_sched_pcpu->ecaps is added, the transposed effective copy: the set of cap bits the sched holds on that cpu which can be accessed with a single read. It is stable under the rq lock. It can also be read locklessly with READ_ONCE(). Grant and revoke only mutate caps[]. They queue a sync request on the target cpu's rq->scx.ecaps_to_sync and kick it, and the cpu recomputes the queued scheds' ecaps from caps[] in balance_one() under its own rq lock. A dying sched runs the sync directly to retire its queued request before freeing. As held references can defer the freeing past the enclosing root scheduler's lifetime, root enable discards leftover sync requests before going live. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 10 ++++ kernel/sched/ext/internal.h | 21 ++++++- kernel/sched/ext/sub.c | 133 +++++++++++++++++++++++++++++++++++++++++++- kernel/sched/ext/sub.h | 16 ++++++ kernel/sched/sched.h | 3 + 5 files changed, 180 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 7599ba9c23ea..f683e93d0ab3 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -2607,6 +2607,8 @@ static int balance_one(struct rq *rq, struct task_struct *prev) rq->scx.flags |= SCX_RQ_IN_BALANCE; rq->scx.flags &= ~SCX_RQ_BAL_KEEP; + scx_process_sync_ecaps(rq); + if ((sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT) && unlikely(rq->scx.cpu_released)) { /* @@ -4664,6 +4666,9 @@ static void scx_sched_free_rcu_work(struct work_struct *work) */ WARN_ON_ONCE(!list_empty(&pcpu->deferred_reenq_local.node)); + /* flush the queued ecaps syncs */ + scx_discard_ecaps_to_sync(cpu, pcpu); + /* * Bypass blocks new kicks. Flush the kick irq_work so this * pcpu's to_kick_node is off the list before it is freed. @@ -6428,6 +6433,9 @@ struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, node = cpu_to_node(cpu); pcpu->sch = sch; INIT_LIST_HEAD(&pcpu->deferred_reenq_local.node); +#ifdef CONFIG_EXT_SUB_SCHED + init_llist_node(&pcpu->ecaps_to_sync_node); +#endif INIT_LIST_HEAD(&pcpu->to_kick_node); if (!zalloc_cpumask_var_node(&pcpu->cpus_to_kick, GFP_KERNEL, node) || !zalloc_cpumask_var_node(&pcpu->cpus_to_kick_if_idle, GFP_KERNEL, node) || @@ -6772,6 +6780,8 @@ static void scx_root_enable_workfn(struct kthread_work *work) rq->scx.cpuperf_target = SCX_CPUPERF_ONE; } + scx_discard_stale_ecaps_syncs(); + /* * Keep CPUs stable during enable so that the BPF scheduler can track * online CPUs by watching ->on/offline_cpu() after ->init(). diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 2bbe93eb86fa..f512475d1c03 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1182,6 +1182,24 @@ struct scx_sched_pcpu { cpumask_var_t cpus_to_wait; struct list_head to_kick_node; +#ifdef CONFIG_EXT_SUB_SCHED + /* + * pshard->caps[cap_bit] is the set of cids the sched holds that one + * cap on. ecaps is its transpose: the set of SCX_CAP_* bits the sched + * holds on this cpu, collected so that the hot-path check is a single + * read. + * + * While pshard->caps[] under pshard->lock is the target configuration, + * ecaps is the effective copy owned by the cpu. It is written under the + * rq lock while processing rq->ecaps_to_sync. Can also be read with + * READ_ONCE() outside rq lock. + * + * See queue_sync_ecaps() and scx_process_sync_ecaps(). + */ + u64 ecaps; + struct llist_node ecaps_to_sync_node; +#endif + /* * The event counters are in a per-CPU variable to minimize the * accounting overhead. A system-wide view on the event counter is @@ -1291,7 +1309,8 @@ struct scx_pshard { /* * Per-cap cmask, inline via TRAILING_OVERLAP so cmask.bits[] overlaps - * the trailing _bits[] storage. Access as &caps[i].cmask. + * the trailing _bits[] storage. Access as &caps[i].cmask. See + * scx_sched_pcpu->ecaps. */ TRAILING_OVERLAP(struct scx_cmask, cmask, bits, u64 _bits[SCX_CMASK_NR_WORDS(SCX_CID_SHARD_MAX_CPUS)]; diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 07972f2972f3..136d8dabe4ad 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -297,6 +297,125 @@ static void scx_sub_seed_caps(struct scx_sched *sch) caps_updated_deliver(&to_deliver); } +static u64 calc_effective_caps(struct scx_pshard *ps, s32 cid) +{ + u64 ecaps = 0; + u32 cap_bit; + + for (cap_bit = 0; cap_bit < __SCX_NR_CAPS; cap_bit++) + if (scx_cmask_test(cid, &ps->caps[cap_bit].cmask)) + ecaps |= BIT_U64(cap_bit) | scx_caps_implied(BIT_U64(cap_bit)); + return ecaps; +} + +/** + * queue_sync_ecaps - Queue ecaps update for a (sch, cid) pair + * @sch: sched to update + * @cid: cid to update + * + * Queue an ecaps update for @sch's @cid and kick the cpu so that it syncs in + * balance_one(). + */ +static void queue_sync_ecaps(struct scx_sched *sch, s32 cid) +{ + s32 cpu = __scx_cid_to_cpu(cid); + struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); + + /* + * Pairs with smp_mb() in scx_process_sync_ecaps(). Either the check + * below sees the node off the list and queues it, or the in-flight sync + * sees the caps[] update made before this call. + */ + smp_mb(); + + /* @cid's pshard->lock excludes concurrent queueing attempts */ + if (llist_on_list(&pcpu->ecaps_to_sync_node)) + return; + if (llist_add(&pcpu->ecaps_to_sync_node, &cpu_rq(cpu)->scx.ecaps_to_sync)) + scx_kick_cpu(scx_root, cpu, 0); +} + +/* discard @rq's queued ecaps syncs */ +static void discard_queued_syncs(struct rq *rq) +{ + struct llist_node *pos, *tmp; + + lockdep_assert_rq_held(rq); + + llist_for_each_safe(pos, tmp, llist_del_all(&rq->scx.ecaps_to_sync)) + init_llist_node(pos); +} + +/** + * scx_process_sync_ecaps - Sync this cpu's ecaps to pshard->caps[] + * @rq: the cid's cpu rq + * + * pshard->caps[] is the target configuration. pcpu->ecaps is the effective + * transposed copy owned by the cid's cpu and written only here under @rq's + * lock. + */ +void scx_process_sync_ecaps(struct rq *rq) +{ + s32 cid = __scx_cpu_to_cid(cpu_of(rq)); + s32 shard = scx_cid_to_shard[cid]; + struct llist_node *batch, *pos, *tmp; + + lockdep_assert_rq_held(rq); + + if (likely(llist_empty(&rq->scx.ecaps_to_sync))) + return; + + batch = llist_del_all(&rq->scx.ecaps_to_sync); + llist_for_each_safe(pos, tmp, batch) { + struct scx_sched_pcpu *pcpu = + container_of(pos, struct scx_sched_pcpu, ecaps_to_sync_node); + struct scx_pshard *ps = pcpu->sch->pshard[shard]; + + init_llist_node(pos); + + /* pairs with smp_mb() in queue_sync_ecaps(), see there */ + smp_mb(); + + WRITE_ONCE(pcpu->ecaps, calc_effective_caps(ps, cid)); + } +} + +/* + * @pcpu's sched was unhashed before the grace period, so nothing new queues. + * Flush its pending sync so the pcpu can be freed. scx_process_sync_ecaps() + * takes nodes off the list before syncing and acquiring the rq lock waits for + * any in-flight walk. + */ +void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu) +{ + scoped_guard (rq_lock_irqsave, cpu_rq(cpu)) + scx_process_sync_ecaps(cpu_rq(cpu)); + + WARN_ON_ONCE(llist_on_list(&pcpu->ecaps_to_sync_node)); +} + +/** + * scx_discard_stale_ecaps_syncs - Discard ecaps syncs from earlier schedulers + * + * To be called during root enable before the scheduler goes live. An earlier + * root's sub-sched may not have gone through its RCU free path yet (e.g. a + * still-open link fd defers it) and can leave queued ecaps syncs behind. + * Processing them would decode the dead sched's pshards with the current cid + * layout. Discard them instead. The backing scx_sched_pcpu's are still + * allocated as the free path drains ecaps_to_sync_node before freeing. + */ +void scx_discard_stale_ecaps_syncs(void) +{ + s32 cpu; + + for_each_possible_cpu(cpu) { + struct rq *rq = cpu_rq(cpu); + + guard(rq_lock_irqsave)(rq); + discard_queued_syncs(rq); + } +} + static DECLARE_WAIT_QUEUE_HEAD(scx_unlink_waitq); void drain_descendants(struct scx_sched *sch) @@ -1059,9 +1178,14 @@ __bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, granted_caps |= BIT_U64(cap_bit); } - if (granted_caps) + if (granted_caps) { + s32 cid; + caps_updated_record(cps, changed_cids, granted_caps, &to_deliver); + scx_cmask_for_each_cid(cid, changed_cids) + queue_sync_ecaps(child, cid); + } } /* record cids that didn't make it through into @denied_out */ @@ -1154,9 +1278,14 @@ __bpf_kfunc void scx_bpf_sub_revoke(u64 cgroup_id, u64 caps, revoked_caps |= BIT_U64(cap_bit); } - if (revoked_caps) + if (revoked_caps) { + s32 cid; + caps_updated_record(ps, changed_cids, revoked_caps, &to_deliver); + scx_cmask_for_each_cid(cid, changed_cids) + queue_sync_ecaps(pos, cid); + } } if (revoked_caps) diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index 822118b91edc..9bf700df8c82 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -29,6 +29,9 @@ bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux *aux); void scx_free_pshards(struct scx_sched *sch); s32 scx_alloc_pshards(struct scx_sched *sch); void scx_init_root_caps(struct scx_sched *sch); +void scx_process_sync_ecaps(struct rq *rq); +void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu); +void scx_discard_stale_ecaps_syncs(void); static inline const char *sch_cgrp_path(struct scx_sched *sch) { @@ -48,6 +51,9 @@ static inline void scx_sub_disable(struct scx_sched *sch) { } static inline void scx_free_pshards(struct scx_sched *sch) {} static inline s32 scx_alloc_pshards(struct scx_sched *sch) { return 0; } static inline void scx_init_root_caps(struct scx_sched *sch) {} +static inline void scx_process_sync_ecaps(struct rq *rq) {} +static inline void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu) {} +static inline void scx_discard_stale_ecaps_syncs(void) {} #endif /* CONFIG_EXT_SUB_SCHED */ @@ -64,6 +70,16 @@ static inline void scx_init_root_caps(struct scx_sched *sch) {} for ((pos) = scx_next_descendant_pre(NULL, (root)); (pos); \ (pos) = scx_next_descendant_pre((pos), (root))) +#ifdef CONFIG_EXT_SUB_SCHED + +/* caps implied by holding @cap */ +static inline u64 scx_caps_implied(u64 cap) +{ + return 0; +} + +#endif /* CONFIG_EXT_SUB_SCHED */ + /* * One user of this function is scx_bpf_dispatch() which can be called * recursively as sub-sched dispatches nest. Always inline to reduce stack usage diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 9e41a23ef49e..72299c105487 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -806,6 +806,9 @@ struct scx_rq { u32 flags; u32 nr_immed; /* ENQ_IMMED tasks on local_dsq */ u64 clock; /* current per-rq clock -- see scx_bpf_now() */ +#ifdef CONFIG_EXT_SUB_SCHED + struct llist_head ecaps_to_sync; /* pending ecaps syncs */ +#endif cpumask_var_t cpus_to_sync; bool kick_sync_pending; unsigned long kick_sync; -- cgit v1.2.3 From b81a6c018cdeb91897696489694653c56fa60eb5 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 13 Jul 2026 22:18:43 -1000 Subject: sched_ext: Add sub_ecaps_updated() effective-cap change notifier A sub-scheduler that gains or loses effective caps on a cpu may want to act on it right away - e.g. place or preempt on a newly usable cpu. The existing ops.sub_caps_updated() doesn't fit as it is delivered asynchronously to scheduling operations and can arrive before the per-cpu effective caps go live. Add ops.sub_ecaps_updated(cid, before, after), a cid-form callback fired from scx_process_sync_ecaps() when a sub-sched's effective caps on a cid change. It runs in dispatch context so the sched can insert, kick or preempt on the cid directly. @before is the caps as of the last delivery. Cpu hotplug rides the same machinery. Going down zeroes each sched's ecaps on the cpu's cid, with queued syncs discarded at consumption while the cpu is inactive. Coming back up queues a sync for every sched. reported_ecaps is kept across the down/up cycle, so the resync fires the callback only if ownership actually changed while the cpu was down. v2: Compute cid below the active-cpu guard; discard queued syncs on !cpu_active(). (sashiko AI) Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 13 ++++- kernel/sched/ext/internal.h | 19 ++++++- kernel/sched/ext/sub.c | 129 ++++++++++++++++++++++++++++++++++++++++---- kernel/sched/ext/sub.h | 8 ++- 4 files changed, 154 insertions(+), 15 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index f683e93d0ab3..e96b0ced5a4d 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -2607,7 +2607,7 @@ static int balance_one(struct rq *rq, struct task_struct *prev) rq->scx.flags |= SCX_RQ_IN_BALANCE; rq->scx.flags &= ~SCX_RQ_BAL_KEEP; - scx_process_sync_ecaps(rq); + scx_process_sync_ecaps(rq, prev); if ((sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT) && unlikely(rq->scx.cpu_released)) { @@ -3146,6 +3146,11 @@ static void handle_hotplug(struct rq *rq, bool online) if (scx_enabled()) scx_idle_update_selcpu_topology(&sch->ops); + if (online) + scx_online_ecaps(rq); + else + scx_offline_ecaps(rq); + if (online && SCX_HAS_OP(sch, cpu_online)) SCX_CALL_OP(sch, cpu_online, NULL, scx_cpu_arg(cpu)); else if (!online && SCX_HAS_OP(sch, cpu_offline)) @@ -4666,7 +4671,7 @@ static void scx_sched_free_rcu_work(struct work_struct *work) */ WARN_ON_ONCE(!list_empty(&pcpu->deferred_reenq_local.node)); - /* flush the queued ecaps syncs */ + /* retire the queued ecaps syncs so the pcpu can be freed */ scx_discard_ecaps_to_sync(cpu, pcpu); /* @@ -7543,6 +7548,7 @@ static struct bpf_struct_ops bpf_sched_ext_ops = { static void sched_ext_ops_cid__set_cmask(struct task_struct *p, const struct scx_cmask *cmask) {} static void sched_ext_ops__sub_caps_updated(const struct scx_cmask *cmask, u64 caps) {} +static void sched_ext_ops__sub_ecaps_updated(s32 cid, u64 before, u64 after) {} static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = { .select_cid = sched_ext_ops__select_cpu, @@ -7576,6 +7582,7 @@ static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = { .sub_attach = sched_ext_ops__sub_attach, .sub_detach = sched_ext_ops__sub_detach, .sub_caps_updated = sched_ext_ops__sub_caps_updated, + .sub_ecaps_updated = sched_ext_ops__sub_ecaps_updated, .cid_online = sched_ext_ops__cpu_online, .cid_offline = sched_ext_ops__cpu_offline, .init_cids = sched_ext_ops__init_cids, @@ -9892,6 +9899,7 @@ static const u32 scx_kf_allow_flags[] = { #endif /* CONFIG_EXT_GROUP_SCHED */ [SCX_OP_IDX(sub_attach)] = SCX_KF_ALLOW_UNLOCKED, [SCX_OP_IDX(sub_detach)] = SCX_KF_ALLOW_UNLOCKED, + [SCX_OP_IDX(sub_ecaps_updated)] = SCX_KF_ALLOW_ENQUEUE | SCX_KF_ALLOW_DISPATCH, [SCX_OP_IDX(cpu_online)] = SCX_KF_ALLOW_UNLOCKED, [SCX_OP_IDX(cpu_offline)] = SCX_KF_ALLOW_UNLOCKED, [SCX_OP_IDX(init_cids)] = SCX_KF_ALLOW_UNLOCKED | SCX_KF_ALLOW_INIT_CIDS, @@ -10031,6 +10039,7 @@ static int __init scx_init(void) CID_OFFSET_MATCH(sub_attach, sub_attach); CID_OFFSET_MATCH(sub_detach, sub_detach); CID_OFFSET_MATCH(sub_caps_updated, sub_caps_updated); + CID_OFFSET_MATCH(sub_ecaps_updated, sub_ecaps_updated); CID_OFFSET_MATCH(init_cids, init_cids); CID_OFFSET_MATCH(init, init); CID_OFFSET_MATCH(exit, exit); diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index f512475d1c03..b1b3937168fb 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -770,12 +770,26 @@ struct sched_ext_ops { * Delivered asynchronously after the change is recorded, and may run * before it takes effect on any given cpu. Use it to track which caps * the sub-sched holds and propagate to its own children, not to decide - * if a task can run on a cpu now. + * if a task can run on a cpu now. sub_ecaps_updated() reports that per + * cpu, once it is in effect. * * May call scx_bpf_sub_grant() / scx_bpf_sub_revoke() on children. */ void (*sub_caps_updated)(const struct scx_cmask *cmask, u64 caps); + /** + * @sub_ecaps_updated: This sub-sched's effective caps on a cid changed + * @cid: the cid whose effective caps changed + * @before: effective caps as of the last delivery + * @after: effective caps now + * + * Invoked when this sub-sched's effective caps on @cid change, once the + * change is in effect on the cpu. Runs in dispatch context with rq lock + * held, and can perform all operations allowed in ops.dispatch() + * including inserting/moving tasks. + */ + void (*sub_ecaps_updated)(s32 cid, u64 before, u64 after); + /* * All online ops must come before ops.cpu_online(). */ @@ -997,6 +1011,7 @@ struct sched_ext_ops_cid { s32 (*sub_attach)(struct scx_sub_attach_args *args); void (*sub_detach)(struct scx_sub_detach_args *args); void (*sub_caps_updated)(const struct scx_cmask *cmask, u64 caps); + void (*sub_ecaps_updated)(s32 cid, u64 before, u64 after); void (*cid_online)(s32 cid); void (*cid_offline)(s32 cid); s32 (*init_cids)(void); @@ -1198,6 +1213,8 @@ struct scx_sched_pcpu { */ u64 ecaps; struct llist_node ecaps_to_sync_node; + /* effective caps as of the last sub_ecaps_updated() delivery */ + u64 reported_ecaps; #endif /* diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 136d8dabe4ad..a86ecfb623c4 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -13,6 +13,7 @@ * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. * Copyright (c) 2026 Tejun Heo */ +#include #include #include "internal.h" #include "cid.h" @@ -349,15 +350,16 @@ static void discard_queued_syncs(struct rq *rq) /** * scx_process_sync_ecaps - Sync this cpu's ecaps to pshard->caps[] * @rq: the cid's cpu rq + * @prev: @rq's previous task from the in-progress balance * * pshard->caps[] is the target configuration. pcpu->ecaps is the effective * transposed copy owned by the cid's cpu and written only here under @rq's * lock. */ -void scx_process_sync_ecaps(struct rq *rq) +void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) { - s32 cid = __scx_cpu_to_cid(cpu_of(rq)); - s32 shard = scx_cid_to_shard[cid]; + s32 cpu = cpu_of(rq); + s32 cid, shard; struct llist_node *batch, *pos, *tmp; lockdep_assert_rq_held(rq); @@ -365,33 +367,140 @@ void scx_process_sync_ecaps(struct rq *rq) if (likely(llist_empty(&rq->scx.ecaps_to_sync))) return; + /* + * ecaps are zeroed while the cpu is inactive and must stay zero. + * Discard queued syncs instead of processing them - the + * scx_online_ecaps() reseed re-syncs every sched on activation. + * cpu_active() clears before the offline zeroing and sets before the + * reseed is queued, so this test can neither miss a racing sync nor + * eat the reseed. + */ + if (unlikely(!cpu_active(cpu))) { + discard_queued_syncs(rq); + return; + } + + /* @cid is valid here: the cpu is active with queued syncs */ + cid = __scx_cpu_to_cid(cpu); + shard = scx_cid_to_shard[cid]; + batch = llist_del_all(&rq->scx.ecaps_to_sync); llist_for_each_safe(pos, tmp, batch) { struct scx_sched_pcpu *pcpu = container_of(pos, struct scx_sched_pcpu, ecaps_to_sync_node); struct scx_pshard *ps = pcpu->sch->pshard[shard]; + u64 ecaps; init_llist_node(pos); /* pairs with smp_mb() in queue_sync_ecaps(), see there */ smp_mb(); - WRITE_ONCE(pcpu->ecaps, calc_effective_caps(ps, cid)); + ecaps = calc_effective_caps(ps, cid); + WRITE_ONCE(pcpu->ecaps, ecaps); + + /* tell the sched its effective caps on this cid changed */ + if (ecaps != pcpu->reported_ecaps && + SCX_HAS_OP(pcpu->sch, sub_ecaps_updated) && + !scx_bypassing(pcpu->sch, cpu)) { + struct scx_dsp_ctx *dspc = &pcpu->dsp_ctx; + + dspc->rq = rq; + /* stash @prev so nested dispatches can access it */ + rq->scx.sub_dispatch_prev = prev; + SCX_CALL_OP(pcpu->sch, sub_ecaps_updated, rq, scx_cpu_arg(cpu), + pcpu->reported_ecaps, ecaps); + rq->scx.sub_dispatch_prev = NULL; + scx_flush_dispatch_buf(pcpu->sch, rq); + pcpu->reported_ecaps = ecaps; + } + } +} + +/* + * A cpu came back. Re-seed each sub-sched's ecaps on the cpu's cid. The sync + * recomputes effective caps from the pshard and fires ops.sub_ecaps_updated() + * only on a real change since offline. + */ +void scx_online_ecaps(struct rq *rq) +{ + s32 cid = __scx_cpu_to_cid(cpu_of(rq)); + s32 shard = scx_cid_to_shard[cid]; + struct scx_sched *pos; + + guard(rq_lock_irqsave)(rq); + + scx_for_each_descendant_pre(pos, scx_root) { + struct scx_pshard *ps; + + /* root holds every cap and never uses ecaps */ + if (pos == scx_root) + continue; + + ps = pos->pshard[shard]; + guard(raw_spinlock)(&ps->lock); + queue_sync_ecaps(pos, cid); + } +} + +/* + * A cpu is going down. Zero each sub-sched's in-effect ecaps so cap checks + * treat the cpu as capless while offline. Pending and late-queued syncs are + * discarded at consumption by scx_process_sync_ecaps() while the cpu is + * inactive. Leave reported_ecaps. Ownership is unchanged, so the + * scx_online_ecaps() reseed reports only a genuine delta. No callback fires + * here. + */ +void scx_offline_ecaps(struct rq *rq) +{ + s32 cpu = cpu_of(rq); + struct scx_sched *pos; + + guard(rq_lock_irqsave)(rq); + + scx_for_each_descendant_pre(pos, scx_root) { + /* root holds every cap and never uses ecaps */ + if (pos == scx_root) + continue; + + WRITE_ONCE(per_cpu_ptr(pos->pcpu, cpu)->ecaps, 0); } } /* * @pcpu's sched was unhashed before the grace period, so nothing new queues. - * Flush its pending sync so the pcpu can be freed. scx_process_sync_ecaps() - * takes nodes off the list before syncing and acquiring the rq lock waits for - * any in-flight walk. + * Flush its pending sync so the pcpu can be freed. If the cpu is online and + * scx is enabled, drain via balance_one(). Otherwise, discard under the rq + * lock. */ void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu) { - scoped_guard (rq_lock_irqsave, cpu_rq(cpu)) - scx_process_sync_ecaps(cpu_rq(cpu)); + struct rq *rq = cpu_rq(cpu); - WARN_ON_ONCE(llist_on_list(&pcpu->ecaps_to_sync_node)); + while (true) { + scoped_guard (rq_lock_irqsave, rq) { + /* + * scx_process_sync_ecaps() takes the node off the list + * before it is done accessing @pcpu but does all of it + * under the rq lock. Off-list observed under the rq + * lock guarantees that the sync is complete. + */ + if (!llist_on_list(&pcpu->ecaps_to_sync_node)) + return; + /* + * Discard only when the cpu is truly down. cpu_active() + * is already set when scx_online_ecaps() queues an online + * resync while SCX_RQ_ONLINE is not - so test cpu_active(), + * or that resync would be dropped. + */ + if (!scx_enabled() || !cpu_active(cpu)) { + discard_queued_syncs(rq); + return; + } + } + resched_cpu(cpu); + msleep(1); + } } /** diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index 9bf700df8c82..d28d16d84cd5 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -29,7 +29,9 @@ bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux *aux); void scx_free_pshards(struct scx_sched *sch); s32 scx_alloc_pshards(struct scx_sched *sch); void scx_init_root_caps(struct scx_sched *sch); -void scx_process_sync_ecaps(struct rq *rq); +void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev); +void scx_online_ecaps(struct rq *rq); +void scx_offline_ecaps(struct rq *rq); void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu); void scx_discard_stale_ecaps_syncs(void); @@ -51,7 +53,9 @@ static inline void scx_sub_disable(struct scx_sched *sch) { } static inline void scx_free_pshards(struct scx_sched *sch) {} static inline s32 scx_alloc_pshards(struct scx_sched *sch) { return 0; } static inline void scx_init_root_caps(struct scx_sched *sch) {} -static inline void scx_process_sync_ecaps(struct rq *rq) {} +static inline void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) {} +static inline void scx_online_ecaps(struct rq *rq) {} +static inline void scx_offline_ecaps(struct rq *rq) {} static inline void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu) {} static inline void scx_discard_stale_ecaps_syncs(void) {} -- cgit v1.2.3 From bf6cf1886a56eaf929d8193a3f4b30be59a778e2 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 13 Jul 2026 22:18:43 -1000 Subject: sched_ext: Generalize local-DSQ handling to rq-owned DSQs The local DSQ is synchronized by the containing rq lock rather than its own dsq->lock. A later patch adds a second such DSQ. In preparation, factor the "rq owns the lock" test into dsq_is_rq_owned() and rename local_dsq_post_enq() to rq_owned_post_enq(), taking @rq explicitly. No behavior change. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index e96b0ced5a4d..71219b818b4f 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -96,6 +96,17 @@ static atomic_long_t scx_hotplug_seq = ATOMIC_LONG_INIT(0); /* Global cursor for the per-CPU tid allocator. Starts at 1; tid 0 is reserved. */ static atomic64_t scx_tid_cursor = ATOMIC64_INIT(1); +/* is @dsq synchronized by the containing rq lock instead of dsq->lock? */ +static bool dsq_is_rq_owned(struct scx_dispatch_q *dsq) +{ + switch (dsq->id) { + case SCX_DSQ_LOCAL: + return true; + default: + return false; + } +} + #ifdef CONFIG_EXT_SUB_SCHED /* * The sub sched being enabled. Used by scx_disable_and_exit_task() to exit @@ -1270,11 +1281,10 @@ static void call_task_dequeue(struct scx_sched *sch, struct rq *rq, p->scx.flags &= ~SCX_TASK_IN_CUSTODY; } -static void local_dsq_post_enq(struct scx_sched *sch, struct scx_dispatch_q *dsq, - struct task_struct *p, u64 enq_flags) +static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq, + struct scx_dispatch_q *dsq, struct task_struct *p, + u64 enq_flags) { - struct rq *rq = container_of(dsq, struct rq, scx.local_dsq); - call_task_dequeue(sch, rq, p, 0); /* @@ -1334,13 +1344,13 @@ static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq, struct scx_dispatch_q *dsq, struct task_struct *p, u64 enq_flags) { - bool is_local = dsq->id == SCX_DSQ_LOCAL; + bool is_rq_owned = dsq_is_rq_owned(dsq); WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node)); WARN_ON_ONCE((p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) || !RB_EMPTY_NODE(&p->scx.dsq_priq)); - if (!is_local) { + if (!is_rq_owned) { raw_spin_lock_nested(&dsq->lock, (enq_flags & SCX_ENQ_NESTED) ? SINGLE_DEPTH_NESTING : 0); @@ -1435,8 +1445,8 @@ static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq, * ops_state first, both sides would modify p->scx.flags * concurrently in a non-atomic way. */ - if (is_local) { - local_dsq_post_enq(sch, dsq, p, enq_flags); + if (is_rq_owned) { + rq_owned_post_enq(sch, rq, dsq, p, enq_flags); } else { /* * Global and bypass DSQs are terminal - the task leaves the @@ -1487,7 +1497,7 @@ static void task_unlink_from_dsq(struct task_struct *p, static void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p) { struct scx_dispatch_q *dsq = p->scx.dsq; - bool is_local = dsq == &rq->scx.local_dsq; + bool is_rq_owned = dsq && dsq_is_rq_owned(dsq); lockdep_assert_rq_held(rq); @@ -1511,7 +1521,7 @@ static void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p) return; } - if (!is_local) + if (!is_rq_owned) raw_spin_lock(&dsq->lock); /* @@ -1533,7 +1543,7 @@ static void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p) } p->scx.dsq = NULL; - if (!is_local) + if (!is_rq_owned) raw_spin_unlock(&dsq->lock); } @@ -2085,7 +2095,7 @@ static void move_local_task_to_local_dsq(struct scx_sched *sch, dsq_inc_nr(dst_dsq, p, enq_flags); p->scx.dsq = dst_dsq; - local_dsq_post_enq(sch, dst_dsq, p, enq_flags); + rq_owned_post_enq(sch, dst_rq, dst_dsq, p, enq_flags); } /** -- cgit v1.2.3 From 75a8c8202c918f567e7a7b16add40b0be02c2113 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 13 Jul 2026 22:18:43 -1000 Subject: sched_ext: Add reject DSQ for cap-rejected dispatches When a sub-scheduler dispatches a task to a CPU it lacks the required capability on, the task must be rejected rather than allowed to run. Add the machinery for that. Each rq gets a reject DSQ, a kernel-internal holding queue that is never run and that the BPF scheduler cannot reach. An insert that must be refused is diverted there instead of the local DSQ, and a deferred requeue then hands the parked tasks back to the BPF scheduler to re-decide. A cap revoke extends this to already-queued tasks. When the revoke reaches the cpu's effective caps, the cpu scans its local DSQ and reenqueues the tasks that no longer qualify. A migration-disabled task must run on its cpu, so a capless one is admitted anyway and counted in the new SCX_EV_SUB_FORCED_ADMIT event. This is preparation for the actual sub-sched cap enforcement. The divert is wired but inert here. v2: Admit offline-rq and migration_pending inserts to local, not reject. (sashiko AI) Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- include/linux/sched/ext.h | 15 +++++- kernel/sched/ext/ext.c | 42 ++++++++++++--- kernel/sched/ext/internal.h | 19 ++++++- kernel/sched/ext/sub.c | 127 +++++++++++++++++++++++++++++++++++++++++++- kernel/sched/ext/sub.h | 49 +++++++++++++++++ kernel/sched/sched.h | 3 ++ 6 files changed, 244 insertions(+), 11 deletions(-) (limited to 'kernel') diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index 75cb8b119fb7..7e3f6b33f4a8 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -58,6 +58,7 @@ enum scx_dsq_id_flags { SCX_DSQ_GLOBAL = SCX_DSQ_FLAG_BUILTIN | 1, SCX_DSQ_LOCAL = SCX_DSQ_FLAG_BUILTIN | 2, SCX_DSQ_BYPASS = SCX_DSQ_FLAG_BUILTIN | 3, + SCX_DSQ_REJECT = SCX_DSQ_FLAG_BUILTIN | 4, /* internal - see find_dsq_for_dispatch() */ SCX_DSQ_LOCAL_ON = SCX_DSQ_FLAG_BUILTIN | SCX_DSQ_FLAG_LOCAL_ON, SCX_DSQ_LOCAL_CPU_MASK = 0xffffffffLLU, }; @@ -124,7 +125,7 @@ enum scx_ent_flags { SCX_TASK_DEAD = 5 << SCX_TASK_STATE_SHIFT, /* - * Bits 12 and 13 are used to carry reenqueue reason. In addition to + * Bits 12 to 14 are used to carry reenqueue reason. In addition to * %SCX_ENQ_REENQ flag, ops.enqueue() can also test for * %SCX_TASK_REENQ_REASON_NONE to distinguish reenqueues. * @@ -132,15 +133,17 @@ enum scx_ent_flags { * KFUNC reenqueued by scx_bpf_dsq_reenq() and friends * IMMED reenqueued due to failed ENQ_IMMED * PREEMPTED preempted while running + * CAP sub-sched cap miss, see p->scx.reenq_reason_* */ SCX_TASK_REENQ_REASON_SHIFT = 12, - SCX_TASK_REENQ_REASON_BITS = 2, + SCX_TASK_REENQ_REASON_BITS = 3, SCX_TASK_REENQ_REASON_MASK = ((1 << SCX_TASK_REENQ_REASON_BITS) - 1) << SCX_TASK_REENQ_REASON_SHIFT, SCX_TASK_REENQ_NONE = 0 << SCX_TASK_REENQ_REASON_SHIFT, SCX_TASK_REENQ_KFUNC = 1 << SCX_TASK_REENQ_REASON_SHIFT, SCX_TASK_REENQ_IMMED = 2 << SCX_TASK_REENQ_REASON_SHIFT, SCX_TASK_REENQ_PREEMPTED = 3 << SCX_TASK_REENQ_REASON_SHIFT, + SCX_TASK_REENQ_CAP = 4 << SCX_TASK_REENQ_REASON_SHIFT, /* iteration cursor, not a task */ SCX_TASK_CURSOR = 1 << 31, @@ -239,6 +242,14 @@ struct sched_ext_entity { */ u64 dsq_vtime; + /* + * Sub-sched cap rejected reenq context, valid only while + * %SCX_TASK_REENQ_CAP is set. @reenq_reason_caps is the SCX_CAP_* bits + * that were needed but missing. @reenq_reason_cid is the target cid. + */ + u64 reenq_reason_caps; + s32 reenq_reason_cid; + /* * If set, reject future sched_setscheduler(2) calls updating the policy * to %SCHED_EXT with -%EACCES. diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 71219b818b4f..547468af9674 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -101,6 +101,7 @@ static bool dsq_is_rq_owned(struct scx_dispatch_q *dsq) { switch (dsq->id) { case SCX_DSQ_LOCAL: + case SCX_DSQ_REJECT: return true; default: return false; @@ -1287,6 +1288,12 @@ static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq, { call_task_dequeue(sch, rq, p, 0); + /* rejected: kick the deferred reenq, skip wakeup/preemption */ + if (unlikely(dsq->id == SCX_DSQ_REJECT)) { + schedule_deferred_locked(rq); + return; + } + /* * Note that @rq's lock may be dropped between this enqueue and @p * actually getting on CPU. This gives higher-class tasks (e.g. RT) @@ -1344,7 +1351,12 @@ static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq, struct scx_dispatch_q *dsq, struct task_struct *p, u64 enq_flags) { - bool is_rq_owned = dsq_is_rq_owned(dsq); + bool is_rq_owned = false; + + if (dsq->id == SCX_DSQ_LOCAL) { + dsq = scx_local_or_reject_dsq(sch, rq, p, &enq_flags); + is_rq_owned = true; + } WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node)); WARN_ON_ONCE((p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) || @@ -1494,7 +1506,7 @@ static void task_unlink_from_dsq(struct task_struct *p, } } -static void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p) +void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p) { struct scx_dispatch_q *dsq = p->scx.dsq; bool is_rq_owned = dsq && dsq_is_rq_owned(dsq); @@ -1584,6 +1596,10 @@ static struct scx_dispatch_q *find_dsq_for_dispatch(struct scx_sched *sch, else dsq = find_user_dsq(sch, dsq_id); + /* + * Built-in DSQs are never inserted into dsq_hash, so REJECT hits the + * error below. It cannot be reached with an ID. + */ if (unlikely(!dsq)) { scx_error(sch, "non-existent DSQ 0x%llx", dsq_id); return find_global_dsq(sch, tcpu); @@ -1709,8 +1725,8 @@ bool scx_rq_online(struct rq *rq) return likely((rq->scx.flags & SCX_RQ_ONLINE) && cpu_active(cpu_of(rq))); } -static void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, - int sticky_cpu) +void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, + int sticky_cpu) { struct scx_sched *sch = scx_task_sched(p); struct task_struct **ddsp_taskp; @@ -2079,7 +2095,7 @@ static void move_local_task_to_local_dsq(struct scx_sched *sch, struct scx_dispatch_q *src_dsq, struct rq *dst_rq) { - struct scx_dispatch_q *dst_dsq = &dst_rq->scx.local_dsq; + struct scx_dispatch_q *dst_dsq = scx_local_or_reject_dsq(sch, dst_rq, p, &enq_flags); /* @dsq is locked and @p is on @dst_rq */ lockdep_assert_held(&src_dsq->lock); @@ -3808,7 +3824,8 @@ static void process_ddsp_deferred_locals(struct rq *rq) * another reenq cycle. Repetitions are bounded by %SCX_REENQ_LOCAL_MAX_REPEAT * in process_deferred_reenq_locals(). */ -static bool local_task_should_reenq(struct task_struct *p, u64 *reenq_flags, u32 *reason) +static bool local_task_should_reenq(struct rq *rq, struct task_struct *p, + u64 *reenq_flags, u32 *reason) { bool first; @@ -3824,6 +3841,12 @@ static bool local_task_should_reenq(struct task_struct *p, u64 *reenq_flags, u32 return true; } + if ((*reenq_flags & SCX_REENQ_CAP_REVOKE) && + scx_task_reenq_on_cap_revoke(rq, p)) { + *reason = SCX_TASK_REENQ_CAP; + return true; + } + return *reenq_flags & SCX_REENQ_ANY; } @@ -3867,7 +3890,7 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) if (!scx_is_descendant(task_sch, sch)) continue; - if (!local_task_should_reenq(p, &reenq_flags, &reason)) + if (!local_task_should_reenq(rq, p, &reenq_flags, &reason)) continue; scx_dispatch_dequeue(rq, p); @@ -4063,6 +4086,8 @@ static void run_deferred(struct rq *rq) if (!list_empty(&rq->scx.deferred_reenq_users)) process_deferred_reenq_users(rq); + + scx_reenq_reject(rq); } #ifdef CONFIG_NO_HZ_FULL @@ -7896,6 +7921,9 @@ void __init init_sched_ext_class(void) /* local_dsq's sch will be set during scx_root_enable() */ BUG_ON(init_dsq(&rq->scx.local_dsq, SCX_DSQ_LOCAL, NULL)); +#ifdef CONFIG_EXT_SUB_SCHED + BUG_ON(init_dsq(&rq->scx.reject_dsq, SCX_DSQ_REJECT, NULL)); +#endif INIT_LIST_HEAD(&rq->scx.runnable_list); INIT_LIST_HEAD(&rq->scx.ddsp_deferred_locals); diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index b1b3937168fb..a360d312702b 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1135,6 +1135,13 @@ struct scx_event_stats { * from sub_bypass_dsq's. */ s64 SCX_EV_SUB_BYPASS_DISPATCH; + + /* + * The number of times a migration-disabled task lacking the cap for its + * cid was allowed onto the local DSQ. It must run on its pinned CPU, so + * it can't be rejected. The violation is counted here. + */ + s64 SCX_EV_SUB_FORCED_ADMIT; }; #define SCX_EVENTS_LIST(SCX_EVENT) \ @@ -1150,7 +1157,8 @@ struct scx_event_stats { SCX_EVENT(SCX_EV_BYPASS_DISPATCH); \ SCX_EVENT(SCX_EV_BYPASS_ACTIVATE); \ SCX_EVENT(SCX_EV_INSERT_NOT_OWNED); \ - SCX_EVENT(SCX_EV_SUB_BYPASS_DISPATCH) + SCX_EVENT(SCX_EV_SUB_BYPASS_DISPATCH); \ + SCX_EVENT(SCX_EV_SUB_FORCED_ADMIT) struct scx_sched; @@ -1270,6 +1278,9 @@ enum scx_cap_flags { __SCX_CAP_ALL = BIT_U64(__SCX_NR_CAPS) - 1, SCX_CAP_DUMMY = BIT_U64(__SCX_CAP_DUMMY), + + /* caps whose loss strands queued tasks, see scx_process_sync_ecaps() */ + SCX_CAPS_REENQ_ON_LOSS = 0, }; #ifdef CONFIG_EXT_SUB_SCHED @@ -1583,6 +1594,9 @@ enum scx_reenq_flags { /* low 16bits determine which tasks should be reenqueued */ SCX_REENQ_ANY = 1LLU << 0, /* all tasks */ + /* internal: kernel-issued on cap revoke, not accepted from BPF */ + SCX_REENQ_CAP_REVOKE = 1LLU << 1, + __SCX_REENQ_FILTER_MASK = 0xffffLLU, __SCX_REENQ_USER_MASK = SCX_REENQ_ANY, @@ -1835,6 +1849,9 @@ void scx_task_iter_start(struct scx_task_iter *iter, struct cgroup *cgrp); void scx_task_iter_unlock(struct scx_task_iter *iter); void scx_task_iter_stop(struct scx_task_iter *iter); struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter); +void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p); +void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, + int sticky_cpu); bool scx_consume_dispatch_q(struct scx_sched *sch, struct rq *rq, struct scx_dispatch_q *dsq, u64 enq_flags); bool scx_consume_global_dsq(struct scx_sched *sch, struct rq *rq); diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index a86ecfb623c4..bbd068bfd86c 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -216,6 +216,118 @@ void scx_init_root_caps(struct scx_sched *sch) } } +/** + * scx_local_or_reject_dsq - Pick the local or reject DSQ for an insert + * @sch: enqueuing sub-sched + * @rq: rq whose local DSQ @p targets + * @p: task being inserted + * @enq_flags: in/out; %SCX_ENQ_IMMED is cleared when diverting to reject + * + * Return @rq's local DSQ if @sch holds the required caps on @rq's cid, + * otherwise @rq's reject DSQ after recording the reenq reason on @p. + * + * Bypass doesn't need special-casing as a bypassing sched's tasks are enqueued + * to and run by its nearest non-bypassing ancestor. If root is bypassing, it + * always holds all caps. + */ +struct scx_dispatch_q *scx_local_or_reject_dsq(struct scx_sched *sch, struct rq *rq, + struct task_struct *p, u64 *enq_flags) +{ + s32 cid = __scx_cpu_to_cid(cpu_of(rq)); + u64 missing = scx_missing_caps(sch, cpu_of(rq), scx_caps_for_enq(*enq_flags)); + + /* requirements met */ + if (likely(!missing)) + return &rq->scx.local_dsq; + + /* + * The task must run on this CPU regardless of caps: the rq is draining + * offline (BPF scheduler bypassed), the task is migration-disabled, or a + * migration is pending. Admit despite the missing caps and count it. + */ + if (unlikely(!scx_rq_online(rq) || is_migration_disabled(p) || + p->migration_pending)) { + __scx_add_event(sch, SCX_EV_SUB_FORCED_ADMIT, 1); + return &rq->scx.local_dsq; + } + + p->scx.reenq_reason_caps = missing; + p->scx.reenq_reason_cid = cid; + + /* + * Only local DSQ can honor IMMED and dsq_inc_nr() WARNs on IMMED into + * others. Strip both the enq flag and the sticky task flag - the + * latter can carry in from an earlier admitted IMMED insert. + */ + *enq_flags &= ~SCX_ENQ_IMMED; + p->scx.flags &= ~SCX_TASK_IMMED; + + return &rq->scx.reject_dsq; +} + +/* @p lost the caps needed to stay on @rq's local DSQ? Record reason if so. */ +bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p) +{ + u64 missing; + + /* migration-disabled tasks are admitted regardless of caps */ + if (is_migration_disabled(p)) + return false; + + missing = scx_missing_caps(scx_task_sched(p), cpu_of(rq), scx_caps_for_task(p)); + if (likely(!missing)) + return false; + + p->scx.reenq_reason_caps = missing; + p->scx.reenq_reason_cid = __scx_cpu_to_cid(cpu_of(rq)); + return true; +} + +/* + * Drain @rq->scx.reject_dsq, reenqueueing each task so the BPF re-decides + * from p->scx.reenq_reason_*. + * + * A task can be re-rejected repeatedly, and there's no repeat limit here. + * Rejection can't happen for root, and sub-scheds can be safely ejected after + * triggering the stall watchdog. + */ +void scx_reenq_reject(struct rq *rq) +{ + LIST_HEAD(tasks); + struct task_struct *p, *n; + + lockdep_assert_rq_held(rq); + + if (list_empty(&rq->scx.reject_dsq.list)) + return; + + /* + * Move to a private list so a task re-rejected by the + * scx_do_enqueue_task() below isn't revisited this round. + */ + list_for_each_entry_safe(p, n, &rq->scx.reject_dsq.list, scx.dsq_list.node) { + /* migration_pending tasks should have bypassed to local DSQ */ + if (WARN_ON_ONCE(p->migration_pending)) + continue; + + scx_dispatch_dequeue(rq, p); + + if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) + p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; + p->scx.flags |= SCX_TASK_REENQ_CAP; + + list_add_tail(&p->scx.dsq_list.node, &tasks); + } + + list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) { + list_del_init(&p->scx.dsq_list.node); + + scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); + + p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; + } +} + /* record a caps change, see struct scx_caps_updated */ static void caps_updated_record(struct scx_pshard *ps, const struct scx_cmask *cids, u64 caps, struct list_head *to_deliver) @@ -361,6 +473,7 @@ void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) s32 cpu = cpu_of(rq); s32 cid, shard; struct llist_node *batch, *pos, *tmp; + u64 lost_all = 0; lockdep_assert_rq_held(rq); @@ -389,16 +502,20 @@ void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) struct scx_sched_pcpu *pcpu = container_of(pos, struct scx_sched_pcpu, ecaps_to_sync_node); struct scx_pshard *ps = pcpu->sch->pshard[shard]; - u64 ecaps; + u64 old, ecaps, lost; init_llist_node(pos); /* pairs with smp_mb() in queue_sync_ecaps(), see there */ smp_mb(); + old = READ_ONCE(pcpu->ecaps); ecaps = calc_effective_caps(ps, cid); WRITE_ONCE(pcpu->ecaps, ecaps); + lost = old & ~ecaps; + lost_all |= lost; + /* tell the sched its effective caps on this cid changed */ if (ecaps != pcpu->reported_ecaps && SCX_HAS_OP(pcpu->sch, sub_ecaps_updated) && @@ -415,6 +532,14 @@ void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) pcpu->reported_ecaps = ecaps; } } + + /* + * Losing a cap can strand already-queued tasks. Schedule a reenq scan + * to move the now-capless ones off the local DSQ. The scan tests + * against the effective caps and thus must come after the ecaps sync. + */ + if (lost_all & SCX_CAPS_REENQ_ON_LOSS) + scx_schedule_reenq_local(rq, SCX_REENQ_CAP_REVOKE); } /* diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index d28d16d84cd5..9fc95d27f393 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -34,6 +34,10 @@ void scx_online_ecaps(struct rq *rq); void scx_offline_ecaps(struct rq *rq); void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu); void scx_discard_stale_ecaps_syncs(void); +struct scx_dispatch_q *scx_local_or_reject_dsq(struct scx_sched *sch, struct rq *rq, + struct task_struct *p, u64 *enq_flags); +bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p); +void scx_reenq_reject(struct rq *rq); static inline const char *sch_cgrp_path(struct scx_sched *sch) { @@ -58,6 +62,9 @@ static inline void scx_online_ecaps(struct rq *rq) {} static inline void scx_offline_ecaps(struct rq *rq) {} static inline void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu) {} static inline void scx_discard_stale_ecaps_syncs(void) {} +static inline struct scx_dispatch_q *scx_local_or_reject_dsq(struct scx_sched *sch, struct rq *rq, struct task_struct *p, u64 *enq_flags) { return &rq->scx.local_dsq; } +static inline bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p) { return false; } +static inline void scx_reenq_reject(struct rq *rq) {} #endif /* CONFIG_EXT_SUB_SCHED */ @@ -76,12 +83,54 @@ static inline void scx_discard_stale_ecaps_syncs(void) {} #ifdef CONFIG_EXT_SUB_SCHED +/** + * scx_missing_caps - The caps in @needed that @sch lacks on @cpu + * @sch: sched to test + * @cpu: cpu to test on + * @needed: bitmask of SCX_CAP_* values + * + * Return the caps in @needed that @sch lacks for @cpu, 0 if it holds them all. + */ +static inline u64 scx_missing_caps(struct scx_sched *sch, s32 cpu, u64 needed) +{ + u64 ecaps; + + /* root holds every cap on every cpu */ + if (!sch->level) + return 0; + + ecaps = READ_ONCE(per_cpu_ptr(sch->pcpu, cpu)->ecaps); + + return needed & ~ecaps; +} + +/* + * Cap semantics: which caps an action requires, and which caps a cap implies. + * Keep all such mappings collected here. + */ + +/* map @enq_flags to the SCX_CAP_* bit required for the local-DSQ insert */ +static inline u64 scx_caps_for_enq(u64 enq_flags) +{ + return 0; +} + +/* map queued @p to the SCX_CAP_* bit required to stay on its local DSQ */ +static inline u64 scx_caps_for_task(struct task_struct *p) +{ + return 0; +} + /* caps implied by holding @cap */ static inline u64 scx_caps_implied(u64 cap) { return 0; } +#else /* CONFIG_EXT_SUB_SCHED */ + +static inline u64 scx_missing_caps(struct scx_sched *sch, s32 cpu, u64 needed) { return 0; } + #endif /* CONFIG_EXT_SUB_SCHED */ /* diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 72299c105487..c0a4699a6c0a 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -794,6 +794,9 @@ enum scx_rq_flags { struct scx_rq { struct scx_dispatch_q local_dsq; +#ifdef CONFIG_EXT_SUB_SCHED + struct scx_dispatch_q reject_dsq; /* staging for cap-rejected tasks */ +#endif struct list_head runnable_list; /* runnable tasks on this rq */ struct list_head ddsp_deferred_locals; /* deferred ddsps from enq */ unsigned long ops_qseq; -- cgit v1.2.3 From 8b17523479437fadb26f649e00a0d848c624c07e Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 13 Jul 2026 22:18:43 -1000 Subject: sched_ext: Add SCX_ENQ_IGNORE_CAPS for in-place restore A SAVE/RESTORE requeue re-inserts a running task in place and is immediately followed by set_next_task_scx(). It is not a real scheduling event: the task is already admitted to its cid and must return to the local DSQ unconditionally. scx_caps_for_enq() maps an enqueue to the cap its local-DSQ insert requires. Add SCX_ENQ_IGNORE_CAPS, set it on the RESTORE-in-place branch of enqueue_task_scx(), and have scx_caps_for_enq() require no caps for it, so the cid admission gate never diverts an in-place restore to the reject DSQ. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 7 +++++-- kernel/sched/ext/internal.h | 1 + kernel/sched/ext/sub.h | 3 +++ 3 files changed, 9 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 547468af9674..2da3a5f44b4a 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1878,10 +1878,13 @@ static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_ * Restoring a running task will be immediately followed by * set_next_task_scx() which expects the task to not be on the BPF * scheduler as tasks can only start running through local DSQs. Force - * direct-dispatch into the local DSQ by setting the sticky_cpu. + * direct-dispatch into the local DSQ by setting the sticky_cpu. Mark + * IGNORE_CAPS to force entry into the local DSQ. */ - if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) + if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) { sticky_cpu = cpu_of(rq); + enq_flags |= SCX_ENQ_IGNORE_CAPS; + } if (p->scx.flags & SCX_TASK_QUEUED) { WARN_ON_ONCE(!task_runnable(p)); diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index a360d312702b..079ee941b343 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1568,6 +1568,7 @@ enum scx_enq_flags { SCX_ENQ_DSQ_PRIQ = 1LLU << 57, SCX_ENQ_NESTED = 1LLU << 58, SCX_ENQ_GDSQ_FALLBACK = 1LLU << 59, /* fell back to global DSQ */ + SCX_ENQ_IGNORE_CAPS = 1LLU << 60, /* admit to local DSQ ignoring caps */ }; enum scx_deq_flags { diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index 9fc95d27f393..3c22ae40349d 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -112,6 +112,9 @@ static inline u64 scx_missing_caps(struct scx_sched *sch, s32 cpu, u64 needed) /* map @enq_flags to the SCX_CAP_* bit required for the local-DSQ insert */ static inline u64 scx_caps_for_enq(u64 enq_flags) { + /* a restored task must be put into the local DSQ regardless of caps */ + if (enq_flags & SCX_ENQ_IGNORE_CAPS) + return 0; return 0; } -- cgit v1.2.3 From 147d1885f390bcfb929210814f86dc22b24c2631 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 13 Jul 2026 22:18:43 -1000 Subject: sched_ext: Add the SCX_CAP_ENQ_IMMED cap Replace the __SCX_CAP_DUMMY placeholder with SCX_CAP_ENQ_IMMED, which gates inserting IMMED tasks onto a cid's local DSQ. An IMMED enqueue is guaranteed to either get its task running on the cpu at once or hand it back to the scheduler, so IMMED work can never pile up on the cpu's queue and a cpu can be shared across sub-scheds through IMMED access without any of them swamping it. That makes ENQ_IMMED the natural baseline, the minimal cap to make any use of a cpu. SCX_CAP_BASE aliases it so gates on basic cpu access can state the intention instead of naming ENQ_IMMED. Enforcement covers inserts and queued tasks. An insert without the cap is diverted to the reject DSQ, and queued tasks are reenqueued when the cap is lost. scx_bpf_sub_dispatch() skips a child that lacks the cap on the cpu, as its inserts would only be rejected. Vacating the running task on cap loss lands in a later patch. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 2 +- kernel/sched/ext/internal.h | 11 ++++++++--- kernel/sched/ext/sub.c | 8 ++++++++ kernel/sched/ext/sub.h | 4 ++-- 4 files changed, 19 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 2da3a5f44b4a..151befdc9602 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -4792,7 +4792,7 @@ SCX_ATTR(events); #ifdef CONFIG_EXT_SUB_SCHED static const char *scx_cap_names[__SCX_NR_CAPS] = { - [__SCX_CAP_DUMMY] = "dummy", + [__SCX_CAP_ENQ_IMMED] = "enq_immed", }; static ssize_t scx_attr_caps_show(struct kobject *kobj, diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 079ee941b343..e735812e77a5 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1270,17 +1270,22 @@ struct scx_sched_pnode { * topology-aligned and likely to serve as the locality unit when cids are * allocated to schedulers, so per-shard lock granularity scales naturally with * the allocation pattern. + * + * ENQ_IMMED insert an IMMED task onto the cid's local DSQ */ enum scx_cap_flags { - __SCX_CAP_DUMMY = 0, + __SCX_CAP_ENQ_IMMED = 0, __SCX_NR_CAPS, __SCX_CAP_ALL = BIT_U64(__SCX_NR_CAPS) - 1, - SCX_CAP_DUMMY = BIT_U64(__SCX_CAP_DUMMY), + SCX_CAP_ENQ_IMMED = BIT_U64(__SCX_CAP_ENQ_IMMED), + + /* alias for minimal cap to make any use of a cpu */ + SCX_CAP_BASE = SCX_CAP_ENQ_IMMED, /* caps whose loss strands queued tasks, see scx_process_sync_ecaps() */ - SCX_CAPS_REENQ_ON_LOSS = 0, + SCX_CAPS_REENQ_ON_LOSS = SCX_CAP_ENQ_IMMED, }; #ifdef CONFIG_EXT_SUB_SCHED diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index bbd068bfd86c..11fda1d3828d 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -1279,6 +1279,14 @@ __bpf_kfunc bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux * return false; } + /* + * Skip a child that does not effectively hold the base cap on this cpu: + * its inserts would only be rejected. ecaps are synced at the top of + * balance_one() before dispatch, so this reflects the in-effect state. + */ + if (scx_missing_caps(child, cpu_of(this_rq), SCX_CAP_BASE)) + return false; + return scx_dispatch_sched(child, this_rq, this_rq->scx.sub_dispatch_prev, true); } diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index 3c22ae40349d..13e9dec56a6a 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -115,13 +115,13 @@ static inline u64 scx_caps_for_enq(u64 enq_flags) /* a restored task must be put into the local DSQ regardless of caps */ if (enq_flags & SCX_ENQ_IGNORE_CAPS) return 0; - return 0; + return SCX_CAP_ENQ_IMMED; } /* map queued @p to the SCX_CAP_* bit required to stay on its local DSQ */ static inline u64 scx_caps_for_task(struct task_struct *p) { - return 0; + return SCX_CAP_ENQ_IMMED; } /* caps implied by holding @cap */ -- cgit v1.2.3 From 61d564fcdfcd664f623d4175738a6ccee654ca02 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 13 Jul 2026 22:18:43 -1000 Subject: sched_ext: Assign a unique id to each scheduler instance Neither a scx_sched pointer nor its cgroup id uniquely identifies a scheduler instance. A freed sched's memory can be reallocated, and a cgroup can detach one sched and attach another. Add a monotonic, never-reused u64 id. A later patch compares it to drop a slice request that outlived a change of a task's owning scheduler. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 4 ++++ kernel/sched/ext/internal.h | 3 +++ 2 files changed, 7 insertions(+) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 151befdc9602..14775e288a6e 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -108,6 +108,9 @@ static bool dsq_is_rq_owned(struct scx_dispatch_q *dsq) } } +/* Cursor for unique scx_sched instance ids. id 0 is reserved. */ +static atomic64_t scx_sched_id_cursor = ATOMIC64_INIT(0); + #ifdef CONFIG_EXT_SUB_SCHED /* * The sub sched being enabled. Used by scx_disable_and_exit_task() to exit @@ -6502,6 +6505,7 @@ struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, level * sizeof(parent->ancestors[0])); sch->ancestors[level] = sch; sch->level = level; + sch->id = atomic64_inc_return(&scx_sched_id_cursor); if (ops->timeout_ms) sch->watchdog_timeout = msecs_to_jiffies(ops->timeout_ms); diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index e735812e77a5..ad98e3469b12 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1445,6 +1445,9 @@ struct scx_sched { struct list_head all; + /* unique instance id, monotonic and never reused */ + u64 id; + #ifdef CONFIG_EXT_SUB_SCHED struct rhash_head hash_node; -- cgit v1.2.3 From c00158ef28b60bf3d793edc66eac74f7a20cc9d3 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 13 Jul 2026 22:18:43 -1000 Subject: sched_ext: Route task slice writes through set_task_slice() A later change makes set_task_slice() also drop a pending out-of-band slice request, so the BPF-triggered writes to p->scx.slice need to funnel through one place. Introduce set_task_slice() and route those writes through it. update_curr_scx() decrements curr->scx.slice directly for accounting and is left alone. No functional change - the helper only assigns p->scx.slice. v2: Reword the set_task_slice comment to "BPF-triggered writes". (Andrea) Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 14775e288a6e..35aca50796fd 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1165,6 +1165,12 @@ static void touch_core_sched_dispatch(struct rq *rq, struct task_struct *p) #endif } +/* set @p's slice, BPF-triggered writes to p->scx.slice go through here */ +static void set_task_slice(struct task_struct *p, u64 slice) +{ + p->scx.slice = slice; +} + static void update_curr_scx(struct rq *rq) { struct task_struct *curr = rq->curr; @@ -1250,7 +1256,7 @@ static void dsq_dec_nr(struct scx_dispatch_q *dsq, struct task_struct *p) static void refill_task_slice_dfl(struct scx_sched *sch, struct task_struct *p) { - p->scx.slice = READ_ONCE(sch->slice_dfl); + set_task_slice(p, READ_ONCE(sch->slice_dfl)); __scx_add_event(sch, SCX_EV_REFILL_SLICE_DFL, 1); } @@ -1345,7 +1351,7 @@ static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq, if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr && rq->curr->sched_class == &ext_sched_class) { - rq->curr->scx.slice = 0; + set_task_slice(rq->curr, 0); resched_curr(rq); } } @@ -2058,7 +2064,7 @@ static void yield_task_scx(struct rq *rq) if (SCX_HAS_OP(sch, yield)) SCX_CALL_OP_2TASKS_RET(sch, yield, rq, p, NULL); else - p->scx.slice = 0; + set_task_slice(p, 0); } static bool yield_to_task_scx(struct rq *rq, struct task_struct *to) @@ -3302,7 +3308,7 @@ static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued) * we can't trust the slice management or ops.core_sched_before(). */ if (scx_bypassing(sch, cpu_of(rq))) { - curr->scx.slice = 0; + set_task_slice(curr, 0); touch_core_sched(rq, curr); } else if (SCX_HAS_OP(sch, tick)) { SCX_CALL_OP_TASK(sch, tick, rq, curr); @@ -3441,7 +3447,7 @@ static void scx_disable_task(struct scx_sched *sch, struct task_struct *p) * control, after ops.disable() has observed their final values. */ p->scx.dsq_vtime = 0; - p->scx.slice = 0; + set_task_slice(p, 0); /* * Verify the task is not in BPF scheduler's custody. If flag @@ -7093,7 +7099,7 @@ static void scx_root_enable_workfn(struct kthread_work *work) queue_flags |= DEQUEUE_CLASS; scoped_guard (sched_change, p, queue_flags) { - p->scx.slice = READ_ONCE(sch->slice_dfl); + set_task_slice(p, READ_ONCE(sch->slice_dfl)); p->sched_class = new_class; } } @@ -7736,7 +7742,7 @@ static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_r !sched_class_above(cur_class, &ext_sched_class)) { if (cpumask_test_cpu(cpu, pcpu->cpus_to_preempt)) { if (cur_class == &ext_sched_class) - rq->curr->scx.slice = 0; + set_task_slice(rq->curr, 0); cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt); } @@ -8086,9 +8092,9 @@ __bpf_kfunc bool scx_bpf_dsq_insert___v2(struct task_struct *p, u64 dsq_id, return false; if (slice) - p->scx.slice = slice; + set_task_slice(p, slice); else - p->scx.slice = p->scx.slice ?: 1; + set_task_slice(p, p->scx.slice ?: 1); scx_dsq_insert_commit(sch, p, dsq_id, enq_flags); @@ -8112,9 +8118,9 @@ static bool scx_dsq_insert_vtime(struct scx_sched *sch, struct task_struct *p, return false; if (slice) - p->scx.slice = slice; + set_task_slice(p, slice); else - p->scx.slice = p->scx.slice ?: 1; + set_task_slice(p, p->scx.slice ?: 1); p->scx.dsq_vtime = vtime; @@ -8296,7 +8302,7 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_VTIME) p->scx.dsq_vtime = kit->vtime; if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_SLICE) - p->scx.slice = kit->slice; + set_task_slice(p, kit->slice); /* execute move */ locked_rq = move_task_between_dsqs(sch, p, enq_flags, src_dsq, dst_dsq); @@ -8689,7 +8695,7 @@ __bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice, if (unlikely(!sch || !scx_task_on_sched(sch, p))) return false; - p->scx.slice = slice; + set_task_slice(p, slice); return true; } -- cgit v1.2.3 From d5b8f4cdd17dc42fd51048785fef1e2a7dd1117f Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 13 Jul 2026 22:18:43 -1000 Subject: sched_ext: Track the cpu a task is runnable on Add p->scx.runnable_cpu, the cpu @p is runnable on, or -1 when it is not. It is stamped as @p joins the runnable_list (set_task_runnable()) and cleared as it leaves (clr_task_runnable()), both under the rq lock. task_cpu() can't answer "is @p on this rq" reliably: a remote wakeup changes it under @p's pi_lock alone, without the source rq lock, so it can read as the locked rq while @p is really elsewhere. runnable_cpu changes only under the rq lock, so a caller holding an rq lock can compare against it to know whether that is @p's current rq. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- include/linux/sched/ext.h | 1 + init/init_task.c | 1 + kernel/sched/ext/ext.c | 9 +++++++++ 3 files changed, 11 insertions(+) (limited to 'kernel') diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index 7e3f6b33f4a8..853f03b63133 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -201,6 +201,7 @@ struct sched_ext_entity { s32 sticky_cpu; s32 holding_cpu; s32 selected_cpu; + s32 runnable_cpu; /* cpu @p is runnable on, -1 if not */ struct task_struct *kf_tasks[2]; /* see SCX_CALL_OP_TASK() */ struct list_head runnable_node; /* rq->scx.runnable_list */ diff --git a/init/init_task.c b/init/init_task.c index b67ef6040a65..5c7ad50ac685 100644 --- a/init/init_task.c +++ b/init/init_task.c @@ -145,6 +145,7 @@ struct task_struct init_task __aligned(L1_CACHE_BYTES) = { .dsq_list.node = LIST_HEAD_INIT(init_task.scx.dsq_list.node), .sticky_cpu = -1, .holding_cpu = -1, + .runnable_cpu = -1, .runnable_node = LIST_HEAD_INIT(init_task.scx.runnable_node), .runnable_at = INITIAL_JIFFIES, .ddsp_dsq_id = SCX_DSQ_INVALID, diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 35aca50796fd..9ae8d78738b4 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1865,11 +1865,19 @@ static void set_task_runnable(struct rq *rq, struct task_struct *p) * appended to the runnable_list. */ list_add_tail(&p->scx.runnable_node, &rq->scx.runnable_list); + + /* + * Record the rq @p is runnable on, maintained under the rq lock so it + * stays valid unlike task_cpu(), which a remote wakeup can move under + * pi_lock alone. + */ + WRITE_ONCE(p->scx.runnable_cpu, cpu_of(rq)); } static void clr_task_runnable(struct task_struct *p, bool reset_runnable_at) { list_del_init(&p->scx.runnable_node); + WRITE_ONCE(p->scx.runnable_cpu, -1); if (reset_runnable_at) p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT; } @@ -3531,6 +3539,7 @@ void init_scx_entity(struct sched_ext_entity *scx) RB_CLEAR_NODE(&scx->dsq_priq); scx->sticky_cpu = -1; scx->holding_cpu = -1; + scx->runnable_cpu = -1; INIT_LIST_HEAD(&scx->runnable_node); scx->runnable_at = jiffies; scx->ddsp_dsq_id = SCX_DSQ_INVALID; -- cgit v1.2.3 From 46a85ae6fe5b468107baa9f21f073a940208d9ff Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 13 Jul 2026 22:18:43 -1000 Subject: sched_ext: Tie cpu occupancy to SCX_CAP_BASE through the task slice A task's slice grants it cpu occupancy - how long it holds its cpu. In a sub-scheduler hierarchy cpu access is delegated through revocable capabilities, so a task's occupancy must follow them. Only its own scheduler sets its slice, and extending the slice is allowed only while that scheduler holds baseline cpu access (SCX_CAP_BASE) on the cpu. Otherwise a scheduler could keep occupying a cpu it has been denied simply by handing out long slices. The cap check reads effective caps, which are coherent only under the task's rq lock, and the kernel decrements the slice under that lock as the task runs, so a running task's slice can be changed only there while a queued task's can be set directly. Make scx_bpf_task_set_slice() apply the slice under the rq lock. Synchronously when the caller already holds it, otherwise by stashing it in the new p->scx.slice_oob, tagged with the scheduler's id so a request that outlived a reassignment is dropped. Whether the caller holds @p's current rq lock is tested with p->scx.runnable_cpu. Revocation is enforced through the same grant. When a cpu's effective caps lose SCX_CAP_BASE, the cap-revoke reenq scan also checks the running task and zeroes its slice to evict it. The scan runs as a balance callback after the pick, so this catches both the task that was running when the revoke landed and a capless task the pick just promoted off the local DSQ. The paths that keep a task on its cpu - holding on to the last runnable task in balance, the ENQ_LAST reinsertion and the slice refill on pick - skip tasks lacking baseline access. A migration-disabled task is exempt, mirroring its capless admission on insert. v4: Test rq ownership with p->scx.runnable_cpu, closing a remote-wakeup TOCTOU. (sashiko AI) v3: Keep a pending out-of-band slice request across refill and preserve. (sashiko AI) v2: Only write slice directly when @p is queued on the held rq. (sashiko AI) Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- include/linux/sched/ext.h | 17 +++- kernel/sched/ext/ext.c | 200 +++++++++++++++++++++++++++++++++++++++++--- kernel/sched/ext/internal.h | 19 ++++- kernel/sched/ext/sub.h | 11 +++ 4 files changed, 231 insertions(+), 16 deletions(-) (limited to 'kernel') diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index 853f03b63133..803da0f1e509 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -223,10 +223,11 @@ struct sched_ext_entity { /* BPF scheduler modifiable fields */ /* - * Runtime budget in nsecs. This is usually set through - * scx_bpf_dsq_insert() but can also be modified directly by the BPF - * scheduler. Automatically decreased by SCX as the task executes. On - * depletion, a scheduling event is triggered. + * Runtime budget in nsecs - how long the task may hold its cpu. Owned + * by the task's scheduler. Set it when enqueuing via + * scx_bpf_dsq_insert(), or otherwise via scx_bpf_task_set_slice(). + * Automatically decreased as the task executes. On depletion a + * scheduling event is triggered. * * This value is cleared to zero if the task is preempted by * %SCX_KICK_PREEMPT and shouldn't be used to determine how long the @@ -243,6 +244,14 @@ struct sched_ext_entity { */ u64 dsq_vtime; + /* + * Out-of-band slice request from scx_bpf_task_set_slice() when the + * caller does not hold the rq lock, applied under the rq lock at the + * next slice consideration. One atomic64 packs the pending flag, the + * issuing sch's id, and the requested slice. See scx_slice_oob_consts. + */ + atomic64_t slice_oob; + /* * Sub-sched cap rejected reenq context, valid only while * %SCX_TASK_REENQ_CAP is set. @reenq_reason_caps is the SCX_CAP_* bits diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 9ae8d78738b4..39b17626c398 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1165,9 +1165,134 @@ static void touch_core_sched_dispatch(struct rq *rq, struct task_struct *p) #endif } -/* set @p's slice, BPF-triggered writes to p->scx.slice go through here */ +/* + * p->scx.slice_oob packs an out-of-band slice request into one atomic64. A zero + * word means no request. Otherwise the fields are: + * + * 63 SCX_SLICE_OOB_PENDING, set on every request + * 62-43 lower bits of issuing scheduler's id + * 42-0 requested slice duration in nsecs + * + * A duration of SCX_SLICE_OOB_DUR_MASK means SCX_SLICE_INF. A finite dur + * saturates at SCX_SLICE_OOB_DUR_MASK - 1. The id is used to detect and ignore + * a request that outlived a task ownership change. + * + * Only the low 20 bits of sch->id are packed, which is enough to make + * collisions practically impossible. A theoretical collision just lets a stale + * request through once. + */ +enum scx_slice_oob_consts { + SCX_SLICE_OOB_DUR_BITS = 43, + SCX_SLICE_OOB_ID_BITS = 64 - SCX_SLICE_OOB_DUR_BITS - 1, + + SCX_SLICE_OOB_DUR_MASK = (1LLU << SCX_SLICE_OOB_DUR_BITS) - 1, + SCX_SLICE_OOB_ID_SHIFT = SCX_SLICE_OOB_DUR_BITS, + SCX_SLICE_OOB_ID_MASK = (1LLU << SCX_SLICE_OOB_ID_BITS) - 1, + SCX_SLICE_OOB_PENDING = 1LLU << 63, +}; + +/* + * Slice write rules + * + * A task's slice - how long it may hold its cpu - is an occupancy grant owned + * by the task's scheduler. How it may be written depends on whether the task is + * running. + * + * Queued, not running: the slice grants no occupancy yet and nothing consumes + * it, so the owner writes it directly - via scx_bpf_dsq_insert(), the dsq move + * kfuncs, or scx_bpf_task_set_slice(). Serializing its own writers is then the + * scheduler's job, not the kernel's. + * + * Running: the slice must be changed under the task's rq lock, because: + * + * - Raising it extends occupancy, allowed only with %SCX_CAP_BASE on the cpu, + * and that cap check is coherent only under the rq lock. Shortening is always + * allowed. + * + * - The kernel decrements it there as the task runs. The decrement is a + * read-modify-write, so a racing write can be clobbered. + * + * scx_bpf_task_set_slice() writes directly only when @p is queued or running + * on the rq lock it holds. That is the only state where the lock keeps us @p's + * owner: @p can't move to another rq without it. A task that isn't queued here + * can instead be woken onto a different rq without taking this lock, and that + * dispatch sets its slice - so a direct write would race. Those cases stash + * into p->scx.slice_oob to be applied under @p's actual rq lock. A later in-band + * write supersedes a stash, and a stash whose scheduler id no longer matches + * @p's owner is dropped. + */ + +/* clear a pending slice request */ +static void clear_task_slice_oob(struct task_struct *p) +{ + if (unlikely(atomic64_read(&p->scx.slice_oob))) + atomic64_set(&p->scx.slice_oob, 0); +} + +/* set @p's slice, leaving any pending out-of-band request in place */ +static void set_task_slice_keep_oob(struct task_struct *p, u64 slice) +{ + p->scx.slice = slice; +} + +/* set @p's slice, superseding any pending out-of-band request */ static void set_task_slice(struct task_struct *p, u64 slice) { + set_task_slice_keep_oob(p, slice); + clear_task_slice_oob(p); +} + +/* request @p's slice to be set to @slice, see the slice write rules above */ +static void set_task_slice_oob(struct scx_sched *sch, struct task_struct *p, u64 slice) +{ + u64 dur; + + if (slice == SCX_SLICE_INF) { + dur = SCX_SLICE_OOB_DUR_MASK; + } else if (unlikely(slice >= SCX_SLICE_OOB_DUR_MASK)) { + dur = SCX_SLICE_OOB_DUR_MASK - 1; + scx_add_event(sch, SCX_EV_SLICE_CLAMPED, 1); + } else { + dur = slice; + } + + atomic64_set(&p->scx.slice_oob, SCX_SLICE_OOB_PENDING | + ((sch->id & SCX_SLICE_OOB_ID_MASK) << SCX_SLICE_OOB_ID_SHIFT) | dur); +} + +/* + * Apply a pending out-of-band slice request under @rq's lock. A request whose + * packed id no longer matches @p's current owner is dropped. An extension needs + * baseline cpu access on @p's cid. %SCX_EV_SLICE_DENIED counts the denials. + * Shortening is always allowed. See the slice write rules above. + */ +static void apply_task_slice_oob(struct rq *rq, struct task_struct *p) +{ + u64 oob, dur, slice; + + lockdep_assert_rq_held(rq); + + if (likely(!atomic64_read(&p->scx.slice_oob))) + return; + + oob = atomic64_xchg(&p->scx.slice_oob, 0); + if (unlikely(!oob)) + return; + + /* the issuing scheduler no longer owns @p, drop the request */ + if (unlikely(((oob >> SCX_SLICE_OOB_ID_SHIFT) & SCX_SLICE_OOB_ID_MASK) != + (scx_task_sched(p)->id & SCX_SLICE_OOB_ID_MASK))) + return; + + dur = oob & SCX_SLICE_OOB_DUR_MASK; + slice = dur == SCX_SLICE_OOB_DUR_MASK ? SCX_SLICE_INF : dur; + + if (slice > p->scx.slice && + unlikely(scx_missing_caps(scx_task_sched(p), cpu_of(rq), SCX_CAP_BASE))) { + __scx_add_event(scx_task_sched(p), SCX_EV_SLICE_DENIED, 1); + return; + } + p->scx.slice = slice; } @@ -1176,6 +1301,9 @@ static void update_curr_scx(struct rq *rq) struct task_struct *curr = rq->curr; s64 delta_exec; + /* apply even on 0 delta_exec, callers may still act on the slice */ + apply_task_slice_oob(rq, curr); + delta_exec = update_curr_common(rq); if (unlikely(delta_exec <= 0)) return; @@ -1256,7 +1384,11 @@ static void dsq_dec_nr(struct scx_dispatch_q *dsq, struct task_struct *p) static void refill_task_slice_dfl(struct scx_sched *sch, struct task_struct *p) { - set_task_slice(p, READ_ONCE(sch->slice_dfl)); + /* + * A default refill is not an explicit request, so it must not drop a + * pending out-of-band one, which is applied when @p next runs. + */ + set_task_slice_keep_oob(p, READ_ONCE(sch->slice_dfl)); __scx_add_event(sch, SCX_EV_REFILL_SLICE_DFL, 1); } @@ -2700,7 +2832,8 @@ static int balance_one(struct rq *rq, struct task_struct *prev) * %SCX_OPS_ENQ_LAST is in effect. */ if ((prev->scx.flags & SCX_TASK_QUEUED) && - (!(sch->ops.flags & SCX_OPS_ENQ_LAST) || scx_bypassing(sch, cpu))) { + (!(sch->ops.flags & SCX_OPS_ENQ_LAST) || scx_bypassing(sch, cpu)) && + scx_task_can_stay_on_cpu(rq, prev)) { rq->scx.flags |= SCX_RQ_BAL_KEEP; __scx_add_event(sch, SCX_EV_DISPATCH_KEEP_LAST, 1); goto has_tasks; @@ -2747,6 +2880,9 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first) clr_task_runnable(p, true); + /* apply any pending out-of-band slice request before the tick decision */ + apply_task_slice_oob(rq, p); + /* * @p is getting newly scheduled or got kicked after someone updated its * slice. Update SCX_RQ_CAN_STOP_TICK to reflect whether the tick can be @@ -2875,12 +3011,14 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, * sched_class, %SCX_OPS_ENQ_LAST must be set. Tell * ops.enqueue() that @p is the only one available for this cpu, * which should trigger an explicit follow-up scheduling event. + * This doesn't apply if the baseline access on the CPU is lost. * * Core scheduling can force this CPU idle while @p stays * runnable. @p's cookie then won't match the core's, so skip * the warning in that case. */ - if (next && sched_class_above(&ext_sched_class, next->sched_class)) { + if (next && sched_class_above(&ext_sched_class, next->sched_class) && + scx_task_can_stay_on_cpu(rq, p)) { WARN_ON_ONCE(sched_cpu_cookie_match(rq, p) && !(sch->ops.flags & SCX_OPS_ENQ_LAST)); scx_do_enqueue_task(rq, p, SCX_ENQ_LAST, -1); @@ -3002,7 +3140,7 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) if (!p) return NULL; - if (unlikely(!p->scx.slice)) { + if (unlikely(!p->scx.slice) && scx_task_can_stay_on_cpu(rq, p)) { struct scx_sched *sch = scx_task_sched(p); if (!scx_bypassing(sch, cpu_of(rq)) && @@ -3932,6 +4070,20 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) nr_enqueued++; } + /* + * The revoke that scheduled this scan may have raced the pick: curr + * may be a now-capless task, either one that kept running or one + * promoted off the local DSQ between the ecaps sync and this scan. + * Zero the slice to evict it. The enqueue gate blocks new capless + * inserts, so no later pick can slip through after the scan. + */ + if ((reenq_flags & SCX_REENQ_CAP_REVOKE) && + rq->curr->sched_class == &ext_sched_class && + scx_task_reenq_on_cap_revoke(rq, rq->curr)) { + set_task_slice(rq->curr, 0); + resched_curr(rq); + } + return nr_enqueued; } @@ -8103,7 +8255,7 @@ __bpf_kfunc bool scx_bpf_dsq_insert___v2(struct task_struct *p, u64 dsq_id, if (slice) set_task_slice(p, slice); else - set_task_slice(p, p->scx.slice ?: 1); + set_task_slice_keep_oob(p, p->scx.slice ?: 1); scx_dsq_insert_commit(sch, p, dsq_id, enq_flags); @@ -8129,7 +8281,7 @@ static bool scx_dsq_insert_vtime(struct scx_sched *sch, struct task_struct *p, if (slice) set_task_slice(p, slice); else - set_task_slice(p, p->scx.slice ?: 1); + set_task_slice_keep_oob(p, p->scx.slice ?: 1); p->scx.dsq_vtime = vtime; @@ -8691,20 +8843,48 @@ __bpf_kfunc_start_defs(); * @slice: time slice to set in nsecs * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * - * Set @p's time slice to @slice. Returns %true on success, %false if the - * calling scheduler doesn't have authority over @p. + * Set @p's time slice. @p must be on the calling scheduler. The value is + * applied whether or not the caller holds @p's rq lock - see the slice write + * rules above for the ownership model. + * + * Raising the slice is honored only while the scheduler holds %SCX_CAP_BASE on + * @p's cpu, otherwise it is counted in %SCX_EV_SLICE_DENIED. Shortening is + * always allowed. On the stashed path the slice is packed into an atomic64_t + * with the scheduler id and a flag bit, so a slice too large to fit is clamped + * and counted in %SCX_EV_SLICE_CLAMPED. %SCX_SLICE_INF is preserved. + * + * Return %true on success, %false if @p is not on the calling scheduler. */ __bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice, const struct bpf_prog_aux *aux) { struct scx_sched *sch; + struct rq *locked_rq; guard(rcu)(); sch = scx_prog_sched(aux); if (unlikely(!sch || !scx_task_on_sched(sch, p))) return false; - set_task_slice(p, slice); + /* + * Directly write only when we hold the lock of the rq @p is queued or + * running on. See the slice write rules above. + */ + locked_rq = scx_locked_rq(); + if (!locked_rq || + (READ_ONCE(p->scx.runnable_cpu) != cpu_of(locked_rq) && + !task_current(locked_rq, p))) { + set_task_slice_oob(sch, p, slice); + return true; + } + + /* under the rq lock: apply now, extensions gated on baseline access */ + if (slice > p->scx.slice && + unlikely(scx_missing_caps(sch, cpu_of(locked_rq), SCX_CAP_BASE))) + __scx_add_event(sch, SCX_EV_SLICE_DENIED, 1); + else + set_task_slice(p, slice); + return true; } diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index ad98e3469b12..ab1dfad28cb5 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -400,8 +400,9 @@ struct sched_ext_ops { * @p: task running currently * * This operation is called every 1/HZ seconds on CPUs which are - * executing an SCX task. Setting @p->scx.slice to 0 will trigger an - * immediate dispatch cycle on the CPU. + * executing an SCX task. Setting a slice of 0 for @p with + * scx_bpf_task_set_slice() will trigger an immediate dispatch cycle on + * the CPU. */ void (*tick)(struct task_struct *p); @@ -1103,6 +1104,18 @@ struct scx_event_stats { */ s64 SCX_EV_REFILL_SLICE_DFL; + /* + * The number of times an out-of-band slice request exceeded the maximum + * representable value and was clamped. + */ + s64 SCX_EV_SLICE_CLAMPED; + + /* + * The number of times a slice extension was denied because the + * scheduler lacked baseline cpu access on the task's cpu. + */ + s64 SCX_EV_SLICE_DENIED; + /* * The total duration of bypass modes in nanoseconds. */ @@ -1153,6 +1166,8 @@ struct scx_event_stats { SCX_EVENT(SCX_EV_REENQ_IMMED); \ SCX_EVENT(SCX_EV_REENQ_LOCAL_REPEAT); \ SCX_EVENT(SCX_EV_REFILL_SLICE_DFL); \ + SCX_EVENT(SCX_EV_SLICE_CLAMPED); \ + SCX_EVENT(SCX_EV_SLICE_DENIED); \ SCX_EVENT(SCX_EV_BYPASS_DURATION); \ SCX_EVENT(SCX_EV_BYPASS_DISPATCH); \ SCX_EVENT(SCX_EV_BYPASS_ACTIVATE); \ diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index 13e9dec56a6a..3b15a10b8c8f 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -130,9 +130,20 @@ static inline u64 scx_caps_implied(u64 cap) return 0; } +/* may @p keep running on @rq's cpu? requires baseline cpu access */ +static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p) +{ + /* a migration-disabled task is let in without caps, keep it likewise */ + if (unlikely(is_migration_disabled(p))) + return true; + + return likely(!scx_missing_caps(scx_task_sched(p), cpu_of(rq), SCX_CAP_BASE)); +} + #else /* CONFIG_EXT_SUB_SCHED */ static inline u64 scx_missing_caps(struct scx_sched *sch, s32 cpu, u64 needed) { return 0; } +static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p) { return true; } #endif /* CONFIG_EXT_SUB_SCHED */ -- cgit v1.2.3 From 701b7bcad8681b161de791579aeb3de285f9a20f Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 13 Jul 2026 22:18:43 -1000 Subject: sched_ext: Add the SCX_CAP_ENQ cap Add SCX_CAP_ENQ, which gates inserting tasks onto a cid's local DSQ. Unlike IMMED enqueue, plain enqueues can pile up, so ENQ is the stronger cap and implies ENQ_IMMED. Losing ENQ also triggers the reenq scan. The scan tests each queued task and the running task against the cap each needs via scx_caps_for_task(), so an ENQ-only loss reenqueues plain tasks, evicting a running one, while IMMED tasks, which need only ENQ_IMMED, stay put. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 1 + kernel/sched/ext/internal.h | 14 +++++++++++--- kernel/sched/ext/sub.h | 12 ++++++++++-- 3 files changed, 22 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 39b17626c398..90dc3ef4b71f 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -4963,6 +4963,7 @@ SCX_ATTR(events); #ifdef CONFIG_EXT_SUB_SCHED static const char *scx_cap_names[__SCX_NR_CAPS] = { [__SCX_CAP_ENQ_IMMED] = "enq_immed", + [__SCX_CAP_ENQ] = "enq", }; static ssize_t scx_attr_caps_show(struct kobject *kobj, diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index ab1dfad28cb5..86f9db116a8e 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1224,8 +1224,8 @@ struct scx_sched_pcpu { /* * pshard->caps[cap_bit] is the set of cids the sched holds that one * cap on. ecaps is its transpose: the set of SCX_CAP_* bits the sched - * holds on this cpu, collected so that the hot-path check is a single - * read. + * effectively holds on this cpu, with implied caps folded in, so that + * the hot-path check is a single read. * * While pshard->caps[] under pshard->lock is the target configuration, * ecaps is the effective copy owned by the cpu. It is written under the @@ -1287,20 +1287,28 @@ struct scx_sched_pnode { * the allocation pattern. * * ENQ_IMMED insert an IMMED task onto the cid's local DSQ + * + * ENQ insert any task onto the cid's local DSQ (implies ENQ_IMMED) + * + * Implied caps apply to the holder's own use of a cid, not to delegation. + * scx_bpf_sub_grant() delegates literally-held caps, so a cap held only through + * implication is usable but cannot be re-delegated to a child. */ enum scx_cap_flags { __SCX_CAP_ENQ_IMMED = 0, + __SCX_CAP_ENQ = 1, __SCX_NR_CAPS, __SCX_CAP_ALL = BIT_U64(__SCX_NR_CAPS) - 1, SCX_CAP_ENQ_IMMED = BIT_U64(__SCX_CAP_ENQ_IMMED), + SCX_CAP_ENQ = BIT_U64(__SCX_CAP_ENQ), /* alias for minimal cap to make any use of a cpu */ SCX_CAP_BASE = SCX_CAP_ENQ_IMMED, /* caps whose loss strands queued tasks, see scx_process_sync_ecaps() */ - SCX_CAPS_REENQ_ON_LOSS = SCX_CAP_ENQ_IMMED, + SCX_CAPS_REENQ_ON_LOSS = SCX_CAP_ENQ_IMMED | SCX_CAP_ENQ, }; #ifdef CONFIG_EXT_SUB_SCHED diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index 3b15a10b8c8f..e531273b8708 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -115,18 +115,26 @@ static inline u64 scx_caps_for_enq(u64 enq_flags) /* a restored task must be put into the local DSQ regardless of caps */ if (enq_flags & SCX_ENQ_IGNORE_CAPS) return 0; - return SCX_CAP_ENQ_IMMED; + if (enq_flags & SCX_ENQ_IMMED) + return SCX_CAP_ENQ_IMMED; + return SCX_CAP_ENQ; } /* map queued @p to the SCX_CAP_* bit required to stay on its local DSQ */ static inline u64 scx_caps_for_task(struct task_struct *p) { - return SCX_CAP_ENQ_IMMED; + if (p->scx.flags & SCX_TASK_IMMED) + return SCX_CAP_ENQ_IMMED; + return SCX_CAP_ENQ; } /* caps implied by holding @cap */ static inline u64 scx_caps_implied(u64 cap) { + switch (cap) { + case SCX_CAP_ENQ: + return SCX_CAP_ENQ_IMMED; + } return 0; } -- cgit v1.2.3 From 6ea3be36808100336dffb4b04a6e5a483a26a649 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 13 Jul 2026 22:18:44 -1000 Subject: sched_ext: Gate kicks on SCX_CAP_BASE and preemption on SCX_CAP_PREEMPT A kick forces a scheduling event on the target cpu, and a preemption also evicts the running task. Gate both on caps. Any kick requires baseline access on the cid, and preempting a task the sub-sched does not own - whether by a SCX_ENQ_PREEMPT insert or a SCX_KICK_PREEMPT kick - requires the new SCX_CAP_PREEMPT. Gating either alone would leave a hole - the weakest cap authorizing preempting kicks, or plain kicks disturbing cpus the kicker has no access to. Preempting the sched's own subtree is always allowed, and the cap extends the right to any task on the cid. PREEMPT implies ENQ, and so ENQ_IMMED. A preempting insert tests the running task under the target rq lock and is rejected and reenqueued unless the victim is in the inserter's subtree or it holds PREEMPT. A migration-disabled task is admitted regardless, but with SCX_ENQ_PREEMPT stripped. Kicks are enforced on the delivery path, where the effective caps can be read coherently under the target rq's lock. A kick from a sub-sched lacking SCX_CAP_BASE on the cid is dropped, and a SCX_KICK_PREEMPT kick without PREEMPT for a task outside the kicker's subtree degrades to a plain reschedule. Unlike the enqueue caps, PREEMPT is checked only at the instant of the insert or kick, never as a standing property of a queued task. v2: Clear SCX_ENQ_PREEMPT on the offline and migration_pending force-admits. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 34 +++++++++++++++++++++++++--------- kernel/sched/ext/internal.h | 24 ++++++++++++++++++++++-- kernel/sched/ext/sub.c | 19 +++++++++++++++---- kernel/sched/ext/sub.h | 15 +++++++++++++++ 4 files changed, 77 insertions(+), 15 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 90dc3ef4b71f..154ef4ada614 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -294,7 +294,7 @@ static bool u32_before(u32 a, u32 b) * * Test whether @sch is a descendant of @ancestor. */ -static bool scx_is_descendant(struct scx_sched *sch, struct scx_sched *ancestor) +bool scx_is_descendant(struct scx_sched *sch, struct scx_sched *ancestor) { if (sch->level < ancestor->level) return false; @@ -4964,6 +4964,7 @@ SCX_ATTR(events); static const char *scx_cap_names[__SCX_NR_CAPS] = { [__SCX_CAP_ENQ_IMMED] = "enq_immed", [__SCX_CAP_ENQ] = "enq", + [__SCX_CAP_PREEMPT] = "preempt", }; static ssize_t scx_attr_caps_show(struct kobject *kobj, @@ -7898,13 +7899,22 @@ static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_r * During CPU hotplug, a CPU may depend on kicking itself to make * forward progress. Allow kicking self regardless of online state. If * @cpu is running a higher class task, we have no control over @cpu. - * Skip kicking. + * Skip kicking. A sub-sched lacking baseline access on @cid has no + * business forcing a reschedule there - skip. This is the authoritative + * cap check: ecaps is read here under @rq's lock. */ if ((cpu_online(cpu) || cpu == cpu_of(this_rq)) && - !sched_class_above(cur_class, &ext_sched_class)) { + !sched_class_above(cur_class, &ext_sched_class) && + !scx_missing_caps(pcpu->sch, cpu, SCX_CAP_BASE)) { if (cpumask_test_cpu(cpu, pcpu->cpus_to_preempt)) { - if (cur_class == &ext_sched_class) - set_task_slice(rq->curr, 0); + if (cur_class == &ext_sched_class) { + if (likely(!scx_missing_caps(pcpu->sch, cpu, + scx_caps_for_preempt(pcpu->sch, rq)))) + set_task_slice(rq->curr, 0); + else + __scx_add_event(pcpu->sch, + SCX_EV_SUB_PREEMPT_DENIED, 1); + } cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt); } @@ -7928,15 +7938,18 @@ static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_r return should_wait; } -static void kick_one_cpu_if_idle(s32 cpu, struct rq *this_rq) +static void kick_one_cpu_if_idle(s32 cpu, struct scx_sched_pcpu *pcpu, + struct rq *this_rq) { struct rq *rq = cpu_rq(cpu); unsigned long flags; raw_spin_rq_lock_irqsave(rq, flags); + /* idle kicks need baseline access too, see kick_one_cpu() */ if (!can_skip_idle_kick(rq) && - (cpu_online(cpu) || cpu == cpu_of(this_rq))) + (cpu_online(cpu) || cpu == cpu_of(this_rq)) && + !scx_missing_caps(pcpu->sch, cpu, SCX_CAP_BASE)) resched_curr(rq); raw_spin_rq_unlock_irqrestore(rq, flags); @@ -7973,7 +7986,7 @@ static void kick_cpus_irq_workfn(struct irq_work *irq_work) } for_each_cpu(cpu, pcpu->cpus_to_kick_if_idle) { - kick_one_cpu_if_idle(cpu, this_rq); + kick_one_cpu_if_idle(cpu, pcpu, this_rq); cpumask_clear_cpu(cpu, pcpu->cpus_to_kick_if_idle); } } @@ -9010,7 +9023,10 @@ __bpf_kfunc void scx_bpf_kick_cpu(s32 cpu, u64 flags, const struct bpf_prog_aux * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * * cid-addressed equivalent of scx_bpf_kick_cpu(). An invalid @cid aborts the - * scheduler via scx_cid_to_cpu(). + * scheduler via scx_cid_to_cpu(). Caps are enforced on the delivery path: a + * kick is dropped if the caller lacks baseline access on @cid, and a + * %SCX_KICK_PREEMPT degrades to a plain reschedule if the caller lacks + * %SCX_CAP_PREEMPT for a task outside its subtree. */ __bpf_kfunc void scx_bpf_kick_cid(s32 cid, u64 flags, const struct bpf_prog_aux *aux) { diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 86f9db116a8e..abbcaab8f35a 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1155,6 +1155,13 @@ struct scx_event_stats { * it can't be rejected. The violation is counted here. */ s64 SCX_EV_SUB_FORCED_ADMIT; + + /* + * The number of times a preempting kick was refused because the + * sub-sched lacked SCX_CAP_PREEMPT for a task outside its subtree. The + * kick degrades to a plain reschedule. + */ + s64 SCX_EV_SUB_PREEMPT_DENIED; }; #define SCX_EVENTS_LIST(SCX_EVENT) \ @@ -1173,7 +1180,8 @@ struct scx_event_stats { SCX_EVENT(SCX_EV_BYPASS_ACTIVATE); \ SCX_EVENT(SCX_EV_INSERT_NOT_OWNED); \ SCX_EVENT(SCX_EV_SUB_BYPASS_DISPATCH); \ - SCX_EVENT(SCX_EV_SUB_FORCED_ADMIT) + SCX_EVENT(SCX_EV_SUB_FORCED_ADMIT); \ + SCX_EVENT(SCX_EV_SUB_PREEMPT_DENIED) struct scx_sched; @@ -1287,22 +1295,33 @@ struct scx_sched_pnode { * the allocation pattern. * * ENQ_IMMED insert an IMMED task onto the cid's local DSQ + * - kick the cid's cpu (except SCX_KICK_PREEMPT) * * ENQ insert any task onto the cid's local DSQ (implies ENQ_IMMED) * + * PREEMPT preempt any task running on the cid regardless of the owning + * sched (implies ENQ). Preempting a task in the sched's own subtree + * doesn't require any cap. + * - SCX_ENQ_PREEMPT inserts + * - SCX_KICK_PREEMPT kicks + * * Implied caps apply to the holder's own use of a cid, not to delegation. * scx_bpf_sub_grant() delegates literally-held caps, so a cap held only through - * implication is usable but cannot be re-delegated to a child. + * implication is usable but cannot be re-delegated to a child. When granting a + * cap, it usually makes sense to delegate its implied caps explicitly alongside + * it. */ enum scx_cap_flags { __SCX_CAP_ENQ_IMMED = 0, __SCX_CAP_ENQ = 1, + __SCX_CAP_PREEMPT = 2, __SCX_NR_CAPS, __SCX_CAP_ALL = BIT_U64(__SCX_NR_CAPS) - 1, SCX_CAP_ENQ_IMMED = BIT_U64(__SCX_CAP_ENQ_IMMED), SCX_CAP_ENQ = BIT_U64(__SCX_CAP_ENQ), + SCX_CAP_PREEMPT = BIT_U64(__SCX_CAP_PREEMPT), /* alias for minimal cap to make any use of a cpu */ SCX_CAP_BASE = SCX_CAP_ENQ_IMMED, @@ -1914,6 +1933,7 @@ struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, struct scx_sched *parent); int scx_validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops); int scx_sched_sysfs_add(struct scx_sched *sch); +bool scx_is_descendant(struct scx_sched *sch, struct scx_sched *ancestor); extern raw_spinlock_t scx_sched_lock; extern struct mutex scx_enable_mutex; diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 11fda1d3828d..270473c67a41 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -221,11 +221,14 @@ void scx_init_root_caps(struct scx_sched *sch) * @sch: enqueuing sub-sched * @rq: rq whose local DSQ @p targets * @p: task being inserted - * @enq_flags: in/out; %SCX_ENQ_IMMED is cleared when diverting to reject + * @enq_flags: in/out, unhonored flags are cleared * * Return @rq's local DSQ if @sch holds the required caps on @rq's cid, * otherwise @rq's reject DSQ after recording the reenq reason on @p. * + * %SCX_ENQ_IMMED and %SCX_ENQ_PREEMPT are cleared when diverting to reject. + * %SCX_ENQ_PREEMPT is also cleared on a fallback migration-disabled admission. + * * Bypass doesn't need special-casing as a bypassing sched's tasks are enqueued * to and run by its nearest non-bypassing ancestor. If root is bypassing, it * always holds all caps. @@ -234,7 +237,12 @@ struct scx_dispatch_q *scx_local_or_reject_dsq(struct scx_sched *sch, struct rq struct task_struct *p, u64 *enq_flags) { s32 cid = __scx_cpu_to_cid(cpu_of(rq)); - u64 missing = scx_missing_caps(sch, cpu_of(rq), scx_caps_for_enq(*enq_flags)); + u64 needed = scx_caps_for_enq(*enq_flags); + u64 missing; + + if (*enq_flags & SCX_ENQ_PREEMPT) + needed |= scx_caps_for_preempt(sch, rq); + missing = scx_missing_caps(sch, cpu_of(rq), needed); /* requirements met */ if (likely(!missing)) @@ -244,10 +252,12 @@ struct scx_dispatch_q *scx_local_or_reject_dsq(struct scx_sched *sch, struct rq * The task must run on this CPU regardless of caps: the rq is draining * offline (BPF scheduler bypassed), the task is migration-disabled, or a * migration is pending. Admit despite the missing caps and count it. + * Refuse preemptions. */ if (unlikely(!scx_rq_online(rq) || is_migration_disabled(p) || p->migration_pending)) { __scx_add_event(sch, SCX_EV_SUB_FORCED_ADMIT, 1); + *enq_flags &= ~SCX_ENQ_PREEMPT; return &rq->scx.local_dsq; } @@ -257,9 +267,10 @@ struct scx_dispatch_q *scx_local_or_reject_dsq(struct scx_sched *sch, struct rq /* * Only local DSQ can honor IMMED and dsq_inc_nr() WARNs on IMMED into * others. Strip both the enq flag and the sticky task flag - the - * latter can carry in from an earlier admitted IMMED insert. + * latter can carry in from an earlier admitted IMMED insert. Strip + * PREEMPT too. */ - *enq_flags &= ~SCX_ENQ_IMMED; + *enq_flags &= ~(SCX_ENQ_IMMED | SCX_ENQ_PREEMPT); p->scx.flags &= ~SCX_TASK_IMMED; return &rq->scx.reject_dsq; diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index e531273b8708..dc3e07520024 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -128,10 +128,24 @@ static inline u64 scx_caps_for_task(struct task_struct *p) return SCX_CAP_ENQ; } +/* the cap @sch needs to preempt @rq's current task, 0 if none */ +static inline u64 scx_caps_for_preempt(struct scx_sched *sch, struct rq *rq) +{ + struct task_struct *curr = rq->curr; + + /* a non-ext task can't be preempted by ext, own-subtree needs no cap */ + if (curr->sched_class != &ext_sched_class || + scx_is_descendant(scx_task_sched(curr), sch)) + return 0; + return SCX_CAP_PREEMPT; +} + /* caps implied by holding @cap */ static inline u64 scx_caps_implied(u64 cap) { switch (cap) { + case SCX_CAP_PREEMPT: + return SCX_CAP_ENQ | SCX_CAP_ENQ_IMMED; case SCX_CAP_ENQ: return SCX_CAP_ENQ_IMMED; } @@ -151,6 +165,7 @@ static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p #else /* CONFIG_EXT_SUB_SCHED */ static inline u64 scx_missing_caps(struct scx_sched *sch, s32 cpu, u64 needed) { return 0; } +static inline u64 scx_caps_for_preempt(struct scx_sched *sch, struct rq *rq) { return 0; } static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p) { return true; } #endif /* CONFIG_EXT_SUB_SCHED */ -- cgit v1.2.3 From f2c9f5155ba8ee798e95b7a656d600cc2de687a0 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 13 Jul 2026 22:18:44 -1000 Subject: sched_ext: Authorize remote-move inserts against the placing scheduler scx_local_or_reject_dsq() authorizes a local-DSQ insert against the caps of the scheduler doing the insert. On the consume/dispatch paths that is the scheduler running balance_one(), passed down through scx_consume_dispatch_q() and move_local_task_to_local_dsq(), so the check is correct. The remote-move path loses it. move_remote_task_to_local_dsq() re-activates @p on the destination rq through enqueue_task_scx(), which reconstructs the scheduler from the task, i.e. @p's owner. When an ancestor places a descendant's task - e.g. draining a bypassed sub-scheduler - the owner is a sub-scheduler of the placer, so authorizing against the owner checks a narrower cap set and can spuriously reject a task the placer is entitled to run. Carry the placing scheduler across the activate_task() boundary the same way enq_flags already are, via a per-rq field set only for the duration of the re-activation, and have scx_local_or_reject_dsq() authorize against it. The placer's caps are a superset of the owner's, so this admits what the placer may run and keeps rejecting what it may not. v2: Document @sch in move_remote_task_to_local_dsq()'s kerneldoc. (Andrea) Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 28 ++++++++++++++++------------ kernel/sched/ext/sub.c | 9 +++++++-- kernel/sched/sched.h | 3 ++- 3 files changed, 25 insertions(+), 15 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 154ef4ada614..af46c3fee610 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -2268,6 +2268,7 @@ static void move_local_task_to_local_dsq(struct scx_sched *sch, /** * move_remote_task_to_local_dsq - Move a task from a foreign rq to a local DSQ + * @sch: scheduler placing @p * @p: task to move * @enq_flags: %SCX_ENQ_* * @src_rq: rq to move the task from, locked on entry, released on return @@ -2275,7 +2276,8 @@ static void move_local_task_to_local_dsq(struct scx_sched *sch, * * Move @p which is currently on @src_rq to @dst_rq's local DSQ. */ -static void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags, +static void move_remote_task_to_local_dsq(struct scx_sched *sch, + struct task_struct *p, u64 enq_flags, struct rq *src_rq, struct rq *dst_rq) { lockdep_assert_rq_held(src_rq); @@ -2291,15 +2293,19 @@ static void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags, switch_rq_lock(src_rq, dst_rq); /* - * We want to pass scx-specific enq_flags but activate_task() will - * truncate the upper 32 bit. As we own @rq, we can pass them through - * @rq->scx.remote_activate_enq_flags instead. + * activate_task() below truncates enq_flags to 32 bits and re-derives + * @p's owner, dropping our scx flags and the placing @sch. We own @rq, + * so stash both across the call. The enqueue reads them back, keeping + * the scx flags and checking caps against the placer, not the owner. */ WARN_ON_ONCE(!cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr)); - WARN_ON_ONCE(dst_rq->scx.remote_activate_enq_flags); + WARN_ON_ONCE(dst_rq->scx.remote_activate_enq_flags || + dst_rq->scx.remote_activate_sch); dst_rq->scx.remote_activate_enq_flags = enq_flags; + dst_rq->scx.remote_activate_sch = sch; activate_task(dst_rq, p, 0); dst_rq->scx.remote_activate_enq_flags = 0; + dst_rq->scx.remote_activate_sch = NULL; } /* @@ -2431,12 +2437,12 @@ static bool unlink_dsq_and_switch_rq_lock(struct task_struct *p, !WARN_ON_ONCE(src_rq != task_rq(p)); } -static bool consume_remote_task(struct rq *this_rq, +static bool consume_remote_task(struct scx_sched *sch, struct rq *this_rq, struct task_struct *p, u64 enq_flags, struct scx_dispatch_q *dsq, struct rq *src_rq) { if (unlink_dsq_and_switch_rq_lock(p, dsq, this_rq, src_rq)) { - move_remote_task_to_local_dsq(p, enq_flags, src_rq, this_rq); + move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, this_rq); return true; } else { switch_rq_lock(src_rq, this_rq); @@ -2497,8 +2503,7 @@ static struct rq *move_task_between_dsqs(struct scx_sched *sch, raw_spin_unlock(&src_dsq->lock); } else { raw_spin_unlock(&src_dsq->lock); - move_remote_task_to_local_dsq(p, enq_flags, - src_rq, dst_rq); + move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, dst_rq); } } else { /* @@ -2551,7 +2556,7 @@ retry: } if (task_can_run_on_remote_rq(sch, p, rq, false)) { - if (likely(consume_remote_task(rq, p, enq_flags, dsq, task_rq))) + if (likely(consume_remote_task(sch, rq, p, enq_flags, dsq, task_rq))) return true; goto retry; } @@ -2644,8 +2649,7 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, scx_dispatch_enqueue(sch, src_rq, find_global_dsq(sch, task_cpu(p)), p, enq_flags | SCX_ENQ_GDSQ_FALLBACK); } else { - move_remote_task_to_local_dsq(p, enq_flags, - src_rq, dst_rq); + move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, dst_rq); /* task has been moved to dst_rq, which is now locked */ locked_rq = dst_rq; } diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 270473c67a41..655632b5511f 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -237,12 +237,17 @@ struct scx_dispatch_q *scx_local_or_reject_dsq(struct scx_sched *sch, struct rq struct task_struct *p, u64 *enq_flags) { s32 cid = __scx_cpu_to_cid(cpu_of(rq)); + struct scx_sched *asch = rq->scx.remote_activate_sch ?: sch; u64 needed = scx_caps_for_enq(*enq_flags); u64 missing; + /* + * On a remote activation the scheduling sched (@asch) differs from + * @p's owner (@sch). Check caps against the scheduling sched. + */ if (*enq_flags & SCX_ENQ_PREEMPT) - needed |= scx_caps_for_preempt(sch, rq); - missing = scx_missing_caps(sch, cpu_of(rq), needed); + needed |= scx_caps_for_preempt(asch, rq); + missing = scx_missing_caps(asch, cpu_of(rq), needed); /* requirements met */ if (likely(!missing)) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index c0a4699a6c0a..cdd21c814008 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -800,8 +800,9 @@ struct scx_rq { struct list_head runnable_list; /* runnable tasks on this rq */ struct list_head ddsp_deferred_locals; /* deferred ddsps from enq */ unsigned long ops_qseq; - /* see move_remote_task_to_local_dsq() */ + /* both stashed across the activate_task() in move_remote_task_to_local_dsq() */ u64 remote_activate_enq_flags; + struct scx_sched *remote_activate_sch; u32 nr_running; u32 cpuperf_target; /* [0, SCHED_CAPACITY_SCALE] */ bool in_select_cpu; -- cgit v1.2.3 From ca3aec453d64d2ad89156c10a2e1b15a93b9fe50 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 13 Jul 2026 22:18:44 -1000 Subject: sched_ext: Route ops.update_idle() to sub-schedulers and re-notify owed scheds __scx_update_idle() notified only the root scheduler. A sub-scheduler that holds a cid needs that cid's idle state to place and kick on it. Deliver ops.update_idle() to every scheduler that holds SCX_CAP_BASE on the transitioning cid. The root holds every cap, so a real transition always reaches it. Real transitions are not enough on their own. A cid that is already idle when a sub-sched gains baseline access produces no transition, so the new holder would never learn it is idle. The ecaps sync arms a re-notify on the gain, and the next idle pick delivers ops.update_idle() to just that sched, leaving holders that already track the cpu untouched. A matching loss of baseline access drops any pending re-notify. Bypass suppresses ops.update_idle() too, so a cpu that goes idle during a bypass window and stays idle yields no transition to re-deliver on un-bypass. Arm the same re-notify for every sched leaving bypass. The acute case is a child granted cids during its own ops.sub_attach(). The grant lands while the child is bypassed and the notify walk skips it, so on un-bypass it holds cids it never saw go idle. The root is owed the same and is armed through a separate per-rq flag, which keeps this working when sub-schedulers are compiled out. v2: Gate the idle catch-up in pick_task_idle() to avoid a double ops.update_idle(). (sashiko AI) Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 40 +++++++++++++++++++-- kernel/sched/ext/idle.c | 86 ++++++++++++++++++++++++++++++++------------- kernel/sched/ext/internal.h | 2 ++ kernel/sched/ext/sub.c | 24 ++++++++++++- kernel/sched/idle.c | 8 ++++- kernel/sched/sched.h | 2 ++ 6 files changed, 133 insertions(+), 29 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index af46c3fee610..591b7ee5349f 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -5541,6 +5541,38 @@ void scx_disable_bypass_dsp(struct scx_sched *sch) } } +/** + * unbypass_renotify_idle - Arm an idle re-notify for a sched leaving bypass + * @rq: rq of the cpu leaving bypass + * @pos: scheduler that just left bypass on @rq's cpu + * @pcpu: @pos's per-cpu state for @rq's cpu + * + * A sched leaving bypass is owed the ops.update_idle() calls suppressed while + * bypassing. A cpu that goes idle during the bypass window and stays idle won't + * produce a notification. Arm a re-notify that scx_bypass()'s resched flushes + * on the next idle pick. + * + * An acute case is ops.sub_attach(). If the parent grants the child cids while + * attaching, when attach is complete and bypass is lifted, the child may hold + * idle cids it never saw go idle. + * + * The root is no exception as bypass suppresses its notifications the same way. + * However, the root uses a separate per-rq flag so its re-notify keeps working + * even when !CONFIG_EXT_SUB_SCHED. + */ +static void unbypass_renotify_idle(struct rq *rq, struct scx_sched *pos, + struct scx_sched_pcpu *pcpu) +{ + if (pos == scx_root) { + rq->scx.flags |= SCX_RQ_ROOT_IDLE_RENOTIFY; + return; + } +#ifdef CONFIG_EXT_SUB_SCHED + pcpu->idle_renotify = true; + rq->scx.flags |= SCX_RQ_SUB_IDLE_RENOTIFY; +#endif +} + /** * scx_bypass - [Un]bypass scx_ops and guarantee forward progress * @sch: sched to bypass @@ -5624,11 +5656,15 @@ void scx_bypass(struct scx_sched *sch, bool bypass) scx_for_each_descendant_pre(pos, sch) { struct scx_sched_pcpu *pcpu = per_cpu_ptr(pos->pcpu, cpu); + bool was_bypassing = pcpu->flags & SCX_SCHED_PCPU_BYPASSING; - if (pos->bypass_depth) + if (pos->bypass_depth) { pcpu->flags |= SCX_SCHED_PCPU_BYPASSING; - else + } else { pcpu->flags &= ~SCX_SCHED_PCPU_BYPASSING; + if (was_bypassing) + unbypass_renotify_idle(rq, pos, pcpu); + } } raw_spin_unlock(&scx_sched_lock); diff --git a/kernel/sched/ext/idle.c b/kernel/sched/ext/idle.c index 8e8c6201b7df..16ebe3ab8647 100644 --- a/kernel/sched/ext/idle.c +++ b/kernel/sched/ext/idle.c @@ -12,6 +12,7 @@ #include "internal.h" #include "cid.h" #include "idle.h" +#include "sub.h" /* Enable/disable built-in idle CPU selection policy */ static DEFINE_STATIC_KEY_FALSE(scx_builtin_idle_enabled); @@ -730,6 +731,46 @@ static void update_builtin_idle(int cpu, bool idle) } } +/* + * Notify schedulers of an idle transition on @cpu's cid, delivering to every + * sched that holds %SCX_CAP_BASE on the cid (the root holds every cap). A real + * transition (@do_notify) reaches all holders. A forced one (@root_renotify for + * the root, a sub-sched's idle_renotify marker for a sub) reaches only the owed + * scheds. + */ +static void scx_idle_notify(struct rq *rq, bool idle, bool do_notify, bool root_renotify) +{ + s32 cpu = cpu_of(rq); + s32 cid = scx_cpu_arg(cpu); + struct scx_sched *pos; + + lockdep_assert_rq_held(rq); + + pos = scx_next_descendant_pre(NULL, scx_root); + while (pos) { + bool forced = false; + + if (unlikely(scx_missing_caps(pos, cpu, SCX_CAP_BASE))) { + pos = scx_skip_subtree_pre(pos, scx_root); + continue; + } + + if (pos == scx_root) { + forced = root_renotify; + } +#ifdef CONFIG_EXT_SUB_SCHED + else if (per_cpu_ptr(pos->pcpu, cpu)->idle_renotify) { + per_cpu_ptr(pos->pcpu, cpu)->idle_renotify = false; + forced = true; + } +#endif + if ((do_notify || forced) && SCX_HAS_OP(pos, update_idle) && + !scx_bypassing(pos, cpu)) + SCX_CALL_OP(pos, update_idle, rq, cid, idle); + pos = scx_next_descendant_pre(pos, scx_root); + } +} + /* * Update the idle state of a CPU to @idle. * @@ -748,44 +789,39 @@ static void update_builtin_idle(int cpu, bool idle) */ void __scx_update_idle(struct rq *rq, bool idle, bool do_notify) { - struct scx_sched *sch = scx_root; int cpu = cpu_of(rq); lockdep_assert_rq_held(rq); /* - * Update the idle masks: - * - for real idle transitions (do_notify == true) - * - for idle-to-idle transitions (indicated by the previous task - * being the idle thread, managed by pick_task_idle()) - * - * Skip updating idle masks if the previous task is not the idle - * thread, since set_next_task_idle() has already handled it when - * transitioning from a task to the idle thread (calling this - * function with do_notify == true). - * - * In this way we can avoid updating the idle masks twice, - * unnecessarily. + * pick_task_idle() calls here only on an idle-to-idle re-pick and the + * transitions call with @do_notify, so every reaching call updates the + * masks. */ if (static_branch_likely(&scx_builtin_idle_enabled)) - if (do_notify || is_idle_task(rq->curr)) - update_builtin_idle(cpu, idle); + update_builtin_idle(cpu, idle); /* - * Trigger ops.update_idle() only when transitioning from a task to - * the idle thread and vice versa. - * - * Idle transitions are indicated by do_notify being set to true, - * managed by put_prev_task_idle()/set_next_task_idle(). + * ops.update_idle() fires on real idle transitions, indicated by + * @do_notify and managed by put_prev_task_idle()/set_next_task_idle(). + * An idle pick also fires it to flush a forced notify owed to a sched + * that missed transitions while bypassed or on a cid it just gained. + * unbypass_renotify_idle() and scx_process_sync_ecaps() arm the per-rq + * gates, and scx_idle_notify() targets the owed scheds. * - * This must come after builtin idle update so that BPF schedulers can - * create interlocking between ops.update_idle() and ops.enqueue() - + * This must come after the builtin idle update so that BPF schedulers + * can create interlocking between ops.update_idle() and ops.enqueue() - * either enqueue() sees the idle bit or update_idle() sees the task * that enqueue() queued. */ - if (SCX_HAS_OP(sch, update_idle) && do_notify && - !scx_bypassing(sch, cpu_of(rq))) - SCX_CALL_OP(sch, update_idle, rq, scx_cpu_arg(cpu_of(rq)), idle); + if (do_notify || + (idle && (rq->scx.flags & + (SCX_RQ_SUB_IDLE_RENOTIFY | SCX_RQ_ROOT_IDLE_RENOTIFY)))) { + bool root_renotify = rq->scx.flags & SCX_RQ_ROOT_IDLE_RENOTIFY; + + rq->scx.flags &= ~(SCX_RQ_SUB_IDLE_RENOTIFY | SCX_RQ_ROOT_IDLE_RENOTIFY); + scx_idle_notify(rq, idle, do_notify, root_renotify); + } } static void reset_idle_masks(struct sched_ext_ops *ops) diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index abbcaab8f35a..97d7ce89dd5f 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1244,6 +1244,8 @@ struct scx_sched_pcpu { */ u64 ecaps; struct llist_node ecaps_to_sync_node; + /* owed a forced update_idle() re-notify on this cpu */ + bool idle_renotify; /* effective caps as of the last sub_ecaps_updated() delivery */ u64 reported_ecaps; #endif diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 655632b5511f..a91480debcc2 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -483,6 +483,10 @@ static void discard_queued_syncs(struct rq *rq) * pshard->caps[] is the target configuration. pcpu->ecaps is the effective * transposed copy owned by the cid's cpu and written only here under @rq's * lock. + * + * A sched that newly gains baseline access here is owed an update_idle() so it + * learns the cid's idle state. Such a gain arms the per-rq + * %SCX_RQ_SUB_IDLE_RENOTIFY gate so the next idle pick delivers it. */ void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) { @@ -518,7 +522,7 @@ void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) struct scx_sched_pcpu *pcpu = container_of(pos, struct scx_sched_pcpu, ecaps_to_sync_node); struct scx_pshard *ps = pcpu->sch->pshard[shard]; - u64 old, ecaps, lost; + u64 old, ecaps, lost, gained; init_llist_node(pos); @@ -530,6 +534,7 @@ void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) WRITE_ONCE(pcpu->ecaps, ecaps); lost = old & ~ecaps; + gained = ecaps & ~old; lost_all |= lost; /* tell the sched its effective caps on this cid changed */ @@ -547,6 +552,18 @@ void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) scx_flush_dispatch_buf(pcpu->sch, rq); pcpu->reported_ecaps = ecaps; } + + /* + * Gaining baseline access owes an update_idle() so the sched + * learns the cpu's idle state. Arm the per-rq gate so the next + * idle pick flushes it. Losing access drops any pending notify. + */ + if (gained & SCX_CAP_BASE) { + pcpu->idle_renotify = true; + rq->scx.flags |= SCX_RQ_SUB_IDLE_RENOTIFY; + } else if (lost & SCX_CAP_BASE) { + pcpu->idle_renotify = false; + } } /* @@ -1441,6 +1458,11 @@ __bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, caps_updated_record(cps, changed_cids, granted_caps, &to_deliver); + /* + * The sync arms an update_idle() re-notify if + * the cid gains baseline access, so the holder + * learns of an already-idle cid. + */ scx_cmask_for_each_cid(cid, changed_cids) queue_sync_ecaps(child, cid); } diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c index 052435f4d3e3..eb73b65ce6c4 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -503,7 +503,13 @@ static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool fir struct task_struct *pick_task_idle(struct rq *rq, struct rq_flags *rf) { - scx_update_idle(rq, true, false); + /* + * Notify scx only on an idle-to-idle re-pick (the cpu was already idle). + * A real task->idle transition is delivered by set_next_task_idle(), so + * calling here too would duplicate it. + */ + if (scx_enabled() && is_idle_task(rq->curr)) + scx_update_idle(rq, true, false); return rq->idle; } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index cdd21c814008..54cff94556c0 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -787,6 +787,8 @@ enum scx_rq_flags { SCX_RQ_BAL_KEEP = 1 << 3, /* balance decided to keep current */ SCX_RQ_CLK_VALID = 1 << 5, /* RQ clock is fresh and valid */ SCX_RQ_BAL_CB_PENDING = 1 << 6, /* must queue a cb after dispatching */ + SCX_RQ_SUB_IDLE_RENOTIFY = 1 << 7, /* sub-scheds are owed update_idle() */ + SCX_RQ_ROOT_IDLE_RENOTIFY = 1 << 8, /* the root is owed update_idle() */ SCX_RQ_IN_WAKEUP = 1 << 16, SCX_RQ_IN_BALANCE = 1 << 17, -- cgit v1.2.3 From 75c268ed57f2197a4f1fd9259f42cb6065dd311b Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 13 Jul 2026 22:18:44 -1000 Subject: sched_ext: Replay ecaps notifications suppressed by bypass scx_process_sync_ecaps() consumes ecaps syncs while the sched is bypassing without delivering ops.sub_ecaps_updated(), leaving reported_ecaps stale. Nothing re-queued a sync when bypass lifted, so a cid whose caps never change again would never be notified. Attach-time initial grants hit this every time: they are consumed during the enable bypass window, so a sched never learned its initial effective caps through the callback. Re-queue a sync for every (sched, cpu) with an undelivered delta at the per-cpu bypass exit in scx_bypass(), next to the idle renotify catch-up. The next balance on the cpu then delivers the pending delta with proper dispatch context. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 4 +++- kernel/sched/ext/sub.c | 35 +++++++++++++++++++++++++++++++++++ kernel/sched/ext/sub.h | 2 ++ 3 files changed, 40 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 591b7ee5349f..945f6192ca41 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -5662,8 +5662,10 @@ void scx_bypass(struct scx_sched *sch, bool bypass) pcpu->flags |= SCX_SCHED_PCPU_BYPASSING; } else { pcpu->flags &= ~SCX_SCHED_PCPU_BYPASSING; - if (was_bypassing) + if (was_bypassing) { unbypass_renotify_idle(rq, pos, pcpu); + scx_unbypass_replay_ecaps(rq, pos); + } } } diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index a91480debcc2..bdcbde37d0d2 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -575,6 +575,41 @@ void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) scx_schedule_reenq_local(rq, SCX_REENQ_CAP_REVOKE); } +/** + * scx_unbypass_replay_ecaps - Replay a bypass-suppressed ecaps notification + * @rq: rq of the cpu leaving bypass + * @sch: scheduler that just left bypass on @rq's cpu + * + * scx_process_sync_ecaps() consumes syncs while bypassing without delivering + * ops.sub_ecaps_updated(), leaving reported_ecaps stale. Nothing re-queues a + * sync when bypass lifts, so without a replay a cid that never changes again + * would never be notified. The attach-time initial grants are the acute case + * as they are consumed during the enable bypass window. Re-queue a sync for + * any undelivered delta so the next balance delivers it. + */ +void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch) +{ + s32 cpu = cpu_of(rq); + struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); + struct scx_pshard *ps; + s32 cid; + + lockdep_assert_rq_held(rq); + + /* root holds every cap and never uses ecaps */ + if (!sch->level) + return; + + if (READ_ONCE(pcpu->ecaps) == pcpu->reported_ecaps) + return; + + cid = __scx_cpu_to_cid(cpu); + ps = sch->pshard[scx_cid_to_shard[cid]]; + + guard(raw_spinlock)(&ps->lock); + queue_sync_ecaps(sch, cid); +} + /* * A cpu came back. Re-seed each sub-sched's ecaps on the cpu's cid. The sync * recomputes effective caps from the pshard and fires ops.sub_ecaps_updated() diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index dc3e07520024..f72c18a5972a 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -30,6 +30,7 @@ void scx_free_pshards(struct scx_sched *sch); s32 scx_alloc_pshards(struct scx_sched *sch); void scx_init_root_caps(struct scx_sched *sch); void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev); +void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch); void scx_online_ecaps(struct rq *rq); void scx_offline_ecaps(struct rq *rq); void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu); @@ -58,6 +59,7 @@ static inline void scx_free_pshards(struct scx_sched *sch) {} static inline s32 scx_alloc_pshards(struct scx_sched *sch) { return 0; } static inline void scx_init_root_caps(struct scx_sched *sch) {} static inline void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) {} +static inline void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch) {} static inline void scx_online_ecaps(struct rq *rq) {} static inline void scx_offline_ecaps(struct rq *rq) {} static inline void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu) {} -- cgit v1.2.3 From b0a2ca6ae0ed03c625cd343d662d83bc94d84217 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 13 Jul 2026 22:18:44 -1000 Subject: sched_ext: Add scx_bpf_sub_kill() to evict a child sub-scheduler A cid-form scheduler can grant caps to and revoke them from its child sub-schedulers but has no way to tear one down. Add scx_bpf_sub_kill() to evict a direct child with a printf-style reason that reaches the child's scx_exit_info. No exit code is taken because the child is a separate scheduler whose exit-code semantics the parent cannot know. The child and its subtree are disabled through the usual async path under a new exit kind, SCX_EXIT_PARENT_KILL. The bstr formatting infrastructure in ext.c is exposed through internal.h with scx_ prefixes so the kfunc, which lives in sub.c, can format the reason. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 28 +++++++++------- kernel/sched/ext/internal.h | 11 ++++++ kernel/sched/ext/sub.c | 57 ++++++++++++++++++++++++++++++++ tools/sched_ext/include/scx/common.bpf.h | 18 ++++++++++ 4 files changed, 102 insertions(+), 12 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 945f6192ca41..f8f8597b6f78 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -187,14 +187,8 @@ static const struct rhashtable_params dsq_hash_params = { static LLIST_HEAD(dsqs_to_free); -/* string formatting from BPF */ -struct scx_bstr_buf { - u64 data[MAX_BPRINTF_VARARGS]; - char line[SCX_EXIT_MSG_LEN]; -}; - -static DEFINE_RAW_SPINLOCK(scx_exit_bstr_buf_lock); -static struct scx_bstr_buf scx_exit_bstr_buf; +DEFINE_RAW_SPINLOCK(scx_exit_bstr_buf_lock); +struct scx_bstr_buf scx_exit_bstr_buf; /* ops debug dump */ static DEFINE_RAW_SPINLOCK(scx_dump_lock); @@ -5755,6 +5749,8 @@ static const char *scx_exit_reason(enum scx_exit_kind kind) return "disabled by sysrq-S"; case SCX_EXIT_PARENT: return "parent exiting"; + case SCX_EXIT_PARENT_KILL: + return "killed by parent scheduler"; case SCX_EXIT_ERROR: return "runtime error"; case SCX_EXIT_ERROR_BPF: @@ -9370,8 +9366,8 @@ static s32 __bstr_format(struct scx_sched *sch, u64 *data_buf, char *line_buf, } __printf(3, 0) -static s32 bstr_format(struct scx_sched *sch, struct scx_bstr_buf *buf, - char *fmt, unsigned long long *data, u32 data__sz) +s32 scx_bstr_format(struct scx_sched *sch, struct scx_bstr_buf *buf, + char *fmt, unsigned long long *data, u32 data__sz) { return __bstr_format(sch, buf->data, buf->line, sizeof(buf->line), fmt, data, data__sz); @@ -9401,7 +9397,7 @@ __bpf_kfunc void scx_bpf_exit_bstr(s64 exit_code, char *fmt, raw_spin_lock_irqsave(&scx_exit_bstr_buf_lock, flags); sch = scx_prog_sched(aux); if (likely(sch) && - bstr_format(sch, &scx_exit_bstr_buf, fmt, data, data__sz) >= 0) + scx_bstr_format(sch, &scx_exit_bstr_buf, fmt, data, data__sz) >= 0) scx_exit(sch, SCX_EXIT_UNREG_BPF, exit_code, "%s", scx_exit_bstr_buf.line); raw_spin_unlock_irqrestore(&scx_exit_bstr_buf_lock, flags); } @@ -9426,7 +9422,7 @@ __bpf_kfunc void scx_bpf_error_bstr(char *fmt, unsigned long long *data, raw_spin_lock_irqsave(&scx_exit_bstr_buf_lock, flags); sch = scx_prog_sched(aux); if (likely(sch) && - bstr_format(sch, &scx_exit_bstr_buf, fmt, data, data__sz) >= 0) + scx_bstr_format(sch, &scx_exit_bstr_buf, fmt, data, data__sz) >= 0) scx_exit(sch, SCX_EXIT_ERROR_BPF, 0, "%s", scx_exit_bstr_buf.line); raw_spin_unlock_irqrestore(&scx_exit_bstr_buf_lock, flags); } @@ -10062,6 +10058,13 @@ __bpf_kfunc s32 scx_bpf_sub_caps(u64 cgroup_id, u64 caps, struct scx_cmask *out_ { return -EOPNOTSUPP; } + +__bpf_kfunc s32 scx_bpf_sub_kill_bstr(u64 cgroup_id, char *fmt, + unsigned long long *data, u32 data__sz, + const struct bpf_prog_aux *aux) +{ + return -EOPNOTSUPP; +} #endif /* !CONFIG_EXT_SUB_SCHED */ __bpf_kfunc_end_defs(); @@ -10111,6 +10114,7 @@ BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_IMPLICIT_ARGS | KF_RCU | KF_ACQUIRE) BTF_ID_FLAGS(func, scx_bpf_sub_grant, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_sub_revoke, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_sub_caps, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_sub_kill_bstr, KF_IMPLICIT_ARGS) BTF_KFUNCS_END(scx_kfunc_ids_any) static const struct btf_kfunc_id_set scx_kfunc_set_any = { diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 97d7ce89dd5f..d7a1d6a14ebf 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -51,6 +51,7 @@ enum scx_exit_kind { SCX_EXIT_UNREG_KERN, /* kernel-initiated unregistration */ SCX_EXIT_SYSRQ, /* requested by 'S' sysrq */ SCX_EXIT_PARENT, /* parent exiting */ + SCX_EXIT_PARENT_KILL, /* killed by parent scheduler */ SCX_EXIT_ERROR = 1024, /* runtime error, error msg contains details */ SCX_EXIT_ERROR_BPF, /* ERROR but triggered through scx_bpf_error() */ @@ -1876,6 +1877,12 @@ struct scx_enable_cmd { int ret; }; +/* string formatting from BPF */ +struct scx_bstr_buf { + u64 data[MAX_BPRINTF_VARARGS]; + char line[SCX_EXIT_MSG_LEN]; +}; + extern struct scx_sched __rcu *scx_root; DECLARE_PER_CPU(struct rq *, scx_locked_rq_state); @@ -1936,10 +1943,14 @@ struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, int scx_validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops); int scx_sched_sysfs_add(struct scx_sched *sch); bool scx_is_descendant(struct scx_sched *sch, struct scx_sched *ancestor); +__printf(3, 0) s32 scx_bstr_format(struct scx_sched *sch, struct scx_bstr_buf *buf, + char *fmt, unsigned long long *data, u32 data__sz); extern raw_spinlock_t scx_sched_lock; extern struct mutex scx_enable_mutex; extern struct percpu_rw_semaphore scx_fork_rwsem; +extern raw_spinlock_t scx_exit_bstr_buf_lock; +extern struct scx_bstr_buf scx_exit_bstr_buf; #ifdef CONFIG_EXT_SUB_SCHED extern const struct rhashtable_params scx_sched_hash_params; extern struct rhashtable scx_sched_hash; diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index bdcbde37d0d2..68ab3675337f 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -1694,6 +1694,63 @@ __bpf_kfunc s32 scx_bpf_sub_caps(u64 cgroup_id, u64 caps, struct scx_cmask *out_ return 0; } +/** + * scx_bpf_sub_kill_bstr - Kill a direct child sub-scheduler + * @cgroup_id: cgroup id of the direct child to kill + * @fmt: reason message format string + * @data: format string parameters packaged using ___bpf_fill() macro + * @data__sz: @data len, must end in '__sz' for the verifier + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Evict a direct child sub-scheduler, disabling it with the supplied reason. + * The child and its subtree are torn down asynchronously through the usual + * disable path. + * + * Unlike scx_bpf_exit(), no exit code is taken: the child is a separate + * scheduler with its own exit-code semantics, so a code chosen by the parent + * would have no defined meaning. The reason string carries the intent. + * + * Return 0 on success or -ENODEV if @cgroup_id names no sub-scheduler, which + * can race with the child detaching on its own and so is not a scheduler error. + * Naming a sched that exists but is not a direct child aborts the parent. + */ +__printf(2, 0) +__bpf_kfunc s32 scx_bpf_sub_kill_bstr(u64 cgroup_id, char *fmt, + unsigned long long *data, u32 data__sz, + const struct bpf_prog_aux *aux) +{ + struct scx_sched *parent, *child; + s32 ret; + + guard(rcu)(); + + parent = scx_prog_sched(aux); + if (unlikely(!parent)) + return -ENODEV; + + if (!scx_is_cid_type()) { + scx_error(parent, "sub-cap kfuncs require a cid-form scheduler"); + return -EOPNOTSUPP; + } + + child = scx_find_sub_sched(cgroup_id); + if (unlikely(!child)) + return -ENODEV; + + if (unlikely(scx_parent(child) != parent)) { + scx_error(parent, "%s: sub-%llu is not a direct child", + parent->cgrp_path, cgroup_id); + return -EINVAL; + } + + guard(raw_spinlock_irqsave)(&scx_exit_bstr_buf_lock); + ret = scx_bstr_format(parent, &scx_exit_bstr_buf, fmt, data, data__sz); + if (ret < 0) + return ret; + scx_exit(child, SCX_EXIT_PARENT_KILL, 0, "%s", scx_exit_bstr_buf.line); + return 0; +} + __bpf_kfunc_end_defs(); #endif /* CONFIG_EXT_SUB_SCHED */ diff --git a/tools/sched_ext/include/scx/common.bpf.h b/tools/sched_ext/include/scx/common.bpf.h index 09c21602b2ed..acc2b131ea8f 100644 --- a/tools/sched_ext/include/scx/common.bpf.h +++ b/tools/sched_ext/include/scx/common.bpf.h @@ -119,6 +119,8 @@ s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, const struct scx_cmask *cmask, struct scx_cmask *denied) __ksym __weak; void scx_bpf_sub_revoke(u64 cgroup_id, u64 caps, const struct scx_cmask *cmask) __ksym __weak; s32 scx_bpf_sub_caps(u64 cgroup_id, u64 caps, struct scx_cmask *out) __ksym __weak; +s32 scx_bpf_sub_kill_bstr(u64 cgroup_id, char *fmt, + unsigned long long *data, u32 data__sz) __ksym __weak; /* * Use the following as @it__iter when calling scx_bpf_dsq_move[_vtime]() from @@ -165,6 +167,22 @@ void ___scx_bpf_bstr_format_checker(const char *fmt, ...) {} ___scx_bpf_bstr_format_checker(fmt, ##args); \ }) +/* + * scx_bpf_sub_kill() wraps the scx_bpf_sub_kill_bstr() kfunc with variadic + * arguments instead of an array of u64. It kills the direct child sub-scheduler + * @cgid, passing the formatted reason to its user space, and evaluates to the + * kfunc's return value. On a kernel without sub-scheduler support the kfunc is + * absent and it returns -EOPNOTSUPP. + */ +#define scx_bpf_sub_kill(cgid, fmt, args...) \ +({ \ + scx_bpf_bstr_preamble(fmt, args) \ + ___scx_bpf_bstr_format_checker(fmt, ##args); \ + bpf_ksym_exists(scx_bpf_sub_kill_bstr) ? \ + scx_bpf_sub_kill_bstr((cgid), ___fmt, ___param, \ + sizeof(___param)) : -EOPNOTSUPP; \ +}) + /* * scx_bpf_error() wraps the scx_bpf_error_bstr() kfunc with variadic arguments * instead of an array of u64. Invoking this macro will cause the scheduler to -- cgit v1.2.3 From 34e0fbfe6737f461d1ee0e52ac1179221b95272f Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 15 Jul 2026 13:37:59 -1000 Subject: sched_ext: Remove queued ecaps syncs directly on sched teardown scx_discard_ecaps_to_sync() waited for balance_one() to consume a dying sched's queued ecaps sync, polling with resched_cpu() + msleep(). The wait is unbounded - the ext dl_server forces picks through sustained fair or RT load only while ext tasks are queued, so an ext-idle cpu monopolized by a higher class can stall the teardown indefinitely. Remove the node directly instead: take all queued nodes, drop the dying sched's and resplice the rest. Consumption runs under the rq lock and batch nodes read as on-list throughout, so the producer-side dedup stays correct. A node that an in-flight scx_process_sync_ecaps() batch holds across a dispatch-induced rq unlock still needs a wait, but one bounded by that batch completing rather than by a future balance. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 2 +- kernel/sched/ext/sub.c | 67 +++++++++++++++++++++++++++++++------------------- 2 files changed, 43 insertions(+), 26 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index f8f8597b6f78..428f2bfb2cda 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -4877,7 +4877,7 @@ static void scx_sched_free_rcu_work(struct work_struct *work) */ WARN_ON_ONCE(!list_empty(&pcpu->deferred_reenq_local.node)); - /* retire the queued ecaps syncs so the pcpu can be freed */ + /* remove the queued ecaps sync so the pcpu can be freed */ scx_discard_ecaps_to_sync(cpu, pcpu); /* diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 68ab3675337f..198063a78a3a 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -13,7 +13,6 @@ * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. * Copyright (c) 2026 Tejun Heo */ -#include #include #include "internal.h" #include "cid.h" @@ -537,7 +536,12 @@ void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) gained = ecaps & ~old; lost_all |= lost; - /* tell the sched its effective caps on this cid changed */ + /* + * Tell the sched its effective caps on this cid changed. The + * invocation is equivalent to the dispatch path and may drop + * and re-acquire the rq lock temporarily while the rest of + * @batch is held privately, see scx_discard_ecaps_to_sync(). + */ if (ecaps != pcpu->reported_ecaps && SCX_HAS_OP(pcpu->sch, sub_ecaps_updated) && !scx_bypassing(pcpu->sch, cpu)) { @@ -661,38 +665,51 @@ void scx_offline_ecaps(struct rq *rq) } /* - * @pcpu's sched was unhashed before the grace period, so nothing new queues. - * Flush its pending sync so the pcpu can be freed. If the cpu is online and - * scx is enabled, drain via balance_one(). Otherwise, discard under the rq - * lock. + * @pcpu's sched was unhashed before the grace period, so nothing re-queues its + * sync node. Remove the node from @rq's pending list so the pcpu can be freed. */ void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu) { struct rq *rq = cpu_rq(cpu); + struct llist_node *head = NULL, *tail = NULL; + struct llist_node *pos, *tmp; + + /* + * llist can't unlink a single node. Take all queued nodes, drop @pcpu's + * and resplice the rest. Nodes in the taken batch read as on-list + * throughout, so queue_sync_ecaps() stays correct. + */ + if (llist_on_list(&pcpu->ecaps_to_sync_node)) { + scoped_guard (rq_lock_irqsave, rq) { + llist_for_each_safe(pos, tmp, llist_del_all(&rq->scx.ecaps_to_sync)) { + if (pos == &pcpu->ecaps_to_sync_node) { + init_llist_node(pos); + } else { + pos->next = head; + head = pos; + if (!tail) + tail = pos; + } + } + if (head) + llist_add_batch(head, tail, &rq->scx.ecaps_to_sync); + } + } + /* + * An in-flight scx_process_sync_ecaps() batch may still hold the node + * privately across dispatch-induced rq unlocks, reading as on-list. + * + * Because a bypassing sched gets no op call, init_llist_node() and all + * @pcpu accesses share one contiguous lock hold, off-list under the rq + * lock means @pcpu won't be accessed again. + */ while (true) { scoped_guard (rq_lock_irqsave, rq) { - /* - * scx_process_sync_ecaps() takes the node off the list - * before it is done accessing @pcpu but does all of it - * under the rq lock. Off-list observed under the rq - * lock guarantees that the sync is complete. - */ if (!llist_on_list(&pcpu->ecaps_to_sync_node)) return; - /* - * Discard only when the cpu is truly down. cpu_active() - * is already set when scx_online_ecaps() queues an online - * resync while SCX_RQ_ONLINE is not - so test cpu_active(), - * or that resync would be dropped. - */ - if (!scx_enabled() || !cpu_active(cpu)) { - discard_queued_syncs(rq); - return; - } } - resched_cpu(cpu); - msleep(1); + cpu_relax(); } } @@ -704,7 +721,7 @@ void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu) * still-open link fd defers it) and can leave queued ecaps syncs behind. * Processing them would decode the dead sched's pshards with the current cid * layout. Discard them instead. The backing scx_sched_pcpu's are still - * allocated as the free path drains ecaps_to_sync_node before freeing. + * allocated as the free path removes ecaps_to_sync_node before freeing. */ void scx_discard_stale_ecaps_syncs(void) { -- cgit v1.2.3 From 7f480f34b78a0f482fbaa2aadece036aad275a74 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 15 Jul 2026 13:37:59 -1000 Subject: sched_ext: Move scx_dispatch_sched() to a new inlines.h scx_dispatch_sched() is common dispatch machinery and looks out of place in sub.h, but it needs scx_cpu_arg() from cid.h and can't move into internal.h without creating a circular include. Add inlines.h on top of internal.h and cid.h, and move the function there. The function was sub.h's only cid.h user, so drop that include. Pure code move, no functional change. v2: Host the function in a new inlines.h instead of at internal.h's tail, which formed a circular include with cid.h. Drop sub.h's now-unused cid.h include. (sashiko AI) Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/build_policy.c | 1 + kernel/sched/ext/ext.c | 1 + kernel/sched/ext/inlines.h | 125 ++++++++++++++++++++++++++++++++++++++++++++ kernel/sched/ext/sub.c | 1 + kernel/sched/ext/sub.h | 110 -------------------------------------- 5 files changed, 128 insertions(+), 110 deletions(-) create mode 100644 kernel/sched/ext/inlines.h (limited to 'kernel') diff --git a/kernel/sched/build_policy.c b/kernel/sched/build_policy.c index 01dc7bf89af8..2a828725a7f9 100644 --- a/kernel/sched/build_policy.c +++ b/kernel/sched/build_policy.c @@ -67,6 +67,7 @@ # include "ext/arena.h" # include "ext/idle.h" # include "ext/sub.h" +# include "ext/inlines.h" # include "ext/ext.c" # include "ext/cid.c" # include "ext/arena.c" diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 428f2bfb2cda..f9958b8bd8f4 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -20,6 +20,7 @@ #include "arena.h" #include "idle.h" #include "sub.h" +#include "inlines.h" DEFINE_RAW_SPINLOCK(scx_sched_lock); diff --git a/kernel/sched/ext/inlines.h b/kernel/sched/ext/inlines.h new file mode 100644 index 000000000000..45c657bdad50 --- /dev/null +++ b/kernel/sched/ext/inlines.h @@ -0,0 +1,125 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst + * + * Inline definitions layered on top of internal.h and cid.h. + * + * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2026 Tejun Heo + */ +#ifndef _KERNEL_SCHED_EXT_INLINES_H +#define _KERNEL_SCHED_EXT_INLINES_H + +#include "internal.h" +#include "cid.h" + +/* + * One user of this function is scx_bpf_dispatch() which can be called + * recursively as sub-sched dispatches nest. Always inline to reduce stack usage + * from the call frame. + */ +static __always_inline bool +scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, + struct task_struct *prev, bool nested) +{ + struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; + int nr_loops = SCX_DSP_MAX_LOOPS; + s32 cpu = cpu_of(rq); + bool prev_on_sch = (prev->sched_class == &ext_sched_class) && + scx_task_on_sched(sch, prev); + + if (scx_consume_global_dsq(sch, rq)) + return true; + + if (scx_bypass_dsp_enabled(sch)) { + /* if @sch is bypassing, only the bypass DSQs are active */ + if (scx_bypassing(sch, cpu)) + return scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0); + +#ifdef CONFIG_EXT_SUB_SCHED + /* + * If @sch isn't bypassing but its children are, @sch is + * responsible for making forward progress for both its own + * tasks that aren't bypassing and the bypassing descendants' + * tasks. The following implements a simple built-in behavior - + * let each CPU try to run the bypass DSQ every Nth time. + * + * Later, if necessary, we can add an ops flag to suppress the + * auto-consumption and a kfunc to consume the bypass DSQ and, + * so that the BPF scheduler can fully control scheduling of + * bypassed tasks. + */ + struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); + + if (!(pcpu->bypass_host_seq++ % SCX_BYPASS_HOST_NTH) && + scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0)) { + __scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1); + return true; + } +#endif /* CONFIG_EXT_SUB_SCHED */ + } + + if (unlikely(!SCX_HAS_OP(sch, dispatch)) || !scx_rq_online(rq)) + return false; + + dspc->rq = rq; + + /* + * The dispatch loop. Because scx_flush_dispatch_buf() may drop the rq + * lock, the local DSQ might still end up empty after a successful + * ops.dispatch(). If the local DSQ is empty even after ops.dispatch() + * produced some tasks, retry. The BPF scheduler may depend on this + * looping behavior to simplify its implementation. + */ + do { + dspc->nr_tasks = 0; + + if (nested) { + SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu), + prev_on_sch ? prev : NULL); + } else { + /* stash @prev so that nested invocations can access it */ + rq->scx.sub_dispatch_prev = prev; + SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu), + prev_on_sch ? prev : NULL); + rq->scx.sub_dispatch_prev = NULL; + } + + scx_flush_dispatch_buf(sch, rq); + + if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice) { + rq->scx.flags |= SCX_RQ_BAL_KEEP; + return true; + } + if (rq->scx.local_dsq.nr) + return true; + if (scx_consume_global_dsq(sch, rq)) + return true; + + /* + * ops.dispatch() can trap us in this loop by repeatedly + * dispatching ineligible tasks. Break out once in a while to + * allow the watchdog to run. As IRQ can't be enabled in + * balance(), we want to complete this scheduling cycle and then + * start a new one. IOW, we want to call resched_curr() on the + * next, most likely idle, task, not the current one. Use + * __scx_bpf_kick_cpu() for deferred kicking. + */ + if (unlikely(!--nr_loops)) { + scx_kick_cpu(sch, cpu, 0); + break; + } + } while (dspc->nr_tasks); + + /* + * Prevent the CPU from going idle while bypassed descendants have tasks + * queued. Without this fallback, bypassed tasks could stall if the host + * scheduler's ops.dispatch() doesn't yield any tasks. + */ + if (scx_bypass_dsp_enabled(sch)) + return scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0); + + return false; +} + +#endif /* _KERNEL_SCHED_EXT_INLINES_H */ diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 198063a78a3a..0875659d43c7 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -18,6 +18,7 @@ #include "cid.h" #include "arena.h" #include "sub.h" +#include "inlines.h" #ifdef CONFIG_EXT_SUB_SCHED diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index f72c18a5972a..0db2d2ea0fd1 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -11,7 +11,6 @@ #define _KERNEL_SCHED_EXT_SUB_H #include "internal.h" -#include "cid.h" #ifdef CONFIG_EXT_SUB_SCHED @@ -172,113 +171,4 @@ static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p #endif /* CONFIG_EXT_SUB_SCHED */ -/* - * One user of this function is scx_bpf_dispatch() which can be called - * recursively as sub-sched dispatches nest. Always inline to reduce stack usage - * from the call frame. - */ -static __always_inline bool -scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, - struct task_struct *prev, bool nested) -{ - struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; - int nr_loops = SCX_DSP_MAX_LOOPS; - s32 cpu = cpu_of(rq); - bool prev_on_sch = (prev->sched_class == &ext_sched_class) && - scx_task_on_sched(sch, prev); - - if (scx_consume_global_dsq(sch, rq)) - return true; - - if (scx_bypass_dsp_enabled(sch)) { - /* if @sch is bypassing, only the bypass DSQs are active */ - if (scx_bypassing(sch, cpu)) - return scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0); - -#ifdef CONFIG_EXT_SUB_SCHED - /* - * If @sch isn't bypassing but its children are, @sch is - * responsible for making forward progress for both its own - * tasks that aren't bypassing and the bypassing descendants' - * tasks. The following implements a simple built-in behavior - - * let each CPU try to run the bypass DSQ every Nth time. - * - * Later, if necessary, we can add an ops flag to suppress the - * auto-consumption and a kfunc to consume the bypass DSQ and, - * so that the BPF scheduler can fully control scheduling of - * bypassed tasks. - */ - struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); - - if (!(pcpu->bypass_host_seq++ % SCX_BYPASS_HOST_NTH) && - scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0)) { - __scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1); - return true; - } -#endif /* CONFIG_EXT_SUB_SCHED */ - } - - if (unlikely(!SCX_HAS_OP(sch, dispatch)) || !scx_rq_online(rq)) - return false; - - dspc->rq = rq; - - /* - * The dispatch loop. Because scx_flush_dispatch_buf() may drop the rq - * lock, the local DSQ might still end up empty after a successful - * ops.dispatch(). If the local DSQ is empty even after ops.dispatch() - * produced some tasks, retry. The BPF scheduler may depend on this - * looping behavior to simplify its implementation. - */ - do { - dspc->nr_tasks = 0; - - if (nested) { - SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu), - prev_on_sch ? prev : NULL); - } else { - /* stash @prev so that nested invocations can access it */ - rq->scx.sub_dispatch_prev = prev; - SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu), - prev_on_sch ? prev : NULL); - rq->scx.sub_dispatch_prev = NULL; - } - - scx_flush_dispatch_buf(sch, rq); - - if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice) { - rq->scx.flags |= SCX_RQ_BAL_KEEP; - return true; - } - if (rq->scx.local_dsq.nr) - return true; - if (scx_consume_global_dsq(sch, rq)) - return true; - - /* - * ops.dispatch() can trap us in this loop by repeatedly - * dispatching ineligible tasks. Break out once in a while to - * allow the watchdog to run. As IRQ can't be enabled in - * balance(), we want to complete this scheduling cycle and then - * start a new one. IOW, we want to call resched_curr() on the - * next, most likely idle, task, not the current one. Use - * __scx_bpf_kick_cpu() for deferred kicking. - */ - if (unlikely(!--nr_loops)) { - scx_kick_cpu(sch, cpu, 0); - break; - } - } while (dspc->nr_tasks); - - /* - * Prevent the CPU from going idle while bypassed descendants have tasks - * queued. Without this fallback, bypassed tasks could stall if the host - * scheduler's ops.dispatch() doesn't yield any tasks. - */ - if (scx_bypass_dsp_enabled(sch)) - return scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0); - - return false; -} - #endif /* _KERNEL_SCHED_EXT_SUB_H */ -- cgit v1.2.3 From 6ba3bd6f22314072d36949767e1d1192b97e5ba4 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 15 Jul 2026 13:37:59 -1000 Subject: sched_ext: Gate sub_dispatch_prev with CONFIG_EXT_SUB_SCHED rq->scx.sub_dispatch_prev is sub-sched-only but was left unconditional. Move it into the CONFIG_EXT_SUB_SCHED block next to ecaps_to_sync and gate its updates. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/inlines.h | 19 +++++++++++-------- kernel/sched/sched.h | 3 +-- 2 files changed, 12 insertions(+), 10 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/inlines.h b/kernel/sched/ext/inlines.h index 45c657bdad50..d2d074cee1cb 100644 --- a/kernel/sched/ext/inlines.h +++ b/kernel/sched/ext/inlines.h @@ -74,16 +74,19 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, do { dspc->nr_tasks = 0; - if (nested) { - SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu), - prev_on_sch ? prev : NULL); - } else { - /* stash @prev so that nested invocations can access it */ +#ifdef CONFIG_EXT_SUB_SCHED + /* stash @prev so that nested invocations can access it */ + if (!nested) rq->scx.sub_dispatch_prev = prev; - SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu), - prev_on_sch ? prev : NULL); +#endif + + SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu), + prev_on_sch ? prev : NULL); + +#ifdef CONFIG_EXT_SUB_SCHED + if (!nested) rq->scx.sub_dispatch_prev = NULL; - } +#endif scx_flush_dispatch_buf(sch, rq); diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 54cff94556c0..64d79e9efc3d 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -814,6 +814,7 @@ struct scx_rq { u64 clock; /* current per-rq clock -- see scx_bpf_now() */ #ifdef CONFIG_EXT_SUB_SCHED struct llist_head ecaps_to_sync; /* pending ecaps syncs */ + struct task_struct *sub_dispatch_prev; #endif cpumask_var_t cpus_to_sync; bool kick_sync_pending; @@ -821,8 +822,6 @@ struct scx_rq { struct list_head sched_pcpus_to_kick; /* see kick_cpus_irq_workfn() */ - struct task_struct *sub_dispatch_prev; - raw_spinlock_t deferred_reenq_lock; u64 deferred_reenq_locals_seq; struct list_head deferred_reenq_locals; /* scheds requesting reenq of local DSQ */ -- cgit v1.2.3 From 8946dbd3aa91acfb75b1633859290ba248e313b2 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 15 Jul 2026 13:37:59 -1000 Subject: sched_ext: Add the scx_has_subs static key and gate sub-sched hot paths With CONFIG_EXT_SUB_SCHED=y but no sub-scheduler attached - the common case - hot paths still pay for sub-sched bookkeeping. Gate it behind __scx_has_subs, a static key counting live sub-schedulers, so that a root-only system stops paying. Most conversions are simple skip-if-no-sub tests. scx_idle_notify() is special - it's a hierarchy walk, so give it a fast path which notifies the root directly using the same tests as the walk. A pending SCX_RQ_SUB_IDLE_RENOTIFY can be ignored as no sub can be owed one and the caller clears the flag either way. Suggested-by: Andrea Righi Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 4 ++++ kernel/sched/ext/idle.c | 10 ++++++++++ kernel/sched/ext/internal.h | 19 +++++++++++++++++-- kernel/sched/ext/sub.c | 21 +++++++++++++++++++-- kernel/sched/ext/sub.h | 15 +++++++++++++++ 5 files changed, 65 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index f9958b8bd8f4..b3b8cf95e0f7 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -4920,6 +4920,10 @@ static void scx_sched_free_rcu_work(struct work_struct *work) scx_arena_pool_destroy(sch); if (sch->arena_map) bpf_map_put(sch->arena_map); + + /* @sch is completely inactive by now */ + scx_dec_has_subs(sch); + kfree(sch); } diff --git a/kernel/sched/ext/idle.c b/kernel/sched/ext/idle.c index 16ebe3ab8647..d08166de03d8 100644 --- a/kernel/sched/ext/idle.c +++ b/kernel/sched/ext/idle.c @@ -746,6 +746,16 @@ static void scx_idle_notify(struct rq *rq, bool idle, bool do_notify, bool root_ lockdep_assert_rq_held(rq); + /* with no sub-sched, only the root can be owed a notification */ + if (!scx_has_subs()) { + struct scx_sched *sch = scx_root; + + if ((do_notify || root_renotify) && + SCX_HAS_OP(sch, update_idle) && !scx_bypassing(sch, cpu)) + SCX_CALL_OP(sch, update_idle, rq, cid, idle); + return; + } + pos = scx_next_descendant_pre(NULL, scx_root); while (pos) { bool forced = false; diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index d7a1d6a14ebf..4f4130f0d121 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -2110,7 +2110,7 @@ do { \ */ #define SCX_CALL_OP_TASK(sch, op, locked_rq, task, args...) \ do { \ - WARN_ON_ONCE((sch) != scx_task_sched_rcu(task)); \ + WARN_ON_ONCE(scx_has_subs() && (sch) != scx_task_sched_rcu(task)); \ __SCX_CALL_OP_TASK((sch), ops, op, locked_rq, task, ##args); \ } while (0) @@ -2125,7 +2125,7 @@ do { \ #define SCX_CALL_OP_TASK_RET(sch, op, locked_rq, task, args...) \ ({ \ __typeof__((sch)->ops.op(task, ##args)) __ret; \ - WARN_ON_ONCE((sch) != scx_task_sched_rcu(task)); \ + WARN_ON_ONCE(scx_has_subs() && (sch) != scx_task_sched_rcu(task)); \ WARN_ON_ONCE(current->scx.kf_tasks[0]); \ current->scx.kf_tasks[0] = task; \ __ret = SCX_CALL_OP_RET((sch), op, locked_rq, task, ##args); \ @@ -2165,6 +2165,19 @@ static inline bool scx_bypassing(struct scx_sched *sch, s32 cpu) } #ifdef CONFIG_EXT_SUB_SCHED +DECLARE_STATIC_KEY_FALSE(__scx_has_subs); + +/** + * scx_has_subs - Whether any sub-scheduler exists + * + * Gates the sub-sched portions of hot paths so that a root-only system doesn't + * pay for them. See scx_sub_enable_workfn() and scx_sched_free_rcu_work(). + */ +static inline bool scx_has_subs(void) +{ + return static_branch_unlikely(&__scx_has_subs); +} + /** * scx_task_sched - Find scx_sched scheduling a task * @p: task of interest @@ -2259,6 +2272,8 @@ static inline struct scx_sched *scx_parent(struct scx_sched *sch) return NULL; } #else /* CONFIG_EXT_SUB_SCHED */ +static inline bool scx_has_subs(void) { return false; } + static inline struct scx_sched *scx_task_sched(const struct task_struct *p) { return rcu_dereference_protected(scx_root, diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 0875659d43c7..3cc6d2633f73 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -22,6 +22,12 @@ #ifdef CONFIG_EXT_SUB_SCHED +/* + * On while any sub-scheduler exists so that a root-only system doesn't pay for + * the sub-sched portions of hot paths. See scx_has_subs(). + */ +DEFINE_STATIC_KEY_FALSE(__scx_has_subs); + /** * scx_skip_subtree_pre - Skip @pos's subtree in a pre-order walk * @pos: current position @@ -236,6 +242,9 @@ void scx_init_root_caps(struct scx_sched *sch) struct scx_dispatch_q *scx_local_or_reject_dsq(struct scx_sched *sch, struct rq *rq, struct task_struct *p, u64 *enq_flags) { + if (!scx_has_subs()) + return &rq->scx.local_dsq; + s32 cid = __scx_cpu_to_cid(cpu_of(rq)); struct scx_sched *asch = rq->scx.remote_activate_sch ?: sch; u64 needed = scx_caps_for_enq(*enq_flags); @@ -314,7 +323,7 @@ void scx_reenq_reject(struct rq *rq) lockdep_assert_rq_held(rq); - if (list_empty(&rq->scx.reject_dsq.list)) + if (!scx_has_subs() || list_empty(&rq->scx.reject_dsq.list)) return; /* @@ -497,7 +506,7 @@ void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) lockdep_assert_rq_held(rq); - if (likely(llist_empty(&rq->scx.ecaps_to_sync))) + if (!scx_has_subs() || likely(llist_empty(&rq->scx.ecaps_to_sync))) return; /* @@ -1016,10 +1025,18 @@ void scx_sub_enable_workfn(struct kthread_work *work) kobject_get(&parent->kobj); raw_spin_unlock_irq(&scx_sched_lock); + /* + * Flip the hot-path gates before ops->priv is published - the sub's + * programs can e.g. kick cpus from that point on. The matching dec is + * at the end of scx_sched_free_rcu_work(). + */ + static_branch_inc(&__scx_has_subs); + /* scx_alloc_and_add_sched() consumes @cgrp whether it succeeds or not */ sch = scx_alloc_and_add_sched(cmd, cgrp, parent); kobject_put(&parent->kobj); if (IS_ERR(sch)) { + static_branch_dec(&__scx_has_subs); ret = PTR_ERR(sch); goto out_unlock; } diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index 0db2d2ea0fd1..625d7ce334aa 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -44,6 +44,13 @@ static inline const char *sch_cgrp_path(struct scx_sched *sch) return sch->cgrp_path; } +/* a dying sub's hot-path influence ends in scx_sched_free_rcu_work() */ +static inline void scx_dec_has_subs(struct scx_sched *sch) +{ + if (sch->level) + static_branch_dec(&__scx_has_subs); +} + #else /* CONFIG_EXT_SUB_SCHED */ static inline struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; } @@ -66,6 +73,7 @@ static inline void scx_discard_stale_ecaps_syncs(void) {} static inline struct scx_dispatch_q *scx_local_or_reject_dsq(struct scx_sched *sch, struct rq *rq, struct task_struct *p, u64 *enq_flags) { return &rq->scx.local_dsq; } static inline bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p) { return false; } static inline void scx_reenq_reject(struct rq *rq) {} +static inline void scx_dec_has_subs(struct scx_sched *sch) {} #endif /* CONFIG_EXT_SUB_SCHED */ @@ -96,6 +104,10 @@ static inline u64 scx_missing_caps(struct scx_sched *sch, s32 cpu, u64 needed) { u64 ecaps; + /* no sub-scheds, no missing caps */ + if (!scx_has_subs()) + return 0; + /* root holds every cap on every cpu */ if (!sch->level) return 0; @@ -156,6 +168,9 @@ static inline u64 scx_caps_implied(u64 cap) /* may @p keep running on @rq's cpu? requires baseline cpu access */ static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p) { + if (!scx_has_subs()) + return true; + /* a migration-disabled task is let in without caps, keep it likewise */ if (unlikely(is_migration_disabled(p))) return true; -- cgit v1.2.3 From 52478777b37ba56bb11d40025df0a03ef65c6acb Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 17 Jul 2026 22:12:20 -1000 Subject: cgroup: Add cgroup_task_notifier and task migration events A subsystem can attach to the cgroup hierarchy itself, independent of which controllers are enabled where - BPF hooks already behave this way and sched_ext sub-schedulers do too. Controller callbacks can't track task migrations for them: sched_ext must re-home a task whose migration crosses a sub-scheduler boundary, but the cpu controller's attach callbacks fire only when the task_group changes and miss moves whenever the controller topology is coarser than the sub-scheduler topology. Add cgroup_task_notifier with per-task migration events mirroring the can_attach/attach/cancel_attach phases so that a consumer which prepares per-task state can also veto a migration: CGROUP_TASK_MIGRATING fires pre-commit, CGROUP_TASK_MIGRATED post-commit and CGROUP_TASK_MIGRATE_CANCELED unwinds a failed migration. Only migrations that change a task's dfl cgroup are reported. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- include/linux/cgroup.h | 26 ++++++++++++++ kernel/cgroup/cgroup.c | 93 +++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 110 insertions(+), 9 deletions(-) (limited to 'kernel') diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h index f2aa46a4f871..aa92db5f05de 100644 --- a/include/linux/cgroup.h +++ b/include/linux/cgroup.h @@ -82,12 +82,38 @@ enum cgroup_lifetime_events { CGROUP_LIFETIME_OFFLINE, }; +/* + * Events on cgroup_task_notifier, data is struct cgroup_task_migrate_ctx. + * MIGRATING fires per task before the migration commits and an error return + * from the chain fails the migration, in which case tasks that were already + * notified receive MIGRATE_CANCELED. MIGRATED fires per task after the + * migration is committed and can't fail. Only migrations that change a task's + * dfl cgroup are reported. + */ +enum cgroup_task_events { + CGROUP_TASK_MIGRATING, + CGROUP_TASK_MIGRATED, + CGROUP_TASK_MIGRATE_CANCELED, +}; + +/* + * @src_dcgrp and @dst_dcgrp are @task's dfl cgroups before and after the + * migration. @src_dcgrp is NULL for CGROUP_TASK_MIGRATED as per-task sources + * are not tracked past the commit point. + */ +struct cgroup_task_migrate_ctx { + struct task_struct *task; + struct cgroup *src_dcgrp; + struct cgroup *dst_dcgrp; +}; + extern struct file_system_type cgroup_fs_type; extern struct cgroup_root cgrp_dfl_root; extern struct css_set init_css_set; extern struct mutex cgroup_mutex; extern spinlock_t css_set_lock; extern struct blocking_notifier_head cgroup_lifetime_notifier; +extern struct blocking_notifier_head cgroup_task_notifier; #define SUBSYS(_x) extern struct cgroup_subsys _x ## _cgrp_subsys; #include diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index 38f8d9df8fbc..2f6b634c84ca 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -88,6 +88,8 @@ EXPORT_SYMBOL_GPL(css_set_lock); struct blocking_notifier_head cgroup_lifetime_notifier = BLOCKING_NOTIFIER_INIT(cgroup_lifetime_notifier); +struct blocking_notifier_head cgroup_task_notifier = + BLOCKING_NOTIFIER_INIT(cgroup_task_notifier); DEFINE_SPINLOCK(trace_cgroup_path_lock); char trace_cgroup_path[TRACE_CGROUP_PATH_LEN]; @@ -2676,14 +2678,27 @@ struct task_struct *cgroup_taskset_next(struct cgroup_taskset *tset, return NULL; } +static void cgroup_migrate_notify_canceled(struct css_set *src_cset, + struct task_struct *task) +{ + struct cgroup_task_migrate_ctx ctx = { + .task = task, + .src_dcgrp = src_cset->dfl_cgrp, + .dst_dcgrp = src_cset->mg_dst_cset->dfl_cgrp, + }; + + blocking_notifier_call_chain(&cgroup_task_notifier, + CGROUP_TASK_MIGRATE_CANCELED, &ctx); +} + /** * cgroup_migrate_execute - migrate a taskset * @mgctx: migration context * - * Migrate tasks in @mgctx as setup by migration preparation functions. - * This function fails iff one of the ->can_attach callbacks fails and - * guarantees that either all or none of the tasks in @mgctx are migrated. - * @mgctx is consumed regardless of success. + * Migrate tasks in @mgctx as setup by migration preparation functions. This + * function fails iff one of the ->can_attach callbacks or CGROUP_TASK_MIGRATING + * notifications fails and guarantees that either all or none of the tasks in + * @mgctx are migrated. @mgctx is consumed regardless of success. */ static int cgroup_migrate_execute(struct cgroup_mgctx *mgctx) { @@ -2691,6 +2706,7 @@ static int cgroup_migrate_execute(struct cgroup_mgctx *mgctx) struct cgroup_subsys *ss; struct task_struct *task, *tmp_task; struct css_set *cset, *tmp_cset; + bool dfl_migration = false; int ssid, failed_ssid, ret; /* check that we can legitimately attach to the cgroup */ @@ -2707,6 +2723,33 @@ static int cgroup_migrate_execute(struct cgroup_mgctx *mgctx) } while_each_subsys_mask(); } + /* + * Notify each task about the impending migration. An error return fails + * the migration. Only migrations on the default hierarchy are reported: + * a migration modifies either every moved task's dfl cgroup or, on + * cgroup1 or for subtree_control writes, none. + */ + list_for_each_entry(cset, &tset->src_csets, mg_node) { + if (cset->dfl_cgrp == cset->mg_dst_cset->dfl_cgrp) + continue; + dfl_migration = true; + list_for_each_entry(task, &cset->mg_tasks, cg_list) { + struct cgroup_task_migrate_ctx ctx = { + .task = task, + .src_dcgrp = cset->dfl_cgrp, + .dst_dcgrp = cset->mg_dst_cset->dfl_cgrp, + }; + + ret = blocking_notifier_call_chain_robust(&cgroup_task_notifier, + CGROUP_TASK_MIGRATING, + CGROUP_TASK_MIGRATE_CANCELED, + &ctx); + ret = notifier_to_errno(ret); + if (ret) + goto out_cancel_migrating; + } + } + /* * Now that we're guaranteed success, proceed to move all tasks to * the new cgroup. There are no failure cases after here, so this @@ -2750,9 +2793,41 @@ static int cgroup_migrate_execute(struct cgroup_mgctx *mgctx) } while_each_subsys_mask(); } + /* + * Notify each task after successful migration. The operation can no + * longer fail and the return value is ignored. The MIGRATING loop + * above explains why only dfl migrations are reported. Per-task + * sources are not tracked past the commit point, so src_dcgrp is + * NULL. + */ + if (dfl_migration) { + list_for_each_entry(cset, &tset->dst_csets, mg_node) { + list_for_each_entry(task, &cset->mg_tasks, cg_list) { + struct cgroup_task_migrate_ctx ctx = { + .task = task, + .dst_dcgrp = cset->dfl_cgrp, + }; + + blocking_notifier_call_chain( + &cgroup_task_notifier, + CGROUP_TASK_MIGRATED, &ctx); + } + } + } + ret = 0; goto out_release_tset; +out_cancel_migrating: + list_for_each_entry_continue_reverse(task, &cset->mg_tasks, cg_list) + cgroup_migrate_notify_canceled(cset, task); + list_for_each_entry_continue_reverse(cset, &tset->src_csets, mg_node) { + if (cset->dfl_cgrp == cset->mg_dst_cset->dfl_cgrp) + continue; + list_for_each_entry_reverse(task, &cset->mg_tasks, cg_list) + cgroup_migrate_notify_canceled(cset, task); + } + failed_ssid = CGROUP_SUBSYS_COUNT; out_cancel_attach: if (tset->nr_tasks) { do_each_subsys_mask(ss, ssid, mgctx->ss_mask) { @@ -2976,11 +3051,11 @@ int cgroup_migrate_prepare_dst(struct cgroup_mgctx *mgctx) * cgroup_migrate_prepare_dst() on the targets before invoking this * function and following up with cgroup_migrate_finish(). * - * As long as a controller's ->can_attach() doesn't fail, this function is - * guaranteed to succeed. This means that, excluding ->can_attach() - * failure, when migrating multiple targets, the success or failure can be - * decided for all targets by invoking group_migrate_prepare_dst() before - * actually starting migrating. + * As long as a controller's ->can_attach() or a CGROUP_TASK_MIGRATING + * notification doesn't fail, this function is guaranteed to succeed. This + * means that, excluding those failures, when migrating multiple targets, + * the success or failure can be decided for all targets by invoking + * group_migrate_prepare_dst() before actually starting migrating. */ int cgroup_migrate(struct task_struct *leader, bool threadgroup, struct cgroup_mgctx *mgctx) -- cgit v1.2.3 From 0dc90ce1be18ea6b18b545f4752de64cfc140ccd Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 17 Jul 2026 22:12:20 -1000 Subject: sched_ext: Factor out scx_rehome_task() and scx_punt_task() Factor out scx_rehome_task() and scx_punt_task() from the sub-disable re-home loop and scx_fail_parent(). The upcoming cgroup migration re-homing also needs scx_rehome_task(). No functional changes. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/sub.c | 84 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 56 insertions(+), 28 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index c7f70cf877d1..5f7ac6696d17 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -758,6 +758,57 @@ void drain_descendants(struct scx_sched *sch) wait_event(scx_unlink_waitq, list_empty(&sch->children)); } +/** + * scx_rehome_task - Move a task to a sched it has been initialized for + * @to: sched taking over @p, @p's init on it already complete + * @p: task to re-home + * + * Exit @p from its current sched and switch it over to @to, overriding the + * state to %SCX_TASK_READY to account for the already completed init. A task + * on a non-ext class, possible under an %SCX_OPS_SWITCH_PARTIAL root, stays + * %READY and is enabled by switching_to_scx() if it switches over. + */ +static void scx_rehome_task(struct scx_sched *to, struct task_struct *p) +{ + lockdep_assert_held(&p->pi_lock); + lockdep_assert_rq_held(task_rq(p)); + + scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { + scx_disable_and_exit_task(scx_task_sched(p), p); + scx_set_task_state(p, SCX_TASK_INIT_BEGIN); + scx_set_task_state(p, SCX_TASK_INIT); + scx_set_task_sched(p, to); + scx_set_task_state(p, SCX_TASK_READY); + if (p->sched_class == &ext_sched_class) + scx_enable_task(to, p); + } +} + +/** + * scx_punt_task - Hand a task to a failed sched without initialization + * @to: failed and bypassed sched taking custody of @p + * @p: task to punt + * + * Take @p off its current sched and put it on @to at %SCX_TASK_NONE. @to is + * dying and its teardown will re-home @p properly. + * + * Used when @to must take over @p but failed to initialize it. Bypass keeps + * scheduling decisions away from @to but @p can still trigger its task ops, + * which may confuse the BPF side. @to is dying anyway. The exit paths skip + * %NONE tasks (see __scx_disable_and_exit_task() and switched_from_scx()). + */ +static void scx_punt_task(struct scx_sched *to, struct task_struct *p) +{ + lockdep_assert_held(&p->pi_lock); + lockdep_assert_rq_held(task_rq(p)); + WARN_ON_ONCE(!READ_ONCE(to->bypass_depth)); + + scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { + scx_disable_and_exit_task(scx_task_sched(p), p); + scx_set_task_sched(p, to); + } +} + static void scx_fail_parent(struct scx_sched *sch, struct task_struct *failed, s32 fail_code) { @@ -769,9 +820,9 @@ static void scx_fail_parent(struct scx_sched *sch, fail_code, failed->comm, failed->pid); /* - * Once $parent is bypassed, it's safe to put SCX_TASK_NONE tasks into - * it. This may cause downstream failures on the BPF side but $parent is - * dying anyway. + * Once $parent is bypassed, tasks can be punted into it. This may + * cause downstream failures on the BPF side but $parent is dying + * anyway. */ scx_bypass(parent, true); @@ -780,10 +831,7 @@ static void scx_fail_parent(struct scx_sched *sch, if (scx_task_on_sched(parent, p)) continue; - scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { - scx_disable_and_exit_task(sch, p); - scx_set_task_sched(p, parent); - } + scx_punt_task(parent, p); } scx_task_iter_stop(&sti); } @@ -881,27 +929,7 @@ void scx_sub_disable(struct scx_sched *sch) continue; } - scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { - /* - * $p is initialized for $parent and still attached to - * @sch. Disable and exit for @sch, switch over to - * $parent and override the state to READY to account - * for $p having already been initialized. - */ - scx_disable_and_exit_task(sch, p); - scx_set_task_state(p, SCX_TASK_INIT_BEGIN); - scx_set_task_state(p, SCX_TASK_INIT); - scx_set_task_sched(p, parent); - scx_set_task_state(p, SCX_TASK_READY); - - /* - * A task on a non-ext class, possible under an - * %SCX_OPS_SWITCH_PARTIAL root, stays READY and is - * enabled by switching_to_scx() if it switches over. - */ - if (p->sched_class == &ext_sched_class) - scx_enable_task(parent, p); - } + scx_rehome_task(parent, p); task_rq_unlock(rq, p, &rf); put_task_struct(p); -- cgit v1.2.3 From 54880aeb872b870904454ae20d179d32622494dc Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 17 Jul 2026 22:12:20 -1000 Subject: sched_ext: Relocate scx_cgroup_enabled scx_cgroup_enabled is in the CONFIG_EXT_GROUP_SCHED block. The upcoming cgroup migration re-homing needs the gate outside the block. Move the definition and flag flips outside CONFIG_EXT_GROUP_SCHED. No functional changes. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 18 ++++++++++++------ kernel/sched/ext/internal.h | 1 + 2 files changed, 13 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 145bda57b6fe..cd99947b4229 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -80,6 +80,14 @@ static bool scx_switching_all; DEFINE_STATIC_KEY_FALSE(__scx_switched_all); static DEFINE_STATIC_KEY_FALSE(__scx_tid_to_task_enabled); +/* + * Gates cgroup ops delivery. Set at the end of the cgroup init phase of root + * enable and cleared before root disable starts tearing down tasks, both under + * scx_cgroup_lock(). Holding cgroup_lock() and seeing %true guarantees no race + * against root tearing down tasks. + */ +bool scx_cgroup_enabled; + /* * True once SCX_OPS_TID_TO_TASK has been negotiated with the root scheduler * and the tid->task table is live. Wraps the static key so callers don't @@ -4309,7 +4317,6 @@ bool scx_can_stop_tick(struct rq *rq) #ifdef CONFIG_EXT_GROUP_SCHED DEFINE_STATIC_PERCPU_RWSEM(scx_cgroup_ops_rwsem); -static bool scx_cgroup_enabled; void scx_tg_init(struct task_group *tg) { @@ -4689,8 +4696,6 @@ static void scx_cgroup_exit(struct scx_sched *sch) { struct cgroup_subsys_state *css; - scx_cgroup_enabled = false; - /* * scx_tg_on/offline() are excluded through cgroup_lock(). If we walk * cgroups and exit all the inited ones, all online cgroups are exited. @@ -4745,9 +4750,6 @@ static int scx_cgroup_init(struct scx_sched *sch) tg->scx.flags |= SCX_TG_INITED; } - WARN_ON_ONCE(scx_cgroup_enabled); - scx_cgroup_enabled = true; - return 0; } @@ -5971,6 +5973,7 @@ static void scx_root_disable(struct scx_sched *sch) * doesn't race against scx_disable_and_exit_task(). */ scx_cgroup_lock(); + scx_cgroup_enabled = false; scx_cgroup_exit(sch); scx_cgroup_unlock(); @@ -7234,6 +7237,9 @@ static void scx_root_enable_workfn(struct kthread_work *work) if (ret) goto err_disable_unlock_all; + WARN_ON_ONCE(scx_cgroup_enabled); + scx_cgroup_enabled = true; + scx_task_iter_start(&sti, NULL); while ((p = scx_task_iter_next_locked(&sti))) { /* diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 4f4130f0d121..d76ac22019af 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1949,6 +1949,7 @@ __printf(3, 0) s32 scx_bstr_format(struct scx_sched *sch, struct scx_bstr_buf *b extern raw_spinlock_t scx_sched_lock; extern struct mutex scx_enable_mutex; extern struct percpu_rw_semaphore scx_fork_rwsem; +extern bool scx_cgroup_enabled; extern raw_spinlock_t scx_exit_bstr_buf_lock; extern struct scx_bstr_buf scx_exit_bstr_buf; #ifdef CONFIG_EXT_SUB_SCHED -- cgit v1.2.3 From bf9dee58ab565a2b4bfd26cf6b235989d527f7a4 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 17 Jul 2026 22:12:20 -1000 Subject: sched_ext: Re-home tasks on cgroup migration A task's sched (p->scx.sched) must match its cgroup's owner (cgrp->scx_sched). cgroup migration breaks the invariant: scx_cgroup_move_task() only fires root's ops.cgroup_move() and never re-homes the task, leading to wrong-sched scheduling and, once the stale sched is freed, a use-after-free. Hook into the new cgroup task migration events and re-home each task whose destination cgroup is owned by a different sched. The events map naturally to the transfer: MIGRATING runs the fallible init for the destination sched, letting it reject the migration the same way ops.cgroup_prep_move() can, MIGRATED does the re-home, which can't fail, and CANCELED undoes the init when the migration falls through. Pre-commit, the task's task_group still reflects the source, so __scx_init_task() grows an explicit cgroup argument for the migration path to hand ops.init_task() the destination cgroup. Signed-off-by: Tejun Heo Closes: https://lore.kernel.org/r/alnxrsexEe_nQwqL@gpd4 Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 29 +++++++--- kernel/sched/ext/internal.h | 3 +- kernel/sched/ext/sub.c | 135 ++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 152 insertions(+), 15 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index cd99947b4229..6bac68758704 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -3485,15 +3485,28 @@ static struct cgroup *tg_cgrp(struct task_group *tg) return &cgrp_dfl_root.cgrp; } -#define SCX_INIT_TASK_ARGS_CGROUP(tg) .cgroup = tg_cgrp(tg), +#define SCX_INIT_TASK_ARGS_CGROUP(cgrp) .cgroup = (cgrp), #else /* CONFIG_EXT_GROUP_SCHED */ -#define SCX_INIT_TASK_ARGS_CGROUP(tg) +#define SCX_INIT_TASK_ARGS_CGROUP(cgrp) #endif /* CONFIG_EXT_GROUP_SCHED */ -int __scx_init_task(struct scx_sched *sch, struct task_struct *p, bool fork) +/** + * __scx_init_task - Initialize a task for a sched + * @sch: sched to initialize @p for + * @p: task of interest + * @cgrp: cgroup @p is joining, %NULL for @p's current task_group's cgroup + * @fork: %true if @p is being forked + * + * Pre-commit cgroup migration passes @cgrp explicitly as @p's task_group + * still reflects the source. + * + * Return 0 on success, -errno on failure. + */ +int __scx_init_task(struct scx_sched *sch, struct task_struct *p, + struct cgroup *cgrp, bool fork) { int ret; @@ -3501,7 +3514,7 @@ int __scx_init_task(struct scx_sched *sch, struct task_struct *p, bool fork) if (SCX_HAS_OP(sch, init_task)) { struct scx_init_task_args args = { - SCX_INIT_TASK_ARGS_CGROUP(task_group(p)) + SCX_INIT_TASK_ARGS_CGROUP(cgrp ?: tg_cgrp(task_group(p))) .fork = fork, }; @@ -3747,7 +3760,7 @@ int scx_fork(struct task_struct *p, struct kernel_clone_args *kargs) struct scx_sched *sch = scx_root; #endif scx_set_task_state(p, SCX_TASK_INIT_BEGIN); - ret = __scx_init_task(sch, p, true); + ret = __scx_init_task(sch, p, NULL, true); if (unlikely(ret)) { scx_set_task_state(p, SCX_TASK_NONE); return ret; @@ -5969,8 +5982,8 @@ static void scx_root_disable(struct scx_sched *sch) WRITE_ONCE(scx_switching_all, false); /* - * Shut down cgroup support before tasks so that the cgroup attach path - * doesn't race against scx_disable_and_exit_task(). + * Shut down cgroup support before tasks so that the cgroup attach and + * migration paths don't race against scx_disable_and_exit_task(). */ scx_cgroup_lock(); scx_cgroup_enabled = false; @@ -7263,7 +7276,7 @@ static void scx_root_enable_workfn(struct kthread_work *work) scx_set_task_state(p, SCX_TASK_INIT_BEGIN); scx_task_iter_unlock(&sti); - ret = __scx_init_task(sch, p, false); + ret = __scx_init_task(sch, p, NULL, false); scx_task_iter_relock(&sti, p); diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index d76ac22019af..23fc95502ea6 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1920,7 +1920,8 @@ void scx_flush_dispatch_buf(struct scx_sched *sch, struct rq *rq); void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags); void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, u64 reenq_flags, struct rq *locked_rq); -int __scx_init_task(struct scx_sched *sch, struct task_struct *p, bool fork); +int __scx_init_task(struct scx_sched *sch, struct task_struct *p, + struct cgroup *cgrp, bool fork); void scx_enable_task(struct scx_sched *sch, struct task_struct *p); void __scx_disable_and_exit_task(struct scx_sched *sch, struct task_struct *p); void scx_sub_init_cancel_task(struct scx_sched *sch, struct task_struct *p); diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 5f7ac6696d17..393dbd00d2f7 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -906,7 +906,7 @@ void scx_sub_disable(struct scx_sched *sch) * parent. A child can't directly affect the parent through its * own failures. */ - ret = __scx_init_task(parent, p, false); + ret = __scx_init_task(parent, p, NULL, false); if (ret) { scx_fail_parent(sch, p, ret); put_task_struct(p); @@ -1212,7 +1212,7 @@ void scx_sub_enable_workfn(struct kthread_work *work) * As $p is still on $parent, it can't be transitioned to INIT. * Let's worry about task state later. Use __scx_init_task(). */ - ret = __scx_init_task(sch, p, false); + ret = __scx_init_task(sch, p, NULL, false); if (ret) goto abort; @@ -1336,6 +1336,99 @@ err_disable: cmd->ret = 0; } +/** + * scx_cgroup_task_migrating - Prepare a task for a cgroup migration + * @ctx: migration being prepared + * + * A task's sched must match its cgroup's owner, so a migration that crosses a + * sched boundary re-homes the task once committed. Run the fallible part here, + * before the migration commits: initialize the task for the destination sched. + * A rejection fails the cgroup.procs write. + */ +static s32 scx_cgroup_task_migrating(struct cgroup_task_migrate_ctx *ctx) +{ + struct task_struct *p = ctx->task; + struct scx_sched *to; + int ret; + + /* + * Cleared under scx_cgroup_lock() before root disable starts tearing + * down tasks. As cgroup_mutex is held, a set flag guarantees that the + * teardown loop is not running concurrently. + */ + if (!scx_cgroup_enabled) + return NOTIFY_OK; + + to = ctx->dst_dcgrp->scx_sched; + if (scx_task_on_sched(to, p)) + return NOTIFY_OK; + + ret = __scx_init_task(to, p, ctx->dst_dcgrp, false); + if (ret) + return notifier_from_errno(ret); + + return NOTIFY_OK; +} + +/** + * scx_cgroup_task_migrated - Re-home a task that changed cgroups + * @ctx: committed migration + * + * Move the task to its new cgroup's sched, which scx_cgroup_task_migrating() + * already initialized it for. Can't fail. + * + * This is safe against all phases of the destination sched's destruction. A + * disable resets cgroup ownership to the parent and re-homes tasks in one + * scx_cgroup_lock() section. If that section already ran, the destination would + * be the parent. Otherwise, the re-home loop is still ahead and guaranteed to + * visit the task, now in the destination cgroup. + */ +static void scx_cgroup_task_migrated(struct cgroup_task_migrate_ctx *ctx) +{ + struct task_struct *p = ctx->task; + struct scx_sched *to; + struct rq *rq; + struct rq_flags rf; + + if (!scx_cgroup_enabled) + return; + + to = ctx->dst_dcgrp->scx_sched; + if (scx_task_on_sched(to, p)) + return; + + rq = task_rq_lock(p, &rf); + scx_rehome_task(to, p); + task_rq_unlock(rq, p, &rf); +} + +/** + * scx_cgroup_task_migrate_canceled - Undo migration preparation + * @ctx: canceled migration + * + * The migration failed after scx_cgroup_task_migrating() initialized the task + * for the destination sched. The task stays on its current sched in the source + * cgroup. Undo the destination's init. + */ +static void scx_cgroup_task_migrate_canceled(struct cgroup_task_migrate_ctx *ctx) +{ + struct task_struct *p = ctx->task; + struct scx_sched *to; + struct rq *rq; + struct rq_flags rf; + + if (!scx_cgroup_enabled) + return; + + to = ctx->dst_dcgrp->scx_sched; + if (scx_task_on_sched(to, p)) + return; + + rq = task_rq_lock(p, &rf); + scx_sub_init_cancel_task(to, p); + task_rq_unlock(rq, p, &rf); +} + static s32 scx_cgroup_lifetime_notify(struct notifier_block *nb, unsigned long action, void *data) { @@ -1367,12 +1460,42 @@ static struct notifier_block scx_cgroup_lifetime_nb = { .notifier_call = scx_cgroup_lifetime_notify, }; -static s32 __init scx_cgroup_lifetime_notifier_init(void) +static s32 scx_cgroup_task_notify(struct notifier_block *nb, + unsigned long action, void *data) +{ + struct cgroup_task_migrate_ctx *ctx = data; + + switch (action) { + case CGROUP_TASK_MIGRATING: + return scx_cgroup_task_migrating(ctx); + case CGROUP_TASK_MIGRATED: + scx_cgroup_task_migrated(ctx); + break; + case CGROUP_TASK_MIGRATE_CANCELED: + scx_cgroup_task_migrate_canceled(ctx); + break; + } + + return NOTIFY_OK; +} + +static struct notifier_block scx_cgroup_task_nb = { + .notifier_call = scx_cgroup_task_notify, +}; + +static s32 __init scx_cgroup_notifier_init(void) { - return blocking_notifier_chain_register(&cgroup_lifetime_notifier, - &scx_cgroup_lifetime_nb); + s32 ret; + + ret = blocking_notifier_chain_register(&cgroup_lifetime_notifier, + &scx_cgroup_lifetime_nb); + if (ret) + return ret; + + return blocking_notifier_chain_register(&cgroup_task_notifier, + &scx_cgroup_task_nb); } -core_initcall(scx_cgroup_lifetime_notifier_init); +core_initcall(scx_cgroup_notifier_init); static void scx_pstack_recursion(struct bpf_prog *prog, const char *op) { -- cgit v1.2.3 From 46932bc5fd7ea1156a6742ad1b9306383e0cfb6f Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 17 Jul 2026 22:12:20 -1000 Subject: sched_ext: Deliver cgroup ops to each task_group's sched With sub-schedulers claiming cgroup subtrees, cgroup ops must be delivered to each task_group's sched rather than always to root. Add tg->scx.sched to track which sched initialized the task_group. It is set and cleared together with SCX_TG_INITED. Deliver the ops accordingly: - ops.cgroup_exit() goes to the sched whose ops.cgroup_init() it pairs with. - ops.cgroup_prep_move/move/cancel_move() go to the task's sched, and only for moves that don't re-home the task. A re-homing move is reported through the ops.exit_task/init_task() pair instead. The cgroups passed to the move ops can be outside the sched's inited set as the cpu controller can be coarser than the sub-scheduler topology. - Knobs of a cgroup belong to the parent, so ops.set_weight/idle/bandwidth() go to the parent task_group's sched. All task_groups currently resolve to the root sched, so no behavior changes until sub-schedulers start claiming cgroups. While at it, scx_cgroup_init() is restructured so both paths share the recording. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- include/linux/sched/ext.h | 2 + kernel/sched/ext/ext.c | 157 ++++++++++++++++++++++++++++++++------------ kernel/sched/ext/internal.h | 34 ++++++++-- 3 files changed, 146 insertions(+), 47 deletions(-) (limited to 'kernel') diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index cce42b21f5f5..a6db5d300f30 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -298,6 +298,8 @@ static inline bool scx_rcu_cpu_stall(const struct cpumask *stalled_mask) { retur struct scx_task_group { #ifdef CONFIG_EXT_GROUP_SCHED + struct scx_sched *sched; + u32 flags; /* SCX_TG_* */ u32 weight; u64 bw_period_us; diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 6bac68758704..b542900da5a3 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -4339,6 +4339,51 @@ void scx_tg_init(struct task_group *tg) tg->scx.idle = false; } +/** + * scx_tg_sched - Resolve a task_group's sched + * @tg: task_group of interest + * + * Return the sched that @tg's ops.cgroup_init() succeeded on, %NULL if @tg + * isn't inited. An autogroup tg has no cgroup of its own and resolves to the + * root sched. + * + * Safe for callers read-locking the ops rwsem. tg->scx.sched rewrites + * write-lock it, and tg on/offline can't overlap such callers as a css's files + * are created after online and drained before offline. + */ +static struct scx_sched *scx_tg_sched(struct task_group *tg) +{ + lockdep_assert(lockdep_is_held(&cgroup_mutex) || + lockdep_is_held(&scx_cgroup_ops_rwsem)); + + if (!tg->css.cgroup) + tg = &root_task_group; + return tg->scx.sched; +} + +/** + * scx_tg_knob_sched - Resolve the sched receiving a task_group's knob updates + * @tg: task_group of interest + * + * Knobs of a cgroup belong to the parent. Deliver the set_* ops to the + * parent task_group's sched, which equals @tg's own sched everywhere except + * at a sub-scheduler attach point, where the sub's parent sched receives + * them. + * + * The callers sit in @tg's cgroup file writes holding the ops rwsem read + * side. That extends scx_tg_sched()'s file-write argument to the parent's + * sched read: a parent css outlives its children's files. + */ +static struct scx_sched *scx_tg_knob_sched(struct task_group *tg) +{ + lockdep_assert(lockdep_is_held(&cgroup_mutex) || + lockdep_is_held(&scx_cgroup_ops_rwsem)); + + if (!tg->css.cgroup || !tg->css.parent) + return scx_tg_sched(&root_task_group); + return scx_tg_sched(css_tg(tg->css.parent)); +} + int scx_tg_online(struct task_group *tg) { struct scx_sched *sch = scx_root; @@ -4359,8 +4404,10 @@ int scx_tg_online(struct task_group *tg) if (ret) ret = scx_ops_sanitize_err(sch, "cgroup_init", ret); } - if (ret == 0) + if (ret == 0) { + tg->scx.sched = sch; tg->scx.flags |= SCX_TG_ONLINE | SCX_TG_INITED; + } } else { tg->scx.flags |= SCX_TG_ONLINE; } @@ -4370,19 +4417,30 @@ int scx_tg_online(struct task_group *tg) void scx_tg_offline(struct task_group *tg) { - struct scx_sched *sch = scx_root; + struct scx_sched *sch = tg->scx.sched; WARN_ON_ONCE(!(tg->scx.flags & SCX_TG_ONLINE)); - if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_exit) && - (tg->scx.flags & SCX_TG_INITED)) + /* INITED implies non-NULL @sch, test before SCX_HAS_OP() derefs */ + if (scx_cgroup_enabled && (tg->scx.flags & SCX_TG_INITED) && + SCX_HAS_OP(sch, cgroup_exit)) SCX_CALL_OP(sch, cgroup_exit, NULL, tg->css.cgroup); + tg->scx.sched = NULL; tg->scx.flags &= ~(SCX_TG_ONLINE | SCX_TG_INITED); } +/* + * @p's sched for the cgroup migration paths. Stable as re-homes happen either + * at CGROUP_TASK_MIGRATED of the same migration or under scx_cgroup_lock(), + * both while holding cgroup_mutex. + */ +static struct scx_sched *scx_cgroup_task_sched(struct task_struct *p) +{ + return rcu_dereference_protected(p->scx.sched, lockdep_is_held(&cgroup_mutex)); +} + int scx_cgroup_can_attach(struct cgroup_taskset *tset) { - struct scx_sched *sch = scx_root; struct cgroup_subsys_state *css; struct task_struct *p; int ret; @@ -4391,6 +4449,7 @@ int scx_cgroup_can_attach(struct cgroup_taskset *tset) return 0; cgroup_taskset_for_each(p, css, tset) { + struct scx_sched *sch = scx_cgroup_task_sched(p); struct cgroup *from = tg_cgrp(task_group(p)); struct cgroup *to = tg_cgrp(css_tg(css)); @@ -4404,11 +4463,22 @@ int scx_cgroup_can_attach(struct cgroup_taskset *tset) if (from == to) continue; + /* + * The cgroup_move ops are delivered to @p's sched, and only for + * moves that don't re-home @p. A re-homing move changes the dfl + * cgroup's sched and is reported through the + * exit_task/init_task pair that the re-homing generates. + */ + if (!sch || sch != task_css_set(p)->mg_dst_cset->dfl_cgrp->scx_sched) + continue; + if (SCX_HAS_OP(sch, cgroup_prep_move)) { ret = SCX_CALL_OP_RET(sch, cgroup_prep_move, NULL, p, from, css->cgroup); - if (ret) + if (ret) { + ret = scx_ops_sanitize_err(sch, "cgroup_prep_move", ret); goto err; + } } p->scx.cgrp_moving_from = from; @@ -4418,41 +4488,41 @@ int scx_cgroup_can_attach(struct cgroup_taskset *tset) err: cgroup_taskset_for_each(p, css, tset) { - if (SCX_HAS_OP(sch, cgroup_cancel_move) && - p->scx.cgrp_moving_from) + struct scx_sched *sch = scx_cgroup_task_sched(p); + + /* cgrp_moving_from implies non-NULL @sch, test it first */ + if (p->scx.cgrp_moving_from && SCX_HAS_OP(sch, cgroup_cancel_move)) SCX_CALL_OP(sch, cgroup_cancel_move, NULL, p, p->scx.cgrp_moving_from, css->cgroup); p->scx.cgrp_moving_from = NULL; } - return scx_ops_sanitize_err(sch, "cgroup_prep_move", ret); + return ret; } void scx_cgroup_move_task(struct task_struct *p) { - struct scx_sched *sch = scx_root; + struct scx_sched *sch; if (!scx_cgroup_enabled) return; /* - * scx_cgroup_can_attach() sets cgrp_moving_from only when the task's - * cgroup changes. Migration keys off css rather than cgroup identity, - * so it can hand an unchanged-cgroup task here with cgrp_moving_from - * NULL. Nothing to report to the BPF scheduler then, so skip it and - * keep prep_move and move paired. Cgroup ops run on the root sched, - * dispatch on the explicit @sch. + * Migration keys off css rather than cgroup identity, so it can hand an + * unchanged-cgroup task here with cgrp_moving_from NULL. Nothing to + * report to the BPF scheduler then, so skip it and keep prep_move and + * move paired. */ - if (SCX_HAS_OP(sch, cgroup_move) && p->scx.cgrp_moving_from) - __SCX_CALL_OP_TASK(sch, ops, cgroup_move, task_rq(p), - p, p->scx.cgrp_moving_from, - tg_cgrp(task_group(p))); + sch = scx_cgroup_task_sched(p); + if (p->scx.cgrp_moving_from && SCX_HAS_OP(sch, cgroup_move)) + SCX_CALL_OP_TASK(sch, cgroup_move, task_rq(p), + p, p->scx.cgrp_moving_from, + tg_cgrp(task_group(p))); p->scx.cgrp_moving_from = NULL; } void scx_cgroup_cancel_attach(struct cgroup_taskset *tset) { - struct scx_sched *sch = scx_root; struct cgroup_subsys_state *css; struct task_struct *p; @@ -4460,8 +4530,10 @@ void scx_cgroup_cancel_attach(struct cgroup_taskset *tset) return; cgroup_taskset_for_each(p, css, tset) { - if (SCX_HAS_OP(sch, cgroup_cancel_move) && - p->scx.cgrp_moving_from) + struct scx_sched *sch = scx_cgroup_task_sched(p); + + /* cgrp_moving_from implies non-NULL @sch, test it first */ + if (p->scx.cgrp_moving_from && SCX_HAS_OP(sch, cgroup_cancel_move)) SCX_CALL_OP(sch, cgroup_cancel_move, NULL, p, p->scx.cgrp_moving_from, css->cgroup); p->scx.cgrp_moving_from = NULL; @@ -4473,7 +4545,7 @@ void scx_group_set_weight(struct task_group *tg, unsigned long weight) struct scx_sched *sch; percpu_down_read(&scx_cgroup_ops_rwsem); - sch = scx_root; + sch = scx_tg_knob_sched(tg); if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_weight) && tg->scx.weight != weight) @@ -4489,7 +4561,7 @@ void scx_group_set_idle(struct task_group *tg, bool idle) struct scx_sched *sch; percpu_down_read(&scx_cgroup_ops_rwsem); - sch = scx_root; + sch = scx_tg_knob_sched(tg); if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_idle)) SCX_CALL_OP(sch, cgroup_set_idle, NULL, tg_cgrp(tg), idle); @@ -4506,7 +4578,7 @@ void scx_group_set_bandwidth(struct task_group *tg, struct scx_sched *sch; percpu_down_read(&scx_cgroup_ops_rwsem); - sch = scx_root; + sch = scx_tg_knob_sched(tg); if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_bandwidth) && (tg->scx.bw_period_us != period_us || @@ -4718,6 +4790,7 @@ static void scx_cgroup_exit(struct scx_sched *sch) if (!(tg->scx.flags & SCX_TG_INITED)) continue; + tg->scx.sched = NULL; tg->scx.flags &= ~SCX_TG_INITED; if (!sch->ops.cgroup_exit) @@ -4738,28 +4811,26 @@ static int scx_cgroup_init(struct scx_sched *sch) */ css_for_each_descendant_pre(css, &root_task_group.css) { struct task_group *tg = css_tg(css); - struct scx_cgroup_init_args args = { - .weight = tg->scx.weight, - .bw_period_us = tg->scx.bw_period_us, - .bw_quota_us = tg->scx.bw_quota_us, - .bw_burst_us = tg->scx.bw_burst_us, - }; - if ((tg->scx.flags & - (SCX_TG_ONLINE | SCX_TG_INITED)) != SCX_TG_ONLINE) + if ((tg->scx.flags & (SCX_TG_ONLINE | SCX_TG_INITED)) != SCX_TG_ONLINE) continue; - if (!sch->ops.cgroup_init) { - tg->scx.flags |= SCX_TG_INITED; - continue; - } + if (sch->ops.cgroup_init) { + struct scx_cgroup_init_args args = { + .weight = tg->scx.weight, + .bw_period_us = tg->scx.bw_period_us, + .bw_quota_us = tg->scx.bw_quota_us, + .bw_burst_us = tg->scx.bw_burst_us, + }; - ret = SCX_CALL_OP_RET(sch, cgroup_init, NULL, - css->cgroup, &args); - if (ret) { - scx_error(sch, "ops.cgroup_init() failed (%d)", ret); - return ret; + ret = SCX_CALL_OP_RET(sch, cgroup_init, NULL, css->cgroup, &args); + if (ret) { + scx_error(sch, "ops.cgroup_init() failed (%d)", ret); + return ret; + } } + + tg->scx.sched = sch; tg->scx.flags |= SCX_TG_INITED; } diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 23fc95502ea6..ad90d4645a33 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -645,8 +645,14 @@ struct sched_ext_ops { * @cgrp: cgroup being initialized * @args: init arguments, see the struct definition * - * Either the BPF scheduler is being loaded or @cgrp created, initialize - * @cgrp for sched_ext. This operation may block. + * Initialize @cgrp for sched_ext, delivered to @cgrp's sched either + * when the BPF scheduler is being loaded or when @cgrp is created. This + * operation may block. + * + * When the BPF scheduler is being loaded or cgroups are being handed + * over, @cgrp may already have been removed by userspace: a removed + * cgroup stays schedulable until its dying tasks finish their final + * context switches. * * Return 0 for success, -errno for failure. An error return while * loading will abort loading of the BPF scheduler. During cgroup @@ -659,8 +665,13 @@ struct sched_ext_ops { * @cgroup_exit: Exit a cgroup * @cgrp: cgroup being exited * - * Either the BPF scheduler is being unloaded or @cgrp destroyed, exit - * @cgrp for sched_ext. This operation my block. + * Exit @cgrp for sched_ext, delivered to the sched whose + * ops.cgroup_init() it pairs with, either when the BPF scheduler is + * being unloaded or when @cgrp is destroyed. This operation may block. + * + * For a destroyed @cgrp, delivery follows the last scheduling event on + * it: a removed cgroup stays schedulable until its dying tasks finish + * their final context switches. */ void (*cgroup_exit)(struct cgroup *cgrp); @@ -673,6 +684,12 @@ struct sched_ext_ops { * Prepare @p for move from cgroup @from to @to. This operation may * block and can be used for allocations. * + * The cgroup_move ops are delivered to @p's sched, and only for moves + * that don't re-home @p. A re-homing move is reported through + * ops.exit_task() and ops.init_task() instead. @from and @to can + * reference cgroups the sched never received ops.cgroup_init() for, as + * the cpu controller can be coarser than the sub-scheduler topology. + * * Return 0 for success, -errno for failure. An error return aborts the * migration. */ @@ -708,6 +725,11 @@ struct sched_ext_ops { * @weight: new weight [1..10000] * * Update @cgrp's weight to @weight. + * + * Knobs of a cgroup belong to the parent, so the set_* ops are + * delivered to @cgrp's parent's sched. That sched may never have seen + * ops.cgroup_init() for @cgrp - at a sub-scheduler attach point, the + * parent sched tracks @cgrp through ops.sub_attach() instead. */ void (*cgroup_set_weight)(struct cgroup *cgrp, u32 weight); @@ -728,6 +750,8 @@ struct sched_ext_ops { * burst temporarily. The specific control mechanism and thus the * interpretation of @period_us and burstiness is up to the BPF * scheduler. + * + * Delivery follows the same rule as cgroup_set_weight(). */ void (*cgroup_set_bandwidth)(struct cgroup *cgrp, u64 period_us, u64 quota_us, u64 burst_us); @@ -740,6 +764,8 @@ struct sched_ext_ops { * Update @cgrp's idle state to @idle. This callback is invoked when * a cgroup transitions between idle and non-idle states, allowing the * BPF scheduler to adjust its behavior accordingly. + * + * Delivery follows the same rule as cgroup_set_weight(). */ void (*cgroup_set_idle)(struct cgroup *cgrp, bool idle); -- cgit v1.2.3 From a6ec0b62c589c5b5518e0e3d9eaac6398a82b6f4 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 17 Jul 2026 22:12:20 -1000 Subject: sched_ext: Hand over cgroups at sub-scheduler enable/disable Sub-schedulers don't get cgroups yet: every task_group is inited on the root sched and the routing added by the previous patches always resolves to it. Add the handover: an enabling sub-scheduler takes over the cgroups in its subtree and a disabling one returns them to its parent. scx_cgroup_claim_subtree() runs while the sub enables, after the subtree's cgrp->scx_sched's are set and before any task is claimed. It inits each subtree task_group on the sub, exits it from the parent and updates tg->scx.sched. A failed ops.cgroup_init() unwinds the sub-side inits and aborts the enable with the parent untouched. Disabling reverses it with scx_cgroup_return_subtree(): exit each cgroup from the sub, then re-init it on the parent with the current tg->scx.* values, resyncing weight and bandwidth changes made while the sub had it. When a re-init fails, the parent is failed and the remaining task_groups still transfer uninited and get no cgroup ops - the same punting done for tasks. The dying parent's own disable moves them onward. The handover walks include dying but not yet offlined task_groups, the same as root's bulk walks: a removed cgroup keeps hosting scheduling events until its dying tasks finish their final context switches, and its ops.cgroup_exit() must follow the last of them. tg on/offlining is excluded through cgroup_lock(), so either ordering against an rmdir of a subtree cgroup delivers balanced init/exit pairs. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- include/linux/sched/ext.h | 13 +++ kernel/sched/ext/ext.c | 45 +++++++--- kernel/sched/ext/internal.h | 6 ++ kernel/sched/ext/sub.c | 199 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 250 insertions(+), 13 deletions(-) (limited to 'kernel') diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index a6db5d300f30..78b2f289cb98 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -298,6 +298,19 @@ static inline bool scx_rcu_cpu_stall(const struct cpumask *stalled_mask) { retur struct scx_task_group { #ifdef CONFIG_EXT_GROUP_SCHED + /* + * The sched this tg is on, NULL if none. SCX_TG_INITED tracks whether + * ops.cgroup_init() succeeded on it. When a child sched exits and its + * tgs move to the parent, a failed init leaves the tg on the parent + * with INITED clear (see scx_cgroup_return_subtree()). + * + * This is tracked separately from cgrp->scx_sched because the tg + * hierarchy can diverge from the cgroup2 hierarchy in both lifetime and + * shape. A tg stays online past its cgroup's removal while the + * cgrp->scx_sched rewrites visit only live cgroups, leaving a removed + * cgroup's pointer stale. The cpu controller can also be mounted on + * cgroup1. + */ struct scx_sched *sched; u32 flags; /* SCX_TG_* */ diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index b542900da5a3..58cd971e5fc5 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -4347,6 +4347,11 @@ void scx_tg_init(struct task_group *tg) * isn't inited. An autogroup tg has no cgroup of its own and resolves to the * root sched. * + * When a child sched exits, its task_groups are moved to the parent and + * re-inited on it. A failed re-init fails the parent in turn and leaves the + * task_group without a sched it's inited on, resolving to %NULL. See + * scx_cgroup_return_subtree(). + * * Safe for callers read-locking the ops rwsem. tg->scx.sched rewrites * write-lock it, and tg on/offline can't overlap such callers as a css's files * are created after online and drained before offline. @@ -4358,7 +4363,8 @@ static struct scx_sched *scx_tg_sched(struct task_group *tg) if (!tg->css.cgroup) tg = &root_task_group; - return tg->scx.sched; + /* INITED means ops.cgroup_init() succeeded on @tg->scx.sched */ + return (tg->scx.flags & SCX_TG_INITED) ? tg->scx.sched : NULL; } /** @@ -4370,6 +4376,9 @@ static struct scx_sched *scx_tg_sched(struct task_group *tg) * at a sub-scheduler attach point, where the sub's parent sched receives * them. * + * Return %NULL if the parent task_group has no sched. That can happen when the + * parent's ops.cgroup_init() fails while a sub-scheduler is being disabled. + * * The callers sit in @tg's cgroup file writes holding the ops rwsem read * side. That extends scx_tg_sched()'s file-write argument to the parent's * sched read: a parent css outlives its children's files. @@ -4386,12 +4395,24 @@ static struct scx_sched *scx_tg_knob_sched(struct task_group *tg) int scx_tg_online(struct task_group *tg) { - struct scx_sched *sch = scx_root; int ret = 0; WARN_ON_ONCE(tg->scx.flags & (SCX_TG_ONLINE | SCX_TG_INITED)); if (scx_cgroup_enabled) { + struct scx_sched *sch; + + /* + * The cgroup lifetime notifier populates cgrp->scx_sched before + * css_online, but only on the default hierarchy. Sub-scheds are + * attached to the cgroup2 hierarchy, so a cgroup1 task_group + * always belongs to the root sched. + */ + if (cgroup_on_dfl(tg->css.cgroup)) + sch = tg->css.cgroup->scx_sched; + else + sch = scx_tg_sched(&root_task_group); + if (SCX_HAS_OP(sch, cgroup_init)) { struct scx_cgroup_init_args args = { .weight = tg->scx.weight, @@ -4547,7 +4568,7 @@ void scx_group_set_weight(struct task_group *tg, unsigned long weight) percpu_down_read(&scx_cgroup_ops_rwsem); sch = scx_tg_knob_sched(tg); - if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_weight) && + if (scx_cgroup_enabled && sch && SCX_HAS_OP(sch, cgroup_set_weight) && tg->scx.weight != weight) SCX_CALL_OP(sch, cgroup_set_weight, NULL, tg_cgrp(tg), weight); @@ -4563,7 +4584,7 @@ void scx_group_set_idle(struct task_group *tg, bool idle) percpu_down_read(&scx_cgroup_ops_rwsem); sch = scx_tg_knob_sched(tg); - if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_idle)) + if (scx_cgroup_enabled && sch && SCX_HAS_OP(sch, cgroup_set_idle)) SCX_CALL_OP(sch, cgroup_set_idle, NULL, tg_cgrp(tg), idle); /* Update the task group's idle state */ @@ -4580,7 +4601,7 @@ void scx_group_set_bandwidth(struct task_group *tg, percpu_down_read(&scx_cgroup_ops_rwsem); sch = scx_tg_knob_sched(tg); - if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_bandwidth) && + if (scx_cgroup_enabled && sch && SCX_HAS_OP(sch, cgroup_set_bandwidth) && (tg->scx.bw_period_us != period_us || tg->scx.bw_quota_us != quota_us || tg->scx.bw_burst_us != burst_us)) @@ -4788,15 +4809,13 @@ static void scx_cgroup_exit(struct scx_sched *sch) css_for_each_descendant_post(css, &root_task_group.css) { struct task_group *tg = css_tg(css); - if (!(tg->scx.flags & SCX_TG_INITED)) - continue; + /* also clear the sched of tgs whose ops.cgroup_init() failed */ tg->scx.sched = NULL; - tg->scx.flags &= ~SCX_TG_INITED; - - if (!sch->ops.cgroup_exit) - continue; - - SCX_CALL_OP(sch, cgroup_exit, NULL, css->cgroup); + if (tg->scx.flags & SCX_TG_INITED) { + tg->scx.flags &= ~SCX_TG_INITED; + if (sch->ops.cgroup_exit) + SCX_CALL_OP(sch, cgroup_exit, NULL, css->cgroup); + } } } diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index ad90d4645a33..0c0a8fdaa2c4 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -649,6 +649,11 @@ struct sched_ext_ops { * when the BPF scheduler is being loaded or when @cgrp is created. This * operation may block. * + * Cgroup handovers also generate these ops: an enabling sub-scheduler + * receives ops.cgroup_init() for every cgroup in its subtree while the + * previous sched receives ops.cgroup_exit(), and disabling reverses the + * two. + * * When the BPF scheduler is being loaded or cgroups are being handed * over, @cgrp may already have been removed by userspace: a removed * cgroup stays schedulable until its dying tasks finish their final @@ -1722,6 +1727,7 @@ enum scx_kick_flags { enum scx_tg_flags { SCX_TG_ONLINE = 1U << 0, SCX_TG_INITED = 1U << 1, + SCX_TG_SUB_INIT = 1U << 2, /* see scx_cgroup_claim_subtree() */ }; enum scx_enable_state { diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 393dbd00d2f7..8d8737149bc0 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -836,6 +836,191 @@ static void scx_fail_parent(struct scx_sched *sch, scx_task_iter_stop(&sti); } +#ifdef CONFIG_EXT_GROUP_SCHED +/** + * scx_cgroup_claim_subtree - Claim the subtree's cgroups for an enabling sub + * @sch: sub-scheduler being enabled + * + * Called while enabling @sch, after the subtree's cgrp->scx_sched's are pointed + * at @sch and before any task is claimed. This mirrors root enable's + * cgroups-before-tasks order. The ops.init_task() args are task_group-granular + * and can still reference a cgroup outside the handed-over set when the cpu + * controller is coarser than the sub topology or mounted on cgroup1. + * + * First init each of the parent sched's subtree cgroups on @sch, and only then + * exit them from the parent, so that a failed init can be unwound with the + * parent untouched. The both-inited transient is invisible outside + * scx_cgroup_lock(). %SCX_TG_SUB_INIT tracks the first pass's progress. + * %SCX_TG_INITED stays set throughout, except for a task_group whose + * ops.cgroup_init() failed on the parent (see scx_cgroup_return_subtree()): + * there is nothing to exit from the parent and %SCX_TG_INITED is set back with + * the transfer. + * + * Dying but not yet offlined task_groups are included: a removed cgroup keeps + * hosting scheduling events until its dying tasks finish their final context + * switches, so it still needs to be inited on a sched, and its offline-time + * ops.cgroup_exit() follows the last of those events. + * + * Return 0 on success, -errno on failure. On failure, @sch has been + * scx_error()'d and is left with no cgroups. + */ +static s32 scx_cgroup_claim_subtree(struct scx_sched *sch) +{ + struct cgroup *sub_cgrp = sch_cgroup(sch); + struct cgroup_subsys_state *ecss = cgroup_e_css(sub_cgrp, &cpu_cgrp_subsys); + struct scx_sched *parent = scx_parent(sch); + struct cgroup_subsys_state *css; + int ret; + + css_for_each_descendant_pre(css, ecss) { + struct task_group *tg = css_tg(css); + struct scx_cgroup_init_args args = { + .weight = tg->scx.weight, + .bw_period_us = tg->scx.bw_period_us, + .bw_quota_us = tg->scx.bw_quota_us, + .bw_burst_us = tg->scx.bw_burst_us, + }; + + if (tg->scx.sched != parent || + !cgroup_is_descendant(css->cgroup, sub_cgrp)) + continue; + + if (SCX_HAS_OP(sch, cgroup_init)) { + ret = SCX_CALL_OP_RET(sch, cgroup_init, NULL, css->cgroup, &args); + if (ret) { + scx_error(sch, "ops.cgroup_init() failed (%d)", ret); + goto err; + } + } + tg->scx.flags |= SCX_TG_SUB_INIT; + } + + css_for_each_descendant_post(css, ecss) { + struct task_group *tg = css_tg(css); + + /* + * SUB_INIT is pass 1's progress mark: pass 2 and the err path + * must visit exactly the tgs pass 1 inited. + */ + if (!(tg->scx.flags & SCX_TG_SUB_INIT)) + continue; + + /* skip the exit if the parent's ops.cgroup_init() failed */ + if ((tg->scx.flags & SCX_TG_INITED) && SCX_HAS_OP(parent, cgroup_exit)) + SCX_CALL_OP(parent, cgroup_exit, NULL, css->cgroup); + tg->scx.sched = sch; + tg->scx.flags |= SCX_TG_INITED; + tg->scx.flags &= ~SCX_TG_SUB_INIT; + } + + return 0; + +err: + css_for_each_descendant_post(css, ecss) { + struct task_group *tg = css_tg(css); + + if (!(tg->scx.flags & SCX_TG_SUB_INIT)) + continue; + + if (SCX_HAS_OP(sch, cgroup_exit)) + SCX_CALL_OP(sch, cgroup_exit, NULL, css->cgroup); + tg->scx.flags &= ~SCX_TG_SUB_INIT; + } + return ret; +} + +/** + * scx_cgroup_return_subtree - Return the subtree's cgroups to the parent sched + * @sch: sub-scheduler being disabled + * + * Called while disabling @sch, after the subtree's cgrp->scx_sched's are reset + * to the parent sched and before tasks are re-homed, mirroring root disable's + * cgroups-before-tasks teardown order. The reverse of + * scx_cgroup_claim_subtree(): exit @sch's cgroups from @sch, then init them on + * the parent with the current tg->scx.* values, resyncing settings that changed + * while @sch had them. + * + * When an init on the parent fails, the parent is failed - the same policy as + * task re-homing. The remaining task_groups are punted: they move to the parent + * anyway with %SCX_TG_INITED cleared, as ops.cgroup_init() failed or never ran + * for them. A punted task_group gets no cgroup ops. The dying parent's own + * disable moves it one sched up, initing it there. Root ends the chain: root + * teardown drops cgroup ops entirely and the next enable's bulk init re-inits + * every online task_group. + * + * The task re-home that follows still delivers ops.init_task() to the dying + * parent, including for tasks in punted cgroups it never inited - tolerated + * like the downstream failures of task punting (see scx_punt_task()). + */ +static void scx_cgroup_return_subtree(struct scx_sched *sch) +{ + struct cgroup *sub_cgrp = sch_cgroup(sch); + struct cgroup_subsys_state *ecss = cgroup_e_css(sub_cgrp, &cpu_cgrp_subsys); + struct scx_sched *parent = scx_parent(sch); + struct cgroup_subsys_state *css; + bool parent_failed = false; + int ret; + + css_for_each_descendant_post(css, ecss) { + struct task_group *tg = css_tg(css); + + if (tg->scx.sched != sch || + !cgroup_is_descendant(css->cgroup, sub_cgrp)) + continue; + + /* skip the exit if @sch's ops.cgroup_init() failed for the tg */ + if ((tg->scx.flags & SCX_TG_INITED) && SCX_HAS_OP(sch, cgroup_exit)) + SCX_CALL_OP(sch, cgroup_exit, NULL, css->cgroup); + tg->scx.sched = parent; + tg->scx.flags |= SCX_TG_SUB_INIT; + } + + css_for_each_descendant_pre(css, ecss) { + struct task_group *tg = css_tg(css); + struct scx_cgroup_init_args args = { + .weight = tg->scx.weight, + .bw_period_us = tg->scx.bw_period_us, + .bw_quota_us = tg->scx.bw_quota_us, + .bw_burst_us = tg->scx.bw_burst_us, + }; + + /* the first pass must have transferred everything */ + WARN_ON_ONCE(tg->scx.sched == sch); + + /* + * SUB_INIT distinguishes the tgs pass 1 moved. The sched test + * can't: a tg punted to the parent by an earlier failure would + * also match. + */ + if (!(tg->scx.flags & SCX_TG_SUB_INIT)) + continue; + tg->scx.flags &= ~(SCX_TG_SUB_INIT | SCX_TG_INITED); + + /* + * A re-init on $parent failed. The task_groups from here on are + * punted: they stay on the dying $parent with INITED clear and + * move onward when it disables. + */ + if (parent_failed) + continue; + + if (SCX_HAS_OP(parent, cgroup_init)) { + ret = SCX_CALL_OP_RET(parent, cgroup_init, NULL, css->cgroup, &args); + if (ret) { + scx_error(parent, "ops.cgroup_init() failed (%d) while disabling a sub-scheduler", + ret); + parent_failed = true; + continue; + } + } + tg->scx.flags |= SCX_TG_INITED; + } +} +#else +static inline s32 scx_cgroup_claim_subtree(struct scx_sched *sch) { return 0; } +static inline void scx_cgroup_return_subtree(struct scx_sched *sch) {} +#endif + void scx_sub_disable(struct scx_sched *sch) { struct scx_sched *parent = scx_parent(sch); @@ -871,6 +1056,12 @@ void scx_sub_disable(struct scx_sched *sch) set_cgroup_sched(sch_cgroup(sch), parent); + /* + * Return the subtree's cgroups before re-homing tasks so that any + * ops.init_task() on $parent only sees cgroups it has initialized. + */ + scx_cgroup_return_subtree(sch); + scx_task_iter_start(&sti, sch->cgrp); while ((p = scx_task_iter_next_locked(&sti))) { struct rq *rq; @@ -1172,6 +1363,14 @@ void scx_sub_enable_workfn(struct kthread_work *work) goto err_unlock_and_disable; } + /* + * Take over the subtree's cgroups before any task is claimed, + * mirroring root enable's cgroups-before-tasks order. + */ + ret = scx_cgroup_claim_subtree(sch); + if (ret) + goto err_unlock_and_disable; + /* * Initialize tasks for the new child $sch without exiting them for * $parent so that the tasks can always be reverted back to $parent -- cgit v1.2.3 From b20dfde5ec543597f7f178f7b96cf6ebcc50e5ee Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 18 Jul 2026 00:04:25 -1000 Subject: sched_ext: Rename the cid-form cgroup ops to cpuctl_* Two unrelated things go by "cgroup" in the cid form. Sub-schedulers attach to cgroups, and the cgroup_*() ops deliver cpu controller events. While the ops names suggest cgroup2 hierarchy, they actually operate on the cpu controller. Rename them to cpuctl_* in struct sched_ext_ops_cid, which has no users outside scx_qmap yet. The cpu form is deployed ABI and keeps the old names. The layout is unchanged and the kernel keeps calling through the cpu-form union view. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 36 ++++++++++++++++++------------------ kernel/sched/ext/internal.h | 25 ++++++++++++------------- tools/sched_ext/scx_qmap.bpf.c | 22 +++++++++++----------- tools/sched_ext/scx_qmap.c | 2 +- tools/sched_ext/scx_qmap.h | 2 +- 5 files changed, 43 insertions(+), 44 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 58cd971e5fc5..b730eac4b13f 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -7952,14 +7952,14 @@ static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = { .enable = sched_ext_ops__enable, .disable = sched_ext_ops__disable, #ifdef CONFIG_EXT_GROUP_SCHED - .cgroup_init = sched_ext_ops__cgroup_init, - .cgroup_exit = sched_ext_ops__cgroup_exit, - .cgroup_prep_move = sched_ext_ops__cgroup_prep_move, - .cgroup_move = sched_ext_ops__cgroup_move, - .cgroup_cancel_move = sched_ext_ops__cgroup_cancel_move, - .cgroup_set_weight = sched_ext_ops__cgroup_set_weight, - .cgroup_set_bandwidth = sched_ext_ops__cgroup_set_bandwidth, - .cgroup_set_idle = sched_ext_ops__cgroup_set_idle, + .cpuctl_init = sched_ext_ops__cgroup_init, + .cpuctl_exit = sched_ext_ops__cgroup_exit, + .cpuctl_prep_move = sched_ext_ops__cgroup_prep_move, + .cpuctl_move = sched_ext_ops__cgroup_move, + .cpuctl_cancel_move = sched_ext_ops__cgroup_cancel_move, + .cpuctl_set_weight = sched_ext_ops__cgroup_set_weight, + .cpuctl_set_bandwidth = sched_ext_ops__cgroup_set_bandwidth, + .cpuctl_set_idle = sched_ext_ops__cgroup_set_idle, #endif .sub_attach = sched_ext_ops__sub_attach, .sub_detach = sched_ext_ops__sub_detach, @@ -10479,22 +10479,22 @@ static int __init scx_init(void) CID_OFFSET_MATCH(init_cids, init_cids); CID_OFFSET_MATCH(init, init); CID_OFFSET_MATCH(exit, exit); -#ifdef CONFIG_EXT_GROUP_SCHED - CID_OFFSET_MATCH(cgroup_init, cgroup_init); - CID_OFFSET_MATCH(cgroup_exit, cgroup_exit); - CID_OFFSET_MATCH(cgroup_prep_move, cgroup_prep_move); - CID_OFFSET_MATCH(cgroup_move, cgroup_move); - CID_OFFSET_MATCH(cgroup_cancel_move, cgroup_cancel_move); - CID_OFFSET_MATCH(cgroup_set_weight, cgroup_set_weight); - CID_OFFSET_MATCH(cgroup_set_bandwidth, cgroup_set_bandwidth); - CID_OFFSET_MATCH(cgroup_set_idle, cgroup_set_idle); -#endif /* renamed callbacks must occupy the same slot as their cpu-form sibling */ CID_OFFSET_MATCH(select_cpu, select_cid); CID_OFFSET_MATCH(set_cpumask, set_cmask); CID_OFFSET_MATCH(cpu_online, cid_online); CID_OFFSET_MATCH(cpu_offline, cid_offline); CID_OFFSET_MATCH(dump_cpu, dump_cid); +#ifdef CONFIG_EXT_GROUP_SCHED + CID_OFFSET_MATCH(cgroup_init, cpuctl_init); + CID_OFFSET_MATCH(cgroup_exit, cpuctl_exit); + CID_OFFSET_MATCH(cgroup_prep_move, cpuctl_prep_move); + CID_OFFSET_MATCH(cgroup_move, cpuctl_move); + CID_OFFSET_MATCH(cgroup_cancel_move, cpuctl_cancel_move); + CID_OFFSET_MATCH(cgroup_set_weight, cpuctl_set_weight); + CID_OFFSET_MATCH(cgroup_set_bandwidth, cpuctl_set_bandwidth); + CID_OFFSET_MATCH(cgroup_set_idle, cpuctl_set_idle); +#endif /* @priv tail must align since both share the same data block */ CID_OFFSET_MATCH(priv, priv); /* diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 0c0a8fdaa2c4..26bfda216524 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -992,6 +992,7 @@ struct sched_ext_ops { * - cpu_online -> cid_online * - cpu_offline -> cid_offline * - dump_cpu -> dump_cid + * - cgroup_* -> cpuctl_* (they track the cgroup cpu controller) * - cpu_acquire/cpu_release -> not present (deprecated in sched_ext_ops) * * BPF schedulers using this type cannot call cpu-form scx_bpf_* kfuncs; @@ -1027,19 +1028,17 @@ struct sched_ext_ops_cid { void (*dump_cid)(struct scx_dump_ctx *ctx, s32 cid, bool idle); void (*dump_task)(struct scx_dump_ctx *ctx, struct task_struct *p); #ifdef CONFIG_EXT_GROUP_SCHED - s32 (*cgroup_init)(struct cgroup *cgrp, - struct scx_cgroup_init_args *args); - void (*cgroup_exit)(struct cgroup *cgrp); - s32 (*cgroup_prep_move)(struct task_struct *p, - struct cgroup *from, struct cgroup *to); - void (*cgroup_move)(struct task_struct *p, - struct cgroup *from, struct cgroup *to); - void (*cgroup_cancel_move)(struct task_struct *p, - struct cgroup *from, struct cgroup *to); - void (*cgroup_set_weight)(struct cgroup *cgrp, u32 weight); - void (*cgroup_set_bandwidth)(struct cgroup *cgrp, - u64 period_us, u64 quota_us, u64 burst_us); - void (*cgroup_set_idle)(struct cgroup *cgrp, bool idle); + s32 (*cpuctl_init)(struct cgroup *cgrp, struct scx_cgroup_init_args *args); + void (*cpuctl_exit)(struct cgroup *cgrp); + s32 (*cpuctl_prep_move)(struct task_struct *p, struct cgroup *from, + struct cgroup *to); + void (*cpuctl_move)(struct task_struct *p, struct cgroup *from, struct cgroup *to); + void (*cpuctl_cancel_move)(struct task_struct *p, struct cgroup *from, + struct cgroup *to); + void (*cpuctl_set_weight)(struct cgroup *cgrp, u32 weight); + void (*cpuctl_set_bandwidth)(struct cgroup *cgrp, u64 period_us, u64 quota_us, + u64 burst_us); + void (*cpuctl_set_idle)(struct cgroup *cgrp, bool idle); #endif /* CONFIG_EXT_GROUP_SCHED */ s32 (*sub_attach)(struct scx_sub_attach_args *args); void (*sub_detach)(struct scx_sub_detach_args *args); diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c index 925ae1a1d440..aead17658573 100644 --- a/tools/sched_ext/scx_qmap.bpf.c +++ b/tools/sched_ext/scx_qmap.bpf.c @@ -1014,7 +1014,7 @@ void BPF_STRUCT_OPS(qmap_dump_task, struct scx_dump_ctx *dctx, struct task_struc taskc->force_local, taskc->core_sched_seq); } -s32 BPF_STRUCT_OPS(qmap_cgroup_init, struct cgroup *cgrp, struct scx_cgroup_init_args *args) +s32 BPF_STRUCT_OPS(qmap_cpuctl_init, struct cgroup *cgrp, struct scx_cgroup_init_args *args) { QMAP_TOUCH_ARENA(); @@ -1036,7 +1036,7 @@ s32 BPF_STRUCT_OPS(qmap_cgroup_init, struct cgroup *cgrp, struct scx_cgroup_init static void redistribute(void); -void BPF_STRUCT_OPS(qmap_cgroup_set_weight, struct cgroup *cgrp, u32 weight) +void BPF_STRUCT_OPS(qmap_cpuctl_set_weight, struct cgroup *cgrp, u32 weight) { u64 cgid = cgrp->kn->id; s32 i; @@ -1062,16 +1062,16 @@ void BPF_STRUCT_OPS(qmap_cgroup_set_weight, struct cgroup *cgrp, u32 weight) } } -void BPF_STRUCT_OPS(qmap_cgroup_set_bandwidth, struct cgroup *cgrp, - u64 period_us, u64 quota_us, u64 burst_us) +void BPF_STRUCT_OPS(qmap_cpuctl_set_bandwidth, struct cgroup *cgrp, u64 period_us, + u64 quota_us, u64 burst_us) { if (print_msgs) bpf_printk("CGRP SET %llu period=%lu quota=%ld burst=%lu", cgrp->kn->id, period_us, quota_us, burst_us); } -void BPF_STRUCT_OPS(qmap_cgroup_move, struct task_struct *p, - struct cgroup *from, struct cgroup *to) +void BPF_STRUCT_OPS(qmap_cpuctl_move, struct task_struct *p, struct cgroup *from, + struct cgroup *to) { if (print_msgs) bpf_printk("CGRP MOVE %d %llu -> %llu", @@ -1890,7 +1890,7 @@ void BPF_STRUCT_OPS(qmap_exit, struct scx_exit_info *ei) /* * Seed a new sub slot with the cgroup's current weight. The kernel delivers - * ops.cgroup_set_weight() only on value-changing writes, so a weight set + * ops.cpuctl_set_weight() only on value-changing writes, so a weight set * before the sub attached would otherwise go unnoticed. */ static u32 cgrp_cur_weight(u64 cgid) @@ -1989,10 +1989,10 @@ SCX_OPS_CID_DEFINE(qmap_ops, .dump = (void *)qmap_dump, .dump_cid = (void *)qmap_dump_cid, .dump_task = (void *)qmap_dump_task, - .cgroup_init = (void *)qmap_cgroup_init, - .cgroup_set_weight = (void *)qmap_cgroup_set_weight, - .cgroup_set_bandwidth = (void *)qmap_cgroup_set_bandwidth, - .cgroup_move = (void *)qmap_cgroup_move, + .cpuctl_init = (void *)qmap_cpuctl_init, + .cpuctl_set_weight = (void *)qmap_cpuctl_set_weight, + .cpuctl_set_bandwidth = (void *)qmap_cpuctl_set_bandwidth, + .cpuctl_move = (void *)qmap_cpuctl_move, .sub_attach = (void *)qmap_sub_attach, .sub_detach = (void *)qmap_sub_detach, .sub_caps_updated = (void *)qmap_sub_caps_updated, diff --git a/tools/sched_ext/scx_qmap.c b/tools/sched_ext/scx_qmap.c index ddb524b4946c..27ffda1c519e 100644 --- a/tools/sched_ext/scx_qmap.c +++ b/tools/sched_ext/scx_qmap.c @@ -70,7 +70,7 @@ const char help_fmt[] = " -i SEC Stats interval, seconds (default 5)\n" " -R MS Round-robin period for time-shared cpus, ms (default 200)\n" " -J MODE Fault injection (wrong-cid: dispatch to a cid not held,\n" -" init-fail/cgrp-init-fail: fail init_task/cgroup_init for\n" +" init-fail/cgrp-init-fail: fail init_task/cpuctl_init for\n" " \"qmfail*\" comms/cgroups)\n" " -v Print libbpf debug messages\n" " -h Display this help and exit\n"; diff --git a/tools/sched_ext/scx_qmap.h b/tools/sched_ext/scx_qmap.h index cc2840e7aa3c..c42f7ef74b89 100644 --- a/tools/sched_ext/scx_qmap.h +++ b/tools/sched_ext/scx_qmap.h @@ -65,7 +65,7 @@ enum qmap_inject { QMAP_INJ_OFF = 0, QMAP_INJ_WRONG_CID = 1, /* dispatch to a cid we don't hold */ QMAP_INJ_INIT_FAIL = 2, /* fail init_task for "qmfail*" comms */ - QMAP_INJ_CGRP_INIT_FAIL = 3, /* fail cgroup_init for "qmfail*" cgroups */ + QMAP_INJ_CGRP_INIT_FAIL = 3, /* fail cpuctl_init for "qmfail*" cgroups */ }; /* -- cgit v1.2.3 From 2c91169377f44e2f2b0b79bc0365f0722987a650 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Tue, 21 Jul 2026 11:25:14 -1000 Subject: sched_ext: Blame the DSQ's owning scheduler for a runnable stall check_rq_for_timeouts() blames a runnable stall on the task's owner. Under a sub-scheduler hierarchy the stalled task can be sitting on a DSQ that a different scheduler has to drain, e.g. an ancestor's bypass DSQ while the owner is bypassing. The drainer then escapes blame while the owner is exited, and when the owner's exit is already claimed, nothing actionable is reported at all. Blame the DSQ's owning scheduler instead. The local DSQ is consumed by the cpu itself and keeps blame on the owner. Detection keeps the owner's timeout and single-scheduler behavior is unchanged. Reviewed-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index b730eac4b13f..96389ec31ab8 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -3391,8 +3391,19 @@ static bool check_rq_for_timeouts(struct rq *rq) if (unlikely(time_after(jiffies, last_runnable + READ_ONCE(sch->watchdog_timeout)))) { + struct scx_dispatch_q *dsq = READ_ONCE(p->scx.dsq); u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); + /* + * A task can be stuck on a DSQ that a sched other than + * its owner is responsible for draining, e.g. an + * ancestor's bypass DSQ while the owner is bypassing. + * Blame the drainer. The local DSQ is consumed by the + * cpu itself and keeps blame on the owner. + */ + if (dsq && dsq->sched && dsq->id != SCX_DSQ_LOCAL) + sch = dsq->sched; + __scx_exit(sch, SCX_EXIT_ERROR_STALL, 0, cpu_of(rq), "%s[%d] failed to run for %u.%03us", p->comm, p->pid, dur_ms / 1000, -- cgit v1.2.3 From 5f860e647de21269c3b7f03be9d892d6af46b972 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Tue, 21 Jul 2026 11:25:14 -1000 Subject: sched_ext: Skip the default CPU selection while bypassing select_task_rq_scx() falls into the default path when the scheduler has no ops.select_cpu or is bypassing. There it calls scx_select_cpu_dfl() and direct-dispatches to the picked CPU's local DSQ. While bypassing, neither does anything: the enqueue path routes the task to a bypass DSQ before consulting the direct-dispatch target, so the direct dispatch never happens, and the CPU pick at most shifts which CPU's bypass DSQ receives the task. Worse, when the scheduler does its own idle tracking, the built-in idle cpumasks the pick consults are not even updated, so it doesn't work anyway. Return prev_cpu without the default selection while bypassing and let the bypass enqueue place the task. Reviewed-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 96389ec31ab8..e6c4cbe1f182 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -3277,6 +3277,19 @@ static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flag } else { s32 cpu; + /* + * While bypassing, the enqueue path routes @p to a bypass DSQ + * without consulting the direct-dispatch target, making the + * default selection pointless. It doesn't work anyway when the + * scheduler does its own idle tracking and the built-in idle + * cpumasks are not updated. Leave @p on @prev_cpu. + */ + if (bypassing) { + __scx_add_event(sch, SCX_EV_BYPASS_DISPATCH, 1); + p->scx.selected_cpu = prev_cpu; + return prev_cpu; + } + cpu = scx_select_cpu_dfl(p, prev_cpu, wake_flags, NULL, 0); if (cpu >= 0) { refill_task_slice_dfl(sch, p); @@ -3286,8 +3299,6 @@ static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flag } p->scx.selected_cpu = cpu; - if (bypassing) - __scx_add_event(sch, SCX_EV_BYPASS_DISPATCH, 1); return cpu; } } -- cgit v1.2.3 From 8cc4909134f290fdf0a1848cf3e109d015918f09 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Tue, 21 Jul 2026 11:25:14 -1000 Subject: sched_ext: Drop unused scx_cpumask_to_cmask() scx_cpumask_to_cmask() has no callers. Reviewed-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/cid.c | 21 --------------------- kernel/sched/ext/cid.h | 1 - 2 files changed, 22 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/cid.c b/kernel/sched/ext/cid.c index 9dfd242be34f..5f990d95735f 100644 --- a/kernel/sched/ext/cid.c +++ b/kernel/sched/ext/cid.c @@ -371,27 +371,6 @@ void scx_cmask_fill(struct scx_cmask *m) m->bits[nr_words - 1] &= (1ULL << tail_bits) - 1; } -/** - * scx_cpumask_to_cmask - Translate a kernel cpumask into a cmask - * @src: source cpumask - * @dst: cmask to write - * - * Clear @dst's active range and set the bit for each cid whose cpu is in - * @src and lies within that range. Out-of-range cids are silently ignored. - */ -void scx_cpumask_to_cmask(const struct cpumask *src, struct scx_cmask *dst) -{ - s32 cpu; - - scx_cmask_clear(dst); - for_each_cpu(cpu, src) { - s32 cid = __scx_cpu_to_cid(cpu); - - if (cid >= 0) - __scx_cmask_set(cid, dst); - } -} - /* * Return the index of the largest entry in @counts, or NUMA_NO_NODE if all * entries are zero. Ties resolve to the lowest index. diff --git a/kernel/sched/ext/cid.h b/kernel/sched/ext/cid.h index b36a1a28eac8..470ac9224da4 100644 --- a/kernel/sched/ext/cid.h +++ b/kernel/sched/ext/cid.h @@ -68,7 +68,6 @@ bool scx_cmask_intersects(const struct scx_cmask *a, const struct scx_cmask *b); bool scx_cmask_empty(const struct scx_cmask *m); s32 scx_cid_init(struct scx_sched *sch); int scx_cid_kfunc_init(void); -void scx_cpumask_to_cmask(const struct cpumask *src, struct scx_cmask *dst); /** * cid_valid - Verify a cid value, to be used on ops input args -- cgit v1.2.3 From 3a773220d39ba993dfe5d135f8b610be10794d5d Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Tue, 21 Jul 2026 22:22:18 -1000 Subject: sched_ext: Build the cid tables privately and publish them with RCU The cid tables are visible to the cid kfuncs while being modified: the first enable publishes the global pointers before filling them, ops.init_cids() overrides rewrite them in place, and re-enables rebuild them in place. A racing TRACING or SYSCALL program can read unfilled entries, including uninitialized memory in the kmalloc'd tables, or torn topo updates. Tie the tables' lifetimes to the root sched instead: each root enable builds a fresh set privately and publishes the per-table __rcu globals once the layout is final, and root disable unpublishes and RCU-frees the set. A non-NULL global is now always a fully built table which stays valid for the reader's RCU read section, and lookups stay two loads. Kfuncs treat NULL as no-mapping, also after the scheduler exits instead of reporting the stale last mapping. The cid kfuncs are available whether the root scheduler is cid-form or cpu-form, the latter to allow gradual migration to cids. Every root therefore builds and publishes a default mapping. Every reader must either be gated on scheduler liveness or NULL-check inside an RCU read section. Fix the two kfuncs that were neither: scx_bpf_this_cid() read the table with no RCU or preemption protection and scx_bpf_task_cid() relied on KF_RCU, which doesn't put a sleepable program in an RCU read section. The hotplug callbacks are instead serialized by retiring the tables inside the cpus_read_lock() section that clears scx_root. v2: Document why every root builds the tables (desc + cid.c comment). Reported-by: Andrea Righi Closes: https://lore.kernel.org/r/al3tLtPZZkFjMveK@gpd4 Reviewed-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/cid.c | 260 ++++++++++++++++++++++++++++---------------- kernel/sched/ext/cid.h | 57 ++++++---- kernel/sched/ext/ext.c | 54 ++++++--- kernel/sched/ext/internal.h | 5 +- kernel/sched/ext/sub.c | 42 +++++-- 5 files changed, 276 insertions(+), 142 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/cid.c b/kernel/sched/ext/cid.c index 5f990d95735f..231c8562d7c7 100644 --- a/kernel/sched/ext/cid.c +++ b/kernel/sched/ext/cid.c @@ -11,20 +11,22 @@ #include "cid.h" /* - * cid tables. - * - * Pointers are allocated on first enable and never freed. During root enable, - * the default mapping is populated and then ops.init_cids() is called which can - * use scx_bpf_cid_override() to change the mapping. The mapping stays stable - * until the root is disabled. + * cid tables. The cid kfuncs are available whether the root scheduler is + * cid-form or cpu-form, the latter to allow gradual migration to cids, so every + * root builds a default mapping. Each root enable allocates a fresh set, builds + * it privately and publishes the __rcu globals below once the layout is final. + * Root disable unpublishes and RCU-frees the set. kfuncs may run before the + * tables are published and must check for NULL. */ u32 scx_nr_cid_shards; -s16 *scx_cid_to_cpu_tbl; -s16 *scx_cpu_to_cid_tbl; -s32 *scx_cid_to_shard; -s32 *scx_shard_node; -struct scx_cid_shard *scx_cid_shard_ranges; -struct scx_cid_topo *scx_cid_topo; +s16 __rcu *scx_cid_to_cpu_tbl; +s16 __rcu *scx_cpu_to_cid_tbl; +s32 __rcu *scx_cid_to_shard; +s32 __rcu *scx_shard_node; +struct scx_cid_shard __rcu *scx_cid_shard_ranges; +struct scx_cid_topo __rcu *scx_cid_topo; + +static struct scx_cid_tables *scx_cid_tables; /* used only during alloc/free */ #define SCX_CID_TOPO_NEG (struct scx_cid_topo) { \ .core_cid = -1, .core_idx = -1, .llc_cid = -1, .llc_idx = -1, \ @@ -73,50 +75,102 @@ static void calc_shard_layout(const struct cpumask *llc_cpus, u32 shard_size, *nr_large_p = nr_cores % nr_shards; } -/* Allocate the cid tables once on first enable; never freed. */ -static s32 scx_cid_arrays_alloc(void) +static void scx_cid_tables_free(struct scx_cid_tables *tbls) +{ + if (!tbls) + return; + kvfree(tbls->cid_to_cpu); + kvfree(tbls->cpu_to_cid); + kvfree(tbls->cid_to_shard); + kvfree(tbls->shard_node); + kvfree(tbls->shard_ranges); + kvfree(tbls->topo); + kfree(tbls); +} + +static void scx_cid_tables_free_rcufn(struct rcu_head *rcu) +{ + scx_cid_tables_free(container_of(rcu, struct scx_cid_tables, rcu)); +} + +static struct scx_cid_tables *scx_cid_alloc_tables(void) { u32 npossible = num_possible_cpus(); - s16 *cid_to_cpu, *cpu_to_cid; - s32 *cid_to_shard; - s32 *shard_node; - struct scx_cid_shard *cid_shard_ranges; - struct scx_cid_topo *cid_topo; - - if (scx_cid_to_cpu_tbl) - return 0; - - cid_to_cpu = kzalloc_objs(*scx_cid_to_cpu_tbl, npossible, GFP_KERNEL); - cpu_to_cid = kzalloc_objs(*scx_cpu_to_cid_tbl, nr_cpu_ids, GFP_KERNEL); - cid_to_shard = kzalloc_objs(*scx_cid_to_shard, npossible, GFP_KERNEL); - shard_node = kmalloc_objs(*scx_shard_node, npossible, GFP_KERNEL); - cid_shard_ranges = kzalloc_objs(*scx_cid_shard_ranges, npossible, GFP_KERNEL); - cid_topo = kmalloc_objs(*scx_cid_topo, npossible, GFP_KERNEL); - - if (!cid_to_cpu || !cpu_to_cid || !cid_to_shard || !shard_node || - !cid_shard_ranges || !cid_topo) { - kfree(cid_to_cpu); - kfree(cpu_to_cid); - kfree(cid_to_shard); - kfree(shard_node); - kfree(cid_shard_ranges); - kfree(cid_topo); - return -ENOMEM; + struct scx_cid_tables *tbls; + + tbls = kzalloc_obj(*tbls, GFP_KERNEL); + if (!tbls) + return NULL; + + tbls->cid_to_cpu = kvcalloc(npossible, sizeof(*tbls->cid_to_cpu), GFP_KERNEL); + tbls->cpu_to_cid = kvcalloc(nr_cpu_ids, sizeof(*tbls->cpu_to_cid), GFP_KERNEL); + tbls->cid_to_shard = kvcalloc(npossible, sizeof(*tbls->cid_to_shard), GFP_KERNEL); + tbls->shard_node = kvcalloc(npossible, sizeof(*tbls->shard_node), GFP_KERNEL); + tbls->shard_ranges = kvcalloc(npossible, sizeof(*tbls->shard_ranges), GFP_KERNEL); + tbls->topo = kvcalloc(npossible, sizeof(*tbls->topo), GFP_KERNEL); + + if (!tbls->cid_to_cpu || !tbls->cpu_to_cid || !tbls->cid_to_shard || + !tbls->shard_node || !tbls->shard_ranges || !tbls->topo) { + scx_cid_tables_free(tbls); + return NULL; } - WRITE_ONCE(scx_cid_to_cpu_tbl, cid_to_cpu); - WRITE_ONCE(scx_cpu_to_cid_tbl, cpu_to_cid); - WRITE_ONCE(scx_cid_to_shard, cid_to_shard); - WRITE_ONCE(scx_shard_node, shard_node); - WRITE_ONCE(scx_cid_shard_ranges, cid_shard_ranges); - WRITE_ONCE(scx_cid_topo, cid_topo); - return 0; + return tbls; +} + +/** + * scx_cid_publish_tables - Publish the tables scx_cid_init() built + * + * Called after ops.init_cids() where the layout is final. + */ +void scx_cid_publish_tables(void) +{ + struct scx_cid_tables *tbls = scx_cid_tables; + + lockdep_assert_held(&scx_enable_mutex); + + scx_nr_cid_shards = tbls->nr_shards; + rcu_assign_pointer(scx_cid_to_cpu_tbl, tbls->cid_to_cpu); + rcu_assign_pointer(scx_cpu_to_cid_tbl, tbls->cpu_to_cid); + rcu_assign_pointer(scx_cid_to_shard, tbls->cid_to_shard); + rcu_assign_pointer(scx_shard_node, tbls->shard_node); + rcu_assign_pointer(scx_cid_shard_ranges, tbls->shard_ranges); + rcu_assign_pointer(scx_cid_topo, tbls->topo); +} + +/** + * scx_cid_retire_tables - Unpublish and retire the cid tables + * + * Called by root disable after the readers which dereference without NULL + * checks are drained, inside cpus_read_lock() to exclude the hotplug path. + */ +void scx_cid_retire_tables(void) +{ + struct scx_cid_tables *tbls = scx_cid_tables; + + lockdep_assert_held(&scx_enable_mutex); + lockdep_assert_cpus_held(); + + if (!tbls) + return; + + scx_cid_tables = NULL; + RCU_INIT_POINTER(scx_cid_to_cpu_tbl, NULL); + RCU_INIT_POINTER(scx_cpu_to_cid_tbl, NULL); + RCU_INIT_POINTER(scx_cid_to_shard, NULL); + RCU_INIT_POINTER(scx_shard_node, NULL); + RCU_INIT_POINTER(scx_cid_shard_ranges, NULL); + RCU_INIT_POINTER(scx_cid_topo, NULL); + call_rcu(&tbls->rcu, scx_cid_tables_free_rcufn); } /** * scx_cid_init - build the cid mapping * @sch: the scx_sched being initialized; used as the scx_error() target * + * Build a fresh table set. It becomes visible through scx_cid_publish_tables() + * and is retired by scx_cid_retire_tables() at disable. + * * See "Topological CPU IDs" in cid.h for the model. Walk online cpus by * intersection at each level (parent_scratch & this_level_mask), which keeps * containment correct by construction and naturally splits a physical LLC @@ -131,30 +185,32 @@ s32 scx_cid_init(struct scx_sched *sch) cpumask_var_t core_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL; cpumask_var_t llc_fallback __free(free_cpumask_var) = CPUMASK_VAR_NULL; cpumask_var_t online_no_topo __free(free_cpumask_var) = CPUMASK_VAR_NULL; + struct scx_cid_tables *tbls; u32 next_cid = 0; s32 next_node_idx = 0, next_llc_idx = 0, next_core_idx = 0; s32 next_shard_idx = 0; u32 shard_size, max_cids; u32 notopo_in_shard; s32 notopo_shard_cid, notopo_shard_idx; - s32 cpu, cid, si, ret; + s32 cpu, cid, si; /* CMASK_MAX_WORDS in cid.bpf.h covers NR_CPUS up to 8192 */ BUILD_BUG_ON(NR_CPUS > 8192); lockdep_assert_cpus_held(); + lockdep_assert_held(&scx_enable_mutex); shard_size = sch->ops.cid_shard_size ?: SCX_CID_SHARD_SIZE_DFL; max_cids = min_t(u32, shard_size, SCX_CID_SHARD_MAX_CPUS); - ret = scx_cid_arrays_alloc(); - if (ret) - return ret; + tbls = scx_cid_alloc_tables(); + if (!tbls) + return -ENOMEM; + + scx_cid_tables = tbls; - /* clear shard ranges and reset shard_node for repopulate */ - memset(scx_cid_shard_ranges, 0, num_possible_cpus() * sizeof(*scx_cid_shard_ranges)); for (si = 0; si < num_possible_cpus(); si++) - scx_shard_node[si] = NUMA_NO_NODE; + tbls->shard_node[si] = NUMA_NO_NODE; if (!zalloc_cpumask_var(&to_walk, GFP_KERNEL) || !zalloc_cpumask_var(&node_scratch, GFP_KERNEL) || @@ -166,7 +222,7 @@ s32 scx_cid_init(struct scx_sched *sch) /* -1 sentinels for sparse-possible cpu id holes (0 is a valid cid) */ for (cpu = 0; cpu < nr_cpu_ids; cpu++) - scx_cpu_to_cid_tbl[cpu] = -1; + tbls->cpu_to_cid[cpu] = -1; cpumask_copy(to_walk, cpu_online_mask); @@ -209,7 +265,7 @@ s32 scx_cid_init(struct scx_sched *sch) calc_shard_layout(llc_scratch, shard_size, &cores_per_shard, &nr_large); shard_cid = next_cid; shard_idx = next_shard_idx++; - scx_shard_node[shard_idx] = nid; + tbls->shard_node[shard_idx] = nid; while (!cpumask_empty(llc_scratch)) { s32 lcpu = cpumask_first(llc_scratch); @@ -240,7 +296,7 @@ s32 scx_cid_init(struct scx_sched *sch) cids_in_shard = 0; shard_cid = next_cid; shard_idx = next_shard_idx++; - scx_shard_node[shard_idx] = nid; + tbls->shard_node[shard_idx] = nid; } cores_in_shard++; cids_in_shard += cids_in_core; @@ -248,10 +304,10 @@ s32 scx_cid_init(struct scx_sched *sch) for_each_cpu(ccpu, core_scratch) { s32 cid = next_cid++; - scx_cid_to_cpu_tbl[cid] = ccpu; - scx_cpu_to_cid_tbl[ccpu] = cid; - scx_cid_to_shard[cid] = shard_idx; - scx_cid_topo[cid] = (struct scx_cid_topo){ + tbls->cid_to_cpu[cid] = ccpu; + tbls->cpu_to_cid[ccpu] = cid; + tbls->cid_to_shard[cid] = shard_idx; + tbls->topo[cid] = (struct scx_cid_topo){ .core_cid = core_cid, .core_idx = core_idx, .llc_cid = llc_cid, @@ -283,14 +339,14 @@ s32 scx_cid_init(struct scx_sched *sch) notopo_shard_idx = -1; for_each_cpu(cpu, cpu_possible_mask) { - if (__scx_cpu_to_cid(cpu) != -1) + if (tbls->cpu_to_cid[cpu] != -1) continue; if (cpu_online(cpu)) cpumask_set_cpu(cpu, online_no_topo); cid = next_cid++; - scx_cid_to_cpu_tbl[cid] = cpu; - scx_cpu_to_cid_tbl[cpu] = cid; + tbls->cid_to_cpu[cid] = cpu; + tbls->cpu_to_cid[cpu] = cid; if (notopo_in_shard >= min_t(u32, shard_size, SCX_CID_SHARD_MAX_CPUS)) { notopo_shard_cid = cid; @@ -299,10 +355,10 @@ s32 scx_cid_init(struct scx_sched *sch) } notopo_in_shard++; - scx_cid_to_shard[cid] = notopo_shard_idx; - scx_cid_topo[cid] = SCX_CID_TOPO_NEG; - scx_cid_topo[cid].shard_cid = notopo_shard_cid; - scx_cid_topo[cid].shard_idx = notopo_shard_idx; + tbls->cid_to_shard[cid] = notopo_shard_idx; + tbls->topo[cid] = SCX_CID_TOPO_NEG; + tbls->topo[cid].shard_cid = notopo_shard_cid; + tbls->topo[cid].shard_idx = notopo_shard_idx; } if (!cpumask_empty(llc_fallback)) @@ -318,14 +374,14 @@ s32 scx_cid_init(struct scx_sched *sch) * shard, nr_cids is the count. */ for (cid = 0; cid < next_cid; cid++) { - s32 sidx = scx_cid_to_shard[cid]; + s32 sidx = tbls->cid_to_shard[cid]; - if (scx_cid_shard_ranges[sidx].nr_cids == 0) - scx_cid_shard_ranges[sidx].base_cid = cid; - scx_cid_shard_ranges[sidx].nr_cids++; + if (tbls->shard_ranges[sidx].nr_cids == 0) + tbls->shard_ranges[sidx].base_cid = cid; + tbls->shard_ranges[sidx].nr_cids++; } - scx_nr_cid_shards = next_shard_idx; + tbls->nr_shards = next_shard_idx; return 0; } @@ -417,6 +473,7 @@ __bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid_src, u32 cpu_to_cid_ s32 *cpu_to_cid __free(kfree) = NULL; s32 *shard_start __free(kfree) = NULL; u32 npossible = num_possible_cpus(); + struct scx_cid_tables *tbls; struct scx_sched *sch; u32 nr_shards; bool alloced; @@ -438,6 +495,10 @@ __bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid_src, u32 cpu_to_cid_ if (unlikely(!sch)) return; + /* called from ops.init_cids(), so the tables exist and are unpublished */ + lockdep_assert_held(&scx_enable_mutex); + tbls = scx_cid_tables; + if (!alloced || !node_counts || !cpu_to_cid || !shard_start) { scx_error(sch, "scx_bpf_cid_override: allocation failed"); return; @@ -488,7 +549,7 @@ __bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid_src, u32 cpu_to_cid_ return; } - /* Validate first so that invalid input leaves globals untouched. */ + /* validate first so that invalid input leaves the tables untouched */ for_each_possible_cpu(cpu) { s32 c = cpu_to_cid[cpu]; @@ -503,12 +564,12 @@ __bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid_src, u32 cpu_to_cid_ for_each_possible_cpu(cpu) { s32 c = cpu_to_cid[cpu]; - scx_cpu_to_cid_tbl[cpu] = c; - scx_cid_to_cpu_tbl[c] = cpu; + tbls->cpu_to_cid[cpu] = c; + tbls->cid_to_cpu[c] = cpu; } /* - * Derive scx_shard_node[] by majority count: an overridden shard may + * Derive shard_node[] by majority count: an overridden shard may * span NUMA nodes, so assign each to the node that owns the most cpus. */ for (si = 0; si < nr_shards; si++) { @@ -516,12 +577,12 @@ __bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid_src, u32 cpu_to_cid_ memset(node_counts, 0, nr_node_ids * sizeof(*node_counts)); for (cid = shard_start[si]; cid < end; cid++) { - s32 node = cpu_to_node(scx_cid_to_cpu_tbl[cid]); + s32 node = cpu_to_node(tbls->cid_to_cpu[cid]); if (numa_valid_node(node)) node_counts[node]++; } - scx_shard_node[si] = pick_max_node(node_counts, nr_node_ids); + tbls->shard_node[si] = pick_max_node(node_counts, nr_node_ids); } /* @@ -532,22 +593,22 @@ __bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid_src, u32 cpu_to_cid_ for (cid = 0; cid < npossible; cid++) { if (si + 1 < nr_shards && cid >= shard_start[si + 1]) si++; - scx_cid_to_shard[cid] = si; - scx_cid_topo[cid] = SCX_CID_TOPO_NEG; - scx_cid_topo[cid].shard_cid = shard_start[si]; - scx_cid_topo[cid].shard_idx = si; + tbls->cid_to_shard[cid] = si; + tbls->topo[cid] = SCX_CID_TOPO_NEG; + tbls->topo[cid].shard_cid = shard_start[si]; + tbls->topo[cid].shard_idx = si; } - /* Rebuild scx_cid_shard_ranges[] for the new layout. */ - memset(scx_cid_shard_ranges, 0, npossible * sizeof(*scx_cid_shard_ranges)); + /* Rebuild shard_ranges[] for the new layout. */ + memset(tbls->shard_ranges, 0, npossible * sizeof(*tbls->shard_ranges)); for (si = 0; si < nr_shards; si++) { u32 end = (si + 1 < nr_shards) ? shard_start[si + 1] : npossible; - scx_cid_shard_ranges[si].base_cid = shard_start[si]; - scx_cid_shard_ranges[si].nr_cids = end - shard_start[si]; + tbls->shard_ranges[si].base_cid = shard_start[si]; + tbls->shard_ranges[si].nr_cids = end - shard_start[si]; } - scx_nr_cid_shards = nr_shards; + tbls->nr_shards = nr_shards; } /** @@ -849,22 +910,25 @@ bool scx_cmask_empty(const struct scx_cmask *m) * * Fill @out__uninit with the topology info for @cid. Trigger scx_error() if * @cid is out of range. If @cid is valid but in the no-topo section, all fields - * are set to -1. + * are set to -1. All fields are also set to -1 when no cid tables have been + * published yet, which a program may observe while racing the root enable. */ __bpf_kfunc void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out__uninit, const struct bpf_prog_aux *aux) { + struct scx_cid_topo *topo; struct scx_sched *sch; guard(rcu)(); sch = scx_prog_sched(aux); - if (unlikely(!sch) || !cid_valid(sch, cid)) { + topo = rcu_dereference(scx_cid_topo); + if (unlikely(!sch) || !cid_valid(sch, cid) || unlikely(!topo)) { *out__uninit = SCX_CID_TOPO_NEG; return; } - *out__uninit = READ_ONCE(scx_cid_topo)[cid]; + *out__uninit = topo[cid]; } __bpf_kfunc_end_defs(); @@ -907,6 +971,7 @@ int scx_cmask_ref_init(struct scx_sched *sch, const struct scx_cmask *src, { struct scx_cmask *kern_src = scx_arena_to_kaddr(sch, src); u32 base, nr_cids, alloc_words, npossible = num_possible_cpus(); + s32 *cid_to_shard; base = READ_ONCE(kern_src->base); nr_cids = READ_ONCE(kern_src->nr_cids); @@ -921,9 +986,10 @@ int scx_cmask_ref_init(struct scx_sched *sch, const struct scx_cmask *src, ref->base = base; ref->nr_cids = nr_cids; - ref->shard_first = scx_cid_to_shard[base]; + cid_to_shard = rcu_dereference_all(scx_cid_to_shard); + ref->shard_first = cid_to_shard[base]; if (likely(nr_cids)) - ref->shard_end = scx_cid_to_shard[base + nr_cids - 1] + 1; + ref->shard_end = cid_to_shard[base + nr_cids - 1] + 1; else ref->shard_end = ref->shard_first; @@ -946,6 +1012,8 @@ int scx_cmask_ref_init(struct scx_sched *sch, const struct scx_cmask *src, void scx_cmask_ref_init_kern(struct scx_sched *sch, struct scx_cmask *m, u32 base, u32 nr_cids, struct scx_cmask_ref *ref) { + s32 *cid_to_shard; + WRITE_ONCE(m->base, base); WRITE_ONCE(m->nr_cids, nr_cids); WRITE_ONCE(m->alloc_words, SCX_CMASK_NR_WORDS(nr_cids)); @@ -955,9 +1023,10 @@ void scx_cmask_ref_init_kern(struct scx_sched *sch, struct scx_cmask *m, ref->base = base; ref->nr_cids = nr_cids; - ref->shard_first = scx_cid_to_shard[base]; + cid_to_shard = rcu_dereference_all(scx_cid_to_shard); + ref->shard_first = cid_to_shard[base]; if (likely(nr_cids)) - ref->shard_end = scx_cid_to_shard[base + nr_cids - 1] + 1; + ref->shard_end = cid_to_shard[base + nr_cids - 1] + 1; else ref->shard_end = ref->shard_first; } @@ -976,7 +1045,8 @@ void scx_cmask_ref_init_kern(struct scx_sched *sch, struct scx_cmask *m, void scx_cmask_ref_shard(const struct scx_cmask_ref *ref, s32 shard_idx, struct scx_cmask *out) { - const struct scx_cid_shard *shard = &scx_cid_shard_ranges[shard_idx]; + const struct scx_cid_shard *shard = + &rcu_dereference_all(scx_cid_shard_ranges)[shard_idx]; u32 shard_base = shard->base_cid; u32 shard_end = shard_base + shard->nr_cids; u32 isect_base, isect_end, nr_words, src_off, wi; diff --git a/kernel/sched/ext/cid.h b/kernel/sched/ext/cid.h index 470ac9224da4..1f74d1f331f5 100644 --- a/kernel/sched/ext/cid.h +++ b/kernel/sched/ext/cid.h @@ -48,13 +48,24 @@ struct scx_sched; * See the comment above the table definitions in cid.c for the * memory-ordering and visibility contract. */ +struct scx_cid_tables { + u32 nr_shards; + s16 *cid_to_cpu; /* [num_possible_cpus()] */ + s16 *cpu_to_cid; /* [nr_cpu_ids] */ + s32 *cid_to_shard; /* [num_possible_cpus()] */ + s32 *shard_node; /* [num_possible_cpus()] */ + struct scx_cid_shard *shard_ranges; /* [num_possible_cpus()] */ + struct scx_cid_topo *topo; /* [num_possible_cpus()] */ + struct rcu_head rcu; +}; + extern u32 scx_nr_cid_shards; -extern s16 *scx_cid_to_cpu_tbl; -extern s16 *scx_cpu_to_cid_tbl; -extern s32 *scx_cid_to_shard; -extern s32 *scx_shard_node; -extern struct scx_cid_shard *scx_cid_shard_ranges; -extern struct scx_cid_topo *scx_cid_topo; +extern s16 __rcu *scx_cid_to_cpu_tbl; +extern s16 __rcu *scx_cpu_to_cid_tbl; +extern s32 __rcu *scx_cid_to_shard; +extern s32 __rcu *scx_shard_node; +extern struct scx_cid_shard __rcu *scx_cid_shard_ranges; +extern struct scx_cid_topo __rcu *scx_cid_topo; extern struct btf_id_set8 scx_kfunc_ids_init_cids; void scx_cmask_clear(struct scx_cmask *m); @@ -67,6 +78,8 @@ bool scx_cmask_subset(const struct scx_cmask *sub, const struct scx_cmask *super bool scx_cmask_intersects(const struct scx_cmask *a, const struct scx_cmask *b); bool scx_cmask_empty(const struct scx_cmask *m); s32 scx_cid_init(struct scx_sched *sch); +void scx_cid_publish_tables(void); +void scx_cid_retire_tables(void); int scx_cid_kfunc_init(void); /** @@ -89,14 +102,12 @@ static inline bool cid_valid(struct scx_sched *sch, s32 cid) * __scx_cid_to_cpu - Unchecked cid->cpu table lookup * @cid: cid to look up. Must be in [0, num_possible_cpus()). * - * Intended for callsites that have already validated @cid and that hold a - * non-NULL @sch from scx_prog_sched() - a live sched implies the table has - * been allocated, so no NULL check is needed here. + * Intended for callsites that have already validated @cid and that run on a + * live scheduler, which guarantees the tables are published and stable. */ static inline s32 __scx_cid_to_cpu(s32 cid) { - /* READ_ONCE pairs with WRITE_ONCE in scx_cid_arrays_alloc() */ - return READ_ONCE(scx_cid_to_cpu_tbl)[cid]; + return rcu_dereference_all(scx_cid_to_cpu_tbl)[cid]; } /** @@ -107,7 +118,7 @@ static inline s32 __scx_cid_to_cpu(s32 cid) */ static inline s32 __scx_cpu_to_cid(s32 cpu) { - return READ_ONCE(scx_cpu_to_cid_tbl)[cpu]; + return rcu_dereference_all(scx_cpu_to_cid_tbl)[cpu]; } /** @@ -116,15 +127,19 @@ static inline s32 __scx_cpu_to_cid(s32 cpu) * @cid: cid to look up * * Return the cpu for @cid or a negative errno on failure. Invalid cid triggers - * scx_error() on @sch. The cid arrays are allocated on first scheduler enable - * and never freed, so the returned cpu is stable for the lifetime of the loaded - * scheduler. + * scx_error() on @sch. The mapping is stable while the scheduler is live. + * + * Return -EINVAL without triggering scx_error() if no tables have been + * published yet, which a prog-facing kfunc can observe while racing the root + * scheduler enable. */ static inline s32 scx_cid_to_cpu(struct scx_sched *sch, s32 cid) { - if (!cid_valid(sch, cid)) + s16 *tbl = rcu_dereference_all(scx_cid_to_cpu_tbl); + + if (!cid_valid(sch, cid) || unlikely(!tbl)) return -EINVAL; - return __scx_cid_to_cpu(cid); + return tbl[cid]; } /** @@ -133,13 +148,15 @@ static inline s32 scx_cid_to_cpu(struct scx_sched *sch, s32 cid) * @cpu: cpu to look up * * Return the cid for @cpu or a negative errno on failure. Invalid cpu triggers - * scx_error() on @sch. Same lifetime guarantee as scx_cid_to_cpu(). + * scx_error() on @sch. Same usage rules as scx_cid_to_cpu(). */ static inline s32 scx_cpu_to_cid(struct scx_sched *sch, s32 cpu) { - if (!scx_cpu_valid(sch, cpu, NULL)) + s16 *tbl = rcu_dereference_all(scx_cpu_to_cid_tbl); + + if (!scx_cpu_valid(sch, cpu, NULL) || unlikely(!tbl)) return -EINVAL; - return __scx_cpu_to_cid(cpu); + return tbl[cpu]; } /** diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index e6c4cbe1f182..6a4e3a4f6098 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -3334,6 +3334,7 @@ static void handle_hotplug(struct rq *rq, bool online) { struct scx_sched *sch = scx_root; s32 cpu = cpu_of(rq); + s32 cpu_or_cid = cpu; atomic_long_inc(&scx_hotplug_seq); @@ -3353,10 +3354,26 @@ static void handle_hotplug(struct rq *rq, bool online) else scx_offline_ecaps(rq); + /* + * The tables can't be retired while this function is running as the + * retirement is inside cpus_read_lock. However, scx_cpu_arg() is + * awkward here as the tables can be NULL after root enable failure and + * lockdep would trigger without surrounding rcu_read_lock(). Open code + * the translation. If the table is NULL, the ops are also cleared and + * @cpu_or_cid goes unused. + */ + if (scx_is_cid_type()) { + s16 *tbl = rcu_dereference_check(scx_cpu_to_cid_tbl, + lockdep_is_cpus_held()); + + if (tbl) + cpu_or_cid = tbl[cpu]; + } + if (online && SCX_HAS_OP(sch, cpu_online)) - SCX_CALL_OP(sch, cpu_online, NULL, scx_cpu_arg(cpu)); + SCX_CALL_OP(sch, cpu_online, NULL, cpu_or_cid); else if (!online && SCX_HAS_OP(sch, cpu_offline)) - SCX_CALL_OP(sch, cpu_offline, NULL, scx_cpu_arg(cpu)); + SCX_CALL_OP(sch, cpu_offline, NULL, cpu_or_cid); else scx_exit(sch, SCX_EXIT_UNREG_KERN, SCX_ECODE_ACT_RESTART | SCX_ECODE_RSN_HOTPLUG, @@ -6197,11 +6214,12 @@ static void scx_root_disable(struct scx_sched *sch) scx_unlink_sched(sch); /* - * scx_root clearing must be inside cpus_read_lock(). See - * handle_hotplug(). + * scx_root clearing and cid table retirement must be inside + * cpus_read_lock(). See handle_hotplug(). */ cpus_read_lock(); RCU_INIT_POINTER(scx_root, NULL); + scx_cid_retire_tables(); cpus_read_unlock(); /* @@ -7195,10 +7213,9 @@ static void scx_root_enable_workfn(struct kthread_work *work) cpus_read_lock(); /* - * Build the cid mapping before publishing scx_root. The cid kfuncs - * dereference the cid arrays unconditionally once scx_prog_sched() - * returns non-NULL; the rcu_assign_pointer() below pairs with their - * rcu_dereference() to make the populated arrays visible. + * Build the cid mapping into a private under-construction set. It + * becomes visible to readers only through scx_cid_publish_tables() once + * ops.init_cids() has finalized the layout. */ ret = scx_cid_init(sch); if (ret) { @@ -7235,6 +7252,9 @@ static void scx_root_enable_workfn(struct kthread_work *work) } } + /* the cid layout is final, expose it to readers */ + scx_cid_publish_tables(); + ret = scx_arena_pool_init(sch); if (ret) { cpus_read_unlock(); @@ -9872,13 +9892,15 @@ __bpf_kfunc u32 scx_bpf_nr_online_cids(void) * * cid-addressed equivalent of bpf_get_smp_processor_id() for scx programs. * The current cpu is trivially valid, so this is just a table lookup. Return - * -EINVAL if called from a non-SCX program before any scheduler has ever - * been enabled (the cid table is still unallocated at that point). + * -EINVAL if called before any scheduler has ever published its cid tables. */ __bpf_kfunc s32 scx_bpf_this_cid(void) { - s16 *tbl = READ_ONCE(scx_cpu_to_cid_tbl); + s16 *tbl; + + guard(rcu)(); + tbl = rcu_dereference(scx_cpu_to_cid_tbl); if (!tbl) return -EINVAL; return tbl[raw_smp_processor_id()]; @@ -9937,13 +9959,17 @@ __bpf_kfunc s32 scx_bpf_task_cpu(const struct task_struct *p) * @p: task of interest * * cid-addressed equivalent of scx_bpf_task_cpu(). task_cpu(p) is always a - * valid cpu, so this is just a table lookup. Return -EINVAL if called from - * a non-SCX program before any scheduler has ever been enabled. + * valid cpu, so this is just a table lookup. Return -EINVAL if called before + * any scheduler has ever published its cid tables. */ __bpf_kfunc s32 scx_bpf_task_cid(const struct task_struct *p) { - s16 *tbl = READ_ONCE(scx_cpu_to_cid_tbl); + s16 *tbl; + + /* KF_RCU covers only @p - a sleepable program holds no RCU lock */ + guard(rcu)(); + tbl = rcu_dereference(scx_cpu_to_cid_tbl); if (!tbl) return -EINVAL; return tbl[task_cpu(p)]; diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 26bfda216524..a9a853a71061 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1504,8 +1504,9 @@ struct scx_sched { #ifdef CONFIG_EXT_SUB_SCHED /* * pshard[] size captured at enable for the async RCU free path - - * scx_nr_cid_shards may be rewritten by a later scx_cid_init() before - * free runs. While sch is active, use the global. + * scx_nr_cid_shards may be rewritten by a later enable's + * scx_cid_publish_tables() before free runs. While sch is active, use + * the global. */ u32 nr_pshards; #endif diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 8d8737149bc0..d7842a609d96 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -138,7 +138,9 @@ void scx_free_pshards(struct scx_sched *sch) static struct scx_pshard *alloc_pshard(struct scx_sched *sch, s32 shard_idx, s32 node) { - const struct scx_cid_shard *shard = &scx_cid_shard_ranges[shard_idx]; + const struct scx_cid_shard *shard = + &rcu_dereference_protected(scx_cid_shard_ranges, + lockdep_is_held(&scx_enable_mutex))[shard_idx]; size_t cmask_size = struct_size_t(struct scx_cmask, bits, SCX_CMASK_NR_WORDS(shard->nr_cids)); struct scx_pshard *pshard; @@ -176,17 +178,21 @@ static struct scx_pshard *alloc_pshard(struct scx_sched *sch, s32 shard_idx, s32 s32 scx_alloc_pshards(struct scx_sched *sch) { struct scx_pshard **pshard; + s32 *shard_node; s32 si; if (!sch->is_cid_type || !sch->arena_pool) return 0; + shard_node = rcu_dereference_protected(scx_shard_node, + lockdep_is_held(&scx_enable_mutex)); + pshard = kzalloc_objs(pshard[0], scx_nr_cid_shards, GFP_KERNEL); if (!pshard) return -ENOMEM; for (si = 0; si < scx_nr_cid_shards; si++) { - pshard[si] = alloc_pshard(sch, si, scx_shard_node[si]); + pshard[si] = alloc_pshard(sch, si, shard_node[si]); if (!pshard[si]) { while (--si >= 0) free_pshard(pshard[si]); @@ -198,8 +204,9 @@ s32 scx_alloc_pshards(struct scx_sched *sch) sch->nr_pshards = scx_nr_cid_shards; /* * Publish only after every entry is built so a reader observing - * @sch->pshard never sees a partially-filled array. Pair the store - * with a barrier and READ_ONCE() on the read side. + * @sch->pshard never sees a partially-filled array or unpublished cid + * tables. Pair the store with a barrier and an acquire load on the + * read side. */ smp_wmb(); WRITE_ONCE(sch->pshard, pshard); @@ -524,7 +531,7 @@ void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) /* @cid is valid here: the cpu is active with queued syncs */ cid = __scx_cpu_to_cid(cpu); - shard = scx_cid_to_shard[cid]; + shard = rcu_dereference_all(scx_cid_to_shard)[cid]; batch = llist_del_all(&rq->scx.ecaps_to_sync); llist_for_each_safe(pos, tmp, batch) { @@ -618,7 +625,7 @@ void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch) return; cid = __scx_cpu_to_cid(cpu); - ps = sch->pshard[scx_cid_to_shard[cid]]; + ps = sch->pshard[rcu_dereference_all(scx_cid_to_shard)[cid]]; guard(raw_spinlock)(&ps->lock); queue_sync_ecaps(sch, cid); @@ -631,12 +638,23 @@ void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch) */ void scx_online_ecaps(struct rq *rq) { - s32 cid = __scx_cpu_to_cid(cpu_of(rq)); - s32 shard = scx_cid_to_shard[cid]; struct scx_sched *pos; + s32 cid, shard; + + /* + * Only a live hierarchy can have ecaps to reseed. This also keeps the + * table reads below away from an enable that failed before publishing + * the tables. A concurrent disable can't retire them, see + * handle_hotplug(). + */ + if (!scx_enabled()) + return; guard(rq_lock_irqsave)(rq); + cid = __scx_cpu_to_cid(cpu_of(rq)); + shard = rcu_dereference_all(scx_cid_to_shard)[cid]; + scx_for_each_descendant_pre(pos, scx_root) { struct scx_pshard *ps; @@ -2074,9 +2092,10 @@ __bpf_kfunc s32 scx_bpf_sub_caps(u64 cgroup_id, u64 caps, struct scx_cmask *out_ /* * The target's caps storage may not be set up yet (e.g. a self-read * during ops.init_cids()). Pairs with the publish in - * scx_alloc_pshards(): a non-NULL pshard has every element set. + * scx_alloc_pshards(): a non-NULL pshard has every element set and the + * acquire also orders the cid table reads below against it. */ - pshard = READ_ONCE(target->pshard); + pshard = smp_load_acquire(&target->pshard); if (unlikely(!pshard)) { scx_error(sch, "scx_bpf_sub_caps() called before caps storage is initialized"); return -ENODEV; @@ -2089,7 +2108,8 @@ __bpf_kfunc s32 scx_bpf_sub_caps(u64 cgroup_id, u64 caps, struct scx_cmask *out_ } for (si = ref.shard_first; si < ref.shard_end; si++) { - const struct scx_cid_shard *shard = &scx_cid_shard_ranges[si]; + const struct scx_cid_shard *shard = + &rcu_dereference_all(scx_cid_shard_ranges)[si]; SCX_CMASK_DEFINE_SHARD(local_out, shard->base_cid, shard->nr_cids); u32 cap_bit; -- cgit v1.2.3 From 00a08ddfb48bb85885a8665bc56716c4d2f9bd82 Mon Sep 17 00:00:00 2001 From: Cui Jian Date: Wed, 22 Jul 2026 18:03:52 +0800 Subject: sched_ext: Fix stale errno in scx_sub_enable_workfn() The nesting depth check and the cgroup online check in scx_sub_enable_workfn() reach err_disable without setting ret, so the fallback error added by commit db4e9defd2e8 ("sched_ext: Record an error on errno-only sub-enable failure") reports "scx_sub_enable() failed (0)". This is currently harmless because both paths record their own scx_error() first and the first error wins, but it leaves the fallback broken for these paths. Set -EINVAL and -ENODEV there so the fallback always reports a real errno. v2: The validate_ops() path from v1 is already fixed in for-7.3 (sub.c already has ret = scx_validate_ops()), so only the two remaining paths are addressed. Signed-off-by: Cui Jian Reviewed-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/sub.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'kernel') diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index d7842a609d96..6da6c91e4287 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -1331,6 +1331,7 @@ void scx_sub_enable_workfn(struct kthread_work *work) if (sch->level >= SCX_SUB_MAX_DEPTH) { scx_error(sch, "max nesting depth %d violated", SCX_SUB_MAX_DEPTH); + ret = -EINVAL; goto err_disable; } @@ -1378,6 +1379,7 @@ void scx_sub_enable_workfn(struct kthread_work *work) set_cgroup_sched(sch_cgroup(sch), sch); if (!(cgrp->self.flags & CSS_ONLINE)) { scx_error(sch, "cgroup is not online"); + ret = -ENODEV; goto err_unlock_and_disable; } -- cgit v1.2.3 From 94ca9591108a948649ad1ba5c069ae632f16b917 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Thu, 23 Jul 2026 10:31:18 -0700 Subject: sched_ext: Repair kernel-doc comments Add missing function parameter descriptions and use the correct function name in kernel-doc comments to avoid kernel-doc warnings: Warning: kernel/sched/ext/ext.c:2692 function parameter 'sch' not described in 'finish_dispatch' Warning: kernel/sched/ext/ext.c:5309 function parameter 'stalled_mask' not described in 'scx_rcu_cpu_stall' Warning: kernel/sched/ext/ext.c:5405 function parameter 'cpu' not described in 'scx_hardlockup' Warning: kernel/sched/ext/ext.c:8470 expecting prototype for scx_bpf_dsq_insert(). Prototype was for scx_bpf_dsq_insert___v2() instead Warning: kernel/sched/ext/ext.c:8784 expecting prototype for scx_bpf_dsq_move_to_local(). Prototype was for scx_bpf_dsq_move_to_local___v2() instead Warning: kernel/sched/ext/ext.c:9498 expecting prototype for scx_bpf_reenqueue_local(). Prototype was for scx_bpf_reenqueue_local___v2() instead Signed-off-by: Randy Dunlap Reviewed-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 6a4e3a4f6098..81c2d8eeae41 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -2669,6 +2669,7 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, /** * finish_dispatch - Asynchronously finish dispatching a task + * @sch: the scheduler * @rq: current rq which is locked * @p: task to finish dispatching * @qseq_at_dispatch: qseq when @p started getting dispatched @@ -5296,6 +5297,7 @@ static __printf(2, 3) bool handle_lockup(int exit_cpu, const char *fmt, ...) /** * scx_rcu_cpu_stall - sched_ext RCU CPU stall handler + * @stalled_mask: bit mask of stalled CPUs * * While there are various reasons why RCU CPU stalls can occur on a system * that may not be caused by the current BPF scheduler, try kicking out the @@ -5390,6 +5392,7 @@ static DEFINE_IRQ_WORK(scx_hardlockup_irq_work, scx_hardlockup_irq_workfn); /** * scx_hardlockup - sched_ext hardlockup handler + * @cpu: the target CPU * * A poorly behaving BPF scheduler can trigger hard lockup by e.g. putting * numerous affinitized tasks in a single queue and directing all CPUs at it. @@ -8426,7 +8429,7 @@ static void scx_dsq_insert_commit(struct scx_sched *sch, struct task_struct *p, __bpf_kfunc_start_defs(); /** - * scx_bpf_dsq_insert - Insert a task into the FIFO queue of a DSQ + * scx_bpf_dsq_insert___v2 - Insert a task into the FIFO queue of a DSQ * @p: task_struct to insert * @dsq_id: DSQ to insert into * @slice: duration @p can run for in nsecs, 0 to keep the current value @@ -8756,7 +8759,7 @@ __bpf_kfunc void scx_bpf_dispatch_cancel(const struct bpf_prog_aux *aux) } /** - * scx_bpf_dsq_move_to_local - move a task from a DSQ to the current CPU's local DSQ + * scx_bpf_dsq_move_to_local___v2 - move a task from a DSQ to the current CPU's local DSQ * @dsq_id: DSQ to move task from. Must be a user-created DSQ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * @enq_flags: %SCX_ENQ_* @@ -9485,7 +9488,7 @@ __bpf_kfunc void scx_bpf_dsq_reenq(u64 dsq_id, u64 reenq_flags, } /** - * scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ + * scx_bpf_reenqueue_local___v2 - Re-enqueue tasks on a local DSQ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * * Iterate over all of the tasks currently enqueued on the local DSQ of the -- cgit v1.2.3 From c509d81070263f4bade65e217c5c397399837f4b Mon Sep 17 00:00:00 2001 From: Liang Luo Date: Fri, 24 Jul 2026 16:08:24 +0800 Subject: sched_ext: Fix incorrect SCX_PICK_IDLE_CPU_* flag prefix in kernel-doc The flags passed to the pick-idle kfuncs are values from the scx_pick_idle_cpu_flags enum, whose members are prefixed SCX_PICK_IDLE_ (SCX_PICK_IDLE_CORE, SCX_PICK_IDLE_IN_NODE). Three kernel-doc comments in idle.c erroneously used %SCX_PICK_IDLE_CPU_* which does not correspond to any defined flag name, while the adjacent scx_bpf_pick_idle_cpu_node() correctly documents %SCX_PICK_IDLE_*. Fix the three occurrences to use the correct SCX_PICK_IDLE_* prefix. Signed-off-by: Liang Luo Signed-off-by: Tejun Heo --- kernel/sched/ext/idle.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/idle.c b/kernel/sched/ext/idle.c index d08166de03d8..ae28f583f54c 100644 --- a/kernel/sched/ext/idle.c +++ b/kernel/sched/ext/idle.c @@ -1362,7 +1362,7 @@ __bpf_kfunc s32 scx_bpf_pick_idle_cpu_node(const struct cpumask *cpus_allowed, /** * scx_bpf_pick_idle_cpu - Pick and claim an idle cpu * @cpus_allowed: Allowed cpumask - * @flags: %SCX_PICK_IDLE_CPU_* flags + * @flags: %SCX_PICK_IDLE_* flags * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * * Pick and claim an idle cpu in @cpus_allowed. Returns the picked idle cpu @@ -1409,7 +1409,7 @@ __bpf_kfunc s32 scx_bpf_pick_idle_cpu(const struct cpumask *cpus_allowed, * or pick any CPU from @node * @cpus_allowed: Allowed cpumask * @node: target NUMA node - * @flags: %SCX_PICK_IDLE_CPU_* flags + * @flags: %SCX_PICK_IDLE_* flags * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * * Pick and claim an idle cpu in @cpus_allowed. If none is available, pick any @@ -1460,7 +1460,7 @@ __bpf_kfunc s32 scx_bpf_pick_any_cpu_node(const struct cpumask *cpus_allowed, /** * scx_bpf_pick_any_cpu - Pick and claim an idle cpu if available or pick any CPU * @cpus_allowed: Allowed cpumask - * @flags: %SCX_PICK_IDLE_CPU_* flags + * @flags: %SCX_PICK_IDLE_* flags * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * * Pick and claim an idle cpu in @cpus_allowed. If none is available, pick any -- cgit v1.2.3 From f879519db8a2f1b3922f9023daec1642f68df51d Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 24 Jul 2026 09:09:28 -1000 Subject: sched_ext: Gate local DSQ reenq on baseline cid access scx_bpf_dsq_reenq() with an SCX_DSQ_LOCAL_ON target schedules deferred reenq work on the cid's cpu, raising an IPI when the target rq isn't the locked one. Nothing checks caps along the way, so a sub-sched holding no cap at all on a cid can force its cpu to take IPIs and rq lock cycles at will. The analogous scx_bpf_kick_cid() path gates delivery on SCX_CAP_BASE in kick_one_cpu() to prevent exactly this. Apply the same rule at the reenq scheduling point: if the calling sched lacks SCX_CAP_BASE on the target cid, drop the reenq and count it in the new SCX_EV_SUB_REENQ_DENIED event. The check is lockless, which is fine: a reenq slipping through right after a revoke is harmless, and a wrong denial can't happen - if the caller has seen its ownership of the cpu, the check sees it too. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 12 ++++++++++++ kernel/sched/ext/internal.h | 9 ++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 81c2d8eeae41..fd88b4d4f12a 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1071,6 +1071,18 @@ void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, if (dsq->id == SCX_DSQ_LOCAL) { rq = container_of(dsq, struct rq, scx.local_dsq); + /* + * A sub-sched lacking baseline access on the target cid has no + * business triggering IPIs. The lockless test is fine: slipping + * through right after a revoke is harmless and a wrong denial + * can't happen - if the caller has seen its ownership, so does + * this test. + */ + if (unlikely(scx_missing_caps(sch, cpu_of(rq), SCX_CAP_BASE))) { + __scx_add_event(sch, SCX_EV_SUB_REENQ_DENIED, 1); + return; + } + struct scx_sched_pcpu *sch_pcpu = per_cpu_ptr(sch->pcpu, cpu_of(rq)); struct scx_deferred_reenq_local *drl = &sch_pcpu->deferred_reenq_local; diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index a9a853a71061..1bdf34f3fc3c 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1193,6 +1193,12 @@ struct scx_event_stats { * kick degrades to a plain reschedule. */ s64 SCX_EV_SUB_PREEMPT_DENIED; + + /* + * The number of times a local DSQ reenq was dropped because the + * sub-sched lacked baseline access on the target cid. + */ + s64 SCX_EV_SUB_REENQ_DENIED; }; #define SCX_EVENTS_LIST(SCX_EVENT) \ @@ -1212,7 +1218,8 @@ struct scx_event_stats { SCX_EVENT(SCX_EV_INSERT_NOT_OWNED); \ SCX_EVENT(SCX_EV_SUB_BYPASS_DISPATCH); \ SCX_EVENT(SCX_EV_SUB_FORCED_ADMIT); \ - SCX_EVENT(SCX_EV_SUB_PREEMPT_DENIED) + SCX_EVENT(SCX_EV_SUB_PREEMPT_DENIED); \ + SCX_EVENT(SCX_EV_SUB_REENQ_DENIED) struct scx_sched; -- cgit v1.2.3 From 7c80408fa9433778d3b506cee4ec50d52639f5cd Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 24 Jul 2026 09:09:28 -1000 Subject: sched_ext: Count kicks denied for lacking baseline cid access kick_one_cpu() silently skips a kick when the kicking sub-sched lacks SCX_CAP_BASE on the target cid, as does kick_one_cpu_if_idle() for idle kicks. The skips are sound with the same logic as the reenq gate but are invisible today, unlike the preempt degradation counted in SCX_EV_SUB_PREEMPT_DENIED. Count them in a new SCX_EV_SUB_KICK_DENIED event so every cap denial is observable. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 20 ++++++++++++++------ kernel/sched/ext/internal.h | 8 ++++++++ 2 files changed, 22 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index fd88b4d4f12a..5d02bb99c09d 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -8118,6 +8118,7 @@ static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_r struct scx_rq *this_scx = &this_rq->scx; const struct sched_class *cur_class; bool should_wait = false; + bool kickable; unsigned long flags; raw_spin_rq_lock_irqsave(rq, flags); @@ -8131,9 +8132,10 @@ static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_r * business forcing a reschedule there - skip. This is the authoritative * cap check: ecaps is read here under @rq's lock. */ - if ((cpu_online(cpu) || cpu == cpu_of(this_rq)) && - !sched_class_above(cur_class, &ext_sched_class) && - !scx_missing_caps(pcpu->sch, cpu, SCX_CAP_BASE)) { + kickable = (cpu_online(cpu) || cpu == cpu_of(this_rq)) && + !sched_class_above(cur_class, &ext_sched_class); + + if (kickable && !scx_missing_caps(pcpu->sch, cpu, SCX_CAP_BASE)) { if (cpumask_test_cpu(cpu, pcpu->cpus_to_preempt)) { if (cur_class == &ext_sched_class) { if (likely(!scx_missing_caps(pcpu->sch, cpu, @@ -8157,6 +8159,9 @@ static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_r resched_curr(rq); } else { + /* a kickable cpu was skipped solely for the missing caps */ + if (kickable) + __scx_add_event(pcpu->sch, SCX_EV_SUB_KICK_DENIED, 1); cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt); cpumask_clear_cpu(cpu, pcpu->cpus_to_wait); } @@ -8176,9 +8181,12 @@ static void kick_one_cpu_if_idle(s32 cpu, struct scx_sched_pcpu *pcpu, /* idle kicks need baseline access too, see kick_one_cpu() */ if (!can_skip_idle_kick(rq) && - (cpu_online(cpu) || cpu == cpu_of(this_rq)) && - !scx_missing_caps(pcpu->sch, cpu, SCX_CAP_BASE)) - resched_curr(rq); + (cpu_online(cpu) || cpu == cpu_of(this_rq))) { + if (likely(!scx_missing_caps(pcpu->sch, cpu, SCX_CAP_BASE))) + resched_curr(rq); + else + __scx_add_event(pcpu->sch, SCX_EV_SUB_KICK_DENIED, 1); + } raw_spin_rq_unlock_irqrestore(rq, flags); } diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 1bdf34f3fc3c..308d16320818 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1194,6 +1194,13 @@ struct scx_event_stats { */ s64 SCX_EV_SUB_PREEMPT_DENIED; + /* + * The number of times a kick was skipped because the sub-sched lacked + * baseline access on the target cid. The preempt-part degradation of a + * delivered kick is counted in SCX_EV_SUB_PREEMPT_DENIED instead. + */ + s64 SCX_EV_SUB_KICK_DENIED; + /* * The number of times a local DSQ reenq was dropped because the * sub-sched lacked baseline access on the target cid. @@ -1219,6 +1226,7 @@ struct scx_event_stats { SCX_EVENT(SCX_EV_SUB_BYPASS_DISPATCH); \ SCX_EVENT(SCX_EV_SUB_FORCED_ADMIT); \ SCX_EVENT(SCX_EV_SUB_PREEMPT_DENIED); \ + SCX_EVENT(SCX_EV_SUB_KICK_DENIED); \ SCX_EVENT(SCX_EV_SUB_REENQ_DENIED) struct scx_sched; -- cgit v1.2.3 From 49d6247d64aec7ba1590223d277c47ae09037d4d Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 24 Jul 2026 09:09:28 -1000 Subject: sched_ext: Factor out scx_cpuperf_set() Factor the cpuperf target write out of scx_bpf_cpuperf_set() into scx_cpuperf_set() which takes the acting sched and returns 0 or -errno, and flatten the nested validation into early returns. No functional change. Prep for gating the write behind a cap and reporting the outcome from the cid-form kfunc. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 78 +++++++++++++++++++++++++++++--------------------- 1 file changed, 45 insertions(+), 33 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 5d02bb99c09d..e11c66bd23e7 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -9781,6 +9781,50 @@ __bpf_kfunc u32 scx_bpf_cidperf_cur(s32 cid, const struct bpf_prog_aux *aux) return arch_scale_freq_capacity(cpu); } +/* validate and apply a cpuperf target, see scx_bpf_cpuperf_set() */ +static s32 scx_cpuperf_set(struct scx_sched *sch, s32 cpu, u32 perf) +{ + struct rq *rq, *locked_rq; + struct rq_flags rf; + + if (unlikely(perf > SCX_CPUPERF_ONE)) { + scx_error(sch, "Invalid cpuperf target %u for CPU %d", perf, cpu); + return -EINVAL; + } + + if (!scx_cpu_valid(sch, cpu, NULL)) + return -EINVAL; + + rq = cpu_rq(cpu); + locked_rq = scx_locked_rq(); + + /* + * When called with an rq lock held, restrict the operation to the + * corresponding CPU to prevent ABBA deadlocks. + */ + if (locked_rq && rq != locked_rq) { + scx_error(sch, "Invalid target CPU %d", cpu); + return -EINVAL; + } + + /* + * If no rq lock is held, allow to operate on any CPU by acquiring + * the corresponding rq lock. + */ + if (!locked_rq) { + rq_lock_irqsave(rq, &rf); + update_rq_clock(rq); + } + + rq->scx.cpuperf_target = perf; + cpufreq_update_util(rq, 0); + + if (!locked_rq) + rq_unlock_irqrestore(rq, &rf); + + return 0; +} + /** * scx_bpf_cpuperf_set - Set the relative performance target of a CPU * @cpu: CPU of interest @@ -9806,39 +9850,7 @@ __bpf_kfunc void scx_bpf_cpuperf_set(s32 cpu, u32 perf, const struct bpf_prog_au if (unlikely(!sch)) return; - if (unlikely(perf > SCX_CPUPERF_ONE)) { - scx_error(sch, "Invalid cpuperf target %u for CPU %d", perf, cpu); - return; - } - - if (scx_cpu_valid(sch, cpu, NULL)) { - struct rq *rq = cpu_rq(cpu), *locked_rq = scx_locked_rq(); - struct rq_flags rf; - - /* - * When called with an rq lock held, restrict the operation - * to the corresponding CPU to prevent ABBA deadlocks. - */ - if (locked_rq && rq != locked_rq) { - scx_error(sch, "Invalid target CPU %d", cpu); - return; - } - - /* - * If no rq lock is held, allow to operate on any CPU by - * acquiring the corresponding rq lock. - */ - if (!locked_rq) { - rq_lock_irqsave(rq, &rf); - update_rq_clock(rq); - } - - rq->scx.cpuperf_target = perf; - cpufreq_update_util(rq, 0); - - if (!locked_rq) - rq_unlock_irqrestore(rq, &rf); - } + scx_cpuperf_set(sch, cpu, perf); } /** -- cgit v1.2.3 From 3a21e34eb2588e135190cf5c48e64397bc18d0b5 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 24 Jul 2026 09:09:28 -1000 Subject: sched_ext: Gate scx_bpf_cidperf_set() behind a new SCX_CAP_PERF scx_bpf_cidperf_set() reaches cpufreq with no cap check, so any cid-form sub-sched can steer the frequency of any cid in its view, including ones it holds nothing on. Gate it behind a new SCX_CAP_PERF rather than SCX_CAP_BASE: hardware control is a separate axis from queue access - a parent may well delegate scheduling on a cid without handing over its frequency. PERF neither implies nor is implied by the other caps. The check runs under the target rq's lock, which ecaps updates are also folded under, so it is authoritative - a write can never land after a revoke has taken effect. Denials are counted in SCX_EV_SUB_CIDPERF_DENIED. The operation is synchronous and the outcome is reported to the caller: scx_bpf_cidperf_set() now returns 0 or -errno, -EACCES on denial. The cid-form interface is still under initial development, so the signature is changed in place without versioning. scx_qmap grants PERF alongside its existing cid grants so the cpuperf demo keeps working in sub-scheds. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 35 ++++++++++++++++++++++++-------- kernel/sched/ext/internal.h | 16 ++++++++++++++- tools/sched_ext/include/scx/common.bpf.h | 2 +- tools/sched_ext/scx_qmap.bpf.c | 17 ++++++++++------ 4 files changed, 53 insertions(+), 17 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index e11c66bd23e7..aca8d2380509 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -5144,6 +5144,7 @@ static const char *scx_cap_names[__SCX_NR_CAPS] = { [__SCX_CAP_ENQ_IMMED] = "enq_immed", [__SCX_CAP_ENQ] = "enq", [__SCX_CAP_PREEMPT] = "preempt", + [__SCX_CAP_PERF] = "perf", }; static ssize_t scx_attr_caps_show(struct kobject *kobj, @@ -9786,6 +9787,7 @@ static s32 scx_cpuperf_set(struct scx_sched *sch, s32 cpu, u32 perf) { struct rq *rq, *locked_rq; struct rq_flags rf; + s32 ret; if (unlikely(perf > SCX_CPUPERF_ONE)) { scx_error(sch, "Invalid cpuperf target %u for CPU %d", perf, cpu); @@ -9816,13 +9818,24 @@ static s32 scx_cpuperf_set(struct scx_sched *sch, s32 cpu, u32 perf) update_rq_clock(rq); } - rq->scx.cpuperf_target = perf; - cpufreq_update_util(rq, 0); + /* + * ecaps updates are folded under the rq lock, making this test + * authoritative: a write can never land after a revoke has taken + * effect on @cpu. + */ + if (likely(!scx_missing_caps(sch, cpu, SCX_CAP_PERF))) { + rq->scx.cpuperf_target = perf; + cpufreq_update_util(rq, 0); + ret = 0; + } else { + __scx_add_event(sch, SCX_EV_SUB_CIDPERF_DENIED, 1); + ret = -EACCES; + } if (!locked_rq) rq_unlock_irqrestore(rq, &rf); - return 0; + return ret; } /** @@ -9859,10 +9872,13 @@ __bpf_kfunc void scx_bpf_cpuperf_set(s32 cpu, u32 perf, const struct bpf_prog_au * @perf: target performance level [0, %SCX_CPUPERF_ONE] * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * - * cid-addressed equivalent of scx_bpf_cpuperf_set(). + * cid-addressed equivalent of scx_bpf_cpuperf_set(). A sub-sched needs + * SCX_CAP_PERF on @cid. Returns 0 if the target was applied, -%EACCES if + * the write was denied for missing caps, other -errnos if @cid didn't + * resolve. */ -__bpf_kfunc void scx_bpf_cidperf_set(s32 cid, u32 perf, - const struct bpf_prog_aux *aux) +__bpf_kfunc s32 scx_bpf_cidperf_set(s32 cid, u32 perf, + const struct bpf_prog_aux *aux) { struct scx_sched *sch; s32 cpu; @@ -9871,11 +9887,12 @@ __bpf_kfunc void scx_bpf_cidperf_set(s32 cid, u32 perf, sch = scx_prog_sched(aux); if (unlikely(!sch)) - return; + return -ENODEV; cpu = scx_cid_to_cpu(sch, cid); if (cpu < 0) - return; - scx_bpf_cpuperf_set(cpu, perf, aux); + return cpu; + + return scx_cpuperf_set(sch, cpu, perf); } /** diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 308d16320818..886f1d132e6b 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1206,6 +1206,12 @@ struct scx_event_stats { * sub-sched lacked baseline access on the target cid. */ s64 SCX_EV_SUB_REENQ_DENIED; + + /* + * The number of times scx_bpf_cidperf_set() was denied because the + * sub-sched lacked SCX_CAP_PERF on the target cid. + */ + s64 SCX_EV_SUB_CIDPERF_DENIED; }; #define SCX_EVENTS_LIST(SCX_EVENT) \ @@ -1227,7 +1233,8 @@ struct scx_event_stats { SCX_EVENT(SCX_EV_SUB_FORCED_ADMIT); \ SCX_EVENT(SCX_EV_SUB_PREEMPT_DENIED); \ SCX_EVENT(SCX_EV_SUB_KICK_DENIED); \ - SCX_EVENT(SCX_EV_SUB_REENQ_DENIED) + SCX_EVENT(SCX_EV_SUB_REENQ_DENIED); \ + SCX_EVENT(SCX_EV_SUB_CIDPERF_DENIED) struct scx_sched; @@ -1353,6 +1360,11 @@ struct scx_sched_pnode { * - SCX_ENQ_PREEMPT inserts * - SCX_KICK_PREEMPT kicks * + * PERF control the cid's cpu power/perf management state, currently the + * cpufreq target set through scx_bpf_cidperf_set(). Hardware + * control is a separate axis from queue access: PERF neither + * implies nor is implied by the caps above. + * * Implied caps apply to the holder's own use of a cid, not to delegation. * scx_bpf_sub_grant() delegates literally-held caps, so a cap held only through * implication is usable but cannot be re-delegated to a child. When granting a @@ -1363,6 +1375,7 @@ enum scx_cap_flags { __SCX_CAP_ENQ_IMMED = 0, __SCX_CAP_ENQ = 1, __SCX_CAP_PREEMPT = 2, + __SCX_CAP_PERF = 3, __SCX_NR_CAPS, __SCX_CAP_ALL = BIT_U64(__SCX_NR_CAPS) - 1, @@ -1370,6 +1383,7 @@ enum scx_cap_flags { SCX_CAP_ENQ_IMMED = BIT_U64(__SCX_CAP_ENQ_IMMED), SCX_CAP_ENQ = BIT_U64(__SCX_CAP_ENQ), SCX_CAP_PREEMPT = BIT_U64(__SCX_CAP_PREEMPT), + SCX_CAP_PERF = BIT_U64(__SCX_CAP_PERF), /* alias for minimal cap to make any use of a cpu */ SCX_CAP_BASE = SCX_CAP_ENQ_IMMED, diff --git a/tools/sched_ext/include/scx/common.bpf.h b/tools/sched_ext/include/scx/common.bpf.h index acc2b131ea8f..6035d007c43f 100644 --- a/tools/sched_ext/include/scx/common.bpf.h +++ b/tools/sched_ext/include/scx/common.bpf.h @@ -112,7 +112,7 @@ u32 scx_bpf_nr_cids(void) __ksym __weak; u32 scx_bpf_nr_online_cids(void) __ksym __weak; u32 scx_bpf_cidperf_cap(s32 cid) __ksym __weak; u32 scx_bpf_cidperf_cur(s32 cid) __ksym __weak; -void scx_bpf_cidperf_set(s32 cid, u32 perf) __ksym __weak; +s32 scx_bpf_cidperf_set(s32 cid, u32 perf) __ksym __weak; /* sub-scheduler cap control, scx_bpf_sub_caps() cgroup_id 0 == self */ s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, const struct scx_cmask *cmask, diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c index aead17658573..8822ed11c0d8 100644 --- a/tools/sched_ext/scx_qmap.bpf.c +++ b/tools/sched_ext/scx_qmap.bpf.c @@ -1585,11 +1585,13 @@ __noinline void apply_partition(void) cmask_copy(&qa.to_grant_cids.mask, &ssc->granted_cids.mask); cmask_andnot(&qa.to_grant_cids.mask, &ssc->prev_granted.mask); - scx_bpf_sub_revoke(cgid, SCX_CAP_ENQ_IMMED, + scx_bpf_sub_revoke(cgid, SCX_CAP_ENQ_IMMED | SCX_CAP_PERF, (void *)(long)&qa.prev_rr_cids.mask); - scx_bpf_sub_revoke(cgid, SCX_CAP_ENQ | SCX_CAP_PREEMPT | SCX_CAP_ENQ_IMMED, + scx_bpf_sub_revoke(cgid, SCX_CAP_ENQ | SCX_CAP_PREEMPT | + SCX_CAP_ENQ_IMMED | SCX_CAP_PERF, (void *)(long)&qa.to_revoke_cids.mask); - scx_bpf_sub_grant(cgid, SCX_CAP_ENQ | SCX_CAP_PREEMPT | SCX_CAP_ENQ_IMMED, + scx_bpf_sub_grant(cgid, SCX_CAP_ENQ | SCX_CAP_PREEMPT | + SCX_CAP_ENQ_IMMED | SCX_CAP_PERF, (void *)(long)&qa.to_grant_cids.mask, NULL); } @@ -1605,7 +1607,8 @@ __noinline void apply_partition(void) holder_cgid = qa.part.rr_slots[pos]; /* 0 = self, nothing to grant */ if (holder_cgid) - scx_bpf_sub_grant(holder_cgid, SCX_CAP_ENQ_IMMED, + scx_bpf_sub_grant(holder_cgid, + SCX_CAP_ENQ_IMMED | SCX_CAP_PERF, (void *)(long)&qa.rr_cids.mask, NULL); } } @@ -1696,10 +1699,12 @@ static void rr_advance(void) * time-share. */ if (old_cgid) - scx_bpf_sub_revoke(old_cgid, SCX_CAP_ENQ_IMMED, + scx_bpf_sub_revoke(old_cgid, + SCX_CAP_ENQ_IMMED | SCX_CAP_PERF, (void *)(long)&qa.rr_cids.mask); if (new_cgid) - scx_bpf_sub_grant(new_cgid, SCX_CAP_ENQ_IMMED, + scx_bpf_sub_grant(new_cgid, + SCX_CAP_ENQ_IMMED | SCX_CAP_PERF, (void *)(long)&qa.rr_cids.mask, NULL); } -- cgit v1.2.3 From 7947442047cff1d296ab615b8c6ddbc7c9b7bf21 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Thu, 23 Jul 2026 15:07:56 -1000 Subject: sched_ext: Add scx_cgroup_sched() for cgrp->scx_sched reads cgrp->scx_sched is __rcu and published with rcu_assign_pointer() but every reader loads it with a plain access, so sparse flags all of them. The reads are lock-protected: enable/disable paths rewrite the field under all of scx_enable_mutex, scx_fork_rwsem and cgroup_mutex, and cgroup creation inherits the parent's sched under cgroup_mutex before the new cgroup is reachable, so holding any one of the three locks makes the read stable. Add scx_cgroup_sched() which states the protection with rcu_dereference_check() and convert the readers. No functional changes. Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 6 +++--- kernel/sched/ext/sub.c | 16 +++++++++------- kernel/sched/ext/sub.h | 14 ++++++++++++++ 3 files changed, 26 insertions(+), 10 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index aca8d2380509..08cadca29c85 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -3807,7 +3807,7 @@ int scx_fork(struct task_struct *p, struct kernel_clone_args *kargs) if (scx_init_task_enabled) { #ifdef CONFIG_EXT_SUB_SCHED - struct scx_sched *sch = kargs->cset->dfl_cgrp->scx_sched; + struct scx_sched *sch = scx_cgroup_sched(kargs->cset->dfl_cgrp); #else struct scx_sched *sch = scx_root; #endif @@ -4461,7 +4461,7 @@ int scx_tg_online(struct task_group *tg) * always belongs to the root sched. */ if (cgroup_on_dfl(tg->css.cgroup)) - sch = tg->css.cgroup->scx_sched; + sch = scx_cgroup_sched(tg->css.cgroup); else sch = scx_tg_sched(&root_task_group); @@ -4542,7 +4542,7 @@ int scx_cgroup_can_attach(struct cgroup_taskset *tset) * cgroup's sched and is reported through the * exit_task/init_task pair that the re-homing generates. */ - if (!sch || sch != task_css_set(p)->mg_dst_cset->dfl_cgrp->scx_sched) + if (!sch || sch != scx_cgroup_sched(task_css_set(p)->mg_dst_cset->dfl_cgrp)) continue; if (SCX_HAS_OP(sch, cgroup_prep_move)) { diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 6da6c91e4287..2a8c979c7976 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -1204,7 +1204,7 @@ dump: /* verify that a scheduler can be attached to @cgrp and return the parent */ static struct scx_sched *find_parent_sched(struct cgroup *cgrp) { - struct scx_sched *parent = cgrp->scx_sched; + struct scx_sched *parent = scx_cgroup_sched(cgrp); struct scx_sched *pos; lockdep_assert_held(&scx_sched_lock); @@ -1578,7 +1578,7 @@ static s32 scx_cgroup_task_migrating(struct cgroup_task_migrate_ctx *ctx) if (!scx_cgroup_enabled) return NOTIFY_OK; - to = ctx->dst_dcgrp->scx_sched; + to = scx_cgroup_sched(ctx->dst_dcgrp); if (scx_task_on_sched(to, p)) return NOTIFY_OK; @@ -1612,7 +1612,7 @@ static void scx_cgroup_task_migrated(struct cgroup_task_migrate_ctx *ctx) if (!scx_cgroup_enabled) return; - to = ctx->dst_dcgrp->scx_sched; + to = scx_cgroup_sched(ctx->dst_dcgrp); if (scx_task_on_sched(to, p)) return; @@ -1639,7 +1639,7 @@ static void scx_cgroup_task_migrate_canceled(struct cgroup_task_migrate_ctx *ctx if (!scx_cgroup_enabled) return; - to = ctx->dst_dcgrp->scx_sched; + to = scx_cgroup_sched(ctx->dst_dcgrp); if (scx_task_on_sched(to, p)) return; @@ -1653,6 +1653,7 @@ static s32 scx_cgroup_lifetime_notify(struct notifier_block *nb, { struct cgroup *cgrp = data; struct cgroup *parent = cgroup_parent(cgrp); + struct scx_sched *sch; if (!cgroup_on_dfl(cgrp)) return NOTIFY_OK; @@ -1661,12 +1662,13 @@ static s32 scx_cgroup_lifetime_notify(struct notifier_block *nb, case CGROUP_LIFETIME_ONLINE: /* inherit ->scx_sched from $parent */ if (parent) - rcu_assign_pointer(cgrp->scx_sched, parent->scx_sched); + rcu_assign_pointer(cgrp->scx_sched, scx_cgroup_sched(parent)); break; case CGROUP_LIFETIME_OFFLINE: /* if there is a sched attached, shoot it down */ - if (cgrp->scx_sched && cgrp->scx_sched->cgrp == cgrp) - scx_exit(cgrp->scx_sched, SCX_EXIT_UNREG_KERN, + sch = scx_cgroup_sched(cgrp); + if (sch && sch->cgrp == cgrp) + scx_exit(sch, SCX_EXIT_UNREG_KERN, SCX_ECODE_RSN_CGROUP_OFFLINE, "cgroup %llu going offline", cgroup_id(cgrp)); break; diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index 625d7ce334aa..0019b75a2560 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -39,6 +39,20 @@ struct scx_dispatch_q *scx_local_or_reject_dsq(struct scx_sched *sch, struct rq bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p); void scx_reenq_reject(struct rq *rq); +/* + * cgrp->scx_sched is written by root/sub enable/disable under all of + * scx_enable_mutex, scx_fork_rwsem and cgroup_mutex. A new cgroup inherits the + * parent's sched under just cgroup_mutex but is not yet reachable by the other + * two lock holders. Any one of the three locks stabilizes the association. + */ +static inline struct scx_sched *scx_cgroup_sched(struct cgroup *cgrp) +{ + return rcu_dereference_check(cgrp->scx_sched, + lockdep_is_held(&cgroup_mutex) || + percpu_rwsem_is_held(&scx_fork_rwsem) || + lockdep_is_held(&scx_enable_mutex)); +} + static inline const char *sch_cgrp_path(struct scx_sched *sch) { return sch->cgrp_path; -- cgit v1.2.3 From ce228343010df67e0a35bd1a9980abcfdf9cf95f Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Thu, 23 Jul 2026 15:07:56 -1000 Subject: sched_ext: Resolve most remaining scx_root accesses scx_root is __rcu and naked accesses were left as transitional markers for the multi-scheduler transition, to be converted to accesses through the associated scheduler instances. Most accesses have since been converted to resolve the sched from the program or task at hand. The remaining naked sites divide into ones that semantically always want the root sched, which this patch resolves, and one that is left to a later patch. The resolved sites: - The SCX_OPS_TID_TO_TASK validation and the ecaps sync kick already hold a sched whose ancestors[] pins the root as entry 0 with plain pointers stable for the sched's lifetime. Reach the root through the sched at hand. - The dispatch entry, class switch, idle notification and fork init paths only execute while the scheduler is live and scx_root never changes inside the live window, so no update can race them. Add scx_root_protected_live() which documents that invariant and resolves with a plain load. - The hotplug path, including the ecaps reseeds, runs with the hotplug lock held, which excludes the scx_root writers. Add scx_root_protected(), which accepts either the hotplug lock or scx_enable_mutex. - Is-root tests use a zero level instead of comparing against the global. touch_core_sched_dispatch() stays naked, to be resolved by a later patch. Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 12 ++++++------ kernel/sched/ext/idle.c | 15 +++++++-------- kernel/sched/ext/internal.h | 27 +++++++++++++++++++++++++++ kernel/sched/ext/sub.c | 17 ++++++++++------- 4 files changed, 50 insertions(+), 21 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 08cadca29c85..2002ab0f7c9b 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -2798,7 +2798,7 @@ static inline void maybe_queue_balance_callback(struct rq *rq) static int balance_one(struct rq *rq, struct task_struct *prev) { - struct scx_sched *sch = scx_root; + struct scx_sched *sch = scx_root_protected_live(); s32 cpu = cpu_of(rq); lockdep_assert_rq_held(rq); @@ -2954,7 +2954,7 @@ preempt_reason_from_class(const struct sched_class *class) static void switch_class(struct rq *rq, struct task_struct *next) { - struct scx_sched *sch = scx_root; + struct scx_sched *sch = scx_root_protected_live(); const struct sched_class *next_class = next->sched_class; if (!(sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT)) @@ -3345,7 +3345,7 @@ static void set_cpus_allowed_scx(struct task_struct *p, static void handle_hotplug(struct rq *rq, bool online) { - struct scx_sched *sch = scx_root; + struct scx_sched *sch = scx_root_protected(); s32 cpu = cpu_of(rq); s32 cpu_or_cid = cpu; @@ -3809,7 +3809,7 @@ int scx_fork(struct task_struct *p, struct kernel_clone_args *kargs) #ifdef CONFIG_EXT_SUB_SCHED struct scx_sched *sch = scx_cgroup_sched(kargs->cset->dfl_cgrp); #else - struct scx_sched *sch = scx_root; + struct scx_sched *sch = scx_root_protected_live(); #endif scx_set_task_state(p, SCX_TASK_INIT_BEGIN); ret = __scx_init_task(sch, p, NULL, true); @@ -5741,7 +5741,7 @@ void scx_disable_bypass_dsp(struct scx_sched *sch) static void unbypass_renotify_idle(struct rq *rq, struct scx_sched *pos, struct scx_sched_pcpu *pcpu) { - if (pos == scx_root) { + if (!pos->level) { rq->scx.flags |= SCX_RQ_ROOT_IDLE_RENOTIFY; return; } @@ -7109,7 +7109,7 @@ int scx_validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops) * enabled it. */ if ((ops->flags & SCX_OPS_TID_TO_TASK) && scx_parent(sch) && - !(scx_root->ops.flags & SCX_OPS_TID_TO_TASK)) { + !(sch->ancestors[0]->ops.flags & SCX_OPS_TID_TO_TASK)) { scx_error(sch, "SCX_OPS_TID_TO_TASK requires root scheduler to enable it"); return -EINVAL; } diff --git a/kernel/sched/ext/idle.c b/kernel/sched/ext/idle.c index ae28f583f54c..3e9d6a44bf43 100644 --- a/kernel/sched/ext/idle.c +++ b/kernel/sched/ext/idle.c @@ -742,30 +742,29 @@ static void scx_idle_notify(struct rq *rq, bool idle, bool do_notify, bool root_ { s32 cpu = cpu_of(rq); s32 cid = scx_cpu_arg(cpu); + struct scx_sched *root = scx_root_protected_live(); struct scx_sched *pos; lockdep_assert_rq_held(rq); /* with no sub-sched, only the root can be owed a notification */ if (!scx_has_subs()) { - struct scx_sched *sch = scx_root; - if ((do_notify || root_renotify) && - SCX_HAS_OP(sch, update_idle) && !scx_bypassing(sch, cpu)) - SCX_CALL_OP(sch, update_idle, rq, cid, idle); + SCX_HAS_OP(root, update_idle) && !scx_bypassing(root, cpu)) + SCX_CALL_OP(root, update_idle, rq, cid, idle); return; } - pos = scx_next_descendant_pre(NULL, scx_root); + pos = scx_next_descendant_pre(NULL, root); while (pos) { bool forced = false; if (unlikely(scx_missing_caps(pos, cpu, SCX_CAP_BASE))) { - pos = scx_skip_subtree_pre(pos, scx_root); + pos = scx_skip_subtree_pre(pos, root); continue; } - if (pos == scx_root) { + if (!pos->level) { forced = root_renotify; } #ifdef CONFIG_EXT_SUB_SCHED @@ -777,7 +776,7 @@ static void scx_idle_notify(struct rq *rq, bool idle, bool do_notify, bool root_ if ((do_notify || forced) && SCX_HAS_OP(pos, update_idle) && !scx_bypassing(pos, cpu)) SCX_CALL_OP(pos, update_idle, rq, cid, idle); - pos = scx_next_descendant_pre(pos, scx_root); + pos = scx_next_descendant_pre(pos, root); } } diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 886f1d132e6b..39dddcbb3b7d 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -2025,6 +2025,33 @@ extern struct scx_sched *scx_enabling_sub_sched; #define scx_error(sch, fmt, args...) \ scx_exit((sch), SCX_EXIT_ERROR, 0, fmt, ##args) +/** + * scx_root_protected_live - Root sched for paths that only run while live + * + * scx_root is published before the scheduler goes live and cleared only after + * it is fully drained, so a path that only executes while the scheduler is live + * can never race an update. Return the root sched with a plain load, never + * %NULL. + */ +static inline struct scx_sched *scx_root_protected_live(void) +{ + return rcu_dereference_protected(scx_root, true); +} + +/** + * scx_root_protected - Root sched for contexts that exclude its updates + * + * Both scx_root updates run under the locks checked below, so holding one + * excludes them. Return the root sched with a plain load, %NULL if no scheduler + * is loaded. + */ +static inline struct scx_sched *scx_root_protected(void) +{ + return rcu_dereference_protected(scx_root, + lockdep_is_cpus_held() || + lockdep_is_held(&scx_enable_mutex)); +} + static inline struct scx_dispatch_q *scx_bypass_dsq(struct scx_sched *sch, s32 cpu) { return &per_cpu_ptr(sch->pcpu, cpu)->bypass_dsq; diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 2a8c979c7976..824fe35f00ee 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -477,7 +477,7 @@ static void queue_sync_ecaps(struct scx_sched *sch, s32 cid) if (llist_on_list(&pcpu->ecaps_to_sync_node)) return; if (llist_add(&pcpu->ecaps_to_sync_node, &cpu_rq(cpu)->scx.ecaps_to_sync)) - scx_kick_cpu(scx_root, cpu, 0); + scx_kick_cpu(sch->ancestors[0], cpu, 0); } /* discard @rq's queued ecaps syncs */ @@ -638,7 +638,7 @@ void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch) */ void scx_online_ecaps(struct rq *rq) { - struct scx_sched *pos; + struct scx_sched *root, *pos; s32 cid, shard; /* @@ -652,14 +652,15 @@ void scx_online_ecaps(struct rq *rq) guard(rq_lock_irqsave)(rq); + root = scx_root_protected(); cid = __scx_cpu_to_cid(cpu_of(rq)); shard = rcu_dereference_all(scx_cid_to_shard)[cid]; - scx_for_each_descendant_pre(pos, scx_root) { + scx_for_each_descendant_pre(pos, root) { struct scx_pshard *ps; /* root holds every cap and never uses ecaps */ - if (pos == scx_root) + if (!pos->level) continue; ps = pos->pshard[shard]; @@ -679,13 +680,15 @@ void scx_online_ecaps(struct rq *rq) void scx_offline_ecaps(struct rq *rq) { s32 cpu = cpu_of(rq); - struct scx_sched *pos; + struct scx_sched *root, *pos; guard(rq_lock_irqsave)(rq); - scx_for_each_descendant_pre(pos, scx_root) { + root = scx_root_protected(); + + scx_for_each_descendant_pre(pos, root) { /* root holds every cap and never uses ecaps */ - if (pos == scx_root) + if (!pos->level) continue; WRITE_ONCE(per_cpu_ptr(pos->pcpu, cpu)->ecaps, 0); -- cgit v1.2.3 From 6ee471b646e09203f8dca27ffff78bb454dc97ac Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Thu, 23 Jul 2026 15:07:56 -1000 Subject: sched_ext: Use rcu_access_pointer() for the first_task comparison dsq->first_task is __rcu for the lockless scx_bpf_dsq_peek(). The task removal path compares it against the departing task with a plain load, which sparse flags. The comparison runs under the dsq lock and only tests identity, so rcu_access_pointer() is the fit. Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 2002ab0f7c9b..79dc258515e7 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1654,7 +1654,7 @@ static void task_unlink_from_dsq(struct task_struct *p, list_del_init(&p->scx.dsq_list.node); dsq_dec_nr(dsq, p); - if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN) && dsq->first_task == p) { + if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN) && rcu_access_pointer(dsq->first_task) == p) { struct task_struct *first_task; first_task = nldsq_next_task(dsq, NULL, false); -- cgit v1.2.3 From 7706d6e4f2e3ad7dfb92b84cacd0c16e6e3c8381 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sun, 26 Jul 2026 11:11:51 -1000 Subject: sched_ext: Bound per-task reenqueues and eject the owning scheduler Unlike local reenqueues, cap rejections have no repeat limit. A malfunctioning scheduler can keep re-inserting a task to a cid it lacks caps on, cycling the task through reject and reenqueue. This was assumed safe because a task that never runs trips the stall watchdog. However, the reenqueue irq_work re-arms itself and outranks the timer vector, blocking everything else on the CPU including stall detection and recovery, until the NMI hardlockup detector fires. Local reenqueues already have a repeat cap, SCX_REENQ_LOCAL_MAX_REPEAT, which needs generalizing to cover all reenqueues. It also has an attribution problem. Counted per-cpu on root, it tears down the whole hierarchy even when a sub-scheduler caused the repeated reenqueues. Generalize by bounding every reenqueue with one per-task counter. reenq_cnt is bumped in scx_do_enqueue_task() on each SCX_ENQ_REENQ, the single path every reenqueue producer passes through, and cleared in clr_task_runnable() when the task is picked to run and in scx_disable_task() when it leaves the scheduler's control. Past SCX_REENQ_MAX_REPEAT the task's owning scheduler is ejected with a new SCX_EXIT_ERROR_REENQ and the task is left stranded to be picked up during sched exit. The SCX_EV_REENQ_LOCAL_REPEAT event becomes SCX_EV_REENQ_REPEAT, counting repeat reenqueues from all sources. v2: Count SCX_EV_REENQ_REPEAT only when a reenqueue leads to another reenqueue, not on every reenqueue. v3: - Also clear reenq_cnt in scx_disable_task() so that the count doesn't carry over to the next owner across sched class switches, scheduler replacement or sub-scheduler rehoming (Andrea Righi). - Update the stale SCX_EV_REENQ_LOCAL_REPEAT references in sched-ext.rst (Andrea Righi). Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- Documentation/scheduler/sched-ext.rst | 8 ++--- include/linux/sched/ext.h | 1 + kernel/sched/ext/ext.c | 58 ++++++++++++++++++++--------------- kernel/sched/ext/internal.h | 19 +++++------- kernel/sched/ext/sub.c | 6 ++-- kernel/sched/ext/types.h | 2 +- kernel/sched/sched.h | 1 - 7 files changed, 51 insertions(+), 44 deletions(-) (limited to 'kernel') diff --git a/Documentation/scheduler/sched-ext.rst b/Documentation/scheduler/sched-ext.rst index 2771ea4cc14a..ad2fff3c0593 100644 --- a/Documentation/scheduler/sched-ext.rst +++ b/Documentation/scheduler/sched-ext.rst @@ -106,7 +106,7 @@ counters. Each counter occupies one ``name value`` line: SCX_EV_ENQ_SKIP_EXITING 0 SCX_EV_ENQ_SKIP_MIGRATION_DISABLED 0 SCX_EV_REENQ_IMMED 0 - SCX_EV_REENQ_LOCAL_REPEAT 0 + SCX_EV_REENQ_REPEAT 0 SCX_EV_REFILL_SLICE_DFL 456789 SCX_EV_BYPASS_DURATION 0 SCX_EV_BYPASS_DISPATCH 0 @@ -129,9 +129,9 @@ The counters are described in ``kernel/sched/ext/internal.h``; briefly: ``SCX_OPS_ENQ_MIGRATION_DISABLED`` is not set). * ``SCX_EV_REENQ_IMMED``: a task dispatched with ``SCX_ENQ_IMMED`` was re-enqueued because the target CPU was not available for immediate execution. -* ``SCX_EV_REENQ_LOCAL_REPEAT``: a reenqueue of the local DSQ triggered - another reenqueue; recurring counts indicate incorrect ``SCX_ENQ_REENQ`` - handling in the BPF scheduler. +* ``SCX_EV_REENQ_REPEAT``: a reenqueue led to another reenqueue without the + task running in between; recurring counts indicate that the BPF scheduler + keeps re-deciding placements it can't honor. * ``SCX_EV_REFILL_SLICE_DFL``: a task's time slice was refilled with the default value (``SCX_SLICE_DFL``). * ``SCX_EV_BYPASS_DURATION``: total nanoseconds spent in bypass mode. diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index 78b2f289cb98..bd9c4059e8fc 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -198,6 +198,7 @@ struct sched_ext_entity { u32 dsq_flags; /* protected by DSQ lock */ u32 flags; /* protected by rq lock */ u32 weight; + u32 reenq_cnt; /* reenqueues since last run */ s32 sticky_cpu; s32 holding_cpu; s32 selected_cpu; diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 79dc258515e7..ff9151232f58 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1903,6 +1903,24 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, */ p->scx.flags &= ~SCX_TASK_IMMED; + /* + * A task reenqueued too many times without running means the scheduler + * keeps re-deciding a placement it can't honor, e.g. re-inserting to a + * cid it lacks caps on. Eject the owning scheduler and strand the task + * to be picked up during sched exit. + */ + if (enq_flags & SCX_ENQ_REENQ) { + if (++p->scx.reenq_cnt > 1) + __scx_add_event(sch, SCX_EV_REENQ_REPEAT, 1); + + if (unlikely(p->scx.reenq_cnt > SCX_REENQ_MAX_REPEAT)) { + __scx_exit(sch, SCX_EXIT_ERROR_REENQ, 0, cpu_of(rq), + "%s[%d] reenqueued %u times without running", + p->comm, p->pid, p->scx.reenq_cnt); + return; + } + } + /* * If !scx_rq_online(), we already told the BPF scheduler that the CPU * is offline and are just running the hotplug path. Don't bother the @@ -2025,8 +2043,10 @@ static void clr_task_runnable(struct task_struct *p, bool reset_runnable_at) { list_del_init(&p->scx.runnable_node); WRITE_ONCE(p->scx.runnable_cpu, -1); - if (reset_runnable_at) + if (reset_runnable_at) { p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT; + p->scx.reenq_cnt = 0; + } } static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_flags) @@ -3669,6 +3689,7 @@ static void scx_disable_task(struct scx_sched *sch, struct task_struct *p) */ p->scx.dsq_vtime = 0; set_task_slice(p, 0); + p->scx.reenq_cnt = 0; /* * Verify the task is not in BPF scheduler's custody. If flag @@ -4066,8 +4087,8 @@ static void process_ddsp_deferred_locals(struct rq *rq) * Reenqueued tasks go through ops.enqueue() with %SCX_ENQ_REENQ | * %SCX_TASK_REENQ_IMMED. If the BPF scheduler dispatches back to the same local * DSQ with %SCX_ENQ_IMMED while the CPU is still unavailable, this triggers - * another reenq cycle. Repetitions are bounded by %SCX_REENQ_LOCAL_MAX_REPEAT - * in process_deferred_reenq_locals(). + * another reenq cycle. Repetitions are bounded by %SCX_REENQ_MAX_REPEAT in + * scx_do_enqueue_task(), which ejects the task's owning scheduler. */ static bool local_task_should_reenq(struct rq *rq, struct task_struct *p, u64 *reenq_flags, u32 *reason) @@ -4175,14 +4196,16 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) static void process_deferred_reenq_locals(struct rq *rq) { - u64 seq = ++rq->scx.deferred_reenq_locals_seq; - lockdep_assert_rq_held(rq); + /* + * A task can be re-queued within this loop when a reenqueued task + * bounces straight back to the local DSQ. That recursion is bounded by + * the per-task reenqueue cap in scx_do_enqueue_task(). + */ while (true) { struct scx_sched *sch; u64 reenq_flags; - bool skip = false; scoped_guard (raw_spinlock, &rq->scx.deferred_reenq_lock) { struct scx_deferred_reenq_local *drl = @@ -4201,27 +4224,12 @@ static void process_deferred_reenq_locals(struct rq *rq) reenq_flags = drl->flags; WRITE_ONCE(drl->flags, 0); list_del_init(&drl->node); - - if (likely(drl->seq != seq)) { - drl->seq = seq; - drl->cnt = 0; - } else { - if (unlikely(++drl->cnt > SCX_REENQ_LOCAL_MAX_REPEAT)) { - scx_error(sch, "SCX_ENQ_REENQ on SCX_DSQ_LOCAL repeated %u times", - drl->cnt); - skip = true; - } - - __scx_add_event(sch, SCX_EV_REENQ_LOCAL_REPEAT, 1); - } } - if (!skip) { - /* see schedule_dsq_reenq() */ - smp_mb(); + /* see schedule_dsq_reenq() */ + smp_mb(); - reenq_local(sch, rq, reenq_flags); - } + reenq_local(sch, rq, reenq_flags); } } @@ -5941,6 +5949,8 @@ static const char *scx_exit_reason(enum scx_exit_kind kind) return "scx_bpf_error"; case SCX_EXIT_ERROR_STALL: return "runnable task stall"; + case SCX_EXIT_ERROR_REENQ: + return "reenqueue limit"; default: return ""; } diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 39dddcbb3b7d..144e962d3342 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -56,6 +56,7 @@ enum scx_exit_kind { SCX_EXIT_ERROR = 1024, /* runtime error, error msg contains details */ SCX_EXIT_ERROR_BPF, /* ERROR but triggered through scx_bpf_error() */ SCX_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */ + SCX_EXIT_ERROR_REENQ, /* task hit reenqueue limit without running */ }; /* @@ -1119,15 +1120,13 @@ struct scx_event_stats { s64 SCX_EV_REENQ_IMMED; /* - * The number of times a reenq of local DSQ caused another reenq of - * local DSQ. This can happen when %SCX_ENQ_IMMED races against a higher - * priority class task even if the BPF scheduler always satisfies the - * prerequisites for %SCX_ENQ_IMMED at the time of enqueue. However, - * that scenario is very unlikely and this count going up regularly - * indicates that the BPF scheduler is handling %SCX_ENQ_REENQ - * incorrectly causing recursive reenqueues. + * The number of times a reenqueue (%SCX_ENQ_REENQ) led to another + * reenqueue without the task running in between. This count climbing + * rapidly indicates that the BPF scheduler keeps re-deciding placements + * it can't honor. A single task reenqueued more than + * %SCX_REENQ_MAX_REPEAT times gets its owning scheduler ejected. */ - s64 SCX_EV_REENQ_LOCAL_REPEAT; + s64 SCX_EV_REENQ_REPEAT; /* * Total number of times a task's time slice was refilled with the @@ -1221,7 +1220,7 @@ struct scx_event_stats { SCX_EVENT(SCX_EV_ENQ_SKIP_EXITING); \ SCX_EVENT(SCX_EV_ENQ_SKIP_MIGRATION_DISABLED); \ SCX_EVENT(SCX_EV_REENQ_IMMED); \ - SCX_EVENT(SCX_EV_REENQ_LOCAL_REPEAT); \ + SCX_EVENT(SCX_EV_REENQ_REPEAT); \ SCX_EVENT(SCX_EV_REFILL_SLICE_DFL); \ SCX_EVENT(SCX_EV_SLICE_CLAMPED); \ SCX_EVENT(SCX_EV_SLICE_DENIED); \ @@ -1260,8 +1259,6 @@ struct scx_dsp_ctx { struct scx_deferred_reenq_local { struct list_head node; u64 flags; - u64 seq; - u32 cnt; }; struct scx_sched_pcpu { diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 824fe35f00ee..7265f32bd8f5 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -319,9 +319,9 @@ bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p) * Drain @rq->scx.reject_dsq, reenqueueing each task so the BPF re-decides * from p->scx.reenq_reason_*. * - * A task can be re-rejected repeatedly, and there's no repeat limit here. - * Rejection can't happen for root, and sub-scheds can be safely ejected after - * triggering the stall watchdog. + * A task can be re-rejected repeatedly. The reenqueue is bounded per task in + * scx_do_enqueue_task(), which ejects the owning sub past SCX_REENQ_MAX_REPEAT. + * Rejection can't happen for root. */ void scx_reenq_reject(struct rq *rq) { diff --git a/kernel/sched/ext/types.h b/kernel/sched/ext/types.h index a1a05820725e..b94ddee21c57 100644 --- a/kernel/sched/ext/types.h +++ b/kernel/sched/ext/types.h @@ -41,7 +41,7 @@ enum scx_consts { SCX_BYPASS_LB_MIN_DELTA_DIV = 4, SCX_BYPASS_LB_BATCH = 256, - SCX_REENQ_LOCAL_MAX_REPEAT = 256, + SCX_REENQ_MAX_REPEAT = 256, SCX_SUB_MAX_DEPTH = 4, }; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 64d79e9efc3d..c0cb879d75f0 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -823,7 +823,6 @@ struct scx_rq { struct list_head sched_pcpus_to_kick; /* see kick_cpus_irq_workfn() */ raw_spinlock_t deferred_reenq_lock; - u64 deferred_reenq_locals_seq; struct list_head deferred_reenq_locals; /* scheds requesting reenq of local DSQ */ struct list_head deferred_reenq_users; /* user DSQs requesting reenq */ struct balance_callback deferred_bal_cb; -- cgit v1.2.3 From f883dbb64ca53f75d9006d1e73180c9d9ecfc9a2 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 27 Jul 2026 11:20:23 -1000 Subject: sched_ext: Make exit claiming lock-free scx_claim_exit() claims descendants' exits by walking the subtree under scx_sched_lock, making exit claiming, and thus scx_error(), unusable from NMI and from under scx_sched_lock. However, kfuncs raising errors can run from NMI-attached BPF progs, the hardlockup handler runs in NMI, and scx_link_sched() wants to report failures under the lock. The walk does two things with different urgencies: ->aborting must be asserted synchronously to break IRQs-off dispatch-path live-locks, while the descendants' exit_kind claims can happen later. Split them: sweep ->aborting locklessly under RCU to unwedge the system and defer the locked SCX_EXIT_PARENT walk to a new irq_work, both of which are NMI-safe. The sweep stores each node's ->aborting and then reads its children list while scx_link_sched() inserts and then checks the parent's ->aborting, the two sides paired by full barriers - one side always sees the other. A link that sees ->aborting undoes its insert and fails. As the undo's list_del_rcu() leaves ->sibling non-empty, list_empty() can no longer identify a never-linked sched during teardown - add sch->linked instead. trace_sched_ext_exit can now fire from NMI and is called after the ->aborting stores so that its callbacks don't hold up live-lock recovery. The exit backtrace is skipped for NMI exits as stack_trace_save()'s NMI-safety is arch-dependent and undocumented. v2: Move trace_sched_ext_exit() after the ->aborting stores (Andrea). Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 110 ++++++++++++++++++++++++++++---------------- kernel/sched/ext/internal.h | 2 + kernel/sched/ext/sub.c | 12 +++-- 3 files changed, 79 insertions(+), 45 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index ff9151232f58..dfa624b8607e 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -5035,6 +5035,7 @@ static void scx_sched_free_rcu_work(struct work_struct *work) struct scx_dispatch_q *dsq; int cpu, node; + irq_work_sync(&sch->propagate_exit_irq_work); irq_work_sync(&sch->disable_irq_work); kthread_destroy_worker(sch->helper); timer_shutdown_sync(&sch->bypass_lb_timer); @@ -6003,18 +6004,6 @@ s32 scx_link_sched(struct scx_sched *sch) struct scx_sched *parent = scx_parent(sch); if (parent) { - /* - * scx_claim_exit() propagates exit_kind transition to - * its sub-scheds while holding scx_sched_lock - either - * we can see the parent's non-NONE exit_kind or the - * parent can shoot us down. - */ - if (atomic_read(&parent->exit_kind) != SCX_EXIT_NONE) { - err_msg = "parent disabled"; - ret = -ENOENT; - break; - } - /* * Bypass state is spread across per-cpu flags and a * depth count, so inheriting it is tricky and has no @@ -6034,6 +6023,23 @@ s32 scx_link_sched(struct scx_sched *sch) } list_add_tail_rcu(&sch->sibling, &parent->children); + + /* + * Pairs with the mb after the ->aborting assertion in + * scx_claim_exit(). Either we see ->aborting and back + * out, or the exit path sees us and exits us. + */ + smp_mb(); + if (unlikely(READ_ONCE(parent->aborting))) { + rhashtable_remove_fast(&scx_sched_hash, &sch->hash_node, + scx_sched_hash_params); + list_del_rcu(&sch->sibling); + err_msg = "parent disabled"; + ret = -ENOENT; + break; + } + + sch->linked = true; } #endif /* CONFIG_EXT_SUB_SCHED */ @@ -6057,10 +6063,11 @@ void scx_unlink_sched(struct scx_sched *sch) { scoped_guard(raw_spinlock_irq, &scx_sched_lock) { #ifdef CONFIG_EXT_SUB_SCHED - if (scx_parent(sch)) { + if (sch->linked) { rhashtable_remove_fast(&scx_sched_hash, &sch->hash_node, scx_sched_hash_params); list_del_rcu(&sch->sibling); + sch->linked = false; } #endif /* CONFIG_EXT_SUB_SCHED */ list_del_rcu(&sch->all); @@ -6270,12 +6277,36 @@ done: scx_bypass(sch, false); } +/** + * scx_propagate_exit_irq_workfn - Claim SCX_EXIT_PARENT on the exiting subtree + * @irq_work: &scx_sched.propagate_exit_irq_work + * + * Queued by scx_claim_exit() after a non-PARENT claim. Claims SCX_EXIT_PARENT + * on each descendant, giving every one its own disable work - most of disabling + * is serialized but ops.exit() can take arbitrarily long and running them in + * separate helper kthreads parallelizes it. No recursion as only non-PARENT + * claims propagate. + */ +static void scx_propagate_exit_irq_workfn(struct irq_work *irq_work) +{ + struct scx_sched *sch = container_of(irq_work, struct scx_sched, + propagate_exit_irq_work); + struct scx_sched *pos; + + scoped_guard (raw_spinlock_irqsave, &scx_sched_lock) { + scx_for_each_descendant_pre(pos, sch) + scx_disable(pos, SCX_EXIT_PARENT); + } +} + /* * Claim the exit on @sch. The caller must ensure that the helper kthread work * is kicked before the current task can be preempted. Once exit_kind is * claimed, scx_error() can no longer trigger, so if the current task gets * preempted and the BPF scheduler fails to schedule it back, the helper work * will never be kicked and the whole system can wedge. + * + * Lock-free and safe to call from any context including NMI. */ static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind) { @@ -6289,37 +6320,31 @@ static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind) if (!atomic_try_cmpxchg(&sch->exit_kind, &none, kind)) return false; - /* - * Some CPUs may be trapped in the dispatch paths. Set the aborting - * flag to break potential live-lock scenarios, ensuring we can - * successfully reach scx_bypass(). - */ - WRITE_ONCE(sch->aborting, true); - - trace_sched_ext_exit(sch, kind); + if (kind == SCX_EXIT_PARENT) { + /* an ancestor is already sweeping the subtree */ + WRITE_ONCE(sch->aborting, true); + } else { + struct scx_sched *pos; - /* - * Propagate exits to descendants immediately. Each has a dedicated - * helper kthread and can run in parallel. While most of disabling is - * serialized, running them in separate threads allows parallelizing - * ops.exit(), which can take arbitrarily long prolonging bypass mode. - * - * To guarantee forward progress, this propagation must be in-line so - * that ->aborting is synchronously asserted for all sub-scheds. The - * propagation is also the interlocking point against sub-sched - * attachment. See scx_link_sched(). - * - * This doesn't cause recursions as propagation only takes place for - * non-propagation exits. - */ - if (kind != SCX_EXIT_PARENT) { - scoped_guard (raw_spinlock_irqsave, &scx_sched_lock) { - struct scx_sched *pos; + /* + * CPUs may be live-locked in the dispatch paths of @sch or its + * descendants, which ->aborting breaks. Sweep the subtree + * locklessly so that this works from NMI. smp_store_mb() orders + * each node's ->aborting store before its children are walked - + * either we see a racing scx_link_sched() on ->children or it + * sees ->aborting. + */ + scoped_guard (rcu) { scx_for_each_descendant_pre(pos, sch) - scx_disable(pos, SCX_EXIT_PARENT); + smp_store_mb(pos->aborting, true); } + + irq_work_queue(&sch->propagate_exit_irq_work); } + /* fired after ->aborting is set so callbacks can't delay recovery */ + trace_sched_ext_exit(sch, kind); + return true; } @@ -6748,7 +6773,11 @@ bool scx_vexit(struct scx_sched *sch, ei->exit_code = exit_code; #ifdef CONFIG_STACKTRACE - if (kind >= SCX_EXIT_ERROR) + /* + * stack_trace_save()'s NMI-safety is arch-dependent and undocumented. + * Skip the backtrace when exiting from NMI. + */ + if (kind >= SCX_EXIT_ERROR && !in_nmi()) ei->bt_len = stack_trace_save(ei->bt, SCX_EXIT_BT_LEN, 1); #endif vscnprintf(ei->msg, SCX_EXIT_MSG_LEN, fmt, args); @@ -6918,6 +6947,7 @@ struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, sch->slice_dfl = SCX_SLICE_DFL; atomic_set(&sch->exit_kind, SCX_EXIT_NONE); sch->disable_irq_work = IRQ_WORK_INIT_HARD(scx_disable_irq_workfn); + sch->propagate_exit_irq_work = IRQ_WORK_INIT_HARD(scx_propagate_exit_irq_workfn); kthread_init_work(&sch->disable_work, scx_disable_workfn); timer_setup(&sch->bypass_lb_timer, scx_bypass_lb_timerfn, 0); diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 144e962d3342..d9ab6571d7ad 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1559,6 +1559,7 @@ struct scx_sched { char *cgrp_path; struct kset *sub_kset; + bool linked; /* on ->children, see scx_link_sched() */ bool sub_attached; #endif /* CONFIG_EXT_SUB_SCHED */ @@ -1577,6 +1578,7 @@ struct scx_sched { struct kthread_worker *helper; struct irq_work disable_irq_work; struct kthread_work disable_work; + struct irq_work propagate_exit_irq_work; /* see scx_claim_exit() */ struct timer_list bypass_lb_timer; cpumask_var_t bypass_lb_donee_cpumask; cpumask_var_t bypass_lb_resched_cpumask; diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 7265f32bd8f5..7a5eb3a6f488 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -1067,12 +1067,14 @@ void scx_sub_disable(struct scx_sched *sch) scx_cgroup_lock(); /* - * An enable that failed before scx_link_sched() never owned a cgroup or - * task and won't be waited on by an ancestor's drain_descendants(). - * Nothing to reparent and walking the tasks can misbehave as the task - * ownership invariant (either owned by self or parent) does not hold. + * An enable that failed before scx_link_sched() succeeded never owned a + * cgroup or task and won't be waited on by an ancestor's + * drain_descendants(). Nothing to reparent and walking the tasks can + * misbehave as the task ownership invariant (either owned by self or + * parent) does not hold. ->sibling can't identify this case - an undone + * link leaves it non-empty. */ - if (list_empty(&sch->sibling)) + if (!sch->linked) goto dump; set_cgroup_sched(sch_cgroup(sch), parent); -- cgit v1.2.3 From 1bf623ebd50315260ce6da9601e4cd3e79659152 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 27 Jul 2026 11:20:32 -1000 Subject: sched_ext: Format bstr exit messages after claiming the exit The bstr exit kfuncs format the message into a shared static buffer under a raw spinlock before initiating the exit. The lock can't be taken from NMI and needlessly serializes all bstr exits system-wide. Now that exit claiming is lock-free, reverse the order: claim the exit first and format directly into the exit_info message buffer which the claim winner owns exclusively. The new scx_exit_bstr() implements the sequence, replacing scx_bstr_format(), and the shared buffer and lock are deleted; the formatter itself is what bpf_trace_printk() already runs from NMI. scx_prog_sched() callers were relying on the lock for RCU protection, which is now provided explicitly. A malformed format no longer changes or fails the requested operation: scx_bpf_exit_bstr() keeps its graceful exit kind and scx_bpf_sub_kill_bstr() still kills the child, with a fallback message carrying the formatting errno, while the sched that supplied the bad format is aborted for its bug. Before this and the previous patch, an "any" category kfunc called from NMI context could trigger scx_error() and deadlock - e.g. a tracing prog attached to a function running in NMI calling scx_bpf_dsq_peek() on a non-existent DSQ would try to grab scx_sched_lock, which may be held by the interrupted CPU. This and the previous patch fix the deadlock: scx_error() and the bstr exit kfuncs, and thus scx_bpf_error() and scx_bpf_exit(), are now safe to call from any context including NMI. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 90 ++++++++++++++++++++++++++++++--------------- kernel/sched/ext/internal.h | 7 ++-- kernel/sched/ext/sub.c | 7 +--- 3 files changed, 64 insertions(+), 40 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index dfa624b8607e..cc526776b3cc 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -196,9 +196,6 @@ static const struct rhashtable_params dsq_hash_params = { static LLIST_HEAD(dsqs_to_free); -DEFINE_RAW_SPINLOCK(scx_exit_bstr_buf_lock); -struct scx_bstr_buf scx_exit_bstr_buf; - /* ops debug dump */ static DEFINE_RAW_SPINLOCK(scx_dump_lock); @@ -6760,17 +6757,12 @@ static void scx_disable_irq_workfn(struct irq_work *irq_work) kthread_queue_work(sch->helper, &sch->disable_work); } -bool scx_vexit(struct scx_sched *sch, - enum scx_exit_kind kind, s64 exit_code, s32 exit_cpu, - const char *fmt, va_list args) +/* finish exit_info and kick the disable work, ei->msg must already be set */ +static void scx_finish_exit(struct scx_sched *sch, enum scx_exit_kind kind, + s64 exit_code, s32 exit_cpu) { struct scx_exit_info *ei = sch->exit_info; - guard(preempt)(); - - if (!scx_claim_exit(sch, kind)) - return false; - ei->exit_code = exit_code; #ifdef CONFIG_STACKTRACE /* @@ -6780,8 +6772,6 @@ bool scx_vexit(struct scx_sched *sch, if (kind >= SCX_EXIT_ERROR && !in_nmi()) ei->bt_len = stack_trace_save(ei->bt, SCX_EXIT_BT_LEN, 1); #endif - vscnprintf(ei->msg, SCX_EXIT_MSG_LEN, fmt, args); - /* * Set ei->kind and ->reason for scx_dump_state(). They'll be set again * in scx_disable_workfn(). @@ -6791,6 +6781,22 @@ bool scx_vexit(struct scx_sched *sch, ei->exit_cpu = exit_cpu; irq_work_queue(&sch->disable_irq_work); +} + +bool scx_vexit(struct scx_sched *sch, + enum scx_exit_kind kind, s64 exit_code, s32 exit_cpu, + const char *fmt, va_list args) +{ + struct scx_exit_info *ei = sch->exit_info; + + guard(preempt)(); + + if (!scx_claim_exit(sch, kind)) + return false; + + vscnprintf(ei->msg, SCX_EXIT_MSG_LEN, fmt, args); + + scx_finish_exit(sch, kind, exit_code, exit_cpu); return true; } @@ -9604,12 +9610,38 @@ static s32 __bstr_format(struct scx_sched *sch, u64 *data_buf, char *line_buf, return ret; } -__printf(3, 0) -s32 scx_bstr_format(struct scx_sched *sch, struct scx_bstr_buf *buf, - char *fmt, unsigned long long *data, u32 data__sz) +/* + * Exit @sch with the reason formatted from a BPF-supplied bstr format. The exit + * is claimed first and the reason is formatted directly into the winner-owned + * exit_info buffer, which allows use from any context including NMI. + * + * @fmt_blame is the sched blamed for formatting failures through the + * scx_error() calls in __bstr_format() and differs from @sch when a parent + * supplies the kill reason for a child. A formatting failure doesn't revert the + * claim - @sch still exits with the claimed kind and a fallback message. + */ +__printf(5, 0) +bool scx_exit_bstr(struct scx_sched *sch, enum scx_exit_kind kind, + s64 exit_code, struct scx_sched *fmt_blame, char *fmt, + unsigned long long *data, u32 data__sz) { - return __bstr_format(sch, buf->data, buf->line, sizeof(buf->line), - fmt, data, data__sz); + struct scx_exit_info *ei = sch->exit_info; + u64 data_buf[MAX_BPRINTF_VARARGS]; + s32 ret; + + guard(preempt)(); + + if (!scx_claim_exit(sch, kind)) + return false; + + ret = __bstr_format(fmt_blame, data_buf, ei->msg, SCX_EXIT_MSG_LEN, + fmt, data, data__sz); + if (ret < 0) + scnprintf(ei->msg, SCX_EXIT_MSG_LEN, + "exit message formatting failed (%d)", ret); + + scx_finish_exit(sch, kind, exit_code, raw_smp_processor_id()); + return true; } __bpf_kfunc_start_defs(); @@ -9631,14 +9663,13 @@ __bpf_kfunc void scx_bpf_exit_bstr(s64 exit_code, char *fmt, const struct bpf_prog_aux *aux) { struct scx_sched *sch; - unsigned long flags; - raw_spin_lock_irqsave(&scx_exit_bstr_buf_lock, flags); + guard(rcu)(); + sch = scx_prog_sched(aux); - if (likely(sch) && - scx_bstr_format(sch, &scx_exit_bstr_buf, fmt, data, data__sz) >= 0) - scx_exit(sch, SCX_EXIT_UNREG_BPF, exit_code, "%s", scx_exit_bstr_buf.line); - raw_spin_unlock_irqrestore(&scx_exit_bstr_buf_lock, flags); + if (likely(sch)) + scx_exit_bstr(sch, SCX_EXIT_UNREG_BPF, exit_code, sch, fmt, + data, data__sz); } /** @@ -9656,14 +9687,13 @@ __bpf_kfunc void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data__sz, const struct bpf_prog_aux *aux) { struct scx_sched *sch; - unsigned long flags; - raw_spin_lock_irqsave(&scx_exit_bstr_buf_lock, flags); + guard(rcu)(); + sch = scx_prog_sched(aux); - if (likely(sch) && - scx_bstr_format(sch, &scx_exit_bstr_buf, fmt, data, data__sz) >= 0) - scx_exit(sch, SCX_EXIT_ERROR_BPF, 0, "%s", scx_exit_bstr_buf.line); - raw_spin_unlock_irqrestore(&scx_exit_bstr_buf_lock, flags); + if (likely(sch)) + scx_exit_bstr(sch, SCX_EXIT_ERROR_BPF, 0, sch, fmt, data, + data__sz); } /** diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index d9ab6571d7ad..5b73ffb0251a 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -2004,15 +2004,14 @@ struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, int scx_validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops); int scx_sched_sysfs_add(struct scx_sched *sch); bool scx_is_descendant(struct scx_sched *sch, struct scx_sched *ancestor); -__printf(3, 0) s32 scx_bstr_format(struct scx_sched *sch, struct scx_bstr_buf *buf, - char *fmt, unsigned long long *data, u32 data__sz); +__printf(5, 0) bool scx_exit_bstr(struct scx_sched *sch, enum scx_exit_kind kind, + s64 exit_code, struct scx_sched *fmt_blame, + char *fmt, unsigned long long *data, u32 data__sz); extern raw_spinlock_t scx_sched_lock; extern struct mutex scx_enable_mutex; extern struct percpu_rw_semaphore scx_fork_rwsem; extern bool scx_cgroup_enabled; -extern raw_spinlock_t scx_exit_bstr_buf_lock; -extern struct scx_bstr_buf scx_exit_bstr_buf; #ifdef CONFIG_EXT_SUB_SCHED extern const struct rhashtable_params scx_sched_hash_params; extern struct rhashtable scx_sched_hash; diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 7a5eb3a6f488..daf7fbd3d0c3 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -2155,7 +2155,6 @@ __bpf_kfunc s32 scx_bpf_sub_kill_bstr(u64 cgroup_id, char *fmt, const struct bpf_prog_aux *aux) { struct scx_sched *parent, *child; - s32 ret; guard(rcu)(); @@ -2178,11 +2177,7 @@ __bpf_kfunc s32 scx_bpf_sub_kill_bstr(u64 cgroup_id, char *fmt, return -EINVAL; } - guard(raw_spinlock_irqsave)(&scx_exit_bstr_buf_lock); - ret = scx_bstr_format(parent, &scx_exit_bstr_buf, fmt, data, data__sz); - if (ret < 0) - return ret; - scx_exit(child, SCX_EXIT_PARENT_KILL, 0, "%s", scx_exit_bstr_buf.line); + scx_exit_bstr(child, SCX_EXIT_PARENT_KILL, 0, parent, fmt, data, data__sz); return 0; } -- cgit v1.2.3 From e06ece82d7b078b511d36fbaca46df231c647e16 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 27 Jul 2026 11:20:32 -1000 Subject: sched_ext: Report NMI kicks with scx_error() The per-cpu kick lists are protected by IRQ masking which doesn't stop NMIs, so scx_bpf_kick_cpu() from NMI silently drops the kick after a one-time warning. A dropped kick can leave a CPU idle when the scheduler believes it was woken, which is a correctness problem for the scheduler even if the kernel is fine. Now that scx_error() works from NMI, abort the scheduler instead so that the bug is surfaced deterministically. The warned_nmi_kick tracking is no longer needed. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 6 +----- kernel/sched/ext/internal.h | 1 - 2 files changed, 1 insertion(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index cc526776b3cc..cff55fa3d58f 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -9219,11 +9219,7 @@ void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags) * not mask NMIs, so kicking from NMI could corrupt it and is unsupported. */ if (unlikely(in_nmi())) { - if (!sch->warned_nmi_kick) { - sch->warned_nmi_kick = true; - pr_warn("sched_ext: %s: scx_bpf_kick_cpu() from NMI ignored\n", - sch->ops.name); - } + scx_error(sch, "scx_bpf_kick_cpu() called from NMI"); return; } diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 5b73ffb0251a..7501ec28958b 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1543,7 +1543,6 @@ struct scx_sched { */ bool warned_zero_slice:1; bool warned_unassoc_progs:1; - bool warned_nmi_kick:1; struct list_head all; -- cgit v1.2.3 From 3c4b38064937a761ebbf85b1649e812db85eb59e Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 27 Jul 2026 11:20:32 -1000 Subject: sched_ext: Abort directly from the hardlockup handler scx_hardlockup() defers the abort to an irq_work because exit claiming used to take scx_sched_lock and couldn't run from NMI. The deferral is now unnecessary - claiming is NMI-safe and asserting ->aborting is exactly what breaks the live-locks that hard-lock CPUs. Call handle_lockup() directly and drop the irq_work. This also makes the self-detected case recoverable: the perf watchdog fires on the hard-locked CPU itself, where a queued irq_work never runs with IRQs off. Also fix the return value: %true used to be returned whenever sched_ext was loaded, suppressing the kernel's hardlockup report even when the abort was refused. Return %true only when this call initiated the abort. Fixes: bd2d76455b65 ("sched_ext: Defer scx_hardlockup() out of NMI") Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 35 +++++++++-------------------------- 1 file changed, 9 insertions(+), 26 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index cff55fa3d58f..f99c2b8557e6 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -5390,25 +5390,6 @@ void scx_softlockup(u32 dur_s) cpu, dur_s); } -/* - * scx_hardlockup() runs from NMI and eventually calls scx_claim_exit(), - * which takes scx_sched_lock. scx_sched_lock isn't NMI-safe and grabbing - * it from NMI context can lead to deadlocks. Defer via irq_work; the - * disable path runs off irq_work anyway. - */ -static atomic_t scx_hardlockup_cpu = ATOMIC_INIT(-1); - -static void scx_hardlockup_irq_workfn(struct irq_work *work) -{ - int cpu = atomic_xchg(&scx_hardlockup_cpu, -1); - - if (cpu >= 0 && handle_lockup(cpu, "hard lockup - CPU %d", cpu)) - printk_deferred(KERN_ERR "sched_ext: Hard lockup - CPU %d, disabling BPF scheduler\n", - cpu); -} - -static DEFINE_IRQ_WORK(scx_hardlockup_irq_work, scx_hardlockup_irq_workfn); - /** * scx_hardlockup - sched_ext hardlockup handler * @cpu: the target CPU @@ -5418,19 +5399,21 @@ static DEFINE_IRQ_WORK(scx_hardlockup_irq_work, scx_hardlockup_irq_workfn); * Try kicking out the current scheduler in an attempt to recover the system to * a good state before taking more drastic actions. * - * Queues an irq_work; the handle_lockup() call happens in IRQ context (see - * scx_hardlockup_irq_workfn). + * Called from NMI. Aborting the scheduler sets ->aborting throughout the + * hierarchy before returning, which is what breaks the dispatch-path live-locks + * that can hard-lock CPUs. * - * Returns %true if sched_ext is enabled and the work was queued, %false - * otherwise. + * Returns %true if sched_ext is enabled and abort was initiated, which may + * resolve the lockup. %false if sched_ext is not enabled or abort was already + * initiated by someone else. */ bool scx_hardlockup(int cpu) { - if (!rcu_access_pointer(scx_root)) + if (!handle_lockup(cpu, "hard lockup - CPU %d", cpu)) return false; - atomic_cmpxchg(&scx_hardlockup_cpu, -1, cpu); - irq_work_queue(&scx_hardlockup_irq_work); + printk_deferred(KERN_ERR "sched_ext: Hard lockup - CPU %d, disabling BPF scheduler\n", + cpu); return true; } -- cgit v1.2.3 From ee7aece608178e322ba150a73613ebace31df885 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 27 Jul 2026 11:20:32 -1000 Subject: sched_ext: Report scx_link_sched() failures inline scx_link_sched() carries each failure out of the locked section through err_msg and ret because scx_error() used to take scx_sched_lock and couldn't be called under it. That restriction is gone, so report each failure at the site it's detected and return directly. The scx_error() here claims the exit on the sched being linked, which has no descendants yet, and the locked propagation walk is deferred, so nothing reacquires scx_sched_lock inline. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 29 +++++++++-------------------- 1 file changed, 9 insertions(+), 20 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index f99c2b8557e6..312938179e52 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -5975,31 +5975,30 @@ static void refresh_watchdog(void) s32 scx_link_sched(struct scx_sched *sch) { - const char *err_msg = ""; - s32 ret = 0; - scoped_guard(raw_spinlock_irqsave, &scx_bypass_lock) /* for the parent bypass check */ scoped_guard(raw_spinlock, &scx_sched_lock) { #ifdef CONFIG_EXT_SUB_SCHED struct scx_sched *parent = scx_parent(sch); if (parent) { + s32 ret; + /* * Bypass state is spread across per-cpu flags and a * depth count, so inheriting it is tricky and has no * valid use case. Refuse it. */ if (READ_ONCE(parent->bypass_depth)) { - err_msg = "parent bypassing"; - ret = -EBUSY; - break; + scx_error(sch, "parent bypassing (%d)", -EBUSY); + return -EBUSY; } ret = rhashtable_lookup_insert_fast(&scx_sched_hash, &sch->hash_node, scx_sched_hash_params); if (ret) { - err_msg = "failed to insert into scx_sched_hash"; - break; + scx_error(sch, "failed to insert into scx_sched_hash (%d)", + ret); + return ret; } list_add_tail_rcu(&sch->sibling, &parent->children); @@ -6014,9 +6013,8 @@ s32 scx_link_sched(struct scx_sched *sch) rhashtable_remove_fast(&scx_sched_hash, &sch->hash_node, scx_sched_hash_params); list_del_rcu(&sch->sibling); - err_msg = "parent disabled"; - ret = -ENOENT; - break; + scx_error(sch, "parent disabled (%d)", -ENOENT); + return -ENOENT; } sch->linked = true; @@ -6026,15 +6024,6 @@ s32 scx_link_sched(struct scx_sched *sch) list_add_tail_rcu(&sch->all, &scx_sched_all); } - /* - * scx_error() takes scx_sched_lock via scx_claim_exit(), so it must run after - * the guard above is released. - */ - if (ret) { - scx_error(sch, "%s (%d)", err_msg, ret); - return ret; - } - refresh_watchdog(); return 0; } -- cgit v1.2.3 From 680e0718b9f0b9f3067b2c3947088c2878f55442 Mon Sep 17 00:00:00 2001 From: Liang Luo Date: Fri, 31 Jul 2026 10:30:04 +0800 Subject: sched_ext: Fix stale @cgroup_id in sched_ext_ops kernel-doc The kernel-doc comment for sched_ext_ops::sub_cgroup_id uses the old @cgroup_id name, which no longer matches the struct member. This produces two kernel-doc warnings: Warning: struct member sub_cgroup_id not described in sched_ext_ops Warning: Excess struct member cgroup_id description in sched_ext_ops Update the @param name to match the actual member. Signed-off-by: Liang Luo Signed-off-by: Tejun Heo --- kernel/sched/ext/internal.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 7501ec28958b..741d8f6b5199 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -925,8 +925,8 @@ struct sched_ext_ops { u32 cid_shard_size; /** - * @cgroup_id: When >1, attach the scheduler as a sub-scheduler on the - * specified cgroup. + * @sub_cgroup_id: When >1, attach the scheduler as a sub-scheduler + * on the specified cgroup. */ u64 sub_cgroup_id; -- cgit v1.2.3 From c5b9316cf3d5371d53022352c7c9d248e14b801f Mon Sep 17 00:00:00 2001 From: Liang Luo Date: Tue, 28 Jul 2026 14:09:55 +0800 Subject: sched_ext: Set errno on ENABLING -> ENABLED transition failure If the SCX_ENABLING -> SCX_ENABLED cmpxchg at the tail of scx_root_enable_workfn() fails, the function jumps to err_disable without setting ret. At that point ret still holds the return value of the last successful __scx_init_task() call, which is 0, so the err_disable fallback reports the meaningless message: scx_root_enable() failed (0) Set ret = -EBUSY, consistent with the other enable-state guards at the top of the same function, so the fallback always reports a real errno. Signed-off-by: Liang Luo Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 1 + 1 file changed, 1 insertion(+) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 312938179e52..6a3e94707c8b 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -7520,6 +7520,7 @@ static void scx_root_enable_workfn(struct kthread_work *work) if (!scx_tryset_enable_state(SCX_ENABLED, SCX_ENABLING)) { WARN_ON_ONCE(atomic_read(&sch->exit_kind) == SCX_EXIT_NONE); + ret = -EBUSY; goto err_disable; } -- cgit v1.2.3 From ba190ed3f47f0e177e4be525d4a6330bd2fbd1f1 Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Mon, 3 Aug 2026 08:08:12 +0200 Subject: sched_ext: Initialize idle masks as busy The built-in idle masks are reset with all online CPUs marked idle before sched_ext is enabled. Busy CPUs can therefore be incorrectly advertised as idle until their next idle transition. Initialize the masks empty so that the initial state is conservative. When bypass is lifted, every CPU is rescheduled and idle-to-idle re-picks populate the masks with CPUs that are actually idle. Later idle transitions keep the masks up to date. Suggested-by: Tejun Heo Signed-off-by: Andrea Righi Reviewed-by: Kuba Piecuch Signed-off-by: Tejun Heo --- kernel/sched/ext/idle.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/idle.c b/kernel/sched/ext/idle.c index 3e9d6a44bf43..33bd51ab3ced 100644 --- a/kernel/sched/ext/idle.c +++ b/kernel/sched/ext/idle.c @@ -838,20 +838,20 @@ static void reset_idle_masks(struct sched_ext_ops *ops) int node; /* - * Consider all online cpus idle. Should converge to the actual state - * quickly. + * Start with all CPUs marked busy. The idle masks are populated when + * bypass is lifted and each idle CPU is forced through an idle re-pick. + * This may temporarily omit idle CPUs but never advertises a busy CPU as + * idle. */ if (!(ops->flags & SCX_OPS_BUILTIN_IDLE_PER_NODE)) { - cpumask_copy(idle_cpumask(NUMA_NO_NODE)->cpu, cpu_online_mask); - cpumask_copy(idle_cpumask(NUMA_NO_NODE)->smt, cpu_online_mask); + cpumask_clear(idle_cpumask(NUMA_NO_NODE)->cpu); + cpumask_clear(idle_cpumask(NUMA_NO_NODE)->smt); return; } for_each_node(node) { - const struct cpumask *node_mask = cpumask_of_node(node); - - cpumask_and(idle_cpumask(node)->cpu, cpu_online_mask, node_mask); - cpumask_and(idle_cpumask(node)->smt, cpu_online_mask, node_mask); + cpumask_clear(idle_cpumask(node)->cpu); + cpumask_clear(idle_cpumask(node)->smt); } } -- cgit v1.2.3 From 8b3b8522c9139c18b8dbbafbeb0c903609e5a27d Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 3 Aug 2026 11:00:31 -1000 Subject: sched_ext: Rename scx_local_or_reject_dsq() to scx_resolve_local_dsq() The following rescue execution addition gives the function a third possible destination, making a name that enumerates the outcomes a poor fit. Rename to the destination-neutral scx_resolve_local_dsq(). No functional changes. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 4 ++-- kernel/sched/ext/sub.c | 6 +++--- kernel/sched/ext/sub.h | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 6a3e94707c8b..8e5c7e45520e 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1507,7 +1507,7 @@ static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq, bool is_rq_owned = false; if (dsq->id == SCX_DSQ_LOCAL) { - dsq = scx_local_or_reject_dsq(sch, rq, p, &enq_flags); + dsq = scx_resolve_local_dsq(sch, rq, p, &enq_flags); is_rq_owned = true; } @@ -2279,7 +2279,7 @@ static void move_local_task_to_local_dsq(struct scx_sched *sch, struct scx_dispatch_q *src_dsq, struct rq *dst_rq) { - struct scx_dispatch_q *dst_dsq = scx_local_or_reject_dsq(sch, dst_rq, p, &enq_flags); + struct scx_dispatch_q *dst_dsq = scx_resolve_local_dsq(sch, dst_rq, p, &enq_flags); /* @dsq is locked and @p is on @dst_rq */ lockdep_assert_held(&src_dsq->lock); diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index daf7fbd3d0c3..2ea9a690e986 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -230,7 +230,7 @@ void scx_init_root_caps(struct scx_sched *sch) } /** - * scx_local_or_reject_dsq - Pick the local or reject DSQ for an insert + * scx_resolve_local_dsq - Pick the local or reject DSQ for an insert * @sch: enqueuing sub-sched * @rq: rq whose local DSQ @p targets * @p: task being inserted @@ -246,8 +246,8 @@ void scx_init_root_caps(struct scx_sched *sch) * to and run by its nearest non-bypassing ancestor. If root is bypassing, it * always holds all caps. */ -struct scx_dispatch_q *scx_local_or_reject_dsq(struct scx_sched *sch, struct rq *rq, - struct task_struct *p, u64 *enq_flags) +struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, + struct task_struct *p, u64 *enq_flags) { if (!scx_has_subs()) return &rq->scx.local_dsq; diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index 0019b75a2560..db449559bbe8 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -34,8 +34,8 @@ void scx_online_ecaps(struct rq *rq); void scx_offline_ecaps(struct rq *rq); void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu); void scx_discard_stale_ecaps_syncs(void); -struct scx_dispatch_q *scx_local_or_reject_dsq(struct scx_sched *sch, struct rq *rq, - struct task_struct *p, u64 *enq_flags); +struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, + struct task_struct *p, u64 *enq_flags); bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p); void scx_reenq_reject(struct rq *rq); @@ -84,7 +84,7 @@ static inline void scx_online_ecaps(struct rq *rq) {} static inline void scx_offline_ecaps(struct rq *rq) {} static inline void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu) {} static inline void scx_discard_stale_ecaps_syncs(void) {} -static inline struct scx_dispatch_q *scx_local_or_reject_dsq(struct scx_sched *sch, struct rq *rq, struct task_struct *p, u64 *enq_flags) { return &rq->scx.local_dsq; } +static inline struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, struct task_struct *p, u64 *enq_flags) { return &rq->scx.local_dsq; } static inline bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p) { return false; } static inline void scx_reenq_reject(struct rq *rq) {} static inline void scx_dec_has_subs(struct scx_sched *sch) {} -- cgit v1.2.3 From 2d091012a41ad3e29ca66afd3c6a160e485ba8ac Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 3 Aug 2026 11:00:39 -1000 Subject: sched_ext: Make several ext.c helpers available outside ext.c set_task_slice(), task_unlink_from_dsq(), move_local_task_to_local_dsq(), init_dsq() and dump_line() will be used outside ext.c. Add the scx_ prefix and declare them in internal.h. The scx_sched_all list will also be used outside ext.c, drop its static. No functional changes. v2: Declare scx_sched_all outside the CONFIG_EXT_SUB_SCHED block - the definition is unconditional. (sashiko AI) Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 172 +++++++++++++++++++++----------------------- kernel/sched/ext/internal.h | 8 +++ 2 files changed, 90 insertions(+), 90 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 8e5c7e45520e..4245a737592a 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -38,7 +38,7 @@ struct scx_sched __rcu *scx_root; * All scheds, writers must hold both scx_enable_mutex and scx_sched_lock. * Readers can hold either or rcu_read_lock(). */ -static LIST_HEAD(scx_sched_all); +LIST_HEAD(scx_sched_all); #ifdef CONFIG_EXT_SUB_SCHED const struct rhashtable_params scx_sched_hash_params = { @@ -900,7 +900,7 @@ struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter) * @kind: a kind of event to dump */ #define scx_dump_event(s, events, kind) do { \ - dump_line(&(s), "%40s: %16lld", #kind, (events)->kind); \ + scx_dump_line(&(s), "%40s: %16lld", #kind, (events)->kind); \ } while (0) @@ -1248,7 +1248,7 @@ static void set_task_slice_keep_oob(struct task_struct *p, u64 slice) } /* set @p's slice, superseding any pending out-of-band request */ -static void set_task_slice(struct task_struct *p, u64 slice) +void scx_set_task_slice(struct task_struct *p, u64 slice) { set_task_slice_keep_oob(p, slice); clear_task_slice_oob(p); @@ -1495,7 +1495,7 @@ static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq, if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr && rq->curr->sched_class == &ext_sched_class) { - set_task_slice(rq->curr, 0); + scx_set_task_slice(rq->curr, 0); resched_curr(rq); } } @@ -1637,8 +1637,7 @@ static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq, atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); } -static void task_unlink_from_dsq(struct task_struct *p, - struct scx_dispatch_q *dsq) +void scx_task_unlink_from_dsq(struct task_struct *p, struct scx_dispatch_q *dsq) { WARN_ON_ONCE(list_empty(&p->scx.dsq_list.node)); @@ -1695,7 +1694,7 @@ void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p) */ if (p->scx.holding_cpu < 0) { /* @p must still be on @dsq, dequeue */ - task_unlink_from_dsq(p, dsq); + scx_task_unlink_from_dsq(p, dsq); } else { /* * We're racing against dispatch_to_local_dsq() which already @@ -1722,7 +1721,7 @@ static void dispatch_dequeue_locked(struct task_struct *p, lockdep_assert_rq_held(task_rq(p)); lockdep_assert_held(&dsq->lock); - task_unlink_from_dsq(p, dsq); + scx_task_unlink_from_dsq(p, dsq); p->scx.dsq = NULL; } @@ -2236,7 +2235,7 @@ static void yield_task_scx(struct rq *rq) if (SCX_HAS_OP(sch, yield)) SCX_CALL_OP_2TASKS_RET(sch, yield, rq, p, NULL); else - set_task_slice(p, 0); + scx_set_task_slice(p, 0); } static bool yield_to_task_scx(struct rq *rq, struct task_struct *to) @@ -2274,10 +2273,9 @@ static void wakeup_preempt_scx(struct rq *rq, struct task_struct *p, int wake_fl scx_schedule_reenq_local(rq, 0); } -static void move_local_task_to_local_dsq(struct scx_sched *sch, - struct task_struct *p, u64 enq_flags, - struct scx_dispatch_q *src_dsq, - struct rq *dst_rq) +void scx_move_local_task_to_local_dsq(struct scx_sched *sch, struct task_struct *p, + u64 enq_flags, struct scx_dispatch_q *src_dsq, + struct rq *dst_rq) { struct scx_dispatch_q *dst_dsq = scx_resolve_local_dsq(sch, dst_rq, p, &enq_flags); @@ -2458,7 +2456,7 @@ static bool unlink_dsq_and_switch_rq_lock(struct task_struct *p, lockdep_assert_rq_held(locked_rq); WARN_ON_ONCE(p->scx.holding_cpu >= 0); - task_unlink_from_dsq(p, dsq); + scx_task_unlink_from_dsq(p, dsq); p->scx.holding_cpu = cpu; raw_spin_unlock(&dsq->lock); @@ -2529,9 +2527,8 @@ static struct rq *move_task_between_dsqs(struct scx_sched *sch, if (dst_dsq->id == SCX_DSQ_LOCAL) { /* @p is going from a non-local DSQ to a local DSQ */ if (src_rq == dst_rq) { - task_unlink_from_dsq(p, src_dsq); - move_local_task_to_local_dsq(sch, p, enq_flags, - src_dsq, dst_rq); + scx_task_unlink_from_dsq(p, src_dsq); + scx_move_local_task_to_local_dsq(sch, p, enq_flags, src_dsq, dst_rq); raw_spin_unlock(&src_dsq->lock); } else { raw_spin_unlock(&src_dsq->lock); @@ -2581,8 +2578,8 @@ retry: break; if (rq == task_rq) { - task_unlink_from_dsq(p, dsq); - move_local_task_to_local_dsq(sch, p, enq_flags, dsq, rq); + scx_task_unlink_from_dsq(p, dsq); + scx_move_local_task_to_local_dsq(sch, p, enq_flags, dsq, rq); raw_spin_unlock(&dsq->lock); return true; } @@ -3530,7 +3527,7 @@ static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued) * we can't trust the slice management or ops.core_sched_before(). */ if (scx_bypassing(sch, cpu_of(rq))) { - set_task_slice(curr, 0); + scx_set_task_slice(curr, 0); touch_core_sched(rq, curr); } else if (SCX_HAS_OP(sch, tick)) { SCX_CALL_OP_TASK(sch, tick, rq, curr); @@ -3685,7 +3682,7 @@ static void scx_disable_task(struct scx_sched *sch, struct task_struct *p) * control, after ops.disable() has observed their final values. */ p->scx.dsq_vtime = 0; - set_task_slice(p, 0); + scx_set_task_slice(p, 0); p->scx.reenq_cnt = 0; /* @@ -4184,7 +4181,7 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) if ((reenq_flags & SCX_REENQ_CAP_REVOKE) && rq->curr->sched_class == &ext_sched_class && scx_task_reenq_on_cap_revoke(rq, rq->curr)) { - set_task_slice(rq->curr, 0); + scx_set_task_slice(rq->curr, 0); resched_curr(rq); } @@ -4748,8 +4745,7 @@ DEFINE_SCHED_CLASS(ext) = { #endif }; -static s32 init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id, - struct scx_sched *sch) +s32 scx_init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id, struct scx_sched *sch) { s32 cpu; @@ -6375,7 +6371,7 @@ static void dump_newline(struct seq_buf *s) seq_buf_putc(s, '\n'); } -static __printf(2, 3) void dump_line(struct seq_buf *s, const char *fmt, ...) +__printf(2, 3) void scx_dump_line(struct seq_buf *s, const char *fmt, ...) { va_list args; @@ -6407,7 +6403,7 @@ static void dump_stack_trace(struct seq_buf *s, const char *prefix, unsigned int i; for (i = 0; i < len; i++) - dump_line(s, "%s%pS", prefix, (void *)bt[i]); + scx_dump_line(s, "%s%pS", prefix, (void *)bt[i]); } static void ops_dump_init(struct seq_buf *s, const char *prefix) @@ -6457,7 +6453,7 @@ static void ops_dump_flush(void) */ c = *end; *end = '\0'; - dump_line(dd->s, "%s%s", dd->prefix, line); + scx_dump_line(dd->s, "%s%s", dd->prefix, line); if (c == '\0') break; @@ -6501,21 +6497,19 @@ static void scx_dump_task(struct scx_sched *sch, struct seq_buf *s, struct scx_d (unsigned long long)p->scx.dsq->id); dump_newline(s); - dump_line(s, " %c%c %s[%d] %s%s %+ldms", - marker, task_state_to_char(p), p->comm, p->pid, - own_marker, sch_id_buf, - jiffies_delta_msecs(p->scx.runnable_at, dctx->at_jiffies)); - dump_line(s, " scx_state/flags=%u/0x%x dsq_flags=0x%x ops_state/qseq=%lu/%lu", - scx_get_task_state(p) >> SCX_TASK_STATE_SHIFT, - p->scx.flags & ~SCX_TASK_STATE_MASK, - p->scx.dsq_flags, ops_state & SCX_OPSS_STATE_MASK, - ops_state >> SCX_OPSS_QSEQ_SHIFT); - dump_line(s, " sticky/holding_cpu=%d/%d dsq_id=%s", - p->scx.sticky_cpu, p->scx.holding_cpu, dsq_id_buf); - dump_line(s, " dsq_vtime=%llu slice=%llu weight=%u", - p->scx.dsq_vtime, p->scx.slice, p->scx.weight); - dump_line(s, " cpus=%*pb no_mig=%u", cpumask_pr_args(p->cpus_ptr), - p->migration_disabled); + scx_dump_line(s, " %c%c %s[%d] %s%s %+ldms", + marker, task_state_to_char(p), p->comm, p->pid, own_marker, sch_id_buf, + jiffies_delta_msecs(p->scx.runnable_at, dctx->at_jiffies)); + scx_dump_line(s, " scx_state/flags=%u/0x%x dsq_flags=0x%x ops_state/qseq=%lu/%lu", + scx_get_task_state(p) >> SCX_TASK_STATE_SHIFT, + p->scx.flags & ~SCX_TASK_STATE_MASK, p->scx.dsq_flags, + ops_state & SCX_OPSS_STATE_MASK, ops_state >> SCX_OPSS_QSEQ_SHIFT); + scx_dump_line(s, " sticky/holding_cpu=%d/%d dsq_id=%s", + p->scx.sticky_cpu, p->scx.holding_cpu, dsq_id_buf); + scx_dump_line(s, " dsq_vtime=%llu slice=%llu weight=%u", + p->scx.dsq_vtime, p->scx.slice, p->scx.weight); + scx_dump_line(s, " cpus=%*pb no_mig=%u", cpumask_pr_args(p->cpus_ptr), + p->migration_disabled); if (SCX_HAS_OP(sch, dump_task)) { ops_dump_init(s, " "); @@ -6563,28 +6557,26 @@ static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s, seq_buf_init(&ns, buf, avail); dump_newline(&ns); - dump_line(&ns, "CPU %-4d: nr_run=%u flags=0x%x cpu_rel=%d ops_qseq=%lu ksync=%lu", - cpu, rq->scx.nr_running, rq->scx.flags, - rq->scx.cpu_released, rq->scx.ops_qseq, - rq->scx.kick_sync); - dump_line(&ns, " curr=%s[%d] class=%ps", - rq->curr->comm, rq->curr->pid, - rq->curr->sched_class); + scx_dump_line(&ns, "CPU %-4d: nr_run=%u flags=0x%x cpu_rel=%d ops_qseq=%lu ksync=%lu", + cpu, rq->scx.nr_running, rq->scx.flags, rq->scx.cpu_released, + rq->scx.ops_qseq, rq->scx.kick_sync); + scx_dump_line(&ns, " curr=%s[%d] class=%ps", + rq->curr->comm, rq->curr->pid, rq->curr->sched_class); if (!cpumask_empty(pcpu->cpus_to_kick)) - dump_line(&ns, " cpus_to_kick : %*pb", - cpumask_pr_args(pcpu->cpus_to_kick)); + scx_dump_line(&ns, " cpus_to_kick : %*pb", + cpumask_pr_args(pcpu->cpus_to_kick)); if (!cpumask_empty(pcpu->cpus_to_kick_if_idle)) - dump_line(&ns, " idle_to_kick : %*pb", - cpumask_pr_args(pcpu->cpus_to_kick_if_idle)); + scx_dump_line(&ns, " idle_to_kick : %*pb", + cpumask_pr_args(pcpu->cpus_to_kick_if_idle)); if (!cpumask_empty(pcpu->cpus_to_preempt)) - dump_line(&ns, " cpus_to_preempt: %*pb", - cpumask_pr_args(pcpu->cpus_to_preempt)); + scx_dump_line(&ns, " cpus_to_preempt: %*pb", + cpumask_pr_args(pcpu->cpus_to_preempt)); if (!cpumask_empty(pcpu->cpus_to_wait)) - dump_line(&ns, " cpus_to_wait : %*pb", - cpumask_pr_args(pcpu->cpus_to_wait)); + scx_dump_line(&ns, " cpus_to_wait : %*pb", + cpumask_pr_args(pcpu->cpus_to_wait)); if (!cpumask_empty(rq->scx.cpus_to_sync)) - dump_line(&ns, " cpus_to_sync : %*pb", - cpumask_pr_args(rq->scx.cpus_to_sync)); + scx_dump_line(&ns, " cpus_to_sync : %*pb", + cpumask_pr_args(rq->scx.cpus_to_sync)); used = seq_buf_used(&ns); if (SCX_HAS_OP(sch, dump_cpu)) { @@ -6652,25 +6644,25 @@ static void scx_dump_state(struct scx_sched *sch, struct scx_exit_info *ei, #ifdef CONFIG_EXT_SUB_SCHED if (sch->level == 0) - dump_line(&s, "%s: root", sch->ops.name); + scx_dump_line(&s, "%s: root", sch->ops.name); else - dump_line(&s, "%s: sub%d-%llu %s", - sch->ops.name, sch->level, sch->ops.sub_cgroup_id, - sch->cgrp_path); + scx_dump_line(&s, "%s: sub%d-%llu %s", + sch->ops.name, sch->level, sch->ops.sub_cgroup_id, + sch->cgrp_path); #endif if (ei->kind == SCX_EXIT_NONE) { - dump_line(&s, "Debug dump triggered by %s", ei->reason); + scx_dump_line(&s, "Debug dump triggered by %s", ei->reason); } else { if (ei->exit_cpu >= 0) - dump_line(&s, "%s[%d] triggered exit kind %d on CPU %d:", - current->comm, current->pid, ei->kind, - ei->exit_cpu); + scx_dump_line(&s, "%s[%d] triggered exit kind %d on CPU %d:", + current->comm, current->pid, ei->kind, + ei->exit_cpu); else - dump_line(&s, "%s[%d] triggered exit kind %d:", - current->comm, current->pid, ei->kind); - dump_line(&s, " %s (%s)", ei->reason, ei->msg); + scx_dump_line(&s, "%s[%d] triggered exit kind %d:", + current->comm, current->pid, ei->kind); + scx_dump_line(&s, " %s (%s)", ei->reason, ei->msg); dump_newline(&s); - dump_line(&s, "Backtrace:"); + scx_dump_line(&s, "Backtrace:"); dump_stack_trace(&s, " ", ei->bt, ei->bt_len); } @@ -6681,8 +6673,8 @@ static void scx_dump_state(struct scx_sched *sch, struct scx_exit_info *ei, } dump_newline(&s); - dump_line(&s, "CPU states"); - dump_line(&s, "----------"); + scx_dump_line(&s, "CPU states"); + scx_dump_line(&s, "----------"); /* * Dump stalled CPUs first so they aren't lost to dump truncation, then @@ -6705,8 +6697,8 @@ static void scx_dump_state(struct scx_sched *sch, struct scx_exit_info *ei, } dump_newline(&s); - dump_line(&s, "Event counters"); - dump_line(&s, "--------------"); + scx_dump_line(&s, "Event counters"); + scx_dump_line(&s, "--------------"); scx_read_events(sch, &events); #define SCX_EVENT(name) scx_dump_event(s, &events, name) @@ -6815,7 +6807,7 @@ static struct scx_sched_pnode *alloc_pnode(struct scx_sched *sch, int node) if (!pnode) return NULL; - if (init_dsq(&pnode->global_dsq, SCX_DSQ_GLOBAL, sch)) { + if (scx_init_dsq(&pnode->global_dsq, SCX_DSQ_GLOBAL, sch)) { kfree(pnode); return NULL; } @@ -6876,7 +6868,7 @@ struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, } for_each_possible_cpu(cpu) { - ret = init_dsq(scx_bypass_dsq(sch, cpu), SCX_DSQ_BYPASS, sch); + ret = scx_init_dsq(scx_bypass_dsq(sch, cpu), SCX_DSQ_BYPASS, sch); if (ret) { bypass_fail_cpu = cpu; goto err_free_pcpu; @@ -7509,7 +7501,7 @@ static void scx_root_enable_workfn(struct kthread_work *work) queue_flags |= DEQUEUE_CLASS; scoped_guard (sched_change, p, queue_flags) { - set_task_slice(p, READ_ONCE(sch->slice_dfl)); + scx_set_task_slice(p, READ_ONCE(sch->slice_dfl)); p->sched_class = new_class; } } @@ -8160,7 +8152,7 @@ static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_r if (cur_class == &ext_sched_class) { if (likely(!scx_missing_caps(pcpu->sch, cpu, scx_caps_for_preempt(pcpu->sch, rq)))) - set_task_slice(rq->curr, 0); + scx_set_task_slice(rq->curr, 0); else __scx_add_event(pcpu->sch, SCX_EV_SUB_PREEMPT_DENIED, 1); @@ -8364,9 +8356,9 @@ void __init init_sched_ext_class(void) int n = cpu_to_node(cpu); /* local_dsq's sch will be set during scx_root_enable() */ - BUG_ON(init_dsq(&rq->scx.local_dsq, SCX_DSQ_LOCAL, NULL)); + BUG_ON(scx_init_dsq(&rq->scx.local_dsq, SCX_DSQ_LOCAL, NULL)); #ifdef CONFIG_EXT_SUB_SCHED - BUG_ON(init_dsq(&rq->scx.reject_dsq, SCX_DSQ_REJECT, NULL)); + BUG_ON(scx_init_dsq(&rq->scx.reject_dsq, SCX_DSQ_REJECT, NULL)); #endif INIT_LIST_HEAD(&rq->scx.runnable_list); @@ -8523,7 +8515,7 @@ __bpf_kfunc bool scx_bpf_dsq_insert___v2(struct task_struct *p, u64 dsq_id, return false; if (slice) - set_task_slice(p, slice); + scx_set_task_slice(p, slice); else set_task_slice_keep_oob(p, p->scx.slice ?: 1); @@ -8549,7 +8541,7 @@ static bool scx_dsq_insert_vtime(struct scx_sched *sch, struct task_struct *p, return false; if (slice) - set_task_slice(p, slice); + scx_set_task_slice(p, slice); else set_task_slice_keep_oob(p, p->scx.slice ?: 1); @@ -8733,7 +8725,7 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_VTIME) p->scx.dsq_vtime = kit->vtime; if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_SLICE) - set_task_slice(p, kit->slice); + scx_set_task_slice(p, kit->slice); /* execute move */ locked_rq = move_task_between_dsqs(sch, p, enq_flags, src_dsq, dst_dsq); @@ -9056,10 +9048,10 @@ __bpf_kfunc s32 scx_bpf_create_dsq(u64 dsq_id, s32 node, const struct bpf_prog_a return -ENOMEM; /* - * init_dsq() must be called in GFP_KERNEL context. Init it with NULL - * @sch and update afterwards. + * scx_init_dsq() must be called in GFP_KERNEL context. Init it with + * NULL @sch and update afterwards. */ - ret = init_dsq(dsq, dsq_id, NULL); + ret = scx_init_dsq(dsq, dsq_id, NULL); if (ret) { kfree(dsq); return ret; @@ -9153,7 +9145,7 @@ __bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice, unlikely(scx_missing_caps(sch, cpu_of(locked_rq), SCX_CAP_BASE))) __scx_add_event(sch, SCX_EV_SLICE_DENIED, 1); else - set_task_slice(p, slice); + scx_set_task_slice(p, slice); return true; } @@ -9702,8 +9694,8 @@ __bpf_kfunc void scx_bpf_dump_bstr(char *fmt, unsigned long long *data, ret = __bstr_format(sch, buf->data, buf->line + dd->cursor, sizeof(buf->line) - dd->cursor, fmt, data, data__sz); if (ret < 0) { - dump_line(dd->s, "%s[!] (\"%s\", %p, %u) failed to format (%d)", - dd->prefix, fmt, data, data__sz, ret); + scx_dump_line(dd->s, "%s[!] (\"%s\", %p, %u) failed to format (%d)", + dd->prefix, fmt, data, data__sz, ret); return; } diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 741d8f6b5199..1d226aa8a003 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1968,14 +1968,21 @@ void scx_task_iter_start(struct scx_task_iter *iter, struct cgroup *cgrp); void scx_task_iter_unlock(struct scx_task_iter *iter); void scx_task_iter_stop(struct scx_task_iter *iter); struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter); +void scx_set_task_slice(struct task_struct *p, u64 slice); +void scx_task_unlink_from_dsq(struct task_struct *p, struct scx_dispatch_q *dsq); void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p); void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, int sticky_cpu); +void scx_move_local_task_to_local_dsq(struct scx_sched *sch, struct task_struct *p, + u64 enq_flags, struct scx_dispatch_q *src_dsq, + struct rq *dst_rq); bool scx_consume_dispatch_q(struct scx_sched *sch, struct rq *rq, struct scx_dispatch_q *dsq, u64 enq_flags); bool scx_consume_global_dsq(struct scx_sched *sch, struct rq *rq); bool scx_rq_online(struct rq *rq); void scx_flush_dispatch_buf(struct scx_sched *sch, struct rq *rq); +s32 scx_init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id, struct scx_sched *sch); +__printf(2, 3) void scx_dump_line(struct seq_buf *s, const char *fmt, ...); void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags); void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, u64 reenq_flags, struct rq *locked_rq); @@ -2011,6 +2018,7 @@ extern raw_spinlock_t scx_sched_lock; extern struct mutex scx_enable_mutex; extern struct percpu_rw_semaphore scx_fork_rwsem; extern bool scx_cgroup_enabled; +extern struct list_head scx_sched_all; #ifdef CONFIG_EXT_SUB_SCHED extern const struct rhashtable_params scx_sched_hash_params; extern struct rhashtable scx_sched_hash; -- cgit v1.2.3 From f82b16b8e8f91e973e7c4ca4f6ac84b6385e48eb Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 3 Aug 2026 11:00:47 -1000 Subject: sched_ext: Factor out __scx_bpf_now() scx_bpf_now() couples the valid-or-fresh rq clock read to the current rq. The read is useful for kernel-internal timing against a specific rq, including a remotely locked one. Factor it out into __scx_bpf_now(). Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 59 ++++++++++++++++++++++----------------------- kernel/sched/ext/internal.h | 1 + 2 files changed, 30 insertions(+), 30 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 4245a737592a..777ae515c88e 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -10160,6 +10160,27 @@ __bpf_kfunc struct task_struct *scx_bpf_tid_to_task(u64 tid) return container_of(scx, struct task_struct, scx); } +u64 __scx_bpf_now(struct rq *rq) +{ + /* the caller must be on @rq's cpu or hold its lock */ + lockdep_assert((rq == this_rq() && !preemptible()) || + lockdep_is_held(__rq_lockp(rq))); + + if (smp_load_acquire(&rq->scx.flags) & SCX_RQ_CLK_VALID) { + /* if the rq clock is valid, use the cached rq clock */ + return READ_ONCE(rq->scx.clock); + } else { + /* + * Otherwise, return a fresh rq clock. + * + * The rq clock is updated outside of the rq lock. + * In this case, keep the updated rq clock invalid so the next + * read outside the rq lock gets a fresh rq clock. + */ + return sched_clock_cpu(cpu_of(rq)); + } +} + /** * scx_bpf_now - Returns a high-performance monotonically non-decreasing * clock for the current CPU. The clock returned is in nanoseconds. @@ -10190,36 +10211,14 @@ __bpf_kfunc struct task_struct *scx_bpf_tid_to_task(u64 tid) */ __bpf_kfunc u64 scx_bpf_now(void) { - struct rq *rq; - u64 clock; - - preempt_disable(); - - rq = this_rq(); - if (smp_load_acquire(&rq->scx.flags) & SCX_RQ_CLK_VALID) { - /* - * If the rq clock is valid, use the cached rq clock. - * - * Note that scx_bpf_now() is re-entrant between a process - * context and an interrupt context (e.g., timer interrupt). - * However, we don't need to consider the race between them - * because such race is not observable from a caller. - */ - clock = READ_ONCE(rq->scx.clock); - } else { - /* - * Otherwise, return a fresh rq clock. - * - * The rq clock is updated outside of the rq lock. - * In this case, keep the updated rq clock invalid so the next - * kfunc call outside the rq lock gets a fresh rq clock. - */ - clock = sched_clock_cpu(cpu_of(rq)); - } - - preempt_enable(); - - return clock; + /* + * Note that scx_bpf_now() is re-entrant between a process context and + * an interrupt context (e.g., timer interrupt). However, we don't need + * to consider the race between them because such race is not observable + * from a caller. + */ + guard(preempt)(); + return __scx_bpf_now(this_rq()); } static void scx_read_events(struct scx_sched *sch, struct scx_event_stats *events) diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 1d226aa8a003..a0a2294f1dc2 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1984,6 +1984,7 @@ void scx_flush_dispatch_buf(struct scx_sched *sch, struct rq *rq); s32 scx_init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id, struct scx_sched *sch); __printf(2, 3) void scx_dump_line(struct seq_buf *s, const char *fmt, ...); void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags); +u64 __scx_bpf_now(struct rq *rq); void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, u64 reenq_flags, struct rq *locked_rq); int __scx_init_task(struct scx_sched *sch, struct task_struct *p, -- cgit v1.2.3 From 1fd50778b18d44095adfe5dab2f2a84fcc0fb7a4 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 3 Aug 2026 11:00:57 -1000 Subject: sched_ext: Reject internal enq_flags in the dsq move kfuncs The dsq insert kfuncs reject __SCX_ENQ_INTERNAL_MASK bits in scx_dsq_insert_preamble() instead of scx_vet_enq_flags(). A scheduler can smuggle internal flags such as SCX_ENQ_CLEAR_OPSS through the dsq move kfuncs and corrupt the dispatch protocol. Move the rejection into scx_vet_enq_flags(). The vtime move wrapper OR'd the internal SCX_ENQ_DSQ_PRIQ bit into enq_flags before the vet; the bit now goes in inside scx_dsq_move() after the vet. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 777ae515c88e..1577a063d63f 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -8394,6 +8394,11 @@ static bool scx_vet_enq_flags(struct scx_sched *sch, u64 dsq_id, u64 *enq_flags) bool is_local = dsq_id == SCX_DSQ_LOCAL || (dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON; + if (unlikely(*enq_flags & __SCX_ENQ_INTERNAL_MASK)) { + scx_error(sch, "invalid enq_flags 0x%llx", *enq_flags); + return false; + } + if (*enq_flags & SCX_ENQ_IMMED) { if (unlikely(!is_local)) { scx_error(sch, "SCX_ENQ_IMMED on a non-local DSQ 0x%llx", dsq_id); @@ -8416,11 +8421,6 @@ static bool scx_dsq_insert_preamble(struct scx_sched *sch, struct task_struct *p return false; } - if (unlikely(*enq_flags & __SCX_ENQ_INTERNAL_MASK)) { - scx_error(sch, "invalid enq_flags 0x%llx", *enq_flags); - return false; - } - /* see SCX_EV_INSERT_NOT_OWNED definition */ if (unlikely(!scx_task_on_sched(sch, p))) { __scx_add_event(sch, SCX_EV_INSERT_NOT_OWNED, 1); @@ -8652,7 +8652,8 @@ static const struct btf_kfunc_id_set scx_kfunc_set_enqueue_dispatch = { }; static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, - struct task_struct *p, u64 dsq_id, u64 enq_flags) + struct task_struct *p, u64 dsq_id, u64 enq_flags, + bool priq) { struct scx_dispatch_q *src_dsq = kit->dsq, *dst_dsq; struct scx_sched *sch; @@ -8674,6 +8675,10 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, if (!scx_vet_enq_flags(sch, dsq_id, &enq_flags)) return false; + /* internal bit, can only go in after @enq_flags is vetted */ + if (priq) + enq_flags |= SCX_ENQ_DSQ_PRIQ; + /* * If the BPF scheduler keeps calling this function repeatedly, it can * cause similar live-lock conditions as scx_consume_dispatch_q(). @@ -8930,7 +8935,7 @@ __bpf_kfunc bool scx_bpf_dsq_move(struct bpf_iter_scx_dsq *it__iter, u64 enq_flags) { return scx_dsq_move((struct bpf_iter_scx_dsq_kern *)it__iter, - p, dsq_id, enq_flags); + p, dsq_id, enq_flags, false); } /** @@ -8955,7 +8960,7 @@ __bpf_kfunc bool scx_bpf_dsq_move_vtime(struct bpf_iter_scx_dsq *it__iter, u64 enq_flags) { return scx_dsq_move((struct bpf_iter_scx_dsq_kern *)it__iter, - p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ); + p, dsq_id, enq_flags, true); } __bpf_kfunc_end_defs(); -- cgit v1.2.3 From 78f8d726e62e0b72a4b11e5778d2d7e252b076c0 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 3 Aug 2026 11:01:07 -1000 Subject: sched_ext: Make SCX_ENQ_IGNORE_CAPS waive the preemption cap too SCX_ENQ_IGNORE_CAPS is kernel-internal and marks a placement the kernel forces. scx_caps_for_enq() waives the enqueue cap for it, but a PREEMPT insert still picks up the preemption cap requirement from scx_caps_for_preempt(). Update scx_caps_for_preempt() to take enq_flags and require nothing when SCX_ENQ_IGNORE_CAPS is set. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 5 +++-- kernel/sched/ext/sub.c | 2 +- kernel/sched/ext/sub.h | 9 ++++++--- 3 files changed, 10 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 1577a063d63f..c603b90f16a1 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -8150,8 +8150,9 @@ static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_r if (kickable && !scx_missing_caps(pcpu->sch, cpu, SCX_CAP_BASE)) { if (cpumask_test_cpu(cpu, pcpu->cpus_to_preempt)) { if (cur_class == &ext_sched_class) { - if (likely(!scx_missing_caps(pcpu->sch, cpu, - scx_caps_for_preempt(pcpu->sch, rq)))) + u64 caps = scx_caps_for_preempt(pcpu->sch, rq, 0); + + if (likely(!scx_missing_caps(pcpu->sch, cpu, caps))) scx_set_task_slice(rq->curr, 0); else __scx_add_event(pcpu->sch, diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 2ea9a690e986..c30f48ee07f9 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -262,7 +262,7 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r * @p's owner (@sch). Check caps against the scheduling sched. */ if (*enq_flags & SCX_ENQ_PREEMPT) - needed |= scx_caps_for_preempt(asch, rq); + needed |= scx_caps_for_preempt(asch, rq, *enq_flags); missing = scx_missing_caps(asch, cpu_of(rq), needed); /* requirements met */ diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index db449559bbe8..fe1d82e6c1d5 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -140,7 +140,7 @@ static inline u64 scx_missing_caps(struct scx_sched *sch, s32 cpu, u64 needed) static inline u64 scx_caps_for_enq(u64 enq_flags) { /* a restored task must be put into the local DSQ regardless of caps */ - if (enq_flags & SCX_ENQ_IGNORE_CAPS) + if (unlikely(enq_flags & SCX_ENQ_IGNORE_CAPS)) return 0; if (enq_flags & SCX_ENQ_IMMED) return SCX_CAP_ENQ_IMMED; @@ -156,10 +156,13 @@ static inline u64 scx_caps_for_task(struct task_struct *p) } /* the cap @sch needs to preempt @rq's current task, 0 if none */ -static inline u64 scx_caps_for_preempt(struct scx_sched *sch, struct rq *rq) +static inline u64 scx_caps_for_preempt(struct scx_sched *sch, struct rq *rq, u64 enq_flags) { struct task_struct *curr = rq->curr; + /* a kernel-forced placement preempts regardless of caps */ + if (unlikely(enq_flags & SCX_ENQ_IGNORE_CAPS)) + return 0; /* a non-ext task can't be preempted by ext, own-subtree needs no cap */ if (curr->sched_class != &ext_sched_class || scx_is_descendant(scx_task_sched(curr), sch)) @@ -195,7 +198,7 @@ static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p #else /* CONFIG_EXT_SUB_SCHED */ static inline u64 scx_missing_caps(struct scx_sched *sch, s32 cpu, u64 needed) { return 0; } -static inline u64 scx_caps_for_preempt(struct scx_sched *sch, struct rq *rq) { return 0; } +static inline u64 scx_caps_for_preempt(struct scx_sched *sch, struct rq *rq, u64 enq_flags) { return 0; } static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p) { return true; } #endif /* CONFIG_EXT_SUB_SCHED */ -- cgit v1.2.3 From 13f1eae3b66257625f865babd4fb7c251c8c981e Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 3 Aug 2026 11:01:14 -1000 Subject: sched_ext: Synchronize slice and dsq_vtime writes p->scx.slice and p->scx.dsq_vtime writes have no synchronization rules. The dsq insert kfuncs write both fields synchronously from whatever context they're called in - a direct dispatch from ops.select_cpu() writes with only pi_lock held - and, as the kfuncs are safe to call spuriously with the invalid dispatch discarded later, a scheduler can modify any task's slice by spuriously calling them. The latter stands in the way of an upcoming patch which adds kernel-granted slices that the schedulers must not be able to modify. Give both fields explicit rules. While the task is running, sleeping or queued on an rq-owned DSQ, the rq lock protects them - these are the states where the kernel consumes the slice. While queued on a user DSQ or on the BPF side, the kernel neither consumes nor decides on the fields and every writer acts for the BPF scheduler - synchronizing the writers is the scheduler's responsibility and whichever write lands last wins. To conform, an insert kfunc no longer writes the fields when called. The values travel with the dispatch and take effect when the task is inserted. A discarded dispatch has no side effects. The rq lock rule is asserted at the slice store. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- include/linux/sched/ext.h | 2 + kernel/sched/ext/ext.c | 184 +++++++++++++++++++++++++++----------------- kernel/sched/ext/internal.h | 4 + 3 files changed, 120 insertions(+), 70 deletions(-) (limited to 'kernel') diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index bd9c4059e8fc..3166a0c3d892 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -192,6 +192,8 @@ struct sched_ext_entity { atomic_long_t ops_state; u64 ddsp_dsq_id; u64 ddsp_enq_flags; + u64 ddsp_slice; + u64 ddsp_vtime; struct scx_dsq_list_node dsq_list; /* dispatch order */ struct rb_node dsq_priq; /* p->scx.dsq_vtime order */ u32 dsq_seq; diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index c603b90f16a1..6e59f2669c47 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1204,34 +1204,29 @@ enum scx_slice_oob_consts { }; /* - * Slice write rules + * Slice and dsq_vtime write rules * - * A task's slice - how long it may hold its cpu - is an occupancy grant owned - * by the task's scheduler. How it may be written depends on whether the task is - * running. + * While @p is running, sleeping or queued on an rq-owned DSQ, both fields are + * protected by the rq lock. While running, the rq lock is required because + * update_curr_scx() RMWs the slice and the cap check for slice extension is + * only reliable under the rq lock. * - * Queued, not running: the slice grants no occupancy yet and nothing consumes - * it, so the owner writes it directly - via scx_bpf_dsq_insert(), the dsq move - * kfuncs, or scx_bpf_task_set_slice(). Serializing its own writers is then the - * scheduler's job, not the kernel's. + * While @p is queued on a user DSQ or on the BPF side, the kernel neither + * consumes nor decides on the fields. Synchronizing the writers is the BPF + * scheduler's responsibility. An rq-locked scx_bpf_task_set_slice() write and a + * concurrent DSQ insertion commit can race each other and whichever lands last + * wins. * - * Running: the slice must be changed under the task's rq lock, because: + * A DSQ insert kfunc doesn't update the fields directly. The verdict carries + * the values and apply_slice_vtime() commits them at the insertion. * - * - Raising it extends occupancy, allowed only with %SCX_CAP_BASE on the cpu, - * and that cap check is coherent only under the rq lock. Shortening is always - * allowed. + * scx_bpf_task_set_slice() may be called from any context and writes directly + * only if @p's rq lock is already held, otherwise it bounces through + * p->scx.slice_oob, applied under @p's rq lock at the next slice consideration. * - * - The kernel decrements it there as the task runs. The decrement is a - * read-modify-write, so a racing write can be clobbered. - * - * scx_bpf_task_set_slice() writes directly only when @p is queued or running - * on the rq lock it holds. That is the only state where the lock keeps us @p's - * owner: @p can't move to another rq without it. A task that isn't queued here - * can instead be woken onto a different rq without taking this lock, and that - * dispatch sets its slice - so a direct write would race. Those cases stash - * into p->scx.slice_oob to be applied under @p's actual rq lock. A later in-band - * write supersedes a stash, and a stash whose scheduler id no longer matches - * @p's owner is dropped. + * dsq_vtime orders the next PRIQ insertion and has no running-side consumer, so + * scx_bpf_task_set_dsq_vtime() writes it directly. Fork-time init and direct + * BPF stores from non-cid-form schedulers are outside these rules. */ /* clear a pending slice request */ @@ -1244,6 +1239,7 @@ static void clear_task_slice_oob(struct task_struct *p) /* set @p's slice, leaving any pending out-of-band request in place */ static void set_task_slice_keep_oob(struct task_struct *p, u64 slice) { + lockdep_assert_rq_held(task_rq(p)); p->scx.slice = slice; } @@ -1254,7 +1250,7 @@ void scx_set_task_slice(struct task_struct *p, u64 slice) clear_task_slice_oob(p); } -/* request @p's slice to be set to @slice, see the slice write rules above */ +/* request @p's slice to be set to @slice, see the write rules above */ static void set_task_slice_oob(struct scx_sched *sch, struct task_struct *p, u64 slice) { u64 dur; @@ -1276,7 +1272,7 @@ static void set_task_slice_oob(struct scx_sched *sch, struct task_struct *p, u64 * Apply a pending out-of-band slice request under @rq's lock. A request whose * packed id no longer matches @p's current owner is dropped. An extension needs * baseline cpu access on @p's cid. %SCX_EV_SLICE_DENIED counts the denials. - * Shortening is always allowed. See the slice write rules above. + * Shortening is always allowed. See the write rules above. */ static void apply_task_slice_oob(struct rq *rq, struct task_struct *p) { @@ -1305,7 +1301,30 @@ static void apply_task_slice_oob(struct rq *rq, struct task_struct *p) return; } - p->scx.slice = slice; + set_task_slice_keep_oob(p, slice); +} + +/* + * A dsq insert kfunc doesn't write slice or dsq_vtime. The verdict carries them + * and they are committed here, at the insertion. A zero @slice keeps the + * current value, floored at 1 so the task isn't treated as expired. + */ +static void apply_slice_vtime(struct task_struct *p, u64 slice, u64 vtime, u64 enq_flags) +{ + if (slice) { + p->scx.slice = slice; + /* + * An explicit slice supersedes a pending oob request. A carried + * default refill is not an explicit request and must keep it. + */ + if (!(enq_flags & SCX_ENQ_SLICE_DFL)) + clear_task_slice_oob(p); + } else if (!p->scx.slice) { + p->scx.slice = 1; + } + + if (enq_flags & SCX_ENQ_DSQ_PRIQ) + p->scx.dsq_vtime = vtime; } static void update_curr_scx(struct rq *rq) @@ -1502,7 +1521,7 @@ static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq, static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq, struct scx_dispatch_q *dsq, struct task_struct *p, - u64 enq_flags) + u64 slice, u64 vtime, u64 enq_flags) { bool is_rq_owned = false; @@ -1541,6 +1560,13 @@ static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq, enq_flags &= ~SCX_ENQ_DSQ_PRIQ; } + /* + * @dsq is locked and @enq_flags is sanitized. Commit the carried slice + * and vtime before the PRIQ insertion below reads the new dsq_vtime. + */ + if (enq_flags & SCX_ENQ_APPLY_SLICE) + apply_slice_vtime(p, slice, vtime, enq_flags); + if (enq_flags & SCX_ENQ_DSQ_PRIQ) { struct rb_node *rbp; @@ -1763,7 +1789,7 @@ static struct scx_dispatch_q *find_dsq_for_dispatch(struct scx_sched *sch, static void mark_direct_dispatch(struct scx_sched *sch, struct task_struct *ddsp_task, struct task_struct *p, u64 dsq_id, - u64 enq_flags) + u64 slice, u64 vtime, u64 enq_flags) { /* * Mark that dispatch already happened from ops.select_cpu() or @@ -1787,6 +1813,8 @@ static void mark_direct_dispatch(struct scx_sched *sch, WARN_ON_ONCE(p->scx.ddsp_dsq_id != SCX_DSQ_INVALID); WARN_ON_ONCE(p->scx.ddsp_enq_flags); + p->scx.ddsp_slice = slice; + p->scx.ddsp_vtime = vtime; p->scx.ddsp_dsq_id = dsq_id; p->scx.ddsp_enq_flags = enq_flags; } @@ -1818,7 +1846,7 @@ static void direct_dispatch(struct scx_sched *sch, struct task_struct *p, struct rq *rq = task_rq(p); struct scx_dispatch_q *dsq = find_dsq_for_dispatch(sch, rq, p->scx.ddsp_dsq_id, task_cpu(p)); - u64 ddsp_enq_flags; + u64 ddsp_enq_flags, slice, vtime; touch_core_sched_dispatch(rq, p); @@ -1860,9 +1888,12 @@ static void direct_dispatch(struct scx_sched *sch, struct task_struct *p, } ddsp_enq_flags = p->scx.ddsp_enq_flags; + slice = p->scx.ddsp_slice; + vtime = p->scx.ddsp_vtime; clear_direct_dispatch(p); - scx_dispatch_enqueue(sch, rq, dsq, p, ddsp_enq_flags | SCX_ENQ_CLEAR_OPSS); + scx_dispatch_enqueue(sch, rq, dsq, p, slice, vtime, + ddsp_enq_flags | SCX_ENQ_APPLY_SLICE | SCX_ENQ_CLEAR_OPSS); } bool scx_rq_online(struct rq *rq) @@ -1983,7 +2014,7 @@ direct: direct_dispatch(sch, p, enq_flags); return; local_norefill: - scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, enq_flags); + scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0, enq_flags); return; local: dsq = &rq->scx.local_dsq; @@ -2004,7 +2035,7 @@ enqueue: touch_core_sched(rq, p); refill_task_slice_dfl(sch, p); clear_direct_dispatch(p); - scx_dispatch_enqueue(sch, rq, dsq, p, enq_flags); + scx_dispatch_enqueue(sch, rq, dsq, p, 0, 0, enq_flags); } static bool task_runnable(const struct task_struct *p) @@ -2542,7 +2573,7 @@ static struct rq *move_task_between_dsqs(struct scx_sched *sch, dispatch_dequeue_locked(p, src_dsq); raw_spin_unlock(&src_dsq->lock); - scx_dispatch_enqueue(sch, dst_rq, dst_dsq, p, enq_flags); + scx_dispatch_enqueue(sch, dst_rq, dst_dsq, p, 0, 0, enq_flags); } return dst_rq; @@ -2608,6 +2639,8 @@ bool scx_consume_global_dsq(struct scx_sched *sch, struct rq *rq) * @rq: current rq which is locked * @dst_dsq: destination DSQ * @p: task to dispatch + * @slice: slice carried by the insert verdict, 0 keeps the current value + * @vtime: vtime carried by the insert verdict, committed on PRIQ inserts * @enq_flags: %SCX_ENQ_* * * We're holding @rq lock and want to dispatch @p to @dst_dsq which is a local @@ -2618,8 +2651,8 @@ bool scx_consume_global_dsq(struct scx_sched *sch, struct rq *rq) * %SCX_OPSS_DISPATCHING). */ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, - struct scx_dispatch_q *dst_dsq, - struct task_struct *p, u64 enq_flags) + struct scx_dispatch_q *dst_dsq, struct task_struct *p, + u64 slice, u64 vtime, u64 enq_flags) { struct rq *src_rq = task_rq(p); struct rq *dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq); @@ -2632,8 +2665,8 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, * If dispatching to @rq that @p is already on, no lock dancing needed. */ if (rq == src_rq && rq == dst_rq) { - scx_dispatch_enqueue(sch, rq, dst_dsq, p, - enq_flags | SCX_ENQ_CLEAR_OPSS); + scx_dispatch_enqueue(sch, rq, dst_dsq, p, slice, vtime, + enq_flags | SCX_ENQ_APPLY_SLICE | SCX_ENQ_CLEAR_OPSS); return; } @@ -2671,13 +2704,16 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, if (src_rq == dst_rq) { p->scx.holding_cpu = -1; scx_dispatch_enqueue(sch, dst_rq, &dst_rq->scx.local_dsq, p, - enq_flags); + slice, vtime, enq_flags | SCX_ENQ_APPLY_SLICE); } else if (unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) { p->scx.holding_cpu = -1; fallback = true; scx_dispatch_enqueue(sch, src_rq, find_global_dsq(sch, task_cpu(p)), - p, enq_flags | SCX_ENQ_GDSQ_FALLBACK); + p, slice, vtime, + enq_flags | SCX_ENQ_APPLY_SLICE | + SCX_ENQ_GDSQ_FALLBACK); } else { + apply_slice_vtime(p, slice, vtime, enq_flags); move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, dst_rq); /* task has been moved to dst_rq, which is now locked */ locked_rq = dst_rq; @@ -2713,10 +2749,9 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, * was valid in the first place. Make sure that the task is still owned by the * BPF scheduler and claim the ownership before dispatching. */ -static void finish_dispatch(struct scx_sched *sch, struct rq *rq, - struct task_struct *p, - unsigned long qseq_at_dispatch, - u64 dsq_id, u64 enq_flags) +static void finish_dispatch(struct scx_sched *sch, struct rq *rq, struct task_struct *p, + unsigned long qseq_at_dispatch, u64 dsq_id, + u64 slice, u64 vtime, u64 enq_flags) { struct scx_dispatch_q *dsq; unsigned long opss; @@ -2776,9 +2811,10 @@ retry: dsq = find_dsq_for_dispatch(sch, this_rq(), dsq_id, task_cpu(p)); if (dsq->id == SCX_DSQ_LOCAL) - dispatch_to_local_dsq(sch, rq, dsq, p, enq_flags); + dispatch_to_local_dsq(sch, rq, dsq, p, slice, vtime, enq_flags); else - scx_dispatch_enqueue(sch, rq, dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); + scx_dispatch_enqueue(sch, rq, dsq, p, slice, vtime, + enq_flags | SCX_ENQ_APPLY_SLICE | SCX_ENQ_CLEAR_OPSS); } void scx_flush_dispatch_buf(struct scx_sched *sch, struct rq *rq) @@ -2790,7 +2826,7 @@ void scx_flush_dispatch_buf(struct scx_sched *sch, struct rq *rq) struct scx_dsp_buf_ent *ent = &dspc->buf[u]; finish_dispatch(sch, rq, ent->task, ent->qseq, ent->dsq_id, - ent->enq_flags); + ent->slice, ent->vtime, ent->enq_flags); } dspc->nr_tasks += dspc->cursor; @@ -3035,7 +3071,8 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; } else { - scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, SCX_ENQ_HEAD); + scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0, + SCX_ENQ_HEAD); } goto switch_class; } @@ -3319,7 +3356,13 @@ static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flag cpu = scx_select_cpu_dfl(p, prev_cpu, wake_flags, NULL, 0); if (cpu >= 0) { - refill_task_slice_dfl(sch, p); + /* + * Carry the slice refill and let the insertion commit + * it under rq lock. See the write rules. + */ + __scx_add_event(sch, SCX_EV_REFILL_SLICE_DFL, 1); + p->scx.ddsp_slice = READ_ONCE(sch->slice_dfl); + p->scx.ddsp_enq_flags = SCX_ENQ_SLICE_DFL; p->scx.ddsp_dsq_id = SCX_DSQ_LOCAL; } else { cpu = prev_cpu; @@ -4051,13 +4094,15 @@ static void process_ddsp_deferred_locals(struct rq *rq) struct scx_dispatch_q *dsq; u64 dsq_id = p->scx.ddsp_dsq_id; u64 enq_flags = p->scx.ddsp_enq_flags; + u64 slice = p->scx.ddsp_slice; + u64 vtime = p->scx.ddsp_vtime; list_del_init(&p->scx.dsq_list.node); clear_direct_dispatch(p); dsq = find_dsq_for_dispatch(sch, rq, dsq_id, task_cpu(p)); if (!WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL)) - dispatch_to_local_dsq(sch, rq, dsq, p, enq_flags); + dispatch_to_local_dsq(sch, rq, dsq, p, slice, vtime, enq_flags); } } @@ -5487,7 +5532,7 @@ resume: * between bypass DSQs. */ dispatch_dequeue_locked(p, donor_dsq); - scx_dispatch_enqueue(sch, cpu_rq(donee), donee_dsq, p, SCX_ENQ_NESTED); + scx_dispatch_enqueue(sch, cpu_rq(donee), donee_dsq, p, 0, 0, SCX_ENQ_NESTED); /* * $donee might have been idle and need to be woken up. No need @@ -8435,14 +8480,14 @@ static bool scx_dsq_insert_preamble(struct scx_sched *sch, struct task_struct *p } static void scx_dsq_insert_commit(struct scx_sched *sch, struct task_struct *p, - u64 dsq_id, u64 enq_flags) + u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) { struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; struct task_struct *ddsp_task; ddsp_task = __this_cpu_read(direct_dispatch_task); if (ddsp_task) { - mark_direct_dispatch(sch, ddsp_task, p, dsq_id, enq_flags); + mark_direct_dispatch(sch, ddsp_task, p, dsq_id, slice, vtime, enq_flags); return; } @@ -8455,6 +8500,8 @@ static void scx_dsq_insert_commit(struct scx_sched *sch, struct task_struct *p, .task = p, .qseq = atomic_long_read(&p->scx.ops_state) & SCX_OPSS_QSEQ_MASK, .dsq_id = dsq_id, + .slice = slice, + .vtime = vtime, .enq_flags = enq_flags, }; } @@ -8515,12 +8562,7 @@ __bpf_kfunc bool scx_bpf_dsq_insert___v2(struct task_struct *p, u64 dsq_id, if (!scx_dsq_insert_preamble(sch, p, dsq_id, &enq_flags)) return false; - if (slice) - scx_set_task_slice(p, slice); - else - set_task_slice_keep_oob(p, p->scx.slice ?: 1); - - scx_dsq_insert_commit(sch, p, dsq_id, enq_flags); + scx_dsq_insert_commit(sch, p, dsq_id, slice, 0, enq_flags); return true; } @@ -8541,14 +8583,7 @@ static bool scx_dsq_insert_vtime(struct scx_sched *sch, struct task_struct *p, if (!scx_dsq_insert_preamble(sch, p, dsq_id, &enq_flags)) return false; - if (slice) - scx_set_task_slice(p, slice); - else - set_task_slice_keep_oob(p, p->scx.slice ?: 1); - - p->scx.dsq_vtime = vtime; - - scx_dsq_insert_commit(sch, p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ); + scx_dsq_insert_commit(sch, p, dsq_id, slice, vtime, enq_flags | SCX_ENQ_DSQ_PRIQ); return true; } @@ -8724,9 +8759,9 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, dst_dsq = find_dsq_for_dispatch(sch, this_rq, dsq_id, task_cpu(p)); /* - * Apply vtime and slice updates before moving so that the new time is - * visible before inserting into $dst_dsq. @p is still on $src_dsq but - * this is safe as we're locking it. + * Apply vtime and slice updates before moving. @p is still on $src_dsq + * with both $src_dsq and its task_rq locked, satisfying the write + * rules, and the PRIQ insertion into $dst_dsq reads the new vtime. */ if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_VTIME) p->scx.dsq_vtime = kit->vtime; @@ -9136,7 +9171,16 @@ __bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice, /* * Directly write only when we hold the lock of the rq @p is queued or - * running on. See the slice write rules above. + * running on. See the write rules above. + * + * While @p is queued on a user DSQ or in the BPF scheduler, + * synchronization is the scheduler's responsibility. This write can + * race a concurrent dispatch's commit, see apply_slice_vtime(). + * + * Making this kfunc always go through the oob stash would leave the + * commit as the only direct writer and close the race, but that would + * require two more oob application points - the dispatch keep-prev test + * and the tick-time expiry check. */ locked_rq = scx_locked_rq(); if (!locked_rq || diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index a0a2294f1dc2..a11ed6e1e028 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1246,6 +1246,8 @@ struct scx_dsp_buf_ent { struct task_struct *task; unsigned long qseq; u64 dsq_id; + u64 slice; + u64 vtime; u64 enq_flags; }; @@ -1680,6 +1682,8 @@ enum scx_enq_flags { SCX_ENQ_NESTED = 1LLU << 58, SCX_ENQ_GDSQ_FALLBACK = 1LLU << 59, /* fell back to global DSQ */ SCX_ENQ_IGNORE_CAPS = 1LLU << 60, /* admit to local DSQ ignoring caps */ + SCX_ENQ_APPLY_SLICE = 1LLU << 61, /* apply carried slice/vtime at insertion */ + SCX_ENQ_SLICE_DFL = 1LLU << 62, /* carried slice is a default refill */ }; enum scx_deq_flags { -- cgit v1.2.3 From 9cfc6ab34a3184b3683d27dcccc5c05bede41c37 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 3 Aug 2026 11:01:20 -1000 Subject: sched_ext: Add SCX_TASK_PROTECTED A BPF scheduler can displace any of its tasks at will - cut a running one's slice with an SCX_ENQ_PREEMPT dispatch, an SCX_KICK_PREEMPT kick or a direct shortening, and jump a queued one with HEAD insertions. Sometimes the kernel needs a slice and a DSQ position to stick regardless. Add SCX_TASK_PROTECTED, guarding both: - The slice becomes immutable. Every scheduler-reachable write is refused and counted as SCX_EV_SLICE_DENIED. Higher scheduling classes are unaffected. PREEMPT|IMMED can't preempt a running protected task and gets reenqueued. - A protected task that reached the head of its DSQ keeps it - HEAD insertions land behind the leading run of protected tasks and reenqueue sweeps skip them. Only rq-owned DSQs can hold protected tasks, so the walk runs only for them. The bit lives in p->scx.flags so that both the refusal and the head walk read it under the rq lock that protects it. Protection ends when the slice is consumed, when the task leaves the rq except for a save/restore on the running task, on a yield, when the scheduler enters bypass, and when the task leaves scx. The flag is kernel-internal and not used yet. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- include/linux/sched/ext.h | 1 + kernel/sched/ext/ext.c | 165 ++++++++++++++++++++++++++++++++++++++------ kernel/sched/ext/internal.h | 2 +- 3 files changed, 144 insertions(+), 24 deletions(-) (limited to 'kernel') diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index 3166a0c3d892..b519fbc88e17 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -102,6 +102,7 @@ enum scx_ent_flags { SCX_TASK_DEQD_FOR_SLEEP = 1 << 3, /* last dequeue was for SLEEP */ SCX_TASK_SUB_INIT = 1 << 4, /* task being initialized for a sub sched */ SCX_TASK_IMMED = 1 << 5, /* task is on local DSQ with %SCX_ENQ_IMMED */ + SCX_TASK_PROTECTED = 1 << 6, /* slice and DSQ head position protected */ /* * Bits 8 to 10 are used to carry task state: diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 6e59f2669c47..0121ecec8b25 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -386,8 +386,16 @@ static bool rq_is_open(struct rq *rq, u64 enq_flags) * so allow it to avoid spuriously triggering reenq on a combined * PREEMPT|IMMED insertion. */ - if (enq_flags & SCX_ENQ_PREEMPT) - return true; + if (enq_flags & SCX_ENQ_PREEMPT) { + struct task_struct *curr = rq->curr; + + /* + * A protected slice refuses the preemption and the cpu stays + * occupied. See rq_owned_post_enq(). + */ + return curr->sched_class != &ext_sched_class || + likely(!(curr->scx.flags & SCX_TASK_PROTECTED)); + } /* * @rq is either in transition to or running an SCX task and can't go @@ -1224,6 +1232,9 @@ enum scx_slice_oob_consts { * only if @p's rq lock is already held, otherwise it bounces through * p->scx.slice_oob, applied under @p's rq lock at the next slice consideration. * + * While %SCX_TASK_PROTECTED is set, every scheduler-reachable slice update is + * refused. See set_task_slice_keep_oob(). + * * dsq_vtime orders the next PRIQ insertion and has no running-side consumer, so * scx_bpf_task_set_dsq_vtime() writes it directly. Fork-time init and direct * BPF stores from non-cid-form schedulers are outside these rules. @@ -1236,18 +1247,92 @@ static void clear_task_slice_oob(struct task_struct *p) atomic64_set(&p->scx.slice_oob, 0); } -/* set @p's slice, leaving any pending out-of-band request in place */ -static void set_task_slice_keep_oob(struct task_struct *p, u64 slice) +/** + * dsq_insert_head - FIFO head insertion honoring %SCX_TASK_PROTECTED + * @dsq: DSQ to insert into + * @p: task being inserted + * + * A HEAD insert should land behind any leading protected tasks. Return %true + * indicates whether @p became the first entry. + */ +static bool dsq_insert_head(struct scx_dispatch_q *dsq, struct task_struct *p) +{ + struct list_head *pos = &dsq->list; + struct scx_dsq_list_node *node; + + /* + * Only rq-owned DSQs can hold protected tasks and the associated rq + * lock keeps their flags stable. + */ + if (!dsq_is_rq_owned(dsq)) { + list_add(&p->scx.dsq_list.node, &dsq->list); + return true; + } + + list_for_each_entry(node, &dsq->list, node) { + struct task_struct *q; + + if (WARN_ON_ONCE(node->flags & SCX_DSQ_LNODE_ITER_CURSOR)) + continue; + + q = container_of(node, struct task_struct, scx.dsq_list); + if (!(q->scx.flags & SCX_TASK_PROTECTED)) + break; + + pos = &node->node; + } + + list_add(&p->scx.dsq_list.node, pos); + + return pos == &dsq->list; +} + +/** + * set_task_slice_keep_oob - Set @p's slice, leaving any pending oob request + * @p: task of interest + * @slice: slice to set + * + * While %SCX_TASK_PROTECTED is set, BPF schedulers may not modify the slice. + * Refuse and return %false. + */ +static bool set_task_slice_keep_oob(struct task_struct *p, u64 slice) { lockdep_assert_rq_held(task_rq(p)); + + if (unlikely(p->scx.flags & SCX_TASK_PROTECTED)) + return false; + p->scx.slice = slice; + return true; } /* set @p's slice, superseding any pending out-of-band request */ -void scx_set_task_slice(struct task_struct *p, u64 slice) +bool scx_set_task_slice(struct task_struct *p, u64 slice) { - set_task_slice_keep_oob(p, slice); + if (!set_task_slice_keep_oob(p, slice)) + return false; clear_task_slice_oob(p); + return true; +} + +/** + * scx_task_slice_ended - @p's slice is consumed or given up + * @rq: rq @p is on + * @p: task of interest + * + * End what rides on the slice - the protection. + * + * A dequeue normally ends the slice too. The exception is a save/restore pair + * on the running task. Attribute changes like renice cycle the task through + * dequeue and enqueue while it keeps executing, so the slice continues. A + * queued task instead loses its DSQ position on any dequeue and the slice ends + * with it. + */ +static void scx_task_slice_ended(struct rq *rq, struct task_struct *p) +{ + lockdep_assert_rq_held(rq); + + p->scx.flags &= ~SCX_TASK_PROTECTED; } /* request @p's slice to be set to @slice, see the write rules above */ @@ -1271,8 +1356,9 @@ static void set_task_slice_oob(struct scx_sched *sch, struct task_struct *p, u64 /* * Apply a pending out-of-band slice request under @rq's lock. A request whose * packed id no longer matches @p's current owner is dropped. An extension needs - * baseline cpu access on @p's cid. %SCX_EV_SLICE_DENIED counts the denials. - * Shortening is always allowed. See the write rules above. + * baseline cpu access on @p's cid, shortening is always allowed, and a + * protected slice refuses both. %SCX_EV_SLICE_DENIED counts the denials. See + * the write rules above. */ static void apply_task_slice_oob(struct rq *rq, struct task_struct *p) { @@ -1301,7 +1387,8 @@ static void apply_task_slice_oob(struct rq *rq, struct task_struct *p) return; } - set_task_slice_keep_oob(p, slice); + if (unlikely(!set_task_slice_keep_oob(p, slice))) + __scx_add_event(scx_task_sched(p), SCX_EV_SLICE_DENIED, 1); } /* @@ -1514,8 +1601,10 @@ static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq, if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr && rq->curr->sched_class == &ext_sched_class) { - scx_set_task_slice(rq->curr, 0); - resched_curr(rq); + if (likely(scx_set_task_slice(rq->curr, 0))) + resched_curr(rq); + else + __scx_add_event(sch, SCX_EV_SLICE_DENIED, 1); } } @@ -1606,9 +1695,8 @@ static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq, dsq->id); if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) { - list_add(&p->scx.dsq_list.node, &dsq->list); /* new task inserted at head - use fastpath */ - if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN)) + if (dsq_insert_head(dsq, p) && !(dsq->id & SCX_DSQ_FLAG_BUILTIN)) rcu_assign_pointer(dsq->first_task, p); } else { /* @@ -2254,6 +2342,11 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_ sub_nr_running(rq, 1); scx_dispatch_dequeue(rq, p); + + /* see scx_task_slice_ended() for the save/restore exception */ + if (!((deq_flags & DEQUEUE_SAVE) && task_current(rq, p))) + scx_task_slice_ended(rq, p); + clear_direct_dispatch(p); return true; } @@ -2263,6 +2356,9 @@ static void yield_task_scx(struct rq *rq) struct task_struct *p = rq->donor; struct scx_sched *sch = scx_task_sched(p); + /* a yield gives the slice up */ + scx_task_slice_ended(rq, p); + if (SCX_HAS_OP(sch, yield)) SCX_CALL_OP_2TASKS_RET(sch, yield, rq, p, NULL); else @@ -2274,6 +2370,9 @@ static bool yield_to_task_scx(struct rq *rq, struct task_struct *to) struct task_struct *from = rq->donor; struct scx_sched *sch = scx_task_sched(from); + /* like a plain yield, giving the slice up ends the protection */ + scx_task_slice_ended(rq, from); + if (SCX_HAS_OP(sch, yield) && sch == scx_task_sched(to)) return SCX_CALL_OP_2TASKS_RET(sch, yield, rq, from, to); else @@ -2317,7 +2416,7 @@ void scx_move_local_task_to_local_dsq(struct scx_sched *sch, struct task_struct WARN_ON_ONCE(p->scx.holding_cpu >= 0); if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) - list_add(&p->scx.dsq_list.node, &dst_dsq->list); + dsq_insert_head(dst_dsq, p); else list_add_tail(&p->scx.dsq_list.node, &dst_dsq->list); @@ -3051,6 +3150,10 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, update_curr_scx(rq); + /* the slice is consumed, protection ends with it */ + if (!p->scx.slice) + scx_task_slice_ended(rq, p); + /* see dequeue_task_scx() on why we skip when !QUEUED */ if (SCX_HAS_OP(sch, stopping) && (p->scx.flags & SCX_TASK_QUEUED)) SCX_CALL_OP_TASK(sch, stopping, rq, p, true); @@ -3204,8 +3307,11 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) */ if (keep_prev) { p = prev; - if (!p->scx.slice) + if (!p->scx.slice) { + /* the slice is consumed, protection ends */ + scx_task_slice_ended(rq, p); refill_task_slice_dfl(scx_task_sched(p), p); + } } else { p = first_local_task(rq); if (!p) @@ -3725,6 +3831,7 @@ static void scx_disable_task(struct scx_sched *sch, struct task_struct *p) * control, after ops.disable() has observed their final values. */ p->scx.dsq_vtime = 0; + scx_task_slice_ended(rq, p); scx_set_task_slice(p, 0); p->scx.reenq_cnt = 0; @@ -4137,6 +4244,9 @@ static bool local_task_should_reenq(struct rq *rq, struct task_struct *p, first = !(*reenq_flags & SCX_REENQ_TSR_NOT_FIRST); *reenq_flags |= SCX_REENQ_TSR_NOT_FIRST; + if (unlikely(p->scx.flags & SCX_TASK_PROTECTED)) + return false; + *reason = SCX_TASK_REENQ_KFUNC; if ((p->scx.flags & SCX_TASK_IMMED) && @@ -5901,6 +6011,13 @@ void scx_bypass(struct scx_sched *sch, bool bypass) if (!scx_is_descendant(scx_task_sched(p), sch)) continue; + /* + * Bypass trumps protection. Cycling clears for queued + * tasks but current task needs explicit stripping. + */ + if (bypass && task_current(rq, p)) + scx_task_slice_ended(rq, p); + /* cycling deq/enq is enough, see the function comment */ scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { /* nothing */ ; @@ -8197,11 +8314,10 @@ static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_r if (cur_class == &ext_sched_class) { u64 caps = scx_caps_for_preempt(pcpu->sch, rq, 0); - if (likely(!scx_missing_caps(pcpu->sch, cpu, caps))) - scx_set_task_slice(rq->curr, 0); - else - __scx_add_event(pcpu->sch, - SCX_EV_SUB_PREEMPT_DENIED, 1); + if (unlikely(scx_missing_caps(pcpu->sch, cpu, caps))) + __scx_add_event(pcpu->sch, SCX_EV_SUB_PREEMPT_DENIED, 1); + else if (unlikely(!scx_set_task_slice(rq->curr, 0))) + __scx_add_event(pcpu->sch, SCX_EV_SLICE_DENIED, 1); } cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt); } @@ -9192,10 +9308,13 @@ __bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice, /* under the rq lock: apply now, extensions gated on baseline access */ if (slice > p->scx.slice && - unlikely(scx_missing_caps(sch, cpu_of(locked_rq), SCX_CAP_BASE))) + unlikely(scx_missing_caps(sch, cpu_of(locked_rq), SCX_CAP_BASE))) { + __scx_add_event(sch, SCX_EV_SLICE_DENIED, 1); + return true; + } + + if (unlikely(!scx_set_task_slice(p, slice))) __scx_add_event(sch, SCX_EV_SLICE_DENIED, 1); - else - scx_set_task_slice(p, slice); return true; } diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index a11ed6e1e028..d418935f1e6b 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1972,7 +1972,7 @@ void scx_task_iter_start(struct scx_task_iter *iter, struct cgroup *cgrp); void scx_task_iter_unlock(struct scx_task_iter *iter); void scx_task_iter_stop(struct scx_task_iter *iter); struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter); -void scx_set_task_slice(struct task_struct *p, u64 slice); +bool scx_set_task_slice(struct task_struct *p, u64 slice); void scx_task_unlink_from_dsq(struct task_struct *p, struct scx_dispatch_q *dsq); void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p); void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, -- cgit v1.2.3 From 5fd501744b10814f5c12899ce86d223cee2c51ca Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 3 Aug 2026 11:01:29 -1000 Subject: sched_ext: Add bandwidth-limited rescue execution for stranded tasks A local DSQ insert lacking the needed caps is diverted to the reject DSQ and bounced back through ops.enqueue() so the scheduler can re-decide. That recovery assumes the scheduler has somewhere legal to send the task. When it doesn't, e.g. when the task's affinity is restricted to cids delegated away, the task starves until the stall watchdog ejects the scheduler. An exiting task is worse - it skips ops.enqueue() and the rejection becomes a self-requeuing cycle that burns the CPU until the watchdog fires. Add SCX_ENQ_RESCUE, a fallback modifier on local DSQ inserts. When the insert would be rejected for missing caps, the kernel takes over and runs the task on the target CPU without consulting the owning scheduler. The kernel sets the flag itself when enqueueing an exiting task. Rescue is a last-resort forward-progress backstop with a persistent disadvantage, not a way around cap enforcement. A per-CPU token bucket accrues rescue_bandwidth_ppt (default 2%) of CPU time and rescues run one at a time in arrival order. Each is granted a slice of the rescue_quantum_us (default 5ms) quantum divided across the waiters, waits at the tail of the local DSQ claiming no priority, and rejoins its scheduler as a fresh arrival once the slice is served. The schedulers keep their normal control over an admitted rescuee and may preempt or reslice it. Service is measured on CPU time actually received, so neither shortens the rescue. Prolonged denial escalates - the remaining slice turns into protected execution (SCX_TASK_PROTECTED) and the rescuee preempts the current task. Escalation is paced by the same bucket, and delivered service converges on the configured bandwidth no matter how aggressively the schedulers dispatch. Both knobs are root-only and SCX_RESCUE_DISABLE turns rescue off, making SCX_ENQ_RESCUE inserts reject as usual. v2: - Add SCX_OPS_OPEN() fix-ups for the new ops fields so cpu-form schedulers setting them still load on older kernels. (Andrea) Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- include/linux/sched/ext.h | 1 + kernel/sched/ext/ext.c | 112 +++++++++-- kernel/sched/ext/internal.h | 56 +++++- kernel/sched/ext/sub.c | 376 +++++++++++++++++++++++++++++++++-- kernel/sched/ext/sub.h | 26 +++ kernel/sched/ext/types.h | 9 + kernel/sched/sched.h | 12 ++ tools/sched_ext/include/scx/compat.h | 11 + 8 files changed, 570 insertions(+), 33 deletions(-) (limited to 'kernel') diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index b519fbc88e17..a6aabbefd185 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -59,6 +59,7 @@ enum scx_dsq_id_flags { SCX_DSQ_LOCAL = SCX_DSQ_FLAG_BUILTIN | 2, SCX_DSQ_BYPASS = SCX_DSQ_FLAG_BUILTIN | 3, SCX_DSQ_REJECT = SCX_DSQ_FLAG_BUILTIN | 4, /* internal - see find_dsq_for_dispatch() */ + SCX_DSQ_RESCUE = SCX_DSQ_FLAG_BUILTIN | 5, /* internal - see find_dsq_for_dispatch() */ SCX_DSQ_LOCAL_ON = SCX_DSQ_FLAG_BUILTIN | SCX_DSQ_FLAG_LOCAL_ON, SCX_DSQ_LOCAL_CPU_MASK = 0xffffffffLLU, }; diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 0121ecec8b25..7f86d4adcd0d 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -111,6 +111,7 @@ static bool dsq_is_rq_owned(struct scx_dispatch_q *dsq) switch (dsq->id) { case SCX_DSQ_LOCAL: case SCX_DSQ_REJECT: + case SCX_DSQ_RESCUE: return true; default: return false; @@ -1320,7 +1321,8 @@ bool scx_set_task_slice(struct task_struct *p, u64 slice) * @rq: rq @p is on * @p: task of interest * - * End what rides on the slice - the protection. + * End what rides on the slice - the protection, and the rescue if @p is being + * rescued. * * A dequeue normally ends the slice too. The exception is a save/restore pair * on the running task. Attribute changes like renice cycle the task through @@ -1328,11 +1330,13 @@ bool scx_set_task_slice(struct task_struct *p, u64 slice) * queued task instead loses its DSQ position on any dequeue and the slice ends * with it. */ -static void scx_task_slice_ended(struct rq *rq, struct task_struct *p) +void scx_task_slice_ended(struct rq *rq, struct task_struct *p) { lockdep_assert_rq_held(rq); p->scx.flags &= ~SCX_TASK_PROTECTED; + if (unlikely(p == scx_rescuee(rq))) + scx_rescue_end(rq); } /* request @p's slice to be set to @slice, see the write rules above */ @@ -1432,6 +1436,9 @@ static void update_curr_scx(struct rq *rq) touch_core_sched(rq, curr); } + if (unlikely(curr == scx_rescuee(rq))) + scx_rescue_charge(rq, delta_exec); + dl_server_update(&rq->ext_server, delta_exec); } @@ -1547,9 +1554,13 @@ static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq, { call_task_dequeue(sch, rq, p, 0); - /* rejected: kick the deferred reenq, skip wakeup/preemption */ - if (unlikely(dsq->id == SCX_DSQ_REJECT)) { - schedule_deferred_locked(rq); + /* + * Only local inserts get the wakeup treatment below. Rejects kick the + * deferred reenq and rescue parks are paced by the rescue timer. + */ + if (unlikely(dsq->id != SCX_DSQ_LOCAL)) { + if (dsq->id == SCX_DSQ_REJECT) + schedule_deferred_locked(rq); return; } @@ -1863,8 +1874,8 @@ static struct scx_dispatch_q *find_dsq_for_dispatch(struct scx_sched *sch, dsq = find_user_dsq(sch, dsq_id); /* - * Built-in DSQs are never inserted into dsq_hash, so REJECT hits the - * error below. It cannot be reached with an ID. + * Built-in DSQs are never inserted into dsq_hash, so REJECT and RESCUE + * hit the error below. They cannot be reached with an ID. */ if (unlikely(!dsq)) { scx_error(sch, "non-existent DSQ 0x%llx", dsq_id); @@ -2056,6 +2067,7 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, if (!(sch->ops.flags & SCX_OPS_ENQ_EXITING) && unlikely(p->flags & PF_EXITING)) { __scx_add_event(sch, SCX_EV_ENQ_SKIP_EXITING, 1); + enq_flags |= SCX_ENQ_RESCUE; /* avoid looping on cap rejection */ goto local; } @@ -2324,9 +2336,11 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_ * information meaningful to the BPF scheduler and can be suppressed by * skipping the callbacks if the task is !QUEUED. */ - if (SCX_HAS_OP(sch, stopping) && task_current(rq, p)) { + if (task_current(rq, p) && + (SCX_HAS_OP(sch, stopping) || unlikely(p == scx_rescuee(rq)))) { update_curr_scx(rq); - SCX_CALL_OP_TASK(sch, stopping, rq, p, false); + if (SCX_HAS_OP(sch, stopping)) + SCX_CALL_OP_TASK(sch, stopping, rq, p, false); } if (SCX_HAS_OP(sch, quiescent) && !task_on_rq_migrating(p)) @@ -2409,8 +2423,9 @@ void scx_move_local_task_to_local_dsq(struct scx_sched *sch, struct task_struct { struct scx_dispatch_q *dst_dsq = scx_resolve_local_dsq(sch, dst_rq, p, &enq_flags); - /* @dsq is locked and @p is on @dst_rq */ - lockdep_assert_held(&src_dsq->lock); + /* @p is on @dst_rq, an rq-owned @src_dsq is covered by the rq lock */ + if (!dsq_is_rq_owned(src_dsq)) + lockdep_assert_held(&src_dsq->lock); lockdep_assert_rq_held(dst_rq); WARN_ON_ONCE(p->scx.holding_cpu >= 0); @@ -3144,15 +3159,23 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, struct task_struct *next) { struct scx_sched *sch = scx_task_sched(p); + bool rescue_keep = false; /* see kick_sync_wait_bal_cb() */ smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); update_curr_scx(rq); - /* the slice is consumed, protection ends with it */ - if (!p->scx.slice) - scx_task_slice_ended(rq, p); + /* + * If the slice is consumed, protection ends with it. A rescuee + * preempted beforehand keeps going, see scx_rescue_keep(). + */ + if (!p->scx.slice) { + if (unlikely(p == scx_rescuee(rq))) + rescue_keep = scx_rescue_keep(rq, p); + if (!rescue_keep) + scx_task_slice_ended(rq, p); + } /* see dequeue_task_scx() on why we skip when !QUEUED */ if (SCX_HAS_OP(sch, stopping) && (p->scx.flags & SCX_TASK_QUEUED)) @@ -3167,15 +3190,34 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, * forcing a different task. Leave it at the head of the local * DSQ unless it was an IMMED task. IMMED tasks should not * linger on a busy CPU, reenqueue them to the BPF scheduler. + * + * An open rescue must keep @p on the local DSQ even if the + * scheduler zeroed the slice in ops.stopping() above. */ - if (p->scx.slice && !scx_bypassing(sch, cpu_of(rq))) { + if ((p->scx.slice || unlikely(p == scx_rescuee(rq))) && + !scx_bypassing(sch, cpu_of(rq))) { if (p->scx.flags & SCX_TASK_IMMED) { p->scx.flags |= SCX_TASK_REENQ_PREEMPTED; scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; } else { + u64 enq_flags = 0; + + /* + * Keep a preempted rescue going. If preempted + * by another SCX task, append to the local DSQ, + * see scx_rescue_keep(). + */ + if (unlikely(p == scx_rescuee(rq))) { + enq_flags |= SCX_ENQ_IGNORE_CAPS; + if (!rescue_keep) + enq_flags |= SCX_ENQ_HEAD; + } else { + enq_flags |= SCX_ENQ_HEAD; + } + scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0, - SCX_ENQ_HEAD); + enq_flags); } goto switch_class; } @@ -3575,6 +3617,7 @@ static void rq_online_scx(struct rq *rq) static void rq_offline_scx(struct rq *rq) { rq->scx.flags &= ~SCX_RQ_ONLINE; + scx_rescue_flush(rq); } static bool check_rq_for_timeouts(struct rq *rq) @@ -4244,7 +4287,7 @@ static bool local_task_should_reenq(struct rq *rq, struct task_struct *p, first = !(*reenq_flags & SCX_REENQ_TSR_NOT_FIRST); *reenq_flags |= SCX_REENQ_TSR_NOT_FIRST; - if (unlikely(p->scx.flags & SCX_TASK_PROTECTED)) + if (unlikely((p->scx.flags & SCX_TASK_PROTECTED) || p == scx_rescuee(rq))) return false; *reason = SCX_TASK_REENQ_KFUNC; @@ -4527,6 +4570,13 @@ bool scx_can_stop_tick(struct rq *rq) if (scx_bypassing(sch, cpu_of(rq))) return false; + /* + * A running rescuee's charging and expiry are tick-driven, see + * scx_rescue_charge(). Keep the tick while rescue is in progress. + */ + if (unlikely(p == scx_rescuee(rq))) + return false; + /* * @rq can dispatch from different DSQs, so we can't tell whether it * needs the tick or not by looking at nr_running. Allow stopping ticks @@ -6722,6 +6772,7 @@ static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s, scx_dump_line(&ns, "CPU %-4d: nr_run=%u flags=0x%x cpu_rel=%d ops_qseq=%lu ksync=%lu", cpu, rq->scx.nr_running, rq->scx.flags, rq->scx.cpu_released, rq->scx.ops_qseq, rq->scx.kick_sync); + scx_rescue_dump(&ns, rq); scx_dump_line(&ns, " curr=%s[%d] class=%ps", rq->curr->comm, rq->curr->pid, rq->curr->sched_class); if (!cpumask_empty(pcpu->cpus_to_kick)) @@ -7393,6 +7444,7 @@ static void scx_root_enable_workfn(struct kthread_work *work) } scx_discard_stale_ecaps_syncs(); + scx_rescue_set_knobs(sch); /* * Keep CPUs stable during enable so that the BPF scheduler can track @@ -7899,6 +7951,24 @@ static int bpf_scx_init_member(const struct btf_type *t, case offsetof(struct sched_ext_ops, cid_shard_size): ops->cid_shard_size = *(u32 *)(udata + moff); return 1; + case offsetof(struct sched_ext_ops, rescue_bandwidth_ppt): { + u32 bw_ppt = *(u32 *)(udata + moff); + + if (bw_ppt > SCX_RESCUE_MAX_BW_PPT && bw_ppt != SCX_RESCUE_DISABLE) + return -E2BIG; + ops->rescue_bandwidth_ppt = bw_ppt; + return 1; + } + case offsetof(struct sched_ext_ops, rescue_quantum_us): { + u32 quantum_us = *(u32 *)(udata + moff); + + if (quantum_us > SCX_RESCUE_MAX_QUANTUM_US) + return -E2BIG; + if (quantum_us && quantum_us < SCX_RESCUE_MIN_QUANTUM_US) + return -EINVAL; + ops->rescue_quantum_us = quantum_us; + return 1; + } #ifdef CONFIG_EXT_SUB_SCHED case offsetof(struct sched_ext_ops, sub_cgroup_id): ops->sub_cgroup_id = *(u64 *)(udata + moff); @@ -8521,6 +8591,7 @@ void __init init_sched_ext_class(void) BUG_ON(scx_init_dsq(&rq->scx.local_dsq, SCX_DSQ_LOCAL, NULL)); #ifdef CONFIG_EXT_SUB_SCHED BUG_ON(scx_init_dsq(&rq->scx.reject_dsq, SCX_DSQ_REJECT, NULL)); + scx_rescue_init(rq); #endif INIT_LIST_HEAD(&rq->scx.runnable_list); @@ -8570,6 +8641,11 @@ static bool scx_vet_enq_flags(struct scx_sched *sch, u64 dsq_id, u64 *enq_flags) *enq_flags |= SCX_ENQ_IMMED; } + if (unlikely((*enq_flags & SCX_ENQ_RESCUE) && !is_local)) { + scx_error(sch, "SCX_ENQ_RESCUE on a non-local DSQ 0x%llx", dsq_id); + return false; + } + return true; } @@ -10751,6 +10827,8 @@ static int __init scx_init(void) CID_OFFSET_MATCH(exit_dump_len, exit_dump_len); CID_OFFSET_MATCH(hotplug_seq, hotplug_seq); CID_OFFSET_MATCH(cid_shard_size, cid_shard_size); + CID_OFFSET_MATCH(rescue_bandwidth_ppt, rescue_bandwidth_ppt); + CID_OFFSET_MATCH(rescue_quantum_us, rescue_quantum_us); CID_OFFSET_MATCH(sub_cgroup_id, sub_cgroup_id); /* shared callbacks: the union view requires byte-for-byte offset match */ CID_OFFSET_MATCH(enqueue, enqueue); diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index d418935f1e6b..18983dbe81f4 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -924,6 +924,37 @@ struct sched_ext_ops { */ u32 cid_shard_size; + /** + * @rescue_bandwidth_ppt: Rescue execution bandwidth in parts per thousand + * + * The fraction of each CPU's time that may be consumed running tasks + * from its rescue DSQ. A higher bandwidth admits and escalates rescues + * faster, see @rescue_quantum_us. + * + * Only the root scheduler's value is used. 0 means the default of 20 + * (2%). May not exceed 250 (25%). %SCX_RESCUE_DISABLE disables rescue - + * %SCX_ENQ_RESCUE inserts are then rejected like any other insert + * lacking the caps. + */ + u32 rescue_bandwidth_ppt; + + /** + * @rescue_quantum_us: Rescue execution quantum in microseconds + * + * How much CPU time each rescue gets. Rescues run one at a time per CPU + * and admissions are paced to keep rescue execution within + * @rescue_bandwidth_ppt - with the defaults, one 5ms rescue every + * 250ms. A crowded queue round-robins on the quantum divided across the + * waiters, floored at 1ms. A stuck rescue eventually escalates to + * forced execution. A larger quantum interrupts the CPU less often but + * for longer and spaces rescues further apart. + * + * Only the root scheduler's value is used. 0 means the default (5000). + * Non-zero values must be within [1000, 100000]. Values too short for + * the kernel to meter are lifted silently. + */ + u32 rescue_quantum_us; + /** * @sub_cgroup_id: When >1, attach the scheduler as a sub-scheduler * on the specified cgroup. @@ -1058,6 +1089,8 @@ struct sched_ext_ops_cid { u32 exit_dump_len; u64 hotplug_seq; u32 cid_shard_size; + u32 rescue_bandwidth_ppt; + u32 rescue_quantum_us; u64 sub_cgroup_id; char name[SCX_OPS_NAME_LEN]; @@ -1211,6 +1244,12 @@ struct scx_event_stats { * sub-sched lacked SCX_CAP_PERF on the target cid. */ s64 SCX_EV_SUB_CIDPERF_DENIED; + + /* + * The number of times an insert carrying %SCX_ENQ_RESCUE lacked the + * caps for its cid and the task entered the rescue path. + */ + s64 SCX_EV_SUB_RESCUE; }; #define SCX_EVENTS_LIST(SCX_EVENT) \ @@ -1233,7 +1272,8 @@ struct scx_event_stats { SCX_EVENT(SCX_EV_SUB_PREEMPT_DENIED); \ SCX_EVENT(SCX_EV_SUB_KICK_DENIED); \ SCX_EVENT(SCX_EV_SUB_REENQ_DENIED); \ - SCX_EVENT(SCX_EV_SUB_CIDPERF_DENIED) + SCX_EVENT(SCX_EV_SUB_CIDPERF_DENIED); \ + SCX_EVENT(SCX_EV_SUB_RESCUE) struct scx_sched; @@ -1656,6 +1696,17 @@ enum scx_enq_flags { */ SCX_ENQ_IMMED = 1LLU << 33, + /* + * Only allowed on local DSQs. If the insert lacks the caps for the + * target cid, divert the task to the CPU's rescue path instead of + * rejecting and reenqueueing, e.g. when the task's affinity is + * restricted to cids the scheduler doesn't hold. The kernel runs + * rescued tasks on the target CPU. Rescue execution is guaranteed to + * make forward progress and is bandwidth-limited, see the + * rescue_bandwidth_ppt and rescue_quantum_us ops fields. + */ + SCX_ENQ_RESCUE = 1LLU << 34, + /* * The task being enqueued was previously enqueued on a DSQ, but was * removed and is being re-enqueued. See SCX_TASK_REENQ_* flags to find @@ -1973,6 +2024,7 @@ void scx_task_iter_unlock(struct scx_task_iter *iter); void scx_task_iter_stop(struct scx_task_iter *iter); struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter); bool scx_set_task_slice(struct task_struct *p, u64 slice); +void scx_task_slice_ended(struct rq *rq, struct task_struct *p); void scx_task_unlink_from_dsq(struct task_struct *p, struct scx_dispatch_q *dsq); void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p); void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, @@ -2371,6 +2423,7 @@ static inline struct scx_sched *scx_parent(struct scx_sched *sch) else return NULL; } + #else /* CONFIG_EXT_SUB_SCHED */ static inline bool scx_has_subs(void) { return false; } @@ -2402,6 +2455,7 @@ static inline struct scx_sched *scx_prog_sched(const struct bpf_prog_aux *aux) } static inline struct scx_sched *scx_parent(struct scx_sched *sch) { return NULL; } + #endif /* CONFIG_EXT_SUB_SCHED */ #endif /* _KERNEL_SCHED_EXT_INTERNAL_H */ diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index c30f48ee07f9..3c1f11268e7f 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -28,6 +28,11 @@ */ DEFINE_STATIC_KEY_FALSE(__scx_has_subs); +/* latched at root enable before any rescue runs */ +static s32 scx_rescue_bw_1024; +static s64 scx_rescue_quantum_ns; +static s64 scx_rescue_sat_delta_ns; + /** * scx_skip_subtree_pre - Skip @pos's subtree in a pre-order walk * @pos: current position @@ -229,18 +234,350 @@ void scx_init_root_caps(struct scx_sched *sch) } } +/* unserved remainder of @rq's rescuee's admitted slice, 0 once fully served */ +static s64 scx_rescue_slice_remaining(struct rq *rq) +{ + s64 served = rq->scx.rescue.curr->se.sum_exec_runtime - rq->scx.rescue.exec_snap; + + return max(rq->scx.rescue.slice - served, 0); +} + +/** + * scx_rescue_charge - Charge the rescuee's runtime + * @rq: rq the rescuee is running on + * @delta_exec: runtime being charged + * + * Also ends the rescue once the admitted slice has been served in full. Ending + * on served time rather than slice exhaustion bounds both the rescue and the + * charging when a scheduler extends the rescuee's slice. + */ +void scx_rescue_charge(struct rq *rq, s64 delta_exec) +{ + lockdep_assert_rq_held(rq); + + /* + * A rescue slice is bounded by one quantum and tick-driven expiry can + * overshoot by up to a tick. Clamp to avoid wild over-charges on VMs. + */ + delta_exec = min_t(s64, delta_exec, scx_rescue_quantum_ns + TICK_NSEC); + + rq->scx.rescue.budget -= delta_exec; + + if (!scx_rescue_slice_remaining(rq)) + scx_task_slice_ended(rq, rq->scx.rescue.curr); +} + /** - * scx_resolve_local_dsq - Pick the local or reject DSQ for an insert + * scx_rescue_end - End the rescue execution on @rq + * @rq: rq of interest + * + * When no rescuee is left pending, the session is over and the balance above + * one quantum dies with it - it would otherwise become a banked license to + * preempt the cid owner long after the starvation ended. While waiters remain, + * the accrued deficit belongs to the queue and carries into the next rescue. + */ +void scx_rescue_end(struct rq *rq) +{ + lockdep_assert_rq_held(rq); + + rq->scx.rescue.curr = NULL; + if (list_empty(&rq->scx.rescue.dsq.list)) + rq->scx.rescue.budget = min(rq->scx.rescue.budget, scx_rescue_quantum_ns); +} + +/** + * scx_rescue_keep - Keep the rescue going for a preempted-out rescuee + * @rq: rq @p is running on + * @p: task under rescue whose slice is exhausted + * + * Called from put_prev_task_scx() to decide what an exhausted slice means for + * the rescuee. scx_rescue_charge() ends the rescue the moment the admitted + * slice is fully served, so arriving here with the rescue still open means @p + * was preempted. Restore the unserved remainder and return %true - @p stays the + * rescuee and the caller reinserts it at the tail of the local DSQ, behind + * whatever preempted the rescuee. + * + * Return %false to end the rescue instead - the slice is already fully served, + * @p is leaving the rq or bypass is dismantling rescues. + */ +bool scx_rescue_keep(struct rq *rq, struct task_struct *p) +{ + s64 remaining = scx_rescue_slice_remaining(rq); + + lockdep_assert_rq_held(rq); + + if (!remaining || !(p->scx.flags & SCX_TASK_QUEUED) || + scx_bypassing(scx_task_sched(p), cpu_of(rq))) + return false; + + scx_set_task_slice(p, remaining); + return true; +} + +/** + * scx_rescue_accrue - Accrue budget at the configured fraction of elapsed time + * @rq: rq of interest + * + * A session spans from the first arrival until no rescuee is left, pending or + * admitted. While one is active the cap is three quanta and the balance drives + * escalation, see scx_rescue_timerfn(). Outside a session the cap is one + * quantum, so an idle gap funds the next arrival's admission but never an + * escalation. + */ +static void scx_rescue_accrue(struct rq *rq) +{ + bool in_session = rq->scx.rescue.curr || !list_empty(&rq->scx.rescue.dsq.list); + s64 cap = in_session ? 3 * scx_rescue_quantum_ns : scx_rescue_quantum_ns; + s64 delta; + u64 now; + + lockdep_assert_rq_held(rq); + + /* not every path here holds an updated rq clock, use __scx_bpf_now() */ + now = __scx_bpf_now(rq); + delta = now - rq->scx.rescue.clock; + rq->scx.rescue.clock = now; + + /* + * Avoid multiplication overflows by taking a shortcut when the gap is + * large enough to fill the budget. + */ + if (delta >= scx_rescue_sat_delta_ns) + rq->scx.rescue.budget = cap; + else + rq->scx.rescue.budget = + min(cap, rq->scx.rescue.budget + + ((delta * scx_rescue_bw_1024) >> SCHED_CAPACITY_SHIFT)); +} + +/* + * The slice for the next admission - the quantum divided across the stranded + * tasks so that a crowded queue round-robins on shorter slices. + */ +static s64 scx_rescue_next_slice(struct rq *rq) +{ + s64 min_slice = max_t(s64, SCX_RESCUE_MIN_SLICE_US * NSEC_PER_USEC, TICK_NSEC); + u32 depth = rq->scx.rescue.dsq.nr ?: 1; + + return clamp(div_s64(scx_rescue_quantum_ns, depth), min_slice, scx_rescue_quantum_ns); +} + +static void scx_rescue_timer_arm(struct rq *rq) +{ + struct timer_list *timer = &rq->scx.rescue.timer; + s64 delay = scx_rescue_quantum_ns / 4; /* should be granular enough */ + + if (timer_pending(timer)) + return; + + /* + * While the head waiter can't be admitted because the bucket is short + * of a full quantum, stretch to the full funding delay. + */ + if (!rq->scx.rescue.curr && rq->scx.rescue.budget < scx_rescue_quantum_ns) { + s64 deficit = scx_rescue_quantum_ns - rq->scx.rescue.budget; + + delay = max(delay, + div_s64(deficit << SCHED_CAPACITY_SHIFT, scx_rescue_bw_1024)); + } + + /* +1 rounds up so the beat is due by the time the timer fires */ + timer->expires = jiffies + nsecs_to_jiffies(delay) + 1; + add_timer_on(timer, cpu_of(rq)); +} + +/** + * scx_rescue_admit - Start rescuing @p on @rq + * @rq: rq @p is being admitted on + * @p: task being admitted, off any DSQ + * @slice: CPU time to grant + * + * The schedulers keep their normal control over @p and may preempt or reslice + * it. @slice is measured on served CPU time against the snapshot taken here, so + * neither shortens the rescue, see scx_rescue_charge() and scx_rescue_keep(). + * Prolonged denial escalates into protected execution, see + * scx_rescue_timerfn(). + */ +static void scx_rescue_admit(struct rq *rq, struct task_struct *p, s64 slice) +{ + lockdep_assert_rq_held(rq); + WARN_ON_ONCE(rq->scx.rescue.curr); + + rq->scx.rescue.curr = p; + rq->scx.rescue.slice = slice; + rq->scx.rescue.exec_snap = p->se.sum_exec_runtime; + scx_set_task_slice(p, slice); + scx_rescue_timer_arm(rq); +} + +/** + * scx_rescue_try_admit - Try to admit a freshly stranded task + * @rq: rq @p is being inserted on + * @p: stranded task being diverted to rescue + * + * One rescue at a time and earlier arrivals go first. Admission needs a full + * quantum of budget, spent as the rescue runs. Return %true if @p was admitted + * and should be inserted at the tail of @rq's local DSQ, %false if it has to + * park on the rescue DSQ, with the timer armed to admit it later. + */ +static bool scx_rescue_try_admit(struct rq *rq, struct task_struct *p) +{ + scx_rescue_accrue(rq); + + if (!rq->scx.rescue.curr && list_empty(&rq->scx.rescue.dsq.list) && + rq->scx.rescue.budget >= scx_rescue_quantum_ns) { + scx_rescue_admit(rq, p, scx_rescue_quantum_ns); + return true; + } + + scx_rescue_timer_arm(rq); + return false; +} + +/** + * scx_rescue_timerfn - Drive and pace rescue execution + * @timer: rq->scx.rescue.timer + * + * Runs every quarter quantum while a rescuee exists, pending or admitted, see + * scx_rescue_timer_arm(). The head waiter is admitted once the bucket holds a + * full quantum and granted its slice, see scx_rescue_next_slice(). A session + * whose budget accumulates over two quanta with the admitted rescuee still + * waiting escalates - the rescuee's remaining slice turns into protected + * execution and it preempts the current task. + */ +static void scx_rescue_timerfn(struct timer_list *timer) +{ + struct rq *rq = timer_container_of(rq, timer, scx.rescue.timer); + struct task_struct *p; + + guard(rq_lock_irqsave)(rq); + + p = rq->scx.rescue.curr; + if (!p && list_empty(&rq->scx.rescue.dsq.list)) + return; + + scx_rescue_accrue(rq); + + if (!p) { + s64 slice = scx_rescue_next_slice(rq); + + /* no rescue in progress */ + if (rq->scx.rescue.budget < scx_rescue_quantum_ns) + goto out_arm; + + /* there's enough budget to start rescuing the next one */ + p = list_first_entry(&rq->scx.rescue.dsq.list, struct task_struct, + scx.dsq_list.node); + scx_task_unlink_from_dsq(p, &rq->scx.rescue.dsq); + scx_rescue_admit(rq, p, slice); + scx_move_local_task_to_local_dsq(scx_task_sched(p), p, SCX_ENQ_IGNORE_CAPS, + &rq->scx.rescue.dsq, rq); + if (sched_class_above(&ext_sched_class, rq->curr->sched_class)) + resched_curr(rq); + } else if (p->scx.dsq && rq->scx.rescue.budget > 2 * scx_rescue_quantum_ns) { + /* + * The rescuee waited for the CPU for too long. Escalate - grant + * the unserved remainder, protect it from the schedulers and + * preempt the current task. The slice is set before the + * protection. Repeat beats only repeat the head move - the + * slice write is refused on a protected task. + */ + scx_set_task_slice(p, scx_rescue_slice_remaining(rq)); + p->scx.flags |= SCX_TASK_PROTECTED; + scx_task_unlink_from_dsq(p, &rq->scx.local_dsq); + scx_move_local_task_to_local_dsq(scx_task_sched(p), p, + SCX_ENQ_HEAD | SCX_ENQ_PREEMPT | SCX_ENQ_IGNORE_CAPS, + &rq->scx.local_dsq, rq); + } +out_arm: + scx_rescue_timer_arm(rq); +} + +/* flush out tasks waiting for rescue before a CPU goes down */ +void scx_rescue_flush(struct rq *rq) +{ + struct task_struct *p, *n; + + lockdep_assert_rq_held(rq); + + /* sched domain rebuilds call rq_offline with the CPU staying alive */ + if (cpu_active(cpu_of(rq))) + return; + + /* end the current rescue */ + if (rq->scx.rescue.curr) + scx_task_slice_ended(rq, rq->scx.rescue.curr); + + /* and flush out all pending ones */ + list_for_each_entry_safe(p, n, &rq->scx.rescue.dsq.list, scx.dsq_list.node) { + scx_task_unlink_from_dsq(p, &rq->scx.rescue.dsq); + scx_move_local_task_to_local_dsq(scx_task_sched(p), p, SCX_ENQ_IGNORE_CAPS, + &rq->scx.rescue.dsq, rq); + } + + timer_delete(&rq->scx.rescue.timer); +} + +void scx_rescue_dump(struct seq_buf *s, struct rq *rq) +{ + struct task_struct *p = rq->scx.rescue.curr; + + scx_dump_line(s, " rescue=%u budget=%lldus rescuing=%s[%d]", + rq->scx.rescue.dsq.nr, + div_s64(rq->scx.rescue.budget, NSEC_PER_USEC), + p ? p->comm : "none", p ? p->pid : -1); +} + +/* latch the rescue parameters on root scheduler enable */ +void scx_rescue_set_knobs(struct scx_sched *sch) +{ + s32 bw_ppt = sch->ops.rescue_bandwidth_ppt ?: SCX_RESCUE_DFL_BW_PPT; + s64 quantum_us = sch->ops.rescue_quantum_us ?: SCX_RESCUE_DFL_QUANTUM_US; + + if (sch->ops.rescue_bandwidth_ppt == SCX_RESCUE_DISABLE) { + scx_rescue_bw_1024 = 0; + return; + } + + scx_rescue_bw_1024 = bw_ppt * SCHED_CAPACITY_SCALE / 1000; + scx_rescue_quantum_ns = max(quantum_us * NSEC_PER_USEC, TICK_NSEC); + scx_rescue_sat_delta_ns = + div_s64((4 * scx_rescue_quantum_ns + TICK_NSEC) << SCHED_CAPACITY_SHIFT, + scx_rescue_bw_1024); + + /* + * A rescued task is guaranteed to run after two full periods - one to + * be admitted, one more to escalate. Require the two periods to fit in + * a quarter of the watchdog timeout, so one full period may take at + * most an eighth. + */ + if (div_s64(scx_rescue_quantum_ns << SCHED_CAPACITY_SHIFT, scx_rescue_bw_1024) > + jiffies_to_nsecs(sch->watchdog_timeout) / 8) + pr_warn("sched_ext: rescue may not run a stuck task before the %ums watchdog timeout, decrease rescue_quantum_us or increase rescue_bandwidth_ppt\n", + jiffies_to_msecs(sch->watchdog_timeout)); +} + +void scx_rescue_init(struct rq *rq) +{ + BUG_ON(scx_init_dsq(&rq->scx.rescue.dsq, SCX_DSQ_RESCUE, NULL)); + timer_setup(&rq->scx.rescue.timer, scx_rescue_timerfn, TIMER_PINNED); +} + +/** + * scx_resolve_local_dsq - Pick the local, rescue or reject DSQ for an insert * @sch: enqueuing sub-sched * @rq: rq whose local DSQ @p targets * @p: task being inserted * @enq_flags: in/out, unhonored flags are cleared * - * Return @rq's local DSQ if @sch holds the required caps on @rq's cid, - * otherwise @rq's reject DSQ after recording the reenq reason on @p. + * Return @rq's local DSQ if @sch holds the required caps on @rq's cid. + * Otherwise, return @rq's rescue DSQ if the insert carries %SCX_ENQ_RESCUE and + * rescue is enabled, or @rq's reject DSQ after recording the reenq reason on + * @p. * - * %SCX_ENQ_IMMED and %SCX_ENQ_PREEMPT are cleared when diverting to reject. - * %SCX_ENQ_PREEMPT is also cleared on a fallback migration-disabled admission. + * %SCX_ENQ_IMMED, %SCX_ENQ_PREEMPT and %SCX_ENQ_HEAD are cleared when diverting + * to rescue or reject. %SCX_ENQ_PREEMPT is also cleared on a fallback + * migration-disabled admission. * * Bypass doesn't need special-casing as a bypassing sched's tasks are enqueued * to and run by its nearest non-bypassing ancestor. If root is bypassing, it @@ -282,18 +619,27 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r return &rq->scx.local_dsq; } - p->scx.reenq_reason_caps = missing; - p->scx.reenq_reason_cid = cid; - /* - * Only local DSQ can honor IMMED and dsq_inc_nr() WARNs on IMMED into - * others. Strip both the enq flag and the sticky task flag - the - * latter can carry in from an earlier admitted IMMED insert. Strip - * PREEMPT too. + * Diverting to rescue or reject, neither of which honors IMMED, PREEMPT + * or HEAD - a diversion has no priority and IMMED is not allowed on + * non-local DSQs. Strip the enq and task flags along with the slice. */ - *enq_flags &= ~(SCX_ENQ_IMMED | SCX_ENQ_PREEMPT); + *enq_flags &= ~(SCX_ENQ_IMMED | SCX_ENQ_PREEMPT | SCX_ENQ_HEAD | + SCX_ENQ_APPLY_SLICE | SCX_ENQ_SLICE_DFL); p->scx.flags &= ~SCX_TASK_IMMED; + /* the enqueuer opted for rescue instead of rejection and reenqueue */ + if ((*enq_flags & SCX_ENQ_RESCUE) && likely(scx_rescue_bw_1024)) { + __scx_add_event(sch, SCX_EV_SUB_RESCUE, 1); + if (scx_rescue_try_admit(rq, p)) + return &rq->scx.local_dsq; + else + return &rq->scx.rescue.dsq; + } + + p->scx.reenq_reason_caps = missing; + p->scx.reenq_reason_cid = cid; + return &rq->scx.reject_dsq; } @@ -302,8 +648,8 @@ bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p) { u64 missing; - /* migration-disabled tasks are admitted regardless of caps */ - if (is_migration_disabled(p)) + /* migration-disabled tasks and the rescuee are admitted capless */ + if (is_migration_disabled(p) || p == scx_rescuee(rq)) return false; missing = scx_missing_caps(scx_task_sched(p), cpu_of(rq), scx_caps_for_task(p)); diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index fe1d82e6c1d5..f7bcdfda8dd8 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -38,6 +38,13 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r struct task_struct *p, u64 *enq_flags); bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p); void scx_reenq_reject(struct rq *rq); +void scx_rescue_charge(struct rq *rq, s64 delta_exec); +void scx_rescue_end(struct rq *rq); +bool scx_rescue_keep(struct rq *rq, struct task_struct *p); +void scx_rescue_flush(struct rq *rq); +void scx_rescue_dump(struct seq_buf *s, struct rq *rq); +void scx_rescue_set_knobs(struct scx_sched *sch); +void scx_rescue_init(struct rq *rq); /* * cgrp->scx_sched is written by root/sub enable/disable under all of @@ -87,6 +94,13 @@ static inline void scx_discard_stale_ecaps_syncs(void) {} static inline struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, struct task_struct *p, u64 *enq_flags) { return &rq->scx.local_dsq; } static inline bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p) { return false; } static inline void scx_reenq_reject(struct rq *rq) {} +static inline void scx_rescue_charge(struct rq *rq, s64 delta_exec) {} +static inline void scx_rescue_end(struct rq *rq) {} +static inline bool scx_rescue_keep(struct rq *rq, struct task_struct *p) { return false; } +static inline void scx_rescue_flush(struct rq *rq) {} +static inline void scx_rescue_dump(struct seq_buf *s, struct rq *rq) {} +static inline void scx_rescue_set_knobs(struct scx_sched *sch) {} +static inline void scx_rescue_init(struct rq *rq) {} static inline void scx_dec_has_subs(struct scx_sched *sch) {} #endif /* CONFIG_EXT_SUB_SCHED */ @@ -195,11 +209,23 @@ static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p return likely(!scx_missing_caps(scx_task_sched(p), cpu_of(rq), SCX_CAP_BASE)); } +/* the task admitted for rescue on @rq, NULL if none */ +static inline struct task_struct *scx_rescuee(struct rq *rq) +{ + lockdep_assert_rq_held(rq); + + if (!scx_has_subs()) + return NULL; + + return rq->scx.rescue.curr; +} + #else /* CONFIG_EXT_SUB_SCHED */ static inline u64 scx_missing_caps(struct scx_sched *sch, s32 cpu, u64 needed) { return 0; } static inline u64 scx_caps_for_preempt(struct scx_sched *sch, struct rq *rq, u64 enq_flags) { return 0; } static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p) { return true; } +static inline struct task_struct *scx_rescuee(struct rq *rq) { return NULL; } #endif /* CONFIG_EXT_SUB_SCHED */ diff --git a/kernel/sched/ext/types.h b/kernel/sched/ext/types.h index b94ddee21c57..d39588717e9b 100644 --- a/kernel/sched/ext/types.h +++ b/kernel/sched/ext/types.h @@ -19,6 +19,15 @@ enum scx_consts { SCX_DSP_MAX_LOOPS = 32, SCX_WATCHDOG_MAX_TIMEOUT = 30 * HZ, + /* rescue knob defaults and limits, see scx_rescue_timerfn() */ + SCX_RESCUE_DFL_BW_PPT = 20, /* parts per thousand, 2% */ + SCX_RESCUE_MAX_BW_PPT = 250, /* 25% */ + SCX_RESCUE_DISABLE = U32_MAX, /* disables rescue */ + SCX_RESCUE_DFL_QUANTUM_US = 5000, + SCX_RESCUE_MIN_QUANTUM_US = 1000, + SCX_RESCUE_MAX_QUANTUM_US = 100000, + SCX_RESCUE_MIN_SLICE_US = 1000, /* floor of the divided slice */ + /* per-CPU chunk size for p->scx.tid allocation, see scx_alloc_tid() */ SCX_TID_CHUNK = 1024, diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index c0cb879d75f0..289e298df628 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -794,10 +794,22 @@ enum scx_rq_flags { SCX_RQ_IN_BALANCE = 1 << 17, }; +/* per-rq rescue execution state, see scx_rescue_timerfn() */ +struct scx_rq_rescue { + struct scx_dispatch_q dsq; /* stranded tasks awaiting rescue */ + s64 budget; /* execution token bucket, ns */ + u64 clock; /* last budget accrual timestamp */ + struct task_struct *curr; /* task being rescued, one at a time */ + s64 slice; /* curr's admitted slice */ + u64 exec_snap; /* sum_exec_runtime at admission */ + struct timer_list timer; /* paces admission and escalation */ +}; + struct scx_rq { struct scx_dispatch_q local_dsq; #ifdef CONFIG_EXT_SUB_SCHED struct scx_dispatch_q reject_dsq; /* staging for cap-rejected tasks */ + struct scx_rq_rescue rescue; #endif struct list_head runnable_list; /* runnable tasks on this rq */ struct list_head ddsp_deferred_locals; /* deferred ddsps from enq */ diff --git a/tools/sched_ext/include/scx/compat.h b/tools/sched_ext/include/scx/compat.h index 7757252d52e2..d2e4384df5af 100644 --- a/tools/sched_ext/include/scx/compat.h +++ b/tools/sched_ext/include/scx/compat.h @@ -175,6 +175,7 @@ static inline long scx_hotplug_seq(void) * - v6.17: ops.cgroup_set_bandwidth() * - v6.19: ops.cgroup_set_idle() * - v7.1: ops.sub_attach(), ops.sub_detach(), ops.sub_cgroup_id + * - v7.3: ops.rescue_bandwidth_ppt, ops.rescue_quantum_us */ #define __SCX_OPS_OPEN(__ops_name, __scx_name, __ops_struct) ({ \ struct __scx_name *__oskel; \ @@ -218,6 +219,16 @@ static inline long scx_hotplug_seq(void) fprintf(stderr, "WARNING: kernel doesn't support ops.sub_cgroup_id\n"); \ __skel->struct_ops.__ops_name->sub_cgroup_id = 0; \ } \ + if (__skel->struct_ops.__ops_name->rescue_bandwidth_ppt > 0 && \ + !__COMPAT_struct_has_field("sched_ext_ops", "rescue_bandwidth_ppt")) { \ + fprintf(stderr, "WARNING: kernel doesn't support ops.rescue_bandwidth_ppt\n"); \ + __skel->struct_ops.__ops_name->rescue_bandwidth_ppt = 0; \ + } \ + if (__skel->struct_ops.__ops_name->rescue_quantum_us > 0 && \ + !__COMPAT_struct_has_field("sched_ext_ops", "rescue_quantum_us")) { \ + fprintf(stderr, "WARNING: kernel doesn't support ops.rescue_quantum_us\n"); \ + __skel->struct_ops.__ops_name->rescue_quantum_us = 0; \ + } \ __skel; \ }) -- cgit v1.2.3 From bb70e4fb626b70895b7917ee97c256f24d019c34 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 3 Aug 2026 11:01:36 -1000 Subject: sched_ext: Eject the top rescue consumer on overload When rescue demand on a cpu persistently exceeds the configured bandwidth, tasks age on that cpu's rescue DSQ until the stall watchdog fires. The watchdog blames the waiting task's owner, but the misbehaving party is whoever floods the queue, not whoever happens to time out. Track each sched's recent rescue consumption per cpu as a decaying average. Once the oldest waiter on a cpu's rescue DSQ has been queued past a threshold derived from the rescue knobs (4s at the defaults), the rescue timer ejects the sub with the highest recent consumption on that cpu with SCX_EXIT_ERROR_RESCUE. With no recent consumer there is no victim and nothing is ejected - the generic stall watchdog eventually blames the waiter's owner instead. Ejections on a cpu are spaced one threshold apart so the freed bandwidth can drain the backlog before another sub is judged. The overload check only wins the race against the stall watchdog when the watchdog timeout clears the threshold, and a single in-budget wait must not cross the trigger on its own. Warn on a scheduler whose timeout doesn't fit and on knobs whose funding period exceeds half the threshold. v2: - Track kill_at in jiffies_64 - on 32-bit, the time_before() grace check wraps 2^31 ticks after the last ejection and suppresses ejections. (sashiko AI) - Track rescue_avg_at in jiffies_64 likewise - the unsigned long decay delta truncates mod 2^32 on 32-bit and can revive a weeks-old usage average in the victim pick. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- include/linux/sched/ext.h | 3 + kernel/sched/ext/ext.c | 2 + kernel/sched/ext/internal.h | 9 +++ kernel/sched/ext/sub.c | 137 ++++++++++++++++++++++++++++++++++++++++---- kernel/sched/ext/types.h | 3 + kernel/sched/sched.h | 1 + 6 files changed, 144 insertions(+), 11 deletions(-) (limited to 'kernel') diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index a6aabbefd185..a3ec980e2925 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -215,6 +215,9 @@ struct sched_ext_entity { #ifdef CONFIG_SCHED_CORE u64 core_sched_at; /* see scx_prio_less() */ #endif +#ifdef CONFIG_EXT_SUB_SCHED + unsigned long rescue_at; /* queued on a rescue DSQ at, jiffies */ +#endif /* * Unique non-zero task ID assigned at fork. Persists across exec and diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 7f86d4adcd0d..0bbe144c9811 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -6140,6 +6140,8 @@ static const char *scx_exit_reason(enum scx_exit_kind kind) return "runnable task stall"; case SCX_EXIT_ERROR_REENQ: return "reenqueue limit"; + case SCX_EXIT_ERROR_RESCUE: + return "rescue bandwidth overload"; default: return ""; } diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 18983dbe81f4..b699e7c1103f 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -57,6 +57,7 @@ enum scx_exit_kind { SCX_EXIT_ERROR_BPF, /* ERROR but triggered through scx_bpf_error() */ SCX_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */ SCX_EXIT_ERROR_REENQ, /* task hit reenqueue limit without running */ + SCX_EXIT_ERROR_RESCUE, /* ejected for overloading rescue execution */ }; /* @@ -1340,6 +1341,14 @@ struct scx_sched_pcpu { bool idle_renotify; /* effective caps as of the last sub_ecaps_updated() delivery */ u64 reported_ecaps; + + /* + * Decaying rescue runtime consumed on this cpu, see + * scx_rescue_decay_avg(). Overload on this cpu ejects the sub with the + * largest value. Accessed only under this cpu's rq lock. + */ + u64 rescue_avg; + u64 rescue_avg_at; /* last decay, jiffies_64 */ #endif /* diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 3c1f11268e7f..b81254be1b04 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -32,6 +32,8 @@ DEFINE_STATIC_KEY_FALSE(__scx_has_subs); static s32 scx_rescue_bw_1024; static s64 scx_rescue_quantum_ns; static s64 scx_rescue_sat_delta_ns; +static unsigned long scx_rescue_decay_halflife; +static unsigned long scx_rescue_overload_after; /** * scx_skip_subtree_pre - Skip @pos's subtree in a pre-order walk @@ -242,6 +244,23 @@ static s64 scx_rescue_slice_remaining(struct rq *rq) return max(rq->scx.rescue.slice - served, 0); } +/* + * Decay @pcpu's rescue usage average in place, halving per the knob-derived + * halflife, see scx_rescue_set_knobs(). The timestamp advances only by whole + * halflives. + */ +static u64 scx_rescue_decay_avg(struct scx_sched_pcpu *pcpu) +{ + unsigned long halflife = scx_rescue_decay_halflife; + u64 n = div_u64(get_jiffies_64() - pcpu->rescue_avg_at, halflife); + + if (n) { + pcpu->rescue_avg = n < 64 ? pcpu->rescue_avg >> n : 0; + pcpu->rescue_avg_at += n * halflife; + } + return pcpu->rescue_avg; +} + /** * scx_rescue_charge - Charge the rescuee's runtime * @rq: rq the rescuee is running on @@ -253,6 +272,8 @@ static s64 scx_rescue_slice_remaining(struct rq *rq) */ void scx_rescue_charge(struct rq *rq, s64 delta_exec) { + struct scx_sched_pcpu *pcpu; + lockdep_assert_rq_held(rq); /* @@ -263,6 +284,10 @@ void scx_rescue_charge(struct rq *rq, s64 delta_exec) rq->scx.rescue.budget -= delta_exec; + /* per-cpu usage average feeds the overload victim pick */ + pcpu = per_cpu_ptr(scx_task_sched(rq->curr)->pcpu, cpu_of(rq)); + pcpu->rescue_avg = scx_rescue_decay_avg(pcpu) + delta_exec; + if (!scx_rescue_slice_remaining(rq)) scx_task_slice_ended(rq, rq->scx.rescue.curr); } @@ -434,6 +459,64 @@ static bool scx_rescue_try_admit(struct rq *rq, struct task_struct *p) return false; } +/** + * scx_rescue_check_overload - Eject the top rescue consumer on a stuck rescue + * @rq: rq whose rescue timer fired + * + * If the oldest waiter on @rq's rescue DSQ has been queued for too long, rescue + * demand on this cpu persistently exceeds the configured bandwidth. Eject the + * sub with the highest recent rescue consumption instead of letting the + * scheduler stall path blame the waiter's owner, who may just be crowded out. + */ +static void scx_rescue_check_overload(struct rq *rq) +{ + struct scx_sched *victim = NULL, *pos; + struct task_struct *p; + int cpu = cpu_of(rq); + u64 max_avg = 0; + u32 dur_ms; + + lockdep_assert_rq_held(rq); + + p = list_first_entry_or_null(&rq->scx.rescue.dsq.list, struct task_struct, + scx.dsq_list.node); + if (!p) + return; + + /* has the head waiter been queued for longer than the threshold? */ + if (time_before(jiffies, p->scx.rescue_at + scx_rescue_overload_after)) + return; + + /* + * Grace period after the last ejection on this cpu - the freed + * bandwidth gets one threshold's worth of time to drain the backlog + * before another sub is judged. + */ + if (time_before64(get_jiffies_64(), rq->scx.rescue.kill_at + + scx_rescue_overload_after)) + return; + + list_for_each_entry_rcu(pos, &scx_sched_all, all) { + u64 avg = scx_rescue_decay_avg(per_cpu_ptr(pos->pcpu, cpu)); + + /* skip an already-exiting sub, else the ejection is wasted */ + if (pos->level && avg > max_avg && + atomic_read(&pos->exit_kind) == SCX_EXIT_NONE) { + max_avg = avg; + victim = pos; + } + } + if (!victim) + return; + + rq->scx.rescue.kill_at = get_jiffies_64(); + dur_ms = jiffies_to_msecs(jiffies - p->scx.rescue_at); + __scx_exit(victim, SCX_EXIT_ERROR_RESCUE, 0, cpu, + "used too much rescue CPU time (%llums) while %s[%d] waited %u.%03us to be rescued", + div_u64(max_avg, NSEC_PER_MSEC), p->comm, p->pid, dur_ms / 1000, + dur_ms % 1000); +} + /** * scx_rescue_timerfn - Drive and pace rescue execution * @timer: rq->scx.rescue.timer @@ -443,7 +526,8 @@ static bool scx_rescue_try_admit(struct rq *rq, struct task_struct *p) * full quantum and granted its slice, see scx_rescue_next_slice(). A session * whose budget accumulates over two quanta with the admitted rescuee still * waiting escalates - the rescuee's remaining slice turns into protected - * execution and it preempts the current task. + * execution and it preempts the current task. An overloaded rescue queue ejects + * the top consumer, see scx_rescue_check_overload(). */ static void scx_rescue_timerfn(struct timer_list *timer) { @@ -457,6 +541,7 @@ static void scx_rescue_timerfn(struct timer_list *timer) return; scx_rescue_accrue(rq); + scx_rescue_check_overload(rq); if (!p) { s64 slice = scx_rescue_next_slice(rq); @@ -528,11 +613,28 @@ void scx_rescue_dump(struct seq_buf *s, struct rq *rq) p ? p->comm : "none", p ? p->pid : -1); } +/* + * A scheduler whose stall watchdog is shorter than the overload threshold gets + * stall-killed over its parked waiters before the overload check can eject the + * actual top consumer. The root's knobs set the threshold, warn on any + * scheduler that doesn't fit it. + */ +static void scx_rescue_check_timeout(struct scx_sched *sch) +{ + if (!scx_rescue_bw_1024 || sch->watchdog_timeout > scx_rescue_overload_after) + return; + + pr_warn("sched_ext: %s: watchdog timeout %ums <= rescue overload threshold %ums\n", + sch->ops.name, jiffies_to_msecs(sch->watchdog_timeout), + jiffies_to_msecs(scx_rescue_overload_after)); +} + /* latch the rescue parameters on root scheduler enable */ void scx_rescue_set_knobs(struct scx_sched *sch) { s32 bw_ppt = sch->ops.rescue_bandwidth_ppt ?: SCX_RESCUE_DFL_BW_PPT; s64 quantum_us = sch->ops.rescue_quantum_us ?: SCX_RESCUE_DFL_QUANTUM_US; + s64 period_ns; if (sch->ops.rescue_bandwidth_ppt == SCX_RESCUE_DISABLE) { scx_rescue_bw_1024 = 0; @@ -546,21 +648,30 @@ void scx_rescue_set_knobs(struct scx_sched *sch) scx_rescue_bw_1024); /* - * A rescued task is guaranteed to run after two full periods - one to - * be admitted, one more to escalate. Require the two periods to fit in - * a quarter of the watchdog timeout, so one full period may take at - * most an eighth. + * The overload threshold and the decay halflife scale with the funding + * period - the time the bucket takes to fund one full quantum. */ - if (div_s64(scx_rescue_quantum_ns << SCHED_CAPACITY_SHIFT, scx_rescue_bw_1024) > - jiffies_to_nsecs(sch->watchdog_timeout) / 8) - pr_warn("sched_ext: rescue may not run a stuck task before the %ums watchdog timeout, decrease rescue_quantum_us or increase rescue_bandwidth_ppt\n", - jiffies_to_msecs(sch->watchdog_timeout)); + period_ns = div_s64(scx_rescue_quantum_ns << SCHED_CAPACITY_SHIFT, scx_rescue_bw_1024); + scx_rescue_overload_after = + clamp(nsecs_to_jiffies(SCX_RESCUE_OVERLOAD_MULT * period_ns), + msecs_to_jiffies(SCX_RESCUE_MIN_OVERLOAD_MS), + msecs_to_jiffies(SCX_RESCUE_MAX_OVERLOAD_MS)); + scx_rescue_decay_halflife = scx_rescue_overload_after / 4; + + /* a single in-budget wait must not cross the overload trigger */ + if (nsecs_to_jiffies(period_ns) > scx_rescue_overload_after / 2) + pr_warn("sched_ext: %s: rescue funding period %lldms > overload threshold %ums / 2\n", + sch->ops.name, div_s64(period_ns, NSEC_PER_MSEC), + jiffies_to_msecs(scx_rescue_overload_after)); + + scx_rescue_check_timeout(sch); } void scx_rescue_init(struct rq *rq) { BUG_ON(scx_init_dsq(&rq->scx.rescue.dsq, SCX_DSQ_RESCUE, NULL)); timer_setup(&rq->scx.rescue.timer, scx_rescue_timerfn, TIMER_PINNED); + rq->scx.rescue.kill_at = get_jiffies_64(); } /** @@ -633,8 +744,10 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r __scx_add_event(sch, SCX_EV_SUB_RESCUE, 1); if (scx_rescue_try_admit(rq, p)) return &rq->scx.local_dsq; - else - return &rq->scx.rescue.dsq; + + /* queueing, the overload trigger measures the wait from here */ + p->scx.rescue_at = jiffies; + return &rq->scx.rescue.dsq; } p->scx.reenq_reason_caps = missing; @@ -1654,6 +1767,8 @@ void scx_sub_enable_workfn(struct kthread_work *work) if (ret) goto err_disable; + scx_rescue_check_timeout(sch); + /* * Allocate pshard[] before scx_link_sched() publishes @sch into the * parent's RCU children list. A concurrent revoke walking the tree diff --git a/kernel/sched/ext/types.h b/kernel/sched/ext/types.h index d39588717e9b..1eb3ac8508f6 100644 --- a/kernel/sched/ext/types.h +++ b/kernel/sched/ext/types.h @@ -27,6 +27,9 @@ enum scx_consts { SCX_RESCUE_MIN_QUANTUM_US = 1000, SCX_RESCUE_MAX_QUANTUM_US = 100000, SCX_RESCUE_MIN_SLICE_US = 1000, /* floor of the divided slice */ + SCX_RESCUE_OVERLOAD_MULT = 16, /* overload threshold in funding periods */ + SCX_RESCUE_MIN_OVERLOAD_MS = 1000, + SCX_RESCUE_MAX_OVERLOAD_MS = 15000, /* per-CPU chunk size for p->scx.tid allocation, see scx_alloc_tid() */ SCX_TID_CHUNK = 1024, diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 289e298df628..63786712a115 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -803,6 +803,7 @@ struct scx_rq_rescue { s64 slice; /* curr's admitted slice */ u64 exec_snap; /* sum_exec_runtime at admission */ struct timer_list timer; /* paces admission and escalation */ + u64 kill_at; /* last ejection, init before any */ }; struct scx_rq { -- cgit v1.2.3 From c10b216a072ff5c57bc880a05f87eb519aecc529 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 7 Aug 2026 11:02:16 -1000 Subject: sched/core: Handle pick_task() releasing the rq lock Core scheduling's pick_next_task() breaks when a ->pick_task() implementation can release the rq lock. The selection state derived on entry is only valid while the lock is held continuously. Once a pick can drop the lock, an interleaving selection can invalidate all of it: the single-CPU fast path can commit an uncookied pick although the core went cookied during the release, and forceidle committed by the interleaving selection skews the restarted pass's accounting. Fix it by restarting the whole selection when a pick returns RETRY_TASK after releasing the lock: a single restart point above the state derivation replaces the per-loop restart labels, so a retry picks up state committed by interleaving selections and accounts and resets forceidle like a fresh selection would. need_sync and fi_before latch across retries. Clock validity can't be re-derived - there is no program-ordered way to tell whether the own and core rq clocks are still updated after the lock was released, as other lockers' pin cycles may or may not have invalidated them. When restarting, clear core_clock_updated so that the sibling loop re-updates the core rq, and update the own rq clock if invalidated. Fixes: 4c95380701f5 ("sched/ext: Fold balance_scx() into pick_task_scx()") Cc: stable@vger.kernel.org # v6.19+ Signed-off-by: Tejun Heo Acked-by: Peter Zijlstra (Intel) --- kernel/sched/core.c | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 3cc6fb1d2054..84ef83316562 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -6217,7 +6217,7 @@ pick_next_task(struct rq *rq, struct rq_flags *rf) unsigned long cookie; int i, cpu, occ = 0; struct rq *rq_i; - bool need_sync; + bool need_sync = false; if (!sched_core_enabled(rq)) return __pick_next_task(rq, rf); @@ -6260,7 +6260,9 @@ pick_next_task(struct rq *rq, struct rq_flags *rf) prev_balance(rq, rf); smt_mask = cpu_smt_mask(cpu); - need_sync = !!rq->core->core_cookie; + +restart: + need_sync |= !!rq->core->core_cookie; /* reset state */ rq->core->core_cookie = 0UL; @@ -6295,10 +6297,15 @@ pick_next_task(struct rq *rq, struct rq_flags *rf) * and there are no cookied tasks running on siblings. */ if (!need_sync) { -restart_single: next = pick_task(rq, rf); - if (unlikely(next == RETRY_TASK)) - goto restart_single; + if (unlikely(next == RETRY_TASK)) { + /* rq lock may have been dropped, clocks invalidated */ + core_clock_updated = false; + if (!(rq->clock_update_flags & RQCF_UPDATED)) + update_rq_clock(rq); + goto restart; + } + if (!next->core_cookie) { rq->core_pick = NULL; rq->core_dl_server = NULL; @@ -6318,7 +6325,6 @@ restart_single: * * Tie-break prio towards the current CPU */ -restart_multi: max = NULL; for_each_cpu_wrap(i, smt_mask, cpu) { rq_i = cpu_rq(i); @@ -6332,8 +6338,13 @@ restart_multi: update_rq_clock(rq_i); p = pick_task(rq_i, rf); - if (unlikely(p == RETRY_TASK)) - goto restart_multi; + if (unlikely(p == RETRY_TASK)) { + /* rq lock may have been dropped, clocks invalidated */ + core_clock_updated = false; + if (!(rq->clock_update_flags & RQCF_UPDATED)) + update_rq_clock(rq); + goto restart; + } rq_i->core_pick = p; rq_i->core_dl_server = rq_i->dl_server; -- cgit v1.2.3 From f3629c63a4af3e491381780bc6c123cb498c4c40 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 7 Aug 2026 11:02:17 -1000 Subject: sched/core: Make core-sched flips wait for in-flight selections Core scheduling's pick_next_task() operates on all sibling rqs under one acquisition of the shared core-wide lock. A ->pick_task() that releases the rq lock leaves every sibling __lock momentarily free, letting __sched_core_flip(false) complete mid-selection and rebind rq_lockp() under it. The selection resumes on the split locks, touching sibling state it no longer protects, and __schedule() finally releases a lock that was never taken while leaking the one that was. Count in-flight core-wide selections in the leader's rq->core_pick_in_flight and make __sched_core_flip() wait for the count to drain. The count only changes under the shared lock, which the flip holds while sampling, so no other ordering is needed. The wait can repeat while selections overlap, but the flip backs off between samples and flips are rare cookie-lifetime events. sched_core_cpu_deactivate() moves the count to the new leader - a stale copy left behind would bias it forever if that CPU later returns as its own leader. Fixes: 539f65125d20 ("sched: Add core wide task selection and scheduling") Cc: stable@vger.kernel.org # v5.14+ Signed-off-by: Tejun Heo Acked-by: Peter Zijlstra (Intel) --- kernel/sched/core.c | 22 ++++++++++++++++++++++ kernel/sched/sched.h | 1 + 2 files changed, 23 insertions(+) (limited to 'kernel') diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 84ef83316562..0130463798f8 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -443,6 +443,17 @@ static void __sched_core_flip(bool enabled) sched_core_lock(cpu, &flags); + /* + * A core-wide selection may have the shared rq lock temporarily + * released by a lock-dropping ->pick_task(). Flipping would + * rebind rq_lockp() under it. Wait it out. + */ + while (cpu_rq(cpu)->core->core_pick_in_flight) { + sched_core_unlock(cpu, &flags); + cpu_relax(); + sched_core_lock(cpu, &flags); + } + for_each_cpu(t, smt_mask) cpu_rq(t)->core_enabled = enabled; @@ -6236,6 +6247,8 @@ pick_next_task(struct rq *rq, struct rq_flags *rf) return __pick_next_task(rq, rf); } + rq->core->core_pick_in_flight++; + /* * If there were no {en,de}queues since we picked (IOW, the task * pointers are all still valid), and we haven't scheduled the last @@ -6450,6 +6463,7 @@ restart: } out_set_next: + rq->core->core_pick_in_flight--; put_prev_set_next_task(rq, rq->donor, next); if (rq->core->core_forceidle_count && next == rq->idle) queue_core_balance(rq); @@ -6644,6 +6658,13 @@ static void sched_core_cpu_deactivate(unsigned int cpu) core_rq->core_forceidle_seq = rq->core_forceidle_seq; core_rq->core_forceidle_occupation = rq->core_forceidle_occupation; + /* + * A stale leftover would bias the count forever if this CPU later + * returns as its own leader. Move, don't copy. + */ + core_rq->core_pick_in_flight = rq->core_pick_in_flight; + rq->core_pick_in_flight = 0; + /* * Accounting edge for forced idle is handled in pick_next_task(). * Don't need another one here, since the hotplug thread shouldn't @@ -9060,6 +9081,7 @@ void __init sched_init(void) rq->core_forceidle_count = 0; rq->core_forceidle_occupation = 0; rq->core_forceidle_start = 0; + rq->core_pick_in_flight = 0; rq->core_cookie = 0UL; #endif diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 56acf502ba26..450b6a04669f 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1358,6 +1358,7 @@ struct rq { unsigned int core_forceidle_seq; unsigned int core_forceidle_occupation; u64 core_forceidle_start; + unsigned int core_pick_in_flight; #endif /* CONFIG_SCHED_CORE */ /* Scratch cpumask to be temporarily used under rq_lock */ -- cgit v1.2.3 From ffaab58d217581cb75353168f8812a16e10463fc Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 7 Aug 2026 11:02:18 -1000 Subject: sched_ext: Replace SCX_RQ_BAL_KEEP with a dispatch verdict return SCX_RQ_BAL_KEEP tells the pick to keep running the previous task, a leftover from when balancing and picking were separate operations. An rq-level flag only works while dispatches and picks pair up one to one, which core scheduling breaks: selections interleave through dispatch's lock drops and a pick can consume a stale flag, keeping a task that has since been dequeued. Fixing core scheduling support requires the decision to travel with the dispatch that made it. Make scx_dispatch_sched() and balance_one() return an explicit verdict instead and drop the flag's plumbing from the tools autogen enum headers. Also factor the pick-side invocation, its follow-up queueing and the post-dispatch checks out of do_pick_task_scx() into dispatch_pick(). No functional changes intended. v2: Drop the SCX_RQ_BAL_KEEP plumbing from the tools autogen enum headers as well (Andrea). Fixes: 4c95380701f5 ("sched/ext: Fold balance_scx() into pick_task_scx()") Cc: stable@vger.kernel.org # v6.19+ Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 124 ++++++++++++++---------- kernel/sched/sched.h | 1 - tools/sched_ext/include/scx/enum_defs.autogen.h | 1 - tools/sched_ext/include/scx/enums.autogen.bpf.h | 3 - tools/sched_ext/include/scx/enums.autogen.h | 1 - 5 files changed, 73 insertions(+), 57 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 18183062f751..ffbe4f7edc99 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -2774,12 +2774,19 @@ static inline void maybe_queue_balance_callback(struct rq *rq) rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING; } +/* what dispatch concluded, consumed by the pick that follows */ +enum scx_dsp_verdict { + SCX_DSP_NONE, /* nothing to run */ + SCX_DSP_LOCAL, /* local DSQ has tasks */ + SCX_DSP_PREV, /* keep running @prev */ +}; + /* * One user of this function is scx_bpf_dispatch() which can be called * recursively as sub-sched dispatches nest. Always inline to reduce stack usage * from the call frame. */ -static __always_inline bool +static __always_inline enum scx_dsp_verdict scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, struct task_struct *prev, bool nested) { @@ -2790,12 +2797,15 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, scx_task_on_sched(sch, prev); if (consume_global_dsq(sch, rq)) - return true; + return SCX_DSP_LOCAL; if (bypass_dsp_enabled(sch)) { /* if @sch is bypassing, only the bypass DSQs are active */ - if (scx_bypassing(sch, cpu)) - return consume_dispatch_q(sch, rq, bypass_dsq(sch, cpu), 0); + if (scx_bypassing(sch, cpu)) { + if (consume_dispatch_q(sch, rq, bypass_dsq(sch, cpu), 0)) + return SCX_DSP_LOCAL; + return SCX_DSP_NONE; + } #ifdef CONFIG_EXT_SUB_SCHED /* @@ -2815,13 +2825,13 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, if (!(pcpu->bypass_host_seq++ % SCX_BYPASS_HOST_NTH) && consume_dispatch_q(sch, rq, bypass_dsq(sch, cpu), 0)) { __scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1); - return true; + return SCX_DSP_LOCAL; } #endif /* CONFIG_EXT_SUB_SCHED */ } if (unlikely(!SCX_HAS_OP(sch, dispatch)) || !scx_rq_online(rq)) - return false; + return SCX_DSP_NONE; dspc->rq = rq; @@ -2848,14 +2858,12 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, flush_dispatch_buf(sch, rq); - if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice) { - rq->scx.flags |= SCX_RQ_BAL_KEEP; - return true; - } + if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice) + return SCX_DSP_PREV; if (rq->scx.local_dsq.nr) - return true; + return SCX_DSP_LOCAL; if (consume_global_dsq(sch, rq)) - return true; + return SCX_DSP_LOCAL; /* * ops.dispatch() can trap us in this loop by repeatedly @@ -2877,20 +2885,20 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, * queued. Without this fallback, bypassed tasks could stall if the host * scheduler's ops.dispatch() doesn't yield any tasks. */ - if (bypass_dsp_enabled(sch)) - return consume_dispatch_q(sch, rq, bypass_dsq(sch, cpu), 0); + if (bypass_dsp_enabled(sch) && consume_dispatch_q(sch, rq, bypass_dsq(sch, cpu), 0)) + return SCX_DSP_LOCAL; - return false; + return SCX_DSP_NONE; } -static int balance_one(struct rq *rq, struct task_struct *prev) +static enum scx_dsp_verdict balance_one(struct rq *rq, struct task_struct *prev) { struct scx_sched *sch = scx_root; + enum scx_dsp_verdict verdict; s32 cpu = cpu_of(rq); lockdep_assert_rq_held(rq); rq->scx.flags |= SCX_RQ_IN_BALANCE; - rq->scx.flags &= ~SCX_RQ_BAL_KEEP; if ((sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT) && unlikely(rq->scx.cpu_released)) { @@ -2920,16 +2928,19 @@ static int balance_one(struct rq *rq, struct task_struct *prev) */ if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice && !scx_bypassing(sch, cpu)) { - rq->scx.flags |= SCX_RQ_BAL_KEEP; + verdict = SCX_DSP_PREV; goto has_tasks; } } /* if there already are tasks to run, nothing to do */ - if (rq->scx.local_dsq.nr) + if (rq->scx.local_dsq.nr) { + verdict = SCX_DSP_LOCAL; goto has_tasks; + } - if (scx_dispatch_sched(sch, rq, prev, false)) + verdict = scx_dispatch_sched(sch, rq, prev, false); + if (verdict != SCX_DSP_NONE) goto has_tasks; /* @@ -2938,12 +2949,12 @@ static int balance_one(struct rq *rq, struct task_struct *prev) */ if ((prev->scx.flags & SCX_TASK_QUEUED) && (!(sch->ops.flags & SCX_OPS_ENQ_LAST) || scx_bypassing(sch, cpu))) { - rq->scx.flags |= SCX_RQ_BAL_KEEP; __scx_add_event(sch, SCX_EV_DISPATCH_KEEP_LAST, 1); + verdict = SCX_DSP_PREV; goto has_tasks; } rq->scx.flags &= ~SCX_RQ_IN_BALANCE; - return false; + return SCX_DSP_NONE; has_tasks: /* @@ -2960,7 +2971,7 @@ has_tasks: schedule_reenq_local(rq, 0); rq->scx.flags &= ~SCX_RQ_IN_BALANCE; - return true; + return verdict; } static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first) @@ -3179,27 +3190,23 @@ static struct task_struct *first_local_task(struct rq *rq) struct task_struct, scx.dsq_list.node); } -static struct task_struct * -do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) +/* + * Run dispatch and queue the follow-up work for a pick. + */ +static enum scx_dsp_verdict dispatch_pick(struct rq *rq, struct rq_flags *rf, + struct task_struct *prev) { - struct task_struct *prev = rq->curr; - bool keep_prev; - struct task_struct *p; - - /* see kick_sync_wait_bal_cb() */ - smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); - - rq_modified_begin(rq, &ext_sched_class); + enum scx_dsp_verdict verdict; rq_unpin_lock(rq, rf); - balance_one(rq, prev); + verdict = balance_one(rq, prev); rq_repin_lock(rq, rf); maybe_queue_balance_callback(rq); /* - * Defer to a balance callback which can drop rq lock and enable - * IRQs. Waiting directly in the pick path would deadlock against - * CPUs sending us IPIs (e.g. TLB flushes) while we wait for them. + * Defer to a balance callback which can drop rq lock and enable IRQs. + * Waiting directly in the pick path would deadlock against CPUs sending + * us IPIs (e.g. TLB flushes) while we wait for them. */ if (unlikely(rq->scx.kick_sync_pending)) { rq->scx.kick_sync_pending = false; @@ -3207,10 +3214,32 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) kick_sync_wait_bal_cb); } + if (unlikely(verdict == SCX_DSP_PREV && prev->sched_class != &ext_sched_class)) { + WARN_ON_ONCE(scx_enable_state() == SCX_ENABLED); + verdict = SCX_DSP_LOCAL; + } + + return verdict; +} + +static struct task_struct * +do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) +{ + struct task_struct *prev = rq->curr; + enum scx_dsp_verdict verdict; + struct task_struct *p; + + /* see kick_sync_wait_bal_cb() */ + smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); + + rq_modified_begin(rq, &ext_sched_class); + + verdict = dispatch_pick(rq, rf, prev); + /* - * If any higher-priority sched class enqueued a runnable task on - * this rq during balance_one(), abort and return RETRY_TASK, so - * that the scheduler loop can restart. + * If any higher-priority sched class enqueued a runnable task on this + * rq during balance_one(), abort and return RETRY_TASK, so that the + * scheduler loop can restart. * * If @force_scx is true, always try to pick a SCHED_EXT task, * regardless of any higher-priority sched classes activity. @@ -3218,19 +3247,12 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) if (!force_scx && rq_modified_above(rq, &ext_sched_class)) return RETRY_TASK; - keep_prev = rq->scx.flags & SCX_RQ_BAL_KEEP; - if (unlikely(keep_prev && - prev->sched_class != &ext_sched_class)) { - WARN_ON_ONCE(scx_enable_state() == SCX_ENABLED); - keep_prev = false; - } - /* * If balance_one() is telling us to keep running @prev, replenish slice * if necessary and keep running @prev. Otherwise, pop the first one * from the local DSQ. */ - if (keep_prev) { + if (verdict == SCX_DSP_PREV) { p = prev; if (!p->scx.slice) refill_task_slice_dfl(scx_task_sched(p), p); @@ -5573,7 +5595,7 @@ static void disable_bypass_dsp(struct scx_sched *sch) * * - ops.dispatch() is ignored. * - * - balance_one() does not set %SCX_RQ_BAL_KEEP on non-zero slice as slice + * - balance_one() does not report %SCX_DSP_PREV on non-zero slice as slice * can't be trusted. Whenever a tick triggers, the running task is rotated to * the tail of the queue with core_sched_at touched. * @@ -9201,8 +9223,8 @@ __bpf_kfunc bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux * return false; } - return scx_dispatch_sched(child, this_rq, this_rq->scx.sub_dispatch_prev, - true); + return scx_dispatch_sched(child, this_rq, this_rq->scx.sub_dispatch_prev, true) != + SCX_DSP_NONE; } #endif /* CONFIG_EXT_SUB_SCHED */ diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 450b6a04669f..14c2df6fd9af 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -784,7 +784,6 @@ enum scx_rq_flags { */ SCX_RQ_ONLINE = 1 << 0, SCX_RQ_CAN_STOP_TICK = 1 << 1, - SCX_RQ_BAL_KEEP = 1 << 3, /* balance decided to keep current */ SCX_RQ_CLK_VALID = 1 << 5, /* RQ clock is fresh and valid */ SCX_RQ_BAL_CB_PENDING = 1 << 6, /* must queue a cb after dispatching */ diff --git a/tools/sched_ext/include/scx/enum_defs.autogen.h b/tools/sched_ext/include/scx/enum_defs.autogen.h index da4b459820fd..8e287c88ad90 100644 --- a/tools/sched_ext/include/scx/enum_defs.autogen.h +++ b/tools/sched_ext/include/scx/enum_defs.autogen.h @@ -143,7 +143,6 @@ #define HAVE___SCX_REENQ_TSR_MASK #define HAVE_SCX_RQ_ONLINE #define HAVE_SCX_RQ_CAN_STOP_TICK -#define HAVE_SCX_RQ_BAL_KEEP #define HAVE_SCX_RQ_CLK_VALID #define HAVE_SCX_RQ_BAL_CB_PENDING #define HAVE_SCX_RQ_IN_WAKEUP diff --git a/tools/sched_ext/include/scx/enums.autogen.bpf.h b/tools/sched_ext/include/scx/enums.autogen.bpf.h index dafccbb6b69d..d02abed94e8e 100644 --- a/tools/sched_ext/include/scx/enums.autogen.bpf.h +++ b/tools/sched_ext/include/scx/enums.autogen.bpf.h @@ -22,9 +22,6 @@ const volatile u64 __SCX_RQ_CAN_STOP_TICK __weak; const volatile u64 __SCX_RQ_BAL_PENDING __weak; #define SCX_RQ_BAL_PENDING __SCX_RQ_BAL_PENDING -const volatile u64 __SCX_RQ_BAL_KEEP __weak; -#define SCX_RQ_BAL_KEEP __SCX_RQ_BAL_KEEP - const volatile u64 __SCX_RQ_BYPASSING __weak; #define SCX_RQ_BYPASSING __SCX_RQ_BYPASSING diff --git a/tools/sched_ext/include/scx/enums.autogen.h b/tools/sched_ext/include/scx/enums.autogen.h index bbd4901f4fce..c26934a3d8e9 100644 --- a/tools/sched_ext/include/scx/enums.autogen.h +++ b/tools/sched_ext/include/scx/enums.autogen.h @@ -11,7 +11,6 @@ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_ONLINE); \ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_CAN_STOP_TICK); \ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_BAL_PENDING); \ - SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_BAL_KEEP); \ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_BYPASSING); \ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_CLK_VALID); \ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_IN_WAKEUP); \ -- cgit v1.2.3 From 3dd52416e44a70bc993adb96d2e0d71b9ea21359 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 7 Aug 2026 11:02:19 -1000 Subject: sched_ext: Fix this_rq() assumptions in dispatch kfuncs Under core scheduling, dispatch runs from within the core-wide pick and can target a sibling rq, so ops.dispatch() may execute on a CPU different from the dispatched rq's. Several kfunc paths assumed the two always coincide: - scx_dsq_move() decided whether an rq lock is held by testing this_rq()'s rq flags and lock-danced accordingly. A dispatch for a sibling took the unlocked-context branch and acquired the source rq lock on top of the already held dispatched rq lock which could deadlock. - scx_bpf_sub_dispatch() dispatched this_rq() with its stashed sub_dispatch_prev, which is NULL when dispatching for a sibling. - finish_dispatch(), scx_bpf_dsq_reenq() and scx_bpf_dsq_nr_queued() resolved SCX_DSQ_LOCAL to this CPU's local DSQ rather than the dispatched rq's. The latter two are callable from other rq-locked operations too, where SCX_DSQ_LOCAL now likewise resolves to the op's rq. This changes behavior also without core scheduling, e.g. for ops.enqueue() running a remote wakeup on the waking CPU, and is intended: which CPU happens to execute an operation is incidental, the op's rq is what it is operating on, and the resolution now matches the insert side where SCX_DSQ_LOCAL dispatches land on the task's rq. Use the rq tracked by scx_locked_rq(), which is set to the dispatched rq around ops invocations and NULL in unlocked contexts. Fixes: 4c95380701f5 ("sched/ext: Fold balance_scx() into pick_task_scx()") Cc: stable@vger.kernel.org # v6.19+ Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 58 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 23 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index ffbe4f7edc99..84ec71d28b61 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -2737,7 +2737,7 @@ retry: BUG_ON(!(p->scx.flags & SCX_TASK_QUEUED)); - dsq = find_dsq_for_dispatch(sch, this_rq(), dsq_id, task_cpu(p)); + dsq = find_dsq_for_dispatch(sch, rq, dsq_id, task_cpu(p)); if (dsq->id == SCX_DSQ_LOCAL) dispatch_to_local_dsq(sch, rq, dsq, p, enq_flags); @@ -8887,9 +8887,8 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, { struct scx_dispatch_q *src_dsq = kit->dsq, *dst_dsq; struct scx_sched *sch; - struct rq *this_rq, *src_rq, *locked_rq; + struct rq *p_rq, *src_rq, *locked_rq; bool dispatched = false; - bool in_balance; unsigned long flags; /* @@ -8919,24 +8918,28 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, } /* - * Can be called from either ops.dispatch() locking this_rq() or any - * context where no rq lock is held. If latter, lock @p's task_rq which - * we'll likely need anyway. + * Can be called from either ops.dispatch() holding the dispatched rq's + * lock or any context where no rq lock is held. If latter, lock @p's + * task_rq which we'll likely need anyway. */ src_rq = task_rq(p); local_irq_save(flags); - this_rq = this_rq(); - in_balance = this_rq->scx.flags & SCX_RQ_IN_BALANCE; - if (in_balance) { - if (this_rq != src_rq) - switch_rq_lock(this_rq, src_rq); + /* + * Under core scheduling, dispatch can run for a sibling rq, so the + * locked rq is not necessarily this CPU's. + */ + locked_rq = scx_locked_rq(); + + if (locked_rq) { + if (locked_rq != src_rq) + switch_rq_lock(locked_rq, src_rq); } else { raw_spin_rq_lock(src_rq); } - locked_rq = src_rq; + p_rq = src_rq; raw_spin_lock(&src_dsq->lock); /* did someone else get to it while we dropped the locks? */ @@ -8946,7 +8949,7 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, } /* @p is still on $src_dsq and stable, determine the destination */ - dst_dsq = find_dsq_for_dispatch(sch, this_rq, dsq_id, task_cpu(p)); + dst_dsq = find_dsq_for_dispatch(sch, locked_rq ?: this_rq(), dsq_id, task_cpu(p)); /* * Apply vtime and slice updates before moving so that the new time is @@ -8959,14 +8962,14 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, p->scx.slice = kit->slice; /* execute move */ - locked_rq = move_task_between_dsqs(sch, p, enq_flags, src_dsq, dst_dsq); + p_rq = move_task_between_dsqs(sch, p, enq_flags, src_dsq, dst_dsq); dispatched = true; out: - if (in_balance) { - if (this_rq != locked_rq) - switch_rq_lock(locked_rq, this_rq); + if (locked_rq) { + if (locked_rq != p_rq) + switch_rq_lock(p_rq, locked_rq); } else { - raw_spin_rq_unlock_irqrestore(locked_rq, flags); + raw_spin_rq_unlock_irqrestore(p_rq, flags); } kit->cursor.flags &= ~(__SCX_DSQ_ITER_HAS_SLICE | @@ -9204,7 +9207,7 @@ __bpf_kfunc bool scx_bpf_dsq_move_vtime(struct bpf_iter_scx_dsq *it__iter, */ __bpf_kfunc bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux *aux) { - struct rq *this_rq = this_rq(); + struct rq *rq = scx_locked_rq(); struct scx_sched *parent, *child; guard(rcu)(); @@ -9223,7 +9226,7 @@ __bpf_kfunc bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux * return false; } - return scx_dispatch_sched(child, this_rq, this_rq->scx.sub_dispatch_prev, true) != + return scx_dispatch_sched(child, rq, rq->scx.sub_dispatch_prev, true) != SCX_DSP_NONE; } #endif /* CONFIG_EXT_SUB_SCHED */ @@ -9518,6 +9521,10 @@ __bpf_kfunc s32 scx_bpf_kick_cid(s32 cid, u64 flags, const struct bpf_prog_aux * * * Return the number of tasks in the DSQ matching @dsq_id. If not found, * -%ENOENT is returned. + * + * %SCX_DSQ_LOCAL resolves to the local DSQ of the rq the current scheduler + * operation is locked to - e.g. the rq being dispatched for in ops.dispatch() - + * or the calling CPU's when no rq is locked. */ __bpf_kfunc s32 scx_bpf_dsq_nr_queued(u64 dsq_id, const struct bpf_prog_aux *aux) { @@ -9534,7 +9541,7 @@ __bpf_kfunc s32 scx_bpf_dsq_nr_queued(u64 dsq_id, const struct bpf_prog_aux *aux } if (dsq_id == SCX_DSQ_LOCAL) { - ret = READ_ONCE(this_rq()->scx.local_dsq.nr); + ret = READ_ONCE((scx_locked_rq() ?: this_rq())->scx.local_dsq.nr); goto out; } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { s32 cpu = scx_cpu_ret(sch, dsq_id & SCX_DSQ_LOCAL_CPU_MASK); @@ -9713,10 +9720,15 @@ __bpf_kfunc struct task_struct *scx_bpf_dsq_peek(u64 dsq_id, * - User DSQs * * Re-enqueues are performed asynchronously. Can be called from anywhere. + * + * %SCX_DSQ_LOCAL resolves to the local DSQ of the rq the current scheduler + * operation is locked to - e.g. the rq being dispatched for in ops.dispatch() - + * or the calling CPU's when no rq is locked. */ __bpf_kfunc void scx_bpf_dsq_reenq(u64 dsq_id, u64 reenq_flags, const struct bpf_prog_aux *aux) { + struct rq *locked_rq = scx_locked_rq(); struct scx_sched *sch; struct scx_dispatch_q *dsq; @@ -9735,8 +9747,8 @@ __bpf_kfunc void scx_bpf_dsq_reenq(u64 dsq_id, u64 reenq_flags, if (!(reenq_flags & __SCX_REENQ_FILTER_MASK)) reenq_flags |= SCX_REENQ_ANY; - dsq = find_dsq_for_dispatch(sch, this_rq(), dsq_id, smp_processor_id()); - schedule_dsq_reenq(sch, dsq, reenq_flags, scx_locked_rq()); + dsq = find_dsq_for_dispatch(sch, locked_rq ?: this_rq(), dsq_id, smp_processor_id()); + schedule_dsq_reenq(sch, dsq, reenq_flags, locked_rq); } /** -- cgit v1.2.3 From f2da9587118d5da41a3f81a59642d5bb4782ddcc Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 7 Aug 2026 11:02:20 -1000 Subject: sched_ext: Count rq lock releases in rq->scx.lock_drop_seq Under core scheduling, pick_next_task() selects for all SMT siblings under one continuous hold of the shared core-wide rq lock, and sched_ext's dispatch can release that lock from inside the pick. In preparation for making the core-sched pick detect the releases and retry, add rq->scx.lock_drop_seq and bump it at every site that can release an rq lock while a dispatch may be in flight. The counter is only maintained while core scheduling is enabled. No functional changes. Fixes: 4c95380701f5 ("sched/ext: Fold balance_scx() into pick_task_scx()") Cc: stable@vger.kernel.org # v6.19+ Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 38 ++++++++++++++++++++++++++++++++++++-- kernel/sched/sched.h | 3 +++ 2 files changed, 39 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 84ec71d28b61..ebe40953acf0 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -479,12 +479,27 @@ static bool rq_is_open(struct rq *rq, u64 enq_flags) */ DEFINE_PER_CPU(struct rq *, scx_locked_rq_state); +/* + * Under core scheduling, a pick that releases the rq lock invalidates the + * core-wide selection it is part of. Count the releases so that the core-sched + * pick can tell whether one happened across dispatch. + */ +static void scx_rq_lock_drop(struct rq *rq) +{ + lockdep_assert_rq_held(rq); +#ifdef CONFIG_SCHED_CORE + if (sched_core_enabled(rq)) + rq->scx.lock_drop_seq++; +#endif +} + static void switch_rq_lock(struct rq *from, struct rq *to) { bool tracked = scx_locked_rq() == from; if (tracked) update_locked_rq(NULL); + scx_rq_lock_drop(from); raw_spin_rq_unlock(from); raw_spin_rq_lock(to); if (tracked) @@ -1133,6 +1148,7 @@ static void deferred_irq_workfn(struct irq_work *irq_work) raw_spin_rq_lock(rq); run_deferred(rq); + scx_rq_lock_drop(rq); raw_spin_rq_unlock(rq); } @@ -2430,12 +2446,14 @@ static bool consume_remote_task(struct rq *this_rq, struct task_struct *p, u64 enq_flags, struct scx_dispatch_q *dsq, struct rq *src_rq) { + scx_rq_lock_drop(this_rq); raw_spin_rq_unlock(this_rq); if (unlink_dsq_and_lock_src_rq(p, dsq, src_rq)) { move_remote_task_to_local_dsq(p, enq_flags, src_rq, this_rq); return true; } else { + scx_rq_lock_drop(src_rq); raw_spin_rq_unlock(src_rq); raw_spin_rq_lock(this_rq); return false; @@ -3171,6 +3189,7 @@ retry: continue; } + scx_rq_lock_drop(rq); raw_spin_rq_unlock_irq(rq); while (READ_ONCE(cpu_rq(cpu)->scx.kick_sync) == ksyncs[cpu]) { smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); @@ -4280,8 +4299,10 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag task_rq = task_rq(p); if (locked_rq != task_rq) { - if (locked_rq) + if (locked_rq) { + scx_rq_lock_drop(locked_rq); raw_spin_rq_unlock(locked_rq); + } if (unlikely(!raw_spin_rq_trylock(task_rq))) { raw_spin_unlock(&dsq->lock); raw_spin_rq_lock(task_rq); @@ -4307,6 +4328,7 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; if (!(++nr_enqueued % SCX_TASK_ITER_BATCH)) { + scx_rq_lock_drop(locked_rq); raw_spin_rq_unlock(locked_rq); locked_rq = NULL; cpu_relax(); @@ -4319,8 +4341,10 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag raw_spin_unlock(&dsq->lock); if (locked_rq != rq) { - if (locked_rq) + if (locked_rq) { + scx_rq_lock_drop(locked_rq); raw_spin_rq_unlock(locked_rq); + } raw_spin_rq_lock(rq); } } @@ -5372,6 +5396,7 @@ resume: if (!(nr_balanced % SCX_BYPASS_LB_BATCH) && n) { list_move_tail(&cursor.node, &n->scx.dsq_list.node); raw_spin_unlock(&donor_dsq->lock); + scx_rq_lock_drop(donor_rq); raw_spin_rq_unlock_irq(donor_rq); cpu_relax(); raw_spin_rq_lock_irq(donor_rq); @@ -5382,6 +5407,7 @@ resume: list_del_init(&cursor.node); raw_spin_unlock(&donor_dsq->lock); + scx_rq_lock_drop(donor_rq); raw_spin_rq_unlock_irq(donor_rq); return nr_balanced; @@ -5672,6 +5698,7 @@ static void scx_bypass(struct scx_sched *sch, bool bypass) * sees scx_bypassing() before moving tasks to SCX. */ if (!scx_enabled()) { + scx_rq_lock_drop(rq); raw_spin_rq_unlock(rq); continue; } @@ -5698,6 +5725,7 @@ static void scx_bypass(struct scx_sched *sch, bool bypass) if (cpu_online(cpu) || cpu == smp_processor_id()) resched_curr(rq); + scx_rq_lock_drop(rq); raw_spin_rq_unlock(rq); } @@ -8425,6 +8453,7 @@ static bool kick_one_cpu(s32 cpu, struct rq *this_rq, unsigned long *ksyncs) cpumask_clear_cpu(cpu, this_scx->cpus_to_wait); } + scx_rq_lock_drop(rq); raw_spin_rq_unlock_irqrestore(rq, flags); return should_wait; @@ -8441,6 +8470,7 @@ static void kick_one_cpu_if_idle(s32 cpu, struct rq *this_rq) (cpu_online(cpu) || cpu == cpu_of(this_rq))) resched_curr(rq); + scx_rq_lock_drop(rq); raw_spin_rq_unlock_irqrestore(rq, flags); } @@ -8478,6 +8508,7 @@ static void kick_cpus_irq_workfn(struct irq_work *irq_work) raw_spin_rq_lock(this_rq); this_scx->kick_sync_pending = true; resched_curr(this_rq); + scx_rq_lock_drop(this_rq); raw_spin_rq_unlock(this_rq); } } @@ -8969,6 +9000,7 @@ out: if (locked_rq != p_rq) switch_rq_lock(p_rq, locked_rq); } else { + scx_rq_lock_drop(p_rq); raw_spin_rq_unlock_irqrestore(p_rq, flags); } @@ -9448,9 +9480,11 @@ static void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags) if (raw_spin_rq_trylock(target_rq)) { if (can_skip_idle_kick(target_rq)) { + scx_rq_lock_drop(target_rq); raw_spin_rq_unlock(target_rq); goto out; } + scx_rq_lock_drop(target_rq); raw_spin_rq_unlock(target_rq); } cpumask_set_cpu(cpu, this_rq->scx.cpus_to_kick_if_idle); diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 14c2df6fd9af..f76e6f7f4a01 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -803,6 +803,9 @@ struct scx_rq { bool cpu_released; u32 flags; u32 nr_immed; /* ENQ_IMMED tasks on local_dsq */ +#ifdef CONFIG_SCHED_CORE + u32 lock_drop_seq; /* nr dispatch lock releases */ +#endif u64 clock; /* current per-rq clock -- see scx_bpf_now() */ cpumask_var_t cpus_to_kick; cpumask_var_t cpus_to_kick_if_idle; -- cgit v1.2.3 From d954004205c1a1d3f59ce8482b559266c15600fa Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 7 Aug 2026 11:02:21 -1000 Subject: sched_ext: Fix rq->core_pick corruption under core scheduling Core scheduling's pick_next_task() picks what to run on every SMT sibling of the core in a single pass under the shared core-wide rq lock. The selection state is consistent only while the lock is held continuously, so ->pick_task() originally could not release it. However, since 4c95380701f5 ("sched/ext: Fold balance_scx() into pick_task_scx()"), sched_ext runs dispatch from inside the pick and dispatching can drop the rq lock. To support this, pick_next_task() has been updated to restart the whole selection when a pick returns RETRY_TASK after releasing the lock. When selections on the same core interleave through the dropped lock, they corrupt each other's state: one clears the other's rq->core_pick leading to a NULL deref, or invalidates its keep-the-previous-task decision leaving a dequeued task running, which deadlocks the next wakeup and matches the reported hard hangs. A cookied ping-pong load on an SMT machine makes the interleavings frequent and kills the kernel within seconds. Fix it by making the pick return RETRY_TASK whenever dispatch released the rq lock, so that a selection only ever commits picks made under a continuously held lock. The previous patch's rq->scx.lock_drop_seq counts the releases. A dispatch that touched nothing never releases the lock and its verdict, including "nothing to run", stands: retries are bounded, each following a dispatch that actually did something, and an idle CPU does not loop. If another dispatch is already in flight on the rq, skip dispatching and pick from what is already queued locally - the in-flight dispatch has released the lock, so its own selection will retry and re-pick this rq, while returning RETRY_TASK here would only spin on the lock that dispatch needs to finish. Balance callbacks must run in the context that queued them, so they can only be queued on the CPU's own rq. When dispatching for another rq, run the deferred work directly instead - that rq may consume all its picks through the core-sched fast path and never queue the callback itself. The put_prev_task_scx() warning about a runnable task being left behind assumed that dispatch ran as part of the very pick that is switching away. That now only holds on the non-core path, so gate it and drop the cookie-match test, which is always true without core scheduling, from its condition. Fixes: 4c95380701f5 ("sched/ext: Fold balance_scx() into pick_task_scx()") Cc: stable@vger.kernel.org # v6.19+ Reported-by: ElXreno Link: https://github.com/sched-ext/scx/issues/3715 Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 86 +++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 78 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index ebe40953acf0..c70458885487 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -2797,6 +2797,7 @@ enum scx_dsp_verdict { SCX_DSP_NONE, /* nothing to run */ SCX_DSP_LOCAL, /* local DSQ has tasks */ SCX_DSP_PREV, /* keep running @prev */ + SCX_DSP_RETRY, /* pick helpers only: restart the pick */ }; /* @@ -3142,12 +3143,12 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, * ops.enqueue() that @p is the only one available for this cpu, * which should trigger an explicit follow-up scheduling event. * - * Core scheduling can force this CPU idle while @p stays - * runnable. @p's cookie then won't match the core's, so skip - * the warning in that case. + * Under core scheduling, a pick dispatches only when nothing is + * locally runnable and can legitimately go idle with @p still + * runnable (see do_pick_task_scx()). */ if (next && sched_class_above(&ext_sched_class, next->sched_class)) { - WARN_ON_ONCE(sched_cpu_cookie_match(rq, p) && + WARN_ON_ONCE(!sched_core_enabled(rq) && !(sch->ops.flags & SCX_OPS_ENQ_LAST)); do_enqueue_task(rq, p, SCX_ENQ_LAST, -1); } else { @@ -3241,6 +3242,70 @@ static enum scx_dsp_verdict dispatch_pick(struct rq *rq, struct rq_flags *rf, return verdict; } +#ifdef CONFIG_SCHED_CORE +/* + * Dispatch for a pick when core scheduling is enabled. The selection picks for + * all SMT siblings and the rq_i->core_pick state it builds must stay atomic + * throughout. If the dispatch released the rq lock, anything can have happened + * in between - return %SCX_DSP_RETRY to restart the selection against current + * state. + */ +static enum scx_dsp_verdict dispatch_core_pick(struct rq *rq, struct rq_flags *rf, + struct task_struct *prev) +{ + enum scx_dsp_verdict verdict; + u32 seq = rq->scx.lock_drop_seq; + + /* another dispatch is in flight on @rq, let that handle it */ + if (rq->scx.flags & SCX_RQ_IN_BALANCE) + return SCX_DSP_NONE; + + rq_unpin_lock(rq, rf); + + verdict = balance_one(rq, prev); + + if (cpu_of(rq) == smp_processor_id()) { + maybe_queue_balance_callback(rq); + + /* see dispatch_pick() */ + if (unlikely(rq->scx.kick_sync_pending)) { + rq->scx.kick_sync_pending = false; + queue_balance_callback(rq, &rq->scx.kick_sync_bal_cb, + kick_sync_wait_bal_cb); + } + } else if (unlikely(rq->scx.flags & SCX_RQ_BAL_CB_PENDING)) { + /* + * Balance callbacks must run in the context that queued them, + * so they can't be queued on another CPU's rq. Run the deferred + * work directly instead. + */ + rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING; + run_deferred(rq); + } + + rq_repin_lock(rq, rf); + + /* if balance_one() released the rq lock, restart the selection */ + if (rq->scx.lock_drop_seq != seq) + return SCX_DSP_RETRY; + + /* see dispatch_pick() */ + if (unlikely(verdict == SCX_DSP_PREV && + prev->sched_class != &ext_sched_class)) { + WARN_ON_ONCE(scx_enable_state() == SCX_ENABLED); + verdict = SCX_DSP_LOCAL; + } + + return verdict; +} +#else /* CONFIG_SCHED_CORE */ +static enum scx_dsp_verdict dispatch_core_pick(struct rq *rq, struct rq_flags *rf, + struct task_struct *prev) +{ + return SCX_DSP_NONE; +} +#endif /* CONFIG_SCHED_CORE */ + static struct task_struct * do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) { @@ -3253,7 +3318,13 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) rq_modified_begin(rq, &ext_sched_class); - verdict = dispatch_pick(rq, rf, prev); + if (sched_core_enabled(rq)) + verdict = dispatch_core_pick(rq, rf, prev); + else + verdict = dispatch_pick(rq, rf, prev); + + if (verdict == SCX_DSP_RETRY) + return RETRY_TASK; /* * If any higher-priority sched class enqueued a runnable task on this @@ -3267,9 +3338,8 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) return RETRY_TASK; /* - * If balance_one() is telling us to keep running @prev, replenish slice - * if necessary and keep running @prev. Otherwise, pop the first one - * from the local DSQ. + * If we're keeping @prev, replenish slice if necessary and keep running + * @prev. Otherwise, pop the first one from the local DSQ. */ if (verdict == SCX_DSP_PREV) { p = prev; -- cgit v1.2.3 From 0c09d1ad81ed55b02f64f4dd2e2e7c83161e5740 Mon Sep 17 00:00:00 2001 From: fangqiurong Date: Wed, 12 Aug 2026 14:11:16 +0800 Subject: sched_ext: Gate cid kfuncs behind the SCX struct_ops check scx_bpf_cid_to_cpu(), scx_bpf_cpu_to_cid() and scx_bpf_cid_topo() live in the scx_kfunc_ids_cid set, but scx_kfunc_context_filter() doesn't check that set. The filter's first test treats any kfunc outside its known sets as non-SCX and allows it, so these three kfuncs can be called from any struct_ops program - e.g. a TCP congestion control program. Add scx_kfunc_ids_cid to the filter's known sets, matching how in_any and in_idle are handled. Fixes: e9b55af47edf ("sched_ext: Add topological CPU IDs (cids)") Assisted-by: Z.ai:glm-5.2 Signed-off-by: fangqiurong Signed-off-by: Tejun Heo --- kernel/sched/ext/cid.h | 1 + kernel/sched/ext/ext.c | 9 +++++---- 2 files changed, 6 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/cid.h b/kernel/sched/ext/cid.h index 1f74d1f331f5..2fe2311a0f99 100644 --- a/kernel/sched/ext/cid.h +++ b/kernel/sched/ext/cid.h @@ -67,6 +67,7 @@ extern s32 __rcu *scx_shard_node; extern struct scx_cid_shard __rcu *scx_cid_shard_ranges; extern struct scx_cid_topo __rcu *scx_cid_topo; extern struct btf_id_set8 scx_kfunc_ids_init_cids; +extern struct btf_id_set8 scx_kfunc_ids_cid; void scx_cmask_clear(struct scx_cmask *m); void scx_cmask_fill(struct scx_cmask *m); diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 313db9fac1f9..61b156d7fc4b 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -10866,19 +10866,20 @@ int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id) bool in_idle = btf_id_set8_contains(&scx_kfunc_ids_idle, kfunc_id); bool in_any = btf_id_set8_contains(&scx_kfunc_ids_any, kfunc_id); bool in_cpu_only = btf_id_set8_contains(&scx_kfunc_ids_cpu_only, kfunc_id); + bool in_cid = btf_id_set8_contains(&scx_kfunc_ids_cid, kfunc_id); u32 moff, flags; /* Not an SCX kfunc - allow. */ if (!(in_unlocked || in_init_cids || in_select_cpu || in_enqueue || in_dispatch || - in_cpu_release || in_idle || in_any)) + in_cpu_release || in_idle || in_any || in_cid)) return 0; /* SYSCALL progs (e.g. BPF test_run()) may call unlocked and select_cpu kfuncs. */ if (prog->type == BPF_PROG_TYPE_SYSCALL) - return (in_unlocked || in_select_cpu || in_idle || in_any) ? 0 : -EACCES; + return (in_unlocked || in_select_cpu || in_idle || in_any || in_cid) ? 0 : -EACCES; if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) - return (in_any || in_idle) ? 0 : -EACCES; + return (in_any || in_idle || in_cid) ? 0 : -EACCES; /* * add_subprog_and_kfunc() collects all kfunc calls, including dead code @@ -10913,7 +10914,7 @@ int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id) return -EACCES; /* SCX struct_ops: check the per-op allow list. */ - if (in_any || in_idle) + if (in_any || in_idle || in_cid) return 0; moff = prog->aux->attach_st_ops_member_off; -- cgit v1.2.3 From 67f1f4a48c24974e392188602b477741186fa8ce Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 12 Aug 2026 09:55:33 -1000 Subject: sched_ext: Pass kernel arena pointers to ops_cid callbacks The cid-form set_cmask() and sub_caps_updated() callbacks receive cmasks that the kernel builds in the arena, and the kernel converts the kernel addresses to the BPF arena pointer form by hand before each call. BPF now translates between BPF and kernel arena addresses for __arena arguments. Tag the arguments __arena in the cfi stubs and the ops_cid member declarations and pass the kernel arena addresses directly, dropping the manual scx_kaddr_to_arena() conversions and the now-unused helper. The delivered value is unchanged and existing BPF-side code works as before. The arena argument address translation is currently implemented only on x86-64. cid-form schedulers implementing these callbacks load only there for now. Signed-off-by: Tejun Heo --- kernel/sched/ext/arena.c | 3 +-- kernel/sched/ext/ext.c | 14 +++++++------- kernel/sched/ext/internal.h | 19 ++++--------------- kernel/sched/ext/sub.c | 4 +--- 4 files changed, 13 insertions(+), 27 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/arena.c b/kernel/sched/ext/arena.c index b0e6a0be4913..f7a9f633f435 100644 --- a/kernel/sched/ext/arena.c +++ b/kernel/sched/ext/arena.c @@ -6,8 +6,7 @@ * * Each chunk added to @sch->arena_pool comes from one * bpf_arena_alloc_pages_sleepable() call and is registered at the - * kernel-side mapping address. Callers translate to the BPF-arena form - * themselves if needed. + * kernel-side mapping address. * * Allocations grow the pool on demand. Underlying arena pages are released * when the arena map itself is torn down. diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 61b156d7fc4b..c12327b6d541 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -467,13 +467,14 @@ static inline void scx_call_op_set_cpumask(struct scx_sched *sch, struct rq *rq, /* * Build the per-cpu arena cmask from kernel geometry via @ref, - * never reading its BPF-writable header, and hand BPF the arena - * address. The rq lock makes this cpu the sole kernel writer. + * never reading its BPF-writable header. set_cmask()'s __arena + * argument takes the kernel address and the struct_ops + * trampoline rebases it into BPF's arena pointer form. The rq + * lock makes this cpu the sole kernel writer. */ scx_cmask_ref_init_kern(sch, kern_va, 0, num_possible_cpus(), &ref); scx_cmask_ref_from_cpumask(&ref, cpumask); - SCX_CALL_CID_OP_TASK(sch, set_cmask, rq, task, - scx_kaddr_to_arena(sch, kern_va)); + SCX_CALL_CID_OP_TASK(sch, set_cmask, rq, task, kern_va); } else { SCX_CALL_OP_TASK(sch, set_cpumask, rq, task, cpumask); } @@ -8339,9 +8340,8 @@ static struct bpf_struct_ops bpf_sched_ext_ops = { * fresh stubs, set_cmask due to an argument type difference and the sub-sched * notifiers because no cpu-form stub exists to reuse. */ -static void sched_ext_ops_cid__set_cmask(struct task_struct *p, - const struct scx_cmask *cmask) {} -static void sched_ext_ops__sub_caps_updated(const struct scx_cmask *cmask, u64 caps) {} +static void sched_ext_ops_cid__set_cmask(struct task_struct *p, const struct scx_cmask *cmask__arena) {} +static void sched_ext_ops__sub_caps_updated(const struct scx_cmask *cmask__arena, u64 caps) {} static void sched_ext_ops__sub_ecaps_updated(s32 cid, u64 before, u64 after) {} static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = { diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index b699e7c1103f..35b5bf8c5c66 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1049,7 +1049,7 @@ struct sched_ext_ops_cid { struct task_struct *b); void (*set_weight)(struct task_struct *p, u32 weight); void (*set_cmask)(struct task_struct *p, - const struct scx_cmask *cmask); + const struct scx_cmask *cmask__arena); void (*update_idle)(s32 cid, bool idle); s32 (*init_task)(struct task_struct *p, struct scx_init_task_args *args); @@ -1075,7 +1075,7 @@ struct sched_ext_ops_cid { #endif /* CONFIG_EXT_GROUP_SCHED */ s32 (*sub_attach)(struct scx_sub_attach_args *args); void (*sub_detach)(struct scx_sub_detach_args *args); - void (*sub_caps_updated)(const struct scx_cmask *cmask, u64 caps); + void (*sub_caps_updated)(const struct scx_cmask *cmask__arena, u64 caps); void (*sub_ecaps_updated)(s32 cid, u64 before, u64 after); void (*cid_online)(s32 cid); void (*cid_offline)(s32 cid); @@ -1534,8 +1534,7 @@ struct scx_sched { * * @arena_pool sub-allocates @arena_map. Each gen_pool chunk is added * at the kernel-side mapping address. @arena_kern_base is the start - * of the arena's kern_vm range. See scx_arena_to_kaddr() and - * scx_kaddr_to_arena(). + * of the arena's kern_vm range. See scx_arena_to_kaddr(). */ struct bpf_map *arena_map; struct gen_pool *arena_pool; @@ -1544,7 +1543,7 @@ struct scx_sched { /* * Per-CPU arena cmask used by scx_call_op_set_cpumask() to hand a cmask * to ops_cid.set_cmask(). The kernel writes through the stored kern_va - * and hands BPF its arena pointer via scx_kaddr_to_arena(). + * and passes it to the callback's __arena argument. */ struct scx_cmask * __percpu *set_cmask_scratch; @@ -1654,16 +1653,6 @@ static inline void *scx_arena_to_kaddr(struct scx_sched *sch, const void *bpf_pt return (void *)(sch->arena_kern_base + (u32)(uintptr_t)bpf_ptr); } -/** - * scx_kaddr_to_arena - Translate a kernel arena address to its BPF form - * @sch: scheduler whose arena hosts @kaddr - * @kaddr: kernel-side arena address, supplied by trusted kernel code - */ -static inline void *scx_kaddr_to_arena(struct scx_sched *sch, const void *kaddr) -{ - return (void *)((uintptr_t)kaddr - sch->arena_kern_base); -} - enum scx_wake_flags { /* expose select WF_* flags as enums */ SCX_WAKE_FORK = WF_FORK, diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 3bd12c1ff0a3..0978581d8c8e 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -870,9 +870,7 @@ static void caps_updated_deliver(struct list_head *to_deliver) break; /* caps != 0 only when deliverable (has_op, above) */ - SCX_CALL_OP(sch, sub_caps_updated, NULL, - scx_kaddr_to_arena(sch, cu->cmask_arena_out), - caps); + SCX_CALL_OP(sch, sub_caps_updated, NULL, cu->cmask_arena_out, caps); } } } -- cgit v1.2.3 From a8dc810968af02190f55cc7574bc87c93156f266 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 12 Aug 2026 09:55:33 -1000 Subject: sched_ext: Convert sub-cap kfuncs to __arena cmask arguments The sub-cap kfuncs take their cmask arguments as __ign pointers. The values cross the kfunc boundary as unchecked scalars and scx_cmask_ref_init() rebases them into the arena by hand. BPF now translates between BPF and kernel arena addresses for __arena arguments. Tag the cmask arguments __arena so the kfuncs receive kernel addresses and scx_cmask_ref_init() loses the hand-rolled conversion. The optional denied_out keeps its NULL not-provided signal via __arena__nullable. The mandatory masks use plain __arena. scx_qmap's call sites drop the (void *)(long) casts since the BPF-side declarations type the cmask arguments __arena and take arena pointers directly. The arena argument address translation is currently implemented only on x86-64. Schedulers calling these kfuncs load only there for now. Signed-off-by: Tejun Heo --- kernel/sched/ext/cid.c | 11 ++++--- kernel/sched/ext/ext.c | 8 +++--- kernel/sched/ext/sub.c | 49 ++++++++++++++++---------------- kernel/sched/ext/types.h | 6 ++-- tools/sched_ext/include/scx/common.bpf.h | 11 ++++--- tools/sched_ext/scx_qmap.bpf.c | 20 ++++++------- 6 files changed, 51 insertions(+), 54 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/cid.c b/kernel/sched/ext/cid.c index 231c8562d7c7..11fe9be80f1e 100644 --- a/kernel/sched/ext/cid.c +++ b/kernel/sched/ext/cid.c @@ -957,7 +957,7 @@ static const struct btf_kfunc_id_set scx_kfunc_set_cid = { /** * scx_cmask_ref_init - Bind a scx_cmask_ref to a BPF-arena cmask * @sch: scheduler whose arena hosts @src - * @src: BPF-supplied cmask pointer + * @src: BPF-supplied cmask, rebased to its kernel address * @ref: output ref * * Snapshot @src's @base, @nr_cids and @alloc_words. The snapshot is necessary @@ -969,20 +969,19 @@ static const struct btf_kfunc_id_set scx_kfunc_set_cid = { int scx_cmask_ref_init(struct scx_sched *sch, const struct scx_cmask *src, struct scx_cmask_ref *ref) { - struct scx_cmask *kern_src = scx_arena_to_kaddr(sch, src); u32 base, nr_cids, alloc_words, npossible = num_possible_cpus(); s32 *cid_to_shard; - base = READ_ONCE(kern_src->base); - nr_cids = READ_ONCE(kern_src->nr_cids); - alloc_words = READ_ONCE(kern_src->alloc_words); + base = READ_ONCE(src->base); + nr_cids = READ_ONCE(src->nr_cids); + alloc_words = READ_ONCE(src->alloc_words); if (unlikely(base >= npossible || nr_cids > npossible - base || SCX_CMASK_NR_WORDS(nr_cids) > alloc_words)) return -EINVAL; ref->sch = sch; - ref->src = kern_src; + ref->src = (struct scx_cmask *)src; ref->base = base; ref->nr_cids = nr_cids; diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index c12327b6d541..26dbbbfe5a87 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -10677,20 +10677,20 @@ out: #ifndef CONFIG_EXT_SUB_SCHED __bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, - const struct scx_cmask *cmask__ign, - struct scx_cmask *denied_out__ign, + const struct scx_cmask *cmask__arena, + struct scx_cmask *denied_out__arena__nullable, const struct bpf_prog_aux *aux) { return -EOPNOTSUPP; } __bpf_kfunc void scx_bpf_sub_revoke(u64 cgroup_id, u64 caps, - const struct scx_cmask *cmask__ign, + const struct scx_cmask *cmask__arena, const struct bpf_prog_aux *aux) { } -__bpf_kfunc s32 scx_bpf_sub_caps(u64 cgroup_id, u64 caps, struct scx_cmask *out__ign, +__bpf_kfunc s32 scx_bpf_sub_caps(u64 cgroup_id, u64 caps, struct scx_cmask *out__arena, const struct bpf_prog_aux *aux) { return -EOPNOTSUPP; diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 0978581d8c8e..d874ad41a8ed 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -2287,26 +2287,26 @@ static s32 sub_cap_preamble(u64 cgroup_id, u64 caps, const struct bpf_prog_aux * } /** - * scx_bpf_sub_grant - Grant @caps on @cmask__ign's cids to a direct child + * scx_bpf_sub_grant - Grant @caps on a cmask's cids to a direct child * @cgroup_id: cgroup id of the direct child sub-sched * @caps: bitmask of SCX_CAP_* to grant - * @cmask__ign: cid cmask to grant @caps on (arena pointer) - * @denied_out__ign: optional arena cmask accumulating refused cids + * @cmask__arena: cid cmask to grant @caps on + * @denied_out__arena__nullable: optional cmask accumulating refused cids * @aux: implicit BPF argument * - * A cid in @cmask__ign is granted to the child only if the parent holds every - * requested cap on it. Refused cids are OR'd into @denied_out__ign when - * provided. Refusals outside @denied_out__ign's range are not recorded. + * A cid in @cmask__arena is granted to the child only if the parent holds every + * requested cap on it. Refused cids are OR'd into the denied mask when + * provided. Refusals outside the denied mask's range are not recorded. * - * All-or-nothing keeps the caller-visible result binary per cid, so - * @denied_out__ign is one mask to interpret rather than a per-cap matrix. + * All-or-nothing keeps the caller-visible result binary per cid, so the denied + * mask is one mask to interpret rather than a per-cap matrix. * * Return 0 on full success, -EPERM if any cid was refused, or a negative * errno on other failures. */ __bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, - const struct scx_cmask *cmask__ign, - struct scx_cmask *denied_out__ign, + const struct scx_cmask *cmask__arena, + struct scx_cmask *denied_out__arena__nullable, const struct bpf_prog_aux *aux) { struct scx_cmask_ref ref, denied_ref; @@ -2321,14 +2321,14 @@ __bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, if (ret) return ret; - ret = scx_cmask_ref_init(parent, cmask__ign, &ref); + ret = scx_cmask_ref_init(parent, cmask__arena, &ref); if (ret) { scx_error(parent, "invalid cmask (%d)", ret); return ret; } - if (denied_out__ign) { - ret = scx_cmask_ref_init(parent, denied_out__ign, &denied_ref); + if (denied_out__arena__nullable) { + ret = scx_cmask_ref_init(parent, denied_out__arena__nullable, &denied_ref); if (ret) { scx_error(parent, "invalid denied_out (%d)", ret); return ret; @@ -2395,10 +2395,10 @@ __bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, } } - /* record cids that didn't make it through into @denied_out */ + /* record cids that didn't make it into the denied mask */ if (!scx_cmask_subset(slice, granted_cids)) { any_denied = true; - if (denied_out__ign) { + if (denied_out__arena__nullable) { SCX_CMASK_DEFINE_SHARD(denied, slice->base, slice->nr_cids); scx_cmask_copy(denied, slice); @@ -2414,19 +2414,18 @@ __bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, } /** - * scx_bpf_sub_revoke - Revoke @caps on @cmask__ign's cids from @child + * scx_bpf_sub_revoke - Revoke @caps on a cmask's cids from a direct child * @cgroup_id: cgroup id of the direct child sub-sched * @caps: bitmask of SCX_CAP_* to revoke - * @cmask__ign: cid cmask to revoke @caps on (arena pointer) + * @cmask__arena: cid cmask to revoke @caps on * @aux: implicit BPF argument * - * Clear @caps bits on @cmask__ign from the child named by @cgroup_id and all + * Clear @caps bits on @cmask__arena from the child named by @cgroup_id and all * its descendants. The origin parent's pshard lock is held across the subtree - * walk so a concurrent grant from the origin parent observes the revoked - * state. + * walk so a concurrent grant from the origin parent observes the revoked state. */ __bpf_kfunc void scx_bpf_sub_revoke(u64 cgroup_id, u64 caps, - const struct scx_cmask *cmask__ign, + const struct scx_cmask *cmask__arena, const struct bpf_prog_aux *aux) { struct scx_cmask_ref ref; @@ -2439,7 +2438,7 @@ __bpf_kfunc void scx_bpf_sub_revoke(u64 cgroup_id, u64 caps, if (sub_cap_preamble(cgroup_id, caps, aux, &parent, &child)) return; - ret = scx_cmask_ref_init(parent, cmask__ign, &ref); + ret = scx_cmask_ref_init(parent, cmask__arena, &ref); if (ret) { scx_error(parent, "invalid cmask (%d)", ret); return; @@ -2509,7 +2508,7 @@ __bpf_kfunc void scx_bpf_sub_revoke(u64 cgroup_id, u64 caps, * scx_bpf_sub_caps - Read self's or a direct child's cap cmasks * @cgroup_id: 0 for self, or a direct child's cgroup id * @caps: one or more SCX_CAP_* bits - * @out__ign: arena cmask to receive the union of @caps within its range + * @out__arena: cmask to receive the union of @caps within its range * @aux: implicit BPF argument * * Read the cap cmasks granted on each cid for self (@cgroup_id 0) or a direct @@ -2519,7 +2518,7 @@ __bpf_kfunc void scx_bpf_sub_revoke(u64 cgroup_id, u64 caps, * Return 0, -ENODEV if @cgroup_id names no direct child, or -EINVAL on bad * inputs. */ -__bpf_kfunc s32 scx_bpf_sub_caps(u64 cgroup_id, u64 caps, struct scx_cmask *out__ign, +__bpf_kfunc s32 scx_bpf_sub_caps(u64 cgroup_id, u64 caps, struct scx_cmask *out__arena, const struct bpf_prog_aux *aux) { struct scx_cmask_ref ref; @@ -2569,7 +2568,7 @@ __bpf_kfunc s32 scx_bpf_sub_caps(u64 cgroup_id, u64 caps, struct scx_cmask *out_ return -ENODEV; } - ret = scx_cmask_ref_init(sch, out__ign, &ref); + ret = scx_cmask_ref_init(sch, out__arena, &ref); if (ret) { scx_error(sch, "invalid out (%d)", ret); return ret; diff --git a/kernel/sched/ext/types.h b/kernel/sched/ext/types.h index 1eb3ac8508f6..943d8d429a2c 100644 --- a/kernel/sched/ext/types.h +++ b/kernel/sched/ext/types.h @@ -187,9 +187,9 @@ struct scx_cmask { /* * scx_cmask_ref: validated reference to a BPF-arena cmask. * - * scx_cmask_ref_init() normalizes the pointer into the arena and snapshots - * @base/@nr_cids. The snapshot is what downstream code uses for sizing - the - * live header can be mutated concurrently by BPF. + * scx_cmask_ref_init() snapshots @base/@nr_cids. The snapshot is what + * downstream code uses for sizing - the live header can be mutated concurrently + * by BPF. * * scx_cmask_ref_shard() reads one shard into a cmask. scx_cmask_ref_or() and * scx_cmask_ref_copy() write back into the referenced arena cmask, bounded by diff --git a/tools/sched_ext/include/scx/common.bpf.h b/tools/sched_ext/include/scx/common.bpf.h index 6035d007c43f..979d4cabfaf9 100644 --- a/tools/sched_ext/include/scx/common.bpf.h +++ b/tools/sched_ext/include/scx/common.bpf.h @@ -26,6 +26,7 @@ #include #include "user_exit_info.bpf.h" #include "enum_defs.autogen.h" +#include "bpf_arena_common.bpf.h" #define PF_IDLE 0x00000002 /* I am an IDLE thread */ #define PF_IO_WORKER 0x00000010 /* Task is an IO worker */ @@ -115,12 +116,10 @@ u32 scx_bpf_cidperf_cur(s32 cid) __ksym __weak; s32 scx_bpf_cidperf_set(s32 cid, u32 perf) __ksym __weak; /* sub-scheduler cap control, scx_bpf_sub_caps() cgroup_id 0 == self */ -s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, const struct scx_cmask *cmask, - struct scx_cmask *denied) __ksym __weak; -void scx_bpf_sub_revoke(u64 cgroup_id, u64 caps, const struct scx_cmask *cmask) __ksym __weak; -s32 scx_bpf_sub_caps(u64 cgroup_id, u64 caps, struct scx_cmask *out) __ksym __weak; -s32 scx_bpf_sub_kill_bstr(u64 cgroup_id, char *fmt, - unsigned long long *data, u32 data__sz) __ksym __weak; +s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, const struct scx_cmask __arena *cmask__arena, struct scx_cmask __arena *denied_out__arena__nullable) __ksym __weak; +void scx_bpf_sub_revoke(u64 cgroup_id, u64 caps, const struct scx_cmask __arena *cmask__arena) __ksym __weak; +s32 scx_bpf_sub_caps(u64 cgroup_id, u64 caps, struct scx_cmask __arena *out__arena) __ksym __weak; +s32 scx_bpf_sub_kill_bstr(u64 cgroup_id, char *fmt, unsigned long long *data, u32 data__sz) __ksym __weak; /* * Use the following as @it__iter when calling scx_bpf_dsq_move[_vtime]() from diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c index dd0434437835..61dc748742a2 100644 --- a/tools/sched_ext/scx_qmap.bpf.c +++ b/tools/sched_ext/scx_qmap.bpf.c @@ -1385,8 +1385,8 @@ __noinline void compute_partition(void) } /* find out the cids we hold */ - scx_bpf_sub_caps(0, SCX_CAP_ENQ, (void *)(long)&qa.held_excl.mask); - scx_bpf_sub_caps(0, SCX_CAP_ENQ_IMMED, (void *)(long)&qa.held_shared.mask); + scx_bpf_sub_caps(0, SCX_CAP_ENQ, &qa.held_excl.mask); + scx_bpf_sub_caps(0, SCX_CAP_ENQ_IMMED, &qa.held_shared.mask); cmask_andnot(&qa.held_shared.mask, &qa.held_excl.mask); /* held only as ENQ_IMMED */ qa.part.nr_shared = 0; @@ -1613,13 +1613,13 @@ __noinline void apply_partition(void) cmask_andnot(&qa.to_grant_cids.mask, &ssc->prev_granted.mask); scx_bpf_sub_revoke(cgid, SCX_CAP_ENQ_IMMED | SCX_CAP_PERF, - (void *)(long)&qa.prev_rr_cids.mask); + &qa.prev_rr_cids.mask); scx_bpf_sub_revoke(cgid, SCX_CAP_ENQ | SCX_CAP_PREEMPT | SCX_CAP_ENQ_IMMED | SCX_CAP_PERF, - (void *)(long)&qa.to_revoke_cids.mask); + &qa.to_revoke_cids.mask); scx_bpf_sub_grant(cgid, SCX_CAP_ENQ | SCX_CAP_PREEMPT | SCX_CAP_ENQ_IMMED | SCX_CAP_PERF, - (void *)(long)&qa.to_grant_cids.mask, NULL); + &qa.to_grant_cids.mask, NULL); } /* the current holder of the shared pool gets ENQ_IMMED on all of it */ @@ -1636,7 +1636,7 @@ __noinline void apply_partition(void) if (holder_cgid) scx_bpf_sub_grant(holder_cgid, SCX_CAP_ENQ_IMMED | SCX_CAP_PERF, - (void *)(long)&qa.rr_cids.mask, NULL); + &qa.rr_cids.mask, NULL); } } @@ -1728,11 +1728,11 @@ static void rr_advance(void) if (old_cgid) scx_bpf_sub_revoke(old_cgid, SCX_CAP_ENQ_IMMED | SCX_CAP_PERF, - (void *)(long)&qa.rr_cids.mask); + &qa.rr_cids.mask); if (new_cgid) scx_bpf_sub_grant(new_cgid, SCX_CAP_ENQ_IMMED | SCX_CAP_PERF, - (void *)(long)&qa.rr_cids.mask, NULL); + &qa.rr_cids.mask, NULL); } part_end(); @@ -1840,8 +1840,8 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(qmap_init) cmask_init(&qa.held_excl.mask, 0, nr_cids); cmask_init(&qa.held_shared.mask, 0, nr_cids); - scx_bpf_sub_caps(0, SCX_CAP_ENQ, (void *)(long)&qa.held_excl.mask); - scx_bpf_sub_caps(0, SCX_CAP_ENQ_IMMED, (void *)(long)&qa.held_shared.mask); + scx_bpf_sub_caps(0, SCX_CAP_ENQ, &qa.held_excl.mask); + scx_bpf_sub_caps(0, SCX_CAP_ENQ_IMMED, &qa.held_shared.mask); cmask_andnot(&qa.held_shared.mask, &qa.held_excl.mask); bpf_for(i, 0, MAX_SUB_SCHEDS) { -- cgit v1.2.3 From a05c5b5cb5cfc2c2b27ce05a690dd0af1bcdf099 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 12 Aug 2026 09:55:33 -1000 Subject: sched_ext: Convert scx_bpf_cid_override() to __arena array arguments scx_bpf_cid_override() predates the cid-form arena transition and takes its arrays as verifier-checked mem+size buffers, forcing scx_qmap to keep the cpu_to_cid and shard_start arrays in writable bss while the rest of its state lives in the arena. Unify on arena arguments before cid-form schedulers start seeing real use. BPF now translates between BPF and kernel arena addresses for __arena arguments. Take the arrays as __arena arguments, with the counts passed in entries. The counts now size the snapshot copies and are bounds-checked before them. scx_qmap moves the arrays into struct qmap_arena. As the arena is mmapped at load, the loader populates them between load and attach instead of before load. The arena argument address translation is currently implemented only on x86-64. Schedulers calling this kfunc load only there for now. Signed-off-by: Tejun Heo --- kernel/sched/ext/cid.c | 49 +++++++++++++---------- tools/sched_ext/include/scx/compat.bpf.h | 14 ++++--- tools/sched_ext/scx_qmap.bpf.c | 16 ++------ tools/sched_ext/scx_qmap.c | 68 +++++++++++++++++++------------- tools/sched_ext/scx_qmap.h | 4 ++ 5 files changed, 84 insertions(+), 67 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/cid.c b/kernel/sched/ext/cid.c index 11fe9be80f1e..39f88deb94bc 100644 --- a/kernel/sched/ext/cid.c +++ b/kernel/sched/ext/cid.c @@ -449,10 +449,10 @@ __bpf_kfunc_start_defs(); /** * scx_bpf_cid_override - Install an explicit cpu->cid mapping with shard info - * @cpu_to_cid_src: array of nr_cpu_ids s32 entries (cid for each cpu) - * @cpu_to_cid_src__sz: must be nr_cpu_ids * sizeof(s32) bytes - * @shard_start_src: array of first-cid-of-each-shard, strictly increasing from 0 - * @shard_start_src__sz: nr_shards * sizeof(s32) bytes + * @cpu_to_cid__arena: array of nr_cpu_ids s32 entries (cid for each cpu) + * @cpu_to_cid_cnt: number of entries, must be nr_cpu_ids + * @shard_start__arena: array of first-cid-of-each-shard, one entry per shard + * @shard_start_cnt: number of shards * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * * May only be called from ops.init_cids() of the root scheduler. Replace the @@ -464,9 +464,9 @@ __bpf_kfunc_start_defs(); * (core/LLC/node) is cleared and the shard layout is set from the input. On * invalid input, abort the scheduler. */ -__bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid_src, u32 cpu_to_cid_src__sz, - const s32 *shard_start_src, u32 shard_start_src__sz, - const struct bpf_prog_aux *aux) +__bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid__arena, u32 cpu_to_cid_cnt, + const s32 *shard_start__arena, u32 shard_start_cnt, + const struct bpf_prog_aux *aux) { cpumask_var_t seen __free(free_cpumask_var) = CPUMASK_VAR_NULL; u32 *node_counts __free(kfree) = NULL; @@ -475,19 +475,28 @@ __bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid_src, u32 cpu_to_cid_ u32 npossible = num_possible_cpus(); struct scx_cid_tables *tbls; struct scx_sched *sch; - u32 nr_shards; + u32 nr_shards = shard_start_cnt; bool alloced; s32 cpu, cid, si; /* * GFP_KERNEL allocs must happen before the rcu read section. Snapshot - * the BPF-supplied arrays so a concurrent map mutation can't change + * the BPF-supplied arrays so a concurrent arena write can't change * them between validation and use. + * + * The BPF-supplied counts size the snapshots and thus the arena reads. + * Gate the copies on the count bounds, reported below once @sch is + * available. The bounded reads, at most 32KB, stay within the guard + * region that arena fault recovery covers. */ alloced = zalloc_cpumask_var(&seen, GFP_KERNEL); node_counts = kcalloc(nr_node_ids, sizeof(*node_counts), GFP_KERNEL); - cpu_to_cid = kmemdup(cpu_to_cid_src, cpu_to_cid_src__sz, GFP_KERNEL); - shard_start = kmemdup(shard_start_src, shard_start_src__sz, GFP_KERNEL); + if (cpu_to_cid_cnt == nr_cpu_ids) + cpu_to_cid = kmemdup(cpu_to_cid__arena, cpu_to_cid_cnt * sizeof(s32), + GFP_KERNEL); + if (nr_shards && nr_shards <= npossible) + shard_start = kmemdup(shard_start__arena, nr_shards * sizeof(s32), + GFP_KERNEL); guard(rcu)(); @@ -499,25 +508,23 @@ __bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid_src, u32 cpu_to_cid_ lockdep_assert_held(&scx_enable_mutex); tbls = scx_cid_tables; - if (!alloced || !node_counts || !cpu_to_cid || !shard_start) { - scx_error(sch, "scx_bpf_cid_override: allocation failed"); + if (cpu_to_cid_cnt != nr_cpu_ids) { + scx_error(sch, "scx_bpf_cid_override: cpu_to_cid expected %u entries, got %u", + nr_cpu_ids, cpu_to_cid_cnt); return; } - if (cpu_to_cid_src__sz != nr_cpu_ids * sizeof(s32)) { - scx_error(sch, "scx_bpf_cid_override: cpu_to_cid expected %zu bytes, got %u", - nr_cpu_ids * sizeof(s32), cpu_to_cid_src__sz); + if (!nr_shards || nr_shards > npossible) { + scx_error(sch, "scx_bpf_cid_override: invalid shard_start count %u", + nr_shards); return; } - if (!shard_start_src__sz || shard_start_src__sz % sizeof(s32)) { - scx_error(sch, "scx_bpf_cid_override: invalid shard_start size %u", - shard_start_src__sz); + if (!alloced || !node_counts || !cpu_to_cid || !shard_start) { + scx_error(sch, "scx_bpf_cid_override: allocation failed"); return; } - nr_shards = shard_start_src__sz / sizeof(s32); - /* validate shard_start[]: starts at 0, strictly increasing, in range */ if (shard_start[0] != 0) { scx_error(sch, "scx_bpf_cid_override: shard_start[0] must be 0, got %d", diff --git a/tools/sched_ext/include/scx/compat.bpf.h b/tools/sched_ext/include/scx/compat.bpf.h index 1b45655a64a3..7c735eb09c07 100644 --- a/tools/sched_ext/include/scx/compat.bpf.h +++ b/tools/sched_ext/include/scx/compat.bpf.h @@ -125,15 +125,17 @@ static inline bool scx_bpf_sub_dispatch(u64 cgroup_id) * v7.3: scx_bpf_cid_override() for explicit cid and shard mapping. Ignore if * missing. */ -void scx_bpf_cid_override___compat(const s32 *cpu_to_cid, u32 cpu_to_cid__sz, - const s32 *shard_start, u32 shard_start__sz) __ksym __weak; +void scx_bpf_cid_override___compat(const s32 __arena *cpu_to_cid__arena, + u32 cpu_to_cid_cnt, + const s32 __arena *shard_start__arena, + u32 shard_start_cnt) __ksym __weak; -static inline void scx_bpf_cid_override(const s32 *cpu_to_cid, u32 cpu_to_cid__sz, - const s32 *shard_start, u32 shard_start__sz) +static inline void scx_bpf_cid_override(const s32 __arena *cpu_to_cid, u32 cpu_to_cid_cnt, + const s32 __arena *shard_start, u32 shard_start_cnt) { if (bpf_ksym_exists(scx_bpf_cid_override___compat)) - scx_bpf_cid_override___compat(cpu_to_cid, cpu_to_cid__sz, - shard_start, shard_start__sz); + scx_bpf_cid_override___compat(cpu_to_cid, cpu_to_cid_cnt, + shard_start, shard_start_cnt); } /** diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c index 61dc748742a2..d9ca9106a5ce 100644 --- a/tools/sched_ext/scx_qmap.bpf.c +++ b/tools/sched_ext/scx_qmap.bpf.c @@ -80,14 +80,6 @@ const volatile u64 round_robin_ns; */ const volatile u32 cid_override_mode; const volatile u32 cid_override_nr_shards; -/* - * Arrays live in bss (writable) because scx_bpf_cid_override()'s BPF - * verifier signature treats its len-paired pointers as read/write - rodata - * fails verification with "write into map forbidden". Userspace populates - * them before SCX_OPS_LOAD, same as rodata, and nothing writes them after. - */ -s32 cid_override_cpu_to_cid[SCX_QMAP_MAX_CPUS]; -s32 cid_override_shard_start[SCX_QMAP_MAX_CPUS]; UEI_DEFINE(uei); @@ -1761,17 +1753,15 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(qmap_init_cids) if (!cid_override_mode) return 0; - /* bound the count so the verifier accepts cpu_to_cid's mem/len pair */ + /* the arena arrays are sized SCX_QMAP_MAX_CPUS */ if (nr_cpu_ids > SCX_QMAP_MAX_CPUS) { scx_bpf_error("nr_cpu_ids=%u exceeds SCX_QMAP_MAX_CPUS=%d", nr_cpu_ids, SCX_QMAP_MAX_CPUS); return -EINVAL; } - scx_bpf_cid_override((const s32 *)cid_override_cpu_to_cid, - nr_cpu_ids * sizeof(s32), - (const s32 *)cid_override_shard_start, - cid_override_nr_shards * sizeof(s32)); + scx_bpf_cid_override(qa.cid_override_cpu_to_cid, nr_cpu_ids, + qa.cid_override_shard_start, cid_override_nr_shards); return 0; } diff --git a/tools/sched_ext/scx_qmap.c b/tools/sched_ext/scx_qmap.c index 988b6931633e..5bb5f687e579 100644 --- a/tools/sched_ext/scx_qmap.c +++ b/tools/sched_ext/scx_qmap.c @@ -242,6 +242,7 @@ int main(int argc, char **argv) char tbuf[32]; u32 inject_mode = 0; u64 own_cgid = 0; + s32 cid_override_shard_sz = 4; libbpf_set_print(libbpf_print_fn); signal(SIGINT, sigint_handler); @@ -328,8 +329,7 @@ restart: break; case 'C': { u32 nr_cpus = libbpf_num_possible_cpus(); - u32 mode, i; - s32 shard_sz = 4; + u32 mode; if (!strcmp(optarg, "shuffle")) mode = QMAP_CID_OVR_SHUFFLE; @@ -344,18 +344,7 @@ restart: return 1; } skel->rodata->cid_override_mode = mode; - - /* shuffle: reversed cpu_to_cid; others: identity */ - for (i = 0; i < nr_cpus; i++) { - if (mode == QMAP_CID_OVR_SHUFFLE) - skel->bss->cid_override_cpu_to_cid[i] = nr_cpus - 1 - i; - else - skel->bss->cid_override_cpu_to_cid[i] = i; - } - if (mode == QMAP_CID_OVR_BAD_DUP && nr_cpus >= 2) - skel->bss->cid_override_cpu_to_cid[1] = 0; - if (mode == QMAP_CID_OVR_BAD_RANGE) - skel->bss->cid_override_cpu_to_cid[0] = (s32)nr_cpus; + cid_override_shard_sz = 4; /* * bad-mono needs >= 3 shards to build a 0-based but @@ -368,21 +357,12 @@ restart: nr_cpus); return 1; } - shard_sz = nr_cpus / 3; + cid_override_shard_sz = nr_cpus / 3; } /* shards of shard_sz each */ - skel->rodata->cid_override_nr_shards = (nr_cpus + shard_sz - 1) / shard_sz; - for (i = 0; i < skel->rodata->cid_override_nr_shards; i++) - skel->bss->cid_override_shard_start[i] = i * shard_sz; - - if (mode == QMAP_CID_OVR_BAD_MONO) { - /* swap [1] and [2] to break monotonicity */ - s32 tmp = skel->bss->cid_override_shard_start[1]; - skel->bss->cid_override_shard_start[1] = - skel->bss->cid_override_shard_start[2]; - skel->bss->cid_override_shard_start[2] = tmp; - } + skel->rodata->cid_override_nr_shards = + (nr_cpus + cid_override_shard_sz - 1) / cid_override_shard_sz; break; } case 'i': @@ -428,9 +408,43 @@ restart: skel->rodata->round_robin_ns = (u64)round_robin_ms * 1000000; SCX_OPS_LOAD(skel, qmap_ops, scx_qmap, uei); - link = SCX_OPS_ATTACH(skel, qmap_ops, scx_qmap); qa = &skel->arena->qa; + + /* + * The cid-override arrays live in the arena, which is mmapped at load. + * Populate them before qmap_init_cids() consumes them at attach. + */ + if (skel->rodata->cid_override_mode) { + u32 mode = skel->rodata->cid_override_mode; + u32 nr_cpus = libbpf_num_possible_cpus(); + u32 i; + + /* shuffle: reversed cpu_to_cid; others: identity */ + for (i = 0; i < nr_cpus; i++) { + if (mode == QMAP_CID_OVR_SHUFFLE) + qa->cid_override_cpu_to_cid[i] = nr_cpus - 1 - i; + else + qa->cid_override_cpu_to_cid[i] = i; + } + if (mode == QMAP_CID_OVR_BAD_DUP && nr_cpus >= 2) + qa->cid_override_cpu_to_cid[1] = 0; + if (mode == QMAP_CID_OVR_BAD_RANGE) + qa->cid_override_cpu_to_cid[0] = (s32)nr_cpus; + + for (i = 0; i < skel->rodata->cid_override_nr_shards; i++) + qa->cid_override_shard_start[i] = i * cid_override_shard_sz; + + if (mode == QMAP_CID_OVR_BAD_MONO) { + /* swap [1] and [2] to break monotonicity */ + s32 tmp = qa->cid_override_shard_start[1]; + qa->cid_override_shard_start[1] = qa->cid_override_shard_start[2]; + qa->cid_override_shard_start[2] = tmp; + } + } + + link = SCX_OPS_ATTACH(skel, qmap_ops, scx_qmap); + qa->test_error_cnt = test_error_cnt; qa->inject_mode = inject_mode; diff --git a/tools/sched_ext/scx_qmap.h b/tools/sched_ext/scx_qmap.h index c8f602d58ca3..c78d61806b39 100644 --- a/tools/sched_ext/scx_qmap.h +++ b/tools/sched_ext/scx_qmap.h @@ -131,6 +131,10 @@ struct qmap_arena { struct cpu_ctx cpu_ctxs[SCX_QMAP_MAX_CPUS]; + /* cid-override test input, populated by the loader before attach */ + __s32 cid_override_cpu_to_cid[SCX_QMAP_MAX_CPUS]; + __s32 cid_override_shard_start[SCX_QMAP_MAX_CPUS]; + /* task_ctx slab; allocated and threaded by qmap_init() */ struct task_ctx __arena *task_ctxs; struct task_ctx __arena *task_free_head; -- cgit v1.2.3 From d7832ba1a4a4f462ce94c81960adf10f7e87ae7a Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 14 Aug 2026 07:46:46 -1000 Subject: sched_ext: Set up ops.sub_ecaps_updated() dispatch context on the executing CPU scx_process_sync_ecaps() sets up the dispatch context for ops.sub_ecaps_updated() in the target cpu's pcpu context recovered from the llist node. However, the context is per executing cpu: the dispatch kfuncs resolve it with this_cpu_ptr() and the dispatch buffer lives in it. What the dispatches target is determined by the rq recorded in the context, not by which cpu's context it is. Under core scheduling the pick runs balance_one() for sibling rqs, so a sync processed for a sibling invokes the op with the executing cpu's context not set up and its dispatch kfuncs misoperate on a NULL or stale rq. Set up the executing cpu's dsp_ctx instead, matching scx_dispatch_sched(). The recorded rq keeps the dispatches targeting the synced cpu. Fixes: b81a6c018cde ("sched_ext: Add sub_ecaps_updated() effective-cap change notifier") Reported-by: David Carlier Link: https://lore.kernel.org/all/20260813045931.8691-1-devnexen@gmail.com/ Signed-off-by: Tejun Heo --- kernel/sched/ext/sub.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 3bd12c1ff0a3..ec4729c99763 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -1017,11 +1017,15 @@ void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) * invocation is equivalent to the dispatch path and may drop * and re-acquire the rq lock temporarily while the rest of * @batch is held privately, see scx_discard_ecaps_to_sync(). + * The dispatch kfuncs resolve their context on the executing + * cpu, which under core scheduling can differ from @rq's cpu, + * so the context is set up there. The rq recorded in it keeps + * the dispatches targeting @rq. */ if (ecaps != pcpu->reported_ecaps && SCX_HAS_OP(pcpu->sch, sub_ecaps_updated) && !scx_bypassing(pcpu->sch, cpu)) { - struct scx_dsp_ctx *dspc = &pcpu->dsp_ctx; + struct scx_dsp_ctx *dspc = &this_cpu_ptr(pcpu->sch->pcpu)->dsp_ctx; dspc->rq = rq; /* stash @prev so nested dispatches can access it */ -- cgit v1.2.3 From 1d0a73dd0c4f5a904ece163fdd24c7e03de0fb1b Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Fri, 14 Aug 2026 07:01:23 +0800 Subject: sched_ext: fix stale references in doc comments - inlines.h: scx_bpf_dispatch() doesn't exist; the comment means scx_bpf_sub_dispatch() - internal.h: name %SCX_DEQ_SCHED_CHANGE instead of the never-defined %SCX_DEQ_SAVE - internal.h: @name shows up in the ops file in the scheduler's sysfs directory, not a "kernel.sched_ext_ops" sysctl Signed-off-by: Tao Cui Signed-off-by: Tejun Heo --- kernel/sched/ext/inlines.h | 2 +- kernel/sched/ext/internal.h | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/inlines.h b/kernel/sched/ext/inlines.h index 72d5ccd819d1..8f3be59863e5 100644 --- a/kernel/sched/ext/inlines.h +++ b/kernel/sched/ext/inlines.h @@ -22,7 +22,7 @@ enum scx_dsp_verdict { }; /* - * One user of this function is scx_bpf_dispatch() which can be called + * One user of this function is scx_bpf_sub_dispatch() which can be called * recursively as sub-sched dispatches nest. Always inline to reduce stack usage * from the call frame. */ diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index b699e7c1103f..c91296c53225 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -485,7 +485,7 @@ struct sched_ext_ops { * - sleeping (%SCX_DEQ_SLEEP) * - being moved to another CPU * - being temporarily taken off the queue for an attribute change - * (%SCX_DEQ_SAVE) + * (%SCX_DEQ_SCHED_CHANGE) * * This and ->dequeue() are related but not coupled. This operation * notifies @p's state transition and may not be preceded by ->dequeue() @@ -966,8 +966,9 @@ struct sched_ext_ops { * @name: BPF scheduler's name * * Must be a non-zero valid BPF object name including only isalnum(), - * '_' and '.' chars. Shows up in kernel.sched_ext_ops sysctl while the - * BPF scheduler is enabled. + * '_' and '.' chars. Exposed via the ops file in the scheduler's sysfs + * directory, /sys/kernel/sched_ext/root/ops for the root scheduler, + * while the BPF scheduler is enabled. */ char name[SCX_OPS_NAME_LEN]; -- cgit v1.2.3 From 03506edca637a8465dba9f635c50e9884fbcaf4e Mon Sep 17 00:00:00 2001 From: fangqiurong Date: Fri, 14 Aug 2026 11:26:23 +0800 Subject: sched_ext: Fix exit_task leak on fork failure during enable scx_fork() initializes tasks when scx_init_task_enabled is set, but scx_cancel_fork() only exits them when scx_enabled() is true. A fork that fails in the enable window (between releasing scx_fork_rwsem and setting __scx_enabled) runs ops.init_task() but never ops.exit_task(). Gate scx_cancel_fork() on scx_init_task_enabled. Fixes: 4269c603cc26 ("sched_ext: Enable scx_ops_init_task() separately") Cc: stable@vger.kernel.org # v6.12+ Signed-off-by: fangqiurong Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 61b156d7fc4b..2adf2bde0cb1 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -4163,7 +4163,7 @@ void scx_post_fork(struct task_struct *p) void scx_cancel_fork(struct task_struct *p) { - if (scx_enabled()) { + if (scx_init_task_enabled) { struct rq *rq; struct rq_flags rf; -- cgit v1.2.3 From d44093323131a42a85184315bb9b6da4813cc39b Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 14 Aug 2026 23:47:39 -1000 Subject: sched_ext: Drop unlocked scx_rq_clock_invalidate() from scx_root_disable() scx_root_disable() invalidates each rq's clock before taking the rq lock. scx_rq_clock_invalidate() is a plain read-modify-write of rq->scx.flags and every other writer of the word runs under the rq lock, so the unlocked update can race a concurrent flags update and lose one side's bits. The invalidation doesn't matter in the first place. The cached clock is read only by scx_bpf_now() from a loaded scheduler's BPF programs, nothing can re-validate the clock while sched_ext is disabled as scx_rq_clock_update() is gated on scx_enabled() too, and the usual rq lock cycles under the next scheduler refresh or invalidate it before it's practically observable. Drop the invalidation instead of fixing the locking. v2: Description and comment updated - the invalidation is unnecessary rather than subsumed by the rq lock cycle below. Fixes: 3a9910b5904d ("sched_ext: Implement scx_bpf_now()") Signed-off-by: Tejun Heo Cc: Changwoo Min --- kernel/sched/ext/ext.c | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 2adf2bde0cb1..0e136398e715 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -6476,12 +6476,9 @@ static void scx_root_disable(struct scx_sched *sch) percpu_up_write(&scx_fork_rwsem); /* - * Invalidate all the rq clocks to prevent getting outdated - * rq clocks from a previous scx scheduler. - * - * Also re-balance the dl_server bandwidth reservations: detach - * ext_server (no more sched_ext tasks) and reinstate fair_server if it - * was previously detached because we were running in full mode. + * Re-balance the dl_server bandwidth reservations: detach ext_server + * (no more sched_ext tasks) and reinstate fair_server if it was + * previously detached because we were running in full mode. * * Unlike the enable path, this runs on a recovery path that cannot * fail, so we use dl_server_swap_bw() to atomically free ext_server's @@ -6494,8 +6491,6 @@ static void scx_root_disable(struct scx_sched *sch) for_each_possible_cpu(cpu) { struct rq *rq = cpu_rq(cpu); - scx_rq_clock_invalidate(rq); - scoped_guard(rq_lock_irqsave, rq) { update_rq_clock(rq); if (was_switched_all) { -- cgit v1.2.3 From b27dfc7d8db91a24c772545e9f32739027ab49c9 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 14 Aug 2026 18:26:37 -1000 Subject: sched_ext: Make scx_bpf_events() read the calling scheduler's counters scx_bpf_events() always reads the root scheduler's event counters, so a sub-scheduler program querying its own events silently gets the root's instead and has no BPF-visible way to read its own (the per-scheduler sysfs "events" file is the only interface). Resolve the scheduler from the calling program with scx_prog_sched(). Unassociated programs follow the usual scx_prog_sched() resolution: the root scheduler under a pre-sub-attach compat root and zeroed counters otherwise. Also fix up the malformed comment into proper kerneldoc. Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 0e136398e715..0b31a9976e97 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -10604,19 +10604,23 @@ static void scx_read_events(struct scx_sched *sch, struct scx_event_stats *event } } -/* - * scx_bpf_events - Get a system-wide event counter to +/** + * scx_bpf_events - Read the event counters of the calling scheduler * @events: output buffer from a BPF program - * @events__sz: @events len, must end in '__sz'' for the verifier + * @events__sz: @events len, must end in '__sz' for the verifier + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Read the event counters of the scheduler associated with the calling program. + * @events is zeroed when no scheduler can be resolved. */ -__bpf_kfunc void scx_bpf_events(struct scx_event_stats *events, - size_t events__sz) +__bpf_kfunc void scx_bpf_events(struct scx_event_stats *events, size_t events__sz, + const struct bpf_prog_aux *aux) { struct scx_sched *sch; struct scx_event_stats e_sys; rcu_read_lock(); - sch = rcu_dereference(scx_root); + sch = scx_prog_sched(aux); if (sch) scx_read_events(sch, &e_sys); else @@ -10739,7 +10743,7 @@ BTF_ID_FLAGS(func, scx_bpf_cpu_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PRO BTF_ID_FLAGS(func, scx_bpf_cid_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED) BTF_ID_FLAGS(func, scx_bpf_tid_to_task, KF_RET_NULL | KF_RCU_PROTECTED) BTF_ID_FLAGS(func, scx_bpf_now) -BTF_ID_FLAGS(func, scx_bpf_events) +BTF_ID_FLAGS(func, scx_bpf_events, KF_IMPLICIT_ARGS) #ifdef CONFIG_CGROUP_SCHED BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_IMPLICIT_ARGS | KF_RCU | KF_ACQUIRE) #endif -- cgit v1.2.3 From 8d8dd8ae89eaa78b37fc85528e926029f5facbdf Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Sat, 15 Aug 2026 10:20:17 +0800 Subject: sched_ext: Don't BUG_ON a destroyed DSQ in process_deferred_reenq_users scx_bpf_dsq_reenq() queues a deferred reenq (dru) that runs from run_deferred(), not ops.dispatch(). If the DSQ is destroyed before the dru runs, process_deferred_reenq_users() sees dsq->id == SCX_DSQ_INVALID and hits the BUG_ON. destroy_dsq() doesn't flush pending drus, so just skip. tj: Read dsq->id once with READ_ONCE(). Reading it separately in the INVALID check and the BUG_ON would leave a window where destroy_dsq() can invalidate the id between the two reads and still trigger the BUG_ON. Fixes: 84b1a0ea0b7c ("sched_ext: Implement scx_bpf_dsq_reenq() for user DSQs") Cc: stable@vger.kernel.org # v7.1+ Signed-off-by: Tao Cui Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 0b31a9976e97..950d98508463 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -4613,7 +4613,7 @@ static void process_deferred_reenq_users(struct rq *rq) while (true) { struct scx_dispatch_q *dsq; - u64 reenq_flags; + u64 dsq_id, reenq_flags; scoped_guard (raw_spinlock, &rq->scx.deferred_reenq_lock) { struct scx_deferred_reenq_user *dru = @@ -4636,7 +4636,12 @@ static void process_deferred_reenq_users(struct rq *rq) /* see schedule_dsq_reenq() */ smp_mb(); - BUG_ON(dsq->id & SCX_DSQ_FLAG_BUILTIN); + /* destroy_dsq() may have raced and invalidated @dsq, nothing to reenq */ + dsq_id = READ_ONCE(dsq->id); + if (unlikely(dsq_id == SCX_DSQ_INVALID)) + continue; + + BUG_ON(dsq_id & SCX_DSQ_FLAG_BUILTIN); reenq_user(rq, dsq, reenq_flags); } } -- cgit v1.2.3 From 524ab50763af33d65e6e042cf7034cd8f82d437b Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 14 Aug 2026 23:44:52 -1000 Subject: sched_ext: Make SCHED_CLASS_EXT select GENERIC_ALLOCATOR kernel/sched/ext/arena.c uses the gen_pool allocator, which is built only when GENERIC_ALLOCATOR is set. SCHED_CLASS_EXT doesn't select it, so on configs where nothing else does, the build fails to link: build_policy.o: undefined reference to `gen_pool_create' build_policy.o: undefined reference to `gen_pool_for_each_chunk' build_policy.o: undefined reference to `gen_pool_destroy' Fixes: 9eca087deb0b ("sched_ext: Sub-allocator over kernel-claimed BPF arena pages") Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202608151315.tvN3X0Oq-lkp@intel.com/ Closes: https://lore.kernel.org/oe-kbuild-all/202608151632.3p91bTQj-lkp@intel.com/ Signed-off-by: Tejun Heo --- kernel/Kconfig.preempt | 1 + 1 file changed, 1 insertion(+) (limited to 'kernel') diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt index 88c594c6d7fc..f95859e4e3f0 100644 --- a/kernel/Kconfig.preempt +++ b/kernel/Kconfig.preempt @@ -169,6 +169,7 @@ config SCHED_CORE config SCHED_CLASS_EXT bool "Extensible Scheduling Class" depends on BPF_SYSCALL && BPF_JIT && DEBUG_INFO_BTF + select GENERIC_ALLOCATOR select STACKTRACE if STACKTRACE_SUPPORT help This option enables a new scheduler class sched_ext (SCX), which -- cgit v1.2.3 From e0253dd04beb03e79477c5ef4768b11135687206 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 14 Aug 2026 14:56:56 -1000 Subject: sched_ext: Keep kick_sync waiting on the rq's own CPU kick_sync_wait_bal_cb() assumes it runs on the rq's CPU from the __schedule() tail: the snapshots it compares against live in that CPU's percpu area and the busy-wait runs with the rq lock dropped and IRQs enabled. However, dispatch can now drop the rq lock while the callback sits queued, and rq lock takers in that window (the sched class change paths, the scx task iterator) flush pending balance callbacks on release, running the callback on a foreign CPU. Such a run compares against unrelated snapshots and can deadlock when the executing CPU is itself a wait target. Bail on a foreign CPU and leave the wait state alone. The wait only observes progress that the resched kicks already guarantee and the rq's next wait picks up the stale cpus_to_sync bits. Fixes: 4c95380701f5 ("sched/ext: Fold balance_scx() into pick_task_scx()") Cc: stable@vger.kernel.org # v6.19+ Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 950d98508463..743eba10650a 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -3269,11 +3269,26 @@ switch_class: static void kick_sync_wait_bal_cb(struct rq *rq) { - struct scx_kick_syncs __rcu *ks = __this_cpu_read(scx_kick_syncs); - unsigned long *ksyncs = rcu_dereference_sched(ks)->syncs; + struct scx_kick_syncs __rcu *ks; + unsigned long *ksyncs; bool waited; s32 cpu; + /* + * This callback is queued and normally flushed within @rq's own + * scheduling pass. However, dispatch can drop the rq lock while it sits + * queued, and lock takers in that window (the sched class change paths, + * the scx task iterator) flush pending balance callbacks on release, + * running this one on a foreign CPU whose snapshots are unrelated. The + * kicked CPUs are already on their way to advance the kick_syncs being + * waited on. Don't get in the way. + */ + if (unlikely(cpu_of(rq) != smp_processor_id())) + return; + + ks = __this_cpu_read(scx_kick_syncs); + ksyncs = rcu_dereference_sched(ks)->syncs; + /* * Drop rq lock and enable IRQs while waiting. IRQs must be enabled * — a target CPU may be waiting for us to process an IPI (e.g. TLB -- cgit v1.2.3 From 307a7b7ae59257027388d6fa19f507ff2f562336 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 14 Aug 2026 14:56:56 -1000 Subject: sched_ext: Drop the stale keep_prev fixup in dispatch_pick() The fixup demoting a keep verdict when @prev is not on ext_sched_class guarded against the rq-level SCX_RQ_BAL_KEEP flag going stale back when balancing and picking were separate operations. The verdict now travels in the return value, created and consumed in one invocation against the @prev it evaluated, and every keep decision tests SCX_TASK_QUEUED under the rq lock, which implies ext_sched_class as a class switch dequeues first. Drop the fixup along with dispatch_core_pick()'s copy. Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 12 ------------ 1 file changed, 12 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 743eba10650a..91fd81f3fe20 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -3355,11 +3355,6 @@ static enum scx_dsp_verdict dispatch_pick(struct rq *rq, struct rq_flags *rf, kick_sync_wait_bal_cb); } - if (unlikely(verdict == SCX_DSP_PREV && prev->sched_class != &ext_sched_class)) { - WARN_ON_ONCE(scx_enable_state() == SCX_ENABLED); - verdict = SCX_DSP_LOCAL; - } - return verdict; } @@ -3410,13 +3405,6 @@ static enum scx_dsp_verdict dispatch_core_pick(struct rq *rq, struct rq_flags *r if (rq->scx.lock_drop_seq != seq) return SCX_DSP_RETRY; - /* see dispatch_pick() */ - if (unlikely(verdict == SCX_DSP_PREV && - prev->sched_class != &ext_sched_class)) { - WARN_ON_ONCE(scx_enable_state() == SCX_ENABLED); - verdict = SCX_DSP_LOCAL; - } - return verdict; } #else /* CONFIG_SCHED_CORE */ -- cgit v1.2.3 From 3167bd3e0c22b1821df9987b9f4509e147cdad1f Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 14 Aug 2026 14:56:56 -1000 Subject: sched_ext: Rename balance-era identifiers to dispatch terms sched_class->balance() is gone from sched_ext and what balance_one() does is run dispatch to produce something pickable. Update the balance-era names to dispatch terms: - balance_one() -> dispatch_one() - SCX_RQ_IN_BALANCE -> SCX_RQ_IN_DISPATCH No BPF scheduler reads the flag. The enum autogen headers gain the new name with the old entry retained like other removed enumerators, zero-filling at load time. No functional changes. Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 52 ++++++++++++------------- kernel/sched/ext/inlines.h | 2 +- kernel/sched/ext/sub.c | 8 ++-- kernel/sched/sched.h | 2 +- tools/sched_ext/include/scx/enum_defs.autogen.h | 1 + tools/sched_ext/include/scx/enums.autogen.bpf.h | 3 ++ tools/sched_ext/include/scx/enums.autogen.h | 1 + 7 files changed, 37 insertions(+), 32 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 91fd81f3fe20..cbbf3fa7462b 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -377,9 +377,9 @@ static bool rq_is_open(struct rq *rq, u64 enq_flags) * If we're in the dispatch path holding rq lock, $curr may or may not * be ready depending on whether the on-going dispatch decides to extend * $curr's slice. We say yes here and resolve it at the end of dispatch. - * See balance_one(). + * See dispatch_one(). */ - if (rq->scx.flags & SCX_RQ_IN_BALANCE) + if (rq->scx.flags & SCX_RQ_IN_DISPATCH) return true; /* @@ -1019,7 +1019,7 @@ static void schedule_deferred(struct rq *rq) /* * This is the fallback when schedule_deferred_locked() can't use * the cheaper balance callback or wakeup hook paths (the target - * CPU is not in balance or wakeup). Currently, this is primarily + * CPU is not in dispatch or wakeup). Currently, this is primarily * hit by reenqueue operations targeting a remote CPU. * * Queue on the target CPU. The deferred work can run from any CPU @@ -1055,25 +1055,25 @@ static void schedule_deferred_locked(struct rq *rq) return; /* - * If in balance, the balance callbacks will be called before rq lock is - * released. Schedule one. + * If in dispatch, the balance callbacks will be called before rq lock + * is released. Schedule one. * * * We can't directly insert the callback into the * rq's list: The call can drop its lock and make the pending balance * callback visible to unrelated code paths that call rq_pin_lock(). * - * Just let balance_one() know that it must do it itself. + * Just let dispatch_one() know that it must do it itself. */ - if (rq->scx.flags & SCX_RQ_IN_BALANCE) { + if (rq->scx.flags & SCX_RQ_IN_DISPATCH) { rq->scx.flags |= SCX_RQ_BAL_CB_PENDING; return; } /* * No scheduler hooks available. Use the generic irq_work path. The - * above WAKEUP and BALANCE paths should cover most of the cases and the - * time to IRQ re-enable shouldn't be long. + * above WAKEUP and DISPATCH paths should cover most of the cases and + * the time to IRQ re-enable shouldn't be long. */ schedule_deferred(rq); } @@ -1616,14 +1616,14 @@ static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq, wakeup_preempt(rq, p, 0); /* - * If @rq is in balance, the CPU is already vacant and looking for the + * If @rq is in dispatch, the CPU is already vacant and looking for the * next task to run. No need to preempt or trigger resched after moving * @p into its local DSQ. * Note that the wakeup_preempt() above may have already triggered * a resched if @rq->next_class was idle. It's harmless, since * need_resched is cleared immediately after task pick. */ - if (rq->scx.flags & SCX_RQ_IN_BALANCE) + if (rq->scx.flags & SCX_RQ_IN_DISPATCH) return; if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr && @@ -2348,7 +2348,7 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_ * * @p may go through multiple stopping <-> running transitions between * here and put_prev_task_scx() if task attribute changes occur while - * balance_one() leaves @rq unlocked. However, they don't contain any + * dispatch_one() leaves @rq unlocked. However, they don't contain any * information meaningful to the BPF scheduler and can be suppressed by * skipping the callbacks if the task is !QUEUED. */ @@ -2976,14 +2976,14 @@ static inline void maybe_queue_balance_callback(struct rq *rq) rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING; } -static enum scx_dsp_verdict balance_one(struct rq *rq, struct task_struct *prev) +static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev) { struct scx_sched *sch = scx_root_protected_live(); enum scx_dsp_verdict verdict; s32 cpu = cpu_of(rq); lockdep_assert_rq_held(rq); - rq->scx.flags |= SCX_RQ_IN_BALANCE; + rq->scx.flags |= SCX_RQ_IN_DISPATCH; scx_process_sync_ecaps(rq, prev); @@ -3041,7 +3041,7 @@ static enum scx_dsp_verdict balance_one(struct rq *rq, struct task_struct *prev) verdict = SCX_DSP_PREV; goto has_tasks; } - rq->scx.flags &= ~SCX_RQ_IN_BALANCE; + rq->scx.flags &= ~SCX_RQ_IN_DISPATCH; return SCX_DSP_NONE; has_tasks: @@ -3058,7 +3058,7 @@ has_tasks: if (unlikely(rq->scx.local_dsq.nr > 1 && rq->scx.nr_immed)) scx_schedule_reenq_local(rq, 0); - rq->scx.flags &= ~SCX_RQ_IN_BALANCE; + rq->scx.flags &= ~SCX_RQ_IN_DISPATCH; return verdict; } @@ -3159,7 +3159,7 @@ static void switch_class(struct rq *rq, struct task_struct *next) * preempted, and it regaining control of the CPU. * * ->cpu_release() complements ->cpu_acquire(), which is emitted the - * next time that balance_one() is invoked. + * next time that dispatch_one() is invoked. */ if (!rq->scx.cpu_released) { if (sch->ops.cpu_release) { @@ -3340,7 +3340,7 @@ static enum scx_dsp_verdict dispatch_pick(struct rq *rq, struct rq_flags *rf, enum scx_dsp_verdict verdict; rq_unpin_lock(rq, rf); - verdict = balance_one(rq, prev); + verdict = dispatch_one(rq, prev); rq_repin_lock(rq, rf); maybe_queue_balance_callback(rq); @@ -3373,12 +3373,12 @@ static enum scx_dsp_verdict dispatch_core_pick(struct rq *rq, struct rq_flags *r u32 seq = rq->scx.lock_drop_seq; /* another dispatch is in flight on @rq, let that handle it */ - if (rq->scx.flags & SCX_RQ_IN_BALANCE) + if (rq->scx.flags & SCX_RQ_IN_DISPATCH) return SCX_DSP_NONE; rq_unpin_lock(rq, rf); - verdict = balance_one(rq, prev); + verdict = dispatch_one(rq, prev); if (cpu_of(rq) == smp_processor_id()) { maybe_queue_balance_callback(rq); @@ -3401,7 +3401,7 @@ static enum scx_dsp_verdict dispatch_core_pick(struct rq *rq, struct rq_flags *r rq_repin_lock(rq, rf); - /* if balance_one() released the rq lock, restart the selection */ + /* if dispatch_one() released the rq lock, restart the selection */ if (rq->scx.lock_drop_seq != seq) return SCX_DSP_RETRY; @@ -3437,7 +3437,7 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) /* * If any higher-priority sched class enqueued a runnable task on this - * rq during balance_one(), abort and return RETRY_TASK, so that the + * rq during dispatch_one(), abort and return RETRY_TASK, so that the * scheduler loop can restart. * * If @force_scx is true, always try to pick a SCHED_EXT task, @@ -6077,7 +6077,7 @@ static void unbypass_renotify_idle(struct rq *rq, struct scx_sched *pos, * * - ops.dispatch() is ignored. * - * - balance_one() does not report %SCX_DSP_PREV on non-zero slice as slice + * - dispatch_one() does not report %SCX_DSP_PREV on non-zero slice as slice * can't be trusted. Whenever a tick triggers, the running task is rotated to * the tail of the queue with core_sched_at touched. * @@ -8460,13 +8460,13 @@ static bool can_skip_idle_kick(struct rq *rq) * We can skip idle kicking if @rq is going to go through at least one * full SCX scheduling cycle before going idle. Just checking whether * curr is not idle is insufficient because we could be racing - * balance_one() trying to pull the next task from a remote rq, which + * dispatch_one() trying to pull the next task from a remote rq, which * may fail, and @rq may become idle afterwards. * * The race window is small and we don't and can't guarantee that @rq is * only kicked while idle anyway. Skip only when sure. */ - return !is_idle_task(rq->curr) && !(rq->scx.flags & SCX_RQ_IN_BALANCE); + return !is_idle_task(rq->curr) && !(rq->scx.flags & SCX_RQ_IN_DISPATCH); } static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_rq, @@ -9199,7 +9199,7 @@ __bpf_kfunc bool scx_bpf_dsq_move_to_local___v2(u64 dsq_id, u64 enq_flags, /* * A successfully consumed task can be dequeued before it starts * running while the CPU is trying to migrate other dispatched - * tasks. Bump nr_tasks to tell balance_one() to retry on empty + * tasks. Bump nr_tasks to tell dispatch_one() to retry on empty * local DSQ. */ dspc->nr_tasks++; diff --git a/kernel/sched/ext/inlines.h b/kernel/sched/ext/inlines.h index 8f3be59863e5..ed423bcc26b8 100644 --- a/kernel/sched/ext/inlines.h +++ b/kernel/sched/ext/inlines.h @@ -112,7 +112,7 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, * ops.dispatch() can trap us in this loop by repeatedly * dispatching ineligible tasks. Break out once in a while to * allow the watchdog to run. As IRQ can't be enabled in - * balance(), we want to complete this scheduling cycle and then + * dispatch, we want to complete this scheduling cycle and then * start a new one. IOW, we want to call resched_curr() on the * next, most likely idle, task, not the current one. Use * __scx_bpf_kick_cpu() for deferred kicking. diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index ec4729c99763..a7b38c90d095 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -918,7 +918,7 @@ static u64 calc_effective_caps(struct scx_pshard *ps, s32 cid) * @cid: cid to update * * Queue an ecaps update for @sch's @cid and kick the cpu so that it syncs in - * balance_one(). + * dispatch_one(). */ static void queue_sync_ecaps(struct scx_sched *sch, s32 cid) { @@ -953,7 +953,7 @@ static void discard_queued_syncs(struct rq *rq) /** * scx_process_sync_ecaps - Sync this cpu's ecaps to pshard->caps[] * @rq: the cid's cpu rq - * @prev: @rq's previous task from the in-progress balance + * @prev: @rq's previous task from the in-progress dispatch * * pshard->caps[] is the target configuration. pcpu->ecaps is the effective * transposed copy owned by the cid's cpu and written only here under @rq's @@ -1069,7 +1069,7 @@ void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) * sync when bypass lifts, so without a replay a cid that never changes again * would never be notified. The attach-time initial grants are the acute case * as they are consumed during the enable bypass window. Re-queue a sync for - * any undelivered delta so the next balance delivers it. + * any undelivered delta so the next dispatch delivers it. */ void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch) { @@ -2248,7 +2248,7 @@ __bpf_kfunc bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux * /* * Skip a child that does not effectively hold the base cap on this cpu: * its inserts would only be rejected. ecaps are synced at the top of - * balance_one() before dispatch, so this reflects the in-effect state. + * dispatch_one() before dispatch, so this reflects the in-effect state. */ if (scx_missing_caps(child, cpu_of(rq), SCX_CAP_BASE)) return false; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 5ae2d6d2d35b..7701a5a60972 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -790,7 +790,7 @@ enum scx_rq_flags { SCX_RQ_ROOT_IDLE_RENOTIFY = 1 << 8, /* the root is owed update_idle() */ SCX_RQ_IN_WAKEUP = 1 << 16, - SCX_RQ_IN_BALANCE = 1 << 17, + SCX_RQ_IN_DISPATCH = 1 << 17, }; /* per-rq rescue execution state, see scx_rescue_timerfn() */ diff --git a/tools/sched_ext/include/scx/enum_defs.autogen.h b/tools/sched_ext/include/scx/enum_defs.autogen.h index d609f369a337..19aa1de3e700 100644 --- a/tools/sched_ext/include/scx/enum_defs.autogen.h +++ b/tools/sched_ext/include/scx/enum_defs.autogen.h @@ -189,6 +189,7 @@ #define HAVE_SCX_RQ_ROOT_IDLE_RENOTIFY #define HAVE_SCX_RQ_IN_WAKEUP #define HAVE_SCX_RQ_IN_BALANCE +#define HAVE_SCX_RQ_IN_DISPATCH #define HAVE_SCX_SCHED_PCPU_BYPASSING #define HAVE_SCX_SLICE_OOB_DUR_BITS #define HAVE_SCX_SLICE_OOB_ID_BITS diff --git a/tools/sched_ext/include/scx/enums.autogen.bpf.h b/tools/sched_ext/include/scx/enums.autogen.bpf.h index d74b901688f1..7268131010de 100644 --- a/tools/sched_ext/include/scx/enums.autogen.bpf.h +++ b/tools/sched_ext/include/scx/enums.autogen.bpf.h @@ -34,6 +34,9 @@ const volatile u64 __SCX_RQ_IN_WAKEUP __weak; const volatile u64 __SCX_RQ_IN_BALANCE __weak; #define SCX_RQ_IN_BALANCE __SCX_RQ_IN_BALANCE +const volatile u64 __SCX_RQ_IN_DISPATCH __weak; +#define SCX_RQ_IN_DISPATCH __SCX_RQ_IN_DISPATCH + const volatile u64 __SCX_DSQ_FLAG_BUILTIN __weak; #define SCX_DSQ_FLAG_BUILTIN __SCX_DSQ_FLAG_BUILTIN diff --git a/tools/sched_ext/include/scx/enums.autogen.h b/tools/sched_ext/include/scx/enums.autogen.h index d58f3e59680e..e61632654517 100644 --- a/tools/sched_ext/include/scx/enums.autogen.h +++ b/tools/sched_ext/include/scx/enums.autogen.h @@ -15,6 +15,7 @@ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_CLK_VALID); \ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_IN_WAKEUP); \ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_IN_BALANCE); \ + SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_IN_DISPATCH); \ SCX_ENUM_SET(skel, scx_dsq_id_flags, SCX_DSQ_FLAG_BUILTIN); \ SCX_ENUM_SET(skel, scx_dsq_id_flags, SCX_DSQ_FLAG_LOCAL_ON); \ SCX_ENUM_SET(skel, scx_dsq_id_flags, SCX_DSQ_INVALID); \ -- cgit v1.2.3 From c384ab8a0b13741982669790a36a152acb2ede82 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 15 Aug 2026 14:06:50 -1000 Subject: sched_ext: Move the config-off sub-cap kfunc stubs into sub.c The EOPNOTSUPP stubs for the sub-cap kfuncs live in ext.c under #ifndef CONFIG_EXT_SUB_SCHED while the real definitions live in sub.c. Move the stubs into sub.c so all sub kfunc definitions live in one file. Pure code move, no functional change. Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 29 ----------------------------- kernel/sched/ext/sub.c | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 29 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index cbbf3fa7462b..2ae1df58939d 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -10682,35 +10682,6 @@ out: } #endif /* CONFIG_CGROUP_SCHED */ -#ifndef CONFIG_EXT_SUB_SCHED -__bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, - const struct scx_cmask *cmask__ign, - struct scx_cmask *denied_out__ign, - const struct bpf_prog_aux *aux) -{ - return -EOPNOTSUPP; -} - -__bpf_kfunc void scx_bpf_sub_revoke(u64 cgroup_id, u64 caps, - const struct scx_cmask *cmask__ign, - const struct bpf_prog_aux *aux) -{ -} - -__bpf_kfunc s32 scx_bpf_sub_caps(u64 cgroup_id, u64 caps, struct scx_cmask *out__ign, - const struct bpf_prog_aux *aux) -{ - return -EOPNOTSUPP; -} - -__bpf_kfunc s32 scx_bpf_sub_kill_bstr(u64 cgroup_id, char *fmt, - unsigned long long *data, u32 data__sz, - const struct bpf_prog_aux *aux) -{ - return -EOPNOTSUPP; -} -#endif /* !CONFIG_EXT_SUB_SCHED */ - __bpf_kfunc_end_defs(); BTF_KFUNCS_START(scx_kfunc_ids_any) diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index a7b38c90d095..92720a00b711 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -2648,4 +2648,37 @@ __bpf_kfunc s32 scx_bpf_sub_kill_bstr(u64 cgroup_id, char *fmt, __bpf_kfunc_end_defs(); +#else /* !CONFIG_EXT_SUB_SCHED */ + +__bpf_kfunc_start_defs(); + +__bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, + const struct scx_cmask *cmask__ign, + struct scx_cmask *denied_out__ign, + const struct bpf_prog_aux *aux) +{ + return -EOPNOTSUPP; +} + +__bpf_kfunc void scx_bpf_sub_revoke(u64 cgroup_id, u64 caps, + const struct scx_cmask *cmask__ign, + const struct bpf_prog_aux *aux) +{ +} + +__bpf_kfunc s32 scx_bpf_sub_caps(u64 cgroup_id, u64 caps, struct scx_cmask *out__ign, + const struct bpf_prog_aux *aux) +{ + return -EOPNOTSUPP; +} + +__bpf_kfunc s32 scx_bpf_sub_kill_bstr(u64 cgroup_id, char *fmt, + unsigned long long *data, u32 data__sz, + const struct bpf_prog_aux *aux) +{ + return -EOPNOTSUPP; +} + +__bpf_kfunc_end_defs(); + #endif /* CONFIG_EXT_SUB_SCHED */ -- cgit v1.2.3 From f7b6d128dd49a6eec09066ecfd29095f12588786 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 15 Aug 2026 13:08:28 -1000 Subject: sched_ext: Fix inverted ops.core_sched_before() invocation scx_prio_less() implements prio_less() semantics - %true means that @a is the lower priority and should run after @b. ops.core_sched_before() is documented to return %true when @a should run before @b. scx_prio_less() returns the op's value as-is, inverting the documented semantics at runtime. Call the op with the arguments swapped. scx_qmap followed the wiring instead of the documentation and returned %true for the younger task, so the two inversions canceled out and it behaved as intended. Flip its comparison to match. scx_qmap is likely the only current user in or out of the kernel tree. Any scheduler written the same way needs the same flip, while schedulers following the documentation are fixed by this change. Fixes: 7b0888b7cc19 ("sched_ext: Implement core-sched support") Cc: stable@vger.kernel.org # v6.12+ Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 10 ++++++++-- tools/sched_ext/scx_qmap.bpf.c | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 2ae1df58939d..9014c814a00a 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -3528,6 +3528,8 @@ void ext_server_init(struct rq *rq) * * When ops.core_sched_before() is enabled, @p->scx.core_sched_at is used to * implement FIFO ordering within each local DSQ. See pick_task_scx(). + * + * Return: %true if @a should run after @b. */ bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, bool in_fi) @@ -3536,6 +3538,10 @@ bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, struct scx_sched *sch_b = scx_task_sched(b); /* + * scx_prio_less() returns whether @a should run after @b while + * ops.core_sched_before() returns whether its first argument should run + * before the second. Swap the arguments. + * * The const qualifiers are dropped from task_struct pointers when * calling ops.core_sched_before(). Accesses are controlled by the * verifier. @@ -3544,8 +3550,8 @@ bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, !scx_bypassing(sch_a, task_cpu(a))) return SCX_CALL_OP_2TASKS_RET(sch_a, core_sched_before, task_rq(a), - (struct task_struct *)a, - (struct task_struct *)b); + (struct task_struct *)b, + (struct task_struct *)a); else return time_after64(a->scx.core_sched_at, b->scx.core_sched_at); } diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c index dd0434437835..723f45fe0cbc 100644 --- a/tools/sched_ext/scx_qmap.bpf.c +++ b/tools/sched_ext/scx_qmap.bpf.c @@ -900,7 +900,7 @@ static s64 task_qdist(struct task_struct *p) bool BPF_STRUCT_OPS(qmap_core_sched_before, struct task_struct *a, struct task_struct *b) { - return task_qdist(a) > task_qdist(b); + return task_qdist(a) < task_qdist(b); } /* -- cgit v1.2.3 From 0ec5dd0669291c8ffbee096367e078c26cbcc332 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 15 Aug 2026 13:08:28 -1000 Subject: sched_ext: Use runnable_at for the default core-sched task ordering The default core-sched ordering runs the longest waiting task first by comparing p->scx.core_sched_at stamps. The stamp is maintained under two rules. touch_core_sched() stamps when a task starts waiting for a CPU and when its slice runs out. If the scheduler implements ops.core_sched_before(), touch_core_sched_dispatch() re-stamps on every dispatch. A comparison can see one stamp taken under each rule, which isn't a meaningful ordering. The dispatch rule also buys little - it only aligns bypass-mode comparisons with the local DSQ order. Multiple schedulers make the mixed comparisons more common. Wait time is what p->scx.runnable_at already tracks for the stall watchdog. Delete core_sched_at with both touch functions and compare runnable_at in the scx_prio_less() fallback. runnable_at is refreshed only on enqueue and goes stale while a task keeps occupying its CPU. Instead of re-stamping, order a running task after every waiting task as it is the most recently serviced. Signed-off-by: Tejun Heo --- include/linux/sched/ext.h | 3 -- kernel/sched/ext/ext.c | 105 +++++++++++----------------------------------- 2 files changed, 25 insertions(+), 83 deletions(-) (limited to 'kernel') diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index a3ec980e2925..582d7cd4a983 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -212,9 +212,6 @@ struct sched_ext_entity { struct list_head runnable_node; /* rq->scx.runnable_list */ unsigned long runnable_at; -#ifdef CONFIG_SCHED_CORE - u64 core_sched_at; /* see scx_prio_less() */ -#endif #ifdef CONFIG_EXT_SUB_SCHED unsigned long rescue_at; /* queued on a rescue DSQ at, jiffies */ #endif diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 9014c814a00a..24663ae713a4 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1155,53 +1155,6 @@ void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, schedule_deferred(rq); } -/** - * touch_core_sched - Update timestamp used for core-sched task ordering - * @rq: rq to read clock from, must be locked - * @p: task to update the timestamp for - * - * Update @p->scx.core_sched_at timestamp. This is used by scx_prio_less() to - * implement global or local-DSQ FIFO ordering for core-sched. Should be called - * when a task becomes runnable and its turn on the CPU ends (e.g. slice - * exhaustion). - */ -static void touch_core_sched(struct rq *rq, struct task_struct *p) -{ - lockdep_assert_rq_held(rq); - -#ifdef CONFIG_SCHED_CORE - /* - * It's okay to update the timestamp spuriously. Use - * sched_core_disabled() which is cheaper than enabled(). - * - * As this is used to determine ordering between tasks of sibling CPUs, - * it may be better to use per-core dispatch sequence instead. - */ - if (!sched_core_disabled()) - p->scx.core_sched_at = sched_clock_cpu(cpu_of(rq)); -#endif -} - -/** - * touch_core_sched_dispatch - Update core-sched timestamp on dispatch - * @rq: rq to read clock from, must be locked - * @p: task being dispatched - * - * If the BPF scheduler implements custom core-sched ordering via - * ops.core_sched_before(), @p->scx.core_sched_at is used to implement FIFO - * ordering within each local DSQ. This function is called from dispatch paths - * and updates @p->scx.core_sched_at if custom core-sched ordering is in effect. - */ -static void touch_core_sched_dispatch(struct rq *rq, struct task_struct *p) -{ - lockdep_assert_rq_held(rq); - -#ifdef CONFIG_SCHED_CORE - if (unlikely(SCX_HAS_OP(scx_root, core_sched_before))) - touch_core_sched(rq, p); -#endif -} - /* * p->scx.slice_oob packs an out-of-band slice request into one atomic64. A zero * word means no request. Otherwise the fields are: @@ -1446,11 +1399,8 @@ static void update_curr_scx(struct rq *rq) if (unlikely(delta_exec <= 0)) return; - if (curr->scx.slice != SCX_SLICE_INF) { + if (curr->scx.slice != SCX_SLICE_INF) curr->scx.slice -= min_t(u64, curr->scx.slice, delta_exec); - if (!curr->scx.slice) - touch_core_sched(rq, curr); - } if (unlikely(curr == scx_rescuee(rq))) scx_rescue_charge(rq, delta_exec); @@ -1963,8 +1913,6 @@ static void direct_dispatch(struct scx_sched *sch, struct task_struct *p, find_dsq_for_dispatch(sch, rq, p->scx.ddsp_dsq_id, task_cpu(p)); u64 ddsp_enq_flags, slice, vtime; - touch_core_sched_dispatch(rq, p); - p->scx.ddsp_enq_flags |= enq_flags; /* @@ -2143,12 +2091,6 @@ bypass: goto enqueue; enqueue: - /* - * For task-ordering, slice refill must be treated as implying the end - * of the current slice. Otherwise, the longer @p stays on the CPU, the - * higher priority it becomes from scx_prio_less()'s POV. - */ - touch_core_sched(rq, p); refill_task_slice_dfl(sch, p); clear_direct_dispatch(p); scx_dispatch_enqueue(sch, rq, dsq, p, 0, 0, enq_flags); @@ -2226,9 +2168,6 @@ static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_ if (SCX_HAS_OP(sch, runnable) && !task_on_rq_migrating(p)) SCX_CALL_OP_TASK(sch, runnable, rq, p, enq_flags); - if (enq_flags & SCX_ENQ_WAKEUP) - touch_core_sched(rq, p); - /* Start dl_server if this is the first task being enqueued */ if (rq->scx.nr_running == 1) dl_server_start(&rq->ext_server); @@ -2886,7 +2825,6 @@ static void finish_dispatch(struct scx_sched *sch, struct rq *rq, struct task_st struct scx_dispatch_q *dsq; unsigned long opss; - touch_core_sched_dispatch(rq, p); retry: /* * No need for _acquire here. @p is accessed only after a successful @@ -3521,13 +3459,10 @@ void ext_server_init(struct rq *rq) * usual sched_class'es and needs to find out the expected task ordering. For * SCX, core-sched calls this function to interrogate the task ordering. * - * Unless overridden by ops.core_sched_before(), @p->scx.core_sched_at is used - * to implement the default task ordering. The older the timestamp, the higher - * priority the task - the global FIFO ordering matching the default scheduling - * behavior. - * - * When ops.core_sched_before() is enabled, @p->scx.core_sched_at is used to - * implement FIFO ordering within each local DSQ. See pick_task_scx(). + * Unless overridden by ops.core_sched_before(), the default task ordering runs + * the task which has been waiting longer first. A running task counts as the + * most recently serviced and orders after every waiting task. Waiting tasks are + * compared by @p->scx.runnable_at. * * Return: %true if @a should run after @b. */ @@ -3536,6 +3471,7 @@ bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, { struct scx_sched *sch_a = scx_task_sched(a); struct scx_sched *sch_b = scx_task_sched(b); + bool a_running, b_running; /* * scx_prio_less() returns whether @a should run after @b while @@ -3552,8 +3488,19 @@ bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, task_rq(a), (struct task_struct *)b, (struct task_struct *)a); - else - return time_after64(a->scx.core_sched_at, b->scx.core_sched_at); + + /* + * runnable_at is refreshed only on enqueue, so a task which keeps + * occupying its CPU carries a stale stamp. A running task is the most + * recently serviced whatever its stamp says. Order it after every + * waiting task. + */ + a_running = a->on_cpu; + b_running = b->on_cpu; + if (a_running != b_running) + return a_running; + + return time_after(a->scx.runnable_at, b->scx.runnable_at); } #endif /* CONFIG_SCHED_CORE */ @@ -3824,15 +3771,13 @@ static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued) update_curr_scx(rq); /* - * While disabling, always resched and refresh core-sched timestamp as - * we can't trust the slice management or ops.core_sched_before(). + * While disabling, always resched as we can't trust the slice + * management. */ - if (scx_bypassing(sch, cpu_of(rq))) { + if (scx_bypassing(sch, cpu_of(rq))) scx_set_task_slice(curr, 0); - touch_core_sched(rq, curr); - } else if (SCX_HAS_OP(sch, tick)) { + else if (SCX_HAS_OP(sch, tick)) SCX_CALL_OP_TASK(sch, tick, rq, curr); - } if (!curr->scx.slice) resched_curr(rq); @@ -6085,14 +6030,14 @@ static void unbypass_renotify_idle(struct rq *rq, struct scx_sched *pos, * * - dispatch_one() does not report %SCX_DSP_PREV on non-zero slice as slice * can't be trusted. Whenever a tick triggers, the running task is rotated to - * the tail of the queue with core_sched_at touched. + * the tail of the queue. * * - pick_next_task() suppresses zero slice warning. * * - scx_kick_cpu() is disabled to avoid irq_work malfunction during PM * operations. * - * - scx_prio_less() reverts to the default core_sched_at order. + * - scx_prio_less() reverts to the default runnable_at order. */ void scx_bypass(struct scx_sched *sch, bool bypass) { -- cgit v1.2.3 From 006dd4d04b379f4d76c0439b3f4b15d1216dac18 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 15 Aug 2026 13:08:28 -1000 Subject: sched_ext: Make core-sched task ordering hierarchy-aware With sub-schedulers, tasks of different schedulers routinely share rqs and SMT siblings, but scx_prio_less() consults ops.core_sched_before() only when both tasks belong to the same scheduler. Every pair spanning two schedulers falls back to the default ordering, so no scheduler can express ordering across a scheduler boundary, including a root over its sub-schedulers' tasks. Order a pair spanning schedulers by the nearest common ancestor that implements ops.core_sched_before(): both tasks are in its subtree, making this the one op where a scheduler is called on tasks it delegated to its sub-schedulers and may not be scheduling anymore. Same-scheduler pairs keep using the owning scheduler's op so a parent never orders inside a subtree it delegated. The op is skipped when the deciding scheduler is bypassing on either task's CPU. Update scx_qmap to fall back to the kernel's default ordering when handed a delegated task it has no task_ctx for. Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 38 ++++++++++++++++++++++++++++++-------- kernel/sched/ext/internal.h | 5 +++++ tools/sched_ext/scx_qmap.bpf.c | 23 ++++++++++++++++------- 3 files changed, 51 insertions(+), 15 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 24663ae713a4..694ac1ea417b 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -3459,10 +3459,16 @@ void ext_server_init(struct rq *rq) * usual sched_class'es and needs to find out the expected task ordering. For * SCX, core-sched calls this function to interrogate the task ordering. * - * Unless overridden by ops.core_sched_before(), the default task ordering runs - * the task which has been waiting longer first. A running task counts as the - * most recently serviced and orders after every waiting task. Waiting tasks are - * compared by @p->scx.runnable_at. + * A pair of tasks owned by one scheduler is ordered by the owner's + * ops.core_sched_before(). A pair spanning two schedulers is ordered by their + * nearest common ancestor which implements the op - the one case where the op + * is called on tasks that the scheduler delegated to its sub-schedulers and may + * not be scheduling anymore. + * + * When neither applies, or the deciding scheduler is bypassing on either task's + * CPU, the default ordering runs the task which has been waiting longer first. + * A running task counts as the most recently serviced and orders after every + * waiting task. Waiting tasks are compared by @p->scx.runnable_at. * * Return: %true if @a should run after @b. */ @@ -3471,8 +3477,26 @@ bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, { struct scx_sched *sch_a = scx_task_sched(a); struct scx_sched *sch_b = scx_task_sched(b); + struct scx_sched *sch = NULL; bool a_running, b_running; + if (sch_a == sch_b) { + if (SCX_HAS_OP(sch_a, core_sched_before)) + sch = sch_a; + } else { + s32 level; + + for (level = min(sch_a->level, sch_b->level); level >= 0; level--) { + struct scx_sched *anc = sch_a->ancestors[level]; + + if (anc == sch_b->ancestors[level] && + SCX_HAS_OP(anc, core_sched_before)) { + sch = anc; + break; + } + } + } + /* * scx_prio_less() returns whether @a should run after @b while * ops.core_sched_before() returns whether its first argument should run @@ -3482,10 +3506,8 @@ bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, * calling ops.core_sched_before(). Accesses are controlled by the * verifier. */ - if (sch_a == sch_b && SCX_HAS_OP(sch_a, core_sched_before) && - !scx_bypassing(sch_a, task_cpu(a))) - return SCX_CALL_OP_2TASKS_RET(sch_a, core_sched_before, - task_rq(a), + if (sch && !scx_bypassing(sch, task_cpu(a)) && !scx_bypassing(sch, task_cpu(b))) + return SCX_CALL_OP_2TASKS_RET(sch, core_sched_before, task_rq(a), (struct task_struct *)b, (struct task_struct *)a); diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index c91296c53225..fa20cac3ab61 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -521,6 +521,11 @@ struct sched_ext_ops { * the BPF scheduler. Should return %true if @a should run before @b. * %false if there's no required ordering or @b should run before @a. * + * In a scheduler hierarchy, a pair spanning two schedulers is ordered + * by the nearest common ancestor implementing this op, so the op may be + * called on tasks that the scheduler delegated to its sub-schedulers + * and is not scheduling anymore. See scx_prio_less(). + * * If not specified, the default is ordering them according to when they * became runnable. */ diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c index 723f45fe0cbc..a5f666716d80 100644 --- a/tools/sched_ext/scx_qmap.bpf.c +++ b/tools/sched_ext/scx_qmap.bpf.c @@ -866,16 +866,11 @@ void BPF_STRUCT_OPS(qmap_tick, struct task_struct *p) * The distance from the head of the queue scaled by the weight of the queue. * The lower the number, the older the task and the higher the priority. */ -static s64 task_qdist(struct task_struct *p) +static s64 task_qdist(struct task_struct *p, task_ctx_t *taskc) { int idx = weight_to_idx(p->scx.weight); - task_ctx_t *taskc; s64 qdist; - taskc = lookup_task_ctx(p); - if (!taskc) - return 0; - qdist = taskc->core_sched_seq - qa.core_sched_head_seqs[idx]; /* @@ -900,7 +895,21 @@ static s64 task_qdist(struct task_struct *p) bool BPF_STRUCT_OPS(qmap_core_sched_before, struct task_struct *a, struct task_struct *b) { - return task_qdist(a) < task_qdist(b); + task_ctx_t *taskc_a = lookup_task_ctx(a); + task_ctx_t *taskc_b = lookup_task_ctx(b); + + /* + * A task delegated to a sub-scheduler has no task_ctx here. Order such + * pairs by the kernel's default ordering - a running task after every + * waiting task, then by runnable_at. + */ + if (!taskc_a || !taskc_b) { + if (a->on_cpu != b->on_cpu) + return b->on_cpu; + return time_before(a->scx.runnable_at, b->scx.runnable_at); + } + + return task_qdist(a, taskc_a) < task_qdist(b, taskc_b); } /* -- cgit v1.2.3 From d023aa69c3b5f22a442fb67a37f17b04602eb43f Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 15 Aug 2026 13:08:28 -1000 Subject: sched_ext: Drop the dead SCX_DEQ_CORE_SCHED_EXEC test in dequeue_task_scx() dequeue_task_scx() masks SCX_DEQ_CORE_SCHED_EXEC out of the SCX_DEQ_SCHED_CHANGE decision, but the test can never fire: the incoming flags are an int of generic DEQUEUE_* bits while the flag is bit 32, and the core-sched execute path never goes through class dequeue anyway - set_next_task_scx() calls ops_dequeue() with the flag directly. The test was live when the SCX_DEQ_SCHED_CHANGE computation sat in ops_dequeue() and became dead when 03f5304aad0f ("sched_ext: Pass full dequeue flags to ops.quiescent()") moved the computation here. Drop it. Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 694ac1ea417b..532c979a1de4 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -2267,9 +2267,9 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_ /* * Set %SCX_DEQ_SCHED_CHANGE when the dequeue is due to a property - * change (not sleep or core-sched pick). + * change (not sleep). */ - if (!(deq_flags & (DEQUEUE_SLEEP | SCX_DEQ_CORE_SCHED_EXEC))) + if (!(deq_flags & DEQUEUE_SLEEP)) deq_flags |= SCX_DEQ_SCHED_CHANGE; if (!(p->scx.flags & SCX_TASK_QUEUED)) { -- cgit v1.2.3