From 252891d03dfb239fb76a78ab06b0e5a8719e0f86 Mon Sep 17 00:00:00 2001 From: Cheng-Yang Chou Date: Wed, 10 Jun 2026 23:26:56 +0800 Subject: sched_ext, rcu: Upgrade RCU stall paths to report cpumask of stalled CPUs scx_rcu_cpu_stall() previously recorded the detector CPU rather than the stalled one, and the expedited grace period path had no stalled CPU to report at all. Thread a cpumask through panic_on_rcu_stall() and scx_rcu_cpu_stall() to capture all stalled CPUs. Report cpumask_first() as exit_cpu and the full CPU list in the exit message. Task-only stalls yield exit_cpu = -1. Store the stall mask in scx_sched rather than scx_exit_info, keeping the BPF-visible struct unchanged. scx_dump_state() reads sch->stall_cpus directly and dumps all stalled CPUs first to avoid losing them to truncation. Signed-off-by: Cheng-Yang Chou Reviewed-by: Paul E. McKenney Reviewed-by: Andrea Righi Signed-off-by: Tejun Heo --- include/linux/sched/ext.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index 20b2343aa344..75cb8b119fb7 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -263,7 +263,7 @@ void sched_ext_dead(struct task_struct *p); void print_scx_info(const char *log_lvl, struct task_struct *p); void scx_softlockup(u32 dur_s); bool scx_hardlockup(int cpu); -bool scx_rcu_cpu_stall(void); +bool scx_rcu_cpu_stall(const struct cpumask *stalled_mask); #else /* !CONFIG_SCHED_CLASS_EXT */ @@ -271,7 +271,7 @@ static inline void sched_ext_dead(struct task_struct *p) {} static inline void print_scx_info(const char *log_lvl, struct task_struct *p) {} static inline void scx_softlockup(u32 dur_s) {} static inline bool scx_hardlockup(int cpu) { return false; } -static inline bool scx_rcu_cpu_stall(void) { return false; } +static inline bool scx_rcu_cpu_stall(const struct cpumask *stalled_mask) { return false; } #endif /* CONFIG_SCHED_CLASS_EXT */ -- cgit v1.2.3 From 75a8c8202c918f567e7a7b16add40b0be02c2113 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 13 Jul 2026 22:18:43 -1000 Subject: sched_ext: Add reject DSQ for cap-rejected dispatches When a sub-scheduler dispatches a task to a CPU it lacks the required capability on, the task must be rejected rather than allowed to run. Add the machinery for that. Each rq gets a reject DSQ, a kernel-internal holding queue that is never run and that the BPF scheduler cannot reach. An insert that must be refused is diverted there instead of the local DSQ, and a deferred requeue then hands the parked tasks back to the BPF scheduler to re-decide. A cap revoke extends this to already-queued tasks. When the revoke reaches the cpu's effective caps, the cpu scans its local DSQ and reenqueues the tasks that no longer qualify. A migration-disabled task must run on its cpu, so a capless one is admitted anyway and counted in the new SCX_EV_SUB_FORCED_ADMIT event. This is preparation for the actual sub-sched cap enforcement. The divert is wired but inert here. v2: Admit offline-rq and migration_pending inserts to local, not reject. (sashiko AI) Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- include/linux/sched/ext.h | 15 +++++- kernel/sched/ext/ext.c | 42 ++++++++++++--- kernel/sched/ext/internal.h | 19 ++++++- kernel/sched/ext/sub.c | 127 +++++++++++++++++++++++++++++++++++++++++++- kernel/sched/ext/sub.h | 49 +++++++++++++++++ kernel/sched/sched.h | 3 ++ 6 files changed, 244 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index 75cb8b119fb7..7e3f6b33f4a8 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -58,6 +58,7 @@ enum scx_dsq_id_flags { SCX_DSQ_GLOBAL = SCX_DSQ_FLAG_BUILTIN | 1, SCX_DSQ_LOCAL = SCX_DSQ_FLAG_BUILTIN | 2, SCX_DSQ_BYPASS = SCX_DSQ_FLAG_BUILTIN | 3, + SCX_DSQ_REJECT = SCX_DSQ_FLAG_BUILTIN | 4, /* internal - see find_dsq_for_dispatch() */ SCX_DSQ_LOCAL_ON = SCX_DSQ_FLAG_BUILTIN | SCX_DSQ_FLAG_LOCAL_ON, SCX_DSQ_LOCAL_CPU_MASK = 0xffffffffLLU, }; @@ -124,7 +125,7 @@ enum scx_ent_flags { SCX_TASK_DEAD = 5 << SCX_TASK_STATE_SHIFT, /* - * Bits 12 and 13 are used to carry reenqueue reason. In addition to + * Bits 12 to 14 are used to carry reenqueue reason. In addition to * %SCX_ENQ_REENQ flag, ops.enqueue() can also test for * %SCX_TASK_REENQ_REASON_NONE to distinguish reenqueues. * @@ -132,15 +133,17 @@ enum scx_ent_flags { * KFUNC reenqueued by scx_bpf_dsq_reenq() and friends * IMMED reenqueued due to failed ENQ_IMMED * PREEMPTED preempted while running + * CAP sub-sched cap miss, see p->scx.reenq_reason_* */ SCX_TASK_REENQ_REASON_SHIFT = 12, - SCX_TASK_REENQ_REASON_BITS = 2, + SCX_TASK_REENQ_REASON_BITS = 3, SCX_TASK_REENQ_REASON_MASK = ((1 << SCX_TASK_REENQ_REASON_BITS) - 1) << SCX_TASK_REENQ_REASON_SHIFT, SCX_TASK_REENQ_NONE = 0 << SCX_TASK_REENQ_REASON_SHIFT, SCX_TASK_REENQ_KFUNC = 1 << SCX_TASK_REENQ_REASON_SHIFT, SCX_TASK_REENQ_IMMED = 2 << SCX_TASK_REENQ_REASON_SHIFT, SCX_TASK_REENQ_PREEMPTED = 3 << SCX_TASK_REENQ_REASON_SHIFT, + SCX_TASK_REENQ_CAP = 4 << SCX_TASK_REENQ_REASON_SHIFT, /* iteration cursor, not a task */ SCX_TASK_CURSOR = 1 << 31, @@ -239,6 +242,14 @@ struct sched_ext_entity { */ u64 dsq_vtime; + /* + * Sub-sched cap rejected reenq context, valid only while + * %SCX_TASK_REENQ_CAP is set. @reenq_reason_caps is the SCX_CAP_* bits + * that were needed but missing. @reenq_reason_cid is the target cid. + */ + u64 reenq_reason_caps; + s32 reenq_reason_cid; + /* * If set, reject future sched_setscheduler(2) calls updating the policy * to %SCHED_EXT with -%EACCES. diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 71219b818b4f..547468af9674 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -101,6 +101,7 @@ static bool dsq_is_rq_owned(struct scx_dispatch_q *dsq) { switch (dsq->id) { case SCX_DSQ_LOCAL: + case SCX_DSQ_REJECT: return true; default: return false; @@ -1287,6 +1288,12 @@ static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq, { call_task_dequeue(sch, rq, p, 0); + /* rejected: kick the deferred reenq, skip wakeup/preemption */ + if (unlikely(dsq->id == SCX_DSQ_REJECT)) { + schedule_deferred_locked(rq); + return; + } + /* * Note that @rq's lock may be dropped between this enqueue and @p * actually getting on CPU. This gives higher-class tasks (e.g. RT) @@ -1344,7 +1351,12 @@ static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq, struct scx_dispatch_q *dsq, struct task_struct *p, u64 enq_flags) { - bool is_rq_owned = dsq_is_rq_owned(dsq); + bool is_rq_owned = false; + + if (dsq->id == SCX_DSQ_LOCAL) { + dsq = scx_local_or_reject_dsq(sch, rq, p, &enq_flags); + is_rq_owned = true; + } WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node)); WARN_ON_ONCE((p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) || @@ -1494,7 +1506,7 @@ static void task_unlink_from_dsq(struct task_struct *p, } } -static void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p) +void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p) { struct scx_dispatch_q *dsq = p->scx.dsq; bool is_rq_owned = dsq && dsq_is_rq_owned(dsq); @@ -1584,6 +1596,10 @@ static struct scx_dispatch_q *find_dsq_for_dispatch(struct scx_sched *sch, else dsq = find_user_dsq(sch, dsq_id); + /* + * Built-in DSQs are never inserted into dsq_hash, so REJECT hits the + * error below. It cannot be reached with an ID. + */ if (unlikely(!dsq)) { scx_error(sch, "non-existent DSQ 0x%llx", dsq_id); return find_global_dsq(sch, tcpu); @@ -1709,8 +1725,8 @@ bool scx_rq_online(struct rq *rq) return likely((rq->scx.flags & SCX_RQ_ONLINE) && cpu_active(cpu_of(rq))); } -static void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, - int sticky_cpu) +void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, + int sticky_cpu) { struct scx_sched *sch = scx_task_sched(p); struct task_struct **ddsp_taskp; @@ -2079,7 +2095,7 @@ static void move_local_task_to_local_dsq(struct scx_sched *sch, struct scx_dispatch_q *src_dsq, struct rq *dst_rq) { - struct scx_dispatch_q *dst_dsq = &dst_rq->scx.local_dsq; + struct scx_dispatch_q *dst_dsq = scx_local_or_reject_dsq(sch, dst_rq, p, &enq_flags); /* @dsq is locked and @p is on @dst_rq */ lockdep_assert_held(&src_dsq->lock); @@ -3808,7 +3824,8 @@ static void process_ddsp_deferred_locals(struct rq *rq) * another reenq cycle. Repetitions are bounded by %SCX_REENQ_LOCAL_MAX_REPEAT * in process_deferred_reenq_locals(). */ -static bool local_task_should_reenq(struct task_struct *p, u64 *reenq_flags, u32 *reason) +static bool local_task_should_reenq(struct rq *rq, struct task_struct *p, + u64 *reenq_flags, u32 *reason) { bool first; @@ -3824,6 +3841,12 @@ static bool local_task_should_reenq(struct task_struct *p, u64 *reenq_flags, u32 return true; } + if ((*reenq_flags & SCX_REENQ_CAP_REVOKE) && + scx_task_reenq_on_cap_revoke(rq, p)) { + *reason = SCX_TASK_REENQ_CAP; + return true; + } + return *reenq_flags & SCX_REENQ_ANY; } @@ -3867,7 +3890,7 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) if (!scx_is_descendant(task_sch, sch)) continue; - if (!local_task_should_reenq(p, &reenq_flags, &reason)) + if (!local_task_should_reenq(rq, p, &reenq_flags, &reason)) continue; scx_dispatch_dequeue(rq, p); @@ -4063,6 +4086,8 @@ static void run_deferred(struct rq *rq) if (!list_empty(&rq->scx.deferred_reenq_users)) process_deferred_reenq_users(rq); + + scx_reenq_reject(rq); } #ifdef CONFIG_NO_HZ_FULL @@ -7896,6 +7921,9 @@ void __init init_sched_ext_class(void) /* local_dsq's sch will be set during scx_root_enable() */ BUG_ON(init_dsq(&rq->scx.local_dsq, SCX_DSQ_LOCAL, NULL)); +#ifdef CONFIG_EXT_SUB_SCHED + BUG_ON(init_dsq(&rq->scx.reject_dsq, SCX_DSQ_REJECT, NULL)); +#endif INIT_LIST_HEAD(&rq->scx.runnable_list); INIT_LIST_HEAD(&rq->scx.ddsp_deferred_locals); diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index b1b3937168fb..a360d312702b 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1135,6 +1135,13 @@ struct scx_event_stats { * from sub_bypass_dsq's. */ s64 SCX_EV_SUB_BYPASS_DISPATCH; + + /* + * The number of times a migration-disabled task lacking the cap for its + * cid was allowed onto the local DSQ. It must run on its pinned CPU, so + * it can't be rejected. The violation is counted here. + */ + s64 SCX_EV_SUB_FORCED_ADMIT; }; #define SCX_EVENTS_LIST(SCX_EVENT) \ @@ -1150,7 +1157,8 @@ struct scx_event_stats { SCX_EVENT(SCX_EV_BYPASS_DISPATCH); \ SCX_EVENT(SCX_EV_BYPASS_ACTIVATE); \ SCX_EVENT(SCX_EV_INSERT_NOT_OWNED); \ - SCX_EVENT(SCX_EV_SUB_BYPASS_DISPATCH) + SCX_EVENT(SCX_EV_SUB_BYPASS_DISPATCH); \ + SCX_EVENT(SCX_EV_SUB_FORCED_ADMIT) struct scx_sched; @@ -1270,6 +1278,9 @@ enum scx_cap_flags { __SCX_CAP_ALL = BIT_U64(__SCX_NR_CAPS) - 1, SCX_CAP_DUMMY = BIT_U64(__SCX_CAP_DUMMY), + + /* caps whose loss strands queued tasks, see scx_process_sync_ecaps() */ + SCX_CAPS_REENQ_ON_LOSS = 0, }; #ifdef CONFIG_EXT_SUB_SCHED @@ -1583,6 +1594,9 @@ enum scx_reenq_flags { /* low 16bits determine which tasks should be reenqueued */ SCX_REENQ_ANY = 1LLU << 0, /* all tasks */ + /* internal: kernel-issued on cap revoke, not accepted from BPF */ + SCX_REENQ_CAP_REVOKE = 1LLU << 1, + __SCX_REENQ_FILTER_MASK = 0xffffLLU, __SCX_REENQ_USER_MASK = SCX_REENQ_ANY, @@ -1835,6 +1849,9 @@ void scx_task_iter_start(struct scx_task_iter *iter, struct cgroup *cgrp); void scx_task_iter_unlock(struct scx_task_iter *iter); void scx_task_iter_stop(struct scx_task_iter *iter); struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter); +void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p); +void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, + int sticky_cpu); bool scx_consume_dispatch_q(struct scx_sched *sch, struct rq *rq, struct scx_dispatch_q *dsq, u64 enq_flags); bool scx_consume_global_dsq(struct scx_sched *sch, struct rq *rq); diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index a86ecfb623c4..bbd068bfd86c 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -216,6 +216,118 @@ void scx_init_root_caps(struct scx_sched *sch) } } +/** + * scx_local_or_reject_dsq - Pick the local or reject DSQ for an insert + * @sch: enqueuing sub-sched + * @rq: rq whose local DSQ @p targets + * @p: task being inserted + * @enq_flags: in/out; %SCX_ENQ_IMMED is cleared when diverting to reject + * + * Return @rq's local DSQ if @sch holds the required caps on @rq's cid, + * otherwise @rq's reject DSQ after recording the reenq reason on @p. + * + * Bypass doesn't need special-casing as a bypassing sched's tasks are enqueued + * to and run by its nearest non-bypassing ancestor. If root is bypassing, it + * always holds all caps. + */ +struct scx_dispatch_q *scx_local_or_reject_dsq(struct scx_sched *sch, struct rq *rq, + struct task_struct *p, u64 *enq_flags) +{ + s32 cid = __scx_cpu_to_cid(cpu_of(rq)); + u64 missing = scx_missing_caps(sch, cpu_of(rq), scx_caps_for_enq(*enq_flags)); + + /* requirements met */ + if (likely(!missing)) + return &rq->scx.local_dsq; + + /* + * The task must run on this CPU regardless of caps: the rq is draining + * offline (BPF scheduler bypassed), the task is migration-disabled, or a + * migration is pending. Admit despite the missing caps and count it. + */ + if (unlikely(!scx_rq_online(rq) || is_migration_disabled(p) || + p->migration_pending)) { + __scx_add_event(sch, SCX_EV_SUB_FORCED_ADMIT, 1); + return &rq->scx.local_dsq; + } + + p->scx.reenq_reason_caps = missing; + p->scx.reenq_reason_cid = cid; + + /* + * Only local DSQ can honor IMMED and dsq_inc_nr() WARNs on IMMED into + * others. Strip both the enq flag and the sticky task flag - the + * latter can carry in from an earlier admitted IMMED insert. + */ + *enq_flags &= ~SCX_ENQ_IMMED; + p->scx.flags &= ~SCX_TASK_IMMED; + + return &rq->scx.reject_dsq; +} + +/* @p lost the caps needed to stay on @rq's local DSQ? Record reason if so. */ +bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p) +{ + u64 missing; + + /* migration-disabled tasks are admitted regardless of caps */ + if (is_migration_disabled(p)) + return false; + + missing = scx_missing_caps(scx_task_sched(p), cpu_of(rq), scx_caps_for_task(p)); + if (likely(!missing)) + return false; + + p->scx.reenq_reason_caps = missing; + p->scx.reenq_reason_cid = __scx_cpu_to_cid(cpu_of(rq)); + return true; +} + +/* + * Drain @rq->scx.reject_dsq, reenqueueing each task so the BPF re-decides + * from p->scx.reenq_reason_*. + * + * A task can be re-rejected repeatedly, and there's no repeat limit here. + * Rejection can't happen for root, and sub-scheds can be safely ejected after + * triggering the stall watchdog. + */ +void scx_reenq_reject(struct rq *rq) +{ + LIST_HEAD(tasks); + struct task_struct *p, *n; + + lockdep_assert_rq_held(rq); + + if (list_empty(&rq->scx.reject_dsq.list)) + return; + + /* + * Move to a private list so a task re-rejected by the + * scx_do_enqueue_task() below isn't revisited this round. + */ + list_for_each_entry_safe(p, n, &rq->scx.reject_dsq.list, scx.dsq_list.node) { + /* migration_pending tasks should have bypassed to local DSQ */ + if (WARN_ON_ONCE(p->migration_pending)) + continue; + + scx_dispatch_dequeue(rq, p); + + if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) + p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; + p->scx.flags |= SCX_TASK_REENQ_CAP; + + list_add_tail(&p->scx.dsq_list.node, &tasks); + } + + list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) { + list_del_init(&p->scx.dsq_list.node); + + scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); + + p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; + } +} + /* record a caps change, see struct scx_caps_updated */ static void caps_updated_record(struct scx_pshard *ps, const struct scx_cmask *cids, u64 caps, struct list_head *to_deliver) @@ -361,6 +473,7 @@ void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) s32 cpu = cpu_of(rq); s32 cid, shard; struct llist_node *batch, *pos, *tmp; + u64 lost_all = 0; lockdep_assert_rq_held(rq); @@ -389,16 +502,20 @@ void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) struct scx_sched_pcpu *pcpu = container_of(pos, struct scx_sched_pcpu, ecaps_to_sync_node); struct scx_pshard *ps = pcpu->sch->pshard[shard]; - u64 ecaps; + u64 old, ecaps, lost; init_llist_node(pos); /* pairs with smp_mb() in queue_sync_ecaps(), see there */ smp_mb(); + old = READ_ONCE(pcpu->ecaps); ecaps = calc_effective_caps(ps, cid); WRITE_ONCE(pcpu->ecaps, ecaps); + lost = old & ~ecaps; + lost_all |= lost; + /* tell the sched its effective caps on this cid changed */ if (ecaps != pcpu->reported_ecaps && SCX_HAS_OP(pcpu->sch, sub_ecaps_updated) && @@ -415,6 +532,14 @@ void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) pcpu->reported_ecaps = ecaps; } } + + /* + * Losing a cap can strand already-queued tasks. Schedule a reenq scan + * to move the now-capless ones off the local DSQ. The scan tests + * against the effective caps and thus must come after the ecaps sync. + */ + if (lost_all & SCX_CAPS_REENQ_ON_LOSS) + scx_schedule_reenq_local(rq, SCX_REENQ_CAP_REVOKE); } /* diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index d28d16d84cd5..9fc95d27f393 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -34,6 +34,10 @@ void scx_online_ecaps(struct rq *rq); void scx_offline_ecaps(struct rq *rq); void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu); void scx_discard_stale_ecaps_syncs(void); +struct scx_dispatch_q *scx_local_or_reject_dsq(struct scx_sched *sch, struct rq *rq, + struct task_struct *p, u64 *enq_flags); +bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p); +void scx_reenq_reject(struct rq *rq); static inline const char *sch_cgrp_path(struct scx_sched *sch) { @@ -58,6 +62,9 @@ static inline void scx_online_ecaps(struct rq *rq) {} static inline void scx_offline_ecaps(struct rq *rq) {} static inline void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu) {} static inline void scx_discard_stale_ecaps_syncs(void) {} +static inline struct scx_dispatch_q *scx_local_or_reject_dsq(struct scx_sched *sch, struct rq *rq, struct task_struct *p, u64 *enq_flags) { return &rq->scx.local_dsq; } +static inline bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p) { return false; } +static inline void scx_reenq_reject(struct rq *rq) {} #endif /* CONFIG_EXT_SUB_SCHED */ @@ -76,12 +83,54 @@ static inline void scx_discard_stale_ecaps_syncs(void) {} #ifdef CONFIG_EXT_SUB_SCHED +/** + * scx_missing_caps - The caps in @needed that @sch lacks on @cpu + * @sch: sched to test + * @cpu: cpu to test on + * @needed: bitmask of SCX_CAP_* values + * + * Return the caps in @needed that @sch lacks for @cpu, 0 if it holds them all. + */ +static inline u64 scx_missing_caps(struct scx_sched *sch, s32 cpu, u64 needed) +{ + u64 ecaps; + + /* root holds every cap on every cpu */ + if (!sch->level) + return 0; + + ecaps = READ_ONCE(per_cpu_ptr(sch->pcpu, cpu)->ecaps); + + return needed & ~ecaps; +} + +/* + * Cap semantics: which caps an action requires, and which caps a cap implies. + * Keep all such mappings collected here. + */ + +/* map @enq_flags to the SCX_CAP_* bit required for the local-DSQ insert */ +static inline u64 scx_caps_for_enq(u64 enq_flags) +{ + return 0; +} + +/* map queued @p to the SCX_CAP_* bit required to stay on its local DSQ */ +static inline u64 scx_caps_for_task(struct task_struct *p) +{ + return 0; +} + /* caps implied by holding @cap */ static inline u64 scx_caps_implied(u64 cap) { return 0; } +#else /* CONFIG_EXT_SUB_SCHED */ + +static inline u64 scx_missing_caps(struct scx_sched *sch, s32 cpu, u64 needed) { return 0; } + #endif /* CONFIG_EXT_SUB_SCHED */ /* diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 72299c105487..c0a4699a6c0a 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -794,6 +794,9 @@ enum scx_rq_flags { struct scx_rq { struct scx_dispatch_q local_dsq; +#ifdef CONFIG_EXT_SUB_SCHED + struct scx_dispatch_q reject_dsq; /* staging for cap-rejected tasks */ +#endif struct list_head runnable_list; /* runnable tasks on this rq */ struct list_head ddsp_deferred_locals; /* deferred ddsps from enq */ unsigned long ops_qseq; -- cgit v1.2.3 From d5b8f4cdd17dc42fd51048785fef1e2a7dd1117f Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 13 Jul 2026 22:18:43 -1000 Subject: sched_ext: Track the cpu a task is runnable on Add p->scx.runnable_cpu, the cpu @p is runnable on, or -1 when it is not. It is stamped as @p joins the runnable_list (set_task_runnable()) and cleared as it leaves (clr_task_runnable()), both under the rq lock. task_cpu() can't answer "is @p on this rq" reliably: a remote wakeup changes it under @p's pi_lock alone, without the source rq lock, so it can read as the locked rq while @p is really elsewhere. runnable_cpu changes only under the rq lock, so a caller holding an rq lock can compare against it to know whether that is @p's current rq. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- include/linux/sched/ext.h | 1 + init/init_task.c | 1 + kernel/sched/ext/ext.c | 9 +++++++++ 3 files changed, 11 insertions(+) (limited to 'include/linux') diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index 7e3f6b33f4a8..853f03b63133 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -201,6 +201,7 @@ struct sched_ext_entity { s32 sticky_cpu; s32 holding_cpu; s32 selected_cpu; + s32 runnable_cpu; /* cpu @p is runnable on, -1 if not */ struct task_struct *kf_tasks[2]; /* see SCX_CALL_OP_TASK() */ struct list_head runnable_node; /* rq->scx.runnable_list */ diff --git a/init/init_task.c b/init/init_task.c index b67ef6040a65..5c7ad50ac685 100644 --- a/init/init_task.c +++ b/init/init_task.c @@ -145,6 +145,7 @@ struct task_struct init_task __aligned(L1_CACHE_BYTES) = { .dsq_list.node = LIST_HEAD_INIT(init_task.scx.dsq_list.node), .sticky_cpu = -1, .holding_cpu = -1, + .runnable_cpu = -1, .runnable_node = LIST_HEAD_INIT(init_task.scx.runnable_node), .runnable_at = INITIAL_JIFFIES, .ddsp_dsq_id = SCX_DSQ_INVALID, diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 35aca50796fd..9ae8d78738b4 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1865,11 +1865,19 @@ static void set_task_runnable(struct rq *rq, struct task_struct *p) * appended to the runnable_list. */ list_add_tail(&p->scx.runnable_node, &rq->scx.runnable_list); + + /* + * Record the rq @p is runnable on, maintained under the rq lock so it + * stays valid unlike task_cpu(), which a remote wakeup can move under + * pi_lock alone. + */ + WRITE_ONCE(p->scx.runnable_cpu, cpu_of(rq)); } static void clr_task_runnable(struct task_struct *p, bool reset_runnable_at) { list_del_init(&p->scx.runnable_node); + WRITE_ONCE(p->scx.runnable_cpu, -1); if (reset_runnable_at) p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT; } @@ -3531,6 +3539,7 @@ void init_scx_entity(struct sched_ext_entity *scx) RB_CLEAR_NODE(&scx->dsq_priq); scx->sticky_cpu = -1; scx->holding_cpu = -1; + scx->runnable_cpu = -1; INIT_LIST_HEAD(&scx->runnable_node); scx->runnable_at = jiffies; scx->ddsp_dsq_id = SCX_DSQ_INVALID; -- cgit v1.2.3 From 46a85ae6fe5b468107baa9f21f073a940208d9ff Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 13 Jul 2026 22:18:43 -1000 Subject: sched_ext: Tie cpu occupancy to SCX_CAP_BASE through the task slice A task's slice grants it cpu occupancy - how long it holds its cpu. In a sub-scheduler hierarchy cpu access is delegated through revocable capabilities, so a task's occupancy must follow them. Only its own scheduler sets its slice, and extending the slice is allowed only while that scheduler holds baseline cpu access (SCX_CAP_BASE) on the cpu. Otherwise a scheduler could keep occupying a cpu it has been denied simply by handing out long slices. The cap check reads effective caps, which are coherent only under the task's rq lock, and the kernel decrements the slice under that lock as the task runs, so a running task's slice can be changed only there while a queued task's can be set directly. Make scx_bpf_task_set_slice() apply the slice under the rq lock. Synchronously when the caller already holds it, otherwise by stashing it in the new p->scx.slice_oob, tagged with the scheduler's id so a request that outlived a reassignment is dropped. Whether the caller holds @p's current rq lock is tested with p->scx.runnable_cpu. Revocation is enforced through the same grant. When a cpu's effective caps lose SCX_CAP_BASE, the cap-revoke reenq scan also checks the running task and zeroes its slice to evict it. The scan runs as a balance callback after the pick, so this catches both the task that was running when the revoke landed and a capless task the pick just promoted off the local DSQ. The paths that keep a task on its cpu - holding on to the last runnable task in balance, the ENQ_LAST reinsertion and the slice refill on pick - skip tasks lacking baseline access. A migration-disabled task is exempt, mirroring its capless admission on insert. v4: Test rq ownership with p->scx.runnable_cpu, closing a remote-wakeup TOCTOU. (sashiko AI) v3: Keep a pending out-of-band slice request across refill and preserve. (sashiko AI) v2: Only write slice directly when @p is queued on the held rq. (sashiko AI) Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- include/linux/sched/ext.h | 17 +++- kernel/sched/ext/ext.c | 200 +++++++++++++++++++++++++++++++++++++++++--- kernel/sched/ext/internal.h | 19 ++++- kernel/sched/ext/sub.h | 11 +++ 4 files changed, 231 insertions(+), 16 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index 853f03b63133..803da0f1e509 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -223,10 +223,11 @@ struct sched_ext_entity { /* BPF scheduler modifiable fields */ /* - * Runtime budget in nsecs. This is usually set through - * scx_bpf_dsq_insert() but can also be modified directly by the BPF - * scheduler. Automatically decreased by SCX as the task executes. On - * depletion, a scheduling event is triggered. + * Runtime budget in nsecs - how long the task may hold its cpu. Owned + * by the task's scheduler. Set it when enqueuing via + * scx_bpf_dsq_insert(), or otherwise via scx_bpf_task_set_slice(). + * Automatically decreased as the task executes. On depletion a + * scheduling event is triggered. * * This value is cleared to zero if the task is preempted by * %SCX_KICK_PREEMPT and shouldn't be used to determine how long the @@ -243,6 +244,14 @@ struct sched_ext_entity { */ u64 dsq_vtime; + /* + * Out-of-band slice request from scx_bpf_task_set_slice() when the + * caller does not hold the rq lock, applied under the rq lock at the + * next slice consideration. One atomic64 packs the pending flag, the + * issuing sch's id, and the requested slice. See scx_slice_oob_consts. + */ + atomic64_t slice_oob; + /* * Sub-sched cap rejected reenq context, valid only while * %SCX_TASK_REENQ_CAP is set. @reenq_reason_caps is the SCX_CAP_* bits diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 9ae8d78738b4..39b17626c398 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1165,9 +1165,134 @@ static void touch_core_sched_dispatch(struct rq *rq, struct task_struct *p) #endif } -/* set @p's slice, BPF-triggered writes to p->scx.slice go through here */ +/* + * p->scx.slice_oob packs an out-of-band slice request into one atomic64. A zero + * word means no request. Otherwise the fields are: + * + * 63 SCX_SLICE_OOB_PENDING, set on every request + * 62-43 lower bits of issuing scheduler's id + * 42-0 requested slice duration in nsecs + * + * A duration of SCX_SLICE_OOB_DUR_MASK means SCX_SLICE_INF. A finite dur + * saturates at SCX_SLICE_OOB_DUR_MASK - 1. The id is used to detect and ignore + * a request that outlived a task ownership change. + * + * Only the low 20 bits of sch->id are packed, which is enough to make + * collisions practically impossible. A theoretical collision just lets a stale + * request through once. + */ +enum scx_slice_oob_consts { + SCX_SLICE_OOB_DUR_BITS = 43, + SCX_SLICE_OOB_ID_BITS = 64 - SCX_SLICE_OOB_DUR_BITS - 1, + + SCX_SLICE_OOB_DUR_MASK = (1LLU << SCX_SLICE_OOB_DUR_BITS) - 1, + SCX_SLICE_OOB_ID_SHIFT = SCX_SLICE_OOB_DUR_BITS, + SCX_SLICE_OOB_ID_MASK = (1LLU << SCX_SLICE_OOB_ID_BITS) - 1, + SCX_SLICE_OOB_PENDING = 1LLU << 63, +}; + +/* + * Slice write rules + * + * A task's slice - how long it may hold its cpu - is an occupancy grant owned + * by the task's scheduler. How it may be written depends on whether the task is + * running. + * + * Queued, not running: the slice grants no occupancy yet and nothing consumes + * it, so the owner writes it directly - via scx_bpf_dsq_insert(), the dsq move + * kfuncs, or scx_bpf_task_set_slice(). Serializing its own writers is then the + * scheduler's job, not the kernel's. + * + * Running: the slice must be changed under the task's rq lock, because: + * + * - Raising it extends occupancy, allowed only with %SCX_CAP_BASE on the cpu, + * and that cap check is coherent only under the rq lock. Shortening is always + * allowed. + * + * - The kernel decrements it there as the task runs. The decrement is a + * read-modify-write, so a racing write can be clobbered. + * + * scx_bpf_task_set_slice() writes directly only when @p is queued or running + * on the rq lock it holds. That is the only state where the lock keeps us @p's + * owner: @p can't move to another rq without it. A task that isn't queued here + * can instead be woken onto a different rq without taking this lock, and that + * dispatch sets its slice - so a direct write would race. Those cases stash + * into p->scx.slice_oob to be applied under @p's actual rq lock. A later in-band + * write supersedes a stash, and a stash whose scheduler id no longer matches + * @p's owner is dropped. + */ + +/* clear a pending slice request */ +static void clear_task_slice_oob(struct task_struct *p) +{ + if (unlikely(atomic64_read(&p->scx.slice_oob))) + atomic64_set(&p->scx.slice_oob, 0); +} + +/* set @p's slice, leaving any pending out-of-band request in place */ +static void set_task_slice_keep_oob(struct task_struct *p, u64 slice) +{ + p->scx.slice = slice; +} + +/* set @p's slice, superseding any pending out-of-band request */ static void set_task_slice(struct task_struct *p, u64 slice) { + set_task_slice_keep_oob(p, slice); + clear_task_slice_oob(p); +} + +/* request @p's slice to be set to @slice, see the slice write rules above */ +static void set_task_slice_oob(struct scx_sched *sch, struct task_struct *p, u64 slice) +{ + u64 dur; + + if (slice == SCX_SLICE_INF) { + dur = SCX_SLICE_OOB_DUR_MASK; + } else if (unlikely(slice >= SCX_SLICE_OOB_DUR_MASK)) { + dur = SCX_SLICE_OOB_DUR_MASK - 1; + scx_add_event(sch, SCX_EV_SLICE_CLAMPED, 1); + } else { + dur = slice; + } + + atomic64_set(&p->scx.slice_oob, SCX_SLICE_OOB_PENDING | + ((sch->id & SCX_SLICE_OOB_ID_MASK) << SCX_SLICE_OOB_ID_SHIFT) | dur); +} + +/* + * Apply a pending out-of-band slice request under @rq's lock. A request whose + * packed id no longer matches @p's current owner is dropped. An extension needs + * baseline cpu access on @p's cid. %SCX_EV_SLICE_DENIED counts the denials. + * Shortening is always allowed. See the slice write rules above. + */ +static void apply_task_slice_oob(struct rq *rq, struct task_struct *p) +{ + u64 oob, dur, slice; + + lockdep_assert_rq_held(rq); + + if (likely(!atomic64_read(&p->scx.slice_oob))) + return; + + oob = atomic64_xchg(&p->scx.slice_oob, 0); + if (unlikely(!oob)) + return; + + /* the issuing scheduler no longer owns @p, drop the request */ + if (unlikely(((oob >> SCX_SLICE_OOB_ID_SHIFT) & SCX_SLICE_OOB_ID_MASK) != + (scx_task_sched(p)->id & SCX_SLICE_OOB_ID_MASK))) + return; + + dur = oob & SCX_SLICE_OOB_DUR_MASK; + slice = dur == SCX_SLICE_OOB_DUR_MASK ? SCX_SLICE_INF : dur; + + if (slice > p->scx.slice && + unlikely(scx_missing_caps(scx_task_sched(p), cpu_of(rq), SCX_CAP_BASE))) { + __scx_add_event(scx_task_sched(p), SCX_EV_SLICE_DENIED, 1); + return; + } + p->scx.slice = slice; } @@ -1176,6 +1301,9 @@ static void update_curr_scx(struct rq *rq) struct task_struct *curr = rq->curr; s64 delta_exec; + /* apply even on 0 delta_exec, callers may still act on the slice */ + apply_task_slice_oob(rq, curr); + delta_exec = update_curr_common(rq); if (unlikely(delta_exec <= 0)) return; @@ -1256,7 +1384,11 @@ static void dsq_dec_nr(struct scx_dispatch_q *dsq, struct task_struct *p) static void refill_task_slice_dfl(struct scx_sched *sch, struct task_struct *p) { - set_task_slice(p, READ_ONCE(sch->slice_dfl)); + /* + * A default refill is not an explicit request, so it must not drop a + * pending out-of-band one, which is applied when @p next runs. + */ + set_task_slice_keep_oob(p, READ_ONCE(sch->slice_dfl)); __scx_add_event(sch, SCX_EV_REFILL_SLICE_DFL, 1); } @@ -2700,7 +2832,8 @@ static int balance_one(struct rq *rq, struct task_struct *prev) * %SCX_OPS_ENQ_LAST is in effect. */ if ((prev->scx.flags & SCX_TASK_QUEUED) && - (!(sch->ops.flags & SCX_OPS_ENQ_LAST) || scx_bypassing(sch, cpu))) { + (!(sch->ops.flags & SCX_OPS_ENQ_LAST) || scx_bypassing(sch, cpu)) && + scx_task_can_stay_on_cpu(rq, prev)) { rq->scx.flags |= SCX_RQ_BAL_KEEP; __scx_add_event(sch, SCX_EV_DISPATCH_KEEP_LAST, 1); goto has_tasks; @@ -2747,6 +2880,9 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first) clr_task_runnable(p, true); + /* apply any pending out-of-band slice request before the tick decision */ + apply_task_slice_oob(rq, p); + /* * @p is getting newly scheduled or got kicked after someone updated its * slice. Update SCX_RQ_CAN_STOP_TICK to reflect whether the tick can be @@ -2875,12 +3011,14 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, * sched_class, %SCX_OPS_ENQ_LAST must be set. Tell * ops.enqueue() that @p is the only one available for this cpu, * which should trigger an explicit follow-up scheduling event. + * This doesn't apply if the baseline access on the CPU is lost. * * Core scheduling can force this CPU idle while @p stays * runnable. @p's cookie then won't match the core's, so skip * the warning in that case. */ - if (next && sched_class_above(&ext_sched_class, next->sched_class)) { + if (next && sched_class_above(&ext_sched_class, next->sched_class) && + scx_task_can_stay_on_cpu(rq, p)) { WARN_ON_ONCE(sched_cpu_cookie_match(rq, p) && !(sch->ops.flags & SCX_OPS_ENQ_LAST)); scx_do_enqueue_task(rq, p, SCX_ENQ_LAST, -1); @@ -3002,7 +3140,7 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) if (!p) return NULL; - if (unlikely(!p->scx.slice)) { + if (unlikely(!p->scx.slice) && scx_task_can_stay_on_cpu(rq, p)) { struct scx_sched *sch = scx_task_sched(p); if (!scx_bypassing(sch, cpu_of(rq)) && @@ -3932,6 +4070,20 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) nr_enqueued++; } + /* + * The revoke that scheduled this scan may have raced the pick: curr + * may be a now-capless task, either one that kept running or one + * promoted off the local DSQ between the ecaps sync and this scan. + * Zero the slice to evict it. The enqueue gate blocks new capless + * inserts, so no later pick can slip through after the scan. + */ + if ((reenq_flags & SCX_REENQ_CAP_REVOKE) && + rq->curr->sched_class == &ext_sched_class && + scx_task_reenq_on_cap_revoke(rq, rq->curr)) { + set_task_slice(rq->curr, 0); + resched_curr(rq); + } + return nr_enqueued; } @@ -8103,7 +8255,7 @@ __bpf_kfunc bool scx_bpf_dsq_insert___v2(struct task_struct *p, u64 dsq_id, if (slice) set_task_slice(p, slice); else - set_task_slice(p, p->scx.slice ?: 1); + set_task_slice_keep_oob(p, p->scx.slice ?: 1); scx_dsq_insert_commit(sch, p, dsq_id, enq_flags); @@ -8129,7 +8281,7 @@ static bool scx_dsq_insert_vtime(struct scx_sched *sch, struct task_struct *p, if (slice) set_task_slice(p, slice); else - set_task_slice(p, p->scx.slice ?: 1); + set_task_slice_keep_oob(p, p->scx.slice ?: 1); p->scx.dsq_vtime = vtime; @@ -8691,20 +8843,48 @@ __bpf_kfunc_start_defs(); * @slice: time slice to set in nsecs * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * - * Set @p's time slice to @slice. Returns %true on success, %false if the - * calling scheduler doesn't have authority over @p. + * Set @p's time slice. @p must be on the calling scheduler. The value is + * applied whether or not the caller holds @p's rq lock - see the slice write + * rules above for the ownership model. + * + * Raising the slice is honored only while the scheduler holds %SCX_CAP_BASE on + * @p's cpu, otherwise it is counted in %SCX_EV_SLICE_DENIED. Shortening is + * always allowed. On the stashed path the slice is packed into an atomic64_t + * with the scheduler id and a flag bit, so a slice too large to fit is clamped + * and counted in %SCX_EV_SLICE_CLAMPED. %SCX_SLICE_INF is preserved. + * + * Return %true on success, %false if @p is not on the calling scheduler. */ __bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice, const struct bpf_prog_aux *aux) { struct scx_sched *sch; + struct rq *locked_rq; guard(rcu)(); sch = scx_prog_sched(aux); if (unlikely(!sch || !scx_task_on_sched(sch, p))) return false; - set_task_slice(p, slice); + /* + * Directly write only when we hold the lock of the rq @p is queued or + * running on. See the slice write rules above. + */ + locked_rq = scx_locked_rq(); + if (!locked_rq || + (READ_ONCE(p->scx.runnable_cpu) != cpu_of(locked_rq) && + !task_current(locked_rq, p))) { + set_task_slice_oob(sch, p, slice); + return true; + } + + /* under the rq lock: apply now, extensions gated on baseline access */ + if (slice > p->scx.slice && + unlikely(scx_missing_caps(sch, cpu_of(locked_rq), SCX_CAP_BASE))) + __scx_add_event(sch, SCX_EV_SLICE_DENIED, 1); + else + set_task_slice(p, slice); + return true; } diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index ad98e3469b12..ab1dfad28cb5 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -400,8 +400,9 @@ struct sched_ext_ops { * @p: task running currently * * This operation is called every 1/HZ seconds on CPUs which are - * executing an SCX task. Setting @p->scx.slice to 0 will trigger an - * immediate dispatch cycle on the CPU. + * executing an SCX task. Setting a slice of 0 for @p with + * scx_bpf_task_set_slice() will trigger an immediate dispatch cycle on + * the CPU. */ void (*tick)(struct task_struct *p); @@ -1103,6 +1104,18 @@ struct scx_event_stats { */ s64 SCX_EV_REFILL_SLICE_DFL; + /* + * The number of times an out-of-band slice request exceeded the maximum + * representable value and was clamped. + */ + s64 SCX_EV_SLICE_CLAMPED; + + /* + * The number of times a slice extension was denied because the + * scheduler lacked baseline cpu access on the task's cpu. + */ + s64 SCX_EV_SLICE_DENIED; + /* * The total duration of bypass modes in nanoseconds. */ @@ -1153,6 +1166,8 @@ struct scx_event_stats { SCX_EVENT(SCX_EV_REENQ_IMMED); \ SCX_EVENT(SCX_EV_REENQ_LOCAL_REPEAT); \ SCX_EVENT(SCX_EV_REFILL_SLICE_DFL); \ + SCX_EVENT(SCX_EV_SLICE_CLAMPED); \ + SCX_EVENT(SCX_EV_SLICE_DENIED); \ SCX_EVENT(SCX_EV_BYPASS_DURATION); \ SCX_EVENT(SCX_EV_BYPASS_DISPATCH); \ SCX_EVENT(SCX_EV_BYPASS_ACTIVATE); \ diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index 13e9dec56a6a..3b15a10b8c8f 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -130,9 +130,20 @@ static inline u64 scx_caps_implied(u64 cap) return 0; } +/* may @p keep running on @rq's cpu? requires baseline cpu access */ +static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p) +{ + /* a migration-disabled task is let in without caps, keep it likewise */ + if (unlikely(is_migration_disabled(p))) + return true; + + return likely(!scx_missing_caps(scx_task_sched(p), cpu_of(rq), SCX_CAP_BASE)); +} + #else /* CONFIG_EXT_SUB_SCHED */ static inline u64 scx_missing_caps(struct scx_sched *sch, s32 cpu, u64 needed) { return 0; } +static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p) { return true; } #endif /* CONFIG_EXT_SUB_SCHED */ -- cgit v1.2.3 From 52478777b37ba56bb11d40025df0a03ef65c6acb Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 17 Jul 2026 22:12:20 -1000 Subject: cgroup: Add cgroup_task_notifier and task migration events A subsystem can attach to the cgroup hierarchy itself, independent of which controllers are enabled where - BPF hooks already behave this way and sched_ext sub-schedulers do too. Controller callbacks can't track task migrations for them: sched_ext must re-home a task whose migration crosses a sub-scheduler boundary, but the cpu controller's attach callbacks fire only when the task_group changes and miss moves whenever the controller topology is coarser than the sub-scheduler topology. Add cgroup_task_notifier with per-task migration events mirroring the can_attach/attach/cancel_attach phases so that a consumer which prepares per-task state can also veto a migration: CGROUP_TASK_MIGRATING fires pre-commit, CGROUP_TASK_MIGRATED post-commit and CGROUP_TASK_MIGRATE_CANCELED unwinds a failed migration. Only migrations that change a task's dfl cgroup are reported. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- include/linux/cgroup.h | 26 ++++++++++++++ kernel/cgroup/cgroup.c | 93 +++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 110 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h index f2aa46a4f871..aa92db5f05de 100644 --- a/include/linux/cgroup.h +++ b/include/linux/cgroup.h @@ -82,12 +82,38 @@ enum cgroup_lifetime_events { CGROUP_LIFETIME_OFFLINE, }; +/* + * Events on cgroup_task_notifier, data is struct cgroup_task_migrate_ctx. + * MIGRATING fires per task before the migration commits and an error return + * from the chain fails the migration, in which case tasks that were already + * notified receive MIGRATE_CANCELED. MIGRATED fires per task after the + * migration is committed and can't fail. Only migrations that change a task's + * dfl cgroup are reported. + */ +enum cgroup_task_events { + CGROUP_TASK_MIGRATING, + CGROUP_TASK_MIGRATED, + CGROUP_TASK_MIGRATE_CANCELED, +}; + +/* + * @src_dcgrp and @dst_dcgrp are @task's dfl cgroups before and after the + * migration. @src_dcgrp is NULL for CGROUP_TASK_MIGRATED as per-task sources + * are not tracked past the commit point. + */ +struct cgroup_task_migrate_ctx { + struct task_struct *task; + struct cgroup *src_dcgrp; + struct cgroup *dst_dcgrp; +}; + extern struct file_system_type cgroup_fs_type; extern struct cgroup_root cgrp_dfl_root; extern struct css_set init_css_set; extern struct mutex cgroup_mutex; extern spinlock_t css_set_lock; extern struct blocking_notifier_head cgroup_lifetime_notifier; +extern struct blocking_notifier_head cgroup_task_notifier; #define SUBSYS(_x) extern struct cgroup_subsys _x ## _cgrp_subsys; #include diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index 38f8d9df8fbc..2f6b634c84ca 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -88,6 +88,8 @@ EXPORT_SYMBOL_GPL(css_set_lock); struct blocking_notifier_head cgroup_lifetime_notifier = BLOCKING_NOTIFIER_INIT(cgroup_lifetime_notifier); +struct blocking_notifier_head cgroup_task_notifier = + BLOCKING_NOTIFIER_INIT(cgroup_task_notifier); DEFINE_SPINLOCK(trace_cgroup_path_lock); char trace_cgroup_path[TRACE_CGROUP_PATH_LEN]; @@ -2676,14 +2678,27 @@ struct task_struct *cgroup_taskset_next(struct cgroup_taskset *tset, return NULL; } +static void cgroup_migrate_notify_canceled(struct css_set *src_cset, + struct task_struct *task) +{ + struct cgroup_task_migrate_ctx ctx = { + .task = task, + .src_dcgrp = src_cset->dfl_cgrp, + .dst_dcgrp = src_cset->mg_dst_cset->dfl_cgrp, + }; + + blocking_notifier_call_chain(&cgroup_task_notifier, + CGROUP_TASK_MIGRATE_CANCELED, &ctx); +} + /** * cgroup_migrate_execute - migrate a taskset * @mgctx: migration context * - * Migrate tasks in @mgctx as setup by migration preparation functions. - * This function fails iff one of the ->can_attach callbacks fails and - * guarantees that either all or none of the tasks in @mgctx are migrated. - * @mgctx is consumed regardless of success. + * Migrate tasks in @mgctx as setup by migration preparation functions. This + * function fails iff one of the ->can_attach callbacks or CGROUP_TASK_MIGRATING + * notifications fails and guarantees that either all or none of the tasks in + * @mgctx are migrated. @mgctx is consumed regardless of success. */ static int cgroup_migrate_execute(struct cgroup_mgctx *mgctx) { @@ -2691,6 +2706,7 @@ static int cgroup_migrate_execute(struct cgroup_mgctx *mgctx) struct cgroup_subsys *ss; struct task_struct *task, *tmp_task; struct css_set *cset, *tmp_cset; + bool dfl_migration = false; int ssid, failed_ssid, ret; /* check that we can legitimately attach to the cgroup */ @@ -2707,6 +2723,33 @@ static int cgroup_migrate_execute(struct cgroup_mgctx *mgctx) } while_each_subsys_mask(); } + /* + * Notify each task about the impending migration. An error return fails + * the migration. Only migrations on the default hierarchy are reported: + * a migration modifies either every moved task's dfl cgroup or, on + * cgroup1 or for subtree_control writes, none. + */ + list_for_each_entry(cset, &tset->src_csets, mg_node) { + if (cset->dfl_cgrp == cset->mg_dst_cset->dfl_cgrp) + continue; + dfl_migration = true; + list_for_each_entry(task, &cset->mg_tasks, cg_list) { + struct cgroup_task_migrate_ctx ctx = { + .task = task, + .src_dcgrp = cset->dfl_cgrp, + .dst_dcgrp = cset->mg_dst_cset->dfl_cgrp, + }; + + ret = blocking_notifier_call_chain_robust(&cgroup_task_notifier, + CGROUP_TASK_MIGRATING, + CGROUP_TASK_MIGRATE_CANCELED, + &ctx); + ret = notifier_to_errno(ret); + if (ret) + goto out_cancel_migrating; + } + } + /* * Now that we're guaranteed success, proceed to move all tasks to * the new cgroup. There are no failure cases after here, so this @@ -2750,9 +2793,41 @@ static int cgroup_migrate_execute(struct cgroup_mgctx *mgctx) } while_each_subsys_mask(); } + /* + * Notify each task after successful migration. The operation can no + * longer fail and the return value is ignored. The MIGRATING loop + * above explains why only dfl migrations are reported. Per-task + * sources are not tracked past the commit point, so src_dcgrp is + * NULL. + */ + if (dfl_migration) { + list_for_each_entry(cset, &tset->dst_csets, mg_node) { + list_for_each_entry(task, &cset->mg_tasks, cg_list) { + struct cgroup_task_migrate_ctx ctx = { + .task = task, + .dst_dcgrp = cset->dfl_cgrp, + }; + + blocking_notifier_call_chain( + &cgroup_task_notifier, + CGROUP_TASK_MIGRATED, &ctx); + } + } + } + ret = 0; goto out_release_tset; +out_cancel_migrating: + list_for_each_entry_continue_reverse(task, &cset->mg_tasks, cg_list) + cgroup_migrate_notify_canceled(cset, task); + list_for_each_entry_continue_reverse(cset, &tset->src_csets, mg_node) { + if (cset->dfl_cgrp == cset->mg_dst_cset->dfl_cgrp) + continue; + list_for_each_entry_reverse(task, &cset->mg_tasks, cg_list) + cgroup_migrate_notify_canceled(cset, task); + } + failed_ssid = CGROUP_SUBSYS_COUNT; out_cancel_attach: if (tset->nr_tasks) { do_each_subsys_mask(ss, ssid, mgctx->ss_mask) { @@ -2976,11 +3051,11 @@ int cgroup_migrate_prepare_dst(struct cgroup_mgctx *mgctx) * cgroup_migrate_prepare_dst() on the targets before invoking this * function and following up with cgroup_migrate_finish(). * - * As long as a controller's ->can_attach() doesn't fail, this function is - * guaranteed to succeed. This means that, excluding ->can_attach() - * failure, when migrating multiple targets, the success or failure can be - * decided for all targets by invoking group_migrate_prepare_dst() before - * actually starting migrating. + * As long as a controller's ->can_attach() or a CGROUP_TASK_MIGRATING + * notification doesn't fail, this function is guaranteed to succeed. This + * means that, excluding those failures, when migrating multiple targets, + * the success or failure can be decided for all targets by invoking + * group_migrate_prepare_dst() before actually starting migrating. */ int cgroup_migrate(struct task_struct *leader, bool threadgroup, struct cgroup_mgctx *mgctx) -- cgit v1.2.3 From 46932bc5fd7ea1156a6742ad1b9306383e0cfb6f Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 17 Jul 2026 22:12:20 -1000 Subject: sched_ext: Deliver cgroup ops to each task_group's sched With sub-schedulers claiming cgroup subtrees, cgroup ops must be delivered to each task_group's sched rather than always to root. Add tg->scx.sched to track which sched initialized the task_group. It is set and cleared together with SCX_TG_INITED. Deliver the ops accordingly: - ops.cgroup_exit() goes to the sched whose ops.cgroup_init() it pairs with. - ops.cgroup_prep_move/move/cancel_move() go to the task's sched, and only for moves that don't re-home the task. A re-homing move is reported through the ops.exit_task/init_task() pair instead. The cgroups passed to the move ops can be outside the sched's inited set as the cpu controller can be coarser than the sub-scheduler topology. - Knobs of a cgroup belong to the parent, so ops.set_weight/idle/bandwidth() go to the parent task_group's sched. All task_groups currently resolve to the root sched, so no behavior changes until sub-schedulers start claiming cgroups. While at it, scx_cgroup_init() is restructured so both paths share the recording. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- include/linux/sched/ext.h | 2 + kernel/sched/ext/ext.c | 157 ++++++++++++++++++++++++++++++++------------ kernel/sched/ext/internal.h | 34 ++++++++-- 3 files changed, 146 insertions(+), 47 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index cce42b21f5f5..a6db5d300f30 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -298,6 +298,8 @@ static inline bool scx_rcu_cpu_stall(const struct cpumask *stalled_mask) { retur struct scx_task_group { #ifdef CONFIG_EXT_GROUP_SCHED + struct scx_sched *sched; + u32 flags; /* SCX_TG_* */ u32 weight; u64 bw_period_us; diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 6bac68758704..b542900da5a3 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -4339,6 +4339,51 @@ void scx_tg_init(struct task_group *tg) tg->scx.idle = false; } +/** + * scx_tg_sched - Resolve a task_group's sched + * @tg: task_group of interest + * + * Return the sched that @tg's ops.cgroup_init() succeeded on, %NULL if @tg + * isn't inited. An autogroup tg has no cgroup of its own and resolves to the + * root sched. + * + * Safe for callers read-locking the ops rwsem. tg->scx.sched rewrites + * write-lock it, and tg on/offline can't overlap such callers as a css's files + * are created after online and drained before offline. + */ +static struct scx_sched *scx_tg_sched(struct task_group *tg) +{ + lockdep_assert(lockdep_is_held(&cgroup_mutex) || + lockdep_is_held(&scx_cgroup_ops_rwsem)); + + if (!tg->css.cgroup) + tg = &root_task_group; + return tg->scx.sched; +} + +/** + * scx_tg_knob_sched - Resolve the sched receiving a task_group's knob updates + * @tg: task_group of interest + * + * Knobs of a cgroup belong to the parent. Deliver the set_* ops to the + * parent task_group's sched, which equals @tg's own sched everywhere except + * at a sub-scheduler attach point, where the sub's parent sched receives + * them. + * + * The callers sit in @tg's cgroup file writes holding the ops rwsem read + * side. That extends scx_tg_sched()'s file-write argument to the parent's + * sched read: a parent css outlives its children's files. + */ +static struct scx_sched *scx_tg_knob_sched(struct task_group *tg) +{ + lockdep_assert(lockdep_is_held(&cgroup_mutex) || + lockdep_is_held(&scx_cgroup_ops_rwsem)); + + if (!tg->css.cgroup || !tg->css.parent) + return scx_tg_sched(&root_task_group); + return scx_tg_sched(css_tg(tg->css.parent)); +} + int scx_tg_online(struct task_group *tg) { struct scx_sched *sch = scx_root; @@ -4359,8 +4404,10 @@ int scx_tg_online(struct task_group *tg) if (ret) ret = scx_ops_sanitize_err(sch, "cgroup_init", ret); } - if (ret == 0) + if (ret == 0) { + tg->scx.sched = sch; tg->scx.flags |= SCX_TG_ONLINE | SCX_TG_INITED; + } } else { tg->scx.flags |= SCX_TG_ONLINE; } @@ -4370,19 +4417,30 @@ int scx_tg_online(struct task_group *tg) void scx_tg_offline(struct task_group *tg) { - struct scx_sched *sch = scx_root; + struct scx_sched *sch = tg->scx.sched; WARN_ON_ONCE(!(tg->scx.flags & SCX_TG_ONLINE)); - if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_exit) && - (tg->scx.flags & SCX_TG_INITED)) + /* INITED implies non-NULL @sch, test before SCX_HAS_OP() derefs */ + if (scx_cgroup_enabled && (tg->scx.flags & SCX_TG_INITED) && + SCX_HAS_OP(sch, cgroup_exit)) SCX_CALL_OP(sch, cgroup_exit, NULL, tg->css.cgroup); + tg->scx.sched = NULL; tg->scx.flags &= ~(SCX_TG_ONLINE | SCX_TG_INITED); } +/* + * @p's sched for the cgroup migration paths. Stable as re-homes happen either + * at CGROUP_TASK_MIGRATED of the same migration or under scx_cgroup_lock(), + * both while holding cgroup_mutex. + */ +static struct scx_sched *scx_cgroup_task_sched(struct task_struct *p) +{ + return rcu_dereference_protected(p->scx.sched, lockdep_is_held(&cgroup_mutex)); +} + int scx_cgroup_can_attach(struct cgroup_taskset *tset) { - struct scx_sched *sch = scx_root; struct cgroup_subsys_state *css; struct task_struct *p; int ret; @@ -4391,6 +4449,7 @@ int scx_cgroup_can_attach(struct cgroup_taskset *tset) return 0; cgroup_taskset_for_each(p, css, tset) { + struct scx_sched *sch = scx_cgroup_task_sched(p); struct cgroup *from = tg_cgrp(task_group(p)); struct cgroup *to = tg_cgrp(css_tg(css)); @@ -4404,11 +4463,22 @@ int scx_cgroup_can_attach(struct cgroup_taskset *tset) if (from == to) continue; + /* + * The cgroup_move ops are delivered to @p's sched, and only for + * moves that don't re-home @p. A re-homing move changes the dfl + * cgroup's sched and is reported through the + * exit_task/init_task pair that the re-homing generates. + */ + if (!sch || sch != task_css_set(p)->mg_dst_cset->dfl_cgrp->scx_sched) + continue; + if (SCX_HAS_OP(sch, cgroup_prep_move)) { ret = SCX_CALL_OP_RET(sch, cgroup_prep_move, NULL, p, from, css->cgroup); - if (ret) + if (ret) { + ret = scx_ops_sanitize_err(sch, "cgroup_prep_move", ret); goto err; + } } p->scx.cgrp_moving_from = from; @@ -4418,41 +4488,41 @@ int scx_cgroup_can_attach(struct cgroup_taskset *tset) err: cgroup_taskset_for_each(p, css, tset) { - if (SCX_HAS_OP(sch, cgroup_cancel_move) && - p->scx.cgrp_moving_from) + struct scx_sched *sch = scx_cgroup_task_sched(p); + + /* cgrp_moving_from implies non-NULL @sch, test it first */ + if (p->scx.cgrp_moving_from && SCX_HAS_OP(sch, cgroup_cancel_move)) SCX_CALL_OP(sch, cgroup_cancel_move, NULL, p, p->scx.cgrp_moving_from, css->cgroup); p->scx.cgrp_moving_from = NULL; } - return scx_ops_sanitize_err(sch, "cgroup_prep_move", ret); + return ret; } void scx_cgroup_move_task(struct task_struct *p) { - struct scx_sched *sch = scx_root; + struct scx_sched *sch; if (!scx_cgroup_enabled) return; /* - * scx_cgroup_can_attach() sets cgrp_moving_from only when the task's - * cgroup changes. Migration keys off css rather than cgroup identity, - * so it can hand an unchanged-cgroup task here with cgrp_moving_from - * NULL. Nothing to report to the BPF scheduler then, so skip it and - * keep prep_move and move paired. Cgroup ops run on the root sched, - * dispatch on the explicit @sch. + * Migration keys off css rather than cgroup identity, so it can hand an + * unchanged-cgroup task here with cgrp_moving_from NULL. Nothing to + * report to the BPF scheduler then, so skip it and keep prep_move and + * move paired. */ - if (SCX_HAS_OP(sch, cgroup_move) && p->scx.cgrp_moving_from) - __SCX_CALL_OP_TASK(sch, ops, cgroup_move, task_rq(p), - p, p->scx.cgrp_moving_from, - tg_cgrp(task_group(p))); + sch = scx_cgroup_task_sched(p); + if (p->scx.cgrp_moving_from && SCX_HAS_OP(sch, cgroup_move)) + SCX_CALL_OP_TASK(sch, cgroup_move, task_rq(p), + p, p->scx.cgrp_moving_from, + tg_cgrp(task_group(p))); p->scx.cgrp_moving_from = NULL; } void scx_cgroup_cancel_attach(struct cgroup_taskset *tset) { - struct scx_sched *sch = scx_root; struct cgroup_subsys_state *css; struct task_struct *p; @@ -4460,8 +4530,10 @@ void scx_cgroup_cancel_attach(struct cgroup_taskset *tset) return; cgroup_taskset_for_each(p, css, tset) { - if (SCX_HAS_OP(sch, cgroup_cancel_move) && - p->scx.cgrp_moving_from) + struct scx_sched *sch = scx_cgroup_task_sched(p); + + /* cgrp_moving_from implies non-NULL @sch, test it first */ + if (p->scx.cgrp_moving_from && SCX_HAS_OP(sch, cgroup_cancel_move)) SCX_CALL_OP(sch, cgroup_cancel_move, NULL, p, p->scx.cgrp_moving_from, css->cgroup); p->scx.cgrp_moving_from = NULL; @@ -4473,7 +4545,7 @@ void scx_group_set_weight(struct task_group *tg, unsigned long weight) struct scx_sched *sch; percpu_down_read(&scx_cgroup_ops_rwsem); - sch = scx_root; + sch = scx_tg_knob_sched(tg); if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_weight) && tg->scx.weight != weight) @@ -4489,7 +4561,7 @@ void scx_group_set_idle(struct task_group *tg, bool idle) struct scx_sched *sch; percpu_down_read(&scx_cgroup_ops_rwsem); - sch = scx_root; + sch = scx_tg_knob_sched(tg); if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_idle)) SCX_CALL_OP(sch, cgroup_set_idle, NULL, tg_cgrp(tg), idle); @@ -4506,7 +4578,7 @@ void scx_group_set_bandwidth(struct task_group *tg, struct scx_sched *sch; percpu_down_read(&scx_cgroup_ops_rwsem); - sch = scx_root; + sch = scx_tg_knob_sched(tg); if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_bandwidth) && (tg->scx.bw_period_us != period_us || @@ -4718,6 +4790,7 @@ static void scx_cgroup_exit(struct scx_sched *sch) if (!(tg->scx.flags & SCX_TG_INITED)) continue; + tg->scx.sched = NULL; tg->scx.flags &= ~SCX_TG_INITED; if (!sch->ops.cgroup_exit) @@ -4738,28 +4811,26 @@ static int scx_cgroup_init(struct scx_sched *sch) */ css_for_each_descendant_pre(css, &root_task_group.css) { struct task_group *tg = css_tg(css); - struct scx_cgroup_init_args args = { - .weight = tg->scx.weight, - .bw_period_us = tg->scx.bw_period_us, - .bw_quota_us = tg->scx.bw_quota_us, - .bw_burst_us = tg->scx.bw_burst_us, - }; - if ((tg->scx.flags & - (SCX_TG_ONLINE | SCX_TG_INITED)) != SCX_TG_ONLINE) + if ((tg->scx.flags & (SCX_TG_ONLINE | SCX_TG_INITED)) != SCX_TG_ONLINE) continue; - if (!sch->ops.cgroup_init) { - tg->scx.flags |= SCX_TG_INITED; - continue; - } + if (sch->ops.cgroup_init) { + struct scx_cgroup_init_args args = { + .weight = tg->scx.weight, + .bw_period_us = tg->scx.bw_period_us, + .bw_quota_us = tg->scx.bw_quota_us, + .bw_burst_us = tg->scx.bw_burst_us, + }; - ret = SCX_CALL_OP_RET(sch, cgroup_init, NULL, - css->cgroup, &args); - if (ret) { - scx_error(sch, "ops.cgroup_init() failed (%d)", ret); - return ret; + ret = SCX_CALL_OP_RET(sch, cgroup_init, NULL, css->cgroup, &args); + if (ret) { + scx_error(sch, "ops.cgroup_init() failed (%d)", ret); + return ret; + } } + + tg->scx.sched = sch; tg->scx.flags |= SCX_TG_INITED; } diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 23fc95502ea6..ad90d4645a33 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -645,8 +645,14 @@ struct sched_ext_ops { * @cgrp: cgroup being initialized * @args: init arguments, see the struct definition * - * Either the BPF scheduler is being loaded or @cgrp created, initialize - * @cgrp for sched_ext. This operation may block. + * Initialize @cgrp for sched_ext, delivered to @cgrp's sched either + * when the BPF scheduler is being loaded or when @cgrp is created. This + * operation may block. + * + * When the BPF scheduler is being loaded or cgroups are being handed + * over, @cgrp may already have been removed by userspace: a removed + * cgroup stays schedulable until its dying tasks finish their final + * context switches. * * Return 0 for success, -errno for failure. An error return while * loading will abort loading of the BPF scheduler. During cgroup @@ -659,8 +665,13 @@ struct sched_ext_ops { * @cgroup_exit: Exit a cgroup * @cgrp: cgroup being exited * - * Either the BPF scheduler is being unloaded or @cgrp destroyed, exit - * @cgrp for sched_ext. This operation my block. + * Exit @cgrp for sched_ext, delivered to the sched whose + * ops.cgroup_init() it pairs with, either when the BPF scheduler is + * being unloaded or when @cgrp is destroyed. This operation may block. + * + * For a destroyed @cgrp, delivery follows the last scheduling event on + * it: a removed cgroup stays schedulable until its dying tasks finish + * their final context switches. */ void (*cgroup_exit)(struct cgroup *cgrp); @@ -673,6 +684,12 @@ struct sched_ext_ops { * Prepare @p for move from cgroup @from to @to. This operation may * block and can be used for allocations. * + * The cgroup_move ops are delivered to @p's sched, and only for moves + * that don't re-home @p. A re-homing move is reported through + * ops.exit_task() and ops.init_task() instead. @from and @to can + * reference cgroups the sched never received ops.cgroup_init() for, as + * the cpu controller can be coarser than the sub-scheduler topology. + * * Return 0 for success, -errno for failure. An error return aborts the * migration. */ @@ -708,6 +725,11 @@ struct sched_ext_ops { * @weight: new weight [1..10000] * * Update @cgrp's weight to @weight. + * + * Knobs of a cgroup belong to the parent, so the set_* ops are + * delivered to @cgrp's parent's sched. That sched may never have seen + * ops.cgroup_init() for @cgrp - at a sub-scheduler attach point, the + * parent sched tracks @cgrp through ops.sub_attach() instead. */ void (*cgroup_set_weight)(struct cgroup *cgrp, u32 weight); @@ -728,6 +750,8 @@ struct sched_ext_ops { * burst temporarily. The specific control mechanism and thus the * interpretation of @period_us and burstiness is up to the BPF * scheduler. + * + * Delivery follows the same rule as cgroup_set_weight(). */ void (*cgroup_set_bandwidth)(struct cgroup *cgrp, u64 period_us, u64 quota_us, u64 burst_us); @@ -740,6 +764,8 @@ struct sched_ext_ops { * Update @cgrp's idle state to @idle. This callback is invoked when * a cgroup transitions between idle and non-idle states, allowing the * BPF scheduler to adjust its behavior accordingly. + * + * Delivery follows the same rule as cgroup_set_weight(). */ void (*cgroup_set_idle)(struct cgroup *cgrp, bool idle); -- cgit v1.2.3 From a6ec0b62c589c5b5518e0e3d9eaac6398a82b6f4 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 17 Jul 2026 22:12:20 -1000 Subject: sched_ext: Hand over cgroups at sub-scheduler enable/disable Sub-schedulers don't get cgroups yet: every task_group is inited on the root sched and the routing added by the previous patches always resolves to it. Add the handover: an enabling sub-scheduler takes over the cgroups in its subtree and a disabling one returns them to its parent. scx_cgroup_claim_subtree() runs while the sub enables, after the subtree's cgrp->scx_sched's are set and before any task is claimed. It inits each subtree task_group on the sub, exits it from the parent and updates tg->scx.sched. A failed ops.cgroup_init() unwinds the sub-side inits and aborts the enable with the parent untouched. Disabling reverses it with scx_cgroup_return_subtree(): exit each cgroup from the sub, then re-init it on the parent with the current tg->scx.* values, resyncing weight and bandwidth changes made while the sub had it. When a re-init fails, the parent is failed and the remaining task_groups still transfer uninited and get no cgroup ops - the same punting done for tasks. The dying parent's own disable moves them onward. The handover walks include dying but not yet offlined task_groups, the same as root's bulk walks: a removed cgroup keeps hosting scheduling events until its dying tasks finish their final context switches, and its ops.cgroup_exit() must follow the last of them. tg on/offlining is excluded through cgroup_lock(), so either ordering against an rmdir of a subtree cgroup delivers balanced init/exit pairs. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- include/linux/sched/ext.h | 13 +++ kernel/sched/ext/ext.c | 45 +++++++--- kernel/sched/ext/internal.h | 6 ++ kernel/sched/ext/sub.c | 199 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 250 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index a6db5d300f30..78b2f289cb98 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -298,6 +298,19 @@ static inline bool scx_rcu_cpu_stall(const struct cpumask *stalled_mask) { retur struct scx_task_group { #ifdef CONFIG_EXT_GROUP_SCHED + /* + * The sched this tg is on, NULL if none. SCX_TG_INITED tracks whether + * ops.cgroup_init() succeeded on it. When a child sched exits and its + * tgs move to the parent, a failed init leaves the tg on the parent + * with INITED clear (see scx_cgroup_return_subtree()). + * + * This is tracked separately from cgrp->scx_sched because the tg + * hierarchy can diverge from the cgroup2 hierarchy in both lifetime and + * shape. A tg stays online past its cgroup's removal while the + * cgrp->scx_sched rewrites visit only live cgroups, leaving a removed + * cgroup's pointer stale. The cpu controller can also be mounted on + * cgroup1. + */ struct scx_sched *sched; u32 flags; /* SCX_TG_* */ diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index b542900da5a3..58cd971e5fc5 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -4347,6 +4347,11 @@ void scx_tg_init(struct task_group *tg) * isn't inited. An autogroup tg has no cgroup of its own and resolves to the * root sched. * + * When a child sched exits, its task_groups are moved to the parent and + * re-inited on it. A failed re-init fails the parent in turn and leaves the + * task_group without a sched it's inited on, resolving to %NULL. See + * scx_cgroup_return_subtree(). + * * Safe for callers read-locking the ops rwsem. tg->scx.sched rewrites * write-lock it, and tg on/offline can't overlap such callers as a css's files * are created after online and drained before offline. @@ -4358,7 +4363,8 @@ static struct scx_sched *scx_tg_sched(struct task_group *tg) if (!tg->css.cgroup) tg = &root_task_group; - return tg->scx.sched; + /* INITED means ops.cgroup_init() succeeded on @tg->scx.sched */ + return (tg->scx.flags & SCX_TG_INITED) ? tg->scx.sched : NULL; } /** @@ -4370,6 +4376,9 @@ static struct scx_sched *scx_tg_sched(struct task_group *tg) * at a sub-scheduler attach point, where the sub's parent sched receives * them. * + * Return %NULL if the parent task_group has no sched. That can happen when the + * parent's ops.cgroup_init() fails while a sub-scheduler is being disabled. + * * The callers sit in @tg's cgroup file writes holding the ops rwsem read * side. That extends scx_tg_sched()'s file-write argument to the parent's * sched read: a parent css outlives its children's files. @@ -4386,12 +4395,24 @@ static struct scx_sched *scx_tg_knob_sched(struct task_group *tg) int scx_tg_online(struct task_group *tg) { - struct scx_sched *sch = scx_root; int ret = 0; WARN_ON_ONCE(tg->scx.flags & (SCX_TG_ONLINE | SCX_TG_INITED)); if (scx_cgroup_enabled) { + struct scx_sched *sch; + + /* + * The cgroup lifetime notifier populates cgrp->scx_sched before + * css_online, but only on the default hierarchy. Sub-scheds are + * attached to the cgroup2 hierarchy, so a cgroup1 task_group + * always belongs to the root sched. + */ + if (cgroup_on_dfl(tg->css.cgroup)) + sch = tg->css.cgroup->scx_sched; + else + sch = scx_tg_sched(&root_task_group); + if (SCX_HAS_OP(sch, cgroup_init)) { struct scx_cgroup_init_args args = { .weight = tg->scx.weight, @@ -4547,7 +4568,7 @@ void scx_group_set_weight(struct task_group *tg, unsigned long weight) percpu_down_read(&scx_cgroup_ops_rwsem); sch = scx_tg_knob_sched(tg); - if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_weight) && + if (scx_cgroup_enabled && sch && SCX_HAS_OP(sch, cgroup_set_weight) && tg->scx.weight != weight) SCX_CALL_OP(sch, cgroup_set_weight, NULL, tg_cgrp(tg), weight); @@ -4563,7 +4584,7 @@ void scx_group_set_idle(struct task_group *tg, bool idle) percpu_down_read(&scx_cgroup_ops_rwsem); sch = scx_tg_knob_sched(tg); - if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_idle)) + if (scx_cgroup_enabled && sch && SCX_HAS_OP(sch, cgroup_set_idle)) SCX_CALL_OP(sch, cgroup_set_idle, NULL, tg_cgrp(tg), idle); /* Update the task group's idle state */ @@ -4580,7 +4601,7 @@ void scx_group_set_bandwidth(struct task_group *tg, percpu_down_read(&scx_cgroup_ops_rwsem); sch = scx_tg_knob_sched(tg); - if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_bandwidth) && + if (scx_cgroup_enabled && sch && SCX_HAS_OP(sch, cgroup_set_bandwidth) && (tg->scx.bw_period_us != period_us || tg->scx.bw_quota_us != quota_us || tg->scx.bw_burst_us != burst_us)) @@ -4788,15 +4809,13 @@ static void scx_cgroup_exit(struct scx_sched *sch) css_for_each_descendant_post(css, &root_task_group.css) { struct task_group *tg = css_tg(css); - if (!(tg->scx.flags & SCX_TG_INITED)) - continue; + /* also clear the sched of tgs whose ops.cgroup_init() failed */ tg->scx.sched = NULL; - tg->scx.flags &= ~SCX_TG_INITED; - - if (!sch->ops.cgroup_exit) - continue; - - SCX_CALL_OP(sch, cgroup_exit, NULL, css->cgroup); + if (tg->scx.flags & SCX_TG_INITED) { + tg->scx.flags &= ~SCX_TG_INITED; + if (sch->ops.cgroup_exit) + SCX_CALL_OP(sch, cgroup_exit, NULL, css->cgroup); + } } } diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index ad90d4645a33..0c0a8fdaa2c4 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -649,6 +649,11 @@ struct sched_ext_ops { * when the BPF scheduler is being loaded or when @cgrp is created. This * operation may block. * + * Cgroup handovers also generate these ops: an enabling sub-scheduler + * receives ops.cgroup_init() for every cgroup in its subtree while the + * previous sched receives ops.cgroup_exit(), and disabling reverses the + * two. + * * When the BPF scheduler is being loaded or cgroups are being handed * over, @cgrp may already have been removed by userspace: a removed * cgroup stays schedulable until its dying tasks finish their final @@ -1722,6 +1727,7 @@ enum scx_kick_flags { enum scx_tg_flags { SCX_TG_ONLINE = 1U << 0, SCX_TG_INITED = 1U << 1, + SCX_TG_SUB_INIT = 1U << 2, /* see scx_cgroup_claim_subtree() */ }; enum scx_enable_state { diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 393dbd00d2f7..8d8737149bc0 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -836,6 +836,191 @@ static void scx_fail_parent(struct scx_sched *sch, scx_task_iter_stop(&sti); } +#ifdef CONFIG_EXT_GROUP_SCHED +/** + * scx_cgroup_claim_subtree - Claim the subtree's cgroups for an enabling sub + * @sch: sub-scheduler being enabled + * + * Called while enabling @sch, after the subtree's cgrp->scx_sched's are pointed + * at @sch and before any task is claimed. This mirrors root enable's + * cgroups-before-tasks order. The ops.init_task() args are task_group-granular + * and can still reference a cgroup outside the handed-over set when the cpu + * controller is coarser than the sub topology or mounted on cgroup1. + * + * First init each of the parent sched's subtree cgroups on @sch, and only then + * exit them from the parent, so that a failed init can be unwound with the + * parent untouched. The both-inited transient is invisible outside + * scx_cgroup_lock(). %SCX_TG_SUB_INIT tracks the first pass's progress. + * %SCX_TG_INITED stays set throughout, except for a task_group whose + * ops.cgroup_init() failed on the parent (see scx_cgroup_return_subtree()): + * there is nothing to exit from the parent and %SCX_TG_INITED is set back with + * the transfer. + * + * Dying but not yet offlined task_groups are included: a removed cgroup keeps + * hosting scheduling events until its dying tasks finish their final context + * switches, so it still needs to be inited on a sched, and its offline-time + * ops.cgroup_exit() follows the last of those events. + * + * Return 0 on success, -errno on failure. On failure, @sch has been + * scx_error()'d and is left with no cgroups. + */ +static s32 scx_cgroup_claim_subtree(struct scx_sched *sch) +{ + struct cgroup *sub_cgrp = sch_cgroup(sch); + struct cgroup_subsys_state *ecss = cgroup_e_css(sub_cgrp, &cpu_cgrp_subsys); + struct scx_sched *parent = scx_parent(sch); + struct cgroup_subsys_state *css; + int ret; + + css_for_each_descendant_pre(css, ecss) { + struct task_group *tg = css_tg(css); + struct scx_cgroup_init_args args = { + .weight = tg->scx.weight, + .bw_period_us = tg->scx.bw_period_us, + .bw_quota_us = tg->scx.bw_quota_us, + .bw_burst_us = tg->scx.bw_burst_us, + }; + + if (tg->scx.sched != parent || + !cgroup_is_descendant(css->cgroup, sub_cgrp)) + continue; + + if (SCX_HAS_OP(sch, cgroup_init)) { + ret = SCX_CALL_OP_RET(sch, cgroup_init, NULL, css->cgroup, &args); + if (ret) { + scx_error(sch, "ops.cgroup_init() failed (%d)", ret); + goto err; + } + } + tg->scx.flags |= SCX_TG_SUB_INIT; + } + + css_for_each_descendant_post(css, ecss) { + struct task_group *tg = css_tg(css); + + /* + * SUB_INIT is pass 1's progress mark: pass 2 and the err path + * must visit exactly the tgs pass 1 inited. + */ + if (!(tg->scx.flags & SCX_TG_SUB_INIT)) + continue; + + /* skip the exit if the parent's ops.cgroup_init() failed */ + if ((tg->scx.flags & SCX_TG_INITED) && SCX_HAS_OP(parent, cgroup_exit)) + SCX_CALL_OP(parent, cgroup_exit, NULL, css->cgroup); + tg->scx.sched = sch; + tg->scx.flags |= SCX_TG_INITED; + tg->scx.flags &= ~SCX_TG_SUB_INIT; + } + + return 0; + +err: + css_for_each_descendant_post(css, ecss) { + struct task_group *tg = css_tg(css); + + if (!(tg->scx.flags & SCX_TG_SUB_INIT)) + continue; + + if (SCX_HAS_OP(sch, cgroup_exit)) + SCX_CALL_OP(sch, cgroup_exit, NULL, css->cgroup); + tg->scx.flags &= ~SCX_TG_SUB_INIT; + } + return ret; +} + +/** + * scx_cgroup_return_subtree - Return the subtree's cgroups to the parent sched + * @sch: sub-scheduler being disabled + * + * Called while disabling @sch, after the subtree's cgrp->scx_sched's are reset + * to the parent sched and before tasks are re-homed, mirroring root disable's + * cgroups-before-tasks teardown order. The reverse of + * scx_cgroup_claim_subtree(): exit @sch's cgroups from @sch, then init them on + * the parent with the current tg->scx.* values, resyncing settings that changed + * while @sch had them. + * + * When an init on the parent fails, the parent is failed - the same policy as + * task re-homing. The remaining task_groups are punted: they move to the parent + * anyway with %SCX_TG_INITED cleared, as ops.cgroup_init() failed or never ran + * for them. A punted task_group gets no cgroup ops. The dying parent's own + * disable moves it one sched up, initing it there. Root ends the chain: root + * teardown drops cgroup ops entirely and the next enable's bulk init re-inits + * every online task_group. + * + * The task re-home that follows still delivers ops.init_task() to the dying + * parent, including for tasks in punted cgroups it never inited - tolerated + * like the downstream failures of task punting (see scx_punt_task()). + */ +static void scx_cgroup_return_subtree(struct scx_sched *sch) +{ + struct cgroup *sub_cgrp = sch_cgroup(sch); + struct cgroup_subsys_state *ecss = cgroup_e_css(sub_cgrp, &cpu_cgrp_subsys); + struct scx_sched *parent = scx_parent(sch); + struct cgroup_subsys_state *css; + bool parent_failed = false; + int ret; + + css_for_each_descendant_post(css, ecss) { + struct task_group *tg = css_tg(css); + + if (tg->scx.sched != sch || + !cgroup_is_descendant(css->cgroup, sub_cgrp)) + continue; + + /* skip the exit if @sch's ops.cgroup_init() failed for the tg */ + if ((tg->scx.flags & SCX_TG_INITED) && SCX_HAS_OP(sch, cgroup_exit)) + SCX_CALL_OP(sch, cgroup_exit, NULL, css->cgroup); + tg->scx.sched = parent; + tg->scx.flags |= SCX_TG_SUB_INIT; + } + + css_for_each_descendant_pre(css, ecss) { + struct task_group *tg = css_tg(css); + struct scx_cgroup_init_args args = { + .weight = tg->scx.weight, + .bw_period_us = tg->scx.bw_period_us, + .bw_quota_us = tg->scx.bw_quota_us, + .bw_burst_us = tg->scx.bw_burst_us, + }; + + /* the first pass must have transferred everything */ + WARN_ON_ONCE(tg->scx.sched == sch); + + /* + * SUB_INIT distinguishes the tgs pass 1 moved. The sched test + * can't: a tg punted to the parent by an earlier failure would + * also match. + */ + if (!(tg->scx.flags & SCX_TG_SUB_INIT)) + continue; + tg->scx.flags &= ~(SCX_TG_SUB_INIT | SCX_TG_INITED); + + /* + * A re-init on $parent failed. The task_groups from here on are + * punted: they stay on the dying $parent with INITED clear and + * move onward when it disables. + */ + if (parent_failed) + continue; + + if (SCX_HAS_OP(parent, cgroup_init)) { + ret = SCX_CALL_OP_RET(parent, cgroup_init, NULL, css->cgroup, &args); + if (ret) { + scx_error(parent, "ops.cgroup_init() failed (%d) while disabling a sub-scheduler", + ret); + parent_failed = true; + continue; + } + } + tg->scx.flags |= SCX_TG_INITED; + } +} +#else +static inline s32 scx_cgroup_claim_subtree(struct scx_sched *sch) { return 0; } +static inline void scx_cgroup_return_subtree(struct scx_sched *sch) {} +#endif + void scx_sub_disable(struct scx_sched *sch) { struct scx_sched *parent = scx_parent(sch); @@ -871,6 +1056,12 @@ void scx_sub_disable(struct scx_sched *sch) set_cgroup_sched(sch_cgroup(sch), parent); + /* + * Return the subtree's cgroups before re-homing tasks so that any + * ops.init_task() on $parent only sees cgroups it has initialized. + */ + scx_cgroup_return_subtree(sch); + scx_task_iter_start(&sti, sch->cgrp); while ((p = scx_task_iter_next_locked(&sti))) { struct rq *rq; @@ -1172,6 +1363,14 @@ void scx_sub_enable_workfn(struct kthread_work *work) goto err_unlock_and_disable; } + /* + * Take over the subtree's cgroups before any task is claimed, + * mirroring root enable's cgroups-before-tasks order. + */ + ret = scx_cgroup_claim_subtree(sch); + if (ret) + goto err_unlock_and_disable; + /* * Initialize tasks for the new child $sch without exiting them for * $parent so that the tasks can always be reverted back to $parent -- cgit v1.2.3 From 7706d6e4f2e3ad7dfb92b84cacd0c16e6e3c8381 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sun, 26 Jul 2026 11:11:51 -1000 Subject: sched_ext: Bound per-task reenqueues and eject the owning scheduler Unlike local reenqueues, cap rejections have no repeat limit. A malfunctioning scheduler can keep re-inserting a task to a cid it lacks caps on, cycling the task through reject and reenqueue. This was assumed safe because a task that never runs trips the stall watchdog. However, the reenqueue irq_work re-arms itself and outranks the timer vector, blocking everything else on the CPU including stall detection and recovery, until the NMI hardlockup detector fires. Local reenqueues already have a repeat cap, SCX_REENQ_LOCAL_MAX_REPEAT, which needs generalizing to cover all reenqueues. It also has an attribution problem. Counted per-cpu on root, it tears down the whole hierarchy even when a sub-scheduler caused the repeated reenqueues. Generalize by bounding every reenqueue with one per-task counter. reenq_cnt is bumped in scx_do_enqueue_task() on each SCX_ENQ_REENQ, the single path every reenqueue producer passes through, and cleared in clr_task_runnable() when the task is picked to run and in scx_disable_task() when it leaves the scheduler's control. Past SCX_REENQ_MAX_REPEAT the task's owning scheduler is ejected with a new SCX_EXIT_ERROR_REENQ and the task is left stranded to be picked up during sched exit. The SCX_EV_REENQ_LOCAL_REPEAT event becomes SCX_EV_REENQ_REPEAT, counting repeat reenqueues from all sources. v2: Count SCX_EV_REENQ_REPEAT only when a reenqueue leads to another reenqueue, not on every reenqueue. v3: - Also clear reenq_cnt in scx_disable_task() so that the count doesn't carry over to the next owner across sched class switches, scheduler replacement or sub-scheduler rehoming (Andrea Righi). - Update the stale SCX_EV_REENQ_LOCAL_REPEAT references in sched-ext.rst (Andrea Righi). Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- Documentation/scheduler/sched-ext.rst | 8 ++--- include/linux/sched/ext.h | 1 + kernel/sched/ext/ext.c | 58 ++++++++++++++++++++--------------- kernel/sched/ext/internal.h | 19 +++++------- kernel/sched/ext/sub.c | 6 ++-- kernel/sched/ext/types.h | 2 +- kernel/sched/sched.h | 1 - 7 files changed, 51 insertions(+), 44 deletions(-) (limited to 'include/linux') diff --git a/Documentation/scheduler/sched-ext.rst b/Documentation/scheduler/sched-ext.rst index 2771ea4cc14a..ad2fff3c0593 100644 --- a/Documentation/scheduler/sched-ext.rst +++ b/Documentation/scheduler/sched-ext.rst @@ -106,7 +106,7 @@ counters. Each counter occupies one ``name value`` line: SCX_EV_ENQ_SKIP_EXITING 0 SCX_EV_ENQ_SKIP_MIGRATION_DISABLED 0 SCX_EV_REENQ_IMMED 0 - SCX_EV_REENQ_LOCAL_REPEAT 0 + SCX_EV_REENQ_REPEAT 0 SCX_EV_REFILL_SLICE_DFL 456789 SCX_EV_BYPASS_DURATION 0 SCX_EV_BYPASS_DISPATCH 0 @@ -129,9 +129,9 @@ The counters are described in ``kernel/sched/ext/internal.h``; briefly: ``SCX_OPS_ENQ_MIGRATION_DISABLED`` is not set). * ``SCX_EV_REENQ_IMMED``: a task dispatched with ``SCX_ENQ_IMMED`` was re-enqueued because the target CPU was not available for immediate execution. -* ``SCX_EV_REENQ_LOCAL_REPEAT``: a reenqueue of the local DSQ triggered - another reenqueue; recurring counts indicate incorrect ``SCX_ENQ_REENQ`` - handling in the BPF scheduler. +* ``SCX_EV_REENQ_REPEAT``: a reenqueue led to another reenqueue without the + task running in between; recurring counts indicate that the BPF scheduler + keeps re-deciding placements it can't honor. * ``SCX_EV_REFILL_SLICE_DFL``: a task's time slice was refilled with the default value (``SCX_SLICE_DFL``). * ``SCX_EV_BYPASS_DURATION``: total nanoseconds spent in bypass mode. diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index 78b2f289cb98..bd9c4059e8fc 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -198,6 +198,7 @@ struct sched_ext_entity { u32 dsq_flags; /* protected by DSQ lock */ u32 flags; /* protected by rq lock */ u32 weight; + u32 reenq_cnt; /* reenqueues since last run */ s32 sticky_cpu; s32 holding_cpu; s32 selected_cpu; diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 79dc258515e7..ff9151232f58 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1903,6 +1903,24 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, */ p->scx.flags &= ~SCX_TASK_IMMED; + /* + * A task reenqueued too many times without running means the scheduler + * keeps re-deciding a placement it can't honor, e.g. re-inserting to a + * cid it lacks caps on. Eject the owning scheduler and strand the task + * to be picked up during sched exit. + */ + if (enq_flags & SCX_ENQ_REENQ) { + if (++p->scx.reenq_cnt > 1) + __scx_add_event(sch, SCX_EV_REENQ_REPEAT, 1); + + if (unlikely(p->scx.reenq_cnt > SCX_REENQ_MAX_REPEAT)) { + __scx_exit(sch, SCX_EXIT_ERROR_REENQ, 0, cpu_of(rq), + "%s[%d] reenqueued %u times without running", + p->comm, p->pid, p->scx.reenq_cnt); + return; + } + } + /* * If !scx_rq_online(), we already told the BPF scheduler that the CPU * is offline and are just running the hotplug path. Don't bother the @@ -2025,8 +2043,10 @@ static void clr_task_runnable(struct task_struct *p, bool reset_runnable_at) { list_del_init(&p->scx.runnable_node); WRITE_ONCE(p->scx.runnable_cpu, -1); - if (reset_runnable_at) + if (reset_runnable_at) { p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT; + p->scx.reenq_cnt = 0; + } } static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_flags) @@ -3669,6 +3689,7 @@ static void scx_disable_task(struct scx_sched *sch, struct task_struct *p) */ p->scx.dsq_vtime = 0; set_task_slice(p, 0); + p->scx.reenq_cnt = 0; /* * Verify the task is not in BPF scheduler's custody. If flag @@ -4066,8 +4087,8 @@ static void process_ddsp_deferred_locals(struct rq *rq) * Reenqueued tasks go through ops.enqueue() with %SCX_ENQ_REENQ | * %SCX_TASK_REENQ_IMMED. If the BPF scheduler dispatches back to the same local * DSQ with %SCX_ENQ_IMMED while the CPU is still unavailable, this triggers - * another reenq cycle. Repetitions are bounded by %SCX_REENQ_LOCAL_MAX_REPEAT - * in process_deferred_reenq_locals(). + * another reenq cycle. Repetitions are bounded by %SCX_REENQ_MAX_REPEAT in + * scx_do_enqueue_task(), which ejects the task's owning scheduler. */ static bool local_task_should_reenq(struct rq *rq, struct task_struct *p, u64 *reenq_flags, u32 *reason) @@ -4175,14 +4196,16 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) static void process_deferred_reenq_locals(struct rq *rq) { - u64 seq = ++rq->scx.deferred_reenq_locals_seq; - lockdep_assert_rq_held(rq); + /* + * A task can be re-queued within this loop when a reenqueued task + * bounces straight back to the local DSQ. That recursion is bounded by + * the per-task reenqueue cap in scx_do_enqueue_task(). + */ while (true) { struct scx_sched *sch; u64 reenq_flags; - bool skip = false; scoped_guard (raw_spinlock, &rq->scx.deferred_reenq_lock) { struct scx_deferred_reenq_local *drl = @@ -4201,27 +4224,12 @@ static void process_deferred_reenq_locals(struct rq *rq) reenq_flags = drl->flags; WRITE_ONCE(drl->flags, 0); list_del_init(&drl->node); - - if (likely(drl->seq != seq)) { - drl->seq = seq; - drl->cnt = 0; - } else { - if (unlikely(++drl->cnt > SCX_REENQ_LOCAL_MAX_REPEAT)) { - scx_error(sch, "SCX_ENQ_REENQ on SCX_DSQ_LOCAL repeated %u times", - drl->cnt); - skip = true; - } - - __scx_add_event(sch, SCX_EV_REENQ_LOCAL_REPEAT, 1); - } } - if (!skip) { - /* see schedule_dsq_reenq() */ - smp_mb(); + /* see schedule_dsq_reenq() */ + smp_mb(); - reenq_local(sch, rq, reenq_flags); - } + reenq_local(sch, rq, reenq_flags); } } @@ -5941,6 +5949,8 @@ static const char *scx_exit_reason(enum scx_exit_kind kind) return "scx_bpf_error"; case SCX_EXIT_ERROR_STALL: return "runnable task stall"; + case SCX_EXIT_ERROR_REENQ: + return "reenqueue limit"; default: return ""; } diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 39dddcbb3b7d..144e962d3342 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -56,6 +56,7 @@ enum scx_exit_kind { SCX_EXIT_ERROR = 1024, /* runtime error, error msg contains details */ SCX_EXIT_ERROR_BPF, /* ERROR but triggered through scx_bpf_error() */ SCX_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */ + SCX_EXIT_ERROR_REENQ, /* task hit reenqueue limit without running */ }; /* @@ -1119,15 +1120,13 @@ struct scx_event_stats { s64 SCX_EV_REENQ_IMMED; /* - * The number of times a reenq of local DSQ caused another reenq of - * local DSQ. This can happen when %SCX_ENQ_IMMED races against a higher - * priority class task even if the BPF scheduler always satisfies the - * prerequisites for %SCX_ENQ_IMMED at the time of enqueue. However, - * that scenario is very unlikely and this count going up regularly - * indicates that the BPF scheduler is handling %SCX_ENQ_REENQ - * incorrectly causing recursive reenqueues. + * The number of times a reenqueue (%SCX_ENQ_REENQ) led to another + * reenqueue without the task running in between. This count climbing + * rapidly indicates that the BPF scheduler keeps re-deciding placements + * it can't honor. A single task reenqueued more than + * %SCX_REENQ_MAX_REPEAT times gets its owning scheduler ejected. */ - s64 SCX_EV_REENQ_LOCAL_REPEAT; + s64 SCX_EV_REENQ_REPEAT; /* * Total number of times a task's time slice was refilled with the @@ -1221,7 +1220,7 @@ struct scx_event_stats { SCX_EVENT(SCX_EV_ENQ_SKIP_EXITING); \ SCX_EVENT(SCX_EV_ENQ_SKIP_MIGRATION_DISABLED); \ SCX_EVENT(SCX_EV_REENQ_IMMED); \ - SCX_EVENT(SCX_EV_REENQ_LOCAL_REPEAT); \ + SCX_EVENT(SCX_EV_REENQ_REPEAT); \ SCX_EVENT(SCX_EV_REFILL_SLICE_DFL); \ SCX_EVENT(SCX_EV_SLICE_CLAMPED); \ SCX_EVENT(SCX_EV_SLICE_DENIED); \ @@ -1260,8 +1259,6 @@ struct scx_dsp_ctx { struct scx_deferred_reenq_local { struct list_head node; u64 flags; - u64 seq; - u32 cnt; }; struct scx_sched_pcpu { diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 824fe35f00ee..7265f32bd8f5 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -319,9 +319,9 @@ bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p) * Drain @rq->scx.reject_dsq, reenqueueing each task so the BPF re-decides * from p->scx.reenq_reason_*. * - * A task can be re-rejected repeatedly, and there's no repeat limit here. - * Rejection can't happen for root, and sub-scheds can be safely ejected after - * triggering the stall watchdog. + * A task can be re-rejected repeatedly. The reenqueue is bounded per task in + * scx_do_enqueue_task(), which ejects the owning sub past SCX_REENQ_MAX_REPEAT. + * Rejection can't happen for root. */ void scx_reenq_reject(struct rq *rq) { diff --git a/kernel/sched/ext/types.h b/kernel/sched/ext/types.h index a1a05820725e..b94ddee21c57 100644 --- a/kernel/sched/ext/types.h +++ b/kernel/sched/ext/types.h @@ -41,7 +41,7 @@ enum scx_consts { SCX_BYPASS_LB_MIN_DELTA_DIV = 4, SCX_BYPASS_LB_BATCH = 256, - SCX_REENQ_LOCAL_MAX_REPEAT = 256, + SCX_REENQ_MAX_REPEAT = 256, SCX_SUB_MAX_DEPTH = 4, }; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 64d79e9efc3d..c0cb879d75f0 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -823,7 +823,6 @@ struct scx_rq { struct list_head sched_pcpus_to_kick; /* see kick_cpus_irq_workfn() */ raw_spinlock_t deferred_reenq_lock; - u64 deferred_reenq_locals_seq; struct list_head deferred_reenq_locals; /* scheds requesting reenq of local DSQ */ struct list_head deferred_reenq_users; /* user DSQs requesting reenq */ struct balance_callback deferred_bal_cb; -- cgit v1.2.3 From 13f1eae3b66257625f865babd4fb7c251c8c981e Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 3 Aug 2026 11:01:14 -1000 Subject: sched_ext: Synchronize slice and dsq_vtime writes p->scx.slice and p->scx.dsq_vtime writes have no synchronization rules. The dsq insert kfuncs write both fields synchronously from whatever context they're called in - a direct dispatch from ops.select_cpu() writes with only pi_lock held - and, as the kfuncs are safe to call spuriously with the invalid dispatch discarded later, a scheduler can modify any task's slice by spuriously calling them. The latter stands in the way of an upcoming patch which adds kernel-granted slices that the schedulers must not be able to modify. Give both fields explicit rules. While the task is running, sleeping or queued on an rq-owned DSQ, the rq lock protects them - these are the states where the kernel consumes the slice. While queued on a user DSQ or on the BPF side, the kernel neither consumes nor decides on the fields and every writer acts for the BPF scheduler - synchronizing the writers is the scheduler's responsibility and whichever write lands last wins. To conform, an insert kfunc no longer writes the fields when called. The values travel with the dispatch and take effect when the task is inserted. A discarded dispatch has no side effects. The rq lock rule is asserted at the slice store. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- include/linux/sched/ext.h | 2 + kernel/sched/ext/ext.c | 184 +++++++++++++++++++++++++++----------------- kernel/sched/ext/internal.h | 4 + 3 files changed, 120 insertions(+), 70 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index bd9c4059e8fc..3166a0c3d892 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -192,6 +192,8 @@ struct sched_ext_entity { atomic_long_t ops_state; u64 ddsp_dsq_id; u64 ddsp_enq_flags; + u64 ddsp_slice; + u64 ddsp_vtime; struct scx_dsq_list_node dsq_list; /* dispatch order */ struct rb_node dsq_priq; /* p->scx.dsq_vtime order */ u32 dsq_seq; diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index c603b90f16a1..6e59f2669c47 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1204,34 +1204,29 @@ enum scx_slice_oob_consts { }; /* - * Slice write rules + * Slice and dsq_vtime write rules * - * A task's slice - how long it may hold its cpu - is an occupancy grant owned - * by the task's scheduler. How it may be written depends on whether the task is - * running. + * While @p is running, sleeping or queued on an rq-owned DSQ, both fields are + * protected by the rq lock. While running, the rq lock is required because + * update_curr_scx() RMWs the slice and the cap check for slice extension is + * only reliable under the rq lock. * - * Queued, not running: the slice grants no occupancy yet and nothing consumes - * it, so the owner writes it directly - via scx_bpf_dsq_insert(), the dsq move - * kfuncs, or scx_bpf_task_set_slice(). Serializing its own writers is then the - * scheduler's job, not the kernel's. + * While @p is queued on a user DSQ or on the BPF side, the kernel neither + * consumes nor decides on the fields. Synchronizing the writers is the BPF + * scheduler's responsibility. An rq-locked scx_bpf_task_set_slice() write and a + * concurrent DSQ insertion commit can race each other and whichever lands last + * wins. * - * Running: the slice must be changed under the task's rq lock, because: + * A DSQ insert kfunc doesn't update the fields directly. The verdict carries + * the values and apply_slice_vtime() commits them at the insertion. * - * - Raising it extends occupancy, allowed only with %SCX_CAP_BASE on the cpu, - * and that cap check is coherent only under the rq lock. Shortening is always - * allowed. + * scx_bpf_task_set_slice() may be called from any context and writes directly + * only if @p's rq lock is already held, otherwise it bounces through + * p->scx.slice_oob, applied under @p's rq lock at the next slice consideration. * - * - The kernel decrements it there as the task runs. The decrement is a - * read-modify-write, so a racing write can be clobbered. - * - * scx_bpf_task_set_slice() writes directly only when @p is queued or running - * on the rq lock it holds. That is the only state where the lock keeps us @p's - * owner: @p can't move to another rq without it. A task that isn't queued here - * can instead be woken onto a different rq without taking this lock, and that - * dispatch sets its slice - so a direct write would race. Those cases stash - * into p->scx.slice_oob to be applied under @p's actual rq lock. A later in-band - * write supersedes a stash, and a stash whose scheduler id no longer matches - * @p's owner is dropped. + * dsq_vtime orders the next PRIQ insertion and has no running-side consumer, so + * scx_bpf_task_set_dsq_vtime() writes it directly. Fork-time init and direct + * BPF stores from non-cid-form schedulers are outside these rules. */ /* clear a pending slice request */ @@ -1244,6 +1239,7 @@ static void clear_task_slice_oob(struct task_struct *p) /* set @p's slice, leaving any pending out-of-band request in place */ static void set_task_slice_keep_oob(struct task_struct *p, u64 slice) { + lockdep_assert_rq_held(task_rq(p)); p->scx.slice = slice; } @@ -1254,7 +1250,7 @@ void scx_set_task_slice(struct task_struct *p, u64 slice) clear_task_slice_oob(p); } -/* request @p's slice to be set to @slice, see the slice write rules above */ +/* request @p's slice to be set to @slice, see the write rules above */ static void set_task_slice_oob(struct scx_sched *sch, struct task_struct *p, u64 slice) { u64 dur; @@ -1276,7 +1272,7 @@ static void set_task_slice_oob(struct scx_sched *sch, struct task_struct *p, u64 * Apply a pending out-of-band slice request under @rq's lock. A request whose * packed id no longer matches @p's current owner is dropped. An extension needs * baseline cpu access on @p's cid. %SCX_EV_SLICE_DENIED counts the denials. - * Shortening is always allowed. See the slice write rules above. + * Shortening is always allowed. See the write rules above. */ static void apply_task_slice_oob(struct rq *rq, struct task_struct *p) { @@ -1305,7 +1301,30 @@ static void apply_task_slice_oob(struct rq *rq, struct task_struct *p) return; } - p->scx.slice = slice; + set_task_slice_keep_oob(p, slice); +} + +/* + * A dsq insert kfunc doesn't write slice or dsq_vtime. The verdict carries them + * and they are committed here, at the insertion. A zero @slice keeps the + * current value, floored at 1 so the task isn't treated as expired. + */ +static void apply_slice_vtime(struct task_struct *p, u64 slice, u64 vtime, u64 enq_flags) +{ + if (slice) { + p->scx.slice = slice; + /* + * An explicit slice supersedes a pending oob request. A carried + * default refill is not an explicit request and must keep it. + */ + if (!(enq_flags & SCX_ENQ_SLICE_DFL)) + clear_task_slice_oob(p); + } else if (!p->scx.slice) { + p->scx.slice = 1; + } + + if (enq_flags & SCX_ENQ_DSQ_PRIQ) + p->scx.dsq_vtime = vtime; } static void update_curr_scx(struct rq *rq) @@ -1502,7 +1521,7 @@ static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq, static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq, struct scx_dispatch_q *dsq, struct task_struct *p, - u64 enq_flags) + u64 slice, u64 vtime, u64 enq_flags) { bool is_rq_owned = false; @@ -1541,6 +1560,13 @@ static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq, enq_flags &= ~SCX_ENQ_DSQ_PRIQ; } + /* + * @dsq is locked and @enq_flags is sanitized. Commit the carried slice + * and vtime before the PRIQ insertion below reads the new dsq_vtime. + */ + if (enq_flags & SCX_ENQ_APPLY_SLICE) + apply_slice_vtime(p, slice, vtime, enq_flags); + if (enq_flags & SCX_ENQ_DSQ_PRIQ) { struct rb_node *rbp; @@ -1763,7 +1789,7 @@ static struct scx_dispatch_q *find_dsq_for_dispatch(struct scx_sched *sch, static void mark_direct_dispatch(struct scx_sched *sch, struct task_struct *ddsp_task, struct task_struct *p, u64 dsq_id, - u64 enq_flags) + u64 slice, u64 vtime, u64 enq_flags) { /* * Mark that dispatch already happened from ops.select_cpu() or @@ -1787,6 +1813,8 @@ static void mark_direct_dispatch(struct scx_sched *sch, WARN_ON_ONCE(p->scx.ddsp_dsq_id != SCX_DSQ_INVALID); WARN_ON_ONCE(p->scx.ddsp_enq_flags); + p->scx.ddsp_slice = slice; + p->scx.ddsp_vtime = vtime; p->scx.ddsp_dsq_id = dsq_id; p->scx.ddsp_enq_flags = enq_flags; } @@ -1818,7 +1846,7 @@ static void direct_dispatch(struct scx_sched *sch, struct task_struct *p, struct rq *rq = task_rq(p); struct scx_dispatch_q *dsq = find_dsq_for_dispatch(sch, rq, p->scx.ddsp_dsq_id, task_cpu(p)); - u64 ddsp_enq_flags; + u64 ddsp_enq_flags, slice, vtime; touch_core_sched_dispatch(rq, p); @@ -1860,9 +1888,12 @@ static void direct_dispatch(struct scx_sched *sch, struct task_struct *p, } ddsp_enq_flags = p->scx.ddsp_enq_flags; + slice = p->scx.ddsp_slice; + vtime = p->scx.ddsp_vtime; clear_direct_dispatch(p); - scx_dispatch_enqueue(sch, rq, dsq, p, ddsp_enq_flags | SCX_ENQ_CLEAR_OPSS); + scx_dispatch_enqueue(sch, rq, dsq, p, slice, vtime, + ddsp_enq_flags | SCX_ENQ_APPLY_SLICE | SCX_ENQ_CLEAR_OPSS); } bool scx_rq_online(struct rq *rq) @@ -1983,7 +2014,7 @@ direct: direct_dispatch(sch, p, enq_flags); return; local_norefill: - scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, enq_flags); + scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0, enq_flags); return; local: dsq = &rq->scx.local_dsq; @@ -2004,7 +2035,7 @@ enqueue: touch_core_sched(rq, p); refill_task_slice_dfl(sch, p); clear_direct_dispatch(p); - scx_dispatch_enqueue(sch, rq, dsq, p, enq_flags); + scx_dispatch_enqueue(sch, rq, dsq, p, 0, 0, enq_flags); } static bool task_runnable(const struct task_struct *p) @@ -2542,7 +2573,7 @@ static struct rq *move_task_between_dsqs(struct scx_sched *sch, dispatch_dequeue_locked(p, src_dsq); raw_spin_unlock(&src_dsq->lock); - scx_dispatch_enqueue(sch, dst_rq, dst_dsq, p, enq_flags); + scx_dispatch_enqueue(sch, dst_rq, dst_dsq, p, 0, 0, enq_flags); } return dst_rq; @@ -2608,6 +2639,8 @@ bool scx_consume_global_dsq(struct scx_sched *sch, struct rq *rq) * @rq: current rq which is locked * @dst_dsq: destination DSQ * @p: task to dispatch + * @slice: slice carried by the insert verdict, 0 keeps the current value + * @vtime: vtime carried by the insert verdict, committed on PRIQ inserts * @enq_flags: %SCX_ENQ_* * * We're holding @rq lock and want to dispatch @p to @dst_dsq which is a local @@ -2618,8 +2651,8 @@ bool scx_consume_global_dsq(struct scx_sched *sch, struct rq *rq) * %SCX_OPSS_DISPATCHING). */ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, - struct scx_dispatch_q *dst_dsq, - struct task_struct *p, u64 enq_flags) + struct scx_dispatch_q *dst_dsq, struct task_struct *p, + u64 slice, u64 vtime, u64 enq_flags) { struct rq *src_rq = task_rq(p); struct rq *dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq); @@ -2632,8 +2665,8 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, * If dispatching to @rq that @p is already on, no lock dancing needed. */ if (rq == src_rq && rq == dst_rq) { - scx_dispatch_enqueue(sch, rq, dst_dsq, p, - enq_flags | SCX_ENQ_CLEAR_OPSS); + scx_dispatch_enqueue(sch, rq, dst_dsq, p, slice, vtime, + enq_flags | SCX_ENQ_APPLY_SLICE | SCX_ENQ_CLEAR_OPSS); return; } @@ -2671,13 +2704,16 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, if (src_rq == dst_rq) { p->scx.holding_cpu = -1; scx_dispatch_enqueue(sch, dst_rq, &dst_rq->scx.local_dsq, p, - enq_flags); + slice, vtime, enq_flags | SCX_ENQ_APPLY_SLICE); } else if (unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) { p->scx.holding_cpu = -1; fallback = true; scx_dispatch_enqueue(sch, src_rq, find_global_dsq(sch, task_cpu(p)), - p, enq_flags | SCX_ENQ_GDSQ_FALLBACK); + p, slice, vtime, + enq_flags | SCX_ENQ_APPLY_SLICE | + SCX_ENQ_GDSQ_FALLBACK); } else { + apply_slice_vtime(p, slice, vtime, enq_flags); move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, dst_rq); /* task has been moved to dst_rq, which is now locked */ locked_rq = dst_rq; @@ -2713,10 +2749,9 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, * was valid in the first place. Make sure that the task is still owned by the * BPF scheduler and claim the ownership before dispatching. */ -static void finish_dispatch(struct scx_sched *sch, struct rq *rq, - struct task_struct *p, - unsigned long qseq_at_dispatch, - u64 dsq_id, u64 enq_flags) +static void finish_dispatch(struct scx_sched *sch, struct rq *rq, struct task_struct *p, + unsigned long qseq_at_dispatch, u64 dsq_id, + u64 slice, u64 vtime, u64 enq_flags) { struct scx_dispatch_q *dsq; unsigned long opss; @@ -2776,9 +2811,10 @@ retry: dsq = find_dsq_for_dispatch(sch, this_rq(), dsq_id, task_cpu(p)); if (dsq->id == SCX_DSQ_LOCAL) - dispatch_to_local_dsq(sch, rq, dsq, p, enq_flags); + dispatch_to_local_dsq(sch, rq, dsq, p, slice, vtime, enq_flags); else - scx_dispatch_enqueue(sch, rq, dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); + scx_dispatch_enqueue(sch, rq, dsq, p, slice, vtime, + enq_flags | SCX_ENQ_APPLY_SLICE | SCX_ENQ_CLEAR_OPSS); } void scx_flush_dispatch_buf(struct scx_sched *sch, struct rq *rq) @@ -2790,7 +2826,7 @@ void scx_flush_dispatch_buf(struct scx_sched *sch, struct rq *rq) struct scx_dsp_buf_ent *ent = &dspc->buf[u]; finish_dispatch(sch, rq, ent->task, ent->qseq, ent->dsq_id, - ent->enq_flags); + ent->slice, ent->vtime, ent->enq_flags); } dspc->nr_tasks += dspc->cursor; @@ -3035,7 +3071,8 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; } else { - scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, SCX_ENQ_HEAD); + scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0, + SCX_ENQ_HEAD); } goto switch_class; } @@ -3319,7 +3356,13 @@ static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flag cpu = scx_select_cpu_dfl(p, prev_cpu, wake_flags, NULL, 0); if (cpu >= 0) { - refill_task_slice_dfl(sch, p); + /* + * Carry the slice refill and let the insertion commit + * it under rq lock. See the write rules. + */ + __scx_add_event(sch, SCX_EV_REFILL_SLICE_DFL, 1); + p->scx.ddsp_slice = READ_ONCE(sch->slice_dfl); + p->scx.ddsp_enq_flags = SCX_ENQ_SLICE_DFL; p->scx.ddsp_dsq_id = SCX_DSQ_LOCAL; } else { cpu = prev_cpu; @@ -4051,13 +4094,15 @@ static void process_ddsp_deferred_locals(struct rq *rq) struct scx_dispatch_q *dsq; u64 dsq_id = p->scx.ddsp_dsq_id; u64 enq_flags = p->scx.ddsp_enq_flags; + u64 slice = p->scx.ddsp_slice; + u64 vtime = p->scx.ddsp_vtime; list_del_init(&p->scx.dsq_list.node); clear_direct_dispatch(p); dsq = find_dsq_for_dispatch(sch, rq, dsq_id, task_cpu(p)); if (!WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL)) - dispatch_to_local_dsq(sch, rq, dsq, p, enq_flags); + dispatch_to_local_dsq(sch, rq, dsq, p, slice, vtime, enq_flags); } } @@ -5487,7 +5532,7 @@ resume: * between bypass DSQs. */ dispatch_dequeue_locked(p, donor_dsq); - scx_dispatch_enqueue(sch, cpu_rq(donee), donee_dsq, p, SCX_ENQ_NESTED); + scx_dispatch_enqueue(sch, cpu_rq(donee), donee_dsq, p, 0, 0, SCX_ENQ_NESTED); /* * $donee might have been idle and need to be woken up. No need @@ -8435,14 +8480,14 @@ static bool scx_dsq_insert_preamble(struct scx_sched *sch, struct task_struct *p } static void scx_dsq_insert_commit(struct scx_sched *sch, struct task_struct *p, - u64 dsq_id, u64 enq_flags) + u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) { struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; struct task_struct *ddsp_task; ddsp_task = __this_cpu_read(direct_dispatch_task); if (ddsp_task) { - mark_direct_dispatch(sch, ddsp_task, p, dsq_id, enq_flags); + mark_direct_dispatch(sch, ddsp_task, p, dsq_id, slice, vtime, enq_flags); return; } @@ -8455,6 +8500,8 @@ static void scx_dsq_insert_commit(struct scx_sched *sch, struct task_struct *p, .task = p, .qseq = atomic_long_read(&p->scx.ops_state) & SCX_OPSS_QSEQ_MASK, .dsq_id = dsq_id, + .slice = slice, + .vtime = vtime, .enq_flags = enq_flags, }; } @@ -8515,12 +8562,7 @@ __bpf_kfunc bool scx_bpf_dsq_insert___v2(struct task_struct *p, u64 dsq_id, if (!scx_dsq_insert_preamble(sch, p, dsq_id, &enq_flags)) return false; - if (slice) - scx_set_task_slice(p, slice); - else - set_task_slice_keep_oob(p, p->scx.slice ?: 1); - - scx_dsq_insert_commit(sch, p, dsq_id, enq_flags); + scx_dsq_insert_commit(sch, p, dsq_id, slice, 0, enq_flags); return true; } @@ -8541,14 +8583,7 @@ static bool scx_dsq_insert_vtime(struct scx_sched *sch, struct task_struct *p, if (!scx_dsq_insert_preamble(sch, p, dsq_id, &enq_flags)) return false; - if (slice) - scx_set_task_slice(p, slice); - else - set_task_slice_keep_oob(p, p->scx.slice ?: 1); - - p->scx.dsq_vtime = vtime; - - scx_dsq_insert_commit(sch, p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ); + scx_dsq_insert_commit(sch, p, dsq_id, slice, vtime, enq_flags | SCX_ENQ_DSQ_PRIQ); return true; } @@ -8724,9 +8759,9 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, dst_dsq = find_dsq_for_dispatch(sch, this_rq, dsq_id, task_cpu(p)); /* - * Apply vtime and slice updates before moving so that the new time is - * visible before inserting into $dst_dsq. @p is still on $src_dsq but - * this is safe as we're locking it. + * Apply vtime and slice updates before moving. @p is still on $src_dsq + * with both $src_dsq and its task_rq locked, satisfying the write + * rules, and the PRIQ insertion into $dst_dsq reads the new vtime. */ if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_VTIME) p->scx.dsq_vtime = kit->vtime; @@ -9136,7 +9171,16 @@ __bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice, /* * Directly write only when we hold the lock of the rq @p is queued or - * running on. See the slice write rules above. + * running on. See the write rules above. + * + * While @p is queued on a user DSQ or in the BPF scheduler, + * synchronization is the scheduler's responsibility. This write can + * race a concurrent dispatch's commit, see apply_slice_vtime(). + * + * Making this kfunc always go through the oob stash would leave the + * commit as the only direct writer and close the race, but that would + * require two more oob application points - the dispatch keep-prev test + * and the tick-time expiry check. */ locked_rq = scx_locked_rq(); if (!locked_rq || diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index a0a2294f1dc2..a11ed6e1e028 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1246,6 +1246,8 @@ struct scx_dsp_buf_ent { struct task_struct *task; unsigned long qseq; u64 dsq_id; + u64 slice; + u64 vtime; u64 enq_flags; }; @@ -1680,6 +1682,8 @@ enum scx_enq_flags { SCX_ENQ_NESTED = 1LLU << 58, SCX_ENQ_GDSQ_FALLBACK = 1LLU << 59, /* fell back to global DSQ */ SCX_ENQ_IGNORE_CAPS = 1LLU << 60, /* admit to local DSQ ignoring caps */ + SCX_ENQ_APPLY_SLICE = 1LLU << 61, /* apply carried slice/vtime at insertion */ + SCX_ENQ_SLICE_DFL = 1LLU << 62, /* carried slice is a default refill */ }; enum scx_deq_flags { -- cgit v1.2.3 From 9cfc6ab34a3184b3683d27dcccc5c05bede41c37 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 3 Aug 2026 11:01:20 -1000 Subject: sched_ext: Add SCX_TASK_PROTECTED A BPF scheduler can displace any of its tasks at will - cut a running one's slice with an SCX_ENQ_PREEMPT dispatch, an SCX_KICK_PREEMPT kick or a direct shortening, and jump a queued one with HEAD insertions. Sometimes the kernel needs a slice and a DSQ position to stick regardless. Add SCX_TASK_PROTECTED, guarding both: - The slice becomes immutable. Every scheduler-reachable write is refused and counted as SCX_EV_SLICE_DENIED. Higher scheduling classes are unaffected. PREEMPT|IMMED can't preempt a running protected task and gets reenqueued. - A protected task that reached the head of its DSQ keeps it - HEAD insertions land behind the leading run of protected tasks and reenqueue sweeps skip them. Only rq-owned DSQs can hold protected tasks, so the walk runs only for them. The bit lives in p->scx.flags so that both the refusal and the head walk read it under the rq lock that protects it. Protection ends when the slice is consumed, when the task leaves the rq except for a save/restore on the running task, on a yield, when the scheduler enters bypass, and when the task leaves scx. The flag is kernel-internal and not used yet. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- include/linux/sched/ext.h | 1 + kernel/sched/ext/ext.c | 165 ++++++++++++++++++++++++++++++++++++++------ kernel/sched/ext/internal.h | 2 +- 3 files changed, 144 insertions(+), 24 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index 3166a0c3d892..b519fbc88e17 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -102,6 +102,7 @@ enum scx_ent_flags { SCX_TASK_DEQD_FOR_SLEEP = 1 << 3, /* last dequeue was for SLEEP */ SCX_TASK_SUB_INIT = 1 << 4, /* task being initialized for a sub sched */ SCX_TASK_IMMED = 1 << 5, /* task is on local DSQ with %SCX_ENQ_IMMED */ + SCX_TASK_PROTECTED = 1 << 6, /* slice and DSQ head position protected */ /* * Bits 8 to 10 are used to carry task state: diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 6e59f2669c47..0121ecec8b25 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -386,8 +386,16 @@ static bool rq_is_open(struct rq *rq, u64 enq_flags) * so allow it to avoid spuriously triggering reenq on a combined * PREEMPT|IMMED insertion. */ - if (enq_flags & SCX_ENQ_PREEMPT) - return true; + if (enq_flags & SCX_ENQ_PREEMPT) { + struct task_struct *curr = rq->curr; + + /* + * A protected slice refuses the preemption and the cpu stays + * occupied. See rq_owned_post_enq(). + */ + return curr->sched_class != &ext_sched_class || + likely(!(curr->scx.flags & SCX_TASK_PROTECTED)); + } /* * @rq is either in transition to or running an SCX task and can't go @@ -1224,6 +1232,9 @@ enum scx_slice_oob_consts { * only if @p's rq lock is already held, otherwise it bounces through * p->scx.slice_oob, applied under @p's rq lock at the next slice consideration. * + * While %SCX_TASK_PROTECTED is set, every scheduler-reachable slice update is + * refused. See set_task_slice_keep_oob(). + * * dsq_vtime orders the next PRIQ insertion and has no running-side consumer, so * scx_bpf_task_set_dsq_vtime() writes it directly. Fork-time init and direct * BPF stores from non-cid-form schedulers are outside these rules. @@ -1236,18 +1247,92 @@ static void clear_task_slice_oob(struct task_struct *p) atomic64_set(&p->scx.slice_oob, 0); } -/* set @p's slice, leaving any pending out-of-band request in place */ -static void set_task_slice_keep_oob(struct task_struct *p, u64 slice) +/** + * dsq_insert_head - FIFO head insertion honoring %SCX_TASK_PROTECTED + * @dsq: DSQ to insert into + * @p: task being inserted + * + * A HEAD insert should land behind any leading protected tasks. Return %true + * indicates whether @p became the first entry. + */ +static bool dsq_insert_head(struct scx_dispatch_q *dsq, struct task_struct *p) +{ + struct list_head *pos = &dsq->list; + struct scx_dsq_list_node *node; + + /* + * Only rq-owned DSQs can hold protected tasks and the associated rq + * lock keeps their flags stable. + */ + if (!dsq_is_rq_owned(dsq)) { + list_add(&p->scx.dsq_list.node, &dsq->list); + return true; + } + + list_for_each_entry(node, &dsq->list, node) { + struct task_struct *q; + + if (WARN_ON_ONCE(node->flags & SCX_DSQ_LNODE_ITER_CURSOR)) + continue; + + q = container_of(node, struct task_struct, scx.dsq_list); + if (!(q->scx.flags & SCX_TASK_PROTECTED)) + break; + + pos = &node->node; + } + + list_add(&p->scx.dsq_list.node, pos); + + return pos == &dsq->list; +} + +/** + * set_task_slice_keep_oob - Set @p's slice, leaving any pending oob request + * @p: task of interest + * @slice: slice to set + * + * While %SCX_TASK_PROTECTED is set, BPF schedulers may not modify the slice. + * Refuse and return %false. + */ +static bool set_task_slice_keep_oob(struct task_struct *p, u64 slice) { lockdep_assert_rq_held(task_rq(p)); + + if (unlikely(p->scx.flags & SCX_TASK_PROTECTED)) + return false; + p->scx.slice = slice; + return true; } /* set @p's slice, superseding any pending out-of-band request */ -void scx_set_task_slice(struct task_struct *p, u64 slice) +bool scx_set_task_slice(struct task_struct *p, u64 slice) { - set_task_slice_keep_oob(p, slice); + if (!set_task_slice_keep_oob(p, slice)) + return false; clear_task_slice_oob(p); + return true; +} + +/** + * scx_task_slice_ended - @p's slice is consumed or given up + * @rq: rq @p is on + * @p: task of interest + * + * End what rides on the slice - the protection. + * + * A dequeue normally ends the slice too. The exception is a save/restore pair + * on the running task. Attribute changes like renice cycle the task through + * dequeue and enqueue while it keeps executing, so the slice continues. A + * queued task instead loses its DSQ position on any dequeue and the slice ends + * with it. + */ +static void scx_task_slice_ended(struct rq *rq, struct task_struct *p) +{ + lockdep_assert_rq_held(rq); + + p->scx.flags &= ~SCX_TASK_PROTECTED; } /* request @p's slice to be set to @slice, see the write rules above */ @@ -1271,8 +1356,9 @@ static void set_task_slice_oob(struct scx_sched *sch, struct task_struct *p, u64 /* * Apply a pending out-of-band slice request under @rq's lock. A request whose * packed id no longer matches @p's current owner is dropped. An extension needs - * baseline cpu access on @p's cid. %SCX_EV_SLICE_DENIED counts the denials. - * Shortening is always allowed. See the write rules above. + * baseline cpu access on @p's cid, shortening is always allowed, and a + * protected slice refuses both. %SCX_EV_SLICE_DENIED counts the denials. See + * the write rules above. */ static void apply_task_slice_oob(struct rq *rq, struct task_struct *p) { @@ -1301,7 +1387,8 @@ static void apply_task_slice_oob(struct rq *rq, struct task_struct *p) return; } - set_task_slice_keep_oob(p, slice); + if (unlikely(!set_task_slice_keep_oob(p, slice))) + __scx_add_event(scx_task_sched(p), SCX_EV_SLICE_DENIED, 1); } /* @@ -1514,8 +1601,10 @@ static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq, if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr && rq->curr->sched_class == &ext_sched_class) { - scx_set_task_slice(rq->curr, 0); - resched_curr(rq); + if (likely(scx_set_task_slice(rq->curr, 0))) + resched_curr(rq); + else + __scx_add_event(sch, SCX_EV_SLICE_DENIED, 1); } } @@ -1606,9 +1695,8 @@ static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq, dsq->id); if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) { - list_add(&p->scx.dsq_list.node, &dsq->list); /* new task inserted at head - use fastpath */ - if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN)) + if (dsq_insert_head(dsq, p) && !(dsq->id & SCX_DSQ_FLAG_BUILTIN)) rcu_assign_pointer(dsq->first_task, p); } else { /* @@ -2254,6 +2342,11 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_ sub_nr_running(rq, 1); scx_dispatch_dequeue(rq, p); + + /* see scx_task_slice_ended() for the save/restore exception */ + if (!((deq_flags & DEQUEUE_SAVE) && task_current(rq, p))) + scx_task_slice_ended(rq, p); + clear_direct_dispatch(p); return true; } @@ -2263,6 +2356,9 @@ static void yield_task_scx(struct rq *rq) struct task_struct *p = rq->donor; struct scx_sched *sch = scx_task_sched(p); + /* a yield gives the slice up */ + scx_task_slice_ended(rq, p); + if (SCX_HAS_OP(sch, yield)) SCX_CALL_OP_2TASKS_RET(sch, yield, rq, p, NULL); else @@ -2274,6 +2370,9 @@ static bool yield_to_task_scx(struct rq *rq, struct task_struct *to) struct task_struct *from = rq->donor; struct scx_sched *sch = scx_task_sched(from); + /* like a plain yield, giving the slice up ends the protection */ + scx_task_slice_ended(rq, from); + if (SCX_HAS_OP(sch, yield) && sch == scx_task_sched(to)) return SCX_CALL_OP_2TASKS_RET(sch, yield, rq, from, to); else @@ -2317,7 +2416,7 @@ void scx_move_local_task_to_local_dsq(struct scx_sched *sch, struct task_struct WARN_ON_ONCE(p->scx.holding_cpu >= 0); if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) - list_add(&p->scx.dsq_list.node, &dst_dsq->list); + dsq_insert_head(dst_dsq, p); else list_add_tail(&p->scx.dsq_list.node, &dst_dsq->list); @@ -3051,6 +3150,10 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, update_curr_scx(rq); + /* the slice is consumed, protection ends with it */ + if (!p->scx.slice) + scx_task_slice_ended(rq, p); + /* see dequeue_task_scx() on why we skip when !QUEUED */ if (SCX_HAS_OP(sch, stopping) && (p->scx.flags & SCX_TASK_QUEUED)) SCX_CALL_OP_TASK(sch, stopping, rq, p, true); @@ -3204,8 +3307,11 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) */ if (keep_prev) { p = prev; - if (!p->scx.slice) + if (!p->scx.slice) { + /* the slice is consumed, protection ends */ + scx_task_slice_ended(rq, p); refill_task_slice_dfl(scx_task_sched(p), p); + } } else { p = first_local_task(rq); if (!p) @@ -3725,6 +3831,7 @@ static void scx_disable_task(struct scx_sched *sch, struct task_struct *p) * control, after ops.disable() has observed their final values. */ p->scx.dsq_vtime = 0; + scx_task_slice_ended(rq, p); scx_set_task_slice(p, 0); p->scx.reenq_cnt = 0; @@ -4137,6 +4244,9 @@ static bool local_task_should_reenq(struct rq *rq, struct task_struct *p, first = !(*reenq_flags & SCX_REENQ_TSR_NOT_FIRST); *reenq_flags |= SCX_REENQ_TSR_NOT_FIRST; + if (unlikely(p->scx.flags & SCX_TASK_PROTECTED)) + return false; + *reason = SCX_TASK_REENQ_KFUNC; if ((p->scx.flags & SCX_TASK_IMMED) && @@ -5901,6 +6011,13 @@ void scx_bypass(struct scx_sched *sch, bool bypass) if (!scx_is_descendant(scx_task_sched(p), sch)) continue; + /* + * Bypass trumps protection. Cycling clears for queued + * tasks but current task needs explicit stripping. + */ + if (bypass && task_current(rq, p)) + scx_task_slice_ended(rq, p); + /* cycling deq/enq is enough, see the function comment */ scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { /* nothing */ ; @@ -8197,11 +8314,10 @@ static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_r if (cur_class == &ext_sched_class) { u64 caps = scx_caps_for_preempt(pcpu->sch, rq, 0); - if (likely(!scx_missing_caps(pcpu->sch, cpu, caps))) - scx_set_task_slice(rq->curr, 0); - else - __scx_add_event(pcpu->sch, - SCX_EV_SUB_PREEMPT_DENIED, 1); + if (unlikely(scx_missing_caps(pcpu->sch, cpu, caps))) + __scx_add_event(pcpu->sch, SCX_EV_SUB_PREEMPT_DENIED, 1); + else if (unlikely(!scx_set_task_slice(rq->curr, 0))) + __scx_add_event(pcpu->sch, SCX_EV_SLICE_DENIED, 1); } cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt); } @@ -9192,10 +9308,13 @@ __bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice, /* under the rq lock: apply now, extensions gated on baseline access */ if (slice > p->scx.slice && - unlikely(scx_missing_caps(sch, cpu_of(locked_rq), SCX_CAP_BASE))) + unlikely(scx_missing_caps(sch, cpu_of(locked_rq), SCX_CAP_BASE))) { + __scx_add_event(sch, SCX_EV_SLICE_DENIED, 1); + return true; + } + + if (unlikely(!scx_set_task_slice(p, slice))) __scx_add_event(sch, SCX_EV_SLICE_DENIED, 1); - else - scx_set_task_slice(p, slice); return true; } diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index a11ed6e1e028..d418935f1e6b 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1972,7 +1972,7 @@ void scx_task_iter_start(struct scx_task_iter *iter, struct cgroup *cgrp); void scx_task_iter_unlock(struct scx_task_iter *iter); void scx_task_iter_stop(struct scx_task_iter *iter); struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter); -void scx_set_task_slice(struct task_struct *p, u64 slice); +bool scx_set_task_slice(struct task_struct *p, u64 slice); void scx_task_unlink_from_dsq(struct task_struct *p, struct scx_dispatch_q *dsq); void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p); void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, -- cgit v1.2.3 From 5fd501744b10814f5c12899ce86d223cee2c51ca Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 3 Aug 2026 11:01:29 -1000 Subject: sched_ext: Add bandwidth-limited rescue execution for stranded tasks A local DSQ insert lacking the needed caps is diverted to the reject DSQ and bounced back through ops.enqueue() so the scheduler can re-decide. That recovery assumes the scheduler has somewhere legal to send the task. When it doesn't, e.g. when the task's affinity is restricted to cids delegated away, the task starves until the stall watchdog ejects the scheduler. An exiting task is worse - it skips ops.enqueue() and the rejection becomes a self-requeuing cycle that burns the CPU until the watchdog fires. Add SCX_ENQ_RESCUE, a fallback modifier on local DSQ inserts. When the insert would be rejected for missing caps, the kernel takes over and runs the task on the target CPU without consulting the owning scheduler. The kernel sets the flag itself when enqueueing an exiting task. Rescue is a last-resort forward-progress backstop with a persistent disadvantage, not a way around cap enforcement. A per-CPU token bucket accrues rescue_bandwidth_ppt (default 2%) of CPU time and rescues run one at a time in arrival order. Each is granted a slice of the rescue_quantum_us (default 5ms) quantum divided across the waiters, waits at the tail of the local DSQ claiming no priority, and rejoins its scheduler as a fresh arrival once the slice is served. The schedulers keep their normal control over an admitted rescuee and may preempt or reslice it. Service is measured on CPU time actually received, so neither shortens the rescue. Prolonged denial escalates - the remaining slice turns into protected execution (SCX_TASK_PROTECTED) and the rescuee preempts the current task. Escalation is paced by the same bucket, and delivered service converges on the configured bandwidth no matter how aggressively the schedulers dispatch. Both knobs are root-only and SCX_RESCUE_DISABLE turns rescue off, making SCX_ENQ_RESCUE inserts reject as usual. v2: - Add SCX_OPS_OPEN() fix-ups for the new ops fields so cpu-form schedulers setting them still load on older kernels. (Andrea) Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- include/linux/sched/ext.h | 1 + kernel/sched/ext/ext.c | 112 +++++++++-- kernel/sched/ext/internal.h | 56 +++++- kernel/sched/ext/sub.c | 376 +++++++++++++++++++++++++++++++++-- kernel/sched/ext/sub.h | 26 +++ kernel/sched/ext/types.h | 9 + kernel/sched/sched.h | 12 ++ tools/sched_ext/include/scx/compat.h | 11 + 8 files changed, 570 insertions(+), 33 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index b519fbc88e17..a6aabbefd185 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -59,6 +59,7 @@ enum scx_dsq_id_flags { SCX_DSQ_LOCAL = SCX_DSQ_FLAG_BUILTIN | 2, SCX_DSQ_BYPASS = SCX_DSQ_FLAG_BUILTIN | 3, SCX_DSQ_REJECT = SCX_DSQ_FLAG_BUILTIN | 4, /* internal - see find_dsq_for_dispatch() */ + SCX_DSQ_RESCUE = SCX_DSQ_FLAG_BUILTIN | 5, /* internal - see find_dsq_for_dispatch() */ SCX_DSQ_LOCAL_ON = SCX_DSQ_FLAG_BUILTIN | SCX_DSQ_FLAG_LOCAL_ON, SCX_DSQ_LOCAL_CPU_MASK = 0xffffffffLLU, }; diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 0121ecec8b25..7f86d4adcd0d 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -111,6 +111,7 @@ static bool dsq_is_rq_owned(struct scx_dispatch_q *dsq) switch (dsq->id) { case SCX_DSQ_LOCAL: case SCX_DSQ_REJECT: + case SCX_DSQ_RESCUE: return true; default: return false; @@ -1320,7 +1321,8 @@ bool scx_set_task_slice(struct task_struct *p, u64 slice) * @rq: rq @p is on * @p: task of interest * - * End what rides on the slice - the protection. + * End what rides on the slice - the protection, and the rescue if @p is being + * rescued. * * A dequeue normally ends the slice too. The exception is a save/restore pair * on the running task. Attribute changes like renice cycle the task through @@ -1328,11 +1330,13 @@ bool scx_set_task_slice(struct task_struct *p, u64 slice) * queued task instead loses its DSQ position on any dequeue and the slice ends * with it. */ -static void scx_task_slice_ended(struct rq *rq, struct task_struct *p) +void scx_task_slice_ended(struct rq *rq, struct task_struct *p) { lockdep_assert_rq_held(rq); p->scx.flags &= ~SCX_TASK_PROTECTED; + if (unlikely(p == scx_rescuee(rq))) + scx_rescue_end(rq); } /* request @p's slice to be set to @slice, see the write rules above */ @@ -1432,6 +1436,9 @@ static void update_curr_scx(struct rq *rq) touch_core_sched(rq, curr); } + if (unlikely(curr == scx_rescuee(rq))) + scx_rescue_charge(rq, delta_exec); + dl_server_update(&rq->ext_server, delta_exec); } @@ -1547,9 +1554,13 @@ static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq, { call_task_dequeue(sch, rq, p, 0); - /* rejected: kick the deferred reenq, skip wakeup/preemption */ - if (unlikely(dsq->id == SCX_DSQ_REJECT)) { - schedule_deferred_locked(rq); + /* + * Only local inserts get the wakeup treatment below. Rejects kick the + * deferred reenq and rescue parks are paced by the rescue timer. + */ + if (unlikely(dsq->id != SCX_DSQ_LOCAL)) { + if (dsq->id == SCX_DSQ_REJECT) + schedule_deferred_locked(rq); return; } @@ -1863,8 +1874,8 @@ static struct scx_dispatch_q *find_dsq_for_dispatch(struct scx_sched *sch, dsq = find_user_dsq(sch, dsq_id); /* - * Built-in DSQs are never inserted into dsq_hash, so REJECT hits the - * error below. It cannot be reached with an ID. + * Built-in DSQs are never inserted into dsq_hash, so REJECT and RESCUE + * hit the error below. They cannot be reached with an ID. */ if (unlikely(!dsq)) { scx_error(sch, "non-existent DSQ 0x%llx", dsq_id); @@ -2056,6 +2067,7 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, if (!(sch->ops.flags & SCX_OPS_ENQ_EXITING) && unlikely(p->flags & PF_EXITING)) { __scx_add_event(sch, SCX_EV_ENQ_SKIP_EXITING, 1); + enq_flags |= SCX_ENQ_RESCUE; /* avoid looping on cap rejection */ goto local; } @@ -2324,9 +2336,11 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_ * information meaningful to the BPF scheduler and can be suppressed by * skipping the callbacks if the task is !QUEUED. */ - if (SCX_HAS_OP(sch, stopping) && task_current(rq, p)) { + if (task_current(rq, p) && + (SCX_HAS_OP(sch, stopping) || unlikely(p == scx_rescuee(rq)))) { update_curr_scx(rq); - SCX_CALL_OP_TASK(sch, stopping, rq, p, false); + if (SCX_HAS_OP(sch, stopping)) + SCX_CALL_OP_TASK(sch, stopping, rq, p, false); } if (SCX_HAS_OP(sch, quiescent) && !task_on_rq_migrating(p)) @@ -2409,8 +2423,9 @@ void scx_move_local_task_to_local_dsq(struct scx_sched *sch, struct task_struct { struct scx_dispatch_q *dst_dsq = scx_resolve_local_dsq(sch, dst_rq, p, &enq_flags); - /* @dsq is locked and @p is on @dst_rq */ - lockdep_assert_held(&src_dsq->lock); + /* @p is on @dst_rq, an rq-owned @src_dsq is covered by the rq lock */ + if (!dsq_is_rq_owned(src_dsq)) + lockdep_assert_held(&src_dsq->lock); lockdep_assert_rq_held(dst_rq); WARN_ON_ONCE(p->scx.holding_cpu >= 0); @@ -3144,15 +3159,23 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, struct task_struct *next) { struct scx_sched *sch = scx_task_sched(p); + bool rescue_keep = false; /* see kick_sync_wait_bal_cb() */ smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); update_curr_scx(rq); - /* the slice is consumed, protection ends with it */ - if (!p->scx.slice) - scx_task_slice_ended(rq, p); + /* + * If the slice is consumed, protection ends with it. A rescuee + * preempted beforehand keeps going, see scx_rescue_keep(). + */ + if (!p->scx.slice) { + if (unlikely(p == scx_rescuee(rq))) + rescue_keep = scx_rescue_keep(rq, p); + if (!rescue_keep) + scx_task_slice_ended(rq, p); + } /* see dequeue_task_scx() on why we skip when !QUEUED */ if (SCX_HAS_OP(sch, stopping) && (p->scx.flags & SCX_TASK_QUEUED)) @@ -3167,15 +3190,34 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, * forcing a different task. Leave it at the head of the local * DSQ unless it was an IMMED task. IMMED tasks should not * linger on a busy CPU, reenqueue them to the BPF scheduler. + * + * An open rescue must keep @p on the local DSQ even if the + * scheduler zeroed the slice in ops.stopping() above. */ - if (p->scx.slice && !scx_bypassing(sch, cpu_of(rq))) { + if ((p->scx.slice || unlikely(p == scx_rescuee(rq))) && + !scx_bypassing(sch, cpu_of(rq))) { if (p->scx.flags & SCX_TASK_IMMED) { p->scx.flags |= SCX_TASK_REENQ_PREEMPTED; scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; } else { + u64 enq_flags = 0; + + /* + * Keep a preempted rescue going. If preempted + * by another SCX task, append to the local DSQ, + * see scx_rescue_keep(). + */ + if (unlikely(p == scx_rescuee(rq))) { + enq_flags |= SCX_ENQ_IGNORE_CAPS; + if (!rescue_keep) + enq_flags |= SCX_ENQ_HEAD; + } else { + enq_flags |= SCX_ENQ_HEAD; + } + scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0, - SCX_ENQ_HEAD); + enq_flags); } goto switch_class; } @@ -3575,6 +3617,7 @@ static void rq_online_scx(struct rq *rq) static void rq_offline_scx(struct rq *rq) { rq->scx.flags &= ~SCX_RQ_ONLINE; + scx_rescue_flush(rq); } static bool check_rq_for_timeouts(struct rq *rq) @@ -4244,7 +4287,7 @@ static bool local_task_should_reenq(struct rq *rq, struct task_struct *p, first = !(*reenq_flags & SCX_REENQ_TSR_NOT_FIRST); *reenq_flags |= SCX_REENQ_TSR_NOT_FIRST; - if (unlikely(p->scx.flags & SCX_TASK_PROTECTED)) + if (unlikely((p->scx.flags & SCX_TASK_PROTECTED) || p == scx_rescuee(rq))) return false; *reason = SCX_TASK_REENQ_KFUNC; @@ -4527,6 +4570,13 @@ bool scx_can_stop_tick(struct rq *rq) if (scx_bypassing(sch, cpu_of(rq))) return false; + /* + * A running rescuee's charging and expiry are tick-driven, see + * scx_rescue_charge(). Keep the tick while rescue is in progress. + */ + if (unlikely(p == scx_rescuee(rq))) + return false; + /* * @rq can dispatch from different DSQs, so we can't tell whether it * needs the tick or not by looking at nr_running. Allow stopping ticks @@ -6722,6 +6772,7 @@ static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s, scx_dump_line(&ns, "CPU %-4d: nr_run=%u flags=0x%x cpu_rel=%d ops_qseq=%lu ksync=%lu", cpu, rq->scx.nr_running, rq->scx.flags, rq->scx.cpu_released, rq->scx.ops_qseq, rq->scx.kick_sync); + scx_rescue_dump(&ns, rq); scx_dump_line(&ns, " curr=%s[%d] class=%ps", rq->curr->comm, rq->curr->pid, rq->curr->sched_class); if (!cpumask_empty(pcpu->cpus_to_kick)) @@ -7393,6 +7444,7 @@ static void scx_root_enable_workfn(struct kthread_work *work) } scx_discard_stale_ecaps_syncs(); + scx_rescue_set_knobs(sch); /* * Keep CPUs stable during enable so that the BPF scheduler can track @@ -7899,6 +7951,24 @@ static int bpf_scx_init_member(const struct btf_type *t, case offsetof(struct sched_ext_ops, cid_shard_size): ops->cid_shard_size = *(u32 *)(udata + moff); return 1; + case offsetof(struct sched_ext_ops, rescue_bandwidth_ppt): { + u32 bw_ppt = *(u32 *)(udata + moff); + + if (bw_ppt > SCX_RESCUE_MAX_BW_PPT && bw_ppt != SCX_RESCUE_DISABLE) + return -E2BIG; + ops->rescue_bandwidth_ppt = bw_ppt; + return 1; + } + case offsetof(struct sched_ext_ops, rescue_quantum_us): { + u32 quantum_us = *(u32 *)(udata + moff); + + if (quantum_us > SCX_RESCUE_MAX_QUANTUM_US) + return -E2BIG; + if (quantum_us && quantum_us < SCX_RESCUE_MIN_QUANTUM_US) + return -EINVAL; + ops->rescue_quantum_us = quantum_us; + return 1; + } #ifdef CONFIG_EXT_SUB_SCHED case offsetof(struct sched_ext_ops, sub_cgroup_id): ops->sub_cgroup_id = *(u64 *)(udata + moff); @@ -8521,6 +8591,7 @@ void __init init_sched_ext_class(void) BUG_ON(scx_init_dsq(&rq->scx.local_dsq, SCX_DSQ_LOCAL, NULL)); #ifdef CONFIG_EXT_SUB_SCHED BUG_ON(scx_init_dsq(&rq->scx.reject_dsq, SCX_DSQ_REJECT, NULL)); + scx_rescue_init(rq); #endif INIT_LIST_HEAD(&rq->scx.runnable_list); @@ -8570,6 +8641,11 @@ static bool scx_vet_enq_flags(struct scx_sched *sch, u64 dsq_id, u64 *enq_flags) *enq_flags |= SCX_ENQ_IMMED; } + if (unlikely((*enq_flags & SCX_ENQ_RESCUE) && !is_local)) { + scx_error(sch, "SCX_ENQ_RESCUE on a non-local DSQ 0x%llx", dsq_id); + return false; + } + return true; } @@ -10751,6 +10827,8 @@ static int __init scx_init(void) CID_OFFSET_MATCH(exit_dump_len, exit_dump_len); CID_OFFSET_MATCH(hotplug_seq, hotplug_seq); CID_OFFSET_MATCH(cid_shard_size, cid_shard_size); + CID_OFFSET_MATCH(rescue_bandwidth_ppt, rescue_bandwidth_ppt); + CID_OFFSET_MATCH(rescue_quantum_us, rescue_quantum_us); CID_OFFSET_MATCH(sub_cgroup_id, sub_cgroup_id); /* shared callbacks: the union view requires byte-for-byte offset match */ CID_OFFSET_MATCH(enqueue, enqueue); diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index d418935f1e6b..18983dbe81f4 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -924,6 +924,37 @@ struct sched_ext_ops { */ u32 cid_shard_size; + /** + * @rescue_bandwidth_ppt: Rescue execution bandwidth in parts per thousand + * + * The fraction of each CPU's time that may be consumed running tasks + * from its rescue DSQ. A higher bandwidth admits and escalates rescues + * faster, see @rescue_quantum_us. + * + * Only the root scheduler's value is used. 0 means the default of 20 + * (2%). May not exceed 250 (25%). %SCX_RESCUE_DISABLE disables rescue - + * %SCX_ENQ_RESCUE inserts are then rejected like any other insert + * lacking the caps. + */ + u32 rescue_bandwidth_ppt; + + /** + * @rescue_quantum_us: Rescue execution quantum in microseconds + * + * How much CPU time each rescue gets. Rescues run one at a time per CPU + * and admissions are paced to keep rescue execution within + * @rescue_bandwidth_ppt - with the defaults, one 5ms rescue every + * 250ms. A crowded queue round-robins on the quantum divided across the + * waiters, floored at 1ms. A stuck rescue eventually escalates to + * forced execution. A larger quantum interrupts the CPU less often but + * for longer and spaces rescues further apart. + * + * Only the root scheduler's value is used. 0 means the default (5000). + * Non-zero values must be within [1000, 100000]. Values too short for + * the kernel to meter are lifted silently. + */ + u32 rescue_quantum_us; + /** * @sub_cgroup_id: When >1, attach the scheduler as a sub-scheduler * on the specified cgroup. @@ -1058,6 +1089,8 @@ struct sched_ext_ops_cid { u32 exit_dump_len; u64 hotplug_seq; u32 cid_shard_size; + u32 rescue_bandwidth_ppt; + u32 rescue_quantum_us; u64 sub_cgroup_id; char name[SCX_OPS_NAME_LEN]; @@ -1211,6 +1244,12 @@ struct scx_event_stats { * sub-sched lacked SCX_CAP_PERF on the target cid. */ s64 SCX_EV_SUB_CIDPERF_DENIED; + + /* + * The number of times an insert carrying %SCX_ENQ_RESCUE lacked the + * caps for its cid and the task entered the rescue path. + */ + s64 SCX_EV_SUB_RESCUE; }; #define SCX_EVENTS_LIST(SCX_EVENT) \ @@ -1233,7 +1272,8 @@ struct scx_event_stats { SCX_EVENT(SCX_EV_SUB_PREEMPT_DENIED); \ SCX_EVENT(SCX_EV_SUB_KICK_DENIED); \ SCX_EVENT(SCX_EV_SUB_REENQ_DENIED); \ - SCX_EVENT(SCX_EV_SUB_CIDPERF_DENIED) + SCX_EVENT(SCX_EV_SUB_CIDPERF_DENIED); \ + SCX_EVENT(SCX_EV_SUB_RESCUE) struct scx_sched; @@ -1656,6 +1696,17 @@ enum scx_enq_flags { */ SCX_ENQ_IMMED = 1LLU << 33, + /* + * Only allowed on local DSQs. If the insert lacks the caps for the + * target cid, divert the task to the CPU's rescue path instead of + * rejecting and reenqueueing, e.g. when the task's affinity is + * restricted to cids the scheduler doesn't hold. The kernel runs + * rescued tasks on the target CPU. Rescue execution is guaranteed to + * make forward progress and is bandwidth-limited, see the + * rescue_bandwidth_ppt and rescue_quantum_us ops fields. + */ + SCX_ENQ_RESCUE = 1LLU << 34, + /* * The task being enqueued was previously enqueued on a DSQ, but was * removed and is being re-enqueued. See SCX_TASK_REENQ_* flags to find @@ -1973,6 +2024,7 @@ void scx_task_iter_unlock(struct scx_task_iter *iter); void scx_task_iter_stop(struct scx_task_iter *iter); struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter); bool scx_set_task_slice(struct task_struct *p, u64 slice); +void scx_task_slice_ended(struct rq *rq, struct task_struct *p); void scx_task_unlink_from_dsq(struct task_struct *p, struct scx_dispatch_q *dsq); void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p); void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, @@ -2371,6 +2423,7 @@ static inline struct scx_sched *scx_parent(struct scx_sched *sch) else return NULL; } + #else /* CONFIG_EXT_SUB_SCHED */ static inline bool scx_has_subs(void) { return false; } @@ -2402,6 +2455,7 @@ static inline struct scx_sched *scx_prog_sched(const struct bpf_prog_aux *aux) } static inline struct scx_sched *scx_parent(struct scx_sched *sch) { return NULL; } + #endif /* CONFIG_EXT_SUB_SCHED */ #endif /* _KERNEL_SCHED_EXT_INTERNAL_H */ diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index c30f48ee07f9..3c1f11268e7f 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -28,6 +28,11 @@ */ DEFINE_STATIC_KEY_FALSE(__scx_has_subs); +/* latched at root enable before any rescue runs */ +static s32 scx_rescue_bw_1024; +static s64 scx_rescue_quantum_ns; +static s64 scx_rescue_sat_delta_ns; + /** * scx_skip_subtree_pre - Skip @pos's subtree in a pre-order walk * @pos: current position @@ -229,18 +234,350 @@ void scx_init_root_caps(struct scx_sched *sch) } } +/* unserved remainder of @rq's rescuee's admitted slice, 0 once fully served */ +static s64 scx_rescue_slice_remaining(struct rq *rq) +{ + s64 served = rq->scx.rescue.curr->se.sum_exec_runtime - rq->scx.rescue.exec_snap; + + return max(rq->scx.rescue.slice - served, 0); +} + +/** + * scx_rescue_charge - Charge the rescuee's runtime + * @rq: rq the rescuee is running on + * @delta_exec: runtime being charged + * + * Also ends the rescue once the admitted slice has been served in full. Ending + * on served time rather than slice exhaustion bounds both the rescue and the + * charging when a scheduler extends the rescuee's slice. + */ +void scx_rescue_charge(struct rq *rq, s64 delta_exec) +{ + lockdep_assert_rq_held(rq); + + /* + * A rescue slice is bounded by one quantum and tick-driven expiry can + * overshoot by up to a tick. Clamp to avoid wild over-charges on VMs. + */ + delta_exec = min_t(s64, delta_exec, scx_rescue_quantum_ns + TICK_NSEC); + + rq->scx.rescue.budget -= delta_exec; + + if (!scx_rescue_slice_remaining(rq)) + scx_task_slice_ended(rq, rq->scx.rescue.curr); +} + /** - * scx_resolve_local_dsq - Pick the local or reject DSQ for an insert + * scx_rescue_end - End the rescue execution on @rq + * @rq: rq of interest + * + * When no rescuee is left pending, the session is over and the balance above + * one quantum dies with it - it would otherwise become a banked license to + * preempt the cid owner long after the starvation ended. While waiters remain, + * the accrued deficit belongs to the queue and carries into the next rescue. + */ +void scx_rescue_end(struct rq *rq) +{ + lockdep_assert_rq_held(rq); + + rq->scx.rescue.curr = NULL; + if (list_empty(&rq->scx.rescue.dsq.list)) + rq->scx.rescue.budget = min(rq->scx.rescue.budget, scx_rescue_quantum_ns); +} + +/** + * scx_rescue_keep - Keep the rescue going for a preempted-out rescuee + * @rq: rq @p is running on + * @p: task under rescue whose slice is exhausted + * + * Called from put_prev_task_scx() to decide what an exhausted slice means for + * the rescuee. scx_rescue_charge() ends the rescue the moment the admitted + * slice is fully served, so arriving here with the rescue still open means @p + * was preempted. Restore the unserved remainder and return %true - @p stays the + * rescuee and the caller reinserts it at the tail of the local DSQ, behind + * whatever preempted the rescuee. + * + * Return %false to end the rescue instead - the slice is already fully served, + * @p is leaving the rq or bypass is dismantling rescues. + */ +bool scx_rescue_keep(struct rq *rq, struct task_struct *p) +{ + s64 remaining = scx_rescue_slice_remaining(rq); + + lockdep_assert_rq_held(rq); + + if (!remaining || !(p->scx.flags & SCX_TASK_QUEUED) || + scx_bypassing(scx_task_sched(p), cpu_of(rq))) + return false; + + scx_set_task_slice(p, remaining); + return true; +} + +/** + * scx_rescue_accrue - Accrue budget at the configured fraction of elapsed time + * @rq: rq of interest + * + * A session spans from the first arrival until no rescuee is left, pending or + * admitted. While one is active the cap is three quanta and the balance drives + * escalation, see scx_rescue_timerfn(). Outside a session the cap is one + * quantum, so an idle gap funds the next arrival's admission but never an + * escalation. + */ +static void scx_rescue_accrue(struct rq *rq) +{ + bool in_session = rq->scx.rescue.curr || !list_empty(&rq->scx.rescue.dsq.list); + s64 cap = in_session ? 3 * scx_rescue_quantum_ns : scx_rescue_quantum_ns; + s64 delta; + u64 now; + + lockdep_assert_rq_held(rq); + + /* not every path here holds an updated rq clock, use __scx_bpf_now() */ + now = __scx_bpf_now(rq); + delta = now - rq->scx.rescue.clock; + rq->scx.rescue.clock = now; + + /* + * Avoid multiplication overflows by taking a shortcut when the gap is + * large enough to fill the budget. + */ + if (delta >= scx_rescue_sat_delta_ns) + rq->scx.rescue.budget = cap; + else + rq->scx.rescue.budget = + min(cap, rq->scx.rescue.budget + + ((delta * scx_rescue_bw_1024) >> SCHED_CAPACITY_SHIFT)); +} + +/* + * The slice for the next admission - the quantum divided across the stranded + * tasks so that a crowded queue round-robins on shorter slices. + */ +static s64 scx_rescue_next_slice(struct rq *rq) +{ + s64 min_slice = max_t(s64, SCX_RESCUE_MIN_SLICE_US * NSEC_PER_USEC, TICK_NSEC); + u32 depth = rq->scx.rescue.dsq.nr ?: 1; + + return clamp(div_s64(scx_rescue_quantum_ns, depth), min_slice, scx_rescue_quantum_ns); +} + +static void scx_rescue_timer_arm(struct rq *rq) +{ + struct timer_list *timer = &rq->scx.rescue.timer; + s64 delay = scx_rescue_quantum_ns / 4; /* should be granular enough */ + + if (timer_pending(timer)) + return; + + /* + * While the head waiter can't be admitted because the bucket is short + * of a full quantum, stretch to the full funding delay. + */ + if (!rq->scx.rescue.curr && rq->scx.rescue.budget < scx_rescue_quantum_ns) { + s64 deficit = scx_rescue_quantum_ns - rq->scx.rescue.budget; + + delay = max(delay, + div_s64(deficit << SCHED_CAPACITY_SHIFT, scx_rescue_bw_1024)); + } + + /* +1 rounds up so the beat is due by the time the timer fires */ + timer->expires = jiffies + nsecs_to_jiffies(delay) + 1; + add_timer_on(timer, cpu_of(rq)); +} + +/** + * scx_rescue_admit - Start rescuing @p on @rq + * @rq: rq @p is being admitted on + * @p: task being admitted, off any DSQ + * @slice: CPU time to grant + * + * The schedulers keep their normal control over @p and may preempt or reslice + * it. @slice is measured on served CPU time against the snapshot taken here, so + * neither shortens the rescue, see scx_rescue_charge() and scx_rescue_keep(). + * Prolonged denial escalates into protected execution, see + * scx_rescue_timerfn(). + */ +static void scx_rescue_admit(struct rq *rq, struct task_struct *p, s64 slice) +{ + lockdep_assert_rq_held(rq); + WARN_ON_ONCE(rq->scx.rescue.curr); + + rq->scx.rescue.curr = p; + rq->scx.rescue.slice = slice; + rq->scx.rescue.exec_snap = p->se.sum_exec_runtime; + scx_set_task_slice(p, slice); + scx_rescue_timer_arm(rq); +} + +/** + * scx_rescue_try_admit - Try to admit a freshly stranded task + * @rq: rq @p is being inserted on + * @p: stranded task being diverted to rescue + * + * One rescue at a time and earlier arrivals go first. Admission needs a full + * quantum of budget, spent as the rescue runs. Return %true if @p was admitted + * and should be inserted at the tail of @rq's local DSQ, %false if it has to + * park on the rescue DSQ, with the timer armed to admit it later. + */ +static bool scx_rescue_try_admit(struct rq *rq, struct task_struct *p) +{ + scx_rescue_accrue(rq); + + if (!rq->scx.rescue.curr && list_empty(&rq->scx.rescue.dsq.list) && + rq->scx.rescue.budget >= scx_rescue_quantum_ns) { + scx_rescue_admit(rq, p, scx_rescue_quantum_ns); + return true; + } + + scx_rescue_timer_arm(rq); + return false; +} + +/** + * scx_rescue_timerfn - Drive and pace rescue execution + * @timer: rq->scx.rescue.timer + * + * Runs every quarter quantum while a rescuee exists, pending or admitted, see + * scx_rescue_timer_arm(). The head waiter is admitted once the bucket holds a + * full quantum and granted its slice, see scx_rescue_next_slice(). A session + * whose budget accumulates over two quanta with the admitted rescuee still + * waiting escalates - the rescuee's remaining slice turns into protected + * execution and it preempts the current task. + */ +static void scx_rescue_timerfn(struct timer_list *timer) +{ + struct rq *rq = timer_container_of(rq, timer, scx.rescue.timer); + struct task_struct *p; + + guard(rq_lock_irqsave)(rq); + + p = rq->scx.rescue.curr; + if (!p && list_empty(&rq->scx.rescue.dsq.list)) + return; + + scx_rescue_accrue(rq); + + if (!p) { + s64 slice = scx_rescue_next_slice(rq); + + /* no rescue in progress */ + if (rq->scx.rescue.budget < scx_rescue_quantum_ns) + goto out_arm; + + /* there's enough budget to start rescuing the next one */ + p = list_first_entry(&rq->scx.rescue.dsq.list, struct task_struct, + scx.dsq_list.node); + scx_task_unlink_from_dsq(p, &rq->scx.rescue.dsq); + scx_rescue_admit(rq, p, slice); + scx_move_local_task_to_local_dsq(scx_task_sched(p), p, SCX_ENQ_IGNORE_CAPS, + &rq->scx.rescue.dsq, rq); + if (sched_class_above(&ext_sched_class, rq->curr->sched_class)) + resched_curr(rq); + } else if (p->scx.dsq && rq->scx.rescue.budget > 2 * scx_rescue_quantum_ns) { + /* + * The rescuee waited for the CPU for too long. Escalate - grant + * the unserved remainder, protect it from the schedulers and + * preempt the current task. The slice is set before the + * protection. Repeat beats only repeat the head move - the + * slice write is refused on a protected task. + */ + scx_set_task_slice(p, scx_rescue_slice_remaining(rq)); + p->scx.flags |= SCX_TASK_PROTECTED; + scx_task_unlink_from_dsq(p, &rq->scx.local_dsq); + scx_move_local_task_to_local_dsq(scx_task_sched(p), p, + SCX_ENQ_HEAD | SCX_ENQ_PREEMPT | SCX_ENQ_IGNORE_CAPS, + &rq->scx.local_dsq, rq); + } +out_arm: + scx_rescue_timer_arm(rq); +} + +/* flush out tasks waiting for rescue before a CPU goes down */ +void scx_rescue_flush(struct rq *rq) +{ + struct task_struct *p, *n; + + lockdep_assert_rq_held(rq); + + /* sched domain rebuilds call rq_offline with the CPU staying alive */ + if (cpu_active(cpu_of(rq))) + return; + + /* end the current rescue */ + if (rq->scx.rescue.curr) + scx_task_slice_ended(rq, rq->scx.rescue.curr); + + /* and flush out all pending ones */ + list_for_each_entry_safe(p, n, &rq->scx.rescue.dsq.list, scx.dsq_list.node) { + scx_task_unlink_from_dsq(p, &rq->scx.rescue.dsq); + scx_move_local_task_to_local_dsq(scx_task_sched(p), p, SCX_ENQ_IGNORE_CAPS, + &rq->scx.rescue.dsq, rq); + } + + timer_delete(&rq->scx.rescue.timer); +} + +void scx_rescue_dump(struct seq_buf *s, struct rq *rq) +{ + struct task_struct *p = rq->scx.rescue.curr; + + scx_dump_line(s, " rescue=%u budget=%lldus rescuing=%s[%d]", + rq->scx.rescue.dsq.nr, + div_s64(rq->scx.rescue.budget, NSEC_PER_USEC), + p ? p->comm : "none", p ? p->pid : -1); +} + +/* latch the rescue parameters on root scheduler enable */ +void scx_rescue_set_knobs(struct scx_sched *sch) +{ + s32 bw_ppt = sch->ops.rescue_bandwidth_ppt ?: SCX_RESCUE_DFL_BW_PPT; + s64 quantum_us = sch->ops.rescue_quantum_us ?: SCX_RESCUE_DFL_QUANTUM_US; + + if (sch->ops.rescue_bandwidth_ppt == SCX_RESCUE_DISABLE) { + scx_rescue_bw_1024 = 0; + return; + } + + scx_rescue_bw_1024 = bw_ppt * SCHED_CAPACITY_SCALE / 1000; + scx_rescue_quantum_ns = max(quantum_us * NSEC_PER_USEC, TICK_NSEC); + scx_rescue_sat_delta_ns = + div_s64((4 * scx_rescue_quantum_ns + TICK_NSEC) << SCHED_CAPACITY_SHIFT, + scx_rescue_bw_1024); + + /* + * A rescued task is guaranteed to run after two full periods - one to + * be admitted, one more to escalate. Require the two periods to fit in + * a quarter of the watchdog timeout, so one full period may take at + * most an eighth. + */ + if (div_s64(scx_rescue_quantum_ns << SCHED_CAPACITY_SHIFT, scx_rescue_bw_1024) > + jiffies_to_nsecs(sch->watchdog_timeout) / 8) + pr_warn("sched_ext: rescue may not run a stuck task before the %ums watchdog timeout, decrease rescue_quantum_us or increase rescue_bandwidth_ppt\n", + jiffies_to_msecs(sch->watchdog_timeout)); +} + +void scx_rescue_init(struct rq *rq) +{ + BUG_ON(scx_init_dsq(&rq->scx.rescue.dsq, SCX_DSQ_RESCUE, NULL)); + timer_setup(&rq->scx.rescue.timer, scx_rescue_timerfn, TIMER_PINNED); +} + +/** + * scx_resolve_local_dsq - Pick the local, rescue or reject DSQ for an insert * @sch: enqueuing sub-sched * @rq: rq whose local DSQ @p targets * @p: task being inserted * @enq_flags: in/out, unhonored flags are cleared * - * Return @rq's local DSQ if @sch holds the required caps on @rq's cid, - * otherwise @rq's reject DSQ after recording the reenq reason on @p. + * Return @rq's local DSQ if @sch holds the required caps on @rq's cid. + * Otherwise, return @rq's rescue DSQ if the insert carries %SCX_ENQ_RESCUE and + * rescue is enabled, or @rq's reject DSQ after recording the reenq reason on + * @p. * - * %SCX_ENQ_IMMED and %SCX_ENQ_PREEMPT are cleared when diverting to reject. - * %SCX_ENQ_PREEMPT is also cleared on a fallback migration-disabled admission. + * %SCX_ENQ_IMMED, %SCX_ENQ_PREEMPT and %SCX_ENQ_HEAD are cleared when diverting + * to rescue or reject. %SCX_ENQ_PREEMPT is also cleared on a fallback + * migration-disabled admission. * * Bypass doesn't need special-casing as a bypassing sched's tasks are enqueued * to and run by its nearest non-bypassing ancestor. If root is bypassing, it @@ -282,18 +619,27 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r return &rq->scx.local_dsq; } - p->scx.reenq_reason_caps = missing; - p->scx.reenq_reason_cid = cid; - /* - * Only local DSQ can honor IMMED and dsq_inc_nr() WARNs on IMMED into - * others. Strip both the enq flag and the sticky task flag - the - * latter can carry in from an earlier admitted IMMED insert. Strip - * PREEMPT too. + * Diverting to rescue or reject, neither of which honors IMMED, PREEMPT + * or HEAD - a diversion has no priority and IMMED is not allowed on + * non-local DSQs. Strip the enq and task flags along with the slice. */ - *enq_flags &= ~(SCX_ENQ_IMMED | SCX_ENQ_PREEMPT); + *enq_flags &= ~(SCX_ENQ_IMMED | SCX_ENQ_PREEMPT | SCX_ENQ_HEAD | + SCX_ENQ_APPLY_SLICE | SCX_ENQ_SLICE_DFL); p->scx.flags &= ~SCX_TASK_IMMED; + /* the enqueuer opted for rescue instead of rejection and reenqueue */ + if ((*enq_flags & SCX_ENQ_RESCUE) && likely(scx_rescue_bw_1024)) { + __scx_add_event(sch, SCX_EV_SUB_RESCUE, 1); + if (scx_rescue_try_admit(rq, p)) + return &rq->scx.local_dsq; + else + return &rq->scx.rescue.dsq; + } + + p->scx.reenq_reason_caps = missing; + p->scx.reenq_reason_cid = cid; + return &rq->scx.reject_dsq; } @@ -302,8 +648,8 @@ bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p) { u64 missing; - /* migration-disabled tasks are admitted regardless of caps */ - if (is_migration_disabled(p)) + /* migration-disabled tasks and the rescuee are admitted capless */ + if (is_migration_disabled(p) || p == scx_rescuee(rq)) return false; missing = scx_missing_caps(scx_task_sched(p), cpu_of(rq), scx_caps_for_task(p)); diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index fe1d82e6c1d5..f7bcdfda8dd8 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -38,6 +38,13 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r struct task_struct *p, u64 *enq_flags); bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p); void scx_reenq_reject(struct rq *rq); +void scx_rescue_charge(struct rq *rq, s64 delta_exec); +void scx_rescue_end(struct rq *rq); +bool scx_rescue_keep(struct rq *rq, struct task_struct *p); +void scx_rescue_flush(struct rq *rq); +void scx_rescue_dump(struct seq_buf *s, struct rq *rq); +void scx_rescue_set_knobs(struct scx_sched *sch); +void scx_rescue_init(struct rq *rq); /* * cgrp->scx_sched is written by root/sub enable/disable under all of @@ -87,6 +94,13 @@ static inline void scx_discard_stale_ecaps_syncs(void) {} static inline struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, struct task_struct *p, u64 *enq_flags) { return &rq->scx.local_dsq; } static inline bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p) { return false; } static inline void scx_reenq_reject(struct rq *rq) {} +static inline void scx_rescue_charge(struct rq *rq, s64 delta_exec) {} +static inline void scx_rescue_end(struct rq *rq) {} +static inline bool scx_rescue_keep(struct rq *rq, struct task_struct *p) { return false; } +static inline void scx_rescue_flush(struct rq *rq) {} +static inline void scx_rescue_dump(struct seq_buf *s, struct rq *rq) {} +static inline void scx_rescue_set_knobs(struct scx_sched *sch) {} +static inline void scx_rescue_init(struct rq *rq) {} static inline void scx_dec_has_subs(struct scx_sched *sch) {} #endif /* CONFIG_EXT_SUB_SCHED */ @@ -195,11 +209,23 @@ static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p return likely(!scx_missing_caps(scx_task_sched(p), cpu_of(rq), SCX_CAP_BASE)); } +/* the task admitted for rescue on @rq, NULL if none */ +static inline struct task_struct *scx_rescuee(struct rq *rq) +{ + lockdep_assert_rq_held(rq); + + if (!scx_has_subs()) + return NULL; + + return rq->scx.rescue.curr; +} + #else /* CONFIG_EXT_SUB_SCHED */ static inline u64 scx_missing_caps(struct scx_sched *sch, s32 cpu, u64 needed) { return 0; } static inline u64 scx_caps_for_preempt(struct scx_sched *sch, struct rq *rq, u64 enq_flags) { return 0; } static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p) { return true; } +static inline struct task_struct *scx_rescuee(struct rq *rq) { return NULL; } #endif /* CONFIG_EXT_SUB_SCHED */ diff --git a/kernel/sched/ext/types.h b/kernel/sched/ext/types.h index b94ddee21c57..d39588717e9b 100644 --- a/kernel/sched/ext/types.h +++ b/kernel/sched/ext/types.h @@ -19,6 +19,15 @@ enum scx_consts { SCX_DSP_MAX_LOOPS = 32, SCX_WATCHDOG_MAX_TIMEOUT = 30 * HZ, + /* rescue knob defaults and limits, see scx_rescue_timerfn() */ + SCX_RESCUE_DFL_BW_PPT = 20, /* parts per thousand, 2% */ + SCX_RESCUE_MAX_BW_PPT = 250, /* 25% */ + SCX_RESCUE_DISABLE = U32_MAX, /* disables rescue */ + SCX_RESCUE_DFL_QUANTUM_US = 5000, + SCX_RESCUE_MIN_QUANTUM_US = 1000, + SCX_RESCUE_MAX_QUANTUM_US = 100000, + SCX_RESCUE_MIN_SLICE_US = 1000, /* floor of the divided slice */ + /* per-CPU chunk size for p->scx.tid allocation, see scx_alloc_tid() */ SCX_TID_CHUNK = 1024, diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index c0cb879d75f0..289e298df628 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -794,10 +794,22 @@ enum scx_rq_flags { SCX_RQ_IN_BALANCE = 1 << 17, }; +/* per-rq rescue execution state, see scx_rescue_timerfn() */ +struct scx_rq_rescue { + struct scx_dispatch_q dsq; /* stranded tasks awaiting rescue */ + s64 budget; /* execution token bucket, ns */ + u64 clock; /* last budget accrual timestamp */ + struct task_struct *curr; /* task being rescued, one at a time */ + s64 slice; /* curr's admitted slice */ + u64 exec_snap; /* sum_exec_runtime at admission */ + struct timer_list timer; /* paces admission and escalation */ +}; + struct scx_rq { struct scx_dispatch_q local_dsq; #ifdef CONFIG_EXT_SUB_SCHED struct scx_dispatch_q reject_dsq; /* staging for cap-rejected tasks */ + struct scx_rq_rescue rescue; #endif struct list_head runnable_list; /* runnable tasks on this rq */ struct list_head ddsp_deferred_locals; /* deferred ddsps from enq */ diff --git a/tools/sched_ext/include/scx/compat.h b/tools/sched_ext/include/scx/compat.h index 7757252d52e2..d2e4384df5af 100644 --- a/tools/sched_ext/include/scx/compat.h +++ b/tools/sched_ext/include/scx/compat.h @@ -175,6 +175,7 @@ static inline long scx_hotplug_seq(void) * - v6.17: ops.cgroup_set_bandwidth() * - v6.19: ops.cgroup_set_idle() * - v7.1: ops.sub_attach(), ops.sub_detach(), ops.sub_cgroup_id + * - v7.3: ops.rescue_bandwidth_ppt, ops.rescue_quantum_us */ #define __SCX_OPS_OPEN(__ops_name, __scx_name, __ops_struct) ({ \ struct __scx_name *__oskel; \ @@ -218,6 +219,16 @@ static inline long scx_hotplug_seq(void) fprintf(stderr, "WARNING: kernel doesn't support ops.sub_cgroup_id\n"); \ __skel->struct_ops.__ops_name->sub_cgroup_id = 0; \ } \ + if (__skel->struct_ops.__ops_name->rescue_bandwidth_ppt > 0 && \ + !__COMPAT_struct_has_field("sched_ext_ops", "rescue_bandwidth_ppt")) { \ + fprintf(stderr, "WARNING: kernel doesn't support ops.rescue_bandwidth_ppt\n"); \ + __skel->struct_ops.__ops_name->rescue_bandwidth_ppt = 0; \ + } \ + if (__skel->struct_ops.__ops_name->rescue_quantum_us > 0 && \ + !__COMPAT_struct_has_field("sched_ext_ops", "rescue_quantum_us")) { \ + fprintf(stderr, "WARNING: kernel doesn't support ops.rescue_quantum_us\n"); \ + __skel->struct_ops.__ops_name->rescue_quantum_us = 0; \ + } \ __skel; \ }) -- cgit v1.2.3 From bb70e4fb626b70895b7917ee97c256f24d019c34 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 3 Aug 2026 11:01:36 -1000 Subject: sched_ext: Eject the top rescue consumer on overload When rescue demand on a cpu persistently exceeds the configured bandwidth, tasks age on that cpu's rescue DSQ until the stall watchdog fires. The watchdog blames the waiting task's owner, but the misbehaving party is whoever floods the queue, not whoever happens to time out. Track each sched's recent rescue consumption per cpu as a decaying average. Once the oldest waiter on a cpu's rescue DSQ has been queued past a threshold derived from the rescue knobs (4s at the defaults), the rescue timer ejects the sub with the highest recent consumption on that cpu with SCX_EXIT_ERROR_RESCUE. With no recent consumer there is no victim and nothing is ejected - the generic stall watchdog eventually blames the waiter's owner instead. Ejections on a cpu are spaced one threshold apart so the freed bandwidth can drain the backlog before another sub is judged. The overload check only wins the race against the stall watchdog when the watchdog timeout clears the threshold, and a single in-budget wait must not cross the trigger on its own. Warn on a scheduler whose timeout doesn't fit and on knobs whose funding period exceeds half the threshold. v2: - Track kill_at in jiffies_64 - on 32-bit, the time_before() grace check wraps 2^31 ticks after the last ejection and suppresses ejections. (sashiko AI) - Track rescue_avg_at in jiffies_64 likewise - the unsigned long decay delta truncates mod 2^32 on 32-bit and can revive a weeks-old usage average in the victim pick. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- include/linux/sched/ext.h | 3 + kernel/sched/ext/ext.c | 2 + kernel/sched/ext/internal.h | 9 +++ kernel/sched/ext/sub.c | 137 ++++++++++++++++++++++++++++++++++++++++---- kernel/sched/ext/types.h | 3 + kernel/sched/sched.h | 1 + 6 files changed, 144 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index a6aabbefd185..a3ec980e2925 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -215,6 +215,9 @@ struct sched_ext_entity { #ifdef CONFIG_SCHED_CORE u64 core_sched_at; /* see scx_prio_less() */ #endif +#ifdef CONFIG_EXT_SUB_SCHED + unsigned long rescue_at; /* queued on a rescue DSQ at, jiffies */ +#endif /* * Unique non-zero task ID assigned at fork. Persists across exec and diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 7f86d4adcd0d..0bbe144c9811 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -6140,6 +6140,8 @@ static const char *scx_exit_reason(enum scx_exit_kind kind) return "runnable task stall"; case SCX_EXIT_ERROR_REENQ: return "reenqueue limit"; + case SCX_EXIT_ERROR_RESCUE: + return "rescue bandwidth overload"; default: return ""; } diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 18983dbe81f4..b699e7c1103f 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -57,6 +57,7 @@ enum scx_exit_kind { SCX_EXIT_ERROR_BPF, /* ERROR but triggered through scx_bpf_error() */ SCX_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */ SCX_EXIT_ERROR_REENQ, /* task hit reenqueue limit without running */ + SCX_EXIT_ERROR_RESCUE, /* ejected for overloading rescue execution */ }; /* @@ -1340,6 +1341,14 @@ struct scx_sched_pcpu { bool idle_renotify; /* effective caps as of the last sub_ecaps_updated() delivery */ u64 reported_ecaps; + + /* + * Decaying rescue runtime consumed on this cpu, see + * scx_rescue_decay_avg(). Overload on this cpu ejects the sub with the + * largest value. Accessed only under this cpu's rq lock. + */ + u64 rescue_avg; + u64 rescue_avg_at; /* last decay, jiffies_64 */ #endif /* diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 3c1f11268e7f..b81254be1b04 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -32,6 +32,8 @@ DEFINE_STATIC_KEY_FALSE(__scx_has_subs); static s32 scx_rescue_bw_1024; static s64 scx_rescue_quantum_ns; static s64 scx_rescue_sat_delta_ns; +static unsigned long scx_rescue_decay_halflife; +static unsigned long scx_rescue_overload_after; /** * scx_skip_subtree_pre - Skip @pos's subtree in a pre-order walk @@ -242,6 +244,23 @@ static s64 scx_rescue_slice_remaining(struct rq *rq) return max(rq->scx.rescue.slice - served, 0); } +/* + * Decay @pcpu's rescue usage average in place, halving per the knob-derived + * halflife, see scx_rescue_set_knobs(). The timestamp advances only by whole + * halflives. + */ +static u64 scx_rescue_decay_avg(struct scx_sched_pcpu *pcpu) +{ + unsigned long halflife = scx_rescue_decay_halflife; + u64 n = div_u64(get_jiffies_64() - pcpu->rescue_avg_at, halflife); + + if (n) { + pcpu->rescue_avg = n < 64 ? pcpu->rescue_avg >> n : 0; + pcpu->rescue_avg_at += n * halflife; + } + return pcpu->rescue_avg; +} + /** * scx_rescue_charge - Charge the rescuee's runtime * @rq: rq the rescuee is running on @@ -253,6 +272,8 @@ static s64 scx_rescue_slice_remaining(struct rq *rq) */ void scx_rescue_charge(struct rq *rq, s64 delta_exec) { + struct scx_sched_pcpu *pcpu; + lockdep_assert_rq_held(rq); /* @@ -263,6 +284,10 @@ void scx_rescue_charge(struct rq *rq, s64 delta_exec) rq->scx.rescue.budget -= delta_exec; + /* per-cpu usage average feeds the overload victim pick */ + pcpu = per_cpu_ptr(scx_task_sched(rq->curr)->pcpu, cpu_of(rq)); + pcpu->rescue_avg = scx_rescue_decay_avg(pcpu) + delta_exec; + if (!scx_rescue_slice_remaining(rq)) scx_task_slice_ended(rq, rq->scx.rescue.curr); } @@ -434,6 +459,64 @@ static bool scx_rescue_try_admit(struct rq *rq, struct task_struct *p) return false; } +/** + * scx_rescue_check_overload - Eject the top rescue consumer on a stuck rescue + * @rq: rq whose rescue timer fired + * + * If the oldest waiter on @rq's rescue DSQ has been queued for too long, rescue + * demand on this cpu persistently exceeds the configured bandwidth. Eject the + * sub with the highest recent rescue consumption instead of letting the + * scheduler stall path blame the waiter's owner, who may just be crowded out. + */ +static void scx_rescue_check_overload(struct rq *rq) +{ + struct scx_sched *victim = NULL, *pos; + struct task_struct *p; + int cpu = cpu_of(rq); + u64 max_avg = 0; + u32 dur_ms; + + lockdep_assert_rq_held(rq); + + p = list_first_entry_or_null(&rq->scx.rescue.dsq.list, struct task_struct, + scx.dsq_list.node); + if (!p) + return; + + /* has the head waiter been queued for longer than the threshold? */ + if (time_before(jiffies, p->scx.rescue_at + scx_rescue_overload_after)) + return; + + /* + * Grace period after the last ejection on this cpu - the freed + * bandwidth gets one threshold's worth of time to drain the backlog + * before another sub is judged. + */ + if (time_before64(get_jiffies_64(), rq->scx.rescue.kill_at + + scx_rescue_overload_after)) + return; + + list_for_each_entry_rcu(pos, &scx_sched_all, all) { + u64 avg = scx_rescue_decay_avg(per_cpu_ptr(pos->pcpu, cpu)); + + /* skip an already-exiting sub, else the ejection is wasted */ + if (pos->level && avg > max_avg && + atomic_read(&pos->exit_kind) == SCX_EXIT_NONE) { + max_avg = avg; + victim = pos; + } + } + if (!victim) + return; + + rq->scx.rescue.kill_at = get_jiffies_64(); + dur_ms = jiffies_to_msecs(jiffies - p->scx.rescue_at); + __scx_exit(victim, SCX_EXIT_ERROR_RESCUE, 0, cpu, + "used too much rescue CPU time (%llums) while %s[%d] waited %u.%03us to be rescued", + div_u64(max_avg, NSEC_PER_MSEC), p->comm, p->pid, dur_ms / 1000, + dur_ms % 1000); +} + /** * scx_rescue_timerfn - Drive and pace rescue execution * @timer: rq->scx.rescue.timer @@ -443,7 +526,8 @@ static bool scx_rescue_try_admit(struct rq *rq, struct task_struct *p) * full quantum and granted its slice, see scx_rescue_next_slice(). A session * whose budget accumulates over two quanta with the admitted rescuee still * waiting escalates - the rescuee's remaining slice turns into protected - * execution and it preempts the current task. + * execution and it preempts the current task. An overloaded rescue queue ejects + * the top consumer, see scx_rescue_check_overload(). */ static void scx_rescue_timerfn(struct timer_list *timer) { @@ -457,6 +541,7 @@ static void scx_rescue_timerfn(struct timer_list *timer) return; scx_rescue_accrue(rq); + scx_rescue_check_overload(rq); if (!p) { s64 slice = scx_rescue_next_slice(rq); @@ -528,11 +613,28 @@ void scx_rescue_dump(struct seq_buf *s, struct rq *rq) p ? p->comm : "none", p ? p->pid : -1); } +/* + * A scheduler whose stall watchdog is shorter than the overload threshold gets + * stall-killed over its parked waiters before the overload check can eject the + * actual top consumer. The root's knobs set the threshold, warn on any + * scheduler that doesn't fit it. + */ +static void scx_rescue_check_timeout(struct scx_sched *sch) +{ + if (!scx_rescue_bw_1024 || sch->watchdog_timeout > scx_rescue_overload_after) + return; + + pr_warn("sched_ext: %s: watchdog timeout %ums <= rescue overload threshold %ums\n", + sch->ops.name, jiffies_to_msecs(sch->watchdog_timeout), + jiffies_to_msecs(scx_rescue_overload_after)); +} + /* latch the rescue parameters on root scheduler enable */ void scx_rescue_set_knobs(struct scx_sched *sch) { s32 bw_ppt = sch->ops.rescue_bandwidth_ppt ?: SCX_RESCUE_DFL_BW_PPT; s64 quantum_us = sch->ops.rescue_quantum_us ?: SCX_RESCUE_DFL_QUANTUM_US; + s64 period_ns; if (sch->ops.rescue_bandwidth_ppt == SCX_RESCUE_DISABLE) { scx_rescue_bw_1024 = 0; @@ -546,21 +648,30 @@ void scx_rescue_set_knobs(struct scx_sched *sch) scx_rescue_bw_1024); /* - * A rescued task is guaranteed to run after two full periods - one to - * be admitted, one more to escalate. Require the two periods to fit in - * a quarter of the watchdog timeout, so one full period may take at - * most an eighth. + * The overload threshold and the decay halflife scale with the funding + * period - the time the bucket takes to fund one full quantum. */ - if (div_s64(scx_rescue_quantum_ns << SCHED_CAPACITY_SHIFT, scx_rescue_bw_1024) > - jiffies_to_nsecs(sch->watchdog_timeout) / 8) - pr_warn("sched_ext: rescue may not run a stuck task before the %ums watchdog timeout, decrease rescue_quantum_us or increase rescue_bandwidth_ppt\n", - jiffies_to_msecs(sch->watchdog_timeout)); + period_ns = div_s64(scx_rescue_quantum_ns << SCHED_CAPACITY_SHIFT, scx_rescue_bw_1024); + scx_rescue_overload_after = + clamp(nsecs_to_jiffies(SCX_RESCUE_OVERLOAD_MULT * period_ns), + msecs_to_jiffies(SCX_RESCUE_MIN_OVERLOAD_MS), + msecs_to_jiffies(SCX_RESCUE_MAX_OVERLOAD_MS)); + scx_rescue_decay_halflife = scx_rescue_overload_after / 4; + + /* a single in-budget wait must not cross the overload trigger */ + if (nsecs_to_jiffies(period_ns) > scx_rescue_overload_after / 2) + pr_warn("sched_ext: %s: rescue funding period %lldms > overload threshold %ums / 2\n", + sch->ops.name, div_s64(period_ns, NSEC_PER_MSEC), + jiffies_to_msecs(scx_rescue_overload_after)); + + scx_rescue_check_timeout(sch); } void scx_rescue_init(struct rq *rq) { BUG_ON(scx_init_dsq(&rq->scx.rescue.dsq, SCX_DSQ_RESCUE, NULL)); timer_setup(&rq->scx.rescue.timer, scx_rescue_timerfn, TIMER_PINNED); + rq->scx.rescue.kill_at = get_jiffies_64(); } /** @@ -633,8 +744,10 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r __scx_add_event(sch, SCX_EV_SUB_RESCUE, 1); if (scx_rescue_try_admit(rq, p)) return &rq->scx.local_dsq; - else - return &rq->scx.rescue.dsq; + + /* queueing, the overload trigger measures the wait from here */ + p->scx.rescue_at = jiffies; + return &rq->scx.rescue.dsq; } p->scx.reenq_reason_caps = missing; @@ -1654,6 +1767,8 @@ void scx_sub_enable_workfn(struct kthread_work *work) if (ret) goto err_disable; + scx_rescue_check_timeout(sch); + /* * Allocate pshard[] before scx_link_sched() publishes @sch into the * parent's RCU children list. A concurrent revoke walking the tree diff --git a/kernel/sched/ext/types.h b/kernel/sched/ext/types.h index d39588717e9b..1eb3ac8508f6 100644 --- a/kernel/sched/ext/types.h +++ b/kernel/sched/ext/types.h @@ -27,6 +27,9 @@ enum scx_consts { SCX_RESCUE_MIN_QUANTUM_US = 1000, SCX_RESCUE_MAX_QUANTUM_US = 100000, SCX_RESCUE_MIN_SLICE_US = 1000, /* floor of the divided slice */ + SCX_RESCUE_OVERLOAD_MULT = 16, /* overload threshold in funding periods */ + SCX_RESCUE_MIN_OVERLOAD_MS = 1000, + SCX_RESCUE_MAX_OVERLOAD_MS = 15000, /* per-CPU chunk size for p->scx.tid allocation, see scx_alloc_tid() */ SCX_TID_CHUNK = 1024, diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 289e298df628..63786712a115 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -803,6 +803,7 @@ struct scx_rq_rescue { s64 slice; /* curr's admitted slice */ u64 exec_snap; /* sum_exec_runtime at admission */ struct timer_list timer; /* paces admission and escalation */ + u64 kill_at; /* last ejection, init before any */ }; struct scx_rq { -- cgit v1.2.3 From 0ec5dd0669291c8ffbee096367e078c26cbcc332 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 15 Aug 2026 13:08:28 -1000 Subject: sched_ext: Use runnable_at for the default core-sched task ordering The default core-sched ordering runs the longest waiting task first by comparing p->scx.core_sched_at stamps. The stamp is maintained under two rules. touch_core_sched() stamps when a task starts waiting for a CPU and when its slice runs out. If the scheduler implements ops.core_sched_before(), touch_core_sched_dispatch() re-stamps on every dispatch. A comparison can see one stamp taken under each rule, which isn't a meaningful ordering. The dispatch rule also buys little - it only aligns bypass-mode comparisons with the local DSQ order. Multiple schedulers make the mixed comparisons more common. Wait time is what p->scx.runnable_at already tracks for the stall watchdog. Delete core_sched_at with both touch functions and compare runnable_at in the scx_prio_less() fallback. runnable_at is refreshed only on enqueue and goes stale while a task keeps occupying its CPU. Instead of re-stamping, order a running task after every waiting task as it is the most recently serviced. Signed-off-by: Tejun Heo --- include/linux/sched/ext.h | 3 -- kernel/sched/ext/ext.c | 105 +++++++++++----------------------------------- 2 files changed, 25 insertions(+), 83 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index a3ec980e2925..582d7cd4a983 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -212,9 +212,6 @@ struct sched_ext_entity { struct list_head runnable_node; /* rq->scx.runnable_list */ unsigned long runnable_at; -#ifdef CONFIG_SCHED_CORE - u64 core_sched_at; /* see scx_prio_less() */ -#endif #ifdef CONFIG_EXT_SUB_SCHED unsigned long rescue_at; /* queued on a rescue DSQ at, jiffies */ #endif diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 9014c814a00a..24663ae713a4 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1155,53 +1155,6 @@ void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, schedule_deferred(rq); } -/** - * touch_core_sched - Update timestamp used for core-sched task ordering - * @rq: rq to read clock from, must be locked - * @p: task to update the timestamp for - * - * Update @p->scx.core_sched_at timestamp. This is used by scx_prio_less() to - * implement global or local-DSQ FIFO ordering for core-sched. Should be called - * when a task becomes runnable and its turn on the CPU ends (e.g. slice - * exhaustion). - */ -static void touch_core_sched(struct rq *rq, struct task_struct *p) -{ - lockdep_assert_rq_held(rq); - -#ifdef CONFIG_SCHED_CORE - /* - * It's okay to update the timestamp spuriously. Use - * sched_core_disabled() which is cheaper than enabled(). - * - * As this is used to determine ordering between tasks of sibling CPUs, - * it may be better to use per-core dispatch sequence instead. - */ - if (!sched_core_disabled()) - p->scx.core_sched_at = sched_clock_cpu(cpu_of(rq)); -#endif -} - -/** - * touch_core_sched_dispatch - Update core-sched timestamp on dispatch - * @rq: rq to read clock from, must be locked - * @p: task being dispatched - * - * If the BPF scheduler implements custom core-sched ordering via - * ops.core_sched_before(), @p->scx.core_sched_at is used to implement FIFO - * ordering within each local DSQ. This function is called from dispatch paths - * and updates @p->scx.core_sched_at if custom core-sched ordering is in effect. - */ -static void touch_core_sched_dispatch(struct rq *rq, struct task_struct *p) -{ - lockdep_assert_rq_held(rq); - -#ifdef CONFIG_SCHED_CORE - if (unlikely(SCX_HAS_OP(scx_root, core_sched_before))) - touch_core_sched(rq, p); -#endif -} - /* * p->scx.slice_oob packs an out-of-band slice request into one atomic64. A zero * word means no request. Otherwise the fields are: @@ -1446,11 +1399,8 @@ static void update_curr_scx(struct rq *rq) if (unlikely(delta_exec <= 0)) return; - if (curr->scx.slice != SCX_SLICE_INF) { + if (curr->scx.slice != SCX_SLICE_INF) curr->scx.slice -= min_t(u64, curr->scx.slice, delta_exec); - if (!curr->scx.slice) - touch_core_sched(rq, curr); - } if (unlikely(curr == scx_rescuee(rq))) scx_rescue_charge(rq, delta_exec); @@ -1963,8 +1913,6 @@ static void direct_dispatch(struct scx_sched *sch, struct task_struct *p, find_dsq_for_dispatch(sch, rq, p->scx.ddsp_dsq_id, task_cpu(p)); u64 ddsp_enq_flags, slice, vtime; - touch_core_sched_dispatch(rq, p); - p->scx.ddsp_enq_flags |= enq_flags; /* @@ -2143,12 +2091,6 @@ bypass: goto enqueue; enqueue: - /* - * For task-ordering, slice refill must be treated as implying the end - * of the current slice. Otherwise, the longer @p stays on the CPU, the - * higher priority it becomes from scx_prio_less()'s POV. - */ - touch_core_sched(rq, p); refill_task_slice_dfl(sch, p); clear_direct_dispatch(p); scx_dispatch_enqueue(sch, rq, dsq, p, 0, 0, enq_flags); @@ -2226,9 +2168,6 @@ static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_ if (SCX_HAS_OP(sch, runnable) && !task_on_rq_migrating(p)) SCX_CALL_OP_TASK(sch, runnable, rq, p, enq_flags); - if (enq_flags & SCX_ENQ_WAKEUP) - touch_core_sched(rq, p); - /* Start dl_server if this is the first task being enqueued */ if (rq->scx.nr_running == 1) dl_server_start(&rq->ext_server); @@ -2886,7 +2825,6 @@ static void finish_dispatch(struct scx_sched *sch, struct rq *rq, struct task_st struct scx_dispatch_q *dsq; unsigned long opss; - touch_core_sched_dispatch(rq, p); retry: /* * No need for _acquire here. @p is accessed only after a successful @@ -3521,13 +3459,10 @@ void ext_server_init(struct rq *rq) * usual sched_class'es and needs to find out the expected task ordering. For * SCX, core-sched calls this function to interrogate the task ordering. * - * Unless overridden by ops.core_sched_before(), @p->scx.core_sched_at is used - * to implement the default task ordering. The older the timestamp, the higher - * priority the task - the global FIFO ordering matching the default scheduling - * behavior. - * - * When ops.core_sched_before() is enabled, @p->scx.core_sched_at is used to - * implement FIFO ordering within each local DSQ. See pick_task_scx(). + * Unless overridden by ops.core_sched_before(), the default task ordering runs + * the task which has been waiting longer first. A running task counts as the + * most recently serviced and orders after every waiting task. Waiting tasks are + * compared by @p->scx.runnable_at. * * Return: %true if @a should run after @b. */ @@ -3536,6 +3471,7 @@ bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, { struct scx_sched *sch_a = scx_task_sched(a); struct scx_sched *sch_b = scx_task_sched(b); + bool a_running, b_running; /* * scx_prio_less() returns whether @a should run after @b while @@ -3552,8 +3488,19 @@ bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, task_rq(a), (struct task_struct *)b, (struct task_struct *)a); - else - return time_after64(a->scx.core_sched_at, b->scx.core_sched_at); + + /* + * runnable_at is refreshed only on enqueue, so a task which keeps + * occupying its CPU carries a stale stamp. A running task is the most + * recently serviced whatever its stamp says. Order it after every + * waiting task. + */ + a_running = a->on_cpu; + b_running = b->on_cpu; + if (a_running != b_running) + return a_running; + + return time_after(a->scx.runnable_at, b->scx.runnable_at); } #endif /* CONFIG_SCHED_CORE */ @@ -3824,15 +3771,13 @@ static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued) update_curr_scx(rq); /* - * While disabling, always resched and refresh core-sched timestamp as - * we can't trust the slice management or ops.core_sched_before(). + * While disabling, always resched as we can't trust the slice + * management. */ - if (scx_bypassing(sch, cpu_of(rq))) { + if (scx_bypassing(sch, cpu_of(rq))) scx_set_task_slice(curr, 0); - touch_core_sched(rq, curr); - } else if (SCX_HAS_OP(sch, tick)) { + else if (SCX_HAS_OP(sch, tick)) SCX_CALL_OP_TASK(sch, tick, rq, curr); - } if (!curr->scx.slice) resched_curr(rq); @@ -6085,14 +6030,14 @@ static void unbypass_renotify_idle(struct rq *rq, struct scx_sched *pos, * * - dispatch_one() does not report %SCX_DSP_PREV on non-zero slice as slice * can't be trusted. Whenever a tick triggers, the running task is rotated to - * the tail of the queue with core_sched_at touched. + * the tail of the queue. * * - pick_next_task() suppresses zero slice warning. * * - scx_kick_cpu() is disabled to avoid irq_work malfunction during PM * operations. * - * - scx_prio_less() reverts to the default core_sched_at order. + * - scx_prio_less() reverts to the default runnable_at order. */ void scx_bypass(struct scx_sched *sch, bool bypass) { -- cgit v1.2.3