From e6ac89b8b1c12e9104df45a14a26e2dfa96ded06 Mon Sep 17 00:00:00 2001 From: Kuba Piecuch Date: Sat, 3 Oct 2026 11:53:17 +0000 Subject: sched_ext: Generate qseq from a per-task counter finish_dispatch() uses the qseq embedded in p->scx.ops_state to tell whether the QUEUED instance of a task it's about to claim is the one scx_bpf_dsq_insert() saw. qseq is generated from rq->scx.ops_qseq, but the counters of different rqs are independent, so if a task is dequeued and re-enqueued on a different rq between scx_bpf_dsq_insert() and finish_dispatch(), the new QUEUED instance can end up with the same qseq as the old one: CPU X CPU Z ----- ----- enqueue p on rq A, qseq = N ops.dispatch() scx_bpf_dsq_insert(p) records qseq N sched_setaffinity(p) dequeue p from rq A enqueue p on rq B, qseq = N finish_dispatch(p, N) qseq matches, p is claimed The claim itself is still atomic so the core stays consistent, but an insert issued for a previous QUEUED instance gets applied to a new one which the BPF scheduler has just received through ops.enqueue(). This breaks the guarantee that dispatches targeting a stale instance are ignored. Generate qseq from a per-task counter, p->scx.ops_qseq, instead so that consecutive QUEUED instances of a task never share a qseq regardless of which rq they're on. The counter is only updated in scx_do_enqueue_task() with the task's rq locked, so no additional synchronization is needed, and it fits in an existing hole in struct sched_ext_entity on 64bit. Remove the now unused rq->scx.ops_qseq. Never generate qseq 0. NONE and DISPATCHING don't carry a qseq, so scx_bpf_dsq_insert() on a task in either state records 0. With a per-task counter, every task's first QUEUED instance would otherwise get qseq 0 and could be claimed by such an insert. Wrap the counter where the QSEQ field wraps so that it can't reach a value that shifts to 0 on 32bit either. Fixes: f0e1a0643a59 ("sched_ext: Implement BPF extensible scheduler class") Cc: stable@vger.kernel.org # v6.12+ Assisted-by: Claude:claude-opus-5.5 Signed-off-by: Kuba Piecuch Signed-off-by: Tejun Heo --- include/linux/sched/ext.h | 1 + kernel/sched/ext/ext.c | 14 ++++++++++---- kernel/sched/ext/internal.h | 5 +++++ kernel/sched/sched.h | 1 - 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index 582d7cd4a983..36c04797436c 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -207,6 +207,7 @@ struct sched_ext_entity { s32 holding_cpu; s32 selected_cpu; s32 runnable_cpu; /* cpu @p is runnable on, -1 if not */ + u32 ops_qseq; /* protected by rq lock */ struct task_struct *kf_tasks[2]; /* see SCX_CALL_OP_TASK() */ struct list_head runnable_node; /* rq->scx.runnable_list */ diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 5209510d6a5e..ac8fe70a9fb6 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -2057,8 +2057,14 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, if (unlikely(!SCX_HAS_OP(sch, enqueue))) goto global; - /* DSQ bypass didn't trigger, enqueue on the BPF scheduler */ - qseq = rq->scx.ops_qseq++ << SCX_OPSS_QSEQ_SHIFT; + /* + * DSQ bypass didn't trigger, enqueue on the BPF scheduler. Wrap the + * per-task qseq counter where the QSEQ field wraps and skip 0, which is + * what scx_bpf_dsq_insert() records for a task in NONE or DISPATCHING. + */ + p->scx.ops_qseq = ((p->scx.ops_qseq + 1) & + (SCX_OPSS_QSEQ_MASK >> SCX_OPSS_QSEQ_SHIFT)) ?: 1; + qseq = (unsigned long)p->scx.ops_qseq << SCX_OPSS_QSEQ_SHIFT; WARN_ON_ONCE(atomic_long_read(&p->scx.ops_state) != SCX_OPSS_NONE); atomic_long_set(&p->scx.ops_state, SCX_OPSS_QUEUEING | qseq); @@ -6969,9 +6975,9 @@ static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s, seq_buf_init(&ns, buf, avail); dump_newline(&ns); - scx_dump_line(&ns, "CPU %-4d: nr_run=%u flags=0x%x cpu_rel=%d ops_qseq=%lu ksync=%lu", + scx_dump_line(&ns, "CPU %-4d: nr_run=%u flags=0x%x cpu_rel=%d ksync=%lu", cpu, rq->scx.nr_running, rq->scx.flags, rq->scx.cpu_released, - rq->scx.ops_qseq, rq->scx.kick_sync); + rq->scx.kick_sync); scx_rescue_dump(&ns, rq); scx_dump_line(&ns, " curr=%s[%d] class=%ps", rq->curr->comm, rq->curr->pid, rq->curr->sched_class); diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 1df8f583b0ec..5b37faa532d6 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1998,6 +1998,11 @@ enum scx_ops_state { * dequeue/requeue, the dispatcher can tell whether it still has a claim * on the task being dispatched. * + * QSEQ is generated from the per-task p->scx.ops_qseq counter so that + * it doesn't repeat across QUEUED instances of the same task even if + * the task moves between rqs. 0 is never used as a valid QSEQ since + * NONE and DISPATCHING map to this value. + * * As some 32bit archs can't do 64bit store_release/load_acquire, * p->scx.ops_state is atomic_long_t which leaves 30 bits for QSEQ on * 32bit machines. The dispatch race window QSEQ protects is very narrow diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 742cce9da4e9..9ed0cb605c33 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -813,7 +813,6 @@ struct scx_rq { #endif struct list_head runnable_list; /* runnable tasks on this rq */ struct list_head ddsp_deferred_locals; /* deferred ddsps from enq */ - unsigned long ops_qseq; /* both stashed across the activate_task() in move_remote_task_to_local_dsq() */ u64 remote_activate_enq_flags; struct scx_sched *remote_activate_sch; -- cgit v1.2.3 From 7a9df76679633cf26e0fea882a684012c6ec245a Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 7 Oct 2026 06:26:47 -1000 Subject: sched_ext: Fix stall when a task enqueued with SCX_ENQ_LAST lands back on its CPU's local DSQ With SCX_OPS_ENQ_LAST, the last runnable task on a CPU is passed to ops.enqueue() instead of being kept running. Exiting tasks without SCX_OPS_ENQ_EXITING, migration disabled tasks without SCX_OPS_ENQ_MIGRATION_DISABLED and tasks on an offline rq skip ops.enqueue() and go straight to the local DSQ. This happens in put_prev_task_scx() after the pick has settled on idle, so nothing reschedules the CPU, which idles with the task queued until an unrelated wakeup lands on it or the watchdog fires. scx_mitosis, which sets SCX_OPS_ENQ_LAST without SCX_OPS_ENQ_EXITING, hit this during mass cgroup teardown: exiting tasks stuck in exit_mmap() for over 40 seconds. A task put on the local DSQ runs, as anywhere else. If the SCX_ENQ_LAST enqueue left the task on this CPU's local DSQ, set need_resched on the task being switched in, as proxy_resched_idle() does. resched_curr() would flag the task being put instead, and __schedule() clears that right after the pick. Fixes: f0e1a0643a59 ("sched_ext: Implement BPF extensible scheduler class") Cc: stable@vger.kernel.org # v6.12+ Reported-by: Xiangyu Bu Link: https://github.com/sched-ext/scx/pull/3870 Signed-off-by: Tejun Heo Cc: Dan Schatzberg --- kernel/sched/ext/ext.c | 13 +++++++++++-- kernel/sched/ext/internal.h | 5 +++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index ac8fe70a9fb6..64d937fb2b24 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -3213,8 +3213,7 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, /* * If @p is runnable but we're about to enter a lower * sched_class, %SCX_OPS_ENQ_LAST must be set. Tell - * ops.enqueue() that @p is the only one available for this cpu, - * which should trigger an explicit follow-up scheduling event. + * ops.enqueue() that @p is the only one available for this cpu. * This doesn't apply if the baseline access on the CPU is lost. * * Under core scheduling, a pick dispatches only when nothing is @@ -3226,6 +3225,16 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, WARN_ON_ONCE(!sched_core_enabled(rq) && !(sch->ops.flags & SCX_OPS_ENQ_LAST)); scx_do_enqueue_task(rq, p, SCX_ENQ_LAST, -1); + + /* + * A task put on the local DSQ runs, as anywhere else. + * Here the insert can't reschedule on its own: the pick + * has settled on @next and @p is still curr, so + * resched_curr() would flag @p and __schedule() clears + * that right after the pick. Flag @next instead. + */ + if (p->scx.dsq == &rq->scx.local_dsq) + set_tsk_need_resched(next); } else { scx_do_enqueue_task(rq, p, 0, -1); } diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 5b37faa532d6..85579fcabf52 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1774,8 +1774,9 @@ enum scx_enq_flags { * %SCX_OPS_ENQ_LAST is specified, they're ops.enqueue()'d with the * %SCX_ENQ_LAST flag set. * - * The BPF scheduler is responsible for triggering a follow-up - * scheduling event. Otherwise, Execution may stall. + * If the task is queued on the local DSQ of the CPU it was running on, + * it continues to run. Otherwise, the CPU goes idle. A scheduler that + * wants a full dispatch cycle on the CPU should kick it. */ SCX_ENQ_LAST = 1LLU << 41, -- cgit v1.2.3 From 069403dff337feb192bfc70fb28ddc8a171e20c1 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 9 Oct 2026 12:01:45 -1000 Subject: sched_ext: Fix use-after-free of a destroyed user DSQ by a deferred reenqueue A deferred reenqueue can read a freed user DSQ. The request stays on its CPU's deferred list until run_deferred() takes it, and the RCU callback that frees the DSQ assumes that the grace period has flushed it, so exit_dsq() checks the per-CPU node without the deferred list lock. A grace period waits for readers in flight, not for a pending run_deferred(). A run_deferred() which starts after the grace period began can detach the request right before the callback looks and then read the freed DSQ's id. Cancel the pending requests before the grace period instead. The irq work that frees the DSQ unlinks every CPU's request under the per-CPU deferred list lock, and schedule_dsq_reenq() ignores a new request under the same lock once the id is invalid. Only a run_deferred() that started before the grace period can hold a request, and it runs with IRQs disabled, so the grace period waits for it and nothing touches the DSQ after the callback. Fixes: 84b1a0ea0b7c ("sched_ext: Implement scx_bpf_dsq_reenq() for user DSQs") Cc: stable@vger.kernel.org # v7.1+ Reported-and-tested-by: Hui Su Link: https://lore.kernel.org/all/20261008024753.4096008-1-sh_def@163.com/ Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 64d937fb2b24..a6a72ae7f815 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1149,6 +1149,10 @@ void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock); + /* @dsq is being destroyed, see free_dsq_irq_workfn() */ + if (unlikely(READ_ONCE(dsq->id) == SCX_DSQ_INVALID)) + return; + if (list_empty(&dru->node)) list_move_tail(&dru->node, &rq->scx.deferred_reenq_users); WRITE_ONCE(dru->flags, dru->flags | reenq_flags); @@ -5180,8 +5184,8 @@ static void exit_dsq(struct scx_dispatch_q *dsq) struct rq *rq = cpu_rq(cpu); /* - * There must have been a RCU grace period since the last - * insertion and @dsq should be off the deferred list by now. + * free_dsq_irq_workfn() cancelled the reenqs before the grace + * period and schedule_dsq_reenq() queues nothing after that. */ if (WARN_ON_ONCE(!list_empty(&dru->node))) { guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock); @@ -5205,8 +5209,26 @@ static void free_dsq_irq_workfn(struct irq_work *irq_work) struct llist_node *to_free = llist_del_all(&dsqs_to_free); struct scx_dispatch_q *dsq, *tmp_dsq; - llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node) + llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node) { + s32 cpu; + + /* + * Cancel pending reenqs. Only a run_deferred() that started + * before the grace period can hold one, and it keeps IRQs off, + * so the grace period waits for it. After this sweep, + * schedule_dsq_reenq() sees the invalid id under the same lock + * and queues nothing. + */ + for_each_possible_cpu(cpu) { + struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu, cpu); + struct rq *rq = cpu_rq(cpu); + + guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock); + list_del_init(&pcpu->deferred_reenq_user.node); + } + call_rcu(&dsq->rcu, free_dsq_rcufn); + } } static DEFINE_IRQ_WORK(free_dsq_irq_work, free_dsq_irq_workfn); -- cgit v1.2.3