summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2026-07-13 15:55:17 -0700
committerLinus Torvalds <torvalds@linux-foundation.org>2026-07-13 15:55:17 -0700
commitf7574d3f906a963d7b70f01beff731ee0351d5c5 (patch)
tree162c0dab977c778156bfcffa2d50ba35e3a505fe
parentf94f853f6de0d78a68a7de9f57e3b7f9dcdb728b (diff)
parent0e2f4ab68a89fad42e0f5a9ff4b740738e7aa1d6 (diff)
downloadlinux-next-f7574d3f906a963d7b70f01beff731ee0351d5c5.tar.gz
linux-next-f7574d3f906a963d7b70f01beff731ee0351d5c5.zip
Merge tag 'sched_ext-for-7.2-rc3-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext
Pull sched_ext fixes from Tejun Heo: - Lifecycle fixes for the new sub-scheduler support: two use-after-frees and an enable-failure path that left a half-initialized sub-scheduler linked. - Two dispatch-path locking bugs: a spurious scheduler abort from a migration race, and a lockdep splat from stale runqueue-lock tracking. - Callback and task-state fixes: stale scheduler-owned state on a task leaving SCX, a weight callback running after disable, and a bogus warning on core-scheduling forced idle. - On nohz_full, finite-slice tasks could miss the tick that expires their slice. Enable it when such a task is picked, with a selftest. - Smaller fixes: userspace CPU-mask helpers, ratelimited deprecation warnings, docs and a sparse annotation. * tag 'sched_ext-for-7.2-rc3-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext: sched_ext: Skip ops.set_weight() for disabled tasks tools/sched_ext: scx - Fix cmask_subset(), cmask_equal() and cmask_weight() sched_ext: Fix premature ops->priv publication in scx_alloc_and_add_sched() sched_ext: Record an error on errno-only sub-enable failure selftests/sched_ext: Verify nohz_full tick behavior sched_ext: Enable tick for finite slices on nohz_full sched_ext: Preserve rq tracking across local DSQ dispatch sched_ext: Documentation: Fix ops table header reference sched_ext: Don't warn on core-sched forced idle in put_prev_task_scx() sched_ext: Pin parent scx_sched across a child sub-scheduler's lifetime sched_ext: Annotate ksyncs with __rcu in alloc/free_kick_syncs() sched_ext: Check remote rq eligibility under task's rq lock sched_ext: Reset dsq_vtime and slice when a task leaves SCX sched_ext: Avoid flooding the log with deprecation warnings
-rw-r--r--Documentation/scheduler/sched-ext.rst8
-rw-r--r--kernel/sched/ext/ext.c177
-rw-r--r--kernel/sched/ext/internal.h23
-rw-r--r--tools/sched_ext/include/scx/cid.bpf.h88
-rw-r--r--tools/testing/selftests/sched_ext/Makefile1
-rw-r--r--tools/testing/selftests/sched_ext/nohz_tick.bpf.c65
-rw-r--r--tools/testing/selftests/sched_ext/nohz_tick.c347
7 files changed, 614 insertions, 95 deletions
diff --git a/Documentation/scheduler/sched-ext.rst b/Documentation/scheduler/sched-ext.rst
index 4b1ffd03f516..2771ea4cc14a 100644
--- a/Documentation/scheduler/sched-ext.rst
+++ b/Documentation/scheduler/sched-ext.rst
@@ -493,8 +493,9 @@ a freshly woken up task gets on a CPU.
Where to Look
=============
-* ``include/linux/sched/ext.h`` defines the core data structures, ops table
- and constants.
+* ``include/linux/sched/ext.h`` defines the core data structures and
+ constants, while the ops table (``struct sched_ext_ops``) is defined in
+ ``kernel/sched/ext/internal.h``.
* ``kernel/sched/ext/ext.c`` contains sched_ext core implementation and helpers.
The functions prefixed with ``scx_bpf_`` can be called from the BPF
@@ -555,7 +556,8 @@ ABI Instability
===============
The APIs provided by sched_ext to BPF schedulers programs have no stability
-guarantees. This includes the ops table callbacks and constants defined in
+guarantees. This includes the ops table callbacks defined in
+``kernel/sched/ext/internal.h`` and the constants defined in
``include/linux/sched/ext.h``, as well as the ``scx_bpf_`` kfuncs defined in
``kernel/sched/ext/ext.c`` and ``kernel/sched/ext/idle.c``.
diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c
index 691d53fe0f64..e3fa7b2fac9d 100644
--- a/kernel/sched/ext/ext.c
+++ b/kernel/sched/ext/ext.c
@@ -479,6 +479,18 @@ static bool rq_is_open(struct rq *rq, u64 enq_flags)
*/
DEFINE_PER_CPU(struct rq *, scx_locked_rq_state);
+static void switch_rq_lock(struct rq *from, struct rq *to)
+{
+ bool tracked = scx_locked_rq() == from;
+
+ if (tracked)
+ update_locked_rq(NULL);
+ raw_spin_rq_unlock(from);
+ raw_spin_rq_lock(to);
+ if (tracked)
+ update_locked_rq(to);
+}
+
/*
* Flipped on enable per sch->is_cid_type. Declared in internal.h so
* subsystem inlines can read it.
@@ -2274,8 +2286,7 @@ static void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags,
deactivate_task(src_rq, p, 0);
set_task_cpu(p, cpu_of(dst_rq));
- raw_spin_rq_unlock(src_rq);
- raw_spin_rq_lock(dst_rq);
+ switch_rq_lock(src_rq, dst_rq);
/*
* We want to pass scx-specific enq_flags but activate_task() will
@@ -2307,6 +2318,7 @@ static void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags,
* no to the BPF scheduler initiated migrations while offline.
*
* The caller must ensure that @p and @rq are on different CPUs.
+ * If enforce == true, caller must hold @p's rq lock.
*/
static bool task_can_run_on_remote_rq(struct scx_sched *sch,
struct task_struct *p, struct rq *rq,
@@ -2314,6 +2326,14 @@ static bool task_can_run_on_remote_rq(struct scx_sched *sch,
{
s32 cpu = cpu_of(rq);
+ /*
+ * To prevent races with @p still running on its old CPU while switching
+ * out, make sure we're holding @p's rq lock so as not to risk
+ * erroneously killing the BPF scheduler.
+ */
+ if (enforce)
+ lockdep_assert_rq_held(task_rq(p));
+
WARN_ON_ONCE(task_cpu(p) == cpu);
/*
@@ -2581,13 +2601,6 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq,
return;
}
- if (src_rq != dst_rq &&
- unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) {
- dispatch_enqueue(sch, rq, find_global_dsq(sch, task_cpu(p)), p,
- enq_flags | SCX_ENQ_CLEAR_OPSS | SCX_ENQ_GDSQ_FALLBACK);
- return;
- }
-
/*
* @p is on a possibly remote @src_rq which we need to lock to move the
* task. If dequeue is in progress, it'd be locking @src_rq and waiting
@@ -2606,14 +2619,14 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq,
/* switch to @src_rq lock */
if (locked_rq != src_rq) {
- raw_spin_rq_unlock(locked_rq);
+ switch_rq_lock(locked_rq, src_rq);
locked_rq = src_rq;
- raw_spin_rq_lock(src_rq);
}
/* task_rq couldn't have changed if we're still the holding cpu */
if (likely(p->scx.holding_cpu == raw_smp_processor_id()) &&
!WARN_ON_ONCE(src_rq != task_rq(p))) {
+ bool fallback = false;
/*
* If @p is staying on the same rq, there's no need to go
* through the full deactivate/activate cycle. Optimize by
@@ -2623,6 +2636,11 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq,
p->scx.holding_cpu = -1;
dispatch_enqueue(sch, dst_rq, &dst_rq->scx.local_dsq, p,
enq_flags);
+ } else if (unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) {
+ p->scx.holding_cpu = -1;
+ fallback = true;
+ dispatch_enqueue(sch, src_rq, find_global_dsq(sch, task_cpu(p)),
+ p, enq_flags | SCX_ENQ_GDSQ_FALLBACK);
} else {
move_remote_task_to_local_dsq(p, enq_flags,
src_rq, dst_rq);
@@ -2631,15 +2649,13 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq,
}
/* if the destination CPU is idle, wake it up */
- if (sched_class_above(p->sched_class, dst_rq->curr->sched_class))
+ if (!fallback && sched_class_above(p->sched_class, dst_rq->curr->sched_class))
resched_curr(dst_rq);
}
/* switch back to @rq lock */
- if (locked_rq != rq) {
- raw_spin_rq_unlock(locked_rq);
- raw_spin_rq_lock(rq);
- }
+ if (locked_rq != rq)
+ switch_rq_lock(locked_rq, rq);
}
/**
@@ -2970,24 +2986,38 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first)
/*
* @p is getting newly scheduled or got kicked after someone updated its
- * slice. Refresh whether tick can be stopped. See scx_can_stop_tick().
+ * slice. Update SCX_RQ_CAN_STOP_TICK to reflect whether the tick can be
+ * stopped. See scx_can_stop_tick().
+ *
+ * Moreover, refresh the load_avgs just when transitioning in and out of
+ * nohz. In the future, we might want to add a mechanism to update
+ * load_avgs periodically on tick-stopped CPUs.
*/
- if ((p->scx.slice == SCX_SLICE_INF) !=
- (bool)(rq->scx.flags & SCX_RQ_CAN_STOP_TICK)) {
- if (p->scx.slice == SCX_SLICE_INF)
+ if (p->scx.slice == SCX_SLICE_INF) {
+ if (!(rq->scx.flags & SCX_RQ_CAN_STOP_TICK)) {
+ /*
+ * Bypass mode always assigns finite slices, so @p
+ * can't have an infinite slice while bypassing.
+ * Therefore, sched_update_tick_dependency() can safely
+ * evaluate the outgoing task.
+ */
rq->scx.flags |= SCX_RQ_CAN_STOP_TICK;
- else
- rq->scx.flags &= ~SCX_RQ_CAN_STOP_TICK;
+ sched_update_tick_dependency(rq);
- sched_update_tick_dependency(rq);
+ update_other_load_avgs(rq);
+ }
+ } else {
+ if (rq->scx.flags & SCX_RQ_CAN_STOP_TICK) {
+ rq->scx.flags &= ~SCX_RQ_CAN_STOP_TICK;
+ update_other_load_avgs(rq);
+ }
/*
- * For now, let's refresh the load_avgs just when transitioning
- * in and out of nohz. In the future, we might want to add a
- * mechanism which calls the following periodically on
- * tick-stopped CPUs.
+ * @rq still references the outgoing scheduling context. A finite
+ * slice is sufficient by itself to require the tick.
*/
- update_other_load_avgs(rq);
+ if (tick_nohz_full_cpu(cpu_of(rq)))
+ tick_nohz_dep_set_cpu(cpu_of(rq), TICK_DEP_BIT_SCHED);
}
}
@@ -3082,9 +3112,14 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p,
* sched_class, %SCX_OPS_ENQ_LAST must be set. Tell
* ops.enqueue() that @p is the only one available for this cpu,
* which should trigger an explicit follow-up scheduling event.
+ *
+ * Core scheduling can force this CPU idle while @p stays
+ * runnable. @p's cookie then won't match the core's, so skip
+ * the warning in that case.
*/
if (next && sched_class_above(&ext_sched_class, next->sched_class)) {
- WARN_ON_ONCE(!(sch->ops.flags & SCX_OPS_ENQ_LAST));
+ WARN_ON_ONCE(sched_cpu_cookie_match(rq, p) &&
+ !(sch->ops.flags & SCX_OPS_ENQ_LAST));
do_enqueue_task(rq, p, SCX_ENQ_LAST, -1);
} else {
do_enqueue_task(rq, p, 0, -1);
@@ -3648,6 +3683,13 @@ static void scx_disable_task(struct scx_sched *sch, struct task_struct *p)
scx_set_task_state(p, SCX_TASK_READY);
/*
+ * Reset the SCX-managed fields when @p leaves the BPF scheduler's
+ * control, after ops.disable() has observed their final values.
+ */
+ p->scx.dsq_vtime = 0;
+ p->scx.slice = 0;
+
+ /*
* Verify the task is not in BPF scheduler's custody. If flag
* transitions are consistent, the flag should always be clear
* here.
@@ -3925,6 +3967,17 @@ static void reweight_task_scx(struct rq *rq, struct task_struct *p,
if (task_dead_and_done(p))
return;
+ /*
+ * When switching sched_class away from SCX, reweight_task_scx()
+ * is called _after_ scx_disable_task(). Skip calling ops.set_weight()
+ * since the BPF scheduler may have already forgotten the task in
+ * ops.disable().
+ * p->scx.weight will be recalculated in scx_enable_task() if the task
+ * ever returns to SCX class.
+ */
+ if (scx_get_task_state(p) != SCX_TASK_ENABLED)
+ return;
+
p->scx.weight = sched_weight_to_cgroup(scale_load_down(lw->weight));
if (SCX_HAS_OP(sch, set_weight))
SCX_CALL_OP_TASK(sch, set_weight, rq, p, p->scx.weight);
@@ -4301,6 +4354,15 @@ bool scx_can_stop_tick(struct rq *rq)
if (p->sched_class != &ext_sched_class)
return true;
+ /*
+ * @rq->curr may still reference an outgoing EXT task after it has been
+ * dequeued. If no EXT tasks are accounted on @rq, ignore its stale
+ * slice state. If another task is dispatched from a DSQ,
+ * set_next_task_scx() will update the dependency for the incoming task.
+ */
+ if (!rq->scx.nr_running)
+ return true;
+
if (scx_bypassing(sch, cpu_of(rq)))
return false;
@@ -4901,6 +4963,8 @@ static void scx_sched_free_rcu_work(struct work_struct *work)
cgroup_put(sch_cgroup(sch));
if (sch->sub_kset)
kobject_put(&sch->sub_kset->kobj);
+ if (scx_parent(sch))
+ kobject_put(&scx_parent(sch)->kobj);
#endif /* CONFIG_EXT_SUB_SCHED */
for_each_possible_cpu(cpu) {
@@ -5672,7 +5736,7 @@ static void free_kick_syncs(void)
int cpu;
for_each_possible_cpu(cpu) {
- struct scx_kick_syncs **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu);
+ struct scx_kick_syncs __rcu **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu);
struct scx_kick_syncs *to_free;
to_free = rcu_replace_pointer(*ksyncs, NULL, true);
@@ -6653,7 +6717,7 @@ static int alloc_kick_syncs(void)
* can exceed percpu allocator limits on large machines.
*/
for_each_possible_cpu(cpu) {
- struct scx_kick_syncs **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu);
+ struct scx_kick_syncs __rcu **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu);
struct scx_kick_syncs *new_ksyncs;
WARN_ON_ONCE(rcu_access_pointer(*ksyncs));
@@ -6825,11 +6889,6 @@ static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd,
sch->ops = *cmd->ops;
}
- rcu_assign_pointer(ops->priv, sch);
-
- sch->kobj.kset = scx_kset;
- INIT_LIST_HEAD(&sch->all);
-
#ifdef CONFIG_EXT_SUB_SCHED
char *buf = kzalloc(PATH_MAX, GFP_KERNEL);
if (!buf) {
@@ -6847,13 +6906,32 @@ static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd,
sch->cgrp = cgrp;
INIT_LIST_HEAD(&sch->children);
INIT_LIST_HEAD(&sch->sibling);
+#endif /* CONFIG_EXT_SUB_SCHED */
- if (parent)
+ /*
+ * Publishing makes @sch visible to scx_prog_sched() readers. Failure
+ * paths after this point must free @sch through kobject_put() whose
+ * release path defers the actual freeing by an RCU grace period.
+ */
+ rcu_assign_pointer(ops->priv, sch);
+
+ sch->kobj.kset = scx_kset;
+ INIT_LIST_HEAD(&sch->all);
+
+#ifdef CONFIG_EXT_SUB_SCHED
+ if (parent) {
+ /*
+ * Pin @parent for @sch's lifetime. The kobject hierarchy pins
+ * it only via @parent->sub_kset, which is dropped during
+ * disable. Released in scx_sched_free_rcu_work().
+ */
+ kobject_get(&parent->kobj);
ret = kobject_init_and_add(&sch->kobj, &scx_ktype,
&parent->sub_kset->kobj,
"sub-%llu", cgroup_id(cgrp));
- else
+ } else {
ret = kobject_init_and_add(&sch->kobj, &scx_ktype, NULL, "root");
+ }
if (ret < 0) {
RCU_INIT_POINTER(ops->priv, NULL);
@@ -6895,7 +6973,6 @@ static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd,
#ifdef CONFIG_EXT_SUB_SCHED
err_free_lb_resched:
- RCU_INIT_POINTER(ops->priv, NULL);
free_cpumask_var(sch->bypass_lb_resched_cpumask);
#endif
err_free_lb_cpumask:
@@ -6988,7 +7065,7 @@ static int validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops)
* run past the BPF allocation. Skip for cid-form.
*/
if (!sch->is_cid_type && (ops->cpu_acquire || ops->cpu_release))
- pr_warn("ops->cpu_acquire/release() are deprecated, use sched_switch TP instead\n");
+ pr_warn_ratelimited("ops->cpu_acquire/release() are deprecated, use sched_switch TP instead\n");
/*
* Sub-scheduler support is tied to the cid-form struct_ops. A sub-sched
@@ -7686,6 +7763,12 @@ err_unlock_and_disable:
percpu_up_write(&scx_fork_rwsem);
err_disable:
mutex_unlock(&scx_enable_mutex);
+ /*
+ * Some enable failures only return an errno (e.g. -ENOMEM from an
+ * allocation) without calling scx_error(). Record it so
+ * scx_flush_disable_work() runs the disable and ops.exit() fires.
+ */
+ scx_error(sch, "scx_sub_enable() failed (%d)", ret);
scx_flush_disable_work(sch);
cmd->ret = 0;
}
@@ -7806,7 +7889,7 @@ static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log,
off + size <= offsetofend(struct task_struct, scx.slice)) ||
(off >= offsetof(struct task_struct, scx.dsq_vtime) &&
off + size <= offsetofend(struct task_struct, scx.dsq_vtime))) {
- pr_warn("sched_ext: Writing directly to p->scx.slice/dsq_vtime is deprecated, use scx_bpf_task_set_slice/dsq_vtime()");
+ pr_warn_ratelimited("sched_ext: Writing directly to p->scx.slice/dsq_vtime is deprecated, use scx_bpf_task_set_slice/dsq_vtime()\n");
return SCALAR_VALUE;
}
@@ -8796,10 +8879,8 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit,
in_balance = this_rq->scx.flags & SCX_RQ_IN_BALANCE;
if (in_balance) {
- if (this_rq != src_rq) {
- raw_spin_rq_unlock(this_rq);
- raw_spin_rq_lock(src_rq);
- }
+ if (this_rq != src_rq)
+ switch_rq_lock(this_rq, src_rq);
} else {
raw_spin_rq_lock(src_rq);
}
@@ -8831,10 +8912,8 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit,
dispatched = true;
out:
if (in_balance) {
- if (this_rq != locked_rq) {
- raw_spin_rq_unlock(locked_rq);
- raw_spin_rq_lock(this_rq);
- }
+ if (this_rq != locked_rq)
+ switch_rq_lock(locked_rq, this_rq);
} else {
raw_spin_rq_unlock_irqrestore(locked_rq, flags);
}
diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h
index 145272cb4d8a..673059fa9d72 100644
--- a/kernel/sched/ext/internal.h
+++ b/kernel/sched/ext/internal.h
@@ -1469,21 +1469,24 @@ static const char *scx_enable_state_str[] = {
* The sched_ext core uses a "lock dancing" protocol coordinated by
* p->scx.holding_cpu. When moving a task to a different rq:
*
- * 1. Verify task can be moved (CPU affinity, migration_disabled, etc.)
- * 2. Set p->scx.holding_cpu to the current CPU
- * 3. Set task state to %SCX_OPSS_NONE; dequeue waits while DISPATCHING
+ * 1. Set p->scx.holding_cpu to the current CPU
+ * 2. Set task state to %SCX_OPSS_NONE; dequeue waits while DISPATCHING
* is set, so clearing DISPATCHING first prevents the circular wait
* (safe to lock the rq we need)
- * 4. Unlock the current CPU's rq
- * 5. Lock src_rq (where the task currently lives)
- * 6. Verify p->scx.holding_cpu == current CPU, if not, dequeue won the
+ * 3. Unlock the current CPU's rq
+ * 4. Lock src_rq (where the task currently lives)
+ * 5. Verify p->scx.holding_cpu == current CPU, if not, dequeue won the
* race (dequeue clears holding_cpu to -1 when it takes the task), in
* this case migration is aborted
- * 7. If src_rq == dst_rq: clear holding_cpu and enqueue directly
+ * 6. If src_rq == dst_rq: clear holding_cpu and enqueue directly
* into dst_rq's local DSQ (no lock swap needed)
- * 8. Otherwise: call move_remote_task_to_local_dsq(), which releases
- * src_rq, locks dst_rq, and performs the deactivate/activate
- * migration cycle (dst_rq is held on return)
+ * 7. Otherwise, verify under src_rq lock that the task can be moved to dst_rq
+ * (CPU affinity, migration_disabled, etc.). If not, clear holding_cpu,
+ * leave the task on src_rq, and enqueue it on the fallback DSQ.
+ * 8. Otherwise (i.e. if the task can be moved to dst_rq), call
+ * move_remote_task_to_local_dsq(), which releases src_rq, locks dst_rq,
+ * and performs the deactivate/activate migration cycle
+ * (dst_rq is held on return)
* 9. Unlock dst_rq and re-lock the current CPU's rq to restore
* the lock state expected by the caller
*
diff --git a/tools/sched_ext/include/scx/cid.bpf.h b/tools/sched_ext/include/scx/cid.bpf.h
index db247e42fb45..6b0b4e41b288 100644
--- a/tools/sched_ext/include/scx/cid.bpf.h
+++ b/tools/sched_ext/include/scx/cid.bpf.h
@@ -391,7 +391,9 @@ static __always_inline bool cmask_equal(const struct scx_cmask __arena *a,
if (a->base != b->base || a->nr_cids != b->nr_cids)
return false;
- nr_words = CMASK_NR_WORDS(a->nr_cids);
+ if (a->nr_cids == 0)
+ return true;
+ nr_words = (a->base + a->nr_cids - 1) / 64 - a->base / 64 + 1;
bpf_for(i, 0, CMASK_MAX_WORDS) {
if (i >= nr_words)
@@ -402,36 +404,6 @@ static __always_inline bool cmask_equal(const struct scx_cmask __arena *a,
return true;
}
-/*
- * True iff every bit set in @a is also set in @b over the intersection of
- * their ranges. Bits of @a outside @b's range fail the test.
- */
-static __always_inline bool cmask_subset(const struct scx_cmask __arena *a,
- const struct scx_cmask __arena *b)
-{
- u32 a_end = a->base + a->nr_cids;
- u32 b_end = b->base + b->nr_cids;
- u32 a_wbase = a->base / 64;
- u32 b_wbase = b->base / 64;
- u32 nr_words, i;
-
- /* any bit of @a outside @b's range is a subset violation */
- if (a->base < b->base || a_end > b_end)
- return false;
-
- nr_words = CMASK_NR_WORDS(a->nr_cids);
- bpf_for(i, 0, CMASK_MAX_WORDS) {
- u32 wi_b;
-
- if (i >= nr_words)
- break;
- wi_b = a_wbase + i - b_wbase;
- if (a->bits[i] & ~b->bits[wi_b])
- return false;
- }
- return true;
-}
-
/**
* cmask_next_set - find the first set bit at or after @cid
* @m: cmask to search
@@ -489,15 +461,65 @@ static __always_inline u32 cmask_first_set(const struct scx_cmask __arena *m)
(cid) = cmask_next_set((m), (cid) + 1))
/*
+ * True iff every bit set in @a is also set in @b. Matches the kernel-side
+ * scx_cmask_subset(): ranges don't need to nest, and set bits of @a outside
+ * @b's range fail the test.
+ */
+static __always_inline bool cmask_subset(const struct scx_cmask __arena *a,
+ const struct scx_cmask __arena *b)
+{
+ u32 a_end = a->base + a->nr_cids;
+ u32 b_end = b->base + b->nr_cids;
+ u32 a_wbase = a->base / 64;
+ u32 b_wbase = b->base / 64;
+ u32 lo = a->base > b->base ? a->base : b->base;
+ u32 hi = a_end < b_end ? a_end : b_end;
+ u32 lo_word, hi_word, i;
+
+ /* set bits of @a outside @b's range can't be in @b */
+ if (a->base < b->base &&
+ cmask_next_set(a, a->base) < (b->base < a_end ? b->base : a_end))
+ return false;
+ if (a_end > b_end &&
+ cmask_next_set(a, a->base > b_end ? a->base : b_end) < a_end)
+ return false;
+
+ if (lo >= hi)
+ return true;
+
+ /*
+ * Walk the words the range intersection spans. Plain word tests
+ * suffice: the scans above guarantee @a has no set bit outside @b's
+ * range and padding bits are kept clear by all cmask helpers.
+ */
+ lo_word = lo / 64;
+ hi_word = (hi - 1) / 64;
+
+ bpf_for(i, 0, CMASK_MAX_WORDS) {
+ u32 w = lo_word + i;
+
+ if (w > hi_word)
+ break;
+ if (a->bits[w - a_wbase] & ~b->bits[w - b_wbase])
+ return false;
+ }
+ return true;
+}
+
+/*
* Population count over [base, base + nr_cids). Padding bits in the head/tail
* words are guaranteed zero by the mutating helpers, so a flat popcount over
- * all words is correct.
+ * the words the range spans is correct.
*/
static __always_inline u32 cmask_weight(const struct scx_cmask __arena *m)
{
- u32 nr_words = CMASK_NR_WORDS(m->nr_cids), i;
+ u32 nr_words, i;
u32 count = 0;
+ if (!m->nr_cids)
+ return 0;
+ nr_words = (m->base + m->nr_cids - 1) / 64 - m->base / 64 + 1;
+
bpf_for(i, 0, CMASK_MAX_WORDS) {
if (i >= nr_words)
break;
diff --git a/tools/testing/selftests/sched_ext/Makefile b/tools/testing/selftests/sched_ext/Makefile
index 5d2dffca0e91..3cfe90e0f34f 100644
--- a/tools/testing/selftests/sched_ext/Makefile
+++ b/tools/testing/selftests/sched_ext/Makefile
@@ -176,6 +176,7 @@ auto-test-targets := \
maybe_null \
minimal \
non_scx_kfunc_deny \
+ nohz_tick \
numa \
allowed_cpus \
peek_dsq \
diff --git a/tools/testing/selftests/sched_ext/nohz_tick.bpf.c b/tools/testing/selftests/sched_ext/nohz_tick.bpf.c
new file mode 100644
index 000000000000..6998c5dd6bcb
--- /dev/null
+++ b/tools/testing/selftests/sched_ext/nohz_tick.bpf.c
@@ -0,0 +1,65 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES
+ *
+ * Exercise tick dependency transitions between infinite and finite slices.
+ */
+#include <scx/common.bpf.h>
+
+char _license[] SEC("license") = "GPL";
+
+const volatile s32 test_cpu;
+bool finite_phase;
+u64 nr_inf_running;
+u64 nr_finite_running;
+u64 nr_finite_ticks;
+
+UEI_DEFINE(uei);
+
+s32 BPF_STRUCT_OPS(nohz_tick_select_cpu, struct task_struct *p, s32 prev_cpu,
+ u64 wake_flags)
+{
+ return prev_cpu;
+}
+
+void BPF_STRUCT_OPS(nohz_tick_enqueue, struct task_struct *p, u64 enq_flags)
+{
+ u64 slice = finite_phase ? 1000000ULL : SCX_SLICE_INF;
+
+ scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, slice, enq_flags);
+ if (enq_flags & SCX_ENQ_LAST)
+ scx_bpf_kick_cpu(test_cpu, SCX_KICK_IDLE);
+}
+
+void BPF_STRUCT_OPS(nohz_tick_running, struct task_struct *p)
+{
+ if (bpf_get_smp_processor_id() != test_cpu)
+ return;
+
+ if (finite_phase)
+ __sync_fetch_and_add(&nr_finite_running, 1);
+ else
+ __sync_fetch_and_add(&nr_inf_running, 1);
+}
+
+void BPF_STRUCT_OPS(nohz_tick_tick, struct task_struct *p)
+{
+ if (bpf_get_smp_processor_id() == test_cpu && finite_phase)
+ __sync_fetch_and_add(&nr_finite_ticks, 1);
+}
+
+void BPF_STRUCT_OPS(nohz_tick_exit, struct scx_exit_info *ei)
+{
+ UEI_RECORD(uei, ei);
+}
+
+SEC(".struct_ops.link")
+struct sched_ext_ops nohz_tick_ops = {
+ .select_cpu = (void *)nohz_tick_select_cpu,
+ .enqueue = (void *)nohz_tick_enqueue,
+ .running = (void *)nohz_tick_running,
+ .tick = (void *)nohz_tick_tick,
+ .exit = (void *)nohz_tick_exit,
+ .name = "nohz_tick",
+ .timeout_ms = 1000U,
+};
diff --git a/tools/testing/selftests/sched_ext/nohz_tick.c b/tools/testing/selftests/sched_ext/nohz_tick.c
new file mode 100644
index 000000000000..028f54391c2c
--- /dev/null
+++ b/tools/testing/selftests/sched_ext/nohz_tick.c
@@ -0,0 +1,347 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES
+ *
+ * Validate that a finite-slice EXT task restarts the scheduler tick when it
+ * follows an infinite-slice EXT task and an idle interval on a NOHZ_FULL CPU.
+ */
+#define _GNU_SOURCE
+
+#include <bpf/bpf.h>
+#include <errno.h>
+#include <sched.h>
+#include <signal.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/prctl.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+#include <scx/common.h>
+
+#include "nohz_tick.bpf.skel.h"
+#include "scx_test.h"
+
+#ifndef SCHED_EXT
+#define SCHED_EXT 7
+#endif
+
+#define MIN_FINITE_TICKS 3
+#define PHASE_TIMEOUT_MS 1000
+
+struct nohz_tick_ctx {
+ struct nohz_tick *skel;
+ cpu_set_t original_mask;
+ int test_cpu;
+};
+
+static int first_allowed_cpu(const cpu_set_t *mask, int first, int last)
+{
+ int cpu;
+
+ for (cpu = first; cpu <= last && cpu < CPU_SETSIZE; cpu++)
+ if (CPU_ISSET(cpu, mask))
+ return cpu;
+
+ return -1;
+}
+
+static int find_nohz_full_cpu(const cpu_set_t *allowed)
+{
+ char buf[4096], *cur, *end;
+ FILE *file;
+
+ file = fopen("/sys/devices/system/cpu/nohz_full", "r");
+ if (!file)
+ return -1;
+ if (!fgets(buf, sizeof(buf), file)) {
+ fclose(file);
+ return -1;
+ }
+ fclose(file);
+
+ cur = buf;
+ while (*cur) {
+ long first, last;
+ int cpu;
+
+ while (*cur == ' ' || *cur == '\t' || *cur == ',')
+ cur++;
+ if (*cur < '0' || *cur > '9')
+ break;
+
+ errno = 0;
+ first = strtol(cur, &end, 10);
+ if (errno || end == cur || first < 0 || first >= CPU_SETSIZE)
+ return -1;
+ cur = end;
+ last = first;
+ if (*cur == '-') {
+ cur++;
+ errno = 0;
+ last = strtol(cur, &end, 10);
+ if (errno || end == cur || last < first)
+ return -1;
+ cur = end;
+ }
+
+ cpu = first_allowed_cpu(allowed, first, last);
+ if (cpu >= 0)
+ return cpu;
+ }
+
+ return -1;
+}
+
+static pid_t start_worker(int cpu)
+{
+ struct sched_param param = {};
+ cpu_set_t mask;
+ pid_t parent;
+ pid_t pid;
+
+ parent = getpid();
+ pid = fork();
+ if (pid != 0)
+ return pid;
+ if (prctl(PR_SET_PDEATHSIG, SIGKILL) || getppid() != parent)
+ _exit(1);
+
+ /*
+ * Become EXT before touching the target so it stays idle until wakeup.
+ */
+ if (sched_setscheduler(0, SCHED_EXT, &param))
+ _exit(1);
+
+ CPU_ZERO(&mask);
+ CPU_SET(cpu, &mask);
+ if (sched_setaffinity(0, sizeof(mask), &mask))
+ _exit(1);
+
+ for (;;)
+ asm volatile("" ::: "memory");
+}
+
+static void stop_worker(pid_t pid)
+{
+ if (pid <= 0)
+ return;
+
+ kill(pid, SIGKILL);
+ waitpid(pid, NULL, 0);
+}
+
+static int pause_worker(pid_t pid)
+{
+ int status;
+
+ if (kill(pid, SIGSTOP))
+ return -errno;
+ if (waitpid(pid, &status, WUNTRACED) != pid)
+ return -errno;
+ if (!WIFSTOPPED(status))
+ return -ECHILD;
+
+ return 0;
+}
+
+static bool wait_for_counter(const u64 *counter, u64 value, int timeout_ms)
+{
+ int elapsed;
+
+ for (elapsed = 0; elapsed < timeout_ms; elapsed++) {
+ if (__atomic_load_n(counter, __ATOMIC_RELAXED) >= value)
+ return true;
+ usleep(1000);
+ }
+
+ return false;
+}
+
+static enum scx_test_status setup(void **ctx_ptr)
+{
+ struct nohz_tick_ctx *ctx;
+ cpu_set_t controller_mask;
+ int cpu;
+
+ ctx = calloc(1, sizeof(*ctx));
+ SCX_FAIL_IF(!ctx, "Failed to allocate context");
+ if (sched_getaffinity(0, sizeof(ctx->original_mask),
+ &ctx->original_mask)) {
+ free(ctx);
+ SCX_FAIL("Failed to get affinity (%d)", errno);
+ }
+
+ cpu = find_nohz_full_cpu(&ctx->original_mask);
+ if (cpu < 0) {
+ fprintf(stderr, "SKIP: no allowed NOHZ_FULL CPU\n");
+ free(ctx);
+ return SCX_TEST_SKIP;
+ }
+
+ controller_mask = ctx->original_mask;
+ CPU_CLR(cpu, &controller_mask);
+ if (CPU_COUNT(&controller_mask) == 0) {
+ fprintf(stderr, "SKIP: no housekeeping CPU available\n");
+ free(ctx);
+ return SCX_TEST_SKIP;
+ }
+
+ ctx->test_cpu = cpu;
+ ctx->skel = nohz_tick__open();
+ if (!ctx->skel) {
+ free(ctx);
+ SCX_FAIL("Failed to open skeleton");
+ }
+
+ SCX_ENUM_INIT(ctx->skel);
+ ctx->skel->rodata->test_cpu = cpu;
+ ctx->skel->struct_ops.nohz_tick_ops->flags |= SCX_OPS_SWITCH_PARTIAL |
+ SCX_OPS_ENQ_LAST;
+ if (nohz_tick__load(ctx->skel)) {
+ nohz_tick__destroy(ctx->skel);
+ free(ctx);
+ SCX_FAIL("Failed to load skeleton");
+ }
+
+ if (sched_setaffinity(0, sizeof(controller_mask), &controller_mask)) {
+ nohz_tick__destroy(ctx->skel);
+ free(ctx);
+ SCX_FAIL("Failed to move controller off CPU %d (%d)", cpu, errno);
+ }
+
+ *ctx_ptr = ctx;
+ return SCX_TEST_PASS;
+}
+
+static enum scx_test_status run(void *ctx_ptr)
+{
+ struct nohz_tick_ctx *ctx = ctx_ptr;
+ struct nohz_tick *skel = ctx->skel;
+ struct bpf_link *link = NULL;
+ enum scx_test_status status = SCX_TEST_FAIL;
+ pid_t finite_worker = -1;
+ pid_t inf_worker = -1;
+ u64 finite_running;
+ u64 finite_ticks;
+ int ret;
+
+ link = bpf_map__attach_struct_ops(skel->maps.nohz_tick_ops);
+ if (!link) {
+ SCX_ERR("Failed to attach scheduler");
+ goto out;
+ }
+
+ /*
+ * Establish SCX_RQ_CAN_STOP_TICK with an infinite-slice task.
+ */
+ inf_worker = start_worker(ctx->test_cpu);
+ if (inf_worker < 0) {
+ SCX_ERR("Failed to start infinite-slice worker (%d)", errno);
+ goto out;
+ }
+ if (!wait_for_counter(&skel->bss->nr_inf_running, 1,
+ PHASE_TIMEOUT_MS)) {
+ SCX_ERR("Infinite-slice worker was not scheduled");
+ goto out;
+ }
+
+ /* Block without exiting so the rq retains the infinite-slice state. */
+ ret = pause_worker(inf_worker);
+ if (ret) {
+ SCX_ERR("Failed to stop infinite-slice worker (%d)", ret);
+ goto out;
+ }
+
+ /* Let the target enter idle with its tick stopped. */
+ usleep(100000);
+
+ /*
+ * The next EXT task receives a finite slice and must restart the tick.
+ */
+ __atomic_store_n(&skel->bss->finite_phase, true, __ATOMIC_RELEASE);
+ finite_worker = start_worker(ctx->test_cpu);
+ if (finite_worker < 0) {
+ SCX_ERR("Failed to start finite-slice worker (%d)", errno);
+ goto out;
+ }
+ if (!wait_for_counter(&skel->bss->nr_finite_running, 1,
+ PHASE_TIMEOUT_MS)) {
+ SCX_ERR("Finite-slice worker was not scheduled");
+ goto out;
+ }
+ if (!wait_for_counter(&skel->bss->nr_finite_ticks, MIN_FINITE_TICKS,
+ PHASE_TIMEOUT_MS)) {
+ SCX_ERR("Finite-slice worker received only %llu scheduler ticks",
+ (unsigned long long)skel->bss->nr_finite_ticks);
+ goto out;
+ }
+ stop_worker(finite_worker);
+ finite_worker = -1;
+
+ /*
+ * Leave the CPU idle after a finite-slice task. The next finite-slice
+ * task must restart the tick even though the slice type is unchanged.
+ */
+ usleep(100000);
+ finite_running = __atomic_load_n(&skel->bss->nr_finite_running,
+ __ATOMIC_RELAXED);
+ finite_ticks = __atomic_load_n(&skel->bss->nr_finite_ticks,
+ __ATOMIC_RELAXED);
+
+ finite_worker = start_worker(ctx->test_cpu);
+ if (finite_worker < 0) {
+ SCX_ERR("Failed to start second finite-slice worker (%d)", errno);
+ goto out;
+ }
+ if (!wait_for_counter(&skel->bss->nr_finite_running,
+ finite_running + 1, PHASE_TIMEOUT_MS)) {
+ SCX_ERR("Second finite-slice worker was not scheduled");
+ goto out;
+ }
+ if (!wait_for_counter(&skel->bss->nr_finite_ticks,
+ finite_ticks + MIN_FINITE_TICKS,
+ PHASE_TIMEOUT_MS)) {
+ SCX_ERR("Second finite-slice worker received only %llu scheduler ticks",
+ (unsigned long long)(skel->bss->nr_finite_ticks -
+ finite_ticks));
+ goto out;
+ }
+
+ if (skel->data->uei.kind != EXIT_KIND(SCX_EXIT_NONE)) {
+ SCX_ERR("Scheduler exited unexpectedly (kind=%llu code=%lld)",
+ (unsigned long long)skel->data->uei.kind,
+ (long long)skel->data->uei.exit_code);
+ goto out;
+ }
+
+ fprintf(stderr, "CPU %d received %llu finite-slice ticks\n",
+ ctx->test_cpu,
+ (unsigned long long)skel->bss->nr_finite_ticks);
+ status = SCX_TEST_PASS;
+out:
+ stop_worker(finite_worker);
+ stop_worker(inf_worker);
+ if (link)
+ bpf_link__destroy(link);
+ return status;
+}
+
+static void cleanup(void *ctx_ptr)
+{
+ struct nohz_tick_ctx *ctx = ctx_ptr;
+
+ sched_setaffinity(0, sizeof(ctx->original_mask), &ctx->original_mask);
+ nohz_tick__destroy(ctx->skel);
+ free(ctx);
+}
+
+struct scx_test nohz_tick = {
+ .name = "nohz_tick",
+ .description = "Verify finite EXT slices restart the NOHZ_FULL tick",
+ .setup = setup,
+ .run = run,
+ .cleanup = cleanup,
+};
+REGISTER_SCX_TEST(&nohz_tick)