From 93fccdb0456d963f0a7c515562eaf53dfe07f9c7 Mon Sep 17 00:00:00 2001 From: Shaojie Sun Date: Thu, 3 Sep 2026 14:48:26 +0800 Subject: cgroup: namespace: Drop BUG_ON() in copy_cgroup_ns() The sole caller, create_new_namespaces(), always passes a valid cgroup namespace pointer from the task's nsproxy. The other copy_*_ns() helpers, e.g. copy_time_ns() and copy_pid_ns(), don't perform such a check either. Drop the BUG_ON() for consistency with them. Signed-off-by: Shaojie Sun Reviewed-by: Tao Cui Signed-off-by: Tejun Heo --- kernel/cgroup/namespace.c | 2 -- 1 file changed, 2 deletions(-) (limited to 'kernel/cgroup') diff --git a/kernel/cgroup/namespace.c b/kernel/cgroup/namespace.c index ea4ee13936be..76660332ec64 100644 --- a/kernel/cgroup/namespace.c +++ b/kernel/cgroup/namespace.c @@ -53,8 +53,6 @@ struct cgroup_namespace *copy_cgroup_ns(u64 flags, struct ucounts *ucounts; struct css_set *cset; - BUG_ON(!old_ns); - if (!(flags & CLONE_NEWCGROUP)) { get_cgroup_ns(old_ns); return old_ns; -- cgit v1.2.3 From 22098e3b6f7260860d61523a8dc2964b40559763 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 2 Sep 2026 13:03:11 -1000 Subject: cgroup: Return -ENODEV consistently for a dead cgroup cgroup_kn_lock_live() returns NULL when the cgroup behind a kernfs node has already been removed. Most callers turn that into -ENODEV, but the cgroup.type, cgroup.max.descendants, cgroup.max.depth, cgroup.pressure, cgroup.freeze and cgroup.kill writers return -ENOENT. The divergence is accidental. Return -ENODEV from all of them, matching cgroup.procs and cgroup.subtree_control. Signed-off-by: Tejun Heo --- kernel/cgroup/cgroup.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'kernel/cgroup') diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index 2d532bf2c0c7..8ec358c76031 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -3791,7 +3791,7 @@ static ssize_t cgroup_type_write(struct kernfs_open_file *of, char *buf, /* drain dying csses before we re-apply (threaded) subtree control */ cgrp = cgroup_kn_lock_live(of->kn, true); if (!cgrp) - return -ENOENT; + return -ENODEV; /* threaded can only be enabled */ ret = cgroup_enable_threaded(cgrp); @@ -3834,7 +3834,7 @@ static ssize_t cgroup_max_descendants_write(struct kernfs_open_file *of, cgrp = cgroup_kn_lock_live(of->kn, false); if (!cgrp) - return -ENOENT; + return -ENODEV; WRITE_ONCE(cgrp->max_descendants, descendants); @@ -3877,7 +3877,7 @@ static ssize_t cgroup_max_depth_write(struct kernfs_open_file *of, cgrp = cgroup_kn_lock_live(of->kn, false); if (!cgrp) - return -ENOENT; + return -ENODEV; WRITE_ONCE(cgrp->max_depth, depth); @@ -4193,7 +4193,7 @@ static ssize_t cgroup_pressure_write(struct kernfs_open_file *of, cgrp = cgroup_kn_lock_live(of->kn, false); if (!cgrp) - return -ENOENT; + return -ENODEV; psi = cgroup_psi(cgrp); if (psi->enabled != enable) { @@ -4269,7 +4269,7 @@ static ssize_t cgroup_freeze_write(struct kernfs_open_file *of, cgrp = cgroup_kn_lock_live(of->kn, false); if (!cgrp) - return -ENOENT; + return -ENODEV; cgroup_freeze(cgrp, freeze); @@ -4331,7 +4331,7 @@ static ssize_t cgroup_kill_write(struct kernfs_open_file *of, char *buf, cgrp = cgroup_kn_lock_live(of->kn, false); if (!cgrp) - return -ENOENT; + return -ENODEV; /* * Killing is a process directed operation, i.e. the whole thread-group -- cgit v1.2.3 From 67499cd07ba93dc643ca5624b7dc947f0354ace2 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 2 Sep 2026 13:03:12 -1000 Subject: cgroup: Return ERR_PTR from cgroup_kn_lock_live() cgroup_kn_lock_live() returns NULL when the cgroup is dead and the callers translate that into an errno themselves. Return ERR_PTR(-ENODEV) instead and have the callers pass the errno through, so that failures other than a dead cgroup can be reported through the same return value. No functional change. Signed-off-by: Tejun Heo --- kernel/cgroup/cgroup-v1.c | 8 ++++---- kernel/cgroup/cgroup.c | 50 +++++++++++++++++++++++------------------------ kernel/cgroup/debug.c | 10 +++++----- 3 files changed, 34 insertions(+), 34 deletions(-) (limited to 'kernel/cgroup') diff --git a/kernel/cgroup/cgroup-v1.c b/kernel/cgroup/cgroup-v1.c index a4337c9b5287..7e008867f3ac 100644 --- a/kernel/cgroup/cgroup-v1.c +++ b/kernel/cgroup/cgroup-v1.c @@ -506,8 +506,8 @@ static ssize_t __cgroup1_procs_write(struct kernfs_open_file *of, enum cgroup_attach_lock_mode lock_mode; cgrp = cgroup_kn_lock_live(of->kn, false); - if (!cgrp) - return -ENODEV; + if (IS_ERR(cgrp)) + return PTR_ERR(cgrp); task = cgroup_procs_write_start(buf, threadgroup, &lock_mode); ret = PTR_ERR_OR_ZERO(task); @@ -569,8 +569,8 @@ static ssize_t cgroup_release_agent_write(struct kernfs_open_file *of, return -EPERM; cgrp = cgroup_kn_lock_live(of->kn, false); - if (!cgrp) - return -ENODEV; + if (IS_ERR(cgrp)) + return PTR_ERR(cgrp); spin_lock(&release_agent_path_lock); strscpy(cgrp->root->release_agent_path, strstrip(buf), sizeof(cgrp->root->release_agent_path)); diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index 8ec358c76031..d30816ee47ac 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -1685,8 +1685,8 @@ void cgroup_kn_unlock(struct kernfs_node *kn) * This helper is to be used by a cgroup kernfs method currently servicing * @kn. It breaks the active protection, performs cgroup locking and * verifies that the associated cgroup is alive. Returns the cgroup if - * alive; otherwise, %NULL. A successful return should be undone by a - * matching cgroup_kn_unlock() invocation. If @drain_offline is %true, the + * alive; otherwise, an ERR_PTR value. A successful return should be undone by + * a matching cgroup_kn_unlock() invocation. If @drain_offline is %true, the * cgroup is drained of offlining csses before return. * * Any cgroup kernfs method implementation which requires locking the @@ -1710,7 +1710,7 @@ struct cgroup *cgroup_kn_lock_live(struct kernfs_node *kn, bool drain_offline) * break the active_ref protection. */ if (!cgroup_tryget(cgrp)) - return NULL; + return ERR_PTR(-ENODEV); kernfs_break_active_protection(kn); if (drain_offline) @@ -1722,7 +1722,7 @@ struct cgroup *cgroup_kn_lock_live(struct kernfs_node *kn, bool drain_offline) return cgrp; cgroup_kn_unlock(kn); - return NULL; + return ERR_PTR(-ENODEV); } static void cgroup_rm_file(struct cgroup *cgrp, const struct cftype *cft) @@ -3650,8 +3650,8 @@ static ssize_t cgroup_subtree_control_write(struct kernfs_open_file *of, } cgrp = cgroup_kn_lock_live(of->kn, true); - if (!cgrp) - return -ENODEV; + if (IS_ERR(cgrp)) + return PTR_ERR(cgrp); for_each_subsys(ss, ssid) { if (enable & (1 << ssid)) { @@ -3790,8 +3790,8 @@ static ssize_t cgroup_type_write(struct kernfs_open_file *of, char *buf, /* drain dying csses before we re-apply (threaded) subtree control */ cgrp = cgroup_kn_lock_live(of->kn, true); - if (!cgrp) - return -ENODEV; + if (IS_ERR(cgrp)) + return PTR_ERR(cgrp); /* threaded can only be enabled */ ret = cgroup_enable_threaded(cgrp); @@ -3833,8 +3833,8 @@ static ssize_t cgroup_max_descendants_write(struct kernfs_open_file *of, return -ERANGE; cgrp = cgroup_kn_lock_live(of->kn, false); - if (!cgrp) - return -ENODEV; + if (IS_ERR(cgrp)) + return PTR_ERR(cgrp); WRITE_ONCE(cgrp->max_descendants, descendants); @@ -3876,8 +3876,8 @@ static ssize_t cgroup_max_depth_write(struct kernfs_open_file *of, return -ERANGE; cgrp = cgroup_kn_lock_live(of->kn, false); - if (!cgrp) - return -ENODEV; + if (IS_ERR(cgrp)) + return PTR_ERR(cgrp); WRITE_ONCE(cgrp->max_depth, depth); @@ -4075,8 +4075,8 @@ static ssize_t pressure_write(struct kernfs_open_file *of, char *buf, ssize_t ret = 0; cgrp = cgroup_kn_lock_live(of->kn, false); - if (!cgrp) - return -ENODEV; + if (IS_ERR(cgrp)) + return PTR_ERR(cgrp); ctx = of->priv; if (!ctx) { @@ -4192,8 +4192,8 @@ static ssize_t cgroup_pressure_write(struct kernfs_open_file *of, return -ERANGE; cgrp = cgroup_kn_lock_live(of->kn, false); - if (!cgrp) - return -ENODEV; + if (IS_ERR(cgrp)) + return PTR_ERR(cgrp); psi = cgroup_psi(cgrp); if (psi->enabled != enable) { @@ -4268,8 +4268,8 @@ static ssize_t cgroup_freeze_write(struct kernfs_open_file *of, return -ERANGE; cgrp = cgroup_kn_lock_live(of->kn, false); - if (!cgrp) - return -ENODEV; + if (IS_ERR(cgrp)) + return PTR_ERR(cgrp); cgroup_freeze(cgrp, freeze); @@ -4330,8 +4330,8 @@ static ssize_t cgroup_kill_write(struct kernfs_open_file *of, char *buf, return -ERANGE; cgrp = cgroup_kn_lock_live(of->kn, false); - if (!cgrp) - return -ENODEV; + if (IS_ERR(cgrp)) + return PTR_ERR(cgrp); /* * Killing is a process directed operation, i.e. the whole thread-group @@ -5485,8 +5485,8 @@ static ssize_t __cgroup_procs_write(struct kernfs_open_file *of, char *buf, enum cgroup_attach_lock_mode lock_mode; dst_cgrp = cgroup_kn_lock_live(of->kn, false); - if (!dst_cgrp) - return -ENODEV; + if (IS_ERR(dst_cgrp)) + return PTR_ERR(dst_cgrp); task = cgroup_procs_write_start(buf, threadgroup, &lock_mode); ret = PTR_ERR_OR_ZERO(task); @@ -6120,8 +6120,8 @@ int cgroup_mkdir(struct kernfs_node *parent_kn, const char *name, umode_t mode) return -EINVAL; parent = cgroup_kn_lock_live(parent_kn, false); - if (!parent) - return -ENODEV; + if (IS_ERR(parent)) + return PTR_ERR(parent); if (!cgroup_check_hierarchy_limits(parent)) { ret = -EAGAIN; @@ -6397,7 +6397,7 @@ int cgroup_rmdir(struct kernfs_node *kn) int ret = 0; cgrp = cgroup_kn_lock_live(kn, false); - if (!cgrp) + if (IS_ERR(cgrp)) return 0; ret = cgroup_destroy_locked(cgrp); diff --git a/kernel/cgroup/debug.c b/kernel/cgroup/debug.c index 883347b87842..96004cd65d09 100644 --- a/kernel/cgroup/debug.c +++ b/kernel/cgroup/debug.c @@ -45,7 +45,7 @@ static int current_css_set_read(struct seq_file *seq, void *v) struct cgroup_subsys_state *css; int i, refcnt; - if (!cgroup_kn_lock_live(of->kn, false)) + if (IS_ERR(cgroup_kn_lock_live(of->kn, false))) return -ENODEV; spin_lock_irq(&css_set_lock); @@ -206,8 +206,8 @@ static int cgroup_subsys_states_read(struct seq_file *seq, void *v) int i; cgrp = cgroup_kn_lock_live(of->kn, false); - if (!cgrp) - return -ENODEV; + if (IS_ERR(cgrp)) + return PTR_ERR(cgrp); for_each_subsys(ss, i) { css = rcu_dereference_check(cgrp->subsys[ss->id], true); @@ -254,8 +254,8 @@ static int cgroup_masks_read(struct seq_file *seq, void *v) struct cgroup *cgrp; cgrp = cgroup_kn_lock_live(of->kn, false); - if (!cgrp) - return -ENODEV; + if (IS_ERR(cgrp)) + return PTR_ERR(cgrp); cgroup_masks_read_one(seq, "subtree_control", cgrp->subtree_control); cgroup_masks_read_one(seq, "subtree_ss_mask", cgrp->subtree_ss_mask); -- cgit v1.2.3 From c367ce5b31ffd4d958f3f6c5fe1c9e820dc9f526 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 2 Sep 2026 13:03:13 -1000 Subject: cgroup: Make the offline drain interruptible cgroup_subtree_control_write() and cgroup_type_write() drain dying csses from the subtree before re-applying control, so that a controller being re-enabled gets a fresh css instead of adopting one that is still dying. The drain waits for offline_css() in TASK_UNINTERRUPTIBLE. Since 1dffd95575eb ("cgroup: Defer kill_css_finish() in cgroup_apply_control_disable()"), a disabled css offlines only after every task that still pins it has finished exiting. The root cause of the hang is an indefinite wait in a place where indefinite waits should not happen: a task past exit_signals() can block in its exit path for as long as, say, a FUSE daemon takes to answer a flush, and nothing can interrupt it because prepare_signal() drops every signal for an exiting group. The drain then escalated that into an uninterruptible wait for the writer, which sits in D state for as long as the exiting task does. That is the hang syzbot reports as "task hung in cgroup_subtree_control_write". Make the drain interruptible. Nothing has been modified when it runs, so the write restarts or fails with EINTR without side effects. cgroup1 mount and remount drain the same way and become interruptible too. cgroup_destroy_root() drains from a workqueue where no signal is ever pending and keeps waiting as before. This only stops the escalation. The write still waits for the exiting tasks, and the indefinite wait in the exit path remains a separate problem. Fixes: 1dffd95575eb ("cgroup: Defer kill_css_finish() in cgroup_apply_control_disable()") Reported-by: syzbot+bb2e19a1190a556c01b1@syzkaller.appspotmail.com Link: https://lore.kernel.org/all/6a23a4b4.e4db5ad2.3b7dfb.0000.GAE@google.com/ Signed-off-by: Tejun Heo --- kernel/cgroup/cgroup-internal.h | 2 +- kernel/cgroup/cgroup-v1.c | 8 +++++-- kernel/cgroup/cgroup.c | 48 ++++++++++++++++++++++++++++------------- 3 files changed, 40 insertions(+), 18 deletions(-) (limited to 'kernel/cgroup') diff --git a/kernel/cgroup/cgroup-internal.h b/kernel/cgroup/cgroup-internal.h index 58797123b752..f2684f9f8e0a 100644 --- a/kernel/cgroup/cgroup-internal.h +++ b/kernel/cgroup/cgroup-internal.h @@ -254,7 +254,7 @@ void cgroup_procs_write_finish(struct task_struct *task, enum cgroup_attach_lock_mode lock_mode) __releases(&cgroup_threadgroup_rwsem); -void cgroup_lock_and_drain_offline(struct cgroup *cgrp); +int cgroup_lock_and_drain_offline(struct cgroup *cgrp); int cgroup_mkdir(struct kernfs_node *parent_kn, const char *name, umode_t mode); int cgroup_rmdir(struct kernfs_node *kn); diff --git a/kernel/cgroup/cgroup-v1.c b/kernel/cgroup/cgroup-v1.c index 7e008867f3ac..167bf6555a49 100644 --- a/kernel/cgroup/cgroup-v1.c +++ b/kernel/cgroup/cgroup-v1.c @@ -1097,7 +1097,9 @@ int cgroup1_reconfigure(struct fs_context *fc) int ret = 0; u32 added_mask, removed_mask; - cgroup_lock_and_drain_offline(&cgrp_dfl_root.cgrp); + ret = cgroup_lock_and_drain_offline(&cgrp_dfl_root.cgrp); + if (unlikely(ret)) + return ret; /* See what subsystems are wanted */ ret = check_cgroupfs_options(fc); @@ -1262,7 +1264,9 @@ int cgroup1_get_tree(struct fs_context *fc) if (!ns_capable(ctx->ns->user_ns, CAP_SYS_ADMIN)) return -EPERM; - cgroup_lock_and_drain_offline(&cgrp_dfl_root.cgrp); + ret = cgroup_lock_and_drain_offline(&cgrp_dfl_root.cgrp); + if (unlikely(ret)) + return ret; ret = cgroup1_root_to_use(fc); if (!ret && !percpu_ref_tryget_live(&ctx->root->cgrp.self.refcnt)) diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index d30816ee47ac..ab9746056104 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -1379,7 +1379,10 @@ static void cgroup_destroy_root(struct cgroup_root *root) trace_cgroup_destroy_root(root); - cgroup_lock_and_drain_offline(&cgrp_dfl_root.cgrp); + /* runs off a workqueue, no signal can interrupt the drain */ + ret = cgroup_lock_and_drain_offline(&cgrp_dfl_root.cgrp); + if (WARN_ON_ONCE(ret)) + cgroup_lock(); BUG_ON(atomic_read(&root->nr_cgrps)); BUG_ON(!list_empty(&cgrp->self.children)); @@ -1687,7 +1690,8 @@ void cgroup_kn_unlock(struct kernfs_node *kn) * verifies that the associated cgroup is alive. Returns the cgroup if * alive; otherwise, an ERR_PTR value. A successful return should be undone by * a matching cgroup_kn_unlock() invocation. If @drain_offline is %true, the - * cgroup is drained of offlining csses before return. + * cgroup is drained of offlining csses before return, and an interrupted drain + * fails with -ERESTARTSYS. * * Any cgroup kernfs method implementation which requires locking the * associated cgroup should use this helper. It avoids nesting cgroup @@ -1697,6 +1701,7 @@ void cgroup_kn_unlock(struct kernfs_node *kn) struct cgroup *cgroup_kn_lock_live(struct kernfs_node *kn, bool drain_offline) { struct cgroup *cgrp; + int ret; if (kernfs_type(kn) == KERNFS_DIR) cgrp = kn->priv; @@ -1713,10 +1718,16 @@ struct cgroup *cgroup_kn_lock_live(struct kernfs_node *kn, bool drain_offline) return ERR_PTR(-ENODEV); kernfs_break_active_protection(kn); - if (drain_offline) - cgroup_lock_and_drain_offline(cgrp); - else + if (drain_offline) { + ret = cgroup_lock_and_drain_offline(cgrp); + if (unlikely(ret)) { + kernfs_unbreak_active_protection(kn); + cgroup_put(cgrp); + return ERR_PTR(ret); + } + } else { cgroup_lock(); + } if (!cgroup_is_dead(cgrp)) return cgrp; @@ -3320,16 +3331,20 @@ out_finish: * @cgrp: root of the target subtree * * Because css offlining is asynchronous, userland may try to re-enable a - * controller while the previous css is still around. This function grabs - * cgroup_mutex and drains the previous css instances of @cgrp's subtree. + * controller while the previous css is still around. This function grabs + * cgroup_mutex and waits until no css in @cgrp's subtree is dying. A dying css + * offlines only after every task that still pins it has finished exiting, which + * can take arbitrarily long, so the wait is interruptible. + * + * Returns 0 with cgroup_mutex held once the subtree is drained, or -ERESTARTSYS + * without it if interrupted by a signal. */ -void cgroup_lock_and_drain_offline(struct cgroup *cgrp) - __acquires(&cgroup_mutex) +int cgroup_lock_and_drain_offline(struct cgroup *cgrp) { struct cgroup *dsct; struct cgroup_subsys_state *d_css; struct cgroup_subsys *ss; - int ssid; + int ssid, ret; restart: cgroup_lock(); @@ -3343,17 +3358,20 @@ restart: continue; cgroup_get_live(dsct); - prepare_to_wait(&dsct->offline_waitq, &wait, - TASK_UNINTERRUPTIBLE); - + ret = prepare_to_wait_event(&dsct->offline_waitq, &wait, + TASK_INTERRUPTIBLE); cgroup_unlock(); - schedule(); + if (!ret) + schedule(); finish_wait(&dsct->offline_waitq, &wait); - cgroup_put(dsct); + if (unlikely(ret)) + return ret; goto restart; } } + + return 0; } /** -- cgit v1.2.3 From de99a40c54a6fe009e526657cf49790f7e8fcea0 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Fri, 11 Sep 2026 12:19:00 +0200 Subject: cgroup: Move cgrp_dead_ task+iwork into its own struct MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cgrp_dead_tasks and cgrp_dead_tasks_iwork are independent per-CPU variables and are independently accessed via a this_cpu_ptr(). Having a custom struct with those two members makes it possible to have only one per-CPU accessor and access the second member via an offset. This makes the code a bit more compact and is a micro-optimization. Besides that, it allows cgrp_dead_tasks_iwork_fn() to access the list pointer via the passed iwork pointer. This not only eliminates the this_cpu_ptr() but also makes it possible to be invoked on a different CPU. This does not happen as of today but might if I can flush the irq_work items from another CPU during CPU-hotplug events. Create struct cgroup_dead with the per-CPU variables as members. Access the struct with a per-CPU accessor and use container_of() in the irq_work callback. Signed-off-by: Sebastian Andrzej Siewior Acked-by: Michal Koutný Signed-off-by: Tejun Heo --- kernel/cgroup/cgroup.c | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) (limited to 'kernel/cgroup') diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index ab9746056104..16c0362330ac 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -7212,15 +7212,19 @@ static void do_cgroup_task_dead(struct task_struct *tsk) * the cgroup and task_struct can be pinned indefinitely. Bounce through lazy * irq_work to allow batching while ensuring timely completion. */ -static DEFINE_PER_CPU(struct llist_head, cgrp_dead_tasks); -static DEFINE_PER_CPU(struct irq_work, cgrp_dead_tasks_iwork); +struct cgroup_dead { + struct irq_work iwork; + struct llist_head tasks; +}; +static DEFINE_PER_CPU(struct cgroup_dead, cgroup_dead); static void cgrp_dead_tasks_iwork_fn(struct irq_work *iwork) { + struct cgroup_dead *cgrp_dead = container_of(iwork, struct cgroup_dead, iwork); struct llist_node *lnode; struct task_struct *task, *next; - lnode = llist_del_all(this_cpu_ptr(&cgrp_dead_tasks)); + lnode = llist_del_all(&cgrp_dead->tasks); llist_for_each_entry_safe(task, next, lnode, cg_dead_lnode) { do_cgroup_task_dead(task); put_task_struct(task); @@ -7232,17 +7236,20 @@ static void __init cgroup_rt_init(void) int cpu; for_each_possible_cpu(cpu) { - init_llist_head(per_cpu_ptr(&cgrp_dead_tasks, cpu)); - per_cpu(cgrp_dead_tasks_iwork, cpu) = - IRQ_WORK_INIT_LAZY(cgrp_dead_tasks_iwork_fn); + struct cgroup_dead *cgrp_dead = per_cpu_ptr(&cgroup_dead, cpu); + + init_llist_head(&cgrp_dead->tasks); + cgrp_dead->iwork = IRQ_WORK_INIT_LAZY(cgrp_dead_tasks_iwork_fn); } } void cgroup_task_dead(struct task_struct *task) { + struct cgroup_dead *cgrp_dead = this_cpu_ptr(&cgroup_dead); + get_task_struct(task); - llist_add(&task->cg_dead_lnode, this_cpu_ptr(&cgrp_dead_tasks)); - irq_work_queue(this_cpu_ptr(&cgrp_dead_tasks_iwork)); + llist_add(&task->cg_dead_lnode, &cgrp_dead->tasks); + irq_work_queue(&cgrp_dead->iwork); } #else /* CONFIG_PREEMPT_RT */ static void __init cgroup_rt_init(void) {} -- cgit v1.2.3 From e8f69fe59a104b4a97a3365dbb43c72996c2acde Mon Sep 17 00:00:00 2001 From: Karl Mehltretter Date: Sat, 12 Sep 2026 00:08:33 +0200 Subject: cgroup/dmem: Fix eviction helper reference in dmem_cgroup_try_charge() The kernel-doc comment of dmem_cgroup_try_charge() refers to dmem_cgroup_evict_valuable, which does not exist. The name is dmem_cgroup_state_evict_valuable. Say dmem_cgroup_state_evict_valuable. Fixes: b168ed458dde ("kernel/cgroup: Add "dmem" memory accounting cgroup") Assisted-by: LLM Signed-off-by: Karl Mehltretter Signed-off-by: Tejun Heo --- kernel/cgroup/dmem.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel/cgroup') diff --git a/kernel/cgroup/dmem.c b/kernel/cgroup/dmem.c index 4683f3d68022..10e148d64e85 100644 --- a/kernel/cgroup/dmem.c +++ b/kernel/cgroup/dmem.c @@ -706,8 +706,8 @@ EXPORT_SYMBOL_GPL(dmem_cgroup_uncharge); * * When this function fails with -EAGAIN and @ret_limit_pool is non-null, it * will be set to the pool for which the limit is hit. This can be used for - * eviction as argument to dmem_cgroup_evict_valuable(). This reference must be freed - * with @dmem_cgroup_pool_state_put(). + * eviction as argument to dmem_cgroup_state_evict_valuable(). This reference + * must be freed with @dmem_cgroup_pool_state_put(). * * Return: 0 on success, -EAGAIN on hitting a limit, or a negative errno on failure. */ -- cgit v1.2.3 From a408c6d27f96e401d6da2b69c5b6f0c31a48a923 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Fri, 18 Sep 2026 12:48:52 +0200 Subject: cgroup: Use %p for pointer formatting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since ad67b74d2469 ("printk: hash addresses printed with %p") %p is hashed by default, which is also what %pK prints under the default kptr_restrict=0. Converting the remaining %pK users makes it possible to remove its handling from vsprintf. Documentation/process/deprecated.rst asks that no new %p uses be added, but these pointers are what makes the css_set debug output useful, and debug.c already prints css pointers with %p. Keep the pointer output and use %p. Under kptr_restrict=1 and 2 the files now show the same hashed values as any other %p output: unprivileged readers see hashes instead of zeros, and a CAP_SYSLOG reader under kptr_restrict=1 no longer gets raw addresses. Booting with hash_pointers=never is the remaining way to see them. Signed-off-by: Sebastian Andrzej Siewior Acked-by: Michal Koutný Signed-off-by: Tejun Heo --- kernel/cgroup/debug.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'kernel/cgroup') diff --git a/kernel/cgroup/debug.c b/kernel/cgroup/debug.c index 96004cd65d09..aba243b5fd5f 100644 --- a/kernel/cgroup/debug.c +++ b/kernel/cgroup/debug.c @@ -51,7 +51,7 @@ static int current_css_set_read(struct seq_file *seq, void *v) spin_lock_irq(&css_set_lock); cset = task_css_set(current); refcnt = refcount_read(&cset->refcount); - seq_printf(seq, "css_set %pK %d", cset, refcnt); + seq_printf(seq, "css_set %p %d", cset, refcnt); if (refcnt > cset->nr_tasks) seq_printf(seq, " +%d", refcnt - cset->nr_tasks); seq_puts(seq, "\n"); @@ -125,10 +125,10 @@ static int cgroup_css_links_read(struct seq_file *seq, void *v) * Print out the proc_cset and threaded_cset relationship * and highlight difference between refcount and task_count. */ - seq_printf(seq, "css_set %pK", cset); + seq_printf(seq, "css_set %p", cset); if (rcu_dereference_protected(cset->dom_cset, 1) != cset) { threaded_csets++; - seq_printf(seq, "=>%pK", cset->dom_cset); + seq_printf(seq, "=>%p", cset->dom_cset); } if (!list_empty(&cset->threaded_csets)) { struct css_set *tcset; @@ -137,7 +137,7 @@ static int cgroup_css_links_read(struct seq_file *seq, void *v) list_for_each_entry(tcset, &cset->threaded_csets, threaded_csets_node) { seq_puts(seq, idx ? "," : "<="); - seq_printf(seq, "%pK", tcset); + seq_printf(seq, "%p", tcset); idx++; } } else { -- cgit v1.2.3 From 7dba07752511660ffc17dffa57b516259b3f4016 Mon Sep 17 00:00:00 2001 From: Peng Fan Date: Sun, 20 Sep 2026 10:27:37 +0800 Subject: cgroup: use assign_bit() where applicable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Convert open-coded if/else with set_bit/clear_bit and their non-atomic __set_bit/__clear_bit variants to the assign_bit/__assign_bit API. Done with Coccinelle semantic patch: // set_bit -> clear_bit => assign_bit @@ expression cond, bit, addr; @@ -if (cond) - set_bit(bit, addr); -else - clear_bit(bit, addr); +assign_bit(bit, addr, cond); // clear_bit -> set_bit => assign_bit @@ expression cond, bit, addr; @@ -if (cond) - clear_bit(bit, addr); -else - set_bit(bit, addr); +assign_bit(bit, addr, !cond); // __set_bit -> __clear_bit => __assign_bit @@ expression cond, bit, addr; @@ -if (cond) - __set_bit(bit, addr); -else - __clear_bit(bit, addr); +__assign_bit(bit, addr, cond); // __clear_bit -> __set_bit => __assign_bit @@ expression cond, bit, addr; @@ -if (cond) - __clear_bit(bit, addr); -else - __set_bit(bit, addr); +__assign_bit(bit, addr, !cond); Signed-off-by: Peng Fan Acked-by: Michal Koutný Signed-off-by: Tejun Heo --- kernel/cgroup/cgroup-v1.c | 10 ++-------- kernel/cgroup/cpuset.c | 16 ++++------------ kernel/cgroup/freezer.c | 5 +---- 3 files changed, 7 insertions(+), 24 deletions(-) (limited to 'kernel/cgroup') diff --git a/kernel/cgroup/cgroup-v1.c b/kernel/cgroup/cgroup-v1.c index 167bf6555a49..6825db74abf4 100644 --- a/kernel/cgroup/cgroup-v1.c +++ b/kernel/cgroup/cgroup-v1.c @@ -605,10 +605,7 @@ static u64 cgroup_read_notify_on_release(struct cgroup_subsys_state *css, static int cgroup_write_notify_on_release(struct cgroup_subsys_state *css, struct cftype *cft, u64 val) { - if (val) - set_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags); - else - clear_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags); + assign_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags, val); return 0; } @@ -621,10 +618,7 @@ static u64 cgroup_clone_children_read(struct cgroup_subsys_state *css, static int cgroup_clone_children_write(struct cgroup_subsys_state *css, struct cftype *cft, u64 val) { - if (val) - set_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags); - else - clear_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags); + assign_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags, val); return 0; } diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 2538faac9aba..32e04bc0da64 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -1216,10 +1216,7 @@ static void update_partition_sd_lb(struct cpuset *cs, int old_prs) } if (new_lb != !!is_sched_load_balance(cs)) { rebuild_domains = true; - if (new_lb) - set_bit(CS_SCHED_LOAD_BALANCE, &cs->flags); - else - clear_bit(CS_SCHED_LOAD_BALANCE, &cs->flags); + assign_bit(CS_SCHED_LOAD_BALANCE, &cs->flags, new_lb); } if (rebuild_domains) @@ -2308,10 +2305,8 @@ get_css: */ if (cpuset_v2() && !is_partition_valid(cp) && (is_sched_load_balance(parent) != is_sched_load_balance(cp))) { - if (is_sched_load_balance(parent)) - set_bit(CS_SCHED_LOAD_BALANCE, &cp->flags); - else - clear_bit(CS_SCHED_LOAD_BALANCE, &cp->flags); + assign_bit(CS_SCHED_LOAD_BALANCE, &cp->flags, + is_sched_load_balance(parent)); } /* @@ -2890,10 +2885,7 @@ int cpuset_update_flag(cpuset_flagbits_t bit, struct cpuset *cs, if (!trialcs) return -ENOMEM; - if (turning_on) - set_bit(bit, &trialcs->flags); - else - clear_bit(bit, &trialcs->flags); + assign_bit(bit, &trialcs->flags, turning_on); err = validate_change(cs, trialcs); if (err < 0) diff --git a/kernel/cgroup/freezer.c b/kernel/cgroup/freezer.c index 6c18854bff34..35112d62318a 100644 --- a/kernel/cgroup/freezer.c +++ b/kernel/cgroup/freezer.c @@ -20,10 +20,7 @@ static bool cgroup_update_frozen_flag(struct cgroup *cgrp, bool frozen) if (test_bit(CGRP_FROZEN, &cgrp->flags) == frozen) return false; - if (frozen) - set_bit(CGRP_FROZEN, &cgrp->flags); - else - clear_bit(CGRP_FROZEN, &cgrp->flags); + assign_bit(CGRP_FROZEN, &cgrp->flags, frozen); cgroup_file_notify(&cgrp->events_file); TRACE_CGROUP_PATH(notify_frozen, cgrp, frozen); -- cgit v1.2.3 From 024e82e1d39e2e868543d9bf3e2e28448a0ee7e4 Mon Sep 17 00:00:00 2001 From: Guopeng Zhang Date: Sun, 20 Sep 2026 10:52:56 +0800 Subject: cgroup/cpuset: Remove redundant sched domain rebuild from update_prstate() Both callers of update_prstate(), cpuset_partition_write() and cpuset_css_killed(), call cpuset_update_sd_hk_unlock() immediately afterwards. That helper rebuilds sched domains when force_sd_rebuild is set before releasing the cpuset locks, so the check in update_prstate() is redundant. Commit 3bfe47967191 ("cgroup/cpuset: Move housekeeping_update()/rebuild_sched_domains() together") removed the same check from cpuset_write_resmask(). Remove the remaining one from update_prstate(). Signed-off-by: Guopeng Zhang Reviewed-by: Ridong Chen Reviewed-by: Waiman Long Signed-off-by: Tejun Heo --- kernel/cgroup/cpuset.c | 2 -- 1 file changed, 2 deletions(-) (limited to 'kernel/cgroup') diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 32e04bc0da64..753aa65afcd7 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -3036,8 +3036,6 @@ out: update_partition_sd_lb(cs, old_prs); notify_partition_change(cs, old_prs); - if (force_sd_rebuild) - rebuild_sched_domains_locked(); free_tmpmasks(&tmpmask); return 0; } -- cgit v1.2.3 From 8a1c0f5c82843104d9bef2bc3726c89da70a9b91 Mon Sep 17 00:00:00 2001 From: Waiman Long Date: Sun, 27 Sep 2026 17:53:18 -0400 Subject: cgroup/cpuset: Run SCHED_DEADLINE shrink test on valid partition root only Commit f82f80426f7a ("sched/deadline: Ensure that updates to exclusive cpusets don't break AC") adds a check in validate_change() to make sure that there is enough bandwidth for SCHED_DEADLINE tasks if we shrink a v1 exclusive cpuset that has CS_CPU_EXCLUSIVE flag set. With the introduction of cpuset partition in cgroup v2, we keep setting the CS_CPU_EXCLUSIVE flag for a partition root so that the SCHED_DEADLINE check will continue to work as intended. However it turns out that the current code isn't perfect and there are cases where a cpuset isn't a valid partition root, but the exclusive flag is still incorrectly set. This can leads to SCHED_DEADLINE check being incorrectly triggered when there are deadline tasks in the system. This can result in unexpected -EBUSY failure when making changes to cpuset control files. Fix that by checking for a valid partition root for v2 and is_cpu_exclusive() for v1. It is far easier and less cumbersome than to make sure that the exclusive flag is only set for valid partition roots. Even though commit a86ce68078b2 ("cgroup/cpuset: Extract out CS_CPU_EXCLUSIVE & CS_SCHED_LOAD_BALANCE handling") is marked as a commit to be fixed, the problem may exist before that. Fixes: a86ce68078b2 ("cgroup/cpuset: Extract out CS_CPU_EXCLUSIVE & CS_SCHED_LOAD_BALANCE handling") Signed-off-by: Waiman Long Reviewed-by: Ridong Chen Reviewed-by: Guopeng Zhang Signed-off-by: Tejun Heo --- kernel/cgroup/cpuset.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'kernel/cgroup') diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 753aa65afcd7..104fe10c336d 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -771,8 +771,8 @@ static int validate_change(struct cpuset *cur, struct cpuset *trial) * For v1, effective_cpus == cpus_allowed & user_xcpus() returns * cpus_allowed. * - * For v2, is_cpu_exclusive() & is_sched_load_balance() are true only - * for non-isolated partition root. At this point, the target + * For v2, is_partition_valid(cur) & is_sched_load_balance() are true + * only for non-isolated partition root. At this point, the target * effective_cpus isn't computed yet. user_xcpus() is the best * approximation. * @@ -781,7 +781,8 @@ static int validate_change(struct cpuset *cur, struct cpuset *trial) * becomes an issue. */ ret = -EBUSY; - if (is_cpu_exclusive(cur) && is_sched_load_balance(cur) && + if ((is_partition_valid(cur) || (!cpuset_v2() && is_cpu_exclusive(cur))) && + is_sched_load_balance(cur) && !cpuset_cpumask_can_shrink(cur->effective_cpus, user_xcpus(trial))) goto out; -- cgit v1.2.3 From 85ad3963de964ed58e43b2ea7b6dea8b9a22adb3 Mon Sep 17 00:00:00 2001 From: Waiman Long Date: Sun, 27 Sep 2026 17:53:19 -0400 Subject: cgroup/cpuset: Remove CS_CPU_EXCLUSIVE handling code from v2 As the is_cpu_exclusive() function is only used for DEADLINE shrink test with v1 there is no need to have the CS_CPU_EXCLUSIVE flag set on valid partition root anymore. Remove update_partition_exclusive_flag() and other CS_CPU_EXCLUSIVE flag handling code in cpuset.c. Signed-off-by: Waiman Long Reviewed-by: Ridong Chen Reviewed-by: Guopeng Zhang Signed-off-by: Tejun Heo --- kernel/cgroup/cpuset.c | 52 ++++---------------------------------------------- 1 file changed, 4 insertions(+), 48 deletions(-) (limited to 'kernel/cgroup') diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 104fe10c336d..4a915da37bb3 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -1174,25 +1174,6 @@ enum partition_cmd { static void update_sibling_cpumasks(struct cpuset *parent, struct cpuset *cs, struct tmpmasks *tmp); -/* - * Update partition exclusive flag - * - * Return: 0 if successful, an error code otherwise - */ -static int update_partition_exclusive_flag(struct cpuset *cs, int new_prs) -{ - bool exclusive = (new_prs > PRS_MEMBER); - - if (exclusive && !is_cpu_exclusive(cs)) { - if (cpuset_update_flag(CS_CPU_EXCLUSIVE, cs, 1)) - return PERR_NOTEXCL; - } else if (!exclusive && is_cpu_exclusive(cs)) { - /* Turning off CS_CPU_EXCLUSIVE will not return error */ - cpuset_update_flag(CS_CPU_EXCLUSIVE, cs, 0); - } - return 0; -} - /* * Update partition load balance flag and/or rebuild sched domain * @@ -1248,11 +1229,9 @@ static void reset_partition_data(struct cpuset *cs) lockdep_assert_held(&callback_lock); - if (cpumask_empty(cs->exclusive_cpus)) { + if (cpumask_empty(cs->exclusive_cpus)) cpumask_clear(cs->effective_xcpus); - if (is_cpu_exclusive(cs)) - clear_bit(CS_CPU_EXCLUSIVE, &cs->flags); - } + if (!cpumask_and(cs->effective_cpus, parent->effective_cpus, cs->cpus_allowed)) cpumask_copy(cs->effective_cpus, parent->effective_cpus); } @@ -2028,19 +2007,6 @@ write_error: if (!adding && !deleting && (new_prs == old_prs)) return 0; - /* - * Transitioning between invalid to valid or vice versa may require - * changing CS_CPU_EXCLUSIVE. In the case of partcmd_update, - * validate_change() has already been successfully called and - * CPU lists in cs haven't been updated yet. So defer it to later. - */ - if ((old_prs != new_prs) && (cmd != partcmd_update)) { - int err = update_partition_exclusive_flag(cs, new_prs); - - if (err) - return err; - } - /* * Change the parent's effective_cpus & effective_xcpus (top cpuset * only). @@ -2063,9 +2029,6 @@ write_error: spin_unlock_irq(&callback_lock); - if ((old_prs != new_prs) && (cmd == partcmd_update)) - update_partition_exclusive_flag(cs, new_prs); - if (adding || deleting) { cpuset_update_tasks_cpumask(parent, tmp->addmask); update_sibling_cpumasks(parent, cs, tmp); @@ -2942,10 +2905,6 @@ static int update_prstate(struct cpuset *cs, int new_prs) if (alloc_tmpmasks(&tmpmask)) return -ENOMEM; - err = update_partition_exclusive_flag(cs, new_prs); - if (err) - goto out; - if (!old_prs) { /* * cpus_allowed and exclusive_cpus cannot be both empty. @@ -3009,13 +2968,10 @@ static int update_prstate(struct cpuset *cs, int new_prs) } out: /* - * Make partition invalid & disable CS_CPU_EXCLUSIVE if an error - * happens. + * Make partition invalid if an error happens. */ - if (err) { + if (err) new_prs = -new_prs; - update_partition_exclusive_flag(cs, new_prs); - } spin_lock_irq(&callback_lock); cs->partition_root_state = new_prs; -- cgit v1.2.3 From 5ffe20411787dd9947a35669ef8cb9484053efcf Mon Sep 17 00:00:00 2001 From: Waiman Long Date: Wed, 30 Sep 2026 11:48:00 -0400 Subject: cgroup/cpuset: Use cpuset_v2() in cpuset_num_cpus() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cpuset code has 2 different helpers for determining if it is in a v1 or v2 working environment - cpuset_v2() and is_in_v2_mode(). cpuset_v2() checks the binding of the cpuset subsystem is on a v1 or v2 hierarchy. It can also help to optimize out v1 specific code if CONFIG_CPUSETS_V1 isn't defined. is_in_v2_mode() layers on top of cpuset_v2() and returns true if it is in a v2 hierarchy or in a v1 hierarchy with "cpuset_v2_mode" mount option enabled to make "cpuset.cpus" and "cpuset.mems" behave in a way similar to their v2 counterparts. However the cpuset hierarchy may still be a standalone one separated from the other cgroup hierarchies. Commit 90ac22ffef48 ("sched/fair: Add cgroup_mode: max") adds a new cpuset_num_cpus() helper to return the number of effective CPUs in a given cgroup. It is part of the "sched: Flatten the pick" patch series to help flatten the cpu cgroup runqueue hierarchies. cpuset_num_cpus() currently uses is_in_v2_mode() to determine if it is in a v2 environment. However is_in_v2_mode() can be true in a v1 environment where the cpuset and cpu cgroup subsystems can be in two separate hierarchies. The current use case [1] for cpuset_num_cpus() assumes that it can return the number of effective CPUs present in the current cpu cgroup which may not be the case. Fix that by switching to use cpuset_v2() which signifies that it is in a unified hierarchy where both cpuset and cpu cgroup subsystems share the same cgroup hierarchy if it returns true. [1] https://lore.kernel.org/lkml/20260930081125.GF88198@noisy.programming.kicks-ass.net Fixes: 90ac22ffef48 ("sched/fair: Add cgroup_mode: max") Signed-off-by: Waiman Long Reviewed-by: Andrea Righi Reviewed-by: Michal Koutný Signed-off-by: Tejun Heo --- kernel/cgroup/cpuset.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel/cgroup') diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 4a915da37bb3..cc8ff0723495 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -4258,7 +4258,7 @@ int cpuset_num_cpus(struct cgroup *cgrp) int nr = num_online_cpus(); struct cpuset *cs; - if (is_in_v2_mode()) { + if (cpuset_v2()) { guard(rcu)(); cs = css_cs(cgroup_e_css(cgrp, &cpuset_cgrp_subsys)); if (cs) -- cgit v1.2.3 From 5a711e2fc9d48e54d7ad4279cfaf4cf8d084452f Mon Sep 17 00:00:00 2001 From: Ridong Chen Date: Wed, 30 Sep 2026 10:31:39 +0800 Subject: cgroup/cpuset: Move cpuset_update_flag() to cpuset-v1.c cpuset_update_flag() is only used by the legacy (v1) interface: its callers are the write handlers of cpuset1_files via cpuset_write_u64(), and cpuset_css_offline() which invokes it only on the legacy hierarchy (!cpuset_v2() && is_sched_load_balance()). Move it into cpuset-v1.c, where its sole helper cpuset1_update_tasks_flags() already lives, so both can go back to being static and drop out of cpuset-internal.h. As a result: - dup_or_alloc_cpuset(), free_cpuset() and validate_change() are now referenced from cpuset-v1.c, so they lose their static qualifier and gain declarations in cpuset-internal.h. - The v2 branch in cpuset_update_flag() that called cpuset_force_rebuild() is dead on the v1-only call paths and is dropped; the remaining rebuild_sched_domains_locked() path is kept. callback_lock, a static in cpuset.c, is taken through the existing cpuset_callback_lock_irq() wrapper. - The open-coded "!cpuset_v2() && is_sched_load_balance()" test in cpuset_css_offline() is wrapped in a new cpuset1_offline_css() helper (a no-op stub when CONFIG_CPUSETS_V1=n), removing the last cpuset_v2() branch from that path in cpuset.c. No functional change intended. Assisted-by: LLM Signed-off-by: Ridong Chen Reviewed-by: Waiman Long Signed-off-by: Tejun Heo --- kernel/cgroup/cpuset-internal.h | 8 +++-- kernel/cgroup/cpuset-v1.c | 59 ++++++++++++++++++++++++++++++++++- kernel/cgroup/cpuset.c | 68 +++-------------------------------------- 3 files changed, 67 insertions(+), 68 deletions(-) (limited to 'kernel/cgroup') diff --git a/kernel/cgroup/cpuset-internal.h b/kernel/cgroup/cpuset-internal.h index e7d010661fd3..5da9c91295b0 100644 --- a/kernel/cgroup/cpuset-internal.h +++ b/kernel/cgroup/cpuset-internal.h @@ -304,7 +304,9 @@ void cpuset_callback_lock_irq(void); void cpuset_callback_unlock_irq(void); void cpuset_update_tasks_cpumask(struct cpuset *cs, struct cpumask *new_cpus); void cpuset_update_tasks_nodemask(struct cpuset *cs); -int cpuset_update_flag(cpuset_flagbits_t bit, struct cpuset *cs, int turning_on); +struct cpuset *dup_or_alloc_cpuset(struct cpuset *cs); +void free_cpuset(struct cpuset *cs); +int validate_change(struct cpuset *cur, struct cpuset *trial); ssize_t cpuset_write_resmask(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off); int cpuset_common_seq_show(struct seq_file *sf, void *v); @@ -318,7 +320,6 @@ void cpuset_full_unlock(void); extern struct cftype cpuset1_files[]; void cpuset1_update_task_spread_flags(struct cpuset *cs, struct task_struct *tsk); -void cpuset1_update_tasks_flags(struct cpuset *cs); void cpuset1_hotplug_update_tasks(struct cpuset *cs, struct cpumask *new_cpus, nodemask_t *new_mems, bool cpus_updated, bool mems_updated); @@ -328,11 +329,11 @@ void cpuset1_init(struct cpuset *cs); void cpuset1_online_css(struct cgroup_subsys_state *css); int cpuset1_generate_sched_domains(cpumask_var_t **domains, struct sched_domain_attr **attributes); +void cpuset1_offline_css(struct cpuset *cs); #else static inline void cpuset1_update_task_spread_flags(struct cpuset *cs, struct task_struct *tsk) {} -static inline void cpuset1_update_tasks_flags(struct cpuset *cs) {} static inline void cpuset1_hotplug_update_tasks(struct cpuset *cs, struct cpumask *new_cpus, nodemask_t *new_mems, bool cpus_updated, bool mems_updated) {} @@ -344,6 +345,7 @@ static inline void cpuset1_init(struct cpuset *cs) {} static inline void cpuset1_online_css(struct cgroup_subsys_state *css) {} static inline int cpuset1_generate_sched_domains(cpumask_var_t **domains, struct sched_domain_attr **attributes) { return 0; }; +static inline void cpuset1_offline_css(struct cpuset *cs) {} #endif /* CONFIG_CPUSETS_V1 */ diff --git a/kernel/cgroup/cpuset-v1.c b/kernel/cgroup/cpuset-v1.c index 562ad35f00d0..7c7f2887dfaf 100644 --- a/kernel/cgroup/cpuset-v1.c +++ b/kernel/cgroup/cpuset-v1.c @@ -229,7 +229,7 @@ void cpuset1_update_task_spread_flags(struct cpuset *cs, * function is called with cpuset_mutex held, cpuset membership stays * stable. */ -void cpuset1_update_tasks_flags(struct cpuset *cs) +static void cpuset1_update_tasks_flags(struct cpuset *cs) { struct css_task_iter it; struct task_struct *task; @@ -473,6 +473,51 @@ static u64 cpuset_read_u64(struct cgroup_subsys_state *css, struct cftype *cft) return 0; } +/* + * cpuset_update_flag - read a 0 or a 1 in a file and update associated flag + * bit: the bit to update (see cpuset_flagbits_t) + * cs: the cpuset to update + * turning_on: whether the flag is being set or cleared + * + * Call with cpuset_mutex held. + */ +static int cpuset_update_flag(cpuset_flagbits_t bit, struct cpuset *cs, + int turning_on) +{ + struct cpuset *trialcs; + int balance_flag_changed; + int spread_page_changed; + int err; + + trialcs = dup_or_alloc_cpuset(cs); + if (!trialcs) + return -ENOMEM; + + assign_bit(bit, &trialcs->flags, turning_on); + + err = validate_change(cs, trialcs); + if (err < 0) + goto out; + + balance_flag_changed = (is_sched_load_balance(cs) != + is_sched_load_balance(trialcs)); + + spread_page_changed = is_spread_page(cs) != is_spread_page(trialcs); + + cpuset_callback_lock_irq(); + cs->flags = trialcs->flags; + cpuset_callback_unlock_irq(); + + if (!cpumask_empty(trialcs->cpus_allowed) && balance_flag_changed) + rebuild_sched_domains_locked(); + + if (spread_page_changed) + cpuset1_update_tasks_flags(cs); +out: + free_cpuset(trialcs); + return err; +} + static int cpuset_write_u64(struct cgroup_subsys_state *css, struct cftype *cft, u64 val) { @@ -581,6 +626,18 @@ void cpuset1_online_css(struct cgroup_subsys_state *css) cpuset_callback_unlock_irq(); } +/* + * Legacy-hierarchy handling when a cpuset is taken offline: if it had + * sched_load_balance enabled, turn it off so the scheduler domains are + * rebuilt. A no-op on the default hierarchy. + */ +void cpuset1_offline_css(struct cpuset *cs) +{ + if (!cgroup_subsys_on_dfl(cpuset_cgrp_subsys) && + is_sched_load_balance(cs)) + cpuset_update_flag(CS_SCHED_LOAD_BALANCE, cs, 0); +} + static void update_domain_attr(struct sched_domain_attr *dattr, struct cpuset *c) { diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index cc8ff0723495..aaf39d1ba112 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -162,7 +162,6 @@ static cpumask_var_t isolated_hk_cpus; /* T */ * It can be set in * - update_partition_sd_lb() * - update_cpumasks_hier() - * - cpuset_update_flag() * - cpuset_hotplug_update_tasks() * - cpuset_handle_hotplug() * @@ -608,7 +607,7 @@ static inline void free_tmpmasks(struct tmpmasks *tmp) * * Return: Pointer to newly allocated cpuset on success, NULL on failure */ -static struct cpuset *dup_or_alloc_cpuset(struct cpuset *cs) +struct cpuset *dup_or_alloc_cpuset(struct cpuset *cs) { struct cpuset *trial; @@ -649,7 +648,7 @@ static struct cpuset *dup_or_alloc_cpuset(struct cpuset *cs) * free_cpuset - free the cpuset * @cs: the cpuset to be freed */ -static inline void free_cpuset(struct cpuset *cs) +void free_cpuset(struct cpuset *cs) { free_cpumask_var(cs->cpus_allowed); free_cpumask_var(cs->effective_cpus); @@ -743,7 +742,7 @@ static inline bool mems_excl_conflict(struct cpuset *cs1, struct cpuset *cs2) * Return 0 if valid, -errno if not. */ -static int validate_change(struct cpuset *cur, struct cpuset *trial) +int validate_change(struct cpuset *cur, struct cpuset *trial) { struct cgroup_subsys_state *css; struct cpuset *c, *par; @@ -2828,56 +2827,6 @@ bool current_cpuset_is_being_rebound(void) return ret; } -/* - * cpuset_update_flag - read a 0 or a 1 in a file and update associated flag - * bit: the bit to update (see cpuset_flagbits_t) - * cs: the cpuset to update - * turning_on: whether the flag is being set or cleared - * - * Call with cpuset_mutex held. - */ - -int cpuset_update_flag(cpuset_flagbits_t bit, struct cpuset *cs, - int turning_on) -{ - struct cpuset *trialcs; - int balance_flag_changed; - int spread_page_changed; - int err; - - trialcs = dup_or_alloc_cpuset(cs); - if (!trialcs) - return -ENOMEM; - - assign_bit(bit, &trialcs->flags, turning_on); - - err = validate_change(cs, trialcs); - if (err < 0) - goto out; - - balance_flag_changed = (is_sched_load_balance(cs) != - is_sched_load_balance(trialcs)); - - spread_page_changed = is_spread_page(cs) != is_spread_page(trialcs); - - spin_lock_irq(&callback_lock); - cs->flags = trialcs->flags; - spin_unlock_irq(&callback_lock); - - if (!cpumask_empty(trialcs->cpus_allowed) && balance_flag_changed) { - if (cpuset_v2()) - cpuset_force_rebuild(); - else - rebuild_sched_domains_locked(); - } - - if (spread_page_changed) - cpuset1_update_tasks_flags(cs); -out: - free_cpuset(trialcs); - return err; -} - /** * update_prstate - update partition_root_state * @cs: the cpuset to update @@ -3631,21 +3580,12 @@ static int cpuset_css_online(struct cgroup_subsys_state *css) return 0; } -/* - * If the cpuset being removed has its flag 'sched_load_balance' - * enabled, then simulate turning sched_load_balance off, which - * will call rebuild_sched_domains_locked(). That is not needed - * in the default hierarchy where only changes in partition - * will cause repartitioning. - */ static void cpuset_css_offline(struct cgroup_subsys_state *css) { struct cpuset *cs = css_cs(css); cpuset_full_lock(); - if (!cpuset_v2() && is_sched_load_balance(cs)) - cpuset_update_flag(CS_SCHED_LOAD_BALANCE, cs, 0); - + cpuset1_offline_css(cs); cpuset_dec(); cpuset_full_unlock(); } -- cgit v1.2.3 From d5632ae015028376668ef29e3a24c099611a55c7 Mon Sep 17 00:00:00 2001 From: Peng Yu Date: Tue, 29 Sep 2026 16:24:10 -0700 Subject: cgroup: Track the effective css in each cgroup cgroup_e_css() and cgroup_get_e_css() find the effective css by walking up the hierarchy until they reach a cgroup that has the subsystem enabled. Per-I/O users such as nvmet need this lookup to be O(1). Track the effective css in each cgroup and make cgroup_e_css() and cgroup_get_e_css() use it. When a css is brought online or offline, update the effective css of its cgroup and of all descendants. Signed-off-by: Peng Yu Assisted-by: Claude:claude-fable-5 [Claude Code] Assisted-by: Claude:claude-opus-5-5 [Claude Code] Signed-off-by: Tejun Heo --- include/linux/cgroup-defs.h | 8 ++++++ kernel/cgroup/cgroup.c | 60 +++++++++++++++++++++++++++++---------------- 2 files changed, 47 insertions(+), 21 deletions(-) (limited to 'kernel/cgroup') diff --git a/include/linux/cgroup-defs.h b/include/linux/cgroup-defs.h index 3754d697854b..cbfe8937647c 100644 --- a/include/linux/cgroup-defs.h +++ b/include/linux/cgroup-defs.h @@ -555,6 +555,14 @@ struct cgroup { /* Private pointers for each registered subsystem */ struct cgroup_subsys_state __rcu *subsys[CGROUP_SUBSYS_COUNT]; + /* + * Effective css for each subsystem: the css of the nearest ancestor, + * including this cgroup, that has the subsystem enabled, or the root + * css if the subsystem isn't bound to this hierarchy. Updated under + * cgroup_mutex and read under RCU. + */ + struct cgroup_subsys_state __rcu *e_css[CGROUP_SUBSYS_COUNT]; + /* * Keep track of total number of dying CSSes at and below this cgroup. * Protected by cgroup_mutex. diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index 16c0362330ac..d4f3fcb2dbf6 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -548,20 +548,11 @@ static struct cgroup_subsys_state *cgroup_e_css_by_mask(struct cgroup *cgrp, struct cgroup_subsys_state *cgroup_e_css(struct cgroup *cgrp, struct cgroup_subsys *ss) { - struct cgroup_subsys_state *css; - if (!CGROUP_HAS_SUBSYS_CONFIG) return NULL; - do { - css = cgroup_css(cgrp, ss); - - if (css) - return css; - cgrp = cgroup_parent(cgrp); - } while (cgrp); - - return init_css_set.subsys[ss->id]; + return rcu_dereference_check(cgrp->e_css[ss->id], + lockdep_is_held(&cgroup_mutex)); } /** @@ -585,17 +576,10 @@ struct cgroup_subsys_state *cgroup_get_e_css(struct cgroup *cgrp, rcu_read_lock(); - do { - css = cgroup_css(cgrp, ss); - - if (css && css_tryget_online(css)) - goto out_unlock; - cgrp = cgroup_parent(cgrp); - } while (cgrp); + css = cgroup_e_css(cgrp, ss); + while (!css_tryget_online(css)) + css = cgroup_e_css(cgroup_parent(css->cgroup), ss); - css = init_css_set.subsys[ss->id]; - css_get(css); -out_unlock: rcu_read_unlock(); return css; } @@ -2142,12 +2126,24 @@ void init_cgroup_root(struct cgroup_fs_context *ctx) { struct cgroup_root *root = ctx->root; struct cgroup *cgrp = &root->cgrp; + struct cgroup_subsys *ss; + int ssid; INIT_LIST_HEAD_RCU(&root->root_list); atomic_set(&root->nr_cgrps, 1); cgrp->root = root; init_cgroup_housekeeping(cgrp); + /* + * A root cgroup's effective css is always the root css in + * init_css_set.subsys[], whichever hierarchy the subsystem is bound + * to, so rebind_subsystems() doesn't need to update e_css[]. For + * cgrp_dfl_root this runs before the root csses exist, and + * online_css() sets the entries when they come online. + */ + for_each_subsys(ss, ssid) + RCU_INIT_POINTER(cgrp->e_css[ssid], init_css_set.subsys[ssid]); + /* DYNMODS must be modified through cgroup_favor_dynmods() */ root->flags = ctx->flags & ~CGRP_ROOT_FAVOR_DYNMODS; if (ctx->release_agent) @@ -5874,6 +5870,22 @@ static void init_and_link_css(struct cgroup_subsys_state *css, BUG_ON(cgroup_css(cgrp, ss)); } +static void cgroup_update_e_css(struct cgroup *cgrp, struct cgroup_subsys *ss) +{ + struct cgroup_subsys_state *d_css; + + lockdep_assert_held(&cgroup_mutex); + + css_for_each_descendant_pre(d_css, &cgrp->self) { + struct cgroup *dsct = d_css->cgroup; + struct cgroup_subsys_state *css = cgroup_css(dsct, ss); + + if (!css) + css = cgroup_e_css(cgroup_parent(dsct), ss); + rcu_assign_pointer(dsct->e_css[ss->id], css); + } +} + /* invoke ->css_online() on a new CSS and mark it online if successful */ static int online_css(struct cgroup_subsys_state *css) { @@ -5887,6 +5899,7 @@ static int online_css(struct cgroup_subsys_state *css) if (!ret) { css->flags |= CSS_ONLINE; rcu_assign_pointer(css->cgroup->subsys[ss->id], css); + cgroup_update_e_css(css->cgroup, ss); atomic_inc(&css->online_cnt); if (css->parent) { @@ -5913,6 +5926,7 @@ static void offline_css(struct cgroup_subsys_state *css) css->flags &= ~CSS_ONLINE; RCU_INIT_POINTER(css->cgroup->subsys[ss->id], NULL); + cgroup_update_e_css(css->cgroup, ss); wake_up_all(&css->cgroup->offline_waitq); } @@ -5984,6 +5998,7 @@ static struct cgroup *cgroup_create(struct cgroup *parent, const char *name, { struct cgroup_root *root = parent->root; struct cgroup *cgrp, *tcgrp; + struct cgroup_subsys *ss; struct kernfs_node *kn; int i, level = parent->level + 1; int ret; @@ -6028,6 +6043,9 @@ static struct cgroup *cgroup_create(struct cgroup *parent, const char *name, for (tcgrp = cgrp; tcgrp; tcgrp = cgroup_parent(tcgrp)) cgrp->ancestors[tcgrp->level] = tcgrp; + for_each_subsys(ss, i) + RCU_INIT_POINTER(cgrp->e_css[i], cgroup_e_css(parent, ss)); + /* * New cgroup inherits effective freeze counter, and * if the parent has to be frozen, the child has too. -- cgit v1.2.3