From 31e33de1f4e159c7c6c44001fbd3398d53b0311d Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Tue, 1 Sep 2026 11:09:28 -1000 Subject: workqueue: Maintain pwq->total_in_flight pwq_busy() scans all of pwq->nr_in_flight[] to tell whether anything is in flight. Maintain the sum in pwq->total_in_flight, which fits in existing padding, and test that instead. Signed-off-by: Tejun Heo --- kernel/workqueue.c | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) (limited to 'kernel/workqueue.c') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 298f22c4caad..322de4f74e2f 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -275,6 +275,7 @@ struct pool_workqueue { int work_color; /* L: current color */ int flush_color; /* L: flushing color */ int refcnt; /* L: reference count */ + int total_in_flight; /* L: sum of nr_in_flight[] */ int nr_in_flight[WORK_NR_COLORS]; /* L: nr of in_flight works */ bool plugged; /* L: execution suspended */ @@ -2089,6 +2090,22 @@ static void pwq_dec_nr_active(struct pool_workqueue *pwq) node_activate_pending_pwq(nna, pool); } +/** + * pwq_inc_nr_in_flight - increment pwq's nr_in_flight + * @pwq: pwq of interest + * @work_color: color of the work item being queued + * + * A work item or a barrier with @work_color is being queued to @pwq. + * + * CONTEXT: + * raw_spin_lock_irq(pool->lock). + */ +static void pwq_inc_nr_in_flight(struct pool_workqueue *pwq, int work_color) +{ + pwq->nr_in_flight[work_color]++; + pwq->total_in_flight++; +} + /** * pwq_dec_nr_in_flight - decrement pwq's nr_in_flight * @pwq: pwq of interest @@ -2113,6 +2130,7 @@ static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, unsigned long work_ pwq_dec_nr_active(pwq); pwq->nr_in_flight[color]--; + pwq->total_in_flight--; /* is flush in progress and are we at the flushing tip? */ if (likely(pwq->flush_color != color)) @@ -2462,7 +2480,7 @@ retry: if (WARN_ON(!list_empty(&work->entry))) goto out; - pwq->nr_in_flight[pwq->work_color]++; + pwq_inc_nr_in_flight(pwq, pwq->work_color); work_flags = work_color_to_flags(pwq->work_color); /* @@ -4062,7 +4080,7 @@ static void insert_wq_barrier(struct pool_workqueue *pwq, __set_bit(WORK_STRUCT_LINKED_BIT, bits); } - pwq->nr_in_flight[work_color]++; + pwq_inc_nr_in_flight(pwq, work_color); work_flags |= work_color_to_flags(work_color); insert_work(pwq, &barr->work, head, work_flags); @@ -6129,11 +6147,8 @@ EXPORT_SYMBOL_GPL(alloc_workqueue_lockdep_map); static bool pwq_busy(struct pool_workqueue *pwq) { - int i; - - for (i = 0; i < WORK_NR_COLORS; i++) - if (pwq->nr_in_flight[i]) - return true; + if (pwq->total_in_flight) + return true; if ((pwq != rcu_access_pointer(pwq->wq->dfl_pwq)) && (pwq->refcnt > 1)) return true; -- cgit v1.2.3