diff options
| author | Jens Axboe <axboe@kernel.dk> | 2022-04-25 19:49:02 -0600 |
|---|---|---|
| committer | Jens Axboe <axboe@kernel.dk> | 2022-04-30 08:39:54 -0600 |
| commit | 9f010507bbc1be19dbeedc1a254209fea44adc14 (patch) | |
| tree | 1853223814ff1616df47663a544258258c20c903 /fs/io_uring.c | |
| parent | 6cf5862e3c2caafd11f39a373b98b00c5a04b83a (diff) | |
| download | lwn-9f010507bbc1be19dbeedc1a254209fea44adc14.tar.gz lwn-9f010507bbc1be19dbeedc1a254209fea44adc14.zip | |
io_uring: set task_work notify method at init time
While doing so, switch SQPOLL to TWA_SIGNAL_NO_IPI as well, as that
just does a task wakeup and then we can remove the special wakeup we
have in task_work_add.
Reviewed-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/20220426014904.60384-5-axboe@kernel.dk
Signed-off-by: Jens Axboe <axboe@kernel.dk>
Diffstat (limited to 'fs/io_uring.c')
| -rw-r--r-- | fs/io_uring.c | 23 |
1 files changed, 11 insertions, 12 deletions
diff --git a/fs/io_uring.c b/fs/io_uring.c index e92029bd3936..3c669d8f5e57 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -366,6 +366,7 @@ struct io_ring_ctx { struct io_rings *rings; unsigned int flags; + enum task_work_notify_mode notify_method; unsigned int compat: 1; unsigned int drain_next: 1; unsigned int restricted: 1; @@ -2621,8 +2622,8 @@ static void tctx_task_work(struct callback_head *cb) static void io_req_task_work_add(struct io_kiocb *req, bool priority) { struct task_struct *tsk = req->task; + struct io_ring_ctx *ctx = req->ctx; struct io_uring_task *tctx = tsk->io_uring; - enum task_work_notify_mode notify; struct io_wq_work_node *node; unsigned long flags; bool running; @@ -2645,18 +2646,8 @@ static void io_req_task_work_add(struct io_kiocb *req, bool priority) if (running) return; - /* - * SQPOLL kernel thread doesn't need notification, just a wakeup. For - * all other cases, use TWA_SIGNAL unconditionally to ensure we're - * processing task_work. There's no reliable way to tell if TWA_RESUME - * will do the job. - */ - notify = (req->ctx->flags & IORING_SETUP_SQPOLL) ? TWA_NONE : TWA_SIGNAL; - if (likely(!task_work_add(tsk, &tctx->task_work, notify))) { - if (notify == TWA_NONE) - wake_up_process(tsk); + if (likely(!task_work_add(tsk, &tctx->task_work, ctx->notify_method))) return; - } spin_lock_irqsave(&tctx->task_lock, flags); tctx->task_running = false; @@ -11336,6 +11327,14 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p, ctx->user = get_uid(current_user()); /* + * For SQPOLL, we just need a wakeup, always. + */ + if (ctx->flags & IORING_SETUP_SQPOLL) + ctx->notify_method = TWA_SIGNAL_NO_IPI; + else + ctx->notify_method = TWA_SIGNAL; + + /* * This is just grabbed for accounting purposes. When a process exits, * the mm is exited and dropped before the files, hence we need to hang * on to this mm purely for the purposes of being able to unaccount |