diff options
| author | Pavel Begunkov <asml.silence@gmail.com> | 2021-03-22 01:58:29 +0000 |
|---|---|---|
| committer | Jens Axboe <axboe@kernel.dk> | 2021-04-11 17:41:59 -0600 |
| commit | e1d675df1a36e33e43c614e01d9f714618ac121e (patch) | |
| tree | 11e8cbc11b0c9b6df10155bb0540a63d52159031 /fs/io_uring.c | |
| parent | 8418f22a53795f4478a302aaec3d056795f56089 (diff) | |
| download | lwn-e1d675df1a36e33e43c614e01d9f714618ac121e.tar.gz lwn-e1d675df1a36e33e43c614e01d9f714618ac121e.zip | |
io_uring: don't init req->work fully in advance
req->work is mostly unused unless it's punted, and io_init_req() is too
hot for fully initialising it. Fortunately, we can skip init work.next
as it's controlled by io-wq, and can not touch work.flags by moving
everything related into io_prep_async_work(). The only field left is
req->work.creds, but there is nothing can be done, keep maintaining it.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
Diffstat (limited to 'fs/io_uring.c')
| -rw-r--r-- | fs/io_uring.c | 25 |
1 files changed, 14 insertions, 11 deletions
diff --git a/fs/io_uring.c b/fs/io_uring.c index 90b26221ba31..74e665953ab2 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1199,6 +1199,8 @@ static void io_prep_async_work(struct io_kiocb *req) if (!req->work.creds) req->work.creds = get_current_cred(); + req->work.list.next = NULL; + req->work.flags = 0; if (req->flags & REQ_F_FORCE_ASYNC) req->work.flags |= IO_WQ_WORK_CONCURRENT; @@ -1209,6 +1211,18 @@ static void io_prep_async_work(struct io_kiocb *req) if (def->unbound_nonreg_file) req->work.flags |= IO_WQ_WORK_UNBOUND; } + + switch (req->opcode) { + case IORING_OP_SPLICE: + case IORING_OP_TEE: + /* + * Splice operation will be punted aync, and here need to + * modify io_wq_work.flags, so initialize io_wq_work firstly. + */ + if (!S_ISREG(file_inode(req->splice.file_in)->i_mode)) + req->work.flags |= IO_WQ_WORK_UNBOUND; + break; + } } static void io_prep_async_link(struct io_kiocb *req) @@ -3593,15 +3607,6 @@ static int __io_splice_prep(struct io_kiocb *req, if (!sp->file_in) return -EBADF; req->flags |= REQ_F_NEED_CLEANUP; - - if (!S_ISREG(file_inode(sp->file_in)->i_mode)) { - /* - * Splice operation will be punted aync, and here need to - * modify io_wq_work.flags, so initialize io_wq_work firstly. - */ - req->work.flags |= IO_WQ_WORK_UNBOUND; - } - return 0; } @@ -6389,9 +6394,7 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, atomic_set(&req->refs, 2); req->task = current; req->result = 0; - req->work.list.next = NULL; req->work.creds = NULL; - req->work.flags = 0; /* enforce forwards compatibility on users */ if (unlikely(sqe_flags & ~SQE_VALID_FLAGS)) { |