diff options
| author | Pavel Begunkov <asml.silence@gmail.com> | 2022-03-21 22:02:22 +0000 |
|---|---|---|
| committer | Jens Axboe <axboe@kernel.dk> | 2022-04-24 17:34:16 -0600 |
| commit | a538be5be328229d4da3343d4d6514bb4d5c3d5d (patch) | |
| tree | 786c3125d1fa4d04af507c3346fd28b939c3e982 /fs/io_uring.c | |
| parent | 7819a1f6ac0393f1f7861dfece6ffd5ef010f0f9 (diff) | |
| download | lwn-a538be5be328229d4da3343d4d6514bb4d5c3d5d.tar.gz lwn-a538be5be328229d4da3343d4d6514bb4d5c3d5d.zip | |
io_uring: optimise io_free_batch_list
We do several req->flags checks in the fast path of
io_free_batch_list(). One explicit check of REQ_F_REFCOUNT, and two
other hidden in io_queue_next() and io_dismantle_req(). Moreover, there
is a io_req_put_rsrc_locked() call in between, so there is no hope
req->flags will be preserved in registers.
All those flags if not a slow path than definitely a slower path, so
put them all under a single flags mask check and save several mem
reloads and ifs.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/0fb493f73f2009aea395c570c2932fecaa4e1244.1647897811.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
Diffstat (limited to 'fs/io_uring.c')
| -rw-r--r-- | fs/io_uring.c | 20 |
1 files changed, 14 insertions, 6 deletions
diff --git a/fs/io_uring.c b/fs/io_uring.c index 1859fff53ddf..fddfbf0f9310 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -113,6 +113,9 @@ #define IO_REQ_CLEAN_FLAGS (REQ_F_BUFFER_SELECTED | REQ_F_NEED_CLEANUP | \ REQ_F_POLLED | REQ_F_CREDS | REQ_F_ASYNC_DATA) +#define IO_REQ_CLEAN_SLOW_FLAGS (REQ_F_REFCOUNT | REQ_F_LINK | REQ_F_HARDLINK |\ + IO_REQ_CLEAN_FLAGS) + #define IO_TCTX_REFS_CACHE_NR (1U << 10) struct io_uring { @@ -2626,15 +2629,20 @@ static void io_free_batch_list(struct io_ring_ctx *ctx, struct io_kiocb *req = container_of(node, struct io_kiocb, comp_list); - if (unlikely(req->flags & REQ_F_REFCOUNT)) { - node = req->comp_list.next; - if (!req_ref_put_and_test(req)) - continue; + if (unlikely(req->flags & IO_REQ_CLEAN_SLOW_FLAGS)) { + if (req->flags & REQ_F_REFCOUNT) { + node = req->comp_list.next; + if (!req_ref_put_and_test(req)) + continue; + } + io_queue_next(req); + if (unlikely(req->flags & IO_REQ_CLEAN_FLAGS)) + io_clean_op(req); } + if (!(req->flags & REQ_F_FIXED_FILE)) + io_put_file(req->file); io_req_put_rsrc_locked(req, ctx); - io_queue_next(req); - io_dismantle_req(req); if (req->task != task) { if (task) |