diff options
| author | Usama Arif <usama.arif@bytedance.com> | 2022-02-04 14:51:15 +0000 |
|---|---|---|
| committer | Jens Axboe <axboe@kernel.dk> | 2022-03-10 06:32:49 -0700 |
| commit | c75312dd592b31427caa88170b61fdc3ae5b2891 (patch) | |
| tree | 1c0d7db092564493b06549991bcb9a099f0f29f2 /fs/io_uring.c | |
| parent | 77bc59b498174ea4f120d477d2cd0cf90fc58235 (diff) | |
| download | lwn-c75312dd592b31427caa88170b61fdc3ae5b2891.tar.gz lwn-c75312dd592b31427caa88170b61fdc3ae5b2891.zip | |
io_uring: avoid ring quiesce while registering async eventfd
This is done using the RCU data structure (io_ev_fd). eventfd_async is
moved from io_ring_ctx to io_ev_fd which is RCU protected hence avoiding
ring quiesce which is much more expensive than an RCU lock. The place
where eventfd_async is read is already under rcu_read_lock so there is no
extra RCU read-side critical section needed.
Signed-off-by: Usama Arif <usama.arif@bytedance.com>
Link: https://lore.kernel.org/r/20220204145117.1186568-4-usama.arif@bytedance.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
Diffstat (limited to 'fs/io_uring.c')
| -rw-r--r-- | fs/io_uring.c | 22 |
1 files changed, 12 insertions, 10 deletions
diff --git a/fs/io_uring.c b/fs/io_uring.c index 6c35e88be4ea..961866b24b4f 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -328,6 +328,7 @@ struct io_submit_state { struct io_ev_fd { struct eventfd_ctx *cq_ev_fd; + unsigned int eventfd_async: 1; struct rcu_head rcu; }; @@ -340,7 +341,6 @@ struct io_ring_ctx { unsigned int flags; unsigned int compat: 1; unsigned int drain_next: 1; - unsigned int eventfd_async: 1; unsigned int restricted: 1; unsigned int off_timeout_used: 1; unsigned int drain_active: 1; @@ -1756,7 +1756,7 @@ static void io_eventfd_signal(struct io_ring_ctx *ctx) if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED) goto out; - if (!ctx->eventfd_async || io_wq_current_is_worker()) + if (!ev_fd->eventfd_async || io_wq_current_is_worker()) eventfd_signal(ev_fd->cq_ev_fd, 1); out: @@ -9385,7 +9385,8 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx, return done ? done : err; } -static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg) +static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg, + unsigned int eventfd_async) { struct io_ev_fd *ev_fd; __s32 __user *fds = arg; @@ -9409,6 +9410,7 @@ static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg) kfree(ev_fd); return ret; } + ev_fd->eventfd_async = eventfd_async; rcu_assign_pointer(ctx->io_ev_fd, ev_fd); return ret; @@ -11014,6 +11016,7 @@ static bool io_register_op_must_quiesce(int op) case IORING_UNREGISTER_FILES: case IORING_REGISTER_FILES_UPDATE: case IORING_REGISTER_EVENTFD: + case IORING_REGISTER_EVENTFD_ASYNC: case IORING_UNREGISTER_EVENTFD: case IORING_REGISTER_PROBE: case IORING_REGISTER_PERSONALITY: @@ -11114,17 +11117,16 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, ret = io_register_files_update(ctx, arg, nr_args); break; case IORING_REGISTER_EVENTFD: - case IORING_REGISTER_EVENTFD_ASYNC: ret = -EINVAL; if (nr_args != 1) break; - ret = io_eventfd_register(ctx, arg); - if (ret) + ret = io_eventfd_register(ctx, arg, 0); + break; + case IORING_REGISTER_EVENTFD_ASYNC: + ret = -EINVAL; + if (nr_args != 1) break; - if (opcode == IORING_REGISTER_EVENTFD_ASYNC) - ctx->eventfd_async = 1; - else - ctx->eventfd_async = 0; + ret = io_eventfd_register(ctx, arg, 1); break; case IORING_UNREGISTER_EVENTFD: ret = -EINVAL; |