From 88e41cf928a6e1a0eb5a9492e2d091ec6193cce4 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Mon, 22 Feb 2021 22:08:01 -0700 Subject: io_uring: add multishot mode for IORING_OP_POLL_ADD The default io_uring poll mode is one-shot, where once the event triggers, the poll command is completed and won't trigger any further events. If we're doing repeated polling on the same file or socket, then it can be more efficient to do multishot, where we keep triggering whenever the event becomes true. This deviates from the usual norm of having one CQE per SQE submitted. Add a CQE flag, IORING_CQE_F_MORE, which tells the application to expect further completion events from the submitted SQE. Right now the only user of this is POLL_ADD in multishot mode. Since sqe->poll_events is using the space that we normally use for adding flags to commands, use sqe->len for the flag space for POLL_ADD. Multishot mode is selected by setting IORING_POLL_ADD_MULTI in sqe->len. An application should expect more CQEs for the specificed SQE if the CQE is flagged with IORING_CQE_F_MORE. In multishot mode, only cancelation or an error will terminate the poll request, in which case the flag will be cleared. Signed-off-by: Jens Axboe --- include/uapi/linux/io_uring.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'include/uapi') diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index 2514eb6b1cf2..76c967621601 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -159,6 +159,16 @@ enum { */ #define SPLICE_F_FD_IN_FIXED (1U << 31) /* the last bit of __u32 */ +/* + * POLL_ADD flags. Note that since sqe->poll_events is the flag space, the + * command flags for POLL_ADD are stored in sqe->len. + * + * IORING_POLL_ADD_MULTI Multishot poll. Sets IORING_CQE_F_MORE if + * the poll handler will continue to report + * CQEs on behalf of the same SQE. + */ +#define IORING_POLL_ADD_MULTI (1U << 0) + /* * IO completion data structure (Completion Queue Entry) */ @@ -172,8 +182,10 @@ struct io_uring_cqe { * cqe->flags * * IORING_CQE_F_BUFFER If set, the upper 16 bits are the buffer ID + * IORING_CQE_F_MORE If set, parent SQE will generate more CQE entries */ #define IORING_CQE_F_BUFFER (1U << 0) +#define IORING_CQE_F_MORE (1U << 1) enum { IORING_CQE_BUFFER_SHIFT = 16, -- cgit v1.2.3 From b69de288e913030082bed3a324ddc58be6c1e983 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 17 Mar 2021 08:37:41 -0600 Subject: io_uring: allow events and user_data update of running poll requests This adds two new POLL_ADD flags, IORING_POLL_UPDATE_EVENTS and IORING_POLL_UPDATE_USER_DATA. As with the other POLL_ADD flag, these are masked into sqe->len. If set, the POLL_ADD will have the following behavior: - sqe->addr must contain the the user_data of the poll request that needs to be modified. This field is otherwise invalid for a POLL_ADD command. - If IORING_POLL_UPDATE_EVENTS is set, sqe->poll_events must contain the new mask for the existing poll request. There are no checks for whether these are identical or not, if a matching poll request is found, then it is re-armed with the new mask. - If IORING_POLL_UPDATE_USER_DATA is set, sqe->off must contain the new user_data for the existing poll request. A POLL_ADD with any of these flags set may complete with any of the following results: 1) 0, which means that we successfully found the existing poll request specified, and performed the re-arm procedure. Any error from that re-arm will be exposed as a completion event for that original poll request, not for the update request. 2) -ENOENT, if no existing poll request was found with the given user_data. 3) -EALREADY, if the existing poll request was already in the process of being removed/canceled/completing. 4) -EACCES, if an attempt was made to modify an internal poll request (eg not one originally issued ass IORING_OP_POLL_ADD). The usual -EINVAL cases apply as well, if any invalid fields are set in the sqe for this command type. Signed-off-by: Jens Axboe --- fs/io_uring.c | 95 +++++++++++++++++++++++++++++++++++++++---- include/uapi/linux/io_uring.h | 5 +++ 2 files changed, 92 insertions(+), 8 deletions(-) (limited to 'include/uapi') diff --git a/fs/io_uring.c b/fs/io_uring.c index 3c54e8c9f81f..eeb165253491 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -486,7 +486,15 @@ struct io_poll_iocb { __poll_t events; bool done; bool canceled; - struct wait_queue_entry wait; + bool update_events; + bool update_user_data; + union { + struct wait_queue_entry wait; + struct { + u64 old_user_data; + u64 new_user_data; + }; + }; }; struct io_poll_remove { @@ -4911,8 +4919,9 @@ static bool io_poll_complete(struct io_kiocb *req, __poll_t mask, int error) } if (!error) error = mangle_poll(mask); - if (!__io_cqring_fill_event(req, error, flags) || - (req->poll.events & EPOLLONESHOT)) { + if (req->poll.events & EPOLLONESHOT) + flags = 0; + if (!__io_cqring_fill_event(req, error, flags)) { io_poll_remove_waitqs(req); req->poll.done = true; flags = 0; @@ -4992,6 +5001,7 @@ static void io_init_poll_iocb(struct io_poll_iocb *poll, __poll_t events, poll->head = NULL; poll->done = false; poll->canceled = false; + poll->update_events = poll->update_user_data = false; #define IO_POLL_UNMASK (EPOLLERR|EPOLLHUP|EPOLLNVAL|EPOLLRDHUP) /* mask in events that we always want/need */ poll->events = events | IO_POLL_UNMASK; @@ -5370,24 +5380,36 @@ static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) return -EINVAL; - if (sqe->addr || sqe->ioprio || sqe->off || sqe->buf_index) + if (sqe->ioprio || sqe->buf_index) return -EINVAL; flags = READ_ONCE(sqe->len); - if (flags & ~IORING_POLL_ADD_MULTI) + if (flags & ~(IORING_POLL_ADD_MULTI | IORING_POLL_UPDATE_EVENTS | + IORING_POLL_UPDATE_USER_DATA)) return -EINVAL; - events = READ_ONCE(sqe->poll32_events); #ifdef __BIG_ENDIAN events = swahw32(events); #endif - if (!flags) + if (!(flags & IORING_POLL_ADD_MULTI)) events |= EPOLLONESHOT; + poll->update_events = poll->update_user_data = false; + if (flags & IORING_POLL_UPDATE_EVENTS) { + poll->update_events = true; + poll->old_user_data = READ_ONCE(sqe->addr); + } + if (flags & IORING_POLL_UPDATE_USER_DATA) { + poll->update_user_data = true; + poll->new_user_data = READ_ONCE(sqe->off); + } + if (!(poll->update_events || poll->update_user_data) && + (sqe->off || sqe->addr)) + return -EINVAL; poll->events = demangle_poll(events) | (events & (EPOLLEXCLUSIVE|EPOLLONESHOT)); return 0; } -static int io_poll_add(struct io_kiocb *req, unsigned int issue_flags) +static int __io_poll_add(struct io_kiocb *req) { struct io_poll_iocb *poll = &req->poll; struct io_ring_ctx *ctx = req->ctx; @@ -5413,6 +5435,63 @@ static int io_poll_add(struct io_kiocb *req, unsigned int issue_flags) return ipt.error; } +static int io_poll_update(struct io_kiocb *req) +{ + struct io_ring_ctx *ctx = req->ctx; + struct io_kiocb *preq; + int ret; + + spin_lock_irq(&ctx->completion_lock); + preq = io_poll_find(ctx, req->poll.old_user_data); + if (!preq) { + ret = -ENOENT; + goto err; + } else if (preq->opcode != IORING_OP_POLL_ADD) { + /* don't allow internal poll updates */ + ret = -EACCES; + goto err; + } + if (!__io_poll_remove_one(preq, &preq->poll)) { + /* in process of completing/removal */ + ret = -EALREADY; + goto err; + } + /* we now have a detached poll request. reissue. */ + ret = 0; +err: + spin_unlock_irq(&ctx->completion_lock); + if (ret < 0) { + req_set_fail_links(req); + io_req_complete(req, ret); + return 0; + } + /* only mask one event flags, keep behavior flags */ + if (req->poll.update_events) { + preq->poll.events &= ~0xffff; + preq->poll.events |= req->poll.events & 0xffff; + preq->poll.events |= IO_POLL_UNMASK; + } + if (req->poll.update_user_data) + preq->user_data = req->poll.new_user_data; + + /* complete update request, we're done with it */ + io_req_complete(req, ret); + + ret = __io_poll_add(preq); + if (ret < 0) { + req_set_fail_links(preq); + io_req_complete(preq, ret); + } + return 0; +} + +static int io_poll_add(struct io_kiocb *req, unsigned int issue_flags) +{ + if (!req->poll.update_events && !req->poll.update_user_data) + return __io_poll_add(req); + return io_poll_update(req); +} + static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer) { struct io_timeout_data *data = container_of(timer, diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index 76c967621601..5beaa6bbc6db 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -166,8 +166,13 @@ enum { * IORING_POLL_ADD_MULTI Multishot poll. Sets IORING_CQE_F_MORE if * the poll handler will continue to report * CQEs on behalf of the same SQE. + * + * IORING_POLL_UPDATE Update existing poll request, matching + * sqe->addr as the old user_data field. */ #define IORING_POLL_ADD_MULTI (1U << 0) +#define IORING_POLL_UPDATE_EVENTS (1U << 1) +#define IORING_POLL_UPDATE_USER_DATA (1U << 2) /* * IO completion data structure (Completion Queue Entry) -- cgit v1.2.3 From fdecb66281e165927059419c3b1de09ffe4f8369 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sun, 25 Apr 2021 14:32:20 +0100 Subject: io_uring: enumerate dynamic resources As resources are getting more support and common parts, it'll be more convenient to index resources and use it for indexing. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/f0be63e9310212d5601d36277c2946ff7a040485.1619356238.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 16 ++++++++-------- include/uapi/linux/io_uring.h | 4 ++++ 2 files changed, 12 insertions(+), 8 deletions(-) (limited to 'include/uapi') diff --git a/fs/io_uring.c b/fs/io_uring.c index 86a957d1059b..36b9651588f9 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1035,7 +1035,7 @@ static void io_dismantle_req(struct io_kiocb *req); static void io_put_task(struct task_struct *task, int nr); static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req); static void io_queue_linked_timeout(struct io_kiocb *req); -static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned opcode, +static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type, struct io_uring_rsrc_update *up, unsigned nr_args); static void io_clean_op(struct io_kiocb *req); @@ -5824,7 +5824,7 @@ static int io_files_update(struct io_kiocb *req, unsigned int issue_flags) up.data = req->rsrc_update.arg; mutex_lock(&ctx->uring_lock); - ret = __io_register_rsrc_update(ctx, IORING_REGISTER_FILES_UPDATE, + ret = __io_register_rsrc_update(ctx, IORING_RSRC_FILE, &up, req->rsrc_update.nr_args); mutex_unlock(&ctx->uring_lock); @@ -9709,7 +9709,7 @@ static int io_register_enable_rings(struct io_ring_ctx *ctx) return 0; } -static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned opcode, +static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type, struct io_uring_rsrc_update *up, unsigned nr_args) { @@ -9722,14 +9722,14 @@ static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned opcode, if (err) return err; - switch (opcode) { - case IORING_REGISTER_FILES_UPDATE: + switch (type) { + case IORING_RSRC_FILE: return __io_sqe_files_update(ctx, up, nr_args); } return -EINVAL; } -static int io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned opcode, +static int io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type, void __user *arg, unsigned nr_args) { struct io_uring_rsrc_update up; @@ -9740,7 +9740,7 @@ static int io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned opcode, return -EFAULT; if (up.resv) return -EINVAL; - return __io_register_rsrc_update(ctx, opcode, &up, nr_args); + return __io_register_rsrc_update(ctx, type, &up, nr_args); } static bool io_register_op_must_quiesce(int op) @@ -9829,7 +9829,7 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, ret = io_sqe_files_unregister(ctx); break; case IORING_REGISTER_FILES_UPDATE: - ret = io_register_rsrc_update(ctx, opcode, arg, nr_args); + ret = io_register_rsrc_update(ctx, IORING_RSRC_FILE, arg, nr_args); break; case IORING_REGISTER_EVENTFD: case IORING_REGISTER_EVENTFD_ASYNC: diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index 5beaa6bbc6db..d363e0c4fd21 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -316,6 +316,10 @@ struct io_uring_rsrc_update { __aligned_u64 data; }; +enum { + IORING_RSRC_FILE = 0, +}; + /* Skip updating fd indexes set to this value in the fd table */ #define IORING_REGISTER_FILES_SKIP (-2) -- cgit v1.2.3 From 792e35824be9af9fb4dac956229fb97bda04e25e Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sun, 25 Apr 2021 14:32:21 +0100 Subject: io_uring: add IORING_REGISTER_RSRC Add a new io_uring_register() opcode for rsrc registeration. Instead of accepting a pointer to resources, fds or iovecs, it @arg is now pointing to a struct io_uring_rsrc_register, and the second argument tells how large that struct is to make it easily extendible by adding new fields. All that is done mainly to be able to pass in a pointer with tags. Pass it in and enable CQE posting for file resources. Doesn't support setting tags on update yet. A design choice made here is to not post CQEs on rsrc de-registration, but only when we updated-removed it by rsrc dynamic update. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/c498aaec32a4bb277b2406b9069662c02cdda98c.1619356238.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 45 ++++++++++++++++++++++++++++++++++++++----- include/uapi/linux/io_uring.h | 8 ++++++++ 2 files changed, 48 insertions(+), 5 deletions(-) (limited to 'include/uapi') diff --git a/fs/io_uring.c b/fs/io_uring.c index 36b9651588f9..0088eeef82ec 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7589,7 +7589,7 @@ static struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx) } static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, - unsigned nr_args) + unsigned nr_args, u64 __user *tags) { __s32 __user *fds = (__s32 __user *) arg; struct file *file; @@ -7616,17 +7616,24 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, goto out_free; for (i = 0; i < nr_args; i++, ctx->nr_user_files++) { - if (copy_from_user(&fd, &fds[i], sizeof(fd))) { + u64 tag = 0; + + if ((tags && copy_from_user(&tag, &tags[i], sizeof(tag))) || + copy_from_user(&fd, &fds[i], sizeof(fd))) { ret = -EFAULT; goto out_fput; } /* allow sparse sets */ - if (fd == -1) + if (fd == -1) { + ret = -EINVAL; + if (unlikely(tag)) + goto out_fput; continue; + } file = fget(fd); ret = -EBADF; - if (!file) + if (unlikely(!file)) goto out_fput; /* @@ -7640,6 +7647,7 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, fput(file); goto out_fput; } + ctx->file_data->tags[i] = tag; io_fixed_file_set(io_fixed_file_slot(&ctx->file_table, i), file); } @@ -9743,6 +9751,29 @@ static int io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type, return __io_register_rsrc_update(ctx, type, &up, nr_args); } +static int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg, + unsigned int size) +{ + struct io_uring_rsrc_register rr; + + /* keep it extendible */ + if (size != sizeof(rr)) + return -EINVAL; + + memset(&rr, 0, sizeof(rr)); + if (copy_from_user(&rr, arg, size)) + return -EFAULT; + if (!rr.nr) + return -EINVAL; + + switch (rr.type) { + case IORING_RSRC_FILE: + return io_sqe_files_register(ctx, u64_to_user_ptr(rr.data), + rr.nr, u64_to_user_ptr(rr.tags)); + } + return -EINVAL; +} + static bool io_register_op_must_quiesce(int op) { switch (op) { @@ -9752,6 +9783,7 @@ static bool io_register_op_must_quiesce(int op) case IORING_REGISTER_PROBE: case IORING_REGISTER_PERSONALITY: case IORING_UNREGISTER_PERSONALITY: + case IORING_REGISTER_RSRC: return false; default: return true; @@ -9820,7 +9852,7 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, ret = io_sqe_buffers_unregister(ctx); break; case IORING_REGISTER_FILES: - ret = io_sqe_files_register(ctx, arg, nr_args); + ret = io_sqe_files_register(ctx, arg, nr_args, NULL); break; case IORING_UNREGISTER_FILES: ret = -EINVAL; @@ -9877,6 +9909,9 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, case IORING_REGISTER_RESTRICTIONS: ret = io_register_restrictions(ctx, arg, nr_args); break; + case IORING_REGISTER_RSRC: + ret = io_register_rsrc(ctx, arg, nr_args); + break; default: ret = -EINVAL; break; diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index d363e0c4fd21..ce7b2fce6713 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -298,6 +298,7 @@ enum { IORING_UNREGISTER_PERSONALITY = 10, IORING_REGISTER_RESTRICTIONS = 11, IORING_REGISTER_ENABLE_RINGS = 12, + IORING_REGISTER_RSRC = 13, /* this goes last */ IORING_REGISTER_LAST @@ -320,6 +321,13 @@ enum { IORING_RSRC_FILE = 0, }; +struct io_uring_rsrc_register { + __u32 type; + __u32 nr; + __aligned_u64 data; + __aligned_u64 tags; +}; + /* Skip updating fd indexes set to this value in the fd table */ #define IORING_REGISTER_FILES_SKIP (-2) -- cgit v1.2.3 From c3bdad0271834214be01c1d687c262bf80da6eb0 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sun, 25 Apr 2021 14:32:22 +0100 Subject: io_uring: add generic rsrc update with tags Add IORING_REGISTER_RSRC_UPDATE, which also supports passing in rsrc tags. Implement it for registered files. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/d4dc66df204212f64835ffca2c4eb5e8363f2f05.1619356238.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 52 ++++++++++++++++++++++++++++++++++--------- include/uapi/linux/io_uring.h | 22 +++++++++++++----- 2 files changed, 57 insertions(+), 17 deletions(-) (limited to 'include/uapi') diff --git a/fs/io_uring.c b/fs/io_uring.c index 0088eeef82ec..a4c37a2cc690 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1036,7 +1036,7 @@ static void io_put_task(struct task_struct *task, int nr); static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req); static void io_queue_linked_timeout(struct io_kiocb *req); static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type, - struct io_uring_rsrc_update *up, + struct io_uring_rsrc_update2 *up, unsigned nr_args); static void io_clean_op(struct io_kiocb *req); static struct file *io_file_get(struct io_submit_state *state, @@ -5814,7 +5814,7 @@ static int io_rsrc_update_prep(struct io_kiocb *req, static int io_files_update(struct io_kiocb *req, unsigned int issue_flags) { struct io_ring_ctx *ctx = req->ctx; - struct io_uring_rsrc_update up; + struct io_uring_rsrc_update2 up; int ret; if (issue_flags & IO_URING_F_NONBLOCK) @@ -5822,6 +5822,8 @@ static int io_files_update(struct io_kiocb *req, unsigned int issue_flags) up.offset = req->rsrc_update.offset; up.data = req->rsrc_update.arg; + up.nr = 0; + up.tags = 0; mutex_lock(&ctx->uring_lock); ret = __io_register_rsrc_update(ctx, IORING_RSRC_FILE, @@ -7732,9 +7734,10 @@ static int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx, } static int __io_sqe_files_update(struct io_ring_ctx *ctx, - struct io_uring_rsrc_update *up, + struct io_uring_rsrc_update2 *up, unsigned nr_args) { + u64 __user *tags = u64_to_user_ptr(up->tags); __s32 __user *fds = u64_to_user_ptr(up->data); struct io_rsrc_data *data = ctx->file_data; struct io_fixed_file *file_slot; @@ -7749,10 +7752,17 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, return -EINVAL; for (done = 0; done < nr_args; done++) { - if (copy_from_user(&fd, &fds[done], sizeof(fd))) { + u64 tag = 0; + + if ((tags && copy_from_user(&tag, &tags[done], sizeof(tag))) || + copy_from_user(&fd, &fds[done], sizeof(fd))) { err = -EFAULT; break; } + if ((fd == IORING_REGISTER_FILES_SKIP || fd == -1) && tag) { + err = -EINVAL; + break; + } if (fd == IORING_REGISTER_FILES_SKIP) continue; @@ -7787,6 +7797,7 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, err = -EBADF; break; } + data->tags[up->offset + done] = tag; io_fixed_file_set(file_slot, file); err = io_sqe_file_register(ctx, file, i); if (err) { @@ -9718,12 +9729,14 @@ static int io_register_enable_rings(struct io_ring_ctx *ctx) } static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type, - struct io_uring_rsrc_update *up, + struct io_uring_rsrc_update2 *up, unsigned nr_args) { __u32 tmp; int err; + if (up->resv) + return -EINVAL; if (check_add_overflow(up->offset, nr_args, &tmp)) return -EOVERFLOW; err = io_rsrc_node_switch_start(ctx); @@ -9737,18 +9750,31 @@ static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type, return -EINVAL; } -static int io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type, - void __user *arg, unsigned nr_args) +static int io_register_files_update(struct io_ring_ctx *ctx, void __user *arg, + unsigned nr_args) { - struct io_uring_rsrc_update up; + struct io_uring_rsrc_update2 up; if (!nr_args) return -EINVAL; + memset(&up, 0, sizeof(up)); + if (copy_from_user(&up, arg, sizeof(struct io_uring_rsrc_update))) + return -EFAULT; + return __io_register_rsrc_update(ctx, IORING_RSRC_FILE, &up, nr_args); +} + +static int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg, + unsigned size) +{ + struct io_uring_rsrc_update2 up; + + if (size != sizeof(up)) + return -EINVAL; if (copy_from_user(&up, arg, sizeof(up))) return -EFAULT; - if (up.resv) + if (!up.nr) return -EINVAL; - return __io_register_rsrc_update(ctx, type, &up, nr_args); + return __io_register_rsrc_update(ctx, up.type, &up, up.nr); } static int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg, @@ -9784,6 +9810,7 @@ static bool io_register_op_must_quiesce(int op) case IORING_REGISTER_PERSONALITY: case IORING_UNREGISTER_PERSONALITY: case IORING_REGISTER_RSRC: + case IORING_REGISTER_RSRC_UPDATE: return false; default: return true; @@ -9861,7 +9888,7 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, ret = io_sqe_files_unregister(ctx); break; case IORING_REGISTER_FILES_UPDATE: - ret = io_register_rsrc_update(ctx, IORING_RSRC_FILE, arg, nr_args); + ret = io_register_files_update(ctx, arg, nr_args); break; case IORING_REGISTER_EVENTFD: case IORING_REGISTER_EVENTFD_ASYNC: @@ -9912,6 +9939,9 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, case IORING_REGISTER_RSRC: ret = io_register_rsrc(ctx, arg, nr_args); break; + case IORING_REGISTER_RSRC_UPDATE: + ret = io_register_rsrc_update(ctx, arg, nr_args); + break; default: ret = -EINVAL; break; diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index ce7b2fce6713..6d8360b5b9c5 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -299,6 +299,7 @@ enum { IORING_REGISTER_RESTRICTIONS = 11, IORING_REGISTER_ENABLE_RINGS = 12, IORING_REGISTER_RSRC = 13, + IORING_REGISTER_RSRC_UPDATE = 14, /* this goes last */ IORING_REGISTER_LAST @@ -311,12 +312,6 @@ struct io_uring_files_update { __aligned_u64 /* __s32 * */ fds; }; -struct io_uring_rsrc_update { - __u32 offset; - __u32 resv; - __aligned_u64 data; -}; - enum { IORING_RSRC_FILE = 0, }; @@ -328,6 +323,21 @@ struct io_uring_rsrc_register { __aligned_u64 tags; }; +struct io_uring_rsrc_update { + __u32 offset; + __u32 resv; + __aligned_u64 data; +}; + +struct io_uring_rsrc_update2 { + __u32 offset; + __u32 resv; + __aligned_u64 data; + __aligned_u64 tags; + __u32 type; + __u32 nr; +}; + /* Skip updating fd indexes set to this value in the fd table */ #define IORING_REGISTER_FILES_SKIP (-2) -- cgit v1.2.3 From 634d00df5e1cfc4a707b629a814bd607f726bd52 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sun, 25 Apr 2021 14:32:26 +0100 Subject: io_uring: add full-fledged dynamic buffers support Hook buffers into all rsrc infrastructure, including tagging and updates. Suggested-by: Bijan Mottahedeh Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/119ed51d68a491dae87eb55fb467a47870c86aad.1619356238.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 76 ++++++++++++++++++++++++++++++++++++++++--- include/uapi/linux/io_uring.h | 1 + 2 files changed, 73 insertions(+), 4 deletions(-) (limited to 'include/uapi') diff --git a/fs/io_uring.c b/fs/io_uring.c index 0680d241fedc..0ab09f7a44e7 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -8114,8 +8114,8 @@ static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_mapped_ubuf **slo static void io_rsrc_buf_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc) { - /* no updates yet, so not used */ - WARN_ON_ONCE(1); + io_buffer_unmap(ctx, &prsrc->buf); + prsrc->buf = NULL; } static void __io_sqe_buffers_unregister(struct io_ring_ctx *ctx) @@ -8359,7 +8359,7 @@ static int io_buffer_validate(struct iovec *iov) } static int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, - unsigned int nr_args) + unsigned int nr_args, u64 __user *tags) { struct page *last_hpage = NULL; struct io_rsrc_data *data; @@ -8383,6 +8383,12 @@ static int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, } for (i = 0; i < nr_args; i++, ctx->nr_user_bufs++) { + u64 tag = 0; + + if (tags && copy_from_user(&tag, &tags[i], sizeof(tag))) { + ret = -EFAULT; + break; + } ret = io_copy_iov(ctx, &iov, arg, i); if (ret) break; @@ -8394,6 +8400,7 @@ static int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, &last_hpage); if (ret) break; + data->tags[i] = tag; } WARN_ON_ONCE(ctx->buf_data); @@ -8406,6 +8413,62 @@ static int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, return ret; } +static int __io_sqe_buffers_update(struct io_ring_ctx *ctx, + struct io_uring_rsrc_update2 *up, + unsigned int nr_args) +{ + u64 __user *tags = u64_to_user_ptr(up->tags); + struct iovec iov, __user *iovs = u64_to_user_ptr(up->data); + struct io_mapped_ubuf *imu; + struct page *last_hpage = NULL; + bool needs_switch = false; + __u32 done; + int i, err; + + if (!ctx->buf_data) + return -ENXIO; + if (up->offset + nr_args > ctx->nr_user_bufs) + return -EINVAL; + + for (done = 0; done < nr_args; done++) { + u64 tag = 0; + + err = io_copy_iov(ctx, &iov, iovs, done); + if (err) + break; + if (tags && copy_from_user(&tag, &tags[done], sizeof(tag))) { + err = -EFAULT; + break; + } + + i = array_index_nospec(up->offset + done, ctx->nr_user_bufs); + imu = ctx->user_bufs[i]; + if (imu) { + err = io_queue_rsrc_removal(ctx->buf_data, up->offset + done, + ctx->rsrc_node, imu); + if (err) + break; + ctx->user_bufs[i] = NULL; + needs_switch = true; + } + + if (iov.iov_base || iov.iov_len) { + err = io_buffer_validate(&iov); + if (err) + break; + err = io_sqe_buffer_register(ctx, &iov, &ctx->user_bufs[i], + &last_hpage); + if (err) + break; + ctx->buf_data->tags[up->offset + done] = tag; + } + } + + if (needs_switch) + io_rsrc_node_switch(ctx, ctx->buf_data); + return done ? done : err; +} + static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg) { __s32 __user *fds = arg; @@ -9807,6 +9870,8 @@ static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type, switch (type) { case IORING_RSRC_FILE: return __io_sqe_files_update(ctx, up, nr_args); + case IORING_RSRC_BUFFER: + return __io_sqe_buffers_update(ctx, up, nr_args); } return -EINVAL; } @@ -9857,6 +9922,9 @@ static int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg, case IORING_RSRC_FILE: return io_sqe_files_register(ctx, u64_to_user_ptr(rr.data), rr.nr, u64_to_user_ptr(rr.tags)); + case IORING_RSRC_BUFFER: + return io_sqe_buffers_register(ctx, u64_to_user_ptr(rr.data), + rr.nr, u64_to_user_ptr(rr.tags)); } return -EINVAL; } @@ -9933,7 +10001,7 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, switch (opcode) { case IORING_REGISTER_BUFFERS: - ret = io_sqe_buffers_register(ctx, arg, nr_args); + ret = io_sqe_buffers_register(ctx, arg, nr_args, NULL); break; case IORING_UNREGISTER_BUFFERS: ret = -EINVAL; diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index 6d8360b5b9c5..e1ae46683301 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -314,6 +314,7 @@ struct io_uring_files_update { enum { IORING_RSRC_FILE = 0, + IORING_RSRC_BUFFER = 1, }; struct io_uring_rsrc_register { -- cgit v1.2.3