diff options
| -rw-r--r-- | Documentation/block/biovecs.rst | 1 | ||||
| -rw-r--r-- | include/linux/bio.h | 6 | ||||
| -rw-r--r-- | include/linux/io_uring_types.h | 2 | ||||
| -rw-r--r-- | io_uring/io-wq.c | 5 | ||||
| -rw-r--r-- | io_uring/io_uring.c | 2 | ||||
| -rw-r--r-- | io_uring/kbuf.c | 5 | ||||
| -rw-r--r-- | io_uring/net.c | 1 | ||||
| -rw-r--r-- | io_uring/sqpoll.c | 7 |
8 files changed, 27 insertions, 2 deletions
diff --git a/Documentation/block/biovecs.rst b/Documentation/block/biovecs.rst index 11126ed6f40f..b9dc0c9dbee4 100644 --- a/Documentation/block/biovecs.rst +++ b/Documentation/block/biovecs.rst @@ -135,6 +135,7 @@ Usage of helpers: bio_first_bvec_all() bio_first_page_all() bio_first_folio_all() + bio_last_bvec_all() * The following helpers iterate over single-page segment. The passed 'struct bio_vec' will contain a single-page IO vector during the iteration:: diff --git a/include/linux/bio.h b/include/linux/bio.h index 8f33f717b14f..3a0e9cb6c2f6 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -256,6 +256,12 @@ static inline struct folio *bio_first_folio_all(struct bio *bio) return page_folio(bio_first_page_all(bio)); } +static inline struct bio_vec *bio_last_bvec_all(struct bio *bio) +{ + WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)); + return &bio->bi_io_vec[bio->bi_vcnt - 1]; +} + /** * struct folio_iter - State for iterating all folios in a bio. * @folio: The current folio we're iterating. NULL after the last folio. diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 87151a5b62c1..a2c623a67a25 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -534,6 +534,8 @@ struct io_ring_ctx { struct io_mapped_region ring_region; /* used for optimised request parameter and wait argument passing */ struct io_mapped_region param_region; + + struct kcov_common_handle_id kcov_handle; }; /* diff --git a/io_uring/io-wq.c b/io_uring/io-wq.c index 2e14880eef92..be8d75731d24 100644 --- a/io_uring/io-wq.c +++ b/io_uring/io-wq.c @@ -19,6 +19,7 @@ #include <linux/mmu_context.h> #include <linux/sched/sysctl.h> #include <uapi/linux/io_uring.h> +#include <linux/kcov.h> #include "io-wq.h" #include "slist.h" @@ -643,13 +644,17 @@ static void io_worker_handle_work(struct io_wq_acct *acct, unsigned int hash = __io_wq_is_hashed(work_flags) ? __io_get_work_hash(work_flags) : -1U; + struct io_kiocb *req; next_hashed = wq_next_work(work); if (do_kill && (work_flags & IO_WQ_WORK_UNBOUND)) atomic_or(IO_WQ_WORK_CANCEL, &work->flags); + req = container_of(work, struct io_kiocb, work); + kcov_remote_start_common(req->ctx->kcov_handle); io_wq_submit_work(work); + kcov_remote_stop(); io_assign_current_work(worker, NULL); linked = io_wq_free_work(work); diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 1ea2fca34a36..1279e27c2c6d 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -59,6 +59,7 @@ #include <linux/audit.h> #include <linux/security.h> #include <linux/jump_label.h> +#include <linux/kcov.h> #define CREATE_TRACE_POINTS #include <trace/events/io_uring.h> @@ -293,6 +294,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) INIT_HLIST_HEAD(&ctx->cancelable_uring_cmd); io_napi_init(ctx); mutex_init(&ctx->mmap_lock); + ctx->kcov_handle = kcov_common_handle(); return ctx; diff --git a/io_uring/kbuf.c b/io_uring/kbuf.c index de0129bceaba..1cf5be62bb65 100644 --- a/io_uring/kbuf.c +++ b/io_uring/kbuf.c @@ -266,6 +266,9 @@ static int io_ring_buffers_peek(struct io_kiocb *req, struct buf_sel_arg *arg, if (unlikely(!nr_avail)) return -ENOBUFS; + /* MAX_RW_COUNT is the universal Linux per-call IO maximum */ + arg->max_len = min_t(size_t, arg->max_len, MAX_RW_COUNT); + buf = io_ring_head_to_buf(br, head, bl->mask); if (arg->max_len) { u32 len = READ_ONCE(buf->len); @@ -295,7 +298,7 @@ static int io_ring_buffers_peek(struct io_kiocb *req, struct buf_sel_arg *arg, /* set it to max, if not set, so we can use it unconditionally */ if (!arg->max_len) - arg->max_len = INT_MAX; + arg->max_len = MAX_RW_COUNT; req->buf_index = READ_ONCE(buf->bid); do { diff --git a/io_uring/net.c b/io_uring/net.c index 00a7df803b99..a74d15f7b7d2 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -445,6 +445,7 @@ int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) req->flags |= REQ_F_NOWAIT; if (req->flags & REQ_F_BUFFER_SELECT) sr->buf_group = req->buf_index; + sr->mshot_total_len = sr->mshot_len = 0; if (sr->flags & IORING_RECVSEND_BUNDLE) { if (req->opcode == IORING_OP_SENDMSG) return -EINVAL; diff --git a/io_uring/sqpoll.c b/io_uring/sqpoll.c index 2460bd605266..ad42e8eb1002 100644 --- a/io_uring/sqpoll.c +++ b/io_uring/sqpoll.c @@ -13,6 +13,7 @@ #include <linux/cpuset.h> #include <linux/sched/cputime.h> #include <linux/io_uring.h> +#include <linux/kcov.h> #include <uapi/linux/io_uring.h> @@ -332,10 +333,14 @@ static int io_sq_thread(void *data) cap_entries = !list_is_singular(&sqd->ctx_list); list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) { - int ret = __io_sq_thread(ctx, sqd, cap_entries, &ist); + int ret; + + kcov_remote_start_common(ctx->kcov_handle); + ret = __io_sq_thread(ctx, sqd, cap_entries, &ist); if (!sqt_spin && (ret > 0 || !list_empty(&ctx->iopoll_list))) sqt_spin = true; + kcov_remote_stop(); } if (io_sq_tw(IORING_TW_CAP_ENTRIES_VALUE)) sqt_spin = true; |