diff options
| author | Pavel Begunkov <asml.silence@gmail.com> | 2021-10-27 13:21:07 +0100 |
|---|---|---|
| committer | Jens Axboe <axboe@kernel.dk> | 2021-10-27 06:54:58 -0600 |
| commit | 1bb6b81029456f4e2e6727c5167f43bdfc34bee5 (patch) | |
| tree | 02ea0da43b7ad152375b9fb59cd2c90ed0acb1cf | |
| parent | a2247f19ee1c5ad75ef095cdfb909a3244b88aa8 (diff) | |
| download | lwn-1bb6b81029456f4e2e6727c5167f43bdfc34bee5.tar.gz lwn-1bb6b81029456f4e2e6727c5167f43bdfc34bee5.zip | |
block: avoid extra iter advance with async iocb
Nobody cares about iov iterators state if we return -EIOCBQUEUED, so as
the we now have __blkdev_direct_IO_async(), which gets pages only once,
we can skip expensive iov_iter_advance(). It's around 1-2% of all CPU
spent.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/a6158edfbfa2ae3bc24aed29a72f035df18fad2f.1635337135.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
| -rw-r--r-- | block/bio.c | 2 | ||||
| -rw-r--r-- | block/fops.c | 20 | ||||
| -rw-r--r-- | include/linux/bio.h | 1 |
3 files changed, 17 insertions, 6 deletions
diff --git a/block/bio.c b/block/bio.c index ead1f8a9ff5e..15ab0d6d1c06 100644 --- a/block/bio.c +++ b/block/bio.c @@ -1046,7 +1046,7 @@ void __bio_release_pages(struct bio *bio, bool mark_dirty) } EXPORT_SYMBOL_GPL(__bio_release_pages); -static void bio_iov_bvec_set(struct bio *bio, struct iov_iter *iter) +void bio_iov_bvec_set(struct bio *bio, struct iov_iter *iter) { size_t size = iov_iter_count(iter); diff --git a/block/fops.c b/block/fops.c index a7b328296912..092e5079e827 100644 --- a/block/fops.c +++ b/block/fops.c @@ -352,11 +352,21 @@ static ssize_t __blkdev_direct_IO_async(struct kiocb *iocb, bio->bi_end_io = blkdev_bio_end_io_async; bio->bi_ioprio = iocb->ki_ioprio; - ret = bio_iov_iter_get_pages(bio, iter); - if (unlikely(ret)) { - bio->bi_status = BLK_STS_IOERR; - bio_endio(bio); - return ret; + if (iov_iter_is_bvec(iter)) { + /* + * Users don't rely on the iterator being in any particular + * state for async I/O returning -EIOCBQUEUED, hence we can + * avoid expensive iov_iter_advance(). Bypass + * bio_iov_iter_get_pages() and set the bvec directly. + */ + bio_iov_bvec_set(bio, iter); + } else { + ret = bio_iov_iter_get_pages(bio, iter); + if (unlikely(ret)) { + bio->bi_status = BLK_STS_IOERR; + bio_endio(bio); + return ret; + } } dio->size = bio->bi_iter.bi_size; diff --git a/include/linux/bio.h b/include/linux/bio.h index c88700d1bdc3..fe6bdfbbef66 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -417,6 +417,7 @@ int bio_add_zone_append_page(struct bio *bio, struct page *page, void __bio_add_page(struct bio *bio, struct page *page, unsigned int len, unsigned int off); int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter); +void bio_iov_bvec_set(struct bio *bio, struct iov_iter *iter); void __bio_release_pages(struct bio *bio, bool mark_dirty); extern void bio_set_pages_dirty(struct bio *bio); extern void bio_check_pages_dirty(struct bio *bio); |