From f6fe52a7b18675d76d7f7dae0c16f412a4e33f9a Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 27 May 2026 17:10:22 +0200 Subject: bvec: make the bvec_iter helpers inline functions The macros are impossible to follow due to the lack of visual type information and all the braces. Replace them with inline helpers to improve on that. Because the calling conventions are a bit problematic with a lot of passing structures by value, all the helpers are marked as __always_inline so that they are force inlined. Signed-off-by: Christoph Hellwig Reviewed-by: Bart Van Assche Reviewed-by: Caleb Sander Mateos Reviewed-by: Ming Lei Reviewed-by: Chaitanya Kulkarni Reviewed-by: Keith Busch Link: https://patch.msgid.link/20260527151043.2349900-4-hch@lst.de Signed-off-by: Jens Axboe --- include/linux/bvec.h | 101 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 64 insertions(+), 37 deletions(-) (limited to 'include/linux/bvec.h') diff --git a/include/linux/bvec.h b/include/linux/bvec.h index d36dd476feda..f4c7ec282ac9 100644 --- a/include/linux/bvec.h +++ b/include/linux/bvec.h @@ -104,51 +104,78 @@ struct bvec_iter_all { unsigned done; }; -/* - * various member access, note that bio_data should of course not be used - * on highmem page vectors - */ -#define __bvec_iter_bvec(bvec, iter) (&(bvec)[(iter).bi_idx]) +static __always_inline const struct bio_vec * +__bvec_iter_bvec(const struct bio_vec *bvecs, const struct bvec_iter iter) +{ + return bvecs + iter.bi_idx; +} /* multi-page (mp_bvec) helpers */ -#define mp_bvec_iter_page(bvec, iter) \ - (__bvec_iter_bvec((bvec), (iter))->bv_page) +static __always_inline struct page * +mp_bvec_iter_page(const struct bio_vec *bvecs, const struct bvec_iter iter) +{ + return __bvec_iter_bvec(bvecs, iter)->bv_page; +} -#define mp_bvec_iter_len(bvec, iter) \ - min((iter).bi_size, \ - __bvec_iter_bvec((bvec), (iter))->bv_len - (iter).bi_bvec_done) +static __always_inline unsigned int +mp_bvec_iter_len(const struct bio_vec *bvecs, const struct bvec_iter iter) +{ + return min(__bvec_iter_bvec(bvecs, iter)->bv_len - iter.bi_bvec_done, + iter.bi_size); +} -#define mp_bvec_iter_offset(bvec, iter) \ - (__bvec_iter_bvec((bvec), (iter))->bv_offset + (iter).bi_bvec_done) +static __always_inline unsigned int +mp_bvec_iter_offset(const struct bio_vec *bvecs, const struct bvec_iter iter) +{ + return __bvec_iter_bvec(bvecs, iter)->bv_offset + iter.bi_bvec_done; +} -#define mp_bvec_iter_page_idx(bvec, iter) \ - (mp_bvec_iter_offset((bvec), (iter)) / PAGE_SIZE) +static __always_inline unsigned int +mp_bvec_iter_page_idx(const struct bio_vec *bvecs, const struct bvec_iter iter) +{ + return mp_bvec_iter_offset(bvecs, iter) / PAGE_SIZE; +} -#define mp_bvec_iter_bvec(bvec, iter) \ -((struct bio_vec) { \ - .bv_page = mp_bvec_iter_page((bvec), (iter)), \ - .bv_len = mp_bvec_iter_len((bvec), (iter)), \ - .bv_offset = mp_bvec_iter_offset((bvec), (iter)), \ -}) +static __always_inline struct bio_vec +mp_bvec_iter_bvec(const struct bio_vec *bvecs, const struct bvec_iter iter) +{ + return (struct bio_vec) { + .bv_page = mp_bvec_iter_page(bvecs, iter), + .bv_len = mp_bvec_iter_len(bvecs, iter), + .bv_offset = mp_bvec_iter_offset(bvecs, iter), + }; +} /* For building single-page bvec in flight */ - #define bvec_iter_offset(bvec, iter) \ - (mp_bvec_iter_offset((bvec), (iter)) % PAGE_SIZE) - -#define bvec_iter_len(bvec, iter) \ - min_t(unsigned, mp_bvec_iter_len((bvec), (iter)), \ - PAGE_SIZE - bvec_iter_offset((bvec), (iter))) - -#define bvec_iter_page(bvec, iter) \ - (mp_bvec_iter_page((bvec), (iter)) + \ - mp_bvec_iter_page_idx((bvec), (iter))) - -#define bvec_iter_bvec(bvec, iter) \ -((struct bio_vec) { \ - .bv_page = bvec_iter_page((bvec), (iter)), \ - .bv_len = bvec_iter_len((bvec), (iter)), \ - .bv_offset = bvec_iter_offset((bvec), (iter)), \ -}) +static __always_inline unsigned int +bvec_iter_offset(const struct bio_vec *bvecs, const struct bvec_iter iter) +{ + return mp_bvec_iter_offset(bvecs, iter) % PAGE_SIZE; +} + +static __always_inline unsigned int +bvec_iter_len(const struct bio_vec *bvecs, const struct bvec_iter iter) +{ + return min(mp_bvec_iter_len(bvecs, iter), + PAGE_SIZE - bvec_iter_offset(bvecs, iter)); +} + +static __always_inline struct page * +bvec_iter_page(const struct bio_vec *bvecs, const struct bvec_iter iter) +{ + return mp_bvec_iter_page(bvecs, iter) + + mp_bvec_iter_page_idx(bvecs, iter); +} + +static __always_inline struct bio_vec +bvec_iter_bvec(const struct bio_vec *bvecs, const struct bvec_iter iter) +{ + return (struct bio_vec) { + .bv_page = bvec_iter_page(bvecs, iter), + .bv_len = bvec_iter_len(bvecs, iter), + .bv_offset = bvec_iter_offset(bvecs, iter), + }; +} static inline bool bvec_iter_advance(const struct bio_vec *bv, struct bvec_iter *iter, unsigned bytes) -- cgit v1.2.3 From ca36c1aaee095d009d73f6623fb5e536ada80032 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 28 May 2026 18:59:03 +0100 Subject: block: Add bvec_folio() This is a simple helper which replaces page_folio(bvec->bv_page). Minor improvement in readability, but the real motivation is to reduce the number of references to bvec->bv_page so that it can be changed with less work. Signed-off-by: Matthew Wilcox (Oracle) Cc: Leon Romanovsky Reviewed-by: Hannes Reinecke Reviewed-by: Christoph Hellwig Reviewed-by: William Kucharski Link: https://patch.msgid.link/20260528175905.1102280-2-willy@infradead.org Signed-off-by: Jens Axboe --- block/bio.c | 6 +++--- include/linux/bio.h | 2 +- include/linux/bvec.h | 15 +++++++++++++++ io_uring/rsrc.c | 2 +- mm/page_io.c | 4 ++-- 5 files changed, 22 insertions(+), 7 deletions(-) (limited to 'include/linux/bvec.h') diff --git a/block/bio.c b/block/bio.c index e9944dab0132..c3d0a00a27f1 100644 --- a/block/bio.c +++ b/block/bio.c @@ -1299,7 +1299,7 @@ static void bio_free_folios(struct bio *bio) int i; bio_for_each_bvec_all(bv, bio, i) { - struct folio *folio = page_folio(bv->bv_page); + struct folio *folio = bvec_folio(bv); if (!is_zero_folio(folio)) folio_put(folio); @@ -1406,7 +1406,7 @@ int bio_iov_iter_bounce(struct bio *bio, struct iov_iter *iter, size_t maxlen) static void bvec_unpin(struct bio_vec *bv, bool mark_dirty) { - struct folio *folio = page_folio(bv->bv_page); + struct folio *folio = bvec_folio(bv); size_t nr_pages = (bv->bv_offset + bv->bv_len - 1) / PAGE_SIZE - bv->bv_offset / PAGE_SIZE + 1; @@ -1440,7 +1440,7 @@ static void bio_iov_iter_unbounce_read(struct bio *bio, bool is_error, bvec_unpin(&bio->bi_io_vec[1 + i], mark_dirty); } - folio_put(page_folio(bio->bi_io_vec[0].bv_page)); + folio_put(bvec_folio(&bio->bi_io_vec[0])); } /** diff --git a/include/linux/bio.h b/include/linux/bio.h index 97f993d4d914..26ebfff93a16 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -283,7 +283,7 @@ static inline void bio_first_folio(struct folio_iter *fi, struct bio *bio, return; } - fi->folio = page_folio(bvec->bv_page); + fi->folio = bvec_folio(bvec); fi->offset = bvec->bv_offset + PAGE_SIZE * folio_page_idx(fi->folio, bvec->bv_page); fi->_seg_count = bvec->bv_len; diff --git a/include/linux/bvec.h b/include/linux/bvec.h index f4c7ec282ac9..c939f2b3be87 100644 --- a/include/linux/bvec.h +++ b/include/linux/bvec.h @@ -74,6 +74,21 @@ static inline void bvec_set_virt(struct bio_vec *bv, void *vaddr, bvec_set_page(bv, virt_to_page(vaddr), len, offset_in_page(vaddr)); } +/** + * bvec_folio - Return the first folio referenced by this bvec + * @bv: bvec to access + * + * A bvec can contain non-folio memory, so this should only be called by + * the creator of the bvec; drivers have no business looking at the owner + * of the memory. It may not even be the right interface for the caller + * to use as a bvec can span multiple folios. You may be better off using + * something like bio_for_each_folio_all() which iterates over all folios. + */ +static inline struct folio *bvec_folio(const struct bio_vec *bv) +{ + return page_folio(bv->bv_page); +} + struct bvec_iter { /* * Current device address in 512 byte sectors. Only updated by the bio diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 650303626be6..5d792f70ec1e 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -102,7 +102,7 @@ static void io_release_ubuf(void *priv) unsigned int i; for (i = 0; i < imu->nr_bvecs; i++) { - struct folio *folio = page_folio(imu->bvec[i].bv_page); + struct folio *folio = bvec_folio(&imu->bvec[i]); unpin_user_folio(folio, 1); } diff --git a/mm/page_io.c b/mm/page_io.c index 70cea9e24d2f..a59b73f8bdd9 100644 --- a/mm/page_io.c +++ b/mm/page_io.c @@ -490,7 +490,7 @@ static void sio_read_complete(struct kiocb *iocb, long ret) if (ret == sio->len) { for (p = 0; p < sio->pages; p++) { - struct folio *folio = page_folio(sio->bvec[p].bv_page); + struct folio *folio = bvec_folio(&sio->bvec[p]); count_mthp_stat(folio_order(folio), MTHP_STAT_SWPIN); count_memcg_folio_events(folio, PSWPIN, folio_nr_pages(folio)); @@ -500,7 +500,7 @@ static void sio_read_complete(struct kiocb *iocb, long ret) count_vm_events(PSWPIN, sio->len >> PAGE_SHIFT); } else { for (p = 0; p < sio->pages; p++) { - struct folio *folio = page_folio(sio->bvec[p].bv_page); + struct folio *folio = bvec_folio(&sio->bvec[p]); folio_unlock(folio); } -- cgit v1.2.3 From 11ff85db51e05cf8cbeea36be1085fdacedeab55 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 28 May 2026 18:59:04 +0100 Subject: block: Include bvec.h kernel-doc in the htmldocs People have gone to the trouble of writing this kernel-doc; the least we can do is publish it. Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Hannes Reinecke Reviewed-by: Christoph Hellwig Reviewed-by: William Kucharski Link: https://patch.msgid.link/20260528175905.1102280-3-willy@infradead.org Signed-off-by: Jens Axboe --- Documentation/core-api/kernel-api.rst | 1 + include/linux/bvec.h | 2 ++ 2 files changed, 3 insertions(+) (limited to 'include/linux/bvec.h') diff --git a/Documentation/core-api/kernel-api.rst b/Documentation/core-api/kernel-api.rst index e8211c4ca662..4c4a57c1c094 100644 --- a/Documentation/core-api/kernel-api.rst +++ b/Documentation/core-api/kernel-api.rst @@ -307,6 +307,7 @@ Accounting Framework Block Devices ============= +.. kernel-doc:: include/linux/bvec.h .. kernel-doc:: include/linux/bio.h .. kernel-doc:: block/blk-core.c :export: diff --git a/include/linux/bvec.h b/include/linux/bvec.h index c939f2b3be87..92837e2743f1 100644 --- a/include/linux/bvec.h +++ b/include/linux/bvec.h @@ -289,6 +289,7 @@ static inline void *bvec_kmap_local(struct bio_vec *bvec) /** * memcpy_from_bvec - copy data from a bvec + * @to: Kernel virtual address to copy to. * @bvec: bvec to copy from * * Must be called on single-page bvecs only. @@ -301,6 +302,7 @@ static inline void memcpy_from_bvec(char *to, struct bio_vec *bvec) /** * memcpy_to_bvec - copy data to a bvec * @bvec: bvec to copy to + * @from: Kernel virtual address to copy from. * * Must be called on single-page bvecs only. */ -- cgit v1.2.3