From 1e1fdfe165317af81976083f37211f6978c7628c Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 24 Jun 2026 18:42:26 +0100 Subject: iomap: Remove FGP_NOFS from iomap_get_folio() FGP_NOFS is legacy; filesystems should be using memalloc_nofs_save/restore instead. We have it here in iomap because it was buried in grab_cache_page_write_begin() and we didn't want to change this behaviour as part of the folio transition. I have tested this with XFS and see no issues. Other filesystems (cc'd) may need to make adjustments. Please test with lockdep enabled. Cc: Darrick J. Wong Cc: Jens Axboe Cc: Namjae Jeon Cc: Sungjong Seo Cc: Yuezhang Mo Cc: Miklos Szeredi Cc: Andreas Gruenbacher Cc: Hyunchul Lee Cc: Konstantin Komarov Cc: Carlos Maiolino Cc: Damien Le Moal Cc: Naohiro Aota Cc: Johannes Thumshirn Cc: linux-xfs@vger.kernel.org Cc: linux-fsdevel@vger.kernel.org Cc: linux-block@vger.kernel.org Cc: fuse-devel@lists.linux.dev Cc: gfs2@lists.linux.dev Cc: ntfs3@lists.linux.dev Signed-off-by: Matthew Wilcox (Oracle) Link: https://patch.msgid.link/20260624174228.2015893-1-willy@infradead.org Signed-off-by: Christian Brauner (Amutable) --- fs/iomap/buffered-io.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs/iomap') diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c index 8d4806dc46d4..27bc2455a98d 100644 --- a/fs/iomap/buffered-io.c +++ b/fs/iomap/buffered-io.c @@ -768,7 +768,7 @@ EXPORT_SYMBOL_GPL(iomap_is_partially_uptodate); */ struct folio *iomap_get_folio(struct iomap_iter *iter, loff_t pos, size_t len) { - fgf_t fgp = FGP_WRITEBEGIN | FGP_NOFS; + fgf_t fgp = FGP_WRITEBEGIN; if (iter->flags & IOMAP_NOWAIT) fgp |= FGP_NOWAIT; -- cgit v1.2.3 From 562e5ff95de15ae94eff169417366564ef14735d Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 29 Jun 2026 14:52:29 +0200 Subject: iomap: add a separate bio_set for iomap_split_ioend iomap_split_ioend can split bios that already come from iomap_ioend_bioset and thus deadlock when the bioset is exhausted. Add a separate bio_set to avoid this deadlock. Christian Brauner says: Mark iomap_ioend_split_bioset static as it is only used in ioend.c, fixing the sparse warning reported by the kernel test robot. Fixes: 5fcbd555d483 ("iomap: split bios to zone append limits in the submission handlers") Signed-off-by: Christoph Hellwig Link: https://patch.msgid.link/20260629125229.3400726-1-hch@lst.de Signed-off-by: Christian Brauner (Amutable) --- fs/iomap/ioend.c | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) (limited to 'fs/iomap') diff --git a/fs/iomap/ioend.c b/fs/iomap/ioend.c index f7c3e0c70fd7..82eeeadac6d6 100644 --- a/fs/iomap/ioend.c +++ b/fs/iomap/ioend.c @@ -13,6 +13,7 @@ struct bio_set iomap_ioend_bioset; EXPORT_SYMBOL_GPL(iomap_ioend_bioset); +static struct bio_set iomap_ioend_split_bioset; struct iomap_ioend *iomap_init_ioend(struct inode *inode, struct bio *bio, loff_t file_offset, u16 ioend_flags) @@ -482,7 +483,8 @@ struct iomap_ioend *iomap_split_ioend(struct iomap_ioend *ioend, sector_offset = ALIGN_DOWN(sector_offset << SECTOR_SHIFT, i_blocksize(ioend->io_inode)) >> SECTOR_SHIFT; - split = bio_split(bio, sector_offset, GFP_NOFS, &iomap_ioend_bioset); + split = bio_split(bio, sector_offset, GFP_NOFS, + &iomap_ioend_split_bioset); if (IS_ERR(split)) return ERR_CAST(split); split->bi_private = bio->bi_private; @@ -505,8 +507,23 @@ EXPORT_SYMBOL_GPL(iomap_split_ioend); static int __init iomap_ioend_init(void) { - return bioset_init(&iomap_ioend_bioset, 4 * (PAGE_SIZE / SECTOR_SIZE), + const unsigned int nr_mempool_entries = 4 * (PAGE_SIZE / SECTOR_SIZE); + int error; + + error = bioset_init(&iomap_ioend_bioset, nr_mempool_entries, offsetof(struct iomap_ioend, io_bio), BIOSET_NEED_BVECS); + if (error) + return error; + error = bioset_init(&iomap_ioend_split_bioset, nr_mempool_entries, + offsetof(struct iomap_ioend, io_bio), + BIOSET_NEED_BVECS); + if (error) + goto out_exit_ioend_bioset; + return 0; + +out_exit_ioend_bioset: + bioset_exit(&iomap_ioend_bioset); + return error; } fs_initcall(iomap_ioend_init); -- cgit v1.2.3 From d12fd544419535aa31c40e213161a3fd17bb77ba Mon Sep 17 00:00:00 2001 From: Fengnan Chang Date: Wed, 1 Jul 2026 11:32:51 +0800 Subject: iomap: factor out iomap_dio_alignment helper Extract the alignment computation from iomap_dio_bio_iter() into a standalone helper so the upcoming simple direct I/O path can reuse it without requiring a struct iomap_dio. No functional change. Signed-off-by: Fengnan Chang Link: https://patch.msgid.link/20260701033253.46420-2-changfengnan@bytedance.com Reviewed-by: Christoph Hellwig Signed-off-by: Christian Brauner (Amutable) --- fs/iomap/direct-io.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'fs/iomap') diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c index b485e3b191da..487c4763f3fd 100644 --- a/fs/iomap/direct-io.c +++ b/fs/iomap/direct-io.c @@ -398,6 +398,14 @@ out_put_bio: return ret; } +static inline unsigned int iomap_dio_alignment(struct inode *inode, + struct block_device *bdev, unsigned int dio_flags) +{ + if (dio_flags & IOMAP_DIO_FSBLOCK_ALIGNED) + return i_blocksize(inode); + return bdev_logical_block_size(bdev); +} + static int iomap_dio_bio_iter(struct iomap_iter *iter, struct iomap_dio *dio) { const struct iomap *iomap = &iter->iomap; @@ -416,10 +424,7 @@ static int iomap_dio_bio_iter(struct iomap_iter *iter, struct iomap_dio *dio) * File systems that write out of place and always allocate new blocks * need each bio to be block aligned as that's the unit of allocation. */ - if (dio->flags & IOMAP_DIO_FSBLOCK_ALIGNED) - alignment = fs_block_size; - else - alignment = bdev_logical_block_size(iomap->bdev); + alignment = iomap_dio_alignment(inode, iomap->bdev, dio->flags); if ((pos | length) & (alignment - 1)) return -EINVAL; -- cgit v1.2.3 From a10ab54c15e98e7045b45d2400d445f5b3a0a7d4 Mon Sep 17 00:00:00 2001 From: Fengnan Chang Date: Wed, 1 Jul 2026 11:32:52 +0800 Subject: iomap: pass error code to should_report_dio_fserror directly Change should_report_dio_fserror() to take an error code instead of the full struct iomap_dio, decoupling it for reuse by the upcoming simple direct I/O path. No functional change. Signed-off-by: Fengnan Chang Link: https://patch.msgid.link/20260701033253.46420-3-changfengnan@bytedance.com Reviewed-by: Christoph Hellwig Signed-off-by: Christian Brauner (Amutable) --- fs/iomap/direct-io.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'fs/iomap') diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c index 487c4763f3fd..1b9abdd831d0 100644 --- a/fs/iomap/direct-io.c +++ b/fs/iomap/direct-io.c @@ -88,9 +88,9 @@ static inline enum fserror_type iomap_dio_err_type(const struct iomap_dio *dio) return FSERR_DIRECTIO_READ; } -static inline bool should_report_dio_fserror(const struct iomap_dio *dio) +static inline bool should_report_dio_fserror(int error) { - switch (dio->error) { + switch (error) { case 0: case -EAGAIN: case -ENOTBLK: @@ -110,7 +110,7 @@ ssize_t iomap_dio_complete(struct iomap_dio *dio) if (dops && dops->end_io) ret = dops->end_io(iocb, dio->size, ret, dio->flags); - if (should_report_dio_fserror(dio)) + if (should_report_dio_fserror(dio->error)) fserror_report_io(file_inode(iocb->ki_filp), iomap_dio_err_type(dio), offset, dio->size, dio->error, GFP_NOFS); -- cgit v1.2.3 From c45650e4f8ba25d02e860094a77bf85ec3a35f13 Mon Sep 17 00:00:00 2001 From: Fengnan Chang Date: Wed, 1 Jul 2026 11:32:53 +0800 Subject: iomap: add simple dio path for small direct I/O When running 4K random read workloads on high-performance Gen5 NVMe SSDs, the software overhead in the iomap direct I/O path (__iomap_dio_rw) becomes a significant bottleneck. Using io_uring with poll mode for a 4K randread test on a raw block device: taskset -c 30 ./t/io_uring -p1 -d512 -b4096 -s32 -c32 -F1 -B1 -R1 -X1 -n1 -P1 /dev/nvme10n1 Result: ~3.2M IOPS Running the exact same workload on ext4 and XFS: taskset -c 30 ./t/io_uring -p1 -d512 -b4096 -s32 -c32 -F1 -B1 -R1 -X1 -n1 -P1 /mnt/testfile Result: ~1.92M IOPS Profiling the ext4 workload reveals that a significant portion of CPU time is spent on memory allocation and the iomap state machine iteration: 5.33% [kernel] [k] __iomap_dio_rw 3.26% [kernel] [k] iomap_iter 2.37% [kernel] [k] iomap_dio_bio_iter 2.35% [kernel] [k] kfree 1.33% [kernel] [k] iomap_dio_complete Introduce a simple dio path to reduce the overhead of iomap. It is triggered when the request satisfies all of: - a READ request whose I/O size is <= inode blocksize (fits in a single block, no splits); - no custom iomap_dio_ops (dops) registered by the filesystem; - no caller-accumulated residual (done_before == 0); - none of IOMAP_DIO_FORCE_WAIT / IOMAP_DIO_PARTIAL / IOMAP_DIO_BOUNCE set, the range is within i_size, and the inode is not encrypted. The bio is allocated from a dedicated bioset whose front_pad embeds struct iomap_dio_simple, so the whole request lives in a single cacheline-aligned allocation and no separate struct iomap_dio is needed. Completion is handled inline from ->bi_end_io for the common success case, and only punted to the s_dio_done_wq workqueue on error. After this optimization, the heavy generic functions disappear from the profile, replaced by a single streamlined execution path: 4.83% [kernel] [k] iomap_dio_simple With this patch, 4K random read IOPS on ext4 increases from 1.92M to 2.19M in the original single-core io_uring poll-mode workload. Below are the test results using fio: fs workload qd simple=0 simple=1 gain ext4 libaio 1 18,740 18,761 +0.11% ext4 libaio 64 462,850 480,587 +3.83% ext4 libaio 128 459,498 478,824 +4.21% ext4 libaio 256 459,938 480,156 +4.40% ext4 io_uring 1 18,836 18,880 +0.24% ext4 io_uring 64 568,193 600,625 +5.71% ext4 io_uring 128 570,998 602,148 +5.46% ext4 io_uring 256 572,052 602,536 +5.33% ext4 io_uring_poll 1 19,283 19,272 -0.06% ext4 io_uring_poll 64 989,735 1,013,342 +2.39% ext4 io_uring_poll 128 1,467,336 1,538,444 +4.85% ext4 io_uring_poll 256 1,663,498 1,830,842 +10.06% xfs libaio 1 18,764 18,776 +0.06% xfs libaio 64 462,408 480,860 +3.99% xfs libaio 128 461,280 480,819 +4.24% xfs libaio 256 461,626 480,190 +4.02% xfs io_uring 1 18,871 18,903 +0.17% xfs io_uring 64 570,383 597,399 +4.74% xfs io_uring 128 568,290 597,370 +5.12% xfs io_uring 256 570,616 598,775 +4.93% xfs io_uring_poll 1 19,211 19,315 +0.54% xfs io_uring_poll 64 989,726 1,008,455 +1.89% xfs io_uring_poll 128 1,430,426 1,513,064 +5.78% xfs io_uring_poll 256 1,587,339 1,742,220 +9.76% Signed-off-by: Fengnan Chang Link: https://patch.msgid.link/20260701033253.46420-4-changfengnan@bytedance.com Reviewed-by: Christoph Hellwig Signed-off-by: Christian Brauner (Amutable) --- fs/iomap/direct-io.c | 274 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 274 insertions(+) (limited to 'fs/iomap') diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c index 1b9abdd831d0..ca790239e5eb 100644 --- a/fs/iomap/direct-io.c +++ b/fs/iomap/direct-io.c @@ -10,6 +10,7 @@ #include #include #include +#include #include "internal.h" #include "trace.h" @@ -893,12 +894,277 @@ out_free_dio: } EXPORT_SYMBOL_GPL(__iomap_dio_rw); +struct iomap_dio_simple { + struct kiocb *iocb; + size_t size; + unsigned int dio_flags; + struct work_struct work; + /* + * Align @bio to a cacheline boundary so that, combined with the + * front_pad passed to bioset_init(), the bio sits at the start of + * a cacheline in memory returned by the (HWCACHE-aligned) bio + * slab. This keeps the hot fields block layer touches on submit + * and completion (bi_iter, bi_status, ...) within a single line. + */ + struct bio bio ____cacheline_aligned_in_smp; +}; + +static struct bio_set iomap_dio_simple_pool; + +static ssize_t iomap_dio_simple_complete(struct iomap_dio_simple *sr) +{ + struct bio *bio = &sr->bio; + struct kiocb *iocb = sr->iocb; + struct inode *inode = file_inode(iocb->ki_filp); + ssize_t ret; + + if (unlikely(bio->bi_status)) { + ret = blk_status_to_errno(bio->bi_status); + if (should_report_dio_fserror(ret)) + fserror_report_io(inode, FSERR_DIRECTIO_READ, + iocb->ki_pos, sr->size, ret, + GFP_NOFS); + } else { + ret = sr->size; + iocb->ki_pos += ret; + } + + if (sr->dio_flags & IOMAP_DIO_USER_BACKED) { + bio_check_pages_dirty(bio); + } else { + bio_release_pages(bio, false); + bio_put(bio); + } + inode_dio_end(inode); + trace_iomap_dio_complete(iocb, ret < 0 ? ret : 0, ret); + return ret; +} + +static void iomap_dio_simple_complete_work(struct work_struct *work) +{ + struct iomap_dio_simple *sr = + container_of(work, struct iomap_dio_simple, work); + struct kiocb *iocb = sr->iocb; + + WRITE_ONCE(iocb->private, NULL); + iocb->ki_complete(iocb, iomap_dio_simple_complete(sr)); +} + +static void iomap_dio_simple_end_io(struct bio *bio) +{ + struct iomap_dio_simple *sr = + container_of(bio, struct iomap_dio_simple, bio); + struct kiocb *iocb = sr->iocb; + + if (unlikely(sr->bio.bi_status)) { + struct inode *inode = file_inode(iocb->ki_filp); + + INIT_WORK(&sr->work, iomap_dio_simple_complete_work); + queue_work(inode->i_sb->s_dio_done_wq, &sr->work); + return; + } + + WRITE_ONCE(iocb->private, NULL); + iocb->ki_complete(iocb, iomap_dio_simple_complete(sr)); +} + +static inline bool +iomap_dio_simple_supported(struct kiocb *iocb, struct iov_iter *iter, + const struct iomap_dio_ops *dops, + unsigned int dio_flags, size_t done_before) +{ + struct inode *inode = file_inode(iocb->ki_filp); + size_t count = iov_iter_count(iter); + + if (dops || done_before) + return false; + if (iov_iter_rw(iter) != READ) + return false; + if (!count) + return false; + /* + * Simple dio is an optimization for small IO. Filter out large IO + * early as it's the most common case to fail for typical direct IO + * workloads. + */ + if (count > inode->i_sb->s_blocksize) + return false; + if (dio_flags & (IOMAP_DIO_FORCE_WAIT | IOMAP_DIO_PARTIAL | + IOMAP_DIO_BOUNCE)) + return false; + if (iocb->ki_pos + count > i_size_read(inode)) + return false; + if (IS_ENCRYPTED(inode)) + return false; + + return true; +} + +/* + * Fast path for small, block-aligned direct I/Os that map to a single + * contiguous on-disk extent. + * + * iomap_dio_simple_supported() enforces the cheap up-front constraints before + * entering this path. + * + * @dops must be NULL: a non-NULL @dops means the caller wants its + * ->end_io / ->submit_io hooks invoked, and in particular wants its bios to be + * allocated from the filesystem-private @dops->bio_set (whose front_pad sizes a + * filesystem-private wrapper around the bio). The fast path instead allocates + * from the shared iomap_dio_simple_pool, whose front_pad matches struct + * iomap_dio_simple; the two wrappers are not interchangeable, so we must fall + * back to __iomap_dio_rw() in that case. + * + * @done_before must be zero: a non-zero caller-accumulated residual cannot be + * carried through a single-bio inline completion. + * + * @iter must describe a non-empty READ no larger than the inode block size: + * writes, zero-length I/O, and larger requests need the generic iomap direct + * I/O path. + * + * @dio_flags must not request IOMAP_DIO_FORCE_WAIT, IOMAP_DIO_PARTIAL, or + * IOMAP_DIO_BOUNCE: this path does not support forced waiting, partial direct + * I/O, or bouncing. The range must also stay within i_size and encrypted + * inodes must use the generic iomap direct I/O path. + * + * -ENOTBLK is the private sentinel returned by iomap_dio_simple() when it + * decides the request does not fit the fast path. In that case we proceed to + * the generic __iomap_dio_rw() slow path. Any other errno is a real result and + * is propagated as-is, in particular -EAGAIN for IOCB_NOWAIT must reach the + * caller. + */ +static ssize_t +iomap_dio_simple(struct kiocb *iocb, struct iov_iter *iter, + const struct iomap_ops *ops, void *private, + unsigned int dio_flags) +{ + struct inode *inode = file_inode(iocb->ki_filp); + size_t count = iov_iter_count(iter); + bool wait_for_completion = is_sync_kiocb(iocb); + struct iomap_iter iomi = { + .inode = inode, + .pos = iocb->ki_pos, + .len = count, + .flags = IOMAP_DIRECT, + .private = private, + }; + struct iomap_dio_simple *sr; + unsigned int alignment; + struct bio *bio; + ssize_t ret; + + if (iocb->ki_flags & IOCB_NOWAIT) + iomi.flags |= IOMAP_NOWAIT; + + ret = kiocb_write_and_wait(iocb, count); + if (ret) + return ret; + + inode_dio_begin(inode); + + ret = ops->iomap_begin(inode, iomi.pos, count, iomi.flags, + &iomi.iomap, &iomi.srcmap); + if (ret) { + inode_dio_end(inode); + return ret; + } + + if (iomi.iomap.type != IOMAP_MAPPED || + iomi.iomap.offset + iomi.iomap.length < iomi.pos + count || + (iomi.iomap.flags & IOMAP_F_INTEGRITY)) { + ret = -ENOTBLK; + goto out_iomap_end; + } + + alignment = iomap_dio_alignment(inode, iomi.iomap.bdev, dio_flags); + if ((iomi.pos | count) & (alignment - 1)) { + ret = -EINVAL; + goto out_iomap_end; + } + + if (!wait_for_completion && unlikely(!inode->i_sb->s_dio_done_wq)) { + ret = sb_init_dio_done_wq(inode->i_sb); + if (ret < 0) + goto out_iomap_end; + } + + trace_iomap_dio_rw_begin(iocb, iter, dio_flags, 0); + + if (user_backed_iter(iter)) + dio_flags |= IOMAP_DIO_USER_BACKED; + + bio = bio_alloc_bioset(iomi.iomap.bdev, + bio_iov_vecs_to_alloc(iter, BIO_MAX_VECS), + REQ_OP_READ, GFP_KERNEL, &iomap_dio_simple_pool); + sr = container_of(bio, struct iomap_dio_simple, bio); + sr->iocb = iocb; + sr->dio_flags = dio_flags; + + bio->bi_iter.bi_sector = iomap_sector(&iomi.iomap, iomi.pos); + bio->bi_ioprio = iocb->ki_ioprio; + + ret = bio_iov_iter_get_pages(bio, iter, alignment - 1); + if (unlikely(ret)) + goto out_bio_put; + + if (bio->bi_iter.bi_size != count) { + iov_iter_revert(iter, bio->bi_iter.bi_size); + ret = -ENOTBLK; + goto out_bio_release_pages; + } + + sr->size = bio->bi_iter.bi_size; + + if (dio_flags & IOMAP_DIO_USER_BACKED) + bio_set_pages_dirty(bio); + + if (iocb->ki_flags & IOCB_NOWAIT) + bio->bi_opf |= REQ_NOWAIT; + if ((iocb->ki_flags & IOCB_HIPRI) && !wait_for_completion) { + bio->bi_opf |= REQ_POLLED; + WRITE_ONCE(iocb->private, bio); + } + + if (ops->iomap_end) + ops->iomap_end(inode, iomi.pos, count, count, iomi.flags, + &iomi.iomap); + + if (!wait_for_completion) { + bio->bi_end_io = iomap_dio_simple_end_io; + submit_bio(bio); + trace_iomap_dio_rw_queued(inode, iomi.pos, count); + return -EIOCBQUEUED; + } + + submit_bio_wait(bio); + return iomap_dio_simple_complete(sr); + +out_bio_release_pages: + bio_release_pages(bio, false); +out_bio_put: + bio_put(bio); +out_iomap_end: + if (ops->iomap_end) + ops->iomap_end(inode, iomi.pos, count, 0, iomi.flags, + &iomi.iomap); + inode_dio_end(inode); + return ret; +} + ssize_t iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, const struct iomap_ops *ops, const struct iomap_dio_ops *dops, unsigned int dio_flags, void *private, size_t done_before) { struct iomap_dio *dio; + ssize_t ret; + + if (iomap_dio_simple_supported(iocb, iter, dops, dio_flags, + done_before)) { + ret = iomap_dio_simple(iocb, iter, ops, private, dio_flags); + if (ret != -ENOTBLK) + return ret; + } dio = __iomap_dio_rw(iocb, iter, ops, dops, dio_flags, private, done_before); @@ -907,3 +1173,11 @@ iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, return iomap_dio_complete(dio); } EXPORT_SYMBOL_GPL(iomap_dio_rw); + +static int __init iomap_dio_init(void) +{ + return bioset_init(&iomap_dio_simple_pool, 4, + offsetof(struct iomap_dio_simple, bio), + BIOSET_NEED_BVECS | BIOSET_PERCPU_CACHE); +} +fs_initcall(iomap_dio_init); -- cgit v1.2.3