fuse: add zero-copy over io-uring

Implement zero-copy in fuse io-uring to eliminate memory copies between
the application, kernel, and server for read/write operations. The
server can directly access client pages or page cache folios without
copying data through an intermediary buffer. When a fuse request arrives,
the kernel registers the relevant pages into a sparse slot in the
server's io_uring registered buffer table. The server can then operate
on these pages directly using io-uring fixed buffer operations (eg
read_fixed/write_fixed) and the kernel unregisters these pages when the
request completes. Non-page-backed args (eg op out headers) will go
through the payload buffer as normal. The server can specify which open
files should have their reads/writes go through zero-copy, by setting
the FOPEN_IO_URING_ZERO_COPY flag when servicing opens.

This requires CAP_SYS_ADMIN and bufpools. This is gated behind
CAP_SYS_ADMIN because zero-copy allows the server direct access to the
client's underlying pages, rather than operating on an intermediary
buffer that the contents of the client's pages were copied into or on
page cache folios.

The request flow for the zero-copy direct-io write path (client writes
data, server reads it) is as follows:
=======================================================================
|  Kernel                                   |  FUSE server
|                                           |
|  "write(fd, buf, 1MB)"                    |
|                                           |
|  >sys_write()                             |
|    >fuse_file_write_iter()                |
|      >fuse_send_one()                     |
|        [req->args->in_pages = true]       |
|        [folios hold client write data]    |
|                                           |
|  >fuse_uring_copy_to_ring()               |
|    >copy_header_to_ring(IN_OUT)           |
|      [memcpy fuse_in_header]              |
|    >copy_header_to_ring(OP)               |
|      [memcpy write_in header]             |
|                                           |
|    >fuse_uring_args_to_ring()             |
|      >setup_fuse_copy_state()             |
|        [skip_folio_copy = true]           |
|                                           |
|      >fuse_uring_set_up_zero_copy()       |
|        [folio_get for each client folio]  |
|        [build bio_vec array from folios]  |
|        >io_buffer_register_bvec()         |
|          [register pages at
                 ent->zero_copy_index]      |
|        [ent->zero_copied = true]          |
|                                           |
|      >fuse_copy_args()                    |
|        [skip_folio_copy => return 0       |
|         for page arg, skip data copy]     |
|                                           |
|    >copy_header_to_ring(RING_ENT)         |
|      [memcpy ent_in_out]                  |
|    >io_uring_cmd_done()                   |
|                                           |
|                                           | [CQE received]
|                                           |
|                                           | [issue io_uring READ at
|                                           |  ent->zero_copy_index]
|                                           | [reads directly from
|                                           |client's pages (ZERO_COPY)]
|                                           |
|                                           | [write data to backing
|                                           | store]
|                                           |  [submit COMMIT AND FETCH]
|                                           |
|  >fuse_uring_commit_fetch()               |
|    >fuse_uring_commit()                   |
|      >fuse_uring_copy_from_ring()         |
|    >fuse_uring_req_end()                  |
|      >io_buffer_unregister(ent->zero_copy_index) |
|        [unregister pages from index]      |
|      >fuse_zero_copy_release()            |
|        [folio_put for each folio]         |
|      [ent->zero_copied = false]           |
|      >fuse_request_end()                  |
|        [wake up client]                   |

The zero-copy read path is analogous.

Some requests may have both page-backed args and non-page-backed args.
For these requests, the page-backed args are zero-copied while the
non-page-backed args are copied to the buffer selected from the buffer
pool:
    zero-copy: pages registered via io_buffer_register_bvec()
    non-page-backed: copied to payload buffer via fuse_copy_args()

For a request whose payload is zero-copied, the
registration/unregistration path looks like:

    register:  fuse_uring_set_up_zero_copy()
                 folio_get() for each folio
                 io_buffer_register_bvec(ent->zero_copy_index)

    unregister: fuse_uring_req_end()
                  io_buffer_unregister(ent->zero_copy_index)
                  -> fuse_zero_copy_release() callback
                     folio_put() for each folio

Please note that on abort for in-flight zero-copied requests that have
been sent to userspace, the registered bvec slot remains occupied and
its folios remain pinned until the io-uring ring is destroyed, at which
point io-uring unregisters all buffers and the fuse_zero_copy_release()
callback drops the folio references. Unregistering at teardown would
require operating on the ring context directly, whose validity is hard
to ascertain; this is deemed not worth the complexity for the abort
race, since everything is freed when the ring is torn down.

The throughput improvement from zero-copy depends on how much of the
per-request latency is spent on data copying vs backing I/O. The gain
comes from eliminating the payload-buffer memcpy,  but accessing the
zero-copied pages requires the server to issue the read/write as an
IORING_OP_READ/WRITE_FIXED operation. The benefit is largest when the
mempcy is a meaningful fraction of per-request latency while backing i/o
is still noticable enough that the extra io-uring op's overhead doesn't
dominate.

Benchmarked with passthrough_hp (--nopassthrough, q_depth=8) on a
2-socket Intel Xeon Gold 6138 (40 cores / 80 threads), using fio (sync
engine, bs=1M, O_DIRECT, numjobs=2, 30s run + 10s ramp, 3 runs) where
direct-I/O throughput is against a RAM-backed (tmpfs) source (backing
I/O is not the bottleneck):

		baseline   registered-buf   zero-copy   (zc vs base)
direct read     ~5.1 GB/s  ~5.4 GB/s        ~8.9 GB/s   (+75%)
direct write    ~3.4 GB/s  ~4.8 GB/s        ~5.1 GB/s   (+50%)

Reads end up higher than writes because the backing store reads faster
than it writes (the baseline shows the same read>write gap, and the raw
device does too). On a device-bound NVMe (~2 GB/s reads) the read gain
shrinks to ~10-16% (and no measurable gains for writes), as backing I/O
rather than the eliminated copy dominates latency.

The benefit overall scales with how much of the
per-request latency is the data copy versus backing I/O.

Signed-off-by: Joanne Koong <joannelkoong@gmail.com>
Reviewed-by: Bernd Schubert <bernd@bsbernd.com>
Signed-off-by: Miklos Szeredi <mszeredi@redhat.com>

1 files changed, 34 insertions, 0 deletions

diff --git a/include/uapi/linux/fuse.h b/include/uapi/linux/fuse.h
index 538d844da099..7435e09c87fe 100644
--- a/include/uapi/linux/fuse.h
+++ b/include/uapi/linux/fuse.h
@@ -246,6 +246,8 @@
  *  - add FUSE_HAS_IO_URING_BUFPOOL
  *  - add fuse_uring_cmd_req bufpool struct
  *  - add bufpool offset field to fuse_uring_ent_in_out struct
+ *  - add FUSE_URING_ZERO_COPY, FUSE_URING_ENT_ZERO_COPY, and
+ *    FOPEN_IO_URING_ZERO_COPY flag
  */
 
 #ifndef _LINUX_FUSE_H
@@ -389,6 +391,12 @@ struct fuse_file_lock {
  * FOPEN_NOFLUSH: don't flush data cache on close (unless FUSE_WRITEBACK_CACHE)
  * FOPEN_PARALLEL_DIRECT_WRITES: Allow concurrent direct writes on the same inode
  * FOPEN_PASSTHROUGH: passthrough read/write io for this open file
+ * FOPEN_IO_URING_ZERO_COPY: use io-uring zero-copy for reads/writes on this
+ *                           open file. Honored only when the serving io-uring
+ *                           queue was set up for zero-copy
+ *                           (FUSE_URING_ZERO_COPY) and the request carries page
+ *                           payload. Otherwise reads/writes fall back to
+ *                           copying.
  */
 #define FOPEN_DIRECT_IO		(1 << 0)
 #define FOPEN_KEEP_CACHE	(1 << 1)
@@ -398,6 +406,7 @@ struct fuse_file_lock {
 #define FOPEN_NOFLUSH		(1 << 5)
 #define FOPEN_PARALLEL_DIRECT_WRITES	(1 << 6)
 #define FOPEN_PASSTHROUGH	(1 << 7)
+#define FOPEN_IO_URING_ZERO_COPY (1 << 8)
 
 /**
  * INIT request/reply flags
@@ -1259,6 +1268,13 @@ struct fuse_supp_groups {
 #define FUSE_URING_IN_OUT_HEADER_SZ 128
 #define FUSE_URING_OP_IN_OUT_SZ 128
 
+/**
+ * fuse_uring_ent_in_out flags
+ *
+ * FUSE_URING_ENT_ZERO_COPY: Set if the ent's payload is zero-copied
+ */
+#define FUSE_URING_ENT_ZERO_COPY	(1 << 0)
+
 /* Used as part of the fuse_uring_req_header */
 struct fuse_uring_ent_in_out {
 	uint64_t flags;
@@ -1310,6 +1326,14 @@ enum fuse_uring_cmd {
 	FUSE_IO_URING_CMD_ADD_BUFPOOL = 4,
 };
 
+/*
+ * fuse_uring_cmd_req flags for FUSE_IO_URING_CMD_ADD_QUEUE
+ *
+ * FUSE_URING_ZERO_COPY is only supported for queues with bufpools on privileged
+ * servers
+ */
+#define FUSE_URING_ZERO_COPY		(1 << 0)
+
 /**
  * In the 80B command area of the SQE.
  */
@@ -1330,6 +1354,16 @@ struct fuse_uring_cmd_req {
 			uint32_t len;
 			uint32_t reserved;
 		} bufpool;
+
+		/*
+		 * Index of this entry's slot in the server's io_uring
+		 * registered buffer table, where the kernel registers the
+		 * request's pages for zero-copy. Set for
+		 * FUSE_IO_URING_CMD_REGISTER cmds only, and only on queues
+		 * created with FUSE_URING_ZERO_COPY. On a non-zero-copy queue
+		 * this must be 0
+		 */
+		uint16_t ent_zero_copy_buf_index;
 	};
 };