diff options
| author | Pavel Begunkov <asml.silence@gmail.com> | 2022-06-20 15:27:35 +0100 |
|---|---|---|
| committer | Jens Axboe <axboe@kernel.dk> | 2022-07-24 18:39:15 -0600 |
| commit | 4a0fef62788b69df09267c8e3f3f11d4bb9d50e7 (patch) | |
| tree | ac71025cad11239d4ecc1c31bb8c2af811954fb2 | |
| parent | bce5d70cd64a5d48aff613334b8a5fac450b9753 (diff) | |
| download | lwn-4a0fef62788b69df09267c8e3f3f11d4bb9d50e7.tar.gz lwn-4a0fef62788b69df09267c8e3f3f11d4bb9d50e7.zip | |
io_uring: optimize io_uring_task layout
task_work bits of io_uring_task are split into two cache lines causing
extra cache bouncing, place them into a separate cache line. Also move
the most used submission path fields closer together, so there are hot.
Cc: stable@vger.kernel.org # 5.15+
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
| -rw-r--r-- | io_uring/tctx.h | 34 |
1 files changed, 18 insertions, 16 deletions
diff --git a/io_uring/tctx.h b/io_uring/tctx.h index dde82ce4d8e2..dead0ed00429 100644 --- a/io_uring/tctx.h +++ b/io_uring/tctx.h @@ -7,22 +7,24 @@ struct io_uring_task { /* submission side */ - int cached_refs; - struct xarray xa; - struct wait_queue_head wait; - const struct io_ring_ctx *last; - struct io_wq *io_wq; - struct percpu_counter inflight; - atomic_t inflight_tracked; - atomic_t in_idle; - - spinlock_t task_lock; - struct io_wq_work_list task_list; - struct io_wq_work_list prio_task_list; - struct callback_head task_work; - bool task_running; - - struct file *registered_rings[IO_RINGFD_REG_MAX]; + int cached_refs; + const struct io_ring_ctx *last; + struct io_wq *io_wq; + struct file *registered_rings[IO_RINGFD_REG_MAX]; + + struct xarray xa; + struct wait_queue_head wait; + atomic_t in_idle; + atomic_t inflight_tracked; + struct percpu_counter inflight; + + struct { /* task_work */ + spinlock_t task_lock; + bool task_running; + struct io_wq_work_list task_list; + struct io_wq_work_list prio_task_list; + struct callback_head task_work; + } ____cacheline_aligned_in_smp; }; struct io_tctx_node { |