diff options
| author | Pavel Begunkov <asml.silence@gmail.com> | 2026-08-07 14:19:31 +0100 |
|---|---|---|
| committer | Jens Axboe <axboe@kernel.dk> | 2026-08-15 17:29:57 -0600 |
| commit | 7d5d94fe9af1b402271236ff8536252789f3eb17 (patch) | |
| tree | b977d1db8c16e5215264e92ae08aafd528fec491 /io_uring | |
| parent | 6bc4d9dec9059f1373eb140fc80b6f64fd153ecd (diff) | |
| download | lwn-7d5d94fe9af1b402271236ff8536252789f3eb17.tar.gz lwn-7d5d94fe9af1b402271236ff8536252789f3eb17.zip | |
io_uring/zcrx: move freelist lock to struct zcrx
freelist_lock, which protects slow path allocations, is currently stored
in struct io_zcrx_area. Once we add support for multiple queues, we'll
need a lock in the zcrx ctx, move it there.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://patch.msgid.link/be7feadff1e210861433f99fe44170aa27ec5b33.1786108672.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
Diffstat (limited to 'io_uring')
| -rw-r--r-- | io_uring/zcrx.c | 14 | ||||
| -rw-r--r-- | io_uring/zcrx.h | 2 |
2 files changed, 8 insertions, 8 deletions
diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index 8f9654d4a6de..7fda74475c46 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -542,7 +542,6 @@ static int __zcrx_create_area(struct io_zcrx_ifq *ifq, /* we're only supporting one area per ifq for now */ area->area_id = 0; area_reg->rq_area_token = zcrx_area_id_to_token(area->area_id); - spin_lock_init(&area->freelist_lock); ret = io_zcrx_append_area(ifq, area); if (!ret) @@ -573,6 +572,7 @@ static struct io_zcrx_ifq *io_zcrx_ifq_alloc(struct io_ring_ctx *ctx) ifq->if_rxq = -1; spin_lock_init(&ifq->ctx_lock); spin_lock_init(&ifq->rq.lock); + spin_lock_init(&ifq->alloc_lock); mutex_init(&ifq->pp_lock); refcount_set(&ifq->refs, 1); refcount_set(&ifq->user_refs, 1); @@ -647,8 +647,9 @@ static void io_put_zcrx_ifq(struct io_zcrx_ifq *ifq) static void io_zcrx_return_niov_freelist(struct net_iov *niov) { struct io_zcrx_area *area = io_zcrx_iov_to_area(niov); + struct io_zcrx_ifq *ifq = area->ifq; - guard(spinlock_bh)(&area->freelist_lock); + guard(spinlock_bh)(&ifq->alloc_lock); if (WARN_ON_ONCE(area->free_count >= area->nia.num_niovs)) return; area->freelist[area->free_count++] = net_iov_idx(niov); @@ -658,7 +659,7 @@ static struct net_iov *zcrx_get_free_niov(struct io_zcrx_area *area) { unsigned niov_idx; - lockdep_assert_held(&area->freelist_lock); + lockdep_assert_held(&area->ifq->alloc_lock); if (unlikely(!area->free_count)) return NULL; @@ -1250,7 +1251,7 @@ static unsigned io_zcrx_refill_slow(struct page_pool *pp, struct io_zcrx_ifq *if struct io_zcrx_area *area = ifq->area; unsigned allocated = 0; - guard(spinlock_bh)(&area->freelist_lock); + guard(spinlock_bh)(&ifq->alloc_lock); for (allocated = 0; allocated < to_alloc; allocated++) { struct net_iov *niov = zcrx_get_free_niov(area); @@ -1563,14 +1564,13 @@ static bool io_zcrx_queue_cqe(struct io_kiocb *req, struct net_iov *niov, static struct net_iov *io_alloc_fallback_niov(struct io_zcrx_ifq *ifq) { - struct io_zcrx_area *area = ifq->area; struct net_iov *niov = NULL; if (!ifq->kern_readable) return NULL; - scoped_guard(spinlock_bh, &area->freelist_lock) - niov = zcrx_get_free_niov(area); + scoped_guard(spinlock_bh, &ifq->alloc_lock) + niov = zcrx_get_free_niov(ifq->area); if (niov) page_pool_fragment_netmem(net_iov_to_netmem(niov), 1); diff --git a/io_uring/zcrx.h b/io_uring/zcrx.h index 0eb7ea35a9ff..302659669ba4 100644 --- a/io_uring/zcrx.h +++ b/io_uring/zcrx.h @@ -36,7 +36,6 @@ struct io_zcrx_area { u16 area_id; /* freelist */ - spinlock_t freelist_lock ____cacheline_aligned_in_smp; u32 free_count; u32 *freelist; @@ -65,6 +64,7 @@ struct io_zcrx_ifq { bool kern_readable; struct zcrx_rq rq ____cacheline_aligned_in_smp; + spinlock_t alloc_lock ____cacheline_aligned_in_smp; u32 if_rxq; struct device *dev; |