diff options
| author | Pavel Begunkov <asml.silence@gmail.com> | 2023-04-13 15:28:11 +0100 |
|---|---|---|
| committer | Jens Axboe <axboe@kernel.dk> | 2023-04-15 14:44:57 -0600 |
| commit | 2f2af35f8e5a1ed552ed02e47277d50092a2b9f6 (patch) | |
| tree | 2db31a557c74f122c74acab020e862c9aa768218 /io_uring | |
| parent | 0b222eeb6514ba6c3457b667fa4f3645032e1fc9 (diff) | |
| download | lwn-2f2af35f8e5a1ed552ed02e47277d50092a2b9f6.tar.gz lwn-2f2af35f8e5a1ed552ed02e47277d50092a2b9f6.zip | |
io_uring/rsrc: inline switch_start fast path
Inline the part of io_rsrc_node_switch_start() that checks whether the
cache is empty or not, as most of the times it will have some number of
entries in there.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/9619c1717a0e01f22c5fce2f1ba2735f804da0f2.1681395792.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
Diffstat (limited to 'io_uring')
| -rw-r--r-- | io_uring/rsrc.c | 12 | ||||
| -rw-r--r-- | io_uring/rsrc.h | 9 |
2 files changed, 13 insertions, 8 deletions
diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 5415a18844e0..bfa0b382c6c6 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -230,15 +230,13 @@ void io_rsrc_node_switch(struct io_ring_ctx *ctx, ctx->rsrc_node = backup; } -int io_rsrc_node_switch_start(struct io_ring_ctx *ctx) +int __io_rsrc_node_switch_start(struct io_ring_ctx *ctx) { - if (io_alloc_cache_empty(&ctx->rsrc_node_cache)) { - struct io_rsrc_node *node = kzalloc(sizeof(*node), GFP_KERNEL); + struct io_rsrc_node *node = kzalloc(sizeof(*node), GFP_KERNEL); - if (!node) - return -ENOMEM; - io_alloc_cache_put(&ctx->rsrc_node_cache, &node->cache); - } + if (!node) + return -ENOMEM; + io_alloc_cache_put(&ctx->rsrc_node_cache, &node->cache); return 0; } diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h index 5dd2fcb28069..732496afed4c 100644 --- a/io_uring/rsrc.h +++ b/io_uring/rsrc.h @@ -69,7 +69,7 @@ void io_rsrc_put_tw(struct callback_head *cb); void io_rsrc_node_ref_zero(struct io_rsrc_node *node); void io_rsrc_put_work(struct work_struct *work); void io_rsrc_node_destroy(struct io_ring_ctx *ctx, struct io_rsrc_node *ref_node); -int io_rsrc_node_switch_start(struct io_ring_ctx *ctx); +int __io_rsrc_node_switch_start(struct io_ring_ctx *ctx); struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx); int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx, struct io_rsrc_node *node, void *rsrc); @@ -111,6 +111,13 @@ static inline int io_scm_file_account(struct io_ring_ctx *ctx, return __io_scm_file_account(ctx, file); } +static inline int io_rsrc_node_switch_start(struct io_ring_ctx *ctx) +{ + if (unlikely(io_alloc_cache_empty(&ctx->rsrc_node_cache))) + return __io_rsrc_node_switch_start(ctx); + return 0; +} + int io_register_files_update(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args); int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg, |