io_uring/rsrc: improve code generation for fixed file assignment
Jens Axboe
maintainerPATCH 1/1 io_uring/rsrc: improve code generation for fixed file assignment
For the normal read/write path, we have already locked the ring submission side when assigning the file. This causes branch mispredictions when we then check and try and lock again in io_req_set_rsrc_node(). As this is a very hot path, this matters.
Add a basic helper that already assumes we already have it locked, and use that in io_file_get_fixed().
Signed-off-by Jens Axboe <[email protected]>
---
io_uring/io_uring.c +3 -2
@@ -2000,9 +2000,10 @@ inline struct file *io_file_get_fixed(struct io_kiocb *req, int fd, goto out; fd =array_index_nospec (fd, ctx->nr_user_files); slot =io_fixed_file_slot (&ctx->file_table, fd); - file =io_slot_file (slot); +if (!req->rsrc_node) +__io_req_set_rsrc_node (req, ctx); req->flags |=io_slot_flags (slot); -io_req_set_rsrc_node (req, ctx,0 ); + file =io_slot_file (slot); out:io_ring_submit_unlock (ctx, issue_flags);return file;
io_uring/rsrc.h +9 -5
@@ -102,17 +102,21 @@ static inline void io_charge_rsrc_node(struct io_ring_ctx *ctx, node->refs++; } +static inline void __io_req_set_rsrc_node (struct io_kiocb *req, +struct io_ring_ctx *ctx) +{ +lockdep_assert_held (&ctx->uring_lock); + req->rsrc_node = ctx->rsrc_node; +io_charge_rsrc_node (ctx, ctx->rsrc_node); +} +static inline void io_req_set_rsrc_node (struct io_kiocb *req,struct io_ring_ctx *ctx,unsigned int issue_flags) {if (!req->rsrc_node) {io_ring_submit_lock (ctx, issue_flags); - -lockdep_assert_held (&ctx->uring_lock); - - req->rsrc_node = ctx->rsrc_node; -io_charge_rsrc_node (ctx, ctx->rsrc_node); +__io_req_set_rsrc_node (req, ctx);io_ring_submit_unlock (ctx, issue_flags); } }
Signature · 1 line
Jens Axboe