io_rsrc_node cleanups
Jens Axboe
maintainercover io_rsrc_node cleanups
Hi,
Nothing major here - just a patch reclaiming a bit of space in struct io_rsrc_node. Nothing that yields anything yet, but may as well free up the 'type' to have more future room.
2nd patch reclaims 8b from struct io_kiocb, by taking advantage of the fact that provided and registered buffers cannot currently be used together. This may change in the future, but it's true for now.
include/linux/io_uring_types.h | 7 ++++++- io_uring/io_uring.c | 6 +++--- io_uring/net.c | 3 ++- io_uring/nop.c | 3 ++- io_uring/notif.c | 4 ++-- io_uring/rsrc.c | 11 +++++------ io_uring/rsrc.h | 31 +++++++++++++++++++++++-------- io_uring/rw.c | 3 ++- io_uring/uring_cmd.c | 4 ++-- 9 files changed, 47 insertions(+), 25 deletions(-)
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerQuoted 10 lines · Jens Axboe
Hi,
Nothing major here - just a patch reclaiming a bit of space in struct
io_rsrc_node. Nothing that yields anything yet, but may as well free up the 'type' to have more future room.2nd patch reclaims 8b from struct
io_kiocb, by taking advantage of the fact that provided and registered buffers cannot currently be used together. This may change in the future, but it's true for now.
Quoted 10 lines
include/linux/io_uring_types.h | 7 ++++++- io_uring/io_uring.c | 6 +++--- io_uring/net.c | 3 ++- io_uring/nop.c | 3 ++- io_uring/notif.c | 4 ++-- io_uring/rsrc.c | 11 +++++------ io_uring/rsrc.h | 31 +++++++++++++++++++++++-------- io_uring/rw.c | 3 ++- io_uring/uring_cmd.c | 4 ++-- 9 files changed, 47 insertions(+), 25 deletions(-)
Not sure what happened with git send-email here, but here are the two patches.
Signature · 1 line
Jens Axboe
From 0c81a9455e0f5b440bb29efcef35225841328a91 Mon Sep 17 00:00:00 2001
From: Jens Axboe <[email protected]>
Date: Sun, 3 Nov 2024 08:46:07 -0700
Subject: [PATCH 2/2] io_uring/rsrc: split io_kiocb node type assignments
Currently the io_rsrc_node assignment in io_kiocb is an array of two pointers, as two nodes may be assigned to a request - one file node, and one buffer node. However, the buffer node can co-exist with the provided buffers, as currently it's not supported to use both provided and registered buffers at the same time.
This crucially brings struct io_kiocb down to 4 cache lines again, as before it spilled into the 5th cacheline.
--- include/linux/io_uring_types.h | 7 ++++++- io_uring/io_uring.c | 6 +++--- io_uring/net.c | 3 ++- io_uring/nop.c | 3 ++- io_uring/notif.c | 4 ++-- io_uring/rsrc.h | 16 ++++++++++------ io_uring/rw.c | 3 ++- io_uring/uring_cmd.c | 4 ++-- 8 files changed, 29 insertions(+), 17 deletions(-)
include/linux/io_uring_types.h +6 -1
@@ -479,6 +479,7 @@ enum {REQ_F_BL_NO_RECYCLE_BIT ,REQ_F_BUFFERS_COMMIT_BIT ,REQ_F_GROUP_LEADER_BIT , +REQ_F_BUF_NODE_BIT ,/* not a real bit, just to check we're not overflowing the space */ __REQ_F_LAST_BIT,
@@ -561,6 +562,8 @@ enum {REQ_F_BUFFERS_COMMIT =IO_REQ_FLAG (REQ_F_BUFFERS_COMMIT_BIT ),/* sqe group lead */ REQ_F_GROUP_LEADER =IO_REQ_FLAG (REQ_F_GROUP_LEADER_BIT ), +/* buf node is valid */ + REQ_F_BUF_NODE = IO_REQ_FLAG(REQ_F_BUF_NODE_BIT), };typedef void (*io_req_tw_func_t )(struct io_kiocb *req,struct io_tw_state *ts);
@@ -641,6 +644,8 @@ struct io_kiocb {* REQ_F_BUFFER_RING is set. */ struct io_buffer_list *buf_list; + +struct io_rsrc_node *buf_node; };union {
@@ -650,7 +655,7 @@ struct io_kiocb {__poll_t apoll_events; }; -struct io_rsrc_node *rsrc_nodes[2 ]; +struct io_rsrc_node *file_node;atomic_t refs;bool cancel_seq_set;
io_uring/io_uring.c +3 -3
@@ -1081,8 +1081,8 @@ void io_req_defer_failed(struct io_kiocb *req, s32 res)static void io_preinit_req (struct io_kiocb *req,struct io_ring_ctx *ctx) { req->ctx = ctx; - req->rsrc_nodes[IORING_RSRC_FILE ] =NULL ; - req->rsrc_nodes[IORING_RSRC_BUFFER ] =NULL ; + req->buf_node =NULL ; + req->file_node =NULL ; req->link =NULL ; req->async_data =NULL ;/* not necessary, but safer to zero */
@@ -2044,7 +2044,7 @@ inline struct file *io_file_get_fixed(struct io_kiocb *req, int fd,io_ring_submit_lock (ctx, issue_flags); node =io_rsrc_node_lookup (&ctx->file_table.data, fd);if (node) { -io_req_assign_rsrc_node (req, node); +io_req_assign_rsrc_node (&req->file_node, node); req->flags |=io_slot_flags (node); file =io_slot_file (node); }
io_uring/net.c +2 -1
@@ -1348,7 +1348,8 @@ static int io_send_zc_import(struct io_kiocb *req, unsigned int issue_flags)io_ring_submit_lock (ctx, issue_flags); node =io_rsrc_node_lookup (&ctx->buf_table, sr->buf_index);if (node) { -io_req_assign_rsrc_node (sr->notif, node); +io_req_assign_rsrc_node (&sr->notif->buf_node, node); + sr->notif->flags |=REQ_F_BUF_NODE ; ret =0 ; }io_ring_submit_unlock (ctx, issue_flags);
io_uring/nop.c +2 -1
@@ -67,7 +67,8 @@ int io_nop(struct io_kiocb *req, unsigned int issue_flags)io_ring_submit_lock (ctx, issue_flags); node =io_rsrc_node_lookup (&ctx->buf_table, nop->buffer);if (node) { -io_req_assign_rsrc_node (req, node); +io_req_assign_rsrc_node (&req->buf_node, node); + req->flags |=REQ_F_BUF_NODE ; ret =0 ; }io_ring_submit_unlock (ctx, issue_flags);
io_uring/notif.c +2 -2
@@ -117,8 +117,8 @@ struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx) notif->file =NULL ; notif->task = current;io_get_task_refs (1 ); - notif->rsrc_nodes[IORING_RSRC_FILE ] =NULL ; - notif->rsrc_nodes[IORING_RSRC_BUFFER ] =NULL ; + notif->file_node =NULL ; + notif->buf_node =NULL ; nd =io_notif_to_data (notif); nd->zc_report = false;
io_uring/rsrc.h +10 -6
@@ -95,10 +95,14 @@ static inline bool io_reset_rsrc_node(struct io_rsrc_data *data, int index)static inline void io_req_put_rsrc_nodes (struct io_kiocb *req) { -io_put_rsrc_node (req->rsrc_nodes[IORING_RSRC_FILE ]); -io_put_rsrc_node (req->rsrc_nodes[IORING_RSRC_BUFFER ]); - req->rsrc_nodes[IORING_RSRC_FILE ] =NULL ; - req->rsrc_nodes[IORING_RSRC_BUFFER ] =NULL ; +if (req->file_node) { +io_put_rsrc_node (req->file_node); + req->file_node =NULL ; + } +if (req->flags &REQ_F_BUF_NODE ) { +io_put_rsrc_node (req->buf_node); + req->buf_node =NULL ; + } }static inline struct io_ring_ctx *io_rsrc_node_ctx (struct io_rsrc_node *node)
@@ -111,11 +115,11 @@ static inline int io_rsrc_node_type(struct io_rsrc_node *node)return node->ctx_ptr &IORING_RSRC_TYPE_MASK ; } -static inline void io_req_assign_rsrc_node (struct io_kiocb *req, +static inline void io_req_assign_rsrc_node (struct io_rsrc_node **dst_node,struct io_rsrc_node *node) { node->refs++; - req->rsrc_nodes[io_rsrc_node_type (node)] = node; + *dst_node = node; }int io_files_update (struct io_kiocb *req,unsigned int issue_flags);
io_uring/rw.c +2 -1
@@ -341,7 +341,8 @@ static int io_prep_rw_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe node =io_rsrc_node_lookup (&ctx->buf_table, req->buf_index);if (!node)return -EFAULT ; -io_req_assign_rsrc_node (req, node); +io_req_assign_rsrc_node (&req->buf_node, node); + req->flags |=REQ_F_BUF_NODE ; io = req->async_data; ret =io_import_fixed (ddir, &io->iter, node->buf, rw->addr, rw->len);
io_uring/uring_cmd.c +2 -2
@@ -219,7 +219,7 @@ int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)* being called. This prevents destruction of the mapped buffer * we'll need at actual import time. */ -io_req_assign_rsrc_node (req, node); +io_req_assign_rsrc_node (&req->buf_node, node); } ioucmd->cmd_op =READ_ONCE (sqe->cmd_op);
@@ -275,7 +275,7 @@ int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw, struct iov_iter *iter, void *ioucmd) { struct io_kiocb *req = cmd_to_io_kiocb(ioucmd); -struct io_rsrc_node *node = req->rsrc_nodes[IORING_RSRC_BUFFER ]; +struct io_rsrc_node *node = req->buf_node;/* Must have had rsrc_node assigned at prep time */ if (node)
Signature · 1 line
2.45.2
From 020853e1546816a308067e6690d39b1a9c31a69d Mon Sep 17 00:00:00 2001
From: Jens Axboe <[email protected]>
Date: Sun, 3 Nov 2024 08:17:28 -0700
Subject: [PATCH 1/2] io_uring/rsrc: encode node type and ctx together
Rather than keep the type field separate rom ctx, use the fact that we can encode up to 4 types of nodes in the LSB of the ctx pointer. Doesn't reclaim any space right now on 64-bit archs, but it leaves a full int for future use.
--- io_uring/rsrc.c | 11 +++++------ io_uring/rsrc.h | 17 ++++++++++++++--- 2 files changed, 19 insertions(+), 9 deletions(-)
io_uring/rsrc.c +5 -6
@@ -124,9 +124,8 @@ struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx, int type) node =kzalloc (sizeof (*node),GFP_KERNEL );if (node) { - node->ctx = ctx; + node->ctx_ptr = (unsigned long ) ctx | type; node->refs =1 ; - node->type = type; }return node; }
@@ -445,21 +444,21 @@ int io_files_update(struct io_kiocb *req, unsigned int issue_flags)void io_free_rsrc_node (struct io_rsrc_node *node) { -struct io_ring_ctx *ctx = node->ctx; +struct io_ring_ctx *ctx =io_rsrc_node_ctx (node);lockdep_assert_held (&ctx->uring_lock);if (node->tag) -io_post_aux_cqe (node->ctx, node->tag,0 ,0 ); +io_post_aux_cqe (ctx, node->tag,0 ,0 ); -switch (node->type) { +switch (io_rsrc_node_type (node)) {case IORING_RSRC_FILE :if (io_slot_file (node))fput (io_slot_file (node));break ;case IORING_RSRC_BUFFER :if (node->buf) -io_buffer_unmap (node->ctx, node); +io_buffer_unmap (ctx, node);break ;default :WARN_ON_ONCE (1 );
io_uring/rsrc.h +14 -3
@@ -11,12 +11,13 @@enum {IORING_RSRC_FILE =0 ,IORING_RSRC_BUFFER =1 , + +IORING_RSRC_TYPE_MASK =0x3UL , };struct io_rsrc_node { -struct io_ring_ctx *ctx; +unsigned long ctx_ptr;int refs; -u16 type;u64 tag;union {
@@ -100,11 +101,21 @@ static inline void io_req_put_rsrc_nodes(struct io_kiocb *req) req->rsrc_nodes[IORING_RSRC_BUFFER ] =NULL ; } +static inline struct io_ring_ctx *io_rsrc_node_ctx (struct io_rsrc_node *node) +{ +return (struct io_ring_ctx *) (node->ctx_ptr & ~IORING_RSRC_TYPE_MASK ); +} + +static inline int io_rsrc_node_type (struct io_rsrc_node *node) +{ +return node->ctx_ptr &IORING_RSRC_TYPE_MASK ; +} +static inline void io_req_assign_rsrc_node (struct io_kiocb *req,struct io_rsrc_node *node) { node->refs++; - req->rsrc_nodes[node->type] = node; + req->rsrc_nodes[io_rsrc_node_type (node)] = node; }int io_files_update (struct io_kiocb *req,unsigned int issue_flags);
Signature · 1 line
2.45.2