io_uring: split req init from submit
Subsystems (2)
PATCH 1/3 io_uring: split req init from submit
From: Keith Busch <[email protected]>
Split the req initialization and link handling from the submit. This simplifies the submit path since everything that can fail is separate from it, and makes it easier to create batched submissions later.
--- io_uring/io_uring.c | 66 +++++++++++++++++++++++++-------------------- 1 file changed, 37 insertions(+), 29 deletions(-)
io_uring/io_uring.c +37 -29
@@ -2279,18 +2279,20 @@ static __cold int io_submit_fail_init(const struct io_uring_sqe *sqe,return 0 ; } -static inline int io_submit_sqe (struct io_ring_ctx *ctx,struct io_kiocb *req, -const struct io_uring_sqe *sqe) - __must_hold(&ctx->uring_lock) +static inline void io_submit_sqe (struct io_kiocb *req) { -struct io_submit_link *link = &ctx->submit_state.link; -int ret; +trace_io_uring_submit_req (req); - ret =io_init_req (ctx, req, sqe); -if (unlikely (ret)) -return io_submit_fail_init (sqe, req, ret); +if (unlikely (req->flags & (REQ_F_FORCE_ASYNC |REQ_F_FAIL ))) +io_queue_sqe_fallback (req); +else +io_queue_sqe (req); +} -trace_io_uring_submit_req (req); +static int io_setup_link (struct io_submit_link *link,struct io_kiocb **orig) +{ +struct io_kiocb *req = *orig; +int ret;/* * If we already have a head request, queue this one for async
@@ -2300,35 +2302,28 @@ static inline int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,* conditions are true (normal request), then just queue it. */ if (unlikely (link->head)) { + *orig =NULL ; + ret =io_req_prep_async (req);if (unlikely (ret)) -return io_submit_fail_init (sqe, req, ret); +return ret;trace_io_uring_link (req, link->head); link->last->link = req; link->last = req; -if (req->flags &IO_REQ_LINK_FLAGS )return 0 ; +/* last request of the link, flush it */ - req = link->head; + *orig = link->head; link->head =NULL ; -if (req->flags & (REQ_F_FORCE_ASYNC |REQ_F_FAIL )) - goto fallback; - - }else if (unlikely (req->flags & (IO_REQ_LINK_FLAGS | -REQ_F_FORCE_ASYNC |REQ_F_FAIL ))) { -if (req->flags &IO_REQ_LINK_FLAGS ) { - link->head = req; - link->last = req; - }else { -fallback: -io_queue_sqe_fallback (req); - } -return 0 ; + }else if (unlikely (req->flags &IO_REQ_LINK_FLAGS )) { + link->head = req; + link->last = req; + *orig =NULL ; +return 0 ; } -io_queue_sqe (req);return 0 ; }
@@ -2412,9 +2407,10 @@ static bool io_get_sqe(struct io_ring_ctx *ctx, const struct io_uring_sqe **sqe)int io_submit_sqes (struct io_ring_ctx *ctx,unsigned int nr) __must_hold(&ctx->uring_lock) { +struct io_submit_link *link = &ctx->submit_state.link;unsigned int entries =io_sqring_entries (ctx);unsigned int left; -int ret; +int ret, err;if (unlikely (!entries))return 0 ;
@@ -2434,12 +2430,24 @@ int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr)break ; } + err =io_init_req (ctx, req, sqe); +if (unlikely (err)) + goto error; + + err =io_setup_link (link, &req); +if (unlikely (err)) + goto error; + +if (likely (req)) +io_submit_sqe (req); +continue ; +error:/* * Continue submitting even for sqe failure if the * ring was setup with IORING_SETUP_SUBMIT_ALL */ -if (unlikely (io_submit_sqe (ctx, req, sqe)) && - !(ctx->flags &IORING_SETUP_SUBMIT_ALL )) { + err =io_submit_fail_init (sqe, req, err); +if (err && !(ctx->flags &IORING_SETUP_SUBMIT_ALL )) { left--;break ; }
Signature · 1 line
2.34.1
PATCH 2/3 io_uring: split req prep and submit loops
From: Keith Busch <[email protected]>
Do all the prep work up front, then dispatch all synchronous requests at once. This will make it easier to count batches for plugging.
--- io_uring/io_uring.c | 26 ++++++++++++++++++-------- io_uring/slist.h | 4 ++++ 2 files changed, 22 insertions(+), 8 deletions(-)
io_uring/io_uring.c +18 -8
@@ -1082,6 +1082,7 @@ static void io_preinit_req(struct io_kiocb *req, struct io_ring_ctx *ctx) req->ctx = ctx; req->link =NULL ; req->async_data =NULL ; + req->comp_list.next =NULL ;/* not necessary, but safer to zero */ req->cqe.res =0 ; }
@@ -2282,11 +2283,7 @@ static __cold int io_submit_fail_init(const struct io_uring_sqe *sqe,static inline void io_submit_sqe (struct io_kiocb *req) {trace_io_uring_submit_req (req); - -if (unlikely (req->flags & (REQ_F_FORCE_ASYNC |REQ_F_FAIL ))) -io_queue_sqe_fallback (req); -else -io_queue_sqe (req); +io_queue_sqe (req); }static int io_setup_link (struct io_submit_link *link,struct io_kiocb **orig)
@@ -2409,6 +2406,9 @@ int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) {struct io_submit_link *link = &ctx->submit_state.link;unsigned int entries =io_sqring_entries (ctx); +struct io_wq_work_node *pos, *next; +struct io_wq_work_list req_list; +struct io_kiocb *req;unsigned int left;int ret, err;
@@ -2419,6 +2419,7 @@ int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr)io_get_task_refs (left);io_submit_state_start (&ctx->submit_state, left); +INIT_WQ_LIST (&req_list); do { const struct io_uring_sqe *sqe; struct io_kiocb *req;
@@ -2437,9 +2438,12 @@ int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) err =io_setup_link (link, &req);if (unlikely (err)) goto error; - -if (likely (req)) -io_submit_sqe (req); +else if (unlikely (!req)) +continue ; +else if (unlikely (req->flags & (REQ_F_FORCE_ASYNC |REQ_F_FAIL ))) +io_queue_sqe_fallback (req); +else +wq_list_add_tail (&req->comp_list, &req_list);continue ; error:/*
@@ -2453,6 +2457,12 @@ int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) } }while (--left); +wq_list_for_each_safe (pos, next, &req_list) { + req = container_of(pos, struct io_kiocb, comp_list); + req->comp_list.next =NULL ; +io_submit_sqe (req); + } +if (unlikely (left)) { ret -= left;/* try again if it submitted nothing and can't allocate a req */
io_uring/slist.h +4 -0
@@ -12,6 +12,10 @@#define wq_list_for_each_resume (pos, prv) \ for (; pos; prv = pos, pos = (pos)->next) +#define wq_list_for_each_safe (pos, n, head) \ + for (pos = (head)->first, n = pos ? pos->next : NULL; \ + pos; pos = n, n = pos ? pos->next : NULL) +#define wq_list_empty (list) (READ_ONCE((list)->first) == NULL)#define INIT_WQ_LIST (list) do { \
Signature · 1 line
2.34.1
PATCH 3/3 io_uring: set plug tags for same file
In reply to Keith BuschFrom: Keith Busch <[email protected]>
io_uring tries to optimize allocating tags by hinting to the plug how many it expects to need for a batch instead of allocating each tag individually. But io_uring submission queueus may have a mix of many devices for io, so the number of io's counted may be overestimated. This can lead to allocating too many tags, which adds overhead to finding that many contiguous tags, freeing up the ones we didn't use, and may starve out other users that can actually use them.
When starting a new batch of uring commands, count only commands that match the file descriptor of the first seen for this optimization. This avoids have to call the unlikely "blk_mq_free_plug_rqs()" at the end of a submission when multiple devices are used in a batch.
--- block/blk-core.c | 49 +++++++++++++++------------------- block/blk-mq.c | 6 +++-- include/linux/io_uring_types.h | 1 + io_uring/io_uring.c | 19 ++++++++----- 4 files changed, 40 insertions(+), 35 deletions(-)
block/blk-core.c +22 -27
@@ -1043,32 +1043,6 @@ int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork, }EXPORT_SYMBOL (kblockd_mod_delayed_work_on); -void blk_start_plug_nr_ios (struct blk_plug *plug,unsigned short nr_ios) -{ -struct task_struct *tsk = current; - -/* -* If this is a nested plug, don't actually assign it. -*/ -if (tsk->plug) -return ; - - plug->mq_list =NULL ; - plug->cached_rq =NULL ; - plug->nr_ios = min_t(unsigned short, nr_ios, BLK_MAX_REQUEST_COUNT); - plug->rq_count =0 ; - plug->multiple_queues = false; - plug->has_elevator = false; - plug->nowait = false; -INIT_LIST_HEAD (&plug->cb_list); - -/* -* Store ordering should not be needed here, since a potential -* preempt will imply a full memory barrier -*/ - tsk->plug = plug; -} -/** * blk_start_plug - initialize blk_plug and track it inside the task_struct * @plug: The &struct blk_plug that needs to be initialized
@@ -1094,7 +1068,28 @@ void blk_start_plug_nr_ios(struct blk_plug *plug, unsigned short nr_ios)*/ void blk_start_plug (struct blk_plug *plug) { -blk_start_plug_nr_ios (plug,1 ); +struct task_struct *tsk = current; + +/* +* If this is a nested plug, don't actually assign it. +*/ +if (tsk->plug) +return ; + + plug->mq_list =NULL ; + plug->cached_rq =NULL ; + plug->nr_ios =1 ; + plug->rq_count =0 ; + plug->multiple_queues = false; + plug->has_elevator = false; + plug->nowait = false; +INIT_LIST_HEAD (&plug->cb_list); + +/* +* Store ordering should not be needed here, since a potential +* preempt will imply a full memory barrier +*/ + tsk->plug = plug; }EXPORT_SYMBOL (blk_start_plug);
block/blk-mq.c +4 -2
@@ -524,7 +524,8 @@ static struct request *blk_mq_rq_cache_fill(struct request_queue *q, .q = q, .flags = flags, .cmd_flags = opf, - .nr_tags = plug->nr_ios, + .nr_tags = min_t(unsigned int, plug->nr_ios, + BLK_MAX_REQUEST_COUNT), .cached_rq = &plug->cached_rq, };struct request *rq;
@@ -2867,7 +2868,8 @@ static struct request *blk_mq_get_new_requests(struct request_queue *q,rq_qos_throttle (q, bio);if (plug) { - data.nr_tags = plug->nr_ios; + data.nr_tags = min_t(unsigned int, plug->nr_ios, +BLK_MAX_REQUEST_COUNT ); plug->nr_ios =1 ; data.cached_rq = &plug->cached_rq; }
include/linux/io_uring_types.h +1 -0
@@ -175,6 +175,7 @@ struct io_submit_state {bool need_plug;unsigned short submit_nr;unsigned int cqes_count; +int fd;struct blk_plug plug;struct io_uring_cqe cqes[16 ]; };
io_uring/io_uring.c +13 -6
@@ -2209,18 +2209,25 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,return -EINVAL ;if (def->needs_file) { -struct io_submit_state *state = &ctx->submit_state; - req->cqe.fd =READ_ONCE (sqe->fd);/* * Plug now if we have more than 2 IO left after this, and the * target is potentially a read/write to block based storage. */ -if (state->need_plug && def->plug) { - state->plug_started = true; - state->need_plug = false; -blk_start_plug_nr_ios (&state->plug, state->submit_nr); +if (def->plug) { +struct io_submit_state *state = &ctx->submit_state; + +if (state->need_plug) { + state->plug_started = true; + state->need_plug = false; + state->fd = req->cqe.fd; +blk_start_plug (&state->plug); + }else if (state->plug_started && + state->fd == req->cqe.fd && + !state->link.head) { + state->plug.nr_ios++; + } } }
Signature · 1 line
2.34.1
kernel test robot
build botRe: [PATCH 3/3] io_uring: set plug tags for same file
kernel test robot noticed the following build errors:
Hi Keith,
kernel test robot noticed the following build errors:
[auto build test ERROR on axboe-block/for-next]
[also build test ERROR on linus/master v6.5-rc3 next-20230728] [If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Keith-Busch/io_uring-split-req-prep-and-submit-loops/20230729-042238 base: https://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux-block.git for-next patch link: https://lore.kernel.org/r/20230728201449.3350962-3-kbusch%40meta.com patch subject: [PATCH 3/3] io_uring: set plug tags for same file config: nios2-randconfig-r033-20230728 (https://download.01.org/0day-ci/archive/20230729/[email protected]/config) compiler: nios2-linux-gcc (GCC) 12.3.0 reproduce: (https://download.01.org/0day-ci/archive/20230729/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All errors (new ones prefixed by >>):
io_uring/io_uring.c: In function 'io_init_req':
>> io_uring/io_uring.c:2229:44: error: 'struct blk_plug' has no member named 'nr_ios'
2229 | state->plug.nr_ios++;
| ^vim +2229 io_uring/io_uring.c
2151
2152 static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
2153 const struct io_uring_sqe *sqe)
2154 __must_hold(&ctx->uring_lock)
2155 {
2156 const struct io_issue_def *def;
2157 unsigned int sqe_flags;
2158 int personality;
2159 u8 opcode;
2160
2161 /* req is partially pre-initialised, see io_preinit_req() */
2162 req->opcode = opcode = READ_ONCE(sqe->opcode);
2163 /* same numerical values with corresponding REQ_F_*, safe to copy */
2164 req->flags = sqe_flags = READ_ONCE(sqe->flags);
2165 req->cqe.user_data = READ_ONCE(sqe->user_data);
2166 req->file = NULL;
2167 req->rsrc_node = NULL;
2168 req->task = current;
2169
2170 if (unlikely(opcode >= IORING_OP_LAST)) {
2171 req->opcode = 0;
2172 return -EINVAL;
2173 }
2174 def = &io_issue_defs[opcode];
2175 if (unlikely(sqe_flags & ~SQE_COMMON_FLAGS)) {
2176 /* enforce forwards compatibility on users */
2177 if (sqe_flags & ~SQE_VALID_FLAGS)
2178 return -EINVAL;
2179 if (sqe_flags & IOSQE_BUFFER_SELECT) {
2180 if (!def->buffer_select)
2181 return -EOPNOTSUPP;
2182 req->buf_index = READ_ONCE(sqe->buf_group);
2183 }
2184 if (sqe_flags & IOSQE_CQE_SKIP_SUCCESS)
2185 ctx->drain_disabled = true;
2186 if (sqe_flags & IOSQE_IO_DRAIN) {
2187 if (ctx->drain_disabled)
2188 return -EOPNOTSUPP;
2189 io_init_req_drain(req);
2190 }
2191 }
2192 if (unlikely(ctx->restricted || ctx->drain_active || ctx->drain_next)) {
2193 if (ctx->restricted && !io_check_restriction(ctx, req, sqe_flags))
2194 return -EACCES;
2195 /* knock it to the slow queue path, will be drained there */
2196 if (ctx->drain_active)
2197 req->flags |= REQ_F_FORCE_ASYNC;
2198 /* if there is no link, we're at "next" request and need to drain */
2199 if (unlikely(ctx->drain_next) && !ctx->submit_state.link.head) {
2200 ctx->drain_next = false;
2201 ctx->drain_active = true;
2202 req->flags |= REQ_F_IO_DRAIN | REQ_F_FORCE_ASYNC;
2203 }
2204 }
2205
2206 if (!def->ioprio && sqe->ioprio)
2207 return -EINVAL;
2208 if (!def->iopoll && (ctx->flags & IORING_SETUP_IOPOLL))
2209 return -EINVAL;
2210
2211 if (def->needs_file) {
2212 req->cqe.fd = READ_ONCE(sqe->fd);
2213
2214 /*
2215 * Plug now if we have more than 2 IO left after this, and the
2216 * target is potentially a read/write to block based storage.
2217 */
2218 if (def->plug) {
2219 struct io_submit_state *state = &ctx->submit_state;
2220
2221 if (state->need_plug) {
2222 state->plug_started = true;
2223 state->need_plug = false;
2224 state->fd = req->cqe.fd;
2225 blk_start_plug(&state->plug);
2226 } else if (state->plug_started &&
2227 state->fd == req->cqe.fd &&
2228 !state->link.head) {
> 2229 state->plug.nr_ios++;
2230 }
2231 }
2232 }
2233
2234 personality = READ_ONCE(sqe->personality);
2235 if (personality) {
2236 int ret;
2237
2238 req->creds = xa_load(&ctx->personalities, personality);
2239 if (!req->creds)
2240 return -EINVAL;
2241 get_cred(req->creds);
2242 ret = security_uring_override_creds(req->creds);
2243 if (ret) {
2244 put_cred(req->creds);
2245 return ret;
2246 }
2247 req->flags |= REQ_F_CREDS;
2248 }
2249
2250 return def->prep(req, sqe);
2251 }
2252
Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki
kernel test robot
build botRe: [PATCH 3/3] io_uring: set plug tags for same file
In reply to Keith Buschkernel test robot noticed the following build errors:
Hi Keith,
kernel test robot noticed the following build errors:
[auto build test ERROR on axboe-block/for-next]
[also build test ERROR on linus/master v6.5-rc3 next-20230728] [If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Keith-Busch/io_uring-split-req-prep-and-submit-loops/20230729-042238 base: https://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux-block.git for-next patch link: https://lore.kernel.org/r/20230728201449.3350962-3-kbusch%40meta.com patch subject: [PATCH 3/3] io_uring: set plug tags for same file config: s390-randconfig-r022-20230728 (https://download.01.org/0day-ci/archive/20230729/[email protected]/config) compiler: clang version 15.0.7 (https://github.com/llvm/llvm-project.git 8dfdcc7b7bf66834a761bd8de445840ef68e4d1a) reproduce: (https://download.01.org/0day-ci/archive/20230729/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All errors (new ones prefixed by >>):
In file included from io_uring/io_uring.c:61:
In file included from include/net/sock.h:46:
In file included from include/linux/netdevice.h:38:
In file included from include/net/net_namespace.h:43:
In file included from include/linux/skbuff.h:28:
In file included from include/linux/dma-mapping.h:11:
In file included from include/linux/scatterlist.h:9:
In file included from arch/s390/include/asm/io.h:75:
include/asm-generic/io.h:547:31: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
val = __raw_readb(PCI_IOBASE + addr);
~~~~~~~~~~ ^
include/asm-generic/io.h:560:61: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
val = __le16_to_cpu((__le16 __force)__raw_readw(PCI_IOBASE + addr));
~~~~~~~~~~ ^
include/uapi/linux/byteorder/big_endian.h:37:59: note: expanded from macro '__le16_to_cpu'
#define __le16_to_cpu(x) __swab16((__force __u16)(__le16)(x))
^
include/uapi/linux/swab.h:102:54: note: expanded from macro '__swab16'
#define __swab16(x) (__u16)__builtin_bswap16((__u16)(x))
^
In file included from io_uring/io_uring.c:61:
In file included from include/net/sock.h:46:
In file included from include/linux/netdevice.h:38:
In file included from include/net/net_namespace.h:43:
In file included from include/linux/skbuff.h:28:
In file included from include/linux/dma-mapping.h:11:
In file included from include/linux/scatterlist.h:9:
In file included from arch/s390/include/asm/io.h:75:
include/asm-generic/io.h:573:61: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
val = __le32_to_cpu((__le32 __force)__raw_readl(PCI_IOBASE + addr));
~~~~~~~~~~ ^
include/uapi/linux/byteorder/big_endian.h:35:59: note: expanded from macro '__le32_to_cpu'
#define __le32_to_cpu(x) __swab32((__force __u32)(__le32)(x))
^
include/uapi/linux/swab.h:115:54: note: expanded from macro '__swab32'
#define __swab32(x) (__u32)__builtin_bswap32((__u32)(x))
^
In file included from io_uring/io_uring.c:61:
In file included from include/net/sock.h:46:
In file included from include/linux/netdevice.h:38:
In file included from include/net/net_namespace.h:43:
In file included from include/linux/skbuff.h:28:
In file included from include/linux/dma-mapping.h:11:
In file included from include/linux/scatterlist.h:9:
In file included from arch/s390/include/asm/io.h:75:
include/asm-generic/io.h:584:33: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
__raw_writeb(value, PCI_IOBASE + addr);
~~~~~~~~~~ ^
include/asm-generic/io.h:594:59: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
__raw_writew((u16 __force)cpu_to_le16(value), PCI_IOBASE + addr);
~~~~~~~~~~ ^
include/asm-generic/io.h:604:59: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
__raw_writel((u32 __force)cpu_to_le32(value), PCI_IOBASE + addr);
~~~~~~~~~~ ^
include/asm-generic/io.h:692:20: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
readsb(PCI_IOBASE + addr, buffer, count);
~~~~~~~~~~ ^
include/asm-generic/io.h:700:20: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
readsw(PCI_IOBASE + addr, buffer, count);
~~~~~~~~~~ ^
include/asm-generic/io.h:708:20: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
readsl(PCI_IOBASE + addr, buffer, count);
~~~~~~~~~~ ^
include/asm-generic/io.h:717:21: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
writesb(PCI_IOBASE + addr, buffer, count);
~~~~~~~~~~ ^
include/asm-generic/io.h:726:21: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
writesw(PCI_IOBASE + addr, buffer, count);
~~~~~~~~~~ ^
include/asm-generic/io.h:735:21: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
writesl(PCI_IOBASE + addr, buffer, count);
~~~~~~~~~~ ^
>> io_uring/io_uring.c:2229:17: error: no member named 'nr_ios' in 'struct blk_plug'
state->plug.nr_ios++;
~~~~~~~~~~~ ^12 warnings and 1 error generated.
vim +2229 io_uring/io_uring.c
2151
2152 static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
2153 const struct io_uring_sqe *sqe)
2154 __must_hold(&ctx->uring_lock)
2155 {
2156 const struct io_issue_def *def;
2157 unsigned int sqe_flags;
2158 int personality;
2159 u8 opcode;
2160
2161 /* req is partially pre-initialised, see io_preinit_req() */
2162 req->opcode = opcode = READ_ONCE(sqe->opcode);
2163 /* same numerical values with corresponding REQ_F_*, safe to copy */
2164 req->flags = sqe_flags = READ_ONCE(sqe->flags);
2165 req->cqe.user_data = READ_ONCE(sqe->user_data);
2166 req->file = NULL;
2167 req->rsrc_node = NULL;
2168 req->task = current;
2169
2170 if (unlikely(opcode >= IORING_OP_LAST)) {
2171 req->opcode = 0;
2172 return -EINVAL;
2173 }
2174 def = &io_issue_defs[opcode];
2175 if (unlikely(sqe_flags & ~SQE_COMMON_FLAGS)) {
2176 /* enforce forwards compatibility on users */
2177 if (sqe_flags & ~SQE_VALID_FLAGS)
2178 return -EINVAL;
2179 if (sqe_flags & IOSQE_BUFFER_SELECT) {
2180 if (!def->buffer_select)
2181 return -EOPNOTSUPP;
2182 req->buf_index = READ_ONCE(sqe->buf_group);
2183 }
2184 if (sqe_flags & IOSQE_CQE_SKIP_SUCCESS)
2185 ctx->drain_disabled = true;
2186 if (sqe_flags & IOSQE_IO_DRAIN) {
2187 if (ctx->drain_disabled)
2188 return -EOPNOTSUPP;
2189 io_init_req_drain(req);
2190 }
2191 }
2192 if (unlikely(ctx->restricted || ctx->drain_active || ctx->drain_next)) {
2193 if (ctx->restricted && !io_check_restriction(ctx, req, sqe_flags))
2194 return -EACCES;
2195 /* knock it to the slow queue path, will be drained there */
2196 if (ctx->drain_active)
2197 req->flags |= REQ_F_FORCE_ASYNC;
2198 /* if there is no link, we're at "next" request and need to drain */
2199 if (unlikely(ctx->drain_next) && !ctx->submit_state.link.head) {
2200 ctx->drain_next = false;
2201 ctx->drain_active = true;
2202 req->flags |= REQ_F_IO_DRAIN | REQ_F_FORCE_ASYNC;
2203 }
2204 }
2205
2206 if (!def->ioprio && sqe->ioprio)
2207 return -EINVAL;
2208 if (!def->iopoll && (ctx->flags & IORING_SETUP_IOPOLL))
2209 return -EINVAL;
2210
2211 if (def->needs_file) {
2212 req->cqe.fd = READ_ONCE(sqe->fd);
2213
2214 /*
2215 * Plug now if we have more than 2 IO left after this, and the
2216 * target is potentially a read/write to block based storage.
2217 */
2218 if (def->plug) {
2219 struct io_submit_state *state = &ctx->submit_state;
2220
2221 if (state->need_plug) {
2222 state->plug_started = true;
2223 state->need_plug = false;
2224 state->fd = req->cqe.fd;
2225 blk_start_plug(&state->plug);
2226 } else if (state->plug_started &&
2227 state->fd == req->cqe.fd &&
2228 !state->link.head) {
> 2229 state->plug.nr_ios++;
2230 }
2231 }
2232 }
2233
2234 personality = READ_ONCE(sqe->personality);
2235 if (personality) {
2236 int ret;
2237
2238 req->creds = xa_load(&ctx->personalities, personality);
2239 if (!req->creds)
2240 return -EINVAL;
2241 get_cred(req->creds);
2242 ret = security_uring_override_creds(req->creds);
2243 if (ret) {
2244 put_cred(req->creds);
2245 return ret;
2246 }
2247 req->flags |= REQ_F_CREDS;
2248 }
2249
2250 return def->prep(req, sqe);
2251 }
2252
Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki
Quoted 5 lines · Keith Busch
From: Keith Busch <[email protected]>
Split the req initialization and link handling from the submit. This simplifies the submit path since everything that can fail is separate from it, and makes it easier to create batched submissions later.
Keith, I don't think this prep patch does us any good, I'd rather shove the link assembling code further out of the common path. I like the first version more (see [1]). I'd suggest to merge it, and do cleaning up after.
I'll also say that IMHO the overhead is well justified. It's not only about having multiple nvmes, the problem slows down cases mixing storage with net and the rest of IO in a single ring.
[1] https://lore.kernel.org/io-uring/[email protected]/
Signature · 1 line
Pavel Begunkov
Jens Axboe
maintainerQuoted 12 lines · Pavel Begunkov
Quoted 5 lines · Keith Busch
From: Keith Busch <[email protected]>
Split the req initialization and link handling from the submit. This simplifies the submit path since everything that can fail is separate from it, and makes it easier to create batched submissions later.
Keith, I don't think this prep patch does us any good, I'd rather shove the link assembling code further out of the common path. I like the first version more (see [1]). I'd suggest to merge it, and do cleaning up after.
Quoted 5 lines
I'll also say that IMHO the overhead is well justified. It's not only about having multiple nvmes, the problem slows down cases mixing storage with net and the rest of IO in a single ring.
The downside of that one, to me, is that it just serializes all of it and we end up looping over the submission list twice. With alloc+init split, at least we get some locality wins by grouping the setup side of the requests.
Signature · 1 line
Jens Axboe
Quoted 13 lines · Jens Axboe
Quoted 12 lines · Pavel Begunkov
Quoted 5 lines · Keith Busch
From: Keith Busch <[email protected]>
Split the req initialization and link handling from the submit. This simplifies the submit path since everything that can fail is separate from it, and makes it easier to create batched submissions later.
Keith, I don't think this prep patch does us any good, I'd rather shove the link assembling code further out of the common path. I like the first version more (see [1]). I'd suggest to merge it, and do cleaning up after.
Quoted 8 lines
Quoted 5 lines
I'll also say that IMHO the overhead is well justified. It's not only about having multiple nvmes, the problem slows down cases mixing storage with net and the rest of IO in a single ring.
The downside of that one, to me, is that it just serializes all of it and we end up looping over the submission list twice.
Right, and there is nothing can be done if we want to know about all requests in advance, at least without changing uapi and/or adding userspace hints.
Quoted 3 lines
With alloc+init
split, at least we get some locality wins by grouping the setup side of the requests.
I don't think I follow, what grouping do you mean? As far as I see, v1 and v2 are essentially same with the difference of whether you have a helper for setting up links or not, see io_setup_link() from v2. In both cases it's executed in the same sequence:
1) init (generic init + opcode init + link setup) each request and put into a temporary list.
2) go go over the list and submit them one by one
And after inlining they should look pretty close.
Signature · 1 line
Pavel Begunkov
Quoted 36 lines · Pavel Begunkov
Quoted 13 lines · Jens Axboe
Quoted 12 lines · Pavel Begunkov
Quoted 5 lines · Keith Busch
From: Keith Busch <[email protected]>
Split the req initialization and link handling from the submit. This simplifies the submit path since everything that can fail is separate from it, and makes it easier to create batched submissions later.
Keith, I don't think this prep patch does us any good, I'd rather shove the link assembling code further out of the common path. I like the first version more (see [1]). I'd suggest to merge it, and do cleaning up after.
Quoted 8 lines
Quoted 5 lines
I'll also say that IMHO the overhead is well justified. It's not only about having multiple nvmes, the problem slows down cases mixing storage with net and the rest of IO in a single ring.
The downside of that one, to me, is that it just serializes all of it and we end up looping over the submission list twice.
Right, and there is nothing can be done if we want to know about all requests in advance, at least without changing uapi and/or adding userspace hints.
Quoted 3 lines
With alloc+init
split, at least we get some locality wins by grouping the setup side of the requests.I don't think I follow, what grouping do you mean? As far as I see, v1 and v2 are essentially same with the difference of whether you have a helper for setting up links or not, see
io_setup_link()from v2. In both cases it's executed in the same sequence:
Quoted 5 lines
1) init (generic init + opcode init + link setup) each request and put into a temporary list.
2) go go over the list and submit them one by oneAnd after inlining they should look pretty close.
The main difference in this one compared to the original version is that everything in the 2nd loop is just for the final dispatch. Anything that can fail, fallback, or defer to async happens in the first loop. I'm not sure that makes a difference in runtime, but having the 2nd loop handle only fast-path requests was what I set out to do for this version.
Quoted 41 lines · Keith Busch
Quoted 40 lines · Pavel Begunkov
Quoted 13 lines · Jens Axboe
Quoted 12 lines · Pavel Begunkov
Quoted 5 lines · Keith Busch
From: Keith Busch <[email protected]>
Split the req initialization and link handling from the submit. This simplifies the submit path since everything that can fail is separate from it, and makes it easier to create batched submissions later.
Keith, I don't think this prep patch does us any good, I'd rather shove the link assembling code further out of the common path. I like the first version more (see [1]). I'd suggest to merge it, and do cleaning up after.
Quoted 8 lines
Quoted 5 lines
I'll also say that IMHO the overhead is well justified. It's not only about having multiple nvmes, the problem slows down cases mixing storage with net and the rest of IO in a single ring.
The downside of that one, to me, is that it just serializes all of it and we end up looping over the submission list twice.
Right, and there is nothing can be done if we want to know about all requests in advance, at least without changing uapi and/or adding userspace hints.
Quoted 3 lines
With alloc+init
split, at least we get some locality wins by grouping the setup side of the requests.I don't think I follow, what grouping do you mean? As far as I see, v1 and v2 are essentially same with the difference of whether you have a helper for setting up links or not, see
io_setup_link()from v2. In both cases it's executed in the same sequence:1) init (generic init + opcode init + link setup) each request and put into a temporary list.
2) go go over the list and submit them one by one
Quoted 7 lines
Quoted 1 line
And after inlining they should look pretty close.
The main difference in this one compared to the original version is that everything in the 2nd loop is just for the final dispatch. Anything that can fail, fallback, or defer to async happens in the first loop. I'm not sure that makes a difference in runtime, but having the 2nd loop handle only fast-path requests was what I set out to do for this version.
For performance it doesn't matter, it's a very slow path and we should not be hitting it. And it only smears single req submission over multiple places, for instance it won't be legal to use io_submit_sqe() without those extra checks. Those are all minor points, but I don't think it's anyhow better than v1 in this aspect.
Signature · 1 line
Pavel Begunkov