liburing: support for mixed sqes
v6 cover liburing: support for mixed sqes
From: Keith Busch <[email protected]>
Changes from v5:
- Fixed up braces and line code style
- Added fdinfo mixed sqe tests
- Fixed up errors when CONFIG_USE_SANITIZER is enabled
- Changed api name to io_uring_get_sqe128 from
io_uring_get_sqe128_mixed
Keith Busch (6):
liburing: provide uring_cmd prep function Add support IORING_SETUP_SQE_MIXED
test: add nop testing for IORING_SETUP_SQE_MIXED
test: add mixed sqe test for uring commands test/fdinfo: flush sq prior to reading test/fdinfo: add mixed sqe option to fdinfo test
src/include/liburing.h | 84 ++++++++++++++++++- src/include/liburing/io_uring.h | 8 ++ src/sanitize.c | 4 +- test/Makefile | 3 + test/fdinfo.c | 51 +++++++++--- test/io_uring_passthrough.c | 18 ++-- test/sqe-mixed-bad-wrap.c | 87 ++++++++++++++++++++ test/sqe-mixed-nop.c | 82 +++++++++++++++++++ test/sqe-mixed-uring_cmd.c | 140 ++++++++++++++++++++++++++++++++ 9 files changed, 448 insertions(+), 29 deletions(-) create mode 100644 test/sqe-mixed-bad-wrap.c create mode 100644 test/sqe-mixed-nop.c create mode 100644 test/sqe-mixed-uring_cmd.c
Signature · 1 line
2.47.3
PATCH v6 1/6 liburing: provide uring_cmd prep function
From: Keith Busch <[email protected]>
The uring_cmd needs to ensure __pad1 is clear, which is a reserved field for opcode. If a prior submission for a different opcode at that same entry did use that field, the command would fail the kernel's checks if the caller didn't explicity clear it. Provide an init helper for this opcode.
Also, the nvme uring_cmd tests had a couple places setting the sqe addr and length, which are unused fields, so they shouldn't have been doing that, though it had been harmless to do so since the kernel isn't checking these fields.
Provide a helper function specific to the uring_cmd preparation.
--- src/include/liburing.h | 19 +++++++++++++++---- test/io_uring_passthrough.c | 18 ++++++------------ 2 files changed, 21 insertions(+), 16 deletions(-)
src/include/liburing.h +15 -4
@@ -1517,6 +1517,19 @@ IOURINGINLINE void io_uring_prep_socket_direct_alloc(struct io_uring_sqe *sqe,__io_uring_set_target_fixed_file (sqe,IORING_FILE_INDEX_ALLOC -1 ); } +IOURINGINLINE void io_uring_prep_uring_cmd(struct io_uring_sqe *sqe, +__u32 cmd_op, +int fd) +LIBURING_NOEXCEPT +{ + sqe->opcode =IORING_OP_URING_CMD ; + sqe->fd = fd; + sqe->cmd_op = cmd_op; + sqe->__pad1 =0 ; + sqe->addr =0ul ; + sqe->len =0 ; +} +/* * Prepare commands for sockets */
@@ -1529,11 +1542,10 @@ IOURINGINLINE void io_uring_prep_cmd_sock(struct io_uring_sqe *sqe, int optlen)LIBURING_NOEXCEPT { -io_uring_prep_rw (IORING_OP_URING_CMD , sqe, fd,NULL ,0 ,0 ); +io_uring_prep_uring_cmd (sqe, cmd_op, fd); sqe->optval = (unsigned long ) (uintptr_t ) optval; sqe->optname = optname; sqe->optlen = optlen; - sqe->cmd_op = cmd_op; sqe->level = level; }
@@ -1607,8 +1619,7 @@ IOURINGINLINE void io_uring_prep_cmd_discard(struct io_uring_sqe *sqe, uint64_t offset, uint64_t nbytes) LIBURING_NOEXCEPT { - io_uring_prep_rw(IORING_OP_URING_CMD, sqe, fd, 0, 0, 0); - sqe->cmd_op =BLOCK_URING_CMD_DISCARD ; +io_uring_prep_uring_cmd (sqe ,BLOCK_URING_CMD_DISCARD ,fd ); sqe->addr = offset; sqe->addr3 = nbytes; }
test/io_uring_passthrough.c +6 -12
@@ -141,18 +141,16 @@ static int __test_io(const char *file, struct io_uring *ring, int tc, int read,if (sqthread) use_fd =0 ; +if (nonvec) +io_uring_prep_uring_cmd (sqe,NVME_URING_CMD_IO , use_fd); +else +io_uring_prep_uring_cmd (sqe,NVME_URING_CMD_IO_VEC , use_fd);if (fixed && (i &1 )) do_fixed =0 ;if (do_fixed) sqe->buf_index =0 ;if (async) sqe->flags |=IOSQE_ASYNC ; -if (nonvec) - sqe->cmd_op =NVME_URING_CMD_IO ; -else - sqe->cmd_op =NVME_URING_CMD_IO_VEC ; - sqe->fd = use_fd; - sqe->opcode =IORING_OP_URING_CMD ;if (do_fixed) sqe->uring_cmd_flags |=IORING_URING_CMD_FIXED ; sqe->user_data = ((uint64_t )offset <<32 ) | i;
@@ -328,9 +326,7 @@ static int test_invalid_passthru_submit(const char *file) } sqe =io_uring_get_sqe (&ring); -io_uring_prep_read (sqe, fd, vecs[0 ].iov_base, vecs[0 ].iov_len,0 ); - sqe->cmd_op =NVME_URING_CMD_IO ; - sqe->opcode =IORING_OP_URING_CMD ; +io_uring_prep_uring_cmd (sqe,NVME_URING_CMD_IO , fd); sqe->user_data =1 ; cmd = (struct nvme_uring_cmd *)sqe->cmd;memset (cmd,0 ,sizeof (struct nvme_uring_cmd ));
@@ -401,10 +397,8 @@ static int test_io_uring_submit_enters(const char *file)__u32 nlb; sqe =io_uring_get_sqe (&ring); -io_uring_prep_readv (sqe, fd, &vecs[i],1 , offset); +io_uring_prep_uring_cmd (sqe,NVME_URING_CMD_IO , fd); sqe->user_data = i; - sqe->opcode =IORING_OP_URING_CMD ; - sqe->cmd_op =NVME_URING_CMD_IO ; cmd = (struct nvme_uring_cmd *)sqe->cmd;memset (cmd,0 ,sizeof (struct nvme_uring_cmd ));
Signature · 1 line
2.47.3
PATCH v6 2/6 Add support IORING_SETUP_SQE_MIXED
In reply to Keith BuschFrom: Keith Busch <[email protected]>
This adds core support for mixed sized SQEs in the same SQ ring. Before this, SQEs were either 64b in size (the normal size), or 128b if IORING_SETUP_SQE128 was set in the ring initialization. With the mixed support, an SQE may be either 64b or 128b on the same SQ ring. If the SQE is 128b in size, then a 128b opcode will be set in the sqe op. When acquiring a large sqe at the end of the sq, the client may post a NOP SQE with IOSQE_CQE_SKIP_SUCCESS set that the kernel will process and skip posting a CQE.
--- src/include/liburing.h | 73 +++++++++++++++++++++++++++++++-- src/include/liburing/io_uring.h | 8 ++++ src/sanitize.c | 4 +- 3 files changed, 80 insertions(+), 5 deletions(-)
src/include/liburing.h +69 -4
@@ -800,6 +800,12 @@ IOURINGINLINE void io_uring_prep_nop(struct io_uring_sqe *sqe)io_uring_prep_rw (IORING_OP_NOP , sqe,-1 ,NULL ,0 ,0 ); } +IOURINGINLINE void io_uring_prep_nop128(struct io_uring_sqe *sqe) +LIBURING_NOEXCEPT +{ +io_uring_prep_rw (IORING_OP_NOP128 , sqe,-1 ,NULL ,0 ,0 ); +} + IOURINGINLINE void io_uring_prep_timeout(struct io_uring_sqe *sqe,const struct __kernel_timespec *ts,unsigned count,unsigned flags)
@@ -1517,12 +1523,13 @@ IOURINGINLINE void io_uring_prep_socket_direct_alloc(struct io_uring_sqe *sqe,__io_uring_set_target_fixed_file (sqe,IORING_FILE_INDEX_ALLOC -1 ); } -IOURINGINLINE void io_uring_prep_uring_cmd(struct io_uring_sqe *sqe, -__u32 cmd_op, -int fd) +IOURINGINLINE void __io_uring_prep_uring_cmd(struct io_uring_sqe *sqe, +int op, +__u32 cmd_op, +int fd)LIBURING_NOEXCEPT { - sqe->opcode =IORING_OP_URING_CMD ; + sqe->opcode = (__u8 ) op; sqe->fd = fd; sqe->cmd_op = cmd_op; sqe->__pad1 =0 ;
@@ -1530,6 +1537,22 @@ IOURINGINLINE void io_uring_prep_uring_cmd(struct io_uring_sqe *sqe, sqe->len =0 ; } +IOURINGINLINE void io_uring_prep_uring_cmd(struct io_uring_sqe *sqe, +int cmd_op, +int fd) +LIBURING_NOEXCEPT +{ +__io_uring_prep_uring_cmd (sqe,IORING_OP_URING_CMD , cmd_op, fd); +} + +IOURINGINLINE void io_uring_prep_uring_cmd128(struct io_uring_sqe *sqe, +int cmd_op, +int fd) +LIBURING_NOEXCEPT +{ +__io_uring_prep_uring_cmd (sqe,IORING_OP_URING_CMD128 , cmd_op, fd); +} +/* * Prepare commands for sockets */
@@ -2007,6 +2030,48 @@ IOURINGINLINE struct io_uring_sqe *io_uring_get_sqe(struct io_uring *ring)struct io_uring_sqe *io_uring_get_sqe (struct io_uring *ring);#endif + +/* +* Return a 128B sqe to fill. Applications must later call io_uring_submit() +* when it's ready to tell the kernel about it. The caller may call this +* function multiple times before calling io_uring_submit(). +* +* Returns a vacant 128B sqe, or NULL if we're full. If the current tail is the +* last entry in the ring, this function will insert a nop + skip complete such +* that the 128b entry wraps back to the beginning of the queue for a +* contiguous big sq entry. It's up to the caller to use a 128b opcode in order +* for the kernel to know how to advance its sq head pointer. +*/ +IOURINGINLINE struct io_uring_sqe *io_uring_get_sqe128(struct io_uring *ring) +LIBURING_NOEXCEPT +{ +struct io_uring_sq *sq = &ring->sq; +unsigned head =io_uring_load_sq_head (ring), tail = sq->sqe_tail; +struct io_uring_sqe *sqe; + +if (ring->flags &IORING_SETUP_SQE128 ) +return io_uring_get_sqe (ring); +if (!(ring->flags &IORING_SETUP_SQE_MIXED )) +return NULL ; + +if (((tail +1 ) & sq->ring_mask) ==0 ) { +if ((tail +2 ) - head >= sq->ring_entries) +return NULL ; + + sqe =_io_uring_get_sqe (ring); +io_uring_prep_nop (sqe); + sqe->flags |=IOSQE_CQE_SKIP_SUCCESS ; + tail = sq->sqe_tail; + }else if ((tail +1 ) - head >= sq->ring_entries) { +return NULL ; + } + + sqe = &sq->sqes[tail & sq->ring_mask]; + sq->sqe_tail = tail +2 ; +io_uring_initialize_sqe (sqe); +return sqe; +} + ssize_t io_uring_mlock_size(unsigned entries, unsigned flags) LIBURING_NOEXCEPT;ssize_t io_uring_mlock_size_params (unsigned entries,struct io_uring_params *p)
src/include/liburing/io_uring.h +8 -0
@@ -211,6 +211,12 @@ enum io_uring_sqe_flags_bit {*/ #define IORING_SETUP_CQE_MIXED (1U << 18) +/* +* Allow both 64b and 128b SQEs. If a 128b SQE is posted, it will use a 128b +* opcode. +*/ +#define IORING_SETUP_SQE_MIXED (1U << 19) +enum io_uring_op {IORING_OP_NOP ,IORING_OP_READV ,
@@ -275,6 +281,8 @@ enum io_uring_op {IORING_OP_READV_FIXED ,IORING_OP_WRITEV_FIXED ,IORING_OP_PIPE , +IORING_OP_NOP128 , +IORING_OP_URING_CMD128 ,/* this goes last, obviously */ IORING_OP_LAST,
src/sanitize.c +3 -1
@@ -120,7 +120,9 @@ static inline void initialize_sanitize_handlers() sanitize_handlers[IORING_OP_READV_FIXED ] = sanitize_sqe_addr; sanitize_handlers[IORING_OP_WRITEV_FIXED ] = sanitize_sqe_addr; sanitize_handlers[IORING_OP_PIPE ] = sanitize_sqe_addr; -_Static_assert (IORING_OP_PIPE +1 ==IORING_OP_LAST ,"Need an implementation for all IORING_OP_* codes" ); + sanitize_handlers[IORING_OP_NOP128 ] = sanitize_sqe_nop; + sanitize_handlers[IORING_OP_URING_CMD128 ] = sanitize_sqe_optval; +_Static_assert (IORING_OP_URING_CMD128 +1 ==IORING_OP_LAST ,"Need an implementation for all IORING_OP_* codes" ); sanitize_handlers_initialized = true; }
Signature · 1 line
2.47.3
Jens Axboe
maintainerRe: [PATCHv6 2/6] Add support IORING_SETUP_SQE_MIXED
Quoted 31 lines · Keith Busch
src/include/liburing.h +31
-0+ +/* +* Return a 128B sqe to fill. Applications must later call io_uring_submit() +* when it's ready to tell the kernel about it. The caller may call this +* function multiple times before calling io_uring_submit(). +* +* Returns a vacant 128B sqe, or NULL if we're full. If the current tail is the +* last entry in the ring, this function will insert a nop + skip complete such +* that the 128b entry wraps back to the beginning of the queue for a +* contiguous big sq entry. It's up to the caller to use a 128b opcode in order +* for the kernel to know how to advance its sq head pointer. +*/ +IOURINGINLINE struct io_uring_sqe *io_uring_get_sqe128(struct io_uring *ring) +LIBURING_NOEXCEPT +{ +struct io_uring_sq *sq = &ring->sq; +unsigned head =io_uring_load_sq_head (ring), tail = sq->sqe_tail; +struct io_uring_sqe *sqe; + +if (ring->flags &IORING_SETUP_SQE128 ) +return io_uring_get_sqe (ring); +if (!(ring->flags &IORING_SETUP_SQE_MIXED )) +return NULL ; + +if (((tail +1 ) & sq->ring_mask) ==0 ) { +if ((tail +2 ) - head >= sq->ring_entries) +return NULL ; + + sqe =_io_uring_get_sqe (ring); +io_uring_prep_nop (sqe); + sqe->flags |=IOSQE_CQE_SKIP_SUCCESS ;
src/include/liburing.h IOURINGINLINE struct io_uring_sqe *io_uring_get_sqe(struct io_uring *ring) · line 2073
Quoted 10 lines
src/include/liburing.h +10
-0+ tail = sq->sqe_tail; + }else if ((tail +1 ) - head >= sq->ring_entries) { +return NULL ; + } + + sqe = &sq->sqes[tail & sq->ring_mask]; + sq->sqe_tail = tail +2 ; +io_uring_initialize_sqe (sqe); +return sqe; +}
I did apply the patches, but can you add a man page addition for this one?
Signature · 1 line
Jens Axboe
PATCH v6 3/6 test: add nop testing for IORING_SETUP_SQE_MIXED
In reply to Keith BuschFrom: Keith Busch <[email protected]>
Test mixing 64 and 128 byte sqe entries on a queue.
Also introduce a negative test case that inserts a bad 128b operation at the end of a mixed sqe to test the kernel's invalid entry detection.
--- test/Makefile | 2 + test/sqe-mixed-bad-wrap.c | 87 ++++++++++++++++++++++++++++++++++++++ test/sqe-mixed-nop.c | 82 +++++++++++++++++++++++++++++++++++ test/sqe-mixed-uring_cmd.c | 0 4 files changed, 171 insertions(+) create mode 100644 test/sqe-mixed-bad-wrap.c create mode 100644 test/sqe-mixed-nop.c create mode 100644 test/sqe-mixed-uring_cmd.c
test/Makefile +2 -0
@@ -237,6 +237,8 @@ test_srcs := \ sq-poll-share.c \ sqpoll-sleep.c \ sq-space_left.c \ + sqe-mixed-nop.c \ + sqe-mixed-bad-wrap.c \ sqwait.c \ stdout.c \ submit-and-wait.c \
test/sqe-mixed-bad-wrap.c new file +87 -0
87 lines not on this page · raw message
test/sqe-mixed-nop.c new file +82 -0
82 lines not on this page · raw message
test/sqe-mixed-uring_cmd.c new file +0 -0
Signature · 1 line
2.47.3
PATCH v6 4/6 test: add mixed sqe test for uring commands
In reply to Keith BuschFrom: Keith Busch <[email protected]>
--- test/Makefile | 1 + test/sqe-mixed-uring_cmd.c | 140 +++++++++++++++++++++++++++++++++++++ 2 files changed, 141 insertions(+)
test/Makefile +1 -0
@@ -239,6 +239,7 @@ test_srcs := \ sq-space_left.c \ sqe-mixed-nop.c \ sqe-mixed-bad-wrap.c \ + sqe-mixed-uring_cmd.c \ sqwait.c \ stdout.c \ submit-and-wait.c \
test/sqe-mixed-uring_cmd.c +140 -0
140 lines not on this page · raw message
Signature · 1 line
2.47.3
PATCH v6 5/6 test/fdinfo: flush sq prior to reading
In reply to Keith BuschFrom: Keith Busch <[email protected]>
Ensure the kernel can see the updated tail value before reading the ring's fdinfo. This tests the kernel's handling for walking the submission queue.
--- test/fdinfo.c | 1 + 1 file changed, 1 insertion(+)
test/fdinfo.c +1 -0
@@ -167,6 +167,7 @@ static int __test_io(const char *file, struct io_uring *ring, int write, offset +=BS ; } +__io_uring_flush_sq (ring);fdinfo_read (ring); ret =io_uring_submit (ring);
Signature · 1 line
2.47.3
PATCH v6 6/6 test/fdinfo: add mixed sqe option to fdinfo test
In reply to Keith BuschFrom: Keith Busch <[email protected]>
--- test/fdinfo.c | 50 ++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 12 deletions(-)
test/fdinfo.c +38 -12
@@ -57,7 +57,7 @@ static void fdinfo_read(struct io_uring *ring)static int __test_io (const char *file,struct io_uring *ring,int write,int buffered,int sqthread,int fixed,int nonvec, -int buf_select,int seq,int exp_len) +int buf_select,int seq,int exp_len,int mixed) {struct io_uring_sqe *sqe;struct io_uring_cqe *cqe;
@@ -66,9 +66,9 @@ static int __test_io(const char *file, struct io_uring *ring, int write,off_t offset;#ifdef VERBOSE -fprintf (stdout,"%s: start %d/%d/%d/%d/%d: " , __FUNCTION__, write, +fprintf (stdout,"%s: start %d/%d/%d/%d/%d/%d: " , __FUNCTION__, write, buffered, sqthread, - fixed, nonvec); + fixed, nonvec, mixed);#endif if (write) open_flags =O_WRONLY ;
@@ -107,6 +107,17 @@ static int __test_io(const char *file, struct io_uring *ring, int write, offset =0 ;for (i =0 ; i <BUFFERS ; i++) { +if (mixed && i &1 ) { + sqe =io_uring_get_sqe128 (ring); +if (!sqe) { +fprintf (stderr,"sqe get failed\n" ); + goto err; + } +io_uring_prep_nop128 (sqe); + sqe->user_data = i; +continue ; + } + sqe =io_uring_get_sqe (ring);if (!sqe) {fprintf (stderr,"sqe get failed\n" );
@@ -171,9 +182,13 @@ static int __test_io(const char *file, struct io_uring *ring, int write,fdinfo_read (ring); ret =io_uring_submit (ring); -if (ret !=BUFFERS ) { +if (!mixed && ret !=BUFFERS ) {fprintf (stderr,"submit got %d, wanted %d\n" , ret,BUFFERS ); goto err; + }else if (mixed && ret !=BUFFERS + (BUFFERS >>1 )) { +fprintf (stderr,"submit got %d, wanted %d\n" , ret, +BUFFERS + (BUFFERS >>1 )); + goto err; }for (i =0 ; i <10 ; i++) {
@@ -187,7 +202,13 @@ static int __test_io(const char *file, struct io_uring *ring, int write,fprintf (stderr,"wait_cqe=%d\n" , ret); goto err; } -if (cqe->res == -EINVAL && nonvec) { +if (mixed && cqe->user_data &1 ) { +if (cqe->user_data >32 ) { +fprintf (stderr,"cqe user data %lld, max expected %d\n" , + cqe->user_data,BUFFERS -1 ); + goto err; + } + }else if (cqe->res == -EINVAL && nonvec) {if (!warned) {fprintf (stdout,"Non-vectored IO not " "supported, skipping\n" );
@@ -253,15 +274,19 @@ err:return 1 ; }static int test_io (const char *file,int write,int buffered,int sqthread, -int fixed,int nonvec,int exp_len) +int fixed,int nonvec,int exp_len,int mixed) {struct io_uring ring; -int ret, ring_flags =0 ; +int ret, ring_flags =0 , entries =64 ;if (sqthread) ring_flags =IORING_SETUP_SQPOLL ; +if (mixed) { + ring_flags |=IORING_SETUP_SQE_MIXED ; + entries =128 ; + } - ret =t_create_ring (64 , &ring, ring_flags); + ret =t_create_ring (entries, &ring, ring_flags);if (ret ==T_SETUP_SKIP )return 0 ;if (ret !=T_SETUP_OK ) {
@@ -270,7 +295,7 @@ static int test_io(const char *file, int write, int buffered, int sqthread, } ret =__test_io (file, &ring, write, buffered, sqthread, fixed, nonvec, -0 ,0 , exp_len); +0 ,0 , exp_len, mixed);io_uring_queue_exit (&ring);return ret; }
@@ -380,17 +405,18 @@ int main(int argc, char *argv[]) vecs =t_create_buffers (BUFFERS ,BS );/* if we don't have nonvec read, skip testing that */ - nr =has_nonvec_read () ?32 :16 ; + nr =has_nonvec_read () ?64 :32 ;for (i =0 ; i < nr; i++) {int write = (i &1 ) !=0 ;int buffered = (i &2 ) !=0 ;int sqthread = (i &4 ) !=0 ;int fixed = (i &8 ) !=0 ; -int nonvec = (i &16 ) !=0 ; +int mixed = (i &16 ) !=0 ; +int nonvec = (i &32 ) !=0 ; ret =test_io (fname, write, buffered, sqthread, fixed, nonvec, -BS ); +BS , mixed);if (ret ==T_EXIT_SKIP )continue ;if (ret) {
Signature · 1 line
2.47.3
Quoted 9 lines · Keith Busch
Changes from v5:
- Fixed up braces and line code style
- Added fdinfo mixed sqe tests
- Fixed up errors when
CONFIG_USE_SANITIZERis enabled[...]
Applied, thanks!
[1/6] liburing: provide uring_cmd prep function
commit: 45c13470e7fd050fcd94b99d00ed58576b7fc4bf
[2/6] Add support IORING_SETUP_SQE_MIXED
commit: e1453a03f3e5f568f94e9cf9008fa1862154f7c1
[3/6] test: add nop testing for IORING_SETUP_SQE_MIXED
commit: 293e571185b78098ba4e95cf1723b0bad0cbfad9
[4/6] test: add mixed sqe test for uring commands
commit: 43f0b21ade9d0b69197dad4f1517a113150e7b47
[5/6] test/fdinfo: flush sq prior to reading
commit: d118e9e77c2e4a4017f3d000af349e59d6adf686
[6/6] test/fdinfo: add mixed sqe option to fdinfo test
commit: a41ea23839da482d9585b83e46398147e4c4d1aaBest regards,
Signature · 1 line
Jens Axboe