liburing: add support for large CQE sizes
v2 cover liburing: add support for large CQE sizes
This adds support for large CQE sizes in the liburing layer. The large CQE sizes double the size compared to the default CQE size.
To support larger CQE sizes the mmap call needs to be modified to map a larger memory region for large CQE's. For default CQE's the size of the mapping stays the same.
Also the ring size calculation needs to change.
Finally when large CQE's are indexed, they need to take into account the bigger CQE size. The index manipulation remains unchanged, only when the CQE array is accessed, the offset is changed for large CQE's.
The nop test has been modified to test that the new values are set correctly.
Testing:
The liburing test suite has been run with the four different configurations:
- default
- large SQE
- large CQE
- large SQE & large CQE
To do this the default setting has been changed for the test run to the above values.:
To use these
changes, also the corresponding kernel changes are required.
Changes:
V2: the changed kernel definition of io_uring_cqe_extra has been applied to
the first patch in this patch series.
Stefan Roesch (6):
liburing: Update io_uring.h with large CQE kernel changes
liburing: increase mmap size for large CQE's
liburing: return correct ring size for large CQE's
liburing: index large CQE's correctly
liburing: add large CQE tests to nop test
liburing: Test all configurations with NOP test
src/include/liburing.h | 18 +++++++- src/include/liburing/io_uring.h | 13 ++++++ src/queue.c | 6 ++- src/setup.c | 13 ++++-- test/nop.c | 74 ++++++++++++++++++++------------- test/test.h | 35 ++++++++++++++++ 6 files changed, 123 insertions(+), 36 deletions(-) create mode 100644 test/test.h
base-commit: c0ba24d1215e9f2b08266b28b35436988c6f3543
Signature · 1 line
2.30.2
PATCH v2 1/6 liburing: Update io_uring.h with large CQE kernel changes
This updates the io_uring.h file with the changes in the kernel.
--- src/include/liburing/io_uring.h | 13 +++++++++++++ 1 file changed, 13 insertions(+)
src/include/liburing/io_uring.h +13 -0
@@ -113,6 +113,7 @@ enum {#define IORING_SETUP_R_DISABLED (1U << 6)/* start with ring disabled */ #define IORING_SETUP_SUBMIT_ALL (1U << 7)/* continue submit on error */ #define IORING_SETUP_SQE128 (1U << 8)/* SQEs are 128b */ +#define IORING_SETUP_CQE32 (1U << 9)/* CQEs are 32b */ enum {IORING_OP_NOP ,
@@ -198,6 +199,12 @@ enum {#define IORING_POLL_UPDATE_EVENTS (1U << 1)#define IORING_POLL_UPDATE_USER_DATA (1U << 2) +/* Extra padding for large CQEs. */ +struct io_uring_cqe_extra { +__u64 extra1; +__u64 extra2; +}; +/* * IO completion data structure (Completion Queue Entry) */
@@ -205,6 +212,12 @@ struct io_uring_cqe {__u64 user_data;/* sqe->data submission passed back */ __s32 res;/* result code for this event */ __u32 flags; + +/* +* If the ring is initialized wit IORING_SETUP_CQE32, then this field +* contains 16-bytes of padding, doubling the size fo the CQE. +*/ +struct io_uring_cqe_extra b[]; };/*
Signature · 1 line
2.30.2
PATCH v2 2/6 liburing: increase mmap size for large CQE's
In reply to Stefan RoeschThis doubles the mmap size for large CQE's.
--- src/setup.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-)
src/setup.c +5 -1
@@ -21,8 +21,12 @@ static int io_uring_mmap(int fd, struct io_uring_params *p,size_t size;int ret; + size =sizeof (struct io_uring_cqe ); +if (p->flags &IORING_SETUP_CQE32 ) + size +=sizeof (struct io_uring_cqe ); + sq->ring_sz = p->sq_off.array + p->sq_entries *sizeof (unsigned ); - cq->ring_sz = p->cq_off.cqes + p->cq_entries *sizeof (struct io_uring_cqe ); + cq->ring_sz = p->cq_off.cqes + p->cq_entries * size;if (p->features &IORING_FEAT_SINGLE_MMAP ) {if (cq->ring_sz > sq->ring_sz)
Signature · 1 line
2.30.2
PATCH v2 3/6 liburing: return correct ring size for large CQE's
In reply to Stefan RoeschReturn the correct ring_size when large CQE's are used.
--- src/setup.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-)
src/setup.c +5 -2
@@ -257,8 +257,11 @@ static size_t rings_size(struct io_uring_params *p, unsigned entries, {size_t pages, sq_size, cq_size; - cq_size =KRING_SIZE ; - cq_size += cq_entries *sizeof (struct io_uring_cqe ); + cq_size =sizeof (struct io_uring_cqe ); +if (p->flags &IORING_SETUP_CQE32 ) + cq_size +=sizeof (struct io_uring_cqe ); + cq_size *= cq_entries; + cq_size +=KRING_SIZE ; cq_size = (cq_size +63 ) & ~63UL ; pages = (size_t )1 <<npages (cq_size, page_size);
Signature · 1 line
2.30.2
PATCH v2 4/6 liburing: index large CQE's correctly
In reply to Stefan RoeschLarge CQE's need to take into account that each CQE has double the size. When the CQE array is indexed, the offset into the array needs to be changed accordingly.
--- src/include/liburing.h | 18 ++++++++++++++++-- src/queue.c | 6 +++++- 2 files changed, 21 insertions(+), 3 deletions(-)
src/include/liburing.h +16 -2
@@ -188,6 +188,16 @@ int __io_uring_get_cqe(struct io_uring *ring,#define LIBURING_UDATA_TIMEOUT ((__u64) -1) +/* +* Calculates the step size for CQE iteration. +* For standard CQE's its 1, for big CQE's its two. +*/ +#define io_uring_cqe_shift (ring) \ + (!!((ring)->flags & IORING_SETUP_CQE32)) + +#define io_uring_cqe_index (ring,ptr,mask) \ + (((ptr) & (mask)) << io_uring_cqe_shift(ring)) +#define io_uring_for_each_cqe (ring, head, cqe) \/* \ * io_uring_smp_load_acquire() enforces the order of tail \
@@ -195,7 +205,7 @@ int __io_uring_get_cqe(struct io_uring *ring,*/ \for (head = *(ring)->cq.khead; \ (cqe = (head !=io_uring_smp_load_acquire ((ring)->cq.ktail) ? \ - &(ring)->cq.cqes[head & (*(ring)->cq.kring_mask)] :NULL )); \ + &(ring)->cq.cqes[io_uring_cqe_index (ring, head, *(ring)->cq.kring_mask)] :NULL )); \ head++) \/*
@@ -844,6 +854,10 @@ static inline int __io_uring_peek_cqe(struct io_uring *ring,int err =0 ;unsigned available;unsigned mask = *ring->cq.kring_mask; +int shift =0 ; + +if (ring->flags &IORING_SETUP_CQE32 ) + shift =1 ; do { unsigned tail = io_uring_smp_load_acquire(ring->cq.ktail);
@@ -854,7 +868,7 @@ static inline int __io_uring_peek_cqe(struct io_uring *ring,if (!available)break ; - cqe = &ring->cq.cqes[head & mask]; + cqe = &ring->cq.cqes[(head & mask) << shift];if (!(ring->features &IORING_FEAT_EXT_ARG ) && cqe->user_data ==LIBURING_UDATA_TIMEOUT ) {if (cqe->res <0 )
src/queue.c +5 -1
@@ -132,6 +132,10 @@ unsigned io_uring_peek_batch_cqe(struct io_uring *ring, {unsigned ready;bool overflow_checked = false; +int shift =0 ; + +if (ring->flags &IORING_SETUP_CQE32 ) + shift =1 ; again: ready =io_uring_cq_ready (ring);
@@ -144,7 +148,7 @@ again: count = count > ready ? ready : count; last = head + count;for (;head != last; head++, i++) - cqes[i] = &ring->cq.cqes[head & mask]; + cqes[i] = &ring->cq.cqes[(head & mask) << shift];return count; }
Signature · 1 line
2.30.2
PATCH v2 5/6 liburing: add large CQE tests to nop test
In reply to Stefan RoeschThis adds two test cases for large CQE's:
- Single NOP test, which checks that the new extra1 and extra2 fields are set.
- Multiple NOP submission test which also checks for the new fields.
--- test/nop.c | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+)
test/nop.c +31 -0
@@ -19,6 +19,7 @@ static int test_single_nop(struct io_uring *ring)struct io_uring_cqe *cqe;struct io_uring_sqe *sqe;int ret; +bool cqe32 = (ring->flags &IORING_SETUP_CQE32 ); sqe =io_uring_get_sqe (ring);if (!sqe) {
@@ -27,6 +28,10 @@ static int test_single_nop(struct io_uring *ring) }io_uring_prep_nop (sqe); +if (cqe32) { + sqe->addr =1234 ; + sqe->addr2 =5678 ; + } sqe->user_data = ++seq; ret =io_uring_submit (ring);
@@ -44,6 +49,17 @@ static int test_single_nop(struct io_uring *ring)fprintf (stderr,"Unexpected 0 user_data\n" ); goto err; } +if (cqe32) { +if (cqe->b[0 ].extra1 !=1234 ) { +fprintf (stderr,"Unexpected extra1\n" ); + goto err; + + } +if (cqe->b[0 ].extra2 !=5678 ) { +fprintf (stderr,"Unexpected extra2\n" ); + goto err; + } + }io_uring_cqe_seen (ring, cqe);return 0 ; err:
@@ -55,6 +71,7 @@ static int test_barrier_nop(struct io_uring *ring)struct io_uring_cqe *cqe;struct io_uring_sqe *sqe;int ret, i; +bool cqe32 = (ring->flags &IORING_SETUP_CQE32 );for (i =0 ; i <8 ; i++) { sqe =io_uring_get_sqe (ring);
@@ -66,6 +83,10 @@ static int test_barrier_nop(struct io_uring *ring)io_uring_prep_nop (sqe);if (i ==4 ) sqe->flags =IOSQE_IO_DRAIN ; +if (cqe32) { + sqe->addr =1234 ; + sqe->addr2 =5678 ; + } sqe->user_data = ++seq; }
@@ -88,6 +109,16 @@ static int test_barrier_nop(struct io_uring *ring)fprintf (stderr,"Unexpected 0 user_data\n" ); goto err; } +if (cqe32) { +if (cqe->b[0 ].extra1 !=1234 ) { +fprintf (stderr,"Unexpected extra1\n" ); + goto err; + } +if (cqe->b[0 ].extra2 !=5678 ) { +fprintf (stderr,"Unexpected extra2\n" ); + goto err; + } + }io_uring_cqe_seen (ring, cqe); }
Signature · 1 line
2.30.2
PATCH v2 6/6 liburing: Test all configurations with NOP test
In reply to Stefan RoeschThis runs the NOP test with all four configurations:
- default SQE and CQE size
- large SQE size
- large CQE size
- large SQE and large CQE size
--- test/nop.c | 43 +++++++++++++------------------------------ test/test.h | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 30 deletions(-) create mode 100644 test/test.h
test/nop.c +13 -30
@@ -11,6 +11,7 @@#include <fcntl.h> #include "liburing.h" +#include "test.h" static int seq;
@@ -127,12 +128,14 @@ err:return 1 ; } -static int test_p (struct io_uring_params *p) +static int test_ring (unsigned flags) {struct io_uring ring; +struct io_uring_params p = { };int ret; - ret =io_uring_queue_init_params (8 , &ring, p); + p.flags = flags; + ret =io_uring_queue_init_params (8 , &ring, &p);if (ret) {fprintf (stderr,"ring setup failed: %d\n" , ret);return 1 ;
@@ -150,29 +153,11 @@ static int test_p(struct io_uring_params *p) goto err; } -io_uring_queue_exit (&ring); -return 0 ; err:io_uring_queue_exit (&ring);return ret; } -static int test_normal_ring (void ) -{ -struct io_uring_params p = { }; - -return test_p (&p); -} - -static int test_big_ring (void ) -{ -struct io_uring_params p = { }; - - p.flags =IORING_SETUP_SQE128 ; -return test_p (&p); -} - -int main (int argc,char *argv[]) {int ret;
@@ -180,17 +165,15 @@ int main(int argc, char *argv[])if (argc >1 )return 0 ; - ret =test_normal_ring (); -if (ret) { -fprintf (stderr,"Normal ring test failed\n" ); -return ret; - } - - ret =test_big_ring (); -if (ret) { -fprintf (stderr,"Big ring test failed\n" ); -return ret; + FOR_ALL_TEST_CONFIGS { +fprintf (stderr,"Testing %s config\n" ,IORING_GET_TEST_CONFIG_DESCRIPTION ()); + ret =test_ring (IORING_GET_TEST_CONFIG_FLAGS ()); +if (ret) { +fprintf (stderr,"Normal ring test failed\n" ); +return ret; + } } +fprintf (stderr,"PASS\n" );return 0 ; }
test/test.h new file +35 -0
@@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: MIT */ +/* +* Description: Test configs for tests. +*/ +#ifndef LIBURING_TEST_H +#define LIBURING_TEST_H + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct io_uring_test_config { +unsigned int flags; +const char *description; +}io_uring_test_config ; + +io_uring_test_config io_uring_test_configs[] = { + {0 ,"default" }, + {IORING_SETUP_SQE128 ,"large SQE" }, + {IORING_SETUP_CQE32 ,"large CQE" }, + {IORING_SETUP_SQE128 |IORING_SETUP_CQE32 ,"large SQE/CQE" }, +}; + +#define FOR_ALL_TEST_CONFIGS \ + for (int i = 0; i < sizeof(io_uring_test_configs) / sizeof(io_uring_test_configs[0]); i++) + +#define IORING_GET_TEST_CONFIG_FLAGS () (io_uring_test_configs[i].flags) +#define IORING_GET_TEST_CONFIG_DESCRIPTION () (io_uring_test_configs[i].description) + + +#ifdef __cplusplus +} +#endif + +#endif
Signature · 1 line
2.30.2
Quoted 9 lines · Stefan Roesch
This adds support for large CQE sizes in the liburing layer. The large CQE sizes double the size compared to the default CQE size.
To support larger CQE sizes the mmap call needs to be modified to map a larger memory region for large CQE's. For default CQE's the size of the mapping stays the same.
Also the ring size calculation needs to change.[...]
Applied, thanks!
[1/6] liburing: Update io_uring.h with large CQE kernel changes
commit: 80222605fa597ee72c3d89f9a7d59c6c24d5153a
[2/6] liburing: increase mmap size for large CQE's
commit: b2f684deb583070ca0693ba630f98c166f977662
[3/6] liburing: return correct ring size for large CQE's
commit: b4595bf204f5abbae94449f9f693d26e38bb67c1
[4/6] liburing: index large CQE's correctly
commit: 5950be3737d71c67c4784bad0fa73801b82cada2
[5/6] liburing: add large CQE tests to nop test
commit: b0d2bbcb0ed13347a4156000d6a5816247694ffc
[6/6] liburing: Test all configurations with NOP test
commit: 0281392aa21209a97b7024f4fca26b717ad213c3Best regards,
Signature · 1 line
Jens Axboe