ring: convert to C11 atomics where practical
cover ring: convert to C11 atomics where practical
This is split out from the atomic deprecation series. Convert lib/ring off rte_atomic32 and onto the C11 memory model, except where C11 version has noticable performance drop on x86 with GCC.
The pre-existing C11 and GCC-builtin paths lived in separate headers with substantial duplication. After this series, only the MP head CAS (__rte_ring_headtail_move_head_mt) retains separate implementations; everything else is shared. Patch 2 documents the reason for keeping the GCC builtin on the MP head CAS.
The default RTE_USE_C11_MEM_MODEL selection per architecture is unchanged.
Stephen Hemminger (5):
ring: split single thread vs multi-thread cases
ring: use GCC builtin as alternative to rte_atomic32
ring: use C11 for update_tail
ring: drop unused arg to update_tail
ring: use C11 for single thread move head
lib/ring/meson.build | 2 +-
lib/ring/rte_ring_c11_pvt.h | 61 +++------
lib/ring/rte_ring_elem_pvt.h | 116 ++++++++++++++++--
..._ring_generic_pvt.h => rte_ring_gcc_pvt.h} | 62 +++-------
lib/ring/rte_ring_hts_elem_pvt.h | 8 +-
lib/ring/soring.c | 34 ++---
6 files changed, 161 insertions(+), 122 deletions(-)
rename lib/ring/{rte_ring_generic_pvt.h => rte_ring_gcc_pvt.h} (63%)Signature · 1 line
2.53.0
PATCH 1/5 ring: split single thread vs multi-thread cases
The move head function has optimization for updating when being used on single threaded ring. Code is cleaner if the two cases are split into separate functions.
--- lib/ring/rte_ring_c11_pvt.h | 100 +++++++++++++++++++++++++------- lib/ring/rte_ring_elem_pvt.h | 16 +++-- lib/ring/rte_ring_generic_pvt.h | 77 ++++++++++++++++++++---- lib/ring/soring.c | 24 +++++--- 4 files changed, 171 insertions(+), 46 deletions(-)
lib/ring/rte_ring_c11_pvt.h +79 -21
@@ -46,6 +46,7 @@ __rte_ring_update_tail(struct rte_ring_headtail *ht, uint32_t old_val,/** * @internal This is a helper function that moves the producer/consumer head +* optimized for single threaded case * * @param d * A pointer to the headtail structure with head value to be moved
@@ -54,8 +55,6 @@ __rte_ring_update_tail(struct rte_ring_headtail *ht, uint32_t old_val,* function only reads tail value from it * @param capacity * Either ring capacity value (for producer), or zero (for consumer) -* @param is_st -* Indicates whether multi-thread safe path is needed or not * @param n * The number of elements we want to move head value on * @param behavior
@@ -72,14 +71,77 @@ __rte_ring_update_tail(struct rte_ring_headtail *ht, uint32_t old_val,* If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only */ static __rte_always_inline unsigned int -__rte_ring_headtail_move_head (struct rte_ring_headtail *d, +__rte_ring_headtail_move_head_st (struct rte_ring_headtail *d,const struct rte_ring_headtail *s,uint32_t capacity, -unsigned int is_st,unsigned int n, +unsigned int n,enum rte_ring_queue_behavior behavior,uint32_t *old_head,uint32_t *new_head,uint32_t *entries) {uint32_t stail; -int success; + +/* Single producer: only this thread writes d->head, +* so a relaxed load is sufficient. +*/ + *old_head =rte_atomic_load_explicit (&d->head, rte_memory_order_relaxed); + +/* Acquire pairs with the consumer's release-store of tail in __rte_ring_update_tail, +* ensuring the consumer's ring-element reads are complete before +* we observe the updated tail. +*/ + stail =rte_atomic_load_explicit (&s->tail, rte_memory_order_acquire); + +/* Unsigned subtraction is modulo 2^32, so entries is always in +* [0, capacity) even if old_head > stail. +*/ + *entries = capacity + stail - *old_head; + +/* check that we have enough room in ring */ +if (unlikely (n > *entries)) + n = (behavior ==RTE_RING_QUEUE_FIXED ) ?0 : *entries; + +if (n >0 ) { + *new_head = *old_head + n; +rte_atomic_store_explicit (&d->head, *new_head, rte_memory_order_relaxed); + } + +return n; +} + +/** +* @internal This is a helper function that moves the producer/consumer head +* for use in multi-thread safe path +* +* @param d +* A pointer to the headtail structure with head value to be moved +* @param s +* A pointer to the counter-part headtail structure. Note that this +* function only reads tail value from it +* @param capacity +* Either ring capacity value (for producer), or zero (for consumer) +* @param n +* The number of elements we want to move head value on +* @param behavior +* RTE_RING_QUEUE_FIXED: Move on a fixed number of items +* RTE_RING_QUEUE_VARIABLE: Move on as many items as possible +* @param old_head +* Returns head value as it was before the move +* @param new_head +* Returns the new head value +* @param entries +* Returns the number of ring entries available BEFORE head was moved +* @return +* Actual number of objects the head was moved on +* If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only +*/ +static __rte_always_inline unsigned int +__rte_ring_headtail_move_head_mt (struct rte_ring_headtail *d, +const struct rte_ring_headtail *s,uint32_t capacity, +unsigned int n, +enum rte_ring_queue_behavior behavior, +uint32_t *old_head,uint32_t *new_head,uint32_t *entries) +{ +uint32_t stail; +bool success;unsigned int max = n;/*
@@ -120,25 +182,21 @@ __rte_ring_headtail_move_head(struct rte_ring_headtail *d,return 0 ; *new_head = *old_head + n; -if (is_st) { - d->head = *new_head; - success =1 ; - }else -/* on failure, *old_head is updated */ -/* -* R1/A2. -* R1: Establishes a synchronizing edge with A0 of a -* different thread. -* A2: Establishes a synchronizing edge with R1 of a -* different thread to observe same value for stail -* observed by that thread on CAS failure (to retry -* with an updated *old_head). -*/ - success =rte_atomic_compare_exchange_strong_explicit ( +/* on failure, *old_head is updated */ +/* +* R1/A2. +* R1: Establishes a synchronizing edge with A0 of a +* different thread. +* A2: Establishes a synchronizing edge with R1 of a +* different thread to observe same value for stail +* observed by that thread on CAS failure (to retry +* with an updated *old_head). +*/ + success =rte_atomic_compare_exchange_strong_explicit ( &d->head, old_head, *new_head, rte_memory_order_release, rte_memory_order_acquire); - }while (unlikely (success ==0 )); + }while (unlikely (!success));return n; }
lib/ring/rte_ring_elem_pvt.h +12 -4
@@ -341,8 +341,12 @@ __rte_ring_move_prod_head(struct rte_ring *r, unsigned int is_sp, uint32_t *old_head, uint32_t *new_head, uint32_t *free_entries) { - return __rte_ring_headtail_move_head(&r->prod, &r->cons, r->capacity, -is_sp ,n ,behavior ,old_head ,new_head ,free_entries ); + if (is_sp) + return __rte_ring_headtail_move_head_st(&r->prod, &r->cons, r->capacity, + n, behavior, old_head, new_head, free_entries); + else + return __rte_ring_headtail_move_head_mt(&r->prod, &r->cons, r->capacity, +n ,behavior ,old_head ,new_head ,free_entries ); }/**
@@ -374,8 +378,12 @@ __rte_ring_move_cons_head(struct rte_ring *r, unsigned int is_sc, uint32_t *old_head, uint32_t *new_head, uint32_t *entries) { - return __rte_ring_headtail_move_head(&r->cons, &r->prod, 0, -is_sc ,n ,behavior ,old_head ,new_head ,entries ); + if (is_sc) + return __rte_ring_headtail_move_head_st(&r->cons, &r->prod, 0, + n, behavior, old_head, new_head, entries); + else + return __rte_ring_headtail_move_head_mt(&r->cons, &r->prod, 0, +n ,behavior ,old_head ,new_head ,entries ); }/**
lib/ring/rte_ring_generic_pvt.h +65 -12
104 lines not on this page · raw message
lib/ring/soring.c +15 -9
@@ -135,9 +135,12 @@ __rte_soring_move_prod_head(struct rte_soring *r, uint32_t num,switch (st) {case RTE_RING_SYNC_ST : + n =__rte_ring_headtail_move_head_st (&r->prod.ht, &r->cons.ht, + r->capacity, num, behavior, head, next, free); +break ;case RTE_RING_SYNC_MT : - n =__rte_ring_headtail_move_head (&r->prod.ht, &r->cons.ht, - r->capacity, st, num, behavior, head, next, free); + n =__rte_ring_headtail_move_head_mt (&r->prod.ht, &r->cons.ht, + r->capacity, num, behavior, head, next, free);break ;case RTE_RING_SYNC_MT_RTS : n = __rte_ring_rts_move_head(&r->prod.rts, &r->cons.ht,
@@ -168,9 +171,13 @@ __rte_soring_move_cons_head(struct rte_soring *r, uint32_t stage, uint32_t num,switch (st) {case RTE_RING_SYNC_ST : + n =__rte_ring_headtail_move_head_st (&r->cons.ht, + &r->stage[stage].ht,0 , num, behavior, + head, next, avail); +break ;case RTE_RING_SYNC_MT : - n =__rte_ring_headtail_move_head (&r->cons.ht, - &r->stage[stage].ht,0 , st, num, behavior, + n =__rte_ring_headtail_move_head_mt (&r->cons.ht, + &r->stage[stage].ht,0 , num, behavior, head, next, avail);break ;case RTE_RING_SYNC_MT_RTS :
@@ -309,9 +316,8 @@ soring_enqueue_start(struct rte_soring *r, uint32_t num,switch (st) {case RTE_RING_SYNC_ST : - n =__rte_ring_headtail_move_head (&r->prod.ht, &r->cons.ht, - r->capacity,RTE_RING_SYNC_ST , num, behavior, - &head, &next, &free); + n =__rte_ring_headtail_move_head_st (&r->prod.ht, &r->cons.ht, + r->capacity, num, behavior, &head, &next, &free);break ;case RTE_RING_SYNC_MT_HTS : n = __rte_ring_hts_move_head(&r->prod.hts, &r->cons.ht,
@@ -419,8 +425,8 @@ soring_dequeue_start(struct rte_soring *r, void *objs, void *meta,switch (st) {case RTE_RING_SYNC_ST : - n =__rte_ring_headtail_move_head (&r->cons.ht, &r->stage[ns].ht, -0 ,RTE_RING_SYNC_ST , num, behavior, &head, &next, + n =__rte_ring_headtail_move_head_st (&r->cons.ht, &r->stage[ns].ht, +0 , num, behavior, &head, &next, &avail);break ;case RTE_RING_SYNC_MT_HTS :
Signature · 1 line
2.53.0
RE: [PATCH 1/5] ring: split single thread vs multi-thread cases
Quoted 381 lines
The move head function has optimization for updating when being used on single threaded ring. Code is cleaner if the two cases are split into separate functions.
Signed-off-by Stephen Hemminger <[email protected]>--- lib/ring/rte_ring_c11_pvt.h | 100 +++++++++++++++++++++++++------- lib/ring/rte_ring_elem_pvt.h | 16 +++-- lib/ring/rte_ring_generic_pvt.h | 77 ++++++++++++++++++++---- lib/ring/soring.c | 24 +++++--- 4 files changed, 171 insertions(+), 46 deletions(-)lib/ring/rte_ring_c11_pvt.h +1
-0@@ -46,6 +46,7 @@ __rte_ring_update_tail(struct rte_ring_headtail *ht, uint32_t old_val,/** * @internal This is a helper function that moves the producer/consumer head +* optimized for single threaded case * * @param d * A pointer to the headtail structure with head value to be moved
lib/ring/rte_ring_c11_pvt.h +0
-2@@ -54,8 +55,6 @@ __rte_ring_update_tail(struct rte_ring_headtail *ht,uint32_t old_val, * function only reads tail value from it * @param capacity * Either ring capacity value (for producer), or zero (for consumer) - * @param is_st - * Indicates whether multi-thread safe path is needed or not * @param n * The number of elements we want to move head value on* @param behavior
lib/ring/rte_ring_c11_pvt.h +26
-3@@ -72,14 +71,77 @@ __rte_ring_update_tail(struct rte_ring_headtail *ht,uint32_t old_val, * If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only */ static __rte_always_inline unsigned int -__rte_ring_headtail_move_head (struct rte_ring_headtail *d, +__rte_ring_headtail_move_head_st (struct rte_ring_headtail *d,const struct rte_ring_headtail *s,uint32_t capacity, -unsigned int is_st,unsigned int n, +unsigned int n,enum rte_ring_queue_behavior behavior,uint32_t *old_head,uint32_t *new_head,uint32_t *entries) {uint32_t stail; -int success; + +/* Single producer: only this thread writes d->head, +* so a relaxed load is sufficient. +*/ + *old_head =rte_atomic_load_explicit (&d->head, rte_memory_order_relaxed); + +/* Acquire pairs with the consumer's release-store of tail in __rte_ring_update_tail, +* ensuring the consumer's ring-element reads are complete before +* we observe the updated tail. +*/ + stail =rte_atomic_load_explicit (&s->tail, rte_memory_order_acquire); + +/* Unsigned subtraction is modulo 2^32, so entries is always in +* [0, capacity) even if old_head > stail. +*/ + *entries = capacity + stail - *old_head; + +/* check that we have enough room in ring */ +if (unlikely (n > *entries)) + n = (behavior ==RTE_RING_QUEUE_FIXED ) ?0 : *entries; + +if (n >0 ) { + *new_head = *old_head + n; + rte_atomic_store_explicit(&d->head, *new_head,
rte_memory_order_relaxed);lib/ring/rte_ring_c11_pvt.h +40
-15+ } + +return n; +} + +/** +* @internal This is a helper function that moves the producer/consumer head +* for use in multi-thread safe path +* +* @param d +* A pointer to the headtail structure with head value to be moved +* @param s +* A pointer to the counter-part headtail structure. Note that this +* function only reads tail value from it +* @param capacity +* Either ring capacity value (for producer), or zero (for consumer) +* @param n +* The number of elements we want to move head value on +* @param behavior +* RTE_RING_QUEUE_FIXED: Move on a fixed number of items +* RTE_RING_QUEUE_VARIABLE: Move on as many items as possible +* @param old_head +* Returns head value as it was before the move +* @param new_head +* Returns the new head value +* @param entries +* Returns the number of ring entries available BEFORE head was moved +* @return +* Actual number of objects the head was moved on +* If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only +*/ +static __rte_always_inline unsigned int +__rte_ring_headtail_move_head_mt (struct rte_ring_headtail *d, +const struct rte_ring_headtail *s,uint32_t capacity, +unsigned int n, +enum rte_ring_queue_behavior behavior, +uint32_t *old_head,uint32_t *new_head,uint32_t *entries) +{ +uint32_t stail; +bool success;unsigned int max = n;/* @@ -120,25 +182,21 @@ __rte_ring_headtail_move_head(structrte_ring_headtail *d, return 0; *new_head = *old_head + n; -if (is_st) { - d->head = *new_head; - success =1 ; - }else -/* on failure, *old_head is updated */ -/* -* R1/A2. -* R1: Establishes a synchronizing edge with A0 of a -* different thread. -* A2: Establishes a synchronizing edge with R1 of a -* different thread to observe same value for stail -* observed by that thread on CAS failure (to retry -* with an updated *old_head). -*/ - success =rte_atomic_compare_exchange_strong_explicit(
lib/ring/rte_ring_c11_pvt.h +12
-1+/* on failure, *old_head is updated */ +/* +* R1/A2. +* R1: Establishes a synchronizing edge with A0 of a +* different thread. +* A2: Establishes a synchronizing edge with R1 of a +* different thread to observe same value for stail +* observed by that thread on CAS failure (to retry +* with an updated *old_head). +*/ + success =rte_atomic_compare_exchange_strong_explicit ( &d->head, old_head, *new_head, rte_memory_order_release, rte_memory_order_acquire); - }while (unlikely (success ==0 )); + }while (unlikely (!success));return n; }lib/ring/rte_ring_elem_pvt.h +6
-2@@ -341,8 +341,12 @@ __rte_ring_move_prod_head(struct rte_ring *r,unsigned int is_sp, uint32_t *old_head, uint32_t *new_head, uint32_t *free_entries) { - return __rte_ring_headtail_move_head(&r->prod, &r->cons, r->capacity, -is_sp ,n ,behavior ,old_head ,new_head ,free_entries ); + if (is_sp) + return __rte_ring_headtail_move_head_st(&r->prod, &r->cons, r->capacity, + n, behavior, old_head, new_head, free_entries); + else + return __rte_ring_headtail_move_head_mt(&r->prod, &r->cons, r->capacity, +n ,behavior ,old_head ,new_head ,free_entries );}
/**
lib/ring/rte_ring_elem_pvt.h +6
-2@@ -374,8 +378,12 @@ __rte_ring_move_cons_head(struct rte_ring *r,unsigned int is_sc, uint32_t *old_head, uint32_t *new_head, uint32_t *entries) { - return __rte_ring_headtail_move_head(&r->cons, &r->prod, 0, -is_sc ,n ,behavior ,old_head ,new_head ,entries ); + if (is_sc) + return __rte_ring_headtail_move_head_st(&r->cons, &r->prod, 0, + n, behavior, old_head, new_head, entries); + else + return __rte_ring_headtail_move_head_mt(&r->cons, &r->prod, 0, +n ,behavior ,old_head ,new_head ,entries );}
/**
lib/ring/rte_ring_generic_pvt.h +1
-0@@ -42,6 +42,7 @@ __rte_ring_update_tail(struct rte_ring_headtail *ht, uint32_t old_val,/** * @internal This is a helper function that moves the producer/consumer head +* for use in multi-thread safe path * * @param d * A pointer to the headtail structure with head value to be moved
lib/ring/rte_ring_generic_pvt.h +0
-2@@ -50,8 +51,6 @@ __rte_ring_update_tail(struct rte_ring_headtail *ht,uint32_t old_val, * function only reads tail value from it * @param capacity * Either ring capacity value (for producer), or zero (for consumer) - * @param is_st - * Indicates whether multi-thread safe path is needed or not * @param n * The number of elements we want to move head value on* @param behavior
lib/ring/rte_ring_generic_pvt.h +2
-3@@ -68,10 +67,9 @@ __rte_ring_update_tail(struct rte_ring_headtail *ht,uint32_t old_val, * If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only */ static __rte_always_inline unsigned int -__rte_ring_headtail_move_head (struct rte_ring_headtail *d, +__rte_ring_headtail_move_head_mt (struct rte_ring_headtail *d,const struct rte_ring_headtail *s,uint32_t capacity, -unsigned int is_st,unsigned int n, -enum rte_ring_queue_behavior behavior, +unsigned int n,enum rte_ring_queue_behavior behavior,uint32_t *old_head,uint32_t *new_head,uint32_t *entries) {lib/ring/rte_ring_generic_pvt.h +62
-7unsigned int max = n;@@ -105,15 +103,70 @@ __rte_ring_headtail_move_head(structrte_ring_headtail *d, return 0; *new_head = *old_head + n; -if (is_st) { - d->head = *new_head; - success =1 ; - }else - success =rte_atomic32_cmpset ( - (uint32_t *)(uintptr_t )&d->head, - *old_head, *new_head); + success =rte_atomic32_cmpset ( + (uint32_t *)(uintptr_t )&d->head, + *old_head, *new_head); }while (unlikely (success ==0 ));return n; } +/** +* @internal This is a helper function that moves the producer/consumer head +* optimized for single threaded case +* +* @param d +* A pointer to the headtail structure with head value to be moved +* @param s +* A pointer to the counter-part headtail structure. Note that this +* function only reads tail value from it +* @param capacity +* Either ring capacity value (for producer), or zero (for consumer) +* @param n +* The number of elements we want to move head value on +* @param behavior +* RTE_RING_QUEUE_FIXED: Move on a fixed number of items +* RTE_RING_QUEUE_VARIABLE: Move on as many items as possible +* @param old_head +* Returns head value as it was before the move +* @param new_head +* Returns the new head value +* @param entries +* Returns the number of ring entries available BEFORE head was moved +* @return +* Actual number of objects the head was moved on +* If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only +*/ +static __rte_always_inline unsigned int +__rte_ring_headtail_move_head_st (struct rte_ring_headtail *d, +const struct rte_ring_headtail *s,uint32_t capacity, +unsigned int n, +enum rte_ring_queue_behavior behavior, +uint32_t *old_head,uint32_t *new_head,uint32_t *entries) +{ + *old_head = d->head; + +/* add rmb barrier to avoid load/load reorder in weak +* memory model. It is noop on x86 +*/ +rte_smp_rmb (); + +/* +* The subtraction is done between two unsigned 32bits value +* (the result is always modulo 32 bits even if we have +* *old_head > s->tail). So 'entries' is always between 0 +* and capacity (which is < size). +*/ + *entries = (capacity + s->tail - *old_head); + +/* check that we have enough room in ring */ +if (unlikely (n > *entries)) + n = (behavior ==RTE_RING_QUEUE_FIXED ) ?0 : *entries; + +if (likely (n >0 )) { + *new_head = *old_head + n; + d->head = *new_head; + } +return n; +} +#endif /* _RTE_RING_GENERIC_PVT_H_ */
lib/ring/soring.c +5
-2@@ -135,9 +135,12 @@ __rte_soring_move_prod_head(struct rte_soring *r, uint32_t num, switch (st) {case RTE_RING_SYNC_ST : + n =__rte_ring_headtail_move_head_st (&r->prod.ht, &r- >cons.ht, + r->capacity, num, behavior, head, next, free); +break ;case RTE_RING_SYNC_MT : - n =__rte_ring_headtail_move_head (&r->prod.ht, &r->cons.ht, - r->capacity, st, num, behavior, head, next, free); + n =__rte_ring_headtail_move_head_mt (&r->prod.ht, &r- >cons.ht, + r->capacity, num, behavior, head, next, free);lib/ring/soring.c +6
-2break ;case RTE_RING_SYNC_MT_RTS : n = __rte_ring_rts_move_head(&r->prod.rts, &r->cons.ht,@@ -168,9 +171,13 @@ __rte_soring_move_cons_head(struct rte_soring *r, uint32_t stage, uint32_t num, switch (st) {case RTE_RING_SYNC_ST : + n =__rte_ring_headtail_move_head_st (&r->cons.ht, + &r->stage[stage].ht,0 , num, behavior, + head, next, avail); +break ;case RTE_RING_SYNC_MT : - n =__rte_ring_headtail_move_head (&r->cons.ht, - &r->stage[stage].ht,0 , st, num, behavior, + n =__rte_ring_headtail_move_head_mt (&r->cons.ht, + &r->stage[stage].ht,0 , num, behavior, head, next, avail);break ;lib/ring/soring.c +2
-3case RTE_RING_SYNC_MT_RTS :@@ -309,9 +316,8 @@ soring_enqueue_start(struct rte_soring *r, uint32_t num,switch (st) {case RTE_RING_SYNC_ST : - n =__rte_ring_headtail_move_head (&r->prod.ht, &r->cons.ht, - r->capacity,RTE_RING_SYNC_ST , num, behavior, - &head, &next, &free); + n =__rte_ring_headtail_move_head_st (&r->prod.ht, &r- >cons.ht, + r->capacity, num, behavior, &head, &next, &free);break ;case RTE_RING_SYNC_MT_HTS :lib/ring/soring.c +2
-2n = __rte_ring_hts_move_head(&r->prod.hts, &r->cons.ht,@@ -419,8 +425,8 @@ soring_dequeue_start(struct rte_soring *r, void *objs, void *meta, switch (st) {case RTE_RING_SYNC_ST : - n = __rte_ring_headtail_move_head(&r->cons.ht, &r- >stage[ns].ht, - 0, RTE_RING_SYNC_ST, num, behavior, &head, &next, + n = __rte_ring_headtail_move_head_st(&r->cons.ht, &r- >stage[ns].ht, + 0, num, behavior, &head, &next,
lib/ring/soring.c soring_dequeue_start(struct rte_soring *r, void *objs, void *meta,
Quoted 4 lines
lib/ring/soring.c +0
-1&avail);break ;case RTE_RING_SYNC_MT_HTS : --
Quoted 1 line
2.53.0
PATCH 2/5 ring: use GCC builtin as alternative to rte_atomic32
In reply to Stephen HemmingerThis patch replaces use of the deprecated rte_atomic32 code with GCC builtin atomic operations.
Although it would be preferable to use C11 version on all architectures, there is a performance loss if we do it that way:
Measured on i9-13900H, two physical cores MP/MC bulk n=128, 10 runs:
with C11 builtin: 5.86 cycles/elem
with __sync builtin: 5.36 cycles/elem (-9.4%)
The C11 __atomic_compare_exchange_n builtin writes the actual value back to its expected pointer on failure. On x86 this forces GCC to emit extra instructions on the critical path between the CAS and the success-test.
__sync_bool_compare_and_swap returns a plain bool with no pointer writeback, allowing GCC to emit tighter code.
---
lib/ring/meson.build | 2 +-
lib/ring/rte_ring_c11_pvt.h | 3 +-
lib/ring/rte_ring_elem_pvt.h | 2 +-
..._ring_generic_pvt.h => rte_ring_gcc_pvt.h} | 37 +++++++++++--------
4 files changed, 24 insertions(+), 20 deletions(-)
rename lib/ring/{rte_ring_generic_pvt.h => rte_ring_gcc_pvt.h} (87%)lib/ring/meson.build +1 -1
@@ -9,7 +9,7 @@ indirect_headers += files ( 'rte_ring_elem.h', 'rte_ring_elem_pvt.h', 'rte_ring_c11_pvt.h', - 'rte_ring_generic_pvt.h', + 'rte_ring_gcc_pvt.h', 'rte_ring_hts.h', 'rte_ring_hts_elem_pvt.h', 'rte_ring_peek.h',
lib/ring/rte_ring_c11_pvt.h +1 -2
@@ -43,7 +43,6 @@ __rte_ring_update_tail(struct rte_ring_headtail *ht, uint32_t old_val,*/ rte_atomic_store_explicit (&ht->tail, new_val, rte_memory_order_release); } -/** * @internal This is a helper function that moves the producer/consumer head * optimized for single threaded case
@@ -82,7 +81,7 @@ __rte_ring_headtail_move_head_st(struct rte_ring_headtail *d,/* Single producer: only this thread writes d->head, * so a relaxed load is sufficient. */ - *old_head =rte_atomic_load_explicit (&d->head, rte_memory_order_relaxed); + *old_head =rte_atomic_load_explicit (&d->head, rte_memory_order_acquire);/* Acquire pairs with the consumer's release-store of tail in __rte_ring_update_tail, * ensuring the consumer's ring-element reads are complete before
lib/ring/rte_ring_elem_pvt.h +1 -1
@@ -309,7 +309,7 @@ __rte_ring_dequeue_elems(struct rte_ring *r, uint32_t cons_head,#ifdef RTE_USE_C11_MEM_MODEL #include "rte_ring_c11_pvt.h" #else -#include "rte_ring_generic_pvt.h" +#include "rte_ring_gcc_pvt.h" #endif /**
lib/ring/rte_ring_gcc_pvt.h renamed from lib/ring/rte_ring_generic_pvt.h +21 -16
@@ -7,11 +7,11 @@* Used as BSD-3 Licensed with permission from Kip Macy. */ -#ifndef _RTE_RING_GENERIC_PVT_H_ -#define _RTE_RING_GENERIC_PVT_H_ +#ifndef _RTE_RING_GCC_PVT_H_ +#define _RTE_RING_GCC_PVT_H_/** -* @file rte_ring_generic_pvt.h +* @file rte_ring_gcc_pvt.h * It is not recommended to include this file directly, * include <rte_ring.h> instead. * Contains internal helper functions for MP/SP and MC/SC ring modes.
@@ -25,10 +25,8 @@ static __rte_always_inline void __rte_ring_update_tail(struct rte_ring_headtail *ht, uint32_t old_val, uint32_t new_val, uint32_t single, uint32_t enqueue) { - if (enqueue) - rte_smp_wmb(); - else -rte_smp_rmb (); +RTE_SET_USED (enqueue); +/* * If there are other enqueues/dequeues in progress that preceded us, * we need to wait for them to complete
@@ -37,7 +35,12 @@ __rte_ring_update_tail(struct rte_ring_headtail *ht, uint32_t old_val,rte_wait_until_equal_32 ((volatile uint32_t *)(uintptr_t )&ht->tail, old_val, rte_memory_order_relaxed); - ht->tail = new_val; +/* +* R0: Establishes a synchronizing edge with load-acquire of tail at A1. +* Ensures that memory effects by this thread on ring elements array +* is observed by a different thread of the other type. +*/ +__atomic_store_n (&ht->tail, new_val, __ATOMIC_RELEASE); }/**
@@ -72,8 +75,8 @@ __rte_ring_headtail_move_head_mt(struct rte_ring_headtail *d, unsigned int n, enum rte_ring_queue_behavior behavior, uint32_t *old_head, uint32_t *new_head, uint32_t *entries) { + bool success;unsigned int max = n; -int success; do { /* Reset n to the initial burst count */
@@ -81,10 +84,10 @@ __rte_ring_headtail_move_head_mt(struct rte_ring_headtail *d, *old_head = d->head; -/* add rmb barrier to avoid load/load reorder in weak +/* add fence to avoid load/load reorder in weak * memory model. It is noop on x86 */ -rte_smp_rmb (); +__atomic_thread_fence (__ATOMIC_ACQUIRE);/* * The subtraction is done between two unsigned 32bits value
@@ -92,7 +95,7 @@ __rte_ring_headtail_move_head_mt(struct rte_ring_headtail *d,* *old_head > s->tail). So 'entries' is always between 0 * and capacity (which is < size). */ - *entries = (capacity + s->tail - *old_head); + *entries = capacity + s->tail - *old_head;/* check that we have enough room in ring */ if (unlikely (n > *entries))
@@ -100,13 +103,15 @@ __rte_ring_headtail_move_head_mt(struct rte_ring_headtail *d, 0 : *entries;if (n ==0 ) -return 0 ; +break ; *new_head = *old_head + n; - success =rte_atomic32_cmpset ( + + success =__sync_bool_compare_and_swap ( (uint32_t *)(uintptr_t )&d->head, *old_head, *new_head); - }while (unlikely (success ==0 )); + }while (unlikely (!success)); +return n; }
@@ -169,4 +174,4 @@ __rte_ring_headtail_move_head_st(struct rte_ring_headtail *d, return n; } -#endif /* _RTE_RING_GENERIC_PVT_H_ */ +#endif /* _RTE_RING_GCC_PVT_H_ */
Signature · 1 line
2.53.0
RE: [PATCH 2/5] ring: use GCC builtin as alternative to rte_atomic32
Quoted 57 lines
This patch replaces use of the deprecated
rte_atomic32code with GCC builtin atomic operations.Although it would be preferable to use C11 version on all architectures, there is a performance loss if we do it that way:
Measured on i9-13900H, two physical cores MP/MC bulk n=128, 10 runs:
with C11 builtin: 5.86 cycles/elem
with__syncbuiltin: 5.36 cycles/elem (-9.4%)The C11
__atomic_compare_exchange_nbuiltin writes the actual value back to its expected pointer on failure. On x86 this forces GCC to emit extra instructions on the critical path between the CAS and the success-test.
__sync_bool_compare_and_swapreturns a plain bool with no pointer writeback, allowing GCC to emit tighter code.Signed-off-by Stephen Hemminger <[email protected]>--- lib/ring/meson.build | 2 +- lib/ring/rte_ring_c11_pvt.h | 3 +- lib/ring/rte_ring_elem_pvt.h | 2 +- ..._ring_generic_pvt.h => rte_ring_gcc_pvt.h} | 37 +++++++++++-------- 4 files changed, 24 insertions(+), 20 deletions(-) rename lib/ring/{rte_ring_generic_pvt.h => rte_ring_gcc_pvt.h} (87%)lib/ring/meson.build +1
-1@@ -9,7 +9,7 @@ indirect_headers += files ( 'rte_ring_elem.h', 'rte_ring_elem_pvt.h', 'rte_ring_c11_pvt.h', - 'rte_ring_generic_pvt.h', + 'rte_ring_gcc_pvt.h', 'rte_ring_hts.h', 'rte_ring_hts_elem_pvt.h', 'rte_ring_peek.h',lib/ring/rte_ring_c11_pvt.h +0
-1@@ -43,7 +43,6 @@ __rte_ring_update_tail(struct rte_ring_headtail *ht,uint32_t old_val, */ rte_atomic_store_explicit (&ht->tail, new_val, rte_memory_order_release); } -/** * @internal This is a helper function that moves the producer/consumer head
* optimized for single threaded case
lib/ring/rte_ring_c11_pvt.h __rte_ring_headtail_move_head_st(struct rte_ring_headtail · line 87
Quoted 8 lines
lib/ring/rte_ring_c11_pvt.h +1
-1@@ -82,7 +81,7 @@ __rte_ring_headtail_move_head_st(struct rte_ring_headtail *d,/* Single producer: only this thread writes d->head, * so a relaxed load is sufficient. */ - *old_head =rte_atomic_load_explicit (&d->head, rte_memory_order_relaxed); + *old_head =rte_atomic_load_explicit (&d->head, rte_memory_order_acquire);
Not sure, why it had changed to 'acquire' here?
Looks like just patch splitting mistake, no?
Quoted 4 lines
/* Acquire pairs with the consumer's release-store of tail in
__rte_ring_update_tail,
* ensuring the consumer's ring-element reads are complete before
Re: [PATCH 2/5] ring: use GCC builtin as alternative to rte_atomic32
Quoted 7 lines · On Thu, 4 Jun 2026 15:11:25 +0000 Konstantin Ananyev…
Quoted 7 lines
/**
* @internal This is a helper function that moves the producer/consumer head
* optimized for single threaded caselib/ring/rte_ring_c11_pvt.h +0
-0@@ -82,7 +81,7 @@ __rte_ring_headtail_move_head_st(struct rte_ring_headtail *d,/* Single producer: only this thread writes d->head, * so a relaxed load is sufficient.
Quoted 8 lines
Review __rte_ring_headtail_move_head_st(struct rte_ring_headtail · line 87
Quoted 5 lines
(quoted diff) +1
-1*/ - *old_head = rte_atomic_load_explicit(&d->head, rte_memory_order_relaxed); + *old_head = rte_atomic_load_explicit(&d->head, rte_memory_order_acquire);Not sure, why it had changed to 'acquire' here?
Looks like just patch splitting mistake, no?
I should have kept it as relaxed for the first load.
RE: [PATCH 2/5] ring: use GCC builtin as alternative to rte_atomic32
Quoted 22 lines
Quoted 2 lines · On Thu, 4 Jun 2026 15:11:25 +0000 Konstantin Ananyev…
Quoted 2 lines
/**
* @internal This is a helper function that moves the producer/consumerhead
Quoted 2 lines
Review __rte_ring_headtail_move_head_st(struct
Quoted 2 lines
* optimized for single threaded case
(quoted diff) +0
-0@@ -82,7 +81,7 @@ __rte_ring_headtail_move_head_st(struct
rte_ring_headtailQuoted 11 lines
Review __rte_ring_headtail_move_head_st(struct · line 87
Quoted 8 lines
(quoted diff) +1
-1*d, /* Single producer: only this thread writes d->head, * so a relaxed load is sufficient. */ - *old_head = rte_atomic_load_explicit(&d->head, rte_memory_order_relaxed); + *old_head = rte_atomic_load_explicit(&d->head, rte_memory_order_acquire);Not sure, why it had changed to 'acquire' here?
Looks like just patch splitting mistake, no?
Quoted 1 line
I should have kept it as relaxed for the first load.
Yes, I believe so.
In fact, I reverted it back to 'relaxed' in final version (after applying all 5 patches) and run both stress_ring_autotest and stress_soring_autotest on ARM box in our lab. All passed.
Konstantin
PATCH 3/5 ring: use C11 for update_tail
In reply to Stephen HemmingerThe GCC builtin atomic special case is not needed for updating tail. The performance is the same with C11 memory model.
--- lib/ring/rte_ring_c11_pvt.h | 24 ------------------------ lib/ring/rte_ring_elem_pvt.h | 22 ++++++++++++++++++++++ lib/ring/rte_ring_gcc_pvt.h | 25 ------------------------- 3 files changed, 22 insertions(+), 49 deletions(-)
lib/ring/rte_ring_c11_pvt.h +0 -24
@@ -19,30 +19,6 @@* For more information please refer to <rte_ring.h>. */ -/** -* @internal This function updates tail values. -*/ -static __rte_always_inline void -__rte_ring_update_tail (struct rte_ring_headtail *ht,uint32_t old_val, -uint32_t new_val,uint32_t single,uint32_t enqueue) -{ -RTE_SET_USED (enqueue); - -/* -* If there are other enqueues/dequeues in progress that preceded us, -* we need to wait for them to complete -*/ -if (!single) -rte_wait_until_equal_32 ((uint32_t *)(uintptr_t )&ht->tail, old_val, - rte_memory_order_relaxed); - -/* -* R0: Establishes a synchronizing edge with load-acquire of tail at A1. -* Ensures that memory effects by this thread on ring elements array -* is observed by a different thread of the other type. -*/ -rte_atomic_store_explicit (&ht->tail, new_val, rte_memory_order_release); -}/** * @internal This is a helper function that moves the producer/consumer head * optimized for single threaded case
lib/ring/rte_ring_elem_pvt.h +22 -0
@@ -299,6 +299,28 @@ __rte_ring_dequeue_elems(struct rte_ring *r, uint32_t cons_head, cons_head & r->mask, esize, num); } +static __rte_always_inline void +__rte_ring_update_tail (struct rte_ring_headtail *ht,uint32_t old_val, +uint32_t new_val,uint32_t single,uint32_t enqueue) +{ +RTE_SET_USED (enqueue); + +/* +* If there are other enqueues/dequeues in progress that preceded us, +* we need to wait for them to complete +*/ +if (!single) +rte_wait_until_equal_32 ((uint32_t *)(uintptr_t )&ht->tail, old_val, + rte_memory_order_relaxed); + +/* +* R0: Establishes a synchronizing edge with load-acquire of tail at A1. +* Ensures that memory effects by this thread on ring elements array +* is observed by a different thread of the other type. +*/ +rte_atomic_store_explicit (&ht->tail, new_val, rte_memory_order_release); +} +/* Between load and load. there might be cpu reorder in weak model * (powerpc/arm). * There are 2 choices for the users
lib/ring/rte_ring_gcc_pvt.h +0 -25
@@ -18,31 +18,6 @@* For more information please refer to <rte_ring.h>. */ -/** -* @internal This function updates tail values. -*/ -static __rte_always_inline void -__rte_ring_update_tail (struct rte_ring_headtail *ht,uint32_t old_val, -uint32_t new_val,uint32_t single,uint32_t enqueue) -{ -RTE_SET_USED (enqueue); - -/* -* If there are other enqueues/dequeues in progress that preceded us, -* we need to wait for them to complete -*/ -if (!single) -rte_wait_until_equal_32 ((volatile uint32_t *)(uintptr_t )&ht->tail, old_val, - rte_memory_order_relaxed); - -/* -* R0: Establishes a synchronizing edge with load-acquire of tail at A1. -* Ensures that memory effects by this thread on ring elements array -* is observed by a different thread of the other type. -*/ -__atomic_store_n (&ht->tail, new_val, __ATOMIC_RELEASE); -} -/** * @internal This is a helper function that moves the producer/consumer head * for use in multi-thread safe path
Signature · 1 line
2.53.0
RE: [PATCH 3/5] ring: use C11 for update_tail
Quoted 5 lines
The GCC builtin atomic special case is not needed for updating tail. The performance is the same with C11 memory model.
Quoted 8 lines
Signed-off-by Stephen Hemminger <[email protected]>--- lib/ring/rte_ring_c11_pvt.h | 24 ------------------------ lib/ring/rte_ring_elem_pvt.h | 22 ++++++++++++++++++++++ lib/ring/rte_ring_gcc_pvt.h | 25 ------------------------- 3 files changed, 22 insertions(+), 49 deletions(-)--
Quoted 1 line
2.53.0
PATCH 4/5 ring: drop unused arg to update_tail
In reply to Stephen HemmingerThe internal functions to update tail of ring no longer use the enqueue flag argument.
--- lib/ring/rte_ring_elem_pvt.h | 8 +++----- lib/ring/rte_ring_hts_elem_pvt.h | 8 +++----- lib/ring/soring.c | 10 +++++----- 3 files changed, 11 insertions(+), 15 deletions(-)
lib/ring/rte_ring_elem_pvt.h +3 -5
@@ -301,10 +301,8 @@ __rte_ring_dequeue_elems(struct rte_ring *r, uint32_t cons_head, static __rte_always_inline void__rte_ring_update_tail (struct rte_ring_headtail *ht,uint32_t old_val, -uint32_t new_val,uint32_t single,uint32_t enqueue) +uint32_t new_val,uint32_t single) { -RTE_SET_USED (enqueue); -/* * If there are other enqueues/dequeues in progress that preceded us, * we need to wait for them to complete
@@ -448,7 +446,7 @@ __rte_ring_do_enqueue_elem(struct rte_ring *r, const void *obj_table,__rte_ring_enqueue_elems (r, prod_head, obj_table, esize, n); -__rte_ring_update_tail (&r->prod, prod_head, prod_next, is_sp,1 ); +__rte_ring_update_tail (&r->prod, prod_head, prod_next, is_sp); end:if (free_space !=NULL ) *free_space = free_entries - n;
@@ -495,7 +493,7 @@ __rte_ring_do_dequeue_elem(struct rte_ring *r, void *obj_table,__rte_ring_dequeue_elems (r, cons_head, obj_table, esize, n); -__rte_ring_update_tail (&r->cons, cons_head, cons_next, is_sc,0 ); +__rte_ring_update_tail (&r->cons, cons_head, cons_next, is_sc); end:if (available !=NULL )
lib/ring/rte_ring_hts_elem_pvt.h +3 -5
@@ -25,12 +25,10 @@*/ static __rte_always_inline void__rte_ring_hts_update_tail (struct rte_ring_hts_headtail *ht,uint32_t old_tail, -uint32_t num,uint32_t enqueue) +uint32_t num) {uint32_t tail; -RTE_SET_USED (enqueue); - tail = old_tail + num;/*
@@ -217,7 +215,7 @@ __rte_ring_do_hts_enqueue_elem(struct rte_ring *r, const void *obj_table,if (n !=0 ) {__rte_ring_enqueue_elems (r, head, obj_table, esize, n); -__rte_ring_hts_update_tail (&r->hts_prod, head, n,1 ); +__rte_ring_hts_update_tail (&r->hts_prod, head, n); }if (free_space !=NULL )
@@ -258,7 +256,7 @@ __rte_ring_do_hts_dequeue_elem(struct rte_ring *r, void *obj_table,if (n !=0 ) {__rte_ring_dequeue_elems (r, head, obj_table, esize, n); -__rte_ring_hts_update_tail (&r->hts_cons, head, n,0 ); +__rte_ring_hts_update_tail (&r->hts_cons, head, n); }if (available !=NULL )
lib/ring/soring.c +5 -5
@@ -202,21 +202,21 @@ __rte_soring_move_cons_head(struct rte_soring *r, uint32_t stage, uint32_t num, static __rte_always_inline void__rte_soring_update_tail (struct __rte_ring_headtail *rht, -enum rte_ring_sync_type st,uint32_t head,uint32_t next,uint32_t enq) +enum rte_ring_sync_type st,uint32_t head,uint32_t next) {uint32_t n;switch (st) {case RTE_RING_SYNC_ST :case RTE_RING_SYNC_MT : -__rte_ring_update_tail (&rht->ht, head, next, st, enq); +__rte_ring_update_tail (&rht->ht, head, next, st);break ;case RTE_RING_SYNC_MT_RTS :__rte_ring_rts_update_tail (&rht->rts);break ;case RTE_RING_SYNC_MT_HTS : n = next - head; -__rte_ring_hts_update_tail (&rht->hts, head, n, enq); +__rte_ring_hts_update_tail (&rht->hts, head, n);break ;default :/* unsupported mode, shouldn't be here */
@@ -295,7 +295,7 @@ soring_enqueue(struct rte_soring *r, const void *objs, &prod_head, &prod_next, &nb_free);if (n !=0 ) {__enqueue_elems (r, objs, meta, prod_head, n); -__rte_soring_update_tail (&r->prod, st, prod_head, prod_next,1 ); +__rte_soring_update_tail (&r->prod, st, prod_head, prod_next); }if (free_space !=NULL )
@@ -401,7 +401,7 @@ soring_dequeue(struct rte_soring *r, void *objs, void *meta,/* we have some elems to consume */ if (n !=0 ) {__dequeue_elems (r, objs, meta, cons_head, n); -__rte_soring_update_tail (&r->cons, st, cons_head, cons_next,0 ); +__rte_soring_update_tail (&r->cons, st, cons_head, cons_next); }if (available !=NULL )
Signature · 1 line
2.53.0
RE: [PATCH 4/5] ring: drop unused arg to update_tail
Quoted 5 lines
The internal functions to update tail of ring no longer use the enqueue flag argument.
Quoted 8 lines
Signed-off-by Stephen Hemminger <[email protected]>--- lib/ring/rte_ring_elem_pvt.h | 8 +++----- lib/ring/rte_ring_hts_elem_pvt.h | 8 +++----- lib/ring/soring.c | 10 +++++----- 3 files changed, 11 insertions(+), 15 deletions(-)--
Quoted 1 line
2.53.0
PATCH 5/5 ring: use C11 for single thread move head
In reply to Stephen HemmingerThe function to move head for single threaded case can always use the C11 code, there is no performance difference from GCC intrinsics.
This reduces the exception code to just one function.
--- lib/ring/rte_ring_c11_pvt.h | 62 ------------------------------ lib/ring/rte_ring_elem_pvt.h | 74 +++++++++++++++++++++++++++++++++--- lib/ring/rte_ring_gcc_pvt.h | 59 ---------------------------- 3 files changed, 68 insertions(+), 127 deletions(-)
lib/ring/rte_ring_c11_pvt.h +0 -62
@@ -19,68 +19,6 @@* For more information please refer to <rte_ring.h>. */ -/** -* @internal This is a helper function that moves the producer/consumer head -* optimized for single threaded case -* -* @param d -* A pointer to the headtail structure with head value to be moved -* @param s -* A pointer to the counter-part headtail structure. Note that this -* function only reads tail value from it -* @param capacity -* Either ring capacity value (for producer), or zero (for consumer) -* @param n -* The number of elements we want to move head value on -* @param behavior -* RTE_RING_QUEUE_FIXED: Move on a fixed number of items -* RTE_RING_QUEUE_VARIABLE: Move on as many items as possible -* @param old_head -* Returns head value as it was before the move -* @param new_head -* Returns the new head value -* @param entries -* Returns the number of ring entries available BEFORE head was moved -* @return -* Actual number of objects the head was moved on -* If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only -*/ -static __rte_always_inline unsigned int -__rte_ring_headtail_move_head_st (struct rte_ring_headtail *d, -const struct rte_ring_headtail *s,uint32_t capacity, -unsigned int n, -enum rte_ring_queue_behavior behavior, -uint32_t *old_head,uint32_t *new_head,uint32_t *entries) -{ -uint32_t stail; - -/* Single producer: only this thread writes d->head, -* so a relaxed load is sufficient. -*/ - *old_head =rte_atomic_load_explicit (&d->head, rte_memory_order_acquire); - -/* Acquire pairs with the consumer's release-store of tail in __rte_ring_update_tail, -* ensuring the consumer's ring-element reads are complete before -* we observe the updated tail. -*/ - stail =rte_atomic_load_explicit (&s->tail, rte_memory_order_acquire); - -/* Unsigned subtraction is modulo 2^32, so entries is always in -* [0, capacity) even if old_head > stail. -*/ - *entries = capacity + stail - *old_head; - -/* check that we have enough room in ring */ -if (unlikely (n > *entries)) - n = (behavior ==RTE_RING_QUEUE_FIXED ) ?0 : *entries; - -if (n >0 ) { - *new_head = *old_head + n; -rte_atomic_store_explicit (&d->head, *new_head, rte_memory_order_relaxed); - } - -return n; -}/** * @internal This is a helper function that moves the producer/consumer head
lib/ring/rte_ring_elem_pvt.h +68 -6
80 lines not on this page · raw message
lib/ring/rte_ring_gcc_pvt.h +0 -59
63 lines not on this page · raw message
Signature · 1 line
2.53.0
RE: [PATCH 5/5] ring: use C11 for single thread move head
Quoted 6 lines
The function to move head for single threaded case can always use the C11 code, there is no performance difference from GCC intrinsics.
Quoted 9 lines
This reduces the exception code to just one function.
Signed-off-by Stephen Hemminger <[email protected]>--- lib/ring/rte_ring_c11_pvt.h | 62 ------------------------------ lib/ring/rte_ring_elem_pvt.h | 74 +++++++++++++++++++++++++++++++++--- lib/ring/rte_ring_gcc_pvt.h | 59 ---------------------------- 3 files changed, 68 insertions(+), 127 deletions(-)
Quoted 2 lines
--
2.53.0
PATCH v2 1/1 ring: convert to C11 atomics where practical
In reply to Stephen HemmingerThis is split out from the atomic deprecation series. It converts lib/ring off rte_atomic32 and onto the C11 memory model, except where the C11 version has a noticeable performance drop on x86 with GCC.
The pre-existing C11 and GCC-builtin paths lived in separate headers with substantial duplication. After this series, only the MP head CAS (__rte_ring_headtail_move_head_mt) retains separate implementations; everything else is shared. Patch 2 documents the reason for keeping the GCC builtin on the MP head CAS.
The default RTE_USE_C11_MEM_MODEL selection per architecture is unchanged.
v2 - consolidate cleanup patches
- fix the memory order on first load in _st case. it was going back/forth across the patches
Stephen Hemminger (3):
ring: split single thread vs multi-thread cases
ring: use GCC builtin as alternative to rte_atomic32
ring: cleanup the C11 code
lib/ring/meson.build | 2 +-
lib/ring/rte_ring_c11_pvt.h | 62 +++-------
lib/ring/rte_ring_elem_pvt.h | 116 ++++++++++++++++--
..._ring_generic_pvt.h => rte_ring_gcc_pvt.h} | 58 +++------
lib/ring/rte_ring_hts_elem_pvt.h | 8 +-
lib/ring/soring.c | 34 ++---
6 files changed, 159 insertions(+), 121 deletions(-)
rename lib/ring/{rte_ring_generic_pvt.h => rte_ring_gcc_pvt.h} (65%)Signature · 1 line
2.53.0
PATCH v2 1/3 ring: split single thread vs multi-thread cases
The move head function has optimization for updating when being used on single threaded ring. Code is cleaner if the two cases are split into separate functions.
--- lib/ring/rte_ring_c11_pvt.h | 100 +++++++++++++++++++++++++------- lib/ring/rte_ring_elem_pvt.h | 16 +++-- lib/ring/rte_ring_generic_pvt.h | 77 ++++++++++++++++++++---- lib/ring/soring.c | 24 +++++--- 4 files changed, 171 insertions(+), 46 deletions(-)
lib/ring/rte_ring_c11_pvt.h +79 -21
@@ -46,6 +46,7 @@ __rte_ring_update_tail(struct rte_ring_headtail *ht, uint32_t old_val,/** * @internal This is a helper function that moves the producer/consumer head +* optimized for single threaded case * * @param d * A pointer to the headtail structure with head value to be moved
@@ -54,8 +55,6 @@ __rte_ring_update_tail(struct rte_ring_headtail *ht, uint32_t old_val,* function only reads tail value from it * @param capacity * Either ring capacity value (for producer), or zero (for consumer) -* @param is_st -* Indicates whether multi-thread safe path is needed or not * @param n * The number of elements we want to move head value on * @param behavior
@@ -72,14 +71,77 @@ __rte_ring_update_tail(struct rte_ring_headtail *ht, uint32_t old_val,* If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only */ static __rte_always_inline unsigned int -__rte_ring_headtail_move_head (struct rte_ring_headtail *d, +__rte_ring_headtail_move_head_st (struct rte_ring_headtail *d,const struct rte_ring_headtail *s,uint32_t capacity, -unsigned int is_st,unsigned int n, +unsigned int n,enum rte_ring_queue_behavior behavior,uint32_t *old_head,uint32_t *new_head,uint32_t *entries) {uint32_t stail; -int success; + +/* Single producer: only this thread writes d->head, +* so a relaxed load is sufficient. +*/ + *old_head =rte_atomic_load_explicit (&d->head, rte_memory_order_relaxed); + +/* Acquire pairs with the consumer's release-store of tail in __rte_ring_update_tail, +* ensuring the consumer's ring-element reads are complete before +* we observe the updated tail. +*/ + stail =rte_atomic_load_explicit (&s->tail, rte_memory_order_acquire); + +/* Unsigned subtraction is modulo 2^32, so entries is always in +* [0, capacity) even if old_head > stail. +*/ + *entries = capacity + stail - *old_head; + +/* check that we have enough room in ring */ +if (unlikely (n > *entries)) + n = (behavior ==RTE_RING_QUEUE_FIXED ) ?0 : *entries; + +if (n >0 ) { + *new_head = *old_head + n; +rte_atomic_store_explicit (&d->head, *new_head, rte_memory_order_relaxed); + } + +return n; +} + +/** +* @internal This is a helper function that moves the producer/consumer head +* for use in multi-thread safe path +* +* @param d +* A pointer to the headtail structure with head value to be moved +* @param s +* A pointer to the counter-part headtail structure. Note that this +* function only reads tail value from it +* @param capacity +* Either ring capacity value (for producer), or zero (for consumer) +* @param n +* The number of elements we want to move head value on +* @param behavior +* RTE_RING_QUEUE_FIXED: Move on a fixed number of items +* RTE_RING_QUEUE_VARIABLE: Move on as many items as possible +* @param old_head +* Returns head value as it was before the move +* @param new_head +* Returns the new head value +* @param entries +* Returns the number of ring entries available BEFORE head was moved +* @return +* Actual number of objects the head was moved on +* If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only +*/ +static __rte_always_inline unsigned int +__rte_ring_headtail_move_head_mt (struct rte_ring_headtail *d, +const struct rte_ring_headtail *s,uint32_t capacity, +unsigned int n, +enum rte_ring_queue_behavior behavior, +uint32_t *old_head,uint32_t *new_head,uint32_t *entries) +{ +uint32_t stail; +bool success;unsigned int max = n;/*
@@ -120,25 +182,21 @@ __rte_ring_headtail_move_head(struct rte_ring_headtail *d,return 0 ; *new_head = *old_head + n; -if (is_st) { - d->head = *new_head; - success =1 ; - }else -/* on failure, *old_head is updated */ -/* -* R1/A2. -* R1: Establishes a synchronizing edge with A0 of a -* different thread. -* A2: Establishes a synchronizing edge with R1 of a -* different thread to observe same value for stail -* observed by that thread on CAS failure (to retry -* with an updated *old_head). -*/ - success =rte_atomic_compare_exchange_strong_explicit ( +/* on failure, *old_head is updated */ +/* +* R1/A2. +* R1: Establishes a synchronizing edge with A0 of a +* different thread. +* A2: Establishes a synchronizing edge with R1 of a +* different thread to observe same value for stail +* observed by that thread on CAS failure (to retry +* with an updated *old_head). +*/ + success =rte_atomic_compare_exchange_strong_explicit ( &d->head, old_head, *new_head, rte_memory_order_release, rte_memory_order_acquire); - }while (unlikely (success ==0 )); + }while (unlikely (!success));return n; }
lib/ring/rte_ring_elem_pvt.h +12 -4
@@ -341,8 +341,12 @@ __rte_ring_move_prod_head(struct rte_ring *r, unsigned int is_sp, uint32_t *old_head, uint32_t *new_head, uint32_t *free_entries) { - return __rte_ring_headtail_move_head(&r->prod, &r->cons, r->capacity, -is_sp ,n ,behavior ,old_head ,new_head ,free_entries ); + if (is_sp) + return __rte_ring_headtail_move_head_st(&r->prod, &r->cons, r->capacity, + n, behavior, old_head, new_head, free_entries); + else + return __rte_ring_headtail_move_head_mt(&r->prod, &r->cons, r->capacity, +n ,behavior ,old_head ,new_head ,free_entries ); }/**
@@ -374,8 +378,12 @@ __rte_ring_move_cons_head(struct rte_ring *r, unsigned int is_sc, uint32_t *old_head, uint32_t *new_head, uint32_t *entries) { - return __rte_ring_headtail_move_head(&r->cons, &r->prod, 0, -is_sc ,n ,behavior ,old_head ,new_head ,entries ); + if (is_sc) + return __rte_ring_headtail_move_head_st(&r->cons, &r->prod, 0, + n, behavior, old_head, new_head, entries); + else + return __rte_ring_headtail_move_head_mt(&r->cons, &r->prod, 0, +n ,behavior ,old_head ,new_head ,entries ); }/**
lib/ring/rte_ring_generic_pvt.h +65 -12
104 lines not on this page · raw message
lib/ring/soring.c +15 -9
@@ -135,9 +135,12 @@ __rte_soring_move_prod_head(struct rte_soring *r, uint32_t num,switch (st) {case RTE_RING_SYNC_ST : + n =__rte_ring_headtail_move_head_st (&r->prod.ht, &r->cons.ht, + r->capacity, num, behavior, head, next, free); +break ;case RTE_RING_SYNC_MT : - n =__rte_ring_headtail_move_head (&r->prod.ht, &r->cons.ht, - r->capacity, st, num, behavior, head, next, free); + n =__rte_ring_headtail_move_head_mt (&r->prod.ht, &r->cons.ht, + r->capacity, num, behavior, head, next, free);break ;case RTE_RING_SYNC_MT_RTS : n = __rte_ring_rts_move_head(&r->prod.rts, &r->cons.ht,
@@ -168,9 +171,13 @@ __rte_soring_move_cons_head(struct rte_soring *r, uint32_t stage, uint32_t num,switch (st) {case RTE_RING_SYNC_ST : + n =__rte_ring_headtail_move_head_st (&r->cons.ht, + &r->stage[stage].ht,0 , num, behavior, + head, next, avail); +break ;case RTE_RING_SYNC_MT : - n =__rte_ring_headtail_move_head (&r->cons.ht, - &r->stage[stage].ht,0 , st, num, behavior, + n =__rte_ring_headtail_move_head_mt (&r->cons.ht, + &r->stage[stage].ht,0 , num, behavior, head, next, avail);break ;case RTE_RING_SYNC_MT_RTS :
@@ -309,9 +316,8 @@ soring_enqueue_start(struct rte_soring *r, uint32_t num,switch (st) {case RTE_RING_SYNC_ST : - n =__rte_ring_headtail_move_head (&r->prod.ht, &r->cons.ht, - r->capacity,RTE_RING_SYNC_ST , num, behavior, - &head, &next, &free); + n =__rte_ring_headtail_move_head_st (&r->prod.ht, &r->cons.ht, + r->capacity, num, behavior, &head, &next, &free);break ;case RTE_RING_SYNC_MT_HTS : n = __rte_ring_hts_move_head(&r->prod.hts, &r->cons.ht,
@@ -419,8 +425,8 @@ soring_dequeue_start(struct rte_soring *r, void *objs, void *meta,switch (st) {case RTE_RING_SYNC_ST : - n =__rte_ring_headtail_move_head (&r->cons.ht, &r->stage[ns].ht, -0 ,RTE_RING_SYNC_ST , num, behavior, &head, &next, + n =__rte_ring_headtail_move_head_st (&r->cons.ht, &r->stage[ns].ht, +0 , num, behavior, &head, &next, &avail);break ;case RTE_RING_SYNC_MT_HTS :
Signature · 1 line
2.53.0
PATCH v2 2/3 ring: use GCC builtin as alternative to rte_atomic32
In reply to Stephen HemmingerThis patch replaces use of the deprecated rte_atomic32 code with GCC builtin atomic operations.
Although it would be preferable to use C11 version on all architectures, there is a performance loss if we do it that way:
Measured on i9-13900H, two physical cores MP/MC bulk n=128, 10 runs:
with C11 builtin: 5.86 cycles/elem
with __sync builtin: 5.36 cycles/elem (-9.4%)
The C11 __atomic_compare_exchange_n builtin writes the actual value back to its expected pointer on failure. On x86 this forces GCC to emit extra instructions on the critical path between the CAS and the success-test.
__sync_bool_compare_and_swap returns a plain bool with no pointer writeback, allowing GCC to emit tighter code.
---
lib/ring/meson.build | 2 +-
lib/ring/rte_ring_elem_pvt.h | 2 +-
..._ring_generic_pvt.h => rte_ring_gcc_pvt.h} | 33 +++++++++++--------
3 files changed, 21 insertions(+), 16 deletions(-)
rename lib/ring/{rte_ring_generic_pvt.h => rte_ring_gcc_pvt.h} (88%)lib/ring/meson.build +1 -1
@@ -9,7 +9,7 @@ indirect_headers += files ( 'rte_ring_elem.h', 'rte_ring_elem_pvt.h', 'rte_ring_c11_pvt.h', - 'rte_ring_generic_pvt.h', + 'rte_ring_gcc_pvt.h', 'rte_ring_hts.h', 'rte_ring_hts_elem_pvt.h', 'rte_ring_peek.h',
lib/ring/rte_ring_elem_pvt.h +1 -1
@@ -309,7 +309,7 @@ __rte_ring_dequeue_elems(struct rte_ring *r, uint32_t cons_head,#ifdef RTE_USE_C11_MEM_MODEL #include "rte_ring_c11_pvt.h" #else -#include "rte_ring_generic_pvt.h" +#include "rte_ring_gcc_pvt.h" #endif /**
lib/ring/rte_ring_gcc_pvt.h renamed from lib/ring/rte_ring_generic_pvt.h +19 -14
@@ -7,11 +7,11 @@* Used as BSD-3 Licensed with permission from Kip Macy. */ -#ifndef _RTE_RING_GENERIC_PVT_H_ -#define _RTE_RING_GENERIC_PVT_H_ +#ifndef _RTE_RING_GCC_PVT_H_ +#define _RTE_RING_GCC_PVT_H_/** -* @file rte_ring_generic_pvt.h +* @file rte_ring_gcc_pvt.h * It is not recommended to include this file directly, * include <rte_ring.h> instead. * Contains internal helper functions for MP/SP and MC/SC ring modes.
@@ -25,10 +25,8 @@ static __rte_always_inline void __rte_ring_update_tail(struct rte_ring_headtail *ht, uint32_t old_val, uint32_t new_val, uint32_t single, uint32_t enqueue) { - if (enqueue) - rte_smp_wmb(); - else -rte_smp_rmb (); +RTE_SET_USED (enqueue); +/* * If there are other enqueues/dequeues in progress that preceded us, * we need to wait for them to complete
@@ -37,7 +35,12 @@ __rte_ring_update_tail(struct rte_ring_headtail *ht, uint32_t old_val,rte_wait_until_equal_32 ((volatile uint32_t *)(uintptr_t )&ht->tail, old_val, rte_memory_order_relaxed); - ht->tail = new_val; +/* +* R0: Establishes a synchronizing edge with load-acquire of tail at A1. +* Ensures that memory effects by this thread on ring elements array +* is observed by a different thread of the other type. +*/ +__atomic_store_n (&ht->tail, new_val, __ATOMIC_RELEASE); }/**
@@ -73,7 +76,7 @@ __rte_ring_headtail_move_head_mt(struct rte_ring_headtail *d, uint32_t *old_head, uint32_t *new_head, uint32_t *entries) { unsigned int max = n; -int success; +bool success; do { /* Reset n to the initial burst count */
@@ -81,10 +84,10 @@ __rte_ring_headtail_move_head_mt(struct rte_ring_headtail *d, *old_head = d->head; -/* add rmb barrier to avoid load/load reorder in weak +/* add fence to avoid load/load reorder in weak * memory model. It is noop on x86 */ -rte_smp_rmb (); +__atomic_thread_fence (__ATOMIC_ACQUIRE);/* * The subtraction is done between two unsigned 32bits value
@@ -103,10 +106,12 @@ __rte_ring_headtail_move_head_mt(struct rte_ring_headtail *d,return 0 ; *new_head = *old_head + n; - success =rte_atomic32_cmpset ( + + success =__sync_bool_compare_and_swap ( (uint32_t *)(uintptr_t )&d->head, *old_head, *new_head); - }while (unlikely (success ==0 )); + }while (unlikely (!success)); +return n; }
@@ -169,4 +174,4 @@ __rte_ring_headtail_move_head_st(struct rte_ring_headtail *d, return n; } -#endif /* _RTE_RING_GENERIC_PVT_H_ */ +#endif /* _RTE_RING_GCC_PVT_H_ */
Signature · 1 line
2.53.0
RE: [PATCH v2 2/3] ring: use GCC builtin as alternative to rte_atomic32
Quoted 146 lines
This patch replaces use of the deprecated
rte_atomic32code with GCC builtin atomic operations.Although it would be preferable to use C11 version on all architectures, there is a performance loss if we do it that way:
Measured on i9-13900H, two physical cores MP/MC bulk n=128, 10 runs:
with C11 builtin: 5.86 cycles/elem
with__syncbuiltin: 5.36 cycles/elem (-9.4%)The C11
__atomic_compare_exchange_nbuiltin writes the actual value back to its expected pointer on failure. On x86 this forces GCC to emit extra instructions on the critical path between the CAS and the success-test.
__sync_bool_compare_and_swapreturns a plain bool with no pointer writeback, allowing GCC to emit tighter code.Signed-off-by Stephen Hemminger <[email protected]>--- lib/ring/meson.build | 2 +- lib/ring/rte_ring_elem_pvt.h | 2 +- ..._ring_generic_pvt.h => rte_ring_gcc_pvt.h} | 33 +++++++++++-------- 3 files changed, 21 insertions(+), 16 deletions(-) rename lib/ring/{rte_ring_generic_pvt.h => rte_ring_gcc_pvt.h} (88%)lib/ring/meson.build +1
-1@@ -9,7 +9,7 @@ indirect_headers += files ( 'rte_ring_elem.h', 'rte_ring_elem_pvt.h', 'rte_ring_c11_pvt.h', - 'rte_ring_generic_pvt.h', + 'rte_ring_gcc_pvt.h', 'rte_ring_hts.h', 'rte_ring_hts_elem_pvt.h', 'rte_ring_peek.h',lib/ring/rte_ring_elem_pvt.h +1
-1@@ -309,7 +309,7 @@ __rte_ring_dequeue_elems(struct rte_ring *r, uint32_t cons_head,#ifdef RTE_USE_C11_MEM_MODEL #include "rte_ring_c11_pvt.h" #else -#include "rte_ring_generic_pvt.h" +#include "rte_ring_gcc_pvt.h" #endif /**
lib/ring/rte_ring_gcc_pvt.h renamed from lib/ring/rte_ring_generic_pvt.h +11
-8@@ -7,11 +7,11 @@* Used as BSD-3 Licensed with permission from Kip Macy. */ -#ifndef _RTE_RING_GENERIC_PVT_H_ -#define _RTE_RING_GENERIC_PVT_H_ +#ifndef _RTE_RING_GCC_PVT_H_ +#define _RTE_RING_GCC_PVT_H_/** -* @file rte_ring_generic_pvt.h +* @file rte_ring_gcc_pvt.h * It is not recommended to include this file directly, * include <rte_ring.h> instead. * Contains internal helper functions for MP/SP and MC/SC ring modes. @@ -25,10 +25,8 @@ static __rte_always_inline void __rte_ring_update_tail(struct rte_ring_headtail *ht, uint32_t old_val, uint32_t new_val, uint32_t single, uint32_t enqueue) { - if (enqueue) - rte_smp_wmb(); - else -rte_smp_rmb (); +RTE_SET_USED (enqueue); +/* * If there are other enqueues/dequeues in progress that preceded us, * we need to wait for them to complete @@ -37,7 +35,12 @@ __rte_ring_update_tail(struct rte_ring_headtail *ht,uint32_t old_val, rte_wait_until_equal_32((volatile uint32_t *)(uintptr_t)&ht->tail, old_val, rte_memory_order_relaxed); - ht->tail = new_val; +/* +* R0: Establishes a synchronizing edge with load-acquire of tail at A1. +* Ensures that memory effects by this thread on ring elements array +* is observed by a different thread of the other type. +*/ +__atomic_store_n (&ht->tail, new_val, __ATOMIC_RELEASE); }/**
lib/ring/rte_ring_gcc_pvt.h renamed from lib/ring/rte_ring_generic_pvt.h +1
-1@@ -73,7 +76,7 @@ __rte_ring_headtail_move_head_mt(struct rte_ring_headtail *d, uint32_t *old_head, uint32_t *new_head, uint32_t *entries) {unsigned int max = n; -int success; +bool success; do {lib/ring/rte_ring_gcc_pvt.h renamed from lib/ring/rte_ring_generic_pvt.h +2
-2/* Reset n to the initial burst count */ @@ -81,10 +84,10 @@ __rte_ring_headtail_move_head_mt(structrte_ring_headtail *d, *old_head = d->head; -/* add rmb barrier to avoid load/load reorder in weak +/* add fence to avoid load/load reorder in weak * memory model. It is noop on x86 */ -rte_smp_rmb (); +__atomic_thread_fence (__ATOMIC_ACQUIRE);/* lib/ring/rte_ring_gcc_pvt.h renamed from lib/ring/rte_ring_generic_pvt.h +4
-2* The subtraction is done between two unsigned 32bits value @@ -103,10 +106,12 @@ __rte_ring_headtail_move_head_mt(structrte_ring_headtail *d, return 0; *new_head = *old_head + n; - success =rte_atomic32_cmpset ( + + success =__sync_bool_compare_and_swap ( (uint32_t *)(uintptr_t )&d->head, *old_head, *new_head); - }while (unlikely (success ==0 )); + }while (unlikely (!success)); +return n; }
lib/ring/rte_ring_gcc_pvt.h __rte_ring_headtail_move_head_st(struct
Quoted 7 lines
lib/ring/rte_ring_gcc_pvt.h renamed from lib/ring/rte_ring_generic_pvt.h +1
-2@@ -169,4 +174,4 @@ __rte_ring_headtail_move_head_st(structrte_ring_headtail *d, return n; } -#endif /* _RTE_RING_GENERIC_PVT_H_ */ +#endif /* _RTE_RING_GCC_PVT_H_ */ --
Quoted 1 line
2.53.0
Re: [PATCH v2 2/3] ring: use GCC builtin as alternative to rte_atomic32
In reply to Stephen Hemminger04/06/2026 18:32, Stephen Hemminger:
Quoted 2 lines
This patch replaces use of the deprecated
rte_atomic32code with GCC builtin atomic operations.
It compiles fine with GCC, but there is an issue with clang:
ninja: Entering directory `build-gcc-static'
ninja: no work to do.
ninja: Entering directory `build-gcc-shared'
ninja: no work to do.
ninja: Entering directory `build-clang-static'
ninja: no work to do.
ninja: Entering directory `build-clang-shared'
[1/3069] Compiling C object lib/librte_ring.a.p/ring_soring.c.o
rte_ring_gcc_pvt.h:43:2: error: address argument to atomic operation must be a pointer to integer or pointer ('volatile _Atomic(uint32_t) *' invalid)
43 | __atomic_store_n(&ht->tail, new_val, __ATOMIC_RELEASE);
| ^ ~~~~~~~~~Quoted 6 lines
Although it would be preferable to use C11 version on all architectures, there is a performance loss if we do it that way:
Measured on i9-13900H, two physical cores MP/MC bulk n=128, 10 runs:
with C11 builtin: 5.86 cycles/elem
with__syncbuiltin: 5.36 cycles/elem (-9.4%)
You don't compare with the current rte_atomic functions?
PATCH v2 3/3 ring: cleanup the C11 code
In reply to Stephen HemmingerPut the C11 code in the rte_ring_elem_pvt.h file and only have the GCC vs C11 code split in separate includes.
The internal functions to update tail of ring no longer use the enqueue flag argument.
--- lib/ring/rte_ring_c11_pvt.h | 88 ---------------------------- lib/ring/rte_ring_elem_pvt.h | 98 +++++++++++++++++++++++++++++--- lib/ring/rte_ring_gcc_pvt.h | 84 --------------------------- lib/ring/rte_ring_hts_elem_pvt.h | 8 +-- lib/ring/soring.c | 10 ++-- 5 files changed, 98 insertions(+), 190 deletions(-)
lib/ring/rte_ring_c11_pvt.h +0 -88
@@ -19,94 +19,6 @@* For more information please refer to <rte_ring.h>. */ -/** -* @internal This function updates tail values. -*/ -static __rte_always_inline void -__rte_ring_update_tail (struct rte_ring_headtail *ht,uint32_t old_val, -uint32_t new_val,uint32_t single,uint32_t enqueue) -{ -RTE_SET_USED (enqueue); - -/* -* If there are other enqueues/dequeues in progress that preceded us, -* we need to wait for them to complete -*/ -if (!single) -rte_wait_until_equal_32 ((uint32_t *)(uintptr_t )&ht->tail, old_val, - rte_memory_order_relaxed); - -/* -* R0: Establishes a synchronizing edge with load-acquire of tail at A1. -* Ensures that memory effects by this thread on ring elements array -* is observed by a different thread of the other type. -*/ -rte_atomic_store_explicit (&ht->tail, new_val, rte_memory_order_release); -} - -/** -* @internal This is a helper function that moves the producer/consumer head -* optimized for single threaded case -* -* @param d -* A pointer to the headtail structure with head value to be moved -* @param s -* A pointer to the counter-part headtail structure. Note that this -* function only reads tail value from it -* @param capacity -* Either ring capacity value (for producer), or zero (for consumer) -* @param n -* The number of elements we want to move head value on -* @param behavior -* RTE_RING_QUEUE_FIXED: Move on a fixed number of items -* RTE_RING_QUEUE_VARIABLE: Move on as many items as possible -* @param old_head -* Returns head value as it was before the move -* @param new_head -* Returns the new head value -* @param entries -* Returns the number of ring entries available BEFORE head was moved -* @return -* Actual number of objects the head was moved on -* If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only -*/ -static __rte_always_inline unsigned int -__rte_ring_headtail_move_head_st (struct rte_ring_headtail *d, -const struct rte_ring_headtail *s,uint32_t capacity, -unsigned int n, -enum rte_ring_queue_behavior behavior, -uint32_t *old_head,uint32_t *new_head,uint32_t *entries) -{ -uint32_t stail; - -/* Single producer: only this thread writes d->head, -* so a relaxed load is sufficient. -*/ - *old_head =rte_atomic_load_explicit (&d->head, rte_memory_order_relaxed); - -/* Acquire pairs with the consumer's release-store of tail in __rte_ring_update_tail, -* ensuring the consumer's ring-element reads are complete before -* we observe the updated tail. -*/ - stail =rte_atomic_load_explicit (&s->tail, rte_memory_order_acquire); - -/* Unsigned subtraction is modulo 2^32, so entries is always in -* [0, capacity) even if old_head > stail. -*/ - *entries = capacity + stail - *old_head; - -/* check that we have enough room in ring */ -if (unlikely (n > *entries)) - n = (behavior ==RTE_RING_QUEUE_FIXED ) ?0 : *entries; - -if (n >0 ) { - *new_head = *old_head + n; -rte_atomic_store_explicit (&d->head, *new_head, rte_memory_order_relaxed); - } - -return n; -} -/** * @internal This is a helper function that moves the producer/consumer head * for use in multi-thread safe path
lib/ring/rte_ring_elem_pvt.h +90 -8
116 lines not on this page · raw message
lib/ring/rte_ring_gcc_pvt.h +0 -84
94 lines not on this page · raw message
lib/ring/rte_ring_hts_elem_pvt.h +3 -5
@@ -25,12 +25,10 @@*/ static __rte_always_inline void__rte_ring_hts_update_tail (struct rte_ring_hts_headtail *ht,uint32_t old_tail, -uint32_t num,uint32_t enqueue) +uint32_t num) {uint32_t tail; -RTE_SET_USED (enqueue); - tail = old_tail + num;/*
@@ -217,7 +215,7 @@ __rte_ring_do_hts_enqueue_elem(struct rte_ring *r, const void *obj_table,if (n !=0 ) {__rte_ring_enqueue_elems (r, head, obj_table, esize, n); -__rte_ring_hts_update_tail (&r->hts_prod, head, n,1 ); +__rte_ring_hts_update_tail (&r->hts_prod, head, n); }if (free_space !=NULL )
@@ -258,7 +256,7 @@ __rte_ring_do_hts_dequeue_elem(struct rte_ring *r, void *obj_table,if (n !=0 ) {__rte_ring_dequeue_elems (r, head, obj_table, esize, n); -__rte_ring_hts_update_tail (&r->hts_cons, head, n,0 ); +__rte_ring_hts_update_tail (&r->hts_cons, head, n); }if (available !=NULL )
lib/ring/soring.c +5 -5
@@ -202,21 +202,21 @@ __rte_soring_move_cons_head(struct rte_soring *r, uint32_t stage, uint32_t num, static __rte_always_inline void__rte_soring_update_tail (struct __rte_ring_headtail *rht, -enum rte_ring_sync_type st,uint32_t head,uint32_t next,uint32_t enq) +enum rte_ring_sync_type st,uint32_t head,uint32_t next) {uint32_t n;switch (st) {case RTE_RING_SYNC_ST :case RTE_RING_SYNC_MT : -__rte_ring_update_tail (&rht->ht, head, next, st, enq); +__rte_ring_update_tail (&rht->ht, head, next, st);break ;case RTE_RING_SYNC_MT_RTS :__rte_ring_rts_update_tail (&rht->rts);break ;case RTE_RING_SYNC_MT_HTS : n = next - head; -__rte_ring_hts_update_tail (&rht->hts, head, n, enq); +__rte_ring_hts_update_tail (&rht->hts, head, n);break ;default :/* unsupported mode, shouldn't be here */
@@ -295,7 +295,7 @@ soring_enqueue(struct rte_soring *r, const void *objs, &prod_head, &prod_next, &nb_free);if (n !=0 ) {__enqueue_elems (r, objs, meta, prod_head, n); -__rte_soring_update_tail (&r->prod, st, prod_head, prod_next,1 ); +__rte_soring_update_tail (&r->prod, st, prod_head, prod_next); }if (free_space !=NULL )
@@ -401,7 +401,7 @@ soring_dequeue(struct rte_soring *r, void *objs, void *meta,/* we have some elems to consume */ if (n !=0 ) {__dequeue_elems (r, objs, meta, cons_head, n); -__rte_soring_update_tail (&r->cons, st, cons_head, cons_next,0 ); +__rte_soring_update_tail (&r->cons, st, cons_head, cons_next); }if (available !=NULL )
Signature · 1 line
2.53.0
v3 cover ring: replace use of rte_atomic
In reply to Stephen HemmingerThis is part of the broader rte_atomic32 deprecation work, sent separately because it is the most complex part and benefits from independent review.
Convert lib/ring off rte_atomic32 and onto the C11 memory model, except for the ring head compare-and-swap where special case is needed. On x86 with GCC using C11 atomics produces measurably worse code.
After this series only __rte_ring_headtail_move_head has separate C11 and GCC-builtin implementations; everything else uses the same code on all architectures. The default RTE_USE_C11_MEM_MODEL selection per architecture is unchanged.
v3:
- rebase and squash patches
- keep original code for x86 single thread case
Stephen Hemminger (2):
ring: split single thread vs multi-thread cases
ring: replace rte_atomic32 with __sync builtin
lib/ring/meson.build | 2 +- lib/ring/rte_ring_c11_pvt.h | 107 +++++++++++++-------- lib/ring/rte_ring_elem_pvt.h | 53 ++++++++--- lib/ring/rte_ring_gcc_pvt.h | 155 +++++++++++++++++++++++++++++++ lib/ring/rte_ring_generic_pvt.h | 119 ------------------------ lib/ring/rte_ring_hts_elem_pvt.h | 8 +- lib/ring/soring.c | 34 ++++--- 7 files changed, 289 insertions(+), 189 deletions(-) create mode 100644 lib/ring/rte_ring_gcc_pvt.h delete mode 100644 lib/ring/rte_ring_generic_pvt.h
Signature · 1 line
2.53.0
PATCH v3 1/2 ring: split single thread vs multi-thread cases
The move head function has optimization for updating when being used on single threaded ring. Code is cleaner if the two cases are split into separate functions.
--- lib/ring/rte_ring_c11_pvt.h | 100 +++++++++++++++++++++++++------- lib/ring/rte_ring_elem_pvt.h | 16 +++-- lib/ring/rte_ring_generic_pvt.h | 77 ++++++++++++++++++++---- lib/ring/soring.c | 24 +++++--- 4 files changed, 171 insertions(+), 46 deletions(-)
lib/ring/rte_ring_c11_pvt.h +79 -21
@@ -46,6 +46,7 @@ __rte_ring_update_tail(struct rte_ring_headtail *ht, uint32_t old_val,/** * @internal This is a helper function that moves the producer/consumer head +* optimized for single threaded case * * @param d * A pointer to the headtail structure with head value to be moved
@@ -54,8 +55,6 @@ __rte_ring_update_tail(struct rte_ring_headtail *ht, uint32_t old_val,* function only reads tail value from it * @param capacity * Either ring capacity value (for producer), or zero (for consumer) -* @param is_st -* Indicates whether multi-thread safe path is needed or not * @param n * The number of elements we want to move head value on * @param behavior
@@ -72,14 +71,77 @@ __rte_ring_update_tail(struct rte_ring_headtail *ht, uint32_t old_val,* If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only */ static __rte_always_inline unsigned int -__rte_ring_headtail_move_head (struct rte_ring_headtail *d, +__rte_ring_headtail_move_head_st (struct rte_ring_headtail *d,const struct rte_ring_headtail *s,uint32_t capacity, -unsigned int is_st,unsigned int n, +unsigned int n,enum rte_ring_queue_behavior behavior,uint32_t *old_head,uint32_t *new_head,uint32_t *entries) {uint32_t stail; -int success; + +/* Single producer: only this thread writes d->head, +* so a relaxed load is sufficient. +*/ + *old_head =rte_atomic_load_explicit (&d->head, rte_memory_order_relaxed); + +/* Acquire pairs with the consumer's release-store of tail in __rte_ring_update_tail, +* ensuring the consumer's ring-element reads are complete before +* we observe the updated tail. +*/ + stail =rte_atomic_load_explicit (&s->tail, rte_memory_order_acquire); + +/* Unsigned subtraction is modulo 2^32, so entries is always in +* [0, capacity) even if old_head > stail. +*/ + *entries = capacity + stail - *old_head; + +/* check that we have enough room in ring */ +if (unlikely (n > *entries)) + n = (behavior ==RTE_RING_QUEUE_FIXED ) ?0 : *entries; + +if (n >0 ) { + *new_head = *old_head + n; +rte_atomic_store_explicit (&d->head, *new_head, rte_memory_order_relaxed); + } + +return n; +} + +/** +* @internal This is a helper function that moves the producer/consumer head +* for use in multi-thread safe path +* +* @param d +* A pointer to the headtail structure with head value to be moved +* @param s +* A pointer to the counter-part headtail structure. Note that this +* function only reads tail value from it +* @param capacity +* Either ring capacity value (for producer), or zero (for consumer) +* @param n +* The number of elements we want to move head value on +* @param behavior +* RTE_RING_QUEUE_FIXED: Move on a fixed number of items +* RTE_RING_QUEUE_VARIABLE: Move on as many items as possible +* @param old_head +* Returns head value as it was before the move +* @param new_head +* Returns the new head value +* @param entries +* Returns the number of ring entries available BEFORE head was moved +* @return +* Actual number of objects the head was moved on +* If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only +*/ +static __rte_always_inline unsigned int +__rte_ring_headtail_move_head_mt (struct rte_ring_headtail *d, +const struct rte_ring_headtail *s,uint32_t capacity, +unsigned int n, +enum rte_ring_queue_behavior behavior, +uint32_t *old_head,uint32_t *new_head,uint32_t *entries) +{ +uint32_t stail; +bool success;unsigned int max = n;/*
@@ -120,25 +182,21 @@ __rte_ring_headtail_move_head(struct rte_ring_headtail *d,return 0 ; *new_head = *old_head + n; -if (is_st) { - d->head = *new_head; - success =1 ; - }else -/* on failure, *old_head is updated */ -/* -* R1/A2. -* R1: Establishes a synchronizing edge with A0 of a -* different thread. -* A2: Establishes a synchronizing edge with R1 of a -* different thread to observe same value for stail -* observed by that thread on CAS failure (to retry -* with an updated *old_head). -*/ - success =rte_atomic_compare_exchange_strong_explicit ( +/* on failure, *old_head is updated */ +/* +* R1/A2. +* R1: Establishes a synchronizing edge with A0 of a +* different thread. +* A2: Establishes a synchronizing edge with R1 of a +* different thread to observe same value for stail +* observed by that thread on CAS failure (to retry +* with an updated *old_head). +*/ + success =rte_atomic_compare_exchange_strong_explicit ( &d->head, old_head, *new_head, rte_memory_order_release, rte_memory_order_acquire); - }while (unlikely (success ==0 )); + }while (unlikely (!success));return n; }
lib/ring/rte_ring_elem_pvt.h +12 -4
@@ -341,8 +341,12 @@ __rte_ring_move_prod_head(struct rte_ring *r, unsigned int is_sp, uint32_t *old_head, uint32_t *new_head, uint32_t *free_entries) { - return __rte_ring_headtail_move_head(&r->prod, &r->cons, r->capacity, -is_sp ,n ,behavior ,old_head ,new_head ,free_entries ); + if (is_sp) + return __rte_ring_headtail_move_head_st(&r->prod, &r->cons, r->capacity, + n, behavior, old_head, new_head, free_entries); + else + return __rte_ring_headtail_move_head_mt(&r->prod, &r->cons, r->capacity, +n ,behavior ,old_head ,new_head ,free_entries ); }/**
@@ -374,8 +378,12 @@ __rte_ring_move_cons_head(struct rte_ring *r, unsigned int is_sc, uint32_t *old_head, uint32_t *new_head, uint32_t *entries) { - return __rte_ring_headtail_move_head(&r->cons, &r->prod, 0, -is_sc ,n ,behavior ,old_head ,new_head ,entries ); + if (is_sc) + return __rte_ring_headtail_move_head_st(&r->cons, &r->prod, 0, + n, behavior, old_head, new_head, entries); + else + return __rte_ring_headtail_move_head_mt(&r->cons, &r->prod, 0, +n ,behavior ,old_head ,new_head ,entries ); }/**
lib/ring/rte_ring_generic_pvt.h +65 -12
104 lines not on this page · raw message
lib/ring/soring.c +15 -9
@@ -135,9 +135,12 @@ __rte_soring_move_prod_head(struct rte_soring *r, uint32_t num,switch (st) {case RTE_RING_SYNC_ST : + n =__rte_ring_headtail_move_head_st (&r->prod.ht, &r->cons.ht, + r->capacity, num, behavior, head, next, free); +break ;case RTE_RING_SYNC_MT : - n =__rte_ring_headtail_move_head (&r->prod.ht, &r->cons.ht, - r->capacity, st, num, behavior, head, next, free); + n =__rte_ring_headtail_move_head_mt (&r->prod.ht, &r->cons.ht, + r->capacity, num, behavior, head, next, free);break ;case RTE_RING_SYNC_MT_RTS : n = __rte_ring_rts_move_head(&r->prod.rts, &r->cons.ht,
@@ -168,9 +171,13 @@ __rte_soring_move_cons_head(struct rte_soring *r, uint32_t stage, uint32_t num,switch (st) {case RTE_RING_SYNC_ST : + n =__rte_ring_headtail_move_head_st (&r->cons.ht, + &r->stage[stage].ht,0 , num, behavior, + head, next, avail); +break ;case RTE_RING_SYNC_MT : - n =__rte_ring_headtail_move_head (&r->cons.ht, - &r->stage[stage].ht,0 , st, num, behavior, + n =__rte_ring_headtail_move_head_mt (&r->cons.ht, + &r->stage[stage].ht,0 , num, behavior, head, next, avail);break ;case RTE_RING_SYNC_MT_RTS :
@@ -309,9 +316,8 @@ soring_enqueue_start(struct rte_soring *r, uint32_t num,switch (st) {case RTE_RING_SYNC_ST : - n =__rte_ring_headtail_move_head (&r->prod.ht, &r->cons.ht, - r->capacity,RTE_RING_SYNC_ST , num, behavior, - &head, &next, &free); + n =__rte_ring_headtail_move_head_st (&r->prod.ht, &r->cons.ht, + r->capacity, num, behavior, &head, &next, &free);break ;case RTE_RING_SYNC_MT_HTS : n = __rte_ring_hts_move_head(&r->prod.hts, &r->cons.ht,
@@ -419,8 +425,8 @@ soring_dequeue_start(struct rte_soring *r, void *objs, void *meta,switch (st) {case RTE_RING_SYNC_ST : - n =__rte_ring_headtail_move_head (&r->cons.ht, &r->stage[ns].ht, -0 ,RTE_RING_SYNC_ST , num, behavior, &head, &next, + n =__rte_ring_headtail_move_head_st (&r->cons.ht, &r->stage[ns].ht, +0 , num, behavior, &head, &next, &avail);break ;case RTE_RING_SYNC_MT_HTS :
Signature · 1 line
2.53.0
PATCH v3 2/2 ring: replace rte_atomic32 with __sync builtin
In reply to Stephen HemmingerReplaces use of the deprecated rte_atomic32 code with GCC builtin atomic operations on x86. The C11 version used on other architectures is unchanged.
Although it would be preferable to use C11 on all architectures, there is a performance loss if we do it that way.
On x86 i9-13900H, two physical cores MP/MC (cycles/elem), ring_perf test with GCC 14.2:
n asm sync c11
8 72.86 72.12 89.01 32 18.74 18.80 24.62 64 10.07 9.86 12.41 128 6.99 6.74 9.01 256 6.38 6.20 7.34
Pure C11 regresses 15-30% due to __atomic_compare_exchange_n's failure-writeback semantic.
Drop the now-unused enqueue argument.
---
lib/ring/meson.build | 2 +-
lib/ring/rte_ring_c11_pvt.h | 25 ----
lib/ring/rte_ring_elem_pvt.h | 37 +++--
..._ring_generic_pvt.h => rte_ring_gcc_pvt.h} | 141 ++++++++----------
lib/ring/rte_ring_hts_elem_pvt.h | 8 +-
lib/ring/soring.c | 10 +-
6 files changed, 99 insertions(+), 124 deletions(-)
rename lib/ring/{rte_ring_generic_pvt.h => rte_ring_gcc_pvt.h} (81%)lib/ring/meson.build +1 -1
@@ -9,7 +9,7 @@ indirect_headers += files ( 'rte_ring_elem.h', 'rte_ring_elem_pvt.h', 'rte_ring_c11_pvt.h', - 'rte_ring_generic_pvt.h', + 'rte_ring_gcc_pvt.h', 'rte_ring_hts.h', 'rte_ring_hts_elem_pvt.h', 'rte_ring_peek.h',
lib/ring/rte_ring_c11_pvt.h +0 -25
@@ -19,31 +19,6 @@* For more information please refer to <rte_ring.h>. */ -/** -* @internal This function updates tail values. -*/ -static __rte_always_inline void -__rte_ring_update_tail (struct rte_ring_headtail *ht,uint32_t old_val, -uint32_t new_val,uint32_t single,uint32_t enqueue) -{ -RTE_SET_USED (enqueue); - -/* -* If there are other enqueues/dequeues in progress that preceded us, -* we need to wait for them to complete -*/ -if (!single) -rte_wait_until_equal_32 ((uint32_t *)(uintptr_t )&ht->tail, old_val, - rte_memory_order_relaxed); - -/* -* R0: Establishes a synchronizing edge with load-acquire of tail at A1. -* Ensures that memory effects by this thread on ring elements array -* is observed by a different thread of the other type. -*/ -rte_atomic_store_explicit (&ht->tail, new_val, rte_memory_order_release); -} -/** * @internal This is a helper function that moves the producer/consumer head * optimized for single threaded case
lib/ring/rte_ring_elem_pvt.h +28 -9
@@ -299,17 +299,36 @@ __rte_ring_dequeue_elems(struct rte_ring *r, uint32_t cons_head, cons_head & r->mask, esize, num); } -/* Between load and load. there might be cpu reorder in weak model -* (powerpc/arm). -* There are 2 choices for the users -* 1.use rmb() memory barrier -* 2.use one-direction load_acquire/store_release barrier -* It depends on performance test results. +static __rte_always_inline void +__rte_ring_update_tail (struct rte_ring_headtail *ht,uint32_t old_val, +uint32_t new_val,uint32_t single) +{ +/* +* If there are other enqueues/dequeues in progress that preceded us, +* we need to wait for them to complete +*/ +if (!single) +rte_wait_until_equal_32 ((uint32_t *)(uintptr_t )&ht->tail, old_val, + rte_memory_order_relaxed); + +/* +* R0: Establishes a synchronizing edge with load-acquire of tail at A1. +* Ensures that memory effects by this thread on ring elements array +* is observed by a different thread of the other type. +*/ +rte_atomic_store_explicit (&ht->tail, new_val, rte_memory_order_release); +} + +/* +* The function __rte_ring_headtail_move_head_mt,st has two versions +* based on what is most efficient on a given architecture. +* +* The C11 is preferred but on x86 GCC has 10% performance drop. */ #ifdef RTE_USE_C11_MEM_MODEL #include "rte_ring_c11_pvt.h" #else -#include "rte_ring_generic_pvt.h" +#include "rte_ring_gcc_pvt.h" #endif /**
@@ -426,7 +445,7 @@ __rte_ring_do_enqueue_elem(struct rte_ring *r, const void *obj_table,__rte_ring_enqueue_elems (r, prod_head, obj_table, esize, n); -__rte_ring_update_tail (&r->prod, prod_head, prod_next, is_sp,1 ); +__rte_ring_update_tail (&r->prod, prod_head, prod_next, is_sp); end:if (free_space !=NULL ) *free_space = free_entries - n;
@@ -473,7 +492,7 @@ __rte_ring_do_dequeue_elem(struct rte_ring *r, void *obj_table,__rte_ring_dequeue_elems (r, cons_head, obj_table, esize, n); -__rte_ring_update_tail (&r->cons, cons_head, cons_next, is_sc,0 ); +__rte_ring_update_tail (&r->cons, cons_head, cons_next, is_sc); end:if (available !=NULL )
lib/ring/rte_ring_gcc_pvt.h renamed from lib/ring/rte_ring_generic_pvt.h +62 -79
192 lines not on this page · raw message
lib/ring/rte_ring_hts_elem_pvt.h +3 -5
@@ -25,12 +25,10 @@*/ static __rte_always_inline void__rte_ring_hts_update_tail (struct rte_ring_hts_headtail *ht,uint32_t old_tail, -uint32_t num,uint32_t enqueue) +uint32_t num) {uint32_t tail; -RTE_SET_USED (enqueue); - tail = old_tail + num;/*
@@ -217,7 +215,7 @@ __rte_ring_do_hts_enqueue_elem(struct rte_ring *r, const void *obj_table,if (n !=0 ) {__rte_ring_enqueue_elems (r, head, obj_table, esize, n); -__rte_ring_hts_update_tail (&r->hts_prod, head, n,1 ); +__rte_ring_hts_update_tail (&r->hts_prod, head, n); }if (free_space !=NULL )
@@ -258,7 +256,7 @@ __rte_ring_do_hts_dequeue_elem(struct rte_ring *r, void *obj_table,if (n !=0 ) {__rte_ring_dequeue_elems (r, head, obj_table, esize, n); -__rte_ring_hts_update_tail (&r->hts_cons, head, n,0 ); +__rte_ring_hts_update_tail (&r->hts_cons, head, n); }if (available !=NULL )
lib/ring/soring.c +5 -5
@@ -202,21 +202,21 @@ __rte_soring_move_cons_head(struct rte_soring *r, uint32_t stage, uint32_t num, static __rte_always_inline void__rte_soring_update_tail (struct __rte_ring_headtail *rht, -enum rte_ring_sync_type st,uint32_t head,uint32_t next,uint32_t enq) +enum rte_ring_sync_type st,uint32_t head,uint32_t next) {uint32_t n;switch (st) {case RTE_RING_SYNC_ST :case RTE_RING_SYNC_MT : -__rte_ring_update_tail (&rht->ht, head, next, st, enq); +__rte_ring_update_tail (&rht->ht, head, next, st);break ;case RTE_RING_SYNC_MT_RTS :__rte_ring_rts_update_tail (&rht->rts);break ;case RTE_RING_SYNC_MT_HTS : n = next - head; -__rte_ring_hts_update_tail (&rht->hts, head, n, enq); +__rte_ring_hts_update_tail (&rht->hts, head, n);break ;default :/* unsupported mode, shouldn't be here */
@@ -295,7 +295,7 @@ soring_enqueue(struct rte_soring *r, const void *objs, &prod_head, &prod_next, &nb_free);if (n !=0 ) {__enqueue_elems (r, objs, meta, prod_head, n); -__rte_soring_update_tail (&r->prod, st, prod_head, prod_next,1 ); +__rte_soring_update_tail (&r->prod, st, prod_head, prod_next); }if (free_space !=NULL )
@@ -401,7 +401,7 @@ soring_dequeue(struct rte_soring *r, void *objs, void *meta,/* we have some elems to consume */ if (n !=0 ) {__dequeue_elems (r, objs, meta, cons_head, n); -__rte_soring_update_tail (&r->cons, st, cons_head, cons_next,0 ); +__rte_soring_update_tail (&r->cons, st, cons_head, cons_next); }if (available !=NULL )
Signature · 1 line
2.53.0
Thomas Monjalon
AppliedRe: [PATCH v3 0/2] ring: replace use of rte_atomic
In reply to Stephen Hemminger10/06/2026 20:43, Stephen Hemminger:
Quoted 7 lines
v3:
- rebase and squash patches
- keep original code for x86 single thread caseStephen Hemminger (2):
ring: split single thread vs multi-thread cases
ring: replacerte_atomic32with__syncbuiltin
Applied, thanks.