lib/timer: protect timer subsystem initialized with lock
PATCH 1/2 lib/timer: protect timer subsystem initialized with lock
From: Honnappa Nagarahalli <[email protected]>
rte_timer_subsystem_initialized is a global variable that can be accessed by multiple processes simultaneously. Hence, any access to rte_timer_subsystem_initialized should be protected by rte_mcfg_timer_lock.
--- lib/librte_timer/rte_timer.c | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-)
lib/librte_timer/rte_timer.c +14 -10
@@ -145,11 +145,13 @@ rte_timer_subsystem_init(void)const size_t mem_size = data_arr_size +sizeof (*rte_timer_mz_refcnt);bool do_full_init = true; -if (rte_timer_subsystem_initialized) -return -EALREADY ; -rte_mcfg_timer_lock (); +if (rte_timer_subsystem_initialized) { +rte_mcfg_timer_unlock (); +return -EALREADY ; + } + mz =rte_memzone_lookup (mz_name);if (mz ==NULL ) { mz = rte_memzone_reserve_aligned(mz_name, mem_size,
@@ -183,27 +185,29 @@ rte_timer_subsystem_init(void) rte_timer_data_arr[default_data_id].internal_flags |=FL_ALLOCATED ; (*rte_timer_mz_refcnt)++; -rte_mcfg_timer_unlock (); - rte_timer_subsystem_initialized =1 ; +rte_mcfg_timer_unlock (); +return 0 ; }void rte_timer_subsystem_finalize (void ) { -if (!rte_timer_subsystem_initialized) -return ; -rte_mcfg_timer_lock (); +if (!rte_timer_subsystem_initialized) { +rte_mcfg_timer_unlock (); +return ; + } +if (--(*rte_timer_mz_refcnt) ==0 )rte_memzone_free (rte_timer_data_mz); -rte_mcfg_timer_unlock (); - rte_timer_subsystem_initialized =0 ; + +rte_mcfg_timer_unlock (); }/* Initialize the timer handle tim for use */
Signature · 1 line
2.7.4
PATCH 2/2 lib/timer: relax barrier for status update
Volatile has no ordering semantics. The rte_timer structure defines timer status as a volatile variable and uses the rte_r/wmb barrier to guarantee inter-thread visibility.
This patch optimized the volatile operation with c11 atomic operations and one-way barrier to save the performance penalty. According to the timer_perf_autotest benchmarking results, this patch can uplift 10%~16% timer appending performance, 3%~20% timer resetting performance and 45% timer callbacks scheduling performance on aarch64 and no loss in performance for x86.
--- lib/librte_timer/rte_timer.c | 90 +++++++++++++++++++++++++++++++------------- lib/librte_timer/rte_timer.h | 2 +- 2 files changed, 65 insertions(+), 27 deletions(-)
lib/librte_timer/rte_timer.c +64 -26
@@ -10,7 +10,6 @@#include <assert.h> #include <sys/queue.h> -#include <rte_atomic.h> #include <rte_common.h> #include <rte_cycles.h> #include <rte_eal_memconfig.h>
@@ -218,7 +217,7 @@ rte_timer_init(struct rte_timer *tim) status.state =RTE_TIMER_STOP ; status.owner =RTE_TIMER_NO_OWNER ; - tim->status.u32 = status.u32; +__atomic_store_n (&tim->status.u32, status.u32, __ATOMIC_RELAXED); }/*
@@ -239,9 +238,9 @@ timer_set_config_state(struct rte_timer *tim,/* wait that the timer is in correct status before update, * and mark it as being configured */ -while (success ==0 ) { - prev_status.u32 = tim->status.u32; + prev_status.u32 =__atomic_load_n (&tim->status.u32, __ATOMIC_RELAXED); +while (success ==0 ) {/* timer is running on another core * or ready to run on local core, exit */
@@ -258,9 +257,20 @@ timer_set_config_state(struct rte_timer *tim,* mark it atomically as being configured */ status.state =RTE_TIMER_CONFIG ; status.owner = (int16_t )lcore_id; - success =rte_atomic32_cmpset (&tim->status.u32, - prev_status.u32, - status.u32); +/* If status is observed as RTE_TIMER_CONFIG earlier, +* that's not going to cause any issues because the +* pattern is read for status then read the other members. +* In one of the callers to timer_set_config_state +* (the __rte_timer_reset) we set other members to the +* structure (period, expire, f, arg) we want these +* changes to be observed after our change to status. +* So we need __ATOMIC_ACQUIRE here. +*/ + success =__atomic_compare_exchange_n (&tim->status.u32, + &prev_status.u32, + status.u32,0 , + __ATOMIC_ACQUIRE, + __ATOMIC_RELAXED); } ret_prev_status->u32 = prev_status.u32;
@@ -279,20 +289,27 @@ timer_set_running_state(struct rte_timer *tim)/* wait that the timer is in correct status before update, * and mark it as running */ -while (success ==0 ) { - prev_status.u32 = tim->status.u32; + prev_status.u32 =__atomic_load_n (&tim->status.u32, __ATOMIC_RELAXED); +while (success ==0 ) {/* timer is not pending anymore */ if (prev_status.state !=RTE_TIMER_PENDING )return -1 ;/* here, we know that timer is stopped or pending, -* mark it atomically as being configured */ +* mark it atomically as being running +*/ status.state =RTE_TIMER_RUNNING ; status.owner = (int16_t )lcore_id; - success =rte_atomic32_cmpset (&tim->status.u32, - prev_status.u32, - status.u32); +/* RUNNING states are acting as locked states. If the +* timer is in RUNNING state, the state cannot be changed +* by other threads. So, we should use ACQUIRE here. +*/ + success =__atomic_compare_exchange_n (&tim->status.u32, + &prev_status.u32, + status.u32,0 , + __ATOMIC_ACQUIRE, + __ATOMIC_RELAXED); }return 0 ;
@@ -520,10 +537,12 @@ __rte_timer_reset(struct rte_timer *tim, uint64_t expire,/* update state: as we are in CONFIG state, only us can modify * the state so we don't need to use cmpset() here */ -rte_wmb (); status.state =RTE_TIMER_PENDING ; status.owner = (int16_t )tim_lcore; - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory operations above +* the status update are observed before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, __ATOMIC_RELEASE);if (tim_lcore != lcore_id || !local_is_locked)rte_spinlock_unlock (&priv_timer[tim_lcore].list_lock);
@@ -600,10 +619,12 @@ __rte_timer_stop(struct rte_timer *tim, int local_is_locked, }/* mark timer as stopped */ -rte_wmb (); status.state =RTE_TIMER_STOP ; status.owner =RTE_TIMER_NO_OWNER ; - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory operations above +* the status update are observed before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, __ATOMIC_RELEASE);return 0 ; }
@@ -637,7 +658,8 @@ rte_timer_stop_sync(struct rte_timer *tim)int rte_timer_pending (struct rte_timer *tim) { -return tim->status.state ==RTE_TIMER_PENDING ; +return __atomic_load_n (&tim->status.state, + __ATOMIC_RELAXED) ==RTE_TIMER_PENDING ; }/* must be called periodically, run all timer that expired */
@@ -739,8 +761,12 @@ __rte_timer_manage(struct rte_timer_data *timer_data)/* remove from done list and mark timer as stopped */ status.state =RTE_TIMER_STOP ; status.owner =RTE_TIMER_NO_OWNER ; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory +* operations above the status update are observed +* before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, + __ATOMIC_RELEASE); }else {/* keep it in list and mark timer as pending */
@@ -748,8 +774,12 @@ __rte_timer_manage(struct rte_timer_data *timer_data) status.state =RTE_TIMER_PENDING ;__TIMER_STAT_ADD (priv_timer, pending,1 ); status.owner = (int16_t )lcore_id; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory +* operations above the status update are observed +* before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, + __ATOMIC_RELEASE);__rte_timer_reset (tim, tim->expire + tim->period, tim->period, lcore_id, tim->f, tim->arg,1 , timer_data);
@@ -919,8 +949,12 @@ rte_timer_alt_manage(uint32_t timer_data_id,/* remove from done list and mark timer as stopped */ status.state =RTE_TIMER_STOP ; status.owner =RTE_TIMER_NO_OWNER ; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory +* operations above the status update are observed +* before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, + __ATOMIC_RELEASE); }else {/* keep it in list and mark timer as pending */ rte_spinlock_lock(
@@ -928,8 +962,12 @@ rte_timer_alt_manage(uint32_t timer_data_id, status.state =RTE_TIMER_PENDING ;__TIMER_STAT_ADD (data->priv_timer, pending,1 ); status.owner = (int16_t )this_lcore; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory +* operations above the status update are observed +* before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, + __ATOMIC_RELEASE);__rte_timer_reset (tim, tim->expire + tim->period, tim->period, this_lcore, tim->f, tim->arg,1 , data);
lib/librte_timer/rte_timer.h +1 -1
@@ -101,7 +101,7 @@ struct rte_timer {uint64_t expire;/**< Time when timer expire. */ struct rte_timer *sl_next[MAX_SKIPLIST_DEPTH ]; -volatile union rte_timer_status status;/**< Status of timer. */ +union rte_timer_status status;/**< Status of timer. */ uint64_t period;/**< Period of timer (0 if not periodic). */ rte_timer_cb_t f;/**< Callback function. */ void *arg;/**< Argument to callback function. */
Signature · 1 line
2.7.4
Re: [dpdk-dev] [PATCH 2/2] lib/timer: relax barrier for status update
Quoted 21 lines
-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Monday, February 24, 2020 2:42 PM
To: [email protected]; [email protected]; [email protected]
Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu <[email protected]>; Phil Yang <[email protected]>; nd <[email protected]>
Subject: [PATCH 2/2] lib/timer: relax barrier for status updateVolatile has no ordering semantics. The
rte_timerstructure defines timer status as a volatile variable and uses the rte_r/wmb barrier to guarantee inter-thread visibility.This patch optimized the volatile operation with c11 atomic operations and one-way barrier to save the performance penalty. According to the
timer_perf_autotestbenchmarking results, this patch can uplift 10%~16% timer appending performance, 3%~20% timer resetting performance and 45% timer callbacks scheduling performance on aarch64 and no loss in performance for x86.
Quoted 5 lines
Suggested-by Honnappa Nagarahalli <[email protected]>Signed-off-by Phil Yang <[email protected]>Reviewed-by Gavin Hu <[email protected]>--- lib/librte_timer/rte_timer.c | 90 +++++++++++++++++++++++++++++++----
Ping.
Thanks,
Phil
Re: [dpdk-dev] [PATCH 2/2] lib/timer: relax barrier for status update
In reply to Phil YangQuoted 22 lines
-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Monday, February 24, 2020 12:42 AM
To: [email protected]; Carrillo, Erik G <[email protected]>; [email protected]
Cc: [email protected]; Burakov, Anatoly <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: [PATCH 2/2] lib/timer: relax barrier for status updateVolatile has no ordering semantics. The
rte_timerstructure defines timer status as a volatile variable and uses the rte_r/wmb barrier to guarantee inter-thread visibility.This patch optimized the volatile operation with c11 atomic operations and one-way barrier to save the performance penalty. According to the
timer_perf_autotestbenchmarking results, this patch can uplift 10%~16% timer appending performance, 3%~20% timer resetting performance and 45% timer callbacks scheduling performance on aarch64 and no loss in performance for x86.
Quoted 3 lines
Suggested-by Honnappa Nagarahalli <[email protected]>Signed-off-by Phil Yang <[email protected]>Reviewed-by Gavin Hu <[email protected]>
Hi Phil,
It seems like the consensus is to generally avoid replacing rte_atomic_* interfaces with the GCC builtins directly. In other areas of DPDK that are being patched, are the <std_atomic.h> C11 APIs going to be investigated? It seems like that decision will apply here as well.
Thanks,
Erik
Quoted 222 lines
--- lib/librte_timer/rte_timer.c | 90 +++++++++++++++++++++++++++++++---- --------- lib/librte_timer/rte_timer.h | 2 +- 2 files changed, 65 insertions(+), 27 deletions(-)lib/librte_timer/rte_timer.c index renamed from lib/librte_timer/rte_timer.c +0
-0269e921..be0262d 100644
lib/librte_timer/rte_timer.c +1
-2@@ -10,7 +10,6 @@#include <assert.h> #include <sys/queue.h> -#include <rte_atomic.h> #include <rte_common.h> #include <rte_cycles.h> #include <rte_eal_memconfig.h> @@ -218,7 +217,7 @@ rte_timer_init(struct rte_timer *tim) status.state =RTE_TIMER_STOP ; status.owner =RTE_TIMER_NO_OWNER ; - tim->status.u32 = status.u32; +__atomic_store_n (&tim->status.u32, status.u32, __ATOMIC_RELAXED); }/*
lib/librte_timer/rte_timer.c +2
-2@@ -239,9 +238,9 @@ timer_set_config_state(struct rte_timer *tim,/* wait that the timer is in correct status before update, * and mark it as being configured */ -while (success ==0 ) { - prev_status.u32 = tim->status.u32; + prev_status.u32 =__atomic_load_n (&tim->status.u32, __ATOMIC_RELAXED); +while (success ==0 ) {/* timer is running on another core * or ready to run on local core, exit lib/librte_timer/rte_timer.c +14
-3*/ @@ -258,9 +257,20 @@ timer_set_config_state(struct rte_timer *tim,* mark it atomically as being configured */ status.state =RTE_TIMER_CONFIG ; status.owner = (int16_t )lcore_id; - success =rte_atomic32_cmpset (&tim->status.u32, - prev_status.u32, - status.u32); +/* If status is observed as RTE_TIMER_CONFIG earlier, +* that's not going to cause any issues because the +* pattern is read for status then read the other members. +* In one of the callers to timer_set_config_state +* (the __rte_timer_reset) we set other members to the +* structure (period, expire, f, arg) we want these +* changes to be observed after our change to status. +* So we need __ATOMIC_ACQUIRE here. +*/ + success =__atomic_compare_exchange_n (&tim- >status.u32, + &prev_status.u32, + status.u32,0 , + __ATOMIC_ACQUIRE, + __ATOMIC_RELAXED); }
ret_prev_status->u32= prev_status.u32; @@ -279,20 +289,27 @@timer_set_running_state(struct rte_timer *tim)lib/librte_timer/rte_timer.c +1
-2/* wait that the timer is in correct status before update, * and mark it as running */ -while (success ==0 ) { - prev_status.u32 = tim->status.u32; + prev_status.u32 = __atomic_load_n(&tim->status.u32,
__ATOMIC_RELAXED);lib/librte_timer/rte_timer.c +8
-4+while (success ==0 ) {/* timer is not pending anymore */ if (prev_status.state !=RTE_TIMER_PENDING )return -1 ;/* here, we know that timer is stopped or pending, -* mark it atomically as being configured */ +* mark it atomically as being running +*/ status.state =RTE_TIMER_RUNNING ; status.owner = (int16_t )lcore_id; - success =rte_atomic32_cmpset (&tim->status.u32, - prev_status.u32, - status.u32); +/* RUNNING states are acting as locked states. If the +* timer is in RUNNING state, the state cannot be changed +* by other threads. So, we should use ACQUIRE here. +*/ + success = __atomic_compare_exchange_n(&tim-Quoted 1 line
status.u32,
lib/librte_timer/rte_timer.c +8
-2+ &prev_status.u32, + status.u32,0 , + __ATOMIC_ACQUIRE, + __ATOMIC_RELAXED); }return 0 ;@@ -520,10 +537,12 @@ __rte_timer_reset(struct rte_timer *tim, uint64_t expire,/* update state: as we are in CONFIG state, only us can modify * the state so we don't need to use cmpset() here */ -rte_wmb (); status.state =RTE_TIMER_PENDING ; status.owner = (int16_t )tim_lcore; - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory operations above +* the status update are observed before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, __ATOMIC_RELEASE);lib/librte_timer/rte_timer.c +4
-2if (tim_lcore != lcore_id || !local_is_locked)rte_spinlock_unlock (&priv_timer[tim_lcore].list_lock);@@ -600,10 +619,12 @@ __rte_timer_stop(struct rte_timer *tim, int local_is_locked, }/* mark timer as stopped */ -rte_wmb (); status.state =RTE_TIMER_STOP ; status.owner =RTE_TIMER_NO_OWNER ; - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory operations above +* the status update are observed before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, __ATOMIC_RELEASE);lib/librte_timer/rte_timer.c +3
-1return 0 ; }@@ -637,7 +658,8 @@ rte_timer_stop_sync(struct rte_timer *tim) int rte_timer_pending(struct rte_timer *tim) { -return tim->status.state ==RTE_TIMER_PENDING ; +return __atomic_load_n (&tim->status.state, + __ATOMIC_RELAXED) ==RTE_TIMER_PENDING ; } /* must be called periodically, run all timer that expired */ @@ -739,8 +761,12 @@ __rte_timer_manage(struct rte_timer_data *timer_data)/* remove from done list and mark timer as stopped */
lib/librte_timer/rte_timer.c +6
-2status.state =RTE_TIMER_STOP ; status.owner =RTE_TIMER_NO_OWNER ; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory +* operations above the status update are observed +* before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, + __ATOMIC_RELEASE); }else { /* keep it in list and mark timer as pending */ @@ -748,8 +774,12 @@
__rte_timer_manage(struct rte_timer_data *timer_data)lib/librte_timer/rte_timer.c +12
-4status.state =RTE_TIMER_PENDING ;__TIMER_STAT_ADD (priv_timer, pending,1 ); status.owner = (int16_t )lcore_id; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory +* operations above the status update are observed +* before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, + __ATOMIC_RELEASE);__rte_timer_reset (tim, tim->expire + tim->period, tim->period, lcore_id, tim->f, tim->arg,1 , timer_data);@@ -919,8 +949,12 @@ rte_timer_alt_manage(uint32_t timer_data_id,/* remove from done list and mark timer as stopped */ status.state =RTE_TIMER_STOP ; status.owner =RTE_TIMER_NO_OWNER ; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory +* operations above the status update are observed +* before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, + __ATOMIC_RELEASE); }else {/* keep it in list and mark timer as pending */ lib/librte_timer/rte_timer.c +6
-2rte_spinlock_lock(@@ -928,8 +962,12 @@ rte_timer_alt_manage(uint32_t timer_data_id, status.state =RTE_TIMER_PENDING ;__TIMER_STAT_ADD (data->priv_timer, pending,1 ); status.owner = (int16_t )this_lcore; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory +* operations above the status update are observed +* before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, + __ATOMIC_RELEASE);__rte_timer_reset (tim, tim->expire + tim->period, tim->period, this_lcore, tim->f, tim->arg,1 , data);lib/librte_timer/rte_timer.h index renamed from lib/librte_timer/rte_timer.h +0
-0c6b3d45..df533fa 100644
lib/librte_timer/rte_timer.h +1
-2@@ -101,7 +101,7 @@ struct rte_timer {uint64_t expire;/**< Time when timer expire. */ struct rte_timer *sl_next[MAX_SKIPLIST_DEPTH ]; -volatile union rte_timer_status status;/**< Status of timer. */ +union rte_timer_status status;/**< Status of timer. */ uint64_t period;/**< Period of timer (0 if not periodic). */ rte_timer_cb_t f;/**< Callback function. */ void *arg;/**< Argument to callback function. */ --2.7.4
Re: [dpdk-dev] [PATCH 2/2] lib/timer: relax barrier for status update
<snip>
Quoted 17 lines
Quoted 13 lines
Subject: [PATCH 2/2] lib/timer: relax barrier for status update
Volatile has no ordering semantics. The
rte_timerstructure defines timer status as a volatile variable and uses the rte_r/wmb barrier to guarantee inter-thread visibility.This patch optimized the volatile operation with c11 atomic operations and one-way barrier to save the performance penalty. According to the
timer_perf_autotestbenchmarking results, this patch can uplift 10%~16% timer appending performance, 3%~20% timer resetting performance and 45% timer callbacks scheduling performance on aarch64 and no loss in performance for x86.Quoted 3 lines
Suggested-by Honnappa Nagarahalli <[email protected]>Signed-off-by Phil Yang <[email protected]>Reviewed-by Gavin Hu <[email protected]>
Quoted 6 lines
Hi Phil,
It seems like the consensus is to generally avoid replacing rte_atomic_*
interfaces with the GCC builtins directly. In other areas of DPDK that are
being patched, are the <std_atomic.h> C11 APIs going to be investigated? It
seems like that decision will apply here as well.
Agree. The new APIs are going to be 1 to 1 mapped with the built-in intrinsics (the memory orderings used themselves will not change). We should go ahead with the review and conclude any issues. Once the decision is made on what APIs to use, we can submit the next version using the APIs decided.
Quoted 227 lines
Thanks,
ErikQuoted 147 lines
--- lib/librte_timer/rte_timer.c | 90 +++++++++++++++++++++++++++++++---- --------- lib/librte_timer/rte_timer.h | 2 +- 2 files changed, 65 insertions(+), 27 deletions(-)(quoted diff) +0
-0
b/lib/librte_timer/rte_timer.cindex 269e921..be0262d 100644lib/librte_timer/rte_timer.c +1
-2@@ -10,7 +10,6 @@#include <assert.h> #include <sys/queue.h> -#include <rte_atomic.h> #include <rte_common.h> #include <rte_cycles.h> #include <rte_eal_memconfig.h> @@ -218,7 +217,7 @@ rte_timer_init(struct rte_timer *tim) status.state =RTE_TIMER_STOP ; status.owner =RTE_TIMER_NO_OWNER ; - tim->status.u32 = status.u32; +__atomic_store_n (&tim->status.u32, status.u32, __ATOMIC_RELAXED); }/*
lib/librte_timer/rte_timer.c +2
-2@@ -239,9 +238,9 @@ timer_set_config_state(struct rte_timer *tim,/* wait that the timer is in correct status before update, * and mark it as being configured */ -while (success ==0 ) { - prev_status.u32 = tim->status.u32; + prev_status.u32 =__atomic_load_n (&tim->status.u32, __ATOMIC_RELAXED); +while (success ==0 ) {/* timer is running on another core * or ready to run on local core, exit lib/librte_timer/rte_timer.c +14
-3*/ @@ -258,9 +257,20 @@ timer_set_config_state(struct rte_timer *tim,* mark it atomically as being configured */ status.state =RTE_TIMER_CONFIG ; status.owner = (int16_t )lcore_id; - success =rte_atomic32_cmpset (&tim->status.u32, - prev_status.u32, - status.u32); +/* If status is observed as RTE_TIMER_CONFIG earlier, +* that's not going to cause any issues because the +* pattern is read for status then read the other members. +* In one of the callers to timer_set_config_state +* (the __rte_timer_reset) we set other members to the +* structure (period, expire, f, arg) we want these +* changes to be observed after our change to status. +* So we need __ATOMIC_ACQUIRE here. +*/ + success =__atomic_compare_exchange_n (&tim- >status.u32, + &prev_status.u32, + status.u32,0 , + __ATOMIC_ACQUIRE, + __ATOMIC_RELAXED); }
ret_prev_status->u32= prev_status.u32; @@ -279,20 +289,27 @@timer_set_running_state(struct rte_timer *tim)lib/librte_timer/rte_timer.c +1
-2/* wait that the timer is in correct status before update, * and mark it as running */ -while (success ==0 ) { - prev_status.u32 = tim->status.u32; + prev_status.u32 = __atomic_load_n(&tim->status.u32,
__ATOMIC_RELAXED);lib/librte_timer/rte_timer.c +8
-4+while (success ==0 ) {/* timer is not pending anymore */ if (prev_status.state !=RTE_TIMER_PENDING )return -1 ;/* here, we know that timer is stopped or pending, -* mark it atomically as being configured */ +* mark it atomically as being running +*/ status.state =RTE_TIMER_RUNNING ; status.owner = (int16_t )lcore_id; - success =rte_atomic32_cmpset (&tim->status.u32, - prev_status.u32, - status.u32); +/* RUNNING states are acting as locked states. If the +* timer is in RUNNING state, the state cannot be changed +* by other threads. So, we should use ACQUIRE here. +*/ + success = __atomic_compare_exchange_n(&tim-Quoted 1 line
status.u32,
lib/librte_timer/rte_timer.c +8
-2+ &prev_status.u32, + status.u32,0 , + __ATOMIC_ACQUIRE, + __ATOMIC_RELAXED); }return 0 ;@@ -520,10 +537,12 @@ __rte_timer_reset(struct rte_timer *tim,uint64_t expire,/* update state: as we are in CONFIG state, only us can modify * the state so we don't need to use cmpset() here */ -rte_wmb (); status.state = RTE_TIMER_PENDING; status.owner = (int16_t )tim_lcore; - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory operations above +* the status update are observed before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, __ATOMIC_RELEASE);lib/librte_timer/rte_timer.c +4
-2if (tim_lcore != lcore_id || !local_is_locked)rte_spinlock_unlock (&priv_timer[tim_lcore].list_lock);@@ -600,10 +619,12 @@ __rte_timer_stop(struct rte_timer *tim, int local_is_locked, }/* mark timer as stopped */ -rte_wmb (); status.state =RTE_TIMER_STOP ; status.owner =RTE_TIMER_NO_OWNER ; - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory operations above +* the status update are observed before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, __ATOMIC_RELEASE);lib/librte_timer/rte_timer.c +3
-2return 0 ; }@@ -637,7 +658,8 @@ rte_timer_stop_sync(struct rte_timer *tim) int rte_timer_pending(struct rte_timer *tim) { -return tim->status.state ==RTE_TIMER_PENDING ; +return __atomic_load_n (&tim->status.state, + __ATOMIC_RELAXED) ==RTE_TIMER_PENDING ; } /* must be called periodically, run all timer that expired */ @@ -739 ,8 +761,12 @@ __rte_timer_manage(struct rte_timer_data *timer_data)Quoted 1 line
/* remove from done list and mark timer as stopped
*/
Quoted 29 lines
lib/librte_timer/rte_timer.c +6
-2status.state =RTE_TIMER_STOP ; status.owner =RTE_TIMER_NO_OWNER ; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory +* operations above the status update are observed +* before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, + __ATOMIC_RELEASE); }else { /* keep it in list and mark timer as pending */ @@ -748,8 +774,12 @@
__rte_timer_manage(struct rte_timer_data *timer_data)lib/librte_timer/rte_timer.c +6
-2status.state =RTE_TIMER_PENDING ;__TIMER_STAT_ADD (priv_timer, pending,1 ); status.owner = (int16_t )lcore_id; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory +* operations above the status update are observed +* before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, + __ATOMIC_RELEASE);__rte_timer_reset (tim, tim->expire + tim->period, tim->period, lcore_id, tim->f, tim->arg,1 , timer_data);lib/librte_timer/rte_timer.c rte_timer_alt_manage(uint32_t timer_data_id, · line 949
Quoted 1 line
lib/librte_timer/rte_timer.c +0
-0@@ -919,8 +949,12 @@ rte_timer_alt_manage(uint32_t timer_data_id,/* remove from done list and mark timer as stopped */
Quoted 43 lines
lib/librte_timer/rte_timer.c +12
-4status.state =RTE_TIMER_STOP ; status.owner =RTE_TIMER_NO_OWNER ; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory +* operations above the status update are observed +* before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, + __ATOMIC_RELEASE); }else {/* keep it in list and mark timer as pending */ rte_spinlock_lock(@@ -928,8 +962,12 @@ rte_timer_alt_manage(uint32_t timer_data_id, status.state =RTE_TIMER_PENDING ;__TIMER_STAT_ADD (data->priv_timer, pending,1 ); status.owner = (int16_t )this_lcore; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory +* operations above the status update are observed +* before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, + __ATOMIC_RELEASE);__rte_timer_reset (tim, tim->expire + tim->period, tim->period, this_lcore, tim->f, tim->arg,1 , data);(quoted diff) +0
-0
b/lib/librte_timer/rte_timer.hindex c6b3d45..df533fa 100644lib/librte_timer/rte_timer.h +1
-2@@ -101,7 +101,7 @@ struct rte_timer {uint64_t expire;/**< Time when timer expire. */ struct rte_timer *sl_next[MAX_SKIPLIST_DEPTH ]; -volatile union rte_timer_status status;/**< Status of timer. */ +union rte_timer_status status;/**< Status of timer. */ uint64_t period;/**< Period of timer (0 if not periodic). */ rte_timer_cb_t f;/**< Callback function. */ void *arg;/**< Argument to callback function. */ --2.7.4
Re: [dpdk-dev] [PATCH 2/2] lib/timer: relax barrier for status update
Quoted 36 lines
-----Original Message-----
From: Honnappa Nagarahalli <[email protected]>
Sent: Wednesday, April 8, 2020 4:16 PM
To: Carrillo, Erik G <[email protected]>; Phil Yang <[email protected]>; [email protected]; [email protected]
Cc: [email protected]; Burakov, Anatoly <[email protected]>; [email protected]; [email protected]; [email protected]; Gavin Hu <[email protected]>; nd <[email protected]>; Honnappa Nagarahalli <[email protected]>; nd <[email protected]>
Subject: RE: [PATCH 2/2] lib/timer: relax barrier for status update<snip>
Quoted 17 lines
Quoted 13 lines
Subject: [PATCH 2/2] lib/timer: relax barrier for status update
Volatile has no ordering semantics. The
rte_timerstructure defines timer status as a volatile variable and uses the rte_r/wmb barrier to guarantee inter-thread visibility.This patch optimized the volatile operation with c11 atomic operations and one-way barrier to save the performance penalty. According to the
timer_perf_autotestbenchmarking results, this patch can uplift 10%~16% timer appending performance, 3%~20% timer resetting performance and 45% timer callbacks scheduling performance on aarch64 and no loss in performance for x86.Quoted 3 lines
Suggested-by Honnappa Nagarahalli <[email protected]>Signed-off-by Phil Yang <[email protected]>Reviewed-by Gavin Hu <[email protected]>Quoted 5 lines
Hi Phil,
It seems like the consensus is to generally avoid replacing rte_atomic_*
interfaces with the GCC builtins directly. In other areas of DPDK that are
being patched, are the <std_atomic.h> C11 APIs going to be investigated?
Quoted 7 lines
It
Quoted 1 line
seems like that decision will apply here as well.
Agree. The new APIs are going to be 1 to 1 mapped with the built-in intrinsics (the memory orderings used themselves will not change). We should go ahead with the review and conclude any issues. Once the decision is made on what APIs to use, we can submit the next version using the APIs decided.
Thanks, Honnappa.
I have reviewed the memory orderings and I see no issues with them. I do have a question regarding a comment - I'll pose it inline:
Quoted 42 lines
Quoted 42 lines
Thanks,
ErikQuoted 38 lines
--- lib/librte_timer/rte_timer.c | 90 +++++++++++++++++++++++++++++++---- --------- lib/librte_timer/rte_timer.h | 2 +- 2 files changed, 65 insertions(+), 27 deletions(-)(quoted diff) +0
-0
b/lib/librte_timer/rte_timer.cindex 269e921..be0262d 100644lib/librte_timer/rte_timer.c +1
-2@@ -10,7 +10,6 @@#include <assert.h> #include <sys/queue.h> -#include <rte_atomic.h> #include <rte_common.h> #include <rte_cycles.h> #include <rte_eal_memconfig.h> @@ -218,7 +217,7 @@ rte_timer_init(struct rte_timer *tim) status.state =RTE_TIMER_STOP ; status.owner =RTE_TIMER_NO_OWNER ; - tim->status.u32 = status.u32; +__atomic_store_n (&tim->status.u32, status.u32, __ATOMIC_RELAXED); }/*
lib/librte_timer/rte_timer.c +1
-2@@ -239,9 +238,9 @@ timer_set_config_state(struct rte_timer *tim,/* wait that the timer is in correct status before update, * and mark it as being configured */ -while (success ==0 ) { - prev_status.u32 = tim->status.u32; + prev_status.u32 =__atomic_load_n (&tim->status.u32, __ATOMIC_RELAXED);
Quoted 14 lines
Quoted 6 lines
Quoted 6 lines
lib/librte_timer/rte_timer.c +1
-0+while (success ==0 ) {/* timer is running on another core * or ready to run on local core, exit lib/librte_timer/rte_timer.c +0
-0*/ @@ -258,9 +257,20 @@ timer_set_config_state(struct rte_timer *tim,* mark it atomically as being configured */ Quoted 8 lines
lib/librte_timer/rte_timer.c timer_set_config_state(struct rte_timer *tim, · line 262
Quoted 8 lines
lib/librte_timer/rte_timer.c +3
-3status.state =RTE_TIMER_CONFIG ; status.owner = (int16_t )lcore_id; - success =rte_atomic32_cmpset (&tim->status.u32, - prev_status.u32, - status.u32); +/* If status is observed as RTE_TIMER_CONFIG earlier, +* that's not going to cause any issues because the +* pattern is read for status then read the other members.
I don't follow the above comment. What is meant by "earlier"?
Thanks,
Erik
Quoted 172 lines
Quoted 98 lines
Quoted 96 lines
lib/librte_timer/rte_timer.c +11
-0+* In one of the callers to timer_set_config_state +* (the __rte_timer_reset) we set other members to the +* structure (period, expire, f, arg) we want these +* changes to be observed after our change to status. +* So we need __ATOMIC_ACQUIRE here. +*/ + success =__atomic_compare_exchange_n (&tim- >status.u32, + &prev_status.u32, + status.u32,0 , + __ATOMIC_ACQUIRE, + __ATOMIC_RELAXED); }
ret_prev_status->u32= prev_status.u32; @@ -279,20 +289,27 @@timer_set_running_state(struct rte_timer *tim)lib/librte_timer/rte_timer.c +1
-2/* wait that the timer is in correct status before update, * and mark it as running */ -while (success ==0 ) { - prev_status.u32 = tim->status.u32; + prev_status.u32 = __atomic_load_n(&tim->status.u32,
__ATOMIC_RELAXED);lib/librte_timer/rte_timer.c +8
-4+while (success ==0 ) {/* timer is not pending anymore */ if (prev_status.state !=RTE_TIMER_PENDING )return -1 ;/* here, we know that timer is stopped or pending, -* mark it atomically as being configured */ +* mark it atomically as being running +*/ status.state =RTE_TIMER_RUNNING ; status.owner = (int16_t )lcore_id; - success =rte_atomic32_cmpset (&tim->status.u32, - prev_status.u32, - status.u32); +/* RUNNING states are acting as locked states. If the +* timer is in RUNNING state, the state cannot be changed +* by other threads. So, we should use ACQUIRE here. +*/ + success = __atomic_compare_exchange_n(&tim-Quoted 1 line
status.u32,
lib/librte_timer/rte_timer.c +8
-2+ &prev_status.u32, + status.u32,0 , + __ATOMIC_ACQUIRE, + __ATOMIC_RELAXED); }return 0 ;@@ -520,10 +537,12 @@ __rte_timer_reset(struct rte_timer *tim,uint64_t expire,/* update state: as we are in CONFIG state, only us can modify * the state so we don't need to use cmpset() here */ -rte_wmb (); status.state = RTE_TIMER_PENDING; status.owner = (int16_t )tim_lcore; - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory operations above +* the status update are observed before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, __ATOMIC_RELEASE);lib/librte_timer/rte_timer.c +4
-2if (tim_lcore != lcore_id || !local_is_locked)rte_spinlock_unlock (&priv_timer[tim_lcore].list_lock);@@ -600,10 +619,12 @@ __rte_timer_stop(struct rte_timer *tim, int local_is_locked, }/* mark timer as stopped */ -rte_wmb (); status.state =RTE_TIMER_STOP ; status.owner =RTE_TIMER_NO_OWNER ; - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory operations above +* the status update are observed before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, __ATOMIC_RELEASE);lib/librte_timer/rte_timer.c +3
-2return 0 ; }@@ -637,7 +658,8 @@ rte_timer_stop_sync(struct rte_timer *tim) int rte_timer_pending(struct rte_timer *tim) { -return tim->status.state ==RTE_TIMER_PENDING ; +return __atomic_load_n (&tim->status.state, + __ATOMIC_RELAXED) ==RTE_TIMER_PENDING ; } /* must be called periodically, run all timer that expired */ @@ -739 ,8 +761,12 @@ __rte_timer_manage(struct rte_timer_data *timer_data)Quoted 1 line
/* remove from done list and mark timer as stopped
*/
Quoted 14 lines
Quoted 13 lines
lib/librte_timer/rte_timer.c +6
-2status.state =RTE_TIMER_STOP ; status.owner =RTE_TIMER_NO_OWNER ; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory +* operations above the status update are observed +* before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, + __ATOMIC_RELEASE); }else { /* keep it in list and mark timer as pending */ @@ -Quoted 1 line
748,8 +774,12 @@ __rte_timer_manage(struct
rte_timer_data
*timer_data)Quoted 59 lines
Quoted 15 lines
lib/librte_timer/rte_timer.c +6
-2status.state =RTE_TIMER_PENDING ;__TIMER_STAT_ADD (priv_timer, pending,1 ); status.owner = (int16_t )lcore_id; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory +* operations above the status update are observed +* before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, + __ATOMIC_RELEASE);__rte_timer_reset (tim, tim->expire + tim->period, tim->period, lcore_id, tim->f, tim->arg,1 , timer_data);lib/librte_timer/rte_timer.c rte_timer_alt_manage(uint32_t timer_data_id, · line 949
Quoted 1 line
lib/librte_timer/rte_timer.c +0
-0@@ -919,8 +949,12 @@ rte_timer_alt_manage(uint32_t timer_data_id,/* remove from done list and mark timer as stopped */
Quoted 42 lines
lib/librte_timer/rte_timer.c +12
-4status.state =RTE_TIMER_STOP ; status.owner =RTE_TIMER_NO_OWNER ; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory +* operations above the status update are observed +* before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, + __ATOMIC_RELEASE); }else {/* keep it in list and mark timer as pending */ rte_spinlock_lock(@@ -928,8 +962,12 @@ rte_timer_alt_manage(uint32_t timer_data_id, status.state =RTE_TIMER_PENDING ;__TIMER_STAT_ADD (data->priv_timer, pending,1 ); status.owner = (int16_t )this_lcore; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory +* operations above the status update are observed +* before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, + __ATOMIC_RELEASE);__rte_timer_reset (tim, tim->expire + tim->period, tim->period, this_lcore, tim->f, tim->arg,1 , data);(quoted diff) +0
-0
b/lib/librte_timer/rte_timer.hindex c6b3d45..df533fa 100644lib/librte_timer/rte_timer.h +1
-2@@ -101,7 +101,7 @@ struct rte_timer {uint64_t expire;/**< Time when timer expire. */ struct rte_timer *sl_next[MAX_SKIPLIST_DEPTH ]; -volatile union rte_timer_status status;/**< Status of timer. */ +union rte_timer_status status;/**< Status of timer. */ uint64_t period;/**< Period of timer (0 if not periodic). */ rte_timer_cb_t f;/**< Callback function. */ void *arg;/**< Argument to callback function. */ --2.7.4
Re: [dpdk-dev] [PATCH 2/2] lib/timer: relax barrier for status update
<snip>
Quoted 29 lines
Quoted 23 lines
Quoted 18 lines
Quoted 14 lines
Subject: [PATCH 2/2] lib/timer: relax barrier for status update
Volatile has no ordering semantics. The
rte_timerstructure defines timer status as a volatile variable and uses the rte_r/wmb barrier to guarantee inter-thread visibility.This patch optimized the volatile operation with c11 atomic operations and one-way barrier to save the performance penalty. According to the
timer_perf_autotestbenchmarking results, this patch can uplift 10%~16% timer appending performance, 3%~20% timer resetting performance and 45% timer callbacks scheduling performance on aarch64 and no loss in performance for x86.Quoted 3 lines
Suggested-by Honnappa Nagarahalli <[email protected]>Signed-off-by Phil Yang <[email protected]>Reviewed-by Gavin Hu <[email protected]>Quoted 5 lines
Hi Phil,
It seems like the consensus is to generally avoid replacing rte_atomic_*
interfaces with the GCC builtins directly. In other areas of DPDK that are
being patched, are the <std_atomic.h> C11 APIs going to be investigated?Quoted 6 lines
It
Quoted 1 line
seems like that decision will apply here as well.
Agree. The new APIs are going to be 1 to 1 mapped with the built-in intrinsics (the memory orderings used themselves will not change). We should go ahead with the review and conclude any issues. Once the decision is made on what APIs to use, we can submit the next version using
Quoted 6 lines
the APIs decided.
Quoted 1 line
Thanks, Honnappa.
I have reviewed the memory orderings and I see no issues with them. I do
have a question regarding a comment - I'll pose it inline:
Fantastic, thank you.
I have an unrelated (to this patch) question for you below.
Quoted 60 lines
Quoted 43 lines
Quoted 43 lines
Thanks,
ErikQuoted 38 lines
--- lib/librte_timer/rte_timer.c | 90 +++++++++++++++++++++++++++++++---- --------- lib/librte_timer/rte_timer.h | 2 +- 2 files changed, 65 insertions(+), 27 deletions(-)(quoted diff) +0
-0
b/lib/librte_timer/rte_timer.cindex 269e921..be0262d 100644lib/librte_timer/rte_timer.c +1
-2@@ -10,7 +10,6 @@#include <assert.h> #include <sys/queue.h> -#include <rte_atomic.h> #include <rte_common.h> #include <rte_cycles.h> #include <rte_eal_memconfig.h> @@ -218,7 +217,7 @@ rte_timer_init(struct rte_timer *tim) status.state =RTE_TIMER_STOP ; status.owner =RTE_TIMER_NO_OWNER ; - tim->status.u32 = status.u32; +__atomic_store_n (&tim->status.u32, status.u32, __ATOMIC_RELAXED); }/*
lib/librte_timer/rte_timer.c +1
-2@@ -239,9 +238,9 @@ timer_set_config_state(struct rte_timer *tim,/* wait that the timer is in correct status before update, * and mark it as being configured */ -while (success ==0 ) { - prev_status.u32 = tim->status.u32; + prev_status.u32 =__atomic_load_n (&tim->status.u32, __ATOMIC_RELAXED);Quoted 14 lines
Quoted 6 lines
Quoted 6 lines
lib/librte_timer/rte_timer.c +1
-0+while (success ==0 ) {/* timer is running on another core * or ready to run on local core, exit lib/librte_timer/rte_timer.c +0
-0*/ @@ -258,9 +257,20 @@ timer_set_config_state(struct rte_timer *tim,* mark it atomically as being configured */ Quoted 8 lines
lib/librte_timer/rte_timer.c timer_set_config_state(struct rte_timer *tim, · line 262
Quoted 8 lines
lib/librte_timer/rte_timer.c +3
-3status.state =RTE_TIMER_CONFIG ; status.owner = (int16_t )lcore_id; - success =rte_atomic32_cmpset (&tim->status.u32, - prev_status.u32, - status.u32); +/* If status is observed as RTE_TIMER_CONFIG earlier, +* that's not going to cause any issues because the +* pattern is read for status then read the other members. I don't follow the above comment. What is meant by "earlier"?
Quoted 2 lines
Thanks,
Erik
I would rather change this comment to something similar to what is mentioned while changing to 'RUNNING' state. 'CONFIG' is also a locking state. I think it is much easier to understand.
Quoted 114 lines
Quoted 114 lines
Quoted 99 lines
Quoted 97 lines
lib/librte_timer/rte_timer.c +11
-0+* In one of the callers to timer_set_config_state +* (the __rte_timer_reset) we set other members to the +* structure (period, expire, f, arg) we want these +* changes to be observed after our change to status. +* So we need __ATOMIC_ACQUIRE here. +*/ + success =__atomic_compare_exchange_n (&tim- >status.u32, + &prev_status.u32, + status.u32,0 , + __ATOMIC_ACQUIRE, + __ATOMIC_RELAXED); }
ret_prev_status->u32= prev_status.u32; @@ -279,20 +289,27 @@timer_set_running_state(struct rte_timer *tim)lib/librte_timer/rte_timer.c +1
-2/* wait that the timer is in correct status before update, * and mark it as running */ -while (success ==0 ) { - prev_status.u32 = tim->status.u32; + prev_status.u32 = __atomic_load_n(&tim->status.u32,
__ATOMIC_RELAXED);lib/librte_timer/rte_timer.c +8
-4+while (success ==0 ) {/* timer is not pending anymore */ if (prev_status.state !=RTE_TIMER_PENDING )return -1 ;/* here, we know that timer is stopped or pending, -* mark it atomically as being configured */ +* mark it atomically as being running +*/ status.state =RTE_TIMER_RUNNING ; status.owner = (int16_t )lcore_id; - success =rte_atomic32_cmpset (&tim->status.u32, - prev_status.u32, - status.u32); +/* RUNNING states are acting as locked states. If the +* timer is in RUNNING state, the state cannot be changed +* by other threads. So, we should use ACQUIRE here. +*/ + success = __atomic_compare_exchange_n(&tim-Quoted 1 line
status.u32,
lib/librte_timer/rte_timer.c +8
-2+ &prev_status.u32, + status.u32,0 , + __ATOMIC_ACQUIRE, + __ATOMIC_RELAXED); }return 0 ;@@ -520,10 +537,12 @@ __rte_timer_reset(struct rte_timer *tim,uint64_t expire,/* update state: as we are in CONFIG state, only us can modify * the state so we don't need to use cmpset() here */ -rte_wmb (); status.state = RTE_TIMER_PENDING; status.owner = (int16_t )tim_lcore; - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory operations above +* the status update are observed before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, __ATOMIC_RELEASE);lib/librte_timer/rte_timer.c +4
-2if (tim_lcore != lcore_id || !local_is_locked)rte_spinlock_unlock (&priv_timer[tim_lcore].list_lock);@@ -600,10 +619,12 @@ __rte_timer_stop(struct rte_timer *tim, int local_is_locked, }/* mark timer as stopped */ -rte_wmb (); status.state =RTE_TIMER_STOP ; status.owner =RTE_TIMER_NO_OWNER ; - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory operations above +* the status update are observed before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, __ATOMIC_RELEASE);lib/librte_timer/rte_timer.c +3
-2return 0 ; }@@ -637,7 +658,8 @@ rte_timer_stop_sync(struct rte_timer *tim)int rte_timer_pending (struct rte_timer *tim) { -return tim->status.state ==RTE_TIMER_PENDING ; +return __atomic_load_n (&tim->status.state, + __ATOMIC_RELAXED) ==RTE_TIMER_PENDING ; } /* must be called periodically, run all timer that expired */ @@ -739 ,8 +761,12 @@ __rte_timer_manage(struct rte_timer_data *timer_data)Quoted 1 line
/* remove from done list and mark timer as stopped
*/
Quoted 14 lines
Quoted 13 lines
lib/librte_timer/rte_timer.c +6
-2status.state =RTE_TIMER_STOP ; status.owner =RTE_TIMER_NO_OWNER ; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory +* operations above the status update are observed +* before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, + __ATOMIC_RELEASE); }else { /* keep it in list and mark timer as pending */ @@ -Quoted 1 line
748,8 +774,12 @@ __rte_timer_manage(struct
rte_timer_data
*timer_data)
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
status.state =
RTE_TIMER_PENDING;
Is it better to set this to STOPPED since it is out of the run list? I think it is better for the understanding as well.
Quoted 58 lines
Quoted 58 lines
Quoted 58 lines
Quoted 14 lines
lib/librte_timer/rte_timer.c +6
-2__TIMER_STAT_ADD (priv_timer, pending,1 ); status.owner = (int16_t )lcore_id; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory +* operations above the status update are observed +* before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, + __ATOMIC_RELEASE);__rte_timer_reset (tim, tim->expire + tim->period, tim->period, lcore_id, tim->f, tim->arg,1 , timer_data);lib/librte_timer/rte_timer.c rte_timer_alt_manage(uint32_t timer_data_id, · line 949
Quoted 1 line
lib/librte_timer/rte_timer.c +0
-0@@ -919,8 +949,12 @@ rte_timer_alt_manage(uint32_t timer_data_id,/* remove from done list and mark timer as stopped */
Quoted 42 lines
lib/librte_timer/rte_timer.c +12
-4status.state =RTE_TIMER_STOP ; status.owner =RTE_TIMER_NO_OWNER ; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory +* operations above the status update are observed +* before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, + __ATOMIC_RELEASE); }else {/* keep it in list and mark timer as pending */ rte_spinlock_lock(@@ -928,8 +962,12 @@ rte_timer_alt_manage(uint32_t timer_data_id, status.state =RTE_TIMER_PENDING ;__TIMER_STAT_ADD (data->priv_timer, pending,1 ); status.owner = (int16_t )this_lcore; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory +* operations above the status update are observed +* before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, + __ATOMIC_RELEASE);__rte_timer_reset (tim, tim->expire + tim->period, tim->period, this_lcore, tim->f, tim->arg,1 , data);(quoted diff) +0
-0
b/lib/librte_timer/rte_timer.hindex c6b3d45..df533fa 100644lib/librte_timer/rte_timer.h +1
-2@@ -101,7 +101,7 @@ struct rte_timer {uint64_t expire;/**< Time when timer expire. */ struct rte_timer *sl_next[MAX_SKIPLIST_DEPTH ]; -volatile union rte_timer_status status;/**< Status of timer. */ +union rte_timer_status status;/**< Status of timer. */ uint64_t period;/**< Period of timer (0 if not periodic). */ rte_timer_cb_t f;/**< Callback function. */ void *arg;/**< Argument to callback function. */ --2.7.4
Re: [dpdk-dev] [PATCH 2/2] lib/timer: relax barrier for status update
Quoted 90 lines
-----Original Message-----
From: Honnappa Nagarahalli <[email protected]>
Sent: Wednesday, April 8, 2020 4:56 PM
To: Carrillo, Erik G <[email protected]>; Phil Yang <[email protected]>; [email protected]; [email protected]
Cc: [email protected]; Burakov, Anatoly <[email protected]>; [email protected]; [email protected]; [email protected]; Gavin Hu <[email protected]>; nd <[email protected]>; Honnappa Nagarahalli <[email protected]>; nd <[email protected]>
Subject: RE: [PATCH 2/2] lib/timer: relax barrier for status update<snip>
Quoted 19 lines
Quoted 19 lines
Quoted 19 lines
Quoted 18 lines
Subject: [PATCH 2/2] lib/timer: relax barrier for status update
Volatile has no ordering semantics. The
rte_timerstructure defines timer status as a volatile variable and uses the rte_r/wmb barrier to guarantee inter-thread visibility.This patch optimized the volatile operation with c11 atomic operations and one-way barrier to save the performance penalty. According to the
timer_perf_autotestbenchmarking results, this patch can uplift 10%~16% timer appending performance, 3%~20% timer resetting performance and 45% timer callbacks scheduling performance on aarch64 and no loss in performance for x86.Suggested-by: Honnappa Nagarahalli
<[email protected]>Signed-off-by Phil Yang <[email protected]>Reviewed-by Gavin Hu <[email protected]>Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Hi Phil,
It seems like the consensus is to generally avoid replacing rte_atomic_*
interfaces with the GCC builtins directly. In other areas of DPDK thatare
Quoted 1 line
Quoted 1 line
Quoted 1 line
being patched, are the <
std_atomic.h> C11 APIs going to beinvestigated?
Quoted 7 lines
Quoted 7 lines
It
Quoted 1 line
seems like that decision will apply here as well.
Agree. The new APIs are going to be 1 to 1 mapped with the built-in intrinsics (the memory orderings used themselves will not change). We should go ahead with the review and conclude any issues. Once the decision is made on what APIs to use, we can submit the next version using
Quoted 6 lines
the APIs decided.
Quoted 1 line
Thanks, Honnappa.
I have reviewed the memory orderings and I see no issues with them. I do
have a question regarding a comment - I'll pose it inline:Fantastic, thank you.
I have an unrelated (to this patch) question for you below.Quoted 34 lines
Quoted 34 lines
Quoted 34 lines
Thanks,
ErikQuoted 29 lines
--- lib/librte_timer/rte_timer.c | 90 +++++++++++++++++++++++++++++++---- --------- lib/librte_timer/rte_timer.h | 2 +- 2 files changed, 65 insertions(+), 27 deletions(-)(quoted diff) +0
-0
b/lib/librte_timer/rte_timer.cindex 269e921..be0262d 100644lib/librte_timer/rte_timer.c +1
-2@@ -10,7 +10,6 @@#include <assert.h> #include <sys/queue.h> -#include <rte_atomic.h> #include <rte_common.h> #include <rte_cycles.h> #include <rte_eal_memconfig.h> @@ -218,7 +217,7 @@ rte_timer_init(struct rte_timer *tim) status.state =RTE_TIMER_STOP ; status.owner =RTE_TIMER_NO_OWNER ; - tim->status.u32 = status.u32; +__atomic_store_n (&tim->status.u32, status.u32, __ATOMIC_RELAXED); }/*
<... snipped ...>
Quoted 14 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
lib/librte_timer/rte_timer.c timer_set_config_state(struct rte_timer
Quoted 1 line
lib/librte_timer/rte_timer.c +0
-0@@ -258,9 +257,20 @@ timer_set_config_state(struct rte_timer*tim,
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
lib/librte_timer/rte_timer.c timer_set_config_state(struct rte_timer *tim,
Quoted 7 lines
lib/librte_timer/rte_timer.c +1
-3* mark it atomically as being configured */ status.state =RTE_TIMER_CONFIG ; status.owner = (int16_t )lcore_id; - success =rte_atomic32_cmpset (&tim->status.u32, - prev_status.u32, - status.u32); +/* If status is observed as RTE_TIMER_CONFIG earlier,
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
lib/librte_timer/rte_timer.c timer_set_config_state(struct rte_timer · line 262
Quoted 2 lines
lib/librte_timer/rte_timer.c +2
-0+* that's not going to cause any issues because the +* pattern is read for status then read the other members.
Quoted 1 line
Quoted 8 lines
Quoted 4 lines
I don't follow the above comment. What is meant by "earlier"?
Thanks,
ErikI would rather change this comment to something similar to what is mentioned while changing to 'RUNNING' state.
'CONFIG' is also a locking state. I think it is much easier to understand.
Ok, thanks - that makes sense.
< ... snipped ...>
Quoted 6 lines
Quoted 3 lines
Quoted 3 lines
Quoted 1 line
Quoted 1 line
748,8 +774,12 @@ __rte_timer_manage(struct
rte_timer_data
*timer_data)Quoted 1 line
Quoted 1 line
status.state =
RTE_TIMER_PENDING;Is it better to set this to STOPPED since it is out of the run list? I think it is better for the understanding as well.
In this location, we are dealing with periodic timers, and we are about to restart the current timer after it just expired and its callback was executed. As I understand it, setting the state back to PENDING here will cause the timer_reset() call below to remove this timer from the list (run list) it's still in (and fix up the links from the previous to the next elements), update other bits of the data structure, and update stats. That behavior would change if we set the state to STOPPED. At least to me, it also seems like the PENDING state is still accurate conceptually since the periodic timer wasn't explicitly stopped by this processing.
Thanks,
Erik
Quoted 76 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
lib/librte_timer/rte_timer.c __rte_timer_manage(struct rte_timer_data *timer_data)
Quoted 5 lines
lib/librte_timer/rte_timer.c +1
-2__TIMER_STAT_ADD (priv_timer, pending,1 ); status.owner = (int16_t )lcore_id; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory
Quoted 1 line
Quoted 1 line
Quoted 1 line
lib/librte_timer/rte_timer.c
Quoted 1 line
lib/librte_timer/rte_timer.c +1
-0+* operations above the status update are observed
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
lib/librte_timer/rte_timer.c __rte_timer_manage(struct rte_timer_data *timer_data)
Quoted 3 lines
lib/librte_timer/rte_timer.c +3
-0+* before the update by all threads +*/ + __atomic_store_n(&tim->status.u32,status.u32,
Quoted 7 lines
Quoted 2 lines
Quoted 2 lines
lib/librte_timer/rte_timer.c
Quoted 2 lines
lib/librte_timer/rte_timer.c +1
-0+ __ATOMIC_RELEASE); __rte_timer_reset(tim, tim->expire + tim-period,
Quoted 1 line
Quoted 1 line
Quoted 1 line
tim->period,lcore_id,tim->f, tim-arg, 1,
Quoted 2 lines
Quoted 2 lines
lib/librte_timer/rte_timer.c rte_timer_alt_manage(uint32_t
Quoted 2 lines
lib/librte_timer/rte_timer.c +0
-0timer_data);@@ -919,8 +949,12 @@ rte_timer_alt_manage(uint32_t
timer_data_id,Quoted 1 line
Quoted 1 line
Quoted 1 line
lib/librte_timer/rte_timer.c rte_timer_alt_manage(uint32_t · line 949
Quoted 1 line
lib/librte_timer/rte_timer.c +0
-0/* remove from done list and mark timer as stopped
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
*/
lib/librte_timer/rte_timer.c __rte_timer_manage(struct rte_timer_data *timer_data)
Quoted 5 lines
lib/librte_timer/rte_timer.c +1
-2status.state =RTE_TIMER_STOP ; status.owner =RTE_TIMER_NO_OWNER ; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory
Quoted 1 line
Quoted 1 line
Quoted 1 line
lib/librte_timer/rte_timer.c rte_timer_alt_manage(uint32_t · line 953
Quoted 1 line
lib/librte_timer/rte_timer.c +1
-0+* operations above the status update are observed
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
lib/librte_timer/rte_timer.c __rte_timer_manage(struct rte_timer_data *timer_data)
Quoted 3 lines
lib/librte_timer/rte_timer.c +3
-0+* before the update by all threads +*/ + __atomic_store_n(&tim->status.u32,status.u32,
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
lib/librte_timer/rte_timer.c rte_timer_alt_manage(uint32_t
Quoted 5 lines
lib/librte_timer/rte_timer.c +1
-0+ __ATOMIC_RELEASE); }else {/* keep it in list and mark timer as pending */ rte_spinlock_lock(@@ -928,8 +962,12 @@ rte_timer_alt_manage(uint32_t
timer_data_id,Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
lib/librte_timer/rte_timer.c rte_timer_alt_manage(uint32_t · line 963
Quoted 2 lines
lib/librte_timer/rte_timer.c +0
-0status.state =RTE_TIMER_PENDING ; __TIMER_STAT_ADD(data->priv_timer,pending, 1);
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
lib/librte_timer/rte_timer.c rte_timer_alt_manage(uint32_t timer_data_id,
Quoted 4 lines
lib/librte_timer/rte_timer.c +1
-2status.owner = (int16_t )this_lcore; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory
Quoted 1 line
Quoted 1 line
Quoted 1 line
lib/librte_timer/rte_timer.c rte_timer_alt_manage(uint32_t · line 966
Quoted 1 line
lib/librte_timer/rte_timer.c +1
-0+* operations above the status update are observed
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
lib/librte_timer/rte_timer.c __rte_timer_manage(struct rte_timer_data *timer_data)
Quoted 3 lines
lib/librte_timer/rte_timer.c +3
-0+* before the update by all threads +*/ + __atomic_store_n(&tim->status.u32,status.u32,
Quoted 5 lines
Quoted 2 lines
Quoted 2 lines
lib/librte_timer/rte_timer.c rte_timer_alt_manage(uint32_t · line 971
Quoted 2 lines
lib/librte_timer/rte_timer.c +1
-0+ __ATOMIC_RELEASE); __rte_timer_reset(tim, tim->expire + tim-period,
Quoted 1 line
Quoted 1 line
lib/librte_timer/rte_timer.c rte_timer_alt_manage(uint32_t · line 972
Quoted 1 line
lib/librte_timer/rte_timer.c +0
-0tim->period, this_lcore, tim->f, tim-arg, 1,
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
lib/librte_timer/rte_timer.h struct rte_timer { · line 103
Quoted 9 lines
lib/librte_timer/rte_timer.c +0
-0data);(quoted diff) +0
-0
b/lib/librte_timer/rte_timer.hindex c6b3d45..df533fa 100644lib/librte_timer/rte_timer.h +0
-1@@ -101,7 +101,7 @@ struct rte_timer {uint64_t expire;/**< Time when timer expire. */ struct rte_timer *sl_next[MAX_SKIPLIST_DEPTH ]; -volatile union rte_timer_status status;/**< Status of timer. */
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
lib/librte_timer/rte_timer.h +1
-1+union rte_timer_status status;/**< Status of timer. */ uint64_t period;/**< Period of timer (0 if not periodic). */ rte_timer_cb_t f;/**< Callback function. */ void *arg;/**< Argument to callback function. */ --2.7.4
Re: [dpdk-dev] [PATCH 2/2] lib/timer: relax barrier for status update
Quoted 123 lines
-----Original Message-----
From: Carrillo, Erik G <[email protected]>
Sent: Friday, April 10, 2020 3:29 AM
To: Honnappa Nagarahalli <[email protected]>; Phil Yang <[email protected]>; [email protected]; [email protected]
Cc: [email protected]; Burakov, Anatoly <[email protected]>; [email protected]; [email protected]; [email protected]; Gavin Hu <[email protected]>; nd <[email protected]>; nd <[email protected]>
Subject: RE: [PATCH 2/2] lib/timer: relax barrier for status updateQuoted 7 lines
-----Original Message-----
From: Honnappa Nagarahalli <[email protected]>
Sent: Wednesday, April 8, 2020 4:56 PM
To: Carrillo, Erik G <[email protected]>; Phil Yang <[email protected]>; [email protected]; [email protected]
Cc: [email protected]; Burakov, Anatoly <[email protected]>; [email protected];Quoted 5 lines
[email protected]; Gavin Hu <[email protected]>; nd <[email protected]>; Honnappa Nagarahalli <[email protected]>; nd <[email protected]>
Subject: RE: [PATCH 2/2] lib/timer: relax barrier for status updateQuoted 24 lines
<snip>
Quoted 22 lines
Quoted 22 lines
Quoted 22 lines
Quoted 14 lines
Subject: [PATCH 2/2] lib/timer: relax barrier for status update
Volatile has no ordering semantics. The
rte_timerstructure defines timer status as a volatile variable and uses the rte_r/wmb barrier to guarantee inter-thread visibility.This patch optimized the volatile operation with c11 atomic operations and one-way barrier to save the performance penalty. According to the
timer_perf_autotestbenchmarking results, this patch can uplift 10%~16% timer appending performance, 3%~20% timer resetting performance and 45% timer callbacks scheduling performance on aarch64 and no loss in performance for x86.Quoted 4 lines
Suggested-by: Honnappa Nagarahalli
<[email protected]>Signed-off-by Phil Yang <[email protected]>Reviewed-by Gavin Hu <[email protected]>Hi Phil,
It seems like the consensus is to generally avoid replacing
rte_atomic_*
Quoted 16 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
interfaces with the GCC builtins directly. In other areas of DPDK that
are
Quoted 1 line
Quoted 1 line
Quoted 1 line
being patched, are the <
std_atomic.h> C11 APIs going to beinvestigated?
Quoted 12 lines
Quoted 7 lines
It
Quoted 1 line
seems like that decision will apply here as well.
Agree. The new APIs are going to be 1 to 1 mapped with the built-in intrinsics (the memory orderings used themselves will not change). We should go ahead with the review and conclude any issues. Once the decision is made on what APIs to use, we can submit the next version using
the APIs decided.
Quoted 1 line
Thanks, Honnappa.
I have reviewed the memory orderings and I see no issues with them. I
do
Quoted 38 lines
Quoted 1 line
have a question regarding a comment - I'll pose it inline:
Fantastic, thank you.
I have an unrelated (to this patch) question for you below.Quoted 34 lines
Quoted 34 lines
Quoted 34 lines
Thanks,
ErikQuoted 29 lines
--- lib/librte_timer/rte_timer.c | 90 +++++++++++++++++++++++++++++++---- --------- lib/librte_timer/rte_timer.h | 2 +- 2 files changed, 65 insertions(+), 27 deletions(-)(quoted diff) +0
-0
b/lib/librte_timer/rte_timer.cindex 269e921..be0262d 100644lib/librte_timer/rte_timer.c +1
-2@@ -10,7 +10,6 @@#include <assert.h> #include <sys/queue.h> -#include <rte_atomic.h> #include <rte_common.h> #include <rte_cycles.h> #include <rte_eal_memconfig.h> @@ -218,7 +217,7 @@ rte_timer_init(struct rte_timer *tim) status.state =RTE_TIMER_STOP ; status.owner =RTE_TIMER_NO_OWNER ; - tim->status.u32 = status.u32; +__atomic_store_n (&tim->status.u32, status.u32, __ATOMIC_RELAXED); }/*
<... snipped ...>
Quoted 16 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
lib/librte_timer/rte_timer.c timer_set_config_state(struct rte_timer
Quoted 1 line
lib/librte_timer/rte_timer.c +0
-0@@ -258,9 +257,20 @@ timer_set_config_state(struct rte_timer*tim,
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
lib/librte_timer/rte_timer.c timer_set_config_state(struct rte_timer *tim,
Quoted 7 lines
lib/librte_timer/rte_timer.c +1
-3* mark it atomically as being configured */ status.state =RTE_TIMER_CONFIG ; status.owner = (int16_t )lcore_id; - success =rte_atomic32_cmpset (&tim->status.u32, - prev_status.u32, - status.u32); +/* If status is observed as RTE_TIMER_CONFIG earlier,
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
lib/librte_timer/rte_timer.c timer_set_config_state(struct rte_timer · line 262
Quoted 2 lines
lib/librte_timer/rte_timer.c +2
-0+* that's not going to cause any issues because the +* pattern is read for status then read the other members.
Quoted 3 lines
I don't follow the above comment. What is meant by "earlier"?
Quoted 8 lines
Quoted 6 lines
Quoted 2 lines
Thanks,
ErikI would rather change this comment to something similar to what is mentioned while changing to 'RUNNING' state.
'CONFIG' is also a locking state. I think it is much easier to understand.Ok, thanks - that makes sense.
OK, thanks.
I will modify the comments in V2 to:
"CONFIG states are acting as locked states. If the timer is in CONFIG state, the state cannot be changed by other threads. So, we should use ACQUIRE here."
Thanks,
Phil
Quoted 10 lines
< ... snipped ...>
Quoted 6 lines
Quoted 3 lines
Quoted 3 lines
Quoted 1 line
Quoted 1 line
748,8 +774,12 @@ __rte_timer_manage(struct
rte_timer_data
*timer_data)Quoted 1 line
Quoted 1 line
status.state =
RTE_TIMER_PENDING;Is it better to set this to STOPPED since it is out of the run list? I think it is better for the understanding as well.
Quoted 9 lines
In this location, we are dealing with periodic timers, and we are about to restart the current timer after it just expired and its callback was executed. As I understand it, setting the state back to PENDING here will cause the
timer_reset()call below to remove this timer from the list (run list) it's still in (and fix up the links from the previous to the next elements), update other
bits of the data structure, and update stats. That behavior would change if
we set the state to STOPPED. At least to me, it also seems like the PENDING state is still accurate conceptually since the periodic timer wasn't explicitly stopped by this processing.
Yes. +1 for this.
Quoted 80 lines
Thanks,
ErikQuoted 76 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
lib/librte_timer/rte_timer.c __rte_timer_manage(struct rte_timer_data *timer_data)
Quoted 5 lines
lib/librte_timer/rte_timer.c +1
-2__TIMER_STAT_ADD (priv_timer, pending,1 ); status.owner = (int16_t )lcore_id; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory
Quoted 1 line
Quoted 1 line
Quoted 1 line
lib/librte_timer/rte_timer.c
Quoted 1 line
lib/librte_timer/rte_timer.c +1
-0+* operations above the status update are observed
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
lib/librte_timer/rte_timer.c __rte_timer_manage(struct rte_timer_data *timer_data)
Quoted 3 lines
lib/librte_timer/rte_timer.c +3
-0+* before the update by all threads +*/ + __atomic_store_n(&tim->status.u32,status.u32,
Quoted 7 lines
Quoted 2 lines
Quoted 2 lines
lib/librte_timer/rte_timer.c
Quoted 2 lines
lib/librte_timer/rte_timer.c +1
-0+ __ATOMIC_RELEASE); __rte_timer_reset(tim, tim->expire + tim-period,
Quoted 1 line
Quoted 1 line
Quoted 1 line
tim->period,lcore_id,tim->f, tim-arg, 1,
Quoted 2 lines
Quoted 2 lines
lib/librte_timer/rte_timer.c rte_timer_alt_manage(uint32_t
Quoted 2 lines
lib/librte_timer/rte_timer.c +0
-0timer_data);@@ -919,8 +949,12 @@ rte_timer_alt_manage(uint32_t
timer_data_id,Quoted 1 line
Quoted 1 line
Quoted 1 line
lib/librte_timer/rte_timer.c rte_timer_alt_manage(uint32_t · line 949
Quoted 1 line
lib/librte_timer/rte_timer.c +0
-0/* remove from done list and mark timer as stopped
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
*/
lib/librte_timer/rte_timer.c __rte_timer_manage(struct rte_timer_data *timer_data)
Quoted 5 lines
lib/librte_timer/rte_timer.c +1
-2status.state =RTE_TIMER_STOP ; status.owner =RTE_TIMER_NO_OWNER ; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory
Quoted 1 line
Quoted 1 line
Quoted 1 line
lib/librte_timer/rte_timer.c rte_timer_alt_manage(uint32_t · line 953
Quoted 1 line
lib/librte_timer/rte_timer.c +1
-0+* operations above the status update are observed
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
lib/librte_timer/rte_timer.c __rte_timer_manage(struct rte_timer_data *timer_data)
Quoted 3 lines
lib/librte_timer/rte_timer.c +3
-0+* before the update by all threads +*/ + __atomic_store_n(&tim->status.u32,status.u32,
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
lib/librte_timer/rte_timer.c rte_timer_alt_manage(uint32_t
Quoted 5 lines
lib/librte_timer/rte_timer.c +1
-0+ __ATOMIC_RELEASE); }else {/* keep it in list and mark timer as pending */ rte_spinlock_lock(@@ -928,8 +962,12 @@ rte_timer_alt_manage(uint32_t
timer_data_id,Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
lib/librte_timer/rte_timer.c rte_timer_alt_manage(uint32_t · line 963
Quoted 2 lines
lib/librte_timer/rte_timer.c +0
-0status.state =RTE_TIMER_PENDING ; __TIMER_STAT_ADD(data->priv_timer,pending, 1);
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
lib/librte_timer/rte_timer.c rte_timer_alt_manage(uint32_t timer_data_id,
Quoted 4 lines
lib/librte_timer/rte_timer.c +1
-2status.owner = (int16_t )this_lcore; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory
Quoted 1 line
Quoted 1 line
Quoted 1 line
lib/librte_timer/rte_timer.c rte_timer_alt_manage(uint32_t · line 966
Quoted 1 line
lib/librte_timer/rte_timer.c +1
-0+* operations above the status update are observed
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
lib/librte_timer/rte_timer.c __rte_timer_manage(struct rte_timer_data *timer_data)
Quoted 3 lines
lib/librte_timer/rte_timer.c +3
-0+* before the update by all threads +*/ + __atomic_store_n(&tim->status.u32,status.u32,
Quoted 5 lines
Quoted 2 lines
Quoted 2 lines
lib/librte_timer/rte_timer.c rte_timer_alt_manage(uint32_t · line 971
Quoted 2 lines
lib/librte_timer/rte_timer.c +1
-0+ __ATOMIC_RELEASE); __rte_timer_reset(tim, tim->expire + tim-period,
Quoted 1 line
Quoted 1 line
lib/librte_timer/rte_timer.c rte_timer_alt_manage(uint32_t · line 972
Quoted 1 line
lib/librte_timer/rte_timer.c +0
-0tim->period, this_lcore, tim->f, tim-arg, 1,
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
lib/librte_timer/rte_timer.h struct rte_timer { · line 103
Quoted 9 lines
lib/librte_timer/rte_timer.c +0
-0data);(quoted diff) +0
-0
b/lib/librte_timer/rte_timer.hindex c6b3d45..df533fa 100644lib/librte_timer/rte_timer.h +0
-1@@ -101,7 +101,7 @@ struct rte_timer {uint64_t expire;/**< Time when timer expire. */ struct rte_timer *sl_next[MAX_SKIPLIST_DEPTH ]; -volatile union rte_timer_status status;/**< Status of timer. */
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
lib/librte_timer/rte_timer.h +1
-1+union rte_timer_status status;/**< Status of timer. */ uint64_t period;/**< Period of timer (0 if not periodic). */ rte_timer_cb_t f;/**< Callback function. */ void *arg;/**< Argument to callback function. */ --2.7.4
PATCH v2 1/1 lib/timer: relax barrier for status update
In reply to Phil YangVolatile has no ordering semantics. The rte_timer structure defines timer status as a volatile variable and uses the rte_r/wmb barrier to guarantee inter-thread visibility.
This patch optimized the volatile operation with c11 atomic operations and one-way barrier to save the performance penalty. According to the timer_perf_autotest benchmarking results, this patch can uplift 10%~16% timer appending performance, 3%~20% timer resetting performance and 45% timer callbacks scheduling performance on aarch64 and no loss in performance for x86.
---
This patch depends on patch:
http://patchwork.dpdk.org/patch/65997/
v2:
1. Changed the memory ordering comment in timer_set_config_state.
2. It is still using built-ins as the wrapper functions for C11 built-ins are not defined yet.
lib/librte_timer/rte_timer.c | 85 ++++++++++++++++++++++++++++++-------------- lib/librte_timer/rte_timer.h | 2 +- 2 files changed, 60 insertions(+), 27 deletions(-)
lib/librte_timer/rte_timer.c +59 -26
@@ -10,7 +10,6 @@#include <assert.h> #include <sys/queue.h> -#include <rte_atomic.h> #include <rte_common.h> #include <rte_cycles.h> #include <rte_eal_memconfig.h>
@@ -218,7 +217,7 @@ rte_timer_init(struct rte_timer *tim) status.state =RTE_TIMER_STOP ; status.owner =RTE_TIMER_NO_OWNER ; - tim->status.u32 = status.u32; +__atomic_store_n (&tim->status.u32, status.u32, __ATOMIC_RELAXED); }/*
@@ -239,9 +238,9 @@ timer_set_config_state(struct rte_timer *tim,/* wait that the timer is in correct status before update, * and mark it as being configured */ -while (success ==0 ) { - prev_status.u32 = tim->status.u32; + prev_status.u32 =__atomic_load_n (&tim->status.u32, __ATOMIC_RELAXED); +while (success ==0 ) {/* timer is running on another core * or ready to run on local core, exit */
@@ -258,9 +257,15 @@ timer_set_config_state(struct rte_timer *tim,* mark it atomically as being configured */ status.state =RTE_TIMER_CONFIG ; status.owner = (int16_t )lcore_id; - success =rte_atomic32_cmpset (&tim->status.u32, - prev_status.u32, - status.u32); +/* CONFIG states are acting as locked states. If the +* timer is in CONFIG state, the state cannot be changed +* by other threads. So, we should use ACQUIRE here. +*/ + success =__atomic_compare_exchange_n (&tim->status.u32, + &prev_status.u32, + status.u32,0 , + __ATOMIC_ACQUIRE, + __ATOMIC_RELAXED); } ret_prev_status->u32 = prev_status.u32;
@@ -279,20 +284,27 @@ timer_set_running_state(struct rte_timer *tim)/* wait that the timer is in correct status before update, * and mark it as running */ -while (success ==0 ) { - prev_status.u32 = tim->status.u32; + prev_status.u32 =__atomic_load_n (&tim->status.u32, __ATOMIC_RELAXED); +while (success ==0 ) {/* timer is not pending anymore */ if (prev_status.state !=RTE_TIMER_PENDING )return -1 ;/* here, we know that timer is stopped or pending, -* mark it atomically as being configured */ +* mark it atomically as being running +*/ status.state =RTE_TIMER_RUNNING ; status.owner = (int16_t )lcore_id; - success =rte_atomic32_cmpset (&tim->status.u32, - prev_status.u32, - status.u32); +/* RUNNING states are acting as locked states. If the +* timer is in RUNNING state, the state cannot be changed +* by other threads. So, we should use ACQUIRE here. +*/ + success =__atomic_compare_exchange_n (&tim->status.u32, + &prev_status.u32, + status.u32,0 , + __ATOMIC_ACQUIRE, + __ATOMIC_RELAXED); }return 0 ;
@@ -520,10 +532,12 @@ __rte_timer_reset(struct rte_timer *tim, uint64_t expire,/* update state: as we are in CONFIG state, only us can modify * the state so we don't need to use cmpset() here */ -rte_wmb (); status.state =RTE_TIMER_PENDING ; status.owner = (int16_t )tim_lcore; - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory operations above +* the status update are observed before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, __ATOMIC_RELEASE);if (tim_lcore != lcore_id || !local_is_locked)rte_spinlock_unlock (&priv_timer[tim_lcore].list_lock);
@@ -600,10 +614,12 @@ __rte_timer_stop(struct rte_timer *tim, int local_is_locked, }/* mark timer as stopped */ -rte_wmb (); status.state =RTE_TIMER_STOP ; status.owner =RTE_TIMER_NO_OWNER ; - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory operations above +* the status update are observed before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, __ATOMIC_RELEASE);return 0 ; }
@@ -637,7 +653,8 @@ rte_timer_stop_sync(struct rte_timer *tim)int rte_timer_pending (struct rte_timer *tim) { -return tim->status.state ==RTE_TIMER_PENDING ; +return __atomic_load_n (&tim->status.state, + __ATOMIC_RELAXED) ==RTE_TIMER_PENDING ; }/* must be called periodically, run all timer that expired */
@@ -739,8 +756,12 @@ __rte_timer_manage(struct rte_timer_data *timer_data)/* remove from done list and mark timer as stopped */ status.state =RTE_TIMER_STOP ; status.owner =RTE_TIMER_NO_OWNER ; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory +* operations above the status update are observed +* before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, + __ATOMIC_RELEASE); }else {/* keep it in list and mark timer as pending */
@@ -748,8 +769,12 @@ __rte_timer_manage(struct rte_timer_data *timer_data) status.state =RTE_TIMER_PENDING ;__TIMER_STAT_ADD (priv_timer, pending,1 ); status.owner = (int16_t )lcore_id; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory +* operations above the status update are observed +* before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, + __ATOMIC_RELEASE);__rte_timer_reset (tim, tim->expire + tim->period, tim->period, lcore_id, tim->f, tim->arg,1 , timer_data);
@@ -919,8 +944,12 @@ rte_timer_alt_manage(uint32_t timer_data_id,/* remove from done list and mark timer as stopped */ status.state =RTE_TIMER_STOP ; status.owner =RTE_TIMER_NO_OWNER ; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory +* operations above the status update are observed +* before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, + __ATOMIC_RELEASE); }else {/* keep it in list and mark timer as pending */ rte_spinlock_lock(
@@ -928,8 +957,12 @@ rte_timer_alt_manage(uint32_t timer_data_id, status.state =RTE_TIMER_PENDING ;__TIMER_STAT_ADD (data->priv_timer, pending,1 ); status.owner = (int16_t )this_lcore; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory +* operations above the status update are observed +* before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, + __ATOMIC_RELEASE);__rte_timer_reset (tim, tim->expire + tim->period, tim->period, this_lcore, tim->f, tim->arg,1 , data);
lib/librte_timer/rte_timer.h +1 -1
@@ -101,7 +101,7 @@ struct rte_timer {uint64_t expire;/**< Time when timer expire. */ struct rte_timer *sl_next[MAX_SKIPLIST_DEPTH ]; -volatile union rte_timer_status status;/**< Status of timer. */ +union rte_timer_status status;/**< Status of timer. */ uint64_t period;/**< Period of timer (0 if not periodic). */ rte_timer_cb_t f;/**< Callback function. */ void *arg;/**< Argument to callback function. */
Signature · 1 line
2.7.4
Re: [dpdk-dev] [PATCH v2] lib/timer: relax barrier for status update
Hi Erik,
Quoted 17 lines
Subject: [PATCH v2] lib/timer: relax barrier for status update
Volatile has no ordering semantics. The
rte_timerstructure defines timer status as a volatile variable and uses the rte_r/wmb barrier to guarantee inter-thread visibility.This patch optimized the volatile operation with c11 atomic operations and one-way barrier to save the performance penalty. According to the
timer_perf_autotestbenchmarking results, this patch can uplift 10%~16% timer appending performance, 3%~20% timer resetting performance and 45% timer callbacks scheduling performance on aarch64 and no loss in performance for x86.Suggested-by Honnappa Nagarahalli <[email protected]>Signed-off-by Phil Yang <[email protected]>Reviewed-by Gavin Hu <[email protected]>
Quoted 8 lines
---
This patch depends on patch:
http://patchwork.dpdk.org/patch/65997/v2:
1. Changed the memory ordering comment intimer_set_config_state.
2. It is still using built-ins as the wrapper functions for C11 built-ins are not defined yet.
It is too late to get the wrapper functions done for 20.05. It was decided in yesterday's tech board meeting to go ahead with C11 atomic built-ins (since there is lot of code in DPDK that uses C11 built-ins). If there are no further comments, can you please provide your ack?
Quoted 213 lines
lib/librte_timer/rte_timer.c | 85 ++++++++++++++++++++++++++++++----------- --- lib/librte_timer/rte_timer.h | 2 +- 2 files changed, 60 insertions(+), 27 deletions(-)lib/librte_timer/rte_timer.c index renamed from lib/librte_timer/rte_timer.c +0
-0269e921..ba17216 100644
lib/librte_timer/rte_timer.c +1
-2@@ -10,7 +10,6 @@#include <assert.h> #include <sys/queue.h> -#include <rte_atomic.h> #include <rte_common.h> #include <rte_cycles.h> #include <rte_eal_memconfig.h> @@ -218,7 +217,7 @@ rte_timer_init(struct rte_timer *tim) status.state =RTE_TIMER_STOP ; status.owner =RTE_TIMER_NO_OWNER ; - tim->status.u32 = status.u32; +__atomic_store_n (&tim->status.u32, status.u32, __ATOMIC_RELAXED); }/*
lib/librte_timer/rte_timer.c +2
-2@@ -239,9 +238,9 @@ timer_set_config_state(struct rte_timer *tim,/* wait that the timer is in correct status before update, * and mark it as being configured */ -while (success ==0 ) { - prev_status.u32 = tim->status.u32; + prev_status.u32 =__atomic_load_n (&tim->status.u32, __ATOMIC_RELAXED); +while (success ==0 ) {/* timer is running on another core * or ready to run on local core, exit lib/librte_timer/rte_timer.c +9
-3*/ @@ -258,9 +257,15 @@ timer_set_config_state(struct rte_timer *tim,* mark it atomically as being configured */ status.state =RTE_TIMER_CONFIG ; status.owner = (int16_t )lcore_id; - success =rte_atomic32_cmpset (&tim->status.u32, - prev_status.u32, - status.u32); +/* CONFIG states are acting as locked states. If the +* timer is in CONFIG state, the state cannot be changed +* by other threads. So, we should use ACQUIRE here. +*/ + success =__atomic_compare_exchange_n (&tim->status.u32, + &prev_status.u32, + status.u32,0 , + __ATOMIC_ACQUIRE, + __ATOMIC_RELAXED); } ret_prev_status->u32 = prev_status.u32; @@ -279,20 +284,27 @@
timer_set_running_state(struct rte_timer *tim)lib/librte_timer/rte_timer.c +1
-2/* wait that the timer is in correct status before update, * and mark it as running */ -while (success ==0 ) { - prev_status.u32 = tim->status.u32; + prev_status.u32 = __atomic_load_n(&tim->status.u32,
__ATOMIC_RELAXED);lib/librte_timer/rte_timer.c +16
-6+while (success ==0 ) {/* timer is not pending anymore */ if (prev_status.state !=RTE_TIMER_PENDING )return -1 ;/* here, we know that timer is stopped or pending, -* mark it atomically as being configured */ +* mark it atomically as being running +*/ status.state =RTE_TIMER_RUNNING ; status.owner = (int16_t )lcore_id; - success =rte_atomic32_cmpset (&tim->status.u32, - prev_status.u32, - status.u32); +/* RUNNING states are acting as locked states. If the +* timer is in RUNNING state, the state cannot be changed +* by other threads. So, we should use ACQUIRE here. +*/ + success =__atomic_compare_exchange_n (&tim->status.u32, + &prev_status.u32, + status.u32,0 , + __ATOMIC_ACQUIRE, + __ATOMIC_RELAXED); }return 0 ;@@ -520,10 +532,12 @@ __rte_timer_reset(struct rte_timer *tim, uint64_t expire,/* update state: as we are in CONFIG state, only us can modify * the state so we don't need to use cmpset() here */ -rte_wmb (); status.state =RTE_TIMER_PENDING ; status.owner = (int16_t )tim_lcore; - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory operations above +* the status update are observed before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, __ATOMIC_RELEASE);if (tim_lcore != lcore_id || !local_is_locked)lib/librte_timer/rte_timer.c +4
-2rte_spinlock_unlock (&priv_timer[tim_lcore].list_lock);@@ -600,10 +614,12 @@ __rte_timer_stop(struct rte_timer *tim, int local_is_locked, }/* mark timer as stopped */ -rte_wmb (); status.state =RTE_TIMER_STOP ; status.owner =RTE_TIMER_NO_OWNER ; - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory operations above +* the status update are observed before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, __ATOMIC_RELEASE);return 0 ;}
lib/librte_timer/rte_timer.c +3
-1@@ -637,7 +653,8 @@ rte_timer_stop_sync(struct rte_timer *tim) int rte_timer_pending(struct rte_timer *tim) { -return tim->status.state ==RTE_TIMER_PENDING ; +return __atomic_load_n (&tim->status.state, + __ATOMIC_RELAXED) ==RTE_TIMER_PENDING ; } /* must be called periodically, run all timer that expired */ @@ -739,8 +756,12 @@ __rte_timer_manage(struct rte_timer_data *timer_data)/* remove from done list and mark timer as stopped */
lib/librte_timer/rte_timer.c +6
-2status.state =RTE_TIMER_STOP ; status.owner =RTE_TIMER_NO_OWNER ; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory +* operations above the status update are observed +* before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, + __ATOMIC_RELEASE); }else { /* keep it in list and mark timer as pending */ @@ -748,8 +769,12 @@
__rte_timer_manage(struct rte_timer_data *timer_data)lib/librte_timer/rte_timer.c +12
-4status.state =RTE_TIMER_PENDING ;__TIMER_STAT_ADD (priv_timer, pending,1 ); status.owner = (int16_t )lcore_id; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory +* operations above the status update are observed +* before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, + __ATOMIC_RELEASE);__rte_timer_reset (tim, tim->expire + tim->period, tim->period, lcore_id, tim->f, tim->arg,1 , timer_data);@@ -919,8 +944,12 @@ rte_timer_alt_manage(uint32_t timer_data_id,/* remove from done list and mark timer as stopped */ status.state =RTE_TIMER_STOP ; status.owner =RTE_TIMER_NO_OWNER ; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory +* operations above the status update are observed +* before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, + __ATOMIC_RELEASE); }else {/* keep it in list and mark timer as pending */ lib/librte_timer/rte_timer.c +6
-2rte_spinlock_lock(@@ -928,8 +957,12 @@ rte_timer_alt_manage(uint32_t timer_data_id, status.state =RTE_TIMER_PENDING ;__TIMER_STAT_ADD (data->priv_timer, pending,1 ); status.owner = (int16_t )this_lcore; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory +* operations above the status update are observed +* before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, + __ATOMIC_RELEASE);__rte_timer_reset (tim, tim->expire + tim->period, tim->period, this_lcore, tim->f, tim->arg,1 , data);lib/librte_timer/rte_timer.h index renamed from lib/librte_timer/rte_timer.h +0
-0c6b3d45..df533fa 100644
lib/librte_timer/rte_timer.h +1
-2@@ -101,7 +101,7 @@ struct rte_timer {uint64_t expire;/**< Time when timer expire. */ struct rte_timer *sl_next[MAX_SKIPLIST_DEPTH ]; -volatile union rte_timer_status status;/**< Status of timer. */ +union rte_timer_status status;/**< Status of timer. */ uint64_t period;/**< Period of timer (0 if not periodic). */ rte_timer_cb_t f;/**< Callback function. */ void *arg;/**< Argument to callback function. */ --2.7.4
Carrillo, Erik G
Acked-byRe: [dpdk-dev] [PATCH v2] lib/timer: relax barrier for status update
Hi Honnappa,
Quoted 35 lines
-----Original Message-----
From: Honnappa Nagarahalli <[email protected]>
Sent: Thursday, April 23, 2020 3:06 PM
To: Phil Yang <[email protected]>; Carrillo, Erik G <[email protected]>; [email protected]; [email protected]
Cc: [email protected]; [email protected]; Ananyev, Konstantin <[email protected]>; [email protected]; [email protected]; Gavin Hu <[email protected]>; nd <[email protected]>; Honnappa Nagarahalli <[email protected]>; nd <[email protected]>
Subject: RE: [PATCH v2] lib/timer: relax barrier for status updateHi Erik,
Quoted 21 lines
Subject: [PATCH v2] lib/timer: relax barrier for status update
Volatile has no ordering semantics. The
rte_timerstructure defines timer status as a volatile variable and uses the rte_r/wmb barrier to guarantee inter-thread visibility.This patch optimized the volatile operation with c11 atomic operations and one-way barrier to save the performance penalty. According to the
timer_perf_autotestbenchmarking results, this patch can uplift 10%~16% timer appending performance, 3%~20% timer resetting performance and 45% timer callbacks scheduling performance on aarch64 and no loss in performance for x86.Suggested-by Honnappa Nagarahalli <[email protected]>Signed-off-by Phil Yang <[email protected]>Reviewed-by Gavin Hu <[email protected]>---
This patch depends on patch:
http://patchwork.dpdk.org/patch/65997/
Quoted 9 lines
Quoted 4 lines
v2:
1. Changed the memory ordering comment intimer_set_config_state.
2. It is still using built-ins as the wrapper functions for C11 built-ins are not defined yet.It is too late to get the wrapper functions done for 20.05. It was decided in yesterday's tech board meeting to go ahead with C11 atomic built-ins (since there is lot of code in DPDK that uses C11 built-ins). If there are no further comments, can you please provide your ack?
Ok, thanks for letting me know. Based on that decision, I've taken another look and done some testing and it looks good to me. I've made one comment in-line below and acked it.
<... snipped ...>
Quoted 29 lines
Quoted 29 lines
lib/librte_timer/rte_timer.c +9
-3@@ -258,9 +257,15 @@ timer_set_config_state(struct rte_timer *tim,* mark it atomically as being configured */ status.state =RTE_TIMER_CONFIG ; status.owner = (int16_t )lcore_id; - success =rte_atomic32_cmpset (&tim->status.u32, - prev_status.u32, - status.u32); +/* CONFIG states are acting as locked states. If the +* timer is in CONFIG state, the state cannot be changed +* by other threads. So, we should use ACQUIRE here. +*/ + success =__atomic_compare_exchange_n (&tim- status.u32, + &prev_status.u32, + status.u32,0 , + __ATOMIC_ACQUIRE, + __ATOMIC_RELAXED); }
ret_prev_status->u32= prev_status.u32; @@ -279,20 +284,27 @@timer_set_running_state(struct rte_timer *tim)lib/librte_timer/rte_timer.c +1
-2/* wait that the timer is in correct status before update, * and mark it as running */ -while (success ==0 ) { - prev_status.u32 = tim->status.u32; + prev_status.u32 = __atomic_load_n(&tim->status.u32,
__ATOMIC_RELAXED);
Quoted 6 lines
lib/librte_timer/rte_timer.c timer_set_running_state(struct rte_timer *tim) · line 294
Quoted 6 lines
lib/librte_timer/rte_timer.c +1
-0+while (success ==0 ) {/* timer is not pending anymore */ if (prev_status.state !=RTE_TIMER_PENDING )return -1 ;/* here, we know that timer is stopped or pending,
We know that the timer will be pending at this point... Since we're correcting the comment below, we can correct this part too.
With that change:
Quoted 20 lines
Quoted 20 lines
lib/librte_timer/rte_timer.c +7
-4-* mark it atomically as being configured */ +* mark it atomically as being running +*/ status.state =RTE_TIMER_RUNNING ; status.owner = (int16_t )lcore_id; - success =rte_atomic32_cmpset (&tim->status.u32, - prev_status.u32, - status.u32); +/* RUNNING states are acting as locked states. If the +* timer is in RUNNING state, the state cannot be changed +* by other threads. So, we should use ACQUIRE here. +*/ + success = __atomic_compare_exchange_n(&tim-status.u32,
lib/librte_timer/rte_timer.c +4
-0+ &prev_status.u32, + status.u32,0 , + __ATOMIC_ACQUIRE, + __ATOMIC_RELAXED); }
Quoted 1 line
Quoted 1 line
return 0;
Thanks,
Erik
Re: [dpdk-dev] [PATCH v2] lib/timer: relax barrier for status update
Quoted 100 lines
-----Original Message-----
From: Carrillo, Erik G <[email protected]>
Sent: Friday, April 24, 2020 9:27 AM
To: Honnappa Nagarahalli <[email protected]>; Phil Yang <[email protected]>; [email protected]; [email protected]
Cc: [email protected]; [email protected]; Ananyev, Konstantin <[email protected]>; [email protected]; [email protected]; Gavin Hu <[email protected]>; nd <[email protected]>; nd <[email protected]>
Subject: RE: [PATCH v2] lib/timer: relax barrier for status updateHi Honnappa,
Quoted 35 lines
-----Original Message-----
From: Honnappa Nagarahalli <[email protected]>
Sent: Thursday, April 23, 2020 3:06 PM
To: Phil Yang <[email protected]>; Carrillo, Erik G <[email protected]>; [email protected]; [email protected]
Cc: [email protected]; [email protected]; Ananyev, Konstantin <[email protected]>; [email protected]; [email protected]; Gavin Hu <[email protected]>; nd <[email protected]>; Honnappa Nagarahalli <[email protected]>; nd <[email protected]>
Subject: RE: [PATCH v2] lib/timer: relax barrier for status updateHi Erik,
Quoted 21 lines
Subject: [PATCH v2] lib/timer: relax barrier for status update
Volatile has no ordering semantics. The
rte_timerstructure defines timer status as a volatile variable and uses the rte_r/wmb barrier to guarantee inter-thread visibility.This patch optimized the volatile operation with c11 atomic operations and one-way barrier to save the performance penalty. According to the
timer_perf_autotestbenchmarking results, this patch can uplift 10%~16% timer appending performance, 3%~20% timer resetting performance and 45% timer callbacks scheduling performance on aarch64 and no loss in performance for x86.Suggested-by Honnappa Nagarahalli <[email protected]>Signed-off-by Phil Yang <[email protected]>Reviewed-by Gavin Hu <[email protected]>---
This patch depends on patch:
http://patchwork.dpdk.org/patch/65997/Quoted 9 lines
Quoted 4 lines
v2:
1. Changed the memory ordering comment intimer_set_config_state.
2. It is still using built-ins as the wrapper functions for C11 built-ins are not defined yet.It is too late to get the wrapper functions done for 20.05. It was decided in yesterday's tech board meeting to go ahead with C11 atomic built-ins (since there is lot of code in DPDK that uses C11 built-ins). If there are no further comments, can you please provide your ack?
Ok, thanks for letting me know. Based on that decision, I've taken another look
and done some testing and it looks good to me. I've made one comment in- line
below and acked it.<... snipped ...>
Quoted 34 lines
Quoted 34 lines
lib/librte_timer/rte_timer.c +9
-3@@ -258,9 +257,15 @@ timer_set_config_state(struct rte_timer *tim,* mark it atomically as being configured */ status.state =RTE_TIMER_CONFIG ; status.owner = (int16_t )lcore_id; - success =rte_atomic32_cmpset (&tim->status.u32, - prev_status.u32, - status.u32); +/* CONFIG states are acting as locked states. If the +* timer is in CONFIG state, the state cannot be changed +* by other threads. So, we should use ACQUIRE here. +*/ + success =__atomic_compare_exchange_n (&tim- status.u32, + &prev_status.u32, + status.u32,0 , + __ATOMIC_ACQUIRE, + __ATOMIC_RELAXED); }
ret_prev_status->u32= prev_status.u32; @@ -279,20 +284,27 @@timer_set_running_state(struct rte_timer *tim)lib/librte_timer/rte_timer.c +1
-2/* wait that the timer is in correct status before update, * and mark it as running */ -while (success ==0 ) { - prev_status.u32 = tim->status.u32; + prev_status.u32 = __atomic_load_n(&tim->status.u32,
__ATOMIC_RELAXED);lib/librte_timer/rte_timer.c +1
-0+while (success ==0 ) {/* timer is not pending anymore */ if (prev_status.state !=RTE_TIMER_PENDING )return -1 ;
Quoted 7 lines
Quoted 1 line
Quoted 1 line
/* here, we know that timer is stopped or pending,
We know that the timer will be pending at this point... Since we're correcting the comment below, we can correct this part too.
With that change:
Thanks Erik.
Updated in v3.
Thanks,
Phil
Quoted 25 lines
Quoted 21 lines
Quoted 21 lines
lib/librte_timer/rte_timer.c +7
-4-* mark it atomically as being configured */ +* mark it atomically as being running +*/ status.state =RTE_TIMER_RUNNING ; status.owner = (int16_t )lcore_id; - success =rte_atomic32_cmpset (&tim->status.u32, - prev_status.u32, - status.u32); +/* RUNNING states are acting as locked states. If the +* timer is in RUNNING state, the state cannot be changed +* by other threads. So, we should use ACQUIRE here. +*/ + success = __atomic_compare_exchange_n(&tim-status.u32,
lib/librte_timer/rte_timer.c +4
-0+ &prev_status.u32, + status.u32,0 , + __ATOMIC_ACQUIRE, + __ATOMIC_RELAXED); }Quoted 1 line
Quoted 1 line
return 0;
Thanks,
Erik
PATCH v3 1/1 lib/timer: relax barrier for status update
In reply to Phil YangVolatile has no ordering semantics. The rte_timer structure defines timer status as a volatile variable and uses the rte_r/wmb barrier to guarantee inter-thread visibility.
This patch optimized the volatile operation with c11 atomic operations and one-way barrier to save the performance penalty. According to the timer_perf_autotest benchmarking results, this patch can uplift 10%~16% timer appending performance, 3%~20% timer resetting performance and 45% timer callbacks scheduling performance on aarch64 and no loss in performance for x86.
---
This patch depends on patch:
http://patchwork.dpdk.org/patch/65997/
It is still using built-ins as the wrapper functions for C11 built-ins are not defined yet
v3: fix typo.
v2:
1. Changed the memory ordering comment in timer_set_config_state.
lib/librte_timer/rte_timer.c | 87 ++++++++++++++++++++++++++++++-------------- lib/librte_timer/rte_timer.h | 2 +- 2 files changed, 61 insertions(+), 28 deletions(-)
lib/librte_timer/rte_timer.c +60 -27
@@ -10,7 +10,6 @@#include <assert.h> #include <sys/queue.h> -#include <rte_atomic.h> #include <rte_common.h> #include <rte_cycles.h> #include <rte_eal_memconfig.h>
@@ -218,7 +217,7 @@ rte_timer_init(struct rte_timer *tim) status.state =RTE_TIMER_STOP ; status.owner =RTE_TIMER_NO_OWNER ; - tim->status.u32 = status.u32; +__atomic_store_n (&tim->status.u32, status.u32, __ATOMIC_RELAXED); }/*
@@ -239,9 +238,9 @@ timer_set_config_state(struct rte_timer *tim,/* wait that the timer is in correct status before update, * and mark it as being configured */ -while (success ==0 ) { - prev_status.u32 = tim->status.u32; + prev_status.u32 =__atomic_load_n (&tim->status.u32, __ATOMIC_RELAXED); +while (success ==0 ) {/* timer is running on another core * or ready to run on local core, exit */
@@ -258,9 +257,15 @@ timer_set_config_state(struct rte_timer *tim,* mark it atomically as being configured */ status.state =RTE_TIMER_CONFIG ; status.owner = (int16_t )lcore_id; - success =rte_atomic32_cmpset (&tim->status.u32, - prev_status.u32, - status.u32); +/* CONFIG states are acting as locked states. If the +* timer is in CONFIG state, the state cannot be changed +* by other threads. So, we should use ACQUIRE here. +*/ + success =__atomic_compare_exchange_n (&tim->status.u32, + &prev_status.u32, + status.u32,0 , + __ATOMIC_ACQUIRE, + __ATOMIC_RELAXED); } ret_prev_status->u32 = prev_status.u32;
@@ -279,20 +284,27 @@ timer_set_running_state(struct rte_timer *tim)/* wait that the timer is in correct status before update, * and mark it as running */ -while (success ==0 ) { - prev_status.u32 = tim->status.u32; + prev_status.u32 =__atomic_load_n (&tim->status.u32, __ATOMIC_RELAXED); +while (success ==0 ) {/* timer is not pending anymore */ if (prev_status.state !=RTE_TIMER_PENDING )return -1 ; -/* here, we know that timer is stopped or pending, -* mark it atomically as being configured */ +/* we know that the timer will be pending at this point, +* mark it atomically as being running +*/ status.state =RTE_TIMER_RUNNING ; status.owner = (int16_t )lcore_id; - success =rte_atomic32_cmpset (&tim->status.u32, - prev_status.u32, - status.u32); +/* RUNNING states are acting as locked states. If the +* timer is in RUNNING state, the state cannot be changed +* by other threads. So, we should use ACQUIRE here. +*/ + success =__atomic_compare_exchange_n (&tim->status.u32, + &prev_status.u32, + status.u32,0 , + __ATOMIC_ACQUIRE, + __ATOMIC_RELAXED); }return 0 ;
@@ -520,10 +532,12 @@ __rte_timer_reset(struct rte_timer *tim, uint64_t expire,/* update state: as we are in CONFIG state, only us can modify * the state so we don't need to use cmpset() here */ -rte_wmb (); status.state =RTE_TIMER_PENDING ; status.owner = (int16_t )tim_lcore; - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory operations above +* the status update are observed before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, __ATOMIC_RELEASE);if (tim_lcore != lcore_id || !local_is_locked)rte_spinlock_unlock (&priv_timer[tim_lcore].list_lock);
@@ -600,10 +614,12 @@ __rte_timer_stop(struct rte_timer *tim, int local_is_locked, }/* mark timer as stopped */ -rte_wmb (); status.state =RTE_TIMER_STOP ; status.owner =RTE_TIMER_NO_OWNER ; - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory operations above +* the status update are observed before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, __ATOMIC_RELEASE);return 0 ; }
@@ -637,7 +653,8 @@ rte_timer_stop_sync(struct rte_timer *tim)int rte_timer_pending (struct rte_timer *tim) { -return tim->status.state ==RTE_TIMER_PENDING ; +return __atomic_load_n (&tim->status.state, + __ATOMIC_RELAXED) ==RTE_TIMER_PENDING ; }/* must be called periodically, run all timer that expired */
@@ -739,8 +756,12 @@ __rte_timer_manage(struct rte_timer_data *timer_data)/* remove from done list and mark timer as stopped */ status.state =RTE_TIMER_STOP ; status.owner =RTE_TIMER_NO_OWNER ; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory +* operations above the status update are observed +* before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, + __ATOMIC_RELEASE); }else {/* keep it in list and mark timer as pending */
@@ -748,8 +769,12 @@ __rte_timer_manage(struct rte_timer_data *timer_data) status.state =RTE_TIMER_PENDING ;__TIMER_STAT_ADD (priv_timer, pending,1 ); status.owner = (int16_t )lcore_id; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory +* operations above the status update are observed +* before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, + __ATOMIC_RELEASE);__rte_timer_reset (tim, tim->expire + tim->period, tim->period, lcore_id, tim->f, tim->arg,1 , timer_data);
@@ -919,8 +944,12 @@ rte_timer_alt_manage(uint32_t timer_data_id,/* remove from done list and mark timer as stopped */ status.state =RTE_TIMER_STOP ; status.owner =RTE_TIMER_NO_OWNER ; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory +* operations above the status update are observed +* before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, + __ATOMIC_RELEASE); }else {/* keep it in list and mark timer as pending */ rte_spinlock_lock(
@@ -928,8 +957,12 @@ rte_timer_alt_manage(uint32_t timer_data_id, status.state =RTE_TIMER_PENDING ;__TIMER_STAT_ADD (data->priv_timer, pending,1 ); status.owner = (int16_t )this_lcore; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory +* operations above the status update are observed +* before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, + __ATOMIC_RELEASE);__rte_timer_reset (tim, tim->expire + tim->period, tim->period, this_lcore, tim->f, tim->arg,1 , data);
lib/librte_timer/rte_timer.h +1 -1
@@ -101,7 +101,7 @@ struct rte_timer {uint64_t expire;/**< Time when timer expire. */ struct rte_timer *sl_next[MAX_SKIPLIST_DEPTH ]; -volatile union rte_timer_status status;/**< Status of timer. */ +union rte_timer_status status;/**< Status of timer. */ uint64_t period;/**< Period of timer (0 if not periodic). */ rte_timer_cb_t f;/**< Callback function. */ void *arg;/**< Argument to callback function. */
Signature · 1 line
2.7.4
Re: [dpdk-dev] [PATCH v3] lib/timer: relax barrier for status update
24/04/2020 09:24, Phil Yang:
Quoted 15 lines
Volatile has no ordering semantics. The
rte_timerstructure defines timer status as a volatile variable and uses the rte_r/wmb barrier to guarantee inter-thread visibility.This patch optimized the volatile operation with c11 atomic operations and one-way barrier to save the performance penalty. According to the
timer_perf_autotestbenchmarking results, this patch can uplift 10%~16% timer appending performance, 3%~20% timer resetting performance and 45% timer callbacks scheduling performance on aarch64 and no loss in performance for x86.Suggested-by Honnappa Nagarahalli <[email protected]>Signed-off-by Phil Yang <[email protected]>Reviewed-by Gavin Hu <[email protected]>
[...]
lib/librte_timer/rte_timer.h struct rte_timer · line 101
Quoted 5 lines
lib/librte_timer/rte_timer.h +1
-1@@ -101,7 +101,7 @@ struct rte_timer -volatile union rte_timer_status status;/**< Status of timer. */ +union rte_timer_status status;/**< Status of timer. */
Unfortunately, I cannot merge this patch because it breaks the ABI:
[C]'function void rte_timer_init(rte_timer*)' at rte_timer.c:214:1 has some indirect sub-type changes:
parameter 1 of type 'rte_timer*' has sub-type changes:
in pointed to type 'struct rte_timer' at rte_timer.h:100:1:
type size hasn't changed
1 data member changes (2 filtered):
type of 'volatile rte_timer_status rte_timer::status' changed:
entity changed from 'volatile rte_timer_status' to 'union rte_timer_status' at rte_timer.h:67:1 type size hasn't changed
Re: [dpdk-dev] [PATCH v3] lib/timer: relax barrier for status update
Quoted 36 lines
-----Original Message-----
From: Thomas Monjalon <[email protected]>
Sent: Sunday, April 26, 2020 1:18 AM
To: Phil Yang <[email protected]>
Cc: [email protected]; [email protected]; [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu <[email protected]>; nd <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v3] lib/timer: relax barrier for status update24/04/2020 09:24, Phil Yang:
Quoted 8 lines
Volatile has no ordering semantics. The
rte_timerstructure defines timer status as a volatile variable and uses the rte_r/wmb barrier to guarantee inter-thread visibility.This patch optimized the volatile operation with c11 atomic operations and one-way barrier to save the performance penalty. According to the
timer_perf_autotestbenchmarking results, this patch can uplift 10%~16% timer appending performance, 3%~20% timer resetting performance and45%
Quoted 7 lines
timer callbacks scheduling performance on aarch64 and no loss in performance for x86.
Suggested-by Honnappa Nagarahalli <[email protected]>Signed-off-by Phil Yang <[email protected]>Reviewed-by Gavin Hu <[email protected]>[...]
lib/librte_timer/rte_timer.h struct rte_timer · line 101
Quoted 5 lines
lib/librte_timer/rte_timer.h +1
-1@@ -101,7 +101,7 @@ struct rte_timer -volatile union rte_timer_status status;/**< Status of timer. */ +union rte_timer_status status;/**< Status of timer. */ Unfortunately, I cannot merge this patch because it breaks the ABI:
Quoted 11 lines
[C]'function void
rte_timer_init(rte_timer*)' at rte_timer.c:214:1 has some
indirect sub-type changes:
parameter 1 of type 'rte_timer*' has sub-type changes:
in pointed to type 'structrte_timer' at rte_timer.h:100:1:
type size hasn't changed
1 data member changes (2 filtered):
type of 'volatilerte_timer_statusrte_timer::status' changed:
entity changed from 'volatilerte_timer_status' to 'unionrte_timer_status' at rte_timer.h:67:1
type size hasn't changed
I think we can revert it to the original definition of rte_timer and keep the union rte_timer_status volatile-qualified. Because with or without this 'volatile' qualify, it generates the same code on aarch64 and x86.
So it seems acceptable to ignore it to make the ABI compatible?
Thank,
Phil
Re: [dpdk-dev] [PATCH v3] lib/timer: relax barrier for status update
Quoted 62 lines
-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Sunday, April 26, 2020 2:36 AM
To: [email protected]
Cc: Carrillo, Erik G <[email protected]>; [email protected]; [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu <[email protected]>; nd <[email protected]>; nd <[email protected]>
Subject: RE: [dpdk-dev] [PATCH v3] lib/timer: relax barrier for status updateQuoted 38 lines
-----Original Message-----
From: Thomas Monjalon <[email protected]>
Sent: Sunday, April 26, 2020 1:18 AM
To: Phil Yang <[email protected]>
Cc: [email protected]; [email protected]; [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu <[email protected]>; nd <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v3] lib/timer: relax barrier for status update24/04/2020 09:24, Phil Yang:
Quoted 9 lines
Volatile has no ordering semantics. The
rte_timerstructure defines timer status as a volatile variable and uses the rte_r/wmb barrier to guarantee inter-thread visibility.This patch optimized the volatile operation with c11 atomic operations and one-way barrier to save the performance penalty. According to the
timer_perf_autotestbenchmarking results, this patch can uplift 10%~16% timer appending performance, 3%~20% timer resetting performance and45%
Quoted 7 lines
timer callbacks scheduling performance on aarch64 and no loss in performance for x86.
Suggested-by Honnappa Nagarahalli <[email protected]>Signed-off-by Phil Yang <[email protected]>Reviewed-by Gavin Hu <[email protected]>[...]
lib/librte_timer/rte_timer.h struct rte_timer · line 101
Quoted 5 lines
lib/librte_timer/rte_timer.h +1
-1@@ -101,7 +101,7 @@ struct rte_timer -volatile union rte_timer_status status;/**< Status of timer. */ +union rte_timer_status status;/**< Status of timer. */ Unfortunately, I cannot merge this patch because it breaks the ABI:
Quoted 11 lines
[C]'function void
rte_timer_init(rte_timer*)' at rte_timer.c:214:1
has some indirect sub-type changes:
parameter 1 of type 'rte_timer*' has sub-type changes:
in pointed to type 'structrte_timer' at rte_timer.h:100:1:
type size hasn't changed
1 data member changes (2 filtered):
type of 'volatilerte_timer_statusrte_timer::status' changed:
entity changed from 'volatilerte_timer_status' to 'unionrte_timer_status' at rte_timer.h:67:1
type size hasn't changed
Quoted 8 lines
I think we can revert it to the original definition of
rte_timerand keep the unionrte_timer_statusvolatile-qualified.
Because with or without this 'volatile' qualify, it generates the same code on aarch64 and x86.
So it seems acceptable to ignore it to make the ABI compatible?Thank,
Phil
I was wondering about this also. Is the performance improvement on aarch64 still the same in that case?
Thanks,
Erik
Re: [dpdk-dev] [PATCH v3] lib/timer: relax barrier for status update
Quoted 76 lines
-----Original Message-----
From: Carrillo, Erik G <[email protected]>
Sent: Sunday, April 26, 2020 8:19 PM
To: Phil Yang <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu <[email protected]>; nd <[email protected]>; nd <[email protected]>
Subject: RE: [dpdk-dev] [PATCH v3] lib/timer: relax barrier for status updateQuoted 9 lines
-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Sunday, April 26, 2020 2:36 AM
To: [email protected]
Cc: Carrillo, Erik G <[email protected]>; [email protected]; [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu <[email protected]>; nd <[email protected]>; nd <[email protected]>
Subject: RE: [dpdk-dev] [PATCH v3] lib/timer: relax barrier for statusupdate
Quoted 51 lines
Quoted 39 lines
-----Original Message-----
From: Thomas Monjalon <[email protected]>
Sent: Sunday, April 26, 2020 1:18 AM
To: Phil Yang <[email protected]>
Cc: [email protected]; [email protected]; [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu <[email protected]>; nd <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v3] lib/timer: relax barrier for status update24/04/2020 09:24, Phil Yang:
Quoted 9 lines
Volatile has no ordering semantics. The
rte_timerstructure defines timer status as a volatile variable and uses the rte_r/wmb barrier to guarantee inter-thread visibility.This patch optimized the volatile operation with c11 atomic operations and one-way barrier to save the performance penalty. According to the
timer_perf_autotestbenchmarking results, this patch can uplift 10%~16% timer appending performance, 3%~20% timer resetting performance and45%
Quoted 7 lines
timer callbacks scheduling performance on aarch64 and no loss in performance for x86.
Suggested-by Honnappa Nagarahalli <[email protected]>Signed-off-by Phil Yang <[email protected]>Reviewed-by Gavin Hu <[email protected]>[...]
lib/librte_timer/rte_timer.h struct rte_timer · line 101
Quoted 5 lines
lib/librte_timer/rte_timer.h +1
-1@@ -101,7 +101,7 @@ struct rte_timer -volatile union rte_timer_status status;/**< Status of timer. */ +union rte_timer_status status;/**< Status of timer. */ Unfortunately, I cannot merge this patch because it breaks the ABI:
Quoted 11 lines
[C]'function void
rte_timer_init(rte_timer*)' at rte_timer.c:214:1
has some indirect sub-type changes:
parameter 1 of type 'rte_timer*' has sub-type changes:
in pointed to type 'structrte_timer' at rte_timer.h:100:1:
type size hasn't changed
1 data member changes (2 filtered):
type of 'volatilerte_timer_statusrte_timer::status' changed:
entity changed from 'volatilerte_timer_status' to 'unionrte_timer_status' at rte_timer.h:67:1
type size hasn't changedQuoted 3 lines
I think we can revert it to the original definition of
rte_timerand keep the unionrte_timer_statusvolatile-qualified.
Because with or without this 'volatile' qualify, it generates the same codeon
Quoted 8 lines
Quoted 5 lines
aarch64 and x86.
So it seems acceptable to ignore it to make the ABI compatible?Thank,
PhilI was wondering about this also. Is the performance improvement on aarch64 still the same in that case?
Yes. it is.
It got the same performance improvement on aarch64 and no performance loss on x86.
I will update it in v4.
Re: [dpdk-dev] [PATCH v3] lib/timer: relax barrier for status update
Quoted 100 lines
-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Sunday, April 26, 2020 9:20 AM
To: Carrillo, Erik G <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu <[email protected]>; nd <[email protected]>; nd <[email protected]>; nd <[email protected]>
Subject: RE: [dpdk-dev] [PATCH v3] lib/timer: relax barrier for status updateQuoted 22 lines
-----Original Message-----
From: Carrillo, Erik G <[email protected]>
Sent: Sunday, April 26, 2020 8:19 PM
To: Phil Yang <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu <[email protected]>; nd <[email protected]>; nd <[email protected]>
Subject: RE: [dpdk-dev] [PATCH v3] lib/timer: relax barrier for status updateQuoted 10 lines
-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Sunday, April 26, 2020 2:36 AM
To: [email protected]
Cc: Carrillo, Erik G <[email protected]>; [email protected]; [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu <[email protected]>; nd <[email protected]>; nd <[email protected]>
Subject: RE: [dpdk-dev] [PATCH v3] lib/timer: relax barrier for statusQuoted 9 lines
update
Quoted 8 lines
Quoted 8 lines
-----Original Message-----
From: Thomas Monjalon <[email protected]>
Sent: Sunday, April 26, 2020 1:18 AM
To: Phil Yang <[email protected]>
Cc: [email protected]; [email protected]; [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu <[email protected]>;nd
Quoted 23 lines
Quoted 23 lines
Quoted 23 lines
<[email protected]>
Subject: Re: [dpdk-dev] [PATCH v3] lib/timer: relax barrier for status update24/04/2020 09:24, Phil Yang:
Quoted 9 lines
Volatile has no ordering semantics. The
rte_timerstructure defines timer status as a volatile variable and uses the rte_r/wmb barrier to guarantee inter-thread visibility.This patch optimized the volatile operation with c11 atomic operations and one-way barrier to save the performance penalty. According to the
timer_perf_autotestbenchmarking results, this patch can uplift 10%~16% timer appending performance, 3%~20% timer resetting performance and45%
Quoted 8 lines
timer callbacks scheduling performance on aarch64 and no loss in performance for x86.
Suggested-by: Honnappa Nagarahalli
<[email protected]>Signed-off-by Phil Yang <[email protected]>Reviewed-by Gavin Hu <[email protected]>Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
[...]
lib/librte_timer/rte_timer.h struct rte_timer · line 101
Quoted 4 lines
lib/librte_timer/rte_timer.h +0
-1@@ -101,7 +101,7 @@ struct rte_timer -volatile union rte_timer_status status;/**< Status of timer. */
Quoted 27 lines
Quoted 16 lines
Quoted 4 lines
lib/librte_timer/rte_timer.h struct rte_timer · line 104
Quoted 1 line
lib/librte_timer/rte_timer.h +1
-0+union rte_timer_status status;/**< Status of timer. */ Unfortunately, I cannot merge this patch because it breaks the ABI:
Quoted 11 lines
[C]'function void
rte_timer_init(rte_timer*)' at
rte_timer.c:214:1 has some indirect sub-type changes:
parameter 1 of type 'rte_timer*' has sub-type changes:
in pointed to type 'structrte_timer' at rte_timer.h:100:1:
type size hasn't changed
1 data member changes (2 filtered):
type of 'volatilerte_timer_statusrte_timer::status' changed:
entity changed from 'volatilerte_timer_status' to
'unionrte_timer_status' at rte_timer.h:67:1
type size hasn't changedQuoted 4 lines
I think we can revert it to the original definition of
rte_timerand keep the unionrte_timer_statusvolatile-qualified. Because with or without this 'volatile' qualify, it generates the same codeon
Quoted 5 lines
aarch64 and x86.
So it seems acceptable to ignore it to make the ABI compatible?Thank,
Phil
Quoted 8 lines
Quoted 2 lines
I was wondering about this also. Is the performance improvement on aarch64 still the same in that case?
Yes. it is.
It got the same performance improvement on aarch64 and no performance loss on x86.I will update it in v4.
Great - thanks, Phil.
PATCH v4 1/1 lib/timer: relax barrier for status update
In reply to Phil YangVolatile has no ordering semantics. The rte_timer structure defines timer status as a volatile variable and uses the rte_r/wmb barrier to guarantee inter-thread visibility.
This patch optimized the volatile operation with c11 atomic operations and one-way barrier to save the performance penalty. According to the timer_perf_autotest benchmarking results, this patch can uplift 10%~16% timer appending performance, 3%~20% timer resetting performance and 45% timer callbacks scheduling performance on aarch64 and no loss in performance for x86.
---
It is still using built-ins as the wrapper functions for C11 built-ins are not defined yet
v4:
fix ABI break in rte_timer struct.
v3:
fix typo.
v2:
Changed the memory ordering comment in timer_set_config_state.
lib/librte_timer/rte_timer.c | 87 ++++++++++++++++++++++++++++++-------------- 1 file changed, 60 insertions(+), 27 deletions(-)
lib/librte_timer/rte_timer.c +60 -27
@@ -10,7 +10,6 @@#include <assert.h> #include <sys/queue.h> -#include <rte_atomic.h> #include <rte_common.h> #include <rte_cycles.h> #include <rte_eal_memconfig.h>
@@ -218,7 +217,7 @@ rte_timer_init(struct rte_timer *tim) status.state =RTE_TIMER_STOP ; status.owner =RTE_TIMER_NO_OWNER ; - tim->status.u32 = status.u32; +__atomic_store_n (&tim->status.u32, status.u32, __ATOMIC_RELAXED); }/*
@@ -239,9 +238,9 @@ timer_set_config_state(struct rte_timer *tim,/* wait that the timer is in correct status before update, * and mark it as being configured */ -while (success ==0 ) { - prev_status.u32 = tim->status.u32; + prev_status.u32 =__atomic_load_n (&tim->status.u32, __ATOMIC_RELAXED); +while (success ==0 ) {/* timer is running on another core * or ready to run on local core, exit */
@@ -258,9 +257,15 @@ timer_set_config_state(struct rte_timer *tim,* mark it atomically as being configured */ status.state =RTE_TIMER_CONFIG ; status.owner = (int16_t )lcore_id; - success =rte_atomic32_cmpset (&tim->status.u32, - prev_status.u32, - status.u32); +/* CONFIG states are acting as locked states. If the +* timer is in CONFIG state, the state cannot be changed +* by other threads. So, we should use ACQUIRE here. +*/ + success =__atomic_compare_exchange_n (&tim->status.u32, + &prev_status.u32, + status.u32,0 , + __ATOMIC_ACQUIRE, + __ATOMIC_RELAXED); } ret_prev_status->u32 = prev_status.u32;
@@ -279,20 +284,27 @@ timer_set_running_state(struct rte_timer *tim)/* wait that the timer is in correct status before update, * and mark it as running */ -while (success ==0 ) { - prev_status.u32 = tim->status.u32; + prev_status.u32 =__atomic_load_n (&tim->status.u32, __ATOMIC_RELAXED); +while (success ==0 ) {/* timer is not pending anymore */ if (prev_status.state !=RTE_TIMER_PENDING )return -1 ; -/* here, we know that timer is stopped or pending, -* mark it atomically as being configured */ +/* we know that the timer will be pending at this point +* mark it atomically as being running +*/ status.state =RTE_TIMER_RUNNING ; status.owner = (int16_t )lcore_id; - success =rte_atomic32_cmpset (&tim->status.u32, - prev_status.u32, - status.u32); +/* RUNNING states are acting as locked states. If the +* timer is in RUNNING state, the state cannot be changed +* by other threads. So, we should use ACQUIRE here. +*/ + success =__atomic_compare_exchange_n (&tim->status.u32, + &prev_status.u32, + status.u32,0 , + __ATOMIC_ACQUIRE, + __ATOMIC_RELAXED); }return 0 ;
@@ -520,10 +532,12 @@ __rte_timer_reset(struct rte_timer *tim, uint64_t expire,/* update state: as we are in CONFIG state, only us can modify * the state so we don't need to use cmpset() here */ -rte_wmb (); status.state =RTE_TIMER_PENDING ; status.owner = (int16_t )tim_lcore; - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory operations above +* the status update are observed before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, __ATOMIC_RELEASE);if (tim_lcore != lcore_id || !local_is_locked)rte_spinlock_unlock (&priv_timer[tim_lcore].list_lock);
@@ -600,10 +614,12 @@ __rte_timer_stop(struct rte_timer *tim, int local_is_locked, }/* mark timer as stopped */ -rte_wmb (); status.state =RTE_TIMER_STOP ; status.owner =RTE_TIMER_NO_OWNER ; - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory operations above +* the status update are observed before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, __ATOMIC_RELEASE);return 0 ; }
@@ -637,7 +653,8 @@ rte_timer_stop_sync(struct rte_timer *tim)int rte_timer_pending (struct rte_timer *tim) { -return tim->status.state ==RTE_TIMER_PENDING ; +return __atomic_load_n (&tim->status.state, + __ATOMIC_RELAXED) ==RTE_TIMER_PENDING ; }/* must be called periodically, run all timer that expired */
@@ -739,8 +756,12 @@ __rte_timer_manage(struct rte_timer_data *timer_data)/* remove from done list and mark timer as stopped */ status.state =RTE_TIMER_STOP ; status.owner =RTE_TIMER_NO_OWNER ; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory +* operations above the status update are observed +* before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, + __ATOMIC_RELEASE); }else {/* keep it in list and mark timer as pending */
@@ -748,8 +769,12 @@ __rte_timer_manage(struct rte_timer_data *timer_data) status.state =RTE_TIMER_PENDING ;__TIMER_STAT_ADD (priv_timer, pending,1 ); status.owner = (int16_t )lcore_id; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory +* operations above the status update are observed +* before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, + __ATOMIC_RELEASE);__rte_timer_reset (tim, tim->expire + tim->period, tim->period, lcore_id, tim->f, tim->arg,1 , timer_data);
@@ -919,8 +944,12 @@ rte_timer_alt_manage(uint32_t timer_data_id,/* remove from done list and mark timer as stopped */ status.state =RTE_TIMER_STOP ; status.owner =RTE_TIMER_NO_OWNER ; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory +* operations above the status update are observed +* before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, + __ATOMIC_RELEASE); }else {/* keep it in list and mark timer as pending */ rte_spinlock_lock(
@@ -928,8 +957,12 @@ rte_timer_alt_manage(uint32_t timer_data_id, status.state =RTE_TIMER_PENDING ;__TIMER_STAT_ADD (data->priv_timer, pending,1 ); status.owner = (int16_t )this_lcore; -rte_wmb (); - tim->status.u32 = status.u32; +/* The "RELEASE" ordering guarantees the memory +* operations above the status update are observed +* before the update by all threads +*/ +__atomic_store_n (&tim->status.u32, status.u32, + __ATOMIC_RELEASE);__rte_timer_reset (tim, tim->expire + tim->period, tim->period, this_lcore, tim->f, tim->arg,1 , data);
Signature · 1 line
2.7.4
Thomas Monjalon
AppliedRe: [dpdk-dev] [PATCH v4] lib/timer: relax barrier for status update
26/04/2020 16:45, Phil Yang:
Quoted 11 lines
Volatile has no ordering semantics. The
rte_timerstructure defines timer status as a volatile variable and uses the rte_r/wmb barrier to guarantee inter-thread visibility.This patch optimized the volatile operation with c11 atomic operations and one-way barrier to save the performance penalty. According to the
timer_perf_autotestbenchmarking results, this patch can uplift 10%~16% timer appending performance, 3%~20% timer resetting performance and 45% timer callbacks scheduling performance on aarch64 and no loss in performance for x86.
Quoted 4 lines
Suggested-by Honnappa Nagarahalli <[email protected]>Signed-off-by Phil Yang <[email protected]>Reviewed-by Gavin Hu <[email protected]>
Applied, thanks
Re: [dpdk-dev] [PATCH v2] lib/timer: relax barrier for status update
In reply to Phil Yang20/04/2020 18:05, Phil Yang:
Quoted 2 lines
This patch depends on patch:
http://patchwork.dpdk.org/patch/65997/
In order to ease patch tracking, you should have kept the first patch in the next version of your series. We don't split series in general.
Re: [dpdk-dev] [PATCH v2] lib/timer: relax barrier for status update
Quoted 11 lines
-----Original Message-----
From: Thomas Monjalon <[email protected]>
Sent: Saturday, April 25, 2020 10:36 PM
To: Phil Yang <[email protected]>
Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu <[email protected]>; nd <[email protected]>; [email protected]
Subject: Re: [dpdk-dev] [PATCH v2] lib/timer: relax barrier for status update
Quoted 6 lines
20/04/2020 18:05, Phil Yang:
Quoted 2 lines
This patch depends on patch:
http://patchwork.dpdk.org/patch/65997/In order to ease patch tracking, you should have kept the first patch in the next version of your series. We don't split series in general.
Thanks for reminding me.
I will update the patch series in the new version.
Thanks,
Phil
Quoted 2 lines
Re: [dpdk-dev] [PATCH v2] lib/timer: relax barrier for status update
25/04/2020 17:51, Phil Yang:
Quoted 10 lines
From: Thomas Monjalon <[email protected]>
Quoted 6 lines
20/04/2020 18:05, Phil Yang:
Quoted 2 lines
This patch depends on patch:
http://patchwork.dpdk.org/patch/65997/In order to ease patch tracking, you should have kept the first patch in the next version of your series. We don't split series in general.
Thanks for reminding me.
I will update the patch series in the new version.
No that's fine, I'm merging it already.
Next time :-)
Quoted 19 lines
-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Monday, February 24, 2020 12:42 AM
To: [email protected]; Carrillo, Erik G <[email protected]>; [email protected]
Cc: [email protected]; Burakov, Anatoly <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; [email protected]
Subject: [PATCH 1/2] lib/timer: protect timer subsystem initialized with lockFrom: Honnappa Nagarahalli <[email protected]>
rte_timer_subsystem_initializedis a global variable that can be accessed by multiple processes simultaneously. Hence, any access torte_timer_subsystem_initializedshould be protected byrte_mcfg_timer_lock.
Quoted 6 lines
Fixes f9d6cd8bfe9e ("timer: fix resource leak in finalize")Signed-off-by Honnappa Nagarahalli <[email protected]>Reviewed-by Gavin Hu <[email protected]>Reviewed-by Phil Yang <[email protected]>
Quoted 5 lines
Quoted 5 lines
rte_timer_subsystem_initializedis a global variable that can be accessed by multiple processes simultaneously. Hence, any access torte_timer_subsystem_initializedshould be protected byrte_mcfg_timer_lock.
Quoted 7 lines
Quoted 6 lines
Fixes f9d6cd8bfe9e ("timer: fix resource leak in finalize")Signed-off-by Honnappa Nagarahalli <[email protected]>Reviewed-by Gavin Hu <[email protected]>Reviewed-by Phil Yang <[email protected]>
Applied (without patch 2), thanks.