Skip to contentSkip to conversation

use rte optional stdatomic API

PATCH Superseded Tyler Retzlaff 9 participants 85 messages Oct 16, 2023 – Oct 30, 2023 Network drivers

lore
Share & tools
lore
#0 lore raw

cover use rte optional stdatomic API

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API.

Note the first 2 patches are redundant and will be removed before merge. The series is a work in progress and is being submitted to get initial CI results.

Tyler Retzlaff (21):
power: fix use of rte stdatomic
event/cnxk: remove single use of rte stdatomic
power: use rte optional stdatomic API
bbdev: use rte optional stdatomic API
eal: use rte optional stdatomic API
eventdev: use rte optional stdatomic API
gpudev: use rte optional stdatomic API
ipsec: use rte optional stdatomic API
mbuf: use rte optional stdatomic API
mempool: use rte optional stdatomic API
rcu: use rte optional stdatomic API
pdump: use rte optional stdatomic API
stack: use rte optional stdatomic API
telemetry: use rte optional stdatomic API
vhost: use rte optional stdatomic API
cryptodev: use rte optional stdatomic API
distributor: use rte optional stdatomic API
ethdev: use rte optional stdatomic API
hash: use rte optional stdatomic API
timer: use rte optional stdatomic API
ring: use rte optional stdatomic API

 drivers/event/cnxk/cnxk_tim_worker.c   |   2 +-
 drivers/event/cnxk/cnxk_tim_worker.h   |   4 +-
 drivers/net/mlx5/mlx5_hws_cnt.h        |   2 +-
 lib/bbdev/rte_bbdev.c                  |   6 +-
 lib/bbdev/rte_bbdev.h                  |   2 +-
 lib/cryptodev/rte_cryptodev.c          |  22 +++---
 lib/cryptodev/rte_cryptodev.h          |  16 ++---
 lib/distributor/distributor_private.h  |   4 +-
 lib/distributor/rte_distributor.c      |  54 +++++++--------
 lib/eal/common/eal_common_launch.c     |  10 +--
 lib/eal/common/eal_common_mcfg.c       |   2 +-
 lib/eal/common/eal_common_proc.c       |  14 ++--
 lib/eal/common/eal_common_thread.c     |  26 +++----
 lib/eal/common/eal_common_trace.c      |   8 +--
 lib/eal/common/eal_common_trace_ctf.c  |   4 +-
 lib/eal/common/eal_memcfg.h            |   2 +-
 lib/eal/common/eal_private.h           |   4 +-
 lib/eal/common/eal_trace.h             |   4 +-
 lib/eal/common/rte_service.c           | 122 ++++++++++++++++-----------------
 lib/eal/freebsd/eal.c                  |  20 +++---
 lib/eal/include/rte_epoll.h            |   3 +-
 lib/eal/linux/eal.c                    |  26 +++----
 lib/eal/linux/eal_interrupts.c         |  42 ++++++------
 lib/eal/ppc/include/rte_atomic.h       |   6 +-
 lib/eal/windows/rte_thread.c           |   8 ++-
 lib/ethdev/ethdev_driver.h             |  16 ++---
 lib/ethdev/ethdev_private.c            |   6 +-
 lib/ethdev/rte_ethdev.c                |  24 +++----
 lib/ethdev/rte_ethdev.h                |  16 ++---
 lib/ethdev/rte_ethdev_core.h           |   2 +-
 lib/eventdev/rte_event_timer_adapter.c |  66 +++++++++---------
 lib/eventdev/rte_event_timer_adapter.h |   2 +-
 lib/gpudev/gpudev.c                    |   6 +-
 lib/gpudev/gpudev_driver.h             |   2 +-
 lib/hash/rte_cuckoo_hash.c             | 116 +++++++++++++++----------------
 lib/hash/rte_cuckoo_hash.h             |   6 +-
 lib/ipsec/ipsec_sqn.h                  |   2 +-
 lib/ipsec/sa.h                         |   2 +-
 lib/mbuf/rte_mbuf.h                    |  20 +++---
 lib/mbuf/rte_mbuf_core.h               |   4 +-
 lib/mempool/rte_mempool.h              |   4 +-
 lib/pdump/rte_pdump.c                  |  14 ++--
 lib/pdump/rte_pdump.h                  |   8 +--
 lib/power/power_acpi_cpufreq.c         |  33 ++++-----
 lib/power/power_amd_pstate_cpufreq.c   |  14 ++--
 lib/power/power_cppc_cpufreq.c         |  25 +++----
 lib/power/power_pstate_cpufreq.c       |  31 +++++----
 lib/rcu/rte_rcu_qsbr.c                 |  48 ++++++-------
 lib/rcu/rte_rcu_qsbr.h                 |  68 +++++++++---------
 lib/ring/rte_ring_c11_pvt.h            |  33 ++++-----
 lib/ring/rte_ring_core.h               |  10 +--
 lib/ring/rte_ring_generic_pvt.h        |   3 +-
 lib/ring/rte_ring_hts_elem_pvt.h       |  22 +++---
 lib/ring/rte_ring_peek_elem_pvt.h      |   6 +-
 lib/ring/rte_ring_rts_elem_pvt.h       |  27 ++++----
 lib/stack/rte_stack.h                  |   2 +-
 lib/stack/rte_stack_lf_c11.h           |  24 +++----
 lib/stack/rte_stack_lf_generic.h       |  18 ++---
 lib/telemetry/telemetry.c              |  18 ++---
 lib/timer/rte_timer.c                  |  50 +++++++-------
 lib/timer/rte_timer.h                  |   6 +-
 lib/vhost/vdpa.c                       |   3 +-
 lib/vhost/vhost.c                      |  42 ++++++------
 lib/vhost/vhost.h                      |  39 ++++++-----
 lib/vhost/vhost_user.c                 |   6 +-
 lib/vhost/virtio_net.c                 |  58 +++++++++-------
 lib/vhost/virtio_net_ctrl.c            |   6 +-
 67 files changed, 674 insertions(+), 647 deletions(-)
Signature · 1 line
1.8.3.1
#1 lore raw

PATCH 1/21 power: fix use of rte stdatomic

* rte stdatomic functions operate on RTE_ATOMIC(T) specified types not regular T add missing specifier for amd_pstate_power_info state field
* use rte_memory_order_xxx instead of __ATOMIC_XXX

Fixes 1ed04d33cf19 ("power: support amd-pstate cpufreq driver")
---
 lib/power/power_amd_pstate_cpufreq.c | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)
lib/power/power_amd_pstate_cpufreq.c +7 -7
index ee58395..dbd9d2b 100644
@@ -47,7 +47,7 @@ enum power_state {
  */
 struct amd_pstate_power_info {
 	uint32_t lcore_id;                   /**< Logical core id */
-	uint32_t state;                      /**< Power in use state */
+	RTE_ATOMIC(uint32_t) state;          /**< Power in use state */
 	FILE *f;                             /**< FD of scaling_setspeed */
 	char governor_ori[28];               /**< Original governor name */
 	uint32_t curr_idx;                   /**< Freq index in freqs array */
@@ -370,7 +370,7 @@ struct amd_pstate_power_info {
 	 */
 	if (!rte_atomic_compare_exchange_strong_explicit(&(pi->state),
 					&exp_state, POWER_ONGOING,
-					__ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
+					rte_memory_order_acquire, rte_memory_order_relaxed)) {
 		RTE_LOG(INFO, POWER, "Power management of lcore %u is "
 				"in use\n", lcore_id);
 		return -1;
@@ -408,12 +408,12 @@ struct amd_pstate_power_info {
 	RTE_LOG(INFO, POWER, "Initialized successfully for lcore %u "
 			"power management\n", lcore_id);
 
-	rte_atomic_store_explicit(&(pi->state), POWER_USED, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&(pi->state), POWER_USED, rte_memory_order_release);
 
 	return 0;
 
 fail:
-	rte_atomic_store_explicit(&(pi->state), POWER_UNKNOWN, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&(pi->state), POWER_UNKNOWN, rte_memory_order_release);
 	return -1;
 }
 
@@ -448,7 +448,7 @@ struct amd_pstate_power_info {
 	 */
 	if (!rte_atomic_compare_exchange_strong_explicit(&(pi->state),
 					&exp_state, POWER_ONGOING,
-					__ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
+					rte_memory_order_acquire, rte_memory_order_relaxed)) {
 		RTE_LOG(INFO, POWER, "Power management of lcore %u is "
 				"not used\n", lcore_id);
 		return -1;
@@ -468,12 +468,12 @@ struct amd_pstate_power_info {
 	RTE_LOG(INFO, POWER, "Power management of lcore %u has exited from "
 			"'userspace' mode and been set back to the "
 			"original\n", lcore_id);
-	rte_atomic_store_explicit(&(pi->state), POWER_IDLE, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&(pi->state), POWER_IDLE, rte_memory_order_release);
 
 	return 0;
 
 fail:
-	rte_atomic_store_explicit(&(pi->state), POWER_UNKNOWN, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&(pi->state), POWER_UNKNOWN, rte_memory_order_release);
 
 	return -1;
 }
Signature · 1 line
1.8.3.1
#2 lore raw

PATCH 2/21 event/cnxk: remove single use of rte stdatomic

In reply to Tyler Retzlaff

The variable operated on by the single use of rte stdatomic was not RTE_ATOMIC(T) specified. Remove the use of stdatomic for now to fix LLVM build with enable_stdatomic=true. event/cnxk will be converted to rte stdatomic in a later series.

Fixes 14a4aa9eae71 ("event/cnxk: support get remaining ticks")
---
 drivers/event/cnxk/cnxk_tim_worker.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/event/cnxk/cnxk_tim_worker.c +1 -1
index ae4bf33..944490d 100644
@@ -193,7 +193,7 @@
 		return -ENOENT;
 
 	bkt = (struct cnxk_tim_bkt *)evtim->impl_opaque[1];
-	sema = rte_atomic_load_explicit(&bkt->w1, rte_memory_order_acquire);
+	sema = __atomic_load_n(&bkt->w1, rte_memory_order_acquire);
 	if (cnxk_tim_bkt_get_hbt(sema) || !cnxk_tim_bkt_get_nent(sema))
 		return -ENOENT;
 
Signature · 1 line
1.8.3.1
#3 lore raw

PATCH 3/21 power: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/power/power_acpi_cpufreq.c   | 33 +++++++++++++++++----------------
 lib/power/power_cppc_cpufreq.c   | 25 +++++++++++++------------
 lib/power/power_pstate_cpufreq.c | 31 ++++++++++++++++---------------
 3 files changed, 46 insertions(+), 43 deletions(-)
lib/power/power_acpi_cpufreq.c +17 -16
index 6e57aca..8b55f19 100644
@@ -7,6 +7,7 @@
 #include <stdlib.h>
 
 #include <rte_memcpy.h>
+#include <rte_stdatomic.h>
 #include <rte_string_fns.h>
 
 #include "power_acpi_cpufreq.h"
@@ -41,13 +42,13 @@ enum power_state {
  * Power info per lcore.
  */
 struct acpi_power_info {
-	unsigned int lcore_id;                   /**< Logical core id */
+	unsigned int lcore_id;               /**< Logical core id */
 	uint32_t freqs[RTE_MAX_LCORE_FREQS]; /**< Frequency array */
 	uint32_t nb_freqs;                   /**< number of available freqs */
 	FILE *f;                             /**< FD of scaling_setspeed */
 	char governor_ori[32];               /**< Original governor name */
 	uint32_t curr_idx;                   /**< Freq index in freqs array */
-	uint32_t state;                      /**< Power in use state */
+	RTE_ATOMIC(uint32_t) state;          /**< Power in use state */
 	uint16_t turbo_available;            /**< Turbo Boost available */
 	uint16_t turbo_enable;               /**< Turbo Boost enable/disable */
 } __rte_cache_aligned;
@@ -249,9 +250,9 @@ struct acpi_power_info {
 	 * ordering below as lock to make sure the frequency operations
 	 * in the critical section are done under the correct state.
 	 */
-	if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
-					POWER_ONGOING, 0,
-					__ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
+	if (!rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state,
+					POWER_ONGOING,
+					rte_memory_order_acquire, rte_memory_order_relaxed)) {
 		RTE_LOG(INFO, POWER, "Power management of lcore %u is "
 				"in use\n", lcore_id);
 		return -1;
@@ -289,15 +290,15 @@ struct acpi_power_info {
 	RTE_LOG(INFO, POWER, "Initialized successfully for lcore %u "
 			"power management\n", lcore_id);
 	exp_state = POWER_ONGOING;
-	__atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_USED,
-				    0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
+	rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state, POWER_USED,
+				    rte_memory_order_release, rte_memory_order_relaxed);
 
 	return 0;
 
 fail:
 	exp_state = POWER_ONGOING;
-	__atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_UNKNOWN,
-				    0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
+	rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state, POWER_UNKNOWN,
+				    rte_memory_order_release, rte_memory_order_relaxed);
 
 	return -1;
 }
@@ -321,9 +322,9 @@ struct acpi_power_info {
 	 * ordering below as lock to make sure the frequency operations
 	 * in the critical section are done under the correct state.
 	 */
-	if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
-					POWER_ONGOING, 0,
-					__ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
+	if (!rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state,
+					POWER_ONGOING,
+					rte_memory_order_acquire, rte_memory_order_relaxed)) {
 		RTE_LOG(INFO, POWER, "Power management of lcore %u is "
 				"not used\n", lcore_id);
 		return -1;
@@ -344,15 +345,15 @@ struct acpi_power_info {
 			"'userspace' mode and been set back to the "
 			"original\n", lcore_id);
 	exp_state = POWER_ONGOING;
-	__atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_IDLE,
-				    0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
+	rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state, POWER_IDLE,
+				    rte_memory_order_release, rte_memory_order_relaxed);
 
 	return 0;
 
 fail:
 	exp_state = POWER_ONGOING;
-	__atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_UNKNOWN,
-				    0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
+	rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state, POWER_UNKNOWN,
+				    rte_memory_order_release, rte_memory_order_relaxed);
 
 	return -1;
 }
lib/power/power_cppc_cpufreq.c +13 -12
index fc9cffe..bb70f6a 100644
@@ -6,6 +6,7 @@
 #include <stdlib.h>
 
 #include <rte_memcpy.h>
+#include <rte_stdatomic.h>
 
 #include "power_cppc_cpufreq.h"
 #include "power_common.h"
@@ -49,8 +50,8 @@ enum power_state {
  * Power info per lcore.
  */
 struct cppc_power_info {
-	unsigned int lcore_id;                   /**< Logical core id */
-	uint32_t state;                      /**< Power in use state */
+	unsigned int lcore_id;               /**< Logical core id */
+	RTE_ATOMIC(uint32_t) state;          /**< Power in use state */
 	FILE *f;                             /**< FD of scaling_setspeed */
 	char governor_ori[32];               /**< Original governor name */
 	uint32_t curr_idx;                   /**< Freq index in freqs array */
@@ -353,9 +354,9 @@ struct cppc_power_info {
 	 * ordering below as lock to make sure the frequency operations
 	 * in the critical section are done under the correct state.
 	 */
-	if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
-					POWER_ONGOING, 0,
-					__ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
+	if (!rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state,
+					POWER_ONGOING,
+					rte_memory_order_acquire, rte_memory_order_relaxed)) {
 		RTE_LOG(INFO, POWER, "Power management of lcore %u is "
 				"in use\n", lcore_id);
 		return -1;
@@ -393,12 +394,12 @@ struct cppc_power_info {
 	RTE_LOG(INFO, POWER, "Initialized successfully for lcore %u "
 			"power management\n", lcore_id);
 
-	__atomic_store_n(&(pi->state), POWER_USED, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&(pi->state), POWER_USED, rte_memory_order_release);
 
 	return 0;
 
 fail:
-	__atomic_store_n(&(pi->state), POWER_UNKNOWN, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&(pi->state), POWER_UNKNOWN, rte_memory_order_release);
 	return -1;
 }
 
@@ -431,9 +432,9 @@ struct cppc_power_info {
 	 * ordering below as lock to make sure the frequency operations
 	 * in the critical section are done under the correct state.
 	 */
-	if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
-					POWER_ONGOING, 0,
-					__ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
+	if (!rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state,
+					POWER_ONGOING,
+					rte_memory_order_acquire, rte_memory_order_relaxed)) {
 		RTE_LOG(INFO, POWER, "Power management of lcore %u is "
 				"not used\n", lcore_id);
 		return -1;
@@ -453,12 +454,12 @@ struct cppc_power_info {
 	RTE_LOG(INFO, POWER, "Power management of lcore %u has exited from "
 			"'userspace' mode and been set back to the "
 			"original\n", lcore_id);
-	__atomic_store_n(&(pi->state), POWER_IDLE, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&(pi->state), POWER_IDLE, rte_memory_order_release);
 
 	return 0;
 
 fail:
-	__atomic_store_n(&(pi->state), POWER_UNKNOWN, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&(pi->state), POWER_UNKNOWN, rte_memory_order_release);
 
 	return -1;
 }
lib/power/power_pstate_cpufreq.c +16 -15
index 52aa645..5ca5f60 100644
@@ -12,6 +12,7 @@
 #include <inttypes.h>
 
 #include <rte_memcpy.h>
+#include <rte_stdatomic.h>
 
 #include "rte_power_pmd_mgmt.h"
 #include "power_pstate_cpufreq.h"
@@ -59,7 +60,7 @@ struct pstate_power_info {
 	uint32_t non_turbo_max_ratio;        /**< Non Turbo Max ratio  */
 	uint32_t sys_max_freq;               /**< system wide max freq  */
 	uint32_t core_base_freq;             /**< core base freq  */
-	uint32_t state;                      /**< Power in use state */
+	RTE_ATOMIC(uint32_t) state;          /**< Power in use state */
 	uint16_t turbo_available;            /**< Turbo Boost available */
 	uint16_t turbo_enable;               /**< Turbo Boost enable/disable */
 	uint16_t priority_core;              /**< High Performance core */
@@ -555,9 +556,9 @@ struct pstate_power_info {
 	 * ordering below as lock to make sure the frequency operations
 	 * in the critical section are done under the correct state.
 	 */
-	if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
-					POWER_ONGOING, 0,
-					__ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
+	if (!rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state,
+					POWER_ONGOING,
+					rte_memory_order_acquire, rte_memory_order_relaxed)) {
 		RTE_LOG(INFO, POWER, "Power management of lcore %u is "
 				"in use\n", lcore_id);
 		return -1;
@@ -600,15 +601,15 @@ struct pstate_power_info {
 	RTE_LOG(INFO, POWER, "Initialized successfully for lcore %u "
 			"power management\n", lcore_id);
 	exp_state = POWER_ONGOING;
-	__atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_USED,
-				    0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
+	rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state, POWER_USED,
+				    rte_memory_order_release, rte_memory_order_relaxed);
 
 	return 0;
 
 fail:
 	exp_state = POWER_ONGOING;
-	__atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_UNKNOWN,
-				    0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
+	rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state, POWER_UNKNOWN,
+				    rte_memory_order_release, rte_memory_order_relaxed);
 
 	return -1;
 }
@@ -633,9 +634,9 @@ struct pstate_power_info {
 	 * ordering below as lock to make sure the frequency operations
 	 * in the critical section are under done the correct state.
 	 */
-	if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
-					POWER_ONGOING, 0,
-					__ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
+	if (!rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state,
+					POWER_ONGOING,
+					rte_memory_order_acquire, rte_memory_order_relaxed)) {
 		RTE_LOG(INFO, POWER, "Power management of lcore %u is "
 				"not used\n", lcore_id);
 		return -1;
@@ -658,15 +659,15 @@ struct pstate_power_info {
 			"'performance' mode and been set back to the "
 			"original\n", lcore_id);
 	exp_state = POWER_ONGOING;
-	__atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_IDLE,
-				    0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
+	rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state, POWER_IDLE,
+				    rte_memory_order_release, rte_memory_order_relaxed);
 
 	return 0;
 
 fail:
 	exp_state = POWER_ONGOING;
-	__atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_UNKNOWN,
-				    0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
+	rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state, POWER_UNKNOWN,
+				    rte_memory_order_release, rte_memory_order_relaxed);
 
 	return -1;
 }
Signature · 1 line
1.8.3.1
#4 lore raw

PATCH 4/21 bbdev: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/bbdev/rte_bbdev.c | 6 +++---
 lib/bbdev/rte_bbdev.h | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)
lib/bbdev/rte_bbdev.c +3 -3
index 155323e..cfebea0 100644
@@ -208,7 +208,7 @@ struct rte_bbdev *
 		return NULL;
 	}
 
-	__atomic_fetch_add(&bbdev->data->process_cnt, 1, __ATOMIC_RELAXED);
+	rte_atomic_fetch_add_explicit(&bbdev->data->process_cnt, 1, rte_memory_order_relaxed);
 	bbdev->data->dev_id = dev_id;
 	bbdev->state = RTE_BBDEV_INITIALIZED;
 
@@ -250,8 +250,8 @@ struct rte_bbdev *
 	}
 
 	/* clear shared BBDev Data if no process is using the device anymore */
-	if (__atomic_fetch_sub(&bbdev->data->process_cnt, 1,
-			      __ATOMIC_RELAXED) - 1 == 0)
+	if (rte_atomic_fetch_sub_explicit(&bbdev->data->process_cnt, 1,
+			      rte_memory_order_relaxed) - 1 == 0)
 		memset(bbdev->data, 0, sizeof(*bbdev->data));
 
 	memset(bbdev, 0, sizeof(*bbdev));
lib/bbdev/rte_bbdev.h +1 -1
index d12e2e7..e1aee08 100644
@@ -482,7 +482,7 @@ struct rte_bbdev_data {
 	uint16_t dev_id;  /**< Device ID */
 	int socket_id;  /**< NUMA socket that device is on */
 	bool started;  /**< Device run-time state */
-	uint16_t process_cnt;  /** Counter of processes using the device */
+	RTE_ATOMIC(uint16_t) process_cnt;  /** Counter of processes using the device */
 };
 
 /* Forward declarations */
Signature · 1 line
1.8.3.1
#5 lore raw

PATCH 5/21 eal: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/eal/common/eal_common_launch.c    |  10 +--
 lib/eal/common/eal_common_mcfg.c      |   2 +-
 lib/eal/common/eal_common_proc.c      |  14 ++--
 lib/eal/common/eal_common_thread.c    |  26 ++++----
 lib/eal/common/eal_common_trace.c     |   8 +--
 lib/eal/common/eal_common_trace_ctf.c |   4 +-
 lib/eal/common/eal_memcfg.h           |   2 +-
 lib/eal/common/eal_private.h          |   4 +-
 lib/eal/common/eal_trace.h            |   4 +-
 lib/eal/common/rte_service.c          | 122 +++++++++++++++++-----------------
 lib/eal/freebsd/eal.c                 |  20 +++---
 lib/eal/include/rte_epoll.h           |   3 +-
 lib/eal/linux/eal.c                   |  26 ++++----
 lib/eal/linux/eal_interrupts.c        |  42 ++++++------
 lib/eal/ppc/include/rte_atomic.h      |   6 +-
 lib/eal/windows/rte_thread.c          |   8 ++-
 16 files changed, 152 insertions(+), 149 deletions(-)
lib/eal/common/eal_common_launch.c +5 -5
index 0504598..5320c3b 100644
@@ -18,8 +18,8 @@
 int
 rte_eal_wait_lcore(unsigned worker_id)
 {
-	while (__atomic_load_n(&lcore_config[worker_id].state,
-			__ATOMIC_ACQUIRE) != WAIT)
+	while (rte_atomic_load_explicit(&lcore_config[worker_id].state,
+			rte_memory_order_acquire) != WAIT)
 		rte_pause();
 
 	return lcore_config[worker_id].ret;
@@ -38,8 +38,8 @@
 	/* Check if the worker is in 'WAIT' state. Use acquire order
 	 * since 'state' variable is used as the guard variable.
 	 */
-	if (__atomic_load_n(&lcore_config[worker_id].state,
-			__ATOMIC_ACQUIRE) != WAIT)
+	if (rte_atomic_load_explicit(&lcore_config[worker_id].state,
+			rte_memory_order_acquire) != WAIT)
 		goto finish;
 
 	lcore_config[worker_id].arg = arg;
@@ -47,7 +47,7 @@
 	 * before the worker thread starts running the function.
 	 * Use worker thread function as the guard variable.
 	 */
-	__atomic_store_n(&lcore_config[worker_id].f, f, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&lcore_config[worker_id].f, f, rte_memory_order_release);
 
 	rc = eal_thread_wake_worker(worker_id);
 
lib/eal/common/eal_common_mcfg.c +1 -1
index 2a785e7..dabb80e 100644
@@ -30,7 +30,7 @@
 	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
 
 	/* wait until shared mem_config finish initialising */
-	rte_wait_until_equal_32(&mcfg->magic, RTE_MAGIC, __ATOMIC_RELAXED);
+	rte_wait_until_equal_32(&mcfg->magic, RTE_MAGIC, rte_memory_order_relaxed);
 }
 
 int
lib/eal/common/eal_common_proc.c +7 -7
index f20a348..728815c 100644
@@ -33,7 +33,7 @@
 #include "eal_filesystem.h"
 #include "eal_internal_cfg.h"
 
-static int mp_fd = -1;
+static RTE_ATOMIC(int) mp_fd = -1;
 static rte_thread_t mp_handle_tid;
 static char mp_filter[PATH_MAX];   /* Filter for secondary process sockets */
 static char mp_dir_path[PATH_MAX]; /* The directory path for all mp sockets */
@@ -404,7 +404,7 @@ struct pending_request {
 	struct sockaddr_un sa;
 	int fd;
 
-	while ((fd = __atomic_load_n(&mp_fd, __ATOMIC_RELAXED)) >= 0) {
+	while ((fd = rte_atomic_load_explicit(&mp_fd, rte_memory_order_relaxed)) >= 0) {
 		int ret;
 
 		ret = read_msg(fd, &msg, &sa);
@@ -652,7 +652,7 @@ enum async_action {
 		RTE_LOG(ERR, EAL, "failed to create mp thread: %s\n",
 			strerror(errno));
 		close(dir_fd);
-		close(__atomic_exchange_n(&mp_fd, -1, __ATOMIC_RELAXED));
+		close(rte_atomic_exchange_explicit(&mp_fd, -1, rte_memory_order_relaxed));
 		return -1;
 	}
 
@@ -668,7 +668,7 @@ enum async_action {
 {
 	int fd;
 
-	fd = __atomic_exchange_n(&mp_fd, -1, __ATOMIC_RELAXED);
+	fd = rte_atomic_exchange_explicit(&mp_fd, -1, rte_memory_order_relaxed);
 	if (fd < 0)
 		return;
 
@@ -1282,11 +1282,11 @@ enum mp_status {
 
 	expected = MP_STATUS_UNKNOWN;
 	desired = status;
-	if (__atomic_compare_exchange_n(&mcfg->mp_status, &expected, desired,
-			false, __ATOMIC_RELAXED, __ATOMIC_RELAXED))
+	if (rte_atomic_compare_exchange_strong_explicit(&mcfg->mp_status, &expected, desired,
+			rte_memory_order_relaxed, rte_memory_order_relaxed))
 		return true;
 
-	return __atomic_load_n(&mcfg->mp_status, __ATOMIC_RELAXED) == desired;
+	return rte_atomic_load_explicit(&mcfg->mp_status, rte_memory_order_relaxed) == desired;
 }
 
 bool
lib/eal/common/eal_common_thread.c +13 -13
index 668b1ed..c422ea8 100644
@@ -191,8 +191,8 @@ unsigned rte_socket_id(void)
 		/* Set the state to 'RUNNING'. Use release order
 		 * since 'state' variable is used as the guard variable.
 		 */
-		__atomic_store_n(&lcore_config[lcore_id].state, RUNNING,
-			__ATOMIC_RELEASE);
+		rte_atomic_store_explicit(&lcore_config[lcore_id].state, RUNNING,
+			rte_memory_order_release);
 
 		eal_thread_ack_command();
 
@@ -201,8 +201,8 @@ unsigned rte_socket_id(void)
 		 * are accessed only after update to 'f' is visible.
 		 * Wait till the update to 'f' is visible to the worker.
 		 */
-		while ((f = __atomic_load_n(&lcore_config[lcore_id].f,
-				__ATOMIC_ACQUIRE)) == NULL)
+		while ((f = rte_atomic_load_explicit(&lcore_config[lcore_id].f,
+				rte_memory_order_acquire)) == NULL)
 			rte_pause();
 
 		rte_eal_trace_thread_lcore_running(lcore_id, f);
@@ -219,8 +219,8 @@ unsigned rte_socket_id(void)
 		 * are completed before the state is updated.
 		 * Use 'state' as the guard variable.
 		 */
-		__atomic_store_n(&lcore_config[lcore_id].state, WAIT,
-			__ATOMIC_RELEASE);
+		rte_atomic_store_explicit(&lcore_config[lcore_id].state, WAIT,
+			rte_memory_order_release);
 
 		rte_eal_trace_thread_lcore_stopped(lcore_id);
 	}
@@ -242,7 +242,7 @@ struct control_thread_params {
 	/* Control thread status.
 	 * If the status is CTRL_THREAD_ERROR, 'ret' has the error code.
 	 */
-	enum __rte_ctrl_thread_status status;
+	RTE_ATOMIC(enum __rte_ctrl_thread_status) status;
 };
 
 static int control_thread_init(void *arg)
@@ -259,13 +259,13 @@ static int control_thread_init(void *arg)
 	RTE_PER_LCORE(_socket_id) = SOCKET_ID_ANY;
 	params->ret = rte_thread_set_affinity_by_id(rte_thread_self(), cpuset);
 	if (params->ret != 0) {
-		__atomic_store_n(&params->status,
-			CTRL_THREAD_ERROR, __ATOMIC_RELEASE);
+		rte_atomic_store_explicit(&params->status,
+			CTRL_THREAD_ERROR, rte_memory_order_release);
 		return 1;
 	}
 
-	__atomic_store_n(&params->status,
-		CTRL_THREAD_RUNNING, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&params->status,
+		CTRL_THREAD_RUNNING, rte_memory_order_release);
 
 	return 0;
 }
@@ -310,8 +310,8 @@ static uint32_t control_thread_start(void *arg)
 
 	/* Wait for the control thread to initialize successfully */
 	while ((ctrl_thread_status =
-			__atomic_load_n(&params->status,
-			__ATOMIC_ACQUIRE)) == CTRL_THREAD_LAUNCHING) {
+			rte_atomic_load_explicit(&params->status,
+			rte_memory_order_acquire)) == CTRL_THREAD_LAUNCHING) {
 		rte_delay_us_sleep(1);
 	}
 
lib/eal/common/eal_common_trace.c +4 -4
index d2eac2d..6ad87fc 100644
@@ -97,7 +97,7 @@ struct trace_point_head *
 bool
 rte_trace_is_enabled(void)
 {
-	return __atomic_load_n(&trace.status, __ATOMIC_ACQUIRE) != 0;
+	return rte_atomic_load_explicit(&trace.status, rte_memory_order_acquire) != 0;
 }
 
 static void
@@ -157,7 +157,7 @@ rte_trace_mode rte_trace_mode_get(void)
 	prev = rte_atomic_fetch_or_explicit(t, __RTE_TRACE_FIELD_ENABLE_MASK,
 		rte_memory_order_release);
 	if ((prev & __RTE_TRACE_FIELD_ENABLE_MASK) == 0)
-		__atomic_fetch_add(&trace.status, 1, __ATOMIC_RELEASE);
+		rte_atomic_fetch_add_explicit(&trace.status, 1, rte_memory_order_release);
 	return 0;
 }
 
@@ -172,7 +172,7 @@ rte_trace_mode rte_trace_mode_get(void)
 	prev = rte_atomic_fetch_and_explicit(t, ~__RTE_TRACE_FIELD_ENABLE_MASK,
 		rte_memory_order_release);
 	if ((prev & __RTE_TRACE_FIELD_ENABLE_MASK) != 0)
-		__atomic_fetch_sub(&trace.status, 1, __ATOMIC_RELEASE);
+		rte_atomic_fetch_sub_explicit(&trace.status, 1, rte_memory_order_release);
 	return 0;
 }
 
@@ -526,7 +526,7 @@ rte_trace_mode rte_trace_mode_get(void)
 
 	/* Add the trace point at tail */
 	STAILQ_INSERT_TAIL(&tp_list, tp, next);
-	__atomic_thread_fence(__ATOMIC_RELEASE);
+	__atomic_thread_fence(rte_memory_order_release);
 
 	/* All Good !!! */
 	return 0;
lib/eal/common/eal_common_trace_ctf.c +2 -2
index c6775c3..04c4f71 100644
@@ -361,10 +361,10 @@
 	if (ctf_meta == NULL)
 		return -EINVAL;
 
-	if (!__atomic_load_n(&trace->ctf_fixup_done, __ATOMIC_SEQ_CST) &&
+	if (!rte_atomic_load_explicit(&trace->ctf_fixup_done, rte_memory_order_seq_cst) &&
 				rte_get_timer_hz()) {
 		meta_fixup(trace, ctf_meta);
-		__atomic_store_n(&trace->ctf_fixup_done, 1, __ATOMIC_SEQ_CST);
+		rte_atomic_store_explicit(&trace->ctf_fixup_done, 1, rte_memory_order_seq_cst);
 	}
 
 	rc = fprintf(f, "%s", ctf_meta);
lib/eal/common/eal_memcfg.h +1 -1
index d5c63e2..60e2089 100644
@@ -42,7 +42,7 @@ struct rte_mem_config {
 	rte_rwlock_t memory_hotplug_lock;
 	/**< Indicates whether memory hotplug request is in progress. */
 
-	uint8_t mp_status; /**< Multiprocess status. */
+	RTE_ATOMIC(uint8_t) mp_status; /**< Multiprocess status. */
 
 	/* memory segments and zones */
 	struct rte_fbarray memzones; /**< Memzone descriptors. */
lib/eal/common/eal_private.h +2 -2
index ebd496b..4d2e806 100644
@@ -24,11 +24,11 @@ struct lcore_config {
 	int pipe_main2worker[2];   /**< communication pipe with main */
 	int pipe_worker2main[2];   /**< communication pipe with main */
 
-	lcore_function_t * volatile f; /**< function to call */
+	RTE_ATOMIC(lcore_function_t *) volatile f; /**< function to call */
 	void * volatile arg;       /**< argument of function */
 	volatile int ret;          /**< return value of function */
 
-	volatile enum rte_lcore_state_t state; /**< lcore state */
+	volatile RTE_ATOMIC(enum rte_lcore_state_t) state; /**< lcore state */
 	unsigned int socket_id;    /**< physical socket id for this lcore */
 	unsigned int core_id;      /**< core number on socket for this lcore */
 	int core_index;            /**< relative index, starting from 0 */
lib/eal/common/eal_trace.h +2 -2
index d66bcfe..ace2ef3 100644
@@ -50,7 +50,7 @@ struct trace_arg {
 struct trace {
 	char *dir;
 	int register_errno;
-	uint32_t status;
+	RTE_ATOMIC(uint32_t) status;
 	enum rte_trace_mode mode;
 	rte_uuid_t uuid;
 	uint32_t buff_len;
@@ -65,7 +65,7 @@ struct trace {
 	uint32_t ctf_meta_offset_freq;
 	uint32_t ctf_meta_offset_freq_off_s;
 	uint32_t ctf_meta_offset_freq_off;
-	uint16_t ctf_fixup_done;
+	RTE_ATOMIC(uint16_t) ctf_fixup_done;
 	rte_spinlock_t lock;
 };
 
lib/eal/common/rte_service.c +61 -61

307 lines not on this page · raw message

lib/eal/freebsd/eal.c +10 -10
index 39a2868..568e06e 100644
@@ -597,8 +597,8 @@ static void rte_eal_init_alert(const char *msg)
 		return -1;
 	}
 
-	if (!__atomic_compare_exchange_n(&run_once, &has_run, 1, 0,
-					__ATOMIC_RELAXED, __ATOMIC_RELAXED)) {
+	if (!rte_atomic_compare_exchange_strong_explicit(&run_once, &has_run, 1,
+					rte_memory_order_relaxed, rte_memory_order_relaxed)) {
 		rte_eal_init_alert("already called initialization.");
 		rte_errno = EALREADY;
 		return -1;
@@ -622,7 +622,7 @@ static void rte_eal_init_alert(const char *msg)
 	if (fctret < 0) {
 		rte_eal_init_alert("Invalid 'command line' arguments.");
 		rte_errno = EINVAL;
-		__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
+		rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed);
 		return -1;
 	}
 
@@ -636,20 +636,20 @@ static void rte_eal_init_alert(const char *msg)
 	if (eal_plugins_init() < 0) {
 		rte_eal_init_alert("Cannot init plugins");
 		rte_errno = EINVAL;
-		__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
+		rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed);
 		return -1;
 	}
 
 	if (eal_trace_init() < 0) {
 		rte_eal_init_alert("Cannot init trace");
 		rte_errno = EFAULT;
-		__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
+		rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed);
 		return -1;
 	}
 
 	if (eal_option_device_parse()) {
 		rte_errno = ENODEV;
-		__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
+		rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed);
 		return -1;
 	}
 
@@ -683,7 +683,7 @@ static void rte_eal_init_alert(const char *msg)
 	if (rte_bus_scan()) {
 		rte_eal_init_alert("Cannot scan the buses for devices");
 		rte_errno = ENODEV;
-		__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
+		rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed);
 		return -1;
 	}
 
@@ -736,7 +736,7 @@ static void rte_eal_init_alert(const char *msg)
 		if (ret < 0) {
 			rte_eal_init_alert("Cannot get hugepage information.");
 			rte_errno = EACCES;
-			__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
+			rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed);
 			return -1;
 		}
 	}
@@ -915,8 +915,8 @@ static void rte_eal_init_alert(const char *msg)
 	static uint32_t run_once;
 	uint32_t has_run = 0;
 
-	if (!__atomic_compare_exchange_n(&run_once, &has_run, 1, 0,
-			__ATOMIC_RELAXED, __ATOMIC_RELAXED)) {
+	if (!rte_atomic_compare_exchange_strong_explicit(&run_once, &has_run, 1,
+			rte_memory_order_relaxed, rte_memory_order_relaxed)) {
 		RTE_LOG(WARNING, EAL, "Already called cleanup\n");
 		rte_errno = EALREADY;
 		return -1;
lib/eal/include/rte_epoll.h +2 -1
index 01525f5..ae0cf20 100644
@@ -13,6 +13,7 @@
 
 #include <stdint.h>
 
+#include <rte_stdatomic.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -38,7 +39,7 @@ enum {
 
 /** interrupt epoll event obj, taken by epoll_event.ptr */
 struct rte_epoll_event {
-	uint32_t status;           /**< OUT: event status */
+	RTE_ATOMIC(uint32_t) status;           /**< OUT: event status */
 	int fd;                    /**< OUT: event fd */
 	int epfd;       /**< OUT: epoll instance the ev associated with */
 	struct rte_epoll_data epdata;
lib/eal/linux/eal.c +13 -13
index 5f4b2fb..57da058 100644
@@ -967,7 +967,7 @@ static void rte_eal_init_alert(const char *msg)
 rte_eal_init(int argc, char **argv)
 {
 	int i, fctret, ret;
-	static uint32_t run_once;
+	static RTE_ATOMIC(uint32_t) run_once;
 	uint32_t has_run = 0;
 	char cpuset[RTE_CPU_AFFINITY_STR_LEN];
 	char thread_name[RTE_THREAD_NAME_SIZE];
@@ -983,8 +983,8 @@ static void rte_eal_init_alert(const char *msg)
 		return -1;
 	}
 
-	if (!__atomic_compare_exchange_n(&run_once, &has_run, 1, 0,
-					__ATOMIC_RELAXED, __ATOMIC_RELAXED)) {
+	if (!rte_atomic_compare_exchange_strong_explicit(&run_once, &has_run, 1,
+					rte_memory_order_relaxed, rte_memory_order_relaxed)) {
 		rte_eal_init_alert("already called initialization.");
 		rte_errno = EALREADY;
 		return -1;
@@ -1008,14 +1008,14 @@ static void rte_eal_init_alert(const char *msg)
 	if (fctret < 0) {
 		rte_eal_init_alert("Invalid 'command line' arguments.");
 		rte_errno = EINVAL;
-		__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
+		rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed);
 		return -1;
 	}
 
 	if (eal_plugins_init() < 0) {
 		rte_eal_init_alert("Cannot init plugins");
 		rte_errno = EINVAL;
-		__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
+		rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed);
 		return -1;
 	}
 
@@ -1027,7 +1027,7 @@ static void rte_eal_init_alert(const char *msg)
 
 	if (eal_option_device_parse()) {
 		rte_errno = ENODEV;
-		__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
+		rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed);
 		return -1;
 	}
 
@@ -1061,7 +1061,7 @@ static void rte_eal_init_alert(const char *msg)
 	if (rte_bus_scan()) {
 		rte_eal_init_alert("Cannot scan the buses for devices");
 		rte_errno = ENODEV;
-		__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
+		rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed);
 		return -1;
 	}
 
@@ -1125,7 +1125,7 @@ static void rte_eal_init_alert(const char *msg)
 		if (ret < 0) {
 			rte_eal_init_alert("Cannot get hugepage information.");
 			rte_errno = EACCES;
-			__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
+			rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed);
 			return -1;
 		}
 	}
@@ -1150,7 +1150,7 @@ static void rte_eal_init_alert(const char *msg)
 			 internal_conf->syslog_facility) < 0) {
 		rte_eal_init_alert("Cannot init logging.");
 		rte_errno = ENOMEM;
-		__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
+		rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed);
 		return -1;
 	}
 
@@ -1158,7 +1158,7 @@ static void rte_eal_init_alert(const char *msg)
 	if (rte_eal_vfio_setup() < 0) {
 		rte_eal_init_alert("Cannot init VFIO");
 		rte_errno = EAGAIN;
-		__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
+		rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed);
 		return -1;
 	}
 #endif
@@ -1345,11 +1345,11 @@ static void rte_eal_init_alert(const char *msg)
 int
 rte_eal_cleanup(void)
 {
-	static uint32_t run_once;
+	static RTE_ATOMIC(uint32_t) run_once;
 	uint32_t has_run = 0;
 
-	if (!__atomic_compare_exchange_n(&run_once, &has_run, 1, 0,
-					__ATOMIC_RELAXED, __ATOMIC_RELAXED)) {
+	if (!rte_atomic_compare_exchange_strong_explicit(&run_once, &has_run, 1,
+					rte_memory_order_relaxed, rte_memory_order_relaxed)) {
 		RTE_LOG(WARNING, EAL, "Already called cleanup\n");
 		rte_errno = EALREADY;
 		return -1;
lib/eal/linux/eal_interrupts.c +21 -21

93 lines not on this page · raw message

lib/eal/ppc/include/rte_atomic.h +3 -3
index 7382412..645c713 100644
@@ -48,7 +48,7 @@
 static inline int
 rte_atomic16_cmpset(volatile uint16_t *dst, uint16_t exp, uint16_t src)
 {
-	return __atomic_compare_exchange(dst, &exp, &src, 0, rte_memory_order_acquire,
+	return rte_atomic_compare_exchange_strong_explicit(dst, &exp, src, rte_memory_order_acquire,
 		rte_memory_order_acquire) ? 1 : 0;
 }
 
@@ -90,7 +90,7 @@ static inline int rte_atomic16_dec_and_test(rte_atomic16_t *v)
 static inline int
 rte_atomic32_cmpset(volatile uint32_t *dst, uint32_t exp, uint32_t src)
 {
-	return __atomic_compare_exchange(dst, &exp, &src, 0, rte_memory_order_acquire,
+	return rte_atomic_compare_exchange_strong_explicit(dst, &exp, src, rte_memory_order_acquire,
 		rte_memory_order_acquire) ? 1 : 0;
 }
 
@@ -132,7 +132,7 @@ static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v)
 static inline int
 rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src)
 {
-	return __atomic_compare_exchange(dst, &exp, &src, 0, rte_memory_order_acquire,
+	return rte_atomic_compare_exchange_strong_explicit(dst, &exp, src, rte_memory_order_acquire,
 		rte_memory_order_acquire) ? 1 : 0;
 }
 
lib/eal/windows/rte_thread.c +5 -3
index acf6484..145ac4b 100644
@@ -9,6 +9,7 @@
 #include <rte_eal.h>
 #include <rte_common.h>
 #include <rte_errno.h>
+#include <rte_stdatomic.h>
 #include <rte_thread.h>
 
 #include "eal_windows.h"
@@ -19,7 +20,7 @@ struct eal_tls_key {
 
 struct thread_routine_ctx {
 	rte_thread_func thread_func;
-	bool thread_init_failed;
+	RTE_ATOMIC(bool) thread_init_failed;
 	void *routine_args;
 };
 
@@ -168,7 +169,8 @@ struct thread_routine_ctx {
 thread_func_wrapper(void *arg)
 {
 	struct thread_routine_ctx ctx = *(struct thread_routine_ctx *)arg;
-	const bool thread_exit = __atomic_load_n(&ctx.thread_init_failed, __ATOMIC_ACQUIRE);
+	const bool thread_exit = rte_atomic_load_explicit(
+		&ctx.thread_init_failed, rte_memory_order_acquire);
 
 	free(arg);
 
@@ -237,7 +239,7 @@ struct thread_routine_ctx {
 	}
 
 resume_thread:
-	__atomic_store_n(&ctx->thread_init_failed, thread_exit, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&ctx->thread_init_failed, thread_exit, rte_memory_order_release);
 
 	if (ResumeThread(thread_handle) == (DWORD)-1) {
 		ret = thread_log_last_error("ResumeThread()");
Signature · 1 line
1.8.3.1
#6 lore raw

PATCH 6/21 eventdev: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 drivers/event/cnxk/cnxk_tim_worker.h   |  4 +--
 lib/eventdev/rte_event_timer_adapter.c | 66 +++++++++++++++++-----------------
 lib/eventdev/rte_event_timer_adapter.h |  2 +-
 3 files changed, 36 insertions(+), 36 deletions(-)
drivers/event/cnxk/cnxk_tim_worker.h +2 -2
index f0857f2..f530d8c 100644
@@ -314,7 +314,7 @@
 
 	tim->impl_opaque[0] = (uintptr_t)chunk;
 	tim->impl_opaque[1] = (uintptr_t)bkt;
-	__atomic_store_n(&tim->state, RTE_EVENT_TIMER_ARMED, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&tim->state, RTE_EVENT_TIMER_ARMED, rte_memory_order_release);
 	cnxk_tim_bkt_inc_nent(bkt);
 	cnxk_tim_bkt_dec_lock_relaxed(bkt);
 
@@ -425,7 +425,7 @@
 
 	tim->impl_opaque[0] = (uintptr_t)chunk;
 	tim->impl_opaque[1] = (uintptr_t)bkt;
-	__atomic_store_n(&tim->state, RTE_EVENT_TIMER_ARMED, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&tim->state, RTE_EVENT_TIMER_ARMED, rte_memory_order_release);
 	cnxk_tim_bkt_inc_nent(bkt);
 	cnxk_tim_bkt_dec_lock_relaxed(bkt);
 
lib/eventdev/rte_event_timer_adapter.c +33 -33

162 lines not on this page · raw message

lib/eventdev/rte_event_timer_adapter.h +1 -1
index fbdddf8..49e646a 100644
@@ -498,7 +498,7 @@ struct rte_event_timer {
 	 * implementation specific values to share between the arm and cancel
 	 * operations.  The application should not modify this field.
 	 */
-	enum rte_event_timer_state state;
+	RTE_ATOMIC(enum rte_event_timer_state) state;
 	/**< State of the event timer. */
 	uint8_t user_meta[];
 	/**< Memory to store user specific metadata.
Signature · 1 line
1.8.3.1
#7 lore raw

PATCH 7/21 gpudev: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/gpudev/gpudev.c        | 6 +++---
 lib/gpudev/gpudev_driver.h | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)
lib/gpudev/gpudev.c +3 -3
index 8f12abe..6845d18 100644
@@ -228,7 +228,7 @@ struct rte_gpu *
 	dev->mpshared->info.numa_node = -1;
 	dev->mpshared->info.parent = RTE_GPU_ID_NONE;
 	TAILQ_INIT(&dev->callbacks);
-	__atomic_fetch_add(&dev->mpshared->process_refcnt, 1, __ATOMIC_RELAXED);
+	rte_atomic_fetch_add_explicit(&dev->mpshared->process_refcnt, 1, rte_memory_order_relaxed);
 
 	gpu_count++;
 	GPU_LOG(DEBUG, "new device %s (id %d) of total %d",
@@ -277,7 +277,7 @@ struct rte_gpu *
 
 	TAILQ_INIT(&dev->callbacks);
 	dev->mpshared = shared_dev;
-	__atomic_fetch_add(&dev->mpshared->process_refcnt, 1, __ATOMIC_RELAXED);
+	rte_atomic_fetch_add_explicit(&dev->mpshared->process_refcnt, 1, rte_memory_order_relaxed);
 
 	gpu_count++;
 	GPU_LOG(DEBUG, "attached device %s (id %d) of total %d",
@@ -340,7 +340,7 @@ struct rte_gpu *
 
 	gpu_free_callbacks(dev);
 	dev->process_state = RTE_GPU_STATE_UNUSED;
-	__atomic_fetch_sub(&dev->mpshared->process_refcnt, 1, __ATOMIC_RELAXED);
+	rte_atomic_fetch_sub_explicit(&dev->mpshared->process_refcnt, 1, rte_memory_order_relaxed);
 	gpu_count--;
 
 	return 0;
lib/gpudev/gpudev_driver.h +1 -1
index 42898c7..0b1e7f2 100644
@@ -69,7 +69,7 @@ struct rte_gpu_mpshared {
 	/* Device info structure. */
 	struct rte_gpu_info info;
 	/* Counter of processes using the device. */
-	uint16_t process_refcnt; /* Updated by this library. */
+	RTE_ATOMIC(uint16_t) process_refcnt; /* Updated by this library. */
 };
 
 struct rte_gpu {
Signature · 1 line
1.8.3.1
#8 lore raw

PATCH 8/21 ipsec: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/ipsec/ipsec_sqn.h | 2 +-
 lib/ipsec/sa.h        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
lib/ipsec/ipsec_sqn.h +1 -1
index 505950e..984a9dd 100644
@@ -128,7 +128,7 @@
 
 	n = *num;
 	if (SQN_ATOMIC(sa))
-		sqn = __atomic_fetch_add(&sa->sqn.outb, n, __ATOMIC_RELAXED) + n;
+		sqn = rte_atomic_fetch_add_explicit(&sa->sqn.outb, n, rte_memory_order_relaxed) + n;
 	else {
 		sqn = sa->sqn.outb + n;
 		sa->sqn.outb = sqn;
lib/ipsec/sa.h +1 -1
index ce4af8c..4b30bea 100644
@@ -124,7 +124,7 @@ struct rte_ipsec_sa {
 	 * place from other frequently accessed data.
 	 */
 	union {
-		uint64_t outb;
+		RTE_ATOMIC(uint64_t) outb;
 		struct {
 			uint32_t rdidx; /* read index */
 			uint32_t wridx; /* write index */
Signature · 1 line
1.8.3.1
#9 lore raw

PATCH 9/21 mbuf: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/mbuf/rte_mbuf.h      | 20 ++++++++++----------
 lib/mbuf/rte_mbuf_core.h |  4 ++--
 2 files changed, 12 insertions(+), 12 deletions(-)
lib/mbuf/rte_mbuf.h +10 -10
index 913c459..b8ab477 100644
@@ -361,7 +361,7 @@ struct rte_pktmbuf_pool_private {
 static inline uint16_t
 rte_mbuf_refcnt_read(const struct rte_mbuf *m)
 {
-	return __atomic_load_n(&m->refcnt, __ATOMIC_RELAXED);
+	return rte_atomic_load_explicit(&m->refcnt, rte_memory_order_relaxed);
 }
 
 /**
@@ -374,15 +374,15 @@ struct rte_pktmbuf_pool_private {
 static inline void
 rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
 {
-	__atomic_store_n(&m->refcnt, new_value, __ATOMIC_RELAXED);
+	rte_atomic_store_explicit(&m->refcnt, new_value, rte_memory_order_relaxed);
 }
 
 /* internal */
 static inline uint16_t
 __rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
 {
-	return __atomic_fetch_add(&m->refcnt, value,
-				 __ATOMIC_ACQ_REL) + value;
+	return rte_atomic_fetch_add_explicit(&m->refcnt, value,
+				 rte_memory_order_acq_rel) + value;
 }
 
 /**
@@ -463,7 +463,7 @@ struct rte_pktmbuf_pool_private {
 static inline uint16_t
 rte_mbuf_ext_refcnt_read(const struct rte_mbuf_ext_shared_info *shinfo)
 {
-	return __atomic_load_n(&shinfo->refcnt, __ATOMIC_RELAXED);
+	return rte_atomic_load_explicit(&shinfo->refcnt, rte_memory_order_relaxed);
 }
 
 /**
@@ -478,7 +478,7 @@ struct rte_pktmbuf_pool_private {
 rte_mbuf_ext_refcnt_set(struct rte_mbuf_ext_shared_info *shinfo,
 	uint16_t new_value)
 {
-	__atomic_store_n(&shinfo->refcnt, new_value, __ATOMIC_RELAXED);
+	rte_atomic_store_explicit(&shinfo->refcnt, new_value, rte_memory_order_relaxed);
 }
 
 /**
@@ -502,8 +502,8 @@ struct rte_pktmbuf_pool_private {
 		return (uint16_t)value;
 	}
 
-	return __atomic_fetch_add(&shinfo->refcnt, value,
-				 __ATOMIC_ACQ_REL) + value;
+	return rte_atomic_fetch_add_explicit(&shinfo->refcnt, value,
+				 rte_memory_order_acq_rel) + value;
 }
 
 /** Mbuf prefetch */
@@ -1315,8 +1315,8 @@ static inline int __rte_pktmbuf_pinned_extbuf_decref(struct rte_mbuf *m)
 	 * Direct usage of add primitive to avoid
 	 * duplication of comparing with one.
 	 */
-	if (likely(__atomic_fetch_add(&shinfo->refcnt, -1,
-				     __ATOMIC_ACQ_REL) - 1))
+	if (likely(rte_atomic_fetch_add_explicit(&shinfo->refcnt, -1,
+				     rte_memory_order_acq_rel) - 1))
 		return 1;
 
 	/* Reinitialize counter before mbuf freeing. */
lib/mbuf/rte_mbuf_core.h +2 -2
index e9bc0d1..bf761f8 100644
@@ -497,7 +497,7 @@ struct rte_mbuf {
 	 * rte_mbuf_refcnt_set(). The functionality of these functions (atomic,
 	 * or non-atomic) is controlled by the RTE_MBUF_REFCNT_ATOMIC flag.
 	 */
-	uint16_t refcnt;
+	RTE_ATOMIC(uint16_t) refcnt;
 
 	/**
 	 * Number of segments. Only valid for the first segment of an mbuf
@@ -674,7 +674,7 @@ struct rte_mbuf {
 struct rte_mbuf_ext_shared_info {
 	rte_mbuf_extbuf_free_callback_t free_cb; /**< Free callback function */
 	void *fcb_opaque;                        /**< Free callback argument */
-	uint16_t refcnt;
+	RTE_ATOMIC(uint16_t) refcnt;
 };
 
 /** Maximum number of nb_segs allowed. */
Signature · 1 line
1.8.3.1
#10 lore raw

PATCH 10/21 mempool: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/mempool/rte_mempool.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
lib/mempool/rte_mempool.h +2 -2
index f70bf36..df87cd2 100644
@@ -327,8 +327,8 @@ struct rte_mempool {
 		if (likely(__lcore_id < RTE_MAX_LCORE))                         \
 			(mp)->stats[__lcore_id].name += (n);                    \
 		else                                                            \
-			__atomic_fetch_add(&((mp)->stats[RTE_MAX_LCORE].name),  \
-					   (n), __ATOMIC_RELAXED);              \
+			rte_atomic_fetch_add_explicit(&((mp)->stats[RTE_MAX_LCORE].name),  \
+					   (n), rte_memory_order_relaxed);              \
 	} while (0)
 #else
 #define RTE_MEMPOOL_STAT_ADD(mp, name, n) do {} while (0)
Signature · 1 line
1.8.3.1
#11 lore raw

PATCH 11/21 rcu: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/rcu/rte_rcu_qsbr.c | 48 +++++++++++++++++------------------
 lib/rcu/rte_rcu_qsbr.h | 68 +++++++++++++++++++++++++-------------------------
 2 files changed, 58 insertions(+), 58 deletions(-)
lib/rcu/rte_rcu_qsbr.c +24 -24
index 17be93e..4dc7714 100644
@@ -102,21 +102,21 @@
 	 * go out of sync. Hence, additional checks are required.
 	 */
 	/* Check if the thread is already registered */
-	old_bmap = __atomic_load_n(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					__ATOMIC_RELAXED);
+	old_bmap = rte_atomic_load_explicit(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
+					rte_memory_order_relaxed);
 	if (old_bmap & 1UL << id)
 		return 0;
 
 	do {
 		new_bmap = old_bmap | (1UL << id);
-		success = __atomic_compare_exchange(
+		success = rte_atomic_compare_exchange_strong_explicit(
 					__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					&old_bmap, &new_bmap, 0,
-					__ATOMIC_RELEASE, __ATOMIC_RELAXED);
+					&old_bmap, new_bmap,
+					rte_memory_order_release, rte_memory_order_relaxed);
 
 		if (success)
-			__atomic_fetch_add(&v->num_threads,
-						1, __ATOMIC_RELAXED);
+			rte_atomic_fetch_add_explicit(&v->num_threads,
+						1, rte_memory_order_relaxed);
 		else if (old_bmap & (1UL << id))
 			/* Someone else registered this thread.
 			 * Counter should not be incremented.
@@ -154,8 +154,8 @@
 	 * go out of sync. Hence, additional checks are required.
 	 */
 	/* Check if the thread is already unregistered */
-	old_bmap = __atomic_load_n(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					__ATOMIC_RELAXED);
+	old_bmap = rte_atomic_load_explicit(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
+					rte_memory_order_relaxed);
 	if (!(old_bmap & (1UL << id)))
 		return 0;
 
@@ -165,14 +165,14 @@
 		 * completed before removal of the thread from the list of
 		 * reporting threads.
 		 */
-		success = __atomic_compare_exchange(
+		success = rte_atomic_compare_exchange_strong_explicit(
 					__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					&old_bmap, &new_bmap, 0,
-					__ATOMIC_RELEASE, __ATOMIC_RELAXED);
+					&old_bmap, new_bmap,
+					rte_memory_order_release, rte_memory_order_relaxed);
 
 		if (success)
-			__atomic_fetch_sub(&v->num_threads,
-						1, __ATOMIC_RELAXED);
+			rte_atomic_fetch_sub_explicit(&v->num_threads,
+						1, rte_memory_order_relaxed);
 		else if (!(old_bmap & (1UL << id)))
 			/* Someone else unregistered this thread.
 			 * Counter should not be incremented.
@@ -227,8 +227,8 @@
 
 	fprintf(f, "  Registered thread IDs = ");
 	for (i = 0; i < v->num_elems; i++) {
-		bmap = __atomic_load_n(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					__ATOMIC_ACQUIRE);
+		bmap = rte_atomic_load_explicit(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
+					rte_memory_order_acquire);
 		id = i << __RTE_QSBR_THRID_INDEX_SHIFT;
 		while (bmap) {
 			t = __builtin_ctzl(bmap);
@@ -241,26 +241,26 @@
 	fprintf(f, "\n");
 
 	fprintf(f, "  Token = %" PRIu64 "\n",
-			__atomic_load_n(&v->token, __ATOMIC_ACQUIRE));
+			rte_atomic_load_explicit(&v->token, rte_memory_order_acquire));
 
 	fprintf(f, "  Least Acknowledged Token = %" PRIu64 "\n",
-			__atomic_load_n(&v->acked_token, __ATOMIC_ACQUIRE));
+			rte_atomic_load_explicit(&v->acked_token, rte_memory_order_acquire));
 
 	fprintf(f, "Quiescent State Counts for readers:\n");
 	for (i = 0; i < v->num_elems; i++) {
-		bmap = __atomic_load_n(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					__ATOMIC_ACQUIRE);
+		bmap = rte_atomic_load_explicit(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
+					rte_memory_order_acquire);
 		id = i << __RTE_QSBR_THRID_INDEX_SHIFT;
 		while (bmap) {
 			t = __builtin_ctzl(bmap);
 			fprintf(f, "thread ID = %u, count = %" PRIu64 ", lock count = %u\n",
 				id + t,
-				__atomic_load_n(
+				rte_atomic_load_explicit(
 					&v->qsbr_cnt[id + t].cnt,
-					__ATOMIC_RELAXED),
-				__atomic_load_n(
+					rte_memory_order_relaxed),
+				rte_atomic_load_explicit(
 					&v->qsbr_cnt[id + t].lock_cnt,
-					__ATOMIC_RELAXED));
+					rte_memory_order_relaxed));
 			bmap &= ~(1UL << t);
 		}
 	}
lib/rcu/rte_rcu_qsbr.h +34 -34

195 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#12 lore raw

PATCH 12/21 pdump: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/pdump/rte_pdump.c | 14 +++++++-------
 lib/pdump/rte_pdump.h |  8 ++++----
 2 files changed, 11 insertions(+), 11 deletions(-)
lib/pdump/rte_pdump.c +7 -7
index 53cca10..80b90c6 100644
@@ -110,8 +110,8 @@ struct pdump_response {
 		 * then packet doesn't match the filter (will be ignored).
 		 */
 		if (cbs->filter && rcs[i] == 0) {
-			__atomic_fetch_add(&stats->filtered,
-					   1, __ATOMIC_RELAXED);
+			rte_atomic_fetch_add_explicit(&stats->filtered,
+					   1, rte_memory_order_relaxed);
 			continue;
 		}
 
@@ -127,18 +127,18 @@ struct pdump_response {
 			p = rte_pktmbuf_copy(pkts[i], mp, 0, cbs->snaplen);
 
 		if (unlikely(p == NULL))
-			__atomic_fetch_add(&stats->nombuf, 1, __ATOMIC_RELAXED);
+			rte_atomic_fetch_add_explicit(&stats->nombuf, 1, rte_memory_order_relaxed);
 		else
 			dup_bufs[d_pkts++] = p;
 	}
 
-	__atomic_fetch_add(&stats->accepted, d_pkts, __ATOMIC_RELAXED);
+	rte_atomic_fetch_add_explicit(&stats->accepted, d_pkts, rte_memory_order_relaxed);
 
 	ring_enq = rte_ring_enqueue_burst(ring, (void *)&dup_bufs[0], d_pkts, NULL);
 	if (unlikely(ring_enq < d_pkts)) {
 		unsigned int drops = d_pkts - ring_enq;
 
-		__atomic_fetch_add(&stats->ringfull, drops, __ATOMIC_RELAXED);
+		rte_atomic_fetch_add_explicit(&stats->ringfull, drops, rte_memory_order_relaxed);
 		rte_pktmbuf_free_bulk(&dup_bufs[ring_enq], drops);
 	}
 }
@@ -720,10 +720,10 @@ struct pdump_response {
 	uint16_t qid;
 
 	for (qid = 0; qid < nq; qid++) {
-		const uint64_t *perq = (const uint64_t *)&stats[port][qid];
+		const RTE_ATOMIC(uint64_t) *perq = (const uint64_t __rte_atomic *)&stats[port][qid];
 
 		for (i = 0; i < sizeof(*total) / sizeof(uint64_t); i++) {
-			val = __atomic_load_n(&perq[i], __ATOMIC_RELAXED);
+			val = rte_atomic_load_explicit(&perq[i], rte_memory_order_relaxed);
 			sum[i] += val;
 		}
 	}
lib/pdump/rte_pdump.h +4 -4
index b1a3918..7feb2b6 100644
@@ -233,10 +233,10 @@ enum {
  * The statistics are sum of both receive and transmit queues.
  */
 struct rte_pdump_stats {
-	uint64_t accepted; /**< Number of packets accepted by filter. */
-	uint64_t filtered; /**< Number of packets rejected by filter. */
-	uint64_t nombuf;   /**< Number of mbuf allocation failures. */
-	uint64_t ringfull; /**< Number of missed packets due to ring full. */
+	RTE_ATOMIC(uint64_t) accepted; /**< Number of packets accepted by filter. */
+	RTE_ATOMIC(uint64_t) filtered; /**< Number of packets rejected by filter. */
+	RTE_ATOMIC(uint64_t) nombuf;   /**< Number of mbuf allocation failures. */
+	RTE_ATOMIC(uint64_t) ringfull; /**< Number of missed packets due to ring full. */
 
 	uint64_t reserved[4]; /**< Reserved and pad to cache line */
 };
Signature · 1 line
1.8.3.1
#13 lore raw

PATCH 13/21 stack: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/stack/rte_stack.h            |  2 +-
 lib/stack/rte_stack_lf_c11.h     | 24 ++++++++++++------------
 lib/stack/rte_stack_lf_generic.h | 18 +++++++++---------
 3 files changed, 22 insertions(+), 22 deletions(-)
lib/stack/rte_stack.h +1 -1
index 921d29a..a379300 100644
@@ -44,7 +44,7 @@ struct rte_stack_lf_list {
 	/** List head */
 	struct rte_stack_lf_head head __rte_aligned(16);
 	/** List len */
-	uint64_t len;
+	RTE_ATOMIC(uint64_t) len;
 };
 
 /* Structure containing two lock-free LIFO lists: the stack itself and a list
lib/stack/rte_stack_lf_c11.h +12 -12
index 687a6f6..9cb6998 100644
@@ -26,8 +26,8 @@
 	 * elements. If the mempool is near-empty to the point that this is a
 	 * concern, the user should consider increasing the mempool size.
 	 */
-	return (unsigned int)__atomic_load_n(&s->stack_lf.used.len,
-					     __ATOMIC_RELAXED);
+	return (unsigned int)rte_atomic_load_explicit(&s->stack_lf.used.len,
+					     rte_memory_order_relaxed);
 }
 
 static __rte_always_inline void
@@ -59,14 +59,14 @@
 				(rte_int128_t *)&list->head,
 				(rte_int128_t *)&old_head,
 				(rte_int128_t *)&new_head,
-				1, __ATOMIC_RELEASE,
-				__ATOMIC_RELAXED);
+				1, rte_memory_order_release,
+				rte_memory_order_relaxed);
 	} while (success == 0);
 
 	/* Ensure the stack modifications are not reordered with respect
 	 * to the LIFO len update.
 	 */
-	__atomic_fetch_add(&list->len, num, __ATOMIC_RELEASE);
+	rte_atomic_fetch_add_explicit(&list->len, num, rte_memory_order_release);
 }
 
 static __rte_always_inline struct rte_stack_lf_elem *
@@ -80,7 +80,7 @@
 	int success;
 
 	/* Reserve num elements, if available */
-	len = __atomic_load_n(&list->len, __ATOMIC_RELAXED);
+	len = rte_atomic_load_explicit(&list->len, rte_memory_order_relaxed);
 
 	while (1) {
 		/* Does the list contain enough elements? */
@@ -88,10 +88,10 @@
 			return NULL;
 
 		/* len is updated on failure */
-		if (__atomic_compare_exchange_n(&list->len,
+		if (rte_atomic_compare_exchange_weak_explicit(&list->len,
 						&len, len - num,
-						1, __ATOMIC_ACQUIRE,
-						__ATOMIC_RELAXED))
+						rte_memory_order_acquire,
+						rte_memory_order_relaxed))
 			break;
 	}
 
@@ -110,7 +110,7 @@
 		 * elements are properly ordered with respect to the head
 		 * pointer read.
 		 */
-		__atomic_thread_fence(__ATOMIC_ACQUIRE);
+		__atomic_thread_fence(rte_memory_order_acquire);
 
 		rte_prefetch0(old_head.top);
 
@@ -159,8 +159,8 @@
 				(rte_int128_t *)&list->head,
 				(rte_int128_t *)&old_head,
 				(rte_int128_t *)&new_head,
-				0, __ATOMIC_RELAXED,
-				__ATOMIC_RELAXED);
+				0, rte_memory_order_relaxed,
+				rte_memory_order_relaxed);
 	} while (success == 0);
 
 	return old_head.top;
lib/stack/rte_stack_lf_generic.h +9 -9
index 39f7ff3..cc69e4d 100644
@@ -27,7 +27,7 @@
 	 * concern, the user should consider increasing the mempool size.
 	 */
 	/* NOTE: review for potential ordering optimization */
-	return __atomic_load_n(&s->stack_lf.used.len, __ATOMIC_SEQ_CST);
+	return rte_atomic_load_explicit(&s->stack_lf.used.len, rte_memory_order_seq_cst);
 }
 
 static __rte_always_inline void
@@ -64,11 +64,11 @@
 				(rte_int128_t *)&list->head,
 				(rte_int128_t *)&old_head,
 				(rte_int128_t *)&new_head,
-				1, __ATOMIC_RELEASE,
-				__ATOMIC_RELAXED);
+				1, rte_memory_order_release,
+				rte_memory_order_relaxed);
 	} while (success == 0);
 	/* NOTE: review for potential ordering optimization */
-	__atomic_fetch_add(&list->len, num, __ATOMIC_SEQ_CST);
+	rte_atomic_fetch_add_explicit(&list->len, num, rte_memory_order_seq_cst);
 }
 
 static __rte_always_inline struct rte_stack_lf_elem *
@@ -83,15 +83,15 @@
 	/* Reserve num elements, if available */
 	while (1) {
 		/* NOTE: review for potential ordering optimization */
-		uint64_t len = __atomic_load_n(&list->len, __ATOMIC_SEQ_CST);
+		uint64_t len = rte_atomic_load_explicit(&list->len, rte_memory_order_seq_cst);
 
 		/* Does the list contain enough elements? */
 		if (unlikely(len < num))
 			return NULL;
 
 		/* NOTE: review for potential ordering optimization */
-		if (__atomic_compare_exchange_n(&list->len, &len, len - num,
-				0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST))
+		if (rte_atomic_compare_exchange_strong_explicit(&list->len, &len, len - num,
+				rte_memory_order_seq_cst, rte_memory_order_seq_cst))
 			break;
 	}
 
@@ -143,8 +143,8 @@
 				(rte_int128_t *)&list->head,
 				(rte_int128_t *)&old_head,
 				(rte_int128_t *)&new_head,
-				1, __ATOMIC_RELEASE,
-				__ATOMIC_RELAXED);
+				1, rte_memory_order_release,
+				rte_memory_order_relaxed);
 	} while (success == 0);
 
 	return old_head.top;
Signature · 1 line
1.8.3.1
#14 lore raw

PATCH 14/21 telemetry: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/telemetry/telemetry.c | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)
lib/telemetry/telemetry.c +9 -9
index aeb078c..9298284 100644
@@ -45,7 +45,7 @@ struct socket {
 	int sock;
 	char path[sizeof(((struct sockaddr_un *)0)->sun_path)];
 	handler fn;
-	uint16_t *num_clients;
+	RTE_ATOMIC(uint16_t) *num_clients;
 };
 static struct socket v2_socket; /* socket for v2 telemetry */
 static struct socket v1_socket; /* socket for v1 telemetry */
@@ -64,7 +64,7 @@ struct socket {
 /* Used when accessing or modifying list of command callbacks */
 static rte_spinlock_t callback_sl = RTE_SPINLOCK_INITIALIZER;
 #ifndef RTE_EXEC_ENV_WINDOWS
-static uint16_t v2_clients;
+static RTE_ATOMIC(uint16_t) v2_clients;
 #endif /* !RTE_EXEC_ENV_WINDOWS */
 
 int
@@ -404,7 +404,7 @@ struct socket {
 		bytes = read(s, buffer, sizeof(buffer) - 1);
 	}
 	close(s);
-	__atomic_fetch_sub(&v2_clients, 1, __ATOMIC_RELAXED);
+	rte_atomic_fetch_sub_explicit(&v2_clients, 1, rte_memory_order_relaxed);
 	return NULL;
 }
 
@@ -421,14 +421,14 @@ struct socket {
 			return NULL;
 		}
 		if (s->num_clients != NULL) {
-			uint16_t conns = __atomic_load_n(s->num_clients,
-					__ATOMIC_RELAXED);
+			uint16_t conns = rte_atomic_load_explicit(s->num_clients,
+					rte_memory_order_relaxed);
 			if (conns >= MAX_CONNECTIONS) {
 				close(s_accepted);
 				continue;
 			}
-			__atomic_fetch_add(s->num_clients, 1,
-					__ATOMIC_RELAXED);
+			rte_atomic_fetch_add_explicit(s->num_clients, 1,
+					rte_memory_order_relaxed);
 		}
 		rc = pthread_create(&th, NULL, s->fn,
 				    (void *)(uintptr_t)s_accepted);
@@ -437,8 +437,8 @@ struct socket {
 				 strerror(rc));
 			close(s_accepted);
 			if (s->num_clients != NULL)
-				__atomic_fetch_sub(s->num_clients, 1,
-						   __ATOMIC_RELAXED);
+				rte_atomic_fetch_sub_explicit(s->num_clients, 1,
+						   rte_memory_order_relaxed);
 			continue;
 		}
 		pthread_detach(th);
Signature · 1 line
1.8.3.1
#15 lore raw

PATCH 15/21 vhost: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/vhost/vdpa.c            |  3 ++-
 lib/vhost/vhost.c           | 42 ++++++++++++++++----------------
 lib/vhost/vhost.h           | 39 ++++++++++++++++--------------
 lib/vhost/vhost_user.c      |  6 ++---
 lib/vhost/virtio_net.c      | 58 +++++++++++++++++++++++++--------------------
 lib/vhost/virtio_net_ctrl.c |  6 +++--
 6 files changed, 84 insertions(+), 70 deletions(-)
lib/vhost/vdpa.c +2 -1
index 6284ea2..219eef8 100644
@@ -235,7 +235,8 @@ struct rte_vdpa_device *
 	}
 
 	/* used idx is the synchronization point for the split vring */
-	__atomic_store_n(&vq->used->idx, idx_m, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit((unsigned short __rte_atomic *)&vq->used->idx,
+		idx_m, rte_memory_order_release);
 
 	if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))
 		vring_used_event(s_vring) = idx_m;
lib/vhost/vhost.c +22 -20

124 lines not on this page · raw message

lib/vhost/vhost.h +21 -18
index 5fc9035..f8624fb 100644
@@ -158,9 +158,9 @@ struct virtqueue_stats {
 	uint64_t inflight_completed;
 	uint64_t guest_notifications_suppressed;
 	/* Counters below are atomic, and should be incremented as such. */
-	uint64_t guest_notifications;
-	uint64_t guest_notifications_offloaded;
-	uint64_t guest_notifications_error;
+	RTE_ATOMIC(uint64_t) guest_notifications;
+	RTE_ATOMIC(uint64_t) guest_notifications_offloaded;
+	RTE_ATOMIC(uint64_t) guest_notifications_error;
 };
 
 /**
@@ -348,7 +348,7 @@ struct vhost_virtqueue {
 	struct vhost_vring_addr ring_addrs;
 	struct virtqueue_stats	stats;
 
-	bool irq_pending;
+	RTE_ATOMIC(bool) irq_pending;
 } __rte_cache_aligned;
 
 /* Virtio device status as per Virtio specification */
@@ -486,7 +486,7 @@ struct virtio_net {
 	uint32_t		flags;
 	uint16_t		vhost_hlen;
 	/* to tell if we need broadcast rarp packet */
-	int16_t			broadcast_rarp;
+	RTE_ATOMIC(int16_t)	broadcast_rarp;
 	uint32_t		nr_vring;
 	int			async_copy;
 
@@ -557,7 +557,8 @@ struct virtio_net {
 static inline bool
 desc_is_avail(struct vring_packed_desc *desc, bool wrap_counter)
 {
-	uint16_t flags = __atomic_load_n(&desc->flags, __ATOMIC_ACQUIRE);
+	uint16_t flags = rte_atomic_load_explicit((unsigned short __rte_atomic *)&desc->flags,
+		rte_memory_order_acquire);
 
 	return wrap_counter == !!(flags & VRING_DESC_F_AVAIL) &&
 		wrap_counter != !!(flags & VRING_DESC_F_USED);
@@ -914,17 +915,19 @@ uint64_t translate_log_addr(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	bool expected = false;
 
 	if (dev->notify_ops->guest_notify) {
-		if (__atomic_compare_exchange_n(&vq->irq_pending, &expected, true, 0,
-				  __ATOMIC_RELEASE, __ATOMIC_RELAXED)) {
+		if (rte_atomic_compare_exchange_strong_explicit(&vq->irq_pending, &expected, true,
+				  rte_memory_order_release, rte_memory_order_relaxed)) {
 			if (dev->notify_ops->guest_notify(dev->vid, vq->index)) {
 				if (dev->flags & VIRTIO_DEV_STATS_ENABLED)
-					__atomic_fetch_add(&vq->stats.guest_notifications_offloaded,
-						1, __ATOMIC_RELAXED);
+					rte_atomic_fetch_add_explicit(
+						&vq->stats.guest_notifications_offloaded,
+						1, rte_memory_order_relaxed);
 				return;
 			}
 
 			/* Offloading failed, fallback to direct IRQ injection */
-			__atomic_store_n(&vq->irq_pending, false, __ATOMIC_RELEASE);
+			rte_atomic_store_explicit(&vq->irq_pending, false,
+				rte_memory_order_release);
 		} else {
 			vq->stats.guest_notifications_suppressed++;
 			return;
@@ -933,14 +936,14 @@ uint64_t translate_log_addr(struct virtio_net *dev, struct vhost_virtqueue *vq,
 
 	if (dev->backend_ops->inject_irq(dev, vq)) {
 		if (dev->flags & VIRTIO_DEV_STATS_ENABLED)
-			__atomic_fetch_add(&vq->stats.guest_notifications_error,
-				1, __ATOMIC_RELAXED);
+			rte_atomic_fetch_add_explicit(&vq->stats.guest_notifications_error,
+				1, rte_memory_order_relaxed);
 		return;
 	}
 
 	if (dev->flags & VIRTIO_DEV_STATS_ENABLED)
-		__atomic_fetch_add(&vq->stats.guest_notifications,
-			1, __ATOMIC_RELAXED);
+		rte_atomic_fetch_add_explicit(&vq->stats.guest_notifications,
+			1, rte_memory_order_relaxed);
 	if (dev->notify_ops->guest_notified)
 		dev->notify_ops->guest_notified(dev->vid);
 }
@@ -949,7 +952,7 @@ uint64_t translate_log_addr(struct virtio_net *dev, struct vhost_virtqueue *vq,
 vhost_vring_call_split(struct virtio_net *dev, struct vhost_virtqueue *vq)
 {
 	/* Flush used->idx update before we read avail->flags. */
-	rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
+	rte_atomic_thread_fence(rte_memory_order_seq_cst);
 
 	/* Don't kick guest if we don't reach index specified by guest. */
 	if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) {
@@ -981,7 +984,7 @@ uint64_t translate_log_addr(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	bool signalled_used_valid, kick = false;
 
 	/* Flush used desc update. */
-	rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
+	rte_atomic_thread_fence(rte_memory_order_seq_cst);
 
 	if (!(dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))) {
 		if (vq->driver_event->flags !=
@@ -1007,7 +1010,7 @@ uint64_t translate_log_addr(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		goto kick;
 	}
 
-	rte_atomic_thread_fence(__ATOMIC_ACQUIRE);
+	rte_atomic_thread_fence(rte_memory_order_acquire);
 
 	off_wrap = vq->driver_event->off_wrap;
 	off = off_wrap & ~(1 << 15);
lib/vhost/vhost_user.c +3 -3
index 901a80b..e363121 100644
@@ -1914,7 +1914,7 @@ static int vhost_user_set_vring_err(struct virtio_net **pdev,
 
 	if (inflight_split->used_idx != used->idx) {
 		inflight_split->desc[last_io].inflight = 0;
-		rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
+		rte_atomic_thread_fence(rte_memory_order_seq_cst);
 		inflight_split->used_idx = used->idx;
 	}
 
@@ -2418,10 +2418,10 @@ static int vhost_user_set_log_fd(struct virtio_net **pdev,
 	 * Set the flag to inject a RARP broadcast packet at
 	 * rte_vhost_dequeue_burst().
 	 *
-	 * __ATOMIC_RELEASE ordering is for making sure the mac is
+	 * rte_memory_order_release ordering is for making sure the mac is
 	 * copied before the flag is set.
 	 */
-	__atomic_store_n(&dev->broadcast_rarp, 1, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&dev->broadcast_rarp, 1, rte_memory_order_release);
 	vdpa_dev = dev->vdpa_dev;
 	if (vdpa_dev && vdpa_dev->ops->migration_done)
 		vdpa_dev->ops->migration_done(dev->vid);
lib/vhost/virtio_net.c +32 -26

166 lines not on this page · raw message

lib/vhost/virtio_net_ctrl.c +4 -2
index 6b583a0..c4847f8 100644
@@ -33,7 +33,8 @@ struct virtio_net_ctrl_elem {
 	uint8_t *ctrl_req;
 	struct vring_desc *descs;
 
-	avail_idx = __atomic_load_n(&cvq->avail->idx, __ATOMIC_ACQUIRE);
+	avail_idx = rte_atomic_load_explicit((unsigned short __rte_atomic *)&cvq->avail->idx,
+		rte_memory_order_acquire);
 	if (avail_idx == cvq->last_avail_idx) {
 		VHOST_LOG_CONFIG(dev->ifname, DEBUG, "Control queue empty\n");
 		return 0;
@@ -236,7 +237,8 @@ struct virtio_net_ctrl_elem {
 	if (cvq->last_used_idx >= cvq->size)
 		cvq->last_used_idx -= cvq->size;
 
-	__atomic_store_n(&cvq->used->idx, cvq->last_used_idx, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit((unsigned short __rte_atomic *)&cvq->used->idx,
+		cvq->last_used_idx, rte_memory_order_release);
 
 	vhost_vring_call_split(dev, dev->cvq);
 
Signature · 1 line
1.8.3.1
#16 lore raw

PATCH 16/21 cryptodev: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/cryptodev/rte_cryptodev.c | 22 ++++++++++++----------
 lib/cryptodev/rte_cryptodev.h | 16 ++++++++--------
 2 files changed, 20 insertions(+), 18 deletions(-)
lib/cryptodev/rte_cryptodev.c +12 -10
index 314710b..b258827 100644
@@ -1535,12 +1535,12 @@ struct rte_cryptodev_cb *
 		/* Stores to cb->fn and cb->param should complete before
 		 * cb is visible to data plane.
 		 */
-		__atomic_store_n(&tail->next, cb, __ATOMIC_RELEASE);
+		rte_atomic_store_explicit(&tail->next, cb, rte_memory_order_release);
 	} else {
 		/* Stores to cb->fn and cb->param should complete before
 		 * cb is visible to data plane.
 		 */
-		__atomic_store_n(&list->next, cb, __ATOMIC_RELEASE);
+		rte_atomic_store_explicit(&list->next, cb, rte_memory_order_release);
 	}
 
 	rte_spinlock_unlock(&rte_cryptodev_callback_lock);
@@ -1555,7 +1555,8 @@ struct rte_cryptodev_cb *
 				  struct rte_cryptodev_cb *cb)
 {
 	struct rte_cryptodev *dev;
-	struct rte_cryptodev_cb **prev_cb, *curr_cb;
+	RTE_ATOMIC(struct rte_cryptodev_cb *) *prev_cb;
+	struct rte_cryptodev_cb *curr_cb;
 	struct rte_cryptodev_cb_rcu *list;
 	int ret;
 
@@ -1601,8 +1602,8 @@ struct rte_cryptodev_cb *
 		curr_cb = *prev_cb;
 		if (curr_cb == cb) {
 			/* Remove the user cb from the callback list. */
-			__atomic_store_n(prev_cb, curr_cb->next,
-				__ATOMIC_RELAXED);
+			rte_atomic_store_explicit(prev_cb, curr_cb->next,
+				rte_memory_order_relaxed);
 			ret = 0;
 			break;
 		}
@@ -1673,12 +1674,12 @@ struct rte_cryptodev_cb *
 		/* Stores to cb->fn and cb->param should complete before
 		 * cb is visible to data plane.
 		 */
-		__atomic_store_n(&tail->next, cb, __ATOMIC_RELEASE);
+		rte_atomic_store_explicit(&tail->next, cb, rte_memory_order_release);
 	} else {
 		/* Stores to cb->fn and cb->param should complete before
 		 * cb is visible to data plane.
 		 */
-		__atomic_store_n(&list->next, cb, __ATOMIC_RELEASE);
+		rte_atomic_store_explicit(&list->next, cb, rte_memory_order_release);
 	}
 
 	rte_spinlock_unlock(&rte_cryptodev_callback_lock);
@@ -1694,7 +1695,8 @@ struct rte_cryptodev_cb *
 				  struct rte_cryptodev_cb *cb)
 {
 	struct rte_cryptodev *dev;
-	struct rte_cryptodev_cb **prev_cb, *curr_cb;
+	RTE_ATOMIC(struct rte_cryptodev_cb *) *prev_cb;
+	struct rte_cryptodev_cb *curr_cb;
 	struct rte_cryptodev_cb_rcu *list;
 	int ret;
 
@@ -1740,8 +1742,8 @@ struct rte_cryptodev_cb *
 		curr_cb = *prev_cb;
 		if (curr_cb == cb) {
 			/* Remove the user cb from the callback list. */
-			__atomic_store_n(prev_cb, curr_cb->next,
-				__ATOMIC_RELAXED);
+			rte_atomic_store_explicit(prev_cb, curr_cb->next,
+				rte_memory_order_relaxed);
 			ret = 0;
 			break;
 		}
lib/cryptodev/rte_cryptodev.h +8 -8
index be0698c..9092118 100644
@@ -979,7 +979,7 @@ struct rte_cryptodev_config {
  * queue pair on enqueue/dequeue.
  */
 struct rte_cryptodev_cb {
-	struct rte_cryptodev_cb *next;
+	RTE_ATOMIC(struct rte_cryptodev_cb *) next;
 	/**< Pointer to next callback */
 	rte_cryptodev_callback_fn fn;
 	/**< Pointer to callback function */
@@ -992,7 +992,7 @@ struct rte_cryptodev_cb {
  * Structure used to hold information about the RCU for a queue pair.
  */
 struct rte_cryptodev_cb_rcu {
-	struct rte_cryptodev_cb *next;
+	RTE_ATOMIC(struct rte_cryptodev_cb *) next;
 	/**< Pointer to next callback */
 	struct rte_rcu_qsbr *qsbr;
 	/**< RCU QSBR variable per queue pair */
@@ -1947,15 +1947,15 @@ int rte_cryptodev_remove_deq_callback(uint8_t dev_id,
 		struct rte_cryptodev_cb_rcu *list;
 		struct rte_cryptodev_cb *cb;
 
-		/* __ATOMIC_RELEASE memory order was used when the
+		/* rte_memory_order_release memory order was used when the
 		 * call back was inserted into the list.
 		 * Since there is a clear dependency between loading
-		 * cb and cb->fn/cb->next, __ATOMIC_ACQUIRE memory order is
+		 * cb and cb->fn/cb->next, rte_memory_order_acquire memory order is
 		 * not required.
 		 */
 		list = &fp_ops->qp.deq_cb[qp_id];
 		rte_rcu_qsbr_thread_online(list->qsbr, 0);
-		cb = __atomic_load_n(&list->next, __ATOMIC_RELAXED);
+		cb = rte_atomic_load_explicit(&list->next, rte_memory_order_relaxed);
 
 		while (cb != NULL) {
 			nb_ops = cb->fn(dev_id, qp_id, ops, nb_ops,
@@ -2014,15 +2014,15 @@ int rte_cryptodev_remove_deq_callback(uint8_t dev_id,
 		struct rte_cryptodev_cb_rcu *list;
 		struct rte_cryptodev_cb *cb;
 
-		/* __ATOMIC_RELEASE memory order was used when the
+		/* rte_memory_order_release memory order was used when the
 		 * call back was inserted into the list.
 		 * Since there is a clear dependency between loading
-		 * cb and cb->fn/cb->next, __ATOMIC_ACQUIRE memory order is
+		 * cb and cb->fn/cb->next, rte_memory_order_acquire memory order is
 		 * not required.
 		 */
 		list = &fp_ops->qp.enq_cb[qp_id];
 		rte_rcu_qsbr_thread_online(list->qsbr, 0);
-		cb = __atomic_load_n(&list->next, __ATOMIC_RELAXED);
+		cb = rte_atomic_load_explicit(&list->next, rte_memory_order_relaxed);
 
 		while (cb != NULL) {
 			nb_ops = cb->fn(dev_id, qp_id, ops, nb_ops,
Signature · 1 line
1.8.3.1
#17 lore raw

PATCH 17/21 distributor: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/distributor/distributor_private.h |  4 +--
 lib/distributor/rte_distributor.c     | 54 +++++++++++++++++------------------
 2 files changed, 29 insertions(+), 29 deletions(-)
lib/distributor/distributor_private.h +2 -2
index 2f29343..dfeb9b5 100644
@@ -113,12 +113,12 @@ enum rte_distributor_match_function {
  * There is a separate cacheline for returns in the burst API.
  */
 struct rte_distributor_buffer {
-	volatile int64_t bufptr64[RTE_DIST_BURST_SIZE]
+	volatile RTE_ATOMIC(int64_t) bufptr64[RTE_DIST_BURST_SIZE]
 		__rte_cache_aligned; /* <= outgoing to worker */
 
 	int64_t pad1 __rte_cache_aligned;    /* <= one cache line  */
 
-	volatile int64_t retptr64[RTE_DIST_BURST_SIZE]
+	volatile RTE_ATOMIC(int64_t) retptr64[RTE_DIST_BURST_SIZE]
 		__rte_cache_aligned; /* <= incoming from worker */
 
 	int64_t pad2 __rte_cache_aligned;    /* <= one cache line  */
lib/distributor/rte_distributor.c +27 -27

161 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#18 lore raw

PATCH 18/21 ethdev: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/ethdev/ethdev_driver.h   | 16 ++++++++--------
 lib/ethdev/ethdev_private.c  |  6 +++---
 lib/ethdev/rte_ethdev.c      | 24 ++++++++++++------------
 lib/ethdev/rte_ethdev.h      | 16 ++++++++--------
 lib/ethdev/rte_ethdev_core.h |  2 +-
 5 files changed, 32 insertions(+), 32 deletions(-)
lib/ethdev/ethdev_driver.h +8 -8
index deb23ad..b482cd1 100644
@@ -30,7 +30,7 @@
  * queue on Rx and Tx.
  */
 struct rte_eth_rxtx_callback {
-	struct rte_eth_rxtx_callback *next;
+	RTE_ATOMIC(struct rte_eth_rxtx_callback *) next;
 	union{
 		rte_rx_callback_fn rx;
 		rte_tx_callback_fn tx;
@@ -80,12 +80,12 @@ struct rte_eth_dev {
 	 * User-supplied functions called from rx_burst to post-process
 	 * received packets before passing them to the user
 	 */
-	struct rte_eth_rxtx_callback *post_rx_burst_cbs[RTE_MAX_QUEUES_PER_PORT];
+	RTE_ATOMIC(struct rte_eth_rxtx_callback *) post_rx_burst_cbs[RTE_MAX_QUEUES_PER_PORT];
 	/**
 	 * User-supplied functions called from tx_burst to pre-process
 	 * received packets before passing them to the driver for transmission
 	 */
-	struct rte_eth_rxtx_callback *pre_tx_burst_cbs[RTE_MAX_QUEUES_PER_PORT];
+	RTE_ATOMIC(struct rte_eth_rxtx_callback *) pre_tx_burst_cbs[RTE_MAX_QUEUES_PER_PORT];
 
 	enum rte_eth_dev_state state; /**< Flag indicating the port state */
 	void *security_ctx; /**< Context for security ops */
@@ -1655,7 +1655,7 @@ int rte_eth_dev_callback_process(struct rte_eth_dev *dev,
 rte_eth_linkstatus_set(struct rte_eth_dev *dev,
 		       const struct rte_eth_link *new_link)
 {
-	uint64_t *dev_link = (uint64_t *)&(dev->data->dev_link);
+	RTE_ATOMIC(uint64_t) *dev_link = (uint64_t __rte_atomic *)&(dev->data->dev_link);
 	union {
 		uint64_t val64;
 		struct rte_eth_link link;
@@ -1663,8 +1663,8 @@ int rte_eth_dev_callback_process(struct rte_eth_dev *dev,
 
 	RTE_BUILD_BUG_ON(sizeof(*new_link) != sizeof(uint64_t));
 
-	orig.val64 = __atomic_exchange_n(dev_link, *(const uint64_t *)new_link,
-					__ATOMIC_SEQ_CST);
+	orig.val64 = rte_atomic_exchange_explicit(dev_link, *(const uint64_t *)new_link,
+					rte_memory_order_seq_cst);
 
 	return (orig.link.link_status == new_link->link_status) ? -1 : 0;
 }
@@ -1682,12 +1682,12 @@ int rte_eth_dev_callback_process(struct rte_eth_dev *dev,
 rte_eth_linkstatus_get(const struct rte_eth_dev *dev,
 		       struct rte_eth_link *link)
 {
-	uint64_t *src = (uint64_t *)&(dev->data->dev_link);
+	RTE_ATOMIC(uint64_t) *src = (uint64_t __rte_atomic *)&(dev->data->dev_link);
 	uint64_t *dst = (uint64_t *)link;
 
 	RTE_BUILD_BUG_ON(sizeof(*link) != sizeof(uint64_t));
 
-	*dst = __atomic_load_n(src, __ATOMIC_SEQ_CST);
+	*dst = rte_atomic_load_explicit(src, rte_memory_order_seq_cst);
 }
 
 /**
lib/ethdev/ethdev_private.c +3 -3
index 7cc7f28..82e2568 100644
@@ -245,7 +245,7 @@ struct dummy_queue {
 void
 eth_dev_fp_ops_reset(struct rte_eth_fp_ops *fpo)
 {
-	static void *dummy_data[RTE_MAX_QUEUES_PER_PORT];
+	static RTE_ATOMIC(void *) dummy_data[RTE_MAX_QUEUES_PER_PORT];
 	uintptr_t port_id = fpo - rte_eth_fp_ops;
 
 	per_port_queues[port_id].rx_warn_once = false;
@@ -278,10 +278,10 @@ struct dummy_queue {
 	fpo->recycle_rx_descriptors_refill = dev->recycle_rx_descriptors_refill;
 
 	fpo->rxq.data = dev->data->rx_queues;
-	fpo->rxq.clbk = (void **)(uintptr_t)dev->post_rx_burst_cbs;
+	fpo->rxq.clbk = (void * __rte_atomic *)(uintptr_t)dev->post_rx_burst_cbs;
 
 	fpo->txq.data = dev->data->tx_queues;
-	fpo->txq.clbk = (void **)(uintptr_t)dev->pre_tx_burst_cbs;
+	fpo->txq.clbk = (void * __rte_atomic *)(uintptr_t)dev->pre_tx_burst_cbs;
 }
 
 uint16_t
lib/ethdev/rte_ethdev.c +12 -12
index 9dabcb5..af23ac0 100644
@@ -5654,9 +5654,9 @@ int rte_eth_rx_avail_thresh_query(uint16_t port_id, uint16_t *queue_id,
 		/* Stores to cb->fn and cb->param should complete before
 		 * cb is visible to data plane.
 		 */
-		__atomic_store_n(
+		rte_atomic_store_explicit(
 			&rte_eth_devices[port_id].post_rx_burst_cbs[queue_id],
-			cb, __ATOMIC_RELEASE);
+			cb, rte_memory_order_release);
 
 	} else {
 		while (tail->next)
@@ -5664,7 +5664,7 @@ int rte_eth_rx_avail_thresh_query(uint16_t port_id, uint16_t *queue_id,
 		/* Stores to cb->fn and cb->param should complete before
 		 * cb is visible to data plane.
 		 */
-		__atomic_store_n(&tail->next, cb, __ATOMIC_RELEASE);
+		rte_atomic_store_explicit(&tail->next, cb, rte_memory_order_release);
 	}
 	rte_spinlock_unlock(&eth_dev_rx_cb_lock);
 
@@ -5704,9 +5704,9 @@ int rte_eth_rx_avail_thresh_query(uint16_t port_id, uint16_t *queue_id,
 	/* Stores to cb->fn, cb->param and cb->next should complete before
 	 * cb is visible to data plane threads.
 	 */
-	__atomic_store_n(
+	rte_atomic_store_explicit(
 		&rte_eth_devices[port_id].post_rx_burst_cbs[queue_id],
-		cb, __ATOMIC_RELEASE);
+		cb, rte_memory_order_release);
 	rte_spinlock_unlock(&eth_dev_rx_cb_lock);
 
 	rte_eth_trace_add_first_rx_callback(port_id, queue_id, fn, user_param,
@@ -5757,9 +5757,9 @@ int rte_eth_rx_avail_thresh_query(uint16_t port_id, uint16_t *queue_id,
 		/* Stores to cb->fn and cb->param should complete before
 		 * cb is visible to data plane.
 		 */
-		__atomic_store_n(
+		rte_atomic_store_explicit(
 			&rte_eth_devices[port_id].pre_tx_burst_cbs[queue_id],
-			cb, __ATOMIC_RELEASE);
+			cb, rte_memory_order_release);
 
 	} else {
 		while (tail->next)
@@ -5767,7 +5767,7 @@ int rte_eth_rx_avail_thresh_query(uint16_t port_id, uint16_t *queue_id,
 		/* Stores to cb->fn and cb->param should complete before
 		 * cb is visible to data plane.
 		 */
-		__atomic_store_n(&tail->next, cb, __ATOMIC_RELEASE);
+		rte_atomic_store_explicit(&tail->next, cb, rte_memory_order_release);
 	}
 	rte_spinlock_unlock(&eth_dev_tx_cb_lock);
 
@@ -5791,7 +5791,7 @@ int rte_eth_rx_avail_thresh_query(uint16_t port_id, uint16_t *queue_id,
 
 	struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 	struct rte_eth_rxtx_callback *cb;
-	struct rte_eth_rxtx_callback **prev_cb;
+	RTE_ATOMIC(struct rte_eth_rxtx_callback *) *prev_cb;
 	int ret = -EINVAL;
 
 	rte_spinlock_lock(&eth_dev_rx_cb_lock);
@@ -5800,7 +5800,7 @@ int rte_eth_rx_avail_thresh_query(uint16_t port_id, uint16_t *queue_id,
 		cb = *prev_cb;
 		if (cb == user_cb) {
 			/* Remove the user cb from the callback list. */
-			__atomic_store_n(prev_cb, cb->next, __ATOMIC_RELAXED);
+			rte_atomic_store_explicit(prev_cb, cb->next, rte_memory_order_relaxed);
 			ret = 0;
 			break;
 		}
@@ -5828,7 +5828,7 @@ int rte_eth_rx_avail_thresh_query(uint16_t port_id, uint16_t *queue_id,
 	struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 	int ret = -EINVAL;
 	struct rte_eth_rxtx_callback *cb;
-	struct rte_eth_rxtx_callback **prev_cb;
+	RTE_ATOMIC(struct rte_eth_rxtx_callback *) *prev_cb;
 
 	rte_spinlock_lock(&eth_dev_tx_cb_lock);
 	prev_cb = &dev->pre_tx_burst_cbs[queue_id];
@@ -5836,7 +5836,7 @@ int rte_eth_rx_avail_thresh_query(uint16_t port_id, uint16_t *queue_id,
 		cb = *prev_cb;
 		if (cb == user_cb) {
 			/* Remove the user cb from the callback list. */
-			__atomic_store_n(prev_cb, cb->next, __ATOMIC_RELAXED);
+			rte_atomic_store_explicit(prev_cb, cb->next, rte_memory_order_relaxed);
 			ret = 0;
 			break;
 		}
lib/ethdev/rte_ethdev.h +8 -8
index f949dfc..ec48b24 100644
@@ -6018,14 +6018,14 @@ uint16_t rte_eth_call_rx_callbacks(uint16_t port_id, uint16_t queue_id,
 	{
 		void *cb;
 
-		/* __ATOMIC_RELEASE memory order was used when the
+		/* rte_memory_order_release memory order was used when the
 		 * call back was inserted into the list.
 		 * Since there is a clear dependency between loading
-		 * cb and cb->fn/cb->next, __ATOMIC_ACQUIRE memory order is
+		 * cb and cb->fn/cb->next, rte_memory_order_acquire memory order is
 		 * not required.
 		 */
-		cb = __atomic_load_n((void **)&p->rxq.clbk[queue_id],
-				__ATOMIC_RELAXED);
+		cb = rte_atomic_load_explicit(&p->rxq.clbk[queue_id],
+				rte_memory_order_relaxed);
 		if (unlikely(cb != NULL))
 			nb_rx = rte_eth_call_rx_callbacks(port_id, queue_id,
 					rx_pkts, nb_rx, nb_pkts, cb);
@@ -6355,14 +6355,14 @@ uint16_t rte_eth_call_tx_callbacks(uint16_t port_id, uint16_t queue_id,
 	{
 		void *cb;
 
-		/* __ATOMIC_RELEASE memory order was used when the
+		/* rte_memory_order_release memory order was used when the
 		 * call back was inserted into the list.
 		 * Since there is a clear dependency between loading
-		 * cb and cb->fn/cb->next, __ATOMIC_ACQUIRE memory order is
+		 * cb and cb->fn/cb->next, rte_memory_order_acquire memory order is
 		 * not required.
 		 */
-		cb = __atomic_load_n((void **)&p->txq.clbk[queue_id],
-				__ATOMIC_RELAXED);
+		cb = rte_atomic_load_explicit(&p->txq.clbk[queue_id],
+				rte_memory_order_relaxed);
 		if (unlikely(cb != NULL))
 			nb_pkts = rte_eth_call_tx_callbacks(port_id, queue_id,
 					tx_pkts, nb_pkts, cb);
lib/ethdev/rte_ethdev_core.h +1 -1
index 32f5f73..4bfaf79 100644
@@ -71,7 +71,7 @@ struct rte_ethdev_qdata {
 	/** points to array of internal queue data pointers */
 	void **data;
 	/** points to array of queue callback data pointers */
-	void **clbk;
+	RTE_ATOMIC(void *) *clbk;
 };
 
 /**
Signature · 1 line
1.8.3.1
#19 lore raw

PATCH 19/21 hash: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/hash/rte_cuckoo_hash.c | 116 ++++++++++++++++++++++-----------------------
 lib/hash/rte_cuckoo_hash.h |   6 +--
 2 files changed, 61 insertions(+), 61 deletions(-)
lib/hash/rte_cuckoo_hash.c +58 -58
index 19b23f2..b2cf60d 100644
@@ -149,7 +149,7 @@ struct rte_hash *
 	unsigned int writer_takes_lock = 0;
 	unsigned int no_free_on_del = 0;
 	uint32_t *ext_bkt_to_free = NULL;
-	uint32_t *tbl_chng_cnt = NULL;
+	RTE_ATOMIC(uint32_t) *tbl_chng_cnt = NULL;
 	struct lcore_cache *local_free_slots = NULL;
 	unsigned int readwrite_concur_lf_support = 0;
 	uint32_t i;
@@ -713,9 +713,9 @@ struct rte_hash *
 				 * variable. Release the application data
 				 * to the readers.
 				 */
-				__atomic_store_n(&k->pdata,
+				rte_atomic_store_explicit(&k->pdata,
 					data,
-					__ATOMIC_RELEASE);
+					rte_memory_order_release);
 				/*
 				 * Return index where key is stored,
 				 * subtracting the first dummy index
@@ -776,9 +776,9 @@ struct rte_hash *
 			 * key_idx is the guard variable for signature
 			 * and key.
 			 */
-			__atomic_store_n(&prim_bkt->key_idx[i],
+			rte_atomic_store_explicit(&prim_bkt->key_idx[i],
 					 new_idx,
-					 __ATOMIC_RELEASE);
+					 rte_memory_order_release);
 			break;
 		}
 	}
@@ -851,9 +851,9 @@ struct rte_hash *
 		if (unlikely(&h->buckets[prev_alt_bkt_idx]
 				!= curr_bkt)) {
 			/* revert it to empty, otherwise duplicated keys */
-			__atomic_store_n(&curr_bkt->key_idx[curr_slot],
+			rte_atomic_store_explicit(&curr_bkt->key_idx[curr_slot],
 				EMPTY_SLOT,
-				__ATOMIC_RELEASE);
+				rte_memory_order_release);
 			__hash_rw_writer_unlock(h);
 			return -1;
 		}
@@ -865,13 +865,13 @@ struct rte_hash *
 			 * Since there is one writer, load acquires on
 			 * tbl_chng_cnt are not required.
 			 */
-			__atomic_store_n(h->tbl_chng_cnt,
+			rte_atomic_store_explicit(h->tbl_chng_cnt,
 					 *h->tbl_chng_cnt + 1,
-					 __ATOMIC_RELEASE);
+					 rte_memory_order_release);
 			/* The store to sig_current should not
 			 * move above the store to tbl_chng_cnt.
 			 */
-			__atomic_thread_fence(__ATOMIC_RELEASE);
+			__atomic_thread_fence(rte_memory_order_release);
 		}
 
 		/* Need to swap current/alt sig to allow later
@@ -881,9 +881,9 @@ struct rte_hash *
 		curr_bkt->sig_current[curr_slot] =
 			prev_bkt->sig_current[prev_slot];
 		/* Release the updated bucket entry */
-		__atomic_store_n(&curr_bkt->key_idx[curr_slot],
+		rte_atomic_store_explicit(&curr_bkt->key_idx[curr_slot],
 			prev_bkt->key_idx[prev_slot],
-			__ATOMIC_RELEASE);
+			rte_memory_order_release);
 
 		curr_slot = prev_slot;
 		curr_node = prev_node;
@@ -897,20 +897,20 @@ struct rte_hash *
 		 * Since there is one writer, load acquires on
 		 * tbl_chng_cnt are not required.
 		 */
-		__atomic_store_n(h->tbl_chng_cnt,
+		rte_atomic_store_explicit(h->tbl_chng_cnt,
 				 *h->tbl_chng_cnt + 1,
-				 __ATOMIC_RELEASE);
+				 rte_memory_order_release);
 		/* The store to sig_current should not
 		 * move above the store to tbl_chng_cnt.
 		 */
-		__atomic_thread_fence(__ATOMIC_RELEASE);
+		__atomic_thread_fence(rte_memory_order_release);
 	}
 
 	curr_bkt->sig_current[curr_slot] = sig;
 	/* Release the new bucket entry */
-	__atomic_store_n(&curr_bkt->key_idx[curr_slot],
+	rte_atomic_store_explicit(&curr_bkt->key_idx[curr_slot],
 			 new_idx,
-			 __ATOMIC_RELEASE);
+			 rte_memory_order_release);
 
 	__hash_rw_writer_unlock(h);
 
@@ -1076,9 +1076,9 @@ struct rte_hash *
 	 * not leak after the store of pdata in the key store. i.e. pdata is
 	 * the guard variable. Release the application data to the readers.
 	 */
-	__atomic_store_n(&new_k->pdata,
+	rte_atomic_store_explicit(&new_k->pdata,
 		data,
-		__ATOMIC_RELEASE);
+		rte_memory_order_release);
 	/* Copy key */
 	memcpy(new_k->key, key, h->key_len);
 
@@ -1149,9 +1149,9 @@ struct rte_hash *
 				 * key_idx is the guard variable for signature
 				 * and key.
 				 */
-				__atomic_store_n(&cur_bkt->key_idx[i],
+				rte_atomic_store_explicit(&cur_bkt->key_idx[i],
 						 slot_id,
-						 __ATOMIC_RELEASE);
+						 rte_memory_order_release);
 				__hash_rw_writer_unlock(h);
 				return slot_id - 1;
 			}
@@ -1185,9 +1185,9 @@ struct rte_hash *
 	 * the store to key_idx. i.e. key_idx is the guard variable
 	 * for signature and key.
 	 */
-	__atomic_store_n(&(h->buckets_ext[ext_bkt_id - 1]).key_idx[0],
+	rte_atomic_store_explicit(&(h->buckets_ext[ext_bkt_id - 1]).key_idx[0],
 			 slot_id,
-			 __ATOMIC_RELEASE);
+			 rte_memory_order_release);
 	/* Link the new bucket to sec bucket linked list */
 	last = rte_hash_get_last_bkt(sec_bkt);
 	last->next = &h->buckets_ext[ext_bkt_id - 1];
@@ -1290,17 +1290,17 @@ struct rte_hash *
 		 * key comparison will ensure that the lookup fails.
 		 */
 		if (bkt->sig_current[i] == sig) {
-			key_idx = __atomic_load_n(&bkt->key_idx[i],
-					  __ATOMIC_ACQUIRE);
+			key_idx = rte_atomic_load_explicit(&bkt->key_idx[i],
+					  rte_memory_order_acquire);
 			if (key_idx != EMPTY_SLOT) {
 				k = (struct rte_hash_key *) ((char *)keys +
 						key_idx * h->key_entry_size);
 
 				if (rte_hash_cmp_eq(key, k->key, h) == 0) {
 					if (data != NULL) {
-						*data = __atomic_load_n(
+						*data = rte_atomic_load_explicit(
 							&k->pdata,
-							__ATOMIC_ACQUIRE);
+							rte_memory_order_acquire);
 					}
 					/*
 					 * Return index where key is stored,
@@ -1374,8 +1374,8 @@ struct rte_hash *
 		 * starts. Acquire semantics will make sure that
 		 * loads in search_one_bucket are not hoisted.
 		 */
-		cnt_b = __atomic_load_n(h->tbl_chng_cnt,
-				__ATOMIC_ACQUIRE);
+		cnt_b = rte_atomic_load_explicit(h->tbl_chng_cnt,
+				rte_memory_order_acquire);
 
 		/* Check if key is in primary location */
 		bkt = &h->buckets[prim_bucket_idx];
@@ -1396,7 +1396,7 @@ struct rte_hash *
 		/* The loads of sig_current in search_one_bucket
 		 * should not move below the load from tbl_chng_cnt.
 		 */
-		__atomic_thread_fence(__ATOMIC_ACQUIRE);
+		__atomic_thread_fence(rte_memory_order_acquire);
 		/* Re-read the table change counter to check if the
 		 * table has changed during search. If yes, re-do
 		 * the search.
@@ -1405,8 +1405,8 @@ struct rte_hash *
 		 * and key index in secondary bucket will make sure
 		 * that it does not get hoisted.
 		 */
-		cnt_a = __atomic_load_n(h->tbl_chng_cnt,
-					__ATOMIC_ACQUIRE);
+		cnt_a = rte_atomic_load_explicit(h->tbl_chng_cnt,
+					rte_memory_order_acquire);
 	} while (cnt_b != cnt_a);
 
 	return -ENOENT;
@@ -1611,26 +1611,26 @@ struct rte_hash *
 	for (i = RTE_HASH_BUCKET_ENTRIES - 1; i >= 0; i--) {
 		if (last_bkt->key_idx[i] != EMPTY_SLOT) {
 			cur_bkt->sig_current[pos] = last_bkt->sig_current[i];
-			__atomic_store_n(&cur_bkt->key_idx[pos],
+			rte_atomic_store_explicit(&cur_bkt->key_idx[pos],
 					 last_bkt->key_idx[i],
-					 __ATOMIC_RELEASE);
+					 rte_memory_order_release);
 			if (h->readwrite_concur_lf_support) {
 				/* Inform the readers that the table has changed
 				 * Since there is one writer, load acquire on
 				 * tbl_chng_cnt is not required.
 				 */
-				__atomic_store_n(h->tbl_chng_cnt,
+				rte_atomic_store_explicit(h->tbl_chng_cnt,
 					 *h->tbl_chng_cnt + 1,
-					 __ATOMIC_RELEASE);
+					 rte_memory_order_release);
 				/* The store to sig_current should
 				 * not move above the store to tbl_chng_cnt.
 				 */
-				__atomic_thread_fence(__ATOMIC_RELEASE);
+				__atomic_thread_fence(rte_memory_order_release);
 			}
 			last_bkt->sig_current[i] = NULL_SIGNATURE;
-			__atomic_store_n(&last_bkt->key_idx[i],
+			rte_atomic_store_explicit(&last_bkt->key_idx[i],
 					 EMPTY_SLOT,
-					 __ATOMIC_RELEASE);
+					 rte_memory_order_release);
 			return;
 		}
 	}
@@ -1650,8 +1650,8 @@ struct rte_hash *
 
 	/* Check if key is in bucket */
 	for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
-		key_idx = __atomic_load_n(&bkt->key_idx[i],
-					  __ATOMIC_ACQUIRE);
+		key_idx = rte_atomic_load_explicit(&bkt->key_idx[i],
+					  rte_memory_order_acquire);
 		if (bkt->sig_current[i] == sig && key_idx != EMPTY_SLOT) {
 			k = (struct rte_hash_key *) ((char *)keys +
 					key_idx * h->key_entry_size);
@@ -1663,9 +1663,9 @@ struct rte_hash *
 				if (!h->no_free_on_del)
 					remove_entry(h, bkt, i);
 
-				__atomic_store_n(&bkt->key_idx[i],
+				rte_atomic_store_explicit(&bkt->key_idx[i],
 						 EMPTY_SLOT,
-						 __ATOMIC_RELEASE);
+						 rte_memory_order_release);
 
 				*pos = i;
 				/*
@@ -2077,8 +2077,8 @@ struct rte_hash *
 		 * starts. Acquire semantics will make sure that
 		 * loads in compare_signatures are not hoisted.
 		 */
-		cnt_b = __atomic_load_n(h->tbl_chng_cnt,
-					__ATOMIC_ACQUIRE);
+		cnt_b = rte_atomic_load_explicit(h->tbl_chng_cnt,
+					rte_memory_order_acquire);
 
 		/* Compare signatures and prefetch key slot of first hit */
 		for (i = 0; i < num_keys; i++) {
@@ -2121,9 +2121,9 @@ struct rte_hash *
 						__builtin_ctzl(prim_hitmask[i])
 						>> 1;
 				uint32_t key_idx =
-				__atomic_load_n(
+				rte_atomic_load_explicit(
 					&primary_bkt[i]->key_idx[hit_index],
-					__ATOMIC_ACQUIRE);
+					rte_memory_order_acquire);
 				const struct rte_hash_key *key_slot =
 					(const struct rte_hash_key *)(
 					(const char *)h->key_store +
@@ -2137,9 +2137,9 @@ struct rte_hash *
 					!rte_hash_cmp_eq(
 						key_slot->key, keys[i], h)) {
 					if (data != NULL)
-						data[i] = __atomic_load_n(
+						data[i] = rte_atomic_load_explicit(
 							&key_slot->pdata,
-							__ATOMIC_ACQUIRE);
+							rte_memory_order_acquire);
 
 					hits |= 1ULL << i;
 					positions[i] = key_idx - 1;
@@ -2153,9 +2153,9 @@ struct rte_hash *
 						__builtin_ctzl(sec_hitmask[i])
 						>> 1;
 				uint32_t key_idx =
-				__atomic_load_n(
+				rte_atomic_load_explicit(
 					&secondary_bkt[i]->key_idx[hit_index],
-					__ATOMIC_ACQUIRE);
+					rte_memory_order_acquire);
 				const struct rte_hash_key *key_slot =
 					(const struct rte_hash_key *)(
 					(const char *)h->key_store +
@@ -2170,9 +2170,9 @@ struct rte_hash *
 					!rte_hash_cmp_eq(
 						key_slot->key, keys[i], h)) {
 					if (data != NULL)
-						data[i] = __atomic_load_n(
+						data[i] = rte_atomic_load_explicit(
 							&key_slot->pdata,
-							__ATOMIC_ACQUIRE);
+							rte_memory_order_acquire);
 
 					hits |= 1ULL << i;
 					positions[i] = key_idx - 1;
@@ -2216,7 +2216,7 @@ struct rte_hash *
 		/* The loads of sig_current in compare_signatures
 		 * should not move below the load from tbl_chng_cnt.
 		 */
-		__atomic_thread_fence(__ATOMIC_ACQUIRE);
+		__atomic_thread_fence(rte_memory_order_acquire);
 		/* Re-read the table change counter to check if the
 		 * table has changed during search. If yes, re-do
 		 * the search.
@@ -2225,8 +2225,8 @@ struct rte_hash *
 		 * key index will make sure that it does not get
 		 * hoisted.
 		 */
-		cnt_a = __atomic_load_n(h->tbl_chng_cnt,
-					__ATOMIC_ACQUIRE);
+		cnt_a = rte_atomic_load_explicit(h->tbl_chng_cnt,
+					rte_memory_order_acquire);
 	} while (cnt_b != cnt_a);
 
 	if (hit_mask != NULL)
@@ -2498,8 +2498,8 @@ struct rte_hash *
 	idx = *next % RTE_HASH_BUCKET_ENTRIES;
 
 	/* If current position is empty, go to the next one */
-	while ((position = __atomic_load_n(&h->buckets[bucket_idx].key_idx[idx],
-					__ATOMIC_ACQUIRE)) == EMPTY_SLOT) {
+	while ((position = rte_atomic_load_explicit(&h->buckets[bucket_idx].key_idx[idx],
+					rte_memory_order_acquire)) == EMPTY_SLOT) {
 		(*next)++;
 		/* End of table */
 		if (*next == total_entries_main)
lib/hash/rte_cuckoo_hash.h +3 -3
index eb2644f..f7afc4d 100644
@@ -137,7 +137,7 @@ struct lcore_cache {
 struct rte_hash_key {
 	union {
 		uintptr_t idata;
-		void *pdata;
+		RTE_ATOMIC(void *) pdata;
 	};
 	/* Variable key size */
 	char key[0];
@@ -155,7 +155,7 @@ enum rte_hash_sig_compare_function {
 struct rte_hash_bucket {
 	uint16_t sig_current[RTE_HASH_BUCKET_ENTRIES];
 
-	uint32_t key_idx[RTE_HASH_BUCKET_ENTRIES];
+	RTE_ATOMIC(uint32_t) key_idx[RTE_HASH_BUCKET_ENTRIES];
 
 	uint8_t flag[RTE_HASH_BUCKET_ENTRIES];
 
@@ -229,7 +229,7 @@ struct rte_hash {
 	 * is piggy-backed to freeing of the key index.
 	 */
 	uint32_t *ext_bkt_to_free;
-	uint32_t *tbl_chng_cnt;
+	RTE_ATOMIC(uint32_t) *tbl_chng_cnt;
 	/**< Indicates if the hash table changed from last read. */
 } __rte_cache_aligned;
 
Signature · 1 line
1.8.3.1
#20 lore raw

PATCH 20/21 timer: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/timer/rte_timer.c | 50 +++++++++++++++++++++++++-------------------------
 lib/timer/rte_timer.h |  6 +++---
 2 files changed, 28 insertions(+), 28 deletions(-)
lib/timer/rte_timer.c +25 -25
index 85d6757..53ed221 100644
@@ -210,7 +210,7 @@ struct rte_timer_data {
 
 	status.state = RTE_TIMER_STOP;
 	status.owner = RTE_TIMER_NO_OWNER;
-	__atomic_store_n(&tim->status.u32, status.u32, __ATOMIC_RELAXED);
+	rte_atomic_store_explicit(&tim->status.u32, status.u32, rte_memory_order_relaxed);
 }
 
 /*
@@ -231,7 +231,7 @@ struct rte_timer_data {
 
 	/* wait that the timer is in correct status before update,
 	 * and mark it as being configured */
-	prev_status.u32 = __atomic_load_n(&tim->status.u32, __ATOMIC_RELAXED);
+	prev_status.u32 = rte_atomic_load_explicit(&tim->status.u32, rte_memory_order_relaxed);
 
 	while (success == 0) {
 		/* timer is running on another core
@@ -254,11 +254,11 @@ struct rte_timer_data {
 		 * timer is in CONFIG state, the state cannot be changed
 		 * by other threads. So, we should use ACQUIRE here.
 		 */
-		success = __atomic_compare_exchange_n(&tim->status.u32,
-					      &prev_status.u32,
-					      status.u32, 0,
-					      __ATOMIC_ACQUIRE,
-					      __ATOMIC_RELAXED);
+		success = rte_atomic_compare_exchange_strong_explicit(&tim->status.u32,
+					      (uint32_t *)(uintptr_t)&prev_status.u32,
+					      status.u32,
+					      rte_memory_order_acquire,
+					      rte_memory_order_relaxed);
 	}
 
 	ret_prev_status->u32 = prev_status.u32;
@@ -277,7 +277,7 @@ struct rte_timer_data {
 
 	/* wait that the timer is in correct status before update,
 	 * and mark it as running */
-	prev_status.u32 = __atomic_load_n(&tim->status.u32, __ATOMIC_RELAXED);
+	prev_status.u32 = rte_atomic_load_explicit(&tim->status.u32, rte_memory_order_relaxed);
 
 	while (success == 0) {
 		/* timer is not pending anymore */
@@ -293,11 +293,11 @@ struct rte_timer_data {
 		 * timer is in RUNNING state, the state cannot be changed
 		 * by other threads. So, we should use ACQUIRE here.
 		 */
-		success = __atomic_compare_exchange_n(&tim->status.u32,
-					      &prev_status.u32,
-					      status.u32, 0,
-					      __ATOMIC_ACQUIRE,
-					      __ATOMIC_RELAXED);
+		success = rte_atomic_compare_exchange_strong_explicit(&tim->status.u32,
+					      (uint32_t *)(uintptr_t)&prev_status.u32,
+					      status.u32,
+					      rte_memory_order_acquire,
+					      rte_memory_order_relaxed);
 	}
 
 	return 0;
@@ -530,7 +530,7 @@ struct rte_timer_data {
 	/* The "RELEASE" ordering guarantees the memory operations above
 	 * the status update are observed before the update by all threads
 	 */
-	__atomic_store_n(&tim->status.u32, status.u32, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&tim->status.u32, status.u32, rte_memory_order_release);
 
 	if (tim_lcore != lcore_id || !local_is_locked)
 		rte_spinlock_unlock(&priv_timer[tim_lcore].list_lock);
@@ -612,7 +612,7 @@ struct rte_timer_data {
 	/* The "RELEASE" ordering guarantees the memory operations above
 	 * the status update are observed before the update by all threads
 	 */
-	__atomic_store_n(&tim->status.u32, status.u32, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&tim->status.u32, status.u32, rte_memory_order_release);
 
 	return 0;
 }
@@ -646,8 +646,8 @@ struct rte_timer_data {
 int
 rte_timer_pending(struct rte_timer *tim)
 {
-	return __atomic_load_n(&tim->status.state,
-				__ATOMIC_RELAXED) == RTE_TIMER_PENDING;
+	return rte_atomic_load_explicit(&tim->status.state,
+				rte_memory_order_relaxed) == RTE_TIMER_PENDING;
 }
 
 /* must be called periodically, run all timer that expired */
@@ -753,8 +753,8 @@ struct rte_timer_data {
 			 * operations above the status update are observed
 			 * before the update by all threads
 			 */
-			__atomic_store_n(&tim->status.u32, status.u32,
-				__ATOMIC_RELEASE);
+			rte_atomic_store_explicit(&tim->status.u32, status.u32,
+				rte_memory_order_release);
 		}
 		else {
 			/* keep it in list and mark timer as pending */
@@ -766,8 +766,8 @@ struct rte_timer_data {
 			 * operations above the status update are observed
 			 * before the update by all threads
 			 */
-			__atomic_store_n(&tim->status.u32, status.u32,
-				__ATOMIC_RELEASE);
+			rte_atomic_store_explicit(&tim->status.u32, status.u32,
+				rte_memory_order_release);
 			__rte_timer_reset(tim, tim->expire + tim->period,
 				tim->period, lcore_id, tim->f, tim->arg, 1,
 				timer_data);
@@ -941,8 +941,8 @@ struct rte_timer_data {
 			 * operations above the status update are observed
 			 * before the update by all threads
 			 */
-			__atomic_store_n(&tim->status.u32, status.u32,
-				__ATOMIC_RELEASE);
+			rte_atomic_store_explicit(&tim->status.u32, status.u32,
+				rte_memory_order_release);
 		} else {
 			/* keep it in list and mark timer as pending */
 			rte_spinlock_lock(
@@ -954,8 +954,8 @@ struct rte_timer_data {
 			 * operations above the status update are observed
 			 * before the update by all threads
 			 */
-			__atomic_store_n(&tim->status.u32, status.u32,
-				__ATOMIC_RELEASE);
+			rte_atomic_store_explicit(&tim->status.u32, status.u32,
+				rte_memory_order_release);
 			__rte_timer_reset(tim, tim->expire + tim->period,
 				tim->period, this_lcore, tim->f, tim->arg, 1,
 				data);
lib/timer/rte_timer.h +3 -3
index d3927d5..a35bc08 100644
@@ -65,10 +65,10 @@ enum rte_timer_type {
  */
 union rte_timer_status {
 	struct {
-		uint16_t state;  /**< Stop, pending, running, config. */
-		int16_t owner;   /**< The lcore that owns the timer. */
+		RTE_ATOMIC(uint16_t) state;  /**< Stop, pending, running, config. */
+		RTE_ATOMIC(int16_t) owner;   /**< The lcore that owns the timer. */
 	};
-	uint32_t u32;            /**< To atomic-set status + owner. */
+	RTE_ATOMIC(uint32_t) u32;            /**< To atomic-set status + owner. */
 };
 
 #ifdef RTE_LIBRTE_TIMER_DEBUG
Signature · 1 line
1.8.3.1
#21 lore raw

PATCH 21/21 ring: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 drivers/net/mlx5/mlx5_hws_cnt.h   |  2 +-
 lib/ring/rte_ring_c11_pvt.h       | 33 +++++++++++++++++----------------
 lib/ring/rte_ring_core.h          | 10 +++++-----
 lib/ring/rte_ring_generic_pvt.h   |  3 ++-
 lib/ring/rte_ring_hts_elem_pvt.h  | 22 ++++++++++++----------
 lib/ring/rte_ring_peek_elem_pvt.h |  6 +++---
 lib/ring/rte_ring_rts_elem_pvt.h  | 27 ++++++++++++++-------------
 7 files changed, 54 insertions(+), 49 deletions(-)
drivers/net/mlx5/mlx5_hws_cnt.h +1 -1
index f462665..cc9ac10 100644
@@ -394,7 +394,7 @@ struct mlx5_hws_age_param {
 	__rte_ring_get_elem_addr(r, revert2head, sizeof(cnt_id_t), n,
 			&zcd->ptr1, &zcd->n1, &zcd->ptr2);
 	/* Update tail */
-	__atomic_store_n(&r->prod.tail, revert2head, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&r->prod.tail, revert2head, rte_memory_order_release);
 	return n;
 }
 
lib/ring/rte_ring_c11_pvt.h +17 -16
index f895950..f8be538 100644
@@ -22,9 +22,10 @@
 	 * we need to wait for them to complete
 	 */
 	if (!single)
-		rte_wait_until_equal_32(&ht->tail, old_val, __ATOMIC_RELAXED);
+		rte_wait_until_equal_32((volatile uint32_t *)(uintptr_t)&ht->tail, old_val,
+			rte_memory_order_relaxed);
 
-	__atomic_store_n(&ht->tail, new_val, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&ht->tail, new_val, rte_memory_order_release);
 }
 
 /**
@@ -61,19 +62,19 @@
 	unsigned int max = n;
 	int success;
 
-	*old_head = __atomic_load_n(&r->prod.head, __ATOMIC_RELAXED);
+	*old_head = rte_atomic_load_explicit(&r->prod.head, rte_memory_order_relaxed);
 	do {
 		/* Reset n to the initial burst count */
 		n = max;
 
 		/* Ensure the head is read before tail */
-		__atomic_thread_fence(__ATOMIC_ACQUIRE);
+		__atomic_thread_fence(rte_memory_order_acquire);
 
 		/* load-acquire synchronize with store-release of ht->tail
 		 * in update_tail.
 		 */
-		cons_tail = __atomic_load_n(&r->cons.tail,
-					__ATOMIC_ACQUIRE);
+		cons_tail = rte_atomic_load_explicit(&r->cons.tail,
+					rte_memory_order_acquire);
 
 		/* The subtraction is done between two unsigned 32bits value
 		 * (the result is always modulo 32 bits even if we have
@@ -95,10 +96,10 @@
 			r->prod.head = *new_head, success = 1;
 		else
 			/* on failure, *old_head is updated */
-			success = __atomic_compare_exchange_n(&r->prod.head,
+			success = rte_atomic_compare_exchange_strong_explicit(&r->prod.head,
 					old_head, *new_head,
-					0, __ATOMIC_RELAXED,
-					__ATOMIC_RELAXED);
+					rte_memory_order_relaxed,
+					rte_memory_order_relaxed);
 	} while (unlikely(success == 0));
 	return n;
 }
@@ -137,19 +138,19 @@
 	int success;
 
 	/* move cons.head atomically */
-	*old_head = __atomic_load_n(&r->cons.head, __ATOMIC_RELAXED);
+	*old_head = rte_atomic_load_explicit(&r->cons.head, rte_memory_order_relaxed);
 	do {
 		/* Restore n as it may change every loop */
 		n = max;
 
 		/* Ensure the head is read before tail */
-		__atomic_thread_fence(__ATOMIC_ACQUIRE);
+		__atomic_thread_fence(rte_memory_order_acquire);
 
 		/* this load-acquire synchronize with store-release of ht->tail
 		 * in update_tail.
 		 */
-		prod_tail = __atomic_load_n(&r->prod.tail,
-					__ATOMIC_ACQUIRE);
+		prod_tail = rte_atomic_load_explicit(&r->prod.tail,
+					rte_memory_order_acquire);
 
 		/* The subtraction is done between two unsigned 32bits value
 		 * (the result is always modulo 32 bits even if we have
@@ -170,10 +171,10 @@
 			r->cons.head = *new_head, success = 1;
 		else
 			/* on failure, *old_head will be updated */
-			success = __atomic_compare_exchange_n(&r->cons.head,
+			success = rte_atomic_compare_exchange_strong_explicit(&r->cons.head,
 							old_head, *new_head,
-							0, __ATOMIC_RELAXED,
-							__ATOMIC_RELAXED);
+							rte_memory_order_relaxed,
+							rte_memory_order_relaxed);
 	} while (unlikely(success == 0));
 	return n;
 }
lib/ring/rte_ring_core.h +5 -5
index 327fdcf..7a2b577 100644
@@ -67,7 +67,7 @@ enum rte_ring_sync_type {
  */
 struct rte_ring_headtail {
 	volatile uint32_t head;      /**< prod/consumer head. */
-	volatile uint32_t tail;      /**< prod/consumer tail. */
+	volatile RTE_ATOMIC(uint32_t) tail;      /**< prod/consumer tail. */
 	union {
 		/** sync type of prod/cons */
 		enum rte_ring_sync_type sync_type;
@@ -78,7 +78,7 @@ struct rte_ring_headtail {
 
 union __rte_ring_rts_poscnt {
 	/** raw 8B value to read/write *cnt* and *pos* as one atomic op */
-	uint64_t raw __rte_aligned(8);
+	RTE_ATOMIC(uint64_t) raw __rte_aligned(8);
 	struct {
 		uint32_t cnt; /**< head/tail reference counter */
 		uint32_t pos; /**< head/tail position */
@@ -94,10 +94,10 @@ struct rte_ring_rts_headtail {
 
 union __rte_ring_hts_pos {
 	/** raw 8B value to read/write *head* and *tail* as one atomic op */
-	uint64_t raw __rte_aligned(8);
+	RTE_ATOMIC(uint64_t) raw __rte_aligned(8);
 	struct {
-		uint32_t head; /**< head position */
-		uint32_t tail; /**< tail position */
+		RTE_ATOMIC(uint32_t) head; /**< head position */
+		RTE_ATOMIC(uint32_t) tail; /**< tail position */
 	} pos;
 };
 
lib/ring/rte_ring_generic_pvt.h +2 -1
index 5acb6e5..ffb3654 100644
@@ -23,7 +23,8 @@
 	 * we need to wait for them to complete
 	 */
 	if (!single)
-		rte_wait_until_equal_32(&ht->tail, old_val, __ATOMIC_RELAXED);
+		rte_wait_until_equal_32((volatile uint32_t *)(uintptr_t)&ht->tail, old_val,
+			rte_memory_order_relaxed);
 
 	ht->tail = new_val;
 }
lib/ring/rte_ring_hts_elem_pvt.h +12 -10
index a8678d3..91f5eec 100644
@@ -10,6 +10,8 @@
 #ifndef _RTE_RING_HTS_ELEM_PVT_H_
 #define _RTE_RING_HTS_ELEM_PVT_H_
 
+#include <rte_stdatomic.h>
+
 /**
  * @file rte_ring_hts_elem_pvt.h
  * It is not recommended to include this file directly,
@@ -30,7 +32,7 @@
 	RTE_SET_USED(enqueue);
 
 	tail = old_tail + num;
-	__atomic_store_n(&ht->ht.pos.tail, tail, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&ht->ht.pos.tail, tail, rte_memory_order_release);
 }
 
 /**
@@ -44,7 +46,7 @@
 {
 	while (p->pos.head != p->pos.tail) {
 		rte_pause();
-		p->raw = __atomic_load_n(&ht->ht.raw, __ATOMIC_ACQUIRE);
+		p->raw = rte_atomic_load_explicit(&ht->ht.raw, rte_memory_order_acquire);
 	}
 }
 
@@ -61,7 +63,7 @@
 
 	const uint32_t capacity = r->capacity;
 
-	op.raw = __atomic_load_n(&r->hts_prod.ht.raw, __ATOMIC_ACQUIRE);
+	op.raw = rte_atomic_load_explicit(&r->hts_prod.ht.raw, rte_memory_order_acquire);
 
 	do {
 		/* Reset n to the initial burst count */
@@ -98,9 +100,9 @@
 	 *  - OOO reads of cons tail value
 	 *  - OOO copy of elems from the ring
 	 */
-	} while (__atomic_compare_exchange_n(&r->hts_prod.ht.raw,
-			&op.raw, np.raw,
-			0, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE) == 0);
+	} while (rte_atomic_compare_exchange_strong_explicit(&r->hts_prod.ht.raw,
+			(uint64_t *)(uintptr_t)&op.raw, np.raw,
+			rte_memory_order_acquire, rte_memory_order_acquire) == 0);
 
 	*old_head = op.pos.head;
 	return n;
@@ -117,7 +119,7 @@
 	uint32_t n;
 	union __rte_ring_hts_pos np, op;
 
-	op.raw = __atomic_load_n(&r->hts_cons.ht.raw, __ATOMIC_ACQUIRE);
+	op.raw = rte_atomic_load_explicit(&r->hts_cons.ht.raw, rte_memory_order_acquire);
 
 	/* move cons.head atomically */
 	do {
@@ -153,9 +155,9 @@
 	 *  - OOO reads of prod tail value
 	 *  - OOO copy of elems from the ring
 	 */
-	} while (__atomic_compare_exchange_n(&r->hts_cons.ht.raw,
-			&op.raw, np.raw,
-			0, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE) == 0);
+	} while (rte_atomic_compare_exchange_strong_explicit(&r->hts_cons.ht.raw,
+			(uint64_t *)(uintptr_t)&op.raw, np.raw,
+			rte_memory_order_acquire, rte_memory_order_acquire) == 0);
 
 	*old_head = op.pos.head;
 	return n;
lib/ring/rte_ring_peek_elem_pvt.h +3 -3
index bb0a7d5..b5f0822 100644
@@ -59,7 +59,7 @@
 
 	pos = tail + num;
 	ht->head = pos;
-	__atomic_store_n(&ht->tail, pos, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&ht->tail, pos, rte_memory_order_release);
 }
 
 /**
@@ -78,7 +78,7 @@
 	uint32_t n;
 	union __rte_ring_hts_pos p;
 
-	p.raw = __atomic_load_n(&ht->ht.raw, __ATOMIC_RELAXED);
+	p.raw = rte_atomic_load_explicit(&ht->ht.raw, rte_memory_order_relaxed);
 	n = p.pos.head - p.pos.tail;
 
 	RTE_ASSERT(n >= num);
@@ -104,7 +104,7 @@
 	p.pos.head = tail + num;
 	p.pos.tail = p.pos.head;
 
-	__atomic_store_n(&ht->ht.raw, p.raw, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&ht->ht.raw, p.raw, rte_memory_order_release);
 }
 
 /**
lib/ring/rte_ring_rts_elem_pvt.h +14 -13
index 7164213..1226503 100644
@@ -31,18 +31,19 @@
 	 * might preceded us, then don't update tail with new value.
 	 */
 
-	ot.raw = __atomic_load_n(&ht->tail.raw, __ATOMIC_ACQUIRE);
+	ot.raw = rte_atomic_load_explicit(&ht->tail.raw, rte_memory_order_acquire);
 
 	do {
 		/* on 32-bit systems we have to do atomic read here */
-		h.raw = __atomic_load_n(&ht->head.raw, __ATOMIC_RELAXED);
+		h.raw = rte_atomic_load_explicit(&ht->head.raw, rte_memory_order_relaxed);
 
 		nt.raw = ot.raw;
 		if (++nt.val.cnt == h.val.cnt)
 			nt.val.pos = h.val.pos;
 
-	} while (__atomic_compare_exchange_n(&ht->tail.raw, &ot.raw, nt.raw,
-			0, __ATOMIC_RELEASE, __ATOMIC_ACQUIRE) == 0);
+	} while (rte_atomic_compare_exchange_strong_explicit(&ht->tail.raw,
+			(uint64_t *)(uintptr_t)&ot.raw, nt.raw,
+			rte_memory_order_release, rte_memory_order_acquire) == 0);
 }
 
 /**
@@ -59,7 +60,7 @@
 
 	while (h->val.pos - ht->tail.val.pos > max) {
 		rte_pause();
-		h->raw = __atomic_load_n(&ht->head.raw, __ATOMIC_ACQUIRE);
+		h->raw = rte_atomic_load_explicit(&ht->head.raw, rte_memory_order_acquire);
 	}
 }
 
@@ -76,7 +77,7 @@
 
 	const uint32_t capacity = r->capacity;
 
-	oh.raw = __atomic_load_n(&r->rts_prod.head.raw, __ATOMIC_ACQUIRE);
+	oh.raw = rte_atomic_load_explicit(&r->rts_prod.head.raw, rte_memory_order_acquire);
 
 	do {
 		/* Reset n to the initial burst count */
@@ -113,9 +114,9 @@
 	 *  - OOO reads of cons tail value
 	 *  - OOO copy of elems to the ring
 	 */
-	} while (__atomic_compare_exchange_n(&r->rts_prod.head.raw,
-			&oh.raw, nh.raw,
-			0, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE) == 0);
+	} while (rte_atomic_compare_exchange_strong_explicit(&r->rts_prod.head.raw,
+			(uint64_t *)(uintptr_t)&oh.raw, nh.raw,
+			rte_memory_order_acquire, rte_memory_order_acquire) == 0);
 
 	*old_head = oh.val.pos;
 	return n;
@@ -132,7 +133,7 @@
 	uint32_t n;
 	union __rte_ring_rts_poscnt nh, oh;
 
-	oh.raw = __atomic_load_n(&r->rts_cons.head.raw, __ATOMIC_ACQUIRE);
+	oh.raw = rte_atomic_load_explicit(&r->rts_cons.head.raw, rte_memory_order_acquire);
 
 	/* move cons.head atomically */
 	do {
@@ -168,9 +169,9 @@
 	 *  - OOO reads of prod tail value
 	 *  - OOO copy of elems from the ring
 	 */
-	} while (__atomic_compare_exchange_n(&r->rts_cons.head.raw,
-			&oh.raw, nh.raw,
-			0, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE) == 0);
+	} while (rte_atomic_compare_exchange_strong_explicit(&r->rts_cons.head.raw,
+			(uint64_t *)(uintptr_t)&oh.raw, nh.raw,
+			rte_memory_order_acquire, rte_memory_order_acquire) == 0);
 
 	*old_head = oh.val.pos;
 	return n;
Signature · 1 line
1.8.3.1
#22 lore raw

v2 cover use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API.

v2:
* add #include <rte_stdatomic.h> to rte_mbuf_core.h
* remove first two patches which were fixes that have been merged in another series

Tyler Retzlaff (19):
power: use rte optional stdatomic API
bbdev: use rte optional stdatomic API
eal: use rte optional stdatomic API
eventdev: use rte optional stdatomic API
gpudev: use rte optional stdatomic API
ipsec: use rte optional stdatomic API
mbuf: use rte optional stdatomic API
mempool: use rte optional stdatomic API
rcu: use rte optional stdatomic API
pdump: use rte optional stdatomic API
stack: use rte optional stdatomic API
telemetry: use rte optional stdatomic API
vhost: use rte optional stdatomic API
cryptodev: use rte optional stdatomic API
distributor: use rte optional stdatomic API
ethdev: use rte optional stdatomic API
hash: use rte optional stdatomic API
timer: use rte optional stdatomic API
ring: use rte optional stdatomic API

 drivers/event/cnxk/cnxk_tim_worker.h   |   4 +-
 drivers/net/mlx5/mlx5_hws_cnt.h        |   2 +-
 lib/bbdev/rte_bbdev.c                  |   6 +-
 lib/bbdev/rte_bbdev.h                  |   2 +-
 lib/cryptodev/rte_cryptodev.c          |  22 +++---
 lib/cryptodev/rte_cryptodev.h          |  16 ++---
 lib/distributor/distributor_private.h  |   4 +-
 lib/distributor/rte_distributor.c      |  54 +++++++--------
 lib/eal/common/eal_common_launch.c     |  10 +--
 lib/eal/common/eal_common_mcfg.c       |   2 +-
 lib/eal/common/eal_common_proc.c       |  14 ++--
 lib/eal/common/eal_common_thread.c     |  26 +++----
 lib/eal/common/eal_common_trace.c      |   8 +--
 lib/eal/common/eal_common_trace_ctf.c  |   4 +-
 lib/eal/common/eal_memcfg.h            |   2 +-
 lib/eal/common/eal_private.h           |   4 +-
 lib/eal/common/eal_trace.h             |   4 +-
 lib/eal/common/rte_service.c           | 122 ++++++++++++++++-----------------
 lib/eal/freebsd/eal.c                  |  20 +++---
 lib/eal/include/rte_epoll.h            |   3 +-
 lib/eal/linux/eal.c                    |  26 +++----
 lib/eal/linux/eal_interrupts.c         |  42 ++++++------
 lib/eal/ppc/include/rte_atomic.h       |   6 +-
 lib/eal/windows/rte_thread.c           |   8 ++-
 lib/ethdev/ethdev_driver.h             |  16 ++---
 lib/ethdev/ethdev_private.c            |   6 +-
 lib/ethdev/rte_ethdev.c                |  24 +++----
 lib/ethdev/rte_ethdev.h                |  16 ++---
 lib/ethdev/rte_ethdev_core.h           |   2 +-
 lib/eventdev/rte_event_timer_adapter.c |  66 +++++++++---------
 lib/eventdev/rte_event_timer_adapter.h |   2 +-
 lib/gpudev/gpudev.c                    |   6 +-
 lib/gpudev/gpudev_driver.h             |   2 +-
 lib/hash/rte_cuckoo_hash.c             | 116 +++++++++++++++----------------
 lib/hash/rte_cuckoo_hash.h             |   6 +-
 lib/ipsec/ipsec_sqn.h                  |   2 +-
 lib/ipsec/sa.h                         |   2 +-
 lib/mbuf/rte_mbuf.h                    |  20 +++---
 lib/mbuf/rte_mbuf_core.h               |   5 +-
 lib/mempool/rte_mempool.h              |   4 +-
 lib/pdump/rte_pdump.c                  |  14 ++--
 lib/pdump/rte_pdump.h                  |   8 +--
 lib/power/power_acpi_cpufreq.c         |  33 ++++-----
 lib/power/power_cppc_cpufreq.c         |  25 +++----
 lib/power/power_pstate_cpufreq.c       |  31 +++++----
 lib/rcu/rte_rcu_qsbr.c                 |  48 ++++++-------
 lib/rcu/rte_rcu_qsbr.h                 |  68 +++++++++---------
 lib/ring/rte_ring_c11_pvt.h            |  33 ++++-----
 lib/ring/rte_ring_core.h               |  10 +--
 lib/ring/rte_ring_generic_pvt.h        |   3 +-
 lib/ring/rte_ring_hts_elem_pvt.h       |  22 +++---
 lib/ring/rte_ring_peek_elem_pvt.h      |   6 +-
 lib/ring/rte_ring_rts_elem_pvt.h       |  27 ++++----
 lib/stack/rte_stack.h                  |   2 +-
 lib/stack/rte_stack_lf_c11.h           |  24 +++----
 lib/stack/rte_stack_lf_generic.h       |  18 ++---
 lib/telemetry/telemetry.c              |  18 ++---
 lib/timer/rte_timer.c                  |  50 +++++++-------
 lib/timer/rte_timer.h                  |   6 +-
 lib/vhost/vdpa.c                       |   3 +-
 lib/vhost/vhost.c                      |  42 ++++++------
 lib/vhost/vhost.h                      |  39 ++++++-----
 lib/vhost/vhost_user.c                 |   6 +-
 lib/vhost/virtio_net.c                 |  58 +++++++++-------
 lib/vhost/virtio_net_ctrl.c            |   6 +-
 65 files changed, 667 insertions(+), 639 deletions(-)
Signature · 1 line
1.8.3.1
#23 lore raw

PATCH v2 1/19 power: use rte optional stdatomic API

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/power/power_acpi_cpufreq.c   | 33 +++++++++++++++++----------------
 lib/power/power_cppc_cpufreq.c   | 25 +++++++++++++------------
 lib/power/power_pstate_cpufreq.c | 31 ++++++++++++++++---------------
 3 files changed, 46 insertions(+), 43 deletions(-)
lib/power/power_acpi_cpufreq.c +17 -16
index 6e57aca..8b55f19 100644
@@ -7,6 +7,7 @@
 #include <stdlib.h>
 
 #include <rte_memcpy.h>
+#include <rte_stdatomic.h>
 #include <rte_string_fns.h>
 
 #include "power_acpi_cpufreq.h"
@@ -41,13 +42,13 @@ enum power_state {
  * Power info per lcore.
  */
 struct acpi_power_info {
-	unsigned int lcore_id;                   /**< Logical core id */
+	unsigned int lcore_id;               /**< Logical core id */
 	uint32_t freqs[RTE_MAX_LCORE_FREQS]; /**< Frequency array */
 	uint32_t nb_freqs;                   /**< number of available freqs */
 	FILE *f;                             /**< FD of scaling_setspeed */
 	char governor_ori[32];               /**< Original governor name */
 	uint32_t curr_idx;                   /**< Freq index in freqs array */
-	uint32_t state;                      /**< Power in use state */
+	RTE_ATOMIC(uint32_t) state;          /**< Power in use state */
 	uint16_t turbo_available;            /**< Turbo Boost available */
 	uint16_t turbo_enable;               /**< Turbo Boost enable/disable */
 } __rte_cache_aligned;
@@ -249,9 +250,9 @@ struct acpi_power_info {
 	 * ordering below as lock to make sure the frequency operations
 	 * in the critical section are done under the correct state.
 	 */
-	if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
-					POWER_ONGOING, 0,
-					__ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
+	if (!rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state,
+					POWER_ONGOING,
+					rte_memory_order_acquire, rte_memory_order_relaxed)) {
 		RTE_LOG(INFO, POWER, "Power management of lcore %u is "
 				"in use\n", lcore_id);
 		return -1;
@@ -289,15 +290,15 @@ struct acpi_power_info {
 	RTE_LOG(INFO, POWER, "Initialized successfully for lcore %u "
 			"power management\n", lcore_id);
 	exp_state = POWER_ONGOING;
-	__atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_USED,
-				    0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
+	rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state, POWER_USED,
+				    rte_memory_order_release, rte_memory_order_relaxed);
 
 	return 0;
 
 fail:
 	exp_state = POWER_ONGOING;
-	__atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_UNKNOWN,
-				    0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
+	rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state, POWER_UNKNOWN,
+				    rte_memory_order_release, rte_memory_order_relaxed);
 
 	return -1;
 }
@@ -321,9 +322,9 @@ struct acpi_power_info {
 	 * ordering below as lock to make sure the frequency operations
 	 * in the critical section are done under the correct state.
 	 */
-	if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
-					POWER_ONGOING, 0,
-					__ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
+	if (!rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state,
+					POWER_ONGOING,
+					rte_memory_order_acquire, rte_memory_order_relaxed)) {
 		RTE_LOG(INFO, POWER, "Power management of lcore %u is "
 				"not used\n", lcore_id);
 		return -1;
@@ -344,15 +345,15 @@ struct acpi_power_info {
 			"'userspace' mode and been set back to the "
 			"original\n", lcore_id);
 	exp_state = POWER_ONGOING;
-	__atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_IDLE,
-				    0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
+	rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state, POWER_IDLE,
+				    rte_memory_order_release, rte_memory_order_relaxed);
 
 	return 0;
 
 fail:
 	exp_state = POWER_ONGOING;
-	__atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_UNKNOWN,
-				    0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
+	rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state, POWER_UNKNOWN,
+				    rte_memory_order_release, rte_memory_order_relaxed);
 
 	return -1;
 }
lib/power/power_cppc_cpufreq.c +13 -12
index fc9cffe..bb70f6a 100644
@@ -6,6 +6,7 @@
 #include <stdlib.h>
 
 #include <rte_memcpy.h>
+#include <rte_stdatomic.h>
 
 #include "power_cppc_cpufreq.h"
 #include "power_common.h"
@@ -49,8 +50,8 @@ enum power_state {
  * Power info per lcore.
  */
 struct cppc_power_info {
-	unsigned int lcore_id;                   /**< Logical core id */
-	uint32_t state;                      /**< Power in use state */
+	unsigned int lcore_id;               /**< Logical core id */
+	RTE_ATOMIC(uint32_t) state;          /**< Power in use state */
 	FILE *f;                             /**< FD of scaling_setspeed */
 	char governor_ori[32];               /**< Original governor name */
 	uint32_t curr_idx;                   /**< Freq index in freqs array */
@@ -353,9 +354,9 @@ struct cppc_power_info {
 	 * ordering below as lock to make sure the frequency operations
 	 * in the critical section are done under the correct state.
 	 */
-	if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
-					POWER_ONGOING, 0,
-					__ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
+	if (!rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state,
+					POWER_ONGOING,
+					rte_memory_order_acquire, rte_memory_order_relaxed)) {
 		RTE_LOG(INFO, POWER, "Power management of lcore %u is "
 				"in use\n", lcore_id);
 		return -1;
@@ -393,12 +394,12 @@ struct cppc_power_info {
 	RTE_LOG(INFO, POWER, "Initialized successfully for lcore %u "
 			"power management\n", lcore_id);
 
-	__atomic_store_n(&(pi->state), POWER_USED, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&(pi->state), POWER_USED, rte_memory_order_release);
 
 	return 0;
 
 fail:
-	__atomic_store_n(&(pi->state), POWER_UNKNOWN, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&(pi->state), POWER_UNKNOWN, rte_memory_order_release);
 	return -1;
 }
 
@@ -431,9 +432,9 @@ struct cppc_power_info {
 	 * ordering below as lock to make sure the frequency operations
 	 * in the critical section are done under the correct state.
 	 */
-	if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
-					POWER_ONGOING, 0,
-					__ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
+	if (!rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state,
+					POWER_ONGOING,
+					rte_memory_order_acquire, rte_memory_order_relaxed)) {
 		RTE_LOG(INFO, POWER, "Power management of lcore %u is "
 				"not used\n", lcore_id);
 		return -1;
@@ -453,12 +454,12 @@ struct cppc_power_info {
 	RTE_LOG(INFO, POWER, "Power management of lcore %u has exited from "
 			"'userspace' mode and been set back to the "
 			"original\n", lcore_id);
-	__atomic_store_n(&(pi->state), POWER_IDLE, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&(pi->state), POWER_IDLE, rte_memory_order_release);
 
 	return 0;
 
 fail:
-	__atomic_store_n(&(pi->state), POWER_UNKNOWN, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&(pi->state), POWER_UNKNOWN, rte_memory_order_release);
 
 	return -1;
 }
lib/power/power_pstate_cpufreq.c +16 -15
index 52aa645..5ca5f60 100644
@@ -12,6 +12,7 @@
 #include <inttypes.h>
 
 #include <rte_memcpy.h>
+#include <rte_stdatomic.h>
 
 #include "rte_power_pmd_mgmt.h"
 #include "power_pstate_cpufreq.h"
@@ -59,7 +60,7 @@ struct pstate_power_info {
 	uint32_t non_turbo_max_ratio;        /**< Non Turbo Max ratio  */
 	uint32_t sys_max_freq;               /**< system wide max freq  */
 	uint32_t core_base_freq;             /**< core base freq  */
-	uint32_t state;                      /**< Power in use state */
+	RTE_ATOMIC(uint32_t) state;          /**< Power in use state */
 	uint16_t turbo_available;            /**< Turbo Boost available */
 	uint16_t turbo_enable;               /**< Turbo Boost enable/disable */
 	uint16_t priority_core;              /**< High Performance core */
@@ -555,9 +556,9 @@ struct pstate_power_info {
 	 * ordering below as lock to make sure the frequency operations
 	 * in the critical section are done under the correct state.
 	 */
-	if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
-					POWER_ONGOING, 0,
-					__ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
+	if (!rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state,
+					POWER_ONGOING,
+					rte_memory_order_acquire, rte_memory_order_relaxed)) {
 		RTE_LOG(INFO, POWER, "Power management of lcore %u is "
 				"in use\n", lcore_id);
 		return -1;
@@ -600,15 +601,15 @@ struct pstate_power_info {
 	RTE_LOG(INFO, POWER, "Initialized successfully for lcore %u "
 			"power management\n", lcore_id);
 	exp_state = POWER_ONGOING;
-	__atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_USED,
-				    0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
+	rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state, POWER_USED,
+				    rte_memory_order_release, rte_memory_order_relaxed);
 
 	return 0;
 
 fail:
 	exp_state = POWER_ONGOING;
-	__atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_UNKNOWN,
-				    0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
+	rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state, POWER_UNKNOWN,
+				    rte_memory_order_release, rte_memory_order_relaxed);
 
 	return -1;
 }
@@ -633,9 +634,9 @@ struct pstate_power_info {
 	 * ordering below as lock to make sure the frequency operations
 	 * in the critical section are under done the correct state.
 	 */
-	if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
-					POWER_ONGOING, 0,
-					__ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
+	if (!rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state,
+					POWER_ONGOING,
+					rte_memory_order_acquire, rte_memory_order_relaxed)) {
 		RTE_LOG(INFO, POWER, "Power management of lcore %u is "
 				"not used\n", lcore_id);
 		return -1;
@@ -658,15 +659,15 @@ struct pstate_power_info {
 			"'performance' mode and been set back to the "
 			"original\n", lcore_id);
 	exp_state = POWER_ONGOING;
-	__atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_IDLE,
-				    0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
+	rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state, POWER_IDLE,
+				    rte_memory_order_release, rte_memory_order_relaxed);
 
 	return 0;
 
 fail:
 	exp_state = POWER_ONGOING;
-	__atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_UNKNOWN,
-				    0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
+	rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state, POWER_UNKNOWN,
+				    rte_memory_order_release, rte_memory_order_relaxed);
 
 	return -1;
 }
Signature · 1 line
1.8.3.1
#24 lore raw

PATCH v2 2/19 bbdev: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/bbdev/rte_bbdev.c | 6 +++---
 lib/bbdev/rte_bbdev.h | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)
lib/bbdev/rte_bbdev.c +3 -3
index 155323e..cfebea0 100644
@@ -208,7 +208,7 @@ struct rte_bbdev *
 		return NULL;
 	}
 
-	__atomic_fetch_add(&bbdev->data->process_cnt, 1, __ATOMIC_RELAXED);
+	rte_atomic_fetch_add_explicit(&bbdev->data->process_cnt, 1, rte_memory_order_relaxed);
 	bbdev->data->dev_id = dev_id;
 	bbdev->state = RTE_BBDEV_INITIALIZED;
 
@@ -250,8 +250,8 @@ struct rte_bbdev *
 	}
 
 	/* clear shared BBDev Data if no process is using the device anymore */
-	if (__atomic_fetch_sub(&bbdev->data->process_cnt, 1,
-			      __ATOMIC_RELAXED) - 1 == 0)
+	if (rte_atomic_fetch_sub_explicit(&bbdev->data->process_cnt, 1,
+			      rte_memory_order_relaxed) - 1 == 0)
 		memset(bbdev->data, 0, sizeof(*bbdev->data));
 
 	memset(bbdev, 0, sizeof(*bbdev));
lib/bbdev/rte_bbdev.h +1 -1
index d12e2e7..e1aee08 100644
@@ -482,7 +482,7 @@ struct rte_bbdev_data {
 	uint16_t dev_id;  /**< Device ID */
 	int socket_id;  /**< NUMA socket that device is on */
 	bool started;  /**< Device run-time state */
-	uint16_t process_cnt;  /** Counter of processes using the device */
+	RTE_ATOMIC(uint16_t) process_cnt;  /** Counter of processes using the device */
 };
 
 /* Forward declarations */
Signature · 1 line
1.8.3.1
#25 lore raw

PATCH v2 3/19 eal: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/eal/common/eal_common_launch.c    |  10 +--
 lib/eal/common/eal_common_mcfg.c      |   2 +-
 lib/eal/common/eal_common_proc.c      |  14 ++--
 lib/eal/common/eal_common_thread.c    |  26 ++++----
 lib/eal/common/eal_common_trace.c     |   8 +--
 lib/eal/common/eal_common_trace_ctf.c |   4 +-
 lib/eal/common/eal_memcfg.h           |   2 +-
 lib/eal/common/eal_private.h          |   4 +-
 lib/eal/common/eal_trace.h            |   4 +-
 lib/eal/common/rte_service.c          | 122 +++++++++++++++++-----------------
 lib/eal/freebsd/eal.c                 |  20 +++---
 lib/eal/include/rte_epoll.h           |   3 +-
 lib/eal/linux/eal.c                   |  26 ++++----
 lib/eal/linux/eal_interrupts.c        |  42 ++++++------
 lib/eal/ppc/include/rte_atomic.h      |   6 +-
 lib/eal/windows/rte_thread.c          |   8 ++-
 16 files changed, 152 insertions(+), 149 deletions(-)
lib/eal/common/eal_common_launch.c +5 -5
index 0504598..5320c3b 100644
@@ -18,8 +18,8 @@
 int
 rte_eal_wait_lcore(unsigned worker_id)
 {
-	while (__atomic_load_n(&lcore_config[worker_id].state,
-			__ATOMIC_ACQUIRE) != WAIT)
+	while (rte_atomic_load_explicit(&lcore_config[worker_id].state,
+			rte_memory_order_acquire) != WAIT)
 		rte_pause();
 
 	return lcore_config[worker_id].ret;
@@ -38,8 +38,8 @@
 	/* Check if the worker is in 'WAIT' state. Use acquire order
 	 * since 'state' variable is used as the guard variable.
 	 */
-	if (__atomic_load_n(&lcore_config[worker_id].state,
-			__ATOMIC_ACQUIRE) != WAIT)
+	if (rte_atomic_load_explicit(&lcore_config[worker_id].state,
+			rte_memory_order_acquire) != WAIT)
 		goto finish;
 
 	lcore_config[worker_id].arg = arg;
@@ -47,7 +47,7 @@
 	 * before the worker thread starts running the function.
 	 * Use worker thread function as the guard variable.
 	 */
-	__atomic_store_n(&lcore_config[worker_id].f, f, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&lcore_config[worker_id].f, f, rte_memory_order_release);
 
 	rc = eal_thread_wake_worker(worker_id);
 
lib/eal/common/eal_common_mcfg.c +1 -1
index 2a785e7..dabb80e 100644
@@ -30,7 +30,7 @@
 	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
 
 	/* wait until shared mem_config finish initialising */
-	rte_wait_until_equal_32(&mcfg->magic, RTE_MAGIC, __ATOMIC_RELAXED);
+	rte_wait_until_equal_32(&mcfg->magic, RTE_MAGIC, rte_memory_order_relaxed);
 }
 
 int
lib/eal/common/eal_common_proc.c +7 -7
index f20a348..728815c 100644
@@ -33,7 +33,7 @@
 #include "eal_filesystem.h"
 #include "eal_internal_cfg.h"
 
-static int mp_fd = -1;
+static RTE_ATOMIC(int) mp_fd = -1;
 static rte_thread_t mp_handle_tid;
 static char mp_filter[PATH_MAX];   /* Filter for secondary process sockets */
 static char mp_dir_path[PATH_MAX]; /* The directory path for all mp sockets */
@@ -404,7 +404,7 @@ struct pending_request {
 	struct sockaddr_un sa;
 	int fd;
 
-	while ((fd = __atomic_load_n(&mp_fd, __ATOMIC_RELAXED)) >= 0) {
+	while ((fd = rte_atomic_load_explicit(&mp_fd, rte_memory_order_relaxed)) >= 0) {
 		int ret;
 
 		ret = read_msg(fd, &msg, &sa);
@@ -652,7 +652,7 @@ enum async_action {
 		RTE_LOG(ERR, EAL, "failed to create mp thread: %s\n",
 			strerror(errno));
 		close(dir_fd);
-		close(__atomic_exchange_n(&mp_fd, -1, __ATOMIC_RELAXED));
+		close(rte_atomic_exchange_explicit(&mp_fd, -1, rte_memory_order_relaxed));
 		return -1;
 	}
 
@@ -668,7 +668,7 @@ enum async_action {
 {
 	int fd;
 
-	fd = __atomic_exchange_n(&mp_fd, -1, __ATOMIC_RELAXED);
+	fd = rte_atomic_exchange_explicit(&mp_fd, -1, rte_memory_order_relaxed);
 	if (fd < 0)
 		return;
 
@@ -1282,11 +1282,11 @@ enum mp_status {
 
 	expected = MP_STATUS_UNKNOWN;
 	desired = status;
-	if (__atomic_compare_exchange_n(&mcfg->mp_status, &expected, desired,
-			false, __ATOMIC_RELAXED, __ATOMIC_RELAXED))
+	if (rte_atomic_compare_exchange_strong_explicit(&mcfg->mp_status, &expected, desired,
+			rte_memory_order_relaxed, rte_memory_order_relaxed))
 		return true;
 
-	return __atomic_load_n(&mcfg->mp_status, __ATOMIC_RELAXED) == desired;
+	return rte_atomic_load_explicit(&mcfg->mp_status, rte_memory_order_relaxed) == desired;
 }
 
 bool
lib/eal/common/eal_common_thread.c +13 -13
index 668b1ed..c422ea8 100644
@@ -191,8 +191,8 @@ unsigned rte_socket_id(void)
 		/* Set the state to 'RUNNING'. Use release order
 		 * since 'state' variable is used as the guard variable.
 		 */
-		__atomic_store_n(&lcore_config[lcore_id].state, RUNNING,
-			__ATOMIC_RELEASE);
+		rte_atomic_store_explicit(&lcore_config[lcore_id].state, RUNNING,
+			rte_memory_order_release);
 
 		eal_thread_ack_command();
 
@@ -201,8 +201,8 @@ unsigned rte_socket_id(void)
 		 * are accessed only after update to 'f' is visible.
 		 * Wait till the update to 'f' is visible to the worker.
 		 */
-		while ((f = __atomic_load_n(&lcore_config[lcore_id].f,
-				__ATOMIC_ACQUIRE)) == NULL)
+		while ((f = rte_atomic_load_explicit(&lcore_config[lcore_id].f,
+				rte_memory_order_acquire)) == NULL)
 			rte_pause();
 
 		rte_eal_trace_thread_lcore_running(lcore_id, f);
@@ -219,8 +219,8 @@ unsigned rte_socket_id(void)
 		 * are completed before the state is updated.
 		 * Use 'state' as the guard variable.
 		 */
-		__atomic_store_n(&lcore_config[lcore_id].state, WAIT,
-			__ATOMIC_RELEASE);
+		rte_atomic_store_explicit(&lcore_config[lcore_id].state, WAIT,
+			rte_memory_order_release);
 
 		rte_eal_trace_thread_lcore_stopped(lcore_id);
 	}
@@ -242,7 +242,7 @@ struct control_thread_params {
 	/* Control thread status.
 	 * If the status is CTRL_THREAD_ERROR, 'ret' has the error code.
 	 */
-	enum __rte_ctrl_thread_status status;
+	RTE_ATOMIC(enum __rte_ctrl_thread_status) status;
 };
 
 static int control_thread_init(void *arg)
@@ -259,13 +259,13 @@ static int control_thread_init(void *arg)
 	RTE_PER_LCORE(_socket_id) = SOCKET_ID_ANY;
 	params->ret = rte_thread_set_affinity_by_id(rte_thread_self(), cpuset);
 	if (params->ret != 0) {
-		__atomic_store_n(&params->status,
-			CTRL_THREAD_ERROR, __ATOMIC_RELEASE);
+		rte_atomic_store_explicit(&params->status,
+			CTRL_THREAD_ERROR, rte_memory_order_release);
 		return 1;
 	}
 
-	__atomic_store_n(&params->status,
-		CTRL_THREAD_RUNNING, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&params->status,
+		CTRL_THREAD_RUNNING, rte_memory_order_release);
 
 	return 0;
 }
@@ -310,8 +310,8 @@ static uint32_t control_thread_start(void *arg)
 
 	/* Wait for the control thread to initialize successfully */
 	while ((ctrl_thread_status =
-			__atomic_load_n(&params->status,
-			__ATOMIC_ACQUIRE)) == CTRL_THREAD_LAUNCHING) {
+			rte_atomic_load_explicit(&params->status,
+			rte_memory_order_acquire)) == CTRL_THREAD_LAUNCHING) {
 		rte_delay_us_sleep(1);
 	}
 
lib/eal/common/eal_common_trace.c +4 -4
index d2eac2d..6ad87fc 100644
@@ -97,7 +97,7 @@ struct trace_point_head *
 bool
 rte_trace_is_enabled(void)
 {
-	return __atomic_load_n(&trace.status, __ATOMIC_ACQUIRE) != 0;
+	return rte_atomic_load_explicit(&trace.status, rte_memory_order_acquire) != 0;
 }
 
 static void
@@ -157,7 +157,7 @@ rte_trace_mode rte_trace_mode_get(void)
 	prev = rte_atomic_fetch_or_explicit(t, __RTE_TRACE_FIELD_ENABLE_MASK,
 		rte_memory_order_release);
 	if ((prev & __RTE_TRACE_FIELD_ENABLE_MASK) == 0)
-		__atomic_fetch_add(&trace.status, 1, __ATOMIC_RELEASE);
+		rte_atomic_fetch_add_explicit(&trace.status, 1, rte_memory_order_release);
 	return 0;
 }
 
@@ -172,7 +172,7 @@ rte_trace_mode rte_trace_mode_get(void)
 	prev = rte_atomic_fetch_and_explicit(t, ~__RTE_TRACE_FIELD_ENABLE_MASK,
 		rte_memory_order_release);
 	if ((prev & __RTE_TRACE_FIELD_ENABLE_MASK) != 0)
-		__atomic_fetch_sub(&trace.status, 1, __ATOMIC_RELEASE);
+		rte_atomic_fetch_sub_explicit(&trace.status, 1, rte_memory_order_release);
 	return 0;
 }
 
@@ -526,7 +526,7 @@ rte_trace_mode rte_trace_mode_get(void)
 
 	/* Add the trace point at tail */
 	STAILQ_INSERT_TAIL(&tp_list, tp, next);
-	__atomic_thread_fence(__ATOMIC_RELEASE);
+	__atomic_thread_fence(rte_memory_order_release);
 
 	/* All Good !!! */
 	return 0;
lib/eal/common/eal_common_trace_ctf.c +2 -2
index c6775c3..04c4f71 100644
@@ -361,10 +361,10 @@
 	if (ctf_meta == NULL)
 		return -EINVAL;
 
-	if (!__atomic_load_n(&trace->ctf_fixup_done, __ATOMIC_SEQ_CST) &&
+	if (!rte_atomic_load_explicit(&trace->ctf_fixup_done, rte_memory_order_seq_cst) &&
 				rte_get_timer_hz()) {
 		meta_fixup(trace, ctf_meta);
-		__atomic_store_n(&trace->ctf_fixup_done, 1, __ATOMIC_SEQ_CST);
+		rte_atomic_store_explicit(&trace->ctf_fixup_done, 1, rte_memory_order_seq_cst);
 	}
 
 	rc = fprintf(f, "%s", ctf_meta);
lib/eal/common/eal_memcfg.h +1 -1
index d5c63e2..60e2089 100644
@@ -42,7 +42,7 @@ struct rte_mem_config {
 	rte_rwlock_t memory_hotplug_lock;
 	/**< Indicates whether memory hotplug request is in progress. */
 
-	uint8_t mp_status; /**< Multiprocess status. */
+	RTE_ATOMIC(uint8_t) mp_status; /**< Multiprocess status. */
 
 	/* memory segments and zones */
 	struct rte_fbarray memzones; /**< Memzone descriptors. */
lib/eal/common/eal_private.h +2 -2
index ebd496b..4d2e806 100644
@@ -24,11 +24,11 @@ struct lcore_config {
 	int pipe_main2worker[2];   /**< communication pipe with main */
 	int pipe_worker2main[2];   /**< communication pipe with main */
 
-	lcore_function_t * volatile f; /**< function to call */
+	RTE_ATOMIC(lcore_function_t *) volatile f; /**< function to call */
 	void * volatile arg;       /**< argument of function */
 	volatile int ret;          /**< return value of function */
 
-	volatile enum rte_lcore_state_t state; /**< lcore state */
+	volatile RTE_ATOMIC(enum rte_lcore_state_t) state; /**< lcore state */
 	unsigned int socket_id;    /**< physical socket id for this lcore */
 	unsigned int core_id;      /**< core number on socket for this lcore */
 	int core_index;            /**< relative index, starting from 0 */
lib/eal/common/eal_trace.h +2 -2
index d66bcfe..ace2ef3 100644
@@ -50,7 +50,7 @@ struct trace_arg {
 struct trace {
 	char *dir;
 	int register_errno;
-	uint32_t status;
+	RTE_ATOMIC(uint32_t) status;
 	enum rte_trace_mode mode;
 	rte_uuid_t uuid;
 	uint32_t buff_len;
@@ -65,7 +65,7 @@ struct trace {
 	uint32_t ctf_meta_offset_freq;
 	uint32_t ctf_meta_offset_freq_off_s;
 	uint32_t ctf_meta_offset_freq_off;
-	uint16_t ctf_fixup_done;
+	RTE_ATOMIC(uint16_t) ctf_fixup_done;
 	rte_spinlock_t lock;
 };
 
lib/eal/common/rte_service.c +61 -61

307 lines not on this page · raw message

lib/eal/freebsd/eal.c +10 -10
index 39a2868..568e06e 100644
@@ -597,8 +597,8 @@ static void rte_eal_init_alert(const char *msg)
 		return -1;
 	}
 
-	if (!__atomic_compare_exchange_n(&run_once, &has_run, 1, 0,
-					__ATOMIC_RELAXED, __ATOMIC_RELAXED)) {
+	if (!rte_atomic_compare_exchange_strong_explicit(&run_once, &has_run, 1,
+					rte_memory_order_relaxed, rte_memory_order_relaxed)) {
 		rte_eal_init_alert("already called initialization.");
 		rte_errno = EALREADY;
 		return -1;
@@ -622,7 +622,7 @@ static void rte_eal_init_alert(const char *msg)
 	if (fctret < 0) {
 		rte_eal_init_alert("Invalid 'command line' arguments.");
 		rte_errno = EINVAL;
-		__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
+		rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed);
 		return -1;
 	}
 
@@ -636,20 +636,20 @@ static void rte_eal_init_alert(const char *msg)
 	if (eal_plugins_init() < 0) {
 		rte_eal_init_alert("Cannot init plugins");
 		rte_errno = EINVAL;
-		__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
+		rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed);
 		return -1;
 	}
 
 	if (eal_trace_init() < 0) {
 		rte_eal_init_alert("Cannot init trace");
 		rte_errno = EFAULT;
-		__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
+		rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed);
 		return -1;
 	}
 
 	if (eal_option_device_parse()) {
 		rte_errno = ENODEV;
-		__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
+		rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed);
 		return -1;
 	}
 
@@ -683,7 +683,7 @@ static void rte_eal_init_alert(const char *msg)
 	if (rte_bus_scan()) {
 		rte_eal_init_alert("Cannot scan the buses for devices");
 		rte_errno = ENODEV;
-		__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
+		rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed);
 		return -1;
 	}
 
@@ -736,7 +736,7 @@ static void rte_eal_init_alert(const char *msg)
 		if (ret < 0) {
 			rte_eal_init_alert("Cannot get hugepage information.");
 			rte_errno = EACCES;
-			__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
+			rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed);
 			return -1;
 		}
 	}
@@ -915,8 +915,8 @@ static void rte_eal_init_alert(const char *msg)
 	static uint32_t run_once;
 	uint32_t has_run = 0;
 
-	if (!__atomic_compare_exchange_n(&run_once, &has_run, 1, 0,
-			__ATOMIC_RELAXED, __ATOMIC_RELAXED)) {
+	if (!rte_atomic_compare_exchange_strong_explicit(&run_once, &has_run, 1,
+			rte_memory_order_relaxed, rte_memory_order_relaxed)) {
 		RTE_LOG(WARNING, EAL, "Already called cleanup\n");
 		rte_errno = EALREADY;
 		return -1;
lib/eal/include/rte_epoll.h +2 -1
index 01525f5..ae0cf20 100644
@@ -13,6 +13,7 @@
 
 #include <stdint.h>
 
+#include <rte_stdatomic.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -38,7 +39,7 @@ enum {
 
 /** interrupt epoll event obj, taken by epoll_event.ptr */
 struct rte_epoll_event {
-	uint32_t status;           /**< OUT: event status */
+	RTE_ATOMIC(uint32_t) status;           /**< OUT: event status */
 	int fd;                    /**< OUT: event fd */
 	int epfd;       /**< OUT: epoll instance the ev associated with */
 	struct rte_epoll_data epdata;
lib/eal/linux/eal.c +13 -13
index 5f4b2fb..57da058 100644
@@ -967,7 +967,7 @@ static void rte_eal_init_alert(const char *msg)
 rte_eal_init(int argc, char **argv)
 {
 	int i, fctret, ret;
-	static uint32_t run_once;
+	static RTE_ATOMIC(uint32_t) run_once;
 	uint32_t has_run = 0;
 	char cpuset[RTE_CPU_AFFINITY_STR_LEN];
 	char thread_name[RTE_THREAD_NAME_SIZE];
@@ -983,8 +983,8 @@ static void rte_eal_init_alert(const char *msg)
 		return -1;
 	}
 
-	if (!__atomic_compare_exchange_n(&run_once, &has_run, 1, 0,
-					__ATOMIC_RELAXED, __ATOMIC_RELAXED)) {
+	if (!rte_atomic_compare_exchange_strong_explicit(&run_once, &has_run, 1,
+					rte_memory_order_relaxed, rte_memory_order_relaxed)) {
 		rte_eal_init_alert("already called initialization.");
 		rte_errno = EALREADY;
 		return -1;
@@ -1008,14 +1008,14 @@ static void rte_eal_init_alert(const char *msg)
 	if (fctret < 0) {
 		rte_eal_init_alert("Invalid 'command line' arguments.");
 		rte_errno = EINVAL;
-		__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
+		rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed);
 		return -1;
 	}
 
 	if (eal_plugins_init() < 0) {
 		rte_eal_init_alert("Cannot init plugins");
 		rte_errno = EINVAL;
-		__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
+		rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed);
 		return -1;
 	}
 
@@ -1027,7 +1027,7 @@ static void rte_eal_init_alert(const char *msg)
 
 	if (eal_option_device_parse()) {
 		rte_errno = ENODEV;
-		__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
+		rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed);
 		return -1;
 	}
 
@@ -1061,7 +1061,7 @@ static void rte_eal_init_alert(const char *msg)
 	if (rte_bus_scan()) {
 		rte_eal_init_alert("Cannot scan the buses for devices");
 		rte_errno = ENODEV;
-		__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
+		rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed);
 		return -1;
 	}
 
@@ -1125,7 +1125,7 @@ static void rte_eal_init_alert(const char *msg)
 		if (ret < 0) {
 			rte_eal_init_alert("Cannot get hugepage information.");
 			rte_errno = EACCES;
-			__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
+			rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed);
 			return -1;
 		}
 	}
@@ -1150,7 +1150,7 @@ static void rte_eal_init_alert(const char *msg)
 			 internal_conf->syslog_facility) < 0) {
 		rte_eal_init_alert("Cannot init logging.");
 		rte_errno = ENOMEM;
-		__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
+		rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed);
 		return -1;
 	}
 
@@ -1158,7 +1158,7 @@ static void rte_eal_init_alert(const char *msg)
 	if (rte_eal_vfio_setup() < 0) {
 		rte_eal_init_alert("Cannot init VFIO");
 		rte_errno = EAGAIN;
-		__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
+		rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed);
 		return -1;
 	}
 #endif
@@ -1345,11 +1345,11 @@ static void rte_eal_init_alert(const char *msg)
 int
 rte_eal_cleanup(void)
 {
-	static uint32_t run_once;
+	static RTE_ATOMIC(uint32_t) run_once;
 	uint32_t has_run = 0;
 
-	if (!__atomic_compare_exchange_n(&run_once, &has_run, 1, 0,
-					__ATOMIC_RELAXED, __ATOMIC_RELAXED)) {
+	if (!rte_atomic_compare_exchange_strong_explicit(&run_once, &has_run, 1,
+					rte_memory_order_relaxed, rte_memory_order_relaxed)) {
 		RTE_LOG(WARNING, EAL, "Already called cleanup\n");
 		rte_errno = EALREADY;
 		return -1;
lib/eal/linux/eal_interrupts.c +21 -21

93 lines not on this page · raw message

lib/eal/ppc/include/rte_atomic.h +3 -3
index 7382412..645c713 100644
@@ -48,7 +48,7 @@
 static inline int
 rte_atomic16_cmpset(volatile uint16_t *dst, uint16_t exp, uint16_t src)
 {
-	return __atomic_compare_exchange(dst, &exp, &src, 0, rte_memory_order_acquire,
+	return rte_atomic_compare_exchange_strong_explicit(dst, &exp, src, rte_memory_order_acquire,
 		rte_memory_order_acquire) ? 1 : 0;
 }
 
@@ -90,7 +90,7 @@ static inline int rte_atomic16_dec_and_test(rte_atomic16_t *v)
 static inline int
 rte_atomic32_cmpset(volatile uint32_t *dst, uint32_t exp, uint32_t src)
 {
-	return __atomic_compare_exchange(dst, &exp, &src, 0, rte_memory_order_acquire,
+	return rte_atomic_compare_exchange_strong_explicit(dst, &exp, src, rte_memory_order_acquire,
 		rte_memory_order_acquire) ? 1 : 0;
 }
 
@@ -132,7 +132,7 @@ static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v)
 static inline int
 rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src)
 {
-	return __atomic_compare_exchange(dst, &exp, &src, 0, rte_memory_order_acquire,
+	return rte_atomic_compare_exchange_strong_explicit(dst, &exp, src, rte_memory_order_acquire,
 		rte_memory_order_acquire) ? 1 : 0;
 }
 
lib/eal/windows/rte_thread.c +5 -3
index acf6484..145ac4b 100644
@@ -9,6 +9,7 @@
 #include <rte_eal.h>
 #include <rte_common.h>
 #include <rte_errno.h>
+#include <rte_stdatomic.h>
 #include <rte_thread.h>
 
 #include "eal_windows.h"
@@ -19,7 +20,7 @@ struct eal_tls_key {
 
 struct thread_routine_ctx {
 	rte_thread_func thread_func;
-	bool thread_init_failed;
+	RTE_ATOMIC(bool) thread_init_failed;
 	void *routine_args;
 };
 
@@ -168,7 +169,8 @@ struct thread_routine_ctx {
 thread_func_wrapper(void *arg)
 {
 	struct thread_routine_ctx ctx = *(struct thread_routine_ctx *)arg;
-	const bool thread_exit = __atomic_load_n(&ctx.thread_init_failed, __ATOMIC_ACQUIRE);
+	const bool thread_exit = rte_atomic_load_explicit(
+		&ctx.thread_init_failed, rte_memory_order_acquire);
 
 	free(arg);
 
@@ -237,7 +239,7 @@ struct thread_routine_ctx {
 	}
 
 resume_thread:
-	__atomic_store_n(&ctx->thread_init_failed, thread_exit, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&ctx->thread_init_failed, thread_exit, rte_memory_order_release);
 
 	if (ResumeThread(thread_handle) == (DWORD)-1) {
 		ret = thread_log_last_error("ResumeThread()");
Signature · 1 line
1.8.3.1
#26 lore raw

PATCH v2 4/19 eventdev: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 drivers/event/cnxk/cnxk_tim_worker.h   |  4 +--
 lib/eventdev/rte_event_timer_adapter.c | 66 +++++++++++++++++-----------------
 lib/eventdev/rte_event_timer_adapter.h |  2 +-
 3 files changed, 36 insertions(+), 36 deletions(-)
drivers/event/cnxk/cnxk_tim_worker.h +2 -2
index f0857f2..f530d8c 100644
@@ -314,7 +314,7 @@
 
 	tim->impl_opaque[0] = (uintptr_t)chunk;
 	tim->impl_opaque[1] = (uintptr_t)bkt;
-	__atomic_store_n(&tim->state, RTE_EVENT_TIMER_ARMED, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&tim->state, RTE_EVENT_TIMER_ARMED, rte_memory_order_release);
 	cnxk_tim_bkt_inc_nent(bkt);
 	cnxk_tim_bkt_dec_lock_relaxed(bkt);
 
@@ -425,7 +425,7 @@
 
 	tim->impl_opaque[0] = (uintptr_t)chunk;
 	tim->impl_opaque[1] = (uintptr_t)bkt;
-	__atomic_store_n(&tim->state, RTE_EVENT_TIMER_ARMED, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&tim->state, RTE_EVENT_TIMER_ARMED, rte_memory_order_release);
 	cnxk_tim_bkt_inc_nent(bkt);
 	cnxk_tim_bkt_dec_lock_relaxed(bkt);
 
lib/eventdev/rte_event_timer_adapter.c +33 -33

162 lines not on this page · raw message

lib/eventdev/rte_event_timer_adapter.h +1 -1
index fbdddf8..49e646a 100644
@@ -498,7 +498,7 @@ struct rte_event_timer {
 	 * implementation specific values to share between the arm and cancel
 	 * operations.  The application should not modify this field.
 	 */
-	enum rte_event_timer_state state;
+	RTE_ATOMIC(enum rte_event_timer_state) state;
 	/**< State of the event timer. */
 	uint8_t user_meta[];
 	/**< Memory to store user specific metadata.
Signature · 1 line
1.8.3.1
#27 lore raw

PATCH v2 5/19 gpudev: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/gpudev/gpudev.c        | 6 +++---
 lib/gpudev/gpudev_driver.h | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)
lib/gpudev/gpudev.c +3 -3
index 8f12abe..6845d18 100644
@@ -228,7 +228,7 @@ struct rte_gpu *
 	dev->mpshared->info.numa_node = -1;
 	dev->mpshared->info.parent = RTE_GPU_ID_NONE;
 	TAILQ_INIT(&dev->callbacks);
-	__atomic_fetch_add(&dev->mpshared->process_refcnt, 1, __ATOMIC_RELAXED);
+	rte_atomic_fetch_add_explicit(&dev->mpshared->process_refcnt, 1, rte_memory_order_relaxed);
 
 	gpu_count++;
 	GPU_LOG(DEBUG, "new device %s (id %d) of total %d",
@@ -277,7 +277,7 @@ struct rte_gpu *
 
 	TAILQ_INIT(&dev->callbacks);
 	dev->mpshared = shared_dev;
-	__atomic_fetch_add(&dev->mpshared->process_refcnt, 1, __ATOMIC_RELAXED);
+	rte_atomic_fetch_add_explicit(&dev->mpshared->process_refcnt, 1, rte_memory_order_relaxed);
 
 	gpu_count++;
 	GPU_LOG(DEBUG, "attached device %s (id %d) of total %d",
@@ -340,7 +340,7 @@ struct rte_gpu *
 
 	gpu_free_callbacks(dev);
 	dev->process_state = RTE_GPU_STATE_UNUSED;
-	__atomic_fetch_sub(&dev->mpshared->process_refcnt, 1, __ATOMIC_RELAXED);
+	rte_atomic_fetch_sub_explicit(&dev->mpshared->process_refcnt, 1, rte_memory_order_relaxed);
 	gpu_count--;
 
 	return 0;
lib/gpudev/gpudev_driver.h +1 -1
index 42898c7..0b1e7f2 100644
@@ -69,7 +69,7 @@ struct rte_gpu_mpshared {
 	/* Device info structure. */
 	struct rte_gpu_info info;
 	/* Counter of processes using the device. */
-	uint16_t process_refcnt; /* Updated by this library. */
+	RTE_ATOMIC(uint16_t) process_refcnt; /* Updated by this library. */
 };
 
 struct rte_gpu {
Signature · 1 line
1.8.3.1
#28 lore raw

PATCH v2 6/19 ipsec: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/ipsec/ipsec_sqn.h | 2 +-
 lib/ipsec/sa.h        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
lib/ipsec/ipsec_sqn.h +1 -1
index 505950e..984a9dd 100644
@@ -128,7 +128,7 @@
 
 	n = *num;
 	if (SQN_ATOMIC(sa))
-		sqn = __atomic_fetch_add(&sa->sqn.outb, n, __ATOMIC_RELAXED) + n;
+		sqn = rte_atomic_fetch_add_explicit(&sa->sqn.outb, n, rte_memory_order_relaxed) + n;
 	else {
 		sqn = sa->sqn.outb + n;
 		sa->sqn.outb = sqn;
lib/ipsec/sa.h +1 -1
index ce4af8c..4b30bea 100644
@@ -124,7 +124,7 @@ struct rte_ipsec_sa {
 	 * place from other frequently accessed data.
 	 */
 	union {
-		uint64_t outb;
+		RTE_ATOMIC(uint64_t) outb;
 		struct {
 			uint32_t rdidx; /* read index */
 			uint32_t wridx; /* write index */
Signature · 1 line
1.8.3.1
Acked-by on v2 6/19
#29 lore raw

Re: [PATCH v2 06/19] ipsec: use rte optional stdatomic API

17.10.2023 21:31, Tyler Retzlaff пишет:

Quoted 27 lines

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
  lib/ipsec/ipsec_sqn.h | 2 +-
  lib/ipsec/sa.h        | 2 +-
  2 files changed, 2 insertions(+), 2 deletions(-)
lib/ipsec/ipsec_sqn.h +1 -1
index 505950e..984a9dd 100644
@@ -128,7 +128,7 @@
 
  	n = *num;
  	if (SQN_ATOMIC(sa))
-		sqn = __atomic_fetch_add(&sa->sqn.outb, n, __ATOMIC_RELAXED) + n;
+		sqn = rte_atomic_fetch_add_explicit(&sa->sqn.outb, n, rte_memory_order_relaxed) + n;
  	else {
  		sqn = sa->sqn.outb + n;
  		sa->sqn.outb = sqn;
lib/ipsec/sa.h +0 -0
index ce4af8c..4b30bea 100644

lib/ipsec/sa.h struct rte_ipsec_sa { · line 130

Quoted 8 lines
lib/ipsec/sa.h +1 -1
@@ -124,7 +124,7 @@ struct rte_ipsec_sa {
  	 * place from other frequently accessed data.
  	 */
  	union {
-		uint64_t outb;
+		RTE_ATOMIC(uint64_t) outb;
  		struct {
  			uint32_t rdidx; /* read index */
  			uint32_t wridx; /* write index */
#30 lore raw

PATCH v2 7/19 mbuf: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/mbuf/rte_mbuf.h      | 20 ++++++++++----------
 lib/mbuf/rte_mbuf_core.h |  5 +++--
 2 files changed, 13 insertions(+), 12 deletions(-)
lib/mbuf/rte_mbuf.h +10 -10
index 913c459..b8ab477 100644
@@ -361,7 +361,7 @@ struct rte_pktmbuf_pool_private {
 static inline uint16_t
 rte_mbuf_refcnt_read(const struct rte_mbuf *m)
 {
-	return __atomic_load_n(&m->refcnt, __ATOMIC_RELAXED);
+	return rte_atomic_load_explicit(&m->refcnt, rte_memory_order_relaxed);
 }
 
 /**
@@ -374,15 +374,15 @@ struct rte_pktmbuf_pool_private {
 static inline void
 rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
 {
-	__atomic_store_n(&m->refcnt, new_value, __ATOMIC_RELAXED);
+	rte_atomic_store_explicit(&m->refcnt, new_value, rte_memory_order_relaxed);
 }
 
 /* internal */
 static inline uint16_t
 __rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
 {
-	return __atomic_fetch_add(&m->refcnt, value,
-				 __ATOMIC_ACQ_REL) + value;
+	return rte_atomic_fetch_add_explicit(&m->refcnt, value,
+				 rte_memory_order_acq_rel) + value;
 }
 
 /**
@@ -463,7 +463,7 @@ struct rte_pktmbuf_pool_private {
 static inline uint16_t
 rte_mbuf_ext_refcnt_read(const struct rte_mbuf_ext_shared_info *shinfo)
 {
-	return __atomic_load_n(&shinfo->refcnt, __ATOMIC_RELAXED);
+	return rte_atomic_load_explicit(&shinfo->refcnt, rte_memory_order_relaxed);
 }
 
 /**
@@ -478,7 +478,7 @@ struct rte_pktmbuf_pool_private {
 rte_mbuf_ext_refcnt_set(struct rte_mbuf_ext_shared_info *shinfo,
 	uint16_t new_value)
 {
-	__atomic_store_n(&shinfo->refcnt, new_value, __ATOMIC_RELAXED);
+	rte_atomic_store_explicit(&shinfo->refcnt, new_value, rte_memory_order_relaxed);
 }
 
 /**
@@ -502,8 +502,8 @@ struct rte_pktmbuf_pool_private {
 		return (uint16_t)value;
 	}
 
-	return __atomic_fetch_add(&shinfo->refcnt, value,
-				 __ATOMIC_ACQ_REL) + value;
+	return rte_atomic_fetch_add_explicit(&shinfo->refcnt, value,
+				 rte_memory_order_acq_rel) + value;
 }
 
 /** Mbuf prefetch */
@@ -1315,8 +1315,8 @@ static inline int __rte_pktmbuf_pinned_extbuf_decref(struct rte_mbuf *m)
 	 * Direct usage of add primitive to avoid
 	 * duplication of comparing with one.
 	 */
-	if (likely(__atomic_fetch_add(&shinfo->refcnt, -1,
-				     __ATOMIC_ACQ_REL) - 1))
+	if (likely(rte_atomic_fetch_add_explicit(&shinfo->refcnt, -1,
+				     rte_memory_order_acq_rel) - 1))
 		return 1;
 
 	/* Reinitialize counter before mbuf freeing. */
lib/mbuf/rte_mbuf_core.h +3 -2
index e9bc0d1..5688683 100644
@@ -19,6 +19,7 @@
 #include <stdint.h>
 
 #include <rte_byteorder.h>
+#include <rte_stdatomic.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -497,7 +498,7 @@ struct rte_mbuf {
 	 * rte_mbuf_refcnt_set(). The functionality of these functions (atomic,
 	 * or non-atomic) is controlled by the RTE_MBUF_REFCNT_ATOMIC flag.
 	 */
-	uint16_t refcnt;
+	RTE_ATOMIC(uint16_t) refcnt;
 
 	/**
 	 * Number of segments. Only valid for the first segment of an mbuf
@@ -674,7 +675,7 @@ struct rte_mbuf {
 struct rte_mbuf_ext_shared_info {
 	rte_mbuf_extbuf_free_callback_t free_cb; /**< Free callback function */
 	void *fcb_opaque;                        /**< Free callback argument */
-	uint16_t refcnt;
+	RTE_ATOMIC(uint16_t) refcnt;
 };
 
 /** Maximum number of nb_segs allowed. */
Signature · 1 line
1.8.3.1
Acked-by on v2 7/19
#31 lore raw

Re: [PATCH v2 07/19] mbuf: use rte optional stdatomic API

17.10.2023 21:31, Tyler Retzlaff пишет:

Quoted 103 lines

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
  lib/mbuf/rte_mbuf.h      | 20 ++++++++++----------
  lib/mbuf/rte_mbuf_core.h |  5 +++--
  2 files changed, 13 insertions(+), 12 deletions(-)
lib/mbuf/rte_mbuf.h +10 -10
index 913c459..b8ab477 100644
@@ -361,7 +361,7 @@ struct rte_pktmbuf_pool_private {
  static inline uint16_t
  rte_mbuf_refcnt_read(const struct rte_mbuf *m)
  {
-	return __atomic_load_n(&m->refcnt, __ATOMIC_RELAXED);
+	return rte_atomic_load_explicit(&m->refcnt, rte_memory_order_relaxed);
  }
 
  /**
@@ -374,15 +374,15 @@ struct rte_pktmbuf_pool_private {
  static inline void
  rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
  {
-	__atomic_store_n(&m->refcnt, new_value, __ATOMIC_RELAXED);
+	rte_atomic_store_explicit(&m->refcnt, new_value, rte_memory_order_relaxed);
  }
 
  /* internal */
  static inline uint16_t
  __rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
  {
-	return __atomic_fetch_add(&m->refcnt, value,
-				 __ATOMIC_ACQ_REL) + value;
+	return rte_atomic_fetch_add_explicit(&m->refcnt, value,
+				 rte_memory_order_acq_rel) + value;
  }
 
  /**
@@ -463,7 +463,7 @@ struct rte_pktmbuf_pool_private {
  static inline uint16_t
  rte_mbuf_ext_refcnt_read(const struct rte_mbuf_ext_shared_info *shinfo)
  {
-	return __atomic_load_n(&shinfo->refcnt, __ATOMIC_RELAXED);
+	return rte_atomic_load_explicit(&shinfo->refcnt, rte_memory_order_relaxed);
  }
 
  /**
@@ -478,7 +478,7 @@ struct rte_pktmbuf_pool_private {
  rte_mbuf_ext_refcnt_set(struct rte_mbuf_ext_shared_info *shinfo,
  	uint16_t new_value)
  {
-	__atomic_store_n(&shinfo->refcnt, new_value, __ATOMIC_RELAXED);
+	rte_atomic_store_explicit(&shinfo->refcnt, new_value, rte_memory_order_relaxed);
  }
 
  /**
@@ -502,8 +502,8 @@ struct rte_pktmbuf_pool_private {
  		return (uint16_t)value;
  	}
 
-	return __atomic_fetch_add(&shinfo->refcnt, value,
-				 __ATOMIC_ACQ_REL) + value;
+	return rte_atomic_fetch_add_explicit(&shinfo->refcnt, value,
+				 rte_memory_order_acq_rel) + value;
  }
 
  /** Mbuf prefetch */
@@ -1315,8 +1315,8 @@ static inline int __rte_pktmbuf_pinned_extbuf_decref(struct rte_mbuf *m)
  	 * Direct usage of add primitive to avoid
  	 * duplication of comparing with one.
  	 */
-	if (likely(__atomic_fetch_add(&shinfo->refcnt, -1,
-				     __ATOMIC_ACQ_REL) - 1))
+	if (likely(rte_atomic_fetch_add_explicit(&shinfo->refcnt, -1,
+				     rte_memory_order_acq_rel) - 1))
  		return 1;
 
  	/* Reinitialize counter before mbuf freeing. */
lib/mbuf/rte_mbuf_core.h +2 -1
index e9bc0d1..5688683 100644
@@ -19,6 +19,7 @@
  #include <stdint.h>
 
  #include <rte_byteorder.h>
+#include <rte_stdatomic.h>
 
  #ifdef __cplusplus
  extern "C" {
@@ -497,7 +498,7 @@ struct rte_mbuf {
  	 * rte_mbuf_refcnt_set(). The functionality of these functions (atomic,
  	 * or non-atomic) is controlled by the RTE_MBUF_REFCNT_ATOMIC flag.
  	 */
-	uint16_t refcnt;
+	RTE_ATOMIC(uint16_t) refcnt;
 
  	/**
  	 * Number of segments. Only valid for the first segment of an mbuf

lib/mbuf/rte_mbuf_core.h struct rte_mbuf { · line 681

Quoted 8 lines
lib/mbuf/rte_mbuf_core.h +1 -1
@@ -674,7 +675,7 @@ struct rte_mbuf {
  struct rte_mbuf_ext_shared_info {
  	rte_mbuf_extbuf_free_callback_t free_cb; /**< Free callback function */
  	void *fcb_opaque;                        /**< Free callback argument */
-	uint16_t refcnt;
+	RTE_ATOMIC(uint16_t) refcnt;
  };
 
  /** Maximum number of nb_segs allowed. */
#32 lore raw

PATCH v2 8/19 mempool: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/mempool/rte_mempool.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
lib/mempool/rte_mempool.h +2 -2
index f70bf36..df87cd2 100644
@@ -327,8 +327,8 @@ struct rte_mempool {
 		if (likely(__lcore_id < RTE_MAX_LCORE))                         \
 			(mp)->stats[__lcore_id].name += (n);                    \
 		else                                                            \
-			__atomic_fetch_add(&((mp)->stats[RTE_MAX_LCORE].name),  \
-					   (n), __ATOMIC_RELAXED);              \
+			rte_atomic_fetch_add_explicit(&((mp)->stats[RTE_MAX_LCORE].name),  \
+					   (n), rte_memory_order_relaxed);              \
 	} while (0)
 #else
 #define RTE_MEMPOOL_STAT_ADD(mp, name, n) do {} while (0)
Signature · 1 line
1.8.3.1
Acked-by on v2 8/19
#33 lore raw

Re: [PATCH v2 08/19] mempool: use rte optional stdatomic API

17.10.2023 21:31, Tyler Retzlaff пишет:

Quoted 13 lines

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
  lib/mempool/rte_mempool.h | 4 ++--
  1 file changed, 2 insertions(+), 2 deletions(-)
lib/mempool/rte_mempool.h +0 -0
index f70bf36..df87cd2 100644

lib/mempool/rte_mempool.h struct rte_mempool { · line 334

Quoted 10 lines
lib/mempool/rte_mempool.h +2 -2
@@ -327,8 +327,8 @@ struct rte_mempool {
  		if (likely(__lcore_id < RTE_MAX_LCORE))                         \
  			(mp)->stats[__lcore_id].name += (n);                    \
  		else                                                            \
-			__atomic_fetch_add(&((mp)->stats[RTE_MAX_LCORE].name),  \
-					   (n), __ATOMIC_RELAXED);              \
+			rte_atomic_fetch_add_explicit(&((mp)->stats[RTE_MAX_LCORE].name),  \
+					   (n), rte_memory_order_relaxed);              \
  	} while (0)
  #else
  #define RTE_MEMPOOL_STAT_ADD(mp, name, n) do {} while (0)
#34 lore raw

PATCH v2 9/19 rcu: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/rcu/rte_rcu_qsbr.c | 48 +++++++++++++++++------------------
 lib/rcu/rte_rcu_qsbr.h | 68 +++++++++++++++++++++++++-------------------------
 2 files changed, 58 insertions(+), 58 deletions(-)
lib/rcu/rte_rcu_qsbr.c +24 -24
index 17be93e..4dc7714 100644
@@ -102,21 +102,21 @@
 	 * go out of sync. Hence, additional checks are required.
 	 */
 	/* Check if the thread is already registered */
-	old_bmap = __atomic_load_n(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					__ATOMIC_RELAXED);
+	old_bmap = rte_atomic_load_explicit(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
+					rte_memory_order_relaxed);
 	if (old_bmap & 1UL << id)
 		return 0;
 
 	do {
 		new_bmap = old_bmap | (1UL << id);
-		success = __atomic_compare_exchange(
+		success = rte_atomic_compare_exchange_strong_explicit(
 					__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					&old_bmap, &new_bmap, 0,
-					__ATOMIC_RELEASE, __ATOMIC_RELAXED);
+					&old_bmap, new_bmap,
+					rte_memory_order_release, rte_memory_order_relaxed);
 
 		if (success)
-			__atomic_fetch_add(&v->num_threads,
-						1, __ATOMIC_RELAXED);
+			rte_atomic_fetch_add_explicit(&v->num_threads,
+						1, rte_memory_order_relaxed);
 		else if (old_bmap & (1UL << id))
 			/* Someone else registered this thread.
 			 * Counter should not be incremented.
@@ -154,8 +154,8 @@
 	 * go out of sync. Hence, additional checks are required.
 	 */
 	/* Check if the thread is already unregistered */
-	old_bmap = __atomic_load_n(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					__ATOMIC_RELAXED);
+	old_bmap = rte_atomic_load_explicit(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
+					rte_memory_order_relaxed);
 	if (!(old_bmap & (1UL << id)))
 		return 0;
 
@@ -165,14 +165,14 @@
 		 * completed before removal of the thread from the list of
 		 * reporting threads.
 		 */
-		success = __atomic_compare_exchange(
+		success = rte_atomic_compare_exchange_strong_explicit(
 					__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					&old_bmap, &new_bmap, 0,
-					__ATOMIC_RELEASE, __ATOMIC_RELAXED);
+					&old_bmap, new_bmap,
+					rte_memory_order_release, rte_memory_order_relaxed);
 
 		if (success)
-			__atomic_fetch_sub(&v->num_threads,
-						1, __ATOMIC_RELAXED);
+			rte_atomic_fetch_sub_explicit(&v->num_threads,
+						1, rte_memory_order_relaxed);
 		else if (!(old_bmap & (1UL << id)))
 			/* Someone else unregistered this thread.
 			 * Counter should not be incremented.
@@ -227,8 +227,8 @@
 
 	fprintf(f, "  Registered thread IDs = ");
 	for (i = 0; i < v->num_elems; i++) {
-		bmap = __atomic_load_n(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					__ATOMIC_ACQUIRE);
+		bmap = rte_atomic_load_explicit(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
+					rte_memory_order_acquire);
 		id = i << __RTE_QSBR_THRID_INDEX_SHIFT;
 		while (bmap) {
 			t = __builtin_ctzl(bmap);
@@ -241,26 +241,26 @@
 	fprintf(f, "\n");
 
 	fprintf(f, "  Token = %" PRIu64 "\n",
-			__atomic_load_n(&v->token, __ATOMIC_ACQUIRE));
+			rte_atomic_load_explicit(&v->token, rte_memory_order_acquire));
 
 	fprintf(f, "  Least Acknowledged Token = %" PRIu64 "\n",
-			__atomic_load_n(&v->acked_token, __ATOMIC_ACQUIRE));
+			rte_atomic_load_explicit(&v->acked_token, rte_memory_order_acquire));
 
 	fprintf(f, "Quiescent State Counts for readers:\n");
 	for (i = 0; i < v->num_elems; i++) {
-		bmap = __atomic_load_n(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					__ATOMIC_ACQUIRE);
+		bmap = rte_atomic_load_explicit(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
+					rte_memory_order_acquire);
 		id = i << __RTE_QSBR_THRID_INDEX_SHIFT;
 		while (bmap) {
 			t = __builtin_ctzl(bmap);
 			fprintf(f, "thread ID = %u, count = %" PRIu64 ", lock count = %u\n",
 				id + t,
-				__atomic_load_n(
+				rte_atomic_load_explicit(
 					&v->qsbr_cnt[id + t].cnt,
-					__ATOMIC_RELAXED),
-				__atomic_load_n(
+					rte_memory_order_relaxed),
+				rte_atomic_load_explicit(
 					&v->qsbr_cnt[id + t].lock_cnt,
-					__ATOMIC_RELAXED));
+					rte_memory_order_relaxed));
 			bmap &= ~(1UL << t);
 		}
 	}
lib/rcu/rte_rcu_qsbr.h +34 -34

195 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#35 lore raw

RE: [PATCH v2 09/19] rcu: use rte optional stdatomic API

Quoted 150 lines

-----Original Message-----
From: Tyler Retzlaff <[email protected]>
Sent: Wednesday, October 18, 2023 4:31 AM
To: [email protected]
Cc: Akhil Goyal <[email protected]>; Anatoly Burakov <[email protected]>; Andrew Rybchenko <[email protected]>; Bruce Richardson <[email protected]>; Chenbo Xia <[email protected]>; Ciara Power <[email protected]>; David Christensen <[email protected]>; David Hunt <[email protected]>; Dmitry Kozlyuk <[email protected]>; Dmitry Malloy <[email protected]>; Elena Agostini <[email protected]>; Erik Gabriel Carrillo <[email protected]>; Fan Zhang <[email protected]>; Ferruh Yigit <[email protected]>; Harman Kalra <[email protected]>; Harry van Haaren <[email protected]>; Honnappa Nagarahalli <[email protected]>; [email protected]; Konstantin Ananyev <[email protected]>; Matan Azrad <[email protected]>; Maxime Coquelin <[email protected]>; Narcisa Ana Maria Vasile <[email protected]>; Nicolas Chautru <[email protected]>; Olivier Matz <[email protected]>; Ori Kam <[email protected]>; Pallavi Kadam <[email protected]>; Pavan Nikhilesh <[email protected]>; Reshma Pattan <[email protected]>; Sameh Gobriel <[email protected]>; Shijith Thotton <[email protected]>; Sivaprasad Tummala <[email protected]>; Stephen Hemminger <[email protected]>; Suanming Mou <[email protected]>; Sunil Kumar Kori <[email protected]>; [email protected]; Viacheslav Ovsiienko <[email protected]>; Vladimir Medvedkin <[email protected]>; Yipeng Wang <[email protected]>; Tyler Retzlaff <[email protected]>
Subject: [PATCH v2 09/19] rcu: use rte optional stdatomic API

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/rcu/rte_rcu_qsbr.c | 48 +++++++++++++++++------------------
 lib/rcu/rte_rcu_qsbr.h | 68 +++++++++++++++++++++++++-------------------------
 2 files changed, 58 insertions(+), 58 deletions(-)
lib/rcu/rte_rcu_qsbr.c +25 -24
@@ -102,21 +102,21 @@
 	 * go out of sync. Hence, additional checks are required.
 	 */
 	/* Check if the thread is already registered */
-	old_bmap = __atomic_load_n(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					__ATOMIC_RELAXED);
+	old_bmap = rte_atomic_load_explicit(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
+					rte_memory_order_relaxed);
 	if (old_bmap & 1UL << id)
 		return 0;
 
 	do {
 		new_bmap = old_bmap | (1UL << id);
-		success = __atomic_compare_exchange(
+		success = rte_atomic_compare_exchange_strong_explicit(
 					__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					&old_bmap, &new_bmap, 0,
-					__ATOMIC_RELEASE, __ATOMIC_RELAXED);
+					&old_bmap, new_bmap,
+					rte_memory_order_release, rte_memory_order_relaxed);
 
 		if (success)
-			__atomic_fetch_add(&v->num_threads,
-						1, __ATOMIC_RELAXED);
+			rte_atomic_fetch_add_explicit(&v->num_threads,
+						1, rte_memory_order_relaxed);
 		else if (old_bmap & (1UL << id))
 			/* Someone else registered this thread.
 			 * Counter should not be incremented.
@@ -154,8 +154,8 @@
 	 * go out of sync. Hence, additional checks are required.
 	 */
 	/* Check if the thread is already unregistered */
-	old_bmap = __atomic_load_n(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					__ATOMIC_RELAXED);
+	old_bmap = rte_atomic_load_explicit(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
+					rte_memory_order_relaxed);
 	if (!(old_bmap & (1UL << id)))
 		return 0;
 
@@ -165,14 +165,14 @@
 		 * completed before removal of the thread from the list of
 		 * reporting threads.
 		 */
-		success = __atomic_compare_exchange(
+		success = rte_atomic_compare_exchange_strong_explicit(
 					__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					&old_bmap, &new_bmap, 0,
-					__ATOMIC_RELEASE, __ATOMIC_RELAXED);
+					&old_bmap, new_bmap,
+					rte_memory_order_release, rte_memory_order_relaxed);
 
 		if (success)
-			__atomic_fetch_sub(&v->num_threads,
-						1, __ATOMIC_RELAXED);
+			rte_atomic_fetch_sub_explicit(&v->num_threads,
+						1, rte_memory_order_relaxed);
 		else if (!(old_bmap & (1UL << id)))
 			/* Someone else unregistered this thread.
 			 * Counter should not be incremented.
@@ -227,8 +227,8 @@
 
 	fprintf(f, "  Registered thread IDs = ");
 	for (i = 0; i < v->num_elems; i++) {
-		bmap = __atomic_load_n(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					__ATOMIC_ACQUIRE);
+		bmap = rte_atomic_load_explicit(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
+					rte_memory_order_acquire);
 		id = i << __RTE_QSBR_THRID_INDEX_SHIFT;
 		while (bmap) {
 			t = __builtin_ctzl(bmap);
@@ -241,26 +241,26 @@
 	fprintf(f, "\n");
 
 	fprintf(f, "  Token = %" PRIu64 "\n",
-			__atomic_load_n(&v->token, __ATOMIC_ACQUIRE));
+			rte_atomic_load_explicit(&v->token, rte_memory_order_acquire));
 
 	fprintf(f, "  Least Acknowledged Token = %" PRIu64 "\n",
-			__atomic_load_n(&v->acked_token, __ATOMIC_ACQUIRE));
+			rte_atomic_load_explicit(&v->acked_token,
+rte_memory_order_acquire));
 
 	fprintf(f, "Quiescent State Counts for readers:\n");
 	for (i = 0; i < v->num_elems; i++) {
-		bmap = __atomic_load_n(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					__ATOMIC_ACQUIRE);
+		bmap = rte_atomic_load_explicit(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
+					rte_memory_order_acquire);
 		id = i << __RTE_QSBR_THRID_INDEX_SHIFT;
 		while (bmap) {
 			t = __builtin_ctzl(bmap);
 			fprintf(f, "thread ID = %u, count = %" PRIu64 ", lock count = %u\n",
 				id + t,
-				__atomic_load_n(
+				rte_atomic_load_explicit(
 					&v->qsbr_cnt[id + t].cnt,
-					__ATOMIC_RELAXED),
-				__atomic_load_n(
+					rte_memory_order_relaxed),
+				rte_atomic_load_explicit(
 					&v->qsbr_cnt[id + t].lock_cnt,
-					__ATOMIC_RELAXED));
+					rte_memory_order_relaxed));
 			bmap &= ~(1UL << t);
 		}
 	}
lib/rcu/rte_rcu_qsbr.h +0 -0
@@ -63,11 +63,11 @@
  * Given thread id needs to be converted to index into the array and

lib/rcu/rte_rcu_qsbr.h line 72

Quoted 10 lines
lib/rcu/rte_rcu_qsbr.h +3 -1
  * the id within the array element.
  */
-#define __RTE_QSBR_THRID_ARRAY_ELM_SIZE (sizeof(uint64_t) * 8)
+#define __RTE_QSBR_THRID_ARRAY_ELM_SIZE (sizeof(RTE_ATOMIC(uint64_t)) *
+8)
 #define __RTE_QSBR_THRID_ARRAY_SIZE(max_threads) \
 	RTE_ALIGN(RTE_ALIGN_MUL_CEIL(max_threads, \
 		__RTE_QSBR_THRID_ARRAY_ELM_SIZE) >> 3, RTE_CACHE_LINE_SIZE) -#define
 __RTE_QSBR_THRID_ARRAY_ELM(v, i) ((uint64_t *) \
+#define __RTE_QSBR_THRID_ARRAY_ELM(v, i) ((uint64_t __rte_atomic *) \

Is it equivalent to ((RTE_ATOMIC(uint64_t) *)?

Quoted 3 lines
lib/rcu/rte_rcu_qsbr.h +0 -0
 	((struct rte_rcu_qsbr_cnt *)(v + 1) + v->max_threads) + i)  #define

__RTE_QSBR_THRID_INDEX_SHIFT 6 #define __RTE_QSBR_THRID_MASK 0x3f @@ -75,13 +75,13 @@

<snip>

#36 lore raw

Re: [PATCH v2 09/19] rcu: use rte optional stdatomic API

Quoted 154 lines · Ruifeng Wang
Quoted 154 lines

-----Original Message-----
From: Tyler Retzlaff <[email protected]>
Sent: Wednesday, October 18, 2023 4:31 AM
To: [email protected]
Cc: Akhil Goyal <[email protected]>; Anatoly Burakov <[email protected]>; Andrew Rybchenko <[email protected]>; Bruce Richardson <[email protected]>; Chenbo Xia <[email protected]>; Ciara Power <[email protected]>; David Christensen <[email protected]>; David Hunt <[email protected]>; Dmitry Kozlyuk <[email protected]>; Dmitry Malloy <[email protected]>; Elena Agostini <[email protected]>; Erik Gabriel Carrillo <[email protected]>; Fan Zhang <[email protected]>; Ferruh Yigit <[email protected]>; Harman Kalra <[email protected]>; Harry van Haaren <[email protected]>; Honnappa Nagarahalli <[email protected]>; [email protected]; Konstantin Ananyev <[email protected]>; Matan Azrad <[email protected]>; Maxime Coquelin <[email protected]>; Narcisa Ana Maria Vasile <[email protected]>; Nicolas Chautru <[email protected]>; Olivier Matz <[email protected]>; Ori Kam <[email protected]>; Pallavi Kadam <[email protected]>; Pavan Nikhilesh <[email protected]>; Reshma Pattan <[email protected]>; Sameh Gobriel <[email protected]>; Shijith Thotton <[email protected]>; Sivaprasad Tummala <[email protected]>; Stephen Hemminger <[email protected]>; Suanming Mou <[email protected]>; Sunil Kumar Kori <[email protected]>; [email protected]; Viacheslav Ovsiienko <[email protected]>; Vladimir Medvedkin <[email protected]>; Yipeng Wang <[email protected]>; Tyler Retzlaff <[email protected]>
Subject: [PATCH v2 09/19] rcu: use rte optional stdatomic API

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/rcu/rte_rcu_qsbr.c | 48 +++++++++++++++++------------------
 lib/rcu/rte_rcu_qsbr.h | 68 +++++++++++++++++++++++++-------------------------
 2 files changed, 58 insertions(+), 58 deletions(-)
lib/rcu/rte_rcu_qsbr.c +25 -24
@@ -102,21 +102,21 @@
 	 * go out of sync. Hence, additional checks are required.
 	 */
 	/* Check if the thread is already registered */
-	old_bmap = __atomic_load_n(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					__ATOMIC_RELAXED);
+	old_bmap = rte_atomic_load_explicit(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
+					rte_memory_order_relaxed);
 	if (old_bmap & 1UL << id)
 		return 0;
 
 	do {
 		new_bmap = old_bmap | (1UL << id);
-		success = __atomic_compare_exchange(
+		success = rte_atomic_compare_exchange_strong_explicit(
 					__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					&old_bmap, &new_bmap, 0,
-					__ATOMIC_RELEASE, __ATOMIC_RELAXED);
+					&old_bmap, new_bmap,
+					rte_memory_order_release, rte_memory_order_relaxed);
 
 		if (success)
-			__atomic_fetch_add(&v->num_threads,
-						1, __ATOMIC_RELAXED);
+			rte_atomic_fetch_add_explicit(&v->num_threads,
+						1, rte_memory_order_relaxed);
 		else if (old_bmap & (1UL << id))
 			/* Someone else registered this thread.
 			 * Counter should not be incremented.
@@ -154,8 +154,8 @@
 	 * go out of sync. Hence, additional checks are required.
 	 */
 	/* Check if the thread is already unregistered */
-	old_bmap = __atomic_load_n(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					__ATOMIC_RELAXED);
+	old_bmap = rte_atomic_load_explicit(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
+					rte_memory_order_relaxed);
 	if (!(old_bmap & (1UL << id)))
 		return 0;
 
@@ -165,14 +165,14 @@
 		 * completed before removal of the thread from the list of
 		 * reporting threads.
 		 */
-		success = __atomic_compare_exchange(
+		success = rte_atomic_compare_exchange_strong_explicit(
 					__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					&old_bmap, &new_bmap, 0,
-					__ATOMIC_RELEASE, __ATOMIC_RELAXED);
+					&old_bmap, new_bmap,
+					rte_memory_order_release, rte_memory_order_relaxed);
 
 		if (success)
-			__atomic_fetch_sub(&v->num_threads,
-						1, __ATOMIC_RELAXED);
+			rte_atomic_fetch_sub_explicit(&v->num_threads,
+						1, rte_memory_order_relaxed);
 		else if (!(old_bmap & (1UL << id)))
 			/* Someone else unregistered this thread.
 			 * Counter should not be incremented.
@@ -227,8 +227,8 @@
 
 	fprintf(f, "  Registered thread IDs = ");
 	for (i = 0; i < v->num_elems; i++) {
-		bmap = __atomic_load_n(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					__ATOMIC_ACQUIRE);
+		bmap = rte_atomic_load_explicit(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
+					rte_memory_order_acquire);
 		id = i << __RTE_QSBR_THRID_INDEX_SHIFT;
 		while (bmap) {
 			t = __builtin_ctzl(bmap);
@@ -241,26 +241,26 @@
 	fprintf(f, "\n");
 
 	fprintf(f, "  Token = %" PRIu64 "\n",
-			__atomic_load_n(&v->token, __ATOMIC_ACQUIRE));
+			rte_atomic_load_explicit(&v->token, rte_memory_order_acquire));
 
 	fprintf(f, "  Least Acknowledged Token = %" PRIu64 "\n",
-			__atomic_load_n(&v->acked_token, __ATOMIC_ACQUIRE));
+			rte_atomic_load_explicit(&v->acked_token,
+rte_memory_order_acquire));
 
 	fprintf(f, "Quiescent State Counts for readers:\n");
 	for (i = 0; i < v->num_elems; i++) {
-		bmap = __atomic_load_n(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					__ATOMIC_ACQUIRE);
+		bmap = rte_atomic_load_explicit(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
+					rte_memory_order_acquire);
 		id = i << __RTE_QSBR_THRID_INDEX_SHIFT;
 		while (bmap) {
 			t = __builtin_ctzl(bmap);
 			fprintf(f, "thread ID = %u, count = %" PRIu64 ", lock count = %u\n",
 				id + t,
-				__atomic_load_n(
+				rte_atomic_load_explicit(
 					&v->qsbr_cnt[id + t].cnt,
-					__ATOMIC_RELAXED),
-				__atomic_load_n(
+					rte_memory_order_relaxed),
+				rte_atomic_load_explicit(
 					&v->qsbr_cnt[id + t].lock_cnt,
-					__ATOMIC_RELAXED));
+					rte_memory_order_relaxed));
 			bmap &= ~(1UL << t);
 		}
 	}
lib/rcu/rte_rcu_qsbr.h +1 -1
@@ -63,11 +63,11 @@
  * Given thread id needs to be converted to index into the array and
  * the id within the array element.
  */
-#define __RTE_QSBR_THRID_ARRAY_ELM_SIZE (sizeof(uint64_t) * 8)
+#define __RTE_QSBR_THRID_ARRAY_ELM_SIZE (sizeof(RTE_ATOMIC(uint64_t)) *
Quoted 8 lines

lib/rcu/rte_rcu_qsbr.h line 70

Quoted 6 lines
lib/rcu/rte_rcu_qsbr.h +2 -0
+8)
 #define __RTE_QSBR_THRID_ARRAY_SIZE(max_threads) \
 	RTE_ALIGN(RTE_ALIGN_MUL_CEIL(max_threads, \
 		__RTE_QSBR_THRID_ARRAY_ELM_SIZE) >> 3, RTE_CACHE_LINE_SIZE) -#define
 __RTE_QSBR_THRID_ARRAY_ELM(v, i) ((uint64_t *) \
+#define __RTE_QSBR_THRID_ARRAY_ELM(v, i) ((uint64_t __rte_atomic *) \

Is it equivalent to ((RTE_ATOMIC(uint64_t) *)?

i'm not sure if you're asking about the resultant type of the expression or not?

in this context we aren't specifying an atomic type but rather adding the atomic qualifier to what should already be a variable that has an atomic specified type with a cast which is why we use __rte_atomic.

Quoted 6 lines
Quoted 4 lines
lib/rcu/rte_rcu_qsbr.h +0 -0
 	((struct rte_rcu_qsbr_cnt *)(v + 1) + v->max_threads) + i)  #define

__RTE_QSBR_THRID_INDEX_SHIFT 6 #define __RTE_QSBR_THRID_MASK 0x3f @@ -75,13 +75,13 @@

<snip>

#37 lore raw

RE: [PATCH v2 09/19] rcu: use rte optional stdatomic API

Quoted 204 lines

-----Original Message-----
From: Tyler Retzlaff <[email protected]>
Sent: Thursday, October 26, 2023 6:38 AM
To: Ruifeng Wang <[email protected]>
Cc: [email protected]; Akhil Goyal <[email protected]>; Anatoly Burakov <[email protected]>; Andrew Rybchenko <[email protected]>; Bruce Richardson <[email protected]>; Chenbo Xia <[email protected]>; Ciara Power <[email protected]>; David Christensen <[email protected]>; David Hunt <[email protected]>; Dmitry Kozlyuk <[email protected]>; Dmitry Malloy <[email protected]>; Elena Agostini <[email protected]>; Erik Gabriel Carrillo <[email protected]>; Fan Zhang <[email protected]>; Ferruh Yigit <[email protected]>; Harman Kalra <[email protected]>; Harry van Haaren <[email protected]>; Honnappa Nagarahalli <[email protected]>; [email protected]; Konstantin Ananyev <[email protected]>; Matan Azrad <[email protected]>; Maxime Coquelin <[email protected]>; Narcisa Ana Maria Vasile <[email protected]>; Nicolas Chautru <[email protected]>; Olivier Matz <[email protected]>; Ori Kam <[email protected]>; Pallavi Kadam <[email protected]>; Pavan Nikhilesh <[email protected]>; Reshma Pattan <[email protected]>; Sameh Gobriel <[email protected]>; Shijith Thotton <[email protected]>; Sivaprasad Tummala <[email protected]>; Stephen Hemminger <[email protected]>; Suanming Mou <[email protected]>; Sunil Kumar Kori <[email protected]>; [email protected]; Viacheslav Ovsiienko <[email protected]>; Vladimir Medvedkin <[email protected]>; Yipeng Wang <[email protected]>; nd <[email protected]>
Subject: Re: [PATCH v2 09/19] rcu: use rte optional stdatomic API

Quoted 168 lines · Ruifeng Wang
Quoted 168 lines

-----Original Message-----
From: Tyler Retzlaff <[email protected]>
Sent: Wednesday, October 18, 2023 4:31 AM
To: [email protected]
Cc: Akhil Goyal <[email protected]>; Anatoly Burakov <[email protected]>; Andrew Rybchenko <[email protected]>; Bruce Richardson <[email protected]>; Chenbo Xia <[email protected]>; Ciara Power <[email protected]>; David Christensen <[email protected]>; David Hunt <[email protected]>; Dmitry Kozlyuk <[email protected]>; Dmitry Malloy <[email protected]>; Elena Agostini <[email protected]>; Erik Gabriel Carrillo <[email protected]>; Fan Zhang <[email protected]>; Ferruh Yigit <[email protected]>; Harman Kalra <[email protected]>; Harry van Haaren <[email protected]>; Honnappa Nagarahalli <[email protected]>; [email protected]; Konstantin Ananyev <[email protected]>; Matan Azrad <[email protected]>; Maxime Coquelin <[email protected]>; Narcisa Ana Maria Vasile <[email protected]>; Nicolas Chautru <[email protected]>; Olivier Matz <[email protected]>; Ori Kam <[email protected]>; Pallavi Kadam <[email protected]>; Pavan Nikhilesh <[email protected]>; Reshma Pattan <[email protected]>; Sameh Gobriel <[email protected]>; Shijith Thotton <[email protected]>; Sivaprasad Tummala <[email protected]>; Stephen Hemminger <[email protected]>; Suanming Mou <[email protected]>; Sunil Kumar Kori <[email protected]>; [email protected]; Viacheslav Ovsiienko <[email protected]>; Vladimir Medvedkin <[email protected]>; Yipeng Wang <[email protected]>; Tyler Retzlaff <[email protected]>
Subject: [PATCH v2 09/19] rcu: use rte optional stdatomic API

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/rcu/rte_rcu_qsbr.c | 48 +++++++++++++++++------------------
 lib/rcu/rte_rcu_qsbr.h | 68
+++++++++++++++++++++++++-------------------------
 2 files changed, 58 insertions(+), 58 deletions(-)
lib/rcu/rte_rcu_qsbr.c index renamed from lib/rcu/rte_rcu_qsbr.c +0 -0

17be93e..4dc7714 100644

lib/rcu/rte_rcu_qsbr.c +25 -24
@@ -102,21 +102,21 @@
 	 * go out of sync. Hence, additional checks are required.
 	 */
 	/* Check if the thread is already registered */
-	old_bmap = __atomic_load_n(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					__ATOMIC_RELAXED);
+	old_bmap = rte_atomic_load_explicit(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
+					rte_memory_order_relaxed);
 	if (old_bmap & 1UL << id)
 		return 0;
 
 	do {
 		new_bmap = old_bmap | (1UL << id);
-		success = __atomic_compare_exchange(
+		success = rte_atomic_compare_exchange_strong_explicit(
 					__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					&old_bmap, &new_bmap, 0,
-					__ATOMIC_RELEASE, __ATOMIC_RELAXED);
+					&old_bmap, new_bmap,
+					rte_memory_order_release, rte_memory_order_relaxed);
 
 		if (success)
-			__atomic_fetch_add(&v->num_threads,
-						1, __ATOMIC_RELAXED);
+			rte_atomic_fetch_add_explicit(&v->num_threads,
+						1, rte_memory_order_relaxed);
 		else if (old_bmap & (1UL << id))
 			/* Someone else registered this thread.
 			 * Counter should not be incremented.
@@ -154,8 +154,8 @@
 	 * go out of sync. Hence, additional checks are required.
 	 */
 	/* Check if the thread is already unregistered */
-	old_bmap = __atomic_load_n(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					__ATOMIC_RELAXED);
+	old_bmap = rte_atomic_load_explicit(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
+					rte_memory_order_relaxed);
 	if (!(old_bmap & (1UL << id)))
 		return 0;
 
@@ -165,14 +165,14 @@
 		 * completed before removal of the thread from the list of
 		 * reporting threads.
 		 */
-		success = __atomic_compare_exchange(
+		success = rte_atomic_compare_exchange_strong_explicit(
 					__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					&old_bmap, &new_bmap, 0,
-					__ATOMIC_RELEASE, __ATOMIC_RELAXED);
+					&old_bmap, new_bmap,
+					rte_memory_order_release, rte_memory_order_relaxed);
 
 		if (success)
-			__atomic_fetch_sub(&v->num_threads,
-						1, __ATOMIC_RELAXED);
+			rte_atomic_fetch_sub_explicit(&v->num_threads,
+						1, rte_memory_order_relaxed);
 		else if (!(old_bmap & (1UL << id)))
 			/* Someone else unregistered this thread.
 			 * Counter should not be incremented.
@@ -227,8 +227,8 @@
 
 	fprintf(f, "  Registered thread IDs = ");
 	for (i = 0; i < v->num_elems; i++) {
-		bmap = __atomic_load_n(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					__ATOMIC_ACQUIRE);
+		bmap = rte_atomic_load_explicit(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
+					rte_memory_order_acquire);
 		id = i << __RTE_QSBR_THRID_INDEX_SHIFT;
 		while (bmap) {
 			t = __builtin_ctzl(bmap);
@@ -241,26 +241,26 @@
 	fprintf(f, "\n");
 
 	fprintf(f, "  Token = %" PRIu64 "\n",
-			__atomic_load_n(&v->token, __ATOMIC_ACQUIRE));
+			rte_atomic_load_explicit(&v->token, rte_memory_order_acquire));
 
 	fprintf(f, "  Least Acknowledged Token = %" PRIu64 "\n",
-			__atomic_load_n(&v->acked_token, __ATOMIC_ACQUIRE));
+			rte_atomic_load_explicit(&v->acked_token,
+rte_memory_order_acquire));
 
 	fprintf(f, "Quiescent State Counts for readers:\n");
 	for (i = 0; i < v->num_elems; i++) {
-		bmap = __atomic_load_n(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					__ATOMIC_ACQUIRE);
+		bmap = rte_atomic_load_explicit(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
+					rte_memory_order_acquire);
 		id = i << __RTE_QSBR_THRID_INDEX_SHIFT;
 		while (bmap) {
 			t = __builtin_ctzl(bmap);
 			fprintf(f, "thread ID = %u, count = %" PRIu64 ", lock count = %u\n",
 				id + t,
-				__atomic_load_n(
+				rte_atomic_load_explicit(
 					&v->qsbr_cnt[id + t].cnt,
-					__ATOMIC_RELAXED),
-				__atomic_load_n(
+					rte_memory_order_relaxed),
+				rte_atomic_load_explicit(
 					&v->qsbr_cnt[id + t].lock_cnt,
-					__ATOMIC_RELAXED));
+					rte_memory_order_relaxed));
 			bmap &= ~(1UL << t);
 		}
 	}
lib/rcu/rte_rcu_qsbr.h index renamed from lib/rcu/rte_rcu_qsbr.h +0 -0

87e1b55..9f4aed2 100644

lib/rcu/rte_rcu_qsbr.h +3 -1
@@ -63,11 +63,11 @@
  * Given thread id needs to be converted to index into the array and
  * the id within the array element.
  */
-#define __RTE_QSBR_THRID_ARRAY_ELM_SIZE (sizeof(uint64_t) * 8)
+#define __RTE_QSBR_THRID_ARRAY_ELM_SIZE
+(sizeof(RTE_ATOMIC(uint64_t)) *
+8)
Quoted 8 lines

lib/rcu/rte_rcu_qsbr.h

Quoted 6 lines
lib/rcu/rte_rcu_qsbr.h +2 -1
 #define __RTE_QSBR_THRID_ARRAY_SIZE(max_threads) \
 	RTE_ALIGN(RTE_ALIGN_MUL_CEIL(max_threads, \
 		__RTE_QSBR_THRID_ARRAY_ELM_SIZE) >> 3, RTE_CACHE_LINE_SIZE)
-#define __RTE_QSBR_THRID_ARRAY_ELM(v, i) ((uint64_t *) \
+#define __RTE_QSBR_THRID_ARRAY_ELM(v, i) ((uint64_t __rte_atomic *)
+\

Is it equivalent to ((RTE_ATOMIC(uint64_t) *)?

Quoted 1 line

i'm not sure if you're asking about the resultant type of the expression or not?

I see other places are using specifier hence the question.

Quoted 4 lines

in this context we aren't specifying an atomic type but rather adding the atomic qualifier to what should already be a variable that has an atomic specified type with a cast which is why we use __rte_atomic.

I read from document [1] that atomic qualified type may have a different size from the original type. If that is the case, the size difference could cause issue in bitmap array accessing. Did I misunderstand?

[1] https://en.cppreference.com/w/c/language/atomic

Quoted 8 lines
Quoted 8 lines
Quoted 6 lines

((struct rte_rcu_qsbr_cnt *)(v + 1) + v->max_threads) + i)
#define __RTE_QSBR_THRID_INDEX_SHIFT 6 #define __RTE_QSBR_THRID_MASK 0x3f @@ -75,13 +75,13 @@

<snip>

#38 lore raw

Re: [PATCH v2 09/19] rcu: use rte optional stdatomic API

Quoted 213 lines · Ruifeng Wang
Quoted 204 lines

-----Original Message-----
From: Tyler Retzlaff <[email protected]>
Sent: Thursday, October 26, 2023 6:38 AM
To: Ruifeng Wang <[email protected]>
Cc: [email protected]; Akhil Goyal <[email protected]>; Anatoly Burakov <[email protected]>; Andrew Rybchenko <[email protected]>; Bruce Richardson <[email protected]>; Chenbo Xia <[email protected]>; Ciara Power <[email protected]>; David Christensen <[email protected]>; David Hunt <[email protected]>; Dmitry Kozlyuk <[email protected]>; Dmitry Malloy <[email protected]>; Elena Agostini <[email protected]>; Erik Gabriel Carrillo <[email protected]>; Fan Zhang <[email protected]>; Ferruh Yigit <[email protected]>; Harman Kalra <[email protected]>; Harry van Haaren <[email protected]>; Honnappa Nagarahalli <[email protected]>; [email protected]; Konstantin Ananyev <[email protected]>; Matan Azrad <[email protected]>; Maxime Coquelin <[email protected]>; Narcisa Ana Maria Vasile <[email protected]>; Nicolas Chautru <[email protected]>; Olivier Matz <[email protected]>; Ori Kam <[email protected]>; Pallavi Kadam <[email protected]>; Pavan Nikhilesh <[email protected]>; Reshma Pattan <[email protected]>; Sameh Gobriel <[email protected]>; Shijith Thotton <[email protected]>; Sivaprasad Tummala <[email protected]>; Stephen Hemminger <[email protected]>; Suanming Mou <[email protected]>; Sunil Kumar Kori <[email protected]>; [email protected]; Viacheslav Ovsiienko <[email protected]>; Vladimir Medvedkin <[email protected]>; Yipeng Wang <[email protected]>; nd <[email protected]>
Subject: Re: [PATCH v2 09/19] rcu: use rte optional stdatomic API

Quoted 168 lines · Ruifeng Wang
Quoted 168 lines

-----Original Message-----
From: Tyler Retzlaff <[email protected]>
Sent: Wednesday, October 18, 2023 4:31 AM
To: [email protected]
Cc: Akhil Goyal <[email protected]>; Anatoly Burakov <[email protected]>; Andrew Rybchenko <[email protected]>; Bruce Richardson <[email protected]>; Chenbo Xia <[email protected]>; Ciara Power <[email protected]>; David Christensen <[email protected]>; David Hunt <[email protected]>; Dmitry Kozlyuk <[email protected]>; Dmitry Malloy <[email protected]>; Elena Agostini <[email protected]>; Erik Gabriel Carrillo <[email protected]>; Fan Zhang <[email protected]>; Ferruh Yigit <[email protected]>; Harman Kalra <[email protected]>; Harry van Haaren <[email protected]>; Honnappa Nagarahalli <[email protected]>; [email protected]; Konstantin Ananyev <[email protected]>; Matan Azrad <[email protected]>; Maxime Coquelin <[email protected]>; Narcisa Ana Maria Vasile <[email protected]>; Nicolas Chautru <[email protected]>; Olivier Matz <[email protected]>; Ori Kam <[email protected]>; Pallavi Kadam <[email protected]>; Pavan Nikhilesh <[email protected]>; Reshma Pattan <[email protected]>; Sameh Gobriel <[email protected]>; Shijith Thotton <[email protected]>; Sivaprasad Tummala <[email protected]>; Stephen Hemminger <[email protected]>; Suanming Mou <[email protected]>; Sunil Kumar Kori <[email protected]>; [email protected]; Viacheslav Ovsiienko <[email protected]>; Vladimir Medvedkin <[email protected]>; Yipeng Wang <[email protected]>; Tyler Retzlaff <[email protected]>
Subject: [PATCH v2 09/19] rcu: use rte optional stdatomic API

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/rcu/rte_rcu_qsbr.c | 48 +++++++++++++++++------------------
 lib/rcu/rte_rcu_qsbr.h | 68
+++++++++++++++++++++++++-------------------------
 2 files changed, 58 insertions(+), 58 deletions(-)
lib/rcu/rte_rcu_qsbr.c index renamed from lib/rcu/rte_rcu_qsbr.c +0 -0

17be93e..4dc7714 100644

lib/rcu/rte_rcu_qsbr.c +25 -24
@@ -102,21 +102,21 @@
 	 * go out of sync. Hence, additional checks are required.
 	 */
 	/* Check if the thread is already registered */
-	old_bmap = __atomic_load_n(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					__ATOMIC_RELAXED);
+	old_bmap = rte_atomic_load_explicit(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
+					rte_memory_order_relaxed);
 	if (old_bmap & 1UL << id)
 		return 0;
 
 	do {
 		new_bmap = old_bmap | (1UL << id);
-		success = __atomic_compare_exchange(
+		success = rte_atomic_compare_exchange_strong_explicit(
 					__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					&old_bmap, &new_bmap, 0,
-					__ATOMIC_RELEASE, __ATOMIC_RELAXED);
+					&old_bmap, new_bmap,
+					rte_memory_order_release, rte_memory_order_relaxed);
 
 		if (success)
-			__atomic_fetch_add(&v->num_threads,
-						1, __ATOMIC_RELAXED);
+			rte_atomic_fetch_add_explicit(&v->num_threads,
+						1, rte_memory_order_relaxed);
 		else if (old_bmap & (1UL << id))
 			/* Someone else registered this thread.
 			 * Counter should not be incremented.
@@ -154,8 +154,8 @@
 	 * go out of sync. Hence, additional checks are required.
 	 */
 	/* Check if the thread is already unregistered */
-	old_bmap = __atomic_load_n(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					__ATOMIC_RELAXED);
+	old_bmap = rte_atomic_load_explicit(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
+					rte_memory_order_relaxed);
 	if (!(old_bmap & (1UL << id)))
 		return 0;
 
@@ -165,14 +165,14 @@
 		 * completed before removal of the thread from the list of
 		 * reporting threads.
 		 */
-		success = __atomic_compare_exchange(
+		success = rte_atomic_compare_exchange_strong_explicit(
 					__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					&old_bmap, &new_bmap, 0,
-					__ATOMIC_RELEASE, __ATOMIC_RELAXED);
+					&old_bmap, new_bmap,
+					rte_memory_order_release, rte_memory_order_relaxed);
 
 		if (success)
-			__atomic_fetch_sub(&v->num_threads,
-						1, __ATOMIC_RELAXED);
+			rte_atomic_fetch_sub_explicit(&v->num_threads,
+						1, rte_memory_order_relaxed);
 		else if (!(old_bmap & (1UL << id)))
 			/* Someone else unregistered this thread.
 			 * Counter should not be incremented.
@@ -227,8 +227,8 @@
 
 	fprintf(f, "  Registered thread IDs = ");
 	for (i = 0; i < v->num_elems; i++) {
-		bmap = __atomic_load_n(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					__ATOMIC_ACQUIRE);
+		bmap = rte_atomic_load_explicit(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
+					rte_memory_order_acquire);
 		id = i << __RTE_QSBR_THRID_INDEX_SHIFT;
 		while (bmap) {
 			t = __builtin_ctzl(bmap);
@@ -241,26 +241,26 @@
 	fprintf(f, "\n");
 
 	fprintf(f, "  Token = %" PRIu64 "\n",
-			__atomic_load_n(&v->token, __ATOMIC_ACQUIRE));
+			rte_atomic_load_explicit(&v->token, rte_memory_order_acquire));
 
 	fprintf(f, "  Least Acknowledged Token = %" PRIu64 "\n",
-			__atomic_load_n(&v->acked_token, __ATOMIC_ACQUIRE));
+			rte_atomic_load_explicit(&v->acked_token,
+rte_memory_order_acquire));
 
 	fprintf(f, "Quiescent State Counts for readers:\n");
 	for (i = 0; i < v->num_elems; i++) {
-		bmap = __atomic_load_n(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					__ATOMIC_ACQUIRE);
+		bmap = rte_atomic_load_explicit(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
+					rte_memory_order_acquire);
 		id = i << __RTE_QSBR_THRID_INDEX_SHIFT;
 		while (bmap) {
 			t = __builtin_ctzl(bmap);
 			fprintf(f, "thread ID = %u, count = %" PRIu64 ", lock count = %u\n",
 				id + t,
-				__atomic_load_n(
+				rte_atomic_load_explicit(
 					&v->qsbr_cnt[id + t].cnt,
-					__ATOMIC_RELAXED),
-				__atomic_load_n(
+					rte_memory_order_relaxed),
+				rte_atomic_load_explicit(
 					&v->qsbr_cnt[id + t].lock_cnt,
-					__ATOMIC_RELAXED));
+					rte_memory_order_relaxed));
 			bmap &= ~(1UL << t);
 		}
 	}
lib/rcu/rte_rcu_qsbr.h index renamed from lib/rcu/rte_rcu_qsbr.h +0 -0

87e1b55..9f4aed2 100644

lib/rcu/rte_rcu_qsbr.h +3 -1
@@ -63,11 +63,11 @@
  * Given thread id needs to be converted to index into the array and
  * the id within the array element.
  */
-#define __RTE_QSBR_THRID_ARRAY_ELM_SIZE (sizeof(uint64_t) * 8)
+#define __RTE_QSBR_THRID_ARRAY_ELM_SIZE
+(sizeof(RTE_ATOMIC(uint64_t)) *
+8)
Quoted 8 lines

lib/rcu/rte_rcu_qsbr.h

Quoted 6 lines
lib/rcu/rte_rcu_qsbr.h +2 -1
 #define __RTE_QSBR_THRID_ARRAY_SIZE(max_threads) \
 	RTE_ALIGN(RTE_ALIGN_MUL_CEIL(max_threads, \
 		__RTE_QSBR_THRID_ARRAY_ELM_SIZE) >> 3, RTE_CACHE_LINE_SIZE)
-#define __RTE_QSBR_THRID_ARRAY_ELM(v, i) ((uint64_t *) \
+#define __RTE_QSBR_THRID_ARRAY_ELM(v, i) ((uint64_t __rte_atomic *)
+\

Is it equivalent to ((RTE_ATOMIC(uint64_t) *)?

Quoted 1 line

i'm not sure if you're asking about the resultant type of the expression or not?

I see other places are using specifier hence the question.

Quoted 4 lines

in this context we aren't specifying an atomic type but rather adding the atomic qualifier to what should already be a variable that has an atomic specified type with a cast which is why we use __rte_atomic.

Quoted 6 lines

I read from document [1] that atomic qualified type may have a different size from the original type. If that is the case, the size difference could cause issue in bitmap array accessing. Did I misunderstand?

[1] https://en.cppreference.com/w/c/language/atomic

you do not misunderstand, the standard allows atomic specified type sizes to differ from their ordinary native type sizes. though i have some issues with how cppreference is wording things here as compared with the actual standard.

one of the reasons is it allows all standard atomic functions to be 'generic' which means they can be used on objects of arbitrary size instead of just integer and pointer types. i.e. you can use it on struct, union and array types.

it's implementation defined how the operations are made atomic and is obviously target processor dependent, but in cases when the processor has no intrinsic support to perform the operation atomically the toolchain may generate the code that uses a hidden lock. you can test whether this is the case for arbitrary objects using standard specified atomic_is_lock_free.
https://en.cppreference.com/w/c/atomic/atomic_is_lock_free

so that's the long answer form of why they *may* be different size, alignment etc.. but the real question is in this instance will it be?

probably not.

mainly because it wouldn't make a lot of sense for clang/gcc to suddenly decide that sizeof(uint64_t) != sizeof(_Atomic(uint64_t)) or that they should need to use a lock on amd64 processor to load/store atomically (assuming native alignment) etc..

a lot of the above is why we had a lot of discussion about how and when we could enable the use of standard C11 atomics in dpdk. as you've probably noticed for existing platforms, toolchains and targets it is actually defaulted off, but it does allow binary packagers or users to build with it on.

for compatibility only the strictest of guarantees can be made when dpdk and the application are both built consistently to use or not use standard atomics. it is strongly cautioned that applications should not attempt to use an unmatched atomic operation on a dpdk atomic object. i.e. if you enabled standard atomics, don't use __atomic_load_n directly on a field from a public dpdk structure, instead use rte_atomic_load_explicit and make sure your application defines RTE_ENABLE_STDATOMIC.

hope this explanation helps.

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 6 lines

((struct rte_rcu_qsbr_cnt *)(v + 1) + v->max_threads) + i)
#define __RTE_QSBR_THRID_INDEX_SHIFT 6 #define __RTE_QSBR_THRID_MASK 0x3f @@ -75,13 +75,13 @@

<snip>

#39 lore raw

PATCH v2 10/19 pdump: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/pdump/rte_pdump.c | 14 +++++++-------
 lib/pdump/rte_pdump.h |  8 ++++----
 2 files changed, 11 insertions(+), 11 deletions(-)
lib/pdump/rte_pdump.c +7 -7
index 53cca10..80b90c6 100644
@@ -110,8 +110,8 @@ struct pdump_response {
 		 * then packet doesn't match the filter (will be ignored).
 		 */
 		if (cbs->filter && rcs[i] == 0) {
-			__atomic_fetch_add(&stats->filtered,
-					   1, __ATOMIC_RELAXED);
+			rte_atomic_fetch_add_explicit(&stats->filtered,
+					   1, rte_memory_order_relaxed);
 			continue;
 		}
 
@@ -127,18 +127,18 @@ struct pdump_response {
 			p = rte_pktmbuf_copy(pkts[i], mp, 0, cbs->snaplen);
 
 		if (unlikely(p == NULL))
-			__atomic_fetch_add(&stats->nombuf, 1, __ATOMIC_RELAXED);
+			rte_atomic_fetch_add_explicit(&stats->nombuf, 1, rte_memory_order_relaxed);
 		else
 			dup_bufs[d_pkts++] = p;
 	}
 
-	__atomic_fetch_add(&stats->accepted, d_pkts, __ATOMIC_RELAXED);
+	rte_atomic_fetch_add_explicit(&stats->accepted, d_pkts, rte_memory_order_relaxed);
 
 	ring_enq = rte_ring_enqueue_burst(ring, (void *)&dup_bufs[0], d_pkts, NULL);
 	if (unlikely(ring_enq < d_pkts)) {
 		unsigned int drops = d_pkts - ring_enq;
 
-		__atomic_fetch_add(&stats->ringfull, drops, __ATOMIC_RELAXED);
+		rte_atomic_fetch_add_explicit(&stats->ringfull, drops, rte_memory_order_relaxed);
 		rte_pktmbuf_free_bulk(&dup_bufs[ring_enq], drops);
 	}
 }
@@ -720,10 +720,10 @@ struct pdump_response {
 	uint16_t qid;
 
 	for (qid = 0; qid < nq; qid++) {
-		const uint64_t *perq = (const uint64_t *)&stats[port][qid];
+		const RTE_ATOMIC(uint64_t) *perq = (const uint64_t __rte_atomic *)&stats[port][qid];
 
 		for (i = 0; i < sizeof(*total) / sizeof(uint64_t); i++) {
-			val = __atomic_load_n(&perq[i], __ATOMIC_RELAXED);
+			val = rte_atomic_load_explicit(&perq[i], rte_memory_order_relaxed);
 			sum[i] += val;
 		}
 	}
lib/pdump/rte_pdump.h +4 -4
index b1a3918..7feb2b6 100644
@@ -233,10 +233,10 @@ enum {
  * The statistics are sum of both receive and transmit queues.
  */
 struct rte_pdump_stats {
-	uint64_t accepted; /**< Number of packets accepted by filter. */
-	uint64_t filtered; /**< Number of packets rejected by filter. */
-	uint64_t nombuf;   /**< Number of mbuf allocation failures. */
-	uint64_t ringfull; /**< Number of missed packets due to ring full. */
+	RTE_ATOMIC(uint64_t) accepted; /**< Number of packets accepted by filter. */
+	RTE_ATOMIC(uint64_t) filtered; /**< Number of packets rejected by filter. */
+	RTE_ATOMIC(uint64_t) nombuf;   /**< Number of mbuf allocation failures. */
+	RTE_ATOMIC(uint64_t) ringfull; /**< Number of missed packets due to ring full. */
 
 	uint64_t reserved[4]; /**< Reserved and pad to cache line */
 };
Signature · 1 line
1.8.3.1
#40 lore raw

PATCH v2 11/19 stack: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/stack/rte_stack.h            |  2 +-
 lib/stack/rte_stack_lf_c11.h     | 24 ++++++++++++------------
 lib/stack/rte_stack_lf_generic.h | 18 +++++++++---------
 3 files changed, 22 insertions(+), 22 deletions(-)
lib/stack/rte_stack.h +1 -1
index 921d29a..a379300 100644
@@ -44,7 +44,7 @@ struct rte_stack_lf_list {
 	/** List head */
 	struct rte_stack_lf_head head __rte_aligned(16);
 	/** List len */
-	uint64_t len;
+	RTE_ATOMIC(uint64_t) len;
 };
 
 /* Structure containing two lock-free LIFO lists: the stack itself and a list
lib/stack/rte_stack_lf_c11.h +12 -12
index 687a6f6..9cb6998 100644
@@ -26,8 +26,8 @@
 	 * elements. If the mempool is near-empty to the point that this is a
 	 * concern, the user should consider increasing the mempool size.
 	 */
-	return (unsigned int)__atomic_load_n(&s->stack_lf.used.len,
-					     __ATOMIC_RELAXED);
+	return (unsigned int)rte_atomic_load_explicit(&s->stack_lf.used.len,
+					     rte_memory_order_relaxed);
 }
 
 static __rte_always_inline void
@@ -59,14 +59,14 @@
 				(rte_int128_t *)&list->head,
 				(rte_int128_t *)&old_head,
 				(rte_int128_t *)&new_head,
-				1, __ATOMIC_RELEASE,
-				__ATOMIC_RELAXED);
+				1, rte_memory_order_release,
+				rte_memory_order_relaxed);
 	} while (success == 0);
 
 	/* Ensure the stack modifications are not reordered with respect
 	 * to the LIFO len update.
 	 */
-	__atomic_fetch_add(&list->len, num, __ATOMIC_RELEASE);
+	rte_atomic_fetch_add_explicit(&list->len, num, rte_memory_order_release);
 }
 
 static __rte_always_inline struct rte_stack_lf_elem *
@@ -80,7 +80,7 @@
 	int success;
 
 	/* Reserve num elements, if available */
-	len = __atomic_load_n(&list->len, __ATOMIC_RELAXED);
+	len = rte_atomic_load_explicit(&list->len, rte_memory_order_relaxed);
 
 	while (1) {
 		/* Does the list contain enough elements? */
@@ -88,10 +88,10 @@
 			return NULL;
 
 		/* len is updated on failure */
-		if (__atomic_compare_exchange_n(&list->len,
+		if (rte_atomic_compare_exchange_weak_explicit(&list->len,
 						&len, len - num,
-						1, __ATOMIC_ACQUIRE,
-						__ATOMIC_RELAXED))
+						rte_memory_order_acquire,
+						rte_memory_order_relaxed))
 			break;
 	}
 
@@ -110,7 +110,7 @@
 		 * elements are properly ordered with respect to the head
 		 * pointer read.
 		 */
-		__atomic_thread_fence(__ATOMIC_ACQUIRE);
+		__atomic_thread_fence(rte_memory_order_acquire);
 
 		rte_prefetch0(old_head.top);
 
@@ -159,8 +159,8 @@
 				(rte_int128_t *)&list->head,
 				(rte_int128_t *)&old_head,
 				(rte_int128_t *)&new_head,
-				0, __ATOMIC_RELAXED,
-				__ATOMIC_RELAXED);
+				0, rte_memory_order_relaxed,
+				rte_memory_order_relaxed);
 	} while (success == 0);
 
 	return old_head.top;
lib/stack/rte_stack_lf_generic.h +9 -9
index 39f7ff3..cc69e4d 100644
@@ -27,7 +27,7 @@
 	 * concern, the user should consider increasing the mempool size.
 	 */
 	/* NOTE: review for potential ordering optimization */
-	return __atomic_load_n(&s->stack_lf.used.len, __ATOMIC_SEQ_CST);
+	return rte_atomic_load_explicit(&s->stack_lf.used.len, rte_memory_order_seq_cst);
 }
 
 static __rte_always_inline void
@@ -64,11 +64,11 @@
 				(rte_int128_t *)&list->head,
 				(rte_int128_t *)&old_head,
 				(rte_int128_t *)&new_head,
-				1, __ATOMIC_RELEASE,
-				__ATOMIC_RELAXED);
+				1, rte_memory_order_release,
+				rte_memory_order_relaxed);
 	} while (success == 0);
 	/* NOTE: review for potential ordering optimization */
-	__atomic_fetch_add(&list->len, num, __ATOMIC_SEQ_CST);
+	rte_atomic_fetch_add_explicit(&list->len, num, rte_memory_order_seq_cst);
 }
 
 static __rte_always_inline struct rte_stack_lf_elem *
@@ -83,15 +83,15 @@
 	/* Reserve num elements, if available */
 	while (1) {
 		/* NOTE: review for potential ordering optimization */
-		uint64_t len = __atomic_load_n(&list->len, __ATOMIC_SEQ_CST);
+		uint64_t len = rte_atomic_load_explicit(&list->len, rte_memory_order_seq_cst);
 
 		/* Does the list contain enough elements? */
 		if (unlikely(len < num))
 			return NULL;
 
 		/* NOTE: review for potential ordering optimization */
-		if (__atomic_compare_exchange_n(&list->len, &len, len - num,
-				0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST))
+		if (rte_atomic_compare_exchange_strong_explicit(&list->len, &len, len - num,
+				rte_memory_order_seq_cst, rte_memory_order_seq_cst))
 			break;
 	}
 
@@ -143,8 +143,8 @@
 				(rte_int128_t *)&list->head,
 				(rte_int128_t *)&old_head,
 				(rte_int128_t *)&new_head,
-				1, __ATOMIC_RELEASE,
-				__ATOMIC_RELAXED);
+				1, rte_memory_order_release,
+				rte_memory_order_relaxed);
 	} while (success == 0);
 
 	return old_head.top;
Signature · 1 line
1.8.3.1
Acked-by on v2 11/19
#41 lore raw

Re: [PATCH v2 11/19] stack: use rte optional stdatomic API

17.10.2023 21:31, Tyler Retzlaff пишет:

Quoted 147 lines

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
  lib/stack/rte_stack.h            |  2 +-
  lib/stack/rte_stack_lf_c11.h     | 24 ++++++++++++------------
  lib/stack/rte_stack_lf_generic.h | 18 +++++++++---------
  3 files changed, 22 insertions(+), 22 deletions(-)
lib/stack/rte_stack.h +1 -1
index 921d29a..a379300 100644
@@ -44,7 +44,7 @@ struct rte_stack_lf_list {
  	/** List head */
  	struct rte_stack_lf_head head __rte_aligned(16);
  	/** List len */
-	uint64_t len;
+	RTE_ATOMIC(uint64_t) len;
  };
 
  /* Structure containing two lock-free LIFO lists: the stack itself and a list
lib/stack/rte_stack_lf_c11.h +12 -12
index 687a6f6..9cb6998 100644
@@ -26,8 +26,8 @@
  	 * elements. If the mempool is near-empty to the point that this is a
  	 * concern, the user should consider increasing the mempool size.
  	 */
-	return (unsigned int)__atomic_load_n(&s->stack_lf.used.len,
-					     __ATOMIC_RELAXED);
+	return (unsigned int)rte_atomic_load_explicit(&s->stack_lf.used.len,
+					     rte_memory_order_relaxed);
  }
 
  static __rte_always_inline void
@@ -59,14 +59,14 @@
  				(rte_int128_t *)&list->head,
  				(rte_int128_t *)&old_head,
  				(rte_int128_t *)&new_head,
-				1, __ATOMIC_RELEASE,
-				__ATOMIC_RELAXED);
+				1, rte_memory_order_release,
+				rte_memory_order_relaxed);
  	} while (success == 0);
 
  	/* Ensure the stack modifications are not reordered with respect
  	 * to the LIFO len update.
  	 */
-	__atomic_fetch_add(&list->len, num, __ATOMIC_RELEASE);
+	rte_atomic_fetch_add_explicit(&list->len, num, rte_memory_order_release);
  }
 
  static __rte_always_inline struct rte_stack_lf_elem *
@@ -80,7 +80,7 @@
  	int success;
 
  	/* Reserve num elements, if available */
-	len = __atomic_load_n(&list->len, __ATOMIC_RELAXED);
+	len = rte_atomic_load_explicit(&list->len, rte_memory_order_relaxed);
 
  	while (1) {
  		/* Does the list contain enough elements? */
@@ -88,10 +88,10 @@
  			return NULL;
 
  		/* len is updated on failure */
-		if (__atomic_compare_exchange_n(&list->len,
+		if (rte_atomic_compare_exchange_weak_explicit(&list->len,
  						&len, len - num,
-						1, __ATOMIC_ACQUIRE,
-						__ATOMIC_RELAXED))
+						rte_memory_order_acquire,
+						rte_memory_order_relaxed))
  			break;
  	}
 
@@ -110,7 +110,7 @@
  		 * elements are properly ordered with respect to the head
  		 * pointer read.
  		 */
-		__atomic_thread_fence(__ATOMIC_ACQUIRE);
+		__atomic_thread_fence(rte_memory_order_acquire);
 
  		rte_prefetch0(old_head.top);
 
@@ -159,8 +159,8 @@
  				(rte_int128_t *)&list->head,
  				(rte_int128_t *)&old_head,
  				(rte_int128_t *)&new_head,
-				0, __ATOMIC_RELAXED,
-				__ATOMIC_RELAXED);
+				0, rte_memory_order_relaxed,
+				rte_memory_order_relaxed);
  	} while (success == 0);
 
  	return old_head.top;
lib/stack/rte_stack_lf_generic.h +7 -7
index 39f7ff3..cc69e4d 100644
@@ -27,7 +27,7 @@
  	 * concern, the user should consider increasing the mempool size.
  	 */
  	/* NOTE: review for potential ordering optimization */
-	return __atomic_load_n(&s->stack_lf.used.len, __ATOMIC_SEQ_CST);
+	return rte_atomic_load_explicit(&s->stack_lf.used.len, rte_memory_order_seq_cst);
  }
 
  static __rte_always_inline void
@@ -64,11 +64,11 @@
  				(rte_int128_t *)&list->head,
  				(rte_int128_t *)&old_head,
  				(rte_int128_t *)&new_head,
-				1, __ATOMIC_RELEASE,
-				__ATOMIC_RELAXED);
+				1, rte_memory_order_release,
+				rte_memory_order_relaxed);
  	} while (success == 0);
  	/* NOTE: review for potential ordering optimization */
-	__atomic_fetch_add(&list->len, num, __ATOMIC_SEQ_CST);
+	rte_atomic_fetch_add_explicit(&list->len, num, rte_memory_order_seq_cst);
  }
 
  static __rte_always_inline struct rte_stack_lf_elem *
@@ -83,15 +83,15 @@
  	/* Reserve num elements, if available */
  	while (1) {
  		/* NOTE: review for potential ordering optimization */
-		uint64_t len = __atomic_load_n(&list->len, __ATOMIC_SEQ_CST);
+		uint64_t len = rte_atomic_load_explicit(&list->len, rte_memory_order_seq_cst);
 
  		/* Does the list contain enough elements? */
  		if (unlikely(len < num))
  			return NULL;
 
  		/* NOTE: review for potential ordering optimization */
-		if (__atomic_compare_exchange_n(&list->len, &len, len - num,
-				0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST))
+		if (rte_atomic_compare_exchange_strong_explicit(&list->len, &len, len - num,
+				rte_memory_order_seq_cst, rte_memory_order_seq_cst))
  			break;
  	}
 

lib/stack/rte_stack_lf_generic.h line 150

Quoted 10 lines
lib/stack/rte_stack_lf_generic.h +2 -2
@@ -143,8 +143,8 @@
  				(rte_int128_t *)&list->head,
  				(rte_int128_t *)&old_head,
  				(rte_int128_t *)&new_head,
-				1, __ATOMIC_RELEASE,
-				__ATOMIC_RELAXED);
+				1, rte_memory_order_release,
+				rte_memory_order_relaxed);
  	} while (success == 0);
 
  	return old_head.top;
#42 lore raw

PATCH v2 12/19 telemetry: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/telemetry/telemetry.c | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)
lib/telemetry/telemetry.c +9 -9
index aeb078c..9298284 100644
@@ -45,7 +45,7 @@ struct socket {
 	int sock;
 	char path[sizeof(((struct sockaddr_un *)0)->sun_path)];
 	handler fn;
-	uint16_t *num_clients;
+	RTE_ATOMIC(uint16_t) *num_clients;
 };
 static struct socket v2_socket; /* socket for v2 telemetry */
 static struct socket v1_socket; /* socket for v1 telemetry */
@@ -64,7 +64,7 @@ struct socket {
 /* Used when accessing or modifying list of command callbacks */
 static rte_spinlock_t callback_sl = RTE_SPINLOCK_INITIALIZER;
 #ifndef RTE_EXEC_ENV_WINDOWS
-static uint16_t v2_clients;
+static RTE_ATOMIC(uint16_t) v2_clients;
 #endif /* !RTE_EXEC_ENV_WINDOWS */
 
 int
@@ -404,7 +404,7 @@ struct socket {
 		bytes = read(s, buffer, sizeof(buffer) - 1);
 	}
 	close(s);
-	__atomic_fetch_sub(&v2_clients, 1, __ATOMIC_RELAXED);
+	rte_atomic_fetch_sub_explicit(&v2_clients, 1, rte_memory_order_relaxed);
 	return NULL;
 }
 
@@ -421,14 +421,14 @@ struct socket {
 			return NULL;
 		}
 		if (s->num_clients != NULL) {
-			uint16_t conns = __atomic_load_n(s->num_clients,
-					__ATOMIC_RELAXED);
+			uint16_t conns = rte_atomic_load_explicit(s->num_clients,
+					rte_memory_order_relaxed);
 			if (conns >= MAX_CONNECTIONS) {
 				close(s_accepted);
 				continue;
 			}
-			__atomic_fetch_add(s->num_clients, 1,
-					__ATOMIC_RELAXED);
+			rte_atomic_fetch_add_explicit(s->num_clients, 1,
+					rte_memory_order_relaxed);
 		}
 		rc = pthread_create(&th, NULL, s->fn,
 				    (void *)(uintptr_t)s_accepted);
@@ -437,8 +437,8 @@ struct socket {
 				 strerror(rc));
 			close(s_accepted);
 			if (s->num_clients != NULL)
-				__atomic_fetch_sub(s->num_clients, 1,
-						   __ATOMIC_RELAXED);
+				rte_atomic_fetch_sub_explicit(s->num_clients, 1,
+						   rte_memory_order_relaxed);
 			continue;
 		}
 		pthread_detach(th);
Signature · 1 line
1.8.3.1
#43 lore raw

PATCH v2 13/19 vhost: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/vhost/vdpa.c            |  3 ++-
 lib/vhost/vhost.c           | 42 ++++++++++++++++----------------
 lib/vhost/vhost.h           | 39 ++++++++++++++++--------------
 lib/vhost/vhost_user.c      |  6 ++---
 lib/vhost/virtio_net.c      | 58 +++++++++++++++++++++++++--------------------
 lib/vhost/virtio_net_ctrl.c |  6 +++--
 6 files changed, 84 insertions(+), 70 deletions(-)
lib/vhost/vdpa.c +2 -1
index 6284ea2..219eef8 100644
@@ -235,7 +235,8 @@ struct rte_vdpa_device *
 	}
 
 	/* used idx is the synchronization point for the split vring */
-	__atomic_store_n(&vq->used->idx, idx_m, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit((unsigned short __rte_atomic *)&vq->used->idx,
+		idx_m, rte_memory_order_release);
 
 	if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))
 		vring_used_event(s_vring) = idx_m;
lib/vhost/vhost.c +22 -20

124 lines not on this page · raw message

lib/vhost/vhost.h +21 -18
index 5fc9035..f8624fb 100644
@@ -158,9 +158,9 @@ struct virtqueue_stats {
 	uint64_t inflight_completed;
 	uint64_t guest_notifications_suppressed;
 	/* Counters below are atomic, and should be incremented as such. */
-	uint64_t guest_notifications;
-	uint64_t guest_notifications_offloaded;
-	uint64_t guest_notifications_error;
+	RTE_ATOMIC(uint64_t) guest_notifications;
+	RTE_ATOMIC(uint64_t) guest_notifications_offloaded;
+	RTE_ATOMIC(uint64_t) guest_notifications_error;
 };
 
 /**
@@ -348,7 +348,7 @@ struct vhost_virtqueue {
 	struct vhost_vring_addr ring_addrs;
 	struct virtqueue_stats	stats;
 
-	bool irq_pending;
+	RTE_ATOMIC(bool) irq_pending;
 } __rte_cache_aligned;
 
 /* Virtio device status as per Virtio specification */
@@ -486,7 +486,7 @@ struct virtio_net {
 	uint32_t		flags;
 	uint16_t		vhost_hlen;
 	/* to tell if we need broadcast rarp packet */
-	int16_t			broadcast_rarp;
+	RTE_ATOMIC(int16_t)	broadcast_rarp;
 	uint32_t		nr_vring;
 	int			async_copy;
 
@@ -557,7 +557,8 @@ struct virtio_net {
 static inline bool
 desc_is_avail(struct vring_packed_desc *desc, bool wrap_counter)
 {
-	uint16_t flags = __atomic_load_n(&desc->flags, __ATOMIC_ACQUIRE);
+	uint16_t flags = rte_atomic_load_explicit((unsigned short __rte_atomic *)&desc->flags,
+		rte_memory_order_acquire);
 
 	return wrap_counter == !!(flags & VRING_DESC_F_AVAIL) &&
 		wrap_counter != !!(flags & VRING_DESC_F_USED);
@@ -914,17 +915,19 @@ uint64_t translate_log_addr(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	bool expected = false;
 
 	if (dev->notify_ops->guest_notify) {
-		if (__atomic_compare_exchange_n(&vq->irq_pending, &expected, true, 0,
-				  __ATOMIC_RELEASE, __ATOMIC_RELAXED)) {
+		if (rte_atomic_compare_exchange_strong_explicit(&vq->irq_pending, &expected, true,
+				  rte_memory_order_release, rte_memory_order_relaxed)) {
 			if (dev->notify_ops->guest_notify(dev->vid, vq->index)) {
 				if (dev->flags & VIRTIO_DEV_STATS_ENABLED)
-					__atomic_fetch_add(&vq->stats.guest_notifications_offloaded,
-						1, __ATOMIC_RELAXED);
+					rte_atomic_fetch_add_explicit(
+						&vq->stats.guest_notifications_offloaded,
+						1, rte_memory_order_relaxed);
 				return;
 			}
 
 			/* Offloading failed, fallback to direct IRQ injection */
-			__atomic_store_n(&vq->irq_pending, false, __ATOMIC_RELEASE);
+			rte_atomic_store_explicit(&vq->irq_pending, false,
+				rte_memory_order_release);
 		} else {
 			vq->stats.guest_notifications_suppressed++;
 			return;
@@ -933,14 +936,14 @@ uint64_t translate_log_addr(struct virtio_net *dev, struct vhost_virtqueue *vq,
 
 	if (dev->backend_ops->inject_irq(dev, vq)) {
 		if (dev->flags & VIRTIO_DEV_STATS_ENABLED)
-			__atomic_fetch_add(&vq->stats.guest_notifications_error,
-				1, __ATOMIC_RELAXED);
+			rte_atomic_fetch_add_explicit(&vq->stats.guest_notifications_error,
+				1, rte_memory_order_relaxed);
 		return;
 	}
 
 	if (dev->flags & VIRTIO_DEV_STATS_ENABLED)
-		__atomic_fetch_add(&vq->stats.guest_notifications,
-			1, __ATOMIC_RELAXED);
+		rte_atomic_fetch_add_explicit(&vq->stats.guest_notifications,
+			1, rte_memory_order_relaxed);
 	if (dev->notify_ops->guest_notified)
 		dev->notify_ops->guest_notified(dev->vid);
 }
@@ -949,7 +952,7 @@ uint64_t translate_log_addr(struct virtio_net *dev, struct vhost_virtqueue *vq,
 vhost_vring_call_split(struct virtio_net *dev, struct vhost_virtqueue *vq)
 {
 	/* Flush used->idx update before we read avail->flags. */
-	rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
+	rte_atomic_thread_fence(rte_memory_order_seq_cst);
 
 	/* Don't kick guest if we don't reach index specified by guest. */
 	if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) {
@@ -981,7 +984,7 @@ uint64_t translate_log_addr(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	bool signalled_used_valid, kick = false;
 
 	/* Flush used desc update. */
-	rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
+	rte_atomic_thread_fence(rte_memory_order_seq_cst);
 
 	if (!(dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))) {
 		if (vq->driver_event->flags !=
@@ -1007,7 +1010,7 @@ uint64_t translate_log_addr(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		goto kick;
 	}
 
-	rte_atomic_thread_fence(__ATOMIC_ACQUIRE);
+	rte_atomic_thread_fence(rte_memory_order_acquire);
 
 	off_wrap = vq->driver_event->off_wrap;
 	off = off_wrap & ~(1 << 15);
lib/vhost/vhost_user.c +3 -3
index 901a80b..e363121 100644
@@ -1914,7 +1914,7 @@ static int vhost_user_set_vring_err(struct virtio_net **pdev,
 
 	if (inflight_split->used_idx != used->idx) {
 		inflight_split->desc[last_io].inflight = 0;
-		rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
+		rte_atomic_thread_fence(rte_memory_order_seq_cst);
 		inflight_split->used_idx = used->idx;
 	}
 
@@ -2418,10 +2418,10 @@ static int vhost_user_set_log_fd(struct virtio_net **pdev,
 	 * Set the flag to inject a RARP broadcast packet at
 	 * rte_vhost_dequeue_burst().
 	 *
-	 * __ATOMIC_RELEASE ordering is for making sure the mac is
+	 * rte_memory_order_release ordering is for making sure the mac is
 	 * copied before the flag is set.
 	 */
-	__atomic_store_n(&dev->broadcast_rarp, 1, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&dev->broadcast_rarp, 1, rte_memory_order_release);
 	vdpa_dev = dev->vdpa_dev;
 	if (vdpa_dev && vdpa_dev->ops->migration_done)
 		vdpa_dev->ops->migration_done(dev->vid);
lib/vhost/virtio_net.c +32 -26

166 lines not on this page · raw message

lib/vhost/virtio_net_ctrl.c +4 -2
index 6b583a0..c4847f8 100644
@@ -33,7 +33,8 @@ struct virtio_net_ctrl_elem {
 	uint8_t *ctrl_req;
 	struct vring_desc *descs;
 
-	avail_idx = __atomic_load_n(&cvq->avail->idx, __ATOMIC_ACQUIRE);
+	avail_idx = rte_atomic_load_explicit((unsigned short __rte_atomic *)&cvq->avail->idx,
+		rte_memory_order_acquire);
 	if (avail_idx == cvq->last_avail_idx) {
 		VHOST_LOG_CONFIG(dev->ifname, DEBUG, "Control queue empty\n");
 		return 0;
@@ -236,7 +237,8 @@ struct virtio_net_ctrl_elem {
 	if (cvq->last_used_idx >= cvq->size)
 		cvq->last_used_idx -= cvq->size;
 
-	__atomic_store_n(&cvq->used->idx, cvq->last_used_idx, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit((unsigned short __rte_atomic *)&cvq->used->idx,
+		cvq->last_used_idx, rte_memory_order_release);
 
 	vhost_vring_call_split(dev, dev->cvq);
 
Signature · 1 line
1.8.3.1
#44 lore raw

PATCH v2 14/19 cryptodev: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/cryptodev/rte_cryptodev.c | 22 ++++++++++++----------
 lib/cryptodev/rte_cryptodev.h | 16 ++++++++--------
 2 files changed, 20 insertions(+), 18 deletions(-)
lib/cryptodev/rte_cryptodev.c +12 -10
index 314710b..b258827 100644
@@ -1535,12 +1535,12 @@ struct rte_cryptodev_cb *
 		/* Stores to cb->fn and cb->param should complete before
 		 * cb is visible to data plane.
 		 */
-		__atomic_store_n(&tail->next, cb, __ATOMIC_RELEASE);
+		rte_atomic_store_explicit(&tail->next, cb, rte_memory_order_release);
 	} else {
 		/* Stores to cb->fn and cb->param should complete before
 		 * cb is visible to data plane.
 		 */
-		__atomic_store_n(&list->next, cb, __ATOMIC_RELEASE);
+		rte_atomic_store_explicit(&list->next, cb, rte_memory_order_release);
 	}
 
 	rte_spinlock_unlock(&rte_cryptodev_callback_lock);
@@ -1555,7 +1555,8 @@ struct rte_cryptodev_cb *
 				  struct rte_cryptodev_cb *cb)
 {
 	struct rte_cryptodev *dev;
-	struct rte_cryptodev_cb **prev_cb, *curr_cb;
+	RTE_ATOMIC(struct rte_cryptodev_cb *) *prev_cb;
+	struct rte_cryptodev_cb *curr_cb;
 	struct rte_cryptodev_cb_rcu *list;
 	int ret;
 
@@ -1601,8 +1602,8 @@ struct rte_cryptodev_cb *
 		curr_cb = *prev_cb;
 		if (curr_cb == cb) {
 			/* Remove the user cb from the callback list. */
-			__atomic_store_n(prev_cb, curr_cb->next,
-				__ATOMIC_RELAXED);
+			rte_atomic_store_explicit(prev_cb, curr_cb->next,
+				rte_memory_order_relaxed);
 			ret = 0;
 			break;
 		}
@@ -1673,12 +1674,12 @@ struct rte_cryptodev_cb *
 		/* Stores to cb->fn and cb->param should complete before
 		 * cb is visible to data plane.
 		 */
-		__atomic_store_n(&tail->next, cb, __ATOMIC_RELEASE);
+		rte_atomic_store_explicit(&tail->next, cb, rte_memory_order_release);
 	} else {
 		/* Stores to cb->fn and cb->param should complete before
 		 * cb is visible to data plane.
 		 */
-		__atomic_store_n(&list->next, cb, __ATOMIC_RELEASE);
+		rte_atomic_store_explicit(&list->next, cb, rte_memory_order_release);
 	}
 
 	rte_spinlock_unlock(&rte_cryptodev_callback_lock);
@@ -1694,7 +1695,8 @@ struct rte_cryptodev_cb *
 				  struct rte_cryptodev_cb *cb)
 {
 	struct rte_cryptodev *dev;
-	struct rte_cryptodev_cb **prev_cb, *curr_cb;
+	RTE_ATOMIC(struct rte_cryptodev_cb *) *prev_cb;
+	struct rte_cryptodev_cb *curr_cb;
 	struct rte_cryptodev_cb_rcu *list;
 	int ret;
 
@@ -1740,8 +1742,8 @@ struct rte_cryptodev_cb *
 		curr_cb = *prev_cb;
 		if (curr_cb == cb) {
 			/* Remove the user cb from the callback list. */
-			__atomic_store_n(prev_cb, curr_cb->next,
-				__ATOMIC_RELAXED);
+			rte_atomic_store_explicit(prev_cb, curr_cb->next,
+				rte_memory_order_relaxed);
 			ret = 0;
 			break;
 		}
lib/cryptodev/rte_cryptodev.h +8 -8
index be0698c..9092118 100644
@@ -979,7 +979,7 @@ struct rte_cryptodev_config {
  * queue pair on enqueue/dequeue.
  */
 struct rte_cryptodev_cb {
-	struct rte_cryptodev_cb *next;
+	RTE_ATOMIC(struct rte_cryptodev_cb *) next;
 	/**< Pointer to next callback */
 	rte_cryptodev_callback_fn fn;
 	/**< Pointer to callback function */
@@ -992,7 +992,7 @@ struct rte_cryptodev_cb {
  * Structure used to hold information about the RCU for a queue pair.
  */
 struct rte_cryptodev_cb_rcu {
-	struct rte_cryptodev_cb *next;
+	RTE_ATOMIC(struct rte_cryptodev_cb *) next;
 	/**< Pointer to next callback */
 	struct rte_rcu_qsbr *qsbr;
 	/**< RCU QSBR variable per queue pair */
@@ -1947,15 +1947,15 @@ int rte_cryptodev_remove_deq_callback(uint8_t dev_id,
 		struct rte_cryptodev_cb_rcu *list;
 		struct rte_cryptodev_cb *cb;
 
-		/* __ATOMIC_RELEASE memory order was used when the
+		/* rte_memory_order_release memory order was used when the
 		 * call back was inserted into the list.
 		 * Since there is a clear dependency between loading
-		 * cb and cb->fn/cb->next, __ATOMIC_ACQUIRE memory order is
+		 * cb and cb->fn/cb->next, rte_memory_order_acquire memory order is
 		 * not required.
 		 */
 		list = &fp_ops->qp.deq_cb[qp_id];
 		rte_rcu_qsbr_thread_online(list->qsbr, 0);
-		cb = __atomic_load_n(&list->next, __ATOMIC_RELAXED);
+		cb = rte_atomic_load_explicit(&list->next, rte_memory_order_relaxed);
 
 		while (cb != NULL) {
 			nb_ops = cb->fn(dev_id, qp_id, ops, nb_ops,
@@ -2014,15 +2014,15 @@ int rte_cryptodev_remove_deq_callback(uint8_t dev_id,
 		struct rte_cryptodev_cb_rcu *list;
 		struct rte_cryptodev_cb *cb;
 
-		/* __ATOMIC_RELEASE memory order was used when the
+		/* rte_memory_order_release memory order was used when the
 		 * call back was inserted into the list.
 		 * Since there is a clear dependency between loading
-		 * cb and cb->fn/cb->next, __ATOMIC_ACQUIRE memory order is
+		 * cb and cb->fn/cb->next, rte_memory_order_acquire memory order is
 		 * not required.
 		 */
 		list = &fp_ops->qp.enq_cb[qp_id];
 		rte_rcu_qsbr_thread_online(list->qsbr, 0);
-		cb = __atomic_load_n(&list->next, __ATOMIC_RELAXED);
+		cb = rte_atomic_load_explicit(&list->next, rte_memory_order_relaxed);
 
 		while (cb != NULL) {
 			nb_ops = cb->fn(dev_id, qp_id, ops, nb_ops,
Signature · 1 line
1.8.3.1
#45 lore raw

PATCH v2 15/19 distributor: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/distributor/distributor_private.h |  4 +--
 lib/distributor/rte_distributor.c     | 54 +++++++++++++++++------------------
 2 files changed, 29 insertions(+), 29 deletions(-)
lib/distributor/distributor_private.h +2 -2
index 2f29343..dfeb9b5 100644
@@ -113,12 +113,12 @@ enum rte_distributor_match_function {
  * There is a separate cacheline for returns in the burst API.
  */
 struct rte_distributor_buffer {
-	volatile int64_t bufptr64[RTE_DIST_BURST_SIZE]
+	volatile RTE_ATOMIC(int64_t) bufptr64[RTE_DIST_BURST_SIZE]
 		__rte_cache_aligned; /* <= outgoing to worker */
 
 	int64_t pad1 __rte_cache_aligned;    /* <= one cache line  */
 
-	volatile int64_t retptr64[RTE_DIST_BURST_SIZE]
+	volatile RTE_ATOMIC(int64_t) retptr64[RTE_DIST_BURST_SIZE]
 		__rte_cache_aligned; /* <= incoming from worker */
 
 	int64_t pad2 __rte_cache_aligned;    /* <= one cache line  */
lib/distributor/rte_distributor.c +27 -27

161 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#46 lore raw

PATCH v2 16/19 ethdev: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/ethdev/ethdev_driver.h   | 16 ++++++++--------
 lib/ethdev/ethdev_private.c  |  6 +++---
 lib/ethdev/rte_ethdev.c      | 24 ++++++++++++------------
 lib/ethdev/rte_ethdev.h      | 16 ++++++++--------
 lib/ethdev/rte_ethdev_core.h |  2 +-
 5 files changed, 32 insertions(+), 32 deletions(-)
lib/ethdev/ethdev_driver.h +8 -8
index deb23ad..b482cd1 100644
@@ -30,7 +30,7 @@
  * queue on Rx and Tx.
  */
 struct rte_eth_rxtx_callback {
-	struct rte_eth_rxtx_callback *next;
+	RTE_ATOMIC(struct rte_eth_rxtx_callback *) next;
 	union{
 		rte_rx_callback_fn rx;
 		rte_tx_callback_fn tx;
@@ -80,12 +80,12 @@ struct rte_eth_dev {
 	 * User-supplied functions called from rx_burst to post-process
 	 * received packets before passing them to the user
 	 */
-	struct rte_eth_rxtx_callback *post_rx_burst_cbs[RTE_MAX_QUEUES_PER_PORT];
+	RTE_ATOMIC(struct rte_eth_rxtx_callback *) post_rx_burst_cbs[RTE_MAX_QUEUES_PER_PORT];
 	/**
 	 * User-supplied functions called from tx_burst to pre-process
 	 * received packets before passing them to the driver for transmission
 	 */
-	struct rte_eth_rxtx_callback *pre_tx_burst_cbs[RTE_MAX_QUEUES_PER_PORT];
+	RTE_ATOMIC(struct rte_eth_rxtx_callback *) pre_tx_burst_cbs[RTE_MAX_QUEUES_PER_PORT];
 
 	enum rte_eth_dev_state state; /**< Flag indicating the port state */
 	void *security_ctx; /**< Context for security ops */
@@ -1655,7 +1655,7 @@ int rte_eth_dev_callback_process(struct rte_eth_dev *dev,
 rte_eth_linkstatus_set(struct rte_eth_dev *dev,
 		       const struct rte_eth_link *new_link)
 {
-	uint64_t *dev_link = (uint64_t *)&(dev->data->dev_link);
+	RTE_ATOMIC(uint64_t) *dev_link = (uint64_t __rte_atomic *)&(dev->data->dev_link);
 	union {
 		uint64_t val64;
 		struct rte_eth_link link;
@@ -1663,8 +1663,8 @@ int rte_eth_dev_callback_process(struct rte_eth_dev *dev,
 
 	RTE_BUILD_BUG_ON(sizeof(*new_link) != sizeof(uint64_t));
 
-	orig.val64 = __atomic_exchange_n(dev_link, *(const uint64_t *)new_link,
-					__ATOMIC_SEQ_CST);
+	orig.val64 = rte_atomic_exchange_explicit(dev_link, *(const uint64_t *)new_link,
+					rte_memory_order_seq_cst);
 
 	return (orig.link.link_status == new_link->link_status) ? -1 : 0;
 }
@@ -1682,12 +1682,12 @@ int rte_eth_dev_callback_process(struct rte_eth_dev *dev,
 rte_eth_linkstatus_get(const struct rte_eth_dev *dev,
 		       struct rte_eth_link *link)
 {
-	uint64_t *src = (uint64_t *)&(dev->data->dev_link);
+	RTE_ATOMIC(uint64_t) *src = (uint64_t __rte_atomic *)&(dev->data->dev_link);
 	uint64_t *dst = (uint64_t *)link;
 
 	RTE_BUILD_BUG_ON(sizeof(*link) != sizeof(uint64_t));
 
-	*dst = __atomic_load_n(src, __ATOMIC_SEQ_CST);
+	*dst = rte_atomic_load_explicit(src, rte_memory_order_seq_cst);
 }
 
 /**
lib/ethdev/ethdev_private.c +3 -3
index 7cc7f28..82e2568 100644
@@ -245,7 +245,7 @@ struct dummy_queue {
 void
 eth_dev_fp_ops_reset(struct rte_eth_fp_ops *fpo)
 {
-	static void *dummy_data[RTE_MAX_QUEUES_PER_PORT];
+	static RTE_ATOMIC(void *) dummy_data[RTE_MAX_QUEUES_PER_PORT];
 	uintptr_t port_id = fpo - rte_eth_fp_ops;
 
 	per_port_queues[port_id].rx_warn_once = false;
@@ -278,10 +278,10 @@ struct dummy_queue {
 	fpo->recycle_rx_descriptors_refill = dev->recycle_rx_descriptors_refill;
 
 	fpo->rxq.data = dev->data->rx_queues;
-	fpo->rxq.clbk = (void **)(uintptr_t)dev->post_rx_burst_cbs;
+	fpo->rxq.clbk = (void * __rte_atomic *)(uintptr_t)dev->post_rx_burst_cbs;
 
 	fpo->txq.data = dev->data->tx_queues;
-	fpo->txq.clbk = (void **)(uintptr_t)dev->pre_tx_burst_cbs;
+	fpo->txq.clbk = (void * __rte_atomic *)(uintptr_t)dev->pre_tx_burst_cbs;
 }
 
 uint16_t
lib/ethdev/rte_ethdev.c +12 -12
index 9dabcb5..af23ac0 100644
@@ -5654,9 +5654,9 @@ int rte_eth_rx_avail_thresh_query(uint16_t port_id, uint16_t *queue_id,
 		/* Stores to cb->fn and cb->param should complete before
 		 * cb is visible to data plane.
 		 */
-		__atomic_store_n(
+		rte_atomic_store_explicit(
 			&rte_eth_devices[port_id].post_rx_burst_cbs[queue_id],
-			cb, __ATOMIC_RELEASE);
+			cb, rte_memory_order_release);
 
 	} else {
 		while (tail->next)
@@ -5664,7 +5664,7 @@ int rte_eth_rx_avail_thresh_query(uint16_t port_id, uint16_t *queue_id,
 		/* Stores to cb->fn and cb->param should complete before
 		 * cb is visible to data plane.
 		 */
-		__atomic_store_n(&tail->next, cb, __ATOMIC_RELEASE);
+		rte_atomic_store_explicit(&tail->next, cb, rte_memory_order_release);
 	}
 	rte_spinlock_unlock(&eth_dev_rx_cb_lock);
 
@@ -5704,9 +5704,9 @@ int rte_eth_rx_avail_thresh_query(uint16_t port_id, uint16_t *queue_id,
 	/* Stores to cb->fn, cb->param and cb->next should complete before
 	 * cb is visible to data plane threads.
 	 */
-	__atomic_store_n(
+	rte_atomic_store_explicit(
 		&rte_eth_devices[port_id].post_rx_burst_cbs[queue_id],
-		cb, __ATOMIC_RELEASE);
+		cb, rte_memory_order_release);
 	rte_spinlock_unlock(&eth_dev_rx_cb_lock);
 
 	rte_eth_trace_add_first_rx_callback(port_id, queue_id, fn, user_param,
@@ -5757,9 +5757,9 @@ int rte_eth_rx_avail_thresh_query(uint16_t port_id, uint16_t *queue_id,
 		/* Stores to cb->fn and cb->param should complete before
 		 * cb is visible to data plane.
 		 */
-		__atomic_store_n(
+		rte_atomic_store_explicit(
 			&rte_eth_devices[port_id].pre_tx_burst_cbs[queue_id],
-			cb, __ATOMIC_RELEASE);
+			cb, rte_memory_order_release);
 
 	} else {
 		while (tail->next)
@@ -5767,7 +5767,7 @@ int rte_eth_rx_avail_thresh_query(uint16_t port_id, uint16_t *queue_id,
 		/* Stores to cb->fn and cb->param should complete before
 		 * cb is visible to data plane.
 		 */
-		__atomic_store_n(&tail->next, cb, __ATOMIC_RELEASE);
+		rte_atomic_store_explicit(&tail->next, cb, rte_memory_order_release);
 	}
 	rte_spinlock_unlock(&eth_dev_tx_cb_lock);
 
@@ -5791,7 +5791,7 @@ int rte_eth_rx_avail_thresh_query(uint16_t port_id, uint16_t *queue_id,
 
 	struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 	struct rte_eth_rxtx_callback *cb;
-	struct rte_eth_rxtx_callback **prev_cb;
+	RTE_ATOMIC(struct rte_eth_rxtx_callback *) *prev_cb;
 	int ret = -EINVAL;
 
 	rte_spinlock_lock(&eth_dev_rx_cb_lock);
@@ -5800,7 +5800,7 @@ int rte_eth_rx_avail_thresh_query(uint16_t port_id, uint16_t *queue_id,
 		cb = *prev_cb;
 		if (cb == user_cb) {
 			/* Remove the user cb from the callback list. */
-			__atomic_store_n(prev_cb, cb->next, __ATOMIC_RELAXED);
+			rte_atomic_store_explicit(prev_cb, cb->next, rte_memory_order_relaxed);
 			ret = 0;
 			break;
 		}
@@ -5828,7 +5828,7 @@ int rte_eth_rx_avail_thresh_query(uint16_t port_id, uint16_t *queue_id,
 	struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 	int ret = -EINVAL;
 	struct rte_eth_rxtx_callback *cb;
-	struct rte_eth_rxtx_callback **prev_cb;
+	RTE_ATOMIC(struct rte_eth_rxtx_callback *) *prev_cb;
 
 	rte_spinlock_lock(&eth_dev_tx_cb_lock);
 	prev_cb = &dev->pre_tx_burst_cbs[queue_id];
@@ -5836,7 +5836,7 @@ int rte_eth_rx_avail_thresh_query(uint16_t port_id, uint16_t *queue_id,
 		cb = *prev_cb;
 		if (cb == user_cb) {
 			/* Remove the user cb from the callback list. */
-			__atomic_store_n(prev_cb, cb->next, __ATOMIC_RELAXED);
+			rte_atomic_store_explicit(prev_cb, cb->next, rte_memory_order_relaxed);
 			ret = 0;
 			break;
 		}
lib/ethdev/rte_ethdev.h +8 -8
index f949dfc..ec48b24 100644
@@ -6018,14 +6018,14 @@ uint16_t rte_eth_call_rx_callbacks(uint16_t port_id, uint16_t queue_id,
 	{
 		void *cb;
 
-		/* __ATOMIC_RELEASE memory order was used when the
+		/* rte_memory_order_release memory order was used when the
 		 * call back was inserted into the list.
 		 * Since there is a clear dependency between loading
-		 * cb and cb->fn/cb->next, __ATOMIC_ACQUIRE memory order is
+		 * cb and cb->fn/cb->next, rte_memory_order_acquire memory order is
 		 * not required.
 		 */
-		cb = __atomic_load_n((void **)&p->rxq.clbk[queue_id],
-				__ATOMIC_RELAXED);
+		cb = rte_atomic_load_explicit(&p->rxq.clbk[queue_id],
+				rte_memory_order_relaxed);
 		if (unlikely(cb != NULL))
 			nb_rx = rte_eth_call_rx_callbacks(port_id, queue_id,
 					rx_pkts, nb_rx, nb_pkts, cb);
@@ -6355,14 +6355,14 @@ uint16_t rte_eth_call_tx_callbacks(uint16_t port_id, uint16_t queue_id,
 	{
 		void *cb;
 
-		/* __ATOMIC_RELEASE memory order was used when the
+		/* rte_memory_order_release memory order was used when the
 		 * call back was inserted into the list.
 		 * Since there is a clear dependency between loading
-		 * cb and cb->fn/cb->next, __ATOMIC_ACQUIRE memory order is
+		 * cb and cb->fn/cb->next, rte_memory_order_acquire memory order is
 		 * not required.
 		 */
-		cb = __atomic_load_n((void **)&p->txq.clbk[queue_id],
-				__ATOMIC_RELAXED);
+		cb = rte_atomic_load_explicit(&p->txq.clbk[queue_id],
+				rte_memory_order_relaxed);
 		if (unlikely(cb != NULL))
 			nb_pkts = rte_eth_call_tx_callbacks(port_id, queue_id,
 					tx_pkts, nb_pkts, cb);
lib/ethdev/rte_ethdev_core.h +1 -1
index 32f5f73..4bfaf79 100644
@@ -71,7 +71,7 @@ struct rte_ethdev_qdata {
 	/** points to array of internal queue data pointers */
 	void **data;
 	/** points to array of queue callback data pointers */
-	void **clbk;
+	RTE_ATOMIC(void *) *clbk;
 };
 
 /**
Signature · 1 line
1.8.3.1
#47 lore raw

PATCH v2 17/19 hash: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/hash/rte_cuckoo_hash.c | 116 ++++++++++++++++++++++-----------------------
 lib/hash/rte_cuckoo_hash.h |   6 +--
 2 files changed, 61 insertions(+), 61 deletions(-)
lib/hash/rte_cuckoo_hash.c +58 -58
index 19b23f2..b2cf60d 100644
@@ -149,7 +149,7 @@ struct rte_hash *
 	unsigned int writer_takes_lock = 0;
 	unsigned int no_free_on_del = 0;
 	uint32_t *ext_bkt_to_free = NULL;
-	uint32_t *tbl_chng_cnt = NULL;
+	RTE_ATOMIC(uint32_t) *tbl_chng_cnt = NULL;
 	struct lcore_cache *local_free_slots = NULL;
 	unsigned int readwrite_concur_lf_support = 0;
 	uint32_t i;
@@ -713,9 +713,9 @@ struct rte_hash *
 				 * variable. Release the application data
 				 * to the readers.
 				 */
-				__atomic_store_n(&k->pdata,
+				rte_atomic_store_explicit(&k->pdata,
 					data,
-					__ATOMIC_RELEASE);
+					rte_memory_order_release);
 				/*
 				 * Return index where key is stored,
 				 * subtracting the first dummy index
@@ -776,9 +776,9 @@ struct rte_hash *
 			 * key_idx is the guard variable for signature
 			 * and key.
 			 */
-			__atomic_store_n(&prim_bkt->key_idx[i],
+			rte_atomic_store_explicit(&prim_bkt->key_idx[i],
 					 new_idx,
-					 __ATOMIC_RELEASE);
+					 rte_memory_order_release);
 			break;
 		}
 	}
@@ -851,9 +851,9 @@ struct rte_hash *
 		if (unlikely(&h->buckets[prev_alt_bkt_idx]
 				!= curr_bkt)) {
 			/* revert it to empty, otherwise duplicated keys */
-			__atomic_store_n(&curr_bkt->key_idx[curr_slot],
+			rte_atomic_store_explicit(&curr_bkt->key_idx[curr_slot],
 				EMPTY_SLOT,
-				__ATOMIC_RELEASE);
+				rte_memory_order_release);
 			__hash_rw_writer_unlock(h);
 			return -1;
 		}
@@ -865,13 +865,13 @@ struct rte_hash *
 			 * Since there is one writer, load acquires on
 			 * tbl_chng_cnt are not required.
 			 */
-			__atomic_store_n(h->tbl_chng_cnt,
+			rte_atomic_store_explicit(h->tbl_chng_cnt,
 					 *h->tbl_chng_cnt + 1,
-					 __ATOMIC_RELEASE);
+					 rte_memory_order_release);
 			/* The store to sig_current should not
 			 * move above the store to tbl_chng_cnt.
 			 */
-			__atomic_thread_fence(__ATOMIC_RELEASE);
+			__atomic_thread_fence(rte_memory_order_release);
 		}
 
 		/* Need to swap current/alt sig to allow later
@@ -881,9 +881,9 @@ struct rte_hash *
 		curr_bkt->sig_current[curr_slot] =
 			prev_bkt->sig_current[prev_slot];
 		/* Release the updated bucket entry */
-		__atomic_store_n(&curr_bkt->key_idx[curr_slot],
+		rte_atomic_store_explicit(&curr_bkt->key_idx[curr_slot],
 			prev_bkt->key_idx[prev_slot],
-			__ATOMIC_RELEASE);
+			rte_memory_order_release);
 
 		curr_slot = prev_slot;
 		curr_node = prev_node;
@@ -897,20 +897,20 @@ struct rte_hash *
 		 * Since there is one writer, load acquires on
 		 * tbl_chng_cnt are not required.
 		 */
-		__atomic_store_n(h->tbl_chng_cnt,
+		rte_atomic_store_explicit(h->tbl_chng_cnt,
 				 *h->tbl_chng_cnt + 1,
-				 __ATOMIC_RELEASE);
+				 rte_memory_order_release);
 		/* The store to sig_current should not
 		 * move above the store to tbl_chng_cnt.
 		 */
-		__atomic_thread_fence(__ATOMIC_RELEASE);
+		__atomic_thread_fence(rte_memory_order_release);
 	}
 
 	curr_bkt->sig_current[curr_slot] = sig;
 	/* Release the new bucket entry */
-	__atomic_store_n(&curr_bkt->key_idx[curr_slot],
+	rte_atomic_store_explicit(&curr_bkt->key_idx[curr_slot],
 			 new_idx,
-			 __ATOMIC_RELEASE);
+			 rte_memory_order_release);
 
 	__hash_rw_writer_unlock(h);
 
@@ -1076,9 +1076,9 @@ struct rte_hash *
 	 * not leak after the store of pdata in the key store. i.e. pdata is
 	 * the guard variable. Release the application data to the readers.
 	 */
-	__atomic_store_n(&new_k->pdata,
+	rte_atomic_store_explicit(&new_k->pdata,
 		data,
-		__ATOMIC_RELEASE);
+		rte_memory_order_release);
 	/* Copy key */
 	memcpy(new_k->key, key, h->key_len);
 
@@ -1149,9 +1149,9 @@ struct rte_hash *
 				 * key_idx is the guard variable for signature
 				 * and key.
 				 */
-				__atomic_store_n(&cur_bkt->key_idx[i],
+				rte_atomic_store_explicit(&cur_bkt->key_idx[i],
 						 slot_id,
-						 __ATOMIC_RELEASE);
+						 rte_memory_order_release);
 				__hash_rw_writer_unlock(h);
 				return slot_id - 1;
 			}
@@ -1185,9 +1185,9 @@ struct rte_hash *
 	 * the store to key_idx. i.e. key_idx is the guard variable
 	 * for signature and key.
 	 */
-	__atomic_store_n(&(h->buckets_ext[ext_bkt_id - 1]).key_idx[0],
+	rte_atomic_store_explicit(&(h->buckets_ext[ext_bkt_id - 1]).key_idx[0],
 			 slot_id,
-			 __ATOMIC_RELEASE);
+			 rte_memory_order_release);
 	/* Link the new bucket to sec bucket linked list */
 	last = rte_hash_get_last_bkt(sec_bkt);
 	last->next = &h->buckets_ext[ext_bkt_id - 1];
@@ -1290,17 +1290,17 @@ struct rte_hash *
 		 * key comparison will ensure that the lookup fails.
 		 */
 		if (bkt->sig_current[i] == sig) {
-			key_idx = __atomic_load_n(&bkt->key_idx[i],
-					  __ATOMIC_ACQUIRE);
+			key_idx = rte_atomic_load_explicit(&bkt->key_idx[i],
+					  rte_memory_order_acquire);
 			if (key_idx != EMPTY_SLOT) {
 				k = (struct rte_hash_key *) ((char *)keys +
 						key_idx * h->key_entry_size);
 
 				if (rte_hash_cmp_eq(key, k->key, h) == 0) {
 					if (data != NULL) {
-						*data = __atomic_load_n(
+						*data = rte_atomic_load_explicit(
 							&k->pdata,
-							__ATOMIC_ACQUIRE);
+							rte_memory_order_acquire);
 					}
 					/*
 					 * Return index where key is stored,
@@ -1374,8 +1374,8 @@ struct rte_hash *
 		 * starts. Acquire semantics will make sure that
 		 * loads in search_one_bucket are not hoisted.
 		 */
-		cnt_b = __atomic_load_n(h->tbl_chng_cnt,
-				__ATOMIC_ACQUIRE);
+		cnt_b = rte_atomic_load_explicit(h->tbl_chng_cnt,
+				rte_memory_order_acquire);
 
 		/* Check if key is in primary location */
 		bkt = &h->buckets[prim_bucket_idx];
@@ -1396,7 +1396,7 @@ struct rte_hash *
 		/* The loads of sig_current in search_one_bucket
 		 * should not move below the load from tbl_chng_cnt.
 		 */
-		__atomic_thread_fence(__ATOMIC_ACQUIRE);
+		__atomic_thread_fence(rte_memory_order_acquire);
 		/* Re-read the table change counter to check if the
 		 * table has changed during search. If yes, re-do
 		 * the search.
@@ -1405,8 +1405,8 @@ struct rte_hash *
 		 * and key index in secondary bucket will make sure
 		 * that it does not get hoisted.
 		 */
-		cnt_a = __atomic_load_n(h->tbl_chng_cnt,
-					__ATOMIC_ACQUIRE);
+		cnt_a = rte_atomic_load_explicit(h->tbl_chng_cnt,
+					rte_memory_order_acquire);
 	} while (cnt_b != cnt_a);
 
 	return -ENOENT;
@@ -1611,26 +1611,26 @@ struct rte_hash *
 	for (i = RTE_HASH_BUCKET_ENTRIES - 1; i >= 0; i--) {
 		if (last_bkt->key_idx[i] != EMPTY_SLOT) {
 			cur_bkt->sig_current[pos] = last_bkt->sig_current[i];
-			__atomic_store_n(&cur_bkt->key_idx[pos],
+			rte_atomic_store_explicit(&cur_bkt->key_idx[pos],
 					 last_bkt->key_idx[i],
-					 __ATOMIC_RELEASE);
+					 rte_memory_order_release);
 			if (h->readwrite_concur_lf_support) {
 				/* Inform the readers that the table has changed
 				 * Since there is one writer, load acquire on
 				 * tbl_chng_cnt is not required.
 				 */
-				__atomic_store_n(h->tbl_chng_cnt,
+				rte_atomic_store_explicit(h->tbl_chng_cnt,
 					 *h->tbl_chng_cnt + 1,
-					 __ATOMIC_RELEASE);
+					 rte_memory_order_release);
 				/* The store to sig_current should
 				 * not move above the store to tbl_chng_cnt.
 				 */
-				__atomic_thread_fence(__ATOMIC_RELEASE);
+				__atomic_thread_fence(rte_memory_order_release);
 			}
 			last_bkt->sig_current[i] = NULL_SIGNATURE;
-			__atomic_store_n(&last_bkt->key_idx[i],
+			rte_atomic_store_explicit(&last_bkt->key_idx[i],
 					 EMPTY_SLOT,
-					 __ATOMIC_RELEASE);
+					 rte_memory_order_release);
 			return;
 		}
 	}
@@ -1650,8 +1650,8 @@ struct rte_hash *
 
 	/* Check if key is in bucket */
 	for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
-		key_idx = __atomic_load_n(&bkt->key_idx[i],
-					  __ATOMIC_ACQUIRE);
+		key_idx = rte_atomic_load_explicit(&bkt->key_idx[i],
+					  rte_memory_order_acquire);
 		if (bkt->sig_current[i] == sig && key_idx != EMPTY_SLOT) {
 			k = (struct rte_hash_key *) ((char *)keys +
 					key_idx * h->key_entry_size);
@@ -1663,9 +1663,9 @@ struct rte_hash *
 				if (!h->no_free_on_del)
 					remove_entry(h, bkt, i);
 
-				__atomic_store_n(&bkt->key_idx[i],
+				rte_atomic_store_explicit(&bkt->key_idx[i],
 						 EMPTY_SLOT,
-						 __ATOMIC_RELEASE);
+						 rte_memory_order_release);
 
 				*pos = i;
 				/*
@@ -2077,8 +2077,8 @@ struct rte_hash *
 		 * starts. Acquire semantics will make sure that
 		 * loads in compare_signatures are not hoisted.
 		 */
-		cnt_b = __atomic_load_n(h->tbl_chng_cnt,
-					__ATOMIC_ACQUIRE);
+		cnt_b = rte_atomic_load_explicit(h->tbl_chng_cnt,
+					rte_memory_order_acquire);
 
 		/* Compare signatures and prefetch key slot of first hit */
 		for (i = 0; i < num_keys; i++) {
@@ -2121,9 +2121,9 @@ struct rte_hash *
 						__builtin_ctzl(prim_hitmask[i])
 						>> 1;
 				uint32_t key_idx =
-				__atomic_load_n(
+				rte_atomic_load_explicit(
 					&primary_bkt[i]->key_idx[hit_index],
-					__ATOMIC_ACQUIRE);
+					rte_memory_order_acquire);
 				const struct rte_hash_key *key_slot =
 					(const struct rte_hash_key *)(
 					(const char *)h->key_store +
@@ -2137,9 +2137,9 @@ struct rte_hash *
 					!rte_hash_cmp_eq(
 						key_slot->key, keys[i], h)) {
 					if (data != NULL)
-						data[i] = __atomic_load_n(
+						data[i] = rte_atomic_load_explicit(
 							&key_slot->pdata,
-							__ATOMIC_ACQUIRE);
+							rte_memory_order_acquire);
 
 					hits |= 1ULL << i;
 					positions[i] = key_idx - 1;
@@ -2153,9 +2153,9 @@ struct rte_hash *
 						__builtin_ctzl(sec_hitmask[i])
 						>> 1;
 				uint32_t key_idx =
-				__atomic_load_n(
+				rte_atomic_load_explicit(
 					&secondary_bkt[i]->key_idx[hit_index],
-					__ATOMIC_ACQUIRE);
+					rte_memory_order_acquire);
 				const struct rte_hash_key *key_slot =
 					(const struct rte_hash_key *)(
 					(const char *)h->key_store +
@@ -2170,9 +2170,9 @@ struct rte_hash *
 					!rte_hash_cmp_eq(
 						key_slot->key, keys[i], h)) {
 					if (data != NULL)
-						data[i] = __atomic_load_n(
+						data[i] = rte_atomic_load_explicit(
 							&key_slot->pdata,
-							__ATOMIC_ACQUIRE);
+							rte_memory_order_acquire);
 
 					hits |= 1ULL << i;
 					positions[i] = key_idx - 1;
@@ -2216,7 +2216,7 @@ struct rte_hash *
 		/* The loads of sig_current in compare_signatures
 		 * should not move below the load from tbl_chng_cnt.
 		 */
-		__atomic_thread_fence(__ATOMIC_ACQUIRE);
+		__atomic_thread_fence(rte_memory_order_acquire);
 		/* Re-read the table change counter to check if the
 		 * table has changed during search. If yes, re-do
 		 * the search.
@@ -2225,8 +2225,8 @@ struct rte_hash *
 		 * key index will make sure that it does not get
 		 * hoisted.
 		 */
-		cnt_a = __atomic_load_n(h->tbl_chng_cnt,
-					__ATOMIC_ACQUIRE);
+		cnt_a = rte_atomic_load_explicit(h->tbl_chng_cnt,
+					rte_memory_order_acquire);
 	} while (cnt_b != cnt_a);
 
 	if (hit_mask != NULL)
@@ -2498,8 +2498,8 @@ struct rte_hash *
 	idx = *next % RTE_HASH_BUCKET_ENTRIES;
 
 	/* If current position is empty, go to the next one */
-	while ((position = __atomic_load_n(&h->buckets[bucket_idx].key_idx[idx],
-					__ATOMIC_ACQUIRE)) == EMPTY_SLOT) {
+	while ((position = rte_atomic_load_explicit(&h->buckets[bucket_idx].key_idx[idx],
+					rte_memory_order_acquire)) == EMPTY_SLOT) {
 		(*next)++;
 		/* End of table */
 		if (*next == total_entries_main)
lib/hash/rte_cuckoo_hash.h +3 -3
index eb2644f..f7afc4d 100644
@@ -137,7 +137,7 @@ struct lcore_cache {
 struct rte_hash_key {
 	union {
 		uintptr_t idata;
-		void *pdata;
+		RTE_ATOMIC(void *) pdata;
 	};
 	/* Variable key size */
 	char key[0];
@@ -155,7 +155,7 @@ enum rte_hash_sig_compare_function {
 struct rte_hash_bucket {
 	uint16_t sig_current[RTE_HASH_BUCKET_ENTRIES];
 
-	uint32_t key_idx[RTE_HASH_BUCKET_ENTRIES];
+	RTE_ATOMIC(uint32_t) key_idx[RTE_HASH_BUCKET_ENTRIES];
 
 	uint8_t flag[RTE_HASH_BUCKET_ENTRIES];
 
@@ -229,7 +229,7 @@ struct rte_hash {
 	 * is piggy-backed to freeing of the key index.
 	 */
 	uint32_t *ext_bkt_to_free;
-	uint32_t *tbl_chng_cnt;
+	RTE_ATOMIC(uint32_t) *tbl_chng_cnt;
 	/**< Indicates if the hash table changed from last read. */
 } __rte_cache_aligned;
 
Signature · 1 line
1.8.3.1
#48 lore raw

PATCH v2 18/19 timer: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/timer/rte_timer.c | 50 +++++++++++++++++++++++++-------------------------
 lib/timer/rte_timer.h |  6 +++---
 2 files changed, 28 insertions(+), 28 deletions(-)
lib/timer/rte_timer.c +25 -25
index 85d6757..53ed221 100644
@@ -210,7 +210,7 @@ struct rte_timer_data {
 
 	status.state = RTE_TIMER_STOP;
 	status.owner = RTE_TIMER_NO_OWNER;
-	__atomic_store_n(&tim->status.u32, status.u32, __ATOMIC_RELAXED);
+	rte_atomic_store_explicit(&tim->status.u32, status.u32, rte_memory_order_relaxed);
 }
 
 /*
@@ -231,7 +231,7 @@ struct rte_timer_data {
 
 	/* wait that the timer is in correct status before update,
 	 * and mark it as being configured */
-	prev_status.u32 = __atomic_load_n(&tim->status.u32, __ATOMIC_RELAXED);
+	prev_status.u32 = rte_atomic_load_explicit(&tim->status.u32, rte_memory_order_relaxed);
 
 	while (success == 0) {
 		/* timer is running on another core
@@ -254,11 +254,11 @@ struct rte_timer_data {
 		 * timer is in CONFIG state, the state cannot be changed
 		 * by other threads. So, we should use ACQUIRE here.
 		 */
-		success = __atomic_compare_exchange_n(&tim->status.u32,
-					      &prev_status.u32,
-					      status.u32, 0,
-					      __ATOMIC_ACQUIRE,
-					      __ATOMIC_RELAXED);
+		success = rte_atomic_compare_exchange_strong_explicit(&tim->status.u32,
+					      (uint32_t *)(uintptr_t)&prev_status.u32,
+					      status.u32,
+					      rte_memory_order_acquire,
+					      rte_memory_order_relaxed);
 	}
 
 	ret_prev_status->u32 = prev_status.u32;
@@ -277,7 +277,7 @@ struct rte_timer_data {
 
 	/* wait that the timer is in correct status before update,
 	 * and mark it as running */
-	prev_status.u32 = __atomic_load_n(&tim->status.u32, __ATOMIC_RELAXED);
+	prev_status.u32 = rte_atomic_load_explicit(&tim->status.u32, rte_memory_order_relaxed);
 
 	while (success == 0) {
 		/* timer is not pending anymore */
@@ -293,11 +293,11 @@ struct rte_timer_data {
 		 * timer is in RUNNING state, the state cannot be changed
 		 * by other threads. So, we should use ACQUIRE here.
 		 */
-		success = __atomic_compare_exchange_n(&tim->status.u32,
-					      &prev_status.u32,
-					      status.u32, 0,
-					      __ATOMIC_ACQUIRE,
-					      __ATOMIC_RELAXED);
+		success = rte_atomic_compare_exchange_strong_explicit(&tim->status.u32,
+					      (uint32_t *)(uintptr_t)&prev_status.u32,
+					      status.u32,
+					      rte_memory_order_acquire,
+					      rte_memory_order_relaxed);
 	}
 
 	return 0;
@@ -530,7 +530,7 @@ struct rte_timer_data {
 	/* The "RELEASE" ordering guarantees the memory operations above
 	 * the status update are observed before the update by all threads
 	 */
-	__atomic_store_n(&tim->status.u32, status.u32, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&tim->status.u32, status.u32, rte_memory_order_release);
 
 	if (tim_lcore != lcore_id || !local_is_locked)
 		rte_spinlock_unlock(&priv_timer[tim_lcore].list_lock);
@@ -612,7 +612,7 @@ struct rte_timer_data {
 	/* The "RELEASE" ordering guarantees the memory operations above
 	 * the status update are observed before the update by all threads
 	 */
-	__atomic_store_n(&tim->status.u32, status.u32, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&tim->status.u32, status.u32, rte_memory_order_release);
 
 	return 0;
 }
@@ -646,8 +646,8 @@ struct rte_timer_data {
 int
 rte_timer_pending(struct rte_timer *tim)
 {
-	return __atomic_load_n(&tim->status.state,
-				__ATOMIC_RELAXED) == RTE_TIMER_PENDING;
+	return rte_atomic_load_explicit(&tim->status.state,
+				rte_memory_order_relaxed) == RTE_TIMER_PENDING;
 }
 
 /* must be called periodically, run all timer that expired */
@@ -753,8 +753,8 @@ struct rte_timer_data {
 			 * operations above the status update are observed
 			 * before the update by all threads
 			 */
-			__atomic_store_n(&tim->status.u32, status.u32,
-				__ATOMIC_RELEASE);
+			rte_atomic_store_explicit(&tim->status.u32, status.u32,
+				rte_memory_order_release);
 		}
 		else {
 			/* keep it in list and mark timer as pending */
@@ -766,8 +766,8 @@ struct rte_timer_data {
 			 * operations above the status update are observed
 			 * before the update by all threads
 			 */
-			__atomic_store_n(&tim->status.u32, status.u32,
-				__ATOMIC_RELEASE);
+			rte_atomic_store_explicit(&tim->status.u32, status.u32,
+				rte_memory_order_release);
 			__rte_timer_reset(tim, tim->expire + tim->period,
 				tim->period, lcore_id, tim->f, tim->arg, 1,
 				timer_data);
@@ -941,8 +941,8 @@ struct rte_timer_data {
 			 * operations above the status update are observed
 			 * before the update by all threads
 			 */
-			__atomic_store_n(&tim->status.u32, status.u32,
-				__ATOMIC_RELEASE);
+			rte_atomic_store_explicit(&tim->status.u32, status.u32,
+				rte_memory_order_release);
 		} else {
 			/* keep it in list and mark timer as pending */
 			rte_spinlock_lock(
@@ -954,8 +954,8 @@ struct rte_timer_data {
 			 * operations above the status update are observed
 			 * before the update by all threads
 			 */
-			__atomic_store_n(&tim->status.u32, status.u32,
-				__ATOMIC_RELEASE);
+			rte_atomic_store_explicit(&tim->status.u32, status.u32,
+				rte_memory_order_release);
 			__rte_timer_reset(tim, tim->expire + tim->period,
 				tim->period, this_lcore, tim->f, tim->arg, 1,
 				data);
lib/timer/rte_timer.h +3 -3
index d3927d5..a35bc08 100644
@@ -65,10 +65,10 @@ enum rte_timer_type {
  */
 union rte_timer_status {
 	struct {
-		uint16_t state;  /**< Stop, pending, running, config. */
-		int16_t owner;   /**< The lcore that owns the timer. */
+		RTE_ATOMIC(uint16_t) state;  /**< Stop, pending, running, config. */
+		RTE_ATOMIC(int16_t) owner;   /**< The lcore that owns the timer. */
 	};
-	uint32_t u32;            /**< To atomic-set status + owner. */
+	RTE_ATOMIC(uint32_t) u32;            /**< To atomic-set status + owner. */
 };
 
 #ifdef RTE_LIBRTE_TIMER_DEBUG
Signature · 1 line
1.8.3.1
#49 lore raw

PATCH v2 19/19 ring: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 drivers/net/mlx5/mlx5_hws_cnt.h   |  2 +-
 lib/ring/rte_ring_c11_pvt.h       | 33 +++++++++++++++++----------------
 lib/ring/rte_ring_core.h          | 10 +++++-----
 lib/ring/rte_ring_generic_pvt.h   |  3 ++-
 lib/ring/rte_ring_hts_elem_pvt.h  | 22 ++++++++++++----------
 lib/ring/rte_ring_peek_elem_pvt.h |  6 +++---
 lib/ring/rte_ring_rts_elem_pvt.h  | 27 ++++++++++++++-------------
 7 files changed, 54 insertions(+), 49 deletions(-)
drivers/net/mlx5/mlx5_hws_cnt.h +1 -1
index f462665..cc9ac10 100644
@@ -394,7 +394,7 @@ struct mlx5_hws_age_param {
 	__rte_ring_get_elem_addr(r, revert2head, sizeof(cnt_id_t), n,
 			&zcd->ptr1, &zcd->n1, &zcd->ptr2);
 	/* Update tail */
-	__atomic_store_n(&r->prod.tail, revert2head, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&r->prod.tail, revert2head, rte_memory_order_release);
 	return n;
 }
 
lib/ring/rte_ring_c11_pvt.h +17 -16
index f895950..f8be538 100644
@@ -22,9 +22,10 @@
 	 * we need to wait for them to complete
 	 */
 	if (!single)
-		rte_wait_until_equal_32(&ht->tail, old_val, __ATOMIC_RELAXED);
+		rte_wait_until_equal_32((volatile uint32_t *)(uintptr_t)&ht->tail, old_val,
+			rte_memory_order_relaxed);
 
-	__atomic_store_n(&ht->tail, new_val, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&ht->tail, new_val, rte_memory_order_release);
 }
 
 /**
@@ -61,19 +62,19 @@
 	unsigned int max = n;
 	int success;
 
-	*old_head = __atomic_load_n(&r->prod.head, __ATOMIC_RELAXED);
+	*old_head = rte_atomic_load_explicit(&r->prod.head, rte_memory_order_relaxed);
 	do {
 		/* Reset n to the initial burst count */
 		n = max;
 
 		/* Ensure the head is read before tail */
-		__atomic_thread_fence(__ATOMIC_ACQUIRE);
+		__atomic_thread_fence(rte_memory_order_acquire);
 
 		/* load-acquire synchronize with store-release of ht->tail
 		 * in update_tail.
 		 */
-		cons_tail = __atomic_load_n(&r->cons.tail,
-					__ATOMIC_ACQUIRE);
+		cons_tail = rte_atomic_load_explicit(&r->cons.tail,
+					rte_memory_order_acquire);
 
 		/* The subtraction is done between two unsigned 32bits value
 		 * (the result is always modulo 32 bits even if we have
@@ -95,10 +96,10 @@
 			r->prod.head = *new_head, success = 1;
 		else
 			/* on failure, *old_head is updated */
-			success = __atomic_compare_exchange_n(&r->prod.head,
+			success = rte_atomic_compare_exchange_strong_explicit(&r->prod.head,
 					old_head, *new_head,
-					0, __ATOMIC_RELAXED,
-					__ATOMIC_RELAXED);
+					rte_memory_order_relaxed,
+					rte_memory_order_relaxed);
 	} while (unlikely(success == 0));
 	return n;
 }
@@ -137,19 +138,19 @@
 	int success;
 
 	/* move cons.head atomically */
-	*old_head = __atomic_load_n(&r->cons.head, __ATOMIC_RELAXED);
+	*old_head = rte_atomic_load_explicit(&r->cons.head, rte_memory_order_relaxed);
 	do {
 		/* Restore n as it may change every loop */
 		n = max;
 
 		/* Ensure the head is read before tail */
-		__atomic_thread_fence(__ATOMIC_ACQUIRE);
+		__atomic_thread_fence(rte_memory_order_acquire);
 
 		/* this load-acquire synchronize with store-release of ht->tail
 		 * in update_tail.
 		 */
-		prod_tail = __atomic_load_n(&r->prod.tail,
-					__ATOMIC_ACQUIRE);
+		prod_tail = rte_atomic_load_explicit(&r->prod.tail,
+					rte_memory_order_acquire);
 
 		/* The subtraction is done between two unsigned 32bits value
 		 * (the result is always modulo 32 bits even if we have
@@ -170,10 +171,10 @@
 			r->cons.head = *new_head, success = 1;
 		else
 			/* on failure, *old_head will be updated */
-			success = __atomic_compare_exchange_n(&r->cons.head,
+			success = rte_atomic_compare_exchange_strong_explicit(&r->cons.head,
 							old_head, *new_head,
-							0, __ATOMIC_RELAXED,
-							__ATOMIC_RELAXED);
+							rte_memory_order_relaxed,
+							rte_memory_order_relaxed);
 	} while (unlikely(success == 0));
 	return n;
 }
lib/ring/rte_ring_core.h +5 -5
index 327fdcf..7a2b577 100644
@@ -67,7 +67,7 @@ enum rte_ring_sync_type {
  */
 struct rte_ring_headtail {
 	volatile uint32_t head;      /**< prod/consumer head. */
-	volatile uint32_t tail;      /**< prod/consumer tail. */
+	volatile RTE_ATOMIC(uint32_t) tail;      /**< prod/consumer tail. */
 	union {
 		/** sync type of prod/cons */
 		enum rte_ring_sync_type sync_type;
@@ -78,7 +78,7 @@ struct rte_ring_headtail {
 
 union __rte_ring_rts_poscnt {
 	/** raw 8B value to read/write *cnt* and *pos* as one atomic op */
-	uint64_t raw __rte_aligned(8);
+	RTE_ATOMIC(uint64_t) raw __rte_aligned(8);
 	struct {
 		uint32_t cnt; /**< head/tail reference counter */
 		uint32_t pos; /**< head/tail position */
@@ -94,10 +94,10 @@ struct rte_ring_rts_headtail {
 
 union __rte_ring_hts_pos {
 	/** raw 8B value to read/write *head* and *tail* as one atomic op */
-	uint64_t raw __rte_aligned(8);
+	RTE_ATOMIC(uint64_t) raw __rte_aligned(8);
 	struct {
-		uint32_t head; /**< head position */
-		uint32_t tail; /**< tail position */
+		RTE_ATOMIC(uint32_t) head; /**< head position */
+		RTE_ATOMIC(uint32_t) tail; /**< tail position */
 	} pos;
 };
 
lib/ring/rte_ring_generic_pvt.h +2 -1
index 5acb6e5..ffb3654 100644
@@ -23,7 +23,8 @@
 	 * we need to wait for them to complete
 	 */
 	if (!single)
-		rte_wait_until_equal_32(&ht->tail, old_val, __ATOMIC_RELAXED);
+		rte_wait_until_equal_32((volatile uint32_t *)(uintptr_t)&ht->tail, old_val,
+			rte_memory_order_relaxed);
 
 	ht->tail = new_val;
 }
lib/ring/rte_ring_hts_elem_pvt.h +12 -10
index a8678d3..91f5eec 100644
@@ -10,6 +10,8 @@
 #ifndef _RTE_RING_HTS_ELEM_PVT_H_
 #define _RTE_RING_HTS_ELEM_PVT_H_
 
+#include <rte_stdatomic.h>
+
 /**
  * @file rte_ring_hts_elem_pvt.h
  * It is not recommended to include this file directly,
@@ -30,7 +32,7 @@
 	RTE_SET_USED(enqueue);
 
 	tail = old_tail + num;
-	__atomic_store_n(&ht->ht.pos.tail, tail, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&ht->ht.pos.tail, tail, rte_memory_order_release);
 }
 
 /**
@@ -44,7 +46,7 @@
 {
 	while (p->pos.head != p->pos.tail) {
 		rte_pause();
-		p->raw = __atomic_load_n(&ht->ht.raw, __ATOMIC_ACQUIRE);
+		p->raw = rte_atomic_load_explicit(&ht->ht.raw, rte_memory_order_acquire);
 	}
 }
 
@@ -61,7 +63,7 @@
 
 	const uint32_t capacity = r->capacity;
 
-	op.raw = __atomic_load_n(&r->hts_prod.ht.raw, __ATOMIC_ACQUIRE);
+	op.raw = rte_atomic_load_explicit(&r->hts_prod.ht.raw, rte_memory_order_acquire);
 
 	do {
 		/* Reset n to the initial burst count */
@@ -98,9 +100,9 @@
 	 *  - OOO reads of cons tail value
 	 *  - OOO copy of elems from the ring
 	 */
-	} while (__atomic_compare_exchange_n(&r->hts_prod.ht.raw,
-			&op.raw, np.raw,
-			0, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE) == 0);
+	} while (rte_atomic_compare_exchange_strong_explicit(&r->hts_prod.ht.raw,
+			(uint64_t *)(uintptr_t)&op.raw, np.raw,
+			rte_memory_order_acquire, rte_memory_order_acquire) == 0);
 
 	*old_head = op.pos.head;
 	return n;
@@ -117,7 +119,7 @@
 	uint32_t n;
 	union __rte_ring_hts_pos np, op;
 
-	op.raw = __atomic_load_n(&r->hts_cons.ht.raw, __ATOMIC_ACQUIRE);
+	op.raw = rte_atomic_load_explicit(&r->hts_cons.ht.raw, rte_memory_order_acquire);
 
 	/* move cons.head atomically */
 	do {
@@ -153,9 +155,9 @@
 	 *  - OOO reads of prod tail value
 	 *  - OOO copy of elems from the ring
 	 */
-	} while (__atomic_compare_exchange_n(&r->hts_cons.ht.raw,
-			&op.raw, np.raw,
-			0, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE) == 0);
+	} while (rte_atomic_compare_exchange_strong_explicit(&r->hts_cons.ht.raw,
+			(uint64_t *)(uintptr_t)&op.raw, np.raw,
+			rte_memory_order_acquire, rte_memory_order_acquire) == 0);
 
 	*old_head = op.pos.head;
 	return n;
lib/ring/rte_ring_peek_elem_pvt.h +3 -3
index bb0a7d5..b5f0822 100644
@@ -59,7 +59,7 @@
 
 	pos = tail + num;
 	ht->head = pos;
-	__atomic_store_n(&ht->tail, pos, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&ht->tail, pos, rte_memory_order_release);
 }
 
 /**
@@ -78,7 +78,7 @@
 	uint32_t n;
 	union __rte_ring_hts_pos p;
 
-	p.raw = __atomic_load_n(&ht->ht.raw, __ATOMIC_RELAXED);
+	p.raw = rte_atomic_load_explicit(&ht->ht.raw, rte_memory_order_relaxed);
 	n = p.pos.head - p.pos.tail;
 
 	RTE_ASSERT(n >= num);
@@ -104,7 +104,7 @@
 	p.pos.head = tail + num;
 	p.pos.tail = p.pos.head;
 
-	__atomic_store_n(&ht->ht.raw, p.raw, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&ht->ht.raw, p.raw, rte_memory_order_release);
 }
 
 /**
lib/ring/rte_ring_rts_elem_pvt.h +14 -13
index 7164213..1226503 100644
@@ -31,18 +31,19 @@
 	 * might preceded us, then don't update tail with new value.
 	 */
 
-	ot.raw = __atomic_load_n(&ht->tail.raw, __ATOMIC_ACQUIRE);
+	ot.raw = rte_atomic_load_explicit(&ht->tail.raw, rte_memory_order_acquire);
 
 	do {
 		/* on 32-bit systems we have to do atomic read here */
-		h.raw = __atomic_load_n(&ht->head.raw, __ATOMIC_RELAXED);
+		h.raw = rte_atomic_load_explicit(&ht->head.raw, rte_memory_order_relaxed);
 
 		nt.raw = ot.raw;
 		if (++nt.val.cnt == h.val.cnt)
 			nt.val.pos = h.val.pos;
 
-	} while (__atomic_compare_exchange_n(&ht->tail.raw, &ot.raw, nt.raw,
-			0, __ATOMIC_RELEASE, __ATOMIC_ACQUIRE) == 0);
+	} while (rte_atomic_compare_exchange_strong_explicit(&ht->tail.raw,
+			(uint64_t *)(uintptr_t)&ot.raw, nt.raw,
+			rte_memory_order_release, rte_memory_order_acquire) == 0);
 }
 
 /**
@@ -59,7 +60,7 @@
 
 	while (h->val.pos - ht->tail.val.pos > max) {
 		rte_pause();
-		h->raw = __atomic_load_n(&ht->head.raw, __ATOMIC_ACQUIRE);
+		h->raw = rte_atomic_load_explicit(&ht->head.raw, rte_memory_order_acquire);
 	}
 }
 
@@ -76,7 +77,7 @@
 
 	const uint32_t capacity = r->capacity;
 
-	oh.raw = __atomic_load_n(&r->rts_prod.head.raw, __ATOMIC_ACQUIRE);
+	oh.raw = rte_atomic_load_explicit(&r->rts_prod.head.raw, rte_memory_order_acquire);
 
 	do {
 		/* Reset n to the initial burst count */
@@ -113,9 +114,9 @@
 	 *  - OOO reads of cons tail value
 	 *  - OOO copy of elems to the ring
 	 */
-	} while (__atomic_compare_exchange_n(&r->rts_prod.head.raw,
-			&oh.raw, nh.raw,
-			0, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE) == 0);
+	} while (rte_atomic_compare_exchange_strong_explicit(&r->rts_prod.head.raw,
+			(uint64_t *)(uintptr_t)&oh.raw, nh.raw,
+			rte_memory_order_acquire, rte_memory_order_acquire) == 0);
 
 	*old_head = oh.val.pos;
 	return n;
@@ -132,7 +133,7 @@
 	uint32_t n;
 	union __rte_ring_rts_poscnt nh, oh;
 
-	oh.raw = __atomic_load_n(&r->rts_cons.head.raw, __ATOMIC_ACQUIRE);
+	oh.raw = rte_atomic_load_explicit(&r->rts_cons.head.raw, rte_memory_order_acquire);
 
 	/* move cons.head atomically */
 	do {
@@ -168,9 +169,9 @@
 	 *  - OOO reads of prod tail value
 	 *  - OOO copy of elems from the ring
 	 */
-	} while (__atomic_compare_exchange_n(&r->rts_cons.head.raw,
-			&oh.raw, nh.raw,
-			0, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE) == 0);
+	} while (rte_atomic_compare_exchange_strong_explicit(&r->rts_cons.head.raw,
+			(uint64_t *)(uintptr_t)&oh.raw, nh.raw,
+			rte_memory_order_acquire, rte_memory_order_acquire) == 0);
 
 	*old_head = oh.val.pos;
 	return n;
Signature · 1 line
1.8.3.1
#50 lore raw

Re: [PATCH v2 19/19] ring: use rte optional stdatomic API

17.10.2023 21:31, Tyler Retzlaff пишет:

Quoted 125 lines

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
  drivers/net/mlx5/mlx5_hws_cnt.h   |  2 +-
  lib/ring/rte_ring_c11_pvt.h       | 33 +++++++++++++++++----------------
  lib/ring/rte_ring_core.h          | 10 +++++-----
  lib/ring/rte_ring_generic_pvt.h   |  3 ++-
  lib/ring/rte_ring_hts_elem_pvt.h  | 22 ++++++++++++----------
  lib/ring/rte_ring_peek_elem_pvt.h |  6 +++---
  lib/ring/rte_ring_rts_elem_pvt.h  | 27 ++++++++++++++-------------
  7 files changed, 54 insertions(+), 49 deletions(-)
drivers/net/mlx5/mlx5_hws_cnt.h +1 -1
index f462665..cc9ac10 100644
@@ -394,7 +394,7 @@ struct mlx5_hws_age_param {
  	__rte_ring_get_elem_addr(r, revert2head, sizeof(cnt_id_t), n,
  			&zcd->ptr1, &zcd->n1, &zcd->ptr2);
  	/* Update tail */
-	__atomic_store_n(&r->prod.tail, revert2head, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&r->prod.tail, revert2head, rte_memory_order_release);
  	return n;
  }
 
lib/ring/rte_ring_c11_pvt.h +17 -16
index f895950..f8be538 100644
@@ -22,9 +22,10 @@
  	 * we need to wait for them to complete
  	 */
  	if (!single)
-		rte_wait_until_equal_32(&ht->tail, old_val, __ATOMIC_RELAXED);
+		rte_wait_until_equal_32((volatile uint32_t *)(uintptr_t)&ht->tail, old_val,
+			rte_memory_order_relaxed);
 
-	__atomic_store_n(&ht->tail, new_val, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&ht->tail, new_val, rte_memory_order_release);
  }
 
  /**
@@ -61,19 +62,19 @@
  	unsigned int max = n;
  	int success;
 
-	*old_head = __atomic_load_n(&r->prod.head, __ATOMIC_RELAXED);
+	*old_head = rte_atomic_load_explicit(&r->prod.head, rte_memory_order_relaxed);
  	do {
  		/* Reset n to the initial burst count */
  		n = max;
 
  		/* Ensure the head is read before tail */
-		__atomic_thread_fence(__ATOMIC_ACQUIRE);
+		__atomic_thread_fence(rte_memory_order_acquire);
 
  		/* load-acquire synchronize with store-release of ht->tail
  		 * in update_tail.
  		 */
-		cons_tail = __atomic_load_n(&r->cons.tail,
-					__ATOMIC_ACQUIRE);
+		cons_tail = rte_atomic_load_explicit(&r->cons.tail,
+					rte_memory_order_acquire);
 
  		/* The subtraction is done between two unsigned 32bits value
  		 * (the result is always modulo 32 bits even if we have
@@ -95,10 +96,10 @@
  			r->prod.head = *new_head, success = 1;
  		else
  			/* on failure, *old_head is updated */
-			success = __atomic_compare_exchange_n(&r->prod.head,
+			success = rte_atomic_compare_exchange_strong_explicit(&r->prod.head,
  					old_head, *new_head,
-					0, __ATOMIC_RELAXED,
-					__ATOMIC_RELAXED);
+					rte_memory_order_relaxed,
+					rte_memory_order_relaxed);
  	} while (unlikely(success == 0));
  	return n;
  }
@@ -137,19 +138,19 @@
  	int success;
 
  	/* move cons.head atomically */
-	*old_head = __atomic_load_n(&r->cons.head, __ATOMIC_RELAXED);
+	*old_head = rte_atomic_load_explicit(&r->cons.head, rte_memory_order_relaxed);
  	do {
  		/* Restore n as it may change every loop */
  		n = max;
 
  		/* Ensure the head is read before tail */
-		__atomic_thread_fence(__ATOMIC_ACQUIRE);
+		__atomic_thread_fence(rte_memory_order_acquire);
 
  		/* this load-acquire synchronize with store-release of ht->tail
  		 * in update_tail.
  		 */
-		prod_tail = __atomic_load_n(&r->prod.tail,
-					__ATOMIC_ACQUIRE);
+		prod_tail = rte_atomic_load_explicit(&r->prod.tail,
+					rte_memory_order_acquire);
 
  		/* The subtraction is done between two unsigned 32bits value
  		 * (the result is always modulo 32 bits even if we have
@@ -170,10 +171,10 @@
  			r->cons.head = *new_head, success = 1;
  		else
  			/* on failure, *old_head will be updated */
-			success = __atomic_compare_exchange_n(&r->cons.head,
+			success = rte_atomic_compare_exchange_strong_explicit(&r->cons.head,
  							old_head, *new_head,
-							0, __ATOMIC_RELAXED,
-							__ATOMIC_RELAXED);
+							rte_memory_order_relaxed,
+							rte_memory_order_relaxed);
  	} while (unlikely(success == 0));
  	return n;
  }
lib/ring/rte_ring_core.h +0 -0
index 327fdcf..7a2b577 100644

lib/ring/rte_ring_core.h enum rte_ring_sync_type { · line 70

Quoted 5 lines
lib/ring/rte_ring_core.h +1 -1
@@ -67,7 +67,7 @@ enum rte_ring_sync_type {
   */
  struct rte_ring_headtail {
  	volatile uint32_t head;      /**< prod/consumer head. */
-	volatile uint32_t tail;      /**< prod/consumer tail. */
+	volatile RTE_ATOMIC(uint32_t) tail;      /**< prod/consumer tail. */

Probably a stupid q:
why we do need RTE_ATOMIC() around tail only?
Why head is not affected?

Quoted 31 lines
lib/ring/rte_ring_core.h +4 -4
  	union {
  		/** sync type of prod/cons */
  		enum rte_ring_sync_type sync_type;
@@ -78,7 +78,7 @@ struct rte_ring_headtail {
 
  union __rte_ring_rts_poscnt {
  	/** raw 8B value to read/write *cnt* and *pos* as one atomic op */
-	uint64_t raw __rte_aligned(8);
+	RTE_ATOMIC(uint64_t) raw __rte_aligned(8);
  	struct {
  		uint32_t cnt; /**< head/tail reference counter */
  		uint32_t pos; /**< head/tail position */
@@ -94,10 +94,10 @@ struct rte_ring_rts_headtail {
 
  union __rte_ring_hts_pos {
  	/** raw 8B value to read/write *head* and *tail* as one atomic op */
-	uint64_t raw __rte_aligned(8);
+	RTE_ATOMIC(uint64_t) raw __rte_aligned(8);
  	struct {
-		uint32_t head; /**< head position */
-		uint32_t tail; /**< tail position */
+		RTE_ATOMIC(uint32_t) head; /**< head position */
+		RTE_ATOMIC(uint32_t) tail; /**< tail position */
  	} pos;
  };
 
lib/ring/rte_ring_generic_pvt.h +0 -0
index 5acb6e5..ffb3654 100644

lib/ring/rte_ring_generic_pvt.h line 26

Quoted 5 lines
lib/ring/rte_ring_generic_pvt.h +1 -1
@@ -23,7 +23,8 @@
  	 * we need to wait for them to complete
  	 */
  	if (!single)
-		rte_wait_until_equal_32(&ht->tail, old_val, __ATOMIC_RELAXED);
+		rte_wait_until_equal_32((volatile uint32_t *)(uintptr_t)&ht->tail, old_val,

I suppose we do need that double type conversion only for atomic types right?

Quoted 191 lines
lib/ring/rte_ring_generic_pvt.h +1 -0
+			rte_memory_order_relaxed);
 
  	ht->tail = new_val;
  }
lib/ring/rte_ring_hts_elem_pvt.h +12 -10
index a8678d3..91f5eec 100644
@@ -10,6 +10,8 @@
  #ifndef _RTE_RING_HTS_ELEM_PVT_H_
  #define _RTE_RING_HTS_ELEM_PVT_H_
 
+#include <rte_stdatomic.h>
+
  /**
   * @file rte_ring_hts_elem_pvt.h
   * It is not recommended to include this file directly,
@@ -30,7 +32,7 @@
  	RTE_SET_USED(enqueue);
 
  	tail = old_tail + num;
-	__atomic_store_n(&ht->ht.pos.tail, tail, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&ht->ht.pos.tail, tail, rte_memory_order_release);
  }
 
  /**
@@ -44,7 +46,7 @@
  {
  	while (p->pos.head != p->pos.tail) {
  		rte_pause();
-		p->raw = __atomic_load_n(&ht->ht.raw, __ATOMIC_ACQUIRE);
+		p->raw = rte_atomic_load_explicit(&ht->ht.raw, rte_memory_order_acquire);
  	}
  }
 
@@ -61,7 +63,7 @@
 
  	const uint32_t capacity = r->capacity;
 
-	op.raw = __atomic_load_n(&r->hts_prod.ht.raw, __ATOMIC_ACQUIRE);
+	op.raw = rte_atomic_load_explicit(&r->hts_prod.ht.raw, rte_memory_order_acquire);
 
  	do {
  		/* Reset n to the initial burst count */
@@ -98,9 +100,9 @@
  	 *  - OOO reads of cons tail value
  	 *  - OOO copy of elems from the ring
  	 */
-	} while (__atomic_compare_exchange_n(&r->hts_prod.ht.raw,
-			&op.raw, np.raw,
-			0, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE) == 0);
+	} while (rte_atomic_compare_exchange_strong_explicit(&r->hts_prod.ht.raw,
+			(uint64_t *)(uintptr_t)&op.raw, np.raw,
+			rte_memory_order_acquire, rte_memory_order_acquire) == 0);
 
  	*old_head = op.pos.head;
  	return n;
@@ -117,7 +119,7 @@
  	uint32_t n;
  	union __rte_ring_hts_pos np, op;
 
-	op.raw = __atomic_load_n(&r->hts_cons.ht.raw, __ATOMIC_ACQUIRE);
+	op.raw = rte_atomic_load_explicit(&r->hts_cons.ht.raw, rte_memory_order_acquire);
 
  	/* move cons.head atomically */
  	do {
@@ -153,9 +155,9 @@
  	 *  - OOO reads of prod tail value
  	 *  - OOO copy of elems from the ring
  	 */
-	} while (__atomic_compare_exchange_n(&r->hts_cons.ht.raw,
-			&op.raw, np.raw,
-			0, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE) == 0);
+	} while (rte_atomic_compare_exchange_strong_explicit(&r->hts_cons.ht.raw,
+			(uint64_t *)(uintptr_t)&op.raw, np.raw,
+			rte_memory_order_acquire, rte_memory_order_acquire) == 0);
 
  	*old_head = op.pos.head;
  	return n;
lib/ring/rte_ring_peek_elem_pvt.h +3 -3
index bb0a7d5..b5f0822 100644
@@ -59,7 +59,7 @@
 
  	pos = tail + num;
  	ht->head = pos;
-	__atomic_store_n(&ht->tail, pos, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&ht->tail, pos, rte_memory_order_release);
  }
 
  /**
@@ -78,7 +78,7 @@
  	uint32_t n;
  	union __rte_ring_hts_pos p;
 
-	p.raw = __atomic_load_n(&ht->ht.raw, __ATOMIC_RELAXED);
+	p.raw = rte_atomic_load_explicit(&ht->ht.raw, rte_memory_order_relaxed);
  	n = p.pos.head - p.pos.tail;
 
  	RTE_ASSERT(n >= num);
@@ -104,7 +104,7 @@
  	p.pos.head = tail + num;
  	p.pos.tail = p.pos.head;
 
-	__atomic_store_n(&ht->ht.raw, p.raw, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&ht->ht.raw, p.raw, rte_memory_order_release);
  }
 
  /**
lib/ring/rte_ring_rts_elem_pvt.h +14 -13
index 7164213..1226503 100644
@@ -31,18 +31,19 @@
  	 * might preceded us, then don't update tail with new value.
  	 */
 
-	ot.raw = __atomic_load_n(&ht->tail.raw, __ATOMIC_ACQUIRE);
+	ot.raw = rte_atomic_load_explicit(&ht->tail.raw, rte_memory_order_acquire);
 
  	do {
  		/* on 32-bit systems we have to do atomic read here */
-		h.raw = __atomic_load_n(&ht->head.raw, __ATOMIC_RELAXED);
+		h.raw = rte_atomic_load_explicit(&ht->head.raw, rte_memory_order_relaxed);
 
  		nt.raw = ot.raw;
  		if (++nt.val.cnt == h.val.cnt)
  			nt.val.pos = h.val.pos;
 
-	} while (__atomic_compare_exchange_n(&ht->tail.raw, &ot.raw, nt.raw,
-			0, __ATOMIC_RELEASE, __ATOMIC_ACQUIRE) == 0);
+	} while (rte_atomic_compare_exchange_strong_explicit(&ht->tail.raw,
+			(uint64_t *)(uintptr_t)&ot.raw, nt.raw,
+			rte_memory_order_release, rte_memory_order_acquire) == 0);
  }
 
  /**
@@ -59,7 +60,7 @@
 
  	while (h->val.pos - ht->tail.val.pos > max) {
  		rte_pause();
-		h->raw = __atomic_load_n(&ht->head.raw, __ATOMIC_ACQUIRE);
+		h->raw = rte_atomic_load_explicit(&ht->head.raw, rte_memory_order_acquire);
  	}
  }
 
@@ -76,7 +77,7 @@
 
  	const uint32_t capacity = r->capacity;
 
-	oh.raw = __atomic_load_n(&r->rts_prod.head.raw, __ATOMIC_ACQUIRE);
+	oh.raw = rte_atomic_load_explicit(&r->rts_prod.head.raw, rte_memory_order_acquire);
 
  	do {
  		/* Reset n to the initial burst count */
@@ -113,9 +114,9 @@
  	 *  - OOO reads of cons tail value
  	 *  - OOO copy of elems to the ring
  	 */
-	} while (__atomic_compare_exchange_n(&r->rts_prod.head.raw,
-			&oh.raw, nh.raw,
-			0, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE) == 0);
+	} while (rte_atomic_compare_exchange_strong_explicit(&r->rts_prod.head.raw,
+			(uint64_t *)(uintptr_t)&oh.raw, nh.raw,
+			rte_memory_order_acquire, rte_memory_order_acquire) == 0);
 
  	*old_head = oh.val.pos;
  	return n;
@@ -132,7 +133,7 @@
  	uint32_t n;
  	union __rte_ring_rts_poscnt nh, oh;
 
-	oh.raw = __atomic_load_n(&r->rts_cons.head.raw, __ATOMIC_ACQUIRE);
+	oh.raw = rte_atomic_load_explicit(&r->rts_cons.head.raw, rte_memory_order_acquire);
 
  	/* move cons.head atomically */
  	do {
@@ -168,9 +169,9 @@
  	 *  - OOO reads of prod tail value
  	 *  - OOO copy of elems from the ring
  	 */
-	} while (__atomic_compare_exchange_n(&r->rts_cons.head.raw,
-			&oh.raw, nh.raw,
-			0, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE) == 0);
+	} while (rte_atomic_compare_exchange_strong_explicit(&r->rts_cons.head.raw,
+			(uint64_t *)(uintptr_t)&oh.raw, nh.raw,
+			rte_memory_order_acquire, rte_memory_order_acquire) == 0);
 
  	*old_head = oh.val.pos;
  	return n;
#51 lore raw

RE: [PATCH v2 19/19] ring: use rte optional stdatomic API

Quoted 9 lines

From: Konstantin Ananyev [mailto:[email protected]]
Sent: Tuesday, 24 October 2023 10.43

17.10.2023 21:31, Tyler Retzlaff пишет:

Quoted 5 lines

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---

[...]

Quoted 8 lines
Quoted 4 lines

if (!single)

(quoted diff) +1 -1
-		rte_wait_until_equal_32(&ht->tail, old_val, __ATOMIC_RELAXED);
+		rte_wait_until_equal_32((volatile uint32_t *)(uintptr_t)&ht-

tail, old_val,

I suppose we do need that double type conversion only for atomic types right?

Quoted 4 lines

lib/ring/rte_ring_generic_pvt.h line 30

Quoted 4 lines
lib/ring/rte_ring_generic_pvt.h +1 -0
+			rte_memory_order_relaxed);
 
  	ht->tail = new_val;
  }

This got me thinking...

Do we want to cast away the value's atomic attribute like this, or should we introduce new rte_atomic_wait_XX() functions with the parameters being pointers to atomic values, instead of pointers to simple values?

Just a thought.

The initial rte_atomic_wait_XX() implementations could simply cast a away the atomic attribute like here.

#52 lore raw

Re: [PATCH v2 19/19] ring: use rte optional stdatomic API

Quoted 20 lines · Morten Brørup
Quoted 9 lines

From: Konstantin Ananyev [mailto:[email protected]]
Sent: Tuesday, 24 October 2023 10.43

17.10.2023 21:31, Tyler Retzlaff пишет:

Quoted 5 lines

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---

[...]

Quoted 8 lines
Quoted 4 lines

if (!single)

(quoted diff) +1 -1
-		rte_wait_until_equal_32(&ht->tail, old_val, __ATOMIC_RELAXED);
+		rte_wait_until_equal_32((volatile uint32_t *)(uintptr_t)&ht-

tail, old_val,

I suppose we do need that double type conversion only for atomic types right?

Quoted 8 lines
Quoted 4 lines

lib/ring/rte_ring_generic_pvt.h line 30

Quoted 4 lines
lib/ring/rte_ring_generic_pvt.h +1 -0
+			rte_memory_order_relaxed);
 
  	ht->tail = new_val;
  }

This got me thinking...

Do we want to cast away the value's atomic attribute like this, or should we introduce new rte_atomic_wait_XX() functions with the parameters being pointers to atomic values, instead of pointers to simple values?

just some notes here.

so first let me start with it's okay to do this cast but only because we have knowledge of the internal implementation detail and this series has to do this in a few places.

basically internally the actual atomic operation is fed back into an intrinsic/builtin that is either re-qualified as __rte_atomic or doesn't require qualification. i agree it isn't optimal since we have to take care should we ever alter the implementation to avoid compatibility problems but unlikely for it to be changed.

we could provide new api but i'm not sure we can do that this late in the release cycle. notably i think it would be nicer if it *could* be made to be 'generic' as used literally in the atomics documentation which means it may operate on non-integer and non-pointer types.

Quoted 5 lines

Just a thought.

The initial rte_atomic_wait_XX() implementations could simply cast a away the atomic attribute like here.

#53 lore raw

RE: [PATCH v2 19/19] ring: use rte optional stdatomic API

Quoted 51 lines

From: Tyler Retzlaff [mailto:[email protected]]
Sent: Tuesday, 24 October 2023 17.59

Quoted 7 lines · Morten Brørup
Quoted 7 lines

From: Konstantin Ananyev [mailto:[email protected]]
Sent: Tuesday, 24 October 2023 10.43

17.10.2023 21:31, Tyler Retzlaff пишет:

Quoted 3 lines

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

Quoted 7 lines
Quoted 2 lines
Quoted 2 lines

---

[...]

Quoted 2 lines

Review

Quoted 2 lines

if (!single)

(quoted diff) +0 -1
-		rte_wait_until_equal_32(&ht->tail, old_val,

__ATOMIC_RELAXED);

Quoted 1 line
Quoted 1 line

Review

Quoted 1 line
(quoted diff) +1 -0
+		rte_wait_until_equal_32((volatile uint32_t

*)(uintptr_t)&ht-

Quoted 3 lines
Quoted 3 lines
Quoted 1 line

tail, old_val,

I suppose we do need that double type conversion only for atomic

types

Quoted 10 lines
Quoted 6 lines

right?

lib/ring/rte_ring_generic_pvt.h line 30

Quoted 4 lines
lib/ring/rte_ring_generic_pvt.h +1 -0
+			rte_memory_order_relaxed);
 
  	ht->tail = new_val;
  }

This got me thinking...

Do we want to cast away the value's atomic attribute like this, or

should we introduce new rte_atomic_wait_XX() functions with the parameters being pointers to atomic values, instead of pointers to simple values?

just some notes here.

so first let me start with it's okay to do this cast but only because we have knowledge of the internal implementation detail and this series has to do this in a few places.

basically internally the actual atomic operation is fed back into an intrinsic/builtin that is either re-qualified as __rte_atomic or doesn't require qualification. i agree it isn't optimal since we have to take care should we ever alter the implementation to avoid compatibility problems but unlikely for it to be changed.

Quoted 4 lines

we could provide new api but i'm not sure we can do that this late in the release cycle. notably i think it would be nicer if it *could* be made to be 'generic' as used literally in the atomics documentation which means it may operate on non-integer and non-pointer types.

I agree with all of the above, incl. the conclusion:
Future proofing this (for a very distant future) is not worth the effort - and added APIs - at this time.

Thank you for elaborating, Tyler.

Quoted 7 lines
Quoted 5 lines

Just a thought.

The initial rte_atomic_wait_XX() implementations could simply cast a

away the atomic attribute like here.

Quoted 1 line
#54 lore raw

Re: [PATCH v2 19/19] ring: use rte optional stdatomic API

In reply to Konstantin Ananyev
Quoted 116 lines · Konstantin Ananyev

17.10.2023 21:31, Tyler Retzlaff пишет:

Quoted 115 lines

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 drivers/net/mlx5/mlx5_hws_cnt.h   |  2 +-
 lib/ring/rte_ring_c11_pvt.h       | 33 +++++++++++++++++----------------
 lib/ring/rte_ring_core.h          | 10 +++++-----
 lib/ring/rte_ring_generic_pvt.h   |  3 ++-
 lib/ring/rte_ring_hts_elem_pvt.h  | 22 ++++++++++++----------
 lib/ring/rte_ring_peek_elem_pvt.h |  6 +++---
 lib/ring/rte_ring_rts_elem_pvt.h  | 27 ++++++++++++++-------------
 7 files changed, 54 insertions(+), 49 deletions(-)
drivers/net/mlx5/mlx5_hws_cnt.h +1 -1
index f462665..cc9ac10 100644
@@ -394,7 +394,7 @@ struct mlx5_hws_age_param {
 	__rte_ring_get_elem_addr(r, revert2head, sizeof(cnt_id_t), n,
 			&zcd->ptr1, &zcd->n1, &zcd->ptr2);
 	/* Update tail */
-	__atomic_store_n(&r->prod.tail, revert2head, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&r->prod.tail, revert2head, rte_memory_order_release);
 	return n;
 }
lib/ring/rte_ring_c11_pvt.h +17 -16
index f895950..f8be538 100644
@@ -22,9 +22,10 @@
 	 * we need to wait for them to complete
 	 */
 	if (!single)
-		rte_wait_until_equal_32(&ht->tail, old_val, __ATOMIC_RELAXED);
+		rte_wait_until_equal_32((volatile uint32_t *)(uintptr_t)&ht->tail, old_val,
+			rte_memory_order_relaxed);
-	__atomic_store_n(&ht->tail, new_val, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&ht->tail, new_val, rte_memory_order_release);
 }
 /**
@@ -61,19 +62,19 @@
 	unsigned int max = n;
 	int success;
-	*old_head = __atomic_load_n(&r->prod.head, __ATOMIC_RELAXED);
+	*old_head = rte_atomic_load_explicit(&r->prod.head, rte_memory_order_relaxed);
 	do {
 		/* Reset n to the initial burst count */
 		n = max;
 		/* Ensure the head is read before tail */
-		__atomic_thread_fence(__ATOMIC_ACQUIRE);
+		__atomic_thread_fence(rte_memory_order_acquire);
 		/* load-acquire synchronize with store-release of ht->tail
 		 * in update_tail.
 		 */
-		cons_tail = __atomic_load_n(&r->cons.tail,
-					__ATOMIC_ACQUIRE);
+		cons_tail = rte_atomic_load_explicit(&r->cons.tail,
+					rte_memory_order_acquire);
 		/* The subtraction is done between two unsigned 32bits value
 		 * (the result is always modulo 32 bits even if we have
@@ -95,10 +96,10 @@
 			r->prod.head = *new_head, success = 1;
 		else
 			/* on failure, *old_head is updated */
-			success = __atomic_compare_exchange_n(&r->prod.head,
+			success = rte_atomic_compare_exchange_strong_explicit(&r->prod.head,
 					old_head, *new_head,
-					0, __ATOMIC_RELAXED,
-					__ATOMIC_RELAXED);
+					rte_memory_order_relaxed,
+					rte_memory_order_relaxed);
 	} while (unlikely(success == 0));
 	return n;
 }
@@ -137,19 +138,19 @@
 	int success;
 	/* move cons.head atomically */
-	*old_head = __atomic_load_n(&r->cons.head, __ATOMIC_RELAXED);
+	*old_head = rte_atomic_load_explicit(&r->cons.head, rte_memory_order_relaxed);
 	do {
 		/* Restore n as it may change every loop */
 		n = max;
 		/* Ensure the head is read before tail */
-		__atomic_thread_fence(__ATOMIC_ACQUIRE);
+		__atomic_thread_fence(rte_memory_order_acquire);
 		/* this load-acquire synchronize with store-release of ht->tail
 		 * in update_tail.
 		 */
-		prod_tail = __atomic_load_n(&r->prod.tail,
-					__ATOMIC_ACQUIRE);
+		prod_tail = rte_atomic_load_explicit(&r->prod.tail,
+					rte_memory_order_acquire);
 		/* The subtraction is done between two unsigned 32bits value
 		 * (the result is always modulo 32 bits even if we have
@@ -170,10 +171,10 @@
 			r->cons.head = *new_head, success = 1;
 		else
 			/* on failure, *old_head will be updated */
-			success = __atomic_compare_exchange_n(&r->cons.head,
+			success = rte_atomic_compare_exchange_strong_explicit(&r->cons.head,
 							old_head, *new_head,
-							0, __ATOMIC_RELAXED,
-							__ATOMIC_RELAXED);
+							rte_memory_order_relaxed,
+							rte_memory_order_relaxed);
 	} while (unlikely(success == 0));
 	return n;
 }
lib/ring/rte_ring_core.h +0 -0
index 327fdcf..7a2b577 100644
@@ -67,7 +67,7 @@ enum rte_ring_sync_type {
  */
Quoted 8 lines

lib/ring/rte_ring_core.h enum rte_ring_sync_type { · line 70

Quoted 4 lines
lib/ring/rte_ring_core.h +1 -1
 struct rte_ring_headtail {
 	volatile uint32_t head;      /**< prod/consumer head. */
-	volatile uint32_t tail;      /**< prod/consumer tail. */
+	volatile RTE_ATOMIC(uint32_t) tail;      /**< prod/consumer tail. */

Probably a stupid q:
why we do need RTE_ATOMIC() around tail only?
Why head is not affected?

you have a good eye and this is a slightly common issue that i've seen and there appear to be some interesting things showing up.

the field being qualified has atomic operation performed on it the other field does not in the implementation. it may be an indication of a bug in the existing code or it may be intentional.

case 1. atomics should be used but they aren't.

there are fields in structures and variables that were accessed in a 'mixed' manner. that is in some instances __atomic_op_xxx was being used on them and in other instances not. sometimes it is the initialization case so it is probably okay, sometimes maybe not...

case 2. broader scope atomic operation, or we don't care if narrower
access is atomic.

e.g.
union {

   struct {
       uint32_t head;
       RTE_ATOMIC(uint32_t) tail;
    }
    RTE_ATOMIC(uint64_t) combined;
}

again, could be an indication of missing use of atomic, often the operation on the `combined' field consistently uses atomics but one of the head/tail fields will not be. on purpose? maybe if we are just doing == comparison?

my approach in this series prioritized no functional change. as a result if any of the above are real bugs, they stay real bugs but i have not changed the way the variables are accessed. if i were to change the code and start atomic specifying it has a risk of performance regression (for cases where it isn't a bug) because specifying would result in the compiler code generating for strongest ordering seq_cst for accesses that are not using atomic generic functions that specify ordering.

there is another case which comes up half a dozen times or so that is also concerning to me, but i would need the maintainers of the code to adapt the code to be correct or maybe it is okay...

case 3. qualification discard .. is the existing code really okay?

e.g.

atomic_compare_exchange(*object, *expected, desired, ...)

the issue is with the specification of the memory aliased by expected. gcc doesn't complain or enforce discarding of qualification when using builtin intrinsics. the result is that if expected is an atomic type it may be accessed in a non-atomic manner by the code generated for the atomic operation.

again, i have chosen to maintain existing behavior by casting away the qualification if present on the expected argument.

i feel that in terms of mutating the source tree it is best to separate conversion to atomic specified/qualified types into this separate series and then follow up with additional changes that may have functional/performance impact if not for any other reason that it narrows where you have to look if there is a change. certainly conversion to atomics has made these cases far easier to spot in the code.

finally in terms of most of the toolchain/targets all of this is pretty moot because most of them are defaulting to enable_stdatomics=false so most likely if there are problems they will manifest on windows built with msvc only.

thoughts?

Quoted 197 lines
Quoted 29 lines
lib/ring/rte_ring_core.h +4 -4
 	union {
 		/** sync type of prod/cons */
 		enum rte_ring_sync_type sync_type;
@@ -78,7 +78,7 @@ struct rte_ring_headtail {
 union __rte_ring_rts_poscnt {
 	/** raw 8B value to read/write *cnt* and *pos* as one atomic op */
-	uint64_t raw __rte_aligned(8);
+	RTE_ATOMIC(uint64_t) raw __rte_aligned(8);
 	struct {
 		uint32_t cnt; /**< head/tail reference counter */
 		uint32_t pos; /**< head/tail position */
@@ -94,10 +94,10 @@ struct rte_ring_rts_headtail {
 union __rte_ring_hts_pos {
 	/** raw 8B value to read/write *head* and *tail* as one atomic op */
-	uint64_t raw __rte_aligned(8);
+	RTE_ATOMIC(uint64_t) raw __rte_aligned(8);
 	struct {
-		uint32_t head; /**< head position */
-		uint32_t tail; /**< tail position */
+		RTE_ATOMIC(uint32_t) head; /**< head position */
+		RTE_ATOMIC(uint32_t) tail; /**< tail position */
 	} pos;
 };
lib/ring/rte_ring_generic_pvt.h +0 -0
index 5acb6e5..ffb3654 100644

lib/ring/rte_ring_generic_pvt.h line 26

Quoted 5 lines
lib/ring/rte_ring_generic_pvt.h +1 -1
@@ -23,7 +23,8 @@
 	 * we need to wait for them to complete
 	 */
 	if (!single)
-		rte_wait_until_equal_32(&ht->tail, old_val, __ATOMIC_RELAXED);
+		rte_wait_until_equal_32((volatile uint32_t *)(uintptr_t)&ht->tail, old_val,

I suppose we do need that double type conversion only for atomic types right?

Quoted 159 lines
lib/ring/rte_ring_generic_pvt.h +1 -0
+			rte_memory_order_relaxed);
 	ht->tail = new_val;
 }
lib/ring/rte_ring_hts_elem_pvt.h +12 -10
index a8678d3..91f5eec 100644
@@ -10,6 +10,8 @@
 #ifndef _RTE_RING_HTS_ELEM_PVT_H_
 #define _RTE_RING_HTS_ELEM_PVT_H_
+#include <rte_stdatomic.h>
+
 /**
  * @file rte_ring_hts_elem_pvt.h
  * It is not recommended to include this file directly,
@@ -30,7 +32,7 @@
 	RTE_SET_USED(enqueue);
 	tail = old_tail + num;
-	__atomic_store_n(&ht->ht.pos.tail, tail, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&ht->ht.pos.tail, tail, rte_memory_order_release);
 }
 /**
@@ -44,7 +46,7 @@
 {
 	while (p->pos.head != p->pos.tail) {
 		rte_pause();
-		p->raw = __atomic_load_n(&ht->ht.raw, __ATOMIC_ACQUIRE);
+		p->raw = rte_atomic_load_explicit(&ht->ht.raw, rte_memory_order_acquire);
 	}
 }
@@ -61,7 +63,7 @@
 	const uint32_t capacity = r->capacity;
-	op.raw = __atomic_load_n(&r->hts_prod.ht.raw, __ATOMIC_ACQUIRE);
+	op.raw = rte_atomic_load_explicit(&r->hts_prod.ht.raw, rte_memory_order_acquire);
 	do {
 		/* Reset n to the initial burst count */
@@ -98,9 +100,9 @@
 	 *  - OOO reads of cons tail value
 	 *  - OOO copy of elems from the ring
 	 */
-	} while (__atomic_compare_exchange_n(&r->hts_prod.ht.raw,
-			&op.raw, np.raw,
-			0, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE) == 0);
+	} while (rte_atomic_compare_exchange_strong_explicit(&r->hts_prod.ht.raw,
+			(uint64_t *)(uintptr_t)&op.raw, np.raw,
+			rte_memory_order_acquire, rte_memory_order_acquire) == 0);
 	*old_head = op.pos.head;
 	return n;
@@ -117,7 +119,7 @@
 	uint32_t n;
 	union __rte_ring_hts_pos np, op;
-	op.raw = __atomic_load_n(&r->hts_cons.ht.raw, __ATOMIC_ACQUIRE);
+	op.raw = rte_atomic_load_explicit(&r->hts_cons.ht.raw, rte_memory_order_acquire);
 	/* move cons.head atomically */
 	do {
@@ -153,9 +155,9 @@
 	 *  - OOO reads of prod tail value
 	 *  - OOO copy of elems from the ring
 	 */
-	} while (__atomic_compare_exchange_n(&r->hts_cons.ht.raw,
-			&op.raw, np.raw,
-			0, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE) == 0);
+	} while (rte_atomic_compare_exchange_strong_explicit(&r->hts_cons.ht.raw,
+			(uint64_t *)(uintptr_t)&op.raw, np.raw,
+			rte_memory_order_acquire, rte_memory_order_acquire) == 0);
 	*old_head = op.pos.head;
 	return n;
lib/ring/rte_ring_peek_elem_pvt.h +3 -3
index bb0a7d5..b5f0822 100644
@@ -59,7 +59,7 @@
 	pos = tail + num;
 	ht->head = pos;
-	__atomic_store_n(&ht->tail, pos, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&ht->tail, pos, rte_memory_order_release);
 }
 /**
@@ -78,7 +78,7 @@
 	uint32_t n;
 	union __rte_ring_hts_pos p;
-	p.raw = __atomic_load_n(&ht->ht.raw, __ATOMIC_RELAXED);
+	p.raw = rte_atomic_load_explicit(&ht->ht.raw, rte_memory_order_relaxed);
 	n = p.pos.head - p.pos.tail;
 	RTE_ASSERT(n >= num);
@@ -104,7 +104,7 @@
 	p.pos.head = tail + num;
 	p.pos.tail = p.pos.head;
-	__atomic_store_n(&ht->ht.raw, p.raw, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&ht->ht.raw, p.raw, rte_memory_order_release);
 }
 /**
lib/ring/rte_ring_rts_elem_pvt.h +14 -13
index 7164213..1226503 100644
@@ -31,18 +31,19 @@
 	 * might preceded us, then don't update tail with new value.
 	 */
-	ot.raw = __atomic_load_n(&ht->tail.raw, __ATOMIC_ACQUIRE);
+	ot.raw = rte_atomic_load_explicit(&ht->tail.raw, rte_memory_order_acquire);
 	do {
 		/* on 32-bit systems we have to do atomic read here */
-		h.raw = __atomic_load_n(&ht->head.raw, __ATOMIC_RELAXED);
+		h.raw = rte_atomic_load_explicit(&ht->head.raw, rte_memory_order_relaxed);
 		nt.raw = ot.raw;
 		if (++nt.val.cnt == h.val.cnt)
 			nt.val.pos = h.val.pos;
-	} while (__atomic_compare_exchange_n(&ht->tail.raw, &ot.raw, nt.raw,
-			0, __ATOMIC_RELEASE, __ATOMIC_ACQUIRE) == 0);
+	} while (rte_atomic_compare_exchange_strong_explicit(&ht->tail.raw,
+			(uint64_t *)(uintptr_t)&ot.raw, nt.raw,
+			rte_memory_order_release, rte_memory_order_acquire) == 0);
 }
 /**
@@ -59,7 +60,7 @@
 	while (h->val.pos - ht->tail.val.pos > max) {
 		rte_pause();
-		h->raw = __atomic_load_n(&ht->head.raw, __ATOMIC_ACQUIRE);
+		h->raw = rte_atomic_load_explicit(&ht->head.raw, rte_memory_order_acquire);
 	}
 }
@@ -76,7 +77,7 @@
 	const uint32_t capacity = r->capacity;
-	oh.raw = __atomic_load_n(&r->rts_prod.head.raw, __ATOMIC_ACQUIRE);
+	oh.raw = rte_atomic_load_explicit(&r->rts_prod.head.raw, rte_memory_order_acquire);
 	do {
 		/* Reset n to the initial burst count */
@@ -113,9 +114,9 @@
 	 *  - OOO reads of cons tail value
 	 *  - OOO copy of elems to the ring
 	 */
-	} while (__atomic_compare_exchange_n(&r->rts_prod.head.raw,
-			&oh.raw, nh.raw,
-			0, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE) == 0);
+	} while (rte_atomic_compare_exchange_strong_explicit(&r->rts_prod.head.raw,
+			(uint64_t *)(uintptr_t)&oh.raw, nh.raw,
+			rte_memory_order_acquire, rte_memory_order_acquire) == 0);
 	*old_head = oh.val.pos;
 	return n;
@@ -132,7 +133,7 @@
 	uint32_t n;
 	union __rte_ring_rts_poscnt nh, oh;
-	oh.raw = __atomic_load_n(&r->rts_cons.head.raw, __ATOMIC_ACQUIRE);
+	oh.raw = rte_atomic_load_explicit(&r->rts_cons.head.raw, rte_memory_order_acquire);
 	/* move cons.head atomically */
 	do {
@@ -168,9 +169,9 @@
 	 *  - OOO reads of prod tail value
 	 *  - OOO copy of elems from the ring
 	 */
-	} while (__atomic_compare_exchange_n(&r->rts_cons.head.raw,
-			&oh.raw, nh.raw,
-			0, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE) == 0);
+	} while (rte_atomic_compare_exchange_strong_explicit(&r->rts_cons.head.raw,
+			(uint64_t *)(uintptr_t)&oh.raw, nh.raw,
+			rte_memory_order_acquire, rte_memory_order_acquire) == 0);
 	*old_head = oh.val.pos;
 	return n;
#55 lore raw

RE: [PATCH v2 19/19] ring: use rte optional stdatomic API

Quoted 129 lines · Konstantin Ananyev
Quoted 116 lines

17.10.2023 21:31, Tyler Retzlaff пишет:

Quoted 115 lines

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 drivers/net/mlx5/mlx5_hws_cnt.h   |  2 +-
 lib/ring/rte_ring_c11_pvt.h       | 33 +++++++++++++++++----------------
 lib/ring/rte_ring_core.h          | 10 +++++-----
 lib/ring/rte_ring_generic_pvt.h   |  3 ++-
 lib/ring/rte_ring_hts_elem_pvt.h  | 22 ++++++++++++----------
 lib/ring/rte_ring_peek_elem_pvt.h |  6 +++---
 lib/ring/rte_ring_rts_elem_pvt.h  | 27 ++++++++++++++-------------
 7 files changed, 54 insertions(+), 49 deletions(-)
drivers/net/mlx5/mlx5_hws_cnt.h +1 -1
index f462665..cc9ac10 100644
@@ -394,7 +394,7 @@ struct mlx5_hws_age_param {
 	__rte_ring_get_elem_addr(r, revert2head, sizeof(cnt_id_t), n,
 			&zcd->ptr1, &zcd->n1, &zcd->ptr2);
 	/* Update tail */
-	__atomic_store_n(&r->prod.tail, revert2head, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&r->prod.tail, revert2head, rte_memory_order_release);
 	return n;
 }
lib/ring/rte_ring_c11_pvt.h +17 -16
index f895950..f8be538 100644
@@ -22,9 +22,10 @@
 	 * we need to wait for them to complete
 	 */
 	if (!single)
-		rte_wait_until_equal_32(&ht->tail, old_val, __ATOMIC_RELAXED);
+		rte_wait_until_equal_32((volatile uint32_t *)(uintptr_t)&ht->tail, old_val,
+			rte_memory_order_relaxed);
-	__atomic_store_n(&ht->tail, new_val, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&ht->tail, new_val, rte_memory_order_release);
 }
 /**
@@ -61,19 +62,19 @@
 	unsigned int max = n;
 	int success;
-	*old_head = __atomic_load_n(&r->prod.head, __ATOMIC_RELAXED);
+	*old_head = rte_atomic_load_explicit(&r->prod.head, rte_memory_order_relaxed);
 	do {
 		/* Reset n to the initial burst count */
 		n = max;
 		/* Ensure the head is read before tail */
-		__atomic_thread_fence(__ATOMIC_ACQUIRE);
+		__atomic_thread_fence(rte_memory_order_acquire);
 		/* load-acquire synchronize with store-release of ht->tail
 		 * in update_tail.
 		 */
-		cons_tail = __atomic_load_n(&r->cons.tail,
-					__ATOMIC_ACQUIRE);
+		cons_tail = rte_atomic_load_explicit(&r->cons.tail,
+					rte_memory_order_acquire);
 		/* The subtraction is done between two unsigned 32bits value
 		 * (the result is always modulo 32 bits even if we have
@@ -95,10 +96,10 @@
 			r->prod.head = *new_head, success = 1;
 		else
 			/* on failure, *old_head is updated */
-			success = __atomic_compare_exchange_n(&r->prod.head,
+			success = rte_atomic_compare_exchange_strong_explicit(&r->prod.head,
 					old_head, *new_head,
-					0, __ATOMIC_RELAXED,
-					__ATOMIC_RELAXED);
+					rte_memory_order_relaxed,
+					rte_memory_order_relaxed);
 	} while (unlikely(success == 0));
 	return n;
 }
@@ -137,19 +138,19 @@
 	int success;
 	/* move cons.head atomically */
-	*old_head = __atomic_load_n(&r->cons.head, __ATOMIC_RELAXED);
+	*old_head = rte_atomic_load_explicit(&r->cons.head, rte_memory_order_relaxed);
 	do {
 		/* Restore n as it may change every loop */
 		n = max;
 		/* Ensure the head is read before tail */
-		__atomic_thread_fence(__ATOMIC_ACQUIRE);
+		__atomic_thread_fence(rte_memory_order_acquire);
 		/* this load-acquire synchronize with store-release of ht->tail
 		 * in update_tail.
 		 */
-		prod_tail = __atomic_load_n(&r->prod.tail,
-					__ATOMIC_ACQUIRE);
+		prod_tail = rte_atomic_load_explicit(&r->prod.tail,
+					rte_memory_order_acquire);
 		/* The subtraction is done between two unsigned 32bits value
 		 * (the result is always modulo 32 bits even if we have
@@ -170,10 +171,10 @@
 			r->cons.head = *new_head, success = 1;
 		else
 			/* on failure, *old_head will be updated */
-			success = __atomic_compare_exchange_n(&r->cons.head,
+			success = rte_atomic_compare_exchange_strong_explicit(&r->cons.head,
 							old_head, *new_head,
-							0, __ATOMIC_RELAXED,
-							__ATOMIC_RELAXED);
+							rte_memory_order_relaxed,
+							rte_memory_order_relaxed);
 	} while (unlikely(success == 0));
 	return n;
 }
lib/ring/rte_ring_core.h +0 -0
index 327fdcf..7a2b577 100644
@@ -67,7 +67,7 @@ enum rte_ring_sync_type {
  */
Quoted 8 lines

lib/ring/rte_ring_core.h enum rte_ring_sync_type { · line 70

Quoted 4 lines
lib/ring/rte_ring_core.h +1 -1
 struct rte_ring_headtail {
 	volatile uint32_t head;      /**< prod/consumer head. */
-	volatile uint32_t tail;      /**< prod/consumer tail. */
+	volatile RTE_ATOMIC(uint32_t) tail;      /**< prod/consumer tail. */

Probably a stupid q:
why we do need RTE_ATOMIC() around tail only?
Why head is not affected?

Quoted 6 lines

you have a good eye and this is a slightly common issue that i've seen and there appear to be some interesting things showing up.

the field being qualified has atomic operation performed on it the other field does not in the implementation. it may be an indication of a bug in the existing code or it may be intentional.

Hmm... but as I can see, we are doing similar operations on both head and tail. For head it would be: atomic_load(), then either atomic_store() or atomic_cas(). For tail it would be: atomic_load(), then atomic_store().
Or is that because of we missed atomic_store(&r->prod.head, ..., RELAXED) here:
static __rte_always_inline unsigned int __rte_ring_move_prod_head(struct rte_ring *r, unsigned int is_sp,

                unsigned int n, enum rte_ring_queue_behavior behavior,
                uint32_t *old_head, uint32_t *new_head,
                uint32_t *free_entries)
{
....
if (is_sp)
                        r->prod.head = *new_head, success = 1;

?

Quoted 264 lines

case 1. atomics should be used but they aren't.

there are fields in structures and variables that were accessed in a 'mixed' manner. that is in some instances __atomic_op_xxx was being used on them and in other instances not. sometimes it is the initialization case so it is probably okay, sometimes maybe not...

case 2. broader scope atomic operation, or we don't care if narrower
access is atomic.

e.g.

union {
   struct {
       uint32_t head;
       RTE_ATOMIC(uint32_t) tail;
    }
    RTE_ATOMIC(uint64_t) combined;
}

again, could be an indication of missing use of atomic, often the operation on the `combined' field consistently uses atomics but one of the head/tail fields will not be. on purpose? maybe if we are just doing == comparison?

my approach in this series prioritized no functional change. as a result if any of the above are real bugs, they stay real bugs but i have not changed the way the variables are accessed. if i were to change the code and start atomic specifying it has a risk of performance regression (for cases where it isn't a bug) because specifying would result in the compiler code generating for strongest ordering seq_cst for accesses that are not using atomic generic functions that specify ordering.

there is another case which comes up half a dozen times or so that is also concerning to me, but i would need the maintainers of the code to adapt the code to be correct or maybe it is okay...

case 3. qualification discard .. is the existing code really okay?

e.g.

atomic_compare_exchange(*object, *expected, desired, ...)

the issue is with the specification of the memory aliased by expected. gcc doesn't complain or enforce discarding of qualification when using builtin intrinsics. the result is that if expected is an atomic type it may be accessed in a non-atomic manner by the code generated for the atomic operation.

again, i have chosen to maintain existing behavior by casting away the qualification if present on the expected argument.

i feel that in terms of mutating the source tree it is best to separate conversion to atomic specified/qualified types into this separate series and then follow up with additional changes that may have functional/performance impact if not for any other reason that it narrows where you have to look if there is a change. certainly conversion to atomics has made these cases far easier to spot in the code.

finally in terms of most of the toolchain/targets all of this is pretty moot because most of them are defaulting to enable_stdatomics=false so most likely if there are problems they will manifest on windows built with msvc only.

thoughts?

Quoted 197 lines
Quoted 29 lines
lib/ring/rte_ring_core.h +4 -4
 	union {
 		/** sync type of prod/cons */
 		enum rte_ring_sync_type sync_type;
@@ -78,7 +78,7 @@ struct rte_ring_headtail {
 union __rte_ring_rts_poscnt {
 	/** raw 8B value to read/write *cnt* and *pos* as one atomic op */
-	uint64_t raw __rte_aligned(8);
+	RTE_ATOMIC(uint64_t) raw __rte_aligned(8);
 	struct {
 		uint32_t cnt; /**< head/tail reference counter */
 		uint32_t pos; /**< head/tail position */
@@ -94,10 +94,10 @@ struct rte_ring_rts_headtail {
 union __rte_ring_hts_pos {
 	/** raw 8B value to read/write *head* and *tail* as one atomic op */
-	uint64_t raw __rte_aligned(8);
+	RTE_ATOMIC(uint64_t) raw __rte_aligned(8);
 	struct {
-		uint32_t head; /**< head position */
-		uint32_t tail; /**< tail position */
+		RTE_ATOMIC(uint32_t) head; /**< head position */
+		RTE_ATOMIC(uint32_t) tail; /**< tail position */
 	} pos;
 };
lib/ring/rte_ring_generic_pvt.h +0 -0
index 5acb6e5..ffb3654 100644

lib/ring/rte_ring_generic_pvt.h line 26

Quoted 5 lines
lib/ring/rte_ring_generic_pvt.h +1 -1
@@ -23,7 +23,8 @@
 	 * we need to wait for them to complete
 	 */
 	if (!single)
-		rte_wait_until_equal_32(&ht->tail, old_val, __ATOMIC_RELAXED);
+		rte_wait_until_equal_32((volatile uint32_t *)(uintptr_t)&ht->tail, old_val,

I suppose we do need that double type conversion only for atomic types right?

Quoted 159 lines
lib/ring/rte_ring_generic_pvt.h +1 -0
+			rte_memory_order_relaxed);
 	ht->tail = new_val;
 }
lib/ring/rte_ring_hts_elem_pvt.h +12 -10
index a8678d3..91f5eec 100644
@@ -10,6 +10,8 @@
 #ifndef _RTE_RING_HTS_ELEM_PVT_H_
 #define _RTE_RING_HTS_ELEM_PVT_H_
+#include <rte_stdatomic.h>
+
 /**
  * @file rte_ring_hts_elem_pvt.h
  * It is not recommended to include this file directly,
@@ -30,7 +32,7 @@
 	RTE_SET_USED(enqueue);
 	tail = old_tail + num;
-	__atomic_store_n(&ht->ht.pos.tail, tail, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&ht->ht.pos.tail, tail, rte_memory_order_release);
 }
 /**
@@ -44,7 +46,7 @@
 {
 	while (p->pos.head != p->pos.tail) {
 		rte_pause();
-		p->raw = __atomic_load_n(&ht->ht.raw, __ATOMIC_ACQUIRE);
+		p->raw = rte_atomic_load_explicit(&ht->ht.raw, rte_memory_order_acquire);
 	}
 }
@@ -61,7 +63,7 @@
 	const uint32_t capacity = r->capacity;
-	op.raw = __atomic_load_n(&r->hts_prod.ht.raw, __ATOMIC_ACQUIRE);
+	op.raw = rte_atomic_load_explicit(&r->hts_prod.ht.raw, rte_memory_order_acquire);
 	do {
 		/* Reset n to the initial burst count */
@@ -98,9 +100,9 @@
 	 *  - OOO reads of cons tail value
 	 *  - OOO copy of elems from the ring
 	 */
-	} while (__atomic_compare_exchange_n(&r->hts_prod.ht.raw,
-			&op.raw, np.raw,
-			0, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE) == 0);
+	} while (rte_atomic_compare_exchange_strong_explicit(&r->hts_prod.ht.raw,
+			(uint64_t *)(uintptr_t)&op.raw, np.raw,
+			rte_memory_order_acquire, rte_memory_order_acquire) == 0);
 	*old_head = op.pos.head;
 	return n;
@@ -117,7 +119,7 @@
 	uint32_t n;
 	union __rte_ring_hts_pos np, op;
-	op.raw = __atomic_load_n(&r->hts_cons.ht.raw, __ATOMIC_ACQUIRE);
+	op.raw = rte_atomic_load_explicit(&r->hts_cons.ht.raw, rte_memory_order_acquire);
 	/* move cons.head atomically */
 	do {
@@ -153,9 +155,9 @@
 	 *  - OOO reads of prod tail value
 	 *  - OOO copy of elems from the ring
 	 */
-	} while (__atomic_compare_exchange_n(&r->hts_cons.ht.raw,
-			&op.raw, np.raw,
-			0, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE) == 0);
+	} while (rte_atomic_compare_exchange_strong_explicit(&r->hts_cons.ht.raw,
+			(uint64_t *)(uintptr_t)&op.raw, np.raw,
+			rte_memory_order_acquire, rte_memory_order_acquire) == 0);
 	*old_head = op.pos.head;
 	return n;
lib/ring/rte_ring_peek_elem_pvt.h +3 -3
index bb0a7d5..b5f0822 100644
@@ -59,7 +59,7 @@
 	pos = tail + num;
 	ht->head = pos;
-	__atomic_store_n(&ht->tail, pos, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&ht->tail, pos, rte_memory_order_release);
 }
 /**
@@ -78,7 +78,7 @@
 	uint32_t n;
 	union __rte_ring_hts_pos p;
-	p.raw = __atomic_load_n(&ht->ht.raw, __ATOMIC_RELAXED);
+	p.raw = rte_atomic_load_explicit(&ht->ht.raw, rte_memory_order_relaxed);
 	n = p.pos.head - p.pos.tail;
 	RTE_ASSERT(n >= num);
@@ -104,7 +104,7 @@
 	p.pos.head = tail + num;
 	p.pos.tail = p.pos.head;
-	__atomic_store_n(&ht->ht.raw, p.raw, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&ht->ht.raw, p.raw, rte_memory_order_release);
 }
 /**
lib/ring/rte_ring_rts_elem_pvt.h +14 -13
index 7164213..1226503 100644
@@ -31,18 +31,19 @@
 	 * might preceded us, then don't update tail with new value.
 	 */
-	ot.raw = __atomic_load_n(&ht->tail.raw, __ATOMIC_ACQUIRE);
+	ot.raw = rte_atomic_load_explicit(&ht->tail.raw, rte_memory_order_acquire);
 	do {
 		/* on 32-bit systems we have to do atomic read here */
-		h.raw = __atomic_load_n(&ht->head.raw, __ATOMIC_RELAXED);
+		h.raw = rte_atomic_load_explicit(&ht->head.raw, rte_memory_order_relaxed);
 		nt.raw = ot.raw;
 		if (++nt.val.cnt == h.val.cnt)
 			nt.val.pos = h.val.pos;
-	} while (__atomic_compare_exchange_n(&ht->tail.raw, &ot.raw, nt.raw,
-			0, __ATOMIC_RELEASE, __ATOMIC_ACQUIRE) == 0);
+	} while (rte_atomic_compare_exchange_strong_explicit(&ht->tail.raw,
+			(uint64_t *)(uintptr_t)&ot.raw, nt.raw,
+			rte_memory_order_release, rte_memory_order_acquire) == 0);
 }
 /**
@@ -59,7 +60,7 @@
 	while (h->val.pos - ht->tail.val.pos > max) {
 		rte_pause();
-		h->raw = __atomic_load_n(&ht->head.raw, __ATOMIC_ACQUIRE);
+		h->raw = rte_atomic_load_explicit(&ht->head.raw, rte_memory_order_acquire);
 	}
 }
@@ -76,7 +77,7 @@
 	const uint32_t capacity = r->capacity;
-	oh.raw = __atomic_load_n(&r->rts_prod.head.raw, __ATOMIC_ACQUIRE);
+	oh.raw = rte_atomic_load_explicit(&r->rts_prod.head.raw, rte_memory_order_acquire);
 	do {
 		/* Reset n to the initial burst count */
@@ -113,9 +114,9 @@
 	 *  - OOO reads of cons tail value
 	 *  - OOO copy of elems to the ring
 	 */
-	} while (__atomic_compare_exchange_n(&r->rts_prod.head.raw,
-			&oh.raw, nh.raw,
-			0, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE) == 0);
+	} while (rte_atomic_compare_exchange_strong_explicit(&r->rts_prod.head.raw,
+			(uint64_t *)(uintptr_t)&oh.raw, nh.raw,
+			rte_memory_order_acquire, rte_memory_order_acquire) == 0);
 	*old_head = oh.val.pos;
 	return n;
@@ -132,7 +133,7 @@
 	uint32_t n;
 	union __rte_ring_rts_poscnt nh, oh;
-	oh.raw = __atomic_load_n(&r->rts_cons.head.raw, __ATOMIC_ACQUIRE);
+	oh.raw = rte_atomic_load_explicit(&r->rts_cons.head.raw, rte_memory_order_acquire);
 	/* move cons.head atomically */
 	do {
@@ -168,9 +169,9 @@
 	 *  - OOO reads of prod tail value
 	 *  - OOO copy of elems from the ring
 	 */
-	} while (__atomic_compare_exchange_n(&r->rts_cons.head.raw,
-			&oh.raw, nh.raw,
-			0, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE) == 0);
+	} while (rte_atomic_compare_exchange_strong_explicit(&r->rts_cons.head.raw,
+			(uint64_t *)(uintptr_t)&oh.raw, nh.raw,
+			rte_memory_order_acquire, rte_memory_order_acquire) == 0);
 	*old_head = oh.val.pos;
 	return n;
#56 lore raw

Re: [PATCH v2 19/19] ring: use rte optional stdatomic API

Quoted 150 lines · Konstantin Ananyev
Quoted 129 lines · Konstantin Ananyev
Quoted 116 lines

17.10.2023 21:31, Tyler Retzlaff пишет:

Quoted 115 lines

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 drivers/net/mlx5/mlx5_hws_cnt.h   |  2 +-
 lib/ring/rte_ring_c11_pvt.h       | 33 +++++++++++++++++----------------
 lib/ring/rte_ring_core.h          | 10 +++++-----
 lib/ring/rte_ring_generic_pvt.h   |  3 ++-
 lib/ring/rte_ring_hts_elem_pvt.h  | 22 ++++++++++++----------
 lib/ring/rte_ring_peek_elem_pvt.h |  6 +++---
 lib/ring/rte_ring_rts_elem_pvt.h  | 27 ++++++++++++++-------------
 7 files changed, 54 insertions(+), 49 deletions(-)
drivers/net/mlx5/mlx5_hws_cnt.h +1 -1
index f462665..cc9ac10 100644
@@ -394,7 +394,7 @@ struct mlx5_hws_age_param {
 	__rte_ring_get_elem_addr(r, revert2head, sizeof(cnt_id_t), n,
 			&zcd->ptr1, &zcd->n1, &zcd->ptr2);
 	/* Update tail */
-	__atomic_store_n(&r->prod.tail, revert2head, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&r->prod.tail, revert2head, rte_memory_order_release);
 	return n;
 }
lib/ring/rte_ring_c11_pvt.h +17 -16
index f895950..f8be538 100644
@@ -22,9 +22,10 @@
 	 * we need to wait for them to complete
 	 */
 	if (!single)
-		rte_wait_until_equal_32(&ht->tail, old_val, __ATOMIC_RELAXED);
+		rte_wait_until_equal_32((volatile uint32_t *)(uintptr_t)&ht->tail, old_val,
+			rte_memory_order_relaxed);
-	__atomic_store_n(&ht->tail, new_val, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&ht->tail, new_val, rte_memory_order_release);
 }
 /**
@@ -61,19 +62,19 @@
 	unsigned int max = n;
 	int success;
-	*old_head = __atomic_load_n(&r->prod.head, __ATOMIC_RELAXED);
+	*old_head = rte_atomic_load_explicit(&r->prod.head, rte_memory_order_relaxed);
 	do {
 		/* Reset n to the initial burst count */
 		n = max;
 		/* Ensure the head is read before tail */
-		__atomic_thread_fence(__ATOMIC_ACQUIRE);
+		__atomic_thread_fence(rte_memory_order_acquire);
 		/* load-acquire synchronize with store-release of ht->tail
 		 * in update_tail.
 		 */
-		cons_tail = __atomic_load_n(&r->cons.tail,
-					__ATOMIC_ACQUIRE);
+		cons_tail = rte_atomic_load_explicit(&r->cons.tail,
+					rte_memory_order_acquire);
 		/* The subtraction is done between two unsigned 32bits value
 		 * (the result is always modulo 32 bits even if we have
@@ -95,10 +96,10 @@
 			r->prod.head = *new_head, success = 1;
 		else
 			/* on failure, *old_head is updated */
-			success = __atomic_compare_exchange_n(&r->prod.head,
+			success = rte_atomic_compare_exchange_strong_explicit(&r->prod.head,
 					old_head, *new_head,
-					0, __ATOMIC_RELAXED,
-					__ATOMIC_RELAXED);
+					rte_memory_order_relaxed,
+					rte_memory_order_relaxed);
 	} while (unlikely(success == 0));
 	return n;
 }
@@ -137,19 +138,19 @@
 	int success;
 	/* move cons.head atomically */
-	*old_head = __atomic_load_n(&r->cons.head, __ATOMIC_RELAXED);
+	*old_head = rte_atomic_load_explicit(&r->cons.head, rte_memory_order_relaxed);
 	do {
 		/* Restore n as it may change every loop */
 		n = max;
 		/* Ensure the head is read before tail */
-		__atomic_thread_fence(__ATOMIC_ACQUIRE);
+		__atomic_thread_fence(rte_memory_order_acquire);
 		/* this load-acquire synchronize with store-release of ht->tail
 		 * in update_tail.
 		 */
-		prod_tail = __atomic_load_n(&r->prod.tail,
-					__ATOMIC_ACQUIRE);
+		prod_tail = rte_atomic_load_explicit(&r->prod.tail,
+					rte_memory_order_acquire);
 		/* The subtraction is done between two unsigned 32bits value
 		 * (the result is always modulo 32 bits even if we have
@@ -170,10 +171,10 @@
 			r->cons.head = *new_head, success = 1;
 		else
 			/* on failure, *old_head will be updated */
-			success = __atomic_compare_exchange_n(&r->cons.head,
+			success = rte_atomic_compare_exchange_strong_explicit(&r->cons.head,
 							old_head, *new_head,
-							0, __ATOMIC_RELAXED,
-							__ATOMIC_RELAXED);
+							rte_memory_order_relaxed,
+							rte_memory_order_relaxed);
 	} while (unlikely(success == 0));
 	return n;
 }
lib/ring/rte_ring_core.h +0 -0
index 327fdcf..7a2b577 100644
@@ -67,7 +67,7 @@ enum rte_ring_sync_type {
  */
Quoted 8 lines

lib/ring/rte_ring_core.h enum rte_ring_sync_type { · line 70

Quoted 4 lines
lib/ring/rte_ring_core.h +1 -1
 struct rte_ring_headtail {
 	volatile uint32_t head;      /**< prod/consumer head. */
-	volatile uint32_t tail;      /**< prod/consumer tail. */
+	volatile RTE_ATOMIC(uint32_t) tail;      /**< prod/consumer tail. */

Probably a stupid q:
why we do need RTE_ATOMIC() around tail only?
Why head is not affected?

Quoted 6 lines

you have a good eye and this is a slightly common issue that i've seen and there appear to be some interesting things showing up.

the field being qualified has atomic operation performed on it the other field does not in the implementation. it may be an indication of a bug in the existing code or it may be intentional.

Hmm... but as I can see, we are doing similar operations on both head and tail. For head it would be: atomic_load(), then either atomic_store() or atomic_cas(). For tail it would be: atomic_load(), then atomic_store().
Or is that because of we missed atomic_store(&r->prod.head, ..., RELAXED) here:
static __rte_always_inline unsigned int __rte_ring_move_prod_head(struct rte_ring *r, unsigned int is_sp,

                unsigned int n, enum rte_ring_queue_behavior behavior,
                uint32_t *old_head, uint32_t *new_head,
                uint32_t *free_entries)
{
....
if (is_sp)
                        r->prod.head = *new_head, success = 1;
Quoted 1 line

?

for this instance you are correct, i need to get an understanding of why this builds successfully because it shouldn't. that it doesn't fail probably isn't harmful but since this is a public header the structure is visible it's best to have it carry the correct RTE_ATOMIC(T).

i'll reply back with what i find.

thanks

Quoted 68 lines
Quoted 68 lines

case 1. atomics should be used but they aren't.

there are fields in structures and variables that were accessed in a 'mixed' manner. that is in some instances __atomic_op_xxx was being used on them and in other instances not. sometimes it is the initialization case so it is probably okay, sometimes maybe not...

case 2. broader scope atomic operation, or we don't care if narrower
access is atomic.

e.g.

union {
   struct {
       uint32_t head;
       RTE_ATOMIC(uint32_t) tail;
    }
    RTE_ATOMIC(uint64_t) combined;
}

again, could be an indication of missing use of atomic, often the operation on the `combined' field consistently uses atomics but one of the head/tail fields will not be. on purpose? maybe if we are just doing == comparison?

my approach in this series prioritized no functional change. as a result if any of the above are real bugs, they stay real bugs but i have not changed the way the variables are accessed. if i were to change the code and start atomic specifying it has a risk of performance regression (for cases where it isn't a bug) because specifying would result in the compiler code generating for strongest ordering seq_cst for accesses that are not using atomic generic functions that specify ordering.

there is another case which comes up half a dozen times or so that is also concerning to me, but i would need the maintainers of the code to adapt the code to be correct or maybe it is okay...

case 3. qualification discard .. is the existing code really okay?

e.g.

atomic_compare_exchange(*object, *expected, desired, ...)

the issue is with the specification of the memory aliased by expected. gcc doesn't complain or enforce discarding of qualification when using builtin intrinsics. the result is that if expected is an atomic type it may be accessed in a non-atomic manner by the code generated for the atomic operation.

again, i have chosen to maintain existing behavior by casting away the qualification if present on the expected argument.

i feel that in terms of mutating the source tree it is best to separate conversion to atomic specified/qualified types into this separate series and then follow up with additional changes that may have functional/performance impact if not for any other reason that it narrows where you have to look if there is a change. certainly conversion to atomics has made these cases far easier to spot in the code.

finally in terms of most of the toolchain/targets all of this is pretty moot because most of them are defaulting to enable_stdatomics=false so most likely if there are problems they will manifest on windows built with msvc only.

thoughts?

#57 lore raw

Re: [PATCH v2 19/19] ring: use rte optional stdatomic API

Quoted 151 lines · Tyler Retzlaff
Quoted 150 lines · Konstantin Ananyev
Quoted 129 lines · Konstantin Ananyev
Quoted 116 lines

17.10.2023 21:31, Tyler Retzlaff пишет:

Quoted 115 lines

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 drivers/net/mlx5/mlx5_hws_cnt.h   |  2 +-
 lib/ring/rte_ring_c11_pvt.h       | 33 +++++++++++++++++----------------
 lib/ring/rte_ring_core.h          | 10 +++++-----
 lib/ring/rte_ring_generic_pvt.h   |  3 ++-
 lib/ring/rte_ring_hts_elem_pvt.h  | 22 ++++++++++++----------
 lib/ring/rte_ring_peek_elem_pvt.h |  6 +++---
 lib/ring/rte_ring_rts_elem_pvt.h  | 27 ++++++++++++++-------------
 7 files changed, 54 insertions(+), 49 deletions(-)
drivers/net/mlx5/mlx5_hws_cnt.h +1 -1
index f462665..cc9ac10 100644
@@ -394,7 +394,7 @@ struct mlx5_hws_age_param {
 	__rte_ring_get_elem_addr(r, revert2head, sizeof(cnt_id_t), n,
 			&zcd->ptr1, &zcd->n1, &zcd->ptr2);
 	/* Update tail */
-	__atomic_store_n(&r->prod.tail, revert2head, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&r->prod.tail, revert2head, rte_memory_order_release);
 	return n;
 }
lib/ring/rte_ring_c11_pvt.h +17 -16
index f895950..f8be538 100644
@@ -22,9 +22,10 @@
 	 * we need to wait for them to complete
 	 */
 	if (!single)
-		rte_wait_until_equal_32(&ht->tail, old_val, __ATOMIC_RELAXED);
+		rte_wait_until_equal_32((volatile uint32_t *)(uintptr_t)&ht->tail, old_val,
+			rte_memory_order_relaxed);
-	__atomic_store_n(&ht->tail, new_val, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&ht->tail, new_val, rte_memory_order_release);
 }
 /**
@@ -61,19 +62,19 @@
 	unsigned int max = n;
 	int success;
-	*old_head = __atomic_load_n(&r->prod.head, __ATOMIC_RELAXED);
+	*old_head = rte_atomic_load_explicit(&r->prod.head, rte_memory_order_relaxed);
 	do {
 		/* Reset n to the initial burst count */
 		n = max;
 		/* Ensure the head is read before tail */
-		__atomic_thread_fence(__ATOMIC_ACQUIRE);
+		__atomic_thread_fence(rte_memory_order_acquire);
 		/* load-acquire synchronize with store-release of ht->tail
 		 * in update_tail.
 		 */
-		cons_tail = __atomic_load_n(&r->cons.tail,
-					__ATOMIC_ACQUIRE);
+		cons_tail = rte_atomic_load_explicit(&r->cons.tail,
+					rte_memory_order_acquire);
 		/* The subtraction is done between two unsigned 32bits value
 		 * (the result is always modulo 32 bits even if we have
@@ -95,10 +96,10 @@
 			r->prod.head = *new_head, success = 1;
 		else
 			/* on failure, *old_head is updated */
-			success = __atomic_compare_exchange_n(&r->prod.head,
+			success = rte_atomic_compare_exchange_strong_explicit(&r->prod.head,
 					old_head, *new_head,
-					0, __ATOMIC_RELAXED,
-					__ATOMIC_RELAXED);
+					rte_memory_order_relaxed,
+					rte_memory_order_relaxed);
 	} while (unlikely(success == 0));
 	return n;
 }
@@ -137,19 +138,19 @@
 	int success;
 	/* move cons.head atomically */
-	*old_head = __atomic_load_n(&r->cons.head, __ATOMIC_RELAXED);
+	*old_head = rte_atomic_load_explicit(&r->cons.head, rte_memory_order_relaxed);
 	do {
 		/* Restore n as it may change every loop */
 		n = max;
 		/* Ensure the head is read before tail */
-		__atomic_thread_fence(__ATOMIC_ACQUIRE);
+		__atomic_thread_fence(rte_memory_order_acquire);
 		/* this load-acquire synchronize with store-release of ht->tail
 		 * in update_tail.
 		 */
-		prod_tail = __atomic_load_n(&r->prod.tail,
-					__ATOMIC_ACQUIRE);
+		prod_tail = rte_atomic_load_explicit(&r->prod.tail,
+					rte_memory_order_acquire);
 		/* The subtraction is done between two unsigned 32bits value
 		 * (the result is always modulo 32 bits even if we have
@@ -170,10 +171,10 @@
 			r->cons.head = *new_head, success = 1;
 		else
 			/* on failure, *old_head will be updated */
-			success = __atomic_compare_exchange_n(&r->cons.head,
+			success = rte_atomic_compare_exchange_strong_explicit(&r->cons.head,
 							old_head, *new_head,
-							0, __ATOMIC_RELAXED,
-							__ATOMIC_RELAXED);
+							rte_memory_order_relaxed,
+							rte_memory_order_relaxed);
 	} while (unlikely(success == 0));
 	return n;
 }
lib/ring/rte_ring_core.h +0 -0
index 327fdcf..7a2b577 100644
@@ -67,7 +67,7 @@ enum rte_ring_sync_type {
  */
Quoted 8 lines

lib/ring/rte_ring_core.h enum rte_ring_sync_type { · line 70

Quoted 4 lines
lib/ring/rte_ring_core.h +1 -1
 struct rte_ring_headtail {
 	volatile uint32_t head;      /**< prod/consumer head. */
-	volatile uint32_t tail;      /**< prod/consumer tail. */
+	volatile RTE_ATOMIC(uint32_t) tail;      /**< prod/consumer tail. */

Probably a stupid q:
why we do need RTE_ATOMIC() around tail only?
Why head is not affected?

Quoted 6 lines

you have a good eye and this is a slightly common issue that i've seen and there appear to be some interesting things showing up.

the field being qualified has atomic operation performed on it the other field does not in the implementation. it may be an indication of a bug in the existing code or it may be intentional.

Hmm... but as I can see, we are doing similar operations on both head and tail. For head it would be: atomic_load(), then either atomic_store() or atomic_cas(). For tail it would be: atomic_load(), then atomic_store().
Or is that because of we missed atomic_store(&r->prod.head, ..., RELAXED) here:
static __rte_always_inline unsigned int __rte_ring_move_prod_head(struct rte_ring *r, unsigned int is_sp,

                unsigned int n, enum rte_ring_queue_behavior behavior,
                uint32_t *old_head, uint32_t *new_head,
                uint32_t *free_entries)
{
....
if (is_sp)
                        r->prod.head = *new_head, success = 1;
Quoted 8 lines
Quoted 1 line

?

for this instance you are correct, i need to get an understanding of why this builds successfully because it shouldn't. that it doesn't fail probably isn't harmful but since this is a public header the structure is visible it's best to have it carry the correct RTE_ATOMIC(T).

i'll reply back with what i find.

okay, circling back to answer. simply put we don't seem to have any CI or convenient way to configure a build with RTE_USE_C11_MEM_MODEL so if we don't build rte_ring_c11_pvt.h that field is not accessed with a standard atomic generic function.

this explains why it doesn't fail to build and why it doesn't stick out as needing to be properly specified. which should be done.

i'll submit a new series that catches the other places the series missed when using RTE_USE_C11_MEM_MODEL

thanks

Quoted 71 lines

thanks

Quoted 68 lines
Quoted 68 lines

case 1. atomics should be used but they aren't.

there are fields in structures and variables that were accessed in a 'mixed' manner. that is in some instances __atomic_op_xxx was being used on them and in other instances not. sometimes it is the initialization case so it is probably okay, sometimes maybe not...

case 2. broader scope atomic operation, or we don't care if narrower
access is atomic.

e.g.

union {
   struct {
       uint32_t head;
       RTE_ATOMIC(uint32_t) tail;
    }
    RTE_ATOMIC(uint64_t) combined;
}

again, could be an indication of missing use of atomic, often the operation on the `combined' field consistently uses atomics but one of the head/tail fields will not be. on purpose? maybe if we are just doing == comparison?

my approach in this series prioritized no functional change. as a result if any of the above are real bugs, they stay real bugs but i have not changed the way the variables are accessed. if i were to change the code and start atomic specifying it has a risk of performance regression (for cases where it isn't a bug) because specifying would result in the compiler code generating for strongest ordering seq_cst for accesses that are not using atomic generic functions that specify ordering.

there is another case which comes up half a dozen times or so that is also concerning to me, but i would need the maintainers of the code to adapt the code to be correct or maybe it is okay...

case 3. qualification discard .. is the existing code really okay?

e.g.

atomic_compare_exchange(*object, *expected, desired, ...)

the issue is with the specification of the memory aliased by expected. gcc doesn't complain or enforce discarding of qualification when using builtin intrinsics. the result is that if expected is an atomic type it may be accessed in a non-atomic manner by the code generated for the atomic operation.

again, i have chosen to maintain existing behavior by casting away the qualification if present on the expected argument.

i feel that in terms of mutating the source tree it is best to separate conversion to atomic specified/qualified types into this separate series and then follow up with additional changes that may have functional/performance impact if not for any other reason that it narrows where you have to look if there is a change. certainly conversion to atomics has made these cases far easier to spot in the code.

finally in terms of most of the toolchain/targets all of this is pretty moot because most of them are defaulting to enable_stdatomics=false so most likely if there are problems they will manifest on windows built with msvc only.

thoughts?

#58 lore raw
In reply to Tyler Retzlaff
Quoted 7 lines · On Tue, 17 Oct 2023 13:30:58 -0700 Tyler Retzlaff…

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API.

v2:
* add #include <rte_stdatomic.h> to rte_mbuf_core.h
* remove first two patches which were fixes that have been merged in another series

Looks good.
It does look like lots of places are doing per-cpu statistics and maybe a set of helpers for that would make it simpler.

Linux has percpu_counter_XXX helpers and NET_INC_STATS() macro.

#59 lore raw

v3 cover use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API.

v3:
* add missing atomic specification on head variable
in struct rte_ring_headtail
* adapt to use rte_atomic_xxx stdatomic API in rte_ring_c11_pvt.h
* split comma operator statement into 2 statements

v2:
* add #include <rte_stdatomic.h> to rte_mbuf_core.h
* remove first two patches which were fixes that have been merged in another series

Tyler Retzlaff (19):
power: use rte optional stdatomic API
bbdev: use rte optional stdatomic API
eal: use rte optional stdatomic API
eventdev: use rte optional stdatomic API
gpudev: use rte optional stdatomic API
ipsec: use rte optional stdatomic API
mbuf: use rte optional stdatomic API
mempool: use rte optional stdatomic API
rcu: use rte optional stdatomic API
pdump: use rte optional stdatomic API
stack: use rte optional stdatomic API
telemetry: use rte optional stdatomic API
vhost: use rte optional stdatomic API
cryptodev: use rte optional stdatomic API
distributor: use rte optional stdatomic API
ethdev: use rte optional stdatomic API
hash: use rte optional stdatomic API
timer: use rte optional stdatomic API
ring: use rte optional stdatomic API

 drivers/event/cnxk/cnxk_tim_worker.h   |   4 +-
 drivers/net/mlx5/mlx5_hws_cnt.h        |   4 +-
 lib/bbdev/rte_bbdev.c                  |   6 +-
 lib/bbdev/rte_bbdev.h                  |   2 +-
 lib/cryptodev/rte_cryptodev.c          |  22 +++---
 lib/cryptodev/rte_cryptodev.h          |  16 ++---
 lib/distributor/distributor_private.h  |   4 +-
 lib/distributor/rte_distributor.c      |  54 +++++++--------
 lib/eal/common/eal_common_launch.c     |  10 +--
 lib/eal/common/eal_common_mcfg.c       |   2 +-
 lib/eal/common/eal_common_proc.c       |  14 ++--
 lib/eal/common/eal_common_thread.c     |  26 +++----
 lib/eal/common/eal_common_trace.c      |   8 +--
 lib/eal/common/eal_common_trace_ctf.c  |   4 +-
 lib/eal/common/eal_memcfg.h            |   2 +-
 lib/eal/common/eal_private.h           |   4 +-
 lib/eal/common/eal_trace.h             |   4 +-
 lib/eal/common/rte_service.c           | 122 ++++++++++++++++-----------------
 lib/eal/freebsd/eal.c                  |  20 +++---
 lib/eal/include/rte_epoll.h            |   3 +-
 lib/eal/linux/eal.c                    |  26 +++----
 lib/eal/linux/eal_interrupts.c         |  42 ++++++------
 lib/eal/ppc/include/rte_atomic.h       |   6 +-
 lib/eal/windows/rte_thread.c           |   8 ++-
 lib/ethdev/ethdev_driver.h             |  16 ++---
 lib/ethdev/ethdev_private.c            |   6 +-
 lib/ethdev/rte_ethdev.c                |  24 +++----
 lib/ethdev/rte_ethdev.h                |  16 ++---
 lib/ethdev/rte_ethdev_core.h           |   2 +-
 lib/eventdev/rte_event_timer_adapter.c |  66 +++++++++---------
 lib/eventdev/rte_event_timer_adapter.h |   2 +-
 lib/gpudev/gpudev.c                    |   6 +-
 lib/gpudev/gpudev_driver.h             |   2 +-
 lib/hash/rte_cuckoo_hash.c             | 116 +++++++++++++++----------------
 lib/hash/rte_cuckoo_hash.h             |   6 +-
 lib/ipsec/ipsec_sqn.h                  |   2 +-
 lib/ipsec/sa.h                         |   2 +-
 lib/mbuf/rte_mbuf.h                    |  20 +++---
 lib/mbuf/rte_mbuf_core.h               |   5 +-
 lib/mempool/rte_mempool.h              |   4 +-
 lib/pdump/rte_pdump.c                  |  14 ++--
 lib/pdump/rte_pdump.h                  |   8 +--
 lib/power/power_acpi_cpufreq.c         |  33 ++++-----
 lib/power/power_cppc_cpufreq.c         |  25 +++----
 lib/power/power_pstate_cpufreq.c       |  31 +++++----
 lib/rcu/rte_rcu_qsbr.c                 |  48 ++++++-------
 lib/rcu/rte_rcu_qsbr.h                 |  68 +++++++++---------
 lib/ring/rte_ring_c11_pvt.h            |  47 +++++++------
 lib/ring/rte_ring_core.h               |  12 ++--
 lib/ring/rte_ring_generic_pvt.h        |  16 +++--
 lib/ring/rte_ring_hts_elem_pvt.h       |  22 +++---
 lib/ring/rte_ring_peek_elem_pvt.h      |   6 +-
 lib/ring/rte_ring_rts_elem_pvt.h       |  27 ++++----
 lib/stack/rte_stack.h                  |   2 +-
 lib/stack/rte_stack_lf_c11.h           |  24 +++----
 lib/stack/rte_stack_lf_generic.h       |  18 ++---
 lib/telemetry/telemetry.c              |  18 ++---
 lib/timer/rte_timer.c                  |  50 +++++++-------
 lib/timer/rte_timer.h                  |   6 +-
 lib/vhost/vdpa.c                       |   3 +-
 lib/vhost/vhost.c                      |  42 ++++++------
 lib/vhost/vhost.h                      |  39 ++++++-----
 lib/vhost/vhost_user.c                 |   6 +-
 lib/vhost/virtio_net.c                 |  58 +++++++++-------
 lib/vhost/virtio_net_ctrl.c            |   6 +-
 65 files changed, 684 insertions(+), 653 deletions(-)
Signature · 1 line
1.8.3.1
#60 lore raw

PATCH v3 1/19 power: use rte optional stdatomic API

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/power/power_acpi_cpufreq.c   | 33 +++++++++++++++++----------------
 lib/power/power_cppc_cpufreq.c   | 25 +++++++++++++------------
 lib/power/power_pstate_cpufreq.c | 31 ++++++++++++++++---------------
 3 files changed, 46 insertions(+), 43 deletions(-)
lib/power/power_acpi_cpufreq.c +17 -16
index 6e57aca..8b55f19 100644
@@ -7,6 +7,7 @@
 #include <stdlib.h>
 
 #include <rte_memcpy.h>
+#include <rte_stdatomic.h>
 #include <rte_string_fns.h>
 
 #include "power_acpi_cpufreq.h"
@@ -41,13 +42,13 @@ enum power_state {
  * Power info per lcore.
  */
 struct acpi_power_info {
-	unsigned int lcore_id;                   /**< Logical core id */
+	unsigned int lcore_id;               /**< Logical core id */
 	uint32_t freqs[RTE_MAX_LCORE_FREQS]; /**< Frequency array */
 	uint32_t nb_freqs;                   /**< number of available freqs */
 	FILE *f;                             /**< FD of scaling_setspeed */
 	char governor_ori[32];               /**< Original governor name */
 	uint32_t curr_idx;                   /**< Freq index in freqs array */
-	uint32_t state;                      /**< Power in use state */
+	RTE_ATOMIC(uint32_t) state;          /**< Power in use state */
 	uint16_t turbo_available;            /**< Turbo Boost available */
 	uint16_t turbo_enable;               /**< Turbo Boost enable/disable */
 } __rte_cache_aligned;
@@ -249,9 +250,9 @@ struct acpi_power_info {
 	 * ordering below as lock to make sure the frequency operations
 	 * in the critical section are done under the correct state.
 	 */
-	if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
-					POWER_ONGOING, 0,
-					__ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
+	if (!rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state,
+					POWER_ONGOING,
+					rte_memory_order_acquire, rte_memory_order_relaxed)) {
 		RTE_LOG(INFO, POWER, "Power management of lcore %u is "
 				"in use\n", lcore_id);
 		return -1;
@@ -289,15 +290,15 @@ struct acpi_power_info {
 	RTE_LOG(INFO, POWER, "Initialized successfully for lcore %u "
 			"power management\n", lcore_id);
 	exp_state = POWER_ONGOING;
-	__atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_USED,
-				    0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
+	rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state, POWER_USED,
+				    rte_memory_order_release, rte_memory_order_relaxed);
 
 	return 0;
 
 fail:
 	exp_state = POWER_ONGOING;
-	__atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_UNKNOWN,
-				    0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
+	rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state, POWER_UNKNOWN,
+				    rte_memory_order_release, rte_memory_order_relaxed);
 
 	return -1;
 }
@@ -321,9 +322,9 @@ struct acpi_power_info {
 	 * ordering below as lock to make sure the frequency operations
 	 * in the critical section are done under the correct state.
 	 */
-	if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
-					POWER_ONGOING, 0,
-					__ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
+	if (!rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state,
+					POWER_ONGOING,
+					rte_memory_order_acquire, rte_memory_order_relaxed)) {
 		RTE_LOG(INFO, POWER, "Power management of lcore %u is "
 				"not used\n", lcore_id);
 		return -1;
@@ -344,15 +345,15 @@ struct acpi_power_info {
 			"'userspace' mode and been set back to the "
 			"original\n", lcore_id);
 	exp_state = POWER_ONGOING;
-	__atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_IDLE,
-				    0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
+	rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state, POWER_IDLE,
+				    rte_memory_order_release, rte_memory_order_relaxed);
 
 	return 0;
 
 fail:
 	exp_state = POWER_ONGOING;
-	__atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_UNKNOWN,
-				    0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
+	rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state, POWER_UNKNOWN,
+				    rte_memory_order_release, rte_memory_order_relaxed);
 
 	return -1;
 }
lib/power/power_cppc_cpufreq.c +13 -12
index fc9cffe..bb70f6a 100644
@@ -6,6 +6,7 @@
 #include <stdlib.h>
 
 #include <rte_memcpy.h>
+#include <rte_stdatomic.h>
 
 #include "power_cppc_cpufreq.h"
 #include "power_common.h"
@@ -49,8 +50,8 @@ enum power_state {
  * Power info per lcore.
  */
 struct cppc_power_info {
-	unsigned int lcore_id;                   /**< Logical core id */
-	uint32_t state;                      /**< Power in use state */
+	unsigned int lcore_id;               /**< Logical core id */
+	RTE_ATOMIC(uint32_t) state;          /**< Power in use state */
 	FILE *f;                             /**< FD of scaling_setspeed */
 	char governor_ori[32];               /**< Original governor name */
 	uint32_t curr_idx;                   /**< Freq index in freqs array */
@@ -353,9 +354,9 @@ struct cppc_power_info {
 	 * ordering below as lock to make sure the frequency operations
 	 * in the critical section are done under the correct state.
 	 */
-	if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
-					POWER_ONGOING, 0,
-					__ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
+	if (!rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state,
+					POWER_ONGOING,
+					rte_memory_order_acquire, rte_memory_order_relaxed)) {
 		RTE_LOG(INFO, POWER, "Power management of lcore %u is "
 				"in use\n", lcore_id);
 		return -1;
@@ -393,12 +394,12 @@ struct cppc_power_info {
 	RTE_LOG(INFO, POWER, "Initialized successfully for lcore %u "
 			"power management\n", lcore_id);
 
-	__atomic_store_n(&(pi->state), POWER_USED, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&(pi->state), POWER_USED, rte_memory_order_release);
 
 	return 0;
 
 fail:
-	__atomic_store_n(&(pi->state), POWER_UNKNOWN, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&(pi->state), POWER_UNKNOWN, rte_memory_order_release);
 	return -1;
 }
 
@@ -431,9 +432,9 @@ struct cppc_power_info {
 	 * ordering below as lock to make sure the frequency operations
 	 * in the critical section are done under the correct state.
 	 */
-	if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
-					POWER_ONGOING, 0,
-					__ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
+	if (!rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state,
+					POWER_ONGOING,
+					rte_memory_order_acquire, rte_memory_order_relaxed)) {
 		RTE_LOG(INFO, POWER, "Power management of lcore %u is "
 				"not used\n", lcore_id);
 		return -1;
@@ -453,12 +454,12 @@ struct cppc_power_info {
 	RTE_LOG(INFO, POWER, "Power management of lcore %u has exited from "
 			"'userspace' mode and been set back to the "
 			"original\n", lcore_id);
-	__atomic_store_n(&(pi->state), POWER_IDLE, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&(pi->state), POWER_IDLE, rte_memory_order_release);
 
 	return 0;
 
 fail:
-	__atomic_store_n(&(pi->state), POWER_UNKNOWN, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&(pi->state), POWER_UNKNOWN, rte_memory_order_release);
 
 	return -1;
 }
lib/power/power_pstate_cpufreq.c +16 -15
index 52aa645..5ca5f60 100644
@@ -12,6 +12,7 @@
 #include <inttypes.h>
 
 #include <rte_memcpy.h>
+#include <rte_stdatomic.h>
 
 #include "rte_power_pmd_mgmt.h"
 #include "power_pstate_cpufreq.h"
@@ -59,7 +60,7 @@ struct pstate_power_info {
 	uint32_t non_turbo_max_ratio;        /**< Non Turbo Max ratio  */
 	uint32_t sys_max_freq;               /**< system wide max freq  */
 	uint32_t core_base_freq;             /**< core base freq  */
-	uint32_t state;                      /**< Power in use state */
+	RTE_ATOMIC(uint32_t) state;          /**< Power in use state */
 	uint16_t turbo_available;            /**< Turbo Boost available */
 	uint16_t turbo_enable;               /**< Turbo Boost enable/disable */
 	uint16_t priority_core;              /**< High Performance core */
@@ -555,9 +556,9 @@ struct pstate_power_info {
 	 * ordering below as lock to make sure the frequency operations
 	 * in the critical section are done under the correct state.
 	 */
-	if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
-					POWER_ONGOING, 0,
-					__ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
+	if (!rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state,
+					POWER_ONGOING,
+					rte_memory_order_acquire, rte_memory_order_relaxed)) {
 		RTE_LOG(INFO, POWER, "Power management of lcore %u is "
 				"in use\n", lcore_id);
 		return -1;
@@ -600,15 +601,15 @@ struct pstate_power_info {
 	RTE_LOG(INFO, POWER, "Initialized successfully for lcore %u "
 			"power management\n", lcore_id);
 	exp_state = POWER_ONGOING;
-	__atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_USED,
-				    0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
+	rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state, POWER_USED,
+				    rte_memory_order_release, rte_memory_order_relaxed);
 
 	return 0;
 
 fail:
 	exp_state = POWER_ONGOING;
-	__atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_UNKNOWN,
-				    0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
+	rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state, POWER_UNKNOWN,
+				    rte_memory_order_release, rte_memory_order_relaxed);
 
 	return -1;
 }
@@ -633,9 +634,9 @@ struct pstate_power_info {
 	 * ordering below as lock to make sure the frequency operations
 	 * in the critical section are under done the correct state.
 	 */
-	if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
-					POWER_ONGOING, 0,
-					__ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
+	if (!rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state,
+					POWER_ONGOING,
+					rte_memory_order_acquire, rte_memory_order_relaxed)) {
 		RTE_LOG(INFO, POWER, "Power management of lcore %u is "
 				"not used\n", lcore_id);
 		return -1;
@@ -658,15 +659,15 @@ struct pstate_power_info {
 			"'performance' mode and been set back to the "
 			"original\n", lcore_id);
 	exp_state = POWER_ONGOING;
-	__atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_IDLE,
-				    0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
+	rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state, POWER_IDLE,
+				    rte_memory_order_release, rte_memory_order_relaxed);
 
 	return 0;
 
 fail:
 	exp_state = POWER_ONGOING;
-	__atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_UNKNOWN,
-				    0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
+	rte_atomic_compare_exchange_strong_explicit(&(pi->state), &exp_state, POWER_UNKNOWN,
+				    rte_memory_order_release, rte_memory_order_relaxed);
 
 	return -1;
 }
Signature · 1 line
1.8.3.1
#61 lore raw

PATCH v3 2/19 bbdev: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/bbdev/rte_bbdev.c | 6 +++---
 lib/bbdev/rte_bbdev.h | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)
lib/bbdev/rte_bbdev.c +3 -3
index 155323e..cfebea0 100644
@@ -208,7 +208,7 @@ struct rte_bbdev *
 		return NULL;
 	}
 
-	__atomic_fetch_add(&bbdev->data->process_cnt, 1, __ATOMIC_RELAXED);
+	rte_atomic_fetch_add_explicit(&bbdev->data->process_cnt, 1, rte_memory_order_relaxed);
 	bbdev->data->dev_id = dev_id;
 	bbdev->state = RTE_BBDEV_INITIALIZED;
 
@@ -250,8 +250,8 @@ struct rte_bbdev *
 	}
 
 	/* clear shared BBDev Data if no process is using the device anymore */
-	if (__atomic_fetch_sub(&bbdev->data->process_cnt, 1,
-			      __ATOMIC_RELAXED) - 1 == 0)
+	if (rte_atomic_fetch_sub_explicit(&bbdev->data->process_cnt, 1,
+			      rte_memory_order_relaxed) - 1 == 0)
 		memset(bbdev->data, 0, sizeof(*bbdev->data));
 
 	memset(bbdev, 0, sizeof(*bbdev));
lib/bbdev/rte_bbdev.h +1 -1
index d12e2e7..e1aee08 100644
@@ -482,7 +482,7 @@ struct rte_bbdev_data {
 	uint16_t dev_id;  /**< Device ID */
 	int socket_id;  /**< NUMA socket that device is on */
 	bool started;  /**< Device run-time state */
-	uint16_t process_cnt;  /** Counter of processes using the device */
+	RTE_ATOMIC(uint16_t) process_cnt;  /** Counter of processes using the device */
 };
 
 /* Forward declarations */
Signature · 1 line
1.8.3.1
Reviewed-by on v3 2/19
#62 lore raw

Re: [PATCH v3 02/19] bbdev: use rte optional stdatomic API

Quoted 38 lines · Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
  lib/bbdev/rte_bbdev.c | 6 +++---
  lib/bbdev/rte_bbdev.h | 2 +-
  2 files changed, 4 insertions(+), 4 deletions(-)
lib/bbdev/rte_bbdev.c +3 -3
index 155323e..cfebea0 100644
@@ -208,7 +208,7 @@ struct rte_bbdev *
  		return NULL;
  	}
 
-	__atomic_fetch_add(&bbdev->data->process_cnt, 1, __ATOMIC_RELAXED);
+	rte_atomic_fetch_add_explicit(&bbdev->data->process_cnt, 1, rte_memory_order_relaxed);
  	bbdev->data->dev_id = dev_id;
  	bbdev->state = RTE_BBDEV_INITIALIZED;
 
@@ -250,8 +250,8 @@ struct rte_bbdev *
  	}
 
  	/* clear shared BBDev Data if no process is using the device anymore */
-	if (__atomic_fetch_sub(&bbdev->data->process_cnt, 1,
-			      __ATOMIC_RELAXED) - 1 == 0)
+	if (rte_atomic_fetch_sub_explicit(&bbdev->data->process_cnt, 1,
+			      rte_memory_order_relaxed) - 1 == 0)
  		memset(bbdev->data, 0, sizeof(*bbdev->data));
 
  	memset(bbdev, 0, sizeof(*bbdev));
lib/bbdev/rte_bbdev.h +0 -0
index d12e2e7..e1aee08 100644

lib/bbdev/rte_bbdev.h struct rte_bbdev_data { · line 488

Quoted 8 lines
lib/bbdev/rte_bbdev.h +1 -1
@@ -482,7 +482,7 @@ struct rte_bbdev_data {
  	uint16_t dev_id;  /**< Device ID */
  	int socket_id;  /**< NUMA socket that device is on */
  	bool started;  /**< Device run-time state */
-	uint16_t process_cnt;  /** Counter of processes using the device */
+	RTE_ATOMIC(uint16_t) process_cnt;  /** Counter of processes using the device */
  };
 
  /* Forward declarations */

Thanks,
Maxime

#63 lore raw

PATCH v3 3/19 eal: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/eal/common/eal_common_launch.c    |  10 +--
 lib/eal/common/eal_common_mcfg.c      |   2 +-
 lib/eal/common/eal_common_proc.c      |  14 ++--
 lib/eal/common/eal_common_thread.c    |  26 ++++----
 lib/eal/common/eal_common_trace.c     |   8 +--
 lib/eal/common/eal_common_trace_ctf.c |   4 +-
 lib/eal/common/eal_memcfg.h           |   2 +-
 lib/eal/common/eal_private.h          |   4 +-
 lib/eal/common/eal_trace.h            |   4 +-
 lib/eal/common/rte_service.c          | 122 +++++++++++++++++-----------------
 lib/eal/freebsd/eal.c                 |  20 +++---
 lib/eal/include/rte_epoll.h           |   3 +-
 lib/eal/linux/eal.c                   |  26 ++++----
 lib/eal/linux/eal_interrupts.c        |  42 ++++++------
 lib/eal/ppc/include/rte_atomic.h      |   6 +-
 lib/eal/windows/rte_thread.c          |   8 ++-
 16 files changed, 152 insertions(+), 149 deletions(-)
lib/eal/common/eal_common_launch.c +5 -5
index 0504598..5320c3b 100644
@@ -18,8 +18,8 @@
 int
 rte_eal_wait_lcore(unsigned worker_id)
 {
-	while (__atomic_load_n(&lcore_config[worker_id].state,
-			__ATOMIC_ACQUIRE) != WAIT)
+	while (rte_atomic_load_explicit(&lcore_config[worker_id].state,
+			rte_memory_order_acquire) != WAIT)
 		rte_pause();
 
 	return lcore_config[worker_id].ret;
@@ -38,8 +38,8 @@
 	/* Check if the worker is in 'WAIT' state. Use acquire order
 	 * since 'state' variable is used as the guard variable.
 	 */
-	if (__atomic_load_n(&lcore_config[worker_id].state,
-			__ATOMIC_ACQUIRE) != WAIT)
+	if (rte_atomic_load_explicit(&lcore_config[worker_id].state,
+			rte_memory_order_acquire) != WAIT)
 		goto finish;
 
 	lcore_config[worker_id].arg = arg;
@@ -47,7 +47,7 @@
 	 * before the worker thread starts running the function.
 	 * Use worker thread function as the guard variable.
 	 */
-	__atomic_store_n(&lcore_config[worker_id].f, f, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&lcore_config[worker_id].f, f, rte_memory_order_release);
 
 	rc = eal_thread_wake_worker(worker_id);
 
lib/eal/common/eal_common_mcfg.c +1 -1
index 2a785e7..dabb80e 100644
@@ -30,7 +30,7 @@
 	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
 
 	/* wait until shared mem_config finish initialising */
-	rte_wait_until_equal_32(&mcfg->magic, RTE_MAGIC, __ATOMIC_RELAXED);
+	rte_wait_until_equal_32(&mcfg->magic, RTE_MAGIC, rte_memory_order_relaxed);
 }
 
 int
lib/eal/common/eal_common_proc.c +7 -7
index f20a348..728815c 100644
@@ -33,7 +33,7 @@
 #include "eal_filesystem.h"
 #include "eal_internal_cfg.h"
 
-static int mp_fd = -1;
+static RTE_ATOMIC(int) mp_fd = -1;
 static rte_thread_t mp_handle_tid;
 static char mp_filter[PATH_MAX];   /* Filter for secondary process sockets */
 static char mp_dir_path[PATH_MAX]; /* The directory path for all mp sockets */
@@ -404,7 +404,7 @@ struct pending_request {
 	struct sockaddr_un sa;
 	int fd;
 
-	while ((fd = __atomic_load_n(&mp_fd, __ATOMIC_RELAXED)) >= 0) {
+	while ((fd = rte_atomic_load_explicit(&mp_fd, rte_memory_order_relaxed)) >= 0) {
 		int ret;
 
 		ret = read_msg(fd, &msg, &sa);
@@ -652,7 +652,7 @@ enum async_action {
 		RTE_LOG(ERR, EAL, "failed to create mp thread: %s\n",
 			strerror(errno));
 		close(dir_fd);
-		close(__atomic_exchange_n(&mp_fd, -1, __ATOMIC_RELAXED));
+		close(rte_atomic_exchange_explicit(&mp_fd, -1, rte_memory_order_relaxed));
 		return -1;
 	}
 
@@ -668,7 +668,7 @@ enum async_action {
 {
 	int fd;
 
-	fd = __atomic_exchange_n(&mp_fd, -1, __ATOMIC_RELAXED);
+	fd = rte_atomic_exchange_explicit(&mp_fd, -1, rte_memory_order_relaxed);
 	if (fd < 0)
 		return;
 
@@ -1282,11 +1282,11 @@ enum mp_status {
 
 	expected = MP_STATUS_UNKNOWN;
 	desired = status;
-	if (__atomic_compare_exchange_n(&mcfg->mp_status, &expected, desired,
-			false, __ATOMIC_RELAXED, __ATOMIC_RELAXED))
+	if (rte_atomic_compare_exchange_strong_explicit(&mcfg->mp_status, &expected, desired,
+			rte_memory_order_relaxed, rte_memory_order_relaxed))
 		return true;
 
-	return __atomic_load_n(&mcfg->mp_status, __ATOMIC_RELAXED) == desired;
+	return rte_atomic_load_explicit(&mcfg->mp_status, rte_memory_order_relaxed) == desired;
 }
 
 bool
lib/eal/common/eal_common_thread.c +13 -13
index 668b1ed..c422ea8 100644
@@ -191,8 +191,8 @@ unsigned rte_socket_id(void)
 		/* Set the state to 'RUNNING'. Use release order
 		 * since 'state' variable is used as the guard variable.
 		 */
-		__atomic_store_n(&lcore_config[lcore_id].state, RUNNING,
-			__ATOMIC_RELEASE);
+		rte_atomic_store_explicit(&lcore_config[lcore_id].state, RUNNING,
+			rte_memory_order_release);
 
 		eal_thread_ack_command();
 
@@ -201,8 +201,8 @@ unsigned rte_socket_id(void)
 		 * are accessed only after update to 'f' is visible.
 		 * Wait till the update to 'f' is visible to the worker.
 		 */
-		while ((f = __atomic_load_n(&lcore_config[lcore_id].f,
-				__ATOMIC_ACQUIRE)) == NULL)
+		while ((f = rte_atomic_load_explicit(&lcore_config[lcore_id].f,
+				rte_memory_order_acquire)) == NULL)
 			rte_pause();
 
 		rte_eal_trace_thread_lcore_running(lcore_id, f);
@@ -219,8 +219,8 @@ unsigned rte_socket_id(void)
 		 * are completed before the state is updated.
 		 * Use 'state' as the guard variable.
 		 */
-		__atomic_store_n(&lcore_config[lcore_id].state, WAIT,
-			__ATOMIC_RELEASE);
+		rte_atomic_store_explicit(&lcore_config[lcore_id].state, WAIT,
+			rte_memory_order_release);
 
 		rte_eal_trace_thread_lcore_stopped(lcore_id);
 	}
@@ -242,7 +242,7 @@ struct control_thread_params {
 	/* Control thread status.
 	 * If the status is CTRL_THREAD_ERROR, 'ret' has the error code.
 	 */
-	enum __rte_ctrl_thread_status status;
+	RTE_ATOMIC(enum __rte_ctrl_thread_status) status;
 };
 
 static int control_thread_init(void *arg)
@@ -259,13 +259,13 @@ static int control_thread_init(void *arg)
 	RTE_PER_LCORE(_socket_id) = SOCKET_ID_ANY;
 	params->ret = rte_thread_set_affinity_by_id(rte_thread_self(), cpuset);
 	if (params->ret != 0) {
-		__atomic_store_n(&params->status,
-			CTRL_THREAD_ERROR, __ATOMIC_RELEASE);
+		rte_atomic_store_explicit(&params->status,
+			CTRL_THREAD_ERROR, rte_memory_order_release);
 		return 1;
 	}
 
-	__atomic_store_n(&params->status,
-		CTRL_THREAD_RUNNING, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&params->status,
+		CTRL_THREAD_RUNNING, rte_memory_order_release);
 
 	return 0;
 }
@@ -310,8 +310,8 @@ static uint32_t control_thread_start(void *arg)
 
 	/* Wait for the control thread to initialize successfully */
 	while ((ctrl_thread_status =
-			__atomic_load_n(&params->status,
-			__ATOMIC_ACQUIRE)) == CTRL_THREAD_LAUNCHING) {
+			rte_atomic_load_explicit(&params->status,
+			rte_memory_order_acquire)) == CTRL_THREAD_LAUNCHING) {
 		rte_delay_us_sleep(1);
 	}
 
lib/eal/common/eal_common_trace.c +4 -4
index d2eac2d..6ad87fc 100644
@@ -97,7 +97,7 @@ struct trace_point_head *
 bool
 rte_trace_is_enabled(void)
 {
-	return __atomic_load_n(&trace.status, __ATOMIC_ACQUIRE) != 0;
+	return rte_atomic_load_explicit(&trace.status, rte_memory_order_acquire) != 0;
 }
 
 static void
@@ -157,7 +157,7 @@ rte_trace_mode rte_trace_mode_get(void)
 	prev = rte_atomic_fetch_or_explicit(t, __RTE_TRACE_FIELD_ENABLE_MASK,
 		rte_memory_order_release);
 	if ((prev & __RTE_TRACE_FIELD_ENABLE_MASK) == 0)
-		__atomic_fetch_add(&trace.status, 1, __ATOMIC_RELEASE);
+		rte_atomic_fetch_add_explicit(&trace.status, 1, rte_memory_order_release);
 	return 0;
 }
 
@@ -172,7 +172,7 @@ rte_trace_mode rte_trace_mode_get(void)
 	prev = rte_atomic_fetch_and_explicit(t, ~__RTE_TRACE_FIELD_ENABLE_MASK,
 		rte_memory_order_release);
 	if ((prev & __RTE_TRACE_FIELD_ENABLE_MASK) != 0)
-		__atomic_fetch_sub(&trace.status, 1, __ATOMIC_RELEASE);
+		rte_atomic_fetch_sub_explicit(&trace.status, 1, rte_memory_order_release);
 	return 0;
 }
 
@@ -526,7 +526,7 @@ rte_trace_mode rte_trace_mode_get(void)
 
 	/* Add the trace point at tail */
 	STAILQ_INSERT_TAIL(&tp_list, tp, next);
-	__atomic_thread_fence(__ATOMIC_RELEASE);
+	__atomic_thread_fence(rte_memory_order_release);
 
 	/* All Good !!! */
 	return 0;
lib/eal/common/eal_common_trace_ctf.c +2 -2
index c6775c3..04c4f71 100644
@@ -361,10 +361,10 @@
 	if (ctf_meta == NULL)
 		return -EINVAL;
 
-	if (!__atomic_load_n(&trace->ctf_fixup_done, __ATOMIC_SEQ_CST) &&
+	if (!rte_atomic_load_explicit(&trace->ctf_fixup_done, rte_memory_order_seq_cst) &&
 				rte_get_timer_hz()) {
 		meta_fixup(trace, ctf_meta);
-		__atomic_store_n(&trace->ctf_fixup_done, 1, __ATOMIC_SEQ_CST);
+		rte_atomic_store_explicit(&trace->ctf_fixup_done, 1, rte_memory_order_seq_cst);
 	}
 
 	rc = fprintf(f, "%s", ctf_meta);
lib/eal/common/eal_memcfg.h +1 -1
index d5c63e2..60e2089 100644
@@ -42,7 +42,7 @@ struct rte_mem_config {
 	rte_rwlock_t memory_hotplug_lock;
 	/**< Indicates whether memory hotplug request is in progress. */
 
-	uint8_t mp_status; /**< Multiprocess status. */
+	RTE_ATOMIC(uint8_t) mp_status; /**< Multiprocess status. */
 
 	/* memory segments and zones */
 	struct rte_fbarray memzones; /**< Memzone descriptors. */
lib/eal/common/eal_private.h +2 -2
index ebd496b..4d2e806 100644
@@ -24,11 +24,11 @@ struct lcore_config {
 	int pipe_main2worker[2];   /**< communication pipe with main */
 	int pipe_worker2main[2];   /**< communication pipe with main */
 
-	lcore_function_t * volatile f; /**< function to call */
+	RTE_ATOMIC(lcore_function_t *) volatile f; /**< function to call */
 	void * volatile arg;       /**< argument of function */
 	volatile int ret;          /**< return value of function */
 
-	volatile enum rte_lcore_state_t state; /**< lcore state */
+	volatile RTE_ATOMIC(enum rte_lcore_state_t) state; /**< lcore state */
 	unsigned int socket_id;    /**< physical socket id for this lcore */
 	unsigned int core_id;      /**< core number on socket for this lcore */
 	int core_index;            /**< relative index, starting from 0 */
lib/eal/common/eal_trace.h +2 -2
index d66bcfe..ace2ef3 100644
@@ -50,7 +50,7 @@ struct trace_arg {
 struct trace {
 	char *dir;
 	int register_errno;
-	uint32_t status;
+	RTE_ATOMIC(uint32_t) status;
 	enum rte_trace_mode mode;
 	rte_uuid_t uuid;
 	uint32_t buff_len;
@@ -65,7 +65,7 @@ struct trace {
 	uint32_t ctf_meta_offset_freq;
 	uint32_t ctf_meta_offset_freq_off_s;
 	uint32_t ctf_meta_offset_freq_off;
-	uint16_t ctf_fixup_done;
+	RTE_ATOMIC(uint16_t) ctf_fixup_done;
 	rte_spinlock_t lock;
 };
 
lib/eal/common/rte_service.c +61 -61

307 lines not on this page · raw message

lib/eal/freebsd/eal.c +10 -10
index 39a2868..568e06e 100644
@@ -597,8 +597,8 @@ static void rte_eal_init_alert(const char *msg)
 		return -1;
 	}
 
-	if (!__atomic_compare_exchange_n(&run_once, &has_run, 1, 0,
-					__ATOMIC_RELAXED, __ATOMIC_RELAXED)) {
+	if (!rte_atomic_compare_exchange_strong_explicit(&run_once, &has_run, 1,
+					rte_memory_order_relaxed, rte_memory_order_relaxed)) {
 		rte_eal_init_alert("already called initialization.");
 		rte_errno = EALREADY;
 		return -1;
@@ -622,7 +622,7 @@ static void rte_eal_init_alert(const char *msg)
 	if (fctret < 0) {
 		rte_eal_init_alert("Invalid 'command line' arguments.");
 		rte_errno = EINVAL;
-		__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
+		rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed);
 		return -1;
 	}
 
@@ -636,20 +636,20 @@ static void rte_eal_init_alert(const char *msg)
 	if (eal_plugins_init() < 0) {
 		rte_eal_init_alert("Cannot init plugins");
 		rte_errno = EINVAL;
-		__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
+		rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed);
 		return -1;
 	}
 
 	if (eal_trace_init() < 0) {
 		rte_eal_init_alert("Cannot init trace");
 		rte_errno = EFAULT;
-		__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
+		rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed);
 		return -1;
 	}
 
 	if (eal_option_device_parse()) {
 		rte_errno = ENODEV;
-		__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
+		rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed);
 		return -1;
 	}
 
@@ -683,7 +683,7 @@ static void rte_eal_init_alert(const char *msg)
 	if (rte_bus_scan()) {
 		rte_eal_init_alert("Cannot scan the buses for devices");
 		rte_errno = ENODEV;
-		__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
+		rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed);
 		return -1;
 	}
 
@@ -736,7 +736,7 @@ static void rte_eal_init_alert(const char *msg)
 		if (ret < 0) {
 			rte_eal_init_alert("Cannot get hugepage information.");
 			rte_errno = EACCES;
-			__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
+			rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed);
 			return -1;
 		}
 	}
@@ -915,8 +915,8 @@ static void rte_eal_init_alert(const char *msg)
 	static uint32_t run_once;
 	uint32_t has_run = 0;
 
-	if (!__atomic_compare_exchange_n(&run_once, &has_run, 1, 0,
-			__ATOMIC_RELAXED, __ATOMIC_RELAXED)) {
+	if (!rte_atomic_compare_exchange_strong_explicit(&run_once, &has_run, 1,
+			rte_memory_order_relaxed, rte_memory_order_relaxed)) {
 		RTE_LOG(WARNING, EAL, "Already called cleanup\n");
 		rte_errno = EALREADY;
 		return -1;
lib/eal/include/rte_epoll.h +2 -1
index 01525f5..ae0cf20 100644
@@ -13,6 +13,7 @@
 
 #include <stdint.h>
 
+#include <rte_stdatomic.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -38,7 +39,7 @@ enum {
 
 /** interrupt epoll event obj, taken by epoll_event.ptr */
 struct rte_epoll_event {
-	uint32_t status;           /**< OUT: event status */
+	RTE_ATOMIC(uint32_t) status;           /**< OUT: event status */
 	int fd;                    /**< OUT: event fd */
 	int epfd;       /**< OUT: epoll instance the ev associated with */
 	struct rte_epoll_data epdata;
lib/eal/linux/eal.c +13 -13
index 5f4b2fb..57da058 100644
@@ -967,7 +967,7 @@ static void rte_eal_init_alert(const char *msg)
 rte_eal_init(int argc, char **argv)
 {
 	int i, fctret, ret;
-	static uint32_t run_once;
+	static RTE_ATOMIC(uint32_t) run_once;
 	uint32_t has_run = 0;
 	char cpuset[RTE_CPU_AFFINITY_STR_LEN];
 	char thread_name[RTE_THREAD_NAME_SIZE];
@@ -983,8 +983,8 @@ static void rte_eal_init_alert(const char *msg)
 		return -1;
 	}
 
-	if (!__atomic_compare_exchange_n(&run_once, &has_run, 1, 0,
-					__ATOMIC_RELAXED, __ATOMIC_RELAXED)) {
+	if (!rte_atomic_compare_exchange_strong_explicit(&run_once, &has_run, 1,
+					rte_memory_order_relaxed, rte_memory_order_relaxed)) {
 		rte_eal_init_alert("already called initialization.");
 		rte_errno = EALREADY;
 		return -1;
@@ -1008,14 +1008,14 @@ static void rte_eal_init_alert(const char *msg)
 	if (fctret < 0) {
 		rte_eal_init_alert("Invalid 'command line' arguments.");
 		rte_errno = EINVAL;
-		__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
+		rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed);
 		return -1;
 	}
 
 	if (eal_plugins_init() < 0) {
 		rte_eal_init_alert("Cannot init plugins");
 		rte_errno = EINVAL;
-		__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
+		rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed);
 		return -1;
 	}
 
@@ -1027,7 +1027,7 @@ static void rte_eal_init_alert(const char *msg)
 
 	if (eal_option_device_parse()) {
 		rte_errno = ENODEV;
-		__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
+		rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed);
 		return -1;
 	}
 
@@ -1061,7 +1061,7 @@ static void rte_eal_init_alert(const char *msg)
 	if (rte_bus_scan()) {
 		rte_eal_init_alert("Cannot scan the buses for devices");
 		rte_errno = ENODEV;
-		__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
+		rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed);
 		return -1;
 	}
 
@@ -1125,7 +1125,7 @@ static void rte_eal_init_alert(const char *msg)
 		if (ret < 0) {
 			rte_eal_init_alert("Cannot get hugepage information.");
 			rte_errno = EACCES;
-			__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
+			rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed);
 			return -1;
 		}
 	}
@@ -1150,7 +1150,7 @@ static void rte_eal_init_alert(const char *msg)
 			 internal_conf->syslog_facility) < 0) {
 		rte_eal_init_alert("Cannot init logging.");
 		rte_errno = ENOMEM;
-		__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
+		rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed);
 		return -1;
 	}
 
@@ -1158,7 +1158,7 @@ static void rte_eal_init_alert(const char *msg)
 	if (rte_eal_vfio_setup() < 0) {
 		rte_eal_init_alert("Cannot init VFIO");
 		rte_errno = EAGAIN;
-		__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
+		rte_atomic_store_explicit(&run_once, 0, rte_memory_order_relaxed);
 		return -1;
 	}
 #endif
@@ -1345,11 +1345,11 @@ static void rte_eal_init_alert(const char *msg)
 int
 rte_eal_cleanup(void)
 {
-	static uint32_t run_once;
+	static RTE_ATOMIC(uint32_t) run_once;
 	uint32_t has_run = 0;
 
-	if (!__atomic_compare_exchange_n(&run_once, &has_run, 1, 0,
-					__ATOMIC_RELAXED, __ATOMIC_RELAXED)) {
+	if (!rte_atomic_compare_exchange_strong_explicit(&run_once, &has_run, 1,
+					rte_memory_order_relaxed, rte_memory_order_relaxed)) {
 		RTE_LOG(WARNING, EAL, "Already called cleanup\n");
 		rte_errno = EALREADY;
 		return -1;
lib/eal/linux/eal_interrupts.c +21 -21

93 lines not on this page · raw message

lib/eal/ppc/include/rte_atomic.h +3 -3
index 7382412..645c713 100644
@@ -48,7 +48,7 @@
 static inline int
 rte_atomic16_cmpset(volatile uint16_t *dst, uint16_t exp, uint16_t src)
 {
-	return __atomic_compare_exchange(dst, &exp, &src, 0, rte_memory_order_acquire,
+	return rte_atomic_compare_exchange_strong_explicit(dst, &exp, src, rte_memory_order_acquire,
 		rte_memory_order_acquire) ? 1 : 0;
 }
 
@@ -90,7 +90,7 @@ static inline int rte_atomic16_dec_and_test(rte_atomic16_t *v)
 static inline int
 rte_atomic32_cmpset(volatile uint32_t *dst, uint32_t exp, uint32_t src)
 {
-	return __atomic_compare_exchange(dst, &exp, &src, 0, rte_memory_order_acquire,
+	return rte_atomic_compare_exchange_strong_explicit(dst, &exp, src, rte_memory_order_acquire,
 		rte_memory_order_acquire) ? 1 : 0;
 }
 
@@ -132,7 +132,7 @@ static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v)
 static inline int
 rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src)
 {
-	return __atomic_compare_exchange(dst, &exp, &src, 0, rte_memory_order_acquire,
+	return rte_atomic_compare_exchange_strong_explicit(dst, &exp, src, rte_memory_order_acquire,
 		rte_memory_order_acquire) ? 1 : 0;
 }
 
lib/eal/windows/rte_thread.c +5 -3
index acf6484..145ac4b 100644
@@ -9,6 +9,7 @@
 #include <rte_eal.h>
 #include <rte_common.h>
 #include <rte_errno.h>
+#include <rte_stdatomic.h>
 #include <rte_thread.h>
 
 #include "eal_windows.h"
@@ -19,7 +20,7 @@ struct eal_tls_key {
 
 struct thread_routine_ctx {
 	rte_thread_func thread_func;
-	bool thread_init_failed;
+	RTE_ATOMIC(bool) thread_init_failed;
 	void *routine_args;
 };
 
@@ -168,7 +169,8 @@ struct thread_routine_ctx {
 thread_func_wrapper(void *arg)
 {
 	struct thread_routine_ctx ctx = *(struct thread_routine_ctx *)arg;
-	const bool thread_exit = __atomic_load_n(&ctx.thread_init_failed, __ATOMIC_ACQUIRE);
+	const bool thread_exit = rte_atomic_load_explicit(
+		&ctx.thread_init_failed, rte_memory_order_acquire);
 
 	free(arg);
 
@@ -237,7 +239,7 @@ struct thread_routine_ctx {
 	}
 
 resume_thread:
-	__atomic_store_n(&ctx->thread_init_failed, thread_exit, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&ctx->thread_init_failed, thread_exit, rte_memory_order_release);
 
 	if (ResumeThread(thread_handle) == (DWORD)-1) {
 		ret = thread_log_last_error("ResumeThread()");
Signature · 1 line
1.8.3.1
#64 lore raw

PATCH v3 4/19 eventdev: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 drivers/event/cnxk/cnxk_tim_worker.h   |  4 +--
 lib/eventdev/rte_event_timer_adapter.c | 66 +++++++++++++++++-----------------
 lib/eventdev/rte_event_timer_adapter.h |  2 +-
 3 files changed, 36 insertions(+), 36 deletions(-)
drivers/event/cnxk/cnxk_tim_worker.h +2 -2
index f0857f2..f530d8c 100644
@@ -314,7 +314,7 @@
 
 	tim->impl_opaque[0] = (uintptr_t)chunk;
 	tim->impl_opaque[1] = (uintptr_t)bkt;
-	__atomic_store_n(&tim->state, RTE_EVENT_TIMER_ARMED, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&tim->state, RTE_EVENT_TIMER_ARMED, rte_memory_order_release);
 	cnxk_tim_bkt_inc_nent(bkt);
 	cnxk_tim_bkt_dec_lock_relaxed(bkt);
 
@@ -425,7 +425,7 @@
 
 	tim->impl_opaque[0] = (uintptr_t)chunk;
 	tim->impl_opaque[1] = (uintptr_t)bkt;
-	__atomic_store_n(&tim->state, RTE_EVENT_TIMER_ARMED, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&tim->state, RTE_EVENT_TIMER_ARMED, rte_memory_order_release);
 	cnxk_tim_bkt_inc_nent(bkt);
 	cnxk_tim_bkt_dec_lock_relaxed(bkt);
 
lib/eventdev/rte_event_timer_adapter.c +33 -33

162 lines not on this page · raw message

lib/eventdev/rte_event_timer_adapter.h +1 -1
index fbdddf8..49e646a 100644
@@ -498,7 +498,7 @@ struct rte_event_timer {
 	 * implementation specific values to share between the arm and cancel
 	 * operations.  The application should not modify this field.
 	 */
-	enum rte_event_timer_state state;
+	RTE_ATOMIC(enum rte_event_timer_state) state;
 	/**< State of the event timer. */
 	uint8_t user_meta[];
 	/**< Memory to store user specific metadata.
Signature · 1 line
1.8.3.1
#65 lore raw

PATCH v3 5/19 gpudev: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/gpudev/gpudev.c        | 6 +++---
 lib/gpudev/gpudev_driver.h | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)
lib/gpudev/gpudev.c +3 -3
index 8f12abe..6845d18 100644
@@ -228,7 +228,7 @@ struct rte_gpu *
 	dev->mpshared->info.numa_node = -1;
 	dev->mpshared->info.parent = RTE_GPU_ID_NONE;
 	TAILQ_INIT(&dev->callbacks);
-	__atomic_fetch_add(&dev->mpshared->process_refcnt, 1, __ATOMIC_RELAXED);
+	rte_atomic_fetch_add_explicit(&dev->mpshared->process_refcnt, 1, rte_memory_order_relaxed);
 
 	gpu_count++;
 	GPU_LOG(DEBUG, "new device %s (id %d) of total %d",
@@ -277,7 +277,7 @@ struct rte_gpu *
 
 	TAILQ_INIT(&dev->callbacks);
 	dev->mpshared = shared_dev;
-	__atomic_fetch_add(&dev->mpshared->process_refcnt, 1, __ATOMIC_RELAXED);
+	rte_atomic_fetch_add_explicit(&dev->mpshared->process_refcnt, 1, rte_memory_order_relaxed);
 
 	gpu_count++;
 	GPU_LOG(DEBUG, "attached device %s (id %d) of total %d",
@@ -340,7 +340,7 @@ struct rte_gpu *
 
 	gpu_free_callbacks(dev);
 	dev->process_state = RTE_GPU_STATE_UNUSED;
-	__atomic_fetch_sub(&dev->mpshared->process_refcnt, 1, __ATOMIC_RELAXED);
+	rte_atomic_fetch_sub_explicit(&dev->mpshared->process_refcnt, 1, rte_memory_order_relaxed);
 	gpu_count--;
 
 	return 0;
lib/gpudev/gpudev_driver.h +1 -1
index 42898c7..0b1e7f2 100644
@@ -69,7 +69,7 @@ struct rte_gpu_mpshared {
 	/* Device info structure. */
 	struct rte_gpu_info info;
 	/* Counter of processes using the device. */
-	uint16_t process_refcnt; /* Updated by this library. */
+	RTE_ATOMIC(uint16_t) process_refcnt; /* Updated by this library. */
 };
 
 struct rte_gpu {
Signature · 1 line
1.8.3.1
#66 lore raw

PATCH v3 6/19 ipsec: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/ipsec/ipsec_sqn.h | 2 +-
 lib/ipsec/sa.h        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
lib/ipsec/ipsec_sqn.h +1 -1
index 505950e..984a9dd 100644
@@ -128,7 +128,7 @@
 
 	n = *num;
 	if (SQN_ATOMIC(sa))
-		sqn = __atomic_fetch_add(&sa->sqn.outb, n, __ATOMIC_RELAXED) + n;
+		sqn = rte_atomic_fetch_add_explicit(&sa->sqn.outb, n, rte_memory_order_relaxed) + n;
 	else {
 		sqn = sa->sqn.outb + n;
 		sa->sqn.outb = sqn;
lib/ipsec/sa.h +1 -1
index ce4af8c..4b30bea 100644
@@ -124,7 +124,7 @@ struct rte_ipsec_sa {
 	 * place from other frequently accessed data.
 	 */
 	union {
-		uint64_t outb;
+		RTE_ATOMIC(uint64_t) outb;
 		struct {
 			uint32_t rdidx; /* read index */
 			uint32_t wridx; /* write index */
Signature · 1 line
1.8.3.1
#68 lore raw

PATCH v3 7/19 mbuf: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/mbuf/rte_mbuf.h      | 20 ++++++++++----------
 lib/mbuf/rte_mbuf_core.h |  5 +++--
 2 files changed, 13 insertions(+), 12 deletions(-)
lib/mbuf/rte_mbuf.h +10 -10
index 913c459..b8ab477 100644
@@ -361,7 +361,7 @@ struct rte_pktmbuf_pool_private {
 static inline uint16_t
 rte_mbuf_refcnt_read(const struct rte_mbuf *m)
 {
-	return __atomic_load_n(&m->refcnt, __ATOMIC_RELAXED);
+	return rte_atomic_load_explicit(&m->refcnt, rte_memory_order_relaxed);
 }
 
 /**
@@ -374,15 +374,15 @@ struct rte_pktmbuf_pool_private {
 static inline void
 rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
 {
-	__atomic_store_n(&m->refcnt, new_value, __ATOMIC_RELAXED);
+	rte_atomic_store_explicit(&m->refcnt, new_value, rte_memory_order_relaxed);
 }
 
 /* internal */
 static inline uint16_t
 __rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
 {
-	return __atomic_fetch_add(&m->refcnt, value,
-				 __ATOMIC_ACQ_REL) + value;
+	return rte_atomic_fetch_add_explicit(&m->refcnt, value,
+				 rte_memory_order_acq_rel) + value;
 }
 
 /**
@@ -463,7 +463,7 @@ struct rte_pktmbuf_pool_private {
 static inline uint16_t
 rte_mbuf_ext_refcnt_read(const struct rte_mbuf_ext_shared_info *shinfo)
 {
-	return __atomic_load_n(&shinfo->refcnt, __ATOMIC_RELAXED);
+	return rte_atomic_load_explicit(&shinfo->refcnt, rte_memory_order_relaxed);
 }
 
 /**
@@ -478,7 +478,7 @@ struct rte_pktmbuf_pool_private {
 rte_mbuf_ext_refcnt_set(struct rte_mbuf_ext_shared_info *shinfo,
 	uint16_t new_value)
 {
-	__atomic_store_n(&shinfo->refcnt, new_value, __ATOMIC_RELAXED);
+	rte_atomic_store_explicit(&shinfo->refcnt, new_value, rte_memory_order_relaxed);
 }
 
 /**
@@ -502,8 +502,8 @@ struct rte_pktmbuf_pool_private {
 		return (uint16_t)value;
 	}
 
-	return __atomic_fetch_add(&shinfo->refcnt, value,
-				 __ATOMIC_ACQ_REL) + value;
+	return rte_atomic_fetch_add_explicit(&shinfo->refcnt, value,
+				 rte_memory_order_acq_rel) + value;
 }
 
 /** Mbuf prefetch */
@@ -1315,8 +1315,8 @@ static inline int __rte_pktmbuf_pinned_extbuf_decref(struct rte_mbuf *m)
 	 * Direct usage of add primitive to avoid
 	 * duplication of comparing with one.
 	 */
-	if (likely(__atomic_fetch_add(&shinfo->refcnt, -1,
-				     __ATOMIC_ACQ_REL) - 1))
+	if (likely(rte_atomic_fetch_add_explicit(&shinfo->refcnt, -1,
+				     rte_memory_order_acq_rel) - 1))
 		return 1;
 
 	/* Reinitialize counter before mbuf freeing. */
lib/mbuf/rte_mbuf_core.h +3 -2
index e9bc0d1..5688683 100644
@@ -19,6 +19,7 @@
 #include <stdint.h>
 
 #include <rte_byteorder.h>
+#include <rte_stdatomic.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -497,7 +498,7 @@ struct rte_mbuf {
 	 * rte_mbuf_refcnt_set(). The functionality of these functions (atomic,
 	 * or non-atomic) is controlled by the RTE_MBUF_REFCNT_ATOMIC flag.
 	 */
-	uint16_t refcnt;
+	RTE_ATOMIC(uint16_t) refcnt;
 
 	/**
 	 * Number of segments. Only valid for the first segment of an mbuf
@@ -674,7 +675,7 @@ struct rte_mbuf {
 struct rte_mbuf_ext_shared_info {
 	rte_mbuf_extbuf_free_callback_t free_cb; /**< Free callback function */
 	void *fcb_opaque;                        /**< Free callback argument */
-	uint16_t refcnt;
+	RTE_ATOMIC(uint16_t) refcnt;
 };
 
 /** Maximum number of nb_segs allowed. */
Signature · 1 line
1.8.3.1
#69 lore raw

PATCH v3 8/19 mempool: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/mempool/rte_mempool.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
lib/mempool/rte_mempool.h +2 -2
index f70bf36..df87cd2 100644
@@ -327,8 +327,8 @@ struct rte_mempool {
 		if (likely(__lcore_id < RTE_MAX_LCORE))                         \
 			(mp)->stats[__lcore_id].name += (n);                    \
 		else                                                            \
-			__atomic_fetch_add(&((mp)->stats[RTE_MAX_LCORE].name),  \
-					   (n), __ATOMIC_RELAXED);              \
+			rte_atomic_fetch_add_explicit(&((mp)->stats[RTE_MAX_LCORE].name),  \
+					   (n), rte_memory_order_relaxed);              \
 	} while (0)
 #else
 #define RTE_MEMPOOL_STAT_ADD(mp, name, n) do {} while (0)
Signature · 1 line
1.8.3.1
#70 lore raw

PATCH v3 9/19 rcu: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/rcu/rte_rcu_qsbr.c | 48 +++++++++++++++++------------------
 lib/rcu/rte_rcu_qsbr.h | 68 +++++++++++++++++++++++++-------------------------
 2 files changed, 58 insertions(+), 58 deletions(-)
lib/rcu/rte_rcu_qsbr.c +24 -24
index 17be93e..4dc7714 100644
@@ -102,21 +102,21 @@
 	 * go out of sync. Hence, additional checks are required.
 	 */
 	/* Check if the thread is already registered */
-	old_bmap = __atomic_load_n(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					__ATOMIC_RELAXED);
+	old_bmap = rte_atomic_load_explicit(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
+					rte_memory_order_relaxed);
 	if (old_bmap & 1UL << id)
 		return 0;
 
 	do {
 		new_bmap = old_bmap | (1UL << id);
-		success = __atomic_compare_exchange(
+		success = rte_atomic_compare_exchange_strong_explicit(
 					__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					&old_bmap, &new_bmap, 0,
-					__ATOMIC_RELEASE, __ATOMIC_RELAXED);
+					&old_bmap, new_bmap,
+					rte_memory_order_release, rte_memory_order_relaxed);
 
 		if (success)
-			__atomic_fetch_add(&v->num_threads,
-						1, __ATOMIC_RELAXED);
+			rte_atomic_fetch_add_explicit(&v->num_threads,
+						1, rte_memory_order_relaxed);
 		else if (old_bmap & (1UL << id))
 			/* Someone else registered this thread.
 			 * Counter should not be incremented.
@@ -154,8 +154,8 @@
 	 * go out of sync. Hence, additional checks are required.
 	 */
 	/* Check if the thread is already unregistered */
-	old_bmap = __atomic_load_n(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					__ATOMIC_RELAXED);
+	old_bmap = rte_atomic_load_explicit(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
+					rte_memory_order_relaxed);
 	if (!(old_bmap & (1UL << id)))
 		return 0;
 
@@ -165,14 +165,14 @@
 		 * completed before removal of the thread from the list of
 		 * reporting threads.
 		 */
-		success = __atomic_compare_exchange(
+		success = rte_atomic_compare_exchange_strong_explicit(
 					__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					&old_bmap, &new_bmap, 0,
-					__ATOMIC_RELEASE, __ATOMIC_RELAXED);
+					&old_bmap, new_bmap,
+					rte_memory_order_release, rte_memory_order_relaxed);
 
 		if (success)
-			__atomic_fetch_sub(&v->num_threads,
-						1, __ATOMIC_RELAXED);
+			rte_atomic_fetch_sub_explicit(&v->num_threads,
+						1, rte_memory_order_relaxed);
 		else if (!(old_bmap & (1UL << id)))
 			/* Someone else unregistered this thread.
 			 * Counter should not be incremented.
@@ -227,8 +227,8 @@
 
 	fprintf(f, "  Registered thread IDs = ");
 	for (i = 0; i < v->num_elems; i++) {
-		bmap = __atomic_load_n(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					__ATOMIC_ACQUIRE);
+		bmap = rte_atomic_load_explicit(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
+					rte_memory_order_acquire);
 		id = i << __RTE_QSBR_THRID_INDEX_SHIFT;
 		while (bmap) {
 			t = __builtin_ctzl(bmap);
@@ -241,26 +241,26 @@
 	fprintf(f, "\n");
 
 	fprintf(f, "  Token = %" PRIu64 "\n",
-			__atomic_load_n(&v->token, __ATOMIC_ACQUIRE));
+			rte_atomic_load_explicit(&v->token, rte_memory_order_acquire));
 
 	fprintf(f, "  Least Acknowledged Token = %" PRIu64 "\n",
-			__atomic_load_n(&v->acked_token, __ATOMIC_ACQUIRE));
+			rte_atomic_load_explicit(&v->acked_token, rte_memory_order_acquire));
 
 	fprintf(f, "Quiescent State Counts for readers:\n");
 	for (i = 0; i < v->num_elems; i++) {
-		bmap = __atomic_load_n(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
-					__ATOMIC_ACQUIRE);
+		bmap = rte_atomic_load_explicit(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
+					rte_memory_order_acquire);
 		id = i << __RTE_QSBR_THRID_INDEX_SHIFT;
 		while (bmap) {
 			t = __builtin_ctzl(bmap);
 			fprintf(f, "thread ID = %u, count = %" PRIu64 ", lock count = %u\n",
 				id + t,
-				__atomic_load_n(
+				rte_atomic_load_explicit(
 					&v->qsbr_cnt[id + t].cnt,
-					__ATOMIC_RELAXED),
-				__atomic_load_n(
+					rte_memory_order_relaxed),
+				rte_atomic_load_explicit(
 					&v->qsbr_cnt[id + t].lock_cnt,
-					__ATOMIC_RELAXED));
+					rte_memory_order_relaxed));
 			bmap &= ~(1UL << t);
 		}
 	}
lib/rcu/rte_rcu_qsbr.h +34 -34

195 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#71 lore raw

PATCH v3 10/19 pdump: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/pdump/rte_pdump.c | 14 +++++++-------
 lib/pdump/rte_pdump.h |  8 ++++----
 2 files changed, 11 insertions(+), 11 deletions(-)
lib/pdump/rte_pdump.c +7 -7
index 53cca10..80b90c6 100644
@@ -110,8 +110,8 @@ struct pdump_response {
 		 * then packet doesn't match the filter (will be ignored).
 		 */
 		if (cbs->filter && rcs[i] == 0) {
-			__atomic_fetch_add(&stats->filtered,
-					   1, __ATOMIC_RELAXED);
+			rte_atomic_fetch_add_explicit(&stats->filtered,
+					   1, rte_memory_order_relaxed);
 			continue;
 		}
 
@@ -127,18 +127,18 @@ struct pdump_response {
 			p = rte_pktmbuf_copy(pkts[i], mp, 0, cbs->snaplen);
 
 		if (unlikely(p == NULL))
-			__atomic_fetch_add(&stats->nombuf, 1, __ATOMIC_RELAXED);
+			rte_atomic_fetch_add_explicit(&stats->nombuf, 1, rte_memory_order_relaxed);
 		else
 			dup_bufs[d_pkts++] = p;
 	}
 
-	__atomic_fetch_add(&stats->accepted, d_pkts, __ATOMIC_RELAXED);
+	rte_atomic_fetch_add_explicit(&stats->accepted, d_pkts, rte_memory_order_relaxed);
 
 	ring_enq = rte_ring_enqueue_burst(ring, (void *)&dup_bufs[0], d_pkts, NULL);
 	if (unlikely(ring_enq < d_pkts)) {
 		unsigned int drops = d_pkts - ring_enq;
 
-		__atomic_fetch_add(&stats->ringfull, drops, __ATOMIC_RELAXED);
+		rte_atomic_fetch_add_explicit(&stats->ringfull, drops, rte_memory_order_relaxed);
 		rte_pktmbuf_free_bulk(&dup_bufs[ring_enq], drops);
 	}
 }
@@ -720,10 +720,10 @@ struct pdump_response {
 	uint16_t qid;
 
 	for (qid = 0; qid < nq; qid++) {
-		const uint64_t *perq = (const uint64_t *)&stats[port][qid];
+		const RTE_ATOMIC(uint64_t) *perq = (const uint64_t __rte_atomic *)&stats[port][qid];
 
 		for (i = 0; i < sizeof(*total) / sizeof(uint64_t); i++) {
-			val = __atomic_load_n(&perq[i], __ATOMIC_RELAXED);
+			val = rte_atomic_load_explicit(&perq[i], rte_memory_order_relaxed);
 			sum[i] += val;
 		}
 	}
lib/pdump/rte_pdump.h +4 -4
index b1a3918..7feb2b6 100644
@@ -233,10 +233,10 @@ enum {
  * The statistics are sum of both receive and transmit queues.
  */
 struct rte_pdump_stats {
-	uint64_t accepted; /**< Number of packets accepted by filter. */
-	uint64_t filtered; /**< Number of packets rejected by filter. */
-	uint64_t nombuf;   /**< Number of mbuf allocation failures. */
-	uint64_t ringfull; /**< Number of missed packets due to ring full. */
+	RTE_ATOMIC(uint64_t) accepted; /**< Number of packets accepted by filter. */
+	RTE_ATOMIC(uint64_t) filtered; /**< Number of packets rejected by filter. */
+	RTE_ATOMIC(uint64_t) nombuf;   /**< Number of mbuf allocation failures. */
+	RTE_ATOMIC(uint64_t) ringfull; /**< Number of missed packets due to ring full. */
 
 	uint64_t reserved[4]; /**< Reserved and pad to cache line */
 };
Signature · 1 line
1.8.3.1
#72 lore raw

PATCH v3 11/19 stack: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/stack/rte_stack.h            |  2 +-
 lib/stack/rte_stack_lf_c11.h     | 24 ++++++++++++------------
 lib/stack/rte_stack_lf_generic.h | 18 +++++++++---------
 3 files changed, 22 insertions(+), 22 deletions(-)
lib/stack/rte_stack.h +1 -1
index 921d29a..a379300 100644
@@ -44,7 +44,7 @@ struct rte_stack_lf_list {
 	/** List head */
 	struct rte_stack_lf_head head __rte_aligned(16);
 	/** List len */
-	uint64_t len;
+	RTE_ATOMIC(uint64_t) len;
 };
 
 /* Structure containing two lock-free LIFO lists: the stack itself and a list
lib/stack/rte_stack_lf_c11.h +12 -12
index 687a6f6..9cb6998 100644
@@ -26,8 +26,8 @@
 	 * elements. If the mempool is near-empty to the point that this is a
 	 * concern, the user should consider increasing the mempool size.
 	 */
-	return (unsigned int)__atomic_load_n(&s->stack_lf.used.len,
-					     __ATOMIC_RELAXED);
+	return (unsigned int)rte_atomic_load_explicit(&s->stack_lf.used.len,
+					     rte_memory_order_relaxed);
 }
 
 static __rte_always_inline void
@@ -59,14 +59,14 @@
 				(rte_int128_t *)&list->head,
 				(rte_int128_t *)&old_head,
 				(rte_int128_t *)&new_head,
-				1, __ATOMIC_RELEASE,
-				__ATOMIC_RELAXED);
+				1, rte_memory_order_release,
+				rte_memory_order_relaxed);
 	} while (success == 0);
 
 	/* Ensure the stack modifications are not reordered with respect
 	 * to the LIFO len update.
 	 */
-	__atomic_fetch_add(&list->len, num, __ATOMIC_RELEASE);
+	rte_atomic_fetch_add_explicit(&list->len, num, rte_memory_order_release);
 }
 
 static __rte_always_inline struct rte_stack_lf_elem *
@@ -80,7 +80,7 @@
 	int success;
 
 	/* Reserve num elements, if available */
-	len = __atomic_load_n(&list->len, __ATOMIC_RELAXED);
+	len = rte_atomic_load_explicit(&list->len, rte_memory_order_relaxed);
 
 	while (1) {
 		/* Does the list contain enough elements? */
@@ -88,10 +88,10 @@
 			return NULL;
 
 		/* len is updated on failure */
-		if (__atomic_compare_exchange_n(&list->len,
+		if (rte_atomic_compare_exchange_weak_explicit(&list->len,
 						&len, len - num,
-						1, __ATOMIC_ACQUIRE,
-						__ATOMIC_RELAXED))
+						rte_memory_order_acquire,
+						rte_memory_order_relaxed))
 			break;
 	}
 
@@ -110,7 +110,7 @@
 		 * elements are properly ordered with respect to the head
 		 * pointer read.
 		 */
-		__atomic_thread_fence(__ATOMIC_ACQUIRE);
+		__atomic_thread_fence(rte_memory_order_acquire);
 
 		rte_prefetch0(old_head.top);
 
@@ -159,8 +159,8 @@
 				(rte_int128_t *)&list->head,
 				(rte_int128_t *)&old_head,
 				(rte_int128_t *)&new_head,
-				0, __ATOMIC_RELAXED,
-				__ATOMIC_RELAXED);
+				0, rte_memory_order_relaxed,
+				rte_memory_order_relaxed);
 	} while (success == 0);
 
 	return old_head.top;
lib/stack/rte_stack_lf_generic.h +9 -9
index 39f7ff3..cc69e4d 100644
@@ -27,7 +27,7 @@
 	 * concern, the user should consider increasing the mempool size.
 	 */
 	/* NOTE: review for potential ordering optimization */
-	return __atomic_load_n(&s->stack_lf.used.len, __ATOMIC_SEQ_CST);
+	return rte_atomic_load_explicit(&s->stack_lf.used.len, rte_memory_order_seq_cst);
 }
 
 static __rte_always_inline void
@@ -64,11 +64,11 @@
 				(rte_int128_t *)&list->head,
 				(rte_int128_t *)&old_head,
 				(rte_int128_t *)&new_head,
-				1, __ATOMIC_RELEASE,
-				__ATOMIC_RELAXED);
+				1, rte_memory_order_release,
+				rte_memory_order_relaxed);
 	} while (success == 0);
 	/* NOTE: review for potential ordering optimization */
-	__atomic_fetch_add(&list->len, num, __ATOMIC_SEQ_CST);
+	rte_atomic_fetch_add_explicit(&list->len, num, rte_memory_order_seq_cst);
 }
 
 static __rte_always_inline struct rte_stack_lf_elem *
@@ -83,15 +83,15 @@
 	/* Reserve num elements, if available */
 	while (1) {
 		/* NOTE: review for potential ordering optimization */
-		uint64_t len = __atomic_load_n(&list->len, __ATOMIC_SEQ_CST);
+		uint64_t len = rte_atomic_load_explicit(&list->len, rte_memory_order_seq_cst);
 
 		/* Does the list contain enough elements? */
 		if (unlikely(len < num))
 			return NULL;
 
 		/* NOTE: review for potential ordering optimization */
-		if (__atomic_compare_exchange_n(&list->len, &len, len - num,
-				0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST))
+		if (rte_atomic_compare_exchange_strong_explicit(&list->len, &len, len - num,
+				rte_memory_order_seq_cst, rte_memory_order_seq_cst))
 			break;
 	}
 
@@ -143,8 +143,8 @@
 				(rte_int128_t *)&list->head,
 				(rte_int128_t *)&old_head,
 				(rte_int128_t *)&new_head,
-				1, __ATOMIC_RELEASE,
-				__ATOMIC_RELAXED);
+				1, rte_memory_order_release,
+				rte_memory_order_relaxed);
 	} while (success == 0);
 
 	return old_head.top;
Signature · 1 line
1.8.3.1
#73 lore raw

PATCH v3 12/19 telemetry: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/telemetry/telemetry.c | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)
lib/telemetry/telemetry.c +9 -9
index aeb078c..9298284 100644
@@ -45,7 +45,7 @@ struct socket {
 	int sock;
 	char path[sizeof(((struct sockaddr_un *)0)->sun_path)];
 	handler fn;
-	uint16_t *num_clients;
+	RTE_ATOMIC(uint16_t) *num_clients;
 };
 static struct socket v2_socket; /* socket for v2 telemetry */
 static struct socket v1_socket; /* socket for v1 telemetry */
@@ -64,7 +64,7 @@ struct socket {
 /* Used when accessing or modifying list of command callbacks */
 static rte_spinlock_t callback_sl = RTE_SPINLOCK_INITIALIZER;
 #ifndef RTE_EXEC_ENV_WINDOWS
-static uint16_t v2_clients;
+static RTE_ATOMIC(uint16_t) v2_clients;
 #endif /* !RTE_EXEC_ENV_WINDOWS */
 
 int
@@ -404,7 +404,7 @@ struct socket {
 		bytes = read(s, buffer, sizeof(buffer) - 1);
 	}
 	close(s);
-	__atomic_fetch_sub(&v2_clients, 1, __ATOMIC_RELAXED);
+	rte_atomic_fetch_sub_explicit(&v2_clients, 1, rte_memory_order_relaxed);
 	return NULL;
 }
 
@@ -421,14 +421,14 @@ struct socket {
 			return NULL;
 		}
 		if (s->num_clients != NULL) {
-			uint16_t conns = __atomic_load_n(s->num_clients,
-					__ATOMIC_RELAXED);
+			uint16_t conns = rte_atomic_load_explicit(s->num_clients,
+					rte_memory_order_relaxed);
 			if (conns >= MAX_CONNECTIONS) {
 				close(s_accepted);
 				continue;
 			}
-			__atomic_fetch_add(s->num_clients, 1,
-					__ATOMIC_RELAXED);
+			rte_atomic_fetch_add_explicit(s->num_clients, 1,
+					rte_memory_order_relaxed);
 		}
 		rc = pthread_create(&th, NULL, s->fn,
 				    (void *)(uintptr_t)s_accepted);
@@ -437,8 +437,8 @@ struct socket {
 				 strerror(rc));
 			close(s_accepted);
 			if (s->num_clients != NULL)
-				__atomic_fetch_sub(s->num_clients, 1,
-						   __ATOMIC_RELAXED);
+				rte_atomic_fetch_sub_explicit(s->num_clients, 1,
+						   rte_memory_order_relaxed);
 			continue;
 		}
 		pthread_detach(th);
Signature · 1 line
1.8.3.1
#74 lore raw

PATCH v3 13/19 vhost: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/vhost/vdpa.c            |  3 ++-
 lib/vhost/vhost.c           | 42 ++++++++++++++++----------------
 lib/vhost/vhost.h           | 39 ++++++++++++++++--------------
 lib/vhost/vhost_user.c      |  6 ++---
 lib/vhost/virtio_net.c      | 58 +++++++++++++++++++++++++--------------------
 lib/vhost/virtio_net_ctrl.c |  6 +++--
 6 files changed, 84 insertions(+), 70 deletions(-)
lib/vhost/vdpa.c +2 -1
index 6284ea2..219eef8 100644
@@ -235,7 +235,8 @@ struct rte_vdpa_device *
 	}
 
 	/* used idx is the synchronization point for the split vring */
-	__atomic_store_n(&vq->used->idx, idx_m, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit((unsigned short __rte_atomic *)&vq->used->idx,
+		idx_m, rte_memory_order_release);
 
 	if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))
 		vring_used_event(s_vring) = idx_m;
lib/vhost/vhost.c +22 -20

124 lines not on this page · raw message

lib/vhost/vhost.h +21 -18
index 5fc9035..f8624fb 100644
@@ -158,9 +158,9 @@ struct virtqueue_stats {
 	uint64_t inflight_completed;
 	uint64_t guest_notifications_suppressed;
 	/* Counters below are atomic, and should be incremented as such. */
-	uint64_t guest_notifications;
-	uint64_t guest_notifications_offloaded;
-	uint64_t guest_notifications_error;
+	RTE_ATOMIC(uint64_t) guest_notifications;
+	RTE_ATOMIC(uint64_t) guest_notifications_offloaded;
+	RTE_ATOMIC(uint64_t) guest_notifications_error;
 };
 
 /**
@@ -348,7 +348,7 @@ struct vhost_virtqueue {
 	struct vhost_vring_addr ring_addrs;
 	struct virtqueue_stats	stats;
 
-	bool irq_pending;
+	RTE_ATOMIC(bool) irq_pending;
 } __rte_cache_aligned;
 
 /* Virtio device status as per Virtio specification */
@@ -486,7 +486,7 @@ struct virtio_net {
 	uint32_t		flags;
 	uint16_t		vhost_hlen;
 	/* to tell if we need broadcast rarp packet */
-	int16_t			broadcast_rarp;
+	RTE_ATOMIC(int16_t)	broadcast_rarp;
 	uint32_t		nr_vring;
 	int			async_copy;
 
@@ -557,7 +557,8 @@ struct virtio_net {
 static inline bool
 desc_is_avail(struct vring_packed_desc *desc, bool wrap_counter)
 {
-	uint16_t flags = __atomic_load_n(&desc->flags, __ATOMIC_ACQUIRE);
+	uint16_t flags = rte_atomic_load_explicit((unsigned short __rte_atomic *)&desc->flags,
+		rte_memory_order_acquire);
 
 	return wrap_counter == !!(flags & VRING_DESC_F_AVAIL) &&
 		wrap_counter != !!(flags & VRING_DESC_F_USED);
@@ -914,17 +915,19 @@ uint64_t translate_log_addr(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	bool expected = false;
 
 	if (dev->notify_ops->guest_notify) {
-		if (__atomic_compare_exchange_n(&vq->irq_pending, &expected, true, 0,
-				  __ATOMIC_RELEASE, __ATOMIC_RELAXED)) {
+		if (rte_atomic_compare_exchange_strong_explicit(&vq->irq_pending, &expected, true,
+				  rte_memory_order_release, rte_memory_order_relaxed)) {
 			if (dev->notify_ops->guest_notify(dev->vid, vq->index)) {
 				if (dev->flags & VIRTIO_DEV_STATS_ENABLED)
-					__atomic_fetch_add(&vq->stats.guest_notifications_offloaded,
-						1, __ATOMIC_RELAXED);
+					rte_atomic_fetch_add_explicit(
+						&vq->stats.guest_notifications_offloaded,
+						1, rte_memory_order_relaxed);
 				return;
 			}
 
 			/* Offloading failed, fallback to direct IRQ injection */
-			__atomic_store_n(&vq->irq_pending, false, __ATOMIC_RELEASE);
+			rte_atomic_store_explicit(&vq->irq_pending, false,
+				rte_memory_order_release);
 		} else {
 			vq->stats.guest_notifications_suppressed++;
 			return;
@@ -933,14 +936,14 @@ uint64_t translate_log_addr(struct virtio_net *dev, struct vhost_virtqueue *vq,
 
 	if (dev->backend_ops->inject_irq(dev, vq)) {
 		if (dev->flags & VIRTIO_DEV_STATS_ENABLED)
-			__atomic_fetch_add(&vq->stats.guest_notifications_error,
-				1, __ATOMIC_RELAXED);
+			rte_atomic_fetch_add_explicit(&vq->stats.guest_notifications_error,
+				1, rte_memory_order_relaxed);
 		return;
 	}
 
 	if (dev->flags & VIRTIO_DEV_STATS_ENABLED)
-		__atomic_fetch_add(&vq->stats.guest_notifications,
-			1, __ATOMIC_RELAXED);
+		rte_atomic_fetch_add_explicit(&vq->stats.guest_notifications,
+			1, rte_memory_order_relaxed);
 	if (dev->notify_ops->guest_notified)
 		dev->notify_ops->guest_notified(dev->vid);
 }
@@ -949,7 +952,7 @@ uint64_t translate_log_addr(struct virtio_net *dev, struct vhost_virtqueue *vq,
 vhost_vring_call_split(struct virtio_net *dev, struct vhost_virtqueue *vq)
 {
 	/* Flush used->idx update before we read avail->flags. */
-	rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
+	rte_atomic_thread_fence(rte_memory_order_seq_cst);
 
 	/* Don't kick guest if we don't reach index specified by guest. */
 	if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) {
@@ -981,7 +984,7 @@ uint64_t translate_log_addr(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	bool signalled_used_valid, kick = false;
 
 	/* Flush used desc update. */
-	rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
+	rte_atomic_thread_fence(rte_memory_order_seq_cst);
 
 	if (!(dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))) {
 		if (vq->driver_event->flags !=
@@ -1007,7 +1010,7 @@ uint64_t translate_log_addr(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		goto kick;
 	}
 
-	rte_atomic_thread_fence(__ATOMIC_ACQUIRE);
+	rte_atomic_thread_fence(rte_memory_order_acquire);
 
 	off_wrap = vq->driver_event->off_wrap;
 	off = off_wrap & ~(1 << 15);
lib/vhost/vhost_user.c +3 -3
index 901a80b..e363121 100644
@@ -1914,7 +1914,7 @@ static int vhost_user_set_vring_err(struct virtio_net **pdev,
 
 	if (inflight_split->used_idx != used->idx) {
 		inflight_split->desc[last_io].inflight = 0;
-		rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
+		rte_atomic_thread_fence(rte_memory_order_seq_cst);
 		inflight_split->used_idx = used->idx;
 	}
 
@@ -2418,10 +2418,10 @@ static int vhost_user_set_log_fd(struct virtio_net **pdev,
 	 * Set the flag to inject a RARP broadcast packet at
 	 * rte_vhost_dequeue_burst().
 	 *
-	 * __ATOMIC_RELEASE ordering is for making sure the mac is
+	 * rte_memory_order_release ordering is for making sure the mac is
 	 * copied before the flag is set.
 	 */
-	__atomic_store_n(&dev->broadcast_rarp, 1, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&dev->broadcast_rarp, 1, rte_memory_order_release);
 	vdpa_dev = dev->vdpa_dev;
 	if (vdpa_dev && vdpa_dev->ops->migration_done)
 		vdpa_dev->ops->migration_done(dev->vid);
lib/vhost/virtio_net.c +32 -26

166 lines not on this page · raw message

lib/vhost/virtio_net_ctrl.c +4 -2
index 6b583a0..c4847f8 100644
@@ -33,7 +33,8 @@ struct virtio_net_ctrl_elem {
 	uint8_t *ctrl_req;
 	struct vring_desc *descs;
 
-	avail_idx = __atomic_load_n(&cvq->avail->idx, __ATOMIC_ACQUIRE);
+	avail_idx = rte_atomic_load_explicit((unsigned short __rte_atomic *)&cvq->avail->idx,
+		rte_memory_order_acquire);
 	if (avail_idx == cvq->last_avail_idx) {
 		VHOST_LOG_CONFIG(dev->ifname, DEBUG, "Control queue empty\n");
 		return 0;
@@ -236,7 +237,8 @@ struct virtio_net_ctrl_elem {
 	if (cvq->last_used_idx >= cvq->size)
 		cvq->last_used_idx -= cvq->size;
 
-	__atomic_store_n(&cvq->used->idx, cvq->last_used_idx, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit((unsigned short __rte_atomic *)&cvq->used->idx,
+		cvq->last_used_idx, rte_memory_order_release);
 
 	vhost_vring_call_split(dev, dev->cvq);
 
Signature · 1 line
1.8.3.1
Reviewed-by on v3 13/19
#75 lore raw

Re: [PATCH v3 13/19] vhost: use rte optional stdatomic API

Quoted 4 lines · Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

Quoted 9 lines
---
  lib/vhost/vdpa.c            |  3 ++-
  lib/vhost/vhost.c           | 42 ++++++++++++++++----------------
  lib/vhost/vhost.h           | 39 ++++++++++++++++--------------
  lib/vhost/vhost_user.c      |  6 ++---
  lib/vhost/virtio_net.c      | 58 +++++++++++++++++++++++++--------------------
  lib/vhost/virtio_net_ctrl.c |  6 +++--
  6 files changed, 84 insertions(+), 70 deletions(-)

Thanks,
Maxime

#76 lore raw

PATCH v3 14/19 cryptodev: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/cryptodev/rte_cryptodev.c | 22 ++++++++++++----------
 lib/cryptodev/rte_cryptodev.h | 16 ++++++++--------
 2 files changed, 20 insertions(+), 18 deletions(-)
lib/cryptodev/rte_cryptodev.c +12 -10
index 314710b..b258827 100644
@@ -1535,12 +1535,12 @@ struct rte_cryptodev_cb *
 		/* Stores to cb->fn and cb->param should complete before
 		 * cb is visible to data plane.
 		 */
-		__atomic_store_n(&tail->next, cb, __ATOMIC_RELEASE);
+		rte_atomic_store_explicit(&tail->next, cb, rte_memory_order_release);
 	} else {
 		/* Stores to cb->fn and cb->param should complete before
 		 * cb is visible to data plane.
 		 */
-		__atomic_store_n(&list->next, cb, __ATOMIC_RELEASE);
+		rte_atomic_store_explicit(&list->next, cb, rte_memory_order_release);
 	}
 
 	rte_spinlock_unlock(&rte_cryptodev_callback_lock);
@@ -1555,7 +1555,8 @@ struct rte_cryptodev_cb *
 				  struct rte_cryptodev_cb *cb)
 {
 	struct rte_cryptodev *dev;
-	struct rte_cryptodev_cb **prev_cb, *curr_cb;
+	RTE_ATOMIC(struct rte_cryptodev_cb *) *prev_cb;
+	struct rte_cryptodev_cb *curr_cb;
 	struct rte_cryptodev_cb_rcu *list;
 	int ret;
 
@@ -1601,8 +1602,8 @@ struct rte_cryptodev_cb *
 		curr_cb = *prev_cb;
 		if (curr_cb == cb) {
 			/* Remove the user cb from the callback list. */
-			__atomic_store_n(prev_cb, curr_cb->next,
-				__ATOMIC_RELAXED);
+			rte_atomic_store_explicit(prev_cb, curr_cb->next,
+				rte_memory_order_relaxed);
 			ret = 0;
 			break;
 		}
@@ -1673,12 +1674,12 @@ struct rte_cryptodev_cb *
 		/* Stores to cb->fn and cb->param should complete before
 		 * cb is visible to data plane.
 		 */
-		__atomic_store_n(&tail->next, cb, __ATOMIC_RELEASE);
+		rte_atomic_store_explicit(&tail->next, cb, rte_memory_order_release);
 	} else {
 		/* Stores to cb->fn and cb->param should complete before
 		 * cb is visible to data plane.
 		 */
-		__atomic_store_n(&list->next, cb, __ATOMIC_RELEASE);
+		rte_atomic_store_explicit(&list->next, cb, rte_memory_order_release);
 	}
 
 	rte_spinlock_unlock(&rte_cryptodev_callback_lock);
@@ -1694,7 +1695,8 @@ struct rte_cryptodev_cb *
 				  struct rte_cryptodev_cb *cb)
 {
 	struct rte_cryptodev *dev;
-	struct rte_cryptodev_cb **prev_cb, *curr_cb;
+	RTE_ATOMIC(struct rte_cryptodev_cb *) *prev_cb;
+	struct rte_cryptodev_cb *curr_cb;
 	struct rte_cryptodev_cb_rcu *list;
 	int ret;
 
@@ -1740,8 +1742,8 @@ struct rte_cryptodev_cb *
 		curr_cb = *prev_cb;
 		if (curr_cb == cb) {
 			/* Remove the user cb from the callback list. */
-			__atomic_store_n(prev_cb, curr_cb->next,
-				__ATOMIC_RELAXED);
+			rte_atomic_store_explicit(prev_cb, curr_cb->next,
+				rte_memory_order_relaxed);
 			ret = 0;
 			break;
 		}
lib/cryptodev/rte_cryptodev.h +8 -8
index be0698c..9092118 100644
@@ -979,7 +979,7 @@ struct rte_cryptodev_config {
  * queue pair on enqueue/dequeue.
  */
 struct rte_cryptodev_cb {
-	struct rte_cryptodev_cb *next;
+	RTE_ATOMIC(struct rte_cryptodev_cb *) next;
 	/**< Pointer to next callback */
 	rte_cryptodev_callback_fn fn;
 	/**< Pointer to callback function */
@@ -992,7 +992,7 @@ struct rte_cryptodev_cb {
  * Structure used to hold information about the RCU for a queue pair.
  */
 struct rte_cryptodev_cb_rcu {
-	struct rte_cryptodev_cb *next;
+	RTE_ATOMIC(struct rte_cryptodev_cb *) next;
 	/**< Pointer to next callback */
 	struct rte_rcu_qsbr *qsbr;
 	/**< RCU QSBR variable per queue pair */
@@ -1947,15 +1947,15 @@ int rte_cryptodev_remove_deq_callback(uint8_t dev_id,
 		struct rte_cryptodev_cb_rcu *list;
 		struct rte_cryptodev_cb *cb;
 
-		/* __ATOMIC_RELEASE memory order was used when the
+		/* rte_memory_order_release memory order was used when the
 		 * call back was inserted into the list.
 		 * Since there is a clear dependency between loading
-		 * cb and cb->fn/cb->next, __ATOMIC_ACQUIRE memory order is
+		 * cb and cb->fn/cb->next, rte_memory_order_acquire memory order is
 		 * not required.
 		 */
 		list = &fp_ops->qp.deq_cb[qp_id];
 		rte_rcu_qsbr_thread_online(list->qsbr, 0);
-		cb = __atomic_load_n(&list->next, __ATOMIC_RELAXED);
+		cb = rte_atomic_load_explicit(&list->next, rte_memory_order_relaxed);
 
 		while (cb != NULL) {
 			nb_ops = cb->fn(dev_id, qp_id, ops, nb_ops,
@@ -2014,15 +2014,15 @@ int rte_cryptodev_remove_deq_callback(uint8_t dev_id,
 		struct rte_cryptodev_cb_rcu *list;
 		struct rte_cryptodev_cb *cb;
 
-		/* __ATOMIC_RELEASE memory order was used when the
+		/* rte_memory_order_release memory order was used when the
 		 * call back was inserted into the list.
 		 * Since there is a clear dependency between loading
-		 * cb and cb->fn/cb->next, __ATOMIC_ACQUIRE memory order is
+		 * cb and cb->fn/cb->next, rte_memory_order_acquire memory order is
 		 * not required.
 		 */
 		list = &fp_ops->qp.enq_cb[qp_id];
 		rte_rcu_qsbr_thread_online(list->qsbr, 0);
-		cb = __atomic_load_n(&list->next, __ATOMIC_RELAXED);
+		cb = rte_atomic_load_explicit(&list->next, rte_memory_order_relaxed);
 
 		while (cb != NULL) {
 			nb_ops = cb->fn(dev_id, qp_id, ops, nb_ops,
Signature · 1 line
1.8.3.1
#78 lore raw

PATCH v3 15/19 distributor: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/distributor/distributor_private.h |  4 +--
 lib/distributor/rte_distributor.c     | 54 +++++++++++++++++------------------
 2 files changed, 29 insertions(+), 29 deletions(-)
lib/distributor/distributor_private.h +2 -2
index 2f29343..dfeb9b5 100644
@@ -113,12 +113,12 @@ enum rte_distributor_match_function {
  * There is a separate cacheline for returns in the burst API.
  */
 struct rte_distributor_buffer {
-	volatile int64_t bufptr64[RTE_DIST_BURST_SIZE]
+	volatile RTE_ATOMIC(int64_t) bufptr64[RTE_DIST_BURST_SIZE]
 		__rte_cache_aligned; /* <= outgoing to worker */
 
 	int64_t pad1 __rte_cache_aligned;    /* <= one cache line  */
 
-	volatile int64_t retptr64[RTE_DIST_BURST_SIZE]
+	volatile RTE_ATOMIC(int64_t) retptr64[RTE_DIST_BURST_SIZE]
 		__rte_cache_aligned; /* <= incoming from worker */
 
 	int64_t pad2 __rte_cache_aligned;    /* <= one cache line  */
lib/distributor/rte_distributor.c +27 -27

161 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#79 lore raw

PATCH v3 16/19 ethdev: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/ethdev/ethdev_driver.h   | 16 ++++++++--------
 lib/ethdev/ethdev_private.c  |  6 +++---
 lib/ethdev/rte_ethdev.c      | 24 ++++++++++++------------
 lib/ethdev/rte_ethdev.h      | 16 ++++++++--------
 lib/ethdev/rte_ethdev_core.h |  2 +-
 5 files changed, 32 insertions(+), 32 deletions(-)
lib/ethdev/ethdev_driver.h +8 -8
index deb23ad..b482cd1 100644
@@ -30,7 +30,7 @@
  * queue on Rx and Tx.
  */
 struct rte_eth_rxtx_callback {
-	struct rte_eth_rxtx_callback *next;
+	RTE_ATOMIC(struct rte_eth_rxtx_callback *) next;
 	union{
 		rte_rx_callback_fn rx;
 		rte_tx_callback_fn tx;
@@ -80,12 +80,12 @@ struct rte_eth_dev {
 	 * User-supplied functions called from rx_burst to post-process
 	 * received packets before passing them to the user
 	 */
-	struct rte_eth_rxtx_callback *post_rx_burst_cbs[RTE_MAX_QUEUES_PER_PORT];
+	RTE_ATOMIC(struct rte_eth_rxtx_callback *) post_rx_burst_cbs[RTE_MAX_QUEUES_PER_PORT];
 	/**
 	 * User-supplied functions called from tx_burst to pre-process
 	 * received packets before passing them to the driver for transmission
 	 */
-	struct rte_eth_rxtx_callback *pre_tx_burst_cbs[RTE_MAX_QUEUES_PER_PORT];
+	RTE_ATOMIC(struct rte_eth_rxtx_callback *) pre_tx_burst_cbs[RTE_MAX_QUEUES_PER_PORT];
 
 	enum rte_eth_dev_state state; /**< Flag indicating the port state */
 	void *security_ctx; /**< Context for security ops */
@@ -1655,7 +1655,7 @@ int rte_eth_dev_callback_process(struct rte_eth_dev *dev,
 rte_eth_linkstatus_set(struct rte_eth_dev *dev,
 		       const struct rte_eth_link *new_link)
 {
-	uint64_t *dev_link = (uint64_t *)&(dev->data->dev_link);
+	RTE_ATOMIC(uint64_t) *dev_link = (uint64_t __rte_atomic *)&(dev->data->dev_link);
 	union {
 		uint64_t val64;
 		struct rte_eth_link link;
@@ -1663,8 +1663,8 @@ int rte_eth_dev_callback_process(struct rte_eth_dev *dev,
 
 	RTE_BUILD_BUG_ON(sizeof(*new_link) != sizeof(uint64_t));
 
-	orig.val64 = __atomic_exchange_n(dev_link, *(const uint64_t *)new_link,
-					__ATOMIC_SEQ_CST);
+	orig.val64 = rte_atomic_exchange_explicit(dev_link, *(const uint64_t *)new_link,
+					rte_memory_order_seq_cst);
 
 	return (orig.link.link_status == new_link->link_status) ? -1 : 0;
 }
@@ -1682,12 +1682,12 @@ int rte_eth_dev_callback_process(struct rte_eth_dev *dev,
 rte_eth_linkstatus_get(const struct rte_eth_dev *dev,
 		       struct rte_eth_link *link)
 {
-	uint64_t *src = (uint64_t *)&(dev->data->dev_link);
+	RTE_ATOMIC(uint64_t) *src = (uint64_t __rte_atomic *)&(dev->data->dev_link);
 	uint64_t *dst = (uint64_t *)link;
 
 	RTE_BUILD_BUG_ON(sizeof(*link) != sizeof(uint64_t));
 
-	*dst = __atomic_load_n(src, __ATOMIC_SEQ_CST);
+	*dst = rte_atomic_load_explicit(src, rte_memory_order_seq_cst);
 }
 
 /**
lib/ethdev/ethdev_private.c +3 -3
index 7cc7f28..82e2568 100644
@@ -245,7 +245,7 @@ struct dummy_queue {
 void
 eth_dev_fp_ops_reset(struct rte_eth_fp_ops *fpo)
 {
-	static void *dummy_data[RTE_MAX_QUEUES_PER_PORT];
+	static RTE_ATOMIC(void *) dummy_data[RTE_MAX_QUEUES_PER_PORT];
 	uintptr_t port_id = fpo - rte_eth_fp_ops;
 
 	per_port_queues[port_id].rx_warn_once = false;
@@ -278,10 +278,10 @@ struct dummy_queue {
 	fpo->recycle_rx_descriptors_refill = dev->recycle_rx_descriptors_refill;
 
 	fpo->rxq.data = dev->data->rx_queues;
-	fpo->rxq.clbk = (void **)(uintptr_t)dev->post_rx_burst_cbs;
+	fpo->rxq.clbk = (void * __rte_atomic *)(uintptr_t)dev->post_rx_burst_cbs;
 
 	fpo->txq.data = dev->data->tx_queues;
-	fpo->txq.clbk = (void **)(uintptr_t)dev->pre_tx_burst_cbs;
+	fpo->txq.clbk = (void * __rte_atomic *)(uintptr_t)dev->pre_tx_burst_cbs;
 }
 
 uint16_t
lib/ethdev/rte_ethdev.c +12 -12
index 9dabcb5..af23ac0 100644
@@ -5654,9 +5654,9 @@ int rte_eth_rx_avail_thresh_query(uint16_t port_id, uint16_t *queue_id,
 		/* Stores to cb->fn and cb->param should complete before
 		 * cb is visible to data plane.
 		 */
-		__atomic_store_n(
+		rte_atomic_store_explicit(
 			&rte_eth_devices[port_id].post_rx_burst_cbs[queue_id],
-			cb, __ATOMIC_RELEASE);
+			cb, rte_memory_order_release);
 
 	} else {
 		while (tail->next)
@@ -5664,7 +5664,7 @@ int rte_eth_rx_avail_thresh_query(uint16_t port_id, uint16_t *queue_id,
 		/* Stores to cb->fn and cb->param should complete before
 		 * cb is visible to data plane.
 		 */
-		__atomic_store_n(&tail->next, cb, __ATOMIC_RELEASE);
+		rte_atomic_store_explicit(&tail->next, cb, rte_memory_order_release);
 	}
 	rte_spinlock_unlock(&eth_dev_rx_cb_lock);
 
@@ -5704,9 +5704,9 @@ int rte_eth_rx_avail_thresh_query(uint16_t port_id, uint16_t *queue_id,
 	/* Stores to cb->fn, cb->param and cb->next should complete before
 	 * cb is visible to data plane threads.
 	 */
-	__atomic_store_n(
+	rte_atomic_store_explicit(
 		&rte_eth_devices[port_id].post_rx_burst_cbs[queue_id],
-		cb, __ATOMIC_RELEASE);
+		cb, rte_memory_order_release);
 	rte_spinlock_unlock(&eth_dev_rx_cb_lock);
 
 	rte_eth_trace_add_first_rx_callback(port_id, queue_id, fn, user_param,
@@ -5757,9 +5757,9 @@ int rte_eth_rx_avail_thresh_query(uint16_t port_id, uint16_t *queue_id,
 		/* Stores to cb->fn and cb->param should complete before
 		 * cb is visible to data plane.
 		 */
-		__atomic_store_n(
+		rte_atomic_store_explicit(
 			&rte_eth_devices[port_id].pre_tx_burst_cbs[queue_id],
-			cb, __ATOMIC_RELEASE);
+			cb, rte_memory_order_release);
 
 	} else {
 		while (tail->next)
@@ -5767,7 +5767,7 @@ int rte_eth_rx_avail_thresh_query(uint16_t port_id, uint16_t *queue_id,
 		/* Stores to cb->fn and cb->param should complete before
 		 * cb is visible to data plane.
 		 */
-		__atomic_store_n(&tail->next, cb, __ATOMIC_RELEASE);
+		rte_atomic_store_explicit(&tail->next, cb, rte_memory_order_release);
 	}
 	rte_spinlock_unlock(&eth_dev_tx_cb_lock);
 
@@ -5791,7 +5791,7 @@ int rte_eth_rx_avail_thresh_query(uint16_t port_id, uint16_t *queue_id,
 
 	struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 	struct rte_eth_rxtx_callback *cb;
-	struct rte_eth_rxtx_callback **prev_cb;
+	RTE_ATOMIC(struct rte_eth_rxtx_callback *) *prev_cb;
 	int ret = -EINVAL;
 
 	rte_spinlock_lock(&eth_dev_rx_cb_lock);
@@ -5800,7 +5800,7 @@ int rte_eth_rx_avail_thresh_query(uint16_t port_id, uint16_t *queue_id,
 		cb = *prev_cb;
 		if (cb == user_cb) {
 			/* Remove the user cb from the callback list. */
-			__atomic_store_n(prev_cb, cb->next, __ATOMIC_RELAXED);
+			rte_atomic_store_explicit(prev_cb, cb->next, rte_memory_order_relaxed);
 			ret = 0;
 			break;
 		}
@@ -5828,7 +5828,7 @@ int rte_eth_rx_avail_thresh_query(uint16_t port_id, uint16_t *queue_id,
 	struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 	int ret = -EINVAL;
 	struct rte_eth_rxtx_callback *cb;
-	struct rte_eth_rxtx_callback **prev_cb;
+	RTE_ATOMIC(struct rte_eth_rxtx_callback *) *prev_cb;
 
 	rte_spinlock_lock(&eth_dev_tx_cb_lock);
 	prev_cb = &dev->pre_tx_burst_cbs[queue_id];
@@ -5836,7 +5836,7 @@ int rte_eth_rx_avail_thresh_query(uint16_t port_id, uint16_t *queue_id,
 		cb = *prev_cb;
 		if (cb == user_cb) {
 			/* Remove the user cb from the callback list. */
-			__atomic_store_n(prev_cb, cb->next, __ATOMIC_RELAXED);
+			rte_atomic_store_explicit(prev_cb, cb->next, rte_memory_order_relaxed);
 			ret = 0;
 			break;
 		}
lib/ethdev/rte_ethdev.h +8 -8
index 85b9af7..d1c10f2 100644
@@ -6023,14 +6023,14 @@ uint16_t rte_eth_call_rx_callbacks(uint16_t port_id, uint16_t queue_id,
 	{
 		void *cb;
 
-		/* __ATOMIC_RELEASE memory order was used when the
+		/* rte_memory_order_release memory order was used when the
 		 * call back was inserted into the list.
 		 * Since there is a clear dependency between loading
-		 * cb and cb->fn/cb->next, __ATOMIC_ACQUIRE memory order is
+		 * cb and cb->fn/cb->next, rte_memory_order_acquire memory order is
 		 * not required.
 		 */
-		cb = __atomic_load_n((void **)&p->rxq.clbk[queue_id],
-				__ATOMIC_RELAXED);
+		cb = rte_atomic_load_explicit(&p->rxq.clbk[queue_id],
+				rte_memory_order_relaxed);
 		if (unlikely(cb != NULL))
 			nb_rx = rte_eth_call_rx_callbacks(port_id, queue_id,
 					rx_pkts, nb_rx, nb_pkts, cb);
@@ -6360,14 +6360,14 @@ uint16_t rte_eth_call_tx_callbacks(uint16_t port_id, uint16_t queue_id,
 	{
 		void *cb;
 
-		/* __ATOMIC_RELEASE memory order was used when the
+		/* rte_memory_order_release memory order was used when the
 		 * call back was inserted into the list.
 		 * Since there is a clear dependency between loading
-		 * cb and cb->fn/cb->next, __ATOMIC_ACQUIRE memory order is
+		 * cb and cb->fn/cb->next, rte_memory_order_acquire memory order is
 		 * not required.
 		 */
-		cb = __atomic_load_n((void **)&p->txq.clbk[queue_id],
-				__ATOMIC_RELAXED);
+		cb = rte_atomic_load_explicit(&p->txq.clbk[queue_id],
+				rte_memory_order_relaxed);
 		if (unlikely(cb != NULL))
 			nb_pkts = rte_eth_call_tx_callbacks(port_id, queue_id,
 					tx_pkts, nb_pkts, cb);
lib/ethdev/rte_ethdev_core.h +1 -1
index 32f5f73..4bfaf79 100644
@@ -71,7 +71,7 @@ struct rte_ethdev_qdata {
 	/** points to array of internal queue data pointers */
 	void **data;
 	/** points to array of queue callback data pointers */
-	void **clbk;
+	RTE_ATOMIC(void *) *clbk;
 };
 
 /**
Signature · 1 line
1.8.3.1
#80 lore raw

PATCH v3 17/19 hash: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/hash/rte_cuckoo_hash.c | 116 ++++++++++++++++++++++-----------------------
 lib/hash/rte_cuckoo_hash.h |   6 +--
 2 files changed, 61 insertions(+), 61 deletions(-)
lib/hash/rte_cuckoo_hash.c +58 -58
index 19b23f2..b2cf60d 100644
@@ -149,7 +149,7 @@ struct rte_hash *
 	unsigned int writer_takes_lock = 0;
 	unsigned int no_free_on_del = 0;
 	uint32_t *ext_bkt_to_free = NULL;
-	uint32_t *tbl_chng_cnt = NULL;
+	RTE_ATOMIC(uint32_t) *tbl_chng_cnt = NULL;
 	struct lcore_cache *local_free_slots = NULL;
 	unsigned int readwrite_concur_lf_support = 0;
 	uint32_t i;
@@ -713,9 +713,9 @@ struct rte_hash *
 				 * variable. Release the application data
 				 * to the readers.
 				 */
-				__atomic_store_n(&k->pdata,
+				rte_atomic_store_explicit(&k->pdata,
 					data,
-					__ATOMIC_RELEASE);
+					rte_memory_order_release);
 				/*
 				 * Return index where key is stored,
 				 * subtracting the first dummy index
@@ -776,9 +776,9 @@ struct rte_hash *
 			 * key_idx is the guard variable for signature
 			 * and key.
 			 */
-			__atomic_store_n(&prim_bkt->key_idx[i],
+			rte_atomic_store_explicit(&prim_bkt->key_idx[i],
 					 new_idx,
-					 __ATOMIC_RELEASE);
+					 rte_memory_order_release);
 			break;
 		}
 	}
@@ -851,9 +851,9 @@ struct rte_hash *
 		if (unlikely(&h->buckets[prev_alt_bkt_idx]
 				!= curr_bkt)) {
 			/* revert it to empty, otherwise duplicated keys */
-			__atomic_store_n(&curr_bkt->key_idx[curr_slot],
+			rte_atomic_store_explicit(&curr_bkt->key_idx[curr_slot],
 				EMPTY_SLOT,
-				__ATOMIC_RELEASE);
+				rte_memory_order_release);
 			__hash_rw_writer_unlock(h);
 			return -1;
 		}
@@ -865,13 +865,13 @@ struct rte_hash *
 			 * Since there is one writer, load acquires on
 			 * tbl_chng_cnt are not required.
 			 */
-			__atomic_store_n(h->tbl_chng_cnt,
+			rte_atomic_store_explicit(h->tbl_chng_cnt,
 					 *h->tbl_chng_cnt + 1,
-					 __ATOMIC_RELEASE);
+					 rte_memory_order_release);
 			/* The store to sig_current should not
 			 * move above the store to tbl_chng_cnt.
 			 */
-			__atomic_thread_fence(__ATOMIC_RELEASE);
+			__atomic_thread_fence(rte_memory_order_release);
 		}
 
 		/* Need to swap current/alt sig to allow later
@@ -881,9 +881,9 @@ struct rte_hash *
 		curr_bkt->sig_current[curr_slot] =
 			prev_bkt->sig_current[prev_slot];
 		/* Release the updated bucket entry */
-		__atomic_store_n(&curr_bkt->key_idx[curr_slot],
+		rte_atomic_store_explicit(&curr_bkt->key_idx[curr_slot],
 			prev_bkt->key_idx[prev_slot],
-			__ATOMIC_RELEASE);
+			rte_memory_order_release);
 
 		curr_slot = prev_slot;
 		curr_node = prev_node;
@@ -897,20 +897,20 @@ struct rte_hash *
 		 * Since there is one writer, load acquires on
 		 * tbl_chng_cnt are not required.
 		 */
-		__atomic_store_n(h->tbl_chng_cnt,
+		rte_atomic_store_explicit(h->tbl_chng_cnt,
 				 *h->tbl_chng_cnt + 1,
-				 __ATOMIC_RELEASE);
+				 rte_memory_order_release);
 		/* The store to sig_current should not
 		 * move above the store to tbl_chng_cnt.
 		 */
-		__atomic_thread_fence(__ATOMIC_RELEASE);
+		__atomic_thread_fence(rte_memory_order_release);
 	}
 
 	curr_bkt->sig_current[curr_slot] = sig;
 	/* Release the new bucket entry */
-	__atomic_store_n(&curr_bkt->key_idx[curr_slot],
+	rte_atomic_store_explicit(&curr_bkt->key_idx[curr_slot],
 			 new_idx,
-			 __ATOMIC_RELEASE);
+			 rte_memory_order_release);
 
 	__hash_rw_writer_unlock(h);
 
@@ -1076,9 +1076,9 @@ struct rte_hash *
 	 * not leak after the store of pdata in the key store. i.e. pdata is
 	 * the guard variable. Release the application data to the readers.
 	 */
-	__atomic_store_n(&new_k->pdata,
+	rte_atomic_store_explicit(&new_k->pdata,
 		data,
-		__ATOMIC_RELEASE);
+		rte_memory_order_release);
 	/* Copy key */
 	memcpy(new_k->key, key, h->key_len);
 
@@ -1149,9 +1149,9 @@ struct rte_hash *
 				 * key_idx is the guard variable for signature
 				 * and key.
 				 */
-				__atomic_store_n(&cur_bkt->key_idx[i],
+				rte_atomic_store_explicit(&cur_bkt->key_idx[i],
 						 slot_id,
-						 __ATOMIC_RELEASE);
+						 rte_memory_order_release);
 				__hash_rw_writer_unlock(h);
 				return slot_id - 1;
 			}
@@ -1185,9 +1185,9 @@ struct rte_hash *
 	 * the store to key_idx. i.e. key_idx is the guard variable
 	 * for signature and key.
 	 */
-	__atomic_store_n(&(h->buckets_ext[ext_bkt_id - 1]).key_idx[0],
+	rte_atomic_store_explicit(&(h->buckets_ext[ext_bkt_id - 1]).key_idx[0],
 			 slot_id,
-			 __ATOMIC_RELEASE);
+			 rte_memory_order_release);
 	/* Link the new bucket to sec bucket linked list */
 	last = rte_hash_get_last_bkt(sec_bkt);
 	last->next = &h->buckets_ext[ext_bkt_id - 1];
@@ -1290,17 +1290,17 @@ struct rte_hash *
 		 * key comparison will ensure that the lookup fails.
 		 */
 		if (bkt->sig_current[i] == sig) {
-			key_idx = __atomic_load_n(&bkt->key_idx[i],
-					  __ATOMIC_ACQUIRE);
+			key_idx = rte_atomic_load_explicit(&bkt->key_idx[i],
+					  rte_memory_order_acquire);
 			if (key_idx != EMPTY_SLOT) {
 				k = (struct rte_hash_key *) ((char *)keys +
 						key_idx * h->key_entry_size);
 
 				if (rte_hash_cmp_eq(key, k->key, h) == 0) {
 					if (data != NULL) {
-						*data = __atomic_load_n(
+						*data = rte_atomic_load_explicit(
 							&k->pdata,
-							__ATOMIC_ACQUIRE);
+							rte_memory_order_acquire);
 					}
 					/*
 					 * Return index where key is stored,
@@ -1374,8 +1374,8 @@ struct rte_hash *
 		 * starts. Acquire semantics will make sure that
 		 * loads in search_one_bucket are not hoisted.
 		 */
-		cnt_b = __atomic_load_n(h->tbl_chng_cnt,
-				__ATOMIC_ACQUIRE);
+		cnt_b = rte_atomic_load_explicit(h->tbl_chng_cnt,
+				rte_memory_order_acquire);
 
 		/* Check if key is in primary location */
 		bkt = &h->buckets[prim_bucket_idx];
@@ -1396,7 +1396,7 @@ struct rte_hash *
 		/* The loads of sig_current in search_one_bucket
 		 * should not move below the load from tbl_chng_cnt.
 		 */
-		__atomic_thread_fence(__ATOMIC_ACQUIRE);
+		__atomic_thread_fence(rte_memory_order_acquire);
 		/* Re-read the table change counter to check if the
 		 * table has changed during search. If yes, re-do
 		 * the search.
@@ -1405,8 +1405,8 @@ struct rte_hash *
 		 * and key index in secondary bucket will make sure
 		 * that it does not get hoisted.
 		 */
-		cnt_a = __atomic_load_n(h->tbl_chng_cnt,
-					__ATOMIC_ACQUIRE);
+		cnt_a = rte_atomic_load_explicit(h->tbl_chng_cnt,
+					rte_memory_order_acquire);
 	} while (cnt_b != cnt_a);
 
 	return -ENOENT;
@@ -1611,26 +1611,26 @@ struct rte_hash *
 	for (i = RTE_HASH_BUCKET_ENTRIES - 1; i >= 0; i--) {
 		if (last_bkt->key_idx[i] != EMPTY_SLOT) {
 			cur_bkt->sig_current[pos] = last_bkt->sig_current[i];
-			__atomic_store_n(&cur_bkt->key_idx[pos],
+			rte_atomic_store_explicit(&cur_bkt->key_idx[pos],
 					 last_bkt->key_idx[i],
-					 __ATOMIC_RELEASE);
+					 rte_memory_order_release);
 			if (h->readwrite_concur_lf_support) {
 				/* Inform the readers that the table has changed
 				 * Since there is one writer, load acquire on
 				 * tbl_chng_cnt is not required.
 				 */
-				__atomic_store_n(h->tbl_chng_cnt,
+				rte_atomic_store_explicit(h->tbl_chng_cnt,
 					 *h->tbl_chng_cnt + 1,
-					 __ATOMIC_RELEASE);
+					 rte_memory_order_release);
 				/* The store to sig_current should
 				 * not move above the store to tbl_chng_cnt.
 				 */
-				__atomic_thread_fence(__ATOMIC_RELEASE);
+				__atomic_thread_fence(rte_memory_order_release);
 			}
 			last_bkt->sig_current[i] = NULL_SIGNATURE;
-			__atomic_store_n(&last_bkt->key_idx[i],
+			rte_atomic_store_explicit(&last_bkt->key_idx[i],
 					 EMPTY_SLOT,
-					 __ATOMIC_RELEASE);
+					 rte_memory_order_release);
 			return;
 		}
 	}
@@ -1650,8 +1650,8 @@ struct rte_hash *
 
 	/* Check if key is in bucket */
 	for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) {
-		key_idx = __atomic_load_n(&bkt->key_idx[i],
-					  __ATOMIC_ACQUIRE);
+		key_idx = rte_atomic_load_explicit(&bkt->key_idx[i],
+					  rte_memory_order_acquire);
 		if (bkt->sig_current[i] == sig && key_idx != EMPTY_SLOT) {
 			k = (struct rte_hash_key *) ((char *)keys +
 					key_idx * h->key_entry_size);
@@ -1663,9 +1663,9 @@ struct rte_hash *
 				if (!h->no_free_on_del)
 					remove_entry(h, bkt, i);
 
-				__atomic_store_n(&bkt->key_idx[i],
+				rte_atomic_store_explicit(&bkt->key_idx[i],
 						 EMPTY_SLOT,
-						 __ATOMIC_RELEASE);
+						 rte_memory_order_release);
 
 				*pos = i;
 				/*
@@ -2077,8 +2077,8 @@ struct rte_hash *
 		 * starts. Acquire semantics will make sure that
 		 * loads in compare_signatures are not hoisted.
 		 */
-		cnt_b = __atomic_load_n(h->tbl_chng_cnt,
-					__ATOMIC_ACQUIRE);
+		cnt_b = rte_atomic_load_explicit(h->tbl_chng_cnt,
+					rte_memory_order_acquire);
 
 		/* Compare signatures and prefetch key slot of first hit */
 		for (i = 0; i < num_keys; i++) {
@@ -2121,9 +2121,9 @@ struct rte_hash *
 						__builtin_ctzl(prim_hitmask[i])
 						>> 1;
 				uint32_t key_idx =
-				__atomic_load_n(
+				rte_atomic_load_explicit(
 					&primary_bkt[i]->key_idx[hit_index],
-					__ATOMIC_ACQUIRE);
+					rte_memory_order_acquire);
 				const struct rte_hash_key *key_slot =
 					(const struct rte_hash_key *)(
 					(const char *)h->key_store +
@@ -2137,9 +2137,9 @@ struct rte_hash *
 					!rte_hash_cmp_eq(
 						key_slot->key, keys[i], h)) {
 					if (data != NULL)
-						data[i] = __atomic_load_n(
+						data[i] = rte_atomic_load_explicit(
 							&key_slot->pdata,
-							__ATOMIC_ACQUIRE);
+							rte_memory_order_acquire);
 
 					hits |= 1ULL << i;
 					positions[i] = key_idx - 1;
@@ -2153,9 +2153,9 @@ struct rte_hash *
 						__builtin_ctzl(sec_hitmask[i])
 						>> 1;
 				uint32_t key_idx =
-				__atomic_load_n(
+				rte_atomic_load_explicit(
 					&secondary_bkt[i]->key_idx[hit_index],
-					__ATOMIC_ACQUIRE);
+					rte_memory_order_acquire);
 				const struct rte_hash_key *key_slot =
 					(const struct rte_hash_key *)(
 					(const char *)h->key_store +
@@ -2170,9 +2170,9 @@ struct rte_hash *
 					!rte_hash_cmp_eq(
 						key_slot->key, keys[i], h)) {
 					if (data != NULL)
-						data[i] = __atomic_load_n(
+						data[i] = rte_atomic_load_explicit(
 							&key_slot->pdata,
-							__ATOMIC_ACQUIRE);
+							rte_memory_order_acquire);
 
 					hits |= 1ULL << i;
 					positions[i] = key_idx - 1;
@@ -2216,7 +2216,7 @@ struct rte_hash *
 		/* The loads of sig_current in compare_signatures
 		 * should not move below the load from tbl_chng_cnt.
 		 */
-		__atomic_thread_fence(__ATOMIC_ACQUIRE);
+		__atomic_thread_fence(rte_memory_order_acquire);
 		/* Re-read the table change counter to check if the
 		 * table has changed during search. If yes, re-do
 		 * the search.
@@ -2225,8 +2225,8 @@ struct rte_hash *
 		 * key index will make sure that it does not get
 		 * hoisted.
 		 */
-		cnt_a = __atomic_load_n(h->tbl_chng_cnt,
-					__ATOMIC_ACQUIRE);
+		cnt_a = rte_atomic_load_explicit(h->tbl_chng_cnt,
+					rte_memory_order_acquire);
 	} while (cnt_b != cnt_a);
 
 	if (hit_mask != NULL)
@@ -2498,8 +2498,8 @@ struct rte_hash *
 	idx = *next % RTE_HASH_BUCKET_ENTRIES;
 
 	/* If current position is empty, go to the next one */
-	while ((position = __atomic_load_n(&h->buckets[bucket_idx].key_idx[idx],
-					__ATOMIC_ACQUIRE)) == EMPTY_SLOT) {
+	while ((position = rte_atomic_load_explicit(&h->buckets[bucket_idx].key_idx[idx],
+					rte_memory_order_acquire)) == EMPTY_SLOT) {
 		(*next)++;
 		/* End of table */
 		if (*next == total_entries_main)
lib/hash/rte_cuckoo_hash.h +3 -3
index eb2644f..f7afc4d 100644
@@ -137,7 +137,7 @@ struct lcore_cache {
 struct rte_hash_key {
 	union {
 		uintptr_t idata;
-		void *pdata;
+		RTE_ATOMIC(void *) pdata;
 	};
 	/* Variable key size */
 	char key[0];
@@ -155,7 +155,7 @@ enum rte_hash_sig_compare_function {
 struct rte_hash_bucket {
 	uint16_t sig_current[RTE_HASH_BUCKET_ENTRIES];
 
-	uint32_t key_idx[RTE_HASH_BUCKET_ENTRIES];
+	RTE_ATOMIC(uint32_t) key_idx[RTE_HASH_BUCKET_ENTRIES];
 
 	uint8_t flag[RTE_HASH_BUCKET_ENTRIES];
 
@@ -229,7 +229,7 @@ struct rte_hash {
 	 * is piggy-backed to freeing of the key index.
 	 */
 	uint32_t *ext_bkt_to_free;
-	uint32_t *tbl_chng_cnt;
+	RTE_ATOMIC(uint32_t) *tbl_chng_cnt;
 	/**< Indicates if the hash table changed from last read. */
 } __rte_cache_aligned;
 
Signature · 1 line
1.8.3.1
#81 lore raw

PATCH v3 18/19 timer: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 lib/timer/rte_timer.c | 50 +++++++++++++++++++++++++-------------------------
 lib/timer/rte_timer.h |  6 +++---
 2 files changed, 28 insertions(+), 28 deletions(-)
lib/timer/rte_timer.c +25 -25
index 85d6757..53ed221 100644
@@ -210,7 +210,7 @@ struct rte_timer_data {
 
 	status.state = RTE_TIMER_STOP;
 	status.owner = RTE_TIMER_NO_OWNER;
-	__atomic_store_n(&tim->status.u32, status.u32, __ATOMIC_RELAXED);
+	rte_atomic_store_explicit(&tim->status.u32, status.u32, rte_memory_order_relaxed);
 }
 
 /*
@@ -231,7 +231,7 @@ struct rte_timer_data {
 
 	/* wait that the timer is in correct status before update,
 	 * and mark it as being configured */
-	prev_status.u32 = __atomic_load_n(&tim->status.u32, __ATOMIC_RELAXED);
+	prev_status.u32 = rte_atomic_load_explicit(&tim->status.u32, rte_memory_order_relaxed);
 
 	while (success == 0) {
 		/* timer is running on another core
@@ -254,11 +254,11 @@ struct rte_timer_data {
 		 * timer is in CONFIG state, the state cannot be changed
 		 * by other threads. So, we should use ACQUIRE here.
 		 */
-		success = __atomic_compare_exchange_n(&tim->status.u32,
-					      &prev_status.u32,
-					      status.u32, 0,
-					      __ATOMIC_ACQUIRE,
-					      __ATOMIC_RELAXED);
+		success = rte_atomic_compare_exchange_strong_explicit(&tim->status.u32,
+					      (uint32_t *)(uintptr_t)&prev_status.u32,
+					      status.u32,
+					      rte_memory_order_acquire,
+					      rte_memory_order_relaxed);
 	}
 
 	ret_prev_status->u32 = prev_status.u32;
@@ -277,7 +277,7 @@ struct rte_timer_data {
 
 	/* wait that the timer is in correct status before update,
 	 * and mark it as running */
-	prev_status.u32 = __atomic_load_n(&tim->status.u32, __ATOMIC_RELAXED);
+	prev_status.u32 = rte_atomic_load_explicit(&tim->status.u32, rte_memory_order_relaxed);
 
 	while (success == 0) {
 		/* timer is not pending anymore */
@@ -293,11 +293,11 @@ struct rte_timer_data {
 		 * timer is in RUNNING state, the state cannot be changed
 		 * by other threads. So, we should use ACQUIRE here.
 		 */
-		success = __atomic_compare_exchange_n(&tim->status.u32,
-					      &prev_status.u32,
-					      status.u32, 0,
-					      __ATOMIC_ACQUIRE,
-					      __ATOMIC_RELAXED);
+		success = rte_atomic_compare_exchange_strong_explicit(&tim->status.u32,
+					      (uint32_t *)(uintptr_t)&prev_status.u32,
+					      status.u32,
+					      rte_memory_order_acquire,
+					      rte_memory_order_relaxed);
 	}
 
 	return 0;
@@ -530,7 +530,7 @@ struct rte_timer_data {
 	/* The "RELEASE" ordering guarantees the memory operations above
 	 * the status update are observed before the update by all threads
 	 */
-	__atomic_store_n(&tim->status.u32, status.u32, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&tim->status.u32, status.u32, rte_memory_order_release);
 
 	if (tim_lcore != lcore_id || !local_is_locked)
 		rte_spinlock_unlock(&priv_timer[tim_lcore].list_lock);
@@ -612,7 +612,7 @@ struct rte_timer_data {
 	/* The "RELEASE" ordering guarantees the memory operations above
 	 * the status update are observed before the update by all threads
 	 */
-	__atomic_store_n(&tim->status.u32, status.u32, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&tim->status.u32, status.u32, rte_memory_order_release);
 
 	return 0;
 }
@@ -646,8 +646,8 @@ struct rte_timer_data {
 int
 rte_timer_pending(struct rte_timer *tim)
 {
-	return __atomic_load_n(&tim->status.state,
-				__ATOMIC_RELAXED) == RTE_TIMER_PENDING;
+	return rte_atomic_load_explicit(&tim->status.state,
+				rte_memory_order_relaxed) == RTE_TIMER_PENDING;
 }
 
 /* must be called periodically, run all timer that expired */
@@ -753,8 +753,8 @@ struct rte_timer_data {
 			 * operations above the status update are observed
 			 * before the update by all threads
 			 */
-			__atomic_store_n(&tim->status.u32, status.u32,
-				__ATOMIC_RELEASE);
+			rte_atomic_store_explicit(&tim->status.u32, status.u32,
+				rte_memory_order_release);
 		}
 		else {
 			/* keep it in list and mark timer as pending */
@@ -766,8 +766,8 @@ struct rte_timer_data {
 			 * operations above the status update are observed
 			 * before the update by all threads
 			 */
-			__atomic_store_n(&tim->status.u32, status.u32,
-				__ATOMIC_RELEASE);
+			rte_atomic_store_explicit(&tim->status.u32, status.u32,
+				rte_memory_order_release);
 			__rte_timer_reset(tim, tim->expire + tim->period,
 				tim->period, lcore_id, tim->f, tim->arg, 1,
 				timer_data);
@@ -941,8 +941,8 @@ struct rte_timer_data {
 			 * operations above the status update are observed
 			 * before the update by all threads
 			 */
-			__atomic_store_n(&tim->status.u32, status.u32,
-				__ATOMIC_RELEASE);
+			rte_atomic_store_explicit(&tim->status.u32, status.u32,
+				rte_memory_order_release);
 		} else {
 			/* keep it in list and mark timer as pending */
 			rte_spinlock_lock(
@@ -954,8 +954,8 @@ struct rte_timer_data {
 			 * operations above the status update are observed
 			 * before the update by all threads
 			 */
-			__atomic_store_n(&tim->status.u32, status.u32,
-				__ATOMIC_RELEASE);
+			rte_atomic_store_explicit(&tim->status.u32, status.u32,
+				rte_memory_order_release);
 			__rte_timer_reset(tim, tim->expire + tim->period,
 				tim->period, this_lcore, tim->f, tim->arg, 1,
 				data);
lib/timer/rte_timer.h +3 -3
index d3927d5..a35bc08 100644
@@ -65,10 +65,10 @@ enum rte_timer_type {
  */
 union rte_timer_status {
 	struct {
-		uint16_t state;  /**< Stop, pending, running, config. */
-		int16_t owner;   /**< The lcore that owns the timer. */
+		RTE_ATOMIC(uint16_t) state;  /**< Stop, pending, running, config. */
+		RTE_ATOMIC(int16_t) owner;   /**< The lcore that owns the timer. */
 	};
-	uint32_t u32;            /**< To atomic-set status + owner. */
+	RTE_ATOMIC(uint32_t) u32;            /**< To atomic-set status + owner. */
 };
 
 #ifdef RTE_LIBRTE_TIMER_DEBUG
Signature · 1 line
1.8.3.1
#82 lore raw

PATCH v3 19/19 ring: use rte optional stdatomic API

In reply to Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API

---
 drivers/net/mlx5/mlx5_hws_cnt.h   |  4 ++--
 lib/ring/rte_ring_c11_pvt.h       | 47 +++++++++++++++++++++------------------
 lib/ring/rte_ring_core.h          | 12 +++++-----
 lib/ring/rte_ring_generic_pvt.h   | 16 +++++++------
 lib/ring/rte_ring_hts_elem_pvt.h  | 22 +++++++++---------
 lib/ring/rte_ring_peek_elem_pvt.h |  6 ++---
 lib/ring/rte_ring_rts_elem_pvt.h  | 27 +++++++++++-----------
 7 files changed, 71 insertions(+), 63 deletions(-)
drivers/net/mlx5/mlx5_hws_cnt.h +2 -2
index f462665..dcd5cec 100644
@@ -386,7 +386,7 @@ struct mlx5_hws_age_param {
 
 	MLX5_ASSERT(r->prod.sync_type == RTE_RING_SYNC_ST);
 	MLX5_ASSERT(r->cons.sync_type == RTE_RING_SYNC_ST);
-	current_head = __atomic_load_n(&r->prod.head, __ATOMIC_RELAXED);
+	current_head = rte_atomic_load_explicit(&r->prod.head, rte_memory_order_relaxed);
 	MLX5_ASSERT(n <= r->capacity);
 	MLX5_ASSERT(n <= rte_ring_count(r));
 	revert2head = current_head - n;
@@ -394,7 +394,7 @@ struct mlx5_hws_age_param {
 	__rte_ring_get_elem_addr(r, revert2head, sizeof(cnt_id_t), n,
 			&zcd->ptr1, &zcd->n1, &zcd->ptr2);
 	/* Update tail */
-	__atomic_store_n(&r->prod.tail, revert2head, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&r->prod.tail, revert2head, rte_memory_order_release);
 	return n;
 }
 
lib/ring/rte_ring_c11_pvt.h +25 -22

100 lines not on this page · raw message

lib/ring/rte_ring_core.h +6 -6
index 327fdcf..b770873 100644
@@ -66,8 +66,8 @@ enum rte_ring_sync_type {
  * but offset for *sync_type* and *tail* values should remain the same.
  */
 struct rte_ring_headtail {
-	volatile uint32_t head;      /**< prod/consumer head. */
-	volatile uint32_t tail;      /**< prod/consumer tail. */
+	volatile RTE_ATOMIC(uint32_t) head;      /**< prod/consumer head. */
+	volatile RTE_ATOMIC(uint32_t) tail;      /**< prod/consumer tail. */
 	union {
 		/** sync type of prod/cons */
 		enum rte_ring_sync_type sync_type;
@@ -78,7 +78,7 @@ struct rte_ring_headtail {
 
 union __rte_ring_rts_poscnt {
 	/** raw 8B value to read/write *cnt* and *pos* as one atomic op */
-	uint64_t raw __rte_aligned(8);
+	RTE_ATOMIC(uint64_t) raw __rte_aligned(8);
 	struct {
 		uint32_t cnt; /**< head/tail reference counter */
 		uint32_t pos; /**< head/tail position */
@@ -94,10 +94,10 @@ struct rte_ring_rts_headtail {
 
 union __rte_ring_hts_pos {
 	/** raw 8B value to read/write *head* and *tail* as one atomic op */
-	uint64_t raw __rte_aligned(8);
+	RTE_ATOMIC(uint64_t) raw __rte_aligned(8);
 	struct {
-		uint32_t head; /**< head position */
-		uint32_t tail; /**< tail position */
+		RTE_ATOMIC(uint32_t) head; /**< head position */
+		RTE_ATOMIC(uint32_t) tail; /**< tail position */
 	} pos;
 };
 
lib/ring/rte_ring_generic_pvt.h +9 -7
index 5acb6e5..457f41d 100644
@@ -23,7 +23,8 @@
 	 * we need to wait for them to complete
 	 */
 	if (!single)
-		rte_wait_until_equal_32(&ht->tail, old_val, __ATOMIC_RELAXED);
+		rte_wait_until_equal_32((volatile uint32_t *)(uintptr_t)&ht->tail, old_val,
+			rte_memory_order_relaxed);
 
 	ht->tail = new_val;
 }
@@ -89,10 +90,11 @@
 			return 0;
 
 		*new_head = *old_head + n;
-		if (is_sp)
-			r->prod.head = *new_head, success = 1;
-		else
-			success = rte_atomic32_cmpset(&r->prod.head,
+		if (is_sp) {
+			r->prod.head = *new_head;
+			success = 1;
+		} else
+			success = rte_atomic32_cmpset((uint32_t *)(uintptr_t)&r->prod.head,
 					*old_head, *new_head);
 	} while (unlikely(success == 0));
 	return n;
@@ -162,8 +164,8 @@
 			rte_smp_rmb();
 			success = 1;
 		} else {
-			success = rte_atomic32_cmpset(&r->cons.head, *old_head,
-					*new_head);
+			success = rte_atomic32_cmpset((uint32_t *)(uintptr_t)&r->cons.head,
+					*old_head, *new_head);
 		}
 	} while (unlikely(success == 0));
 	return n;
lib/ring/rte_ring_hts_elem_pvt.h +12 -10
index a8678d3..91f5eec 100644
@@ -10,6 +10,8 @@
 #ifndef _RTE_RING_HTS_ELEM_PVT_H_
 #define _RTE_RING_HTS_ELEM_PVT_H_
 
+#include <rte_stdatomic.h>
+
 /**
  * @file rte_ring_hts_elem_pvt.h
  * It is not recommended to include this file directly,
@@ -30,7 +32,7 @@
 	RTE_SET_USED(enqueue);
 
 	tail = old_tail + num;
-	__atomic_store_n(&ht->ht.pos.tail, tail, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&ht->ht.pos.tail, tail, rte_memory_order_release);
 }
 
 /**
@@ -44,7 +46,7 @@
 {
 	while (p->pos.head != p->pos.tail) {
 		rte_pause();
-		p->raw = __atomic_load_n(&ht->ht.raw, __ATOMIC_ACQUIRE);
+		p->raw = rte_atomic_load_explicit(&ht->ht.raw, rte_memory_order_acquire);
 	}
 }
 
@@ -61,7 +63,7 @@
 
 	const uint32_t capacity = r->capacity;
 
-	op.raw = __atomic_load_n(&r->hts_prod.ht.raw, __ATOMIC_ACQUIRE);
+	op.raw = rte_atomic_load_explicit(&r->hts_prod.ht.raw, rte_memory_order_acquire);
 
 	do {
 		/* Reset n to the initial burst count */
@@ -98,9 +100,9 @@
 	 *  - OOO reads of cons tail value
 	 *  - OOO copy of elems from the ring
 	 */
-	} while (__atomic_compare_exchange_n(&r->hts_prod.ht.raw,
-			&op.raw, np.raw,
-			0, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE) == 0);
+	} while (rte_atomic_compare_exchange_strong_explicit(&r->hts_prod.ht.raw,
+			(uint64_t *)(uintptr_t)&op.raw, np.raw,
+			rte_memory_order_acquire, rte_memory_order_acquire) == 0);
 
 	*old_head = op.pos.head;
 	return n;
@@ -117,7 +119,7 @@
 	uint32_t n;
 	union __rte_ring_hts_pos np, op;
 
-	op.raw = __atomic_load_n(&r->hts_cons.ht.raw, __ATOMIC_ACQUIRE);
+	op.raw = rte_atomic_load_explicit(&r->hts_cons.ht.raw, rte_memory_order_acquire);
 
 	/* move cons.head atomically */
 	do {
@@ -153,9 +155,9 @@
 	 *  - OOO reads of prod tail value
 	 *  - OOO copy of elems from the ring
 	 */
-	} while (__atomic_compare_exchange_n(&r->hts_cons.ht.raw,
-			&op.raw, np.raw,
-			0, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE) == 0);
+	} while (rte_atomic_compare_exchange_strong_explicit(&r->hts_cons.ht.raw,
+			(uint64_t *)(uintptr_t)&op.raw, np.raw,
+			rte_memory_order_acquire, rte_memory_order_acquire) == 0);
 
 	*old_head = op.pos.head;
 	return n;
lib/ring/rte_ring_peek_elem_pvt.h +3 -3
index bb0a7d5..b5f0822 100644
@@ -59,7 +59,7 @@
 
 	pos = tail + num;
 	ht->head = pos;
-	__atomic_store_n(&ht->tail, pos, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&ht->tail, pos, rte_memory_order_release);
 }
 
 /**
@@ -78,7 +78,7 @@
 	uint32_t n;
 	union __rte_ring_hts_pos p;
 
-	p.raw = __atomic_load_n(&ht->ht.raw, __ATOMIC_RELAXED);
+	p.raw = rte_atomic_load_explicit(&ht->ht.raw, rte_memory_order_relaxed);
 	n = p.pos.head - p.pos.tail;
 
 	RTE_ASSERT(n >= num);
@@ -104,7 +104,7 @@
 	p.pos.head = tail + num;
 	p.pos.tail = p.pos.head;
 
-	__atomic_store_n(&ht->ht.raw, p.raw, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&ht->ht.raw, p.raw, rte_memory_order_release);
 }
 
 /**
lib/ring/rte_ring_rts_elem_pvt.h +14 -13
index 7164213..1226503 100644
@@ -31,18 +31,19 @@
 	 * might preceded us, then don't update tail with new value.
 	 */
 
-	ot.raw = __atomic_load_n(&ht->tail.raw, __ATOMIC_ACQUIRE);
+	ot.raw = rte_atomic_load_explicit(&ht->tail.raw, rte_memory_order_acquire);
 
 	do {
 		/* on 32-bit systems we have to do atomic read here */
-		h.raw = __atomic_load_n(&ht->head.raw, __ATOMIC_RELAXED);
+		h.raw = rte_atomic_load_explicit(&ht->head.raw, rte_memory_order_relaxed);
 
 		nt.raw = ot.raw;
 		if (++nt.val.cnt == h.val.cnt)
 			nt.val.pos = h.val.pos;
 
-	} while (__atomic_compare_exchange_n(&ht->tail.raw, &ot.raw, nt.raw,
-			0, __ATOMIC_RELEASE, __ATOMIC_ACQUIRE) == 0);
+	} while (rte_atomic_compare_exchange_strong_explicit(&ht->tail.raw,
+			(uint64_t *)(uintptr_t)&ot.raw, nt.raw,
+			rte_memory_order_release, rte_memory_order_acquire) == 0);
 }
 
 /**
@@ -59,7 +60,7 @@
 
 	while (h->val.pos - ht->tail.val.pos > max) {
 		rte_pause();
-		h->raw = __atomic_load_n(&ht->head.raw, __ATOMIC_ACQUIRE);
+		h->raw = rte_atomic_load_explicit(&ht->head.raw, rte_memory_order_acquire);
 	}
 }
 
@@ -76,7 +77,7 @@
 
 	const uint32_t capacity = r->capacity;
 
-	oh.raw = __atomic_load_n(&r->rts_prod.head.raw, __ATOMIC_ACQUIRE);
+	oh.raw = rte_atomic_load_explicit(&r->rts_prod.head.raw, rte_memory_order_acquire);
 
 	do {
 		/* Reset n to the initial burst count */
@@ -113,9 +114,9 @@
 	 *  - OOO reads of cons tail value
 	 *  - OOO copy of elems to the ring
 	 */
-	} while (__atomic_compare_exchange_n(&r->rts_prod.head.raw,
-			&oh.raw, nh.raw,
-			0, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE) == 0);
+	} while (rte_atomic_compare_exchange_strong_explicit(&r->rts_prod.head.raw,
+			(uint64_t *)(uintptr_t)&oh.raw, nh.raw,
+			rte_memory_order_acquire, rte_memory_order_acquire) == 0);
 
 	*old_head = oh.val.pos;
 	return n;
@@ -132,7 +133,7 @@
 	uint32_t n;
 	union __rte_ring_rts_poscnt nh, oh;
 
-	oh.raw = __atomic_load_n(&r->rts_cons.head.raw, __ATOMIC_ACQUIRE);
+	oh.raw = rte_atomic_load_explicit(&r->rts_cons.head.raw, rte_memory_order_acquire);
 
 	/* move cons.head atomically */
 	do {
@@ -168,9 +169,9 @@
 	 *  - OOO reads of prod tail value
 	 *  - OOO copy of elems from the ring
 	 */
-	} while (__atomic_compare_exchange_n(&r->rts_cons.head.raw,
-			&oh.raw, nh.raw,
-			0, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE) == 0);
+	} while (rte_atomic_compare_exchange_strong_explicit(&r->rts_cons.head.raw,
+			(uint64_t *)(uintptr_t)&oh.raw, nh.raw,
+			rte_memory_order_acquire, rte_memory_order_acquire) == 0);
 
 	*old_head = oh.val.pos;
 	return n;
Signature · 1 line
1.8.3.1
Acked-by on the v3 series
#83 lore raw
In reply to Tyler Retzlaff
Quoted 94 lines · Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API.

v3:
* add missing atomic specification on head variable
in struct rte_ring_headtail
* adapt to use rte_atomic_xxx stdatomic API in rte_ring_c11_pvt.h
* split comma operator statement into 2 statements

v2:
* add #include <rte_stdatomic.h> to rte_mbuf_core.h
* remove first two patches which were fixes that have been merged in another series

Tyler Retzlaff (19):
power: use rte optional stdatomic API
bbdev: use rte optional stdatomic API
eal: use rte optional stdatomic API
eventdev: use rte optional stdatomic API
gpudev: use rte optional stdatomic API
ipsec: use rte optional stdatomic API
mbuf: use rte optional stdatomic API
mempool: use rte optional stdatomic API
rcu: use rte optional stdatomic API
pdump: use rte optional stdatomic API
stack: use rte optional stdatomic API
telemetry: use rte optional stdatomic API
vhost: use rte optional stdatomic API
cryptodev: use rte optional stdatomic API
distributor: use rte optional stdatomic API
ethdev: use rte optional stdatomic API
hash: use rte optional stdatomic API
timer: use rte optional stdatomic API
ring: use rte optional stdatomic API

 drivers/event/cnxk/cnxk_tim_worker.h   |   4 +-
 drivers/net/mlx5/mlx5_hws_cnt.h        |   4 +-
 lib/bbdev/rte_bbdev.c                  |   6 +-
 lib/bbdev/rte_bbdev.h                  |   2 +-
 lib/cryptodev/rte_cryptodev.c          |  22 +++---
 lib/cryptodev/rte_cryptodev.h          |  16 ++---
 lib/distributor/distributor_private.h  |   4 +-
 lib/distributor/rte_distributor.c      |  54 +++++++--------
 lib/eal/common/eal_common_launch.c     |  10 +--
 lib/eal/common/eal_common_mcfg.c       |   2 +-
 lib/eal/common/eal_common_proc.c       |  14 ++--
 lib/eal/common/eal_common_thread.c     |  26 +++----
 lib/eal/common/eal_common_trace.c      |   8 +--
 lib/eal/common/eal_common_trace_ctf.c  |   4 +-
 lib/eal/common/eal_memcfg.h            |   2 +-
 lib/eal/common/eal_private.h           |   4 +-
 lib/eal/common/eal_trace.h             |   4 +-
 lib/eal/common/rte_service.c           | 122 ++++++++++++++++-----------------
 lib/eal/freebsd/eal.c                  |  20 +++---
 lib/eal/include/rte_epoll.h            |   3 +-
 lib/eal/linux/eal.c                    |  26 +++----
 lib/eal/linux/eal_interrupts.c         |  42 ++++++------
 lib/eal/ppc/include/rte_atomic.h       |   6 +-
 lib/eal/windows/rte_thread.c           |   8 ++-
 lib/ethdev/ethdev_driver.h             |  16 ++---
 lib/ethdev/ethdev_private.c            |   6 +-
 lib/ethdev/rte_ethdev.c                |  24 +++----
 lib/ethdev/rte_ethdev.h                |  16 ++---
 lib/ethdev/rte_ethdev_core.h           |   2 +-
 lib/eventdev/rte_event_timer_adapter.c |  66 +++++++++---------
 lib/eventdev/rte_event_timer_adapter.h |   2 +-
 lib/gpudev/gpudev.c                    |   6 +-
 lib/gpudev/gpudev_driver.h             |   2 +-
 lib/hash/rte_cuckoo_hash.c             | 116 +++++++++++++++----------------
 lib/hash/rte_cuckoo_hash.h             |   6 +-
 lib/ipsec/ipsec_sqn.h                  |   2 +-
 lib/ipsec/sa.h                         |   2 +-
 lib/mbuf/rte_mbuf.h                    |  20 +++---
 lib/mbuf/rte_mbuf_core.h               |   5 +-
 lib/mempool/rte_mempool.h              |   4 +-
 lib/pdump/rte_pdump.c                  |  14 ++--
 lib/pdump/rte_pdump.h                  |   8 +--
 lib/power/power_acpi_cpufreq.c         |  33 ++++-----
 lib/power/power_cppc_cpufreq.c         |  25 +++----
 lib/power/power_pstate_cpufreq.c       |  31 +++++----
 lib/rcu/rte_rcu_qsbr.c                 |  48 ++++++-------
 lib/rcu/rte_rcu_qsbr.h                 |  68 +++++++++---------
 lib/ring/rte_ring_c11_pvt.h            |  47 +++++++------
 lib/ring/rte_ring_core.h               |  12 ++--
 lib/ring/rte_ring_generic_pvt.h        |  16 +++--
 lib/ring/rte_ring_hts_elem_pvt.h       |  22 +++---
 lib/ring/rte_ring_peek_elem_pvt.h      |   6 +-
 lib/ring/rte_ring_rts_elem_pvt.h       |  27 ++++----
 lib/stack/rte_stack.h                  |   2 +-
 lib/stack/rte_stack_lf_c11.h           |  24 +++----
 lib/stack/rte_stack_lf_generic.h       |  18 ++---
 lib/telemetry/telemetry.c              |  18 ++---
 lib/timer/rte_timer.c                  |  50 +++++++-------
Quoted 8 lines
 lib/timer/rte_timer.h                  |   6 +-
 lib/vhost/vdpa.c                       |   3 +-
 lib/vhost/vhost.c                      |  42 ++++++------
 lib/vhost/vhost.h                      |  39 ++++++-----
 lib/vhost/vhost_user.c                 |   6 +-
 lib/vhost/virtio_net.c                 |  58 +++++++++-------
 lib/vhost/virtio_net_ctrl.c            |   6 +-
 65 files changed, 684 insertions(+), 653 deletions(-)

The conversion looks correct to me and the CI looks happy.

There are a few acks (from Konstantin) that are missing in the v3 (but I can fix this myself, no need for a new revision just for this). For the series,

Signature · 1 line
David Marchand
#84 lore raw
In reply to Tyler Retzlaff
Quoted 94 lines · Tyler Retzlaff

Replace the use of gcc builtin __atomic_xxx intrinsics with corresponding rte_atomic_xxx optional stdatomic API.

v3:
* add missing atomic specification on head variable
in struct rte_ring_headtail
* adapt to use rte_atomic_xxx stdatomic API in rte_ring_c11_pvt.h
* split comma operator statement into 2 statements

v2:
* add #include <rte_stdatomic.h> to rte_mbuf_core.h
* remove first two patches which were fixes that have been merged in another series

Tyler Retzlaff (19):
power: use rte optional stdatomic API
bbdev: use rte optional stdatomic API
eal: use rte optional stdatomic API
eventdev: use rte optional stdatomic API
gpudev: use rte optional stdatomic API
ipsec: use rte optional stdatomic API
mbuf: use rte optional stdatomic API
mempool: use rte optional stdatomic API
rcu: use rte optional stdatomic API
pdump: use rte optional stdatomic API
stack: use rte optional stdatomic API
telemetry: use rte optional stdatomic API
vhost: use rte optional stdatomic API
cryptodev: use rte optional stdatomic API
distributor: use rte optional stdatomic API
ethdev: use rte optional stdatomic API
hash: use rte optional stdatomic API
timer: use rte optional stdatomic API
ring: use rte optional stdatomic API

 drivers/event/cnxk/cnxk_tim_worker.h   |   4 +-
 drivers/net/mlx5/mlx5_hws_cnt.h        |   4 +-
 lib/bbdev/rte_bbdev.c                  |   6 +-
 lib/bbdev/rte_bbdev.h                  |   2 +-
 lib/cryptodev/rte_cryptodev.c          |  22 +++---
 lib/cryptodev/rte_cryptodev.h          |  16 ++---
 lib/distributor/distributor_private.h  |   4 +-
 lib/distributor/rte_distributor.c      |  54 +++++++--------
 lib/eal/common/eal_common_launch.c     |  10 +--
 lib/eal/common/eal_common_mcfg.c       |   2 +-
 lib/eal/common/eal_common_proc.c       |  14 ++--
 lib/eal/common/eal_common_thread.c     |  26 +++----
 lib/eal/common/eal_common_trace.c      |   8 +--
 lib/eal/common/eal_common_trace_ctf.c  |   4 +-
 lib/eal/common/eal_memcfg.h            |   2 +-
 lib/eal/common/eal_private.h           |   4 +-
 lib/eal/common/eal_trace.h             |   4 +-
 lib/eal/common/rte_service.c           | 122 ++++++++++++++++-----------------
 lib/eal/freebsd/eal.c                  |  20 +++---
 lib/eal/include/rte_epoll.h            |   3 +-
 lib/eal/linux/eal.c                    |  26 +++----
 lib/eal/linux/eal_interrupts.c         |  42 ++++++------
 lib/eal/ppc/include/rte_atomic.h       |   6 +-
 lib/eal/windows/rte_thread.c           |   8 ++-
 lib/ethdev/ethdev_driver.h             |  16 ++---
 lib/ethdev/ethdev_private.c            |   6 +-
 lib/ethdev/rte_ethdev.c                |  24 +++----
 lib/ethdev/rte_ethdev.h                |  16 ++---
 lib/ethdev/rte_ethdev_core.h           |   2 +-
 lib/eventdev/rte_event_timer_adapter.c |  66 +++++++++---------
 lib/eventdev/rte_event_timer_adapter.h |   2 +-
 lib/gpudev/gpudev.c                    |   6 +-
 lib/gpudev/gpudev_driver.h             |   2 +-
 lib/hash/rte_cuckoo_hash.c             | 116 +++++++++++++++----------------
 lib/hash/rte_cuckoo_hash.h             |   6 +-
 lib/ipsec/ipsec_sqn.h                  |   2 +-
 lib/ipsec/sa.h                         |   2 +-
 lib/mbuf/rte_mbuf.h                    |  20 +++---
 lib/mbuf/rte_mbuf_core.h               |   5 +-
 lib/mempool/rte_mempool.h              |   4 +-
 lib/pdump/rte_pdump.c                  |  14 ++--
 lib/pdump/rte_pdump.h                  |   8 +--
 lib/power/power_acpi_cpufreq.c         |  33 ++++-----
 lib/power/power_cppc_cpufreq.c         |  25 +++----
 lib/power/power_pstate_cpufreq.c       |  31 +++++----
 lib/rcu/rte_rcu_qsbr.c                 |  48 ++++++-------
 lib/rcu/rte_rcu_qsbr.h                 |  68 +++++++++---------
 lib/ring/rte_ring_c11_pvt.h            |  47 +++++++------
 lib/ring/rte_ring_core.h               |  12 ++--
 lib/ring/rte_ring_generic_pvt.h        |  16 +++--
 lib/ring/rte_ring_hts_elem_pvt.h       |  22 +++---
 lib/ring/rte_ring_peek_elem_pvt.h      |   6 +-
 lib/ring/rte_ring_rts_elem_pvt.h       |  27 ++++----
 lib/stack/rte_stack.h                  |   2 +-
 lib/stack/rte_stack_lf_c11.h           |  24 +++----
 lib/stack/rte_stack_lf_generic.h       |  18 ++---
 lib/telemetry/telemetry.c              |  18 ++---
 lib/timer/rte_timer.c                  |  50 +++++++-------
Quoted 9 lines
 lib/timer/rte_timer.h                  |   6 +-
 lib/vhost/vdpa.c                       |   3 +-
 lib/vhost/vhost.c                      |  42 ++++++------
 lib/vhost/vhost.h                      |  39 ++++++-----
 lib/vhost/vhost_user.c                 |   6 +-
 lib/vhost/virtio_net.c                 |  58 +++++++++-------
 lib/vhost/virtio_net_ctrl.c            |   6 +-
 65 files changed, 684 insertions(+), 653 deletions(-)

Series applied, thanks.

Signature · 1 line
David Marchand