RFC optional rte optional stdatomics API

index 400e44e..f17b6ae 100644
@@ -13,6 +13,7 @@
 #define _RTE_CONFIG_H_
 
 #include <rte_build_config.h>
+#include <rte_stdatomic.h>
 
 /* legacy defines */
 #ifdef RTE_EXEC_ENV_LINUX

162 lines not on this page · raw message

index 52e45e7..cc25970 100644
@@ -36,9 +36,9 @@
  *   lock multiple times.
  */
 
-rte_mcslock_t *p_ml;
-rte_mcslock_t *p_ml_try;
-rte_mcslock_t *p_ml_perf;
+rte_mcslock_t * __rte_atomic p_ml;
+rte_mcslock_t * __rte_atomic p_ml_try;
+rte_mcslock_t * __rte_atomic p_ml_perf;
 
 static unsigned int count;
 

index 6047911..ac3cec9 100644
@@ -107,33 +107,33 @@
 	 */
 	RTE_SET_USED(failure);
 	/* Find invalid memory order */
-	RTE_ASSERT(success == __ATOMIC_RELAXED ||
-		success == __ATOMIC_ACQUIRE ||
-		success == __ATOMIC_RELEASE ||
-		success == __ATOMIC_ACQ_REL ||
-		success == __ATOMIC_SEQ_CST);
+	RTE_ASSERT(success == rte_memory_order_relaxed ||
+		success == rte_memory_order_acquire ||
+		success == rte_memory_order_release ||
+		success == rte_memory_order_acq_rel ||
+		success == rte_memory_order_seq_cst);
 
 	rte_int128_t expected = *exp;
 	rte_int128_t desired = *src;
 	rte_int128_t old;
 
 #if defined(__ARM_FEATURE_ATOMICS) || defined(RTE_ARM_FEATURE_ATOMICS)
-	if (success == __ATOMIC_RELAXED)
+	if (success == rte_memory_order_relaxed)
 		__cas_128_relaxed(dst, exp, desired);
-	else if (success == __ATOMIC_ACQUIRE)
+	else if (success == rte_memory_order_acquire)
 		__cas_128_acquire(dst, exp, desired);
-	else if (success == __ATOMIC_RELEASE)
+	else if (success == rte_memory_order_release)
 		__cas_128_release(dst, exp, desired);
 	else
 		__cas_128_acq_rel(dst, exp, desired);
 	old = *exp;
 #else
-#define __HAS_ACQ(mo) ((mo) != __ATOMIC_RELAXED && (mo) != __ATOMIC_RELEASE)
-#define __HAS_RLS(mo) ((mo) == __ATOMIC_RELEASE || (mo) == __ATOMIC_ACQ_REL || \
-		(mo) == __ATOMIC_SEQ_CST)
+#define __HAS_ACQ(mo) ((mo) != rte_memory_order_relaxed && (mo) != rte_memory_order_release)
+#define __HAS_RLS(mo) ((mo) == rte_memory_order_release || (mo) == rte_memory_order_acq_rel || \
+		(mo) == rte_memory_order_seq_cst)
 
-	int ldx_mo = __HAS_ACQ(success) ? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED;
-	int stx_mo = __HAS_RLS(success) ? __ATOMIC_RELEASE : __ATOMIC_RELAXED;
+	int ldx_mo = __HAS_ACQ(success) ? rte_memory_order_acquire : rte_memory_order_relaxed;
+	int stx_mo = __HAS_RLS(success) ? rte_memory_order_release : rte_memory_order_relaxed;
 
 #undef __HAS_ACQ
 #undef __HAS_RLS

@@ -153,7 +153,7 @@
 		: "Q" (src->val[0])       \
 		: "memory"); }
 
-		if (ldx_mo == __ATOMIC_RELAXED)
+		if (ldx_mo == rte_memory_order_relaxed)
 			__LOAD_128("ldxp", dst, old)
 		else
 			__LOAD_128("ldaxp", dst, old)

@@ -170,7 +170,7 @@
 		: "memory"); }
 
 		if (likely(old.int128 == expected.int128)) {
-			if (stx_mo == __ATOMIC_RELAXED)
+			if (stx_mo == rte_memory_order_relaxed)
 				__STORE_128("stxp", dst, desired, ret)
 			else
 				__STORE_128("stlxp", dst, desired, ret)

@@ -181,7 +181,7 @@
 			 * needs to be stored back to ensure it was read
 			 * atomically.
 			 */
-			if (stx_mo == __ATOMIC_RELAXED)
+			if (stx_mo == rte_memory_order_relaxed)
 				__STORE_128("stxp", dst, old, ret)
 			else
 				__STORE_128("stlxp", dst, old, ret)

index 5f70e97..d4daafc 100644
@@ -41,7 +41,7 @@ static inline void rte_pause(void)
  * implicitly to exit WFE.
  */
 #define __RTE_ARM_LOAD_EXC_8(src, dst, memorder) {       \
-	if (memorder == __ATOMIC_RELAXED) {               \
+	if (memorder == rte_memory_order_relaxed) {               \
 		asm volatile("ldxrb %w[tmp], [%x[addr]]"  \
 			: [tmp] "=&r" (dst)               \
 			: [addr] "r" (src)                \

@@ -60,7 +60,7 @@ static inline void rte_pause(void)
  * implicitly to exit WFE.
  */
 #define __RTE_ARM_LOAD_EXC_16(src, dst, memorder) {       \
-	if (memorder == __ATOMIC_RELAXED) {               \
+	if (memorder == rte_memory_order_relaxed) {               \
 		asm volatile("ldxrh %w[tmp], [%x[addr]]"  \
 			: [tmp] "=&r" (dst)               \
 			: [addr] "r" (src)                \

@@ -79,7 +79,7 @@ static inline void rte_pause(void)
  * implicitly to exit WFE.
  */
 #define __RTE_ARM_LOAD_EXC_32(src, dst, memorder) {      \
-	if (memorder == __ATOMIC_RELAXED) {              \
+	if (memorder == rte_memory_order_relaxed) {              \
 		asm volatile("ldxr %w[tmp], [%x[addr]]"  \
 			: [tmp] "=&r" (dst)              \
 			: [addr] "r" (src)               \

@@ -98,7 +98,7 @@ static inline void rte_pause(void)
  * implicitly to exit WFE.
  */
 #define __RTE_ARM_LOAD_EXC_64(src, dst, memorder) {      \
-	if (memorder == __ATOMIC_RELAXED) {              \
+	if (memorder == rte_memory_order_relaxed) {              \
 		asm volatile("ldxr %x[tmp], [%x[addr]]"  \
 			: [tmp] "=&r" (dst)              \
 			: [addr] "r" (src)               \

@@ -118,7 +118,7 @@ static inline void rte_pause(void)
  */
 #define __RTE_ARM_LOAD_EXC_128(src, dst, memorder) {                    \
 	volatile rte_int128_t *dst_128 = (volatile rte_int128_t *)&dst; \
-	if (memorder == __ATOMIC_RELAXED) {                             \
+	if (memorder == rte_memory_order_relaxed) {                             \
 		asm volatile("ldxp %x[tmp0], %x[tmp1], [%x[addr]]"      \
 			: [tmp0] "=&r" (dst_128->val[0]),               \
 			  [tmp1] "=&r" (dst_128->val[1])                \

@@ -153,8 +153,8 @@ static inline void rte_pause(void)
 {
 	uint16_t value;
 
-	RTE_BUILD_BUG_ON(memorder != __ATOMIC_ACQUIRE &&
-		memorder != __ATOMIC_RELAXED);
+	RTE_BUILD_BUG_ON(memorder != rte_memory_order_acquire &&
+		memorder != rte_memory_order_relaxed);
 
 	__RTE_ARM_LOAD_EXC_16(addr, value, memorder)
 	if (value != expected) {

@@ -172,8 +172,8 @@ static inline void rte_pause(void)
 {
 	uint32_t value;
 
-	RTE_BUILD_BUG_ON(memorder != __ATOMIC_ACQUIRE &&
-		memorder != __ATOMIC_RELAXED);
+	RTE_BUILD_BUG_ON(memorder != rte_memory_order_acquire &&
+		memorder != rte_memory_order_relaxed);
 
 	__RTE_ARM_LOAD_EXC_32(addr, value, memorder)
 	if (value != expected) {

@@ -191,8 +191,8 @@ static inline void rte_pause(void)
 {
 	uint64_t value;
 
-	RTE_BUILD_BUG_ON(memorder != __ATOMIC_ACQUIRE &&
-		memorder != __ATOMIC_RELAXED);
+	RTE_BUILD_BUG_ON(memorder != rte_memory_order_acquire &&
+		memorder != rte_memory_order_relaxed);
 
 	__RTE_ARM_LOAD_EXC_64(addr, value, memorder)
 	if (value != expected) {

@@ -206,8 +206,8 @@ static inline void rte_pause(void)
 
 #define RTE_WAIT_UNTIL_MASKED(addr, mask, cond, expected, memorder) do {  \
 	RTE_BUILD_BUG_ON(!__builtin_constant_p(memorder));                \
-	RTE_BUILD_BUG_ON(memorder != __ATOMIC_ACQUIRE &&                  \
-		memorder != __ATOMIC_RELAXED);                            \
+	RTE_BUILD_BUG_ON(memorder != rte_memory_order_acquire &&                  \
+		memorder != rte_memory_order_relaxed);                            \
 	const uint32_t size = sizeof(*(addr)) << 3;                       \
 	typeof(*(addr)) expected_value = (expected);                      \
 	typeof(*(addr)) value;                                            \

index 77b96e4..f54cf59 100644
@@ -33,19 +33,19 @@
 
 	switch (pmc->size) {
 	case sizeof(uint8_t):
-		__RTE_ARM_LOAD_EXC_8(pmc->addr, cur_value, __ATOMIC_RELAXED)
+		__RTE_ARM_LOAD_EXC_8(pmc->addr, cur_value, rte_memory_order_relaxed)
 		__RTE_ARM_WFE()
 		break;
 	case sizeof(uint16_t):
-		__RTE_ARM_LOAD_EXC_16(pmc->addr, cur_value, __ATOMIC_RELAXED)
+		__RTE_ARM_LOAD_EXC_16(pmc->addr, cur_value, rte_memory_order_relaxed)
 		__RTE_ARM_WFE()
 		break;
 	case sizeof(uint32_t):
-		__RTE_ARM_LOAD_EXC_32(pmc->addr, cur_value, __ATOMIC_RELAXED)
+		__RTE_ARM_LOAD_EXC_32(pmc->addr, cur_value, rte_memory_order_relaxed)
 		__RTE_ARM_WFE()
 		break;
 	case sizeof(uint64_t):
-		__RTE_ARM_LOAD_EXC_64(pmc->addr, cur_value, __ATOMIC_RELAXED)
+		__RTE_ARM_LOAD_EXC_64(pmc->addr, cur_value, rte_memory_order_relaxed)
 		__RTE_ARM_WFE()
 		break;
 	default:

index cb980af..c6628dd 100644
@@ -103,11 +103,11 @@ struct trace_point_head *
 trace_mode_set(rte_trace_point_t *t, enum rte_trace_mode mode)
 {
 	if (mode == RTE_TRACE_MODE_OVERWRITE)
-		__atomic_fetch_and(t, ~__RTE_TRACE_FIELD_ENABLE_DISCARD,
-			__ATOMIC_RELEASE);
+		rte_atomic_fetch_and_explicit(t, ~__RTE_TRACE_FIELD_ENABLE_DISCARD,
+			rte_memory_order_release);
 	else
-		__atomic_fetch_or(t, __RTE_TRACE_FIELD_ENABLE_DISCARD,
-			__ATOMIC_RELEASE);
+		rte_atomic_fetch_or_explicit(t, __RTE_TRACE_FIELD_ENABLE_DISCARD,
+			rte_memory_order_release);
 }
 
 void

@@ -141,7 +141,7 @@ rte_trace_mode rte_trace_mode_get(void)
 	if (trace_point_is_invalid(t))
 		return false;
 
-	val = __atomic_load_n(t, __ATOMIC_ACQUIRE);
+	val = rte_atomic_load_explicit(t, rte_memory_order_acquire);
 	return (val & __RTE_TRACE_FIELD_ENABLE_MASK) != 0;
 }
 

@@ -153,7 +153,8 @@ rte_trace_mode rte_trace_mode_get(void)
 	if (trace_point_is_invalid(t))
 		return -ERANGE;
 
-	prev = __atomic_fetch_or(t, __RTE_TRACE_FIELD_ENABLE_MASK, __ATOMIC_RELEASE);
+	prev = rte_atomic_fetch_or_explicit(t, __RTE_TRACE_FIELD_ENABLE_MASK,
+	    rte_memory_order_release);
 	if ((prev & __RTE_TRACE_FIELD_ENABLE_MASK) == 0)
 		__atomic_fetch_add(&trace.status, 1, __ATOMIC_RELEASE);
 	return 0;

@@ -167,7 +168,8 @@ rte_trace_mode rte_trace_mode_get(void)
 	if (trace_point_is_invalid(t))
 		return -ERANGE;
 
-	prev = __atomic_fetch_and(t, ~__RTE_TRACE_FIELD_ENABLE_MASK, __ATOMIC_RELEASE);
+	prev = rte_atomic_fetch_and_explicit(t, ~__RTE_TRACE_FIELD_ENABLE_MASK,
+	    rte_memory_order_release);
 	if ((prev & __RTE_TRACE_FIELD_ENABLE_MASK) != 0)
 		__atomic_fetch_sub(&trace.status, 1, __ATOMIC_RELEASE);
 	return 0;

194 lines not on this page · raw message

index ec1f418..3ea1553 100644
@@ -35,13 +35,13 @@
  *  A 16-bit expected value to be in the memory location.
  * @param memorder
  *  Two different memory orders that can be specified:
- *  __ATOMIC_ACQUIRE and __ATOMIC_RELAXED. These map to
+ *  rte_memory_order_acquire and rte_memory_order_relaxed. These map to
  *  C++11 memory orders with the same names, see the C++11 standard or
  *  the GCC wiki on atomic synchronization for detailed definition.
  */
 static __rte_always_inline void
 rte_wait_until_equal_16(volatile uint16_t *addr, uint16_t expected,
-		int memorder);
+		rte_memory_order memorder);
 
 /**
  * Wait for *addr to be updated with a 32-bit expected value, with a relaxed

@@ -53,13 +53,13 @@
  *  A 32-bit expected value to be in the memory location.
  * @param memorder
  *  Two different memory orders that can be specified:
- *  __ATOMIC_ACQUIRE and __ATOMIC_RELAXED. These map to
+ *  rte_memory_order_acquire and rte_memory_order_relaxed. These map to
  *  C++11 memory orders with the same names, see the C++11 standard or
  *  the GCC wiki on atomic synchronization for detailed definition.
  */
 static __rte_always_inline void
 rte_wait_until_equal_32(volatile uint32_t *addr, uint32_t expected,
-		int memorder);
+		rte_memory_order memorder);
 
 /**
  * Wait for *addr to be updated with a 64-bit expected value, with a relaxed

@@ -71,42 +71,42 @@
  *  A 64-bit expected value to be in the memory location.
  * @param memorder
  *  Two different memory orders that can be specified:
- *  __ATOMIC_ACQUIRE and __ATOMIC_RELAXED. These map to
+ *  rte_memory_order_acquire and rte_memory_order_relaxed. These map to
  *  C++11 memory orders with the same names, see the C++11 standard or
  *  the GCC wiki on atomic synchronization for detailed definition.
  */
 static __rte_always_inline void
 rte_wait_until_equal_64(volatile uint64_t *addr, uint64_t expected,
-		int memorder);
+		rte_memory_order memorder);
 
 #ifndef RTE_WAIT_UNTIL_EQUAL_ARCH_DEFINED
 static __rte_always_inline void
 rte_wait_until_equal_16(volatile uint16_t *addr, uint16_t expected,
-		int memorder)
+		rte_memory_order memorder)
 {
-	assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED);
+	assert(memorder == rte_memory_order_acquire || memorder == rte_memory_order_relaxed);
 
-	while (__atomic_load_n(addr, memorder) != expected)
+	while (rte_atomic_load_explicit(addr, memorder) != expected)
 		rte_pause();
 }
 
 static __rte_always_inline void
 rte_wait_until_equal_32(volatile uint32_t *addr, uint32_t expected,
-		int memorder)
+		rte_memory_order memorder)
 {
-	assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED);
+	assert(memorder == rte_memory_order_acquire || memorder == rte_memory_order_relaxed);
 
-	while (__atomic_load_n(addr, memorder) != expected)
+	while (rte_atomic_load_explicit(addr, memorder) != expected)
 		rte_pause();
 }
 
 static __rte_always_inline void
 rte_wait_until_equal_64(volatile uint64_t *addr, uint64_t expected,
-		int memorder)
+		rte_memory_order memorder)
 {
-	assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED);
+	assert(memorder == rte_memory_order_acquire || memorder == rte_memory_order_relaxed);
 
-	while (__atomic_load_n(addr, memorder) != expected)
+	while (rte_atomic_load_explicit(addr, memorder) != expected)
 		rte_pause();
 }
 

@@ -124,16 +124,16 @@
  *  An expected value to be in the memory location.
  * @param memorder
  *  Two different memory orders that can be specified:
- *  __ATOMIC_ACQUIRE and __ATOMIC_RELAXED. These map to
+ *  rte_memory_order_acquire and rte_memory_order_relaxed. These map to
  *  C++11 memory orders with the same names, see the C++11 standard or
  *  the GCC wiki on atomic synchronization for detailed definition.
  */
 #define RTE_WAIT_UNTIL_MASKED(addr, mask, cond, expected, memorder) do { \
 	RTE_BUILD_BUG_ON(!__builtin_constant_p(memorder));               \
-	RTE_BUILD_BUG_ON(memorder != __ATOMIC_ACQUIRE &&                 \
-		memorder != __ATOMIC_RELAXED);                           \
+	RTE_BUILD_BUG_ON((memorder) != rte_memory_order_acquire &&       \
+		(memorder) != rte_memory_order_relaxed);                 \
 	typeof(*(addr)) expected_value = (expected);                     \
-	while (!((__atomic_load_n((addr), (memorder)) & (mask)) cond     \
+	while (!((rte_atomic_load_explicit((addr), (memorder)) & (mask)) cond     \
 			expected_value))                                 \
 		rte_pause();                                             \
 } while (0)

126 lines not on this page · raw message

index c50ebaa..e5ff348 100644
@@ -28,7 +28,7 @@
  * The rte_spinlock_t type.
  */
 typedef struct __rte_lockable {
-	volatile int locked; /**< lock status 0 = unlocked, 1 = locked */
+	volatile int __rte_atomic locked; /**< lock status 0 = unlocked, 1 = locked */
 } rte_spinlock_t;
 
 /**

@@ -65,10 +65,10 @@
 {
 	int exp = 0;
 
-	while (!__atomic_compare_exchange_n(&sl->locked, &exp, 1, 0,
-				__ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
-		rte_wait_until_equal_32((volatile uint32_t *)&sl->locked,
-			       0, __ATOMIC_RELAXED);
+	while (!rte_atomic_compare_exchange_strong_explicit(&sl->locked, &exp, 1,
+				rte_memory_order_acquire, rte_memory_order_relaxed)) {
+		rte_wait_until_equal_32((volatile uint32_t *)(uintptr_t)&sl->locked,
+			       0, rte_memory_order_relaxed);
 		exp = 0;
 	}
 }

@@ -89,7 +89,7 @@
 rte_spinlock_unlock(rte_spinlock_t *sl)
 	__rte_no_thread_safety_analysis
 {
-	__atomic_store_n(&sl->locked, 0, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&sl->locked, 0, rte_memory_order_release);
 }
 #endif
 

@@ -112,9 +112,8 @@
 	__rte_no_thread_safety_analysis
 {
 	int exp = 0;
-	return __atomic_compare_exchange_n(&sl->locked, &exp, 1,
-				0, /* disallow spurious failure */
-				__ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
+	return rte_atomic_compare_exchange_strong_explicit(&sl->locked, &exp, 1,
+				rte_memory_order_acquire, rte_memory_order_relaxed);
 }
 #endif
 

@@ -128,7 +127,7 @@
  */
 static inline int rte_spinlock_is_locked (rte_spinlock_t *sl)
 {
-	return __atomic_load_n(&sl->locked, __ATOMIC_ACQUIRE);
+	return rte_atomic_load_explicit(&sl->locked, rte_memory_order_acquire);
 }
 
 /**

133 lines not on this page · raw message

index a3f7291..7d51fc9 100644
@@ -40,8 +40,8 @@
  */
 struct rte_pflock {
 	struct {
-		uint16_t in;
-		uint16_t out;
+		uint16_t __rte_atomic in;
+		uint16_t __rte_atomic out;
 	} rd, wr;
 };
 typedef struct rte_pflock rte_pflock_t;

@@ -116,14 +116,14 @@ struct rte_pflock {
 	 * If no writer is present, then the operation has completed
 	 * successfully.
 	 */
-	w = __atomic_fetch_add(&pf->rd.in, RTE_PFLOCK_RINC, __ATOMIC_ACQUIRE)
+	w = rte_atomic_fetch_add_explicit(&pf->rd.in, RTE_PFLOCK_RINC, rte_memory_order_acquire)
 		& RTE_PFLOCK_WBITS;
 	if (w == 0)
 		return;
 
 	/* Wait for current write phase to complete. */
 	RTE_WAIT_UNTIL_MASKED(&pf->rd.in, RTE_PFLOCK_WBITS, !=, w,
-		__ATOMIC_ACQUIRE);
+		rte_memory_order_acquire);
 }
 
 /**

@@ -139,7 +139,7 @@ struct rte_pflock {
 static inline void
 rte_pflock_read_unlock(rte_pflock_t *pf)
 {
-	__atomic_fetch_add(&pf->rd.out, RTE_PFLOCK_RINC, __ATOMIC_RELEASE);
+	rte_atomic_fetch_add_explicit(&pf->rd.out, RTE_PFLOCK_RINC, rte_memory_order_release);
 }
 
 /**

@@ -160,8 +160,9 @@ struct rte_pflock {
 	/* Acquire ownership of write-phase.
 	 * This is same as rte_ticketlock_lock().
 	 */
-	ticket = __atomic_fetch_add(&pf->wr.in, 1, __ATOMIC_RELAXED);
-	rte_wait_until_equal_16(&pf->wr.out, ticket, __ATOMIC_ACQUIRE);
+	ticket = rte_atomic_fetch_add_explicit(&pf->wr.in, 1, rte_memory_order_relaxed);
+	rte_wait_until_equal_16((uint16_t *)(uintptr_t)&pf->wr.out, ticket,
+	    rte_memory_order_acquire);
 
 	/*
 	 * Acquire ticket on read-side in order to allow them

@@ -172,10 +173,11 @@ struct rte_pflock {
 	 * speculatively.
 	 */
 	w = RTE_PFLOCK_PRES | (ticket & RTE_PFLOCK_PHID);
-	ticket = __atomic_fetch_add(&pf->rd.in, w, __ATOMIC_RELAXED);
+	ticket = rte_atomic_fetch_add_explicit(&pf->rd.in, w, rte_memory_order_relaxed);
 
 	/* Wait for any pending readers to flush. */
-	rte_wait_until_equal_16(&pf->rd.out, ticket, __ATOMIC_ACQUIRE);
+	rte_wait_until_equal_16((uint16_t *)(uintptr_t)&pf->rd.out, ticket,
+	    rte_memory_order_acquire);
 }
 
 /**

@@ -192,10 +194,10 @@ struct rte_pflock {
 rte_pflock_write_unlock(rte_pflock_t *pf)
 {
 	/* Migrate from write phase to read phase. */
-	__atomic_fetch_and(&pf->rd.in, RTE_PFLOCK_LSB, __ATOMIC_RELEASE);
+	rte_atomic_fetch_and_explicit(&pf->rd.in, RTE_PFLOCK_LSB, rte_memory_order_release);
 
 	/* Allow other writers to continue. */
-	__atomic_fetch_add(&pf->wr.out, 1, __ATOMIC_RELEASE);
+	rte_atomic_fetch_add_explicit(&pf->wr.out, 1, rte_memory_order_release);
 }
 
 #ifdef __cplusplus

index ff62708..f581908 100644
@@ -31,7 +31,7 @@
  * The RTE seqcount type.
  */
 typedef struct {
-	uint32_t sn; /**< A sequence number for the protected data. */
+	uint32_t __rte_atomic sn; /**< A sequence number for the protected data. */
 } rte_seqcount_t;
 
 /**

@@ -105,11 +105,11 @@
 static inline uint32_t
 rte_seqcount_read_begin(const rte_seqcount_t *seqcount)
 {
-	/* __ATOMIC_ACQUIRE to prevent loads after (in program order)
+	/* rte_memory_order_acquire to prevent loads after (in program order)
 	 * from happening before the sn load. Synchronizes-with the
 	 * store release in rte_seqcount_write_end().
 	 */
-	return __atomic_load_n(&seqcount->sn, __ATOMIC_ACQUIRE);
+	return rte_atomic_load_explicit(&seqcount->sn, rte_memory_order_acquire);
 }
 
 /**

@@ -160,9 +160,9 @@
 		return true;
 
 	/* make sure the data loads happens before the sn load */
-	rte_atomic_thread_fence(__ATOMIC_ACQUIRE);
+	rte_atomic_thread_fence(rte_memory_order_acquire);
 
-	end_sn = __atomic_load_n(&seqcount->sn, __ATOMIC_RELAXED);
+	end_sn = rte_atomic_load_explicit(&seqcount->sn, rte_memory_order_relaxed);
 
 	/* A writer incremented the sequence number during this read
 	 * critical section.

@@ -204,12 +204,12 @@
 
 	sn = seqcount->sn + 1;
 
-	__atomic_store_n(&seqcount->sn, sn, __ATOMIC_RELAXED);
+	rte_atomic_store_explicit(&seqcount->sn, sn, rte_memory_order_relaxed);
 
-	/* __ATOMIC_RELEASE to prevent stores after (in program order)
+	/* rte_memory_order_release to prevent stores after (in program order)
 	 * from happening before the sn store.
 	 */
-	rte_atomic_thread_fence(__ATOMIC_RELEASE);
+	rte_atomic_thread_fence(rte_memory_order_release);
 }
 
 /**

@@ -236,7 +236,7 @@
 	sn = seqcount->sn + 1;
 
 	/* Synchronizes-with the load acquire in rte_seqcount_read_begin(). */
-	__atomic_store_n(&seqcount->sn, sn, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&seqcount->sn, sn, rte_memory_order_release);
 }
 
 #ifdef __cplusplus

115 lines not on this page · raw message

index c6b6fcc..2bcf954 100644
@@ -32,7 +32,7 @@
 #include <rte_uuid.h>
 
 /** The tracepoint object. */
-typedef uint64_t rte_trace_point_t;
+typedef uint64_t __rte_atomic rte_trace_point_t;
 
 /**
  * Macro to define the tracepoint arguments in RTE_TRACE_POINT macro.

@@ -358,7 +358,7 @@ struct __rte_trace_header {
 #define __rte_trace_point_emit_header_generic(t) \
 void *mem; \
 do { \
-	const uint64_t val = __atomic_load_n(t, __ATOMIC_ACQUIRE); \
+	const uint64_t val = rte_atomic_load_explicit(t, rte_memory_order_acquire); \
 	if (likely(!(val & __RTE_TRACE_FIELD_ENABLE_MASK))) \
 		return; \
 	mem = __rte_trace_mem_get(val); \

161 lines not on this page · raw message

index f2ee1a9..aedce9b 100644
@@ -82,14 +82,14 @@
 /**
  * Synchronization fence between threads based on the specified memory order.
  *
- * On x86 the __atomic_thread_fence(__ATOMIC_SEQ_CST) generates full 'mfence'
+ * On x86 the __atomic_thread_fence(rte_memory_order_seq_cst) generates full 'mfence'
  * which is quite expensive. The optimized implementation of rte_smp_mb is
  * used instead.
  */
 static __rte_always_inline void
 rte_atomic_thread_fence(int memorder)
 {
-	if (memorder == __ATOMIC_SEQ_CST)
+	if (memorder == rte_memory_order_seq_cst)
 		rte_smp_mb();
 	else
 		__atomic_thread_fence(memorder);

index 0b20ddf..c76218a 100644
@@ -78,7 +78,7 @@ static inline int rte_tm_supported(void)
 }
 
 static inline int
-rte_try_tm(volatile int *lock)
+rte_try_tm(volatile int __rte_atomic *lock)
 {
 	int i, retries;
 

index f749da9..cf70e33 100644
@@ -23,9 +23,9 @@
 	uint64_t val;
 
 	/* trigger a write but don't change the value */
-	val = __atomic_load_n((volatile uint64_t *)addr, __ATOMIC_RELAXED);
-	__atomic_compare_exchange_n((volatile uint64_t *)addr, &val, val, 0,
-			__ATOMIC_RELAXED, __ATOMIC_RELAXED);
+	val = rte_atomic_load_explicit((volatile uint64_t *)addr, rte_memory_order_relaxed);
+	rte_atomic_compare_exchange_strong_explicit((volatile uint64_t *)addr, &val, val,
+			rte_memory_order_relaxed, rte_memory_order_relaxed);
 }
 
 static bool wait_supported;

index 15a36f3..2c65304 100644
@@ -273,7 +273,8 @@
 static inline void
 rte_atomic16_add(rte_atomic16_t *v, int16_t inc)
 {
-	rte_atomic_fetch_add_explicit(&v->cnt, inc, rte_memory_order_seq_cst);
+	rte_atomic_fetch_add_explicit((volatile int16_t __rte_atomic *)&v->cnt, inc,
+	    rte_memory_order_seq_cst);
 }
 
 /**

@@ -287,7 +288,8 @@
 static inline void
 rte_atomic16_sub(rte_atomic16_t *v, int16_t dec)
 {
-	rte_atomic_fetch_sub_explicit(&v->cnt, dec, rte_memory_order_seq_cst);
+	rte_atomic_fetch_sub_explicit((volatile int16_t __rte_atomic *)&v->cnt, dec,
+	    rte_memory_order_seq_cst);
 }
 
 /**

@@ -340,7 +342,8 @@
 static inline int16_t
 rte_atomic16_add_return(rte_atomic16_t *v, int16_t inc)
 {
-	return rte_atomic_fetch_add_explicit(&v->cnt, inc, rte_memory_order_seq_cst) + inc;
+	return rte_atomic_fetch_add_explicit((volatile int16_t __rte_atomic *)&v->cnt, inc,
+	    rte_memory_order_seq_cst) + inc;
 }
 
 /**

@@ -360,7 +363,8 @@
 static inline int16_t
 rte_atomic16_sub_return(rte_atomic16_t *v, int16_t dec)
 {
-	return rte_atomic_fetch_sub_explicit(&v->cnt, dec, rte_memory_order_seq_cst) - dec;
+	return rte_atomic_fetch_sub_explicit((volatile int16_t __rte_atomic *)&v->cnt, dec,
+	    rte_memory_order_seq_cst) - dec;
 }
 
 /**

@@ -379,7 +383,8 @@
 #ifdef RTE_FORCE_INTRINSICS
 static inline int rte_atomic16_inc_and_test(rte_atomic16_t *v)
 {
-	return rte_atomic_fetch_add_explicit(&v->cnt, 1, rte_memory_order_seq_cst) + 1 == 0;
+	return rte_atomic_fetch_add_explicit((volatile int16_t __rte_atomic *)&v->cnt, 1,
+	    rte_memory_order_seq_cst) + 1 == 0;
 }
 #endif
 

@@ -399,7 +404,8 @@ static inline int rte_atomic16_inc_and_test(rte_atomic16_t *v)
 #ifdef RTE_FORCE_INTRINSICS
 static inline int rte_atomic16_dec_and_test(rte_atomic16_t *v)
 {
-	return rte_atomic_fetch_sub_explicit(&v->cnt, 1, rte_memory_order_seq_cst) - 1 == 0;
+	return rte_atomic_fetch_sub_explicit((volatile int16_t __rte_atomic *)&v->cnt, 1,
+	    rte_memory_order_seq_cst) - 1 == 0;
 }
 #endif
 

@@ -552,7 +558,8 @@ static inline void rte_atomic16_clear(rte_atomic16_t *v)
 static inline void
 rte_atomic32_add(rte_atomic32_t *v, int32_t inc)
 {
-	rte_atomic_fetch_add_explicit(&v->cnt, inc, rte_memory_order_seq_cst);
+	rte_atomic_fetch_add_explicit((volatile int32_t __rte_atomic *)&v->cnt, inc,
+	    rte_memory_order_seq_cst);
 }
 
 /**

@@ -566,7 +573,8 @@ static inline void rte_atomic16_clear(rte_atomic16_t *v)
 static inline void
 rte_atomic32_sub(rte_atomic32_t *v, int32_t dec)
 {
-	rte_atomic_fetch_sub_explicit(&v->cnt, dec, rte_memory_order_seq_cst);
+	rte_atomic_fetch_sub_explicit((volatile int32_t __rte_atomic *)&v->cnt, dec,
+	    rte_memory_order_seq_cst);
 }
 
 /**

@@ -619,7 +627,8 @@ static inline void rte_atomic16_clear(rte_atomic16_t *v)
 static inline int32_t
 rte_atomic32_add_return(rte_atomic32_t *v, int32_t inc)
 {
-	return rte_atomic_fetch_add_explicit(&v->cnt, inc, rte_memory_order_seq_cst) + inc;
+	return rte_atomic_fetch_add_explicit((volatile int32_t __rte_atomic *)&v->cnt, inc,
+	    rte_memory_order_seq_cst) + inc;
 }
 
 /**

@@ -639,7 +648,8 @@ static inline void rte_atomic16_clear(rte_atomic16_t *v)
 static inline int32_t
 rte_atomic32_sub_return(rte_atomic32_t *v, int32_t dec)
 {
-	return rte_atomic_fetch_sub_explicit(&v->cnt, dec, rte_memory_order_seq_cst) - dec;
+	return rte_atomic_fetch_sub_explicit((volatile int32_t __rte_atomic *)&v->cnt, dec,
+	    rte_memory_order_seq_cst) - dec;
 }
 
 /**

@@ -658,7 +668,8 @@ static inline void rte_atomic16_clear(rte_atomic16_t *v)
 #ifdef RTE_FORCE_INTRINSICS
 static inline int rte_atomic32_inc_and_test(rte_atomic32_t *v)
 {
-	return rte_atomic_fetch_add_explicit(&v->cnt, 1, rte_memory_order_seq_cst) + 1 == 0;
+	return rte_atomic_fetch_add_explicit((volatile int32_t __rte_atomic *)&v->cnt, 1,
+	    rte_memory_order_seq_cst) + 1 == 0;
 }
 #endif
 

@@ -678,7 +689,8 @@ static inline int rte_atomic32_inc_and_test(rte_atomic32_t *v)
 #ifdef RTE_FORCE_INTRINSICS
 static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v)
 {
-	return rte_atomic_fetch_sub_explicit(&v->cnt, 1, rte_memory_order_seq_cst) - 1 == 0;
+	return rte_atomic_fetch_sub_explicit((volatile int32_t __rte_atomic *)&v->cnt, 1,
+	    rte_memory_order_seq_cst) - 1 == 0;
 }
 #endif
 

@@ -884,7 +896,8 @@ static inline void rte_atomic32_clear(rte_atomic32_t *v)
 static inline void
 rte_atomic64_add(rte_atomic64_t *v, int64_t inc)
 {
-	rte_atomic_fetch_add_explicit(&v->cnt, inc, rte_memory_order_seq_cst);
+	rte_atomic_fetch_add_explicit((volatile int64_t __rte_atomic *)&v->cnt, inc,
+	    rte_memory_order_seq_cst);
 }
 #endif
 

@@ -903,7 +916,8 @@ static inline void rte_atomic32_clear(rte_atomic32_t *v)
 static inline void
 rte_atomic64_sub(rte_atomic64_t *v, int64_t dec)
 {
-	rte_atomic_fetch_sub_explicit(&v->cnt, dec, rte_memory_order_seq_cst);
+	rte_atomic_fetch_sub_explicit((volatile int64_t __rte_atomic *)&v->cnt, dec,
+	    rte_memory_order_seq_cst);
 }
 #endif
 

@@ -961,7 +975,8 @@ static inline void rte_atomic32_clear(rte_atomic32_t *v)
 static inline int64_t
 rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc)
 {
-	return rte_atomic_fetch_add_explicit(&v->cnt, inc, rte_memory_order_seq_cst) + inc;
+	return rte_atomic_fetch_add_explicit((volatile int64_t __rte_atomic *)&v->cnt, inc,
+	    rte_memory_order_seq_cst) + inc;
 }
 #endif
 

@@ -985,7 +1000,8 @@ static inline void rte_atomic32_clear(rte_atomic32_t *v)
 static inline int64_t
 rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec)
 {
-	return rte_atomic_fetch_sub_explicit(&v->cnt, dec, rte_memory_order_seq_cst) - dec;
+	return rte_atomic_fetch_sub_explicit((volatile int64_t __rte_atomic *)&v->cnt, dec,
+	    rte_memory_order_seq_cst) - dec;
 }
 #endif
 

index 3ea1553..db8a1f8 100644
@@ -86,7 +86,8 @@
 {
 	assert(memorder == rte_memory_order_acquire || memorder == rte_memory_order_relaxed);
 
-	while (rte_atomic_load_explicit(addr, memorder) != expected)
+	while (rte_atomic_load_explicit((volatile uint16_t __rte_atomic *)addr, memorder)
+	    != expected)
 		rte_pause();
 }
 

@@ -96,7 +97,8 @@
 {
 	assert(memorder == rte_memory_order_acquire || memorder == rte_memory_order_relaxed);
 
-	while (rte_atomic_load_explicit(addr, memorder) != expected)
+	while (rte_atomic_load_explicit((volatile uint32_t __rte_atomic *)addr, memorder)
+	    != expected)
 		rte_pause();
 }
 

@@ -106,7 +108,8 @@
 {
 	assert(memorder == rte_memory_order_acquire || memorder == rte_memory_order_relaxed);
 
-	while (rte_atomic_load_explicit(addr, memorder) != expected)
+	while (rte_atomic_load_explicit((volatile uint64_t __rte_atomic *)addr, memorder)
+	    != expected)
 		rte_pause();
 }
 

index cf70e33..6c192f0 100644
@@ -23,9 +23,10 @@
 	uint64_t val;
 
 	/* trigger a write but don't change the value */
-	val = rte_atomic_load_explicit((volatile uint64_t *)addr, rte_memory_order_relaxed);
-	rte_atomic_compare_exchange_strong_explicit((volatile uint64_t *)addr, &val, val,
-			rte_memory_order_relaxed, rte_memory_order_relaxed);
+	val = rte_atomic_load_explicit((volatile uint64_t __rte_atomic *)addr,
+	    rte_memory_order_relaxed);
+	rte_atomic_compare_exchange_strong_explicit((volatile uint64_t __rte_atomic *)addr,
+	    &val, val, rte_memory_order_relaxed, rte_memory_order_relaxed);
 }
 
 static bool wait_supported;

index 7101f63..ffbdae5 100644
@@ -52,7 +52,7 @@
  * Only 64-bits of the memory is actually used though.
  */
 union rte_distributor_buffer_single {
-	volatile int64_t bufptr64;
+	volatile int64_t __rte_atomic bufptr64;
 	char pad[RTE_CACHE_LINE_SIZE*3];
 } __rte_cache_aligned;
 

117 lines not on this page · raw message

index ffd2db7..b300447 100644
@@ -25,7 +25,7 @@
 
 struct bpf_eth_cbi {
 	/* used by both data & control path */
-	uint32_t use;    /*usage counter */
+	uint32_t __rte_atomic use;    /*usage counter */
 	const struct rte_eth_rxtx_callback *cb;  /* callback handle */
 	struct rte_bpf *bpf;
 	struct rte_bpf_jit jit;

@@ -110,8 +110,8 @@ struct bpf_eth_cbh {
 
 	/* in use, busy wait till current RX/TX iteration is finished */
 	if ((puse & BPF_ETH_CBI_INUSE) != 0) {
-		RTE_WAIT_UNTIL_MASKED((uint32_t *)(uintptr_t)&cbi->use,
-			UINT32_MAX, !=, puse, __ATOMIC_RELAXED);
+		RTE_WAIT_UNTIL_MASKED((uint32_t __rte_atomic *)(uintptr_t)&cbi->use,
+			UINT32_MAX, !=, puse, rte_memory_order_relaxed);
 	}
 }
 

index aef44e2..efd29eb 100644
@@ -15,6 +15,7 @@
 #include <stdint.h>
 #include <rte_compat.h>
 #include <rte_common.h>
+#include <rte_stdatomic.h>
 
 #ifdef __DOXYGEN__
 

index ec1f418..bebfa95 100644
@@ -16,6 +16,7 @@
 #include <assert.h>
 #include <rte_common.h>
 #include <rte_atomic.h>
+#include <rte_stdatomic.h>
 
 /**
  * Pause CPU execution for a short while

index 9e083bb..24ebec6 100644
@@ -32,6 +32,7 @@
 #include <rte_common.h>
 #include <rte_lock_annotations.h>
 #include <rte_pause.h>
+#include <rte_stdatomic.h>
 
 /**
  * The rte_rwlock_t type.

index c50ebaa..e18f0cd 100644
@@ -23,6 +23,7 @@
 #endif
 #include <rte_lock_annotations.h>
 #include <rte_pause.h>
+#include <rte_stdatomic.h>
 
 /**
  * The rte_spinlock_t type.

index a805cb2..18e63eb 100644
@@ -27,6 +27,7 @@
 #include <rte_common.h>
 #include <rte_pause.h>
 #include <rte_branch_prediction.h>
+#include <rte_stdatomic.h>
 
 /**
  * The rte_mcslock_t type.

index a3f7291..790be71 100644
@@ -34,6 +34,7 @@
 #include <rte_compat.h>
 #include <rte_common.h>
 #include <rte_pause.h>
+#include <rte_stdatomic.h>
 
 /**
  * The rte_pflock_t type.

index ff62708..098af26 100644
@@ -26,6 +26,7 @@
 #include <rte_atomic.h>
 #include <rte_branch_prediction.h>
 #include <rte_compat.h>
+#include <rte_stdatomic.h>
 
 /**
  * The RTE seqcount type.

182 lines not on this page · raw message

index 5db0d8a..e22d119 100644
@@ -24,6 +24,7 @@
 #include <rte_common.h>
 #include <rte_lcore.h>
 #include <rte_pause.h>
+#include <rte_stdatomic.h>
 
 /**
  * The rte_ticketlock_t type.

index c6b6fcc..d587591 100644
@@ -30,6 +30,7 @@
 #include <rte_per_lcore.h>
 #include <rte_string_fns.h>
 #include <rte_uuid.h>
+#include <rte_stdatomic.h>
 
 /** The tracepoint object. */
 typedef uint64_t rte_trace_point_t;

index 52e45e7..cc25970 100644
@@ -36,9 +36,9 @@
  *   lock multiple times.
  */
 
-rte_mcslock_t *p_ml;
-rte_mcslock_t *p_ml_try;
-rte_mcslock_t *p_ml_perf;
+rte_mcslock_t * __rte_atomic p_ml;
+rte_mcslock_t * __rte_atomic p_ml_try;
+rte_mcslock_t * __rte_atomic p_ml_perf;
 
 static unsigned int count;
 

index c00ab78..62fc337 100644
@@ -34,9 +34,9 @@
 #define rte_io_rmb() rte_rmb()
 
 static __rte_always_inline void
-rte_atomic_thread_fence(int memorder)
+rte_atomic_thread_fence(rte_memory_order memorder)
 {
-	__atomic_thread_fence(memorder);
+	__rte_atomic_thread_fence(memorder);
 }
 
 #ifdef __cplusplus

index 6047911..75d8ba6 100644
@@ -38,9 +38,9 @@
 #define rte_io_rmb() rte_rmb()
 
 static __rte_always_inline void
-rte_atomic_thread_fence(int memorder)
+rte_atomic_thread_fence(rte_memory_order memorder)
 {
-	__atomic_thread_fence(memorder);
+	__rte_atomic_thread_fence(memorder);
 }
 
 /*------------------------ 128 bit atomic operations -------------------------*/

@@ -107,33 +107,33 @@
 	 */
 	RTE_SET_USED(failure);
 	/* Find invalid memory order */
-	RTE_ASSERT(success == __ATOMIC_RELAXED ||
-		success == __ATOMIC_ACQUIRE ||
-		success == __ATOMIC_RELEASE ||
-		success == __ATOMIC_ACQ_REL ||
-		success == __ATOMIC_SEQ_CST);
+	RTE_ASSERT(success == rte_memory_order_relaxed ||
+		success == rte_memory_order_acquire ||
+		success == rte_memory_order_release ||
+		success == rte_memory_order_acq_rel ||
+		success == rte_memory_order_seq_cst);
 
 	rte_int128_t expected = *exp;
 	rte_int128_t desired = *src;
 	rte_int128_t old;
 
 #if defined(__ARM_FEATURE_ATOMICS) || defined(RTE_ARM_FEATURE_ATOMICS)
-	if (success == __ATOMIC_RELAXED)
+	if (success == rte_memory_order_relaxed)
 		__cas_128_relaxed(dst, exp, desired);
-	else if (success == __ATOMIC_ACQUIRE)
+	else if (success == rte_memory_order_acquire)
 		__cas_128_acquire(dst, exp, desired);
-	else if (success == __ATOMIC_RELEASE)
+	else if (success == rte_memory_order_release)
 		__cas_128_release(dst, exp, desired);
 	else
 		__cas_128_acq_rel(dst, exp, desired);
 	old = *exp;
 #else
-#define __HAS_ACQ(mo) ((mo) != __ATOMIC_RELAXED && (mo) != __ATOMIC_RELEASE)
-#define __HAS_RLS(mo) ((mo) == __ATOMIC_RELEASE || (mo) == __ATOMIC_ACQ_REL || \
-		(mo) == __ATOMIC_SEQ_CST)
+#define __HAS_ACQ(mo) ((mo) != rte_memory_order_relaxed && (mo) != rte_memory_order_release)
+#define __HAS_RLS(mo) ((mo) == rte_memory_order_release || (mo) == rte_memory_order_acq_rel || \
+		(mo) == rte_memory_order_seq_cst)
 
-	int ldx_mo = __HAS_ACQ(success) ? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED;
-	int stx_mo = __HAS_RLS(success) ? __ATOMIC_RELEASE : __ATOMIC_RELAXED;
+	int ldx_mo = __HAS_ACQ(success) ? rte_memory_order_acquire : rte_memory_order_relaxed;
+	int stx_mo = __HAS_RLS(success) ? rte_memory_order_release : rte_memory_order_relaxed;
 
 #undef __HAS_ACQ
 #undef __HAS_RLS

@@ -153,7 +153,7 @@
 		: "Q" (src->val[0])       \
 		: "memory"); }
 
-		if (ldx_mo == __ATOMIC_RELAXED)
+		if (ldx_mo == rte_memory_order_relaxed)
 			__LOAD_128("ldxp", dst, old)
 		else
 			__LOAD_128("ldaxp", dst, old)

@@ -170,7 +170,7 @@
 		: "memory"); }
 
 		if (likely(old.int128 == expected.int128)) {
-			if (stx_mo == __ATOMIC_RELAXED)
+			if (stx_mo == rte_memory_order_relaxed)
 				__STORE_128("stxp", dst, desired, ret)
 			else
 				__STORE_128("stlxp", dst, desired, ret)

@@ -181,7 +181,7 @@
 			 * needs to be stored back to ensure it was read
 			 * atomically.
 			 */
-			if (stx_mo == __ATOMIC_RELAXED)
+			if (stx_mo == rte_memory_order_relaxed)
 				__STORE_128("stxp", dst, old, ret)
 			else
 				__STORE_128("stlxp", dst, old, ret)

index 5f70e97..d4daafc 100644
@@ -41,7 +41,7 @@ static inline void rte_pause(void)
  * implicitly to exit WFE.
  */
 #define __RTE_ARM_LOAD_EXC_8(src, dst, memorder) {       \
-	if (memorder == __ATOMIC_RELAXED) {               \
+	if (memorder == rte_memory_order_relaxed) {               \
 		asm volatile("ldxrb %w[tmp], [%x[addr]]"  \
 			: [tmp] "=&r" (dst)               \
 			: [addr] "r" (src)                \

@@ -60,7 +60,7 @@ static inline void rte_pause(void)
  * implicitly to exit WFE.
  */
 #define __RTE_ARM_LOAD_EXC_16(src, dst, memorder) {       \
-	if (memorder == __ATOMIC_RELAXED) {               \
+	if (memorder == rte_memory_order_relaxed) {               \
 		asm volatile("ldxrh %w[tmp], [%x[addr]]"  \
 			: [tmp] "=&r" (dst)               \
 			: [addr] "r" (src)                \

@@ -79,7 +79,7 @@ static inline void rte_pause(void)
  * implicitly to exit WFE.
  */
 #define __RTE_ARM_LOAD_EXC_32(src, dst, memorder) {      \
-	if (memorder == __ATOMIC_RELAXED) {              \
+	if (memorder == rte_memory_order_relaxed) {              \
 		asm volatile("ldxr %w[tmp], [%x[addr]]"  \
 			: [tmp] "=&r" (dst)              \
 			: [addr] "r" (src)               \

@@ -98,7 +98,7 @@ static inline void rte_pause(void)
  * implicitly to exit WFE.
  */
 #define __RTE_ARM_LOAD_EXC_64(src, dst, memorder) {      \
-	if (memorder == __ATOMIC_RELAXED) {              \
+	if (memorder == rte_memory_order_relaxed) {              \
 		asm volatile("ldxr %x[tmp], [%x[addr]]"  \
 			: [tmp] "=&r" (dst)              \
 			: [addr] "r" (src)               \

@@ -118,7 +118,7 @@ static inline void rte_pause(void)
  */
 #define __RTE_ARM_LOAD_EXC_128(src, dst, memorder) {                    \
 	volatile rte_int128_t *dst_128 = (volatile rte_int128_t *)&dst; \
-	if (memorder == __ATOMIC_RELAXED) {                             \
+	if (memorder == rte_memory_order_relaxed) {                             \
 		asm volatile("ldxp %x[tmp0], %x[tmp1], [%x[addr]]"      \
 			: [tmp0] "=&r" (dst_128->val[0]),               \
 			  [tmp1] "=&r" (dst_128->val[1])                \

@@ -153,8 +153,8 @@ static inline void rte_pause(void)
 {
 	uint16_t value;
 
-	RTE_BUILD_BUG_ON(memorder != __ATOMIC_ACQUIRE &&
-		memorder != __ATOMIC_RELAXED);
+	RTE_BUILD_BUG_ON(memorder != rte_memory_order_acquire &&
+		memorder != rte_memory_order_relaxed);
 
 	__RTE_ARM_LOAD_EXC_16(addr, value, memorder)
 	if (value != expected) {

@@ -172,8 +172,8 @@ static inline void rte_pause(void)
 {
 	uint32_t value;
 
-	RTE_BUILD_BUG_ON(memorder != __ATOMIC_ACQUIRE &&
-		memorder != __ATOMIC_RELAXED);
+	RTE_BUILD_BUG_ON(memorder != rte_memory_order_acquire &&
+		memorder != rte_memory_order_relaxed);
 
 	__RTE_ARM_LOAD_EXC_32(addr, value, memorder)
 	if (value != expected) {

@@ -191,8 +191,8 @@ static inline void rte_pause(void)
 {
 	uint64_t value;
 
-	RTE_BUILD_BUG_ON(memorder != __ATOMIC_ACQUIRE &&
-		memorder != __ATOMIC_RELAXED);
+	RTE_BUILD_BUG_ON(memorder != rte_memory_order_acquire &&
+		memorder != rte_memory_order_relaxed);
 
 	__RTE_ARM_LOAD_EXC_64(addr, value, memorder)
 	if (value != expected) {

@@ -206,8 +206,8 @@ static inline void rte_pause(void)
 
 #define RTE_WAIT_UNTIL_MASKED(addr, mask, cond, expected, memorder) do {  \
 	RTE_BUILD_BUG_ON(!__builtin_constant_p(memorder));                \
-	RTE_BUILD_BUG_ON(memorder != __ATOMIC_ACQUIRE &&                  \
-		memorder != __ATOMIC_RELAXED);                            \
+	RTE_BUILD_BUG_ON(memorder != rte_memory_order_acquire &&                  \
+		memorder != rte_memory_order_relaxed);                            \
 	const uint32_t size = sizeof(*(addr)) << 3;                       \
 	typeof(*(addr)) expected_value = (expected);                      \
 	typeof(*(addr)) value;                                            \

index 77b96e4..f54cf59 100644
@@ -33,19 +33,19 @@
 
 	switch (pmc->size) {
 	case sizeof(uint8_t):
-		__RTE_ARM_LOAD_EXC_8(pmc->addr, cur_value, __ATOMIC_RELAXED)
+		__RTE_ARM_LOAD_EXC_8(pmc->addr, cur_value, rte_memory_order_relaxed)
 		__RTE_ARM_WFE()
 		break;
 	case sizeof(uint16_t):
-		__RTE_ARM_LOAD_EXC_16(pmc->addr, cur_value, __ATOMIC_RELAXED)
+		__RTE_ARM_LOAD_EXC_16(pmc->addr, cur_value, rte_memory_order_relaxed)
 		__RTE_ARM_WFE()
 		break;
 	case sizeof(uint32_t):
-		__RTE_ARM_LOAD_EXC_32(pmc->addr, cur_value, __ATOMIC_RELAXED)
+		__RTE_ARM_LOAD_EXC_32(pmc->addr, cur_value, rte_memory_order_relaxed)
 		__RTE_ARM_WFE()
 		break;
 	case sizeof(uint64_t):
-		__RTE_ARM_LOAD_EXC_64(pmc->addr, cur_value, __ATOMIC_RELAXED)
+		__RTE_ARM_LOAD_EXC_64(pmc->addr, cur_value, rte_memory_order_relaxed)
 		__RTE_ARM_WFE()
 		break;
 	default:

index cb980af..c6628dd 100644
@@ -103,11 +103,11 @@ struct trace_point_head *
 trace_mode_set(rte_trace_point_t *t, enum rte_trace_mode mode)
 {
 	if (mode == RTE_TRACE_MODE_OVERWRITE)
-		__atomic_fetch_and(t, ~__RTE_TRACE_FIELD_ENABLE_DISCARD,
-			__ATOMIC_RELEASE);
+		rte_atomic_fetch_and_explicit(t, ~__RTE_TRACE_FIELD_ENABLE_DISCARD,
+			rte_memory_order_release);
 	else
-		__atomic_fetch_or(t, __RTE_TRACE_FIELD_ENABLE_DISCARD,
-			__ATOMIC_RELEASE);
+		rte_atomic_fetch_or_explicit(t, __RTE_TRACE_FIELD_ENABLE_DISCARD,
+			rte_memory_order_release);
 }
 
 void

@@ -141,7 +141,7 @@ rte_trace_mode rte_trace_mode_get(void)
 	if (trace_point_is_invalid(t))
 		return false;
 
-	val = __atomic_load_n(t, __ATOMIC_ACQUIRE);
+	val = rte_atomic_load_explicit(t, rte_memory_order_acquire);
 	return (val & __RTE_TRACE_FIELD_ENABLE_MASK) != 0;
 }
 

@@ -153,7 +153,8 @@ rte_trace_mode rte_trace_mode_get(void)
 	if (trace_point_is_invalid(t))
 		return -ERANGE;
 
-	prev = __atomic_fetch_or(t, __RTE_TRACE_FIELD_ENABLE_MASK, __ATOMIC_RELEASE);
+	prev = rte_atomic_fetch_or_explicit(t, __RTE_TRACE_FIELD_ENABLE_MASK,
+	    rte_memory_order_release);
 	if ((prev & __RTE_TRACE_FIELD_ENABLE_MASK) == 0)
 		__atomic_fetch_add(&trace.status, 1, __ATOMIC_RELEASE);
 	return 0;

@@ -167,7 +168,8 @@ rte_trace_mode rte_trace_mode_get(void)
 	if (trace_point_is_invalid(t))
 		return -ERANGE;
 
-	prev = __atomic_fetch_and(t, ~__RTE_TRACE_FIELD_ENABLE_MASK, __ATOMIC_RELEASE);
+	prev = rte_atomic_fetch_and_explicit(t, ~__RTE_TRACE_FIELD_ENABLE_MASK,
+	    rte_memory_order_release);
 	if ((prev & __RTE_TRACE_FIELD_ENABLE_MASK) != 0)
 		__atomic_fetch_sub(&trace.status, 1, __ATOMIC_RELEASE);
 	return 0;

194 lines not on this page · raw message

index bebfa95..c816e7d 100644
@@ -36,13 +36,13 @@
  *  A 16-bit expected value to be in the memory location.
  * @param memorder
  *  Two different memory orders that can be specified:
- *  __ATOMIC_ACQUIRE and __ATOMIC_RELAXED. These map to
+ *  rte_memory_order_acquire and rte_memory_order_relaxed. These map to
  *  C++11 memory orders with the same names, see the C++11 standard or
  *  the GCC wiki on atomic synchronization for detailed definition.
  */
 static __rte_always_inline void
 rte_wait_until_equal_16(volatile uint16_t *addr, uint16_t expected,
-		int memorder);
+		rte_memory_order memorder);
 
 /**
  * Wait for *addr to be updated with a 32-bit expected value, with a relaxed

@@ -54,13 +54,13 @@
  *  A 32-bit expected value to be in the memory location.
  * @param memorder
  *  Two different memory orders that can be specified:
- *  __ATOMIC_ACQUIRE and __ATOMIC_RELAXED. These map to
+ *  rte_memory_order_acquire and rte_memory_order_relaxed. These map to
  *  C++11 memory orders with the same names, see the C++11 standard or
  *  the GCC wiki on atomic synchronization for detailed definition.
  */
 static __rte_always_inline void
 rte_wait_until_equal_32(volatile uint32_t *addr, uint32_t expected,
-		int memorder);
+		rte_memory_order memorder);
 
 /**
  * Wait for *addr to be updated with a 64-bit expected value, with a relaxed

@@ -72,42 +72,42 @@
  *  A 64-bit expected value to be in the memory location.
  * @param memorder
  *  Two different memory orders that can be specified:
- *  __ATOMIC_ACQUIRE and __ATOMIC_RELAXED. These map to
+ *  rte_memory_order_acquire and rte_memory_order_relaxed. These map to
  *  C++11 memory orders with the same names, see the C++11 standard or
  *  the GCC wiki on atomic synchronization for detailed definition.
  */
 static __rte_always_inline void
 rte_wait_until_equal_64(volatile uint64_t *addr, uint64_t expected,
-		int memorder);
+		rte_memory_order memorder);
 
 #ifndef RTE_WAIT_UNTIL_EQUAL_ARCH_DEFINED
 static __rte_always_inline void
 rte_wait_until_equal_16(volatile uint16_t *addr, uint16_t expected,
-		int memorder)
+		rte_memory_order memorder)
 {
-	assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED);
+	assert(memorder == rte_memory_order_acquire || memorder == rte_memory_order_relaxed);
 
-	while (__atomic_load_n(addr, memorder) != expected)
+	while (rte_atomic_load_explicit(addr, memorder) != expected)
 		rte_pause();
 }
 
 static __rte_always_inline void
 rte_wait_until_equal_32(volatile uint32_t *addr, uint32_t expected,
-		int memorder)
+		rte_memory_order memorder)
 {
-	assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED);
+	assert(memorder == rte_memory_order_acquire || memorder == rte_memory_order_relaxed);
 
-	while (__atomic_load_n(addr, memorder) != expected)
+	while (rte_atomic_load_explicit(addr, memorder) != expected)
 		rte_pause();
 }
 
 static __rte_always_inline void
 rte_wait_until_equal_64(volatile uint64_t *addr, uint64_t expected,
-		int memorder)
+		rte_memory_order memorder)
 {
-	assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED);
+	assert(memorder == rte_memory_order_acquire || memorder == rte_memory_order_relaxed);
 
-	while (__atomic_load_n(addr, memorder) != expected)
+	while (rte_atomic_load_explicit(addr, memorder) != expected)
 		rte_pause();
 }
 

@@ -125,16 +125,16 @@
  *  An expected value to be in the memory location.
  * @param memorder
  *  Two different memory orders that can be specified:
- *  __ATOMIC_ACQUIRE and __ATOMIC_RELAXED. These map to
+ *  rte_memory_order_acquire and rte_memory_order_relaxed. These map to
  *  C++11 memory orders with the same names, see the C++11 standard or
  *  the GCC wiki on atomic synchronization for detailed definition.
  */
 #define RTE_WAIT_UNTIL_MASKED(addr, mask, cond, expected, memorder) do { \
 	RTE_BUILD_BUG_ON(!__builtin_constant_p(memorder));               \
-	RTE_BUILD_BUG_ON(memorder != __ATOMIC_ACQUIRE &&                 \
-		memorder != __ATOMIC_RELAXED);                           \
+	RTE_BUILD_BUG_ON((memorder) != rte_memory_order_acquire &&       \
+		(memorder) != rte_memory_order_relaxed);                 \
 	typeof(*(addr)) expected_value = (expected);                     \
-	while (!((__atomic_load_n((addr), (memorder)) & (mask)) cond     \
+	while (!((rte_atomic_load_explicit((addr), (memorder)) & (mask)) cond     \
 			expected_value))                                 \
 		rte_pause();                                             \
 } while (0)

126 lines not on this page · raw message

index e18f0cd..274616a 100644
@@ -29,7 +29,7 @@
  * The rte_spinlock_t type.
  */
 typedef struct __rte_lockable {
-	volatile int locked; /**< lock status 0 = unlocked, 1 = locked */
+	volatile int __rte_atomic locked; /**< lock status 0 = unlocked, 1 = locked */
 } rte_spinlock_t;
 
 /**

@@ -66,10 +66,10 @@
 {
 	int exp = 0;
 
-	while (!__atomic_compare_exchange_n(&sl->locked, &exp, 1, 0,
-				__ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
-		rte_wait_until_equal_32((volatile uint32_t *)&sl->locked,
-			       0, __ATOMIC_RELAXED);
+	while (!rte_atomic_compare_exchange_strong_explicit(&sl->locked, &exp, 1,
+				rte_memory_order_acquire, rte_memory_order_relaxed)) {
+		rte_wait_until_equal_32((volatile uint32_t *)(uintptr_t)&sl->locked,
+			       0, rte_memory_order_relaxed);
 		exp = 0;
 	}
 }

@@ -90,7 +90,7 @@
 rte_spinlock_unlock(rte_spinlock_t *sl)
 	__rte_no_thread_safety_analysis
 {
-	__atomic_store_n(&sl->locked, 0, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&sl->locked, 0, rte_memory_order_release);
 }
 #endif
 

@@ -113,9 +113,8 @@
 	__rte_no_thread_safety_analysis
 {
 	int exp = 0;
-	return __atomic_compare_exchange_n(&sl->locked, &exp, 1,
-				0, /* disallow spurious failure */
-				__ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
+	return rte_atomic_compare_exchange_strong_explicit(&sl->locked, &exp, 1,
+				rte_memory_order_acquire, rte_memory_order_relaxed);
 }
 #endif
 

@@ -129,7 +128,7 @@
  */
 static inline int rte_spinlock_is_locked (rte_spinlock_t *sl)
 {
-	return __atomic_load_n(&sl->locked, __ATOMIC_ACQUIRE);
+	return rte_atomic_load_explicit(&sl->locked, rte_memory_order_acquire);
 }
 
 /**

133 lines not on this page · raw message

index 790be71..a2375b3 100644
@@ -41,8 +41,8 @@
  */
 struct rte_pflock {
 	struct {
-		uint16_t in;
-		uint16_t out;
+		uint16_t __rte_atomic in;
+		uint16_t __rte_atomic out;
 	} rd, wr;
 };
 typedef struct rte_pflock rte_pflock_t;

@@ -117,14 +117,14 @@ struct rte_pflock {
 	 * If no writer is present, then the operation has completed
 	 * successfully.
 	 */
-	w = __atomic_fetch_add(&pf->rd.in, RTE_PFLOCK_RINC, __ATOMIC_ACQUIRE)
+	w = rte_atomic_fetch_add_explicit(&pf->rd.in, RTE_PFLOCK_RINC, rte_memory_order_acquire)
 		& RTE_PFLOCK_WBITS;
 	if (w == 0)
 		return;
 
 	/* Wait for current write phase to complete. */
 	RTE_WAIT_UNTIL_MASKED(&pf->rd.in, RTE_PFLOCK_WBITS, !=, w,
-		__ATOMIC_ACQUIRE);
+		rte_memory_order_acquire);
 }
 
 /**

@@ -140,7 +140,7 @@ struct rte_pflock {
 static inline void
 rte_pflock_read_unlock(rte_pflock_t *pf)
 {
-	__atomic_fetch_add(&pf->rd.out, RTE_PFLOCK_RINC, __ATOMIC_RELEASE);
+	rte_atomic_fetch_add_explicit(&pf->rd.out, RTE_PFLOCK_RINC, rte_memory_order_release);
 }
 
 /**

@@ -161,8 +161,9 @@ struct rte_pflock {
 	/* Acquire ownership of write-phase.
 	 * This is same as rte_ticketlock_lock().
 	 */
-	ticket = __atomic_fetch_add(&pf->wr.in, 1, __ATOMIC_RELAXED);
-	rte_wait_until_equal_16(&pf->wr.out, ticket, __ATOMIC_ACQUIRE);
+	ticket = rte_atomic_fetch_add_explicit(&pf->wr.in, 1, rte_memory_order_relaxed);
+	rte_wait_until_equal_16((uint16_t *)(uintptr_t)&pf->wr.out, ticket,
+	    rte_memory_order_acquire);
 
 	/*
 	 * Acquire ticket on read-side in order to allow them

@@ -173,10 +174,11 @@ struct rte_pflock {
 	 * speculatively.
 	 */
 	w = RTE_PFLOCK_PRES | (ticket & RTE_PFLOCK_PHID);
-	ticket = __atomic_fetch_add(&pf->rd.in, w, __ATOMIC_RELAXED);
+	ticket = rte_atomic_fetch_add_explicit(&pf->rd.in, w, rte_memory_order_relaxed);
 
 	/* Wait for any pending readers to flush. */
-	rte_wait_until_equal_16(&pf->rd.out, ticket, __ATOMIC_ACQUIRE);
+	rte_wait_until_equal_16((uint16_t *)(uintptr_t)&pf->rd.out, ticket,
+	    rte_memory_order_acquire);
 }
 
 /**

@@ -193,10 +195,10 @@ struct rte_pflock {
 rte_pflock_write_unlock(rte_pflock_t *pf)
 {
 	/* Migrate from write phase to read phase. */
-	__atomic_fetch_and(&pf->rd.in, RTE_PFLOCK_LSB, __ATOMIC_RELEASE);
+	rte_atomic_fetch_and_explicit(&pf->rd.in, RTE_PFLOCK_LSB, rte_memory_order_release);
 
 	/* Allow other writers to continue. */
-	__atomic_fetch_add(&pf->wr.out, 1, __ATOMIC_RELEASE);
+	rte_atomic_fetch_add_explicit(&pf->wr.out, 1, rte_memory_order_release);
 }
 
 #ifdef __cplusplus

index 098af26..a658178 100644
@@ -32,7 +32,7 @@
  * The RTE seqcount type.
  */
 typedef struct {
-	uint32_t sn; /**< A sequence number for the protected data. */
+	uint32_t __rte_atomic sn; /**< A sequence number for the protected data. */
 } rte_seqcount_t;
 
 /**

@@ -106,11 +106,11 @@
 static inline uint32_t
 rte_seqcount_read_begin(const rte_seqcount_t *seqcount)
 {
-	/* __ATOMIC_ACQUIRE to prevent loads after (in program order)
+	/* rte_memory_order_acquire to prevent loads after (in program order)
 	 * from happening before the sn load. Synchronizes-with the
 	 * store release in rte_seqcount_write_end().
 	 */
-	return __atomic_load_n(&seqcount->sn, __ATOMIC_ACQUIRE);
+	return rte_atomic_load_explicit(&seqcount->sn, rte_memory_order_acquire);
 }
 
 /**

@@ -161,9 +161,9 @@
 		return true;
 
 	/* make sure the data loads happens before the sn load */
-	rte_atomic_thread_fence(__ATOMIC_ACQUIRE);
+	rte_atomic_thread_fence(rte_memory_order_acquire);
 
-	end_sn = __atomic_load_n(&seqcount->sn, __ATOMIC_RELAXED);
+	end_sn = rte_atomic_load_explicit(&seqcount->sn, rte_memory_order_relaxed);
 
 	/* A writer incremented the sequence number during this read
 	 * critical section.

@@ -205,12 +205,12 @@
 
 	sn = seqcount->sn + 1;
 
-	__atomic_store_n(&seqcount->sn, sn, __ATOMIC_RELAXED);
+	rte_atomic_store_explicit(&seqcount->sn, sn, rte_memory_order_relaxed);
 
-	/* __ATOMIC_RELEASE to prevent stores after (in program order)
+	/* rte_memory_order_release to prevent stores after (in program order)
 	 * from happening before the sn store.
 	 */
-	rte_atomic_thread_fence(__ATOMIC_RELEASE);
+	rte_atomic_thread_fence(rte_memory_order_release);
 }
 
 /**

@@ -237,7 +237,7 @@
 	sn = seqcount->sn + 1;
 
 	/* Synchronizes-with the load acquire in rte_seqcount_read_begin(). */
-	__atomic_store_n(&seqcount->sn, sn, __ATOMIC_RELEASE);
+	rte_atomic_store_explicit(&seqcount->sn, sn, rte_memory_order_release);
 }
 
 #ifdef __cplusplus

115 lines not on this page · raw message

index d587591..e682109 100644
@@ -33,7 +33,7 @@
 #include <rte_stdatomic.h>
 
 /** The tracepoint object. */
-typedef uint64_t rte_trace_point_t;
+typedef uint64_t __rte_atomic rte_trace_point_t;
 
 /**
  * Macro to define the tracepoint arguments in RTE_TRACE_POINT macro.

@@ -359,7 +359,7 @@ struct __rte_trace_header {
 #define __rte_trace_point_emit_header_generic(t) \
 void *mem; \
 do { \
-	const uint64_t val = __atomic_load_n(t, __ATOMIC_ACQUIRE); \
+	const uint64_t val = rte_atomic_load_explicit(t, rte_memory_order_acquire); \
 	if (likely(!(val & __RTE_TRACE_FIELD_ENABLE_MASK))) \
 		return; \
 	mem = __rte_trace_mem_get(val); \

index 3c82845..0510b8f 100644
@@ -35,9 +35,9 @@
 #define rte_io_rmb()	rte_mb()
 
 static __rte_always_inline void
-rte_atomic_thread_fence(int memorder)
+rte_atomic_thread_fence(rte_memory_order memorder)
 {
-	__atomic_thread_fence(memorder);
+	__rte_atomic_thread_fence(memorder);
 }
 
 #ifdef __cplusplus

172 lines not on this page · raw message

index 4b4633c..2603bc9 100644
@@ -40,9 +40,9 @@
 #define rte_io_rmb()	asm volatile("fence ir, ir" : : : "memory")
 
 static __rte_always_inline void
-rte_atomic_thread_fence(int memorder)
+rte_atomic_thread_fence(rte_memory_order memorder)
 {
-	__atomic_thread_fence(memorder);
+	__rte_atomic_thread_fence(memorder);
 }
 
 #ifdef __cplusplus

index f2ee1a9..3b3a9a4 100644
@@ -82,17 +82,17 @@
 /**
  * Synchronization fence between threads based on the specified memory order.
  *
- * On x86 the __atomic_thread_fence(__ATOMIC_SEQ_CST) generates full 'mfence'
+ * On x86 the __rte_atomic_thread_fence(rte_memory_order_seq_cst) generates full 'mfence'
  * which is quite expensive. The optimized implementation of rte_smp_mb is
  * used instead.
  */
 static __rte_always_inline void
-rte_atomic_thread_fence(int memorder)
+rte_atomic_thread_fence(rte_memory_order memorder)
 {
-	if (memorder == __ATOMIC_SEQ_CST)
+	if (memorder == rte_memory_order_seq_cst)
 		rte_smp_mb();
 	else
-		__atomic_thread_fence(memorder);
+		__rte_atomic_thread_fence(memorder);
 }
 
 /*------------------------- 16 bit atomic operations -------------------------*/

index 0b20ddf..c76218a 100644
@@ -78,7 +78,7 @@ static inline int rte_tm_supported(void)
 }
 
 static inline int
-rte_try_tm(volatile int *lock)
+rte_try_tm(volatile int __rte_atomic *lock)
 {
 	int i, retries;
 

index f749da9..cf70e33 100644
@@ -23,9 +23,9 @@
 	uint64_t val;
 
 	/* trigger a write but don't change the value */
-	val = __atomic_load_n((volatile uint64_t *)addr, __ATOMIC_RELAXED);
-	__atomic_compare_exchange_n((volatile uint64_t *)addr, &val, val, 0,
-			__ATOMIC_RELAXED, __ATOMIC_RELAXED);
+	val = rte_atomic_load_explicit((volatile uint64_t *)addr, rte_memory_order_relaxed);
+	rte_atomic_compare_exchange_strong_explicit((volatile uint64_t *)addr, &val, val,
+			rte_memory_order_relaxed, rte_memory_order_relaxed);
 }
 
 static bool wait_supported;

index f6c4b3e..4f954e0 100644
@@ -274,7 +274,8 @@
 static inline void
 rte_atomic16_add(rte_atomic16_t *v, int16_t inc)
 {
-	rte_atomic_fetch_add_explicit(&v->cnt, inc, rte_memory_order_seq_cst);
+	rte_atomic_fetch_add_explicit((volatile int16_t __rte_atomic *)&v->cnt, inc,
+	    rte_memory_order_seq_cst);
 }
 
 /**

@@ -288,7 +289,8 @@
 static inline void
 rte_atomic16_sub(rte_atomic16_t *v, int16_t dec)
 {
-	rte_atomic_fetch_sub_explicit(&v->cnt, dec, rte_memory_order_seq_cst);
+	rte_atomic_fetch_sub_explicit((volatile int16_t __rte_atomic *)&v->cnt, dec,
+	    rte_memory_order_seq_cst);
 }
 
 /**

@@ -341,7 +343,8 @@
 static inline int16_t
 rte_atomic16_add_return(rte_atomic16_t *v, int16_t inc)
 {
-	return rte_atomic_fetch_add_explicit(&v->cnt, inc, rte_memory_order_seq_cst) + inc;
+	return rte_atomic_fetch_add_explicit((volatile int16_t __rte_atomic *)&v->cnt, inc,
+	    rte_memory_order_seq_cst) + inc;
 }
 
 /**

@@ -361,7 +364,8 @@
 static inline int16_t
 rte_atomic16_sub_return(rte_atomic16_t *v, int16_t dec)
 {
-	return rte_atomic_fetch_sub_explicit(&v->cnt, dec, rte_memory_order_seq_cst) - dec;
+	return rte_atomic_fetch_sub_explicit((volatile int16_t __rte_atomic *)&v->cnt, dec,
+	    rte_memory_order_seq_cst) - dec;
 }
 
 /**

@@ -380,7 +384,8 @@
 #ifdef RTE_FORCE_INTRINSICS
 static inline int rte_atomic16_inc_and_test(rte_atomic16_t *v)
 {
-	return rte_atomic_fetch_add_explicit(&v->cnt, 1, rte_memory_order_seq_cst) + 1 == 0;
+	return rte_atomic_fetch_add_explicit((volatile int16_t __rte_atomic *)&v->cnt, 1,
+	    rte_memory_order_seq_cst) + 1 == 0;
 }
 #endif
 

@@ -400,7 +405,8 @@ static inline int rte_atomic16_inc_and_test(rte_atomic16_t *v)
 #ifdef RTE_FORCE_INTRINSICS
 static inline int rte_atomic16_dec_and_test(rte_atomic16_t *v)
 {
-	return rte_atomic_fetch_sub_explicit(&v->cnt, 1, rte_memory_order_seq_cst) - 1 == 0;
+	return rte_atomic_fetch_sub_explicit((volatile int16_t __rte_atomic *)&v->cnt, 1,
+	    rte_memory_order_seq_cst) - 1 == 0;
 }
 #endif
 

@@ -553,7 +559,8 @@ static inline void rte_atomic16_clear(rte_atomic16_t *v)
 static inline void
 rte_atomic32_add(rte_atomic32_t *v, int32_t inc)
 {
-	rte_atomic_fetch_add_explicit(&v->cnt, inc, rte_memory_order_seq_cst);
+	rte_atomic_fetch_add_explicit((volatile int32_t __rte_atomic *)&v->cnt, inc,
+	    rte_memory_order_seq_cst);
 }
 
 /**

@@ -567,7 +574,8 @@ static inline void rte_atomic16_clear(rte_atomic16_t *v)
 static inline void
 rte_atomic32_sub(rte_atomic32_t *v, int32_t dec)
 {
-	rte_atomic_fetch_sub_explicit(&v->cnt, dec, rte_memory_order_seq_cst);
+	rte_atomic_fetch_sub_explicit((volatile int32_t __rte_atomic *)&v->cnt, dec,
+	    rte_memory_order_seq_cst);
 }
 
 /**

@@ -620,7 +628,8 @@ static inline void rte_atomic16_clear(rte_atomic16_t *v)
 static inline int32_t
 rte_atomic32_add_return(rte_atomic32_t *v, int32_t inc)
 {
-	return rte_atomic_fetch_add_explicit(&v->cnt, inc, rte_memory_order_seq_cst) + inc;
+	return rte_atomic_fetch_add_explicit((volatile int32_t __rte_atomic *)&v->cnt, inc,
+	    rte_memory_order_seq_cst) + inc;
 }
 
 /**

@@ -640,7 +649,8 @@ static inline void rte_atomic16_clear(rte_atomic16_t *v)
 static inline int32_t
 rte_atomic32_sub_return(rte_atomic32_t *v, int32_t dec)
 {
-	return rte_atomic_fetch_sub_explicit(&v->cnt, dec, rte_memory_order_seq_cst) - dec;
+	return rte_atomic_fetch_sub_explicit((volatile int32_t __rte_atomic *)&v->cnt, dec,
+	    rte_memory_order_seq_cst) - dec;
 }
 
 /**

@@ -659,7 +669,8 @@ static inline void rte_atomic16_clear(rte_atomic16_t *v)
 #ifdef RTE_FORCE_INTRINSICS
 static inline int rte_atomic32_inc_and_test(rte_atomic32_t *v)
 {
-	return rte_atomic_fetch_add_explicit(&v->cnt, 1, rte_memory_order_seq_cst) + 1 == 0;
+	return rte_atomic_fetch_add_explicit((volatile int32_t __rte_atomic *)&v->cnt, 1,
+	    rte_memory_order_seq_cst) + 1 == 0;
 }
 #endif
 

@@ -679,7 +690,8 @@ static inline int rte_atomic32_inc_and_test(rte_atomic32_t *v)
 #ifdef RTE_FORCE_INTRINSICS
 static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v)
 {
-	return rte_atomic_fetch_sub_explicit(&v->cnt, 1, rte_memory_order_seq_cst) - 1 == 0;
+	return rte_atomic_fetch_sub_explicit((volatile int32_t __rte_atomic *)&v->cnt, 1,
+	    rte_memory_order_seq_cst) - 1 == 0;
 }
 #endif
 

@@ -885,7 +897,8 @@ static inline void rte_atomic32_clear(rte_atomic32_t *v)
 static inline void
 rte_atomic64_add(rte_atomic64_t *v, int64_t inc)
 {
-	rte_atomic_fetch_add_explicit(&v->cnt, inc, rte_memory_order_seq_cst);
+	rte_atomic_fetch_add_explicit((volatile int64_t __rte_atomic *)&v->cnt, inc,
+	    rte_memory_order_seq_cst);
 }
 #endif
 

@@ -904,7 +917,8 @@ static inline void rte_atomic32_clear(rte_atomic32_t *v)
 static inline void
 rte_atomic64_sub(rte_atomic64_t *v, int64_t dec)
 {
-	rte_atomic_fetch_sub_explicit(&v->cnt, dec, rte_memory_order_seq_cst);
+	rte_atomic_fetch_sub_explicit((volatile int64_t __rte_atomic *)&v->cnt, dec,
+	    rte_memory_order_seq_cst);
 }
 #endif
 

@@ -962,7 +976,8 @@ static inline void rte_atomic32_clear(rte_atomic32_t *v)
 static inline int64_t
 rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc)
 {
-	return rte_atomic_fetch_add_explicit(&v->cnt, inc, rte_memory_order_seq_cst) + inc;
+	return rte_atomic_fetch_add_explicit((volatile int64_t __rte_atomic *)&v->cnt, inc,
+	    rte_memory_order_seq_cst) + inc;
 }
 #endif
 

@@ -986,7 +1001,8 @@ static inline void rte_atomic32_clear(rte_atomic32_t *v)
 static inline int64_t
 rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec)
 {
-	return rte_atomic_fetch_sub_explicit(&v->cnt, dec, rte_memory_order_seq_cst) - dec;
+	return rte_atomic_fetch_sub_explicit((volatile int64_t __rte_atomic *)&v->cnt, dec,
+	    rte_memory_order_seq_cst) - dec;
 }
 #endif
 

index c816e7d..c261689 100644
@@ -87,7 +87,8 @@
 {
 	assert(memorder == rte_memory_order_acquire || memorder == rte_memory_order_relaxed);
 
-	while (rte_atomic_load_explicit(addr, memorder) != expected)
+	while (rte_atomic_load_explicit((volatile uint16_t __rte_atomic *)addr, memorder)
+	    != expected)
 		rte_pause();
 }
 

@@ -97,7 +98,8 @@
 {
 	assert(memorder == rte_memory_order_acquire || memorder == rte_memory_order_relaxed);
 
-	while (rte_atomic_load_explicit(addr, memorder) != expected)
+	while (rte_atomic_load_explicit((volatile uint32_t __rte_atomic *)addr, memorder)
+	    != expected)
 		rte_pause();
 }
 

@@ -107,7 +109,8 @@
 {
 	assert(memorder == rte_memory_order_acquire || memorder == rte_memory_order_relaxed);
 
-	while (rte_atomic_load_explicit(addr, memorder) != expected)
+	while (rte_atomic_load_explicit((volatile uint64_t __rte_atomic *)addr, memorder)
+	    != expected)
 		rte_pause();
 }
 

index cf70e33..6c192f0 100644
@@ -23,9 +23,10 @@
 	uint64_t val;
 
 	/* trigger a write but don't change the value */
-	val = rte_atomic_load_explicit((volatile uint64_t *)addr, rte_memory_order_relaxed);
-	rte_atomic_compare_exchange_strong_explicit((volatile uint64_t *)addr, &val, val,
-			rte_memory_order_relaxed, rte_memory_order_relaxed);
+	val = rte_atomic_load_explicit((volatile uint64_t __rte_atomic *)addr,
+	    rte_memory_order_relaxed);
+	rte_atomic_compare_exchange_strong_explicit((volatile uint64_t __rte_atomic *)addr,
+	    &val, val, rte_memory_order_relaxed, rte_memory_order_relaxed);
 }
 
 static bool wait_supported;

index 7101f63..ffbdae5 100644
@@ -52,7 +52,7 @@
  * Only 64-bits of the memory is actually used though.
  */
 union rte_distributor_buffer_single {
-	volatile int64_t bufptr64;
+	volatile int64_t __rte_atomic bufptr64;
 	char pad[RTE_CACHE_LINE_SIZE*3];
 } __rte_cache_aligned;
 

117 lines not on this page · raw message

index ffd2db7..b300447 100644
@@ -25,7 +25,7 @@
 
 struct bpf_eth_cbi {
 	/* used by both data & control path */
-	uint32_t use;    /*usage counter */
+	uint32_t __rte_atomic use;    /*usage counter */
 	const struct rte_eth_rxtx_callback *cb;  /* callback handle */
 	struct rte_bpf *bpf;
 	struct rte_bpf_jit jit;

@@ -110,8 +110,8 @@ struct bpf_eth_cbh {
 
 	/* in use, busy wait till current RX/TX iteration is finished */
 	if ((puse & BPF_ETH_CBI_INUSE) != 0) {
-		RTE_WAIT_UNTIL_MASKED((uint32_t *)(uintptr_t)&cbi->use,
-			UINT32_MAX, !=, puse, __ATOMIC_RELAXED);
+		RTE_WAIT_UNTIL_MASKED((uint32_t __rte_atomic *)(uintptr_t)&cbi->use,
+			UINT32_MAX, !=, puse, rte_memory_order_relaxed);
 	}
 }