Skip to contentSkip to conversation

eal/arm64: add 128-bit atomic compare exchange

PATCH Superseded Phil Yang 7 participants 88 messages Jun 23, 2019 – Oct 21, 2019

Subsystems (2)
lore
Share & tools
lore

Phil Yang

author
#0 lore raw

PATCH 1/3 eal/arm64: add 128-bit atomic compare exchange

Add 128-bit atomic compare exchange on aarch64.

---
This patch depends on
http://patchwork.dpdk.org/patch/54840/

v2:
eal/stack: fix 'pointer-sign' warning

 .../common/include/arch/arm/rte_atomic_64.h        | 184 +++++++++++++++++++++
 .../common/include/arch/x86/rte_atomic_64.h        |  12 --
 lib/librte_eal/common/include/generic/rte_atomic.h |  15 +-
 3 files changed, 198 insertions(+), 13 deletions(-)
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +184 -0
index 97060e4..4ef900c 100644
@@ -1,5 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright(c) 2015 Cavium, Inc
+ * Copyright(c) 2019 Arm Limited
  */
 
 #ifndef _RTE_ATOMIC_ARM64_H_
@@ -14,6 +15,9 @@ extern "C" {
 #endif
 
 #include "generic/rte_atomic.h"
+#include <rte_branch_prediction.h>
+#include <rte_compat.h>
+#include <rte_debug.h>
 
 #define dsb(opt) asm volatile("dsb " #opt : : : "memory")
 #define dmb(opt) asm volatile("dmb " #opt : : : "memory")
@@ -40,6 +44,186 @@ extern "C" {
 
 #define rte_cio_rmb() dmb(oshld)
 
+/*----------------------- 128 bit atomic operations -------------------------*/
+
+#define RTE_HAS_ACQ(mo) ((mo) != __ATOMIC_RELAXED && (mo) != __ATOMIC_RELEASE)
+#define RTE_HAS_RLS(mo) ((mo) == __ATOMIC_RELEASE || \
+			 (mo) == __ATOMIC_ACQ_REL || \
+			 (mo) == __ATOMIC_SEQ_CST)
+
+#define RTE_MO_LOAD(mo)  (RTE_HAS_ACQ((mo)) \
+		? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED)
+#define RTE_MO_STORE(mo) (RTE_HAS_RLS((mo)) \
+		? __ATOMIC_RELEASE : __ATOMIC_RELAXED)
+
+#ifdef __ARM_FEATURE_ATOMICS
+static inline rte_int128_t
+__rte_casp(rte_int128_t *dst, rte_int128_t old, rte_int128_t updated, int mo)
+{
+
+	/* caspX instructions register pair must start from even-numbered
+	 * register at operand 1.
+	 * So, specify registers for local variables here.
+	 */
+	register uint64_t x0 __asm ("x0") = (uint64_t)old.val[0];
+	register uint64_t x1 __asm ("x1") = (uint64_t)old.val[1];
+	register uint64_t x2 __asm ("x2") = (uint64_t)updated.val[0];
+	register uint64_t x3 __asm ("x3") = (uint64_t)updated.val[1];
+
+	if (mo ==  __ATOMIC_RELAXED) {
+		asm volatile(
+				"casp %[old0], %[old1], %[upd0], %[upd1], [%[dst]]"
+				: [old0] "+r" (x0),
+				  [old1] "+r" (x1)
+				: [upd0] "r" (x2),
+				  [upd1] "r" (x3),
+				  [dst] "r" (dst)
+				: "memory");
+	} else if (mo == __ATOMIC_ACQUIRE) {
+		asm volatile(
+				"caspa %[old0], %[old1], %[upd0], %[upd1], [%[dst]]"
+				: [old0] "+r" (x0),
+				  [old1] "+r" (x1)
+				: [upd0] "r" (x2),
+				  [upd1] "r" (x3),
+				  [dst] "r" (dst)
+				: "memory");
+	} else if (mo == __ATOMIC_ACQ_REL) {
+		asm volatile(
+				"caspal %[old0], %[old1], %[upd0], %[upd1], [%[dst]]"
+				: [old0] "+r" (x0),
+				  [old1] "+r" (x1)
+				: [upd0] "r" (x2),
+				  [upd1] "r" (x3),
+				  [dst] "r" (dst)
+				: "memory");
+	} else if (mo == __ATOMIC_RELEASE) {
+		asm volatile(
+				"caspl %[old0], %[old1], %[upd0], %[upd1], [%[dst]]"
+				: [old0] "+r" (x0),
+				  [old1] "+r" (x1)
+				: [upd0] "r" (x2),
+				  [upd1] "r" (x3),
+				  [dst] "r" (dst)
+				: "memory");
+	} else {
+		rte_panic("Invalid memory order\n");
+	}
+
+	old.val[0] = x0;
+	old.val[1] = x1;
+
+	return old;
+}
+#else
+static inline rte_int128_t
+__rte_ldx128(const rte_int128_t *src, int mo)
+{
+	rte_int128_t ret;
+	if (mo == __ATOMIC_ACQUIRE)
+		asm volatile(
+				"ldaxp %0, %1, %2"
+				: "=&r" (ret.val[0]),
+				  "=&r" (ret.val[1])
+				: "Q" (src->val[0])
+				: "memory");
+	else if (mo == __ATOMIC_RELAXED)
+		asm volatile(
+				"ldxp %0, %1, %2"
+				: "=&r" (ret.val[0]),
+				  "=&r" (ret.val[1])
+				: "Q" (src->val[0])
+				: "memory");
+	else
+		rte_panic("Invalid memory order\n");
+
+	return ret;
+}
+
+static inline uint32_t
+__rte_stx128(rte_int128_t *dst, const rte_int128_t src, int mo)
+{
+	uint32_t ret;
+	if (mo == __ATOMIC_RELEASE)
+		asm volatile(
+				"stlxp %w0, %1, %2, %3"
+				: "=&r" (ret)
+				: "r" (src.val[0]),
+				  "r" (src.val[1]),
+				  "Q" (dst->val[0])
+				: "memory");
+	else if (mo == __ATOMIC_RELAXED)
+		asm volatile(
+				"stxp %w0, %1, %2, %3"
+				: "=&r" (ret)
+				: "r" (src.val[0]),
+				  "r" (src.val[1]),
+				  "Q" (dst->val[0])
+				: "memory");
+	else
+		rte_panic("Invalid memory order\n");
+
+	/* Return 0 on success, 1 on failure */
+	return ret;
+}
+#endif
+
+static inline int __rte_experimental
+rte_atomic128_cmp_exchange(rte_int128_t *dst,
+				rte_int128_t *exp,
+				const rte_int128_t *src,
+				unsigned int weak,
+				int success,
+				int failure)
+{
+	// Always do strong CAS
+	RTE_SET_USED(weak);
+	/* Ignore memory ordering for failure, memory order for
+	 * success must be stronger or equal
+	 */
+	RTE_SET_USED(failure);
+
+#ifdef __ARM_FEATURE_ATOMICS
+	rte_int128_t expected = *exp;
+	rte_int128_t desired = *src;
+	rte_int128_t old;
+
+	old = __rte_casp(dst, expected, desired, success);
+#else
+	int ldx_mo = RTE_MO_LOAD(success);
+	int stx_mo = RTE_MO_STORE(success);
+	uint32_t ret = 1;
+	register rte_int128_t expected = *exp;
+	register rte_int128_t desired = *src;
+	register rte_int128_t old;
+
+	/* ldx128 can not guarantee atomic,
+	 * Must write back src or old to verify atomicity of ldx128;
+	 */
+	do {
+		old = __rte_ldx128(dst, ldx_mo);
+		if (likely(old.int128 == expected.int128))
+			ret = __rte_stx128(dst, desired, stx_mo);
+		else
+			/* In the failure case (since 'weak' is ignored and only
+			 * weak == 0 is implemented), expected should contain the
+			 * atomically read value of dst. This means, 'old' needs
+			 * to be stored back to ensure it was read atomically.
+			 */
+			ret = __rte_stx128(dst, old, stx_mo);
+	} while (unlikely(ret));
+#endif
+
+	/* Unconditionally updating expected removes
+	 * an 'if' statement.
+	 * expected should already be in register if
+	 * not in the cache.
+	 */
+	*exp = old;
+
+	return (old.int128 == expected.int128);
+}
+
 #ifdef __cplusplus
 }
 #endif
lib/librte_eal/common/include/arch/x86/rte_atomic_64.h +0 -12
index 6232c57..23cf48f 100644
@@ -212,18 +212,6 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)
 
 /*------------------------ 128 bit atomic operations -------------------------*/
 
-/**
- * 128-bit integer structure.
- */
-RTE_STD_C11
-typedef struct {
-	RTE_STD_C11
-	union {
-		uint64_t val[2];
-		__extension__ __int128 int128;
-	};
-} __rte_aligned(16) rte_int128_t;
-
 static inline int __rte_experimental
 rte_atomic128_cmp_exchange(rte_int128_t *dst,
 			   rte_int128_t *exp,
lib/librte_eal/common/include/generic/rte_atomic.h +14 -1
index 9958543..7dd1aa4 100644
@@ -1081,6 +1081,18 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)
 
 /*------------------------ 128 bit atomic operations -------------------------*/
 
+/**
+ * 128-bit integer structure.
+ */
+RTE_STD_C11
+typedef struct {
+	RTE_STD_C11
+	union {
+		uint64_t val[2];
+		__extension__ __int128 int128;
+	};
+} __rte_aligned(16) rte_int128_t;
+
 #ifdef __DOXYGEN__
 
 /**
@@ -1093,7 +1105,8 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)
  *     *exp = *dst
  * @endcode
  *
- * @note This function is currently only available for the x86-64 platform.
+ * @note This function is currently available for the x86-64 and aarch64
+ * platforms.
  *
  * @note The success and failure arguments must be one of the __ATOMIC_* values
  * defined in the C++11 standard. For details on their behavior, refer to the
Signature · 1 line
2.7.4
author
#1 lore raw

PATCH 2/3 test/atomic: add 128b compare and swap test

Add 128b atomic compare and swap test for aarch64 and x86_64.

---
 app/test/test_atomic.c | 119 ++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 118 insertions(+), 1 deletion(-)
app/test/test_atomic.c +118 -1
index 43be30e..b248063 100644
@@ -1,5 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright(c) 2010-2014 Intel Corporation
+ * Copyright(c) 2019 Arm Limited
  */
 
 #include <stdio.h>
@@ -20,7 +21,7 @@
  * Atomic Variables
  * ================
  *
- * - The main test function performs three subtests. The first test
+ * - The main test function performs four subtests. The first test
  *   checks that the usual inc/dec/add/sub functions are working
  *   correctly:
  *
@@ -61,6 +62,22 @@
  *       atomic_sub(&count, tmp+1);
  *
  *   - At the end of the test, the *count* value must be 0.
+ *
+ * - Test "128b compare and swap" (aarch64 and x86_64 only)
+ *
+ *   - Initialize 128-bit atomic variables to zero.
+ *
+ *   - Invoke ``test_atomici128_cmp_exchange()`` on each lcore. Before doing
+ *     anything else, the cores are waiting a synchro. Each lcore does
+ *     these compare and swap (CAS) operations several times::
+ *
+ *       Relaxed CAS update counter.val[0] + 2; counter.val[0] + 1;
+ *       Acquired CAS update counter.val[0] + 2; counter.val[0] + 1;
+ *       Released CAS update counter.val[0] + 2; counter.val[0] + 1;
+ *       Acquired_Released CAS update counter.val[0] + 2; counter.val[0] + 1;
+ *
+ *   - At the end of the test, the *count128* first 64-bit value and
+ *     second 64-bit value differ by the total iterations.
  */
 
 #define NUM_ATOMIC_TYPES 3
@@ -73,6 +90,10 @@ static rte_atomic64_t a64;
 static rte_atomic64_t count;
 static rte_atomic32_t synchro;
 
+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)
+static rte_int128_t count128;
+#endif
+
 static int
 test_atomic_usual(__attribute__((unused)) void *arg)
 {
@@ -216,6 +237,71 @@ test_atomic_dec_and_test(__attribute__((unused)) void *arg)
 	return 0;
 }
 
+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)
+/*
+ * rte_atomic128_cmp_exchange() should update a 128 bits counter's first 64
+ * bits by 2 and the second 64 bits by 1 in this test. It should return true
+ * if the compare exchange operation successful.
+ * This test repeat 128 bits compare and swap operations 10K rounds. In each
+ * iteration it runs compare and swap operation with different memory models.
+ */
+static int
+test_atomic128_cmp_exchange(__attribute__((unused)) void *arg)
+{
+	while (rte_atomic32_read(&synchro) == 0)
+		;
+
+	rte_int128_t expected;
+	int success;
+
+	expected = count128;
+
+	for (int i=0; i < N; i++) {
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128, &expected,
+					&desired, 1, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
+		} while (success == 0);
+
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128, &expected,
+					&desired, 1, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
+		} while (success == 0);
+
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128, &expected,
+					&desired, 1, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED);
+		} while (success == 0);
+
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128, &expected,
+					&desired, 1, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+		} while (success == 0);
+	}
+
+	return 0;
+}
+#endif
+
 static int
 test_atomic(void)
 {
@@ -340,6 +426,37 @@ test_atomic(void)
 		return -1;
 	}
 
+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)
+	/*
+	 * This case tests the functionality of rte_atomic128b_cmp_exchange
+	 * API. It calls rte_atomic128b_cmp_exchange with four kinds of memory
+	 * models successively on each slave core. Once each 128-bit atomic
+	 * compare and swap operation is successful, it updates the global
+	 * 128-bit counter by 2 for the first 64-bit and 1 for the second
+	 * 64-bit. Each slave core iterates this test 10K times.
+	 * At the end of test, verify whether the first 64-bits of the 128-bit
+	 * counter and the second 64bits is differ by the total iterations. If
+	 * it is, the test passes.
+	 */
+	printf("128b compare and swap test\n");
+	uint64_t iterations = 0;
+
+	rte_atomic32_clear(&synchro);
+	count128.val[0] = 0;
+	count128.val[1] = 0;
+
+	rte_eal_mp_remote_launch(test_atomic128_cmp_exchange, NULL, SKIP_MASTER);
+	rte_atomic32_set(&synchro, 1);
+	rte_eal_mp_wait_lcore();
+	rte_atomic32_clear(&synchro);
+
+	iterations = count128.val[0] - count128.val[1];
+	if (iterations != 4*N*(rte_lcore_count()-1)) {
+		printf("128b compare and swap failed\n");
+		return -1;
+	}
+#endif
+
 	return 0;
 }
 
Signature · 1 line
2.7.4
author
#2 lore raw

PATCH 3/3 eal/stack: enable lock-free stack for aarch64

In reply to Phil Yang

Enable both c11 atomic and non c11 atomic lock-free stack for aarch64.

---
 doc/guides/rel_notes/release_19_08.rst  | 3 +++
 lib/librte_stack/rte_stack_lf_c11.h     | 4 ++--
 lib/librte_stack/rte_stack_lf_generic.h | 4 ++--
 3 files changed, 7 insertions(+), 4 deletions(-)
doc/guides/rel_notes/release_19_08.rst +3 -0
index 8c3932d..b79ae28 100644
@@ -88,6 +88,9 @@ New Features
   * Added multi-queue support to allow one af_xdp vdev with multiple netdev
     queues
 
+* **Added Lock-free Stack for aarch64.**
+
+  The lock-free stack implementation is enabled for aarch64 platforms.
 
 Removed Items
 -------------
lib/librte_stack/rte_stack_lf_c11.h +2 -2
index 3d677ae..67c21fd 100644
@@ -36,7 +36,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 			  struct rte_stack_lf_elem *last,
 			  unsigned int num)
 {
-#ifndef RTE_ARCH_X86_64
+#if !defined(RTE_ARCH_X86_64) && !defined(RTE_ARCH_ARM64)
 	RTE_SET_USED(first);
 	RTE_SET_USED(last);
 	RTE_SET_USED(list);
@@ -88,7 +88,7 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 void **obj_table,
 			 struct rte_stack_lf_elem **last)
 {
-#ifndef RTE_ARCH_X86_64
+#if !defined(RTE_ARCH_X86_64) && !defined(RTE_ARCH_ARM64)
 	RTE_SET_USED(obj_table);
 	RTE_SET_USED(last);
 	RTE_SET_USED(list);
lib/librte_stack/rte_stack_lf_generic.h +2 -2
index 3182151..488fd9f 100644
@@ -36,7 +36,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 			  struct rte_stack_lf_elem *last,
 			  unsigned int num)
 {
-#ifndef RTE_ARCH_X86_64
+#if !defined(RTE_ARCH_X86_64) && !defined(RTE_ARCH_ARM64)
 	RTE_SET_USED(first);
 	RTE_SET_USED(last);
 	RTE_SET_USED(list);
@@ -84,7 +84,7 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 void **obj_table,
 			 struct rte_stack_lf_elem **last)
 {
-#ifndef RTE_ARCH_X86_64
+#if !defined(RTE_ARCH_X86_64) && !defined(RTE_ARCH_ARM64)
 	RTE_SET_USED(obj_table);
 	RTE_SET_USED(last);
 	RTE_SET_USED(list);
Signature · 1 line
2.7.4
author
#3 lore raw

PATCH v2 1/3 eal/arm64: add 128-bit atomic compare exchange

In reply to Phil Yang

Add 128-bit atomic compare exchange on aarch64.

---
This patch depends on 'eal/stack: fix 'pointer-sign' warning'
http://patchwork.dpdk.org/patch/54840/

v2:
Fixed coding style warning.

 .../common/include/arch/arm/rte_atomic_64.h        | 184 +++++++++++++++++++++
 .../common/include/arch/x86/rte_atomic_64.h        |  12 --
 lib/librte_eal/common/include/generic/rte_atomic.h |  15 +-
 3 files changed, 198 insertions(+), 13 deletions(-)
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +184 -0
index 97060e4..ae29ce6 100644
@@ -1,5 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright(c) 2015 Cavium, Inc
+ * Copyright(c) 2019 Arm Limited
  */
 
 #ifndef _RTE_ATOMIC_ARM64_H_
@@ -14,6 +15,9 @@ extern "C" {
 #endif
 
 #include "generic/rte_atomic.h"
+#include <rte_branch_prediction.h>
+#include <rte_compat.h>
+#include <rte_debug.h>
 
 #define dsb(opt) asm volatile("dsb " #opt : : : "memory")
 #define dmb(opt) asm volatile("dmb " #opt : : : "memory")
@@ -40,6 +44,186 @@ extern "C" {
 
 #define rte_cio_rmb() dmb(oshld)
 
+/*----------------------- 128 bit atomic operations -------------------------*/
+
+#define RTE_HAS_ACQ(mo) ((mo) != __ATOMIC_RELAXED && (mo) != __ATOMIC_RELEASE)
+#define RTE_HAS_RLS(mo) ((mo) == __ATOMIC_RELEASE || \
+			 (mo) == __ATOMIC_ACQ_REL || \
+			 (mo) == __ATOMIC_SEQ_CST)
+
+#define RTE_MO_LOAD(mo)  (RTE_HAS_ACQ((mo)) \
+		? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED)
+#define RTE_MO_STORE(mo) (RTE_HAS_RLS((mo)) \
+		? __ATOMIC_RELEASE : __ATOMIC_RELAXED)
+
+#ifdef __ARM_FEATURE_ATOMICS
+static inline rte_int128_t
+__rte_casp(rte_int128_t *dst, rte_int128_t old, rte_int128_t updated, int mo)
+{
+
+	/* caspX instructions register pair must start from even-numbered
+	 * register at operand 1.
+	 * So, specify registers for local variables here.
+	 */
+	register uint64_t x0 __asm("x0") = (uint64_t)old.val[0];
+	register uint64_t x1 __asm("x1") = (uint64_t)old.val[1];
+	register uint64_t x2 __asm("x2") = (uint64_t)updated.val[0];
+	register uint64_t x3 __asm("x3") = (uint64_t)updated.val[1];
+
+	if (mo ==  __ATOMIC_RELAXED) {
+		asm volatile(
+				"casp %[old0], %[old1], %[upd0], %[upd1], [%[dst]]"
+				: [old0] "+r" (x0),
+				  [old1] "+r" (x1)
+				: [upd0] "r" (x2),
+				  [upd1] "r" (x3),
+				  [dst] "r" (dst)
+				: "memory");
+	} else if (mo == __ATOMIC_ACQUIRE) {
+		asm volatile(
+				"caspa %[old0], %[old1], %[upd0], %[upd1], [%[dst]]"
+				: [old0] "+r" (x0),
+				  [old1] "+r" (x1)
+				: [upd0] "r" (x2),
+				  [upd1] "r" (x3),
+				  [dst] "r" (dst)
+				: "memory");
+	} else if (mo == __ATOMIC_ACQ_REL) {
+		asm volatile(
+				"caspal %[old0], %[old1], %[upd0], %[upd1], [%[dst]]"
+				: [old0] "+r" (x0),
+				  [old1] "+r" (x1)
+				: [upd0] "r" (x2),
+				  [upd1] "r" (x3),
+				  [dst] "r" (dst)
+				: "memory");
+	} else if (mo == __ATOMIC_RELEASE) {
+		asm volatile(
+				"caspl %[old0], %[old1], %[upd0], %[upd1], [%[dst]]"
+				: [old0] "+r" (x0),
+				  [old1] "+r" (x1)
+				: [upd0] "r" (x2),
+				  [upd1] "r" (x3),
+				  [dst] "r" (dst)
+				: "memory");
+	} else {
+		rte_panic("Invalid memory order\n");
+	}
+
+	old.val[0] = x0;
+	old.val[1] = x1;
+
+	return old;
+}
+#else
+static inline rte_int128_t
+__rte_ldx128(const rte_int128_t *src, int mo)
+{
+	rte_int128_t ret;
+	if (mo == __ATOMIC_ACQUIRE)
+		asm volatile(
+				"ldaxp %0, %1, %2"
+				: "=&r" (ret.val[0]),
+				  "=&r" (ret.val[1])
+				: "Q" (src->val[0])
+				: "memory");
+	else if (mo == __ATOMIC_RELAXED)
+		asm volatile(
+				"ldxp %0, %1, %2"
+				: "=&r" (ret.val[0]),
+				  "=&r" (ret.val[1])
+				: "Q" (src->val[0])
+				: "memory");
+	else
+		rte_panic("Invalid memory order\n");
+
+	return ret;
+}
+
+static inline uint32_t
+__rte_stx128(rte_int128_t *dst, const rte_int128_t src, int mo)
+{
+	uint32_t ret;
+	if (mo == __ATOMIC_RELEASE)
+		asm volatile(
+				"stlxp %w0, %1, %2, %3"
+				: "=&r" (ret)
+				: "r" (src.val[0]),
+				  "r" (src.val[1]),
+				  "Q" (dst->val[0])
+				: "memory");
+	else if (mo == __ATOMIC_RELAXED)
+		asm volatile(
+				"stxp %w0, %1, %2, %3"
+				: "=&r" (ret)
+				: "r" (src.val[0]),
+				  "r" (src.val[1]),
+				  "Q" (dst->val[0])
+				: "memory");
+	else
+		rte_panic("Invalid memory order\n");
+
+	/* Return 0 on success, 1 on failure */
+	return ret;
+}
+#endif
+
+static inline int __rte_experimental
+rte_atomic128_cmp_exchange(rte_int128_t *dst,
+				rte_int128_t *exp,
+				const rte_int128_t *src,
+				unsigned int weak,
+				int success,
+				int failure)
+{
+	// Always do strong CAS
+	RTE_SET_USED(weak);
+	/* Ignore memory ordering for failure, memory order for
+	 * success must be stronger or equal
+	 */
+	RTE_SET_USED(failure);
+
+#ifdef __ARM_FEATURE_ATOMICS
+	rte_int128_t expected = *exp;
+	rte_int128_t desired = *src;
+	rte_int128_t old;
+
+	old = __rte_casp(dst, expected, desired, success);
+#else
+	int ldx_mo = RTE_MO_LOAD(success);
+	int stx_mo = RTE_MO_STORE(success);
+	uint32_t ret = 1;
+	register rte_int128_t expected = *exp;
+	register rte_int128_t desired = *src;
+	register rte_int128_t old;
+
+	/* ldx128 can not guarantee atomic,
+	 * Must write back src or old to verify atomicity of ldx128;
+	 */
+	do {
+		old = __rte_ldx128(dst, ldx_mo);
+		if (likely(old.int128 == expected.int128))
+			ret = __rte_stx128(dst, desired, stx_mo);
+		else
+			/* In the failure case (since 'weak' is ignored and only
+			 * weak == 0 is implemented), expected should contain the
+			 * atomically read value of dst. This means, 'old' needs
+			 * to be stored back to ensure it was read atomically.
+			 */
+			ret = __rte_stx128(dst, old, stx_mo);
+	} while (unlikely(ret));
+#endif
+
+	/* Unconditionally updating expected removes
+	 * an 'if' statement.
+	 * expected should already be in register if
+	 * not in the cache.
+	 */
+	*exp = old;
+
+	return (old.int128 == expected.int128);
+}
+
 #ifdef __cplusplus
 }
 #endif
lib/librte_eal/common/include/arch/x86/rte_atomic_64.h +0 -12
index 6232c57..23cf48f 100644
@@ -212,18 +212,6 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)
 
 /*------------------------ 128 bit atomic operations -------------------------*/
 
-/**
- * 128-bit integer structure.
- */
-RTE_STD_C11
-typedef struct {
-	RTE_STD_C11
-	union {
-		uint64_t val[2];
-		__extension__ __int128 int128;
-	};
-} __rte_aligned(16) rte_int128_t;
-
 static inline int __rte_experimental
 rte_atomic128_cmp_exchange(rte_int128_t *dst,
 			   rte_int128_t *exp,
lib/librte_eal/common/include/generic/rte_atomic.h +14 -1
index 9958543..7dd1aa4 100644
@@ -1081,6 +1081,18 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)
 
 /*------------------------ 128 bit atomic operations -------------------------*/
 
+/**
+ * 128-bit integer structure.
+ */
+RTE_STD_C11
+typedef struct {
+	RTE_STD_C11
+	union {
+		uint64_t val[2];
+		__extension__ __int128 int128;
+	};
+} __rte_aligned(16) rte_int128_t;
+
 #ifdef __DOXYGEN__
 
 /**
@@ -1093,7 +1105,8 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)
  *     *exp = *dst
  * @endcode
  *
- * @note This function is currently only available for the x86-64 platform.
+ * @note This function is currently available for the x86-64 and aarch64
+ * platforms.
  *
  * @note The success and failure arguments must be one of the __ATOMIC_* values
  * defined in the C++11 standard. For details on their behavior, refer to the
Signature · 1 line
2.7.4
author
#4 lore raw

PATCH v2 2/3 test/atomic: add 128b compare and swap test

Add 128b atomic compare and swap test for aarch64 and x86_64.

---
 app/test/test_atomic.c | 120 ++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 119 insertions(+), 1 deletion(-)
app/test/test_atomic.c +119 -1
index 43be30e..da09bc4 100644
@@ -1,5 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright(c) 2010-2014 Intel Corporation
+ * Copyright(c) 2019 Arm Limited
  */
 
 #include <stdio.h>
@@ -20,7 +21,7 @@
  * Atomic Variables
  * ================
  *
- * - The main test function performs three subtests. The first test
+ * - The main test function performs four subtests. The first test
  *   checks that the usual inc/dec/add/sub functions are working
  *   correctly:
  *
@@ -61,6 +62,22 @@
  *       atomic_sub(&count, tmp+1);
  *
  *   - At the end of the test, the *count* value must be 0.
+ *
+ * - Test "128b compare and swap" (aarch64 and x86_64 only)
+ *
+ *   - Initialize 128-bit atomic variables to zero.
+ *
+ *   - Invoke ``test_atomici128_cmp_exchange()`` on each lcore. Before doing
+ *     anything else, the cores are waiting a synchro. Each lcore does
+ *     these compare and swap (CAS) operations several times::
+ *
+ *       Relaxed CAS update counter.val[0] + 2; counter.val[0] + 1;
+ *       Acquired CAS update counter.val[0] + 2; counter.val[0] + 1;
+ *       Released CAS update counter.val[0] + 2; counter.val[0] + 1;
+ *       Acquired_Released CAS update counter.val[0] + 2; counter.val[0] + 1;
+ *
+ *   - At the end of the test, the *count128* first 64-bit value and
+ *     second 64-bit value differ by the total iterations.
  */
 
 #define NUM_ATOMIC_TYPES 3
@@ -73,6 +90,10 @@ static rte_atomic64_t a64;
 static rte_atomic64_t count;
 static rte_atomic32_t synchro;
 
+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)
+static rte_int128_t count128;
+#endif
+
 static int
 test_atomic_usual(__attribute__((unused)) void *arg)
 {
@@ -216,6 +237,72 @@ test_atomic_dec_and_test(__attribute__((unused)) void *arg)
 	return 0;
 }
 
+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)
+/*
+ * rte_atomic128_cmp_exchange() should update a 128 bits counter's first 64
+ * bits by 2 and the second 64 bits by 1 in this test. It should return true
+ * if the compare exchange operation successful.
+ * This test repeat 128 bits compare and swap operations 10K rounds. In each
+ * iteration it runs compare and swap operation with different memory models.
+ */
+static int
+test_atomic128_cmp_exchange(__attribute__((unused)) void *arg)
+{
+	rte_int128_t expected;
+	int success;
+	unsigned int i;
+
+	while (rte_atomic32_read(&synchro) == 0)
+		;
+
+	expected = count128;
+
+	for (i = 0; i < N; i++) {
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128, &expected,
+					&desired, 1, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
+		} while (success == 0);
+
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128, &expected,
+					&desired, 1, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
+		} while (success == 0);
+
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128, &expected,
+					&desired, 1, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED);
+		} while (success == 0);
+
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128, &expected,
+					&desired, 1, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+		} while (success == 0);
+	}
+
+	return 0;
+}
+#endif
+
 static int
 test_atomic(void)
 {
@@ -340,6 +427,37 @@ test_atomic(void)
 		return -1;
 	}
 
+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)
+	/*
+	 * This case tests the functionality of rte_atomic128b_cmp_exchange
+	 * API. It calls rte_atomic128b_cmp_exchange with four kinds of memory
+	 * models successively on each slave core. Once each 128-bit atomic
+	 * compare and swap operation is successful, it updates the global
+	 * 128-bit counter by 2 for the first 64-bit and 1 for the second
+	 * 64-bit. Each slave core iterates this test 10K times.
+	 * At the end of test, verify whether the first 64-bits of the 128-bit
+	 * counter and the second 64bits is differ by the total iterations. If
+	 * it is, the test passes.
+	 */
+	printf("128b compare and swap test\n");
+	uint64_t iterations = 0;
+
+	rte_atomic32_clear(&synchro);
+	count128.val[0] = 0;
+	count128.val[1] = 0;
+
+	rte_eal_mp_remote_launch(test_atomic128_cmp_exchange, NULL, SKIP_MASTER);
+	rte_atomic32_set(&synchro, 1);
+	rte_eal_mp_wait_lcore();
+	rte_atomic32_clear(&synchro);
+
+	iterations = count128.val[0] - count128.val[1];
+	if (iterations != 4*N*(rte_lcore_count()-1)) {
+		printf("128b compare and swap failed\n");
+		return -1;
+	}
+#endif
+
 	return 0;
 }
 
Signature · 1 line
2.7.4
#5 lore raw

Re: [dpdk-dev] [PATCH v2 2/3] test/atomic: add 128b compare and swap test

Hi Phil,

Looks good overall, just a few documentation issues.

<snip>

Quoted 10 lines

+ *
+ * - Test "128b compare and swap" (aarch64 and x86_64 only)
+ *
+ * - Initialize 128-bit atomic variables to zero.
+ *
+ * - Invoke ``test_atomici128_cmp_exchange()`` on each lcore. Before
doing
+ * anything else, the cores are waiting a synchro. Each lcore does
+ * these compare and swap (CAS) operations several times::
+ *

Quoted 4 lines
+ *       Relaxed CAS update counter.val[0] + 2; counter.val[0] + 1;
+ *       Acquired CAS update counter.val[0] + 2; counter.val[0] + 1;
+ *       Released CAS update counter.val[0] + 2; counter.val[0] + 1;
+ *       Acquired_Released CAS update counter.val[0] + 2; counter.val[0] + 1;

The array index in "counter.val[0] + 1", is incorrect, I believe.

Just a nitpick, but "Relaxed CAS update" can go last to match the order in the code.

<snip>

Review

Quoted 7 lines
(quoted diff) +7 -0
+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)
+/*
+ * rte_atomic128_cmp_exchange() should update a 128 bits counter's
+first 64
+ * bits by 2 and the second 64 bits by 1 in this test. It should return
+true
+ * if the compare exchange operation successful.

"operation successful" -> "operation is successful"

Quoted 2 lines

+ * This test repeat 128 bits compare and swap operations 10K rounds. In +each

"repeat" -> "repeats"

Thanks,
Gage

#6 lore raw

Re: [dpdk-dev] [PATCH v2 2/3] test/atomic: add 128b compare and swap test

Quoted 9 lines

-----Original Message-----
From: Eads, Gage <[email protected]>
Sent: Monday, June 24, 2019 11:10 PM
To: Phil Yang (Arm Technology China) <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>
Subject: RE: [PATCH v2 2/3] test/atomic: add 128b compare and swap test

Quoted 21 lines

Hi Phil,

Looks good overall, just a few documentation issues.

<snip>

Quoted 14 lines

+ *
+ * - Test "128b compare and swap" (aarch64 and x86_64 only)
+ *
+ * - Initialize 128-bit atomic variables to zero.
+ *
+ * - Invoke ``test_atomici128_cmp_exchange()`` on each lcore. Before
doing
+ * anything else, the cores are waiting a synchro. Each lcore does
+ * these compare and swap (CAS) operations several times::
+ *

+ *       Relaxed CAS update counter.val[0] + 2; counter.val[0] + 1;
+ *       Acquired CAS update counter.val[0] + 2; counter.val[0] + 1;
+ *       Released CAS update counter.val[0] + 2; counter.val[0] + 1;
+ *       Acquired_Released CAS update counter.val[0] + 2; counter.val[0] + 1;

Hi Gage,

Quoted 1 line

The array index in "counter.val[0] + 1", is incorrect, I believe.

Yes, you are correct. I will fix it. It should be 'counter.val[2] + 1'.

Quoted 3 lines

Just a nitpick, but "Relaxed CAS update" can go last to match the order in the code.

Sure. Thank you for your correction.

Quoted 12 lines

<snip>

Review

Quoted 7 lines
(quoted diff) +7 -0
+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)
+/*
+ * rte_atomic128_cmp_exchange() should update a 128 bits counter's
+first 64
+ * bits by 2 and the second 64 bits by 1 in this test. It should
+return true
+ * if the compare exchange operation successful.

"operation successful" -> "operation is successful"

Yes.

Quoted 5 lines
Quoted 3 lines

+ * This test repeat 128 bits compare and swap operations 10K rounds. +In each

"repeat" -> "repeats"

Yes.

Quoted 3 lines

Thanks,
Gage

Thanks,
Phil

author
#7 lore raw

PATCH v2 3/3 eal/stack: enable lock-free stack for aarch64

In reply to Phil Yang

Enable both c11 atomic and non c11 atomic lock-free stack for aarch64.

---
 doc/guides/rel_notes/release_19_08.rst  | 3 +++
 lib/librte_stack/rte_stack_lf_c11.h     | 4 ++--
 lib/librte_stack/rte_stack_lf_generic.h | 4 ++--
 3 files changed, 7 insertions(+), 4 deletions(-)
doc/guides/rel_notes/release_19_08.rst +3 -0
index 8c3932d..b79ae28 100644
@@ -88,6 +88,9 @@ New Features
   * Added multi-queue support to allow one af_xdp vdev with multiple netdev
     queues
 
+* **Added Lock-free Stack for aarch64.**
+
+  The lock-free stack implementation is enabled for aarch64 platforms.
 
 Removed Items
 -------------
lib/librte_stack/rte_stack_lf_c11.h +2 -2
index 3d677ae..67c21fd 100644
@@ -36,7 +36,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 			  struct rte_stack_lf_elem *last,
 			  unsigned int num)
 {
-#ifndef RTE_ARCH_X86_64
+#if !defined(RTE_ARCH_X86_64) && !defined(RTE_ARCH_ARM64)
 	RTE_SET_USED(first);
 	RTE_SET_USED(last);
 	RTE_SET_USED(list);
@@ -88,7 +88,7 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 void **obj_table,
 			 struct rte_stack_lf_elem **last)
 {
-#ifndef RTE_ARCH_X86_64
+#if !defined(RTE_ARCH_X86_64) && !defined(RTE_ARCH_ARM64)
 	RTE_SET_USED(obj_table);
 	RTE_SET_USED(last);
 	RTE_SET_USED(list);
lib/librte_stack/rte_stack_lf_generic.h +2 -2
index 3182151..488fd9f 100644
@@ -36,7 +36,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 			  struct rte_stack_lf_elem *last,
 			  unsigned int num)
 {
-#ifndef RTE_ARCH_X86_64
+#if !defined(RTE_ARCH_X86_64) && !defined(RTE_ARCH_ARM64)
 	RTE_SET_USED(first);
 	RTE_SET_USED(last);
 	RTE_SET_USED(list);
@@ -84,7 +84,7 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 void **obj_table,
 			 struct rte_stack_lf_elem **last)
 {
-#ifndef RTE_ARCH_X86_64
+#if !defined(RTE_ARCH_X86_64) && !defined(RTE_ARCH_ARM64)
 	RTE_SET_USED(obj_table);
 	RTE_SET_USED(last);
 	RTE_SET_USED(list);
Signature · 1 line
2.7.4
#8 lore raw

Re: [dpdk-dev] [PATCH v2 3/3] eal/stack: enable lock-free stack for aarch64

Quoted 4 lines

Subject: [PATCH v2 3/3] eal/stack: enable lock-free stack for aarch64

Enable both c11 atomic and non c11 atomic lock-free stack for aarch64.

Quoted 9 lines
---
 doc/guides/rel_notes/release_19_08.rst  | 3 +++
 lib/librte_stack/rte_stack_lf_c11.h     | 4 ++--
 lib/librte_stack/rte_stack_lf_generic.h | 4 ++--
 3 files changed, 7 insertions(+), 4 deletions(-)

Please update doc/guides/prog_guide/env_abstraction_layer.rst as well -- it states that the lock-free stack is "currently limited to the x86_64 platform."

Thanks,
Gage

#9 lore raw

Re: [dpdk-dev] [PATCH v2 3/3] eal/stack: enable lock-free stack for aarch64

Quoted 23 lines

-----Original Message-----
From: Eads, Gage <[email protected]>
Sent: Monday, June 24, 2019 11:15 PM
To: Phil Yang (Arm Technology China) <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>
Subject: RE: [PATCH v2 3/3] eal/stack: enable lock-free stack for aarch64

Quoted 4 lines

Subject: [PATCH v2 3/3] eal/stack: enable lock-free stack for aarch64

Enable both c11 atomic and non c11 atomic lock-free stack for aarch64.

Quoted 9 lines
---
 doc/guides/rel_notes/release_19_08.rst  | 3 +++
 lib/librte_stack/rte_stack_lf_c11.h     | 4 ++--
 lib/librte_stack/rte_stack_lf_generic.h | 4 ++--
 3 files changed, 7 insertions(+), 4 deletions(-)
Quoted 2 lines

Please update doc/guides/prog_guide/env_abstraction_layer.rst as well -- it states that the lock-free stack is "currently limited to the x86_64 platform."

Thanks, Gage. I will update it in next version.

Quoted 2 lines

Thanks,
Gage

Thanks,
Phil

#10 lore raw
In reply to Phil Yang

Hi Phil,

Quoted 14 lines
lib/librte_eal/common/include/generic/rte_atomic.h +4 -0
index 9958543..7dd1aa4 100644
@@ -1081,6 +1081,18 @@ static inline void
 rte_atomic64_clear(rte_atomic64_t *v)
 
 /*------------------------ 128 bit atomic operations -------------------------*/
 
+/**
+ * 128-bit integer structure.
+ */
+RTE_STD_C11

lib/librte_eal/common/include/generic/rte_atomic.h static inline void · line 1098

Quoted 10 lines
lib/librte_eal/common/include/generic/rte_atomic.h +8 -0
+typedef struct {
+	RTE_STD_C11
+	union {
+		uint64_t val[2];
+		__extension__ __int128 int128;
+	};
+} __rte_aligned(16) rte_int128_t;
+
 #ifdef __DOXYGEN__
 

This change breaks 32-bit x86 builds*. A couple ways to resolve this are 1) with RTE_ARCH_* ifdefs, or 2) keep duplicate definitions of the struct in the aarch64 and x86 header files.

Thanks,
Gage

*http://mails.dpdk.org/archives/test-report/2019-June/086586.html

#11 lore raw
Quoted 22 lines

-----Original Message-----
From: Eads, Gage <[email protected]>
Sent: Monday, June 24, 2019 10:46 PM
To: Phil Yang (Arm Technology China) <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>
Subject: RE: [PATCH v2 1/3] eal/arm64: add 128-bit atomic compare exchange

Hi Phil,

Quoted 11 lines
lib/librte_eal/common/include/generic/rte_atomic.h +0 -1
index 9958543..7dd1aa4 100644
@@ -1081,6 +1081,18 @@ static inline void
 rte_atomic64_clear(rte_atomic64_t *v)
 
 /*------------------------ 128 bit atomic operations
-------------------------*/
 
Quoted 15 lines
Quoted 4 lines
lib/librte_eal/common/include/generic/rte_atomic.h +3 -0
+/**
+ * 128-bit integer structure.
+ */

lib/librte_eal/common/include/generic/rte_atomic.h static inline void rte_atomic64_clear(rte_atomic64_t *v) · line 1096

Quoted 10 lines
lib/librte_eal/common/include/generic/rte_atomic.h +9 -0
+RTE_STD_C11
+typedef struct {
+	RTE_STD_C11
+	union {
+		uint64_t val[2];
+		__extension__ __int128 int128;
+	};
+} __rte_aligned(16) rte_int128_t;
+
 #ifdef __DOXYGEN__

Hi Gage,

Quoted 3 lines

This change breaks 32-bit x86 builds*. A couple ways to resolve this are 1) with RTE_ARCH_* ifdefs, or 2) keep duplicate definitions of the struct in the aarch64 and x86 header files.

OK. Let's follow the first approach. I will update it in the new version. Thanks!

Quoted 5 lines

Thanks,
Gage

*http://mails.dpdk.org/archives/test-report/2019-June/086586.html

Thanks,
Phil

author
#12 lore raw

PATCH v3 1/3 eal/arm64: add 128-bit atomic compare exchange

In reply to Phil Yang

Add 128-bit atomic compare exchange on aarch64.

---
v3:
1. Avoid duplication code with macro. (Jerin Jocob)
2. Make invalid memory order to strongest barrier. (Jerin Jocob)
3. Update doc/guides/prog_guide/env_abstraction_layer.rst. (Eads Gage)
4. Fix 32-bit x86 builds issue. (Eads Gage)
5. Correct documentation issues in UT. (Eads Gage)

 .../common/include/arch/arm/rte_atomic_64.h        | 165 +++++++++++++++++++++
 .../common/include/arch/x86/rte_atomic_64.h        |  12 --
 lib/librte_eal/common/include/generic/rte_atomic.h |  17 ++-
 3 files changed, 181 insertions(+), 13 deletions(-)
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +165 -0
index 97060e4..2080c4d 100644
@@ -1,5 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright(c) 2015 Cavium, Inc
+ * Copyright(c) 2019 Arm Limited
  */
 
 #ifndef _RTE_ATOMIC_ARM64_H_
@@ -14,6 +15,9 @@ extern "C" {
 #endif
 
 #include "generic/rte_atomic.h"
+#include <rte_branch_prediction.h>
+#include <rte_compat.h>
+#include <rte_debug.h>
 
 #define dsb(opt) asm volatile("dsb " #opt : : : "memory")
 #define dmb(opt) asm volatile("dmb " #opt : : : "memory")
@@ -40,6 +44,167 @@ extern "C" {
 
 #define rte_cio_rmb() dmb(oshld)
 
+/*------------------------ 128 bit atomic operations -------------------------*/
+
+#define RTE_HAS_ACQ(mo) ((mo) != __ATOMIC_RELAXED && (mo) != __ATOMIC_RELEASE)
+#define RTE_HAS_RLS(mo) ((mo) == __ATOMIC_RELEASE || \
+			 (mo) == __ATOMIC_ACQ_REL || \
+			 (mo) == __ATOMIC_SEQ_CST)
+
+#define RTE_MO_LOAD(mo)  (RTE_HAS_ACQ((mo)) \
+		? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED)
+#define RTE_MO_STORE(mo) (RTE_HAS_RLS((mo)) \
+		? __ATOMIC_RELEASE : __ATOMIC_RELAXED)
+
+#ifdef __ARM_FEATURE_ATOMICS
+#define __ATOMIC128_CAS_OP(cas_op_name, op_string)                          \
+static inline rte_int128_t                                                  \
+cas_op_name(rte_int128_t *dst, rte_int128_t old,                            \
+		rte_int128_t updated)                                               \
+{                                                                           \
+	/* caspX instructions register pair must start from even-numbered
+	 * register at operand 1.
+	 * So, specify registers for local variables here.
+	 */                                                                     \
+	register uint64_t x0 __asm("x0") = (uint64_t)old.val[0];                \
+	register uint64_t x1 __asm("x1") = (uint64_t)old.val[1];                \
+	register uint64_t x2 __asm("x2") = (uint64_t)updated.val[0];            \
+	register uint64_t x3 __asm("x3") = (uint64_t)updated.val[1];            \
+	asm volatile(                                                           \
+			op_string " %[old0], %[old1], %[upd0], %[upd1], [%[dst]]"       \
+			: [old0] "+r" (x0),                                             \
+			  [old1] "+r" (x1)                                              \
+			: [upd0] "r" (x2),                                              \
+			  [upd1] "r" (x3),                                              \
+			  [dst] "r" (dst)                                               \
+			: "memory");                                                    \
+	old.val[0] = x0;                                                        \
+	old.val[1] = x1;                                                        \
+	return old;                                                             \
+}
+
+__ATOMIC128_CAS_OP(__rte_cas_relaxed, "casp")
+__ATOMIC128_CAS_OP(__rte_cas_acquire, "caspa")
+__ATOMIC128_CAS_OP(__rte_cas_release, "caspl")
+__ATOMIC128_CAS_OP(__rte_cas_acq_rel, "caspal")
+#else
+#define __ATOMIC128_LDX_OP(ldx_op_name, op_string)                          \
+static inline rte_int128_t                                                  \
+ldx_op_name(const rte_int128_t *src)                                        \
+{                                                                           \
+	rte_int128_t ret;                                                       \
+	asm volatile(                                                           \
+			op_string " %0, %1, %2"                                         \
+			: "=&r" (ret.val[0]),                                           \
+			  "=&r" (ret.val[1])                                            \
+			: "Q" (src->val[0])                                             \
+			: "memory");                                                    \
+	return ret;                                                             \
+}
+
+__ATOMIC128_LDX_OP(__rte_ldx_relaxed, "ldxp")
+__ATOMIC128_LDX_OP(__rte_ldx_acquire, "ldaxp")
+
+#define __ATOMIC128_STX_OP(stx_op_name, op_string)                          \
+static inline uint32_t                                                      \
+stx_op_name(rte_int128_t *dst, const rte_int128_t src)                      \
+{                                                                           \
+	uint32_t ret;                                                           \
+	asm volatile(                                                           \
+			op_string " %w0, %1, %2, %3"                                    \
+			: "=&r" (ret)                                                   \
+			: "r" (src.val[0]),                                             \
+			  "r" (src.val[1]),                                             \
+			  "Q" (dst->val[0])                                             \
+			: "memory");                                                    \
+	/* Return 0 on success, 1 on failure */                                 \
+	return ret;                                                             \
+}
+
+__ATOMIC128_STX_OP(__rte_stx_relaxed, "stxp")
+__ATOMIC128_STX_OP(__rte_stx_release, "stlxp")
+#endif
+
+static inline int __rte_experimental
+rte_atomic128_cmp_exchange(rte_int128_t *dst,
+				rte_int128_t *exp,
+				const rte_int128_t *src,
+				unsigned int weak,
+				int success,
+				int failure)
+{
+	/* Always do strong CAS */
+	RTE_SET_USED(weak);
+	/* Ignore memory ordering for failure, memory order for
+	 * success must be stronger or equal
+	 */
+	RTE_SET_USED(failure);
+	/* Find invalid memory order */
+	RTE_ASSERT(success == __ATOMIC_RELAXED
+			|| success == __ATOMIC_ACQUIRE
+			|| success == __ATOMIC_RELEASE
+			|| success == __ATOMIC_ACQ_REL
+			|| success == __ATOMIC_SEQ_CST);
+
+#ifdef __ARM_FEATURE_ATOMICS
+	rte_int128_t expected = *exp;
+	rte_int128_t desired = *src;
+	rte_int128_t old;
+
+	if (success == __ATOMIC_RELAXED)
+		old = __rte_cas_relaxed(dst, expected, desired);
+	else if (success == __ATOMIC_ACQUIRE)
+		old = __rte_cas_acquire(dst, expected, desired);
+	else if (success == __ATOMIC_RELEASE)
+		old = __rte_cas_release(dst, expected, desired);
+	else
+		old = __rte_cas_acq_rel(dst, expected, desired);
+#else
+	int ldx_mo = RTE_MO_LOAD(success);
+	int stx_mo = RTE_MO_STORE(success);
+	uint32_t ret = 1;
+	register rte_int128_t expected = *exp;
+	register rte_int128_t desired = *src;
+	register rte_int128_t old;
+
+	/* ldx128 can not guarantee atomic,
+	 * Must write back src or old to verify atomicity of ldx128;
+	 */
+	do {
+		if (ldx_mo == __ATOMIC_RELAXED)
+			old = __rte_ldx_relaxed(dst);
+		else
+			old = __rte_ldx_acquire(dst);
+
+		if (likely(old.int128 == expected.int128)) {
+			if (stx_mo == __ATOMIC_RELAXED)
+				ret = __rte_stx_relaxed(dst, desired);
+			else
+				ret = __rte_stx_release(dst, desired);
+		} else {
+			/* In the failure case (since 'weak' is ignored and only
+			 * weak == 0 is implemented), expected should contain the
+			 * atomically read value of dst. This means, 'old' needs
+			 * to be stored back to ensure it was read atomically.
+			 */
+			if (stx_mo == __ATOMIC_RELAXED)
+				ret = __rte_stx_relaxed(dst, old);
+			else
+				ret = __rte_stx_release(dst, old);
+		}
+	} while (unlikely(ret));
+#endif
+
+	/* Unconditionally updating expected removes
+	 * an 'if' statement.
+	 * expected should already be in register if
+	 * not in the cache.
+	 */
+	*exp = old;
+
+	return (old.int128 == expected.int128);
+}
+
 #ifdef __cplusplus
 }
 #endif
lib/librte_eal/common/include/arch/x86/rte_atomic_64.h +0 -12
index 6232c57..23cf48f 100644
@@ -212,18 +212,6 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)
 
 /*------------------------ 128 bit atomic operations -------------------------*/
 
-/**
- * 128-bit integer structure.
- */
-RTE_STD_C11
-typedef struct {
-	RTE_STD_C11
-	union {
-		uint64_t val[2];
-		__extension__ __int128 int128;
-	};
-} __rte_aligned(16) rte_int128_t;
-
 static inline int __rte_experimental
 rte_atomic128_cmp_exchange(rte_int128_t *dst,
 			   rte_int128_t *exp,
lib/librte_eal/common/include/generic/rte_atomic.h +16 -1
index 9958543..2355e50 100644
@@ -1081,6 +1081,20 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)
 
 /*------------------------ 128 bit atomic operations -------------------------*/
 
+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)
+/**
+ * 128-bit integer structure.
+ */
+RTE_STD_C11
+typedef struct {
+	RTE_STD_C11
+	union {
+		uint64_t val[2];
+		__extension__ __int128 int128;
+	};
+} __rte_aligned(16) rte_int128_t;
+#endif
+
 #ifdef __DOXYGEN__
 
 /**
@@ -1093,7 +1107,8 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)
  *     *exp = *dst
  * @endcode
  *
- * @note This function is currently only available for the x86-64 platform.
+ * @note This function is currently available for the x86-64 and aarch64
+ * platforms.
  *
  * @note The success and failure arguments must be one of the __ATOMIC_* values
  * defined in the C++11 standard. For details on their behavior, refer to the
Signature · 1 line
2.7.4
author
#13 lore raw

PATCH v3 2/3 test/atomic: add 128b compare and swap test

Add 128b atomic compare and swap test for aarch64 and x86_64.

---
 app/test/test_atomic.c | 122 ++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 120 insertions(+), 2 deletions(-)
app/test/test_atomic.c +120 -2
index 43be30e..78541e0 100644
@@ -1,5 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright(c) 2010-2014 Intel Corporation
+ * Copyright(c) 2019 Arm Limited
  */
 
 #include <stdio.h>
@@ -20,7 +21,7 @@
  * Atomic Variables
  * ================
  *
- * - The main test function performs three subtests. The first test
+ * - The main test function performs four subtests. The first test
  *   checks that the usual inc/dec/add/sub functions are working
  *   correctly:
  *
@@ -61,11 +62,27 @@
  *       atomic_sub(&count, tmp+1);
  *
  *   - At the end of the test, the *count* value must be 0.
+ *
+ * - Test "128b compare and swap" (aarch64 and x86_64 only)
+ *
+ *   - Initialize 128-bit atomic variables to zero.
+ *
+ *   - Invoke ``test_atomici128_cmp_exchange()`` on each lcore. Before doing
+ *     anything else, the cores are waiting a synchro. Each lcore does
+ *     these compare and swap (CAS) operations several times::
+ *
+ *       Acquired CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *       Released CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *       Acquired_Released CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *       Relaxed CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *
+ *   - At the end of the test, the *count128* first 64-bit value and
+ *     second 64-bit value differ by the total iterations.
  */
 
 #define NUM_ATOMIC_TYPES 3
 
-#define N 10000
+#define N 1000000
 
 static rte_atomic16_t a16;
 static rte_atomic32_t a32;
@@ -73,6 +90,10 @@ static rte_atomic64_t a64;
 static rte_atomic64_t count;
 static rte_atomic32_t synchro;
 
+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)
+static rte_int128_t count128;
+#endif
+
 static int
 test_atomic_usual(__attribute__((unused)) void *arg)
 {
@@ -216,6 +237,72 @@ test_atomic_dec_and_test(__attribute__((unused)) void *arg)
 	return 0;
 }
 
+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)
+/*
+ * rte_atomic128_cmp_exchange() should update a 128 bits counter's first 64
+ * bits by 2 and the second 64 bits by 1 in this test. It should return true
+ * if the compare exchange operation is successful.
+ * This test repeats 128 bits compare and swap operations 10K rounds. In each
+ * iteration it runs compare and swap operation with different memory models.
+ */
+static int
+test_atomic128_cmp_exchange(__attribute__((unused)) void *arg)
+{
+	rte_int128_t expected;
+	int success;
+	unsigned int i;
+
+	while (rte_atomic32_read(&synchro) == 0)
+		;
+
+	expected = count128;
+
+	for (i = 0; i < N; i++) {
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128, &expected,
+					&desired, 1, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
+		} while (success == 0);
+
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128, &expected,
+					&desired, 1, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
+		} while (success == 0);
+
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128, &expected,
+					&desired, 1, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED);
+		} while (success == 0);
+
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128, &expected,
+					&desired, 1, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+		} while (success == 0);
+	}
+
+	return 0;
+}
+#endif
+
 static int
 test_atomic(void)
 {
@@ -340,6 +427,37 @@ test_atomic(void)
 		return -1;
 	}
 
+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)
+	/*
+	 * This case tests the functionality of rte_atomic128b_cmp_exchange
+	 * API. It calls rte_atomic128b_cmp_exchange with four kinds of memory
+	 * models successively on each slave core. Once each 128-bit atomic
+	 * compare and swap operation is successful, it updates the global
+	 * 128-bit counter by 2 for the first 64-bit and 1 for the second
+	 * 64-bit. Each slave core iterates this test 10K times.
+	 * At the end of test, verify whether the first 64-bits of the 128-bit
+	 * counter and the second 64bits is differ by the total iterations. If
+	 * it is, the test passes.
+	 */
+	printf("128b compare and swap test\n");
+	uint64_t iterations = 0;
+
+	rte_atomic32_clear(&synchro);
+	count128.val[0] = 0;
+	count128.val[1] = 0;
+
+	rte_eal_mp_remote_launch(test_atomic128_cmp_exchange, NULL, SKIP_MASTER);
+	rte_atomic32_set(&synchro, 1);
+	rte_eal_mp_wait_lcore();
+	rte_atomic32_clear(&synchro);
+
+	iterations = count128.val[0] - count128.val[1];
+	if (iterations != 4*N*(rte_lcore_count()-1)) {
+		printf("128b compare and swap failed\n");
+		return -1;
+	}
+#endif
+
 	return 0;
 }
 
Signature · 1 line
2.7.4
Acked-by Tested-by on v3 2/3
#15 lore raw

Re: [dpdk-dev] [EXT] [PATCH v3 2/3] test/atomic: add 128b compare and swap test

In reply to Phil Yang
Quoted 16 lines

-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Friday, June 28, 2019 1:42 PM
To: [email protected]
Cc: [email protected]; Jerin Jacob Kollanukkaran <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: [EXT] [PATCH v3 2/3] test/atomic: add 128b compare and swap test

Add 128b atomic compare and swap test for aarch64 and x86_64.

---
 app/test/test_atomic.c | 122

Review

Quoted 2 lines
(quoted diff) +2 -0
+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64) static
+rte_int128_t count128;

Moving this to down[1] can reduce one #ifdef cutler.

app/test/test_atomic.c test_atomic_dec_and_test(__attribute__((unused)) void *arg) · line 240

Quoted 8 lines

+
static int
test_atomic_usual(__attribute__((unused)) void *arg) { @@ -216,6 +237,72
@@ test_atomic_dec_and_test(__attribute__((unused)) void *arg)

app/test/test_atomic.c +1 -0
 	return 0;
 }
 
+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)

[1] See above,

With above change,

author
#16 lore raw

PATCH v3 3/3 eal/stack: enable lock-free stack for aarch64

In reply to Phil Yang

Enable both c11 atomic and non c11 atomic lock-free stack for aarch64.

---
 doc/guides/prog_guide/env_abstraction_layer.rst | 4 ++--
 doc/guides/rel_notes/release_19_08.rst          | 3 +++
 lib/librte_stack/rte_stack_lf_c11.h             | 4 ++--
 lib/librte_stack/rte_stack_lf_generic.h         | 4 ++--
 4 files changed, 9 insertions(+), 6 deletions(-)
doc/guides/prog_guide/env_abstraction_layer.rst +2 -2
index f15bcd9..d569f95 100644
@@ -592,8 +592,8 @@ Known Issues
   Alternatively, applications can use the lock-free stack mempool handler. When
   considering this handler, note that:
 
-  - It is currently limited to the x86_64 platform, because it uses an
-    instruction (16-byte compare-and-swap) that is not yet available on other
+  - It is currently limited to the aarch64 and x86_64 platforms, because it uses
+    an instruction (16-byte compare-and-swap) that is not yet available on other
     platforms.
   - It has worse average-case performance than the non-preemptive rte_ring, but
     software caching (e.g. the mempool cache) can mitigate this by reducing the
doc/guides/rel_notes/release_19_08.rst +3 -0
index 3da2667..e2e00b9 100644
@@ -99,6 +99,9 @@ New Features
   Updated ``librte_telemetry`` to fetch the global metrics from the
   ``librte_metrics`` library.
 
+* **Added Lock-free Stack for aarch64.**
+
+  The lock-free stack implementation is enabled for aarch64 platforms.
 
 Removed Items
 -------------
lib/librte_stack/rte_stack_lf_c11.h +2 -2
index 3d677ae..67c21fd 100644
@@ -36,7 +36,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 			  struct rte_stack_lf_elem *last,
 			  unsigned int num)
 {
-#ifndef RTE_ARCH_X86_64
+#if !defined(RTE_ARCH_X86_64) && !defined(RTE_ARCH_ARM64)
 	RTE_SET_USED(first);
 	RTE_SET_USED(last);
 	RTE_SET_USED(list);
@@ -88,7 +88,7 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 void **obj_table,
 			 struct rte_stack_lf_elem **last)
 {
-#ifndef RTE_ARCH_X86_64
+#if !defined(RTE_ARCH_X86_64) && !defined(RTE_ARCH_ARM64)
 	RTE_SET_USED(obj_table);
 	RTE_SET_USED(last);
 	RTE_SET_USED(list);
lib/librte_stack/rte_stack_lf_generic.h +2 -2
index 3182151..488fd9f 100644
@@ -36,7 +36,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 			  struct rte_stack_lf_elem *last,
 			  unsigned int num)
 {
-#ifndef RTE_ARCH_X86_64
+#if !defined(RTE_ARCH_X86_64) && !defined(RTE_ARCH_ARM64)
 	RTE_SET_USED(first);
 	RTE_SET_USED(last);
 	RTE_SET_USED(list);
@@ -84,7 +84,7 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 void **obj_table,
 			 struct rte_stack_lf_elem **last)
 {
-#ifndef RTE_ARCH_X86_64
+#if !defined(RTE_ARCH_X86_64) && !defined(RTE_ARCH_ARM64)
 	RTE_SET_USED(obj_table);
 	RTE_SET_USED(last);
 	RTE_SET_USED(list);
Signature · 1 line
2.7.4
#18 lore raw

Re: [dpdk-dev] [EXT] [PATCH v3 3/3] eal/stack: enable lock-free stack for aarch64

In reply to Phil Yang
Quoted 56 lines

-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Friday, June 28, 2019 1:42 PM
To: [email protected]
Cc: [email protected]; Jerin Jacob Kollanukkaran <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: [EXT] [PATCH v3 3/3] eal/stack: enable lock-free stack for aarch64 Enable both c11 atomic and non c11 atomic lock-free stack for aarch64.

Removed Items
-------------

lib/librte_stack/rte_stack_lf_c11.h +1 -1
index 3d677ae..67c21fd 100644
@@ -36,7 +36,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list
 *list,
 			  struct rte_stack_lf_elem *last,
 			  unsigned int num)
 {
-#ifndef RTE_ARCH_X86_64
+#if !defined(RTE_ARCH_X86_64) && !defined(RTE_ARCH_ARM64)
 	RTE_SET_USED(first);
 	RTE_SET_USED(last);
lib/librte_stack/rte_stack_lf_c11.h +1 -1
 	RTE_SET_USED(list);
@@ -88,7 +88,7 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 void **obj_table,
 			 struct rte_stack_lf_elem **last)
 {
-#ifndef RTE_ARCH_X86_64
+#if !defined(RTE_ARCH_X86_64) && !defined(RTE_ARCH_ARM64)
 	RTE_SET_USED(obj_table);
 	RTE_SET_USED(last);
 	RTE_SET_USED(list);
lib/librte_stack/rte_stack_lf_generic.h +1 -1
index 3182151..488fd9f 100644
@@ -36,7 +36,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list
 *list,
 			  struct rte_stack_lf_elem *last,
 			  unsigned int num)
 {
-#ifndef RTE_ARCH_X86_64
+#if !defined(RTE_ARCH_X86_64) && !defined(RTE_ARCH_ARM64)
 	RTE_SET_USED(first);
 	RTE_SET_USED(last);
lib/librte_stack/rte_stack_lf_generic.h +0 -0
 	RTE_SET_USED(list);

lib/librte_stack/rte_stack_lf_generic.h __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list, · line 90

Quoted 8 lines
lib/librte_stack/rte_stack_lf_generic.h +1 -1
@@ -84,7 +84,7 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 void **obj_table,
 			 struct rte_stack_lf_elem **last)
 {
-#ifndef RTE_ARCH_X86_64
+#if !defined(RTE_ARCH_X86_64) && !defined(RTE_ARCH_ARM64)
 	RTE_SET_USED(obj_table);
 	RTE_SET_USED(last);
 	RTE_SET_USED(list);

Can we remove this all #ifdef clutter by adding the following

$ git diff
lib/librte_stack/rte_stack_lf.h +1 -1
index f5581f0c2..46af08b83 100644
@@ -5,7 +5,7 @@
 #ifndef _RTE_STACK_LF_H_
 #define _RTE_STACK_LF_H_
 
-#ifdef RTE_USE_C11_MEM_MODEL
+#if defined (RTE_USE_C11_MEM_MODEL) && defined(RTE_ARCH_X86_64) && defined(RTE_ARCH_ARM64)
 #include "rte_stack_lf_c11.h"
 #else
 #include "rte_stack_lf_generic.h"
#19 lore raw

Re: [dpdk-dev] [EXT] [PATCH v3 3/3] eal/stack: enable lock-free stack for aarch64

Quoted 95 lines

-----Original Message-----
From: Jerin Jacob Kollanukkaran [mailto:[email protected]]
Sent: Thursday, July 18, 2019 11:18 PM
To: Phil Yang <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; Eads, Gage <[email protected]>
Subject: RE: [EXT] [PATCH v3 3/3] eal/stack: enable lock-free stack for aarch64

Quoted 10 lines

-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Friday, June 28, 2019 1:42 PM
To: [email protected]
Cc: [email protected]; Jerin Jacob Kollanukkaran <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: [EXT] [PATCH v3 3/3] eal/stack: enable lock-free stack for aarch64 Enable both c11 atomic and non c11 atomic lock-free stack for

aarch64.

lib/librte_stack/rte_stack_lf_c11.h __rte_stack_lf_pop_elems(struct rte_stack_lf_list

Quoted 23 lines

Removed Items
-------------

lib/librte_stack/rte_stack_lf_c11.h +1 -1
index 3d677ae..67c21fd 100644
@@ -36,7 +36,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list
 *list,
 			  struct rte_stack_lf_elem *last,
 			  unsigned int num)
 {
-#ifndef RTE_ARCH_X86_64
+#if !defined(RTE_ARCH_X86_64) && !defined(RTE_ARCH_ARM64)
 	RTE_SET_USED(first);
 	RTE_SET_USED(last);
lib/librte_stack/rte_stack_lf_c11.h +0 -0
 	RTE_SET_USED(list);
@@ -88,7 +88,7 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list

*list,

lib/librte_stack/rte_stack_lf_generic.h __rte_stack_lf_pop_elems(struct rte_stack_lf_list

Quoted 24 lines
lib/librte_stack/rte_stack_lf_c11.h +1 -1
 			 void **obj_table,
 			 struct rte_stack_lf_elem **last)
 {
-#ifndef RTE_ARCH_X86_64
+#if !defined(RTE_ARCH_X86_64) && !defined(RTE_ARCH_ARM64)
 	RTE_SET_USED(obj_table);
 	RTE_SET_USED(last);
 	RTE_SET_USED(list);
lib/librte_stack/rte_stack_lf_generic.h +1 -1
index 3182151..488fd9f 100644
@@ -36,7 +36,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list
 *list,
 			  struct rte_stack_lf_elem *last,
 			  unsigned int num)
 {
-#ifndef RTE_ARCH_X86_64
+#if !defined(RTE_ARCH_X86_64) && !defined(RTE_ARCH_ARM64)
 	RTE_SET_USED(first);
 	RTE_SET_USED(last);
lib/librte_stack/rte_stack_lf_generic.h +0 -0
 	RTE_SET_USED(list);
@@ -84,7 +84,7 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list

*list,

lib/librte_stack/rte_stack_lf_generic.h __rte_stack_lf_pop_elems(struct rte_stack_lf_list · line 90

Quoted 8 lines
lib/librte_stack/rte_stack_lf_generic.h +1 -1
 			 void **obj_table,
 			 struct rte_stack_lf_elem **last)
 {
-#ifndef RTE_ARCH_X86_64
+#if !defined(RTE_ARCH_X86_64) && !defined(RTE_ARCH_ARM64)
 	RTE_SET_USED(obj_table);
 	RTE_SET_USED(last);
 	RTE_SET_USED(list);

Can we remove this all #ifdef clutter by adding the following

$ git diff
lib/librte_stack/rte_stack_lf.h +1 -1
index f5581f0c2..46af08b83 100644
@@ -5,7 +5,7 @@
 #ifndef _RTE_STACK_LF_H_
 #define _RTE_STACK_LF_H_
 
-#ifdef RTE_USE_C11_MEM_MODEL
+#if defined (RTE_USE_C11_MEM_MODEL) && defined(RTE_ARCH_X86_64)
Quoted 2 lines

&&
+defined(RTE_ARCH_ARM64)

I assume you meant (defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64))?

Quoted 5 lines

#include "rte_stack_lf_c11.h"
#else
#include "rte_stack_lf_generic.h"

The ifdefs in those two headers prevent DPDK from trying to build rte_atomic128_cmp_exchange() on architectures that don't implement it. So the proposal wouldn't quite work, since rte_stack_lf_generic.h calls rte_atomic128_cmp_exchange().

Something like this could work:

#if !(defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)) #include rte_stack_lf_stubs.h
#else
#ifdef RTE_USE_C11_MEM_MODEL
#include "rte_stack_lf_c11.h"
#else
#include "rte_stack_lf_generic.h"
#endif
#endif

Where rte_stack_lf_stubs.h is a new header containing stub implementations of __rte_stack_lf_count, __rte_stack_lf_push_elems, and __rte_stack_lf_pop_elems. It still has some ifdef clutter, but less overall.

#20 lore raw

Re: [dpdk-dev] [EXT] [PATCH v3 3/3] eal/stack: enable lock-free stack for aarch64

Quoted 11 lines
Quoted 11 lines

Can we remove this all #ifdef clutter by adding the following

$ git diff
(quoted diff) +0 -0

b/lib/librte_stack/rte_stack_lf.h index f5581f0c2..46af08b83 100644

lib/librte_stack/rte_stack_lf.h +0 -0
@@ -5,7 +5,7 @@
 #ifndef _RTE_STACK_LF_H_
 #define _RTE_STACK_LF_H_
 
Quoted 8 lines

lib/librte_stack/rte_stack_lf.h line 8

Quoted 2 lines
lib/librte_stack/rte_stack_lf.h +1 -1
-#ifdef RTE_USE_C11_MEM_MODEL
+#if defined (RTE_USE_C11_MEM_MODEL) &&

defined(RTE_ARCH_X86_64)

Quoted 2 lines

&&
+defined(RTE_ARCH_ARM64)

I assume you meant (defined(RTE_ARCH_X86_64) ||
defined(RTE_ARCH_ARM64))?

Yup.

Quoted 22 lines
Quoted 6 lines

#include "rte_stack_lf_c11.h"
#else
#include "rte_stack_lf_generic.h"

The ifdefs in those two headers prevent DPDK from trying to build rte_atomic128_cmp_exchange() on architectures that don't implement it. So the proposal wouldn't quite work, since rte_stack_lf_generic.h calls rte_atomic128_cmp_exchange().

Something like this could work:

#if !(defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)) #include rte_stack_lf_stubs.h #else #ifdef RTE_USE_C11_MEM_MODEL #include "rte_stack_lf_c11.h"
#else
#include "rte_stack_lf_generic.h"
#endif
#endif

Quoted 3 lines

Where rte_stack_lf_stubs.h is a new header containing stub implementations of __rte_stack_lf_count, __rte_stack_lf_push_elems, and __rte_stack_lf_pop_elems. It still has some ifdef clutter, but less overall.

Agree. I prefer to take this route to reduce the ifdef clutter across generic and c11 files.

#21 lore raw

Re: [dpdk-dev] [EXT] [PATCH v3 3/3] eal/stack: enable lock-free stack for aarch64

Quoted 56 lines

-----Original Message-----
From: Jerin Jacob Kollanukkaran <[email protected]>
Sent: Friday, July 19, 2019 1:03 PM
To: Eads, Gage <[email protected]>; Phil Yang (Arm Technology China) <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>
Subject: RE: [EXT] [PATCH v3 3/3] eal/stack: enable lock-free stack for aarch64

Quoted 11 lines
Quoted 11 lines

Can we remove this all #ifdef clutter by adding the following

$ git diff
(quoted diff) +0 -0

b/lib/librte_stack/rte_stack_lf.h index f5581f0c2..46af08b83 100644

lib/librte_stack/rte_stack_lf.h +0 -0
@@ -5,7 +5,7 @@
 #ifndef _RTE_STACK_LF_H_
 #define _RTE_STACK_LF_H_
 
Quoted 8 lines

lib/librte_stack/rte_stack_lf.h line 8

Quoted 2 lines
lib/librte_stack/rte_stack_lf.h +1 -1
-#ifdef RTE_USE_C11_MEM_MODEL
+#if defined (RTE_USE_C11_MEM_MODEL) &&

defined(RTE_ARCH_X86_64)

Quoted 2 lines

&&
+defined(RTE_ARCH_ARM64)

I assume you meant (defined(RTE_ARCH_X86_64) ||
defined(RTE_ARCH_ARM64))?

Yup.

Quoted 9 lines
Quoted 6 lines

#include "rte_stack_lf_c11.h"
#else
#include "rte_stack_lf_generic.h"

The ifdefs in those two headers prevent DPDK from trying to build rte_atomic128_cmp_exchange() on architectures that don't implement it.

So

Quoted 13 lines

the proposal wouldn't quite work, since rte_stack_lf_generic.h calls rte_atomic128_cmp_exchange().

Something like this could work:

#if !(defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)) #include rte_stack_lf_stubs.h #else #ifdef RTE_USE_C11_MEM_MODEL #include "rte_stack_lf_c11.h"
#else
#include "rte_stack_lf_generic.h"
#endif
#endif

Quoted 7 lines
Quoted 2 lines

Where rte_stack_lf_stubs.h is a new header containing stub implementations of __rte_stack_lf_count, __rte_stack_lf_push_elems,

and

Quoted 1 line

__rte_stack_lf_pop_elems. It still has some ifdef clutter, but less overall.

Agree. I prefer to take this route to reduce the ifdef clutter across generic and c11 files.

Got it. I will update it in the next version.

Thanks,
Phil Yang

#22 lore raw
In reply to Phil Yang
Quoted 68 lines

-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Friday, June 28, 2019 1:42 PM
To: [email protected]
Cc: [email protected]; Jerin Jacob Kollanukkaran <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: [EXT] [PATCH v3 1/3] eal/arm64: add 128-bit atomic compare exchange

Add 128-bit atomic compare exchange on aarch64.

---
v3:
1. Avoid duplication code with macro. (Jerin Jocob) 2. Make invalid memory order to strongest barrier. (Jerin Jocob) 3. Update doc/guides/prog_guide/env_abstraction_layer.rst. (Eads Gage) 4. Fix 32-bit x86 builds issue. (Eads Gage) 5. Correct documentation issues in UT. (Eads Gage)

 .../common/include/arch/arm/rte_atomic_64.h        | 165
+++++++++++++++++++++
 .../common/include/arch/x86/rte_atomic_64.h        |  12 --
 lib/librte_eal/common/include/generic/rte_atomic.h |  17 ++-
 3 files changed, 181 insertions(+), 13 deletions(-)
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +4 -0
index 97060e4..2080c4d 100644
@@ -1,5 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright(c) 2015 Cavium, Inc
+ * Copyright(c) 2019 Arm Limited
  */
 
 #ifndef _RTE_ATOMIC_ARM64_H_
@@ -14,6 +15,9 @@ extern "C" {
 #endif
 
 #include "generic/rte_atomic.h"
+#include <rte_branch_prediction.h>
+#include <rte_compat.h>
+#include <rte_debug.h>
 
 #define dsb(opt) asm volatile("dsb " #opt : : : "memory")  #define dmb(opt) asm
 volatile("dmb " #opt : : : "memory") @@ -40,6 +44,167 @@ extern "C" {

#define rte_cio_rmb() dmb(oshld)

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5 -0
+/*------------------------ 128 bit atomic operations
+-------------------------*/
+
+#define RTE_HAS_ACQ(mo) ((mo) != __ATOMIC_RELAXED && (mo) !=
+__ATOMIC_RELEASE) #define RTE_HAS_RLS(mo) ((mo) == __ATOMIC_RELEASE

|| \

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5 -0
+			 (mo) == __ATOMIC_ACQ_REL || \
+			 (mo) == __ATOMIC_SEQ_CST)
+
+#define RTE_MO_LOAD(mo)  (RTE_HAS_ACQ((mo)) \
+		? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED) #define

RTE_MO_STORE(mo)

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +4 -0
+(RTE_HAS_RLS((mo)) \
+		? __ATOMIC_RELEASE : __ATOMIC_RELAXED)
+
+#ifdef __ARM_FEATURE_ATOMICS

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 69

Quoted 10 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +10 -0
+#define __ATOMIC128_CAS_OP(cas_op_name, op_string)                          \
+static inline rte_int128_t                                                  \
+cas_op_name(rte_int128_t *dst, rte_int128_t old,                            \
+		rte_int128_t updated)                                               \
+{                                                                           \
+	/* caspX instructions register pair must start from even-numbered
+	 * register at operand 1.
+	 * So, specify registers for local variables here.
+	 */                                                                     \
+	register uint64_t x0 __asm("x0") = (uint64_t)old.val[0];                \

I understand CASP limitation on register has to be even and odd. Is there anyway to remove explicit x0 register allocation and choose compiler to decide the register. Some reason with optimize(03) gcc makes correctly but not clang.

Hardcoding to specific register makes compiler to not optimize the stuff, especially if it is inline function.

#23 lore raw
Quoted 99 lines

-----Original Message-----
From: Jerin Jacob Kollanukkaran
Sent: Wednesday, July 3, 2019 5:56 PM
To: Phil Yang <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: RE: [EXT] [PATCH v3 1/3] eal/arm64: add 128-bit atomic compare exchange

Quoted 56 lines

-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Friday, June 28, 2019 1:42 PM
To: [email protected]
Cc: [email protected]; Jerin Jacob Kollanukkaran <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: [EXT] [PATCH v3 1/3] eal/arm64: add 128-bit atomic compare exchange

Add 128-bit atomic compare exchange on aarch64.

---
v3:
1. Avoid duplication code with macro. (Jerin Jocob) 2. Make invalid memory order to strongest barrier. (Jerin Jocob) 3. Update doc/guides/prog_guide/env_abstraction_layer.rst. (Eads Gage) 4. Fix 32-bit x86 builds issue. (Eads Gage) 5. Correct documentation issues in UT. (Eads Gage)

 .../common/include/arch/arm/rte_atomic_64.h        | 165
+++++++++++++++++++++
 .../common/include/arch/x86/rte_atomic_64.h        |  12 --
 lib/librte_eal/common/include/generic/rte_atomic.h |  17 ++-
 3 files changed, 181 insertions(+), 13 deletions(-)
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +4 -0
index 97060e4..2080c4d 100644
@@ -1,5 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright(c) 2015 Cavium, Inc
+ * Copyright(c) 2019 Arm Limited
  */
 
 #ifndef _RTE_ATOMIC_ARM64_H_
@@ -14,6 +15,9 @@ extern "C" {
 #endif
 
 #include "generic/rte_atomic.h"
+#include <rte_branch_prediction.h>
+#include <rte_compat.h>
+#include <rte_debug.h>
 
 #define dsb(opt) asm volatile("dsb " #opt : : : "memory")  #define

dmb(opt) asm volatile("dmb " #opt : : : "memory") @@ -40,6 +44,167 @@ extern "C" {

#define rte_cio_rmb() dmb(oshld)

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h

Quoted 5 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5 -0
+/*------------------------ 128 bit atomic operations
+-------------------------*/
+
+#define RTE_HAS_ACQ(mo) ((mo) != __ATOMIC_RELAXED && (mo) !=
+__ATOMIC_RELEASE) #define RTE_HAS_RLS(mo) ((mo) ==

__ATOMIC_RELEASE

Quoted 11 lines

|| \

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5 -0
+			 (mo) == __ATOMIC_ACQ_REL || \
+			 (mo) == __ATOMIC_SEQ_CST)
+
+#define RTE_MO_LOAD(mo)  (RTE_HAS_ACQ((mo)) \
+		? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED) #define

RTE_MO_STORE(mo)

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +4 -0
+(RTE_HAS_RLS((mo)) \
+		? __ATOMIC_RELEASE : __ATOMIC_RELAXED)
+
+#ifdef __ARM_FEATURE_ATOMICS

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 69

Quoted 10 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +10 -0
+#define __ATOMIC128_CAS_OP(cas_op_name, op_string)                          \
+static inline rte_int128_t                                                  \
+cas_op_name(rte_int128_t *dst, rte_int128_t old,                            \
+		rte_int128_t updated)                                               \
+{                                                                           \
+	/* caspX instructions register pair must start from even-numbered
+	 * register at operand 1.
+	 * So, specify registers for local variables here.
+	 */                                                                     \
+	register uint64_t x0 __asm("x0") = (uint64_t)old.val[0];                \

I understand CASP limitation on register has to be even and odd.
Is there anyway to remove explicit x0 register allocation and choose compiler to decide the register. Some reason with optimize(03) gcc makes correctly but not clang.

Quoted 2 lines

Hardcoding to specific register makes compiler to not optimize the stuff, especially if it is inline function.

It look like the limitation fixed recently in gcc.
https://patches.linaro.org/patch/147991/

Not sure about old gcc and clang. ARM compiler experts may know the exact status

#24 lore raw

<snip>

Quoted 82 lines
Quoted 54 lines
Quoted 48 lines

Subject: [EXT] [PATCH v3 1/3] eal/arm64: add 128-bit atomic compare exchange

Add 128-bit atomic compare exchange on aarch64.

---
v3:
1. Avoid duplication code with macro. (Jerin Jocob) 2. Make invalid memory order to strongest barrier. (Jerin Jocob) 3. Update doc/guides/prog_guide/env_abstraction_layer.rst. (Eads Gage) 4. Fix 32-bit x86 builds issue. (Eads Gage) 5. Correct documentation issues in UT. (Eads Gage)

 .../common/include/arch/arm/rte_atomic_64.h        | 165
+++++++++++++++++++++
 .../common/include/arch/x86/rte_atomic_64.h        |  12 --
 lib/librte_eal/common/include/generic/rte_atomic.h |  17 ++-
 3 files changed, 181 insertions(+), 13 deletions(-)
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +4 -0
index 97060e4..2080c4d 100644
@@ -1,5 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright(c) 2015 Cavium, Inc
+ * Copyright(c) 2019 Arm Limited
  */
 
 #ifndef _RTE_ATOMIC_ARM64_H_
@@ -14,6 +15,9 @@ extern "C" {
 #endif
 
 #include "generic/rte_atomic.h"
+#include <rte_branch_prediction.h>
+#include <rte_compat.h>
+#include <rte_debug.h>
 
 #define dsb(opt) asm volatile("dsb " #opt : : : "memory")  #define

dmb(opt) asm volatile("dmb " #opt : : : "memory") @@ -40,6 +44,167 @@ extern "C" {

#define rte_cio_rmb() dmb(oshld)

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h

Quoted 5 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5 -0
+/*------------------------ 128 bit atomic operations
+-------------------------*/
+
+#define RTE_HAS_ACQ(mo) ((mo) != __ATOMIC_RELAXED && (mo) !=
+__ATOMIC_RELEASE) #define RTE_HAS_RLS(mo) ((mo) ==

__ATOMIC_RELEASE

Quoted 12 lines

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" {

Quoted 12 lines

|| \

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5 -0
+			 (mo) == __ATOMIC_ACQ_REL || \
+			 (mo) == __ATOMIC_SEQ_CST)
+
+#define RTE_MO_LOAD(mo)  (RTE_HAS_ACQ((mo)) \
+		? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED) #define

RTE_MO_STORE(mo)

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5 -0
+(RTE_HAS_RLS((mo)) \
+		? __ATOMIC_RELEASE : __ATOMIC_RELAXED)
+
+#ifdef __ARM_FEATURE_ATOMICS
+#define __ATOMIC128_CAS_OP(cas_op_name, op_string)

\

Quoted 15 lines

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 69

Quoted 9 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +9 -0
+static inline rte_int128_t                                                  \
+cas_op_name(rte_int128_t *dst, rte_int128_t old,                            \
+		rte_int128_t updated)                                               \
+{                                                                           \
+	/* caspX instructions register pair must start from even-numbered
+	 * register at operand 1.
+	 * So, specify registers for local variables here.
+	 */                                                                     \
+	register uint64_t x0 __asm("x0") = (uint64_t)old.val[0];                \

I understand CASP limitation on register has to be even and odd. Is there anyway to remove explicit x0 register allocation and choose compiler to decide the register. Some reason with optimize(03) gcc makes correctly but not clang.

Quoted 9 lines
Quoted 2 lines

Hardcoding to specific register makes compiler to not optimize the stuff, especially if it is inline function.

It look like the limitation fixed recently in gcc.
https://patches.linaro.org/patch/147991/

Not sure about old gcc and clang. ARM compiler experts may know the exact status

We could use syntax as follows, an example is in [1]
static inline rte_int128_t
__rte_casp(rte_int128_t *dst, rte_int128_t old, rte_int128_t updated, int mo)

{
		__asm__ volatile("caspl %0, %H0, %1, %H1, [%2]"
				 : "+r" (old)
				 : "r" (updated), "r" (dst)
				 : "memory");
	return old;       
}

[1] https://godbolt.org/z/EUJnuG

#25 lore raw
Quoted 122 lines

-----Original Message-----
From: dev <[email protected]> On Behalf Of Honnappa Nagarahalli
Sent: Friday, July 5, 2019 9:51 AM
To: Jerin Jacob Kollanukkaran <[email protected]>; Phil Yang (Arm Technology China) <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Gavin Hu (Arm Technology China) <[email protected]>; Honnappa Nagarahalli <[email protected]>; nd <[email protected]>; [email protected]; nd <[email protected]>
Subject: Re: [dpdk-dev] [EXT] [PATCH v3 1/3] eal/arm64: add 128-bit atomic compare exchange

<snip>

Quoted 1 line
Quoted 1 line
Quoted 1 line

Subject: [EXT] [PATCH v3 1/3] eal/arm64: add 128-bit atomic

compare

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

exchange

Add 128-bit atomic compare exchange on aarch64.

Tested-by Honnappa Nagarahalli

<[email protected]>

Quoted 1 line
Quoted 1 line
Quoted 1 line
Reviewed-by Honnappa Nagarahalli

<[email protected]>

Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

---
v3:
1. Avoid duplication code with macro. (Jerin Jocob) 2. Make invalid memory order to strongest barrier. (Jerin Jocob) 3. Update doc/guides/prog_guide/env_abstraction_layer.rst. (Eads Gage) 4.

Fix

Quoted 1 line
Quoted 1 line
Quoted 1 line

32-bit x86 builds issue. (Eads Gage) 5. Correct documentation

issues

Quoted 9 lines
Quoted 9 lines
Quoted 9 lines

in UT. (Eads Gage)

 .../common/include/arch/arm/rte_atomic_64.h        | 165
+++++++++++++++++++++
 .../common/include/arch/x86/rte_atomic_64.h        |  12 --
 lib/librte_eal/common/include/generic/rte_atomic.h |  17 ++-
 3 files changed, 181 insertions(+), 13 deletions(-)

diff --git

a/lib/librte_eal/common/include/arch/arm/rte_atomic_64.h

Quoted 14 lines
Quoted 14 lines
Quoted 14 lines

b/lib/librte_eal/common/include/arch/arm/rte_atomic_64.h index 97060e4..2080c4d 100644

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +1 -0
@@ -1,5 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright(c) 2015 Cavium, Inc
+ * Copyright(c) 2019 Arm Limited
  */
 
 #ifndef _RTE_ATOMIC_ARM64_H_
@@ -14,6 +15,9 @@ extern "C" {
 #endif
 
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +3 -0
 #include "generic/rte_atomic.h"
+#include <rte_branch_prediction.h>
+#include <rte_compat.h>
+#include <rte_debug.h>
 
 #define dsb(opt) asm volatile("dsb " #opt : : : "memory")  #define

dmb(opt) asm volatile("dmb " #opt : : : "memory") @@ -40,6

+44,167

Quoted 8 lines
Quoted 8 lines

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h

Quoted 8 lines

@@ extern "C" {

#define rte_cio_rmb() dmb(oshld)

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +4 -0
+/*------------------------ 128 bit atomic operations
+-------------------------*/
+
+#define RTE_HAS_ACQ(mo) ((mo) != __ATOMIC_RELAXED &&

(mo) !=

Quoted 15 lines
Quoted 14 lines
Quoted 1 line

+__ATOMIC_RELEASE) #define RTE_HAS_RLS(mo) ((mo) ==

__ATOMIC_RELEASE

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" {

Quoted 12 lines

|| \

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5 -0
+			 (mo) == __ATOMIC_ACQ_REL || \
+			 (mo) == __ATOMIC_SEQ_CST)
+
+#define RTE_MO_LOAD(mo)  (RTE_HAS_ACQ((mo)) \
+		? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED) #define

RTE_MO_STORE(mo)

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5 -0
+(RTE_HAS_RLS((mo)) \
+		? __ATOMIC_RELEASE : __ATOMIC_RELAXED)
+
+#ifdef __ARM_FEATURE_ATOMICS
+#define __ATOMIC128_CAS_OP(cas_op_name, op_string)

\

Quoted 5 lines
Quoted 5 lines

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" {

Quoted 5 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5 -0
+static inline rte_int128_t                                                  \
+cas_op_name(rte_int128_t *dst, rte_int128_t old,                            \
+		rte_int128_t updated)                                               \
+{                                                                           \
+	/* caspX instructions register pair must start from even-

numbered

Quoted 10 lines
Quoted 10 lines

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 69

Quoted 4 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +4 -0
+	 * register at operand 1.
+	 * So, specify registers for local variables here.
+	 */                                                                     \
+	register uint64_t x0 __asm("x0") = (uint64_t)old.val[0];                \

I understand CASP limitation on register has to be even and odd. Is there anyway to remove explicit x0 register allocation and choose compiler to decide the register. Some reason with optimize(03) gcc makes correctly but not clang.

Quoted 7 lines
Quoted 2 lines

Hardcoding to specific register makes compiler to not optimize the stuff, especially if it is inline function.

It look like the limitation fixed recently in gcc.
https://patches.linaro.org/patch/147991/

Not sure about old gcc and clang. ARM compiler experts may know

the exact

Quoted 2 lines

status

We could use syntax as follows, an example is in [1] static inline rte_int128_t
__rte_casp(rte_int128_t *dst, rte_int128_t old, rte_int128_t updated, int mo)

Quoted 7 lines
{
		__asm__ volatile("caspl %0, %H0, %1, %H1, [%2]"
				 : "+r" (old)
				 : "r" (updated), "r" (dst)
				 : "memory");
	return old;
}

We have used this format for mempool/octeontx2 but clang wasn't too happy.

dpdk/drivers/mempool/octeontx2/otx2_mempool_ops.c:151:15: error: value size does not match register size specified by the constraint and modifier [-Werror,-Wasm-operand-widths]

                [t0] "=&r" (t0), [t1] "=&r" (t1), [t2] "=&r" (t2),
                            ^
dpdk/drivers/mempool/octeontx2/otx2_mempool_ops.c:82:9: note: use constraint modifier "w"
                "casp %[t0], %H[t0], %[wdata], %H[wdata], [%[loc]]\n"

Had to change it to hand coded asm

http://patches.dpdk.org/patch/56110/

Quoted 2 lines

[1] https://godbolt.org/z/EUJnuG

#26 lore raw
Quoted 13 lines

-----Original Message-----
From: Pavan Nikhilesh Bhagavatula <[email protected]>
Sent: Friday, July 5, 2019 12:37 PM
To: Honnappa Nagarahalli <[email protected]>; [email protected]; Phil Yang (Arm Technology China) <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>; [email protected]; nd <[email protected]>
Subject: RE: [EXT] [PATCH v3 1/3] eal/arm64: add 128-bit atomic compare exchange

<snip>

Quoted 54 lines
Quoted 9 lines
Quoted 8 lines
Quoted 2 lines

Review

Quoted 2 lines
(quoted diff) +2 -0
+#ifdef __ARM_FEATURE_ATOMICS
+#define __ATOMIC128_CAS_OP(cas_op_name, op_string)

\

Quoted 5 lines

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" {

Quoted 5 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5 -0
+static inline rte_int128_t                                                  \
+cas_op_name(rte_int128_t *dst, rte_int128_t old,                            \
+		rte_int128_t updated)                                               \
+{                                                                           \
+	/* caspX instructions register pair must start from even-

numbered

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" {

Quoted 4 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +4 -0
+	 * register at operand 1.
+	 * So, specify registers for local variables here.
+	 */                                                                     \
+	register uint64_t x0 __asm("x0") = (uint64_t)old.val[0];

\

Quoted 20 lines
Quoted 6 lines
Quoted 6 lines

I understand CASP limitation on register has to be even and odd. Is there anyway to remove explicit x0 register allocation and choose compiler to decide the register. Some reason with optimize(03) gcc makes correctly but not clang.

Quoted 7 lines
Quoted 2 lines

Hardcoding to specific register makes compiler to not optimize the stuff, especially if it is inline function.

It look like the limitation fixed recently in gcc.
https://patches.linaro.org/patch/147991/

Not sure about old gcc and clang. ARM compiler experts may know

the exact

Quoted 2 lines

status

We could use syntax as follows, an example is in [1] static inline rte_int128_t
__rte_casp(rte_int128_t *dst, rte_int128_t old, rte_int128_t updated, int mo)

Quoted 7 lines
{
		__asm__ volatile("caspl %0, %H0, %1, %H1, [%2]"
				 : "+r" (old)
				 : "r" (updated), "r" (dst)
				 : "memory");
	return old;
}

We have used this format for mempool/octeontx2 but clang wasn't too happy.

dpdk/drivers/mempool/octeontx2/otx2_mempool_ops.c:151:15: error:
value size does not match register size specified by the constraint and modifier [-Werror,-Wasm-operand-widths]

                [t0] "=&r" (t0), [t1] "=&r" (t1), [t2] "=&r" (t2),
                            ^
dpdk/drivers/mempool/octeontx2/otx2_mempool_ops.c:82:9: note: use
constraint modifier "w"
                "casp %[t0], %H[t0], %[wdata], %H[wdata], [%[loc]]\n"
Quoted 3 lines

Had to change it to hand coded asm

http://patches.dpdk.org/patch/56110/

Hi Jerin,

The update from the compiler team is 'the LSE CASP fix has not been backported to older GCC branches'. So, currently, this seems the only approach works for all versions of GCC and Clang. I think we can add another optimization patch for this once all the compilers were ready.

Thanks,
Phil

Quoted 3 lines
Quoted 3 lines

[1] https://godbolt.org/z/EUJnuG

#27 lore raw
Quoted 84 lines

-----Original Message-----
From: Phil Yang (Arm Technology China) <[email protected]>
Sent: Tuesday, July 9, 2019 2:58 PM
To: Pavan Nikhilesh Bhagavatula <[email protected]>; Honnappa Nagarahalli <[email protected]>; Jerin Jacob Kollanukkaran <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>; [email protected]; nd <[email protected]>; nd <[email protected]>
Subject: RE: [EXT] [PATCH v3 1/3] eal/arm64: add 128-bit atomic compare exchange

Quoted 13 lines

-----Original Message-----
From: Pavan Nikhilesh Bhagavatula <[email protected]>
Sent: Friday, July 5, 2019 12:37 PM
To: Honnappa Nagarahalli <[email protected]>; [email protected]; Phil Yang (Arm Technology China) <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>; [email protected]; nd <[email protected]>
Subject: RE: [EXT] [PATCH v3 1/3] eal/arm64: add 128-bit atomic compare exchange

<snip>

Quoted 52 lines
Quoted 9 lines
Quoted 8 lines
Quoted 2 lines

Review

Quoted 2 lines
(quoted diff) +2 -0
+#ifdef __ARM_FEATURE_ATOMICS
+#define __ATOMIC128_CAS_OP(cas_op_name, op_string)

\

Quoted 5 lines

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" {

Quoted 5 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5 -0
+static inline rte_int128_t                                                  \
+cas_op_name(rte_int128_t *dst, rte_int128_t old,                            \
+		rte_int128_t updated)                                               \
+{                                                                           \
+	/* caspX instructions register pair must start from even-

numbered

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" {

Quoted 4 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +4 -0
+	 * register at operand 1.
+	 * So, specify registers for local variables here.
+	 */                                                                     \
+	register uint64_t x0 __asm("x0") = (uint64_t)old.val[0];

\

Quoted 18 lines
Quoted 6 lines
Quoted 6 lines

I understand CASP limitation on register has to be even and odd. Is there anyway to remove explicit x0 register allocation and choose compiler to decide the register. Some reason with optimize(03) gcc makes correctly but not clang.

Quoted 7 lines
Quoted 2 lines

Hardcoding to specific register makes compiler to not optimize the stuff, especially if it is inline function.

It look like the limitation fixed recently in gcc.
https://patches.linaro.org/patch/147991/

Not sure about old gcc and clang. ARM compiler experts may know

the exact

Quoted 2 lines

status

We could use syntax as follows, an example is in [1] static inline rte_int128_t __rte_casp(rte_int128_t *dst, rte_int128_t old,

Quoted 7 lines
rte_int128_t updated, int mo) {
		__asm__ volatile("caspl %0, %H0, %1, %H1, [%2]"
				 : "+r" (old)
				 : "r" (updated), "r" (dst)
				 : "memory");
	return old;
}

We have used this format for mempool/octeontx2 but clang wasn't too happy.

dpdk/drivers/mempool/octeontx2/otx2_mempool_ops.c:151:15: error:
value size does not match register size specified by the constraint and modifier [-Werror,-Wasm-operand-widths]

                [t0] "=&r" (t0), [t1] "=&r" (t1), [t2] "=&r" (t2),
                            ^
dpdk/drivers/mempool/octeontx2/otx2_mempool_ops.c:82:9: note: use
constraint modifier "w"
                "casp %[t0], %H[t0], %[wdata], %H[wdata], [%[loc]]\n"
Quoted 3 lines

Had to change it to hand coded asm

http://patches.dpdk.org/patch/56110/

Quoted 8 lines

Hi Jerin,

The update from the compiler team is 'the LSE CASP fix has not been backported to older GCC branches'.
So, currently, this seems the only approach works for all versions of GCC and Clang.
I think we can add another optimization patch for this once all the compilers were ready.

We are on same page.

Quoted 6 lines

Thanks,
Phil

Quoted 3 lines
Quoted 3 lines

[1] https://godbolt.org/z/EUJnuG

#28 lore raw
In reply to Phil Yang
Quoted 28 lines

-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Friday, June 28, 2019 1:42 PM
To: [email protected]
Cc: [email protected]; Jerin Jacob Kollanukkaran <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: [EXT] [PATCH v3 1/3] eal/arm64: add 128-bit atomic compare exchange

External Email

----------------------------------------------------------------------
Add 128-bit atomic compare exchange on aarch64.

---

(quoted diff) +2 -0
+#define RTE_HAS_ACQ(mo) ((mo) != __ATOMIC_RELAXED && (mo) !=
+__ATOMIC_RELEASE) #define RTE_HAS_RLS(mo) ((mo) ==

__ATOMIC_RELEASE || \

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5 -0
+			 (mo) == __ATOMIC_ACQ_REL || \
+			 (mo) == __ATOMIC_SEQ_CST)
+
+#define RTE_MO_LOAD(mo)  (RTE_HAS_ACQ((mo)) \
+		? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED) #define

RTE_MO_STORE(mo)

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 58

Quoted 3 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +3 -0
+(RTE_HAS_RLS((mo)) \
+		? __ATOMIC_RELEASE : __ATOMIC_RELAXED)
+

The one starts with RTE_ are public symbols, If it is generic enough, Move to common layer so that every architecturse can use. If you think, otherwise make it internal

Quoted 1 line

+#ifdef __ARM_FEATURE_ATOMICS

This define is added in gcc 9.1 and I believe for clang it is not supported yet. So old gcc and clang this will be undefined.
I think, With meson + native build, we can find the presence of ATOMIC support by running a.out. Not sure about make and cross build case. I don't want block this feature because of this, IMO, We can add this code with existing __ARM_FEATURE_ATOMICS scheme and later find a method to enhance it. But please check how to fix it.

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 69

Quoted 10 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +10 -0
+#define __ATOMIC128_CAS_OP(cas_op_name, op_string)                          \
+static inline rte_int128_t                                                  \
+cas_op_name(rte_int128_t *dst, rte_int128_t old,                            \
+		rte_int128_t updated)                                               \
+{                                                                           \
+	/* caspX instructions register pair must start from even-numbered
+	 * register at operand 1.
+	 * So, specify registers for local variables here.
+	 */                                                                     \
+	register uint64_t x0 __asm("x0") = (uint64_t)old.val[0];                \

Since direct x0 register used in the code and
cas_op_name() and rte_atomic128_cmp_exchange() is inline function, Based on parent function load, we may corrupt x0 register aka Break arm64 ABI. Not sure clobber list will help here or not? Making it as no_inline will help but not sure about the performance impact. May be you can check with compiler team.

We burned our hands with this scheme, see 5b40ec6b966260e0ff66a8a2c689664f75d6a0e6 ("mempool/octeontx2: fix possible arm64 ABI break")

Probably we can choose a scheme for rc2 and adjust as when we have complete clarity.

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 80

Quoted 12 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5 -0
+	register uint64_t x1 __asm("x1") = (uint64_t)old.val[1];                \
+	register uint64_t x2 __asm("x2") = (uint64_t)updated.val[0];            \
+	register uint64_t x3 __asm("x3") = (uint64_t)updated.val[1];            \
+	asm volatile(                                                           \
+			op_string " %[old0], %[old1], %[upd0], %[upd1],

[%[dst]]" \

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +6 -0
+			: [old0] "+r" (x0),                                             \
+			  [old1] "+r" (x1)                                              \
+			: [upd0] "r" (x2),                                              \
+			  [upd1] "r" (x3),                                              \
+			  [dst] "r" (dst)                                               \
+			: "memory");                                                    \

Should n't we add x0,x1, x2, x3 in clobber list?

Quoted 9 lines

static inline int __rte_experimental
rte_atomic128_cmp_exchange(rte_int128_t *dst,

lib/librte_eal/common/include/arch/x86/rte_atomic_64.h +0 -0
 			   rte_int128_t *exp,
lib/librte_eal/common/include/generic/rte_atomic.h +0 -0
index 9958543..2355e50 100644

lib/librte_eal/common/include/generic/rte_atomic.h static inline void · line 1085

Quoted 5 lines
lib/librte_eal/common/include/generic/rte_atomic.h +1 -0
@@ -1081,6 +1081,20 @@ static inline void
 rte_atomic64_clear(rte_atomic64_t *v)
 
 /*------------------------ 128 bit atomic operations -------------------------*/
 
+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)

There is nothing specific to x86 and arm64 here, Can we remove this #ifdef ?

Quoted 14 lines
lib/librte_eal/common/include/generic/rte_atomic.h +13 -0
+/**
+ * 128-bit integer structure.
+ */
+RTE_STD_C11
+typedef struct {
+	RTE_STD_C11
+	union {
+		uint64_t val[2];
+		__extension__ __int128 int128;
+	};
+} __rte_aligned(16) rte_int128_t;
+#endif
+
 #ifdef __DOXYGEN__
#29 lore raw
Quoted 38 lines

-----Original Message-----
From: Jerin Jacob Kollanukkaran <[email protected]>
Sent: Friday, July 19, 2019 2:25 PM
To: Phil Yang (Arm Technology China) <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>; [email protected]
Subject: RE: [EXT] [PATCH v3 1/3] eal/arm64: add 128-bit atomic compare exchange

Quoted 5 lines

-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Friday, June 28, 2019 1:42 PM
To: [email protected]
Cc: [email protected]; Jerin Jacob Kollanukkaran

<[email protected]>;

Quoted 22 lines

[email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: [EXT] [PATCH v3 1/3] eal/arm64: add 128-bit atomic compare exchange

External Email

----------------------------------------------------------------------
Add 128-bit atomic compare exchange on aarch64.

---

(quoted diff) +2 -0
+#define RTE_HAS_ACQ(mo) ((mo) != __ATOMIC_RELAXED && (mo) !=
+__ATOMIC_RELEASE) #define RTE_HAS_RLS(mo) ((mo) ==

__ATOMIC_RELEASE || \

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5 -0
+			 (mo) == __ATOMIC_ACQ_REL || \
+			 (mo) == __ATOMIC_SEQ_CST)
+
+#define RTE_MO_LOAD(mo)  (RTE_HAS_ACQ((mo)) \
+		? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED) #define
Quoted 8 lines

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 58

Quoted 4 lines

RTE_MO_STORE(mo)

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +3 -0
+(RTE_HAS_RLS((mo)) \
+		? __ATOMIC_RELEASE : __ATOMIC_RELAXED)
+

The one starts with RTE_ are public symbols, If it is generic enough, Move to common layer so that every architecturse can use. If you think, otherwise make it internal

Let's keep it internal. I will remove the 'RTE_' tag.

Quoted 5 lines
Quoted 4 lines

+#ifdef __ARM_FEATURE_ATOMICS

Quoted 7 lines

This define is added in gcc 9.1 and I believe for clang it is not supported yet. So old gcc and clang this will be undefined.
I think, With meson + native build, we can find the presence of ATOMIC support by running a.out. Not sure about make and cross build case. I don't want block this feature because of this, IMO, We can add this code with existing __ARM_FEATURE_ATOMICS scheme and later find a method to enhance it. But please check how to fix it.

OK.

Quoted 15 lines

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 69

Quoted 11 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +10 -0
+#define __ATOMIC128_CAS_OP(cas_op_name, op_string)                          \
+static inline rte_int128_t                                                  \
+cas_op_name(rte_int128_t *dst, rte_int128_t old,                            \
+		rte_int128_t updated)                                               \
+{                                                                           \
+	/* caspX instructions register pair must start from even-numbered
+	 * register at operand 1.
+	 * So, specify registers for local variables here.
+	 */                                                                     \
+	register uint64_t x0 __asm("x0") = (uint64_t)old.val[0];                \

Since direct x0 register used in the code and cas_op_name() and rte_atomic128_cmp_exchange() is inline function, Based on parent function load, we may corrupt x0 register aka

Since x0/x1 and x2/x3 are used a lot and often contain live values.
Maybe to change them to some relatively less frequently used registers like x14/x15 and x16/x17 might help for this case? According to the PCS (Procedure Call Standard), x14-x17 are also temporary registers.

Quoted 1 line

Break arm64 ABI. Not sure clobber list will help here or not?

In my understanding, for the register variable, if it contains a live value in the specified register, the compiler will move the live value into a free register. Since x0~x3 are present in the input/output operands and x0/x1's value needs to be restored to the variable 'old' as a return value.
So I didn't add them into the clobber list.

Quoted 10 lines

Making it as no_inline will help but not sure about the performance impact. May be you can check with compiler team.

We burned our hands with this scheme, see 5b40ec6b966260e0ff66a8a2c689664f75d6a0e6 ("mempool/octeontx2: fix possible arm64 ABI break")

Probably we can choose a scheme for rc2 and adjust as when we have complete clarity.

Quoted 14 lines

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 80

Quoted 12 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5 -0
+	register uint64_t x1 __asm("x1") = (uint64_t)old.val[1];                \
+	register uint64_t x2 __asm("x2") = (uint64_t)updated.val[0];            \
+	register uint64_t x3 __asm("x3") = (uint64_t)updated.val[1];            \
+	asm volatile(                                                           \
+			op_string " %[old0], %[old1], %[upd0], %[upd1],

[%[dst]]" \

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +6 -0
+			: [old0] "+r" (x0),                                             \
+			  [old1] "+r" (x1)                                              \
+			: [upd0] "r" (x2),                                              \
+			  [upd1] "r" (x3),                                              \
+			  [dst] "r" (dst)                                               \
+			: "memory");                                                    \

Should n't we add x0,x1, x2, x3 in clobber list?

Same as above.

Quoted 13 lines
Quoted 13 lines

static inline int __rte_experimental
rte_atomic128_cmp_exchange(rte_int128_t *dst,

lib/librte_eal/common/include/arch/x86/rte_atomic_64.h +0 -0
 			   rte_int128_t *exp,
lib/librte_eal/common/include/generic/rte_atomic.h +0 -0
index 9958543..2355e50 100644
@@ -1081,6 +1081,20 @@ static inline void
 rte_atomic64_clear(rte_atomic64_t *v)
 
Quoted 5 lines

lib/librte_eal/common/include/generic/rte_atomic.h static inline void rte_atomic64_clear(rte_atomic64_t *v) · line 1084

Quoted 3 lines
lib/librte_eal/common/include/generic/rte_atomic.h +1 -0
 /*------------------------ 128 bit atomic operations -------------------------*/
 
+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)

There is nothing specific to x86 and arm64 here, Can we remove this #ifdef ?

Without this constraint, it will break 32-bit x86 builds.
http://mails.dpdk.org/archives/test-report/2019-June/086586.html

Quoted 15 lines
Quoted 15 lines
lib/librte_eal/common/include/generic/rte_atomic.h +13 -0
+/**
+ * 128-bit integer structure.
+ */
+RTE_STD_C11
+typedef struct {
+	RTE_STD_C11
+	union {
+		uint64_t val[2];
+		__extension__ __int128 int128;
+	};
+} __rte_aligned(16) rte_int128_t;
+#endif
+
 #ifdef __DOXYGEN__
#30 lore raw
Quoted 17 lines
Quoted 8 lines
Quoted 8 lines
(quoted diff) +2 -0
+#define RTE_HAS_ACQ(mo) ((mo) != __ATOMIC_RELAXED && (mo) !=
+__ATOMIC_RELEASE) #define RTE_HAS_RLS(mo) ((mo) ==

__ATOMIC_RELEASE || \

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5 -0
+			 (mo) == __ATOMIC_ACQ_REL || \
+			 (mo) == __ATOMIC_SEQ_CST)
+
+#define RTE_MO_LOAD(mo)  (RTE_HAS_ACQ((mo)) \
+		? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED) #define
Quoted 8 lines

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 58

Quoted 4 lines

RTE_MO_STORE(mo)

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +3 -0
+(RTE_HAS_RLS((mo)) \
+		? __ATOMIC_RELEASE : __ATOMIC_RELAXED)
+

The one starts with RTE_ are public symbols, If it is generic enough, Move to common layer so that every architecturse can use. If you think, otherwise make it internal

Quoted 1 line

Let's keep it internal. I will remove the 'RTE_' tag.

Probably change to __HAS_ACQ to avoid collision(just in case)

Quoted 13 lines
Quoted 5 lines
Quoted 4 lines

+#ifdef __ARM_FEATURE_ATOMICS

Quoted 7 lines

This define is added in gcc 9.1 and I believe for clang it is not supported yet. So old gcc and clang this will be undefined.
I think, With meson + native build, we can find the presence of ATOMIC support by running a.out. Not sure about make and cross build case. I don't want block this feature because of this, IMO, We can add this code with existing __ARM_FEATURE_ATOMICS scheme and later find a method to enhance it. But please check how to fix it.

Quoted 1 line

OK.

After thinking on this a bit, I think, in order to support old gcc(< gcc 9.1) and clang, We can introduce a config option, where, by default it is disabled and enable In specific config(where we know, lse is supported) and meson config.

i.e
#if defined(__ARM_FEATURE_ATOMICS) || defined(RTE_ARM_FEATURE_ATOMICS)

Quoted 17 lines
Quoted 16 lines

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 69

Quoted 12 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +10 -0
+#define __ATOMIC128_CAS_OP(cas_op_name, op_string)                          \
+static inline rte_int128_t                                                  \
+cas_op_name(rte_int128_t *dst, rte_int128_t old,                            \
+		rte_int128_t updated)                                               \
+{                                                                           \
+	/* caspX instructions register pair must start from even-numbered
+	 * register at operand 1.
+	 * So, specify registers for local variables here.
+	 */                                                                     \
+	register uint64_t x0 __asm("x0") = (uint64_t)old.val[0];                \

Since direct x0 register used in the code and cas_op_name() and rte_atomic128_cmp_exchange() is inline function, Based on parent function load, we may corrupt x0 register aka

Quoted 5 lines

Since x0/x1 and x2/x3 are used a lot and often contain live values. Maybe to change them to some relatively less frequently used registers like x14/x15 and x16/x17 might help for this case?
According to the PCS (Procedure Call Standard), x14-x17 are also temporary registers.

X14-x17 are temporary registers but since
cas_op_name() and rte_atomic128_cmp_exchange() are inline functions, Based on the parent function register usage, it _may_ corrupt.

Quoted 8 lines
Quoted 2 lines

Break arm64 ABI. Not sure clobber list will help here or not?

In my understanding, for the register variable, if it contains a live value in the specified register, the compiler will move the live value into a free register. Since x0~x3 are present in the input/output operands and x0/x1's value needs to be restored to the variable 'old' as a return value.
So I didn't add them into the clobber list.

OK

Quoted 12 lines
Quoted 12 lines

Making it as no_inline will help but not sure about the performance impact. May be you can check with compiler team.

We burned our hands with this scheme, see 5b40ec6b966260e0ff66a8a2c689664f75d6a0e6 ("mempool/octeontx2: fix possible arm64 ABI break")

Probably we can choose a scheme for rc2 and adjust as when we have complete clarity.

Quoted 1 line
Quoted 6 lines
Quoted 3 lines
Quoted 1 line

+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)

There is nothing specific to x86 and arm64 here, Can we remove this #ifdef ?

Without this constraint, it will break 32-bit x86 builds.
http://mails.dpdk.org/archives/test-report/2019-June/086586.html

OK . #ifdef RTE_ARCH_64 would help then.

Quoted 11 lines
Quoted 11 lines

lib/librte_eal/common/include/generic/rte_atomic.h static inline void rte_atomic64_clear(rte_atomic64_t *v) · line 1093

Quoted 11 lines
lib/librte_eal/common/include/generic/rte_atomic.h +9 -0
+/**
+ * 128-bit integer structure.
+ */
+RTE_STD_C11
+typedef struct {
+	RTE_STD_C11
+	union {
+		uint64_t val[2];
+		__extension__ __int128 int128;

Instead of guarding RTE_ARCH_64 on this complete structure, How about it only under
#ifdef RTE_ARCH_64
__extension__ __int128 int128;
#endif
So that it rte_int128_t will be available for 32bit as well.

Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
lib/librte_eal/common/include/generic/rte_atomic.h +4 -0
+	};
+} __rte_aligned(16) rte_int128_t;
+#endif
+
 #ifdef __DOXYGEN__
#31 lore raw
Quoted 28 lines

-----Original Message-----
From: Jerin Jacob Kollanukkaran <[email protected]>
Sent: Friday, July 19, 2019 8:35 PM
To: Phil Yang (Arm Technology China) <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>; [email protected]; nd <[email protected]>
Subject: RE: [EXT] [PATCH v3 1/3] eal/arm64: add 128-bit atomic compare exchange

Quoted 17 lines
Quoted 16 lines

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 58

Quoted 12 lines
(quoted diff) +2 -0
+#define RTE_HAS_ACQ(mo) ((mo) != __ATOMIC_RELAXED && (mo) !=
+__ATOMIC_RELEASE) #define RTE_HAS_RLS(mo) ((mo) ==

__ATOMIC_RELEASE || \

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5 -0
+			 (mo) == __ATOMIC_ACQ_REL || \
+			 (mo) == __ATOMIC_SEQ_CST)
+
+#define RTE_MO_LOAD(mo)  (RTE_HAS_ACQ((mo)) \
+		? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED) #define

RTE_MO_STORE(mo)

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +3 -0
+(RTE_HAS_RLS((mo)) \
+		? __ATOMIC_RELEASE : __ATOMIC_RELAXED)
+

The one starts with RTE_ are public symbols, If it is generic enough, Move to common layer so that every architecturse can use. If you think, otherwise make it internal

Quoted 3 lines
Quoted 1 line

Let's keep it internal. I will remove the 'RTE_' tag.

Probably change to __HAS_ACQ to avoid collision(just in case)

OK.

Quoted 18 lines
Quoted 7 lines
Quoted 7 lines
Quoted 5 lines

+#ifdef __ARM_FEATURE_ATOMICS

This define is added in gcc 9.1 and I believe for clang it is not supported

yet.

Quoted 3 lines
Quoted 3 lines

So old gcc and clang this will be undefined.
I think, With meson + native build, we can find the presence of ATOMIC support by running a.out. Not sure about make and cross build

case.

Quoted 5 lines
Quoted 3 lines

I don't want block this feature because of this, IMO, We can add this code with existing __ARM_FEATURE_ATOMICS scheme and later find a method to enhance it. But please check how to fix it.

OK.

Quoted 8 lines

After thinking on this a bit, I think, in order to support old gcc(< gcc 9.1) and clang,
We can introduce a config option, where, by default it is disabled and enable In specific config(where we know, lse is supported) and meson config.

i.e
#if defined(__ARM_FEATURE_ATOMICS) ||
defined(RTE_ARM_FEATURE_ATOMICS)

Cool

Quoted 22 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

+#define __ATOMIC128_CAS_OP(cas_op_name, op_string)

\

Quoted 14 lines
Quoted 5 lines
Quoted 5 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5 -0
+static inline rte_int128_t                                                  \
+cas_op_name(rte_int128_t *dst, rte_int128_t old,                            \
+		rte_int128_t updated)                                               \
+{                                                                           \
+	/* caspX instructions register pair must start from even-numbered
Quoted 8 lines

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 69

Quoted 4 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +4 -0
+	 * register at operand 1.
+	 * So, specify registers for local variables here.
+	 */                                                                     \
+	register uint64_t x0 __asm("x0") = (uint64_t)old.val[0];                \

Since direct x0 register used in the code and cas_op_name() and rte_atomic128_cmp_exchange() is inline function, Based on parent function load, we may corrupt x0 register aka

Quoted 2 lines

Since x0/x1 and x2/x3 are used a lot and often contain live values. Maybe to change them to some relatively less frequently used registers

Quoted 8 lines

like

Quoted 3 lines

x14/x15 and x16/x17 might help for this case?
According to the PCS (Procedure Call Standard), x14-x17 are also temporary registers.

X14-x17 are temporary registers but since
cas_op_name() and rte_atomic128_cmp_exchange() are inline functions, Based on the parent function register usage, it _may_ corrupt.

Just checked how Linux Kernel does similar things:
https://github.com/torvalds/linux/blob/master/arch/arm64/include/asm/atomic_lse.h#L19

Same methods.

I will finish the benchmarking for the no_inline approach. If it has no significant performance loss, I think we can make it as no_inline.

Quoted 30 lines
Quoted 6 lines
Quoted 4 lines

Break arm64 ABI. Not sure clobber list will help here or not?

In my understanding, for the register variable, if it contains a live value in

the

Quoted 2 lines

specified register, the compiler will move the live value into a free register. Since x0~x3 are present in the input/output operands and x0/x1's value

needs to

Quoted 2 lines

be restored to the variable 'old' as a return value. So I didn't add them into the clobber list.

OK

Quoted 2 lines
Quoted 2 lines

Making it as no_inline will help but not sure about the performance

impact.

Quoted 12 lines
Quoted 12 lines

May be you can check with compiler team.

We burned our hands with this scheme, see 5b40ec6b966260e0ff66a8a2c689664f75d6a0e6 ("mempool/octeontx2: fix possible arm64 ABI break")

Probably we can choose a scheme for rc2 and adjust as when we have complete clarity.

Quoted 2 lines

+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)

Quoted 7 lines
Quoted 1 line
Quoted 1 line

There is nothing specific to x86 and arm64 here, Can we remove this

#ifdef ?

Quoted 3 lines

Without this constraint, it will break 32-bit x86 builds.
http://mails.dpdk.org/archives/test-report/2019-June/086586.html

OK . #ifdef RTE_ARCH_64 would help then.

OK.

Quoted 19 lines
Quoted 12 lines
Quoted 12 lines

lib/librte_eal/common/include/generic/rte_atomic.h static inline void rte_atomic64_clear(rte_atomic64_t *v) · line 1093

Quoted 12 lines
lib/librte_eal/common/include/generic/rte_atomic.h +9 -0
+/**
+ * 128-bit integer structure.
+ */
+RTE_STD_C11
+typedef struct {
+	RTE_STD_C11
+	union {
+		uint64_t val[2];
+		__extension__ __int128 int128;

Instead of guarding RTE_ARCH_64 on this complete structure, How about it only under
#ifdef RTE_ARCH_64
__extension__ __int128 int128;
#endif
So that it rte_int128_t will be available for 32bit as well.

Agree, it should be work. But I am not sure.

Hi Gage,

How do you think about this?

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
lib/librte_eal/common/include/generic/rte_atomic.h +4 -0
+	};
+} __rte_aligned(16) rte_int128_t;
+#endif
+
 #ifdef __DOXYGEN__
#32 lore raw
Quoted 14 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
lib/librte_eal/common/include/generic/rte_atomic.h +5 -0
+/**
+ * 128-bit integer structure.
+ */
+RTE_STD_C11
+typedef struct {
Quoted 8 lines
Quoted 4 lines
Quoted 4 lines

lib/librte_eal/common/include/generic/rte_atomic.h static inline void rte_atomic64_clear(rte_atomic64_t *v) · line 1093

Quoted 4 lines
lib/librte_eal/common/include/generic/rte_atomic.h +4 -0
+	RTE_STD_C11
+	union {
+		uint64_t val[2];
+		__extension__ __int128 int128;

Instead of guarding RTE_ARCH_64 on this complete structure, How about it only under #ifdef RTE_ARCH_64 __extension__ __int128 int128; #endif So that it rte_int128_t will be available for 32bit as well.

Quoted 6 lines

Agree, it should be work. But I am not sure.

Hi Gage,

How do you think about this?

I don't see any harm in that.

author
#33 lore raw

PATCH v4 1/3 eal/arm64: add 128-bit atomic compare exchange

In reply to Phil Yang

Add 128-bit atomic compare exchange on aarch64.

---
V4:
1. Add RTE_ARM_FEATURE_ATOMICS flag to support LSE CASP instructions. (Jerin Jocob)
2. Fix possible arm64 ABI break by making casp_op_name noinline. (Jerin Jocob)
3. Add rte_stack_lf_stubs.h to reduce the ifdef clutter. (Gage Eads/Jerin Jocob)

v3:
1. Avoid duplication code with macro. (Jerin Jocob)
2. Make invalid memory order to strongest barrier. (Jerin Jocob)
3. Update doc/guides/prog_guide/env_abstraction_layer.rst. (Gage Eads)
4. Fix 32-bit x86 builds issue. (Gage Eads)
5. Correct documentation issues in UT. (Gage Eads)

v2:
Initial version.

 config/arm/meson.build                             |   1 +
 config/common_base                                 |   5 +
 config/defconfig_arm64-thunderx2-linuxapp-gcc      |   1 +
 .../common/include/arch/arm/rte_atomic_64.h        | 162 +++++++++++++++++++++
 .../common/include/arch/x86/rte_atomic_64.h        |  12 --
 lib/librte_eal/common/include/generic/rte_atomic.h |  17 ++-
 6 files changed, 185 insertions(+), 13 deletions(-)
config/arm/meson.build +1 -0
index 979018e..a88f21e 100644
@@ -68,6 +68,7 @@ flags_thunderx_extra = [
 	['RTE_USE_C11_MEM_MODEL', false]]
 flags_thunderx2_extra = [
 	['RTE_MACHINE', '"thunderx2"'],
+	['RTE_ARM_FEATURE_ATOMICS', true],
 	['RTE_CACHE_LINE_SIZE', 64],
 	['RTE_MAX_NUMA_NODES', 2],
 	['RTE_MAX_LCORE', 256],
config/common_base +5 -0
index 8ef75c2..8862495 100644
@@ -1067,3 +1067,8 @@ CONFIG_RTE_APP_CRYPTO_PERF=y
 # Compile the eventdev application
 #
 CONFIG_RTE_APP_EVENTDEV=y
+
+#
+# Compile ARM LSE ATOMIC instructions statically
+#
+CONFIG_RTE_ARM_FEATURE_ATOMICS=n
config/defconfig_arm64-thunderx2-linuxapp-gcc +1 -0
index cc5c64b..17b6dec 100644
@@ -6,6 +6,7 @@
 
 CONFIG_RTE_MACHINE="thunderx2"
 
+CONFIG_RTE_ARM_FEATURE_ATOMICS=y
 CONFIG_RTE_CACHE_LINE_SIZE=64
 CONFIG_RTE_MAX_NUMA_NODES=2
 CONFIG_RTE_MAX_LCORE=256
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +162 -0

179 lines not on this page · raw message

lib/librte_eal/common/include/arch/x86/rte_atomic_64.h +0 -12
index e087c6c..1217129 100644
@@ -212,18 +212,6 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)
 
 /*------------------------ 128 bit atomic operations -------------------------*/
 
-/**
- * 128-bit integer structure.
- */
-RTE_STD_C11
-typedef struct {
-	RTE_STD_C11
-	union {
-		uint64_t val[2];
-		__extension__ __int128 int128;
-	};
-} __rte_aligned(16) rte_int128_t;
-
 __rte_experimental
 static inline int
 rte_atomic128_cmp_exchange(rte_int128_t *dst,
lib/librte_eal/common/include/generic/rte_atomic.h +16 -1
index 24ff7dc..e6ab15a 100644
@@ -1081,6 +1081,20 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)
 
 /*------------------------ 128 bit atomic operations -------------------------*/
 
+/**
+ * 128-bit integer structure.
+ */
+RTE_STD_C11
+typedef struct {
+	RTE_STD_C11
+	union {
+		uint64_t val[2];
+#ifdef RTE_ARCH_64
+		__extension__ __int128 int128;
+#endif
+	};
+} __rte_aligned(16) rte_int128_t;
+
 #ifdef __DOXYGEN__
 
 /**
@@ -1093,7 +1107,8 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)
  *     *exp = *dst
  * @endcode
  *
- * @note This function is currently only available for the x86-64 platform.
+ * @note This function is currently available for the x86-64 and aarch64
+ * platforms.
  *
  * @note The success and failure arguments must be one of the __ATOMIC_* values
  * defined in the C++11 standard. For details on their behavior, refer to the
Signature · 1 line
2.7.4
author
#34 lore raw

PATCH v4 2/3 test/atomic: add 128b compare and swap test

Add 128b atomic compare and swap test for aarch64 and x86_64.

---
 app/test/test_atomic.c | 120 ++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 118 insertions(+), 2 deletions(-)
app/test/test_atomic.c +118 -2
index 43be30e..ff6ff88 100644
@@ -1,5 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright(c) 2010-2014 Intel Corporation
+ * Copyright(c) 2019 Arm Limited
  */
 
 #include <stdio.h>
@@ -20,7 +21,7 @@
  * Atomic Variables
  * ================
  *
- * - The main test function performs three subtests. The first test
+ * - The main test function performs four subtests. The first test
  *   checks that the usual inc/dec/add/sub functions are working
  *   correctly:
  *
@@ -61,11 +62,27 @@
  *       atomic_sub(&count, tmp+1);
  *
  *   - At the end of the test, the *count* value must be 0.
+ *
+ * - Test "128b compare and swap" (aarch64 and x86_64 only)
+ *
+ *   - Initialize 128-bit atomic variables to zero.
+ *
+ *   - Invoke ``test_atomici128_cmp_exchange()`` on each lcore. Before doing
+ *     anything else, the cores are waiting a synchro. Each lcore does
+ *     these compare and swap (CAS) operations several times::
+ *
+ *       Acquired CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *       Released CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *       Acquired_Released CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *       Relaxed CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *
+ *   - At the end of the test, the *count128* first 64-bit value and
+ *     second 64-bit value differ by the total iterations.
  */
 
 #define NUM_ATOMIC_TYPES 3
 
-#define N 10000
+#define N 1000000
 
 static rte_atomic16_t a16;
 static rte_atomic32_t a32;
@@ -216,6 +233,74 @@ test_atomic_dec_and_test(__attribute__((unused)) void *arg)
 	return 0;
 }
 
+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)
+static rte_int128_t count128;
+
+/*
+ * rte_atomic128_cmp_exchange() should update a 128 bits counter's first 64
+ * bits by 2 and the second 64 bits by 1 in this test. It should return true
+ * if the compare exchange operation is successful.
+ * This test repeats 128 bits compare and swap operations 10K rounds. In each
+ * iteration it runs compare and swap operation with different memory models.
+ */
+static int
+test_atomic128_cmp_exchange(__attribute__((unused)) void *arg)
+{
+	rte_int128_t expected;
+	int success;
+	unsigned int i;
+
+	while (rte_atomic32_read(&synchro) == 0)
+		;
+
+	expected = count128;
+
+	for (i = 0; i < N; i++) {
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128, &expected,
+					&desired, 1, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
+		} while (success == 0);
+
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128, &expected,
+					&desired, 1, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
+		} while (success == 0);
+
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128, &expected,
+					&desired, 1, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED);
+		} while (success == 0);
+
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128, &expected,
+					&desired, 1, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+		} while (success == 0);
+	}
+
+	return 0;
+}
+#endif
+
 static int
 test_atomic(void)
 {
@@ -340,6 +425,37 @@ test_atomic(void)
 		return -1;
 	}
 
+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)
+	/*
+	 * This case tests the functionality of rte_atomic128b_cmp_exchange
+	 * API. It calls rte_atomic128b_cmp_exchange with four kinds of memory
+	 * models successively on each slave core. Once each 128-bit atomic
+	 * compare and swap operation is successful, it updates the global
+	 * 128-bit counter by 2 for the first 64-bit and 1 for the second
+	 * 64-bit. Each slave core iterates this test 10K times.
+	 * At the end of test, verify whether the first 64-bits of the 128-bit
+	 * counter and the second 64bits is differ by the total iterations. If
+	 * it is, the test passes.
+	 */
+	printf("128b compare and swap test\n");
+	uint64_t iterations = 0;
+
+	rte_atomic32_clear(&synchro);
+	count128.val[0] = 0;
+	count128.val[1] = 0;
+
+	rte_eal_mp_remote_launch(test_atomic128_cmp_exchange, NULL, SKIP_MASTER);
+	rte_atomic32_set(&synchro, 1);
+	rte_eal_mp_wait_lcore();
+	rte_atomic32_clear(&synchro);
+
+	iterations = count128.val[0] - count128.val[1];
+	if (iterations != 4*N*(rte_lcore_count()-1)) {
+		printf("128b compare and swap failed\n");
+		return -1;
+	}
+#endif
+
 	return 0;
 }
 
Signature · 1 line
2.7.4
author
#35 lore raw

PATCH v4 3/3 eal/stack: enable lock-free stack for aarch64

In reply to Phil Yang

Enable both c11 atomic and non c11 atomic lock-free stack for aarch64.

---
 doc/guides/prog_guide/env_abstraction_layer.rst |  4 +-
 doc/guides/rel_notes/release_19_08.rst          |  3 ++
 lib/librte_stack/rte_stack_lf.h                 |  4 ++
 lib/librte_stack/rte_stack_lf_c11.h             | 16 -------
 lib/librte_stack/rte_stack_lf_generic.h         | 16 -------
 lib/librte_stack/rte_stack_lf_stubs.h           | 59 +++++++++++++++++++++++++
 6 files changed, 68 insertions(+), 34 deletions(-)
 create mode 100644 lib/librte_stack/rte_stack_lf_stubs.h
doc/guides/prog_guide/env_abstraction_layer.rst +2 -2
index f15bcd9..d569f95 100644
@@ -592,8 +592,8 @@ Known Issues
   Alternatively, applications can use the lock-free stack mempool handler. When
   considering this handler, note that:
 
-  - It is currently limited to the x86_64 platform, because it uses an
-    instruction (16-byte compare-and-swap) that is not yet available on other
+  - It is currently limited to the aarch64 and x86_64 platforms, because it uses
+    an instruction (16-byte compare-and-swap) that is not yet available on other
     platforms.
   - It has worse average-case performance than the non-preemptive rte_ring, but
     software caching (e.g. the mempool cache) can mitigate this by reducing the
doc/guides/rel_notes/release_19_08.rst +3 -0
index 0a3f840..25d45c1 100644
@@ -212,6 +212,9 @@ New Features
 
   Added multiple cores feature to compression perf tool application.
 
+* **Added Lock-free Stack for aarch64.**
+
+  The lock-free stack implementation is enabled for aarch64 platforms.
 
 Removed Items
 -------------
lib/librte_stack/rte_stack_lf.h +4 -0
index f5581f0..e67630c 100644
@@ -5,11 +5,15 @@
 #ifndef _RTE_STACK_LF_H_
 #define _RTE_STACK_LF_H_
 
+#if !(defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64))
+#include "rte_stack_lf_stubs.h"
+#else
 #ifdef RTE_USE_C11_MEM_MODEL
 #include "rte_stack_lf_c11.h"
 #else
 #include "rte_stack_lf_generic.h"
 #endif
+#endif
 
 /**
  * @internal Push several objects on the lock-free stack (MT-safe).
lib/librte_stack/rte_stack_lf_c11.h +0 -16
index 3d677ae..999359f 100644
@@ -36,12 +36,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 			  struct rte_stack_lf_elem *last,
 			  unsigned int num)
 {
-#ifndef RTE_ARCH_X86_64
-	RTE_SET_USED(first);
-	RTE_SET_USED(last);
-	RTE_SET_USED(list);
-	RTE_SET_USED(num);
-#else
 	struct rte_stack_lf_head old_head;
 	int success;
 
@@ -79,7 +73,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 	 * to the LIFO len update.
 	 */
 	__atomic_add_fetch(&list->len, num, __ATOMIC_RELEASE);
-#endif
 }
 
 static __rte_always_inline struct rte_stack_lf_elem *
@@ -88,14 +81,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 void **obj_table,
 			 struct rte_stack_lf_elem **last)
 {
-#ifndef RTE_ARCH_X86_64
-	RTE_SET_USED(obj_table);
-	RTE_SET_USED(last);
-	RTE_SET_USED(list);
-	RTE_SET_USED(num);
-
-	return NULL;
-#else
 	struct rte_stack_lf_head old_head;
 	uint64_t len;
 	int success;
@@ -169,7 +154,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 	} while (success == 0);
 
 	return old_head.top;
-#endif
 }
 
 #endif /* _RTE_STACK_LF_C11_H_ */
lib/librte_stack/rte_stack_lf_generic.h +0 -16
index 3182151..3abbb53 100644
@@ -36,12 +36,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 			  struct rte_stack_lf_elem *last,
 			  unsigned int num)
 {
-#ifndef RTE_ARCH_X86_64
-	RTE_SET_USED(first);
-	RTE_SET_USED(last);
-	RTE_SET_USED(list);
-	RTE_SET_USED(num);
-#else
 	struct rte_stack_lf_head old_head;
 	int success;
 
@@ -75,7 +69,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 	} while (success == 0);
 
 	rte_atomic64_add((rte_atomic64_t *)&list->len, num);
-#endif
 }
 
 static __rte_always_inline struct rte_stack_lf_elem *
@@ -84,14 +77,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 void **obj_table,
 			 struct rte_stack_lf_elem **last)
 {
-#ifndef RTE_ARCH_X86_64
-	RTE_SET_USED(obj_table);
-	RTE_SET_USED(last);
-	RTE_SET_USED(list);
-	RTE_SET_USED(num);
-
-	return NULL;
-#else
 	struct rte_stack_lf_head old_head;
 	int success;
 
@@ -159,7 +144,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 	} while (success == 0);
 
 	return old_head.top;
-#endif
 }
 
 #endif /* _RTE_STACK_LF_GENERIC_H_ */
lib/librte_stack/rte_stack_lf_stubs.h new file +59 -0

59 lines not on this page · raw message

Signature · 1 line
2.7.4
#36 lore raw

Re: [dpdk-dev] [EXT] [PATCH v4 3/3] eal/stack: enable lock-free stack for aarch64

Quoted 10 lines

-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Monday, July 22, 2019 2:14 PM
To: [email protected]
Cc: [email protected]; Jerin Jacob Kollanukkaran <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: [EXT] [PATCH v4 3/3] eal/stack: enable lock-free stack for aarch64

Enable both c11 atomic and non c11 atomic lock-free stack for aarch64.

Probably tell why "rte_stack_lf_stubs.h" introduced in git commit to have Better history.

#37 lore raw

Re: [dpdk-dev] [EXT] [PATCH v4 3/3] eal/stack: enable lock-free stack for aarch64

Quoted 21 lines

-----Original Message-----
From: Jerin Jacob Kollanukkaran <[email protected]>
Sent: Monday, July 22, 2019 6:22 PM
To: Phil Yang (Arm Technology China) <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>
Subject: RE: [EXT] [PATCH v4 3/3] eal/stack: enable lock-free stack for aarch64

Quoted 5 lines

-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Monday, July 22, 2019 2:14 PM
To: [email protected]
Cc: [email protected]; Jerin Jacob Kollanukkaran

<[email protected]>;

Quoted 4 lines

[email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: [EXT] [PATCH v4 3/3] eal/stack: enable lock-free stack for aarch64

Quoted 5 lines
Quoted 1 line

Enable both c11 atomic and non c11 atomic lock-free stack for aarch64.

Probably tell why "rte_stack_lf_stubs.h" introduced in git commit to have Better history.

Agree. I will update it. Thanks

Thanks,
Phil Yang

#38 lore raw
In reply to Phil Yang
Quoted 14 lines

-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Monday, July 22, 2019 2:14 PM
To: [email protected]
Cc: [email protected]; Jerin Jacob Kollanukkaran <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: [EXT] [PATCH v4 1/3] eal/arm64: add 128-bit atomic compare exchange Add 128-bit atomic compare exchange on aarch64.

config/common_base CONFIG_RTE_APP_CRYPTO_PERF=y

Quoted 3 lines
config/common_base +3 -0
+
+#
+# Compile ARM LSE ATOMIC instructions statically #

There is NO value for the keyword "statically" here. Right?

config/defconfig_arm64-thunderx2-linuxapp-gcc line 9

Quoted 13 lines
config/common_base +1 -0
+CONFIG_RTE_ARM_FEATURE_ATOMICS=n
config/defconfig_arm64-thunderx2-linuxapp-gcc +1 -0
index cc5c64b..17b6dec 100644
@@ -6,6 +6,7 @@
 
 CONFIG_RTE_MACHINE="thunderx2"
 
+CONFIG_RTE_ARM_FEATURE_ATOMICS=y

Add for octeontx2 as well.

Quoted 16 lines
config/defconfig_arm64-thunderx2-linuxapp-gcc +11 -0
 CONFIG_RTE_CACHE_LINE_SIZE=64
 CONFIG_RTE_MAX_NUMA_NODES=2
 CONFIG_RTE_MAX_LCORE=256
 
 
+rte_atomic128_cmp_exchange(rte_int128_t *dst,
+				rte_int128_t *exp,
+				const rte_int128_t *src,
+				unsigned int weak,
+				int success,
+				int failure)
+{
+	/* Always do strong CAS */
+	RTE_SET_USED(weak);
+	/* Ignore memory ordering for failure, memory order for
+	 * success must be stronger or equal

config/defconfig_arm64-thunderx2-linuxapp-gcc

Quoted 10 lines
config/defconfig_arm64-thunderx2-linuxapp-gcc +10 -0
+	 */
+	RTE_SET_USED(failure);
+	/* Find invalid memory order */
+	RTE_ASSERT(success == __ATOMIC_RELAXED
+			|| success == __ATOMIC_ACQUIRE
+			|| success == __ATOMIC_RELEASE
+			|| success == __ATOMIC_ACQ_REL
+			|| success == __ATOMIC_SEQ_CST);
+
+#ifdef __ARM_FEATURE_ATOMICS

Shouldn't it be #if defined(__ARM_FEATURE_ATOMICS) || defined(RTE_ARM_FEATURE_ATOMICS) ?

#39 lore raw
Quoted 23 lines

-----Original Message-----
From: Jerin Jacob Kollanukkaran <[email protected]>
Sent: Monday, July 22, 2019 6:20 PM
To: Phil Yang (Arm Technology China) <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>
Subject: RE: [EXT] [PATCH v4 1/3] eal/arm64: add 128-bit atomic compare exchange

Quoted 5 lines

-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Monday, July 22, 2019 2:14 PM
To: [email protected]
Cc: [email protected]; Jerin Jacob Kollanukkaran

<[email protected]>;

Quoted 3 lines

[email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: [EXT] [PATCH v4 1/3] eal/arm64: add 128-bit atomic compare

exchange

Quoted 2 lines

Add 128-bit atomic compare exchange on aarch64.

Quoted 9 lines

config/common_base CONFIG_RTE_APP_CRYPTO_PERF=y

Quoted 7 lines
config/common_base +3 -0
+
+#
+# Compile ARM LSE ATOMIC instructions statically #

There is NO value for the keyword "statically" here. Right?

Agree. Will remove it.

Quoted 11 lines
Quoted 11 lines
config/common_base +1 -0
+CONFIG_RTE_ARM_FEATURE_ATOMICS=n
config/defconfig_arm64-thunderx2-linuxapp-gcc +0 -0
index cc5c64b..17b6dec 100644
@@ -6,6 +6,7 @@
 
Quoted 6 lines

config/defconfig_arm64-thunderx2-linuxapp-gcc line 9

Quoted 3 lines
config/defconfig_arm64-thunderx2-linuxapp-gcc +1 -0
 CONFIG_RTE_MACHINE="thunderx2"
 
+CONFIG_RTE_ARM_FEATURE_ATOMICS=y

Add for octeontx2 as well.

OK. Will add it in v5.

Quoted 22 lines
Quoted 22 lines
config/defconfig_arm64-thunderx2-linuxapp-gcc +16 -0
 CONFIG_RTE_CACHE_LINE_SIZE=64
 CONFIG_RTE_MAX_NUMA_NODES=2
 CONFIG_RTE_MAX_LCORE=256
 
 
+rte_atomic128_cmp_exchange(rte_int128_t *dst,
+				rte_int128_t *exp,
+				const rte_int128_t *src,
+				unsigned int weak,
+				int success,
+				int failure)
+{
+	/* Always do strong CAS */
+	RTE_SET_USED(weak);
+	/* Ignore memory ordering for failure, memory order for
+	 * success must be stronger or equal
+	 */
+	RTE_SET_USED(failure);
+	/* Find invalid memory order */
+	RTE_ASSERT(success == __ATOMIC_RELAXED
+			|| success == __ATOMIC_ACQUIRE
Quoted 8 lines

config/defconfig_arm64-thunderx2-linuxapp-gcc

Quoted 5 lines
config/defconfig_arm64-thunderx2-linuxapp-gcc +5 -0
+			|| success == __ATOMIC_RELEASE
+			|| success == __ATOMIC_ACQ_REL
+			|| success == __ATOMIC_SEQ_CST);
+
+#ifdef __ARM_FEATURE_ATOMICS

Shouldn't it be #if defined(__ARM_FEATURE_ATOMICS) || defined(RTE_ARM_FEATURE_ATOMICS) ?

Yes. That was a mistake. Will update in the version 5. Thanks.

Thanks,
Phil Yang

author
#40 lore raw

PATCH v5 1/3 eal/arm64: add 128-bit atomic compare exchange

In reply to Phil Yang

Add 128-bit atomic compare exchange on aarch64.

---
v5:
1. Enable RTE_ARM_FEATURE_ATOMICS on octeontx2 in default. (Jerin Jocob)
2. Record the reason of introducing "rte_stack_lf_stubs.h" in git commit. (Jerin, Jocob)
3. Fixed a conditional MACRO error in rte_atomic128_cmp_exchange. (Jerin Jocob)

v4:
1. Add RTE_ARM_FEATURE_ATOMICS flag to support LSE CASP instructions. (Jerin Jocob)
2. Fix possible arm64 ABI break by making casp_op_name noinline. (Jerin Jocob)
3. Add rte_stack_lf_stubs.h to reduce the ifdef clutter. (Gage Eads/Jerin Jocob)

v3:
1. Avoid duplication code with macro. (Jerin Jocob)
2. Make invalid memory order to strongest barrier. (Jerin Jocob)
3. Update doc/guides/prog_guide/env_abstraction_layer.rst. (Gage Eads)
4. Fix 32-bit x86 builds issue. (Gage Eads)
5. Correct documentation issues in UT. (Gage Eads)

v2:
Initial version.

 config/arm/meson.build                             |   2 +
 config/common_base                                 |   5 +
 config/defconfig_arm64-octeontx2-linuxapp-gcc      |   3 +
 config/defconfig_arm64-thunderx2-linuxapp-gcc      |   1 +
 .../common/include/arch/arm/rte_atomic_64.h        | 162 +++++++++++++++++++++
 .../common/include/arch/x86/rte_atomic_64.h        |  12 --
 lib/librte_eal/common/include/generic/rte_atomic.h |  17 ++-
 7 files changed, 189 insertions(+), 13 deletions(-)
config/arm/meson.build +2 -0
index 979018e..1f5f471 100644
@@ -68,6 +68,7 @@ flags_thunderx_extra = [
 	['RTE_USE_C11_MEM_MODEL', false]]
 flags_thunderx2_extra = [
 	['RTE_MACHINE', '"thunderx2"'],
+	['RTE_ARM_FEATURE_ATOMICS', true],
 	['RTE_CACHE_LINE_SIZE', 64],
 	['RTE_MAX_NUMA_NODES', 2],
 	['RTE_MAX_LCORE', 256],
@@ -76,6 +77,7 @@ flags_octeontx2_extra = [
 	['RTE_MACHINE', '"octeontx2"'],
 	['RTE_MAX_NUMA_NODES', 1],
 	['RTE_MAX_LCORE', 24],
+	['RTE_ARM_FEATURE_ATOMICS', true],
 	['RTE_EAL_IGB_UIO', false],
 	['RTE_USE_C11_MEM_MODEL', true]]
 
config/common_base +5 -0
index 8ef75c2..16dea5a 100644
@@ -1067,3 +1067,8 @@ CONFIG_RTE_APP_CRYPTO_PERF=y
 # Compile the eventdev application
 #
 CONFIG_RTE_APP_EVENTDEV=y
+
+#
+# Compile ARM LSE ATOMIC instructions
+#
+CONFIG_RTE_ARM_FEATURE_ATOMICS=n
config/defconfig_arm64-octeontx2-linuxapp-gcc +3 -0
index f20da24..a6508e8 100644
@@ -19,3 +19,6 @@ CONFIG_RTE_EAL_IGB_UIO=n
 
 # Max supported NIX LFs
 CONFIG_RTE_MAX_VFIO_GROUPS=128
+
+# arm64 LSE ATOMIC support
+CONFIG_RTE_ARM_FEATURE_ATOMICS=y
config/defconfig_arm64-thunderx2-linuxapp-gcc +1 -0
index cc5c64b..17b6dec 100644
@@ -6,6 +6,7 @@
 
 CONFIG_RTE_MACHINE="thunderx2"
 
+CONFIG_RTE_ARM_FEATURE_ATOMICS=y
 CONFIG_RTE_CACHE_LINE_SIZE=64
 CONFIG_RTE_MAX_NUMA_NODES=2
 CONFIG_RTE_MAX_LCORE=256
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +162 -0

179 lines not on this page · raw message

lib/librte_eal/common/include/arch/x86/rte_atomic_64.h +0 -12
index e087c6c..1217129 100644
@@ -212,18 +212,6 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)
 
 /*------------------------ 128 bit atomic operations -------------------------*/
 
-/**
- * 128-bit integer structure.
- */
-RTE_STD_C11
-typedef struct {
-	RTE_STD_C11
-	union {
-		uint64_t val[2];
-		__extension__ __int128 int128;
-	};
-} __rte_aligned(16) rte_int128_t;
-
 __rte_experimental
 static inline int
 rte_atomic128_cmp_exchange(rte_int128_t *dst,
lib/librte_eal/common/include/generic/rte_atomic.h +16 -1
index 24ff7dc..e6ab15a 100644
@@ -1081,6 +1081,20 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)
 
 /*------------------------ 128 bit atomic operations -------------------------*/
 
+/**
+ * 128-bit integer structure.
+ */
+RTE_STD_C11
+typedef struct {
+	RTE_STD_C11
+	union {
+		uint64_t val[2];
+#ifdef RTE_ARCH_64
+		__extension__ __int128 int128;
+#endif
+	};
+} __rte_aligned(16) rte_int128_t;
+
 #ifdef __DOXYGEN__
 
 /**
@@ -1093,7 +1107,8 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)
  *     *exp = *dst
  * @endcode
  *
- * @note This function is currently only available for the x86-64 platform.
+ * @note This function is currently available for the x86-64 and aarch64
+ * platforms.
  *
  * @note The success and failure arguments must be one of the __ATOMIC_* values
  * defined in the C++11 standard. For details on their behavior, refer to the
Signature · 1 line
2.7.4
author
#41 lore raw

PATCH v5 2/3 test/atomic: add 128b compare and swap test

Add 128b atomic compare and swap test for aarch64 and x86_64.

---
 app/test/test_atomic.c | 120 ++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 118 insertions(+), 2 deletions(-)
app/test/test_atomic.c +118 -2
index 43be30e..ff6ff88 100644
@@ -1,5 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright(c) 2010-2014 Intel Corporation
+ * Copyright(c) 2019 Arm Limited
  */
 
 #include <stdio.h>
@@ -20,7 +21,7 @@
  * Atomic Variables
  * ================
  *
- * - The main test function performs three subtests. The first test
+ * - The main test function performs four subtests. The first test
  *   checks that the usual inc/dec/add/sub functions are working
  *   correctly:
  *
@@ -61,11 +62,27 @@
  *       atomic_sub(&count, tmp+1);
  *
  *   - At the end of the test, the *count* value must be 0.
+ *
+ * - Test "128b compare and swap" (aarch64 and x86_64 only)
+ *
+ *   - Initialize 128-bit atomic variables to zero.
+ *
+ *   - Invoke ``test_atomici128_cmp_exchange()`` on each lcore. Before doing
+ *     anything else, the cores are waiting a synchro. Each lcore does
+ *     these compare and swap (CAS) operations several times::
+ *
+ *       Acquired CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *       Released CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *       Acquired_Released CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *       Relaxed CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *
+ *   - At the end of the test, the *count128* first 64-bit value and
+ *     second 64-bit value differ by the total iterations.
  */
 
 #define NUM_ATOMIC_TYPES 3
 
-#define N 10000
+#define N 1000000
 
 static rte_atomic16_t a16;
 static rte_atomic32_t a32;
@@ -216,6 +233,74 @@ test_atomic_dec_and_test(__attribute__((unused)) void *arg)
 	return 0;
 }
 
+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)
+static rte_int128_t count128;
+
+/*
+ * rte_atomic128_cmp_exchange() should update a 128 bits counter's first 64
+ * bits by 2 and the second 64 bits by 1 in this test. It should return true
+ * if the compare exchange operation is successful.
+ * This test repeats 128 bits compare and swap operations 10K rounds. In each
+ * iteration it runs compare and swap operation with different memory models.
+ */
+static int
+test_atomic128_cmp_exchange(__attribute__((unused)) void *arg)
+{
+	rte_int128_t expected;
+	int success;
+	unsigned int i;
+
+	while (rte_atomic32_read(&synchro) == 0)
+		;
+
+	expected = count128;
+
+	for (i = 0; i < N; i++) {
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128, &expected,
+					&desired, 1, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
+		} while (success == 0);
+
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128, &expected,
+					&desired, 1, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
+		} while (success == 0);
+
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128, &expected,
+					&desired, 1, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED);
+		} while (success == 0);
+
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128, &expected,
+					&desired, 1, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+		} while (success == 0);
+	}
+
+	return 0;
+}
+#endif
+
 static int
 test_atomic(void)
 {
@@ -340,6 +425,37 @@ test_atomic(void)
 		return -1;
 	}
 
+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)
+	/*
+	 * This case tests the functionality of rte_atomic128b_cmp_exchange
+	 * API. It calls rte_atomic128b_cmp_exchange with four kinds of memory
+	 * models successively on each slave core. Once each 128-bit atomic
+	 * compare and swap operation is successful, it updates the global
+	 * 128-bit counter by 2 for the first 64-bit and 1 for the second
+	 * 64-bit. Each slave core iterates this test 10K times.
+	 * At the end of test, verify whether the first 64-bits of the 128-bit
+	 * counter and the second 64bits is differ by the total iterations. If
+	 * it is, the test passes.
+	 */
+	printf("128b compare and swap test\n");
+	uint64_t iterations = 0;
+
+	rte_atomic32_clear(&synchro);
+	count128.val[0] = 0;
+	count128.val[1] = 0;
+
+	rte_eal_mp_remote_launch(test_atomic128_cmp_exchange, NULL, SKIP_MASTER);
+	rte_atomic32_set(&synchro, 1);
+	rte_eal_mp_wait_lcore();
+	rte_atomic32_clear(&synchro);
+
+	iterations = count128.val[0] - count128.val[1];
+	if (iterations != 4*N*(rte_lcore_count()-1)) {
+		printf("128b compare and swap failed\n");
+		return -1;
+	}
+#endif
+
 	return 0;
 }
 
Signature · 1 line
2.7.4
author
#42 lore raw

PATCH v5 3/3 eal/stack: enable lock-free stack for aarch64

In reply to Phil Yang

Enable both c11 atomic and non c11 atomic lock-free stack for aarch64.

Introduced a new header, rte_stack_lf_stubs.h, to reduce the ifdef clutter across generic and c11 files. The rte_stack_lf_stubs.h contains stub implementations of __rte_stack_lf_count, __rte_stack_lf_push_elems and __rte_stack_lf_pop_elems.

---
 doc/guides/prog_guide/env_abstraction_layer.rst |  4 +-
 doc/guides/rel_notes/release_19_08.rst          |  3 ++
 lib/librte_stack/rte_stack_lf.h                 |  4 ++
 lib/librte_stack/rte_stack_lf_c11.h             | 16 -------
 lib/librte_stack/rte_stack_lf_generic.h         | 16 -------
 lib/librte_stack/rte_stack_lf_stubs.h           | 59 +++++++++++++++++++++++++
 6 files changed, 68 insertions(+), 34 deletions(-)
 create mode 100644 lib/librte_stack/rte_stack_lf_stubs.h
doc/guides/prog_guide/env_abstraction_layer.rst +2 -2
index f15bcd9..d569f95 100644
@@ -592,8 +592,8 @@ Known Issues
   Alternatively, applications can use the lock-free stack mempool handler. When
   considering this handler, note that:
 
-  - It is currently limited to the x86_64 platform, because it uses an
-    instruction (16-byte compare-and-swap) that is not yet available on other
+  - It is currently limited to the aarch64 and x86_64 platforms, because it uses
+    an instruction (16-byte compare-and-swap) that is not yet available on other
     platforms.
   - It has worse average-case performance than the non-preemptive rte_ring, but
     software caching (e.g. the mempool cache) can mitigate this by reducing the
doc/guides/rel_notes/release_19_08.rst +3 -0
index 0a3f840..25d45c1 100644
@@ -212,6 +212,9 @@ New Features
 
   Added multiple cores feature to compression perf tool application.
 
+* **Added Lock-free Stack for aarch64.**
+
+  The lock-free stack implementation is enabled for aarch64 platforms.
 
 Removed Items
 -------------
lib/librte_stack/rte_stack_lf.h +4 -0
index f5581f0..e67630c 100644
@@ -5,11 +5,15 @@
 #ifndef _RTE_STACK_LF_H_
 #define _RTE_STACK_LF_H_
 
+#if !(defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64))
+#include "rte_stack_lf_stubs.h"
+#else
 #ifdef RTE_USE_C11_MEM_MODEL
 #include "rte_stack_lf_c11.h"
 #else
 #include "rte_stack_lf_generic.h"
 #endif
+#endif
 
 /**
  * @internal Push several objects on the lock-free stack (MT-safe).
lib/librte_stack/rte_stack_lf_c11.h +0 -16
index 3d677ae..999359f 100644
@@ -36,12 +36,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 			  struct rte_stack_lf_elem *last,
 			  unsigned int num)
 {
-#ifndef RTE_ARCH_X86_64
-	RTE_SET_USED(first);
-	RTE_SET_USED(last);
-	RTE_SET_USED(list);
-	RTE_SET_USED(num);
-#else
 	struct rte_stack_lf_head old_head;
 	int success;
 
@@ -79,7 +73,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 	 * to the LIFO len update.
 	 */
 	__atomic_add_fetch(&list->len, num, __ATOMIC_RELEASE);
-#endif
 }
 
 static __rte_always_inline struct rte_stack_lf_elem *
@@ -88,14 +81,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 void **obj_table,
 			 struct rte_stack_lf_elem **last)
 {
-#ifndef RTE_ARCH_X86_64
-	RTE_SET_USED(obj_table);
-	RTE_SET_USED(last);
-	RTE_SET_USED(list);
-	RTE_SET_USED(num);
-
-	return NULL;
-#else
 	struct rte_stack_lf_head old_head;
 	uint64_t len;
 	int success;
@@ -169,7 +154,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 	} while (success == 0);
 
 	return old_head.top;
-#endif
 }
 
 #endif /* _RTE_STACK_LF_C11_H_ */
lib/librte_stack/rte_stack_lf_generic.h +0 -16
index 3182151..3abbb53 100644
@@ -36,12 +36,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 			  struct rte_stack_lf_elem *last,
 			  unsigned int num)
 {
-#ifndef RTE_ARCH_X86_64
-	RTE_SET_USED(first);
-	RTE_SET_USED(last);
-	RTE_SET_USED(list);
-	RTE_SET_USED(num);
-#else
 	struct rte_stack_lf_head old_head;
 	int success;
 
@@ -75,7 +69,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 	} while (success == 0);
 
 	rte_atomic64_add((rte_atomic64_t *)&list->len, num);
-#endif
 }
 
 static __rte_always_inline struct rte_stack_lf_elem *
@@ -84,14 +77,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 void **obj_table,
 			 struct rte_stack_lf_elem **last)
 {
-#ifndef RTE_ARCH_X86_64
-	RTE_SET_USED(obj_table);
-	RTE_SET_USED(last);
-	RTE_SET_USED(list);
-	RTE_SET_USED(num);
-
-	return NULL;
-#else
 	struct rte_stack_lf_head old_head;
 	int success;
 
@@ -159,7 +144,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 	} while (success == 0);
 
 	return old_head.top;
-#endif
 }
 
 #endif /* _RTE_STACK_LF_GENERIC_H_ */
lib/librte_stack/rte_stack_lf_stubs.h new file +59 -0

59 lines not on this page · raw message

Signature · 1 line
2.7.4
#43 lore raw

Re: [dpdk-dev] [EXT] [PATCH v5 3/3] eal/stack: enable lock-free stack for aarch64

Quoted 16 lines

-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Monday, July 22, 2019 6:36 PM
To: [email protected]
Cc: [email protected]; Jerin Jacob Kollanukkaran <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: [EXT] [PATCH v5 3/3] eal/stack: enable lock-free stack for aarch64

Enable both c11 atomic and non c11 atomic lock-free stack for aarch64.

Introduced a new header, rte_stack_lf_stubs.h, to reduce the ifdef clutter across generic and c11 files. The rte_stack_lf_stubs.h contains stub implementations of __rte_stack_lf_count, __rte_stack_lf_push_elems and __rte_stack_lf_pop_elems.

Quoted 5 lines

# Build issue with 32 bit build, Looks like new header file missing in SYMLINK-$(CONFIG_RTE_LIBRTE_STACK)-include

In file included from /home/jerin/dpdk.org/build/include/rte_stack.h:98,
from /home/jerin/dpdk.org/drivers/mempool/stack/rte_mempool_stack.c:7:
/home/jerin/dpdk.org/build/include/rte_stack_lf.h:9:10: fatal error: rte_stack_lf_stubs.h: No such file or directory

    9 | #include "rte_stack_lf_stubs.h"
      |          ^~~~~~~~~~~~~~~~~~~~~~
#44 lore raw

Re: [dpdk-dev] [EXT] [PATCH v5 3/3] eal/stack: enable lock-free stack for aarch64

Quoted 38 lines

-----Original Message-----
From: Jerin Jacob Kollanukkaran <[email protected]>
Sent: Monday, July 22, 2019 10:15 PM
To: Phil Yang (Arm Technology China) <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>
Subject: RE: [EXT] [PATCH v5 3/3] eal/stack: enable lock-free stack for aarch64

Quoted 5 lines

-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Monday, July 22, 2019 6:36 PM
To: [email protected]
Cc: [email protected]; Jerin Jacob Kollanukkaran

<[email protected]>;

Quoted 9 lines

[email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: [EXT] [PATCH v5 3/3] eal/stack: enable lock-free stack for aarch64

Enable both c11 atomic and non c11 atomic lock-free stack for aarch64.

Introduced a new header, rte_stack_lf_stubs.h, to reduce the ifdef clutter across generic and c11 files. The rte_stack_lf_stubs.h contains stub implementations of __rte_stack_lf_count, __rte_stack_lf_push_elems

and

Quoted 7 lines

__rte_stack_lf_pop_elems.

# Build issue with 32 bit build, Looks like new header file missing in SYMLINK- $(CONFIG_RTE_LIBRTE_STACK)-include

Quoted 8 lines

In file included from /home/jerin/dpdk.org/build/include/rte_stack.h:98,
from
/home/jerin/dpdk.org/drivers/mempool/stack/rte_mempool_stack.c:7:
/home/jerin/dpdk.org/build/include/rte_stack_lf.h:9:10: fatal error:
rte_stack_lf_stubs.h: No such file or directory

    9 | #include "rte_stack_lf_stubs.h"
      |          ^~~~~~~~~~~~~~~~~~~~~~

Yes. Thanks. Will fix it in next version.

#45 lore raw

Re: [dpdk-dev] [PATCH v5 3/3] eal/stack: enable lock-free stack for aarch64

In reply to Phil Yang
Quoted 36 lines
lib/librte_stack/rte_stack_lf_stubs.h new file +13 -0
new file mode 100644
index 0000000..d924bc6
@@ -0,0 +1,59 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2019 Arm Limited
+ */
+
+#ifndef _RTE_STACK_LF_STUBS_H_
+#define _RTE_STACK_LF_STUBS_H_
+
+#include <rte_common.h>
+#include <rte_atomic.h>
+
+static __rte_always_inline unsigned int __rte_stack_lf_count(struct
+rte_stack *s) {
+	/* stack_lf_push() and stack_lf_pop() do not update the list's

contents

lib/librte_stack/rte_stack_lf_stubs.h new file +5 -0
+	 * and stack_lf->len atomically, which can cause the list to appear
+	 * shorter than it actually is if this function is called while other
+	 * threads are modifying the list.
+	 *
+	 * However, given the inherently approximate nature of the

get_count

lib/librte_stack/rte_stack_lf_stubs.h new file +2 -0
+	 * callback -- even if the list and its size were updated atomically,
+	 * the size could change between when get_count executes and

when the

lib/librte_stack/rte_stack_lf_stubs.h new file +3 -0
+	 * value is returned to the caller -- this is acceptable.
+	 *
+	 * The stack_lf->len updates are placed such that the list may appear

to

lib/librte_stack/rte_stack_lf_stubs.h new file +1 -0
+	 * have fewer elements than it does, but will never appear to have

more

lib/librte_stack/rte_stack_lf_stubs.h line 31

Quoted 6 lines
lib/librte_stack/rte_stack_lf_stubs.h new file +6 -0
+	 * elements. If the mempool is near-empty to the point that this is a
+	 * concern, the user should consider increasing the mempool size.
+	 */
+	return (unsigned int)rte_atomic64_read((rte_atomic64_t *)
+			&s->stack_lf.used.len);
+}

Since the stub functions are not functional, these comments should be removed and the function simply return 0. The rte_atomic.h include can be removed as well.

#46 lore raw

Re: [dpdk-dev] [PATCH v5 3/3] eal/stack: enable lock-free stack for aarch64

Quoted 47 lines

-----Original Message-----
From: Eads, Gage <[email protected]>
Sent: Monday, July 22, 2019 10:35 PM
To: Phil Yang (Arm Technology China) <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>
Subject: RE: [PATCH v5 3/3] eal/stack: enable lock-free stack for aarch64

Quoted 38 lines
lib/librte_stack/rte_stack_lf_stubs.h new file +13 -0
new file mode 100644
index 0000000..d924bc6
@@ -0,0 +1,59 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2019 Arm Limited
+ */
+
+#ifndef _RTE_STACK_LF_STUBS_H_
+#define _RTE_STACK_LF_STUBS_H_
+
+#include <rte_common.h>
+#include <rte_atomic.h>
+
+static __rte_always_inline unsigned int __rte_stack_lf_count(struct
+rte_stack *s) {
+	/* stack_lf_push() and stack_lf_pop() do not update the list's

contents

lib/librte_stack/rte_stack_lf_stubs.h new file +5 -0
+	 * and stack_lf->len atomically, which can cause the list to appear
+	 * shorter than it actually is if this function is called while other
+	 * threads are modifying the list.
+	 *
+	 * However, given the inherently approximate nature of the

get_count

lib/librte_stack/rte_stack_lf_stubs.h new file +2 -0
+	 * callback -- even if the list and its size were updated atomically,
+	 * the size could change between when get_count executes and

when the

lib/librte_stack/rte_stack_lf_stubs.h new file +3 -0
+	 * value is returned to the caller -- this is acceptable.
+	 *
+	 * The stack_lf->len updates are placed such that the list may appear

to

lib/librte_stack/rte_stack_lf_stubs.h new file +1 -0
+	 * have fewer elements than it does, but will never appear to have

more

lib/librte_stack/rte_stack_lf_stubs.h new file +2 -0
+	 * elements. If the mempool is near-empty to the point that this is a
+	 * concern, the user should consider increasing the mempool size.
Quoted 8 lines

lib/librte_stack/rte_stack_lf_stubs.h line 31

Quoted 4 lines
lib/librte_stack/rte_stack_lf_stubs.h new file +4 -0
+	 */
+	return (unsigned int)rte_atomic64_read((rte_atomic64_t *)
+			&s->stack_lf.used.len);
+}

Since the stub functions are not functional, these comments should be removed and the function simply return 0. The rte_atomic.h include can be removed as well.

OK. Will do it. Thanks.

#47 lore raw
In reply to Phil Yang

It looks good. Some minor comments.

Quoted 23 lines

-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Monday, July 22, 2019 6:36 PM
To: [email protected]
Cc: [email protected]; Jerin Jacob Kollanukkaran <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: [EXT] [PATCH v5 1/3] eal/arm64: add 128-bit atomic compare exchange
----------------------------------------------------------------------
Add 128-bit atomic compare exchange on aarch64.

config/common_base index renamed from config/common_base +0 -0

8ef75c2..16dea5a 100644

config/common_base +0 -0
@@ -1067,3 +1067,8 @@ CONFIG_RTE_APP_CRYPTO_PERF=y  # Compile the

eventdev application # CONFIG_RTE_APP_EVENTDEV=y

config/common_base CONFIG_RTE_APP_CRYPTO_PERF=y · line 1074

Quoted 5 lines
config/common_base +5 -0
+
+#
+# Compile ARM LSE ATOMIC instructions
+#
+CONFIG_RTE_ARM_FEATURE_ATOMICS=n

Move this config after "Compile Environment Abstraction Layer" section. Now it is at end of file. Better to group the configs.

config/defconfig_arm64-octeontx2-linuxapp-gcc CONFIG_RTE_EAL_IGB_UIO=n · line 24

Quoted 12 lines
config/defconfig_arm64-octeontx2-linuxapp-gcc +3 -0
index f20da24..a6508e8 100644
@@ -19,3 +19,6 @@ CONFIG_RTE_EAL_IGB_UIO=n
 
 # Max supported NIX LFs
 CONFIG_RTE_MAX_VFIO_GROUPS=128
+
+# arm64 LSE ATOMIC support
+CONFIG_RTE_ARM_FEATURE_ATOMICS=y

Move this config after CONFIG_RTE_MAX_LCORE=24

config/defconfig_arm64-thunderx2-linuxapp-gcc line 12

Quoted 13 lines
config/defconfig_arm64-thunderx2-linuxapp-gcc +1 -0
index cc5c64b..17b6dec 100644
@@ -6,6 +6,7 @@
 
 CONFIG_RTE_MACHINE="thunderx2"
 
+CONFIG_RTE_ARM_FEATURE_ATOMICS=y
 CONFIG_RTE_CACHE_LINE_SIZE=64
 CONFIG_RTE_MAX_NUMA_NODES=2
 CONFIG_RTE_MAX_LCORE=256

Move the new config here

#48 lore raw
Quoted 11 lines

-----Original Message-----
From: Jerin Jacob Kollanukkaran <[email protected]>
Sent: Monday, July 22, 2019 10:20 PM
To: Phil Yang (Arm Technology China) <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>
Subject: RE: [EXT] [PATCH v5 1/3] eal/arm64: add 128-bit atomic compare exchange

Quoted 1 line

It looks good. Some minor comments.

Thanks. Updated in the new version.

Quoted 68 lines
Quoted 6 lines

-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Monday, July 22, 2019 6:36 PM
To: [email protected]
Cc: [email protected]; Jerin Jacob Kollanukkaran

<[email protected]>;

config/common_base CONFIG_RTE_APP_CRYPTO_PERF=y # Compile

Quoted 17 lines

[email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: [EXT] [PATCH v5 1/3] eal/arm64: add 128-bit atomic compare exchange
----------------------------------------------------------------------
Add 128-bit atomic compare exchange on aarch64.

config/common_base index renamed from config/common_base +0 -0

8ef75c2..16dea5a 100644

config/common_base +0 -0
@@ -1067,3 +1067,8 @@ CONFIG_RTE_APP_CRYPTO_PERF=y  # Compile

the

config/common_base CONFIG_RTE_APP_CRYPTO_PERF=y · line 1074

Quoted 6 lines

eventdev application # CONFIG_RTE_APP_EVENTDEV=y

config/common_base +5 -0
+
+#
+# Compile ARM LSE ATOMIC instructions
+#
+CONFIG_RTE_ARM_FEATURE_ATOMICS=n

Move this config after "Compile Environment Abstraction Layer" section. Now it is at end of file. Better to group the configs.

config/defconfig_arm64-octeontx2-linuxapp-gcc CONFIG_RTE_EAL_IGB_UIO=n · line 24

Quoted 12 lines
config/defconfig_arm64-octeontx2-linuxapp-gcc +3 -0
index f20da24..a6508e8 100644
@@ -19,3 +19,6 @@ CONFIG_RTE_EAL_IGB_UIO=n
 
 # Max supported NIX LFs
 CONFIG_RTE_MAX_VFIO_GROUPS=128
+
+# arm64 LSE ATOMIC support
+CONFIG_RTE_ARM_FEATURE_ATOMICS=y

Move this config after CONFIG_RTE_MAX_LCORE=24

config/defconfig_arm64-thunderx2-linuxapp-gcc line 12

Quoted 13 lines
config/defconfig_arm64-thunderx2-linuxapp-gcc +1 -0
index cc5c64b..17b6dec 100644
@@ -6,6 +6,7 @@
 
 CONFIG_RTE_MACHINE="thunderx2"
 
+CONFIG_RTE_ARM_FEATURE_ATOMICS=y
 CONFIG_RTE_CACHE_LINE_SIZE=64
 CONFIG_RTE_MAX_NUMA_NODES=2
 CONFIG_RTE_MAX_LCORE=256

Move the new config here

author
#49 lore raw

PATCH v6 1/3 eal/arm64: add 128-bit atomic compare exchange

In reply to Phil Yang

Add 128-bit atomic compare exchange on aarch64.

---
v6:
1. Put the RTE_ARM_FEATURE_ATOMICS flag into EAL group. (Jerin Jocob)
2. Keep rte_stack_lf_stubs.h doing nothing. (Gage Eads)
3. Fixed 32 bit build issue.

v5:
1. Enable RTE_ARM_FEATURE_ATOMICS on octeontx2 in default. (Jerin Jocob)
2. Record the reason of introducing "rte_stack_lf_stubs.h" in git commit. (Jerin, Jocob)
3. Fixed a conditional MACRO error in rte_atomic128_cmp_exchange. (Jerin Jocob)

v4:
1. Add RTE_ARM_FEATURE_ATOMICS flag to support LSE CASP instructions. (Jerin Jocob)
2. Fix possible arm64 ABI break by making casp_op_name noinline. (Jerin Jocob)
3. Add rte_stack_lf_stubs.h to reduce the ifdef clutter. (Gage Eads/Jerin Jocob)

v3:
1. Avoid duplication code with macro. (Jerin Jocob)
2. Make invalid memory order to strongest barrier. (Jerin Jocob)
3. Update doc/guides/prog_guide/env_abstraction_layer.rst. (Gage Eads)
4. Fix 32-bit x86 builds issue. (Gage Eads)
5. Correct documentation issues in UT. (Gage Eads)

v2:
Initial version.

 config/arm/meson.build                             |   2 +
 config/common_base                                 |   2 +
 config/defconfig_arm64-octeontx2-linuxapp-gcc      |   1 +
 config/defconfig_arm64-thunderx2-linuxapp-gcc      |   1 +
 .../common/include/arch/arm/rte_atomic_64.h        | 162 +++++++++++++++++++++
 .../common/include/arch/x86/rte_atomic_64.h        |  12 --
 lib/librte_eal/common/include/generic/rte_atomic.h |  17 ++-
 7 files changed, 184 insertions(+), 13 deletions(-)
config/arm/meson.build +2 -0
index 979018e..9f28271 100644
@@ -71,11 +71,13 @@ flags_thunderx2_extra = [
 	['RTE_CACHE_LINE_SIZE', 64],
 	['RTE_MAX_NUMA_NODES', 2],
 	['RTE_MAX_LCORE', 256],
+	['RTE_ARM_FEATURE_ATOMICS', true],
 	['RTE_USE_C11_MEM_MODEL', true]]
 flags_octeontx2_extra = [
 	['RTE_MACHINE', '"octeontx2"'],
 	['RTE_MAX_NUMA_NODES', 1],
 	['RTE_MAX_LCORE', 24],
+	['RTE_ARM_FEATURE_ATOMICS', true],
 	['RTE_EAL_IGB_UIO', false],
 	['RTE_USE_C11_MEM_MODEL', true]]
 
config/common_base +2 -0
index 8ef75c2..de6d1e0 100644
@@ -82,6 +82,8 @@ CONFIG_RTE_MAX_LCORE=128
 CONFIG_RTE_MAX_NUMA_NODES=8
 CONFIG_RTE_MAX_HEAPS=32
 CONFIG_RTE_MAX_MEMSEG_LISTS=64
+# Use LSE ATOMIC instructions
+CONFIG_RTE_ARM_FEATURE_ATOMICS=n
 # each memseg list will be limited to either RTE_MAX_MEMSEG_PER_LIST pages
 # or RTE_MAX_MEM_MB_PER_LIST megabytes worth of memory, whichever is smaller
 CONFIG_RTE_MAX_MEMSEG_PER_LIST=8192
config/defconfig_arm64-octeontx2-linuxapp-gcc +1 -0
index f20da24..7687dbe 100644
@@ -9,6 +9,7 @@ CONFIG_RTE_MACHINE="octeontx2"
 CONFIG_RTE_CACHE_LINE_SIZE=128
 CONFIG_RTE_MAX_NUMA_NODES=1
 CONFIG_RTE_MAX_LCORE=24
+CONFIG_RTE_ARM_FEATURE_ATOMICS=y
 
 # Doesn't support NUMA
 CONFIG_RTE_EAL_NUMA_AWARE_HUGEPAGES=n
config/defconfig_arm64-thunderx2-linuxapp-gcc +1 -0
index cc5c64b..af4a89c 100644
@@ -9,3 +9,4 @@ CONFIG_RTE_MACHINE="thunderx2"
 CONFIG_RTE_CACHE_LINE_SIZE=64
 CONFIG_RTE_MAX_NUMA_NODES=2
 CONFIG_RTE_MAX_LCORE=256
+CONFIG_RTE_ARM_FEATURE_ATOMICS=y
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +162 -0

179 lines not on this page · raw message

lib/librte_eal/common/include/arch/x86/rte_atomic_64.h +0 -12
index e087c6c..1217129 100644
@@ -212,18 +212,6 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)
 
 /*------------------------ 128 bit atomic operations -------------------------*/
 
-/**
- * 128-bit integer structure.
- */
-RTE_STD_C11
-typedef struct {
-	RTE_STD_C11
-	union {
-		uint64_t val[2];
-		__extension__ __int128 int128;
-	};
-} __rte_aligned(16) rte_int128_t;
-
 __rte_experimental
 static inline int
 rte_atomic128_cmp_exchange(rte_int128_t *dst,
lib/librte_eal/common/include/generic/rte_atomic.h +16 -1
index 24ff7dc..e6ab15a 100644
@@ -1081,6 +1081,20 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)
 
 /*------------------------ 128 bit atomic operations -------------------------*/
 
+/**
+ * 128-bit integer structure.
+ */
+RTE_STD_C11
+typedef struct {
+	RTE_STD_C11
+	union {
+		uint64_t val[2];
+#ifdef RTE_ARCH_64
+		__extension__ __int128 int128;
+#endif
+	};
+} __rte_aligned(16) rte_int128_t;
+
 #ifdef __DOXYGEN__
 
 /**
@@ -1093,7 +1107,8 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)
  *     *exp = *dst
  * @endcode
  *
- * @note This function is currently only available for the x86-64 platform.
+ * @note This function is currently available for the x86-64 and aarch64
+ * platforms.
  *
  * @note The success and failure arguments must be one of the __ATOMIC_* values
  * defined in the C++11 standard. For details on their behavior, refer to the
Signature · 1 line
2.7.4
author
#50 lore raw

PATCH v6 2/3 test/atomic: add 128b compare and swap test

Add 128b atomic compare and swap test for aarch64 and x86_64.

---
 app/test/test_atomic.c | 120 ++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 118 insertions(+), 2 deletions(-)
app/test/test_atomic.c +118 -2
index 43be30e..ff6ff88 100644
@@ -1,5 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright(c) 2010-2014 Intel Corporation
+ * Copyright(c) 2019 Arm Limited
  */
 
 #include <stdio.h>
@@ -20,7 +21,7 @@
  * Atomic Variables
  * ================
  *
- * - The main test function performs three subtests. The first test
+ * - The main test function performs four subtests. The first test
  *   checks that the usual inc/dec/add/sub functions are working
  *   correctly:
  *
@@ -61,11 +62,27 @@
  *       atomic_sub(&count, tmp+1);
  *
  *   - At the end of the test, the *count* value must be 0.
+ *
+ * - Test "128b compare and swap" (aarch64 and x86_64 only)
+ *
+ *   - Initialize 128-bit atomic variables to zero.
+ *
+ *   - Invoke ``test_atomici128_cmp_exchange()`` on each lcore. Before doing
+ *     anything else, the cores are waiting a synchro. Each lcore does
+ *     these compare and swap (CAS) operations several times::
+ *
+ *       Acquired CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *       Released CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *       Acquired_Released CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *       Relaxed CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *
+ *   - At the end of the test, the *count128* first 64-bit value and
+ *     second 64-bit value differ by the total iterations.
  */
 
 #define NUM_ATOMIC_TYPES 3
 
-#define N 10000
+#define N 1000000
 
 static rte_atomic16_t a16;
 static rte_atomic32_t a32;
@@ -216,6 +233,74 @@ test_atomic_dec_and_test(__attribute__((unused)) void *arg)
 	return 0;
 }
 
+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)
+static rte_int128_t count128;
+
+/*
+ * rte_atomic128_cmp_exchange() should update a 128 bits counter's first 64
+ * bits by 2 and the second 64 bits by 1 in this test. It should return true
+ * if the compare exchange operation is successful.
+ * This test repeats 128 bits compare and swap operations 10K rounds. In each
+ * iteration it runs compare and swap operation with different memory models.
+ */
+static int
+test_atomic128_cmp_exchange(__attribute__((unused)) void *arg)
+{
+	rte_int128_t expected;
+	int success;
+	unsigned int i;
+
+	while (rte_atomic32_read(&synchro) == 0)
+		;
+
+	expected = count128;
+
+	for (i = 0; i < N; i++) {
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128, &expected,
+					&desired, 1, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
+		} while (success == 0);
+
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128, &expected,
+					&desired, 1, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
+		} while (success == 0);
+
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128, &expected,
+					&desired, 1, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED);
+		} while (success == 0);
+
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128, &expected,
+					&desired, 1, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+		} while (success == 0);
+	}
+
+	return 0;
+}
+#endif
+
 static int
 test_atomic(void)
 {
@@ -340,6 +425,37 @@ test_atomic(void)
 		return -1;
 	}
 
+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)
+	/*
+	 * This case tests the functionality of rte_atomic128b_cmp_exchange
+	 * API. It calls rte_atomic128b_cmp_exchange with four kinds of memory
+	 * models successively on each slave core. Once each 128-bit atomic
+	 * compare and swap operation is successful, it updates the global
+	 * 128-bit counter by 2 for the first 64-bit and 1 for the second
+	 * 64-bit. Each slave core iterates this test 10K times.
+	 * At the end of test, verify whether the first 64-bits of the 128-bit
+	 * counter and the second 64bits is differ by the total iterations. If
+	 * it is, the test passes.
+	 */
+	printf("128b compare and swap test\n");
+	uint64_t iterations = 0;
+
+	rte_atomic32_clear(&synchro);
+	count128.val[0] = 0;
+	count128.val[1] = 0;
+
+	rte_eal_mp_remote_launch(test_atomic128_cmp_exchange, NULL, SKIP_MASTER);
+	rte_atomic32_set(&synchro, 1);
+	rte_eal_mp_wait_lcore();
+	rte_atomic32_clear(&synchro);
+
+	iterations = count128.val[0] - count128.val[1];
+	if (iterations != 4*N*(rte_lcore_count()-1)) {
+		printf("128b compare and swap failed\n");
+		return -1;
+	}
+#endif
+
 	return 0;
 }
 
Signature · 1 line
2.7.4
author
#51 lore raw

PATCH v6 3/3 eal/stack: enable lock-free stack for aarch64

In reply to Phil Yang

Enable both c11 atomic and non c11 atomic lock-free stack for aarch64.

Introduced a new header to reduce the ifdef clutter across generic and c11 files. The rte_stack_lf_stubs.h contains stub implementations of __rte_stack_lf_count, __rte_stack_lf_push_elems and __rte_stack_lf_pop_elems.

---
 doc/guides/prog_guide/env_abstraction_layer.rst |  4 +--
 doc/guides/rel_notes/release_19_08.rst          |  3 ++
 lib/librte_stack/Makefile                       |  3 +-
 lib/librte_stack/rte_stack_lf.h                 |  4 +++
 lib/librte_stack/rte_stack_lf_c11.h             | 16 ---------
 lib/librte_stack/rte_stack_lf_generic.h         | 16 ---------
 lib/librte_stack/rte_stack_lf_stubs.h           | 44 +++++++++++++++++++++++++
 7 files changed, 55 insertions(+), 35 deletions(-)
 create mode 100644 lib/librte_stack/rte_stack_lf_stubs.h
doc/guides/prog_guide/env_abstraction_layer.rst +2 -2
index f15bcd9..d569f95 100644
@@ -592,8 +592,8 @@ Known Issues
   Alternatively, applications can use the lock-free stack mempool handler. When
   considering this handler, note that:
 
-  - It is currently limited to the x86_64 platform, because it uses an
-    instruction (16-byte compare-and-swap) that is not yet available on other
+  - It is currently limited to the aarch64 and x86_64 platforms, because it uses
+    an instruction (16-byte compare-and-swap) that is not yet available on other
     platforms.
   - It has worse average-case performance than the non-preemptive rte_ring, but
     software caching (e.g. the mempool cache) can mitigate this by reducing the
doc/guides/rel_notes/release_19_08.rst +3 -0
index 0a3f840..25d45c1 100644
@@ -212,6 +212,9 @@ New Features
 
   Added multiple cores feature to compression perf tool application.
 
+* **Added Lock-free Stack for aarch64.**
+
+  The lock-free stack implementation is enabled for aarch64 platforms.
 
 Removed Items
 -------------
lib/librte_stack/Makefile +2 -1
index 8d18ce5..c337ab7 100644
@@ -24,6 +24,7 @@ SYMLINK-$(CONFIG_RTE_LIBRTE_STACK)-include := rte_stack.h \
 					      rte_stack_std.h \
 					      rte_stack_lf.h \
 					      rte_stack_lf_generic.h \
-					      rte_stack_lf_c11.h
+					      rte_stack_lf_c11.h \
+						  rte_stack_lf_stubs.h
 
 include $(RTE_SDK)/mk/rte.lib.mk
lib/librte_stack/rte_stack_lf.h +4 -0
index f5581f0..e67630c 100644
@@ -5,11 +5,15 @@
 #ifndef _RTE_STACK_LF_H_
 #define _RTE_STACK_LF_H_
 
+#if !(defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64))
+#include "rte_stack_lf_stubs.h"
+#else
 #ifdef RTE_USE_C11_MEM_MODEL
 #include "rte_stack_lf_c11.h"
 #else
 #include "rte_stack_lf_generic.h"
 #endif
+#endif
 
 /**
  * @internal Push several objects on the lock-free stack (MT-safe).
lib/librte_stack/rte_stack_lf_c11.h +0 -16
index 3d677ae..999359f 100644
@@ -36,12 +36,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 			  struct rte_stack_lf_elem *last,
 			  unsigned int num)
 {
-#ifndef RTE_ARCH_X86_64
-	RTE_SET_USED(first);
-	RTE_SET_USED(last);
-	RTE_SET_USED(list);
-	RTE_SET_USED(num);
-#else
 	struct rte_stack_lf_head old_head;
 	int success;
 
@@ -79,7 +73,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 	 * to the LIFO len update.
 	 */
 	__atomic_add_fetch(&list->len, num, __ATOMIC_RELEASE);
-#endif
 }
 
 static __rte_always_inline struct rte_stack_lf_elem *
@@ -88,14 +81,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 void **obj_table,
 			 struct rte_stack_lf_elem **last)
 {
-#ifndef RTE_ARCH_X86_64
-	RTE_SET_USED(obj_table);
-	RTE_SET_USED(last);
-	RTE_SET_USED(list);
-	RTE_SET_USED(num);
-
-	return NULL;
-#else
 	struct rte_stack_lf_head old_head;
 	uint64_t len;
 	int success;
@@ -169,7 +154,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 	} while (success == 0);
 
 	return old_head.top;
-#endif
 }
 
 #endif /* _RTE_STACK_LF_C11_H_ */
lib/librte_stack/rte_stack_lf_generic.h +0 -16
index 3182151..3abbb53 100644
@@ -36,12 +36,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 			  struct rte_stack_lf_elem *last,
 			  unsigned int num)
 {
-#ifndef RTE_ARCH_X86_64
-	RTE_SET_USED(first);
-	RTE_SET_USED(last);
-	RTE_SET_USED(list);
-	RTE_SET_USED(num);
-#else
 	struct rte_stack_lf_head old_head;
 	int success;
 
@@ -75,7 +69,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 	} while (success == 0);
 
 	rte_atomic64_add((rte_atomic64_t *)&list->len, num);
-#endif
 }
 
 static __rte_always_inline struct rte_stack_lf_elem *
@@ -84,14 +77,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 void **obj_table,
 			 struct rte_stack_lf_elem **last)
 {
-#ifndef RTE_ARCH_X86_64
-	RTE_SET_USED(obj_table);
-	RTE_SET_USED(last);
-	RTE_SET_USED(list);
-	RTE_SET_USED(num);
-
-	return NULL;
-#else
 	struct rte_stack_lf_head old_head;
 	int success;
 
@@ -159,7 +144,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 	} while (success == 0);
 
 	return old_head.top;
-#endif
 }
 
 #endif /* _RTE_STACK_LF_GENERIC_H_ */
lib/librte_stack/rte_stack_lf_stubs.h new file +44 -0

44 lines not on this page · raw message

Signature · 1 line
2.7.4
Acked-by on v6 3/3
#52 lore raw

Re: [dpdk-dev] [EXT] [PATCH v6 3/3] eal/stack: enable lock-free stack for aarch64

Quoted 16 lines

-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Monday, July 22, 2019 9:53 PM
To: [email protected]
Cc: [email protected]; Jerin Jacob Kollanukkaran <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: [EXT] [PATCH v6 3/3] eal/stack: enable lock-free stack for aarch64

Enable both c11 atomic and non c11 atomic lock-free stack for aarch64.

Introduced a new header to reduce the ifdef clutter across generic and c11 files. The rte_stack_lf_stubs.h contains stub implementations of __rte_stack_lf_count, __rte_stack_lf_push_elems and __rte_stack_lf_pop_elems.

Quoted 5 lines
#53 lore raw
In reply to Phil Yang
Quoted 23 lines

-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Monday, July 22, 2019 9:53 PM
To: [email protected]
Cc: [email protected]; Jerin Jacob Kollanukkaran <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: [EXT] [PATCH v6 1/3] eal/arm64: add 128-bit atomic compare exchange

Add 128-bit atomic compare exchange on aarch64.

---

config/common_base index renamed from config/common_base +0 -0

8ef75c2..de6d1e0 100644

config/common_base CONFIG_RTE_MAX_LCORE=128 · line 84

Quoted 3 lines
config/common_base +0 -0
@@ -82,6 +82,8 @@ CONFIG_RTE_MAX_LCORE=128
 CONFIG_RTE_MAX_NUMA_NODES=8
 CONFIG_RTE_MAX_HEAPS=32
 CONFIG_RTE_MAX_MEMSEG_LISTS=64

Add new line here.

config/common_base CONFIG_RTE_MAX_LCORE=128 · line 85

Quoted 1 line
config/common_base +1 -0
+# Use LSE ATOMIC instructions

I think, you can change the comment to "Use ARM LSE ATOMIC instructions"

config/common_base CONFIG_RTE_MAX_LCORE=128 · line 86

Quoted 1 line
config/common_base +1 -0
+CONFIG_RTE_ARM_FEATURE_ATOMICS=n

This patches series has following check patch warning. Please fix it

With above fixes you can add my acked-by in 1/3 and 3/3 patches in next revision.

I think, you can ignore following warning. WARNING:MACRO_WITH_FLOW_CONTROL: Macros with flow control statements should be avoided

[master]dell[dpdk.org] $ ./devtools/checkpatches.sh

### eal/arm64: add 128-bit atomic compare exchange

WARNING:LONG_LINE: line over 80 characters
#103: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:60:
+ rte_int128_t updated) \

WARNING:LONG_LINE: line over 80 characters
#108: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:65:
+ */ \

WARNING:LINE_CONTINUATIONS: Avoid unnecessary line continuations #108: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:65:
+ */ \

WARNING:LONG_LINE: line over 80 characters
#109: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:66:
+ register uint64_t x0 __asm("x0") = (uint64_t)old.val[0]; \

WARNING:LONG_LINE: line over 80 characters
#110: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:67:
+ register uint64_t x1 __asm("x1") = (uint64_t)old.val[1]; \

WARNING:LONG_LINE: line over 80 characters
#111: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:68:
+ register uint64_t x2 __asm("x2") = (uint64_t)updated.val[0]; \

WARNING:LONG_LINE: line over 80 characters
#112: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:69:
+ register uint64_t x3 __asm("x3") = (uint64_t)updated.val[1]; \

WARNING:LONG_LINE: line over 80 characters
#113: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:70:
+ asm volatile( \

WARNING:LONG_LINE: line over 80 characters
#115: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:72:
+ : [old0] "+r" (x0), \

WARNING:LONG_LINE: line over 80 characters
#116: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:73:
+ [old1] "+r" (x1) \

WARNING:LONG_LINE: line over 80 characters
#118: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:75:
+ [upd1] "r" (x3), \

WARNING:LONG_LINE: line over 80 characters
#119: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:76:
+ [dst] "r" (dst) \

WARNING:LONG_LINE: line over 80 characters
#120: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:77:
+ : "memory"); \

WARNING:LONG_LINE: line over 80 characters
#121: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:78:
+ old.val[0] = x0; \

WARNING:LONG_LINE: line over 80 characters
#122: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:79:
+ old.val[1] = x1; \

WARNING:LONG_LINE: line over 80 characters
#123: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:80:
+ return old; \

WARNING:LONG_LINE: line over 80 characters
#135: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:92:
+ rte_int128_t ret; \

WARNING:LONG_LINE: line over 80 characters
#136: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:93:
+ asm volatile( \

WARNING:LONG_LINE: line over 80 characters
#138: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:95:
+ : "=&r" (ret.val[0]), \

WARNING:LONG_LINE: line over 80 characters
#139: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:96:
+ "=&r" (ret.val[1]) \

WARNING:LONG_LINE: line over 80 characters
#140: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:97:
+ : "Q" (src->val[0]) \

WARNING:LONG_LINE: line over 80 characters

#54 lore raw
Quoted 42 lines

-----Original Message-----
From: Jerin Jacob Kollanukkaran <[email protected]>
Sent: Tuesday, July 23, 2019 12:57 AM
To: Phil Yang (Arm Technology China) <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>
Subject: RE: [EXT] [PATCH v6 1/3] eal/arm64: add 128-bit atomic compare exchange

Quoted 5 lines

-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Monday, July 22, 2019 9:53 PM
To: [email protected]
Cc: [email protected]; Jerin Jacob Kollanukkaran

<[email protected]>;

Quoted 18 lines

[email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: [EXT] [PATCH v6 1/3] eal/arm64: add 128-bit atomic compare exchange

Add 128-bit atomic compare exchange on aarch64.

---

config/common_base index renamed from config/common_base +0 -0

8ef75c2..de6d1e0 100644

config/common_base CONFIG_RTE_MAX_LCORE=128 · line 84

Quoted 3 lines
config/common_base +0 -0
@@ -82,6 +82,8 @@ CONFIG_RTE_MAX_LCORE=128
 CONFIG_RTE_MAX_NUMA_NODES=8
 CONFIG_RTE_MAX_HEAPS=32
 CONFIG_RTE_MAX_MEMSEG_LISTS=64

Add new line here.

Quoted 3 lines

config/common_base CONFIG_RTE_MAX_LCORE=128 · line 85

Quoted 1 line
config/common_base +1 -0
+# Use LSE ATOMIC instructions

I think, you can change the comment to "Use ARM LSE ATOMIC instructions"

OK. Will fix it.

Quoted 7 lines

config/common_base CONFIG_RTE_MAX_LCORE=128 · line 86

Quoted 2 lines
config/common_base +1 -0
+CONFIG_RTE_ARM_FEATURE_ATOMICS=n

This patches series has following check patch warning. Please fix it

With above fixes you can add my acked-by in 1/3 and 3/3 patches in next revision.

Thank you for your review and your comments.

Quoted 7 lines

I think, you can ignore following warning. WARNING:MACRO_WITH_FLOW_CONTROL: Macros with flow control statements should be avoided

[master]dell[dpdk.org] $ ./devtools/checkpatches.sh

Yes. I did this check before upstream this patch.
I ignored ' WARNING:MACRO_WITH_FLOW_CONTROL: Macros with flow control statements should be avoided ' warnings. However, I didn't see any LONG_LINE warnings in the CI report here. http://mails.dpdk.org/archives/test-report/2019-July/090889.html
BTW, I think these 3 warnings in the report might be ineluctable.

Quoted 4 lines

### eal/arm64: add 128-bit atomic compare exchange

WARNING:LONG_LINE: line over 80 characters

The default tab size is 8. You might need to change it to 4 in the /usr/src/linux/scripts/checkpatch tool. FYI.

scripts/checkpatch.pl +3 -3
index a09333f..d3865a4 100755
@@ -1211,7 +1211,7 @@ sub expand_tabs {
         if ($c eq "\t") {
             $res .= ' ';
             $n++;
-            for (; ($n % 8) != 0; $n++) {
+            for (; ($n % 4) != 0; $n++) {
                 $res .= ' ';
             }
             next;
@@ -3688,11 +3688,11 @@ sub process {
             #print "line<$line> prevline<$prevline> indent<$indent> sindent<$sindent> check<$check> continuation<$continuation> s<$s> 
             cond_lines<$cond_lines> stat_real<$stat_real> stat<$stat>\n";
 
             if ($check && $s ne '' &&
-                (($sindent % 8) != 0 ||
+                (($sindent % 4) != 0 ||
                  ($sindent < $indent) ||
                  ($sindent == $indent &&
                   ($s !~ /^\s*(?:\}|\{|else\b)/)) ||
-                 ($sindent > $indent + 8))) {
+                 ($sindent > $indent + 4))) {
                 WARN("SUSPECT_CODE_INDENT",
                      "suspect code indent for conditional statements ($indent, $sindent)\n" . $herecurr . "$stat_real\n");
             }
Quoted 6 lines

#103: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:60:
+ rte_int128_t updated) \

WARNING:LONG_LINE: line over 80 characters
#108: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:65:
+ */ \

#55 lore raw
Quoted 7 lines
Quoted 2 lines

[master]dell[dpdk.org] $ ./devtools/checkpatches.sh

Yes. I did this check before upstream this patch.
I ignored ' WARNING:MACRO_WITH_FLOW_CONTROL: Macros with flow control statements should be avoided ' warnings. However, I didn't see any LONG_LINE warnings in the CI report here.

Yes. But Not sure why?

I sent v8 with "WARNING:LONG_LINE: line over 80 characters" warning fixes with latest kernel checkpatch.pl. Please review.

Quoted 2 lines

http://mails.dpdk.org/archives/test-report/2019-July/090889.html
BTW, I think these 3 warnings in the report might be ineluctable.

#56 lore raw
Quoted 12 lines

-----Original Message-----
From: Jerin Jacob Kollanukkaran <[email protected]>
Sent: Tuesday, July 23, 2019 3:10 PM
To: Phil Yang (Arm Technology China) <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>; nd <[email protected]>
Subject: RE: [EXT] [PATCH v6 1/3] eal/arm64: add 128-bit atomic compare exchange

Quoted 1 line
Quoted 1 line
Quoted 8 lines
Quoted 6 lines
Quoted 1 line

[master]dell[dpdk.org] $ ./devtools/checkpatches.sh

Yes. I did this check before upstream this patch.
I ignored ' WARNING:MACRO_WITH_FLOW_CONTROL: Macros with flow control statements should be avoided ' warnings. However, I didn't see any LONG_LINE warnings in the CI report here.

Yes. But Not sure why?

Not sure.
It might because DPDK CI changed tabstop width or indent width which is different than Kernel checkpatch.pl.

Quoted 4 lines

I sent v8 with "WARNING:LONG_LINE: line over 80 characters" warning fixes with latest kernel
checkpatch.pl. Please review.

+1
I really appreciate your help.

Quoted 3 lines
Quoted 3 lines

http://mails.dpdk.org/archives/test-report/2019-July/090889.html
BTW, I think these 3 warnings in the report might be ineluctable.

author
#57 lore raw

PATCH v7 1/3 eal/arm64: add 128-bit atomic compare exchange

In reply to Phil Yang

Add 128-bit atomic compare exchange on aarch64.

---
v7:
1. Adjust code comment.

v6:
1. Put the RTE_ARM_FEATURE_ATOMICS flag into EAL group. (Jerin Jocob)
2. Keep rte_stack_lf_stubs.h doing nothing. (Gage Eads)
3. Fixed 32 bit build issue.

v5:
1. Enable RTE_ARM_FEATURE_ATOMICS on octeontx2 in default. (Jerin Jocob)
2. Record the reason of introducing "rte_stack_lf_stubs.h" in git commit. (Jerin, Jocob)
3. Fixed a conditional MACRO error in rte_atomic128_cmp_exchange. (Jerin Jocob)

v4:
1. Add RTE_ARM_FEATURE_ATOMICS flag to support LSE CASP instructions. (Jerin Jocob)
2. Fix possible arm64 ABI break by making casp_op_name noinline. (Jerin Jocob)
3. Add rte_stack_lf_stubs.h to reduce the ifdef clutter. (Gage Eads/Jerin Jocob)

v3:
1. Avoid duplication code with macro. (Jerin Jocob)
2. Make invalid memory order to strongest barrier. (Jerin Jocob)
3. Update doc/guides/prog_guide/env_abstraction_layer.rst. (Gage Eads)
4. Fix 32-bit x86 builds issue. (Gage Eads)
5. Correct documentation issues in UT. (Gage Eads)

v2:
Initial version.

 config/arm/meson.build                             |   2 +
 config/common_base                                 |   3 +
 config/defconfig_arm64-octeontx2-linuxapp-gcc      |   1 +
 config/defconfig_arm64-thunderx2-linuxapp-gcc      |   1 +
 .../common/include/arch/arm/rte_atomic_64.h        | 162 +++++++++++++++++++++
 .../common/include/arch/x86/rte_atomic_64.h        |  12 --
 lib/librte_eal/common/include/generic/rte_atomic.h |  17 ++-
 7 files changed, 185 insertions(+), 13 deletions(-)
config/arm/meson.build +2 -0
index 979018e..9f28271 100644
@@ -71,11 +71,13 @@ flags_thunderx2_extra = [
 	['RTE_CACHE_LINE_SIZE', 64],
 	['RTE_MAX_NUMA_NODES', 2],
 	['RTE_MAX_LCORE', 256],
+	['RTE_ARM_FEATURE_ATOMICS', true],
 	['RTE_USE_C11_MEM_MODEL', true]]
 flags_octeontx2_extra = [
 	['RTE_MACHINE', '"octeontx2"'],
 	['RTE_MAX_NUMA_NODES', 1],
 	['RTE_MAX_LCORE', 24],
+	['RTE_ARM_FEATURE_ATOMICS', true],
 	['RTE_EAL_IGB_UIO', false],
 	['RTE_USE_C11_MEM_MODEL', true]]
 
config/common_base +3 -0
index 8ef75c2..2054480 100644
@@ -82,6 +82,9 @@ CONFIG_RTE_MAX_LCORE=128
 CONFIG_RTE_MAX_NUMA_NODES=8
 CONFIG_RTE_MAX_HEAPS=32
 CONFIG_RTE_MAX_MEMSEG_LISTS=64
+
+# Use ARM LSE ATOMIC instructions
+CONFIG_RTE_ARM_FEATURE_ATOMICS=n
 # each memseg list will be limited to either RTE_MAX_MEMSEG_PER_LIST pages
 # or RTE_MAX_MEM_MB_PER_LIST megabytes worth of memory, whichever is smaller
 CONFIG_RTE_MAX_MEMSEG_PER_LIST=8192
config/defconfig_arm64-octeontx2-linuxapp-gcc +1 -0
index f20da24..7687dbe 100644
@@ -9,6 +9,7 @@ CONFIG_RTE_MACHINE="octeontx2"
 CONFIG_RTE_CACHE_LINE_SIZE=128
 CONFIG_RTE_MAX_NUMA_NODES=1
 CONFIG_RTE_MAX_LCORE=24
+CONFIG_RTE_ARM_FEATURE_ATOMICS=y
 
 # Doesn't support NUMA
 CONFIG_RTE_EAL_NUMA_AWARE_HUGEPAGES=n
config/defconfig_arm64-thunderx2-linuxapp-gcc +1 -0
index cc5c64b..af4a89c 100644
@@ -9,3 +9,4 @@ CONFIG_RTE_MACHINE="thunderx2"
 CONFIG_RTE_CACHE_LINE_SIZE=64
 CONFIG_RTE_MAX_NUMA_NODES=2
 CONFIG_RTE_MAX_LCORE=256
+CONFIG_RTE_ARM_FEATURE_ATOMICS=y
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +162 -0

179 lines not on this page · raw message

lib/librte_eal/common/include/arch/x86/rte_atomic_64.h +0 -12
index e087c6c..1217129 100644
@@ -212,18 +212,6 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)
 
 /*------------------------ 128 bit atomic operations -------------------------*/
 
-/**
- * 128-bit integer structure.
- */
-RTE_STD_C11
-typedef struct {
-	RTE_STD_C11
-	union {
-		uint64_t val[2];
-		__extension__ __int128 int128;
-	};
-} __rte_aligned(16) rte_int128_t;
-
 __rte_experimental
 static inline int
 rte_atomic128_cmp_exchange(rte_int128_t *dst,
lib/librte_eal/common/include/generic/rte_atomic.h +16 -1
index 24ff7dc..e6ab15a 100644
@@ -1081,6 +1081,20 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)
 
 /*------------------------ 128 bit atomic operations -------------------------*/
 
+/**
+ * 128-bit integer structure.
+ */
+RTE_STD_C11
+typedef struct {
+	RTE_STD_C11
+	union {
+		uint64_t val[2];
+#ifdef RTE_ARCH_64
+		__extension__ __int128 int128;
+#endif
+	};
+} __rte_aligned(16) rte_int128_t;
+
 #ifdef __DOXYGEN__
 
 /**
@@ -1093,7 +1107,8 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)
  *     *exp = *dst
  * @endcode
  *
- * @note This function is currently only available for the x86-64 platform.
+ * @note This function is currently available for the x86-64 and aarch64
+ * platforms.
  *
  * @note The success and failure arguments must be one of the __ATOMIC_* values
  * defined in the C++11 standard. For details on their behavior, refer to the
Signature · 1 line
2.7.4
author
#58 lore raw

PATCH v7 2/3 test/atomic: add 128b compare and swap test

Add 128b atomic compare and swap test for aarch64 and x86_64.

---
 app/test/test_atomic.c | 120 ++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 118 insertions(+), 2 deletions(-)
app/test/test_atomic.c +118 -2
index 43be30e..ff6ff88 100644
@@ -1,5 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright(c) 2010-2014 Intel Corporation
+ * Copyright(c) 2019 Arm Limited
  */
 
 #include <stdio.h>
@@ -20,7 +21,7 @@
  * Atomic Variables
  * ================
  *
- * - The main test function performs three subtests. The first test
+ * - The main test function performs four subtests. The first test
  *   checks that the usual inc/dec/add/sub functions are working
  *   correctly:
  *
@@ -61,11 +62,27 @@
  *       atomic_sub(&count, tmp+1);
  *
  *   - At the end of the test, the *count* value must be 0.
+ *
+ * - Test "128b compare and swap" (aarch64 and x86_64 only)
+ *
+ *   - Initialize 128-bit atomic variables to zero.
+ *
+ *   - Invoke ``test_atomici128_cmp_exchange()`` on each lcore. Before doing
+ *     anything else, the cores are waiting a synchro. Each lcore does
+ *     these compare and swap (CAS) operations several times::
+ *
+ *       Acquired CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *       Released CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *       Acquired_Released CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *       Relaxed CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *
+ *   - At the end of the test, the *count128* first 64-bit value and
+ *     second 64-bit value differ by the total iterations.
  */
 
 #define NUM_ATOMIC_TYPES 3
 
-#define N 10000
+#define N 1000000
 
 static rte_atomic16_t a16;
 static rte_atomic32_t a32;
@@ -216,6 +233,74 @@ test_atomic_dec_and_test(__attribute__((unused)) void *arg)
 	return 0;
 }
 
+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)
+static rte_int128_t count128;
+
+/*
+ * rte_atomic128_cmp_exchange() should update a 128 bits counter's first 64
+ * bits by 2 and the second 64 bits by 1 in this test. It should return true
+ * if the compare exchange operation is successful.
+ * This test repeats 128 bits compare and swap operations 10K rounds. In each
+ * iteration it runs compare and swap operation with different memory models.
+ */
+static int
+test_atomic128_cmp_exchange(__attribute__((unused)) void *arg)
+{
+	rte_int128_t expected;
+	int success;
+	unsigned int i;
+
+	while (rte_atomic32_read(&synchro) == 0)
+		;
+
+	expected = count128;
+
+	for (i = 0; i < N; i++) {
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128, &expected,
+					&desired, 1, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
+		} while (success == 0);
+
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128, &expected,
+					&desired, 1, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
+		} while (success == 0);
+
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128, &expected,
+					&desired, 1, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED);
+		} while (success == 0);
+
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128, &expected,
+					&desired, 1, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+		} while (success == 0);
+	}
+
+	return 0;
+}
+#endif
+
 static int
 test_atomic(void)
 {
@@ -340,6 +425,37 @@ test_atomic(void)
 		return -1;
 	}
 
+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)
+	/*
+	 * This case tests the functionality of rte_atomic128b_cmp_exchange
+	 * API. It calls rte_atomic128b_cmp_exchange with four kinds of memory
+	 * models successively on each slave core. Once each 128-bit atomic
+	 * compare and swap operation is successful, it updates the global
+	 * 128-bit counter by 2 for the first 64-bit and 1 for the second
+	 * 64-bit. Each slave core iterates this test 10K times.
+	 * At the end of test, verify whether the first 64-bits of the 128-bit
+	 * counter and the second 64bits is differ by the total iterations. If
+	 * it is, the test passes.
+	 */
+	printf("128b compare and swap test\n");
+	uint64_t iterations = 0;
+
+	rte_atomic32_clear(&synchro);
+	count128.val[0] = 0;
+	count128.val[1] = 0;
+
+	rte_eal_mp_remote_launch(test_atomic128_cmp_exchange, NULL, SKIP_MASTER);
+	rte_atomic32_set(&synchro, 1);
+	rte_eal_mp_wait_lcore();
+	rte_atomic32_clear(&synchro);
+
+	iterations = count128.val[0] - count128.val[1];
+	if (iterations != 4*N*(rte_lcore_count()-1)) {
+		printf("128b compare and swap failed\n");
+		return -1;
+	}
+#endif
+
 	return 0;
 }
 
Signature · 1 line
2.7.4
author
#59 lore raw

PATCH v7 3/3 eal/stack: enable lock-free stack for aarch64

In reply to Phil Yang

Enable both c11 atomic and non c11 atomic lock-free stack for aarch64.

Introduced a new header to reduce the ifdef clutter across generic and c11 files. The rte_stack_lf_stubs.h contains stub implementations of __rte_stack_lf_count, __rte_stack_lf_push_elems and __rte_stack_lf_pop_elems.

---
 doc/guides/prog_guide/env_abstraction_layer.rst |  4 +--
 doc/guides/rel_notes/release_19_08.rst          |  3 ++
 lib/librte_stack/Makefile                       |  3 +-
 lib/librte_stack/rte_stack_lf.h                 |  4 +++
 lib/librte_stack/rte_stack_lf_c11.h             | 16 ---------
 lib/librte_stack/rte_stack_lf_generic.h         | 16 ---------
 lib/librte_stack/rte_stack_lf_stubs.h           | 44 +++++++++++++++++++++++++
 7 files changed, 55 insertions(+), 35 deletions(-)
 create mode 100644 lib/librte_stack/rte_stack_lf_stubs.h
doc/guides/prog_guide/env_abstraction_layer.rst +2 -2
index 1487ea5..2335367 100644
@@ -641,8 +641,8 @@ Known Issues
   Alternatively, applications can use the lock-free stack mempool handler. When
   considering this handler, note that:
 
-  - It is currently limited to the x86_64 platform, because it uses an
-    instruction (16-byte compare-and-swap) that is not yet available on other
+  - It is currently limited to the aarch64 and x86_64 platforms, because it uses
+    an instruction (16-byte compare-and-swap) that is not yet available on other
     platforms.
   - It has worse average-case performance than the non-preemptive rte_ring, but
     software caching (e.g. the mempool cache) can mitigate this by reducing the
doc/guides/rel_notes/release_19_08.rst +3 -0
index 50c83ce..55c792a 100644
@@ -212,6 +212,9 @@ New Features
 
   Added multiple cores feature to compression perf tool application.
 
+* **Added Lock-free Stack for aarch64.**
+
+  The lock-free stack implementation is enabled for aarch64 platforms.
 
 Removed Items
 -------------
lib/librte_stack/Makefile +2 -1
index 8d18ce5..c337ab7 100644
@@ -24,6 +24,7 @@ SYMLINK-$(CONFIG_RTE_LIBRTE_STACK)-include := rte_stack.h \
 					      rte_stack_std.h \
 					      rte_stack_lf.h \
 					      rte_stack_lf_generic.h \
-					      rte_stack_lf_c11.h
+					      rte_stack_lf_c11.h \
+						  rte_stack_lf_stubs.h
 
 include $(RTE_SDK)/mk/rte.lib.mk
lib/librte_stack/rte_stack_lf.h +4 -0
index f5581f0..e67630c 100644
@@ -5,11 +5,15 @@
 #ifndef _RTE_STACK_LF_H_
 #define _RTE_STACK_LF_H_
 
+#if !(defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64))
+#include "rte_stack_lf_stubs.h"
+#else
 #ifdef RTE_USE_C11_MEM_MODEL
 #include "rte_stack_lf_c11.h"
 #else
 #include "rte_stack_lf_generic.h"
 #endif
+#endif
 
 /**
  * @internal Push several objects on the lock-free stack (MT-safe).
lib/librte_stack/rte_stack_lf_c11.h +0 -16
index 3d677ae..999359f 100644
@@ -36,12 +36,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 			  struct rte_stack_lf_elem *last,
 			  unsigned int num)
 {
-#ifndef RTE_ARCH_X86_64
-	RTE_SET_USED(first);
-	RTE_SET_USED(last);
-	RTE_SET_USED(list);
-	RTE_SET_USED(num);
-#else
 	struct rte_stack_lf_head old_head;
 	int success;
 
@@ -79,7 +73,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 	 * to the LIFO len update.
 	 */
 	__atomic_add_fetch(&list->len, num, __ATOMIC_RELEASE);
-#endif
 }
 
 static __rte_always_inline struct rte_stack_lf_elem *
@@ -88,14 +81,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 void **obj_table,
 			 struct rte_stack_lf_elem **last)
 {
-#ifndef RTE_ARCH_X86_64
-	RTE_SET_USED(obj_table);
-	RTE_SET_USED(last);
-	RTE_SET_USED(list);
-	RTE_SET_USED(num);
-
-	return NULL;
-#else
 	struct rte_stack_lf_head old_head;
 	uint64_t len;
 	int success;
@@ -169,7 +154,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 	} while (success == 0);
 
 	return old_head.top;
-#endif
 }
 
 #endif /* _RTE_STACK_LF_C11_H_ */
lib/librte_stack/rte_stack_lf_generic.h +0 -16
index 3182151..3abbb53 100644
@@ -36,12 +36,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 			  struct rte_stack_lf_elem *last,
 			  unsigned int num)
 {
-#ifndef RTE_ARCH_X86_64
-	RTE_SET_USED(first);
-	RTE_SET_USED(last);
-	RTE_SET_USED(list);
-	RTE_SET_USED(num);
-#else
 	struct rte_stack_lf_head old_head;
 	int success;
 
@@ -75,7 +69,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 	} while (success == 0);
 
 	rte_atomic64_add((rte_atomic64_t *)&list->len, num);
-#endif
 }
 
 static __rte_always_inline struct rte_stack_lf_elem *
@@ -84,14 +77,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 void **obj_table,
 			 struct rte_stack_lf_elem **last)
 {
-#ifndef RTE_ARCH_X86_64
-	RTE_SET_USED(obj_table);
-	RTE_SET_USED(last);
-	RTE_SET_USED(list);
-	RTE_SET_USED(num);
-
-	return NULL;
-#else
 	struct rte_stack_lf_head old_head;
 	int success;
 
@@ -159,7 +144,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 	} while (success == 0);
 
 	return old_head.top;
-#endif
 }
 
 #endif /* _RTE_STACK_LF_GENERIC_H_ */
lib/librte_stack/rte_stack_lf_stubs.h new file +44 -0

44 lines not on this page · raw message

Signature · 1 line
2.7.4
#60 lore raw

PATCH v8 1/3 eal/arm64: add 128-bit atomic compare exchange

In reply to Phil Yang

From: Phil Yang <[email protected]>

Add 128-bit atomic compare exchange on aarch64.

---

v8:
Fixed "WARNING:LONG_LINE: line over 80 characters" warnings with latest kernel checkpatch.pl

v7:
1. Adjust code comment.

v6:
1. Put the RTE_ARM_FEATURE_ATOMICS flag into EAL group. (Jerin Jocob)
2. Keep rte_stack_lf_stubs.h doing nothing. (Gage Eads)
3. Fixed 32 bit build issue.

v5:
1. Enable RTE_ARM_FEATURE_ATOMICS on octeontx2 in default. (Jerin Jocob)
2. Record the reason of introducing "rte_stack_lf_stubs.h" in git commit.
(Jerin, Jocob)
3. Fixed a conditional MACRO error in rte_atomic128_cmp_exchange. (Jerin Jocob)

v4:
1. Add RTE_ARM_FEATURE_ATOMICS flag to support LSE CASP instructions. (Jerin Jocob)
2. Fix possible arm64 ABI break by making casp_op_name noinline. (Jerin Jocob)
3. Add rte_stack_lf_stubs.h to reduce the ifdef clutter. (Gage Eads/Jerin Jocob)

v3:
1. Avoid duplication code with macro. (Jerin Jocob)
2. Make invalid memory order to strongest barrier. (Jerin Jocob)
3. Update doc/guides/prog_guide/env_abstraction_layer.rst. (Gage Eads)
4. Fix 32-bit x86 builds issue. (Gage Eads)
5. Correct documentation issues in UT. (Gage Eads)

v2:
Initial version.

 config/arm/meson.build                        |   2 +
 config/common_base                            |   3 +
 config/defconfig_arm64-octeontx2-linuxapp-gcc |   1 +
 config/defconfig_arm64-thunderx2-linuxapp-gcc |   1 +
 .../common/include/arch/arm/rte_atomic_64.h   | 163 ++++++++++++++++++
 .../common/include/arch/x86/rte_atomic_64.h   |  12 --
 .../common/include/generic/rte_atomic.h       |  17 +-
 7 files changed, 186 insertions(+), 13 deletions(-)
config/arm/meson.build +2 -0
index 979018e16..9f2827140 100644
@@ -71,11 +71,13 @@ flags_thunderx2_extra = [
 	['RTE_CACHE_LINE_SIZE', 64],
 	['RTE_MAX_NUMA_NODES', 2],
 	['RTE_MAX_LCORE', 256],
+	['RTE_ARM_FEATURE_ATOMICS', true],
 	['RTE_USE_C11_MEM_MODEL', true]]
 flags_octeontx2_extra = [
 	['RTE_MACHINE', '"octeontx2"'],
 	['RTE_MAX_NUMA_NODES', 1],
 	['RTE_MAX_LCORE', 24],
+	['RTE_ARM_FEATURE_ATOMICS', true],
 	['RTE_EAL_IGB_UIO', false],
 	['RTE_USE_C11_MEM_MODEL', true]]
 
config/common_base +3 -0
index 8ef75c203..205448013 100644
@@ -82,6 +82,9 @@ CONFIG_RTE_MAX_LCORE=128
 CONFIG_RTE_MAX_NUMA_NODES=8
 CONFIG_RTE_MAX_HEAPS=32
 CONFIG_RTE_MAX_MEMSEG_LISTS=64
+
+# Use ARM LSE ATOMIC instructions
+CONFIG_RTE_ARM_FEATURE_ATOMICS=n
 # each memseg list will be limited to either RTE_MAX_MEMSEG_PER_LIST pages
 # or RTE_MAX_MEM_MB_PER_LIST megabytes worth of memory, whichever is smaller
 CONFIG_RTE_MAX_MEMSEG_PER_LIST=8192
config/defconfig_arm64-octeontx2-linuxapp-gcc +1 -0
index f20da2442..7687dbec8 100644
@@ -9,6 +9,7 @@ CONFIG_RTE_MACHINE="octeontx2"
 CONFIG_RTE_CACHE_LINE_SIZE=128
 CONFIG_RTE_MAX_NUMA_NODES=1
 CONFIG_RTE_MAX_LCORE=24
+CONFIG_RTE_ARM_FEATURE_ATOMICS=y
 
 # Doesn't support NUMA
 CONFIG_RTE_EAL_NUMA_AWARE_HUGEPAGES=n
config/defconfig_arm64-thunderx2-linuxapp-gcc +1 -0
index cc5c64ba0..af4a89c48 100644
@@ -9,3 +9,4 @@ CONFIG_RTE_MACHINE="thunderx2"
 CONFIG_RTE_CACHE_LINE_SIZE=64
 CONFIG_RTE_MAX_NUMA_NODES=2
 CONFIG_RTE_MAX_LCORE=256
+CONFIG_RTE_ARM_FEATURE_ATOMICS=y
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +163 -0

180 lines not on this page · raw message

lib/librte_eal/common/include/arch/x86/rte_atomic_64.h +0 -12
index e087c6c32..12171296b 100644
@@ -212,18 +212,6 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)
 
 /*------------------------ 128 bit atomic operations -------------------------*/
 
-/**
- * 128-bit integer structure.
- */
-RTE_STD_C11
-typedef struct {
-	RTE_STD_C11
-	union {
-		uint64_t val[2];
-		__extension__ __int128 int128;
-	};
-} __rte_aligned(16) rte_int128_t;
-
 __rte_experimental
 static inline int
 rte_atomic128_cmp_exchange(rte_int128_t *dst,
lib/librte_eal/common/include/generic/rte_atomic.h +16 -1
index 24ff7dcae..e6ab15a97 100644
@@ -1081,6 +1081,20 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)
 
 /*------------------------ 128 bit atomic operations -------------------------*/
 
+/**
+ * 128-bit integer structure.
+ */
+RTE_STD_C11
+typedef struct {
+	RTE_STD_C11
+	union {
+		uint64_t val[2];
+#ifdef RTE_ARCH_64
+		__extension__ __int128 int128;
+#endif
+	};
+} __rte_aligned(16) rte_int128_t;
+
 #ifdef __DOXYGEN__
 
 /**
@@ -1093,7 +1107,8 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)
  *     *exp = *dst
  * @endcode
  *
- * @note This function is currently only available for the x86-64 platform.
+ * @note This function is currently available for the x86-64 and aarch64
+ * platforms.
  *
  * @note The success and failure arguments must be one of the __ATOMIC_* values
  * defined in the C++11 standard. For details on their behavior, refer to the
Signature · 1 line
2.22.0
#61 lore raw

PATCH v8 2/3 test/atomic: add 128b compare and swap test

From: Phil Yang <[email protected]>

Add 128b atomic compare and swap test for aarch64 and x86_64.

---
 app/test/test_atomic.c | 125 ++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 123 insertions(+), 2 deletions(-)
app/test/test_atomic.c +123 -2
index 43be30ec0..0dad92387 100644
@@ -1,5 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright(c) 2010-2014 Intel Corporation
+ * Copyright(c) 2019 Arm Limited
  */
 
 #include <stdio.h>
@@ -20,7 +21,7 @@
  * Atomic Variables
  * ================
  *
- * - The main test function performs three subtests. The first test
+ * - The main test function performs four subtests. The first test
  *   checks that the usual inc/dec/add/sub functions are working
  *   correctly:
  *
@@ -61,11 +62,27 @@
  *       atomic_sub(&count, tmp+1);
  *
  *   - At the end of the test, the *count* value must be 0.
+ *
+ * - Test "128b compare and swap" (aarch64 and x86_64 only)
+ *
+ *   - Initialize 128-bit atomic variables to zero.
+ *
+ *   - Invoke ``test_atomici128_cmp_exchange()`` on each lcore. Before doing
+ *     anything else, the cores are waiting a synchro. Each lcore does
+ *     these compare and swap (CAS) operations several times::
+ *
+ *       Acquired CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *       Released CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *       Acquired_Released CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *       Relaxed CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *
+ *   - At the end of the test, the *count128* first 64-bit value and
+ *     second 64-bit value differ by the total iterations.
  */
 
 #define NUM_ATOMIC_TYPES 3
 
-#define N 10000
+#define N 1000000
 
 static rte_atomic16_t a16;
 static rte_atomic32_t a32;
@@ -216,6 +233,78 @@ test_atomic_dec_and_test(__attribute__((unused)) void *arg)
 	return 0;
 }
 
+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)
+static rte_int128_t count128;
+
+/*
+ * rte_atomic128_cmp_exchange() should update a 128 bits counter's first 64
+ * bits by 2 and the second 64 bits by 1 in this test. It should return true
+ * if the compare exchange operation is successful.
+ * This test repeats 128 bits compare and swap operations 10K rounds. In each
+ * iteration it runs compare and swap operation with different memory models.
+ */
+static int
+test_atomic128_cmp_exchange(__attribute__((unused)) void *arg)
+{
+	rte_int128_t expected;
+	int success;
+	unsigned int i;
+
+	while (rte_atomic32_read(&synchro) == 0)
+		;
+
+	expected = count128;
+
+	for (i = 0; i < N; i++) {
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128,
+				&expected, &desired, 1,
+				__ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
+		} while (success == 0);
+
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128,
+					&expected, &desired, 1,
+					__ATOMIC_RELEASE, __ATOMIC_RELAXED);
+		} while (success == 0);
+
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128,
+					&expected, &desired, 1,
+					__ATOMIC_ACQ_REL, __ATOMIC_RELAXED);
+		} while (success == 0);
+
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128,
+					&expected, &desired, 1,
+					__ATOMIC_RELAXED, __ATOMIC_RELAXED);
+		} while (success == 0);
+	}
+
+	return 0;
+}
+#endif
+
 static int
 test_atomic(void)
 {
@@ -340,6 +429,38 @@ test_atomic(void)
 		return -1;
 	}
 
+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)
+	/*
+	 * This case tests the functionality of rte_atomic128b_cmp_exchange
+	 * API. It calls rte_atomic128b_cmp_exchange with four kinds of memory
+	 * models successively on each slave core. Once each 128-bit atomic
+	 * compare and swap operation is successful, it updates the global
+	 * 128-bit counter by 2 for the first 64-bit and 1 for the second
+	 * 64-bit. Each slave core iterates this test 10K times.
+	 * At the end of test, verify whether the first 64-bits of the 128-bit
+	 * counter and the second 64bits is differ by the total iterations. If
+	 * it is, the test passes.
+	 */
+	printf("128b compare and swap test\n");
+	uint64_t iterations = 0;
+
+	rte_atomic32_clear(&synchro);
+	count128.val[0] = 0;
+	count128.val[1] = 0;
+
+	rte_eal_mp_remote_launch(test_atomic128_cmp_exchange, NULL,
+				 SKIP_MASTER);
+	rte_atomic32_set(&synchro, 1);
+	rte_eal_mp_wait_lcore();
+	rte_atomic32_clear(&synchro);
+
+	iterations = count128.val[0] - count128.val[1];
+	if (iterations != 4*N*(rte_lcore_count()-1)) {
+		printf("128b compare and swap failed\n");
+		return -1;
+	}
+#endif
+
 	return 0;
 }
 
Signature · 1 line
2.22.0
#62 lore raw

PATCH v8 3/3 eal/stack: enable lock-free stack for aarch64

In reply to [email protected]

From: Phil Yang <[email protected]>

Enable both c11 atomic and non c11 atomic lock-free stack for aarch64.

Introduced a new header to reduce the ifdef clutter across generic and c11 files. The rte_stack_lf_stubs.h contains stub implementations of __rte_stack_lf_count, __rte_stack_lf_push_elems and __rte_stack_lf_pop_elems.

---
 .../prog_guide/env_abstraction_layer.rst      |  4 +-
 doc/guides/rel_notes/release_19_08.rst        |  3 ++
 lib/librte_stack/Makefile                     |  3 +-
 lib/librte_stack/rte_stack_lf.h               |  4 ++
 lib/librte_stack/rte_stack_lf_c11.h           | 16 -------
 lib/librte_stack/rte_stack_lf_generic.h       | 16 -------
 lib/librte_stack/rte_stack_lf_stubs.h         | 44 +++++++++++++++++++
 7 files changed, 55 insertions(+), 35 deletions(-)
 create mode 100644 lib/librte_stack/rte_stack_lf_stubs.h
doc/guides/prog_guide/env_abstraction_layer.rst +2 -2
index 1487ea550..23353678d 100644
@@ -641,8 +641,8 @@ Known Issues
   Alternatively, applications can use the lock-free stack mempool handler. When
   considering this handler, note that:
 
-  - It is currently limited to the x86_64 platform, because it uses an
-    instruction (16-byte compare-and-swap) that is not yet available on other
+  - It is currently limited to the aarch64 and x86_64 platforms, because it uses
+    an instruction (16-byte compare-and-swap) that is not yet available on other
     platforms.
   - It has worse average-case performance than the non-preemptive rte_ring, but
     software caching (e.g. the mempool cache) can mitigate this by reducing the
doc/guides/rel_notes/release_19_08.rst +3 -0
index 50c83ce51..55c792af1 100644
@@ -212,6 +212,9 @@ New Features
 
   Added multiple cores feature to compression perf tool application.
 
+* **Added Lock-free Stack for aarch64.**
+
+  The lock-free stack implementation is enabled for aarch64 platforms.
 
 Removed Items
 -------------
lib/librte_stack/Makefile +2 -1
index 8d18ce520..c337ab767 100644
@@ -24,6 +24,7 @@ SYMLINK-$(CONFIG_RTE_LIBRTE_STACK)-include := rte_stack.h \
 					      rte_stack_std.h \
 					      rte_stack_lf.h \
 					      rte_stack_lf_generic.h \
-					      rte_stack_lf_c11.h
+					      rte_stack_lf_c11.h \
+						  rte_stack_lf_stubs.h
 
 include $(RTE_SDK)/mk/rte.lib.mk
lib/librte_stack/rte_stack_lf.h +4 -0
index f5581f0c2..e67630c27 100644
@@ -5,11 +5,15 @@
 #ifndef _RTE_STACK_LF_H_
 #define _RTE_STACK_LF_H_
 
+#if !(defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64))
+#include "rte_stack_lf_stubs.h"
+#else
 #ifdef RTE_USE_C11_MEM_MODEL
 #include "rte_stack_lf_c11.h"
 #else
 #include "rte_stack_lf_generic.h"
 #endif
+#endif
 
 /**
  * @internal Push several objects on the lock-free stack (MT-safe).
lib/librte_stack/rte_stack_lf_c11.h +0 -16
index 3d677aed1..999359f08 100644
@@ -36,12 +36,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 			  struct rte_stack_lf_elem *last,
 			  unsigned int num)
 {
-#ifndef RTE_ARCH_X86_64
-	RTE_SET_USED(first);
-	RTE_SET_USED(last);
-	RTE_SET_USED(list);
-	RTE_SET_USED(num);
-#else
 	struct rte_stack_lf_head old_head;
 	int success;
 
@@ -79,7 +73,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 	 * to the LIFO len update.
 	 */
 	__atomic_add_fetch(&list->len, num, __ATOMIC_RELEASE);
-#endif
 }
 
 static __rte_always_inline struct rte_stack_lf_elem *
@@ -88,14 +81,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 void **obj_table,
 			 struct rte_stack_lf_elem **last)
 {
-#ifndef RTE_ARCH_X86_64
-	RTE_SET_USED(obj_table);
-	RTE_SET_USED(last);
-	RTE_SET_USED(list);
-	RTE_SET_USED(num);
-
-	return NULL;
-#else
 	struct rte_stack_lf_head old_head;
 	uint64_t len;
 	int success;
@@ -169,7 +154,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 	} while (success == 0);
 
 	return old_head.top;
-#endif
 }
 
 #endif /* _RTE_STACK_LF_C11_H_ */
lib/librte_stack/rte_stack_lf_generic.h +0 -16
index 31821514f..3abbb5342 100644
@@ -36,12 +36,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 			  struct rte_stack_lf_elem *last,
 			  unsigned int num)
 {
-#ifndef RTE_ARCH_X86_64
-	RTE_SET_USED(first);
-	RTE_SET_USED(last);
-	RTE_SET_USED(list);
-	RTE_SET_USED(num);
-#else
 	struct rte_stack_lf_head old_head;
 	int success;
 
@@ -75,7 +69,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 	} while (success == 0);
 
 	rte_atomic64_add((rte_atomic64_t *)&list->len, num);
-#endif
 }
 
 static __rte_always_inline struct rte_stack_lf_elem *
@@ -84,14 +77,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 void **obj_table,
 			 struct rte_stack_lf_elem **last)
 {
-#ifndef RTE_ARCH_X86_64
-	RTE_SET_USED(obj_table);
-	RTE_SET_USED(last);
-	RTE_SET_USED(list);
-	RTE_SET_USED(num);
-
-	return NULL;
-#else
 	struct rte_stack_lf_head old_head;
 	int success;
 
@@ -159,7 +144,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 	} while (success == 0);
 
 	return old_head.top;
-#endif
 }
 
 #endif /* _RTE_STACK_LF_GENERIC_H_ */
lib/librte_stack/rte_stack_lf_stubs.h new file +44 -0

44 lines not on this page · raw message

Signature · 1 line
2.22.0
author
#63 lore raw

PATCH v9 1/3 eal/arm64: add 128-bit atomic compare exchange

In reply to [email protected]

Add 128-bit atomic compare exchange on aarch64.

---

v9:
Updated 19.11 release note.

v8:
Fixed "WARNING:LONG_LINE: line over 80 characters" warnings with latest kernel checkpatch.pl

v7:
1. Adjust code comment.

v6:
1. Put the RTE_ARM_FEATURE_ATOMICS flag into EAL group. (Jerin Jocob)
2. Keep rte_stack_lf_stubs.h doing nothing. (Gage Eads)
3. Fixed 32 bit build issue.

v5:
1. Enable RTE_ARM_FEATURE_ATOMICS on octeontx2 in default. (Jerin Jocob)
2. Record the reason of introducing "rte_stack_lf_stubs.h" in git commit.
(Jerin, Jocob)
3. Fixed a conditional MACRO error in rte_atomic128_cmp_exchange. (Jerin Jocob)

v4:
1. Add RTE_ARM_FEATURE_ATOMICS flag to support LSE CASP instructions. (Jerin Jocob)
2. Fix possible arm64 ABI break by making casp_op_name noinline. (Jerin Jocob)
3. Add rte_stack_lf_stubs.h to reduce the ifdef clutter. (Gage Eads/Jerin Jocob)

v3:
1. Avoid duplication code with macro. (Jerin Jocob)
2. Make invalid memory order to strongest barrier. (Jerin Jocob)
3. Update doc/guides/prog_guide/env_abstraction_layer.rst. (Gage Eads)
4. Fix 32-bit x86 builds issue. (Gage Eads)
5. Correct documentation issues in UT. (Gage Eads)

v2:
Initial version.

 config/arm/meson.build                             |   2 +
 config/common_base                                 |   3 +
 config/defconfig_arm64-octeontx2-linuxapp-gcc      |   1 +
 config/defconfig_arm64-thunderx2-linuxapp-gcc      |   1 +
 .../common/include/arch/arm/rte_atomic_64.h        | 163 +++++++++++++++++++++
 .../common/include/arch/x86/rte_atomic_64.h        |  12 --
 lib/librte_eal/common/include/generic/rte_atomic.h |  17 ++-
 7 files changed, 186 insertions(+), 13 deletions(-)
config/arm/meson.build +2 -0
index 979018e..9f28271 100644
@@ -71,11 +71,13 @@ flags_thunderx2_extra = [
 	['RTE_CACHE_LINE_SIZE', 64],
 	['RTE_MAX_NUMA_NODES', 2],
 	['RTE_MAX_LCORE', 256],
+	['RTE_ARM_FEATURE_ATOMICS', true],
 	['RTE_USE_C11_MEM_MODEL', true]]
 flags_octeontx2_extra = [
 	['RTE_MACHINE', '"octeontx2"'],
 	['RTE_MAX_NUMA_NODES', 1],
 	['RTE_MAX_LCORE', 24],
+	['RTE_ARM_FEATURE_ATOMICS', true],
 	['RTE_EAL_IGB_UIO', false],
 	['RTE_USE_C11_MEM_MODEL', true]]
 
config/common_base +3 -0
index 8ef75c2..2054480 100644
@@ -82,6 +82,9 @@ CONFIG_RTE_MAX_LCORE=128
 CONFIG_RTE_MAX_NUMA_NODES=8
 CONFIG_RTE_MAX_HEAPS=32
 CONFIG_RTE_MAX_MEMSEG_LISTS=64
+
+# Use ARM LSE ATOMIC instructions
+CONFIG_RTE_ARM_FEATURE_ATOMICS=n
 # each memseg list will be limited to either RTE_MAX_MEMSEG_PER_LIST pages
 # or RTE_MAX_MEM_MB_PER_LIST megabytes worth of memory, whichever is smaller
 CONFIG_RTE_MAX_MEMSEG_PER_LIST=8192
config/defconfig_arm64-octeontx2-linuxapp-gcc +1 -0
index f20da24..7687dbe 100644
@@ -9,6 +9,7 @@ CONFIG_RTE_MACHINE="octeontx2"
 CONFIG_RTE_CACHE_LINE_SIZE=128
 CONFIG_RTE_MAX_NUMA_NODES=1
 CONFIG_RTE_MAX_LCORE=24
+CONFIG_RTE_ARM_FEATURE_ATOMICS=y
 
 # Doesn't support NUMA
 CONFIG_RTE_EAL_NUMA_AWARE_HUGEPAGES=n
config/defconfig_arm64-thunderx2-linuxapp-gcc +1 -0
index cc5c64b..af4a89c 100644
@@ -9,3 +9,4 @@ CONFIG_RTE_MACHINE="thunderx2"
 CONFIG_RTE_CACHE_LINE_SIZE=64
 CONFIG_RTE_MAX_NUMA_NODES=2
 CONFIG_RTE_MAX_LCORE=256
+CONFIG_RTE_ARM_FEATURE_ATOMICS=y
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +163 -0

180 lines not on this page · raw message

lib/librte_eal/common/include/arch/x86/rte_atomic_64.h +0 -12
index 1335d92..cfe7067 100644
@@ -183,18 +183,6 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)
 
 /*------------------------ 128 bit atomic operations -------------------------*/
 
-/**
- * 128-bit integer structure.
- */
-RTE_STD_C11
-typedef struct {
-	RTE_STD_C11
-	union {
-		uint64_t val[2];
-		__extension__ __int128 int128;
-	};
-} __rte_aligned(16) rte_int128_t;
-
 __rte_experimental
 static inline int
 rte_atomic128_cmp_exchange(rte_int128_t *dst,
lib/librte_eal/common/include/generic/rte_atomic.h +16 -1
index 24ff7dc..e6ab15a 100644
@@ -1081,6 +1081,20 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)
 
 /*------------------------ 128 bit atomic operations -------------------------*/
 
+/**
+ * 128-bit integer structure.
+ */
+RTE_STD_C11
+typedef struct {
+	RTE_STD_C11
+	union {
+		uint64_t val[2];
+#ifdef RTE_ARCH_64
+		__extension__ __int128 int128;
+#endif
+	};
+} __rte_aligned(16) rte_int128_t;
+
 #ifdef __DOXYGEN__
 
 /**
@@ -1093,7 +1107,8 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)
  *     *exp = *dst
  * @endcode
  *
- * @note This function is currently only available for the x86-64 platform.
+ * @note This function is currently available for the x86-64 and aarch64
+ * platforms.
  *
  * @note The success and failure arguments must be one of the __ATOMIC_* values
  * defined in the C++11 standard. For details on their behavior, refer to the
Signature · 1 line
2.7.4
author
#64 lore raw

PATCH v9 2/3 test/atomic: add 128b compare and swap test

Add 128b atomic compare and swap test for aarch64 and x86_64.

---
 app/test/test_atomic.c | 125 ++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 123 insertions(+), 2 deletions(-)
app/test/test_atomic.c +123 -2
index 43be30e..0dad923 100644
@@ -1,5 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright(c) 2010-2014 Intel Corporation
+ * Copyright(c) 2019 Arm Limited
  */
 
 #include <stdio.h>
@@ -20,7 +21,7 @@
  * Atomic Variables
  * ================
  *
- * - The main test function performs three subtests. The first test
+ * - The main test function performs four subtests. The first test
  *   checks that the usual inc/dec/add/sub functions are working
  *   correctly:
  *
@@ -61,11 +62,27 @@
  *       atomic_sub(&count, tmp+1);
  *
  *   - At the end of the test, the *count* value must be 0.
+ *
+ * - Test "128b compare and swap" (aarch64 and x86_64 only)
+ *
+ *   - Initialize 128-bit atomic variables to zero.
+ *
+ *   - Invoke ``test_atomici128_cmp_exchange()`` on each lcore. Before doing
+ *     anything else, the cores are waiting a synchro. Each lcore does
+ *     these compare and swap (CAS) operations several times::
+ *
+ *       Acquired CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *       Released CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *       Acquired_Released CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *       Relaxed CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *
+ *   - At the end of the test, the *count128* first 64-bit value and
+ *     second 64-bit value differ by the total iterations.
  */
 
 #define NUM_ATOMIC_TYPES 3
 
-#define N 10000
+#define N 1000000
 
 static rte_atomic16_t a16;
 static rte_atomic32_t a32;
@@ -216,6 +233,78 @@ test_atomic_dec_and_test(__attribute__((unused)) void *arg)
 	return 0;
 }
 
+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)
+static rte_int128_t count128;
+
+/*
+ * rte_atomic128_cmp_exchange() should update a 128 bits counter's first 64
+ * bits by 2 and the second 64 bits by 1 in this test. It should return true
+ * if the compare exchange operation is successful.
+ * This test repeats 128 bits compare and swap operations 10K rounds. In each
+ * iteration it runs compare and swap operation with different memory models.
+ */
+static int
+test_atomic128_cmp_exchange(__attribute__((unused)) void *arg)
+{
+	rte_int128_t expected;
+	int success;
+	unsigned int i;
+
+	while (rte_atomic32_read(&synchro) == 0)
+		;
+
+	expected = count128;
+
+	for (i = 0; i < N; i++) {
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128,
+				&expected, &desired, 1,
+				__ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
+		} while (success == 0);
+
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128,
+					&expected, &desired, 1,
+					__ATOMIC_RELEASE, __ATOMIC_RELAXED);
+		} while (success == 0);
+
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128,
+					&expected, &desired, 1,
+					__ATOMIC_ACQ_REL, __ATOMIC_RELAXED);
+		} while (success == 0);
+
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128,
+					&expected, &desired, 1,
+					__ATOMIC_RELAXED, __ATOMIC_RELAXED);
+		} while (success == 0);
+	}
+
+	return 0;
+}
+#endif
+
 static int
 test_atomic(void)
 {
@@ -340,6 +429,38 @@ test_atomic(void)
 		return -1;
 	}
 
+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)
+	/*
+	 * This case tests the functionality of rte_atomic128b_cmp_exchange
+	 * API. It calls rte_atomic128b_cmp_exchange with four kinds of memory
+	 * models successively on each slave core. Once each 128-bit atomic
+	 * compare and swap operation is successful, it updates the global
+	 * 128-bit counter by 2 for the first 64-bit and 1 for the second
+	 * 64-bit. Each slave core iterates this test 10K times.
+	 * At the end of test, verify whether the first 64-bits of the 128-bit
+	 * counter and the second 64bits is differ by the total iterations. If
+	 * it is, the test passes.
+	 */
+	printf("128b compare and swap test\n");
+	uint64_t iterations = 0;
+
+	rte_atomic32_clear(&synchro);
+	count128.val[0] = 0;
+	count128.val[1] = 0;
+
+	rte_eal_mp_remote_launch(test_atomic128_cmp_exchange, NULL,
+				 SKIP_MASTER);
+	rte_atomic32_set(&synchro, 1);
+	rte_eal_mp_wait_lcore();
+	rte_atomic32_clear(&synchro);
+
+	iterations = count128.val[0] - count128.val[1];
+	if (iterations != 4*N*(rte_lcore_count()-1)) {
+		printf("128b compare and swap failed\n");
+		return -1;
+	}
+#endif
+
 	return 0;
 }
 
Signature · 1 line
2.7.4
#65 lore raw

Re: [dpdk-dev] [PATCH v9 2/3] test/atomic: add 128b compare and swap test

Quoted 33 lines · Phil Yang

Add 128b atomic compare and swap test for aarch64 and x86_64.

---
 app/test/test_atomic.c | 125 ++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 123 insertions(+), 2 deletions(-)
app/test/test_atomic.c +2 -1
index 43be30e..0dad923 100644
@@ -1,5 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright(c) 2010-2014 Intel Corporation
+ * Copyright(c) 2019 Arm Limited
  */
 
 #include <stdio.h>
@@ -20,7 +21,7 @@
  * Atomic Variables
  * ================
  *
- * - The main test function performs three subtests. The first test
+ * - The main test function performs four subtests. The first test
  *   checks that the usual inc/dec/add/sub functions are working
  *   correctly:
  *

app/test/test_atomic.c line 70

Quoted 9 lines
app/test/test_atomic.c +6 -0
@@ -61,11 +62,27 @@
  *       atomic_sub(&count, tmp+1);
  *
  *   - At the end of the test, the *count* value must be 0.
+ *
+ * - Test "128b compare and swap" (aarch64 and x86_64 only)
+ *
+ *   - Initialize 128-bit atomic variables to zero.
+ *
+ *   - Invoke ``test_atomici128_cmp_exchange()`` on each lcore. Before doing

Typo, atomic128.

Quoted 6 lines
app/test/test_atomic.c +6 -0
+ *     anything else, the cores are waiting a synchro. Each lcore does
+ *     these compare and swap (CAS) operations several times::
+ *
+ *       Acquired CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *       Released CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *       Acquired_Released CAS update counter.val[0] + 2; counter.val[1] + 1;

app/test/test_atomic.c line 85

Quoted 10 lines
app/test/test_atomic.c +5 -1
+ *       Relaxed CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *
+ *   - At the end of the test, the *count128* first 64-bit value and
+ *     second 64-bit value differ by the total iterations.
  */
 
 #define NUM_ATOMIC_TYPES 3
 
-#define N 10000
+#define N 1000000

This change the number of iterations for this test.
Did you evaluate the impact on the test duration?
I suppose this is fairly quick, but could you explain why this has been extended?
The commitlog does not give hints.

Quoted 5 lines
app/test/test_atomic.c +0 -0
 static rte_atomic16_t a16;
 static rte_atomic32_t a32;
@@ -216,6 +233,78 @@ test_atomic_dec_and_test(__attribute__((unused)) void *arg)
        return 0;

app/test/test_atomic.c test_atomic_dec_and_test(__attribute__((unused)) void *arg) · line 243

Quoted 10 lines
app/test/test_atomic.c +8 -0
 }
 
+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)
+static rte_int128_t count128;
+
+/*
+ * rte_atomic128_cmp_exchange() should update a 128 bits counter's first 64
+ * bits by 2 and the second 64 bits by 1 in this test. It should return true
+ * if the compare exchange operation is successful.
+ * This test repeats 128 bits compare and swap operations 10K rounds. In each

s/10K/N/

Quoted 69 lines
app/test/test_atomic.c +64 -0
+ * iteration it runs compare and swap operation with different memory models.
+ */
+static int
+test_atomic128_cmp_exchange(__attribute__((unused)) void *arg)
+{
+       rte_int128_t expected;
+       int success;
+       unsigned int i;
+
+       while (rte_atomic32_read(&synchro) == 0)
+               ;
+
+       expected = count128;
+
+       for (i = 0; i < N; i++) {
+               do {
+                       rte_int128_t desired;
+
+                       desired.val[0] = expected.val[0] + 2;
+                       desired.val[1] = expected.val[1] + 1;
+
+                       success = rte_atomic128_cmp_exchange(&count128,
+                               &expected, &desired, 1,
+                               __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
+               } while (success == 0);
+
+               do {
+                       rte_int128_t desired;
+
+                       desired.val[0] = expected.val[0] + 2;
+                       desired.val[1] = expected.val[1] + 1;
+
+                       success = rte_atomic128_cmp_exchange(&count128,
+                                       &expected, &desired, 1,
+                                       __ATOMIC_RELEASE, __ATOMIC_RELAXED);
+               } while (success == 0);
+
+               do {
+                       rte_int128_t desired;
+
+                       desired.val[0] = expected.val[0] + 2;
+                       desired.val[1] = expected.val[1] + 1;
+
+                       success = rte_atomic128_cmp_exchange(&count128,
+                                       &expected, &desired, 1,
+                                       __ATOMIC_ACQ_REL, __ATOMIC_RELAXED);
+               } while (success == 0);
+
+               do {
+                       rte_int128_t desired;
+
+                       desired.val[0] = expected.val[0] + 2;
+                       desired.val[1] = expected.val[1] + 1;
+
+                       success = rte_atomic128_cmp_exchange(&count128,
+                                       &expected, &desired, 1,
+                                       __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+               } while (success == 0);
+       }
+
+       return 0;
+}
+#endif
+
 static int
 test_atomic(void)
 {
@@ -340,6 +429,38 @@ test_atomic(void)
                return -1;

app/test/test_atomic.c test_atomic(void) · line 439

Quoted 10 lines
app/test/test_atomic.c +8 -0
        }
 
+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)
+       /*
+        * This case tests the functionality of rte_atomic128b_cmp_exchange
+        * API. It calls rte_atomic128b_cmp_exchange with four kinds of memory
+        * models successively on each slave core. Once each 128-bit atomic
+        * compare and swap operation is successful, it updates the global
+        * 128-bit counter by 2 for the first 64-bit and 1 for the second
+        * 64-bit. Each slave core iterates this test 10K times.

N times.

Quoted 28 lines
app/test/test_atomic.c +24 -1
+        * At the end of test, verify whether the first 64-bits of the 128-bit
+        * counter and the second 64bits is differ by the total iterations. If
+        * it is, the test passes.
+        */
+       printf("128b compare and swap test\n");
+       uint64_t iterations = 0;
+
+       rte_atomic32_clear(&synchro);
+       count128.val[0] = 0;
+       count128.val[1] = 0;
+
+       rte_eal_mp_remote_launch(test_atomic128_cmp_exchange, NULL,
+                                SKIP_MASTER);
+       rte_atomic32_set(&synchro, 1);
+       rte_eal_mp_wait_lcore();
+       rte_atomic32_clear(&synchro);
+
+       iterations = count128.val[0] - count128.val[1];
+       if (iterations != 4*N*(rte_lcore_count()-1)) {
+               printf("128b compare and swap failed\n");
+               return -1;
+       }
+#endif
+
        return 0;
 }
 
--
Quoted 2 lines

2.7.4

--
David Marchand

#66 lore raw

Re: [dpdk-dev] [PATCH v9 2/3] test/atomic: add 128b compare and swap test

Quoted 57 lines

-----Original Message-----
From: David Marchand <[email protected]>
Sent: Monday, October 14, 2019 11:45 PM
To: Phil Yang (Arm Technology China) <[email protected]>
Cc: [email protected]; [email protected]; Gage Eads <[email protected]>; dev <[email protected]>; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v9 2/3] test/atomic: add 128b compare and swap test

Quoted 10 lines · Phil Yang

Add 128b atomic compare and swap test for aarch64 and x86_64.

---
 app/test/test_atomic.c | 125

++++++++++++++++++++++++++++++++++++++++++++++++-

Quoted 23 lines
 1 file changed, 123 insertions(+), 2 deletions(-)
app/test/test_atomic.c +2 -1
index 43be30e..0dad923 100644
@@ -1,5 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright(c) 2010-2014 Intel Corporation
+ * Copyright(c) 2019 Arm Limited
  */
 
 #include <stdio.h>
@@ -20,7 +21,7 @@
  * Atomic Variables
  * ================
  *
- * - The main test function performs three subtests. The first test
+ * - The main test function performs four subtests. The first test
  *   checks that the usual inc/dec/add/sub functions are working
  *   correctly:
  *

app/test/test_atomic.c line 70

Quoted 9 lines
app/test/test_atomic.c +6 -0
@@ -61,11 +62,27 @@
  *       atomic_sub(&count, tmp+1);
  *
  *   - At the end of the test, the *count* value must be 0.
+ *
+ * - Test "128b compare and swap" (aarch64 and x86_64 only)
+ *
+ *   - Initialize 128-bit atomic variables to zero.
+ *
+ *   - Invoke ``test_atomici128_cmp_exchange()`` on each lcore. Before

doing

Quoted 1 line

Typo, atomic128.

Updated in v10.

Quoted 16 lines
Quoted 16 lines
app/test/test_atomic.c +10 -0
+ *     anything else, the cores are waiting a synchro. Each lcore does
+ *     these compare and swap (CAS) operations several times::
+ *
+ *       Acquired CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *       Released CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *       Acquired_Released CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *       Relaxed CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *
+ *   - At the end of the test, the *count128* first 64-bit value and
+ *     second 64-bit value differ by the total iterations.
  */
 
 #define NUM_ATOMIC_TYPES 3
 
Quoted 7 lines

app/test/test_atomic.c line 85

Quoted 2 lines
app/test/test_atomic.c +1 -1
-#define N 10000
+#define N 1000000

This change the number of iterations for this test.
Did you evaluate the impact on the test duration?
I suppose this is fairly quick, but could you explain why this has been extended?

By extending the iterations to 1 million times it can test the stability of these atomic APIs, especial for the new added 128bit atomics.
Yes, I did the evaluation. It has no impact on the test duration as the test case is simple.

Quoted 1 line

The commitlog does not give hints.

Thanks for pointing out this. I have updated this in the v10 commitlog.

Quoted 16 lines

app/test/test_atomic.c

Quoted 6 lines
app/test/test_atomic.c +0 -0
 static rte_atomic16_t a16;
 static rte_atomic32_t a32;
@@ -216,6 +233,78 @@

test_atomic_dec_and_test(__attribute__((unused)) void *arg)

app/test/test_atomic.c test_atomic_dec_and_test(__attribute__((unused)) void *arg)

Quoted 8 lines
app/test/test_atomic.c +5 -0
        return 0;
 }
 
+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)
+static rte_int128_t count128;
+
+/*
+ * rte_atomic128_cmp_exchange() should update a 128 bits counter's first

64

Quoted 6 lines

app/test/test_atomic.c test_atomic_dec_and_test(__attribute__((unused)) void *arg)

Quoted 3 lines
app/test/test_atomic.c +3 -0
+ * bits by 2 and the second 64 bits by 1 in this test. It should return true
+ * if the compare exchange operation is successful.
+ * This test repeats 128 bits compare and swap operations 10K rounds. In

each

s/10K/N/

Updated in v10.

<snip>

Quoted 11 lines

app/test/test_atomic.c test_atomic(void)

Quoted 4 lines
app/test/test_atomic.c +4 -0
+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)
+       /*
+        * This case tests the functionality of rte_atomic128b_cmp_exchange
+        * API. It calls rte_atomic128b_cmp_exchange with four kinds of

memory

app/test/test_atomic.c test_atomic(void) · line 439

Quoted 4 lines
app/test/test_atomic.c +4 -0
+        * models successively on each slave core. Once each 128-bit atomic
+        * compare and swap operation is successful, it updates the global
+        * 128-bit counter by 2 for the first 64-bit and 1 for the second
+        * 64-bit. Each slave core iterates this test 10K times.

N times.

Updated in v10.

Thanks,
Phil

Quoted 36 lines
Quoted 30 lines
app/test/test_atomic.c +24 -1
+        * At the end of test, verify whether the first 64-bits of the 128-bit
+        * counter and the second 64bits is differ by the total iterations. If
+        * it is, the test passes.
+        */
+       printf("128b compare and swap test\n");
+       uint64_t iterations = 0;
+
+       rte_atomic32_clear(&synchro);
+       count128.val[0] = 0;
+       count128.val[1] = 0;
+
+       rte_eal_mp_remote_launch(test_atomic128_cmp_exchange, NULL,
+                                SKIP_MASTER);
+       rte_atomic32_set(&synchro, 1);
+       rte_eal_mp_wait_lcore();
+       rte_atomic32_clear(&synchro);
+
+       iterations = count128.val[0] - count128.val[1];
+       if (iterations != 4*N*(rte_lcore_count()-1)) {
+               printf("128b compare and swap failed\n");
+               return -1;
+       }
+#endif
+
        return 0;
 }
 
--
Quoted 2 lines

2.7.4

--
David Marchand

author
#67 lore raw

PATCH v9 3/3 eal/stack: enable lock-free stack for aarch64

In reply to Phil Yang

Enable both c11 atomic and non c11 atomic lock-free stack for aarch64.

Introduced a new header to reduce the ifdef clutter across generic and c11 files. The rte_stack_lf_stubs.h contains stub implementations of __rte_stack_lf_count, __rte_stack_lf_push_elems and __rte_stack_lf_pop_elems.

---
 doc/guides/prog_guide/env_abstraction_layer.rst |  4 +--
 doc/guides/rel_notes/release_19_11.rst          |  3 ++
 lib/librte_stack/Makefile                       |  3 +-
 lib/librte_stack/rte_stack_lf.h                 |  4 +++
 lib/librte_stack/rte_stack_lf_c11.h             | 16 ---------
 lib/librte_stack/rte_stack_lf_generic.h         | 16 ---------
 lib/librte_stack/rte_stack_lf_stubs.h           | 44 +++++++++++++++++++++++++
 7 files changed, 55 insertions(+), 35 deletions(-)
 create mode 100644 lib/librte_stack/rte_stack_lf_stubs.h
doc/guides/prog_guide/env_abstraction_layer.rst +2 -2
index 94f30fd..6e59fae 100644
@@ -648,8 +648,8 @@ Known Issues
   Alternatively, applications can use the lock-free stack mempool handler. When
   considering this handler, note that:
 
-  - It is currently limited to the x86_64 platform, because it uses an
-    instruction (16-byte compare-and-swap) that is not yet available on other
+  - It is currently limited to the aarch64 and x86_64 platforms, because it uses
+    an instruction (16-byte compare-and-swap) that is not yet available on other
     platforms.
   - It has worse average-case performance than the non-preemptive rte_ring, but
     software caching (e.g. the mempool cache) can mitigate this by reducing the
doc/guides/rel_notes/release_19_11.rst +3 -0
index 8490d89..60ffd70 100644
@@ -56,6 +56,9 @@ New Features
      Also, make sure to start the actual text at the margin.
      =========================================================
 
+* **Added Lock-free Stack for aarch64.**
+
+  The lock-free stack implementation is enabled for aarch64 platforms.
 
 Removed Items
 -------------
lib/librte_stack/Makefile +2 -1
index 8d18ce5..c337ab7 100644
@@ -24,6 +24,7 @@ SYMLINK-$(CONFIG_RTE_LIBRTE_STACK)-include := rte_stack.h \
 					      rte_stack_std.h \
 					      rte_stack_lf.h \
 					      rte_stack_lf_generic.h \
-					      rte_stack_lf_c11.h
+					      rte_stack_lf_c11.h \
+						  rte_stack_lf_stubs.h
 
 include $(RTE_SDK)/mk/rte.lib.mk
lib/librte_stack/rte_stack_lf.h +4 -0
index f5581f0..e67630c 100644
@@ -5,11 +5,15 @@
 #ifndef _RTE_STACK_LF_H_
 #define _RTE_STACK_LF_H_
 
+#if !(defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64))
+#include "rte_stack_lf_stubs.h"
+#else
 #ifdef RTE_USE_C11_MEM_MODEL
 #include "rte_stack_lf_c11.h"
 #else
 #include "rte_stack_lf_generic.h"
 #endif
+#endif
 
 /**
  * @internal Push several objects on the lock-free stack (MT-safe).
lib/librte_stack/rte_stack_lf_c11.h +0 -16
index 3d677ae..999359f 100644
@@ -36,12 +36,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 			  struct rte_stack_lf_elem *last,
 			  unsigned int num)
 {
-#ifndef RTE_ARCH_X86_64
-	RTE_SET_USED(first);
-	RTE_SET_USED(last);
-	RTE_SET_USED(list);
-	RTE_SET_USED(num);
-#else
 	struct rte_stack_lf_head old_head;
 	int success;
 
@@ -79,7 +73,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 	 * to the LIFO len update.
 	 */
 	__atomic_add_fetch(&list->len, num, __ATOMIC_RELEASE);
-#endif
 }
 
 static __rte_always_inline struct rte_stack_lf_elem *
@@ -88,14 +81,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 void **obj_table,
 			 struct rte_stack_lf_elem **last)
 {
-#ifndef RTE_ARCH_X86_64
-	RTE_SET_USED(obj_table);
-	RTE_SET_USED(last);
-	RTE_SET_USED(list);
-	RTE_SET_USED(num);
-
-	return NULL;
-#else
 	struct rte_stack_lf_head old_head;
 	uint64_t len;
 	int success;
@@ -169,7 +154,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 	} while (success == 0);
 
 	return old_head.top;
-#endif
 }
 
 #endif /* _RTE_STACK_LF_C11_H_ */
lib/librte_stack/rte_stack_lf_generic.h +0 -16
index 3182151..3abbb53 100644
@@ -36,12 +36,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 			  struct rte_stack_lf_elem *last,
 			  unsigned int num)
 {
-#ifndef RTE_ARCH_X86_64
-	RTE_SET_USED(first);
-	RTE_SET_USED(last);
-	RTE_SET_USED(list);
-	RTE_SET_USED(num);
-#else
 	struct rte_stack_lf_head old_head;
 	int success;
 
@@ -75,7 +69,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 	} while (success == 0);
 
 	rte_atomic64_add((rte_atomic64_t *)&list->len, num);
-#endif
 }
 
 static __rte_always_inline struct rte_stack_lf_elem *
@@ -84,14 +77,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 void **obj_table,
 			 struct rte_stack_lf_elem **last)
 {
-#ifndef RTE_ARCH_X86_64
-	RTE_SET_USED(obj_table);
-	RTE_SET_USED(last);
-	RTE_SET_USED(list);
-	RTE_SET_USED(num);
-
-	return NULL;
-#else
 	struct rte_stack_lf_head old_head;
 	int success;
 
@@ -159,7 +144,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 	} while (success == 0);
 
 	return old_head.top;
-#endif
 }
 
 #endif /* _RTE_STACK_LF_GENERIC_H_ */
lib/librte_stack/rte_stack_lf_stubs.h new file +44 -0

44 lines not on this page · raw message

Signature · 1 line
2.7.4
#68 lore raw

Re: [dpdk-dev] [PATCH v9 3/3] eal/stack: enable lock-free stack for aarch64

Quoted 59 lines · Phil Yang

Enable both c11 atomic and non c11 atomic lock-free stack for aarch64.

Introduced a new header to reduce the ifdef clutter across generic and c11 files. The rte_stack_lf_stubs.h contains stub implementations of __rte_stack_lf_count, __rte_stack_lf_push_elems and __rte_stack_lf_pop_elems.

---
 doc/guides/prog_guide/env_abstraction_layer.rst |  4 +--
 doc/guides/rel_notes/release_19_11.rst          |  3 ++
 lib/librte_stack/Makefile                       |  3 +-
 lib/librte_stack/rte_stack_lf.h                 |  4 +++
 lib/librte_stack/rte_stack_lf_c11.h             | 16 ---------
 lib/librte_stack/rte_stack_lf_generic.h         | 16 ---------
 lib/librte_stack/rte_stack_lf_stubs.h           | 44 +++++++++++++++++++++++++
 7 files changed, 55 insertions(+), 35 deletions(-)
 create mode 100644 lib/librte_stack/rte_stack_lf_stubs.h
doc/guides/prog_guide/env_abstraction_layer.rst +2 -2
index 94f30fd..6e59fae 100644
@@ -648,8 +648,8 @@ Known Issues
   Alternatively, applications can use the lock-free stack mempool handler. When
   considering this handler, note that:
 
-  - It is currently limited to the x86_64 platform, because it uses an
-    instruction (16-byte compare-and-swap) that is not yet available on other
+  - It is currently limited to the aarch64 and x86_64 platforms, because it uses
+    an instruction (16-byte compare-and-swap) that is not yet available on other
     platforms.
   - It has worse average-case performance than the non-preemptive rte_ring, but
     software caching (e.g. the mempool cache) can mitigate this by reducing the
doc/guides/rel_notes/release_19_11.rst +3 -0
index 8490d89..60ffd70 100644
@@ -56,6 +56,9 @@ New Features
      Also, make sure to start the actual text at the margin.
      =========================================================
 
+* **Added Lock-free Stack for aarch64.**
+
+  The lock-free stack implementation is enabled for aarch64 platforms.
 
 Removed Items
 -------------
lib/librte_stack/Makefile +0 -0
index 8d18ce5..c337ab7 100644

lib/librte_stack/Makefile SYMLINK-$(CONFIG_RTE_LIBRTE_STACK)-include := rte_stack.h \ · line 28

Quoted 6 lines
lib/librte_stack/Makefile +2 -1
@@ -24,6 +24,7 @@ SYMLINK-$(CONFIG_RTE_LIBRTE_STACK)-include := rte_stack.h \
                                              rte_stack_std.h \
                                              rte_stack_lf.h \
                                              rte_stack_lf_generic.h \
-                                             rte_stack_lf_c11.h
+                                             rte_stack_lf_c11.h \
+                                                 rte_stack_lf_stubs.h

Please, use the same indentation type than the other lines.

Quoted 171 lines
lib/librte_stack/Makefile +0 -0
 include $(RTE_SDK)/mk/rte.lib.mk
lib/librte_stack/rte_stack_lf.h +4 -0
index f5581f0..e67630c 100644
@@ -5,11 +5,15 @@
 #ifndef _RTE_STACK_LF_H_
 #define _RTE_STACK_LF_H_
 
+#if !(defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64))
+#include "rte_stack_lf_stubs.h"
+#else
 #ifdef RTE_USE_C11_MEM_MODEL
 #include "rte_stack_lf_c11.h"
 #else
 #include "rte_stack_lf_generic.h"
 #endif
+#endif
 
 /**
  * @internal Push several objects on the lock-free stack (MT-safe).
lib/librte_stack/rte_stack_lf_c11.h +0 -16
index 3d677ae..999359f 100644
@@ -36,12 +36,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
                          struct rte_stack_lf_elem *last,
                          unsigned int num)
 {
-#ifndef RTE_ARCH_X86_64
-       RTE_SET_USED(first);
-       RTE_SET_USED(last);
-       RTE_SET_USED(list);
-       RTE_SET_USED(num);
-#else
        struct rte_stack_lf_head old_head;
        int success;
 
@@ -79,7 +73,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
         * to the LIFO len update.
         */
        __atomic_add_fetch(&list->len, num, __ATOMIC_RELEASE);
-#endif
 }
 
 static __rte_always_inline struct rte_stack_lf_elem *
@@ -88,14 +81,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
                         void **obj_table,
                         struct rte_stack_lf_elem **last)
 {
-#ifndef RTE_ARCH_X86_64
-       RTE_SET_USED(obj_table);
-       RTE_SET_USED(last);
-       RTE_SET_USED(list);
-       RTE_SET_USED(num);
-
-       return NULL;
-#else
        struct rte_stack_lf_head old_head;
        uint64_t len;
        int success;
@@ -169,7 +154,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
        } while (success == 0);
 
        return old_head.top;
-#endif
 }
 
 #endif /* _RTE_STACK_LF_C11_H_ */
lib/librte_stack/rte_stack_lf_generic.h +0 -16
index 3182151..3abbb53 100644
@@ -36,12 +36,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
                          struct rte_stack_lf_elem *last,
                          unsigned int num)
 {
-#ifndef RTE_ARCH_X86_64
-       RTE_SET_USED(first);
-       RTE_SET_USED(last);
-       RTE_SET_USED(list);
-       RTE_SET_USED(num);
-#else
        struct rte_stack_lf_head old_head;
        int success;
 
@@ -75,7 +69,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
        } while (success == 0);
 
        rte_atomic64_add((rte_atomic64_t *)&list->len, num);
-#endif
 }
 
 static __rte_always_inline struct rte_stack_lf_elem *
@@ -84,14 +77,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
                         void **obj_table,
                         struct rte_stack_lf_elem **last)
 {
-#ifndef RTE_ARCH_X86_64
-       RTE_SET_USED(obj_table);
-       RTE_SET_USED(last);
-       RTE_SET_USED(list);
-       RTE_SET_USED(num);
-
-       return NULL;
-#else
        struct rte_stack_lf_head old_head;
        int success;
 
@@ -159,7 +144,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
        } while (success == 0);
 
        return old_head.top;
-#endif
 }
 
 #endif /* _RTE_STACK_LF_GENERIC_H_ */
lib/librte_stack/rte_stack_lf_stubs.h new file +44 -1
new file mode 100644
index 0000000..a05abf1
@@ -0,0 +1,44 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2019 Arm Limited
+ */
+
+#ifndef _RTE_STACK_LF_STUBS_H_
+#define _RTE_STACK_LF_STUBS_H_
+
+#include <rte_common.h>
+
+static __rte_always_inline unsigned int
+__rte_stack_lf_count(struct rte_stack *s)
+{
+       RTE_SET_USED(s);
+
+       return 0;
+}
+
+static __rte_always_inline void
+__rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
+                         struct rte_stack_lf_elem *first,
+                         struct rte_stack_lf_elem *last,
+                         unsigned int num)
+{
+       RTE_SET_USED(first);
+       RTE_SET_USED(last);
+       RTE_SET_USED(list);
+       RTE_SET_USED(num);
+}
+
+static __rte_always_inline struct rte_stack_lf_elem *
+__rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
+                        unsigned int num,
+                        void **obj_table,
+                        struct rte_stack_lf_elem **last)
+{
+       RTE_SET_USED(obj_table);
+       RTE_SET_USED(last);
+       RTE_SET_USED(list);
+       RTE_SET_USED(num);
+
+       return NULL;
+}
+
+#endif /* _RTE_STACK_LF_STUBS_H_ */
--

2.7.4

#69 lore raw

Re: [dpdk-dev] [PATCH v9 3/3] eal/stack: enable lock-free stack for aarch64

Quoted 27 lines

-----Original Message-----
From: David Marchand <[email protected]>
Sent: Monday, October 14, 2019 11:45 PM
To: Phil Yang (Arm Technology China) <[email protected]>
Cc: [email protected]; [email protected]; Gage Eads <[email protected]>; dev <[email protected]>; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v9 3/3] eal/stack: enable lock-free stack for aarch64

Quoted 15 lines · Phil Yang

Enable both c11 atomic and non c11 atomic lock-free stack for aarch64.

Introduced a new header to reduce the ifdef clutter across generic and c11 files. The rte_stack_lf_stubs.h contains stub implementations of __rte_stack_lf_count, __rte_stack_lf_push_elems and __rte_stack_lf_pop_elems.

---

<snip>

Quoted 7 lines

lib/librte_stack/Makefile SYMLINK-$(CONFIG_RTE_LIBRTE_STACK)-include :=

Quoted 6 lines

-------------

lib/librte_stack/Makefile +0 -0
index 8d18ce5..c337ab7 100644
@@ -24,6 +24,7 @@ SYMLINK-$(CONFIG_RTE_LIBRTE_STACK)-include :=

rte_stack.h \

Quoted 8 lines

lib/librte_stack/Makefile SYMLINK-$(CONFIG_RTE_LIBRTE_STACK)-include := rte_stack.h \ · line 28

Quoted 6 lines
lib/librte_stack/Makefile +2 -1
                                              rte_stack_std.h \
                                              rte_stack_lf.h \
                                              rte_stack_lf_generic.h \
-                                             rte_stack_lf_c11.h
+                                             rte_stack_lf_c11.h \
+                                                 rte_stack_lf_stubs.h

Please, use the same indentation type than the other lines.

Updated in v10.

Thanks,
Phil

#70 lore raw
In reply to Phil Yang
Quoted 2 lines · Phil Yang

Add 128-bit atomic compare exchange on aarch64.

A bit short, seeing the complexity of the code and the additional RTE_ARM_FEATURE_ATOMICS config flag.

Comments inline.

Quoted 134 lines

---

v9:
Updated 19.11 release note.

v8:
Fixed "WARNING:LONG_LINE: line over 80 characters" warnings with latest kernel checkpatch.pl

v7:
1. Adjust code comment.

v6:
1. Put the RTE_ARM_FEATURE_ATOMICS flag into EAL group. (Jerin Jocob)
2. Keep rte_stack_lf_stubs.h doing nothing. (Gage Eads)
3. Fixed 32 bit build issue.

v5:
1. Enable RTE_ARM_FEATURE_ATOMICS on octeontx2 in default. (Jerin Jocob)
2. Record the reason of introducing "rte_stack_lf_stubs.h" in git commit.
(Jerin, Jocob)
3. Fixed a conditional MACRO error in rte_atomic128_cmp_exchange. (Jerin Jocob)

v4:
1. Add RTE_ARM_FEATURE_ATOMICS flag to support LSE CASP instructions. (Jerin Jocob)
2. Fix possible arm64 ABI break by making casp_op_name noinline. (Jerin Jocob)
3. Add rte_stack_lf_stubs.h to reduce the ifdef clutter. (Gage Eads/Jerin Jocob)

v3:
1. Avoid duplication code with macro. (Jerin Jocob)
2. Make invalid memory order to strongest barrier. (Jerin Jocob)
3. Update doc/guides/prog_guide/env_abstraction_layer.rst. (Gage Eads)
4. Fix 32-bit x86 builds issue. (Gage Eads)
5. Correct documentation issues in UT. (Gage Eads)

v2:
Initial version.

 config/arm/meson.build                             |   2 +
 config/common_base                                 |   3 +
 config/defconfig_arm64-octeontx2-linuxapp-gcc      |   1 +
 config/defconfig_arm64-thunderx2-linuxapp-gcc      |   1 +
 .../common/include/arch/arm/rte_atomic_64.h        | 163 +++++++++++++++++++++
 .../common/include/arch/x86/rte_atomic_64.h        |  12 --
 lib/librte_eal/common/include/generic/rte_atomic.h |  17 ++-
 7 files changed, 186 insertions(+), 13 deletions(-)
config/arm/meson.build +2 -0
index 979018e..9f28271 100644
@@ -71,11 +71,13 @@ flags_thunderx2_extra = [
        ['RTE_CACHE_LINE_SIZE', 64],
        ['RTE_MAX_NUMA_NODES', 2],
        ['RTE_MAX_LCORE', 256],
+       ['RTE_ARM_FEATURE_ATOMICS', true],
        ['RTE_USE_C11_MEM_MODEL', true]]
 flags_octeontx2_extra = [
        ['RTE_MACHINE', '"octeontx2"'],
        ['RTE_MAX_NUMA_NODES', 1],
        ['RTE_MAX_LCORE', 24],
+       ['RTE_ARM_FEATURE_ATOMICS', true],
        ['RTE_EAL_IGB_UIO', false],
        ['RTE_USE_C11_MEM_MODEL', true]]
 
config/common_base +3 -0
index 8ef75c2..2054480 100644
@@ -82,6 +82,9 @@ CONFIG_RTE_MAX_LCORE=128
 CONFIG_RTE_MAX_NUMA_NODES=8
 CONFIG_RTE_MAX_HEAPS=32
 CONFIG_RTE_MAX_MEMSEG_LISTS=64
+
+# Use ARM LSE ATOMIC instructions
+CONFIG_RTE_ARM_FEATURE_ATOMICS=n
 # each memseg list will be limited to either RTE_MAX_MEMSEG_PER_LIST pages
 # or RTE_MAX_MEM_MB_PER_LIST megabytes worth of memory, whichever is smaller
 CONFIG_RTE_MAX_MEMSEG_PER_LIST=8192
config/defconfig_arm64-octeontx2-linuxapp-gcc +1 -0
index f20da24..7687dbe 100644
@@ -9,6 +9,7 @@ CONFIG_RTE_MACHINE="octeontx2"
 CONFIG_RTE_CACHE_LINE_SIZE=128
 CONFIG_RTE_MAX_NUMA_NODES=1
 CONFIG_RTE_MAX_LCORE=24
+CONFIG_RTE_ARM_FEATURE_ATOMICS=y
 
 # Doesn't support NUMA
 CONFIG_RTE_EAL_NUMA_AWARE_HUGEPAGES=n
config/defconfig_arm64-thunderx2-linuxapp-gcc +1 -0
index cc5c64b..af4a89c 100644
@@ -9,3 +9,4 @@ CONFIG_RTE_MACHINE="thunderx2"
 CONFIG_RTE_CACHE_LINE_SIZE=64
 CONFIG_RTE_MAX_NUMA_NODES=2
 CONFIG_RTE_MAX_LCORE=256
+CONFIG_RTE_ARM_FEATURE_ATOMICS=y
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +4 -0
index 97060e4..14d869b 100644
@@ -1,5 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright(c) 2015 Cavium, Inc
+ * Copyright(c) 2019 Arm Limited
  */
 
 #ifndef _RTE_ATOMIC_ARM64_H_
@@ -14,6 +15,9 @@ extern "C" {
 #endif
 
 #include "generic/rte_atomic.h"
+#include <rte_branch_prediction.h>
+#include <rte_compat.h>
+#include <rte_debug.h>
 
 #define dsb(opt) asm volatile("dsb " #opt : : : "memory")
 #define dmb(opt) asm volatile("dmb " #opt : : : "memory")
@@ -40,6 +44,165 @@ extern "C" {
 

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 54

Quoted 10 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +8 -0
 #define rte_cio_rmb() dmb(oshld)
 
+/*------------------------ 128 bit atomic operations -------------------------*/
+
+#define __HAS_ACQ(mo) ((mo) != __ATOMIC_RELAXED && (mo) != __ATOMIC_RELEASE)
+#define __HAS_RLS(mo) ((mo) == __ATOMIC_RELEASE || (mo) == __ATOMIC_ACQ_REL || \
+                                         (mo) == __ATOMIC_SEQ_CST)
+
+#define __MO_LOAD(mo)  (__HAS_ACQ((mo)) ? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED)
+#define __MO_STORE(mo) (__HAS_RLS((mo)) ? __ATOMIC_RELEASE : __ATOMIC_RELAXED)

Those 4 first macros only make sense when LSE is not available (see below [1]). Besides, they are used only once, why not directly use those conditions where needed?

Quoted 22 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +22 -0
+
+#if defined(__ARM_FEATURE_ATOMICS) || defined(RTE_ARM_FEATURE_ATOMICS)
+#define __ATOMIC128_CAS_OP(cas_op_name, op_string)                          \
+static __rte_noinline rte_int128_t                                          \
+cas_op_name(rte_int128_t *dst, rte_int128_t old,                            \
+               rte_int128_t updated)                                       \
+{                                                                           \
+       /* caspX instructions register pair must start from even-numbered
+        * register at operand 1.
+        * So, specify registers for local variables here.
+        */                                                                 \
+       register uint64_t x0 __asm("x0") = (uint64_t)old.val[0];            \
+       register uint64_t x1 __asm("x1") = (uint64_t)old.val[1];            \
+       register uint64_t x2 __asm("x2") = (uint64_t)updated.val[0];        \
+       register uint64_t x3 __asm("x3") = (uint64_t)updated.val[1];        \
+       asm volatile(                                                       \
+               op_string " %[old0], %[old1], %[upd0], %[upd1], [%[dst]]"   \
+               : [old0] "+r" (x0),                                         \
+               [old1] "+r" (x1)                                            \
+               : [upd0] "r" (x2),                                          \
+               [upd1] "r" (x3),                                            \
+               [dst] "r" (dst)                                             \

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 86

Quoted 10 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +10 -0
+               : "memory");                                                \
+       old.val[0] = x0;                                                    \
+       old.val[1] = x1;                                                    \
+       return old;                                                         \
+}
+
+__ATOMIC128_CAS_OP(__rte_cas_relaxed, "casp")
+__ATOMIC128_CAS_OP(__rte_cas_acquire, "caspa")
+__ATOMIC128_CAS_OP(__rte_cas_release, "caspl")
+__ATOMIC128_CAS_OP(__rte_cas_acq_rel, "caspal")

If LSE is available, we expose __rte_cas_XX (explicitely) *non* inlined functions, while without LSE, we expose inlined __rte_ldr_XX and __rte_stx_XX functions.
So we have a first disparity with non-inlined vs inlined functions depending on a #ifdef.
Then, we have a second disparity with two sets of "apis" depending on this #ifdef.

And we expose those sets with a rte_ prefix, meaning people will try to use them, but those are not part of a public api.

Can't we do without them ? (see below [2] for a proposal with ldr/stx, cas should be the same)

Quoted 29 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +29 -0
+#else
+#define __ATOMIC128_LDX_OP(ldx_op_name, op_string)                          \
+static inline rte_int128_t                                                  \
+ldx_op_name(const rte_int128_t *src)                                        \
+{                                                                           \
+       rte_int128_t ret;                                                   \
+       asm volatile(                                                       \
+                       op_string " %0, %1, %2"                             \
+                       : "=&r" (ret.val[0]),                               \
+                         "=&r" (ret.val[1])                                \
+                       : "Q" (src->val[0])                                 \
+                       : "memory");                                        \
+       return ret;                                                         \
+}
+
+__ATOMIC128_LDX_OP(__rte_ldx_relaxed, "ldxp")
+__ATOMIC128_LDX_OP(__rte_ldx_acquire, "ldaxp")
+
+#define __ATOMIC128_STX_OP(stx_op_name, op_string)                          \
+static inline uint32_t                                                      \
+stx_op_name(rte_int128_t *dst, const rte_int128_t src)                      \
+{                                                                           \
+       uint32_t ret;                                                       \
+       asm volatile(                                                       \
+                       op_string " %w0, %1, %2, %3"                        \
+                       : "=&r" (ret)                                       \
+                       : "r" (src.val[0]),                                 \
+                         "r" (src.val[1]),                                 \
+                         "Q" (dst->val[0])                                 \

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 125

Quoted 10 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +10 -0
+                       : "memory");                                        \
+       /* Return 0 on success, 1 on failure */                             \
+       return ret;                                                         \
+}
+
+__ATOMIC128_STX_OP(__rte_stx_relaxed, "stxp")
+__ATOMIC128_STX_OP(__rte_stx_release, "stlxp")
+#endif
+
+static inline int __rte_experimental

The __rte_experimental tag comes first.

Quoted 24 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +24 -0
+rte_atomic128_cmp_exchange(rte_int128_t *dst,
+                               rte_int128_t *exp,
+                               const rte_int128_t *src,
+                               unsigned int weak,
+                               int success,
+                               int failure)
+{
+       /* Always do strong CAS */
+       RTE_SET_USED(weak);
+       /* Ignore memory ordering for failure, memory order for
+        * success must be stronger or equal
+        */
+       RTE_SET_USED(failure);
+       /* Find invalid memory order */
+       RTE_ASSERT(success == __ATOMIC_RELAXED
+                       || success == __ATOMIC_ACQUIRE
+                       || success == __ATOMIC_RELEASE
+                       || success == __ATOMIC_ACQ_REL
+                       || success == __ATOMIC_SEQ_CST);
+
+#if defined(__ARM_FEATURE_ATOMICS) || defined(RTE_ARM_FEATURE_ATOMICS)
+       rte_int128_t expected = *exp;
+       rte_int128_t desired = *src;
+       rte_int128_t old;

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 159

Quoted 10 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +10 -0
+
+       if (success == __ATOMIC_RELAXED)
+               old = __rte_cas_relaxed(dst, expected, desired);
+       else if (success == __ATOMIC_ACQUIRE)
+               old = __rte_cas_acquire(dst, expected, desired);
+       else if (success == __ATOMIC_RELEASE)
+               old = __rte_cas_release(dst, expected, desired);
+       else
+               old = __rte_cas_acq_rel(dst, expected, desired);
+#else

1: the four first macros (on the memory ordering constraints) can be moved here then undef'd once unused.
Or you can just do without them.

Quoted 5 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5 -0
+       int ldx_mo = __MO_LOAD(success);
+       int stx_mo = __MO_STORE(success);
+       uint32_t ret = 1;
+       register rte_int128_t expected = *exp;
+       register rte_int128_t desired = *src;

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 174

Quoted 10 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +10 -0
+       register rte_int128_t old;
+
+       /* ldx128 can not guarantee atomic,
+        * Must write back src or old to verify atomicity of ldx128;
+        */
+       do {
+               if (ldx_mo == __ATOMIC_RELAXED)
+                       old = __rte_ldx_relaxed(dst);
+               else
+                       old = __rte_ldx_acquire(dst);

2: how about using a simple macro that gets passed the op string?

Something like (untested):

#define __READ_128(op_string, src, dst) \
    asm volatile(                      \
        op_string " %0, %1, %2"    \
        : "=&r" (dst.val[0]),      \
          "=&r" (dst.val[1])       \
        : "Q" (src->val[0])        \
        : "memory")

Then used like this:

        if (ldx_mo == __ATOMIC_RELAXED)
            __READ_128("ldxp", dst, old);
        else
            __READ_128("ldaxp", dst, old);

#undef __READ_128

Quoted 7 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +7 -0
+
+               if (likely(old.int128 == expected.int128)) {
+                       if (stx_mo == __ATOMIC_RELAXED)
+                               ret = __rte_stx_relaxed(dst, desired);
+                       else
+                               ret = __rte_stx_release(dst, desired);
+               } else {

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 191

Quoted 10 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +10 -0
+                       /* In the failure case (since 'weak' is ignored and only
+                        * weak == 0 is implemented), expected should contain
+                        * the atomically read value of dst. This means, 'old'
+                        * needs to be stored back to ensure it was read
+                        * atomically.
+                        */
+                       if (stx_mo == __ATOMIC_RELAXED)
+                               ret = __rte_stx_relaxed(dst, old);
+                       else
+                               ret = __rte_stx_release(dst, old);

And:

#define __STORE_128(op_string, dst, val, ret) \
    asm volatile(                        \
        op_string " %w0, %1, %2, %3"     \
        : "=&r" (ret)                    \
        : "r" (val.val[0]),              \
          "r" (val.val[1]),              \
          "Q" (dst->val[0])              \
        : "memory")

Used like this:

        if (likely(old.int128 == expected.int128)) {
            if (stx_mo == __ATOMIC_RELAXED)
                __STORE_128("stxp", dst, desired, ret);
            else
                __STORE_128("stlxp", dst, desired, ret);
        } else {
            /* In the failure case (since 'weak' is ignored and only
             * weak == 0 is implemented), expected should contain
             * the atomically read value of dst. This means, 'old'
             * needs to be stored back to ensure it was read
             * atomically.
             */
            if (stx_mo == __ATOMIC_RELAXED)
                __STORE_128("stxp", dst, old, ret);
            else
                __STORE_128("stlxp", dst, old, ret);
        }

#undef __STORE_128

Quoted 49 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +14 -0
+               }
+       } while (unlikely(ret));
+#endif
+
+       /* Unconditionally updating expected removes
+        * an 'if' statement.
+        * expected should already be in register if
+        * not in the cache.
+        */
+       *exp = old;
+
+       return (old.int128 == expected.int128);
+}
+
 #ifdef __cplusplus
 }
 #endif
lib/librte_eal/common/include/arch/x86/rte_atomic_64.h +0 -12
index 1335d92..cfe7067 100644
@@ -183,18 +183,6 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)
 
 /*------------------------ 128 bit atomic operations -------------------------*/
 
-/**
- * 128-bit integer structure.
- */
-RTE_STD_C11
-typedef struct {
-       RTE_STD_C11
-       union {
-               uint64_t val[2];
-               __extension__ __int128 int128;
-       };
-} __rte_aligned(16) rte_int128_t;
-
 __rte_experimental
 static inline int
 rte_atomic128_cmp_exchange(rte_int128_t *dst,
lib/librte_eal/common/include/generic/rte_atomic.h +1 -0
index 24ff7dc..e6ab15a 100644
@@ -1081,6 +1081,20 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)
 
 /*------------------------ 128 bit atomic operations -------------------------*/
 
+/**

lib/librte_eal/common/include/generic/rte_atomic.h static inline void rte_atomic64_clear(rte_atomic64_t *v) · line 1094

Quoted 10 lines
lib/librte_eal/common/include/generic/rte_atomic.h +10 -0
+ * 128-bit integer structure.
+ */
+RTE_STD_C11
+typedef struct {
+       RTE_STD_C11
+       union {
+               uint64_t val[2];
+#ifdef RTE_ARCH_64
+               __extension__ __int128 int128;
+#endif

You hid this field for x86.
What is the reason?

Quoted 17 lines
lib/librte_eal/common/include/generic/rte_atomic.h +5 -2
+       };
+} __rte_aligned(16) rte_int128_t;
+
 #ifdef __DOXYGEN__
 
 /**
@@ -1093,7 +1107,8 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)
  *     *exp = *dst
  * @endcode
  *
- * @note This function is currently only available for the x86-64 platform.
+ * @note This function is currently available for the x86-64 and aarch64
+ * platforms.
  *
  * @note The success and failure arguments must be one of the __ATOMIC_* values
  * defined in the C++11 standard. For details on their behavior, refer to the
--
Quoted 2 lines

2.7.4

--
David Marchand

#71 lore raw

Hi David,

Thanks for your comments. I have addressed most of them in v10. Please review it. Some comments inline.

Quoted 11 lines

-----Original Message-----
From: David Marchand <[email protected]>
Sent: Monday, October 14, 2019 11:44 PM
To: Phil Yang (Arm Technology China) <[email protected]>
Cc: [email protected]; [email protected]; Gage Eads <[email protected]>; dev <[email protected]>; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v9 1/3] eal/arm64: add 128-bit atomic compare exchange

Quoted 6 lines
Quoted 2 lines · Phil Yang

Add 128-bit atomic compare exchange on aarch64.

A bit short, seeing the complexity of the code and the additional RTE_ARM_FEATURE_ATOMICS config flag.

Updated in v10.

<snip>

Quoted 11 lines

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" {

Quoted 4 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +3 -0
+/*------------------------ 128 bit atomic operations -------------------------*/
+
+#define __HAS_ACQ(mo) ((mo) != __ATOMIC_RELAXED && (mo) !=

__ATOMIC_RELEASE)

Quoted 1 line

+#define __HAS_RLS(mo) ((mo) == __ATOMIC_RELEASE || (mo) ==

__ATOMIC_ACQ_REL || \

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" {

Quoted 3 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +3 -0
+                                         (mo) == __ATOMIC_SEQ_CST)
+
+#define __MO_LOAD(mo)  (__HAS_ACQ((mo)) ? __ATOMIC_ACQUIRE :

__ATOMIC_RELAXED)

Quoted 7 lines
Quoted 1 line

+#define __MO_STORE(mo) (__HAS_RLS((mo)) ? __ATOMIC_RELEASE :

__ATOMIC_RELAXED)

Those 4 first macros only make sense when LSE is not available (see below [1]).
Besides, they are used only once, why not directly use those conditions where needed?

Agree. I removed __MO_LOAD and __MO_STORE in v10 and kept the __HAS_ACQ and __HAS_REL under the non-LSE condition branch in v10. I think they can make the code easy to read.

Quoted 44 lines

Review

Quoted 4 lines
(quoted diff) +2 -0
+
+#if defined(__ARM_FEATURE_ATOMICS) ||

defined(RTE_ARM_FEATURE_ATOMICS)

Quoted 20 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +20 -0
+#define __ATOMIC128_CAS_OP(cas_op_name, op_string)                          \
+static __rte_noinline rte_int128_t                                          \
+cas_op_name(rte_int128_t *dst, rte_int128_t old,                            \
+               rte_int128_t updated)                                       \
+{                                                                           \
+       /* caspX instructions register pair must start from even-numbered
+        * register at operand 1.
+        * So, specify registers for local variables here.
+        */                                                                 \
+       register uint64_t x0 __asm("x0") = (uint64_t)old.val[0];            \
+       register uint64_t x1 __asm("x1") = (uint64_t)old.val[1];            \
+       register uint64_t x2 __asm("x2") = (uint64_t)updated.val[0];        \
+       register uint64_t x3 __asm("x3") = (uint64_t)updated.val[1];        \
+       asm volatile(                                                       \
+               op_string " %[old0], %[old1], %[upd0], %[upd1], [%[dst]]"   \
+               : [old0] "+r" (x0),                                         \
+               [old1] "+r" (x1)                                            \
+               : [upd0] "r" (x2),                                          \
+               [upd1] "r" (x3),                                            \
+               [dst] "r" (dst)                                             \

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 86

Quoted 10 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +10 -0
+               : "memory");                                                \
+       old.val[0] = x0;                                                    \
+       old.val[1] = x1;                                                    \
+       return old;                                                         \
+}
+
+__ATOMIC128_CAS_OP(__rte_cas_relaxed, "casp")
+__ATOMIC128_CAS_OP(__rte_cas_acquire, "caspa")
+__ATOMIC128_CAS_OP(__rte_cas_release, "caspl")
+__ATOMIC128_CAS_OP(__rte_cas_acq_rel, "caspal")

If LSE is available, we expose __rte_cas_XX (explicitely) *non* inlined functions, while without LSE, we expose inlined __rte_ldr_XX and __rte_stx_XX functions.
So we have a first disparity with non-inlined vs inlined functions depending on a #ifdef.
Then, we have a second disparity with two sets of "apis" depending on this #ifdef.

Quoted 5 lines

And we expose those sets with a rte_ prefix, meaning people will try to use them, but those are not part of a public api.

Can't we do without them ? (see below [2] for a proposal with ldr/stx, cas should be the same)

No, it doesn't work.
Because we need to verify the return value at the end of the loop for these macros.

Quoted 35 lines
Quoted 35 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +33 -0
+#else
+#define __ATOMIC128_LDX_OP(ldx_op_name, op_string)                          \
+static inline rte_int128_t                                                  \
+ldx_op_name(const rte_int128_t *src)                                        \
+{                                                                           \
+       rte_int128_t ret;                                                   \
+       asm volatile(                                                       \
+                       op_string " %0, %1, %2"                             \
+                       : "=&r" (ret.val[0]),                               \
+                         "=&r" (ret.val[1])                                \
+                       : "Q" (src->val[0])                                 \
+                       : "memory");                                        \
+       return ret;                                                         \
+}
+
+__ATOMIC128_LDX_OP(__rte_ldx_relaxed, "ldxp")
+__ATOMIC128_LDX_OP(__rte_ldx_acquire, "ldaxp")
+
+#define __ATOMIC128_STX_OP(stx_op_name, op_string)                          \
+static inline uint32_t                                                      \
+stx_op_name(rte_int128_t *dst, const rte_int128_t src)                      \
+{                                                                           \
+       uint32_t ret;                                                       \
+       asm volatile(                                                       \
+                       op_string " %w0, %1, %2, %3"                        \
+                       : "=&r" (ret)                                       \
+                       : "r" (src.val[0]),                                 \
+                         "r" (src.val[1]),                                 \
+                         "Q" (dst->val[0])                                 \
+                       : "memory");                                        \
+       /* Return 0 on success, 1 on failure */                             \
+       return ret;                                                         \
+}
Quoted 8 lines

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 125

Quoted 6 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +6 -0
+
+__ATOMIC128_STX_OP(__rte_stx_relaxed, "stxp")
+__ATOMIC128_STX_OP(__rte_stx_release, "stlxp")
+#endif
+
+static inline int __rte_experimental

The __rte_experimental tag comes first.

Updated in v10.

Quoted 33 lines
Quoted 13 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +11 -0
+rte_atomic128_cmp_exchange(rte_int128_t *dst,
+                               rte_int128_t *exp,
+                               const rte_int128_t *src,
+                               unsigned int weak,
+                               int success,
+                               int failure)
+{
+       /* Always do strong CAS */
+       RTE_SET_USED(weak);
+       /* Ignore memory ordering for failure, memory order for
+        * success must be stronger or equal

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" {

Quoted 10 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +10 -0
+        */
+       RTE_SET_USED(failure);
+       /* Find invalid memory order */
+       RTE_ASSERT(success == __ATOMIC_RELAXED
+                       || success == __ATOMIC_ACQUIRE
+                       || success == __ATOMIC_RELEASE
+                       || success == __ATOMIC_ACQ_REL
+                       || success == __ATOMIC_SEQ_CST);
+
+#if defined(__ARM_FEATURE_ATOMICS) ||

defined(RTE_ARM_FEATURE_ATOMICS)

Quoted 9 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +9 -0
+       rte_int128_t expected = *exp;
+       rte_int128_t desired = *src;
+       rte_int128_t old;
+
+       if (success == __ATOMIC_RELAXED)
+               old = __rte_cas_relaxed(dst, expected, desired);
+       else if (success == __ATOMIC_ACQUIRE)
+               old = __rte_cas_acquire(dst, expected, desired);
+       else if (success == __ATOMIC_RELEASE)
Quoted 8 lines

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 159

Quoted 4 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +4 -0
+               old = __rte_cas_release(dst, expected, desired);
+       else
+               old = __rte_cas_acq_rel(dst, expected, desired);
+#else

1: the four first macros (on the memory ordering constraints) can be moved here then undef'd once unused.
Or you can just do without them.

Updated in v10.

Quoted 143 lines
Quoted 7 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5 -0
+       int ldx_mo = __MO_LOAD(success);
+       int stx_mo = __MO_STORE(success);
+       uint32_t ret = 1;
+       register rte_int128_t expected = *exp;
+       register rte_int128_t desired = *src;

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 174

Quoted 10 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +10 -0
+       register rte_int128_t old;
+
+       /* ldx128 can not guarantee atomic,
+        * Must write back src or old to verify atomicity of ldx128;
+        */
+       do {
+               if (ldx_mo == __ATOMIC_RELAXED)
+                       old = __rte_ldx_relaxed(dst);
+               else
+                       old = __rte_ldx_acquire(dst);

2: how about using a simple macro that gets passed the op string?

Something like (untested):

#define __READ_128(op_string, src, dst) \
    asm volatile(                      \
        op_string " %0, %1, %2"    \
        : "=&r" (dst.val[0]),      \
          "=&r" (dst.val[1])       \
        : "Q" (src->val[0])        \
        : "memory")

Then used like this:

        if (ldx_mo == __ATOMIC_RELAXED)
            __READ_128("ldxp", dst, old);
        else
            __READ_128("ldaxp", dst, old);

#undef __READ_128

Quoted 7 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +7 -0
+
+               if (likely(old.int128 == expected.int128)) {
+                       if (stx_mo == __ATOMIC_RELAXED)
+                               ret = __rte_stx_relaxed(dst, desired);
+                       else
+                               ret = __rte_stx_release(dst, desired);
+               } else {

lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 191

Quoted 10 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +10 -0
+                       /* In the failure case (since 'weak' is ignored and only
+                        * weak == 0 is implemented), expected should contain
+                        * the atomically read value of dst. This means, 'old'
+                        * needs to be stored back to ensure it was read
+                        * atomically.
+                        */
+                       if (stx_mo == __ATOMIC_RELAXED)
+                               ret = __rte_stx_relaxed(dst, old);
+                       else
+                               ret = __rte_stx_release(dst, old);

And:

#define __STORE_128(op_string, dst, val, ret) \
    asm volatile(                        \
        op_string " %w0, %1, %2, %3"     \
        : "=&r" (ret)                    \
        : "r" (val.val[0]),              \
          "r" (val.val[1]),              \
          "Q" (dst->val[0])              \
        : "memory")

Used like this:

        if (likely(old.int128 == expected.int128)) {
            if (stx_mo == __ATOMIC_RELAXED)
                __STORE_128("stxp", dst, desired, ret);
            else
                __STORE_128("stlxp", dst, desired, ret);
        } else {
            /* In the failure case (since 'weak' is ignored and only
             * weak == 0 is implemented), expected should contain
             * the atomically read value of dst. This means, 'old'
             * needs to be stored back to ensure it was read
             * atomically.
             */
            if (stx_mo == __ATOMIC_RELAXED)
                __STORE_128("stxp", dst, old, ret);
            else
                __STORE_128("stlxp", dst, old, ret);
        }

#undef __STORE_128

Review

Quoted 18 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +14 -0
+               }
+       } while (unlikely(ret));
+#endif
+
+       /* Unconditionally updating expected removes
+        * an 'if' statement.
+        * expected should already be in register if
+        * not in the cache.
+        */
+       *exp = old;
+
+       return (old.int128 == expected.int128);
+}
+
 #ifdef __cplusplus
 }
 #endif
(quoted diff) +0 -0

b/lib/librte_eal/common/include/arch/x86/rte_atomic_64.h

lib/librte_eal/common/include/arch/x86/rte_atomic_64.h static inline void

Quoted 4 lines

index 1335d92..cfe7067 100644

lib/librte_eal/common/include/arch/x86/rte_atomic_64.h +0 -0
@@ -183,18 +183,6 @@ static inline void

rte_atomic64_clear(rte_atomic64_t *v)

Review

Quoted 19 lines
lib/librte_eal/common/include/arch/x86/rte_atomic_64.h +0 -12
 /*------------------------ 128 bit atomic operations -------------------------*/
 
-/**
- * 128-bit integer structure.
- */
-RTE_STD_C11
-typedef struct {
-       RTE_STD_C11
-       union {
-               uint64_t val[2];
-               __extension__ __int128 int128;
-       };
-} __rte_aligned(16) rte_int128_t;
-
 __rte_experimental
 static inline int
 rte_atomic128_cmp_exchange(rte_int128_t *dst,
(quoted diff) +0 -0

b/lib/librte_eal/common/include/generic/rte_atomic.h

lib/librte_eal/common/include/generic/rte_atomic.h static inline void

Quoted 4 lines

index 24ff7dc..e6ab15a 100644

lib/librte_eal/common/include/generic/rte_atomic.h +0 -0
@@ -1081,6 +1081,20 @@ static inline void

rte_atomic64_clear(rte_atomic64_t *v)

Quoted 3 lines
lib/librte_eal/common/include/generic/rte_atomic.h +0 -0
 /*------------------------ 128 bit atomic operations -------------------------*/
 
Quoted 14 lines

lib/librte_eal/common/include/generic/rte_atomic.h static inline void rte_atomic64_clear(rte_atomic64_t *v) · line 1094

Quoted 11 lines
lib/librte_eal/common/include/generic/rte_atomic.h +11 -0
+/**
+ * 128-bit integer structure.
+ */
+RTE_STD_C11
+typedef struct {
+       RTE_STD_C11
+       union {
+               uint64_t val[2];
+#ifdef RTE_ARCH_64
+               __extension__ __int128 int128;
+#endif

You hid this field for x86.
What is the reason?

No, we are not hid it for x86. The RTE_ARCH_64 flag covered x86 as well.

Thanks,
Phil

#72 lore raw
Quoted 7 lines · Phil Yang (Arm Technology China)
Quoted 7 lines

-----Original Message-----
From: David Marchand <[email protected]>
If LSE is available, we expose __rte_cas_XX (explicitely) *non* inlined functions, while without LSE, we expose inlined __rte_ldr_XX and __rte_stx_XX functions.
So we have a first disparity with non-inlined vs inlined functions depending on a #ifdef.

You did not comment on the inline / no inline part and I still see this in the v10.
Is this __rte_noinline on the CAS function intentional?

Quoted 11 lines
Quoted 8 lines

Then, we have a second disparity with two sets of "apis" depending on this #ifdef.

And we expose those sets with a rte_ prefix, meaning people will try to use them, but those are not part of a public api.

Can't we do without them ? (see below [2] for a proposal with ldr/stx, cas should be the same)

No, it doesn't work.
Because we need to verify the return value at the end of the loop for these macros.

Do you mean the return value for the stores?

Quoted 8 lines
Quoted 8 lines
#define __STORE_128(op_string, dst, val, ret) \
    asm volatile(                        \
        op_string " %w0, %1, %2, %3"     \
        : "=&r" (ret)                    \
        : "r" (val.val[0]),              \
          "r" (val.val[1]),              \
          "Q" (dst->val[0])              \
        : "memory")

The ret variable is still passed in this macro and the while loop can check it later.

Quoted 24 lines
Quoted 10 lines

Review

Quoted 1 line
(quoted diff) +0 -0

b/lib/librte_eal/common/include/generic/rte_atomic.h

lib/librte_eal/common/include/generic/rte_atomic.h static inline void

Quoted 4 lines

index 24ff7dc..e6ab15a 100644

lib/librte_eal/common/include/generic/rte_atomic.h +0 -0
@@ -1081,6 +1081,20 @@ static inline void

rte_atomic64_clear(rte_atomic64_t *v)

Quoted 3 lines
lib/librte_eal/common/include/generic/rte_atomic.h +0 -0
 /*------------------------ 128 bit atomic operations -------------------------*/
 
Quoted 14 lines

lib/librte_eal/common/include/generic/rte_atomic.h static inline void rte_atomic64_clear(rte_atomic64_t *v) · line 1094

Quoted 11 lines
lib/librte_eal/common/include/generic/rte_atomic.h +11 -0
+/**
+ * 128-bit integer structure.
+ */
+RTE_STD_C11
+typedef struct {
+       RTE_STD_C11
+       union {
+               uint64_t val[2];
+#ifdef RTE_ARCH_64
+               __extension__ __int128 int128;
+#endif

You hid this field for x86.
What is the reason?

Quoted 1 line

No, we are not hid it for x86. The RTE_ARCH_64 flag covered x86 as well.

Ah indeed, I read it wrong, ARCH_64 ... AARCH64 ... :-)

--
David Marchand

#73 lore raw
Quoted 21 lines

-----Original Message-----
From: David Marchand <[email protected]>
Sent: Tuesday, October 15, 2019 8:16 PM
To: Phil Yang (Arm Technology China) <[email protected]>
Cc: [email protected]; [email protected]; Gage Eads <[email protected]>; dev <[email protected]>; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v9 1/3] eal/arm64: add 128-bit atomic compare exchange

Quoted 7 lines · Phil Yang (Arm Technology China)
Quoted 7 lines

-----Original Message-----
From: David Marchand <[email protected]>
If LSE is available, we expose __rte_cas_XX (explicitely) *non* inlined functions, while without LSE, we expose inlined __rte_ldr_XX and __rte_stx_XX functions.
So we have a first disparity with non-inlined vs inlined functions depending on a #ifdef.

Quoted 3 lines

You did not comment on the inline / no inline part and I still see this in the v10.
Is this __rte_noinline on the CAS function intentional?

Apologize for missing this item. Yes, it is to avoid ABI break.
Please check
5b40ec6b966260e0ff66a8a2c689664f75d6a0e6 ("mempool/octeontx2: fix possible arm64 ABI break")

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

Then, we have a second disparity with two sets of "apis" depending on this #ifdef.

And we expose those sets with a rte_ prefix, meaning people will try to use them, but those are not part of a public api.

Quoted 8 lines
Quoted 5 lines
Quoted 2 lines

Can't we do without them ? (see below [2] for a proposal with ldr/stx, cas should be the same)

No, it doesn't work.
Because we need to verify the return value at the end of the loop for these

macros.

Do you mean the return value for the stores?

It is my bad. I missed the ret option in the macro. This approach works.

However, I suggest to keep them as static inline functions rather than a piece of macro in the rte_atomic128_cmp_exchange API.
One reason is APIs name can indicate the memory ordering of these operations.
Moreover, it uses the register type to pass the value in the inline function, so it should not have too much cost comparing with the macro.
I also think these 128bit load and store functions can be used in other places, once it has been proved valuable in rte_atomic128_cmp_exchange API. But let's keep them private for the current stage. BTW, Linux kernel implemented in the same way. https://github.com/torvalds/linux/blob/master/arch/arm64/include/asm/atomic_lse.h#L19

Quoted 45 lines
Quoted 8 lines
Quoted 8 lines
#define __STORE_128(op_string, dst, val, ret) \
    asm volatile(                        \
        op_string " %w0, %1, %2, %3"     \
        : "=&r" (ret)                    \
        : "r" (val.val[0]),              \
          "r" (val.val[1]),              \
          "Q" (dst->val[0])              \
        : "memory")

The ret variable is still passed in this macro and the while loop can check it later.

Quoted 24 lines
Quoted 10 lines

Review

Quoted 1 line
(quoted diff) +0 -0

b/lib/librte_eal/common/include/generic/rte_atomic.h

lib/librte_eal/common/include/generic/rte_atomic.h static inline void

Quoted 4 lines

index 24ff7dc..e6ab15a 100644

lib/librte_eal/common/include/generic/rte_atomic.h +0 -0
@@ -1081,6 +1081,20 @@ static inline void

rte_atomic64_clear(rte_atomic64_t *v)

Quoted 3 lines
lib/librte_eal/common/include/generic/rte_atomic.h +0 -0
 /*------------------------ 128 bit atomic operations -------------------------*/
 
Quoted 14 lines

lib/librte_eal/common/include/generic/rte_atomic.h static inline void rte_atomic64_clear(rte_atomic64_t *v) · line 1094

Quoted 11 lines
lib/librte_eal/common/include/generic/rte_atomic.h +11 -0
+/**
+ * 128-bit integer structure.
+ */
+RTE_STD_C11
+typedef struct {
+       RTE_STD_C11
+       union {
+               uint64_t val[2];
+#ifdef RTE_ARCH_64
+               __extension__ __int128 int128;
+#endif

You hid this field for x86.
What is the reason?

Quoted 1 line

No, we are not hid it for x86. The RTE_ARCH_64 flag covered x86 as well.

Ah indeed, I read it wrong, ARCH_64 ... AARCH64 ... :-)

--
David Marchand

#74 lore raw
Quoted 22 lines · Phil Yang (Arm Technology China)
Quoted 22 lines

-----Original Message-----
From: David Marchand <[email protected]>
Sent: Tuesday, October 15, 2019 8:16 PM
To: Phil Yang (Arm Technology China) <[email protected]>
Cc: [email protected]; [email protected]; Gage Eads <[email protected]>; dev <[email protected]>; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v9 1/3] eal/arm64: add 128-bit atomic compare exchange

Quoted 7 lines · Phil Yang (Arm Technology China)
Quoted 7 lines

-----Original Message-----
From: David Marchand <[email protected]>
If LSE is available, we expose __rte_cas_XX (explicitely) *non* inlined functions, while without LSE, we expose inlined __rte_ldr_XX and __rte_stx_XX functions.
So we have a first disparity with non-inlined vs inlined functions depending on a #ifdef.

Quoted 7 lines
Quoted 3 lines

You did not comment on the inline / no inline part and I still see this in the v10.
Is this __rte_noinline on the CAS function intentional?

Apologize for missing this item. Yes, it is to avoid ABI break.
Please check
5b40ec6b966260e0ff66a8a2c689664f75d6a0e6 ("mempool/octeontx2: fix possible arm64 ABI break")

Looked at the kernel parts on LSE CAS (thanks for the pointer) but I see inlines are used:
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/arch/arm64/include/asm/atomic_lse.h#n365?h=v5.4-rc3

What is special in the kernel or in dpdk that makes this different?

Quoted 12 lines
Quoted 12 lines
Quoted 12 lines
Quoted 11 lines

Then, we have a second disparity with two sets of "apis" depending on this #ifdef.

And we expose those sets with a rte_ prefix, meaning people will try to use them, but those are not part of a public api.

Can't we do without them ? (see below [2] for a proposal with ldr/stx, cas should be the same)

Quoted 7 lines
Quoted 5 lines
Quoted 2 lines

No, it doesn't work.
Because we need to verify the return value at the end of the loop for these

macros.

Do you mean the return value for the stores?

It is my bad. I missed the ret option in the macro. This approach works.

Ok, thanks for confirming.

Quoted 3 lines

However, I suggest to keep them as static inline functions rather than a piece of macro in the rte_atomic128_cmp_exchange API. One reason is APIs name can indicate the memory ordering of these operations.

API?
Those inlines are not part of a public API and we agree this patch is not about adding 128 bits load/store apis.

My proposal gives us small code that looks like:

        if (ldx_mo == __ATOMIC_RELAXED)
            __READ_128("ldxp", dst, old);
        else
            __READ_128("ldaxp", dst, old);

I am not a memory order guru, but with this, I can figure the asm instruction depends on it.
And, since we are looking at internals of an implementation, this is mainly for people looking at/maintaining these low level details.

Quoted 1 line

Moreover, it uses the register type to pass the value in the inline function, so it should not have too much cost comparing with the macro.

This is not a problem of cost, this is about hiding architecture details from the final user.
If you expose something, you can expect someone will start using it and will complain later if you break it.

Quoted 1 line

I also think these 128bit load and store functions can be used in other places, once it has been proved valuable in rte_atomic128_cmp_exchange API. But let's keep them private for the current stage.

Yes I agree this could be introduced in the future.

Quoted 1 line

BTW, Linux kernel implemented in the same way. https://github.com/torvalds/linux/blob/master/arch/arm64/include/asm/atomic_lse.h#L19

Ok kernel exposes its internals, but I think kernel developpers are more vigilant than dpdk developpers on what is part of the public API and what is internal.

--
David Marchand

author
#75 lore raw

PATCH v10 1/3 eal/arm64: add 128-bit atomic compare exchange

In reply to Phil Yang

This patch adds the implementation of the 128-bit atomic compare exchange API on AArch64. Using 64-bit 'ldxp/stxp' instructions can perform this operation. Moreover, on the LSE atomic extension accelerated platforms, it implemented by 'casp' instructions for better performance.

Since the '__ARM_FEATURE_ATOMICS' flag only supports GCC-9, so this patch adds a new config flag 'RTE_ARM_FEATURE_ATOMICS' to enable the 'cas' version on elder version compilers.

---
v10:
1.Removed all the rte tag for internal functions. 2.Removed __MO_LOAD and _MO_STORE macros and keep define __HAS_ACQ and __HAS_REL under non LSE conditional branch.
3.Undef the macro once it is unused.
4.Reword the 1/3 and 2/3 patches' commitlog more specific.

v9:
Updated 19.11 release note.

v8:
Fixed "WARNING:LONG_LINE: line over 80 characters" warnings with latest kernel checkpatch.pl

v7:
1. Adjust code comment.

v6:
1. Put the RTE_ARM_FEATURE_ATOMICS flag into EAL group. (Jerin Jocob)
2. Keep rte_stack_lf_stubs.h doing nothing. (Gage Eads)
3. Fixed 32 bit build issue.

v5:
1. Enable RTE_ARM_FEATURE_ATOMICS on octeontx2 in default. (Jerin Jocob)
2. Record the reason of introducing "rte_stack_lf_stubs.h" in git commit.
(Jerin, Jocob)
3. Fixed a conditional MACRO error in rte_atomic128_cmp_exchange. (Jerin Jocob)

v4:
1. Add RTE_ARM_FEATURE_ATOMICS flag to support LSE CASP instructions. (Jerin Jocob)
2. Fix possible arm64 ABI break by making casp_op_name noinline. (Jerin Jocob)
3. Add rte_stack_lf_stubs.h to reduce the ifdef clutter. (Gage Eads/Jerin Jocob)

v3:
1. Avoid duplication code with macro. (Jerin Jocob)
2. Make invalid memory order to strongest barrier. (Jerin Jocob)
3. Update doc/guides/prog_guide/env_abstraction_layer.rst. (Gage Eads)
4. Fix 32-bit x86 builds issue. (Gage Eads)
5. Correct documentation issues in UT. (Gage Eads)

v2:
Initial version.

 config/arm/meson.build                             |   2 +
 config/common_base                                 |   3 +
 config/defconfig_arm64-octeontx2-linuxapp-gcc      |   1 +
 config/defconfig_arm64-thunderx2-linuxapp-gcc      |   1 +
 .../common/include/arch/arm/rte_atomic_64.h        | 173 +++++++++++++++++++++
 .../common/include/arch/x86/rte_atomic_64.h        |  12 --
 lib/librte_eal/common/include/generic/rte_atomic.h |  17 +-
 7 files changed, 196 insertions(+), 13 deletions(-)
config/arm/meson.build +2 -0
index 979018e..9f28271 100644
@@ -71,11 +71,13 @@ flags_thunderx2_extra = [
 	['RTE_CACHE_LINE_SIZE', 64],
 	['RTE_MAX_NUMA_NODES', 2],
 	['RTE_MAX_LCORE', 256],
+	['RTE_ARM_FEATURE_ATOMICS', true],
 	['RTE_USE_C11_MEM_MODEL', true]]
 flags_octeontx2_extra = [
 	['RTE_MACHINE', '"octeontx2"'],
 	['RTE_MAX_NUMA_NODES', 1],
 	['RTE_MAX_LCORE', 24],
+	['RTE_ARM_FEATURE_ATOMICS', true],
 	['RTE_EAL_IGB_UIO', false],
 	['RTE_USE_C11_MEM_MODEL', true]]
 
config/common_base +3 -0
index e843a21..a96beb9 100644
@@ -82,6 +82,9 @@ CONFIG_RTE_MAX_LCORE=128
 CONFIG_RTE_MAX_NUMA_NODES=8
 CONFIG_RTE_MAX_HEAPS=32
 CONFIG_RTE_MAX_MEMSEG_LISTS=64
+
+# Use ARM LSE ATOMIC instructions
+CONFIG_RTE_ARM_FEATURE_ATOMICS=n
 # each memseg list will be limited to either RTE_MAX_MEMSEG_PER_LIST pages
 # or RTE_MAX_MEM_MB_PER_LIST megabytes worth of memory, whichever is smaller
 CONFIG_RTE_MAX_MEMSEG_PER_LIST=8192
config/defconfig_arm64-octeontx2-linuxapp-gcc +1 -0
index f20da24..7687dbe 100644
@@ -9,6 +9,7 @@ CONFIG_RTE_MACHINE="octeontx2"
 CONFIG_RTE_CACHE_LINE_SIZE=128
 CONFIG_RTE_MAX_NUMA_NODES=1
 CONFIG_RTE_MAX_LCORE=24
+CONFIG_RTE_ARM_FEATURE_ATOMICS=y
 
 # Doesn't support NUMA
 CONFIG_RTE_EAL_NUMA_AWARE_HUGEPAGES=n
config/defconfig_arm64-thunderx2-linuxapp-gcc +1 -0
index cc5c64b..af4a89c 100644
@@ -9,3 +9,4 @@ CONFIG_RTE_MACHINE="thunderx2"
 CONFIG_RTE_CACHE_LINE_SIZE=64
 CONFIG_RTE_MAX_NUMA_NODES=2
 CONFIG_RTE_MAX_LCORE=256
+CONFIG_RTE_ARM_FEATURE_ATOMICS=y
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +173 -0

190 lines not on this page · raw message

lib/librte_eal/common/include/arch/x86/rte_atomic_64.h +0 -12
index 1335d92..cfe7067 100644
@@ -183,18 +183,6 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)
 
 /*------------------------ 128 bit atomic operations -------------------------*/
 
-/**
- * 128-bit integer structure.
- */
-RTE_STD_C11
-typedef struct {
-	RTE_STD_C11
-	union {
-		uint64_t val[2];
-		__extension__ __int128 int128;
-	};
-} __rte_aligned(16) rte_int128_t;
-
 __rte_experimental
 static inline int
 rte_atomic128_cmp_exchange(rte_int128_t *dst,
lib/librte_eal/common/include/generic/rte_atomic.h +16 -1
index 24ff7dc..e6ab15a 100644
@@ -1081,6 +1081,20 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)
 
 /*------------------------ 128 bit atomic operations -------------------------*/
 
+/**
+ * 128-bit integer structure.
+ */
+RTE_STD_C11
+typedef struct {
+	RTE_STD_C11
+	union {
+		uint64_t val[2];
+#ifdef RTE_ARCH_64
+		__extension__ __int128 int128;
+#endif
+	};
+} __rte_aligned(16) rte_int128_t;
+
 #ifdef __DOXYGEN__
 
 /**
@@ -1093,7 +1107,8 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)
  *     *exp = *dst
  * @endcode
  *
- * @note This function is currently only available for the x86-64 platform.
+ * @note This function is currently available for the x86-64 and aarch64
+ * platforms.
  *
  * @note The success and failure arguments must be one of the __ATOMIC_* values
  * defined in the C++11 standard. For details on their behavior, refer to the
Signature · 1 line
2.7.4
author
#76 lore raw

PATCH v10 2/3 test/atomic: add 128b compare and swap test

Add 128b atomic compare and swap test for aarch64 and x86_64. Extend the test iteration from 10 thousand to 1 million times to test the stability of the atomic APIs.

---
 app/test/test_atomic.c | 125 ++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 123 insertions(+), 2 deletions(-)
app/test/test_atomic.c +123 -2
index 43be30e..14bd3a8 100644
@@ -1,5 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright(c) 2010-2014 Intel Corporation
+ * Copyright(c) 2019 Arm Limited
  */
 
 #include <stdio.h>
@@ -20,7 +21,7 @@
  * Atomic Variables
  * ================
  *
- * - The main test function performs three subtests. The first test
+ * - The main test function performs four subtests. The first test
  *   checks that the usual inc/dec/add/sub functions are working
  *   correctly:
  *
@@ -61,11 +62,27 @@
  *       atomic_sub(&count, tmp+1);
  *
  *   - At the end of the test, the *count* value must be 0.
+ *
+ * - Test "128b compare and swap" (aarch64 and x86_64 only)
+ *
+ *   - Initialize 128-bit atomic variables to zero.
+ *
+ *   - Invoke ``test_atomic128_cmp_exchange()`` on each lcore. Before doing
+ *     anything else, the cores are waiting a synchro. Each lcore does
+ *     these compare and swap (CAS) operations several times::
+ *
+ *       Acquired CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *       Released CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *       Acquired_Released CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *       Relaxed CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *
+ *   - At the end of the test, the *count128* first 64-bit value and
+ *     second 64-bit value differ by the total iterations.
  */
 
 #define NUM_ATOMIC_TYPES 3
 
-#define N 10000
+#define N 1000000
 
 static rte_atomic16_t a16;
 static rte_atomic32_t a32;
@@ -216,6 +233,78 @@ test_atomic_dec_and_test(__attribute__((unused)) void *arg)
 	return 0;
 }
 
+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)
+static rte_int128_t count128;
+
+/*
+ * rte_atomic128_cmp_exchange() should update a 128 bits counter's first 64
+ * bits by 2 and the second 64 bits by 1 in this test. It should return true
+ * if the compare exchange operation is successful.
+ * This test repeats 128 bits compare and swap operations N rounds. In each
+ * iteration it runs compare and swap operation with different memory models.
+ */
+static int
+test_atomic128_cmp_exchange(__attribute__((unused)) void *arg)
+{
+	rte_int128_t expected;
+	int success;
+	unsigned int i;
+
+	while (rte_atomic32_read(&synchro) == 0)
+		;
+
+	expected = count128;
+
+	for (i = 0; i < N; i++) {
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128,
+				&expected, &desired, 1,
+				__ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
+		} while (success == 0);
+
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128,
+					&expected, &desired, 1,
+					__ATOMIC_RELEASE, __ATOMIC_RELAXED);
+		} while (success == 0);
+
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128,
+					&expected, &desired, 1,
+					__ATOMIC_ACQ_REL, __ATOMIC_RELAXED);
+		} while (success == 0);
+
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128,
+					&expected, &desired, 1,
+					__ATOMIC_RELAXED, __ATOMIC_RELAXED);
+		} while (success == 0);
+	}
+
+	return 0;
+}
+#endif
+
 static int
 test_atomic(void)
 {
@@ -340,6 +429,38 @@ test_atomic(void)
 		return -1;
 	}
 
+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)
+	/*
+	 * This case tests the functionality of rte_atomic128b_cmp_exchange
+	 * API. It calls rte_atomic128b_cmp_exchange with four kinds of memory
+	 * models successively on each slave core. Once each 128-bit atomic
+	 * compare and swap operation is successful, it updates the global
+	 * 128-bit counter by 2 for the first 64-bit and 1 for the second
+	 * 64-bit. Each slave core iterates this test N times.
+	 * At the end of test, verify whether the first 64-bits of the 128-bit
+	 * counter and the second 64bits is differ by the total iterations. If
+	 * it is, the test passes.
+	 */
+	printf("128b compare and swap test\n");
+	uint64_t iterations = 0;
+
+	rte_atomic32_clear(&synchro);
+	count128.val[0] = 0;
+	count128.val[1] = 0;
+
+	rte_eal_mp_remote_launch(test_atomic128_cmp_exchange, NULL,
+				 SKIP_MASTER);
+	rte_atomic32_set(&synchro, 1);
+	rte_eal_mp_wait_lcore();
+	rte_atomic32_clear(&synchro);
+
+	iterations = count128.val[0] - count128.val[1];
+	if (iterations != 4*N*(rte_lcore_count()-1)) {
+		printf("128b compare and swap failed\n");
+		return -1;
+	}
+#endif
+
 	return 0;
 }
 
Signature · 1 line
2.7.4
author
#77 lore raw

PATCH v10 3/3 eal/stack: enable lock-free stack for aarch64

In reply to Phil Yang

Enable both c11 atomic and non c11 atomic lock-free stack for aarch64.

Introduced a new header to reduce the ifdef clutter across generic and c11 files. The rte_stack_lf_stubs.h contains stub implementations of __rte_stack_lf_count, __rte_stack_lf_push_elems and __rte_stack_lf_pop_elems.

---
 doc/guides/prog_guide/env_abstraction_layer.rst |  4 +--
 doc/guides/rel_notes/release_19_11.rst          |  3 ++
 lib/librte_stack/Makefile                       |  3 +-
 lib/librte_stack/rte_stack_lf.h                 |  4 +++
 lib/librte_stack/rte_stack_lf_c11.h             | 16 ---------
 lib/librte_stack/rte_stack_lf_generic.h         | 16 ---------
 lib/librte_stack/rte_stack_lf_stubs.h           | 44 +++++++++++++++++++++++++
 7 files changed, 55 insertions(+), 35 deletions(-)
 create mode 100644 lib/librte_stack/rte_stack_lf_stubs.h
doc/guides/prog_guide/env_abstraction_layer.rst +2 -2
index 94f30fd..6e59fae 100644
@@ -648,8 +648,8 @@ Known Issues
   Alternatively, applications can use the lock-free stack mempool handler. When
   considering this handler, note that:
 
-  - It is currently limited to the x86_64 platform, because it uses an
-    instruction (16-byte compare-and-swap) that is not yet available on other
+  - It is currently limited to the aarch64 and x86_64 platforms, because it uses
+    an instruction (16-byte compare-and-swap) that is not yet available on other
     platforms.
   - It has worse average-case performance than the non-preemptive rte_ring, but
     software caching (e.g. the mempool cache) can mitigate this by reducing the
doc/guides/rel_notes/release_19_11.rst +3 -0
index 85953b9..4f82f54 100644
@@ -115,6 +115,9 @@ New Features
   Added eBPF JIT support for arm64 architecture to improve the eBPF program
   performance.
 
+* **Added Lock-free Stack for aarch64.**
+
+  The lock-free stack implementation is enabled for aarch64 platforms.
 
 Removed Items
 -------------
lib/librte_stack/Makefile +2 -1
index 8d18ce5..b5e5bed 100644
@@ -24,6 +24,7 @@ SYMLINK-$(CONFIG_RTE_LIBRTE_STACK)-include := rte_stack.h \
 					      rte_stack_std.h \
 					      rte_stack_lf.h \
 					      rte_stack_lf_generic.h \
-					      rte_stack_lf_c11.h
+					      rte_stack_lf_c11.h \
+					      rte_stack_lf_stubs.h
 
 include $(RTE_SDK)/mk/rte.lib.mk
lib/librte_stack/rte_stack_lf.h +4 -0
index f5581f0..e67630c 100644
@@ -5,11 +5,15 @@
 #ifndef _RTE_STACK_LF_H_
 #define _RTE_STACK_LF_H_
 
+#if !(defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64))
+#include "rte_stack_lf_stubs.h"
+#else
 #ifdef RTE_USE_C11_MEM_MODEL
 #include "rte_stack_lf_c11.h"
 #else
 #include "rte_stack_lf_generic.h"
 #endif
+#endif
 
 /**
  * @internal Push several objects on the lock-free stack (MT-safe).
lib/librte_stack/rte_stack_lf_c11.h +0 -16
index 3d677ae..999359f 100644
@@ -36,12 +36,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 			  struct rte_stack_lf_elem *last,
 			  unsigned int num)
 {
-#ifndef RTE_ARCH_X86_64
-	RTE_SET_USED(first);
-	RTE_SET_USED(last);
-	RTE_SET_USED(list);
-	RTE_SET_USED(num);
-#else
 	struct rte_stack_lf_head old_head;
 	int success;
 
@@ -79,7 +73,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 	 * to the LIFO len update.
 	 */
 	__atomic_add_fetch(&list->len, num, __ATOMIC_RELEASE);
-#endif
 }
 
 static __rte_always_inline struct rte_stack_lf_elem *
@@ -88,14 +81,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 void **obj_table,
 			 struct rte_stack_lf_elem **last)
 {
-#ifndef RTE_ARCH_X86_64
-	RTE_SET_USED(obj_table);
-	RTE_SET_USED(last);
-	RTE_SET_USED(list);
-	RTE_SET_USED(num);
-
-	return NULL;
-#else
 	struct rte_stack_lf_head old_head;
 	uint64_t len;
 	int success;
@@ -169,7 +154,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 	} while (success == 0);
 
 	return old_head.top;
-#endif
 }
 
 #endif /* _RTE_STACK_LF_C11_H_ */
lib/librte_stack/rte_stack_lf_generic.h +0 -16
index 3182151..3abbb53 100644
@@ -36,12 +36,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 			  struct rte_stack_lf_elem *last,
 			  unsigned int num)
 {
-#ifndef RTE_ARCH_X86_64
-	RTE_SET_USED(first);
-	RTE_SET_USED(last);
-	RTE_SET_USED(list);
-	RTE_SET_USED(num);
-#else
 	struct rte_stack_lf_head old_head;
 	int success;
 
@@ -75,7 +69,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 	} while (success == 0);
 
 	rte_atomic64_add((rte_atomic64_t *)&list->len, num);
-#endif
 }
 
 static __rte_always_inline struct rte_stack_lf_elem *
@@ -84,14 +77,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 void **obj_table,
 			 struct rte_stack_lf_elem **last)
 {
-#ifndef RTE_ARCH_X86_64
-	RTE_SET_USED(obj_table);
-	RTE_SET_USED(last);
-	RTE_SET_USED(list);
-	RTE_SET_USED(num);
-
-	return NULL;
-#else
 	struct rte_stack_lf_head old_head;
 	int success;
 
@@ -159,7 +144,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 	} while (success == 0);
 
 	return old_head.top;
-#endif
 }
 
 #endif /* _RTE_STACK_LF_GENERIC_H_ */
lib/librte_stack/rte_stack_lf_stubs.h new file +44 -0

44 lines not on this page · raw message

Signature · 1 line
2.7.4
author
#78 lore raw

PATCH v11 1/3 eal/arm64: add 128-bit atomic compare exchange

In reply to Phil Yang

This patch adds the implementation of the 128-bit atomic compare exchange API on AArch64. Using 64-bit 'ldxp/stxp' instructions can perform this operation. Moreover, on the LSE atomic extension accelerated platforms, it implemented by 'casp' instructions for better performance.

Since the '__ARM_FEATURE_ATOMICS' flag only supports GCC-9, so this patch adds a new config flag 'RTE_ARM_FEATURE_ATOMICS' to enable the 'cas' version on elder version compilers.

Since direct x0 register used in the code and cas_op_name() and rte_atomic128_cmp_exchange() is inline function, based on parent function load, it may corrupt x0 register aka Break arm64 ABI. Define CAS operations as rte_noinline functions to avoid the ABI break[1].

[1]5b40ec6b9662 ("mempool/octeontx2: fix possible arm64 ABI break").

---
v11:
1. Renamed cas_op_name by adding the data width tag 128.
2. Replaced __ldx/__stx inline functions with macros.
3. Illustrate the reason of define cas operations as non-inline functions in commitlog.

v10:
1.Removed all the rte tag for internal functions. 2.Removed __MO_LOAD and _MO_STORE macros and keep define __HAS_ACQ and __HAS_REL under non LSE conditional branch.
3.Undef the macro once it is unused.
4.Reword the 1/3 and 2/3 patches' commitlog more specific.

v9:
Updated 19.11 release note.

v8:
Fixed "WARNING:LONG_LINE: line over 80 characters" warnings with latest kernel checkpatch.pl

v7:
1. Adjust code comment.

v6:
1. Put the RTE_ARM_FEATURE_ATOMICS flag into EAL group. (Jerin Jocob)
2. Keep rte_stack_lf_stubs.h doing nothing. (Gage Eads)
3. Fixed 32 bit build issue.

v5:
1. Enable RTE_ARM_FEATURE_ATOMICS on octeontx2 in default. (Jerin Jocob)
2. Record the reason of introducing "rte_stack_lf_stubs.h" in git commit.
(Jerin, Jocob)
3. Fixed a conditional MACRO error in rte_atomic128_cmp_exchange. (Jerin Jocob)

v4:
1. Add RTE_ARM_FEATURE_ATOMICS flag to support LSE CASP instructions. (Jerin Jocob)
2. Fix possible arm64 ABI break by making casp_op_name noinline. (Jerin Jocob)
3. Add rte_stack_lf_stubs.h to reduce the ifdef clutter. (Gage Eads/Jerin Jocob)

v3:
1. Avoid duplication code with macro. (Jerin Jocob)
2. Make invalid memory order to strongest barrier. (Jerin Jocob)
3. Update doc/guides/prog_guide/env_abstraction_layer.rst. (Gage Eads)
4. Fix 32-bit x86 builds issue. (Gage Eads)
5. Correct documentation issues in UT. (Gage Eads)

v2:
Initial version.

 config/arm/meson.build                             |   2 +
 config/common_base                                 |   3 +
 config/defconfig_arm64-octeontx2-linuxapp-gcc      |   1 +
 config/defconfig_arm64-thunderx2-linuxapp-gcc      |   1 +
 .../common/include/arch/arm/rte_atomic_64.h        | 151 +++++++++++++++++++++
 .../common/include/arch/x86/rte_atomic_64.h        |  12 --
 lib/librte_eal/common/include/generic/rte_atomic.h |  17 ++-
 7 files changed, 174 insertions(+), 13 deletions(-)
config/arm/meson.build +2 -0
index 979018e..9f28271 100644
@@ -71,11 +71,13 @@ flags_thunderx2_extra = [
 	['RTE_CACHE_LINE_SIZE', 64],
 	['RTE_MAX_NUMA_NODES', 2],
 	['RTE_MAX_LCORE', 256],
+	['RTE_ARM_FEATURE_ATOMICS', true],
 	['RTE_USE_C11_MEM_MODEL', true]]
 flags_octeontx2_extra = [
 	['RTE_MACHINE', '"octeontx2"'],
 	['RTE_MAX_NUMA_NODES', 1],
 	['RTE_MAX_LCORE', 24],
+	['RTE_ARM_FEATURE_ATOMICS', true],
 	['RTE_EAL_IGB_UIO', false],
 	['RTE_USE_C11_MEM_MODEL', true]]
 
config/common_base +3 -0
index e843a21..a96beb9 100644
@@ -82,6 +82,9 @@ CONFIG_RTE_MAX_LCORE=128
 CONFIG_RTE_MAX_NUMA_NODES=8
 CONFIG_RTE_MAX_HEAPS=32
 CONFIG_RTE_MAX_MEMSEG_LISTS=64
+
+# Use ARM LSE ATOMIC instructions
+CONFIG_RTE_ARM_FEATURE_ATOMICS=n
 # each memseg list will be limited to either RTE_MAX_MEMSEG_PER_LIST pages
 # or RTE_MAX_MEM_MB_PER_LIST megabytes worth of memory, whichever is smaller
 CONFIG_RTE_MAX_MEMSEG_PER_LIST=8192
config/defconfig_arm64-octeontx2-linuxapp-gcc +1 -0
index f20da24..7687dbe 100644
@@ -9,6 +9,7 @@ CONFIG_RTE_MACHINE="octeontx2"
 CONFIG_RTE_CACHE_LINE_SIZE=128
 CONFIG_RTE_MAX_NUMA_NODES=1
 CONFIG_RTE_MAX_LCORE=24
+CONFIG_RTE_ARM_FEATURE_ATOMICS=y
 
 # Doesn't support NUMA
 CONFIG_RTE_EAL_NUMA_AWARE_HUGEPAGES=n
config/defconfig_arm64-thunderx2-linuxapp-gcc +1 -0
index cc5c64b..af4a89c 100644
@@ -9,3 +9,4 @@ CONFIG_RTE_MACHINE="thunderx2"
 CONFIG_RTE_CACHE_LINE_SIZE=64
 CONFIG_RTE_MAX_NUMA_NODES=2
 CONFIG_RTE_MAX_LCORE=256
+CONFIG_RTE_ARM_FEATURE_ATOMICS=y
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +151 -0

168 lines not on this page · raw message

lib/librte_eal/common/include/arch/x86/rte_atomic_64.h +0 -12
index 1335d92..cfe7067 100644
@@ -183,18 +183,6 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)
 
 /*------------------------ 128 bit atomic operations -------------------------*/
 
-/**
- * 128-bit integer structure.
- */
-RTE_STD_C11
-typedef struct {
-	RTE_STD_C11
-	union {
-		uint64_t val[2];
-		__extension__ __int128 int128;
-	};
-} __rte_aligned(16) rte_int128_t;
-
 __rte_experimental
 static inline int
 rte_atomic128_cmp_exchange(rte_int128_t *dst,
lib/librte_eal/common/include/generic/rte_atomic.h +16 -1
index 24ff7dc..e6ab15a 100644
@@ -1081,6 +1081,20 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)
 
 /*------------------------ 128 bit atomic operations -------------------------*/
 
+/**
+ * 128-bit integer structure.
+ */
+RTE_STD_C11
+typedef struct {
+	RTE_STD_C11
+	union {
+		uint64_t val[2];
+#ifdef RTE_ARCH_64
+		__extension__ __int128 int128;
+#endif
+	};
+} __rte_aligned(16) rte_int128_t;
+
 #ifdef __DOXYGEN__
 
 /**
@@ -1093,7 +1107,8 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)
  *     *exp = *dst
  * @endcode
  *
- * @note This function is currently only available for the x86-64 platform.
+ * @note This function is currently available for the x86-64 and aarch64
+ * platforms.
  *
  * @note The success and failure arguments must be one of the __ATOMIC_* values
  * defined in the C++11 standard. For details on their behavior, refer to the
Signature · 1 line
2.7.4
author
#79 lore raw

PATCH v11 2/3 test/atomic: add 128b compare and swap test

Add 128b atomic compare and swap test for aarch64 and x86_64. Extend the test iteration from 10 thousand to 1 million times to test the stability of the atomic APIs.

---
 app/test/test_atomic.c | 125 ++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 123 insertions(+), 2 deletions(-)
app/test/test_atomic.c +123 -2
index 43be30e..14bd3a8 100644
@@ -1,5 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright(c) 2010-2014 Intel Corporation
+ * Copyright(c) 2019 Arm Limited
  */
 
 #include <stdio.h>
@@ -20,7 +21,7 @@
  * Atomic Variables
  * ================
  *
- * - The main test function performs three subtests. The first test
+ * - The main test function performs four subtests. The first test
  *   checks that the usual inc/dec/add/sub functions are working
  *   correctly:
  *
@@ -61,11 +62,27 @@
  *       atomic_sub(&count, tmp+1);
  *
  *   - At the end of the test, the *count* value must be 0.
+ *
+ * - Test "128b compare and swap" (aarch64 and x86_64 only)
+ *
+ *   - Initialize 128-bit atomic variables to zero.
+ *
+ *   - Invoke ``test_atomic128_cmp_exchange()`` on each lcore. Before doing
+ *     anything else, the cores are waiting a synchro. Each lcore does
+ *     these compare and swap (CAS) operations several times::
+ *
+ *       Acquired CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *       Released CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *       Acquired_Released CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *       Relaxed CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *
+ *   - At the end of the test, the *count128* first 64-bit value and
+ *     second 64-bit value differ by the total iterations.
  */
 
 #define NUM_ATOMIC_TYPES 3
 
-#define N 10000
+#define N 1000000
 
 static rte_atomic16_t a16;
 static rte_atomic32_t a32;
@@ -216,6 +233,78 @@ test_atomic_dec_and_test(__attribute__((unused)) void *arg)
 	return 0;
 }
 
+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)
+static rte_int128_t count128;
+
+/*
+ * rte_atomic128_cmp_exchange() should update a 128 bits counter's first 64
+ * bits by 2 and the second 64 bits by 1 in this test. It should return true
+ * if the compare exchange operation is successful.
+ * This test repeats 128 bits compare and swap operations N rounds. In each
+ * iteration it runs compare and swap operation with different memory models.
+ */
+static int
+test_atomic128_cmp_exchange(__attribute__((unused)) void *arg)
+{
+	rte_int128_t expected;
+	int success;
+	unsigned int i;
+
+	while (rte_atomic32_read(&synchro) == 0)
+		;
+
+	expected = count128;
+
+	for (i = 0; i < N; i++) {
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128,
+				&expected, &desired, 1,
+				__ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
+		} while (success == 0);
+
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128,
+					&expected, &desired, 1,
+					__ATOMIC_RELEASE, __ATOMIC_RELAXED);
+		} while (success == 0);
+
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128,
+					&expected, &desired, 1,
+					__ATOMIC_ACQ_REL, __ATOMIC_RELAXED);
+		} while (success == 0);
+
+		do {
+			rte_int128_t desired;
+
+			desired.val[0] = expected.val[0] + 2;
+			desired.val[1] = expected.val[1] + 1;
+
+			success = rte_atomic128_cmp_exchange(&count128,
+					&expected, &desired, 1,
+					__ATOMIC_RELAXED, __ATOMIC_RELAXED);
+		} while (success == 0);
+	}
+
+	return 0;
+}
+#endif
+
 static int
 test_atomic(void)
 {
@@ -340,6 +429,38 @@ test_atomic(void)
 		return -1;
 	}
 
+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)
+	/*
+	 * This case tests the functionality of rte_atomic128b_cmp_exchange
+	 * API. It calls rte_atomic128b_cmp_exchange with four kinds of memory
+	 * models successively on each slave core. Once each 128-bit atomic
+	 * compare and swap operation is successful, it updates the global
+	 * 128-bit counter by 2 for the first 64-bit and 1 for the second
+	 * 64-bit. Each slave core iterates this test N times.
+	 * At the end of test, verify whether the first 64-bits of the 128-bit
+	 * counter and the second 64bits is differ by the total iterations. If
+	 * it is, the test passes.
+	 */
+	printf("128b compare and swap test\n");
+	uint64_t iterations = 0;
+
+	rte_atomic32_clear(&synchro);
+	count128.val[0] = 0;
+	count128.val[1] = 0;
+
+	rte_eal_mp_remote_launch(test_atomic128_cmp_exchange, NULL,
+				 SKIP_MASTER);
+	rte_atomic32_set(&synchro, 1);
+	rte_eal_mp_wait_lcore();
+	rte_atomic32_clear(&synchro);
+
+	iterations = count128.val[0] - count128.val[1];
+	if (iterations != 4*N*(rte_lcore_count()-1)) {
+		printf("128b compare and swap failed\n");
+		return -1;
+	}
+#endif
+
 	return 0;
 }
 
Signature · 1 line
2.7.4
#80 lore raw

Re: [dpdk-dev] [PATCH v11 2/3] test/atomic: add 128b compare and swap test

Quoted 182 lines · Phil Yang

Add 128b atomic compare and swap test for aarch64 and x86_64. Extend the test iteration from 10 thousand to 1 million times to test the stability of the atomic APIs.

---
 app/test/test_atomic.c | 125 ++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 123 insertions(+), 2 deletions(-)
app/test/test_atomic.c +123 -3
index 43be30e..14bd3a8 100644
@@ -1,5 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright(c) 2010-2014 Intel Corporation
+ * Copyright(c) 2019 Arm Limited
  */
 
 #include <stdio.h>
@@ -20,7 +21,7 @@
  * Atomic Variables
  * ================
  *
- * - The main test function performs three subtests. The first test
+ * - The main test function performs four subtests. The first test
  *   checks that the usual inc/dec/add/sub functions are working
  *   correctly:
  *
@@ -61,11 +62,27 @@
  *       atomic_sub(&count, tmp+1);
  *
  *   - At the end of the test, the *count* value must be 0.
+ *
+ * - Test "128b compare and swap" (aarch64 and x86_64 only)
+ *
+ *   - Initialize 128-bit atomic variables to zero.
+ *
+ *   - Invoke ``test_atomic128_cmp_exchange()`` on each lcore. Before doing
+ *     anything else, the cores are waiting a synchro. Each lcore does
+ *     these compare and swap (CAS) operations several times::
+ *
+ *       Acquired CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *       Released CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *       Acquired_Released CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *       Relaxed CAS update counter.val[0] + 2; counter.val[1] + 1;
+ *
+ *   - At the end of the test, the *count128* first 64-bit value and
+ *     second 64-bit value differ by the total iterations.
  */
 
 #define NUM_ATOMIC_TYPES 3
 
-#define N 10000
+#define N 1000000
 
 static rte_atomic16_t a16;
 static rte_atomic32_t a32;
@@ -216,6 +233,78 @@ test_atomic_dec_and_test(__attribute__((unused)) void *arg)
        return 0;
 }
 
+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)
+static rte_int128_t count128;
+
+/*
+ * rte_atomic128_cmp_exchange() should update a 128 bits counter's first 64
+ * bits by 2 and the second 64 bits by 1 in this test. It should return true
+ * if the compare exchange operation is successful.
+ * This test repeats 128 bits compare and swap operations N rounds. In each
+ * iteration it runs compare and swap operation with different memory models.
+ */
+static int
+test_atomic128_cmp_exchange(__attribute__((unused)) void *arg)
+{
+       rte_int128_t expected;
+       int success;
+       unsigned int i;
+
+       while (rte_atomic32_read(&synchro) == 0)
+               ;
+
+       expected = count128;
+
+       for (i = 0; i < N; i++) {
+               do {
+                       rte_int128_t desired;
+
+                       desired.val[0] = expected.val[0] + 2;
+                       desired.val[1] = expected.val[1] + 1;
+
+                       success = rte_atomic128_cmp_exchange(&count128,
+                               &expected, &desired, 1,
+                               __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
+               } while (success == 0);
+
+               do {
+                       rte_int128_t desired;
+
+                       desired.val[0] = expected.val[0] + 2;
+                       desired.val[1] = expected.val[1] + 1;
+
+                       success = rte_atomic128_cmp_exchange(&count128,
+                                       &expected, &desired, 1,
+                                       __ATOMIC_RELEASE, __ATOMIC_RELAXED);
+               } while (success == 0);
+
+               do {
+                       rte_int128_t desired;
+
+                       desired.val[0] = expected.val[0] + 2;
+                       desired.val[1] = expected.val[1] + 1;
+
+                       success = rte_atomic128_cmp_exchange(&count128,
+                                       &expected, &desired, 1,
+                                       __ATOMIC_ACQ_REL, __ATOMIC_RELAXED);
+               } while (success == 0);
+
+               do {
+                       rte_int128_t desired;
+
+                       desired.val[0] = expected.val[0] + 2;
+                       desired.val[1] = expected.val[1] + 1;
+
+                       success = rte_atomic128_cmp_exchange(&count128,
+                                       &expected, &desired, 1,
+                                       __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+               } while (success == 0);
+       }
+
+       return 0;
+}
+#endif
+
 static int
 test_atomic(void)
 {
@@ -340,6 +429,38 @@ test_atomic(void)
                return -1;
        }
 
+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)
+       /*
+        * This case tests the functionality of rte_atomic128b_cmp_exchange
+        * API. It calls rte_atomic128b_cmp_exchange with four kinds of memory
+        * models successively on each slave core. Once each 128-bit atomic
+        * compare and swap operation is successful, it updates the global
+        * 128-bit counter by 2 for the first 64-bit and 1 for the second
+        * 64-bit. Each slave core iterates this test N times.
+        * At the end of test, verify whether the first 64-bits of the 128-bit
+        * counter and the second 64bits is differ by the total iterations. If
+        * it is, the test passes.
+        */
+       printf("128b compare and swap test\n");
+       uint64_t iterations = 0;
+
+       rte_atomic32_clear(&synchro);
+       count128.val[0] = 0;
+       count128.val[1] = 0;
+
+       rte_eal_mp_remote_launch(test_atomic128_cmp_exchange, NULL,
+                                SKIP_MASTER);
+       rte_atomic32_set(&synchro, 1);
+       rte_eal_mp_wait_lcore();
+       rte_atomic32_clear(&synchro);
+
+       iterations = count128.val[0] - count128.val[1];
+       if (iterations != 4*N*(rte_lcore_count()-1)) {
+               printf("128b compare and swap failed\n");
+               return -1;
+       }
+#endif
+
        return 0;
 }
 
--
Quoted 2 lines

2.7.4

Applied, thanks.

Signature · 1 line
David Marchand
author
#81 lore raw

PATCH v11 3/3 eal/stack: enable lock-free stack for aarch64

In reply to Phil Yang

Enable both c11 atomic and non c11 atomic lock-free stack for aarch64.

Introduced a new header to reduce the ifdef clutter across generic and c11 files. The rte_stack_lf_stubs.h contains stub implementations of __rte_stack_lf_count, __rte_stack_lf_push_elems and __rte_stack_lf_pop_elems.

---
 doc/guides/prog_guide/env_abstraction_layer.rst |  4 +--
 doc/guides/rel_notes/release_19_11.rst          |  3 ++
 lib/librte_stack/Makefile                       |  3 +-
 lib/librte_stack/rte_stack_lf.h                 |  4 +++
 lib/librte_stack/rte_stack_lf_c11.h             | 16 ---------
 lib/librte_stack/rte_stack_lf_generic.h         | 16 ---------
 lib/librte_stack/rte_stack_lf_stubs.h           | 44 +++++++++++++++++++++++++
 7 files changed, 55 insertions(+), 35 deletions(-)
 create mode 100644 lib/librte_stack/rte_stack_lf_stubs.h
doc/guides/prog_guide/env_abstraction_layer.rst +2 -2
index 94f30fd..6e59fae 100644
@@ -648,8 +648,8 @@ Known Issues
   Alternatively, applications can use the lock-free stack mempool handler. When
   considering this handler, note that:
 
-  - It is currently limited to the x86_64 platform, because it uses an
-    instruction (16-byte compare-and-swap) that is not yet available on other
+  - It is currently limited to the aarch64 and x86_64 platforms, because it uses
+    an instruction (16-byte compare-and-swap) that is not yet available on other
     platforms.
   - It has worse average-case performance than the non-preemptive rte_ring, but
     software caching (e.g. the mempool cache) can mitigate this by reducing the
doc/guides/rel_notes/release_19_11.rst +3 -0
index 85953b9..4f82f54 100644
@@ -115,6 +115,9 @@ New Features
   Added eBPF JIT support for arm64 architecture to improve the eBPF program
   performance.
 
+* **Added Lock-free Stack for aarch64.**
+
+  The lock-free stack implementation is enabled for aarch64 platforms.
 
 Removed Items
 -------------
lib/librte_stack/Makefile +2 -1
index 8d18ce5..b5e5bed 100644
@@ -24,6 +24,7 @@ SYMLINK-$(CONFIG_RTE_LIBRTE_STACK)-include := rte_stack.h \
 					      rte_stack_std.h \
 					      rte_stack_lf.h \
 					      rte_stack_lf_generic.h \
-					      rte_stack_lf_c11.h
+					      rte_stack_lf_c11.h \
+					      rte_stack_lf_stubs.h
 
 include $(RTE_SDK)/mk/rte.lib.mk
lib/librte_stack/rte_stack_lf.h +4 -0
index f5581f0..e67630c 100644
@@ -5,11 +5,15 @@
 #ifndef _RTE_STACK_LF_H_
 #define _RTE_STACK_LF_H_
 
+#if !(defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64))
+#include "rte_stack_lf_stubs.h"
+#else
 #ifdef RTE_USE_C11_MEM_MODEL
 #include "rte_stack_lf_c11.h"
 #else
 #include "rte_stack_lf_generic.h"
 #endif
+#endif
 
 /**
  * @internal Push several objects on the lock-free stack (MT-safe).
lib/librte_stack/rte_stack_lf_c11.h +0 -16
index 3d677ae..999359f 100644
@@ -36,12 +36,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 			  struct rte_stack_lf_elem *last,
 			  unsigned int num)
 {
-#ifndef RTE_ARCH_X86_64
-	RTE_SET_USED(first);
-	RTE_SET_USED(last);
-	RTE_SET_USED(list);
-	RTE_SET_USED(num);
-#else
 	struct rte_stack_lf_head old_head;
 	int success;
 
@@ -79,7 +73,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 	 * to the LIFO len update.
 	 */
 	__atomic_add_fetch(&list->len, num, __ATOMIC_RELEASE);
-#endif
 }
 
 static __rte_always_inline struct rte_stack_lf_elem *
@@ -88,14 +81,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 void **obj_table,
 			 struct rte_stack_lf_elem **last)
 {
-#ifndef RTE_ARCH_X86_64
-	RTE_SET_USED(obj_table);
-	RTE_SET_USED(last);
-	RTE_SET_USED(list);
-	RTE_SET_USED(num);
-
-	return NULL;
-#else
 	struct rte_stack_lf_head old_head;
 	uint64_t len;
 	int success;
@@ -169,7 +154,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 	} while (success == 0);
 
 	return old_head.top;
-#endif
 }
 
 #endif /* _RTE_STACK_LF_C11_H_ */
lib/librte_stack/rte_stack_lf_generic.h +0 -16
index 3182151..3abbb53 100644
@@ -36,12 +36,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 			  struct rte_stack_lf_elem *last,
 			  unsigned int num)
 {
-#ifndef RTE_ARCH_X86_64
-	RTE_SET_USED(first);
-	RTE_SET_USED(last);
-	RTE_SET_USED(list);
-	RTE_SET_USED(num);
-#else
 	struct rte_stack_lf_head old_head;
 	int success;
 
@@ -75,7 +69,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 	} while (success == 0);
 
 	rte_atomic64_add((rte_atomic64_t *)&list->len, num);
-#endif
 }
 
 static __rte_always_inline struct rte_stack_lf_elem *
@@ -84,14 +77,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 void **obj_table,
 			 struct rte_stack_lf_elem **last)
 {
-#ifndef RTE_ARCH_X86_64
-	RTE_SET_USED(obj_table);
-	RTE_SET_USED(last);
-	RTE_SET_USED(list);
-	RTE_SET_USED(num);
-
-	return NULL;
-#else
 	struct rte_stack_lf_head old_head;
 	int success;
 
@@ -159,7 +144,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 	} while (success == 0);
 
 	return old_head.top;
-#endif
 }
 
 #endif /* _RTE_STACK_LF_GENERIC_H_ */
lib/librte_stack/rte_stack_lf_stubs.h new file +44 -0

44 lines not on this page · raw message

Signature · 1 line
2.7.4
#82 lore raw

Re: [dpdk-dev] [PATCH v11 3/3] eal/stack: enable lock-free stack for aarch64

Quoted 45 lines · Phil Yang

Enable both c11 atomic and non c11 atomic lock-free stack for aarch64.

Introduced a new header to reduce the ifdef clutter across generic and c11 files. The rte_stack_lf_stubs.h contains stub implementations of __rte_stack_lf_count, __rte_stack_lf_push_elems and __rte_stack_lf_pop_elems.

---
 doc/guides/prog_guide/env_abstraction_layer.rst |  4 +--
 doc/guides/rel_notes/release_19_11.rst          |  3 ++
 lib/librte_stack/Makefile                       |  3 +-
 lib/librte_stack/rte_stack_lf.h                 |  4 +++
 lib/librte_stack/rte_stack_lf_c11.h             | 16 ---------
 lib/librte_stack/rte_stack_lf_generic.h         | 16 ---------
 lib/librte_stack/rte_stack_lf_stubs.h           | 44 +++++++++++++++++++++++++
 7 files changed, 55 insertions(+), 35 deletions(-)
 create mode 100644 lib/librte_stack/rte_stack_lf_stubs.h
doc/guides/prog_guide/env_abstraction_layer.rst +2 -2
index 94f30fd..6e59fae 100644
@@ -648,8 +648,8 @@ Known Issues
   Alternatively, applications can use the lock-free stack mempool handler. When
   considering this handler, note that:
 
-  - It is currently limited to the x86_64 platform, because it uses an
-    instruction (16-byte compare-and-swap) that is not yet available on other
+  - It is currently limited to the aarch64 and x86_64 platforms, because it uses
+    an instruction (16-byte compare-and-swap) that is not yet available on other
     platforms.
   - It has worse average-case performance than the non-preemptive rte_ring, but
     software caching (e.g. the mempool cache) can mitigate this by reducing the
doc/guides/rel_notes/release_19_11.rst +0 -0
index 85953b9..4f82f54 100644

doc/guides/rel_notes/release_19_11.rst New Features · line 120

Quoted 6 lines
doc/guides/rel_notes/release_19_11.rst +3 -0
@@ -115,6 +115,9 @@ New Features
   Added eBPF JIT support for arm64 architecture to improve the eBPF program
   performance.
 
+* **Added Lock-free Stack for aarch64.**
+
+  The lock-free stack implementation is enabled for aarch64 platforms.

Missing line (paragraphs are separated with two empty lines). This entry should go with Core libs entries, so at the beginning of this list.

I did the changes.

Quoted 183 lines
doc/guides/rel_notes/release_19_11.rst +0 -0
 Removed Items
 -------------
lib/librte_stack/Makefile +2 -1
index 8d18ce5..b5e5bed 100644
@@ -24,6 +24,7 @@ SYMLINK-$(CONFIG_RTE_LIBRTE_STACK)-include := rte_stack.h \
                                              rte_stack_std.h \
                                              rte_stack_lf.h \
                                              rte_stack_lf_generic.h \
-                                             rte_stack_lf_c11.h
+                                             rte_stack_lf_c11.h \
+                                             rte_stack_lf_stubs.h
 
 include $(RTE_SDK)/mk/rte.lib.mk
lib/librte_stack/rte_stack_lf.h +4 -0
index f5581f0..e67630c 100644
@@ -5,11 +5,15 @@
 #ifndef _RTE_STACK_LF_H_
 #define _RTE_STACK_LF_H_
 
+#if !(defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64))
+#include "rte_stack_lf_stubs.h"
+#else
 #ifdef RTE_USE_C11_MEM_MODEL
 #include "rte_stack_lf_c11.h"
 #else
 #include "rte_stack_lf_generic.h"
 #endif
+#endif
 
 /**
  * @internal Push several objects on the lock-free stack (MT-safe).
lib/librte_stack/rte_stack_lf_c11.h +0 -16
index 3d677ae..999359f 100644
@@ -36,12 +36,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
                          struct rte_stack_lf_elem *last,
                          unsigned int num)
 {
-#ifndef RTE_ARCH_X86_64
-       RTE_SET_USED(first);
-       RTE_SET_USED(last);
-       RTE_SET_USED(list);
-       RTE_SET_USED(num);
-#else
        struct rte_stack_lf_head old_head;
        int success;
 
@@ -79,7 +73,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
         * to the LIFO len update.
         */
        __atomic_add_fetch(&list->len, num, __ATOMIC_RELEASE);
-#endif
 }
 
 static __rte_always_inline struct rte_stack_lf_elem *
@@ -88,14 +81,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
                         void **obj_table,
                         struct rte_stack_lf_elem **last)
 {
-#ifndef RTE_ARCH_X86_64
-       RTE_SET_USED(obj_table);
-       RTE_SET_USED(last);
-       RTE_SET_USED(list);
-       RTE_SET_USED(num);
-
-       return NULL;
-#else
        struct rte_stack_lf_head old_head;
        uint64_t len;
        int success;
@@ -169,7 +154,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
        } while (success == 0);
 
        return old_head.top;
-#endif
 }
 
 #endif /* _RTE_STACK_LF_C11_H_ */
lib/librte_stack/rte_stack_lf_generic.h +0 -16
index 3182151..3abbb53 100644
@@ -36,12 +36,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
                          struct rte_stack_lf_elem *last,
                          unsigned int num)
 {
-#ifndef RTE_ARCH_X86_64
-       RTE_SET_USED(first);
-       RTE_SET_USED(last);
-       RTE_SET_USED(list);
-       RTE_SET_USED(num);
-#else
        struct rte_stack_lf_head old_head;
        int success;
 
@@ -75,7 +69,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
        } while (success == 0);
 
        rte_atomic64_add((rte_atomic64_t *)&list->len, num);
-#endif
 }
 
 static __rte_always_inline struct rte_stack_lf_elem *
@@ -84,14 +77,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
                         void **obj_table,
                         struct rte_stack_lf_elem **last)
 {
-#ifndef RTE_ARCH_X86_64
-       RTE_SET_USED(obj_table);
-       RTE_SET_USED(last);
-       RTE_SET_USED(list);
-       RTE_SET_USED(num);
-
-       return NULL;
-#else
        struct rte_stack_lf_head old_head;
        int success;
 
@@ -159,7 +144,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
        } while (success == 0);
 
        return old_head.top;
-#endif
 }
 
 #endif /* _RTE_STACK_LF_GENERIC_H_ */
lib/librte_stack/rte_stack_lf_stubs.h new file +44 -1
new file mode 100644
index 0000000..a05abf1
@@ -0,0 +1,44 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2019 Arm Limited
+ */
+
+#ifndef _RTE_STACK_LF_STUBS_H_
+#define _RTE_STACK_LF_STUBS_H_
+
+#include <rte_common.h>
+
+static __rte_always_inline unsigned int
+__rte_stack_lf_count(struct rte_stack *s)
+{
+       RTE_SET_USED(s);
+
+       return 0;
+}
+
+static __rte_always_inline void
+__rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
+                         struct rte_stack_lf_elem *first,
+                         struct rte_stack_lf_elem *last,
+                         unsigned int num)
+{
+       RTE_SET_USED(first);
+       RTE_SET_USED(last);
+       RTE_SET_USED(list);
+       RTE_SET_USED(num);
+}
+
+static __rte_always_inline struct rte_stack_lf_elem *
+__rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
+                        unsigned int num,
+                        void **obj_table,
+                        struct rte_stack_lf_elem **last)
+{
+       RTE_SET_USED(obj_table);
+       RTE_SET_USED(last);
+       RTE_SET_USED(list);
+       RTE_SET_USED(num);
+
+       return NULL;
+}
+
+#endif /* _RTE_STACK_LF_STUBS_H_ */
--
Quoted 2 lines

2.7.4

Applied, thanks.

Signature · 1 line
David Marchand
#83 lore raw
In reply to Phil Yang
Quoted 10 lines · Phil Yang

This patch adds the implementation of the 128-bit atomic compare exchange API on AArch64. Using 64-bit 'ldxp/stxp' instructions can perform this operation. Moreover, on the LSE atomic extension accelerated platforms, it implemented by 'casp' instructions for better performance.

Since the '__ARM_FEATURE_ATOMICS' flag only supports GCC-9, so this patch adds a new config flag 'RTE_ARM_FEATURE_ATOMICS' to enable the 'cas' version on elder version compilers.

Jerin, Phil,

I am getting a build error on the octeontx2 target:

{standard input}: Assembler messages:
{standard input}:672: Error: selected processor does not support `casp x0,x1,x2,x3,[x4]'
{standard input}:690: Error: selected processor does not support `caspa x0,x1,x2,x3,[x4]'
{standard input}:708: Error: selected processor does not support `caspl x0,x1,x2,x3,[x4]'
{standard input}:726: Error: selected processor does not support `caspal x0,x1,x2,x3,[x4]'
ninja: build stopped: subcommand failed.

Looking into the meson logs, I can see:

Native C compiler: ccache gcc (gcc 9.2.1 "gcc (GCC) 9.2.1 20190827 (Red Hat 9.2.1-1)")
Cross C compiler: aarch64-linux-gnu-gcc (gcc 8.2.1) Host machine cpu family: aarch64
Host machine cpu: armv8-a
Target machine cpu family: aarch64
Target machine cpu: armv8-a
Build machine cpu family: x86_64
Build machine cpu: x86_64
...
Message: Implementer : Cavium
Compiler for C supports arguments -mcpu=octeontx2: NO
Message: []
Fetching value of define "__ARM_NEON" : 1
Fetching value of define "__ARM_FEATURE_CRC32" :
Fetching value of define "__ARM_FEATURE_CRYPTO" :

My toolchain does not support the octeontx2 target, but RTE_ARM_FEATURE_ATOMICS ends up being set in the configuration anyway. Tried with Linaro toolchains (4.7.1, 7.4) mentionned in the dpdk documentation, same result.

Looking at config/arm/meson.build, the "extra machine specific flags" are appended to the configuration, regardless of what the compiler replied when testing the machine args.

Signature · 1 line
David Marchand
#84 lore raw
Quoted 30 lines · David Marchand
Quoted 10 lines · Phil Yang

This patch adds the implementation of the 128-bit atomic compare exchange API on AArch64. Using 64-bit 'ldxp/stxp' instructions can perform this operation. Moreover, on the LSE atomic extension accelerated platforms, it implemented by 'casp' instructions for better performance.

Since the '__ARM_FEATURE_ATOMICS' flag only supports GCC-9, so this patch adds a new config flag 'RTE_ARM_FEATURE_ATOMICS' to enable the 'cas' version on elder version compilers.

Jerin, Phil,

I am getting a build error on the octeontx2 target:

{standard input}: Assembler messages:
{standard input}:672: Error: selected processor does not support `casp x0,x1,x2,x3,[x4]'
{standard input}:690: Error: selected processor does not support `caspa x0,x1,x2,x3,[x4]'
{standard input}:708: Error: selected processor does not support `caspl x0,x1,x2,x3,[x4]'
{standard input}:726: Error: selected processor does not support `caspal x0,x1,x2,x3,[x4]'
ninja: build stopped: subcommand failed.

Looking into the meson logs, I can see:

Quoted 12 lines

Native C compiler: ccache gcc (gcc 9.2.1 "gcc (GCC) 9.2.1 20190827 (Red Hat 9.2.1-1)")
Cross C compiler: aarch64-linux-gnu-gcc (gcc 8.2.1) Host machine cpu family: aarch64
Host machine cpu: armv8-a
Target machine cpu family: aarch64
Target machine cpu: armv8-a
Build machine cpu family: x86_64
Build machine cpu: x86_64
...
Message: Implementer : Cavium
Compiler for C supports arguments -mcpu=octeontx2: NO

The compiler needs either +lse or mcpu=octeontx2 to generate casp instruction. Could you try this patch, I can submit a patch if it works for you.

[master][dpdk-next-net-mrvl] $ git diff

config/arm/meson.build +0 -0
index 979018e16..466522786 100644
@@ -96,7 +96,7 @@ machine_args_cavium = [
        ['0xa2', ['-mcpu=thunderxt81'], flags_thunderx_extra],
        ['0xa3', ['-mcpu=thunderxt83'], flags_thunderx_extra],
        ['0xaf', ['-march=armv8.1-a+crc+crypto','-mcpu=thunderx2t99'],

flags_thunderx2_extra],
- ['0xb2', ['-mcpu=octeontx2'], flags_octeontx2_extra]]
+ ['0xb2',
['-march=armv8.2-a+crc+crypto+lse','-mcpu=octeontx2'], flags_octeontx2_extra]]

## Arm implementer ID (ARM DDI 0487C.a, Section G7.2.106, Page G7-5321) impl_generic = ['Generic armv8', flags_generic, machine_args_generic]

#85 lore raw
Quoted 46 lines · Jerin Jacob
Quoted 30 lines · David Marchand
Quoted 10 lines · Phil Yang

This patch adds the implementation of the 128-bit atomic compare exchange API on AArch64. Using 64-bit 'ldxp/stxp' instructions can perform this operation. Moreover, on the LSE atomic extension accelerated platforms, it implemented by 'casp' instructions for better performance.

Since the '__ARM_FEATURE_ATOMICS' flag only supports GCC-9, so this patch adds a new config flag 'RTE_ARM_FEATURE_ATOMICS' to enable the 'cas' version on elder version compilers.

Jerin, Phil,

I am getting a build error on the octeontx2 target:

{standard input}: Assembler messages:
{standard input}:672: Error: selected processor does not support `casp x0,x1,x2,x3,[x4]'
{standard input}:690: Error: selected processor does not support `caspa x0,x1,x2,x3,[x4]'
{standard input}:708: Error: selected processor does not support `caspl x0,x1,x2,x3,[x4]'
{standard input}:726: Error: selected processor does not support `caspal x0,x1,x2,x3,[x4]'
ninja: build stopped: subcommand failed.

Looking into the meson logs, I can see:

Quoted 12 lines

Native C compiler: ccache gcc (gcc 9.2.1 "gcc (GCC) 9.2.1 20190827 (Red Hat 9.2.1-1)")
Cross C compiler: aarch64-linux-gnu-gcc (gcc 8.2.1) Host machine cpu family: aarch64
Host machine cpu: armv8-a
Target machine cpu family: aarch64
Target machine cpu: armv8-a
Build machine cpu family: x86_64
Build machine cpu: x86_64
...
Message: Implementer : Cavium
Compiler for C supports arguments -mcpu=octeontx2: NO

Quoted 2 lines

The compiler needs either +lse or mcpu=octeontx2 to generate casp instruction. Could you try this patch, I can submit a patch if it works for you.

Ah cool, I was looking at the march stuff.
Tried your patch, it works fine.

I'd say we can squash your bits in the current patch, since this was unneeded before this patch.
Is this okay for you?

Quoted 16 lines

[master][dpdk-next-net-mrvl] $ git diff

config/arm/meson.build +1 -1
index 979018e16..466522786 100644
@@ -96,7 +96,7 @@ machine_args_cavium = [
        ['0xa2', ['-mcpu=thunderxt81'], flags_thunderx_extra],
        ['0xa3', ['-mcpu=thunderxt83'], flags_thunderx_extra],
        ['0xaf', ['-march=armv8.1-a+crc+crypto','-mcpu=thunderx2t99'],
 flags_thunderx2_extra],
-       ['0xb2', ['-mcpu=octeontx2'], flags_octeontx2_extra]]
+       ['0xb2',
 ['-march=armv8.2-a+crc+crypto+lse','-mcpu=octeontx2'],

flags_octeontx2_extra]]

Quoted 2 lines

## Arm implementer ID (ARM DDI 0487C.a, Section G7.2.106, Page G7-5321) impl_generic = ['Generic armv8', flags_generic, machine_args_generic]

Thanks for the quick reply.

Signature · 1 line
David Marchand
#86 lore raw
Quoted 51 lines · David Marchand
Quoted 46 lines · Jerin Jacob
Quoted 30 lines · David Marchand
Quoted 10 lines · Phil Yang

This patch adds the implementation of the 128-bit atomic compare exchange API on AArch64. Using 64-bit 'ldxp/stxp' instructions can perform this operation. Moreover, on the LSE atomic extension accelerated platforms, it implemented by 'casp' instructions for better performance.

Since the '__ARM_FEATURE_ATOMICS' flag only supports GCC-9, so this patch adds a new config flag 'RTE_ARM_FEATURE_ATOMICS' to enable the 'cas' version on elder version compilers.

Jerin, Phil,

I am getting a build error on the octeontx2 target:

{standard input}: Assembler messages:
{standard input}:672: Error: selected processor does not support `casp x0,x1,x2,x3,[x4]'
{standard input}:690: Error: selected processor does not support `caspa x0,x1,x2,x3,[x4]'
{standard input}:708: Error: selected processor does not support `caspl x0,x1,x2,x3,[x4]'
{standard input}:726: Error: selected processor does not support `caspal x0,x1,x2,x3,[x4]'
ninja: build stopped: subcommand failed.

Looking into the meson logs, I can see:

Quoted 12 lines

Native C compiler: ccache gcc (gcc 9.2.1 "gcc (GCC) 9.2.1 20190827 (Red Hat 9.2.1-1)")
Cross C compiler: aarch64-linux-gnu-gcc (gcc 8.2.1) Host machine cpu family: aarch64
Host machine cpu: armv8-a
Target machine cpu family: aarch64
Target machine cpu: armv8-a
Build machine cpu family: x86_64
Build machine cpu: x86_64
...
Message: Implementer : Cavium
Compiler for C supports arguments -mcpu=octeontx2: NO

Quoted 2 lines

The compiler needs either +lse or mcpu=octeontx2 to generate casp instruction. Could you try this patch, I can submit a patch if it works for you.

Quoted 6 lines

Ah cool, I was looking at the march stuff.
Tried your patch, it works fine.

I'd say we can squash your bits in the current patch, since this was unneeded before this patch.
Is this okay for you?

Yup.

Quoted 27 lines
Quoted 18 lines

[master][dpdk-next-net-mrvl] $ git diff

config/arm/meson.build +1 -1
index 979018e16..466522786 100644
@@ -96,7 +96,7 @@ machine_args_cavium = [
        ['0xa2', ['-mcpu=thunderxt81'], flags_thunderx_extra],
        ['0xa3', ['-mcpu=thunderxt83'], flags_thunderx_extra],
        ['0xaf', ['-march=armv8.1-a+crc+crypto','-mcpu=thunderx2t99'],
 flags_thunderx2_extra],
-       ['0xb2', ['-mcpu=octeontx2'], flags_octeontx2_extra]]
+       ['0xb2',
 ['-march=armv8.2-a+crc+crypto+lse','-mcpu=octeontx2'],

flags_octeontx2_extra]]

Quoted 2 lines

## Arm implementer ID (ARM DDI 0487C.a, Section G7.2.106, Page G7-5321) impl_generic = ['Generic armv8', flags_generic, machine_args_generic]

Thanks for the quick reply.

--
David Marchand

#87 lore raw
Quoted 54 lines · Jerin Jacob
Quoted 51 lines · David Marchand
Quoted 48 lines · Jerin Jacob
Quoted 30 lines · David Marchand
Quoted 10 lines · Phil Yang

This patch adds the implementation of the 128-bit atomic compare exchange API on AArch64. Using 64-bit 'ldxp/stxp' instructions can perform this operation. Moreover, on the LSE atomic extension accelerated platforms, it implemented by 'casp' instructions for better performance.

Since the '__ARM_FEATURE_ATOMICS' flag only supports GCC-9, so this patch adds a new config flag 'RTE_ARM_FEATURE_ATOMICS' to enable the 'cas' version on elder version compilers.

Jerin, Phil,

I am getting a build error on the octeontx2 target:

{standard input}: Assembler messages:
{standard input}:672: Error: selected processor does not support `casp x0,x1,x2,x3,[x4]'
{standard input}:690: Error: selected processor does not support `caspa x0,x1,x2,x3,[x4]'
{standard input}:708: Error: selected processor does not support `caspl x0,x1,x2,x3,[x4]'
{standard input}:726: Error: selected processor does not support `caspal x0,x1,x2,x3,[x4]'
ninja: build stopped: subcommand failed.

Looking into the meson logs, I can see:

Quoted 12 lines

Native C compiler: ccache gcc (gcc 9.2.1 "gcc (GCC) 9.2.1 20190827 (Red Hat 9.2.1-1)")
Cross C compiler: aarch64-linux-gnu-gcc (gcc 8.2.1) Host machine cpu family: aarch64
Host machine cpu: armv8-a
Target machine cpu family: aarch64
Target machine cpu: armv8-a
Build machine cpu family: x86_64
Build machine cpu: x86_64
...
Message: Implementer : Cavium
Compiler for C supports arguments -mcpu=octeontx2: NO

The compiler needs either +lse or mcpu=octeontx2 to generate casp instruction. Could you try this patch, I can submit a patch if it works for you.

Quoted 32 lines
Quoted 6 lines

Ah cool, I was looking at the march stuff.
Tried your patch, it works fine.

I'd say we can squash your bits in the current patch, since this was unneeded before this patch.
Is this okay for you?

Yup.

Quoted 23 lines
Quoted 18 lines

[master][dpdk-next-net-mrvl] $ git diff

config/arm/meson.build +1 -1
index 979018e16..466522786 100644
@@ -96,7 +96,7 @@ machine_args_cavium = [
        ['0xa2', ['-mcpu=thunderxt81'], flags_thunderx_extra],
        ['0xa3', ['-mcpu=thunderxt83'], flags_thunderx_extra],
        ['0xaf', ['-march=armv8.1-a+crc+crypto','-mcpu=thunderx2t99'],
 flags_thunderx2_extra],
-       ['0xb2', ['-mcpu=octeontx2'], flags_octeontx2_extra]]
+       ['0xb2',
 ['-march=armv8.2-a+crc+crypto+lse','-mcpu=octeontx2'],

flags_octeontx2_extra]]

Quoted 2 lines

## Arm implementer ID (ARM DDI 0487C.a, Section G7.2.106, Page G7-5321) impl_generic = ['Generic armv8', flags_generic, machine_args_generic]

Thanks for the quick reply.

Applied with above fix.
Thanks.

Signature · 1 line
David Marchand