eal/arm64: add 128-bit atomic compare exchange
Subsystems (2)
PATCH 1/3 eal/arm64: add 128-bit atomic compare exchange
Add 128-bit atomic compare exchange on aarch64.
---
This patch depends on
http://patchwork.dpdk.org/patch/54840/
v2:
eal/stack: fix 'pointer-sign' warning
.../common/include/arch/arm/rte_atomic_64.h | 184 +++++++++++++++++++++ .../common/include/arch/x86/rte_atomic_64.h | 12 -- lib/librte_eal/common/include/generic/rte_atomic.h | 15 +- 3 files changed, 198 insertions(+), 13 deletions(-)
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +184 -0
@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2015 Cavium, Inc +* Copyright(c) 2019 Arm Limited */ #ifndef _RTE_ATOMIC_ARM64_H_
@@ -14,6 +15,9 @@ extern "C" {#endif #include "generic/rte_atomic.h" +#include <rte_branch_prediction.h> +#include <rte_compat.h> +#include <rte_debug.h> #define dsb (opt) asm volatile("dsb " #opt : : : "memory")#define dmb (opt) asm volatile("dmb " #opt : : : "memory")
@@ -40,6 +44,186 @@ extern "C" {#define rte_cio_rmb () dmb(oshld) +/*----------------------- 128 bit atomic operations -------------------------*/ + +#define RTE_HAS_ACQ (mo) ((mo) != __ATOMIC_RELAXED && (mo) != __ATOMIC_RELEASE) +#define RTE_HAS_RLS (mo) ((mo) == __ATOMIC_RELEASE || \ + (mo) == __ATOMIC_ACQ_REL || \ + (mo) == __ATOMIC_SEQ_CST) + +#define RTE_MO_LOAD (mo) (RTE_HAS_ACQ((mo)) \ + ? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED) +#define RTE_MO_STORE (mo) (RTE_HAS_RLS((mo)) \ + ? __ATOMIC_RELEASE : __ATOMIC_RELAXED) + +#ifdef __ARM_FEATURE_ATOMICS +static inline rte_int128_t +__rte_casp (rte_int128_t *dst,rte_int128_t old,rte_int128_t updated,int mo) +{ + +/* caspX instructions register pair must start from even-numbered +* register at operand 1. +* So, specify registers for local variables here. +*/ + register uint64_t x0 __asm ("x0") = (uint64_t)old.val[0]; + register uint64_t x1 __asm ("x1") = (uint64_t)old.val[1]; + register uint64_t x2 __asm ("x2") = (uint64_t)updated.val[0]; + register uint64_t x3 __asm ("x3") = (uint64_t)updated.val[1]; + +if (mo == __ATOMIC_RELAXED) { + asmvolatile ( +"casp %[old0], %[old1], %[upd0], %[upd1], [%[dst]]" + : [old0]"+r" (x0), + [old1]"+r" (x1) + : [upd0]"r" (x2), + [upd1]"r" (x3), + [dst]"r" (dst) + :"memory" ); + }else if (mo == __ATOMIC_ACQUIRE) { + asmvolatile ( +"caspa %[old0], %[old1], %[upd0], %[upd1], [%[dst]]" + : [old0]"+r" (x0), + [old1]"+r" (x1) + : [upd0]"r" (x2), + [upd1]"r" (x3), + [dst]"r" (dst) + :"memory" ); + }else if (mo == __ATOMIC_ACQ_REL) { + asmvolatile ( +"caspal %[old0], %[old1], %[upd0], %[upd1], [%[dst]]" + : [old0]"+r" (x0), + [old1]"+r" (x1) + : [upd0]"r" (x2), + [upd1]"r" (x3), + [dst]"r" (dst) + :"memory" ); + }else if (mo == __ATOMIC_RELEASE) { + asmvolatile ( +"caspl %[old0], %[old1], %[upd0], %[upd1], [%[dst]]" + : [old0]"+r" (x0), + [old1]"+r" (x1) + : [upd0]"r" (x2), + [upd1]"r" (x3), + [dst]"r" (dst) + :"memory" ); + }else { +rte_panic ("Invalid memory order\n" ); + } + + old.val[0 ] = x0; + old.val[1 ] = x1; + +return old; +} +#else +static inline rte_int128_t +__rte_ldx128 (const rte_int128_t *src,int mo) +{ +rte_int128_t ret; +if (mo == __ATOMIC_ACQUIRE) + asmvolatile ( +"ldaxp %0, %1, %2" + :"=&r" (ret.val[0 ]), +"=&r" (ret.val[1 ]) + :"Q" (src->val[0 ]) + :"memory" ); +else if (mo == __ATOMIC_RELAXED) + asmvolatile ( +"ldxp %0, %1, %2" + :"=&r" (ret.val[0 ]), +"=&r" (ret.val[1 ]) + :"Q" (src->val[0 ]) + :"memory" ); +else +rte_panic ("Invalid memory order\n" ); + +return ret; +} + +static inline uint32_t +__rte_stx128 (rte_int128_t *dst,const rte_int128_t src,int mo) +{ +uint32_t ret; +if (mo == __ATOMIC_RELEASE) + asmvolatile ( +"stlxp %w0, %1, %2, %3" + :"=&r" (ret) + :"r" (src.val[0 ]), +"r" (src.val[1 ]), +"Q" (dst->val[0 ]) + :"memory" ); +else if (mo == __ATOMIC_RELAXED) + asmvolatile ( +"stxp %w0, %1, %2, %3" + :"=&r" (ret) + :"r" (src.val[0 ]), +"r" (src.val[1 ]), +"Q" (dst->val[0 ]) + :"memory" ); +else +rte_panic ("Invalid memory order\n" ); + +/* Return 0 on success, 1 on failure */ +return ret; +} +#endif + +static inline int __rte_experimental +rte_atomic128_cmp_exchange (rte_int128_t *dst, +rte_int128_t *exp, +const rte_int128_t *src, +unsigned int weak, +int success, +int failure) +{ +// Always do strong CAS +RTE_SET_USED (weak); +/* Ignore memory ordering for failure, memory order for +* success must be stronger or equal +*/ +RTE_SET_USED (failure); + +#ifdef __ARM_FEATURE_ATOMICS +rte_int128_t expected = *exp; +rte_int128_t desired = *src; +rte_int128_t old; + + old =__rte_casp (dst, expected, desired, success); +#else +int ldx_mo =RTE_MO_LOAD (success); +int stx_mo =RTE_MO_STORE (success); +uint32_t ret =1 ; + registerrte_int128_t expected = *exp; + registerrte_int128_t desired = *src; + registerrte_int128_t old; + +/* ldx128 can not guarantee atomic, +* Must write back src or old to verify atomicity of ldx128; +*/ +do { + old =__rte_ldx128 (dst, ldx_mo); +if (likely (old.int128 == expected.int128)) + ret =__rte_stx128 (dst, desired, stx_mo); +else +/* In the failure case (since 'weak' is ignored and only +* weak == 0 is implemented), expected should contain the +* atomically read value of dst. This means, 'old' needs +* to be stored back to ensure it was read atomically. +*/ + ret =__rte_stx128 (dst, old, stx_mo); + }while (unlikely (ret)); +#endif + +/* Unconditionally updating expected removes +* an 'if' statement. +* expected should already be in register if +* not in the cache. +*/ + *exp = old; + +return (old.int128 == expected.int128); +} + #ifdef __cplusplus }#endif
lib/librte_eal/common/include/arch/x86/rte_atomic_64.h +0 -12
@@ -212,18 +212,6 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)/*------------------------ 128 bit atomic operations -------------------------*/ -/** -* 128-bit integer structure. -*/ -RTE_STD_C11 -typedef struct { -RTE_STD_C11 - union { -uint64_t val[2 ]; - __extension____int128 int128; - }; -} __rte_aligned(16) rte_int128_t; - static inline int __rte_experimental rte_atomic128_cmp_exchange(rte_int128_t *dst, rte_int128_t *exp,
lib/librte_eal/common/include/generic/rte_atomic.h +14 -1
@@ -1081,6 +1081,18 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)/*------------------------ 128 bit atomic operations -------------------------*/ +/** +* 128-bit integer structure. +*/ +RTE_STD_C11 +typedef struct { +RTE_STD_C11 + union { +uint64_t val[2 ]; + __extension____int128 int128; + }; +} __rte_aligned(16) rte_int128_t; +#ifdef __DOXYGEN__/**
@@ -1093,7 +1105,8 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)* *exp = *dst * @endcode * -* @note This function is currently only available for the x86-64 platform. +* @note This function is currently available for the x86-64 and aarch64 +* platforms. * * @note The success and failure arguments must be one of the __ATOMIC_* values * defined in the C++11 standard. For details on their behavior, refer to the
Signature · 1 line
2.7.4
PATCH 2/3 test/atomic: add 128b compare and swap test
Add 128b atomic compare and swap test for aarch64 and x86_64.
--- app/test/test_atomic.c | 119 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 118 insertions(+), 1 deletion(-)
app/test/test_atomic.c +118 -1
@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2019 Arm Limited */ #include <stdio.h>
@@ -20,7 +21,7 @@* Atomic Variables * ================ * -* - The main test function performs three subtests. The first test +* - The main test function performs four subtests. The first test * checks that the usual inc/dec/add/sub functions are working * correctly: *
@@ -61,6 +62,22 @@* atomic_sub(&count, tmp+1); * * - At the end of the test, the *count* value must be 0. +* +* - Test "128b compare and swap" (aarch64 and x86_64 only) +* +* - Initialize 128-bit atomic variables to zero. +* +* - Invoke ``test_atomici128_cmp_exchange()`` on each lcore. Before doing +* anything else, the cores are waiting a synchro. Each lcore does +* these compare and swap (CAS) operations several times:: +* +* Relaxed CAS update counter.val[0] + 2; counter.val[0] + 1; +* Acquired CAS update counter.val[0] + 2; counter.val[0] + 1; +* Released CAS update counter.val[0] + 2; counter.val[0] + 1; +* Acquired_Released CAS update counter.val[0] + 2; counter.val[0] + 1; +* +* - At the end of the test, the *count128* first 64-bit value and +* second 64-bit value differ by the total iterations. */ #define NUM_ATOMIC_TYPES 3
@@ -73,6 +90,10 @@ static rte_atomic64_t a64;static rte_atomic64_t count;static rte_atomic32_t synchro; +#if defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 ) +static rte_int128_t count128; +#endif +static int test_atomic_usual (__attribute__((unused))void *arg) {
@@ -216,6 +237,71 @@ test_atomic_dec_and_test(__attribute__((unused)) void *arg)return 0 ; } +#if defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 ) +/* +* rte_atomic128_cmp_exchange() should update a 128 bits counter's first 64 +* bits by 2 and the second 64 bits by 1 in this test. It should return true +* if the compare exchange operation successful. +* This test repeat 128 bits compare and swap operations 10K rounds. In each +* iteration it runs compare and swap operation with different memory models. +*/ +static int +test_atomic128_cmp_exchange (__attribute__((unused))void *arg) +{ +while (rte_atomic32_read (&synchro) ==0 ) + ; + +rte_int128_t expected; +int success; + + expected = count128; + +for (int i=0 ; i <N ; i++) { +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, &expected, + &desired,1 , __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); + }while (success ==0 ); + +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, &expected, + &desired,1 , __ATOMIC_RELEASE, __ATOMIC_RELAXED); + }while (success ==0 ); + +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, &expected, + &desired,1 , __ATOMIC_ACQ_REL, __ATOMIC_RELAXED); + }while (success ==0 ); + +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, &expected, + &desired,1 , __ATOMIC_RELAXED, __ATOMIC_RELAXED); + }while (success ==0 ); + } + +return 0 ; +} +#endif +static int test_atomic (void ) {
@@ -340,6 +426,37 @@ test_atomic(void)return -1 ; } +#if defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 ) +/* +* This case tests the functionality of rte_atomic128b_cmp_exchange +* API. It calls rte_atomic128b_cmp_exchange with four kinds of memory +* models successively on each slave core. Once each 128-bit atomic +* compare and swap operation is successful, it updates the global +* 128-bit counter by 2 for the first 64-bit and 1 for the second +* 64-bit. Each slave core iterates this test 10K times. +* At the end of test, verify whether the first 64-bits of the 128-bit +* counter and the second 64bits is differ by the total iterations. If +* it is, the test passes. +*/ +printf ("128b compare and swap test\n" ); +uint64_t iterations =0 ; + +rte_atomic32_clear (&synchro); + count128.val[0 ] =0 ; + count128.val[1 ] =0 ; + +rte_eal_mp_remote_launch (test_atomic128_cmp_exchange,NULL ,SKIP_MASTER ); +rte_atomic32_set (&synchro,1 ); +rte_eal_mp_wait_lcore (); +rte_atomic32_clear (&synchro); + + iterations = count128.val[0 ] - count128.val[1 ]; +if (iterations !=4 *N *(rte_lcore_count ()-1 )) { +printf ("128b compare and swap failed\n" ); +return -1 ; + } +#endif +return 0 ; }
Signature · 1 line
2.7.4
PATCH 3/3 eal/stack: enable lock-free stack for aarch64
In reply to Phil YangEnable both c11 atomic and non c11 atomic lock-free stack for aarch64.
--- doc/guides/rel_notes/release_19_08.rst | 3 +++ lib/librte_stack/rte_stack_lf_c11.h | 4 ++-- lib/librte_stack/rte_stack_lf_generic.h | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-)
doc/guides/rel_notes/release_19_08.rst +3 -0
@@ -88,6 +88,9 @@ New Features * Added multi-queue support to allow one af_xdp vdev with multiple netdev queues +* **Added Lock-free Stack for aarch64.** + + The lock-free stack implementation is enabled for aarch64 platforms. Removed Items -------------
lib/librte_stack/rte_stack_lf_c11.h +2 -2
@@ -36,7 +36,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list, struct rte_stack_lf_elem *last, unsigned int num) { -#ifndef RTE_ARCH_X86_64 +#if !defined(RTE_ARCH_X86_64 ) && !defined(RTE_ARCH_ARM64 )RTE_SET_USED (first);RTE_SET_USED (last);RTE_SET_USED (list);
@@ -88,7 +88,7 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list, void **obj_table, struct rte_stack_lf_elem **last) { -#ifndef RTE_ARCH_X86_64 +#if !defined(RTE_ARCH_X86_64 ) && !defined(RTE_ARCH_ARM64 )RTE_SET_USED (obj_table);RTE_SET_USED (last);RTE_SET_USED (list);
lib/librte_stack/rte_stack_lf_generic.h +2 -2
@@ -36,7 +36,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list, struct rte_stack_lf_elem *last, unsigned int num) { -#ifndef RTE_ARCH_X86_64 +#if !defined(RTE_ARCH_X86_64 ) && !defined(RTE_ARCH_ARM64 )RTE_SET_USED (first);RTE_SET_USED (last);RTE_SET_USED (list);
@@ -84,7 +84,7 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list, void **obj_table, struct rte_stack_lf_elem **last) { -#ifndef RTE_ARCH_X86_64 +#if !defined(RTE_ARCH_X86_64 ) && !defined(RTE_ARCH_ARM64 )RTE_SET_USED (obj_table);RTE_SET_USED (last);RTE_SET_USED (list);
Signature · 1 line
2.7.4
PATCH v2 1/3 eal/arm64: add 128-bit atomic compare exchange
In reply to Phil YangAdd 128-bit atomic compare exchange on aarch64.
---
This patch depends on 'eal/stack: fix 'pointer-sign' warning'
http://patchwork.dpdk.org/patch/54840/
v2:
Fixed coding style warning.
.../common/include/arch/arm/rte_atomic_64.h | 184 +++++++++++++++++++++ .../common/include/arch/x86/rte_atomic_64.h | 12 -- lib/librte_eal/common/include/generic/rte_atomic.h | 15 +- 3 files changed, 198 insertions(+), 13 deletions(-)
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +184 -0
@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2015 Cavium, Inc +* Copyright(c) 2019 Arm Limited */ #ifndef _RTE_ATOMIC_ARM64_H_
@@ -14,6 +15,9 @@ extern "C" {#endif #include "generic/rte_atomic.h" +#include <rte_branch_prediction.h> +#include <rte_compat.h> +#include <rte_debug.h> #define dsb (opt) asm volatile("dsb " #opt : : : "memory")#define dmb (opt) asm volatile("dmb " #opt : : : "memory")
@@ -40,6 +44,186 @@ extern "C" {#define rte_cio_rmb () dmb(oshld) +/*----------------------- 128 bit atomic operations -------------------------*/ + +#define RTE_HAS_ACQ (mo) ((mo) != __ATOMIC_RELAXED && (mo) != __ATOMIC_RELEASE) +#define RTE_HAS_RLS (mo) ((mo) == __ATOMIC_RELEASE || \ + (mo) == __ATOMIC_ACQ_REL || \ + (mo) == __ATOMIC_SEQ_CST) + +#define RTE_MO_LOAD (mo) (RTE_HAS_ACQ((mo)) \ + ? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED) +#define RTE_MO_STORE (mo) (RTE_HAS_RLS((mo)) \ + ? __ATOMIC_RELEASE : __ATOMIC_RELAXED) + +#ifdef __ARM_FEATURE_ATOMICS +static inline rte_int128_t +__rte_casp (rte_int128_t *dst,rte_int128_t old,rte_int128_t updated,int mo) +{ + +/* caspX instructions register pair must start from even-numbered +* register at operand 1. +* So, specify registers for local variables here. +*/ + register uint64_t x0 __asm("x0") = (uint64_t)old.val[0]; + register uint64_t x1 __asm("x1") = (uint64_t)old.val[1]; + register uint64_t x2 __asm("x2") = (uint64_t)updated.val[0]; + register uint64_t x3 __asm("x3") = (uint64_t)updated.val[1]; + +if (mo == __ATOMIC_RELAXED) { + asmvolatile ( +"casp %[old0], %[old1], %[upd0], %[upd1], [%[dst]]" + : [old0]"+r" (x0), + [old1]"+r" (x1) + : [upd0]"r" (x2), + [upd1]"r" (x3), + [dst]"r" (dst) + :"memory" ); + }else if (mo == __ATOMIC_ACQUIRE) { + asmvolatile ( +"caspa %[old0], %[old1], %[upd0], %[upd1], [%[dst]]" + : [old0]"+r" (x0), + [old1]"+r" (x1) + : [upd0]"r" (x2), + [upd1]"r" (x3), + [dst]"r" (dst) + :"memory" ); + }else if (mo == __ATOMIC_ACQ_REL) { + asmvolatile ( +"caspal %[old0], %[old1], %[upd0], %[upd1], [%[dst]]" + : [old0]"+r" (x0), + [old1]"+r" (x1) + : [upd0]"r" (x2), + [upd1]"r" (x3), + [dst]"r" (dst) + :"memory" ); + }else if (mo == __ATOMIC_RELEASE) { + asmvolatile ( +"caspl %[old0], %[old1], %[upd0], %[upd1], [%[dst]]" + : [old0]"+r" (x0), + [old1]"+r" (x1) + : [upd0]"r" (x2), + [upd1]"r" (x3), + [dst]"r" (dst) + :"memory" ); + }else { +rte_panic ("Invalid memory order\n" ); + } + + old.val[0 ] = x0; + old.val[1 ] = x1; + +return old; +} +#else +static inline rte_int128_t +__rte_ldx128 (const rte_int128_t *src,int mo) +{ +rte_int128_t ret; +if (mo == __ATOMIC_ACQUIRE) + asmvolatile ( +"ldaxp %0, %1, %2" + :"=&r" (ret.val[0 ]), +"=&r" (ret.val[1 ]) + :"Q" (src->val[0 ]) + :"memory" ); +else if (mo == __ATOMIC_RELAXED) + asmvolatile ( +"ldxp %0, %1, %2" + :"=&r" (ret.val[0 ]), +"=&r" (ret.val[1 ]) + :"Q" (src->val[0 ]) + :"memory" ); +else +rte_panic ("Invalid memory order\n" ); + +return ret; +} + +static inline uint32_t +__rte_stx128 (rte_int128_t *dst,const rte_int128_t src,int mo) +{ +uint32_t ret; +if (mo == __ATOMIC_RELEASE) + asmvolatile ( +"stlxp %w0, %1, %2, %3" + :"=&r" (ret) + :"r" (src.val[0 ]), +"r" (src.val[1 ]), +"Q" (dst->val[0 ]) + :"memory" ); +else if (mo == __ATOMIC_RELAXED) + asmvolatile ( +"stxp %w0, %1, %2, %3" + :"=&r" (ret) + :"r" (src.val[0 ]), +"r" (src.val[1 ]), +"Q" (dst->val[0 ]) + :"memory" ); +else +rte_panic ("Invalid memory order\n" ); + +/* Return 0 on success, 1 on failure */ +return ret; +} +#endif + +static inline int __rte_experimental +rte_atomic128_cmp_exchange (rte_int128_t *dst, +rte_int128_t *exp, +const rte_int128_t *src, +unsigned int weak, +int success, +int failure) +{ +// Always do strong CAS +RTE_SET_USED (weak); +/* Ignore memory ordering for failure, memory order for +* success must be stronger or equal +*/ +RTE_SET_USED (failure); + +#ifdef __ARM_FEATURE_ATOMICS +rte_int128_t expected = *exp; +rte_int128_t desired = *src; +rte_int128_t old; + + old =__rte_casp (dst, expected, desired, success); +#else +int ldx_mo =RTE_MO_LOAD (success); +int stx_mo =RTE_MO_STORE (success); +uint32_t ret =1 ; + registerrte_int128_t expected = *exp; + registerrte_int128_t desired = *src; + registerrte_int128_t old; + +/* ldx128 can not guarantee atomic, +* Must write back src or old to verify atomicity of ldx128; +*/ +do { + old =__rte_ldx128 (dst, ldx_mo); +if (likely (old.int128 == expected.int128)) + ret =__rte_stx128 (dst, desired, stx_mo); +else +/* In the failure case (since 'weak' is ignored and only +* weak == 0 is implemented), expected should contain the +* atomically read value of dst. This means, 'old' needs +* to be stored back to ensure it was read atomically. +*/ + ret =__rte_stx128 (dst, old, stx_mo); + }while (unlikely (ret)); +#endif + +/* Unconditionally updating expected removes +* an 'if' statement. +* expected should already be in register if +* not in the cache. +*/ + *exp = old; + +return (old.int128 == expected.int128); +} + #ifdef __cplusplus }#endif
lib/librte_eal/common/include/arch/x86/rte_atomic_64.h +0 -12
@@ -212,18 +212,6 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)/*------------------------ 128 bit atomic operations -------------------------*/ -/** -* 128-bit integer structure. -*/ -RTE_STD_C11 -typedef struct { -RTE_STD_C11 - union { -uint64_t val[2 ]; - __extension____int128 int128; - }; -} __rte_aligned(16) rte_int128_t; - static inline int __rte_experimental rte_atomic128_cmp_exchange(rte_int128_t *dst, rte_int128_t *exp,
lib/librte_eal/common/include/generic/rte_atomic.h +14 -1
@@ -1081,6 +1081,18 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)/*------------------------ 128 bit atomic operations -------------------------*/ +/** +* 128-bit integer structure. +*/ +RTE_STD_C11 +typedef struct { +RTE_STD_C11 + union { +uint64_t val[2 ]; + __extension____int128 int128; + }; +} __rte_aligned(16) rte_int128_t; +#ifdef __DOXYGEN__/**
@@ -1093,7 +1105,8 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)* *exp = *dst * @endcode * -* @note This function is currently only available for the x86-64 platform. +* @note This function is currently available for the x86-64 and aarch64 +* platforms. * * @note The success and failure arguments must be one of the __ATOMIC_* values * defined in the C++11 standard. For details on their behavior, refer to the
Signature · 1 line
2.7.4
PATCH v2 2/3 test/atomic: add 128b compare and swap test
Add 128b atomic compare and swap test for aarch64 and x86_64.
--- app/test/test_atomic.c | 120 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 119 insertions(+), 1 deletion(-)
app/test/test_atomic.c +119 -1
@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2019 Arm Limited */ #include <stdio.h>
@@ -20,7 +21,7 @@* Atomic Variables * ================ * -* - The main test function performs three subtests. The first test +* - The main test function performs four subtests. The first test * checks that the usual inc/dec/add/sub functions are working * correctly: *
@@ -61,6 +62,22 @@* atomic_sub(&count, tmp+1); * * - At the end of the test, the *count* value must be 0. +* +* - Test "128b compare and swap" (aarch64 and x86_64 only) +* +* - Initialize 128-bit atomic variables to zero. +* +* - Invoke ``test_atomici128_cmp_exchange()`` on each lcore. Before doing +* anything else, the cores are waiting a synchro. Each lcore does +* these compare and swap (CAS) operations several times:: +* +* Relaxed CAS update counter.val[0] + 2; counter.val[0] + 1; +* Acquired CAS update counter.val[0] + 2; counter.val[0] + 1; +* Released CAS update counter.val[0] + 2; counter.val[0] + 1; +* Acquired_Released CAS update counter.val[0] + 2; counter.val[0] + 1; +* +* - At the end of the test, the *count128* first 64-bit value and +* second 64-bit value differ by the total iterations. */ #define NUM_ATOMIC_TYPES 3
@@ -73,6 +90,10 @@ static rte_atomic64_t a64;static rte_atomic64_t count;static rte_atomic32_t synchro; +#if defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 ) +static rte_int128_t count128; +#endif +static int test_atomic_usual (__attribute__((unused))void *arg) {
@@ -216,6 +237,72 @@ test_atomic_dec_and_test(__attribute__((unused)) void *arg)return 0 ; } +#if defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 ) +/* +* rte_atomic128_cmp_exchange() should update a 128 bits counter's first 64 +* bits by 2 and the second 64 bits by 1 in this test. It should return true +* if the compare exchange operation successful. +* This test repeat 128 bits compare and swap operations 10K rounds. In each +* iteration it runs compare and swap operation with different memory models. +*/ +static int +test_atomic128_cmp_exchange (__attribute__((unused))void *arg) +{ +rte_int128_t expected; +int success; +unsigned int i; + +while (rte_atomic32_read (&synchro) ==0 ) + ; + + expected = count128; + +for (i =0 ; i <N ; i++) { +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, &expected, + &desired,1 , __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); + }while (success ==0 ); + +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, &expected, + &desired,1 , __ATOMIC_RELEASE, __ATOMIC_RELAXED); + }while (success ==0 ); + +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, &expected, + &desired,1 , __ATOMIC_ACQ_REL, __ATOMIC_RELAXED); + }while (success ==0 ); + +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, &expected, + &desired,1 , __ATOMIC_RELAXED, __ATOMIC_RELAXED); + }while (success ==0 ); + } + +return 0 ; +} +#endif +static int test_atomic (void ) {
@@ -340,6 +427,37 @@ test_atomic(void)return -1 ; } +#if defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 ) +/* +* This case tests the functionality of rte_atomic128b_cmp_exchange +* API. It calls rte_atomic128b_cmp_exchange with four kinds of memory +* models successively on each slave core. Once each 128-bit atomic +* compare and swap operation is successful, it updates the global +* 128-bit counter by 2 for the first 64-bit and 1 for the second +* 64-bit. Each slave core iterates this test 10K times. +* At the end of test, verify whether the first 64-bits of the 128-bit +* counter and the second 64bits is differ by the total iterations. If +* it is, the test passes. +*/ +printf ("128b compare and swap test\n" ); +uint64_t iterations =0 ; + +rte_atomic32_clear (&synchro); + count128.val[0 ] =0 ; + count128.val[1 ] =0 ; + +rte_eal_mp_remote_launch (test_atomic128_cmp_exchange,NULL ,SKIP_MASTER ); +rte_atomic32_set (&synchro,1 ); +rte_eal_mp_wait_lcore (); +rte_atomic32_clear (&synchro); + + iterations = count128.val[0 ] - count128.val[1 ]; +if (iterations !=4 *N *(rte_lcore_count ()-1 )) { +printf ("128b compare and swap failed\n" ); +return -1 ; + } +#endif +return 0 ; }
Signature · 1 line
2.7.4
Re: [dpdk-dev] [PATCH v2 2/3] test/atomic: add 128b compare and swap test
Hi Phil,
Looks good overall, just a few documentation issues.
<snip>
Quoted 10 lines
+ *
+ * - Test "128b compare and swap" (aarch64 andx86_64only)
+ *
+ * - Initialize 128-bit atomic variables to zero.
+ *
+ * - Invoke ``test_atomici128_cmp_exchange()`` on each lcore. Before
doing
+ * anything else, the cores are waiting a synchro. Each lcore does
+ * these compare and swap (CAS) operations several times::
+ *
Quoted 4 lines
+ * Relaxed CAS update counter.val[0] + 2; counter.val[0] + 1; + * Acquired CAS update counter.val[0] + 2; counter.val[0] + 1; + * Released CAS update counter.val[0] + 2; counter.val[0] + 1; + * Acquired_Released CAS update counter.val[0] + 2; counter.val[0] + 1;
The array index in "counter.val[0] + 1", is incorrect, I believe.
Just a nitpick, but "Relaxed CAS update" can go last to match the order in the code.
<snip>
Review
Quoted 7 lines
(quoted diff) +7
-0+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64) +/* + * rte_atomic128_cmp_exchange() should update a 128 bits counter's +first 64 + * bits by 2 and the second 64 bits by 1 in this test. It should return +true + * if the compare exchange operation successful.
"operation successful" -> "operation is successful"
Quoted 2 lines
+ * This test repeat 128 bits compare and swap operations 10K rounds. In +each
"repeat" -> "repeats"
Thanks,
Gage
Re: [dpdk-dev] [PATCH v2 2/3] test/atomic: add 128b compare and swap test
Quoted 9 lines
-----Original Message-----
From: Eads, Gage <[email protected]>
Sent: Monday, June 24, 2019 11:10 PM
To: Phil Yang (Arm Technology China) <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>
Subject: RE: [PATCH v2 2/3] test/atomic: add 128b compare and swap test
Quoted 21 lines
Hi Phil,
Looks good overall, just a few documentation issues.
<snip>
Quoted 14 lines
+ *
+ * - Test "128b compare and swap" (aarch64 andx86_64only)
+ *
+ * - Initialize 128-bit atomic variables to zero.
+ *
+ * - Invoke ``test_atomici128_cmp_exchange()`` on each lcore. Before
doing
+ * anything else, the cores are waiting a synchro. Each lcore does
+ * these compare and swap (CAS) operations several times::
+ *+ * Relaxed CAS update counter.val[0] + 2; counter.val[0] + 1; + * Acquired CAS update counter.val[0] + 2; counter.val[0] + 1; + * Released CAS update counter.val[0] + 2; counter.val[0] + 1; + * Acquired_Released CAS update counter.val[0] + 2; counter.val[0] + 1;
Hi Gage,
Quoted 1 line
The array index in "counter.val[0] + 1", is incorrect, I believe.
Yes, you are correct. I will fix it. It should be 'counter.val[2] + 1'.
Quoted 3 lines
Just a nitpick, but "Relaxed CAS update" can go last to match the order in the code.
Sure. Thank you for your correction.
Quoted 12 lines
<snip>
Review
Quoted 7 lines
(quoted diff) +7
-0+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64) +/* + * rte_atomic128_cmp_exchange() should update a 128 bits counter's +first 64 + * bits by 2 and the second 64 bits by 1 in this test. It should +return true + * if the compare exchange operation successful."operation successful" -> "operation is successful"
Yes.
Quoted 5 lines
Quoted 3 lines
+ * This test repeat 128 bits compare and swap operations 10K rounds. +In each
"repeat" -> "repeats"
Yes.
Quoted 3 lines
Thanks,
Gage
Thanks,
Phil
PATCH v2 3/3 eal/stack: enable lock-free stack for aarch64
In reply to Phil YangEnable both c11 atomic and non c11 atomic lock-free stack for aarch64.
--- doc/guides/rel_notes/release_19_08.rst | 3 +++ lib/librte_stack/rte_stack_lf_c11.h | 4 ++-- lib/librte_stack/rte_stack_lf_generic.h | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-)
doc/guides/rel_notes/release_19_08.rst +3 -0
@@ -88,6 +88,9 @@ New Features * Added multi-queue support to allow one af_xdp vdev with multiple netdev queues +* **Added Lock-free Stack for aarch64.** + + The lock-free stack implementation is enabled for aarch64 platforms. Removed Items -------------
lib/librte_stack/rte_stack_lf_c11.h +2 -2
@@ -36,7 +36,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list, struct rte_stack_lf_elem *last, unsigned int num) { -#ifndef RTE_ARCH_X86_64 +#if !defined(RTE_ARCH_X86_64 ) && !defined(RTE_ARCH_ARM64 )RTE_SET_USED (first);RTE_SET_USED (last);RTE_SET_USED (list);
@@ -88,7 +88,7 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list, void **obj_table, struct rte_stack_lf_elem **last) { -#ifndef RTE_ARCH_X86_64 +#if !defined(RTE_ARCH_X86_64 ) && !defined(RTE_ARCH_ARM64 )RTE_SET_USED (obj_table);RTE_SET_USED (last);RTE_SET_USED (list);
lib/librte_stack/rte_stack_lf_generic.h +2 -2
@@ -36,7 +36,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list, struct rte_stack_lf_elem *last, unsigned int num) { -#ifndef RTE_ARCH_X86_64 +#if !defined(RTE_ARCH_X86_64 ) && !defined(RTE_ARCH_ARM64 )RTE_SET_USED (first);RTE_SET_USED (last);RTE_SET_USED (list);
@@ -84,7 +84,7 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list, void **obj_table, struct rte_stack_lf_elem **last) { -#ifndef RTE_ARCH_X86_64 +#if !defined(RTE_ARCH_X86_64 ) && !defined(RTE_ARCH_ARM64 )RTE_SET_USED (obj_table);RTE_SET_USED (last);RTE_SET_USED (list);
Signature · 1 line
2.7.4
Re: [dpdk-dev] [PATCH v2 3/3] eal/stack: enable lock-free stack for aarch64
Quoted 4 lines
Subject: [PATCH v2 3/3] eal/stack: enable lock-free stack for aarch64
Enable both c11 atomic and non c11 atomic lock-free stack for aarch64.
Quoted 9 lines
Signed-off-by Phil Yang <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>Tested-by Honnappa Nagarahalli <[email protected]>--- doc/guides/rel_notes/release_19_08.rst | 3 +++ lib/librte_stack/rte_stack_lf_c11.h | 4 ++-- lib/librte_stack/rte_stack_lf_generic.h | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-)
Please update doc/guides/prog_guide/env_abstraction_layer.rst as well -- it states that the lock-free stack is "currently limited to the x86_64 platform."
Thanks,
Gage
Re: [dpdk-dev] [PATCH v2 3/3] eal/stack: enable lock-free stack for aarch64
Quoted 23 lines
-----Original Message-----
From: Eads, Gage <[email protected]>
Sent: Monday, June 24, 2019 11:15 PM
To: Phil Yang (Arm Technology China) <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>
Subject: RE: [PATCH v2 3/3] eal/stack: enable lock-free stack for aarch64Quoted 4 lines
Subject: [PATCH v2 3/3] eal/stack: enable lock-free stack for aarch64
Enable both c11 atomic and non c11 atomic lock-free stack for aarch64.
Quoted 9 lines
Signed-off-by Phil Yang <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>Tested-by Honnappa Nagarahalli <[email protected]>--- doc/guides/rel_notes/release_19_08.rst | 3 +++ lib/librte_stack/rte_stack_lf_c11.h | 4 ++-- lib/librte_stack/rte_stack_lf_generic.h | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-)
Quoted 2 lines
Please update
doc/guides/prog_guide/env_abstraction_layer.rstas well -- it states that the lock-free stack is "currently limited to thex86_64platform."
Thanks, Gage. I will update it in next version.
Quoted 2 lines
Thanks,
Gage
Thanks,
Phil
Hi Phil,
Quoted 14 lines
lib/librte_eal/common/include/generic/rte_atomic.h +4
-0@@ -1081,6 +1081,18 @@ static inline voidrte_atomic64_clear (rte_atomic64_t *v)/*------------------------ 128 bit atomic operations -------------------------*/ +/** +* 128-bit integer structure. + */ +RTE_STD_C11
lib/librte_eal/common/include/generic/rte_atomic.h static inline void · line 1098
Quoted 10 lines
lib/librte_eal/common/include/generic/rte_atomic.h +8
-0+typedef struct { +RTE_STD_C11 + union { + uint64_t val[2]; + __extension____int128 int128; + }; +} __rte_aligned(16) rte_int128_t; +#ifdef __DOXYGEN__
This change breaks 32-bit x86 builds*. A couple ways to resolve this are 1) with RTE_ARCH_* ifdefs, or 2) keep duplicate definitions of the struct in the aarch64 and x86 header files.
Thanks,
Gage
*http://mails.dpdk.org/archives/test-report/2019-June/086586.html
Quoted 22 lines
-----Original Message-----
From: Eads, Gage <[email protected]>
Sent: Monday, June 24, 2019 10:46 PM
To: Phil Yang (Arm Technology China) <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>
Subject: RE: [PATCH v2 1/3] eal/arm64: add 128-bit atomic compare exchangeHi Phil,
Quoted 11 lines
lib/librte_eal/common/include/generic/rte_atomic.h +0
-1@@ -1081,6 +1081,18 @@ static inline voidrte_atomic64_clear (rte_atomic64_t *v)/*------------------------ 128 bit atomic operations -------------------------*/
Quoted 15 lines
Quoted 4 lines
lib/librte_eal/common/include/generic/rte_atomic.h +3
-0+/** +* 128-bit integer structure. +*/ lib/librte_eal/common/include/generic/rte_atomic.h static inline void rte_atomic64_clear(rte_atomic64_t *v) · line 1096
Quoted 10 lines
lib/librte_eal/common/include/generic/rte_atomic.h +9
-0+RTE_STD_C11 +typedef struct { +RTE_STD_C11 + union { +uint64_t val[2 ]; + __extension____int128 int128; + }; +} __rte_aligned(16) rte_int128_t; +#ifdef __DOXYGEN__
Hi Gage,
Quoted 3 lines
This change breaks 32-bit x86 builds*. A couple ways to resolve this are 1) with RTE_ARCH_* ifdefs, or 2) keep duplicate definitions of the struct in the aarch64 and x86 header files.
OK. Let's follow the first approach. I will update it in the new version. Thanks!
Quoted 5 lines
Thanks,
Gage*http://mails.dpdk.org/archives/test-report/2019-June/086586.html
Thanks,
Phil
PATCH v3 1/3 eal/arm64: add 128-bit atomic compare exchange
In reply to Phil YangAdd 128-bit atomic compare exchange on aarch64.
---
v3:
1. Avoid duplication code with macro. (Jerin Jocob)
2. Make invalid memory order to strongest barrier. (Jerin Jocob)
3. Update doc/guides/prog_guide/env_abstraction_layer.rst. (Eads Gage)
4. Fix 32-bit x86 builds issue. (Eads Gage)
5. Correct documentation issues in UT. (Eads Gage)
.../common/include/arch/arm/rte_atomic_64.h | 165 +++++++++++++++++++++ .../common/include/arch/x86/rte_atomic_64.h | 12 -- lib/librte_eal/common/include/generic/rte_atomic.h | 17 ++- 3 files changed, 181 insertions(+), 13 deletions(-)
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +165 -0
@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2015 Cavium, Inc +* Copyright(c) 2019 Arm Limited */ #ifndef _RTE_ATOMIC_ARM64_H_
@@ -14,6 +15,9 @@ extern "C" {#endif #include "generic/rte_atomic.h" +#include <rte_branch_prediction.h> +#include <rte_compat.h> +#include <rte_debug.h> #define dsb (opt) asm volatile("dsb " #opt : : : "memory")#define dmb (opt) asm volatile("dmb " #opt : : : "memory")
@@ -40,6 +44,167 @@ extern "C" {#define rte_cio_rmb () dmb(oshld) +/*------------------------ 128 bit atomic operations -------------------------*/ + +#define RTE_HAS_ACQ (mo) ((mo) != __ATOMIC_RELAXED && (mo) != __ATOMIC_RELEASE) +#define RTE_HAS_RLS (mo) ((mo) == __ATOMIC_RELEASE || \ + (mo) == __ATOMIC_ACQ_REL || \ + (mo) == __ATOMIC_SEQ_CST) + +#define RTE_MO_LOAD (mo) (RTE_HAS_ACQ((mo)) \ + ? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED) +#define RTE_MO_STORE (mo) (RTE_HAS_RLS((mo)) \ + ? __ATOMIC_RELEASE : __ATOMIC_RELAXED) + +#ifdef __ARM_FEATURE_ATOMICS +#define __ATOMIC128_CAS_OP (cas_op_name, op_string) \ +static inline rte_int128_t \ +cas_op_name(rte_int128_t *dst, rte_int128_t old, \ + rte_int128_t updated) \ +{ \ +/* caspX instructions register pair must start from even-numbered +* register at operand 1. +* So, specify registers for local variables here. +*/ \ + register uint64_t x0 __asm("x0") = (uint64_t)old.val[0]; \ + register uint64_t x1 __asm("x1") = (uint64_t)old.val[1]; \ + register uint64_t x2 __asm("x2") = (uint64_t)updated.val[0]; \ + register uint64_t x3 __asm("x3") = (uint64_t)updated.val[1]; \ + asmvolatile ( \ + op_string" %[old0], %[old1], %[upd0], %[upd1], [%[dst]]" \ + : [old0]"+r" (x0), \ + [old1]"+r" (x1) \ + : [upd0]"r" (x2), \ + [upd1]"r" (x3), \ + [dst]"r" (dst) \ + :"memory" ); \ + old.val[0 ] = x0; \ + old.val[1 ] = x1; \ +return old; \ +} + +__ATOMIC128_CAS_OP(__rte_cas_relaxed, "casp") +__ATOMIC128_CAS_OP(__rte_cas_acquire, "caspa") +__ATOMIC128_CAS_OP(__rte_cas_release, "caspl") +__ATOMIC128_CAS_OP(__rte_cas_acq_rel, "caspal") +#else +#define __ATOMIC128_LDX_OP (ldx_op_name, op_string) \ +static inline rte_int128_t \ +ldx_op_name(const rte_int128_t *src) \ +{ \ + rte_int128_t ret; \ + asm volatile( \ + op_string " %0, %1, %2" \ + : "=&r" (ret.val[0]), \ + "=&r" (ret.val[1]) \ + : "Q" (src->val[0]) \ + : "memory"); \ + return ret; \ +} + +__ATOMIC128_LDX_OP(__rte_ldx_relaxed, "ldxp") +__ATOMIC128_LDX_OP(__rte_ldx_acquire, "ldaxp") + +#define __ATOMIC128_STX_OP (stx_op_name, op_string) \ +static inline uint32_t \ +stx_op_name(rte_int128_t *dst, const rte_int128_t src) \ +{ \ + uint32_t ret; \ + asm volatile( \ + op_string " %w0, %1, %2, %3" \ + : "=&r" (ret) \ + : "r" (src.val[0]), \ + "r" (src.val[1]), \ + "Q" (dst->val[0]) \ + : "memory"); \ +/* Return 0 on success, 1 on failure */ \ +return ret; \ +} + +__ATOMIC128_STX_OP(__rte_stx_relaxed, "stxp") +__ATOMIC128_STX_OP(__rte_stx_release, "stlxp") +#endif + +static inline int __rte_experimental +rte_atomic128_cmp_exchange (rte_int128_t *dst, +rte_int128_t *exp, +const rte_int128_t *src, +unsigned int weak, +int success, +int failure) +{ +/* Always do strong CAS */ +RTE_SET_USED (weak); +/* Ignore memory ordering for failure, memory order for +* success must be stronger or equal +*/ +RTE_SET_USED (failure); +/* Find invalid memory order */ +RTE_ASSERT (success == __ATOMIC_RELAXED + || success == __ATOMIC_ACQUIRE + || success == __ATOMIC_RELEASE + || success == __ATOMIC_ACQ_REL + || success == __ATOMIC_SEQ_CST); + +#ifdef __ARM_FEATURE_ATOMICS +rte_int128_t expected = *exp; +rte_int128_t desired = *src; +rte_int128_t old; + +if (success == __ATOMIC_RELAXED) + old =__rte_cas_relaxed (dst, expected, desired); +else if (success == __ATOMIC_ACQUIRE) + old =__rte_cas_acquire (dst, expected, desired); +else if (success == __ATOMIC_RELEASE) + old =__rte_cas_release (dst, expected, desired); +else + old =__rte_cas_acq_rel (dst, expected, desired); +#else +int ldx_mo =RTE_MO_LOAD (success); +int stx_mo =RTE_MO_STORE (success); +uint32_t ret =1 ; + registerrte_int128_t expected = *exp; + registerrte_int128_t desired = *src; + registerrte_int128_t old; + +/* ldx128 can not guarantee atomic, +* Must write back src or old to verify atomicity of ldx128; +*/ +do { +if (ldx_mo == __ATOMIC_RELAXED) + old =__rte_ldx_relaxed (dst); +else + old =__rte_ldx_acquire (dst); + +if (likely (old.int128 == expected.int128)) { +if (stx_mo == __ATOMIC_RELAXED) + ret =__rte_stx_relaxed (dst, desired); +else + ret =__rte_stx_release (dst, desired); + }else { +/* In the failure case (since 'weak' is ignored and only +* weak == 0 is implemented), expected should contain the +* atomically read value of dst. This means, 'old' needs +* to be stored back to ensure it was read atomically. +*/ +if (stx_mo == __ATOMIC_RELAXED) + ret =__rte_stx_relaxed (dst, old); +else + ret =__rte_stx_release (dst, old); + } + }while (unlikely (ret)); +#endif + +/* Unconditionally updating expected removes +* an 'if' statement. +* expected should already be in register if +* not in the cache. +*/ + *exp = old; + +return (old.int128 == expected.int128); +} + #ifdef __cplusplus }#endif
lib/librte_eal/common/include/arch/x86/rte_atomic_64.h +0 -12
@@ -212,18 +212,6 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)/*------------------------ 128 bit atomic operations -------------------------*/ -/** -* 128-bit integer structure. -*/ -RTE_STD_C11 -typedef struct { -RTE_STD_C11 - union { -uint64_t val[2 ]; - __extension____int128 int128; - }; -} __rte_aligned(16) rte_int128_t; - static inline int __rte_experimental rte_atomic128_cmp_exchange(rte_int128_t *dst, rte_int128_t *exp,
lib/librte_eal/common/include/generic/rte_atomic.h +16 -1
@@ -1081,6 +1081,20 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)/*------------------------ 128 bit atomic operations -------------------------*/ +#if defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 ) +/** +* 128-bit integer structure. +*/ +RTE_STD_C11 +typedef struct { +RTE_STD_C11 + union { +uint64_t val[2 ]; + __extension____int128 int128; + }; +} __rte_aligned(16) rte_int128_t; +#endif +#ifdef __DOXYGEN__/**
@@ -1093,7 +1107,8 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)* *exp = *dst * @endcode * -* @note This function is currently only available for the x86-64 platform. +* @note This function is currently available for the x86-64 and aarch64 +* platforms. * * @note The success and failure arguments must be one of the __ATOMIC_* values * defined in the C++11 standard. For details on their behavior, refer to the
Signature · 1 line
2.7.4
PATCH v3 2/3 test/atomic: add 128b compare and swap test
Add 128b atomic compare and swap test for aarch64 and x86_64.
--- app/test/test_atomic.c | 122 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 120 insertions(+), 2 deletions(-)
app/test/test_atomic.c +120 -2
@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2019 Arm Limited */ #include <stdio.h>
@@ -20,7 +21,7 @@* Atomic Variables * ================ * -* - The main test function performs three subtests. The first test +* - The main test function performs four subtests. The first test * checks that the usual inc/dec/add/sub functions are working * correctly: *
@@ -61,11 +62,27 @@* atomic_sub(&count, tmp+1); * * - At the end of the test, the *count* value must be 0. +* +* - Test "128b compare and swap" (aarch64 and x86_64 only) +* +* - Initialize 128-bit atomic variables to zero. +* +* - Invoke ``test_atomici128_cmp_exchange()`` on each lcore. Before doing +* anything else, the cores are waiting a synchro. Each lcore does +* these compare and swap (CAS) operations several times:: +* +* Acquired CAS update counter.val[0] + 2; counter.val[1] + 1; +* Released CAS update counter.val[0] + 2; counter.val[1] + 1; +* Acquired_Released CAS update counter.val[0] + 2; counter.val[1] + 1; +* Relaxed CAS update counter.val[0] + 2; counter.val[1] + 1; +* +* - At the end of the test, the *count128* first 64-bit value and +* second 64-bit value differ by the total iterations. */ #define NUM_ATOMIC_TYPES 3 -#define N 10000 +#define N 1000000static rte_atomic16_t a16;static rte_atomic32_t a32;
@@ -73,6 +90,10 @@ static rte_atomic64_t a64;static rte_atomic64_t count;static rte_atomic32_t synchro; +#if defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 ) +static rte_int128_t count128; +#endif +static int test_atomic_usual (__attribute__((unused))void *arg) {
@@ -216,6 +237,72 @@ test_atomic_dec_and_test(__attribute__((unused)) void *arg)return 0 ; } +#if defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 ) +/* +* rte_atomic128_cmp_exchange() should update a 128 bits counter's first 64 +* bits by 2 and the second 64 bits by 1 in this test. It should return true +* if the compare exchange operation is successful. +* This test repeats 128 bits compare and swap operations 10K rounds. In each +* iteration it runs compare and swap operation with different memory models. +*/ +static int +test_atomic128_cmp_exchange (__attribute__((unused))void *arg) +{ +rte_int128_t expected; +int success; +unsigned int i; + +while (rte_atomic32_read (&synchro) ==0 ) + ; + + expected = count128; + +for (i =0 ; i <N ; i++) { +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, &expected, + &desired,1 , __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); + }while (success ==0 ); + +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, &expected, + &desired,1 , __ATOMIC_RELEASE, __ATOMIC_RELAXED); + }while (success ==0 ); + +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, &expected, + &desired,1 , __ATOMIC_ACQ_REL, __ATOMIC_RELAXED); + }while (success ==0 ); + +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, &expected, + &desired,1 , __ATOMIC_RELAXED, __ATOMIC_RELAXED); + }while (success ==0 ); + } + +return 0 ; +} +#endif +static int test_atomic (void ) {
@@ -340,6 +427,37 @@ test_atomic(void)return -1 ; } +#if defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 ) +/* +* This case tests the functionality of rte_atomic128b_cmp_exchange +* API. It calls rte_atomic128b_cmp_exchange with four kinds of memory +* models successively on each slave core. Once each 128-bit atomic +* compare and swap operation is successful, it updates the global +* 128-bit counter by 2 for the first 64-bit and 1 for the second +* 64-bit. Each slave core iterates this test 10K times. +* At the end of test, verify whether the first 64-bits of the 128-bit +* counter and the second 64bits is differ by the total iterations. If +* it is, the test passes. +*/ +printf ("128b compare and swap test\n" ); +uint64_t iterations =0 ; + +rte_atomic32_clear (&synchro); + count128.val[0 ] =0 ; + count128.val[1 ] =0 ; + +rte_eal_mp_remote_launch (test_atomic128_cmp_exchange,NULL ,SKIP_MASTER ); +rte_atomic32_set (&synchro,1 ); +rte_eal_mp_wait_lcore (); +rte_atomic32_clear (&synchro); + + iterations = count128.val[0 ] - count128.val[1 ]; +if (iterations !=4 *N *(rte_lcore_count ()-1 )) { +printf ("128b compare and swap failed\n" ); +return -1 ; + } +#endif +return 0 ; }
Signature · 1 line
2.7.4
Re: [dpdk-dev] [PATCH v3 2/3] test/atomic: add 128b compare and swap test
Quoted 6 lines
Add 128b atomic compare and swap test for aarch64 and
x86_64.Signed-off-by Phil Yang <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>
Re: [dpdk-dev] [EXT] [PATCH v3 2/3] test/atomic: add 128b compare and swap test
In reply to Phil YangQuoted 16 lines
-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Friday, June 28, 2019 1:42 PM
To: [email protected]
Cc: [email protected]; Jerin Jacob Kollanukkaran <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: [EXT] [PATCH v3 2/3] test/atomic: add 128b compare and swap testAdd 128b atomic compare and swap test for aarch64 and
x86_64.Signed-off-by Phil Yang <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>--- app/test/test_atomic.c | 122
Review
Quoted 2 lines
(quoted diff) +2
-0+#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64) static +rte_int128_t count128;
Moving this to down[1] can reduce one #ifdef cutler.
app/test/test_atomic.c test_atomic_dec_and_test(__attribute__((unused)) void *arg) · line 240
Quoted 8 lines
+
static int
test_atomic_usual(__attribute__((unused)) void *arg){ @@ -216,6 +237,72
@@test_atomic_dec_and_test(__attribute__((unused)) void *arg)app/test/test_atomic.c +1
-0return 0 ; } +#if defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 )
[1] See above,
With above change,
PATCH v3 3/3 eal/stack: enable lock-free stack for aarch64
In reply to Phil YangEnable both c11 atomic and non c11 atomic lock-free stack for aarch64.
--- doc/guides/prog_guide/env_abstraction_layer.rst | 4 ++-- doc/guides/rel_notes/release_19_08.rst | 3 +++ lib/librte_stack/rte_stack_lf_c11.h | 4 ++-- lib/librte_stack/rte_stack_lf_generic.h | 4 ++-- 4 files changed, 9 insertions(+), 6 deletions(-)
doc/guides/prog_guide/env_abstraction_layer.rst +2 -2
@@ -592,8 +592,8 @@ Known Issues Alternatively, applications can use the lock-free stack mempool handler. When considering this handler, note that: - - It is currently limited to the x86_64 platform, because it uses an - instruction (16-byte compare-and-swap) that is not yet available on other + - It is currently limited to the aarch64 and x86_64 platforms, because it uses + an instruction (16-byte compare-and-swap) that is not yet available on other platforms. - It has worse average-case performance than the non-preemptive rte_ring, but software caching (e.g. the mempool cache) can mitigate this by reducing the
doc/guides/rel_notes/release_19_08.rst +3 -0
@@ -99,6 +99,9 @@ New Features Updated ``librte_telemetry`` to fetch the global metrics from the ``librte_metrics`` library. +* **Added Lock-free Stack for aarch64.** + + The lock-free stack implementation is enabled for aarch64 platforms. Removed Items -------------
lib/librte_stack/rte_stack_lf_c11.h +2 -2
@@ -36,7 +36,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list, struct rte_stack_lf_elem *last, unsigned int num) { -#ifndef RTE_ARCH_X86_64 +#if !defined(RTE_ARCH_X86_64 ) && !defined(RTE_ARCH_ARM64 )RTE_SET_USED (first);RTE_SET_USED (last);RTE_SET_USED (list);
@@ -88,7 +88,7 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list, void **obj_table, struct rte_stack_lf_elem **last) { -#ifndef RTE_ARCH_X86_64 +#if !defined(RTE_ARCH_X86_64 ) && !defined(RTE_ARCH_ARM64 )RTE_SET_USED (obj_table);RTE_SET_USED (last);RTE_SET_USED (list);
lib/librte_stack/rte_stack_lf_generic.h +2 -2
@@ -36,7 +36,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list, struct rte_stack_lf_elem *last, unsigned int num) { -#ifndef RTE_ARCH_X86_64 +#if !defined(RTE_ARCH_X86_64 ) && !defined(RTE_ARCH_ARM64 )RTE_SET_USED (first);RTE_SET_USED (last);RTE_SET_USED (list);
@@ -84,7 +84,7 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list, void **obj_table, struct rte_stack_lf_elem **last) { -#ifndef RTE_ARCH_X86_64 +#if !defined(RTE_ARCH_X86_64 ) && !defined(RTE_ARCH_ARM64 )RTE_SET_USED (obj_table);RTE_SET_USED (last);RTE_SET_USED (list);
Signature · 1 line
2.7.4
Re: [dpdk-dev] [PATCH v3 3/3] eal/stack: enable lock-free stack for aarch64
Quoted 5 lines
Enable both c11 atomic and non c11 atomic lock-free stack for aarch64.
Signed-off-by Phil Yang <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>Tested-by Honnappa Nagarahalli <[email protected]>
Thanks,
Gage
Re: [dpdk-dev] [EXT] [PATCH v3 3/3] eal/stack: enable lock-free stack for aarch64
In reply to Phil YangQuoted 56 lines
-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Friday, June 28, 2019 1:42 PM
To: [email protected]
Cc: [email protected]; Jerin Jacob Kollanukkaran <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: [EXT] [PATCH v3 3/3] eal/stack: enable lock-free stack for aarch64 Enable both c11 atomic and non c11 atomic lock-free stack for aarch64.Signed-off-by Phil Yang <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>Tested-by Honnappa Nagarahalli <[email protected]>Removed Items
-------------lib/librte_stack/rte_stack_lf_c11.h +1
-1@@ -36,7 +36,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list, struct rte_stack_lf_elem *last, unsigned int num) { -#ifndef RTE_ARCH_X86_64 +#if !defined(RTE_ARCH_X86_64 ) && !defined(RTE_ARCH_ARM64 )RTE_SET_USED (first);RTE_SET_USED (last);lib/librte_stack/rte_stack_lf_c11.h +1
-1RTE_SET_USED (list);@@ -88,7 +88,7 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list, void **obj_table, struct rte_stack_lf_elem **last) { -#ifndef RTE_ARCH_X86_64 +#if !defined(RTE_ARCH_X86_64 ) && !defined(RTE_ARCH_ARM64 )RTE_SET_USED (obj_table);RTE_SET_USED (last);RTE_SET_USED (list);lib/librte_stack/rte_stack_lf_generic.h +1
-1@@ -36,7 +36,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list, struct rte_stack_lf_elem *last, unsigned int num) { -#ifndef RTE_ARCH_X86_64 +#if !defined(RTE_ARCH_X86_64 ) && !defined(RTE_ARCH_ARM64 )RTE_SET_USED (first);RTE_SET_USED (last);lib/librte_stack/rte_stack_lf_generic.h +0
-0RTE_SET_USED (list);
lib/librte_stack/rte_stack_lf_generic.h __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list, · line 90
Quoted 8 lines
lib/librte_stack/rte_stack_lf_generic.h +1
-1@@ -84,7 +84,7 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list, void **obj_table, struct rte_stack_lf_elem **last) { -#ifndef RTE_ARCH_X86_64 +#if !defined(RTE_ARCH_X86_64 ) && !defined(RTE_ARCH_ARM64 )RTE_SET_USED (obj_table);RTE_SET_USED (last);RTE_SET_USED (list);
Can we remove this all #ifdef clutter by adding the following
$ git diff
lib/librte_stack/rte_stack_lf.h +1 -1
@@ -5,7 +5,7 @@#ifndef _RTE_STACK_LF_H_#define _RTE_STACK_LF_H_ -#ifdef RTE_USE_C11_MEM_MODEL +#if defined (RTE_USE_C11_MEM_MODEL ) && defined(RTE_ARCH_X86_64 ) && defined(RTE_ARCH_ARM64 )#include "rte_stack_lf_c11.h" #else #include "rte_stack_lf_generic.h"
Re: [dpdk-dev] [EXT] [PATCH v3 3/3] eal/stack: enable lock-free stack for aarch64
Quoted 95 lines
-----Original Message-----
From: Jerin Jacob Kollanukkaran [mailto:[email protected]]
Sent: Thursday, July 18, 2019 11:18 PM
To: Phil Yang <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; Eads, Gage <[email protected]>
Subject: RE: [EXT] [PATCH v3 3/3] eal/stack: enable lock-free stack for aarch64Quoted 10 lines
-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Friday, June 28, 2019 1:42 PM
To: [email protected]
Cc: [email protected]; Jerin Jacob Kollanukkaran <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: [EXT] [PATCH v3 3/3] eal/stack: enable lock-free stack for aarch64 Enable both c11 atomic and non c11 atomic lock-free stack foraarch64.
lib/librte_stack/rte_stack_lf_c11.h __rte_stack_lf_pop_elems(struct rte_stack_lf_list
Quoted 23 lines
Signed-off-by Phil Yang <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>Tested-by Honnappa Nagarahalli <[email protected]>Removed Items
-------------lib/librte_stack/rte_stack_lf_c11.h +1
-1@@ -36,7 +36,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list, struct rte_stack_lf_elem *last, unsigned int num) { -#ifndef RTE_ARCH_X86_64 +#if !defined(RTE_ARCH_X86_64 ) && !defined(RTE_ARCH_ARM64 )RTE_SET_USED (first);RTE_SET_USED (last);lib/librte_stack/rte_stack_lf_c11.h +0
-0RTE_SET_USED (list);@@ -88,7 +88,7 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list*list,
lib/librte_stack/rte_stack_lf_generic.h __rte_stack_lf_pop_elems(struct rte_stack_lf_list
Quoted 24 lines
lib/librte_stack/rte_stack_lf_c11.h +1
-1void **obj_table, struct rte_stack_lf_elem **last) { -#ifndef RTE_ARCH_X86_64 +#if !defined(RTE_ARCH_X86_64 ) && !defined(RTE_ARCH_ARM64 )RTE_SET_USED (obj_table);RTE_SET_USED (last);RTE_SET_USED (list);lib/librte_stack/rte_stack_lf_generic.h +1
-1@@ -36,7 +36,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list, struct rte_stack_lf_elem *last, unsigned int num) { -#ifndef RTE_ARCH_X86_64 +#if !defined(RTE_ARCH_X86_64 ) && !defined(RTE_ARCH_ARM64 )RTE_SET_USED (first);RTE_SET_USED (last);lib/librte_stack/rte_stack_lf_generic.h +0
-0RTE_SET_USED (list);@@ -84,7 +84,7 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list*list,
lib/librte_stack/rte_stack_lf_generic.h __rte_stack_lf_pop_elems(struct rte_stack_lf_list · line 90
Quoted 8 lines
lib/librte_stack/rte_stack_lf_generic.h +1
-1void **obj_table, struct rte_stack_lf_elem **last) { -#ifndef RTE_ARCH_X86_64 +#if !defined(RTE_ARCH_X86_64 ) && !defined(RTE_ARCH_ARM64 )RTE_SET_USED (obj_table);RTE_SET_USED (last);RTE_SET_USED (list);Can we remove this all #ifdef clutter by adding the following
$ git difflib/librte_stack/rte_stack_lf.h +1
-1@@ -5,7 +5,7 @@#ifndef _RTE_STACK_LF_H_#define _RTE_STACK_LF_H_ -#ifdef RTE_USE_C11_MEM_MODEL +#if defined (RTE_USE_C11_MEM_MODEL ) && defined(RTE_ARCH_X86_64 )
Quoted 2 lines
&&
+defined(RTE_ARCH_ARM64)
I assume you meant (defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64))?
Quoted 5 lines
#include "
rte_stack_lf_c11.h"
#else
#include "rte_stack_lf_generic.h"
The ifdefs in those two headers prevent DPDK from trying to build rte_atomic128_cmp_exchange() on architectures that don't implement it. So the proposal wouldn't quite work, since rte_stack_lf_generic.h calls rte_atomic128_cmp_exchange().
Something like this could work:
#if !(defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64)) #include rte_stack_lf_stubs.h
#else
#ifdef RTE_USE_C11_MEM_MODEL
#include "rte_stack_lf_c11.h"
#else
#include "rte_stack_lf_generic.h"
#endif
#endif
Where rte_stack_lf_stubs.h is a new header containing stub implementations of __rte_stack_lf_count, __rte_stack_lf_push_elems, and __rte_stack_lf_pop_elems. It still has some ifdef clutter, but less overall.
Re: [dpdk-dev] [EXT] [PATCH v3 3/3] eal/stack: enable lock-free stack for aarch64
Quoted 11 lines
Quoted 11 lines
Can we remove this all #ifdef clutter by adding the following
$ git diff(quoted diff) +0
-0
b/lib/librte_stack/rte_stack_lf.hindexf5581f0c2..46af08b83100644lib/librte_stack/rte_stack_lf.h +0
-0@@ -5,7 +5,7 @@#ifndef _RTE_STACK_LF_H_#define _RTE_STACK_LF_H_
Quoted 8 lines
lib/librte_stack/rte_stack_lf.h line 8
Quoted 2 lines
lib/librte_stack/rte_stack_lf.h +1
-1-#ifdef RTE_USE_C11_MEM_MODEL +#if defined (RTE_USE_C11_MEM_MODEL) &&
defined(RTE_ARCH_X86_64)Quoted 2 lines
&&
+defined(RTE_ARCH_ARM64)I assume you meant (
defined(RTE_ARCH_X86_64)||defined(RTE_ARCH_ARM64))?
Yup.
Quoted 22 lines
Quoted 6 lines
#include "
rte_stack_lf_c11.h"
#else
#include "rte_stack_lf_generic.h"The ifdefs in those two headers prevent DPDK from trying to build
rte_atomic128_cmp_exchange()on architectures that don't implement it. So the proposal wouldn't quite work, sincerte_stack_lf_generic.hcallsrte_atomic128_cmp_exchange().Something like this could work:
#if !(defined(RTE_ARCH_X86_64) ||
defined(RTE_ARCH_ARM64)) #includerte_stack_lf_stubs.h#else #ifdefRTE_USE_C11_MEM_MODEL#include "rte_stack_lf_c11.h"
#else
#include "rte_stack_lf_generic.h"
#endif
#endif
Quoted 3 lines
Where
rte_stack_lf_stubs.his a new header containing stub implementations of__rte_stack_lf_count,__rte_stack_lf_push_elems, and__rte_stack_lf_pop_elems. It still has some ifdef clutter, but less overall.
Agree. I prefer to take this route to reduce the ifdef clutter across generic and c11 files.
Re: [dpdk-dev] [EXT] [PATCH v3 3/3] eal/stack: enable lock-free stack for aarch64
Quoted 56 lines
-----Original Message-----
From: Jerin Jacob Kollanukkaran <[email protected]>
Sent: Friday, July 19, 2019 1:03 PM
To: Eads, Gage <[email protected]>; Phil Yang (Arm Technology China) <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>
Subject: RE: [EXT] [PATCH v3 3/3] eal/stack: enable lock-free stack for aarch64Quoted 11 lines
Quoted 11 lines
Can we remove this all #ifdef clutter by adding the following
$ git diff(quoted diff) +0
-0
b/lib/librte_stack/rte_stack_lf.hindexf5581f0c2..46af08b83100644lib/librte_stack/rte_stack_lf.h +0
-0@@ -5,7 +5,7 @@#ifndef _RTE_STACK_LF_H_#define _RTE_STACK_LF_H_Quoted 8 lines
lib/librte_stack/rte_stack_lf.h line 8
Quoted 2 lines
lib/librte_stack/rte_stack_lf.h +1
-1-#ifdef RTE_USE_C11_MEM_MODEL +#if defined (RTE_USE_C11_MEM_MODEL) &&
defined(RTE_ARCH_X86_64)Quoted 2 lines
&&
+defined(RTE_ARCH_ARM64)I assume you meant (
defined(RTE_ARCH_X86_64)||defined(RTE_ARCH_ARM64))?Yup.
Quoted 9 lines
Quoted 6 lines
#include "
rte_stack_lf_c11.h"
#else
#include "rte_stack_lf_generic.h"The ifdefs in those two headers prevent DPDK from trying to build
rte_atomic128_cmp_exchange()on architectures that don't implement it.So
Quoted 13 lines
the proposal wouldn't quite work, since
rte_stack_lf_generic.hcallsrte_atomic128_cmp_exchange().Something like this could work:
#if !(defined(RTE_ARCH_X86_64) ||
defined(RTE_ARCH_ARM64)) #includerte_stack_lf_stubs.h#else #ifdefRTE_USE_C11_MEM_MODEL#include "rte_stack_lf_c11.h"
#else
#include "rte_stack_lf_generic.h"
#endif
#endif
Quoted 7 lines
Quoted 2 lines
Where
rte_stack_lf_stubs.his a new header containing stub implementations of__rte_stack_lf_count,__rte_stack_lf_push_elems,and
Quoted 1 line
__rte_stack_lf_pop_elems. It still has some ifdef clutter, but less overall.Agree. I prefer to take this route to reduce the ifdef clutter across generic and c11 files.
Got it. I will update it in the next version.
Thanks,
Phil Yang
Quoted 68 lines
-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Friday, June 28, 2019 1:42 PM
To: [email protected]
Cc: [email protected]; Jerin Jacob Kollanukkaran <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: [EXT] [PATCH v3 1/3] eal/arm64: add 128-bit atomic compare exchangeAdd 128-bit atomic compare exchange on aarch64.
Signed-off-by Phil Yang <[email protected]>Tested-by Honnappa Nagarahalli <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>---
v3:
1. Avoid duplication code with macro. (Jerin Jocob) 2. Make invalid memory order to strongest barrier. (Jerin Jocob) 3. Updatedoc/guides/prog_guide/env_abstraction_layer.rst. (Eads Gage) 4. Fix 32-bit x86 builds issue. (Eads Gage) 5. Correct documentation issues in UT. (Eads Gage).../common/include/arch/arm/rte_atomic_64.h | 165 +++++++++++++++++++++ .../common/include/arch/x86/rte_atomic_64.h | 12 -- lib/librte_eal/common/include/generic/rte_atomic.h | 17 ++- 3 files changed, 181 insertions(+), 13 deletions(-)lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +4
-0@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2015 Cavium, Inc +* Copyright(c) 2019 Arm Limited */ #ifndef _RTE_ATOMIC_ARM64_H_@@ -14,6 +15,9 @@ extern "C" {#endif #include "generic/rte_atomic.h" +#include <rte_branch_prediction.h> +#include <rte_compat.h> +#include <rte_debug.h> #define dsb (opt) asm volatile("dsb " #opt : : : "memory") #define dmb(opt) asm volatile("dmb " #opt : : : "memory") @@ -40,6 +44,167 @@ extern "C" {#define
rte_cio_rmb()dmb(oshld)lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5
-0+/*------------------------ 128 bit atomic operations +-------------------------*/ + +#define RTE_HAS_ACQ (mo) ((mo) != __ATOMIC_RELAXED && (mo) != +__ATOMIC_RELEASE) #define RTE_HAS_RLS(mo) ((mo) == __ATOMIC_RELEASE|| \
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5
-0+ (mo) == __ATOMIC_ACQ_REL || \ + (mo) == __ATOMIC_SEQ_CST) + +#define RTE_MO_LOAD (mo) (RTE_HAS_ACQ((mo)) \ + ? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED) #define
RTE_MO_STORE(mo)lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +4
-0+(RTE_HAS_RLS ((mo)) \ + ? __ATOMIC_RELEASE : __ATOMIC_RELAXED) + +#ifdef __ARM_FEATURE_ATOMICS
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 69
Quoted 10 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +10
-0+#define __ATOMIC128_CAS_OP (cas_op_name, op_string) \ +static inline rte_int128_t \ +cas_op_name(rte_int128_t *dst, rte_int128_t old, \ + rte_int128_t updated) \ +{ \ +/* caspX instructions register pair must start from even-numbered +* register at operand 1. +* So, specify registers for local variables here. +*/ \ + register uint64_t x0 __asm("x0") = (uint64_t)old.val[0]; \
I understand CASP limitation on register has to be even and odd. Is there anyway to remove explicit x0 register allocation and choose compiler to decide the register. Some reason with optimize(03) gcc makes correctly but not clang.
Hardcoding to specific register makes compiler to not optimize the stuff, especially if it is inline function.
Quoted 99 lines
-----Original Message-----
From: Jerin Jacob Kollanukkaran
Sent: Wednesday, July 3, 2019 5:56 PM
To: Phil Yang <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: RE: [EXT] [PATCH v3 1/3] eal/arm64: add 128-bit atomic compare exchangeQuoted 56 lines
-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Friday, June 28, 2019 1:42 PM
To: [email protected]
Cc: [email protected]; Jerin Jacob Kollanukkaran <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: [EXT] [PATCH v3 1/3] eal/arm64: add 128-bit atomic compare exchangeAdd 128-bit atomic compare exchange on aarch64.
Signed-off-by Phil Yang <[email protected]>Tested-by Honnappa Nagarahalli <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>---
v3:
1. Avoid duplication code with macro. (Jerin Jocob) 2. Make invalid memory order to strongest barrier. (Jerin Jocob) 3. Updatedoc/guides/prog_guide/env_abstraction_layer.rst. (Eads Gage) 4. Fix 32-bit x86 builds issue. (Eads Gage) 5. Correct documentation issues in UT. (Eads Gage).../common/include/arch/arm/rte_atomic_64.h | 165 +++++++++++++++++++++ .../common/include/arch/x86/rte_atomic_64.h | 12 -- lib/librte_eal/common/include/generic/rte_atomic.h | 17 ++- 3 files changed, 181 insertions(+), 13 deletions(-)lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +4
-0@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2015 Cavium, Inc +* Copyright(c) 2019 Arm Limited */ #ifndef _RTE_ATOMIC_ARM64_H_@@ -14,6 +15,9 @@ extern "C" {#endif #include "generic/rte_atomic.h" +#include <rte_branch_prediction.h> +#include <rte_compat.h> +#include <rte_debug.h> #define dsb (opt) asm volatile("dsb " #opt : : : "memory") #definedmb(opt) asm
volatile("dmb " #opt : : : "memory")@@ -40,6 +44,167 @@ extern "C" {#define
rte_cio_rmb()dmb(oshld)lib/librte_eal/common/include/arch/arm/rte_atomic_64.h
Quoted 5 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5
-0+/*------------------------ 128 bit atomic operations +-------------------------*/ + +#define RTE_HAS_ACQ (mo) ((mo) != __ATOMIC_RELAXED && (mo) != +__ATOMIC_RELEASE) #define RTE_HAS_RLS(mo) ((mo) ==
__ATOMIC_RELEASEQuoted 11 lines
|| \
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5
-0+ (mo) == __ATOMIC_ACQ_REL || \ + (mo) == __ATOMIC_SEQ_CST) + +#define RTE_MO_LOAD (mo) (RTE_HAS_ACQ((mo)) \ + ? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED) #define
RTE_MO_STORE(mo)lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +4
-0+(RTE_HAS_RLS ((mo)) \ + ? __ATOMIC_RELEASE : __ATOMIC_RELAXED) + +#ifdef __ARM_FEATURE_ATOMICSlib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 69
Quoted 10 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +10
-0+#define __ATOMIC128_CAS_OP (cas_op_name, op_string) \ +static inline rte_int128_t \ +cas_op_name(rte_int128_t *dst, rte_int128_t old, \ + rte_int128_t updated) \ +{ \ +/* caspX instructions register pair must start from even-numbered +* register at operand 1. +* So, specify registers for local variables here. +*/ \ + register uint64_t x0 __asm("x0") = (uint64_t)old.val[0]; \I understand CASP limitation on register has to be even and odd.
Is there anyway to remove explicit x0 register allocation and choose compiler to decide the register. Some reason with optimize(03) gcc makes correctly but not clang.
Quoted 2 lines
Hardcoding to specific register makes compiler to not optimize the stuff, especially if it is inline function.
It look like the limitation fixed recently in gcc.
https://patches.linaro.org/patch/147991/
Not sure about old gcc and clang. ARM compiler experts may know the exact status
<snip>
Quoted 82 lines
Quoted 54 lines
Quoted 48 lines
Subject: [EXT] [PATCH v3 1/3] eal/arm64: add 128-bit atomic compare exchange
Add 128-bit atomic compare exchange on aarch64.
Signed-off-by Phil Yang <[email protected]>Tested-by Honnappa Nagarahalli <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>---
v3:
1. Avoid duplication code with macro. (Jerin Jocob) 2. Make invalid memory order to strongest barrier. (Jerin Jocob) 3. Updatedoc/guides/prog_guide/env_abstraction_layer.rst. (Eads Gage) 4. Fix 32-bit x86 builds issue. (Eads Gage) 5. Correct documentation issues in UT. (Eads Gage).../common/include/arch/arm/rte_atomic_64.h | 165 +++++++++++++++++++++ .../common/include/arch/x86/rte_atomic_64.h | 12 -- lib/librte_eal/common/include/generic/rte_atomic.h | 17 ++- 3 files changed, 181 insertions(+), 13 deletions(-)lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +4
-0@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2015 Cavium, Inc +* Copyright(c) 2019 Arm Limited */ #ifndef _RTE_ATOMIC_ARM64_H_@@ -14,6 +15,9 @@ extern "C" {#endif #include "generic/rte_atomic.h" +#include <rte_branch_prediction.h> +#include <rte_compat.h> +#include <rte_debug.h> #define dsb (opt) asm volatile("dsb " #opt : : : "memory") #definedmb(opt) asm
volatile("dmb " #opt : : : "memory")@@ -40,6 +44,167 @@ extern "C" {#define
rte_cio_rmb()dmb(oshld)lib/librte_eal/common/include/arch/arm/rte_atomic_64.h
Quoted 5 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5
-0+/*------------------------ 128 bit atomic operations +-------------------------*/ + +#define RTE_HAS_ACQ (mo) ((mo) != __ATOMIC_RELAXED && (mo) != +__ATOMIC_RELEASE) #define RTE_HAS_RLS(mo) ((mo) ==
__ATOMIC_RELEASEQuoted 12 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" {
Quoted 12 lines
|| \
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5
-0+ (mo) == __ATOMIC_ACQ_REL || \ + (mo) == __ATOMIC_SEQ_CST) + +#define RTE_MO_LOAD (mo) (RTE_HAS_ACQ((mo)) \ + ? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED) #define
RTE_MO_STORE(mo)lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5
-0+(RTE_HAS_RLS ((mo)) \ + ? __ATOMIC_RELEASE : __ATOMIC_RELAXED) + +#ifdef __ARM_FEATURE_ATOMICS +#define __ATOMIC128_CAS_OP (cas_op_name, op_string)\
Quoted 15 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 69
Quoted 9 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +9
-0+static inline rte_int128_t \ +cas_op_name (rte_int128_t *dst,rte_int128_t old, \ +rte_int128_t updated) \ +{ \ +/* caspX instructions register pair must start from even-numbered +* register at operand 1. +* So, specify registers for local variables here. +*/ \ + register uint64_t x0 __asm("x0") = (uint64_t)old.val[0]; \I understand CASP limitation on register has to be even and odd. Is there anyway to remove explicit x0 register allocation and choose compiler to decide the register. Some reason with optimize(03) gcc makes correctly but not clang.
Quoted 9 lines
Quoted 2 lines
Hardcoding to specific register makes compiler to not optimize the stuff, especially if it is inline function.
It look like the limitation fixed recently in gcc.
https://patches.linaro.org/patch/147991/Not sure about old gcc and clang. ARM compiler experts may know the exact status
We could use syntax as follows, an example is in [1]
static inline rte_int128_t__rte_casp(rte_int128_t *dst, rte_int128_t old, rte_int128_t updated, int mo)
{
__asm__ volatile("caspl %0, %H0, %1, %H1, [%2]"
: "+r" (old)
: "r" (updated), "r" (dst)
: "memory");
return old;
}Quoted 122 lines
-----Original Message-----
From: dev <[email protected]> On Behalf Of Honnappa Nagarahalli
Sent: Friday, July 5, 2019 9:51 AM
To: Jerin Jacob Kollanukkaran <[email protected]>; Phil Yang (Arm Technology China) <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Gavin Hu (Arm Technology China) <[email protected]>; Honnappa Nagarahalli <[email protected]>; nd <[email protected]>; [email protected]; nd <[email protected]>
Subject: Re: [dpdk-dev] [EXT] [PATCH v3 1/3] eal/arm64: add 128-bit atomic compare exchange<snip>
Quoted 1 line
Quoted 1 line
Quoted 1 line
Subject: [EXT] [PATCH v3 1/3] eal/arm64: add 128-bit atomic
compare
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
exchange
Add 128-bit atomic compare exchange on aarch64.
Signed-off-by Phil Yang <[email protected]>Tested-by Honnappa NagarahalliQuoted 1 line
Quoted 1 line
Quoted 1 line
Reviewed-by Honnappa NagarahalliQuoted 5 lines
Quoted 5 lines
Quoted 5 lines
---
v3:
1. Avoid duplication code with macro. (Jerin Jocob) 2. Make invalid memory order to strongest barrier. (Jerin Jocob) 3. Updatedoc/guides/prog_guide/env_abstraction_layer.rst. (Eads Gage) 4.Fix
Quoted 1 line
Quoted 1 line
Quoted 1 line
32-bit x86 builds issue. (Eads Gage) 5. Correct documentation
issues
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
in UT. (Eads Gage)
.../common/include/arch/arm/rte_atomic_64.h | 165 +++++++++++++++++++++ .../common/include/arch/x86/rte_atomic_64.h | 12 -- lib/librte_eal/common/include/generic/rte_atomic.h | 17 ++- 3 files changed, 181 insertions(+), 13 deletions(-)diff --git
a/lib/librte_eal/common/include/arch/arm/rte_atomic_64.hQuoted 14 lines
Quoted 14 lines
Quoted 14 lines
b/lib/librte_eal/common/include/arch/arm/rte_atomic_64.hindex 97060e4..2080c4d 100644lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +1
-0@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2015 Cavium, Inc +* Copyright(c) 2019 Arm Limited */ #ifndef _RTE_ATOMIC_ARM64_H_@@ -14,6 +15,9 @@ extern "C" {#endif Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +3
-0#include "generic/rte_atomic.h" +#include <rte_branch_prediction.h> +#include <rte_compat.h> +#include <rte_debug.h> #define dsb (opt) asm volatile("dsb " #opt : : : "memory") #definedmb(opt) asm
volatile("dmb " #opt : : : "memory")@@ -40,6+44,167
Quoted 8 lines
Quoted 8 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h
Quoted 8 lines
@@ extern "C" {
#define
rte_cio_rmb()dmb(oshld)lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +4
-0+/*------------------------ 128 bit atomic operations +-------------------------*/ + +#define RTE_HAS_ACQ (mo) ((mo) != __ATOMIC_RELAXED &&(mo) !=
Quoted 15 lines
Quoted 14 lines
Quoted 1 line
+__ATOMIC_RELEASE) #define
RTE_HAS_RLS(mo)((mo) ==
__ATOMIC_RELEASElib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" {
Quoted 12 lines
|| \
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5
-0+ (mo) == __ATOMIC_ACQ_REL || \ + (mo) == __ATOMIC_SEQ_CST) + +#define RTE_MO_LOAD (mo) (RTE_HAS_ACQ((mo)) \ + ? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED) #define
RTE_MO_STORE(mo)lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5
-0+(RTE_HAS_RLS ((mo)) \ + ? __ATOMIC_RELEASE : __ATOMIC_RELAXED) + +#ifdef __ARM_FEATURE_ATOMICS +#define __ATOMIC128_CAS_OP (cas_op_name, op_string)\
Quoted 5 lines
Quoted 5 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" {
Quoted 5 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5
-0+static inline rte_int128_t \ +cas_op_name (rte_int128_t *dst,rte_int128_t old, \ +rte_int128_t updated) \ +{ \ +/* caspX instructions register pair must start from even- numbered
Quoted 10 lines
Quoted 10 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 69
Quoted 4 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +4
-0+* register at operand 1. +* So, specify registers for local variables here. +*/ \ + register uint64_t x0 __asm("x0") = (uint64_t)old.val[0]; \I understand CASP limitation on register has to be even and odd. Is there anyway to remove explicit x0 register allocation and choose compiler to decide the register. Some reason with optimize(03) gcc makes correctly but not clang.
Quoted 7 lines
Quoted 2 lines
Hardcoding to specific register makes compiler to not optimize the stuff, especially if it is inline function.
It look like the limitation fixed recently in gcc.
https://patches.linaro.org/patch/147991/Not sure about old gcc and clang. ARM compiler experts may know
the exact
Quoted 2 lines
status
We could use syntax as follows, an example is in [1] static inline
rte_int128_t__rte_casp(rte_int128_t *dst, rte_int128_t old, rte_int128_t updated, int mo)
Quoted 7 lines
{
__asm__ volatile("caspl %0, %H0, %1, %H1, [%2]"
: "+r" (old)
: "r" (updated), "r" (dst)
: "memory");
return old;
}We have used this format for mempool/octeontx2 but clang wasn't too happy.
dpdk/drivers/mempool/octeontx2/otx2_mempool_ops.c:151:15: error: value size does not match register size specified by the constraint and modifier [-Werror,-Wasm-operand-widths]
[t0] "=&r" (t0), [t1] "=&r" (t1), [t2] "=&r" (t2),
^
dpdk/drivers/mempool/octeontx2/otx2_mempool_ops.c:82:9: note: use constraint modifier "w"
"casp %[t0], %H[t0], %[wdata], %H[wdata], [%[loc]]\n"Had to change it to hand coded asm
http://patches.dpdk.org/patch/56110/
Quoted 2 lines
Quoted 13 lines
-----Original Message-----
From: Pavan Nikhilesh Bhagavatula <[email protected]>
Sent: Friday, July 5, 2019 12:37 PM
To: Honnappa Nagarahalli <[email protected]>; [email protected]; Phil Yang (Arm Technology China) <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>; [email protected]; nd <[email protected]>
Subject: RE: [EXT] [PATCH v3 1/3] eal/arm64: add 128-bit atomic compare exchange
<snip>
Quoted 54 lines
Quoted 9 lines
Quoted 8 lines
Quoted 2 lines
Review
Quoted 2 lines
(quoted diff) +2
-0+#ifdef __ARM_FEATURE_ATOMICS +#define __ATOMIC128_CAS_OP(cas_op_name, op_string)\
Quoted 5 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" {
Quoted 5 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5
-0+static inline rte_int128_t \ +cas_op_name (rte_int128_t *dst,rte_int128_t old, \ +rte_int128_t updated) \ +{ \ +/* caspX instructions register pair must start from even- numbered
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" {
Quoted 4 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +4
-0+* register at operand 1. +* So, specify registers for local variables here. +*/ \ + register uint64_t x0 __asm("x0") = (uint64_t)old.val[0];\
Quoted 20 lines
Quoted 6 lines
Quoted 6 lines
I understand CASP limitation on register has to be even and odd. Is there anyway to remove explicit x0 register allocation and choose compiler to decide the register. Some reason with optimize(03) gcc makes correctly but not clang.
Quoted 7 lines
Quoted 2 lines
Hardcoding to specific register makes compiler to not optimize the stuff, especially if it is inline function.
It look like the limitation fixed recently in gcc.
https://patches.linaro.org/patch/147991/Not sure about old gcc and clang. ARM compiler experts may know
the exact
Quoted 2 lines
status
We could use syntax as follows, an example is in [1] static inline
rte_int128_t__rte_casp(rte_int128_t *dst, rte_int128_t old, rte_int128_t updated, int mo)Quoted 7 lines
{ __asm__ volatile("caspl %0, %H0, %1, %H1, [%2]" : "+r" (old) : "r" (updated), "r" (dst) : "memory"); return old; }We have used this format for mempool/octeontx2 but clang wasn't too happy.
dpdk/drivers/mempool/octeontx2/otx2_mempool_ops.c:151:15: error:
value size does not match register size specified by the constraint and modifier [-Werror,-Wasm-operand-widths][t0] "=&r" (t0), [t1] "=&r" (t1), [t2] "=&r" (t2), ^ dpdk/drivers/mempool/octeontx2/otx2_mempool_ops.c:82:9: note: use constraint modifier "w" "casp %[t0], %H[t0], %[wdata], %H[wdata], [%[loc]]\n"
Quoted 3 lines
Had to change it to hand coded asm
Hi Jerin,
The update from the compiler team is 'the LSE CASP fix has not been backported to older GCC branches'. So, currently, this seems the only approach works for all versions of GCC and Clang. I think we can add another optimization patch for this once all the compilers were ready.
Thanks,
Phil
Quoted 3 lines
Quoted 3 lines
Quoted 84 lines
-----Original Message-----
From: Phil Yang (Arm Technology China) <[email protected]>
Sent: Tuesday, July 9, 2019 2:58 PM
To: Pavan Nikhilesh Bhagavatula <[email protected]>; Honnappa Nagarahalli <[email protected]>; Jerin Jacob Kollanukkaran <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>; [email protected]; nd <[email protected]>; nd <[email protected]>
Subject: RE: [EXT] [PATCH v3 1/3] eal/arm64: add 128-bit atomic compare exchangeQuoted 13 lines
-----Original Message-----
From: Pavan Nikhilesh Bhagavatula <[email protected]>
Sent: Friday, July 5, 2019 12:37 PM
To: Honnappa Nagarahalli <[email protected]>; [email protected]; Phil Yang (Arm Technology China) <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>; [email protected]; nd <[email protected]>
Subject: RE: [EXT] [PATCH v3 1/3] eal/arm64: add 128-bit atomic compare exchange<snip>
Quoted 52 lines
Quoted 9 lines
Quoted 8 lines
Quoted 2 lines
Review
Quoted 2 lines
(quoted diff) +2
-0+#ifdef __ARM_FEATURE_ATOMICS +#define __ATOMIC128_CAS_OP(cas_op_name, op_string)\
Quoted 5 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" {
Quoted 5 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5
-0+static inline rte_int128_t \ +cas_op_name (rte_int128_t *dst,rte_int128_t old, \ +rte_int128_t updated) \ +{ \ +/* caspX instructions register pair must start from even- numbered
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" {
Quoted 4 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +4
-0+* register at operand 1. +* So, specify registers for local variables here. +*/ \ + register uint64_t x0 __asm("x0") = (uint64_t)old.val[0];\
Quoted 18 lines
Quoted 6 lines
Quoted 6 lines
I understand CASP limitation on register has to be even and odd. Is there anyway to remove explicit x0 register allocation and choose compiler to decide the register. Some reason with optimize(03) gcc makes correctly but not clang.
Quoted 7 lines
Quoted 2 lines
Hardcoding to specific register makes compiler to not optimize the stuff, especially if it is inline function.
It look like the limitation fixed recently in gcc.
https://patches.linaro.org/patch/147991/Not sure about old gcc and clang. ARM compiler experts may know
the exact
Quoted 2 lines
status
We could use syntax as follows, an example is in [1] static inline
rte_int128_t__rte_casp(rte_int128_t *dst,rte_int128_told,Quoted 7 lines
rte_int128_t updated, int mo) { __asm__ volatile("caspl %0, %H0, %1, %H1, [%2]" : "+r" (old) : "r" (updated), "r" (dst) : "memory"); return old; }We have used this format for mempool/octeontx2 but clang wasn't too happy.
dpdk/drivers/mempool/octeontx2/otx2_mempool_ops.c:151:15: error:
value size does not match register size specified by the constraint and modifier [-Werror,-Wasm-operand-widths][t0] "=&r" (t0), [t1] "=&r" (t1), [t2] "=&r" (t2), ^ dpdk/drivers/mempool/octeontx2/otx2_mempool_ops.c:82:9: note: use constraint modifier "w" "casp %[t0], %H[t0], %[wdata], %H[wdata], [%[loc]]\n"Quoted 3 lines
Had to change it to hand coded asm
Quoted 8 lines
Hi Jerin,
The update from the compiler team is 'the LSE CASP fix has not been backported to older GCC branches'.
So, currently, this seems the only approach works for all versions of GCC and Clang.
I think we can add another optimization patch for this once all the compilers were ready.
We are on same page.
Quoted 6 lines
Thanks,
PhilQuoted 3 lines
Quoted 3 lines
Quoted 28 lines
-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Friday, June 28, 2019 1:42 PM
To: [email protected]
Cc: [email protected]; Jerin Jacob Kollanukkaran <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: [EXT] [PATCH v3 1/3] eal/arm64: add 128-bit atomic compare exchangeExternal Email
----------------------------------------------------------------------
Add 128-bit atomic compare exchange on aarch64.Signed-off-by Phil Yang <[email protected]>Tested-by Honnappa Nagarahalli <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>---
(quoted diff) +2
-0+#define RTE_HAS_ACQ(mo) ((mo) != __ATOMIC_RELAXED && (mo) != +__ATOMIC_RELEASE) #define RTE_HAS_RLS(mo) ((mo) ==
__ATOMIC_RELEASE|| \lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5
-0+ (mo) == __ATOMIC_ACQ_REL || \ + (mo) == __ATOMIC_SEQ_CST) + +#define RTE_MO_LOAD (mo) (RTE_HAS_ACQ((mo)) \ + ? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED) #define
RTE_MO_STORE(mo)
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 58
Quoted 3 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +3
-0+(RTE_HAS_RLS ((mo)) \ + ? __ATOMIC_RELEASE : __ATOMIC_RELAXED) +
The one starts with RTE_ are public symbols, If it is generic enough, Move to common layer so that every architecturse can use. If you think, otherwise make it internal
Quoted 1 line
+#ifdef
__ARM_FEATURE_ATOMICS
This define is added in gcc 9.1 and I believe for clang it is not supported yet. So old gcc and clang this will be undefined.
I think, With meson + native build, we can find the presence of ATOMIC support by running a.out. Not sure about make and cross build case. I don't want block this feature because of this, IMO, We can add this code with existing __ARM_FEATURE_ATOMICS scheme and later find a method to enhance it. But please check how to fix it.
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 69
Quoted 10 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +10
-0+#define __ATOMIC128_CAS_OP (cas_op_name, op_string) \ +static inline rte_int128_t \ +cas_op_name(rte_int128_t *dst, rte_int128_t old, \ + rte_int128_t updated) \ +{ \ +/* caspX instructions register pair must start from even-numbered +* register at operand 1. +* So, specify registers for local variables here. +*/ \ + register uint64_t x0 __asm("x0") = (uint64_t)old.val[0]; \
Since direct x0 register used in the code andcas_op_name() and rte_atomic128_cmp_exchange() is inline function, Based on parent function load, we may corrupt x0 register aka Break arm64 ABI. Not sure clobber list will help here or not? Making it as no_inline will help but not sure about the performance impact. May be you can check with compiler team.
We burned our hands with this scheme, see 5b40ec6b966260e0ff66a8a2c689664f75d6a0e6 ("mempool/octeontx2: fix possible arm64 ABI break")
Probably we can choose a scheme for rc2 and adjust as when we have complete clarity.
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 80
Quoted 12 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5
-0+ register uint64_t x1 __asm("x1") = (uint64_t)old.val[1]; \ + register uint64_t x2 __asm("x2") = (uint64_t)updated.val[0]; \ + register uint64_t x3 __asm("x3") = (uint64_t)updated.val[1]; \ + asm volatile( \ + op_string " %[old0], %[old1], %[upd0], %[upd1],[%[dst]]" \
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +6
-0+ : [old0] "+r" (x0), \ + [old1] "+r" (x1) \ + : [upd0] "r" (x2), \ +[upd1] " r" (x3), \ +[dst] " r" (dst) \ +: " memory"); \
Should n't we add x0,x1, x2, x3 in clobber list?
Quoted 9 lines
static inline int
__rte_experimental
rte_atomic128_cmp_exchange(rte_int128_t *dst,lib/librte_eal/common/include/arch/x86/rte_atomic_64.h +0
-0rte_int128_t *exp,lib/librte_eal/common/include/generic/rte_atomic.h +0
-0
lib/librte_eal/common/include/generic/rte_atomic.h static inline void · line 1085
Quoted 5 lines
lib/librte_eal/common/include/generic/rte_atomic.h +1
-0@@ -1081,6 +1081,20 @@ static inline voidrte_atomic64_clear (rte_atomic64_t *v) /*------------------------ 128 bit atomic operations -------------------------*/ +#if defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 )
There is nothing specific to x86 and arm64 here, Can we remove this #ifdef ?
Quoted 14 lines
lib/librte_eal/common/include/generic/rte_atomic.h +13
-0+/** +* 128-bit integer structure. +*/ +RTE_STD_C11 +typedef struct { +RTE_STD_C11 + union { +uint64_t val[2 ]; + __extension____int128 int128; + }; +} __rte_aligned(16) rte_int128_t; +#endif +#ifdef __DOXYGEN__
Quoted 38 lines
-----Original Message-----
From: Jerin Jacob Kollanukkaran <[email protected]>
Sent: Friday, July 19, 2019 2:25 PM
To: Phil Yang (Arm Technology China) <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>; [email protected]
Subject: RE: [EXT] [PATCH v3 1/3] eal/arm64: add 128-bit atomic compare exchangeQuoted 5 lines
-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Friday, June 28, 2019 1:42 PM
To: [email protected]
Cc: [email protected]; Jerin Jacob KollanukkaranQuoted 22 lines
[email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: [EXT] [PATCH v3 1/3] eal/arm64: add 128-bit atomic compare exchangeExternal Email
----------------------------------------------------------------------
Add 128-bit atomic compare exchange on aarch64.Signed-off-by Phil Yang <[email protected]>Tested-by Honnappa Nagarahalli <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>---
(quoted diff) +2
-0+#define RTE_HAS_ACQ(mo) ((mo) != __ATOMIC_RELAXED && (mo) != +__ATOMIC_RELEASE) #define RTE_HAS_RLS(mo) ((mo) ==
__ATOMIC_RELEASE|| \lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5
-0+ (mo) == __ATOMIC_ACQ_REL || \ + (mo) == __ATOMIC_SEQ_CST) + +#define RTE_MO_LOAD (mo) (RTE_HAS_ACQ((mo)) \ + ? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED) #define
Quoted 8 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 58
Quoted 4 lines
RTE_MO_STORE(mo)lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +3
-0+(RTE_HAS_RLS ((mo)) \ + ? __ATOMIC_RELEASE : __ATOMIC_RELAXED) +The one starts with
RTE_are public symbols, If it is generic enough, Move to common layer so that every architecturse can use. If you think, otherwise make it internal
Let's keep it internal. I will remove the 'RTE_' tag.
Quoted 5 lines
Quoted 4 lines
+#ifdef
__ARM_FEATURE_ATOMICS
Quoted 7 lines
This define is added in gcc 9.1 and I believe for clang it is not supported yet. So old gcc and clang this will be undefined.
I think, With meson + native build, we can find the presence of ATOMIC support by running a.out. Not sure about make and cross build case. I don't want block this feature because of this, IMO, We can add this code with existing__ARM_FEATURE_ATOMICSscheme and later find a method to enhance it. But please check how to fix it.
OK.
Quoted 15 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 69
Quoted 11 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +10
-0+#define __ATOMIC128_CAS_OP (cas_op_name, op_string) \ +static inline rte_int128_t \ +cas_op_name(rte_int128_t *dst, rte_int128_t old, \ + rte_int128_t updated) \ +{ \ +/* caspX instructions register pair must start from even-numbered +* register at operand 1. +* So, specify registers for local variables here. +*/ \ + register uint64_t x0 __asm("x0") = (uint64_t)old.val[0]; \Since direct x0 register used in the code and
cas_op_name()andrte_atomic128_cmp_exchange()is inline function, Based on parent function load, we may corrupt x0 register aka
Since x0/x1 and x2/x3 are used a lot and often contain live values.
Maybe to change them to some relatively less frequently used registers like x14/x15 and x16/x17 might help for this case? According to the PCS (Procedure Call Standard), x14-x17 are also temporary registers.
Quoted 1 line
Break arm64 ABI. Not sure clobber list will help here or not?
In my understanding, for the register variable, if it contains a live value in the specified register, the compiler will move the live value into a free register. Since x0~x3 are present in the input/output operands and x0/x1's value needs to be restored to the variable 'old' as a return value.
So I didn't add them into the clobber list.
Quoted 10 lines
Making it as
no_inlinewill help but not sure about the performance impact. May be you can check with compiler team.We burned our hands with this scheme, see
5b40ec6b966260e0ff66a8a2c689664f75d6a0e6("mempool/octeontx2: fix possible arm64 ABI break")Probably we can choose a scheme for rc2 and adjust as when we have complete clarity.
Quoted 14 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 80
Quoted 12 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5
-0+ register uint64_t x1 __asm("x1") = (uint64_t)old.val[1]; \ + register uint64_t x2 __asm("x2") = (uint64_t)updated.val[0]; \ + register uint64_t x3 __asm("x3") = (uint64_t)updated.val[1]; \ + asm volatile( \ + op_string " %[old0], %[old1], %[upd0], %[upd1],[%[dst]]" \
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +6
-0+ : [old0] "+r" (x0), \ + [old1] "+r" (x1) \ + : [upd0] "r" (x2), \ +[upd1] " r" (x3), \ +[dst] " r" (dst) \ +: " memory"); \ Should n't we add x0,x1, x2, x3 in clobber list?
Same as above.
Quoted 13 lines
Quoted 13 lines
static inline int
__rte_experimental
rte_atomic128_cmp_exchange(rte_int128_t *dst,lib/librte_eal/common/include/arch/x86/rte_atomic_64.h +0
-0rte_int128_t *exp,lib/librte_eal/common/include/generic/rte_atomic.h +0
-0@@ -1081,6 +1081,20 @@ static inline voidrte_atomic64_clear (rte_atomic64_t *v)
Quoted 5 lines
lib/librte_eal/common/include/generic/rte_atomic.h static inline void rte_atomic64_clear(rte_atomic64_t *v) · line 1084
Quoted 3 lines
lib/librte_eal/common/include/generic/rte_atomic.h +1
-0/*------------------------ 128 bit atomic operations -------------------------*/ +#if defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 )There is nothing specific to x86 and arm64 here, Can we remove this #ifdef ?
Without this constraint, it will break 32-bit x86 builds.
http://mails.dpdk.org/archives/test-report/2019-June/086586.html
Quoted 15 lines
Quoted 15 lines
lib/librte_eal/common/include/generic/rte_atomic.h +13
-0+/** +* 128-bit integer structure. +*/ +RTE_STD_C11 +typedef struct { +RTE_STD_C11 + union { +uint64_t val[2 ]; + __extension____int128 int128; + }; +} __rte_aligned(16) rte_int128_t; +#endif +#ifdef __DOXYGEN__
Quoted 17 lines
Quoted 8 lines
Quoted 8 lines
(quoted diff) +2
-0+#define RTE_HAS_ACQ(mo) ((mo) != __ATOMIC_RELAXED && (mo) != +__ATOMIC_RELEASE) #define RTE_HAS_RLS(mo) ((mo) ==
__ATOMIC_RELEASE|| \lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5
-0+ (mo) == __ATOMIC_ACQ_REL || \ + (mo) == __ATOMIC_SEQ_CST) + +#define RTE_MO_LOAD (mo) (RTE_HAS_ACQ((mo)) \ + ? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED) #defineQuoted 8 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 58
Quoted 4 lines
RTE_MO_STORE(mo)lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +3
-0+(RTE_HAS_RLS ((mo)) \ + ? __ATOMIC_RELEASE : __ATOMIC_RELAXED) +The one starts with
RTE_are public symbols, If it is generic enough, Move to common layer so that every architecturse can use. If you think, otherwise make it internal
Quoted 1 line
Let's keep it internal. I will remove the '
RTE_' tag.
Probably change to __HAS_ACQ to avoid collision(just in case)
Quoted 13 lines
Quoted 5 lines
Quoted 4 lines
+#ifdef
__ARM_FEATURE_ATOMICSQuoted 7 lines
This define is added in gcc 9.1 and I believe for clang it is not supported yet. So old gcc and clang this will be undefined.
I think, With meson + native build, we can find the presence of ATOMIC support by running a.out. Not sure about make and cross build case. I don't want block this feature because of this, IMO, We can add this code with existing__ARM_FEATURE_ATOMICSscheme and later find a method to enhance it. But please check how to fix it.
Quoted 1 line
OK.
After thinking on this a bit, I think, in order to support old gcc(< gcc 9.1) and clang, We can introduce a config option, where, by default it is disabled and enable In specific config(where we know, lse is supported) and meson config.
i.e
#if defined(__ARM_FEATURE_ATOMICS) || defined(RTE_ARM_FEATURE_ATOMICS)
Quoted 17 lines
Quoted 16 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 69
Quoted 12 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +10
-0+#define __ATOMIC128_CAS_OP (cas_op_name, op_string) \ +static inline rte_int128_t \ +cas_op_name(rte_int128_t *dst, rte_int128_t old, \ + rte_int128_t updated) \ +{ \ +/* caspX instructions register pair must start from even-numbered +* register at operand 1. +* So, specify registers for local variables here. +*/ \ + register uint64_t x0 __asm("x0") = (uint64_t)old.val[0]; \Since direct x0 register used in the code and
cas_op_name()andrte_atomic128_cmp_exchange()is inline function, Based on parent function load, we may corrupt x0 register aka
Quoted 5 lines
Since x0/x1 and x2/x3 are used a lot and often contain live values. Maybe to change them to some relatively less frequently used registers like x14/x15 and x16/x17 might help for this case?
According to the PCS (Procedure Call Standard), x14-x17 are also temporary registers.
X14-x17 are temporary registers but sincecas_op_name() and rte_atomic128_cmp_exchange() are inline functions, Based on the parent function register usage, it _may_ corrupt.
Quoted 8 lines
Quoted 2 lines
Break arm64 ABI. Not sure clobber list will help here or not?
In my understanding, for the register variable, if it contains a live value in the specified register, the compiler will move the live value into a free register. Since x0~x3 are present in the input/output operands and x0/x1's value needs to be restored to the variable 'old' as a return value.
So I didn't add them into the clobber list.
OK
Quoted 12 lines
Quoted 12 lines
Making it as
no_inlinewill help but not sure about the performance impact. May be you can check with compiler team.We burned our hands with this scheme, see
5b40ec6b966260e0ff66a8a2c689664f75d6a0e6("mempool/octeontx2: fix possible arm64 ABI break")Probably we can choose a scheme for rc2 and adjust as when we have complete clarity.
Quoted 1 line
Quoted 6 lines
Quoted 3 lines
Quoted 1 line
+#if
defined(RTE_ARCH_X86_64)||defined(RTE_ARCH_ARM64)There is nothing specific to x86 and arm64 here, Can we remove this #ifdef ?
Without this constraint, it will break 32-bit x86 builds.
http://mails.dpdk.org/archives/test-report/2019-June/086586.html
OK . #ifdef RTE_ARCH_64 would help then.
Quoted 11 lines
Quoted 11 lines
lib/librte_eal/common/include/generic/rte_atomic.h static inline void rte_atomic64_clear(rte_atomic64_t *v) · line 1093
Quoted 11 lines
lib/librte_eal/common/include/generic/rte_atomic.h +9
-0+/** +* 128-bit integer structure. +*/ +RTE_STD_C11 +typedef struct { +RTE_STD_C11 + union { +uint64_t val[2 ]; + __extension____int128 int128;
Instead of guarding RTE_ARCH_64 on this complete structure, How about it only under
#ifdef RTE_ARCH_64__extension__ __int128 int128;
#endif
So that it rte_int128_t will be available for 32bit as well.
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
lib/librte_eal/common/include/generic/rte_atomic.h +4
-0+ }; +} __rte_aligned(16) rte_int128_t; +#endif +#ifdef __DOXYGEN__
Quoted 28 lines
-----Original Message-----
From: Jerin Jacob Kollanukkaran <[email protected]>
Sent: Friday, July 19, 2019 8:35 PM
To: Phil Yang (Arm Technology China) <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>; [email protected]; nd <[email protected]>
Subject: RE: [EXT] [PATCH v3 1/3] eal/arm64: add 128-bit atomic compare exchangeQuoted 17 lines
Quoted 16 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 58
Quoted 12 lines
(quoted diff) +2
-0+#define RTE_HAS_ACQ(mo) ((mo) != __ATOMIC_RELAXED && (mo) != +__ATOMIC_RELEASE) #define RTE_HAS_RLS(mo) ((mo) ==
__ATOMIC_RELEASE|| \lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5
-0+ (mo) == __ATOMIC_ACQ_REL || \ + (mo) == __ATOMIC_SEQ_CST) + +#define RTE_MO_LOAD (mo) (RTE_HAS_ACQ((mo)) \ + ? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED) #define
RTE_MO_STORE(mo)lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +3
-0+(RTE_HAS_RLS ((mo)) \ + ? __ATOMIC_RELEASE : __ATOMIC_RELAXED) +The one starts with
RTE_are public symbols, If it is generic enough, Move to common layer so that every architecturse can use. If you think, otherwise make it internal
Quoted 3 lines
Quoted 1 line
Let's keep it internal. I will remove the '
RTE_' tag.Probably change to
__HAS_ACQto avoid collision(just in case)
OK.
Quoted 18 lines
Quoted 7 lines
Quoted 7 lines
Quoted 5 lines
+#ifdef
__ARM_FEATURE_ATOMICSThis define is added in gcc 9.1 and I believe for clang it is not supported
yet.
Quoted 3 lines
Quoted 3 lines
So old gcc and clang this will be undefined.
I think, With meson + native build, we can find the presence of ATOMIC support by running a.out. Not sure about make and cross buildcase.
Quoted 5 lines
Quoted 3 lines
I don't want block this feature because of this, IMO, We can add this code with existing
__ARM_FEATURE_ATOMICSscheme and later find a method to enhance it. But please check how to fix it.OK.
Quoted 8 lines
After thinking on this a bit, I think, in order to support old gcc(< gcc 9.1) and clang,
We can introduce a config option, where, by default it is disabled and enable In specificconfig(where we know, lse is supported)and meson config.i.e
#ifdefined(__ARM_FEATURE_ATOMICS)||defined(RTE_ARM_FEATURE_ATOMICS)
Cool
Quoted 22 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
+#define
__ATOMIC128_CAS_OP(cas_op_name, op_string)\
Quoted 14 lines
Quoted 5 lines
Quoted 5 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5
-0+static inline rte_int128_t \ +cas_op_name (rte_int128_t *dst,rte_int128_t old, \ +rte_int128_t updated) \ +{ \ +/* caspX instructions register pair must start from even-numbered Quoted 8 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 69
Quoted 4 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +4
-0+* register at operand 1. +* So, specify registers for local variables here. +*/ \ + register uint64_t x0 __asm("x0") = (uint64_t)old.val[0]; \Since direct x0 register used in the code and
cas_op_name()andrte_atomic128_cmp_exchange()is inline function, Based on parent function load, we may corrupt x0 register akaQuoted 2 lines
Since x0/x1 and x2/x3 are used a lot and often contain live values. Maybe to change them to some relatively less frequently used registers
Quoted 8 lines
like
Quoted 3 lines
x14/x15 and x16/x17 might help for this case?
According to the PCS (Procedure Call Standard), x14-x17 are also temporary registers.X14-x17 are temporary registers but since
cas_op_name()andrte_atomic128_cmp_exchange()are inline functions, Based on the parent function register usage, it _may_ corrupt.
Just checked how Linux Kernel does similar things:
https://github.com/torvalds/linux/blob/master/arch/arm64/include/asm/atomic_lse.h#L19
Same methods.
I will finish the benchmarking for the no_inline approach. If it has no significant performance loss, I think we can make it as no_inline.
Quoted 30 lines
Quoted 6 lines
Quoted 4 lines
Break arm64 ABI. Not sure clobber list will help here or not?
In my understanding, for the register variable, if it contains a live value in
the
Quoted 2 lines
specified register, the compiler will move the live value into a free register. Since x0~x3 are present in the input/output operands and x0/x1's value
needs to
Quoted 2 lines
be restored to the variable 'old' as a return value. So I didn't add them into the clobber list.
OK
Quoted 2 lines
Quoted 2 lines
Making it as
no_inlinewill help but not sure about the performanceimpact.
Quoted 12 lines
Quoted 12 lines
May be you can check with compiler team.
We burned our hands with this scheme, see
5b40ec6b966260e0ff66a8a2c689664f75d6a0e6("mempool/octeontx2: fix possible arm64 ABI break")Probably we can choose a scheme for rc2 and adjust as when we have complete clarity.
Quoted 2 lines
+#if
defined(RTE_ARCH_X86_64)||defined(RTE_ARCH_ARM64)
Quoted 7 lines
Quoted 1 line
Quoted 1 line
There is nothing specific to x86 and arm64 here, Can we remove this
#ifdef ?
Quoted 3 lines
Without this constraint, it will break 32-bit x86 builds.
http://mails.dpdk.org/archives/test-report/2019-June/086586.htmlOK . #ifdef
RTE_ARCH_64would help then.
OK.
Quoted 19 lines
Quoted 12 lines
Quoted 12 lines
lib/librte_eal/common/include/generic/rte_atomic.h static inline void rte_atomic64_clear(rte_atomic64_t *v) · line 1093
Quoted 12 lines
lib/librte_eal/common/include/generic/rte_atomic.h +9
-0+/** +* 128-bit integer structure. +*/ +RTE_STD_C11 +typedef struct { +RTE_STD_C11 + union { +uint64_t val[2 ]; + __extension____int128 int128;Instead of guarding
RTE_ARCH_64on this complete structure, How about it only under
#ifdefRTE_ARCH_64__extension____int128int128;
#endif
So that itrte_int128_twill be available for 32bit as well.
Agree, it should be work. But I am not sure.
Hi Gage,
How do you think about this?
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
lib/librte_eal/common/include/generic/rte_atomic.h +4
-0+ }; +} __rte_aligned(16) rte_int128_t; +#endif +#ifdef __DOXYGEN__
Quoted 14 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
lib/librte_eal/common/include/generic/rte_atomic.h +5
-0+/** +* 128-bit integer structure. +*/ +RTE_STD_C11 +typedef struct {Quoted 8 lines
Quoted 4 lines
Quoted 4 lines
lib/librte_eal/common/include/generic/rte_atomic.h static inline void rte_atomic64_clear(rte_atomic64_t *v) · line 1093
Quoted 4 lines
lib/librte_eal/common/include/generic/rte_atomic.h +4
-0+RTE_STD_C11 + union { +uint64_t val[2 ]; + __extension____int128 int128;Instead of guarding
RTE_ARCH_64on this complete structure, How about it only under #ifdefRTE_ARCH_64__extension____int128int128; #endif So that itrte_int128_twill be available for 32bit as well.
Quoted 6 lines
Agree, it should be work. But I am not sure.
Hi Gage,
How do you think about this?
I don't see any harm in that.
PATCH v4 1/3 eal/arm64: add 128-bit atomic compare exchange
In reply to Phil YangAdd 128-bit atomic compare exchange on aarch64.
---
V4:
1. Add RTE_ARM_FEATURE_ATOMICS flag to support LSE CASP instructions. (Jerin Jocob)
2. Fix possible arm64 ABI break by making casp_op_name noinline. (Jerin Jocob)
3. Add rte_stack_lf_stubs.h to reduce the ifdef clutter. (Gage Eads/Jerin Jocob)
v3:
1. Avoid duplication code with macro. (Jerin Jocob)
2. Make invalid memory order to strongest barrier. (Jerin Jocob)
3. Update doc/guides/prog_guide/env_abstraction_layer.rst. (Gage Eads)
4. Fix 32-bit x86 builds issue. (Gage Eads)
5. Correct documentation issues in UT. (Gage Eads)
v2:
Initial version.
config/arm/meson.build | 1 + config/common_base | 5 + config/defconfig_arm64-thunderx2-linuxapp-gcc | 1 + .../common/include/arch/arm/rte_atomic_64.h | 162 +++++++++++++++++++++ .../common/include/arch/x86/rte_atomic_64.h | 12 -- lib/librte_eal/common/include/generic/rte_atomic.h | 17 ++- 6 files changed, 185 insertions(+), 13 deletions(-)
config/arm/meson.build +1 -0
@@ -68,6 +68,7 @@ flags_thunderx_extra = [ ['RTE_USE_C11_MEM_MODEL', false]] flags_thunderx2_extra = [ ['RTE_MACHINE', '"thunderx2"'], + ['RTE_ARM_FEATURE_ATOMICS', true], ['RTE_CACHE_LINE_SIZE', 64], ['RTE_MAX_NUMA_NODES', 2], ['RTE_MAX_LCORE', 256],
config/common_base +5 -0
@@ -1067,3 +1067,8 @@ CONFIG_RTE_APP_CRYPTO_PERF=y # Compile the eventdev application # CONFIG_RTE_APP_EVENTDEV=y + +# +# Compile ARM LSE ATOMIC instructions statically +# +CONFIG_RTE_ARM_FEATURE_ATOMICS=n
config/defconfig_arm64-thunderx2-linuxapp-gcc +1 -0
@@ -6,6 +6,7 @@ CONFIG_RTE_MACHINE="thunderx2" +CONFIG_RTE_ARM_FEATURE_ATOMICS=y CONFIG_RTE_CACHE_LINE_SIZE=64 CONFIG_RTE_MAX_NUMA_NODES=2 CONFIG_RTE_MAX_LCORE=256
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +162 -0
179 lines not on this page · raw message
lib/librte_eal/common/include/arch/x86/rte_atomic_64.h +0 -12
@@ -212,18 +212,6 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)/*------------------------ 128 bit atomic operations -------------------------*/ -/** -* 128-bit integer structure. -*/ -RTE_STD_C11 -typedef struct { -RTE_STD_C11 - union { -uint64_t val[2 ]; - __extension____int128 int128; - }; -} __rte_aligned(16) rte_int128_t; - __rte_experimental static inline int rte_atomic128_cmp_exchange(rte_int128_t *dst,
lib/librte_eal/common/include/generic/rte_atomic.h +16 -1
@@ -1081,6 +1081,20 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)/*------------------------ 128 bit atomic operations -------------------------*/ +/** +* 128-bit integer structure. +*/ +RTE_STD_C11 +typedef struct { +RTE_STD_C11 + union { +uint64_t val[2 ]; +#ifdef RTE_ARCH_64 + __extension____int128 int128; +#endif + }; +} __rte_aligned(16) rte_int128_t; +#ifdef __DOXYGEN__/**
@@ -1093,7 +1107,8 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)* *exp = *dst * @endcode * -* @note This function is currently only available for the x86-64 platform. +* @note This function is currently available for the x86-64 and aarch64 +* platforms. * * @note The success and failure arguments must be one of the __ATOMIC_* values * defined in the C++11 standard. For details on their behavior, refer to the
Signature · 1 line
2.7.4
PATCH v4 2/3 test/atomic: add 128b compare and swap test
Add 128b atomic compare and swap test for aarch64 and x86_64.
--- app/test/test_atomic.c | 120 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 118 insertions(+), 2 deletions(-)
app/test/test_atomic.c +118 -2
@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2019 Arm Limited */ #include <stdio.h>
@@ -20,7 +21,7 @@* Atomic Variables * ================ * -* - The main test function performs three subtests. The first test +* - The main test function performs four subtests. The first test * checks that the usual inc/dec/add/sub functions are working * correctly: *
@@ -61,11 +62,27 @@* atomic_sub(&count, tmp+1); * * - At the end of the test, the *count* value must be 0. +* +* - Test "128b compare and swap" (aarch64 and x86_64 only) +* +* - Initialize 128-bit atomic variables to zero. +* +* - Invoke ``test_atomici128_cmp_exchange()`` on each lcore. Before doing +* anything else, the cores are waiting a synchro. Each lcore does +* these compare and swap (CAS) operations several times:: +* +* Acquired CAS update counter.val[0] + 2; counter.val[1] + 1; +* Released CAS update counter.val[0] + 2; counter.val[1] + 1; +* Acquired_Released CAS update counter.val[0] + 2; counter.val[1] + 1; +* Relaxed CAS update counter.val[0] + 2; counter.val[1] + 1; +* +* - At the end of the test, the *count128* first 64-bit value and +* second 64-bit value differ by the total iterations. */ #define NUM_ATOMIC_TYPES 3 -#define N 10000 +#define N 1000000static rte_atomic16_t a16;static rte_atomic32_t a32;
@@ -216,6 +233,74 @@ test_atomic_dec_and_test(__attribute__((unused)) void *arg)return 0 ; } +#if defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 ) +static rte_int128_t count128; + +/* +* rte_atomic128_cmp_exchange() should update a 128 bits counter's first 64 +* bits by 2 and the second 64 bits by 1 in this test. It should return true +* if the compare exchange operation is successful. +* This test repeats 128 bits compare and swap operations 10K rounds. In each +* iteration it runs compare and swap operation with different memory models. +*/ +static int +test_atomic128_cmp_exchange (__attribute__((unused))void *arg) +{ +rte_int128_t expected; +int success; +unsigned int i; + +while (rte_atomic32_read (&synchro) ==0 ) + ; + + expected = count128; + +for (i =0 ; i <N ; i++) { +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, &expected, + &desired,1 , __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); + }while (success ==0 ); + +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, &expected, + &desired,1 , __ATOMIC_RELEASE, __ATOMIC_RELAXED); + }while (success ==0 ); + +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, &expected, + &desired,1 , __ATOMIC_ACQ_REL, __ATOMIC_RELAXED); + }while (success ==0 ); + +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, &expected, + &desired,1 , __ATOMIC_RELAXED, __ATOMIC_RELAXED); + }while (success ==0 ); + } + +return 0 ; +} +#endif +static int test_atomic (void ) {
@@ -340,6 +425,37 @@ test_atomic(void)return -1 ; } +#if defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 ) +/* +* This case tests the functionality of rte_atomic128b_cmp_exchange +* API. It calls rte_atomic128b_cmp_exchange with four kinds of memory +* models successively on each slave core. Once each 128-bit atomic +* compare and swap operation is successful, it updates the global +* 128-bit counter by 2 for the first 64-bit and 1 for the second +* 64-bit. Each slave core iterates this test 10K times. +* At the end of test, verify whether the first 64-bits of the 128-bit +* counter and the second 64bits is differ by the total iterations. If +* it is, the test passes. +*/ +printf ("128b compare and swap test\n" ); +uint64_t iterations =0 ; + +rte_atomic32_clear (&synchro); + count128.val[0 ] =0 ; + count128.val[1 ] =0 ; + +rte_eal_mp_remote_launch (test_atomic128_cmp_exchange,NULL ,SKIP_MASTER ); +rte_atomic32_set (&synchro,1 ); +rte_eal_mp_wait_lcore (); +rte_atomic32_clear (&synchro); + + iterations = count128.val[0 ] - count128.val[1 ]; +if (iterations !=4 *N *(rte_lcore_count ()-1 )) { +printf ("128b compare and swap failed\n" ); +return -1 ; + } +#endif +return 0 ; }
Signature · 1 line
2.7.4
PATCH v4 3/3 eal/stack: enable lock-free stack for aarch64
In reply to Phil YangEnable both c11 atomic and non c11 atomic lock-free stack for aarch64.
--- doc/guides/prog_guide/env_abstraction_layer.rst | 4 +- doc/guides/rel_notes/release_19_08.rst | 3 ++ lib/librte_stack/rte_stack_lf.h | 4 ++ lib/librte_stack/rte_stack_lf_c11.h | 16 ------- lib/librte_stack/rte_stack_lf_generic.h | 16 ------- lib/librte_stack/rte_stack_lf_stubs.h | 59 +++++++++++++++++++++++++ 6 files changed, 68 insertions(+), 34 deletions(-) create mode 100644 lib/librte_stack/rte_stack_lf_stubs.h
doc/guides/prog_guide/env_abstraction_layer.rst +2 -2
@@ -592,8 +592,8 @@ Known Issues Alternatively, applications can use the lock-free stack mempool handler. When considering this handler, note that: - - It is currently limited to the x86_64 platform, because it uses an - instruction (16-byte compare-and-swap) that is not yet available on other + - It is currently limited to the aarch64 and x86_64 platforms, because it uses + an instruction (16-byte compare-and-swap) that is not yet available on other platforms. - It has worse average-case performance than the non-preemptive rte_ring, but software caching (e.g. the mempool cache) can mitigate this by reducing the
doc/guides/rel_notes/release_19_08.rst +3 -0
@@ -212,6 +212,9 @@ New Features Added multiple cores feature to compression perf tool application. +* **Added Lock-free Stack for aarch64.** + + The lock-free stack implementation is enabled for aarch64 platforms. Removed Items -------------
lib/librte_stack/rte_stack_lf.h +4 -0
@@ -5,11 +5,15 @@#ifndef _RTE_STACK_LF_H_#define _RTE_STACK_LF_H_ +#if !(defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 )) +#include "rte_stack_lf_stubs.h" +#else #ifdef RTE_USE_C11_MEM_MODEL #include "rte_stack_lf_c11.h" #else #include "rte_stack_lf_generic.h" #endif +#endif /** * @internal Push several objects on the lock-free stack (MT-safe).
lib/librte_stack/rte_stack_lf_c11.h +0 -16
@@ -36,12 +36,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list, struct rte_stack_lf_elem *last, unsigned int num) { -#ifndef RTE_ARCH_X86_64 -RTE_SET_USED (first); -RTE_SET_USED (last); -RTE_SET_USED (list); -RTE_SET_USED (num); -#else struct rte_stack_lf_head old_head;int success;
@@ -79,7 +73,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,* to the LIFO len update. */ __atomic_add_fetch (&list->len, num, __ATOMIC_RELEASE); -#endif } static __rte_always_inline struct rte_stack_lf_elem *
@@ -88,14 +81,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,void **obj_table, struct rte_stack_lf_elem **last) { -#ifndef RTE_ARCH_X86_64 -RTE_SET_USED (obj_table ); -RTE_SET_USED (last); -RTE_SET_USED (list); -RTE_SET_USED (num); - -return NULL ; -#else struct rte_stack_lf_head old_head;uint64_t len;int success;
@@ -169,7 +154,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list, } while (success == 0);return old_head.top; -#endif }#endif /* _RTE_STACK_LF_C11_H_ */
lib/librte_stack/rte_stack_lf_generic.h +0 -16
@@ -36,12 +36,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list, struct rte_stack_lf_elem *last, unsigned int num) { -#ifndef RTE_ARCH_X86_64 -RTE_SET_USED (first); -RTE_SET_USED (last); -RTE_SET_USED (list); -RTE_SET_USED (num); -#else struct rte_stack_lf_head old_head;int success;
@@ -75,7 +69,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list, }while (success ==0 );rte_atomic64_add ((rte_atomic64_t *)&list->len, num); -#endif } static __rte_always_inline struct rte_stack_lf_elem *
@@ -84,14 +77,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,void **obj_table, struct rte_stack_lf_elem **last) { -#ifndef RTE_ARCH_X86_64 -RTE_SET_USED (obj_table ); -RTE_SET_USED (last); -RTE_SET_USED (list); -RTE_SET_USED (num); - -return NULL ; -#else struct rte_stack_lf_head old_head;int success;
@@ -159,7 +144,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list, } while (success == 0);return old_head.top; -#endif }#endif /* _RTE_STACK_LF_GENERIC_H_ */
lib/librte_stack/rte_stack_lf_stubs.h new file +59 -0
59 lines not on this page · raw message
Signature · 1 line
2.7.4
Re: [dpdk-dev] [EXT] [PATCH v4 3/3] eal/stack: enable lock-free stack for aarch64
Quoted 10 lines
-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Monday, July 22, 2019 2:14 PM
To: [email protected]
Cc: [email protected]; Jerin Jacob Kollanukkaran <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: [EXT] [PATCH v4 3/3] eal/stack: enable lock-free stack for aarch64Enable both c11 atomic and non c11 atomic lock-free stack for aarch64.
Probably tell why "rte_stack_lf_stubs.h" introduced in git commit to have Better history.
Re: [dpdk-dev] [EXT] [PATCH v4 3/3] eal/stack: enable lock-free stack for aarch64
Quoted 21 lines
-----Original Message-----
From: Jerin Jacob Kollanukkaran <[email protected]>
Sent: Monday, July 22, 2019 6:22 PM
To: Phil Yang (Arm Technology China) <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>
Subject: RE: [EXT] [PATCH v4 3/3] eal/stack: enable lock-free stack for aarch64Quoted 5 lines
-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Monday, July 22, 2019 2:14 PM
To: [email protected]
Cc: [email protected]; Jerin Jacob KollanukkaranQuoted 4 lines
[email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: [EXT] [PATCH v4 3/3] eal/stack: enable lock-free stack for aarch64
Quoted 5 lines
Quoted 1 line
Enable both c11 atomic and non c11 atomic lock-free stack for aarch64.
Probably tell why "
rte_stack_lf_stubs.h" introduced in git commit to have Better history.
Agree. I will update it. Thanks
Thanks,
Phil Yang
Quoted 14 lines
-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Monday, July 22, 2019 2:14 PM
To: [email protected]
Cc: [email protected]; Jerin Jacob Kollanukkaran <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: [EXT] [PATCH v4 1/3] eal/arm64: add 128-bit atomic compare exchange Add 128-bit atomic compare exchange on aarch64.Suggested-by Jerin Jacob <[email protected]>Signed-off-by Phil Yang <[email protected]>Tested-by Honnappa Nagarahalli <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>
config/common_base CONFIG_RTE_APP_CRYPTO_PERF=y
Quoted 3 lines
config/common_base +3
-0+ +# +# Compile ARM LSE ATOMIC instructions statically #
There is NO value for the keyword "statically" here. Right?
config/defconfig_arm64-thunderx2-linuxapp-gcc line 9
Quoted 13 lines
config/common_base +1
-0+CONFIG_RTE_ARM_FEATURE_ATOMICS=nconfig/defconfig_arm64-thunderx2-linuxapp-gcc +1
-0@@ -6,6 +6,7 @@ CONFIG_RTE_MACHINE="thunderx2" +CONFIG_RTE_ARM_FEATURE_ATOMICS=y
Add for octeontx2 as well.
Quoted 16 lines
config/defconfig_arm64-thunderx2-linuxapp-gcc +11
-0CONFIG_RTE_CACHE_LINE_SIZE=64 CONFIG_RTE_MAX_NUMA_NODES=2 CONFIG_RTE_MAX_LCORE=256 +rte_atomic128_cmp_exchange(rte_int128_t *dst, + rte_int128_t *exp, + const rte_int128_t *src, + unsigned int weak, + int success, + int failure) +{ + /* Always do strong CAS */ + RTE_SET_USED(weak); + /* Ignore memory ordering for failure, memory order for + * success must be stronger or equal
config/defconfig_arm64-thunderx2-linuxapp-gcc
Quoted 10 lines
config/defconfig_arm64-thunderx2-linuxapp-gcc +10
-0+ */ + RTE_SET_USED(failure); + /* Find invalid memory order */ + RTE_ASSERT(success == __ATOMIC_RELAXED + || success == __ATOMIC_ACQUIRE + || success == __ATOMIC_RELEASE + || success == __ATOMIC_ACQ_REL + || success == __ATOMIC_SEQ_CST); + +#ifdef __ARM_FEATURE_ATOMICS
Shouldn't it be #if defined(__ARM_FEATURE_ATOMICS) || defined(RTE_ARM_FEATURE_ATOMICS) ?
Quoted 23 lines
-----Original Message-----
From: Jerin Jacob Kollanukkaran <[email protected]>
Sent: Monday, July 22, 2019 6:20 PM
To: Phil Yang (Arm Technology China) <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>
Subject: RE: [EXT] [PATCH v4 1/3] eal/arm64: add 128-bit atomic compare exchangeQuoted 5 lines
-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Monday, July 22, 2019 2:14 PM
To: [email protected]
Cc: [email protected]; Jerin Jacob KollanukkaranQuoted 3 lines
[email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: [EXT] [PATCH v4 1/3] eal/arm64: add 128-bit atomic compareexchange
Quoted 2 lines
Add 128-bit atomic compare exchange on aarch64.
Quoted 9 lines
config/common_base CONFIG_RTE_APP_CRYPTO_PERF=y
Quoted 7 lines
Suggested-by Jerin Jacob <[email protected]>Signed-off-by Phil Yang <[email protected]>Tested-by Honnappa Nagarahalli <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>config/common_base +3
-0+ +# +# Compile ARM LSE ATOMIC instructions statically #There is NO value for the keyword "statically" here. Right?
Agree. Will remove it.
Quoted 11 lines
Quoted 11 lines
config/common_base +1
-0+CONFIG_RTE_ARM_FEATURE_ATOMICS=nconfig/defconfig_arm64-thunderx2-linuxapp-gcc +0
-0@@ -6,6 +6,7 @@
Quoted 6 lines
config/defconfig_arm64-thunderx2-linuxapp-gcc line 9
Quoted 3 lines
config/defconfig_arm64-thunderx2-linuxapp-gcc +1
-0CONFIG_RTE_MACHINE="thunderx2" +CONFIG_RTE_ARM_FEATURE_ATOMICS=yAdd for octeontx2 as well.
OK. Will add it in v5.
Quoted 22 lines
Quoted 22 lines
config/defconfig_arm64-thunderx2-linuxapp-gcc +16
-0CONFIG_RTE_CACHE_LINE_SIZE=64 CONFIG_RTE_MAX_NUMA_NODES=2 CONFIG_RTE_MAX_LCORE=256 +rte_atomic128_cmp_exchange(rte_int128_t *dst, + rte_int128_t *exp, + const rte_int128_t *src, + unsigned int weak, + int success, + int failure) +{ + /* Always do strong CAS */ + RTE_SET_USED(weak); + /* Ignore memory ordering for failure, memory order for + * success must be stronger or equal + */ + RTE_SET_USED(failure); + /* Find invalid memory order */ + RTE_ASSERT(success == __ATOMIC_RELAXED + || success == __ATOMIC_ACQUIRE
Quoted 8 lines
config/defconfig_arm64-thunderx2-linuxapp-gcc
Quoted 5 lines
config/defconfig_arm64-thunderx2-linuxapp-gcc +5
-0+ || success == __ATOMIC_RELEASE + || success == __ATOMIC_ACQ_REL + || success == __ATOMIC_SEQ_CST); + +#ifdef __ARM_FEATURE_ATOMICSShouldn't it be #if
defined(__ARM_FEATURE_ATOMICS)||defined(RTE_ARM_FEATURE_ATOMICS)?
Yes. That was a mistake. Will update in the version 5. Thanks.
Thanks,
Phil Yang
PATCH v5 1/3 eal/arm64: add 128-bit atomic compare exchange
In reply to Phil YangAdd 128-bit atomic compare exchange on aarch64.
---
v5:
1. Enable RTE_ARM_FEATURE_ATOMICS on octeontx2 in default. (Jerin Jocob)
2. Record the reason of introducing "rte_stack_lf_stubs.h" in git commit. (Jerin, Jocob)
3. Fixed a conditional MACRO error in rte_atomic128_cmp_exchange. (Jerin Jocob)
v4:
1. Add RTE_ARM_FEATURE_ATOMICS flag to support LSE CASP instructions. (Jerin Jocob)
2. Fix possible arm64 ABI break by making casp_op_name noinline. (Jerin Jocob)
3. Add rte_stack_lf_stubs.h to reduce the ifdef clutter. (Gage Eads/Jerin Jocob)
v3:
1. Avoid duplication code with macro. (Jerin Jocob)
2. Make invalid memory order to strongest barrier. (Jerin Jocob)
3. Update doc/guides/prog_guide/env_abstraction_layer.rst. (Gage Eads)
4. Fix 32-bit x86 builds issue. (Gage Eads)
5. Correct documentation issues in UT. (Gage Eads)
v2:
Initial version.
config/arm/meson.build | 2 + config/common_base | 5 + config/defconfig_arm64-octeontx2-linuxapp-gcc | 3 + config/defconfig_arm64-thunderx2-linuxapp-gcc | 1 + .../common/include/arch/arm/rte_atomic_64.h | 162 +++++++++++++++++++++ .../common/include/arch/x86/rte_atomic_64.h | 12 -- lib/librte_eal/common/include/generic/rte_atomic.h | 17 ++- 7 files changed, 189 insertions(+), 13 deletions(-)
config/arm/meson.build +2 -0
@@ -68,6 +68,7 @@ flags_thunderx_extra = [ ['RTE_USE_C11_MEM_MODEL', false]] flags_thunderx2_extra = [ ['RTE_MACHINE', '"thunderx2"'], + ['RTE_ARM_FEATURE_ATOMICS', true], ['RTE_CACHE_LINE_SIZE', 64], ['RTE_MAX_NUMA_NODES', 2], ['RTE_MAX_LCORE', 256],
@@ -76,6 +77,7 @@ flags_octeontx2_extra = [ ['RTE_MACHINE', '"octeontx2"'], ['RTE_MAX_NUMA_NODES', 1], ['RTE_MAX_LCORE', 24], + ['RTE_ARM_FEATURE_ATOMICS', true], ['RTE_EAL_IGB_UIO', false], ['RTE_USE_C11_MEM_MODEL', true]]
config/common_base +5 -0
@@ -1067,3 +1067,8 @@ CONFIG_RTE_APP_CRYPTO_PERF=y # Compile the eventdev application # CONFIG_RTE_APP_EVENTDEV=y + +# +# Compile ARM LSE ATOMIC instructions +# +CONFIG_RTE_ARM_FEATURE_ATOMICS=n
config/defconfig_arm64-octeontx2-linuxapp-gcc +3 -0
@@ -19,3 +19,6 @@ CONFIG_RTE_EAL_IGB_UIO=n # Max supported NIX LFs CONFIG_RTE_MAX_VFIO_GROUPS=128 + +# arm64 LSE ATOMIC support +CONFIG_RTE_ARM_FEATURE_ATOMICS=y
config/defconfig_arm64-thunderx2-linuxapp-gcc +1 -0
@@ -6,6 +6,7 @@ CONFIG_RTE_MACHINE="thunderx2" +CONFIG_RTE_ARM_FEATURE_ATOMICS=y CONFIG_RTE_CACHE_LINE_SIZE=64 CONFIG_RTE_MAX_NUMA_NODES=2 CONFIG_RTE_MAX_LCORE=256
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +162 -0
179 lines not on this page · raw message
lib/librte_eal/common/include/arch/x86/rte_atomic_64.h +0 -12
@@ -212,18 +212,6 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)/*------------------------ 128 bit atomic operations -------------------------*/ -/** -* 128-bit integer structure. -*/ -RTE_STD_C11 -typedef struct { -RTE_STD_C11 - union { -uint64_t val[2 ]; - __extension____int128 int128; - }; -} __rte_aligned(16) rte_int128_t; - __rte_experimental static inline int rte_atomic128_cmp_exchange(rte_int128_t *dst,
lib/librte_eal/common/include/generic/rte_atomic.h +16 -1
@@ -1081,6 +1081,20 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)/*------------------------ 128 bit atomic operations -------------------------*/ +/** +* 128-bit integer structure. +*/ +RTE_STD_C11 +typedef struct { +RTE_STD_C11 + union { +uint64_t val[2 ]; +#ifdef RTE_ARCH_64 + __extension____int128 int128; +#endif + }; +} __rte_aligned(16) rte_int128_t; +#ifdef __DOXYGEN__/**
@@ -1093,7 +1107,8 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)* *exp = *dst * @endcode * -* @note This function is currently only available for the x86-64 platform. +* @note This function is currently available for the x86-64 and aarch64 +* platforms. * * @note The success and failure arguments must be one of the __ATOMIC_* values * defined in the C++11 standard. For details on their behavior, refer to the
Signature · 1 line
2.7.4
PATCH v5 2/3 test/atomic: add 128b compare and swap test
Add 128b atomic compare and swap test for aarch64 and x86_64.
--- app/test/test_atomic.c | 120 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 118 insertions(+), 2 deletions(-)
app/test/test_atomic.c +118 -2
@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2019 Arm Limited */ #include <stdio.h>
@@ -20,7 +21,7 @@* Atomic Variables * ================ * -* - The main test function performs three subtests. The first test +* - The main test function performs four subtests. The first test * checks that the usual inc/dec/add/sub functions are working * correctly: *
@@ -61,11 +62,27 @@* atomic_sub(&count, tmp+1); * * - At the end of the test, the *count* value must be 0. +* +* - Test "128b compare and swap" (aarch64 and x86_64 only) +* +* - Initialize 128-bit atomic variables to zero. +* +* - Invoke ``test_atomici128_cmp_exchange()`` on each lcore. Before doing +* anything else, the cores are waiting a synchro. Each lcore does +* these compare and swap (CAS) operations several times:: +* +* Acquired CAS update counter.val[0] + 2; counter.val[1] + 1; +* Released CAS update counter.val[0] + 2; counter.val[1] + 1; +* Acquired_Released CAS update counter.val[0] + 2; counter.val[1] + 1; +* Relaxed CAS update counter.val[0] + 2; counter.val[1] + 1; +* +* - At the end of the test, the *count128* first 64-bit value and +* second 64-bit value differ by the total iterations. */ #define NUM_ATOMIC_TYPES 3 -#define N 10000 +#define N 1000000static rte_atomic16_t a16;static rte_atomic32_t a32;
@@ -216,6 +233,74 @@ test_atomic_dec_and_test(__attribute__((unused)) void *arg)return 0 ; } +#if defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 ) +static rte_int128_t count128; + +/* +* rte_atomic128_cmp_exchange() should update a 128 bits counter's first 64 +* bits by 2 and the second 64 bits by 1 in this test. It should return true +* if the compare exchange operation is successful. +* This test repeats 128 bits compare and swap operations 10K rounds. In each +* iteration it runs compare and swap operation with different memory models. +*/ +static int +test_atomic128_cmp_exchange (__attribute__((unused))void *arg) +{ +rte_int128_t expected; +int success; +unsigned int i; + +while (rte_atomic32_read (&synchro) ==0 ) + ; + + expected = count128; + +for (i =0 ; i <N ; i++) { +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, &expected, + &desired,1 , __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); + }while (success ==0 ); + +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, &expected, + &desired,1 , __ATOMIC_RELEASE, __ATOMIC_RELAXED); + }while (success ==0 ); + +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, &expected, + &desired,1 , __ATOMIC_ACQ_REL, __ATOMIC_RELAXED); + }while (success ==0 ); + +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, &expected, + &desired,1 , __ATOMIC_RELAXED, __ATOMIC_RELAXED); + }while (success ==0 ); + } + +return 0 ; +} +#endif +static int test_atomic (void ) {
@@ -340,6 +425,37 @@ test_atomic(void)return -1 ; } +#if defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 ) +/* +* This case tests the functionality of rte_atomic128b_cmp_exchange +* API. It calls rte_atomic128b_cmp_exchange with four kinds of memory +* models successively on each slave core. Once each 128-bit atomic +* compare and swap operation is successful, it updates the global +* 128-bit counter by 2 for the first 64-bit and 1 for the second +* 64-bit. Each slave core iterates this test 10K times. +* At the end of test, verify whether the first 64-bits of the 128-bit +* counter and the second 64bits is differ by the total iterations. If +* it is, the test passes. +*/ +printf ("128b compare and swap test\n" ); +uint64_t iterations =0 ; + +rte_atomic32_clear (&synchro); + count128.val[0 ] =0 ; + count128.val[1 ] =0 ; + +rte_eal_mp_remote_launch (test_atomic128_cmp_exchange,NULL ,SKIP_MASTER ); +rte_atomic32_set (&synchro,1 ); +rte_eal_mp_wait_lcore (); +rte_atomic32_clear (&synchro); + + iterations = count128.val[0 ] - count128.val[1 ]; +if (iterations !=4 *N *(rte_lcore_count ()-1 )) { +printf ("128b compare and swap failed\n" ); +return -1 ; + } +#endif +return 0 ; }
Signature · 1 line
2.7.4
PATCH v5 3/3 eal/stack: enable lock-free stack for aarch64
In reply to Phil YangEnable both c11 atomic and non c11 atomic lock-free stack for aarch64.
Introduced a new header, rte_stack_lf_stubs.h, to reduce the ifdef clutter across generic and c11 files. The rte_stack_lf_stubs.h contains stub implementations of __rte_stack_lf_count, __rte_stack_lf_push_elems and __rte_stack_lf_pop_elems.
--- doc/guides/prog_guide/env_abstraction_layer.rst | 4 +- doc/guides/rel_notes/release_19_08.rst | 3 ++ lib/librte_stack/rte_stack_lf.h | 4 ++ lib/librte_stack/rte_stack_lf_c11.h | 16 ------- lib/librte_stack/rte_stack_lf_generic.h | 16 ------- lib/librte_stack/rte_stack_lf_stubs.h | 59 +++++++++++++++++++++++++ 6 files changed, 68 insertions(+), 34 deletions(-) create mode 100644 lib/librte_stack/rte_stack_lf_stubs.h
doc/guides/prog_guide/env_abstraction_layer.rst +2 -2
@@ -592,8 +592,8 @@ Known Issues Alternatively, applications can use the lock-free stack mempool handler. When considering this handler, note that: - - It is currently limited to the x86_64 platform, because it uses an - instruction (16-byte compare-and-swap) that is not yet available on other + - It is currently limited to the aarch64 and x86_64 platforms, because it uses + an instruction (16-byte compare-and-swap) that is not yet available on other platforms. - It has worse average-case performance than the non-preemptive rte_ring, but software caching (e.g. the mempool cache) can mitigate this by reducing the
doc/guides/rel_notes/release_19_08.rst +3 -0
@@ -212,6 +212,9 @@ New Features Added multiple cores feature to compression perf tool application. +* **Added Lock-free Stack for aarch64.** + + The lock-free stack implementation is enabled for aarch64 platforms. Removed Items -------------
lib/librte_stack/rte_stack_lf.h +4 -0
@@ -5,11 +5,15 @@#ifndef _RTE_STACK_LF_H_#define _RTE_STACK_LF_H_ +#if !(defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 )) +#include "rte_stack_lf_stubs.h" +#else #ifdef RTE_USE_C11_MEM_MODEL #include "rte_stack_lf_c11.h" #else #include "rte_stack_lf_generic.h" #endif +#endif /** * @internal Push several objects on the lock-free stack (MT-safe).
lib/librte_stack/rte_stack_lf_c11.h +0 -16
@@ -36,12 +36,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list, struct rte_stack_lf_elem *last, unsigned int num) { -#ifndef RTE_ARCH_X86_64 -RTE_SET_USED (first); -RTE_SET_USED (last); -RTE_SET_USED (list); -RTE_SET_USED (num); -#else struct rte_stack_lf_head old_head;int success;
@@ -79,7 +73,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,* to the LIFO len update. */ __atomic_add_fetch (&list->len, num, __ATOMIC_RELEASE); -#endif } static __rte_always_inline struct rte_stack_lf_elem *
@@ -88,14 +81,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,void **obj_table, struct rte_stack_lf_elem **last) { -#ifndef RTE_ARCH_X86_64 -RTE_SET_USED (obj_table ); -RTE_SET_USED (last); -RTE_SET_USED (list); -RTE_SET_USED (num); - -return NULL ; -#else struct rte_stack_lf_head old_head;uint64_t len;int success;
@@ -169,7 +154,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list, } while (success == 0);return old_head.top; -#endif }#endif /* _RTE_STACK_LF_C11_H_ */
lib/librte_stack/rte_stack_lf_generic.h +0 -16
@@ -36,12 +36,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list, struct rte_stack_lf_elem *last, unsigned int num) { -#ifndef RTE_ARCH_X86_64 -RTE_SET_USED (first); -RTE_SET_USED (last); -RTE_SET_USED (list); -RTE_SET_USED (num); -#else struct rte_stack_lf_head old_head;int success;
@@ -75,7 +69,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list, }while (success ==0 );rte_atomic64_add ((rte_atomic64_t *)&list->len, num); -#endif } static __rte_always_inline struct rte_stack_lf_elem *
@@ -84,14 +77,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,void **obj_table, struct rte_stack_lf_elem **last) { -#ifndef RTE_ARCH_X86_64 -RTE_SET_USED (obj_table ); -RTE_SET_USED (last); -RTE_SET_USED (list); -RTE_SET_USED (num); - -return NULL ; -#else struct rte_stack_lf_head old_head;int success;
@@ -159,7 +144,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list, } while (success == 0);return old_head.top; -#endif }#endif /* _RTE_STACK_LF_GENERIC_H_ */
lib/librte_stack/rte_stack_lf_stubs.h new file +59 -0
59 lines not on this page · raw message
Signature · 1 line
2.7.4
Re: [dpdk-dev] [EXT] [PATCH v5 3/3] eal/stack: enable lock-free stack for aarch64
Quoted 16 lines
-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Monday, July 22, 2019 6:36 PM
To: [email protected]
Cc: [email protected]; Jerin Jacob Kollanukkaran <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: [EXT] [PATCH v5 3/3] eal/stack: enable lock-free stack for aarch64Enable both c11 atomic and non c11 atomic lock-free stack for aarch64.
Introduced a new header,
rte_stack_lf_stubs.h, to reduce the ifdef clutter across generic and c11 files. Therte_stack_lf_stubs.hcontains stub implementations of__rte_stack_lf_count,__rte_stack_lf_push_elemsand__rte_stack_lf_pop_elems.
Quoted 5 lines
Suggested-by Gage Eads <[email protected]>Suggested-by Jerin Jacob <[email protected]>Signed-off-by Phil Yang <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>Tested-by Honnappa Nagarahalli <[email protected]>
# Build issue with 32 bit build, Looks like new header file missing in SYMLINK-$(CONFIG_RTE_LIBRTE_STACK)-include
In file included from /home/jerin/dpdk.org/build/include/rte_stack.h:98,
from /home/jerin/dpdk.org/drivers/mempool/stack/rte_mempool_stack.c:7:/home/jerin/dpdk.org/build/include/rte_stack_lf.h:9:10: fatal error: rte_stack_lf_stubs.h: No such file or directory
9 | #include "rte_stack_lf_stubs.h"
| ^~~~~~~~~~~~~~~~~~~~~~Re: [dpdk-dev] [EXT] [PATCH v5 3/3] eal/stack: enable lock-free stack for aarch64
Quoted 38 lines
-----Original Message-----
From: Jerin Jacob Kollanukkaran <[email protected]>
Sent: Monday, July 22, 2019 10:15 PM
To: Phil Yang (Arm Technology China) <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>
Subject: RE: [EXT] [PATCH v5 3/3] eal/stack: enable lock-free stack for aarch64Quoted 5 lines
-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Monday, July 22, 2019 6:36 PM
To: [email protected]
Cc: [email protected]; Jerin Jacob KollanukkaranQuoted 9 lines
[email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: [EXT] [PATCH v5 3/3] eal/stack: enable lock-free stack for aarch64Enable both c11 atomic and non c11 atomic lock-free stack for aarch64.
Introduced a new header,
rte_stack_lf_stubs.h, to reduce the ifdef clutter across generic and c11 files. Therte_stack_lf_stubs.hcontains stub implementations of__rte_stack_lf_count,__rte_stack_lf_push_elemsand
Quoted 7 lines
__rte_stack_lf_pop_elems.Suggested-by Gage Eads <[email protected]>Suggested-by Jerin Jacob <[email protected]>Signed-off-by Phil Yang <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>Tested-by Honnappa Nagarahalli <[email protected]># Build issue with 32 bit build, Looks like new header file missing in SYMLINK- $(CONFIG_RTE_LIBRTE_STACK)-include
Quoted 8 lines
In file included from
/home/jerin/dpdk.org/build/include/rte_stack.h:98,
from/home/jerin/dpdk.org/drivers/mempool/stack/rte_mempool_stack.c:7:/home/jerin/dpdk.org/build/include/rte_stack_lf.h:9:10: fatal error:rte_stack_lf_stubs.h: No such file or directory9 | #include "rte_stack_lf_stubs.h" | ^~~~~~~~~~~~~~~~~~~~~~
Yes. Thanks. Will fix it in next version.
Re: [dpdk-dev] [PATCH v5 3/3] eal/stack: enable lock-free stack for aarch64
In reply to Phil YangQuoted 36 lines
lib/librte_stack/rte_stack_lf_stubs.h new file +13
-0@@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2019 Arm Limited +*/ + +#ifndef _RTE_STACK_LF_STUBS_H_ +#define _RTE_STACK_LF_STUBS_H_ + +#include <rte_common.h> +#include <rte_atomic.h> + +static __rte_always_inline unsigned int __rte_stack_lf_count(struct +rte_stack *s) { +/* stack_lf_push() and stack_lf_pop() do not update the list's contents
lib/librte_stack/rte_stack_lf_stubs.h new file +5
-0+* and stack_lf->len atomically, which can cause the list to appear +* shorter than it actually is if this function is called while other +* threads are modifying the list. +* +* However, given the inherently approximate nature of the
get_countlib/librte_stack/rte_stack_lf_stubs.h new file +2
-0+* callback -- even if the list and its size were updated atomically, +* the size could change between when get_count executes and when the
lib/librte_stack/rte_stack_lf_stubs.h new file +3
-0+* value is returned to the caller -- this is acceptable. +* +* The stack_lf->len updates are placed such that the list may appear to
lib/librte_stack/rte_stack_lf_stubs.h new file +1
-0+* have fewer elements than it does, but will never appear to have more
lib/librte_stack/rte_stack_lf_stubs.h line 31
Quoted 6 lines
lib/librte_stack/rte_stack_lf_stubs.h new file +6
-0+* elements. If the mempool is near-empty to the point that this is a +* concern, the user should consider increasing the mempool size. +*/ +return (unsigned int )rte_atomic64_read ((rte_atomic64_t *) + &s->stack_lf.used.len); +}
Since the stub functions are not functional, these comments should be removed and the function simply return 0. The rte_atomic.h include can be removed as well.
Re: [dpdk-dev] [PATCH v5 3/3] eal/stack: enable lock-free stack for aarch64
Quoted 47 lines
-----Original Message-----
From: Eads, Gage <[email protected]>
Sent: Monday, July 22, 2019 10:35 PM
To: Phil Yang (Arm Technology China) <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>
Subject: RE: [PATCH v5 3/3] eal/stack: enable lock-free stack for aarch64Quoted 38 lines
lib/librte_stack/rte_stack_lf_stubs.h new file +13
-0@@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2019 Arm Limited +*/ + +#ifndef _RTE_STACK_LF_STUBS_H_ +#define _RTE_STACK_LF_STUBS_H_ + +#include <rte_common.h> +#include <rte_atomic.h> + +static __rte_always_inline unsigned int __rte_stack_lf_count(struct +rte_stack *s) { +/* stack_lf_push() and stack_lf_pop() do not update the list's contents
lib/librte_stack/rte_stack_lf_stubs.h new file +5
-0+* and stack_lf->len atomically, which can cause the list to appear +* shorter than it actually is if this function is called while other +* threads are modifying the list. +* +* However, given the inherently approximate nature of the
get_countlib/librte_stack/rte_stack_lf_stubs.h new file +2
-0+* callback -- even if the list and its size were updated atomically, +* the size could change between when get_count executes and when the
lib/librte_stack/rte_stack_lf_stubs.h new file +3
-0+* value is returned to the caller -- this is acceptable. +* +* The stack_lf->len updates are placed such that the list may appear to
lib/librte_stack/rte_stack_lf_stubs.h new file +1
-0+* have fewer elements than it does, but will never appear to have more
lib/librte_stack/rte_stack_lf_stubs.h new file +2
-0+* elements. If the mempool is near-empty to the point that this is a +* concern, the user should consider increasing the mempool size.
Quoted 8 lines
lib/librte_stack/rte_stack_lf_stubs.h line 31
Quoted 4 lines
lib/librte_stack/rte_stack_lf_stubs.h new file +4
-0+*/ +return (unsigned int )rte_atomic64_read ((rte_atomic64_t *) + &s->stack_lf.used.len); +}Since the stub functions are not functional, these comments should be removed and the function simply return 0. The
rte_atomic.hinclude can be removed as well.
OK. Will do it. Thanks.
It looks good. Some minor comments.
Quoted 23 lines
-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Monday, July 22, 2019 6:36 PM
To: [email protected]
Cc: [email protected]; Jerin Jacob Kollanukkaran <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: [EXT] [PATCH v5 1/3] eal/arm64: add 128-bit atomic compare exchange
----------------------------------------------------------------------
Add 128-bit atomic compare exchange on aarch64.Suggested-by Jerin Jacob <[email protected]>Signed-off-by Phil Yang <[email protected]>Tested-by Honnappa Nagarahalli <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>config/common_base index renamed from config/common_base +0
-08ef75c2..16dea5a 100644
config/common_base +0
-0@@ -1067,3 +1067,8 @@ CONFIG_RTE_APP_CRYPTO_PERF=y # Compile theeventdev application #
CONFIG_RTE_APP_EVENTDEV=y
config/common_base CONFIG_RTE_APP_CRYPTO_PERF=y · line 1074
Quoted 5 lines
config/common_base +5
-0+ +# +# Compile ARM LSE ATOMIC instructions +# +CONFIG_RTE_ARM_FEATURE_ATOMICS=n
Move this config after "Compile Environment Abstraction Layer" section. Now it is at end of file. Better to group the configs.
config/defconfig_arm64-octeontx2-linuxapp-gcc CONFIG_RTE_EAL_IGB_UIO=n · line 24
Quoted 12 lines
config/defconfig_arm64-octeontx2-linuxapp-gcc +3
-0@@ -19,3 +19,6 @@ CONFIG_RTE_EAL_IGB_UIO=n # Max supported NIX LFs CONFIG_RTE_MAX_VFIO_GROUPS=128 + +# arm64 LSE ATOMIC support +CONFIG_RTE_ARM_FEATURE_ATOMICS=y
Move this config after CONFIG_RTE_MAX_LCORE=24
config/defconfig_arm64-thunderx2-linuxapp-gcc line 12
Quoted 13 lines
config/defconfig_arm64-thunderx2-linuxapp-gcc +1
-0@@ -6,6 +6,7 @@ CONFIG_RTE_MACHINE="thunderx2" +CONFIG_RTE_ARM_FEATURE_ATOMICS=y CONFIG_RTE_CACHE_LINE_SIZE=64 CONFIG_RTE_MAX_NUMA_NODES=2 CONFIG_RTE_MAX_LCORE=256
Move the new config here
Quoted 11 lines
-----Original Message-----
From: Jerin Jacob Kollanukkaran <[email protected]>
Sent: Monday, July 22, 2019 10:20 PM
To: Phil Yang (Arm Technology China) <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>
Subject: RE: [EXT] [PATCH v5 1/3] eal/arm64: add 128-bit atomic compare exchange
Quoted 1 line
It looks good. Some minor comments.
Thanks. Updated in the new version.
Quoted 68 lines
Quoted 6 lines
-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Monday, July 22, 2019 6:36 PM
To: [email protected]
Cc: [email protected]; Jerin Jacob Kollanukkaranconfig/common_base CONFIG_RTE_APP_CRYPTO_PERF=y # Compile
Quoted 17 lines
[email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: [EXT] [PATCH v5 1/3] eal/arm64: add 128-bit atomic compare exchange
----------------------------------------------------------------------
Add 128-bit atomic compare exchange on aarch64.Suggested-by Jerin Jacob <[email protected]>Signed-off-by Phil Yang <[email protected]>Tested-by Honnappa Nagarahalli <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>config/common_base index renamed from config/common_base +0
-08ef75c2..16dea5a 100644
config/common_base +0
-0@@ -1067,3 +1067,8 @@ CONFIG_RTE_APP_CRYPTO_PERF=y # Compilethe
config/common_base CONFIG_RTE_APP_CRYPTO_PERF=y · line 1074
Quoted 6 lines
eventdev application #
CONFIG_RTE_APP_EVENTDEV=yconfig/common_base +5
-0+ +# +# Compile ARM LSE ATOMIC instructions +# +CONFIG_RTE_ARM_FEATURE_ATOMICS=nMove this config after "Compile Environment Abstraction Layer" section. Now it is at end of file. Better to group the configs.
config/defconfig_arm64-octeontx2-linuxapp-gcc CONFIG_RTE_EAL_IGB_UIO=n · line 24
Quoted 12 lines
config/defconfig_arm64-octeontx2-linuxapp-gcc +3
-0@@ -19,3 +19,6 @@ CONFIG_RTE_EAL_IGB_UIO=n # Max supported NIX LFs CONFIG_RTE_MAX_VFIO_GROUPS=128 + +# arm64 LSE ATOMIC support +CONFIG_RTE_ARM_FEATURE_ATOMICS=yMove this config after
CONFIG_RTE_MAX_LCORE=24config/defconfig_arm64-thunderx2-linuxapp-gcc line 12
Quoted 13 lines
config/defconfig_arm64-thunderx2-linuxapp-gcc +1
-0@@ -6,6 +6,7 @@ CONFIG_RTE_MACHINE="thunderx2" +CONFIG_RTE_ARM_FEATURE_ATOMICS=y CONFIG_RTE_CACHE_LINE_SIZE=64 CONFIG_RTE_MAX_NUMA_NODES=2 CONFIG_RTE_MAX_LCORE=256Move the new config here
PATCH v6 1/3 eal/arm64: add 128-bit atomic compare exchange
In reply to Phil YangAdd 128-bit atomic compare exchange on aarch64.
---
v6:
1. Put the RTE_ARM_FEATURE_ATOMICS flag into EAL group. (Jerin Jocob)
2. Keep rte_stack_lf_stubs.h doing nothing. (Gage Eads)
3. Fixed 32 bit build issue.
v5:
1. Enable RTE_ARM_FEATURE_ATOMICS on octeontx2 in default. (Jerin Jocob)
2. Record the reason of introducing "rte_stack_lf_stubs.h" in git commit. (Jerin, Jocob)
3. Fixed a conditional MACRO error in rte_atomic128_cmp_exchange. (Jerin Jocob)
v4:
1. Add RTE_ARM_FEATURE_ATOMICS flag to support LSE CASP instructions. (Jerin Jocob)
2. Fix possible arm64 ABI break by making casp_op_name noinline. (Jerin Jocob)
3. Add rte_stack_lf_stubs.h to reduce the ifdef clutter. (Gage Eads/Jerin Jocob)
v3:
1. Avoid duplication code with macro. (Jerin Jocob)
2. Make invalid memory order to strongest barrier. (Jerin Jocob)
3. Update doc/guides/prog_guide/env_abstraction_layer.rst. (Gage Eads)
4. Fix 32-bit x86 builds issue. (Gage Eads)
5. Correct documentation issues in UT. (Gage Eads)
v2:
Initial version.
config/arm/meson.build | 2 + config/common_base | 2 + config/defconfig_arm64-octeontx2-linuxapp-gcc | 1 + config/defconfig_arm64-thunderx2-linuxapp-gcc | 1 + .../common/include/arch/arm/rte_atomic_64.h | 162 +++++++++++++++++++++ .../common/include/arch/x86/rte_atomic_64.h | 12 -- lib/librte_eal/common/include/generic/rte_atomic.h | 17 ++- 7 files changed, 184 insertions(+), 13 deletions(-)
config/arm/meson.build +2 -0
@@ -71,11 +71,13 @@ flags_thunderx2_extra = [ ['RTE_CACHE_LINE_SIZE', 64], ['RTE_MAX_NUMA_NODES', 2], ['RTE_MAX_LCORE', 256], + ['RTE_ARM_FEATURE_ATOMICS', true], ['RTE_USE_C11_MEM_MODEL', true]] flags_octeontx2_extra = [ ['RTE_MACHINE', '"octeontx2"'], ['RTE_MAX_NUMA_NODES', 1], ['RTE_MAX_LCORE', 24], + ['RTE_ARM_FEATURE_ATOMICS', true], ['RTE_EAL_IGB_UIO', false], ['RTE_USE_C11_MEM_MODEL', true]]
config/common_base +2 -0
@@ -82,6 +82,8 @@ CONFIG_RTE_MAX_LCORE=128 CONFIG_RTE_MAX_NUMA_NODES=8 CONFIG_RTE_MAX_HEAPS=32 CONFIG_RTE_MAX_MEMSEG_LISTS=64 +# Use LSE ATOMIC instructions +CONFIG_RTE_ARM_FEATURE_ATOMICS=n # each memseg list will be limited to either RTE_MAX_MEMSEG_PER_LIST pages # or RTE_MAX_MEM_MB_PER_LIST megabytes worth of memory, whichever is smaller CONFIG_RTE_MAX_MEMSEG_PER_LIST=8192
config/defconfig_arm64-octeontx2-linuxapp-gcc +1 -0
@@ -9,6 +9,7 @@ CONFIG_RTE_MACHINE="octeontx2" CONFIG_RTE_CACHE_LINE_SIZE=128 CONFIG_RTE_MAX_NUMA_NODES=1 CONFIG_RTE_MAX_LCORE=24 +CONFIG_RTE_ARM_FEATURE_ATOMICS=y # Doesn't support NUMA CONFIG_RTE_EAL_NUMA_AWARE_HUGEPAGES=n
config/defconfig_arm64-thunderx2-linuxapp-gcc +1 -0
@@ -9,3 +9,4 @@ CONFIG_RTE_MACHINE="thunderx2" CONFIG_RTE_CACHE_LINE_SIZE=64 CONFIG_RTE_MAX_NUMA_NODES=2 CONFIG_RTE_MAX_LCORE=256 +CONFIG_RTE_ARM_FEATURE_ATOMICS=y
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +162 -0
179 lines not on this page · raw message
lib/librte_eal/common/include/arch/x86/rte_atomic_64.h +0 -12
@@ -212,18 +212,6 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)/*------------------------ 128 bit atomic operations -------------------------*/ -/** -* 128-bit integer structure. -*/ -RTE_STD_C11 -typedef struct { -RTE_STD_C11 - union { -uint64_t val[2 ]; - __extension____int128 int128; - }; -} __rte_aligned(16) rte_int128_t; - __rte_experimental static inline int rte_atomic128_cmp_exchange(rte_int128_t *dst,
lib/librte_eal/common/include/generic/rte_atomic.h +16 -1
@@ -1081,6 +1081,20 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)/*------------------------ 128 bit atomic operations -------------------------*/ +/** +* 128-bit integer structure. +*/ +RTE_STD_C11 +typedef struct { +RTE_STD_C11 + union { +uint64_t val[2 ]; +#ifdef RTE_ARCH_64 + __extension____int128 int128; +#endif + }; +} __rte_aligned(16) rte_int128_t; +#ifdef __DOXYGEN__/**
@@ -1093,7 +1107,8 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)* *exp = *dst * @endcode * -* @note This function is currently only available for the x86-64 platform. +* @note This function is currently available for the x86-64 and aarch64 +* platforms. * * @note The success and failure arguments must be one of the __ATOMIC_* values * defined in the C++11 standard. For details on their behavior, refer to the
Signature · 1 line
2.7.4
PATCH v6 2/3 test/atomic: add 128b compare and swap test
Add 128b atomic compare and swap test for aarch64 and x86_64.
--- app/test/test_atomic.c | 120 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 118 insertions(+), 2 deletions(-)
app/test/test_atomic.c +118 -2
@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2019 Arm Limited */ #include <stdio.h>
@@ -20,7 +21,7 @@* Atomic Variables * ================ * -* - The main test function performs three subtests. The first test +* - The main test function performs four subtests. The first test * checks that the usual inc/dec/add/sub functions are working * correctly: *
@@ -61,11 +62,27 @@* atomic_sub(&count, tmp+1); * * - At the end of the test, the *count* value must be 0. +* +* - Test "128b compare and swap" (aarch64 and x86_64 only) +* +* - Initialize 128-bit atomic variables to zero. +* +* - Invoke ``test_atomici128_cmp_exchange()`` on each lcore. Before doing +* anything else, the cores are waiting a synchro. Each lcore does +* these compare and swap (CAS) operations several times:: +* +* Acquired CAS update counter.val[0] + 2; counter.val[1] + 1; +* Released CAS update counter.val[0] + 2; counter.val[1] + 1; +* Acquired_Released CAS update counter.val[0] + 2; counter.val[1] + 1; +* Relaxed CAS update counter.val[0] + 2; counter.val[1] + 1; +* +* - At the end of the test, the *count128* first 64-bit value and +* second 64-bit value differ by the total iterations. */ #define NUM_ATOMIC_TYPES 3 -#define N 10000 +#define N 1000000static rte_atomic16_t a16;static rte_atomic32_t a32;
@@ -216,6 +233,74 @@ test_atomic_dec_and_test(__attribute__((unused)) void *arg)return 0 ; } +#if defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 ) +static rte_int128_t count128; + +/* +* rte_atomic128_cmp_exchange() should update a 128 bits counter's first 64 +* bits by 2 and the second 64 bits by 1 in this test. It should return true +* if the compare exchange operation is successful. +* This test repeats 128 bits compare and swap operations 10K rounds. In each +* iteration it runs compare and swap operation with different memory models. +*/ +static int +test_atomic128_cmp_exchange (__attribute__((unused))void *arg) +{ +rte_int128_t expected; +int success; +unsigned int i; + +while (rte_atomic32_read (&synchro) ==0 ) + ; + + expected = count128; + +for (i =0 ; i <N ; i++) { +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, &expected, + &desired,1 , __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); + }while (success ==0 ); + +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, &expected, + &desired,1 , __ATOMIC_RELEASE, __ATOMIC_RELAXED); + }while (success ==0 ); + +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, &expected, + &desired,1 , __ATOMIC_ACQ_REL, __ATOMIC_RELAXED); + }while (success ==0 ); + +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, &expected, + &desired,1 , __ATOMIC_RELAXED, __ATOMIC_RELAXED); + }while (success ==0 ); + } + +return 0 ; +} +#endif +static int test_atomic (void ) {
@@ -340,6 +425,37 @@ test_atomic(void)return -1 ; } +#if defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 ) +/* +* This case tests the functionality of rte_atomic128b_cmp_exchange +* API. It calls rte_atomic128b_cmp_exchange with four kinds of memory +* models successively on each slave core. Once each 128-bit atomic +* compare and swap operation is successful, it updates the global +* 128-bit counter by 2 for the first 64-bit and 1 for the second +* 64-bit. Each slave core iterates this test 10K times. +* At the end of test, verify whether the first 64-bits of the 128-bit +* counter and the second 64bits is differ by the total iterations. If +* it is, the test passes. +*/ +printf ("128b compare and swap test\n" ); +uint64_t iterations =0 ; + +rte_atomic32_clear (&synchro); + count128.val[0 ] =0 ; + count128.val[1 ] =0 ; + +rte_eal_mp_remote_launch (test_atomic128_cmp_exchange,NULL ,SKIP_MASTER ); +rte_atomic32_set (&synchro,1 ); +rte_eal_mp_wait_lcore (); +rte_atomic32_clear (&synchro); + + iterations = count128.val[0 ] - count128.val[1 ]; +if (iterations !=4 *N *(rte_lcore_count ()-1 )) { +printf ("128b compare and swap failed\n" ); +return -1 ; + } +#endif +return 0 ; }
Signature · 1 line
2.7.4
PATCH v6 3/3 eal/stack: enable lock-free stack for aarch64
In reply to Phil YangEnable both c11 atomic and non c11 atomic lock-free stack for aarch64.
Introduced a new header to reduce the ifdef clutter across generic and c11 files. The rte_stack_lf_stubs.h contains stub implementations of __rte_stack_lf_count, __rte_stack_lf_push_elems and __rte_stack_lf_pop_elems.
--- doc/guides/prog_guide/env_abstraction_layer.rst | 4 +-- doc/guides/rel_notes/release_19_08.rst | 3 ++ lib/librte_stack/Makefile | 3 +- lib/librte_stack/rte_stack_lf.h | 4 +++ lib/librte_stack/rte_stack_lf_c11.h | 16 --------- lib/librte_stack/rte_stack_lf_generic.h | 16 --------- lib/librte_stack/rte_stack_lf_stubs.h | 44 +++++++++++++++++++++++++ 7 files changed, 55 insertions(+), 35 deletions(-) create mode 100644 lib/librte_stack/rte_stack_lf_stubs.h
doc/guides/prog_guide/env_abstraction_layer.rst +2 -2
@@ -592,8 +592,8 @@ Known Issues Alternatively, applications can use the lock-free stack mempool handler. When considering this handler, note that: - - It is currently limited to the x86_64 platform, because it uses an - instruction (16-byte compare-and-swap) that is not yet available on other + - It is currently limited to the aarch64 and x86_64 platforms, because it uses + an instruction (16-byte compare-and-swap) that is not yet available on other platforms. - It has worse average-case performance than the non-preemptive rte_ring, but software caching (e.g. the mempool cache) can mitigate this by reducing the
doc/guides/rel_notes/release_19_08.rst +3 -0
@@ -212,6 +212,9 @@ New Features Added multiple cores feature to compression perf tool application. +* **Added Lock-free Stack for aarch64.** + + The lock-free stack implementation is enabled for aarch64 platforms. Removed Items -------------
lib/librte_stack/Makefile +2 -1
@@ -24,6 +24,7 @@ SYMLINK-$(CONFIG_RTE_LIBRTE_STACK)-include := rte_stack.h \ rte_stack_std.h \ rte_stack_lf.h \ rte_stack_lf_generic.h \ - rte_stack_lf_c11.h + rte_stack_lf_c11.h \ + rte_stack_lf_stubs.h include $(RTE_SDK)/mk/rte.lib.mk
lib/librte_stack/rte_stack_lf.h +4 -0
@@ -5,11 +5,15 @@#ifndef _RTE_STACK_LF_H_#define _RTE_STACK_LF_H_ +#if !(defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 )) +#include "rte_stack_lf_stubs.h" +#else #ifdef RTE_USE_C11_MEM_MODEL #include "rte_stack_lf_c11.h" #else #include "rte_stack_lf_generic.h" #endif +#endif /** * @internal Push several objects on the lock-free stack (MT-safe).
lib/librte_stack/rte_stack_lf_c11.h +0 -16
@@ -36,12 +36,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list, struct rte_stack_lf_elem *last, unsigned int num) { -#ifndef RTE_ARCH_X86_64 -RTE_SET_USED (first); -RTE_SET_USED (last); -RTE_SET_USED (list); -RTE_SET_USED (num); -#else struct rte_stack_lf_head old_head;int success;
@@ -79,7 +73,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,* to the LIFO len update. */ __atomic_add_fetch (&list->len, num, __ATOMIC_RELEASE); -#endif } static __rte_always_inline struct rte_stack_lf_elem *
@@ -88,14 +81,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,void **obj_table, struct rte_stack_lf_elem **last) { -#ifndef RTE_ARCH_X86_64 -RTE_SET_USED (obj_table ); -RTE_SET_USED (last); -RTE_SET_USED (list); -RTE_SET_USED (num); - -return NULL ; -#else struct rte_stack_lf_head old_head;uint64_t len;int success;
@@ -169,7 +154,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list, } while (success == 0);return old_head.top; -#endif }#endif /* _RTE_STACK_LF_C11_H_ */
lib/librte_stack/rte_stack_lf_generic.h +0 -16
@@ -36,12 +36,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list, struct rte_stack_lf_elem *last, unsigned int num) { -#ifndef RTE_ARCH_X86_64 -RTE_SET_USED (first); -RTE_SET_USED (last); -RTE_SET_USED (list); -RTE_SET_USED (num); -#else struct rte_stack_lf_head old_head;int success;
@@ -75,7 +69,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list, }while (success ==0 );rte_atomic64_add ((rte_atomic64_t *)&list->len, num); -#endif } static __rte_always_inline struct rte_stack_lf_elem *
@@ -84,14 +77,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,void **obj_table, struct rte_stack_lf_elem **last) { -#ifndef RTE_ARCH_X86_64 -RTE_SET_USED (obj_table ); -RTE_SET_USED (last); -RTE_SET_USED (list); -RTE_SET_USED (num); - -return NULL ; -#else struct rte_stack_lf_head old_head;int success;
@@ -159,7 +144,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list, } while (success == 0);return old_head.top; -#endif }#endif /* _RTE_STACK_LF_GENERIC_H_ */
lib/librte_stack/rte_stack_lf_stubs.h new file +44 -0
44 lines not on this page · raw message
Signature · 1 line
2.7.4
Re: [dpdk-dev] [EXT] [PATCH v6 3/3] eal/stack: enable lock-free stack for aarch64
Quoted 16 lines
-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Monday, July 22, 2019 9:53 PM
To: [email protected]
Cc: [email protected]; Jerin Jacob Kollanukkaran <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: [EXT] [PATCH v6 3/3] eal/stack: enable lock-free stack for aarch64Enable both c11 atomic and non c11 atomic lock-free stack for aarch64.
Introduced a new header to reduce the ifdef clutter across generic and c11 files. The
rte_stack_lf_stubs.hcontains stub implementations of__rte_stack_lf_count,__rte_stack_lf_push_elemsand__rte_stack_lf_pop_elems.
Quoted 5 lines
Suggested-by Gage Eads <[email protected]>Suggested-by Jerin Jacob <[email protected]>Signed-off-by Phil Yang <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>Tested-by Honnappa Nagarahalli <[email protected]>
Quoted 23 lines
-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Monday, July 22, 2019 9:53 PM
To: [email protected]
Cc: [email protected]; Jerin Jacob Kollanukkaran <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: [EXT] [PATCH v6 1/3] eal/arm64: add 128-bit atomic compare exchangeAdd 128-bit atomic compare exchange on aarch64.
Suggested-by Jerin Jacob <[email protected]>Signed-off-by Phil Yang <[email protected]>Tested-by Honnappa Nagarahalli <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>---
config/common_base index renamed from config/common_base +0
-08ef75c2..de6d1e0 100644
config/common_base CONFIG_RTE_MAX_LCORE=128 · line 84
Quoted 3 lines
config/common_base +0
-0@@ -82,6 +82,8 @@ CONFIG_RTE_MAX_LCORE=128 CONFIG_RTE_MAX_NUMA_NODES=8 CONFIG_RTE_MAX_HEAPS=32 CONFIG_RTE_MAX_MEMSEG_LISTS=64
Add new line here.
config/common_base CONFIG_RTE_MAX_LCORE=128 · line 85
Quoted 1 line
config/common_base +1
-0+# Use LSE ATOMIC instructions
I think, you can change the comment to "Use ARM LSE ATOMIC instructions"
config/common_base CONFIG_RTE_MAX_LCORE=128 · line 86
Quoted 1 line
config/common_base +1
-0+CONFIG_RTE_ARM_FEATURE_ATOMICS=n
This patches series has following check patch warning. Please fix it
With above fixes you can add my acked-by in 1/3 and 3/3 patches in next revision.
I think, you can ignore following warning. WARNING:MACRO_WITH_FLOW_CONTROL: Macros with flow control statements should be avoided
[master]dell[dpdk.org] $ ./devtools/checkpatches.sh
### eal/arm64: add 128-bit atomic compare exchange
WARNING:LONG_LINE: line over 80 characters
#103: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:60:
+ rte_int128_t updated) \
WARNING:LONG_LINE: line over 80 characters
#108: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:65:
+ */ \
WARNING:LINE_CONTINUATIONS: Avoid unnecessary line continuations #108: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:65:
+ */ \
WARNING:LONG_LINE: line over 80 characters
#109: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:66:
+ register uint64_t x0 __asm("x0") = (uint64_t)old.val[0]; \
WARNING:LONG_LINE: line over 80 characters
#110: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:67:
+ register uint64_t x1 __asm("x1") = (uint64_t)old.val[1]; \
WARNING:LONG_LINE: line over 80 characters
#111: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:68:
+ register uint64_t x2 __asm("x2") = (uint64_t)updated.val[0]; \
WARNING:LONG_LINE: line over 80 characters
#112: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:69:
+ register uint64_t x3 __asm("x3") = (uint64_t)updated.val[1]; \
WARNING:LONG_LINE: line over 80 characters
#113: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:70:
+ asm volatile( \
WARNING:LONG_LINE: line over 80 characters
#115: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:72:
+ : [old0] "+r" (x0), \
WARNING:LONG_LINE: line over 80 characters
#116: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:73:
+ [old1] "+r" (x1) \
WARNING:LONG_LINE: line over 80 characters
#118: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:75:
+ [upd1] "r" (x3), \
WARNING:LONG_LINE: line over 80 characters
#119: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:76:
+ [dst] "r" (dst) \
WARNING:LONG_LINE: line over 80 characters
#120: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:77:
+ : "memory"); \
WARNING:LONG_LINE: line over 80 characters
#121: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:78:
+ old.val[0] = x0; \
WARNING:LONG_LINE: line over 80 characters
#122: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:79:
+ old.val[1] = x1; \
WARNING:LONG_LINE: line over 80 characters
#123: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:80:
+ return old; \
WARNING:LONG_LINE: line over 80 characters
#135: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:92:
+ rte_int128_t ret; \
WARNING:LONG_LINE: line over 80 characters
#136: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:93:
+ asm volatile( \
WARNING:LONG_LINE: line over 80 characters
#138: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:95:
+ : "=&r" (ret.val[0]), \
WARNING:LONG_LINE: line over 80 characters
#139: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:96:
+ "=&r" (ret.val[1]) \
WARNING:LONG_LINE: line over 80 characters
#140: FILE: lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:97:
+ : "Q" (src->val[0]) \
WARNING:LONG_LINE: line over 80 characters
Quoted 42 lines
-----Original Message-----
From: Jerin Jacob Kollanukkaran <[email protected]>
Sent: Tuesday, July 23, 2019 12:57 AM
To: Phil Yang (Arm Technology China) <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>
Subject: RE: [EXT] [PATCH v6 1/3] eal/arm64: add 128-bit atomic compare exchangeQuoted 5 lines
-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Monday, July 22, 2019 9:53 PM
To: [email protected]
Cc: [email protected]; Jerin Jacob KollanukkaranQuoted 18 lines
[email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: [EXT] [PATCH v6 1/3] eal/arm64: add 128-bit atomic compare exchangeAdd 128-bit atomic compare exchange on aarch64.
Suggested-by Jerin Jacob <[email protected]>Signed-off-by Phil Yang <[email protected]>Tested-by Honnappa Nagarahalli <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>---
config/common_base index renamed from config/common_base +0
-08ef75c2..de6d1e0 100644
config/common_base CONFIG_RTE_MAX_LCORE=128 · line 84
Quoted 3 lines
config/common_base +0
-0@@ -82,6 +82,8 @@ CONFIG_RTE_MAX_LCORE=128 CONFIG_RTE_MAX_NUMA_NODES=8 CONFIG_RTE_MAX_HEAPS=32 CONFIG_RTE_MAX_MEMSEG_LISTS=64Add new line here.
Quoted 3 lines
config/common_base CONFIG_RTE_MAX_LCORE=128 · line 85
Quoted 1 line
config/common_base +1
-0+# Use LSE ATOMIC instructionsI think, you can change the comment to "Use ARM LSE ATOMIC instructions"
OK. Will fix it.
Quoted 7 lines
config/common_base CONFIG_RTE_MAX_LCORE=128 · line 86
Quoted 2 lines
config/common_base +1
-0+CONFIG_RTE_ARM_FEATURE_ATOMICS=nThis patches series has following check patch warning. Please fix it
With above fixes you can add my acked-by in 1/3 and 3/3 patches in next revision.
Thank you for your review and your comments.
Quoted 7 lines
I think, you can ignore following warning. WARNING:MACRO_WITH_FLOW_CONTROL: Macros with flow control statements should be avoided
[master]dell[dpdk.org] $
./devtools/checkpatches.sh
Yes. I did this check before upstream this patch.
I ignored ' WARNING:MACRO_WITH_FLOW_CONTROL: Macros with flow control statements should be avoided ' warnings. However, I didn't see any LONG_LINE warnings in the CI report here. http://mails.dpdk.org/archives/test-report/2019-July/090889.html
BTW, I think these 3 warnings in the report might be ineluctable.
Quoted 4 lines
### eal/arm64: add 128-bit atomic compare exchange
WARNING:LONG_LINE: line over 80 characters
The default tab size is 8. You might need to change it to 4 in the /usr/src/linux/scripts/checkpatch tool. FYI.
scripts/checkpatch.pl +3 -3
@@ -1211,7 +1211,7 @@ sub expand_tabs { if ($c eq "\t") { $res .= ' '; $n++; - for (; ($n % 8) != 0; $n++) { + for (; ($n % 4) != 0; $n++) { $res .= ' '; } next;
@@ -3688,11 +3688,11 @@ sub process { #print "line<$line> prevline<$prevline> indent<$indent> sindent<$sindent> check<$check> continuation<$continuation> s<$s> cond_lines<$cond_lines> stat_real<$stat_real> stat<$stat>\n"; if ($check && $s ne '' && - (($sindent % 8) != 0 || + (($sindent % 4) != 0 || ($sindent < $indent) || ($sindent == $indent && ($s !~ /^\s*(?:\}|\{|else\b)/)) || - ($sindent > $indent + 8))) { + ($sindent > $indent + 4))) { WARN("SUSPECT_CODE_INDENT", "suspect code indent for conditional statements ($indent, $sindent)\n" . $herecurr . "$stat_real\n");
}
Quoted 6 lines
#103: FILE:
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:60:
+rte_int128_tupdated) \WARNING:LONG_LINE: line over 80 characters
#108: FILE:lib/librte_eal/common/include/arch/arm/rte_atomic_64.h:65:
+ */ \
Quoted 7 lines
Quoted 2 lines
[master]dell[dpdk.org] $
./devtools/checkpatches.shYes. I did this check before upstream this patch.
I ignored ' WARNING:MACRO_WITH_FLOW_CONTROL: Macros with flow control statements should be avoided ' warnings. However, I didn't see anyLONG_LINEwarnings in the CI report here.
Yes. But Not sure why?
I sent v8 with "WARNING:LONG_LINE: line over 80 characters" warning fixes with latest kernel checkpatch.pl. Please review.
Quoted 2 lines
http://mails.dpdk.org/archives/test-report/2019-July/090889.html
BTW, I think these 3 warnings in the report might be ineluctable.
Quoted 12 lines
-----Original Message-----
From: Jerin Jacob Kollanukkaran <[email protected]>
Sent: Tuesday, July 23, 2019 3:10 PM
To: Phil Yang (Arm Technology China) <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>; nd <[email protected]>
Subject: RE: [EXT] [PATCH v6 1/3] eal/arm64: add 128-bit atomic compare exchangeQuoted 1 line
Quoted 1 line
Quoted 8 lines
Quoted 6 lines
Quoted 1 line
[master]dell[dpdk.org] $
./devtools/checkpatches.shYes. I did this check before upstream this patch.
I ignored ' WARNING:MACRO_WITH_FLOW_CONTROL: Macros with flow control statements should be avoided ' warnings. However, I didn't see anyLONG_LINEwarnings in the CI report here.Yes. But Not sure why?
Not sure.
It might because DPDK CI changed tabstop width or indent width which is different than Kernel checkpatch.pl.
Quoted 4 lines
I sent v8 with "WARNING:LONG_LINE: line over 80 characters" warning fixes with latest kernel
checkpatch.pl. Please review.
+1
I really appreciate your help.
Quoted 3 lines
Quoted 3 lines
http://mails.dpdk.org/archives/test-report/2019-July/090889.html
BTW, I think these 3 warnings in the report might be ineluctable.
PATCH v7 1/3 eal/arm64: add 128-bit atomic compare exchange
In reply to Phil YangAdd 128-bit atomic compare exchange on aarch64.
---
v7:
1. Adjust code comment.
v6:
1. Put the RTE_ARM_FEATURE_ATOMICS flag into EAL group. (Jerin Jocob)
2. Keep rte_stack_lf_stubs.h doing nothing. (Gage Eads)
3. Fixed 32 bit build issue.
v5:
1. Enable RTE_ARM_FEATURE_ATOMICS on octeontx2 in default. (Jerin Jocob)
2. Record the reason of introducing "rte_stack_lf_stubs.h" in git commit. (Jerin, Jocob)
3. Fixed a conditional MACRO error in rte_atomic128_cmp_exchange. (Jerin Jocob)
v4:
1. Add RTE_ARM_FEATURE_ATOMICS flag to support LSE CASP instructions. (Jerin Jocob)
2. Fix possible arm64 ABI break by making casp_op_name noinline. (Jerin Jocob)
3. Add rte_stack_lf_stubs.h to reduce the ifdef clutter. (Gage Eads/Jerin Jocob)
v3:
1. Avoid duplication code with macro. (Jerin Jocob)
2. Make invalid memory order to strongest barrier. (Jerin Jocob)
3. Update doc/guides/prog_guide/env_abstraction_layer.rst. (Gage Eads)
4. Fix 32-bit x86 builds issue. (Gage Eads)
5. Correct documentation issues in UT. (Gage Eads)
v2:
Initial version.
config/arm/meson.build | 2 + config/common_base | 3 + config/defconfig_arm64-octeontx2-linuxapp-gcc | 1 + config/defconfig_arm64-thunderx2-linuxapp-gcc | 1 + .../common/include/arch/arm/rte_atomic_64.h | 162 +++++++++++++++++++++ .../common/include/arch/x86/rte_atomic_64.h | 12 -- lib/librte_eal/common/include/generic/rte_atomic.h | 17 ++- 7 files changed, 185 insertions(+), 13 deletions(-)
config/arm/meson.build +2 -0
@@ -71,11 +71,13 @@ flags_thunderx2_extra = [ ['RTE_CACHE_LINE_SIZE', 64], ['RTE_MAX_NUMA_NODES', 2], ['RTE_MAX_LCORE', 256], + ['RTE_ARM_FEATURE_ATOMICS', true], ['RTE_USE_C11_MEM_MODEL', true]] flags_octeontx2_extra = [ ['RTE_MACHINE', '"octeontx2"'], ['RTE_MAX_NUMA_NODES', 1], ['RTE_MAX_LCORE', 24], + ['RTE_ARM_FEATURE_ATOMICS', true], ['RTE_EAL_IGB_UIO', false], ['RTE_USE_C11_MEM_MODEL', true]]
config/common_base +3 -0
@@ -82,6 +82,9 @@ CONFIG_RTE_MAX_LCORE=128 CONFIG_RTE_MAX_NUMA_NODES=8 CONFIG_RTE_MAX_HEAPS=32 CONFIG_RTE_MAX_MEMSEG_LISTS=64 + +# Use ARM LSE ATOMIC instructions +CONFIG_RTE_ARM_FEATURE_ATOMICS=n # each memseg list will be limited to either RTE_MAX_MEMSEG_PER_LIST pages # or RTE_MAX_MEM_MB_PER_LIST megabytes worth of memory, whichever is smaller CONFIG_RTE_MAX_MEMSEG_PER_LIST=8192
config/defconfig_arm64-octeontx2-linuxapp-gcc +1 -0
@@ -9,6 +9,7 @@ CONFIG_RTE_MACHINE="octeontx2" CONFIG_RTE_CACHE_LINE_SIZE=128 CONFIG_RTE_MAX_NUMA_NODES=1 CONFIG_RTE_MAX_LCORE=24 +CONFIG_RTE_ARM_FEATURE_ATOMICS=y # Doesn't support NUMA CONFIG_RTE_EAL_NUMA_AWARE_HUGEPAGES=n
config/defconfig_arm64-thunderx2-linuxapp-gcc +1 -0
@@ -9,3 +9,4 @@ CONFIG_RTE_MACHINE="thunderx2" CONFIG_RTE_CACHE_LINE_SIZE=64 CONFIG_RTE_MAX_NUMA_NODES=2 CONFIG_RTE_MAX_LCORE=256 +CONFIG_RTE_ARM_FEATURE_ATOMICS=y
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +162 -0
179 lines not on this page · raw message
lib/librte_eal/common/include/arch/x86/rte_atomic_64.h +0 -12
@@ -212,18 +212,6 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)/*------------------------ 128 bit atomic operations -------------------------*/ -/** -* 128-bit integer structure. -*/ -RTE_STD_C11 -typedef struct { -RTE_STD_C11 - union { -uint64_t val[2 ]; - __extension____int128 int128; - }; -} __rte_aligned(16) rte_int128_t; - __rte_experimental static inline int rte_atomic128_cmp_exchange(rte_int128_t *dst,
lib/librte_eal/common/include/generic/rte_atomic.h +16 -1
@@ -1081,6 +1081,20 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)/*------------------------ 128 bit atomic operations -------------------------*/ +/** +* 128-bit integer structure. +*/ +RTE_STD_C11 +typedef struct { +RTE_STD_C11 + union { +uint64_t val[2 ]; +#ifdef RTE_ARCH_64 + __extension____int128 int128; +#endif + }; +} __rte_aligned(16) rte_int128_t; +#ifdef __DOXYGEN__/**
@@ -1093,7 +1107,8 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)* *exp = *dst * @endcode * -* @note This function is currently only available for the x86-64 platform. +* @note This function is currently available for the x86-64 and aarch64 +* platforms. * * @note The success and failure arguments must be one of the __ATOMIC_* values * defined in the C++11 standard. For details on their behavior, refer to the
Signature · 1 line
2.7.4
PATCH v7 2/3 test/atomic: add 128b compare and swap test
Add 128b atomic compare and swap test for aarch64 and x86_64.
--- app/test/test_atomic.c | 120 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 118 insertions(+), 2 deletions(-)
app/test/test_atomic.c +118 -2
@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2019 Arm Limited */ #include <stdio.h>
@@ -20,7 +21,7 @@* Atomic Variables * ================ * -* - The main test function performs three subtests. The first test +* - The main test function performs four subtests. The first test * checks that the usual inc/dec/add/sub functions are working * correctly: *
@@ -61,11 +62,27 @@* atomic_sub(&count, tmp+1); * * - At the end of the test, the *count* value must be 0. +* +* - Test "128b compare and swap" (aarch64 and x86_64 only) +* +* - Initialize 128-bit atomic variables to zero. +* +* - Invoke ``test_atomici128_cmp_exchange()`` on each lcore. Before doing +* anything else, the cores are waiting a synchro. Each lcore does +* these compare and swap (CAS) operations several times:: +* +* Acquired CAS update counter.val[0] + 2; counter.val[1] + 1; +* Released CAS update counter.val[0] + 2; counter.val[1] + 1; +* Acquired_Released CAS update counter.val[0] + 2; counter.val[1] + 1; +* Relaxed CAS update counter.val[0] + 2; counter.val[1] + 1; +* +* - At the end of the test, the *count128* first 64-bit value and +* second 64-bit value differ by the total iterations. */ #define NUM_ATOMIC_TYPES 3 -#define N 10000 +#define N 1000000static rte_atomic16_t a16;static rte_atomic32_t a32;
@@ -216,6 +233,74 @@ test_atomic_dec_and_test(__attribute__((unused)) void *arg)return 0 ; } +#if defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 ) +static rte_int128_t count128; + +/* +* rte_atomic128_cmp_exchange() should update a 128 bits counter's first 64 +* bits by 2 and the second 64 bits by 1 in this test. It should return true +* if the compare exchange operation is successful. +* This test repeats 128 bits compare and swap operations 10K rounds. In each +* iteration it runs compare and swap operation with different memory models. +*/ +static int +test_atomic128_cmp_exchange (__attribute__((unused))void *arg) +{ +rte_int128_t expected; +int success; +unsigned int i; + +while (rte_atomic32_read (&synchro) ==0 ) + ; + + expected = count128; + +for (i =0 ; i <N ; i++) { +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, &expected, + &desired,1 , __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); + }while (success ==0 ); + +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, &expected, + &desired,1 , __ATOMIC_RELEASE, __ATOMIC_RELAXED); + }while (success ==0 ); + +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, &expected, + &desired,1 , __ATOMIC_ACQ_REL, __ATOMIC_RELAXED); + }while (success ==0 ); + +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, &expected, + &desired,1 , __ATOMIC_RELAXED, __ATOMIC_RELAXED); + }while (success ==0 ); + } + +return 0 ; +} +#endif +static int test_atomic (void ) {
@@ -340,6 +425,37 @@ test_atomic(void)return -1 ; } +#if defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 ) +/* +* This case tests the functionality of rte_atomic128b_cmp_exchange +* API. It calls rte_atomic128b_cmp_exchange with four kinds of memory +* models successively on each slave core. Once each 128-bit atomic +* compare and swap operation is successful, it updates the global +* 128-bit counter by 2 for the first 64-bit and 1 for the second +* 64-bit. Each slave core iterates this test 10K times. +* At the end of test, verify whether the first 64-bits of the 128-bit +* counter and the second 64bits is differ by the total iterations. If +* it is, the test passes. +*/ +printf ("128b compare and swap test\n" ); +uint64_t iterations =0 ; + +rte_atomic32_clear (&synchro); + count128.val[0 ] =0 ; + count128.val[1 ] =0 ; + +rte_eal_mp_remote_launch (test_atomic128_cmp_exchange,NULL ,SKIP_MASTER ); +rte_atomic32_set (&synchro,1 ); +rte_eal_mp_wait_lcore (); +rte_atomic32_clear (&synchro); + + iterations = count128.val[0 ] - count128.val[1 ]; +if (iterations !=4 *N *(rte_lcore_count ()-1 )) { +printf ("128b compare and swap failed\n" ); +return -1 ; + } +#endif +return 0 ; }
Signature · 1 line
2.7.4
PATCH v7 3/3 eal/stack: enable lock-free stack for aarch64
In reply to Phil YangEnable both c11 atomic and non c11 atomic lock-free stack for aarch64.
Introduced a new header to reduce the ifdef clutter across generic and c11 files. The rte_stack_lf_stubs.h contains stub implementations of __rte_stack_lf_count, __rte_stack_lf_push_elems and __rte_stack_lf_pop_elems.
--- doc/guides/prog_guide/env_abstraction_layer.rst | 4 +-- doc/guides/rel_notes/release_19_08.rst | 3 ++ lib/librte_stack/Makefile | 3 +- lib/librte_stack/rte_stack_lf.h | 4 +++ lib/librte_stack/rte_stack_lf_c11.h | 16 --------- lib/librte_stack/rte_stack_lf_generic.h | 16 --------- lib/librte_stack/rte_stack_lf_stubs.h | 44 +++++++++++++++++++++++++ 7 files changed, 55 insertions(+), 35 deletions(-) create mode 100644 lib/librte_stack/rte_stack_lf_stubs.h
doc/guides/prog_guide/env_abstraction_layer.rst +2 -2
@@ -641,8 +641,8 @@ Known Issues Alternatively, applications can use the lock-free stack mempool handler. When considering this handler, note that: - - It is currently limited to the x86_64 platform, because it uses an - instruction (16-byte compare-and-swap) that is not yet available on other + - It is currently limited to the aarch64 and x86_64 platforms, because it uses + an instruction (16-byte compare-and-swap) that is not yet available on other platforms. - It has worse average-case performance than the non-preemptive rte_ring, but software caching (e.g. the mempool cache) can mitigate this by reducing the
doc/guides/rel_notes/release_19_08.rst +3 -0
@@ -212,6 +212,9 @@ New Features Added multiple cores feature to compression perf tool application. +* **Added Lock-free Stack for aarch64.** + + The lock-free stack implementation is enabled for aarch64 platforms. Removed Items -------------
lib/librte_stack/Makefile +2 -1
@@ -24,6 +24,7 @@ SYMLINK-$(CONFIG_RTE_LIBRTE_STACK)-include := rte_stack.h \ rte_stack_std.h \ rte_stack_lf.h \ rte_stack_lf_generic.h \ - rte_stack_lf_c11.h + rte_stack_lf_c11.h \ + rte_stack_lf_stubs.h include $(RTE_SDK)/mk/rte.lib.mk
lib/librte_stack/rte_stack_lf.h +4 -0
@@ -5,11 +5,15 @@#ifndef _RTE_STACK_LF_H_#define _RTE_STACK_LF_H_ +#if !(defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 )) +#include "rte_stack_lf_stubs.h" +#else #ifdef RTE_USE_C11_MEM_MODEL #include "rte_stack_lf_c11.h" #else #include "rte_stack_lf_generic.h" #endif +#endif /** * @internal Push several objects on the lock-free stack (MT-safe).
lib/librte_stack/rte_stack_lf_c11.h +0 -16
@@ -36,12 +36,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list, struct rte_stack_lf_elem *last, unsigned int num) { -#ifndef RTE_ARCH_X86_64 -RTE_SET_USED (first); -RTE_SET_USED (last); -RTE_SET_USED (list); -RTE_SET_USED (num); -#else struct rte_stack_lf_head old_head;int success;
@@ -79,7 +73,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,* to the LIFO len update. */ __atomic_add_fetch (&list->len, num, __ATOMIC_RELEASE); -#endif } static __rte_always_inline struct rte_stack_lf_elem *
@@ -88,14 +81,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,void **obj_table, struct rte_stack_lf_elem **last) { -#ifndef RTE_ARCH_X86_64 -RTE_SET_USED (obj_table ); -RTE_SET_USED (last); -RTE_SET_USED (list); -RTE_SET_USED (num); - -return NULL ; -#else struct rte_stack_lf_head old_head;uint64_t len;int success;
@@ -169,7 +154,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list, } while (success == 0);return old_head.top; -#endif }#endif /* _RTE_STACK_LF_C11_H_ */
lib/librte_stack/rte_stack_lf_generic.h +0 -16
@@ -36,12 +36,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list, struct rte_stack_lf_elem *last, unsigned int num) { -#ifndef RTE_ARCH_X86_64 -RTE_SET_USED (first); -RTE_SET_USED (last); -RTE_SET_USED (list); -RTE_SET_USED (num); -#else struct rte_stack_lf_head old_head;int success;
@@ -75,7 +69,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list, }while (success ==0 );rte_atomic64_add ((rte_atomic64_t *)&list->len, num); -#endif } static __rte_always_inline struct rte_stack_lf_elem *
@@ -84,14 +77,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,void **obj_table, struct rte_stack_lf_elem **last) { -#ifndef RTE_ARCH_X86_64 -RTE_SET_USED (obj_table ); -RTE_SET_USED (last); -RTE_SET_USED (list); -RTE_SET_USED (num); - -return NULL ; -#else struct rte_stack_lf_head old_head;int success;
@@ -159,7 +144,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list, } while (success == 0);return old_head.top; -#endif }#endif /* _RTE_STACK_LF_GENERIC_H_ */
lib/librte_stack/rte_stack_lf_stubs.h new file +44 -0
44 lines not on this page · raw message
Signature · 1 line
2.7.4
PATCH v8 1/3 eal/arm64: add 128-bit atomic compare exchange
In reply to Phil YangFrom: Phil Yang <[email protected]>
Add 128-bit atomic compare exchange on aarch64.
---
v8:
Fixed "WARNING:LONG_LINE: line over 80 characters" warnings with latest kernel checkpatch.pl
v7:
1. Adjust code comment.
v6:
1. Put the RTE_ARM_FEATURE_ATOMICS flag into EAL group. (Jerin Jocob)
2. Keep rte_stack_lf_stubs.h doing nothing. (Gage Eads)
3. Fixed 32 bit build issue.
v5:
1. Enable RTE_ARM_FEATURE_ATOMICS on octeontx2 in default. (Jerin Jocob)
2. Record the reason of introducing "rte_stack_lf_stubs.h" in git commit.
(Jerin, Jocob)
3. Fixed a conditional MACRO error in rte_atomic128_cmp_exchange. (Jerin Jocob)
v4:
1. Add RTE_ARM_FEATURE_ATOMICS flag to support LSE CASP instructions. (Jerin Jocob)
2. Fix possible arm64 ABI break by making casp_op_name noinline. (Jerin Jocob)
3. Add rte_stack_lf_stubs.h to reduce the ifdef clutter. (Gage Eads/Jerin Jocob)
v3:
1. Avoid duplication code with macro. (Jerin Jocob)
2. Make invalid memory order to strongest barrier. (Jerin Jocob)
3. Update doc/guides/prog_guide/env_abstraction_layer.rst. (Gage Eads)
4. Fix 32-bit x86 builds issue. (Gage Eads)
5. Correct documentation issues in UT. (Gage Eads)
v2:
Initial version.
config/arm/meson.build | 2 + config/common_base | 3 + config/defconfig_arm64-octeontx2-linuxapp-gcc | 1 + config/defconfig_arm64-thunderx2-linuxapp-gcc | 1 + .../common/include/arch/arm/rte_atomic_64.h | 163 ++++++++++++++++++ .../common/include/arch/x86/rte_atomic_64.h | 12 -- .../common/include/generic/rte_atomic.h | 17 +- 7 files changed, 186 insertions(+), 13 deletions(-)
config/arm/meson.build +2 -0
@@ -71,11 +71,13 @@ flags_thunderx2_extra = [ ['RTE_CACHE_LINE_SIZE', 64], ['RTE_MAX_NUMA_NODES', 2], ['RTE_MAX_LCORE', 256], + ['RTE_ARM_FEATURE_ATOMICS', true], ['RTE_USE_C11_MEM_MODEL', true]] flags_octeontx2_extra = [ ['RTE_MACHINE', '"octeontx2"'], ['RTE_MAX_NUMA_NODES', 1], ['RTE_MAX_LCORE', 24], + ['RTE_ARM_FEATURE_ATOMICS', true], ['RTE_EAL_IGB_UIO', false], ['RTE_USE_C11_MEM_MODEL', true]]
config/common_base +3 -0
@@ -82,6 +82,9 @@ CONFIG_RTE_MAX_LCORE=128 CONFIG_RTE_MAX_NUMA_NODES=8 CONFIG_RTE_MAX_HEAPS=32 CONFIG_RTE_MAX_MEMSEG_LISTS=64 + +# Use ARM LSE ATOMIC instructions +CONFIG_RTE_ARM_FEATURE_ATOMICS=n # each memseg list will be limited to either RTE_MAX_MEMSEG_PER_LIST pages # or RTE_MAX_MEM_MB_PER_LIST megabytes worth of memory, whichever is smaller CONFIG_RTE_MAX_MEMSEG_PER_LIST=8192
config/defconfig_arm64-octeontx2-linuxapp-gcc +1 -0
@@ -9,6 +9,7 @@ CONFIG_RTE_MACHINE="octeontx2" CONFIG_RTE_CACHE_LINE_SIZE=128 CONFIG_RTE_MAX_NUMA_NODES=1 CONFIG_RTE_MAX_LCORE=24 +CONFIG_RTE_ARM_FEATURE_ATOMICS=y # Doesn't support NUMA CONFIG_RTE_EAL_NUMA_AWARE_HUGEPAGES=n
config/defconfig_arm64-thunderx2-linuxapp-gcc +1 -0
@@ -9,3 +9,4 @@ CONFIG_RTE_MACHINE="thunderx2" CONFIG_RTE_CACHE_LINE_SIZE=64 CONFIG_RTE_MAX_NUMA_NODES=2 CONFIG_RTE_MAX_LCORE=256 +CONFIG_RTE_ARM_FEATURE_ATOMICS=y
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +163 -0
180 lines not on this page · raw message
lib/librte_eal/common/include/arch/x86/rte_atomic_64.h +0 -12
@@ -212,18 +212,6 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)/*------------------------ 128 bit atomic operations -------------------------*/ -/** -* 128-bit integer structure. -*/ -RTE_STD_C11 -typedef struct { -RTE_STD_C11 - union { -uint64_t val[2 ]; - __extension____int128 int128; - }; -} __rte_aligned(16) rte_int128_t; - __rte_experimental static inline int rte_atomic128_cmp_exchange(rte_int128_t *dst,
lib/librte_eal/common/include/generic/rte_atomic.h +16 -1
@@ -1081,6 +1081,20 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)/*------------------------ 128 bit atomic operations -------------------------*/ +/** +* 128-bit integer structure. +*/ +RTE_STD_C11 +typedef struct { +RTE_STD_C11 + union { +uint64_t val[2 ]; +#ifdef RTE_ARCH_64 + __extension____int128 int128; +#endif + }; +} __rte_aligned(16) rte_int128_t; +#ifdef __DOXYGEN__/**
@@ -1093,7 +1107,8 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)* *exp = *dst * @endcode * -* @note This function is currently only available for the x86-64 platform. +* @note This function is currently available for the x86-64 and aarch64 +* platforms. * * @note The success and failure arguments must be one of the __ATOMIC_* values * defined in the C++11 standard. For details on their behavior, refer to the
Signature · 1 line
2.22.0
PATCH v8 2/3 test/atomic: add 128b compare and swap test
From: Phil Yang <[email protected]>
Add 128b atomic compare and swap test for aarch64 and x86_64.
--- app/test/test_atomic.c | 125 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 123 insertions(+), 2 deletions(-)
app/test/test_atomic.c +123 -2
@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2019 Arm Limited */ #include <stdio.h>
@@ -20,7 +21,7 @@* Atomic Variables * ================ * -* - The main test function performs three subtests. The first test +* - The main test function performs four subtests. The first test * checks that the usual inc/dec/add/sub functions are working * correctly: *
@@ -61,11 +62,27 @@* atomic_sub(&count, tmp+1); * * - At the end of the test, the *count* value must be 0. +* +* - Test "128b compare and swap" (aarch64 and x86_64 only) +* +* - Initialize 128-bit atomic variables to zero. +* +* - Invoke ``test_atomici128_cmp_exchange()`` on each lcore. Before doing +* anything else, the cores are waiting a synchro. Each lcore does +* these compare and swap (CAS) operations several times:: +* +* Acquired CAS update counter.val[0] + 2; counter.val[1] + 1; +* Released CAS update counter.val[0] + 2; counter.val[1] + 1; +* Acquired_Released CAS update counter.val[0] + 2; counter.val[1] + 1; +* Relaxed CAS update counter.val[0] + 2; counter.val[1] + 1; +* +* - At the end of the test, the *count128* first 64-bit value and +* second 64-bit value differ by the total iterations. */ #define NUM_ATOMIC_TYPES 3 -#define N 10000 +#define N 1000000static rte_atomic16_t a16;static rte_atomic32_t a32;
@@ -216,6 +233,78 @@ test_atomic_dec_and_test(__attribute__((unused)) void *arg)return 0 ; } +#if defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 ) +static rte_int128_t count128; + +/* +* rte_atomic128_cmp_exchange() should update a 128 bits counter's first 64 +* bits by 2 and the second 64 bits by 1 in this test. It should return true +* if the compare exchange operation is successful. +* This test repeats 128 bits compare and swap operations 10K rounds. In each +* iteration it runs compare and swap operation with different memory models. +*/ +static int +test_atomic128_cmp_exchange (__attribute__((unused))void *arg) +{ +rte_int128_t expected; +int success; +unsigned int i; + +while (rte_atomic32_read (&synchro) ==0 ) + ; + + expected = count128; + +for (i =0 ; i <N ; i++) { +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, + &expected, &desired,1 , + __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); + }while (success ==0 ); + +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, + &expected, &desired,1 , + __ATOMIC_RELEASE, __ATOMIC_RELAXED); + }while (success ==0 ); + +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, + &expected, &desired,1 , + __ATOMIC_ACQ_REL, __ATOMIC_RELAXED); + }while (success ==0 ); + +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, + &expected, &desired,1 , + __ATOMIC_RELAXED, __ATOMIC_RELAXED); + }while (success ==0 ); + } + +return 0 ; +} +#endif +static int test_atomic (void ) {
@@ -340,6 +429,38 @@ test_atomic(void)return -1 ; } +#if defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 ) +/* +* This case tests the functionality of rte_atomic128b_cmp_exchange +* API. It calls rte_atomic128b_cmp_exchange with four kinds of memory +* models successively on each slave core. Once each 128-bit atomic +* compare and swap operation is successful, it updates the global +* 128-bit counter by 2 for the first 64-bit and 1 for the second +* 64-bit. Each slave core iterates this test 10K times. +* At the end of test, verify whether the first 64-bits of the 128-bit +* counter and the second 64bits is differ by the total iterations. If +* it is, the test passes. +*/ +printf ("128b compare and swap test\n" ); +uint64_t iterations =0 ; + +rte_atomic32_clear (&synchro); + count128.val[0 ] =0 ; + count128.val[1 ] =0 ; + +rte_eal_mp_remote_launch (test_atomic128_cmp_exchange,NULL , +SKIP_MASTER ); +rte_atomic32_set (&synchro,1 ); +rte_eal_mp_wait_lcore (); +rte_atomic32_clear (&synchro); + + iterations = count128.val[0 ] - count128.val[1 ]; +if (iterations !=4 *N *(rte_lcore_count ()-1 )) { +printf ("128b compare and swap failed\n" ); +return -1 ; + } +#endif +return 0 ; }
Signature · 1 line
2.22.0
PATCH v8 3/3 eal/stack: enable lock-free stack for aarch64
In reply to [email protected]From: Phil Yang <[email protected]>
Enable both c11 atomic and non c11 atomic lock-free stack for aarch64.
Introduced a new header to reduce the ifdef clutter across generic and c11 files. The rte_stack_lf_stubs.h contains stub implementations of __rte_stack_lf_count, __rte_stack_lf_push_elems and __rte_stack_lf_pop_elems.
--- .../prog_guide/env_abstraction_layer.rst | 4 +- doc/guides/rel_notes/release_19_08.rst | 3 ++ lib/librte_stack/Makefile | 3 +- lib/librte_stack/rte_stack_lf.h | 4 ++ lib/librte_stack/rte_stack_lf_c11.h | 16 ------- lib/librte_stack/rte_stack_lf_generic.h | 16 ------- lib/librte_stack/rte_stack_lf_stubs.h | 44 +++++++++++++++++++ 7 files changed, 55 insertions(+), 35 deletions(-) create mode 100644 lib/librte_stack/rte_stack_lf_stubs.h
doc/guides/prog_guide/env_abstraction_layer.rst +2 -2
@@ -641,8 +641,8 @@ Known Issues Alternatively, applications can use the lock-free stack mempool handler. When considering this handler, note that: - - It is currently limited to the x86_64 platform, because it uses an - instruction (16-byte compare-and-swap) that is not yet available on other + - It is currently limited to the aarch64 and x86_64 platforms, because it uses + an instruction (16-byte compare-and-swap) that is not yet available on other platforms. - It has worse average-case performance than the non-preemptive rte_ring, but software caching (e.g. the mempool cache) can mitigate this by reducing the
doc/guides/rel_notes/release_19_08.rst +3 -0
@@ -212,6 +212,9 @@ New Features Added multiple cores feature to compression perf tool application. +* **Added Lock-free Stack for aarch64.** + + The lock-free stack implementation is enabled for aarch64 platforms. Removed Items -------------
lib/librte_stack/Makefile +2 -1
@@ -24,6 +24,7 @@ SYMLINK-$(CONFIG_RTE_LIBRTE_STACK)-include := rte_stack.h \ rte_stack_std.h \ rte_stack_lf.h \ rte_stack_lf_generic.h \ - rte_stack_lf_c11.h + rte_stack_lf_c11.h \ + rte_stack_lf_stubs.h include $(RTE_SDK)/mk/rte.lib.mk
lib/librte_stack/rte_stack_lf.h +4 -0
@@ -5,11 +5,15 @@#ifndef _RTE_STACK_LF_H_#define _RTE_STACK_LF_H_ +#if !(defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 )) +#include "rte_stack_lf_stubs.h" +#else #ifdef RTE_USE_C11_MEM_MODEL #include "rte_stack_lf_c11.h" #else #include "rte_stack_lf_generic.h" #endif +#endif /** * @internal Push several objects on the lock-free stack (MT-safe).
lib/librte_stack/rte_stack_lf_c11.h +0 -16
@@ -36,12 +36,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list, struct rte_stack_lf_elem *last, unsigned int num) { -#ifndef RTE_ARCH_X86_64 -RTE_SET_USED (first); -RTE_SET_USED (last); -RTE_SET_USED (list); -RTE_SET_USED (num); -#else struct rte_stack_lf_head old_head;int success;
@@ -79,7 +73,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,* to the LIFO len update. */ __atomic_add_fetch (&list->len, num, __ATOMIC_RELEASE); -#endif } static __rte_always_inline struct rte_stack_lf_elem *
@@ -88,14 +81,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,void **obj_table, struct rte_stack_lf_elem **last) { -#ifndef RTE_ARCH_X86_64 -RTE_SET_USED (obj_table ); -RTE_SET_USED (last); -RTE_SET_USED (list); -RTE_SET_USED (num); - -return NULL ; -#else struct rte_stack_lf_head old_head;uint64_t len;int success;
@@ -169,7 +154,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list, } while (success == 0);return old_head.top; -#endif }#endif /* _RTE_STACK_LF_C11_H_ */
lib/librte_stack/rte_stack_lf_generic.h +0 -16
@@ -36,12 +36,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list, struct rte_stack_lf_elem *last, unsigned int num) { -#ifndef RTE_ARCH_X86_64 -RTE_SET_USED (first); -RTE_SET_USED (last); -RTE_SET_USED (list); -RTE_SET_USED (num); -#else struct rte_stack_lf_head old_head;int success;
@@ -75,7 +69,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list, }while (success ==0 );rte_atomic64_add ((rte_atomic64_t *)&list->len, num); -#endif } static __rte_always_inline struct rte_stack_lf_elem *
@@ -84,14 +77,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,void **obj_table, struct rte_stack_lf_elem **last) { -#ifndef RTE_ARCH_X86_64 -RTE_SET_USED (obj_table ); -RTE_SET_USED (last); -RTE_SET_USED (list); -RTE_SET_USED (num); - -return NULL ; -#else struct rte_stack_lf_head old_head;int success;
@@ -159,7 +144,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list, } while (success == 0);return old_head.top; -#endif }#endif /* _RTE_STACK_LF_GENERIC_H_ */
lib/librte_stack/rte_stack_lf_stubs.h new file +44 -0
44 lines not on this page · raw message
Signature · 1 line
2.22.0
PATCH v9 1/3 eal/arm64: add 128-bit atomic compare exchange
In reply to [email protected]Add 128-bit atomic compare exchange on aarch64.
---
v9:
Updated 19.11 release note.
v8:
Fixed "WARNING:LONG_LINE: line over 80 characters" warnings with latest kernel checkpatch.pl
v7:
1. Adjust code comment.
v6:
1. Put the RTE_ARM_FEATURE_ATOMICS flag into EAL group. (Jerin Jocob)
2. Keep rte_stack_lf_stubs.h doing nothing. (Gage Eads)
3. Fixed 32 bit build issue.
v5:
1. Enable RTE_ARM_FEATURE_ATOMICS on octeontx2 in default. (Jerin Jocob)
2. Record the reason of introducing "rte_stack_lf_stubs.h" in git commit.
(Jerin, Jocob)
3. Fixed a conditional MACRO error in rte_atomic128_cmp_exchange. (Jerin Jocob)
v4:
1. Add RTE_ARM_FEATURE_ATOMICS flag to support LSE CASP instructions. (Jerin Jocob)
2. Fix possible arm64 ABI break by making casp_op_name noinline. (Jerin Jocob)
3. Add rte_stack_lf_stubs.h to reduce the ifdef clutter. (Gage Eads/Jerin Jocob)
v3:
1. Avoid duplication code with macro. (Jerin Jocob)
2. Make invalid memory order to strongest barrier. (Jerin Jocob)
3. Update doc/guides/prog_guide/env_abstraction_layer.rst. (Gage Eads)
4. Fix 32-bit x86 builds issue. (Gage Eads)
5. Correct documentation issues in UT. (Gage Eads)
v2:
Initial version.
config/arm/meson.build | 2 + config/common_base | 3 + config/defconfig_arm64-octeontx2-linuxapp-gcc | 1 + config/defconfig_arm64-thunderx2-linuxapp-gcc | 1 + .../common/include/arch/arm/rte_atomic_64.h | 163 +++++++++++++++++++++ .../common/include/arch/x86/rte_atomic_64.h | 12 -- lib/librte_eal/common/include/generic/rte_atomic.h | 17 ++- 7 files changed, 186 insertions(+), 13 deletions(-)
config/arm/meson.build +2 -0
@@ -71,11 +71,13 @@ flags_thunderx2_extra = [ ['RTE_CACHE_LINE_SIZE', 64], ['RTE_MAX_NUMA_NODES', 2], ['RTE_MAX_LCORE', 256], + ['RTE_ARM_FEATURE_ATOMICS', true], ['RTE_USE_C11_MEM_MODEL', true]] flags_octeontx2_extra = [ ['RTE_MACHINE', '"octeontx2"'], ['RTE_MAX_NUMA_NODES', 1], ['RTE_MAX_LCORE', 24], + ['RTE_ARM_FEATURE_ATOMICS', true], ['RTE_EAL_IGB_UIO', false], ['RTE_USE_C11_MEM_MODEL', true]]
config/common_base +3 -0
@@ -82,6 +82,9 @@ CONFIG_RTE_MAX_LCORE=128 CONFIG_RTE_MAX_NUMA_NODES=8 CONFIG_RTE_MAX_HEAPS=32 CONFIG_RTE_MAX_MEMSEG_LISTS=64 + +# Use ARM LSE ATOMIC instructions +CONFIG_RTE_ARM_FEATURE_ATOMICS=n # each memseg list will be limited to either RTE_MAX_MEMSEG_PER_LIST pages # or RTE_MAX_MEM_MB_PER_LIST megabytes worth of memory, whichever is smaller CONFIG_RTE_MAX_MEMSEG_PER_LIST=8192
config/defconfig_arm64-octeontx2-linuxapp-gcc +1 -0
@@ -9,6 +9,7 @@ CONFIG_RTE_MACHINE="octeontx2" CONFIG_RTE_CACHE_LINE_SIZE=128 CONFIG_RTE_MAX_NUMA_NODES=1 CONFIG_RTE_MAX_LCORE=24 +CONFIG_RTE_ARM_FEATURE_ATOMICS=y # Doesn't support NUMA CONFIG_RTE_EAL_NUMA_AWARE_HUGEPAGES=n
config/defconfig_arm64-thunderx2-linuxapp-gcc +1 -0
@@ -9,3 +9,4 @@ CONFIG_RTE_MACHINE="thunderx2" CONFIG_RTE_CACHE_LINE_SIZE=64 CONFIG_RTE_MAX_NUMA_NODES=2 CONFIG_RTE_MAX_LCORE=256 +CONFIG_RTE_ARM_FEATURE_ATOMICS=y
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +163 -0
180 lines not on this page · raw message
lib/librte_eal/common/include/arch/x86/rte_atomic_64.h +0 -12
@@ -183,18 +183,6 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)/*------------------------ 128 bit atomic operations -------------------------*/ -/** -* 128-bit integer structure. -*/ -RTE_STD_C11 -typedef struct { -RTE_STD_C11 - union { -uint64_t val[2 ]; - __extension____int128 int128; - }; -} __rte_aligned(16) rte_int128_t; - __rte_experimental static inline int rte_atomic128_cmp_exchange(rte_int128_t *dst,
lib/librte_eal/common/include/generic/rte_atomic.h +16 -1
@@ -1081,6 +1081,20 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)/*------------------------ 128 bit atomic operations -------------------------*/ +/** +* 128-bit integer structure. +*/ +RTE_STD_C11 +typedef struct { +RTE_STD_C11 + union { +uint64_t val[2 ]; +#ifdef RTE_ARCH_64 + __extension____int128 int128; +#endif + }; +} __rte_aligned(16) rte_int128_t; +#ifdef __DOXYGEN__/**
@@ -1093,7 +1107,8 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)* *exp = *dst * @endcode * -* @note This function is currently only available for the x86-64 platform. +* @note This function is currently available for the x86-64 and aarch64 +* platforms. * * @note The success and failure arguments must be one of the __ATOMIC_* values * defined in the C++11 standard. For details on their behavior, refer to the
Signature · 1 line
2.7.4
PATCH v9 2/3 test/atomic: add 128b compare and swap test
Add 128b atomic compare and swap test for aarch64 and x86_64.
--- app/test/test_atomic.c | 125 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 123 insertions(+), 2 deletions(-)
app/test/test_atomic.c +123 -2
@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2019 Arm Limited */ #include <stdio.h>
@@ -20,7 +21,7 @@* Atomic Variables * ================ * -* - The main test function performs three subtests. The first test +* - The main test function performs four subtests. The first test * checks that the usual inc/dec/add/sub functions are working * correctly: *
@@ -61,11 +62,27 @@* atomic_sub(&count, tmp+1); * * - At the end of the test, the *count* value must be 0. +* +* - Test "128b compare and swap" (aarch64 and x86_64 only) +* +* - Initialize 128-bit atomic variables to zero. +* +* - Invoke ``test_atomici128_cmp_exchange()`` on each lcore. Before doing +* anything else, the cores are waiting a synchro. Each lcore does +* these compare and swap (CAS) operations several times:: +* +* Acquired CAS update counter.val[0] + 2; counter.val[1] + 1; +* Released CAS update counter.val[0] + 2; counter.val[1] + 1; +* Acquired_Released CAS update counter.val[0] + 2; counter.val[1] + 1; +* Relaxed CAS update counter.val[0] + 2; counter.val[1] + 1; +* +* - At the end of the test, the *count128* first 64-bit value and +* second 64-bit value differ by the total iterations. */ #define NUM_ATOMIC_TYPES 3 -#define N 10000 +#define N 1000000static rte_atomic16_t a16;static rte_atomic32_t a32;
@@ -216,6 +233,78 @@ test_atomic_dec_and_test(__attribute__((unused)) void *arg)return 0 ; } +#if defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 ) +static rte_int128_t count128; + +/* +* rte_atomic128_cmp_exchange() should update a 128 bits counter's first 64 +* bits by 2 and the second 64 bits by 1 in this test. It should return true +* if the compare exchange operation is successful. +* This test repeats 128 bits compare and swap operations 10K rounds. In each +* iteration it runs compare and swap operation with different memory models. +*/ +static int +test_atomic128_cmp_exchange (__attribute__((unused))void *arg) +{ +rte_int128_t expected; +int success; +unsigned int i; + +while (rte_atomic32_read (&synchro) ==0 ) + ; + + expected = count128; + +for (i =0 ; i <N ; i++) { +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, + &expected, &desired,1 , + __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); + }while (success ==0 ); + +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, + &expected, &desired,1 , + __ATOMIC_RELEASE, __ATOMIC_RELAXED); + }while (success ==0 ); + +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, + &expected, &desired,1 , + __ATOMIC_ACQ_REL, __ATOMIC_RELAXED); + }while (success ==0 ); + +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, + &expected, &desired,1 , + __ATOMIC_RELAXED, __ATOMIC_RELAXED); + }while (success ==0 ); + } + +return 0 ; +} +#endif +static int test_atomic (void ) {
@@ -340,6 +429,38 @@ test_atomic(void)return -1 ; } +#if defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 ) +/* +* This case tests the functionality of rte_atomic128b_cmp_exchange +* API. It calls rte_atomic128b_cmp_exchange with four kinds of memory +* models successively on each slave core. Once each 128-bit atomic +* compare and swap operation is successful, it updates the global +* 128-bit counter by 2 for the first 64-bit and 1 for the second +* 64-bit. Each slave core iterates this test 10K times. +* At the end of test, verify whether the first 64-bits of the 128-bit +* counter and the second 64bits is differ by the total iterations. If +* it is, the test passes. +*/ +printf ("128b compare and swap test\n" ); +uint64_t iterations =0 ; + +rte_atomic32_clear (&synchro); + count128.val[0 ] =0 ; + count128.val[1 ] =0 ; + +rte_eal_mp_remote_launch (test_atomic128_cmp_exchange,NULL , +SKIP_MASTER ); +rte_atomic32_set (&synchro,1 ); +rte_eal_mp_wait_lcore (); +rte_atomic32_clear (&synchro); + + iterations = count128.val[0 ] - count128.val[1 ]; +if (iterations !=4 *N *(rte_lcore_count ()-1 )) { +printf ("128b compare and swap failed\n" ); +return -1 ; + } +#endif +return 0 ; }
Signature · 1 line
2.7.4
Re: [dpdk-dev] [PATCH v9 2/3] test/atomic: add 128b compare and swap test
Quoted 33 lines · Phil Yang
Add 128b atomic compare and swap test for aarch64 and
x86_64.Signed-off-by Phil Yang <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>Acked-by Gage Eads <[email protected]>Acked-by Jerin Jacob <[email protected]>Tested-by Jerin Jacob <[email protected]>--- app/test/test_atomic.c | 125 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 123 insertions(+), 2 deletions(-)app/test/test_atomic.c +2
-1@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2019 Arm Limited */ #include <stdio.h> @@ -20,7 +21,7 @@* Atomic Variables * ================ * -* - The main test function performs three subtests. The first test +* - The main test function performs four subtests. The first test * checks that the usual inc/dec/add/sub functions are working * correctly: *
app/test/test_atomic.c line 70
Quoted 9 lines
app/test/test_atomic.c +6
-0@@ -61,11 +62,27 @@* atomic_sub(&count, tmp+1); * * - At the end of the test, the *count* value must be 0. +* +* - Test "128b compare and swap" (aarch64 and x86_64 only) +* +* - Initialize 128-bit atomic variables to zero. +* +* - Invoke ``test_atomici128_cmp_exchange()`` on each lcore. Before doing
Typo, atomic128.
Quoted 6 lines
app/test/test_atomic.c +6
-0+* anything else, the cores are waiting a synchro. Each lcore does +* these compare and swap (CAS) operations several times:: +* +* Acquired CAS update counter.val[0] + 2; counter.val[1] + 1; +* Released CAS update counter.val[0] + 2; counter.val[1] + 1; +* Acquired_Released CAS update counter.val[0] + 2; counter.val[1] + 1;
app/test/test_atomic.c line 85
Quoted 10 lines
app/test/test_atomic.c +5
-1+* Relaxed CAS update counter.val[0] + 2; counter.val[1] + 1; +* +* - At the end of the test, the *count128* first 64-bit value and +* second 64-bit value differ by the total iterations. */ #define NUM_ATOMIC_TYPES 3 -#define N 10000 +#define N 1000000
This change the number of iterations for this test.
Did you evaluate the impact on the test duration?
I suppose this is fairly quick, but could you explain why this has been extended?
The commitlog does not give hints.
Quoted 5 lines
app/test/test_atomic.c +0
-0static rte_atomic16_t a16;static rte_atomic32_t a32;@@ -216,6 +233,78 @@ test_atomic_dec_and_test(__attribute__((unused)) void *arg)return 0 ;
app/test/test_atomic.c test_atomic_dec_and_test(__attribute__((unused)) void *arg) · line 243
Quoted 10 lines
app/test/test_atomic.c +8
-0} +#if defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 ) +static rte_int128_t count128; + +/* +* rte_atomic128_cmp_exchange() should update a 128 bits counter's first 64 +* bits by 2 and the second 64 bits by 1 in this test. It should return true +* if the compare exchange operation is successful. +* This test repeats 128 bits compare and swap operations 10K rounds. In each
s/10K/N/
Quoted 69 lines
app/test/test_atomic.c +64
-0+* iteration it runs compare and swap operation with different memory models. +*/ +static int +test_atomic128_cmp_exchange (__attribute__((unused))void *arg) +{ +rte_int128_t expected; +int success; +unsigned int i; + +while (rte_atomic32_read (&synchro) ==0 ) + ; + + expected = count128; + +for (i =0 ; i <N ; i++) { +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, + &expected, &desired,1 , + __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); + }while (success ==0 ); + +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, + &expected, &desired,1 , + __ATOMIC_RELEASE, __ATOMIC_RELAXED); + }while (success ==0 ); + +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, + &expected, &desired,1 , + __ATOMIC_ACQ_REL, __ATOMIC_RELAXED); + }while (success ==0 ); + +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, + &expected, &desired,1 , + __ATOMIC_RELAXED, __ATOMIC_RELAXED); + }while (success ==0 ); + } + +return 0 ; +} +#endif +static int test_atomic (void ) {@@ -340,6 +429,38 @@ test_atomic(void)return -1 ;
app/test/test_atomic.c test_atomic(void) · line 439
Quoted 10 lines
app/test/test_atomic.c +8
-0} +#if defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 ) +/* +* This case tests the functionality of rte_atomic128b_cmp_exchange +* API. It calls rte_atomic128b_cmp_exchange with four kinds of memory +* models successively on each slave core. Once each 128-bit atomic +* compare and swap operation is successful, it updates the global +* 128-bit counter by 2 for the first 64-bit and 1 for the second +* 64-bit. Each slave core iterates this test 10K times.
N times.
Quoted 28 lines
app/test/test_atomic.c +24
-1+* At the end of test, verify whether the first 64-bits of the 128-bit +* counter and the second 64bits is differ by the total iterations. If +* it is, the test passes. +*/ +printf ("128b compare and swap test\n" ); +uint64_t iterations =0 ; + +rte_atomic32_clear (&synchro); + count128.val[0 ] =0 ; + count128.val[1 ] =0 ; + +rte_eal_mp_remote_launch (test_atomic128_cmp_exchange,NULL , +SKIP_MASTER ); +rte_atomic32_set (&synchro,1 ); +rte_eal_mp_wait_lcore (); +rte_atomic32_clear (&synchro); + + iterations = count128.val[0 ] - count128.val[1 ]; +if (iterations !=4 *N *(rte_lcore_count ()-1 )) { +printf ("128b compare and swap failed\n" ); +return -1 ; + } +#endif +return 0 ; } --
Quoted 2 lines
2.7.4
--
David Marchand
Re: [dpdk-dev] [PATCH v9 2/3] test/atomic: add 128b compare and swap test
Quoted 57 lines
-----Original Message-----
From: David Marchand <[email protected]>
Sent: Monday, October 14, 2019 11:45 PM
To: Phil Yang (Arm Technology China) <[email protected]>
Cc: [email protected]; [email protected]; Gage Eads <[email protected]>; dev <[email protected]>; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v9 2/3] test/atomic: add 128b compare and swap testQuoted 10 lines · Phil Yang
Add 128b atomic compare and swap test for aarch64 and
x86_64.Signed-off-by Phil Yang <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>Acked-by Gage Eads <[email protected]>Acked-by Jerin Jacob <[email protected]>Tested-by Jerin Jacob <[email protected]>--- app/test/test_atomic.c | 125++++++++++++++++++++++++++++++++++++++++++++++++-
Quoted 23 lines
1 file changed, 123 insertions(+), 2 deletions(-)app/test/test_atomic.c +2
-1@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2019 Arm Limited */ #include <stdio.h> @@ -20,7 +21,7 @@* Atomic Variables * ================ * -* - The main test function performs three subtests. The first test +* - The main test function performs four subtests. The first test * checks that the usual inc/dec/add/sub functions are working * correctly: * app/test/test_atomic.c line 70
Quoted 9 lines
app/test/test_atomic.c +6
-0@@ -61,11 +62,27 @@* atomic_sub(&count, tmp+1); * * - At the end of the test, the *count* value must be 0. +* +* - Test "128b compare and swap" (aarch64 and x86_64 only) +* +* - Initialize 128-bit atomic variables to zero. +* +* - Invoke ``test_atomici128_cmp_exchange()`` on each lcore. Before doing
Quoted 1 line
Typo, atomic128.
Updated in v10.
Quoted 16 lines
Quoted 16 lines
app/test/test_atomic.c +10
-0+* anything else, the cores are waiting a synchro. Each lcore does +* these compare and swap (CAS) operations several times:: +* +* Acquired CAS update counter.val[0] + 2; counter.val[1] + 1; +* Released CAS update counter.val[0] + 2; counter.val[1] + 1; +* Acquired_Released CAS update counter.val[0] + 2; counter.val[1] + 1; +* Relaxed CAS update counter.val[0] + 2; counter.val[1] + 1; +* +* - At the end of the test, the *count128* first 64-bit value and +* second 64-bit value differ by the total iterations. */ #define NUM_ATOMIC_TYPES 3
Quoted 7 lines
app/test/test_atomic.c line 85
Quoted 2 lines
app/test/test_atomic.c +1
-1-#define N 10000 +#define N 1000000This change the number of iterations for this test.
Did you evaluate the impact on the test duration?
I suppose this is fairly quick, but could you explain why this has been extended?
By extending the iterations to 1 million times it can test the stability of these atomic APIs, especial for the new added 128bit atomics.
Yes, I did the evaluation. It has no impact on the test duration as the test case is simple.
Quoted 1 line
The commitlog does not give hints.
Thanks for pointing out this. I have updated this in the v10 commitlog.
Quoted 16 lines
app/test/test_atomic.c
Quoted 6 lines
app/test/test_atomic.c +0
-0static rte_atomic16_t a16;static rte_atomic32_t a32;@@ -216,6 +233,78 @@
test_atomic_dec_and_test(__attribute__((unused)) void *arg)app/test/test_atomic.c test_atomic_dec_and_test(__attribute__((unused)) void *arg)
Quoted 8 lines
app/test/test_atomic.c +5
-0return 0 ; } +#if defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 ) +static rte_int128_t count128; + +/* +* rte_atomic128_cmp_exchange() should update a 128 bits counter's first 64
Quoted 6 lines
app/test/test_atomic.c test_atomic_dec_and_test(__attribute__((unused)) void *arg)
Quoted 3 lines
app/test/test_atomic.c +3
-0+* bits by 2 and the second 64 bits by 1 in this test. It should return true +* if the compare exchange operation is successful. +* This test repeats 128 bits compare and swap operations 10K rounds. In each
s/10K/N/
Updated in v10.
<snip>
Quoted 11 lines
app/test/test_atomic.c test_atomic(void)
Quoted 4 lines
app/test/test_atomic.c +4
-0+#if defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 ) +/* +* This case tests the functionality of rte_atomic128b_cmp_exchange +* API. It calls rte_atomic128b_cmp_exchange with four kinds of memory
app/test/test_atomic.c test_atomic(void) · line 439
Quoted 4 lines
app/test/test_atomic.c +4
-0+* models successively on each slave core. Once each 128-bit atomic +* compare and swap operation is successful, it updates the global +* 128-bit counter by 2 for the first 64-bit and 1 for the second +* 64-bit. Each slave core iterates this test 10K times. N times.
Updated in v10.
Thanks,
Phil
Quoted 36 lines
Quoted 30 lines
app/test/test_atomic.c +24
-1+* At the end of test, verify whether the first 64-bits of the 128-bit +* counter and the second 64bits is differ by the total iterations. If +* it is, the test passes. +*/ +printf ("128b compare and swap test\n" ); +uint64_t iterations =0 ; + +rte_atomic32_clear (&synchro); + count128.val[0 ] =0 ; + count128.val[1 ] =0 ; + +rte_eal_mp_remote_launch (test_atomic128_cmp_exchange,NULL , +SKIP_MASTER ); +rte_atomic32_set (&synchro,1 ); +rte_eal_mp_wait_lcore (); +rte_atomic32_clear (&synchro); + + iterations = count128.val[0 ] - count128.val[1 ]; +if (iterations !=4 *N *(rte_lcore_count ()-1 )) { +printf ("128b compare and swap failed\n" ); +return -1 ; + } +#endif +return 0 ; } --Quoted 2 lines
2.7.4
--
David Marchand
PATCH v9 3/3 eal/stack: enable lock-free stack for aarch64
In reply to Phil YangEnable both c11 atomic and non c11 atomic lock-free stack for aarch64.
Introduced a new header to reduce the ifdef clutter across generic and c11 files. The rte_stack_lf_stubs.h contains stub implementations of __rte_stack_lf_count, __rte_stack_lf_push_elems and __rte_stack_lf_pop_elems.
--- doc/guides/prog_guide/env_abstraction_layer.rst | 4 +-- doc/guides/rel_notes/release_19_11.rst | 3 ++ lib/librte_stack/Makefile | 3 +- lib/librte_stack/rte_stack_lf.h | 4 +++ lib/librte_stack/rte_stack_lf_c11.h | 16 --------- lib/librte_stack/rte_stack_lf_generic.h | 16 --------- lib/librte_stack/rte_stack_lf_stubs.h | 44 +++++++++++++++++++++++++ 7 files changed, 55 insertions(+), 35 deletions(-) create mode 100644 lib/librte_stack/rte_stack_lf_stubs.h
doc/guides/prog_guide/env_abstraction_layer.rst +2 -2
@@ -648,8 +648,8 @@ Known Issues Alternatively, applications can use the lock-free stack mempool handler. When considering this handler, note that: - - It is currently limited to the x86_64 platform, because it uses an - instruction (16-byte compare-and-swap) that is not yet available on other + - It is currently limited to the aarch64 and x86_64 platforms, because it uses + an instruction (16-byte compare-and-swap) that is not yet available on other platforms. - It has worse average-case performance than the non-preemptive rte_ring, but software caching (e.g. the mempool cache) can mitigate this by reducing the
doc/guides/rel_notes/release_19_11.rst +3 -0
@@ -56,6 +56,9 @@ New Features Also, make sure to start the actual text at the margin. ========================================================= +* **Added Lock-free Stack for aarch64.** + + The lock-free stack implementation is enabled for aarch64 platforms. Removed Items -------------
lib/librte_stack/Makefile +2 -1
@@ -24,6 +24,7 @@ SYMLINK-$(CONFIG_RTE_LIBRTE_STACK)-include := rte_stack.h \ rte_stack_std.h \ rte_stack_lf.h \ rte_stack_lf_generic.h \ - rte_stack_lf_c11.h + rte_stack_lf_c11.h \ + rte_stack_lf_stubs.h include $(RTE_SDK)/mk/rte.lib.mk
lib/librte_stack/rte_stack_lf.h +4 -0
@@ -5,11 +5,15 @@#ifndef _RTE_STACK_LF_H_#define _RTE_STACK_LF_H_ +#if !(defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 )) +#include "rte_stack_lf_stubs.h" +#else #ifdef RTE_USE_C11_MEM_MODEL #include "rte_stack_lf_c11.h" #else #include "rte_stack_lf_generic.h" #endif +#endif /** * @internal Push several objects on the lock-free stack (MT-safe).
lib/librte_stack/rte_stack_lf_c11.h +0 -16
@@ -36,12 +36,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list, struct rte_stack_lf_elem *last, unsigned int num) { -#ifndef RTE_ARCH_X86_64 -RTE_SET_USED (first); -RTE_SET_USED (last); -RTE_SET_USED (list); -RTE_SET_USED (num); -#else struct rte_stack_lf_head old_head;int success;
@@ -79,7 +73,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,* to the LIFO len update. */ __atomic_add_fetch (&list->len, num, __ATOMIC_RELEASE); -#endif } static __rte_always_inline struct rte_stack_lf_elem *
@@ -88,14 +81,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,void **obj_table, struct rte_stack_lf_elem **last) { -#ifndef RTE_ARCH_X86_64 -RTE_SET_USED (obj_table ); -RTE_SET_USED (last); -RTE_SET_USED (list); -RTE_SET_USED (num); - -return NULL ; -#else struct rte_stack_lf_head old_head;uint64_t len;int success;
@@ -169,7 +154,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list, } while (success == 0);return old_head.top; -#endif }#endif /* _RTE_STACK_LF_C11_H_ */
lib/librte_stack/rte_stack_lf_generic.h +0 -16
@@ -36,12 +36,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list, struct rte_stack_lf_elem *last, unsigned int num) { -#ifndef RTE_ARCH_X86_64 -RTE_SET_USED (first); -RTE_SET_USED (last); -RTE_SET_USED (list); -RTE_SET_USED (num); -#else struct rte_stack_lf_head old_head;int success;
@@ -75,7 +69,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list, }while (success ==0 );rte_atomic64_add ((rte_atomic64_t *)&list->len, num); -#endif } static __rte_always_inline struct rte_stack_lf_elem *
@@ -84,14 +77,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,void **obj_table, struct rte_stack_lf_elem **last) { -#ifndef RTE_ARCH_X86_64 -RTE_SET_USED (obj_table ); -RTE_SET_USED (last); -RTE_SET_USED (list); -RTE_SET_USED (num); - -return NULL ; -#else struct rte_stack_lf_head old_head;int success;
@@ -159,7 +144,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list, } while (success == 0);return old_head.top; -#endif }#endif /* _RTE_STACK_LF_GENERIC_H_ */
lib/librte_stack/rte_stack_lf_stubs.h new file +44 -0
44 lines not on this page · raw message
Signature · 1 line
2.7.4
Re: [dpdk-dev] [PATCH v9 3/3] eal/stack: enable lock-free stack for aarch64
Quoted 59 lines · Phil Yang
Enable both c11 atomic and non c11 atomic lock-free stack for aarch64.
Introduced a new header to reduce the ifdef clutter across generic and c11 files. The
rte_stack_lf_stubs.hcontains stub implementations of__rte_stack_lf_count,__rte_stack_lf_push_elemsand__rte_stack_lf_pop_elems.Suggested-by Gage Eads <[email protected]>Suggested-by Jerin Jacob <[email protected]>Signed-off-by Phil Yang <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>Tested-by Honnappa Nagarahalli <[email protected]>Acked-by Jerin Jacob <[email protected]>--- doc/guides/prog_guide/env_abstraction_layer.rst | 4 +-- doc/guides/rel_notes/release_19_11.rst | 3 ++ lib/librte_stack/Makefile | 3 +- lib/librte_stack/rte_stack_lf.h | 4 +++ lib/librte_stack/rte_stack_lf_c11.h | 16 --------- lib/librte_stack/rte_stack_lf_generic.h | 16 --------- lib/librte_stack/rte_stack_lf_stubs.h | 44 +++++++++++++++++++++++++ 7 files changed, 55 insertions(+), 35 deletions(-) create mode 100644 lib/librte_stack/rte_stack_lf_stubs.hdoc/guides/prog_guide/env_abstraction_layer.rst +2
-2@@ -648,8 +648,8 @@ Known Issues Alternatively, applications can use the lock-free stack mempool handler. When considering this handler, note that: - - It is currently limited to the x86_64 platform, because it uses an - instruction (16-byte compare-and-swap) that is not yet available on other + - It is currently limited to the aarch64 and x86_64 platforms, because it uses + an instruction (16-byte compare-and-swap) that is not yet available on other platforms. - It has worse average-case performance than the non-preemptive rte_ring, but software caching (e.g. the mempool cache) can mitigate this by reducing thedoc/guides/rel_notes/release_19_11.rst +3
-0@@ -56,6 +56,9 @@ New Features Also, make sure to start the actual text at the margin. ========================================================= +* **Added Lock-free Stack for aarch64.** + + The lock-free stack implementation is enabled for aarch64 platforms. Removed Items -------------lib/librte_stack/Makefile +0
-0
lib/librte_stack/Makefile SYMLINK-$(CONFIG_RTE_LIBRTE_STACK)-include := rte_stack.h \ · line 28
Quoted 6 lines
lib/librte_stack/Makefile +2
-1@@ -24,6 +24,7 @@ SYMLINK-$(CONFIG_RTE_LIBRTE_STACK)-include := rte_stack.h \ rte_stack_std.h \ rte_stack_lf.h \ rte_stack_lf_generic.h \ - rte_stack_lf_c11.h + rte_stack_lf_c11.h \ + rte_stack_lf_stubs.h
Please, use the same indentation type than the other lines.
Quoted 171 lines
lib/librte_stack/Makefile +0
-0include $(RTE_SDK)/mk/rte.lib.mklib/librte_stack/rte_stack_lf.h +4
-0@@ -5,11 +5,15 @@#ifndef _RTE_STACK_LF_H_#define _RTE_STACK_LF_H_ +#if !(defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 )) +#include "rte_stack_lf_stubs.h" +#else #ifdef RTE_USE_C11_MEM_MODEL #include "rte_stack_lf_c11.h" #else #include "rte_stack_lf_generic.h" #endif +#endif /** * @internal Push several objects on the lock-free stack (MT-safe). lib/librte_stack/rte_stack_lf_c11.h +0
-16@@ -36,12 +36,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list, struct rte_stack_lf_elem *last, unsigned int num) { -#ifndef RTE_ARCH_X86_64 -RTE_SET_USED (first); -RTE_SET_USED (last); -RTE_SET_USED (list); -RTE_SET_USED (num); -#else struct rte_stack_lf_head old_head;int success;@@ -79,7 +73,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,* to the LIFO len update. */ __atomic_add_fetch (&list->len, num, __ATOMIC_RELEASE); -#endif } static __rte_always_inline struct rte_stack_lf_elem *@@ -88,14 +81,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,void **obj_table, struct rte_stack_lf_elem **last) { -#ifndef RTE_ARCH_X86_64 -RTE_SET_USED (obj_table ); -RTE_SET_USED (last); -RTE_SET_USED (list); -RTE_SET_USED (num); - -return NULL ; -#else struct rte_stack_lf_head old_head;uint64_t len;int success;@@ -169,7 +154,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list, } while (success == 0);return old_head.top; -#endif }#endif /* _RTE_STACK_LF_C11_H_ */ lib/librte_stack/rte_stack_lf_generic.h +0
-16@@ -36,12 +36,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list, struct rte_stack_lf_elem *last, unsigned int num) { -#ifndef RTE_ARCH_X86_64 -RTE_SET_USED (first); -RTE_SET_USED (last); -RTE_SET_USED (list); -RTE_SET_USED (num); -#else struct rte_stack_lf_head old_head;int success;@@ -75,7 +69,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list, }while (success ==0 );rte_atomic64_add ((rte_atomic64_t *)&list->len, num); -#endif } static __rte_always_inline struct rte_stack_lf_elem *@@ -84,14 +77,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,void **obj_table, struct rte_stack_lf_elem **last) { -#ifndef RTE_ARCH_X86_64 -RTE_SET_USED (obj_table ); -RTE_SET_USED (last); -RTE_SET_USED (list); -RTE_SET_USED (num); - -return NULL ; -#else struct rte_stack_lf_head old_head;int success;@@ -159,7 +144,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list, } while (success == 0);return old_head.top; -#endif }#endif /* _RTE_STACK_LF_GENERIC_H_ */ lib/librte_stack/rte_stack_lf_stubs.h new file +44
-1@@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2019 Arm Limited +*/ + +#ifndef _RTE_STACK_LF_STUBS_H_ +#define _RTE_STACK_LF_STUBS_H_ + +#include <rte_common.h> + +static __rte_always_inline unsigned int +__rte_stack_lf_count (struct rte_stack *s) +{ +RTE_SET_USED (s); + +return 0 ; +} + +static __rte_always_inline void +__rte_stack_lf_push_elems (struct rte_stack_lf_list *list, +struct rte_stack_lf_elem *first, +struct rte_stack_lf_elem *last, +unsigned int num) +{ +RTE_SET_USED (first); +RTE_SET_USED (last); +RTE_SET_USED (list); +RTE_SET_USED (num); +} + +static __rte_always_inline struct rte_stack_lf_elem * +__rte_stack_lf_pop_elems (struct rte_stack_lf_list *list, +unsigned int num, +void **obj_table, +struct rte_stack_lf_elem **last) +{ +RTE_SET_USED (obj_table); +RTE_SET_USED (last); +RTE_SET_USED (list); +RTE_SET_USED (num); + +return NULL ; +} + +#endif /* _RTE_STACK_LF_STUBS_H_ */ --2.7.4
Re: [dpdk-dev] [PATCH v9 3/3] eal/stack: enable lock-free stack for aarch64
Quoted 27 lines
-----Original Message-----
From: David Marchand <[email protected]>
Sent: Monday, October 14, 2019 11:45 PM
To: Phil Yang (Arm Technology China) <[email protected]>
Cc: [email protected]; [email protected]; Gage Eads <[email protected]>; dev <[email protected]>; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v9 3/3] eal/stack: enable lock-free stack for aarch64Quoted 15 lines · Phil Yang
Enable both c11 atomic and non c11 atomic lock-free stack for aarch64.
Introduced a new header to reduce the ifdef clutter across generic and c11 files. The
rte_stack_lf_stubs.hcontains stub implementations of__rte_stack_lf_count,__rte_stack_lf_push_elemsand__rte_stack_lf_pop_elems.Suggested-by Gage Eads <[email protected]>Suggested-by Jerin Jacob <[email protected]>Signed-off-by Phil Yang <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>Tested-by Honnappa Nagarahalli <[email protected]>Acked-by Jerin Jacob <[email protected]>---
<snip>
Quoted 7 lines
lib/librte_stack/Makefile SYMLINK-$(CONFIG_RTE_LIBRTE_STACK)-include :=
Quoted 6 lines
-------------
lib/librte_stack/Makefile +0
-0@@ -24,6 +24,7 @@ SYMLINK-$(CONFIG_RTE_LIBRTE_STACK)-include :=
rte_stack.h\
Quoted 8 lines
lib/librte_stack/Makefile SYMLINK-$(CONFIG_RTE_LIBRTE_STACK)-include := rte_stack.h \ · line 28
Quoted 6 lines
lib/librte_stack/Makefile +2
-1rte_stack_std.h \ rte_stack_lf.h \ rte_stack_lf_generic.h \ - rte_stack_lf_c11.h + rte_stack_lf_c11.h \ + rte_stack_lf_stubs.hPlease, use the same indentation type than the other lines.
Updated in v10.
Thanks,
Phil
Quoted 2 lines · Phil Yang
Add 128-bit atomic compare exchange on aarch64.
A bit short, seeing the complexity of the code and the additional RTE_ARM_FEATURE_ATOMICS config flag.
Comments inline.
Quoted 134 lines
Suggested-by Jerin Jacob <[email protected]>Signed-off-by Phil Yang <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>Tested-by Honnappa Nagarahalli <[email protected]>Acked-by Jerin Jacob <[email protected]>---
v9:
Updated 19.11 release note.v8:
Fixed "WARNING:LONG_LINE: line over 80 characters" warnings with latest kernel checkpatch.plv7:
1. Adjust code comment.v6:
1. Put theRTE_ARM_FEATURE_ATOMICSflag into EAL group. (Jerin Jocob)
2. Keeprte_stack_lf_stubs.hdoing nothing. (Gage Eads)
3. Fixed 32 bit build issue.v5:
1. EnableRTE_ARM_FEATURE_ATOMICSon octeontx2 in default. (Jerin Jocob)
2. Record the reason of introducing "rte_stack_lf_stubs.h" in git commit.
(Jerin, Jocob)
3. Fixed a conditional MACRO error inrte_atomic128_cmp_exchange. (Jerin Jocob)v4:
1. AddRTE_ARM_FEATURE_ATOMICSflag to support LSE CASP instructions. (Jerin Jocob)
2. Fix possible arm64 ABI break by makingcasp_op_namenoinline. (Jerin Jocob)
3. Addrte_stack_lf_stubs.hto reduce the ifdef clutter. (Gage Eads/Jerin Jocob)v3:
1. Avoid duplication code with macro. (Jerin Jocob)
2. Make invalid memory order to strongest barrier. (Jerin Jocob)
3. Updatedoc/guides/prog_guide/env_abstraction_layer.rst. (Gage Eads)
4. Fix 32-bit x86 builds issue. (Gage Eads)
5. Correct documentation issues in UT. (Gage Eads)v2:
Initial version.config/arm/meson.build | 2 + config/common_base | 3 + config/defconfig_arm64-octeontx2-linuxapp-gcc | 1 + config/defconfig_arm64-thunderx2-linuxapp-gcc | 1 + .../common/include/arch/arm/rte_atomic_64.h | 163 +++++++++++++++++++++ .../common/include/arch/x86/rte_atomic_64.h | 12 -- lib/librte_eal/common/include/generic/rte_atomic.h | 17 ++- 7 files changed, 186 insertions(+), 13 deletions(-)config/arm/meson.build +2
-0@@ -71,11 +71,13 @@ flags_thunderx2_extra = [ ['RTE_CACHE_LINE_SIZE', 64], ['RTE_MAX_NUMA_NODES', 2], ['RTE_MAX_LCORE', 256], + ['RTE_ARM_FEATURE_ATOMICS', true], ['RTE_USE_C11_MEM_MODEL', true]] flags_octeontx2_extra = [ ['RTE_MACHINE', '"octeontx2"'], ['RTE_MAX_NUMA_NODES', 1], ['RTE_MAX_LCORE', 24], + ['RTE_ARM_FEATURE_ATOMICS', true], ['RTE_EAL_IGB_UIO', false], ['RTE_USE_C11_MEM_MODEL', true]]config/common_base +3
-0@@ -82,6 +82,9 @@ CONFIG_RTE_MAX_LCORE=128 CONFIG_RTE_MAX_NUMA_NODES=8 CONFIG_RTE_MAX_HEAPS=32 CONFIG_RTE_MAX_MEMSEG_LISTS=64 + +# Use ARM LSE ATOMIC instructions +CONFIG_RTE_ARM_FEATURE_ATOMICS=n # each memseg list will be limited to either RTE_MAX_MEMSEG_PER_LIST pages # or RTE_MAX_MEM_MB_PER_LIST megabytes worth of memory, whichever is smaller CONFIG_RTE_MAX_MEMSEG_PER_LIST=8192config/defconfig_arm64-octeontx2-linuxapp-gcc +1
-0@@ -9,6 +9,7 @@ CONFIG_RTE_MACHINE="octeontx2" CONFIG_RTE_CACHE_LINE_SIZE=128 CONFIG_RTE_MAX_NUMA_NODES=1 CONFIG_RTE_MAX_LCORE=24 +CONFIG_RTE_ARM_FEATURE_ATOMICS=y # Doesn't support NUMA CONFIG_RTE_EAL_NUMA_AWARE_HUGEPAGES=nconfig/defconfig_arm64-thunderx2-linuxapp-gcc +1
-0@@ -9,3 +9,4 @@ CONFIG_RTE_MACHINE="thunderx2" CONFIG_RTE_CACHE_LINE_SIZE=64 CONFIG_RTE_MAX_NUMA_NODES=2 CONFIG_RTE_MAX_LCORE=256 +CONFIG_RTE_ARM_FEATURE_ATOMICS=ylib/librte_eal/common/include/arch/arm/rte_atomic_64.h +4
-0@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2015 Cavium, Inc +* Copyright(c) 2019 Arm Limited */ #ifndef _RTE_ATOMIC_ARM64_H_@@ -14,6 +15,9 @@ extern "C" {#endif #include "generic/rte_atomic.h" +#include <rte_branch_prediction.h> +#include <rte_compat.h> +#include <rte_debug.h> #define dsb (opt) asm volatile("dsb " #opt : : : "memory")#define dmb (opt) asm volatile("dmb " #opt : : : "memory")@@ -40,6 +44,165 @@ extern "C" {
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 54
Quoted 10 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +8
-0#define rte_cio_rmb () dmb(oshld) +/*------------------------ 128 bit atomic operations -------------------------*/ + +#define __HAS_ACQ (mo) ((mo) != __ATOMIC_RELAXED && (mo) != __ATOMIC_RELEASE) +#define __HAS_RLS (mo) ((mo) == __ATOMIC_RELEASE || (mo) == __ATOMIC_ACQ_REL || \ + (mo) == __ATOMIC_SEQ_CST) + +#define __MO_LOAD (mo) (__HAS_ACQ((mo)) ? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED) +#define __MO_STORE (mo) (__HAS_RLS((mo)) ? __ATOMIC_RELEASE : __ATOMIC_RELAXED)
Those 4 first macros only make sense when LSE is not available (see below [1]). Besides, they are used only once, why not directly use those conditions where needed?
Quoted 22 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +22
-0+ +#if defined(__ARM_FEATURE_ATOMICS) || defined(RTE_ARM_FEATURE_ATOMICS ) +#define __ATOMIC128_CAS_OP (cas_op_name, op_string) \ +static __rte_noinline rte_int128_t \ +cas_op_name(rte_int128_t *dst, rte_int128_t old, \ + rte_int128_t updated) \ +{ \ +/* caspX instructions register pair must start from even-numbered +* register at operand 1. +* So, specify registers for local variables here. +*/ \ + register uint64_t x0 __asm("x0") = (uint64_t)old.val[0]; \ + register uint64_t x1 __asm("x1") = (uint64_t)old.val[1]; \ + register uint64_t x2 __asm("x2") = (uint64_t)updated.val[0]; \ + register uint64_t x3 __asm("x3") = (uint64_t)updated.val[1]; \ + asmvolatile ( \ + op_string" %[old0], %[old1], %[upd0], %[upd1], [%[dst]]" \ + : [old0]"+r" (x0), \ + [old1]"+r" (x1) \ + : [upd0]"r" (x2), \ + [upd1]"r" (x3), \ + [dst] "r" (dst) \
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 86
Quoted 10 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +10
-0+ : "memory"); \ + old.val[0 ] = x0; \ + old.val[1 ] = x1; \ +return old; \ +} + +__ATOMIC128_CAS_OP(__rte_cas_relaxed, "casp") +__ATOMIC128_CAS_OP(__rte_cas_acquire, "caspa") +__ATOMIC128_CAS_OP(__rte_cas_release, "caspl") +__ATOMIC128_CAS_OP (__rte_cas_acq_rel,"caspal" )
If LSE is available, we expose __rte_cas_XX (explicitely) *non* inlined functions, while without LSE, we expose inlined __rte_ldr_XX and __rte_stx_XX functions.
So we have a first disparity with non-inlined vs inlined functions depending on a #ifdef.
Then, we have a second disparity with two sets of "apis" depending on this #ifdef.
And we expose those sets with a rte_ prefix, meaning people will try to use them, but those are not part of a public api.
Can't we do without them ? (see below [2] for a proposal with ldr/stx, cas should be the same)
Quoted 29 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +29
-0+#else +#define __ATOMIC128_LDX_OP (ldx_op_name, op_string) \ +static inline rte_int128_t \ +ldx_op_name(const rte_int128_t *src) \ +{ \ + rte_int128_t ret; \ + asm volatile( \ + op_string " %0, %1, %2" \ + : "=&r" (ret.val[0]), \ + "=&r" (ret.val[1]) \ + : "Q" (src->val[0]) \ + : "memory"); \ + return ret; \ +} + +__ATOMIC128_LDX_OP(__rte_ldx_relaxed, "ldxp") +__ATOMIC128_LDX_OP(__rte_ldx_acquire, "ldaxp") + +#define __ATOMIC128_STX_OP (stx_op_name, op_string) \ +static inline uint32_t \ +stx_op_name(rte_int128_t *dst, const rte_int128_t src) \ +{ \ + uint32_t ret; \ + asm volatile( \ + op_string " %w0, %1, %2, %3" \ + : "=&r" (ret) \ + : "r" (src.val[0]), \ + "r" (src.val[1]), \ + "Q" (dst->val[0]) \
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 125
Quoted 10 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +10
-0+ : "memory"); \ +/* Return 0 on success, 1 on failure */ \ +return ret; \ +} + +__ATOMIC128_STX_OP(__rte_stx_relaxed, "stxp") +__ATOMIC128_STX_OP(__rte_stx_release, "stlxp") +#endif + +static inline int __rte_experimental
The __rte_experimental tag comes first.
Quoted 24 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +24
-0+rte_atomic128_cmp_exchange (rte_int128_t *dst, + rte_int128_t *exp, + const rte_int128_t *src, + unsigned int weak, + int success, + int failure) +{ +/* Always do strong CAS */ +RTE_SET_USED (weak); +/* Ignore memory ordering for failure, memory order for +* success must be stronger or equal +*/ +RTE_SET_USED (failure); +/* Find invalid memory order */ +RTE_ASSERT (success == __ATOMIC_RELAXED + || success == __ATOMIC_ACQUIRE + || success == __ATOMIC_RELEASE + || success == __ATOMIC_ACQ_REL + || success == __ATOMIC_SEQ_CST); + +#if defined(__ARM_FEATURE_ATOMICS) || defined(RTE_ARM_FEATURE_ATOMICS ) +rte_int128_t expected = *exp; +rte_int128_t desired = *src; +rte_int128_t old;
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 159
Quoted 10 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +10
-0+ +if (success == __ATOMIC_RELAXED) + old =__rte_cas_relaxed (dst, expected, desired); +else if (success == __ATOMIC_ACQUIRE) + old =__rte_cas_acquire (dst, expected, desired); +else if (success == __ATOMIC_RELEASE) + old =__rte_cas_release (dst, expected, desired); +else + old =__rte_cas_acq_rel (dst, expected, desired); +#else
1: the four first macros (on the memory ordering constraints) can be moved here then undef'd once unused.
Or you can just do without them.
Quoted 5 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5
-0+int ldx_mo =__MO_LOAD (success); +int stx_mo =__MO_STORE (success); +uint32_t ret =1 ; + registerrte_int128_t expected = *exp; + registerrte_int128_t desired = *src;
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 174
Quoted 10 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +10
-0+ registerrte_int128_t old; + +/* ldx128 can not guarantee atomic, +* Must write back src or old to verify atomicity of ldx128; +*/ + do { + if (ldx_mo == __ATOMIC_RELAXED) + old = __rte_ldx_relaxed(dst); + else + old = __rte_ldx_acquire(dst);
2: how about using a simple macro that gets passed the op string?
Something like (untested):
#define __READ_128(op_string, src, dst) \
asm volatile( \
op_string " %0, %1, %2" \
: "=&r" (dst.val[0]), \
"=&r" (dst.val[1]) \
: "Q" (src->val[0]) \
: "memory")Then used like this:
if (ldx_mo == __ATOMIC_RELAXED)
__READ_128("ldxp", dst, old);
else
__READ_128("ldaxp", dst, old);#undef __READ_128
Quoted 7 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +7
-0+ +if (likely (old.int128 == expected.int128)) { +if (stx_mo == __ATOMIC_RELAXED) + ret =__rte_stx_relaxed (dst, desired); +else + ret =__rte_stx_release (dst, desired); + }else {
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 191
Quoted 10 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +10
-0+/* In the failure case (since 'weak' is ignored and only +* weak == 0 is implemented), expected should contain +* the atomically read value of dst. This means, 'old' +* needs to be stored back to ensure it was read +* atomically. +*/ +if (stx_mo == __ATOMIC_RELAXED) + ret =__rte_stx_relaxed (dst, old); +else + ret =__rte_stx_release (dst, old);
And:
#define __STORE_128(op_string, dst, val, ret) \
asm volatile( \
op_string " %w0, %1, %2, %3" \
: "=&r" (ret) \
: "r" (val.val[0]), \
"r" (val.val[1]), \
"Q" (dst->val[0]) \
: "memory")Used like this:
if (likely(old.int128 == expected.int128)) {
if (stx_mo == __ATOMIC_RELAXED)
__STORE_128("stxp", dst, desired, ret);
else
__STORE_128("stlxp", dst, desired, ret);
} else {
/* In the failure case (since 'weak' is ignored and only
* weak == 0 is implemented), expected should contain
* the atomically read value of dst. This means, 'old'
* needs to be stored back to ensure it was read
* atomically.
*/
if (stx_mo == __ATOMIC_RELAXED)
__STORE_128("stxp", dst, old, ret);
else
__STORE_128("stlxp", dst, old, ret);
}#undef __STORE_128
Quoted 49 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +14
-0+ } + }while (unlikely (ret)); +#endif + +/* Unconditionally updating expected removes +* an 'if' statement. +* expected should already be in register if +* not in the cache. +*/ + *exp = old; + +return (old.int128 == expected.int128); +} + #ifdef __cplusplus }#endif lib/librte_eal/common/include/arch/x86/rte_atomic_64.h +0
-12@@ -183,18 +183,6 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)/*------------------------ 128 bit atomic operations -------------------------*/ -/** -* 128-bit integer structure. -*/ -RTE_STD_C11 -typedef struct { -RTE_STD_C11 - union { -uint64_t val[2 ]; - __extension____int128 int128; - }; -} __rte_aligned(16) rte_int128_t; - __rte_experimental static inline int rte_atomic128_cmp_exchange(rte_int128_t *dst,lib/librte_eal/common/include/generic/rte_atomic.h +1
-0@@ -1081,6 +1081,20 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)/*------------------------ 128 bit atomic operations -------------------------*/ +/**
lib/librte_eal/common/include/generic/rte_atomic.h static inline void rte_atomic64_clear(rte_atomic64_t *v) · line 1094
Quoted 10 lines
lib/librte_eal/common/include/generic/rte_atomic.h +10
-0+* 128-bit integer structure. +*/ +RTE_STD_C11 +typedef struct { +RTE_STD_C11 + union { +uint64_t val[2 ]; +#ifdef RTE_ARCH_64 + __extension____int128 int128; +#endif
You hid this field for x86.
What is the reason?
Quoted 17 lines
lib/librte_eal/common/include/generic/rte_atomic.h +5
-2+ }; +} __rte_aligned(16) rte_int128_t; +#ifdef __DOXYGEN__/** @@ -1093,7 +1107,8 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)* *exp = *dst * @endcode * -* @note This function is currently only available for the x86-64 platform. +* @note This function is currently available for the x86-64 and aarch64 +* platforms. * * @note The success and failure arguments must be one of the __ATOMIC_* values * defined in the C++11 standard. For details on their behavior, refer to the --
Quoted 2 lines
2.7.4
--
David Marchand
Hi David,
Thanks for your comments. I have addressed most of them in v10. Please review it. Some comments inline.
Quoted 11 lines
-----Original Message-----
From: David Marchand <[email protected]>
Sent: Monday, October 14, 2019 11:44 PM
To: Phil Yang (Arm Technology China) <[email protected]>
Cc: [email protected]; [email protected]; Gage Eads <[email protected]>; dev <[email protected]>; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v9 1/3] eal/arm64: add 128-bit atomic compare exchange
Quoted 6 lines
Quoted 2 lines · Phil Yang
Add 128-bit atomic compare exchange on aarch64.
A bit short, seeing the complexity of the code and the additional
RTE_ARM_FEATURE_ATOMICSconfig flag.
Updated in v10.
<snip>
Quoted 11 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" {
Quoted 4 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +3
-0+/*------------------------ 128 bit atomic operations -------------------------*/ + +#define __HAS_ACQ (mo) ((mo) != __ATOMIC_RELAXED && (mo) !=
__ATOMIC_RELEASE)Quoted 1 line
+#define
__HAS_RLS(mo)((mo) ==__ATOMIC_RELEASE|| (mo) ==
__ATOMIC_ACQ_REL|| \lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" {
Quoted 3 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +3
-0+ (mo) == __ATOMIC_SEQ_CST) + +#define __MO_LOAD (mo) (__HAS_ACQ((mo)) ? __ATOMIC_ACQUIRE :
__ATOMIC_RELAXED)
Quoted 7 lines
Quoted 1 line
+#define
__MO_STORE(mo)(__HAS_RLS((mo)) ?__ATOMIC_RELEASE:
__ATOMIC_RELAXED)Those 4 first macros only make sense when LSE is not available (see below [1]).
Besides, they are used only once, why not directly use those conditions where needed?
Agree. I removed __MO_LOAD and __MO_STORE in v10 and kept the __HAS_ACQ and __HAS_REL under the non-LSE condition branch in v10. I think they can make the code easy to read.
Quoted 44 lines
Review
Quoted 4 lines
(quoted diff) +2
-0+ +#if defined(__ARM_FEATURE_ATOMICS) ||
defined(RTE_ARM_FEATURE_ATOMICS)Quoted 20 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +20
-0+#define __ATOMIC128_CAS_OP (cas_op_name, op_string) \ +static __rte_noinline rte_int128_t \ +cas_op_name(rte_int128_t *dst, rte_int128_t old, \ + rte_int128_t updated) \ +{ \ +/* caspX instructions register pair must start from even-numbered +* register at operand 1. +* So, specify registers for local variables here. +*/ \ + register uint64_t x0 __asm("x0") = (uint64_t)old.val[0]; \ + register uint64_t x1 __asm("x1") = (uint64_t)old.val[1]; \ + register uint64_t x2 __asm("x2") = (uint64_t)updated.val[0]; \ + register uint64_t x3 __asm("x3") = (uint64_t)updated.val[1]; \ + asmvolatile ( \ + op_string" %[old0], %[old1], %[upd0], %[upd1], [%[dst]]" \ + : [old0]"+r" (x0), \ + [old1]"+r" (x1) \ + : [upd0]"r" (x2), \ + [upd1]"r" (x3), \ + [dst] "r" (dst) \lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 86
Quoted 10 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +10
-0+ : "memory"); \ + old.val[0 ] = x0; \ + old.val[1 ] = x1; \ +return old; \ +} + +__ATOMIC128_CAS_OP(__rte_cas_relaxed, "casp") +__ATOMIC128_CAS_OP(__rte_cas_acquire, "caspa") +__ATOMIC128_CAS_OP(__rte_cas_release, "caspl") +__ATOMIC128_CAS_OP (__rte_cas_acq_rel,"caspal" )If LSE is available, we expose
__rte_cas_XX(explicitely) *non* inlined functions, while without LSE, we expose inlined__rte_ldr_XXand__rte_stx_XXfunctions.
So we have a first disparity with non-inlined vs inlined functions depending on a #ifdef.
Then, we have a second disparity with two sets of "apis" depending on this #ifdef.
Quoted 5 lines
And we expose those sets with a
rte_prefix, meaning people will try to use them, but those are not part of a public api.Can't we do without them ? (see below [2] for a proposal with ldr/stx, cas should be the same)
No, it doesn't work.
Because we need to verify the return value at the end of the loop for these macros.
Quoted 35 lines
Quoted 35 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +33
-0+#else +#define __ATOMIC128_LDX_OP (ldx_op_name, op_string) \ +static inline rte_int128_t \ +ldx_op_name(const rte_int128_t *src) \ +{ \ + rte_int128_t ret; \ + asm volatile( \ + op_string " %0, %1, %2" \ + : "=&r" (ret.val[0]), \ + "=&r" (ret.val[1]) \ + : "Q" (src->val[0]) \ + : "memory"); \ + return ret; \ +} + +__ATOMIC128_LDX_OP(__rte_ldx_relaxed, "ldxp") +__ATOMIC128_LDX_OP(__rte_ldx_acquire, "ldaxp") + +#define __ATOMIC128_STX_OP (stx_op_name, op_string) \ +static inline uint32_t \ +stx_op_name(rte_int128_t *dst, const rte_int128_t src) \ +{ \ + uint32_t ret; \ + asm volatile( \ + op_string " %w0, %1, %2, %3" \ + : "=&r" (ret) \ + : "r" (src.val[0]), \ + "r" (src.val[1]), \ + "Q" (dst->val[0]) \ + : "memory"); \ +/* Return 0 on success, 1 on failure */ \ +return ret; \ +}
Quoted 8 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 125
Quoted 6 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +6
-0+ +__ATOMIC128_STX_OP(__rte_stx_relaxed, "stxp") +__ATOMIC128_STX_OP(__rte_stx_release, "stlxp") +#endif + +static inline int __rte_experimentalThe
__rte_experimentaltag comes first.
Updated in v10.
Quoted 33 lines
Quoted 13 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +11
-0+rte_atomic128_cmp_exchange (rte_int128_t *dst, + rte_int128_t *exp, + const rte_int128_t *src, + unsigned int weak, + int success, + int failure) +{ +/* Always do strong CAS */ +RTE_SET_USED (weak); +/* Ignore memory ordering for failure, memory order for +* success must be stronger or equal lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" {
Quoted 10 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +10
-0+*/ +RTE_SET_USED (failure); +/* Find invalid memory order */ +RTE_ASSERT (success == __ATOMIC_RELAXED + || success == __ATOMIC_ACQUIRE + || success == __ATOMIC_RELEASE + || success == __ATOMIC_ACQ_REL + || success == __ATOMIC_SEQ_CST); + +#if defined(__ARM_FEATURE_ATOMICS) ||
defined(RTE_ARM_FEATURE_ATOMICS)Quoted 9 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +9
-0+rte_int128_t expected = *exp; +rte_int128_t desired = *src; +rte_int128_t old; + +if (success == __ATOMIC_RELAXED) + old =__rte_cas_relaxed (dst, expected, desired); +else if (success == __ATOMIC_ACQUIRE) + old =__rte_cas_acquire (dst, expected, desired); +else if (success == __ATOMIC_RELEASE)
Quoted 8 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 159
Quoted 4 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +4
-0+ old =__rte_cas_release (dst, expected, desired); + else + old =__rte_cas_acq_rel (dst, expected, desired); +#else 1: the four first macros (on the memory ordering constraints) can be moved here then undef'd once unused.
Or you can just do without them.
Updated in v10.
Quoted 143 lines
Quoted 7 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +5
-0+int ldx_mo =__MO_LOAD (success); +int stx_mo =__MO_STORE (success); +uint32_t ret =1 ; + registerrte_int128_t expected = *exp; + registerrte_int128_t desired = *src;lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 174
Quoted 10 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +10
-0+ registerrte_int128_t old; + +/* ldx128 can not guarantee atomic, +* Must write back src or old to verify atomicity of ldx128; +*/ + do { + if (ldx_mo == __ATOMIC_RELAXED) + old = __rte_ldx_relaxed(dst); + else + old = __rte_ldx_acquire(dst);2: how about using a simple macro that gets passed the op string?
Something like (untested):
#define __READ_128(op_string, src, dst) \ asm volatile( \ op_string " %0, %1, %2" \ : "=&r" (dst.val[0]), \ "=&r" (dst.val[1]) \ : "Q" (src->val[0]) \ : "memory")Then used like this:
if (ldx_mo == __ATOMIC_RELAXED) __READ_128("ldxp", dst, old); else __READ_128("ldaxp", dst, old);#undef
__READ_128Quoted 7 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +7
-0+ +if (likely (old.int128 == expected.int128)) { +if (stx_mo == __ATOMIC_RELAXED) + ret =__rte_stx_relaxed (dst, desired); +else + ret =__rte_stx_release (dst, desired); + }else {lib/librte_eal/common/include/arch/arm/rte_atomic_64.h extern "C" { · line 191
Quoted 10 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +10
-0+/* In the failure case (since 'weak' is ignored and only +* weak == 0 is implemented), expected should contain +* the atomically read value of dst. This means, 'old' +* needs to be stored back to ensure it was read +* atomically. +*/ +if (stx_mo == __ATOMIC_RELAXED) + ret =__rte_stx_relaxed (dst, old); +else + ret =__rte_stx_release (dst, old);And:
#define __STORE_128(op_string, dst, val, ret) \ asm volatile( \ op_string " %w0, %1, %2, %3" \ : "=&r" (ret) \ : "r" (val.val[0]), \ "r" (val.val[1]), \ "Q" (dst->val[0]) \ : "memory")Used like this:
if (likely(old.int128 == expected.int128)) { if (stx_mo == __ATOMIC_RELAXED) __STORE_128("stxp", dst, desired, ret); else __STORE_128("stlxp", dst, desired, ret); } else { /* In the failure case (since 'weak' is ignored and only * weak == 0 is implemented), expected should contain * the atomically read value of dst. This means, 'old' * needs to be stored back to ensure it was read * atomically. */ if (stx_mo == __ATOMIC_RELAXED) __STORE_128("stxp", dst, old, ret); else __STORE_128("stlxp", dst, old, ret); }#undef
__STORE_128Review
Quoted 18 lines
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +14
-0+ } + }while (unlikely (ret)); +#endif + +/* Unconditionally updating expected removes +* an 'if' statement. +* expected should already be in register if +* not in the cache. +*/ + *exp = old; + +return (old.int128 == expected.int128); +} + #ifdef __cplusplus }#endif (quoted diff) +0
-0
b/lib/librte_eal/common/include/arch/x86/rte_atomic_64.hlib/librte_eal/common/include/arch/x86/rte_atomic_64.h static inline void
Quoted 4 lines
index 1335d92..cfe7067 100644
lib/librte_eal/common/include/arch/x86/rte_atomic_64.h +0
-0@@ -183,18 +183,6 @@ static inline void
rte_atomic64_clear(rte_atomic64_t *v)Review
Quoted 19 lines
lib/librte_eal/common/include/arch/x86/rte_atomic_64.h +0
-12/*------------------------ 128 bit atomic operations -------------------------*/ -/** -* 128-bit integer structure. -*/ -RTE_STD_C11 -typedef struct { -RTE_STD_C11 - union { -uint64_t val[2 ]; - __extension____int128 int128; - }; -} __rte_aligned(16) rte_int128_t; - __rte_experimental static inline int rte_atomic128_cmp_exchange(rte_int128_t *dst,(quoted diff) +0
-0
b/lib/librte_eal/common/include/generic/rte_atomic.hlib/librte_eal/common/include/generic/rte_atomic.h static inline void
Quoted 4 lines
index 24ff7dc..e6ab15a 100644
lib/librte_eal/common/include/generic/rte_atomic.h +0
-0@@ -1081,6 +1081,20 @@ static inline void
rte_atomic64_clear(rte_atomic64_t *v)Quoted 3 lines
lib/librte_eal/common/include/generic/rte_atomic.h +0
-0/*------------------------ 128 bit atomic operations -------------------------*/
Quoted 14 lines
lib/librte_eal/common/include/generic/rte_atomic.h static inline void rte_atomic64_clear(rte_atomic64_t *v) · line 1094
Quoted 11 lines
lib/librte_eal/common/include/generic/rte_atomic.h +11
-0+/** +* 128-bit integer structure. +*/ +RTE_STD_C11 +typedef struct { +RTE_STD_C11 + union { +uint64_t val[2 ]; +#ifdef RTE_ARCH_64 + __extension____int128 int128; +#endif You hid this field for x86.
What is the reason?
No, we are not hid it for x86. The RTE_ARCH_64 flag covered x86 as well.
Thanks,
Phil
Quoted 7 lines · Phil Yang (Arm Technology China)
Quoted 7 lines
-----Original Message-----
From: David Marchand <[email protected]>
If LSE is available, we expose__rte_cas_XX(explicitely) *non* inlined functions, while without LSE, we expose inlined__rte_ldr_XXand__rte_stx_XXfunctions.
So we have a first disparity with non-inlined vs inlined functions depending on a #ifdef.
You did not comment on the inline / no inline part and I still see this in the v10.
Is this __rte_noinline on the CAS function intentional?
Quoted 11 lines
Quoted 8 lines
Then, we have a second disparity with two sets of "apis" depending on this #ifdef.
And we expose those sets with a
rte_prefix, meaning people will try to use them, but those are not part of a public api.Can't we do without them ? (see below [2] for a proposal with ldr/stx, cas should be the same)
No, it doesn't work.
Because we need to verify the return value at the end of the loop for these macros.
Do you mean the return value for the stores?
Quoted 8 lines
Quoted 8 lines
#define __STORE_128(op_string, dst, val, ret) \ asm volatile( \ op_string " %w0, %1, %2, %3" \ : "=&r" (ret) \ : "r" (val.val[0]), \ "r" (val.val[1]), \ "Q" (dst->val[0]) \ : "memory")
The ret variable is still passed in this macro and the while loop can check it later.
Quoted 24 lines
Quoted 10 lines
Review
Quoted 1 line
(quoted diff) +0
-0
b/lib/librte_eal/common/include/generic/rte_atomic.hlib/librte_eal/common/include/generic/rte_atomic.h static inline void
Quoted 4 lines
index 24ff7dc..e6ab15a 100644
lib/librte_eal/common/include/generic/rte_atomic.h +0
-0@@ -1081,6 +1081,20 @@ static inline void
rte_atomic64_clear(rte_atomic64_t *v)Quoted 3 lines
lib/librte_eal/common/include/generic/rte_atomic.h +0
-0/*------------------------ 128 bit atomic operations -------------------------*/ Quoted 14 lines
lib/librte_eal/common/include/generic/rte_atomic.h static inline void rte_atomic64_clear(rte_atomic64_t *v) · line 1094
Quoted 11 lines
lib/librte_eal/common/include/generic/rte_atomic.h +11
-0+/** +* 128-bit integer structure. +*/ +RTE_STD_C11 +typedef struct { +RTE_STD_C11 + union { +uint64_t val[2 ]; +#ifdef RTE_ARCH_64 + __extension____int128 int128; +#endif You hid this field for x86.
What is the reason?
Quoted 1 line
No, we are not hid it for x86. The
RTE_ARCH_64flag covered x86 as well.
Ah indeed, I read it wrong, ARCH_64 ... AARCH64 ... :-)
--
David Marchand
Quoted 21 lines
-----Original Message-----
From: David Marchand <[email protected]>
Sent: Tuesday, October 15, 2019 8:16 PM
To: Phil Yang (Arm Technology China) <[email protected]>
Cc: [email protected]; [email protected]; Gage Eads <[email protected]>; dev <[email protected]>; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v9 1/3] eal/arm64: add 128-bit atomic compare exchangeQuoted 7 lines · Phil Yang (Arm Technology China)
Quoted 7 lines
-----Original Message-----
From: David Marchand <[email protected]>
If LSE is available, we expose__rte_cas_XX(explicitely) *non* inlined functions, while without LSE, we expose inlined__rte_ldr_XXand__rte_stx_XXfunctions.
So we have a first disparity with non-inlined vs inlined functions depending on a #ifdef.
Quoted 3 lines
You did not comment on the inline / no inline part and I still see this in the v10.
Is this__rte_noinlineon the CAS function intentional?
Apologize for missing this item. Yes, it is to avoid ABI break.
Please check5b40ec6b966260e0ff66a8a2c689664f75d6a0e6 ("mempool/octeontx2: fix possible arm64 ABI break")
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Then, we have a second disparity with two sets of "apis" depending on this #ifdef.
And we expose those sets with a
rte_prefix, meaning people will try to use them, but those are not part of a public api.
Quoted 8 lines
Quoted 5 lines
Quoted 2 lines
Can't we do without them ? (see below [2] for a proposal with ldr/stx, cas should be the same)
No, it doesn't work.
Because we need to verify the return value at the end of the loop for thesemacros.
Do you mean the return value for the stores?
It is my bad. I missed the ret option in the macro. This approach works.
However, I suggest to keep them as static inline functions rather than a piece of macro in the rte_atomic128_cmp_exchange API.
One reason is APIs name can indicate the memory ordering of these operations.
Moreover, it uses the register type to pass the value in the inline function, so it should not have too much cost comparing with the macro.
I also think these 128bit load and store functions can be used in other places, once it has been proved valuable in rte_atomic128_cmp_exchange API. But let's keep them private for the current stage. BTW, Linux kernel implemented in the same way. https://github.com/torvalds/linux/blob/master/arch/arm64/include/asm/atomic_lse.h#L19
Quoted 45 lines
Quoted 8 lines
Quoted 8 lines
#define __STORE_128(op_string, dst, val, ret) \ asm volatile( \ op_string " %w0, %1, %2, %3" \ : "=&r" (ret) \ : "r" (val.val[0]), \ "r" (val.val[1]), \ "Q" (dst->val[0]) \ : "memory")The ret variable is still passed in this macro and the while loop can check it later.
Quoted 24 lines
Quoted 10 lines
Review
Quoted 1 line
(quoted diff) +0
-0
b/lib/librte_eal/common/include/generic/rte_atomic.hlib/librte_eal/common/include/generic/rte_atomic.h static inline void
Quoted 4 lines
index 24ff7dc..e6ab15a 100644
lib/librte_eal/common/include/generic/rte_atomic.h +0
-0@@ -1081,6 +1081,20 @@ static inline void
rte_atomic64_clear(rte_atomic64_t *v)Quoted 3 lines
lib/librte_eal/common/include/generic/rte_atomic.h +0
-0/*------------------------ 128 bit atomic operations -------------------------*/ Quoted 14 lines
lib/librte_eal/common/include/generic/rte_atomic.h static inline void rte_atomic64_clear(rte_atomic64_t *v) · line 1094
Quoted 11 lines
lib/librte_eal/common/include/generic/rte_atomic.h +11
-0+/** +* 128-bit integer structure. +*/ +RTE_STD_C11 +typedef struct { +RTE_STD_C11 + union { +uint64_t val[2 ]; +#ifdef RTE_ARCH_64 + __extension____int128 int128; +#endif You hid this field for x86.
What is the reason?Quoted 1 line
No, we are not hid it for x86. The
RTE_ARCH_64flag covered x86 as well.Ah indeed, I read it wrong,
ARCH_64... AARCH64 ... :-)--
David Marchand
Quoted 22 lines · Phil Yang (Arm Technology China)
Quoted 22 lines
-----Original Message-----
From: David Marchand <[email protected]>
Sent: Tuesday, October 15, 2019 8:16 PM
To: Phil Yang (Arm Technology China) <[email protected]>
Cc: [email protected]; [email protected]; Gage Eads <[email protected]>; dev <[email protected]>; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v9 1/3] eal/arm64: add 128-bit atomic compare exchangeQuoted 7 lines · Phil Yang (Arm Technology China)
Quoted 7 lines
-----Original Message-----
From: David Marchand <[email protected]>
If LSE is available, we expose__rte_cas_XX(explicitely) *non* inlined functions, while without LSE, we expose inlined__rte_ldr_XXand__rte_stx_XXfunctions.
So we have a first disparity with non-inlined vs inlined functions depending on a #ifdef.
Quoted 7 lines
Quoted 3 lines
You did not comment on the inline / no inline part and I still see this in the v10.
Is this__rte_noinlineon the CAS function intentional?Apologize for missing this item. Yes, it is to avoid ABI break.
Please check5b40ec6b966260e0ff66a8a2c689664f75d6a0e6("mempool/octeontx2: fix possible arm64 ABI break")
Looked at the kernel parts on LSE CAS (thanks for the pointer) but I see inlines are used:
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/arch/arm64/include/asm/atomic_lse.h#n365?h=v5.4-rc3
What is special in the kernel or in dpdk that makes this different?
Quoted 12 lines
Quoted 12 lines
Quoted 12 lines
Quoted 11 lines
Then, we have a second disparity with two sets of "apis" depending on this #ifdef.
And we expose those sets with a
rte_prefix, meaning people will try to use them, but those are not part of a public api.Can't we do without them ? (see below [2] for a proposal with ldr/stx, cas should be the same)
Quoted 7 lines
Quoted 5 lines
Quoted 2 lines
No, it doesn't work.
Because we need to verify the return value at the end of the loop for thesemacros.
Do you mean the return value for the stores?
It is my bad. I missed the ret option in the macro. This approach works.
Ok, thanks for confirming.
Quoted 3 lines
However, I suggest to keep them as static inline functions rather than a piece of macro in the
rte_atomic128_cmp_exchangeAPI. One reason is APIs name can indicate the memory ordering of these operations.
API?
Those inlines are not part of a public API and we agree this patch is not about adding 128 bits load/store apis.
My proposal gives us small code that looks like:
if (ldx_mo == __ATOMIC_RELAXED)
__READ_128("ldxp", dst, old);
else
__READ_128("ldaxp", dst, old);I am not a memory order guru, but with this, I can figure the asm instruction depends on it.
And, since we are looking at internals of an implementation, this is mainly for people looking at/maintaining these low level details.
Quoted 1 line
Moreover, it uses the register type to pass the value in the inline function, so it should not have too much cost comparing with the macro.
This is not a problem of cost, this is about hiding architecture details from the final user.
If you expose something, you can expect someone will start using it and will complain later if you break it.
Quoted 1 line
I also think these 128bit load and store functions can be used in other places, once it has been proved valuable in
rte_atomic128_cmp_exchangeAPI. But let's keep them private for the current stage.
Yes I agree this could be introduced in the future.
Quoted 1 line
BTW, Linux kernel implemented in the same way. https://github.com/torvalds/linux/blob/master/arch/arm64/include/asm/atomic_lse.h#L19
Ok kernel exposes its internals, but I think kernel developpers are more vigilant than dpdk developpers on what is part of the public API and what is internal.
--
David Marchand
PATCH v10 1/3 eal/arm64: add 128-bit atomic compare exchange
In reply to Phil YangThis patch adds the implementation of the 128-bit atomic compare exchange API on AArch64. Using 64-bit 'ldxp/stxp' instructions can perform this operation. Moreover, on the LSE atomic extension accelerated platforms, it implemented by 'casp' instructions for better performance.
Since the '__ARM_FEATURE_ATOMICS' flag only supports GCC-9, so this patch adds a new config flag 'RTE_ARM_FEATURE_ATOMICS' to enable the 'cas' version on elder version compilers.
---
v10:
1.Removed all the rte tag for internal functions. 2.Removed __MO_LOAD and _MO_STORE macros and keep define __HAS_ACQ and __HAS_REL under non LSE conditional branch.
3.Undef the macro once it is unused.
4.Reword the 1/3 and 2/3 patches' commitlog more specific.
v9:
Updated 19.11 release note.
v8:
Fixed "WARNING:LONG_LINE: line over 80 characters" warnings with latest kernel checkpatch.pl
v7:
1. Adjust code comment.
v6:
1. Put the RTE_ARM_FEATURE_ATOMICS flag into EAL group. (Jerin Jocob)
2. Keep rte_stack_lf_stubs.h doing nothing. (Gage Eads)
3. Fixed 32 bit build issue.
v5:
1. Enable RTE_ARM_FEATURE_ATOMICS on octeontx2 in default. (Jerin Jocob)
2. Record the reason of introducing "rte_stack_lf_stubs.h" in git commit.
(Jerin, Jocob)
3. Fixed a conditional MACRO error in rte_atomic128_cmp_exchange. (Jerin Jocob)
v4:
1. Add RTE_ARM_FEATURE_ATOMICS flag to support LSE CASP instructions. (Jerin Jocob)
2. Fix possible arm64 ABI break by making casp_op_name noinline. (Jerin Jocob)
3. Add rte_stack_lf_stubs.h to reduce the ifdef clutter. (Gage Eads/Jerin Jocob)
v3:
1. Avoid duplication code with macro. (Jerin Jocob)
2. Make invalid memory order to strongest barrier. (Jerin Jocob)
3. Update doc/guides/prog_guide/env_abstraction_layer.rst. (Gage Eads)
4. Fix 32-bit x86 builds issue. (Gage Eads)
5. Correct documentation issues in UT. (Gage Eads)
v2:
Initial version.
config/arm/meson.build | 2 + config/common_base | 3 + config/defconfig_arm64-octeontx2-linuxapp-gcc | 1 + config/defconfig_arm64-thunderx2-linuxapp-gcc | 1 + .../common/include/arch/arm/rte_atomic_64.h | 173 +++++++++++++++++++++ .../common/include/arch/x86/rte_atomic_64.h | 12 -- lib/librte_eal/common/include/generic/rte_atomic.h | 17 +- 7 files changed, 196 insertions(+), 13 deletions(-)
config/arm/meson.build +2 -0
@@ -71,11 +71,13 @@ flags_thunderx2_extra = [ ['RTE_CACHE_LINE_SIZE', 64], ['RTE_MAX_NUMA_NODES', 2], ['RTE_MAX_LCORE', 256], + ['RTE_ARM_FEATURE_ATOMICS', true], ['RTE_USE_C11_MEM_MODEL', true]] flags_octeontx2_extra = [ ['RTE_MACHINE', '"octeontx2"'], ['RTE_MAX_NUMA_NODES', 1], ['RTE_MAX_LCORE', 24], + ['RTE_ARM_FEATURE_ATOMICS', true], ['RTE_EAL_IGB_UIO', false], ['RTE_USE_C11_MEM_MODEL', true]]
config/common_base +3 -0
@@ -82,6 +82,9 @@ CONFIG_RTE_MAX_LCORE=128 CONFIG_RTE_MAX_NUMA_NODES=8 CONFIG_RTE_MAX_HEAPS=32 CONFIG_RTE_MAX_MEMSEG_LISTS=64 + +# Use ARM LSE ATOMIC instructions +CONFIG_RTE_ARM_FEATURE_ATOMICS=n # each memseg list will be limited to either RTE_MAX_MEMSEG_PER_LIST pages # or RTE_MAX_MEM_MB_PER_LIST megabytes worth of memory, whichever is smaller CONFIG_RTE_MAX_MEMSEG_PER_LIST=8192
config/defconfig_arm64-octeontx2-linuxapp-gcc +1 -0
@@ -9,6 +9,7 @@ CONFIG_RTE_MACHINE="octeontx2" CONFIG_RTE_CACHE_LINE_SIZE=128 CONFIG_RTE_MAX_NUMA_NODES=1 CONFIG_RTE_MAX_LCORE=24 +CONFIG_RTE_ARM_FEATURE_ATOMICS=y # Doesn't support NUMA CONFIG_RTE_EAL_NUMA_AWARE_HUGEPAGES=n
config/defconfig_arm64-thunderx2-linuxapp-gcc +1 -0
@@ -9,3 +9,4 @@ CONFIG_RTE_MACHINE="thunderx2" CONFIG_RTE_CACHE_LINE_SIZE=64 CONFIG_RTE_MAX_NUMA_NODES=2 CONFIG_RTE_MAX_LCORE=256 +CONFIG_RTE_ARM_FEATURE_ATOMICS=y
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +173 -0
190 lines not on this page · raw message
lib/librte_eal/common/include/arch/x86/rte_atomic_64.h +0 -12
@@ -183,18 +183,6 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)/*------------------------ 128 bit atomic operations -------------------------*/ -/** -* 128-bit integer structure. -*/ -RTE_STD_C11 -typedef struct { -RTE_STD_C11 - union { -uint64_t val[2 ]; - __extension____int128 int128; - }; -} __rte_aligned(16) rte_int128_t; - __rte_experimental static inline int rte_atomic128_cmp_exchange(rte_int128_t *dst,
lib/librte_eal/common/include/generic/rte_atomic.h +16 -1
@@ -1081,6 +1081,20 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)/*------------------------ 128 bit atomic operations -------------------------*/ +/** +* 128-bit integer structure. +*/ +RTE_STD_C11 +typedef struct { +RTE_STD_C11 + union { +uint64_t val[2 ]; +#ifdef RTE_ARCH_64 + __extension____int128 int128; +#endif + }; +} __rte_aligned(16) rte_int128_t; +#ifdef __DOXYGEN__/**
@@ -1093,7 +1107,8 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)* *exp = *dst * @endcode * -* @note This function is currently only available for the x86-64 platform. +* @note This function is currently available for the x86-64 and aarch64 +* platforms. * * @note The success and failure arguments must be one of the __ATOMIC_* values * defined in the C++11 standard. For details on their behavior, refer to the
Signature · 1 line
2.7.4
PATCH v10 2/3 test/atomic: add 128b compare and swap test
Add 128b atomic compare and swap test for aarch64 and x86_64. Extend the test iteration from 10 thousand to 1 million times to test the stability of the atomic APIs.
--- app/test/test_atomic.c | 125 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 123 insertions(+), 2 deletions(-)
app/test/test_atomic.c +123 -2
@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2019 Arm Limited */ #include <stdio.h>
@@ -20,7 +21,7 @@* Atomic Variables * ================ * -* - The main test function performs three subtests. The first test +* - The main test function performs four subtests. The first test * checks that the usual inc/dec/add/sub functions are working * correctly: *
@@ -61,11 +62,27 @@* atomic_sub(&count, tmp+1); * * - At the end of the test, the *count* value must be 0. +* +* - Test "128b compare and swap" (aarch64 and x86_64 only) +* +* - Initialize 128-bit atomic variables to zero. +* +* - Invoke ``test_atomic128_cmp_exchange()`` on each lcore. Before doing +* anything else, the cores are waiting a synchro. Each lcore does +* these compare and swap (CAS) operations several times:: +* +* Acquired CAS update counter.val[0] + 2; counter.val[1] + 1; +* Released CAS update counter.val[0] + 2; counter.val[1] + 1; +* Acquired_Released CAS update counter.val[0] + 2; counter.val[1] + 1; +* Relaxed CAS update counter.val[0] + 2; counter.val[1] + 1; +* +* - At the end of the test, the *count128* first 64-bit value and +* second 64-bit value differ by the total iterations. */ #define NUM_ATOMIC_TYPES 3 -#define N 10000 +#define N 1000000static rte_atomic16_t a16;static rte_atomic32_t a32;
@@ -216,6 +233,78 @@ test_atomic_dec_and_test(__attribute__((unused)) void *arg)return 0 ; } +#if defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 ) +static rte_int128_t count128; + +/* +* rte_atomic128_cmp_exchange() should update a 128 bits counter's first 64 +* bits by 2 and the second 64 bits by 1 in this test. It should return true +* if the compare exchange operation is successful. +* This test repeats 128 bits compare and swap operations N rounds. In each +* iteration it runs compare and swap operation with different memory models. +*/ +static int +test_atomic128_cmp_exchange (__attribute__((unused))void *arg) +{ +rte_int128_t expected; +int success; +unsigned int i; + +while (rte_atomic32_read (&synchro) ==0 ) + ; + + expected = count128; + +for (i =0 ; i <N ; i++) { +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, + &expected, &desired,1 , + __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); + }while (success ==0 ); + +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, + &expected, &desired,1 , + __ATOMIC_RELEASE, __ATOMIC_RELAXED); + }while (success ==0 ); + +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, + &expected, &desired,1 , + __ATOMIC_ACQ_REL, __ATOMIC_RELAXED); + }while (success ==0 ); + +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, + &expected, &desired,1 , + __ATOMIC_RELAXED, __ATOMIC_RELAXED); + }while (success ==0 ); + } + +return 0 ; +} +#endif +static int test_atomic (void ) {
@@ -340,6 +429,38 @@ test_atomic(void)return -1 ; } +#if defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 ) +/* +* This case tests the functionality of rte_atomic128b_cmp_exchange +* API. It calls rte_atomic128b_cmp_exchange with four kinds of memory +* models successively on each slave core. Once each 128-bit atomic +* compare and swap operation is successful, it updates the global +* 128-bit counter by 2 for the first 64-bit and 1 for the second +* 64-bit. Each slave core iterates this test N times. +* At the end of test, verify whether the first 64-bits of the 128-bit +* counter and the second 64bits is differ by the total iterations. If +* it is, the test passes. +*/ +printf ("128b compare and swap test\n" ); +uint64_t iterations =0 ; + +rte_atomic32_clear (&synchro); + count128.val[0 ] =0 ; + count128.val[1 ] =0 ; + +rte_eal_mp_remote_launch (test_atomic128_cmp_exchange,NULL , +SKIP_MASTER ); +rte_atomic32_set (&synchro,1 ); +rte_eal_mp_wait_lcore (); +rte_atomic32_clear (&synchro); + + iterations = count128.val[0 ] - count128.val[1 ]; +if (iterations !=4 *N *(rte_lcore_count ()-1 )) { +printf ("128b compare and swap failed\n" ); +return -1 ; + } +#endif +return 0 ; }
Signature · 1 line
2.7.4
PATCH v10 3/3 eal/stack: enable lock-free stack for aarch64
In reply to Phil YangEnable both c11 atomic and non c11 atomic lock-free stack for aarch64.
Introduced a new header to reduce the ifdef clutter across generic and c11 files. The rte_stack_lf_stubs.h contains stub implementations of __rte_stack_lf_count, __rte_stack_lf_push_elems and __rte_stack_lf_pop_elems.
--- doc/guides/prog_guide/env_abstraction_layer.rst | 4 +-- doc/guides/rel_notes/release_19_11.rst | 3 ++ lib/librte_stack/Makefile | 3 +- lib/librte_stack/rte_stack_lf.h | 4 +++ lib/librte_stack/rte_stack_lf_c11.h | 16 --------- lib/librte_stack/rte_stack_lf_generic.h | 16 --------- lib/librte_stack/rte_stack_lf_stubs.h | 44 +++++++++++++++++++++++++ 7 files changed, 55 insertions(+), 35 deletions(-) create mode 100644 lib/librte_stack/rte_stack_lf_stubs.h
doc/guides/prog_guide/env_abstraction_layer.rst +2 -2
@@ -648,8 +648,8 @@ Known Issues Alternatively, applications can use the lock-free stack mempool handler. When considering this handler, note that: - - It is currently limited to the x86_64 platform, because it uses an - instruction (16-byte compare-and-swap) that is not yet available on other + - It is currently limited to the aarch64 and x86_64 platforms, because it uses + an instruction (16-byte compare-and-swap) that is not yet available on other platforms. - It has worse average-case performance than the non-preemptive rte_ring, but software caching (e.g. the mempool cache) can mitigate this by reducing the
doc/guides/rel_notes/release_19_11.rst +3 -0
@@ -115,6 +115,9 @@ New Features Added eBPF JIT support for arm64 architecture to improve the eBPF program performance. +* **Added Lock-free Stack for aarch64.** + + The lock-free stack implementation is enabled for aarch64 platforms. Removed Items -------------
lib/librte_stack/Makefile +2 -1
@@ -24,6 +24,7 @@ SYMLINK-$(CONFIG_RTE_LIBRTE_STACK)-include := rte_stack.h \ rte_stack_std.h \ rte_stack_lf.h \ rte_stack_lf_generic.h \ - rte_stack_lf_c11.h + rte_stack_lf_c11.h \ + rte_stack_lf_stubs.h include $(RTE_SDK)/mk/rte.lib.mk
lib/librte_stack/rte_stack_lf.h +4 -0
@@ -5,11 +5,15 @@#ifndef _RTE_STACK_LF_H_#define _RTE_STACK_LF_H_ +#if !(defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 )) +#include "rte_stack_lf_stubs.h" +#else #ifdef RTE_USE_C11_MEM_MODEL #include "rte_stack_lf_c11.h" #else #include "rte_stack_lf_generic.h" #endif +#endif /** * @internal Push several objects on the lock-free stack (MT-safe).
lib/librte_stack/rte_stack_lf_c11.h +0 -16
@@ -36,12 +36,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list, struct rte_stack_lf_elem *last, unsigned int num) { -#ifndef RTE_ARCH_X86_64 -RTE_SET_USED (first); -RTE_SET_USED (last); -RTE_SET_USED (list); -RTE_SET_USED (num); -#else struct rte_stack_lf_head old_head;int success;
@@ -79,7 +73,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,* to the LIFO len update. */ __atomic_add_fetch (&list->len, num, __ATOMIC_RELEASE); -#endif } static __rte_always_inline struct rte_stack_lf_elem *
@@ -88,14 +81,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,void **obj_table, struct rte_stack_lf_elem **last) { -#ifndef RTE_ARCH_X86_64 -RTE_SET_USED (obj_table ); -RTE_SET_USED (last); -RTE_SET_USED (list); -RTE_SET_USED (num); - -return NULL ; -#else struct rte_stack_lf_head old_head;uint64_t len;int success;
@@ -169,7 +154,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list, } while (success == 0);return old_head.top; -#endif }#endif /* _RTE_STACK_LF_C11_H_ */
lib/librte_stack/rte_stack_lf_generic.h +0 -16
@@ -36,12 +36,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list, struct rte_stack_lf_elem *last, unsigned int num) { -#ifndef RTE_ARCH_X86_64 -RTE_SET_USED (first); -RTE_SET_USED (last); -RTE_SET_USED (list); -RTE_SET_USED (num); -#else struct rte_stack_lf_head old_head;int success;
@@ -75,7 +69,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list, }while (success ==0 );rte_atomic64_add ((rte_atomic64_t *)&list->len, num); -#endif } static __rte_always_inline struct rte_stack_lf_elem *
@@ -84,14 +77,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,void **obj_table, struct rte_stack_lf_elem **last) { -#ifndef RTE_ARCH_X86_64 -RTE_SET_USED (obj_table ); -RTE_SET_USED (last); -RTE_SET_USED (list); -RTE_SET_USED (num); - -return NULL ; -#else struct rte_stack_lf_head old_head;int success;
@@ -159,7 +144,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list, } while (success == 0);return old_head.top; -#endif }#endif /* _RTE_STACK_LF_GENERIC_H_ */
lib/librte_stack/rte_stack_lf_stubs.h new file +44 -0
44 lines not on this page · raw message
Signature · 1 line
2.7.4
PATCH v11 1/3 eal/arm64: add 128-bit atomic compare exchange
In reply to Phil YangThis patch adds the implementation of the 128-bit atomic compare exchange API on AArch64. Using 64-bit 'ldxp/stxp' instructions can perform this operation. Moreover, on the LSE atomic extension accelerated platforms, it implemented by 'casp' instructions for better performance.
Since the '__ARM_FEATURE_ATOMICS' flag only supports GCC-9, so this patch adds a new config flag 'RTE_ARM_FEATURE_ATOMICS' to enable the 'cas' version on elder version compilers.
Since direct x0 register used in the code and cas_op_name() and rte_atomic128_cmp_exchange() is inline function, based on parent function load, it may corrupt x0 register aka Break arm64 ABI. Define CAS operations as rte_noinline functions to avoid the ABI break[1].
[1]5b40ec6b9662 ("mempool/octeontx2: fix possible arm64 ABI break").
---
v11:
1. Renamed cas_op_name by adding the data width tag 128.
2. Replaced __ldx/__stx inline functions with macros.
3. Illustrate the reason of define cas operations as non-inline functions in commitlog.
v10:
1.Removed all the rte tag for internal functions. 2.Removed __MO_LOAD and _MO_STORE macros and keep define __HAS_ACQ and __HAS_REL under non LSE conditional branch.
3.Undef the macro once it is unused.
4.Reword the 1/3 and 2/3 patches' commitlog more specific.
v9:
Updated 19.11 release note.
v8:
Fixed "WARNING:LONG_LINE: line over 80 characters" warnings with latest kernel checkpatch.pl
v7:
1. Adjust code comment.
v6:
1. Put the RTE_ARM_FEATURE_ATOMICS flag into EAL group. (Jerin Jocob)
2. Keep rte_stack_lf_stubs.h doing nothing. (Gage Eads)
3. Fixed 32 bit build issue.
v5:
1. Enable RTE_ARM_FEATURE_ATOMICS on octeontx2 in default. (Jerin Jocob)
2. Record the reason of introducing "rte_stack_lf_stubs.h" in git commit.
(Jerin, Jocob)
3. Fixed a conditional MACRO error in rte_atomic128_cmp_exchange. (Jerin Jocob)
v4:
1. Add RTE_ARM_FEATURE_ATOMICS flag to support LSE CASP instructions. (Jerin Jocob)
2. Fix possible arm64 ABI break by making casp_op_name noinline. (Jerin Jocob)
3. Add rte_stack_lf_stubs.h to reduce the ifdef clutter. (Gage Eads/Jerin Jocob)
v3:
1. Avoid duplication code with macro. (Jerin Jocob)
2. Make invalid memory order to strongest barrier. (Jerin Jocob)
3. Update doc/guides/prog_guide/env_abstraction_layer.rst. (Gage Eads)
4. Fix 32-bit x86 builds issue. (Gage Eads)
5. Correct documentation issues in UT. (Gage Eads)
v2:
Initial version.
config/arm/meson.build | 2 + config/common_base | 3 + config/defconfig_arm64-octeontx2-linuxapp-gcc | 1 + config/defconfig_arm64-thunderx2-linuxapp-gcc | 1 + .../common/include/arch/arm/rte_atomic_64.h | 151 +++++++++++++++++++++ .../common/include/arch/x86/rte_atomic_64.h | 12 -- lib/librte_eal/common/include/generic/rte_atomic.h | 17 ++- 7 files changed, 174 insertions(+), 13 deletions(-)
config/arm/meson.build +2 -0
@@ -71,11 +71,13 @@ flags_thunderx2_extra = [ ['RTE_CACHE_LINE_SIZE', 64], ['RTE_MAX_NUMA_NODES', 2], ['RTE_MAX_LCORE', 256], + ['RTE_ARM_FEATURE_ATOMICS', true], ['RTE_USE_C11_MEM_MODEL', true]] flags_octeontx2_extra = [ ['RTE_MACHINE', '"octeontx2"'], ['RTE_MAX_NUMA_NODES', 1], ['RTE_MAX_LCORE', 24], + ['RTE_ARM_FEATURE_ATOMICS', true], ['RTE_EAL_IGB_UIO', false], ['RTE_USE_C11_MEM_MODEL', true]]
config/common_base +3 -0
@@ -82,6 +82,9 @@ CONFIG_RTE_MAX_LCORE=128 CONFIG_RTE_MAX_NUMA_NODES=8 CONFIG_RTE_MAX_HEAPS=32 CONFIG_RTE_MAX_MEMSEG_LISTS=64 + +# Use ARM LSE ATOMIC instructions +CONFIG_RTE_ARM_FEATURE_ATOMICS=n # each memseg list will be limited to either RTE_MAX_MEMSEG_PER_LIST pages # or RTE_MAX_MEM_MB_PER_LIST megabytes worth of memory, whichever is smaller CONFIG_RTE_MAX_MEMSEG_PER_LIST=8192
config/defconfig_arm64-octeontx2-linuxapp-gcc +1 -0
@@ -9,6 +9,7 @@ CONFIG_RTE_MACHINE="octeontx2" CONFIG_RTE_CACHE_LINE_SIZE=128 CONFIG_RTE_MAX_NUMA_NODES=1 CONFIG_RTE_MAX_LCORE=24 +CONFIG_RTE_ARM_FEATURE_ATOMICS=y # Doesn't support NUMA CONFIG_RTE_EAL_NUMA_AWARE_HUGEPAGES=n
config/defconfig_arm64-thunderx2-linuxapp-gcc +1 -0
@@ -9,3 +9,4 @@ CONFIG_RTE_MACHINE="thunderx2" CONFIG_RTE_CACHE_LINE_SIZE=64 CONFIG_RTE_MAX_NUMA_NODES=2 CONFIG_RTE_MAX_LCORE=256 +CONFIG_RTE_ARM_FEATURE_ATOMICS=y
lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +151 -0
168 lines not on this page · raw message
lib/librte_eal/common/include/arch/x86/rte_atomic_64.h +0 -12
@@ -183,18 +183,6 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)/*------------------------ 128 bit atomic operations -------------------------*/ -/** -* 128-bit integer structure. -*/ -RTE_STD_C11 -typedef struct { -RTE_STD_C11 - union { -uint64_t val[2 ]; - __extension____int128 int128; - }; -} __rte_aligned(16) rte_int128_t; - __rte_experimental static inline int rte_atomic128_cmp_exchange(rte_int128_t *dst,
lib/librte_eal/common/include/generic/rte_atomic.h +16 -1
@@ -1081,6 +1081,20 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)/*------------------------ 128 bit atomic operations -------------------------*/ +/** +* 128-bit integer structure. +*/ +RTE_STD_C11 +typedef struct { +RTE_STD_C11 + union { +uint64_t val[2 ]; +#ifdef RTE_ARCH_64 + __extension____int128 int128; +#endif + }; +} __rte_aligned(16) rte_int128_t; +#ifdef __DOXYGEN__/**
@@ -1093,7 +1107,8 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)* *exp = *dst * @endcode * -* @note This function is currently only available for the x86-64 platform. +* @note This function is currently available for the x86-64 and aarch64 +* platforms. * * @note The success and failure arguments must be one of the __ATOMIC_* values * defined in the C++11 standard. For details on their behavior, refer to the
Signature · 1 line
2.7.4
PATCH v11 2/3 test/atomic: add 128b compare and swap test
Add 128b atomic compare and swap test for aarch64 and x86_64. Extend the test iteration from 10 thousand to 1 million times to test the stability of the atomic APIs.
--- app/test/test_atomic.c | 125 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 123 insertions(+), 2 deletions(-)
app/test/test_atomic.c +123 -2
@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2019 Arm Limited */ #include <stdio.h>
@@ -20,7 +21,7 @@* Atomic Variables * ================ * -* - The main test function performs three subtests. The first test +* - The main test function performs four subtests. The first test * checks that the usual inc/dec/add/sub functions are working * correctly: *
@@ -61,11 +62,27 @@* atomic_sub(&count, tmp+1); * * - At the end of the test, the *count* value must be 0. +* +* - Test "128b compare and swap" (aarch64 and x86_64 only) +* +* - Initialize 128-bit atomic variables to zero. +* +* - Invoke ``test_atomic128_cmp_exchange()`` on each lcore. Before doing +* anything else, the cores are waiting a synchro. Each lcore does +* these compare and swap (CAS) operations several times:: +* +* Acquired CAS update counter.val[0] + 2; counter.val[1] + 1; +* Released CAS update counter.val[0] + 2; counter.val[1] + 1; +* Acquired_Released CAS update counter.val[0] + 2; counter.val[1] + 1; +* Relaxed CAS update counter.val[0] + 2; counter.val[1] + 1; +* +* - At the end of the test, the *count128* first 64-bit value and +* second 64-bit value differ by the total iterations. */ #define NUM_ATOMIC_TYPES 3 -#define N 10000 +#define N 1000000static rte_atomic16_t a16;static rte_atomic32_t a32;
@@ -216,6 +233,78 @@ test_atomic_dec_and_test(__attribute__((unused)) void *arg)return 0 ; } +#if defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 ) +static rte_int128_t count128; + +/* +* rte_atomic128_cmp_exchange() should update a 128 bits counter's first 64 +* bits by 2 and the second 64 bits by 1 in this test. It should return true +* if the compare exchange operation is successful. +* This test repeats 128 bits compare and swap operations N rounds. In each +* iteration it runs compare and swap operation with different memory models. +*/ +static int +test_atomic128_cmp_exchange (__attribute__((unused))void *arg) +{ +rte_int128_t expected; +int success; +unsigned int i; + +while (rte_atomic32_read (&synchro) ==0 ) + ; + + expected = count128; + +for (i =0 ; i <N ; i++) { +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, + &expected, &desired,1 , + __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); + }while (success ==0 ); + +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, + &expected, &desired,1 , + __ATOMIC_RELEASE, __ATOMIC_RELAXED); + }while (success ==0 ); + +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, + &expected, &desired,1 , + __ATOMIC_ACQ_REL, __ATOMIC_RELAXED); + }while (success ==0 ); + +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, + &expected, &desired,1 , + __ATOMIC_RELAXED, __ATOMIC_RELAXED); + }while (success ==0 ); + } + +return 0 ; +} +#endif +static int test_atomic (void ) {
@@ -340,6 +429,38 @@ test_atomic(void)return -1 ; } +#if defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 ) +/* +* This case tests the functionality of rte_atomic128b_cmp_exchange +* API. It calls rte_atomic128b_cmp_exchange with four kinds of memory +* models successively on each slave core. Once each 128-bit atomic +* compare and swap operation is successful, it updates the global +* 128-bit counter by 2 for the first 64-bit and 1 for the second +* 64-bit. Each slave core iterates this test N times. +* At the end of test, verify whether the first 64-bits of the 128-bit +* counter and the second 64bits is differ by the total iterations. If +* it is, the test passes. +*/ +printf ("128b compare and swap test\n" ); +uint64_t iterations =0 ; + +rte_atomic32_clear (&synchro); + count128.val[0 ] =0 ; + count128.val[1 ] =0 ; + +rte_eal_mp_remote_launch (test_atomic128_cmp_exchange,NULL , +SKIP_MASTER ); +rte_atomic32_set (&synchro,1 ); +rte_eal_mp_wait_lcore (); +rte_atomic32_clear (&synchro); + + iterations = count128.val[0 ] - count128.val[1 ]; +if (iterations !=4 *N *(rte_lcore_count ()-1 )) { +printf ("128b compare and swap failed\n" ); +return -1 ; + } +#endif +return 0 ; }
Signature · 1 line
2.7.4
David Marchand
AppliedRe: [dpdk-dev] [PATCH v11 2/3] test/atomic: add 128b compare and swap test
Quoted 182 lines · Phil Yang
Add 128b atomic compare and swap test for aarch64 and
x86_64. Extend the test iteration from 10 thousand to 1 million times to test the stability of the atomic APIs.Signed-off-by Phil Yang <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>Acked-by Gage Eads <[email protected]>Acked-by Jerin Jacob <[email protected]>Tested-by Jerin Jacob <[email protected]>--- app/test/test_atomic.c | 125 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 123 insertions(+), 2 deletions(-)app/test/test_atomic.c +123
-3@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2019 Arm Limited */ #include <stdio.h> @@ -20,7 +21,7 @@* Atomic Variables * ================ * -* - The main test function performs three subtests. The first test +* - The main test function performs four subtests. The first test * checks that the usual inc/dec/add/sub functions are working * correctly: * @@ -61,11 +62,27 @@* atomic_sub(&count, tmp+1); * * - At the end of the test, the *count* value must be 0. +* +* - Test "128b compare and swap" (aarch64 and x86_64 only) +* +* - Initialize 128-bit atomic variables to zero. +* +* - Invoke ``test_atomic128_cmp_exchange()`` on each lcore. Before doing +* anything else, the cores are waiting a synchro. Each lcore does +* these compare and swap (CAS) operations several times:: +* +* Acquired CAS update counter.val[0] + 2; counter.val[1] + 1; +* Released CAS update counter.val[0] + 2; counter.val[1] + 1; +* Acquired_Released CAS update counter.val[0] + 2; counter.val[1] + 1; +* Relaxed CAS update counter.val[0] + 2; counter.val[1] + 1; +* +* - At the end of the test, the *count128* first 64-bit value and +* second 64-bit value differ by the total iterations. */ #define NUM_ATOMIC_TYPES 3 -#define N 10000 +#define N 1000000static rte_atomic16_t a16;static rte_atomic32_t a32;@@ -216,6 +233,78 @@ test_atomic_dec_and_test(__attribute__((unused)) void *arg)return 0 ; } +#if defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 ) +static rte_int128_t count128; + +/* +* rte_atomic128_cmp_exchange() should update a 128 bits counter's first 64 +* bits by 2 and the second 64 bits by 1 in this test. It should return true +* if the compare exchange operation is successful. +* This test repeats 128 bits compare and swap operations N rounds. In each +* iteration it runs compare and swap operation with different memory models. +*/ +static int +test_atomic128_cmp_exchange (__attribute__((unused))void *arg) +{ +rte_int128_t expected; +int success; +unsigned int i; + +while (rte_atomic32_read (&synchro) ==0 ) + ; + + expected = count128; + +for (i =0 ; i <N ; i++) { +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, + &expected, &desired,1 , + __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); + }while (success ==0 ); + +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, + &expected, &desired,1 , + __ATOMIC_RELEASE, __ATOMIC_RELAXED); + }while (success ==0 ); + +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, + &expected, &desired,1 , + __ATOMIC_ACQ_REL, __ATOMIC_RELAXED); + }while (success ==0 ); + +do { +rte_int128_t desired; + + desired.val[0 ] = expected.val[0 ] +2 ; + desired.val[1 ] = expected.val[1 ] +1 ; + + success =rte_atomic128_cmp_exchange (&count128, + &expected, &desired,1 , + __ATOMIC_RELAXED, __ATOMIC_RELAXED); + }while (success ==0 ); + } + +return 0 ; +} +#endif +static int test_atomic (void ) {@@ -340,6 +429,38 @@ test_atomic(void)return -1 ; } +#if defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 ) +/* +* This case tests the functionality of rte_atomic128b_cmp_exchange +* API. It calls rte_atomic128b_cmp_exchange with four kinds of memory +* models successively on each slave core. Once each 128-bit atomic +* compare and swap operation is successful, it updates the global +* 128-bit counter by 2 for the first 64-bit and 1 for the second +* 64-bit. Each slave core iterates this test N times. +* At the end of test, verify whether the first 64-bits of the 128-bit +* counter and the second 64bits is differ by the total iterations. If +* it is, the test passes. +*/ +printf ("128b compare and swap test\n" ); +uint64_t iterations =0 ; + +rte_atomic32_clear (&synchro); + count128.val[0 ] =0 ; + count128.val[1 ] =0 ; + +rte_eal_mp_remote_launch (test_atomic128_cmp_exchange,NULL , +SKIP_MASTER ); +rte_atomic32_set (&synchro,1 ); +rte_eal_mp_wait_lcore (); +rte_atomic32_clear (&synchro); + + iterations = count128.val[0 ] - count128.val[1 ]; +if (iterations !=4 *N *(rte_lcore_count ()-1 )) { +printf ("128b compare and swap failed\n" ); +return -1 ; + } +#endif +return 0 ; } --
Quoted 2 lines
2.7.4
Applied, thanks.
Signature · 1 line
David Marchand
PATCH v11 3/3 eal/stack: enable lock-free stack for aarch64
In reply to Phil YangEnable both c11 atomic and non c11 atomic lock-free stack for aarch64.
Introduced a new header to reduce the ifdef clutter across generic and c11 files. The rte_stack_lf_stubs.h contains stub implementations of __rte_stack_lf_count, __rte_stack_lf_push_elems and __rte_stack_lf_pop_elems.
--- doc/guides/prog_guide/env_abstraction_layer.rst | 4 +-- doc/guides/rel_notes/release_19_11.rst | 3 ++ lib/librte_stack/Makefile | 3 +- lib/librte_stack/rte_stack_lf.h | 4 +++ lib/librte_stack/rte_stack_lf_c11.h | 16 --------- lib/librte_stack/rte_stack_lf_generic.h | 16 --------- lib/librte_stack/rte_stack_lf_stubs.h | 44 +++++++++++++++++++++++++ 7 files changed, 55 insertions(+), 35 deletions(-) create mode 100644 lib/librte_stack/rte_stack_lf_stubs.h
doc/guides/prog_guide/env_abstraction_layer.rst +2 -2
@@ -648,8 +648,8 @@ Known Issues Alternatively, applications can use the lock-free stack mempool handler. When considering this handler, note that: - - It is currently limited to the x86_64 platform, because it uses an - instruction (16-byte compare-and-swap) that is not yet available on other + - It is currently limited to the aarch64 and x86_64 platforms, because it uses + an instruction (16-byte compare-and-swap) that is not yet available on other platforms. - It has worse average-case performance than the non-preemptive rte_ring, but software caching (e.g. the mempool cache) can mitigate this by reducing the
doc/guides/rel_notes/release_19_11.rst +3 -0
@@ -115,6 +115,9 @@ New Features Added eBPF JIT support for arm64 architecture to improve the eBPF program performance. +* **Added Lock-free Stack for aarch64.** + + The lock-free stack implementation is enabled for aarch64 platforms. Removed Items -------------
lib/librte_stack/Makefile +2 -1
@@ -24,6 +24,7 @@ SYMLINK-$(CONFIG_RTE_LIBRTE_STACK)-include := rte_stack.h \ rte_stack_std.h \ rte_stack_lf.h \ rte_stack_lf_generic.h \ - rte_stack_lf_c11.h + rte_stack_lf_c11.h \ + rte_stack_lf_stubs.h include $(RTE_SDK)/mk/rte.lib.mk
lib/librte_stack/rte_stack_lf.h +4 -0
@@ -5,11 +5,15 @@#ifndef _RTE_STACK_LF_H_#define _RTE_STACK_LF_H_ +#if !(defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 )) +#include "rte_stack_lf_stubs.h" +#else #ifdef RTE_USE_C11_MEM_MODEL #include "rte_stack_lf_c11.h" #else #include "rte_stack_lf_generic.h" #endif +#endif /** * @internal Push several objects on the lock-free stack (MT-safe).
lib/librte_stack/rte_stack_lf_c11.h +0 -16
@@ -36,12 +36,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list, struct rte_stack_lf_elem *last, unsigned int num) { -#ifndef RTE_ARCH_X86_64 -RTE_SET_USED (first); -RTE_SET_USED (last); -RTE_SET_USED (list); -RTE_SET_USED (num); -#else struct rte_stack_lf_head old_head;int success;
@@ -79,7 +73,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,* to the LIFO len update. */ __atomic_add_fetch (&list->len, num, __ATOMIC_RELEASE); -#endif } static __rte_always_inline struct rte_stack_lf_elem *
@@ -88,14 +81,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,void **obj_table, struct rte_stack_lf_elem **last) { -#ifndef RTE_ARCH_X86_64 -RTE_SET_USED (obj_table ); -RTE_SET_USED (last); -RTE_SET_USED (list); -RTE_SET_USED (num); - -return NULL ; -#else struct rte_stack_lf_head old_head;uint64_t len;int success;
@@ -169,7 +154,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list, } while (success == 0);return old_head.top; -#endif }#endif /* _RTE_STACK_LF_C11_H_ */
lib/librte_stack/rte_stack_lf_generic.h +0 -16
@@ -36,12 +36,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list, struct rte_stack_lf_elem *last, unsigned int num) { -#ifndef RTE_ARCH_X86_64 -RTE_SET_USED (first); -RTE_SET_USED (last); -RTE_SET_USED (list); -RTE_SET_USED (num); -#else struct rte_stack_lf_head old_head;int success;
@@ -75,7 +69,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list, }while (success ==0 );rte_atomic64_add ((rte_atomic64_t *)&list->len, num); -#endif } static __rte_always_inline struct rte_stack_lf_elem *
@@ -84,14 +77,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,void **obj_table, struct rte_stack_lf_elem **last) { -#ifndef RTE_ARCH_X86_64 -RTE_SET_USED (obj_table ); -RTE_SET_USED (last); -RTE_SET_USED (list); -RTE_SET_USED (num); - -return NULL ; -#else struct rte_stack_lf_head old_head;int success;
@@ -159,7 +144,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list, } while (success == 0);return old_head.top; -#endif }#endif /* _RTE_STACK_LF_GENERIC_H_ */
lib/librte_stack/rte_stack_lf_stubs.h new file +44 -0
44 lines not on this page · raw message
Signature · 1 line
2.7.4
David Marchand
AppliedRe: [dpdk-dev] [PATCH v11 3/3] eal/stack: enable lock-free stack for aarch64
Quoted 45 lines · Phil Yang
Enable both c11 atomic and non c11 atomic lock-free stack for aarch64.
Introduced a new header to reduce the ifdef clutter across generic and c11 files. The
rte_stack_lf_stubs.hcontains stub implementations of__rte_stack_lf_count,__rte_stack_lf_push_elemsand__rte_stack_lf_pop_elems.Suggested-by Gage Eads <[email protected]>Suggested-by Jerin Jacob <[email protected]>Signed-off-by Phil Yang <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>Tested-by Honnappa Nagarahalli <[email protected]>Acked-by Jerin Jacob <[email protected]>--- doc/guides/prog_guide/env_abstraction_layer.rst | 4 +-- doc/guides/rel_notes/release_19_11.rst | 3 ++ lib/librte_stack/Makefile | 3 +- lib/librte_stack/rte_stack_lf.h | 4 +++ lib/librte_stack/rte_stack_lf_c11.h | 16 --------- lib/librte_stack/rte_stack_lf_generic.h | 16 --------- lib/librte_stack/rte_stack_lf_stubs.h | 44 +++++++++++++++++++++++++ 7 files changed, 55 insertions(+), 35 deletions(-) create mode 100644 lib/librte_stack/rte_stack_lf_stubs.hdoc/guides/prog_guide/env_abstraction_layer.rst +2
-2@@ -648,8 +648,8 @@ Known Issues Alternatively, applications can use the lock-free stack mempool handler. When considering this handler, note that: - - It is currently limited to the x86_64 platform, because it uses an - instruction (16-byte compare-and-swap) that is not yet available on other + - It is currently limited to the aarch64 and x86_64 platforms, because it uses + an instruction (16-byte compare-and-swap) that is not yet available on other platforms. - It has worse average-case performance than the non-preemptive rte_ring, but software caching (e.g. the mempool cache) can mitigate this by reducing thedoc/guides/rel_notes/release_19_11.rst +0
-0
doc/guides/rel_notes/release_19_11.rst New Features · line 120
Quoted 6 lines
doc/guides/rel_notes/release_19_11.rst +3
-0@@ -115,6 +115,9 @@ New Features Added eBPF JIT support for arm64 architecture to improve the eBPF program performance. +* **Added Lock-free Stack for aarch64.** + + The lock-free stack implementation is enabled for aarch64 platforms.
Missing line (paragraphs are separated with two empty lines). This entry should go with Core libs entries, so at the beginning of this list.
I did the changes.
Quoted 183 lines
doc/guides/rel_notes/release_19_11.rst +0
-0Removed Items -------------lib/librte_stack/Makefile +2
-1@@ -24,6 +24,7 @@ SYMLINK-$(CONFIG_RTE_LIBRTE_STACK)-include := rte_stack.h \ rte_stack_std.h \ rte_stack_lf.h \ rte_stack_lf_generic.h \ - rte_stack_lf_c11.h + rte_stack_lf_c11.h \ + rte_stack_lf_stubs.h include $(RTE_SDK)/mk/rte.lib.mklib/librte_stack/rte_stack_lf.h +4
-0@@ -5,11 +5,15 @@#ifndef _RTE_STACK_LF_H_#define _RTE_STACK_LF_H_ +#if !(defined(RTE_ARCH_X86_64 ) || defined(RTE_ARCH_ARM64 )) +#include "rte_stack_lf_stubs.h" +#else #ifdef RTE_USE_C11_MEM_MODEL #include "rte_stack_lf_c11.h" #else #include "rte_stack_lf_generic.h" #endif +#endif /** * @internal Push several objects on the lock-free stack (MT-safe). lib/librte_stack/rte_stack_lf_c11.h +0
-16@@ -36,12 +36,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list, struct rte_stack_lf_elem *last, unsigned int num) { -#ifndef RTE_ARCH_X86_64 -RTE_SET_USED (first); -RTE_SET_USED (last); -RTE_SET_USED (list); -RTE_SET_USED (num); -#else struct rte_stack_lf_head old_head;int success;@@ -79,7 +73,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,* to the LIFO len update. */ __atomic_add_fetch (&list->len, num, __ATOMIC_RELEASE); -#endif } static __rte_always_inline struct rte_stack_lf_elem *@@ -88,14 +81,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,void **obj_table, struct rte_stack_lf_elem **last) { -#ifndef RTE_ARCH_X86_64 -RTE_SET_USED (obj_table ); -RTE_SET_USED (last); -RTE_SET_USED (list); -RTE_SET_USED (num); - -return NULL ; -#else struct rte_stack_lf_head old_head;uint64_t len;int success;@@ -169,7 +154,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list, } while (success == 0);return old_head.top; -#endif }#endif /* _RTE_STACK_LF_C11_H_ */ lib/librte_stack/rte_stack_lf_generic.h +0
-16@@ -36,12 +36,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list, struct rte_stack_lf_elem *last, unsigned int num) { -#ifndef RTE_ARCH_X86_64 -RTE_SET_USED (first); -RTE_SET_USED (last); -RTE_SET_USED (list); -RTE_SET_USED (num); -#else struct rte_stack_lf_head old_head;int success;@@ -75,7 +69,6 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list, }while (success ==0 );rte_atomic64_add ((rte_atomic64_t *)&list->len, num); -#endif } static __rte_always_inline struct rte_stack_lf_elem *@@ -84,14 +77,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,void **obj_table, struct rte_stack_lf_elem **last) { -#ifndef RTE_ARCH_X86_64 -RTE_SET_USED (obj_table ); -RTE_SET_USED (last); -RTE_SET_USED (list); -RTE_SET_USED (num); - -return NULL ; -#else struct rte_stack_lf_head old_head;int success;@@ -159,7 +144,6 @@ __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list, } while (success == 0);return old_head.top; -#endif }#endif /* _RTE_STACK_LF_GENERIC_H_ */ lib/librte_stack/rte_stack_lf_stubs.h new file +44
-1@@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2019 Arm Limited +*/ + +#ifndef _RTE_STACK_LF_STUBS_H_ +#define _RTE_STACK_LF_STUBS_H_ + +#include <rte_common.h> + +static __rte_always_inline unsigned int +__rte_stack_lf_count (struct rte_stack *s) +{ +RTE_SET_USED (s); + +return 0 ; +} + +static __rte_always_inline void +__rte_stack_lf_push_elems (struct rte_stack_lf_list *list, +struct rte_stack_lf_elem *first, +struct rte_stack_lf_elem *last, +unsigned int num) +{ +RTE_SET_USED (first); +RTE_SET_USED (last); +RTE_SET_USED (list); +RTE_SET_USED (num); +} + +static __rte_always_inline struct rte_stack_lf_elem * +__rte_stack_lf_pop_elems (struct rte_stack_lf_list *list, +unsigned int num, +void **obj_table, +struct rte_stack_lf_elem **last) +{ +RTE_SET_USED (obj_table); +RTE_SET_USED (last); +RTE_SET_USED (list); +RTE_SET_USED (num); + +return NULL ; +} + +#endif /* _RTE_STACK_LF_STUBS_H_ */ --
Quoted 2 lines
2.7.4
Applied, thanks.
Signature · 1 line
David Marchand
Quoted 10 lines · Phil Yang
This patch adds the implementation of the 128-bit atomic compare exchange API on AArch64. Using 64-bit 'ldxp/stxp' instructions can perform this operation. Moreover, on the LSE atomic extension accelerated platforms, it implemented by 'casp' instructions for better performance.
Since the '
__ARM_FEATURE_ATOMICS' flag only supports GCC-9, so this patch adds a new config flag 'RTE_ARM_FEATURE_ATOMICS' to enable the 'cas' version on elder version compilers.
Jerin, Phil,
I am getting a build error on the octeontx2 target:
{standard input}: Assembler messages:
{standard input}:672: Error: selected processor does not support `casp x0,x1,x2,x3,[x4]'
{standard input}:690: Error: selected processor does not support `caspa x0,x1,x2,x3,[x4]'
{standard input}:708: Error: selected processor does not support `caspl x0,x1,x2,x3,[x4]'
{standard input}:726: Error: selected processor does not support `caspal x0,x1,x2,x3,[x4]'
ninja: build stopped: subcommand failed.
Looking into the meson logs, I can see:
Native C compiler: ccache gcc (gcc 9.2.1 "gcc (GCC) 9.2.1 20190827 (Red Hat 9.2.1-1)")
Cross C compiler: aarch64-linux-gnu-gcc (gcc 8.2.1) Host machine cpu family: aarch64
Host machine cpu: armv8-a
Target machine cpu family: aarch64
Target machine cpu: armv8-a
Build machine cpu family: x86_64
Build machine cpu: x86_64
...
Message: Implementer : Cavium
Compiler for C supports arguments -mcpu=octeontx2: NO
Message: []
Fetching value of define "__ARM_NEON" : 1
Fetching value of define "__ARM_FEATURE_CRC32" :
Fetching value of define "__ARM_FEATURE_CRYPTO" :
My toolchain does not support the octeontx2 target, but RTE_ARM_FEATURE_ATOMICS ends up being set in the configuration anyway. Tried with Linaro toolchains (4.7.1, 7.4) mentionned in the dpdk documentation, same result.
Looking at config/arm/meson.build, the "extra machine specific flags" are appended to the configuration, regardless of what the compiler replied when testing the machine args.
Signature · 1 line
David Marchand
Quoted 30 lines · David Marchand
Quoted 10 lines · Phil Yang
This patch adds the implementation of the 128-bit atomic compare exchange API on AArch64. Using 64-bit 'ldxp/stxp' instructions can perform this operation. Moreover, on the LSE atomic extension accelerated platforms, it implemented by 'casp' instructions for better performance.
Since the '
__ARM_FEATURE_ATOMICS' flag only supports GCC-9, so this patch adds a new config flag 'RTE_ARM_FEATURE_ATOMICS' to enable the 'cas' version on elder version compilers.Jerin, Phil,
I am getting a build error on the octeontx2 target:
{standard input}: Assembler messages:
{standard input}:672: Error: selected processor does not support `casp x0,x1,x2,x3,[x4]'
{standard input}:690: Error: selected processor does not support `caspa x0,x1,x2,x3,[x4]'
{standard input}:708: Error: selected processor does not support `caspl x0,x1,x2,x3,[x4]'
{standard input}:726: Error: selected processor does not support `caspal x0,x1,x2,x3,[x4]'
ninja: build stopped: subcommand failed.Looking into the meson logs, I can see:
Quoted 12 lines
Native C compiler: ccache gcc (gcc 9.2.1 "gcc (GCC) 9.2.1 20190827 (Red Hat 9.2.1-1)")
Cross C compiler: aarch64-linux-gnu-gcc (gcc 8.2.1) Host machine cpu family: aarch64
Host machine cpu: armv8-a
Target machine cpu family: aarch64
Target machine cpu: armv8-a
Build machine cpu family:x86_64
Build machine cpu:x86_64
...
Message: Implementer : Cavium
Compiler for C supports arguments -mcpu=octeontx2: NO
The compiler needs either +lse or mcpu=octeontx2 to generate casp instruction. Could you try this patch, I can submit a patch if it works for you.
[master][dpdk-next-net-mrvl] $ git diff
config/arm/meson.build +0 -0
@@ -96,7 +96,7 @@ machine_args_cavium = [ ['0xa2', ['-mcpu=thunderxt81'], flags_thunderx_extra], ['0xa3', ['-mcpu=thunderxt83'], flags_thunderx_extra], ['0xaf', ['-march=armv8.1-a+crc+crypto','-mcpu=thunderx2t99'],
flags_thunderx2_extra],
- ['0xb2', ['-mcpu=octeontx2'], flags_octeontx2_extra]]
+ ['0xb2',
['-march=armv8.2-a+crc+crypto+lse','-mcpu=octeontx2'], flags_octeontx2_extra]]
## Arm implementer ID (ARM DDI 0487C.a, Section G7.2.106, Page G7-5321) impl_generic = ['Generic armv8', flags_generic, machine_args_generic]
Quoted 46 lines · Jerin Jacob
Quoted 30 lines · David Marchand
Quoted 10 lines · Phil Yang
This patch adds the implementation of the 128-bit atomic compare exchange API on AArch64. Using 64-bit 'ldxp/stxp' instructions can perform this operation. Moreover, on the LSE atomic extension accelerated platforms, it implemented by 'casp' instructions for better performance.
Since the '
__ARM_FEATURE_ATOMICS' flag only supports GCC-9, so this patch adds a new config flag 'RTE_ARM_FEATURE_ATOMICS' to enable the 'cas' version on elder version compilers.Jerin, Phil,
I am getting a build error on the octeontx2 target:
{standard input}: Assembler messages:
{standard input}:672: Error: selected processor does not support `casp x0,x1,x2,x3,[x4]'
{standard input}:690: Error: selected processor does not support `caspa x0,x1,x2,x3,[x4]'
{standard input}:708: Error: selected processor does not support `caspl x0,x1,x2,x3,[x4]'
{standard input}:726: Error: selected processor does not support `caspal x0,x1,x2,x3,[x4]'
ninja: build stopped: subcommand failed.Looking into the meson logs, I can see:
Quoted 12 lines
Native C compiler: ccache gcc (gcc 9.2.1 "gcc (GCC) 9.2.1 20190827 (Red Hat 9.2.1-1)")
Cross C compiler: aarch64-linux-gnu-gcc (gcc 8.2.1) Host machine cpu family: aarch64
Host machine cpu: armv8-a
Target machine cpu family: aarch64
Target machine cpu: armv8-a
Build machine cpu family:x86_64
Build machine cpu:x86_64
...
Message: Implementer : Cavium
Compiler for C supports arguments -mcpu=octeontx2: NO
Quoted 2 lines
The compiler needs either +lse or mcpu=octeontx2 to generate casp instruction. Could you try this patch, I can submit a patch if it works for you.
Ah cool, I was looking at the march stuff.
Tried your patch, it works fine.
I'd say we can squash your bits in the current patch, since this was unneeded before this patch.
Is this okay for you?
Quoted 16 lines
[master][dpdk-next-net-mrvl] $ git diff
config/arm/meson.build +1
-1@@ -96,7 +96,7 @@ machine_args_cavium = [ ['0xa2', ['-mcpu=thunderxt81'], flags_thunderx_extra], ['0xa3', ['-mcpu=thunderxt83'], flags_thunderx_extra], ['0xaf', ['-march=armv8.1-a+crc+crypto','-mcpu=thunderx2t99'], flags_thunderx2_extra], - ['0xb2', ['-mcpu=octeontx2'], flags_octeontx2_extra]] + ['0xb2', ['-march=armv8.2-a+crc+crypto+lse','-mcpu=octeontx2'],
flags_octeontx2_extra]]
Quoted 2 lines
## Arm implementer ID (ARM DDI 0487C.a, Section G7.2.106, Page G7-5321)
impl_generic= ['Generic armv8',flags_generic,machine_args_generic]
Thanks for the quick reply.
Signature · 1 line
David Marchand
Quoted 51 lines · David Marchand
Quoted 46 lines · Jerin Jacob
Quoted 30 lines · David Marchand
Quoted 10 lines · Phil Yang
This patch adds the implementation of the 128-bit atomic compare exchange API on AArch64. Using 64-bit 'ldxp/stxp' instructions can perform this operation. Moreover, on the LSE atomic extension accelerated platforms, it implemented by 'casp' instructions for better performance.
Since the '
__ARM_FEATURE_ATOMICS' flag only supports GCC-9, so this patch adds a new config flag 'RTE_ARM_FEATURE_ATOMICS' to enable the 'cas' version on elder version compilers.Jerin, Phil,
I am getting a build error on the octeontx2 target:
{standard input}: Assembler messages:
{standard input}:672: Error: selected processor does not support `casp x0,x1,x2,x3,[x4]'
{standard input}:690: Error: selected processor does not support `caspa x0,x1,x2,x3,[x4]'
{standard input}:708: Error: selected processor does not support `caspl x0,x1,x2,x3,[x4]'
{standard input}:726: Error: selected processor does not support `caspal x0,x1,x2,x3,[x4]'
ninja: build stopped: subcommand failed.Looking into the meson logs, I can see:
Quoted 12 lines
Native C compiler: ccache gcc (gcc 9.2.1 "gcc (GCC) 9.2.1 20190827 (Red Hat 9.2.1-1)")
Cross C compiler: aarch64-linux-gnu-gcc (gcc 8.2.1) Host machine cpu family: aarch64
Host machine cpu: armv8-a
Target machine cpu family: aarch64
Target machine cpu: armv8-a
Build machine cpu family:x86_64
Build machine cpu:x86_64
...
Message: Implementer : Cavium
Compiler for C supports arguments -mcpu=octeontx2: NOQuoted 2 lines
The compiler needs either +lse or mcpu=octeontx2 to generate casp instruction. Could you try this patch, I can submit a patch if it works for you.
Quoted 6 lines
Ah cool, I was looking at the march stuff.
Tried your patch, it works fine.I'd say we can squash your bits in the current patch, since this was unneeded before this patch.
Is this okay for you?
Yup.
Quoted 27 lines
Quoted 18 lines
[master][dpdk-next-net-mrvl] $ git diff
config/arm/meson.build +1
-1@@ -96,7 +96,7 @@ machine_args_cavium = [ ['0xa2', ['-mcpu=thunderxt81'], flags_thunderx_extra], ['0xa3', ['-mcpu=thunderxt83'], flags_thunderx_extra], ['0xaf', ['-march=armv8.1-a+crc+crypto','-mcpu=thunderx2t99'], flags_thunderx2_extra], - ['0xb2', ['-mcpu=octeontx2'], flags_octeontx2_extra]] + ['0xb2', ['-march=armv8.2-a+crc+crypto+lse','-mcpu=octeontx2'],
flags_octeontx2_extra]]Quoted 2 lines
## Arm implementer ID (ARM DDI 0487C.a, Section G7.2.106, Page G7-5321)
impl_generic= ['Generic armv8',flags_generic,machine_args_generic]Thanks for the quick reply.
--
David Marchand
Quoted 54 lines · Jerin Jacob
Quoted 51 lines · David Marchand
Quoted 48 lines · Jerin Jacob
Quoted 30 lines · David Marchand
Quoted 10 lines · Phil Yang
This patch adds the implementation of the 128-bit atomic compare exchange API on AArch64. Using 64-bit 'ldxp/stxp' instructions can perform this operation. Moreover, on the LSE atomic extension accelerated platforms, it implemented by 'casp' instructions for better performance.
Since the '
__ARM_FEATURE_ATOMICS' flag only supports GCC-9, so this patch adds a new config flag 'RTE_ARM_FEATURE_ATOMICS' to enable the 'cas' version on elder version compilers.Jerin, Phil,
I am getting a build error on the octeontx2 target:
{standard input}: Assembler messages:
{standard input}:672: Error: selected processor does not support `casp x0,x1,x2,x3,[x4]'
{standard input}:690: Error: selected processor does not support `caspa x0,x1,x2,x3,[x4]'
{standard input}:708: Error: selected processor does not support `caspl x0,x1,x2,x3,[x4]'
{standard input}:726: Error: selected processor does not support `caspal x0,x1,x2,x3,[x4]'
ninja: build stopped: subcommand failed.Looking into the meson logs, I can see:
Quoted 12 lines
Native C compiler: ccache gcc (gcc 9.2.1 "gcc (GCC) 9.2.1 20190827 (Red Hat 9.2.1-1)")
Cross C compiler: aarch64-linux-gnu-gcc (gcc 8.2.1) Host machine cpu family: aarch64
Host machine cpu: armv8-a
Target machine cpu family: aarch64
Target machine cpu: armv8-a
Build machine cpu family:x86_64
Build machine cpu:x86_64
...
Message: Implementer : Cavium
Compiler for C supports arguments -mcpu=octeontx2: NOThe compiler needs either +lse or mcpu=octeontx2 to generate casp instruction. Could you try this patch, I can submit a patch if it works for you.
Quoted 32 lines
Quoted 6 lines
Ah cool, I was looking at the march stuff.
Tried your patch, it works fine.I'd say we can squash your bits in the current patch, since this was unneeded before this patch.
Is this okay for you?Yup.
Quoted 23 lines
Quoted 18 lines
[master][dpdk-next-net-mrvl] $ git diff
config/arm/meson.build +1
-1@@ -96,7 +96,7 @@ machine_args_cavium = [ ['0xa2', ['-mcpu=thunderxt81'], flags_thunderx_extra], ['0xa3', ['-mcpu=thunderxt83'], flags_thunderx_extra], ['0xaf', ['-march=armv8.1-a+crc+crypto','-mcpu=thunderx2t99'], flags_thunderx2_extra], - ['0xb2', ['-mcpu=octeontx2'], flags_octeontx2_extra]] + ['0xb2', ['-march=armv8.2-a+crc+crypto+lse','-mcpu=octeontx2'],
flags_octeontx2_extra]]Quoted 2 lines
## Arm implementer ID (ARM DDI 0487C.a, Section G7.2.106, Page G7-5321)
impl_generic= ['Generic armv8',flags_generic,machine_args_generic]Thanks for the quick reply.
Applied with above fix.
Thanks.
Signature · 1 line
David Marchand