mempool: split statistics from debug
PATCH 1/1 mempool: split statistics from debug
Split statistics from debug, to make mempool statistics available without the performance cost of continuously validating the cookies in the mempool elements.
--- config/rte_config.h | 2 ++ lib/mempool/rte_mempool.c | 10 +++++----- lib/mempool/rte_mempool.h | 14 +++++++------- 3 files changed, 14 insertions(+), 12 deletions(-)
config/rte_config.h +2 -0
@@ -47,6 +47,8 @@/* mempool defines */ #define RTE_MEMPOOL_CACHE_MAX_SIZE 512 +// RTE_LIBRTE_MEMPOOL_STATS is not set +// RTE_LIBRTE_MEMPOOL_DEBUG is not set /* mbuf defines */ #define RTE_MBUF_DEFAULT_MEMPOOL_OPS "ring_mp_mc"
lib/mempool/rte_mempool.c +5 -5
@@ -818,8 +818,8 @@ rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size, RTE_CACHE_LINE_MASK) != 0);RTE_BUILD_BUG_ON ((sizeof (struct rte_mempool_cache ) &RTE_CACHE_LINE_MASK ) !=0 ); -#ifdef RTE_LIBRTE_MEMPOOL_DEBUG -RTE_BUILD_BUG_ON ((sizeof (struct rte_mempool_debug_stats ) & +#ifdef RTE_LIBRTE_MEMPOOL_STATS +RTE_BUILD_BUG_ON ((sizeof (struct rte_mempool_stats ) &RTE_CACHE_LINE_MASK ) !=0 );RTE_BUILD_BUG_ON ((offsetof(struct rte_mempool , stats) &RTE_CACHE_LINE_MASK ) !=0 );
@@ -1221,9 +1221,9 @@ rte_mempool_audit(struct rte_mempool *mp)void rte_mempool_dump (FILE *f,struct rte_mempool *mp) { -#ifdef RTE_LIBRTE_MEMPOOL_DEBUG +#ifdef RTE_LIBRTE_MEMPOOL_STATS struct rte_mempool_info info; -struct rte_mempool_debug_stats sum; +struct rte_mempool_stats sum;unsigned lcore_id;#endif struct rte_mempool_memhdr *memhdr;
@@ -1269,7 +1269,7 @@ rte_mempool_dump(FILE *f, struct rte_mempool *mp)fprintf (f," common_pool_count=%u\n" , common_count);/* sum and dump statistics */ -#ifdef RTE_LIBRTE_MEMPOOL_DEBUG +#ifdef RTE_LIBRTE_MEMPOOL_STATS rte_mempool_ops_get_info (mp, &info);memset (&sum,0 ,sizeof (sum));for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) {
lib/mempool/rte_mempool.h +7 -7
@@ -56,14 +56,14 @@ extern "C" {#define RTE_MEMPOOL_HEADER_COOKIE2 0xf2eef2eedadd2e55ULL/**< Header cookie. */ #define RTE_MEMPOOL_TRAILER_COOKIE 0xadd2e55badbadbadULL/**< Trailer cookie.*/ -#ifdef RTE_LIBRTE_MEMPOOL_DEBUG +#ifdef RTE_LIBRTE_MEMPOOL_STATS /** * A structure that stores the mempool statistics (per-lcore). * Note: Cache stats (put_cache_bulk/objs, get_cache_bulk/objs) are not * captured since they can be calculated from other stats. * For example: put_cache_objs = put_objs - put_common_pool_objs. */ -struct rte_mempool_debug_stats { +struct rte_mempool_stats { uint64_t put_bulk; /**< Number of puts. */ uint64_t put_objs; /**< Number of objects successfully put. */ uint64_t put_common_pool_bulk; /**< Number of bulks enqueued in common pool. */
@@ -237,9 +237,9 @@ struct rte_mempool {uint32_t nb_mem_chunks;/**< Number of memory chunks */ struct rte_mempool_memhdr_list mem_list;/**< List of memory chunks */ -#ifdef RTE_LIBRTE_MEMPOOL_DEBUG +#ifdef RTE_LIBRTE_MEMPOOL_STATS /** Per-lcore statistics. */ -struct rte_mempool_debug_stats stats[RTE_MAX_LCORE ]; +struct rte_mempool_stats stats[RTE_MAX_LCORE ];#endif } __rte_cache_aligned;
@@ -293,16 +293,16 @@ struct rte_mempool { | RTE_MEMPOOL_F_NO_IOVA_CONTIG \ )/** -* @internal When debug is enabled, store some statistics. +* @internal When stats are enabled, store some statistics. * * @param mp * Pointer to the memory pool. * @param name * Name of the statistics field to increment in the memory pool. * @param n -* Number to add to the object-oriented statistics. +* Number to add to the statistics. */ -#ifdef RTE_LIBRTE_MEMPOOL_DEBUG +#ifdef RTE_LIBRTE_MEMPOOL_STATS #define RTE_MEMPOOL_STAT_ADD (mp, name, n) do { \ unsigned __lcore_id = rte_lcore_id(); \ if (__lcore_id < RTE_MAX_LCORE) { \
Signature · 1 line
2.17.1
Quoted 8 lines
From: Morten Brørup [mailto:[email protected]]
Sent: Sunday, 30 October 2022 12.55Split statistics from debug, to make mempool statistics available without
the performance cost of continuously validating the cookies in the mempool
elements.
mempool_perf_autotest shows that the rate_persec drops to a third (-66 %) when enabling full mempool debug - quite prohibitive!
With this patch, the performance cost is much lower. I don't have detailed test results, but the initial tests without mempool cache show a performance drop of -11 %. Significant, but not prohibitive.
Quoted 119 lines
Signed-off-by Morten Brørup <[email protected]>--- config/rte_config.h | 2 ++ lib/mempool/rte_mempool.c | 10 +++++----- lib/mempool/rte_mempool.h | 14 +++++++------- 3 files changed, 14 insertions(+), 12 deletions(-)config/rte_config.h +2
-0@@ -47,6 +47,8 @@/* mempool defines */ #define RTE_MEMPOOL_CACHE_MAX_SIZE 512 +// RTE_LIBRTE_MEMPOOL_STATS is not set +// RTE_LIBRTE_MEMPOOL_DEBUG is not set /* mbuf defines */ #define RTE_MBUF_DEFAULT_MEMPOOL_OPS "ring_mp_mc"lib/mempool/rte_mempool.c +2
-2@@ -818,8 +818,8 @@ rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size, RTE_CACHE_LINE_MASK) != 0);RTE_BUILD_BUG_ON ((sizeof (struct rte_mempool_cache ) &RTE_CACHE_LINE_MASK ) !=0 ); -#ifdef RTE_LIBRTE_MEMPOOL_DEBUG -RTE_BUILD_BUG_ON ((sizeof (struct rte_mempool_debug_stats ) & +#ifdef RTE_LIBRTE_MEMPOOL_STATS +RTE_BUILD_BUG_ON ((sizeof (struct rte_mempool_stats ) &RTE_CACHE_LINE_MASK ) !=0 ); RTE_BUILD_BUG_ON((offsetof(struct rte_mempool, stats) &lib/mempool/rte_mempool.c +3
-3RTE_CACHE_LINE_MASK) != 0);@@ -1221,9 +1221,9 @@ rte_mempool_audit(struct rte_mempool *mp)void rte_mempool_dump (FILE *f,struct rte_mempool *mp) { -#ifdef RTE_LIBRTE_MEMPOOL_DEBUG +#ifdef RTE_LIBRTE_MEMPOOL_STATS struct rte_mempool_info info; -struct rte_mempool_debug_stats sum; +struct rte_mempool_stats sum;unsigned lcore_id;#endif struct rte_mempool_memhdr *memhdr;@@ -1269,7 +1269,7 @@ rte_mempool_dump(FILE *f, struct rte_mempool *mp)fprintf (f," common_pool_count=%u\n" , common_count);/* sum and dump statistics */ -#ifdef RTE_LIBRTE_MEMPOOL_DEBUG +#ifdef RTE_LIBRTE_MEMPOOL_STATS rte_mempool_ops_get_info (mp, &info);memset (&sum,0 ,sizeof (sum));for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) {lib/mempool/rte_mempool.h +0
-0@@ -56,14 +56,14 @@ extern "C" {#define RTE_MEMPOOL_HEADER_COOKIE2 0xf2eef2eedadd2e55ULL/**< Header cookie. */
#defineRTE_MEMPOOL_TRAILER_COOKIE0xadd2e55badbadbadULL/**< Trailer
cookie.*/lib/mempool/rte_mempool.h +1
-1-#ifdef RTE_LIBRTE_MEMPOOL_DEBUG +#ifdef RTE_LIBRTE_MEMPOOL_STATS /** * A structure that stores the mempool statistics (per-lcore). * Note: Cache stats (put_cache_bulk/objs, get_cache_bulk/objs) are not
* captured since they can be calculated from other stats.
* For example:put_cache_objs=put_objs-put_common_pool_objs. */lib/mempool/rte_mempool.h +1
-1-struct rte_mempool_debug_stats { +struct rte_mempool_stats {uint64_t put_bulk;/**< Number of puts. */ uint64_t put_objs;/**< Number of objects successfully put. */
uint64_tput_common_pool_bulk; /**< Number of bulks enqueued in
common pool. */lib/mempool/rte_mempool.h +0
-0@@ -237,9 +237,9 @@ struct rte_mempool {uint32_t nb_mem_chunks;/**< Number of memory chunks */ struct rte_mempool_memhdr_list mem_list;/**< List of memory chunks */
lib/mempool/rte_mempool.h +5
-6-#ifdef RTE_LIBRTE_MEMPOOL_DEBUG +#ifdef RTE_LIBRTE_MEMPOOL_STATS /** Per-lcore statistics. */ -struct rte_mempool_debug_stats stats[RTE_MAX_LCORE ]; +struct rte_mempool_stats stats[RTE_MAX_LCORE ];#endif } __rte_cache_aligned;@@ -293,16 +293,16 @@ struct rte_mempool { | RTE_MEMPOOL_F_NO_IOVA_CONTIG \ )/** -* @internal When debug is enabled, store some statistics. +* @internal When stats are enabled, store some statistics. * * @param mp * Pointer to the memory pool. * @param name * Name of the statistics field to increment in the memory pool. * @param n -* Number to add to the object-oriented statistics. +* Number to add to the statistics. */ -#ifdef RTE_LIBRTE_MEMPOOL_DEBUG +#ifdef RTE_LIBRTE_MEMPOOL_STATS #define RTE_MEMPOOL_STAT_ADD (mp, name, n) do { \ unsigned __lcore_id = rte_lcore_id(); \ if (__lcore_id < RTE_MAX_LCORE) { \ --2.17.1
Quoted 9 lines · On Sun, 30 Oct 2022 15:04:18 +0100 Morten Brørup <[email protected]>…
Quoted 8 lines
From: Morten Brørup [mailto:[email protected]]
Sent: Sunday, 30 October 2022 12.55Split statistics from debug, to make mempool statistics available without
the performance cost of continuously validating the cookies in the mempool
elements.
Quoted 3 lines
mempool_perf_autotestshows that therate_persecdrops to a third (-66 %) when enabling full mempool debug - quite prohibitive!With this patch, the performance cost is much lower. I don't have detailed test results, but the initial tests without mempool cache show a performance drop of -11 %. Significant, but not prohibitive.
One trick to avoid conditional in fast path would be to add a dummy stats[] per core.
Another would be to move the fast path get/put stats into the mempool_cache. The current model has stats[] per cpu always on another cache line.
lib/mempool/rte_mempool.h +5 -6
@@ -236,8 +236,10 @@ struct rte_mempool {struct rte_mempool_memhdr_list mem_list;/**< List of memory chunks */ #ifdef RTE_LIBRTE_MEMPOOL_DEBUG -/** Per-lcore statistics. */ -struct rte_mempool_debug_stats stats[RTE_MAX_LCORE ]; +/** Per-lcore statistics. +* Allocate one additional per-cpu slot for non-DPDK threads +*/ +struct rte_mempool_debug_stats stats[RTE_MAX_LCORE +1 ];#endif } __rte_cache_aligned;
@@ -302,10 +304,7 @@ struct rte_mempool {*/ #ifdef RTE_LIBRTE_MEMPOOL_DEBUG #define RTE_MEMPOOL_STAT_ADD (mp, name, n) do { \ - unsigned __lcore_id = rte_lcore_id(); \ - if (__lcore_id < RTE_MAX_LCORE) { \ - mp->stats[__lcore_id].name += n; \ - } \ + (mp)->stats[rte_lcore_id()].name += n; } while (0) #else#define RTE_MEMPOOL_STAT_ADD (mp, name, n) do {} while (0)
Quoted 23 lines
From: Stephen Hemminger [mailto:[email protected]]
Sent: Sunday, 30 October 2022 17.13Quoted 10 lines · On Sun, 30 Oct 2022 15:04:18 +0100 Morten Brørup <[email protected]>…
Quoted 8 lines
From: Morten Brørup [mailto:[email protected]]
Sent: Sunday, 30 October 2022 12.55Split statistics from debug, to make mempool statistics available without
the performance cost of continuously validating the cookies in the mempool
elements.
mempool_perf_autotestshows that therate_persecdrops to a third (-66 %) when enabling full mempool debug - quite prohibitive!
Quoted 2 lines
With this patch, the performance cost is much lower. I don't have
detailed test results, but the initial tests without mempool cache show a performance drop of -11 %. Significant, but not prohibitive.
Quoted 6 lines
One trick to avoid conditional in fast path would be to add a dummy stats[] per core.
Another would be to move the fast path get/put stats into the
mempool_cache.
The current model has stats[] per cpu always on another cache line.
I submitted a patch to move the likely get/put stats to the mempool cache [1], but retracted it shortly after. I realized that splitting the stats from the debug cookies had much higher performance effect, so we should do this first. We can move statistics counters into the mempool_cache as part two.
[1]: https://patchwork.dpdk.org/project/dpdk/patch/[email protected]/
Also, it might be too late for 22.11 to move stats to the mempool cache. However, splitting the stats from the debug cookies is extremely simple, so that should be accepted for 22.11 (if reviewed properly).
Quoted 10 lines
lib/mempool/rte_mempool.h +0
-0@@ -236,8 +236,10 @@ struct rte_mempool {struct rte_mempool_memhdr_list mem_list;/**< List of memory chunks */
#ifdef
RTE_LIBRTE_MEMPOOL_DEBUG
lib/mempool/rte_mempool.h struct rte_mempool { · line 242
Quoted 6 lines
lib/mempool/rte_mempool.h +4
-2-/** Per-lcore statistics. */ -struct rte_mempool_debug_stats stats[RTE_MAX_LCORE ]; +/** Per-lcore statistics. +* Allocate one additional per-cpu slot for non-DPDK threads +*/ +struct rte_mempool_debug_stats stats[RTE_MAX_LCORE +1 ];
Excellent! As a bonus, this also fixes a bug in the statistics: Non-DPDK thread operations were not counted.
lib/mempool/rte_mempool.h struct rte_mempool { · line 307
Quoted 12 lines
#endif
}__rte_cache_aligned;lib/mempool/rte_mempool.h +1
-4@@ -302,10 +304,7 @@ struct rte_mempool {*/ #ifdef RTE_LIBRTE_MEMPOOL_DEBUG #define RTE_MEMPOOL_STAT_ADD (mp, name, n) do { \ - unsigned __lcore_id = rte_lcore_id(); \ - if (__lcore_id < RTE_MAX_LCORE) { \ - mp->stats[__lcore_id].name += n; \ - } \ + (mp)->stats[rte_lcore_id()].name += n;
I suppose the index must be offset by one, for rte_lcore_id() returning LCORE_ID_ANY (=UINT32_MAX) to be stored at offset zero:
+ (mp)->stats[rte_lcore_id() + 1].name += n;
lib/mempool/rte_mempool.h struct rte_mempool { · line 310
Quoted 3 lines
lib/mempool/rte_mempool.h +0
-0} while (0)#else #define RTE_MEMPOOL_STAT_ADD (mp, name, n) do {} while (0)
I have marked this patch as Changes Requested, and will submit a v2 patch series with this improvement - but not today, considering the local time.
NB: I am aware that the loop in rte_mempool_dump() in rte_mempool.c must also be updated to account for the additional slot in the stats array. I'll check if there are similar effects elsewhere in the library.
PATCH v2 1/3 mempool: split statistics from debug
In reply to Morten BrørupSplit statistics from debug, to make mempool statistics available without the performance cost of continuously validating the cookies in the mempool elements.
mempool_perf_autotest shows the follwing change in rate_persec.
When enabling mempool debug without this patch:
-28.1 % and -74.0 %, respectively without and with cache.
When enabling mempool stats (but not debug) with this patch:
-5.8 % and -21.2 %, respectively without and with cache.
v2:
* Fix checkpatch warning:
Use C style comments in rte_include.h, not C++ style.
* Do not rename the rte_mempool_debug_stats structure.
--- config/rte_config.h | 2 ++ lib/mempool/rte_mempool.c | 6 +++--- lib/mempool/rte_mempool.h | 6 +++--- 3 files changed, 8 insertions(+), 6 deletions(-)
config/rte_config.h +2 -0
@@ -47,6 +47,8 @@/* mempool defines */ #define RTE_MEMPOOL_CACHE_MAX_SIZE 512 +/* RTE_LIBRTE_MEMPOOL_STATS is not set */ +/* RTE_LIBRTE_MEMPOOL_DEBUG is not set */ /* mbuf defines */ #define RTE_MBUF_DEFAULT_MEMPOOL_OPS "ring_mp_mc"
lib/mempool/rte_mempool.c +3 -3
@@ -818,7 +818,7 @@ rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size, RTE_CACHE_LINE_MASK) != 0);RTE_BUILD_BUG_ON ((sizeof (struct rte_mempool_cache ) &RTE_CACHE_LINE_MASK ) !=0 ); -#ifdef RTE_LIBRTE_MEMPOOL_DEBUG +#ifdef RTE_LIBRTE_MEMPOOL_STATS RTE_BUILD_BUG_ON ((sizeof (struct rte_mempool_debug_stats ) &RTE_CACHE_LINE_MASK ) !=0 ); RTE_BUILD_BUG_ON((offsetof(struct rte_mempool, stats) &
@@ -1221,7 +1221,7 @@ rte_mempool_audit(struct rte_mempool *mp)void rte_mempool_dump (FILE *f,struct rte_mempool *mp) { -#ifdef RTE_LIBRTE_MEMPOOL_DEBUG +#ifdef RTE_LIBRTE_MEMPOOL_STATS struct rte_mempool_info info;struct rte_mempool_debug_stats sum;unsigned lcore_id;
@@ -1269,7 +1269,7 @@ rte_mempool_dump(FILE *f, struct rte_mempool *mp)fprintf (f," common_pool_count=%u\n" , common_count);/* sum and dump statistics */ -#ifdef RTE_LIBRTE_MEMPOOL_DEBUG +#ifdef RTE_LIBRTE_MEMPOOL_STATS rte_mempool_ops_get_info (mp, &info);memset (&sum,0 ,sizeof (sum));for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) {
lib/mempool/rte_mempool.h +3 -3
@@ -56,7 +56,7 @@ extern "C" {#define RTE_MEMPOOL_HEADER_COOKIE2 0xf2eef2eedadd2e55ULL/**< Header cookie. */ #define RTE_MEMPOOL_TRAILER_COOKIE 0xadd2e55badbadbadULL/**< Trailer cookie.*/ -#ifdef RTE_LIBRTE_MEMPOOL_DEBUG +#ifdef RTE_LIBRTE_MEMPOOL_STATS /** * A structure that stores the mempool statistics (per-lcore). * Note: Cache stats (put_cache_bulk/objs, get_cache_bulk/objs) are not
@@ -237,7 +237,7 @@ struct rte_mempool {uint32_t nb_mem_chunks;/**< Number of memory chunks */ struct rte_mempool_memhdr_list mem_list;/**< List of memory chunks */ -#ifdef RTE_LIBRTE_MEMPOOL_DEBUG +#ifdef RTE_LIBRTE_MEMPOOL_STATS /** Per-lcore statistics. */ struct rte_mempool_debug_stats stats[RTE_MAX_LCORE ];#endif
@@ -302,7 +302,7 @@ struct rte_mempool {* @param n * Number to add to the object-oriented statistics. */ -#ifdef RTE_LIBRTE_MEMPOOL_DEBUG +#ifdef RTE_LIBRTE_MEMPOOL_STATS #define RTE_MEMPOOL_STAT_ADD (mp, name, n) do { \ unsigned __lcore_id = rte_lcore_id(); \ if (__lcore_id < RTE_MAX_LCORE) { \
Signature · 1 line
2.17.1
PATCH v2 2/3 mempool: include non-DPDK threads in statistics
Offset the stats array index by one, and count non-DPDK threads at index zero.
This patch provides two benefits:
* Non-DPDK threads are also included in the statistics.
* A conditional in the fast path is removed. Static branch prediction was correct, so the performance improvement is negligible.
v2:
* New. No v1 of this patch in the series.
--- lib/mempool/rte_mempool.c | 2 +- lib/mempool/rte_mempool.h | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-)
lib/mempool/rte_mempool.c +1 -1
@@ -1272,7 +1272,7 @@ rte_mempool_dump(FILE *f, struct rte_mempool *mp)#ifdef RTE_LIBRTE_MEMPOOL_STATS rte_mempool_ops_get_info (mp, &info);memset (&sum,0 ,sizeof (sum)); -for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) { +for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE +1 ; lcore_id++) { sum.put_bulk += mp->stats[lcore_id].put_bulk; sum.put_objs += mp->stats[lcore_id].put_objs; sum.put_common_pool_bulk += mp->stats[lcore_id].put_common_pool_bulk;
lib/mempool/rte_mempool.h +6 -6
@@ -238,8 +238,11 @@ struct rte_mempool {struct rte_mempool_memhdr_list mem_list;/**< List of memory chunks */ #ifdef RTE_LIBRTE_MEMPOOL_STATS -/** Per-lcore statistics. */ -struct rte_mempool_debug_stats stats[RTE_MAX_LCORE ]; +/** Per-lcore statistics. +* +* Offset by one, to include non-DPDK threads. +*/ +struct rte_mempool_debug_stats stats[RTE_MAX_LCORE +1 ];#endif } __rte_cache_aligned;
@@ -304,10 +307,7 @@ struct rte_mempool {*/ #ifdef RTE_LIBRTE_MEMPOOL_STATS #define RTE_MEMPOOL_STAT_ADD (mp, name, n) do { \ - unsigned __lcore_id = rte_lcore_id(); \ - if (__lcore_id < RTE_MAX_LCORE) { \ - mp->stats[__lcore_id].name += n; \ - } \ + (mp)->stats[rte_lcore_id() + 1].name += n; \ } while (0)#else #define RTE_MEMPOOL_STAT_ADD (mp, name, n) do {} while (0)
Signature · 1 line
2.17.1
Re: [PATCH v2 2/3] mempool: include non-DPDK threads in statistics
Quoted 50 lines · Morten Brørup
Offset the stats array index by one, and count non-DPDK threads at index zero.
This patch provides two benefits:
* Non-DPDK threads are also included in the statistics.
* A conditional in the fast path is removed. Static branch prediction was correct, so the performance improvement is negligible.v2:
* New. No v1 of this patch in the series.Suggested-by Stephen Hemminger <[email protected]>Signed-off-by Morten Brørup <[email protected]>--- lib/mempool/rte_mempool.c | 2 +- lib/mempool/rte_mempool.h | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-)lib/mempool/rte_mempool.c +1
-1@@ -1272,7 +1272,7 @@ rte_mempool_dump(FILE *f, struct rte_mempool *mp)#ifdef RTE_LIBRTE_MEMPOOL_STATS rte_mempool_ops_get_info (mp, &info);memset (&sum,0 ,sizeof (sum)); -for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) { +for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE +1 ; lcore_id++) { sum.put_bulk += mp->stats[lcore_id].put_bulk; sum.put_objs += mp->stats[lcore_id].put_objs; sum.put_common_pool_bulk += mp->stats[lcore_id].put_common_pool_bulk;lib/mempool/rte_mempool.h +5
-2@@ -238,8 +238,11 @@ struct rte_mempool {struct rte_mempool_memhdr_list mem_list;/**< List of memory chunks */ #ifdef RTE_LIBRTE_MEMPOOL_STATS -/** Per-lcore statistics. */ -struct rte_mempool_debug_stats stats[RTE_MAX_LCORE ]; +/** Per-lcore statistics. +* +* Offset by one, to include non-DPDK threads. +*/ +struct rte_mempool_debug_stats stats[RTE_MAX_LCORE +1 ];#endif } __rte_cache_aligned;
lib/mempool/rte_mempool.h struct rte_mempool { · line 310
Quoted 8 lines
lib/mempool/rte_mempool.h +1
-4@@ -304,10 +307,7 @@ struct rte_mempool {*/ #ifdef RTE_LIBRTE_MEMPOOL_STATS #define RTE_MEMPOOL_STAT_ADD (mp, name, n) do { \ - unsigned __lcore_id = rte_lcore_id(); \ - if (__lcore_id < RTE_MAX_LCORE) { \ - mp->stats[__lcore_id].name += n; \ - } \ + (mp)->stats[rte_lcore_id() + 1].name += n; \
This relies on LCORE_ID_ANY being UINT32_MAX, and a wrap to 0, for an unregistered non-EAL thread? Might be worth a comment, or better a rewrite with an explicit LCORE_ID_ANY comparison.
You anyways need a conditional. An atomic add must be used in the unregistered EAL thread case.
Quoted 3 lines
lib/mempool/rte_mempool.h +0
-0} while (0)#else #define RTE_MEMPOOL_STAT_ADD (mp, name, n) do {} while (0)
RE: [PATCH v2 2/3] mempool: include non-DPDK threads in statistics
Quoted 58 lines
From: Mattias Rönnblom [mailto:[email protected]]
Sent: Wednesday, 2 November 2022 08.53Quoted 1 line · Morten Brørup
Offset the stats array index by one, and count non-DPDK threads at
index
Quoted 5 lines
zero.
This patch provides two benefits:
* Non-DPDK threads are also included in the statistics.
* A conditional in the fast path is removed. Static branch predictionwas
lib/mempool/rte_mempool.c rte_mempool_dump(FILE *f, struct rte_mempool
Quoted 17 lines
correct, so the performance improvement is negligible.
v2:
* New. No v1 of this patch in the series.Suggested-by Stephen Hemminger <[email protected]>Signed-off-by Morten Brørup <[email protected]>--- lib/mempool/rte_mempool.c | 2 +- lib/mempool/rte_mempool.h | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-)lib/mempool/rte_mempool.c +0
-0@@ -1272,7 +1272,7 @@ rte_mempool_dump(FILE *f, struct rte_mempool*mp)
Quoted 14 lines
lib/mempool/rte_mempool.c +1
-1#ifdef RTE_LIBRTE_MEMPOOL_STATS rte_mempool_ops_get_info (mp, &info);memset (&sum,0 ,sizeof (sum)); -for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) { +for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE +1 ; lcore_id++) { sum.put_bulk += mp->stats[lcore_id].put_bulk; sum.put_objs += mp->stats[lcore_id].put_objs; sum.put_common_pool_bulk += mp-stats[lcore_id].put_common_pool_bulk;
lib/mempool/rte_mempool.h +0
-0lib/mempool/rte_mempool.h struct rte_mempool { · line 238
Quoted 1 line
lib/mempool/rte_mempool.h +0
-0@@ -238,8 +238,11 @@ struct rte_mempool {struct rte_mempool_memhdr_list mem_list;/**< List of memory chunks */
Quoted 12 lines
lib/mempool/rte_mempool.h +5
-2#ifdef RTE_LIBRTE_MEMPOOL_STATS -/** Per-lcore statistics. */ -struct rte_mempool_debug_stats stats[RTE_MAX_LCORE ]; +/** Per-lcore statistics. +* +* Offset by one, to include non-DPDK threads. +*/ +struct rte_mempool_debug_stats stats[RTE_MAX_LCORE +1 ];#endif } __rte_cache_aligned;
Quoted 13 lines
lib/mempool/rte_mempool.h struct rte_mempool { · line 310
Quoted 9 lines
lib/mempool/rte_mempool.h +1
-4@@ -304,10 +307,7 @@ struct rte_mempool {*/ #ifdef RTE_LIBRTE_MEMPOOL_STATS #define RTE_MEMPOOL_STAT_ADD (mp, name, n) do { \ - unsigned __lcore_id = rte_lcore_id(); \ - if (__lcore_id < RTE_MAX_LCORE) { \ - mp->stats[__lcore_id].name += n; \ - } \ + (mp)->stats[rte_lcore_id() + 1].name += n; \This relies on
LCORE_ID_ANYbeingUINT32_MAX, and a wrap to 0, for an unregistered non-EAL thread? Might be worth a comment, or better a rewrite with an explicitLCORE_ID_ANYcomparison.
The purpose of this patch is to avoid the comparison here.
Yes, it relies on the wrap to zero, and these conditions:
1. LCORE_ID_ANY being UINT32_MAX, and
2. the return type of rte_lcore_id() being unsigned int, and
3. unsigned int being uint32_t.
When I wrote this, I considered it safe to assume that LCORE_ID_ANY will remain the unsigned equivalent of -1 using the return type of rte_lcore_id(). In other words: If the return type of rte_lcore_id() should change from 32 to 64 bit, LCORE_ID_ANY would be updated accordingly to UINT64_MAX.
Because of this assumption, I didn't use [(rte_lcore_id() + 1) & UINT32_MAX], but just [rte_lcore_id() + 1].
I struggled writing an appropriate comment without making it unacceptably long, but eventually gave up, and settled for the one-line comment in the structure only.
Quoted 3 lines
You anyways need a conditional. An atomic add must be used in the unregistered EAL thread case.
Good point: The "+= n" must be atomic for non-isolated threads.
I just took a look at how software maintained stats are handled elsewhere, and the first thing I found, is the IOAT DMA driver, which also seems to be using non-atomic increment [1] regardless if used by a DPDK thread or not.
[1]: https://elixir.bootlin.com/dpdk/v22.11-rc2/source/drivers/dma/ioat/ioat_dmadev.c#L228
However, doing it wrong elsewhere doesn't make it correct doing it wrong here too. :-)
Atomic increments are costly, so I would rather follow your suggestion and reintroduce the comparison. How about:
#define RTE_MEMPOOL_STAT_ADD(mp, name, n) do { \
unsigned int __lcore_id = rte_lcore_id(); \
if (likely(__lcore_id < RTE_MAX_LCORE)) { \
(mp)->stats[__lcore_id].name += n; \
} else {
rte_atomic64_add( \
(rte_atomic64_t*)&((mp)->stats[RTE_MAX_LCORE].name), n);\
} \
}And the structure comment could be:
* Plus one, for non-DPDK threads.
Re: [PATCH v2 2/3] mempool: include non-DPDK threads in statistics
Quoted 109 lines · On Wed, 2 Nov 2022 10:09:26 +0100 Morten Brørup <[email protected]>…
Quoted 58 lines
From: Mattias Rönnblom [mailto:[email protected]]
Sent: Wednesday, 2 November 2022 08.53Quoted 1 line · Morten Brørup
Offset the stats array index by one, and count non-DPDK threads at
index
Quoted 5 lines
zero.
This patch provides two benefits:
* Non-DPDK threads are also included in the statistics.
* A conditional in the fast path is removed. Static branch predictionwas
lib/mempool/rte_mempool.c rte_mempool_dump(FILE *f, struct rte_mempool
Quoted 17 lines
correct, so the performance improvement is negligible.
v2:
* New. No v1 of this patch in the series.Suggested-by Stephen Hemminger <[email protected]>Signed-off-by Morten Brørup <[email protected]>--- lib/mempool/rte_mempool.c | 2 +- lib/mempool/rte_mempool.h | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-)lib/mempool/rte_mempool.c +0
-0@@ -1272,7 +1272,7 @@ rte_mempool_dump(FILE *f, struct rte_mempool*mp)
Quoted 14 lines
lib/mempool/rte_mempool.c +1
-1#ifdef RTE_LIBRTE_MEMPOOL_STATS rte_mempool_ops_get_info (mp, &info);memset (&sum,0 ,sizeof (sum)); -for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) { +for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE +1 ; lcore_id++) { sum.put_bulk += mp->stats[lcore_id].put_bulk; sum.put_objs += mp->stats[lcore_id].put_objs; sum.put_common_pool_bulk += mp-stats[lcore_id].put_common_pool_bulk;
lib/mempool/rte_mempool.h +0
-0lib/mempool/rte_mempool.h struct rte_mempool { · line 238
Quoted 1 line
lib/mempool/rte_mempool.h +0
-0@@ -238,8 +238,11 @@ struct rte_mempool {struct rte_mempool_memhdr_list mem_list;/**< List of memory chunks */
Quoted 12 lines
lib/mempool/rte_mempool.h +5
-2#ifdef RTE_LIBRTE_MEMPOOL_STATS -/** Per-lcore statistics. */ -struct rte_mempool_debug_stats stats[RTE_MAX_LCORE ]; +/** Per-lcore statistics. +* +* Offset by one, to include non-DPDK threads. +*/ +struct rte_mempool_debug_stats stats[RTE_MAX_LCORE +1 ];#endif } __rte_cache_aligned;Quoted 13 lines
lib/mempool/rte_mempool.h struct rte_mempool { · line 310
Quoted 9 lines
lib/mempool/rte_mempool.h +1
-4@@ -304,10 +307,7 @@ struct rte_mempool {*/ #ifdef RTE_LIBRTE_MEMPOOL_STATS #define RTE_MEMPOOL_STAT_ADD (mp, name, n) do { \ - unsigned __lcore_id = rte_lcore_id(); \ - if (__lcore_id < RTE_MAX_LCORE) { \ - mp->stats[__lcore_id].name += n; \ - } \ + (mp)->stats[rte_lcore_id() + 1].name += n; \This relies on
LCORE_ID_ANYbeingUINT32_MAX, and a wrap to 0, for an unregistered non-EAL thread? Might be worth a comment, or better a rewrite with an explicitLCORE_ID_ANYcomparison.The purpose of this patch is to avoid the comparison here.
Yes, it relies on the wrap to zero, and these conditions:
1.LCORE_ID_ANYbeingUINT32_MAX, and
2. the return type ofrte_lcore_id()being unsigned int, and
3. unsigned int beinguint32_t.When I wrote this, I considered it safe to assume that
LCORE_ID_ANYwill remain the unsigned equivalent of -1 using the return type ofrte_lcore_id(). In other words: If the return type ofrte_lcore_id()should change from 32 to 64 bit,LCORE_ID_ANYwould be updated accordingly toUINT64_MAX.Because of this assumption, I didn't use [(
rte_lcore_id()+ 1) &UINT32_MAX], but just [rte_lcore_id()+ 1].I struggled writing an appropriate comment without making it unacceptably long, but eventually gave up, and settled for the one-line comment in the structure only.
Quoted 3 lines
You anyways need a conditional. An atomic add must be used in the unregistered EAL thread case.
Good point: The "+= n" must be atomic for non-isolated threads.
I just took a look at how software maintained stats are handled elsewhere, and the first thing I found, is the IOAT DMA driver, which also seems to be using non-atomic increment [1] regardless if used by a DPDK thread or not.
[1]: https://elixir.bootlin.com/dpdk/v22.11-rc2/source/drivers/dma/ioat/ioat_dmadev.c#L228
However, doing it wrong elsewhere doesn't make it correct doing it wrong here too. :-)
Atomic increments are costly, so I would rather follow your suggestion and reintroduce the comparison. How about:
#define RTE_MEMPOOL_STAT_ADD(mp, name, n) do { \ unsigned int __lcore_id = rte_lcore_id(); \ if (likely(__lcore_id < RTE_MAX_LCORE)) { \ (mp)->stats[__lcore_id].name += n; \ } else { rte_atomic64_add( \ (rte_atomic64_t*)&((mp)->stats[RTE_MAX_LCORE].name), n);\ } \ }
Quoted 3 lines
And the structure comment could be:
* Plus one, for non-DPDK threads.
Or use rte_lcore_index() instead of rte_lcore_id()
RE: [PATCH v2 2/3] mempool: include non-DPDK threads in statistics
Med venlig hilsen / Kind regards,
-Morten Brørup
Quoted 142 lines
-----Original Message-----
From: Stephen Hemminger [mailto:[email protected]]
Sent: Wednesday, 2 November 2022 16.19
To: Morten Brørup
Cc: Mattias Rönnblom; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: Re: [PATCH v2 2/3] mempool: include non-DPDK threads in statisticsQuoted 5 lines · On Wed, 2 Nov 2022 10:09:26 +0100 Morten Brørup <[email protected]>…
Quoted 5 lines
From: Mattias Rönnblom [mailto:[email protected]]
Sent: Wednesday, 2 November 2022 08.53Quoted 1 line · Morten Brørup
Offset the stats array index by one, and count non-DPDK threads
at
Quoted 6 lines
Quoted 6 lines
index
Quoted 5 lines
zero.
This patch provides two benefits:
* Non-DPDK threads are also included in the statistics.
* A conditional in the fast path is removed. Static branchprediction
Quoted 6 lines
Quoted 6 lines
was
Quoted 5 lines
correct, so the performance improvement is negligible.
v2:
* New. No v1 of this patch in the series.Quoted 8 lines
Quoted 8 lines
Review
Quoted 8 lines
Suggested-by Stephen Hemminger <[email protected]>Signed-off-by Morten Brørup <[email protected]>--- lib/mempool/rte_mempool.c | 2 +- lib/mempool/rte_mempool.h | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-)(quoted diff) +0
-0
b/lib/mempool/rte_mempool.cQuoted 4 lines
Quoted 4 lines
lib/mempool/rte_mempool.c rte_mempool_dump(FILE *f, struct
Quoted 4 lines
index
62d1ce764e..e6208125e0100644lib/mempool/rte_mempool.c +0
-0@@ -1272,7 +1272,7 @@ rte_mempool_dump(FILE *f, struct
rte_mempoolQuoted 6 lines
Quoted 6 lines
*mp)
lib/mempool/rte_mempool.c rte_mempool_dump(FILE *f, struct rte_mempool *mp)
Quoted 5 lines
lib/mempool/rte_mempool.c +1
-1#ifdef RTE_LIBRTE_MEMPOOL_STATS rte_mempool_ops_get_info (mp, &info);memset (&sum,0 ,sizeof (sum)); -for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) { + for (lcore_id = 0; lcore_id < RTE_MAX_LCORE + 1;lcore_id++) {
Quoted 5 lines
Quoted 5 lines
Review
Quoted 5 lines
lib/mempool/rte_mempool.c +0
-0sum.put_bulk += mp->stats[lcore_id].put_bulk; sum.put_objs += mp->stats[lcore_id].put_objs; sum.put_common_pool_bulk += mp-stats[lcore_id].put_common_pool_bulk;
(quoted diff) +0
-0
b/lib/mempool/rte_mempool.hQuoted 5 lines
Quoted 5 lines
lib/mempool/rte_mempool.h struct rte_mempool { · line 238
Quoted 5 lines
index
9c4bf5549f..16e7e62e3c100644lib/mempool/rte_mempool.h +0
-0@@ -238,8 +238,11 @@ struct rte_mempool {struct rte_mempool_memhdr_list mem_list;/**< List of memory
Quoted 13 lines
Quoted 13 lines
chunks */
Quoted 12 lines
lib/mempool/rte_mempool.h +5
-2#ifdef RTE_LIBRTE_MEMPOOL_STATS -/** Per-lcore statistics. */ -struct rte_mempool_debug_stats stats[RTE_MAX_LCORE ]; +/** Per-lcore statistics. +* +* Offset by one, to include non-DPDK threads. +*/ +struct rte_mempool_debug_stats stats[RTE_MAX_LCORE +1 ];#endif } __rte_cache_aligned;Quoted 4 lines
Quoted 4 lines
lib/mempool/rte_mempool.h struct rte_mempool { · line 309
Quoted 4 lines
lib/mempool/rte_mempool.h +0
-0@@ -304,10 +307,7 @@ struct rte_mempool {*/ #ifdef RTE_LIBRTE_MEMPOOL_STATS #define RTE_MEMPOOL_STAT_ADD (mp, name, n) do {\
Quoted 7 lines
Quoted 7 lines
lib/mempool/rte_mempool.h struct rte_mempool { · line 310
Quoted 5 lines
lib/mempool/rte_mempool.h +1
-4-unsigned __lcore_id =rte_lcore_id (); \ -if (__lcore_id <RTE_MAX_LCORE ) { \ - mp->stats[__lcore_id].name += n; \ - } \ + (mp)->stats[rte_lcore_id () +1 ].name += n; \This relies on
LCORE_ID_ANYbeingUINT32_MAX, and a wrap to 0, foran
Quoted 11 lines
Quoted 2 lines
unregistered non-EAL thread? Might be worth a comment, or better a rewrite with an explicit
LCORE_ID_ANYcomparison.The purpose of this patch is to avoid the comparison here.
Yes, it relies on the wrap to zero, and these conditions:
1.LCORE_ID_ANYbeingUINT32_MAX, and
2. the return type ofrte_lcore_id()being unsigned int, and
3. unsigned int beinguint32_t.When I wrote this, I considered it safe to assume that
LCORE_ID_ANYwill remain the unsigned equivalent of -1 using the return type of
rte_lcore_id(). In other words: If the return type ofrte_lcore_id()should change from 32 to 64 bit,LCORE_ID_ANYwould be updated accordingly toUINT64_MAX.Quoted 2 lines
Because of this assumption, I didn't use [(
rte_lcore_id()+ 1) &
UINT32_MAX], but just [rte_lcore_id()+ 1].Quoted 2 lines
I struggled writing an appropriate comment without making it
unacceptably long, but eventually gave up, and settled for the one-line comment in the structure only.
Quoted 8 lines
Quoted 4 lines
You anyways need a conditional. An atomic add must be used in the unregistered EAL thread case.
Good point: The "+= n" must be atomic for non-isolated threads.
I just took a look at how software maintained stats are handled
elsewhere, and the first thing I found, is the IOAT DMA driver, which also seems to be using non-atomic increment [1] regardless if used by a DPDK thread or not.
Quoted 2 lines
rc2/source/drivers/dma/ioat/ioat_dmadev.c#L228
Quoted 2 lines
However, doing it wrong elsewhere doesn't make it correct doing it
wrong here too. :-)
Quoted 2 lines
Atomic increments are costly, so I would rather follow your
suggestion and reintroduce the comparison. How about:
Quoted 8 lines
#define RTE_MEMPOOL_STAT_ADD(mp, name, n) do { \ unsigned int __lcore_id = rte_lcore_id(); \ if (likely(__lcore_id < RTE_MAX_LCORE)) { \ (mp)->stats[__lcore_id].name += n; \ } else { rte_atomic64_add( \ (rte_atomic64_t*)&((mp)->stats[RTE_MAX_LCORE].name),n);\
Quoted 9 lines
Quoted 6 lines
} \ }And the structure comment could be:
* Plus one, for non-DPDK threads.Or use
rte_lcore_index()instead ofrte_lcore_id()
rte_lcore_index() is not inline, so it would impact performance. And looking at its source code [2], it compares and branches anyway. It would effectively just be a longwinded way of converting the rte_lcore_id() return type from unsigned to signed int.
[2]: https://elixir.bootlin.com/dpdk/v22.11-rc2/source/lib/eal/common/eal_common_lcore.c#L27
Re: [PATCH v2 2/3] mempool: include non-DPDK threads in statistics
In reply to Morten BrørupQuoted 83 lines · Morten Brørup
Quoted 58 lines
From: Mattias Rönnblom [mailto:[email protected]]
Sent: Wednesday, 2 November 2022 08.53Quoted 1 line · Morten Brørup
Offset the stats array index by one, and count non-DPDK threads at
index
Quoted 5 lines
zero.
This patch provides two benefits:
* Non-DPDK threads are also included in the statistics.
* A conditional in the fast path is removed. Static branch predictionwas
lib/mempool/rte_mempool.c rte_mempool_dump(FILE *f, struct rte_mempool
Quoted 17 lines
correct, so the performance improvement is negligible.
v2:
* New. No v1 of this patch in the series.Suggested-by Stephen Hemminger <[email protected]>Signed-off-by Morten Brørup <[email protected]>--- lib/mempool/rte_mempool.c | 2 +- lib/mempool/rte_mempool.h | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-)lib/mempool/rte_mempool.c +0
-0@@ -1272,7 +1272,7 @@ rte_mempool_dump(FILE *f, struct rte_mempool*mp)
Quoted 14 lines
lib/mempool/rte_mempool.c +1
-1#ifdef RTE_LIBRTE_MEMPOOL_STATS rte_mempool_ops_get_info (mp, &info);memset (&sum,0 ,sizeof (sum)); -for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) { +for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE +1 ; lcore_id++) { sum.put_bulk += mp->stats[lcore_id].put_bulk; sum.put_objs += mp->stats[lcore_id].put_objs; sum.put_common_pool_bulk += mp-stats[lcore_id].put_common_pool_bulk;
lib/mempool/rte_mempool.h +0
-0lib/mempool/rte_mempool.h struct rte_mempool { · line 238
Quoted 1 line
lib/mempool/rte_mempool.h +0
-0@@ -238,8 +238,11 @@ struct rte_mempool {struct rte_mempool_memhdr_list mem_list;/**< List of memory chunks */
Quoted 12 lines
lib/mempool/rte_mempool.h +5
-2#ifdef RTE_LIBRTE_MEMPOOL_STATS -/** Per-lcore statistics. */ -struct rte_mempool_debug_stats stats[RTE_MAX_LCORE ]; +/** Per-lcore statistics. +* +* Offset by one, to include non-DPDK threads. +*/ +struct rte_mempool_debug_stats stats[RTE_MAX_LCORE +1 ];#endif } __rte_cache_aligned;Quoted 13 lines
lib/mempool/rte_mempool.h struct rte_mempool { · line 310
Quoted 9 lines
lib/mempool/rte_mempool.h +1
-4@@ -304,10 +307,7 @@ struct rte_mempool {*/ #ifdef RTE_LIBRTE_MEMPOOL_STATS #define RTE_MEMPOOL_STAT_ADD (mp, name, n) do { \ - unsigned __lcore_id = rte_lcore_id(); \ - if (__lcore_id < RTE_MAX_LCORE) { \ - mp->stats[__lcore_id].name += n; \ - } \ + (mp)->stats[rte_lcore_id() + 1].name += n; \This relies on
LCORE_ID_ANYbeingUINT32_MAX, and a wrap to 0, for an unregistered non-EAL thread? Might be worth a comment, or better a rewrite with an explicitLCORE_ID_ANYcomparison.The purpose of this patch is to avoid the comparison here.
Yes, it relies on the wrap to zero, and these conditions:
1.LCORE_ID_ANYbeingUINT32_MAX, and
2. the return type ofrte_lcore_id()being unsigned int, and
3. unsigned int beinguint32_t.When I wrote this, I considered it safe to assume that
LCORE_ID_ANYwill remain the unsigned equivalent of -1 using the return type ofrte_lcore_id(). In other words: If the return type ofrte_lcore_id()should change from 32 to 64 bit,LCORE_ID_ANYwould be updated accordingly toUINT64_MAX.Because of this assumption, I didn't use [(
rte_lcore_id()+ 1) &UINT32_MAX], but just [rte_lcore_id()+ 1].
Quoted 8 lines
I struggled writing an appropriate comment without making it unacceptably long, but eventually gave up, and settled for the one-line comment in the structure only.
Quoted 3 lines
You anyways need a conditional. An atomic add must be used in the unregistered EAL thread case.
Good point: The "+= n" must be atomic for non-isolated threads.
If the various unregistered non-EAL threads are run on isolated cores or not does not matter.
Quoted 8 lines
I just took a look at how software maintained stats are handled elsewhere, and the first thing I found, is the IOAT DMA driver, which also seems to be using non-atomic increment [1] regardless if used by a DPDK thread or not.
[1]: https://elixir.bootlin.com/dpdk/v22.11-rc2/source/drivers/dma/ioat/ioat_dmadev.c#L228
However, doing it wrong elsewhere doesn't make it correct doing it wrong here too. :-)
Atomic increments are costly, so I would rather follow your suggestion and reintroduce the comparison. How about:
Quoted 9 lines
#define RTE_MEMPOOL_STAT_ADD(mp, name, n) do { \
unsigned int __lcore_id = rte_lcore_id(); \
if (likely(__lcore_id < RTE_MAX_LCORE)) { \
(mp)->stats[__lcore_id].name += n; \
} else {
rte_atomic64_add( \
(rte_atomic64_t*)&((mp)->stats[RTE_MAX_LCORE].name), n);\
} \
}You are supposed to use GCC C11 intrinsics (e.g., __atomic_fetch_add()).
In addition: technically, you must use an atomic store for the EAL thread case (and an atomic load on the reader side), although there are tons of examples in DPDK where tearing is ignored. (The generated code will likely look the same.)
DPDK coding conventions require there be no braces for a single statement.
Other than that, it looks good.
Quoted 4 lines
And the structure comment could be:
* Plus one, for non-DPDK threads.
"Unregistered non-EAL threads". This is the term the EAL documentation uses.
RE: [PATCH v2 2/3] mempool: include non-DPDK threads in statistics
Quoted 101 lines
From: Mattias Rönnblom [mailto:[email protected]]
Sent: Wednesday, 2 November 2022 18.53Quoted 11 lines · Morten Brørup
Quoted 11 lines
From: Mattias Rönnblom [mailto:[email protected]]
Sent: Wednesday, 2 November 2022 08.53Quoted 1 line · Morten Brørup
Offset the stats array index by one, and count non-DPDK threads at
index
Quoted 5 lines
zero.
This patch provides two benefits:
* Non-DPDK threads are also included in the statistics.
* A conditional in the fast path is removed. Static branchprediction
Quoted 19 lines
Quoted 19 lines
was
lib/mempool/rte_mempool.c rte_mempool_dump(FILE *f, struct rte_mempool
Quoted 17 lines
correct, so the performance improvement is negligible.
v2:
* New. No v1 of this patch in the series.Suggested-by Stephen Hemminger <[email protected]>Signed-off-by Morten Brørup <[email protected]>--- lib/mempool/rte_mempool.c | 2 +- lib/mempool/rte_mempool.h | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-)lib/mempool/rte_mempool.c +0
-0@@ -1272,7 +1272,7 @@ rte_mempool_dump(FILE *f, struct rte_mempool*mp)
Quoted 15 lines
Quoted 7 lines
Quoted 7 lines
lib/mempool/rte_mempool.c +1
-1#ifdef RTE_LIBRTE_MEMPOOL_STATS rte_mempool_ops_get_info (mp, &info);memset (&sum,0 ,sizeof (sum)); -for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) { +for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE +1 ; lcore_id++) { sum.put_bulk += mp->stats[lcore_id].put_bulk; sum.put_objs += mp->stats[lcore_id].put_objs;Quoted 8 lines
lib/mempool/rte_mempool.h struct rte_mempool { · line 238
Quoted 8 lines
lib/mempool/rte_mempool.c +0
-0sum.put_common_pool_bulk += mp-stats[lcore_id].put_common_pool_bulk;
lib/mempool/rte_mempool.h +0
-0@@ -238,8 +238,11 @@ struct rte_mempool {struct rte_mempool_memhdr_list mem_list;/**< List of memory
Quoted 13 lines
Quoted 13 lines
chunks */
Quoted 12 lines
lib/mempool/rte_mempool.h +5
-2#ifdef RTE_LIBRTE_MEMPOOL_STATS -/** Per-lcore statistics. */ -struct rte_mempool_debug_stats stats[RTE_MAX_LCORE ]; +/** Per-lcore statistics. +* +* Offset by one, to include non-DPDK threads. +*/ +struct rte_mempool_debug_stats stats[RTE_MAX_LCORE +1 ];#endif } __rte_cache_aligned;Quoted 4 lines
Quoted 4 lines
lib/mempool/rte_mempool.h struct rte_mempool { · line 309
Quoted 4 lines
lib/mempool/rte_mempool.h +0
-0@@ -304,10 +307,7 @@ struct rte_mempool {*/ #ifdef RTE_LIBRTE_MEMPOOL_STATS #define RTE_MEMPOOL_STAT_ADD (mp, name, n) do {\
Quoted 7 lines
Quoted 7 lines
lib/mempool/rte_mempool.h struct rte_mempool { · line 310
Quoted 5 lines
lib/mempool/rte_mempool.h +1
-4-unsigned __lcore_id =rte_lcore_id (); \ -if (__lcore_id <RTE_MAX_LCORE ) { \ - mp->stats[__lcore_id].name += n; \ - } \ + (mp)->stats[rte_lcore_id () +1 ].name += n; \This relies on
LCORE_ID_ANYbeingUINT32_MAX, and a wrap to 0, foran
Quoted 11 lines
Quoted 2 lines
unregistered non-EAL thread? Might be worth a comment, or better a rewrite with an explicit
LCORE_ID_ANYcomparison.The purpose of this patch is to avoid the comparison here.
Yes, it relies on the wrap to zero, and these conditions:
1.LCORE_ID_ANYbeingUINT32_MAX, and
2. the return type ofrte_lcore_id()being unsigned int, and
3. unsigned int beinguint32_t.When I wrote this, I considered it safe to assume that
LCORE_ID_ANYwill remain the unsigned equivalent of -1 using the return type of
rte_lcore_id(). In other words: If the return type ofrte_lcore_id()should change from 32 to 64 bit,LCORE_ID_ANYwould be updated accordingly toUINT64_MAX.Quoted 2 lines
Because of this assumption, I didn't use [(
rte_lcore_id()+ 1) &
UINT32_MAX], but just [rte_lcore_id()+ 1].Quoted 2 lines
I struggled writing an appropriate comment without making it
unacceptably long, but eventually gave up, and settled for the one-line comment in the structure only.
Quoted 2 lines
Quoted 2 lines
Quoted 8 lines
Quoted 5 lines
Quoted 2 lines
You anyways need a conditional. An atomic add must be used in the unregistered EAL thread case.
Good point: The "+= n" must be atomic for non-isolated threads.
If the various unregistered non-EAL threads are run on isolated cores or not does not matter.
Agree: They all share the same index, and thus may race, regardless which cores they are using.
Rephrasing: The "+= n" must be atomic for the unregistered non-EAL threads.
Quoted 22 lines
Quoted 2 lines
I just took a look at how software maintained stats are handled
elsewhere, and the first thing I found, is the IOAT DMA driver, which also seems to be using non-atomic increment [1] regardless if used by a DPDK thread or not.
Quoted 2 lines
rc2/source/drivers/dma/ioat/ioat_dmadev.c#L228
Quoted 2 lines
However, doing it wrong elsewhere doesn't make it correct doing it
wrong here too. :-)
Quoted 2 lines
Atomic increments are costly, so I would rather follow your
suggestion and reintroduce the comparison. How about:
Quoted 8 lines
#define RTE_MEMPOOL_STAT_ADD(mp, name, n) do { \ unsigned int __lcore_id = rte_lcore_id(); \ if (likely(__lcore_id < RTE_MAX_LCORE)) { \ (mp)->stats[__lcore_id].name += n; \ } else { rte_atomic64_add( \ (rte_atomic64_t*)&((mp)->stats[RTE_MAX_LCORE].name),
Quoted 5 lines
n);\
Quoted 2 lines
} \ }You are supposed to use GCC C11 intrinsics (e.g.,
__atomic_fetch_add()).
Ups. I forgot!
This should be emphasized everywhere in the rte_atomic library, to prevent such mistakes.
Quoted 5 lines
In addition: technically, you must use an atomic store for the EAL thread case (and an atomic load on the reader side), although there are tons of examples in DPDK where tearing is ignored. (The generated code will likely look the same.)
The counters are 64 bit aligned, but tearing could happen on 32 bit architectures.
My initial reaction to your comment was to do it correctly on the EAL threads too, to avoid tearing there too. However, there might be a performance cost for 32 bit architectures, so I will consider that these are only debug counters, and accept the risk of tearing.
Quoted 3 lines
DPDK coding conventions require there be no braces for a single statement.
Will fix.
Quoted 10 lines
Other than that, it looks good.
Quoted 4 lines
And the structure comment could be:
* Plus one, for non-DPDK threads."Unregistered non-EAL threads". This is the term the EAL documentation uses.
Thank you for clarifying. I didn't follow that discussion, so the new terminology for threads hasn't stuck with me yet. :-)
Re: [PATCH v2 2/3] mempool: include non-DPDK threads in statistics
Quoted 147 lines · Morten Brørup
Quoted 101 lines
From: Mattias Rönnblom [mailto:[email protected]]
Sent: Wednesday, 2 November 2022 18.53Quoted 11 lines · Morten Brørup
Quoted 11 lines
From: Mattias Rönnblom [mailto:[email protected]]
Sent: Wednesday, 2 November 2022 08.53Quoted 1 line · Morten Brørup
Offset the stats array index by one, and count non-DPDK threads at
index
Quoted 5 lines
zero.
This patch provides two benefits:
* Non-DPDK threads are also included in the statistics.
* A conditional in the fast path is removed. Static branchprediction
Quoted 19 lines
Quoted 19 lines
was
lib/mempool/rte_mempool.c rte_mempool_dump(FILE *f, struct rte_mempool
Quoted 17 lines
correct, so the performance improvement is negligible.
v2:
* New. No v1 of this patch in the series.Suggested-by Stephen Hemminger <[email protected]>Signed-off-by Morten Brørup <[email protected]>--- lib/mempool/rte_mempool.c | 2 +- lib/mempool/rte_mempool.h | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-)lib/mempool/rte_mempool.c +0
-0@@ -1272,7 +1272,7 @@ rte_mempool_dump(FILE *f, struct rte_mempool*mp)
Quoted 15 lines
Quoted 7 lines
Quoted 7 lines
lib/mempool/rte_mempool.c +1
-1#ifdef RTE_LIBRTE_MEMPOOL_STATS rte_mempool_ops_get_info (mp, &info);memset (&sum,0 ,sizeof (sum)); -for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) { +for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE +1 ; lcore_id++) { sum.put_bulk += mp->stats[lcore_id].put_bulk; sum.put_objs += mp->stats[lcore_id].put_objs;Quoted 8 lines
lib/mempool/rte_mempool.h struct rte_mempool { · line 238
Quoted 8 lines
lib/mempool/rte_mempool.c +0
-0sum.put_common_pool_bulk += mp-stats[lcore_id].put_common_pool_bulk;
lib/mempool/rte_mempool.h +0
-0@@ -238,8 +238,11 @@ struct rte_mempool {struct rte_mempool_memhdr_list mem_list;/**< List of memory
Quoted 13 lines
Quoted 13 lines
chunks */
Quoted 12 lines
lib/mempool/rte_mempool.h +5
-2#ifdef RTE_LIBRTE_MEMPOOL_STATS -/** Per-lcore statistics. */ -struct rte_mempool_debug_stats stats[RTE_MAX_LCORE ]; +/** Per-lcore statistics. +* +* Offset by one, to include non-DPDK threads. +*/ +struct rte_mempool_debug_stats stats[RTE_MAX_LCORE +1 ];#endif } __rte_cache_aligned;Quoted 4 lines
Quoted 4 lines
lib/mempool/rte_mempool.h struct rte_mempool { · line 309
Quoted 4 lines
lib/mempool/rte_mempool.h +0
-0@@ -304,10 +307,7 @@ struct rte_mempool {*/ #ifdef RTE_LIBRTE_MEMPOOL_STATS #define RTE_MEMPOOL_STAT_ADD (mp, name, n) do {\
Quoted 7 lines
Quoted 7 lines
lib/mempool/rte_mempool.h struct rte_mempool { · line 310
Quoted 5 lines
lib/mempool/rte_mempool.h +1
-4-unsigned __lcore_id =rte_lcore_id (); \ -if (__lcore_id <RTE_MAX_LCORE ) { \ - mp->stats[__lcore_id].name += n; \ - } \ + (mp)->stats[rte_lcore_id () +1 ].name += n; \This relies on
LCORE_ID_ANYbeingUINT32_MAX, and a wrap to 0, foran
Quoted 11 lines
Quoted 2 lines
unregistered non-EAL thread? Might be worth a comment, or better a rewrite with an explicit
LCORE_ID_ANYcomparison.The purpose of this patch is to avoid the comparison here.
Yes, it relies on the wrap to zero, and these conditions:
1.LCORE_ID_ANYbeingUINT32_MAX, and
2. the return type ofrte_lcore_id()being unsigned int, and
3. unsigned int beinguint32_t.When I wrote this, I considered it safe to assume that
LCORE_ID_ANYwill remain the unsigned equivalent of -1 using the return type of
rte_lcore_id(). In other words: If the return type ofrte_lcore_id()should change from 32 to 64 bit,LCORE_ID_ANYwould be updated accordingly toUINT64_MAX.Quoted 2 lines
Because of this assumption, I didn't use [(
rte_lcore_id()+ 1) &
UINT32_MAX], but just [rte_lcore_id()+ 1].Quoted 2 lines
I struggled writing an appropriate comment without making it
unacceptably long, but eventually gave up, and settled for the one-line comment in the structure only.
Quoted 2 lines
Quoted 2 lines
Quoted 8 lines
Quoted 5 lines
Quoted 2 lines
You anyways need a conditional. An atomic add must be used in the unregistered EAL thread case.
Good point: The "+= n" must be atomic for non-isolated threads.
If the various unregistered non-EAL threads are run on isolated cores or not does not matter.
Agree: They all share the same index, and thus may race, regardless which cores they are using.
Rephrasing: The "+= n" must be atomic for the unregistered non-EAL threads.
Quoted 22 lines
Quoted 2 lines
I just took a look at how software maintained stats are handled
elsewhere, and the first thing I found, is the IOAT DMA driver, which also seems to be using non-atomic increment [1] regardless if used by a DPDK thread or not.
Quoted 2 lines
rc2/source/drivers/dma/ioat/ioat_dmadev.c#L228
Quoted 2 lines
However, doing it wrong elsewhere doesn't make it correct doing it
wrong here too. :-)
Quoted 2 lines
Atomic increments are costly, so I would rather follow your
suggestion and reintroduce the comparison. How about:
Quoted 8 lines
#define RTE_MEMPOOL_STAT_ADD(mp, name, n) do { \ unsigned int __lcore_id = rte_lcore_id(); \ if (likely(__lcore_id < RTE_MAX_LCORE)) { \ (mp)->stats[__lcore_id].name += n; \ } else { rte_atomic64_add( \ (rte_atomic64_t*)&((mp)->stats[RTE_MAX_LCORE].name),Quoted 5 lines
n);\
Quoted 2 lines
} \ }You are supposed to use GCC C11 intrinsics (e.g.,
__atomic_fetch_add()).Ups. I forgot!
This should be emphasized everywhere in the
rte_atomiclibrary, to prevent such mistakes.Quoted 1 line
Quoted 7 lines
Quoted 4 lines
In addition: technically, you must use an atomic store for the EAL thread case (and an atomic load on the reader side), although there are tons of examples in DPDK where tearing is ignored. (The generated code will likely look the same.)
The counters are 64 bit aligned, but tearing could happen on 32 bit architectures.
The compiler is free to do it on any architecture, but I'm not sure if it happens much in practice.
Quoted 2 lines
My initial reaction to your comment was to do it correctly on the EAL threads too, to avoid tearing there too. However, there might be a performance cost for 32 bit architectures, so I will consider that these are only debug counters, and accept the risk of tearing.
What would that cost consist of?
In theory C11 atomics could be implemented "in software" (i.e., without the proper ISA-level guarantees, with locks), but does any of the DPDK-supported compiler/32-bit ISAs actually do so?
It's also not obvious that if there, for a certain compiler/ISA-combination, is a performance impact to pay for correctness, correctness would have to give way.
Quoted 19 lines
Quoted 3 lines
DPDK coding conventions require there be no braces for a single statement.
Will fix.
Quoted 10 lines
Other than that, it looks good.
Quoted 4 lines
And the structure comment could be:
* Plus one, for non-DPDK threads."Unregistered non-EAL threads". This is the term the EAL documentation uses.
Thank you for clarifying. I didn't follow that discussion, so the new terminology for threads hasn't stuck with me yet. :-)
RE: [PATCH v2 2/3] mempool: include non-DPDK threads in statistics
Quoted 178 lines
From: Mattias Rönnblom [mailto:[email protected]]
Sent: Friday, 4 November 2022 09.59Quoted 9 lines · Morten Brørup
Quoted 9 lines
From: Mattias Rönnblom [mailto:[email protected]]
Sent: Wednesday, 2 November 2022 18.53Quoted 5 lines · Morten Brørup
Quoted 5 lines
From: Mattias Rönnblom [mailto:[email protected]]
Sent: Wednesday, 2 November 2022 08.53Quoted 1 line · Morten Brørup
Offset the stats array index by one, and count non-DPDK threads
at
Quoted 13 lines
Quoted 13 lines
Quoted 6 lines
Quoted 6 lines
index
Quoted 5 lines
zero.
This patch provides two benefits:
* Non-DPDK threads are also included in the statistics.
* A conditional in the fast path is removed. Static branchprediction
Quoted 6 lines
Quoted 6 lines
was
Quoted 5 lines
correct, so the performance improvement is negligible.
v2:
* New. No v1 of this patch in the series.Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Review
Quoted 8 lines
Suggested-by Stephen Hemminger <[email protected]>Signed-off-by Morten Brørup <[email protected]>--- lib/mempool/rte_mempool.c | 2 +- lib/mempool/rte_mempool.h | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-)(quoted diff) +0
-0
b/lib/mempool/rte_mempool.cQuoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
lib/mempool/rte_mempool.c rte_mempool_dump(FILE *f, struct
Quoted 4 lines
index
62d1ce764e..e6208125e0100644lib/mempool/rte_mempool.c +0
-0@@ -1272,7 +1272,7 @@ rte_mempool_dump(FILE *f, struct
rte_mempoolQuoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
*mp)
lib/mempool/rte_mempool.c rte_mempool_dump(FILE *f, struct rte_mempool *mp)
Quoted 5 lines
lib/mempool/rte_mempool.c +1
-1#ifdef RTE_LIBRTE_MEMPOOL_STATS rte_mempool_ops_get_info (mp, &info);memset (&sum,0 ,sizeof (sum)); -for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) { + for (lcore_id = 0; lcore_id < RTE_MAX_LCORE + 1;lcore_id++) {
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Review
Quoted 5 lines
lib/mempool/rte_mempool.c +0
-0sum.put_bulk += mp->stats[lcore_id].put_bulk; sum.put_objs += mp->stats[lcore_id].put_objs; sum.put_common_pool_bulk += mp-stats[lcore_id].put_common_pool_bulk;
(quoted diff) +0
-0
b/lib/mempool/rte_mempool.hQuoted 48 lines
Quoted 48 lines
Quoted 5 lines
Quoted 5 lines
lib/mempool/rte_mempool.h struct rte_mempool { · line 238
Quoted 5 lines
index
9c4bf5549f..16e7e62e3c100644lib/mempool/rte_mempool.h +0
-0@@ -238,8 +238,11 @@ struct rte_mempool {struct rte_mempool_memhdr_list mem_list;/**< List of memory
Quoted 13 lines
Quoted 13 lines
chunks */
Quoted 12 lines
lib/mempool/rte_mempool.h +5
-2#ifdef RTE_LIBRTE_MEMPOOL_STATS -/** Per-lcore statistics. */ -struct rte_mempool_debug_stats stats[RTE_MAX_LCORE ]; +/** Per-lcore statistics. +* +* Offset by one, to include non-DPDK threads. +*/ +struct rte_mempool_debug_stats stats[RTE_MAX_LCORE +1 ];#endif } __rte_cache_aligned;Quoted 4 lines
Quoted 4 lines
lib/mempool/rte_mempool.h struct rte_mempool { · line 309
Quoted 4 lines
lib/mempool/rte_mempool.h +0
-0@@ -304,10 +307,7 @@ struct rte_mempool {*/ #ifdef RTE_LIBRTE_MEMPOOL_STATS #define RTE_MEMPOOL_STAT_ADD (mp, name, n) do {\
Quoted 7 lines
Quoted 7 lines
lib/mempool/rte_mempool.h struct rte_mempool { · line 310
Quoted 5 lines
lib/mempool/rte_mempool.h +1
-4-unsigned __lcore_id =rte_lcore_id (); \ -if (__lcore_id <RTE_MAX_LCORE ) { \ - mp->stats[__lcore_id].name += n; \ - } \ + (mp)->stats[rte_lcore_id () +1 ].name += n; \This relies on
LCORE_ID_ANYbeingUINT32_MAX, and a wrap to 0, foran
Quoted 11 lines
Quoted 2 lines
unregistered non-EAL thread? Might be worth a comment, or better a rewrite with an explicit
LCORE_ID_ANYcomparison.The purpose of this patch is to avoid the comparison here.
Yes, it relies on the wrap to zero, and these conditions:
1.LCORE_ID_ANYbeingUINT32_MAX, and
2. the return type ofrte_lcore_id()being unsigned int, and
3. unsigned int beinguint32_t.When I wrote this, I considered it safe to assume that
LCORE_ID_ANYwill remain the unsigned equivalent of -1 using the return type of
rte_lcore_id(). In other words: If the return type ofrte_lcore_id()should change from 32 to 64 bit,LCORE_ID_ANYwould be updated accordingly toUINT64_MAX.Quoted 1 line
Quoted 5 lines
Quoted 5 lines
Quoted 1 line
Because of this assumption, I didn't use [(
rte_lcore_id()+ 1) &
UINT32_MAX], but just [rte_lcore_id()+ 1].Quoted 2 lines
I struggled writing an appropriate comment without making it
unacceptably long, but eventually gave up, and settled for the one-
line
Quoted 10 lines
Quoted 10 lines
comment in the structure only.
Quoted 7 lines
Quoted 4 lines
You anyways need a conditional. An atomic add must be used in the unregistered EAL thread case.
Good point: The "+= n" must be atomic for non-isolated threads.
If the various unregistered non-EAL threads are run on isolated
cores
Quoted 3 lines
Quoted 1 line
or not does not matter.
Agree: They all share the same index, and thus may race, regardless
which cores they are using.
Quoted 2 lines
Rephrasing: The "+= n" must be atomic for the unregistered non-EAL
threads.
Quoted 4 lines
Quoted 4 lines
Quoted 3 lines
I just took a look at how software maintained stats are handled
elsewhere, and the first thing I found, is the IOAT DMA driver,
which
Quoted 1 line
Quoted 1 line
also seems to be using non-atomic increment [1] regardless if used
by a
Quoted 20 lines
Quoted 18 lines
DPDK thread or not.
Quoted 2 lines
rc2/source/drivers/dma/ioat/ioat_dmadev.c#L228
Quoted 2 lines
However, doing it wrong elsewhere doesn't make it correct doing it
wrong here too. :-)
Quoted 2 lines
Atomic increments are costly, so I would rather follow your
suggestion and reintroduce the comparison. How about:
Quoted 8 lines
#define RTE_MEMPOOL_STAT_ADD(mp, name, n) do { \ unsigned int __lcore_id = rte_lcore_id(); \ if (likely(__lcore_id < RTE_MAX_LCORE)) { \ (mp)->stats[__lcore_id].name += n; \ } else { rte_atomic64_add( \ (rte_atomic64_t*)&((mp)-stats[RTE_MAX_LCORE].name),
Quoted 1 line
n);\
Quoted 8 lines
Quoted 4 lines
Quoted 2 lines
} \ }You are supposed to use GCC C11 intrinsics (e.g.,
__atomic_fetch_add()).Ups. I forgot!
This should be emphasized everywhere in the
rte_atomiclibrary, toprevent such mistakes.
Quoted 4 lines
Quoted 4 lines
In addition: technically, you must use an atomic store for the EAL thread case (and an atomic load on the reader side), although there
are
Quoted 1 line
Quoted 1 line
tons of examples in DPDK where tearing is ignored. (The generated
code
Quoted 3 lines
Quoted 1 line
will likely look the same.)
The counters are 64 bit aligned, but tearing could happen on 32 bit
architectures.
Quoted 1 line
The compiler is free to do it on any architecture, but I'm not sure if it happens much in practice.
Quoted 7 lines
Quoted 1 line
My initial reaction to your comment was to do it correctly on the EAL
threads too, to avoid tearing there too. However, there might be a performance cost for 32 bit architectures, so I will consider that these are only debug counters, and accept the risk of tearing.
Quoted 1 line
What would that cost consist of?
I have tested it this morning on Godbolt: https://godbolt.org/z/fz7f3cv8b
ctr += n becomes:
add QWORD PTR ctr[rip], rdi
Whereas __atomic_fetch_add(&ctr, n, __ATOMIC_RELAXED) becomes:
lock add QWORD PTR ctr[rip], rdi
Quoted 4 lines
In theory C11 atomics could be implemented "in software" (i.e., without the proper ISA-level guarantees, with locks), but does any of the DPDK-supported compiler/32-bit ISAs actually do so?
Adding -m32 to the compiler options, ctr += n becomes:
xor edx, edx
mov eax, DWORD PTR [esp+4]
add DWORD PTR ctr, eax
adc DWORD PTR ctr+4, edxAnd __atomic_fetch_add(&ctr, n, __ATOMIC_RELAXED) becomes:
push edi
xor edi, edi
push esi
push ebx
sub esp, 8
mov esi, DWORD PTR [esp+24]
mov eax, DWORD PTR ctr
mov edx, DWORD PTR ctr+4
.L4:
mov ecx, eax
mov ebx, edx
add ecx, esi
adc ebx, edi
mov DWORD PTR [esp], ecx
mov DWORD PTR [esp+4], ebx
mov ebx, DWORD PTR [esp]
mov ecx, DWORD PTR [esp+4] lock cmpxchg8b QWORD PTR ctr
jne .L4
add esp, 8
pop ebx
pop esi
pop edi
Quoted 4 lines
It's also not obvious that if there, for a certain compiler/ISA-combination, is a performance impact to pay for correctness, correctness would have to give way.
In this particular case, and because they are only debug counters, I will argue that DPDK generally uses non-atomic access to 64 bit statistics counters. This was also the case for these counters before this patch.
I am planning to finish v3 of this patch series today, so I hope you can agree to close the atomicity discussion with another argument: Making statistics counters atomic should be elevated to a general patch across DPDK, and not part of this mempool patch series.
The v3 patch (which I am working on right now) counts atomically for the unregistered non-EAL threads:
#define RTE_MEMPOOL_STAT_ADD(mp, name, n) do { \
unsigned int __lcore_id = rte_lcore_id(); \
if (likely(__lcore_id < RTE_MAX_LCORE)) \
(mp)->stats[__lcore_id].name += n; \
else \
__atomic_fetch_add(&((mp)->stats[RTE_MAX_LCORE].name), \
n, __ATOMIC_RELAXED); \
} while (0)PS: Excellent discussion - thank you for getting involved, Mattias.
Re: [PATCH v2 2/3] mempool: include non-DPDK threads in statistics
Quoted 188 lines · Morten Brørup
Quoted 178 lines
From: Mattias Rönnblom [mailto:[email protected]]
Sent: Friday, 4 November 2022 09.59Quoted 9 lines · Morten Brørup
Quoted 9 lines
From: Mattias Rönnblom [mailto:[email protected]]
Sent: Wednesday, 2 November 2022 18.53Quoted 5 lines · Morten Brørup
Quoted 5 lines
From: Mattias Rönnblom [mailto:[email protected]]
Sent: Wednesday, 2 November 2022 08.53Quoted 1 line · Morten Brørup
Offset the stats array index by one, and count non-DPDK threads
at
Quoted 13 lines
Quoted 13 lines
Quoted 6 lines
Quoted 6 lines
index
Quoted 5 lines
zero.
This patch provides two benefits:
* Non-DPDK threads are also included in the statistics.
* A conditional in the fast path is removed. Static branchprediction
Quoted 6 lines
Quoted 6 lines
was
Quoted 5 lines
correct, so the performance improvement is negligible.
v2:
* New. No v1 of this patch in the series.Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Review
Quoted 8 lines
Suggested-by Stephen Hemminger <[email protected]>Signed-off-by Morten Brørup <[email protected]>--- lib/mempool/rte_mempool.c | 2 +- lib/mempool/rte_mempool.h | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-)(quoted diff) +0
-0
b/lib/mempool/rte_mempool.cQuoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
lib/mempool/rte_mempool.c rte_mempool_dump(FILE *f, struct
Quoted 4 lines
index
62d1ce764e..e6208125e0100644lib/mempool/rte_mempool.c +0
-0@@ -1272,7 +1272,7 @@ rte_mempool_dump(FILE *f, struct
rte_mempoolQuoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
*mp)
lib/mempool/rte_mempool.c rte_mempool_dump(FILE *f, struct · line 1275
Quoted 5 lines
lib/mempool/rte_mempool.c +1
-1#ifdef RTE_LIBRTE_MEMPOOL_STATS rte_mempool_ops_get_info (mp, &info);memset (&sum,0 ,sizeof (sum)); -for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) { + for (lcore_id = 0; lcore_id < RTE_MAX_LCORE + 1;lcore_id++) {
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Review
Quoted 5 lines
lib/mempool/rte_mempool.c +0
-0sum.put_bulk += mp->stats[lcore_id].put_bulk; sum.put_objs += mp->stats[lcore_id].put_objs; sum.put_common_pool_bulk += mp-stats[lcore_id].put_common_pool_bulk;
(quoted diff) +0
-0
b/lib/mempool/rte_mempool.hQuoted 48 lines
Quoted 48 lines
Quoted 5 lines
Quoted 5 lines
lib/mempool/rte_mempool.h struct rte_mempool { · line 238
Quoted 5 lines
index
9c4bf5549f..16e7e62e3c100644lib/mempool/rte_mempool.h +0
-0@@ -238,8 +238,11 @@ struct rte_mempool {struct rte_mempool_memhdr_list mem_list;/**< List of memory
Quoted 13 lines
Quoted 13 lines
chunks */
Quoted 12 lines
lib/mempool/rte_mempool.h +5
-2#ifdef RTE_LIBRTE_MEMPOOL_STATS -/** Per-lcore statistics. */ -struct rte_mempool_debug_stats stats[RTE_MAX_LCORE ]; +/** Per-lcore statistics. +* +* Offset by one, to include non-DPDK threads. +*/ +struct rte_mempool_debug_stats stats[RTE_MAX_LCORE +1 ];#endif } __rte_cache_aligned;Quoted 4 lines
Quoted 4 lines
lib/mempool/rte_mempool.h struct rte_mempool { · line 309
Quoted 4 lines
lib/mempool/rte_mempool.h +0
-0@@ -304,10 +307,7 @@ struct rte_mempool {*/ #ifdef RTE_LIBRTE_MEMPOOL_STATS #define RTE_MEMPOOL_STAT_ADD (mp, name, n) do {\
Quoted 7 lines
Quoted 7 lines
lib/mempool/rte_mempool.h struct rte_mempool { · line 310
Quoted 5 lines
lib/mempool/rte_mempool.h +1
-4-unsigned __lcore_id =rte_lcore_id (); \ -if (__lcore_id <RTE_MAX_LCORE ) { \ - mp->stats[__lcore_id].name += n; \ - } \ + (mp)->stats[rte_lcore_id () +1 ].name += n; \This relies on
LCORE_ID_ANYbeingUINT32_MAX, and a wrap to 0, foran
Quoted 11 lines
Quoted 2 lines
unregistered non-EAL thread? Might be worth a comment, or better a rewrite with an explicit
LCORE_ID_ANYcomparison.The purpose of this patch is to avoid the comparison here.
Yes, it relies on the wrap to zero, and these conditions:
1.LCORE_ID_ANYbeingUINT32_MAX, and
2. the return type ofrte_lcore_id()being unsigned int, and
3. unsigned int beinguint32_t.When I wrote this, I considered it safe to assume that
LCORE_ID_ANYwill remain the unsigned equivalent of -1 using the return type of
rte_lcore_id(). In other words: If the return type ofrte_lcore_id()should change from 32 to 64 bit,LCORE_ID_ANYwould be updated accordingly toUINT64_MAX.Quoted 1 line
Quoted 5 lines
Quoted 5 lines
Quoted 1 line
Because of this assumption, I didn't use [(
rte_lcore_id()+ 1) &
UINT32_MAX], but just [rte_lcore_id()+ 1].Quoted 2 lines
I struggled writing an appropriate comment without making it
unacceptably long, but eventually gave up, and settled for the one-
line
Quoted 10 lines
Quoted 10 lines
comment in the structure only.
Quoted 7 lines
Quoted 4 lines
You anyways need a conditional. An atomic add must be used in the unregistered EAL thread case.
Good point: The "+= n" must be atomic for non-isolated threads.
If the various unregistered non-EAL threads are run on isolated
cores
Quoted 3 lines
Quoted 1 line
or not does not matter.
Agree: They all share the same index, and thus may race, regardless
which cores they are using.
Quoted 2 lines
Rephrasing: The "+= n" must be atomic for the unregistered non-EAL
threads.
Quoted 4 lines
Quoted 4 lines
Quoted 3 lines
I just took a look at how software maintained stats are handled
elsewhere, and the first thing I found, is the IOAT DMA driver,
which
Quoted 1 line
Quoted 1 line
also seems to be using non-atomic increment [1] regardless if used
by a
Quoted 20 lines
Quoted 18 lines
DPDK thread or not.
Quoted 2 lines
rc2/source/drivers/dma/ioat/ioat_dmadev.c#L228
Quoted 2 lines
However, doing it wrong elsewhere doesn't make it correct doing it
wrong here too. :-)
Quoted 2 lines
Atomic increments are costly, so I would rather follow your
suggestion and reintroduce the comparison. How about:
Quoted 8 lines
#define RTE_MEMPOOL_STAT_ADD(mp, name, n) do { \ unsigned int __lcore_id = rte_lcore_id(); \ if (likely(__lcore_id < RTE_MAX_LCORE)) { \ (mp)->stats[__lcore_id].name += n; \ } else { rte_atomic64_add( \ (rte_atomic64_t*)&((mp)-stats[RTE_MAX_LCORE].name),
Quoted 1 line
n);\
Quoted 8 lines
Quoted 4 lines
Quoted 2 lines
} \ }You are supposed to use GCC C11 intrinsics (e.g.,
__atomic_fetch_add()).Ups. I forgot!
This should be emphasized everywhere in the
rte_atomiclibrary, toprevent such mistakes.
Quoted 4 lines
Quoted 4 lines
In addition: technically, you must use an atomic store for the EAL thread case (and an atomic load on the reader side), although there
are
Quoted 1 line
Quoted 1 line
tons of examples in DPDK where tearing is ignored. (The generated
code
Quoted 3 lines
Quoted 1 line
will likely look the same.)
The counters are 64 bit aligned, but tearing could happen on 32 bit
architectures.
Quoted 1 line
The compiler is free to do it on any architecture, but I'm not sure if it happens much in practice.
Quoted 7 lines
Quoted 1 line
My initial reaction to your comment was to do it correctly on the EAL
threads too, to avoid tearing there too. However, there might be a performance cost for 32 bit architectures, so I will consider that these are only debug counters, and accept the risk of tearing.
Quoted 1 line
What would that cost consist of?
I have tested it this morning on Godbolt: https://godbolt.org/z/fz7f3cv8b
Quoted 8 lines
ctr += n becomes:
add QWORD PTR ctr[rip], rdi
Whereas
__atomic_fetch_add(&ctr, n, __ATOMIC_RELAXED)becomes:lock add QWORD PTR ctr[rip], rdi
Since there is a single writer/producer, only the store need be atomic, not the complete load+add+store sequence.
__atomic_store_n(&ctr, ctr + 1, __ATOMIC_RELAXED);
Such a construct will not result in a lock add instruction.
I've happened to finished a draft version of a chapter on this topic, in the data plane book I'm writing:
https://ericsson.github.io/dataplanebook/stats/stats.html#per-core-counters
I will add a section with this "add an extra instance to deal with unregistered non-EAL threads" approach taken in your patch.
Quoted 39 lines
Quoted 4 lines
In theory C11 atomics could be implemented "in software" (i.e., without the proper ISA-level guarantees, with locks), but does any of the DPDK-supported compiler/32-bit ISAs actually do so?
Adding -m32 to the compiler options, ctr += n becomes:
xor edx, edx mov eax, DWORD PTR [esp+4] add DWORD PTR ctr, eax adc DWORD PTR ctr+4, edxAnd
__atomic_fetch_add(&ctr, n, __ATOMIC_RELAXED)becomes:push edi xor edi, edi push esi push ebx sub esp, 8 mov esi, DWORD PTR [esp+24] mov eax, DWORD PTR ctr mov edx, DWORD PTR ctr+4 .L4: mov ecx, eax mov ebx, edx add ecx, esi adc ebx, edi mov DWORD PTR [esp], ecx mov DWORD PTR [esp+4], ebx mov ebx, DWORD PTR [esp] mov ecx, DWORD PTR [esp+4]lock cmpxchg8b QWORD PTR ctr
jne .L4
add esp, 8
pop ebx
pop esi
pop ediQuoted 1 line
Quoted 8 lines
Quoted 3 lines
It's also not obvious that if there, for a certain compiler/ISA-combination, is a performance impact to pay for correctness, correctness would have to give way.
In this particular case, and because they are only debug counters, I will argue that DPDK generally uses non-atomic access to 64 bit statistics counters. This was also the case for these counters before this patch.
I am planning to finish v3 of this patch series today, so I hope you can agree to close the atomicity discussion with another argument: Making statistics counters atomic should be elevated to a general patch across DPDK, and not part of this mempool patch series.
As I think I indicated, I think this is a minor issue. I'm OK with the code as-written.
Quoted 13 lines
The v3 patch (which I am working on right now) counts atomically for the unregistered non-EAL threads:
#define RTE_MEMPOOL_STAT_ADD(mp, name, n) do { \ unsigned int __lcore_id = rte_lcore_id(); \ if (likely(__lcore_id < RTE_MAX_LCORE)) \ (mp)->stats[__lcore_id].name += n; \ else \ __atomic_fetch_add(&((mp)->stats[RTE_MAX_LCORE].name), \ n, __ATOMIC_RELAXED); \ } while (0)PS: Excellent discussion - thank you for getting involved, Mattias.
RE: [PATCH v2 2/3] mempool: include non-DPDK threads in statistics
Quoted 212 lines
From: Mattias Rönnblom [mailto:[email protected]]
Sent: Monday, 7 November 2022 08.27Quoted 78 lines · Morten Brørup
Quoted 78 lines
From: Mattias Rönnblom [mailto:[email protected]]
Sent: Friday, 4 November 2022 09.59Quoted 9 lines · Morten Brørup
Quoted 9 lines
From: Mattias Rönnblom [mailto:[email protected]]
Sent: Wednesday, 2 November 2022 18.53Quoted 5 lines · Morten Brørup
Quoted 5 lines
From: Mattias Rönnblom [mailto:[email protected]]
Sent: Wednesday, 2 November 2022 08.53Quoted 1 line · Morten Brørup
Offset the stats array index by one, and count non-DPDK threads
at
Quoted 13 lines
Quoted 13 lines
Quoted 6 lines
Quoted 6 lines
index
Quoted 5 lines
zero.
This patch provides two benefits:
* Non-DPDK threads are also included in the statistics.
* A conditional in the fast path is removed. Static branchprediction
Quoted 6 lines
Quoted 6 lines
was
Quoted 5 lines
correct, so the performance improvement is negligible.
v2:
* New. No v1 of this patch in the series.Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Review
Quoted 8 lines
Suggested-by Stephen Hemminger <[email protected]>Signed-off-by Morten Brørup <[email protected]>--- lib/mempool/rte_mempool.c | 2 +- lib/mempool/rte_mempool.h | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-)(quoted diff) +0
-0
b/lib/mempool/rte_mempool.cQuoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
lib/mempool/rte_mempool.c rte_mempool_dump(FILE *f, struct
Quoted 4 lines
index
62d1ce764e..e6208125e0100644lib/mempool/rte_mempool.c +0
-0@@ -1272,7 +1272,7 @@ rte_mempool_dump(FILE *f, struct
rte_mempoolQuoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
*mp)
lib/mempool/rte_mempool.c rte_mempool_dump(FILE *f, struct · line 1275
Quoted 5 lines
lib/mempool/rte_mempool.c +1
-1#ifdef RTE_LIBRTE_MEMPOOL_STATS rte_mempool_ops_get_info (mp, &info);memset (&sum,0 ,sizeof (sum)); -for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) { + for (lcore_id = 0; lcore_id < RTE_MAX_LCORE + 1;lcore_id++) {
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Review
Quoted 5 lines
lib/mempool/rte_mempool.c +0
-0sum.put_bulk += mp->stats[lcore_id].put_bulk; sum.put_objs += mp->stats[lcore_id].put_objs; sum.put_common_pool_bulk += mp-stats[lcore_id].put_common_pool_bulk;
(quoted diff) +0
-0
b/lib/mempool/rte_mempool.hQuoted 24 lines
Quoted 24 lines
Quoted 5 lines
Quoted 5 lines
lib/mempool/rte_mempool.h struct rte_mempool { · line 238
Quoted 5 lines
index
9c4bf5549f..16e7e62e3c100644lib/mempool/rte_mempool.h +0
-0@@ -238,8 +238,11 @@ struct rte_mempool {struct rte_mempool_memhdr_list mem_list;/**< List of memory
Quoted 13 lines
Quoted 13 lines
chunks */
Quoted 12 lines
lib/mempool/rte_mempool.h +5
-2#ifdef RTE_LIBRTE_MEMPOOL_STATS -/** Per-lcore statistics. */ -struct rte_mempool_debug_stats stats[RTE_MAX_LCORE ]; +/** Per-lcore statistics. +* +* Offset by one, to include non-DPDK threads. +*/ +struct rte_mempool_debug_stats stats[RTE_MAX_LCORE +1 ];#endif } __rte_cache_aligned;Quoted 4 lines
Quoted 4 lines
lib/mempool/rte_mempool.h struct rte_mempool { · line 309
Quoted 4 lines
lib/mempool/rte_mempool.h +0
-0@@ -304,10 +307,7 @@ struct rte_mempool {*/ #ifdef RTE_LIBRTE_MEMPOOL_STATS #define RTE_MEMPOOL_STAT_ADD (mp, name, n) do {\
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
lib/mempool/rte_mempool.h struct rte_mempool { · line 310
Quoted 5 lines
lib/mempool/rte_mempool.h +1
-4-unsigned __lcore_id =rte_lcore_id (); \ -if (__lcore_id <RTE_MAX_LCORE ) { \ - mp->stats[__lcore_id].name += n; \ - } \ + (mp)->stats[rte_lcore_id () +1 ].name += n; \This relies on
LCORE_ID_ANYbeingUINT32_MAX, and a wrap to 0,for
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
an
Quoted 1 line
Quoted 1 line
unregistered non-EAL thread? Might be worth a comment, or better
a
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 1 line
rewrite with an explicit
LCORE_ID_ANYcomparison.The purpose of this patch is to avoid the comparison here.
Yes, it relies on the wrap to zero, and these conditions:
1.LCORE_ID_ANYbeingUINT32_MAX, and
2. the return type ofrte_lcore_id()being unsigned int, and
3. unsigned int beinguint32_t.When I wrote this, I considered it safe to assume that
LCORE_ID_ANYQuoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
will remain the unsigned equivalent of -1 using the return type of
rte_lcore_id(). In other words: If the return type of
rte_lcore_id()Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
should change from 32 to 64 bit,
LCORE_ID_ANYwould be updated accordingly toUINT64_MAX.Quoted 2 lines
Because of this assumption, I didn't use [(
rte_lcore_id()+ 1) &
UINT32_MAX], but just [rte_lcore_id()+ 1].Quoted 2 lines
I struggled writing an appropriate comment without making it
unacceptably long, but eventually gave up, and settled for the
one-
Quoted 5 lines
Quoted 5 lines
line
Quoted 4 lines
Quoted 4 lines
comment in the structure only.
Quoted 3 lines
Quoted 3 lines
You anyways need a conditional. An atomic add must be used in
the
Quoted 19 lines
Quoted 19 lines
Quoted 6 lines
Quoted 6 lines
Quoted 4 lines
Quoted 1 line
unregistered EAL thread case.
Good point: The "+= n" must be atomic for non-isolated threads.
If the various unregistered non-EAL threads are run on isolated
cores
Quoted 3 lines
Quoted 1 line
or not does not matter.
Agree: They all share the same index, and thus may race, regardless
which cores they are using.
Quoted 2 lines
Rephrasing: The "+= n" must be atomic for the unregistered non-EAL
threads.
Quoted 4 lines
Quoted 4 lines
Quoted 3 lines
I just took a look at how software maintained stats are handled
elsewhere, and the first thing I found, is the IOAT DMA driver,
which
Quoted 8 lines
Quoted 8 lines
Quoted 1 line
Quoted 1 line
also seems to be using non-atomic increment [1] regardless if used
by a
Quoted 6 lines
Quoted 6 lines
DPDK thread or not.
Quoted 2 lines
rc2/source/drivers/dma/ioat/ioat_dmadev.c#L228
Quoted 2 lines
However, doing it wrong elsewhere doesn't make it correct doing
it
Quoted 19 lines
Quoted 19 lines
Quoted 19 lines
Quoted 12 lines
wrong here too. :-)
Quoted 2 lines
Atomic increments are costly, so I would rather follow your
suggestion and reintroduce the comparison. How about:
Quoted 8 lines
#define RTE_MEMPOOL_STAT_ADD(mp, name, n) do { \ unsigned int __lcore_id = rte_lcore_id(); \ if (likely(__lcore_id < RTE_MAX_LCORE)) { \ (mp)->stats[__lcore_id].name += n; \ } else { rte_atomic64_add( \ (rte_atomic64_t*)&((mp)-stats[RTE_MAX_LCORE].name),
Quoted 5 lines
n);\
Quoted 2 lines
} \ }You are supposed to use GCC C11 intrinsics (e.g.,
__atomic_fetch_add()).Quoted 8 lines
Quoted 8 lines
Quoted 3 lines
Ups. I forgot!
This should be emphasized everywhere in the
rte_atomiclibrary, toprevent such mistakes.
Quoted 4 lines
Quoted 4 lines
In addition: technically, you must use an atomic store for the EAL thread case (and an atomic load on the reader side), although
there
Quoted 10 lines
Quoted 10 lines
are
Quoted 1 line
Quoted 1 line
tons of examples in DPDK where tearing is ignored. (The generated
code
Quoted 3 lines
Quoted 1 line
will likely look the same.)
The counters are 64 bit aligned, but tearing could happen on 32 bit
architectures.
Quoted 1 line
The compiler is free to do it on any architecture, but I'm not sure
if
Quoted 3 lines
Quoted 3 lines
it happens much in practice.
Quoted 1 line
My initial reaction to your comment was to do it correctly on the
EAL
Quoted 8 lines
Quoted 6 lines
threads too, to avoid tearing there too. However, there might be a performance cost for 32 bit architectures, so I will consider that these are only debug counters, and accept the risk of tearing.
Quoted 1 line
What would that cost consist of?
I have tested it this morning on Godbolt:
https://godbolt.org/z/fz7f3cv8b
Quoted 9 lines
ctr += n becomes:
add QWORD PTR ctr[rip], rdi
Whereas
__atomic_fetch_add(&ctr, n, __ATOMIC_RELAXED)becomes:lock add QWORD PTR ctr[rip], rdi
Quoted 6 lines
Since there is a single writer/producer, only the store need be atomic, not the complete load+add+store sequence.
__atomic_store_n(&ctr, ctr + 1, __ATOMIC_RELAXED);Such a construct will not result in a lock add instruction.
I was trying to get rid of the conditional by handling both kinds of threads the same way. However, without the conditional, the unregistered non-EAL threads need to use __atomic_fetch_add would spill over to the EAL threads. So I concluded that the conditional should remain.
Quoted 6 lines
I've happened to finished a draft version of a chapter on this topic, in
the data plane book I'm writing:
https://ericsson.github.io/dataplanebook/stats/stats.html#per-core-
counters
Thanks for the link. I just took a look at it, and it looks great!
Quoted 53 lines
I will add a section with this "add an extra instance to deal with unregistered non-EAL threads" approach taken in your patch.
Quoted 2 lines
Quoted 2 lines
In theory C11 atomics could be implemented "in software" (i.e.,
without
Quoted 37 lines
Quoted 2 lines
the proper ISA-level guarantees, with locks), but does any of the DPDK-supported compiler/32-bit ISAs actually do so?
Adding -m32 to the compiler options, ctr += n becomes:
xor edx, edx mov eax, DWORD PTR [esp+4] add DWORD PTR ctr, eax adc DWORD PTR ctr+4, edxAnd
__atomic_fetch_add(&ctr, n, __ATOMIC_RELAXED)becomes:push edi xor edi, edi push esi push ebx sub esp, 8 mov esi, DWORD PTR [esp+24] mov eax, DWORD PTR ctr mov edx, DWORD PTR ctr+4 .L4: mov ecx, eax mov ebx, edx add ecx, esi adc ebx, edi mov DWORD PTR [esp], ecx mov DWORD PTR [esp+4], ebx mov ebx, DWORD PTR [esp] mov ecx, DWORD PTR [esp+4]lock cmpxchg8b QWORD PTR ctr
jne .L4
add esp, 8
pop ebx
pop esi
pop ediQuoted 1 line
Quoted 5 lines
Quoted 3 lines
It's also not obvious that if there, for a certain compiler/ISA-combination, is a performance impact to pay for correctness, correctness would have to give way.
In this particular case, and because they are only debug counters, I
will argue that DPDK generally uses non-atomic access to 64 bit statistics counters. This was also the case for these counters before this patch.
Quoted 1 line
Quoted 8 lines
Quoted 1 line
I am planning to finish v3 of this patch series today, so I hope you
can agree to close the atomicity discussion with another argument:
Making statistics counters atomic should be elevated to a general patch across DPDK, and not part of this mempool patch series.Quoted 1 line
As I think I indicated, I think this is a minor issue. I'm OK with the code as-written.
Thanks. We can always micro-optimize the unregistered non-EAL thread counters later. :-)
Quoted 21 lines
Quoted 2 lines
The v3 patch (which I am working on right now) counts atomically for
the unregistered non-EAL threads:
Quoted 2 lines
#define
RTE_MEMPOOL_STAT_ADD(mp, name, n)do {\
Quoted 1 line
unsigned int __lcore_id = rte_lcore_id();\
Quoted 1 line
if (likely(__lcore_id < RTE_MAX_LCORE))\
Quoted 1 line
(mp)->stats[__lcore_id].name += n;\
Quoted 1 line
else
\
Quoted 7 lines
__atomic_fetch_add(&((mp)- stats[RTE_MAX_LCORE].name), \ n, __ATOMIC_RELAXED); \ } while (0)PS: Excellent discussion - thank you for getting involved, Mattias.
PATCH v2 3/3 mempool: use cache for frequently updated statistics
In reply to Morten BrørupWhen built with statistics enabled (RTE_LIBRTE_MEMPOOL_STATS defined), the performance of mempools with caches is improved as follows.
When accessing objects in the mempool, either the put_bulk and put_objs or the get_success_bulk and get_success_objs statistics counters are likely to be incremented.
By adding an alternative set of these counters to the mempool cache structure, accesing the dedicated statistics structure is avoided in the likely cases where these counters are incremented.
The trick here is that the cache line holding the mempool cache structure is accessed anyway, in order to access the 'len' or 'flushthresh' fields. Updating some statistics counters in the same cache line has lower performance cost than accessing the statistics counters in the dedicated statistics structure, which resides in another cache line.
mempool_perf_autotest with this patch shows the follwing change in rate_persec.
Compared to only spliting statistics from debug:
+1.5 % and +14.4 %, respectively without and with cache.
Compared to not enabling mempool stats:
-4.4 % and -9.9 %, respectively without and with cache.
v2:
* Move the statistics counters into a stats structure.
--- lib/mempool/rte_mempool.c | 9 +++++ lib/mempool/rte_mempool.h | 73 ++++++++++++++++++++++++++++++++------- 2 files changed, 69 insertions(+), 13 deletions(-)
lib/mempool/rte_mempool.c +9 -0
@@ -1286,6 +1286,15 @@ rte_mempool_dump(FILE *f, struct rte_mempool *mp) sum.get_success_blks += mp->stats[lcore_id].get_success_blks; sum.get_fail_blks += mp->stats[lcore_id].get_fail_blks; } +if (mp->cache_size !=0 ) { +/* Add the statistics stored in the mempool caches. */ +for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) { + sum.put_bulk += mp->local_cache[lcore_id].stats.put_bulk; + sum.put_objs += mp->local_cache[lcore_id].stats.put_objs; + sum.get_success_bulk += mp->local_cache[lcore_id].stats.get_success_bulk; + sum.get_success_objs += mp->local_cache[lcore_id].stats.get_success_objs; + } + }fprintf (f," stats:\n" );fprintf (f," put_bulk=%" PRIu64"\n" , sum.put_bulk);fprintf (f," put_objs=%" PRIu64"\n" , sum.put_objs);
lib/mempool/rte_mempool.h +60 -13
129 lines not on this page · raw message
Signature · 1 line
2.17.1
Re: [PATCH v2 3/3] mempool: use cache for frequently updated statistics
Quoted 67 lines · Morten Brørup
When built with statistics enabled (
RTE_LIBRTE_MEMPOOL_STATSdefined), the performance of mempools with caches is improved as follows.When accessing objects in the mempool, either the
put_bulkandput_objsor theget_success_bulkandget_success_objsstatistics counters are likely to be incremented.By adding an alternative set of these counters to the mempool cache structure, accesing the dedicated statistics structure is avoided in the likely cases where these counters are incremented.
The trick here is that the cache line holding the mempool cache structure is accessed anyway, in order to access the 'len' or 'flushthresh' fields. Updating some statistics counters in the same cache line has lower performance cost than accessing the statistics counters in the dedicated statistics structure, which resides in another cache line.
mempool_perf_autotestwith this patch shows the follwing change inrate_persec.Compared to only spliting statistics from debug:
+1.5 % and +14.4 %, respectively without and with cache.Compared to not enabling mempool stats:
-4.4 % and -9.9 %, respectively without and with cache.v2:
* Move the statistics counters into a stats structure.Signed-off-by Morten Brørup <[email protected]>--- lib/mempool/rte_mempool.c | 9 +++++ lib/mempool/rte_mempool.h | 73 ++++++++++++++++++++++++++++++++------- 2 files changed, 69 insertions(+), 13 deletions(-)lib/mempool/rte_mempool.c +9
-0@@ -1286,6 +1286,15 @@ rte_mempool_dump(FILE *f, struct rte_mempool *mp) sum.get_success_blks += mp->stats[lcore_id].get_success_blks; sum.get_fail_blks += mp->stats[lcore_id].get_fail_blks; } +if (mp->cache_size !=0 ) { +/* Add the statistics stored in the mempool caches. */ +for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) { + sum.put_bulk += mp->local_cache[lcore_id].stats.put_bulk; + sum.put_objs += mp->local_cache[lcore_id].stats.put_objs; + sum.get_success_bulk += mp->local_cache[lcore_id].stats.get_success_bulk; + sum.get_success_objs += mp->local_cache[lcore_id].stats.get_success_objs; + } + }fprintf (f," stats:\n" );fprintf (f," put_bulk=%" PRIu64"\n" , sum.put_bulk);fprintf (f," put_objs=%" PRIu64"\n" , sum.put_objs);lib/mempool/rte_mempool.h +4
-0@@ -86,6 +86,21 @@ struct rte_mempool_cache {uint32_t size;/**< Size of the cache */ uint32_t flushthresh;/**< Threshold before we flush excess elements */ uint32_t len;/**< Current cache count */ +uint32_t unused0; +#ifdef RTE_LIBRTE_MEMPOOL_STATS +/* +* Alternative location for the most frequently updated mempool statistics (per-lcore),
lib/mempool/rte_mempool.h struct rte_mempool_cache { · line 102
Quoted 10 lines
lib/mempool/rte_mempool.h +10
-0+* providing faster update access when using a mempool cache. +*/ +struct { +uint64_t put_bulk;/**< Number of puts. */ +uint64_t put_objs;/**< Number of objects successfully put. */ +uint64_t get_success_bulk;/**< Successful allocation number. */ +uint64_t get_success_objs;/**< Objects successfully allocated. */ + } stats;/**< Statistics */ +#else +uint64_t unused1[4 ];
Are a particular DPDK version supposed to be ABI compatible with itself, with different configuration options? E.g., with and without RTE_LIBRTE_MEMPOOL_STATS. Is that why you have those 4 unused uint64_ts?
Quoted 31 lines
lib/mempool/rte_mempool.h +9
-2+#endif /** * Cache objects * @@ -296,14 +311,14 @@ struct rte_mempool { | RTE_MEMPOOL_F_NO_IOVA_CONTIG \ )/** -* @internal When debug is enabled, store some statistics. +* @internal When stats is enabled, store some statistics. * * @param mp * Pointer to the memory pool. * @param name * Name of the statistics field to increment in the memory pool. * @param n -* Number to add to the object-oriented statistics. +* Number to add to the statistics. */ #ifdef RTE_LIBRTE_MEMPOOL_STATS #define RTE_MEMPOOL_STAT_ADD (mp, name, n) do { \@@ -312,6 +327,23 @@ struct rte_mempool {#else #define RTE_MEMPOOL_STAT_ADD (mp, name, n) do {} while (0)#endif +/** +* @internal When stats is enabled, store some statistics. +* +* @param cache +* Pointer to the memory pool cache. +* @param name
lib/mempool/rte_mempool.h struct rte_mempool { · line 345
Quoted 10 lines
lib/mempool/rte_mempool.h +10
-0+* Name of the statistics field to increment in the memory pool cache. +* @param n +* Number to add to the statistics. +*/ +#ifdef RTE_LIBRTE_MEMPOOL_STATS +#define RTE_MEMPOOL_CACHE_STAT_ADD (cache, name, n) do { \ + (cache)->stats.name += n; \ + } while (0) +#else +#define RTE_MEMPOOL_CACHE_STAT_ADD (cache, name, n) do {} while (0)
Somewhat unrelated comment: maybe <rte_common.h> should have a RTE_NOP macro.
Quoted 78 lines
lib/mempool/rte_mempool.h +27
-11+#endif /** * @internal Calculate the size of the mempool header. @@ -1327,13 +1359,17 @@ rte_mempool_do_generic_put(struct rte_mempool *mp, void * const *obj_table, {void **cache_objs; +/* No cache provided */ +if (unlikely (cache ==NULL )) + goto driver_enqueue; +/* increment stat now, adding in mempool always success */ -RTE_MEMPOOL_STAT_ADD (mp, put_bulk,1 ); -RTE_MEMPOOL_STAT_ADD (mp, put_objs, n); +RTE_MEMPOOL_CACHE_STAT_ADD (cache, put_bulk,1 ); +RTE_MEMPOOL_CACHE_STAT_ADD (cache, put_objs, n); -/* No cache provided or the request itself is too big for the cache */ -if (unlikely (cache ==NULL || n > cache->flushthresh)) - goto driver_enqueue; +/* The request itself is too big for the cache */ +if (unlikely (n > cache->flushthresh)) + goto driver_enqueue_stats_incremented;/* * The cache follows the following algorithm: @@ -1358,6 +1394,12 @@ rte_mempool_do_generic_put(struct rte_mempool *mp, void * const *obj_table, driver_enqueue: +/* increment stat now, adding in mempool always success */ +RTE_MEMPOOL_STAT_ADD (mp, put_bulk,1 ); +RTE_MEMPOOL_STAT_ADD (mp, put_objs, n); + +driver_enqueue_stats_incremented: +/* push objects to the backend */ rte_mempool_ops_enqueue_bulk (mp, obj_table, n); }@@ -1464,8 +1506,8 @@ rte_mempool_do_generic_get(struct rte_mempool *mp, void **obj_table,if (remaining ==0 ) {/* The entire request is satisfied from the cache. */ -RTE_MEMPOOL_STAT_ADD (mp, get_success_bulk,1 ); -RTE_MEMPOOL_STAT_ADD (mp, get_success_objs, n); +RTE_MEMPOOL_CACHE_STAT_ADD (cache, get_success_bulk,1 ); +RTE_MEMPOOL_CACHE_STAT_ADD (cache, get_success_objs, n);return 0 ; }@@ -1494,8 +1536,8 @@ rte_mempool_do_generic_get(struct rte_mempool *mp, void **obj_table, cache->len = cache->size; -RTE_MEMPOOL_STAT_ADD (mp, get_success_bulk,1 ); -RTE_MEMPOOL_STAT_ADD (mp, get_success_objs, n); +RTE_MEMPOOL_CACHE_STAT_ADD (cache, get_success_bulk,1 ); +RTE_MEMPOOL_CACHE_STAT_ADD (cache, get_success_objs, n);return 0 ;@@ -1517,8 +1559,13 @@ rte_mempool_do_generic_get(struct rte_mempool *mp, void **obj_table,RTE_MEMPOOL_STAT_ADD (mp, get_fail_bulk,1 );RTE_MEMPOOL_STAT_ADD (mp, get_fail_objs, n); }else { -RTE_MEMPOOL_STAT_ADD (mp, get_success_bulk,1 ); -RTE_MEMPOOL_STAT_ADD (mp, get_success_objs, n); +if (likely (cache !=NULL )) { +RTE_MEMPOOL_CACHE_STAT_ADD (cache, get_success_bulk,1 ); +RTE_MEMPOOL_CACHE_STAT_ADD (cache, get_success_objs, n); + }else { +RTE_MEMPOOL_STAT_ADD (mp, get_success_bulk,1 ); +RTE_MEMPOOL_STAT_ADD (mp, get_success_objs, n); + } }return ret;
RE: [PATCH v2 3/3] mempool: use cache for frequently updated statistics
Quoted 4 lines
From: Mattias Rönnblom [mailto:[email protected]]
Sent: Wednesday, 2 November 2022 09.01On 2022-10-31 12:26, Morten Brørup wrote:
[...]
Quoted 21 lines
Review struct rte_mempool_cache { · line 87
Quoted 4 lines
+++
b/lib/mempool/rte_mempool.h(quoted diff) +0
-0@@ -86,6 +86,21 @@ struct rte_mempool_cache { uint32_t size; /**< Size of the cache */ uint32_t flushthresh; /**< Threshold before we flush excesselements */
lib/mempool/rte_mempool.h struct rte_mempool_cache {
Quoted 5 lines
lib/mempool/rte_mempool.h +4
-0uint32_t len;/**< Current cache count */ +uint32_t unused0; +#ifdef RTE_LIBRTE_MEMPOOL_STATS +/* +* Alternative location for the most frequently updated mempool statistics (per-lcore),
lib/mempool/rte_mempool.h struct rte_mempool_cache {
Quoted 5 lines
lib/mempool/rte_mempool.h +5
-0+* providing faster update access when using a mempool cache. +*/ +struct { +uint64_t put_bulk;/**< Number of puts. */ +uint64_t put_objs;/**< Number of objects successfully put. */
Review struct rte_mempool_cache { · line 98
Quoted 1 line
(quoted diff) +1
-0+ uint64_t get_success_bulk; /**< Successful allocationnumber. */
Review struct rte_mempool_cache { · line 99
Quoted 1 line
(quoted diff) +1
-0+ uint64_t get_success_objs; /**< Objects successfullyallocated. */
Quoted 9 lines
lib/mempool/rte_mempool.h struct rte_mempool_cache { · line 102
Quoted 3 lines
lib/mempool/rte_mempool.h +3
-0+ } stats; /**< Statistics */ +#else +uint64_t unused1[4 ];Are a particular DPDK version supposed to be ABI compatible with itself,
with different configuration options? E.g., with and withoutRTE_LIBRTE_MEMPOOL_STATS. Is that why you have those 4 unuseduint64_ts?
Valid point: There was no ABI compatibility between with and without RTE_LIBRTE_MEMPOOL_STATS before this patch, so there is no need to add partial ABI compatibility here.
The #else part of this structure should be removed.
Does anyone disagree?
Quoted 1 line
Quoted 1 line
(quoted diff) +1
-0+#endif
Re: [PATCH v2 3/3] mempool: use cache for frequently updated statistics
Quoted 38 lines · Morten Brørup
Quoted 4 lines
From: Mattias Rönnblom [mailto:[email protected]]
Sent: Wednesday, 2 November 2022 09.01On 2022-10-31 12:26, Morten Brørup wrote:
[...]
Quoted 21 lines
Review struct rte_mempool_cache { · line 87
Quoted 4 lines
+++
b/lib/mempool/rte_mempool.h(quoted diff) +0
-0@@ -86,6 +86,21 @@ struct rte_mempool_cache { uint32_t size; /**< Size of the cache */ uint32_t flushthresh; /**< Threshold before we flush excesselements */
lib/mempool/rte_mempool.h struct rte_mempool_cache {
Quoted 5 lines
lib/mempool/rte_mempool.h +4
-0uint32_t len;/**< Current cache count */ +uint32_t unused0; +#ifdef RTE_LIBRTE_MEMPOOL_STATS +/* +* Alternative location for the most frequently updated mempool statistics (per-lcore),
lib/mempool/rte_mempool.h struct rte_mempool_cache {
Quoted 5 lines
lib/mempool/rte_mempool.h +5
-0+* providing faster update access when using a mempool cache. +*/ +struct { +uint64_t put_bulk;/**< Number of puts. */ +uint64_t put_objs;/**< Number of objects successfully put. */
Review struct rte_mempool_cache { · line 98
Quoted 1 line
(quoted diff) +1
-0+ uint64_t get_success_bulk; /**< Successful allocationnumber. */
Review struct rte_mempool_cache { · line 99
Quoted 1 line
(quoted diff) +1
-0+ uint64_t get_success_objs; /**< Objects successfullyallocated. */
Quoted 9 lines
lib/mempool/rte_mempool.h struct rte_mempool_cache { · line 102
Quoted 3 lines
lib/mempool/rte_mempool.h +3
-0+ } stats; /**< Statistics */ +#else +uint64_t unused1[4 ];Are a particular DPDK version supposed to be ABI compatible with itself,
with different configuration options? E.g., with and withoutRTE_LIBRTE_MEMPOOL_STATS. Is that why you have those 4 unuseduint64_ts?
Quoted 8 lines
Valid point: There was no ABI compatibility between with and without
RTE_LIBRTE_MEMPOOL_STATSbefore this patch, so there is no need to add partial ABI compatibility here.The #else part of this structure should be removed.
Does anyone disagree?
Quoted 1 line
Quoted 1 line
(quoted diff) +1
-0+#endif
I have no opinion on that matter, but I have another question: if you remove 'unused1', should you also remove the unused0 field?
PATCH v3 1/3 mempool: split stats from debug
In reply to Morten BrørupSplit stats from debug, to make mempool statistics available without the performance cost of continuously validating the debug cookies in the mempool elements.
mempool_perf_autotest shows the following improvements in rate_persec.
The cost of enabling mempool debug without this patch:
-28.1 % and -74.0 %, respectively without and with cache.
The cost of enabling mempool stats (without debug) after this patch:
-5.8 % and -21.2 %, respectively without and with cache.
v3:
* Update the Programmer's Guide.
* Update the description of the RTE_MEMPOOL_STAT_ADD macro. v2:
* Fix checkpatch warning:
Use C style comments in rte_include.h, not C++ style.
* Do not rename the rte_mempool_debug_stats structure.
--- config/rte_config.h | 2 ++ doc/guides/prog_guide/mempool_lib.rst | 6 +++++- lib/mempool/rte_mempool.c | 6 +++--- lib/mempool/rte_mempool.h | 10 +++++----- 4 files changed, 15 insertions(+), 9 deletions(-)
config/rte_config.h +2 -0
@@ -47,6 +47,8 @@/* mempool defines */ #define RTE_MEMPOOL_CACHE_MAX_SIZE 512 +/* RTE_LIBRTE_MEMPOOL_STATS is not set */ +/* RTE_LIBRTE_MEMPOOL_DEBUG is not set */ /* mbuf defines */ #define RTE_MBUF_DEFAULT_MEMPOOL_OPS "ring_mp_mc"
doc/guides/prog_guide/mempool_lib.rst +5 -1
@@ -20,12 +20,16 @@ Cookies In debug mode, cookies are added at the beginning and end of allocated blocks. The allocated objects then contain overwrite protection fields to help debugging buffer overflows. +Debug mode is disabled by default, but can be enabled by setting ``RTE_LIBRTE_MEMPOOL_DEBUG`` in ``config/rte_config.h``. + Stats ----- -In debug mode, statistics about get from/put in the pool are stored in the mempool structure. +In stats mode, statistics about get from/put in the pool are stored in the mempool structure. Statistics are per-lcore to avoid concurrent access to statistics counters. +Stats mode is disabled by default, but can be enabled by setting ``RTE_LIBRTE_MEMPOOL_STATS`` in ``config/rte_config.h``. + Memory Alignment Constraints on x86 architecture ------------------------------------------------
lib/mempool/rte_mempool.c +3 -3
@@ -818,7 +818,7 @@ rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size, RTE_CACHE_LINE_MASK) != 0);RTE_BUILD_BUG_ON ((sizeof (struct rte_mempool_cache ) &RTE_CACHE_LINE_MASK ) !=0 ); -#ifdef RTE_LIBRTE_MEMPOOL_DEBUG +#ifdef RTE_LIBRTE_MEMPOOL_STATS RTE_BUILD_BUG_ON ((sizeof (struct rte_mempool_debug_stats ) &RTE_CACHE_LINE_MASK ) !=0 ); RTE_BUILD_BUG_ON((offsetof(struct rte_mempool, stats) &
@@ -1221,7 +1221,7 @@ rte_mempool_audit(struct rte_mempool *mp)void rte_mempool_dump (FILE *f,struct rte_mempool *mp) { -#ifdef RTE_LIBRTE_MEMPOOL_DEBUG +#ifdef RTE_LIBRTE_MEMPOOL_STATS struct rte_mempool_info info;struct rte_mempool_debug_stats sum;unsigned lcore_id;
@@ -1269,7 +1269,7 @@ rte_mempool_dump(FILE *f, struct rte_mempool *mp)fprintf (f," common_pool_count=%u\n" , common_count);/* sum and dump statistics */ -#ifdef RTE_LIBRTE_MEMPOOL_DEBUG +#ifdef RTE_LIBRTE_MEMPOOL_STATS rte_mempool_ops_get_info (mp, &info);memset (&sum,0 ,sizeof (sum));for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) {
lib/mempool/rte_mempool.h +5 -5
@@ -56,7 +56,7 @@ extern "C" {#define RTE_MEMPOOL_HEADER_COOKIE2 0xf2eef2eedadd2e55ULL/**< Header cookie. */ #define RTE_MEMPOOL_TRAILER_COOKIE 0xadd2e55badbadbadULL/**< Trailer cookie.*/ -#ifdef RTE_LIBRTE_MEMPOOL_DEBUG +#ifdef RTE_LIBRTE_MEMPOOL_STATS /** * A structure that stores the mempool statistics (per-lcore). * Note: Cache stats (put_cache_bulk/objs, get_cache_bulk/objs) are not
@@ -237,7 +237,7 @@ struct rte_mempool {uint32_t nb_mem_chunks;/**< Number of memory chunks */ struct rte_mempool_memhdr_list mem_list;/**< List of memory chunks */ -#ifdef RTE_LIBRTE_MEMPOOL_DEBUG +#ifdef RTE_LIBRTE_MEMPOOL_STATS /** Per-lcore statistics. */ struct rte_mempool_debug_stats stats[RTE_MAX_LCORE ];#endif
@@ -293,16 +293,16 @@ struct rte_mempool { | RTE_MEMPOOL_F_NO_IOVA_CONTIG \ )/** -* @internal When debug is enabled, store some statistics. +* @internal When stats is enabled, store some statistics. * * @param mp * Pointer to the memory pool. * @param name * Name of the statistics field to increment in the memory pool. * @param n -* Number to add to the object-oriented statistics. +* Number to add to the statistics. */ -#ifdef RTE_LIBRTE_MEMPOOL_DEBUG +#ifdef RTE_LIBRTE_MEMPOOL_STATS #define RTE_MEMPOOL_STAT_ADD (mp, name, n) do { \ unsigned __lcore_id = rte_lcore_id(); \ if (__lcore_id < RTE_MAX_LCORE) { \
Signature · 1 line
2.17.1
PATCH v3 2/3 mempool: add stats for unregistered non-EAL threads
This patch adds statistics for unregistered non-EAL threads, which was previously not included in the statistics.
Add one more entry to the stats array, and use the last index for unregistered non-EAL threads.
The unregistered non-EAL thread statistics are incremented atomically.
In theory, the EAL thread counters should also be accessed atomically to avoid tearing on 32 bit architectures. However, it was decided to avoid the performance cost of using atomic operations, because:
1. these are debug counters, and
2. statistics counters in DPDK are usually incremented non-atomically.
v3 (feedback from Mattias Rönnblom):
* Use correct terminology: Unregistered non-EAL threads.
* Use atomic counting for the unregistered non-EAL threads.
* Reintroduce the conditional instead of offsetting the index by one. v2:
* New. No v1 of this patch in the series.
--- lib/mempool/rte_mempool.c | 2 +- lib/mempool/rte_mempool.h | 19 ++++++++++++------- 2 files changed, 13 insertions(+), 8 deletions(-)
lib/mempool/rte_mempool.c +1 -1
@@ -1272,7 +1272,7 @@ rte_mempool_dump(FILE *f, struct rte_mempool *mp)#ifdef RTE_LIBRTE_MEMPOOL_STATS rte_mempool_ops_get_info (mp, &info);memset (&sum,0 ,sizeof (sum)); -for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) { +for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE +1 ; lcore_id++) { sum.put_bulk += mp->stats[lcore_id].put_bulk; sum.put_objs += mp->stats[lcore_id].put_objs; sum.put_common_pool_bulk += mp->stats[lcore_id].put_common_pool_bulk;
lib/mempool/rte_mempool.h +12 -7
@@ -238,8 +238,11 @@ struct rte_mempool {struct rte_mempool_memhdr_list mem_list;/**< List of memory chunks */ #ifdef RTE_LIBRTE_MEMPOOL_STATS -/** Per-lcore statistics. */ -struct rte_mempool_debug_stats stats[RTE_MAX_LCORE ]; +/** Per-lcore statistics. +* +* Plus one, for unregistered non-EAL threads. +*/ +struct rte_mempool_debug_stats stats[RTE_MAX_LCORE +1 ];#endif } __rte_cache_aligned;
@@ -303,11 +306,13 @@ struct rte_mempool {* Number to add to the statistics. */ #ifdef RTE_LIBRTE_MEMPOOL_STATS -#define RTE_MEMPOOL_STAT_ADD (mp, name, n) do { \ - unsigned __lcore_id = rte_lcore_id(); \ - if (__lcore_id < RTE_MAX_LCORE) { \ - mp->stats[__lcore_id].name += n; \ - } \ +#define RTE_MEMPOOL_STAT_ADD (mp, name, n) do { \ + unsigned int __lcore_id = rte_lcore_id(); \ + if (likely(__lcore_id < RTE_MAX_LCORE)) \ + (mp)->stats[__lcore_id].name += n; \ + else \ + __atomic_fetch_add(&((mp)->stats[RTE_MAX_LCORE].name), \ + n, __ATOMIC_RELAXED); \ } while (0)#else #define RTE_MEMPOOL_STAT_ADD (mp, name, n) do {} while (0)
Signature · 1 line
2.17.1
PATCH v3 3/3 mempool: use cache for frequently updated stats
In reply to Morten BrørupWhen built with stats enabled (RTE_LIBRTE_MEMPOOL_STATS defined), the performance of mempools with caches is improved as follows.
When accessing objects in the mempool, either the put_bulk and put_objs or the get_success_bulk and get_success_objs statistics counters are likely to be incremented.
By adding an alternative set of these counters to the mempool cache structure, accesing the dedicated statistics structure is avoided in the likely cases where these counters are incremented.
The trick here is that the cache line holding the mempool cache structure is accessed anyway, in order to access the 'len' or 'flushthresh' fields. Updating some statistics counters in the same cache line has lower performance cost than accessing the statistics counters in the dedicated statistics structure, which resides in another cache line.
mempool_perf_autotest with this patch shows the following improvements in rate_persec.
The cost of enabling mempool stats (without debug) after this patch:
-6.8 % and -6.7 %, respectively without and with cache.
v3:
* Don't update the the description of the RTE_MEMPOOL_STAT_ADD macro. This change belongs in the first patch of the series.
v2:
* Move the statistics counters into a stats structure.
--- lib/mempool/rte_mempool.c | 9 ++++++ lib/mempool/rte_mempool.h | 67 ++++++++++++++++++++++++++++++++------- 2 files changed, 65 insertions(+), 11 deletions(-)
lib/mempool/rte_mempool.c +9 -0
@@ -1286,6 +1286,15 @@ rte_mempool_dump(FILE *f, struct rte_mempool *mp) sum.get_success_blks += mp->stats[lcore_id].get_success_blks; sum.get_fail_blks += mp->stats[lcore_id].get_fail_blks; } +if (mp->cache_size !=0 ) { +/* Add the statistics stored in the mempool caches. */ +for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) { + sum.put_bulk += mp->local_cache[lcore_id].stats.put_bulk; + sum.put_objs += mp->local_cache[lcore_id].stats.put_objs; + sum.get_success_bulk += mp->local_cache[lcore_id].stats.get_success_bulk; + sum.get_success_objs += mp->local_cache[lcore_id].stats.get_success_objs; + } + }fprintf (f," stats:\n" );fprintf (f," put_bulk=%" PRIu64"\n" , sum.put_bulk);fprintf (f," put_objs=%" PRIu64"\n" , sum.put_objs);
lib/mempool/rte_mempool.h +56 -11
111 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH v4 1/3 mempool: split stats from debug
In reply to Morten BrørupSplit stats from debug, to make mempool statistics available without the performance cost of continuously validating the debug cookies in the mempool elements.
mempool_perf_autotest shows the following improvements in rate_persec.
The cost of enabling mempool debug without this patch:
-28.1 % and -74.0 %, respectively without and with cache.
The cost of enabling mempool stats (without debug) after this patch:
-5.8 % and -21.2 %, respectively without and with cache.
v4:
* No changes.
v3:
* Update the Programmer's Guide.
* Update the description of the RTE_MEMPOOL_STAT_ADD macro. v2:
* Fix checkpatch warning:
Use C style comments in rte_include.h, not C++ style.
* Do not rename the rte_mempool_debug_stats structure.
--- config/rte_config.h | 2 ++ doc/guides/prog_guide/mempool_lib.rst | 6 +++++- lib/mempool/rte_mempool.c | 6 +++--- lib/mempool/rte_mempool.h | 11 ++++++----- 4 files changed, 16 insertions(+), 9 deletions(-)
config/rte_config.h +2 -0
@@ -47,6 +47,8 @@/* mempool defines */ #define RTE_MEMPOOL_CACHE_MAX_SIZE 512 +/* RTE_LIBRTE_MEMPOOL_STATS is not set */ +/* RTE_LIBRTE_MEMPOOL_DEBUG is not set */ /* mbuf defines */ #define RTE_MBUF_DEFAULT_MEMPOOL_OPS "ring_mp_mc"
doc/guides/prog_guide/mempool_lib.rst +5 -1
@@ -20,12 +20,16 @@ Cookies In debug mode, cookies are added at the beginning and end of allocated blocks. The allocated objects then contain overwrite protection fields to help debugging buffer overflows. +Debug mode is disabled by default, but can be enabled by setting ``RTE_LIBRTE_MEMPOOL_DEBUG`` in ``config/rte_config.h``. + Stats ----- -In debug mode, statistics about get from/put in the pool are stored in the mempool structure. +In stats mode, statistics about get from/put in the pool are stored in the mempool structure. Statistics are per-lcore to avoid concurrent access to statistics counters. +Stats mode is disabled by default, but can be enabled by setting ``RTE_LIBRTE_MEMPOOL_STATS`` in ``config/rte_config.h``. + Memory Alignment Constraints on x86 architecture ------------------------------------------------
lib/mempool/rte_mempool.c +3 -3
@@ -818,7 +818,7 @@ rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size, RTE_CACHE_LINE_MASK) != 0);RTE_BUILD_BUG_ON ((sizeof (struct rte_mempool_cache ) &RTE_CACHE_LINE_MASK ) !=0 ); -#ifdef RTE_LIBRTE_MEMPOOL_DEBUG +#ifdef RTE_LIBRTE_MEMPOOL_STATS RTE_BUILD_BUG_ON ((sizeof (struct rte_mempool_debug_stats ) &RTE_CACHE_LINE_MASK ) !=0 ); RTE_BUILD_BUG_ON((offsetof(struct rte_mempool, stats) &
@@ -1221,7 +1221,7 @@ rte_mempool_audit(struct rte_mempool *mp)void rte_mempool_dump (FILE *f,struct rte_mempool *mp) { -#ifdef RTE_LIBRTE_MEMPOOL_DEBUG +#ifdef RTE_LIBRTE_MEMPOOL_STATS struct rte_mempool_info info;struct rte_mempool_debug_stats sum;unsigned lcore_id;
@@ -1269,7 +1269,7 @@ rte_mempool_dump(FILE *f, struct rte_mempool *mp)fprintf (f," common_pool_count=%u\n" , common_count);/* sum and dump statistics */ -#ifdef RTE_LIBRTE_MEMPOOL_DEBUG +#ifdef RTE_LIBRTE_MEMPOOL_STATS rte_mempool_ops_get_info (mp, &info);memset (&sum,0 ,sizeof (sum));for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) {
lib/mempool/rte_mempool.h +6 -5
@@ -56,7 +56,7 @@ extern "C" {#define RTE_MEMPOOL_HEADER_COOKIE2 0xf2eef2eedadd2e55ULL/**< Header cookie. */ #define RTE_MEMPOOL_TRAILER_COOKIE 0xadd2e55badbadbadULL/**< Trailer cookie.*/ -#ifdef RTE_LIBRTE_MEMPOOL_DEBUG +#ifdef RTE_LIBRTE_MEMPOOL_STATS /** * A structure that stores the mempool statistics (per-lcore). * Note: Cache stats (put_cache_bulk/objs, get_cache_bulk/objs) are not
@@ -237,7 +237,7 @@ struct rte_mempool {uint32_t nb_mem_chunks;/**< Number of memory chunks */ struct rte_mempool_memhdr_list mem_list;/**< List of memory chunks */ -#ifdef RTE_LIBRTE_MEMPOOL_DEBUG +#ifdef RTE_LIBRTE_MEMPOOL_STATS /** Per-lcore statistics. */ struct rte_mempool_debug_stats stats[RTE_MAX_LCORE ];#endif
@@ -292,17 +292,18 @@ struct rte_mempool { | RTE_MEMPOOL_F_SC_GET \ | RTE_MEMPOOL_F_NO_IOVA_CONTIG \ ) +/** -* @internal When debug is enabled, store some statistics. +* @internal When stats is enabled, store some statistics. * * @param mp * Pointer to the memory pool. * @param name * Name of the statistics field to increment in the memory pool. * @param n -* Number to add to the object-oriented statistics. +* Number to add to the statistics. */ -#ifdef RTE_LIBRTE_MEMPOOL_DEBUG +#ifdef RTE_LIBRTE_MEMPOOL_STATS #define RTE_MEMPOOL_STAT_ADD (mp, name, n) do { \ unsigned __lcore_id = rte_lcore_id(); \ if (__lcore_id < RTE_MAX_LCORE) { \
Signature · 1 line
2.17.1
PATCH v4 2/3 mempool: add stats for unregistered non-EAL threads
This patch adds statistics for unregistered non-EAL threads, which was previously not included in the statistics.
Add one more entry to the stats array, and use the last index for unregistered non-EAL threads.
The unregistered non-EAL thread statistics are incremented atomically.
In theory, the EAL thread counters should also be accessed atomically to avoid tearing on 32 bit architectures. However, it was decided to avoid the performance cost of using atomic operations, because:
1. these are debug counters, and
2. statistics counters in DPDK are usually incremented non-atomically.
v4:
* No changes.
v3 (feedback from Mattias Rönnblom):
* Use correct terminology: Unregistered non-EAL threads.
* Use atomic counting for the unregistered non-EAL threads.
* Reintroduce the conditional instead of offsetting the index by one. v2:
* New. No v1 of this patch in the series.
--- lib/mempool/rte_mempool.c | 2 +- lib/mempool/rte_mempool.h | 19 ++++++++++++------- 2 files changed, 13 insertions(+), 8 deletions(-)
lib/mempool/rte_mempool.c +1 -1
@@ -1272,7 +1272,7 @@ rte_mempool_dump(FILE *f, struct rte_mempool *mp)#ifdef RTE_LIBRTE_MEMPOOL_STATS rte_mempool_ops_get_info (mp, &info);memset (&sum,0 ,sizeof (sum)); -for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) { +for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE +1 ; lcore_id++) { sum.put_bulk += mp->stats[lcore_id].put_bulk; sum.put_objs += mp->stats[lcore_id].put_objs; sum.put_common_pool_bulk += mp->stats[lcore_id].put_common_pool_bulk;
lib/mempool/rte_mempool.h +12 -7
@@ -238,8 +238,11 @@ struct rte_mempool {struct rte_mempool_memhdr_list mem_list;/**< List of memory chunks */ #ifdef RTE_LIBRTE_MEMPOOL_STATS -/** Per-lcore statistics. */ -struct rte_mempool_debug_stats stats[RTE_MAX_LCORE ]; +/** Per-lcore statistics. +* +* Plus one, for unregistered non-EAL threads. +*/ +struct rte_mempool_debug_stats stats[RTE_MAX_LCORE +1 ];#endif } __rte_cache_aligned;
@@ -304,11 +307,13 @@ struct rte_mempool {* Number to add to the statistics. */ #ifdef RTE_LIBRTE_MEMPOOL_STATS -#define RTE_MEMPOOL_STAT_ADD (mp, name, n) do { \ - unsigned __lcore_id = rte_lcore_id(); \ - if (__lcore_id < RTE_MAX_LCORE) { \ - mp->stats[__lcore_id].name += n; \ - } \ +#define RTE_MEMPOOL_STAT_ADD (mp, name, n) do { \ + unsigned int __lcore_id = rte_lcore_id(); \ + if (likely(__lcore_id < RTE_MAX_LCORE)) \ + (mp)->stats[__lcore_id].name += n; \ + else \ + __atomic_fetch_add(&((mp)->stats[RTE_MAX_LCORE].name), \ + n, __ATOMIC_RELAXED); \ } while (0)#else #define RTE_MEMPOOL_STAT_ADD (mp, name, n) do {} while (0)
Signature · 1 line
2.17.1
Re: [PATCH v4 2/3] mempool: add stats for unregistered non-EAL threads
Quoted 23 lines · Morten Brørup
This patch adds statistics for unregistered non-EAL threads, which was previously not included in the statistics.
Add one more entry to the stats array, and use the last index for unregistered non-EAL threads.
The unregistered non-EAL thread statistics are incremented atomically.
In theory, the EAL thread counters should also be accessed atomically to avoid tearing on 32 bit architectures. However, it was decided to avoid the performance cost of using atomic operations, because:
1. these are debug counters, and
2. statistics counters in DPDK are usually incremented non-atomically.v4:
* No changes.
v3 (feedback from Mattias Rönnblom):
* Use correct terminology: Unregistered non-EAL threads.
* Use atomic counting for the unregistered non-EAL threads.
* Reintroduce the conditional instead of offsetting the index by one. v2:
* New. No v1 of this patch in the series.
Quoted 2 lines
Suggested-by Stephen Hemminger <[email protected]>Signed-off-by Morten Brørup <[email protected]>
PATCH v4 3/3 mempool: use cache for frequently updated stats
In reply to Morten BrørupWhen built with stats enabled (RTE_LIBRTE_MEMPOOL_STATS defined), the performance of mempools with caches is improved as follows.
When accessing objects in the mempool, either the put_bulk and put_objs or the get_success_bulk and get_success_objs statistics counters are likely to be incremented.
By adding an alternative set of these counters to the mempool cache structure, accessing the dedicated statistics structure is avoided in the likely cases where these counters are incremented.
The trick here is that the cache line holding the mempool cache structure is accessed anyway, in order to access the 'len' or 'flushthresh' fields. Updating some statistics counters in the same cache line has lower performance cost than accessing the statistics counters in the dedicated statistics structure, which resides in another cache line.
mempool_perf_autotest with this patch shows the following improvements in rate_persec.
The cost of enabling mempool stats (without debug) after this patch:
-6.8 % and -6.7 %, respectively without and with cache.
v4:
* Fix checkpatch warnings:
A couple of typos in the patch description.
The macro to add to a mempool cache stat variable should not use do {} while (0). Personally, I would tend to disagree with this, but whatever keeps the CI happy.
v3:
* Don't update the description of the RTE_MEMPOOL_STAT_ADD macro. This change belongs in the first patch of the series.
v2:
* Move the statistics counters into a stats structure.
--- lib/mempool/rte_mempool.c | 9 ++++++ lib/mempool/rte_mempool.h | 66 ++++++++++++++++++++++++++++++++------- 2 files changed, 64 insertions(+), 11 deletions(-)
lib/mempool/rte_mempool.c +9 -0
@@ -1286,6 +1286,15 @@ rte_mempool_dump(FILE *f, struct rte_mempool *mp) sum.get_success_blks += mp->stats[lcore_id].get_success_blks; sum.get_fail_blks += mp->stats[lcore_id].get_fail_blks; } +if (mp->cache_size !=0 ) { +/* Add the statistics stored in the mempool caches. */ +for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) { + sum.put_bulk += mp->local_cache[lcore_id].stats.put_bulk; + sum.put_objs += mp->local_cache[lcore_id].stats.put_objs; + sum.get_success_bulk += mp->local_cache[lcore_id].stats.get_success_bulk; + sum.get_success_objs += mp->local_cache[lcore_id].stats.get_success_objs; + } + }fprintf (f," stats:\n" );fprintf (f," put_bulk=%" PRIu64"\n" , sum.put_bulk);fprintf (f," put_objs=%" PRIu64"\n" , sum.put_objs);
lib/mempool/rte_mempool.h +55 -11
110 lines not on this page · raw message
Signature · 1 line
2.17.1
Re: [PATCH v4 3/3] mempool: use cache for frequently updated stats
Quoted 35 lines · Morten Brørup
When built with stats enabled (
RTE_LIBRTE_MEMPOOL_STATSdefined), the performance of mempools with caches is improved as follows.When accessing objects in the mempool, either the
put_bulkandput_objsor theget_success_bulkandget_success_objsstatistics counters are likely to be incremented.By adding an alternative set of these counters to the mempool cache structure, accessing the dedicated statistics structure is avoided in the likely cases where these counters are incremented.
The trick here is that the cache line holding the mempool cache structure is accessed anyway, in order to access the 'len' or 'flushthresh' fields. Updating some statistics counters in the same cache line has lower performance cost than accessing the statistics counters in the dedicated statistics structure, which resides in another cache line.
mempool_perf_autotestwith this patch shows the following improvements inrate_persec.The cost of enabling mempool stats (without debug) after this patch:
-6.8 % and -6.7 %, respectively without and with cache.v4:
* Fix checkpatch warnings:
A couple of typos in the patch description.
The macro to add to a mempool cache stat variable should not use do {} while (0). Personally, I would tend to disagree with this, but whatever keeps the CI happy.
v3:
* Don't update the description of theRTE_MEMPOOL_STAT_ADDmacro. This change belongs in the first patch of the series.
v2:
* Move the statistics counters into a stats structure.
Quoted 1 line
Signed-off-by Morten Brørup <[email protected]>
[snip]
lib/mempool/rte_mempool.h struct rte_mempool { · line 346
Quoted 12 lines
lib/mempool/rte_mempool.h +12
-0+/** +* @internal When stats is enabled, store some statistics. +* +* @param cache +* Pointer to the memory pool cache. +* @param name +* Name of the statistics field to increment in the memory pool cache. +* @param n +* Number to add to the statistics. +*/ +#ifdef RTE_LIBRTE_MEMPOOL_STATS +#define RTE_MEMPOOL_CACHE_STAT_ADD (cache, name, n) (cache)->stats.name += n
I'd enclose it in parenthesis.
Quoted 4 lines
lib/mempool/rte_mempool.h +4
-0+#else +#define RTE_MEMPOOL_CACHE_STAT_ADD (cache, name, n) do {} while (0) +#endif +
RE: [PATCH v4 3/3] mempool: use cache for frequently updated stats
Quoted 4 lines
From: Andrew Rybchenko [mailto:[email protected]]
Sent: Sunday, 6 November 2022 12.41On 11/4/22 15:03, Morten Brørup wrote:
[...]
Quoted 8 lines
Quoted 7 lines
+/**
+ * @internal When stats is enabled, store some statistics.
+ *
+ * @param cache
+ * Pointer to the memory pool cache.
+ * @param name
+ * Name of the statistics field to increment in the memory poolcache.
Quoted 8 lines
Quoted 6 lines
+ * @param n
+ * Number to add to the statistics.
+ */(quoted diff) +2
-0+#ifdef RTE_LIBRTE_MEMPOOL_STATS +#define RTE_MEMPOOL_CACHE_STAT_ADD(cache, name, n) (cache)-stats.name += n
I'd enclose it in parenthesis.
Me too! I had it surrounded by "do {...} while (0)" in v3, but checkpatch complained about it [1], so I changed it to the above. Which checkpatch also complains about. :-(
[1]: http://mails.dpdk.org/archives/test-report/2022-November/321316.html
Feel free to modify this macro at your preference when merging!
Quoted 5 lines
Quoted 5 lines
lib/mempool/rte_mempool.h +4
-0+#else +#define RTE_MEMPOOL_CACHE_STAT_ADD (cache, name, n) do {} while (0) +#endif +
Re: [PATCH v4 3/3] mempool: use cache for frequently updated stats
Quoted 24 lines · Morten Brørup
Quoted 4 lines
From: Andrew Rybchenko [mailto:[email protected]]
Sent: Sunday, 6 November 2022 12.41On 11/4/22 15:03, Morten Brørup wrote:
[...]
Quoted 8 lines
Quoted 7 lines
+/**
+ * @internal When stats is enabled, store some statistics.
+ *
+ * @param cache
+ * Pointer to the memory pool cache.
+ * @param name
+ * Name of the statistics field to increment in the memory poolcache.
Quoted 8 lines
Quoted 6 lines
+ * @param n
+ * Number to add to the statistics.
+ */(quoted diff) +2
-0+#ifdef RTE_LIBRTE_MEMPOOL_STATS +#define RTE_MEMPOOL_CACHE_STAT_ADD(cache, name, n) (cache)-stats.name += n
I'd enclose it in parenthesis.
Quoted 1 line
Me too! I had it surrounded by "do {...} while (0)" in v3, but checkpatch complained about it [1], so I changed it to the above. Which checkpatch also complains about. :-(
I mean
#define RTE_MEMPOOL_CACHE_STAT_ADD(cache, name, n) \
((cache)->stats.name += (n))
Quoted 2 lines
[1]: http://mails.dpdk.org/archives/test-report/2022-November/321316.html
Yes, I've seen it.
Quoted 9 lines
Feel free to modify this macro at your preference when merging!
Quoted 5 lines
Quoted 5 lines
lib/mempool/rte_mempool.h +4
-0+#else +#define RTE_MEMPOOL_CACHE_STAT_ADD (cache, name, n) do {} while (0) +#endif +
RE: [PATCH v4 3/3] mempool: use cache for frequently updated stats
Quoted 28 lines
From: Andrew Rybchenko [mailto:[email protected]]
Sent: Sunday, 6 November 2022 13.00Quoted 24 lines · Morten Brørup
Quoted 4 lines
From: Andrew Rybchenko [mailto:[email protected]]
Sent: Sunday, 6 November 2022 12.41On 11/4/22 15:03, Morten Brørup wrote:
[...]
Quoted 16 lines
Quoted 7 lines
+/**
+ * @internal When stats is enabled, store some statistics.
+ *
+ * @param cache
+ * Pointer to the memory pool cache.
+ * @param name
+ * Name of the statistics field to increment in the memory poolcache.
Quoted 6 lines
+ * @param n
+ * Number to add to the statistics.
+ */(quoted diff) +2
-0+#ifdef RTE_LIBRTE_MEMPOOL_STATS +#define RTE_MEMPOOL_CACHE_STAT_ADD(cache, name, n) (cache)-stats.name += n
I'd enclose it in parenthesis.
Quoted 7 lines
Quoted 1 line
Me too! I had it surrounded by "do {...} while (0)" in v3, but
checkpatch complained about it [1], so I changed it to the above. Which checkpatch also complains about. :-(
I mean
#defineRTE_MEMPOOL_CACHE_STAT_ADD(cache, name, n)\
((cache)->stats.name += (n))
Thank you for elaborating, Andrew. I guess this is exactly what the checkpatch error message for v4 [2] says - I just couldn't understand what it wanted me to do.
[2]: http://mails.dpdk.org/archives/test-report/2022-November/321345.html
I don't always agree with checkpatch, but now Thomas has a third option for how to write this macro when merging. :-)
Quoted 15 lines
Quoted 3 lines
November/321316.html
Yes, I've seen it.
Quoted 8 lines
Feel free to modify this macro at your preference when merging!
Quoted 5 lines
Quoted 5 lines
lib/mempool/rte_mempool.h +4
-0+#else +#define RTE_MEMPOOL_CACHE_STAT_ADD (cache, name, n) do {} while (0) +#endif +
Re: [PATCH v4 3/3] mempool: use cache for frequently updated stats
In reply to Morten BrørupQuoted 172 lines · Morten Brørup
When built with stats enabled (
RTE_LIBRTE_MEMPOOL_STATSdefined), the performance of mempools with caches is improved as follows.When accessing objects in the mempool, either the
put_bulkandput_objsor theget_success_bulkandget_success_objsstatistics counters are likely to be incremented.By adding an alternative set of these counters to the mempool cache structure, accessing the dedicated statistics structure is avoided in the likely cases where these counters are incremented.
The trick here is that the cache line holding the mempool cache structure is accessed anyway, in order to access the 'len' or 'flushthresh' fields. Updating some statistics counters in the same cache line has lower performance cost than accessing the statistics counters in the dedicated statistics structure, which resides in another cache line.
mempool_perf_autotestwith this patch shows the following improvements inrate_persec.The cost of enabling mempool stats (without debug) after this patch:
-6.8 % and -6.7 %, respectively without and with cache.v4:
* Fix checkpatch warnings:
A couple of typos in the patch description.
The macro to add to a mempool cache stat variable should not use do {} while (0). Personally, I would tend to disagree with this, but whatever keeps the CI happy.
v3:
* Don't update the description of theRTE_MEMPOOL_STAT_ADDmacro. This change belongs in the first patch of the series.
v2:
* Move the statistics counters into a stats structure.Signed-off-by Morten Brørup <[email protected]>--- lib/mempool/rte_mempool.c | 9 ++++++ lib/mempool/rte_mempool.h | 66 ++++++++++++++++++++++++++++++++------- 2 files changed, 64 insertions(+), 11 deletions(-)lib/mempool/rte_mempool.c +9
-0@@ -1286,6 +1286,15 @@ rte_mempool_dump(FILE *f, struct rte_mempool *mp) sum.get_success_blks += mp->stats[lcore_id].get_success_blks; sum.get_fail_blks += mp->stats[lcore_id].get_fail_blks; } +if (mp->cache_size !=0 ) { +/* Add the statistics stored in the mempool caches. */ +for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) { + sum.put_bulk += mp->local_cache[lcore_id].stats.put_bulk; + sum.put_objs += mp->local_cache[lcore_id].stats.put_objs; + sum.get_success_bulk += mp->local_cache[lcore_id].stats.get_success_bulk; + sum.get_success_objs += mp->local_cache[lcore_id].stats.get_success_objs; + } + }fprintf (f," stats:\n" );fprintf (f," put_bulk=%" PRIu64"\n" , sum.put_bulk);fprintf (f," put_objs=%" PRIu64"\n" , sum.put_objs);lib/mempool/rte_mempool.h +48
-11@@ -86,6 +86,19 @@ struct rte_mempool_cache {uint32_t size;/**< Size of the cache */ uint32_t flushthresh;/**< Threshold before we flush excess elements */ uint32_t len;/**< Current cache count */ +#ifdef RTE_LIBRTE_MEMPOOL_STATS +uint32_t unused; +/* +* Alternative location for the most frequently updated mempool statistics (per-lcore), +* providing faster update access when using a mempool cache. +*/ +struct { +uint64_t put_bulk;/**< Number of puts. */ +uint64_t put_objs;/**< Number of objects successfully put. */ +uint64_t get_success_bulk;/**< Successful allocation number. */ +uint64_t get_success_objs;/**< Objects successfully allocated. */ + } stats;/**< Statistics */ +#endif /** * Cache objects * @@ -319,6 +332,22 @@ struct rte_mempool {#define RTE_MEMPOOL_STAT_ADD (mp, name, n) do {} while (0)#endif +/** +* @internal When stats is enabled, store some statistics. +* +* @param cache +* Pointer to the memory pool cache. +* @param name +* Name of the statistics field to increment in the memory pool cache. +* @param n +* Number to add to the statistics. +*/ +#ifdef RTE_LIBRTE_MEMPOOL_STATS +#define RTE_MEMPOOL_CACHE_STAT_ADD (cache, name, n) (cache)->stats.name += n +#else +#define RTE_MEMPOOL_CACHE_STAT_ADD (cache, name, n) do {} while (0) +#endif +/** * @internal Calculate the size of the mempool header. * @@ -1333,13 +1362,17 @@ rte_mempool_do_generic_put(struct rte_mempool *mp, void * const *obj_table, {void **cache_objs; +/* No cache provided */ +if (unlikely (cache ==NULL )) + goto driver_enqueue; +/* increment stat now, adding in mempool always success */ -RTE_MEMPOOL_STAT_ADD (mp, put_bulk,1 ); -RTE_MEMPOOL_STAT_ADD (mp, put_objs, n); +RTE_MEMPOOL_CACHE_STAT_ADD (cache, put_bulk,1 ); +RTE_MEMPOOL_CACHE_STAT_ADD (cache, put_objs, n); -/* No cache provided or the request itself is too big for the cache */ -if (unlikely (cache ==NULL || n > cache->flushthresh)) - goto driver_enqueue; +/* The request itself is too big for the cache */ +if (unlikely (n > cache->flushthresh)) + goto driver_enqueue_stats_incremented;/* * The cache follows the following algorithm: @@ -1364,6 +1397,12 @@ rte_mempool_do_generic_put(struct rte_mempool *mp, void * const *obj_table, driver_enqueue: +/* increment stat now, adding in mempool always success */ +RTE_MEMPOOL_STAT_ADD (mp, put_bulk,1 ); +RTE_MEMPOOL_STAT_ADD (mp, put_objs, n); + +driver_enqueue_stats_incremented: +/* push objects to the backend */ rte_mempool_ops_enqueue_bulk (mp, obj_table, n); }@@ -1470,8 +1509,8 @@ rte_mempool_do_generic_get(struct rte_mempool *mp, void **obj_table,if (remaining ==0 ) {/* The entire request is satisfied from the cache. */ -RTE_MEMPOOL_STAT_ADD (mp, get_success_bulk,1 ); -RTE_MEMPOOL_STAT_ADD (mp, get_success_objs, n); +RTE_MEMPOOL_CACHE_STAT_ADD (cache, get_success_bulk,1 ); +RTE_MEMPOOL_CACHE_STAT_ADD (cache, get_success_objs, n);return 0 ; }@@ -1500,8 +1539,8 @@ rte_mempool_do_generic_get(struct rte_mempool *mp, void **obj_table, cache->len = cache->size; -RTE_MEMPOOL_STAT_ADD (mp, get_success_bulk,1 ); -RTE_MEMPOOL_STAT_ADD (mp, get_success_objs, n); +RTE_MEMPOOL_CACHE_STAT_ADD (cache, get_success_bulk,1 ); +RTE_MEMPOOL_CACHE_STAT_ADD (cache, get_success_objs, n);return 0 ;@@ -1523,8 +1562,13 @@ rte_mempool_do_generic_get(struct rte_mempool *mp, void **obj_table,RTE_MEMPOOL_STAT_ADD (mp, get_fail_bulk,1 );RTE_MEMPOOL_STAT_ADD (mp, get_fail_objs, n); }else { -RTE_MEMPOOL_STAT_ADD (mp, get_success_bulk,1 ); -RTE_MEMPOOL_STAT_ADD (mp, get_success_objs, n);
lib/mempool/rte_mempool.h rte_mempool_do_generic_get(struct rte_mempool *mp, void **obj_table, · line 1574
Quoted 10 lines
lib/mempool/rte_mempool.h +7
-0+if (likely (cache !=NULL )) { +RTE_MEMPOOL_CACHE_STAT_ADD (cache, get_success_bulk,1 ); +RTE_MEMPOOL_CACHE_STAT_ADD (cache, get_success_objs, n); + }else { +RTE_MEMPOOL_STAT_ADD (mp, get_success_bulk,1 ); +RTE_MEMPOOL_STAT_ADD (mp, get_success_objs, n); + } }return ret;
For the series,
RE: [PATCH v4 3/3] mempool: use cache for frequently updated stats
In reply to Morten BrørupQuoted 92 lines
When built with stats enabled (
RTE_LIBRTE_MEMPOOL_STATSdefined), the performance of mempools with caches is improved as follows.When accessing objects in the mempool, either the
put_bulkandput_objsor theget_success_bulkandget_success_objsstatistics counters are likely to be incremented.By adding an alternative set of these counters to the mempool cache structure, accessing the dedicated statistics structure is avoided in the likely cases where these counters are incremented.
The trick here is that the cache line holding the mempool cache structure is accessed anyway, in order to access the 'len' or 'flushthresh' fields. Updating some statistics counters in the same cache line has lower performance cost than accessing the statistics counters in the dedicated statistics structure, which resides in another cache line.
mempool_perf_autotestwith this patch shows the following improvements inrate_persec.The cost of enabling mempool stats (without debug) after this patch:
-6.8 % and -6.7 %, respectively without and with cache.v4:
* Fix checkpatch warnings:
A couple of typos in the patch description.
The macro to add to a mempool cache stat variable should not use do {} while (0). Personally, I would tend to disagree with this, but whatever keeps the CI happy.
v3:
* Don't update the description of theRTE_MEMPOOL_STAT_ADDmacro. This change belongs in the first patch of the series.
v2:
* Move the statistics counters into a stats structure.Signed-off-by Morten Brørup <[email protected]>--- lib/mempool/rte_mempool.c | 9 ++++++ lib/mempool/rte_mempool.h | 66 ++++++++++++++++++++++++++++++++------- 2 files changed, 64 insertions(+), 11 deletions(-)lib/mempool/rte_mempool.c +9
-0@@ -1286,6 +1286,15 @@ rte_mempool_dump(FILE *f, struct rte_mempool *mp) sum.get_success_blks += mp->stats[lcore_id].get_success_blks; sum.get_fail_blks += mp->stats[lcore_id].get_fail_blks; } +if (mp->cache_size !=0 ) { +/* Add the statistics stored in the mempool caches. */ +for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) { + sum.put_bulk += mp->local_cache[lcore_id].stats.put_bulk; + sum.put_objs += mp->local_cache[lcore_id].stats.put_objs; + sum.get_success_bulk += mp->local_cache[lcore_id].stats.get_success_bulk; + sum.get_success_objs += mp->local_cache[lcore_id].stats.get_success_objs; + } + }fprintf (f," stats:\n" );fprintf (f," put_bulk=%" PRIu64"\n" , sum.put_bulk);fprintf (f," put_objs=%" PRIu64"\n" , sum.put_objs);lib/mempool/rte_mempool.h +15
-0@@ -86,6 +86,19 @@ struct rte_mempool_cache {uint32_t size;/**< Size of the cache */ uint32_t flushthresh;/**< Threshold before we flush excess elements */ uint32_t len;/**< Current cache count */ +#ifdef RTE_LIBRTE_MEMPOOL_STATS +uint32_t unused; +/* +* Alternative location for the most frequently updated mempool statistics (per-lcore), +* providing faster update access when using a mempool cache. +*/ +struct { +uint64_t put_bulk;/**< Number of puts. */ +uint64_t put_objs;/**< Number of objects successfully put. */ +uint64_t get_success_bulk;/**< Successful allocation number. */ +uint64_t get_success_objs;/**< Objects successfully allocated. */ + } stats;/**< Statistics */ +#endif /** * Cache objects * @@ -319,6 +332,22 @@ struct rte_mempool {#define RTE_MEMPOOL_STAT_ADD (mp, name, n) do {} while (0)#endif +/** +* @internal When stats is enabled, store some statistics.
lib/mempool/rte_mempool.h struct rte_mempool { · line 346
Quoted 10 lines
lib/mempool/rte_mempool.h +10
-0+* +* @param cache +* Pointer to the memory pool cache. +* @param name +* Name of the statistics field to increment in the memory pool cache. +* @param n +* Number to add to the statistics. +*/ +#ifdef RTE_LIBRTE_MEMPOOL_STATS +#define RTE_MEMPOOL_CACHE_STAT_ADD (cache, name, n) (cache)->stats.name += n
As Andrew already pointed, it needs to be: ((cache)->stats.name += (n)) Apart from that, LGTM.
Quoted 84 lines
lib/mempool/rte_mempool.h +30
-12+#else +#define RTE_MEMPOOL_CACHE_STAT_ADD (cache, name, n) do {} while (0) +#endif +/** * @internal Calculate the size of the mempool header. * @@ -1333,13 +1362,17 @@ rte_mempool_do_generic_put(struct rte_mempool *mp, void * const *obj_table, {void **cache_objs; +/* No cache provided */ +if (unlikely (cache ==NULL )) + goto driver_enqueue; +/* increment stat now, adding in mempool always success */ -RTE_MEMPOOL_STAT_ADD (mp, put_bulk,1 ); -RTE_MEMPOOL_STAT_ADD (mp, put_objs, n); +RTE_MEMPOOL_CACHE_STAT_ADD (cache, put_bulk,1 ); +RTE_MEMPOOL_CACHE_STAT_ADD (cache, put_objs, n); -/* No cache provided or the request itself is too big for the cache */ -if (unlikely (cache ==NULL || n > cache->flushthresh)) - goto driver_enqueue; +/* The request itself is too big for the cache */ +if (unlikely (n > cache->flushthresh)) + goto driver_enqueue_stats_incremented;/* * The cache follows the following algorithm: @@ -1364,6 +1397,12 @@ rte_mempool_do_generic_put(struct rte_mempool *mp, void * const *obj_table, driver_enqueue: +/* increment stat now, adding in mempool always success */ +RTE_MEMPOOL_STAT_ADD (mp, put_bulk,1 ); +RTE_MEMPOOL_STAT_ADD (mp, put_objs, n); + +driver_enqueue_stats_incremented: +/* push objects to the backend */ rte_mempool_ops_enqueue_bulk (mp, obj_table, n); }@@ -1470,8 +1509,8 @@ rte_mempool_do_generic_get(struct rte_mempool *mp, void **obj_table,if (remaining ==0 ) {/* The entire request is satisfied from the cache. */ -RTE_MEMPOOL_STAT_ADD (mp, get_success_bulk,1 ); -RTE_MEMPOOL_STAT_ADD (mp, get_success_objs, n); +RTE_MEMPOOL_CACHE_STAT_ADD (cache, get_success_bulk,1 ); +RTE_MEMPOOL_CACHE_STAT_ADD (cache, get_success_objs, n);return 0 ; }@@ -1500,8 +1539,8 @@ rte_mempool_do_generic_get(struct rte_mempool *mp, void **obj_table, cache->len = cache->size; -RTE_MEMPOOL_STAT_ADD (mp, get_success_bulk,1 ); -RTE_MEMPOOL_STAT_ADD (mp, get_success_objs, n); +RTE_MEMPOOL_CACHE_STAT_ADD (cache, get_success_bulk,1 ); +RTE_MEMPOOL_CACHE_STAT_ADD (cache, get_success_objs, n);return 0 ;@@ -1523,8 +1562,13 @@ rte_mempool_do_generic_get(struct rte_mempool *mp, void **obj_table,RTE_MEMPOOL_STAT_ADD (mp, get_fail_bulk,1 );RTE_MEMPOOL_STAT_ADD (mp, get_fail_objs, n); }else { -RTE_MEMPOOL_STAT_ADD (mp, get_success_bulk,1 ); -RTE_MEMPOOL_STAT_ADD (mp, get_success_objs, n); +if (likely (cache !=NULL )) { +RTE_MEMPOOL_CACHE_STAT_ADD (cache, get_success_bulk,1 ); +RTE_MEMPOOL_CACHE_STAT_ADD (cache, get_success_objs, n); + }else { +RTE_MEMPOOL_STAT_ADD (mp, get_success_bulk,1 ); +RTE_MEMPOOL_STAT_ADD (mp, get_success_objs, n); + } }return ret; --2.17.1
Morten Brørup
Reviewed-by (mempool maintainer) Andrew Rybchenko Reviewed-by Mattias Rönnblom Acked-by Konstantin AnanyevRE: [PATCH v4 3/3] mempool: use cache for frequently updated stats
Quoted 48 lines
From: Konstantin Ananyev [mailto:[email protected]]
Sent: Tuesday, 8 November 2022 10.20Quoted 4 lines
When built with stats enabled (
RTE_LIBRTE_MEMPOOL_STATSdefined), the performance of mempools with caches is improved as follows.When accessing objects in the mempool, either the
put_bulkand
put_objsorQuoted 1 line
the
get_success_bulkandget_success_objsstatistics counters arelikely
Quoted 4 lines
to be incremented.
By adding an alternative set of these counters to the mempool cache structure, accessing the dedicated statistics structure is avoided in
the
Quoted 3 lines
likely cases where these counters are incremented.
The trick here is that the cache line holding the mempool cache
structure
Quoted 1 line
is accessed anyway, in order to access the 'len' or 'flushthresh'
fields.
Quoted 2 lines
Updating some statistics counters in the same cache line has lower performance cost than accessing the statistics counters in the
dedicated
Quoted 3 lines
statistics structure, which resides in another cache line.
mempool_perf_autotestwith this patch shows the followingimprovements in
Quoted 10 lines
rate_persec.The cost of enabling mempool stats (without debug) after this patch:
-6.8 % and -6.7 %, respectively without and with cache.v4:
* Fix checkpatch warnings:
A couple of typos in the patch description.
The macro to add to a mempool cache stat variable should not use do {} while (0). Personally, I would tend to disagree with this,but
Quoted 9 lines
whatever keeps the CI happy.
v3:
* Don't update the description of theRTE_MEMPOOL_STAT_ADDmacro. This change belongs in the first patch of the series.
v2:
* Move the statistics counters into a stats structure.Signed-off-by Morten Brørup <[email protected]>---
[...]
Quoted 16 lines
Quoted 7 lines
+/**
+ * @internal When stats is enabled, store some statistics.
+ *
+ * @param cache
+ * Pointer to the memory pool cache.
+ * @param name
+ * Name of the statistics field to increment in the memory poolcache.
lib/mempool/rte_mempool.h struct rte_mempool { · line 346
Quoted 5 lines
+ * @param n
+ * Number to add to the statistics.
+ */lib/mempool/rte_mempool.h +2
-0+#ifdef RTE_LIBRTE_MEMPOOL_STATS +#define RTE_MEMPOOL_CACHE_STAT_ADD (cache, name, n) (cache)->stats.name += nAs Andrew already pointed, it needs to be: ((cache)->stats.name += (n)) Apart from that, LGTM.
Quoted 1 line
Series-Acked-by Konstantin Ananyev <[email protected]>
@Thomas, this series should be ready to apply... it now has been:
Please fix the RTE_MEMPOOL_CACHE_STAT_ADD macro while merging, to satisfy checkpatch. ;-)
It should be:
+#ifdef RTE_LIBRTE_MEMPOOL_STATS
+#define RTE_MEMPOOL_CACHE_STAT_ADD(cache, name, n) ((cache)->stats.name += (n)) +#else
+#define RTE_MEMPOOL_CACHE_STAT_ADD(cache, name, n) do {} while (0) +#endif
Re: [PATCH v4 1/3] mempool: split stats from debug
In reply to Morten BrørupQuoted 22 lines · Morten Brørup
Split stats from debug, to make mempool statistics available without the performance cost of continuously validating the debug cookies in the mempool elements.
mempool_perf_autotestshows the following improvements inrate_persec.The cost of enabling mempool debug without this patch:
-28.1 % and -74.0 %, respectively without and with cache.The cost of enabling mempool stats (without debug) after this patch:
-5.8 % and -21.2 %, respectively without and with cache.v4:
* No changes.
v3:
* Update the Programmer's Guide.
* Update the description of theRTE_MEMPOOL_STAT_ADDmacro. v2:
* Fix checkpatch warning:
Use C style comments inrte_include.h, not C++ style.
* Do not rename therte_mempool_debug_statsstructure.
Quoted 1 line
Signed-off-by Morten Brørup <[email protected]>
Good idea, many thanks for the patch.
PATCH v5 1/3 mempool: split stats from debug
In reply to Morten BrørupSplit stats from debug, to make mempool statistics available without the performance cost of continuously validating the debug cookies in the mempool elements.
mempool_perf_autotest shows the following improvements in rate_persec.
The cost of enabling mempool debug without this patch:
-28.1 % and -74.0 %, respectively without and with cache.
The cost of enabling mempool stats (without debug) after this patch:
-5.8 % and -21.2 %, respectively without and with cache.
v5:
* No changes.
v4:
* No changes.
v3:
* Update the Programmer's Guide.
* Update the description of the RTE_MEMPOOL_STAT_ADD macro. v2:
* Fix checkpatch warning:
Use C style comments in rte_include.h, not C++ style.
* Do not rename the rte_mempool_debug_stats structure.
--- config/rte_config.h | 2 ++ doc/guides/prog_guide/mempool_lib.rst | 6 +++++- lib/mempool/rte_mempool.c | 6 +++--- lib/mempool/rte_mempool.h | 11 ++++++----- 4 files changed, 16 insertions(+), 9 deletions(-)
config/rte_config.h +2 -0
@@ -47,6 +47,8 @@/* mempool defines */ #define RTE_MEMPOOL_CACHE_MAX_SIZE 512 +/* RTE_LIBRTE_MEMPOOL_STATS is not set */ +/* RTE_LIBRTE_MEMPOOL_DEBUG is not set */ /* mbuf defines */ #define RTE_MBUF_DEFAULT_MEMPOOL_OPS "ring_mp_mc"
doc/guides/prog_guide/mempool_lib.rst +5 -1
@@ -20,12 +20,16 @@ Cookies In debug mode, cookies are added at the beginning and end of allocated blocks. The allocated objects then contain overwrite protection fields to help debugging buffer overflows. +Debug mode is disabled by default, but can be enabled by setting ``RTE_LIBRTE_MEMPOOL_DEBUG`` in ``config/rte_config.h``. + Stats ----- -In debug mode, statistics about get from/put in the pool are stored in the mempool structure. +In stats mode, statistics about get from/put in the pool are stored in the mempool structure. Statistics are per-lcore to avoid concurrent access to statistics counters. +Stats mode is disabled by default, but can be enabled by setting ``RTE_LIBRTE_MEMPOOL_STATS`` in ``config/rte_config.h``. + Memory Alignment Constraints on x86 architecture ------------------------------------------------
lib/mempool/rte_mempool.c +3 -3
@@ -818,7 +818,7 @@ rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size, RTE_CACHE_LINE_MASK) != 0);RTE_BUILD_BUG_ON ((sizeof (struct rte_mempool_cache ) &RTE_CACHE_LINE_MASK ) !=0 ); -#ifdef RTE_LIBRTE_MEMPOOL_DEBUG +#ifdef RTE_LIBRTE_MEMPOOL_STATS RTE_BUILD_BUG_ON ((sizeof (struct rte_mempool_debug_stats ) &RTE_CACHE_LINE_MASK ) !=0 ); RTE_BUILD_BUG_ON((offsetof(struct rte_mempool, stats) &
@@ -1221,7 +1221,7 @@ rte_mempool_audit(struct rte_mempool *mp)void rte_mempool_dump (FILE *f,struct rte_mempool *mp) { -#ifdef RTE_LIBRTE_MEMPOOL_DEBUG +#ifdef RTE_LIBRTE_MEMPOOL_STATS struct rte_mempool_info info;struct rte_mempool_debug_stats sum;unsigned lcore_id;
@@ -1269,7 +1269,7 @@ rte_mempool_dump(FILE *f, struct rte_mempool *mp)fprintf (f," common_pool_count=%u\n" , common_count);/* sum and dump statistics */ -#ifdef RTE_LIBRTE_MEMPOOL_DEBUG +#ifdef RTE_LIBRTE_MEMPOOL_STATS rte_mempool_ops_get_info (mp, &info);memset (&sum,0 ,sizeof (sum));for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) {
lib/mempool/rte_mempool.h +6 -5
@@ -56,7 +56,7 @@ extern "C" {#define RTE_MEMPOOL_HEADER_COOKIE2 0xf2eef2eedadd2e55ULL/**< Header cookie. */ #define RTE_MEMPOOL_TRAILER_COOKIE 0xadd2e55badbadbadULL/**< Trailer cookie.*/ -#ifdef RTE_LIBRTE_MEMPOOL_DEBUG +#ifdef RTE_LIBRTE_MEMPOOL_STATS /** * A structure that stores the mempool statistics (per-lcore). * Note: Cache stats (put_cache_bulk/objs, get_cache_bulk/objs) are not
@@ -237,7 +237,7 @@ struct rte_mempool {uint32_t nb_mem_chunks;/**< Number of memory chunks */ struct rte_mempool_memhdr_list mem_list;/**< List of memory chunks */ -#ifdef RTE_LIBRTE_MEMPOOL_DEBUG +#ifdef RTE_LIBRTE_MEMPOOL_STATS /** Per-lcore statistics. */ struct rte_mempool_debug_stats stats[RTE_MAX_LCORE ];#endif
@@ -292,17 +292,18 @@ struct rte_mempool { | RTE_MEMPOOL_F_SC_GET \ | RTE_MEMPOOL_F_NO_IOVA_CONTIG \ ) +/** -* @internal When debug is enabled, store some statistics. +* @internal When stats is enabled, store some statistics. * * @param mp * Pointer to the memory pool. * @param name * Name of the statistics field to increment in the memory pool. * @param n -* Number to add to the object-oriented statistics. +* Number to add to the statistics. */ -#ifdef RTE_LIBRTE_MEMPOOL_DEBUG +#ifdef RTE_LIBRTE_MEMPOOL_STATS #define RTE_MEMPOOL_STAT_ADD (mp, name, n) do { \ unsigned __lcore_id = rte_lcore_id(); \ if (__lcore_id < RTE_MAX_LCORE) { \
Signature · 1 line
2.17.1
PATCH v5 2/3 mempool: add stats for unregistered non-EAL threads
This patch adds statistics for unregistered non-EAL threads, which was previously not included in the statistics.
Add one more entry to the stats array, and use the last index for unregistered non-EAL threads.
The unregistered non-EAL thread statistics are incremented atomically.
In theory, the EAL thread counters should also be accessed atomically to avoid tearing on 32 bit architectures. However, it was decided to avoid the performance cost of using atomic operations, because:
1. these are debug counters, and
2. statistics counters in DPDK are usually incremented non-atomically.
v5:
* Add parentheses around n in the RTE_MEMPOOL_STAT_ADD macro. v4:
* No changes.
v3 (feedback from Mattias Rönnblom):
* Use correct terminology: Unregistered non-EAL threads.
* Use atomic counting for the unregistered non-EAL threads.
* Reintroduce the conditional instead of offsetting the index by one. v2:
* New. No v1 of this patch in the series.
--- lib/mempool/rte_mempool.c | 2 +- lib/mempool/rte_mempool.h | 19 ++++++++++++------- 2 files changed, 13 insertions(+), 8 deletions(-)
lib/mempool/rte_mempool.c +1 -1
@@ -1272,7 +1272,7 @@ rte_mempool_dump(FILE *f, struct rte_mempool *mp)#ifdef RTE_LIBRTE_MEMPOOL_STATS rte_mempool_ops_get_info (mp, &info);memset (&sum,0 ,sizeof (sum)); -for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) { +for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE +1 ; lcore_id++) { sum.put_bulk += mp->stats[lcore_id].put_bulk; sum.put_objs += mp->stats[lcore_id].put_objs; sum.put_common_pool_bulk += mp->stats[lcore_id].put_common_pool_bulk;
lib/mempool/rte_mempool.h +12 -7
@@ -238,8 +238,11 @@ struct rte_mempool {struct rte_mempool_memhdr_list mem_list;/**< List of memory chunks */ #ifdef RTE_LIBRTE_MEMPOOL_STATS -/** Per-lcore statistics. */ -struct rte_mempool_debug_stats stats[RTE_MAX_LCORE ]; +/** Per-lcore statistics. +* +* Plus one, for unregistered non-EAL threads. +*/ +struct rte_mempool_debug_stats stats[RTE_MAX_LCORE +1 ];#endif } __rte_cache_aligned;
@@ -304,11 +307,13 @@ struct rte_mempool {* Number to add to the statistics. */ #ifdef RTE_LIBRTE_MEMPOOL_STATS -#define RTE_MEMPOOL_STAT_ADD (mp, name, n) do { \ - unsigned __lcore_id = rte_lcore_id(); \ - if (__lcore_id < RTE_MAX_LCORE) { \ - mp->stats[__lcore_id].name += n; \ - } \ +#define RTE_MEMPOOL_STAT_ADD (mp, name, n) do { \ + unsigned int __lcore_id = rte_lcore_id(); \ + if (likely(__lcore_id < RTE_MAX_LCORE)) \ + (mp)->stats[__lcore_id].name += (n); \ + else \ + __atomic_fetch_add(&((mp)->stats[RTE_MAX_LCORE].name), \ + (n), __ATOMIC_RELAXED); \ } while (0)#else #define RTE_MEMPOOL_STAT_ADD (mp, name, n) do {} while (0)
Signature · 1 line
2.17.1
PATCH v5 3/3 mempool: use cache for frequently updated stats
In reply to Morten BrørupWhen built with stats enabled (RTE_LIBRTE_MEMPOOL_STATS defined), the performance of mempools with caches is improved as follows.
When accessing objects in the mempool, either the put_bulk and put_objs or the get_success_bulk and get_success_objs statistics counters are likely to be incremented.
By adding an alternative set of these counters to the mempool cache structure, accessing the dedicated statistics structure is avoided in the likely cases where these counters are incremented.
The trick here is that the cache line holding the mempool cache structure is accessed anyway, in order to access the 'len' or 'flushthresh' fields. Updating some statistics counters in the same cache line has lower performance cost than accessing the statistics counters in the dedicated statistics structure, which resides in another cache line.
mempool_perf_autotest with this patch shows the following improvements in rate_persec.
The cost of enabling mempool stats (without debug) after this patch:
-6.8 % and -6.7 %, respectively without and with cache.
v5:
* Add SmartShare Systems to the list of copyright holders.
* Add parentheses around n in the RTE_MEMPOOL_CACHE_STAT_ADD macro.
* Fix checkpatch warning:
The macro to add to a mempool cache stat variable should be enclosed in parentheses.
v4:
* Fix checkpatch warnings:
A couple of typos in the patch description.
The macro to add to a mempool cache stat variable should not use do {} while (0). Personally, I would tend to disagree with this, but whatever keeps the CI happy.
v3:
* Don't update the description of the RTE_MEMPOOL_STAT_ADD macro. This change belongs in the first patch of the series.
v2:
* Move the statistics counters into a stats structure.
--- lib/mempool/rte_mempool.c | 10 ++++++ lib/mempool/rte_mempool.h | 67 ++++++++++++++++++++++++++++++++------- 2 files changed, 66 insertions(+), 11 deletions(-)
lib/mempool/rte_mempool.c +10 -0
@@ -1,6 +1,7 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation. * Copyright(c) 2016 6WIND S.A. +* Copyright(c) 2022 SmartShare Systems */ #include <stdbool.h>
@@ -1286,6 +1287,15 @@ rte_mempool_dump(FILE *f, struct rte_mempool *mp) sum.get_success_blks += mp->stats[lcore_id].get_success_blks; sum.get_fail_blks += mp->stats[lcore_id].get_fail_blks; } +if (mp->cache_size !=0 ) { +/* Add the statistics stored in the mempool caches. */ +for (lcore_id =0 ; lcore_id <RTE_MAX_LCORE ; lcore_id++) { + sum.put_bulk += mp->local_cache[lcore_id].stats.put_bulk; + sum.put_objs += mp->local_cache[lcore_id].stats.put_objs; + sum.get_success_bulk += mp->local_cache[lcore_id].stats.get_success_bulk; + sum.get_success_objs += mp->local_cache[lcore_id].stats.get_success_objs; + } + }fprintf (f," stats:\n" );fprintf (f," put_bulk=%" PRIu64"\n" , sum.put_bulk);fprintf (f," put_objs=%" PRIu64"\n" , sum.put_objs);
lib/mempool/rte_mempool.h +56 -11
117 lines not on this page · raw message
Signature · 1 line
2.17.1
Re: [PATCH v5 3/3] mempool: use cache for frequently updated stats
09/11/2022 19:18, Morten Brørup:
Quoted 2 lines
When built with stats enabled (
RTE_LIBRTE_MEMPOOL_STATSdefined), the performance of mempools with caches is improved as follows.
Series applied, thanks.