Skip to contentSkip to conversation

NEW: pile stack and mempool driver

RFC PATCH New Morten Brørup 2 participants 18 messages Aug 1 – Aug 11

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

RFC PATCH 1/1 NEW: pile stack and mempool driver

Early submission of:
- some mempool optimizations,
- a new mempool "pile" driver, and
- its underlying "pile" stack implementation.

For community feedback and CI test.

Needless to say, this must be separated into a series of patches. For now, I'm submitting a snapshot of work in progress.

Some performance numbers from mempool_perf_autotest_2cores, all with cache=1024 cores=2 n_keep=32768:

start performance test (using ring_mp_mc, with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 753985338 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 755805913

start performance test for lf_stack (with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 29132352 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 29276708

start performance test for pile (with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 560159479 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 557910933

Hat tip to Bruce for bringing attention to the ring not being the optimal mempool driver.

---
 app/test/test_mempool.c                   |   3 +-
 app/test/test_stack.c                     |  70 ++++-
 app/test/test_stack_perf.c                |  15 +-
 config/rte_config.h                       |   5 +-
 doc/guides/prog_guide/stack_lib.rst       |  67 ++++-
 drivers/mempool/stack/rte_mempool_stack.c |  40 +++
 drivers/net/bonding/rte_eth_bond_pmd.c    |   2 +-
 drivers/net/intel/cpfl/cpfl_rxtx.h        |   2 +-
 drivers/net/sxe2/sxe2_txrx_vec_avx512.c   |   3 +-
 drivers/net/tap/rte_eth_tap.c             |   2 +-
 lib/eal/include/rte_common.h              |  12 +
 lib/eal/x86/include/rte_memcpy.h          |  29 ++
 lib/mempool/mempool_trace.h               |   1 -
 lib/mempool/rte_mempool.c                 |  51 ++--
 lib/mempool/rte_mempool.h                 |  74 +++--
 lib/stack/meson.build                     |   3 +-
 lib/stack/rte_stack.c                     |  18 +-
 lib/stack/rte_stack.h                     |  77 +++++-
 lib/stack/rte_stack_lf.h                  |   6 +-
 lib/stack/rte_stack_lf_c11.h              |   2 +-
 lib/stack/rte_stack_lf_generic.h          |   2 +-
 lib/stack/rte_stack_lf_stubs.h            |   2 +-
 lib/stack/rte_stack_pile.c                |  33 +++
 lib/stack/rte_stack_pile.h                | 316 ++++++++++++++++++++++
 lib/stack/rte_stack_std.h                 |  41 ++-
 25 files changed, 756 insertions(+), 120 deletions(-)
 create mode 100644 lib/stack/rte_stack_pile.c
 create mode 100644 lib/stack/rte_stack_pile.h
app/test/test_mempool.c +1 -2
index e54249ce61..76d45cea2a 100644
@@ -112,8 +112,7 @@ test_mempool_basic(struct rte_mempool *mp, int use_external_cache)
 		GOTO_ERR(ret, out);
 
 	printf("get private data\n");
-	if (rte_mempool_get_priv(mp) != (char *)mp +
-			RTE_MEMPOOL_HEADER_SIZE(mp, mp->cache_size))
+	if (rte_mempool_get_priv(mp) != (char *)mp + sizeof(struct rte_mempool))
 		GOTO_ERR(ret, out);
 
 #ifndef RTE_EXEC_ENV_FREEBSD /* rte_mem_virt2iova() not supported on bsd */
app/test/test_stack.c +56 -14

106 lines not on this page · raw message

app/test/test_stack_perf.c +13 -2
index 3f17a2606c..586410671f 100644
@@ -14,14 +14,14 @@
 #include "test.h"
 
 #define STACK_NAME "STACK_PERF"
-#define MAX_BURST 32
+#define MAX_BURST RTE_MEMPOOL_CACHE_MAX_SIZE / 2
 #define STACK_SIZE (RTE_MAX_LCORE * MAX_BURST)
 
 /*
  * Push/pop bulk sizes, marked volatile so they aren't treated as compile-time
  * constants.
  */
-static volatile unsigned int bulk_sizes[] = {8, MAX_BURST};
+static volatile unsigned int bulk_sizes[] = {1, 8, 32, MAX_BURST};
 
 static RTE_ATOMIC(uint32_t) lcore_barrier;
 
@@ -354,5 +354,16 @@ test_lf_stack_perf(void)
 #endif
 }
 
+static int
+test_pile_perf(void)
+{
+#if defined(RTE_STACK_PILE_SUPPORTED)
+	return __test_stack_perf(RTE_STACK_F_PILE);
+#else
+	return TEST_SKIPPED;
+#endif
+}
+
 REGISTER_PERF_TEST(stack_perf_autotest, test_stack_perf);
 REGISTER_PERF_TEST(stack_lf_perf_autotest, test_lf_stack_perf);
+REGISTER_PERF_TEST(stack_pile_perf_autotest, test_pile_perf);
config/rte_config.h +4 -1
index 0447cdf2ad..03350660e4 100644
@@ -56,7 +56,7 @@
 #define RTE_CONTIGMEM_DEFAULT_BUF_SIZE (512*1024*1024)
 
 /* mempool defines */
-#define RTE_MEMPOOL_CACHE_MAX_SIZE 512
+#define RTE_MEMPOOL_CACHE_MAX_SIZE 1024
 /* RTE_LIBRTE_MEMPOOL_STATS is not set */
 /* RTE_LIBRTE_MEMPOOL_DEBUG is not set */
 
@@ -64,6 +64,9 @@
 #define RTE_MBUF_DEFAULT_MEMPOOL_OPS "ring_mp_mc"
 /* RTE_MBUF_HISTORY_DEBUG is not set */
 
+/* stack defines */
+#define RTE_STACK_PILE_BULK_SIZE 32
+
 /* ether defines */
 #define RTE_MAX_QUEUES_PER_PORT 1024
 #define RTE_ETHDEV_RXTX_CALLBACKS 1
doc/guides/prog_guide/stack_lib.rst +63 -4

95 lines not on this page · raw message

drivers/mempool/stack/rte_mempool_stack.c +40 -0
index 1476905227..7467b8b39e 100644
@@ -41,6 +41,36 @@ lf_stack_alloc(struct rte_mempool *mp)
 	return __stack_alloc(mp, RTE_STACK_F_LF);
 }
 
+static int
+pile_alloc(struct rte_mempool *mp)
+{
+	return __stack_alloc(mp, RTE_STACK_F_PILE);
+}
+
+static int
+pile_enqueue(struct rte_mempool *mp, void * const *obj_table,
+	      unsigned int n)
+{
+	struct rte_stack *s = mp->pool_data;
+
+	RTE_ASSERT(s != NULL);
+	RTE_ASSERT(obj_table != NULL);
+
+	return __rte_stack_pile_push(s, obj_table, n) == 0 ? -ENOBUFS : 0;
+}
+
+static int
+pile_dequeue(struct rte_mempool *mp, void **obj_table,
+	      unsigned int n)
+{
+	struct rte_stack *s = mp->pool_data;
+
+	RTE_ASSERT(s != NULL);
+	RTE_ASSERT(obj_table != NULL);
+
+	return __rte_stack_pile_pop(s, obj_table, n) == 0 ? -ENOBUFS : 0;
+}
+
 static int
 stack_enqueue(struct rte_mempool *mp, void * const *obj_table,
 	      unsigned int n)
@@ -93,5 +123,15 @@ static struct rte_mempool_ops ops_lf_stack = {
 	.get_count = stack_get_count
 };
 
+static struct rte_mempool_ops ops_pile = {
+	.name = "pile",
+	.alloc = pile_alloc,
+	.free = stack_free,
+	.enqueue = pile_enqueue,
+	.dequeue = pile_dequeue,
+	.get_count = stack_get_count
+};
+
 RTE_MEMPOOL_REGISTER_OPS(ops_stack);
 RTE_MEMPOOL_REGISTER_OPS(ops_lf_stack);
+RTE_MEMPOOL_REGISTER_OPS(ops_pile);
drivers/net/bonding/rte_eth_bond_pmd.c +1 -1
index 6a4f997b5a..92d7f4f4ef 100644
@@ -1702,7 +1702,7 @@ member_configure_slow_queue(struct rte_eth_dev *bonding_eth_dev,
 		snprintf(mem_name, RTE_DIM(mem_name), "member_port%u_slow_pool",
 				member_id);
 		port->slow_pool = rte_pktmbuf_pool_create(mem_name, 8191,
-			250, 0, RTE_MBUF_DEFAULT_BUF_SIZE,
+			256, 0, RTE_MBUF_DEFAULT_BUF_SIZE,
 			member_eth_dev->data->numa_node);
 
 		/* Any memory allocation failure in initialization is critical because
drivers/net/intel/cpfl/cpfl_rxtx.h +1 -1
index 52cdecac88..faf28fd489 100644
@@ -25,7 +25,7 @@
 #define CPFL_P2P_QUEUE_GRP_ID	1
 #define CPFL_P2P_DESC_LEN	16
 #define CPFL_P2P_NB_MBUF	4096
-#define CPFL_P2P_CACHE_SIZE	250
+#define CPFL_P2P_CACHE_SIZE	256
 #define CPFL_P2P_MBUF_SIZE	2048
 #define CPFL_P2P_RING_BUF	128
 
drivers/net/sxe2/sxe2_txrx_vec_avx512.c +2 -1
index a830c7a33b..4ded5cb63e 100644
@@ -67,11 +67,12 @@ static __rte_always_inline int32_t sxe2_tx_bufs_free_vec_avx512(struct sxe2_tx_q
 		}
 		cache->len += rs_thresh;
 
-		if (cache->len >= cache->flushthresh) {
+		if (cache->len >= cache->size) {
 			(void)rte_mempool_ops_enqueue_bulk(mp,
 					&cache->objs[cache->size], cache->len - cache->size);
 			cache->len = cache->size;
 		}
+
 		goto done;
 	}
 
drivers/net/tap/rte_eth_tap.c +1 -1
index b93452f168..b3142561c2 100644
@@ -61,7 +61,7 @@
 #define TAP_MAX_MAC_ADDRS	16
 #define TAP_GSO_MBUFS_PER_CORE	128
 #define TAP_GSO_MBUF_SEG_SIZE	128
-#define TAP_GSO_MBUF_CACHE_SIZE	4
+#define TAP_GSO_MBUF_CACHE_SIZE	32
 #define TAP_GSO_MBUFS_NUM \
 	(TAP_GSO_MBUFS_PER_CORE * TAP_GSO_MBUF_CACHE_SIZE)
 
lib/eal/include/rte_common.h +12 -0
index 79d2a0ab93..0fd0906506 100644
@@ -567,6 +567,15 @@ static void __attribute__((destructor(RTE_PRIO(prio)), used)) func(void)
 #define __rte_assume(condition) __assume(condition)
 #endif
 
+/**
+ * Alignment hint precondition
+ */
+#ifdef RTE_TOOLCHAIN_MSVC
+#define __rte_assume_aligned(ptr, alignment) (ptr)
+#else
+#define __rte_assume_aligned(ptr, alignment) __builtin_assume_aligned(ptr, alignment)
+#endif
+
 /**
  * Disable AddressSanitizer on some code
  */
@@ -775,6 +784,9 @@ rte_is_aligned(const void * const __rte_restrict ptr, const unsigned int align)
 /** Force minimum cache line alignment. */
 #define __rte_cache_min_aligned __rte_aligned(RTE_CACHE_LINE_MIN_SIZE)
 
+/** Cache alignment hint precondition */
+#define __rte_assume_cache_aligned(ptr) __rte_assume_aligned(ptr, RTE_CACHE_LINE_SIZE)
+
 #define _RTE_CACHE_GUARD_HELPER2(unique) \
 	alignas(RTE_CACHE_LINE_SIZE) \
 	char cache_guard_ ## unique[RTE_CACHE_LINE_SIZE * RTE_CACHE_GUARD_LINES]
lib/eal/x86/include/rte_memcpy.h +29 -0
index 8ed8c55010..3fe1e8d247 100644
@@ -707,6 +707,35 @@ rte_memcpy(void *__rte_restrict dst, const void *__rte_restrict src, size_t n)
 #endif
 		return dst;
 	}
+	/* Common way for small copy size of 64-byte blocks */
+#if defined __AVX512F__ && defined RTE_MEMCPY_AVX512
+	if (__rte_constant(n) && (n & 63) == 0 && n <= 512) {
+#elif defined RTE_MEMCPY_AVX
+	if (__rte_constant(n) && (n & 63) == 0 && n <= 256) {
+#else /* SSE implementation */
+	if (__rte_constant(n) && (n & 63) == 0 && n <= 512) {
+#endif
+		void *ret = dst;
+
+		if (n & 512) {
+			rte_mov256((uint8_t *)dst + 0 * 256, (const uint8_t *)src + 0 * 256);
+			rte_mov256((uint8_t *)dst + 1 * 256, (const uint8_t *)src + 1 * 256);
+		}
+		if (n & 256) {
+			rte_mov256((uint8_t *)dst, (const uint8_t *)src);
+			src = (const uint8_t *)src + 256;
+			dst = (uint8_t *)dst + 256;
+		}
+		if (n & 128) {
+			rte_mov128((uint8_t *)dst, (const uint8_t *)src);
+			src = (const uint8_t *)src + 128;
+			dst = (uint8_t *)dst + 128;
+		}
+		if (n & 64)
+			rte_mov64((uint8_t *)dst, (const uint8_t *)src);
+
+		return ret;
+	}
 
 	/* Implementation for size > 64 bytes depends on alignment with vector register size. */
 	if (!(((uintptr_t)dst | (uintptr_t)src) & ALIGNMENT_MASK))
lib/mempool/mempool_trace.h +0 -1
index 23cda1473c..60e47cf67b 100644
@@ -119,7 +119,6 @@ RTE_TRACE_POINT(
 	rte_trace_point_emit_i32(socket_id);
 	rte_trace_point_emit_ptr(cache);
 	rte_trace_point_emit_u32(cache->len);
-	rte_trace_point_emit_u32(cache->flushthresh);
 )
 
 RTE_TRACE_POINT(
lib/mempool/rte_mempool.c +29 -22

117 lines not on this page · raw message

lib/mempool/rte_mempool.h +35 -39

155 lines not on this page · raw message

lib/stack/meson.build +2 -1
index 18177a742f..50e688522e 100644
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2019 Intel Corporation
 
-sources = files('rte_stack.c', 'rte_stack_std.c', 'rte_stack_lf.c')
+sources = files('rte_stack.c', 'rte_stack_std.c', 'rte_stack_lf.c', 'rte_stack_pile.c')
 headers = files('rte_stack.h')
 # subheaders, not for direct inclusion by apps
 indirect_headers += files(
@@ -10,4 +10,5 @@ indirect_headers += files(
         'rte_stack_lf_generic.h',
         'rte_stack_lf_c11.h',
         'rte_stack_lf_stubs.h',
+        'rte_stack_pile.h',
 )
lib/stack/rte_stack.c +17 -1
index 4c78fe4b4b..a4bbf8a4d7 100644
@@ -1,5 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright(c) 2019 Intel Corporation
+ * Copyright(c) 2026 SmartShare Systems
  */
 
 #include <stdalign.h>
@@ -32,6 +33,8 @@ rte_stack_init(struct rte_stack *s, unsigned int count, uint32_t flags)
 
 	if (flags & RTE_STACK_F_LF)
 		rte_stack_lf_init(s, count);
+	else if (flags & RTE_STACK_F_PILE)
+		rte_stack_pile_init(s, count);
 	else
 		rte_stack_std_init(s);
 }
@@ -41,6 +44,8 @@ rte_stack_get_memsize(unsigned int count, uint32_t flags)
 {
 	if (flags & RTE_STACK_F_LF)
 		return rte_stack_lf_get_memsize(count);
+	else if (flags & RTE_STACK_F_PILE)
+		return rte_stack_pile_get_memsize(count);
 	else
 		return rte_stack_std_get_memsize(count);
 }
@@ -58,7 +63,11 @@ rte_stack_create(const char *name, unsigned int count, int socket_id,
 	unsigned int sz;
 	int ret;
 
-	if (flags & ~(RTE_STACK_F_LF)) {
+	if (flags & ~(RTE_STACK_F_LF | RTE_STACK_F_PILE)) {
+		STACK_LOG_ERR("Unsupported stack flags %#x", flags);
+		return NULL;
+	}
+	if ((flags & RTE_STACK_F_LF) && (flags & RTE_STACK_F_PILE)) {
 		STACK_LOG_ERR("Unsupported stack flags %#x", flags);
 		return NULL;
 	}
@@ -73,6 +82,13 @@ rte_stack_create(const char *name, unsigned int count, int socket_id,
 		return NULL;
 	}
 #endif
+#if !defined(RTE_STACK_PILE_SUPPORTED)
+	if (flags & RTE_STACK_F_PILE) {
+		STACK_LOG_ERR("Pile is not supported on your platform");
+		rte_errno = ENOTSUP;
+		return NULL;
+	}
+#endif
 
 	sz = rte_stack_get_memsize(count, flags);
 
lib/stack/rte_stack.h +75 -2

149 lines not on this page · raw message

lib/stack/rte_stack_lf.h +4 -2
index f2b012cd0e..655620aaa9 100644
@@ -34,7 +34,7 @@
  */
 static __rte_always_inline unsigned int
 __rte_stack_lf_push(struct rte_stack *s,
-		    void * const *obj_table,
+		    void * const * __rte_restrict obj_table,
 		    unsigned int n)
 {
 	struct rte_stack_lf_elem *tmp, *first, *last = NULL;
@@ -71,7 +71,8 @@ __rte_stack_lf_push(struct rte_stack *s,
  *   - Actual number of objects popped.
  */
 static __rte_always_inline unsigned int
-__rte_stack_lf_pop(struct rte_stack *s, void **obj_table, unsigned int n)
+__rte_stack_lf_pop(struct rte_stack *s, void ** __rte_restrict obj_table,
+		   unsigned int n)
 {
 	struct rte_stack_lf_elem *first, *last = NULL;
 
@@ -79,6 +80,7 @@ __rte_stack_lf_pop(struct rte_stack *s, void **obj_table, unsigned int n)
 		return 0;
 
 	/* Pop n used elements */
+	__rte_assume(obj_table != NULL);
 	first = __rte_stack_lf_pop_elems(&s->stack_lf.used,
 					 n, obj_table, &last);
 	if (unlikely(first == NULL))
lib/stack/rte_stack_lf_c11.h +1 -1
index b97e02d6a1..501e985a49 100644
@@ -99,7 +99,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 static __rte_always_inline struct rte_stack_lf_elem *
 __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 unsigned int num,
-			 void **obj_table,
+			 void ** __rte_restrict obj_table,
 			 struct rte_stack_lf_elem **last)
 {
 	struct rte_stack_lf_head old_head;
lib/stack/rte_stack_lf_generic.h +1 -1
index cc69e4d168..c4cc4d2c03 100644
@@ -74,7 +74,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 static __rte_always_inline struct rte_stack_lf_elem *
 __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 unsigned int num,
-			 void **obj_table,
+			 void ** __rte_restrict obj_table,
 			 struct rte_stack_lf_elem **last)
 {
 	struct rte_stack_lf_head old_head;
lib/stack/rte_stack_lf_stubs.h +1 -1
index a05abf1f1c..457a415ccf 100644
@@ -30,7 +30,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 static __rte_always_inline struct rte_stack_lf_elem *
 __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 unsigned int num,
-			 void **obj_table,
+			 void ** __rte_restrict obj_table,
 			 struct rte_stack_lf_elem **last)
 {
 	RTE_SET_USED(obj_table);
lib/stack/rte_stack_pile.c new file +33 -0
new file mode 100644
index 0000000000..eb42ba70a9
@@ -0,0 +1,33 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2026 SmartShare Systems
+ */
+
+#include "rte_stack.h"
+
+void
+rte_stack_pile_init(struct rte_stack *s, unsigned int count)
+{
+    unsigned int bulk = (count + RTE_STACK_PILE_BULK_SIZE - 1) / RTE_STACK_PILE_BULK_SIZE;
+	struct rte_stack_pile_bulk_elem * bulk_elems = (struct rte_stack_pile_bulk_elem *)(&s->stack_pile + 1);
+	struct rte_stack_lf_elem * solo_elems = (struct rte_stack_lf_elem *)&bulk_elems[bulk];
+	unsigned int i;
+
+	for (i = 0; i < bulk; i++)
+		__rte_stack_pile_bulk_push_elems(&s->stack_pile.free_bulk,
+					  &bulk_elems[i], &bulk_elems[i], 1);
+	for (i = 0; i < count; i++)
+		__rte_stack_lf_push_elems(&s->stack_pile.free_solo,
+					  &solo_elems[i], &solo_elems[i], 1);
+}
+
+ssize_t
+rte_stack_pile_get_memsize(unsigned int count)
+{
+    unsigned int bulk = (count + RTE_STACK_PILE_BULK_SIZE - 1) / RTE_STACK_PILE_BULK_SIZE;
+	ssize_t sz = sizeof(struct rte_stack); /* Already cache line aligned. */
+	sz += bulk * sizeof(struct rte_stack_pile_bulk_elem); /* Already cache line aligned. */
+	sz += RTE_CACHE_LINE_ROUNDUP(count * sizeof(struct rte_stack_lf_elem));
+	sz += RTE_CACHE_GUARD_LINES * RTE_CACHE_LINE_SIZE;
+
+	return sz;
+}
lib/stack/rte_stack_pile.h new file +316 -0

316 lines not on this page · raw message

lib/stack/rte_stack_std.h +19 -22

80 lines not on this page · raw message

Signature · 1 line
2.43.0
#1 lore raw

RFC PATCH v3 1/1 NEW: pile stack and mempool driver

Early submission of:
- some mempool optimizations,
- a new mempool "pile" driver, and
- its underlying "pile" stack implementation.

For community feedback and CI test.

Needless to say, this must be separated into a series of patches. For now, I'm submitting a snapshot of work in progress.

Some performance numbers from mempool_perf_autotest_2cores, all with cache=1024 cores=2 n_keep=32768:

start performance test (using ring_mp_mc, with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 753985338 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 755805913

start performance test for lf_stack (with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 29132352 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 29276708

start performance test for pile (with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 560159479 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 557910933

Hat tip to Bruce for bringing attention to the ring not being the optimal mempool driver.

---
v2:
* Fix indentation, long lines, etc. (checkpatch)
* Fix label followed by a declaration is a C23 extension.
* Added __rte_internal to pile init and get_memsize. (AI)
* Fix roll back bulk elements in wrong order with fragmentation. (AI)
* Minor changes suggested by AI.
v3:
* Revert rename unused field in rte_stack_pile_bulk_elem structure.

---
 app/test/test_mempool.c                   |   3 +-
 app/test/test_stack.c                     |  67 ++++-
 app/test/test_stack_perf.c                |  15 +-
 config/rte_config.h                       |   5 +-
 doc/guides/prog_guide/stack_lib.rst       |  67 ++++-
 drivers/mempool/stack/rte_mempool_stack.c |  40 +++
 drivers/net/bonding/rte_eth_bond_pmd.c    |   2 +-
 drivers/net/intel/cpfl/cpfl_rxtx.h        |   2 +-
 drivers/net/sxe2/sxe2_txrx_vec_avx512.c   |   3 +-
 drivers/net/tap/rte_eth_tap.c             |   2 +-
 lib/eal/include/rte_common.h              |  12 +
 lib/eal/x86/include/rte_memcpy.h          |  29 ++
 lib/mempool/mempool_trace.h               |   1 -
 lib/mempool/rte_mempool.c                 |  52 ++--
 lib/mempool/rte_mempool.h                 |  74 +++--
 lib/stack/meson.build                     |   3 +-
 lib/stack/rte_stack.c                     |  18 +-
 lib/stack/rte_stack.h                     |  79 +++++-
 lib/stack/rte_stack_lf.h                  |   6 +-
 lib/stack/rte_stack_lf_c11.h              |   2 +-
 lib/stack/rte_stack_lf_generic.h          |   2 +-
 lib/stack/rte_stack_lf_stubs.h            |   2 +-
 lib/stack/rte_stack_pile.c                |  36 +++
 lib/stack/rte_stack_pile.h                | 328 ++++++++++++++++++++++
 lib/stack/rte_stack_std.h                 |  37 ++-
 25 files changed, 771 insertions(+), 116 deletions(-)
 create mode 100644 lib/stack/rte_stack_pile.c
 create mode 100644 lib/stack/rte_stack_pile.h
app/test/test_mempool.c +1 -2
index e54249ce61..76d45cea2a 100644
@@ -112,8 +112,7 @@ test_mempool_basic(struct rte_mempool *mp, int use_external_cache)
 		GOTO_ERR(ret, out);
 
 	printf("get private data\n");
-	if (rte_mempool_get_priv(mp) != (char *)mp +
-			RTE_MEMPOOL_HEADER_SIZE(mp, mp->cache_size))
+	if (rte_mempool_get_priv(mp) != (char *)mp + sizeof(struct rte_mempool))
 		GOTO_ERR(ret, out);
 
 #ifndef RTE_EXEC_ENV_FREEBSD /* rte_mem_virt2iova() not supported on bsd */
app/test/test_stack.c +55 -12

85 lines not on this page · raw message

app/test/test_stack_perf.c +13 -2
index 3f17a2606c..a15f3719c2 100644
@@ -14,14 +14,14 @@
 #include "test.h"
 
 #define STACK_NAME "STACK_PERF"
-#define MAX_BURST 32
+#define MAX_BURST (RTE_MEMPOOL_CACHE_MAX_SIZE / 2)
 #define STACK_SIZE (RTE_MAX_LCORE * MAX_BURST)
 
 /*
  * Push/pop bulk sizes, marked volatile so they aren't treated as compile-time
  * constants.
  */
-static volatile unsigned int bulk_sizes[] = {8, MAX_BURST};
+static volatile unsigned int bulk_sizes[] = {1, 8, 32, MAX_BURST};
 
 static RTE_ATOMIC(uint32_t) lcore_barrier;
 
@@ -354,5 +354,16 @@ test_lf_stack_perf(void)
 #endif
 }
 
+static int
+test_pile_perf(void)
+{
+#if defined(RTE_STACK_PILE_SUPPORTED)
+	return __test_stack_perf(RTE_STACK_F_PILE);
+#else
+	return TEST_SKIPPED;
+#endif
+}
+
 REGISTER_PERF_TEST(stack_perf_autotest, test_stack_perf);
 REGISTER_PERF_TEST(stack_lf_perf_autotest, test_lf_stack_perf);
+REGISTER_PERF_TEST(stack_pile_perf_autotest, test_pile_perf);
config/rte_config.h +4 -1
index 0447cdf2ad..03350660e4 100644
@@ -56,7 +56,7 @@
 #define RTE_CONTIGMEM_DEFAULT_BUF_SIZE (512*1024*1024)
 
 /* mempool defines */
-#define RTE_MEMPOOL_CACHE_MAX_SIZE 512
+#define RTE_MEMPOOL_CACHE_MAX_SIZE 1024
 /* RTE_LIBRTE_MEMPOOL_STATS is not set */
 /* RTE_LIBRTE_MEMPOOL_DEBUG is not set */
 
@@ -64,6 +64,9 @@
 #define RTE_MBUF_DEFAULT_MEMPOOL_OPS "ring_mp_mc"
 /* RTE_MBUF_HISTORY_DEBUG is not set */
 
+/* stack defines */
+#define RTE_STACK_PILE_BULK_SIZE 32
+
 /* ether defines */
 #define RTE_MAX_QUEUES_PER_PORT 1024
 #define RTE_ETHDEV_RXTX_CALLBACKS 1
doc/guides/prog_guide/stack_lib.rst +63 -4

95 lines not on this page · raw message

drivers/mempool/stack/rte_mempool_stack.c +40 -0
index 1476905227..7467b8b39e 100644
@@ -41,6 +41,36 @@ lf_stack_alloc(struct rte_mempool *mp)
 	return __stack_alloc(mp, RTE_STACK_F_LF);
 }
 
+static int
+pile_alloc(struct rte_mempool *mp)
+{
+	return __stack_alloc(mp, RTE_STACK_F_PILE);
+}
+
+static int
+pile_enqueue(struct rte_mempool *mp, void * const *obj_table,
+	      unsigned int n)
+{
+	struct rte_stack *s = mp->pool_data;
+
+	RTE_ASSERT(s != NULL);
+	RTE_ASSERT(obj_table != NULL);
+
+	return __rte_stack_pile_push(s, obj_table, n) == 0 ? -ENOBUFS : 0;
+}
+
+static int
+pile_dequeue(struct rte_mempool *mp, void **obj_table,
+	      unsigned int n)
+{
+	struct rte_stack *s = mp->pool_data;
+
+	RTE_ASSERT(s != NULL);
+	RTE_ASSERT(obj_table != NULL);
+
+	return __rte_stack_pile_pop(s, obj_table, n) == 0 ? -ENOBUFS : 0;
+}
+
 static int
 stack_enqueue(struct rte_mempool *mp, void * const *obj_table,
 	      unsigned int n)
@@ -93,5 +123,15 @@ static struct rte_mempool_ops ops_lf_stack = {
 	.get_count = stack_get_count
 };
 
+static struct rte_mempool_ops ops_pile = {
+	.name = "pile",
+	.alloc = pile_alloc,
+	.free = stack_free,
+	.enqueue = pile_enqueue,
+	.dequeue = pile_dequeue,
+	.get_count = stack_get_count
+};
+
 RTE_MEMPOOL_REGISTER_OPS(ops_stack);
 RTE_MEMPOOL_REGISTER_OPS(ops_lf_stack);
+RTE_MEMPOOL_REGISTER_OPS(ops_pile);
drivers/net/bonding/rte_eth_bond_pmd.c +1 -1
index 6a4f997b5a..92d7f4f4ef 100644
@@ -1702,7 +1702,7 @@ member_configure_slow_queue(struct rte_eth_dev *bonding_eth_dev,
 		snprintf(mem_name, RTE_DIM(mem_name), "member_port%u_slow_pool",
 				member_id);
 		port->slow_pool = rte_pktmbuf_pool_create(mem_name, 8191,
-			250, 0, RTE_MBUF_DEFAULT_BUF_SIZE,
+			256, 0, RTE_MBUF_DEFAULT_BUF_SIZE,
 			member_eth_dev->data->numa_node);
 
 		/* Any memory allocation failure in initialization is critical because
drivers/net/intel/cpfl/cpfl_rxtx.h +1 -1
index 52cdecac88..faf28fd489 100644
@@ -25,7 +25,7 @@
 #define CPFL_P2P_QUEUE_GRP_ID	1
 #define CPFL_P2P_DESC_LEN	16
 #define CPFL_P2P_NB_MBUF	4096
-#define CPFL_P2P_CACHE_SIZE	250
+#define CPFL_P2P_CACHE_SIZE	256
 #define CPFL_P2P_MBUF_SIZE	2048
 #define CPFL_P2P_RING_BUF	128
 
drivers/net/sxe2/sxe2_txrx_vec_avx512.c +2 -1
index a830c7a33b..4ded5cb63e 100644
@@ -67,11 +67,12 @@ static __rte_always_inline int32_t sxe2_tx_bufs_free_vec_avx512(struct sxe2_tx_q
 		}
 		cache->len += rs_thresh;
 
-		if (cache->len >= cache->flushthresh) {
+		if (cache->len >= cache->size) {
 			(void)rte_mempool_ops_enqueue_bulk(mp,
 					&cache->objs[cache->size], cache->len - cache->size);
 			cache->len = cache->size;
 		}
+
 		goto done;
 	}
 
drivers/net/tap/rte_eth_tap.c +1 -1
index b93452f168..b3142561c2 100644
@@ -61,7 +61,7 @@
 #define TAP_MAX_MAC_ADDRS	16
 #define TAP_GSO_MBUFS_PER_CORE	128
 #define TAP_GSO_MBUF_SEG_SIZE	128
-#define TAP_GSO_MBUF_CACHE_SIZE	4
+#define TAP_GSO_MBUF_CACHE_SIZE	32
 #define TAP_GSO_MBUFS_NUM \
 	(TAP_GSO_MBUFS_PER_CORE * TAP_GSO_MBUF_CACHE_SIZE)
 
lib/eal/include/rte_common.h +12 -0
index 79d2a0ab93..0fd0906506 100644
@@ -567,6 +567,15 @@ static void __attribute__((destructor(RTE_PRIO(prio)), used)) func(void)
 #define __rte_assume(condition) __assume(condition)
 #endif
 
+/**
+ * Alignment hint precondition
+ */
+#ifdef RTE_TOOLCHAIN_MSVC
+#define __rte_assume_aligned(ptr, alignment) (ptr)
+#else
+#define __rte_assume_aligned(ptr, alignment) __builtin_assume_aligned(ptr, alignment)
+#endif
+
 /**
  * Disable AddressSanitizer on some code
  */
@@ -775,6 +784,9 @@ rte_is_aligned(const void * const __rte_restrict ptr, const unsigned int align)
 /** Force minimum cache line alignment. */
 #define __rte_cache_min_aligned __rte_aligned(RTE_CACHE_LINE_MIN_SIZE)
 
+/** Cache alignment hint precondition */
+#define __rte_assume_cache_aligned(ptr) __rte_assume_aligned(ptr, RTE_CACHE_LINE_SIZE)
+
 #define _RTE_CACHE_GUARD_HELPER2(unique) \
 	alignas(RTE_CACHE_LINE_SIZE) \
 	char cache_guard_ ## unique[RTE_CACHE_LINE_SIZE * RTE_CACHE_GUARD_LINES]
lib/eal/x86/include/rte_memcpy.h +29 -0
index 8ed8c55010..3fe1e8d247 100644
@@ -707,6 +707,35 @@ rte_memcpy(void *__rte_restrict dst, const void *__rte_restrict src, size_t n)
 #endif
 		return dst;
 	}
+	/* Common way for small copy size of 64-byte blocks */
+#if defined __AVX512F__ && defined RTE_MEMCPY_AVX512
+	if (__rte_constant(n) && (n & 63) == 0 && n <= 512) {
+#elif defined RTE_MEMCPY_AVX
+	if (__rte_constant(n) && (n & 63) == 0 && n <= 256) {
+#else /* SSE implementation */
+	if (__rte_constant(n) && (n & 63) == 0 && n <= 512) {
+#endif
+		void *ret = dst;
+
+		if (n & 512) {
+			rte_mov256((uint8_t *)dst + 0 * 256, (const uint8_t *)src + 0 * 256);
+			rte_mov256((uint8_t *)dst + 1 * 256, (const uint8_t *)src + 1 * 256);
+		}
+		if (n & 256) {
+			rte_mov256((uint8_t *)dst, (const uint8_t *)src);
+			src = (const uint8_t *)src + 256;
+			dst = (uint8_t *)dst + 256;
+		}
+		if (n & 128) {
+			rte_mov128((uint8_t *)dst, (const uint8_t *)src);
+			src = (const uint8_t *)src + 128;
+			dst = (uint8_t *)dst + 128;
+		}
+		if (n & 64)
+			rte_mov64((uint8_t *)dst, (const uint8_t *)src);
+
+		return ret;
+	}
 
 	/* Implementation for size > 64 bytes depends on alignment with vector register size. */
 	if (!(((uintptr_t)dst | (uintptr_t)src) & ALIGNMENT_MASK))
lib/mempool/mempool_trace.h +0 -1
index 23cda1473c..60e47cf67b 100644
@@ -119,7 +119,6 @@ RTE_TRACE_POINT(
 	rte_trace_point_emit_i32(socket_id);
 	rte_trace_point_emit_ptr(cache);
 	rte_trace_point_emit_u32(cache->len);
-	rte_trace_point_emit_u32(cache->flushthresh);
 )
 
 RTE_TRACE_POINT(
lib/mempool/rte_mempool.c +30 -22

118 lines not on this page · raw message

lib/mempool/rte_mempool.h +35 -39

155 lines not on this page · raw message

lib/stack/meson.build +2 -1
index 18177a742f..50e688522e 100644
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2019 Intel Corporation
 
-sources = files('rte_stack.c', 'rte_stack_std.c', 'rte_stack_lf.c')
+sources = files('rte_stack.c', 'rte_stack_std.c', 'rte_stack_lf.c', 'rte_stack_pile.c')
 headers = files('rte_stack.h')
 # subheaders, not for direct inclusion by apps
 indirect_headers += files(
@@ -10,4 +10,5 @@ indirect_headers += files(
         'rte_stack_lf_generic.h',
         'rte_stack_lf_c11.h',
         'rte_stack_lf_stubs.h',
+        'rte_stack_pile.h',
 )
lib/stack/rte_stack.c +17 -1
index 4c78fe4b4b..a4bbf8a4d7 100644
@@ -1,5 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright(c) 2019 Intel Corporation
+ * Copyright(c) 2026 SmartShare Systems
  */
 
 #include <stdalign.h>
@@ -32,6 +33,8 @@ rte_stack_init(struct rte_stack *s, unsigned int count, uint32_t flags)
 
 	if (flags & RTE_STACK_F_LF)
 		rte_stack_lf_init(s, count);
+	else if (flags & RTE_STACK_F_PILE)
+		rte_stack_pile_init(s, count);
 	else
 		rte_stack_std_init(s);
 }
@@ -41,6 +44,8 @@ rte_stack_get_memsize(unsigned int count, uint32_t flags)
 {
 	if (flags & RTE_STACK_F_LF)
 		return rte_stack_lf_get_memsize(count);
+	else if (flags & RTE_STACK_F_PILE)
+		return rte_stack_pile_get_memsize(count);
 	else
 		return rte_stack_std_get_memsize(count);
 }
@@ -58,7 +63,11 @@ rte_stack_create(const char *name, unsigned int count, int socket_id,
 	unsigned int sz;
 	int ret;
 
-	if (flags & ~(RTE_STACK_F_LF)) {
+	if (flags & ~(RTE_STACK_F_LF | RTE_STACK_F_PILE)) {
+		STACK_LOG_ERR("Unsupported stack flags %#x", flags);
+		return NULL;
+	}
+	if ((flags & RTE_STACK_F_LF) && (flags & RTE_STACK_F_PILE)) {
 		STACK_LOG_ERR("Unsupported stack flags %#x", flags);
 		return NULL;
 	}
@@ -73,6 +82,13 @@ rte_stack_create(const char *name, unsigned int count, int socket_id,
 		return NULL;
 	}
 #endif
+#if !defined(RTE_STACK_PILE_SUPPORTED)
+	if (flags & RTE_STACK_F_PILE) {
+		STACK_LOG_ERR("Pile is not supported on your platform");
+		rte_errno = ENOTSUP;
+		return NULL;
+	}
+#endif
 
 	sz = rte_stack_get_memsize(count, flags);
 
lib/stack/rte_stack.h +77 -2

151 lines not on this page · raw message

lib/stack/rte_stack_lf.h +4 -2
index f2b012cd0e..655620aaa9 100644
@@ -34,7 +34,7 @@
  */
 static __rte_always_inline unsigned int
 __rte_stack_lf_push(struct rte_stack *s,
-		    void * const *obj_table,
+		    void * const * __rte_restrict obj_table,
 		    unsigned int n)
 {
 	struct rte_stack_lf_elem *tmp, *first, *last = NULL;
@@ -71,7 +71,8 @@ __rte_stack_lf_push(struct rte_stack *s,
  *   - Actual number of objects popped.
  */
 static __rte_always_inline unsigned int
-__rte_stack_lf_pop(struct rte_stack *s, void **obj_table, unsigned int n)
+__rte_stack_lf_pop(struct rte_stack *s, void ** __rte_restrict obj_table,
+		   unsigned int n)
 {
 	struct rte_stack_lf_elem *first, *last = NULL;
 
@@ -79,6 +80,7 @@ __rte_stack_lf_pop(struct rte_stack *s, void **obj_table, unsigned int n)
 		return 0;
 
 	/* Pop n used elements */
+	__rte_assume(obj_table != NULL);
 	first = __rte_stack_lf_pop_elems(&s->stack_lf.used,
 					 n, obj_table, &last);
 	if (unlikely(first == NULL))
lib/stack/rte_stack_lf_c11.h +1 -1
index b97e02d6a1..501e985a49 100644
@@ -99,7 +99,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 static __rte_always_inline struct rte_stack_lf_elem *
 __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 unsigned int num,
-			 void **obj_table,
+			 void ** __rte_restrict obj_table,
 			 struct rte_stack_lf_elem **last)
 {
 	struct rte_stack_lf_head old_head;
lib/stack/rte_stack_lf_generic.h +1 -1
index cc69e4d168..c4cc4d2c03 100644
@@ -74,7 +74,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 static __rte_always_inline struct rte_stack_lf_elem *
 __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 unsigned int num,
-			 void **obj_table,
+			 void ** __rte_restrict obj_table,
 			 struct rte_stack_lf_elem **last)
 {
 	struct rte_stack_lf_head old_head;
lib/stack/rte_stack_lf_stubs.h +1 -1
index a05abf1f1c..457a415ccf 100644
@@ -30,7 +30,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 static __rte_always_inline struct rte_stack_lf_elem *
 __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 unsigned int num,
-			 void **obj_table,
+			 void ** __rte_restrict obj_table,
 			 struct rte_stack_lf_elem **last)
 {
 	RTE_SET_USED(obj_table);
lib/stack/rte_stack_pile.c new file +36 -0
new file mode 100644
index 0000000000..aa2901c926
@@ -0,0 +1,36 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2026 SmartShare Systems
+ */
+
+#include "rte_stack.h"
+
+RTE_EXPORT_INTERNAL_SYMBOL(rte_stack_pile_init)
+void
+rte_stack_pile_init(struct rte_stack *s, unsigned int count)
+{
+	unsigned int bulk = (count + RTE_STACK_PILE_BULK_SIZE - 1) / RTE_STACK_PILE_BULK_SIZE;
+	struct rte_stack_pile_bulk_elem *bulk_elems =
+			(struct rte_stack_pile_bulk_elem *)(&s->stack_pile + 1);
+	struct rte_stack_lf_elem *solo_elems = (struct rte_stack_lf_elem *)&bulk_elems[bulk];
+	unsigned int i;
+
+	for (i = 0; i < bulk; i++)
+		__rte_stack_pile_bulk_push_elems(&s->stack_pile.free_bulk,
+					  &bulk_elems[i], &bulk_elems[i], 1);
+	for (i = 0; i < count; i++)
+		__rte_stack_lf_push_elems(&s->stack_pile.free_solo,
+					  &solo_elems[i], &solo_elems[i], 1);
+}
+
+RTE_EXPORT_INTERNAL_SYMBOL(rte_stack_pile_get_memsize)
+ssize_t
+rte_stack_pile_get_memsize(unsigned int count)
+{
+	unsigned int bulk = (count + RTE_STACK_PILE_BULK_SIZE - 1) / RTE_STACK_PILE_BULK_SIZE;
+	ssize_t sz = sizeof(struct rte_stack); /* Already cache line aligned. */
+	sz += bulk * sizeof(struct rte_stack_pile_bulk_elem); /* Already cache line aligned. */
+	sz += RTE_CACHE_LINE_ROUNDUP(count * sizeof(struct rte_stack_lf_elem));
+	sz += RTE_CACHE_GUARD_LINES * RTE_CACHE_LINE_SIZE;
+
+	return sz;
+}
lib/stack/rte_stack_pile.h new file +328 -0

328 lines not on this page · raw message

lib/stack/rte_stack_std.h +17 -20
index ae28add5c4..cb815d6cbe 100644
@@ -6,6 +6,7 @@
 #define _RTE_STACK_STD_H_
 
 #include <rte_branch_prediction.h>
+#include <rte_memcpy.h>
 
 /**
  * @internal Push several objects on the stack (MT-safe).
@@ -20,27 +21,24 @@
  *   Actual number of objects pushed (either 0 or *n*).
  */
 static __rte_always_inline unsigned int
-__rte_stack_std_push(struct rte_stack *s, void * const *obj_table,
+__rte_stack_std_push(struct rte_stack *s, void * const * __rte_restrict obj_table,
 		     unsigned int n)
 {
 	struct rte_stack_std *stack = &s->stack_std;
-	unsigned int index;
-	void **cache_objs;
+	void ** __rte_restrict stack_objs;
 
 	rte_spinlock_lock(&stack->lock);
-	cache_objs = &stack->objs[stack->len];
 
-	/* Is there sufficient space in the stack? */
-	if ((stack->len + n) > s->capacity) {
+	if (unlikely((stack->len + n) > s->capacity)) {
+		/* Insufficient room in the stack. */
 		rte_spinlock_unlock(&stack->lock);
 		return 0;
 	}
 
-	/* Add elements back into the cache */
-	for (index = 0; index < n; ++index, obj_table++)
-		cache_objs[index] = *obj_table;
-
+	/* Push objects to the stack */
+	stack_objs = &stack->objs[stack->len];
 	stack->len += n;
+	rte_memcpy(stack_objs, obj_table, sizeof(void *) * n);
 
 	rte_spinlock_unlock(&stack->lock);
 	return n;
@@ -59,28 +57,27 @@ __rte_stack_std_push(struct rte_stack *s, void * const *obj_table,
  *   Actual number of objects popped (either 0 or *n*).
  */
 static __rte_always_inline unsigned int
-__rte_stack_std_pop(struct rte_stack *s, void **obj_table, unsigned int n)
+__rte_stack_std_pop(struct rte_stack *s, void ** __rte_restrict obj_table, unsigned int n)
 {
 	struct rte_stack_std *stack = &s->stack_std;
-	unsigned int index, len;
-	void **cache_objs;
+	unsigned int index;
+	void ** __rte_restrict stack_objs;
 
 	rte_spinlock_lock(&stack->lock);
 
 	if (unlikely(n > stack->len)) {
+		/* Insufficient objects in the stack. */
 		rte_spinlock_unlock(&stack->lock);
 		return 0;
 	}
 
-	cache_objs = stack->objs;
-
-	for (index = 0, len = stack->len - 1; index < n;
-			++index, len--, obj_table++)
-		*obj_table = cache_objs[len];
-
+	/* Pop objects from the stack */
+	stack_objs = &stack->objs[stack->len];
 	stack->len -= n;
-	rte_spinlock_unlock(&stack->lock);
+	for (index = 0; index < n; index++)
+		*obj_table++ = *--stack_objs;
 
+	rte_spinlock_unlock(&stack->lock);
 	return n;
 }
 
Signature · 1 line
2.43.0
#2 lore raw

RFC PATCH v2 1/1 NEW: pile stack and mempool driver

In reply to Morten Brørup

Early submission of:
- some mempool optimizations,
- a new mempool "pile" driver, and
- its underlying "pile" stack implementation.

For community feedback and CI test.

Needless to say, this must be separated into a series of patches. For now, I'm submitting a snapshot of work in progress.

Some performance numbers from mempool_perf_autotest_2cores, all with cache=1024 cores=2 n_keep=32768:

start performance test (using ring_mp_mc, with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 753985338 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 755805913

start performance test for lf_stack (with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 29132352 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 29276708

start performance test for pile (with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 560159479 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 557910933

Hat tip to Bruce for bringing attention to the ring not being the optimal mempool driver.

---
v2:
* Fix indentation, long lines, etc. (checkpatch)
* Fix label followed by a declaration is a C23 extension.
* Added __rte_internal to pile init and get_memsize. (AI)
* Fix roll back bulk elements in wrong order with fragmentation. (AI)
* Minor changes suggested by AI.

---
 app/test/test_mempool.c                   |   3 +-
 app/test/test_stack.c                     |  67 ++++-
 app/test/test_stack_perf.c                |  15 +-
 config/rte_config.h                       |   5 +-
 doc/guides/prog_guide/stack_lib.rst       |  67 ++++-
 drivers/mempool/stack/rte_mempool_stack.c |  40 +++
 drivers/net/bonding/rte_eth_bond_pmd.c    |   2 +-
 drivers/net/intel/cpfl/cpfl_rxtx.h        |   2 +-
 drivers/net/sxe2/sxe2_txrx_vec_avx512.c   |   3 +-
 drivers/net/tap/rte_eth_tap.c             |   2 +-
 lib/eal/include/rte_common.h              |  12 +
 lib/eal/x86/include/rte_memcpy.h          |  29 ++
 lib/mempool/mempool_trace.h               |   1 -
 lib/mempool/rte_mempool.c                 |  52 ++--
 lib/mempool/rte_mempool.h                 |  74 +++--
 lib/stack/meson.build                     |   3 +-
 lib/stack/rte_stack.c                     |  18 +-
 lib/stack/rte_stack.h                     |  79 +++++-
 lib/stack/rte_stack_lf.h                  |   6 +-
 lib/stack/rte_stack_lf_c11.h              |   2 +-
 lib/stack/rte_stack_lf_generic.h          |   2 +-
 lib/stack/rte_stack_lf_stubs.h            |   2 +-
 lib/stack/rte_stack_pile.c                |  36 +++
 lib/stack/rte_stack_pile.h                | 328 ++++++++++++++++++++++
 lib/stack/rte_stack_std.h                 |  37 ++-
 25 files changed, 771 insertions(+), 116 deletions(-)
 create mode 100644 lib/stack/rte_stack_pile.c
 create mode 100644 lib/stack/rte_stack_pile.h
app/test/test_mempool.c +1 -2
index e54249ce61..76d45cea2a 100644
@@ -112,8 +112,7 @@ test_mempool_basic(struct rte_mempool *mp, int use_external_cache)
 		GOTO_ERR(ret, out);
 
 	printf("get private data\n");
-	if (rte_mempool_get_priv(mp) != (char *)mp +
-			RTE_MEMPOOL_HEADER_SIZE(mp, mp->cache_size))
+	if (rte_mempool_get_priv(mp) != (char *)mp + sizeof(struct rte_mempool))
 		GOTO_ERR(ret, out);
 
 #ifndef RTE_EXEC_ENV_FREEBSD /* rte_mem_virt2iova() not supported on bsd */
app/test/test_stack.c +55 -12

85 lines not on this page · raw message

app/test/test_stack_perf.c +13 -2
index 3f17a2606c..a15f3719c2 100644
@@ -14,14 +14,14 @@
 #include "test.h"
 
 #define STACK_NAME "STACK_PERF"
-#define MAX_BURST 32
+#define MAX_BURST (RTE_MEMPOOL_CACHE_MAX_SIZE / 2)
 #define STACK_SIZE (RTE_MAX_LCORE * MAX_BURST)
 
 /*
  * Push/pop bulk sizes, marked volatile so they aren't treated as compile-time
  * constants.
  */
-static volatile unsigned int bulk_sizes[] = {8, MAX_BURST};
+static volatile unsigned int bulk_sizes[] = {1, 8, 32, MAX_BURST};
 
 static RTE_ATOMIC(uint32_t) lcore_barrier;
 
@@ -354,5 +354,16 @@ test_lf_stack_perf(void)
 #endif
 }
 
+static int
+test_pile_perf(void)
+{
+#if defined(RTE_STACK_PILE_SUPPORTED)
+	return __test_stack_perf(RTE_STACK_F_PILE);
+#else
+	return TEST_SKIPPED;
+#endif
+}
+
 REGISTER_PERF_TEST(stack_perf_autotest, test_stack_perf);
 REGISTER_PERF_TEST(stack_lf_perf_autotest, test_lf_stack_perf);
+REGISTER_PERF_TEST(stack_pile_perf_autotest, test_pile_perf);
config/rte_config.h +4 -1
index 0447cdf2ad..03350660e4 100644
@@ -56,7 +56,7 @@
 #define RTE_CONTIGMEM_DEFAULT_BUF_SIZE (512*1024*1024)
 
 /* mempool defines */
-#define RTE_MEMPOOL_CACHE_MAX_SIZE 512
+#define RTE_MEMPOOL_CACHE_MAX_SIZE 1024
 /* RTE_LIBRTE_MEMPOOL_STATS is not set */
 /* RTE_LIBRTE_MEMPOOL_DEBUG is not set */
 
@@ -64,6 +64,9 @@
 #define RTE_MBUF_DEFAULT_MEMPOOL_OPS "ring_mp_mc"
 /* RTE_MBUF_HISTORY_DEBUG is not set */
 
+/* stack defines */
+#define RTE_STACK_PILE_BULK_SIZE 32
+
 /* ether defines */
 #define RTE_MAX_QUEUES_PER_PORT 1024
 #define RTE_ETHDEV_RXTX_CALLBACKS 1
doc/guides/prog_guide/stack_lib.rst +63 -4

95 lines not on this page · raw message

drivers/mempool/stack/rte_mempool_stack.c +40 -0
index 1476905227..7467b8b39e 100644
@@ -41,6 +41,36 @@ lf_stack_alloc(struct rte_mempool *mp)
 	return __stack_alloc(mp, RTE_STACK_F_LF);
 }
 
+static int
+pile_alloc(struct rte_mempool *mp)
+{
+	return __stack_alloc(mp, RTE_STACK_F_PILE);
+}
+
+static int
+pile_enqueue(struct rte_mempool *mp, void * const *obj_table,
+	      unsigned int n)
+{
+	struct rte_stack *s = mp->pool_data;
+
+	RTE_ASSERT(s != NULL);
+	RTE_ASSERT(obj_table != NULL);
+
+	return __rte_stack_pile_push(s, obj_table, n) == 0 ? -ENOBUFS : 0;
+}
+
+static int
+pile_dequeue(struct rte_mempool *mp, void **obj_table,
+	      unsigned int n)
+{
+	struct rte_stack *s = mp->pool_data;
+
+	RTE_ASSERT(s != NULL);
+	RTE_ASSERT(obj_table != NULL);
+
+	return __rte_stack_pile_pop(s, obj_table, n) == 0 ? -ENOBUFS : 0;
+}
+
 static int
 stack_enqueue(struct rte_mempool *mp, void * const *obj_table,
 	      unsigned int n)
@@ -93,5 +123,15 @@ static struct rte_mempool_ops ops_lf_stack = {
 	.get_count = stack_get_count
 };
 
+static struct rte_mempool_ops ops_pile = {
+	.name = "pile",
+	.alloc = pile_alloc,
+	.free = stack_free,
+	.enqueue = pile_enqueue,
+	.dequeue = pile_dequeue,
+	.get_count = stack_get_count
+};
+
 RTE_MEMPOOL_REGISTER_OPS(ops_stack);
 RTE_MEMPOOL_REGISTER_OPS(ops_lf_stack);
+RTE_MEMPOOL_REGISTER_OPS(ops_pile);
drivers/net/bonding/rte_eth_bond_pmd.c +1 -1
index 6a4f997b5a..92d7f4f4ef 100644
@@ -1702,7 +1702,7 @@ member_configure_slow_queue(struct rte_eth_dev *bonding_eth_dev,
 		snprintf(mem_name, RTE_DIM(mem_name), "member_port%u_slow_pool",
 				member_id);
 		port->slow_pool = rte_pktmbuf_pool_create(mem_name, 8191,
-			250, 0, RTE_MBUF_DEFAULT_BUF_SIZE,
+			256, 0, RTE_MBUF_DEFAULT_BUF_SIZE,
 			member_eth_dev->data->numa_node);
 
 		/* Any memory allocation failure in initialization is critical because
drivers/net/intel/cpfl/cpfl_rxtx.h +1 -1
index 52cdecac88..faf28fd489 100644
@@ -25,7 +25,7 @@
 #define CPFL_P2P_QUEUE_GRP_ID	1
 #define CPFL_P2P_DESC_LEN	16
 #define CPFL_P2P_NB_MBUF	4096
-#define CPFL_P2P_CACHE_SIZE	250
+#define CPFL_P2P_CACHE_SIZE	256
 #define CPFL_P2P_MBUF_SIZE	2048
 #define CPFL_P2P_RING_BUF	128
 
drivers/net/sxe2/sxe2_txrx_vec_avx512.c +2 -1
index a830c7a33b..4ded5cb63e 100644
@@ -67,11 +67,12 @@ static __rte_always_inline int32_t sxe2_tx_bufs_free_vec_avx512(struct sxe2_tx_q
 		}
 		cache->len += rs_thresh;
 
-		if (cache->len >= cache->flushthresh) {
+		if (cache->len >= cache->size) {
 			(void)rte_mempool_ops_enqueue_bulk(mp,
 					&cache->objs[cache->size], cache->len - cache->size);
 			cache->len = cache->size;
 		}
+
 		goto done;
 	}
 
drivers/net/tap/rte_eth_tap.c +1 -1
index b93452f168..b3142561c2 100644
@@ -61,7 +61,7 @@
 #define TAP_MAX_MAC_ADDRS	16
 #define TAP_GSO_MBUFS_PER_CORE	128
 #define TAP_GSO_MBUF_SEG_SIZE	128
-#define TAP_GSO_MBUF_CACHE_SIZE	4
+#define TAP_GSO_MBUF_CACHE_SIZE	32
 #define TAP_GSO_MBUFS_NUM \
 	(TAP_GSO_MBUFS_PER_CORE * TAP_GSO_MBUF_CACHE_SIZE)
 
lib/eal/include/rte_common.h +12 -0
index 79d2a0ab93..0fd0906506 100644
@@ -567,6 +567,15 @@ static void __attribute__((destructor(RTE_PRIO(prio)), used)) func(void)
 #define __rte_assume(condition) __assume(condition)
 #endif
 
+/**
+ * Alignment hint precondition
+ */
+#ifdef RTE_TOOLCHAIN_MSVC
+#define __rte_assume_aligned(ptr, alignment) (ptr)
+#else
+#define __rte_assume_aligned(ptr, alignment) __builtin_assume_aligned(ptr, alignment)
+#endif
+
 /**
  * Disable AddressSanitizer on some code
  */
@@ -775,6 +784,9 @@ rte_is_aligned(const void * const __rte_restrict ptr, const unsigned int align)
 /** Force minimum cache line alignment. */
 #define __rte_cache_min_aligned __rte_aligned(RTE_CACHE_LINE_MIN_SIZE)
 
+/** Cache alignment hint precondition */
+#define __rte_assume_cache_aligned(ptr) __rte_assume_aligned(ptr, RTE_CACHE_LINE_SIZE)
+
 #define _RTE_CACHE_GUARD_HELPER2(unique) \
 	alignas(RTE_CACHE_LINE_SIZE) \
 	char cache_guard_ ## unique[RTE_CACHE_LINE_SIZE * RTE_CACHE_GUARD_LINES]
lib/eal/x86/include/rte_memcpy.h +29 -0
index 8ed8c55010..3fe1e8d247 100644
@@ -707,6 +707,35 @@ rte_memcpy(void *__rte_restrict dst, const void *__rte_restrict src, size_t n)
 #endif
 		return dst;
 	}
+	/* Common way for small copy size of 64-byte blocks */
+#if defined __AVX512F__ && defined RTE_MEMCPY_AVX512
+	if (__rte_constant(n) && (n & 63) == 0 && n <= 512) {
+#elif defined RTE_MEMCPY_AVX
+	if (__rte_constant(n) && (n & 63) == 0 && n <= 256) {
+#else /* SSE implementation */
+	if (__rte_constant(n) && (n & 63) == 0 && n <= 512) {
+#endif
+		void *ret = dst;
+
+		if (n & 512) {
+			rte_mov256((uint8_t *)dst + 0 * 256, (const uint8_t *)src + 0 * 256);
+			rte_mov256((uint8_t *)dst + 1 * 256, (const uint8_t *)src + 1 * 256);
+		}
+		if (n & 256) {
+			rte_mov256((uint8_t *)dst, (const uint8_t *)src);
+			src = (const uint8_t *)src + 256;
+			dst = (uint8_t *)dst + 256;
+		}
+		if (n & 128) {
+			rte_mov128((uint8_t *)dst, (const uint8_t *)src);
+			src = (const uint8_t *)src + 128;
+			dst = (uint8_t *)dst + 128;
+		}
+		if (n & 64)
+			rte_mov64((uint8_t *)dst, (const uint8_t *)src);
+
+		return ret;
+	}
 
 	/* Implementation for size > 64 bytes depends on alignment with vector register size. */
 	if (!(((uintptr_t)dst | (uintptr_t)src) & ALIGNMENT_MASK))
lib/mempool/mempool_trace.h +0 -1
index 23cda1473c..60e47cf67b 100644
@@ -119,7 +119,6 @@ RTE_TRACE_POINT(
 	rte_trace_point_emit_i32(socket_id);
 	rte_trace_point_emit_ptr(cache);
 	rte_trace_point_emit_u32(cache->len);
-	rte_trace_point_emit_u32(cache->flushthresh);
 )
 
 RTE_TRACE_POINT(
lib/mempool/rte_mempool.c +30 -22

118 lines not on this page · raw message

lib/mempool/rte_mempool.h +35 -39

155 lines not on this page · raw message

lib/stack/meson.build +2 -1
index 18177a742f..50e688522e 100644
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2019 Intel Corporation
 
-sources = files('rte_stack.c', 'rte_stack_std.c', 'rte_stack_lf.c')
+sources = files('rte_stack.c', 'rte_stack_std.c', 'rte_stack_lf.c', 'rte_stack_pile.c')
 headers = files('rte_stack.h')
 # subheaders, not for direct inclusion by apps
 indirect_headers += files(
@@ -10,4 +10,5 @@ indirect_headers += files(
         'rte_stack_lf_generic.h',
         'rte_stack_lf_c11.h',
         'rte_stack_lf_stubs.h',
+        'rte_stack_pile.h',
 )
lib/stack/rte_stack.c +17 -1
index 4c78fe4b4b..a4bbf8a4d7 100644
@@ -1,5 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright(c) 2019 Intel Corporation
+ * Copyright(c) 2026 SmartShare Systems
  */
 
 #include <stdalign.h>
@@ -32,6 +33,8 @@ rte_stack_init(struct rte_stack *s, unsigned int count, uint32_t flags)
 
 	if (flags & RTE_STACK_F_LF)
 		rte_stack_lf_init(s, count);
+	else if (flags & RTE_STACK_F_PILE)
+		rte_stack_pile_init(s, count);
 	else
 		rte_stack_std_init(s);
 }
@@ -41,6 +44,8 @@ rte_stack_get_memsize(unsigned int count, uint32_t flags)
 {
 	if (flags & RTE_STACK_F_LF)
 		return rte_stack_lf_get_memsize(count);
+	else if (flags & RTE_STACK_F_PILE)
+		return rte_stack_pile_get_memsize(count);
 	else
 		return rte_stack_std_get_memsize(count);
 }
@@ -58,7 +63,11 @@ rte_stack_create(const char *name, unsigned int count, int socket_id,
 	unsigned int sz;
 	int ret;
 
-	if (flags & ~(RTE_STACK_F_LF)) {
+	if (flags & ~(RTE_STACK_F_LF | RTE_STACK_F_PILE)) {
+		STACK_LOG_ERR("Unsupported stack flags %#x", flags);
+		return NULL;
+	}
+	if ((flags & RTE_STACK_F_LF) && (flags & RTE_STACK_F_PILE)) {
 		STACK_LOG_ERR("Unsupported stack flags %#x", flags);
 		return NULL;
 	}
@@ -73,6 +82,13 @@ rte_stack_create(const char *name, unsigned int count, int socket_id,
 		return NULL;
 	}
 #endif
+#if !defined(RTE_STACK_PILE_SUPPORTED)
+	if (flags & RTE_STACK_F_PILE) {
+		STACK_LOG_ERR("Pile is not supported on your platform");
+		rte_errno = ENOTSUP;
+		return NULL;
+	}
+#endif
 
 	sz = rte_stack_get_memsize(count, flags);
 
lib/stack/rte_stack.h +77 -2

151 lines not on this page · raw message

lib/stack/rte_stack_lf.h +4 -2
index f2b012cd0e..655620aaa9 100644
@@ -34,7 +34,7 @@
  */
 static __rte_always_inline unsigned int
 __rte_stack_lf_push(struct rte_stack *s,
-		    void * const *obj_table,
+		    void * const * __rte_restrict obj_table,
 		    unsigned int n)
 {
 	struct rte_stack_lf_elem *tmp, *first, *last = NULL;
@@ -71,7 +71,8 @@ __rte_stack_lf_push(struct rte_stack *s,
  *   - Actual number of objects popped.
  */
 static __rte_always_inline unsigned int
-__rte_stack_lf_pop(struct rte_stack *s, void **obj_table, unsigned int n)
+__rte_stack_lf_pop(struct rte_stack *s, void ** __rte_restrict obj_table,
+		   unsigned int n)
 {
 	struct rte_stack_lf_elem *first, *last = NULL;
 
@@ -79,6 +80,7 @@ __rte_stack_lf_pop(struct rte_stack *s, void **obj_table, unsigned int n)
 		return 0;
 
 	/* Pop n used elements */
+	__rte_assume(obj_table != NULL);
 	first = __rte_stack_lf_pop_elems(&s->stack_lf.used,
 					 n, obj_table, &last);
 	if (unlikely(first == NULL))
lib/stack/rte_stack_lf_c11.h +1 -1
index b97e02d6a1..501e985a49 100644
@@ -99,7 +99,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 static __rte_always_inline struct rte_stack_lf_elem *
 __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 unsigned int num,
-			 void **obj_table,
+			 void ** __rte_restrict obj_table,
 			 struct rte_stack_lf_elem **last)
 {
 	struct rte_stack_lf_head old_head;
lib/stack/rte_stack_lf_generic.h +1 -1
index cc69e4d168..c4cc4d2c03 100644
@@ -74,7 +74,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 static __rte_always_inline struct rte_stack_lf_elem *
 __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 unsigned int num,
-			 void **obj_table,
+			 void ** __rte_restrict obj_table,
 			 struct rte_stack_lf_elem **last)
 {
 	struct rte_stack_lf_head old_head;
lib/stack/rte_stack_lf_stubs.h +1 -1
index a05abf1f1c..457a415ccf 100644
@@ -30,7 +30,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 static __rte_always_inline struct rte_stack_lf_elem *
 __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 unsigned int num,
-			 void **obj_table,
+			 void ** __rte_restrict obj_table,
 			 struct rte_stack_lf_elem **last)
 {
 	RTE_SET_USED(obj_table);
lib/stack/rte_stack_pile.c new file +36 -0
new file mode 100644
index 0000000000..aa2901c926
@@ -0,0 +1,36 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2026 SmartShare Systems
+ */
+
+#include "rte_stack.h"
+
+RTE_EXPORT_INTERNAL_SYMBOL(rte_stack_pile_init)
+void
+rte_stack_pile_init(struct rte_stack *s, unsigned int count)
+{
+	unsigned int bulk = (count + RTE_STACK_PILE_BULK_SIZE - 1) / RTE_STACK_PILE_BULK_SIZE;
+	struct rte_stack_pile_bulk_elem *bulk_elems =
+			(struct rte_stack_pile_bulk_elem *)(&s->stack_pile + 1);
+	struct rte_stack_lf_elem *solo_elems = (struct rte_stack_lf_elem *)&bulk_elems[bulk];
+	unsigned int i;
+
+	for (i = 0; i < bulk; i++)
+		__rte_stack_pile_bulk_push_elems(&s->stack_pile.free_bulk,
+					  &bulk_elems[i], &bulk_elems[i], 1);
+	for (i = 0; i < count; i++)
+		__rte_stack_lf_push_elems(&s->stack_pile.free_solo,
+					  &solo_elems[i], &solo_elems[i], 1);
+}
+
+RTE_EXPORT_INTERNAL_SYMBOL(rte_stack_pile_get_memsize)
+ssize_t
+rte_stack_pile_get_memsize(unsigned int count)
+{
+	unsigned int bulk = (count + RTE_STACK_PILE_BULK_SIZE - 1) / RTE_STACK_PILE_BULK_SIZE;
+	ssize_t sz = sizeof(struct rte_stack); /* Already cache line aligned. */
+	sz += bulk * sizeof(struct rte_stack_pile_bulk_elem); /* Already cache line aligned. */
+	sz += RTE_CACHE_LINE_ROUNDUP(count * sizeof(struct rte_stack_lf_elem));
+	sz += RTE_CACHE_GUARD_LINES * RTE_CACHE_LINE_SIZE;
+
+	return sz;
+}
lib/stack/rte_stack_pile.h new file +328 -0

328 lines not on this page · raw message

lib/stack/rte_stack_std.h +17 -20
index ae28add5c4..cb815d6cbe 100644
@@ -6,6 +6,7 @@
 #define _RTE_STACK_STD_H_
 
 #include <rte_branch_prediction.h>
+#include <rte_memcpy.h>
 
 /**
  * @internal Push several objects on the stack (MT-safe).
@@ -20,27 +21,24 @@
  *   Actual number of objects pushed (either 0 or *n*).
  */
 static __rte_always_inline unsigned int
-__rte_stack_std_push(struct rte_stack *s, void * const *obj_table,
+__rte_stack_std_push(struct rte_stack *s, void * const * __rte_restrict obj_table,
 		     unsigned int n)
 {
 	struct rte_stack_std *stack = &s->stack_std;
-	unsigned int index;
-	void **cache_objs;
+	void ** __rte_restrict stack_objs;
 
 	rte_spinlock_lock(&stack->lock);
-	cache_objs = &stack->objs[stack->len];
 
-	/* Is there sufficient space in the stack? */
-	if ((stack->len + n) > s->capacity) {
+	if (unlikely((stack->len + n) > s->capacity)) {
+		/* Insufficient room in the stack. */
 		rte_spinlock_unlock(&stack->lock);
 		return 0;
 	}
 
-	/* Add elements back into the cache */
-	for (index = 0; index < n; ++index, obj_table++)
-		cache_objs[index] = *obj_table;
-
+	/* Push objects to the stack */
+	stack_objs = &stack->objs[stack->len];
 	stack->len += n;
+	rte_memcpy(stack_objs, obj_table, sizeof(void *) * n);
 
 	rte_spinlock_unlock(&stack->lock);
 	return n;
@@ -59,28 +57,27 @@ __rte_stack_std_push(struct rte_stack *s, void * const *obj_table,
  *   Actual number of objects popped (either 0 or *n*).
  */
 static __rte_always_inline unsigned int
-__rte_stack_std_pop(struct rte_stack *s, void **obj_table, unsigned int n)
+__rte_stack_std_pop(struct rte_stack *s, void ** __rte_restrict obj_table, unsigned int n)
 {
 	struct rte_stack_std *stack = &s->stack_std;
-	unsigned int index, len;
-	void **cache_objs;
+	unsigned int index;
+	void ** __rte_restrict stack_objs;
 
 	rte_spinlock_lock(&stack->lock);
 
 	if (unlikely(n > stack->len)) {
+		/* Insufficient objects in the stack. */
 		rte_spinlock_unlock(&stack->lock);
 		return 0;
 	}
 
-	cache_objs = stack->objs;
-
-	for (index = 0, len = stack->len - 1; index < n;
-			++index, len--, obj_table++)
-		*obj_table = cache_objs[len];
-
+	/* Pop objects from the stack */
+	stack_objs = &stack->objs[stack->len];
 	stack->len -= n;
-	rte_spinlock_unlock(&stack->lock);
+	for (index = 0; index < n; index++)
+		*obj_table++ = *--stack_objs;
 
+	rte_spinlock_unlock(&stack->lock);
 	return n;
 }
 
Signature · 1 line
2.43.0
#3 lore raw

RFC PATCH v4 1/1 NEW: pile stack and mempool driver

In reply to Morten Brørup

Early submission of:
- some mempool optimizations,
- a new mempool "pile" driver, and
- its underlying "pile" stack implementation.

For community feedback and CI test.

Needless to say, this must be separated into a series of patches. For now, I'm submitting a snapshot of work in progress.

Some performance numbers from mempool_perf_autotest_2cores, all with cache=1024 cores=2 n_keep=32768:

start performance test (using ring_mp_mc, with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 753985338 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 755805913

start performance test for lf_stack (with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 29132352 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 29276708

start performance test for pile (with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 560159479 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 557910933

Hat tip to Bruce for bringing attention to the ring not being the optimal mempool driver.

---
v2:
* Fix indentation, long lines, etc. (checkpatch)
* Fix label followed by a declaration is a C23 extension.
* Added __rte_internal to pile init and get_memsize. (AI)
* Fix roll back bulk elements in wrong order with fragmentation. (AI)
* Minor changes suggested by AI.
v3:
* Revert rename unused field in rte_stack_pile_bulk_elem structure. v4:
* Revert add __rte_internal.
Compilation fails, and other stack implementations don't have it.

---
 app/test/test_mempool.c                   |   3 +-
 app/test/test_stack.c                     |  68 ++++-
 app/test/test_stack_perf.c                |  15 +-
 config/rte_config.h                       |   5 +-
 doc/guides/prog_guide/stack_lib.rst       |  67 ++++-
 drivers/mempool/stack/rte_mempool_stack.c |  40 +++
 drivers/net/bonding/rte_eth_bond_pmd.c    |   2 +-
 drivers/net/intel/cpfl/cpfl_rxtx.h        |   2 +-
 drivers/net/sxe2/sxe2_txrx_vec_avx512.c   |   3 +-
 drivers/net/tap/rte_eth_tap.c             |   2 +-
 lib/eal/include/rte_common.h              |  12 +
 lib/eal/x86/include/rte_memcpy.h          |  29 ++
 lib/mempool/mempool_trace.h               |   1 -
 lib/mempool/rte_mempool.c                 |  52 ++--
 lib/mempool/rte_mempool.h                 |  74 +++--
 lib/stack/meson.build                     |   3 +-
 lib/stack/rte_stack.c                     |  18 +-
 lib/stack/rte_stack.h                     |  79 +++++-
 lib/stack/rte_stack_lf.h                  |   6 +-
 lib/stack/rte_stack_lf_c11.h              |   2 +-
 lib/stack/rte_stack_lf_generic.h          |   2 +-
 lib/stack/rte_stack_lf_stubs.h            |   2 +-
 lib/stack/rte_stack_pile.c                |  34 +++
 lib/stack/rte_stack_pile.h                | 326 ++++++++++++++++++++++
 lib/stack/rte_stack_std.h                 |  37 ++-
 25 files changed, 768 insertions(+), 116 deletions(-)
 create mode 100644 lib/stack/rte_stack_pile.c
 create mode 100644 lib/stack/rte_stack_pile.h
app/test/test_mempool.c +1 -2
index e54249ce61..76d45cea2a 100644
@@ -112,8 +112,7 @@ test_mempool_basic(struct rte_mempool *mp, int use_external_cache)
 		GOTO_ERR(ret, out);
 
 	printf("get private data\n");
-	if (rte_mempool_get_priv(mp) != (char *)mp +
-			RTE_MEMPOOL_HEADER_SIZE(mp, mp->cache_size))
+	if (rte_mempool_get_priv(mp) != (char *)mp + sizeof(struct rte_mempool))
 		GOTO_ERR(ret, out);
 
 #ifndef RTE_EXEC_ENV_FREEBSD /* rte_mem_virt2iova() not supported on bsd */
app/test/test_stack.c +56 -12

86 lines not on this page · raw message

app/test/test_stack_perf.c +13 -2
index 3f17a2606c..a15f3719c2 100644
@@ -14,14 +14,14 @@
 #include "test.h"
 
 #define STACK_NAME "STACK_PERF"
-#define MAX_BURST 32
+#define MAX_BURST (RTE_MEMPOOL_CACHE_MAX_SIZE / 2)
 #define STACK_SIZE (RTE_MAX_LCORE * MAX_BURST)
 
 /*
  * Push/pop bulk sizes, marked volatile so they aren't treated as compile-time
  * constants.
  */
-static volatile unsigned int bulk_sizes[] = {8, MAX_BURST};
+static volatile unsigned int bulk_sizes[] = {1, 8, 32, MAX_BURST};
 
 static RTE_ATOMIC(uint32_t) lcore_barrier;
 
@@ -354,5 +354,16 @@ test_lf_stack_perf(void)
 #endif
 }
 
+static int
+test_pile_perf(void)
+{
+#if defined(RTE_STACK_PILE_SUPPORTED)
+	return __test_stack_perf(RTE_STACK_F_PILE);
+#else
+	return TEST_SKIPPED;
+#endif
+}
+
 REGISTER_PERF_TEST(stack_perf_autotest, test_stack_perf);
 REGISTER_PERF_TEST(stack_lf_perf_autotest, test_lf_stack_perf);
+REGISTER_PERF_TEST(stack_pile_perf_autotest, test_pile_perf);
config/rte_config.h +4 -1
index 0447cdf2ad..03350660e4 100644
@@ -56,7 +56,7 @@
 #define RTE_CONTIGMEM_DEFAULT_BUF_SIZE (512*1024*1024)
 
 /* mempool defines */
-#define RTE_MEMPOOL_CACHE_MAX_SIZE 512
+#define RTE_MEMPOOL_CACHE_MAX_SIZE 1024
 /* RTE_LIBRTE_MEMPOOL_STATS is not set */
 /* RTE_LIBRTE_MEMPOOL_DEBUG is not set */
 
@@ -64,6 +64,9 @@
 #define RTE_MBUF_DEFAULT_MEMPOOL_OPS "ring_mp_mc"
 /* RTE_MBUF_HISTORY_DEBUG is not set */
 
+/* stack defines */
+#define RTE_STACK_PILE_BULK_SIZE 32
+
 /* ether defines */
 #define RTE_MAX_QUEUES_PER_PORT 1024
 #define RTE_ETHDEV_RXTX_CALLBACKS 1
doc/guides/prog_guide/stack_lib.rst +63 -4

95 lines not on this page · raw message

drivers/mempool/stack/rte_mempool_stack.c +40 -0
index 1476905227..7467b8b39e 100644
@@ -41,6 +41,36 @@ lf_stack_alloc(struct rte_mempool *mp)
 	return __stack_alloc(mp, RTE_STACK_F_LF);
 }
 
+static int
+pile_alloc(struct rte_mempool *mp)
+{
+	return __stack_alloc(mp, RTE_STACK_F_PILE);
+}
+
+static int
+pile_enqueue(struct rte_mempool *mp, void * const *obj_table,
+	      unsigned int n)
+{
+	struct rte_stack *s = mp->pool_data;
+
+	RTE_ASSERT(s != NULL);
+	RTE_ASSERT(obj_table != NULL);
+
+	return __rte_stack_pile_push(s, obj_table, n) == 0 ? -ENOBUFS : 0;
+}
+
+static int
+pile_dequeue(struct rte_mempool *mp, void **obj_table,
+	      unsigned int n)
+{
+	struct rte_stack *s = mp->pool_data;
+
+	RTE_ASSERT(s != NULL);
+	RTE_ASSERT(obj_table != NULL);
+
+	return __rte_stack_pile_pop(s, obj_table, n) == 0 ? -ENOBUFS : 0;
+}
+
 static int
 stack_enqueue(struct rte_mempool *mp, void * const *obj_table,
 	      unsigned int n)
@@ -93,5 +123,15 @@ static struct rte_mempool_ops ops_lf_stack = {
 	.get_count = stack_get_count
 };
 
+static struct rte_mempool_ops ops_pile = {
+	.name = "pile",
+	.alloc = pile_alloc,
+	.free = stack_free,
+	.enqueue = pile_enqueue,
+	.dequeue = pile_dequeue,
+	.get_count = stack_get_count
+};
+
 RTE_MEMPOOL_REGISTER_OPS(ops_stack);
 RTE_MEMPOOL_REGISTER_OPS(ops_lf_stack);
+RTE_MEMPOOL_REGISTER_OPS(ops_pile);
drivers/net/bonding/rte_eth_bond_pmd.c +1 -1
index 6a4f997b5a..92d7f4f4ef 100644
@@ -1702,7 +1702,7 @@ member_configure_slow_queue(struct rte_eth_dev *bonding_eth_dev,
 		snprintf(mem_name, RTE_DIM(mem_name), "member_port%u_slow_pool",
 				member_id);
 		port->slow_pool = rte_pktmbuf_pool_create(mem_name, 8191,
-			250, 0, RTE_MBUF_DEFAULT_BUF_SIZE,
+			256, 0, RTE_MBUF_DEFAULT_BUF_SIZE,
 			member_eth_dev->data->numa_node);
 
 		/* Any memory allocation failure in initialization is critical because
drivers/net/intel/cpfl/cpfl_rxtx.h +1 -1
index 52cdecac88..faf28fd489 100644
@@ -25,7 +25,7 @@
 #define CPFL_P2P_QUEUE_GRP_ID	1
 #define CPFL_P2P_DESC_LEN	16
 #define CPFL_P2P_NB_MBUF	4096
-#define CPFL_P2P_CACHE_SIZE	250
+#define CPFL_P2P_CACHE_SIZE	256
 #define CPFL_P2P_MBUF_SIZE	2048
 #define CPFL_P2P_RING_BUF	128
 
drivers/net/sxe2/sxe2_txrx_vec_avx512.c +2 -1
index a830c7a33b..4ded5cb63e 100644
@@ -67,11 +67,12 @@ static __rte_always_inline int32_t sxe2_tx_bufs_free_vec_avx512(struct sxe2_tx_q
 		}
 		cache->len += rs_thresh;
 
-		if (cache->len >= cache->flushthresh) {
+		if (cache->len >= cache->size) {
 			(void)rte_mempool_ops_enqueue_bulk(mp,
 					&cache->objs[cache->size], cache->len - cache->size);
 			cache->len = cache->size;
 		}
+
 		goto done;
 	}
 
drivers/net/tap/rte_eth_tap.c +1 -1
index b93452f168..b3142561c2 100644
@@ -61,7 +61,7 @@
 #define TAP_MAX_MAC_ADDRS	16
 #define TAP_GSO_MBUFS_PER_CORE	128
 #define TAP_GSO_MBUF_SEG_SIZE	128
-#define TAP_GSO_MBUF_CACHE_SIZE	4
+#define TAP_GSO_MBUF_CACHE_SIZE	32
 #define TAP_GSO_MBUFS_NUM \
 	(TAP_GSO_MBUFS_PER_CORE * TAP_GSO_MBUF_CACHE_SIZE)
 
lib/eal/include/rte_common.h +12 -0
index 79d2a0ab93..0fd0906506 100644
@@ -567,6 +567,15 @@ static void __attribute__((destructor(RTE_PRIO(prio)), used)) func(void)
 #define __rte_assume(condition) __assume(condition)
 #endif
 
+/**
+ * Alignment hint precondition
+ */
+#ifdef RTE_TOOLCHAIN_MSVC
+#define __rte_assume_aligned(ptr, alignment) (ptr)
+#else
+#define __rte_assume_aligned(ptr, alignment) __builtin_assume_aligned(ptr, alignment)
+#endif
+
 /**
  * Disable AddressSanitizer on some code
  */
@@ -775,6 +784,9 @@ rte_is_aligned(const void * const __rte_restrict ptr, const unsigned int align)
 /** Force minimum cache line alignment. */
 #define __rte_cache_min_aligned __rte_aligned(RTE_CACHE_LINE_MIN_SIZE)
 
+/** Cache alignment hint precondition */
+#define __rte_assume_cache_aligned(ptr) __rte_assume_aligned(ptr, RTE_CACHE_LINE_SIZE)
+
 #define _RTE_CACHE_GUARD_HELPER2(unique) \
 	alignas(RTE_CACHE_LINE_SIZE) \
 	char cache_guard_ ## unique[RTE_CACHE_LINE_SIZE * RTE_CACHE_GUARD_LINES]
lib/eal/x86/include/rte_memcpy.h +29 -0
index 8ed8c55010..3fe1e8d247 100644
@@ -707,6 +707,35 @@ rte_memcpy(void *__rte_restrict dst, const void *__rte_restrict src, size_t n)
 #endif
 		return dst;
 	}
+	/* Common way for small copy size of 64-byte blocks */
+#if defined __AVX512F__ && defined RTE_MEMCPY_AVX512
+	if (__rte_constant(n) && (n & 63) == 0 && n <= 512) {
+#elif defined RTE_MEMCPY_AVX
+	if (__rte_constant(n) && (n & 63) == 0 && n <= 256) {
+#else /* SSE implementation */
+	if (__rte_constant(n) && (n & 63) == 0 && n <= 512) {
+#endif
+		void *ret = dst;
+
+		if (n & 512) {
+			rte_mov256((uint8_t *)dst + 0 * 256, (const uint8_t *)src + 0 * 256);
+			rte_mov256((uint8_t *)dst + 1 * 256, (const uint8_t *)src + 1 * 256);
+		}
+		if (n & 256) {
+			rte_mov256((uint8_t *)dst, (const uint8_t *)src);
+			src = (const uint8_t *)src + 256;
+			dst = (uint8_t *)dst + 256;
+		}
+		if (n & 128) {
+			rte_mov128((uint8_t *)dst, (const uint8_t *)src);
+			src = (const uint8_t *)src + 128;
+			dst = (uint8_t *)dst + 128;
+		}
+		if (n & 64)
+			rte_mov64((uint8_t *)dst, (const uint8_t *)src);
+
+		return ret;
+	}
 
 	/* Implementation for size > 64 bytes depends on alignment with vector register size. */
 	if (!(((uintptr_t)dst | (uintptr_t)src) & ALIGNMENT_MASK))
lib/mempool/mempool_trace.h +0 -1
index 23cda1473c..60e47cf67b 100644
@@ -119,7 +119,6 @@ RTE_TRACE_POINT(
 	rte_trace_point_emit_i32(socket_id);
 	rte_trace_point_emit_ptr(cache);
 	rte_trace_point_emit_u32(cache->len);
-	rte_trace_point_emit_u32(cache->flushthresh);
 )
 
 RTE_TRACE_POINT(
lib/mempool/rte_mempool.c +30 -22

118 lines not on this page · raw message

lib/mempool/rte_mempool.h +35 -39

155 lines not on this page · raw message

lib/stack/meson.build +2 -1
index 18177a742f..50e688522e 100644
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2019 Intel Corporation
 
-sources = files('rte_stack.c', 'rte_stack_std.c', 'rte_stack_lf.c')
+sources = files('rte_stack.c', 'rte_stack_std.c', 'rte_stack_lf.c', 'rte_stack_pile.c')
 headers = files('rte_stack.h')
 # subheaders, not for direct inclusion by apps
 indirect_headers += files(
@@ -10,4 +10,5 @@ indirect_headers += files(
         'rte_stack_lf_generic.h',
         'rte_stack_lf_c11.h',
         'rte_stack_lf_stubs.h',
+        'rte_stack_pile.h',
 )
lib/stack/rte_stack.c +17 -1
index 4c78fe4b4b..a4bbf8a4d7 100644
@@ -1,5 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright(c) 2019 Intel Corporation
+ * Copyright(c) 2026 SmartShare Systems
  */
 
 #include <stdalign.h>
@@ -32,6 +33,8 @@ rte_stack_init(struct rte_stack *s, unsigned int count, uint32_t flags)
 
 	if (flags & RTE_STACK_F_LF)
 		rte_stack_lf_init(s, count);
+	else if (flags & RTE_STACK_F_PILE)
+		rte_stack_pile_init(s, count);
 	else
 		rte_stack_std_init(s);
 }
@@ -41,6 +44,8 @@ rte_stack_get_memsize(unsigned int count, uint32_t flags)
 {
 	if (flags & RTE_STACK_F_LF)
 		return rte_stack_lf_get_memsize(count);
+	else if (flags & RTE_STACK_F_PILE)
+		return rte_stack_pile_get_memsize(count);
 	else
 		return rte_stack_std_get_memsize(count);
 }
@@ -58,7 +63,11 @@ rte_stack_create(const char *name, unsigned int count, int socket_id,
 	unsigned int sz;
 	int ret;
 
-	if (flags & ~(RTE_STACK_F_LF)) {
+	if (flags & ~(RTE_STACK_F_LF | RTE_STACK_F_PILE)) {
+		STACK_LOG_ERR("Unsupported stack flags %#x", flags);
+		return NULL;
+	}
+	if ((flags & RTE_STACK_F_LF) && (flags & RTE_STACK_F_PILE)) {
 		STACK_LOG_ERR("Unsupported stack flags %#x", flags);
 		return NULL;
 	}
@@ -73,6 +82,13 @@ rte_stack_create(const char *name, unsigned int count, int socket_id,
 		return NULL;
 	}
 #endif
+#if !defined(RTE_STACK_PILE_SUPPORTED)
+	if (flags & RTE_STACK_F_PILE) {
+		STACK_LOG_ERR("Pile is not supported on your platform");
+		rte_errno = ENOTSUP;
+		return NULL;
+	}
+#endif
 
 	sz = rte_stack_get_memsize(count, flags);
 
lib/stack/rte_stack.h +77 -2

151 lines not on this page · raw message

lib/stack/rte_stack_lf.h +4 -2
index f2b012cd0e..655620aaa9 100644
@@ -34,7 +34,7 @@
  */
 static __rte_always_inline unsigned int
 __rte_stack_lf_push(struct rte_stack *s,
-		    void * const *obj_table,
+		    void * const * __rte_restrict obj_table,
 		    unsigned int n)
 {
 	struct rte_stack_lf_elem *tmp, *first, *last = NULL;
@@ -71,7 +71,8 @@ __rte_stack_lf_push(struct rte_stack *s,
  *   - Actual number of objects popped.
  */
 static __rte_always_inline unsigned int
-__rte_stack_lf_pop(struct rte_stack *s, void **obj_table, unsigned int n)
+__rte_stack_lf_pop(struct rte_stack *s, void ** __rte_restrict obj_table,
+		   unsigned int n)
 {
 	struct rte_stack_lf_elem *first, *last = NULL;
 
@@ -79,6 +80,7 @@ __rte_stack_lf_pop(struct rte_stack *s, void **obj_table, unsigned int n)
 		return 0;
 
 	/* Pop n used elements */
+	__rte_assume(obj_table != NULL);
 	first = __rte_stack_lf_pop_elems(&s->stack_lf.used,
 					 n, obj_table, &last);
 	if (unlikely(first == NULL))
lib/stack/rte_stack_lf_c11.h +1 -1
index b97e02d6a1..501e985a49 100644
@@ -99,7 +99,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 static __rte_always_inline struct rte_stack_lf_elem *
 __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 unsigned int num,
-			 void **obj_table,
+			 void ** __rte_restrict obj_table,
 			 struct rte_stack_lf_elem **last)
 {
 	struct rte_stack_lf_head old_head;
lib/stack/rte_stack_lf_generic.h +1 -1
index cc69e4d168..c4cc4d2c03 100644
@@ -74,7 +74,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 static __rte_always_inline struct rte_stack_lf_elem *
 __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 unsigned int num,
-			 void **obj_table,
+			 void ** __rte_restrict obj_table,
 			 struct rte_stack_lf_elem **last)
 {
 	struct rte_stack_lf_head old_head;
lib/stack/rte_stack_lf_stubs.h +1 -1
index a05abf1f1c..457a415ccf 100644
@@ -30,7 +30,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 static __rte_always_inline struct rte_stack_lf_elem *
 __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 unsigned int num,
-			 void **obj_table,
+			 void ** __rte_restrict obj_table,
 			 struct rte_stack_lf_elem **last)
 {
 	RTE_SET_USED(obj_table);
lib/stack/rte_stack_pile.c new file +34 -0
new file mode 100644
index 0000000000..920bccd5ab
@@ -0,0 +1,34 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2026 SmartShare Systems
+ */
+
+#include "rte_stack.h"
+
+void
+rte_stack_pile_init(struct rte_stack *s, unsigned int count)
+{
+	unsigned int bulk = (count + RTE_STACK_PILE_BULK_SIZE - 1) / RTE_STACK_PILE_BULK_SIZE;
+	struct rte_stack_pile_bulk_elem *bulk_elems =
+			(struct rte_stack_pile_bulk_elem *)(&s->stack_pile + 1);
+	struct rte_stack_lf_elem *solo_elems = (struct rte_stack_lf_elem *)&bulk_elems[bulk];
+	unsigned int i;
+
+	for (i = 0; i < bulk; i++)
+		__rte_stack_pile_bulk_push_elems(&s->stack_pile.free_bulk,
+					  &bulk_elems[i], &bulk_elems[i], 1);
+	for (i = 0; i < count; i++)
+		__rte_stack_lf_push_elems(&s->stack_pile.free_solo,
+					  &solo_elems[i], &solo_elems[i], 1);
+}
+
+ssize_t
+rte_stack_pile_get_memsize(unsigned int count)
+{
+	unsigned int bulk = (count + RTE_STACK_PILE_BULK_SIZE - 1) / RTE_STACK_PILE_BULK_SIZE;
+	ssize_t sz = sizeof(struct rte_stack); /* Already cache line aligned. */
+	sz += bulk * sizeof(struct rte_stack_pile_bulk_elem); /* Already cache line aligned. */
+	sz += RTE_CACHE_LINE_ROUNDUP(count * sizeof(struct rte_stack_lf_elem));
+	sz += RTE_CACHE_GUARD_LINES * RTE_CACHE_LINE_SIZE;
+
+	return sz;
+}
lib/stack/rte_stack_pile.h new file +326 -0

326 lines not on this page · raw message

lib/stack/rte_stack_std.h +17 -20
index ae28add5c4..cb815d6cbe 100644
@@ -6,6 +6,7 @@
 #define _RTE_STACK_STD_H_
 
 #include <rte_branch_prediction.h>
+#include <rte_memcpy.h>
 
 /**
  * @internal Push several objects on the stack (MT-safe).
@@ -20,27 +21,24 @@
  *   Actual number of objects pushed (either 0 or *n*).
  */
 static __rte_always_inline unsigned int
-__rte_stack_std_push(struct rte_stack *s, void * const *obj_table,
+__rte_stack_std_push(struct rte_stack *s, void * const * __rte_restrict obj_table,
 		     unsigned int n)
 {
 	struct rte_stack_std *stack = &s->stack_std;
-	unsigned int index;
-	void **cache_objs;
+	void ** __rte_restrict stack_objs;
 
 	rte_spinlock_lock(&stack->lock);
-	cache_objs = &stack->objs[stack->len];
 
-	/* Is there sufficient space in the stack? */
-	if ((stack->len + n) > s->capacity) {
+	if (unlikely((stack->len + n) > s->capacity)) {
+		/* Insufficient room in the stack. */
 		rte_spinlock_unlock(&stack->lock);
 		return 0;
 	}
 
-	/* Add elements back into the cache */
-	for (index = 0; index < n; ++index, obj_table++)
-		cache_objs[index] = *obj_table;
-
+	/* Push objects to the stack */
+	stack_objs = &stack->objs[stack->len];
 	stack->len += n;
+	rte_memcpy(stack_objs, obj_table, sizeof(void *) * n);
 
 	rte_spinlock_unlock(&stack->lock);
 	return n;
@@ -59,28 +57,27 @@ __rte_stack_std_push(struct rte_stack *s, void * const *obj_table,
  *   Actual number of objects popped (either 0 or *n*).
  */
 static __rte_always_inline unsigned int
-__rte_stack_std_pop(struct rte_stack *s, void **obj_table, unsigned int n)
+__rte_stack_std_pop(struct rte_stack *s, void ** __rte_restrict obj_table, unsigned int n)
 {
 	struct rte_stack_std *stack = &s->stack_std;
-	unsigned int index, len;
-	void **cache_objs;
+	unsigned int index;
+	void ** __rte_restrict stack_objs;
 
 	rte_spinlock_lock(&stack->lock);
 
 	if (unlikely(n > stack->len)) {
+		/* Insufficient objects in the stack. */
 		rte_spinlock_unlock(&stack->lock);
 		return 0;
 	}
 
-	cache_objs = stack->objs;
-
-	for (index = 0, len = stack->len - 1; index < n;
-			++index, len--, obj_table++)
-		*obj_table = cache_objs[len];
-
+	/* Pop objects from the stack */
+	stack_objs = &stack->objs[stack->len];
 	stack->len -= n;
-	rte_spinlock_unlock(&stack->lock);
+	for (index = 0; index < n; index++)
+		*obj_table++ = *--stack_objs;
 
+	rte_spinlock_unlock(&stack->lock);
 	return n;
 }
 
Signature · 1 line
2.43.0
#4 lore raw

RFC PATCH v5 1/1 pile stack and mempool driver

In reply to Morten Brørup

Early submission of:
- some mempool optimizations,
- a new mempool "pile" driver, and
- its underlying "pile" stack implementation.

For community feedback and CI test.

Needless to say, this must be separated into a series of patches. For now, I'm submitting a snapshot of work in progress.

Some performance numbers from mempool_perf_autotest_2cores, all with cache=1024 cores=2 n_keep=32768:

start performance test (using ring_mp_mc, with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 753985338 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 755805913

start performance test for lf_stack (with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 29132352 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 29276708

start performance test for pile (with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 560159479 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 557910933

Hat tip to Bruce for bringing attention to the ring not being the optimal mempool driver.

---
v2:
* Fix indentation, long lines, etc. (checkpatch)
* Fix label followed by a declaration is a C23 extension. (CI)
* Added __rte_internal to pile init and get_memsize. (AI)
* Fix roll back bulk elements in wrong order with fragmentation. (AI)
* Minor changes suggested by AI.
v3:
* Revert rename unused field in rte_stack_pile_bulk_elem structure. v4:
* Revert add __rte_internal.
Compilation fails, and existing stack implementations don't have it.
v5:
* Remove compiler diagnostic pragmas in stack overflow test case. (Stephen)
* Temporarily remove stack overflow test case until a sufficiently obfuscated variant doesn't trigger a compiler warning about array overrun.
* Revert most changes in rte_stack_std.h, and only change what is necessary.
* Add FIXME about support for the "generic" memory model should be removed in the pile implementation too, if removed in the lock-free stack. (Inspired by Stephen)

---
 app/test/test_mempool.c                   |   3 +-
 app/test/test_stack.c                     |  64 ++++-
 app/test/test_stack_perf.c                |  15 +-
 config/rte_config.h                       |   5 +-
 doc/guides/prog_guide/stack_lib.rst       |  67 ++++-
 drivers/mempool/stack/rte_mempool_stack.c |  40 +++
 drivers/net/bonding/rte_eth_bond_pmd.c    |   2 +-
 drivers/net/intel/cpfl/cpfl_rxtx.h        |   2 +-
 drivers/net/sxe2/sxe2_txrx_vec_avx512.c   |   3 +-
 drivers/net/tap/rte_eth_tap.c             |   2 +-
 lib/eal/include/rte_common.h              |  12 +
 lib/eal/x86/include/rte_memcpy.h          |  29 ++
 lib/mempool/mempool_trace.h               |   1 -
 lib/mempool/rte_mempool.c                 |  52 ++--
 lib/mempool/rte_mempool.h                 |  74 +++--
 lib/stack/meson.build                     |   3 +-
 lib/stack/rte_stack.c                     |  18 +-
 lib/stack/rte_stack.h                     |  79 +++++-
 lib/stack/rte_stack_lf.h                  |   6 +-
 lib/stack/rte_stack_lf_c11.h              |   2 +-
 lib/stack/rte_stack_lf_generic.h          |   2 +-
 lib/stack/rte_stack_lf_stubs.h            |   2 +-
 lib/stack/rte_stack_pile.c                |  34 +++
 lib/stack/rte_stack_pile.h                | 321 ++++++++++++++++++++++
 lib/stack/rte_stack_std.h                 |  24 +-
 25 files changed, 755 insertions(+), 107 deletions(-)
 create mode 100644 lib/stack/rte_stack_pile.c
 create mode 100644 lib/stack/rte_stack_pile.h
app/test/test_mempool.c +1 -2
index e54249ce61..76d45cea2a 100644
@@ -112,8 +112,7 @@ test_mempool_basic(struct rte_mempool *mp, int use_external_cache)
 		GOTO_ERR(ret, out);
 
 	printf("get private data\n");
-	if (rte_mempool_get_priv(mp) != (char *)mp +
-			RTE_MEMPOOL_HEADER_SIZE(mp, mp->cache_size))
+	if (rte_mempool_get_priv(mp) != (char *)mp + sizeof(struct rte_mempool))
 		GOTO_ERR(ret, out);
 
 #ifndef RTE_EXEC_ENV_FREEBSD /* rte_mem_virt2iova() not supported on bsd */
app/test/test_stack.c +52 -12

82 lines not on this page · raw message

app/test/test_stack_perf.c +13 -2
index 3f17a2606c..a15f3719c2 100644
@@ -14,14 +14,14 @@
 #include "test.h"
 
 #define STACK_NAME "STACK_PERF"
-#define MAX_BURST 32
+#define MAX_BURST (RTE_MEMPOOL_CACHE_MAX_SIZE / 2)
 #define STACK_SIZE (RTE_MAX_LCORE * MAX_BURST)
 
 /*
  * Push/pop bulk sizes, marked volatile so they aren't treated as compile-time
  * constants.
  */
-static volatile unsigned int bulk_sizes[] = {8, MAX_BURST};
+static volatile unsigned int bulk_sizes[] = {1, 8, 32, MAX_BURST};
 
 static RTE_ATOMIC(uint32_t) lcore_barrier;
 
@@ -354,5 +354,16 @@ test_lf_stack_perf(void)
 #endif
 }
 
+static int
+test_pile_perf(void)
+{
+#if defined(RTE_STACK_PILE_SUPPORTED)
+	return __test_stack_perf(RTE_STACK_F_PILE);
+#else
+	return TEST_SKIPPED;
+#endif
+}
+
 REGISTER_PERF_TEST(stack_perf_autotest, test_stack_perf);
 REGISTER_PERF_TEST(stack_lf_perf_autotest, test_lf_stack_perf);
+REGISTER_PERF_TEST(stack_pile_perf_autotest, test_pile_perf);
config/rte_config.h +4 -1
index 0447cdf2ad..03350660e4 100644
@@ -56,7 +56,7 @@
 #define RTE_CONTIGMEM_DEFAULT_BUF_SIZE (512*1024*1024)
 
 /* mempool defines */
-#define RTE_MEMPOOL_CACHE_MAX_SIZE 512
+#define RTE_MEMPOOL_CACHE_MAX_SIZE 1024
 /* RTE_LIBRTE_MEMPOOL_STATS is not set */
 /* RTE_LIBRTE_MEMPOOL_DEBUG is not set */
 
@@ -64,6 +64,9 @@
 #define RTE_MBUF_DEFAULT_MEMPOOL_OPS "ring_mp_mc"
 /* RTE_MBUF_HISTORY_DEBUG is not set */
 
+/* stack defines */
+#define RTE_STACK_PILE_BULK_SIZE 32
+
 /* ether defines */
 #define RTE_MAX_QUEUES_PER_PORT 1024
 #define RTE_ETHDEV_RXTX_CALLBACKS 1
doc/guides/prog_guide/stack_lib.rst +63 -4

95 lines not on this page · raw message

drivers/mempool/stack/rte_mempool_stack.c +40 -0
index 1476905227..7467b8b39e 100644
@@ -41,6 +41,36 @@ lf_stack_alloc(struct rte_mempool *mp)
 	return __stack_alloc(mp, RTE_STACK_F_LF);
 }
 
+static int
+pile_alloc(struct rte_mempool *mp)
+{
+	return __stack_alloc(mp, RTE_STACK_F_PILE);
+}
+
+static int
+pile_enqueue(struct rte_mempool *mp, void * const *obj_table,
+	      unsigned int n)
+{
+	struct rte_stack *s = mp->pool_data;
+
+	RTE_ASSERT(s != NULL);
+	RTE_ASSERT(obj_table != NULL);
+
+	return __rte_stack_pile_push(s, obj_table, n) == 0 ? -ENOBUFS : 0;
+}
+
+static int
+pile_dequeue(struct rte_mempool *mp, void **obj_table,
+	      unsigned int n)
+{
+	struct rte_stack *s = mp->pool_data;
+
+	RTE_ASSERT(s != NULL);
+	RTE_ASSERT(obj_table != NULL);
+
+	return __rte_stack_pile_pop(s, obj_table, n) == 0 ? -ENOBUFS : 0;
+}
+
 static int
 stack_enqueue(struct rte_mempool *mp, void * const *obj_table,
 	      unsigned int n)
@@ -93,5 +123,15 @@ static struct rte_mempool_ops ops_lf_stack = {
 	.get_count = stack_get_count
 };
 
+static struct rte_mempool_ops ops_pile = {
+	.name = "pile",
+	.alloc = pile_alloc,
+	.free = stack_free,
+	.enqueue = pile_enqueue,
+	.dequeue = pile_dequeue,
+	.get_count = stack_get_count
+};
+
 RTE_MEMPOOL_REGISTER_OPS(ops_stack);
 RTE_MEMPOOL_REGISTER_OPS(ops_lf_stack);
+RTE_MEMPOOL_REGISTER_OPS(ops_pile);
drivers/net/bonding/rte_eth_bond_pmd.c +1 -1
index 6a4f997b5a..92d7f4f4ef 100644
@@ -1702,7 +1702,7 @@ member_configure_slow_queue(struct rte_eth_dev *bonding_eth_dev,
 		snprintf(mem_name, RTE_DIM(mem_name), "member_port%u_slow_pool",
 				member_id);
 		port->slow_pool = rte_pktmbuf_pool_create(mem_name, 8191,
-			250, 0, RTE_MBUF_DEFAULT_BUF_SIZE,
+			256, 0, RTE_MBUF_DEFAULT_BUF_SIZE,
 			member_eth_dev->data->numa_node);
 
 		/* Any memory allocation failure in initialization is critical because
drivers/net/intel/cpfl/cpfl_rxtx.h +1 -1
index 52cdecac88..faf28fd489 100644
@@ -25,7 +25,7 @@
 #define CPFL_P2P_QUEUE_GRP_ID	1
 #define CPFL_P2P_DESC_LEN	16
 #define CPFL_P2P_NB_MBUF	4096
-#define CPFL_P2P_CACHE_SIZE	250
+#define CPFL_P2P_CACHE_SIZE	256
 #define CPFL_P2P_MBUF_SIZE	2048
 #define CPFL_P2P_RING_BUF	128
 
drivers/net/sxe2/sxe2_txrx_vec_avx512.c +2 -1
index a830c7a33b..4ded5cb63e 100644
@@ -67,11 +67,12 @@ static __rte_always_inline int32_t sxe2_tx_bufs_free_vec_avx512(struct sxe2_tx_q
 		}
 		cache->len += rs_thresh;
 
-		if (cache->len >= cache->flushthresh) {
+		if (cache->len >= cache->size) {
 			(void)rte_mempool_ops_enqueue_bulk(mp,
 					&cache->objs[cache->size], cache->len - cache->size);
 			cache->len = cache->size;
 		}
+
 		goto done;
 	}
 
drivers/net/tap/rte_eth_tap.c +1 -1
index b93452f168..b3142561c2 100644
@@ -61,7 +61,7 @@
 #define TAP_MAX_MAC_ADDRS	16
 #define TAP_GSO_MBUFS_PER_CORE	128
 #define TAP_GSO_MBUF_SEG_SIZE	128
-#define TAP_GSO_MBUF_CACHE_SIZE	4
+#define TAP_GSO_MBUF_CACHE_SIZE	32
 #define TAP_GSO_MBUFS_NUM \
 	(TAP_GSO_MBUFS_PER_CORE * TAP_GSO_MBUF_CACHE_SIZE)
 
lib/eal/include/rte_common.h +12 -0
index 79d2a0ab93..0fd0906506 100644
@@ -567,6 +567,15 @@ static void __attribute__((destructor(RTE_PRIO(prio)), used)) func(void)
 #define __rte_assume(condition) __assume(condition)
 #endif
 
+/**
+ * Alignment hint precondition
+ */
+#ifdef RTE_TOOLCHAIN_MSVC
+#define __rte_assume_aligned(ptr, alignment) (ptr)
+#else
+#define __rte_assume_aligned(ptr, alignment) __builtin_assume_aligned(ptr, alignment)
+#endif
+
 /**
  * Disable AddressSanitizer on some code
  */
@@ -775,6 +784,9 @@ rte_is_aligned(const void * const __rte_restrict ptr, const unsigned int align)
 /** Force minimum cache line alignment. */
 #define __rte_cache_min_aligned __rte_aligned(RTE_CACHE_LINE_MIN_SIZE)
 
+/** Cache alignment hint precondition */
+#define __rte_assume_cache_aligned(ptr) __rte_assume_aligned(ptr, RTE_CACHE_LINE_SIZE)
+
 #define _RTE_CACHE_GUARD_HELPER2(unique) \
 	alignas(RTE_CACHE_LINE_SIZE) \
 	char cache_guard_ ## unique[RTE_CACHE_LINE_SIZE * RTE_CACHE_GUARD_LINES]
lib/eal/x86/include/rte_memcpy.h +29 -0
index 8ed8c55010..3fe1e8d247 100644
@@ -707,6 +707,35 @@ rte_memcpy(void *__rte_restrict dst, const void *__rte_restrict src, size_t n)
 #endif
 		return dst;
 	}
+	/* Common way for small copy size of 64-byte blocks */
+#if defined __AVX512F__ && defined RTE_MEMCPY_AVX512
+	if (__rte_constant(n) && (n & 63) == 0 && n <= 512) {
+#elif defined RTE_MEMCPY_AVX
+	if (__rte_constant(n) && (n & 63) == 0 && n <= 256) {
+#else /* SSE implementation */
+	if (__rte_constant(n) && (n & 63) == 0 && n <= 512) {
+#endif
+		void *ret = dst;
+
+		if (n & 512) {
+			rte_mov256((uint8_t *)dst + 0 * 256, (const uint8_t *)src + 0 * 256);
+			rte_mov256((uint8_t *)dst + 1 * 256, (const uint8_t *)src + 1 * 256);
+		}
+		if (n & 256) {
+			rte_mov256((uint8_t *)dst, (const uint8_t *)src);
+			src = (const uint8_t *)src + 256;
+			dst = (uint8_t *)dst + 256;
+		}
+		if (n & 128) {
+			rte_mov128((uint8_t *)dst, (const uint8_t *)src);
+			src = (const uint8_t *)src + 128;
+			dst = (uint8_t *)dst + 128;
+		}
+		if (n & 64)
+			rte_mov64((uint8_t *)dst, (const uint8_t *)src);
+
+		return ret;
+	}
 
 	/* Implementation for size > 64 bytes depends on alignment with vector register size. */
 	if (!(((uintptr_t)dst | (uintptr_t)src) & ALIGNMENT_MASK))
lib/mempool/mempool_trace.h +0 -1
index 23cda1473c..60e47cf67b 100644
@@ -119,7 +119,6 @@ RTE_TRACE_POINT(
 	rte_trace_point_emit_i32(socket_id);
 	rte_trace_point_emit_ptr(cache);
 	rte_trace_point_emit_u32(cache->len);
-	rte_trace_point_emit_u32(cache->flushthresh);
 )
 
 RTE_TRACE_POINT(
lib/mempool/rte_mempool.c +30 -22

118 lines not on this page · raw message

lib/mempool/rte_mempool.h +35 -39

155 lines not on this page · raw message

lib/stack/meson.build +2 -1
index 18177a742f..50e688522e 100644
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2019 Intel Corporation
 
-sources = files('rte_stack.c', 'rte_stack_std.c', 'rte_stack_lf.c')
+sources = files('rte_stack.c', 'rte_stack_std.c', 'rte_stack_lf.c', 'rte_stack_pile.c')
 headers = files('rte_stack.h')
 # subheaders, not for direct inclusion by apps
 indirect_headers += files(
@@ -10,4 +10,5 @@ indirect_headers += files(
         'rte_stack_lf_generic.h',
         'rte_stack_lf_c11.h',
         'rte_stack_lf_stubs.h',
+        'rte_stack_pile.h',
 )
lib/stack/rte_stack.c +17 -1
index 4c78fe4b4b..a4bbf8a4d7 100644
@@ -1,5 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright(c) 2019 Intel Corporation
+ * Copyright(c) 2026 SmartShare Systems
  */
 
 #include <stdalign.h>
@@ -32,6 +33,8 @@ rte_stack_init(struct rte_stack *s, unsigned int count, uint32_t flags)
 
 	if (flags & RTE_STACK_F_LF)
 		rte_stack_lf_init(s, count);
+	else if (flags & RTE_STACK_F_PILE)
+		rte_stack_pile_init(s, count);
 	else
 		rte_stack_std_init(s);
 }
@@ -41,6 +44,8 @@ rte_stack_get_memsize(unsigned int count, uint32_t flags)
 {
 	if (flags & RTE_STACK_F_LF)
 		return rte_stack_lf_get_memsize(count);
+	else if (flags & RTE_STACK_F_PILE)
+		return rte_stack_pile_get_memsize(count);
 	else
 		return rte_stack_std_get_memsize(count);
 }
@@ -58,7 +63,11 @@ rte_stack_create(const char *name, unsigned int count, int socket_id,
 	unsigned int sz;
 	int ret;
 
-	if (flags & ~(RTE_STACK_F_LF)) {
+	if (flags & ~(RTE_STACK_F_LF | RTE_STACK_F_PILE)) {
+		STACK_LOG_ERR("Unsupported stack flags %#x", flags);
+		return NULL;
+	}
+	if ((flags & RTE_STACK_F_LF) && (flags & RTE_STACK_F_PILE)) {
 		STACK_LOG_ERR("Unsupported stack flags %#x", flags);
 		return NULL;
 	}
@@ -73,6 +82,13 @@ rte_stack_create(const char *name, unsigned int count, int socket_id,
 		return NULL;
 	}
 #endif
+#if !defined(RTE_STACK_PILE_SUPPORTED)
+	if (flags & RTE_STACK_F_PILE) {
+		STACK_LOG_ERR("Pile is not supported on your platform");
+		rte_errno = ENOTSUP;
+		return NULL;
+	}
+#endif
 
 	sz = rte_stack_get_memsize(count, flags);
 
lib/stack/rte_stack.h +77 -2

151 lines not on this page · raw message

lib/stack/rte_stack_lf.h +4 -2
index f2b012cd0e..655620aaa9 100644
@@ -34,7 +34,7 @@
  */
 static __rte_always_inline unsigned int
 __rte_stack_lf_push(struct rte_stack *s,
-		    void * const *obj_table,
+		    void * const * __rte_restrict obj_table,
 		    unsigned int n)
 {
 	struct rte_stack_lf_elem *tmp, *first, *last = NULL;
@@ -71,7 +71,8 @@ __rte_stack_lf_push(struct rte_stack *s,
  *   - Actual number of objects popped.
  */
 static __rte_always_inline unsigned int
-__rte_stack_lf_pop(struct rte_stack *s, void **obj_table, unsigned int n)
+__rte_stack_lf_pop(struct rte_stack *s, void ** __rte_restrict obj_table,
+		   unsigned int n)
 {
 	struct rte_stack_lf_elem *first, *last = NULL;
 
@@ -79,6 +80,7 @@ __rte_stack_lf_pop(struct rte_stack *s, void **obj_table, unsigned int n)
 		return 0;
 
 	/* Pop n used elements */
+	__rte_assume(obj_table != NULL);
 	first = __rte_stack_lf_pop_elems(&s->stack_lf.used,
 					 n, obj_table, &last);
 	if (unlikely(first == NULL))
lib/stack/rte_stack_lf_c11.h +1 -1
index b97e02d6a1..501e985a49 100644
@@ -99,7 +99,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 static __rte_always_inline struct rte_stack_lf_elem *
 __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 unsigned int num,
-			 void **obj_table,
+			 void ** __rte_restrict obj_table,
 			 struct rte_stack_lf_elem **last)
 {
 	struct rte_stack_lf_head old_head;
lib/stack/rte_stack_lf_generic.h +1 -1
index cc69e4d168..c4cc4d2c03 100644
@@ -74,7 +74,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 static __rte_always_inline struct rte_stack_lf_elem *
 __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 unsigned int num,
-			 void **obj_table,
+			 void ** __rte_restrict obj_table,
 			 struct rte_stack_lf_elem **last)
 {
 	struct rte_stack_lf_head old_head;
lib/stack/rte_stack_lf_stubs.h +1 -1
index a05abf1f1c..457a415ccf 100644
@@ -30,7 +30,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 static __rte_always_inline struct rte_stack_lf_elem *
 __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 unsigned int num,
-			 void **obj_table,
+			 void ** __rte_restrict obj_table,
 			 struct rte_stack_lf_elem **last)
 {
 	RTE_SET_USED(obj_table);
lib/stack/rte_stack_pile.c new file +34 -0
new file mode 100644
index 0000000000..920bccd5ab
@@ -0,0 +1,34 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2026 SmartShare Systems
+ */
+
+#include "rte_stack.h"
+
+void
+rte_stack_pile_init(struct rte_stack *s, unsigned int count)
+{
+	unsigned int bulk = (count + RTE_STACK_PILE_BULK_SIZE - 1) / RTE_STACK_PILE_BULK_SIZE;
+	struct rte_stack_pile_bulk_elem *bulk_elems =
+			(struct rte_stack_pile_bulk_elem *)(&s->stack_pile + 1);
+	struct rte_stack_lf_elem *solo_elems = (struct rte_stack_lf_elem *)&bulk_elems[bulk];
+	unsigned int i;
+
+	for (i = 0; i < bulk; i++)
+		__rte_stack_pile_bulk_push_elems(&s->stack_pile.free_bulk,
+					  &bulk_elems[i], &bulk_elems[i], 1);
+	for (i = 0; i < count; i++)
+		__rte_stack_lf_push_elems(&s->stack_pile.free_solo,
+					  &solo_elems[i], &solo_elems[i], 1);
+}
+
+ssize_t
+rte_stack_pile_get_memsize(unsigned int count)
+{
+	unsigned int bulk = (count + RTE_STACK_PILE_BULK_SIZE - 1) / RTE_STACK_PILE_BULK_SIZE;
+	ssize_t sz = sizeof(struct rte_stack); /* Already cache line aligned. */
+	sz += bulk * sizeof(struct rte_stack_pile_bulk_elem); /* Already cache line aligned. */
+	sz += RTE_CACHE_LINE_ROUNDUP(count * sizeof(struct rte_stack_lf_elem));
+	sz += RTE_CACHE_GUARD_LINES * RTE_CACHE_LINE_SIZE;
+
+	return sz;
+}
lib/stack/rte_stack_pile.h new file +321 -0

321 lines not on this page · raw message

lib/stack/rte_stack_std.h +13 -11
index ae28add5c4..e256cf8d83 100644
@@ -20,25 +20,25 @@
  *   Actual number of objects pushed (either 0 or *n*).
  */
 static __rte_always_inline unsigned int
-__rte_stack_std_push(struct rte_stack *s, void * const *obj_table,
+__rte_stack_std_push(struct rte_stack *s, void * const * __rte_restrict obj_table,
 		     unsigned int n)
 {
 	struct rte_stack_std *stack = &s->stack_std;
 	unsigned int index;
-	void **cache_objs;
+	void ** __rte_restrict stack_objs;
 
 	rte_spinlock_lock(&stack->lock);
-	cache_objs = &stack->objs[stack->len];
+	stack_objs = &stack->objs[stack->len];
 
-	/* Is there sufficient space in the stack? */
-	if ((stack->len + n) > s->capacity) {
+	if (unlikely((stack->len + n) > s->capacity)) {
+		/* Insufficient space in the stack. */
 		rte_spinlock_unlock(&stack->lock);
 		return 0;
 	}
 
-	/* Add elements back into the cache */
+	/* Push objects to the stack */
 	for (index = 0; index < n; ++index, obj_table++)
-		cache_objs[index] = *obj_table;
+		stack_objs[index] = *obj_table;
 
 	stack->len += n;
 
@@ -59,24 +59,26 @@ __rte_stack_std_push(struct rte_stack *s, void * const *obj_table,
  *   Actual number of objects popped (either 0 or *n*).
  */
 static __rte_always_inline unsigned int
-__rte_stack_std_pop(struct rte_stack *s, void **obj_table, unsigned int n)
+__rte_stack_std_pop(struct rte_stack *s, void ** __rte_restrict obj_table, unsigned int n)
 {
 	struct rte_stack_std *stack = &s->stack_std;
 	unsigned int index, len;
-	void **cache_objs;
+	void ** __rte_restrict stack_objs;
 
 	rte_spinlock_lock(&stack->lock);
 
 	if (unlikely(n > stack->len)) {
+		/* Insufficient objects in the stack. */
 		rte_spinlock_unlock(&stack->lock);
 		return 0;
 	}
 
-	cache_objs = stack->objs;
+	stack_objs = stack->objs;
 
+	/* Pop objects from the stack */
 	for (index = 0, len = stack->len - 1; index < n;
 			++index, len--, obj_table++)
-		*obj_table = cache_objs[len];
+		*obj_table = stack_objs[len];
 
 	stack->len -= n;
 	rte_spinlock_unlock(&stack->lock);
Signature · 1 line
2.43.0
#5 lore raw

RFC PATCH v6 1/1 pile stack and mempool driver

In reply to Morten Brørup

Early submission of:
- some mempool optimizations,
- a new mempool "pile" driver, and
- its underlying "pile" stack implementation.

For community feedback and CI test.

Needless to say, this must be separated into a series of patches. For now, I'm submitting a snapshot of work in progress.

Some performance numbers from mempool_perf_autotest_2cores, all with cache=1024 cores=2 n_keep=32768:

start performance test (using ring_mp_mc, with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 753985338 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 755805913

start performance test for lf_stack (with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 29132352 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 29276708

start performance test for pile (with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 560159479 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 557910933

Hat tip to Bruce for bringing attention to the ring not being the optimal mempool driver.

---
v2:
* Fix indentation, long lines, etc. (checkpatch)
* Fix label followed by a declaration is a C23 extension. (CI)
* Added __rte_internal to pile init and get_memsize. (AI)
* Fix roll back bulk elements in wrong order with fragmentation. (AI)
* Minor changes suggested by AI.
v3:
* Revert rename unused field in rte_stack_pile_bulk_elem structure. v4:
* Revert add __rte_internal.
Compilation fails, and existing stack implementations don't have it.
v5:
* Remove compiler diagnostic pragmas in stack overflow test case. (Stephen)
* Temporarily remove stack overflow test case until a sufficiently obfuscated variant doesn't trigger a compiler warning about array overrun.
* Revert most changes in rte_stack_std.h, and only change what is necessary.
* Add FIXME in the pile implementation, noting that support for the generic memory model should be removed here too, if removed in the lock-free stack.
(Inspired by Stephen)
v6:
* Add note about roll back of objects in fragmentation element. (AI)
* Move declaration of temporary variable up, to please compilers. (CI)
* Reverted mempool cache size adjustments in some drivers; let the mempool creation function adjust at runtime instead.

---
 app/test/test_mempool.c                   |   3 +-
 app/test/test_stack.c                     |  64 ++++-
 app/test/test_stack_perf.c                |  15 +-
 config/rte_config.h                       |   5 +-
 doc/guides/prog_guide/stack_lib.rst       |  67 ++++-
 drivers/mempool/stack/rte_mempool_stack.c |  40 +++
 drivers/net/sxe2/sxe2_txrx_vec_avx512.c   |   3 +-
 drivers/net/tap/rte_eth_tap.c             |   2 +-
 lib/eal/include/rte_common.h              |  12 +
 lib/eal/x86/include/rte_memcpy.h          |  29 ++
 lib/mempool/mempool_trace.h               |   1 -
 lib/mempool/rte_mempool.c                 |  58 ++--
 lib/mempool/rte_mempool.h                 |  76 +++--
 lib/stack/meson.build                     |   3 +-
 lib/stack/rte_stack.c                     |  18 +-
 lib/stack/rte_stack.h                     |  82 +++++-
 lib/stack/rte_stack_lf.h                  |   6 +-
 lib/stack/rte_stack_lf_c11.h              |   2 +-
 lib/stack/rte_stack_lf_generic.h          |   2 +-
 lib/stack/rte_stack_lf_stubs.h            |   2 +-
 lib/stack/rte_stack_pile.c                |  34 +++
 lib/stack/rte_stack_pile.h                | 326 ++++++++++++++++++++++
 lib/stack/rte_stack_std.h                 |  24 +-
 23 files changed, 769 insertions(+), 105 deletions(-)
 create mode 100644 lib/stack/rte_stack_pile.c
 create mode 100644 lib/stack/rte_stack_pile.h
app/test/test_mempool.c +1 -2
index e54249ce61..76d45cea2a 100644
@@ -112,8 +112,7 @@ test_mempool_basic(struct rte_mempool *mp, int use_external_cache)
 		GOTO_ERR(ret, out);
 
 	printf("get private data\n");
-	if (rte_mempool_get_priv(mp) != (char *)mp +
-			RTE_MEMPOOL_HEADER_SIZE(mp, mp->cache_size))
+	if (rte_mempool_get_priv(mp) != (char *)mp + sizeof(struct rte_mempool))
 		GOTO_ERR(ret, out);
 
 #ifndef RTE_EXEC_ENV_FREEBSD /* rte_mem_virt2iova() not supported on bsd */
app/test/test_stack.c +52 -12

82 lines not on this page · raw message

app/test/test_stack_perf.c +13 -2
index 3f17a2606c..a15f3719c2 100644
@@ -14,14 +14,14 @@
 #include "test.h"
 
 #define STACK_NAME "STACK_PERF"
-#define MAX_BURST 32
+#define MAX_BURST (RTE_MEMPOOL_CACHE_MAX_SIZE / 2)
 #define STACK_SIZE (RTE_MAX_LCORE * MAX_BURST)
 
 /*
  * Push/pop bulk sizes, marked volatile so they aren't treated as compile-time
  * constants.
  */
-static volatile unsigned int bulk_sizes[] = {8, MAX_BURST};
+static volatile unsigned int bulk_sizes[] = {1, 8, 32, MAX_BURST};
 
 static RTE_ATOMIC(uint32_t) lcore_barrier;
 
@@ -354,5 +354,16 @@ test_lf_stack_perf(void)
 #endif
 }
 
+static int
+test_pile_perf(void)
+{
+#if defined(RTE_STACK_PILE_SUPPORTED)
+	return __test_stack_perf(RTE_STACK_F_PILE);
+#else
+	return TEST_SKIPPED;
+#endif
+}
+
 REGISTER_PERF_TEST(stack_perf_autotest, test_stack_perf);
 REGISTER_PERF_TEST(stack_lf_perf_autotest, test_lf_stack_perf);
+REGISTER_PERF_TEST(stack_pile_perf_autotest, test_pile_perf);
config/rte_config.h +4 -1
index 0447cdf2ad..03350660e4 100644
@@ -56,7 +56,7 @@
 #define RTE_CONTIGMEM_DEFAULT_BUF_SIZE (512*1024*1024)
 
 /* mempool defines */
-#define RTE_MEMPOOL_CACHE_MAX_SIZE 512
+#define RTE_MEMPOOL_CACHE_MAX_SIZE 1024
 /* RTE_LIBRTE_MEMPOOL_STATS is not set */
 /* RTE_LIBRTE_MEMPOOL_DEBUG is not set */
 
@@ -64,6 +64,9 @@
 #define RTE_MBUF_DEFAULT_MEMPOOL_OPS "ring_mp_mc"
 /* RTE_MBUF_HISTORY_DEBUG is not set */
 
+/* stack defines */
+#define RTE_STACK_PILE_BULK_SIZE 32
+
 /* ether defines */
 #define RTE_MAX_QUEUES_PER_PORT 1024
 #define RTE_ETHDEV_RXTX_CALLBACKS 1
doc/guides/prog_guide/stack_lib.rst +63 -4

95 lines not on this page · raw message

drivers/mempool/stack/rte_mempool_stack.c +40 -0
index 1476905227..7467b8b39e 100644
@@ -41,6 +41,36 @@ lf_stack_alloc(struct rte_mempool *mp)
 	return __stack_alloc(mp, RTE_STACK_F_LF);
 }
 
+static int
+pile_alloc(struct rte_mempool *mp)
+{
+	return __stack_alloc(mp, RTE_STACK_F_PILE);
+}
+
+static int
+pile_enqueue(struct rte_mempool *mp, void * const *obj_table,
+	      unsigned int n)
+{
+	struct rte_stack *s = mp->pool_data;
+
+	RTE_ASSERT(s != NULL);
+	RTE_ASSERT(obj_table != NULL);
+
+	return __rte_stack_pile_push(s, obj_table, n) == 0 ? -ENOBUFS : 0;
+}
+
+static int
+pile_dequeue(struct rte_mempool *mp, void **obj_table,
+	      unsigned int n)
+{
+	struct rte_stack *s = mp->pool_data;
+
+	RTE_ASSERT(s != NULL);
+	RTE_ASSERT(obj_table != NULL);
+
+	return __rte_stack_pile_pop(s, obj_table, n) == 0 ? -ENOBUFS : 0;
+}
+
 static int
 stack_enqueue(struct rte_mempool *mp, void * const *obj_table,
 	      unsigned int n)
@@ -93,5 +123,15 @@ static struct rte_mempool_ops ops_lf_stack = {
 	.get_count = stack_get_count
 };
 
+static struct rte_mempool_ops ops_pile = {
+	.name = "pile",
+	.alloc = pile_alloc,
+	.free = stack_free,
+	.enqueue = pile_enqueue,
+	.dequeue = pile_dequeue,
+	.get_count = stack_get_count
+};
+
 RTE_MEMPOOL_REGISTER_OPS(ops_stack);
 RTE_MEMPOOL_REGISTER_OPS(ops_lf_stack);
+RTE_MEMPOOL_REGISTER_OPS(ops_pile);
drivers/net/sxe2/sxe2_txrx_vec_avx512.c +2 -1
index a830c7a33b..4ded5cb63e 100644
@@ -67,11 +67,12 @@ static __rte_always_inline int32_t sxe2_tx_bufs_free_vec_avx512(struct sxe2_tx_q
 		}
 		cache->len += rs_thresh;
 
-		if (cache->len >= cache->flushthresh) {
+		if (cache->len >= cache->size) {
 			(void)rte_mempool_ops_enqueue_bulk(mp,
 					&cache->objs[cache->size], cache->len - cache->size);
 			cache->len = cache->size;
 		}
+
 		goto done;
 	}
 
drivers/net/tap/rte_eth_tap.c +1 -1
index b93452f168..b3142561c2 100644
@@ -61,7 +61,7 @@
 #define TAP_MAX_MAC_ADDRS	16
 #define TAP_GSO_MBUFS_PER_CORE	128
 #define TAP_GSO_MBUF_SEG_SIZE	128
-#define TAP_GSO_MBUF_CACHE_SIZE	4
+#define TAP_GSO_MBUF_CACHE_SIZE	32
 #define TAP_GSO_MBUFS_NUM \
 	(TAP_GSO_MBUFS_PER_CORE * TAP_GSO_MBUF_CACHE_SIZE)
 
lib/eal/include/rte_common.h +12 -0
index 79d2a0ab93..0fd0906506 100644
@@ -567,6 +567,15 @@ static void __attribute__((destructor(RTE_PRIO(prio)), used)) func(void)
 #define __rte_assume(condition) __assume(condition)
 #endif
 
+/**
+ * Alignment hint precondition
+ */
+#ifdef RTE_TOOLCHAIN_MSVC
+#define __rte_assume_aligned(ptr, alignment) (ptr)
+#else
+#define __rte_assume_aligned(ptr, alignment) __builtin_assume_aligned(ptr, alignment)
+#endif
+
 /**
  * Disable AddressSanitizer on some code
  */
@@ -775,6 +784,9 @@ rte_is_aligned(const void * const __rte_restrict ptr, const unsigned int align)
 /** Force minimum cache line alignment. */
 #define __rte_cache_min_aligned __rte_aligned(RTE_CACHE_LINE_MIN_SIZE)
 
+/** Cache alignment hint precondition */
+#define __rte_assume_cache_aligned(ptr) __rte_assume_aligned(ptr, RTE_CACHE_LINE_SIZE)
+
 #define _RTE_CACHE_GUARD_HELPER2(unique) \
 	alignas(RTE_CACHE_LINE_SIZE) \
 	char cache_guard_ ## unique[RTE_CACHE_LINE_SIZE * RTE_CACHE_GUARD_LINES]
lib/eal/x86/include/rte_memcpy.h +29 -0
index 8ed8c55010..3fe1e8d247 100644
@@ -707,6 +707,35 @@ rte_memcpy(void *__rte_restrict dst, const void *__rte_restrict src, size_t n)
 #endif
 		return dst;
 	}
+	/* Common way for small copy size of 64-byte blocks */
+#if defined __AVX512F__ && defined RTE_MEMCPY_AVX512
+	if (__rte_constant(n) && (n & 63) == 0 && n <= 512) {
+#elif defined RTE_MEMCPY_AVX
+	if (__rte_constant(n) && (n & 63) == 0 && n <= 256) {
+#else /* SSE implementation */
+	if (__rte_constant(n) && (n & 63) == 0 && n <= 512) {
+#endif
+		void *ret = dst;
+
+		if (n & 512) {
+			rte_mov256((uint8_t *)dst + 0 * 256, (const uint8_t *)src + 0 * 256);
+			rte_mov256((uint8_t *)dst + 1 * 256, (const uint8_t *)src + 1 * 256);
+		}
+		if (n & 256) {
+			rte_mov256((uint8_t *)dst, (const uint8_t *)src);
+			src = (const uint8_t *)src + 256;
+			dst = (uint8_t *)dst + 256;
+		}
+		if (n & 128) {
+			rte_mov128((uint8_t *)dst, (const uint8_t *)src);
+			src = (const uint8_t *)src + 128;
+			dst = (uint8_t *)dst + 128;
+		}
+		if (n & 64)
+			rte_mov64((uint8_t *)dst, (const uint8_t *)src);
+
+		return ret;
+	}
 
 	/* Implementation for size > 64 bytes depends on alignment with vector register size. */
 	if (!(((uintptr_t)dst | (uintptr_t)src) & ALIGNMENT_MASK))
lib/mempool/mempool_trace.h +0 -1
index 23cda1473c..60e47cf67b 100644
@@ -119,7 +119,6 @@ RTE_TRACE_POINT(
 	rte_trace_point_emit_i32(socket_id);
 	rte_trace_point_emit_ptr(cache);
 	rte_trace_point_emit_u32(cache->len);
-	rte_trace_point_emit_u32(cache->flushthresh);
 )
 
 RTE_TRACE_POINT(
lib/mempool/rte_mempool.c +36 -22

124 lines not on this page · raw message

lib/mempool/rte_mempool.h +37 -39

157 lines not on this page · raw message

lib/stack/meson.build +2 -1
index 18177a742f..50e688522e 100644
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2019 Intel Corporation
 
-sources = files('rte_stack.c', 'rte_stack_std.c', 'rte_stack_lf.c')
+sources = files('rte_stack.c', 'rte_stack_std.c', 'rte_stack_lf.c', 'rte_stack_pile.c')
 headers = files('rte_stack.h')
 # subheaders, not for direct inclusion by apps
 indirect_headers += files(
@@ -10,4 +10,5 @@ indirect_headers += files(
         'rte_stack_lf_generic.h',
         'rte_stack_lf_c11.h',
         'rte_stack_lf_stubs.h',
+        'rte_stack_pile.h',
 )
lib/stack/rte_stack.c +17 -1
index 4c78fe4b4b..a4bbf8a4d7 100644
@@ -1,5 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright(c) 2019 Intel Corporation
+ * Copyright(c) 2026 SmartShare Systems
  */
 
 #include <stdalign.h>
@@ -32,6 +33,8 @@ rte_stack_init(struct rte_stack *s, unsigned int count, uint32_t flags)
 
 	if (flags & RTE_STACK_F_LF)
 		rte_stack_lf_init(s, count);
+	else if (flags & RTE_STACK_F_PILE)
+		rte_stack_pile_init(s, count);
 	else
 		rte_stack_std_init(s);
 }
@@ -41,6 +44,8 @@ rte_stack_get_memsize(unsigned int count, uint32_t flags)
 {
 	if (flags & RTE_STACK_F_LF)
 		return rte_stack_lf_get_memsize(count);
+	else if (flags & RTE_STACK_F_PILE)
+		return rte_stack_pile_get_memsize(count);
 	else
 		return rte_stack_std_get_memsize(count);
 }
@@ -58,7 +63,11 @@ rte_stack_create(const char *name, unsigned int count, int socket_id,
 	unsigned int sz;
 	int ret;
 
-	if (flags & ~(RTE_STACK_F_LF)) {
+	if (flags & ~(RTE_STACK_F_LF | RTE_STACK_F_PILE)) {
+		STACK_LOG_ERR("Unsupported stack flags %#x", flags);
+		return NULL;
+	}
+	if ((flags & RTE_STACK_F_LF) && (flags & RTE_STACK_F_PILE)) {
 		STACK_LOG_ERR("Unsupported stack flags %#x", flags);
 		return NULL;
 	}
@@ -73,6 +82,13 @@ rte_stack_create(const char *name, unsigned int count, int socket_id,
 		return NULL;
 	}
 #endif
+#if !defined(RTE_STACK_PILE_SUPPORTED)
+	if (flags & RTE_STACK_F_PILE) {
+		STACK_LOG_ERR("Pile is not supported on your platform");
+		rte_errno = ENOTSUP;
+		return NULL;
+	}
+#endif
 
 	sz = rte_stack_get_memsize(count, flags);
 
lib/stack/rte_stack.h +80 -2

154 lines not on this page · raw message

lib/stack/rte_stack_lf.h +4 -2
index f2b012cd0e..655620aaa9 100644
@@ -34,7 +34,7 @@
  */
 static __rte_always_inline unsigned int
 __rte_stack_lf_push(struct rte_stack *s,
-		    void * const *obj_table,
+		    void * const * __rte_restrict obj_table,
 		    unsigned int n)
 {
 	struct rte_stack_lf_elem *tmp, *first, *last = NULL;
@@ -71,7 +71,8 @@ __rte_stack_lf_push(struct rte_stack *s,
  *   - Actual number of objects popped.
  */
 static __rte_always_inline unsigned int
-__rte_stack_lf_pop(struct rte_stack *s, void **obj_table, unsigned int n)
+__rte_stack_lf_pop(struct rte_stack *s, void ** __rte_restrict obj_table,
+		   unsigned int n)
 {
 	struct rte_stack_lf_elem *first, *last = NULL;
 
@@ -79,6 +80,7 @@ __rte_stack_lf_pop(struct rte_stack *s, void **obj_table, unsigned int n)
 		return 0;
 
 	/* Pop n used elements */
+	__rte_assume(obj_table != NULL);
 	first = __rte_stack_lf_pop_elems(&s->stack_lf.used,
 					 n, obj_table, &last);
 	if (unlikely(first == NULL))
lib/stack/rte_stack_lf_c11.h +1 -1
index b97e02d6a1..501e985a49 100644
@@ -99,7 +99,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 static __rte_always_inline struct rte_stack_lf_elem *
 __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 unsigned int num,
-			 void **obj_table,
+			 void ** __rte_restrict obj_table,
 			 struct rte_stack_lf_elem **last)
 {
 	struct rte_stack_lf_head old_head;
lib/stack/rte_stack_lf_generic.h +1 -1
index cc69e4d168..c4cc4d2c03 100644
@@ -74,7 +74,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 static __rte_always_inline struct rte_stack_lf_elem *
 __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 unsigned int num,
-			 void **obj_table,
+			 void ** __rte_restrict obj_table,
 			 struct rte_stack_lf_elem **last)
 {
 	struct rte_stack_lf_head old_head;
lib/stack/rte_stack_lf_stubs.h +1 -1
index a05abf1f1c..457a415ccf 100644
@@ -30,7 +30,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 static __rte_always_inline struct rte_stack_lf_elem *
 __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 unsigned int num,
-			 void **obj_table,
+			 void ** __rte_restrict obj_table,
 			 struct rte_stack_lf_elem **last)
 {
 	RTE_SET_USED(obj_table);
lib/stack/rte_stack_pile.c new file +34 -0
new file mode 100644
index 0000000000..920bccd5ab
@@ -0,0 +1,34 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2026 SmartShare Systems
+ */
+
+#include "rte_stack.h"
+
+void
+rte_stack_pile_init(struct rte_stack *s, unsigned int count)
+{
+	unsigned int bulk = (count + RTE_STACK_PILE_BULK_SIZE - 1) / RTE_STACK_PILE_BULK_SIZE;
+	struct rte_stack_pile_bulk_elem *bulk_elems =
+			(struct rte_stack_pile_bulk_elem *)(&s->stack_pile + 1);
+	struct rte_stack_lf_elem *solo_elems = (struct rte_stack_lf_elem *)&bulk_elems[bulk];
+	unsigned int i;
+
+	for (i = 0; i < bulk; i++)
+		__rte_stack_pile_bulk_push_elems(&s->stack_pile.free_bulk,
+					  &bulk_elems[i], &bulk_elems[i], 1);
+	for (i = 0; i < count; i++)
+		__rte_stack_lf_push_elems(&s->stack_pile.free_solo,
+					  &solo_elems[i], &solo_elems[i], 1);
+}
+
+ssize_t
+rte_stack_pile_get_memsize(unsigned int count)
+{
+	unsigned int bulk = (count + RTE_STACK_PILE_BULK_SIZE - 1) / RTE_STACK_PILE_BULK_SIZE;
+	ssize_t sz = sizeof(struct rte_stack); /* Already cache line aligned. */
+	sz += bulk * sizeof(struct rte_stack_pile_bulk_elem); /* Already cache line aligned. */
+	sz += RTE_CACHE_LINE_ROUNDUP(count * sizeof(struct rte_stack_lf_elem));
+	sz += RTE_CACHE_GUARD_LINES * RTE_CACHE_LINE_SIZE;
+
+	return sz;
+}
lib/stack/rte_stack_pile.h new file +326 -0

326 lines not on this page · raw message

lib/stack/rte_stack_std.h +13 -11
index ae28add5c4..e256cf8d83 100644
@@ -20,25 +20,25 @@
  *   Actual number of objects pushed (either 0 or *n*).
  */
 static __rte_always_inline unsigned int
-__rte_stack_std_push(struct rte_stack *s, void * const *obj_table,
+__rte_stack_std_push(struct rte_stack *s, void * const * __rte_restrict obj_table,
 		     unsigned int n)
 {
 	struct rte_stack_std *stack = &s->stack_std;
 	unsigned int index;
-	void **cache_objs;
+	void ** __rte_restrict stack_objs;
 
 	rte_spinlock_lock(&stack->lock);
-	cache_objs = &stack->objs[stack->len];
+	stack_objs = &stack->objs[stack->len];
 
-	/* Is there sufficient space in the stack? */
-	if ((stack->len + n) > s->capacity) {
+	if (unlikely((stack->len + n) > s->capacity)) {
+		/* Insufficient space in the stack. */
 		rte_spinlock_unlock(&stack->lock);
 		return 0;
 	}
 
-	/* Add elements back into the cache */
+	/* Push objects to the stack */
 	for (index = 0; index < n; ++index, obj_table++)
-		cache_objs[index] = *obj_table;
+		stack_objs[index] = *obj_table;
 
 	stack->len += n;
 
@@ -59,24 +59,26 @@ __rte_stack_std_push(struct rte_stack *s, void * const *obj_table,
  *   Actual number of objects popped (either 0 or *n*).
  */
 static __rte_always_inline unsigned int
-__rte_stack_std_pop(struct rte_stack *s, void **obj_table, unsigned int n)
+__rte_stack_std_pop(struct rte_stack *s, void ** __rte_restrict obj_table, unsigned int n)
 {
 	struct rte_stack_std *stack = &s->stack_std;
 	unsigned int index, len;
-	void **cache_objs;
+	void ** __rte_restrict stack_objs;
 
 	rte_spinlock_lock(&stack->lock);
 
 	if (unlikely(n > stack->len)) {
+		/* Insufficient objects in the stack. */
 		rte_spinlock_unlock(&stack->lock);
 		return 0;
 	}
 
-	cache_objs = stack->objs;
+	stack_objs = stack->objs;
 
+	/* Pop objects from the stack */
 	for (index = 0, len = stack->len - 1; index < n;
 			++index, len--, obj_table++)
-		*obj_table = cache_objs[len];
+		*obj_table = stack_objs[len];
 
 	stack->len -= n;
 	rte_spinlock_unlock(&stack->lock);
Signature · 1 line
2.43.0
#6 lore raw

Re: [RFC PATCH v6] pile stack and mempool driver

Quoted 25 lines · On Sun, 2 Aug 2026 09:59:54 +0000 Morten Brørup <[email protected]>…

Early submission of:
- some mempool optimizations,
- a new mempool "pile" driver, and
- its underlying "pile" stack implementation.

For community feedback and CI test.

Needless to say, this must be separated into a series of patches. For now, I'm submitting a snapshot of work in progress.

Some performance numbers from mempool_perf_autotest_2cores, all with cache=1024 cores=2 n_keep=32768:

start performance test (using ring_mp_mc, with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 753985338 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 755805913

start performance test for lf_stack (with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 29132352 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 29276708

start performance test for pile (with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 560159479 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 557910933

Quoted 5 lines

Hat tip to Bruce for bringing attention to the ring not being the optimal mempool driver.

---

Since relatively complex, did AI review with more advanced model.

Review of [RFC PATCH v6] pile stack and mempool driver

Errors

1. lib/stack/rte_stack_pile.h, __rte_stack_pile_pop()

The bulk retry loop breaks the invariant that the fragmentation path depends on (n_solo < RTE_STACK_PILE_BULK_SIZE):

	n_solo += RTE_STACK_PILE_BULK_SIZE;
	n_bulk--;
	if (n_bulk > 0)
		goto bulk;
	else
		goto solo;

Each retry adds a whole bulk worth of objects to n_solo, so control can reach the "solo:" label with n_solo >= RTE_STACK_PILE_BULK_SIZE (up to n). If the solo pop then fails and the fragmentation path is taken, four things go wrong:

- __rte_assume(n_solo < RTE_STACK_PILE_BULK_SIZE) is false, which is undefined behavior.

- the copy loop

	for (i = 0; i < n_solo; i++)
		obj_table[n_bulk * RTE_STACK_PILE_BULK_SIZE + i] = obj_frag[i];

reads past the end of obj_frag[RTE_STACK_PILE_BULK_SIZE] whenever n_solo > RTE_STACK_PILE_BULK_SIZE.

- RTE_STACK_PILE_BULK_SIZE - n_solo underflows for n_solo > BULK_SIZE, so free_solo is asked for ~4 billion elements.

- for n_solo == RTE_STACK_PILE_BULK_SIZE exactly, the request becomes a zero-element pop:

	solo_first = __rte_stack_lf_pop_elems(&pile->free_solo, 0, NULL, &solo_last);

__rte_stack_lf_pop_elems() with num == 0 never enters the traversal loop, so it leaves *last untouched and returns old_head.top, which is non-NULL whenever free_solo is not empty. solo_last is therefore still NULL when

	__rte_stack_lf_push_elems(&pile->solo, solo_first, solo_last, 0);

executes "last->next = old_head.top", i.e. a NULL pointer write. It also splices free_solo's current head onto pile->solo.

Reachable path with BULK_SIZE 32: pop of 64 objects, pile->bulk holds one element, pile->solo empty, free_solo non-empty. The pop of 2 bulks fails (n_solo becomes 32, n_bulk 1), the pop of 1 bulk succeeds, the solo pop of 32 fails, and the fragmentation path is entered with n_solo == 32.

The fragmentation path only makes sense for a partial bulk. After the retry loop, split n_solo back into whole bulks plus a remainder and service the whole-bulk part from pile->solo (or fail), keeping the fragmentation path bounded to n_solo < RTE_STACK_PILE_BULK_SIZE.

2. lib/stack/rte_stack_pile.h, __rte_stack_pile_pop()

	__rte_assume(RTE_STACK_PILE_BULK_SIZE - n_solo < RTE_STACK_PILE_BULK_SIZE - 1);

Off by one: with n_solo == 1 the left side is BULK_SIZE - 1, and "BULK_SIZE - 1 < BULK_SIZE - 1" is false. A false __rte_assume() is undefined behavior. Should be "< RTE_STACK_PILE_BULK_SIZE".

3. app/test/test_stack.c, test_stack_push_pop()

	if (memcmp(&obj_table[i],
			&popped_objs[STACK_SIZE - RTE_STACK_PILE_BULK_SIZE - i],
			RTE_STACK_PILE_BULK_SIZE) != 0) {

memcmp() takes a byte count, but RTE_STACK_PILE_BULK_SIZE is an object count. Only the first 4 pointers of each 32-pointer bulk are compared on a 64-bit build. Needs
"RTE_STACK_PILE_BULK_SIZE * sizeof(void *)".

4. lib/mempool/rte_mempool.h, rte_mempool_do_generic_put()

	const size_t move = RTE_ALIGN_MUL_CEIL(
			sizeof(void *) * (cache->len - cache->size / 2), 32);
	rte_memcpy(cache->objs, __rte_assume_cache_aligned(&cache->objs[cache->size / 2]),
			move);

Both the alignment hint and the rounded-up length are only valid when cache->size is a multiple of 32. rte_mempool_create_empty() now enforces that, but rte_mempool_cache_create() is unchanged and still accepts any size in 1..RTE_MEMPOOL_CACHE_MAX_SIZE. A user cache of, say, size 100 gives &objs[50] at a 400-byte offset, and __builtin_assume_aligned() is then told a false precondition - the compiler may emit aligned vector loads and fault. Either apply the same rounding/rejection in rte_mempool_cache_create(), or drop the alignment hint.

Warnings

5. ABI and API changes without deprecation notices

deprecation.rst currently covers only the flushthresh field and the oversize objs array. The patch additionally changes:

- struct rte_mempool: local_cache from pointer to inline local_cache[RTE_MAX_LCORE] array
- removal of the RTE_MEMPOOL_HEADER_SIZE() macro
- RTE_MEMPOOL_CACHE_MAX_SIZE 512 -> 1024
- RTE_MEMPOOL_MAX_OPS_IDX 16 -> 32, which changes the size of the exported rte_mempool_ops_table variable
- cache_size must now be a multiple of 32

The two existing deprecation entries should also be removed by this patch once they are implemented.

6. lib/mempool/rte_mempool.h - mempool header footprint

With local_cache[] inline and RTE_MEMPOOL_CACHE_MAX_SIZE at 1024, the header is roughly RTE_MAX_LCORE * 8.3 KB, i.e. about 1 MB per mempool, and it is now allocated (and memset) unconditionally. Previously RTE_MEMPOOL_HEADER_SIZE(mp, 0) omitted the array entirely for mempools created with cache_size == 0, which is common for control-object pools.

7. lib/mempool/rte_mempool.c, rte_mempool_create_empty()

	if (cache_size & 31) {
		unsigned int rounded = RTE_ALIGN_MUL_FLOOR(cache_size, 32);

Any requested cache_size below 32 is silently rounded to 0, disabling the cache. The doxygen change in rte_mempool.h says the argument "must be divisible by 32", which reads as a rejection, not a silent rounding - doc and code disagree. The log messages should also print the requested and effective values, and "divisble" is misspelled.

8. app/test/test_stack.c

The excess-push test is wrapped in "#if 0 /* FIXME ... */". That removes coverage for the standard and lock-free stacks as well, and it is exactly the test that would exercise the pile capacity behavior in item 9. Dead code should not be committed; either fix the -Warray-bounds trigger (a runtime-computed size in a volatile variable is usually enough) or drop the block and note the gap in the commit message.

9. lib/stack/rte_stack_pile.c, rte_stack_pile_init()

The pile is initialized with ceil(count / BULK_SIZE) bulk elements plus count solo elements, so it can hold up to roughly 2 * count objects, above the declared capacity. __rte_stack_pile_count() only hides this by clamping with RTE_MIN(s->capacity, ...). rte_stack_push() is documented to fail when there is insufficient space; either enforce the capacity or document that the pile does not.

10. lib/stack/rte_stack_pile.c

rte_stack_pile_init() derives the element base from "(&s->stack_pile + 1)" while rte_stack_pile_get_memsize() sizes it from "sizeof(struct rte_stack)". These agree only because rte_stack_pile happens to be the largest member of the union in struct rte_stack. If another member grows, init() writes past the memzone with no diagnostic. Use the same expression in both places, or give struct rte_stack_pile a flexible array member as rte_stack_lf has.

11. lib/stack/rte_stack_pile.h - doxygen

Both __rte_stack_pile_push() and __rte_stack_pile_pop() document "@param pile", but the parameter is "struct rte_stack *s". Doxygen with -Dwerror will flag the undocumented parameter.

12. Missing release notes

doc/guides/rel_notes/release_26_11.rst is not updated for the new pile stack type, the new "pile" mempool driver, the mempool ABI changes, or the new __rte_assume_aligned() / __rte_assume_cache_aligned() EAL macros.

13. doc/guides/mempool/stack.rst

The driver guide lists the "stack" and "lf_stack" modes; the new "pile" mode is not added. PMD/driver documentation must match the registered ops.

14. lib/mempool/mempool_trace.h

Dropping rte_trace_point_emit_u32(cache->flushthresh) changes the recorded trace format for that trace point. Worth a release note entry for consumers parsing the trace output.

15. app/test/test_stack_perf.c

	#define MAX_BURST (RTE_MEMPOOL_CACHE_MAX_SIZE / 2)

A stack library test should not take its burst size from a mempool configuration constant. Use a stack-specific value (or RTE_STACK_PILE_BULK_SIZE multiples).

Info

16. lib/eal/x86/include/rte_memcpy.h

The new constant-size block allows n <= 512 for AVX-512 and for SSE, but only n <= 256 for AVX2 - is the asymmetry intended? Splitting a single "if (" across #if/#elif/#else with the body outside is also hard to read; a per-ISA RTE_MEMCPY_CONST_MAX define and one "if" would be clearer. This change and the __rte_assume_aligned() addition are independent of the pile work and are good candidates for their own patches when the series is split.

17. drivers/net/sxe2/sxe2_txrx_vec_avx512.c

The hunk adds an unrelated blank line before "goto done;".

18. lib/stack/rte_stack_pile.h, __rte_stack_pile_bulk_pop_elems()

The element list is traversed twice: once inside __rte_stack_lf_pop_elems() (to find the new head and set *last) and again to copy the bulk contents. For a pop of 8 bulk elements that is two dependent pointer chases over the same cache lines.

19. drivers/mempool/stack/rte_mempool_stack.c

pile_enqueue() returns -ENOBUFS when the push fails, but rte_mempool_ops_enqueue_bulk() returns void and callers do not recover, so a failed put loses objects. This is the same hazard the lock-free stack already has, but the pile has two independent free lists, so the window in which a concurrent pop leaves neither free_bulk nor free_solo able to satisfy a push is wider. Worth calling out in stack_lib.rst.

20. doc/guides/prog_guide/stack_lib.rst

"performaing" -> "performing". The trailing "Note:" paragraph would render better as a ".. note::" directive.

21. The series mixes at least five independent changes (EAL assume-aligned macro, x86 rte_memcpy fast path, mempool cache/header rework, the pile stack, the pile mempool driver). You already noted this; those look like the natural split points, and the mempool cache rework in particular deserves its own review thread given the ABI impact.

#7 lore raw

RE: [RFC PATCH v6] pile stack and mempool driver

Thank you, Stephen.

Very useful review feedback, especially the first point!

I tried a few different algorithms for handling the mix of solo and bulk objects before arriving at this one. But as the AI review revealed, it still needs some work.
Will follow up with a v7 patch.

Venlig hilsen / Kind regards,
-Morten Brørup

Quoted 292 lines

-----Original Message-----
From: Stephen Hemminger [mailto:[email protected]]
Sent: Sunday, 2 August 2026 17.23

Quoted 25 lines · On Sun, 2 Aug 2026 09:59:54 +0000 Morten Brørup <[email protected]>…

Early submission of:
- some mempool optimizations,
- a new mempool "pile" driver, and
- its underlying "pile" stack implementation.

For community feedback and CI test.

Needless to say, this must be separated into a series of patches. For now, I'm submitting a snapshot of work in progress.

Some performance numbers from mempool_perf_autotest_2cores, all with cache=1024 cores=2 n_keep=32768:

start performance test (using ring_mp_mc, with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 753985338 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 755805913

start performance test for lf_stack (with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 29132352 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 29276708

start performance test for pile (with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 560159479 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 557910933

Quoted 5 lines

Hat tip to Bruce for bringing attention to the ring not being the optimal mempool driver.

---

Since relatively complex, did AI review with more advanced model.

Review of [RFC PATCH v6] pile stack and mempool driver

Errors

1. lib/stack/rte_stack_pile.h, __rte_stack_pile_pop()

The bulk retry loop breaks the invariant that the fragmentation path depends on (n_solo < RTE_STACK_PILE_BULK_SIZE):

	n_solo += RTE_STACK_PILE_BULK_SIZE;
	n_bulk--;
	if (n_bulk > 0)
		goto bulk;
	else
		goto solo;

Each retry adds a whole bulk worth of objects to n_solo, so control can reach the "solo:" label with n_solo >= RTE_STACK_PILE_BULK_SIZE (up to n).
If the solo pop then fails and the fragmentation path is taken, four things go wrong:

- __rte_assume(n_solo < RTE_STACK_PILE_BULK_SIZE) is false, which is undefined behavior.

- the copy loop

	for (i = 0; i < n_solo; i++)
		obj_table[n_bulk * RTE_STACK_PILE_BULK_SIZE + i] =

obj_frag[i];

reads past the end of obj_frag[RTE_STACK_PILE_BULK_SIZE] whenever n_solo > RTE_STACK_PILE_BULK_SIZE.

- RTE_STACK_PILE_BULK_SIZE - n_solo underflows for n_solo > BULK_SIZE, so free_solo is asked for ~4 billion elements.

- for n_solo == RTE_STACK_PILE_BULK_SIZE exactly, the request becomes a zero-element pop:

	solo_first = __rte_stack_lf_pop_elems(&pile->free_solo, 0, NULL,

&solo_last);

__rte_stack_lf_pop_elems() with num == 0 never enters the traversal loop, so it leaves *last untouched and returns old_head.top, which
is
non-NULL whenever free_solo is not empty. solo_last is therefore
still
NULL when

	__rte_stack_lf_push_elems(&pile->solo, solo_first, solo_last, 0);

executes "last->next = old_head.top", i.e. a NULL pointer write. It also splices free_solo's current head onto pile->solo.

Reachable path with BULK_SIZE 32: pop of 64 objects, pile->bulk holds one element, pile->solo empty, free_solo non-empty. The pop of 2 bulks fails (n_solo becomes 32, n_bulk 1), the pop of 1 bulk succeeds, the solo pop of 32 fails, and the fragmentation path is entered with n_solo == 32.

The fragmentation path only makes sense for a partial bulk. After the retry loop, split n_solo back into whole bulks plus a remainder and service the whole-bulk part from pile->solo (or fail), keeping the fragmentation path bounded to n_solo < RTE_STACK_PILE_BULK_SIZE.

2. lib/stack/rte_stack_pile.h, __rte_stack_pile_pop()

__rte_assume(RTE_STACK_PILE_BULK_SIZE - n_solo <
RTE_STACK_PILE_BULK_SIZE - 1);

Off by one: with n_solo == 1 the left side is BULK_SIZE - 1, and "BULK_SIZE - 1 < BULK_SIZE - 1" is false. A false __rte_assume() is undefined behavior. Should be "< RTE_STACK_PILE_BULK_SIZE".

3. app/test/test_stack.c, test_stack_push_pop()

	if (memcmp(&obj_table[i],
			&popped_objs[STACK_SIZE - RTE_STACK_PILE_BULK_SIZE -
i],
			RTE_STACK_PILE_BULK_SIZE) != 0) {

memcmp() takes a byte count, but RTE_STACK_PILE_BULK_SIZE is an object count. Only the first 4 pointers of each 32-pointer bulk are compared on a 64-bit build. Needs
"RTE_STACK_PILE_BULK_SIZE * sizeof(void *)".

4. lib/mempool/rte_mempool.h, rte_mempool_do_generic_put()

	const size_t move = RTE_ALIGN_MUL_CEIL(
			sizeof(void *) * (cache->len - cache->size / 2), 32);
	rte_memcpy(cache->objs, __rte_assume_cache_aligned(&cache-
Quoted 1 line

objs[cache->size / 2]),

			move);

Both the alignment hint and the rounded-up length are only valid when cache->size is a multiple of 32. rte_mempool_create_empty() now enforces
that, but rte_mempool_cache_create() is unchanged and still accepts any size in 1..RTE_MEMPOOL_CACHE_MAX_SIZE. A user cache of, say, size 100 gives &objs[50] at a 400-byte offset, and __builtin_assume_aligned() is then told a false precondition - the compiler may emit aligned vector loads and fault. Either apply the same rounding/rejection in rte_mempool_cache_create(), or drop the alignment hint.

Warnings

5. ABI and API changes without deprecation notices

deprecation.rst currently covers only the flushthresh field and the oversize objs array. The patch additionally changes:

- struct rte_mempool: local_cache from pointer to inline local_cache[RTE_MAX_LCORE] array
- removal of the RTE_MEMPOOL_HEADER_SIZE() macro
- RTE_MEMPOOL_CACHE_MAX_SIZE 512 -> 1024
- RTE_MEMPOOL_MAX_OPS_IDX 16 -> 32, which changes the size of the exported rte_mempool_ops_table variable
- cache_size must now be a multiple of 32

The two existing deprecation entries should also be removed by this patch once they are implemented.

6. lib/mempool/rte_mempool.h - mempool header footprint

With local_cache[] inline and RTE_MEMPOOL_CACHE_MAX_SIZE at 1024, the header is roughly RTE_MAX_LCORE * 8.3 KB, i.e. about 1 MB per mempool, and it is now allocated (and memset) unconditionally. Previously RTE_MEMPOOL_HEADER_SIZE(mp, 0) omitted the array entirely for mempools created with cache_size == 0, which is common for control-object pools.

7. lib/mempool/rte_mempool.c, rte_mempool_create_empty()

	if (cache_size & 31) {
		unsigned int rounded = RTE_ALIGN_MUL_FLOOR(cache_size, 32);

Any requested cache_size below 32 is silently rounded to 0, disabling the cache. The doxygen change in rte_mempool.h says the argument "must be divisible by 32", which reads as a rejection, not a silent rounding -
doc and code disagree. The log messages should also print the requested
and effective values, and "divisble" is misspelled.

8. app/test/test_stack.c

The excess-push test is wrapped in "#if 0 /* FIXME ... */". That removes coverage for the standard and lock-free stacks as well, and it is exactly the test that would exercise the pile capacity behavior in item 9. Dead code should not be committed; either fix the -Warray-bounds trigger (a runtime-computed size in a volatile variable is usually enough) or drop the block and note the gap in the commit message.

9. lib/stack/rte_stack_pile.c, rte_stack_pile_init()

The pile is initialized with ceil(count / BULK_SIZE) bulk elements plus count solo elements, so it can hold up to roughly 2 * count objects, above the declared capacity. __rte_stack_pile_count() only hides this by clamping with RTE_MIN(s->capacity, ...). rte_stack_push() is documented to fail when there is insufficient space; either enforce the capacity or document that the pile does not.

10. lib/stack/rte_stack_pile.c

rte_stack_pile_init() derives the element base from "(&s->stack_pile + 1)" while rte_stack_pile_get_memsize() sizes it from "sizeof(struct rte_stack)". These agree only because rte_stack_pile happens to be the largest member of the union in struct rte_stack. If another member grows, init() writes past the memzone with no diagnostic.
Use the same expression in both places, or give struct rte_stack_pile a flexible array member as rte_stack_lf has.

11. lib/stack/rte_stack_pile.h - doxygen

Both __rte_stack_pile_push() and __rte_stack_pile_pop() document "@param pile", but the parameter is "struct rte_stack *s". Doxygen with
-Dwerror will flag the undocumented parameter.

12. Missing release notes

doc/guides/rel_notes/release_26_11.rst is not updated for the new pile stack type, the new "pile" mempool driver, the mempool ABI changes, or the new __rte_assume_aligned() / __rte_assume_cache_aligned() EAL macros.

13. doc/guides/mempool/stack.rst

The driver guide lists the "stack" and "lf_stack" modes; the new "pile" mode is not added. PMD/driver documentation must match the registered ops.

14. lib/mempool/mempool_trace.h

Dropping rte_trace_point_emit_u32(cache->flushthresh) changes the recorded trace format for that trace point. Worth a release note entry for consumers parsing the trace output.

15. app/test/test_stack_perf.c

	#define MAX_BURST (RTE_MEMPOOL_CACHE_MAX_SIZE / 2)

A stack library test should not take its burst size from a mempool configuration constant. Use a stack-specific value (or RTE_STACK_PILE_BULK_SIZE multiples).

Info

16. lib/eal/x86/include/rte_memcpy.h

The new constant-size block allows n <= 512 for AVX-512 and for SSE, but
only n <= 256 for AVX2 - is the asymmetry intended? Splitting a single "if (" across #if/#elif/#else with the body outside is also hard to read; a per-ISA RTE_MEMCPY_CONST_MAX define and one "if" would be clearer. This change and the __rte_assume_aligned() addition are independent of the pile work and are good candidates for their own patches when the series is split.

17. drivers/net/sxe2/sxe2_txrx_vec_avx512.c

The hunk adds an unrelated blank line before "goto done;".

18. lib/stack/rte_stack_pile.h, __rte_stack_pile_bulk_pop_elems()

The element list is traversed twice: once inside __rte_stack_lf_pop_elems() (to find the new head and set *last) and again to copy the bulk contents. For a pop of 8 bulk elements that is two dependent pointer chases over the same cache lines.

19. drivers/mempool/stack/rte_mempool_stack.c

pile_enqueue() returns -ENOBUFS when the push fails, but rte_mempool_ops_enqueue_bulk() returns void and callers do not recover, so a failed put loses objects. This is the same hazard the lock-free stack already has, but the pile has two independent free lists, so the window in which a concurrent pop leaves neither free_bulk nor free_solo able to satisfy a push is wider. Worth calling out in stack_lib.rst.

20. doc/guides/prog_guide/stack_lib.rst

"performaing" -> "performing". The trailing "Note:" paragraph would render better as a ".. note::" directive.

21. The series mixes at least five independent changes (EAL assume- aligned
macro, x86 rte_memcpy fast path, mempool cache/header rework, the pile stack, the pile mempool driver). You already noted this; those look like the natural split points, and the mempool cache rework in particular deserves its own review thread given the ABI impact.

#8 lore raw

RFC PATCH v7 1/1 pile stack and mempool driver

In reply to Morten Brørup

Early submission of:
- A new "pile" stack-like implementation using the Stack API, and
- an accompanying "pile" mempool driver.
And:
- Some mempool optimizations.
- Deprecating "__rte_restrict" in favor of keyword "__restrict", supported by all relevant C/C++ compilers.
- An x86 rte_memcpy() optimization for some compile time known sizes (64-byte blocks up to 512 or 256 bytes).
- Decorating the object tables in the Stack API with "restrict".

For CI test and community feedback.

Needless to say, this must be separated into a series of patches, or multiple independent series of patches.
For now, I'm submitting a snapshot of work in progress.

The "pile" somewhat resembles the lock-free stack, but operates on bulks (arrays) of objects, to significantly reduce linked list traversal.
With the pile's default bulk size of 32 objects, a mempool cache flush/refill traverses a linked list of only 16 elements, whereas the lock-free stack would traverse a linked list of 512 elements.

Some performance numbers from mempool_perf_autotest_2cores, all with cache=1024 cores=2 n_keep=32768:

start performance test (using ring_mp_mc, with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 753985338 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 755805913

start performance test for lf_stack (with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 29132352 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 29276708

start performance test for pile (with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 560159479 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 557910933

Hat tip to Bruce for bringing attention to the ring not being the optimal mempool driver!

---
v2:
* Fix indentation, long lines, etc. (checkpatch)
* Fix label followed by a declaration is a C23 extension. (CI)
* Added __rte_internal to pile init and get_memsize. (AI)
* Fix roll back bulk elements in wrong order with fragmentation. (AI)
* Minor changes suggested by AI.
v3:
* Revert rename unused field in rte_stack_pile_bulk_elem structure. v4:
* Revert add __rte_internal.
Compilation fails, and existing stack implementations don't have it.
v5:
* Remove compiler diagnostic pragmas in stack overflow test case. (Stephen)
* Temporarily remove stack overflow test case until a sufficiently obfuscated variant doesn't trigger a compiler warning about array overrun.
* Revert most changes in rte_stack_std.h, and only change what is necessary.
* Add FIXME in the pile implementation, noting that support for the generic memory model should be removed here too, if removed in the lock-free stack.
(Inspired by Stephen)
v6:
* Add note about roll back of objects in fragmentation element. (AI)
* Move declaration of temporary variable up, to please compilers. (CI)
* Revert mempool cache size adjustments in some drivers;
let the mempool creation function adjust at runtime instead.
v7:
* Use the "__restrict" keyword, supported by all relevant C/C++ compilers. Degrade "__rte_restrict" to a backwards compatibility macro, document it as deprecated, and check for it in checkpatches.sh.
* Eliminate risk of namespace pollution by prefixing ALIGNMENT_MASK macro in x86 rte_memcpy.h header file.
* Replace conditional code by defining RTE_MEMCPY_BLOCK_64_MAX. (AI)
* Fix off-by-one in assumption when fetching free elements for the excess objects in the fragmentation element. (AI, advanced model)
* Fix memcmp() size in test_stack_push_pop(). (AI, advanced model)
* Do not try fragmentation when we know (from fetching bulk elements above) that no bulk element is available.
(Inspired by AI, advanced model)
* Fix function descriptions mentioning wrong parameter name. (AI, advanced model)
* Fix typo in documentation. (AI, advanced model)
* Mention the pile in the mempool stack documentation.
(AI, advanced model)
* Set the mbuf default pool ops to "pile", for CI test purposes only.
* Select the C11 memory model for x86, for CI test purposes only.

---
 app/test/test_mempool.c                   |   3 +-
 app/test/test_stack.c                     |  64 ++++-
 app/test/test_stack_perf.c                |  15 +-
 config/rte_config.h                       |   7 +-
 config/x86/meson.build                    |   1 +
 devtools/checkpatches.sh                  |   8 +
 doc/guides/mempool/stack.rst              |  12 +-
 doc/guides/prog_guide/stack_lib.rst       |  67 ++++-
 drivers/mempool/stack/rte_mempool_stack.c |  40 +++
 drivers/net/sxe2/sxe2_txrx_vec_avx512.c   |   2 +-
 drivers/net/tap/rte_eth_tap.c             |   2 +-
 lib/eal/include/rte_bitset.h              |   2 +-
 lib/eal/include/rte_common.h              |  21 +-
 lib/eal/x86/include/rte_memcpy.h          |  69 +++--
 lib/mempool/mempool_trace.h               |   1 -
 lib/mempool/rte_mempool.c                 |  75 +++--
 lib/mempool/rte_mempool.h                 |  76 +++--
 lib/stack/meson.build                     |   3 +-
 lib/stack/rte_stack.c                     |  18 +-
 lib/stack/rte_stack.h                     |  83 +++++-
 lib/stack/rte_stack_lf.h                  |   6 +-
 lib/stack/rte_stack_lf_c11.h              |   2 +-
 lib/stack/rte_stack_lf_generic.h          |   2 +-
 lib/stack/rte_stack_lf_stubs.h            |   2 +-
 lib/stack/rte_stack_pile.c                |  35 +++
 lib/stack/rte_stack_pile.h                | 330 ++++++++++++++++++++++
 lib/stack/rte_stack_std.h                 |  24 +-
 27 files changed, 836 insertions(+), 134 deletions(-)
 create mode 100644 lib/stack/rte_stack_pile.c
 create mode 100644 lib/stack/rte_stack_pile.h
app/test/test_mempool.c +1 -2
index e54249ce61..76d45cea2a 100644
@@ -112,8 +112,7 @@ test_mempool_basic(struct rte_mempool *mp, int use_external_cache)
 		GOTO_ERR(ret, out);
 
 	printf("get private data\n");
-	if (rte_mempool_get_priv(mp) != (char *)mp +
-			RTE_MEMPOOL_HEADER_SIZE(mp, mp->cache_size))
+	if (rte_mempool_get_priv(mp) != (char *)mp + sizeof(struct rte_mempool))
 		GOTO_ERR(ret, out);
 
 #ifndef RTE_EXEC_ENV_FREEBSD /* rte_mem_virt2iova() not supported on bsd */
app/test/test_stack.c +52 -12

82 lines not on this page · raw message

app/test/test_stack_perf.c +13 -2
index 3f17a2606c..a15f3719c2 100644
@@ -14,14 +14,14 @@
 #include "test.h"
 
 #define STACK_NAME "STACK_PERF"
-#define MAX_BURST 32
+#define MAX_BURST (RTE_MEMPOOL_CACHE_MAX_SIZE / 2)
 #define STACK_SIZE (RTE_MAX_LCORE * MAX_BURST)
 
 /*
  * Push/pop bulk sizes, marked volatile so they aren't treated as compile-time
  * constants.
  */
-static volatile unsigned int bulk_sizes[] = {8, MAX_BURST};
+static volatile unsigned int bulk_sizes[] = {1, 8, 32, MAX_BURST};
 
 static RTE_ATOMIC(uint32_t) lcore_barrier;
 
@@ -354,5 +354,16 @@ test_lf_stack_perf(void)
 #endif
 }
 
+static int
+test_pile_perf(void)
+{
+#if defined(RTE_STACK_PILE_SUPPORTED)
+	return __test_stack_perf(RTE_STACK_F_PILE);
+#else
+	return TEST_SKIPPED;
+#endif
+}
+
 REGISTER_PERF_TEST(stack_perf_autotest, test_stack_perf);
 REGISTER_PERF_TEST(stack_lf_perf_autotest, test_lf_stack_perf);
+REGISTER_PERF_TEST(stack_pile_perf_autotest, test_pile_perf);
config/rte_config.h +5 -2
index 0447cdf2ad..6085d2e1a0 100644
@@ -56,14 +56,17 @@
 #define RTE_CONTIGMEM_DEFAULT_BUF_SIZE (512*1024*1024)
 
 /* mempool defines */
-#define RTE_MEMPOOL_CACHE_MAX_SIZE 512
+#define RTE_MEMPOOL_CACHE_MAX_SIZE 1024
 /* RTE_LIBRTE_MEMPOOL_STATS is not set */
 /* RTE_LIBRTE_MEMPOOL_DEBUG is not set */
 
 /* mbuf defines */
-#define RTE_MBUF_DEFAULT_MEMPOOL_OPS "ring_mp_mc"
+#define RTE_MBUF_DEFAULT_MEMPOOL_OPS "pile" /* FIXME: Test only. Default: "ring_mp_mc" */
 /* RTE_MBUF_HISTORY_DEBUG is not set */
 
+/* stack defines */
+#define RTE_STACK_PILE_BULK_SIZE 32
+
 /* ether defines */
 #define RTE_MAX_QUEUES_PER_PORT 1024
 #define RTE_ETHDEV_RXTX_CALLBACKS 1
config/x86/meson.build +1 -0
index 124b204847..28be579556 100644
@@ -49,6 +49,7 @@ else
 endif
 
 dpdk_conf.set('RTE_MAX_NUMA_NODES', 32)
+dpdk_conf.set('RTE_USE_C11_MEM_MODEL', true) # FIXME: Test only.
 
 if is_ms_compiler
     subdir_done()
devtools/checkpatches.sh +8 -0
index 18bd825372..63c1ece0b6 100755
@@ -154,6 +154,14 @@ check_forbidden_additions() { # <patch>
 		-f $(dirname $(readlink -f $0))/check-forbidden-tokens.awk \
 		"$1" || res=1
 
+	# refrain from using __rte_restrict
+	awk -v FOLDERS="lib drivers app examples" \
+		-v EXPRESSIONS="__rte_restrict" \
+		-v RET_ON_FAIL=1 \
+		-v MESSAGE='Using __rte_restrict, prefer restrict' \
+		-f $(dirname $(readlink -f $0))/check-forbidden-tokens.awk \
+		"$1" || res=1
+
 	# refrain from using compiler __atomic_xxx builtins
 	awk -v FOLDERS="lib drivers app examples" \
 		-v SKIP_FILES='drivers/common/cnxk/' \
doc/guides/mempool/stack.rst +11 -1
index 80ea07e65d..708f48f952 100644
@@ -1,5 +1,6 @@
 ..  SPDX-License-Identifier: BSD-3-Clause
     Copyright(c) 2020 Intel Corporation.
+    Copyright(c) 2026 SmartShare Systems.
 
 Stack Mempool Driver
 ====================
@@ -28,6 +29,12 @@ can be selected as described in :ref:`Mempool_Handlers`:
   The underlying **rte_stack** operates in lock-free mode. For more
   information please refer to :ref:`Stack_Library_LF_Stack`.
 
+- ``pile``
+
+  The underlying **rte_stack** operates in lock-free mode,
+  and is optimized for bulks of objects.
+  For more information please refer to :ref:`_Stack_Library_Pile`.
+
 The standard stack outperforms the lock-free stack on average, however the
 standard stack is non-preemptive: if a mempool user is preempted while holding
 the stack lock, that thread will block all other mempool accesses until it
@@ -35,9 +42,12 @@ returns and releases the lock. As a result, an application using the standard
 stack whose threads can be preempted can suffer from brief, infrequent
 performance hiccups.
 
-The lock-free stack, by design, is not susceptible to this problem; one thread can
+The lock-free stack and the pile, by design, are not susceptible to this problem; one thread can
 be preempted at any point during a push or pop operation and will not impede
 the progress of any other thread.
 
+The pile is not LIFO per object, but per bulk of objects.
+Although the pile is optimized for bulks of objects, it can handle any request size.
+
 For a more detailed description of the stack implementations, please refer to
 :doc:`/prog_guide/stack_lib`.
doc/guides/prog_guide/stack_lib.rst +63 -4

95 lines not on this page · raw message

drivers/mempool/stack/rte_mempool_stack.c +40 -0
index 1476905227..7467b8b39e 100644
@@ -41,6 +41,36 @@ lf_stack_alloc(struct rte_mempool *mp)
 	return __stack_alloc(mp, RTE_STACK_F_LF);
 }
 
+static int
+pile_alloc(struct rte_mempool *mp)
+{
+	return __stack_alloc(mp, RTE_STACK_F_PILE);
+}
+
+static int
+pile_enqueue(struct rte_mempool *mp, void * const *obj_table,
+	      unsigned int n)
+{
+	struct rte_stack *s = mp->pool_data;
+
+	RTE_ASSERT(s != NULL);
+	RTE_ASSERT(obj_table != NULL);
+
+	return __rte_stack_pile_push(s, obj_table, n) == 0 ? -ENOBUFS : 0;
+}
+
+static int
+pile_dequeue(struct rte_mempool *mp, void **obj_table,
+	      unsigned int n)
+{
+	struct rte_stack *s = mp->pool_data;
+
+	RTE_ASSERT(s != NULL);
+	RTE_ASSERT(obj_table != NULL);
+
+	return __rte_stack_pile_pop(s, obj_table, n) == 0 ? -ENOBUFS : 0;
+}
+
 static int
 stack_enqueue(struct rte_mempool *mp, void * const *obj_table,
 	      unsigned int n)
@@ -93,5 +123,15 @@ static struct rte_mempool_ops ops_lf_stack = {
 	.get_count = stack_get_count
 };
 
+static struct rte_mempool_ops ops_pile = {
+	.name = "pile",
+	.alloc = pile_alloc,
+	.free = stack_free,
+	.enqueue = pile_enqueue,
+	.dequeue = pile_dequeue,
+	.get_count = stack_get_count
+};
+
 RTE_MEMPOOL_REGISTER_OPS(ops_stack);
 RTE_MEMPOOL_REGISTER_OPS(ops_lf_stack);
+RTE_MEMPOOL_REGISTER_OPS(ops_pile);
drivers/net/sxe2/sxe2_txrx_vec_avx512.c +1 -1
index a830c7a33b..2e680f4027 100644
@@ -67,7 +67,7 @@ static __rte_always_inline int32_t sxe2_tx_bufs_free_vec_avx512(struct sxe2_tx_q
 		}
 		cache->len += rs_thresh;
 
-		if (cache->len >= cache->flushthresh) {
+		if (cache->len >= cache->size) {
 			(void)rte_mempool_ops_enqueue_bulk(mp,
 					&cache->objs[cache->size], cache->len - cache->size);
 			cache->len = cache->size;
drivers/net/tap/rte_eth_tap.c +1 -1
index b93452f168..b3142561c2 100644
@@ -61,7 +61,7 @@
 #define TAP_MAX_MAC_ADDRS	16
 #define TAP_GSO_MBUFS_PER_CORE	128
 #define TAP_GSO_MBUF_SEG_SIZE	128
-#define TAP_GSO_MBUF_CACHE_SIZE	4
+#define TAP_GSO_MBUF_CACHE_SIZE	32
 #define TAP_GSO_MBUFS_NUM \
 	(TAP_GSO_MBUFS_PER_CORE * TAP_GSO_MBUF_CACHE_SIZE)
 
lib/eal/include/rte_bitset.h +1 -1
index 4e6d44874a..8fc4a76da7 100644
@@ -978,7 +978,7 @@ rte_bitset_find_clear_wrap(const uint64_t *bitset, size_t size, size_t start_bit
  */
 __rte_experimental
 static inline void
-rte_bitset_copy(uint64_t *__rte_restrict dst_bitset, const uint64_t *__rte_restrict src_bitset,
+rte_bitset_copy(uint64_t * __restrict dst_bitset, const uint64_t * __restrict src_bitset,
 		size_t size)
 {
 	rte_memcpy(dst_bitset, src_bitset, RTE_BITSET_SIZE(size));
lib/eal/include/rte_common.h +16 -5
index 79d2a0ab93..212f43c494 100644
@@ -250,12 +250,11 @@ typedef uint16_t unaligned_uint16_t;
 
 /**
  * Mark pointer as restricted with regard to pointer aliasing.
+ * For backwards compatibility only.
+ * @deprecated
+ * Use the ``__restrict`` keyword (recognized by supported C and C++ compilers) instead.
  */
-#if !defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901L
 #define __rte_restrict __restrict
-#else
-#define __rte_restrict restrict
-#endif
 
 /**
  * definition to mark a variable or function parameter as used so
@@ -567,6 +566,15 @@ static void __attribute__((destructor(RTE_PRIO(prio)), used)) func(void)
 #define __rte_assume(condition) __assume(condition)
 #endif
 
+/**
+ * Alignment hint precondition
+ */
+#ifdef RTE_TOOLCHAIN_MSVC
+#define __rte_assume_aligned(ptr, alignment) (ptr)
+#else
+#define __rte_assume_aligned(ptr, alignment) __builtin_assume_aligned(ptr, alignment)
+#endif
+
 /**
  * Disable AddressSanitizer on some code
  */
@@ -729,7 +737,7 @@ static void __attribute__((destructor(RTE_PRIO(prio)), used)) func(void)
  *   True(1) where the pointer is correctly aligned, false(0) otherwise
  */
 static inline int
-rte_is_aligned(const void * const __rte_restrict ptr, const unsigned int align)
+rte_is_aligned(const void * const __restrict ptr, const unsigned int align)
 {
 	return ((uintptr_t)ptr & (align - 1)) == 0;
 }
@@ -775,6 +783,9 @@ rte_is_aligned(const void * const __rte_restrict ptr, const unsigned int align)
 /** Force minimum cache line alignment. */
 #define __rte_cache_min_aligned __rte_aligned(RTE_CACHE_LINE_MIN_SIZE)
 
+/** Cache alignment hint precondition */
+#define __rte_assume_cache_aligned(ptr) __rte_assume_aligned(ptr, RTE_CACHE_LINE_SIZE)
+
 #define _RTE_CACHE_GUARD_HELPER2(unique) \
 	alignas(RTE_CACHE_LINE_SIZE) \
 	char cache_guard_ ## unique[RTE_CACHE_LINE_SIZE * RTE_CACHE_GUARD_LINES]
lib/eal/x86/include/rte_memcpy.h +48 -21

196 lines not on this page · raw message

lib/mempool/mempool_trace.h +0 -1
index 23cda1473c..60e47cf67b 100644
@@ -119,7 +119,6 @@ RTE_TRACE_POINT(
 	rte_trace_point_emit_i32(socket_id);
 	rte_trace_point_emit_ptr(cache);
 	rte_trace_point_emit_u32(cache->len);
-	rte_trace_point_emit_u32(cache->flushthresh);
 )
 
 RTE_TRACE_POINT(
lib/mempool/rte_mempool.c +53 -22

147 lines not on this page · raw message

lib/mempool/rte_mempool.h +37 -39

157 lines not on this page · raw message

lib/stack/meson.build +2 -1
index 18177a742f..50e688522e 100644
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2019 Intel Corporation
 
-sources = files('rte_stack.c', 'rte_stack_std.c', 'rte_stack_lf.c')
+sources = files('rte_stack.c', 'rte_stack_std.c', 'rte_stack_lf.c', 'rte_stack_pile.c')
 headers = files('rte_stack.h')
 # subheaders, not for direct inclusion by apps
 indirect_headers += files(
@@ -10,4 +10,5 @@ indirect_headers += files(
         'rte_stack_lf_generic.h',
         'rte_stack_lf_c11.h',
         'rte_stack_lf_stubs.h',
+        'rte_stack_pile.h',
 )
lib/stack/rte_stack.c +17 -1
index 4c78fe4b4b..a4bbf8a4d7 100644
@@ -1,5 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright(c) 2019 Intel Corporation
+ * Copyright(c) 2026 SmartShare Systems
  */
 
 #include <stdalign.h>
@@ -32,6 +33,8 @@ rte_stack_init(struct rte_stack *s, unsigned int count, uint32_t flags)
 
 	if (flags & RTE_STACK_F_LF)
 		rte_stack_lf_init(s, count);
+	else if (flags & RTE_STACK_F_PILE)
+		rte_stack_pile_init(s, count);
 	else
 		rte_stack_std_init(s);
 }
@@ -41,6 +44,8 @@ rte_stack_get_memsize(unsigned int count, uint32_t flags)
 {
 	if (flags & RTE_STACK_F_LF)
 		return rte_stack_lf_get_memsize(count);
+	else if (flags & RTE_STACK_F_PILE)
+		return rte_stack_pile_get_memsize(count);
 	else
 		return rte_stack_std_get_memsize(count);
 }
@@ -58,7 +63,11 @@ rte_stack_create(const char *name, unsigned int count, int socket_id,
 	unsigned int sz;
 	int ret;
 
-	if (flags & ~(RTE_STACK_F_LF)) {
+	if (flags & ~(RTE_STACK_F_LF | RTE_STACK_F_PILE)) {
+		STACK_LOG_ERR("Unsupported stack flags %#x", flags);
+		return NULL;
+	}
+	if ((flags & RTE_STACK_F_LF) && (flags & RTE_STACK_F_PILE)) {
 		STACK_LOG_ERR("Unsupported stack flags %#x", flags);
 		return NULL;
 	}
@@ -73,6 +82,13 @@ rte_stack_create(const char *name, unsigned int count, int socket_id,
 		return NULL;
 	}
 #endif
+#if !defined(RTE_STACK_PILE_SUPPORTED)
+	if (flags & RTE_STACK_F_PILE) {
+		STACK_LOG_ERR("Pile is not supported on your platform");
+		rte_errno = ENOTSUP;
+		return NULL;
+	}
+#endif
 
 	sz = rte_stack_get_memsize(count, flags);
 
lib/stack/rte_stack.h +81 -2

155 lines not on this page · raw message

lib/stack/rte_stack_lf.h +4 -2
index f2b012cd0e..e13b2a60d7 100644
@@ -34,7 +34,7 @@
  */
 static __rte_always_inline unsigned int
 __rte_stack_lf_push(struct rte_stack *s,
-		    void * const *obj_table,
+		    void * const * __restrict obj_table,
 		    unsigned int n)
 {
 	struct rte_stack_lf_elem *tmp, *first, *last = NULL;
@@ -71,7 +71,8 @@ __rte_stack_lf_push(struct rte_stack *s,
  *   - Actual number of objects popped.
  */
 static __rte_always_inline unsigned int
-__rte_stack_lf_pop(struct rte_stack *s, void **obj_table, unsigned int n)
+__rte_stack_lf_pop(struct rte_stack *s, void ** __restrict obj_table,
+		   unsigned int n)
 {
 	struct rte_stack_lf_elem *first, *last = NULL;
 
@@ -79,6 +80,7 @@ __rte_stack_lf_pop(struct rte_stack *s, void **obj_table, unsigned int n)
 		return 0;
 
 	/* Pop n used elements */
+	__rte_assume(obj_table != NULL);
 	first = __rte_stack_lf_pop_elems(&s->stack_lf.used,
 					 n, obj_table, &last);
 	if (unlikely(first == NULL))
lib/stack/rte_stack_lf_c11.h +1 -1
index b97e02d6a1..d02bd1aece 100644
@@ -99,7 +99,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 static __rte_always_inline struct rte_stack_lf_elem *
 __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 unsigned int num,
-			 void **obj_table,
+			 void ** __restrict obj_table,
 			 struct rte_stack_lf_elem **last)
 {
 	struct rte_stack_lf_head old_head;
lib/stack/rte_stack_lf_generic.h +1 -1
index cc69e4d168..5187caf586 100644
@@ -74,7 +74,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 static __rte_always_inline struct rte_stack_lf_elem *
 __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 unsigned int num,
-			 void **obj_table,
+			 void ** __restrict obj_table,
 			 struct rte_stack_lf_elem **last)
 {
 	struct rte_stack_lf_head old_head;
lib/stack/rte_stack_lf_stubs.h +1 -1
index a05abf1f1c..5810744d67 100644
@@ -30,7 +30,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 static __rte_always_inline struct rte_stack_lf_elem *
 __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 unsigned int num,
-			 void **obj_table,
+			 void ** __restrict obj_table,
 			 struct rte_stack_lf_elem **last)
 {
 	RTE_SET_USED(obj_table);
lib/stack/rte_stack_pile.c new file +35 -0
new file mode 100644
index 0000000000..ccad544b8c
@@ -0,0 +1,35 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2026 SmartShare Systems
+ */
+
+#include "rte_stack.h"
+
+void
+rte_stack_pile_init(struct rte_stack *s, unsigned int count)
+{
+	unsigned int bulk = (count + RTE_STACK_PILE_BULK_SIZE - 1) / RTE_STACK_PILE_BULK_SIZE;
+	struct rte_stack_pile_bulk_elem *bulk_elems =
+			(struct rte_stack_pile_bulk_elem *)(&s->stack_pile.elems);
+	struct rte_stack_lf_elem *solo_elems = (struct rte_stack_lf_elem *)&bulk_elems[bulk];
+	unsigned int i;
+
+	for (i = 0; i < bulk; i++)
+		__rte_stack_pile_bulk_push_elems(&s->stack_pile.free_bulk,
+					  &bulk_elems[i], &bulk_elems[i], 1);
+	for (i = 0; i < count; i++)
+		__rte_stack_lf_push_elems(&s->stack_pile.free_solo,
+					  &solo_elems[i], &solo_elems[i], 1);
+}
+
+ssize_t
+rte_stack_pile_get_memsize(unsigned int count)
+{
+	unsigned int bulk = (count + RTE_STACK_PILE_BULK_SIZE - 1) / RTE_STACK_PILE_BULK_SIZE;
+	ssize_t sz = offsetof(struct rte_stack, stack_pile.elems);
+	sz += bulk * sizeof(struct rte_stack_pile_bulk_elem);
+	sz += count * sizeof(struct rte_stack_lf_elem);
+	sz += RTE_CACHE_LINE_ROUNDUP(sz);
+	sz += RTE_CACHE_GUARD_LINES * RTE_CACHE_LINE_SIZE;
+
+	return sz;
+}
lib/stack/rte_stack_pile.h new file +330 -0

330 lines not on this page · raw message

lib/stack/rte_stack_std.h +13 -11
index ae28add5c4..8a3ca9edf6 100644
@@ -20,25 +20,25 @@
  *   Actual number of objects pushed (either 0 or *n*).
  */
 static __rte_always_inline unsigned int
-__rte_stack_std_push(struct rte_stack *s, void * const *obj_table,
+__rte_stack_std_push(struct rte_stack *s, void * const * __restrict obj_table,
 		     unsigned int n)
 {
 	struct rte_stack_std *stack = &s->stack_std;
 	unsigned int index;
-	void **cache_objs;
+	void ** __restrict stack_objs;
 
 	rte_spinlock_lock(&stack->lock);
-	cache_objs = &stack->objs[stack->len];
+	stack_objs = &stack->objs[stack->len];
 
-	/* Is there sufficient space in the stack? */
-	if ((stack->len + n) > s->capacity) {
+	if (unlikely((stack->len + n) > s->capacity)) {
+		/* Insufficient space in the stack. */
 		rte_spinlock_unlock(&stack->lock);
 		return 0;
 	}
 
-	/* Add elements back into the cache */
+	/* Push objects to the stack */
 	for (index = 0; index < n; ++index, obj_table++)
-		cache_objs[index] = *obj_table;
+		stack_objs[index] = *obj_table;
 
 	stack->len += n;
 
@@ -59,24 +59,26 @@ __rte_stack_std_push(struct rte_stack *s, void * const *obj_table,
  *   Actual number of objects popped (either 0 or *n*).
  */
 static __rte_always_inline unsigned int
-__rte_stack_std_pop(struct rte_stack *s, void **obj_table, unsigned int n)
+__rte_stack_std_pop(struct rte_stack *s, void ** __restrict obj_table, unsigned int n)
 {
 	struct rte_stack_std *stack = &s->stack_std;
 	unsigned int index, len;
-	void **cache_objs;
+	void ** __restrict stack_objs;
 
 	rte_spinlock_lock(&stack->lock);
 
 	if (unlikely(n > stack->len)) {
+		/* Insufficient objects in the stack. */
 		rte_spinlock_unlock(&stack->lock);
 		return 0;
 	}
 
-	cache_objs = stack->objs;
+	stack_objs = stack->objs;
 
+	/* Pop objects from the stack */
 	for (index = 0, len = stack->len - 1; index < n;
 			++index, len--, obj_table++)
-		*obj_table = cache_objs[len];
+		*obj_table = stack_objs[len];
 
 	stack->len -= n;
 	rte_spinlock_unlock(&stack->lock);
Signature · 1 line
2.43.0
#9 lore raw

Re: [RFC PATCH v7] pile stack and mempool driver

Quoted 39 lines · On Mon, 3 Aug 2026 08:14:46 +0000 Morten Brørup <[email protected]>…

Early submission of:
- A new "pile" stack-like implementation using the Stack API, and
- an accompanying "pile" mempool driver.
And:
- Some mempool optimizations.
- Deprecating "__rte_restrict" in favor of keyword "__restrict", supported by all relevant C/C++ compilers.
- An x86 rte_memcpy() optimization for some compile time known sizes (64-byte blocks up to 512 or 256 bytes).
- Decorating the object tables in the Stack API with "restrict".

For CI test and community feedback.

Needless to say, this must be separated into a series of patches, or multiple independent series of patches.
For now, I'm submitting a snapshot of work in progress.

The "pile" somewhat resembles the lock-free stack, but operates on bulks (arrays) of objects, to significantly reduce linked list traversal.
With the pile's default bulk size of 32 objects, a mempool cache flush/refill traverses a linked list of only 16 elements, whereas the lock-free stack would traverse a linked list of 512 elements.

Some performance numbers from mempool_perf_autotest_2cores, all with cache=1024 cores=2 n_keep=32768:

start performance test (using ring_mp_mc, with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 753985338 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 755805913

start performance test for lf_stack (with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 29132352 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 29276708

start performance test for pile (with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 560159479 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 557910933

Quoted 5 lines

Hat tip to Bruce for bringing attention to the ring not being the optimal mempool driver!

---

Claude Opus AI review still finds lots of issues.

Review of [RFC PATCH v7] pile stack and mempool driver

Fixed since v6: memcmp() size in test_stack_push_pop(), the off-by-one __rte_assume() in the fragmentation path, the n_solo >= BULK_SIZE overrun (now guarded), rte_mempool_cache_create() alignment precondition, the init/get_memsize base-address mismatch (flexible array member), the @param names, the rte_memcpy #if/#elif split, the sxe2 stray blank line, the doc typo and the note directive.

Errors

1. lib/stack/rte_stack_pile.h, __rte_stack_pile_pop(), "fail:" label

	if (n_bulk > 0) {
		/* Attach the fragmentation element after the bulk elements. */
		bulk_last->next = frag;
	} else {
		bulk_first = frag;
		bulk_last = frag;
	}
	n_bulk += 1;
	...
	__rte_stack_pile_bulk_push_elems(&pile->bulk, bulk_first, bulk_last, n_bulk);

In the n_bulk > 0 branch, frag is linked after bulk_last but bulk_last is not advanced to frag. __rte_stack_lf_push_elems() then executes "last->next = old_head.top" on the old bulk_last, overwriting the link to frag. frag is dropped from every list, while list->len is incremented by n_bulk + 1.

Two consequences: the bulk element and its RTE_STACK_PILE_BULK_SIZE objects are leaked permanently, and pile->bulk.len is now one higher than the number of linked elements. A later pop that reserves that phantom element will succeed in the CAS on len, then walk off the end of the list, hit "i != num", re-read list->head and retry - forever. The inflated len never recovers, so this is a hang, not a transient retry.

This is not new in v7; it was present in v6 and I missed it.

Easiest fix is to mirror what the success path already does correctly and prepend instead of append:

	if (frag != NULL) {
		if (n_bulk > 0)
			frag->next = bulk_first;
		else
			bulk_last = frag;
		bulk_first = frag;
		n_bulk++;
	}

2. lib/stack/rte_stack_pile.c, rte_stack_pile_get_memsize()

	sz += count * sizeof(struct rte_stack_lf_elem);
	sz += RTE_CACHE_LINE_ROUNDUP(sz);

RTE_CACHE_LINE_ROUNDUP() returns the rounded value, it does not return the padding, so "+=" roughly doubles the allocation instead of rounding it up. Every pile stack, and therefore every pile mempool, reserves about twice the memzone it needs. Should be:

	sz = RTE_CACHE_LINE_ROUNDUP(sz);

3. doc/guides/mempool/stack.rst

For more information please refer to :ref:_Stack_Library_Pile.

The leading underscore belongs to the label definition, not the reference; compare the existing ":ref:Stack_Library_LF_Stack" two paragraphs above. As written, Sphinx cannot resolve the reference, which fails the docs build with -Dwerror=true.

Warnings

4. __rte_restrict deprecation

Replacing __rte_restrict with the raw __restrict compiler extension and adding a checkpatches.sh rule against the RTE macro is a project-wide policy change and should be a separate patch on its own thread, not a side effect of the pile work. Specific problems as posted:

- The tree still has about 150 uses of __rte_restrict in 10 files that this patch does not convert, 98 of them in drivers/net/mlx5/mlx5_tx.h. The new checkpatch rule will fire on unrelated patches that touch any of those lines.
- The deprecation is announced only in a doxygen comment. There is no entry in doc/guides/rel_notes/deprecation.rst, no release note, and the macro is not marked with RTE_DEPRECATED, so nothing warns at compile time.
- The checkpatch message says "prefer restrict", but the patch uses "__restrict". "restrict" is the C99/C11 keyword; "__restrict" is a compiler extension. Picking between the two per language mode is exactly what the macro existed for, since C++ has no "restrict". If the intent is to standardise on the extension, the message and the doxygen text should say so.

5. Test-only configuration changes

config/rte_config.h sets RTE_MBUF_DEFAULT_MEMPOOL_OPS to "pile" and config/x86/meson.build sets RTE_USE_C11_MEM_MODEL. Both are marked FIXME/test-only, noting them so they are not forgotten before the series is split and reposted.

6. lib/stack/rte_stack_pile.h

The "return 0;" after the fail label is indented with four spaces instead of a tab.

7. rte_mempool_cache_create() and rte_mempool_create_empty() now disagree

Both round cache_size down to a multiple of 32, but for a request below 32 rte_mempool_cache_create() returns EINVAL while rte_mempool_create_empty() logs a warning and silently creates the mempool with no cache at all. Same input, two different outcomes. The doxygen for the cache_size argument says it "must be divisible by 32", which reads as a rejection in both cases.

The silent path is the risky one: any existing caller passing a small cache size loses caching entirely with no error. The TAP change in this patch (TAP_GSO_MBUF_CACHE_SIZE 4 -> 32) is one instance that was caught; call sites passing literals are not. Consider rejecting in both places, or rounding up to 32 rather than down to 0. The log messages should also print the requested and effective values.

8. ABI and API changes still without deprecation notices

deprecation.rst is untouched. Still outstanding from v6:

- struct rte_mempool: local_cache pointer -> inline local_cache[RTE_MAX_LCORE] array
- removal of RTE_MEMPOOL_HEADER_SIZE()
- RTE_MEMPOOL_CACHE_MAX_SIZE 512 -> 1024
- RTE_MEMPOOL_MAX_OPS_IDX 16 -> 32, which changes the size of the exported rte_mempool_ops_table
- the new divisible-by-32 requirement on cache_size

The two entries that this patch does implement (flushthresh, oversize objs array) should be deleted from deprecation.rst by the patch that implements them.

9. Mempool header footprint (unchanged from v6)

With local_cache[] inline and RTE_MEMPOOL_CACHE_MAX_SIZE at 1024 the header is roughly 1 MB per mempool, now allocated and memset unconditionally. Previously RTE_MEMPOOL_HEADER_SIZE(mp, 0) omitted the array entirely for mempools created with cache_size == 0.

10. app/test/test_stack.c (unchanged from v6)

The excess-push test is still inside "#if 0 /* FIXME ... */", which removes the check for the standard and lock-free stacks as well.

11. Pile capacity exceeds the requested count (unchanged from v6)

rte_stack_pile_init() creates ceil(count / BULK_SIZE) bulk elements plus count solo elements, so the pile can hold roughly 2 * count objects. __rte_stack_pile_count() hides this with RTE_MIN(s->capacity, ...). rte_stack_push() is documented to fail when there is insufficient space. Either enforce the capacity or document that the pile does not.

12. Missing release notes

doc/guides/rel_notes/release_26_11.rst is still not updated for the new pile stack type, the new "pile" mempool driver, the mempool ABI changes, the new __rte_assume_aligned()/__rte_assume_cache_aligned() macros, or the __rte_restrict deprecation.

13. lib/mempool/mempool_trace.h (unchanged from v6)

Dropping the flushthresh field changes the recorded format of that trace point; worth a release note for consumers parsing trace output.

14. app/test/test_stack_perf.c (unchanged from v6)

MAX_BURST is still derived from RTE_MEMPOOL_CACHE_MAX_SIZE. A stack library test should not take its burst size from a mempool configuration constant.

Info

15. lib/stack/rte_stack_pile.c

	(struct rte_stack_pile_bulk_elem *)(&s->stack_pile.elems)

"&array" where "array" is meant; the cast hides the type mismatch. s->stack_pile.elems (or &s->stack_pile.elems[0]) is clearer.

16. lib/stack/rte_stack_pile.h

Hoisting obj_frag[] to the top of __rte_stack_pile_pop() (for the C23 label/declaration issue) puts 256 bytes at cache-line alignment in the frame of a function that is __rte_always_inline into the mempool dequeue fast path, whether or not fragmentation is reached. A nested block around the fragmentation code would keep it off the fast path while still avoiding a declaration after a label.

17. lib/stack/rte_stack_pile.h, __rte_stack_pile_bulk_pop_elems()

The element chain is still traversed twice: once inside __rte_stack_lf_pop_elems() to find the new head, and again to copy the bulk contents.

18. drivers/mempool/stack/rte_mempool_stack.c

pile_enqueue() returns -ENOBUFS on failure, but rte_mempool_ops_enqueue_bulk() returns void, so a failed put loses objects. The lock-free stack has the same hazard, but the pile draws from two independent free lists, so the window in which a concurrent pop leaves neither able to satisfy a push is wider. Worth documenting in stack_lib.rst.

19. The series still mixes independent changes (the __rte_restrict and x86 rte_memcpy work, the mempool cache/header rework, the pile stack, the pile mempool driver). The mempool cache rework in particular deserves its own thread given the ABI impact.

#10 lore raw

RFC PATCH v8 1/1 pile stack and mempool driver

In reply to Morten Brørup

Early submission of:
- A new "pile" stack-like implementation using the Stack API, and
- an accompanying "pile" mempool driver.
And:
- Some mempool optimizations.
- Marking the object tables in the Stack API as "__rte_restrict".

For CI test and community feedback.

Needless to say, this must be separated into multiple independent series of patches.
And release notes must be added.
For now, I'm submitting a snapshot of work in progress.

The "pile" somewhat resembles the lock-free stack, but operates on bulks (arrays) of objects, to significantly reduce linked list traversal.
With the pile's default bulk size of 32 objects, a mempool cache flush/refill traverses a linked list of only 16 elements, whereas the lock-free stack would traverse a linked list of 512 elements.

Some performance numbers from mempool_perf_autotest_2cores, all with cache=1024 cores=2 n_keep=32768:

start performance test (using ring_mp_mc, with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 753985338 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 755805913

start performance test for lf_stack (with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 29132352 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 29276708

start performance test for pile (with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 560159479 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 557910933

Hat tip to Bruce for bringing attention to the ring not being the optimal mempool driver!

---
v8:
* Fix bulk_last not set in failure path with bulk elements. (AI, advanced model)
* Fix copy-paste typo (underscore not removed) in documentation reference. (CI)
* Fix memsize cache rounding, assign instead of add. (AI, advanced model)
* Fix indentation using spaces. (CI, advanced model)
* Align behavior of rte_mempool_cache_create() and mempool_create(), so they both fail if trying to create too small mempool cache. (AI, advanced model)
* Remove mempool lib constant from stack perf test. (AI, advanced model)
* Add note about why objects are copied from bulk elements by traversing the linked list once more, instead of inside __rte_stack_lf_pop_elems(). (Inspired by AI, advanced model)
* Revert __rte_restrict replacement by __restrict.
It didn't stop noise from checkpatch.
Move to separate discussion.
* Revert x86 rte_memcpy() optimization.
Move to separate patch.
* Push solo objects in reverse order, don't ignore order.
* Refactor stack overflow test case, and enable it again.
* Correct order of version history.
v7:
* Use the "__restrict" keyword, supported by all relevant C/C++ compilers. Degrade "__rte_restrict" to a backwards compatibility macro, document it as deprecated, and check for it in checkpatches.sh.
* Eliminate risk of namespace pollution by prefixing ALIGNMENT_MASK macro in x86 rte_memcpy.h header file.
* Replace conditional code by defining RTE_MEMCPY_BLOCK_64_MAX. (AI)
* Fix off-by-one in assumption when fetching free elements for the excess objects in the fragmentation element. (AI, advanced model)
* Fix memcmp() size in test_stack_push_pop(). (AI, advanced model)
* Do not try fragmentation when we know (from fetching bulk elements above) that no bulk element is available.
(Inspired by AI, advanced model)
* Fix function descriptions mentioning wrong parameter name. (AI, advanced model)
* Fix typo in documentation. (AI, advanced model)
* Mention the pile in the mempool stack documentation.
(AI, advanced model)
* Set the mbuf default pool ops to "pile", for CI test purposes only.
* Select the C11 memory model for x86, for CI test purposes only. v6:
* Add note about roll back of objects in fragmentation element. (AI)
* Move declaration of temporary variable up, to please compilers. (CI)
* Revert mempool cache size adjustments in some drivers;
let the mempool creation function adjust at runtime instead.
v5:
* Remove compiler diagnostic pragmas in stack overflow test case. (Stephen)
* Temporarily remove stack overflow test case until a sufficiently obfuscated variant doesn't trigger a compiler warning about array overrun.
* Revert most changes in rte_stack_std.h, and only change what is necessary.
* Add FIXME in the pile implementation, noting that support for the generic memory model should be removed here too, if removed in the lock-free stack.
(Inspired by Stephen)
v4:
* Revert add __rte_internal.
Compilation fails, and existing stack implementations don't have it.
v3:
* Revert rename unused field in rte_stack_pile_bulk_elem structure. v2:
* Fix indentation, long lines, etc. (checkpatch)
* Fix label followed by a declaration is a C23 extension. (CI)
* Added __rte_internal to pile init and get_memsize. (AI)
* Fix roll back bulk elements in wrong order with fragmentation. (AI)
* Minor changes suggested by AI.

---
 app/test/test_mempool.c                   |   3 +-
 app/test/test_stack.c                     |  91 +++++-
 app/test/test_stack_perf.c                |  15 +-
 config/rte_config.h                       |   7 +-
 config/x86/meson.build                    |   1 +
 doc/guides/mempool/stack.rst              |  12 +-
 doc/guides/prog_guide/stack_lib.rst       |  67 ++++-
 drivers/mempool/stack/rte_mempool_stack.c |  40 +++
 drivers/net/sxe2/sxe2_txrx_vec_avx512.c   |   2 +-
 drivers/net/tap/rte_eth_tap.c             |   2 +-
 lib/eal/include/rte_common.h              |  12 +
 lib/mempool/mempool_trace.h               |   1 -
 lib/mempool/rte_mempool.c                 |  79 +++--
 lib/mempool/rte_mempool.h                 |  76 +++--
 lib/mempool/rte_mempool_ops.c             |   5 +-
 lib/stack/meson.build                     |   3 +-
 lib/stack/rte_stack.c                     |  18 +-
 lib/stack/rte_stack.h                     |  83 +++++-
 lib/stack/rte_stack_lf.h                  |   6 +-
 lib/stack/rte_stack_lf_c11.h              |   2 +-
 lib/stack/rte_stack_lf_generic.h          |   2 +-
 lib/stack/rte_stack_lf_stubs.h            |   2 +-
 lib/stack/rte_stack_pile.c                |  35 +++
 lib/stack/rte_stack_pile.h                | 334 ++++++++++++++++++++++
 lib/stack/rte_stack_std.h                 |  24 +-
 25 files changed, 816 insertions(+), 106 deletions(-)
 create mode 100644 lib/stack/rte_stack_pile.c
 create mode 100644 lib/stack/rte_stack_pile.h
app/test/test_mempool.c +1 -2
index e54249ce61..76d45cea2a 100644
@@ -112,8 +112,7 @@ test_mempool_basic(struct rte_mempool *mp, int use_external_cache)
 		GOTO_ERR(ret, out);
 
 	printf("get private data\n");
-	if (rte_mempool_get_priv(mp) != (char *)mp +
-			RTE_MEMPOOL_HEADER_SIZE(mp, mp->cache_size))
+	if (rte_mempool_get_priv(mp) != (char *)mp + sizeof(struct rte_mempool))
 		GOTO_ERR(ret, out);
 
 #ifndef RTE_EXEC_ENV_FREEBSD /* rte_mem_virt2iova() not supported on bsd */
app/test/test_stack.c +81 -10

119 lines not on this page · raw message

app/test/test_stack_perf.c +13 -2
index 3f17a2606c..e46251d687 100644
@@ -14,14 +14,14 @@
 #include "test.h"
 
 #define STACK_NAME "STACK_PERF"
-#define MAX_BURST 32
+#define MAX_BURST 512
 #define STACK_SIZE (RTE_MAX_LCORE * MAX_BURST)
 
 /*
  * Push/pop bulk sizes, marked volatile so they aren't treated as compile-time
  * constants.
  */
-static volatile unsigned int bulk_sizes[] = {8, MAX_BURST};
+static volatile unsigned int bulk_sizes[] = {1, 8, 32, MAX_BURST};
 
 static RTE_ATOMIC(uint32_t) lcore_barrier;
 
@@ -354,5 +354,16 @@ test_lf_stack_perf(void)
 #endif
 }
 
+static int
+test_pile_perf(void)
+{
+#if defined(RTE_STACK_PILE_SUPPORTED)
+	return __test_stack_perf(RTE_STACK_F_PILE);
+#else
+	return TEST_SKIPPED;
+#endif
+}
+
 REGISTER_PERF_TEST(stack_perf_autotest, test_stack_perf);
 REGISTER_PERF_TEST(stack_lf_perf_autotest, test_lf_stack_perf);
+REGISTER_PERF_TEST(stack_pile_perf_autotest, test_pile_perf);
config/rte_config.h +5 -2
index 0447cdf2ad..6085d2e1a0 100644
@@ -56,14 +56,17 @@
 #define RTE_CONTIGMEM_DEFAULT_BUF_SIZE (512*1024*1024)
 
 /* mempool defines */
-#define RTE_MEMPOOL_CACHE_MAX_SIZE 512
+#define RTE_MEMPOOL_CACHE_MAX_SIZE 1024
 /* RTE_LIBRTE_MEMPOOL_STATS is not set */
 /* RTE_LIBRTE_MEMPOOL_DEBUG is not set */
 
 /* mbuf defines */
-#define RTE_MBUF_DEFAULT_MEMPOOL_OPS "ring_mp_mc"
+#define RTE_MBUF_DEFAULT_MEMPOOL_OPS "pile" /* FIXME: Test only. Default: "ring_mp_mc" */
 /* RTE_MBUF_HISTORY_DEBUG is not set */
 
+/* stack defines */
+#define RTE_STACK_PILE_BULK_SIZE 32
+
 /* ether defines */
 #define RTE_MAX_QUEUES_PER_PORT 1024
 #define RTE_ETHDEV_RXTX_CALLBACKS 1
config/x86/meson.build +1 -0
index 124b204847..28be579556 100644
@@ -49,6 +49,7 @@ else
 endif
 
 dpdk_conf.set('RTE_MAX_NUMA_NODES', 32)
+dpdk_conf.set('RTE_USE_C11_MEM_MODEL', true) # FIXME: Test only.
 
 if is_ms_compiler
     subdir_done()
doc/guides/mempool/stack.rst +11 -1
index 80ea07e65d..c06ab2dc56 100644
@@ -1,5 +1,6 @@
 ..  SPDX-License-Identifier: BSD-3-Clause
     Copyright(c) 2020 Intel Corporation.
+    Copyright(c) 2026 SmartShare Systems.
 
 Stack Mempool Driver
 ====================
@@ -28,6 +29,12 @@ can be selected as described in :ref:`Mempool_Handlers`:
   The underlying **rte_stack** operates in lock-free mode. For more
   information please refer to :ref:`Stack_Library_LF_Stack`.
 
+- ``pile``
+
+  The underlying **rte_stack** operates in lock-free mode,
+  and is optimized for bulks of objects.
+  For more information please refer to :ref:`Stack_Library_Pile`.
+
 The standard stack outperforms the lock-free stack on average, however the
 standard stack is non-preemptive: if a mempool user is preempted while holding
 the stack lock, that thread will block all other mempool accesses until it
@@ -35,9 +42,12 @@ returns and releases the lock. As a result, an application using the standard
 stack whose threads can be preempted can suffer from brief, infrequent
 performance hiccups.
 
-The lock-free stack, by design, is not susceptible to this problem; one thread can
+The lock-free stack and the pile, by design, are not susceptible to this problem; one thread can
 be preempted at any point during a push or pop operation and will not impede
 the progress of any other thread.
 
+The pile is not LIFO per object, but per bulk of objects.
+Although the pile is optimized for bulks of objects, it can handle any request size.
+
 For a more detailed description of the stack implementations, please refer to
 :doc:`/prog_guide/stack_lib`.
doc/guides/prog_guide/stack_lib.rst +63 -4

95 lines not on this page · raw message

drivers/mempool/stack/rte_mempool_stack.c +40 -0
index 1476905227..7467b8b39e 100644
@@ -41,6 +41,36 @@ lf_stack_alloc(struct rte_mempool *mp)
 	return __stack_alloc(mp, RTE_STACK_F_LF);
 }
 
+static int
+pile_alloc(struct rte_mempool *mp)
+{
+	return __stack_alloc(mp, RTE_STACK_F_PILE);
+}
+
+static int
+pile_enqueue(struct rte_mempool *mp, void * const *obj_table,
+	      unsigned int n)
+{
+	struct rte_stack *s = mp->pool_data;
+
+	RTE_ASSERT(s != NULL);
+	RTE_ASSERT(obj_table != NULL);
+
+	return __rte_stack_pile_push(s, obj_table, n) == 0 ? -ENOBUFS : 0;
+}
+
+static int
+pile_dequeue(struct rte_mempool *mp, void **obj_table,
+	      unsigned int n)
+{
+	struct rte_stack *s = mp->pool_data;
+
+	RTE_ASSERT(s != NULL);
+	RTE_ASSERT(obj_table != NULL);
+
+	return __rte_stack_pile_pop(s, obj_table, n) == 0 ? -ENOBUFS : 0;
+}
+
 static int
 stack_enqueue(struct rte_mempool *mp, void * const *obj_table,
 	      unsigned int n)
@@ -93,5 +123,15 @@ static struct rte_mempool_ops ops_lf_stack = {
 	.get_count = stack_get_count
 };
 
+static struct rte_mempool_ops ops_pile = {
+	.name = "pile",
+	.alloc = pile_alloc,
+	.free = stack_free,
+	.enqueue = pile_enqueue,
+	.dequeue = pile_dequeue,
+	.get_count = stack_get_count
+};
+
 RTE_MEMPOOL_REGISTER_OPS(ops_stack);
 RTE_MEMPOOL_REGISTER_OPS(ops_lf_stack);
+RTE_MEMPOOL_REGISTER_OPS(ops_pile);
drivers/net/sxe2/sxe2_txrx_vec_avx512.c +1 -1
index a830c7a33b..2e680f4027 100644
@@ -67,7 +67,7 @@ static __rte_always_inline int32_t sxe2_tx_bufs_free_vec_avx512(struct sxe2_tx_q
 		}
 		cache->len += rs_thresh;
 
-		if (cache->len >= cache->flushthresh) {
+		if (cache->len >= cache->size) {
 			(void)rte_mempool_ops_enqueue_bulk(mp,
 					&cache->objs[cache->size], cache->len - cache->size);
 			cache->len = cache->size;
drivers/net/tap/rte_eth_tap.c +1 -1
index b93452f168..b3142561c2 100644
@@ -61,7 +61,7 @@
 #define TAP_MAX_MAC_ADDRS	16
 #define TAP_GSO_MBUFS_PER_CORE	128
 #define TAP_GSO_MBUF_SEG_SIZE	128
-#define TAP_GSO_MBUF_CACHE_SIZE	4
+#define TAP_GSO_MBUF_CACHE_SIZE	32
 #define TAP_GSO_MBUFS_NUM \
 	(TAP_GSO_MBUFS_PER_CORE * TAP_GSO_MBUF_CACHE_SIZE)
 
lib/eal/include/rte_common.h +12 -0
index 79d2a0ab93..0fd0906506 100644
@@ -567,6 +567,15 @@ static void __attribute__((destructor(RTE_PRIO(prio)), used)) func(void)
 #define __rte_assume(condition) __assume(condition)
 #endif
 
+/**
+ * Alignment hint precondition
+ */
+#ifdef RTE_TOOLCHAIN_MSVC
+#define __rte_assume_aligned(ptr, alignment) (ptr)
+#else
+#define __rte_assume_aligned(ptr, alignment) __builtin_assume_aligned(ptr, alignment)
+#endif
+
 /**
  * Disable AddressSanitizer on some code
  */
@@ -775,6 +784,9 @@ rte_is_aligned(const void * const __rte_restrict ptr, const unsigned int align)
 /** Force minimum cache line alignment. */
 #define __rte_cache_min_aligned __rte_aligned(RTE_CACHE_LINE_MIN_SIZE)
 
+/** Cache alignment hint precondition */
+#define __rte_assume_cache_aligned(ptr) __rte_assume_aligned(ptr, RTE_CACHE_LINE_SIZE)
+
 #define _RTE_CACHE_GUARD_HELPER2(unique) \
 	alignas(RTE_CACHE_LINE_SIZE) \
 	char cache_guard_ ## unique[RTE_CACHE_LINE_SIZE * RTE_CACHE_GUARD_LINES]
lib/mempool/mempool_trace.h +0 -1
index 23cda1473c..60e47cf67b 100644
@@ -119,7 +119,6 @@ RTE_TRACE_POINT(
 	rte_trace_point_emit_i32(socket_id);
 	rte_trace_point_emit_ptr(cache);
 	rte_trace_point_emit_u32(cache->len);
-	rte_trace_point_emit_u32(cache->flushthresh);
 )
 
 RTE_TRACE_POINT(
lib/mempool/rte_mempool.c +57 -22

151 lines not on this page · raw message

lib/mempool/rte_mempool.h +37 -39

157 lines not on this page · raw message

lib/mempool/rte_mempool_ops.c +4 -1
index 066bec36fc..44fa3a0687 100644
@@ -185,8 +185,11 @@ rte_mempool_set_ops_byname(struct rte_mempool *mp, const char *name,
 		}
 	}
 
-	if (ops == NULL)
+	if (ops == NULL) {
+		RTE_MEMPOOL_LOG(ERR,
+			"Unknown mempool_ops <%s>", name);
 		return -EINVAL;
+	}
 
 	mp->ops_index = i;
 	mp->pool_config = pool_config;
lib/stack/meson.build +2 -1
index 18177a742f..50e688522e 100644
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2019 Intel Corporation
 
-sources = files('rte_stack.c', 'rte_stack_std.c', 'rte_stack_lf.c')
+sources = files('rte_stack.c', 'rte_stack_std.c', 'rte_stack_lf.c', 'rte_stack_pile.c')
 headers = files('rte_stack.h')
 # subheaders, not for direct inclusion by apps
 indirect_headers += files(
@@ -10,4 +10,5 @@ indirect_headers += files(
         'rte_stack_lf_generic.h',
         'rte_stack_lf_c11.h',
         'rte_stack_lf_stubs.h',
+        'rte_stack_pile.h',
 )
lib/stack/rte_stack.c +17 -1
index 4c78fe4b4b..a4bbf8a4d7 100644
@@ -1,5 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright(c) 2019 Intel Corporation
+ * Copyright(c) 2026 SmartShare Systems
  */
 
 #include <stdalign.h>
@@ -32,6 +33,8 @@ rte_stack_init(struct rte_stack *s, unsigned int count, uint32_t flags)
 
 	if (flags & RTE_STACK_F_LF)
 		rte_stack_lf_init(s, count);
+	else if (flags & RTE_STACK_F_PILE)
+		rte_stack_pile_init(s, count);
 	else
 		rte_stack_std_init(s);
 }
@@ -41,6 +44,8 @@ rte_stack_get_memsize(unsigned int count, uint32_t flags)
 {
 	if (flags & RTE_STACK_F_LF)
 		return rte_stack_lf_get_memsize(count);
+	else if (flags & RTE_STACK_F_PILE)
+		return rte_stack_pile_get_memsize(count);
 	else
 		return rte_stack_std_get_memsize(count);
 }
@@ -58,7 +63,11 @@ rte_stack_create(const char *name, unsigned int count, int socket_id,
 	unsigned int sz;
 	int ret;
 
-	if (flags & ~(RTE_STACK_F_LF)) {
+	if (flags & ~(RTE_STACK_F_LF | RTE_STACK_F_PILE)) {
+		STACK_LOG_ERR("Unsupported stack flags %#x", flags);
+		return NULL;
+	}
+	if ((flags & RTE_STACK_F_LF) && (flags & RTE_STACK_F_PILE)) {
 		STACK_LOG_ERR("Unsupported stack flags %#x", flags);
 		return NULL;
 	}
@@ -73,6 +82,13 @@ rte_stack_create(const char *name, unsigned int count, int socket_id,
 		return NULL;
 	}
 #endif
+#if !defined(RTE_STACK_PILE_SUPPORTED)
+	if (flags & RTE_STACK_F_PILE) {
+		STACK_LOG_ERR("Pile is not supported on your platform");
+		rte_errno = ENOTSUP;
+		return NULL;
+	}
+#endif
 
 	sz = rte_stack_get_memsize(count, flags);
 
lib/stack/rte_stack.h +81 -2

155 lines not on this page · raw message

lib/stack/rte_stack_lf.h +4 -2
index f2b012cd0e..655620aaa9 100644
@@ -34,7 +34,7 @@
  */
 static __rte_always_inline unsigned int
 __rte_stack_lf_push(struct rte_stack *s,
-		    void * const *obj_table,
+		    void * const * __rte_restrict obj_table,
 		    unsigned int n)
 {
 	struct rte_stack_lf_elem *tmp, *first, *last = NULL;
@@ -71,7 +71,8 @@ __rte_stack_lf_push(struct rte_stack *s,
  *   - Actual number of objects popped.
  */
 static __rte_always_inline unsigned int
-__rte_stack_lf_pop(struct rte_stack *s, void **obj_table, unsigned int n)
+__rte_stack_lf_pop(struct rte_stack *s, void ** __rte_restrict obj_table,
+		   unsigned int n)
 {
 	struct rte_stack_lf_elem *first, *last = NULL;
 
@@ -79,6 +80,7 @@ __rte_stack_lf_pop(struct rte_stack *s, void **obj_table, unsigned int n)
 		return 0;
 
 	/* Pop n used elements */
+	__rte_assume(obj_table != NULL);
 	first = __rte_stack_lf_pop_elems(&s->stack_lf.used,
 					 n, obj_table, &last);
 	if (unlikely(first == NULL))
lib/stack/rte_stack_lf_c11.h +1 -1
index b97e02d6a1..501e985a49 100644
@@ -99,7 +99,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 static __rte_always_inline struct rte_stack_lf_elem *
 __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 unsigned int num,
-			 void **obj_table,
+			 void ** __rte_restrict obj_table,
 			 struct rte_stack_lf_elem **last)
 {
 	struct rte_stack_lf_head old_head;
lib/stack/rte_stack_lf_generic.h +1 -1
index cc69e4d168..c4cc4d2c03 100644
@@ -74,7 +74,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 static __rte_always_inline struct rte_stack_lf_elem *
 __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 unsigned int num,
-			 void **obj_table,
+			 void ** __rte_restrict obj_table,
 			 struct rte_stack_lf_elem **last)
 {
 	struct rte_stack_lf_head old_head;
lib/stack/rte_stack_lf_stubs.h +1 -1
index a05abf1f1c..457a415ccf 100644
@@ -30,7 +30,7 @@ __rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
 static __rte_always_inline struct rte_stack_lf_elem *
 __rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
 			 unsigned int num,
-			 void **obj_table,
+			 void ** __rte_restrict obj_table,
 			 struct rte_stack_lf_elem **last)
 {
 	RTE_SET_USED(obj_table);
lib/stack/rte_stack_pile.c new file +35 -0
new file mode 100644
index 0000000000..5884163313
@@ -0,0 +1,35 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2026 SmartShare Systems
+ */
+
+#include "rte_stack.h"
+
+void
+rte_stack_pile_init(struct rte_stack *s, unsigned int count)
+{
+	unsigned int bulk = (count + RTE_STACK_PILE_BULK_SIZE - 1) / RTE_STACK_PILE_BULK_SIZE;
+	struct rte_stack_pile_bulk_elem *bulk_elems =
+			(struct rte_stack_pile_bulk_elem *)(s->stack_pile.elems);
+	struct rte_stack_lf_elem *solo_elems = (struct rte_stack_lf_elem *)&bulk_elems[bulk];
+	unsigned int i;
+
+	for (i = 0; i < bulk; i++)
+		__rte_stack_pile_bulk_push_elems(&s->stack_pile.free_bulk,
+					  &bulk_elems[i], &bulk_elems[i], 1);
+	for (i = 0; i < count; i++)
+		__rte_stack_lf_push_elems(&s->stack_pile.free_solo,
+					  &solo_elems[i], &solo_elems[i], 1);
+}
+
+ssize_t
+rte_stack_pile_get_memsize(unsigned int count)
+{
+	unsigned int bulk = (count + RTE_STACK_PILE_BULK_SIZE - 1) / RTE_STACK_PILE_BULK_SIZE;
+	ssize_t sz = offsetof(struct rte_stack, stack_pile.elems);
+	sz += bulk * sizeof(struct rte_stack_pile_bulk_elem);
+	sz += count * sizeof(struct rte_stack_lf_elem);
+	sz = RTE_CACHE_LINE_ROUNDUP(sz);
+	sz += RTE_CACHE_GUARD_LINES * RTE_CACHE_LINE_SIZE;
+
+	return sz;
+}
lib/stack/rte_stack_pile.h new file +334 -0

334 lines not on this page · raw message

lib/stack/rte_stack_std.h +13 -11
index ae28add5c4..e256cf8d83 100644
@@ -20,25 +20,25 @@
  *   Actual number of objects pushed (either 0 or *n*).
  */
 static __rte_always_inline unsigned int
-__rte_stack_std_push(struct rte_stack *s, void * const *obj_table,
+__rte_stack_std_push(struct rte_stack *s, void * const * __rte_restrict obj_table,
 		     unsigned int n)
 {
 	struct rte_stack_std *stack = &s->stack_std;
 	unsigned int index;
-	void **cache_objs;
+	void ** __rte_restrict stack_objs;
 
 	rte_spinlock_lock(&stack->lock);
-	cache_objs = &stack->objs[stack->len];
+	stack_objs = &stack->objs[stack->len];
 
-	/* Is there sufficient space in the stack? */
-	if ((stack->len + n) > s->capacity) {
+	if (unlikely((stack->len + n) > s->capacity)) {
+		/* Insufficient space in the stack. */
 		rte_spinlock_unlock(&stack->lock);
 		return 0;
 	}
 
-	/* Add elements back into the cache */
+	/* Push objects to the stack */
 	for (index = 0; index < n; ++index, obj_table++)
-		cache_objs[index] = *obj_table;
+		stack_objs[index] = *obj_table;
 
 	stack->len += n;
 
@@ -59,24 +59,26 @@ __rte_stack_std_push(struct rte_stack *s, void * const *obj_table,
  *   Actual number of objects popped (either 0 or *n*).
  */
 static __rte_always_inline unsigned int
-__rte_stack_std_pop(struct rte_stack *s, void **obj_table, unsigned int n)
+__rte_stack_std_pop(struct rte_stack *s, void ** __rte_restrict obj_table, unsigned int n)
 {
 	struct rte_stack_std *stack = &s->stack_std;
 	unsigned int index, len;
-	void **cache_objs;
+	void ** __rte_restrict stack_objs;
 
 	rte_spinlock_lock(&stack->lock);
 
 	if (unlikely(n > stack->len)) {
+		/* Insufficient objects in the stack. */
 		rte_spinlock_unlock(&stack->lock);
 		return 0;
 	}
 
-	cache_objs = stack->objs;
+	stack_objs = stack->objs;
 
+	/* Pop objects from the stack */
 	for (index = 0, len = stack->len - 1; index < n;
 			++index, len--, obj_table++)
-		*obj_table = cache_objs[len];
+		*obj_table = stack_objs[len];
 
 	stack->len -= n;
 	rte_spinlock_unlock(&stack->lock);
Signature · 1 line
2.43.0
#11 lore raw

RFC PATCH v9 1/1 pile stack and mempool driver

In reply to Morten Brørup

Early submission of:
- A new "pile" stack-like implementation using the Stack API, and
- an accompanying "pile" mempool driver.
And:
- Some mempool optimizations.

For CI test and community feedback.

Needless to say, this must be separated into multiple independent series of patches.
And release notes must be added.
For now, I'm submitting a snapshot of work in progress.

The "pile" somewhat resembles the lock-free stack, but operates on bulks (arrays) of objects, to significantly reduce linked list traversal.
With the pile's default bulk size of 32 objects, a mempool cache flush/refill traverses a linked list of only 16 elements, whereas the lock-free stack would traverse a linked list of 512 elements.

Some performance numbers from mempool_perf_autotest_2cores, all with cache=1024 cores=2 n_keep=32768:

start performance test (using ring_mp_mc, with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 753985338 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 755805913

start performance test for lf_stack (with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 29132352 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 29276708

start performance test for pile (with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 560159479 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 557910933

Hat tip to Bruce for bringing attention to the ring not being the optimal mempool driver!

---
v9:
* Fix long lines. (checkpatch)
* Log more, to debug why some Github CI tests cannot find the "pile" mempool driver.
* Revert addition of __rte_restrict.
Move to separate discussion. Could be added to other libs too.
v8:
* Fix bulk_last not set in failure path with bulk elements. (AI, advanced model)
* Fix copy-paste typo (underscore not removed) in documentation reference. (CI)
* Fix memsize cache rounding, assign instead of add. (AI, advanced model)
* Fix indentation using spaces. (CI, advanced model)
* Align behavior of rte_mempool_cache_create() and mempool_create(), so they both fail if trying to create too small mempool cache. (AI, advanced model)
* Remove mempool lib constant from stack perf test. (AI, advanced model)
* Add note about why objects are copied from bulk elements by traversing the linked list once more, instead of inside __rte_stack_lf_pop_elems(). (Inspired by AI, advanced model)
* Revert __rte_restrict replacement by __restrict.
It didn't stop noise from checkpatch.
Move to separate discussion.
* Revert x86 rte_memcpy() optimization.
Move to separate patch.
* Push solo objects in reverse order, don't ignore order.
* Refactor stack overflow test case, and enable it again.
* Correct order of version history.
v7:
* Use the "__restrict" keyword, supported by all relevant C/C++ compilers. Degrade "__rte_restrict" to a backwards compatibility macro, document it as deprecated, and check for it in checkpatches.sh.
* Eliminate risk of namespace pollution by prefixing ALIGNMENT_MASK macro in x86 rte_memcpy.h header file.
* Replace conditional code by defining RTE_MEMCPY_BLOCK_64_MAX. (AI)
* Fix off-by-one in assumption when fetching free elements for the excess objects in the fragmentation element. (AI, advanced model)
* Fix memcmp() size in test_stack_push_pop(). (AI, advanced model)
* Do not try fragmentation when we know (from fetching bulk elements above) that no bulk element is available.
(Inspired by AI, advanced model)
* Fix function descriptions mentioning wrong parameter name. (AI, advanced model)
* Fix typo in documentation. (AI, advanced model)
* Mention the pile in the mempool stack documentation.
(AI, advanced model)
* Set the mbuf default pool ops to "pile", for CI test purposes only.
* Select the C11 memory model for x86, for CI test purposes only. v6:
* Add note about roll back of objects in fragmentation element. (AI)
* Move declaration of temporary variable up, to please compilers. (CI)
* Revert mempool cache size adjustments in some drivers;
let the mempool creation function adjust at runtime instead.
v5:
* Remove compiler diagnostic pragmas in stack overflow test case. (Stephen)
* Temporarily remove stack overflow test case until a sufficiently obfuscated variant doesn't trigger a compiler warning about array overrun.
* Revert most changes in rte_stack_std.h, and only change what is necessary.
* Add FIXME in the pile implementation, noting that support for the generic memory model should be removed here too, if removed in the lock-free stack.
(Inspired by Stephen)
v4:
* Revert add __rte_internal.
Compilation fails, and existing stack implementations don't have it.
v3:
* Revert rename unused field in rte_stack_pile_bulk_elem structure. v2:
* Fix indentation, long lines, etc. (checkpatch)
* Fix label followed by a declaration is a C23 extension. (CI)
* Added __rte_internal to pile init and get_memsize. (AI)
* Fix roll back bulk elements in wrong order with fragmentation. (AI)
* Minor changes suggested by AI.

---
 app/test-pmd/testpmd.c                    |   4 +-
 app/test/test_mempool.c                   |   3 +-
 app/test/test_stack.c                     |  93 +++++-
 app/test/test_stack_perf.c                |  15 +-
 config/rte_config.h                       |   7 +-
 config/x86/meson.build                    |   1 +
 doc/guides/mempool/stack.rst              |  12 +-
 doc/guides/prog_guide/stack_lib.rst       |  67 ++++-
 drivers/mempool/stack/rte_mempool_stack.c |  40 +++
 drivers/net/sxe2/sxe2_txrx_vec_avx512.c   |   2 +-
 drivers/net/tap/rte_eth_tap.c             |   2 +-
 lib/eal/include/rte_common.h              |  12 +
 lib/mempool/mempool_trace.h               |   1 -
 lib/mempool/rte_mempool.c                 |  79 +++--
 lib/mempool/rte_mempool.h                 |  78 ++---
 lib/mempool/rte_mempool_ops.c             |  27 +-
 lib/stack/meson.build                     |   3 +-
 lib/stack/rte_stack.c                     |  18 +-
 lib/stack/rte_stack.h                     |  79 +++++
 lib/stack/rte_stack_lf.h                  |   1 +
 lib/stack/rte_stack_pile.c                |  35 +++
 lib/stack/rte_stack_pile.h                | 334 ++++++++++++++++++++++
 lib/stack/rte_stack_std.h                 |  20 +-
 23 files changed, 829 insertions(+), 104 deletions(-)
 create mode 100644 lib/stack/rte_stack_pile.c
 create mode 100644 lib/stack/rte_stack_pile.h
app/test-pmd/testpmd.c +2 -2
index cab2fa1556..9e4ffd28d3 100644
@@ -1283,8 +1283,8 @@ mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 	}
 
 	TESTPMD_LOG(INFO,
-		"create a new mbuf pool <%s>: n=%u, size=%u, socket=%s\n",
-		pool_name, nb_mbuf, mbuf_seg_size,
+		"create a new mbuf pool <%s>: n=%u, cache=%u, size=%u, socket=%s\n",
+		pool_name, nb_mbuf, mb_mempool_cache, mbuf_seg_size,
 		socket_id_str(socket_id, sock_str, sizeof(sock_str)));
 
 	switch (mp_alloc_type) {
app/test/test_mempool.c +1 -2
index e54249ce61..76d45cea2a 100644
@@ -112,8 +112,7 @@ test_mempool_basic(struct rte_mempool *mp, int use_external_cache)
 		GOTO_ERR(ret, out);
 
 	printf("get private data\n");
-	if (rte_mempool_get_priv(mp) != (char *)mp +
-			RTE_MEMPOOL_HEADER_SIZE(mp, mp->cache_size))
+	if (rte_mempool_get_priv(mp) != (char *)mp + sizeof(struct rte_mempool))
 		GOTO_ERR(ret, out);
 
 #ifndef RTE_EXEC_ENV_FREEBSD /* rte_mem_virt2iova() not supported on bsd */
app/test/test_stack.c +83 -10

121 lines not on this page · raw message

app/test/test_stack_perf.c +13 -2
index 3f17a2606c..e46251d687 100644
@@ -14,14 +14,14 @@
 #include "test.h"
 
 #define STACK_NAME "STACK_PERF"
-#define MAX_BURST 32
+#define MAX_BURST 512
 #define STACK_SIZE (RTE_MAX_LCORE * MAX_BURST)
 
 /*
  * Push/pop bulk sizes, marked volatile so they aren't treated as compile-time
  * constants.
  */
-static volatile unsigned int bulk_sizes[] = {8, MAX_BURST};
+static volatile unsigned int bulk_sizes[] = {1, 8, 32, MAX_BURST};
 
 static RTE_ATOMIC(uint32_t) lcore_barrier;
 
@@ -354,5 +354,16 @@ test_lf_stack_perf(void)
 #endif
 }
 
+static int
+test_pile_perf(void)
+{
+#if defined(RTE_STACK_PILE_SUPPORTED)
+	return __test_stack_perf(RTE_STACK_F_PILE);
+#else
+	return TEST_SKIPPED;
+#endif
+}
+
 REGISTER_PERF_TEST(stack_perf_autotest, test_stack_perf);
 REGISTER_PERF_TEST(stack_lf_perf_autotest, test_lf_stack_perf);
+REGISTER_PERF_TEST(stack_pile_perf_autotest, test_pile_perf);
config/rte_config.h +5 -2
index 0447cdf2ad..6085d2e1a0 100644
@@ -56,14 +56,17 @@
 #define RTE_CONTIGMEM_DEFAULT_BUF_SIZE (512*1024*1024)
 
 /* mempool defines */
-#define RTE_MEMPOOL_CACHE_MAX_SIZE 512
+#define RTE_MEMPOOL_CACHE_MAX_SIZE 1024
 /* RTE_LIBRTE_MEMPOOL_STATS is not set */
 /* RTE_LIBRTE_MEMPOOL_DEBUG is not set */
 
 /* mbuf defines */
-#define RTE_MBUF_DEFAULT_MEMPOOL_OPS "ring_mp_mc"
+#define RTE_MBUF_DEFAULT_MEMPOOL_OPS "pile" /* FIXME: Test only. Default: "ring_mp_mc" */
 /* RTE_MBUF_HISTORY_DEBUG is not set */
 
+/* stack defines */
+#define RTE_STACK_PILE_BULK_SIZE 32
+
 /* ether defines */
 #define RTE_MAX_QUEUES_PER_PORT 1024
 #define RTE_ETHDEV_RXTX_CALLBACKS 1
config/x86/meson.build +1 -0
index 124b204847..28be579556 100644
@@ -49,6 +49,7 @@ else
 endif
 
 dpdk_conf.set('RTE_MAX_NUMA_NODES', 32)
+dpdk_conf.set('RTE_USE_C11_MEM_MODEL', true) # FIXME: Test only.
 
 if is_ms_compiler
     subdir_done()
doc/guides/mempool/stack.rst +11 -1
index 80ea07e65d..c06ab2dc56 100644
@@ -1,5 +1,6 @@
 ..  SPDX-License-Identifier: BSD-3-Clause
     Copyright(c) 2020 Intel Corporation.
+    Copyright(c) 2026 SmartShare Systems.
 
 Stack Mempool Driver
 ====================
@@ -28,6 +29,12 @@ can be selected as described in :ref:`Mempool_Handlers`:
   The underlying **rte_stack** operates in lock-free mode. For more
   information please refer to :ref:`Stack_Library_LF_Stack`.
 
+- ``pile``
+
+  The underlying **rte_stack** operates in lock-free mode,
+  and is optimized for bulks of objects.
+  For more information please refer to :ref:`Stack_Library_Pile`.
+
 The standard stack outperforms the lock-free stack on average, however the
 standard stack is non-preemptive: if a mempool user is preempted while holding
 the stack lock, that thread will block all other mempool accesses until it
@@ -35,9 +42,12 @@ returns and releases the lock. As a result, an application using the standard
 stack whose threads can be preempted can suffer from brief, infrequent
 performance hiccups.
 
-The lock-free stack, by design, is not susceptible to this problem; one thread can
+The lock-free stack and the pile, by design, are not susceptible to this problem; one thread can
 be preempted at any point during a push or pop operation and will not impede
 the progress of any other thread.
 
+The pile is not LIFO per object, but per bulk of objects.
+Although the pile is optimized for bulks of objects, it can handle any request size.
+
 For a more detailed description of the stack implementations, please refer to
 :doc:`/prog_guide/stack_lib`.
doc/guides/prog_guide/stack_lib.rst +63 -4

95 lines not on this page · raw message

drivers/mempool/stack/rte_mempool_stack.c +40 -0
index 1476905227..7467b8b39e 100644
@@ -41,6 +41,36 @@ lf_stack_alloc(struct rte_mempool *mp)
 	return __stack_alloc(mp, RTE_STACK_F_LF);
 }
 
+static int
+pile_alloc(struct rte_mempool *mp)
+{
+	return __stack_alloc(mp, RTE_STACK_F_PILE);
+}
+
+static int
+pile_enqueue(struct rte_mempool *mp, void * const *obj_table,
+	      unsigned int n)
+{
+	struct rte_stack *s = mp->pool_data;
+
+	RTE_ASSERT(s != NULL);
+	RTE_ASSERT(obj_table != NULL);
+
+	return __rte_stack_pile_push(s, obj_table, n) == 0 ? -ENOBUFS : 0;
+}
+
+static int
+pile_dequeue(struct rte_mempool *mp, void **obj_table,
+	      unsigned int n)
+{
+	struct rte_stack *s = mp->pool_data;
+
+	RTE_ASSERT(s != NULL);
+	RTE_ASSERT(obj_table != NULL);
+
+	return __rte_stack_pile_pop(s, obj_table, n) == 0 ? -ENOBUFS : 0;
+}
+
 static int
 stack_enqueue(struct rte_mempool *mp, void * const *obj_table,
 	      unsigned int n)
@@ -93,5 +123,15 @@ static struct rte_mempool_ops ops_lf_stack = {
 	.get_count = stack_get_count
 };
 
+static struct rte_mempool_ops ops_pile = {
+	.name = "pile",
+	.alloc = pile_alloc,
+	.free = stack_free,
+	.enqueue = pile_enqueue,
+	.dequeue = pile_dequeue,
+	.get_count = stack_get_count
+};
+
 RTE_MEMPOOL_REGISTER_OPS(ops_stack);
 RTE_MEMPOOL_REGISTER_OPS(ops_lf_stack);
+RTE_MEMPOOL_REGISTER_OPS(ops_pile);
drivers/net/sxe2/sxe2_txrx_vec_avx512.c +1 -1
index a830c7a33b..2e680f4027 100644
@@ -67,7 +67,7 @@ static __rte_always_inline int32_t sxe2_tx_bufs_free_vec_avx512(struct sxe2_tx_q
 		}
 		cache->len += rs_thresh;
 
-		if (cache->len >= cache->flushthresh) {
+		if (cache->len >= cache->size) {
 			(void)rte_mempool_ops_enqueue_bulk(mp,
 					&cache->objs[cache->size], cache->len - cache->size);
 			cache->len = cache->size;
drivers/net/tap/rte_eth_tap.c +1 -1
index b93452f168..b3142561c2 100644
@@ -61,7 +61,7 @@
 #define TAP_MAX_MAC_ADDRS	16
 #define TAP_GSO_MBUFS_PER_CORE	128
 #define TAP_GSO_MBUF_SEG_SIZE	128
-#define TAP_GSO_MBUF_CACHE_SIZE	4
+#define TAP_GSO_MBUF_CACHE_SIZE	32
 #define TAP_GSO_MBUFS_NUM \
 	(TAP_GSO_MBUFS_PER_CORE * TAP_GSO_MBUF_CACHE_SIZE)
 
lib/eal/include/rte_common.h +12 -0
index 79d2a0ab93..0fd0906506 100644
@@ -567,6 +567,15 @@ static void __attribute__((destructor(RTE_PRIO(prio)), used)) func(void)
 #define __rte_assume(condition) __assume(condition)
 #endif
 
+/**
+ * Alignment hint precondition
+ */
+#ifdef RTE_TOOLCHAIN_MSVC
+#define __rte_assume_aligned(ptr, alignment) (ptr)
+#else
+#define __rte_assume_aligned(ptr, alignment) __builtin_assume_aligned(ptr, alignment)
+#endif
+
 /**
  * Disable AddressSanitizer on some code
  */
@@ -775,6 +784,9 @@ rte_is_aligned(const void * const __rte_restrict ptr, const unsigned int align)
 /** Force minimum cache line alignment. */
 #define __rte_cache_min_aligned __rte_aligned(RTE_CACHE_LINE_MIN_SIZE)
 
+/** Cache alignment hint precondition */
+#define __rte_assume_cache_aligned(ptr) __rte_assume_aligned(ptr, RTE_CACHE_LINE_SIZE)
+
 #define _RTE_CACHE_GUARD_HELPER2(unique) \
 	alignas(RTE_CACHE_LINE_SIZE) \
 	char cache_guard_ ## unique[RTE_CACHE_LINE_SIZE * RTE_CACHE_GUARD_LINES]
lib/mempool/mempool_trace.h +0 -1
index 23cda1473c..60e47cf67b 100644
@@ -119,7 +119,6 @@ RTE_TRACE_POINT(
 	rte_trace_point_emit_i32(socket_id);
 	rte_trace_point_emit_ptr(cache);
 	rte_trace_point_emit_u32(cache->len);
-	rte_trace_point_emit_u32(cache->flushthresh);
 )
 
 RTE_TRACE_POINT(
lib/mempool/rte_mempool.c +57 -22

151 lines not on this page · raw message

lib/mempool/rte_mempool.h +39 -39

165 lines not on this page · raw message

lib/mempool/rte_mempool_ops.c +21 -6
index 066bec36fc..49665812b1 100644
@@ -27,7 +27,7 @@ int
 rte_mempool_register_ops(const struct rte_mempool_ops *h)
 {
 	struct rte_mempool_ops *ops;
-	int16_t ops_index;
+	unsigned int ops_index;
 
 	rte_spinlock_lock(&rte_mempool_ops_table.sl);
 
@@ -47,12 +47,20 @@ rte_mempool_register_ops(const struct rte_mempool_ops *h)
 		return -EINVAL;
 	}
 
-	if (strlen(h->name) >= sizeof(ops->name) - 1) {
+	if (strlen(h->name) > sizeof(ops->name) - 1) {
 		rte_spinlock_unlock(&rte_mempool_ops_table.sl);
-		RTE_MEMPOOL_LOG(DEBUG, "%s(): mempool_ops <%s>: name too long",
+		RTE_MEMPOOL_LOG(ERR, "%s(): mempool_ops <%s>: name too long",
 				__func__, h->name);
-		rte_errno = EEXIST;
-		return -EEXIST;
+		return -ENAMETOOLONG;
+	}
+
+	for (ops_index = 0; ops_index < rte_mempool_ops_table.num_ops; ops_index++) {
+		if (!strcmp(h->name,
+				rte_mempool_ops_table.ops[ops_index].name)) {
+			RTE_MEMPOOL_LOG(ERR, "%s(): mempool_ops <%s>: name exists",
+					__func__, h->name);
+			return -EEXIST;
+		}
 	}
 
 	ops_index = rte_mempool_ops_table.num_ops++;
@@ -68,6 +76,10 @@ rte_mempool_register_ops(const struct rte_mempool_ops *h)
 	ops->get_info = h->get_info;
 	ops->dequeue_contig_blocks = h->dequeue_contig_blocks;
 
+	/* FIXME: Test only. Reduce to DEBUG level. */
+	RTE_MEMPOOL_LOG(INFO,
+		"Registered mempool_ops[%u] <%s>", ops_index, h->name);
+
 	rte_spinlock_unlock(&rte_mempool_ops_table.sl);
 
 	return ops_index;
@@ -185,8 +197,11 @@ rte_mempool_set_ops_byname(struct rte_mempool *mp, const char *name,
 		}
 	}
 
-	if (ops == NULL)
+	if (ops == NULL) {
+		RTE_MEMPOOL_LOG(ERR,
+			"Unknown mempool_ops <%s>, %u ops searched", name, i);
 		return -EINVAL;
+	}
 
 	mp->ops_index = i;
 	mp->pool_config = pool_config;
lib/stack/meson.build +2 -1
index 18177a742f..50e688522e 100644
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2019 Intel Corporation
 
-sources = files('rte_stack.c', 'rte_stack_std.c', 'rte_stack_lf.c')
+sources = files('rte_stack.c', 'rte_stack_std.c', 'rte_stack_lf.c', 'rte_stack_pile.c')
 headers = files('rte_stack.h')
 # subheaders, not for direct inclusion by apps
 indirect_headers += files(
@@ -10,4 +10,5 @@ indirect_headers += files(
         'rte_stack_lf_generic.h',
         'rte_stack_lf_c11.h',
         'rte_stack_lf_stubs.h',
+        'rte_stack_pile.h',
 )
lib/stack/rte_stack.c +17 -1
index 4c78fe4b4b..a4bbf8a4d7 100644
@@ -1,5 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright(c) 2019 Intel Corporation
+ * Copyright(c) 2026 SmartShare Systems
  */
 
 #include <stdalign.h>
@@ -32,6 +33,8 @@ rte_stack_init(struct rte_stack *s, unsigned int count, uint32_t flags)
 
 	if (flags & RTE_STACK_F_LF)
 		rte_stack_lf_init(s, count);
+	else if (flags & RTE_STACK_F_PILE)
+		rte_stack_pile_init(s, count);
 	else
 		rte_stack_std_init(s);
 }
@@ -41,6 +44,8 @@ rte_stack_get_memsize(unsigned int count, uint32_t flags)
 {
 	if (flags & RTE_STACK_F_LF)
 		return rte_stack_lf_get_memsize(count);
+	else if (flags & RTE_STACK_F_PILE)
+		return rte_stack_pile_get_memsize(count);
 	else
 		return rte_stack_std_get_memsize(count);
 }
@@ -58,7 +63,11 @@ rte_stack_create(const char *name, unsigned int count, int socket_id,
 	unsigned int sz;
 	int ret;
 
-	if (flags & ~(RTE_STACK_F_LF)) {
+	if (flags & ~(RTE_STACK_F_LF | RTE_STACK_F_PILE)) {
+		STACK_LOG_ERR("Unsupported stack flags %#x", flags);
+		return NULL;
+	}
+	if ((flags & RTE_STACK_F_LF) && (flags & RTE_STACK_F_PILE)) {
 		STACK_LOG_ERR("Unsupported stack flags %#x", flags);
 		return NULL;
 	}
@@ -73,6 +82,13 @@ rte_stack_create(const char *name, unsigned int count, int socket_id,
 		return NULL;
 	}
 #endif
+#if !defined(RTE_STACK_PILE_SUPPORTED)
+	if (flags & RTE_STACK_F_PILE) {
+		STACK_LOG_ERR("Pile is not supported on your platform");
+		rte_errno = ENOTSUP;
+		return NULL;
+	}
+#endif
 
 	sz = rte_stack_get_memsize(count, flags);
 
lib/stack/rte_stack.h +79 -0

139 lines not on this page · raw message

lib/stack/rte_stack_lf.h +1 -0
index f2b012cd0e..1ee9330c57 100644
@@ -79,6 +79,7 @@ __rte_stack_lf_pop(struct rte_stack *s, void **obj_table, unsigned int n)
 		return 0;
 
 	/* Pop n used elements */
+	__rte_assume(obj_table != NULL);
 	first = __rte_stack_lf_pop_elems(&s->stack_lf.used,
 					 n, obj_table, &last);
 	if (unlikely(first == NULL))
lib/stack/rte_stack_pile.c new file +35 -0
new file mode 100644
index 0000000000..5884163313
@@ -0,0 +1,35 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2026 SmartShare Systems
+ */
+
+#include "rte_stack.h"
+
+void
+rte_stack_pile_init(struct rte_stack *s, unsigned int count)
+{
+	unsigned int bulk = (count + RTE_STACK_PILE_BULK_SIZE - 1) / RTE_STACK_PILE_BULK_SIZE;
+	struct rte_stack_pile_bulk_elem *bulk_elems =
+			(struct rte_stack_pile_bulk_elem *)(s->stack_pile.elems);
+	struct rte_stack_lf_elem *solo_elems = (struct rte_stack_lf_elem *)&bulk_elems[bulk];
+	unsigned int i;
+
+	for (i = 0; i < bulk; i++)
+		__rte_stack_pile_bulk_push_elems(&s->stack_pile.free_bulk,
+					  &bulk_elems[i], &bulk_elems[i], 1);
+	for (i = 0; i < count; i++)
+		__rte_stack_lf_push_elems(&s->stack_pile.free_solo,
+					  &solo_elems[i], &solo_elems[i], 1);
+}
+
+ssize_t
+rte_stack_pile_get_memsize(unsigned int count)
+{
+	unsigned int bulk = (count + RTE_STACK_PILE_BULK_SIZE - 1) / RTE_STACK_PILE_BULK_SIZE;
+	ssize_t sz = offsetof(struct rte_stack, stack_pile.elems);
+	sz += bulk * sizeof(struct rte_stack_pile_bulk_elem);
+	sz += count * sizeof(struct rte_stack_lf_elem);
+	sz = RTE_CACHE_LINE_ROUNDUP(sz);
+	sz += RTE_CACHE_GUARD_LINES * RTE_CACHE_LINE_SIZE;
+
+	return sz;
+}
lib/stack/rte_stack_pile.h new file +334 -0

334 lines not on this page · raw message

lib/stack/rte_stack_std.h +11 -9
index ae28add5c4..d5b21defb0 100644
@@ -25,20 +25,20 @@ __rte_stack_std_push(struct rte_stack *s, void * const *obj_table,
 {
 	struct rte_stack_std *stack = &s->stack_std;
 	unsigned int index;
-	void **cache_objs;
+	void **stack_objs;
 
 	rte_spinlock_lock(&stack->lock);
-	cache_objs = &stack->objs[stack->len];
+	stack_objs = &stack->objs[stack->len];
 
-	/* Is there sufficient space in the stack? */
-	if ((stack->len + n) > s->capacity) {
+	if (unlikely((stack->len + n) > s->capacity)) {
+		/* Insufficient space in the stack. */
 		rte_spinlock_unlock(&stack->lock);
 		return 0;
 	}
 
-	/* Add elements back into the cache */
+	/* Push objects to the stack */
 	for (index = 0; index < n; ++index, obj_table++)
-		cache_objs[index] = *obj_table;
+		stack_objs[index] = *obj_table;
 
 	stack->len += n;
 
@@ -63,20 +63,22 @@ __rte_stack_std_pop(struct rte_stack *s, void **obj_table, unsigned int n)
 {
 	struct rte_stack_std *stack = &s->stack_std;
 	unsigned int index, len;
-	void **cache_objs;
+	void **stack_objs;
 
 	rte_spinlock_lock(&stack->lock);
 
 	if (unlikely(n > stack->len)) {
+		/* Insufficient objects in the stack. */
 		rte_spinlock_unlock(&stack->lock);
 		return 0;
 	}
 
-	cache_objs = stack->objs;
+	stack_objs = stack->objs;
 
+	/* Pop objects from the stack */
 	for (index = 0, len = stack->len - 1; index < n;
 			++index, len--, obj_table++)
-		*obj_table = cache_objs[len];
+		*obj_table = stack_objs[len];
 
 	stack->len -= n;
 	rte_spinlock_unlock(&stack->lock);
Signature · 1 line
2.43.0
#12 lore raw

RFC PATCH v10 1/1 pile stack and mempool driver

In reply to Morten Brørup

Early submission of:
- A new "pile" stack-like implementation using the Stack API, and
- an accompanying "pile" mempool driver.
And:
- Some mempool optimizations.

For CI test and community feedback.

Needless to say, this must be separated into multiple independent series of patches.
And release notes must be added.
For now, I'm submitting a snapshot of work in progress.

The "pile" somewhat resembles the lock-free stack, but operates on bulks (arrays) of objects, to significantly reduce linked list traversal.
With the pile's default bulk size of 32 objects, a mempool cache flush/refill traverses a linked list of only 16 elements, whereas the lock-free stack would traverse a linked list of 512 elements.

Some performance numbers from mempool_perf_autotest_2cores, all with cache=1024 cores=2 n_keep=32768:

start performance test (using ring_mp_mc, with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 753985338 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 755805913

start performance test for lf_stack (with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 29132352 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 29276708

start performance test for pile (with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 560159479 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 557910933

Hat tip to Bruce for bringing attention to the ring not being the optimal mempool driver!

Note: The GitHub "mini" tests don't include the "pile" mempool driver, and are expected to fail.

---
v10:
* Add missing spinlock unlock. (CI)
* Fix temporary FIXME to debug why some GitHub CI tests cannot find the "pile" mempool driver.
It is the "mini" builds, where only the ring driver is included.
* Update RTE_MEMPOOL_NAMESIZE to reflect that some mempool drivers have longer memzone prefix name, e.g. "STK_" (stack) instead of "RG_" (ring). Makes RTE_MEMPOOL_NAMESIZE one smaller (25 instead of 26), and doesn't change the offset of the following field in the structure, which is 8-byte aligned (4-byte aligned on 32-bit architecture).
v9:
* Fix long lines. (checkpatch)
* Log more, to debug why some GitHub CI tests cannot find the "pile" mempool driver.
* Revert addition of __rte_restrict.
Move to separate discussion. Could be added to other libs too.
v8:
* Fix bulk_last not set in failure path with bulk elements. (AI, advanced model)
* Fix copy-paste typo (underscore not removed) in documentation reference. (CI)
* Fix memsize cache rounding, assign instead of add. (AI, advanced model)
* Fix indentation using spaces. (CI, advanced model)
* Align behavior of rte_mempool_cache_create() and mempool_create(), so they both fail if trying to create too small mempool cache. (AI, advanced model)
* Remove mempool lib constant from stack perf test. (AI, advanced model)
* Add note about why objects are copied from bulk elements by traversing the linked list once more, instead of inside __rte_stack_lf_pop_elems(). (Inspired by AI, advanced model)
* Revert __rte_restrict replacement by __restrict.
It didn't stop noise from checkpatch.
Move to separate discussion.
* Revert x86 rte_memcpy() optimization.
Move to separate patch.
* Push solo objects in reverse order, don't ignore order.
* Refactor stack overflow test case, and enable it again.
* Correct order of version history.
v7:
* Use the "__restrict" keyword, supported by all relevant C/C++ compilers. Degrade "__rte_restrict" to a backwards compatibility macro, document it as deprecated, and check for it in checkpatches.sh.
* Eliminate risk of namespace pollution by prefixing ALIGNMENT_MASK macro in x86 rte_memcpy.h header file.
* Replace conditional code by defining RTE_MEMCPY_BLOCK_64_MAX. (AI)
* Fix off-by-one in assumption when fetching free elements for the excess objects in the fragmentation element. (AI, advanced model)
* Fix memcmp() size in test_stack_push_pop(). (AI, advanced model)
* Do not try fragmentation when we know (from fetching bulk elements above) that no bulk element is available.
(Inspired by AI, advanced model)
* Fix function descriptions mentioning wrong parameter name. (AI, advanced model)
* Fix typo in documentation. (AI, advanced model)
* Mention the pile in the mempool stack documentation.
(AI, advanced model)
* Set the mbuf default pool ops to "pile", for CI test purposes only.
* Select the C11 memory model for x86, for CI test purposes only. v6:
* Add note about roll back of objects in fragmentation element. (AI)
* Move declaration of temporary variable up, to please compilers. (CI)
* Revert mempool cache size adjustments in some drivers;
let the mempool creation function adjust at runtime instead.
v5:
* Remove compiler diagnostic pragmas in stack overflow test case. (Stephen)
* Temporarily remove stack overflow test case until a sufficiently obfuscated variant doesn't trigger a compiler warning about array overrun.
* Revert most changes in rte_stack_std.h, and only change what is necessary.
* Add FIXME in the pile implementation, noting that support for the generic memory model should be removed here too, if removed in the lock-free stack.
(Inspired by Stephen)
v4:
* Revert add __rte_internal.
Compilation fails, and existing stack implementations don't have it.
v3:
* Revert rename unused field in rte_stack_pile_bulk_elem structure. v2:
* Fix indentation, long lines, etc. (checkpatch)
* Fix label followed by a declaration is a C23 extension. (CI)
* Added __rte_internal to pile init and get_memsize. (AI)
* Fix roll back bulk elements in wrong order with fragmentation. (AI)
* Minor changes suggested by AI.

---
 app/test-pmd/testpmd.c                    |   4 +-
 app/test/test_mempool.c                   |   3 +-
 app/test/test_stack.c                     |  93 +++++-
 app/test/test_stack_perf.c                |  15 +-
 config/rte_config.h                       |   7 +-
 config/x86/meson.build                    |   1 +
 doc/guides/mempool/stack.rst              |  12 +-
 doc/guides/prog_guide/stack_lib.rst       |  67 ++++-
 drivers/mempool/stack/rte_mempool_stack.c |  40 +++
 drivers/net/sxe2/sxe2_txrx_vec_avx512.c   |   2 +-
 drivers/net/tap/rte_eth_tap.c             |   2 +-
 lib/eal/include/rte_common.h              |  12 +
 lib/mempool/mempool_trace.h               |   1 -
 lib/mempool/rte_mempool.c                 |  79 +++--
 lib/mempool/rte_mempool.h                 |  95 +++---
 lib/mempool/rte_mempool_ops.c             |  27 +-
 lib/stack/meson.build                     |   3 +-
 lib/stack/rte_stack.c                     |  18 +-
 lib/stack/rte_stack.h                     |  79 +++++
 lib/stack/rte_stack_lf.h                  |   1 +
 lib/stack/rte_stack_pile.c                |  35 +++
 lib/stack/rte_stack_pile.h                | 334 ++++++++++++++++++++++
 lib/stack/rte_stack_std.h                 |  20 +-
 23 files changed, 843 insertions(+), 107 deletions(-)
 create mode 100644 lib/stack/rte_stack_pile.c
 create mode 100644 lib/stack/rte_stack_pile.h
app/test-pmd/testpmd.c +2 -2
index cab2fa1556..9e4ffd28d3 100644
@@ -1283,8 +1283,8 @@ mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 	}
 
 	TESTPMD_LOG(INFO,
-		"create a new mbuf pool <%s>: n=%u, size=%u, socket=%s\n",
-		pool_name, nb_mbuf, mbuf_seg_size,
+		"create a new mbuf pool <%s>: n=%u, cache=%u, size=%u, socket=%s\n",
+		pool_name, nb_mbuf, mb_mempool_cache, mbuf_seg_size,
 		socket_id_str(socket_id, sock_str, sizeof(sock_str)));
 
 	switch (mp_alloc_type) {
app/test/test_mempool.c +1 -2
index e54249ce61..76d45cea2a 100644
@@ -112,8 +112,7 @@ test_mempool_basic(struct rte_mempool *mp, int use_external_cache)
 		GOTO_ERR(ret, out);
 
 	printf("get private data\n");
-	if (rte_mempool_get_priv(mp) != (char *)mp +
-			RTE_MEMPOOL_HEADER_SIZE(mp, mp->cache_size))
+	if (rte_mempool_get_priv(mp) != (char *)mp + sizeof(struct rte_mempool))
 		GOTO_ERR(ret, out);
 
 #ifndef RTE_EXEC_ENV_FREEBSD /* rte_mem_virt2iova() not supported on bsd */
app/test/test_stack.c +83 -10

121 lines not on this page · raw message

app/test/test_stack_perf.c +13 -2
index 3f17a2606c..e46251d687 100644
@@ -14,14 +14,14 @@
 #include "test.h"
 
 #define STACK_NAME "STACK_PERF"
-#define MAX_BURST 32
+#define MAX_BURST 512
 #define STACK_SIZE (RTE_MAX_LCORE * MAX_BURST)
 
 /*
  * Push/pop bulk sizes, marked volatile so they aren't treated as compile-time
  * constants.
  */
-static volatile unsigned int bulk_sizes[] = {8, MAX_BURST};
+static volatile unsigned int bulk_sizes[] = {1, 8, 32, MAX_BURST};
 
 static RTE_ATOMIC(uint32_t) lcore_barrier;
 
@@ -354,5 +354,16 @@ test_lf_stack_perf(void)
 #endif
 }
 
+static int
+test_pile_perf(void)
+{
+#if defined(RTE_STACK_PILE_SUPPORTED)
+	return __test_stack_perf(RTE_STACK_F_PILE);
+#else
+	return TEST_SKIPPED;
+#endif
+}
+
 REGISTER_PERF_TEST(stack_perf_autotest, test_stack_perf);
 REGISTER_PERF_TEST(stack_lf_perf_autotest, test_lf_stack_perf);
+REGISTER_PERF_TEST(stack_pile_perf_autotest, test_pile_perf);
config/rte_config.h +5 -2
index 0447cdf2ad..6085d2e1a0 100644
@@ -56,14 +56,17 @@
 #define RTE_CONTIGMEM_DEFAULT_BUF_SIZE (512*1024*1024)
 
 /* mempool defines */
-#define RTE_MEMPOOL_CACHE_MAX_SIZE 512
+#define RTE_MEMPOOL_CACHE_MAX_SIZE 1024
 /* RTE_LIBRTE_MEMPOOL_STATS is not set */
 /* RTE_LIBRTE_MEMPOOL_DEBUG is not set */
 
 /* mbuf defines */
-#define RTE_MBUF_DEFAULT_MEMPOOL_OPS "ring_mp_mc"
+#define RTE_MBUF_DEFAULT_MEMPOOL_OPS "pile" /* FIXME: Test only. Default: "ring_mp_mc" */
 /* RTE_MBUF_HISTORY_DEBUG is not set */
 
+/* stack defines */
+#define RTE_STACK_PILE_BULK_SIZE 32
+
 /* ether defines */
 #define RTE_MAX_QUEUES_PER_PORT 1024
 #define RTE_ETHDEV_RXTX_CALLBACKS 1
config/x86/meson.build +1 -0
index 124b204847..28be579556 100644
@@ -49,6 +49,7 @@ else
 endif
 
 dpdk_conf.set('RTE_MAX_NUMA_NODES', 32)
+dpdk_conf.set('RTE_USE_C11_MEM_MODEL', true) # FIXME: Test only.
 
 if is_ms_compiler
     subdir_done()
doc/guides/mempool/stack.rst +11 -1
index 80ea07e65d..c06ab2dc56 100644
@@ -1,5 +1,6 @@
 ..  SPDX-License-Identifier: BSD-3-Clause
     Copyright(c) 2020 Intel Corporation.
+    Copyright(c) 2026 SmartShare Systems.
 
 Stack Mempool Driver
 ====================
@@ -28,6 +29,12 @@ can be selected as described in :ref:`Mempool_Handlers`:
   The underlying **rte_stack** operates in lock-free mode. For more
   information please refer to :ref:`Stack_Library_LF_Stack`.
 
+- ``pile``
+
+  The underlying **rte_stack** operates in lock-free mode,
+  and is optimized for bulks of objects.
+  For more information please refer to :ref:`Stack_Library_Pile`.
+
 The standard stack outperforms the lock-free stack on average, however the
 standard stack is non-preemptive: if a mempool user is preempted while holding
 the stack lock, that thread will block all other mempool accesses until it
@@ -35,9 +42,12 @@ returns and releases the lock. As a result, an application using the standard
 stack whose threads can be preempted can suffer from brief, infrequent
 performance hiccups.
 
-The lock-free stack, by design, is not susceptible to this problem; one thread can
+The lock-free stack and the pile, by design, are not susceptible to this problem; one thread can
 be preempted at any point during a push or pop operation and will not impede
 the progress of any other thread.
 
+The pile is not LIFO per object, but per bulk of objects.
+Although the pile is optimized for bulks of objects, it can handle any request size.
+
 For a more detailed description of the stack implementations, please refer to
 :doc:`/prog_guide/stack_lib`.
doc/guides/prog_guide/stack_lib.rst +63 -4

95 lines not on this page · raw message

drivers/mempool/stack/rte_mempool_stack.c +40 -0
index 1476905227..7467b8b39e 100644
@@ -41,6 +41,36 @@ lf_stack_alloc(struct rte_mempool *mp)
 	return __stack_alloc(mp, RTE_STACK_F_LF);
 }
 
+static int
+pile_alloc(struct rte_mempool *mp)
+{
+	return __stack_alloc(mp, RTE_STACK_F_PILE);
+}
+
+static int
+pile_enqueue(struct rte_mempool *mp, void * const *obj_table,
+	      unsigned int n)
+{
+	struct rte_stack *s = mp->pool_data;
+
+	RTE_ASSERT(s != NULL);
+	RTE_ASSERT(obj_table != NULL);
+
+	return __rte_stack_pile_push(s, obj_table, n) == 0 ? -ENOBUFS : 0;
+}
+
+static int
+pile_dequeue(struct rte_mempool *mp, void **obj_table,
+	      unsigned int n)
+{
+	struct rte_stack *s = mp->pool_data;
+
+	RTE_ASSERT(s != NULL);
+	RTE_ASSERT(obj_table != NULL);
+
+	return __rte_stack_pile_pop(s, obj_table, n) == 0 ? -ENOBUFS : 0;
+}
+
 static int
 stack_enqueue(struct rte_mempool *mp, void * const *obj_table,
 	      unsigned int n)
@@ -93,5 +123,15 @@ static struct rte_mempool_ops ops_lf_stack = {
 	.get_count = stack_get_count
 };
 
+static struct rte_mempool_ops ops_pile = {
+	.name = "pile",
+	.alloc = pile_alloc,
+	.free = stack_free,
+	.enqueue = pile_enqueue,
+	.dequeue = pile_dequeue,
+	.get_count = stack_get_count
+};
+
 RTE_MEMPOOL_REGISTER_OPS(ops_stack);
 RTE_MEMPOOL_REGISTER_OPS(ops_lf_stack);
+RTE_MEMPOOL_REGISTER_OPS(ops_pile);
drivers/net/sxe2/sxe2_txrx_vec_avx512.c +1 -1
index a830c7a33b..2e680f4027 100644
@@ -67,7 +67,7 @@ static __rte_always_inline int32_t sxe2_tx_bufs_free_vec_avx512(struct sxe2_tx_q
 		}
 		cache->len += rs_thresh;
 
-		if (cache->len >= cache->flushthresh) {
+		if (cache->len >= cache->size) {
 			(void)rte_mempool_ops_enqueue_bulk(mp,
 					&cache->objs[cache->size], cache->len - cache->size);
 			cache->len = cache->size;
drivers/net/tap/rte_eth_tap.c +1 -1
index b93452f168..b3142561c2 100644
@@ -61,7 +61,7 @@
 #define TAP_MAX_MAC_ADDRS	16
 #define TAP_GSO_MBUFS_PER_CORE	128
 #define TAP_GSO_MBUF_SEG_SIZE	128
-#define TAP_GSO_MBUF_CACHE_SIZE	4
+#define TAP_GSO_MBUF_CACHE_SIZE	32
 #define TAP_GSO_MBUFS_NUM \
 	(TAP_GSO_MBUFS_PER_CORE * TAP_GSO_MBUF_CACHE_SIZE)
 
lib/eal/include/rte_common.h +12 -0
index 79d2a0ab93..0fd0906506 100644
@@ -567,6 +567,15 @@ static void __attribute__((destructor(RTE_PRIO(prio)), used)) func(void)
 #define __rte_assume(condition) __assume(condition)
 #endif
 
+/**
+ * Alignment hint precondition
+ */
+#ifdef RTE_TOOLCHAIN_MSVC
+#define __rte_assume_aligned(ptr, alignment) (ptr)
+#else
+#define __rte_assume_aligned(ptr, alignment) __builtin_assume_aligned(ptr, alignment)
+#endif
+
 /**
  * Disable AddressSanitizer on some code
  */
@@ -775,6 +784,9 @@ rte_is_aligned(const void * const __rte_restrict ptr, const unsigned int align)
 /** Force minimum cache line alignment. */
 #define __rte_cache_min_aligned __rte_aligned(RTE_CACHE_LINE_MIN_SIZE)
 
+/** Cache alignment hint precondition */
+#define __rte_assume_cache_aligned(ptr) __rte_assume_aligned(ptr, RTE_CACHE_LINE_SIZE)
+
 #define _RTE_CACHE_GUARD_HELPER2(unique) \
 	alignas(RTE_CACHE_LINE_SIZE) \
 	char cache_guard_ ## unique[RTE_CACHE_LINE_SIZE * RTE_CACHE_GUARD_LINES]
lib/mempool/mempool_trace.h +0 -1
index 23cda1473c..60e47cf67b 100644
@@ -119,7 +119,6 @@ RTE_TRACE_POINT(
 	rte_trace_point_emit_i32(socket_id);
 	rte_trace_point_emit_ptr(cache);
 	rte_trace_point_emit_u32(cache->len);
-	rte_trace_point_emit_u32(cache->flushthresh);
 )
 
 RTE_TRACE_POINT(
lib/mempool/rte_mempool.c +57 -22

151 lines not on this page · raw message

lib/mempool/rte_mempool.h +53 -42

188 lines not on this page · raw message

lib/mempool/rte_mempool_ops.c +21 -6
index 066bec36fc..261ad217ee 100644
@@ -27,7 +27,7 @@ int
 rte_mempool_register_ops(const struct rte_mempool_ops *h)
 {
 	struct rte_mempool_ops *ops;
-	int16_t ops_index;
+	unsigned int ops_index;
 
 	rte_spinlock_lock(&rte_mempool_ops_table.sl);
 
@@ -47,12 +47,21 @@ rte_mempool_register_ops(const struct rte_mempool_ops *h)
 		return -EINVAL;
 	}
 
-	if (strlen(h->name) >= sizeof(ops->name) - 1) {
+	if (strlen(h->name) > sizeof(ops->name) - 1) {
 		rte_spinlock_unlock(&rte_mempool_ops_table.sl);
-		RTE_MEMPOOL_LOG(DEBUG, "%s(): mempool_ops <%s>: name too long",
+		RTE_MEMPOOL_LOG(ERR, "%s(): mempool_ops <%s>: name too long",
 				__func__, h->name);
-		rte_errno = EEXIST;
-		return -EEXIST;
+		return -ENAMETOOLONG;
+	}
+
+	for (ops_index = 0; ops_index < rte_mempool_ops_table.num_ops; ops_index++) {
+		if (!strcmp(h->name,
+				rte_mempool_ops_table.ops[ops_index].name)) {
+			rte_spinlock_unlock(&rte_mempool_ops_table.sl);
+			RTE_MEMPOOL_LOG(ERR, "%s(): mempool_ops <%s>: name exists",
+					__func__, h->name);
+			return -EEXIST;
+		}
 	}
 
 	ops_index = rte_mempool_ops_table.num_ops++;
@@ -68,6 +77,9 @@ rte_mempool_register_ops(const struct rte_mempool_ops *h)
 	ops->get_info = h->get_info;
 	ops->dequeue_contig_blocks = h->dequeue_contig_blocks;
 
+	RTE_MEMPOOL_LOG(DEBUG,
+		"Registered mempool_ops[%u] <%s>", ops_index, h->name);
+
 	rte_spinlock_unlock(&rte_mempool_ops_table.sl);
 
 	return ops_index;
@@ -185,8 +197,11 @@ rte_mempool_set_ops_byname(struct rte_mempool *mp, const char *name,
 		}
 	}
 
-	if (ops == NULL)
+	if (ops == NULL) {
+		RTE_MEMPOOL_LOG(ERR,
+			"Unknown mempool_ops <%s>, of %u ops registered", name, i);
 		return -EINVAL;
+	}
 
 	mp->ops_index = i;
 	mp->pool_config = pool_config;
lib/stack/meson.build +2 -1
index 18177a742f..50e688522e 100644
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2019 Intel Corporation
 
-sources = files('rte_stack.c', 'rte_stack_std.c', 'rte_stack_lf.c')
+sources = files('rte_stack.c', 'rte_stack_std.c', 'rte_stack_lf.c', 'rte_stack_pile.c')
 headers = files('rte_stack.h')
 # subheaders, not for direct inclusion by apps
 indirect_headers += files(
@@ -10,4 +10,5 @@ indirect_headers += files(
         'rte_stack_lf_generic.h',
         'rte_stack_lf_c11.h',
         'rte_stack_lf_stubs.h',
+        'rte_stack_pile.h',
 )
lib/stack/rte_stack.c +17 -1
index 4c78fe4b4b..a4bbf8a4d7 100644
@@ -1,5 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright(c) 2019 Intel Corporation
+ * Copyright(c) 2026 SmartShare Systems
  */
 
 #include <stdalign.h>
@@ -32,6 +33,8 @@ rte_stack_init(struct rte_stack *s, unsigned int count, uint32_t flags)
 
 	if (flags & RTE_STACK_F_LF)
 		rte_stack_lf_init(s, count);
+	else if (flags & RTE_STACK_F_PILE)
+		rte_stack_pile_init(s, count);
 	else
 		rte_stack_std_init(s);
 }
@@ -41,6 +44,8 @@ rte_stack_get_memsize(unsigned int count, uint32_t flags)
 {
 	if (flags & RTE_STACK_F_LF)
 		return rte_stack_lf_get_memsize(count);
+	else if (flags & RTE_STACK_F_PILE)
+		return rte_stack_pile_get_memsize(count);
 	else
 		return rte_stack_std_get_memsize(count);
 }
@@ -58,7 +63,11 @@ rte_stack_create(const char *name, unsigned int count, int socket_id,
 	unsigned int sz;
 	int ret;
 
-	if (flags & ~(RTE_STACK_F_LF)) {
+	if (flags & ~(RTE_STACK_F_LF | RTE_STACK_F_PILE)) {
+		STACK_LOG_ERR("Unsupported stack flags %#x", flags);
+		return NULL;
+	}
+	if ((flags & RTE_STACK_F_LF) && (flags & RTE_STACK_F_PILE)) {
 		STACK_LOG_ERR("Unsupported stack flags %#x", flags);
 		return NULL;
 	}
@@ -73,6 +82,13 @@ rte_stack_create(const char *name, unsigned int count, int socket_id,
 		return NULL;
 	}
 #endif
+#if !defined(RTE_STACK_PILE_SUPPORTED)
+	if (flags & RTE_STACK_F_PILE) {
+		STACK_LOG_ERR("Pile is not supported on your platform");
+		rte_errno = ENOTSUP;
+		return NULL;
+	}
+#endif
 
 	sz = rte_stack_get_memsize(count, flags);
 
lib/stack/rte_stack.h +79 -0

139 lines not on this page · raw message

lib/stack/rte_stack_lf.h +1 -0
index f2b012cd0e..1ee9330c57 100644
@@ -79,6 +79,7 @@ __rte_stack_lf_pop(struct rte_stack *s, void **obj_table, unsigned int n)
 		return 0;
 
 	/* Pop n used elements */
+	__rte_assume(obj_table != NULL);
 	first = __rte_stack_lf_pop_elems(&s->stack_lf.used,
 					 n, obj_table, &last);
 	if (unlikely(first == NULL))
lib/stack/rte_stack_pile.c new file +35 -0
new file mode 100644
index 0000000000..5884163313
@@ -0,0 +1,35 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2026 SmartShare Systems
+ */
+
+#include "rte_stack.h"
+
+void
+rte_stack_pile_init(struct rte_stack *s, unsigned int count)
+{
+	unsigned int bulk = (count + RTE_STACK_PILE_BULK_SIZE - 1) / RTE_STACK_PILE_BULK_SIZE;
+	struct rte_stack_pile_bulk_elem *bulk_elems =
+			(struct rte_stack_pile_bulk_elem *)(s->stack_pile.elems);
+	struct rte_stack_lf_elem *solo_elems = (struct rte_stack_lf_elem *)&bulk_elems[bulk];
+	unsigned int i;
+
+	for (i = 0; i < bulk; i++)
+		__rte_stack_pile_bulk_push_elems(&s->stack_pile.free_bulk,
+					  &bulk_elems[i], &bulk_elems[i], 1);
+	for (i = 0; i < count; i++)
+		__rte_stack_lf_push_elems(&s->stack_pile.free_solo,
+					  &solo_elems[i], &solo_elems[i], 1);
+}
+
+ssize_t
+rte_stack_pile_get_memsize(unsigned int count)
+{
+	unsigned int bulk = (count + RTE_STACK_PILE_BULK_SIZE - 1) / RTE_STACK_PILE_BULK_SIZE;
+	ssize_t sz = offsetof(struct rte_stack, stack_pile.elems);
+	sz += bulk * sizeof(struct rte_stack_pile_bulk_elem);
+	sz += count * sizeof(struct rte_stack_lf_elem);
+	sz = RTE_CACHE_LINE_ROUNDUP(sz);
+	sz += RTE_CACHE_GUARD_LINES * RTE_CACHE_LINE_SIZE;
+
+	return sz;
+}
lib/stack/rte_stack_pile.h new file +334 -0

334 lines not on this page · raw message

lib/stack/rte_stack_std.h +11 -9
index ae28add5c4..d5b21defb0 100644
@@ -25,20 +25,20 @@ __rte_stack_std_push(struct rte_stack *s, void * const *obj_table,
 {
 	struct rte_stack_std *stack = &s->stack_std;
 	unsigned int index;
-	void **cache_objs;
+	void **stack_objs;
 
 	rte_spinlock_lock(&stack->lock);
-	cache_objs = &stack->objs[stack->len];
+	stack_objs = &stack->objs[stack->len];
 
-	/* Is there sufficient space in the stack? */
-	if ((stack->len + n) > s->capacity) {
+	if (unlikely((stack->len + n) > s->capacity)) {
+		/* Insufficient space in the stack. */
 		rte_spinlock_unlock(&stack->lock);
 		return 0;
 	}
 
-	/* Add elements back into the cache */
+	/* Push objects to the stack */
 	for (index = 0; index < n; ++index, obj_table++)
-		cache_objs[index] = *obj_table;
+		stack_objs[index] = *obj_table;
 
 	stack->len += n;
 
@@ -63,20 +63,22 @@ __rte_stack_std_pop(struct rte_stack *s, void **obj_table, unsigned int n)
 {
 	struct rte_stack_std *stack = &s->stack_std;
 	unsigned int index, len;
-	void **cache_objs;
+	void **stack_objs;
 
 	rte_spinlock_lock(&stack->lock);
 
 	if (unlikely(n > stack->len)) {
+		/* Insufficient objects in the stack. */
 		rte_spinlock_unlock(&stack->lock);
 		return 0;
 	}
 
-	cache_objs = stack->objs;
+	stack_objs = stack->objs;
 
+	/* Pop objects from the stack */
 	for (index = 0, len = stack->len - 1; index < n;
 			++index, len--, obj_table++)
-		*obj_table = cache_objs[len];
+		*obj_table = stack_objs[len];
 
 	stack->len -= n;
 	rte_spinlock_unlock(&stack->lock);
Signature · 1 line
2.43.0
#13 lore raw

Re: [RFC PATCH v10] pile stack and mempool driver

Quoted 35 lines · On Mon, 10 Aug 2026 15:12:14 +0000 Morten Brørup <[email protected]>…

Early submission of:
- A new "pile" stack-like implementation using the Stack API, and
- an accompanying "pile" mempool driver.
And:
- Some mempool optimizations.

For CI test and community feedback.

Needless to say, this must be separated into multiple independent series of patches.
And release notes must be added.
For now, I'm submitting a snapshot of work in progress.

The "pile" somewhat resembles the lock-free stack, but operates on bulks (arrays) of objects, to significantly reduce linked list traversal.
With the pile's default bulk size of 32 objects, a mempool cache flush/refill traverses a linked list of only 16 elements, whereas the lock-free stack would traverse a linked list of 512 elements.

Some performance numbers from mempool_perf_autotest_2cores, all with cache=1024 cores=2 n_keep=32768:

start performance test (using ring_mp_mc, with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 753985338 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 755805913

start performance test for lf_stack (with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 29132352 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 29276708

start performance test for pile (with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 560159479 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 557910933

Quoted 8 lines

Hat tip to Bruce for bringing attention to the ring not being the optimal mempool driver!

Note: The GitHub "mini" tests don't include the "pile" mempool driver, and are expected to fail.

---

AI review with Claude Fable showed:

Applied to main (c1a46b9), clean build with gcc 13.3 and -Dwerror=true. No remaining references to the removed flushthresh field or the RTE_MEMPOOL_HEADER_SIZE macro anywhere in the tree. Ran the stack, stack_lf, stack_pile and mempool autotests plus a standalone harness exercising the pile push/pop paths directly.

Errors:

1. test_stack.c: the pile fill loop in test_stack_basic() assumes an effective capacity of RTE_STACK_PILE_BULK_SIZE * STACK_SIZE, but the pile only has ceil(count / 32) free bulk elements, i.e. exactly count objects via the bulk path. The first push(STACK_SIZE) consumes all 2048 bulk elements; the second iteration fails. Confirmed at runtime:

[test_stack_basic():193] Fill objects push failed

The drain loop below it has the mirrored assumption. The loop bound should reflect the actual capacity model (one full-bulk fill, or fill bulk and solo separately).

2. test_stack.c: the failure above is masked -- the test still reports "Test OK". In all four new branches ("All objects push failed", "Fill objects push failed", "All objects pop failed", "Drain objects pop failed"), ret holds the return value of the failed push/pop, which is 0, and "goto fail_test" then does "return ret;", returning 0 = success to __test_stack(). Each of these branches needs "ret = -1;" before the goto. Confirmed at runtime: the failure message prints, then the suite reports Test OK. This is presumably why CI never surfaced error 1.

Warnings:

3. Pile capacity contract: a pile created with count N accepts up to 2*N objects (N via bulk elements plus N via solo elements). Verified
empirically: after a full-bulk fill of 65536, another 65536 solo pushes were accepted, while rte_stack_count() clamps at s->capacity and under-reports the real content. Either size the solo free list so total capacity is N, or document the 2x behavior; the current state makes count/free_count inconsistent for raw stack API users. (Not reachable through mempool usage, where the object total is conserved.)

4. Cache sizes 1..31 now fail pool creation outright: rounding down to a multiple of 32 yields 0 and EINVAL. Verified:
rte_mempool_create(cache=4) and rte_mempool_cache_create(4) both fail; 250 is silently rounded to 224 with only a DEBUG log. This breaks existing applications at runtime (the in-tree tap driver needed fixing from 4 to 32; out-of-tree apps get no warning). Consider rounding up for small sizes instead of failing, and this needs a prominent release notes entry either way. The rte_mempool.h doc says cache_size "must be divisible by 32" while the code accepts and rounds -- doc and behavior should agree.

5. struct rte_mempool now embeds local_cache[RTE_MAX_LCORE] unconditionally, so cache-less (cache_size=0) mempools grow by RTE_MAX_LCORE * sizeof(struct rte_mempool_cache) -- roughly 1 MB of memzone per pool at the default RTE_MAX_LCORE=128 with the enlarged objs[1024] array plus cache guard. Applications creating many small cache-less pools pay this in hugepage memory. Deliberate trade-off for removing the hot-path branch, but it should be called out in the release notes.

6. tap: raising TAP_GSO_MBUF_CACHE_SIZE from 4 to 32 also multiplies TAP_GSO_MBUFS_NUM (defined as PER_CORE * CACHE_SIZE) from 512 to 4096 mbufs, an 8x larger GSO pool. If the change is only to satisfy the divisibility constraint, decouple MBUFS_NUM from CACHE_SIZE.

7. The two FIXME test-only settings (RTE_MBUF_DEFAULT_MEMPOOL_OPS "pile", RTE_USE_C11_MEM_MODEL in config/x86/meson.build) must be reverted before merge, as the FIXMEs already note.

Info:

8. RTE_MEMPOOL_DRIVER_REPRESENTOR_MZ_PREFIX: "representor" collides with the port-representor concept; "representative" or "worst-case" would read better.

9. rte_mempool_register_ops(): the old name-too-long path set rte_errno before returning; the new -ENAMETOOLONG and -EEXIST paths return the negative errno without setting rte_errno. Consistent with the other error returns in this function, but a small change in error reporting.

10. test_stack_push_pop() skips content verification entirely when bulk_sz >= RTE_STACK_PILE_BULK_SIZE but not a multiple of it. Latent (only 1 and MAX_BULK=512 are used today), but worth a comment or coverage.

Runtime verification that passed: the fragmentation pop path (push 64, pop 40 as 1 bulk + 8 via a fragmented bulk element, then pop 24) conserves all objects with no duplication or loss; the retry path with odd sizes (push 33+31+32, pop 96) likewise; bulk-level LIFO ordering matches the memcmp formula in the test; the pile mempool handler passes populate/get/put/audit and a 1000-iteration uneven get/put churn through the per-lcore cache; mempool_autotest (which now defaults to pile) and the std/lf stack autotests all pass. The rounded-up rte_memcpy in rte_mempool_do_generic_put stays within objs[] bounds given the divisible-by-32 cache size enforced at creation.

#15 lore raw

RFC PATCH v11 1/1 pile stack and mempool driver

In reply to Morten Brørup

Early submission of:
- A new "pile" stack-like implementation using the Stack API, and
- an accompanying "pile" mempool driver.

For CI test and community feedback.

Needless to say, this must be separated into multiple independent series of patches.
And release notes must be added.
For now, I'm submitting a snapshot of work in progress.

The "pile" somewhat resembles the lock-free stack, but operates on bulks (arrays) of objects, to significantly reduce linked list traversal.
With the pile's default bulk size of 32 objects, a mempool cache flush/refill traverses a linked list of only 16 elements, whereas the lock-free stack would traverse a linked list of 512 elements.

Some performance numbers from mempool_perf_autotest_2cores, all with cache=1024 cores=2 n_keep=32768:

start performance test (using ring_mp_mc, with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 753985338 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 755805913

start performance test for lf_stack (with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 29132352 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 29276708

start performance test for pile (with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 560159479 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 557910933

Hat tip to Bruce for bringing attention to the ring not being the optimal mempool driver!

Note: The GitHub "mini" tests don't include the "pile" mempool driver, and are expected to fail.

---
v11:
* Fix test_stack_basic(), both regarding stack overflow test, and regarding some failures not returning -1. (AI, advanced model) This also simplifies the stack overflow test.
* Don't fail mempool creation for tiny cache; instead, continue with cache disabled. (Inspired by AI, advanced model)
* Partially revert RTE_MEMPOOL_NAMESIZE change.
It causes fib6_autotest to fail, due to a test case using a long mempool name.
Instead, keep it bases on the memzone prefix name of the default mempool driver (Ring), and add more details its description. (CI)
* Renamed macro representing memzone prefix of mempool driver, for clarity. (AI, advanced model)
* Revert rte_stack_std.h local variable name correction.
Move to separate patch.
v10:
* Add missing spinlock unlock. (CI)
* Fix temporary FIXME to debug why some GitHub CI tests cannot find the "pile" mempool driver.
It is the "mini" builds, where only the ring driver is included.
* Update RTE_MEMPOOL_NAMESIZE to reflect that some mempool drivers have longer memzone prefix name, e.g. "STK_" (stack) instead of "RG_" (ring). Makes RTE_MEMPOOL_NAMESIZE one smaller (25 instead of 26), and doesn't change the offset of the following field in the structure, which is 8-byte aligned (4-byte aligned on 32-bit architecture).
v9:
* Fix long lines. (checkpatch)
* Log more, to debug why some GitHub CI tests cannot find the "pile" mempool driver.
* Revert addition of __rte_restrict.
Move to separate discussion. Could be added to other libs too.
v8:
* Fix bulk_last not set in failure path with bulk elements. (AI, advanced model)
* Fix copy-paste typo (underscore not removed) in documentation reference. (CI)
* Fix memsize cache rounding, assign instead of add. (AI, advanced model)
* Fix indentation using spaces. (CI, advanced model)
* Align behavior of rte_mempool_cache_create() and mempool_create(), so they both fail if trying to create too small mempool cache. (AI, advanced model)
* Remove mempool lib constant from stack perf test. (AI, advanced model)
* Add note about why objects are copied from bulk elements by traversing the linked list once more, instead of inside __rte_stack_lf_pop_elems(). (Inspired by AI, advanced model)
* Revert __rte_restrict replacement by __restrict.
It didn't stop noise from checkpatch.
Move to separate discussion.
* Revert x86 rte_memcpy() optimization.
Move to separate patch.
* Push solo objects in reverse order, don't ignore order.
* Refactor stack overflow test case, and enable it again.
* Correct order of version history.
v7:
* Use the "__restrict" keyword, supported by all relevant C/C++ compilers. Degrade "__rte_restrict" to a backwards compatibility macro, document it as deprecated, and check for it in checkpatches.sh.
* Eliminate risk of namespace pollution by prefixing ALIGNMENT_MASK macro in x86 rte_memcpy.h header file.
* Replace conditional code by defining RTE_MEMCPY_BLOCK_64_MAX. (AI)
* Fix off-by-one in assumption when fetching free elements for the excess objects in the fragmentation element. (AI, advanced model)
* Fix memcmp() size in test_stack_push_pop(). (AI, advanced model)
* Do not try fragmentation when we know (from fetching bulk elements above) that no bulk element is available.
(Inspired by AI, advanced model)
* Fix function descriptions mentioning wrong parameter name. (AI, advanced model)
* Fix typo in documentation. (AI, advanced model)
* Mention the pile in the mempool stack documentation.
(AI, advanced model)
* Set the mbuf default pool ops to "pile", for CI test purposes only.
* Select the C11 memory model for x86, for CI test purposes only. v6:
* Add note about roll back of objects in fragmentation element. (AI)
* Move declaration of temporary variable up, to please compilers. (CI)
* Revert mempool cache size adjustments in some drivers;
let the mempool creation function adjust at runtime instead.
v5:
* Remove compiler diagnostic pragmas in stack overflow test case. (Stephen)
* Temporarily remove stack overflow test case until a sufficiently obfuscated variant doesn't trigger a compiler warning about array overrun.
* Revert most changes in rte_stack_std.h, and only change what is necessary.
* Add FIXME in the pile implementation, noting that support for the generic memory model should be removed here too, if removed in the lock-free stack.
(Inspired by Stephen)
v4:
* Revert add __rte_internal.
Compilation fails, and existing stack implementations don't have it.
v3:
* Revert rename unused field in rte_stack_pile_bulk_elem structure. v2:
* Fix indentation, long lines, etc. (checkpatch)
* Fix label followed by a declaration is a C23 extension. (CI)
* Added __rte_internal to pile init and get_memsize. (AI)
* Fix roll back bulk elements in wrong order with fragmentation. (AI)
* Minor changes suggested by AI.

---
 app/test-pmd/testpmd.c                    |   4 +-
 app/test/test_mempool.c                   |   3 +-
 app/test/test_stack.c                     |  71 ++++-
 app/test/test_stack_perf.c                |  15 +-
 config/rte_config.h                       |   7 +-
 config/x86/meson.build                    |   1 +
 doc/guides/mempool/stack.rst              |  12 +-
 doc/guides/prog_guide/stack_lib.rst       |  67 ++++-
 drivers/mempool/stack/rte_mempool_stack.c |  40 +++
 drivers/net/sxe2/sxe2_txrx_vec_avx512.c   |   2 +-
 drivers/net/tap/rte_eth_tap.c             |   2 +-
 lib/eal/include/rte_common.h              |  12 +
 lib/mempool/mempool_trace.h               |   1 -
 lib/mempool/rte_mempool.c                 |  87 ++++--
 lib/mempool/rte_mempool.h                 | 101 ++++---
 lib/mempool/rte_mempool_ops.c             |  27 +-
 lib/stack/meson.build                     |   3 +-
 lib/stack/rte_stack.c                     |  18 +-
 lib/stack/rte_stack.h                     |  79 +++++
 lib/stack/rte_stack_lf.h                  |   1 +
 lib/stack/rte_stack_pile.c                |  35 +++
 lib/stack/rte_stack_pile.h                | 334 ++++++++++++++++++++++
 22 files changed, 823 insertions(+), 99 deletions(-)
 create mode 100644 lib/stack/rte_stack_pile.c
 create mode 100644 lib/stack/rte_stack_pile.h
app/test-pmd/testpmd.c +2 -2
index cab2fa1556..9e4ffd28d3 100644
@@ -1283,8 +1283,8 @@ mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 	}
 
 	TESTPMD_LOG(INFO,
-		"create a new mbuf pool <%s>: n=%u, size=%u, socket=%s\n",
-		pool_name, nb_mbuf, mbuf_seg_size,
+		"create a new mbuf pool <%s>: n=%u, cache=%u, size=%u, socket=%s\n",
+		pool_name, nb_mbuf, mb_mempool_cache, mbuf_seg_size,
 		socket_id_str(socket_id, sock_str, sizeof(sock_str)));
 
 	switch (mp_alloc_type) {
app/test/test_mempool.c +1 -2
index e54249ce61..76d45cea2a 100644
@@ -112,8 +112,7 @@ test_mempool_basic(struct rte_mempool *mp, int use_external_cache)
 		GOTO_ERR(ret, out);
 
 	printf("get private data\n");
-	if (rte_mempool_get_priv(mp) != (char *)mp +
-			RTE_MEMPOOL_HEADER_SIZE(mp, mp->cache_size))
+	if (rte_mempool_get_priv(mp) != (char *)mp + sizeof(struct rte_mempool))
 		GOTO_ERR(ret, out);
 
 #ifndef RTE_EXEC_ENV_FREEBSD /* rte_mem_virt2iova() not supported on bsd */
app/test/test_stack.c +61 -10

107 lines not on this page · raw message

app/test/test_stack_perf.c +13 -2
index 3f17a2606c..e46251d687 100644
@@ -14,14 +14,14 @@
 #include "test.h"
 
 #define STACK_NAME "STACK_PERF"
-#define MAX_BURST 32
+#define MAX_BURST 512
 #define STACK_SIZE (RTE_MAX_LCORE * MAX_BURST)
 
 /*
  * Push/pop bulk sizes, marked volatile so they aren't treated as compile-time
  * constants.
  */
-static volatile unsigned int bulk_sizes[] = {8, MAX_BURST};
+static volatile unsigned int bulk_sizes[] = {1, 8, 32, MAX_BURST};
 
 static RTE_ATOMIC(uint32_t) lcore_barrier;
 
@@ -354,5 +354,16 @@ test_lf_stack_perf(void)
 #endif
 }
 
+static int
+test_pile_perf(void)
+{
+#if defined(RTE_STACK_PILE_SUPPORTED)
+	return __test_stack_perf(RTE_STACK_F_PILE);
+#else
+	return TEST_SKIPPED;
+#endif
+}
+
 REGISTER_PERF_TEST(stack_perf_autotest, test_stack_perf);
 REGISTER_PERF_TEST(stack_lf_perf_autotest, test_lf_stack_perf);
+REGISTER_PERF_TEST(stack_pile_perf_autotest, test_pile_perf);
config/rte_config.h +5 -2
index 0447cdf2ad..6085d2e1a0 100644
@@ -56,14 +56,17 @@
 #define RTE_CONTIGMEM_DEFAULT_BUF_SIZE (512*1024*1024)
 
 /* mempool defines */
-#define RTE_MEMPOOL_CACHE_MAX_SIZE 512
+#define RTE_MEMPOOL_CACHE_MAX_SIZE 1024
 /* RTE_LIBRTE_MEMPOOL_STATS is not set */
 /* RTE_LIBRTE_MEMPOOL_DEBUG is not set */
 
 /* mbuf defines */
-#define RTE_MBUF_DEFAULT_MEMPOOL_OPS "ring_mp_mc"
+#define RTE_MBUF_DEFAULT_MEMPOOL_OPS "pile" /* FIXME: Test only. Default: "ring_mp_mc" */
 /* RTE_MBUF_HISTORY_DEBUG is not set */
 
+/* stack defines */
+#define RTE_STACK_PILE_BULK_SIZE 32
+
 /* ether defines */
 #define RTE_MAX_QUEUES_PER_PORT 1024
 #define RTE_ETHDEV_RXTX_CALLBACKS 1
config/x86/meson.build +1 -0
index 124b204847..28be579556 100644
@@ -49,6 +49,7 @@ else
 endif
 
 dpdk_conf.set('RTE_MAX_NUMA_NODES', 32)
+dpdk_conf.set('RTE_USE_C11_MEM_MODEL', true) # FIXME: Test only.
 
 if is_ms_compiler
     subdir_done()
doc/guides/mempool/stack.rst +11 -1
index 80ea07e65d..c06ab2dc56 100644
@@ -1,5 +1,6 @@
 ..  SPDX-License-Identifier: BSD-3-Clause
     Copyright(c) 2020 Intel Corporation.
+    Copyright(c) 2026 SmartShare Systems.
 
 Stack Mempool Driver
 ====================
@@ -28,6 +29,12 @@ can be selected as described in :ref:`Mempool_Handlers`:
   The underlying **rte_stack** operates in lock-free mode. For more
   information please refer to :ref:`Stack_Library_LF_Stack`.
 
+- ``pile``
+
+  The underlying **rte_stack** operates in lock-free mode,
+  and is optimized for bulks of objects.
+  For more information please refer to :ref:`Stack_Library_Pile`.
+
 The standard stack outperforms the lock-free stack on average, however the
 standard stack is non-preemptive: if a mempool user is preempted while holding
 the stack lock, that thread will block all other mempool accesses until it
@@ -35,9 +42,12 @@ returns and releases the lock. As a result, an application using the standard
 stack whose threads can be preempted can suffer from brief, infrequent
 performance hiccups.
 
-The lock-free stack, by design, is not susceptible to this problem; one thread can
+The lock-free stack and the pile, by design, are not susceptible to this problem; one thread can
 be preempted at any point during a push or pop operation and will not impede
 the progress of any other thread.
 
+The pile is not LIFO per object, but per bulk of objects.
+Although the pile is optimized for bulks of objects, it can handle any request size.
+
 For a more detailed description of the stack implementations, please refer to
 :doc:`/prog_guide/stack_lib`.
doc/guides/prog_guide/stack_lib.rst +63 -4

95 lines not on this page · raw message

drivers/mempool/stack/rte_mempool_stack.c +40 -0
index 1476905227..7467b8b39e 100644
@@ -41,6 +41,36 @@ lf_stack_alloc(struct rte_mempool *mp)
 	return __stack_alloc(mp, RTE_STACK_F_LF);
 }
 
+static int
+pile_alloc(struct rte_mempool *mp)
+{
+	return __stack_alloc(mp, RTE_STACK_F_PILE);
+}
+
+static int
+pile_enqueue(struct rte_mempool *mp, void * const *obj_table,
+	      unsigned int n)
+{
+	struct rte_stack *s = mp->pool_data;
+
+	RTE_ASSERT(s != NULL);
+	RTE_ASSERT(obj_table != NULL);
+
+	return __rte_stack_pile_push(s, obj_table, n) == 0 ? -ENOBUFS : 0;
+}
+
+static int
+pile_dequeue(struct rte_mempool *mp, void **obj_table,
+	      unsigned int n)
+{
+	struct rte_stack *s = mp->pool_data;
+
+	RTE_ASSERT(s != NULL);
+	RTE_ASSERT(obj_table != NULL);
+
+	return __rte_stack_pile_pop(s, obj_table, n) == 0 ? -ENOBUFS : 0;
+}
+
 static int
 stack_enqueue(struct rte_mempool *mp, void * const *obj_table,
 	      unsigned int n)
@@ -93,5 +123,15 @@ static struct rte_mempool_ops ops_lf_stack = {
 	.get_count = stack_get_count
 };
 
+static struct rte_mempool_ops ops_pile = {
+	.name = "pile",
+	.alloc = pile_alloc,
+	.free = stack_free,
+	.enqueue = pile_enqueue,
+	.dequeue = pile_dequeue,
+	.get_count = stack_get_count
+};
+
 RTE_MEMPOOL_REGISTER_OPS(ops_stack);
 RTE_MEMPOOL_REGISTER_OPS(ops_lf_stack);
+RTE_MEMPOOL_REGISTER_OPS(ops_pile);
drivers/net/sxe2/sxe2_txrx_vec_avx512.c +1 -1
index a830c7a33b..2e680f4027 100644
@@ -67,7 +67,7 @@ static __rte_always_inline int32_t sxe2_tx_bufs_free_vec_avx512(struct sxe2_tx_q
 		}
 		cache->len += rs_thresh;
 
-		if (cache->len >= cache->flushthresh) {
+		if (cache->len >= cache->size) {
 			(void)rte_mempool_ops_enqueue_bulk(mp,
 					&cache->objs[cache->size], cache->len - cache->size);
 			cache->len = cache->size;
drivers/net/tap/rte_eth_tap.c +1 -1
index b93452f168..b3142561c2 100644
@@ -61,7 +61,7 @@
 #define TAP_MAX_MAC_ADDRS	16
 #define TAP_GSO_MBUFS_PER_CORE	128
 #define TAP_GSO_MBUF_SEG_SIZE	128
-#define TAP_GSO_MBUF_CACHE_SIZE	4
+#define TAP_GSO_MBUF_CACHE_SIZE	32
 #define TAP_GSO_MBUFS_NUM \
 	(TAP_GSO_MBUFS_PER_CORE * TAP_GSO_MBUF_CACHE_SIZE)
 
lib/eal/include/rte_common.h +12 -0
index 79d2a0ab93..0fd0906506 100644
@@ -567,6 +567,15 @@ static void __attribute__((destructor(RTE_PRIO(prio)), used)) func(void)
 #define __rte_assume(condition) __assume(condition)
 #endif
 
+/**
+ * Alignment hint precondition
+ */
+#ifdef RTE_TOOLCHAIN_MSVC
+#define __rte_assume_aligned(ptr, alignment) (ptr)
+#else
+#define __rte_assume_aligned(ptr, alignment) __builtin_assume_aligned(ptr, alignment)
+#endif
+
 /**
  * Disable AddressSanitizer on some code
  */
@@ -775,6 +784,9 @@ rte_is_aligned(const void * const __rte_restrict ptr, const unsigned int align)
 /** Force minimum cache line alignment. */
 #define __rte_cache_min_aligned __rte_aligned(RTE_CACHE_LINE_MIN_SIZE)
 
+/** Cache alignment hint precondition */
+#define __rte_assume_cache_aligned(ptr) __rte_assume_aligned(ptr, RTE_CACHE_LINE_SIZE)
+
 #define _RTE_CACHE_GUARD_HELPER2(unique) \
 	alignas(RTE_CACHE_LINE_SIZE) \
 	char cache_guard_ ## unique[RTE_CACHE_LINE_SIZE * RTE_CACHE_GUARD_LINES]
lib/mempool/mempool_trace.h +0 -1
index 23cda1473c..60e47cf67b 100644
@@ -119,7 +119,6 @@ RTE_TRACE_POINT(
 	rte_trace_point_emit_i32(socket_id);
 	rte_trace_point_emit_ptr(cache);
 	rte_trace_point_emit_u32(cache->len);
-	rte_trace_point_emit_u32(cache->flushthresh);
 )
 
 RTE_TRACE_POINT(
lib/mempool/rte_mempool.c +64 -23

185 lines not on this page · raw message

lib/mempool/rte_mempool.h +59 -42

194 lines not on this page · raw message

lib/mempool/rte_mempool_ops.c +21 -6
index 066bec36fc..261ad217ee 100644
@@ -27,7 +27,7 @@ int
 rte_mempool_register_ops(const struct rte_mempool_ops *h)
 {
 	struct rte_mempool_ops *ops;
-	int16_t ops_index;
+	unsigned int ops_index;
 
 	rte_spinlock_lock(&rte_mempool_ops_table.sl);
 
@@ -47,12 +47,21 @@ rte_mempool_register_ops(const struct rte_mempool_ops *h)
 		return -EINVAL;
 	}
 
-	if (strlen(h->name) >= sizeof(ops->name) - 1) {
+	if (strlen(h->name) > sizeof(ops->name) - 1) {
 		rte_spinlock_unlock(&rte_mempool_ops_table.sl);
-		RTE_MEMPOOL_LOG(DEBUG, "%s(): mempool_ops <%s>: name too long",
+		RTE_MEMPOOL_LOG(ERR, "%s(): mempool_ops <%s>: name too long",
 				__func__, h->name);
-		rte_errno = EEXIST;
-		return -EEXIST;
+		return -ENAMETOOLONG;
+	}
+
+	for (ops_index = 0; ops_index < rte_mempool_ops_table.num_ops; ops_index++) {
+		if (!strcmp(h->name,
+				rte_mempool_ops_table.ops[ops_index].name)) {
+			rte_spinlock_unlock(&rte_mempool_ops_table.sl);
+			RTE_MEMPOOL_LOG(ERR, "%s(): mempool_ops <%s>: name exists",
+					__func__, h->name);
+			return -EEXIST;
+		}
 	}
 
 	ops_index = rte_mempool_ops_table.num_ops++;
@@ -68,6 +77,9 @@ rte_mempool_register_ops(const struct rte_mempool_ops *h)
 	ops->get_info = h->get_info;
 	ops->dequeue_contig_blocks = h->dequeue_contig_blocks;
 
+	RTE_MEMPOOL_LOG(DEBUG,
+		"Registered mempool_ops[%u] <%s>", ops_index, h->name);
+
 	rte_spinlock_unlock(&rte_mempool_ops_table.sl);
 
 	return ops_index;
@@ -185,8 +197,11 @@ rte_mempool_set_ops_byname(struct rte_mempool *mp, const char *name,
 		}
 	}
 
-	if (ops == NULL)
+	if (ops == NULL) {
+		RTE_MEMPOOL_LOG(ERR,
+			"Unknown mempool_ops <%s>, of %u ops registered", name, i);
 		return -EINVAL;
+	}
 
 	mp->ops_index = i;
 	mp->pool_config = pool_config;
lib/stack/meson.build +2 -1
index 18177a742f..50e688522e 100644
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2019 Intel Corporation
 
-sources = files('rte_stack.c', 'rte_stack_std.c', 'rte_stack_lf.c')
+sources = files('rte_stack.c', 'rte_stack_std.c', 'rte_stack_lf.c', 'rte_stack_pile.c')
 headers = files('rte_stack.h')
 # subheaders, not for direct inclusion by apps
 indirect_headers += files(
@@ -10,4 +10,5 @@ indirect_headers += files(
         'rte_stack_lf_generic.h',
         'rte_stack_lf_c11.h',
         'rte_stack_lf_stubs.h',
+        'rte_stack_pile.h',
 )
lib/stack/rte_stack.c +17 -1
index 4c78fe4b4b..a4bbf8a4d7 100644
@@ -1,5 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright(c) 2019 Intel Corporation
+ * Copyright(c) 2026 SmartShare Systems
  */
 
 #include <stdalign.h>
@@ -32,6 +33,8 @@ rte_stack_init(struct rte_stack *s, unsigned int count, uint32_t flags)
 
 	if (flags & RTE_STACK_F_LF)
 		rte_stack_lf_init(s, count);
+	else if (flags & RTE_STACK_F_PILE)
+		rte_stack_pile_init(s, count);
 	else
 		rte_stack_std_init(s);
 }
@@ -41,6 +44,8 @@ rte_stack_get_memsize(unsigned int count, uint32_t flags)
 {
 	if (flags & RTE_STACK_F_LF)
 		return rte_stack_lf_get_memsize(count);
+	else if (flags & RTE_STACK_F_PILE)
+		return rte_stack_pile_get_memsize(count);
 	else
 		return rte_stack_std_get_memsize(count);
 }
@@ -58,7 +63,11 @@ rte_stack_create(const char *name, unsigned int count, int socket_id,
 	unsigned int sz;
 	int ret;
 
-	if (flags & ~(RTE_STACK_F_LF)) {
+	if (flags & ~(RTE_STACK_F_LF | RTE_STACK_F_PILE)) {
+		STACK_LOG_ERR("Unsupported stack flags %#x", flags);
+		return NULL;
+	}
+	if ((flags & RTE_STACK_F_LF) && (flags & RTE_STACK_F_PILE)) {
 		STACK_LOG_ERR("Unsupported stack flags %#x", flags);
 		return NULL;
 	}
@@ -73,6 +82,13 @@ rte_stack_create(const char *name, unsigned int count, int socket_id,
 		return NULL;
 	}
 #endif
+#if !defined(RTE_STACK_PILE_SUPPORTED)
+	if (flags & RTE_STACK_F_PILE) {
+		STACK_LOG_ERR("Pile is not supported on your platform");
+		rte_errno = ENOTSUP;
+		return NULL;
+	}
+#endif
 
 	sz = rte_stack_get_memsize(count, flags);
 
lib/stack/rte_stack.h +79 -0

139 lines not on this page · raw message

lib/stack/rte_stack_lf.h +1 -0
index f2b012cd0e..1ee9330c57 100644
@@ -79,6 +79,7 @@ __rte_stack_lf_pop(struct rte_stack *s, void **obj_table, unsigned int n)
 		return 0;
 
 	/* Pop n used elements */
+	__rte_assume(obj_table != NULL);
 	first = __rte_stack_lf_pop_elems(&s->stack_lf.used,
 					 n, obj_table, &last);
 	if (unlikely(first == NULL))
lib/stack/rte_stack_pile.c new file +35 -0
new file mode 100644
index 0000000000..5884163313
@@ -0,0 +1,35 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2026 SmartShare Systems
+ */
+
+#include "rte_stack.h"
+
+void
+rte_stack_pile_init(struct rte_stack *s, unsigned int count)
+{
+	unsigned int bulk = (count + RTE_STACK_PILE_BULK_SIZE - 1) / RTE_STACK_PILE_BULK_SIZE;
+	struct rte_stack_pile_bulk_elem *bulk_elems =
+			(struct rte_stack_pile_bulk_elem *)(s->stack_pile.elems);
+	struct rte_stack_lf_elem *solo_elems = (struct rte_stack_lf_elem *)&bulk_elems[bulk];
+	unsigned int i;
+
+	for (i = 0; i < bulk; i++)
+		__rte_stack_pile_bulk_push_elems(&s->stack_pile.free_bulk,
+					  &bulk_elems[i], &bulk_elems[i], 1);
+	for (i = 0; i < count; i++)
+		__rte_stack_lf_push_elems(&s->stack_pile.free_solo,
+					  &solo_elems[i], &solo_elems[i], 1);
+}
+
+ssize_t
+rte_stack_pile_get_memsize(unsigned int count)
+{
+	unsigned int bulk = (count + RTE_STACK_PILE_BULK_SIZE - 1) / RTE_STACK_PILE_BULK_SIZE;
+	ssize_t sz = offsetof(struct rte_stack, stack_pile.elems);
+	sz += bulk * sizeof(struct rte_stack_pile_bulk_elem);
+	sz += count * sizeof(struct rte_stack_lf_elem);
+	sz = RTE_CACHE_LINE_ROUNDUP(sz);
+	sz += RTE_CACHE_GUARD_LINES * RTE_CACHE_LINE_SIZE;
+
+	return sz;
+}
lib/stack/rte_stack_pile.h new file +334 -0

334 lines not on this page · raw message

Signature · 1 line
2.43.0
#16 lore raw

Re: [RFC PATCH v11] pile stack and mempool driver

Quoted 33 lines · On Tue, 11 Aug 2026 12:35:41 +0000 Morten Brørup <[email protected]>…

Early submission of:
- A new "pile" stack-like implementation using the Stack API, and
- an accompanying "pile" mempool driver.

For CI test and community feedback.

Needless to say, this must be separated into multiple independent series of patches.
And release notes must be added.
For now, I'm submitting a snapshot of work in progress.

The "pile" somewhat resembles the lock-free stack, but operates on bulks (arrays) of objects, to significantly reduce linked list traversal.
With the pile's default bulk size of 32 objects, a mempool cache flush/refill traverses a linked list of only 16 elements, whereas the lock-free stack would traverse a linked list of 512 elements.

Some performance numbers from mempool_perf_autotest_2cores, all with cache=1024 cores=2 n_keep=32768:

start performance test (using ring_mp_mc, with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 753985338 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 755805913

start performance test for lf_stack (with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 29132352 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 29276708

start performance test for pile (with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 560159479 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 557910933

Quoted 7 lines

Hat tip to Bruce for bringing attention to the ring not being the optimal mempool driver!

Note: The GitHub "mini" tests don't include the "pile" mempool driver, and are expected to fail.

Right but the 32 bit code fails at run time.

MEMPOOL: Cache size 250 not divisible by 32, rounding down to 224.
STACK: rte_stack_create(): Pile is not supported on your platform

Should be possible to correct this at compile. Like if pile is not available on this platform, something else should be the default.

The CI failures indicate places where it is likely to be a problem for users.

#17 lore raw

RE: [RFC PATCH v11] pile stack and mempool driver

Quoted 44 lines

From: Stephen Hemminger [mailto:[email protected]]
Sent: Tuesday, 11 August 2026 17.25

Quoted 29 lines · On Tue, 11 Aug 2026 12:35:41 +0000 Morten Brørup <[email protected]>…

Early submission of:
- A new "pile" stack-like implementation using the Stack API, and
- an accompanying "pile" mempool driver.

For CI test and community feedback.

Needless to say, this must be separated into multiple independent series of patches.
And release notes must be added.
For now, I'm submitting a snapshot of work in progress.

The "pile" somewhat resembles the lock-free stack, but operates on bulks (arrays) of objects, to significantly reduce linked list traversal.
With the pile's default bulk size of 32 objects, a mempool cache flush/refill traverses a linked list of only 16 elements, whereas the lock-free stack would traverse a linked list of 512 elements.

Some performance numbers from mempool_perf_autotest_2cores, all with cache=1024 cores=2 n_keep=32768:

start performance test (using ring_mp_mc, with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 753985338 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 755805913

start performance test for lf_stack (with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 29132352 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 29276708

Quoted 8 lines

start performance test for pile (with cache) n_get_bulk= 64 n_put_bulk= 64 constant_n=0 rate_persec= 560159479 n_get_bulk=256 n_put_bulk=256 constant_n=0 rate_persec= 557910933

Hat tip to Bruce for bringing attention to the ring not being the optimal mempool driver!

Note: The GitHub "mini" tests don't include the "pile" mempool

driver,

Quoted 8 lines
Quoted 3 lines

and are expected to fail.

Right but the 32 bit code fails at run time.

MEMPOOL: Cache size 250 not divisible by 32, rounding down to 224.
STACK: rte_stack_create(): Pile is not supported on your platform

The "pile" and "lf_stack" stack types (and mempool drivers) rely on a 128-bit CAS instruction, which is only available on x86_64 and arm64, so asking for one of those mempool drivers will fail on 32-bit architectures.

Quoted 3 lines

Should be possible to correct this at compile. Like if pile is not available on this platform, something else should be the default.

Yep. The default will remain the "ring" mempool driver.
I only temporarily selected the "pile" driver as the default to ensure that it got properly tested by the CI.

Quoted 3 lines

The CI failures indicate places where it is likely to be a problem for users.

Other CI failures are related to "mini" builds.
The "mini" builds only include the "ring" mempool driver, so selecting the "pile" mempool driver naturally fails:
MEMPOOL: Unknown mempool_ops <pile>, of 6 ops registered

I think v11 has brought sufficient CI/AI feedback to split it into individual patch series for the "pile" stack/driver and the mempool optimizations. Will proceed with that now.

Thank you for the advanced AI reviews, Stephen; they were helpful.

-Morten