Skip to contentSkip to conversation

app/testpmd: cross NUMA support

PATCH Superseded Maayan Kashani 2 participants 3 messages Jun 17, 2025 – Jul 26, 2025

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 app/testpmd: cross NUMA support

Cross NUMA support means that
if the current NUMA is out of memory,
use another available NUMA memory.

Replace socket id specific initializations with SOCKET_ID_ANY needed for testpmd init when --no-numa flag is set.

---
 app/test-pmd/testpmd.c      | 8 ++++----
 lib/ethdev/ethdev_private.c | 2 +-
 lib/pdump/rte_pdump.c       | 2 +-
 3 files changed, 6 insertions(+), 6 deletions(-)
app/test-pmd/testpmd.c +4 -4
index b00c93c4536..66ebdee6978 100644
@@ -1729,8 +1729,7 @@ init_config(void)
 			mempools[i] = mbuf_pool_create
 					(mbuf_data_size[i],
 					 nb_mbuf_per_pool,
-					 socket_num == UMA_NO_CONFIG ?
-					 0 : socket_num, i);
+					 SOCKET_ID_ANY, i);
 	}
 
 	init_port_config();
@@ -3058,7 +3057,8 @@ start_port(portid_t pid)
 				} else {
 					struct rte_mempool *mp =
 						mbuf_pool_find
-							(port->socket_id, 0);
+							((numa_support ? port->socket_id :
+							(unsigned int)SOCKET_ID_ANY), 0);
 					if (mp == NULL) {
 						fprintf(stderr,
 							"Failed to setup RX queue: No mempool allocation on the socket %d\n",
@@ -4484,7 +4484,7 @@ main(int argc, char** argv)
 
 #ifdef RTE_LIB_METRICS
 	/* Init metrics library */
-	rte_metrics_init(rte_socket_id());
+	rte_metrics_init(SOCKET_ID_ANY);
 #endif
 
 #ifdef RTE_LIB_LATENCYSTATS
lib/ethdev/ethdev_private.c +1 -1
index b96d992ea12..285d377d91f 100644
@@ -343,7 +343,7 @@ eth_dev_shared_data_prepare(void)
 		/* Allocate port data and ownership shared memory. */
 		mz = rte_memzone_reserve(MZ_RTE_ETH_DEV_DATA,
 				sizeof(*eth_dev_shared_data),
-				rte_socket_id(), flags);
+				SOCKET_ID_ANY, flags);
 		if (mz == NULL) {
 			RTE_ETHDEV_LOG_LINE(ERR, "Cannot allocate ethdev shared data");
 			goto out;
lib/pdump/rte_pdump.c +1 -1
index 79b20ce59b2..ba75b828f2e 100644
@@ -426,7 +426,7 @@ rte_pdump_init(void)
 	int ret;
 
 	mz = rte_memzone_reserve(MZ_RTE_PDUMP_STATS, sizeof(*pdump_stats),
-				 rte_socket_id(), 0);
+				 SOCKET_ID_ANY, 0);
 	if (mz == NULL) {
 		PDUMP_LOG_LINE(ERR, "cannot allocate pdump statistics");
 		rte_errno = ENOMEM;
Signature · 1 line
2.21.0
#1 lore raw

PATCH v2 1/1 app/testpmd: cross NUMA support

Cross NUMA support means that
if the current NUMA is out of memory,
use another available NUMA memory.

This patch is meant to support cross NUMA in testpmd, when disabling gro, gro fix for cross NUMA will come later.

Replace pdump, metrics and dev shared data
memory allocations to any socket
instead of port socket id.

Replace mbuf pools allocations to any socket
when --no-numa flag is set.

---
 app/test-pmd/testpmd.c      | 8 ++++----
 lib/ethdev/ethdev_private.c | 2 +-
 lib/pdump/rte_pdump.c       | 2 +-
 3 files changed, 6 insertions(+), 6 deletions(-)
app/test-pmd/testpmd.c +4 -4
index 344c256c050..92a5d581bb9 100644
@@ -1729,8 +1729,7 @@ init_config(void)
 			mempools[i] = mbuf_pool_create
 					(mbuf_data_size[i],
 					 nb_mbuf_per_pool,
-					 socket_num == UMA_NO_CONFIG ?
-					 0 : socket_num, i);
+					 SOCKET_ID_ANY, i);
 	}
 
 	init_port_config();
@@ -3057,7 +3056,8 @@ start_port(portid_t pid)
 				} else {
 					struct rte_mempool *mp =
 						mbuf_pool_find
-							(port->socket_id, 0);
+							((numa_support ? port->socket_id :
+							(unsigned int)SOCKET_ID_ANY), 0);
 					if (mp == NULL) {
 						fprintf(stderr,
 							"Failed to setup RX queue: No mempool allocation on the socket %d\n",
@@ -4476,7 +4476,7 @@ main(int argc, char** argv)
 
 #ifdef RTE_LIB_METRICS
 	/* Init metrics library */
-	rte_metrics_init(rte_socket_id());
+	rte_metrics_init(SOCKET_ID_ANY);
 #endif
 
 #ifdef RTE_LIB_LATENCYSTATS
lib/ethdev/ethdev_private.c +1 -1
index b96d992ea12..285d377d91f 100644
@@ -343,7 +343,7 @@ eth_dev_shared_data_prepare(void)
 		/* Allocate port data and ownership shared memory. */
 		mz = rte_memzone_reserve(MZ_RTE_ETH_DEV_DATA,
 				sizeof(*eth_dev_shared_data),
-				rte_socket_id(), flags);
+				SOCKET_ID_ANY, flags);
 		if (mz == NULL) {
 			RTE_ETHDEV_LOG_LINE(ERR, "Cannot allocate ethdev shared data");
 			goto out;
lib/pdump/rte_pdump.c +1 -1
index 79b20ce59b2..ba75b828f2e 100644
@@ -426,7 +426,7 @@ rte_pdump_init(void)
 	int ret;
 
 	mz = rte_memzone_reserve(MZ_RTE_PDUMP_STATS, sizeof(*pdump_stats),
-				 rte_socket_id(), 0);
+				 SOCKET_ID_ANY, 0);
 	if (mz == NULL) {
 		PDUMP_LOG_LINE(ERR, "cannot allocate pdump statistics");
 		rte_errno = ENOMEM;
Signature · 1 line
2.21.0
#2 lore raw
Quoted 7 lines · Maayan Kashani

Cross NUMA support means that
if the current NUMA is out of memory,
use another available NUMA memory.

This patch is meant to support cross NUMA in testpmd, when disabling gro, gro fix for cross NUMA will come later.

Quoted 6 lines

Replace pdump, metrics and dev shared data
memory allocations to any socket
instead of port socket id.

Replace mbuf pools allocations to any socket
when --no-numa flag is set.

I think that the patch should be split into few patches. First of all it would allow to explain in the description why SOCKET_ID_ANY is the right choice here since it is not always the right choice. So, explanations are required case by case. Second, it would allow to use correct prefix in the summary.

Summary should hint which problem is solved, what is the goal of the patch, since "cross NUMA support" is not the goal.

As I understand the main idea behind that in general rte_socket_id() should not be used if we don't want to strictly bind memory allocation to the current socket. SOCKET_ID_ANY does the same as a priority (as far as I can see), but allows to fallback to memory which belongs to other NUMA nodes.

Quoted 67 lines
---
  app/test-pmd/testpmd.c      | 8 ++++----
  lib/ethdev/ethdev_private.c | 2 +-
  lib/pdump/rte_pdump.c       | 2 +-
  3 files changed, 6 insertions(+), 6 deletions(-)
app/test-pmd/testpmd.c +4 -4
index 344c256c050..92a5d581bb9 100644
@@ -1729,8 +1729,7 @@ init_config(void)
  			mempools[i] = mbuf_pool_create
  					(mbuf_data_size[i],
  					 nb_mbuf_per_pool,
-					 socket_num == UMA_NO_CONFIG ?
-					 0 : socket_num, i);
+					 SOCKET_ID_ANY, i);
  	}
 
  	init_port_config();
@@ -3057,7 +3056,8 @@ start_port(portid_t pid)
  				} else {
  					struct rte_mempool *mp =
  						mbuf_pool_find
-							(port->socket_id, 0);
+							((numa_support ? port->socket_id :
+							(unsigned int)SOCKET_ID_ANY), 0);
  					if (mp == NULL) {
  						fprintf(stderr,
  							"Failed to setup RX queue: No mempool allocation on the socket %d\n",
@@ -4476,7 +4476,7 @@ main(int argc, char** argv)
 
  #ifdef RTE_LIB_METRICS
  	/* Init metrics library */
-	rte_metrics_init(rte_socket_id());
+	rte_metrics_init(SOCKET_ID_ANY);
  #endif
 
  #ifdef RTE_LIB_LATENCYSTATS
lib/ethdev/ethdev_private.c +1 -1
index b96d992ea12..285d377d91f 100644
@@ -343,7 +343,7 @@ eth_dev_shared_data_prepare(void)
  		/* Allocate port data and ownership shared memory. */
  		mz = rte_memzone_reserve(MZ_RTE_ETH_DEV_DATA,
  				sizeof(*eth_dev_shared_data),
-				rte_socket_id(), flags);
+				SOCKET_ID_ANY, flags);
  		if (mz == NULL) {
  			RTE_ETHDEV_LOG_LINE(ERR, "Cannot allocate ethdev shared data");
  			goto out;
lib/pdump/rte_pdump.c +1 -1
index 79b20ce59b2..ba75b828f2e 100644
@@ -426,7 +426,7 @@ rte_pdump_init(void)
  	int ret;
 
  	mz = rte_memzone_reserve(MZ_RTE_PDUMP_STATS, sizeof(*pdump_stats),
-				 rte_socket_id(), 0);
+				 SOCKET_ID_ANY, 0);
  	if (mz == NULL) {
  		PDUMP_LOG_LINE(ERR, "cannot allocate pdump statistics");
  		rte_errno = ENOMEM;