app/testpmd: add GPU memory option in iofwd engine

-----Original Message-----
From: [email protected] <[email protected]>
Sent: Thursday, November 11, 2021 23:42
To: [email protected]
Cc: Elena Agostini <[email protected]>
Subject: [PATCH v2 1/1] app/testpmd: add GPU memory option in iofwd engine

From: eagostini <[email protected]>

Quoted 1 line

app/testpmd: add GPU memory option in iofwd

This patch introduces GPU memory in testpmd through the gpudev library. Testpmd can be used for network benchmarks when using GPU memory instead of regular CPU memory to send and receive packets. This option is currently limited to iofwd engine.

app/test-pmd/cmdline.c +1 -0

 *item_name, unsigned int max_items,
 	unsigned int j;
 	int value_ok;
 	char c;
+	int gpu_mbuf = 0;
 
 	/*

app/test-pmd/cmdline.c +0 -0

 	 * First parse all items in the list and store their value.

app/test-pmd/cmdline.c +7 -0

+			/*
+			 * When this flag is set, mbufs for this segment
+			 * will be created on GPU memory.
+			 */
+			gpu_mbuf = 1;
+			continue;
+		}
 		if (c != ',') {
 			fprintf(stderr, "character %c is not a decimal digit\n",

c);

app/test-pmd/cmdline.c +2 -0

 			return 0;

@@ -3640,6 +3649,8 @@ parse_item_list(const char *str, const char
 *item_name, unsigned int max_items,
 			parsed_items[nb_item] = value;
 			value_ok = 0;
 			value = 0;
+			mbuf_mem_types[nb_item] = gpu_mbuf ?
 MBUF_MEM_GPU : MBUF_MEM_CPU;
+			gpu_mbuf = 0;
 		}

app/test-pmd/cmdline.c +3 -0

 		nb_item++;
 	}

@@ -3648,6 +3659,9 @@ parse_item_list(const char *str, const char
 *item_name, unsigned int max_items,
 			item_name, nb_item + 1, max_items);
 		return 0;
 	}
+
+	mbuf_mem_types[nb_item] = gpu_mbuf ? MBUF_MEM_GPU :
 MBUF_MEM_CPU;
+
 	parsed_items[nb_item++] = value;

app/test-pmd/cmdline.c +0 -0

 	if (! check_unique_values)
 		return nb_item;

app/test-pmd/meson.build index renamed from app/test-pmd/meson.build +0 -0

d5df52c470..5c8ca68c9d 100644

app/test-pmd/meson.build +2 -1

@@ -32,7 +32,7 @@ if dpdk_conf.has('RTE_HAS_JANSSON')
     ext_deps += jansson_dep
 endif
 
-deps += ['ethdev', 'gro', 'gso', 'cmdline', 'metrics', 'bus_pci']
+deps += ['ethdev', 'gro', 'gso', 'cmdline', 'metrics', 'bus_pci',
+'gpudev']
 if dpdk_conf.has('RTE_CRYPTO_SCHEDULER')
     deps += 'crypto_scheduler'
 endif

app/test-pmd/parameters.c index renamed from app/test-pmd/parameters.c +0 -0

0974b0a38f..d41f7f220b 100644

app/test-pmd/parameters.c +1 -0

+
 	printf("  --total-num-mbufs=N: set the number of mbufs to be

allocated "

app/test-pmd/parameters.c +0 -0

 	       "in mbuf pools.\n");
 	printf("  --max-pkt-len=N: set the maximum size of packet to N

bytes.\n"); @@ -595,6 +598,7 @@ launch_args_parse(int argc, char** argv)

app/test-pmd/parameters.c +5 -0

 	struct rte_eth_dev_info dev_info;
 	uint16_t rec_nb_pkts;
 	int ret;
+	uint32_t idx = 0;
 
 	static struct option lgopts[] = {
 		{ "help",			0, 0, 0 },

@@ -1538,4 +1542,11 @@ launch_args_parse(int argc, char** argv)
 				  "ignored\n");
 		mempool_flags = 0;
 	}
+
+	for (idx = 0; idx < mbuf_data_size_n; idx++) {
+		if (mbuf_mem_types[idx] == MBUF_MEM_GPU &&
 strcmp(cur_fwd_eng->fwd_mode_name, "io") != 0) {
+			fprintf(stderr, "GPU memory mbufs can be used with

iofwd engine only\n");

app/test-pmd/parameters.c +1 -0

+			rte_exit(EXIT_FAILURE, "Command line is

incorrect\n");

}

app/test-pmd/testpmd.c index renamed from app/test-pmd/testpmd.c +0 -0

a66dfb297c..1af235c4d8 100644

app/test-pmd/testpmd.c +0 -0

@@ -205,6 +205,12 @@ uint32_t mbuf_data_size_n = 1; /* Number of

specified mbuf sizes. */ uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = {

app/test-pmd/testpmd.c +6 -0

 	DEFAULT_MBUF_DATA_SIZE
 }; /**< Mbuf data space size. */
+
+/* Mbuf memory types. */
+enum mbuf_mem_type mbuf_mem_types[MAX_SEGS_BUFFER_SPLIT];
+/* Pointers to external memory allocated for mempools. */ uintptr_t
+mempools_ext_ptr[MAX_SEGS_BUFFER_SPLIT];
+
 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
                                       * specified on command-line. */  uint16_t

stats_period; /**< Period to show statistics (disabled by default) */ @@ - 543,6 +549,12 @@ int proc_id;
*/
unsigned int num_procs = 1;

+
static void
eth_rx_metadata_negotiate_mp(uint16_t port_id) { @@ -1103,6 +1115,79
@@ setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,

app/test-pmd/testpmd.c +21 -0

 	return ext_num;
 }
 
+static struct rte_mempool *
+gpu_mbuf_pool_create(uint16_t mbuf_seg_size, unsigned int nb_mbuf,
+			unsigned int socket_id, uint16_t port_id,
+			int gpu_id, uintptr_t * mp_addr)
+{
+	int ret = 0;
+	char pool_name[RTE_MEMPOOL_NAMESIZE];
+	struct rte_eth_dev_info dev_info;
+	struct rte_mempool *rte_mp = NULL;
+	struct rte_pktmbuf_extmem gpu_mem;
+	struct rte_gpu_info ginfo;
+	uint8_t gpu_page_shift = 16;
+	uint32_t gpu_page_size = (1UL << gpu_page_shift);
+
+	ret = eth_dev_info_get_print_err(port_id, &dev_info);
+	if (ret != 0)
+		rte_exit(EXIT_FAILURE,
+			"Failed to get device info for port %d\n",
+			port_id);
+
+	mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name),

port_id, MBUF_MEM_GPU);

app/test-pmd/testpmd.c +11 -0

+	if (!is_proc_primary()) {
+		rte_mp = rte_mempool_lookup(pool_name);
+		if (rte_mp == NULL)
+			rte_exit(EXIT_FAILURE,
+				"Get mbuf pool for socket %u failed: %s\n",
+				socket_id, rte_strerror(rte_errno));
+		return rte_mp;
+	}
+
+	if (rte_gpu_info_get(gpu_id, &ginfo))
+		rte_exit(EXIT_FAILURE, "Can't retrieve info about GPU %d -

bye\n",

app/test-pmd/testpmd.c +4 -0

+gpu_id);
+
+	TESTPMD_LOG(INFO,
+		"create a new mbuf pool <%s>: n=%u, size=%u, socket=%u

GPU device=%s\n",

app/test-pmd/testpmd.c +3 -0

+		pool_name, nb_mbuf, mbuf_seg_size, socket_id, ginfo.name);
+
+	/* Create an external memory mempool using memory allocated on

the

app/test-pmd/testpmd.c +4 -0

+GPU. */
+
+	gpu_mem.elt_size = RTE_MBUF_DEFAULT_BUF_SIZE;
+	gpu_mem.buf_len = RTE_ALIGN_CEIL(nb_mbuf * gpu_mem.elt_size,

gpu_page_size);

app/test-pmd/testpmd.c +5 -0

+	gpu_mem.buf_iova = RTE_BAD_IOVA;
+
+	gpu_mem.buf_ptr = rte_gpu_mem_alloc(gpu_id, gpu_mem.buf_len);
+	if (gpu_mem.buf_ptr == NULL)
+		rte_exit(EXIT_FAILURE, "Could not allocate GPU device

memory\n");

app/test-pmd/testpmd.c +2 -0

+
+	ret = rte_extmem_register(gpu_mem.buf_ptr, gpu_mem.buf_len,

NULL, gpu_mem.buf_iova, gpu_page_size);

app/test-pmd/testpmd.c +2 -0

+	if (ret)
+		rte_exit(EXIT_FAILURE, "Unable to register addr 0x%p, ret

%d\n",

app/test-pmd/testpmd.c +8 -0

+gpu_mem.buf_ptr, ret);
+
+	uint16_t pid = 0;
+
+	RTE_ETH_FOREACH_DEV(pid)
+	{
+		ret = rte_dev_dma_map(dev_info.device, gpu_mem.buf_ptr,
+					  gpu_mem.buf_iova,

gpu_mem.buf_len);

app/test-pmd/testpmd.c +2 -0

+		if (ret) {
+			rte_exit(EXIT_FAILURE, "Unable to DMA map addr

0x%p for device %s\n",

app/test-pmd/testpmd.c +1 -0

+					 gpu_mem.buf_ptr, dev_info.device-

Quoted 1 line

name);

app/test-pmd/testpmd.c +4 -0

+		}
+	}
+
+	rte_mp = rte_pktmbuf_pool_create_extbuf(pool_name, nb_mbuf,

mb_mempool_cache, 0, mbuf_seg_size, socket_id, &gpu_mem, 1);

app/test-pmd/testpmd.c +2 -0

+	if (rte_mp == NULL) {
+		rte_exit(EXIT_FAILURE, "Creation of GPU mempool <%s>

failed\n", pool_name);

app/test-pmd/testpmd.c +9 -2

+	}
+
+	*mp_addr = (uintptr_t) gpu_mem.buf_ptr;
+
+	return rte_mp;
+}
+
 /*
  * Configuration initialisation done once at init time.
  */

@@ -1117,7 +1202,7 @@ mbuf_pool_create(uint16_t mbuf_seg_size,
 unsigned nb_mbuf,
 
 	mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;  #endif
-	mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name),
 size_idx);
+	mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name),
 size_idx,
+MBUF_MEM_CPU);
 	if (!is_proc_primary()) {
 		rte_mp = rte_mempool_lookup(pool_name);
 		if (rte_mp == NULL)

@@ -1700,19 +1785,42 @@ init_config(void)
 
 		for (i = 0; i < num_sockets; i++)
 			for (j = 0; j < mbuf_data_size_n; j++)
-				mempools[i * MAX_SEGS_BUFFER_SPLIT + j]

=

app/test-pmd/testpmd.c +2 -3

-					mbuf_pool_create(mbuf_data_size[j],
-							  nb_mbuf_per_pool,
-							  socket_ids[i], j);
+			{
+				if (mbuf_mem_types[j] == MBUF_MEM_GPU)

{

app/test-pmd/testpmd.c +2 -0

+					if (rte_gpu_count_avail() == 0)
+						rte_exit(EXIT_FAILURE, "No

GPU device available.\n");

app/test-pmd/testpmd.c +2 -0

+
+					mempools[i *

MAX_SEGS_BUFFER_SPLIT + j] =
+
gpu_mbuf_pool_create(mbuf_data_size[j],

+
nb_mbuf_per_pool,
+

		 socket_ids[i], j, gpu_id,
+
		 &(mempools_ext_ptr[j]));

app/test-pmd/testpmd.c +2 -0

+				} else {
+					mempools[i *

MAX_SEGS_BUFFER_SPLIT + j] =
+
mbuf_pool_create(mbuf_data_size[j],
+
nb_mbuf_per_pool,

app/test-pmd/testpmd.c +1 -0

+								socket_ids[i],

j);

app/test-pmd/testpmd.c +5 -5

+				}
+			}
 	} else {
 		uint8_t i;
 
 		for (i = 0; i < mbuf_data_size_n; i++)
-			mempools[i] = mbuf_pool_create
-					(mbuf_data_size[i],
-					 nb_mbuf_per_pool,
-					 socket_num == UMA_NO_CONFIG ?
-					 0 : socket_num, i);
+		{
+			if (mbuf_mem_types[i] == MBUF_MEM_GPU) {
+				mempools[i] =

gpu_mbuf_pool_create(mbuf_data_size[i],
+
nb_mbuf_per_pool,
+

			   socket_num == UMA_NO_CONFIG ? 0 : socket_num,
+
			   i, gpu_id,
+
			   &(mempools_ext_ptr[i]));

app/test-pmd/testpmd.c +2 -0

+			} else {
+				mempools[i] =

mbuf_pool_create(mbuf_data_size[i],

app/test-pmd/testpmd.c +2 -0

+							nb_mbuf_per_pool,
+							socket_num ==

UMA_NO_CONFIG ?

app/test-pmd/testpmd.c +8 -1

+							0 : socket_num, i);
+			}
+		}
+
 	}
 
 	init_port_config();

@@ -3415,8 +3523,11 @@ pmd_test_exit(void)
 		}
 	}
 	for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
-		if (mempools[i])
+		if (mempools[i]) {
 			mempool_free_mp(mempools[i]);
+			if (mbuf_mem_types[i] == MBUF_MEM_GPU)
+				rte_gpu_mem_free(gpu_id, (void
 *)mempools_ext_ptr[i]);
+		}
 	}
 	free(xstats_display);

app/test-pmd/testpmd.h index renamed from app/test-pmd/testpmd.h +0 -0

669ce1e87d..9919044372 100644

app/test-pmd/testpmd.h +1 -0

@@ -12,6 +12,7 @@
 #include <rte_gro.h>
 #include <rte_gso.h>
 #include <rte_os_shim.h>
+#include <rte_gpudev.h>
 #include <cmdline.h>
 #include <sys/queue.h>
 #ifdef RTE_HAS_JANSSON

@@ -474,6 +475,11 @@ extern uint8_t dcb_config;  extern uint32_t

mbuf_data_size_n; extern uint16_t
mbuf_data_size[MAX_SEGS_BUFFER_SPLIT];
/**< Mbuf data space size. */

app/test-pmd/testpmd.h +5 -0

+enum mbuf_mem_type {
+	MBUF_MEM_CPU,
+	MBUF_MEM_GPU
+};
+extern enum mbuf_mem_type

mbuf_mem_types[MAX_SEGS_BUFFER_SPLIT];
extern uint32_t param_total_num_mbufs;

extern uint16_t stats_period;

app/test-pmd/testpmd.h +5 -3

@@ -717,14 +723,16 @@ current_fwd_lcore(void)
 /* Mbuf Pools */
 static inline void
 mbuf_poolname_build(unsigned int sock_id, char *mp_name,
-		    int name_size, uint16_t idx)
+		    int name_size, uint16_t idx, enum mbuf_mem_type
 mem_type)
 {
+	const char *suffix = mem_type == MBUF_MEM_GPU ? "_gpu" : "";
+
 	if (!idx)
 		snprintf(mp_name, name_size,
-			 MBUF_POOL_NAME_PFX "_%u", sock_id);
+			 MBUF_POOL_NAME_PFX "_%u%s", sock_id, suffix);
 	else
 		snprintf(mp_name, name_size,
-			 MBUF_POOL_NAME_PFX "_%hu_%hu",
 (uint16_t)sock_id, idx);
+			 MBUF_POOL_NAME_PFX "_%hu_%hu%s",
 (uint16_t)sock_id, idx, suffix);

}

static inline struct rte_mempool *

app/test-pmd/testpmd.h +2 -1

@@ -732,7 +740,7 @@ mbuf_pool_find(unsigned int sock_id, uint16_t idx)  {
 	char pool_name[RTE_MEMPOOL_NAMESIZE];
 
-	mbuf_poolname_build(sock_id, pool_name, sizeof(pool_name), idx);
+	mbuf_poolname_build(sock_id, pool_name, sizeof(pool_name), idx,
+mbuf_mem_types[idx]);
 	return rte_mempool_lookup((const char *)pool_name);  }
 

doc/guides/testpmd_app_ug/run_app.rst +0 -0

index 30edef07ea..ede7b79abb 100644

Could you, please, set "Author" correctly - "Elena Agostini <[email protected]>"? Otherwise, we see in the git log:

"Author: eagostini <[email protected]>"

Compare with:
"Author: Bing Zhao <[email protected]>" "Author: Viacheslav Ovsiienko <[email protected]>"

Also, please, see the codestyle issues, too many code lines far beyond 100 chars. Lines like this:

app/test-pmd/parameters.c +1 -0

+		if (mbuf_mem_types[idx] == MBUF_MEM_GPU && strcmp(cur_fwd_eng->fwd_mode_name, "io") != 0) {

can be easily be splitted.

Quoted 1 line

Quoted 1 line

app/testpmd: add GPU memory option in iofwd

As I see from the patch - it adds the new mbuf pool type (residing on GPU memory). May be, we should reword the title?
" app/testpmd: add GPU memory option for mbuf pools"

Quoted 5 lines

This patch introduces GPU memory in testpmd through the gpudev library. Testpmd can be used for network benchmarks when using GPU memory instead of regular CPU memory to send and receive packets. This option is currently limited to iofwd engine.

Why? Because iofwd the only mode not touching the mbuf data?
Is it critical for functionality? Is GPU mbuf pool memory accessible from CPU side? I would explain the reasons (for supporting iofwd mode only) in commit message.

app/test-pmd/cmdline.c parse_item_list(const char *str, const char

Quoted 20 lines

Signed-off-by Elena Agostini <[email protected]>

Depends-on: series-19465 ("GPU library")
Depends-on: series-20422 ("common/mlx5: fix external memory pool registration")

---
 app/test-pmd/cmdline.c                |  14 +++
 app/test-pmd/meson.build              |   2 +-
 app/test-pmd/parameters.c             |  13 ++-
 app/test-pmd/testpmd.c                | 133 +++++++++++++++++++++++---
 app/test-pmd/testpmd.h                |  16 +++-
 doc/guides/testpmd_app_ug/run_app.rst |   3 +
 6 files changed, 164 insertions(+), 17 deletions(-)

app/test-pmd/cmdline.c index renamed from app/test-pmd/cmdline.c +0 -0

4f51b259fe..36193bc566 100644

app/test-pmd/cmdline.c +0 -0

@@ -3614,6 +3614,7 @@ parse_item_list(const char *str, const char

The "parse_item_list()" is used to parse the list looking like "number0, number1, ...., numberN" and invoked for "core","port", "txtimes", etc. Not sure all of these params need to handle "g" suffix. We should allow "g" processing only for "mbuf-size". We have "item_name" argument to check whether we are invoked on "mbuf-size".

Quoted 9 lines

app/test-pmd/cmdline.c +1 -0

 *item_name, unsigned int max_items,
 	unsigned int j;
 	int value_ok;
 	char c;
+	int gpu_mbuf = 0;
 
 	/*

app/test-pmd/cmdline.c +0 -0

 	 * First parse all items in the list and store their value.

app/test-pmd/cmdline.c parse_item_list(const char *str, const char · line 3633

Quoted 5 lines

app/test-pmd/cmdline.c +1 -0

@@ -3628,6 +3629,14 @@ parse_item_list(const char *str, const char
 *item_name, unsigned int max_items,
 			value_ok = 1;
 			continue;
 		}
+		if (c == 'g') {

We should check whether "g" is the single char suffix (last char). Otherwise, "20g48" and "g20gggg48" would be also valid values.

Quoted 53 lines

app/test-pmd/cmdline.c +7 -0

+			/*
+			 * When this flag is set, mbufs for this segment
+			 * will be created on GPU memory.
+			 */
+			gpu_mbuf = 1;
+			continue;
+		}
 		if (c != ',') {
 			fprintf(stderr, "character %c is not a decimal digit\n",

c);

app/test-pmd/cmdline.c +2 -0

 			return 0;

@@ -3640,6 +3649,8 @@ parse_item_list(const char *str, const char
 *item_name, unsigned int max_items,
 			parsed_items[nb_item] = value;
 			value_ok = 0;
 			value = 0;
+			mbuf_mem_types[nb_item] = gpu_mbuf ?
 MBUF_MEM_GPU : MBUF_MEM_CPU;
+			gpu_mbuf = 0;
 		}

app/test-pmd/cmdline.c +3 -0

 		nb_item++;
 	}

@@ -3648,6 +3659,9 @@ parse_item_list(const char *str, const char
 *item_name, unsigned int max_items,
 			item_name, nb_item + 1, max_items);
 		return 0;
 	}
+
+	mbuf_mem_types[nb_item] = gpu_mbuf ? MBUF_MEM_GPU :
 MBUF_MEM_CPU;
+
 	parsed_items[nb_item++] = value;

app/test-pmd/cmdline.c +0 -0

 	if (! check_unique_values)
 		return nb_item;

app/test-pmd/meson.build index renamed from app/test-pmd/meson.build +0 -0

d5df52c470..5c8ca68c9d 100644

app/test-pmd/meson.build +2 -1

@@ -32,7 +32,7 @@ if dpdk_conf.has('RTE_HAS_JANSSON')
     ext_deps += jansson_dep
 endif
 
-deps += ['ethdev', 'gro', 'gso', 'cmdline', 'metrics', 'bus_pci']
+deps += ['ethdev', 'gro', 'gso', 'cmdline', 'metrics', 'bus_pci',
+'gpudev']
 if dpdk_conf.has('RTE_CRYPTO_SCHEDULER')
     deps += 'crypto_scheduler'
 endif

app/test-pmd/parameters.c index renamed from app/test-pmd/parameters.c +0 -0

0974b0a38f..d41f7f220b 100644

app/test-pmd/parameters.c usage(char* progname) · line 93

Quoted 8 lines

app/test-pmd/parameters.c +3 -1

@@ -87,7 +87,10 @@ usage(char* progname)
 	       "in NUMA mode.\n");
 	printf("  --mbuf-size=N,[N1[,..Nn]: set the data size of mbuf to "
 	       "N bytes. If multiple numbers are specified the extra pools "
-	       "will be created to receive with packet split features\n");
+	       "will be created to receive with packet split features\n"
+		   "Use 'g' suffix for GPU memory.\n"
+		   "If no or an unrecognized suffix is provided, CPU is
 assumed\n");

Unrecognized suffix? I would emit an error and abort the launch.

Quoted 25 lines

app/test-pmd/parameters.c +1 -0

+
 	printf("  --total-num-mbufs=N: set the number of mbufs to be

allocated "

app/test-pmd/parameters.c +0 -0

 	       "in mbuf pools.\n");
 	printf("  --max-pkt-len=N: set the maximum size of packet to N

bytes.\n"); @@ -595,6 +598,7 @@ launch_args_parse(int argc, char** argv)

app/test-pmd/parameters.c +5 -0

 	struct rte_eth_dev_info dev_info;
 	uint16_t rec_nb_pkts;
 	int ret;
+	uint32_t idx = 0;
 
 	static struct option lgopts[] = {
 		{ "help",			0, 0, 0 },

@@ -1538,4 +1542,11 @@ launch_args_parse(int argc, char** argv)
 				  "ignored\n");
 		mempool_flags = 0;
 	}
+
+	for (idx = 0; idx < mbuf_data_size_n; idx++) {
+		if (mbuf_mem_types[idx] == MBUF_MEM_GPU &&
 strcmp(cur_fwd_eng->fwd_mode_name, "io") != 0) {
+			fprintf(stderr, "GPU memory mbufs can be used with

iofwd engine only\n");

app/test-pmd/parameters.c +1 -0

+			rte_exit(EXIT_FAILURE, "Command line is

incorrect\n");

app/test-pmd/parameters.c

Quoted 2 lines

app/test-pmd/parameters.c +2 -0

+		}
+	}

Please, note, the forwarding mode can be changed from interactive prompt with "set fwd <mode>" command. If iofwd mode is crucial for GPU functionality - we should prevent switching to other modes if GPU pools are engaged.

Quoted 23 lines

}

app/test-pmd/testpmd.c index renamed from app/test-pmd/testpmd.c +0 -0

a66dfb297c..1af235c4d8 100644

app/test-pmd/testpmd.c +0 -0

@@ -205,6 +205,12 @@ uint32_t mbuf_data_size_n = 1; /* Number of

specified mbuf sizes. */ uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = {

app/test-pmd/testpmd.c +6 -0

 	DEFAULT_MBUF_DATA_SIZE
 }; /**< Mbuf data space size. */
+
+/* Mbuf memory types. */
+enum mbuf_mem_type mbuf_mem_types[MAX_SEGS_BUFFER_SPLIT];
+/* Pointers to external memory allocated for mempools. */ uintptr_t
+mempools_ext_ptr[MAX_SEGS_BUFFER_SPLIT];
+
 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
                                       * specified on command-line. */  uint16_t

stats_period; /**< Period to show statistics (disabled by default) */ @@ - 543,6 +549,12 @@ int proc_id;
*/
unsigned int num_procs = 1;

app/test-pmd/testpmd.c int proc_id; · line 556

Quoted 5 lines

app/test-pmd/testpmd.c +5 -0

+/*
+ * In case of GPU memory external mbufs use, for simplicity,
+ * the first GPU device in the list.
+ */
+int gpu_id = 0;

It is assigned with zero and never changes. Support the first GPU only? This limitation should be mentioned in documentation.

Quoted 129 lines

+
static void
eth_rx_metadata_negotiate_mp(uint16_t port_id) { @@ -1103,6 +1115,79
@@ setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,

app/test-pmd/testpmd.c +21 -0

 	return ext_num;
 }
 
+static struct rte_mempool *
+gpu_mbuf_pool_create(uint16_t mbuf_seg_size, unsigned int nb_mbuf,
+			unsigned int socket_id, uint16_t port_id,
+			int gpu_id, uintptr_t * mp_addr)
+{
+	int ret = 0;
+	char pool_name[RTE_MEMPOOL_NAMESIZE];
+	struct rte_eth_dev_info dev_info;
+	struct rte_mempool *rte_mp = NULL;
+	struct rte_pktmbuf_extmem gpu_mem;
+	struct rte_gpu_info ginfo;
+	uint8_t gpu_page_shift = 16;
+	uint32_t gpu_page_size = (1UL << gpu_page_shift);
+
+	ret = eth_dev_info_get_print_err(port_id, &dev_info);
+	if (ret != 0)
+		rte_exit(EXIT_FAILURE,
+			"Failed to get device info for port %d\n",
+			port_id);
+
+	mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name),

port_id, MBUF_MEM_GPU);

app/test-pmd/testpmd.c +11 -0

+	if (!is_proc_primary()) {
+		rte_mp = rte_mempool_lookup(pool_name);
+		if (rte_mp == NULL)
+			rte_exit(EXIT_FAILURE,
+				"Get mbuf pool for socket %u failed: %s\n",
+				socket_id, rte_strerror(rte_errno));
+		return rte_mp;
+	}
+
+	if (rte_gpu_info_get(gpu_id, &ginfo))
+		rte_exit(EXIT_FAILURE, "Can't retrieve info about GPU %d -

bye\n",

app/test-pmd/testpmd.c +4 -0

+gpu_id);
+
+	TESTPMD_LOG(INFO,
+		"create a new mbuf pool <%s>: n=%u, size=%u, socket=%u

GPU device=%s\n",

app/test-pmd/testpmd.c +3 -0

+		pool_name, nb_mbuf, mbuf_seg_size, socket_id, ginfo.name);
+
+	/* Create an external memory mempool using memory allocated on

the

app/test-pmd/testpmd.c +4 -0

+GPU. */
+
+	gpu_mem.elt_size = RTE_MBUF_DEFAULT_BUF_SIZE;
+	gpu_mem.buf_len = RTE_ALIGN_CEIL(nb_mbuf * gpu_mem.elt_size,

gpu_page_size);

app/test-pmd/testpmd.c +5 -0

+	gpu_mem.buf_iova = RTE_BAD_IOVA;
+
+	gpu_mem.buf_ptr = rte_gpu_mem_alloc(gpu_id, gpu_mem.buf_len);
+	if (gpu_mem.buf_ptr == NULL)
+		rte_exit(EXIT_FAILURE, "Could not allocate GPU device

memory\n");

app/test-pmd/testpmd.c +2 -0

+
+	ret = rte_extmem_register(gpu_mem.buf_ptr, gpu_mem.buf_len,

NULL, gpu_mem.buf_iova, gpu_page_size);

app/test-pmd/testpmd.c +2 -0

+	if (ret)
+		rte_exit(EXIT_FAILURE, "Unable to register addr 0x%p, ret

%d\n",

app/test-pmd/testpmd.c +8 -0

+gpu_mem.buf_ptr, ret);
+
+	uint16_t pid = 0;
+
+	RTE_ETH_FOREACH_DEV(pid)
+	{
+		ret = rte_dev_dma_map(dev_info.device, gpu_mem.buf_ptr,
+					  gpu_mem.buf_iova,

gpu_mem.buf_len);

app/test-pmd/testpmd.c +2 -0

+		if (ret) {
+			rte_exit(EXIT_FAILURE, "Unable to DMA map addr

0x%p for device %s\n",

app/test-pmd/testpmd.c +1 -0

+					 gpu_mem.buf_ptr, dev_info.device-

Quoted 1 line

name);

app/test-pmd/testpmd.c +4 -0

+		}
+	}
+
+	rte_mp = rte_pktmbuf_pool_create_extbuf(pool_name, nb_mbuf,

mb_mempool_cache, 0, mbuf_seg_size, socket_id, &gpu_mem, 1);

app/test-pmd/testpmd.c +2 -0

+	if (rte_mp == NULL) {
+		rte_exit(EXIT_FAILURE, "Creation of GPU mempool <%s>

failed\n", pool_name);

app/test-pmd/testpmd.c +9 -2

+	}
+
+	*mp_addr = (uintptr_t) gpu_mem.buf_ptr;
+
+	return rte_mp;
+}
+
 /*
  * Configuration initialisation done once at init time.
  */

@@ -1117,7 +1202,7 @@ mbuf_pool_create(uint16_t mbuf_seg_size,
 unsigned nb_mbuf,
 
 	mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;  #endif
-	mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name),
 size_idx);
+	mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name),
 size_idx,
+MBUF_MEM_CPU);
 	if (!is_proc_primary()) {
 		rte_mp = rte_mempool_lookup(pool_name);
 		if (rte_mp == NULL)

@@ -1700,19 +1785,42 @@ init_config(void)
 
 		for (i = 0; i < num_sockets; i++)
 			for (j = 0; j < mbuf_data_size_n; j++)
-				mempools[i * MAX_SEGS_BUFFER_SPLIT + j]

=

app/test-pmd/testpmd.c +2 -3

-					mbuf_pool_create(mbuf_data_size[j],
-							  nb_mbuf_per_pool,
-							  socket_ids[i], j);
+			{
+				if (mbuf_mem_types[j] == MBUF_MEM_GPU)

{

app/test-pmd/testpmd.c +2 -0

+					if (rte_gpu_count_avail() == 0)
+						rte_exit(EXIT_FAILURE, "No

GPU device available.\n");

app/test-pmd/testpmd.c +2 -0

+
+					mempools[i *

Quoted 3 lines

MAX_SEGS_BUFFER_SPLIT + j] =
+
gpu_mbuf_pool_create(mbuf_data_size[j],

What about GPU/CPU adherence ? Do we create one GPU pool per CPU socket? Disregarding hardware topology at all?
We can mitigate the issue by "--socket-mem/--socket-num" parameter though.

Quoted 121 lines

+
nb_mbuf_per_pool,
+

		 socket_ids[i], j, gpu_id,
+
		 &(mempools_ext_ptr[j]));

app/test-pmd/testpmd.c +2 -0

+				} else {
+					mempools[i *

MAX_SEGS_BUFFER_SPLIT + j] =
+
mbuf_pool_create(mbuf_data_size[j],
+
nb_mbuf_per_pool,

app/test-pmd/testpmd.c +1 -0

+								socket_ids[i],

j);

app/test-pmd/testpmd.c +5 -5

+				}
+			}
 	} else {
 		uint8_t i;
 
 		for (i = 0; i < mbuf_data_size_n; i++)
-			mempools[i] = mbuf_pool_create
-					(mbuf_data_size[i],
-					 nb_mbuf_per_pool,
-					 socket_num == UMA_NO_CONFIG ?
-					 0 : socket_num, i);
+		{
+			if (mbuf_mem_types[i] == MBUF_MEM_GPU) {
+				mempools[i] =

gpu_mbuf_pool_create(mbuf_data_size[i],
+
nb_mbuf_per_pool,
+

			   socket_num == UMA_NO_CONFIG ? 0 : socket_num,
+
			   i, gpu_id,
+
			   &(mempools_ext_ptr[i]));

app/test-pmd/testpmd.c +2 -0

+			} else {
+				mempools[i] =

mbuf_pool_create(mbuf_data_size[i],

app/test-pmd/testpmd.c +2 -0

+							nb_mbuf_per_pool,
+							socket_num ==

UMA_NO_CONFIG ?

app/test-pmd/testpmd.c +8 -1

+							0 : socket_num, i);
+			}
+		}
+
 	}
 
 	init_port_config();

@@ -3415,8 +3523,11 @@ pmd_test_exit(void)
 		}
 	}
 	for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
-		if (mempools[i])
+		if (mempools[i]) {
 			mempool_free_mp(mempools[i]);
+			if (mbuf_mem_types[i] == MBUF_MEM_GPU)
+				rte_gpu_mem_free(gpu_id, (void
 *)mempools_ext_ptr[i]);
+		}
 	}
 	free(xstats_display);

app/test-pmd/testpmd.h index renamed from app/test-pmd/testpmd.h +0 -0

669ce1e87d..9919044372 100644

app/test-pmd/testpmd.h +1 -0

@@ -12,6 +12,7 @@
 #include <rte_gro.h>
 #include <rte_gso.h>
 #include <rte_os_shim.h>
+#include <rte_gpudev.h>
 #include <cmdline.h>
 #include <sys/queue.h>
 #ifdef RTE_HAS_JANSSON

@@ -474,6 +475,11 @@ extern uint8_t dcb_config;  extern uint32_t

mbuf_data_size_n; extern uint16_t
mbuf_data_size[MAX_SEGS_BUFFER_SPLIT];
/**< Mbuf data space size. */

app/test-pmd/testpmd.h +5 -0

+enum mbuf_mem_type {
+	MBUF_MEM_CPU,
+	MBUF_MEM_GPU
+};
+extern enum mbuf_mem_type

mbuf_mem_types[MAX_SEGS_BUFFER_SPLIT];
extern uint32_t param_total_num_mbufs;

extern uint16_t stats_period;

app/test-pmd/testpmd.h +5 -3

@@ -717,14 +723,16 @@ current_fwd_lcore(void)
 /* Mbuf Pools */
 static inline void
 mbuf_poolname_build(unsigned int sock_id, char *mp_name,
-		    int name_size, uint16_t idx)
+		    int name_size, uint16_t idx, enum mbuf_mem_type
 mem_type)
 {
+	const char *suffix = mem_type == MBUF_MEM_GPU ? "_gpu" : "";
+
 	if (!idx)
 		snprintf(mp_name, name_size,
-			 MBUF_POOL_NAME_PFX "_%u", sock_id);
+			 MBUF_POOL_NAME_PFX "_%u%s", sock_id, suffix);
 	else
 		snprintf(mp_name, name_size,
-			 MBUF_POOL_NAME_PFX "_%hu_%hu",
 (uint16_t)sock_id, idx);
+			 MBUF_POOL_NAME_PFX "_%hu_%hu%s",
 (uint16_t)sock_id, idx, suffix);

}

static inline struct rte_mempool *

app/test-pmd/testpmd.h +2 -1

@@ -732,7 +740,7 @@ mbuf_pool_find(unsigned int sock_id, uint16_t idx)  {
 	char pool_name[RTE_MEMPOOL_NAMESIZE];
 
-	mbuf_poolname_build(sock_id, pool_name, sizeof(pool_name), idx);
+	mbuf_poolname_build(sock_id, pool_name, sizeof(pool_name), idx,
+mbuf_mem_types[idx]);
 	return rte_mempool_lookup((const char *)pool_name);  }
 

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 124

Quoted 12 lines

doc/guides/testpmd_app_ug/run_app.rst +3 -0

index 30edef07ea..ede7b79abb 100644
@@ -119,6 +119,9 @@ The command line options are:
     The default value is 2048. If multiple mbuf-size values are specified the
     extra memory pools will be created for allocating mbufs to receive packets
     with buffer splitting features.
+    Providing mbuf size with a 'g' suffix (e.g. ``--mbuf-size=2048g``),
+    will cause the mempool to be created on a GPU memory area allocated.
+    This option is currently limited to iofwd engine with the first GPU.

Mmm, if we have multiple GPUs - we have no way to specify on which one we should allocate the pool. Syntax is not complete☹. Possible we should specify GPU port id after the suffix ? Like 2048g2 ? If port index is omitted we should consider we have the only GPU and check this.

Quoted 453 lines

Could you, please, set "Author" correctly - "Elena Agostini <[email protected]>"? Otherwise, we see in the git log:

"Author: eagostini <[email protected]>"

Compare with:
"Author: Bing Zhao <[email protected]>" "Author: Viacheslav Ovsiienko <[email protected]>"

Also, please, see the codestyle issues, too many code lines far beyond 100 chars. Lines like this:
+ if (mbuf_mem_types[idx] == MBUF_MEM_GPU && strcmp(cur_fwd_eng->fwd_mode_name, "io") != 0) {
can be easily be splitted.

Quoted 1 line

Quoted 1 line

app/testpmd: add GPU memory option in iofwd

As I see from the patch - it adds the new mbuf pool type (residing on GPU memory). May be, we should reword the title?
" app/testpmd: add GPU memory option for mbuf pools"

Quoted 5 lines

This patch introduces GPU memory in testpmd through the gpudev library. Testpmd can be used for network benchmarks when using GPU memory instead of regular CPU memory to send and receive packets. This option is currently limited to iofwd engine.

Why? Because iofwd the only mode not touching the mbuf data?
Is it critical for functionality? Is GPU mbuf pool memory accessible from CPU side? I would explain the reasons (for supporting iofwd mode only) in commit message.

app/test-pmd/cmdline.c parse_item_list(const char *str, const char

Quoted 20 lines

Signed-off-by Elena Agostini <[email protected]>

Depends-on: series-19465 ("GPU library")
Depends-on: series-20422 ("common/mlx5: fix external memory pool registration")

---
 app/test-pmd/cmdline.c                |  14 +++
 app/test-pmd/meson.build              |   2 +-
 app/test-pmd/parameters.c             |  13 ++-
 app/test-pmd/testpmd.c                | 133 +++++++++++++++++++++++---
 app/test-pmd/testpmd.h                |  16 +++-
 doc/guides/testpmd_app_ug/run_app.rst |   3 +
 6 files changed, 164 insertions(+), 17 deletions(-)

app/test-pmd/cmdline.c index renamed from app/test-pmd/cmdline.c +0 -0

4f51b259fe..36193bc566 100644

app/test-pmd/cmdline.c +0 -0

@@ -3614,6 +3614,7 @@ parse_item_list(const char *str, const char

The "parse_item_list()" is used to parse the list looking like "number0, number1, ...., numberN" and invoked for "core","port", "txtimes", etc. Not sure all of these params need to handle "g" suffix. We should allow "g" processing only for "mbuf-size". We have "item_name" argument to check whether we are invoked on "mbuf-size".

Quoted 9 lines

app/test-pmd/cmdline.c +1 -0

 *item_name, unsigned int max_items,
    unsigned int j;
    int value_ok;
    char c;
+   int gpu_mbuf = 0;
 
    /*

* First parse all items in the list and store their value.

app/test-pmd/cmdline.c parse_item_list(const char *str, const char · line 3633

Quoted 5 lines

app/test-pmd/cmdline.c +1 -0

@@ -3628,6 +3629,14 @@ parse_item_list(const char *str, const char
 *item_name, unsigned int max_items,
                    value_ok = 1;
                    continue;
            }
+           if (c == 'g') {

We should check whether "g" is the single char suffix (last char). Otherwise, "20g48" and "g20gggg48" would be also valid values.

Quoted 53 lines

app/test-pmd/cmdline.c +7 -0

+                   /*
+                    * When this flag is set, mbufs for this segment
+                    * will be created on GPU memory.
+                    */
+                   gpu_mbuf = 1;
+                   continue;
+           }
            if (c != ',') {
                    fprintf(stderr, "character %c is not a decimal digit\n",

c);
                    return 0;

app/test-pmd/cmdline.c +2 -0

@@ -3640,6 +3649,8 @@ parse_item_list(const char *str, const char
 *item_name, unsigned int max_items,
                    parsed_items[nb_item] = value;
                    value_ok = 0;
                    value = 0;
+                   mbuf_mem_types[nb_item] = gpu_mbuf ?
 MBUF_MEM_GPU : MBUF_MEM_CPU;
+                   gpu_mbuf = 0;
            }

            nb_item++;
    }

app/test-pmd/cmdline.c +3 -0

@@ -3648,6 +3659,9 @@ parse_item_list(const char *str, const char
 *item_name, unsigned int max_items,
                    item_name, nb_item + 1, max_items);
            return 0;
    }
+
+   mbuf_mem_types[nb_item] = gpu_mbuf ? MBUF_MEM_GPU :
 MBUF_MEM_CPU;
+
    parsed_items[nb_item++] = value;

if (! check_unique_values)
return nb_item;

app/test-pmd/meson.build index renamed from app/test-pmd/meson.build +0 -0

d5df52c470..5c8ca68c9d 100644

app/test-pmd/meson.build +2 -1

@@ -32,7 +32,7 @@ if dpdk_conf.has('RTE_HAS_JANSSON')
     ext_deps += jansson_dep
 endif
 
-deps += ['ethdev', 'gro', 'gso', 'cmdline', 'metrics', 'bus_pci']
+deps += ['ethdev', 'gro', 'gso', 'cmdline', 'metrics', 'bus_pci',
+'gpudev']
 if dpdk_conf.has('RTE_CRYPTO_SCHEDULER')
     deps += 'crypto_scheduler'
 endif

app/test-pmd/parameters.c index renamed from app/test-pmd/parameters.c +0 -0

0974b0a38f..d41f7f220b 100644

app/test-pmd/parameters.c usage(char* progname) · line 93

Quoted 8 lines

app/test-pmd/parameters.c +3 -1

@@ -87,7 +87,10 @@ usage(char* progname)
           "in NUMA mode.\n");
    printf("  --mbuf-size=N,[N1[,..Nn]: set the data size of mbuf to "
           "N bytes. If multiple numbers are specified the extra pools "
-          "will be created to receive with packet split features\n");
+          "will be created to receive with packet split features\n"
+              "Use 'g' suffix for GPU memory.\n"
+              "If no or an unrecognized suffix is provided, CPU is
 assumed\n");

Unrecognized suffix? I would emit an error and abort the launch.

Quoted 25 lines

app/test-pmd/parameters.c +1 -0

+
    printf("  --total-num-mbufs=N: set the number of mbufs to be

allocated "
"in mbuf pools.\n");
printf(" --max-pkt-len=N: set the maximum size of packet to N
bytes.\n"); @@ -595,6 +598,7 @@ launch_args_parse(int argc, char** argv)

    struct rte_eth_dev_info dev_info;
    uint16_t rec_nb_pkts;
    int ret;

app/test-pmd/parameters.c +5 -0

+   uint32_t idx = 0;
 
    static struct option lgopts[] = {
            { "help",                       0, 0, 0 },

@@ -1538,4 +1542,11 @@ launch_args_parse(int argc, char** argv)
                              "ignored\n");
            mempool_flags = 0;
    }
+
+   for (idx = 0; idx < mbuf_data_size_n; idx++) {
+           if (mbuf_mem_types[idx] == MBUF_MEM_GPU &&
 strcmp(cur_fwd_eng->fwd_mode_name, "io") != 0) {
+                   fprintf(stderr, "GPU memory mbufs can be used with

iofwd engine only\n");
+ rte_exit(EXIT_FAILURE, "Command line is
incorrect\n");

app/test-pmd/parameters.c

Quoted 2 lines

app/test-pmd/parameters.c +2 -0

+           }
+   }

Please, note, the forwarding mode can be changed from interactive prompt with "set fwd <mode>" command. If iofwd mode is crucial for GPU functionality - we should prevent switching to other modes if GPU pools are engaged.

Quoted 23 lines

}

app/test-pmd/testpmd.c index renamed from app/test-pmd/testpmd.c +0 -0

a66dfb297c..1af235c4d8 100644

app/test-pmd/testpmd.c +0 -0

@@ -205,6 +205,12 @@ uint32_t mbuf_data_size_n = 1; /* Number of

specified mbuf sizes. */ uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = {
DEFAULT_MBUF_DATA_SIZE
}; /**< Mbuf data space size. */

app/test-pmd/testpmd.c +6 -0

+
+/* Mbuf memory types. */
+enum mbuf_mem_type mbuf_mem_types[MAX_SEGS_BUFFER_SPLIT];
+/* Pointers to external memory allocated for mempools. */ uintptr_t
+mempools_ext_ptr[MAX_SEGS_BUFFER_SPLIT];
+
 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
                                       * specified on command-line. */  uint16_t

stats_period; /**< Period to show statistics (disabled by default) */ @@ - 543,6 +549,12 @@ int proc_id;
*/
unsigned int num_procs = 1;

app/test-pmd/testpmd.c int proc_id; · line 556

Quoted 5 lines

app/test-pmd/testpmd.c +5 -0

+/*
+ * In case of GPU memory external mbufs use, for simplicity,
+ * the first GPU device in the list.
+ */
+int gpu_id = 0;

It is assigned with zero and never changes. Support the first GPU only? This limitation should be mentioned in documentation.

Quoted 129 lines

+
static void
eth_rx_metadata_negotiate_mp(uint16_t port_id) { @@ -1103,6 +1115,79
@@ setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,

    return ext_num;

}

app/test-pmd/testpmd.c +21 -0

+static struct rte_mempool *
+gpu_mbuf_pool_create(uint16_t mbuf_seg_size, unsigned int nb_mbuf,
+                   unsigned int socket_id, uint16_t port_id,
+                   int gpu_id, uintptr_t * mp_addr)
+{
+   int ret = 0;
+   char pool_name[RTE_MEMPOOL_NAMESIZE];
+   struct rte_eth_dev_info dev_info;
+   struct rte_mempool *rte_mp = NULL;
+   struct rte_pktmbuf_extmem gpu_mem;
+   struct rte_gpu_info ginfo;
+   uint8_t gpu_page_shift = 16;
+   uint32_t gpu_page_size = (1UL << gpu_page_shift);
+
+   ret = eth_dev_info_get_print_err(port_id, &dev_info);
+   if (ret != 0)
+           rte_exit(EXIT_FAILURE,
+                   "Failed to get device info for port %d\n",
+                   port_id);
+
+   mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name),

port_id, MBUF_MEM_GPU);

app/test-pmd/testpmd.c +11 -0

+   if (!is_proc_primary()) {
+           rte_mp = rte_mempool_lookup(pool_name);
+           if (rte_mp == NULL)
+                   rte_exit(EXIT_FAILURE,
+                           "Get mbuf pool for socket %u failed: %s\n",
+                           socket_id, rte_strerror(rte_errno));
+           return rte_mp;
+   }
+
+   if (rte_gpu_info_get(gpu_id, &ginfo))
+           rte_exit(EXIT_FAILURE, "Can't retrieve info about GPU %d -

bye\n",

app/test-pmd/testpmd.c +4 -0

+gpu_id);
+
+   TESTPMD_LOG(INFO,
+           "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u

GPU device=%s\n",

app/test-pmd/testpmd.c +3 -0

+           pool_name, nb_mbuf, mbuf_seg_size, socket_id, ginfo.name);
+
+   /* Create an external memory mempool using memory allocated on

the

app/test-pmd/testpmd.c +4 -0

+GPU. */
+
+   gpu_mem.elt_size = RTE_MBUF_DEFAULT_BUF_SIZE;
+   gpu_mem.buf_len = RTE_ALIGN_CEIL(nb_mbuf * gpu_mem.elt_size,

gpu_page_size);

app/test-pmd/testpmd.c +5 -0

+   gpu_mem.buf_iova = RTE_BAD_IOVA;
+
+   gpu_mem.buf_ptr = rte_gpu_mem_alloc(gpu_id, gpu_mem.buf_len);
+   if (gpu_mem.buf_ptr == NULL)
+           rte_exit(EXIT_FAILURE, "Could not allocate GPU device

memory\n");

app/test-pmd/testpmd.c +2 -0

+
+   ret = rte_extmem_register(gpu_mem.buf_ptr, gpu_mem.buf_len,

NULL, gpu_mem.buf_iova, gpu_page_size);

app/test-pmd/testpmd.c +2 -0

+   if (ret)
+           rte_exit(EXIT_FAILURE, "Unable to register addr 0x%p, ret

%d\n",

app/test-pmd/testpmd.c +8 -0

+gpu_mem.buf_ptr, ret);
+
+   uint16_t pid = 0;
+
+   RTE_ETH_FOREACH_DEV(pid)
+   {
+           ret = rte_dev_dma_map(dev_info.device, gpu_mem.buf_ptr,
+                                     gpu_mem.buf_iova,

gpu_mem.buf_len);

app/test-pmd/testpmd.c +2 -0

+           if (ret) {
+                   rte_exit(EXIT_FAILURE, "Unable to DMA map addr

0x%p for device %s\n",
+ gpu_mem.buf_ptr, dev_info.device-

Quoted 1 line

name);

app/test-pmd/testpmd.c +4 -0

+           }
+   }
+
+   rte_mp = rte_pktmbuf_pool_create_extbuf(pool_name, nb_mbuf,

mb_mempool_cache, 0, mbuf_seg_size, socket_id, &gpu_mem, 1);

app/test-pmd/testpmd.c +2 -0

+   if (rte_mp == NULL) {
+           rte_exit(EXIT_FAILURE, "Creation of GPU mempool <%s>

failed\n", pool_name);

app/test-pmd/testpmd.c +9 -2

+   }
+
+   *mp_addr = (uintptr_t) gpu_mem.buf_ptr;
+
+   return rte_mp;
+}
+
 /*
  * Configuration initialisation done once at init time.
  */

@@ -1117,7 +1202,7 @@ mbuf_pool_create(uint16_t mbuf_seg_size,
 unsigned nb_mbuf,
 
    mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;  #endif
-   mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name),
 size_idx);
+   mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name),
 size_idx,
+MBUF_MEM_CPU);
    if (!is_proc_primary()) {
            rte_mp = rte_mempool_lookup(pool_name);
            if (rte_mp == NULL)

@@ -1700,19 +1785,42 @@ init_config(void)
 
            for (i = 0; i < num_sockets; i++)
                    for (j = 0; j < mbuf_data_size_n; j++)
-                           mempools[i * MAX_SEGS_BUFFER_SPLIT + j]

=

app/test-pmd/testpmd.c +2 -3

-                                   mbuf_pool_create(mbuf_data_size[j],
-                                                     nb_mbuf_per_pool,
-                                                     socket_ids[i], j);
+                   {
+                           if (mbuf_mem_types[j] == MBUF_MEM_GPU)

{

app/test-pmd/testpmd.c +2 -0

+                                   if (rte_gpu_count_avail() == 0)
+                                           rte_exit(EXIT_FAILURE, "No

GPU device available.\n");

app/test-pmd/testpmd.c +2 -0

+
+                                   mempools[i *

Quoted 3 lines

MAX_SEGS_BUFFER_SPLIT + j] =
+
gpu_mbuf_pool_create(mbuf_data_size[j],

What about GPU/CPU adherence ? Do we create one GPU pool per CPU socket? Disregarding hardware topology at all?
We can mitigate the issue by "--socket-mem/--socket-num" parameter though.

Quoted 121 lines

app/test-pmd/testpmd.c +5 -0

+
             nb_mbuf_per_pool,
+
             socket_ids[i], j, gpu_id,
+
             &(mempools_ext_ptr[j]));
+                           } else {
+                                   mempools[i *

MAX_SEGS_BUFFER_SPLIT + j] =

app/test-pmd/testpmd.c +3 -0

+
    mbuf_pool_create(mbuf_data_size[j],
+
    nb_mbuf_per_pool,
+                                                           socket_ids[i],

j);

app/test-pmd/testpmd.c +5 -5

+                           }
+                   }
    } else {
            uint8_t i;
 
            for (i = 0; i < mbuf_data_size_n; i++)
-                   mempools[i] = mbuf_pool_create
-                                   (mbuf_data_size[i],
-                                    nb_mbuf_per_pool,
-                                    socket_num == UMA_NO_CONFIG ?
-                                    0 : socket_num, i);
+           {
+                   if (mbuf_mem_types[i] == MBUF_MEM_GPU) {
+                           mempools[i] =

gpu_mbuf_pool_create(mbuf_data_size[i],

app/test-pmd/testpmd.c +6 -0

+
                       nb_mbuf_per_pool,
+
                       socket_num == UMA_NO_CONFIG ? 0 : socket_num,
+
                       i, gpu_id,
+
                       &(mempools_ext_ptr[i]));
+                   } else {
+                           mempools[i] =

mbuf_pool_create(mbuf_data_size[i],

app/test-pmd/testpmd.c +2 -0

+                                                   nb_mbuf_per_pool,
+                                                   socket_num ==

UMA_NO_CONFIG ?

app/test-pmd/testpmd.c +8 -1

+                                                   0 : socket_num, i);
+                   }
+           }
+
    }
 
    init_port_config();

@@ -3415,8 +3523,11 @@ pmd_test_exit(void)
            }
    }
    for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
-           if (mempools[i])
+           if (mempools[i]) {
                    mempool_free_mp(mempools[i]);
+                   if (mbuf_mem_types[i] == MBUF_MEM_GPU)
+                           rte_gpu_mem_free(gpu_id, (void
 *)mempools_ext_ptr[i]);
+           }
    }
    free(xstats_display);

app/test-pmd/testpmd.h index renamed from app/test-pmd/testpmd.h +0 -0

669ce1e87d..9919044372 100644

app/test-pmd/testpmd.h +1 -0

@@ -12,6 +12,7 @@
 #include <rte_gro.h>
 #include <rte_gso.h>
 #include <rte_os_shim.h>
+#include <rte_gpudev.h>
 #include <cmdline.h>
 #include <sys/queue.h>
 #ifdef RTE_HAS_JANSSON

@@ -474,6 +475,11 @@ extern uint8_t dcb_config;  extern uint32_t

mbuf_data_size_n; extern uint16_t
mbuf_data_size[MAX_SEGS_BUFFER_SPLIT];
/**< Mbuf data space size. */

app/test-pmd/testpmd.h +5 -0

+enum mbuf_mem_type {
+   MBUF_MEM_CPU,
+   MBUF_MEM_GPU
+};
+extern enum mbuf_mem_type

mbuf_mem_types[MAX_SEGS_BUFFER_SPLIT];
extern uint32_t param_total_num_mbufs;

extern uint16_t stats_period;

app/test-pmd/testpmd.h +5 -3

@@ -717,14 +723,16 @@ current_fwd_lcore(void)
 /* Mbuf Pools */
 static inline void
 mbuf_poolname_build(unsigned int sock_id, char *mp_name,
-               int name_size, uint16_t idx)
+               int name_size, uint16_t idx, enum mbuf_mem_type
 mem_type)
 {
+   const char *suffix = mem_type == MBUF_MEM_GPU ? "_gpu" : "";
+
    if (!idx)
            snprintf(mp_name, name_size,
-                    MBUF_POOL_NAME_PFX "_%u", sock_id);
+                    MBUF_POOL_NAME_PFX "_%u%s", sock_id, suffix);
    else
            snprintf(mp_name, name_size,
-                    MBUF_POOL_NAME_PFX "_%hu_%hu",
 (uint16_t)sock_id, idx);
+                    MBUF_POOL_NAME_PFX "_%hu_%hu%s",
 (uint16_t)sock_id, idx, suffix);

}

static inline struct rte_mempool *

app/test-pmd/testpmd.h +2 -1

@@ -732,7 +740,7 @@ mbuf_pool_find(unsigned int sock_id, uint16_t idx)  {
    char pool_name[RTE_MEMPOOL_NAMESIZE];
 
-   mbuf_poolname_build(sock_id, pool_name, sizeof(pool_name), idx);
+   mbuf_poolname_build(sock_id, pool_name, sizeof(pool_name), idx,
+mbuf_mem_types[idx]);
    return rte_mempool_lookup((const char *)pool_name);  }
 

Quoted 15 lines

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 124

Quoted 12 lines

doc/guides/testpmd_app_ug/run_app.rst +3 -0

index 30edef07ea..ede7b79abb 100644
@@ -119,6 +119,9 @@ The command line options are:
     The default value is 2048. If multiple mbuf-size values are specified the
     extra memory pools will be created for allocating mbufs to receive packets
     with buffer splitting features.
+    Providing mbuf size with a 'g' suffix (e.g. ``--mbuf-size=2048g``),
+    will cause the mempool to be created on a GPU memory area allocated.
+    This option is currently limited to iofwd engine with the first GPU.

Mmm, if we have multiple GPUs - we have no way to specify on which one we should allocate the pool. Syntax is not complete☹. Possible we should specify GPU port id after the suffix ? Like 2048g2 ? If port index is omitted we should consider we have the only GPU and check this.

Do we need to overload --mbuf-size parameter here? Why not add 'gpu' option to --mp-alloc parameter?

From: Ferruh Yigit <[email protected]>
Date: Tuesday, 16 November 2021 at 19:00
To: Elena Agostini <[email protected]>
Cc: [email protected] <[email protected]>, Slava Ovsiienko <[email protected]>
Subject: Re: [PATCH v2 1/1] app/testpmd: add GPU memory option in iofwd engine External email: Use caution opening links or attachments>

Quoted 6 lines · On 11/11/2021 9:41 PM, [email protected] wrote:

app/test-pmd/meson.build +0 -0

@@ -32,7 +32,7 @@ if dpdk_conf.has('RTE_HAS_JANSSON')
      ext_deps += jansson_dep
  endif
 

app/test-pmd/meson.build if dpdk_conf.has('RTE_HAS_JANSSON') · line 35

Quoted 2 lines

app/test-pmd/meson.build +1 -1

-deps += ['ethdev', 'gro', 'gso', 'cmdline', 'metrics', 'bus_pci']
+deps += ['ethdev', 'gro', 'gso', 'cmdline', 'metrics', 'bus_pci', 'gpudev']>

I didn't review the set, but in a very high level do we want to add 'gpudev' as dependency? Isn't this like adding 'rawdev' as dependency.

Quoted 28 lines

From: Ferruh Yigit <[email protected]>

Date: Tuesday, 16 November 2021 at 19:00

To: Elena Agostini <[email protected]>

Cc [email protected] <[email protected]>, Slava Ovsiienko <[email protected]>

Subject: Re: [PATCH v2 1/1] app/testpmd: add GPU memory option in iofwd engine

External email: Use caution opening links or attachments>

Quoted 11 lines · On 11/11/2021 9:41 PM, [email protected] wrote:

--- a/app/test-pmd/meson.build

+++ b/app/test-pmd/meson.build

(quoted diff) +0 -0

@@ -32,7 +32,7 @@ if dpdk_conf.has('RTE_HAS_JANSSON')
 

       ext_deps += jansson_dep

   endif

Quoted 7 lines

Review

Quoted 3 lines

(quoted diff) +1 -1

-deps += ['ethdev', 'gro', 'gso', 'cmdline', 'metrics', 'bus_pci']
 
+deps += ['ethdev', 'gro', 'gso', 'cmdline', 'metrics', 'bus_pci', 'gpudev']>

I didn't review the set, but in a very high level do we want to add

'gpudev' as dependency? Isn't this like adding 'rawdev' as dependency.

gpudev is a library that can be built without a gpu driver as all the other libraries

and itis actually used only in case of GPU memory mempool.

Reasons for this patch are:

- Have an upstreamed benchmark tool to measure network metrics using GPU memory

- Test some DPDK features not really tested anywhere like the external memory mempool feature

Quoted 44 lines · Elena Agostini

Quoted 28 lines

From: Ferruh Yigit <[email protected]>

Date: Tuesday, 16 November 2021 at 19:00

To: Elena Agostini <[email protected]>

Cc [email protected] <[email protected]>, Slava Ovsiienko <[email protected]>

Subject: Re: [PATCH v2 1/1] app/testpmd: add GPU memory option in iofwd engine

External email: Use caution opening links or attachments>

Quoted 11 lines · On 11/11/2021 9:41 PM, [email protected] wrote:

--- a/app/test-pmd/meson.build

+++ b/app/test-pmd/meson.build

app/test-pmd/meson.build +0 -0

@@ -32,7 +32,7 @@ if dpdk_conf.has('RTE_HAS_JANSSON')
 
      ext_deps += jansson_dep
 
  endif
 
 

Quoted 7 lines

Review if dpdk_conf.has('RTE_HAS_JANSSON')

Quoted 3 lines

(quoted diff) +1 -1

-deps += ['ethdev', 'gro', 'gso', 'cmdline', 'metrics', 'bus_pci']
 
+deps += ['ethdev', 'gro', 'gso', 'cmdline', 'metrics', 'bus_pci', 'gpudev']>

I didn't review the set, but in a very high level do we want to add

'gpudev' as dependency? Isn't this like adding 'rawdev' as dependency.

gpudev is a library that can be built without a gpu driver as all the other libraries

and itis actually used only in case of GPU memory mempool.

Reasons for this patch are:

- Have an upstreamed benchmark tool to measure network metrics using GPU memory

Quoted 2 lines

- Test some DPDK features not really tested anywhere like the external memory mempool feature

I can see the reason, that is obvious, yet again why we are not adding rawdev testing to the testpmd? But adding gpudev.
It is easier to add it to the testpmd, and for some testing perspective it makes sense, but still I am not quite sure about this new dependency, I would like to get more feedback.

From: Jerin Jacob <[email protected]>
Date: Tuesday, 16 November 2021 at 20:09
To: Ferruh Yigit <[email protected]>
Cc: Elena Agostini <[email protected]>, [email protected] <[email protected]>, Slava Ovsiienko <[email protected]>
Subject: Re: [PATCH v2 1/1] app/testpmd: add GPU memory option in iofwd engine
External email: Use caution opening links or attachments

Quoted 46 lines · Ferruh Yigit

Quoted 44 lines · Elena Agostini

Quoted 28 lines

From: Ferruh Yigit <[email protected]>

Date: Tuesday, 16 November 2021 at 19:00

To: Elena Agostini <[email protected]>

Cc [email protected] <[email protected]>, Slava Ovsiienko <[email protected]>

Subject: Re: [PATCH v2 1/1] app/testpmd: add GPU memory option in iofwd engine

External email: Use caution opening links or attachments>

Quoted 11 lines · On 11/11/2021 9:41 PM, [email protected] wrote:

--- a/app/test-pmd/meson.build

+++ b/app/test-pmd/meson.build

app/test-pmd/meson.build +0 -0

@@ -32,7 +32,7 @@ if dpdk_conf.has('RTE_HAS_JANSSON')
 
      ext_deps += jansson_dep
 
  endif
 
 

Quoted 7 lines

Review if dpdk_conf.has('RTE_HAS_JANSSON')

Quoted 3 lines

(quoted diff) +1 -1

-deps += ['ethdev', 'gro', 'gso', 'cmdline', 'metrics', 'bus_pci']
 
+deps += ['ethdev', 'gro', 'gso', 'cmdline', 'metrics', 'bus_pci', 'gpudev']>

I didn't review the set, but in a very high level do we want to add

'gpudev' as dependency? Isn't this like adding 'rawdev' as dependency.

gpudev is a library that can be built without a gpu driver as all the other libraries

and itis actually used only in case of GPU memory mempool.

Reasons for this patch are:

- Have an upstreamed benchmark tool to measure network metrics using GPU memory

Quoted 8 lines

Quoted 2 lines

- Test some DPDK features not really tested anywhere like the external memory mempool feature

I can see the reason, that is obvious, yet again why we are not adding rawdev testing to the testpmd? But adding gpudev.
It is easier to add it to the testpmd, and for some testing perspective it makes sense, but still I am not quite sure about this new dependency, I would like to get more feedback.

I had the similar concern earlier. IMO, It is better to have a separate test application for gpudev like
other device classes. For eventdev cases when it needs to work with ethdev for Rx adapter cases,
We have enabled such code in app/test-eventdev to make testpmd focus on ethdev.

Quoted 131 lines

From: Jerin Jacob <[email protected]>

Date: Tuesday, 16 November 2021 at 20:09

To: Ferruh Yigit <[email protected]>

Cc Elena Agostini <[email protected]>, [email protected] <[email protected]>, Slava Ovsiienko <[email protected]>

Subject: Re: [PATCH v2 1/1] app/testpmd: add GPU memory option in iofwd engine

External email: Use caution opening links or attachments

Quoted 100 lines · Ferruh Yigit

Quoted 84 lines · Elena Agostini

Quoted 64 lines

From: Ferruh Yigit <[email protected]>

Date: Tuesday, 16 November 2021 at 19:00

To: Elena Agostini <[email protected]>

Cc [email protected] <[email protected]>, Slava Ovsiienko <[email protected]>

Subject: Re: [PATCH v2 1/1] app/testpmd: add GPU memory option in iofwd engine

External email: Use caution opening links or attachments>

Quoted 24 lines · On 11/11/2021 9:41 PM, [email protected] wrote:

--- a/app/test-pmd/meson.build

+++ b/app/test-pmd/meson.build

(quoted diff) +0 -0

@@ -32,7 +32,7 @@ if dpdk_conf.has('RTE_HAS_JANSSON')
 
 
 
      ext_deps += jansson_dep
 
 
 

endif

Review

Quoted 5 lines

(quoted diff) +1 -1

-deps += ['ethdev', 'gro', 'gso', 'cmdline', 'metrics', 'bus_pci']
 
 
 
+deps += ['ethdev', 'gro', 'gso', 'cmdline', 'metrics', 'bus_pci', 'gpudev']>

Quoted 5 lines

I didn't review the set, but in a very high level do we want to add

'gpudev' as dependency? Isn't this like adding 'rawdev' as dependency.

gpudev is a library that can be built without a gpu driver as all the other libraries

and itis actually used only in case of GPU memory mempool.

Reasons for this patch are:

Quoted 7 lines

- Have an upstreamed benchmark tool to measure network metrics using GPU memory

- Test some DPDK features not really tested anywhere like the external memory mempool feature

I can see the reason, that is obvious, yet again why we are not adding rawdev

Quoted 7 lines

testing to the testpmd? But adding gpudev.

It is easier to add it to the testpmd, and for some testing perspective it

makes sense, but still I am not quite sure about this new dependency, I would

like to get more feedback.

I had the similar concern earlier. IMO, It is better to have a

Quoted 7 lines

separate test application for gpudev like

other device classes. For eventdev cases when it needs to work with

ethdev for Rx adapter cases,

We have enabled such code in app/test-eventdev to make testpmd focus on ethdev.

gpudev already has a test app in app/test-gpudev.

gpudev needs to be also test with network card and today another application

decidated to test gpudev over the network would be very similar to testpmd io.

At this stage, there is no point in reinventing the wheel

Quoted 145 lines · Elena Agostini

Quoted 131 lines

From: Jerin Jacob <[email protected]>

Date: Tuesday, 16 November 2021 at 20:09

To: Ferruh Yigit <[email protected]>

Cc Elena Agostini <[email protected]>, [email protected] <[email protected]>, Slava Ovsiienko <[email protected]>

Subject: Re: [PATCH v2 1/1] app/testpmd: add GPU memory option in iofwd engine

External email: Use caution opening links or attachments

Quoted 100 lines · Ferruh Yigit

Quoted 84 lines · Elena Agostini

Quoted 64 lines

From: Ferruh Yigit <[email protected]>

Date: Tuesday, 16 November 2021 at 19:00

To: Elena Agostini <[email protected]>

Cc [email protected] <[email protected]>, Slava Ovsiienko <[email protected]>

Subject: Re: [PATCH v2 1/1] app/testpmd: add GPU memory option in iofwd engine

External email: Use caution opening links or attachments>

Quoted 24 lines · On 11/11/2021 9:41 PM, [email protected] wrote:

--- a/app/test-pmd/meson.build

+++ b/app/test-pmd/meson.build

(quoted diff) +0 -0

@@ -32,7 +32,7 @@ if dpdk_conf.has('RTE_HAS_JANSSON')
 
 
 
      ext_deps += jansson_dep
 
 
 

endif

Review

Quoted 5 lines

(quoted diff) +1 -1

-deps += ['ethdev', 'gro', 'gso', 'cmdline', 'metrics', 'bus_pci']
 
 
 
+deps += ['ethdev', 'gro', 'gso', 'cmdline', 'metrics', 'bus_pci', 'gpudev']>

Quoted 5 lines

I didn't review the set, but in a very high level do we want to add

'gpudev' as dependency? Isn't this like adding 'rawdev' as dependency.

gpudev is a library that can be built without a gpu driver as all the other libraries

and itis actually used only in case of GPU memory mempool.

Reasons for this patch are:

Quoted 7 lines

- Have an upstreamed benchmark tool to measure network metrics using GPU memory

- Test some DPDK features not really tested anywhere like the external memory mempool feature

I can see the reason, that is obvious, yet again why we are not adding rawdev

Quoted 7 lines

testing to the testpmd? But adding gpudev.

It is easier to add it to the testpmd, and for some testing perspective it

makes sense, but still I am not quite sure about this new dependency, I would

like to get more feedback.

I had the similar concern earlier. IMO, It is better to have a

Quoted 7 lines

separate test application for gpudev like

other device classes. For eventdev cases when it needs to work with

ethdev for Rx adapter cases,

We have enabled such code in app/test-eventdev to make testpmd focus on ethdev.

gpudev already has a test app in app/test-gpudev.

Quoted 7 lines

gpudev needs to be also test with network card and today another application

decidated to test gpudev over the network would be very similar to testpmd io.

At this stage, there is no point in reinventing the wheel

I think, it is not about not reinventing the wheel, It is about maintenance of testpmd,
currently, the feature are specific to ethdev. Adding more cross-device-specific features
will populate the testpmd. I had a similar case when it network cases need to be integrated to eventdev,
I choose to have it test-eventdev so that testpmd focus remains for ethdev.

From: Elena Agostini <[email protected]>

This patch introduces GPU memory in testpmd through the gpudev library. Testpmd can be used for network benchmarks when using GPU memory instead of regular CPU memory to send and receive packets. This option is currently limited to iofwd engine to ensure no workload is applied on packets not accessible from the CPU.

The options chose is --mbuf-size so buffer split feature across different mempools can be enabled.

Signed-off-by Elena Agostini <[email protected]>

External email: Use caution opening links or attachments

Quoted 8 lines · On Wed, 17 Nov 2021 03:04:59 +0000 <[email protected]> wrote:

From: Elena Agostini <[email protected]>

This patch introduces GPU memory in testpmd through the gpudev library. Testpmd can be used for network benchmarks when using GPU memory instead of regular CPU memory to send and receive packets. This option is currently limited to iofwd engine to ensure no workload is applied on packets not accessible from the CPU.

Quoted 4 lines

The options chose is --mbuf-size so buffer split feature across different mempools can be enabled.

Signed-off-by Elena Agostini <[email protected]>

Won't this create a hard dependency of test-pmd on gpudev? I thought gpudev was supposed to be optional

Quoted 38 lines

External email: Use caution opening links or attachments

On Wed, 17 Nov 2021 03:04:59 +0000

Quoted 23 lines · <[email protected]> wrote:

From: Elena Agostini <[email protected]>

This patch introduces GPU memory in testpmd through the gpudev library.

Testpmd can be used for network benchmarks when using GPU memory

instead of regular CPU memory to send and receive packets.

This option is currently limited to iofwd engine to ensure

no workload is applied on packets not accessible from the CPU.

The options chose is --mbuf-size so buffer split feature across

different mempools can be enabled.

Signed-off-by Elena Agostini <[email protected]>

Quoted 3 lines

Won't this create a hard dependency of test-pmd on gpudev?

I thought gpudev was supposed to be optional

Sure, let me submit another patch to make it optional

Quoted 39 lines

Quoted 38 lines

External email: Use caution opening links or attachments

On Wed, 17 Nov 2021 03:04:59 +0000

Quoted 23 lines · <[email protected]> wrote:

From: Elena Agostini <[email protected]>

This patch introduces GPU memory in testpmd through the gpudev library.

Testpmd can be used for network benchmarks when using GPU memory

instead of regular CPU memory to send and receive packets.

This option is currently limited to iofwd engine to ensure

no workload is applied on packets not accessible from the CPU.

The options chose is --mbuf-size so buffer split feature across

different mempools can be enabled.

Signed-off-by Elena Agostini <[email protected]>

Quoted 7 lines

Quoted 3 lines

Won't this create a hard dependency of test-pmd on gpudev?

I thought gpudev was supposed to be optional

Sure, let me submit another patch to make it optional

Why to add yet another compile time macro everywhere in testpmd and make hard to maintain?
Adding iofwd kind of code is very simple to add test/test-gpudev and all GPU specific options
can be added in test-gpudev. It also helps to review the patches as test cases focus on
each device class.

Quoted 108 lines · Elena Agostini

Quoted 91 lines

Quoted 79 lines

External email: Use caution opening links or attachments

On Wed, 17 Nov 2021 03:04:59 +0000

Quoted 44 lines · <[email protected]> wrote:

From: Elena Agostini <[email protected]>

This patch introduces GPU memory in testpmd through the gpudev library.

Testpmd can be used for network benchmarks when using GPU memory

instead of regular CPU memory to send and receive packets.

This option is currently limited to iofwd engine to ensure

no workload is applied on packets not accessible from the CPU.

The options chose is --mbuf-size so buffer split feature across

different mempools can be enabled.

Quoted 1 line

Signed-off-by Elena Agostini <[email protected]>

Quoted 5 lines

Won't this create a hard dependency of test-pmd on gpudev?

I thought gpudev was supposed to be optional

Quoted 1 line

Sure, let me submit another patch to make it optional

Why to add yet another compile time macro everywhere in testpmd and

make hard to maintain?

Adding iofwd kind of code is very simple to add test/test-gpudev and

all GPU specific options

can be added in test-gpudev. It also helps to review the patches as

test cases focus on

Quoted 1 line

each device class.

Test-gpudev is standalone unit test to ensure gpudev functions work correctly.

In testpmd instead, there is a connection between gpudev and the network.

Quoted 115 lines · Elena Agostini

Quoted 103 lines · Elena Agostini

Quoted 91 lines

Quoted 79 lines

External email: Use caution opening links or attachments

On Wed, 17 Nov 2021 03:04:59 +0000

Quoted 44 lines · <[email protected]> wrote:

From: Elena Agostini <[email protected]>

This patch introduces GPU memory in testpmd through the gpudev library.

Testpmd can be used for network benchmarks when using GPU memory

instead of regular CPU memory to send and receive packets.

This option is currently limited to iofwd engine to ensure

no workload is applied on packets not accessible from the CPU.

The options chose is --mbuf-size so buffer split feature across

different mempools can be enabled.

Quoted 1 line

Signed-off-by Elena Agostini <[email protected]>

Quoted 5 lines

Won't this create a hard dependency of test-pmd on gpudev?

I thought gpudev was supposed to be optional

Quoted 1 line

Sure, let me submit another patch to make it optional

Why to add yet another compile time macro everywhere in testpmd and

make hard to maintain?

Adding iofwd kind of code is very simple to add test/test-gpudev and

Quoted 7 lines

all GPU specific options

can be added in test-gpudev. It also helps to review the patches as

test cases focus on

each device class.

Test-gpudev is standalone unit test to ensure gpudev functions work correctly.

Quoted 1 line

In testpmd instead, there is a connection between gpudev and the network.

I understand that. We had the same case with eventdev, where it needs to work with network. Testpmd is already complicated, IMO, we should focus only ethdev
test cases on testpmd, test-gpudev can use ethdev API to enable networking requirements for gpudev.

Quoted 87 lines

Quoted 86 lines

Quoted 73 lines

Quoted 63 lines

Quoted 51 lines

On Wed, 17 Nov 2021 03:04:59 +0000

Quoted 36 lines

This patch introduces GPU memory in testpmd through the gpudev library.

Testpmd can be used for network benchmarks when using GPU memory

instead of regular CPU memory to send and receive packets.

This option is currently limited to iofwd engine to ensure

no workload is applied on packets not accessible from the CPU.

The options chose is --mbuf-size so buffer split feature across

different mempools can be enabled.

Quoted 1 line

Signed-off-by Elena Agostini <[email protected]>

Quoted 5 lines

Won't this create a hard dependency of test-pmd on gpudev?

I thought gpudev was supposed to be optional

Quoted 1 line

Sure, let me submit another patch to make it optional

Why to add yet another compile time macro everywhere in testpmd and

make hard to maintain?

Adding iofwd kind of code is very simple to add test/test-gpudev and

Quoted 7 lines

all GPU specific options

can be added in test-gpudev. It also helps to review the patches as

test cases focus on

each device class.

Test-gpudev is standalone unit test to ensure gpudev functions work correctly.

In testpmd instead, there is a connection between gpudev and the network.

Quoted 6 lines

I understand that. We had the same case with eventdev, where it needs to work with network. Testpmd is already complicated, IMO, we should focus only ethdev
test cases on testpmd, test-gpudev can use ethdev API to enable networking requirements for gpudev.

+1

Quoted 93 lines

Quoted 92 lines

Quoted 78 lines

Quoted 77 lines

Quoted 62 lines

Quoted 50 lines

On Wed, 17 Nov 2021 03:04:59 +0000

Quoted 36 lines

This patch introduces GPU memory in testpmd through the gpudev library.

Testpmd can be used for network benchmarks when using GPU memory

instead of regular CPU memory to send and receive packets.

This option is currently limited to iofwd engine to ensure

no workload is applied on packets not accessible from the CPU.

The options chose is --mbuf-size so buffer split feature across

different mempools can be enabled.

Quoted 1 line

Signed-off-by Elena Agostini <[email protected]>

Quoted 5 lines

Won't this create a hard dependency of test-pmd on gpudev?

I thought gpudev was supposed to be optional

Quoted 1 line

Sure, let me submit another patch to make it optional

Why to add yet another compile time macro everywhere in testpmd and

make hard to maintain?

Adding iofwd kind of code is very simple to add test/test-gpudev and

all GPU specific options

can be added in test-gpudev. It also helps to review the patches as

test cases focus on

Quoted 7 lines

Quoted 1 line

each device class.

Test-gpudev is standalone unit test to ensure gpudev functions work correctly.

In testpmd instead, there is a connection between gpudev and the network.

I understand that. We had the same case with eventdev, where it needs to work with network. Testpmd is already complicated, IMO, we should focus only ethdev
test cases on testpmd, test-gpudev can use ethdev API to enable networking requirements for gpudev.

Quoted 1 line

+1

+1

Quoted 190 lines

Quoted 188 lines

Quoted 175 lines

Quoted 165 lines

Quoted 153 lines

Quoted 125 lines

Quoted 109 lines

On Wed, 17 Nov 2021 03:04:59 +0000

Quoted 72 lines

This patch introduces GPU memory in testpmd through the gpudev library.

Testpmd can be used for network benchmarks when using GPU memory

instead of regular CPU memory to send and receive packets.

This option is currently limited to iofwd engine to ensure

no workload is applied on packets not accessible from the CPU.

The options chose is --mbuf-size so buffer split feature across

different mempools can be enabled.

Quoted 1 line

Signed-off-by Elena Agostini <[email protected]>

Won't this create a hard dependency of test-pmd on gpudev?

Quoted 1 line

I thought gpudev was supposed to be optional

Quoted 1 line

Sure, let me submit another patch to make it optional

Why to add yet another compile time macro everywhere in testpmd and

make hard to maintain?

Adding iofwd kind of code is very simple to add test/test-gpudev and

all GPU specific options

can be added in test-gpudev. It also helps to review the patches as

Quoted 5 lines

test cases focus on

each device class.

Quoted 5 lines

Test-gpudev is standalone unit test to ensure gpudev functions work correctly.

In testpmd instead, there is a connection between gpudev and the network.

I understand that. We had the same case with eventdev, where it needs to

Quoted 9 lines

work with network. Testpmd is already complicated, IMO, we should

focus only ethdev

test cases on testpmd, test-gpudev can use ethdev API to enable

networking requirements for gpudev.

+1

Quoted 1 line

+1

Testpmd already manages different type of memories for mempools.

gpudev is just another type of memory, there is nothing more than that.

Quoted 191 lines

Quoted 190 lines

Quoted 188 lines

Quoted 175 lines

Quoted 165 lines

Quoted 153 lines

Quoted 125 lines

Quoted 109 lines

On Wed, 17 Nov 2021 03:04:59 +0000

Quoted 72 lines

This patch introduces GPU memory in testpmd through the gpudev library.

Testpmd can be used for network benchmarks when using GPU memory

instead of regular CPU memory to send and receive packets.

This option is currently limited to iofwd engine to ensure

no workload is applied on packets not accessible from the CPU.

The options chose is --mbuf-size so buffer split feature across

different mempools can be enabled.

Quoted 1 line

Signed-off-by Elena Agostini <[email protected]>

Won't this create a hard dependency of test-pmd on gpudev?

Quoted 1 line

I thought gpudev was supposed to be optional

Quoted 1 line

Sure, let me submit another patch to make it optional

Why to add yet another compile time macro everywhere in testpmd and

make hard to maintain?

Adding iofwd kind of code is very simple to add test/test-gpudev and

all GPU specific options

can be added in test-gpudev. It also helps to review the patches as

Quoted 5 lines

test cases focus on

each device class.

Quoted 5 lines

Test-gpudev is standalone unit test to ensure gpudev functions work correctly.

In testpmd instead, there is a connection between gpudev and the network.

I understand that. We had the same case with eventdev, where it needs to

Quoted 9 lines

work with network. Testpmd is already complicated, IMO, we should

focus only ethdev

test cases on testpmd, test-gpudev can use ethdev API to enable

networking requirements for gpudev.

+1

Quoted 7 lines

Quoted 1 line

+1

Testpmd already manages different type of memories for mempools.

gpudev is just another type of memory, there is nothing more than that.

Let take this example:
1) New code changes

app/test-pmd/cmdline.c    |  32 +++++++-
app/test-pmd/config.c     |   4 +-
app/test-pmd/icmpecho.c   |   2 +-
app/test-pmd/meson.build  |   2 +-
app/test-pmd/parameters.c |  15 +++-
app/test-pmd/testpmd.c    | 167 +++++++++++++++++++++++++++++++++++---
app/test-pmd/testpmd.h    |  16 +++-
7 files changed, 217 insertions(+), 21 deletions(-)

2) Good amount of code need to go through condition compilation as gpudev is optional that make
testpmd further ugly.

3) It introduces new memtype, now

(quoted diff) +4 -0

+enum mbuf_mem_type {
+ MBUF_MEM_CPU,
+ MBUF_MEM_GPU
+};

The question largely, why testpmd need to pollute for this, testpmd, we are using for testing ethdev device class.
All we are saying is to enable this use case in test-gpudev so that it focuses on GPU specific, Whoever is not
interested in specific libraries do not even need to review the testpmd patches.

Quoted 454 lines · Elena Agostini

Quoted 391 lines

Quoted 383 lines

Quoted 375 lines

Quoted 359 lines

Quoted 339 lines

Quoted 315 lines

Quoted 251 lines

Quoted 219 lines

On Wed, 17 Nov 2021 03:04:59 +0000

Quoted 144 lines

This patch introduces GPU memory in testpmd through the gpudev library.

Testpmd can be used for network benchmarks when using GPU memory

instead of regular CPU memory to send and receive packets.

This option is currently limited to iofwd engine to ensure

no workload is applied on packets not accessible from the CPU.

The options chose is --mbuf-size so buffer split feature across

different mempools can be enabled.

Quoted 1 line

Signed-off-by Elena Agostini <[email protected]>

Won't this create a hard dependency of test-pmd on gpudev?

Quoted 1 line

I thought gpudev was supposed to be optional

Quoted 1 line

Sure, let me submit another patch to make it optional

Why to add yet another compile time macro everywhere in testpmd and

make hard to maintain?

Adding iofwd kind of code is very simple to add test/test-gpudev and

all GPU specific options

can be added in test-gpudev. It also helps to review the patches as

test cases focus on

Quoted 1 line

each device class.

Test-gpudev is standalone unit test to ensure gpudev functions work correctly.

Quoted 1 line

In testpmd instead, there is a connection between gpudev and the network.

I understand that. We had the same case with eventdev, where it needs to

work with network. Testpmd is already complicated, IMO, we should

focus only ethdev

Quoted 9 lines

test cases on testpmd, test-gpudev can use ethdev API to enable

networking requirements for gpudev.

Quoted 1 line

+1

Quoted 1 line

+1

Quoted 5 lines

Testpmd already manages different type of memories for mempools.

gpudev is just another type of memory, there is nothing more than that.

Let take this example:

1) New code changes

app/test-pmd/cmdline.c | 32 +++++++-

app/test-pmd/config.c | 4 +-

app/test-pmd/icmpecho.c | 2 +-

app/test-pmd/meson.build | 2 +-

app/test-pmd/parameters.c | 15 +++-

app/test-pmd/testpmd.c | 167 +++++++++++++++++++++++++++++++++++---

app/test-pmd/testpmd.h | 16 +++-

7 files changed, 217 insertions(+), 21 deletions(-)

2) Good amount of code need to go through condition compilation as

gpudev is optional that make

testpmd further ugly.

3) It introduces new memtype, now

(quoted diff) +4 -0

+enum mbuf_mem_type {
 
+ MBUF_MEM_CPU,
 
+ MBUF_MEM_GPU
 
+};

The question largely, why testpmd need to pollute for this, testpmd,

we are using for testing ethdev device class.

All we are saying is to enable this use case in test-gpudev so that it

focuses on GPU specific, Whoever is not

Quoted 1 line

interested in specific libraries do not even need to review the testpmd patches.

I understand your point. I don’t understand why this testpmd patch is there since Oct 29 but

I'm receiving reviews only few days before rc4 when I have a limited amount of time to get new code accepted.

I can provide a gpudev + ethdev example by end of today (I'd like to keep test-gpudev as it is to test gpudev API standalone).

Is there any chance this new example will be reviewed and eventually accepted in DPDK 21.11?

Quoted 455 lines

Quoted 454 lines · Elena Agostini

Quoted 391 lines

Quoted 383 lines

Quoted 375 lines

Quoted 359 lines

Quoted 339 lines

Quoted 315 lines

Quoted 251 lines

Quoted 219 lines

On Wed, 17 Nov 2021 03:04:59 +0000

Quoted 144 lines

This patch introduces GPU memory in testpmd through the gpudev library.

Testpmd can be used for network benchmarks when using GPU memory

instead of regular CPU memory to send and receive packets.

This option is currently limited to iofwd engine to ensure

no workload is applied on packets not accessible from the CPU.

The options chose is --mbuf-size so buffer split feature across

different mempools can be enabled.

Quoted 1 line

Signed-off-by Elena Agostini <[email protected]>

Won't this create a hard dependency of test-pmd on gpudev?

Quoted 1 line

I thought gpudev was supposed to be optional

Quoted 1 line

Sure, let me submit another patch to make it optional

Why to add yet another compile time macro everywhere in testpmd and

make hard to maintain?

Adding iofwd kind of code is very simple to add test/test-gpudev and

all GPU specific options

can be added in test-gpudev. It also helps to review the patches as

test cases focus on

Quoted 1 line

each device class.

Test-gpudev is standalone unit test to ensure gpudev functions work correctly.

Quoted 1 line

In testpmd instead, there is a connection between gpudev and the network.

I understand that. We had the same case with eventdev, where it needs to

work with network. Testpmd is already complicated, IMO, we should

focus only ethdev

Quoted 9 lines

test cases on testpmd, test-gpudev can use ethdev API to enable

networking requirements for gpudev.

Quoted 1 line

+1

Quoted 1 line

+1

Quoted 5 lines

Testpmd already manages different type of memories for mempools.

gpudev is just another type of memory, there is nothing more than that.

Let take this example:

1) New code changes

app/test-pmd/cmdline.c | 32 +++++++-

app/test-pmd/config.c | 4 +-

app/test-pmd/icmpecho.c | 2 +-

app/test-pmd/meson.build | 2 +-

app/test-pmd/parameters.c | 15 +++-

app/test-pmd/testpmd.c | 167 +++++++++++++++++++++++++++++++++++---

app/test-pmd/testpmd.h | 16 +++-

7 files changed, 217 insertions(+), 21 deletions(-)

2) Good amount of code need to go through condition compilation as

gpudev is optional that make

testpmd further ugly.

3) It introduces new memtype, now

(quoted diff) +4 -0

+enum mbuf_mem_type {
 
+ MBUF_MEM_CPU,
 
+ MBUF_MEM_GPU
 
+};

The question largely, why testpmd need to pollute for this, testpmd,

we are using for testing ethdev device class.

All we are saying is to enable this use case in test-gpudev so that it

focuses on GPU specific, Whoever is not

Quoted 7 lines

Quoted 1 line

interested in specific libraries do not even need to review the testpmd patches.

I understand your point. I don’t understand why this testpmd patch is there since Oct 29 but

I'm receiving reviews only few days before rc4 when I have a limited amount of time to get new code accepted.>

I understand that pain. Welcome to DPDK, we have all gone through this review issue one or another way.>

Quoted 3 lines

Quoted 3 lines

I can provide a gpudev + ethdev example by end of today (I'd like to keep test-gpudev as it is to test gpudev API standalone).

Is there any chance this new example will be reviewed and eventually accepted in DPDK 21.11?>

Why a new example? I don't have any issues in updating app/test-gpudev/.

Quoted 471 lines · Elena Agostini

Quoted 453 lines

Quoted 452 lines · Elena Agostini

Quoted 391 lines

Quoted 383 lines

Quoted 375 lines

Quoted 359 lines

Quoted 339 lines

Quoted 315 lines

Quoted 251 lines

Quoted 219 lines

On Wed, 17 Nov 2021 03:04:59 +0000

Quoted 145 lines

This patch introduces GPU memory in testpmd through the gpudev library.

Testpmd can be used for network benchmarks when using GPU memory

instead of regular CPU memory to send and receive packets.

This option is currently limited to iofwd engine to ensure

no workload is applied on packets not accessible from the CPU.

The options chose is --mbuf-size so buffer split feature across

different mempools can be enabled.

Signed-off-by: Elena Agostini <[email protected]>

Won't this create a hard dependency of test-pmd on gpudev?

Quoted 1 line

I thought gpudev was supposed to be optional

Quoted 1 line

Sure, let me submit another patch to make it optional

Why to add yet another compile time macro everywhere in testpmd and

make hard to maintain?

Adding iofwd kind of code is very simple to add test/test-gpudev and

all GPU specific options

can be added in test-gpudev. It also helps to review the patches as

test cases focus on

Quoted 1 line

each device class.

Test-gpudev is standalone unit test to ensure gpudev functions work correctly.

Quoted 1 line

In testpmd instead, there is a connection between gpudev and the network.

I understand that. We had the same case with eventdev, where it needs to

work with network. Testpmd is already complicated, IMO, we should

focus only ethdev

Quoted 9 lines

test cases on testpmd, test-gpudev can use ethdev API to enable

networking requirements for gpudev.

Quoted 1 line

+1

Quoted 1 line

+1

Quoted 5 lines

Testpmd already manages different type of memories for mempools.

gpudev is just another type of memory, there is nothing more than that.

Let take this example:

1) New code changes

app/test-pmd/cmdline.c | 32 +++++++-

app/test-pmd/config.c | 4 +-

app/test-pmd/icmpecho.c | 2 +-

app/test-pmd/meson.build | 2 +-

app/test-pmd/parameters.c | 15 +++-

app/test-pmd/testpmd.c | 167 +++++++++++++++++++++++++++++++++++---

app/test-pmd/testpmd.h | 16 +++-

7 files changed, 217 insertions(+), 21 deletions(-)

2) Good amount of code need to go through condition compilation as

gpudev is optional that make

testpmd further ugly.

3) It introduces new memtype, now

(quoted diff) +4 -0

+enum mbuf_mem_type {
 
+ MBUF_MEM_CPU,
 
+ MBUF_MEM_GPU
 
+};

The question largely, why testpmd need to pollute for this, testpmd,

we are using for testing ethdev device class.

All we are saying is to enable this use case in test-gpudev so that it

Quoted 8 lines

Quoted 3 lines

focuses on GPU specific, Whoever is not

interested in specific libraries do not even need to review the testpmd patches.

I understand your point. I don’t understand why this testpmd patch is there since Oct 29 but

I'm receiving reviews only few days before rc4 when I have a limited amount of time to get new code > accepted.>

I understand that pain. Welcome to DPDK, we have all gone through this review issue one or another way.>

Quoted 3 lines

Quoted 5 lines

I can provide a gpudev + ethdev example by end of today (I'd like to keep test-gpudev as it is to > test gpudev API standalone).

Quoted 2 lines

Is there any chance this new example will be reviewed and eventually accepted in DPDK 21.11?>

Why a new example? I don't have any issues in updating app/test-gpudev/.

Understood. If this can also help to speed up the acceptance process, I’ll provide a new patch for app/> test-gpudev to introduce ethdev to send and receive packets

From: Elena Agostini <[email protected]>

This patch introduces GPU memory in testpmd through the gpudev library. Testpmd can be used for network benchmarks when using GPU memory instead of regular CPU memory to send and receive packets. This option is currently limited to iofwd engine to ensure no workload is applied on packets not accessible from the CPU.

The options chose is --mbuf-size so buffer split feature across different mempools can be enabled.

Signed-off-by Elena Agostini <[email protected]>

---
 app/test-pmd/cmdline.c    |  36 +++++++-
 app/test-pmd/config.c     |   4 +-
 app/test-pmd/icmpecho.c   |   2 +-
 app/test-pmd/meson.build  |   3 +
 app/test-pmd/parameters.c |  15 +++-
 app/test-pmd/testpmd.c    | 182 +++++++++++++++++++++++++++++++++++---
 app/test-pmd/testpmd.h    |  18 +++-
 7 files changed, 240 insertions(+), 20 deletions(-)

app/test-pmd/cmdline.c +1 -0

index 3faa37db6d..400aa07da8 100644
@@ -3619,6 +3619,7 @@ parse_item_list(const char *str, const char *item_name, unsigned int max_items,
 	unsigned int j;
 	int value_ok;
 	char c;
+	int gpu_mbuf = 0;
 
 	/*
 	 * First parse all items in the list and store their value.

app/test-pmd/cmdline.c +5 -0

 		if ((c >= '0') && (c <= '9')) {
 			value = (unsigned int) (value * 10 + (c - '0'));
 			value_ok = 1;
 			continue;
 		}
+		if (c == 'g') {
+			/*
+			 * When this flag is set, mbufs for this segment
+			 * will be created on GPU memory.
+			 */

app/test-pmd/cmdline.c +4 -0

+			continue;
+		}
 		if (c != ',') {
 			fprintf(stderr, "character %c is not a decimal digit\n", c);
 			return 0;

@@ -3645,6 +3664,8 @@ parse_item_list(const char *str, const char *item_name, unsigned int max_items,
 			parsed_items[nb_item] = value;
 			value_ok = 0;
 			value = 0;
+			mbuf_mem_types[nb_item] = gpu_mbuf ? MBUF_MEM_GPU : MBUF_MEM_CPU;
+			gpu_mbuf = 0;
 		}
 		nb_item++;
 	}

@@ -3653,12 +3674,15 @@ parse_item_list(const char *str, const char *item_name, unsigned int max_items,
 			item_name, nb_item + 1, max_items);
 		return 0;
 	}

app/test-pmd/cmdline.c +0 -0

 	 * No optimization here...
 	 */
 	for (i = 0; i < nb_item; i++) {

app/test-pmd/cmdline.c +3 -0

+			return;
+		}
+	}
 
 	retry_enabled = 0;
 	set_pkt_forwarding_mode(res->mode);

app/test-pmd/config.c +0 -0

index 6fca09527b..9e8f8f1462 100644

app/test-pmd/config.c +1 -1

  * key of the port.
  */
 void

@@ -5255,7 +5255,7 @@ mcast_addr_pool_remove(struct rte_port *port, uint32_t addr_idx)
 {
 	port->mc_addr_nb--;
 	if (addr_idx == port->mc_addr_nb) {
-		/* No need to recompact the set of multicast addresses. */
+		/* No need to recompact the set of multicast addressses. */
 		if (port->mc_addr_nb == 0) {
 			/* free the pool of multicast addresses. */
 			free(port->mc_addr_pool);

app/test-pmd/icmpecho.c +0 -0

index 99c94cb282..3a85ec3dd1 100644

app/test-pmd/icmpecho.c +0 -0

 }
 
 static const char *

app/test-pmd/meson.build +3 -0

index 43130c8856..568660e18f 100644
@@ -43,6 +43,9 @@ if dpdk_conf.has('RTE_LIB_BPF')
     sources += files('bpf_cmd.c')
     deps += 'bpf'
 endif
+if dpdk_conf.has('RTE_LIB_GPUDEV')
+    deps += 'gpudev'
+endif
 if dpdk_conf.has('RTE_LIB_GRO')
     deps += 'gro'
 endif

app/test-pmd/parameters.c +0 -0

index f9185065af..dde5ef6590 100644

app/test-pmd/parameters.c +2 -0

+
 	printf("  --total-num-mbufs=N: set the number of mbufs to be allocated "
 	       "in mbuf pools.\n");
 	printf("  --max-pkt-len=N: set the maximum size of packet to N bytes.\n");

@@ -594,6 +597,7 @@ launch_args_parse(int argc, char** argv)
 	struct rte_eth_dev_info dev_info;
 	uint16_t rec_nb_pkts;
 	int ret;
+	uint32_t idx = 0;
 
 	static struct option lgopts[] = {
 		{ "help",			0, 0, 0 },