Skip to contentSkip to conversation

app/testpmd: support multi-process

RFC Superseded Lijun Ou 9 participants 76 messages Jan 8, 2021 – Sep 8, 2021

lore
Share & tools
lore

Lijun Ou

author
#0 lore raw

RFC 1/1 app/testpmd: support multi-process

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./testpmd -w xxx --file-prefix=xx -l 0-1 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=0 the secondary cmd:
./testpmd -w xxx --file-prefix=xx -l 2-3 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=1

---
 app/test-pmd/cmdline.c    |   6 ++-
 app/test-pmd/config.c     |   9 +++-
 app/test-pmd/parameters.c |   9 ++++
 app/test-pmd/testpmd.c    | 133 ++++++++++++++++++++++++++++++++--------------
 app/test-pmd/testpmd.h    |   7 +++
 5 files changed, 121 insertions(+), 43 deletions(-)
app/test-pmd/cmdline.c +4 -2
index 2ccbaa0..2237c9f 100644
@@ -71,8 +71,6 @@
 #include "cmdline_tm.h"
 #include "bpf_cmd.h"
 
-static struct cmdline *testpmd_cl;
-
 static void cmd_reconfig_device_queue(portid_t id, uint8_t dev, uint8_t queue);
 
 /* *** Help command with introduction. *** */
@@ -17115,6 +17113,10 @@ prompt(void)
 	if (testpmd_cl == NULL)
 		return;
 	cmdline_interact(testpmd_cl);
+	if (unlikely(f_quit == 1)) {
+		dup2(testpmd_fd_copy, testpmd_cl->s_in);
+		close(testpmd_fd_copy);
+	}
 	cmdline_stdin_exit(testpmd_cl);
 }
 
app/test-pmd/config.c +8 -1
index 3f6c864..1702e0d 100644
@@ -3100,6 +3100,8 @@ rss_fwd_config_setup(void)
 	queueid_t  rxq;
 	queueid_t  nb_q;
 	streamid_t  sm_id;
+	int start;
+	int end;
 
 	nb_q = nb_rxq;
 	if (nb_q > nb_txq)
@@ -3117,7 +3119,10 @@ rss_fwd_config_setup(void)
 	init_fwd_streams();
 
 	setup_fwd_config_of_each_lcore(&cur_fwd_config);
-	rxp = 0; rxq = 0;
+	start = proc_id * nb_q / num_procs;
+	end = start + nb_q / num_procs;
+	rxp = 0;
+	rxq = start;
 	for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
 		struct fwd_stream *fs;
 
@@ -3134,6 +3139,8 @@ rss_fwd_config_setup(void)
 			continue;
 		rxp = 0;
 		rxq++;
+		if (rxq >= end)
+			rxq=start;
 	}
 }
 
app/test-pmd/parameters.c +9 -0
index 414a006..7807afa 100644
@@ -45,6 +45,8 @@
 #include <rte_flow.h>
 
 #include "testpmd.h"
+#define PARAM_PROC_ID "proc-id"
+#define PARAM_NUM_PROCS "num-procs"
 
 static void
 usage(char* progname)
@@ -603,6 +605,8 @@ launch_args_parse(int argc, char** argv)
 		{ "rx-mq-mode",                 1, 0, 0 },
 		{ "record-core-cycles",         0, 0, 0 },
 		{ "record-burst-stats",         0, 0, 0 },
+		{ PARAM_NUM_PROCS,		1, 0, 0 },
+		{ PARAM_PROC_ID, 		1, 0, 0 },
 		{ 0, 0, 0, 0 },
 	};
 
@@ -1359,6 +1363,11 @@ launch_args_parse(int argc, char** argv)
 				record_core_cycles = 1;
 			if (!strcmp(lgopts[opt_idx].name, "record-burst-stats"))
 				record_burst_stats = 1;
+
+			if (strncmp(lgopts[opt_idx].name, PARAM_NUM_PROCS, 8) == 0)
+				num_procs = atoi(optarg);
+			if (strncmp(lgopts[opt_idx].name, PARAM_PROC_ID, 7) == 0)
+				proc_id = atoi(optarg);
 			break;
 		case 'h':
 			usage(argv[0]);
app/test-pmd/testpmd.c +93 -40

255 lines not on this page · raw message

app/test-pmd/testpmd.h +7 -0
index 5f23162..8629c57 100644
@@ -13,6 +13,7 @@
 #include <rte_gso.h>
 #include <cmdline.h>
 #include <sys/queue.h>
+#include  "cmdline.h"
 
 #define RTE_PORT_ALL            (~(portid_t)0x0)
 
@@ -24,6 +25,10 @@
 #define RTE_PORT_CLOSED         (uint16_t)2
 #define RTE_PORT_HANDLING       (uint16_t)3
 
+uint8_t f_quit;
+int testpmd_fd_copy;
+struct cmdline *testpmd_cl;
+
 /*
  * It is used to allocate the memory for hash key.
  * The hash key size is NIC dependent.
@@ -421,6 +426,8 @@ extern uint64_t noisy_lkup_mem_sz;
 extern uint64_t noisy_lkup_num_writes;
 extern uint64_t noisy_lkup_num_reads;
 extern uint64_t noisy_lkup_num_reads_writes;
+extern int proc_id;
+extern unsigned num_procs;
 
 extern uint8_t dcb_config;
 extern uint8_t dcb_test;
Signature · 1 line
2.7.4
#1 lore raw
Quoted 9 lines · Lijun Ou

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./testpmd -w xxx --file-prefix=xx -l 0-1 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=0 the secondary cmd:
./testpmd -w xxx --file-prefix=xx -l 2-3 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=1

+1 to add multi-process support to testpmd, missing it reduces multi-process testing a lot, but I guess concern is it may cause lots of changes and create some corner cases in testpmd.

Can you please explain a little why new 'proc-id' testpmd argument is needed, why the regular way of using eal '--proc-type' command and 'RTE_PROC_SECONDARY' checks is not enough?

Also why 'MULTIPLE_PROCESS_HANDLE' define is required?

Thanks,
ferruh

Quoted 411 lines
---
  app/test-pmd/cmdline.c    |   6 ++-
  app/test-pmd/config.c     |   9 +++-
  app/test-pmd/parameters.c |   9 ++++
  app/test-pmd/testpmd.c    | 133 ++++++++++++++++++++++++++++++++--------------
  app/test-pmd/testpmd.h    |   7 +++
  5 files changed, 121 insertions(+), 43 deletions(-)
app/test-pmd/cmdline.c +4 -2
index 2ccbaa0..2237c9f 100644
@@ -71,8 +71,6 @@
  #include "cmdline_tm.h"
  #include "bpf_cmd.h"
 
-static struct cmdline *testpmd_cl;
-
  static void cmd_reconfig_device_queue(portid_t id, uint8_t dev, uint8_t queue);
 
  /* *** Help command with introduction. *** */
@@ -17115,6 +17113,10 @@ prompt(void)
  	if (testpmd_cl == NULL)
  		return;
  	cmdline_interact(testpmd_cl);
+	if (unlikely(f_quit == 1)) {
+		dup2(testpmd_fd_copy, testpmd_cl->s_in);
+		close(testpmd_fd_copy);
+	}
  	cmdline_stdin_exit(testpmd_cl);
  }
 
app/test-pmd/config.c +8 -1
index 3f6c864..1702e0d 100644
@@ -3100,6 +3100,8 @@ rss_fwd_config_setup(void)
  	queueid_t  rxq;
  	queueid_t  nb_q;
  	streamid_t  sm_id;
+	int start;
+	int end;
 
  	nb_q = nb_rxq;
  	if (nb_q > nb_txq)
@@ -3117,7 +3119,10 @@ rss_fwd_config_setup(void)
  	init_fwd_streams();
 
  	setup_fwd_config_of_each_lcore(&cur_fwd_config);
-	rxp = 0; rxq = 0;
+	start = proc_id * nb_q / num_procs;
+	end = start + nb_q / num_procs;
+	rxp = 0;
+	rxq = start;
  	for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
  		struct fwd_stream *fs;
 
@@ -3134,6 +3139,8 @@ rss_fwd_config_setup(void)
  			continue;
  		rxp = 0;
  		rxq++;
+		if (rxq >= end)
+			rxq=start;
  	}
  }
 
app/test-pmd/parameters.c +9 -0
index 414a006..7807afa 100644
@@ -45,6 +45,8 @@
  #include <rte_flow.h>
 
  #include "testpmd.h"
+#define PARAM_PROC_ID "proc-id"
+#define PARAM_NUM_PROCS "num-procs"
 
  static void
  usage(char* progname)
@@ -603,6 +605,8 @@ launch_args_parse(int argc, char** argv)
  		{ "rx-mq-mode",                 1, 0, 0 },
  		{ "record-core-cycles",         0, 0, 0 },
  		{ "record-burst-stats",         0, 0, 0 },
+		{ PARAM_NUM_PROCS,		1, 0, 0 },
+		{ PARAM_PROC_ID, 		1, 0, 0 },
  		{ 0, 0, 0, 0 },
  	};
 
@@ -1359,6 +1363,11 @@ launch_args_parse(int argc, char** argv)
  				record_core_cycles = 1;
  			if (!strcmp(lgopts[opt_idx].name, "record-burst-stats"))
  				record_burst_stats = 1;
+
+			if (strncmp(lgopts[opt_idx].name, PARAM_NUM_PROCS, 8) == 0)
+				num_procs = atoi(optarg);
+			if (strncmp(lgopts[opt_idx].name, PARAM_PROC_ID, 7) == 0)
+				proc_id = atoi(optarg);
  			break;
  		case 'h':
  			usage(argv[0]);
app/test-pmd/testpmd.c +93 -40
index 2b60f6c..7ab5f48 100644
@@ -63,6 +63,8 @@
 
  #include "testpmd.h"
 
+int testpmd_fd_copy = 500; /* the copy of STDIN_FILENO */
+
  #ifndef MAP_HUGETLB
  /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  #define HUGE_FLAG (0x40000)
@@ -79,6 +81,7 @@
 
  #define EXTMEM_HEAP_NAME "extmem"
  #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M
+#define MULTIPLE_PROCESS_HANDLE
 
  uint16_t verbose_level = 0; /**< Silent by default. */
  int testpmd_logtype; /**< Log type for testpmd logs */
@@ -125,6 +128,9 @@ uint8_t port_numa[RTE_MAX_ETHPORTS];
   */
  uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 
+int proc_id = 0;
+unsigned num_procs = 1;
+
  /*
   * Store specified sockets on which TX ring to be used by ports
   * is allocated.
@@ -978,16 +984,36 @@ mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
  			/* wrapper to rte_mempool_create() */
  			TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
  					rte_mbuf_best_mempool_ops());
+#ifdef MULTIPLE_PROCESS_HANDLE
+			if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+				rte_mp = rte_pktmbuf_pool_create(pool_name,
+					 nb_mbuf, mb_mempool_cache, 0,
+					 mbuf_seg_size, socket_id);
+			else
+				rte_mp = rte_mempool_lookup(pool_name);
+#else
  			rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
  				mb_mempool_cache, 0, mbuf_seg_size, socket_id);
+#endif
  			break;
  		}
  	case MP_ALLOC_ANON:
  		{
+#ifdef MULTIPLE_PROCESS_HANDLE
+			if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+				rte_mp = rte_mempool_create_empty(pool_name,
+					nb_mbuf, mb_size,
+					(unsigned int)mb_mempool_cache,
+					sizeof(struct rte_pktmbuf_pool_private),
+					socket_id, 0);
+			else
+				rte_mp = rte_mempool_lookup(pool_name);
+#else
  			rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
  				mb_size, (unsigned int) mb_mempool_cache,
  				sizeof(struct rte_pktmbuf_pool_private),
  				socket_id, mempool_flags);
+#endif
  			if (rte_mp == NULL)
  				goto err;
 
@@ -1017,9 +1043,18 @@ mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 
  			TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
  					rte_mbuf_best_mempool_ops());
+#ifdef MULTIPLE_PROCESS_HANDLE
+			if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+				rte_mp = rte_pktmbuf_pool_create(pool_name,
+					 nb_mbuf, mb_mempool_cache, 0,
+					 mbuf_seg_size, heap_socket);
+			else
+				rte_mp = rte_mempool_lookup(pool_name);
+#else
  			rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
  					mb_mempool_cache, 0, mbuf_seg_size,
  					heap_socket);
+#endif
  			break;
  		}
  	case MP_ALLOC_XBUF:
@@ -2485,21 +2520,28 @@ start_port(portid_t pid)
  				return -1;
  			}
  			/* configure port */
-			diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
+			if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+				diag = rte_eth_dev_configure(pi,
+						     nb_rxq + nb_hairpinq,
  						     nb_txq + nb_hairpinq,
  						     &(port->dev_conf));
-			if (diag != 0) {
-				if (rte_atomic16_cmpset(&(port->port_status),
-				RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
-					printf("Port %d can not be set back "
-							"to stopped\n", pi);
-				printf("Fail to configure port %d\n", pi);
-				/* try to reconfigure port next time */
-				port->need_reconfig = 1;
-				return -1;
+				if (diag != 0) {
+					if (rte_atomic16_cmpset(
+							&(port->port_status),
+							RTE_PORT_HANDLING,
+							RTE_PORT_STOPPED) == 0)
+						printf("Port %d can not be set "
+						       "back to stopped\n", pi);
+					printf("Fail to configure port %d\n",
+						pi);
+					/* try to reconfigure port next time */
+					port->need_reconfig = 1;
+					return -1;
+				}
  			}
  		}
-		if (port->need_reconfig_queues > 0) {
+		if (port->need_reconfig_queues > 0 &&
+		    rte_eal_process_type() == RTE_PROC_PRIMARY) {
  			port->need_reconfig_queues = 0;
  			/* setup tx queues */
  			for (qi = 0; qi < nb_txq; qi++) {
@@ -2600,15 +2642,18 @@ start_port(portid_t pid)
  		cnt_pi++;
 
  		/* start port */
-		if (rte_eth_dev_start(pi) < 0) {
-			printf("Fail to start port %d\n", pi);
-
-			/* Fail to setup rx queue, return */
-			if (rte_atomic16_cmpset(&(port->port_status),
-				RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
-				printf("Port %d can not be set back to "
-							"stopped\n", pi);
-			continue;
+		if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+			diag = rte_eth_dev_start(pi);
+			if (diag < 0) {
+				printf("Fail to start port %d\n", pi);
+
+				/* Fail to setup rx queue, return */
+				if (rte_atomic16_cmpset(&(port->port_status),
+					RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
+					printf("Port %d can not be set back to "
+								"stopped\n", pi);
+				continue;
+			}
  		}
 
  		if (rte_atomic16_cmpset(&(port->port_status),
@@ -2737,7 +2782,7 @@ stop_port(portid_t pid)
  		if (port->flow_list)
  			port_flow_flush(pi);
 
-		if (rte_eth_dev_stop(pi) != 0)
+		if (rte_eal_process_type() == RTE_PROC_PRIMARY && rte_eth_dev_stop(pi) != 0)
  			RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n",
  				pi);
 
@@ -2806,8 +2851,10 @@ close_port(portid_t pid)
  			continue;
  		}
 
-		port_flow_flush(pi);
-		rte_eth_dev_close(pi);
+		if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+			port_flow_flush(pi);
+		if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+			rte_eth_dev_close(pi);
  	}
 
  	remove_invalid_ports();
@@ -3071,7 +3118,7 @@ pmd_test_exit(void)
  		}
  	}
  	for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
-		if (mempools[i])
+		if (rte_eal_process_type() == RTE_PROC_PRIMARY && mempools[i])
  			rte_mempool_free(mempools[i]);
  	}
 
@@ -3519,6 +3566,10 @@ init_port_dcb_config(portid_t pid,
  	int retval;
  	uint16_t i;
 
+	if (num_procs > 1) {
+		printf("The multi-process feature doesn't support dcb.\n");
+		return -ENOTSUP;
+	}
  	rte_port = &ports[pid];
 
  	memset(&port_conf, 0, sizeof(struct rte_eth_conf));
@@ -3617,13 +3668,6 @@ init_port(void)
  }
 
  static void
-force_quit(void)
-{
-	pmd_test_exit();
-	prompt_exit();
-}
-
-static void
  print_stats(void)
  {
  	uint8_t i;
@@ -3654,12 +3698,16 @@ signal_handler(int signum)
  		if (latencystats_enabled != 0)
  			rte_latencystats_uninit();
  #endif
-		force_quit();
  		/* Set flag to indicate the force termination. */
  		f_quit = 1;
-		/* exit with the expected status */
-		signal(signum, SIG_DFL);
-		kill(getpid(), signum);
+		if (interactive == 1) {
+			dup2(testpmd_cl->s_in, testpmd_fd_copy);
+			close(testpmd_cl->s_in);
+		} else {
+			dup2(0, testpmd_fd_copy);
+			close(0);
+		}
+
  	}
  }
 
@@ -3684,10 +3732,6 @@ main(int argc, char** argv)
  		rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
  			 rte_strerror(rte_errno));
 
-	if (rte_eal_process_type() == RTE_PROC_SECONDARY)
-		rte_exit(EXIT_FAILURE,
-			 "Secondary process type not supported.\n");
-
  	ret = register_eth_event_callback();
  	if (ret != 0)
  		rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
@@ -3783,8 +3827,10 @@ main(int argc, char** argv)
  		}
  	}
 
-	if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
+	if (!no_device_start && start_port(RTE_PORT_ALL) != 0) {
+		pmd_test_exit();
  		rte_exit(EXIT_FAILURE, "Start ports failed\n");
+	}
 
  	/* set all ports to promiscuous mode by default */
  	RTE_ETH_FOREACH_DEV(port_id) {
@@ -3830,6 +3876,8 @@ main(int argc, char** argv)
  		}
  		prompt();
  		pmd_test_exit();
+		if (unlikely(f_quit == 1))
+			prompt_exit();
  	} else
  #endif
  	{
@@ -3865,6 +3913,11 @@ main(int argc, char** argv)
  		printf("Press enter to exit\n");
  		rc = read(0, &c, 1);
  		pmd_test_exit();
+		if (unlikely(f_quit == 1)) {
+			dup2(testpmd_fd_copy, 0);
+			close(testpmd_fd_copy);
+			prompt_exit();
+		}
  		if (rc < 0)
  			return 1;
  	}
app/test-pmd/testpmd.h +7 -0
index 5f23162..8629c57 100644
@@ -13,6 +13,7 @@
  #include <rte_gso.h>
  #include <cmdline.h>
  #include <sys/queue.h>
+#include  "cmdline.h"
 
  #define RTE_PORT_ALL            (~(portid_t)0x0)
 
@@ -24,6 +25,10 @@
  #define RTE_PORT_CLOSED         (uint16_t)2
  #define RTE_PORT_HANDLING       (uint16_t)3
 
+uint8_t f_quit;
+int testpmd_fd_copy;
+struct cmdline *testpmd_cl;
+
  /*
   * It is used to allocate the memory for hash key.
   * The hash key size is NIC dependent.
@@ -421,6 +426,8 @@ extern uint64_t noisy_lkup_mem_sz;
  extern uint64_t noisy_lkup_num_writes;
  extern uint64_t noisy_lkup_num_reads;
  extern uint64_t noisy_lkup_num_reads_writes;
+extern int proc_id;
+extern unsigned num_procs;
 
  extern uint8_t dcb_config;
  extern uint8_t dcb_test;
author
#2 lore raw
Quoted 11 lines · Ferruh Yigit
Quoted 9 lines · Lijun Ou

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./testpmd -w xxx --file-prefix=xx -l 0-1 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=0 the secondary cmd:
./testpmd -w xxx --file-prefix=xx -l 2-3 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=1

Quoted 3 lines

+1 to add multi-process support to testpmd, missing it reduces
multi-process testing a lot, but I guess concern is it may cause lots of changes and create some corner cases in testpmd.

Yes, We have not found any other problems.

Quoted 1 line

Can you please explain a little why new 'proc-id' testpmd argument is needed, why the regular way of using eal '--proc-type' command and 'RTE_PROC_SECONDARY' checks is not enough?

We refer to the definition of symmetric_mp,We need him to identify exactly which one from the process.

Quoted 4 lines

Also why 'MULTIPLE_PROCESS_HANDLE' define is required?

Yes, I think it is unncessary. I will delete it.

Quoted 387 lines

Thanks,
ferruh

Quoted 373 lines
---
  app/test-pmd/cmdline.c    |   6 ++-
  app/test-pmd/config.c     |   9 +++-
  app/test-pmd/parameters.c |   9 ++++
  app/test-pmd/testpmd.c    | 133 
++++++++++++++++++++++++++++++++--------------
  app/test-pmd/testpmd.h    |   7 +++
  5 files changed, 121 insertions(+), 43 deletions(-)
app/test-pmd/cmdline.c +4 -2
index 2ccbaa0..2237c9f 100644
@@ -71,8 +71,6 @@
  #include "cmdline_tm.h"
  #include "bpf_cmd.h"
-static struct cmdline *testpmd_cl;
-
  static void cmd_reconfig_device_queue(portid_t id, uint8_t dev, 
 uint8_t queue);
  /* *** Help command with introduction. *** */
@@ -17115,6 +17113,10 @@ prompt(void)
      if (testpmd_cl == NULL)
          return;
      cmdline_interact(testpmd_cl);
+    if (unlikely(f_quit == 1)) {
+        dup2(testpmd_fd_copy, testpmd_cl->s_in);
+        close(testpmd_fd_copy);
+    }
      cmdline_stdin_exit(testpmd_cl);
  }
app/test-pmd/config.c +8 -1
index 3f6c864..1702e0d 100644
@@ -3100,6 +3100,8 @@ rss_fwd_config_setup(void)
      queueid_t  rxq;
      queueid_t  nb_q;
      streamid_t  sm_id;
+    int start;
+    int end;
      nb_q = nb_rxq;
      if (nb_q > nb_txq)
@@ -3117,7 +3119,10 @@ rss_fwd_config_setup(void)
      init_fwd_streams();
      setup_fwd_config_of_each_lcore(&cur_fwd_config);
-    rxp = 0; rxq = 0;
+    start = proc_id * nb_q / num_procs;
+    end = start + nb_q / num_procs;
+    rxp = 0;
+    rxq = start;
      for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
          struct fwd_stream *fs;
@@ -3134,6 +3139,8 @@ rss_fwd_config_setup(void)
              continue;
          rxp = 0;
          rxq++;
+        if (rxq >= end)
+            rxq=start;
      }
  }
app/test-pmd/parameters.c +9 -0
index 414a006..7807afa 100644
@@ -45,6 +45,8 @@
  #include <rte_flow.h>
  #include "testpmd.h"
+#define PARAM_PROC_ID "proc-id"
+#define PARAM_NUM_PROCS "num-procs"
  static void
  usage(char* progname)
@@ -603,6 +605,8 @@ launch_args_parse(int argc, char** argv)
          { "rx-mq-mode",                 1, 0, 0 },
          { "record-core-cycles",         0, 0, 0 },
          { "record-burst-stats",         0, 0, 0 },
+        { PARAM_NUM_PROCS,        1, 0, 0 },
+        { PARAM_PROC_ID,         1, 0, 0 },
          { 0, 0, 0, 0 },
      };
@@ -1359,6 +1363,11 @@ launch_args_parse(int argc, char** argv)
                  record_core_cycles = 1;
              if (!strcmp(lgopts[opt_idx].name, "record-burst-stats"))
                  record_burst_stats = 1;
+
+            if (strncmp(lgopts[opt_idx].name, PARAM_NUM_PROCS, 8) == 0)
+                num_procs = atoi(optarg);
+            if (strncmp(lgopts[opt_idx].name, PARAM_PROC_ID, 7) == 0)
+                proc_id = atoi(optarg);
              break;
          case 'h':
              usage(argv[0]);
app/test-pmd/testpmd.c +93 -40
index 2b60f6c..7ab5f48 100644
@@ -63,6 +63,8 @@
  #include "testpmd.h"
+int testpmd_fd_copy = 500; /* the copy of STDIN_FILENO */
+
  #ifndef MAP_HUGETLB
  /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  #define HUGE_FLAG (0x40000)
@@ -79,6 +81,7 @@
  #define EXTMEM_HEAP_NAME "extmem"
  #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M
+#define MULTIPLE_PROCESS_HANDLE
  uint16_t verbose_level = 0; /**< Silent by default. */
  int testpmd_logtype; /**< Log type for testpmd logs */
@@ -125,6 +128,9 @@ uint8_t port_numa[RTE_MAX_ETHPORTS];
   */
  uint8_t rxring_numa[RTE_MAX_ETHPORTS];
+int proc_id = 0;
+unsigned num_procs = 1;
+
  /*
   * Store specified sockets on which TX ring to be used by ports
   * is allocated.
@@ -978,16 +984,36 @@ mbuf_pool_create(uint16_t mbuf_seg_size, 
 unsigned nb_mbuf,
              /* wrapper to rte_mempool_create() */
              TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
                      rte_mbuf_best_mempool_ops());
+#ifdef MULTIPLE_PROCESS_HANDLE
+            if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+                rte_mp = rte_pktmbuf_pool_create(pool_name,
+                     nb_mbuf, mb_mempool_cache, 0,
+                     mbuf_seg_size, socket_id);
+            else
+                rte_mp = rte_mempool_lookup(pool_name);
+#else
              rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
                  mb_mempool_cache, 0, mbuf_seg_size, socket_id);
+#endif
              break;
          }
      case MP_ALLOC_ANON:
          {
+#ifdef MULTIPLE_PROCESS_HANDLE
+            if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+                rte_mp = rte_mempool_create_empty(pool_name,
+                    nb_mbuf, mb_size,
+                    (unsigned int)mb_mempool_cache,
+                    sizeof(struct rte_pktmbuf_pool_private),
+                    socket_id, 0);
+            else
+                rte_mp = rte_mempool_lookup(pool_name);
+#else
              rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
                  mb_size, (unsigned int) mb_mempool_cache,
                  sizeof(struct rte_pktmbuf_pool_private),
                  socket_id, mempool_flags);
+#endif
              if (rte_mp == NULL)
                  goto err;
@@ -1017,9 +1043,18 @@ mbuf_pool_create(uint16_t mbuf_seg_size, 
 unsigned nb_mbuf,
              TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
                      rte_mbuf_best_mempool_ops());
+#ifdef MULTIPLE_PROCESS_HANDLE
+            if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+                rte_mp = rte_pktmbuf_pool_create(pool_name,
+                     nb_mbuf, mb_mempool_cache, 0,
+                     mbuf_seg_size, heap_socket);
+            else
+                rte_mp = rte_mempool_lookup(pool_name);
+#else
              rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
                      mb_mempool_cache, 0, mbuf_seg_size,
                      heap_socket);
+#endif
              break;
          }
      case MP_ALLOC_XBUF:
@@ -2485,21 +2520,28 @@ start_port(portid_t pid)
                  return -1;
              }
              /* configure port */
-            diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
+            if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+                diag = rte_eth_dev_configure(pi,
+                             nb_rxq + nb_hairpinq,
                               nb_txq + nb_hairpinq,
                               &(port->dev_conf));
-            if (diag != 0) {
-                if (rte_atomic16_cmpset(&(port->port_status),
-                RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
-                    printf("Port %d can not be set back "
-                            "to stopped\n", pi);
-                printf("Fail to configure port %d\n", pi);
-                /* try to reconfigure port next time */
-                port->need_reconfig = 1;
-                return -1;
+                if (diag != 0) {
+                    if (rte_atomic16_cmpset(
+                            &(port->port_status),
+                            RTE_PORT_HANDLING,
+                            RTE_PORT_STOPPED) == 0)
+                        printf("Port %d can not be set "
+                               "back to stopped\n", pi);
+                    printf("Fail to configure port %d\n",
+                        pi);
+                    /* try to reconfigure port next time */
+                    port->need_reconfig = 1;
+                    return -1;
+                }
              }
          }
-        if (port->need_reconfig_queues > 0) {
+        if (port->need_reconfig_queues > 0 &&
+            rte_eal_process_type() == RTE_PROC_PRIMARY) {
              port->need_reconfig_queues = 0;
              /* setup tx queues */
              for (qi = 0; qi < nb_txq; qi++) {
@@ -2600,15 +2642,18 @@ start_port(portid_t pid)
          cnt_pi++;
          /* start port */
-        if (rte_eth_dev_start(pi) < 0) {
-            printf("Fail to start port %d\n", pi);
-
-            /* Fail to setup rx queue, return */
-            if (rte_atomic16_cmpset(&(port->port_status),
-                RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
-                printf("Port %d can not be set back to "
-                            "stopped\n", pi);
-            continue;
+        if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+            diag = rte_eth_dev_start(pi);
+            if (diag < 0) {
+                printf("Fail to start port %d\n", pi);
+
+                /* Fail to setup rx queue, return */
+                if (rte_atomic16_cmpset(&(port->port_status),
+                    RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
+                    printf("Port %d can not be set back to "
+                                "stopped\n", pi);
+                continue;
+            }
          }
          if (rte_atomic16_cmpset(&(port->port_status),
@@ -2737,7 +2782,7 @@ stop_port(portid_t pid)
          if (port->flow_list)
              port_flow_flush(pi);
-        if (rte_eth_dev_stop(pi) != 0)
+        if (rte_eal_process_type() == RTE_PROC_PRIMARY && 
 rte_eth_dev_stop(pi) != 0)
              RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n",
                  pi);
@@ -2806,8 +2851,10 @@ close_port(portid_t pid)
              continue;
          }
-        port_flow_flush(pi);
-        rte_eth_dev_close(pi);
+        if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+            port_flow_flush(pi);
+        if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+            rte_eth_dev_close(pi);
      }
      remove_invalid_ports();
@@ -3071,7 +3118,7 @@ pmd_test_exit(void)
          }
      }
      for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
-        if (mempools[i])
+        if (rte_eal_process_type() == RTE_PROC_PRIMARY && mempools[i])
              rte_mempool_free(mempools[i]);
      }
@@ -3519,6 +3566,10 @@ init_port_dcb_config(portid_t pid,
      int retval;
      uint16_t i;
+    if (num_procs > 1) {
+        printf("The multi-process feature doesn't support dcb.\n");
+        return -ENOTSUP;
+    }
      rte_port = &ports[pid];
      memset(&port_conf, 0, sizeof(struct rte_eth_conf));
@@ -3617,13 +3668,6 @@ init_port(void)
  }
  static void
-force_quit(void)
-{
-    pmd_test_exit();
-    prompt_exit();
-}
-
-static void
  print_stats(void)
  {
      uint8_t i;
@@ -3654,12 +3698,16 @@ signal_handler(int signum)
          if (latencystats_enabled != 0)
              rte_latencystats_uninit();
  #endif
-        force_quit();
          /* Set flag to indicate the force termination. */
          f_quit = 1;
-        /* exit with the expected status */
-        signal(signum, SIG_DFL);
-        kill(getpid(), signum);
+        if (interactive == 1) {
+            dup2(testpmd_cl->s_in, testpmd_fd_copy);
+            close(testpmd_cl->s_in);
+        } else {
+            dup2(0, testpmd_fd_copy);
+            close(0);
+        }
+
      }
  }
@@ -3684,10 +3732,6 @@ main(int argc, char** argv)
          rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
               rte_strerror(rte_errno));
-    if (rte_eal_process_type() == RTE_PROC_SECONDARY)
-        rte_exit(EXIT_FAILURE,
-             "Secondary process type not supported.\n");
-
      ret = register_eth_event_callback();
      if (ret != 0)
          rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
@@ -3783,8 +3827,10 @@ main(int argc, char** argv)
          }
      }
-    if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
+    if (!no_device_start && start_port(RTE_PORT_ALL) != 0) {
+        pmd_test_exit();
          rte_exit(EXIT_FAILURE, "Start ports failed\n");
+    }
      /* set all ports to promiscuous mode by default */
      RTE_ETH_FOREACH_DEV(port_id) {
@@ -3830,6 +3876,8 @@ main(int argc, char** argv)
          }
          prompt();
          pmd_test_exit();
+        if (unlikely(f_quit == 1))
+            prompt_exit();
      } else
  #endif
      {
@@ -3865,6 +3913,11 @@ main(int argc, char** argv)
          printf("Press enter to exit\n");
          rc = read(0, &c, 1);
          pmd_test_exit();
+        if (unlikely(f_quit == 1)) {
+            dup2(testpmd_fd_copy, 0);
+            close(testpmd_fd_copy);
+            prompt_exit();
+        }
          if (rc < 0)
              return 1;
      }
app/test-pmd/testpmd.h +5 -0
index 5f23162..8629c57 100644
@@ -13,6 +13,7 @@
  #include <rte_gso.h>
  #include <cmdline.h>
  #include <sys/queue.h>
+#include  "cmdline.h"
  #define RTE_PORT_ALL            (~(portid_t)0x0)
@@ -24,6 +25,10 @@
  #define RTE_PORT_CLOSED         (uint16_t)2
  #define RTE_PORT_HANDLING       (uint16_t)3
+uint8_t f_quit;
+int testpmd_fd_copy;
+struct cmdline *testpmd_cl;
+
  /*
   * It is used to allocate the memory for hash key.
   * The hash key size is NIC dependent.

app/test-pmd/testpmd.h extern uint64_t noisy_lkup_mem_sz; · line 433

Quoted 8 lines
app/test-pmd/testpmd.h +2 -0
@@ -421,6 +426,8 @@ extern uint64_t noisy_lkup_mem_sz;
  extern uint64_t noisy_lkup_num_writes;
  extern uint64_t noisy_lkup_num_reads;
  extern uint64_t noisy_lkup_num_reads_writes;
+extern int proc_id;
+extern unsigned num_procs;
  extern uint8_t dcb_config;
  extern uint8_t dcb_test;
 

.

#3 lore raw
In reply to Lijun Ou

Hi,

Quoted 18 lines

-----Original Message-----
From: dev <[email protected]> On Behalf Of Lijun Ou
Sent: Friday, January 8, 2021 11:46 AM
To: [email protected]; [email protected]; [email protected]; [email protected]
Cc: [email protected]
Subject: [dpdk-dev] [RFC] app/testpmd: support multi-process

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./testpmd -w xxx --file-prefix=xx -l 0-1 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=0 the secondary cmd:
./testpmd -w xxx --file-prefix=xx -l 2-3 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=1

Quoted 9 lines
---
 app/test-pmd/cmdline.c    |   6 ++-
 app/test-pmd/config.c     |   9 +++-
 app/test-pmd/parameters.c |   9 ++++
 app/test-pmd/testpmd.c    | 133 ++++++++++++++++++++++++++++++++--
------------
 app/test-pmd/testpmd.h    |   7 +++
 5 files changed, 121 insertions(+), 43 deletions(-)

+1 for having this support for testpmd.

Some questions in my mind:
How are the queues distributing here? In example I see 16 defined, are they for one instance or for all? Will all processes have same memory region? If installing one RTE_FLOW in one instance will be active for all? Same question for detaching device in one instance, how it will reflect on others?
There is many other scenarios like this, how it will handle those?

BRs,
Wisam Jaddo

author
#4 lore raw
Quoted 30 lines · Wisam Monther

Hi,

Quoted 18 lines

-----Original Message-----
From: dev <[email protected]> On Behalf Of Lijun Ou
Sent: Friday, January 8, 2021 11:46 AM
To: [email protected]; [email protected]; [email protected]; [email protected]
Cc: [email protected]
Subject: [dpdk-dev] [RFC] app/testpmd: support multi-process

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./testpmd -w xxx --file-prefix=xx -l 0-1 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=0 the secondary cmd:
./testpmd -w xxx --file-prefix=xx -l 2-3 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=1

Quoted 9 lines
---
  app/test-pmd/cmdline.c    |   6 ++-
  app/test-pmd/config.c     |   9 +++-
  app/test-pmd/parameters.c |   9 ++++
  app/test-pmd/testpmd.c    | 133 ++++++++++++++++++++++++++++++++--
------------
  app/test-pmd/testpmd.h    |   7 +++
  5 files changed, 121 insertions(+), 43 deletions(-)
Quoted 8 lines

+1 for having this support for testpmd.

Some questions in my mind:
How are the queues distributing here? In example I see 16 defined, are they for one instance or for all? Will all processes have same memory region? If installing one RTE_FLOW in one instance will be active for all? Same question for detaching device in one instance, how it will reflect on others?
There is many other scenarios like this, how it will handle those?

Hi,Wisam Monther
Firstly, thank you for your questions. According to the current
implementation scheme, all queues are evenly allocated to different processes based on proc_num and proc_id。
The number of receiving queues, number of processes, and process ID are specified for the master and slave processes. After being created by the main process, the sending and receiving queues are evenly distributed to all processes. The following shows the calculation rule for the Testpmd to allocate queues to each process after the proc ID is specified. start(queue start id) = proc_id * nb_q / num_procs;
end(queue end id) = start + nb_q / num_procs;

For example, if support 16 txq and rxq
the 0 ~7 for primary process
the 8 ~15 for secondary process

all process have the same memory region.

Thanks
Lijun Ou

Quoted 4 lines

BRs,
Wisam Jaddo
.

#5 lore raw
Quoted 68 lines

-----Original Message-----
From: oulijun <[email protected]>
Sent: Tuesday, January 12, 2021 4:13 PM
To: Wisam Monther <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]
Cc: [email protected]
Subject: Re: [dpdk-dev] [RFC] app/testpmd: support multi-process

Quoted 25 lines · Wisam Monther

Hi,

Quoted 23 lines

-----Original Message-----
From: dev <[email protected]> On Behalf Of Lijun Ou
Sent: Friday, January 8, 2021 11:46 AM
To: [email protected]; [email protected]; [email protected]; [email protected]
Cc: [email protected]
Subject: [dpdk-dev] [RFC] app/testpmd: support multi-process

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./testpmd -w xxx --file-prefix=xx -l 0-1 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=0 the secondary cmd:
./testpmd -w xxx --file-prefix=xx -l 2-3 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=1

---
  app/test-pmd/cmdline.c    |   6 ++-
  app/test-pmd/config.c     |   9 +++-
  app/test-pmd/parameters.c |   9 ++++
  app/test-pmd/testpmd.c    | 133

++++++++++++++++++++++++++++++++--

Quoted 9 lines
Quoted 4 lines

------------

  app/test-pmd/testpmd.h    |   7 +++
  5 files changed, 121 insertions(+), 43 deletions(-)

+1 for having this support for testpmd.

Some questions in my mind:
How are the queues distributing here? In example I see 16 defined, are

they for one instance or for all?

Quoted 1 line

Will all processes have same memory region? If installing one RTE_FLOW in

one instance will be active for all?

Quoted 1 line

Same question for detaching device in one instance, how it will reflect on

others?

Quoted 2 lines

There is many other scenarios like this, how it will handle those?

Hi,Wisam Monther
Firstly, thank you for your questions. According to the current
implementation scheme, all queues are evenly allocated to different processes based on proc_num and proc_id。
The number of receiving queues, number of processes, and process ID are specified for the master and slave processes. After being created by the main process, the sending and receiving queues are evenly distributed to all processes. The following shows the calculation rule for the Testpmd to allocate queues to each process after the proc ID is specified. start(queue start id) = proc_id * nb_q / num_procs;
end(queue end id) = start + nb_q / num_procs;

Quoted 5 lines

For example, if support 16 txq and rxq
the 0 ~7 for primary process
the 8 ~15 for secondary process

all process have the same memory region.

Ok great, Thanks that' make sense.
What about the impact of installing RTE_FLOW rule into one instance, does it apply on all? I'm talking about SW side, not HW side? The impact of detaching device from secondary/primary only, how it's designed to handle such cases?

Quoted 8 lines

Thanks
Lijun Ou

Quoted 4 lines

BRs,
Wisam Jaddo
.

author
#6 lore raw
Quoted 72 lines · Wisam Monther
Quoted 72 lines

-----Original Message-----
From: oulijun <[email protected]>
Sent: Tuesday, January 12, 2021 4:13 PM
To: Wisam Monther <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]
Cc: [email protected]
Subject: Re: [dpdk-dev] [RFC] app/testpmd: support multi-process

Quoted 25 lines · Wisam Monther

Hi,

Quoted 23 lines

-----Original Message-----
From: dev <[email protected]> On Behalf Of Lijun Ou
Sent: Friday, January 8, 2021 11:46 AM
To: [email protected]; [email protected]; [email protected]; [email protected]
Cc: [email protected]
Subject: [dpdk-dev] [RFC] app/testpmd: support multi-process

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./testpmd -w xxx --file-prefix=xx -l 0-1 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=0 the secondary cmd:
./testpmd -w xxx --file-prefix=xx -l 2-3 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=1

---
   app/test-pmd/cmdline.c    |   6 ++-
   app/test-pmd/config.c     |   9 +++-
   app/test-pmd/parameters.c |   9 ++++
   app/test-pmd/testpmd.c    | 133

++++++++++++++++++++++++++++++++--

Quoted 9 lines
Quoted 4 lines

------------

   app/test-pmd/testpmd.h    |   7 +++
   5 files changed, 121 insertions(+), 43 deletions(-)

+1 for having this support for testpmd.

Some questions in my mind:
How are the queues distributing here? In example I see 16 defined, are

they for one instance or for all?

Quoted 1 line

Will all processes have same memory region? If installing one RTE_FLOW in

one instance will be active for all?

Quoted 1 line

Same question for detaching device in one instance, how it will reflect on

others?

Quoted 2 lines

There is many other scenarios like this, how it will handle those?

Hi,Wisam Monther
Firstly, thank you for your questions. According to the current
implementation scheme, all queues are evenly allocated to different processes based on proc_num and proc_id。
The number of receiving queues, number of processes, and process ID are specified for the master and slave processes. After being created by the main process, the sending and receiving queues are evenly distributed to all processes. The following shows the calculation rule for the Testpmd to allocate queues to each process after the proc ID is specified. start(queue start id) = proc_id * nb_q / num_procs;
end(queue end id) = start + nb_q / num_procs;

For example, if support 16 txq and rxq
the 0 ~7 for primary process
the 8 ~15 for secondary process

Quoted 6 lines
Quoted 1 line

all process have the same memory region.

Ok great, Thanks that' make sense.
What about the impact of installing RTE_FLOW rule into one instance, does it apply on all? I'm talking about SW side, not HW side? The impact of detaching device from secondary/primary only, how it's designed to handle such cases?

It is my understanding that installing RTE_FLOW rule will affect a instance from our modification.But the all instance will changed from HW side. If detaching device from secondary only, it will not affect other processes.

Quoted 10 lines
Quoted 8 lines

Thanks
Lijun Ou

Quoted 4 lines

BRs,
Wisam Jaddo
.

.

author
#7 lore raw

RFC v2 1/1 app/testpmd: support multi-process

In reply to Lijun Ou

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./testpmd -w xxx --file-prefix=xx -l 0-1 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=0 the secondary cmd:
./testpmd -w xxx --file-prefix=xx -l 2-3 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=1

---
 app/test-pmd/cmdline.c    |   6 ++-
 app/test-pmd/config.c     |   9 +++-
 app/test-pmd/parameters.c |   9 ++++
 app/test-pmd/testpmd.c    | 135 +++++++++++++++++++++++++++++-----------------
 app/test-pmd/testpmd.h    |   7 +++
 5 files changed, 114 insertions(+), 52 deletions(-)
app/test-pmd/cmdline.c +4 -2
index 89034c8..48af5cd 100644
@@ -71,8 +71,6 @@
 #include "cmdline_tm.h"
 #include "bpf_cmd.h"
 
-static struct cmdline *testpmd_cl;
-
 static void cmd_reconfig_device_queue(portid_t id, uint8_t dev, uint8_t queue);
 
 /* *** Help command with introduction. *** */
@@ -17124,6 +17122,10 @@ prompt(void)
 	if (testpmd_cl == NULL)
 		return;
 	cmdline_interact(testpmd_cl);
+	if (unlikely(f_quit == 1)) {
+		dup2(testpmd_fd_copy, testpmd_cl->s_in);
+		close(testpmd_fd_copy);
+	}
 	cmdline_stdin_exit(testpmd_cl);
 }
 
app/test-pmd/config.c +8 -1
index 0e2b9f7..f065008 100644
@@ -3100,6 +3100,8 @@ rss_fwd_config_setup(void)
 	queueid_t  rxq;
 	queueid_t  nb_q;
 	streamid_t  sm_id;
+	int start;
+	int end;
 
 	nb_q = nb_rxq;
 	if (nb_q > nb_txq)
@@ -3117,7 +3119,10 @@ rss_fwd_config_setup(void)
 	init_fwd_streams();
 
 	setup_fwd_config_of_each_lcore(&cur_fwd_config);
-	rxp = 0; rxq = 0;
+	start = proc_id * nb_q / num_procs;
+	end = start + nb_q / num_procs;
+	rxp = 0;
+	rxq = start;
 	for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
 		struct fwd_stream *fs;
 
@@ -3134,6 +3139,8 @@ rss_fwd_config_setup(void)
 			continue;
 		rxp = 0;
 		rxq++;
+		if (rxq >= end)
+			rxq = start;
 	}
 }
 
app/test-pmd/parameters.c +9 -0
index df5eb10..ac63854 100644
@@ -45,6 +45,8 @@
 #include <rte_flow.h>
 
 #include "testpmd.h"
+#define PARAM_PROC_ID "proc-id"
+#define PARAM_NUM_PROCS "num-procs"
 
 static void
 usage(char* progname)
@@ -603,6 +605,8 @@ launch_args_parse(int argc, char** argv)
 		{ "rx-mq-mode",                 1, 0, 0 },
 		{ "record-core-cycles",         0, 0, 0 },
 		{ "record-burst-stats",         0, 0, 0 },
+		{ PARAM_NUM_PROCS,		1, 0, 0 },
+		{ PARAM_PROC_ID, 		1, 0, 0 },
 		{ 0, 0, 0, 0 },
 	};
 
@@ -1356,6 +1360,11 @@ launch_args_parse(int argc, char** argv)
 				record_core_cycles = 1;
 			if (!strcmp(lgopts[opt_idx].name, "record-burst-stats"))
 				record_burst_stats = 1;
+
+			if (strncmp(lgopts[opt_idx].name, PARAM_NUM_PROCS, 8) == 0)
+				num_procs = atoi(optarg);
+			if (strncmp(lgopts[opt_idx].name, PARAM_PROC_ID, 7) == 0)
+				proc_id = atoi(optarg);
 			break;
 		case 'h':
 			usage(argv[0]);
app/test-pmd/testpmd.c +86 -49

242 lines not on this page · raw message

app/test-pmd/testpmd.h +7 -0
index 5f23162..8629c57 100644
@@ -13,6 +13,7 @@
 #include <rte_gso.h>
 #include <cmdline.h>
 #include <sys/queue.h>
+#include  "cmdline.h"
 
 #define RTE_PORT_ALL            (~(portid_t)0x0)
 
@@ -24,6 +25,10 @@
 #define RTE_PORT_CLOSED         (uint16_t)2
 #define RTE_PORT_HANDLING       (uint16_t)3
 
+uint8_t f_quit;
+int testpmd_fd_copy;
+struct cmdline *testpmd_cl;
+
 /*
  * It is used to allocate the memory for hash key.
  * The hash key size is NIC dependent.
@@ -421,6 +426,8 @@ extern uint64_t noisy_lkup_mem_sz;
 extern uint64_t noisy_lkup_num_writes;
 extern uint64_t noisy_lkup_num_reads;
 extern uint64_t noisy_lkup_num_reads_writes;
+extern int proc_id;
+extern unsigned num_procs;
 
 extern uint8_t dcb_config;
 extern uint8_t dcb_test;
Signature · 1 line
2.7.4
author
#8 lore raw

PATCH 1/1 app/testpmd: support multi-process

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./testpmd -w xxx --file-prefix=xx -l 0-1 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=0 the secondary cmd:
./testpmd -w xxx --file-prefix=xx -l 2-3 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=1

---
 app/test-pmd/cmdline.c    |  12 +++-
 app/test-pmd/config.c     |   9 ++-
 app/test-pmd/parameters.c |  11 ++++
 app/test-pmd/testpmd.c    | 138 ++++++++++++++++++++++++++++++----------------
 app/test-pmd/testpmd.h    |   7 +++
 5 files changed, 127 insertions(+), 50 deletions(-)
app/test-pmd/cmdline.c +10 -2
index 8b0f7d5..610e850 100644
@@ -72,8 +72,6 @@
 #include "cmdline_tm.h"
 #include "bpf_cmd.h"
 
-static struct cmdline *testpmd_cl;
-
 static void cmd_reconfig_device_queue(portid_t id, uint8_t dev, uint8_t queue);
 
 /* *** Help command with introduction. *** */
@@ -5353,6 +5351,12 @@ cmd_set_flush_rx_parsed(void *parsed_result,
 		__rte_unused void *data)
 {
 	struct cmd_set_flush_rx *res = parsed_result;
+
+	if (num_procs > 1 && (strcmp(res->mode, "on") == 0)) {
+		printf("multi-process doesn't support to flush rx queues.\n");
+		return;
+	}
+
 	no_flush_rx = (uint8_t)((strcmp(res->mode, "on") == 0) ? 0 : 1);
 }
 
@@ -17316,6 +17320,10 @@ prompt(void)
 		printf("Cannot set exit function for cmdline\n");
 
 	cmdline_interact(testpmd_cl);
+	if (unlikely(f_quit == 1)) {
+		dup2(testpmd_fd_copy, testpmd_cl->s_in);
+		close(testpmd_fd_copy);
+	}
 	if (ret != 0)
 		cmdline_stdin_exit(testpmd_cl);
 }
app/test-pmd/config.c +8 -1
index 2435c26..8ab8153 100644
@@ -2827,6 +2827,8 @@ rss_fwd_config_setup(void)
 	queueid_t  rxq;
 	queueid_t  nb_q;
 	streamid_t  sm_id;
+	int start;
+	int end;
 
 	nb_q = nb_rxq;
 	if (nb_q > nb_txq)
@@ -2844,7 +2846,10 @@ rss_fwd_config_setup(void)
 	init_fwd_streams();
 
 	setup_fwd_config_of_each_lcore(&cur_fwd_config);
-	rxp = 0; rxq = 0;
+	start = proc_id * nb_q / num_procs;
+	end = start + nb_q / num_procs;
+	rxp = 0;
+	rxq = start;
 	for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
 		struct fwd_stream *fs;
 
@@ -2861,6 +2866,8 @@ rss_fwd_config_setup(void)
 			continue;
 		rxp = 0;
 		rxq++;
+		if (rxq >= end)
+			rxq = start;
 	}
 }
 
app/test-pmd/parameters.c +11 -0
index c8acd5d..1a5bf25 100644
@@ -45,6 +45,8 @@
 #include <rte_flow.h>
 
 #include "testpmd.h"
+#define PARAM_PROC_ID "proc-id"
+#define PARAM_NUM_PROCS "num-procs"
 
 static void
 usage(char* progname)
@@ -605,6 +607,8 @@ launch_args_parse(int argc, char** argv)
 		{ "rx-mq-mode",                 1, 0, 0 },
 		{ "record-core-cycles",         0, 0, 0 },
 		{ "record-burst-stats",         0, 0, 0 },
+		{ PARAM_NUM_PROCS,              1, 0, 0 },
+		{ PARAM_PROC_ID,                1, 0, 0 },
 		{ 0, 0, 0, 0 },
 	};
 
@@ -1366,6 +1370,13 @@ launch_args_parse(int argc, char** argv)
 				record_core_cycles = 1;
 			if (!strcmp(lgopts[opt_idx].name, "record-burst-stats"))
 				record_burst_stats = 1;
+
+			if (strncmp(lgopts[opt_idx].name,
+				    PARAM_NUM_PROCS, 8) == 0)
+				num_procs = atoi(optarg);
+			if (strncmp(lgopts[opt_idx].name,
+				    PARAM_PROC_ID, 7) == 0)
+				proc_id = atoi(optarg);
 			break;
 		case 'h':
 			usage(argv[0]);
app/test-pmd/testpmd.c +91 -47

253 lines not on this page · raw message

app/test-pmd/testpmd.h +7 -0
index ce83f31..6d13417 100644
@@ -13,6 +13,7 @@
 #include <rte_gso.h>
 #include <cmdline.h>
 #include <sys/queue.h>
+#include  "cmdline.h"
 
 #define RTE_PORT_ALL            (~(portid_t)0x0)
 
@@ -24,6 +25,10 @@
 #define RTE_PORT_CLOSED         (uint16_t)2
 #define RTE_PORT_HANDLING       (uint16_t)3
 
+uint8_t f_quit;
+int testpmd_fd_copy;
+struct cmdline *testpmd_cl;
+
 /*
  * It is used to allocate the memory for hash key.
  * The hash key size is NIC dependent.
@@ -422,6 +427,8 @@ extern uint64_t noisy_lkup_mem_sz;
 extern uint64_t noisy_lkup_num_writes;
 extern uint64_t noisy_lkup_num_reads;
 extern uint64_t noisy_lkup_num_reads_writes;
+extern int proc_id;
+extern unsigned int num_procs;
 
 extern uint8_t dcb_config;
 extern uint8_t dcb_test;
Signature · 1 line
2.7.4
#9 lore raw
Quoted 9 lines · Lijun Ou

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./testpmd -w xxx --file-prefix=xx -l 0-1 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=0 the secondary cmd:
./testpmd -w xxx --file-prefix=xx -l 2-3 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=1

Can you update this in the testpmd usage guide.
There are so many things that testpmd can do.
What will happen to the stats if I quit one of the processes and start again?
In case of multiple Rx queues, how will RSS happen?
It will be good to have the behavior of various features documented - some of the questions have been already answered.
They can be documented as well.

Quoted 3 lines

2.7.4

#10 lore raw
Quoted 11 lines · Ajit Khaparde
Quoted 9 lines · Lijun Ou

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./testpmd -w xxx --file-prefix=xx -l 0-1 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=0 the secondary cmd:
./testpmd -w xxx --file-prefix=xx -l 2-3 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=1

Quoted 1 line

Can you update this in the testpmd usage guide.

OK, I will do it in V2.

Quoted 3 lines

There are so many things that testpmd can do.
What will happen to the stats if I quit one of the processes and start again?

stats will not change when one quit and start, As they share the same buffer to store the stats.

Quoted 1 line

In case of multiple Rx queues, how will RSS happen?

Primary process and secondary process has seperate queues to use, RSS will work in their own queues whether primary and secondary process.

Quoted 3 lines

It will be good to have the behavior of various features documented - some of the questions have been already answered.
They can be documented as well.

OK, I will do it in V2, thanks.

Quoted 2 lines
#11 lore raw

PATCH v2 1/1 app/testpmd: support multi-process

In reply to Lijun Ou

From: Lijun Ou <[email protected]>

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

---
 app/test-pmd/cmdline.c                |  12 ++-
 app/test-pmd/config.c                 |   9 ++-
 app/test-pmd/parameters.c             |  11 +++
 app/test-pmd/testpmd.c                | 138 ++++++++++++++++++++++------------
 app/test-pmd/testpmd.h                |   7 ++
 doc/guides/testpmd_app_ug/run_app.rst |  69 +++++++++++++++++
 6 files changed, 196 insertions(+), 50 deletions(-)
app/test-pmd/cmdline.c +10 -2
index 14110eb..287d7a0 100644
@@ -71,8 +71,6 @@
 #include "cmdline_tm.h"
 #include "bpf_cmd.h"
 
-static struct cmdline *testpmd_cl;
-
 static void cmd_reconfig_device_queue(portid_t id, uint8_t dev, uint8_t queue);
 
 /* *** Help command with introduction. *** */
@@ -5351,6 +5349,12 @@ cmd_set_flush_rx_parsed(void *parsed_result,
 		__rte_unused void *data)
 {
 	struct cmd_set_flush_rx *res = parsed_result;
+
+	if (num_procs > 1 && (strcmp(res->mode, "on") == 0)) {
+		printf("multi-process doesn't support to flush rx queues.\n");
+		return;
+	}
+
 	no_flush_rx = (uint8_t)((strcmp(res->mode, "on") == 0) ? 0 : 1);
 }
 
@@ -17227,6 +17231,10 @@ prompt(void)
 		printf("Cannot set exit function for cmdline\n");
 
 	cmdline_interact(testpmd_cl);
+	if (unlikely(f_quit == 1)) {
+		dup2(testpmd_fd_copy, testpmd_cl->s_in);
+		close(testpmd_fd_copy);
+	}
 	if (ret != 0)
 		cmdline_stdin_exit(testpmd_cl);
 }
app/test-pmd/config.c +8 -1
index 576d5ac..9edf84a 100644
@@ -2827,6 +2827,8 @@ rss_fwd_config_setup(void)
 	queueid_t  rxq;
 	queueid_t  nb_q;
 	streamid_t  sm_id;
+	int start;
+	int end;
 
 	nb_q = nb_rxq;
 	if (nb_q > nb_txq)
@@ -2844,7 +2846,10 @@ rss_fwd_config_setup(void)
 	init_fwd_streams();
 
 	setup_fwd_config_of_each_lcore(&cur_fwd_config);
-	rxp = 0; rxq = 0;
+	start = proc_id * nb_q / num_procs;
+	end = start + nb_q / num_procs;
+	rxp = 0;
+	rxq = start;
 	for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
 		struct fwd_stream *fs;
 
@@ -2861,6 +2866,8 @@ rss_fwd_config_setup(void)
 			continue;
 		rxp = 0;
 		rxq++;
+		if (rxq >= end)
+			rxq = start;
 	}
 }
 
app/test-pmd/parameters.c +11 -0
index c8acd5d..1a5bf25 100644
@@ -45,6 +45,8 @@
 #include <rte_flow.h>
 
 #include "testpmd.h"
+#define PARAM_PROC_ID "proc-id"
+#define PARAM_NUM_PROCS "num-procs"
 
 static void
 usage(char* progname)
@@ -605,6 +607,8 @@ launch_args_parse(int argc, char** argv)
 		{ "rx-mq-mode",                 1, 0, 0 },
 		{ "record-core-cycles",         0, 0, 0 },
 		{ "record-burst-stats",         0, 0, 0 },
+		{ PARAM_NUM_PROCS,              1, 0, 0 },
+		{ PARAM_PROC_ID,                1, 0, 0 },
 		{ 0, 0, 0, 0 },
 	};
 
@@ -1366,6 +1370,13 @@ launch_args_parse(int argc, char** argv)
 				record_core_cycles = 1;
 			if (!strcmp(lgopts[opt_idx].name, "record-burst-stats"))
 				record_burst_stats = 1;
+
+			if (strncmp(lgopts[opt_idx].name,
+				    PARAM_NUM_PROCS, 8) == 0)
+				num_procs = atoi(optarg);
+			if (strncmp(lgopts[opt_idx].name,
+				    PARAM_PROC_ID, 7) == 0)
+				proc_id = atoi(optarg);
 			break;
 		case 'h':
 			usage(argv[0]);
app/test-pmd/testpmd.c +91 -47

253 lines not on this page · raw message

app/test-pmd/testpmd.h +7 -0
index ce83f31..6d13417 100644
@@ -13,6 +13,7 @@
 #include <rte_gso.h>
 #include <cmdline.h>
 #include <sys/queue.h>
+#include  "cmdline.h"
 
 #define RTE_PORT_ALL            (~(portid_t)0x0)
 
@@ -24,6 +25,10 @@
 #define RTE_PORT_CLOSED         (uint16_t)2
 #define RTE_PORT_HANDLING       (uint16_t)3
 
+uint8_t f_quit;
+int testpmd_fd_copy;
+struct cmdline *testpmd_cl;
+
 /*
  * It is used to allocate the memory for hash key.
  * The hash key size is NIC dependent.
@@ -422,6 +427,8 @@ extern uint64_t noisy_lkup_mem_sz;
 extern uint64_t noisy_lkup_num_writes;
 extern uint64_t noisy_lkup_num_reads;
 extern uint64_t noisy_lkup_num_reads_writes;
+extern int proc_id;
+extern unsigned int num_procs;
 
 extern uint8_t dcb_config;
 extern uint8_t dcb_test;
doc/guides/testpmd_app_ug/run_app.rst +69 -0

72 lines not on this page · raw message

Signature · 1 line
2.7.4
#12 lore raw

:::snip::::

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 571

Quoted 10 lines
doc/guides/testpmd_app_ug/run_app.rst +10 -0
+*   ``--proc-type``
+       Specify a given process instance as the primary or secondary DPDK instance. ``auto`` set here is ok.
+
+*   ``-l CORELIST``
+       List of cores to run on. the corelist should be different in primary process and secondary process.
+
+*   ``--rxq=N``
+
+    Set the number of RX queues per port to N, where 1 <= N <= 65535.
+    The default value is 1. N is the sum of queues used by primary and secondary process.

If RxQ is 1, how will it be shared between the primary and secondary processes? Should the number of rings be a multiple of the number of processes?

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 576

Quoted 5 lines
doc/guides/testpmd_app_ug/run_app.rst +5 -0
+
+*   ``--txq=N``
+
+    Set the number of TX queues per port to N, where 1 <= N <= 65535.
+    The default value is 1. N is the sum of queues used by primary and secondary process.

Same as above. How will a single ring be used across primary and secondary processes?

Quoted 34 lines
doc/guides/testpmd_app_ug/run_app.rst +31 -1
+
+*   ``--num-procs=N``
+
+       The number of processes which will be used.
+
+*   ``--proc-id=id``
+
+       The id of the current process (id < num-procs). id should be different in primary process and secondary process.
+
+Calculation rule for queue:
+All queues are allocated to different processes based on proc_num and proc_id.
+Calculation rule for the Testpmd to allocate queues to each process:
+start(queue start id) = proc_id * nb_q / num_procs;
+end(queue end id) = start + nb_q / num_procs;
+
+For example, if supports 4 txq and rxq
+the 0~1 for primary process
+the 2~3 for secondary process
+
+Most dev ops is supported in primary and secondary process. While secondary process is not permitted
+to allocate or release shared memory, so some ops are not supported as follows:
+``dev_start``
+``dev_stop``
+``rx_queue_setup``
+``tx_queue_setup``
+``rx_queue_release``
+``tx_queue_release``
+
+RTE_FLOW supported, it applies only on its own process on SW side, but all on HW size.
+stats supported, stats will not change when one quit and start, As they share the same buffer to store the stats.
+RSS supported, Primary process and secondary process has separate queues to use, RSS will work in their own queues whether primary and secondary process.
--

2.7.4

#13 lore raw
Quoted 13 lines · Ajit Khaparde

:::snip::::

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 571

Quoted 10 lines
doc/guides/testpmd_app_ug/run_app.rst +10 -0
+*   ``--proc-type``
+       Specify a given process instance as the primary or secondary DPDK instance. ``auto`` set here is ok.
+
+*   ``-l CORELIST``
+       List of cores to run on. the corelist should be different in primary process and secondary process.
+
+*   ``--rxq=N``
+
+    Set the number of RX queues per port to N, where 1 <= N <= 65535.
+    The default value is 1. N is the sum of queues used by primary and secondary process.

If RxQ is 1, how will it be shared between the primary and secondary processes?

Yes, your are right.the range of RxQ(or TxQ) is wrong.
As primary instance and secondary instance should have separate queues, and each should occupy at least one queue. So the range of RxQ(or TxQ) shoud be no less than two.

Quoted 1 line

Should the number of rings be a multiple of the number of process The number of rings had better be a multiple of the number of processes.

If not, redundant queues will exist after queues are allocated to
processes. After RSS is enabled, packet loss occurs when traffic is sent to all processes at the same time. Some traffic enters redundant queues and cannot be forwarded.
I will fix it in v4.Thanks.

Quoted 7 lines

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 576

Quoted 5 lines
doc/guides/testpmd_app_ug/run_app.rst +5 -0
+
+*   ``--txq=N``
+
+    Set the number of TX queues per port to N, where 1 <= N <= 65535.
+    The default value is 1. N is the sum of queues used by primary and secondary process.

Same as above. How will a single ring be used across primary and secondary processes?

Just same as above, I will fix it in v4.
Thanks.

Quoted 35 lines
Quoted 35 lines
doc/guides/testpmd_app_ug/run_app.rst +31 -1
+
+*   ``--num-procs=N``
+
+       The number of processes which will be used.
+
+*   ``--proc-id=id``
+
+       The id of the current process (id < num-procs). id should be different in primary process and secondary process.
+
+Calculation rule for queue:
+All queues are allocated to different processes based on proc_num and proc_id.
+Calculation rule for the Testpmd to allocate queues to each process:
+start(queue start id) = proc_id * nb_q / num_procs;
+end(queue end id) = start + nb_q / num_procs;
+
+For example, if supports 4 txq and rxq
+the 0~1 for primary process
+the 2~3 for secondary process
+
+Most dev ops is supported in primary and secondary process. While secondary process is not permitted
+to allocate or release shared memory, so some ops are not supported as follows:
+``dev_start``
+``dev_stop``
+``rx_queue_setup``
+``tx_queue_setup``
+``rx_queue_release``
+``tx_queue_release``
+
+RTE_FLOW supported, it applies only on its own process on SW side, but all on HW size.
+stats supported, stats will not change when one quit and start, As they share the same buffer to store the stats.
+RSS supported, Primary process and secondary process has separate queues to use, RSS will work in their own queues whether primary and secondary process.
--

2.7.4

#14 lore raw

PATCH v14 1/1 app/testpmd: support multi-process

In reply to Min Hu (Connor)

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

---
v14:
* Fixed comments by Andrew Rybchenko.

v13:
* Modified the doc syntax.

v12:
* Updated doc info.

v11:
* Fixed some minor syntax.

v10:
* Hid process type checks behind new functions.
* Added comments.

v9:
* Updated release notes and rst doc.
* Deleted deprecated codes.
* move macro and variable.

v8:
* Added warning info about queue numbers and process numbers.

v7:
* Fixed compiling error for unexpected unindent.

v6:
* Add rte flow description for multiple process.

v5:
* Fixed run_app.rst for multiple process description.
* Fix compiling error.

v4:
* Fixed minimum vlaue of Rxq or Txq in doc.

v3:
* Fixed compiling error using gcc10.0.

v2:
* Added document for this patch.

---
 app/test-pmd/cmdline.c                 |   6 ++
 app/test-pmd/config.c                  |  20 +++++-
 app/test-pmd/parameters.c              |   9 +++
 app/test-pmd/testpmd.c                 | 121 ++++++++++++++++++++++++++-------
 app/test-pmd/testpmd.h                 |   9 +++
 doc/guides/rel_notes/release_21_08.rst |   2 +-
 doc/guides/testpmd_app_ug/run_app.rst  |  70 +++++++++++++++++++
 7 files changed, 212 insertions(+), 25 deletions(-)
app/test-pmd/cmdline.c +6 -0
index 0268b18..a215b12 100644
@@ -5463,6 +5463,12 @@ cmd_set_flush_rx_parsed(void *parsed_result,
 		__rte_unused void *data)
 {
 	struct cmd_set_flush_rx *res = parsed_result;
+
+	if (num_procs > 1 && (strcmp(res->mode, "on") == 0)) {
+		printf("multi-process doesn't support to flush Rx queues.\n");
+		return;
+	}
+
 	no_flush_rx = (uint8_t)((strcmp(res->mode, "on") == 0) ? 0 : 1);
 }
 
app/test-pmd/config.c +19 -1
index 43c79b5..a0c24bb 100644
@@ -2976,6 +2976,8 @@ rss_fwd_config_setup(void)
 	queueid_t  rxq;
 	queueid_t  nb_q;
 	streamid_t  sm_id;
+	int start;
+	int end;
 
 	nb_q = nb_rxq;
 	if (nb_q > nb_txq)
@@ -2993,7 +2995,21 @@ rss_fwd_config_setup(void)
 	init_fwd_streams();
 
 	setup_fwd_config_of_each_lcore(&cur_fwd_config);
-	rxp = 0; rxq = 0;
+
+	if (proc_id > 0 && nb_q % num_procs != 0)
+		printf("Warning! queue numbers should be multiple of processes, or packet loss will happen.\n");
+
+	/**
+	 * In multi-process, All queues are allocated to different
+	 * processes based on num_procs and proc_id. For example:
+	 * if supports 4 queues(nb_q), 2 processes(num_procs),
+	 * the 0~1 queue for primary process.
+	 * the 2~3 queue for secondary process.
+	 */
+	start = proc_id * nb_q / num_procs;
+	end = start + nb_q / num_procs;
+	rxp = 0;
+	rxq = start;
 	for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
 		struct fwd_stream *fs;
 
@@ -3010,6 +3026,8 @@ rss_fwd_config_setup(void)
 			continue;
 		rxp = 0;
 		rxq++;
+		if (rxq >= end)
+			rxq = start;
 	}
 }
 
app/test-pmd/parameters.c +9 -0
index f3954c1..0f09841 100644
@@ -508,6 +508,9 @@ parse_link_speed(int n)
 void
 launch_args_parse(int argc, char** argv)
 {
+#define PARAM_PROC_ID "proc-id"
+#define PARAM_NUM_PROCS "num-procs"
+
 	int n, opt;
 	char **argvopt;
 	int opt_idx;
@@ -625,6 +628,8 @@ launch_args_parse(int argc, char** argv)
 		{ "rx-mq-mode",                 1, 0, 0 },
 		{ "record-core-cycles",         0, 0, 0 },
 		{ "record-burst-stats",         0, 0, 0 },
+		{ PARAM_NUM_PROCS,              1, 0, 0 },
+		{ PARAM_PROC_ID,                1, 0, 0 },
 		{ 0, 0, 0, 0 },
 	};
 
@@ -1391,6 +1396,10 @@ launch_args_parse(int argc, char** argv)
 				record_core_cycles = 1;
 			if (!strcmp(lgopts[opt_idx].name, "record-burst-stats"))
 				record_burst_stats = 1;
+			if (!strcmp(lgopts[opt_idx].name, PARAM_NUM_PROCS))
+				num_procs = atoi(optarg);
+			if (!strcmp(lgopts[opt_idx].name, PARAM_PROC_ID))
+				proc_id = atoi(optarg);
 			break;
 		case 'h':
 			usage(argv[0]);
app/test-pmd/testpmd.c +98 -23

206 lines not on this page · raw message

app/test-pmd/testpmd.h +9 -0
index 283b5e3..76e10b5 100644
@@ -631,6 +631,15 @@ extern enum rte_eth_rx_mq_mode rx_mq_mode;
 
 extern struct rte_flow_action_conntrack conntrack_context;
 
+extern int proc_id;
+extern unsigned int num_procs;
+
+static inline bool
+is_proc_primary(void)
+{
+	return rte_eal_process_type() == RTE_PROC_PRIMARY;
+}
+
 static inline unsigned int
 lcore_num(void)
 {
doc/guides/rel_notes/release_21_08.rst +1 -1
index a6ecfdf..4b1ea4c 100644
@@ -54,7 +54,7 @@ New Features
      This section is a comment. Do not overwrite or remove it.
      Also, make sure to start the actual text at the margin.
      =======================================================
-
+   * Added support multi-process for testpmd.
 
 Removed Items
 -------------
doc/guides/testpmd_app_ug/run_app.rst +70 -0

73 lines not on this page · raw message

Signature · 1 line
2.7.4
#15 lore raw

PATCH v15 1/1 app/testpmd: support multi-process

In reply to Min Hu (Connor)

From: "Min Hu (Connor)" <[email protected]>

For example the following commands run two testpmd processes:

* the primary process:

./dpdk-testpmd --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

* the secondary process:

./dpdk-testpmd --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

---
v15:
* Fixed release notes.
* Cleanup documentation.

v14:
* Fixed comments by Andrew Rybchenko.

v13:
* Modified the doc syntax.

v12:
* Updated doc info.

v11:
* Fixed some minor syntax.

v10:
* Hid process type checks behind new functions.
* Added comments.

v9:
* Updated release notes and rst doc.
* Deleted deprecated codes.
* move macro and variable.

v8:
* Added warning info about queue numbers and process numbers.

v7:
* Fixed compiling error for unexpected unindent.

v6:
* Add rte flow description for multiple process.

v5:
* Fixed run_app.rst for multiple process description.
* Fix compiling error.

v4:
* Fixed minimum vlaue of Rxq or Txq in doc.

v3:
* Fixed compiling error using gcc10.0.

v2:
* Added document for this patch.

---
 app/test-pmd/cmdline.c                 |   6 ++
 app/test-pmd/config.c                  |  20 +++-
 app/test-pmd/parameters.c              |   9 ++
 app/test-pmd/testpmd.c                 | 121 ++++++++++++++++++++-----
 app/test-pmd/testpmd.h                 |   9 ++
 doc/guides/rel_notes/release_21_08.rst |   5 +
 doc/guides/testpmd_app_ug/run_app.rst  |  82 +++++++++++++++++
 7 files changed, 228 insertions(+), 24 deletions(-)
app/test-pmd/cmdline.c +6 -0
index 8468018cf..5b9880dde 100644
@@ -5459,6 +5459,12 @@ cmd_set_flush_rx_parsed(void *parsed_result,
 		__rte_unused void *data)
 {
 	struct cmd_set_flush_rx *res = parsed_result;
+
+	if (num_procs > 1 && (strcmp(res->mode, "on") == 0)) {
+		printf("multi-process doesn't support to flush Rx queues.\n");
+		return;
+	}
+
 	no_flush_rx = (uint8_t)((strcmp(res->mode, "on") == 0) ? 0 : 1);
 }
 
app/test-pmd/config.c +19 -1
index 04ae0feb5..9e4d86b4a 100644
@@ -2976,6 +2976,8 @@ rss_fwd_config_setup(void)
 	queueid_t  rxq;
 	queueid_t  nb_q;
 	streamid_t  sm_id;
+	int start;
+	int end;
 
 	nb_q = nb_rxq;
 	if (nb_q > nb_txq)
@@ -2993,7 +2995,21 @@ rss_fwd_config_setup(void)
 	init_fwd_streams();
 
 	setup_fwd_config_of_each_lcore(&cur_fwd_config);
-	rxp = 0; rxq = 0;
+
+	if (proc_id > 0 && nb_q % num_procs != 0)
+		printf("Warning! queue numbers should be multiple of processes, or packet loss will happen.\n");
+
+	/**
+	 * In multi-process, All queues are allocated to different
+	 * processes based on num_procs and proc_id. For example:
+	 * if supports 4 queues(nb_q), 2 processes(num_procs),
+	 * the 0~1 queue for primary process.
+	 * the 2~3 queue for secondary process.
+	 */
+	start = proc_id * nb_q / num_procs;
+	end = start + nb_q / num_procs;
+	rxp = 0;
+	rxq = start;
 	for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
 		struct fwd_stream *fs;
 
@@ -3010,6 +3026,8 @@ rss_fwd_config_setup(void)
 			continue;
 		rxp = 0;
 		rxq++;
+		if (rxq >= end)
+			rxq = start;
 	}
 }
 
app/test-pmd/parameters.c +9 -0
index 5e69d2aa8..925fca562 100644
@@ -509,6 +509,9 @@ parse_link_speed(int n)
 void
 launch_args_parse(int argc, char** argv)
 {
+#define PARAM_PROC_ID "proc-id"
+#define PARAM_NUM_PROCS "num-procs"
+
 	int n, opt;
 	char **argvopt;
 	int opt_idx;
@@ -628,6 +631,8 @@ launch_args_parse(int argc, char** argv)
 		{ "rx-mq-mode",                 1, 0, 0 },
 		{ "record-core-cycles",         0, 0, 0 },
 		{ "record-burst-stats",         0, 0, 0 },
+		{ PARAM_NUM_PROCS,              1, 0, 0 },
+		{ PARAM_PROC_ID,                1, 0, 0 },
 		{ 0, 0, 0, 0 },
 	};
 
@@ -1394,6 +1399,10 @@ launch_args_parse(int argc, char** argv)
 				record_core_cycles = 1;
 			if (!strcmp(lgopts[opt_idx].name, "record-burst-stats"))
 				record_burst_stats = 1;
+			if (!strcmp(lgopts[opt_idx].name, PARAM_NUM_PROCS))
+				num_procs = atoi(optarg);
+			if (!strcmp(lgopts[opt_idx].name, PARAM_PROC_ID))
+				proc_id = atoi(optarg);
 			break;
 		case 'h':
 			usage(argv[0]);
app/test-pmd/testpmd.c +98 -23

206 lines not on this page · raw message

app/test-pmd/testpmd.h +9 -0
index d61a055bd..11b9451b0 100644
@@ -632,6 +632,15 @@ extern enum rte_eth_rx_mq_mode rx_mq_mode;
 
 extern struct rte_flow_action_conntrack conntrack_context;
 
+extern int proc_id;
+extern unsigned int num_procs;
+
+static inline bool
+is_proc_primary(void)
+{
+	return rte_eal_process_type() == RTE_PROC_PRIMARY;
+}
+
 static inline unsigned int
 lcore_num(void)
 {
doc/guides/rel_notes/release_21_08.rst +5 -0
index a6ecfdf3c..8ba0bfb83 100644
@@ -55,6 +55,11 @@ New Features
      Also, make sure to start the actual text at the margin.
      =======================================================
 
+   * **Added multi-process support to testpmd.**
+
+     Added command-line options to specify total number of processes and
+     current process ID. Each process owns subset of Rx and Tx queues.
+
 
 Removed Items
 -------------
doc/guides/testpmd_app_ug/run_app.rst +82 -0

85 lines not on this page · raw message

Signature · 1 line
2.30.2
#16 lore raw
Quoted 20 lines · Andrew Rybchenko

From: "Min Hu (Connor)" <[email protected]>

For example the following commands run two testpmd processes:

* the primary process:

./dpdk-testpmd --proc-type=auto -l 0-1 -- -i \
   --rxq=4 --txq=4 --num-procs=2 --proc-id=0

* the secondary process:

./dpdk-testpmd --proc-type=auto -l 2-3 -- -i \
   --rxq=4 --txq=4 --num-procs=2 --proc-id=1

[snip]

Quoted 54 lines
app/test-pmd/testpmd.c +46 -0
index 1cdd3cdd1..a5da0c272 100644
@@ -520,6 +520,62 @@ enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS;
  */
 uint32_t eth_link_speed;
 
+/*
+ * ID of the current process in multi-process, used to
+ * configure the queues to be polled.
+ */
+int proc_id;
+
+/*
+ * Number of processes in multi-process, used to
+ * configure the queues to be polled.
+ */
+unsigned int num_procs = 1;
+
+static int
+eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
+		      const struct rte_eth_conf *dev_conf)
+{
+	if (is_proc_primary())
+		return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q,
+					dev_conf);
+	return 0;
+}
+
+static int
+eth_dev_start_mp(uint16_t port_id)
+{
+	if (is_proc_primary())
+		return rte_eth_dev_start(port_id);
+
+	return 0;
+}
+
+static int
+eth_dev_stop_mp(uint16_t port_id)
+{
+	if (is_proc_primary())
+		return rte_eth_dev_stop(port_id);
+
+	return 0;
+}
+
+static void
+mempool_free_mp(struct rte_mempool *mp)
+{
+	if (is_proc_primary())
+		rte_mempool_free(mp);
+}

app/test-pmd/testpmd.c enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS; · line 578

Quoted 10 lines
app/test-pmd/testpmd.c +10 -0
+
+static int
+eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu)
+{
+	if (is_proc_primary())
+		return rte_eth_dev_set_mtu(port_id, mtu);
+
+	return 0;
+}
+

I think above functions should be removed and corresponding checks should be done in caller directly since above functions are used in single place only and just hide what actually happens in the case of secondary process. It is very misleading.

[snip]

Quoted 12 lines
app/test-pmd/testpmd.c +4 -3
@@ -2495,21 +2565,24 @@ start_port(portid_t pid)
 				return -1;
 			}
 			/* configure port */
-			diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
-						     nb_txq + nb_hairpinq,
-						     &(port->dev_conf));
+			diag = eth_dev_configure_mp(pi,
+					     nb_rxq + nb_hairpinq,
+					     nb_txq + nb_hairpinq,
+					     &(port->dev_conf));
 			if (diag != 0) {

app/test-pmd/testpmd.c start_port(portid_t pid) · line 2578

Quoted 10 lines
app/test-pmd/testpmd.c +6 -4
-				if (rte_atomic16_cmpset(&(port->port_status),
-				RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
-					printf("Port %d can not be set back "
-							"to stopped\n", pi);
+				if (rte_atomic16_cmpset(
+						&(port->port_status),
+						RTE_PORT_HANDLING,
+						RTE_PORT_STOPPED) == 0)
+					printf("Port %d cannot be set back to stopped\n",
+						pi);

Unrelated changes in the patch should be avoided since it just makes the review harder.

Quoted 12 lines
app/test-pmd/testpmd.c +1 -1
 				printf("Fail to configure port %d\n", pi);
 				/* try to reconfigure port next time */
 				port->need_reconfig = 1;
 				return -1;
 			}
 		}
-		if (port->need_reconfig_queues > 0) {
+		if (port->need_reconfig_queues > 0 && is_proc_primary()) {
 			port->need_reconfig_queues = 0;
 			/* setup tx queues */
 			for (qi = 0; qi < nb_txq; qi++) {

app/test-pmd/testpmd.c start_port(portid_t pid) · line 2609

Quoted 7 lines
app/test-pmd/testpmd.c +2 -2
@@ -2532,8 +2605,8 @@ start_port(portid_t pid)
 				if (rte_atomic16_cmpset(&(port->port_status),
 							RTE_PORT_HANDLING,
 							RTE_PORT_STOPPED) == 0)
-					printf("Port %d can not be set back "
-							"to stopped\n", pi);
+					printf("Port %d cannot be set back to stopped\n",
+						pi);

Unrelated changes in the patch should be avoided.

Quoted 11 lines
app/test-pmd/testpmd.c +1 -1
 				printf("Fail to configure port %d tx queues\n",
 				       pi);
 				/* try to reconfigure queues next time */
@@ -2610,16 +2683,16 @@ start_port(portid_t pid)
 		cnt_pi++;
 
 		/* start port */
-		diag = rte_eth_dev_start(pi);
+		diag = eth_dev_start_mp(pi);
 		if (diag < 0) {
 			printf("Fail to start port %d: %s\n", pi,

app/test-pmd/testpmd.c start_port(portid_t pid) · line 2695

Quoted 10 lines
app/test-pmd/testpmd.c +3 -3
 			       rte_strerror(-diag));
 
 			/* Fail to setup rx queue, return */
 			if (rte_atomic16_cmpset(&(port->port_status),
-				RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
-				printf("Port %d can not be set back to "
-							"stopped\n", pi);
+			RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
+				printf("Port %d cannot be set back to stopped\n",
+				       pi);

Unrelated changes in the patch should be avoided.

[snip]

Quoted 70 lines
doc/guides/testpmd_app_ug/run_app.rst +62 -0
index eb4831835..348e5fcac 100644
@@ -545,3 +545,85 @@ The command line options are:
 	bit 0 - two hairpin ports loop
 
     The default value is 0. Hairpin will use single port mode and implicit Tx flow mode.
+
+
+Testpmd Multi-Process Command-line Options
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The following are the command-line options for testpmd multi-process support:
+
+*   primary process:
+
+.. code-block:: console
+
+       sudo ./dpdk-testpmd --proc-type=auto -l 0-1 -- -i --rxq=4 --txq=4 \
+            --num-procs=2 --proc-id=0
+
+*   secondary process:
+
+.. code-block:: console
+
+       sudo ./dpdk-testpmd --proc-type=auto -l 2-3 -- -i --rxq=4 --txq=4 \
+            --num-procs=2 --proc-id=1
+
+The command line options are:
+
+*   ``--num-procs=N``
+
+    The number of processes which will be used.
+
+*   ``--proc-id=ID``
+
+    The ID of the current process (ID < num-procs). ID should be different in
+    primary process and secondary process, which starts from '0'.
+
+Calculation rule for queue:
+All queues are allocated to different processes based on ``proc_num`` and
+``proc_id``.
+Calculation rule for the testpmd to allocate queues to each process:
+
+* start(queue start id) = proc_id * nb_q / num_procs
+
+* end(queue end id) = start + nb_q / num_procs
+
+For example, if testpmd is configured to have 4 Tx and Rx queues,
+queues 0 and 1 will be used by the primary process and
+queues 2 and 3 will be used by the secondary process.
+
+The number of queues should be a multiple of the number of processes. If not,
+redundant queues will exist after queues are allocated to processes. If RSS
+is enabled, packet loss occurs when traffic is sent to all processes at the same
+time. Some traffic goes to redundant queues and cannot be forwarded.
+
+All the dev ops is supported in primary process. While secondary process is
+not permitted to allocate or release shared memory, so some ops are not supported
+as follows:
+
+- ``dev_configure``
+- ``dev_start``
+- ``dev_stop``
+- ``rx_queue_setup``
+- ``tx_queue_setup``
+- ``rx_queue_release``
+- ``tx_queue_release``
+

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 619

Quoted 10 lines
doc/guides/testpmd_app_ug/run_app.rst +10 -0
+So, any command from testpmd which calls those APIs will not be supported in
+secondary process, like:
+
+.. code-block:: console
+
+    port config all rxq|txq|rxd|txd <value>
+    port config <port_id> rx_offload xxx on/off
+    port config <port_id> tx_offload xxx on/off
+
+etc.

I did the formatting cleanup, but I still think that testpmd guide should not dive into such level of details. It should rather highlight multi-process behaviour specifics.

Shouldn't testpmd store state in shared memory to avoid problems when primary is stopped while secondary is running etc.

Some testpmd features rely on reconfigure (i.e. simply change configuration and set flag that reconfigure is required), but configure does nothing and will simply ignore new settings. So, it could look very-very confusing from user point of view.

I'm not sure that it is acceptable to apply the patch in such state and open huge number of bugs in testpmd behaviour when multi-process is used.

I'd even consider to exclude unsupported commands from help etc. However, such level of care about user could be excessive for test tool.

IMHO, it should be no requirement to repeat the primary process command-line configuration in the second process command line (see --rxq=4 --txq=4 above). The information should be obtained from shared state. In theory primary process could even change some settings in interactive mode. I think testpmd should guarantee consistent behaviour even in such conditions. I.e. do not allow to stop ports used by forwarding running in secondary processes. Run-time queues setup and deferred start should be very carefully handled as well.

Quoted 11 lines
doc/guides/testpmd_app_ug/run_app.rst +10 -0
+
+Stats is supported, stats will not change when one quits and starts, as they
+share the same buffer to store the stats. Flow rules are maintained in process
+level: primary and secondary has its own flow list (but one flow list in HW).
+The two can see all the queues, so setting the flow rules for the other is OK.
+But in the testpmd primary process receiving or transmitting packets from the
+queue allocated for secondary process is not permitted, and same for secondary
+process.
+
+Flow API and RSS are supported.
#17 lore raw

Hi, Andrew ,

Quoted 86 lines · Andrew Rybchenko
Quoted 20 lines · Andrew Rybchenko

From: "Min Hu (Connor)" <[email protected]>

For example the following commands run two testpmd processes:

* the primary process:

./dpdk-testpmd --proc-type=auto -l 0-1 -- -i \
    --rxq=4 --txq=4 --num-procs=2 --proc-id=0

* the secondary process:

./dpdk-testpmd --proc-type=auto -l 2-3 -- -i \
    --rxq=4 --txq=4 --num-procs=2 --proc-id=1

[snip]

Quoted 62 lines
app/test-pmd/testpmd.c +54 -0
index 1cdd3cdd1..a5da0c272 100644
@@ -520,6 +520,62 @@ enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS;
   */
  uint32_t eth_link_speed;
 
+/*
+ * ID of the current process in multi-process, used to
+ * configure the queues to be polled.
+ */
+int proc_id;
+
+/*
+ * Number of processes in multi-process, used to
+ * configure the queues to be polled.
+ */
+unsigned int num_procs = 1;
+
+static int
+eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
+		      const struct rte_eth_conf *dev_conf)
+{
+	if (is_proc_primary())
+		return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q,
+					dev_conf);
+	return 0;
+}
+
+static int
+eth_dev_start_mp(uint16_t port_id)
+{
+	if (is_proc_primary())
+		return rte_eth_dev_start(port_id);
+
+	return 0;
+}
+
+static int
+eth_dev_stop_mp(uint16_t port_id)
+{
+	if (is_proc_primary())
+		return rte_eth_dev_stop(port_id);
+
+	return 0;
+}
+
+static void
+mempool_free_mp(struct rte_mempool *mp)
+{
+	if (is_proc_primary())
+		rte_mempool_free(mp);
+}
+
+static int
+eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu)
+{
+	if (is_proc_primary())
+		return rte_eth_dev_set_mtu(port_id, mtu);
+
+	return 0;
Quoted 9 lines

app/test-pmd/testpmd.c enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS; · line 578

Quoted 2 lines
app/test-pmd/testpmd.c +2 -0
+}
+

I think above functions should be removed and corresponding checks should be done in caller directly since above functions are used in single place only and just hide what actually happens in the case of secondary process. It is very misleading.

This was done as Ferruh suggested in V9, and this could reduce the complexity for testpmd when added by the multi-process support.

Quoted 41 lines

[snip]

Quoted 12 lines
app/test-pmd/testpmd.c +4 -3
@@ -2495,21 +2565,24 @@ start_port(portid_t pid)
  				return -1;
  			}
  			/* configure port */
-			diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
-						     nb_txq + nb_hairpinq,
-						     &(port->dev_conf));
+			diag = eth_dev_configure_mp(pi,
+					     nb_rxq + nb_hairpinq,
+					     nb_txq + nb_hairpinq,
+					     &(port->dev_conf));
  			if (diag != 0) {

app/test-pmd/testpmd.c start_port(portid_t pid) · line 2578

Quoted 10 lines
app/test-pmd/testpmd.c +6 -4
-				if (rte_atomic16_cmpset(&(port->port_status),
-				RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
-					printf("Port %d can not be set back "
-							"to stopped\n", pi);
+				if (rte_atomic16_cmpset(
+						&(port->port_status),
+						RTE_PORT_HANDLING,
+						RTE_PORT_STOPPED) == 0)
+					printf("Port %d cannot be set back to stopped\n",
+						pi);

Unrelated changes in the patch should be avoided since it just makes the review harder.This will be fixed in v16.

Quoted 13 lines
app/test-pmd/testpmd.c +1 -1
  				printf("Fail to configure port %d\n", pi);
  				/* try to reconfigure port next time */
  				port->need_reconfig = 1;
  				return -1;
  			}
  		}
-		if (port->need_reconfig_queues > 0) {
+		if (port->need_reconfig_queues > 0 && is_proc_primary()) {
  			port->need_reconfig_queues = 0;
  			/* setup tx queues */
  			for (qi = 0; qi < nb_txq; qi++) {
@@ -2532,8 +2605,8 @@ start_port(portid_t pid)
  				if (rte_atomic16_cmpset(&(port->port_status),
Quoted 8 lines

app/test-pmd/testpmd.c start_port(portid_t pid) · line 2609

Quoted 6 lines
app/test-pmd/testpmd.c +2 -2
  							RTE_PORT_HANDLING,
  							RTE_PORT_STOPPED) == 0)
-					printf("Port %d can not be set back "
-							"to stopped\n", pi);
+					printf("Port %d cannot be set back to stopped\n",
+						pi);

Unrelated changes in the patch should be avoided.

This will be fixed in v16.

Quoted 14 lines
Quoted 14 lines
app/test-pmd/testpmd.c +1 -1
  				printf("Fail to configure port %d tx queues\n",
  				       pi);
  				/* try to reconfigure queues next time */
@@ -2610,16 +2683,16 @@ start_port(portid_t pid)
  		cnt_pi++;
 
  		/* start port */
-		diag = rte_eth_dev_start(pi);
+		diag = eth_dev_start_mp(pi);
  		if (diag < 0) {
  			printf("Fail to start port %d: %s\n", pi,
  			       rte_strerror(-diag));
 
Quoted 10 lines

app/test-pmd/testpmd.c start_port(portid_t pid) · line 2695

Quoted 8 lines
app/test-pmd/testpmd.c +3 -3
  			/* Fail to setup rx queue, return */
  			if (rte_atomic16_cmpset(&(port->port_status),
-				RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
-				printf("Port %d can not be set back to "
-							"stopped\n", pi);
+			RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
+				printf("Port %d cannot be set back to stopped\n",
+				       pi);

Unrelated changes in the patch should be avoided.

This will be fixed in v16.

Quoted 88 lines

[snip]

Quoted 70 lines
doc/guides/testpmd_app_ug/run_app.rst +62 -0
index eb4831835..348e5fcac 100644
@@ -545,3 +545,85 @@ The command line options are:
  	bit 0 - two hairpin ports loop
 
      The default value is 0. Hairpin will use single port mode and implicit Tx flow mode.
+
+
+Testpmd Multi-Process Command-line Options
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The following are the command-line options for testpmd multi-process support:
+
+*   primary process:
+
+.. code-block:: console
+
+       sudo ./dpdk-testpmd --proc-type=auto -l 0-1 -- -i --rxq=4 --txq=4 \
+            --num-procs=2 --proc-id=0
+
+*   secondary process:
+
+.. code-block:: console
+
+       sudo ./dpdk-testpmd --proc-type=auto -l 2-3 -- -i --rxq=4 --txq=4 \
+            --num-procs=2 --proc-id=1
+
+The command line options are:
+
+*   ``--num-procs=N``
+
+    The number of processes which will be used.
+
+*   ``--proc-id=ID``
+
+    The ID of the current process (ID < num-procs). ID should be different in
+    primary process and secondary process, which starts from '0'.
+
+Calculation rule for queue:
+All queues are allocated to different processes based on ``proc_num`` and
+``proc_id``.
+Calculation rule for the testpmd to allocate queues to each process:
+
+* start(queue start id) = proc_id * nb_q / num_procs
+
+* end(queue end id) = start + nb_q / num_procs
+
+For example, if testpmd is configured to have 4 Tx and Rx queues,
+queues 0 and 1 will be used by the primary process and
+queues 2 and 3 will be used by the secondary process.
+
+The number of queues should be a multiple of the number of processes. If not,
+redundant queues will exist after queues are allocated to processes. If RSS
+is enabled, packet loss occurs when traffic is sent to all processes at the same
+time. Some traffic goes to redundant queues and cannot be forwarded.
+
+All the dev ops is supported in primary process. While secondary process is
+not permitted to allocate or release shared memory, so some ops are not supported
+as follows:
+
+- ``dev_configure``
+- ``dev_start``
+- ``dev_stop``
+- ``rx_queue_setup``
+- ``tx_queue_setup``
+- ``rx_queue_release``
+- ``tx_queue_release``
+

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 619

Quoted 10 lines
doc/guides/testpmd_app_ug/run_app.rst +10 -0
+So, any command from testpmd which calls those APIs will not be supported in
+secondary process, like:
+
+.. code-block:: console
+
+    port config all rxq|txq|rxd|txd <value>
+    port config <port_id> rx_offload xxx on/off
+    port config <port_id> tx_offload xxx on/off
+
+etc.

I did the formatting cleanup, but I still think that testpmd guide should not dive into such level of details. It should rather highlight multi-process behaviour specifics.

Quoted 2 lines

Shouldn't testpmd store state in shared memory to avoid problems when primary is stopped while secondary is running

This could be taken into consideration in future.

Quoted 6 lines

Some testpmd features rely on reconfigure (i.e. simply change configuration and set flag that reconfigure is required), but configure does nothing and will simply ignore new settings. So, it could look very-very confusing from user point of view.

Quoted 7 lines

I'm not sure that it is acceptable to apply the patch in such state and open huge number of bugs in testpmd behaviour when multi-process is used.

I'd even consider to exclude unsupported commands from help etc. However, such level of care about user could be excessive for test tool.

This has been done in doc.

Quoted 6 lines

IMHO, it should be no requirement to repeat the primary process command-line configuration in the second process command line (see --rxq=4 --txq=4 above). The information should be obtained from shared state. In theory primary process could even change some settings in interactive

We think keeping the command line in consistent between primary and secondary is easy to understand for users. While shared memory for keeping in order or communicating could be performed,but this could be done in future patch.

Quoted 5 lines

mode. I think testpmd should guarantee consistent behaviour even in such conditions. I.e. do not allow to stop ports used by forwarding running in secondary processes. Run-time queues setup and deferred start should be very carefully handled as well.

``dev_stop`` is not allowed in secondary, which has described in doc.

Quoted 11 lines
Quoted 11 lines
doc/guides/testpmd_app_ug/run_app.rst +10 -0
+
+Stats is supported, stats will not change when one quits and starts, as they
+share the same buffer to store the stats. Flow rules are maintained in process
+level: primary and secondary has its own flow list (but one flow list in HW).
+The two can see all the queues, so setting the flow rules for the other is OK.
+But in the testpmd primary process receiving or transmitting packets from the
+queue allocated for secondary process is not permitted, and same for secondary
+process.
+
+Flow API and RSS are supported.

Thanks for your comment,

This patch supports basic function for multi-process support in testpmd. I think other patches in future could enhance or optimize it, thanks.

Quoted 3 lines

.

#18 lore raw
Quoted 303 lines · Min Hu (Connor)

Hi, Andrew ,

Quoted 87 lines · Andrew Rybchenko
Quoted 20 lines · Andrew Rybchenko

From: "Min Hu (Connor)" <[email protected]>

For example the following commands run two testpmd processes:

  * the primary process:

./dpdk-testpmd --proc-type=auto -l 0-1 -- -i \
    --rxq=4 --txq=4 --num-procs=2 --proc-id=0

  * the secondary process:

./dpdk-testpmd --proc-type=auto -l 2-3 -- -i \
    --rxq=4 --txq=4 --num-procs=2 --proc-id=1

[snip]

Quoted 63 lines
app/test-pmd/testpmd.c +0 -0
index 1cdd3cdd1..a5da0c272 100644
@@ -520,6 +520,62 @@ enum rte_eth_rx_mq_mode rx_mq_mode =
 ETH_MQ_RX_VMDQ_DCB_RSS;
    */
   uint32_t eth_link_speed;

  +/*
+ * ID of the current process in multi-process, used to
+ * configure the queues to be polled.
+ */

app/test-pmd/testpmd.c +10 -0
+int proc_id;
+
+/*
+ * Number of processes in multi-process, used to
+ * configure the queues to be polled.
+ */
+unsigned int num_procs = 1;
+
+static int
+eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t

nb_tx_q,
+              const struct rte_eth_conf *dev_conf)

app/test-pmd/testpmd.c +39 -0
+{
+    if (is_proc_primary())
+        return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q,
+                    dev_conf);
+    return 0;
+}
+
+static int
+eth_dev_start_mp(uint16_t port_id)
+{
+    if (is_proc_primary())
+        return rte_eth_dev_start(port_id);
+
+    return 0;
+}
+
+static int
+eth_dev_stop_mp(uint16_t port_id)
+{
+    if (is_proc_primary())
+        return rte_eth_dev_stop(port_id);
+
+    return 0;
+}
+
+static void
+mempool_free_mp(struct rte_mempool *mp)
+{
+    if (is_proc_primary())
+        rte_mempool_free(mp);
+}
+
+static int
+eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu)
+{
+    if (is_proc_primary())
+        return rte_eth_dev_set_mtu(port_id, mtu);
+
+    return 0;
Quoted 9 lines

app/test-pmd/testpmd.c

Quoted 2 lines
app/test-pmd/testpmd.c +2 -0
+}
+

I think above functions should be removed and corresponding checks should be done in caller directly since above functions are used in single place only and just hide what actually happens in the case of secondary process. It is very misleading.

This was done as Ferruh suggested in V9, and this could reduce the complexity for testpmd when added by the multi-process support.

Quoted 41 lines

[snip]

Quoted 12 lines
app/test-pmd/testpmd.c +4 -3
@@ -2495,21 +2565,24 @@ start_port(portid_t pid)
                   return -1;
               }
               /* configure port */
-            diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
-                             nb_txq + nb_hairpinq,
-                             &(port->dev_conf));
+            diag = eth_dev_configure_mp(pi,
+                         nb_rxq + nb_hairpinq,
+                         nb_txq + nb_hairpinq,
+                         &(port->dev_conf));
               if (diag != 0) {

app/test-pmd/testpmd.c start_port(portid_t pid) · line 2578

Quoted 10 lines
app/test-pmd/testpmd.c +6 -4
-                if (rte_atomic16_cmpset(&(port->port_status),
-                RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
-                    printf("Port %d can not be set back "
-                            "to stopped\n", pi);
+                if (rte_atomic16_cmpset(
+                        &(port->port_status),
+                        RTE_PORT_HANDLING,
+                        RTE_PORT_STOPPED) == 0)
+                    printf("Port %d cannot be set back to stopped\n",
+                        pi);

Unrelated changes in the patch should be avoided since it just makes the review harder.This will be fixed in v16.

Quoted 13 lines
app/test-pmd/testpmd.c +1 -1
                   printf("Fail to configure port %d\n", pi);
                   /* try to reconfigure port next time */
                   port->need_reconfig = 1;
                   return -1;
               }
           }
-        if (port->need_reconfig_queues > 0) {
+        if (port->need_reconfig_queues > 0 && is_proc_primary()) {
               port->need_reconfig_queues = 0;
               /* setup tx queues */
               for (qi = 0; qi < nb_txq; qi++) {
@@ -2532,8 +2605,8 @@ start_port(portid_t pid)
                   if (rte_atomic16_cmpset(&(port->port_status),
Quoted 8 lines

app/test-pmd/testpmd.c start_port(portid_t pid) · line 2609

Quoted 6 lines
app/test-pmd/testpmd.c +2 -2
                               RTE_PORT_HANDLING,
                               RTE_PORT_STOPPED) == 0)
-                    printf("Port %d can not be set back "
-                            "to stopped\n", pi);
+                    printf("Port %d cannot be set back to stopped\n",
+                        pi);

Unrelated changes in the patch should be avoided.

This will be fixed in v16.

Quoted 14 lines
Quoted 14 lines
app/test-pmd/testpmd.c +1 -1
                   printf("Fail to configure port %d tx queues\n",
                          pi);
                   /* try to reconfigure queues next time */
@@ -2610,16 +2683,16 @@ start_port(portid_t pid)
           cnt_pi++;
             /* start port */
-        diag = rte_eth_dev_start(pi);
+        diag = eth_dev_start_mp(pi);
           if (diag < 0) {
               printf("Fail to start port %d: %s\n", pi,
                      rte_strerror(-diag));
                 /* Fail to setup rx queue, return */
               if (rte_atomic16_cmpset(&(port->port_status),
Quoted 8 lines

app/test-pmd/testpmd.c start_port(portid_t pid) · line 2695

Quoted 6 lines
app/test-pmd/testpmd.c +3 -3
-                RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
-                printf("Port %d can not be set back to "
-                            "stopped\n", pi);
+            RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
+                printf("Port %d cannot be set back to stopped\n",
+                       pi);

Unrelated changes in the patch should be avoided.

This will be fixed in v16.

Quoted 100 lines

[snip]

Quoted 83 lines
doc/guides/testpmd_app_ug/run_app.rst +0 -0
index eb4831835..348e5fcac 100644
@@ -545,3 +545,85 @@ The command line options are:

      bit 0 - two hairpin ports loop
        The default value is 0. Hairpin will use single port mode and implicit Tx flow mode.

doc/guides/testpmd_app_ug/run_app.rst +6 -0
+
+
+Testpmd Multi-Process Command-line Options
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The following are the command-line options for testpmd multi-process

support:

doc/guides/testpmd_app_ug/run_app.rst +24 -2
+
+*   primary process:
+
+.. code-block:: console
+
+       sudo ./dpdk-testpmd --proc-type=auto -l 0-1 -- -i --rxq=4
--txq=4 \
+            --num-procs=2 --proc-id=0
+
+*   secondary process:
+
+.. code-block:: console
+
+       sudo ./dpdk-testpmd --proc-type=auto -l 2-3 -- -i --rxq=4
--txq=4 \
+            --num-procs=2 --proc-id=1
+
+The command line options are:
+
+*   ``--num-procs=N``
+
+    The number of processes which will be used.
+
+*   ``--proc-id=ID``
+
+    The ID of the current process (ID < num-procs). ID should be

different in
+    primary process and secondary process, which starts from '0'.

doc/guides/testpmd_app_ug/run_app.rst +3 -0
+
+Calculation rule for queue:
+All queues are allocated to different processes based on

``proc_num`` and
+``proc_id``.

doc/guides/testpmd_app_ug/run_app.rst +11 -0
+Calculation rule for the testpmd to allocate queues to each process:
+
+* start(queue start id) = proc_id * nb_q / num_procs
+
+* end(queue end id) = start + nb_q / num_procs
+
+For example, if testpmd is configured to have 4 Tx and Rx queues,
+queues 0 and 1 will be used by the primary process and
+queues 2 and 3 will be used by the secondary process.
+
+The number of queues should be a multiple of the number of

processes. If not,
+redundant queues will exist after queues are allocated to processes. If RSS
+is enabled, packet loss occurs when traffic is sent to all processes at the same

doc/guides/testpmd_app_ug/run_app.rst +3 -0
+time. Some traffic goes to redundant queues and cannot be forwarded.
+
+All the dev ops is supported in primary process. While secondary

process is
+not permitted to allocate or release shared memory, so some ops are not supported

doc/guides/testpmd_app_ug/run_app.rst +11 -0
+as follows:
+
+- ``dev_configure``
+- ``dev_start``
+- ``dev_stop``
+- ``rx_queue_setup``
+- ``tx_queue_setup``
+- ``rx_queue_release``
+- ``tx_queue_release``
+
+So, any command from testpmd which calls those APIs will not be

supported in

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 619

Quoted 9 lines
doc/guides/testpmd_app_ug/run_app.rst +9 -0
+secondary process, like:
+
+.. code-block:: console
+
+    port config all rxq|txq|rxd|txd <value>
+    port config <port_id> rx_offload xxx on/off
+    port config <port_id> tx_offload xxx on/off
+
+etc.

I did the formatting cleanup, but I still think that testpmd guide should not dive into such level of details. It should rather highlight multi-process behaviour specifics.

Quoted 2 lines

Shouldn't testpmd store state in shared memory to avoid problems when primary is stopped while secondary is running

This could be taken into consideration in future.

Quoted 6 lines

Some testpmd features rely on reconfigure (i.e. simply change configuration and set flag that reconfigure is required), but configure does nothing and will simply ignore new settings. So, it could look very-very confusing from user point of view.

Quoted 7 lines

I'm not sure that it is acceptable to apply the patch in such state and open huge number of bugs in testpmd behaviour when multi-process is used.

I'd even consider to exclude unsupported commands from help etc. However, such level of care about user could be excessive for test tool.

This has been done in doc.

Quoted 6 lines

IMHO, it should be no requirement to repeat the primary process command-line configuration in the second process command line (see --rxq=4 --txq=4 above). The information should be obtained from shared state. In theory primary process could even change some settings in interactive

We think keeping the command line in consistent between primary and secondary is easy to understand for users. While shared memory for keeping in order or communicating could be performed,but this could be done in future patch.

Quoted 6 lines
Quoted 5 lines

mode. I think testpmd should guarantee consistent behaviour even in such conditions. I.e. do not allow to stop ports used by forwarding running in secondary processes. Run-time queues setup and deferred start should be very carefully handled as well.

 ``dev_stop`` is not allowed in secondary, which has described in doc.

I'm talking about dev_stop in primary while secondary is running.

Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
doc/guides/testpmd_app_ug/run_app.rst +2 -0
+
+Stats is supported, stats will not change when one quits and starts,

as they
+share the same buffer to store the stats. Flow rules are maintained in process
+level: primary and secondary has its own flow list (but one flow list in HW).
+The two can see all the queues, so setting the flow rules for the other is OK.
+But in the testpmd primary process receiving or transmitting packets from the

Quoted 8 lines
Quoted 6 lines
Quoted 6 lines

+queue allocated for secondary process is not permitted, and same for secondary

doc/guides/testpmd_app_ug/run_app.rst +3 -0
+process.
+
+Flow API and RSS are supported.

Thanks for your comment,

This patch supports basic function for multi-process support in testpmd. I think other patches in future could enhance or optimize it, thanks.

IMHO, as I state above, current state is insufficient to consider is a start point to be applied.

#19 lore raw

Hi,Andrew ,

Quoted 334 lines · Andrew Rybchenko
Quoted 303 lines · Min Hu (Connor)

Hi, Andrew ,

Quoted 87 lines · Andrew Rybchenko
Quoted 20 lines · Andrew Rybchenko

From: "Min Hu (Connor)" <[email protected]>

For example the following commands run two testpmd processes:

  * the primary process:

./dpdk-testpmd --proc-type=auto -l 0-1 -- -i \
    --rxq=4 --txq=4 --num-procs=2 --proc-id=0

  * the secondary process:

./dpdk-testpmd --proc-type=auto -l 2-3 -- -i \
    --rxq=4 --txq=4 --num-procs=2 --proc-id=1

[snip]

Quoted 63 lines
app/test-pmd/testpmd.c +53 -0
index 1cdd3cdd1..a5da0c272 100644
@@ -520,6 +520,62 @@ enum rte_eth_rx_mq_mode rx_mq_mode =
 ETH_MQ_RX_VMDQ_DCB_RSS;
    */
   uint32_t eth_link_speed;
   +/*
+ * ID of the current process in multi-process, used to
+ * configure the queues to be polled.
+ */
+int proc_id;
+
+/*
+ * Number of processes in multi-process, used to
+ * configure the queues to be polled.
+ */
+unsigned int num_procs = 1;
+
+static int
+eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t
 nb_tx_q,
+              const struct rte_eth_conf *dev_conf)
+{
+    if (is_proc_primary())
+        return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q,
+                    dev_conf);
+    return 0;
+}
+
+static int
+eth_dev_start_mp(uint16_t port_id)
+{
+    if (is_proc_primary())
+        return rte_eth_dev_start(port_id);
+
+    return 0;
+}
+
+static int
+eth_dev_stop_mp(uint16_t port_id)
+{
+    if (is_proc_primary())
+        return rte_eth_dev_stop(port_id);
+
+    return 0;
+}
+
+static void
+mempool_free_mp(struct rte_mempool *mp)
+{
+    if (is_proc_primary())
+        rte_mempool_free(mp);
+}
+
+static int
+eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu)
+{
+    if (is_proc_primary())
+        return rte_eth_dev_set_mtu(port_id, mtu);
+
+    return 0;
Quoted 9 lines

app/test-pmd/testpmd.c enum rte_eth_rx_mq_mode rx_mq_mode = · line 579

Quoted 2 lines
app/test-pmd/testpmd.c +2 -0
+}
+

I think above functions should be removed and corresponding checks should be done in caller directly since above functions are used in single place only and just hide what actually happens in the case of secondary process. It is very misleading.

This was done as Ferruh suggested in V9, and this could reduce the complexity for testpmd when added by the multi-process support.

Quoted 41 lines

[snip]

Quoted 12 lines
app/test-pmd/testpmd.c +4 -3
@@ -2495,21 +2565,24 @@ start_port(portid_t pid)
                   return -1;
               }
               /* configure port */
-            diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
-                             nb_txq + nb_hairpinq,
-                             &(port->dev_conf));
+            diag = eth_dev_configure_mp(pi,
+                         nb_rxq + nb_hairpinq,
+                         nb_txq + nb_hairpinq,
+                         &(port->dev_conf));
               if (diag != 0) {

app/test-pmd/testpmd.c start_port(portid_t pid) · line 2578

Quoted 10 lines
app/test-pmd/testpmd.c +6 -4
-                if (rte_atomic16_cmpset(&(port->port_status),
-                RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
-                    printf("Port %d can not be set back "
-                            "to stopped\n", pi);
+                if (rte_atomic16_cmpset(
+                        &(port->port_status),
+                        RTE_PORT_HANDLING,
+                        RTE_PORT_STOPPED) == 0)
+                    printf("Port %d cannot be set back to stopped\n",
+                        pi);

Unrelated changes in the patch should be avoided since it just makes the review harder.This will be fixed in v16.

Quoted 13 lines
app/test-pmd/testpmd.c +1 -1
                   printf("Fail to configure port %d\n", pi);
                   /* try to reconfigure port next time */
                   port->need_reconfig = 1;
                   return -1;
               }
           }
-        if (port->need_reconfig_queues > 0) {
+        if (port->need_reconfig_queues > 0 && is_proc_primary()) {
               port->need_reconfig_queues = 0;
               /* setup tx queues */
               for (qi = 0; qi < nb_txq; qi++) {
@@ -2532,8 +2605,8 @@ start_port(portid_t pid)
                   if (rte_atomic16_cmpset(&(port->port_status),
Quoted 8 lines

app/test-pmd/testpmd.c start_port(portid_t pid) · line 2609

Quoted 6 lines
app/test-pmd/testpmd.c +2 -2
                               RTE_PORT_HANDLING,
                               RTE_PORT_STOPPED) == 0)
-                    printf("Port %d can not be set back "
-                            "to stopped\n", pi);
+                    printf("Port %d cannot be set back to stopped\n",
+                        pi);

Unrelated changes in the patch should be avoided.

This will be fixed in v16.

Quoted 14 lines
Quoted 14 lines
app/test-pmd/testpmd.c +1 -1
                   printf("Fail to configure port %d tx queues\n",
                          pi);
                   /* try to reconfigure queues next time */
@@ -2610,16 +2683,16 @@ start_port(portid_t pid)
           cnt_pi++;
             /* start port */
-        diag = rte_eth_dev_start(pi);
+        diag = eth_dev_start_mp(pi);
           if (diag < 0) {
               printf("Fail to start port %d: %s\n", pi,
                      rte_strerror(-diag));
                 /* Fail to setup rx queue, return */
               if (rte_atomic16_cmpset(&(port->port_status),
Quoted 8 lines

app/test-pmd/testpmd.c start_port(portid_t pid) · line 2695

Quoted 6 lines
app/test-pmd/testpmd.c +3 -3
-                RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
-                printf("Port %d can not be set back to "
-                            "stopped\n", pi);
+            RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
+                printf("Port %d cannot be set back to stopped\n",
+                       pi);

Unrelated changes in the patch should be avoided.

This will be fixed in v16.

Quoted 100 lines

[snip]

Quoted 83 lines
doc/guides/testpmd_app_ug/run_app.rst +0 -0
index eb4831835..348e5fcac 100644
@@ -545,3 +545,85 @@ The command line options are:
       bit 0 - two hairpin ports loop
         The default value is 0. Hairpin will use single port mode and

implicit Tx flow mode.

doc/guides/testpmd_app_ug/run_app.rst +6 -0
+
+
+Testpmd Multi-Process Command-line Options
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The following are the command-line options for testpmd multi-process

support:

doc/guides/testpmd_app_ug/run_app.rst +24 -2
+
+*   primary process:
+
+.. code-block:: console
+
+       sudo ./dpdk-testpmd --proc-type=auto -l 0-1 -- -i --rxq=4
--txq=4 \
+            --num-procs=2 --proc-id=0
+
+*   secondary process:
+
+.. code-block:: console
+
+       sudo ./dpdk-testpmd --proc-type=auto -l 2-3 -- -i --rxq=4
--txq=4 \
+            --num-procs=2 --proc-id=1
+
+The command line options are:
+
+*   ``--num-procs=N``
+
+    The number of processes which will be used.
+
+*   ``--proc-id=ID``
+
+    The ID of the current process (ID < num-procs). ID should be

different in
+    primary process and secondary process, which starts from '0'.

doc/guides/testpmd_app_ug/run_app.rst +3 -0
+
+Calculation rule for queue:
+All queues are allocated to different processes based on

``proc_num`` and
+``proc_id``.

doc/guides/testpmd_app_ug/run_app.rst +11 -0
+Calculation rule for the testpmd to allocate queues to each process:
+
+* start(queue start id) = proc_id * nb_q / num_procs
+
+* end(queue end id) = start + nb_q / num_procs
+
+For example, if testpmd is configured to have 4 Tx and Rx queues,
+queues 0 and 1 will be used by the primary process and
+queues 2 and 3 will be used by the secondary process.
+
+The number of queues should be a multiple of the number of

processes. If not,
+redundant queues will exist after queues are allocated to processes. If RSS
+is enabled, packet loss occurs when traffic is sent to all processes at the same

doc/guides/testpmd_app_ug/run_app.rst +3 -0
+time. Some traffic goes to redundant queues and cannot be forwarded.
+
+All the dev ops is supported in primary process. While secondary

process is
+not permitted to allocate or release shared memory, so some ops are not supported

doc/guides/testpmd_app_ug/run_app.rst +11 -0
+as follows:
+
+- ``dev_configure``
+- ``dev_start``
+- ``dev_stop``
+- ``rx_queue_setup``
+- ``tx_queue_setup``
+- ``rx_queue_release``
+- ``tx_queue_release``
+
+So, any command from testpmd which calls those APIs will not be

supported in

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 619

Quoted 9 lines
doc/guides/testpmd_app_ug/run_app.rst +9 -0
+secondary process, like:
+
+.. code-block:: console
+
+    port config all rxq|txq|rxd|txd <value>
+    port config <port_id> rx_offload xxx on/off
+    port config <port_id> tx_offload xxx on/off
+
+etc.

I did the formatting cleanup, but I still think that testpmd guide should not dive into such level of details. It should rather highlight multi-process behaviour specifics.

Quoted 2 lines

Shouldn't testpmd store state in shared memory to avoid problems when primary is stopped while secondary is running

This could be taken into consideration in future.

Quoted 6 lines

Some testpmd features rely on reconfigure (i.e. simply change configuration and set flag that reconfigure is required), but configure does nothing and will simply ignore new settings. So, it could look very-very confusing from user point of view.

Quoted 7 lines

I'm not sure that it is acceptable to apply the patch in such state and open huge number of bugs in testpmd behaviour when multi-process is used.

I'd even consider to exclude unsupported commands from help etc. However, such level of care about user could be excessive for test tool.

This has been done in doc.

Quoted 6 lines

IMHO, it should be no requirement to repeat the primary process command-line configuration in the second process command line (see --rxq=4 --txq=4 above). The information should be obtained from shared state. In theory primary process could even change some settings in interactive

We think keeping the command line in consistent between primary and secondary is easy to understand for users. While shared memory for keeping in order or communicating could be performed,but this could be done in future patch.

Quoted 6 lines
Quoted 5 lines

mode. I think testpmd should guarantee consistent behaviour even in such conditions. I.e. do not allow to stop ports used by forwarding running in secondary processes. Run-time queues setup and deferred start should be very carefully handled as well.

 ``dev_stop`` is not allowed in secondary, which has described in doc.

I'm talking about dev_stop in primary while secondary is running.

Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
doc/guides/testpmd_app_ug/run_app.rst +2 -0
+
+Stats is supported, stats will not change when one quits and starts,

as they
+share the same buffer to store the stats. Flow rules are maintained in process
+level: primary and secondary has its own flow list (but one flow list in HW).
+The two can see all the queues, so setting the flow rules for the other is OK.
+But in the testpmd primary process receiving or transmitting packets from the

Quoted 8 lines
Quoted 6 lines
Quoted 6 lines

+queue allocated for secondary process is not permitted, and same for secondary

doc/guides/testpmd_app_ug/run_app.rst +3 -0
+process.
+
+Flow API and RSS are supported.

Thanks for your comment,

This patch supports basic function for multi-process support in testpmd. I think other patches in future could enhance or optimize it, thanks.

Quoted 2 lines

IMHO, as I state above, current state is insufficient to consider is a start point to be applied.

OK, what is the critical(the most important) as a start point to be applied? I will take action to fix it,as I will the patch could be applied before V21.08.

Quoted 2 lines

.

#20 lore raw

PATCH v16 1/1 app/testpmd: support multi-process

In reply to Min Hu (Connor)

This patch adds multi-process support for testpmd. For example the following commands run two testpmd processes:

* the primary process:

./dpdk-testpmd --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

* the secondary process:

./dpdk-testpmd --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

---
V16:
* revert unrelated changes.
* add some restrictions in doc.

v15:
* Fixed release notes.
* Cleanup documentation.

v14:
* Fixed comments by Andrew Rybchenko.

v13:
* Modified the doc syntax.

v12:
* Updated doc info.

v11:
* Fixed some minor syntax.

v10:
* Hid process type checks behind new functions.
* Added comments.

v9:
* Updated release notes and rst doc.
* Deleted deprecated codes.
* move macro and variable.

v8:
* Added warning info about queue numbers and process numbers.

v7:
* Fixed compiling error for unexpected unindent.

v6:
* Add rte flow description for multiple process.

v5:
* Fixed run_app.rst for multiple process description.
* Fix compiling error.

v4:
* Fixed minimum vlaue of Rxq or Txq in doc.

v3:
* Fixed compiling error using gcc10.0.

v2:
* Added document for this patch.

---
 app/test-pmd/cmdline.c                 |   6 ++
 app/test-pmd/config.c                  |  20 ++++++-
 app/test-pmd/parameters.c              |   9 +++
 app/test-pmd/testpmd.c                 | 106 ++++++++++++++++++++++++++++-----
 app/test-pmd/testpmd.h                 |   9 +++
 doc/guides/rel_notes/release_21_08.rst |   3 +
 doc/guides/testpmd_app_ug/run_app.rst  |  84 ++++++++++++++++++++++++++
 7 files changed, 220 insertions(+), 17 deletions(-)
app/test-pmd/cmdline.c +6 -0
index 8468018..5b9880d 100644
@@ -5459,6 +5459,12 @@ cmd_set_flush_rx_parsed(void *parsed_result,
 		__rte_unused void *data)
 {
 	struct cmd_set_flush_rx *res = parsed_result;
+
+	if (num_procs > 1 && (strcmp(res->mode, "on") == 0)) {
+		printf("multi-process doesn't support to flush Rx queues.\n");
+		return;
+	}
+
 	no_flush_rx = (uint8_t)((strcmp(res->mode, "on") == 0) ? 0 : 1);
 }
 
app/test-pmd/config.c +19 -1
index 04ae0fe..9e4d86b 100644
@@ -2976,6 +2976,8 @@ rss_fwd_config_setup(void)
 	queueid_t  rxq;
 	queueid_t  nb_q;
 	streamid_t  sm_id;
+	int start;
+	int end;
 
 	nb_q = nb_rxq;
 	if (nb_q > nb_txq)
@@ -2993,7 +2995,21 @@ rss_fwd_config_setup(void)
 	init_fwd_streams();
 
 	setup_fwd_config_of_each_lcore(&cur_fwd_config);
-	rxp = 0; rxq = 0;
+
+	if (proc_id > 0 && nb_q % num_procs != 0)
+		printf("Warning! queue numbers should be multiple of processes, or packet loss will happen.\n");
+
+	/**
+	 * In multi-process, All queues are allocated to different
+	 * processes based on num_procs and proc_id. For example:
+	 * if supports 4 queues(nb_q), 2 processes(num_procs),
+	 * the 0~1 queue for primary process.
+	 * the 2~3 queue for secondary process.
+	 */
+	start = proc_id * nb_q / num_procs;
+	end = start + nb_q / num_procs;
+	rxp = 0;
+	rxq = start;
 	for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
 		struct fwd_stream *fs;
 
@@ -3010,6 +3026,8 @@ rss_fwd_config_setup(void)
 			continue;
 		rxp = 0;
 		rxq++;
+		if (rxq >= end)
+			rxq = start;
 	}
 }
 
app/test-pmd/parameters.c +9 -0
index 5e69d2a..925fca5 100644
@@ -509,6 +509,9 @@ parse_link_speed(int n)
 void
 launch_args_parse(int argc, char** argv)
 {
+#define PARAM_PROC_ID "proc-id"
+#define PARAM_NUM_PROCS "num-procs"
+
 	int n, opt;
 	char **argvopt;
 	int opt_idx;
@@ -628,6 +631,8 @@ launch_args_parse(int argc, char** argv)
 		{ "rx-mq-mode",                 1, 0, 0 },
 		{ "record-core-cycles",         0, 0, 0 },
 		{ "record-burst-stats",         0, 0, 0 },
+		{ PARAM_NUM_PROCS,              1, 0, 0 },
+		{ PARAM_PROC_ID,                1, 0, 0 },
 		{ 0, 0, 0, 0 },
 	};
 
@@ -1394,6 +1399,10 @@ launch_args_parse(int argc, char** argv)
 				record_core_cycles = 1;
 			if (!strcmp(lgopts[opt_idx].name, "record-burst-stats"))
 				record_burst_stats = 1;
+			if (!strcmp(lgopts[opt_idx].name, PARAM_NUM_PROCS))
+				num_procs = atoi(optarg);
+			if (!strcmp(lgopts[opt_idx].name, PARAM_PROC_ID))
+				proc_id = atoi(optarg);
 			break;
 		case 'h':
 			usage(argv[0]);
app/test-pmd/testpmd.c +90 -16

179 lines not on this page · raw message

app/test-pmd/testpmd.h +9 -0
index d61a055..11b9451 100644
@@ -632,6 +632,15 @@ extern enum rte_eth_rx_mq_mode rx_mq_mode;
 
 extern struct rte_flow_action_conntrack conntrack_context;
 
+extern int proc_id;
+extern unsigned int num_procs;
+
+static inline bool
+is_proc_primary(void)
+{
+	return rte_eal_process_type() == RTE_PROC_PRIMARY;
+}
+
 static inline unsigned int
 lcore_num(void)
 {
doc/guides/rel_notes/release_21_08.rst +3 -0
index 0804b00..669cb75 100644
@@ -90,6 +90,9 @@ New Features
   usecases. Configuration happens via standard rawdev enq/deq operations. See
   the :doc:`../rawdevs/cnxk_bphy` rawdev guide for more details on this driver.
 
+* **Added support multi-process for testpmd.**
+  Added command-line options to specify total number of processes and
+  current process ID. Each process owns subset of Rx and Tx queues.
 
 Removed Items
 -------------
doc/guides/testpmd_app_ug/run_app.rst +84 -0

87 lines not on this page · raw message

Signature · 1 line
2.7.4
#21 lore raw

10/07/2021 05:50, Min Hu (Connor):

Quoted 20 lines

This patch adds multi-process support for testpmd. For example the following commands run two testpmd processes:

* the primary process:

./dpdk-testpmd --proc-type=auto -l 0-1 -- -i \
   --rxq=4 --txq=4 --num-procs=2 --proc-id=0

* the secondary process:

./dpdk-testpmd --proc-type=auto -l 2-3 -- -i \
   --rxq=4 --txq=4 --num-procs=2 --proc-id=1
Quoted 4 lines

---
V16:
* revert unrelated changes.
* add some restrictions in doc.

I didn't see clear agreement to integrate this feature in DPDK 21.08.

BTW, the testpmd maintainer was not Cc'ed.

#22 lore raw
Quoted 26 lines · Thomas Monjalon

10/07/2021 05:50, Min Hu (Connor):

Quoted 20 lines

This patch adds multi-process support for testpmd. For example the following commands run two testpmd processes:

* the primary process:

./dpdk-testpmd --proc-type=auto -l 0-1 -- -i \
    --rxq=4 --txq=4 --num-procs=2 --proc-id=0

* the secondary process:

./dpdk-testpmd --proc-type=auto -l 2-3 -- -i \
    --rxq=4 --txq=4 --num-procs=2 --proc-id=1
Quoted 4 lines

---
V16:
* revert unrelated changes.
* add some restrictions in doc.

Quoted 4 lines

I didn't see clear agreement to integrate this feature in DPDK 21.08.

BTW, the testpmd maintainer was not Cc'ed.

Hi, Thomas,
This patch has two "acked" and one "reviewed-by".
And Xiaoyun Li is the maintainer, I has already Cc'ed to him/her, please check it out, thanks.

Quoted 3 lines

.

#23 lore raw

26/07/2021 02:26, Min Hu (Connor):

Quoted 27 lines · Thomas Monjalon
Quoted 26 lines

10/07/2021 05:50, Min Hu (Connor):

Quoted 24 lines

This patch adds multi-process support for testpmd. For example the following commands run two testpmd processes:

* the primary process:

./dpdk-testpmd --proc-type=auto -l 0-1 -- -i \
    --rxq=4 --txq=4 --num-procs=2 --proc-id=0

* the secondary process:

./dpdk-testpmd --proc-type=auto -l 2-3 -- -i \
    --rxq=4 --txq=4 --num-procs=2 --proc-id=1

---
V16:
* revert unrelated changes.
* add some restrictions in doc.

Quoted 8 lines
Quoted 4 lines

I didn't see clear agreement to integrate this feature in DPDK 21.08.

BTW, the testpmd maintainer was not Cc'ed.

Hi, Thomas,
This patch has two "acked" and one "reviewed-by".
And Xiaoyun Li is the maintainer, I has already Cc'ed to him/her, please check it out, thanks.

Andrew's comment from v15:
"
IMHO, as I state above, current state is insufficient to consider is a start point to be applied.
"

#24 lore raw
Quoted 37 lines · Thomas Monjalon

26/07/2021 02:26, Min Hu (Connor):

Quoted 27 lines · Thomas Monjalon
Quoted 26 lines

10/07/2021 05:50, Min Hu (Connor):

Quoted 24 lines

This patch adds multi-process support for testpmd. For example the following commands run two testpmd processes:

* the primary process:

./dpdk-testpmd --proc-type=auto -l 0-1 -- -i \
     --rxq=4 --txq=4 --num-procs=2 --proc-id=0

* the secondary process:

./dpdk-testpmd --proc-type=auto -l 2-3 -- -i \
     --rxq=4 --txq=4 --num-procs=2 --proc-id=1

---
V16:
* revert unrelated changes.
* add some restrictions in doc.

Quoted 8 lines
Quoted 4 lines

I didn't see clear agreement to integrate this feature in DPDK 21.08.

BTW, the testpmd maintainer was not Cc'ed.

Hi, Thomas,
This patch has two "acked" and one "reviewed-by".
And Xiaoyun Li is the maintainer, I has already Cc'ed to him/her, please check it out, thanks.

Quoted 5 lines

Andrew's comment from v15:
"
IMHO, as I state above, current state is insufficient to consider is a start point to be applied.
"

Hi, Andrew, Thomos,
What is the least thing should be done for this patch to
meet the state sufficient to be applied?

Quoted 4 lines

.

#25 lore raw
Quoted 40 lines · Min Hu (Connor)
Quoted 37 lines · Thomas Monjalon

26/07/2021 02:26, Min Hu (Connor):

Quoted 35 lines · Thomas Monjalon
Quoted 26 lines

10/07/2021 05:50, Min Hu (Connor):

Quoted 20 lines

This patch adds multi-process support for testpmd. For example the following commands run two testpmd processes:

   * the primary process:

./dpdk-testpmd --proc-type=auto -l 0-1 -- -i \
     --rxq=4 --txq=4 --num-procs=2 --proc-id=0

   * the secondary process:

./dpdk-testpmd --proc-type=auto -l 2-3 -- -i \
     --rxq=4 --txq=4 --num-procs=2 --proc-id=1

Quoted 4 lines

---
V16:
* revert unrelated changes.
* add some restrictions in doc.

Quoted 4 lines

I didn't see clear agreement to integrate this feature in DPDK 21.08.

BTW, the testpmd maintainer was not Cc'ed.

Hi, Thomas,
This patch has two "acked" and one "reviewed-by".
And Xiaoyun Li is the maintainer, I has already Cc'ed to him/her, please check it out, thanks.

Quoted 8 lines
Quoted 5 lines

Andrew's comment from v15:
"
IMHO, as I state above, current state is insufficient to consider is a start point to be applied.
"

Hi, Andrew, Thomos,
    What is the least thing should be done for this patch to
meet the state sufficient to be applied?

Hi, all,
How about this patch, could it be applied?

Quoted 6 lines
Quoted 5 lines

.

.

#26 lore raw

02/08/2021 03:51, Min Hu (Connor):

Quoted 44 lines · Min Hu (Connor)
Quoted 43 lines · Thomas Monjalon
Quoted 42 lines

26/07/2021 02:26, Min Hu (Connor):

Quoted 27 lines · Thomas Monjalon
Quoted 26 lines

10/07/2021 05:50, Min Hu (Connor):

Quoted 24 lines

This patch adds multi-process support for testpmd. For example the following commands run two testpmd processes:

* the primary process:

./dpdk-testpmd --proc-type=auto -l 0-1 -- -i \
     --rxq=4 --txq=4 --num-procs=2 --proc-id=0

* the secondary process:

./dpdk-testpmd --proc-type=auto -l 2-3 -- -i \
     --rxq=4 --txq=4 --num-procs=2 --proc-id=1

---
V16:
* revert unrelated changes.
* add some restrictions in doc.

Quoted 8 lines
Quoted 4 lines

I didn't see clear agreement to integrate this feature in DPDK 21.08.

BTW, the testpmd maintainer was not Cc'ed.

Hi, Thomas,
This patch has two "acked" and one "reviewed-by".
And Xiaoyun Li is the maintainer, I has already Cc'ed to him/her, please check it out, thanks.

Andrew's comment from v15:
"
IMHO, as I state above, current state is insufficient to consider is a start point to be applied.
"

Quoted 5 lines
Quoted 3 lines

Hi, Andrew, Thomos,
What is the least thing should be done for this patch to
meet the state sufficient to be applied?

Hi, all,
How about this patch, could it be applied?

I was not confident enough to apply it in 21.08. When others will be back, we'll discuss it again for 21.11.

#27 lore raw
Quoted 43 lines · Thomas Monjalon

02/08/2021 03:51, Min Hu (Connor):

Quoted 42 lines · Min Hu (Connor)
Quoted 41 lines · Thomas Monjalon
Quoted 40 lines

26/07/2021 02:26, Min Hu (Connor):

Quoted 26 lines · Thomas Monjalon
Quoted 25 lines

10/07/2021 05:50, Min Hu (Connor):

Quoted 24 lines

This patch adds multi-process support for testpmd. For example the following commands run two testpmd processes:

* the primary process:

./dpdk-testpmd --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

* the secondary process:

./dpdk-testpmd --proc-type=auto -l 2-3 -- -i \
      --rxq=4 --txq=4 --num-procs=2 --proc-id=1

---
V16:
* revert unrelated changes.
* add some restrictions in doc.

Quoted 8 lines
Quoted 4 lines

I didn't see clear agreement to integrate this feature in DPDK 21.08.

BTW, the testpmd maintainer was not Cc'ed.

Hi, Thomas,
This patch has two "acked" and one "reviewed-by".
And Xiaoyun Li is the maintainer, I has already Cc'ed to him/her, please check it out, thanks.

Andrew's comment from v15:
"
IMHO, as I state above, current state is insufficient to consider is a start point to be applied.
"

Quoted 8 lines
Quoted 5 lines
Quoted 3 lines

Hi, Andrew, Thomos,
What is the least thing should be done for this patch to
meet the state sufficient to be applied?

Hi, all,
How about this patch, could it be applied?

I was not confident enough to apply it in 21.08. When others will be back, we'll discuss it again for 21.11.

For 21.11 release.

#28 lore raw
Quoted 48 lines · Aman Deep
Quoted 46 lines · Thomas Monjalon

02/08/2021 03:51, Min Hu (Connor):

Quoted 45 lines · Min Hu (Connor)
Quoted 44 lines · Thomas Monjalon
Quoted 35 lines

26/07/2021 02:26, Min Hu (Connor):

Quoted 26 lines · Thomas Monjalon
Quoted 25 lines

10/07/2021 05:50, Min Hu (Connor):

Quoted 24 lines

This patch adds multi-process support for testpmd. For example the following commands run two testpmd processes:

* the primary process:

./dpdk-testpmd --proc-type=auto -l 0-1 -- -i \
     --rxq=4 --txq=4 --num-procs=2 --proc-id=0

* the secondary process:

./dpdk-testpmd --proc-type=auto -l 2-3 -- -i \
     --rxq=4 --txq=4 --num-procs=2 --proc-id=1

---
V16:
* revert unrelated changes.
* add some restrictions in doc.

Quoted 8 lines
Quoted 4 lines

I didn't see clear agreement to integrate this feature in DPDK 21.08.

BTW, the testpmd maintainer was not Cc'ed.

Hi, Thomas,
This patch has two "acked" and one "reviewed-by".
And Xiaoyun Li is the maintainer, I has already Cc'ed to him/her, please check it out, thanks.

Quoted 5 lines

Andrew's comment from v15:
"
IMHO, as I state above, current state is insufficient to consider is a start point to be applied.
"

Hi, Andrew, Thomos,
What is the least thing should be done for this patch to
meet the state sufficient to be applied?

Quoted 9 lines
Quoted 5 lines
Quoted 2 lines

Hi, all,
How about this patch, could it be applied?

I was not confident enough to apply it in 21.08. When others will be back, we'll discuss it again for 21.11.

For 21.11 release.

Hi Connor,

Can you please rebase it to latest dpdk (v21.11-rc0) so we can proceed with it?

I am for getting the patch as early as possible in the release cycle, specially before it is getting busy for -rc1.

Thanks,
ferruh

#29 lore raw

Thanks Ferruh, I will handle it as quickly as possible.

Quoted 70 lines · Ferruh Yigit
Quoted 48 lines · Aman Deep
Quoted 46 lines · Thomas Monjalon

02/08/2021 03:51, Min Hu (Connor):

Quoted 45 lines · Min Hu (Connor)
Quoted 44 lines · Thomas Monjalon
Quoted 35 lines

26/07/2021 02:26, Min Hu (Connor):

Quoted 26 lines · Thomas Monjalon
Quoted 25 lines

10/07/2021 05:50, Min Hu (Connor):

Quoted 24 lines

This patch adds multi-process support for testpmd. For example the following commands run two testpmd processes:

* the primary process:

./dpdk-testpmd --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

* the secondary process:

./dpdk-testpmd --proc-type=auto -l 2-3 -- -i \
      --rxq=4 --txq=4 --num-procs=2 --proc-id=1

---
V16:
* revert unrelated changes.
* add some restrictions in doc.

Quoted 8 lines
Quoted 4 lines

I didn't see clear agreement to integrate this feature in DPDK 21.08.

BTW, the testpmd maintainer was not Cc'ed.

Hi, Thomas,
This patch has two "acked" and one "reviewed-by".
And Xiaoyun Li is the maintainer, I has already Cc'ed to him/her, please check it out, thanks.

Quoted 5 lines

Andrew's comment from v15:
"
IMHO, as I state above, current state is insufficient to consider is a start point to be applied.
"

Hi, Andrew, Thomos,
What is the least thing should be done for this patch to
meet the state sufficient to be applied?

Quoted 9 lines
Quoted 5 lines
Quoted 2 lines

Hi, all,
How about this patch, could it be applied?

I was not confident enough to apply it in 21.08. When others will be back, we'll discuss it again for 21.11.

For 21.11 release.

Hi Connor,

Can you please rebase it to latest dpdk (v21.11-rc0) so we can proceed with it?

I am for getting the patch as early as possible in the release cycle, specially before it is getting busy for -rc1.

Thanks,
ferruh
.

#30 lore raw

PATCH v17 1/1 app/testpmd: support multi-process

In reply to Min Hu (Connor)

This patch adds multi-process support for testpmd. For example the following commands run two testpmd processes:

* the primary process:

./dpdk-testpmd --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

* the secondary process:

./dpdk-testpmd --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

---
v17:
rebase 21.11

V16:
* revert unrelated changes.
* add some restrictions in doc.

v15:
* Fixed release notes.
* Cleanup documentation.

v14:
* Fixed comments by Andrew Rybchenko.

v13:
* Modified the doc syntax.

v12:
* Updated doc info.

v11:
* Fixed some minor syntax.

v10:
* Hid process type checks behind new functions.
* Added comments.

v9:
* Updated release notes and rst doc.
* Deleted deprecated codes.
* move macro and variable.

v8:
* Added warning info about queue numbers and process numbers.

v7:
* Fixed compiling error for unexpected unindent.

v6:
* Add rte flow description for multiple process.

v5:
* Fixed run_app.rst for multiple process description.
* Fix compiling error.

v4:
* Fixed minimum vlaue of Rxq or Txq in doc.

v3:
* Fixed compiling error using gcc10.0.

v2:
* Added document for this patch.

---
 app/test-pmd/cmdline.c                 |  6 +++
 app/test-pmd/config.c                  | 20 ++++++-
 app/test-pmd/parameters.c              |  9 ++++
 app/test-pmd/testpmd.c                 | 95 +++++++++++++++++++++++++++++-----
 app/test-pmd/testpmd.h                 |  9 ++++
 doc/guides/rel_notes/release_21_11.rst |  4 ++
 doc/guides/testpmd_app_ug/run_app.rst  | 84 ++++++++++++++++++++++++++++++
 7 files changed, 214 insertions(+), 13 deletions(-)
app/test-pmd/cmdline.c +6 -0
index 82253bc..395122b 100644
@@ -5491,6 +5491,12 @@ cmd_set_flush_rx_parsed(void *parsed_result,
 		__rte_unused void *data)
 {
 	struct cmd_set_flush_rx *res = parsed_result;
+
+	if (num_procs > 1 && (strcmp(res->mode, "on") == 0)) {
+		printf("multi-process doesn't support to flush Rx queues.\n");
+		return;
+	}
+
 	no_flush_rx = (uint8_t)((strcmp(res->mode, "on") == 0) ? 0 : 1);
 }
 
app/test-pmd/config.c +19 -1
index 31d8ba1..bd404db 100644
@@ -2998,6 +2998,8 @@ rss_fwd_config_setup(void)
 	queueid_t  rxq;
 	queueid_t  nb_q;
 	streamid_t  sm_id;
+	int start;
+	int end;
 
 	nb_q = nb_rxq;
 	if (nb_q > nb_txq)
@@ -3015,7 +3017,21 @@ rss_fwd_config_setup(void)
 	init_fwd_streams();
 
 	setup_fwd_config_of_each_lcore(&cur_fwd_config);
-	rxp = 0; rxq = 0;
+
+	if (proc_id > 0 && nb_q % num_procs != 0)
+		printf("Warning! queue numbers should be multiple of processes, or packet loss will happen.\n");
+
+	/**
+	 * In multi-process, All queues are allocated to different
+	 * processes based on num_procs and proc_id. For example:
+	 * if supports 4 queues(nb_q), 2 processes(num_procs),
+	 * the 0~1 queue for primary process.
+	 * the 2~3 queue for secondary process.
+	 */
+	start = proc_id * nb_q / num_procs;
+	end = start + nb_q / num_procs;
+	rxp = 0;
+	rxq = start;
 	for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
 		struct fwd_stream *fs;
 
@@ -3032,6 +3048,8 @@ rss_fwd_config_setup(void)
 			continue;
 		rxp = 0;
 		rxq++;
+		if (rxq >= end)
+			rxq = start;
 	}
 }
 
app/test-pmd/parameters.c +9 -0
index 7c13210..26192f1 100644
@@ -512,6 +512,9 @@ parse_link_speed(int n)
 void
 launch_args_parse(int argc, char** argv)
 {
+#define PARAM_PROC_ID "proc-id"
+#define PARAM_NUM_PROCS "num-procs"
+
 	int n, opt;
 	char **argvopt;
 	int opt_idx;
@@ -631,6 +634,8 @@ launch_args_parse(int argc, char** argv)
 		{ "rx-mq-mode",                 1, 0, 0 },
 		{ "record-core-cycles",         0, 0, 0 },
 		{ "record-burst-stats",         0, 0, 0 },
+		{ PARAM_NUM_PROCS,              1, 0, 0 },
+		{ PARAM_PROC_ID,                1, 0, 0 },
 		{ 0, 0, 0, 0 },
 	};
 
@@ -1397,6 +1402,10 @@ launch_args_parse(int argc, char** argv)
 				record_core_cycles = 1;
 			if (!strcmp(lgopts[opt_idx].name, "record-burst-stats"))
 				record_burst_stats = 1;
+			if (!strcmp(lgopts[opt_idx].name, PARAM_NUM_PROCS))
+				num_procs = atoi(optarg);
+			if (!strcmp(lgopts[opt_idx].name, PARAM_PROC_ID))
+				proc_id = atoi(optarg);
 			break;
 		case 'h':
 			usage(argv[0]);
app/test-pmd/testpmd.c +83 -12

167 lines not on this page · raw message

app/test-pmd/testpmd.h +9 -0
index 16a3598..04773c7 100644
@@ -632,6 +632,15 @@ extern enum rte_eth_rx_mq_mode rx_mq_mode;
 
 extern struct rte_flow_action_conntrack conntrack_context;
 
+extern int proc_id;
+extern unsigned int num_procs;
+
+static inline bool
+is_proc_primary(void)
+{
+	return rte_eal_process_type() == RTE_PROC_PRIMARY;
+}
+
 static inline unsigned int
 lcore_num(void)
 {
doc/guides/rel_notes/release_21_11.rst +4 -0
index d707a55..2a9d3b9 100644
@@ -51,6 +51,10 @@ New Features
      * Other libs
      * Apps, Examples, Tools (if significant)
 
+	 * **Added support multi-process for testpmd.**
+       Added command-line options to specify total number of processes and
+       current process ID. Each process owns subset of Rx and Tx queues.
+
      This section is a comment. Do not overwrite or remove it.
      Also, make sure to start the actual text at the margin.
      =======================================================
doc/guides/testpmd_app_ug/run_app.rst +84 -0

87 lines not on this page · raw message

Signature · 1 line
2.7.4
Applied
#31 lore raw
Quoted 14 lines · Min Hu (Connor)

This patch adds multi-process support for testpmd. For example the following commands run two testpmd processes:

* the primary process:

./dpdk-testpmd --proc-type=auto -l 0-1 -- -i \
   --rxq=4 --txq=4 --num-procs=2 --proc-id=0

* the secondary process:

./dpdk-testpmd --proc-type=auto -l 2-3 -- -i \
   --rxq=4 --txq=4 --num-procs=2 --proc-id=1
Quoted 7 lines

Applied to dpdk-next-net/main, thanks.

Thanks Connor for the work,
I won't be surprised if we need a few more tweak/fix for the testpmd secondary process support but right now there is no blocker issue, so proceeding with the patch, please continue your support for further required changes.

#32 lore raw
Quoted 23 lines · Ferruh Yigit
Quoted 14 lines · Min Hu (Connor)

This patch adds multi-process support for testpmd. For example the following commands run two testpmd processes:

* the primary process:

./dpdk-testpmd --proc-type=auto -l 0-1 -- -i \
    --rxq=4 --txq=4 --num-procs=2 --proc-id=0

* the secondary process:

./dpdk-testpmd --proc-type=auto -l 2-3 -- -i \
    --rxq=4 --txq=4 --num-procs=2 --proc-id=1
Quoted 7 lines
Quoted 7 lines

Applied to dpdk-next-net/main, thanks.

Thanks Connor for the work,
I won't be surprised if we need a few more tweak/fix for the testpmd secondary process support but right now there is no blocker issue, so proceeding with the patch, please continue your support for further required changes.

Yes, got it, thanks Ferruh.

Quoted 2 lines

.

#33 lore raw

PATCH v3 1/1 app/testpmd: support multi-process

In reply to Lijun Ou

From: Lijun Ou <[email protected]>

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

---
 app/test-pmd/cmdline.c                |  12 ++-
 app/test-pmd/config.c                 |   9 ++-
 app/test-pmd/parameters.c             |  11 +++
 app/test-pmd/testpmd.c                | 138 ++++++++++++++++++++++------------
 app/test-pmd/testpmd.h                |   7 ++
 doc/guides/testpmd_app_ug/run_app.rst |  69 +++++++++++++++++
 6 files changed, 196 insertions(+), 50 deletions(-)
app/test-pmd/cmdline.c +10 -2
index 14110eb..287d7a0 100644
@@ -71,8 +71,6 @@
 #include "cmdline_tm.h"
 #include "bpf_cmd.h"
 
-static struct cmdline *testpmd_cl;
-
 static void cmd_reconfig_device_queue(portid_t id, uint8_t dev, uint8_t queue);
 
 /* *** Help command with introduction. *** */
@@ -5351,6 +5349,12 @@ cmd_set_flush_rx_parsed(void *parsed_result,
 		__rte_unused void *data)
 {
 	struct cmd_set_flush_rx *res = parsed_result;
+
+	if (num_procs > 1 && (strcmp(res->mode, "on") == 0)) {
+		printf("multi-process doesn't support to flush rx queues.\n");
+		return;
+	}
+
 	no_flush_rx = (uint8_t)((strcmp(res->mode, "on") == 0) ? 0 : 1);
 }
 
@@ -17227,6 +17231,10 @@ prompt(void)
 		printf("Cannot set exit function for cmdline\n");
 
 	cmdline_interact(testpmd_cl);
+	if (unlikely(f_quit == 1)) {
+		dup2(testpmd_fd_copy, testpmd_cl->s_in);
+		close(testpmd_fd_copy);
+	}
 	if (ret != 0)
 		cmdline_stdin_exit(testpmd_cl);
 }
app/test-pmd/config.c +8 -1
index 576d5ac..9edf84a 100644
@@ -2827,6 +2827,8 @@ rss_fwd_config_setup(void)
 	queueid_t  rxq;
 	queueid_t  nb_q;
 	streamid_t  sm_id;
+	int start;
+	int end;
 
 	nb_q = nb_rxq;
 	if (nb_q > nb_txq)
@@ -2844,7 +2846,10 @@ rss_fwd_config_setup(void)
 	init_fwd_streams();
 
 	setup_fwd_config_of_each_lcore(&cur_fwd_config);
-	rxp = 0; rxq = 0;
+	start = proc_id * nb_q / num_procs;
+	end = start + nb_q / num_procs;
+	rxp = 0;
+	rxq = start;
 	for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
 		struct fwd_stream *fs;
 
@@ -2861,6 +2866,8 @@ rss_fwd_config_setup(void)
 			continue;
 		rxp = 0;
 		rxq++;
+		if (rxq >= end)
+			rxq = start;
 	}
 }
 
app/test-pmd/parameters.c +11 -0
index c8acd5d..1a5bf25 100644
@@ -45,6 +45,8 @@
 #include <rte_flow.h>
 
 #include "testpmd.h"
+#define PARAM_PROC_ID "proc-id"
+#define PARAM_NUM_PROCS "num-procs"
 
 static void
 usage(char* progname)
@@ -605,6 +607,8 @@ launch_args_parse(int argc, char** argv)
 		{ "rx-mq-mode",                 1, 0, 0 },
 		{ "record-core-cycles",         0, 0, 0 },
 		{ "record-burst-stats",         0, 0, 0 },
+		{ PARAM_NUM_PROCS,              1, 0, 0 },
+		{ PARAM_PROC_ID,                1, 0, 0 },
 		{ 0, 0, 0, 0 },
 	};
 
@@ -1366,6 +1370,13 @@ launch_args_parse(int argc, char** argv)
 				record_core_cycles = 1;
 			if (!strcmp(lgopts[opt_idx].name, "record-burst-stats"))
 				record_burst_stats = 1;
+
+			if (strncmp(lgopts[opt_idx].name,
+				    PARAM_NUM_PROCS, 8) == 0)
+				num_procs = atoi(optarg);
+			if (strncmp(lgopts[opt_idx].name,
+				    PARAM_PROC_ID, 7) == 0)
+				proc_id = atoi(optarg);
 			break;
 		case 'h':
 			usage(argv[0]);
app/test-pmd/testpmd.c +91 -47

253 lines not on this page · raw message

app/test-pmd/testpmd.h +7 -0
index ce83f31..6d13417 100644
@@ -13,6 +13,7 @@
 #include <rte_gso.h>
 #include <cmdline.h>
 #include <sys/queue.h>
+#include  "cmdline.h"
 
 #define RTE_PORT_ALL            (~(portid_t)0x0)
 
@@ -24,6 +25,10 @@
 #define RTE_PORT_CLOSED         (uint16_t)2
 #define RTE_PORT_HANDLING       (uint16_t)3
 
+uint8_t f_quit;
+int testpmd_fd_copy;
+struct cmdline *testpmd_cl;
+
 /*
  * It is used to allocate the memory for hash key.
  * The hash key size is NIC dependent.
@@ -422,6 +427,8 @@ extern uint64_t noisy_lkup_mem_sz;
 extern uint64_t noisy_lkup_num_writes;
 extern uint64_t noisy_lkup_num_reads;
 extern uint64_t noisy_lkup_num_reads_writes;
+extern int proc_id;
+extern unsigned int num_procs;
 
 extern uint8_t dcb_config;
 extern uint8_t dcb_test;
doc/guides/testpmd_app_ug/run_app.rst +69 -0

72 lines not on this page · raw message

Signature · 1 line
2.7.4
#34 lore raw

Hi, ferruh and all,
have you any comments about this patch?
thanks.

Quoted 514 lines · Min Hu (Connor)

From: Lijun Ou <[email protected]>

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

---
  app/test-pmd/cmdline.c                |  12 ++-
  app/test-pmd/config.c                 |   9 ++-
  app/test-pmd/parameters.c             |  11 +++
  app/test-pmd/testpmd.c                | 138 ++++++++++++++++++++++------------
  app/test-pmd/testpmd.h                |   7 ++
  doc/guides/testpmd_app_ug/run_app.rst |  69 +++++++++++++++++
  6 files changed, 196 insertions(+), 50 deletions(-)
app/test-pmd/cmdline.c +10 -2
index 14110eb..287d7a0 100644
@@ -71,8 +71,6 @@
  #include "cmdline_tm.h"
  #include "bpf_cmd.h"
 
-static struct cmdline *testpmd_cl;
-
  static void cmd_reconfig_device_queue(portid_t id, uint8_t dev, uint8_t queue);
 
  /* *** Help command with introduction. *** */
@@ -5351,6 +5349,12 @@ cmd_set_flush_rx_parsed(void *parsed_result,
  		__rte_unused void *data)
  {
  	struct cmd_set_flush_rx *res = parsed_result;
+
+	if (num_procs > 1 && (strcmp(res->mode, "on") == 0)) {
+		printf("multi-process doesn't support to flush rx queues.\n");
+		return;
+	}
+
  	no_flush_rx = (uint8_t)((strcmp(res->mode, "on") == 0) ? 0 : 1);
  }
 
@@ -17227,6 +17231,10 @@ prompt(void)
  		printf("Cannot set exit function for cmdline\n");
 
  	cmdline_interact(testpmd_cl);
+	if (unlikely(f_quit == 1)) {
+		dup2(testpmd_fd_copy, testpmd_cl->s_in);
+		close(testpmd_fd_copy);
+	}
  	if (ret != 0)
  		cmdline_stdin_exit(testpmd_cl);
  }
app/test-pmd/config.c +8 -1
index 576d5ac..9edf84a 100644
@@ -2827,6 +2827,8 @@ rss_fwd_config_setup(void)
  	queueid_t  rxq;
  	queueid_t  nb_q;
  	streamid_t  sm_id;
+	int start;
+	int end;
 
  	nb_q = nb_rxq;
  	if (nb_q > nb_txq)
@@ -2844,7 +2846,10 @@ rss_fwd_config_setup(void)
  	init_fwd_streams();
 
  	setup_fwd_config_of_each_lcore(&cur_fwd_config);
-	rxp = 0; rxq = 0;
+	start = proc_id * nb_q / num_procs;
+	end = start + nb_q / num_procs;
+	rxp = 0;
+	rxq = start;
  	for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
  		struct fwd_stream *fs;
 
@@ -2861,6 +2866,8 @@ rss_fwd_config_setup(void)
  			continue;
  		rxp = 0;
  		rxq++;
+		if (rxq >= end)
+			rxq = start;
  	}
  }
 
app/test-pmd/parameters.c +11 -0
index c8acd5d..1a5bf25 100644
@@ -45,6 +45,8 @@
  #include <rte_flow.h>
 
  #include "testpmd.h"
+#define PARAM_PROC_ID "proc-id"
+#define PARAM_NUM_PROCS "num-procs"
 
  static void
  usage(char* progname)
@@ -605,6 +607,8 @@ launch_args_parse(int argc, char** argv)
  		{ "rx-mq-mode",                 1, 0, 0 },
  		{ "record-core-cycles",         0, 0, 0 },
  		{ "record-burst-stats",         0, 0, 0 },
+		{ PARAM_NUM_PROCS,              1, 0, 0 },
+		{ PARAM_PROC_ID,                1, 0, 0 },
  		{ 0, 0, 0, 0 },
  	};
 
@@ -1366,6 +1370,13 @@ launch_args_parse(int argc, char** argv)
  				record_core_cycles = 1;
  			if (!strcmp(lgopts[opt_idx].name, "record-burst-stats"))
  				record_burst_stats = 1;
+
+			if (strncmp(lgopts[opt_idx].name,
+				    PARAM_NUM_PROCS, 8) == 0)
+				num_procs = atoi(optarg);
+			if (strncmp(lgopts[opt_idx].name,
+				    PARAM_PROC_ID, 7) == 0)
+				proc_id = atoi(optarg);
  			break;
  		case 'h':
  			usage(argv[0]);
app/test-pmd/testpmd.c +91 -47
index 1a57324..bbd45d9 100644
@@ -63,6 +63,8 @@
 
  #include "testpmd.h"
 
+int testpmd_fd_copy = 500; /* the copy of STDIN_FILENO */
+
  #ifndef MAP_HUGETLB
  /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  #define HUGE_FLAG (0x40000)
@@ -125,6 +127,9 @@ uint8_t port_numa[RTE_MAX_ETHPORTS];
   */
  uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 
+int proc_id;
+unsigned int num_procs = 1;
+
  /*
   * Store specified sockets on which TX ring to be used by ports
   * is allocated.
@@ -982,16 +987,26 @@ mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
  			/* wrapper to rte_mempool_create() */
  			TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
  					rte_mbuf_best_mempool_ops());
-			rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
-				mb_mempool_cache, 0, mbuf_seg_size, socket_id);
+			if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+				rte_mp = rte_pktmbuf_pool_create(pool_name,
+					 nb_mbuf, mb_mempool_cache, 0,
+					 mbuf_seg_size, socket_id);
+			else
+				rte_mp = rte_mempool_lookup(pool_name);
+
  			break;
  		}
  	case MP_ALLOC_ANON:
  		{
-			rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
-				mb_size, (unsigned int) mb_mempool_cache,
-				sizeof(struct rte_pktmbuf_pool_private),
-				socket_id, mempool_flags);
+			if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+				rte_mp = rte_mempool_create_empty(pool_name,
+					nb_mbuf, mb_size,
+					(unsigned int)mb_mempool_cache,
+					sizeof(struct rte_pktmbuf_pool_private),
+					socket_id, mempool_flags);
+			else
+				rte_mp = rte_mempool_lookup(pool_name);
+
  			if (rte_mp == NULL)
  				goto err;
 
@@ -1021,9 +1036,13 @@ mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 
  			TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
  					rte_mbuf_best_mempool_ops());
-			rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
-					mb_mempool_cache, 0, mbuf_seg_size,
-					heap_socket);
+			if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+				rte_mp = rte_pktmbuf_pool_create(pool_name,
+					 nb_mbuf, mb_mempool_cache, 0,
+					 mbuf_seg_size, heap_socket);
+			else
+				rte_mp = rte_mempool_lookup(pool_name);
+
  			break;
  		}
  	case MP_ALLOC_XBUF:
@@ -1994,6 +2013,12 @@ flush_fwd_rx_queues(void)
  	uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
  	uint64_t timer_period;
 
+	if (num_procs > 1) {
+		printf("multi-process not support for flushing fwd rx "
+		       "queues, skip the below lines and return.\n");
+		return;
+	}
+
  	/* convert to number of cycles */
  	timer_period = rte_get_timer_hz(); /* 1 second timeout */
 
@@ -2503,21 +2528,28 @@ start_port(portid_t pid)
  				return -1;
  			}
  			/* configure port */
-			diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
+			if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+				diag = rte_eth_dev_configure(pi,
+						     nb_rxq + nb_hairpinq,
  						     nb_txq + nb_hairpinq,
  						     &(port->dev_conf));
-			if (diag != 0) {
-				if (rte_atomic16_cmpset(&(port->port_status),
-				RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
-					printf("Port %d can not be set back "
-							"to stopped\n", pi);
-				printf("Fail to configure port %d\n", pi);
-				/* try to reconfigure port next time */
-				port->need_reconfig = 1;
-				return -1;
+				if (diag != 0) {
+					if (rte_atomic16_cmpset(
+							&(port->port_status),
+							RTE_PORT_HANDLING,
+							RTE_PORT_STOPPED) == 0)
+						printf("Port %d can not be set "
+						       "back to stopped\n", pi);
+					printf("Fail to configure port %d\n",
+						pi);
+					/* try to reconfigure port next time */
+					port->need_reconfig = 1;
+					return -1;
+				}
  			}
  		}
-		if (port->need_reconfig_queues > 0) {
+		if (port->need_reconfig_queues > 0 &&
+		    rte_eal_process_type() == RTE_PROC_PRIMARY) {
  			port->need_reconfig_queues = 0;
  			/* setup tx queues */
  			for (qi = 0; qi < nb_txq; qi++) {
@@ -2618,15 +2650,18 @@ start_port(portid_t pid)
  		cnt_pi++;
 
  		/* start port */
-		if (rte_eth_dev_start(pi) < 0) {
-			printf("Fail to start port %d\n", pi);
+		if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+			diag = rte_eth_dev_start(pi);
+			if (diag < 0) {
+				printf("Fail to start port %d\n", pi);
 
-			/* Fail to setup rx queue, return */
-			if (rte_atomic16_cmpset(&(port->port_status),
-				RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
-				printf("Port %d can not be set back to "
+				/* Fail to setup rx queue, return */
+				if (rte_atomic16_cmpset(&(port->port_status),
+				    RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
+					printf("Port %d can not be set back to "
  							"stopped\n", pi);
-			continue;
+				continue;
+			}
  		}
 
  		if (rte_atomic16_cmpset(&(port->port_status),
@@ -2755,7 +2790,8 @@ stop_port(portid_t pid)
  		if (port->flow_list)
  			port_flow_flush(pi);
 
-		if (rte_eth_dev_stop(pi) != 0)
+		if (rte_eal_process_type() == RTE_PROC_PRIMARY &&
+		    rte_eth_dev_stop(pi) != 0)
  			RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n",
  				pi);
 
@@ -2824,8 +2860,10 @@ close_port(portid_t pid)
  			continue;
  		}
 
-		port_flow_flush(pi);
-		rte_eth_dev_close(pi);
+		if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+			port_flow_flush(pi);
+		if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+			rte_eth_dev_close(pi);
  	}
 
  	remove_invalid_ports();
@@ -3089,7 +3127,7 @@ pmd_test_exit(void)
  		}
  	}
  	for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
-		if (mempools[i])
+		if (rte_eal_process_type() == RTE_PROC_PRIMARY && mempools[i])
  			rte_mempool_free(mempools[i]);
  	}
 
@@ -3611,6 +3649,10 @@ init_port_dcb_config(portid_t pid,
  	int retval;
  	uint16_t i;
 
+	if (num_procs > 1) {
+		printf("The multi-process feature doesn't support dcb.\n");
+		return -ENOTSUP;
+	}
  	rte_port = &ports[pid];
 
  	memset(&port_conf, 0, sizeof(struct rte_eth_conf));
@@ -3709,13 +3751,6 @@ init_port(void)
  }
 
  static void
-force_quit(void)
-{
-	pmd_test_exit();
-	prompt_exit();
-}
-
-static void
  print_stats(void)
  {
  	uint8_t i;
@@ -3746,12 +3781,16 @@ signal_handler(int signum)
  		if (latencystats_enabled != 0)
  			rte_latencystats_uninit();
  #endif
-		force_quit();
  		/* Set flag to indicate the force termination. */
  		f_quit = 1;
-		/* exit with the expected status */
-		signal(signum, SIG_DFL);
-		kill(getpid(), signum);
+		if (interactive == 1) {
+			dup2(testpmd_cl->s_in, testpmd_fd_copy);
+			close(testpmd_cl->s_in);
+		} else {
+			dup2(0, testpmd_fd_copy);
+			close(0);
+		}
+
  	}
  }
 
@@ -3776,10 +3815,6 @@ main(int argc, char** argv)
  		rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
  			 rte_strerror(rte_errno));
 
-	if (rte_eal_process_type() == RTE_PROC_SECONDARY)
-		rte_exit(EXIT_FAILURE,
-			 "Secondary process type not supported.\n");
-
  	ret = register_eth_event_callback();
  	if (ret != 0)
  		rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
@@ -3875,8 +3910,10 @@ main(int argc, char** argv)
  		}
  	}
 
-	if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
+	if (!no_device_start && start_port(RTE_PORT_ALL) != 0) {
+		pmd_test_exit();
  		rte_exit(EXIT_FAILURE, "Start ports failed\n");
+	}
 
  	/* set all ports to promiscuous mode by default */
  	RTE_ETH_FOREACH_DEV(port_id) {
@@ -3922,6 +3959,8 @@ main(int argc, char** argv)
  		}
  		prompt();
  		pmd_test_exit();
+		if (unlikely(f_quit == 1))
+			prompt_exit();
  	} else
  #endif
  	{
@@ -3957,6 +3996,11 @@ main(int argc, char** argv)
  		printf("Press enter to exit\n");
  		rc = read(0, &c, 1);
  		pmd_test_exit();
+		if (unlikely(f_quit == 1)) {
+			dup2(testpmd_fd_copy, 0);
+			close(testpmd_fd_copy);
+			prompt_exit();
+		}
  		if (rc < 0)
  			return 1;
  	}
app/test-pmd/testpmd.h +7 -0
index ce83f31..6d13417 100644
@@ -13,6 +13,7 @@
  #include <rte_gso.h>
  #include <cmdline.h>
  #include <sys/queue.h>
+#include  "cmdline.h"
 
  #define RTE_PORT_ALL            (~(portid_t)0x0)
 
@@ -24,6 +25,10 @@
  #define RTE_PORT_CLOSED         (uint16_t)2
  #define RTE_PORT_HANDLING       (uint16_t)3
 
+uint8_t f_quit;
+int testpmd_fd_copy;
+struct cmdline *testpmd_cl;
+
  /*
   * It is used to allocate the memory for hash key.
   * The hash key size is NIC dependent.
@@ -422,6 +427,8 @@ extern uint64_t noisy_lkup_mem_sz;
  extern uint64_t noisy_lkup_num_writes;
  extern uint64_t noisy_lkup_num_reads;
  extern uint64_t noisy_lkup_num_reads_writes;
+extern int proc_id;
+extern unsigned int num_procs;
 
  extern uint8_t dcb_config;
  extern uint8_t dcb_test;
doc/guides/testpmd_app_ug/run_app.rst +69 -0
index 6745072..8e27fcd 100644
@@ -536,3 +536,72 @@ The command line options are:
      bit 1 - two hairpin ports paired
      bit 0 - two hairpin ports loop
      The default value is 0. Hairpin will use single port mode and implicit Tx flow mode.
+
+
+Testpmd Support Multi Process Command-line Options
+--------------------------------------------------
+
+The following are the command-line options for the testpmd applications(support multi process).
+They must be separated from the EAL options, shown in the previous section, with a ``--`` separator:
+
+.. code-block:: console
+
+	primary process:
+    sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i --rxq=4 --txq=4 --num-procs=2 --proc-id=0
+
+	secondary process:
+	sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i --rxq=4 --txq=4 --num-procs=2 --proc-id=1
+
+The command line options are:
+
+*   ``-a, --allow``
+
+	Add a device to the allow list. ``xxx`` means device used which should be the same in primary process
+	and secondary process.
+
+*   ``--proc-type``
+	Specify a given process instance as the primary or secondary DPDK instance. ``auto`` set here is ok.
+
+*   ``-l CORELIST``
+	List of cores to run on. the corelist should be different in primary process and secondary process.
+
+*   ``--rxq=N``
+
+    Set the number of RX queues per port to N, where 1 <= N <= 65535.
+    The default value is 1. N is the sum of queues used by primary and secondary process.
+
+*   ``--txq=N``
+
+    Set the number of TX queues per port to N, where 1 <= N <= 65535.
+    The default value is 1. N is the sum of queues used by primary and secondary process.
+
+*   ``--num-procs=N``
+
+	The number of processes which will be used.
+
+*   ``--proc-id=id``
+
+	The id of the current process (id < num-procs). id should be different in primary process and secondary process.
+
+Calculation rule for queue:
+All queues are allocated to different processes based on proc_num and proc_id.
+Calculation rule for the Testpmd to allocate queues to each process:
+start(queue start id) = proc_id * nb_q / num_procs;
+end(queue end id) = start + nb_q / num_procs;
+
+For example, if supports 4 txq and rxq
+the 0~1 for primary process
+the 2~3 for secondary process
+
+Most dev ops is supported in primary and secondary process. While secondary process is not permitted
+to allocate or release shared memory, so some ops are not supported as follows:
+``dev_start``
+``dev_stop``
+``rx_queue_setup``
+``tx_queue_setup``
+``rx_queue_release``
+``tx_queue_release``
+
+RTE_FLOW supported, it applies only on its own process on SW side, but all on HW size.
+stats supported, stats will not change when one quit and start, As they share the same buffer to store the stats.
+RSS supported, Primary process and secondary process has separate queues to use, RSS will work in their own queues whether primary and secondary process.
#35 lore raw

PATCH v4 1/1 app/testpmd: support multi-process

In reply to Lijun Ou

From: Lijun Ou <[email protected]>

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

---
v4:
* Fixed minimum vlaue of Rxq or Txq in doc.

v3:
* Fixed compiling error using gcc10.0.

v2:
* Added document for this patch.

---
 app/test-pmd/cmdline.c                |  12 ++-
 app/test-pmd/config.c                 |   9 ++-
 app/test-pmd/parameters.c             |  11 +++
 app/test-pmd/testpmd.c                | 138 ++++++++++++++++++++++------------
 app/test-pmd/testpmd.h                |   7 ++
 doc/guides/testpmd_app_ug/run_app.rst |  69 +++++++++++++++++
 6 files changed, 196 insertions(+), 50 deletions(-)
app/test-pmd/cmdline.c +10 -2
index 14110eb..287d7a0 100644
@@ -71,8 +71,6 @@
 #include "cmdline_tm.h"
 #include "bpf_cmd.h"
 
-static struct cmdline *testpmd_cl;
-
 static void cmd_reconfig_device_queue(portid_t id, uint8_t dev, uint8_t queue);
 
 /* *** Help command with introduction. *** */
@@ -5351,6 +5349,12 @@ cmd_set_flush_rx_parsed(void *parsed_result,
 		__rte_unused void *data)
 {
 	struct cmd_set_flush_rx *res = parsed_result;
+
+	if (num_procs > 1 && (strcmp(res->mode, "on") == 0)) {
+		printf("multi-process doesn't support to flush rx queues.\n");
+		return;
+	}
+
 	no_flush_rx = (uint8_t)((strcmp(res->mode, "on") == 0) ? 0 : 1);
 }
 
@@ -17227,6 +17231,10 @@ prompt(void)
 		printf("Cannot set exit function for cmdline\n");
 
 	cmdline_interact(testpmd_cl);
+	if (unlikely(f_quit == 1)) {
+		dup2(testpmd_fd_copy, testpmd_cl->s_in);
+		close(testpmd_fd_copy);
+	}
 	if (ret != 0)
 		cmdline_stdin_exit(testpmd_cl);
 }
app/test-pmd/config.c +8 -1
index 576d5ac..9edf84a 100644
@@ -2827,6 +2827,8 @@ rss_fwd_config_setup(void)
 	queueid_t  rxq;
 	queueid_t  nb_q;
 	streamid_t  sm_id;
+	int start;
+	int end;
 
 	nb_q = nb_rxq;
 	if (nb_q > nb_txq)
@@ -2844,7 +2846,10 @@ rss_fwd_config_setup(void)
 	init_fwd_streams();
 
 	setup_fwd_config_of_each_lcore(&cur_fwd_config);
-	rxp = 0; rxq = 0;
+	start = proc_id * nb_q / num_procs;
+	end = start + nb_q / num_procs;
+	rxp = 0;
+	rxq = start;
 	for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
 		struct fwd_stream *fs;
 
@@ -2861,6 +2866,8 @@ rss_fwd_config_setup(void)
 			continue;
 		rxp = 0;
 		rxq++;
+		if (rxq >= end)
+			rxq = start;
 	}
 }
 
app/test-pmd/parameters.c +11 -0
index c8acd5d..1a5bf25 100644
@@ -45,6 +45,8 @@
 #include <rte_flow.h>
 
 #include "testpmd.h"
+#define PARAM_PROC_ID "proc-id"
+#define PARAM_NUM_PROCS "num-procs"
 
 static void
 usage(char* progname)
@@ -605,6 +607,8 @@ launch_args_parse(int argc, char** argv)
 		{ "rx-mq-mode",                 1, 0, 0 },
 		{ "record-core-cycles",         0, 0, 0 },
 		{ "record-burst-stats",         0, 0, 0 },
+		{ PARAM_NUM_PROCS,              1, 0, 0 },
+		{ PARAM_PROC_ID,                1, 0, 0 },
 		{ 0, 0, 0, 0 },
 	};
 
@@ -1366,6 +1370,13 @@ launch_args_parse(int argc, char** argv)
 				record_core_cycles = 1;
 			if (!strcmp(lgopts[opt_idx].name, "record-burst-stats"))
 				record_burst_stats = 1;
+
+			if (strncmp(lgopts[opt_idx].name,
+				    PARAM_NUM_PROCS, 8) == 0)
+				num_procs = atoi(optarg);
+			if (strncmp(lgopts[opt_idx].name,
+				    PARAM_PROC_ID, 7) == 0)
+				proc_id = atoi(optarg);
 			break;
 		case 'h':
 			usage(argv[0]);
app/test-pmd/testpmd.c +91 -47

253 lines not on this page · raw message

app/test-pmd/testpmd.h +7 -0
index ce83f31..6d13417 100644
@@ -13,6 +13,7 @@
 #include <rte_gso.h>
 #include <cmdline.h>
 #include <sys/queue.h>
+#include  "cmdline.h"
 
 #define RTE_PORT_ALL            (~(portid_t)0x0)
 
@@ -24,6 +25,10 @@
 #define RTE_PORT_CLOSED         (uint16_t)2
 #define RTE_PORT_HANDLING       (uint16_t)3
 
+uint8_t f_quit;
+int testpmd_fd_copy;
+struct cmdline *testpmd_cl;
+
 /*
  * It is used to allocate the memory for hash key.
  * The hash key size is NIC dependent.
@@ -422,6 +427,8 @@ extern uint64_t noisy_lkup_mem_sz;
 extern uint64_t noisy_lkup_num_writes;
 extern uint64_t noisy_lkup_num_reads;
 extern uint64_t noisy_lkup_num_reads_writes;
+extern int proc_id;
+extern unsigned int num_procs;
 
 extern uint8_t dcb_config;
 extern uint8_t dcb_test;
doc/guides/testpmd_app_ug/run_app.rst +69 -0

72 lines not on this page · raw message

Signature · 1 line
2.7.4
#36 lore raw
Quoted 18 lines · Min Hu (Connor)

From: Lijun Ou <[email protected]>

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

---
v4:
* Fixed minimum vlaue of Rxq or Txq in doc.

Quoted 5 lines

v3:
* Fixed compiling error using gcc10.0.

v2:
* Added document for this patch.

+Xiaoyun, testpmd maintainer.

#37 lore raw
In reply to Min Hu (Connor)

Hi

Quoted 39 lines

-----Original Message-----
From: dev <[email protected]> On Behalf Of Min Hu (Connor)
Sent: Monday, March 22, 2021 15:07
To: [email protected]
Cc: Yigit, Ferruh <[email protected]>; [email protected]
Subject: [dpdk-dev] [PATCH v4] app/testpmd: support multi-process

From: Lijun Ou <[email protected]>

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

---
v4:
* Fixed minimum vlaue of Rxq or Txq in doc.

v3:
* Fixed compiling error using gcc10.0.

v2:
* Added document for this patch.

---
 app/test-pmd/cmdline.c                |  12 ++-
 app/test-pmd/config.c                 |   9 ++-
 app/test-pmd/parameters.c             |  11 +++
 app/test-pmd/testpmd.c                | 138 ++++++++++++++++++++++------------
 app/test-pmd/testpmd.h                |   7 ++
 doc/guides/testpmd_app_ug/run_app.rst |  69 +++++++++++++++++
 6 files changed, 196 insertions(+), 50 deletions(-)

<snip>

app/test-pmd/testpmd.c mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, · line 1048

Quoted 10 lines
app/test-pmd/testpmd.c +7 -0
+			if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+				rte_mp = rte_pktmbuf_pool_create(pool_name,
+					 nb_mbuf, mb_mempool_cache, 0,
+					 mbuf_seg_size, heap_socket);
+			else
+				rte_mp = rte_mempool_lookup(pool_name);
+
 			break;
 		}
 	case MP_ALLOC_XBUF:

What about this one when users use external bufs? Why not addressing secondary process here?
If it works for all cases, you should add a condition at the start of this function, if it's secondary, goto err to check mp and return.

Quoted 7 lines
app/test-pmd/testpmd.c +3 -0
@@ -1994,6 +2013,12 @@ flush_fwd_rx_queues(void)
 	uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
 	uint64_t timer_period;
 
+	if (num_procs > 1) {
+		printf("multi-process not support for flushing fwd rx "
+		       "queues, skip the below lines and return.\n");

<snip>

app/test-pmd/testpmd.h line 31

Quoted 4 lines
app/test-pmd/testpmd.h +4 -0
+uint8_t f_quit;
+int testpmd_fd_copy;
+struct cmdline *testpmd_cl;
+

Please address the compilation failure on patchwork related to these variables (multiple definitions).

Quoted 10 lines

+.. code-block:: console

doc/guides/testpmd_app_ug/run_app.rst +9 -0
+
+	primary process:
+    sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i --rxq=4
+--txq=4 --num-procs=2 --proc-id=0
+
+	secondary process:
+	sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i --rxq=4
+--txq=4 --num-procs=2 --proc-id=1
+

<snip>

Quoted 6 lines
doc/guides/testpmd_app_ug/run_app.rst +4 -0
+*   ``--rxq=N``
+
+    Set the number of RX queues per port to N, where 1 <= N <= 65535.
+    The default value is 1. N is the sum of queues used by primary and secondary

process.
+

Did you upstream wrong patch?
You said you would address the queue number issue Ajit Khaparde mentioned but you didn't in this patch. The number of queues should be a multiple of the number of processes?

Quoted 6 lines
doc/guides/testpmd_app_ug/run_app.rst +4 -0
+*   ``--txq=N``
+
+    Set the number of TX queues per port to N, where 1 <= N <= 65535.
+    The default value is 1. N is the sum of queues used by primary and secondary

process.
+

Same as above.

Quoted 1 line

+* ``--num-procs=N``

<snip>

Quoted 9 lines
(quoted diff) +2 -0
+Most dev ops is supported in primary and secondary process. While
+secondary process is not permitted to allocate or release shared memory, so

some ops are not supported as follows:
+``dev_start``
+``dev_stop``
+``rx_queue_setup``
+``tx_queue_setup``
+``rx_queue_release``
+``tx_queue_release``

What about some config commands?
Such as "clear port stats all". Should this be allowed by secondary?
And like "port config all rxq". If primary hasn't started ports, should the secondary allowed to change traffic related stuff (offloads, rx/txd, rx/txq and so on)?

Quoted 3 lines
(quoted diff) +2 -0
+
+RTE_FLOW supported, it applies only on its own process on SW side, but all on

HW size.

About rte flow, what do you mean apply only on its own process on SW side?
If I set number-procs=2, rxq=4
Then on secondary process, I set a flow which directs 192.168.0.1 traffic to queue 0. It seems it will directs this kind of traffic to primary process. But I can't see this rule from primary process side. Is this behavior right for multiple process?

Quoted 6 lines

+stats supported, stats will not change when one quit and start, As they share the same buffer to store the stats.
+RSS supported, Primary process and secondary process has separate queues to use, RSS will work in their own queues whether primary and secondary process. --
2.7.4

#38 lore raw
Quoted 42 lines · Xiaoyun

Hi

Quoted 39 lines

-----Original Message-----
From: dev <[email protected]> On Behalf Of Min Hu (Connor)
Sent: Monday, March 22, 2021 15:07
To: [email protected]
Cc: Yigit, Ferruh <[email protected]>; [email protected]
Subject: [dpdk-dev] [PATCH v4] app/testpmd: support multi-process

From: Lijun Ou <[email protected]>

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

---
v4:
* Fixed minimum vlaue of Rxq or Txq in doc.

v3:
* Fixed compiling error using gcc10.0.

v2:
* Added document for this patch.

---
  app/test-pmd/cmdline.c                |  12 ++-
  app/test-pmd/config.c                 |   9 ++-
  app/test-pmd/parameters.c             |  11 +++
  app/test-pmd/testpmd.c                | 138 ++++++++++++++++++++++------------
  app/test-pmd/testpmd.h                |   7 ++
  doc/guides/testpmd_app_ug/run_app.rst |  69 +++++++++++++++++
  6 files changed, 196 insertions(+), 50 deletions(-)

<snip>

Quoted 14 lines

app/test-pmd/testpmd.c mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, · line 1048

Quoted 10 lines
app/test-pmd/testpmd.c +7 -0
+			if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+				rte_mp = rte_pktmbuf_pool_create(pool_name,
+					 nb_mbuf, mb_mempool_cache, 0,
+					 mbuf_seg_size, heap_socket);
+			else
+				rte_mp = rte_mempool_lookup(pool_name);
+
  			break;
  		}
  	case MP_ALLOC_XBUF:

What about this one when users use external bufs? Why not addressing secondary process here?
If it works for all cases, you should add a condition at the start of this function, if it's secondary, goto err to check mp and return.

Yes, your are right, I have fixed it in v5, thanks.

Quoted 9 lines
Quoted 7 lines
app/test-pmd/testpmd.c +3 -0
@@ -1994,6 +2013,12 @@ flush_fwd_rx_queues(void)
  	uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
  	uint64_t timer_period;
 
+	if (num_procs > 1) {
+		printf("multi-process not support for flushing fwd rx "
+		       "queues, skip the below lines and return.\n");

<snip>

Quoted 7 lines

app/test-pmd/testpmd.h line 31

Quoted 4 lines
app/test-pmd/testpmd.h +4 -0
+uint8_t f_quit;
+int testpmd_fd_copy;
+struct cmdline *testpmd_cl;
+

Please address the compilation failure on patchwork related to these variables (multiple definitions).

Done in v5.

Quoted 11 lines
Quoted 10 lines

+.. code-block:: console

doc/guides/testpmd_app_ug/run_app.rst +9 -0
+
+	primary process:
+    sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i --rxq=4
+--txq=4 --num-procs=2 --proc-id=0
+
+	secondary process:
+	sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i --rxq=4
+--txq=4 --num-procs=2 --proc-id=1
+

<snip>

Quoted 11 lines
Quoted 6 lines
doc/guides/testpmd_app_ug/run_app.rst +4 -0
+*   ``--rxq=N``
+
+    Set the number of RX queues per port to N, where 1 <= N <= 65535.
+    The default value is 1. N is the sum of queues used by primary and secondary

process.
+

Did you upstream wrong patch?
You said you would address the queue number issue Ajit Khaparde mentioned but you didn't in this patch. The number of queues should be a multiple of the number of processes?

Done in v5.

Quoted 10 lines
Quoted 6 lines
doc/guides/testpmd_app_ug/run_app.rst +4 -0
+*   ``--txq=N``
+
+    Set the number of TX queues per port to N, where 1 <= N <= 65535.
+    The default value is 1. N is the sum of queues used by primary and secondary

process.
+

Same as above.

Quoted 1 line

+* ``--num-procs=N``

<snip>

Quoted 13 lines
Quoted 9 lines
(quoted diff) +2 -0
+Most dev ops is supported in primary and secondary process. While
+secondary process is not permitted to allocate or release shared memory, so

some ops are not supported as follows:
+``dev_start``
+``dev_stop``
+``rx_queue_setup``
+``tx_queue_setup``
+``rx_queue_release``
+``tx_queue_release``

What about some config commands?
Such as "clear port stats all". Should this be allowed by secondary?

I think so, actually, all the queues is visible to primary and secondary. The only thing we do is to separate queues for different process for io (packets) in Rx/Tx. It is of for secondary "clear port stats all".

Quoted 2 lines

And like "port config all rxq". If primary hasn't started ports, should the secondary allowed to change traffic related stuff (offloads, rx/txd, rx/txq and so on)?

Yes, port config all rxq/txq/rxd/txd/offload is not supported in the secondary process. It has been done in v5.

Quoted 9 lines
Quoted 3 lines
(quoted diff) +2 -0
+
+RTE_FLOW supported, it applies only on its own process on SW side, but all on

HW size.

About rte flow, what do you mean apply only on its own process on SW side?
If I set number-procs=2, rxq=4
Then on secondary process, I set a flow which directs 192.168.0.1 traffic to queue 0. It seems it will directs this kind of traffic to primary process. But I can't see this rule from primary process side. Is this behavior right for multiple process?

According to doc rte_flow.rst, we maintain flow rules in process level:
primary and secondary has its own flow list(but one flow list in HW).
As previously mentioned, the two can see all the queues, so setting the flow rules for the other is OK. Of course, io(receive or transmit packets) in the queue in others is not
permitted.

Quoted 7 lines
Quoted 6 lines

+stats supported, stats will not change when one quit and start, As they share the same buffer to store the stats.
+RSS supported, Primary process and secondary process has separate queues to use, RSS will work in their own queues whether primary and secondary process. --
2.7.4

#39 lore raw
Quoted 141 lines · Min Hu (Connor)
Quoted 53 lines · Xiaoyun

Hi

Quoted 39 lines

-----Original Message-----
From: dev <[email protected]> On Behalf Of Min Hu (Connor)
Sent: Monday, March 22, 2021 15:07
To: [email protected]
Cc: Yigit, Ferruh <[email protected]>; [email protected]
Subject: [dpdk-dev] [PATCH v4] app/testpmd: support multi-process

From: Lijun Ou <[email protected]>

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

---
v4:
* Fixed minimum vlaue of Rxq or Txq in doc.

v3:
* Fixed compiling error using gcc10.0.

v2:
* Added document for this patch.

---
  app/test-pmd/cmdline.c                |  12 ++-
  app/test-pmd/config.c                 |   9 ++-
  app/test-pmd/parameters.c             |  11 +++
  app/test-pmd/testpmd.c                | 138 ++++++++++++++++++++++------------
  app/test-pmd/testpmd.h                |   7 ++
  doc/guides/testpmd_app_ug/run_app.rst |  69 +++++++++++++++++
  6 files changed, 196 insertions(+), 50 deletions(-)

<snip>

app/test-pmd/testpmd.c mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, · line 1048

Quoted 10 lines
app/test-pmd/testpmd.c +7 -0
+                    if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+                            rte_mp = rte_pktmbuf_pool_create(pool_name,
+                                     nb_mbuf, mb_mempool_cache, 0,
+                                     mbuf_seg_size, heap_socket);
+                    else
+                            rte_mp = rte_mempool_lookup(pool_name);
+
                     break;
             }
     case MP_ALLOC_XBUF:
Quoted 3 lines

What about this one when users use external bufs? Why not addressing secondary process here?
If it works for all cases, you should add a condition at the start of this function, if it's secondary, goto err to check mp and return.

Yes, your are right, I have fixed it in v5, thanks.

Quoted 9 lines
Quoted 7 lines
app/test-pmd/testpmd.c +3 -0
@@ -1994,6 +2013,12 @@ flush_fwd_rx_queues(void)
     uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
     uint64_t timer_period;
 
+    if (num_procs > 1) {
+            printf("multi-process not support for flushing fwd rx "
+                   "queues, skip the below lines and return.\n");

<snip>

Quoted 7 lines

app/test-pmd/testpmd.h line 31

Quoted 4 lines
app/test-pmd/testpmd.h +4 -0
+uint8_t f_quit;
+int testpmd_fd_copy;
+struct cmdline *testpmd_cl;
+

Please address the compilation failure on patchwork related to these variables (multiple definitions).

Done in v5.

Quoted 11 lines
Quoted 10 lines

+.. code-block:: console

doc/guides/testpmd_app_ug/run_app.rst +9 -0
+
+    primary process:
+    sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i --rxq=4
+--txq=4 --num-procs=2 --proc-id=0
+
+    secondary process:
+    sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i --rxq=4
+--txq=4 --num-procs=2 --proc-id=1
+

<snip>

Quoted 11 lines
Quoted 6 lines
doc/guides/testpmd_app_ug/run_app.rst +4 -0
+*   ``--rxq=N``
+
+    Set the number of RX queues per port to N, where 1 <= N <= 65535.
+    The default value is 1. N is the sum of queues used by primary and secondary

process.
+

Did you upstream wrong patch?
You said you would address the queue number issue Ajit Khaparde mentioned but you didn't in this patch. The number of queues should be a multiple of the number of processes?

Done in v5.

Quoted 10 lines
Quoted 6 lines
doc/guides/testpmd_app_ug/run_app.rst +4 -0
+*   ``--txq=N``
+
+    Set the number of TX queues per port to N, where 1 <= N <= 65535.
+    The default value is 1. N is the sum of queues used by primary and secondary

process.
+

Same as above.

Quoted 1 line

+* ``--num-procs=N``

<snip>

Quoted 13 lines
Quoted 9 lines
(quoted diff) +2 -0
+Most dev ops is supported in primary and secondary process. While
+secondary process is not permitted to allocate or release shared memory, so

some ops are not supported as follows:
+``dev_start``
+``dev_stop``
+``rx_queue_setup``
+``tx_queue_setup``
+``rx_queue_release``
+``tx_queue_release``

What about some config commands?
Such as "clear port stats all". Should this be allowed by secondary?

I think so, actually, all the queues is visible to primary and secondary. The only thing we do is to separate queues for different process for io (packets) in Rx/Tx. It is of for secondary "clear port stats all".

Quoted 2 lines

And like "port config all rxq". If primary hasn't started ports, should the secondary allowed to change traffic related stuff (offloads, rx/txd, rx/txq and so on)?

Yes, port config all rxq/txq/rxd/txd/offload is not supported in the secondary process. It has been done in v5.

Quoted 9 lines
Quoted 3 lines
(quoted diff) +2 -0
+
+RTE_FLOW supported, it applies only on its own process on SW side, but all on

HW size.

About rte flow, what do you mean apply only on its own process on SW side?
If I set number-procs=2, rxq=4
Then on secondary process, I set a flow which directs 192.168.0.1 traffic to queue 0. It seems it will directs this kind of traffic to primary process. But I can't see this rule from primary process side. Is this behavior right for multiple process?

Quoted 6 lines

According to doc rte_flow.rst, we maintain flow rules in process level:
primary and secondary has its own flow list(but one flow list in HW). As previously mentioned, the two can see all the queues, so setting the flow rules for the other is OK.
Of course, io(receive or transmit packets) in the queue in others is not permitted.

Can you add this behavior as well to the testpmd doc. Further isolation of resources and operations between the primary and secondary processes is possible.
But this is a good start. We can add more if needed.

Quoted 7 lines
Quoted 7 lines
Quoted 6 lines

+stats supported, stats will not change when one quit and start, As they share the same buffer to store the stats.
+RSS supported, Primary process and secondary process has separate queues to use, RSS will work in their own queues whether primary and secondary process. --
2.7.4

#40 lore raw
Quoted 142 lines · Ajit Khaparde
Quoted 141 lines · Min Hu (Connor)
Quoted 53 lines · Xiaoyun

Hi

Quoted 39 lines

-----Original Message-----
From: dev <[email protected]> On Behalf Of Min Hu (Connor)
Sent: Monday, March 22, 2021 15:07
To: [email protected]
Cc: Yigit, Ferruh <[email protected]>; [email protected]
Subject: [dpdk-dev] [PATCH v4] app/testpmd: support multi-process

From: Lijun Ou <[email protected]>

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

---
v4:
* Fixed minimum vlaue of Rxq or Txq in doc.

v3:
* Fixed compiling error using gcc10.0.

v2:
* Added document for this patch.

---
   app/test-pmd/cmdline.c                |  12 ++-
   app/test-pmd/config.c                 |   9 ++-
   app/test-pmd/parameters.c             |  11 +++
   app/test-pmd/testpmd.c                | 138 ++++++++++++++++++++++------------
   app/test-pmd/testpmd.h                |   7 ++
   doc/guides/testpmd_app_ug/run_app.rst |  69 +++++++++++++++++
   6 files changed, 196 insertions(+), 50 deletions(-)

<snip>

app/test-pmd/testpmd.c mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, · line 1048

Quoted 10 lines
app/test-pmd/testpmd.c +7 -0
+                    if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+                            rte_mp = rte_pktmbuf_pool_create(pool_name,
+                                     nb_mbuf, mb_mempool_cache, 0,
+                                     mbuf_seg_size, heap_socket);
+                    else
+                            rte_mp = rte_mempool_lookup(pool_name);
+
                      break;
              }
      case MP_ALLOC_XBUF:
Quoted 3 lines

What about this one when users use external bufs? Why not addressing secondary process here?
If it works for all cases, you should add a condition at the start of this function, if it's secondary, goto err to check mp and return.

Yes, your are right, I have fixed it in v5, thanks.

Quoted 9 lines
Quoted 7 lines
app/test-pmd/testpmd.c +3 -0
@@ -1994,6 +2013,12 @@ flush_fwd_rx_queues(void)
      uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
      uint64_t timer_period;
 
+    if (num_procs > 1) {
+            printf("multi-process not support for flushing fwd rx "
+                   "queues, skip the below lines and return.\n");

<snip>

Quoted 7 lines

app/test-pmd/testpmd.h line 31

Quoted 4 lines
app/test-pmd/testpmd.h +4 -0
+uint8_t f_quit;
+int testpmd_fd_copy;
+struct cmdline *testpmd_cl;
+

Please address the compilation failure on patchwork related to these variables (multiple definitions).

Done in v5.

Quoted 11 lines
Quoted 10 lines

+.. code-block:: console

doc/guides/testpmd_app_ug/run_app.rst +9 -0
+
+    primary process:
+    sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i --rxq=4
+--txq=4 --num-procs=2 --proc-id=0
+
+    secondary process:
+    sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i --rxq=4
+--txq=4 --num-procs=2 --proc-id=1
+

<snip>

Quoted 11 lines
Quoted 6 lines
doc/guides/testpmd_app_ug/run_app.rst +4 -0
+*   ``--rxq=N``
+
+    Set the number of RX queues per port to N, where 1 <= N <= 65535.
+    The default value is 1. N is the sum of queues used by primary and secondary

process.
+

Did you upstream wrong patch?
You said you would address the queue number issue Ajit Khaparde mentioned but you didn't in this patch. The number of queues should be a multiple of the number of processes?

Done in v5.

Quoted 10 lines
Quoted 6 lines
doc/guides/testpmd_app_ug/run_app.rst +4 -0
+*   ``--txq=N``
+
+    Set the number of TX queues per port to N, where 1 <= N <= 65535.
+    The default value is 1. N is the sum of queues used by primary and secondary

process.
+

Same as above.

Quoted 1 line

+* ``--num-procs=N``

<snip>

Quoted 13 lines
Quoted 9 lines
(quoted diff) +2 -0
+Most dev ops is supported in primary and secondary process. While
+secondary process is not permitted to allocate or release shared memory, so

some ops are not supported as follows:
+``dev_start``
+``dev_stop``
+``rx_queue_setup``
+``tx_queue_setup``
+``rx_queue_release``
+``tx_queue_release``

What about some config commands?
Such as "clear port stats all". Should this be allowed by secondary?

I think so, actually, all the queues is visible to primary and secondary. The only thing we do is to separate queues for different process for io (packets) in Rx/Tx. It is of for secondary "clear port stats all".

Quoted 2 lines

And like "port config all rxq". If primary hasn't started ports, should the secondary allowed to change traffic related stuff (offloads, rx/txd, rx/txq and so on)?

Yes, port config all rxq/txq/rxd/txd/offload is not supported in the secondary process. It has been done in v5.

Quoted 9 lines
Quoted 3 lines
(quoted diff) +2 -0
+
+RTE_FLOW supported, it applies only on its own process on SW side, but all on

HW size.

About rte flow, what do you mean apply only on its own process on SW side?
If I set number-procs=2, rxq=4
Then on secondary process, I set a flow which directs 192.168.0.1 traffic to queue 0. It seems it will directs this kind of traffic to primary process. But I can't see this rule from primary process side. Is this behavior right for multiple process?

Quoted 8 lines
Quoted 6 lines

According to doc rte_flow.rst, we maintain flow rules in process level:
primary and secondary has its own flow list(but one flow list in HW). As previously mentioned, the two can see all the queues, so setting the flow rules for the other is OK.
Of course, io(receive or transmit packets) in the queue in others is not permitted.

Can you add this behavior as well to the testpmd doc.

Hi, Ajit, done in v6, thanks.

Quoted 11 lines

Further isolation of resources and operations between the primary and secondary processes is possible.
But this is a good start. We can add more if needed.

Quoted 7 lines
Quoted 7 lines
Quoted 6 lines

+stats supported, stats will not change when one quit and start, As they share the same buffer to store the stats.
+RSS supported, Primary process and secondary process has separate queues to use, RSS will work in their own queues whether primary and secondary process. --
2.7.4

#41 lore raw

PATCH v5 1/1 app/testpmd: support multi-process

In reply to Lijun Ou

From: Lijun Ou <[email protected]>

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

---
v5:
* Fixed run_app.rst for multiple process description.
* Fix compiling error.

v4:
* Fixed minimum vlaue of Rxq or Txq in doc.

v3:
* Fixed compiling error using gcc10.0.

v2:
* Added document for this patch.

---
 app/test-pmd/cmdline.c                |  12 +++-
 app/test-pmd/config.c                 |   9 ++-
 app/test-pmd/parameters.c             |  11 +++
 app/test-pmd/testpmd.c                | 127 ++++++++++++++++++++++------------
 app/test-pmd/testpmd.h                |   7 ++
 doc/guides/testpmd_app_ug/run_app.rst |  95 +++++++++++++++++++++++++
 6 files changed, 215 insertions(+), 46 deletions(-)
app/test-pmd/cmdline.c +10 -2
index 14110eb..287d7a0 100644
@@ -71,8 +71,6 @@
 #include "cmdline_tm.h"
 #include "bpf_cmd.h"
 
-static struct cmdline *testpmd_cl;
-
 static void cmd_reconfig_device_queue(portid_t id, uint8_t dev, uint8_t queue);
 
 /* *** Help command with introduction. *** */
@@ -5351,6 +5349,12 @@ cmd_set_flush_rx_parsed(void *parsed_result,
 		__rte_unused void *data)
 {
 	struct cmd_set_flush_rx *res = parsed_result;
+
+	if (num_procs > 1 && (strcmp(res->mode, "on") == 0)) {
+		printf("multi-process doesn't support to flush rx queues.\n");
+		return;
+	}
+
 	no_flush_rx = (uint8_t)((strcmp(res->mode, "on") == 0) ? 0 : 1);
 }
 
@@ -17227,6 +17231,10 @@ prompt(void)
 		printf("Cannot set exit function for cmdline\n");
 
 	cmdline_interact(testpmd_cl);
+	if (unlikely(f_quit == 1)) {
+		dup2(testpmd_fd_copy, testpmd_cl->s_in);
+		close(testpmd_fd_copy);
+	}
 	if (ret != 0)
 		cmdline_stdin_exit(testpmd_cl);
 }
app/test-pmd/config.c +8 -1
index ef0b978..e48cbd9 100644
@@ -2827,6 +2827,8 @@ rss_fwd_config_setup(void)
 	queueid_t  rxq;
 	queueid_t  nb_q;
 	streamid_t  sm_id;
+	int start;
+	int end;
 
 	nb_q = nb_rxq;
 	if (nb_q > nb_txq)
@@ -2844,7 +2846,10 @@ rss_fwd_config_setup(void)
 	init_fwd_streams();
 
 	setup_fwd_config_of_each_lcore(&cur_fwd_config);
-	rxp = 0; rxq = 0;
+	start = proc_id * nb_q / num_procs;
+	end = start + nb_q / num_procs;
+	rxp = 0;
+	rxq = start;
 	for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
 		struct fwd_stream *fs;
 
@@ -2861,6 +2866,8 @@ rss_fwd_config_setup(void)
 			continue;
 		rxp = 0;
 		rxq++;
+		if (rxq >= end)
+			rxq = start;
 	}
 }
 
app/test-pmd/parameters.c +11 -0
index a326c8c..ec3bc62 100644
@@ -45,6 +45,8 @@
 #include <rte_flow.h>
 
 #include "testpmd.h"
+#define PARAM_PROC_ID "proc-id"
+#define PARAM_NUM_PROCS "num-procs"
 
 static void
 usage(char* progname)
@@ -644,6 +646,8 @@ launch_args_parse(int argc, char** argv)
 		{ "rx-mq-mode",                 1, 0, 0 },
 		{ "record-core-cycles",         0, 0, 0 },
 		{ "record-burst-stats",         0, 0, 0 },
+		{ PARAM_NUM_PROCS,              1, 0, 0 },
+		{ PARAM_PROC_ID,                1, 0, 0 },
 		{ 0, 0, 0, 0 },
 	};
 
@@ -1410,6 +1414,13 @@ launch_args_parse(int argc, char** argv)
 				record_core_cycles = 1;
 			if (!strcmp(lgopts[opt_idx].name, "record-burst-stats"))
 				record_burst_stats = 1;
+
+			if (strncmp(lgopts[opt_idx].name,
+				    PARAM_NUM_PROCS, 8) == 0)
+				num_procs = atoi(optarg);
+			if (strncmp(lgopts[opt_idx].name,
+				    PARAM_PROC_ID, 7) == 0)
+				proc_id = atoi(optarg);
 			break;
 		case 'h':
 			usage(argv[0]);
app/test-pmd/testpmd.c +84 -43

238 lines not on this page · raw message

app/test-pmd/testpmd.h +7 -0
index a87ccb0..640a377 100644
@@ -13,6 +13,7 @@
 #include <rte_gso.h>
 #include <cmdline.h>
 #include <sys/queue.h>
+#include  "cmdline.h"
 
 #define RTE_PORT_ALL            (~(portid_t)0x0)
 
@@ -24,6 +25,10 @@
 #define RTE_PORT_CLOSED         (uint16_t)2
 #define RTE_PORT_HANDLING       (uint16_t)3
 
+extern uint8_t f_quit;
+extern int testpmd_fd_copy;
+extern struct cmdline *testpmd_cl;
+
 /*
  * It is used to allocate the memory for hash key.
  * The hash key size is NIC dependent.
@@ -423,6 +428,8 @@ extern uint64_t noisy_lkup_mem_sz;
 extern uint64_t noisy_lkup_num_writes;
 extern uint64_t noisy_lkup_num_reads;
 extern uint64_t noisy_lkup_num_reads_writes;
+extern int proc_id;
+extern unsigned int num_procs;
 
 extern uint8_t dcb_config;
 extern uint8_t dcb_test;
doc/guides/testpmd_app_ug/run_app.rst +95 -0

98 lines not on this page · raw message

Signature · 1 line
2.7.4
#42 lore raw

PATCH v6 1/1 app/testpmd: support multi-process

In reply to Lijun Ou

From: Lijun Ou <[email protected]>

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

---
v6:
* Add rte flow description for multiple process.

v5:
* Fixed run_app.rst for multiple process description.
* Fix compiling error.

v4:
* Fixed minimum vlaue of Rxq or Txq in doc.

v3:
* Fixed compiling error using gcc10.0.

v2:
* Added document for this patch.

---
 app/test-pmd/cmdline.c                |  12 +++-
 app/test-pmd/config.c                 |   9 ++-
 app/test-pmd/parameters.c             |  11 +++
 app/test-pmd/testpmd.c                | 127 ++++++++++++++++++++++------------
 app/test-pmd/testpmd.h                |   7 ++
 doc/guides/testpmd_app_ug/run_app.rst |  99 ++++++++++++++++++++++++++
 6 files changed, 219 insertions(+), 46 deletions(-)
app/test-pmd/cmdline.c +10 -2
index 14110eb..287d7a0 100644
@@ -71,8 +71,6 @@
 #include "cmdline_tm.h"
 #include "bpf_cmd.h"
 
-static struct cmdline *testpmd_cl;
-
 static void cmd_reconfig_device_queue(portid_t id, uint8_t dev, uint8_t queue);
 
 /* *** Help command with introduction. *** */
@@ -5351,6 +5349,12 @@ cmd_set_flush_rx_parsed(void *parsed_result,
 		__rte_unused void *data)
 {
 	struct cmd_set_flush_rx *res = parsed_result;
+
+	if (num_procs > 1 && (strcmp(res->mode, "on") == 0)) {
+		printf("multi-process doesn't support to flush rx queues.\n");
+		return;
+	}
+
 	no_flush_rx = (uint8_t)((strcmp(res->mode, "on") == 0) ? 0 : 1);
 }
 
@@ -17227,6 +17231,10 @@ prompt(void)
 		printf("Cannot set exit function for cmdline\n");
 
 	cmdline_interact(testpmd_cl);
+	if (unlikely(f_quit == 1)) {
+		dup2(testpmd_fd_copy, testpmd_cl->s_in);
+		close(testpmd_fd_copy);
+	}
 	if (ret != 0)
 		cmdline_stdin_exit(testpmd_cl);
 }
app/test-pmd/config.c +8 -1
index ef0b978..e48cbd9 100644
@@ -2827,6 +2827,8 @@ rss_fwd_config_setup(void)
 	queueid_t  rxq;
 	queueid_t  nb_q;
 	streamid_t  sm_id;
+	int start;
+	int end;
 
 	nb_q = nb_rxq;
 	if (nb_q > nb_txq)
@@ -2844,7 +2846,10 @@ rss_fwd_config_setup(void)
 	init_fwd_streams();
 
 	setup_fwd_config_of_each_lcore(&cur_fwd_config);
-	rxp = 0; rxq = 0;
+	start = proc_id * nb_q / num_procs;
+	end = start + nb_q / num_procs;
+	rxp = 0;
+	rxq = start;
 	for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
 		struct fwd_stream *fs;
 
@@ -2861,6 +2866,8 @@ rss_fwd_config_setup(void)
 			continue;
 		rxp = 0;
 		rxq++;
+		if (rxq >= end)
+			rxq = start;
 	}
 }
 
app/test-pmd/parameters.c +11 -0
index a326c8c..ec3bc62 100644
@@ -45,6 +45,8 @@
 #include <rte_flow.h>
 
 #include "testpmd.h"
+#define PARAM_PROC_ID "proc-id"
+#define PARAM_NUM_PROCS "num-procs"
 
 static void
 usage(char* progname)
@@ -644,6 +646,8 @@ launch_args_parse(int argc, char** argv)
 		{ "rx-mq-mode",                 1, 0, 0 },
 		{ "record-core-cycles",         0, 0, 0 },
 		{ "record-burst-stats",         0, 0, 0 },
+		{ PARAM_NUM_PROCS,              1, 0, 0 },
+		{ PARAM_PROC_ID,                1, 0, 0 },
 		{ 0, 0, 0, 0 },
 	};
 
@@ -1410,6 +1414,13 @@ launch_args_parse(int argc, char** argv)
 				record_core_cycles = 1;
 			if (!strcmp(lgopts[opt_idx].name, "record-burst-stats"))
 				record_burst_stats = 1;
+
+			if (strncmp(lgopts[opt_idx].name,
+				    PARAM_NUM_PROCS, 8) == 0)
+				num_procs = atoi(optarg);
+			if (strncmp(lgopts[opt_idx].name,
+				    PARAM_PROC_ID, 7) == 0)
+				proc_id = atoi(optarg);
 			break;
 		case 'h':
 			usage(argv[0]);
app/test-pmd/testpmd.c +84 -43

238 lines not on this page · raw message

app/test-pmd/testpmd.h +7 -0
index a87ccb0..640a377 100644
@@ -13,6 +13,7 @@
 #include <rte_gso.h>
 #include <cmdline.h>
 #include <sys/queue.h>
+#include  "cmdline.h"
 
 #define RTE_PORT_ALL            (~(portid_t)0x0)
 
@@ -24,6 +25,10 @@
 #define RTE_PORT_CLOSED         (uint16_t)2
 #define RTE_PORT_HANDLING       (uint16_t)3
 
+extern uint8_t f_quit;
+extern int testpmd_fd_copy;
+extern struct cmdline *testpmd_cl;
+
 /*
  * It is used to allocate the memory for hash key.
  * The hash key size is NIC dependent.
@@ -423,6 +428,8 @@ extern uint64_t noisy_lkup_mem_sz;
 extern uint64_t noisy_lkup_num_writes;
 extern uint64_t noisy_lkup_num_reads;
 extern uint64_t noisy_lkup_num_reads_writes;
+extern int proc_id;
+extern unsigned int num_procs;
 
 extern uint8_t dcb_config;
 extern uint8_t dcb_test;
doc/guides/testpmd_app_ug/run_app.rst +99 -0

102 lines not on this page · raw message

Signature · 1 line
2.7.4
#43 lore raw

PATCH v7 1/1 app/testpmd: support multi-process

In reply to Lijun Ou

From: Lijun Ou <[email protected]>

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

---
v7:
* Fixed compiling error for unexpected unindent.

v6:
* Add rte flow description for multiple process.

v5:
* Fixed run_app.rst for multiple process description.
* Fix compiling error.

v4:
* Fixed minimum vlaue of Rxq or Txq in doc.

v3:
* Fixed compiling error using gcc10.0.

v2:
* Added document for this patch.

---
 app/test-pmd/cmdline.c                |  12 +++-
 app/test-pmd/config.c                 |   9 ++-
 app/test-pmd/parameters.c             |  11 +++
 app/test-pmd/testpmd.c                | 127 ++++++++++++++++++++++------------
 app/test-pmd/testpmd.h                |   7 ++
 doc/guides/testpmd_app_ug/run_app.rst |  99 ++++++++++++++++++++++++++
 6 files changed, 219 insertions(+), 46 deletions(-)
app/test-pmd/cmdline.c +10 -2
index 14110eb..287d7a0 100644
@@ -71,8 +71,6 @@
 #include "cmdline_tm.h"
 #include "bpf_cmd.h"
 
-static struct cmdline *testpmd_cl;
-
 static void cmd_reconfig_device_queue(portid_t id, uint8_t dev, uint8_t queue);
 
 /* *** Help command with introduction. *** */
@@ -5351,6 +5349,12 @@ cmd_set_flush_rx_parsed(void *parsed_result,
 		__rte_unused void *data)
 {
 	struct cmd_set_flush_rx *res = parsed_result;
+
+	if (num_procs > 1 && (strcmp(res->mode, "on") == 0)) {
+		printf("multi-process doesn't support to flush rx queues.\n");
+		return;
+	}
+
 	no_flush_rx = (uint8_t)((strcmp(res->mode, "on") == 0) ? 0 : 1);
 }
 
@@ -17227,6 +17231,10 @@ prompt(void)
 		printf("Cannot set exit function for cmdline\n");
 
 	cmdline_interact(testpmd_cl);
+	if (unlikely(f_quit == 1)) {
+		dup2(testpmd_fd_copy, testpmd_cl->s_in);
+		close(testpmd_fd_copy);
+	}
 	if (ret != 0)
 		cmdline_stdin_exit(testpmd_cl);
 }
app/test-pmd/config.c +8 -1
index ef0b978..e48cbd9 100644
@@ -2827,6 +2827,8 @@ rss_fwd_config_setup(void)
 	queueid_t  rxq;
 	queueid_t  nb_q;
 	streamid_t  sm_id;
+	int start;
+	int end;
 
 	nb_q = nb_rxq;
 	if (nb_q > nb_txq)
@@ -2844,7 +2846,10 @@ rss_fwd_config_setup(void)
 	init_fwd_streams();
 
 	setup_fwd_config_of_each_lcore(&cur_fwd_config);
-	rxp = 0; rxq = 0;
+	start = proc_id * nb_q / num_procs;
+	end = start + nb_q / num_procs;
+	rxp = 0;
+	rxq = start;
 	for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
 		struct fwd_stream *fs;
 
@@ -2861,6 +2866,8 @@ rss_fwd_config_setup(void)
 			continue;
 		rxp = 0;
 		rxq++;
+		if (rxq >= end)
+			rxq = start;
 	}
 }
 
app/test-pmd/parameters.c +11 -0
index a326c8c..ec3bc62 100644
@@ -45,6 +45,8 @@
 #include <rte_flow.h>
 
 #include "testpmd.h"
+#define PARAM_PROC_ID "proc-id"
+#define PARAM_NUM_PROCS "num-procs"
 
 static void
 usage(char* progname)
@@ -644,6 +646,8 @@ launch_args_parse(int argc, char** argv)
 		{ "rx-mq-mode",                 1, 0, 0 },
 		{ "record-core-cycles",         0, 0, 0 },
 		{ "record-burst-stats",         0, 0, 0 },
+		{ PARAM_NUM_PROCS,              1, 0, 0 },
+		{ PARAM_PROC_ID,                1, 0, 0 },
 		{ 0, 0, 0, 0 },
 	};
 
@@ -1410,6 +1414,13 @@ launch_args_parse(int argc, char** argv)
 				record_core_cycles = 1;
 			if (!strcmp(lgopts[opt_idx].name, "record-burst-stats"))
 				record_burst_stats = 1;
+
+			if (strncmp(lgopts[opt_idx].name,
+				    PARAM_NUM_PROCS, 8) == 0)
+				num_procs = atoi(optarg);
+			if (strncmp(lgopts[opt_idx].name,
+				    PARAM_PROC_ID, 7) == 0)
+				proc_id = atoi(optarg);
 			break;
 		case 'h':
 			usage(argv[0]);
app/test-pmd/testpmd.c +84 -43

238 lines not on this page · raw message

app/test-pmd/testpmd.h +7 -0
index a87ccb0..640a377 100644
@@ -13,6 +13,7 @@
 #include <rte_gso.h>
 #include <cmdline.h>
 #include <sys/queue.h>
+#include  "cmdline.h"
 
 #define RTE_PORT_ALL            (~(portid_t)0x0)
 
@@ -24,6 +25,10 @@
 #define RTE_PORT_CLOSED         (uint16_t)2
 #define RTE_PORT_HANDLING       (uint16_t)3
 
+extern uint8_t f_quit;
+extern int testpmd_fd_copy;
+extern struct cmdline *testpmd_cl;
+
 /*
  * It is used to allocate the memory for hash key.
  * The hash key size is NIC dependent.
@@ -423,6 +428,8 @@ extern uint64_t noisy_lkup_mem_sz;
 extern uint64_t noisy_lkup_num_writes;
 extern uint64_t noisy_lkup_num_reads;
 extern uint64_t noisy_lkup_num_reads_writes;
+extern int proc_id;
+extern unsigned int num_procs;
 
 extern uint8_t dcb_config;
 extern uint8_t dcb_test;
doc/guides/testpmd_app_ug/run_app.rst +99 -0

102 lines not on this page · raw message

Signature · 1 line
2.7.4
#44 lore raw

Hi

app/test-pmd/cmdline.c

Quoted 55 lines

-----Original Message-----
From: Min Hu (Connor) <[email protected]>
Sent: Friday, March 26, 2021 16:53
To: [email protected]
Cc: Yigit, Ferruh <[email protected]>; [email protected]; Li, Xiaoyun <[email protected]>
Subject: [PATCH v7] app/testpmd: support multi-process

From: Lijun Ou <[email protected]>

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

---
v7:
* Fixed compiling error for unexpected unindent.

v6:
* Add rte flow description for multiple process.

v5:
* Fixed run_app.rst for multiple process description.
* Fix compiling error.

v4:
* Fixed minimum vlaue of Rxq or Txq in doc.

v3:
* Fixed compiling error using gcc10.0.

v2:
* Added document for this patch.

---
 app/test-pmd/cmdline.c                |  12 +++-
 app/test-pmd/config.c                 |   9 ++-
 app/test-pmd/parameters.c             |  11 +++
 app/test-pmd/testpmd.c                | 127 ++++++++++++++++++++++------------
 app/test-pmd/testpmd.h                |   7 ++
 doc/guides/testpmd_app_ug/run_app.rst |  99 ++++++++++++++++++++++++++
 6 files changed, 219 insertions(+), 46 deletions(-)
app/test-pmd/cmdline.c index renamed from app/test-pmd/cmdline.c +0 -0

14110eb..287d7a0 100644

app/test-pmd/cmdline.c +0 -0
@@ -71,8 +71,6 @@

Why not just restrict queue numbers to be multiple of processes here in rss_fwd_config_setup()? Or at least print a warning to tell users if it's not multiple, packet loss will happen.

Quoted 21 lines
app/test-pmd/cmdline.c +9 -1
 	setup_fwd_config_of_each_lcore(&cur_fwd_config);
-	rxp = 0; rxq = 0;
+	start = proc_id * nb_q / num_procs;
+	end = start + nb_q / num_procs;
+	rxp = 0;
+	rxq = start;
 	for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
 		struct fwd_stream *fs;
@@ -2861,6 +2866,8 @@ rss_fwd_config_setup(void)
 			continue;
 		rxp = 0;
 		rxq++;
+		if (rxq >= end)
+			rxq = start;
 	}
 }
 
+*   ``--txq=N``
+
+    Set the number of Tx queues per port to N. N is the sum of queues used by
Quoted 5 lines

primary
+ and secondary process. As primary process and secondary process should
have separate
+ queues, and each should occupy at least one queue.where N should be no
less than two.

Where N should be the multiple of number of processes.

BRs
Xiaoyun

Quoted 2 lines

+
2.7.4

#45 lore raw
Quoted 52 lines · Xiaoyun

Hi

Quoted 50 lines

-----Original Message-----
From: Min Hu (Connor) <[email protected]>
Sent: Friday, March 26, 2021 16:53
To: [email protected]
Cc: Yigit, Ferruh <[email protected]>; [email protected]; Li, Xiaoyun <[email protected]>
Subject: [PATCH v7] app/testpmd: support multi-process

From: Lijun Ou <[email protected]>

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

---
v7:
* Fixed compiling error for unexpected unindent.

v6:
* Add rte flow description for multiple process.

v5:
* Fixed run_app.rst for multiple process description.
* Fix compiling error.

v4:
* Fixed minimum vlaue of Rxq or Txq in doc.

v3:
* Fixed compiling error using gcc10.0.

v2:
* Added document for this patch.

---
  app/test-pmd/cmdline.c                |  12 +++-
  app/test-pmd/config.c                 |   9 ++-
  app/test-pmd/parameters.c             |  11 +++
  app/test-pmd/testpmd.c                | 127 ++++++++++++++++++++++------------
  app/test-pmd/testpmd.h                |   7 ++
  doc/guides/testpmd_app_ug/run_app.rst |  99 ++++++++++++++++++++++++++
  6 files changed, 219 insertions(+), 46 deletions(-)
Quoted 9 lines

app/test-pmd/cmdline.c

Quoted 5 lines
app/test-pmd/cmdline.c index renamed from app/test-pmd/cmdline.c +0 -0

14110eb..287d7a0 100644

app/test-pmd/cmdline.c +0 -0
@@ -71,8 +71,6 @@

Why not just restrict queue numbers to be multiple of processes here in rss_fwd_config_setup()? Or at least print a warning to tell users if it's not multiple, packet loss will happen.

Hi, fixed in v8, thanks.

Quoted 18 lines
Quoted 18 lines
app/test-pmd/cmdline.c +6 -1
  	setup_fwd_config_of_each_lcore(&cur_fwd_config);
-	rxp = 0; rxq = 0;
+	start = proc_id * nb_q / num_procs;
+	end = start + nb_q / num_procs;
+	rxp = 0;
+	rxq = start;
  	for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
  		struct fwd_stream *fs;
@@ -2861,6 +2866,8 @@ rss_fwd_config_setup(void)
  			continue;
  		rxp = 0;
  		rxq++;
+		if (rxq >= end)
+			rxq = start;
  	}
  }
 
Quoted 11 lines
Quoted 8 lines
app/test-pmd/cmdline.c +3 -0
+*   ``--txq=N``
+
+    Set the number of Tx queues per port to N. N is the sum of queues used by

primary
+ and secondary process. As primary process and secondary process should
have separate
+ queues, and each should occupy at least one queue.where N should be no
less than two.

Where N should be the multiple of number of processes.

Fixed in v8, thanks.

Quoted 5 lines

BRs
Xiaoyun

Quoted 2 lines

+
2.7.4

#46 lore raw

PATCH v8 1/1 app/testpmd: support multi-process

In reply to Lijun Ou

From: Lijun Ou <[email protected]>

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

---
v8:
* Added warning info about queue numbers and process numbers.

v7:
* Fixed compiling error for unexpected unindent.

v6:
* Add rte flow description for multiple process.

v5:
* Fixed run_app.rst for multiple process description.
* Fix compiling error.

v4:
* Fixed minimum vlaue of Rxq or Txq in doc.

v3:
* Fixed compiling error using gcc10.0.

v2:
* Added document for this patch.

---
 app/test-pmd/cmdline.c                |  12 +++-
 app/test-pmd/config.c                 |  14 +++-
 app/test-pmd/parameters.c             |  11 +++
 app/test-pmd/testpmd.c                | 127 ++++++++++++++++++++++------------
 app/test-pmd/testpmd.h                |   7 ++
 doc/guides/testpmd_app_ug/run_app.rst | 101 +++++++++++++++++++++++++++
 6 files changed, 226 insertions(+), 46 deletions(-)
app/test-pmd/cmdline.c +10 -2
index f44116b..38c97e3 100644
@@ -71,8 +71,6 @@
 #include "cmdline_tm.h"
 #include "bpf_cmd.h"
 
-static struct cmdline *testpmd_cl;
-
 static void cmd_reconfig_device_queue(portid_t id, uint8_t dev, uint8_t queue);
 
 /* *** Help command with introduction. *** */
@@ -5351,6 +5349,12 @@ cmd_set_flush_rx_parsed(void *parsed_result,
 		__rte_unused void *data)
 {
 	struct cmd_set_flush_rx *res = parsed_result;
+
+	if (num_procs > 1 && (strcmp(res->mode, "on") == 0)) {
+		printf("multi-process doesn't support to flush rx queues.\n");
+		return;
+	}
+
 	no_flush_rx = (uint8_t)((strcmp(res->mode, "on") == 0) ? 0 : 1);
 }
 
@@ -17229,6 +17233,10 @@ prompt(void)
 		printf("Cannot set exit function for cmdline\n");
 
 	cmdline_interact(testpmd_cl);
+	if (unlikely(f_quit == 1)) {
+		dup2(testpmd_fd_copy, testpmd_cl->s_in);
+		close(testpmd_fd_copy);
+	}
 	if (ret != 0)
 		cmdline_stdin_exit(testpmd_cl);
 }
app/test-pmd/config.c +13 -1
index ef0b978..ec7d030 100644
@@ -2827,6 +2827,8 @@ rss_fwd_config_setup(void)
 	queueid_t  rxq;
 	queueid_t  nb_q;
 	streamid_t  sm_id;
+	int start;
+	int end;
 
 	nb_q = nb_rxq;
 	if (nb_q > nb_txq)
@@ -2844,7 +2846,15 @@ rss_fwd_config_setup(void)
 	init_fwd_streams();
 
 	setup_fwd_config_of_each_lcore(&cur_fwd_config);
-	rxp = 0; rxq = 0;
+
+	if (proc_id > 0 && nb_q % num_procs)
+		printf("Warning! queue numbers should be multiple of "
+			"processes, or packet loss will happen.\n");
+
+	start = proc_id * nb_q / num_procs;
+	end = start + nb_q / num_procs;
+	rxp = 0;
+	rxq = start;
 	for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
 		struct fwd_stream *fs;
 
@@ -2861,6 +2871,8 @@ rss_fwd_config_setup(void)
 			continue;
 		rxp = 0;
 		rxq++;
+		if (rxq >= end)
+			rxq = start;
 	}
 }
 
app/test-pmd/parameters.c +11 -0
index a326c8c..ec3bc62 100644
@@ -45,6 +45,8 @@
 #include <rte_flow.h>
 
 #include "testpmd.h"
+#define PARAM_PROC_ID "proc-id"
+#define PARAM_NUM_PROCS "num-procs"
 
 static void
 usage(char* progname)
@@ -644,6 +646,8 @@ launch_args_parse(int argc, char** argv)
 		{ "rx-mq-mode",                 1, 0, 0 },
 		{ "record-core-cycles",         0, 0, 0 },
 		{ "record-burst-stats",         0, 0, 0 },
+		{ PARAM_NUM_PROCS,              1, 0, 0 },
+		{ PARAM_PROC_ID,                1, 0, 0 },
 		{ 0, 0, 0, 0 },
 	};
 
@@ -1410,6 +1414,13 @@ launch_args_parse(int argc, char** argv)
 				record_core_cycles = 1;
 			if (!strcmp(lgopts[opt_idx].name, "record-burst-stats"))
 				record_burst_stats = 1;
+
+			if (strncmp(lgopts[opt_idx].name,
+				    PARAM_NUM_PROCS, 8) == 0)
+				num_procs = atoi(optarg);
+			if (strncmp(lgopts[opt_idx].name,
+				    PARAM_PROC_ID, 7) == 0)
+				proc_id = atoi(optarg);
 			break;
 		case 'h':
 			usage(argv[0]);
app/test-pmd/testpmd.c +84 -43

238 lines not on this page · raw message

app/test-pmd/testpmd.h +7 -0
index a87ccb0..640a377 100644
@@ -13,6 +13,7 @@
 #include <rte_gso.h>
 #include <cmdline.h>
 #include <sys/queue.h>
+#include  "cmdline.h"
 
 #define RTE_PORT_ALL            (~(portid_t)0x0)
 
@@ -24,6 +25,10 @@
 #define RTE_PORT_CLOSED         (uint16_t)2
 #define RTE_PORT_HANDLING       (uint16_t)3
 
+extern uint8_t f_quit;
+extern int testpmd_fd_copy;
+extern struct cmdline *testpmd_cl;
+
 /*
  * It is used to allocate the memory for hash key.
  * The hash key size is NIC dependent.
@@ -423,6 +428,8 @@ extern uint64_t noisy_lkup_mem_sz;
 extern uint64_t noisy_lkup_num_writes;
 extern uint64_t noisy_lkup_num_reads;
 extern uint64_t noisy_lkup_num_reads_writes;
+extern int proc_id;
+extern unsigned int num_procs;
 
 extern uint8_t dcb_config;
 extern uint8_t dcb_test;
doc/guides/testpmd_app_ug/run_app.rst +101 -0

104 lines not on this page · raw message

Signature · 1 line
2.7.4
Acked-by on v8 1/1
#47 lore raw

Hi

Quoted 53 lines

-----Original Message-----
From: Min Hu (Connor) <[email protected]>
Sent: Tuesday, March 30, 2021 09:48
To: [email protected]
Cc: Yigit, Ferruh <[email protected]>; [email protected]; Li, Xiaoyun <[email protected]>
Subject: [PATCH v8] app/testpmd: support multi-process

From: Lijun Ou <[email protected]>

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

---
v8:
* Added warning info about queue numbers and process numbers.

v7:
* Fixed compiling error for unexpected unindent.

v6:
* Add rte flow description for multiple process.

v5:
* Fixed run_app.rst for multiple process description.
* Fix compiling error.

v4:
* Fixed minimum vlaue of Rxq or Txq in doc.

v3:
* Fixed compiling error using gcc10.0.

v2:
* Added document for this patch.

---
 app/test-pmd/cmdline.c                |  12 +++-
 app/test-pmd/config.c                 |  14 +++-
 app/test-pmd/parameters.c             |  11 +++
 app/test-pmd/testpmd.c                | 127 ++++++++++++++++++++++------------
 app/test-pmd/testpmd.h                |   7 ++
 doc/guides/testpmd_app_ug/run_app.rst | 101
+++++++++++++++++++++++++++
 6 files changed, 226 insertions(+), 46 deletions(-)
Quoted 1 line

2.7.4

Many commands shouldn't be allowed but now only have a brief guide in doc. This can be done in the future I think. Overall, it looks good to me.

#48 lore raw
Quoted 57 lines · Xiaoyun

Hi

Quoted 53 lines

-----Original Message-----
From: Min Hu (Connor) <[email protected]>
Sent: Tuesday, March 30, 2021 09:48
To: [email protected]
Cc: Yigit, Ferruh <[email protected]>; [email protected]; Li, Xiaoyun <[email protected]>
Subject: [PATCH v8] app/testpmd: support multi-process

From: Lijun Ou <[email protected]>

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

---
v8:
* Added warning info about queue numbers and process numbers.

v7:
* Fixed compiling error for unexpected unindent.

v6:
* Add rte flow description for multiple process.

v5:
* Fixed run_app.rst for multiple process description.
* Fix compiling error.

v4:
* Fixed minimum vlaue of Rxq or Txq in doc.

v3:
* Fixed compiling error using gcc10.0.

v2:
* Added document for this patch.

---
  app/test-pmd/cmdline.c                |  12 +++-
  app/test-pmd/config.c                 |  14 +++-
  app/test-pmd/parameters.c             |  11 +++
  app/test-pmd/testpmd.c                | 127 ++++++++++++++++++++++------------
  app/test-pmd/testpmd.h                |   7 ++
  doc/guides/testpmd_app_ug/run_app.rst | 101
+++++++++++++++++++++++++++
  6 files changed, 226 insertions(+), 46 deletions(-)
Quoted 1 line

2.7.4

Quoted 3 lines

Many commands shouldn't be allowed but now only have a brief guide in doc. This can be done in the future I think. Overall, it looks good to me.

Thanks xiaoyun.
You are right, Maybe there are more things to do about supporting multiple process for testpmd. We will perfect it continuously in future.

Quoted 2 lines
Acked-by on v8 1/1
#49 lore raw
In reply to Min Hu (Connor)
Quoted 9 lines · Min Hu (Connor)

From: Lijun Ou <[email protected]>

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

Quoted 6 lines

the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

Some minor nits below. Otherwise looks fine to me.

Quoted 4 lines

---
v8:
* Added warning info about queue numbers and process numbers.

:::snip::::

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 595

Quoted 6 lines
doc/guides/testpmd_app_ug/run_app.rst +6 -0
+*   ``--rxq=N``
+
+    Set the number of Rx queues per port to N. N is the sum of queues used by primary
+    and secondary process. Primary process and secondary process should have separate
+    queues, and each should occupy at least one queue. Where N should be the multiple
+    of number of processes.

of the number of processes.

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 602

Quoted 7 lines
doc/guides/testpmd_app_ug/run_app.rst +7 -0
+
+*   ``--txq=N``
+
+    Set the number of Tx queues per port to N. N is the sum of queues used by primary
+    and secondary process. Primary process and secondary process should have separate
+    queues, and each should occupy at least one queue. Where N should be the multiple
+    of number of processes.

of the number of processes.

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 607

Quoted 5 lines
doc/guides/testpmd_app_ug/run_app.rst +5 -0
+
+*   ``--num-procs=N``
+
+    The number of processes which will be used.
+

:::: snip ::::

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 628

Quoted 6 lines
doc/guides/testpmd_app_ug/run_app.rst +6 -0
+The number of rings should be a multiple of the number of processes. If not,
+redundant queues will exist after queues are allocated to processes. After RSS is
+enabled, packet loss occurs when traffic is sent to all processes at the same time.
+Some traffic enters redundant queues and cannot be forwarded.
+
+Most dev ops is supported in primary and secondary process. While secondary process

Most dev ops are supported in the primary and secondary process. While....

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 637

Quoted 9 lines
doc/guides/testpmd_app_ug/run_app.rst +9 -0
+is not permitted to allocate or release shared memory, so some ops are not supported
+as follows:
+``dev_configure``
+``dev_start``
+``dev_stop``
+``rx_queue_setup``
+``tx_queue_setup``
+``rx_queue_release``
+``tx_queue_release``

:::: snip:::

#50 lore raw
Quoted 10 lines · Ajit Khaparde
Quoted 9 lines · Min Hu (Connor)

From: Lijun Ou <[email protected]>

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

Quoted 9 lines
Quoted 6 lines

the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

Some minor nits below. Otherwise looks fine to me.

Thanks Ajit.

Hi, Ferruh,
Should I send v9 to fix the grammar bugs in doc which Ajit point
out or fix it in future?

Quoted 48 lines
Quoted 4 lines

---
v8:
* Added warning info about queue numbers and process numbers.

:::snip::::

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 595

Quoted 6 lines
doc/guides/testpmd_app_ug/run_app.rst +6 -0
+*   ``--rxq=N``
+
+    Set the number of Rx queues per port to N. N is the sum of queues used by primary
+    and secondary process. Primary process and secondary process should have separate
+    queues, and each should occupy at least one queue. Where N should be the multiple
+    of number of processes.

of the number of processes.

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 602

Quoted 7 lines
doc/guides/testpmd_app_ug/run_app.rst +7 -0
+
+*   ``--txq=N``
+
+    Set the number of Tx queues per port to N. N is the sum of queues used by primary
+    and secondary process. Primary process and secondary process should have separate
+    queues, and each should occupy at least one queue. Where N should be the multiple
+    of number of processes.

of the number of processes.

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 607

Quoted 5 lines
doc/guides/testpmd_app_ug/run_app.rst +5 -0
+
+*   ``--num-procs=N``
+
+    The number of processes which will be used.
+

:::: snip ::::

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 628

Quoted 6 lines
doc/guides/testpmd_app_ug/run_app.rst +6 -0
+The number of rings should be a multiple of the number of processes. If not,
+redundant queues will exist after queues are allocated to processes. After RSS is
+enabled, packet loss occurs when traffic is sent to all processes at the same time.
+Some traffic enters redundant queues and cannot be forwarded.
+
+Most dev ops is supported in primary and secondary process. While secondary process

Most dev ops are supported in the primary and secondary process. While....

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 637

Quoted 9 lines
doc/guides/testpmd_app_ug/run_app.rst +9 -0
+is not permitted to allocate or release shared memory, so some ops are not supported
+as follows:
+``dev_configure``
+``dev_start``
+``dev_stop``
+``rx_queue_setup``
+``tx_queue_setup``
+``rx_queue_release``
+``tx_queue_release``

:::: snip:::

#51 lore raw
Quoted 19 lines · Min Hu (Connor)
Quoted 16 lines · Ajit Khaparde
Quoted 15 lines · Min Hu (Connor)

From: Lijun Ou <[email protected]>

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

Quoted 9 lines
Quoted 3 lines

Some minor nits below. Otherwise looks fine to me.

Thanks Ajit.

Hi, Ferruh,
    Should I send v9 to fix the grammar bugs in doc which Ajit point
out or fix it in future?

Hi Connor, if they are only outstanding issues, I can fix them while merging.

Quoted 58 lines
Quoted 58 lines
Quoted 4 lines

---
v8:
* Added warning info about queue numbers and process numbers.

:::snip::::

Quoted 9 lines
doc/guides/testpmd_app_ug/run_app.rst +3 -0
+*   ``--rxq=N``
+
+    Set the number of Rx queues per port to N. N is the sum of queues used 

by primary
+    and secondary process. Primary process and secondary process should have separate
+    queues, and each should occupy at least one queue. Where N should be the multiple
+    of number of processes.

of the number of processes.

Quoted 10 lines
doc/guides/testpmd_app_ug/run_app.rst +4 -0
+
+*   ``--txq=N``
+
+    Set the number of Tx queues per port to N. N is the sum of queues used 

by primary
+    and secondary process. Primary process and secondary process should have separate
+    queues, and each should occupy at least one queue. Where N should be the multiple
+    of number of processes.

of the number of processes.

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 607

Quoted 5 lines
doc/guides/testpmd_app_ug/run_app.rst +5 -0
+
+*   ``--num-procs=N``
+
+    The number of processes which will be used.
+

:::: snip ::::

Quoted 9 lines
(quoted diff) +2 -0
+The number of rings should be a multiple of the number of processes. If not,
+redundant queues will exist after queues are allocated to processes. After 

RSS is
+enabled, packet loss occurs when traffic is sent to all processes at the same time.

doc/guides/testpmd_app_ug/run_app.rst +3 -0
+Some traffic enters redundant queues and cannot be forwarded.
+
+Most dev ops is supported in primary and secondary process. While secondary 

process

Most dev ops are supported in the primary and secondary process. While....

Quoted 10 lines

+is not permitted to allocate or release shared memory, so some ops are not supported
+as follows:
+``dev_configure``
+``dev_start``
+``dev_stop``
+``rx_queue_setup``
+``tx_queue_setup``
+``rx_queue_release``
+``tx_queue_release``

:::: snip:::

#52 lore raw
Quoted 23 lines · Ferruh Yigit
Quoted 22 lines · Min Hu (Connor)
Quoted 19 lines · Ajit Khaparde
Quoted 9 lines · Min Hu (Connor)

From: Lijun Ou <[email protected]>

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

Quoted 6 lines

the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

Some minor nits below. Otherwise looks fine to me.

Quoted 9 lines
Quoted 6 lines

Thanks Ajit.

Hi, Ferruh,
     Should I send v9 to fix the grammar bugs in doc which Ajit point out or fix it in future?

Hi Connor, if they are only outstanding issues, I can fix them while merging.

Thanks Ferruh.

Quoted 61 lines
Quoted 47 lines
Quoted 47 lines
Quoted 4 lines

---
v8:
* Added warning info about queue numbers and process numbers.

:::snip::::

Quoted 9 lines
doc/guides/testpmd_app_ug/run_app.rst +3 -0
+*   ``--rxq=N``
+
+    Set the number of Rx queues per port to N. N is the sum of 

queues used by primary
+    and secondary process. Primary process and secondary process should have separate
+    queues, and each should occupy at least one queue. Where N should be the multiple
+    of number of processes.

of the number of processes.

Quoted 10 lines
doc/guides/testpmd_app_ug/run_app.rst +4 -0
+
+*   ``--txq=N``
+
+    Set the number of Tx queues per port to N. N is the sum of 

queues used by primary
+    and secondary process. Primary process and secondary process should have separate
+    queues, and each should occupy at least one queue. Where N should be the multiple
+    of number of processes.

of the number of processes.

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 607

Quoted 5 lines
doc/guides/testpmd_app_ug/run_app.rst +5 -0
+
+*   ``--num-procs=N``
+
+    The number of processes which will be used.
+

:::: snip ::::

Quoted 10 lines

+The number of rings should be a multiple of the number of processes. If not,
+redundant queues will exist after queues are allocated to processes. After RSS is
+enabled, packet loss occurs when traffic is sent to all processes at the same time.

doc/guides/testpmd_app_ug/run_app.rst +3 -0
+Some traffic enters redundant queues and cannot be forwarded.
+
+Most dev ops is supported in primary and secondary process. While 

secondary process

Most dev ops are supported in the primary and secondary process. While....

Quoted 12 lines
Quoted 12 lines
Quoted 10 lines

+is not permitted to allocate or release shared memory, so some ops are not supported
+as follows:
+``dev_configure``
+``dev_start``
+``dev_stop``
+``rx_queue_setup``
+``tx_queue_setup``
+``rx_queue_release``
+``tx_queue_release``

:::: snip:::

.

Acked-by Xiaoyun Li Acked-by Ajit Khaparde
#53 lore raw

Hi, Ferry and all,
This patch has been acked:

Are there any other comments about that patch?

Quoted 98 lines · Min Hu (Connor)
Quoted 23 lines · Ferruh Yigit
Quoted 22 lines · Min Hu (Connor)
Quoted 19 lines · Ajit Khaparde
Quoted 9 lines · Min Hu (Connor)

From: Lijun Ou <[email protected]>

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

Quoted 6 lines

the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

Some minor nits below. Otherwise looks fine to me.

Quoted 9 lines
Quoted 6 lines

Thanks Ajit.

Hi, Ferruh,
     Should I send v9 to fix the grammar bugs in doc which Ajit point out or fix it in future?

Hi Connor, if they are only outstanding issues, I can fix them while merging.

Thanks Ferruh.

Quoted 60 lines
Quoted 47 lines
Quoted 47 lines
Quoted 4 lines

---
v8:
* Added warning info about queue numbers and process numbers.

:::snip::::

Quoted 9 lines
doc/guides/testpmd_app_ug/run_app.rst +3 -0
+*   ``--rxq=N``
+
+    Set the number of Rx queues per port to N. N is the sum of 

queues used by primary
+    and secondary process. Primary process and secondary process should have separate
+    queues, and each should occupy at least one queue. Where N should be the multiple
+    of number of processes.

of the number of processes.

Quoted 10 lines
doc/guides/testpmd_app_ug/run_app.rst +4 -0
+
+*   ``--txq=N``
+
+    Set the number of Tx queues per port to N. N is the sum of 

queues used by primary
+    and secondary process. Primary process and secondary process should have separate
+    queues, and each should occupy at least one queue. Where N should be the multiple
+    of number of processes.

of the number of processes.

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 607

Quoted 5 lines
doc/guides/testpmd_app_ug/run_app.rst +5 -0
+
+*   ``--num-procs=N``
+
+    The number of processes which will be used.
+

:::: snip ::::

Quoted 10 lines

+The number of rings should be a multiple of the number of processes. If not,
+redundant queues will exist after queues are allocated to processes. After RSS is
+enabled, packet loss occurs when traffic is sent to all processes at the same time.

doc/guides/testpmd_app_ug/run_app.rst +3 -0
+Some traffic enters redundant queues and cannot be forwarded.
+
+Most dev ops is supported in primary and secondary process. While 

secondary process

Most dev ops are supported in the primary and secondary process. While....

Quoted 12 lines
Quoted 12 lines
Quoted 10 lines

+is not permitted to allocate or release shared memory, so some ops are not supported
+as follows:
+``dev_configure``
+``dev_start``
+``dev_stop``
+``rx_queue_setup``
+``tx_queue_setup``
+``rx_queue_release``
+``tx_queue_release``

:::: snip:::

Quoted 1 line

.

.

#54 lore raw
Quoted 7 lines · Min Hu (Connor)

Hi, Ferry and all,
    This patch has been acked:

    Are there any other comments about that patch?

Hi Connor,

In today's release status meeting I asked if more parties can test this patch, Lets wait a little more for it to be tested.

Quoted 99 lines · Min Hu (Connor)
Quoted 97 lines · Ferruh Yigit
Quoted 23 lines · Min Hu (Connor)
Quoted 22 lines · Ajit Khaparde
Quoted 19 lines · Min Hu (Connor)
Quoted 9 lines

From: Lijun Ou <[email protected]>

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

Quoted 6 lines

the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

Some minor nits below. Otherwise looks fine to me.

Quoted 9 lines
Quoted 6 lines

Thanks Ajit.

Hi, Ferruh,
     Should I send v9 to fix the grammar bugs in doc which Ajit point out or fix it in future?

Hi Connor, if they are only outstanding issues, I can fix them while merging.

Thanks Ferruh.

Quoted 59 lines
Quoted 46 lines
Quoted 46 lines
Quoted 4 lines

---
v8:
* Added warning info about queue numbers and process numbers.

:::snip::::

Quoted 9 lines
doc/guides/testpmd_app_ug/run_app.rst +3 -0
+*   ``--rxq=N``
+
+    Set the number of Rx queues per port to N. N is the sum of queues 

used by primary
+    and secondary process. Primary process and secondary process should have separate
+    queues, and each should occupy at least one queue. Where N should be the multiple
+    of number of processes.

of the number of processes.

Quoted 10 lines
doc/guides/testpmd_app_ug/run_app.rst +4 -0
+
+*   ``--txq=N``
+
+    Set the number of Tx queues per port to N. N is the sum of queues 

used by primary
+    and secondary process. Primary process and secondary process should have separate
+    queues, and each should occupy at least one queue. Where N should be the multiple
+    of number of processes.

of the number of processes.

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 607

Quoted 5 lines
doc/guides/testpmd_app_ug/run_app.rst +5 -0
+
+*   ``--num-procs=N``
+
+    The number of processes which will be used.
+

:::: snip ::::

Quoted 9 lines
(quoted diff) +2 -0
+The number of rings should be a multiple of the number of processes. If not,
+redundant queues will exist after queues are allocated to processes. 

After RSS is
+enabled, packet loss occurs when traffic is sent to all processes at the same time.

doc/guides/testpmd_app_ug/run_app.rst +3 -0
+Some traffic enters redundant queues and cannot be forwarded.
+
+Most dev ops is supported in primary and secondary process. While 

secondary process

Most dev ops are supported in the primary and secondary process. While....

Quoted 12 lines
Quoted 12 lines
Quoted 10 lines

+is not permitted to allocate or release shared memory, so some ops are not supported
+as follows:
+``dev_configure``
+``dev_start``
+``dev_stop``
+``rx_queue_setup``
+``tx_queue_setup``
+``rx_queue_release``
+``tx_queue_release``

:::: snip:::

Quoted 1 line

.

.

#55 lore raw
Quoted 6 lines · Ferruh Yigit
Quoted 5 lines · Min Hu (Connor)

Hi, Ferry and all,
     This patch has been acked:

Quoted 8 lines
Quoted 2 lines

     Are there any other comments about that patch?

Hi Connor,

In today's release status meeting I asked if more parties can test this patch, Lets wait a little more for it to be tested.

OK, thanks Ferruh.

Quoted 102 lines
Quoted 98 lines · Min Hu (Connor)
Quoted 96 lines · Ferruh Yigit
Quoted 23 lines · Min Hu (Connor)
Quoted 22 lines · Ajit Khaparde
Quoted 19 lines · Min Hu (Connor)
Quoted 9 lines

From: Lijun Ou <[email protected]>

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

Quoted 6 lines

the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

Some minor nits below. Otherwise looks fine to me.

Quoted 9 lines
Quoted 6 lines

Thanks Ajit.

Hi, Ferruh,
     Should I send v9 to fix the grammar bugs in doc which Ajit point out or fix it in future?

Hi Connor, if they are only outstanding issues, I can fix them while merging.

Thanks Ferruh.

Quoted 60 lines
Quoted 59 lines
Quoted 59 lines
Quoted 4 lines

---
v8:
* Added warning info about queue numbers and process numbers.

:::snip::::

Quoted 9 lines
(quoted diff) +3 -0
+*   ``--rxq=N``
+
+    Set the number of Rx queues per port to N. N is the sum of 

queues used by primary
+    and secondary process. Primary process and secondary process should have separate
+    queues, and each should occupy at least one queue. Where N should be the multiple
+    of number of processes.

of the number of processes.

Quoted 10 lines
(quoted diff) +4 -0
+
+*   ``--txq=N``
+
+    Set the number of Tx queues per port to N. N is the sum of 

queues used by primary
+    and secondary process. Primary process and secondary process should have separate
+    queues, and each should occupy at least one queue. Where N should be the multiple
+    of number of processes.

of the number of processes.

Review

Quoted 5 lines
(quoted diff) +5 -0
+
+*   ``--num-procs=N``
+
+    The number of processes which will be used.
+

:::: snip ::::

Quoted 10 lines

+The number of rings should be a multiple of the number of processes. If not,
+redundant queues will exist after queues are allocated to processes. After RSS is
+enabled, packet loss occurs when traffic is sent to all processes at the same time.

(quoted diff) +3 -0
+Some traffic enters redundant queues and cannot be forwarded.
+
+Most dev ops is supported in primary and secondary process. 

While secondary process

Most dev ops are supported in the primary and secondary process. While....

Quoted 10 lines

+is not permitted to allocate or release shared memory, so some ops are not supported
+as follows:
+``dev_configure``
+``dev_start``
+``dev_stop``
+``rx_queue_setup``
+``tx_queue_setup``
+``rx_queue_release``
+``tx_queue_release``

:::: snip:::

Quoted 2 lines
Quoted 2 lines
Quoted 1 line

.

.

.

#56 lore raw
In reply to Min Hu (Connor)
Quoted 8 lines · Min Hu (Connor)

From: Lijun Ou <[email protected]>

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

Quoted 4 lines

the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

Hi Connor,

Please find a few minor comments below, since they are minor issues please feel free to keep the existing acks in next version.

Quoted 24 lines

---
v8:
* Added warning info about queue numbers and process numbers.

v7:
* Fixed compiling error for unexpected unindent.

v6:
* Add rte flow description for multiple process.

v5:
* Fixed run_app.rst for multiple process description.
* Fix compiling error.

v4:
* Fixed minimum vlaue of Rxq or Txq in doc.

v3:
* Fixed compiling error using gcc10.0.

v2:
* Added document for this patch.

Quoted 7 lines
---
  app/test-pmd/cmdline.c                |  12 +++-
  app/test-pmd/config.c                 |  14 +++-
  app/test-pmd/parameters.c             |  11 +++
  app/test-pmd/testpmd.c                | 127 ++++++++++++++++++++++------------
  app/test-pmd/testpmd.h                |   7 ++
  doc/guides/testpmd_app_ug/run_app.rst | 101 +++++++++++++++++++++++++++

Can you please add a release notes update too?

Quoted 29 lines
  6 files changed, 226 insertions(+), 46 deletions(-)
app/test-pmd/cmdline.c +6 -2
index f44116b..38c97e3 100644
@@ -71,8 +71,6 @@
  #include "cmdline_tm.h"
  #include "bpf_cmd.h"
 
-static struct cmdline *testpmd_cl;
-
  static void cmd_reconfig_device_queue(portid_t id, uint8_t dev, uint8_t queue);
 
  /* *** Help command with introduction. *** */
@@ -5351,6 +5349,12 @@ cmd_set_flush_rx_parsed(void *parsed_result,
  		__rte_unused void *data)
  {
  	struct cmd_set_flush_rx *res = parsed_result;
+
+	if (num_procs > 1 && (strcmp(res->mode, "on") == 0)) {
+		printf("multi-process doesn't support to flush rx queues.\n");
+		return;
+	}
+
  	no_flush_rx = (uint8_t)((strcmp(res->mode, "on") == 0) ? 0 : 1);
  }
 

app/test-pmd/cmdline.c prompt(void) · line 17239

Quoted 7 lines
app/test-pmd/cmdline.c +4 -0
@@ -17229,6 +17233,10 @@ prompt(void)
  		printf("Cannot set exit function for cmdline\n");
 
  	cmdline_interact(testpmd_cl);
+	if (unlikely(f_quit == 1)) {
+		dup2(testpmd_fd_copy, testpmd_cl->s_in);
+		close(testpmd_fd_copy);
+	}

Why this is needed? Can you put some comment on the code to explain why this is needed?

Quoted 47 lines
app/test-pmd/cmdline.c +0 -0
  	if (ret != 0)
  		cmdline_stdin_exit(testpmd_cl);
  }
app/test-pmd/config.c +13 -1
index ef0b978..ec7d030 100644
@@ -2827,6 +2827,8 @@ rss_fwd_config_setup(void)
  	queueid_t  rxq;
  	queueid_t  nb_q;
  	streamid_t  sm_id;
+	int start;
+	int end;
 
  	nb_q = nb_rxq;
  	if (nb_q > nb_txq)
@@ -2844,7 +2846,15 @@ rss_fwd_config_setup(void)
  	init_fwd_streams();
 
  	setup_fwd_config_of_each_lcore(&cur_fwd_config);
-	rxp = 0; rxq = 0;
+
+	if (proc_id > 0 && nb_q % num_procs)
+		printf("Warning! queue numbers should be multiple of "
+			"processes, or packet loss will happen.\n");
+
+	start = proc_id * nb_q / num_procs;
+	end = start + nb_q / num_procs;
+	rxp = 0;
+	rxq = start;
  	for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
  		struct fwd_stream *fs;
 
@@ -2861,6 +2871,8 @@ rss_fwd_config_setup(void)
  			continue;
  		rxp = 0;
  		rxq++;
+		if (rxq >= end)
+			rxq = start;
  	}
  }
 
app/test-pmd/parameters.c +0 -0
index a326c8c..ec3bc62 100644

app/test-pmd/parameters.c line 50

Quoted 6 lines
app/test-pmd/parameters.c +2 -0
@@ -45,6 +45,8 @@
  #include <rte_flow.h>
 
  #include "testpmd.h"
+#define PARAM_PROC_ID "proc-id"
+#define PARAM_NUM_PROCS "num-procs"
 

Macros just after include is not good, can you please move these just after 'launch_args_parse()' to keep locality.

Quoted 12 lines
app/test-pmd/parameters.c +2 -0
  static void
  usage(char* progname)
@@ -644,6 +646,8 @@ launch_args_parse(int argc, char** argv)
  		{ "rx-mq-mode",                 1, 0, 0 },
  		{ "record-core-cycles",         0, 0, 0 },
  		{ "record-burst-stats",         0, 0, 0 },
+		{ PARAM_NUM_PROCS,              1, 0, 0 },
+		{ PARAM_PROC_ID,                1, 0, 0 },
  		{ 0, 0, 0, 0 },
  	};
 

app/test-pmd/parameters.c launch_args_parse(int argc, char** argv) · line 1419

Quoted 6 lines
app/test-pmd/parameters.c +3 -0
@@ -1410,6 +1414,13 @@ launch_args_parse(int argc, char** argv)
  				record_core_cycles = 1;
  			if (!strcmp(lgopts[opt_idx].name, "record-burst-stats"))
  				record_burst_stats = 1;
+
+			if (strncmp(lgopts[opt_idx].name,
+				    PARAM_NUM_PROCS, 8) == 0)

9?
"num-procs"
123456789

Quoted 12 lines
app/test-pmd/parameters.c +4 -0
+				num_procs = atoi(optarg);
+			if (strncmp(lgopts[opt_idx].name,
+				    PARAM_PROC_ID, 7) == 0)
+				proc_id = atoi(optarg);
  			break;
  		case 'h':
  			usage(argv[0]);
app/test-pmd/testpmd.c +0 -0
index 96d2e0f..c31234e 100644

app/test-pmd/testpmd.c line 68

Quoted 6 lines
app/test-pmd/testpmd.c +3 -0
@@ -63,6 +63,9 @@
 
  #include "testpmd.h"
 
+int testpmd_fd_copy; /* the copy of STDIN_FILENO */
+struct cmdline *testpmd_cl;
+

Again the global variables just after include is not good place.

What do you think to keep 'testpmd_cl' in the 'cmdline.c', since the object allocated there and the variable seems match more to that file. 'extern' already added to header, so it can be used here.

'testpmd_fd_copy' can be moved at the bottom of the block that has global variables, and please add more comment on what this variable is for.

app/test-pmd/testpmd.c uint8_t port_numa[RTE_MAX_ETHPORTS]; · line 133

Quoted 10 lines
app/test-pmd/testpmd.c +3 -0
  #ifndef MAP_HUGETLB
  /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
  #define HUGE_FLAG (0x40000)
@@ -125,6 +128,9 @@ uint8_t port_numa[RTE_MAX_ETHPORTS];
   */
  uint8_t rxring_numa[RTE_MAX_ETHPORTS];
 
+int proc_id;
+unsigned int num_procs = 1;
+

Similar comments for these global variables.

The testpmd is already complex, it is good to have multi process support for the testpmd but the concern is additional complexity it bring, it may lead very tangled senarious. Please add much more comment on the multi process related additions.

There is a block at the beggining of the this file, that has all global variables with enough comments on them, can we follow same for these new global variables. Group all multiprocess related variables together, appent to the global variables block, and comment all the variables in detail?

Quoted 127 lines
app/test-pmd/testpmd.c +51 -24
  /*
   * Store specified sockets on which TX ring to be used by ports
   * is allocated.
@@ -977,6 +983,11 @@ mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
  	mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
  	mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx);
 
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+		rte_mp = rte_mempool_lookup(pool_name);
+		goto err;
+	}
+
  	TESTPMD_LOG(INFO,
  		"create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
  		pool_name, nb_mbuf, mbuf_seg_size, socket_id);
@@ -1059,9 +1070,14 @@ mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 
  err:
  	if (rte_mp == NULL) {
-		rte_exit(EXIT_FAILURE,
-			"Creation of mbuf pool for socket %u failed: %s\n",
-			socket_id, rte_strerror(rte_errno));
+		if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+			rte_exit(EXIT_FAILURE,
+				"Creation of mbuf pool for socket %u failed: %s\n",
+				socket_id, rte_strerror(rte_errno));
+		else
+			rte_exit(EXIT_FAILURE,
+				"Get mbuf pool for socket %u failed: %s\n",
+				socket_id, rte_strerror(rte_errno));
  	} else if (verbose_level > 0) {
  		rte_mempool_dump(stdout, rte_mp);
  	}
@@ -2002,6 +2018,12 @@ flush_fwd_rx_queues(void)
  	uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
  	uint64_t timer_period;
 
+	if (num_procs > 1) {
+		printf("multi-process not support for flushing fwd rx "
+		       "queues, skip the below lines and return.\n");
+		return;
+	}
+
  	/* convert to number of cycles */
  	timer_period = rte_get_timer_hz(); /* 1 second timeout */
 
@@ -2511,21 +2533,28 @@ start_port(portid_t pid)
  				return -1;
  			}
  			/* configure port */
-			diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
+			if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+				diag = rte_eth_dev_configure(pi,
+						     nb_rxq + nb_hairpinq,
  						     nb_txq + nb_hairpinq,
  						     &(port->dev_conf));
-			if (diag != 0) {
-				if (rte_atomic16_cmpset(&(port->port_status),
-				RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
-					printf("Port %d can not be set back "
-							"to stopped\n", pi);
-				printf("Fail to configure port %d\n", pi);
-				/* try to reconfigure port next time */
-				port->need_reconfig = 1;
-				return -1;
+				if (diag != 0) {
+					if (rte_atomic16_cmpset(
+							&(port->port_status),
+							RTE_PORT_HANDLING,
+							RTE_PORT_STOPPED) == 0)
+						printf("Port %d can not be set "
+						       "back to stopped\n", pi);
+					printf("Fail to configure port %d\n",
+						pi);
+					/* try to reconfigure port next time */
+					port->need_reconfig = 1;
+					return -1;
+				}
  			}
  		}
-		if (port->need_reconfig_queues > 0) {
+		if (port->need_reconfig_queues > 0 &&
+		    rte_eal_process_type() == RTE_PROC_PRIMARY) {
  			port->need_reconfig_queues = 0;
  			/* setup tx queues */
  			for (qi = 0; qi < nb_txq; qi++) {
@@ -2626,17 +2655,20 @@ start_port(portid_t pid)
  		cnt_pi++;
 
  		/* start port */
-		diag = rte_eth_dev_start(pi);
-		if (diag < 0) {
-			printf("Fail to start port %d: %s\n", pi,
-			       rte_strerror(-diag));
+		if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+			diag = rte_eth_dev_start(pi);
+			if (diag < 0) {
+				printf("Fail to start port %d: %s\n", pi,
+				       rte_strerror(-diag));
 
-			/* Fail to setup rx queue, return */
-			if (rte_atomic16_cmpset(&(port->port_status),
+				/* Fail to setup rx queue, return */
+				if (rte_atomic16_cmpset(&(port->port_status),
  				RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
-				printf("Port %d can not be set back to "
-							"stopped\n", pi);
-			continue;
+					printf("Port %d can not be set back to "
+								"stopped\n",
+						pi);
+				continue;
+			}
  		}
 
  		if (rte_atomic16_cmpset(&(port->port_status),
@@ -2765,7 +2797,8 @@ stop_port(portid_t pid)
  		if (port->flow_list)
  			port_flow_flush(pi);
 
-		if (rte_eth_dev_stop(pi) != 0)
+		if (rte_eal_process_type() == RTE_PROC_PRIMARY &&
+		    rte_eth_dev_stop(pi) != 0)
  			RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n",
  				pi);
 

app/test-pmd/testpmd.c close_port(portid_t pid) · line 2873

Quoted 9 lines
app/test-pmd/testpmd.c +4 -2
@@ -2834,8 +2867,10 @@ close_port(portid_t pid)
  			continue;
  		}
 
-		port_flow_flush(pi);
-		rte_eth_dev_close(pi);
+		if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+			port_flow_flush(pi);
+		if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+			rte_eth_dev_close(pi);

Can combine two calls to prevent redundant check.

Quoted 45 lines
app/test-pmd/testpmd.c +5 -10
  	}
 
  	remove_invalid_ports();
@@ -3099,7 +3134,7 @@ pmd_test_exit(void)
  		}
  	}
  	for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
-		if (mempools[i])
+		if (rte_eal_process_type() == RTE_PROC_PRIMARY && mempools[i])
  			rte_mempool_free(mempools[i]);
  	}
 
@@ -3621,6 +3656,10 @@ init_port_dcb_config(portid_t pid,
  	int retval;
  	uint16_t i;
 
+	if (num_procs > 1) {
+		printf("The multi-process feature doesn't support dcb.\n");
+		return -ENOTSUP;
+	}
  	rte_port = &ports[pid];
 
  	memset(&port_conf, 0, sizeof(struct rte_eth_conf));
@@ -3719,13 +3758,6 @@ init_port(void)
  }
 
  static void
-force_quit(void)
-{
-	pmd_test_exit();
-	prompt_exit();
-}
-
-static void
  print_stats(void)
  {
  	uint8_t i;
@@ -3756,12 +3788,16 @@ signal_handler(int signum)
  		if (latencystats_enabled != 0)
  			rte_latencystats_uninit();
  #endif
-		force_quit();
  		/* Set flag to indicate the force termination. */
  		f_quit = 1;
-		/* exit with the expected status */

app/test-pmd/testpmd.c signal_handler(int signum) · line 3800

Quoted 10 lines
app/test-pmd/testpmd.c +8 -2
-		signal(signum, SIG_DFL);
-		kill(getpid(), signum);
+		if (interactive == 1) {
+			dup2(testpmd_cl->s_in, testpmd_fd_copy);
+			close(testpmd_cl->s_in);
+		} else {
+			dup2(0, testpmd_fd_copy);
+			close(0);
+		}
+

Similarly can you please comment the code why this is needed.

Quoted 27 lines
app/test-pmd/testpmd.c +3 -5
  	}
  }
 
@@ -3786,10 +3822,6 @@ main(int argc, char** argv)
  		rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
  			 rte_strerror(rte_errno));
 
-	if (rte_eal_process_type() == RTE_PROC_SECONDARY)
-		rte_exit(EXIT_FAILURE,
-			 "Secondary process type not supported.\n");
-
  	ret = register_eth_event_callback();
  	if (ret != 0)
  		rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
@@ -3885,8 +3917,10 @@ main(int argc, char** argv)
  		}
  	}
 
-	if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
+	if (!no_device_start && start_port(RTE_PORT_ALL) != 0) {
+		pmd_test_exit();
  		rte_exit(EXIT_FAILURE, "Start ports failed\n");
+	}
 
  	/* set all ports to promiscuous mode by default */
  	RTE_ETH_FOREACH_DEV(port_id) {

app/test-pmd/testpmd.c main(int argc, char** argv) · line 3970

Quoted 5 lines
app/test-pmd/testpmd.c +2 -0
@@ -3932,6 +3966,8 @@ main(int argc, char** argv)
  		}
  		prompt();
  		pmd_test_exit();
+		if (unlikely(f_quit == 1))
+			prompt_exit();

Why additional prompt_exit() is required?

app/test-pmd/testpmd.c main(int argc, char** argv) · line 4010

Quoted 12 lines
app/test-pmd/testpmd.c +5 -0
  	} else
  #endif
  	{
@@ -3967,6 +4003,11 @@ main(int argc, char** argv)
  		printf("Press enter to exit\n");
  		rc = read(0, &c, 1);
  		pmd_test_exit();
+		if (unlikely(f_quit == 1)) {
+			dup2(testpmd_fd_copy, 0);
+			close(testpmd_fd_copy);
+			prompt_exit();
+		}

By definition 'f_quit', force quit, is added to support exiting in container environment, since they can't exit with SIGINT/SIGTERM, so main thing with this global variable should be breaking the loop in main function. I wonder why this variable used extensively in this patch, is it used in different purpose here? will quit/Ctrl-C behaviour differ in multi process?

Quoted 16 lines
app/test-pmd/testpmd.c +0 -0
  		if (rc < 0)
  			return 1;
  	}
app/test-pmd/testpmd.h +1 -0
index a87ccb0..640a377 100644
@@ -13,6 +13,7 @@
  #include <rte_gso.h>
  #include <cmdline.h>
  #include <sys/queue.h>
+#include  "cmdline.h"
 
  #define RTE_PORT_ALL            (~(portid_t)0x0)
 

app/test-pmd/testpmd.h line 31

Quoted 7 lines
app/test-pmd/testpmd.h +4 -0
@@ -24,6 +25,10 @@
  #define RTE_PORT_CLOSED         (uint16_t)2
  #define RTE_PORT_HANDLING       (uint16_t)3
 
+extern uint8_t f_quit;
+extern int testpmd_fd_copy;
+extern struct cmdline *testpmd_cl;
+

Again can you please move them in the file, there is already a block in below for extern variables, please place there with sufficient comments.

app/test-pmd/testpmd.h extern uint64_t noisy_lkup_mem_sz; · line 433

Quoted 10 lines
app/test-pmd/testpmd.h +2 -0
  /*
   * It is used to allocate the memory for hash key.
   * The hash key size is NIC dependent.
@@ -423,6 +428,8 @@ extern uint64_t noisy_lkup_mem_sz;
  extern uint64_t noisy_lkup_num_writes;
  extern uint64_t noisy_lkup_num_reads;
  extern uint64_t noisy_lkup_num_reads_writes;
+extern int proc_id;
+extern unsigned int num_procs;
 

Can you pleaes group multi process related changes together?

Quoted 36 lines
app/test-pmd/testpmd.h +0 -0
  extern uint8_t dcb_config;
  extern uint8_t dcb_test;
doc/guides/testpmd_app_ug/run_app.rst +26 -0
index ec1dc7d..7e60e80 100644
@@ -551,3 +551,104 @@ The command line options are:
      bit 1 - two hairpin ports paired
      bit 0 - two hairpin ports loop
      The default value is 0. Hairpin will use single port mode and implicit Tx flow mode.
+
+
+Testpmd Support Multi Process Command-line Options
+--------------------------------------------------
+
+The following are the command-line options for the testpmd applications(support
+multi process).They must be separated from the EAL options, shown in the previous
+section, with a ``--`` separator:
+
+.. code-block:: console
+
+	primary process:
+	sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i --rxq=4 --txq=4 \
+        --num-procs=2 --proc-id=0
+
+	secondary process:
+	sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i --rxq=4 --txq=4 \
+        --num-procs=2 --proc-id=1
+
+The command line options are:
+
+*   ``-a, --allow``
+
+    Add a device to the allow list. ``xxx`` means device used which should be the
+    same in primary process and secondary process.
+

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 589

Quoted 10 lines
doc/guides/testpmd_app_ug/run_app.rst +10 -0
+*   ``--proc-type``
+
+    Specify a given process instance as the primary or secondary DPDK instance.
+    ``auto`` set here is OK.
+
+*   ``-l CORELIST``
+
+    List of cores to run on. the corelist should be different in primary process and
+    secondary process.
+

+1 for documentation, but above are eal parameters, what about not repeting them here but referencing eal documentation, 'doc/guides/linux_gsg/eal_args.include.rst', for it?

Quoted 66 lines
doc/guides/testpmd_app_ug/run_app.rst +65 -0
+*   ``--rxq=N``
+
+    Set the number of Rx queues per port to N. N is the sum of queues used by primary
+    and secondary process. Primary process and secondary process should have separate
+    queues, and each should occupy at least one queue. Where N should be the multiple
+    of number of processes.
+
+*   ``--txq=N``
+
+    Set the number of Tx queues per port to N. N is the sum of queues used by primary
+    and secondary process. Primary process and secondary process should have separate
+    queues, and each should occupy at least one queue. Where N should be the multiple
+    of number of processes.
+
+*   ``--num-procs=N``
+
+    The number of processes which will be used.
+
+*   ``--proc-id=id``
+
+    The id of the current process (id < num-procs). id should be different in primary
+    process and secondary process.
+
+Calculation rule for queue:
+All queues are allocated to different processes based on proc_num and proc_id.
+Calculation rule for the Testpmd to allocate queues to each process:
+start(queue start id) = proc_id * nb_q / num_procs;
+end(queue end id) = start + nb_q / num_procs;
+
+For example, if supports 4 txq and rxq
+the 0~1 for primary process
+the 2~3 for secondary process
+
+The number of rings should be a multiple of the number of processes. If not,
+redundant queues will exist after queues are allocated to processes. After RSS is
+enabled, packet loss occurs when traffic is sent to all processes at the same time.
+Some traffic enters redundant queues and cannot be forwarded.
+
+Most dev ops is supported in primary and secondary process. While secondary process
+is not permitted to allocate or release shared memory, so some ops are not supported
+as follows:
+``dev_configure``
+``dev_start``
+``dev_stop``
+``rx_queue_setup``
+``tx_queue_setup``
+``rx_queue_release``
+``tx_queue_release``
+
+So, any command from testpmd which calls those APIs will not be supported in secondary
+process, like:
+``port config all rxq|txq|rxd|txd <value>``
+``port config <port_id> rx_offload xxx on/off ``
+``port config <port_id> tx_offload xxx on/off``
+etc.
+
+RTE_FLOW supported, it applies only on its own process on SW side, but all on HW size.
+stats supported, stats will not change when one quit and start, As they share the same
+buffer to store the stats. Flow rules are maintained in process level: primary and secondary
+has its own flow list(but one flow list in HW). The two can see all the queues, so setting
+the flow rules for the other is OK. Of course, io(receive or transmit packets) in the queue
+from others is not permitted.
+
+RSS supported, Primary process and secondary process has separate queues to use, RSS
+will work in their own queues whether primary and secondary process.
#57 lore raw
Quoted 9 lines · Ferruh Yigit
Quoted 8 lines · Min Hu (Connor)

From: Lijun Ou <[email protected]>

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

Quoted 9 lines
Quoted 4 lines

the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

Hi Connor,

Please find a few minor comments below, since they are minor issues please feel free to keep the existing acks in next version.

Reminder of this patch, waiting for minor updates, it would be good to have it before -rc1, so that the change can be tested thoroughly.

Meanwhile I have concern/question on long term testing of the feature. It is possible to break the multi-process support unintentionally and not recognize it. How can we continually verify the feature?

Are you using DTS?
Does it make sense to add a testcase into dts to test testpmd multi-process support so that it is verified on each release?

#58 lore raw
Quoted 15 lines · Ferruh Yigit
Quoted 14 lines · Ferruh Yigit
Quoted 12 lines · Min Hu (Connor)

From: Lijun Ou <[email protected]>

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

Quoted 7 lines
Quoted 4 lines

Hi Connor,

Please find a few minor comments below, since they are minor issues please feel free to keep the existing acks in next version.

Reminder of this patch, waiting for minor updates, it would be good to have it before -rc1, so that the change can be tested thoroughly. Hi, Ferruh,

v9 has been sent, please check it out, thanks.

Quoted 5 lines

Meanwhile I have concern/question on long term testing of the feature. It is possible to break the multi-process support unintentionally and not recognize it. How can we continually verify the feature?

Are you using DTS?
Does it make sense to add a testcase into dts to test testpmd multi-process support so that it is verified on each release? .

Well, we have our own DTS system which contains testcase of multiple process in testpmd:
1. When we upstream patches to DPDK,the patch should pass all the testcase.
2. Before DPDK version release(in RC1) in every three months, we rebase the new DPDK version, and check all the testcase in our DTS system. If some testcase fails, we will fix it and upstream the patch to DPDK, of course, including multi-process support in testpmd.

#59 lore raw

PATCH v9 1/1 app/testpmd: support multi-process

In reply to Lijun Ou

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

---
v9:
* Updated release notes and rst doc.
* Deleted deprecated codes.
* move macro and variable.

v8:
* Added warning info about queue numbers and process numbers.

v7:
* Fixed compiling error for unexpected unindent.

v6:
* Add rte flow description for multiple process.

v5:
* Fixed run_app.rst for multiple process description.
* Fix compiling error.

v4:
* Fixed minimum vlaue of Rxq or Txq in doc.

v3:
* Fixed compiling error using gcc10.0.

v2:
* Added document for this patch.

---
 app/test-pmd/cmdline.c                 |   6 ++
 app/test-pmd/config.c                  |  14 ++++-
 app/test-pmd/parameters.c              |  12 ++++
 app/test-pmd/testpmd.c                 | 108 +++++++++++++++++++++++----------
 app/test-pmd/testpmd.h                 |   3 +
 doc/guides/rel_notes/release_21_05.rst |   1 +
 doc/guides/testpmd_app_ug/run_app.rst  |  86 ++++++++++++++++++++++++++
 7 files changed, 196 insertions(+), 34 deletions(-)
app/test-pmd/cmdline.c +6 -0
index 5bf1497..e465824 100644
@@ -5354,6 +5354,12 @@ cmd_set_flush_rx_parsed(void *parsed_result,
 		__rte_unused void *data)
 {
 	struct cmd_set_flush_rx *res = parsed_result;
+
+	if (num_procs > 1 && (strcmp(res->mode, "on") == 0)) {
+		printf("multi-process doesn't support to flush rx queues.\n");
+		return;
+	}
+
 	no_flush_rx = (uint8_t)((strcmp(res->mode, "on") == 0) ? 0 : 1);
 }
 
app/test-pmd/config.c +13 -1
index 40b2b29..c982c87 100644
@@ -2860,6 +2860,8 @@ rss_fwd_config_setup(void)
 	queueid_t  rxq;
 	queueid_t  nb_q;
 	streamid_t  sm_id;
+	int start;
+	int end;
 
 	nb_q = nb_rxq;
 	if (nb_q > nb_txq)
@@ -2877,7 +2879,15 @@ rss_fwd_config_setup(void)
 	init_fwd_streams();
 
 	setup_fwd_config_of_each_lcore(&cur_fwd_config);
-	rxp = 0; rxq = 0;
+
+	if (proc_id > 0 && nb_q % num_procs)
+		printf("Warning! queue numbers should be multiple of "
+			"processes, or packet loss will happen.\n");
+
+	start = proc_id * nb_q / num_procs;
+	end = start + nb_q / num_procs;
+	rxp = 0;
+	rxq = start;
 	for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
 		struct fwd_stream *fs;
 
@@ -2894,6 +2904,8 @@ rss_fwd_config_setup(void)
 			continue;
 		rxp = 0;
 		rxq++;
+		if (rxq >= end)
+			rxq = start;
 	}
 }
 
app/test-pmd/parameters.c +12 -0
index f3954c1..d86cc21 100644
@@ -508,6 +508,9 @@ parse_link_speed(int n)
 void
 launch_args_parse(int argc, char** argv)
 {
+#define PARAM_PROC_ID "proc-id"
+#define PARAM_NUM_PROCS "num-procs"
+
 	int n, opt;
 	char **argvopt;
 	int opt_idx;
@@ -625,6 +628,8 @@ launch_args_parse(int argc, char** argv)
 		{ "rx-mq-mode",                 1, 0, 0 },
 		{ "record-core-cycles",         0, 0, 0 },
 		{ "record-burst-stats",         0, 0, 0 },
+		{ PARAM_NUM_PROCS,              1, 0, 0 },
+		{ PARAM_PROC_ID,                1, 0, 0 },
 		{ 0, 0, 0, 0 },
 	};
 
@@ -1391,6 +1396,13 @@ launch_args_parse(int argc, char** argv)
 				record_core_cycles = 1;
 			if (!strcmp(lgopts[opt_idx].name, "record-burst-stats"))
 				record_burst_stats = 1;
+
+			if (strncmp(lgopts[opt_idx].name,
+				    PARAM_NUM_PROCS, 9) == 0)
+				num_procs = atoi(optarg);
+			if (strncmp(lgopts[opt_idx].name,
+				    PARAM_PROC_ID, 7) == 0)
+				proc_id = atoi(optarg);
 			break;
 		case 'h':
 			usage(argv[0]);
app/test-pmd/testpmd.c +75 -33

193 lines not on this page · raw message

app/test-pmd/testpmd.h +3 -0
index 36d8535..e04016c 100644
@@ -630,6 +630,9 @@ extern struct mplsoudp_decap_conf mplsoudp_decap_conf;
 
 extern enum rte_eth_rx_mq_mode rx_mq_mode;
 
+extern int proc_id;
+extern unsigned int num_procs;
+
 static inline unsigned int
 lcore_num(void)
 {
doc/guides/rel_notes/release_21_05.rst +1 -0
index 3bd7757..df01d4f 100644
@@ -185,6 +185,7 @@ New Features
     ``show port (port_id) rxq (queue_id) desc used count``
   * Added command to dump internal representation information of single flow.
     ``flow dump (port_id) rule (rule_id)``
+  * Added support multi-process for testpmd.
 
 
 Removed Items
doc/guides/testpmd_app_ug/run_app.rst +86 -0

89 lines not on this page · raw message

Signature · 1 line
2.7.4
#60 lore raw
Quoted 10 lines · Min Hu (Connor)

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

Hi Connor,

Thanks for the update. +Anatoly as multi-process maintainer, and ethdev maintainers.

Quoted 75 lines

---
v9:
* Updated release notes and rst doc.
* Deleted deprecated codes.
* move macro and variable.

v8:
* Added warning info about queue numbers and process numbers.

v7:
* Fixed compiling error for unexpected unindent.

v6:
* Add rte flow description for multiple process.

v5:
* Fixed run_app.rst for multiple process description.
* Fix compiling error.

v4:
* Fixed minimum vlaue of Rxq or Txq in doc.

v3:
* Fixed compiling error using gcc10.0.

v2:
* Added document for this patch.

---
  app/test-pmd/cmdline.c                 |   6 ++
  app/test-pmd/config.c                  |  14 ++++-
  app/test-pmd/parameters.c              |  12 ++++
  app/test-pmd/testpmd.c                 | 108 +++++++++++++++++++++++----------
  app/test-pmd/testpmd.h                 |   3 +
  doc/guides/rel_notes/release_21_05.rst |   1 +
  doc/guides/testpmd_app_ug/run_app.rst  |  86 ++++++++++++++++++++++++++
  7 files changed, 196 insertions(+), 34 deletions(-)
app/test-pmd/cmdline.c +6 -0
index 5bf1497..e465824 100644
@@ -5354,6 +5354,12 @@ cmd_set_flush_rx_parsed(void *parsed_result,
  		__rte_unused void *data)
  {
  	struct cmd_set_flush_rx *res = parsed_result;
+
+	if (num_procs > 1 && (strcmp(res->mode, "on") == 0)) {
+		printf("multi-process doesn't support to flush rx queues.\n");
+		return;
+	}
+
  	no_flush_rx = (uint8_t)((strcmp(res->mode, "on") == 0) ? 0 : 1);
  }
 
app/test-pmd/config.c +2 -0
index 40b2b29..c982c87 100644
@@ -2860,6 +2860,8 @@ rss_fwd_config_setup(void)
  	queueid_t  rxq;
  	queueid_t  nb_q;
  	streamid_t  sm_id;
+	int start;
+	int end;
 
  	nb_q = nb_rxq;
  	if (nb_q > nb_txq)
@@ -2877,7 +2879,15 @@ rss_fwd_config_setup(void)
  	init_fwd_streams();
 
  	setup_fwd_config_of_each_lcore(&cur_fwd_config);

app/test-pmd/config.c rss_fwd_config_setup(void) · line 2890

Quoted 10 lines
app/test-pmd/config.c +9 -1
-	rxp = 0; rxq = 0;
+
+	if (proc_id > 0 && nb_q % num_procs)
+		printf("Warning! queue numbers should be multiple of "
+			"processes, or packet loss will happen.\n");
+
+	start = proc_id * nb_q / num_procs;
+	end = start + nb_q / num_procs;
+	rxp = 0;
+	rxq = start;

Can you put some comment above, on what/why is done, something similar to you already put into the documentation?

Quoted 36 lines
app/test-pmd/config.c +2 -0
  	for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
  		struct fwd_stream *fs;
 
@@ -2894,6 +2904,8 @@ rss_fwd_config_setup(void)
  			continue;
  		rxp = 0;
  		rxq++;
+		if (rxq >= end)
+			rxq = start;
  	}
  }
 
app/test-pmd/parameters.c +5 -0
index f3954c1..d86cc21 100644
@@ -508,6 +508,9 @@ parse_link_speed(int n)
  void
  launch_args_parse(int argc, char** argv)
  {
+#define PARAM_PROC_ID "proc-id"
+#define PARAM_NUM_PROCS "num-procs"
+
  	int n, opt;
  	char **argvopt;
  	int opt_idx;
@@ -625,6 +628,8 @@ launch_args_parse(int argc, char** argv)
  		{ "rx-mq-mode",                 1, 0, 0 },
  		{ "record-core-cycles",         0, 0, 0 },
  		{ "record-burst-stats",         0, 0, 0 },
+		{ PARAM_NUM_PROCS,              1, 0, 0 },
+		{ PARAM_PROC_ID,                1, 0, 0 },
  		{ 0, 0, 0, 0 },
  	};
 

app/test-pmd/parameters.c launch_args_parse(int argc, char** argv) · line 1399

Quoted 4 lines
app/test-pmd/parameters.c +1 -0
@@ -1391,6 +1396,13 @@ launch_args_parse(int argc, char** argv)
  				record_core_cycles = 1;
  			if (!strcmp(lgopts[opt_idx].name, "record-burst-stats"))
  				record_burst_stats = 1;
+

To be consistent for rest, this empty line can be removed.

Quoted 14 lines
app/test-pmd/parameters.c +6 -0
+			if (strncmp(lgopts[opt_idx].name,
+				    PARAM_NUM_PROCS, 9) == 0)
+				num_procs = atoi(optarg);
+			if (strncmp(lgopts[opt_idx].name,
+				    PARAM_PROC_ID, 7) == 0)
+				proc_id = atoi(optarg);
  			break;
  		case 'h':
  			usage(argv[0]);
app/test-pmd/testpmd.c +0 -0
index 96d2e0f..01d0d82 100644

app/test-pmd/testpmd.c enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS; · line 523

Quoted 6 lines
app/test-pmd/testpmd.c +3 -0
@@ -518,6 +518,16 @@ enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS;
   */
  uint32_t eth_link_speed;
 
+/*
+ * Id of the current process.
+ */

Can you please add more info, like this being related to the primary/secondary process support and used to configure the queues to be polled.

app/test-pmd/testpmd.c enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS; · line 530

Quoted 7 lines
app/test-pmd/testpmd.c +7 -0
+int proc_id;
+
+/*
+ * Number of processes.
+ */
+unsigned int num_procs = 1;
+

Ditto.

<...>

Quoted 22 lines
app/test-pmd/testpmd.c +6 -10
@@ -2511,21 +2537,28 @@ start_port(portid_t pid)
  				return -1;
  			}
  			/* configure port */
-			diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
+			if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+				diag = rte_eth_dev_configure(pi,
+						     nb_rxq + nb_hairpinq,
  						     nb_txq + nb_hairpinq,
  						     &(port->dev_conf));
-			if (diag != 0) {
-				if (rte_atomic16_cmpset(&(port->port_status),
-				RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
-					printf("Port %d can not be set back "
-							"to stopped\n", pi);
-				printf("Fail to configure port %d\n", pi);
-				/* try to reconfigure port next time */
-				port->need_reconfig = 1;
-				return -1;
+				if (diag != 0) {
+					if (rte_atomic16_cmpset(
+							&(port->port_status),

app/test-pmd/testpmd.c start_port(portid_t pid) · line 2557

Quoted 10 lines
app/test-pmd/testpmd.c +10 -0
+							RTE_PORT_HANDLING,
+							RTE_PORT_STOPPED) == 0)
+						printf("Port %d can not be set "
+						       "back to stopped\n", pi);
+					printf("Fail to configure port %d\n",
+						pi);
+					/* try to reconfigure port next time */
+					port->need_reconfig = 1;
+					return -1;
+				}

Just an idea,
I am a little worried about the complexity this new process type checks are added to the multiple locations. What do you think hiding process type checks behind new functions, like:
eth_dev_configure_mp(...) {
if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
return rte_eth_dev_configure(..)
}
return 0;
}

and replace it with 'rte_eth_dev_configure()':

-diag = rte_eth_dev_configure(...)
+diag = eth_dev_configure_mp(...)

Do you think does this help reducing the complexity added by the multi-process support?

app/test-pmd/testpmd.c start_port(portid_t pid) · line 2561

Quoted 5 lines
app/test-pmd/testpmd.c +2 -1
  			}
  		}
-		if (port->need_reconfig_queues > 0) {
+		if (port->need_reconfig_queues > 0 &&
+		    rte_eal_process_type() == RTE_PROC_PRIMARY) {

According our coding convention, we don't allign with paranthesis but put double tabs.

And what about creating an inline function for primary process check, like is_proc_primary(), that may help.

Quoted 12 lines
app/test-pmd/testpmd.c +1 -4
  			port->need_reconfig_queues = 0;
  			/* setup tx queues */
  			for (qi = 0; qi < nb_txq; qi++) {
@@ -2626,17 +2659,20 @@ start_port(portid_t pid)
  		cnt_pi++;
 
  		/* start port */
-		diag = rte_eth_dev_start(pi);
-		if (diag < 0) {
-			printf("Fail to start port %d: %s\n", pi,
-			       rte_strerror(-diag));
+		if (rte_eal_process_type() == RTE_PROC_PRIMARY) {

app/test-pmd/testpmd.c start_port(portid_t pid) · line 2670

Quoted 10 lines
app/test-pmd/testpmd.c +6 -2
+			diag = rte_eth_dev_start(pi);
+			if (diag < 0) {
+				printf("Fail to start port %d: %s\n", pi,
+				       rte_strerror(-diag));
 
-			/* Fail to setup rx queue, return */
-			if (rte_atomic16_cmpset(&(port->port_status),
+				/* Fail to setup rx queue, return */
+				if (rte_atomic16_cmpset(&(port->port_status),
  				RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)

Syntax is wrong here after change, but if you go with above idea to hide process type check within functions, you won't need to change these lines at all.

<...>

app/test-pmd/testpmd.c main(int argc, char** argv) · line 3929

Quoted 7 lines
app/test-pmd/testpmd.c +2 -1
@@ -3885,8 +3925,10 @@ main(int argc, char** argv)
  		}
  	}
 
-	if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
+	if (!no_device_start && start_port(RTE_PORT_ALL) != 0) {
+		pmd_test_exit();

Why need to add 'pmd_test_exit()' for the multi-process support, if there is a special case, please add comment for it.

#61 lore raw

Hi, Ferrruh,
All is fixed in v10, please check it out, thanks.

Quoted 272 lines · Ferruh Yigit
Quoted 10 lines · Min Hu (Connor)

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

Hi Connor,

Thanks for the update. +Anatoly as multi-process maintainer, and ethdev maintainers.

Quoted 73 lines

---
v9:
* Updated release notes and rst doc.
* Deleted deprecated codes.
* move macro and variable.

v8:
* Added warning info about queue numbers and process numbers.

v7:
* Fixed compiling error for unexpected unindent.

v6:
* Add rte flow description for multiple process.

v5:
* Fixed run_app.rst for multiple process description.
* Fix compiling error.

v4:
* Fixed minimum vlaue of Rxq or Txq in doc.

v3:
* Fixed compiling error using gcc10.0.

v2:
* Added document for this patch.
---
  app/test-pmd/cmdline.c                 |   6 ++ app/test-pmd/config.c                  |  14 ++++- app/test-pmd/parameters.c              |  12 ++++ app/test-pmd/testpmd.c                 | 108
+++++++++++++++++++++++----------
  app/test-pmd/testpmd.h                 |   3 + doc/guides/rel_notes/release_21_05.rst |   1 + doc/guides/testpmd_app_ug/run_app.rst  |  86 ++++++++++++++++++++++++++

  7 files changed, 196 insertions(+), 34 deletions(-)
app/test-pmd/cmdline.c +6 -0
index 5bf1497..e465824 100644
@@ -5354,6 +5354,12 @@ cmd_set_flush_rx_parsed(void *parsed_result,
           __rte_unused void *data)
   {
       struct cmd_set_flush_rx *res = parsed_result;
+
+    if (num_procs > 1 && (strcmp(res->mode, "on") == 0)) {
+        printf("multi-process doesn't support to flush rx queues.\n");
+        return;
+    }
+
       no_flush_rx = (uint8_t)((strcmp(res->mode, "on") == 0) ? 0 : 1);
   }
app/test-pmd/config.c +2 -0
index 40b2b29..c982c87 100644
@@ -2860,6 +2860,8 @@ rss_fwd_config_setup(void)
       queueid_t  rxq;
       queueid_t  nb_q;
       streamid_t  sm_id;
+    int start;
+    int end;
       nb_q = nb_rxq;
       if (nb_q > nb_txq)
@@ -2877,7 +2879,15 @@ rss_fwd_config_setup(void)
       init_fwd_streams();
       setup_fwd_config_of_each_lcore(&cur_fwd_config);

app/test-pmd/config.c rss_fwd_config_setup(void) · line 2889

Quoted 10 lines
app/test-pmd/config.c +9 -1
-    rxp = 0; rxq = 0;
+
+    if (proc_id > 0 && nb_q % num_procs)
+        printf("Warning! queue numbers should be multiple of "
+            "processes, or packet loss will happen.\n");
+
+    start = proc_id * nb_q / num_procs;
+    end = start + nb_q / num_procs;
+    rxp = 0;
+    rxq = start;

Can you put some comment above, on what/why is done, something similar to you already put into the documentation?

Quoted 33 lines
app/test-pmd/config.c +2 -0
       for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
           struct fwd_stream *fs;
@@ -2894,6 +2904,8 @@ rss_fwd_config_setup(void)
               continue;
           rxp = 0;
           rxq++;
+        if (rxq >= end)
+            rxq = start;
       }
   }
app/test-pmd/parameters.c +0 -0
index f3954c1..d86cc21 100644
@@ -508,6 +508,9 @@ parse_link_speed(int n)

  void
  launch_args_parse(int argc, char** argv) {

app/test-pmd/parameters.c +3 -0
+#define PARAM_PROC_ID "proc-id"
+#define PARAM_NUM_PROCS "num-procs"
+

      int n, opt;
      char **argvopt;
      int opt_idx;

app/test-pmd/parameters.c +2 -0
@@ -625,6 +628,8 @@ launch_args_parse(int argc, char** argv)
           { "rx-mq-mode",                 1, 0, 0 },
           { "record-core-cycles",         0, 0, 0 },
           { "record-burst-stats",         0, 0, 0 },
+        { PARAM_NUM_PROCS,              1, 0, 0 },
+        { PARAM_PROC_ID,                1, 0, 0 },
           { 0, 0, 0, 0 },
       };

app/test-pmd/parameters.c launch_args_parse(int argc, char** argv) · line 1399

Quoted 4 lines
app/test-pmd/parameters.c +1 -0
@@ -1391,6 +1396,13 @@ launch_args_parse(int argc, char** argv)
                   record_core_cycles = 1;
               if (!strcmp(lgopts[opt_idx].name, "record-burst-stats"))
                   record_burst_stats = 1;
+

To be consistent for rest, this empty line can be removed.

Quoted 14 lines
app/test-pmd/parameters.c +6 -0
+            if (strncmp(lgopts[opt_idx].name,
+                    PARAM_NUM_PROCS, 9) == 0)
+                num_procs = atoi(optarg);
+            if (strncmp(lgopts[opt_idx].name,
+                    PARAM_PROC_ID, 7) == 0)
+                proc_id = atoi(optarg);
               break;

          case 'h':
              usage(argv[0]);

app/test-pmd/testpmd.c +0 -0
index 96d2e0f..01d0d82 100644

app/test-pmd/testpmd.c enum rte_eth_rx_mq_mode rx_mq_mode = · line 523

Quoted 6 lines
app/test-pmd/testpmd.c +3 -0
@@ -518,6 +518,16 @@ enum rte_eth_rx_mq_mode rx_mq_mode = 
 ETH_MQ_RX_VMDQ_DCB_RSS;
    */
   uint32_t eth_link_speed;
+/*
+ * Id of the current process.
+ */

Can you please add more info, like this being related to the primary/secondary process support and used to configure the queues to be polled.

app/test-pmd/testpmd.c enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS; · line 530

Quoted 7 lines
app/test-pmd/testpmd.c +7 -0
+int proc_id;
+
+/*
+ * Number of processes.
+ */
+unsigned int num_procs = 1;
+

Ditto.

<...>

Quoted 22 lines
app/test-pmd/testpmd.c +6 -10
@@ -2511,21 +2537,28 @@ start_port(portid_t pid)
                   return -1;
               }
               /* configure port */
-            diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
+            if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+                diag = rte_eth_dev_configure(pi,
+                             nb_rxq + nb_hairpinq,
                                nb_txq + nb_hairpinq,
                                &(port->dev_conf));
-            if (diag != 0) {
-                if (rte_atomic16_cmpset(&(port->port_status),
-                RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
-                    printf("Port %d can not be set back "
-                            "to stopped\n", pi);
-                printf("Fail to configure port %d\n", pi);
-                /* try to reconfigure port next time */
-                port->need_reconfig = 1;
-                return -1;
+                if (diag != 0) {
+                    if (rte_atomic16_cmpset(
+                            &(port->port_status),

app/test-pmd/testpmd.c start_port(portid_t pid) · line 2557

Quoted 10 lines
app/test-pmd/testpmd.c +10 -0
+                            RTE_PORT_HANDLING,
+                            RTE_PORT_STOPPED) == 0)
+                        printf("Port %d can not be set "
+                               "back to stopped\n", pi);
+                    printf("Fail to configure port %d\n",
+                        pi);
+                    /* try to reconfigure port next time */
+                    port->need_reconfig = 1;
+                    return -1;
+                }

Just an idea,
I am a little worried about the complexity this new process type checks are added to the multiple locations. What do you think hiding process type checks behind new functions, like:
eth_dev_configure_mp(...) {
  if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
    return rte_eth_dev_configure(..)
  }
  return 0;
}

and replace it with 'rte_eth_dev_configure()':

 -diag = rte_eth_dev_configure(...)
 +diag = eth_dev_configure_mp(...)

Do you think does this help reducing the complexity added by the multi-process support?

app/test-pmd/testpmd.c start_port(portid_t pid) · line 2561

Quoted 5 lines
app/test-pmd/testpmd.c +2 -1
               }
           }
-        if (port->need_reconfig_queues > 0) {
+        if (port->need_reconfig_queues > 0 &&
+            rte_eal_process_type() == RTE_PROC_PRIMARY) {

According our coding convention, we don't allign with paranthesis but put double tabs.

And what about creating an inline function for primary process check, like is_proc_primary(), that may help.

Quoted 10 lines
app/test-pmd/testpmd.c +0 -4
               port->need_reconfig_queues = 0;
               /* setup tx queues */
               for (qi = 0; qi < nb_txq; qi++) {
@@ -2626,17 +2659,20 @@ start_port(portid_t pid)
           cnt_pi++;
           /* start port */
-        diag = rte_eth_dev_start(pi);
-        if (diag < 0) {
-            printf("Fail to start port %d: %s\n", pi,
-                   rte_strerror(-diag));

app/test-pmd/testpmd.c start_port(portid_t pid) · line 2668

Quoted 10 lines
app/test-pmd/testpmd.c +7 -2
+        if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+            diag = rte_eth_dev_start(pi);
+            if (diag < 0) {
+                printf("Fail to start port %d: %s\n", pi,
+                       rte_strerror(-diag));
-            /* Fail to setup rx queue, return */
-            if (rte_atomic16_cmpset(&(port->port_status),
+                /* Fail to setup rx queue, return */
+                if (rte_atomic16_cmpset(&(port->port_status),
                   RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)

Syntax is wrong here after change, but if you go with above idea to hide process type check within functions, you won't need to change these lines at all.

<...>

app/test-pmd/testpmd.c main(int argc, char** argv) · line 3928

Quoted 6 lines
app/test-pmd/testpmd.c +2 -1
@@ -3885,8 +3925,10 @@ main(int argc, char** argv)
           }
       }
-    if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
+    if (!no_device_start && start_port(RTE_PORT_ALL) != 0) {
+        pmd_test_exit();

Why need to add 'pmd_test_exit()' for the multi-process support, if there is a special case, please add comment for it.

.

#62 lore raw

PATCH v10 1/1 app/testpmd: support multi-process

In reply to Lijun Ou

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

---
v10:
* Hid process type checks behind new functions.
* Added comments.

v9:
* Updated release notes and rst doc.
* Deleted deprecated codes.
* move macro and variable.

v8:
* Added warning info about queue numbers and process numbers.

v7:
* Fixed compiling error for unexpected unindent.

v6:
* Add rte flow description for multiple process.

v5:
* Fixed run_app.rst for multiple process description.
* Fix compiling error.

v4:
* Fixed minimum vlaue of Rxq or Txq in doc.

v3:
* Fixed compiling error using gcc10.0.

v2:
* Added document for this patch.

---
 app/test-pmd/cmdline.c                 |   6 ++
 app/test-pmd/config.c                  |  21 +++++-
 app/test-pmd/parameters.c              |  11 +++
 app/test-pmd/testpmd.c                 | 127 ++++++++++++++++++++++++++-------
 app/test-pmd/testpmd.h                 |   9 +++
 doc/guides/rel_notes/release_21_05.rst |   1 +
 doc/guides/testpmd_app_ug/run_app.rst  |  86 ++++++++++++++++++++++
 7 files changed, 236 insertions(+), 25 deletions(-)
app/test-pmd/cmdline.c +6 -0
index 5bf1497..e465824 100644
@@ -5354,6 +5354,12 @@ cmd_set_flush_rx_parsed(void *parsed_result,
 		__rte_unused void *data)
 {
 	struct cmd_set_flush_rx *res = parsed_result;
+
+	if (num_procs > 1 && (strcmp(res->mode, "on") == 0)) {
+		printf("multi-process doesn't support to flush rx queues.\n");
+		return;
+	}
+
 	no_flush_rx = (uint8_t)((strcmp(res->mode, "on") == 0) ? 0 : 1);
 }
 
app/test-pmd/config.c +20 -1
index 40b2b29..d0e5b29 100644
@@ -2860,6 +2860,8 @@ rss_fwd_config_setup(void)
 	queueid_t  rxq;
 	queueid_t  nb_q;
 	streamid_t  sm_id;
+	int start;
+	int end;
 
 	nb_q = nb_rxq;
 	if (nb_q > nb_txq)
@@ -2877,7 +2879,22 @@ rss_fwd_config_setup(void)
 	init_fwd_streams();
 
 	setup_fwd_config_of_each_lcore(&cur_fwd_config);
-	rxp = 0; rxq = 0;
+
+	if (proc_id > 0 && nb_q % num_procs)
+		printf("Warning! queue numbers should be multiple of "
+			"processes, or packet loss will happen.\n");
+
+	/**
+	 * In multi-process, All queues are allocated to different
+	 * processes based on num_procs and proc_id. For example:
+	 * if supports 4 queues(nb_q), 2 processes(num_procs),
+	 * the 0~1 queue for primary process.
+	 * the 2~3 queue for secondary process.
+	 */
+	start = proc_id * nb_q / num_procs;
+	end = start + nb_q / num_procs;
+	rxp = 0;
+	rxq = start;
 	for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
 		struct fwd_stream *fs;
 
@@ -2894,6 +2911,8 @@ rss_fwd_config_setup(void)
 			continue;
 		rxp = 0;
 		rxq++;
+		if (rxq >= end)
+			rxq = start;
 	}
 }
 
app/test-pmd/parameters.c +11 -0
index f3954c1..ece05c1 100644
@@ -508,6 +508,9 @@ parse_link_speed(int n)
 void
 launch_args_parse(int argc, char** argv)
 {
+#define PARAM_PROC_ID "proc-id"
+#define PARAM_NUM_PROCS "num-procs"
+
 	int n, opt;
 	char **argvopt;
 	int opt_idx;
@@ -625,6 +628,8 @@ launch_args_parse(int argc, char** argv)
 		{ "rx-mq-mode",                 1, 0, 0 },
 		{ "record-core-cycles",         0, 0, 0 },
 		{ "record-burst-stats",         0, 0, 0 },
+		{ PARAM_NUM_PROCS,              1, 0, 0 },
+		{ PARAM_PROC_ID,                1, 0, 0 },
 		{ 0, 0, 0, 0 },
 	};
 
@@ -1391,6 +1396,12 @@ launch_args_parse(int argc, char** argv)
 				record_core_cycles = 1;
 			if (!strcmp(lgopts[opt_idx].name, "record-burst-stats"))
 				record_burst_stats = 1;
+			if (strncmp(lgopts[opt_idx].name,
+				    PARAM_NUM_PROCS, 9) == 0)
+				num_procs = atoi(optarg);
+			if (strncmp(lgopts[opt_idx].name,
+				    PARAM_PROC_ID, 7) == 0)
+				proc_id = atoi(optarg);
 			break;
 		case 'h':
 			usage(argv[0]);
app/test-pmd/testpmd.c +103 -24

212 lines not on this page · raw message

app/test-pmd/testpmd.h +9 -0
index 36d8535..4c8f8bc 100644
@@ -630,6 +630,15 @@ extern struct mplsoudp_decap_conf mplsoudp_decap_conf;
 
 extern enum rte_eth_rx_mq_mode rx_mq_mode;
 
+extern int proc_id;
+extern unsigned int num_procs;
+
+static inline bool
+is_proc_primary(void)
+{
+	return rte_eal_process_type() == RTE_PROC_PRIMARY;
+}
+
 static inline unsigned int
 lcore_num(void)
 {
doc/guides/rel_notes/release_21_05.rst +1 -0
index 3bd7757..df01d4f 100644
@@ -185,6 +185,7 @@ New Features
     ``show port (port_id) rxq (queue_id) desc used count``
   * Added command to dump internal representation information of single flow.
     ``flow dump (port_id) rule (rule_id)``
+  * Added support multi-process for testpmd.
 
 
 Removed Items
doc/guides/testpmd_app_ug/run_app.rst +86 -0

89 lines not on this page · raw message

Signature · 1 line
2.7.4
Reviewed-by on v10 1/1
#63 lore raw
Quoted 6 lines · Min Hu (Connor)

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

Quoted 8 lines

the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

Hi Connor,

I put some minor syntax comments, when they are fixed,

<...>

app/test-pmd/testpmd.c start_port(portid_t pid) · line 2596

Quoted 12 lines
  			if (diag != 0) {
app/test-pmd/testpmd.c +6 -5
-				if (rte_atomic16_cmpset(&(port->port_status),
-				RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
-					printf("Port %d can not be set back "
-							"to stopped\n", pi);
-				printf("Fail to configure port %d\n", pi);
+				if (rte_atomic16_cmpset(
+						&(port->port_status),
+						RTE_PORT_HANDLING,
+						RTE_PORT_STOPPED) == 0)
+					printf("Port %d can not be set "
+					       "back to stopped\n", pi);

It is OK to have long line for the strings, so it can be like printf("Port %d can not be set back to stopped\n",

	pi);

app/test-pmd/testpmd.c start_port(portid_t pid) · line 2598

Quoted 2 lines
app/test-pmd/testpmd.c +2 -0
+				printf("Fail to configure port %d\n",
+					pi);

No need to update this line.

<...>

app/test-pmd/testpmd.c start_port(portid_t pid) · line 2715

Quoted 8 lines

/* Fail to setup rx queue, return */ if (rte_atomic16_cmpset(&(port->port_status),

app/test-pmd/testpmd.c +3 -2
-				RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
+			RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
  				printf("Port %d can not be set back to "
-							"stopped\n", pi);
+							"stopped\n",
+					pi);

These are syntax changes, I think can keep these lines as it is.

#64 lore raw

Thanks Ferruh, fixed in v11, thanks.

Quoted 59 lines · Ferruh Yigit
Quoted 6 lines · Min Hu (Connor)

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

Quoted 8 lines

the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

Hi Connor,

I put some minor syntax comments, when they are fixed,

<...>

Quoted 4 lines

              if (diag != 0) {
-                if (rte_atomic16_cmpset(&(port->port_status), -                RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) -                    printf("Port %d can not be set back "

Quoted 8 lines

-                            "to stopped\n", pi); -                printf("Fail to configure port %d\n", pi); +                if (rte_atomic16_cmpset(
+                        &(port->port_status),
+                        RTE_PORT_HANDLING,
+                        RTE_PORT_STOPPED) == 0)
+                    printf("Port %d can not be set " +                           "back to stopped\n", pi);

It is OK to have long line for the strings, so it can be like printf("Port %d can not be set back to stopped\n",
    pi);

Quoted 2 lines

+                printf("Fail to configure port %d\n", +                    pi);

No need to update this line.

<...>

Quoted 8 lines

              /* Fail to setup rx queue, return */ if (rte_atomic16_cmpset(&(port->port_status), -                RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) +            RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) printf("Port %d can not be set back to " -                            "stopped\n", pi);
+                            "stopped\n",
+                    pi);

These are syntax changes, I think can keep these lines as it is.

.

#65 lore raw

PATCH v11 1/1 app/testpmd: support multi-process

In reply to Lijun Ou

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

---
v11:
* Fixed some minor syntax.

v10:
* Hid process type checks behind new functions.
* Added comments.

v9:
* Updated release notes and rst doc.
* Deleted deprecated codes.
* move macro and variable.

v8:
* Added warning info about queue numbers and process numbers.

v7:
* Fixed compiling error for unexpected unindent.

v6:
* Add rte flow description for multiple process.

v5:
* Fixed run_app.rst for multiple process description.
* Fix compiling error.

v4:
* Fixed minimum vlaue of Rxq or Txq in doc.

v3:
* Fixed compiling error using gcc10.0.

v2:
* Added document for this patch.

---
 app/test-pmd/cmdline.c                 |   6 ++
 app/test-pmd/config.c                  |  21 +++++-
 app/test-pmd/parameters.c              |  11 +++
 app/test-pmd/testpmd.c                 | 129 ++++++++++++++++++++++++++-------
 app/test-pmd/testpmd.h                 |   9 +++
 doc/guides/rel_notes/release_21_05.rst |   1 +
 doc/guides/testpmd_app_ug/run_app.rst  |  86 ++++++++++++++++++++++
 7 files changed, 236 insertions(+), 27 deletions(-)
app/test-pmd/cmdline.c +6 -0
index 5bf1497..e465824 100644
@@ -5354,6 +5354,12 @@ cmd_set_flush_rx_parsed(void *parsed_result,
 		__rte_unused void *data)
 {
 	struct cmd_set_flush_rx *res = parsed_result;
+
+	if (num_procs > 1 && (strcmp(res->mode, "on") == 0)) {
+		printf("multi-process doesn't support to flush rx queues.\n");
+		return;
+	}
+
 	no_flush_rx = (uint8_t)((strcmp(res->mode, "on") == 0) ? 0 : 1);
 }
 
app/test-pmd/config.c +20 -1
index d4b0e85..5d91335 100644
@@ -2859,6 +2859,8 @@ rss_fwd_config_setup(void)
 	queueid_t  rxq;
 	queueid_t  nb_q;
 	streamid_t  sm_id;
+	int start;
+	int end;
 
 	nb_q = nb_rxq;
 	if (nb_q > nb_txq)
@@ -2876,7 +2878,22 @@ rss_fwd_config_setup(void)
 	init_fwd_streams();
 
 	setup_fwd_config_of_each_lcore(&cur_fwd_config);
-	rxp = 0; rxq = 0;
+
+	if (proc_id > 0 && nb_q % num_procs)
+		printf("Warning! queue numbers should be multiple of "
+			"processes, or packet loss will happen.\n");
+
+	/**
+	 * In multi-process, All queues are allocated to different
+	 * processes based on num_procs and proc_id. For example:
+	 * if supports 4 queues(nb_q), 2 processes(num_procs),
+	 * the 0~1 queue for primary process.
+	 * the 2~3 queue for secondary process.
+	 */
+	start = proc_id * nb_q / num_procs;
+	end = start + nb_q / num_procs;
+	rxp = 0;
+	rxq = start;
 	for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
 		struct fwd_stream *fs;
 
@@ -2893,6 +2910,8 @@ rss_fwd_config_setup(void)
 			continue;
 		rxp = 0;
 		rxq++;
+		if (rxq >= end)
+			rxq = start;
 	}
 }
 
app/test-pmd/parameters.c +11 -0
index f3954c1..ece05c1 100644
@@ -508,6 +508,9 @@ parse_link_speed(int n)
 void
 launch_args_parse(int argc, char** argv)
 {
+#define PARAM_PROC_ID "proc-id"
+#define PARAM_NUM_PROCS "num-procs"
+
 	int n, opt;
 	char **argvopt;
 	int opt_idx;
@@ -625,6 +628,8 @@ launch_args_parse(int argc, char** argv)
 		{ "rx-mq-mode",                 1, 0, 0 },
 		{ "record-core-cycles",         0, 0, 0 },
 		{ "record-burst-stats",         0, 0, 0 },
+		{ PARAM_NUM_PROCS,              1, 0, 0 },
+		{ PARAM_PROC_ID,                1, 0, 0 },
 		{ 0, 0, 0, 0 },
 	};
 
@@ -1391,6 +1396,12 @@ launch_args_parse(int argc, char** argv)
 				record_core_cycles = 1;
 			if (!strcmp(lgopts[opt_idx].name, "record-burst-stats"))
 				record_burst_stats = 1;
+			if (strncmp(lgopts[opt_idx].name,
+				    PARAM_NUM_PROCS, 9) == 0)
+				num_procs = atoi(optarg);
+			if (strncmp(lgopts[opt_idx].name,
+				    PARAM_PROC_ID, 7) == 0)
+				proc_id = atoi(optarg);
 			break;
 		case 'h':
 			usage(argv[0]);
app/test-pmd/testpmd.c +103 -26

220 lines not on this page · raw message

app/test-pmd/testpmd.h +9 -0
index 36d8535..4c8f8bc 100644
@@ -630,6 +630,15 @@ extern struct mplsoudp_decap_conf mplsoudp_decap_conf;
 
 extern enum rte_eth_rx_mq_mode rx_mq_mode;
 
+extern int proc_id;
+extern unsigned int num_procs;
+
+static inline bool
+is_proc_primary(void)
+{
+	return rte_eal_process_type() == RTE_PROC_PRIMARY;
+}
+
 static inline unsigned int
 lcore_num(void)
 {
doc/guides/rel_notes/release_21_05.rst +1 -0
index 58272e1..353889a 100644
@@ -192,6 +192,7 @@ New Features
     ``show port (port_id) rxq (queue_id) desc used count``
   * Added command to dump internal representation information of single flow.
     ``flow dump (port_id) rule (rule_id)``
+  * Added support multi-process for testpmd.
 
 
 Removed Items
doc/guides/testpmd_app_ug/run_app.rst +86 -0

89 lines not on this page · raw message

Signature · 1 line
2.7.4
#66 lore raw
Quoted 10 lines · Min Hu (Connor)

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

Hi Connor,

I put some comments on the documentation, can you please check them?

Meanwhile, John, Marko can you please help on the doc review?

Thanks,
ferruh

Quoted 5 lines

<...>

Quoted 8 lines
app/test-pmd/testpmd.c +1 -1
@@ -2626,16 +2701,16 @@ start_port(portid_t pid)
  		cnt_pi++;
 
  		/* start port */
-		diag = rte_eth_dev_start(pi);
+		diag = eth_dev_start_mp(pi);
  		if (diag < 0) {
  			printf("Fail to start port %d: %s\n", pi,

app/test-pmd/testpmd.c start_port(portid_t pid) · line 2713

Quoted 10 lines
app/test-pmd/testpmd.c +3 -3
  			       rte_strerror(-diag));
 
  			/* Fail to setup rx queue, return */
  			if (rte_atomic16_cmpset(&(port->port_status),
-				RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
-				printf("Port %d can not be set back to "
-							"stopped\n", pi);
+			RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
+				printf("Port %d can not be set back to stopped\n",
+					pi);

Indentation is wrong.

<...>

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 549

Quoted 10 lines
doc/guides/testpmd_app_ug/run_app.rst +4 -0
@@ -543,3 +543,89 @@ The command line options are:
      bit 1 - two hairpin ports paired
      bit 0 - two hairpin ports loop
      The default value is 0. Hairpin will use single port mode and implicit Tx flow mode.
+
+
+Testpmd Support Multi Process Command-line Options
+--------------------------------------------------

What do you think making this a sub-section of the "Testpmd Command-line Options"?

And the section name can be "Testpmd Multi-Process Command-line Options"

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 551

Quoted 2 lines
doc/guides/testpmd_app_ug/run_app.rst +2 -0
+
+The following are the command-line options for the testpmd applications(support

Space before '('.

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 553

Quoted 2 lines
doc/guides/testpmd_app_ug/run_app.rst +2 -0
+multi process).They must be separated from the EAL options, shown in the previous
+section, with a ``--`` separator:

If this becomes sub-section of the "Testpmd Command-line Options", above will become duplication, and I think it can be simplified as:

"The following are the command-line options for testpmd multi-process support:"

Quoted 26 lines
doc/guides/testpmd_app_ug/run_app.rst +26 -0
+
+.. code-block:: console
+
+	primary process:
+	sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i --rxq=4 --txq=4 \
+        --num-procs=2 --proc-id=0
+
+	secondary process:
+	sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i --rxq=4 --txq=4 \
+        --num-procs=2 --proc-id=1
+
+The command line options are:
+
+*   ``--rxq=N``
+
+    Set the number of Rx queues per port to N. N is the sum of queues used by primary
+    and secondary process. Primary process and secondary process should have separate
+    queues, and each should occupy at least one queue. Where N should be the multiple
+    of number of processes.
+
+*   ``--txq=N``
+
+    Set the number of Tx queues per port to N. N is the sum of queues used by primary
+    and secondary process. Primary process and secondary process should have separate
+    queues, and each should occupy at least one queue. Where N should be the multiple
+    of number of processes.

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 589

Quoted 10 lines
doc/guides/testpmd_app_ug/run_app.rst +10 -0
+
+*   ``--num-procs=N``
+
+    The number of processes which will be used.
+
+*   ``--proc-id=id``
+
+    The id of the current process (id < num-procs). id should be different in primary
+    process and secondary process.
+

The famous question, does it start from '0' or '1'?

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 591

Quoted 2 lines
doc/guides/testpmd_app_ug/run_app.rst +2 -0
+Calculation rule for queue:
+All queues are allocated to different processes based on proc_num and proc_id.

Can highlight proc_num and proc_id, as ``proc_num`` and ``proc_id``.

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 592

Quoted 1 line
doc/guides/testpmd_app_ug/run_app.rst +1 -0
+Calculation rule for the Testpmd to allocate queues to each process:

Not sure if testpmd should start with uppercase.

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 595

Quoted 3 lines
doc/guides/testpmd_app_ug/run_app.rst +3 -0
+start(queue start id) = proc_id * nb_q / num_procs;
+end(queue end id) = start + nb_q / num_procs;
+

Can you put above into a code-block.

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 596

Quoted 1 line
doc/guides/testpmd_app_ug/run_app.rst +1 -0
+For example, if supports 4 txq and rxq

".., if testpmp supports 4 Tx and Rx queues"

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 605

Quoted 9 lines
doc/guides/testpmd_app_ug/run_app.rst +9 -0
+the 0~1 for primary process
+the 2~3 for secondary process
+
+The number of rings should be a multiple of the number of processes. If not,
+redundant queues will exist after queues are allocated to processes. After RSS is
+enabled, packet loss occurs when traffic is sent to all processes at the same time.
+Some traffic enters redundant queues and cannot be forwarded.
+
+Most dev ops is supported in primary and secondary process. While secondary process

"Most of the device operations are supported in ..."? "While the secondary process ..."?

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 615

Quoted 10 lines
doc/guides/testpmd_app_ug/run_app.rst +10 -0
+is not permitted to allocate or release shared memory, so some ops are not supported
+as follows:
+``dev_configure``
+``dev_start``
+``dev_stop``
+``rx_queue_setup``
+``tx_queue_setup``
+``rx_queue_release``
+``tx_queue_release``
+

This list displayed as single line in the html output, if the intention is to have them listed as been in the source, may need following update:

-as follows:
+as follows::
+

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 621

Quoted 6 lines
doc/guides/testpmd_app_ug/run_app.rst +6 -0
+So, any command from testpmd which calls those APIs will not be supported in secondary
+process, like:
+``port config all rxq|txq|rxd|txd <value>``
+``port config <port_id> rx_offload xxx on/off ``
+``port config <port_id> tx_offload xxx on/off``
+etc.

Same here for the list view.

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 623

Quoted 2 lines
doc/guides/testpmd_app_ug/run_app.rst +2 -0
+
+RTE_FLOW supported, it applies only on its own process on SW side, but all on HW size.

You may use "Flow API", instead of RTE_FLOW, so it becomes: "Flow API is supported."

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 625

Quoted 2 lines
doc/guides/testpmd_app_ug/run_app.rst +2 -0
+stats supported, stats will not change when one quit and start, As they share the same
+buffer to store the stats. 

The above stats related sentences looks like squeezed within the flow API related description, it may be good to move it.

Quoted 2 lines

Flow rules are maintained in process level: primary and secondary
+has its own flow list(but one flow list in HW). The two can see all the queues, so setting

Space before '('.

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 628

Quoted 2 lines
doc/guides/testpmd_app_ug/run_app.rst +2 -0
+the flow rules for the other is OK. Of course, io(receive or transmit packets) in the queue
+from others is not permitted.

I understand what you mean, but wording can be improved, can you please try to reword it?

doc/guides/testpmd_app_ug/run_app.rst

Quoted 2 lines
doc/guides/testpmd_app_ug/run_app.rst +2 -0
+
+RSS supported, Primary process and secondary process has separate queues to use, RSS

"RSS is supported"? And start 'primary' with lowercase.

Quoted 2 lines

+will work in their own queues whether primary and secondary process.

There is a chance that when RSS is enabled, the packet may end up in other process' queues?

#67 lore raw

Hi, Ferruh,

Quoted 136 lines · Ferruh Yigit
Quoted 10 lines · Min Hu (Connor)

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

Hi Connor,

I put some comments on the documentation, can you please check them?

Meanwhile, John, Marko can you please help on the doc review?

Thanks,
ferruh

Quoted 5 lines

<...>

Quoted 6 lines
app/test-pmd/testpmd.c +1 -1
@@ -2626,16 +2701,16 @@ start_port(portid_t pid)
           cnt_pi++;
           /* start port */
-        diag = rte_eth_dev_start(pi);
+        diag = eth_dev_start_mp(pi);
           if (diag < 0) {

app/test-pmd/testpmd.c start_port(portid_t pid) · line 2711

Quoted 10 lines
app/test-pmd/testpmd.c +3 -3
               printf("Fail to start port %d: %s\n", pi,
                      rte_strerror(-diag));
               /* Fail to setup rx queue, return */
               if (rte_atomic16_cmpset(&(port->port_status),
-                RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
-                printf("Port %d can not be set back to "
-                            "stopped\n", pi);
+            RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
+                printf("Port %d can not be set back to stopped\n",
+                    pi);

Indentation is wrong.

<...>

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 549

Quoted 10 lines
doc/guides/testpmd_app_ug/run_app.rst +0 -0
@@ -543,3 +543,89 @@ The command line options are:

      bit 1 - two hairpin ports paired
      bit 0 - two hairpin ports loop
      The default value is 0. Hairpin will use single port mode and implicit Tx flow mode.

doc/guides/testpmd_app_ug/run_app.rst +4 -0
+
+
+Testpmd Support Multi Process Command-line Options
+--------------------------------------------------

What do you think making this a sub-section of the "Testpmd Command-line Options"?

And the section name can be "Testpmd Multi-Process Command-line Options"

Quoted 3 lines
doc/guides/testpmd_app_ug/run_app.rst +2 -0
+
+The following are the command-line options for the testpmd 

applications(support

Space before '('.

Quoted 3 lines

+multi process).They must be separated from the EAL options, shown in the previous
+section, with a ``--`` separator:

If this becomes sub-section of the "Testpmd Command-line Options", above will become duplication, and I think it can be simplified as:

"The following are the command-line options for testpmd multi-process support:"

Quoted 42 lines
doc/guides/testpmd_app_ug/run_app.rst +16 -2
+
+.. code-block:: console
+
+    primary process:
+    sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i --rxq=4 
--txq=4 \
+        --num-procs=2 --proc-id=0
+
+    secondary process:
+    sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i --rxq=4 
--txq=4 \
+        --num-procs=2 --proc-id=1
+
+The command line options are:
+
+*   ``--rxq=N``
+
+    Set the number of Rx queues per port to N. N is the sum of queues 

used by primary
+    and secondary process. Primary process and secondary process should have separate
+    queues, and each should occupy at least one queue. Where N should be the multiple
+    of number of processes.

doc/guides/testpmd_app_ug/run_app.rst +4 -0
+
+*   ``--txq=N``
+
+    Set the number of Tx queues per port to N. N is the sum of queues 

used by primary
+    and secondary process. Primary process and secondary process should have separate
+    queues, and each should occupy at least one queue. Where N should be the multiple
+    of number of processes.

doc/guides/testpmd_app_ug/run_app.rst +8 -0
+
+*   ``--num-procs=N``
+
+    The number of processes which will be used.
+
+*   ``--proc-id=id``
+
+    The id of the current process (id < num-procs). id should be 
Quoted 3 lines

different in primary
+    process and secondary process.
+

The famous question, does it start from '0' or '1'?

Quoted 3 lines
doc/guides/testpmd_app_ug/run_app.rst +2 -0
+Calculation rule for queue:
+All queues are allocated to different processes based on proc_num and 

proc_id.

Can highlight proc_num and proc_id, as ``proc_num`` and ``proc_id``.

Quoted 1 line

+Calculation rule for the Testpmd to allocate queues to each process:

Quoted 8 lines

Not sure if testpmd should start with uppercase.

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 595

Quoted 3 lines
doc/guides/testpmd_app_ug/run_app.rst +3 -0
+start(queue start id) = proc_id * nb_q / num_procs;
+end(queue end id) = start + nb_q / num_procs;
+

Can you put above into a code-block.

Have already put above into a code-block in previous version.

Quoted 79 lines
Quoted 1 line

+For example, if supports 4 txq and rxq

".., if testpmp supports 4 Tx and Rx queues"

Quoted 12 lines
doc/guides/testpmd_app_ug/run_app.rst +4 -0
+the 0~1 for primary process
+the 2~3 for secondary process
+
+The number of rings should be a multiple of the number of processes. 

If not,
+redundant queues will exist after queues are allocated to processes. After RSS is
+enabled, packet loss occurs when traffic is sent to all processes at the same time.

doc/guides/testpmd_app_ug/run_app.rst +3 -0
+Some traffic enters redundant queues and cannot be forwarded.
+
+Most dev ops is supported in primary and secondary process. While 
Quoted 1 line

secondary process

"Most of the device operations are supported in ..."? "While the secondary process ..."?

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 615

Quoted 11 lines

+is not permitted to allocate or release shared memory, so some ops are not supported

doc/guides/testpmd_app_ug/run_app.rst +9 -0
+as follows:
+``dev_configure``
+``dev_start``
+``dev_stop``
+``rx_queue_setup``
+``tx_queue_setup``
+``rx_queue_release``
+``tx_queue_release``
+

This list displayed as single line in the html output, if the intention is to have them listed as been in the source, may need following update:

 -as follows:
 +as follows::
 +

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 621

Quoted 7 lines

+So, any command from testpmd which calls those APIs will not be supported in secondary

doc/guides/testpmd_app_ug/run_app.rst +5 -0
+process, like:
+``port config all rxq|txq|rxd|txd <value>``
+``port config <port_id> rx_offload xxx on/off ``
+``port config <port_id> tx_offload xxx on/off``
+etc.

Same here for the list view.

Quoted 3 lines
doc/guides/testpmd_app_ug/run_app.rst +2 -0
+
+RTE_FLOW supported, it applies only on its own process on SW side, 

but all on HW size.

You may use "Flow API", instead of RTE_FLOW, so it becomes: "Flow API is supported."

Quoted 3 lines

+stats supported, stats will not change when one quit and start, As they share the same
+buffer to store the stats.

The above stats related sentences looks like squeezed within the flow API related description, it may be good to move it.

Quoted 3 lines

Flow rules are maintained in process level: primary and secondary +has its own flow list(but one flow list in HW). The two can see all the queues, so setting

Space before '('.

Quoted 3 lines

+the flow rules for the other is OK. Of course, io(receive or transmit packets) in the queue
+from others is not permitted.

I understand what you mean, but wording can be improved, can you please try to reword it?

Quoted 3 lines
doc/guides/testpmd_app_ug/run_app.rst +2 -0
+
+RSS supported, Primary process and secondary process has separate 

queues to use, RSS

"RSS is supported"? And start 'primary' with lowercase.

Quoted 4 lines
Quoted 2 lines

+will work in their own queues whether primary and secondary process.

There is a chance that when RSS is enabled, the packet may end up in other process' queues?

When RSS is enabled, packet could end up in other process's queues, but primary and secondary receives packerts from their own queues, and they do not interfere with each other, do not conflict with each other.

Quoted 1 line

.

#68 lore raw

PATCH v12 1/1 app/testpmd: support multi-process

In reply to Lijun Ou

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

---
v12:
* Updated doc info.

v11:
* Fixed some minor syntax.

v10:
* Hid process type checks behind new functions.
* Added comments.

v9:
* Updated release notes and rst doc.
* Deleted deprecated codes.
* move macro and variable.

v8:
* Added warning info about queue numbers and process numbers.

v7:
* Fixed compiling error for unexpected unindent.

v6:
* Add rte flow description for multiple process.

v5:
* Fixed run_app.rst for multiple process description.
* Fix compiling error.

v4:
* Fixed minimum vlaue of Rxq or Txq in doc.

v3:
* Fixed compiling error using gcc10.0.

v2:
* Added document for this patch.

---
 app/test-pmd/cmdline.c                 |   6 ++
 app/test-pmd/config.c                  |  21 +++++-
 app/test-pmd/parameters.c              |  11 +++
 app/test-pmd/testpmd.c                 | 129 ++++++++++++++++++++++++++-------
 app/test-pmd/testpmd.h                 |   9 +++
 doc/guides/rel_notes/release_21_05.rst |   1 +
 doc/guides/testpmd_app_ug/run_app.rst  |  84 +++++++++++++++++++++
 7 files changed, 234 insertions(+), 27 deletions(-)
app/test-pmd/cmdline.c +6 -0
index ccaeefa..b6bf4cd 100644
@@ -5357,6 +5357,12 @@ cmd_set_flush_rx_parsed(void *parsed_result,
 		__rte_unused void *data)
 {
 	struct cmd_set_flush_rx *res = parsed_result;
+
+	if (num_procs > 1 && (strcmp(res->mode, "on") == 0)) {
+		printf("multi-process doesn't support to flush rx queues.\n");
+		return;
+	}
+
 	no_flush_rx = (uint8_t)((strcmp(res->mode, "on") == 0) ? 0 : 1);
 }
 
app/test-pmd/config.c +20 -1
index ccb9bd3..3c7a378 100644
@@ -2924,6 +2924,8 @@ rss_fwd_config_setup(void)
 	queueid_t  rxq;
 	queueid_t  nb_q;
 	streamid_t  sm_id;
+	int start;
+	int end;
 
 	nb_q = nb_rxq;
 	if (nb_q > nb_txq)
@@ -2941,7 +2943,22 @@ rss_fwd_config_setup(void)
 	init_fwd_streams();
 
 	setup_fwd_config_of_each_lcore(&cur_fwd_config);
-	rxp = 0; rxq = 0;
+
+	if (proc_id > 0 && nb_q % num_procs)
+		printf("Warning! queue numbers should be multiple of "
+			"processes, or packet loss will happen.\n");
+
+	/**
+	 * In multi-process, All queues are allocated to different
+	 * processes based on num_procs and proc_id. For example:
+	 * if supports 4 queues(nb_q), 2 processes(num_procs),
+	 * the 0~1 queue for primary process.
+	 * the 2~3 queue for secondary process.
+	 */
+	start = proc_id * nb_q / num_procs;
+	end = start + nb_q / num_procs;
+	rxp = 0;
+	rxq = start;
 	for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
 		struct fwd_stream *fs;
 
@@ -2958,6 +2975,8 @@ rss_fwd_config_setup(void)
 			continue;
 		rxp = 0;
 		rxq++;
+		if (rxq >= end)
+			rxq = start;
 	}
 }
 
app/test-pmd/parameters.c +11 -0
index f3954c1..ece05c1 100644
@@ -508,6 +508,9 @@ parse_link_speed(int n)
 void
 launch_args_parse(int argc, char** argv)
 {
+#define PARAM_PROC_ID "proc-id"
+#define PARAM_NUM_PROCS "num-procs"
+
 	int n, opt;
 	char **argvopt;
 	int opt_idx;
@@ -625,6 +628,8 @@ launch_args_parse(int argc, char** argv)
 		{ "rx-mq-mode",                 1, 0, 0 },
 		{ "record-core-cycles",         0, 0, 0 },
 		{ "record-burst-stats",         0, 0, 0 },
+		{ PARAM_NUM_PROCS,              1, 0, 0 },
+		{ PARAM_PROC_ID,                1, 0, 0 },
 		{ 0, 0, 0, 0 },
 	};
 
@@ -1391,6 +1396,12 @@ launch_args_parse(int argc, char** argv)
 				record_core_cycles = 1;
 			if (!strcmp(lgopts[opt_idx].name, "record-burst-stats"))
 				record_burst_stats = 1;
+			if (strncmp(lgopts[opt_idx].name,
+				    PARAM_NUM_PROCS, 9) == 0)
+				num_procs = atoi(optarg);
+			if (strncmp(lgopts[opt_idx].name,
+				    PARAM_PROC_ID, 7) == 0)
+				proc_id = atoi(optarg);
 			break;
 		case 'h':
 			usage(argv[0]);
app/test-pmd/testpmd.c +103 -26

220 lines not on this page · raw message

app/test-pmd/testpmd.h +9 -0
index 9530ec5..e2939ea 100644
@@ -632,6 +632,15 @@ extern enum rte_eth_rx_mq_mode rx_mq_mode;
 
 extern struct rte_flow_action_conntrack conntrack_context;
 
+extern int proc_id;
+extern unsigned int num_procs;
+
+static inline bool
+is_proc_primary(void)
+{
+	return rte_eal_process_type() == RTE_PROC_PRIMARY;
+}
+
 static inline unsigned int
 lcore_num(void)
 {
doc/guides/rel_notes/release_21_05.rst +1 -0
index 668fca8..cc88aea 100644
@@ -230,6 +230,7 @@ New Features
   * Added commands to construct conntrack context and relevant indirect
     action handle creation, update for conntrack action as well as conntrack
     item matching.
+  * Added support multi-process for testpmd.  
 
 * **Updated ipsec-secgw sample application.**
 
doc/guides/testpmd_app_ug/run_app.rst +84 -0

87 lines not on this page · raw message

Signature · 1 line
2.7.4
#69 lore raw

PATCH v13 1/1 app/testpmd: support multi-process

In reply to Lijun Ou

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

---
v13:
* Modified the doc syntax.

v12:
* Updated doc info.

v11:
* Fixed some minor syntax.

v10:
* Hid process type checks behind new functions.
* Added comments.

v9:
* Updated release notes and rst doc.
* Deleted deprecated codes.
* move macro and variable.

v8:
* Added warning info about queue numbers and process numbers.

v7:
* Fixed compiling error for unexpected unindent.

v6:
* Add rte flow description for multiple process.

v5:
* Fixed run_app.rst for multiple process description.
* Fix compiling error.

v4:
* Fixed minimum vlaue of Rxq or Txq in doc.

v3:
* Fixed compiling error using gcc10.0.

v2:
* Added document for this patch.

---
 app/test-pmd/cmdline.c                 |   6 ++
 app/test-pmd/config.c                  |  21 +++++-
 app/test-pmd/parameters.c              |  11 +++
 app/test-pmd/testpmd.c                 | 129 ++++++++++++++++++++++++++-------
 app/test-pmd/testpmd.h                 |   9 +++
 doc/guides/rel_notes/release_21_05.rst |   1 +
 doc/guides/testpmd_app_ug/run_app.rst  |  70 ++++++++++++++++++
 7 files changed, 220 insertions(+), 27 deletions(-)
app/test-pmd/cmdline.c +6 -0
index 12efbc0..f0fa6e8 100644
@@ -5450,6 +5450,12 @@ cmd_set_flush_rx_parsed(void *parsed_result,
 		__rte_unused void *data)
 {
 	struct cmd_set_flush_rx *res = parsed_result;
+
+	if (num_procs > 1 && (strcmp(res->mode, "on") == 0)) {
+		printf("multi-process doesn't support to flush rx queues.\n");
+		return;
+	}
+
 	no_flush_rx = (uint8_t)((strcmp(res->mode, "on") == 0) ? 0 : 1);
 }
 
app/test-pmd/config.c +20 -1
index e189062..9eb1fa7 100644
@@ -2971,6 +2971,8 @@ rss_fwd_config_setup(void)
 	queueid_t  rxq;
 	queueid_t  nb_q;
 	streamid_t  sm_id;
+	int start;
+	int end;
 
 	nb_q = nb_rxq;
 	if (nb_q > nb_txq)
@@ -2988,7 +2990,22 @@ rss_fwd_config_setup(void)
 	init_fwd_streams();
 
 	setup_fwd_config_of_each_lcore(&cur_fwd_config);
-	rxp = 0; rxq = 0;
+
+	if (proc_id > 0 && nb_q % num_procs)
+		printf("Warning! queue numbers should be multiple of "
+			"processes, or packet loss will happen.\n");
+
+	/**
+	 * In multi-process, All queues are allocated to different
+	 * processes based on num_procs and proc_id. For example:
+	 * if supports 4 queues(nb_q), 2 processes(num_procs),
+	 * the 0~1 queue for primary process.
+	 * the 2~3 queue for secondary process.
+	 */
+	start = proc_id * nb_q / num_procs;
+	end = start + nb_q / num_procs;
+	rxp = 0;
+	rxq = start;
 	for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
 		struct fwd_stream *fs;
 
@@ -3005,6 +3022,8 @@ rss_fwd_config_setup(void)
 			continue;
 		rxp = 0;
 		rxq++;
+		if (rxq >= end)
+			rxq = start;
 	}
 }
 
app/test-pmd/parameters.c +11 -0
index f3954c1..ece05c1 100644
@@ -508,6 +508,9 @@ parse_link_speed(int n)
 void
 launch_args_parse(int argc, char** argv)
 {
+#define PARAM_PROC_ID "proc-id"
+#define PARAM_NUM_PROCS "num-procs"
+
 	int n, opt;
 	char **argvopt;
 	int opt_idx;
@@ -625,6 +628,8 @@ launch_args_parse(int argc, char** argv)
 		{ "rx-mq-mode",                 1, 0, 0 },
 		{ "record-core-cycles",         0, 0, 0 },
 		{ "record-burst-stats",         0, 0, 0 },
+		{ PARAM_NUM_PROCS,              1, 0, 0 },
+		{ PARAM_PROC_ID,                1, 0, 0 },
 		{ 0, 0, 0, 0 },
 	};
 
@@ -1391,6 +1396,12 @@ launch_args_parse(int argc, char** argv)
 				record_core_cycles = 1;
 			if (!strcmp(lgopts[opt_idx].name, "record-burst-stats"))
 				record_burst_stats = 1;
+			if (strncmp(lgopts[opt_idx].name,
+				    PARAM_NUM_PROCS, 9) == 0)
+				num_procs = atoi(optarg);
+			if (strncmp(lgopts[opt_idx].name,
+				    PARAM_PROC_ID, 7) == 0)
+				proc_id = atoi(optarg);
 			break;
 		case 'h':
 			usage(argv[0]);
app/test-pmd/testpmd.c +103 -26

220 lines not on this page · raw message

app/test-pmd/testpmd.h +9 -0
index 6ca872d..3318e8f 100644
@@ -632,6 +632,15 @@ extern enum rte_eth_rx_mq_mode rx_mq_mode;
 
 extern struct rte_flow_action_conntrack conntrack_context;
 
+extern int proc_id;
+extern unsigned int num_procs;
+
+static inline bool
+is_proc_primary(void)
+{
+	return rte_eal_process_type() == RTE_PROC_PRIMARY;
+}
+
 static inline unsigned int
 lcore_num(void)
 {
doc/guides/rel_notes/release_21_05.rst +1 -0
index b36c120..9361332 100644
@@ -235,6 +235,7 @@ New Features
     ``port cleanup (port_id) txq (queue_id) (free_cnt)``
   * Added command to show link flow control info.
     ``show port (port_id) flow_ctrl``
+  * Added support multi-process for testpmd.
 
 * **Updated ipsec-secgw sample application.**
 
doc/guides/testpmd_app_ug/run_app.rst +70 -0

73 lines not on this page · raw message

Signature · 1 line
2.7.4
#70 lore raw

@Thomas, @Ferruh, please, see question below.

Quoted 70 lines · Min Hu (Connor)

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

---
v13:
* Modified the doc syntax.

v12:
* Updated doc info.

v11:
* Fixed some minor syntax.

v10:
* Hid process type checks behind new functions.
* Added comments.

v9:
* Updated release notes and rst doc.
* Deleted deprecated codes.
* move macro and variable.

v8:
* Added warning info about queue numbers and process numbers.

v7:
* Fixed compiling error for unexpected unindent.

v6:
* Add rte flow description for multiple process.

v5:
* Fixed run_app.rst for multiple process description.
* Fix compiling error.

v4:
* Fixed minimum vlaue of Rxq or Txq in doc.

v3:
* Fixed compiling error using gcc10.0.

v2:
* Added document for this patch.

---
 app/test-pmd/cmdline.c                 |   6 ++
 app/test-pmd/config.c                  |  21 +++++-
 app/test-pmd/parameters.c              |  11 +++
 app/test-pmd/testpmd.c                 | 129 ++++++++++++++++++++++++++-------
 app/test-pmd/testpmd.h                 |   9 +++
 doc/guides/rel_notes/release_21_05.rst |   1 +
 doc/guides/testpmd_app_ug/run_app.rst  |  70 ++++++++++++++++++
 7 files changed, 220 insertions(+), 27 deletions(-)
app/test-pmd/cmdline.c +0 -0
index 12efbc0..f0fa6e8 100644

app/test-pmd/cmdline.c cmd_set_flush_rx_parsed(void *parsed_result, · line 5455

Quoted 6 lines
app/test-pmd/cmdline.c +3 -0
@@ -5450,6 +5450,12 @@ cmd_set_flush_rx_parsed(void *parsed_result,
 		__rte_unused void *data)
 {
 	struct cmd_set_flush_rx *res = parsed_result;
+
+	if (num_procs > 1 && (strcmp(res->mode, "on") == 0)) {
+		printf("multi-process doesn't support to flush rx queues.\n");

rx -> Rx

Quoted 20 lines
app/test-pmd/cmdline.c +3 -0
+		return;
+	}
+
 	no_flush_rx = (uint8_t)((strcmp(res->mode, "on") == 0) ? 0 : 1);
 }
 
app/test-pmd/config.c +2 -0
index e189062..9eb1fa7 100644
@@ -2971,6 +2971,8 @@ rss_fwd_config_setup(void)
 	queueid_t  rxq;
 	queueid_t  nb_q;
 	streamid_t  sm_id;
+	int start;
+	int end;
 
 	nb_q = nb_rxq;
 	if (nb_q > nb_txq)

app/test-pmd/config.c rss_fwd_config_setup(void) · line 2994

Quoted 6 lines
app/test-pmd/config.c +2 -1
@@ -2988,7 +2990,22 @@ rss_fwd_config_setup(void)
 	init_fwd_streams();
 
 	setup_fwd_config_of_each_lcore(&cur_fwd_config);
-	rxp = 0; rxq = 0;
+
+	if (proc_id > 0 && nb_q % num_procs)

Please, compare result with 0 explicitly.

app/test-pmd/config.c rss_fwd_config_setup(void) · line 2996

Quoted 2 lines
app/test-pmd/config.c +2 -0
+		printf("Warning! queue numbers should be multiple of "
+			"processes, or packet loss will happen.\n");

Do not split format string across multiple lines.

Frankly speaking I don't undertand why. Why is it impossible to serve 2 queues in the first process and 1 queue in the second process if 3 queues and 2 processes are configured.
I think RSS redirection table can perfectly do it.

Quoted 48 lines
app/test-pmd/config.c +14 -0
+
+	/**
+	 * In multi-process, All queues are allocated to different
+	 * processes based on num_procs and proc_id. For example:
+	 * if supports 4 queues(nb_q), 2 processes(num_procs),
+	 * the 0~1 queue for primary process.
+	 * the 2~3 queue for secondary process.
+	 */
+	start = proc_id * nb_q / num_procs;
+	end = start + nb_q / num_procs;
+	rxp = 0;
+	rxq = start;
 	for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
 		struct fwd_stream *fs;
 
@@ -3005,6 +3022,8 @@ rss_fwd_config_setup(void)
 			continue;
 		rxp = 0;
 		rxq++;
+		if (rxq >= end)
+			rxq = start;
 	}
 }
 
app/test-pmd/parameters.c +5 -0
index f3954c1..ece05c1 100644
@@ -508,6 +508,9 @@ parse_link_speed(int n)
 void
 launch_args_parse(int argc, char** argv)
 {
+#define PARAM_PROC_ID "proc-id"
+#define PARAM_NUM_PROCS "num-procs"
+
 	int n, opt;
 	char **argvopt;
 	int opt_idx;
@@ -625,6 +628,8 @@ launch_args_parse(int argc, char** argv)
 		{ "rx-mq-mode",                 1, 0, 0 },
 		{ "record-core-cycles",         0, 0, 0 },
 		{ "record-burst-stats",         0, 0, 0 },
+		{ PARAM_NUM_PROCS,              1, 0, 0 },
+		{ PARAM_PROC_ID,                1, 0, 0 },
 		{ 0, 0, 0, 0 },
 	};
 

app/test-pmd/parameters.c launch_args_parse(int argc, char** argv) · line 1400

Quoted 5 lines
app/test-pmd/parameters.c +2 -0
@@ -1391,6 +1396,12 @@ launch_args_parse(int argc, char** argv)
 				record_core_cycles = 1;
 			if (!strcmp(lgopts[opt_idx].name, "record-burst-stats"))
 				record_burst_stats = 1;
+			if (strncmp(lgopts[opt_idx].name,
+				    PARAM_NUM_PROCS, 9) == 0)

I strongly dislike 9 here and 7 below. Why is strncmp() used here, but just strcmp() is used for all other options. It makes the code inconsistent.

Quoted 12 lines
app/test-pmd/parameters.c +4 -0
+				num_procs = atoi(optarg);
+			if (strncmp(lgopts[opt_idx].name,
+				    PARAM_PROC_ID, 7) == 0)
+				proc_id = atoi(optarg);
 			break;
 		case 'h':
 			usage(argv[0]);
app/test-pmd/testpmd.c +0 -0
index d4be23f..afa2a6b 100644

app/test-pmd/testpmd.c enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS; · line 522

Quoted 5 lines
app/test-pmd/testpmd.c +2 -0
@@ -518,6 +518,62 @@ enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS;
  */
 uint32_t eth_link_speed;
 
+/*
+ * Id of the current process in multi-process, used to

Id -> ID in accordance with devtools/words-case.txt

Quoted 33 lines
app/test-pmd/testpmd.c +33 -0
+ * configure the queues to be polled.
+ */
+int proc_id;
+
+/*
+ * Number of processes in multi-process, used to
+ * configure the queues to be polled.
+ */
+unsigned int num_procs = 1;
+
+static int
+eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
+		      const struct rte_eth_conf *dev_conf)
+{
+	if (is_proc_primary())
+		return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q,
+					dev_conf);
+	return 0;
+}
+
+static int
+eth_dev_start_mp(uint16_t port_id)
+{
+	if (is_proc_primary())
+		return rte_eth_dev_start(port_id);
+
+	return 0;
+}
+
+static int
+eth_dev_stop_mp(uint16_t port_id)
+{
+	if (is_proc_primary())

app/test-pmd/testpmd.c enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS; · line 565

Quoted 10 lines
app/test-pmd/testpmd.c +10 -0
+		return rte_eth_dev_stop(port_id);
+
+	return 0;
+}
+
+static void
+mempool_free_mp(struct rte_mempool *mp)
+{
+	if (is_proc_primary())
+		return rte_mempool_free(mp);

As far as I remember some compilers do not like it for void. Just remove 'return'.

Quoted 15 lines
app/test-pmd/testpmd.c +11 -0
+}
+
+static int
+eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu)
+{
+	if (is_proc_primary())
+		return rte_eth_dev_set_mtu(port_id, mtu);
+
+	return 0;
+}
+
 /* Forward function declarations */
 static void setup_attached_port(portid_t pi);
 static void check_all_ports_link_status(uint32_t port_mask);

app/test-pmd/testpmd.c mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, · line 1038

Quoted 6 lines
app/test-pmd/testpmd.c +3 -0
@@ -977,6 +1033,11 @@ mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 	mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 	mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx);
 
+	if (!is_proc_primary()) {
+		rte_mp = rte_mempool_lookup(pool_name);
+		goto err;

It looks like error path, but it works in the case of success as well. Looks confusing.

Quoted 24 lines
app/test-pmd/testpmd.c +10 -3
+	}
+
 	TESTPMD_LOG(INFO,
 		"create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
 		pool_name, nb_mbuf, mbuf_seg_size, socket_id);
@@ -1059,9 +1120,14 @@ mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 
 err:
 	if (rte_mp == NULL) {
-		rte_exit(EXIT_FAILURE,
-			"Creation of mbuf pool for socket %u failed: %s\n",
-			socket_id, rte_strerror(rte_errno));
+		if (is_proc_primary())
+			rte_exit(EXIT_FAILURE,
+				"Creation of mbuf pool for socket %u failed: %s\n",
+				socket_id, rte_strerror(rte_errno));
+		else
+			rte_exit(EXIT_FAILURE,
+				"Get mbuf pool for socket %u failed: %s\n",
+				socket_id, rte_strerror(rte_errno));
 	} else if (verbose_level > 0) {
 		rte_mempool_dump(stdout, rte_mp);
 	}

app/test-pmd/testpmd.c flush_fwd_rx_queues(void) · line 2072

Quoted 5 lines
app/test-pmd/testpmd.c +2 -0
@@ -2002,6 +2068,12 @@ flush_fwd_rx_queues(void)
 	uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
 	uint64_t timer_period;
 
+	if (num_procs > 1) {
+		printf("multi-process not support for flushing fwd rx "

rx -> Rx
also it is better to avoid like split.

Quoted 18 lines
app/test-pmd/testpmd.c +8 -3
+		       "queues, skip the below lines and return.\n");
+		return;
+	}
+
 	/* convert to number of cycles */
 	timer_period = rte_get_timer_hz(); /* 1 second timeout */
 
@@ -2511,21 +2583,24 @@ start_port(portid_t pid)
 				return -1;
 			}
 			/* configure port */
-			diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
-						     nb_txq + nb_hairpinq,
-						     &(port->dev_conf));
+			diag = eth_dev_configure_mp(pi,
+					     nb_rxq + nb_hairpinq,
+					     nb_txq + nb_hairpinq,
+					     &(port->dev_conf));

app/test-pmd/testpmd.c start_port(portid_t pid) · line 2595

Quoted 10 lines
app/test-pmd/testpmd.c +5 -4
 			if (diag != 0) {
-				if (rte_atomic16_cmpset(&(port->port_status),
-				RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
-					printf("Port %d can not be set back "
-							"to stopped\n", pi);
+				if (rte_atomic16_cmpset(
+						&(port->port_status),
+						RTE_PORT_HANDLING,
+						RTE_PORT_STOPPED) == 0)
+					printf("Port %d can not be set back to stopped\n",

can not -> cannot (since you touch the line anyway)

Quoted 13 lines
app/test-pmd/testpmd.c +2 -1
+						pi);
 				printf("Fail to configure port %d\n", pi);
 				/* try to reconfigure port next time */
 				port->need_reconfig = 1;
 				return -1;
 			}
 		}
-		if (port->need_reconfig_queues > 0) {
+		if (port->need_reconfig_queues > 0 && is_proc_primary()) {
 			port->need_reconfig_queues = 0;
 			/* setup tx queues */
 			for (qi = 0; qi < nb_txq; qi++) {

app/test-pmd/testpmd.c start_port(portid_t pid) · line 2626

Quoted 6 lines
app/test-pmd/testpmd.c +1 -2
@@ -2548,8 +2623,8 @@ start_port(portid_t pid)
 				if (rte_atomic16_cmpset(&(port->port_status),
 							RTE_PORT_HANDLING,
 							RTE_PORT_STOPPED) == 0)
-					printf("Port %d can not be set back "
-							"to stopped\n", pi);
+					printf("Port %d can not be set back to stopped\n",

can not -> cannot

Quoted 11 lines
app/test-pmd/testpmd.c +2 -1
+						pi);
 				printf("Fail to configure port %d tx queues\n",
 				       pi);
 				/* try to reconfigure queues next time */
@@ -2626,16 +2701,16 @@ start_port(portid_t pid)
 		cnt_pi++;
 
 		/* start port */
-		diag = rte_eth_dev_start(pi);
+		diag = eth_dev_start_mp(pi);
 		if (diag < 0) {

app/test-pmd/testpmd.c start_port(portid_t pid) · line 2712

Quoted 10 lines
app/test-pmd/testpmd.c +2 -3
 			printf("Fail to start port %d: %s\n", pi,
 			       rte_strerror(-diag));
 
 			/* Fail to setup rx queue, return */
 			if (rte_atomic16_cmpset(&(port->port_status),
-				RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
-				printf("Port %d can not be set back to "
-							"stopped\n", pi);
+			RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
+				printf("Port %d can not be set back to stopped\n",

can not -> cannot

Quoted 91 lines
app/test-pmd/testpmd.c +12 -9
+				       pi);
 			continue;
 		}
 
@@ -2765,7 +2840,7 @@ stop_port(portid_t pid)
 		if (port->flow_list)
 			port_flow_flush(pi);
 
-		if (rte_eth_dev_stop(pi) != 0)
+		if (eth_dev_stop_mp(pi) != 0)
 			RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n",
 				pi);
 
@@ -2834,8 +2909,10 @@ close_port(portid_t pid)
 			continue;
 		}
 
-		port_flow_flush(pi);
-		rte_eth_dev_close(pi);
+		if (is_proc_primary()) {
+			port_flow_flush(pi);
+			rte_eth_dev_close(pi);
+		}
 	}
 
 	remove_invalid_ports();
@@ -3101,7 +3178,7 @@ pmd_test_exit(void)
 	}
 	for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
 		if (mempools[i])
-			rte_mempool_free(mempools[i]);
+			mempool_free_mp(mempools[i]);
 	}
 
 	printf("\nBye...\n");
@@ -3432,7 +3509,7 @@ update_jumbo_frame_offload(portid_t portid)
 	 * if unset do it here
 	 */
 	if ((rx_offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) == 0) {
-		ret = rte_eth_dev_set_mtu(portid,
+		ret = eth_dev_set_mtu_mp(portid,
 				port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead);
 		if (ret)
 			printf("Failed to set MTU to %u for port %u\n",
@@ -3622,6 +3699,10 @@ init_port_dcb_config(portid_t pid,
 	int retval;
 	uint16_t i;
 
+	if (num_procs > 1) {
+		printf("The multi-process feature doesn't support dcb.\n");
+		return -ENOTSUP;
+	}
 	rte_port = &ports[pid];
 
 	memset(&port_conf, 0, sizeof(struct rte_eth_conf));
@@ -3787,10 +3868,6 @@ main(int argc, char** argv)
 		rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
 			 rte_strerror(rte_errno));
 
-	if (rte_eal_process_type() == RTE_PROC_SECONDARY)
-		rte_exit(EXIT_FAILURE,
-			 "Secondary process type not supported.\n");
-
 	ret = register_eth_event_callback();
 	if (ret != 0)
 		rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
app/test-pmd/testpmd.h +9 -0
index 6ca872d..3318e8f 100644
@@ -632,6 +632,15 @@ extern enum rte_eth_rx_mq_mode rx_mq_mode;
 
 extern struct rte_flow_action_conntrack conntrack_context;
 
+extern int proc_id;
+extern unsigned int num_procs;
+
+static inline bool
+is_proc_primary(void)
+{
+	return rte_eal_process_type() == RTE_PROC_PRIMARY;
+}
+
 static inline unsigned int
 lcore_num(void)
 {
doc/guides/rel_notes/release_21_05.rst +0 -0
index b36c120..9361332 100644

doc/guides/rel_notes/release_21_05.rst New Features · line 238

Quoted 4 lines
doc/guides/rel_notes/release_21_05.rst +1 -0
@@ -235,6 +235,7 @@ New Features
     ``port cleanup (port_id) txq (queue_id) (free_cnt)``
   * Added command to show link flow control info.
     ``show port (port_id) flow_ctrl``
+  * Added support multi-process for testpmd.

Please, rebase to 21.08.

Quoted 16 lines
doc/guides/rel_notes/release_21_05.rst +0 -0
 * **Updated ipsec-secgw sample application.**
 
doc/guides/testpmd_app_ug/run_app.rst +5 -0
index d062165..74efa4f 100644
@@ -543,3 +543,73 @@ The command line options are:
     bit 1 - two hairpin ports paired
     bit 0 - two hairpin ports loop
     The default value is 0. Hairpin will use single port mode and implicit Tx flow mode.
+
+Testpmd Multi-Process Command-line Options
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The following are the command-line options for testpmd multi-process support:

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 560

Quoted 10 lines
doc/guides/testpmd_app_ug/run_app.rst +10 -0
+
+.. code-block:: console
+
+	primary process:
+	sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i --rxq=4 --txq=4 \
+        --num-procs=2 --proc-id=0
+
+	secondary process:
+	sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i --rxq=4 --txq=4 \
+        --num-procs=2 --proc-id=1

Do we really need --rxq=4 --txq=4 in secodary processes? Can't we get it from already configured device in primary process?

Same question is applicable to --num-procs. May be testpmd can publish in shared memory? If yes, I'm OK to either do it in the patch or defer as a later improvement.

Quoted 26 lines
doc/guides/testpmd_app_ug/run_app.rst +26 -0
+
+The command line options are:
+
+*   ``--num-procs=N``
+
+    The number of processes which will be used.
+
+*   ``--proc-id=id``
+
+    The id of the current process (id < num-procs). id should be different in primary
+    process and secondary process, which starts from '0'.
+
+Calculation rule for queue:
+All queues are allocated to different processes based on ``proc_num`` and ``proc_id``.
+Calculation rule for the testpmd to allocate queues to each process:
+start(queue start id) = proc_id * nb_q / num_procs;
+end(queue end id) = start + nb_q / num_procs;
+
+For example, if testpmd supports 4 Tx and Rx queues
+the 0~1 for primary process
+the 2~3 for secondary process
+
+The number of rings should be a multiple of the number of processes. If not,
+redundant queues will exist after queues are allocated to processes. After RSS is
+enabled, packet loss occurs when traffic is sent to all processes at the same time.
+Some traffic enters redundant queues and cannot be forwarded.

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 596

Quoted 10 lines
doc/guides/testpmd_app_ug/run_app.rst +10 -0
+
+All the dev ops is supported in primary process. While secondary process is not permitted
+to allocate or release shared memory, so some ops are not supported as follows::
+``dev_configure``
+``dev_start``
+``dev_stop``
+``rx_queue_setup``
+``tx_queue_setup``
+``rx_queue_release``
+``tx_queue_release``

@Thomas, @Ferrh, shouldn't it be handled on ethdev level as well if it is really that strict.

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 605

Quoted 9 lines
doc/guides/testpmd_app_ug/run_app.rst +9 -0
+
+So, any command from testpmd which calls those APIs will not be supported in secondary
+process, like::
+``port config all rxq|txq|rxd|txd <value>``
+``port config <port_id> rx_offload xxx on/off ``
+``port config <port_id> tx_offload xxx on/off``
+etc.
+
+Flow API is supported, it applies only on its own process on SW side, but all on HW side.

Sorry, I don't understand it.

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 615

Quoted 10 lines
doc/guides/testpmd_app_ug/run_app.rst +10 -0
+
+Stats is supported, stats will not change when one quit and start, as they share the same
+buffer to store the stats. Flow rules are maintained in process level: primary and secondary
+has its own flow list (but one flow list in HW). The two can see all the queues, so setting
+the flow rules for the other is OK. But in the testpmd primary process receiving or transmitting
+packets from the queue allocated for secondary process is not permitted, and same for
+secondary process.
+
+RSS is supported, primary process and secondary process has separate queues to use, RSS
+will work in their own queues whether primary or secondary process.

For me it sounds like secondary process has own RSS configuration. If so, it is false. I guess I simply misunderstand above paragraph.

#71 lore raw

08/06/2021 10:42, Andrew Rybchenko:

Quoted 3 lines · Min Hu (Connor)
Quoted 2 lines
doc/guides/testpmd_app_ug/run_app.rst +0 -0

[...]

Quoted 4 lines
Quoted 4 lines
doc/guides/testpmd_app_ug/run_app.rst +4 -0
+All the dev ops is supported in primary process. While secondary process is not permitted
+to allocate or release shared memory, so some ops are not supported as follows::
+``dev_configure``
+``dev_start``
Quoted 8 lines
Quoted 5 lines

+``dev_stop``
+``rx_queue_setup``
+``tx_queue_setup``
+``rx_queue_release``
+``tx_queue_release``

@Thomas, @Ferrh, shouldn't it be handled on ethdev level as well if it is really that strict.

Yes it should be documented at ethdev level, not testpmd. I think it was kept fuzzy for too long.

#72 lore raw
Quoted 18 lines · Thomas Monjalon

08/06/2021 10:42, Andrew Rybchenko:

Quoted 3 lines · Min Hu (Connor)
Quoted 2 lines
doc/guides/testpmd_app_ug/run_app.rst +0 -0

[...]

Quoted 4 lines
Quoted 4 lines
doc/guides/testpmd_app_ug/run_app.rst +4 -0
+All the dev ops is supported in primary process. While secondary process is not permitted
+to allocate or release shared memory, so some ops are not supported as follows::
+``dev_configure``
+``dev_start``
Quoted 8 lines
Quoted 5 lines

+``dev_stop``
+``rx_queue_setup``
+``tx_queue_setup``
+``rx_queue_release``
+``tx_queue_release``

@Thomas, @Ferrh, shouldn't it be handled on ethdev level as well if it is really that strict.

Quoted 2 lines

Yes it should be documented at ethdev level, not testpmd. I think it was kept fuzzy for too long.

To document is good, but I'm talking about more -
add checks in corresponding API functions and
return error.

#73 lore raw

08/06/2021 12:39, Andrew Rybchenko:

Quoted 19 lines · Thomas Monjalon
Quoted 18 lines

08/06/2021 10:42, Andrew Rybchenko:

Quoted 3 lines · Min Hu (Connor)
Quoted 2 lines
doc/guides/testpmd_app_ug/run_app.rst +0 -0

[...]

Quoted 12 lines

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 596

Quoted 9 lines
doc/guides/testpmd_app_ug/run_app.rst +9 -0
+All the dev ops is supported in primary process. While secondary process is not permitted
+to allocate or release shared memory, so some ops are not supported as follows::
+``dev_configure``
+``dev_start``
+``dev_stop``
+``rx_queue_setup``
+``tx_queue_setup``
+``rx_queue_release``
+``tx_queue_release``

@Thomas, @Ferrh, shouldn't it be handled on ethdev level as well if it is really that strict.

Quoted 6 lines
Quoted 2 lines

Yes it should be documented at ethdev level, not testpmd. I think it was kept fuzzy for too long.

To document is good, but I'm talking about more -
add checks in corresponding API functions and
return error.

Yes I am OK with adding checks of the running process.

#74 lore raw
In reply to Andrew Rybchenko
Quoted 19 lines · Andrew Rybchenko
Quoted 18 lines · Thomas Monjalon

08/06/2021 10:42, Andrew Rybchenko:

Quoted 3 lines · Min Hu (Connor)
Quoted 2 lines
doc/guides/testpmd_app_ug/run_app.rst +0 -0

[...]

Quoted 12 lines

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 596

Quoted 9 lines
doc/guides/testpmd_app_ug/run_app.rst +9 -0
+All the dev ops is supported in primary process. While secondary process is not permitted
+to allocate or release shared memory, so some ops are not supported as follows::
+``dev_configure``
+``dev_start``
+``dev_stop``
+``rx_queue_setup``
+``tx_queue_setup``
+``rx_queue_release``
+``tx_queue_release``

@Thomas, @Ferrh, shouldn't it be handled on ethdev level as well if it is really that strict.

Quoted 7 lines
Quoted 2 lines

Yes it should be documented at ethdev level, not testpmd. I think it was kept fuzzy for too long.

To document is good, but I'm talking about more -
add checks in corresponding API functions and
return error.

+Anatoly as multi process maintainer.

There are already some PMDs have these checks in the PMD level.

But current approach is providing more flexibility, if application takes care of the synchronization, is there any reason some of above APIs can't be used by the secondary process?

I am not sure if this flexibility is used or needed, if not needed +1 have the checks in the ethdev layer and remove the PMD ones.

#75 lore raw
In reply to Andrew Rybchenko

Hi, Andrew,
see replies below, and others without no reply will be fixed in v14.

Quoted 116 lines · Andrew Rybchenko

@Thomas, @Ferruh, please, see question below.

Quoted 70 lines · Min Hu (Connor)

This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0

the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1

---
v13:
* Modified the doc syntax.

v12:
* Updated doc info.

v11:
* Fixed some minor syntax.

v10:
* Hid process type checks behind new functions.
* Added comments.

v9:
* Updated release notes and rst doc.
* Deleted deprecated codes.
* move macro and variable.

v8:
* Added warning info about queue numbers and process numbers.

v7:
* Fixed compiling error for unexpected unindent.

v6:
* Add rte flow description for multiple process.

v5:
* Fixed run_app.rst for multiple process description.
* Fix compiling error.

v4:
* Fixed minimum vlaue of Rxq or Txq in doc.

v3:
* Fixed compiling error using gcc10.0.

v2:
* Added document for this patch.

---
  app/test-pmd/cmdline.c                 |   6 ++
  app/test-pmd/config.c                  |  21 +++++-
  app/test-pmd/parameters.c              |  11 +++
  app/test-pmd/testpmd.c                 | 129 ++++++++++++++++++++++++++-------
  app/test-pmd/testpmd.h                 |   9 +++
  doc/guides/rel_notes/release_21_05.rst |   1 +
  doc/guides/testpmd_app_ug/run_app.rst  |  70 ++++++++++++++++++
  7 files changed, 220 insertions(+), 27 deletions(-)
app/test-pmd/cmdline.c +0 -0
index 12efbc0..f0fa6e8 100644

app/test-pmd/cmdline.c cmd_set_flush_rx_parsed(void *parsed_result, · line 5455

Quoted 6 lines
app/test-pmd/cmdline.c +3 -0
@@ -5450,6 +5450,12 @@ cmd_set_flush_rx_parsed(void *parsed_result,
  		__rte_unused void *data)
  {
  	struct cmd_set_flush_rx *res = parsed_result;
+
+	if (num_procs > 1 && (strcmp(res->mode, "on") == 0)) {
+		printf("multi-process doesn't support to flush rx queues.\n");

rx -> Rx

Quoted 20 lines
app/test-pmd/cmdline.c +3 -0
+		return;
+	}
+
  	no_flush_rx = (uint8_t)((strcmp(res->mode, "on") == 0) ? 0 : 1);
  }
 
app/test-pmd/config.c +2 -0
index e189062..9eb1fa7 100644
@@ -2971,6 +2971,8 @@ rss_fwd_config_setup(void)
  	queueid_t  rxq;
  	queueid_t  nb_q;
  	streamid_t  sm_id;
+	int start;
+	int end;
 
  	nb_q = nb_rxq;
  	if (nb_q > nb_txq)

app/test-pmd/config.c rss_fwd_config_setup(void) · line 2994

Quoted 6 lines
app/test-pmd/config.c +2 -1
@@ -2988,7 +2990,22 @@ rss_fwd_config_setup(void)
  	init_fwd_streams();
 
  	setup_fwd_config_of_each_lcore(&cur_fwd_config);
-	rxp = 0; rxq = 0;
+
+	if (proc_id > 0 && nb_q % num_procs)

Please, compare result with 0 explicitly.

app/test-pmd/config.c rss_fwd_config_setup(void) · line 2996

Quoted 2 lines
app/test-pmd/config.c +2 -0
+		printf("Warning! queue numbers should be multiple of "
+			"processes, or packet loss will happen.\n");

Do not split format string across multiple lines.

Quoted 5 lines

Frankly speaking I don't undertand why. Why is it impossible to serve 2 queues in the first process and 1 queue in the second process if 3 queues and 2 processes are configured.
I think RSS redirection table can perfectly do it.

Well, currently, my patch is one design implementation. I think this can be done for later improvemnet.

Quoted 389 lines
Quoted 48 lines
app/test-pmd/config.c +14 -0
+
+	/**
+	 * In multi-process, All queues are allocated to different
+	 * processes based on num_procs and proc_id. For example:
+	 * if supports 4 queues(nb_q), 2 processes(num_procs),
+	 * the 0~1 queue for primary process.
+	 * the 2~3 queue for secondary process.
+	 */
+	start = proc_id * nb_q / num_procs;
+	end = start + nb_q / num_procs;
+	rxp = 0;
+	rxq = start;
  	for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
  		struct fwd_stream *fs;
 
@@ -3005,6 +3022,8 @@ rss_fwd_config_setup(void)
  			continue;
  		rxp = 0;
  		rxq++;
+		if (rxq >= end)
+			rxq = start;
  	}
  }
 
app/test-pmd/parameters.c +5 -0
index f3954c1..ece05c1 100644
@@ -508,6 +508,9 @@ parse_link_speed(int n)
  void
  launch_args_parse(int argc, char** argv)
  {
+#define PARAM_PROC_ID "proc-id"
+#define PARAM_NUM_PROCS "num-procs"
+
  	int n, opt;
  	char **argvopt;
  	int opt_idx;
@@ -625,6 +628,8 @@ launch_args_parse(int argc, char** argv)
  		{ "rx-mq-mode",                 1, 0, 0 },
  		{ "record-core-cycles",         0, 0, 0 },
  		{ "record-burst-stats",         0, 0, 0 },
+		{ PARAM_NUM_PROCS,              1, 0, 0 },
+		{ PARAM_PROC_ID,                1, 0, 0 },
  		{ 0, 0, 0, 0 },
  	};
 

app/test-pmd/parameters.c launch_args_parse(int argc, char** argv) · line 1400

Quoted 5 lines
app/test-pmd/parameters.c +2 -0
@@ -1391,6 +1396,12 @@ launch_args_parse(int argc, char** argv)
  				record_core_cycles = 1;
  			if (!strcmp(lgopts[opt_idx].name, "record-burst-stats"))
  				record_burst_stats = 1;
+			if (strncmp(lgopts[opt_idx].name,
+				    PARAM_NUM_PROCS, 9) == 0)

I strongly dislike 9 here and 7 below. Why is strncmp() used here, but just strcmp() is used for all other options. It makes the code inconsistent.

Quoted 12 lines
app/test-pmd/parameters.c +4 -0
+				num_procs = atoi(optarg);
+			if (strncmp(lgopts[opt_idx].name,
+				    PARAM_PROC_ID, 7) == 0)
+				proc_id = atoi(optarg);
  			break;
  		case 'h':
  			usage(argv[0]);
app/test-pmd/testpmd.c +0 -0
index d4be23f..afa2a6b 100644

app/test-pmd/testpmd.c enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS; · line 522

Quoted 5 lines
app/test-pmd/testpmd.c +2 -0
@@ -518,6 +518,62 @@ enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS;
   */
  uint32_t eth_link_speed;
 
+/*
+ * Id of the current process in multi-process, used to

Id -> ID in accordance with devtools/words-case.txt

Quoted 33 lines
app/test-pmd/testpmd.c +33 -0
+ * configure the queues to be polled.
+ */
+int proc_id;
+
+/*
+ * Number of processes in multi-process, used to
+ * configure the queues to be polled.
+ */
+unsigned int num_procs = 1;
+
+static int
+eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
+		      const struct rte_eth_conf *dev_conf)
+{
+	if (is_proc_primary())
+		return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q,
+					dev_conf);
+	return 0;
+}
+
+static int
+eth_dev_start_mp(uint16_t port_id)
+{
+	if (is_proc_primary())
+		return rte_eth_dev_start(port_id);
+
+	return 0;
+}
+
+static int
+eth_dev_stop_mp(uint16_t port_id)
+{
+	if (is_proc_primary())

app/test-pmd/testpmd.c enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS; · line 565

Quoted 10 lines
app/test-pmd/testpmd.c +10 -0
+		return rte_eth_dev_stop(port_id);
+
+	return 0;
+}
+
+static void
+mempool_free_mp(struct rte_mempool *mp)
+{
+	if (is_proc_primary())
+		return rte_mempool_free(mp);

As far as I remember some compilers do not like it for void. Just remove 'return'.

Quoted 15 lines
app/test-pmd/testpmd.c +11 -0
+}
+
+static int
+eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu)
+{
+	if (is_proc_primary())
+		return rte_eth_dev_set_mtu(port_id, mtu);
+
+	return 0;
+}
+
  /* Forward function declarations */
  static void setup_attached_port(portid_t pi);
  static void check_all_ports_link_status(uint32_t port_mask);

app/test-pmd/testpmd.c mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, · line 1038

Quoted 6 lines
app/test-pmd/testpmd.c +3 -0
@@ -977,6 +1033,11 @@ mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
  	mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
  	mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx);
 
+	if (!is_proc_primary()) {
+		rte_mp = rte_mempool_lookup(pool_name);
+		goto err;

It looks like error path, but it works in the case of success as well. Looks confusing.

Quoted 24 lines
app/test-pmd/testpmd.c +10 -3
+	}
+
  	TESTPMD_LOG(INFO,
  		"create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
  		pool_name, nb_mbuf, mbuf_seg_size, socket_id);
@@ -1059,9 +1120,14 @@ mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 
  err:
  	if (rte_mp == NULL) {
-		rte_exit(EXIT_FAILURE,
-			"Creation of mbuf pool for socket %u failed: %s\n",
-			socket_id, rte_strerror(rte_errno));
+		if (is_proc_primary())
+			rte_exit(EXIT_FAILURE,
+				"Creation of mbuf pool for socket %u failed: %s\n",
+				socket_id, rte_strerror(rte_errno));
+		else
+			rte_exit(EXIT_FAILURE,
+				"Get mbuf pool for socket %u failed: %s\n",
+				socket_id, rte_strerror(rte_errno));
  	} else if (verbose_level > 0) {
  		rte_mempool_dump(stdout, rte_mp);
  	}

app/test-pmd/testpmd.c flush_fwd_rx_queues(void) · line 2072

Quoted 5 lines
app/test-pmd/testpmd.c +2 -0
@@ -2002,6 +2068,12 @@ flush_fwd_rx_queues(void)
  	uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
  	uint64_t timer_period;
 
+	if (num_procs > 1) {
+		printf("multi-process not support for flushing fwd rx "

rx -> Rx
also it is better to avoid like split.

Quoted 18 lines
app/test-pmd/testpmd.c +8 -3
+		       "queues, skip the below lines and return.\n");
+		return;
+	}
+
  	/* convert to number of cycles */
  	timer_period = rte_get_timer_hz(); /* 1 second timeout */
 
@@ -2511,21 +2583,24 @@ start_port(portid_t pid)
  				return -1;
  			}
  			/* configure port */
-			diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
-						     nb_txq + nb_hairpinq,
-						     &(port->dev_conf));
+			diag = eth_dev_configure_mp(pi,
+					     nb_rxq + nb_hairpinq,
+					     nb_txq + nb_hairpinq,
+					     &(port->dev_conf));

app/test-pmd/testpmd.c start_port(portid_t pid) · line 2595

Quoted 10 lines
app/test-pmd/testpmd.c +5 -4
  			if (diag != 0) {
-				if (rte_atomic16_cmpset(&(port->port_status),
-				RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
-					printf("Port %d can not be set back "
-							"to stopped\n", pi);
+				if (rte_atomic16_cmpset(
+						&(port->port_status),
+						RTE_PORT_HANDLING,
+						RTE_PORT_STOPPED) == 0)
+					printf("Port %d can not be set back to stopped\n",

can not -> cannot (since you touch the line anyway)

Quoted 13 lines
app/test-pmd/testpmd.c +2 -1
+						pi);
  				printf("Fail to configure port %d\n", pi);
  				/* try to reconfigure port next time */
  				port->need_reconfig = 1;
  				return -1;
  			}
  		}
-		if (port->need_reconfig_queues > 0) {
+		if (port->need_reconfig_queues > 0 && is_proc_primary()) {
  			port->need_reconfig_queues = 0;
  			/* setup tx queues */
  			for (qi = 0; qi < nb_txq; qi++) {

app/test-pmd/testpmd.c start_port(portid_t pid) · line 2626

Quoted 6 lines
app/test-pmd/testpmd.c +1 -2
@@ -2548,8 +2623,8 @@ start_port(portid_t pid)
  				if (rte_atomic16_cmpset(&(port->port_status),
  							RTE_PORT_HANDLING,
  							RTE_PORT_STOPPED) == 0)
-					printf("Port %d can not be set back "
-							"to stopped\n", pi);
+					printf("Port %d can not be set back to stopped\n",

can not -> cannot

Quoted 11 lines
app/test-pmd/testpmd.c +2 -1
+						pi);
  				printf("Fail to configure port %d tx queues\n",
  				       pi);
  				/* try to reconfigure queues next time */
@@ -2626,16 +2701,16 @@ start_port(portid_t pid)
  		cnt_pi++;
 
  		/* start port */
-		diag = rte_eth_dev_start(pi);
+		diag = eth_dev_start_mp(pi);
  		if (diag < 0) {

app/test-pmd/testpmd.c start_port(portid_t pid) · line 2712

Quoted 10 lines
app/test-pmd/testpmd.c +2 -3
  			printf("Fail to start port %d: %s\n", pi,
  			       rte_strerror(-diag));
 
  			/* Fail to setup rx queue, return */
  			if (rte_atomic16_cmpset(&(port->port_status),
-				RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
-				printf("Port %d can not be set back to "
-							"stopped\n", pi);
+			RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
+				printf("Port %d can not be set back to stopped\n",

can not -> cannot

Quoted 91 lines
app/test-pmd/testpmd.c +12 -9
+				       pi);
  			continue;
  		}
 
@@ -2765,7 +2840,7 @@ stop_port(portid_t pid)
  		if (port->flow_list)
  			port_flow_flush(pi);
 
-		if (rte_eth_dev_stop(pi) != 0)
+		if (eth_dev_stop_mp(pi) != 0)
  			RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n",
  				pi);
 
@@ -2834,8 +2909,10 @@ close_port(portid_t pid)
  			continue;
  		}
 
-		port_flow_flush(pi);
-		rte_eth_dev_close(pi);
+		if (is_proc_primary()) {
+			port_flow_flush(pi);
+			rte_eth_dev_close(pi);
+		}
  	}
 
  	remove_invalid_ports();
@@ -3101,7 +3178,7 @@ pmd_test_exit(void)
  	}
  	for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
  		if (mempools[i])
-			rte_mempool_free(mempools[i]);
+			mempool_free_mp(mempools[i]);
  	}
 
  	printf("\nBye...\n");
@@ -3432,7 +3509,7 @@ update_jumbo_frame_offload(portid_t portid)
  	 * if unset do it here
  	 */
  	if ((rx_offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) == 0) {
-		ret = rte_eth_dev_set_mtu(portid,
+		ret = eth_dev_set_mtu_mp(portid,
  				port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead);
  		if (ret)
  			printf("Failed to set MTU to %u for port %u\n",
@@ -3622,6 +3699,10 @@ init_port_dcb_config(portid_t pid,
  	int retval;
  	uint16_t i;
 
+	if (num_procs > 1) {
+		printf("The multi-process feature doesn't support dcb.\n");
+		return -ENOTSUP;
+	}
  	rte_port = &ports[pid];
 
  	memset(&port_conf, 0, sizeof(struct rte_eth_conf));
@@ -3787,10 +3868,6 @@ main(int argc, char** argv)
  		rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
  			 rte_strerror(rte_errno));
 
-	if (rte_eal_process_type() == RTE_PROC_SECONDARY)
-		rte_exit(EXIT_FAILURE,
-			 "Secondary process type not supported.\n");
-
  	ret = register_eth_event_callback();
  	if (ret != 0)
  		rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
app/test-pmd/testpmd.h +9 -0
index 6ca872d..3318e8f 100644
@@ -632,6 +632,15 @@ extern enum rte_eth_rx_mq_mode rx_mq_mode;
 
  extern struct rte_flow_action_conntrack conntrack_context;
 
+extern int proc_id;
+extern unsigned int num_procs;
+
+static inline bool
+is_proc_primary(void)
+{
+	return rte_eal_process_type() == RTE_PROC_PRIMARY;
+}
+
  static inline unsigned int
  lcore_num(void)
  {
doc/guides/rel_notes/release_21_05.rst +0 -0
index b36c120..9361332 100644

doc/guides/rel_notes/release_21_05.rst New Features · line 238

Quoted 4 lines
doc/guides/rel_notes/release_21_05.rst +1 -0
@@ -235,6 +235,7 @@ New Features
      ``port cleanup (port_id) txq (queue_id) (free_cnt)``
    * Added command to show link flow control info.
      ``show port (port_id) flow_ctrl``
+  * Added support multi-process for testpmd.

Please, rebase to 21.08.

Quoted 16 lines
doc/guides/rel_notes/release_21_05.rst +0 -0
  * **Updated ipsec-secgw sample application.**
 
doc/guides/testpmd_app_ug/run_app.rst +5 -0
index d062165..74efa4f 100644
@@ -543,3 +543,73 @@ The command line options are:
      bit 1 - two hairpin ports paired
      bit 0 - two hairpin ports loop
      The default value is 0. Hairpin will use single port mode and implicit Tx flow mode.
+
+Testpmd Multi-Process Command-line Options
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The following are the command-line options for testpmd multi-process support:

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 560

Quoted 10 lines
doc/guides/testpmd_app_ug/run_app.rst +10 -0
+
+.. code-block:: console
+
+	primary process:
+	sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i --rxq=4 --txq=4 \
+        --num-procs=2 --proc-id=0
+
+	secondary process:
+	sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i --rxq=4 --txq=4 \
+        --num-procs=2 --proc-id=1

Do we really need --rxq=4 --txq=4 in secodary processes? Can't we get it from already configured device in primary process?

Quoted 4 lines

Same question is applicable to --num-procs. May be testpmd can publish in shared memory? If yes, I'm OK to either do it in the patch or defer as a later improvement.

Yes, agree with you, it can be done for later improvement.

Quoted 40 lines
Quoted 26 lines
doc/guides/testpmd_app_ug/run_app.rst +26 -0
+
+The command line options are:
+
+*   ``--num-procs=N``
+
+    The number of processes which will be used.
+
+*   ``--proc-id=id``
+
+    The id of the current process (id < num-procs). id should be different in primary
+    process and secondary process, which starts from '0'.
+
+Calculation rule for queue:
+All queues are allocated to different processes based on ``proc_num`` and ``proc_id``.
+Calculation rule for the testpmd to allocate queues to each process:
+start(queue start id) = proc_id * nb_q / num_procs;
+end(queue end id) = start + nb_q / num_procs;
+
+For example, if testpmd supports 4 Tx and Rx queues
+the 0~1 for primary process
+the 2~3 for secondary process
+
+The number of rings should be a multiple of the number of processes. If not,
+redundant queues will exist after queues are allocated to processes. After RSS is
+enabled, packet loss occurs when traffic is sent to all processes at the same time.
+Some traffic enters redundant queues and cannot be forwarded.

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 596

Quoted 10 lines
doc/guides/testpmd_app_ug/run_app.rst +10 -0
+
+All the dev ops is supported in primary process. While secondary process is not permitted
+to allocate or release shared memory, so some ops are not supported as follows::
+``dev_configure``
+``dev_start``
+``dev_stop``
+``rx_queue_setup``
+``tx_queue_setup``
+``rx_queue_release``
+``tx_queue_release``

@Thomas, @Ferrh, shouldn't it be handled on ethdev level as
well if it is really that strict.Yes, API modification may be handled as another patch for later improvement.

Quoted 11 lines

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 605

Quoted 9 lines
doc/guides/testpmd_app_ug/run_app.rst +9 -0
+
+So, any command from testpmd which calls those APIs will not be supported in secondary
+process, like::
+``port config all rxq|txq|rxd|txd <value>``
+``port config <port_id> rx_offload xxx on/off ``
+``port config <port_id> tx_offload xxx on/off``
+etc.
+
+Flow API is supported, it applies only on its own process on SW side, but all on HW side.

Sorry, I don't understand it.

This may be confusing, I will delete the lines.
What I mean is Flow API is supported,that is it.

Quoted 15 lines

doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 615

Quoted 11 lines
doc/guides/testpmd_app_ug/run_app.rst +10 -0
+
+Stats is supported, stats will not change when one quit and start, as they share the same
+buffer to store the stats. Flow rules are maintained in process level: primary and secondary
+has its own flow list (but one flow list in HW). The two can see all the queues, so setting
+the flow rules for the other is OK. But in the testpmd primary process receiving or transmitting
+packets from the queue allocated for secondary process is not permitted, and same for
+secondary process.
+
+RSS is supported, primary process and secondary process has separate queues to use, RSS
+will work in their own queues whether primary or secondary process.

For me it sounds like secondary process has own RSS configuration. If so, it is false. I guess I simply misunderstand above paragraph.

This may be confusing, I will delete the lines.
What I mean is that primary process and secondary process has its own queues, but both the queues are processed by RSS.

Quoted 2 lines

.