app/testpmd: support multi-process
RFC 1/1 app/testpmd: support multi-process
This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./testpmd -w xxx --file-prefix=xx -l 0-1 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=0 the secondary cmd:
./testpmd -w xxx --file-prefix=xx -l 2-3 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=1
--- app/test-pmd/cmdline.c | 6 ++- app/test-pmd/config.c | 9 +++- app/test-pmd/parameters.c | 9 ++++ app/test-pmd/testpmd.c | 133 ++++++++++++++++++++++++++++++++-------------- app/test-pmd/testpmd.h | 7 +++ 5 files changed, 121 insertions(+), 43 deletions(-)
app/test-pmd/cmdline.c +4 -2
@@ -71,8 +71,6 @@#include "cmdline_tm.h" #include "bpf_cmd.h" -static struct cmdline *testpmd_cl; -static void cmd_reconfig_device_queue (portid_t id,uint8_t dev,uint8_t queue);/* *** Help command with introduction. *** */
@@ -17115,6 +17113,10 @@ prompt(void)if (testpmd_cl ==NULL )return ;cmdline_interact (testpmd_cl); +if (unlikely (f_quit ==1 )) { +dup2 (testpmd_fd_copy, testpmd_cl->s_in); +close (testpmd_fd_copy); + }cmdline_stdin_exit (testpmd_cl); }
app/test-pmd/config.c +8 -1
@@ -3100,6 +3100,8 @@ rss_fwd_config_setup(void)queueid_t rxq;queueid_t nb_q;streamid_t sm_id; +int start; +int end; nb_q = nb_rxq;if (nb_q > nb_txq)
@@ -3117,7 +3119,10 @@ rss_fwd_config_setup(void)init_fwd_streams ();setup_fwd_config_of_each_lcore (&cur_fwd_config); - rxp =0 ; rxq =0 ; + start = proc_id * nb_q / num_procs; + end = start + nb_q / num_procs; + rxp =0 ; + rxq = start;for (sm_id =0 ; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {struct fwd_stream *fs;
@@ -3134,6 +3139,8 @@ rss_fwd_config_setup(void)continue ; rxp =0 ; rxq++; +if (rxq >= end) + rxq=start; } }
app/test-pmd/parameters.c +9 -0
@@ -45,6 +45,8 @@#include <rte_flow.h> #include "testpmd.h" +#define PARAM_PROC_ID "proc-id" +#define PARAM_NUM_PROCS "num-procs"static void usage (char * progname)
@@ -603,6 +605,8 @@ launch_args_parse(int argc, char** argv) { "rx-mq-mode", 1, 0, 0 }, { "record-core-cycles", 0, 0, 0 }, { "record-burst-stats", 0, 0, 0 }, + { PARAM_NUM_PROCS, 1, 0, 0 }, + { PARAM_PROC_ID, 1, 0, 0 }, { 0, 0, 0, 0 }, };
@@ -1359,6 +1363,11 @@ launch_args_parse(int argc, char** argv) record_core_cycles =1 ;if (!strcmp (lgopts[opt_idx].name,"record-burst-stats" )) record_burst_stats =1 ; + +if (strncmp (lgopts[opt_idx].name,PARAM_NUM_PROCS ,8 ) ==0 ) + num_procs =atoi (optarg); +if (strncmp (lgopts[opt_idx].name,PARAM_PROC_ID ,7 ) ==0 ) + proc_id =atoi (optarg);break ;case 'h' :usage (argv[0 ]);
app/test-pmd/testpmd.c +93 -40
255 lines not on this page · raw message
app/test-pmd/testpmd.h +7 -0
@@ -13,6 +13,7 @@#include <rte_gso.h> #include <cmdline.h> #include <sys/queue.h> +#include "cmdline.h" #define RTE_PORT_ALL (~(portid_t)0x0)
@@ -24,6 +25,10 @@#define RTE_PORT_CLOSED (uint16_t)2#define RTE_PORT_HANDLING (uint16_t)3 +uint8_t f_quit; +int testpmd_fd_copy; +struct cmdline *testpmd_cl; +/* * It is used to allocate the memory for hash key. * The hash key size is NIC dependent.
@@ -421,6 +426,8 @@ extern uint64_t noisy_lkup_mem_sz;extern uint64_t noisy_lkup_num_writes;extern uint64_t noisy_lkup_num_reads;extern uint64_t noisy_lkup_num_reads_writes; +extern int proc_id; +extern unsigned num_procs;extern uint8_t dcb_config;extern uint8_t dcb_test;
Signature · 1 line
2.7.4
Quoted 9 lines · Lijun Ou
This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./testpmd -w xxx --file-prefix=xx -l 0-1 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=0 the secondary cmd:
./testpmd -w xxx --file-prefix=xx -l 2-3 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=1
+1 to add multi-process support to testpmd, missing it reduces multi-process testing a lot, but I guess concern is it may cause lots of changes and create some corner cases in testpmd.
Can you please explain a little why new 'proc-id' testpmd argument is needed, why the regular way of using eal '--proc-type' command and 'RTE_PROC_SECONDARY' checks is not enough?
Also why 'MULTIPLE_PROCESS_HANDLE' define is required?
Thanks,
ferruh
Quoted 411 lines
Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>--- app/test-pmd/cmdline.c | 6 ++- app/test-pmd/config.c | 9 +++- app/test-pmd/parameters.c | 9 ++++ app/test-pmd/testpmd.c | 133 ++++++++++++++++++++++++++++++++-------------- app/test-pmd/testpmd.h | 7 +++ 5 files changed, 121 insertions(+), 43 deletions(-)app/test-pmd/cmdline.c +4
-2@@ -71,8 +71,6 @@#include "cmdline_tm.h" #include "bpf_cmd.h" -static struct cmdline *testpmd_cl; -static void cmd_reconfig_device_queue (portid_t id,uint8_t dev,uint8_t queue);/* *** Help command with introduction. *** */ @@ -17115,6 +17113,10 @@ prompt(void)if (testpmd_cl ==NULL )return ;cmdline_interact (testpmd_cl); +if (unlikely (f_quit ==1 )) { +dup2 (testpmd_fd_copy, testpmd_cl->s_in); +close (testpmd_fd_copy); + }cmdline_stdin_exit (testpmd_cl); }app/test-pmd/config.c +8
-1@@ -3100,6 +3100,8 @@ rss_fwd_config_setup(void)queueid_t rxq;queueid_t nb_q;streamid_t sm_id; +int start; +int end; nb_q = nb_rxq;if (nb_q > nb_txq)@@ -3117,7 +3119,10 @@ rss_fwd_config_setup(void)init_fwd_streams ();setup_fwd_config_of_each_lcore (&cur_fwd_config); - rxp =0 ; rxq =0 ; + start = proc_id * nb_q / num_procs; + end = start + nb_q / num_procs; + rxp =0 ; + rxq = start;for (sm_id =0 ; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {struct fwd_stream *fs;@@ -3134,6 +3139,8 @@ rss_fwd_config_setup(void)continue ; rxp =0 ; rxq++; +if (rxq >= end) + rxq=start; } }app/test-pmd/parameters.c +9
-0@@ -45,6 +45,8 @@#include <rte_flow.h> #include "testpmd.h" +#define PARAM_PROC_ID "proc-id" +#define PARAM_NUM_PROCS "num-procs"static void usage (char * progname)@@ -603,6 +605,8 @@ launch_args_parse(int argc, char** argv) { "rx-mq-mode", 1, 0, 0 }, { "record-core-cycles", 0, 0, 0 }, { "record-burst-stats", 0, 0, 0 }, + { PARAM_NUM_PROCS, 1, 0, 0 }, + { PARAM_PROC_ID, 1, 0, 0 }, { 0, 0, 0, 0 }, };@@ -1359,6 +1363,11 @@ launch_args_parse(int argc, char** argv) record_core_cycles =1 ;if (!strcmp (lgopts[opt_idx].name,"record-burst-stats" )) record_burst_stats =1 ; + +if (strncmp (lgopts[opt_idx].name,PARAM_NUM_PROCS ,8 ) ==0 ) + num_procs =atoi (optarg); +if (strncmp (lgopts[opt_idx].name,PARAM_PROC_ID ,7 ) ==0 ) + proc_id =atoi (optarg);break ;case 'h' :usage (argv[0 ]);app/test-pmd/testpmd.c +93
-40@@ -63,6 +63,8 @@#include "testpmd.h" +int testpmd_fd_copy =500 ;/* the copy of STDIN_FILENO */ +#ifndef MAP_HUGETLB /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ #define HUGE_FLAG (0x40000)@@ -79,6 +81,7 @@#define EXTMEM_HEAP_NAME "extmem"#define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M +#define MULTIPLE_PROCESS_HANDLE uint16_t verbose_level =0 ;/**< Silent by default. */ int testpmd_logtype;/**< Log type for testpmd logs */ @@ -125,6 +128,9 @@ uint8_t port_numa[RTE_MAX_ETHPORTS];*/ uint8_t rxring_numa[RTE_MAX_ETHPORTS ]; +int proc_id =0 ; +unsigned num_procs =1 ; +/* * Store specified sockets on which TX ring to be used by ports * is allocated. @@ -978,16 +984,36 @@ mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,/* wrapper to rte_mempool_create() */ TESTPMD_LOG (INFO ,"preferred mempool ops selected: %s\n" ,rte_mbuf_best_mempool_ops ()); +#ifdef MULTIPLE_PROCESS_HANDLE +if (rte_eal_process_type () ==RTE_PROC_PRIMARY ) + rte_mp =rte_pktmbuf_pool_create (pool_name, + nb_mbuf, mb_mempool_cache,0 , + mbuf_seg_size, socket_id); +else + rte_mp =rte_mempool_lookup (pool_name); +#else rte_mp =rte_pktmbuf_pool_create (pool_name, nb_mbuf, mb_mempool_cache,0 , mbuf_seg_size, socket_id); +#endif break ; }case MP_ALLOC_ANON : { +#ifdef MULTIPLE_PROCESS_HANDLE +if (rte_eal_process_type () ==RTE_PROC_PRIMARY ) + rte_mp =rte_mempool_create_empty (pool_name, + nb_mbuf, mb_size, + (unsigned int )mb_mempool_cache, +sizeof (struct rte_pktmbuf_pool_private ), + socket_id,0 ); +else + rte_mp =rte_mempool_lookup (pool_name); +#else rte_mp =rte_mempool_create_empty (pool_name, nb_mbuf, mb_size, (unsigned int ) mb_mempool_cache,sizeof (struct rte_pktmbuf_pool_private ), socket_id, mempool_flags); +#endif if (rte_mp ==NULL ) goto err;@@ -1017,9 +1043,18 @@ mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,TESTPMD_LOG (INFO ,"preferred mempool ops selected: %s\n" ,rte_mbuf_best_mempool_ops ()); +#ifdef MULTIPLE_PROCESS_HANDLE +if (rte_eal_process_type () ==RTE_PROC_PRIMARY ) + rte_mp =rte_pktmbuf_pool_create (pool_name, + nb_mbuf, mb_mempool_cache,0 , + mbuf_seg_size, heap_socket); +else + rte_mp =rte_mempool_lookup (pool_name); +#else rte_mp =rte_pktmbuf_pool_create (pool_name, nb_mbuf, mb_mempool_cache,0 , mbuf_seg_size, heap_socket); +#endif break ; }case MP_ALLOC_XBUF :@@ -2485,21 +2520,28 @@ start_port(portid_t pid)return -1 ; }/* configure port */ - diag =rte_eth_dev_configure (pi, nb_rxq + nb_hairpinq, +if (rte_eal_process_type () ==RTE_PROC_PRIMARY ) { + diag =rte_eth_dev_configure (pi, + nb_rxq + nb_hairpinq, nb_txq + nb_hairpinq, &(port->dev_conf)); -if (diag !=0 ) { -if (rte_atomic16_cmpset (&(port->port_status), -RTE_PORT_HANDLING ,RTE_PORT_STOPPED ) ==0 ) -printf ("Port %d can not be set back " -"to stopped\n" , pi); -printf ("Fail to configure port %d\n" , pi); -/* try to reconfigure port next time */ - port->need_reconfig =1 ; -return -1 ; +if (diag !=0 ) { +if (rte_atomic16_cmpset ( + &(port->port_status), +RTE_PORT_HANDLING , +RTE_PORT_STOPPED ) ==0 ) +printf ("Port %d can not be set " +"back to stopped\n" , pi); +printf ("Fail to configure port %d\n" , + pi); +/* try to reconfigure port next time */ + port->need_reconfig =1 ; +return -1 ; + } } } -if (port->need_reconfig_queues >0 ) { +if (port->need_reconfig_queues >0 && +rte_eal_process_type () ==RTE_PROC_PRIMARY ) { port->need_reconfig_queues =0 ;/* setup tx queues */ for (qi =0 ; qi < nb_txq; qi++) {@@ -2600,15 +2642,18 @@ start_port(portid_t pid) cnt_pi++;/* start port */ -if (rte_eth_dev_start (pi) <0 ) { -printf ("Fail to start port %d\n" , pi); - -/* Fail to setup rx queue, return */ -if (rte_atomic16_cmpset (&(port->port_status), -RTE_PORT_HANDLING ,RTE_PORT_STOPPED ) ==0 ) -printf ("Port %d can not be set back to " -"stopped\n" , pi); -continue ; +if (rte_eal_process_type () ==RTE_PROC_PRIMARY ) { + diag =rte_eth_dev_start (pi); +if (diag <0 ) { +printf ("Fail to start port %d\n" , pi); + +/* Fail to setup rx queue, return */ +if (rte_atomic16_cmpset (&(port->port_status), +RTE_PORT_HANDLING ,RTE_PORT_STOPPED ) ==0 ) +printf ("Port %d can not be set back to " +"stopped\n" , pi); +continue ; + } } if (rte_atomic16_cmpset(&(port->port_status),@@ -2737,7 +2782,7 @@ stop_port(portid_t pid)if (port->flow_list)port_flow_flush (pi); -if (rte_eth_dev_stop (pi) !=0 ) +if (rte_eal_process_type () ==RTE_PROC_PRIMARY &&rte_eth_dev_stop (pi) !=0 )RTE_LOG (ERR ,EAL ,"rte_eth_dev_stop failed for port %u\n" , pi);@@ -2806,8 +2851,10 @@ close_port(portid_t pid)continue ; } -port_flow_flush (pi); -rte_eth_dev_close (pi); +if (rte_eal_process_type () ==RTE_PROC_PRIMARY ) +port_flow_flush (pi); +if (rte_eal_process_type () ==RTE_PROC_PRIMARY ) +rte_eth_dev_close (pi); }remove_invalid_ports ();@@ -3071,7 +3118,7 @@ pmd_test_exit(void) } }for (i =0 ; i <RTE_DIM (mempools) ; i++) { -if (mempools[i]) +if (rte_eal_process_type () ==RTE_PROC_PRIMARY && mempools[i])rte_mempool_free (mempools[i]); }@@ -3519,6 +3566,10 @@ init_port_dcb_config(portid_t pid,int retval;uint16_t i; +if (num_procs >1 ) { +printf ("The multi-process feature doesn't support dcb.\n" ); +return -ENOTSUP ; + } rte_port = &ports[pid];memset (&port_conf,0 ,sizeof (struct rte_eth_conf ));@@ -3617,13 +3668,6 @@ init_port(void) }static void -force_quit (void ) -{ -pmd_test_exit (); -prompt_exit (); -} - -static void print_stats (void ) {uint8_t i;@@ -3654,12 +3698,16 @@ signal_handler(int signum)if (latencystats_enabled !=0 )rte_latencystats_uninit ();#endif -force_quit ();/* Set flag to indicate the force termination. */ f_quit =1 ; -/* exit with the expected status */ -signal (signum,SIG_DFL ); -kill (getpid (), signum); +if (interactive ==1 ) { +dup2 (testpmd_cl->s_in, testpmd_fd_copy); +close (testpmd_cl->s_in); + }else { +dup2 (0 , testpmd_fd_copy); +close (0 ); + } + } }@@ -3684,10 +3732,6 @@ main(int argc, char** argv)rte_exit (EXIT_FAILURE ,"Cannot init EAL: %s\n" ,rte_strerror (rte_errno)); -if (rte_eal_process_type () ==RTE_PROC_SECONDARY ) -rte_exit (EXIT_FAILURE , -"Secondary process type not supported.\n" ); - ret =register_eth_event_callback ();if (ret !=0 )rte_exit (EXIT_FAILURE ,"Cannot register for ethdev events" );@@ -3783,8 +3827,10 @@ main(int argc, char** argv) } } -if (!no_device_start &&start_port (RTE_PORT_ALL ) !=0 ) +if (!no_device_start &&start_port (RTE_PORT_ALL ) !=0 ) { +pmd_test_exit ();rte_exit (EXIT_FAILURE ,"Start ports failed\n" ); + }/* set all ports to promiscuous mode by default */ RTE_ETH_FOREACH_DEV (port_id) {@@ -3830,6 +3876,8 @@ main(int argc, char** argv) }prompt ();pmd_test_exit (); +if (unlikely (f_quit ==1 )) +prompt_exit (); } else#endif {@@ -3865,6 +3913,11 @@ main(int argc, char** argv)printf ("Press enter to exit\n" ); rc =read (0 , &c,1 );pmd_test_exit (); +if (unlikely (f_quit ==1 )) { +dup2 (testpmd_fd_copy,0 ); +close (testpmd_fd_copy); +prompt_exit (); + }if (rc <0 )return 1 ; }app/test-pmd/testpmd.h +7
-0@@ -13,6 +13,7 @@#include <rte_gso.h> #include <cmdline.h> #include <sys/queue.h> +#include "cmdline.h" #define RTE_PORT_ALL (~(portid_t)0x0)@@ -24,6 +25,10 @@#define RTE_PORT_CLOSED (uint16_t)2#define RTE_PORT_HANDLING (uint16_t)3 +uint8_t f_quit; +int testpmd_fd_copy; +struct cmdline *testpmd_cl; +/* * It is used to allocate the memory for hash key. * The hash key size is NIC dependent. @@ -421,6 +426,8 @@ extern uint64_t noisy_lkup_mem_sz;extern uint64_t noisy_lkup_num_writes;extern uint64_t noisy_lkup_num_reads;extern uint64_t noisy_lkup_num_reads_writes; +extern int proc_id; +extern unsigned num_procs;extern uint8_t dcb_config;extern uint8_t dcb_test;
Quoted 11 lines · Ferruh Yigit
Quoted 9 lines · Lijun Ou
This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./testpmd -w xxx --file-prefix=xx -l 0-1 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=0 the secondary cmd:
./testpmd -w xxx --file-prefix=xx -l 2-3 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=1
Quoted 3 lines
+1 to add multi-process support to testpmd, missing it reduces
multi-process testing a lot, but I guess concern is it may cause lots of changes and create some corner cases in testpmd.
Yes, We have not found any other problems.
Quoted 1 line
Can you please explain a little why new 'proc-id' testpmd argument is needed, why the regular way of using eal '--proc-type' command and '
RTE_PROC_SECONDARY' checks is not enough?
We refer to the definition of symmetric_mp,We need him to identify exactly which one from the process.
Quoted 4 lines
Also why '
MULTIPLE_PROCESS_HANDLE' define is required?
Yes, I think it is unncessary. I will delete it.
Quoted 387 lines
Thanks,
ferruhQuoted 373 lines
Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>--- app/test-pmd/cmdline.c | 6 ++- app/test-pmd/config.c | 9 +++- app/test-pmd/parameters.c | 9 ++++ app/test-pmd/testpmd.c | 133 ++++++++++++++++++++++++++++++++-------------- app/test-pmd/testpmd.h | 7 +++ 5 files changed, 121 insertions(+), 43 deletions(-)app/test-pmd/cmdline.c +4
-2@@ -71,8 +71,6 @@#include "cmdline_tm.h" #include "bpf_cmd.h" -static struct cmdline *testpmd_cl; -static void cmd_reconfig_device_queue (portid_t id,uint8_t dev,uint8_t queue);/* *** Help command with introduction. *** */ @@ -17115,6 +17113,10 @@ prompt(void)if (testpmd_cl ==NULL )return ;cmdline_interact (testpmd_cl); +if (unlikely (f_quit ==1 )) { +dup2 (testpmd_fd_copy, testpmd_cl->s_in); +close (testpmd_fd_copy); + }cmdline_stdin_exit (testpmd_cl); }app/test-pmd/config.c +8
-1@@ -3100,6 +3100,8 @@ rss_fwd_config_setup(void)queueid_t rxq;queueid_t nb_q;streamid_t sm_id; +int start; +int end; nb_q = nb_rxq;if (nb_q > nb_txq)@@ -3117,7 +3119,10 @@ rss_fwd_config_setup(void)init_fwd_streams ();setup_fwd_config_of_each_lcore (&cur_fwd_config); - rxp =0 ; rxq =0 ; + start = proc_id * nb_q / num_procs; + end = start + nb_q / num_procs; + rxp =0 ; + rxq = start;for (sm_id =0 ; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {struct fwd_stream *fs;@@ -3134,6 +3139,8 @@ rss_fwd_config_setup(void)continue ; rxp =0 ; rxq++; +if (rxq >= end) + rxq=start; } }app/test-pmd/parameters.c +9
-0@@ -45,6 +45,8 @@#include <rte_flow.h> #include "testpmd.h" +#define PARAM_PROC_ID "proc-id" +#define PARAM_NUM_PROCS "num-procs"static void usage (char * progname)@@ -603,6 +605,8 @@ launch_args_parse(int argc, char** argv) { "rx-mq-mode", 1, 0, 0 }, { "record-core-cycles", 0, 0, 0 }, { "record-burst-stats", 0, 0, 0 }, + { PARAM_NUM_PROCS, 1, 0, 0 }, + { PARAM_PROC_ID, 1, 0, 0 }, { 0, 0, 0, 0 }, };@@ -1359,6 +1363,11 @@ launch_args_parse(int argc, char** argv) record_core_cycles =1 ;if (!strcmp (lgopts[opt_idx].name,"record-burst-stats" )) record_burst_stats =1 ; + +if (strncmp (lgopts[opt_idx].name,PARAM_NUM_PROCS ,8 ) ==0 ) + num_procs =atoi (optarg); +if (strncmp (lgopts[opt_idx].name,PARAM_PROC_ID ,7 ) ==0 ) + proc_id =atoi (optarg);break ;case 'h' :usage (argv[0 ]);app/test-pmd/testpmd.c +93
-40@@ -63,6 +63,8 @@#include "testpmd.h" +int testpmd_fd_copy =500 ;/* the copy of STDIN_FILENO */ +#ifndef MAP_HUGETLB /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ #define HUGE_FLAG (0x40000)@@ -79,6 +81,7 @@#define EXTMEM_HEAP_NAME "extmem"#define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M +#define MULTIPLE_PROCESS_HANDLE uint16_t verbose_level =0 ;/**< Silent by default. */ int testpmd_logtype;/**< Log type for testpmd logs */ @@ -125,6 +128,9 @@ uint8_t port_numa[RTE_MAX_ETHPORTS];*/ uint8_t rxring_numa[RTE_MAX_ETHPORTS ]; +int proc_id =0 ; +unsigned num_procs =1 ; +/* * Store specified sockets on which TX ring to be used by ports * is allocated. @@ -978,16 +984,36 @@ mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, /* wrapper to rte_mempool_create() */ TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", rte_mbuf_best_mempool_ops()); +#ifdef MULTIPLE_PROCESS_HANDLE +if (rte_eal_process_type () ==RTE_PROC_PRIMARY ) + rte_mp =rte_pktmbuf_pool_create (pool_name, + nb_mbuf, mb_mempool_cache,0 , + mbuf_seg_size, socket_id); +else + rte_mp =rte_mempool_lookup (pool_name); +#else rte_mp =rte_pktmbuf_pool_create (pool_name, nb_mbuf, mb_mempool_cache,0 , mbuf_seg_size, socket_id); +#endif break ; }case MP_ALLOC_ANON : { +#ifdef MULTIPLE_PROCESS_HANDLE +if (rte_eal_process_type () ==RTE_PROC_PRIMARY ) + rte_mp =rte_mempool_create_empty (pool_name, + nb_mbuf, mb_size, + (unsigned int )mb_mempool_cache, +sizeof (struct rte_pktmbuf_pool_private ), + socket_id,0 ); +else + rte_mp =rte_mempool_lookup (pool_name); +#else rte_mp =rte_mempool_create_empty (pool_name, nb_mbuf, mb_size, (unsigned int ) mb_mempool_cache,sizeof (struct rte_pktmbuf_pool_private ), socket_id, mempool_flags); +#endif if (rte_mp ==NULL ) goto err;@@ -1017,9 +1043,18 @@ mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", rte_mbuf_best_mempool_ops()); +#ifdef MULTIPLE_PROCESS_HANDLE +if (rte_eal_process_type () ==RTE_PROC_PRIMARY ) + rte_mp =rte_pktmbuf_pool_create (pool_name, + nb_mbuf, mb_mempool_cache,0 , + mbuf_seg_size, heap_socket); +else + rte_mp =rte_mempool_lookup (pool_name); +#else rte_mp =rte_pktmbuf_pool_create (pool_name, nb_mbuf, mb_mempool_cache,0 , mbuf_seg_size, heap_socket); +#endif break ; }case MP_ALLOC_XBUF :@@ -2485,21 +2520,28 @@ start_port(portid_t pid)return -1 ; }/* configure port */ - diag =rte_eth_dev_configure (pi, nb_rxq + nb_hairpinq, +if (rte_eal_process_type () ==RTE_PROC_PRIMARY ) { + diag =rte_eth_dev_configure (pi, + nb_rxq + nb_hairpinq, nb_txq + nb_hairpinq, &(port->dev_conf)); -if (diag !=0 ) { -if (rte_atomic16_cmpset (&(port->port_status), -RTE_PORT_HANDLING ,RTE_PORT_STOPPED ) ==0 ) -printf ("Port %d can not be set back " -"to stopped\n" , pi); -printf ("Fail to configure port %d\n" , pi); -/* try to reconfigure port next time */ - port->need_reconfig =1 ; -return -1 ; +if (diag !=0 ) { +if (rte_atomic16_cmpset ( + &(port->port_status), +RTE_PORT_HANDLING , +RTE_PORT_STOPPED ) ==0 ) +printf ("Port %d can not be set " +"back to stopped\n" , pi); +printf ("Fail to configure port %d\n" , + pi); +/* try to reconfigure port next time */ + port->need_reconfig =1 ; +return -1 ; + } } } -if (port->need_reconfig_queues >0 ) { +if (port->need_reconfig_queues >0 && +rte_eal_process_type () ==RTE_PROC_PRIMARY ) { port->need_reconfig_queues =0 ;/* setup tx queues */ for (qi =0 ; qi < nb_txq; qi++) {@@ -2600,15 +2642,18 @@ start_port(portid_t pid) cnt_pi++;/* start port */ -if (rte_eth_dev_start (pi) <0 ) { -printf ("Fail to start port %d\n" , pi); - -/* Fail to setup rx queue, return */ -if (rte_atomic16_cmpset (&(port->port_status), -RTE_PORT_HANDLING ,RTE_PORT_STOPPED ) ==0 ) -printf ("Port %d can not be set back to " -"stopped\n" , pi); -continue ; +if (rte_eal_process_type () ==RTE_PROC_PRIMARY ) { + diag =rte_eth_dev_start (pi); +if (diag <0 ) { +printf ("Fail to start port %d\n" , pi); + +/* Fail to setup rx queue, return */ +if (rte_atomic16_cmpset (&(port->port_status), +RTE_PORT_HANDLING ,RTE_PORT_STOPPED ) ==0 ) +printf ("Port %d can not be set back to " +"stopped\n" , pi); +continue ; + } } if (rte_atomic16_cmpset(&(port->port_status),@@ -2737,7 +2782,7 @@ stop_port(portid_t pid)if (port->flow_list)port_flow_flush (pi); -if (rte_eth_dev_stop (pi) !=0 ) +if (rte_eal_process_type () ==RTE_PROC_PRIMARY &&rte_eth_dev_stop (pi) !=0 )RTE_LOG (ERR ,EAL ,"rte_eth_dev_stop failed for port %u\n" , pi);@@ -2806,8 +2851,10 @@ close_port(portid_t pid)continue ; } -port_flow_flush (pi); -rte_eth_dev_close (pi); +if (rte_eal_process_type () ==RTE_PROC_PRIMARY ) +port_flow_flush (pi); +if (rte_eal_process_type () ==RTE_PROC_PRIMARY ) +rte_eth_dev_close (pi); }remove_invalid_ports ();@@ -3071,7 +3118,7 @@ pmd_test_exit(void) } }for (i =0 ; i <RTE_DIM (mempools) ; i++) { -if (mempools[i]) +if (rte_eal_process_type () ==RTE_PROC_PRIMARY && mempools[i])rte_mempool_free (mempools[i]); }@@ -3519,6 +3566,10 @@ init_port_dcb_config(portid_t pid,int retval;uint16_t i; +if (num_procs >1 ) { +printf ("The multi-process feature doesn't support dcb.\n" ); +return -ENOTSUP ; + } rte_port = &ports[pid];memset (&port_conf,0 ,sizeof (struct rte_eth_conf ));@@ -3617,13 +3668,6 @@ init_port(void) }static void -force_quit (void ) -{ -pmd_test_exit (); -prompt_exit (); -} - -static void print_stats (void ) {uint8_t i;@@ -3654,12 +3698,16 @@ signal_handler(int signum)if (latencystats_enabled !=0 )rte_latencystats_uninit ();#endif -force_quit ();/* Set flag to indicate the force termination. */ f_quit =1 ; -/* exit with the expected status */ -signal (signum,SIG_DFL ); -kill (getpid (), signum); +if (interactive ==1 ) { +dup2 (testpmd_cl->s_in, testpmd_fd_copy); +close (testpmd_cl->s_in); + }else { +dup2 (0 , testpmd_fd_copy); +close (0 ); + } + } }@@ -3684,10 +3732,6 @@ main(int argc, char** argv)rte_exit (EXIT_FAILURE ,"Cannot init EAL: %s\n" ,rte_strerror (rte_errno)); -if (rte_eal_process_type () ==RTE_PROC_SECONDARY ) -rte_exit (EXIT_FAILURE , -"Secondary process type not supported.\n" ); - ret =register_eth_event_callback ();if (ret !=0 )rte_exit (EXIT_FAILURE ,"Cannot register for ethdev events" );@@ -3783,8 +3827,10 @@ main(int argc, char** argv) } } -if (!no_device_start &&start_port (RTE_PORT_ALL ) !=0 ) +if (!no_device_start &&start_port (RTE_PORT_ALL ) !=0 ) { +pmd_test_exit ();rte_exit (EXIT_FAILURE ,"Start ports failed\n" ); + }/* set all ports to promiscuous mode by default */ RTE_ETH_FOREACH_DEV (port_id) {@@ -3830,6 +3876,8 @@ main(int argc, char** argv) }prompt ();pmd_test_exit (); +if (unlikely (f_quit ==1 )) +prompt_exit (); } else#endif {@@ -3865,6 +3913,11 @@ main(int argc, char** argv)printf ("Press enter to exit\n" ); rc =read (0 , &c,1 );pmd_test_exit (); +if (unlikely (f_quit ==1 )) { +dup2 (testpmd_fd_copy,0 ); +close (testpmd_fd_copy); +prompt_exit (); + }if (rc <0 )return 1 ; }app/test-pmd/testpmd.h +5
-0@@ -13,6 +13,7 @@#include <rte_gso.h> #include <cmdline.h> #include <sys/queue.h> +#include "cmdline.h" #define RTE_PORT_ALL (~(portid_t)0x0)@@ -24,6 +25,10 @@#define RTE_PORT_CLOSED (uint16_t)2#define RTE_PORT_HANDLING (uint16_t)3 +uint8_t f_quit; +int testpmd_fd_copy; +struct cmdline *testpmd_cl; +/* * It is used to allocate the memory for hash key. * The hash key size is NIC dependent. app/test-pmd/testpmd.h extern uint64_t noisy_lkup_mem_sz; · line 433
Quoted 8 lines
app/test-pmd/testpmd.h +2
-0@@ -421,6 +426,8 @@ extern uint64_t noisy_lkup_mem_sz;extern uint64_t noisy_lkup_num_writes;extern uint64_t noisy_lkup_num_reads;extern uint64_t noisy_lkup_num_reads_writes; +extern int proc_id; +extern unsigned num_procs;extern uint8_t dcb_config;extern uint8_t dcb_test;.
Hi,
Quoted 18 lines
-----Original Message-----
From: dev <[email protected]> On Behalf Of Lijun Ou
Sent: Friday, January 8, 2021 11:46 AM
To: [email protected]; [email protected]; [email protected]; [email protected]
Cc: [email protected]
Subject: [dpdk-dev] [RFC] app/testpmd: support multi-processThis patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./testpmd -w xxx --file-prefix=xx -l 0-1 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=0 the secondary cmd:
./testpmd -w xxx --file-prefix=xx -l 2-3 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=1Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>
Quoted 9 lines
--- app/test-pmd/cmdline.c | 6 ++- app/test-pmd/config.c | 9 +++- app/test-pmd/parameters.c | 9 ++++ app/test-pmd/testpmd.c | 133 ++++++++++++++++++++++++++++++++-- ------------ app/test-pmd/testpmd.h | 7 +++ 5 files changed, 121 insertions(+), 43 deletions(-)
+1 for having this support for testpmd.
Some questions in my mind:
How are the queues distributing here? In example I see 16 defined, are they for one instance or for all? Will all processes have same memory region? If installing one RTE_FLOW in one instance will be active for all? Same question for detaching device in one instance, how it will reflect on others?
There is many other scenarios like this, how it will handle those?
BRs,
Wisam Jaddo
Quoted 30 lines · Wisam Monther
Hi,
Quoted 18 lines
-----Original Message-----
From: dev <[email protected]> On Behalf Of Lijun Ou
Sent: Friday, January 8, 2021 11:46 AM
To: [email protected]; [email protected]; [email protected]; [email protected]
Cc: [email protected]
Subject: [dpdk-dev] [RFC] app/testpmd: support multi-processThis patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./testpmd -w xxx --file-prefix=xx -l 0-1 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=0 the secondary cmd:
./testpmd -w xxx --file-prefix=xx -l 2-3 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=1Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>Quoted 9 lines
--- app/test-pmd/cmdline.c | 6 ++- app/test-pmd/config.c | 9 +++- app/test-pmd/parameters.c | 9 ++++ app/test-pmd/testpmd.c | 133 ++++++++++++++++++++++++++++++++-- ------------ app/test-pmd/testpmd.h | 7 +++ 5 files changed, 121 insertions(+), 43 deletions(-)
Quoted 8 lines
+1 for having this support for testpmd.
Some questions in my mind:
How are the queues distributing here? In example I see 16 defined, are they for one instance or for all? Will all processes have same memory region? If installing oneRTE_FLOWin one instance will be active for all? Same question for detaching device in one instance, how it will reflect on others?
There is many other scenarios like this, how it will handle those?
Hi,Wisam Monther
Firstly, thank you for your questions. According to the current
implementation scheme, all queues are evenly allocated to different processes based on proc_num and proc_id。
The number of receiving queues, number of processes, and process ID are specified for the master and slave processes. After being created by the main process, the sending and receiving queues are evenly distributed to all processes. The following shows the calculation rule for the Testpmd to allocate queues to each process after the proc ID is specified. start(queue start id) = proc_id * nb_q / num_procs;
end(queue end id) = start + nb_q / num_procs;
For example, if support 16 txq and rxq
the 0 ~7 for primary process
the 8 ~15 for secondary process
all process have the same memory region.
Thanks
Lijun Ou
Quoted 4 lines
BRs,
Wisam Jaddo
.
Quoted 68 lines
-----Original Message-----
From: oulijun <[email protected]>
Sent: Tuesday, January 12, 2021 4:13 PM
To: Wisam Monther <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]
Cc: [email protected]
Subject: Re: [dpdk-dev] [RFC] app/testpmd: support multi-processQuoted 25 lines · Wisam Monther
Hi,
Quoted 23 lines
-----Original Message-----
From: dev <[email protected]> On Behalf Of Lijun Ou
Sent: Friday, January 8, 2021 11:46 AM
To: [email protected]; [email protected]; [email protected]; [email protected]
Cc: [email protected]
Subject: [dpdk-dev] [RFC] app/testpmd: support multi-processThis patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./testpmd -w xxx --file-prefix=xx -l 0-1 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=0 the secondary cmd:
./testpmd -w xxx --file-prefix=xx -l 2-3 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=1Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>--- app/test-pmd/cmdline.c | 6 ++- app/test-pmd/config.c | 9 +++- app/test-pmd/parameters.c | 9 ++++ app/test-pmd/testpmd.c | 133++++++++++++++++++++++++++++++++--
Quoted 9 lines
Quoted 4 lines
------------
app/test-pmd/testpmd.h | 7 +++ 5 files changed, 121 insertions(+), 43 deletions(-)+1 for having this support for testpmd.
Some questions in my mind:
How are the queues distributing here? In example I see 16 defined, arethey for one instance or for all?
Quoted 1 line
Will all processes have same memory region? If installing one
RTE_FLOWinone instance will be active for all?
Quoted 1 line
Same question for detaching device in one instance, how it will reflect on
others?
Quoted 2 lines
There is many other scenarios like this, how it will handle those?
Hi,Wisam Monther
Firstly, thank you for your questions. According to the current
implementation scheme, all queues are evenly allocated to different processes based onproc_numand proc_id。
The number of receiving queues, number of processes, and process ID are specified for the master and slave processes. After being created by the main process, the sending and receiving queues are evenly distributed to all processes. The following shows the calculation rule for the Testpmd to allocate queues to each process after the proc ID is specified. start(queue start id) =proc_id*nb_q/ num_procs;
end(queue end id) = start +nb_q/ num_procs;
Quoted 5 lines
For example, if support 16 txq and rxq
the 0 ~7 for primary process
the 8 ~15 for secondary processall process have the same memory region.
Ok great, Thanks that' make sense.
What about the impact of installing RTE_FLOW rule into one instance, does it apply on all? I'm talking about SW side, not HW side? The impact of detaching device from secondary/primary only, how it's designed to handle such cases?
Quoted 8 lines
Thanks
Lijun OuQuoted 4 lines
BRs,
Wisam Jaddo
.
Quoted 72 lines · Wisam Monther
Quoted 72 lines
-----Original Message-----
From: oulijun <[email protected]>
Sent: Tuesday, January 12, 2021 4:13 PM
To: Wisam Monther <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]
Cc: [email protected]
Subject: Re: [dpdk-dev] [RFC] app/testpmd: support multi-processQuoted 25 lines · Wisam Monther
Hi,
Quoted 23 lines
-----Original Message-----
From: dev <[email protected]> On Behalf Of Lijun Ou
Sent: Friday, January 8, 2021 11:46 AM
To: [email protected]; [email protected]; [email protected]; [email protected]
Cc: [email protected]
Subject: [dpdk-dev] [RFC] app/testpmd: support multi-processThis patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./testpmd -w xxx --file-prefix=xx -l 0-1 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=0 the secondary cmd:
./testpmd -w xxx --file-prefix=xx -l 2-3 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=1Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>--- app/test-pmd/cmdline.c | 6 ++- app/test-pmd/config.c | 9 +++- app/test-pmd/parameters.c | 9 ++++ app/test-pmd/testpmd.c | 133++++++++++++++++++++++++++++++++--
Quoted 9 lines
Quoted 4 lines
------------
app/test-pmd/testpmd.h | 7 +++ 5 files changed, 121 insertions(+), 43 deletions(-)+1 for having this support for testpmd.
Some questions in my mind:
How are the queues distributing here? In example I see 16 defined, arethey for one instance or for all?
Quoted 1 line
Will all processes have same memory region? If installing one
RTE_FLOWinone instance will be active for all?
Quoted 1 line
Same question for detaching device in one instance, how it will reflect on
others?
Quoted 2 lines
There is many other scenarios like this, how it will handle those?
Hi,Wisam Monther
Firstly, thank you for your questions. According to the current
implementation scheme, all queues are evenly allocated to different processes based onproc_numand proc_id。
The number of receiving queues, number of processes, and process ID are specified for the master and slave processes. After being created by the main process, the sending and receiving queues are evenly distributed to all processes. The following shows the calculation rule for the Testpmd to allocate queues to each process after the proc ID is specified. start(queue start id) =proc_id*nb_q/ num_procs;
end(queue end id) = start +nb_q/ num_procs;For example, if support 16 txq and rxq
the 0 ~7 for primary process
the 8 ~15 for secondary process
Quoted 6 lines
Quoted 1 line
all process have the same memory region.
Ok great, Thanks that' make sense.
What about the impact of installingRTE_FLOWrule into one instance, does it apply on all? I'm talking about SW side, not HW side? The impact of detaching device from secondary/primary only, how it's designed to handle such cases?
It is my understanding that installing RTE_FLOW rule will affect a instance from our modification.But the all instance will changed from HW side. If detaching device from secondary only, it will not affect other processes.
Quoted 10 lines
Quoted 8 lines
Thanks
Lijun OuQuoted 4 lines
BRs,
Wisam Jaddo
..
RFC v2 1/1 app/testpmd: support multi-process
In reply to Lijun OuThis patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./testpmd -w xxx --file-prefix=xx -l 0-1 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=0 the secondary cmd:
./testpmd -w xxx --file-prefix=xx -l 2-3 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=1
--- app/test-pmd/cmdline.c | 6 ++- app/test-pmd/config.c | 9 +++- app/test-pmd/parameters.c | 9 ++++ app/test-pmd/testpmd.c | 135 +++++++++++++++++++++++++++++----------------- app/test-pmd/testpmd.h | 7 +++ 5 files changed, 114 insertions(+), 52 deletions(-)
app/test-pmd/cmdline.c +4 -2
@@ -71,8 +71,6 @@#include "cmdline_tm.h" #include "bpf_cmd.h" -static struct cmdline *testpmd_cl; -static void cmd_reconfig_device_queue (portid_t id,uint8_t dev,uint8_t queue);/* *** Help command with introduction. *** */
@@ -17124,6 +17122,10 @@ prompt(void)if (testpmd_cl ==NULL )return ;cmdline_interact (testpmd_cl); +if (unlikely (f_quit ==1 )) { +dup2 (testpmd_fd_copy, testpmd_cl->s_in); +close (testpmd_fd_copy); + }cmdline_stdin_exit (testpmd_cl); }
app/test-pmd/config.c +8 -1
@@ -3100,6 +3100,8 @@ rss_fwd_config_setup(void)queueid_t rxq;queueid_t nb_q;streamid_t sm_id; +int start; +int end; nb_q = nb_rxq;if (nb_q > nb_txq)
@@ -3117,7 +3119,10 @@ rss_fwd_config_setup(void)init_fwd_streams ();setup_fwd_config_of_each_lcore (&cur_fwd_config); - rxp =0 ; rxq =0 ; + start = proc_id * nb_q / num_procs; + end = start + nb_q / num_procs; + rxp =0 ; + rxq = start;for (sm_id =0 ; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {struct fwd_stream *fs;
@@ -3134,6 +3139,8 @@ rss_fwd_config_setup(void)continue ; rxp =0 ; rxq++; +if (rxq >= end) + rxq = start; } }
app/test-pmd/parameters.c +9 -0
@@ -45,6 +45,8 @@#include <rte_flow.h> #include "testpmd.h" +#define PARAM_PROC_ID "proc-id" +#define PARAM_NUM_PROCS "num-procs"static void usage (char * progname)
@@ -603,6 +605,8 @@ launch_args_parse(int argc, char** argv) { "rx-mq-mode", 1, 0, 0 }, { "record-core-cycles", 0, 0, 0 }, { "record-burst-stats", 0, 0, 0 }, + { PARAM_NUM_PROCS, 1, 0, 0 }, + { PARAM_PROC_ID, 1, 0, 0 }, { 0, 0, 0, 0 }, };
@@ -1356,6 +1360,11 @@ launch_args_parse(int argc, char** argv) record_core_cycles =1 ;if (!strcmp (lgopts[opt_idx].name,"record-burst-stats" )) record_burst_stats =1 ; + +if (strncmp (lgopts[opt_idx].name,PARAM_NUM_PROCS ,8 ) ==0 ) + num_procs =atoi (optarg); +if (strncmp (lgopts[opt_idx].name,PARAM_PROC_ID ,7 ) ==0 ) + proc_id =atoi (optarg);break ;case 'h' :usage (argv[0 ]);
app/test-pmd/testpmd.c +86 -49
242 lines not on this page · raw message
app/test-pmd/testpmd.h +7 -0
@@ -13,6 +13,7 @@#include <rte_gso.h> #include <cmdline.h> #include <sys/queue.h> +#include "cmdline.h" #define RTE_PORT_ALL (~(portid_t)0x0)
@@ -24,6 +25,10 @@#define RTE_PORT_CLOSED (uint16_t)2#define RTE_PORT_HANDLING (uint16_t)3 +uint8_t f_quit; +int testpmd_fd_copy; +struct cmdline *testpmd_cl; +/* * It is used to allocate the memory for hash key. * The hash key size is NIC dependent.
@@ -421,6 +426,8 @@ extern uint64_t noisy_lkup_mem_sz;extern uint64_t noisy_lkup_num_writes;extern uint64_t noisy_lkup_num_reads;extern uint64_t noisy_lkup_num_reads_writes; +extern int proc_id; +extern unsigned num_procs;extern uint8_t dcb_config;extern uint8_t dcb_test;
Signature · 1 line
2.7.4
PATCH 1/1 app/testpmd: support multi-process
This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./testpmd -w xxx --file-prefix=xx -l 0-1 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=0 the secondary cmd:
./testpmd -w xxx --file-prefix=xx -l 2-3 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=1
--- app/test-pmd/cmdline.c | 12 +++- app/test-pmd/config.c | 9 ++- app/test-pmd/parameters.c | 11 ++++ app/test-pmd/testpmd.c | 138 ++++++++++++++++++++++++++++++---------------- app/test-pmd/testpmd.h | 7 +++ 5 files changed, 127 insertions(+), 50 deletions(-)
app/test-pmd/cmdline.c +10 -2
@@ -72,8 +72,6 @@#include "cmdline_tm.h" #include "bpf_cmd.h" -static struct cmdline *testpmd_cl; -static void cmd_reconfig_device_queue (portid_t id,uint8_t dev,uint8_t queue);/* *** Help command with introduction. *** */
@@ -5353,6 +5351,12 @@ cmd_set_flush_rx_parsed(void *parsed_result, __rte_unused void *data) {struct cmd_set_flush_rx *res = parsed_result; + +if (num_procs >1 && (strcmp (res->mode,"on" ) ==0 )) { +printf ("multi-process doesn't support to flush rx queues.\n" ); +return ; + } + no_flush_rx = (uint8_t )((strcmp (res->mode,"on" ) ==0 ) ?0 :1 ); }
@@ -17316,6 +17320,10 @@ prompt(void)printf ("Cannot set exit function for cmdline\n" );cmdline_interact (testpmd_cl); +if (unlikely (f_quit ==1 )) { +dup2 (testpmd_fd_copy, testpmd_cl->s_in); +close (testpmd_fd_copy); + }if (ret !=0 )cmdline_stdin_exit (testpmd_cl); }
app/test-pmd/config.c +8 -1
@@ -2827,6 +2827,8 @@ rss_fwd_config_setup(void)queueid_t rxq;queueid_t nb_q;streamid_t sm_id; +int start; +int end; nb_q = nb_rxq;if (nb_q > nb_txq)
@@ -2844,7 +2846,10 @@ rss_fwd_config_setup(void)init_fwd_streams ();setup_fwd_config_of_each_lcore (&cur_fwd_config); - rxp =0 ; rxq =0 ; + start = proc_id * nb_q / num_procs; + end = start + nb_q / num_procs; + rxp =0 ; + rxq = start;for (sm_id =0 ; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {struct fwd_stream *fs;
@@ -2861,6 +2866,8 @@ rss_fwd_config_setup(void)continue ; rxp =0 ; rxq++; +if (rxq >= end) + rxq = start; } }
app/test-pmd/parameters.c +11 -0
@@ -45,6 +45,8 @@#include <rte_flow.h> #include "testpmd.h" +#define PARAM_PROC_ID "proc-id" +#define PARAM_NUM_PROCS "num-procs"static void usage (char * progname)
@@ -605,6 +607,8 @@ launch_args_parse(int argc, char** argv) { "rx-mq-mode", 1, 0, 0 }, { "record-core-cycles", 0, 0, 0 }, { "record-burst-stats", 0, 0, 0 }, + { PARAM_NUM_PROCS, 1, 0, 0 }, + { PARAM_PROC_ID, 1, 0, 0 }, { 0, 0, 0, 0 }, };
@@ -1366,6 +1370,13 @@ launch_args_parse(int argc, char** argv) record_core_cycles =1 ;if (!strcmp (lgopts[opt_idx].name,"record-burst-stats" )) record_burst_stats =1 ; + +if (strncmp (lgopts[opt_idx].name, +PARAM_NUM_PROCS ,8 ) ==0 ) + num_procs =atoi (optarg); +if (strncmp (lgopts[opt_idx].name, +PARAM_PROC_ID ,7 ) ==0 ) + proc_id =atoi (optarg);break ;case 'h' :usage (argv[0 ]);
app/test-pmd/testpmd.c +91 -47
253 lines not on this page · raw message
app/test-pmd/testpmd.h +7 -0
@@ -13,6 +13,7 @@#include <rte_gso.h> #include <cmdline.h> #include <sys/queue.h> +#include "cmdline.h" #define RTE_PORT_ALL (~(portid_t)0x0)
@@ -24,6 +25,10 @@#define RTE_PORT_CLOSED (uint16_t)2#define RTE_PORT_HANDLING (uint16_t)3 +uint8_t f_quit; +int testpmd_fd_copy; +struct cmdline *testpmd_cl; +/* * It is used to allocate the memory for hash key. * The hash key size is NIC dependent.
@@ -422,6 +427,8 @@ extern uint64_t noisy_lkup_mem_sz;extern uint64_t noisy_lkup_num_writes;extern uint64_t noisy_lkup_num_reads;extern uint64_t noisy_lkup_num_reads_writes; +extern int proc_id; +extern unsigned int num_procs;extern uint8_t dcb_config;extern uint8_t dcb_test;
Signature · 1 line
2.7.4
Quoted 9 lines · Lijun Ou
This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./testpmd -w xxx --file-prefix=xx -l 0-1 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=0 the secondary cmd:
./testpmd -w xxx --file-prefix=xx -l 2-3 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=1
Can you update this in the testpmd usage guide.
There are so many things that testpmd can do.
What will happen to the stats if I quit one of the processes and start again?
In case of multiple Rx queues, how will RSS happen?
It will be good to have the behavior of various features documented - some of the questions have been already answered.
They can be documented as well.
Quoted 3 lines
2.7.4
Quoted 11 lines · Ajit Khaparde
Quoted 9 lines · Lijun Ou
This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./testpmd -w xxx --file-prefix=xx -l 0-1 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=0 the secondary cmd:
./testpmd -w xxx --file-prefix=xx -l 2-3 -n 2 -- -i\
--rxq=16 --txq=16 --num-procs=2 --proc-id=1
Quoted 1 line
Can you update this in the testpmd usage guide.
OK, I will do it in V2.
Quoted 3 lines
There are so many things that testpmd can do.
What will happen to the stats if I quit one of the processes and start again?
stats will not change when one quit and start, As they share the same buffer to store the stats.
Quoted 1 line
In case of multiple Rx queues, how will RSS happen?
Primary process and secondary process has seperate queues to use, RSS will work in their own queues whether primary and secondary process.
Quoted 3 lines
It will be good to have the behavior of various features documented - some of the questions have been already answered.
They can be documented as well.
OK, I will do it in V2, thanks.
Quoted 2 lines
PATCH v2 1/1 app/testpmd: support multi-process
In reply to Lijun OuFrom: Lijun Ou <[email protected]>
This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0
the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1
--- app/test-pmd/cmdline.c | 12 ++- app/test-pmd/config.c | 9 ++- app/test-pmd/parameters.c | 11 +++ app/test-pmd/testpmd.c | 138 ++++++++++++++++++++++------------ app/test-pmd/testpmd.h | 7 ++ doc/guides/testpmd_app_ug/run_app.rst | 69 +++++++++++++++++ 6 files changed, 196 insertions(+), 50 deletions(-)
app/test-pmd/cmdline.c +10 -2
@@ -71,8 +71,6 @@#include "cmdline_tm.h" #include "bpf_cmd.h" -static struct cmdline *testpmd_cl; -static void cmd_reconfig_device_queue (portid_t id,uint8_t dev,uint8_t queue);/* *** Help command with introduction. *** */
@@ -5351,6 +5349,12 @@ cmd_set_flush_rx_parsed(void *parsed_result, __rte_unused void *data) {struct cmd_set_flush_rx *res = parsed_result; + +if (num_procs >1 && (strcmp (res->mode,"on" ) ==0 )) { +printf ("multi-process doesn't support to flush rx queues.\n" ); +return ; + } + no_flush_rx = (uint8_t )((strcmp (res->mode,"on" ) ==0 ) ?0 :1 ); }
@@ -17227,6 +17231,10 @@ prompt(void)printf ("Cannot set exit function for cmdline\n" );cmdline_interact (testpmd_cl); +if (unlikely (f_quit ==1 )) { +dup2 (testpmd_fd_copy, testpmd_cl->s_in); +close (testpmd_fd_copy); + }if (ret !=0 )cmdline_stdin_exit (testpmd_cl); }
app/test-pmd/config.c +8 -1
@@ -2827,6 +2827,8 @@ rss_fwd_config_setup(void)queueid_t rxq;queueid_t nb_q;streamid_t sm_id; +int start; +int end; nb_q = nb_rxq;if (nb_q > nb_txq)
@@ -2844,7 +2846,10 @@ rss_fwd_config_setup(void)init_fwd_streams ();setup_fwd_config_of_each_lcore (&cur_fwd_config); - rxp =0 ; rxq =0 ; + start = proc_id * nb_q / num_procs; + end = start + nb_q / num_procs; + rxp =0 ; + rxq = start;for (sm_id =0 ; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {struct fwd_stream *fs;
@@ -2861,6 +2866,8 @@ rss_fwd_config_setup(void)continue ; rxp =0 ; rxq++; +if (rxq >= end) + rxq = start; } }
app/test-pmd/parameters.c +11 -0
@@ -45,6 +45,8 @@#include <rte_flow.h> #include "testpmd.h" +#define PARAM_PROC_ID "proc-id" +#define PARAM_NUM_PROCS "num-procs"static void usage (char * progname)
@@ -605,6 +607,8 @@ launch_args_parse(int argc, char** argv) { "rx-mq-mode", 1, 0, 0 }, { "record-core-cycles", 0, 0, 0 }, { "record-burst-stats", 0, 0, 0 }, + { PARAM_NUM_PROCS, 1, 0, 0 }, + { PARAM_PROC_ID, 1, 0, 0 }, { 0, 0, 0, 0 }, };
@@ -1366,6 +1370,13 @@ launch_args_parse(int argc, char** argv) record_core_cycles =1 ;if (!strcmp (lgopts[opt_idx].name,"record-burst-stats" )) record_burst_stats =1 ; + +if (strncmp (lgopts[opt_idx].name, +PARAM_NUM_PROCS ,8 ) ==0 ) + num_procs =atoi (optarg); +if (strncmp (lgopts[opt_idx].name, +PARAM_PROC_ID ,7 ) ==0 ) + proc_id =atoi (optarg);break ;case 'h' :usage (argv[0 ]);
app/test-pmd/testpmd.c +91 -47
253 lines not on this page · raw message
app/test-pmd/testpmd.h +7 -0
@@ -13,6 +13,7 @@#include <rte_gso.h> #include <cmdline.h> #include <sys/queue.h> +#include "cmdline.h" #define RTE_PORT_ALL (~(portid_t)0x0)
@@ -24,6 +25,10 @@#define RTE_PORT_CLOSED (uint16_t)2#define RTE_PORT_HANDLING (uint16_t)3 +uint8_t f_quit; +int testpmd_fd_copy; +struct cmdline *testpmd_cl; +/* * It is used to allocate the memory for hash key. * The hash key size is NIC dependent.
@@ -422,6 +427,8 @@ extern uint64_t noisy_lkup_mem_sz;extern uint64_t noisy_lkup_num_writes;extern uint64_t noisy_lkup_num_reads;extern uint64_t noisy_lkup_num_reads_writes; +extern int proc_id; +extern unsigned int num_procs;extern uint8_t dcb_config;extern uint8_t dcb_test;
doc/guides/testpmd_app_ug/run_app.rst +69 -0
72 lines not on this page · raw message
Signature · 1 line
2.7.4
:::snip::::
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 571
Quoted 10 lines
doc/guides/testpmd_app_ug/run_app.rst +10
-0+* ``--proc-type`` + Specify a given process instance as the primary or secondary DPDK instance. ``auto`` set here is ok. + +* ``-l CORELIST`` + List of cores to run on. the corelist should be different in primary process and secondary process. + +* ``--rxq=N`` + + Set the number of RX queues per port to N, where 1 <= N <= 65535. + The default value is 1. N is the sum of queues used by primary and secondary process.
If RxQ is 1, how will it be shared between the primary and secondary processes? Should the number of rings be a multiple of the number of processes?
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 576
Quoted 5 lines
doc/guides/testpmd_app_ug/run_app.rst +5
-0+ +* ``--txq=N`` + + Set the number of TX queues per port to N, where 1 <= N <= 65535. + The default value is 1. N is the sum of queues used by primary and secondary process.
Same as above. How will a single ring be used across primary and secondary processes?
Quoted 34 lines
doc/guides/testpmd_app_ug/run_app.rst +31
-1+ +* ``--num-procs=N`` + + The number of processes which will be used. + +* ``--proc-id=id`` + + The id of the current process (id < num-procs). id should be different in primary process and secondary process. + +Calculation rule for queue: +All queues are allocated to different processes based on proc_num and proc_id. +Calculation rule for the Testpmd to allocate queues to each process: +start(queue start id) = proc_id * nb_q / num_procs; +end(queue end id) = start + nb_q / num_procs; + +For example, if supports 4 txq and rxq +the 0~1 for primary process +the 2~3 for secondary process + +Most dev ops is supported in primary and secondary process. While secondary process is not permitted +to allocate or release shared memory, so some ops are not supported as follows: +``dev_start`` +``dev_stop`` +``rx_queue_setup`` +``tx_queue_setup`` +``rx_queue_release`` +``tx_queue_release`` + +RTE_FLOW supported, it applies only on its own process on SW side, but all on HW size. +stats supported, stats will not change when one quit and start, As they share the same buffer to store the stats. +RSS supported, Primary process and secondary process has separate queues to use, RSS will work in their own queues whether primary and secondary process. --2.7.4
Quoted 13 lines · Ajit Khaparde
:::snip::::
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 571
Quoted 10 lines
doc/guides/testpmd_app_ug/run_app.rst +10
-0+* ``--proc-type`` + Specify a given process instance as the primary or secondary DPDK instance. ``auto`` set here is ok. + +* ``-l CORELIST`` + List of cores to run on. the corelist should be different in primary process and secondary process. + +* ``--rxq=N`` + + Set the number of RX queues per port to N, where 1 <= N <= 65535. + The default value is 1. N is the sum of queues used by primary and secondary process.If RxQ is 1, how will it be shared between the primary and secondary processes?
Yes, your are right.the range of RxQ(or TxQ) is wrong.
As primary instance and secondary instance should have separate queues, and each should occupy at least one queue. So the range of RxQ(or TxQ) shoud be no less than two.
Quoted 1 line
Should the number of rings be a multiple of the number of process The number of rings had better be a multiple of the number of processes.
If not, redundant queues will exist after queues are allocated to
processes. After RSS is enabled, packet loss occurs when traffic is sent to all processes at the same time. Some traffic enters redundant queues and cannot be forwarded.
I will fix it in v4.Thanks.
Quoted 7 lines
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 576
Quoted 5 lines
doc/guides/testpmd_app_ug/run_app.rst +5
-0+ +* ``--txq=N`` + + Set the number of TX queues per port to N, where 1 <= N <= 65535. + The default value is 1. N is the sum of queues used by primary and secondary process.Same as above. How will a single ring be used across primary and secondary processes?
Just same as above, I will fix it in v4.
Thanks.
Quoted 35 lines
Quoted 35 lines
doc/guides/testpmd_app_ug/run_app.rst +31
-1+ +* ``--num-procs=N`` + + The number of processes which will be used. + +* ``--proc-id=id`` + + The id of the current process (id < num-procs). id should be different in primary process and secondary process. + +Calculation rule for queue: +All queues are allocated to different processes based on proc_num and proc_id. +Calculation rule for the Testpmd to allocate queues to each process: +start(queue start id) = proc_id * nb_q / num_procs; +end(queue end id) = start + nb_q / num_procs; + +For example, if supports 4 txq and rxq +the 0~1 for primary process +the 2~3 for secondary process + +Most dev ops is supported in primary and secondary process. While secondary process is not permitted +to allocate or release shared memory, so some ops are not supported as follows: +``dev_start`` +``dev_stop`` +``rx_queue_setup`` +``tx_queue_setup`` +``rx_queue_release`` +``tx_queue_release`` + +RTE_FLOW supported, it applies only on its own process on SW side, but all on HW size. +stats supported, stats will not change when one quit and start, As they share the same buffer to store the stats. +RSS supported, Primary process and secondary process has separate queues to use, RSS will work in their own queues whether primary and secondary process. --2.7.4
PATCH v14 1/1 app/testpmd: support multi-process
In reply to Min Hu (Connor)This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0
the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1
---
v14:
* Fixed comments by Andrew Rybchenko.
v13:
* Modified the doc syntax.
v12:
* Updated doc info.
v11:
* Fixed some minor syntax.
v10:
* Hid process type checks behind new functions.
* Added comments.
v9:
* Updated release notes and rst doc.
* Deleted deprecated codes.
* move macro and variable.
v8:
* Added warning info about queue numbers and process numbers.
v7:
* Fixed compiling error for unexpected unindent.
v6:
* Add rte flow description for multiple process.
v5:
* Fixed run_app.rst for multiple process description.
* Fix compiling error.
v4:
* Fixed minimum vlaue of Rxq or Txq in doc.
v3:
* Fixed compiling error using gcc10.0.
v2:
* Added document for this patch.
--- app/test-pmd/cmdline.c | 6 ++ app/test-pmd/config.c | 20 +++++- app/test-pmd/parameters.c | 9 +++ app/test-pmd/testpmd.c | 121 ++++++++++++++++++++++++++------- app/test-pmd/testpmd.h | 9 +++ doc/guides/rel_notes/release_21_08.rst | 2 +- doc/guides/testpmd_app_ug/run_app.rst | 70 +++++++++++++++++++ 7 files changed, 212 insertions(+), 25 deletions(-)
app/test-pmd/cmdline.c +6 -0
@@ -5463,6 +5463,12 @@ cmd_set_flush_rx_parsed(void *parsed_result, __rte_unused void *data) {struct cmd_set_flush_rx *res = parsed_result; + +if (num_procs >1 && (strcmp (res->mode,"on" ) ==0 )) { +printf ("multi-process doesn't support to flush Rx queues.\n" ); +return ; + } + no_flush_rx = (uint8_t )((strcmp (res->mode,"on" ) ==0 ) ?0 :1 ); }
app/test-pmd/config.c +19 -1
@@ -2976,6 +2976,8 @@ rss_fwd_config_setup(void)queueid_t rxq;queueid_t nb_q;streamid_t sm_id; +int start; +int end; nb_q = nb_rxq;if (nb_q > nb_txq)
@@ -2993,7 +2995,21 @@ rss_fwd_config_setup(void)init_fwd_streams ();setup_fwd_config_of_each_lcore (&cur_fwd_config); - rxp =0 ; rxq =0 ; + +if (proc_id >0 && nb_q % num_procs !=0 ) +printf ("Warning! queue numbers should be multiple of processes, or packet loss will happen.\n" ); + +/** +* In multi-process, All queues are allocated to different +* processes based on num_procs and proc_id. For example: +* if supports 4 queues(nb_q), 2 processes(num_procs), +* the 0~1 queue for primary process. +* the 2~3 queue for secondary process. +*/ + start = proc_id * nb_q / num_procs; + end = start + nb_q / num_procs; + rxp =0 ; + rxq = start;for (sm_id =0 ; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {struct fwd_stream *fs;
@@ -3010,6 +3026,8 @@ rss_fwd_config_setup(void)continue ; rxp =0 ; rxq++; +if (rxq >= end) + rxq = start; } }
app/test-pmd/parameters.c +9 -0
@@ -508,6 +508,9 @@ parse_link_speed(int n)void launch_args_parse (int argc,char ** argv) { +#define PARAM_PROC_ID "proc-id" +#define PARAM_NUM_PROCS "num-procs" +int n, opt;char **argvopt;int opt_idx;
@@ -625,6 +628,8 @@ launch_args_parse(int argc, char** argv) { "rx-mq-mode", 1, 0, 0 }, { "record-core-cycles", 0, 0, 0 }, { "record-burst-stats", 0, 0, 0 }, + { PARAM_NUM_PROCS, 1, 0, 0 }, + { PARAM_PROC_ID, 1, 0, 0 }, { 0, 0, 0, 0 }, };
@@ -1391,6 +1396,10 @@ launch_args_parse(int argc, char** argv) record_core_cycles =1 ;if (!strcmp (lgopts[opt_idx].name,"record-burst-stats" )) record_burst_stats =1 ; +if (!strcmp (lgopts[opt_idx].name,PARAM_NUM_PROCS )) + num_procs =atoi (optarg); +if (!strcmp (lgopts[opt_idx].name,PARAM_PROC_ID )) + proc_id =atoi (optarg);break ;case 'h' :usage (argv[0 ]);
app/test-pmd/testpmd.c +98 -23
206 lines not on this page · raw message
app/test-pmd/testpmd.h +9 -0
@@ -631,6 +631,15 @@ extern enum rte_eth_rx_mq_mode rx_mq_mode;extern struct rte_flow_action_conntrack conntrack_context; +extern int proc_id; +extern unsigned int num_procs; + +static inline bool +is_proc_primary (void ) +{ +return rte_eal_process_type () ==RTE_PROC_PRIMARY ; +} +static inline unsigned int lcore_num (void ) {
doc/guides/rel_notes/release_21_08.rst +1 -1
@@ -54,7 +54,7 @@ New Features This section is a comment. Do not overwrite or remove it. Also, make sure to start the actual text at the margin. ======================================================= - + * Added support multi-process for testpmd. Removed Items -------------
doc/guides/testpmd_app_ug/run_app.rst +70 -0
73 lines not on this page · raw message
Signature · 1 line
2.7.4
PATCH v15 1/1 app/testpmd: support multi-process
In reply to Min Hu (Connor)From: "Min Hu (Connor)" <[email protected]>
For example the following commands run two testpmd processes:
* the primary process:
./dpdk-testpmd --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0
* the secondary process:
./dpdk-testpmd --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1
---
v15:
* Fixed release notes.
* Cleanup documentation.
v14:
* Fixed comments by Andrew Rybchenko.
v13:
* Modified the doc syntax.
v12:
* Updated doc info.
v11:
* Fixed some minor syntax.
v10:
* Hid process type checks behind new functions.
* Added comments.
v9:
* Updated release notes and rst doc.
* Deleted deprecated codes.
* move macro and variable.
v8:
* Added warning info about queue numbers and process numbers.
v7:
* Fixed compiling error for unexpected unindent.
v6:
* Add rte flow description for multiple process.
v5:
* Fixed run_app.rst for multiple process description.
* Fix compiling error.
v4:
* Fixed minimum vlaue of Rxq or Txq in doc.
v3:
* Fixed compiling error using gcc10.0.
v2:
* Added document for this patch.
--- app/test-pmd/cmdline.c | 6 ++ app/test-pmd/config.c | 20 +++- app/test-pmd/parameters.c | 9 ++ app/test-pmd/testpmd.c | 121 ++++++++++++++++++++----- app/test-pmd/testpmd.h | 9 ++ doc/guides/rel_notes/release_21_08.rst | 5 + doc/guides/testpmd_app_ug/run_app.rst | 82 +++++++++++++++++ 7 files changed, 228 insertions(+), 24 deletions(-)
app/test-pmd/cmdline.c +6 -0
@@ -5459,6 +5459,12 @@ cmd_set_flush_rx_parsed(void *parsed_result, __rte_unused void *data) {struct cmd_set_flush_rx *res = parsed_result; + +if (num_procs >1 && (strcmp (res->mode,"on" ) ==0 )) { +printf ("multi-process doesn't support to flush Rx queues.\n" ); +return ; + } + no_flush_rx = (uint8_t )((strcmp (res->mode,"on" ) ==0 ) ?0 :1 ); }
app/test-pmd/config.c +19 -1
@@ -2976,6 +2976,8 @@ rss_fwd_config_setup(void)queueid_t rxq;queueid_t nb_q;streamid_t sm_id; +int start; +int end; nb_q = nb_rxq;if (nb_q > nb_txq)
@@ -2993,7 +2995,21 @@ rss_fwd_config_setup(void)init_fwd_streams ();setup_fwd_config_of_each_lcore (&cur_fwd_config); - rxp =0 ; rxq =0 ; + +if (proc_id >0 && nb_q % num_procs !=0 ) +printf ("Warning! queue numbers should be multiple of processes, or packet loss will happen.\n" ); + +/** +* In multi-process, All queues are allocated to different +* processes based on num_procs and proc_id. For example: +* if supports 4 queues(nb_q), 2 processes(num_procs), +* the 0~1 queue for primary process. +* the 2~3 queue for secondary process. +*/ + start = proc_id * nb_q / num_procs; + end = start + nb_q / num_procs; + rxp =0 ; + rxq = start;for (sm_id =0 ; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {struct fwd_stream *fs;
@@ -3010,6 +3026,8 @@ rss_fwd_config_setup(void)continue ; rxp =0 ; rxq++; +if (rxq >= end) + rxq = start; } }
app/test-pmd/parameters.c +9 -0
@@ -509,6 +509,9 @@ parse_link_speed(int n)void launch_args_parse (int argc,char ** argv) { +#define PARAM_PROC_ID "proc-id" +#define PARAM_NUM_PROCS "num-procs" +int n, opt;char **argvopt;int opt_idx;
@@ -628,6 +631,8 @@ launch_args_parse(int argc, char** argv) { "rx-mq-mode", 1, 0, 0 }, { "record-core-cycles", 0, 0, 0 }, { "record-burst-stats", 0, 0, 0 }, + { PARAM_NUM_PROCS, 1, 0, 0 }, + { PARAM_PROC_ID, 1, 0, 0 }, { 0, 0, 0, 0 }, };
@@ -1394,6 +1399,10 @@ launch_args_parse(int argc, char** argv) record_core_cycles =1 ;if (!strcmp (lgopts[opt_idx].name,"record-burst-stats" )) record_burst_stats =1 ; +if (!strcmp (lgopts[opt_idx].name,PARAM_NUM_PROCS )) + num_procs =atoi (optarg); +if (!strcmp (lgopts[opt_idx].name,PARAM_PROC_ID )) + proc_id =atoi (optarg);break ;case 'h' :usage (argv[0 ]);
app/test-pmd/testpmd.c +98 -23
206 lines not on this page · raw message
app/test-pmd/testpmd.h +9 -0
@@ -632,6 +632,15 @@ extern enum rte_eth_rx_mq_mode rx_mq_mode;extern struct rte_flow_action_conntrack conntrack_context; +extern int proc_id; +extern unsigned int num_procs; + +static inline bool +is_proc_primary (void ) +{ +return rte_eal_process_type () ==RTE_PROC_PRIMARY ; +} +static inline unsigned int lcore_num (void ) {
doc/guides/rel_notes/release_21_08.rst +5 -0
@@ -55,6 +55,11 @@ New Features Also, make sure to start the actual text at the margin. ======================================================= + * **Added multi-process support to testpmd.** + + Added command-line options to specify total number of processes and + current process ID. Each process owns subset of Rx and Tx queues. + Removed Items -------------
doc/guides/testpmd_app_ug/run_app.rst +82 -0
85 lines not on this page · raw message
Signature · 1 line
2.30.2
Quoted 20 lines · Andrew Rybchenko
From: "Min Hu (Connor)" <[email protected]>
For example the following commands run two testpmd processes:
* the primary process:
./dpdk-testpmd --proc-type=auto -l 0-1 -- -i \ --rxq=4 --txq=4 --num-procs=2 --proc-id=0* the secondary process:
./dpdk-testpmd --proc-type=auto -l 2-3 -- -i \ --rxq=4 --txq=4 --num-procs=2 --proc-id=1Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>Signed-off-by Andrew Rybchenko <[email protected]>Acked-by Xiaoyun Li <[email protected]>Acked-by Ajit Khaparde <[email protected]>Reviewed-by Ferruh Yigit <[email protected]>
[snip]
Quoted 54 lines
app/test-pmd/testpmd.c +46
-0@@ -520,6 +520,62 @@ enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS;*/ uint32_t eth_link_speed; +/* +* ID of the current process in multi-process, used to +* configure the queues to be polled. +*/ +int proc_id; + +/* +* Number of processes in multi-process, used to +* configure the queues to be polled. +*/ +unsigned int num_procs =1 ; + +static int +eth_dev_configure_mp (uint16_t port_id,uint16_t nb_rx_q,uint16_t nb_tx_q, +const struct rte_eth_conf *dev_conf) +{ +if (is_proc_primary ()) +return rte_eth_dev_configure (port_id, nb_rx_q, nb_tx_q, + dev_conf); +return 0 ; +} + +static int +eth_dev_start_mp (uint16_t port_id) +{ +if (is_proc_primary ()) +return rte_eth_dev_start (port_id); + +return 0 ; +} + +static int +eth_dev_stop_mp (uint16_t port_id) +{ +if (is_proc_primary ()) +return rte_eth_dev_stop (port_id); + +return 0 ; +} + +static void +mempool_free_mp (struct rte_mempool *mp) +{ +if (is_proc_primary ()) +rte_mempool_free (mp); +}
app/test-pmd/testpmd.c enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS; · line 578
Quoted 10 lines
app/test-pmd/testpmd.c +10
-0+ +static int +eth_dev_set_mtu_mp (uint16_t port_id,uint16_t mtu) +{ +if (is_proc_primary ()) +return rte_eth_dev_set_mtu (port_id, mtu); + +return 0 ; +} +
I think above functions should be removed and corresponding checks should be done in caller directly since above functions are used in single place only and just hide what actually happens in the case of secondary process. It is very misleading.
[snip]
Quoted 12 lines
app/test-pmd/testpmd.c +4
-3@@ -2495,21 +2565,24 @@ start_port(portid_t pid)return -1 ; }/* configure port */ - diag =rte_eth_dev_configure (pi, nb_rxq + nb_hairpinq, - nb_txq + nb_hairpinq, - &(port->dev_conf)); + diag =eth_dev_configure_mp (pi, + nb_rxq + nb_hairpinq, + nb_txq + nb_hairpinq, + &(port->dev_conf));if (diag !=0 ) {
app/test-pmd/testpmd.c start_port(portid_t pid) · line 2578
Quoted 10 lines
app/test-pmd/testpmd.c +6
-4-if (rte_atomic16_cmpset (&(port->port_status), -RTE_PORT_HANDLING ,RTE_PORT_STOPPED ) ==0 ) -printf ("Port %d can not be set back " -"to stopped\n" , pi); +if (rte_atomic16_cmpset ( + &(port->port_status), +RTE_PORT_HANDLING , +RTE_PORT_STOPPED ) ==0 ) +printf ("Port %d cannot be set back to stopped\n" , + pi);
Unrelated changes in the patch should be avoided since it just makes the review harder.
Quoted 12 lines
app/test-pmd/testpmd.c +1
-1printf ("Fail to configure port %d\n" , pi);/* try to reconfigure port next time */ port->need_reconfig =1 ;return -1 ; } } -if (port->need_reconfig_queues >0 ) { +if (port->need_reconfig_queues >0 &&is_proc_primary ()) { port->need_reconfig_queues =0 ;/* setup tx queues */ for (qi =0 ; qi < nb_txq; qi++) {
app/test-pmd/testpmd.c start_port(portid_t pid) · line 2609
Quoted 7 lines
app/test-pmd/testpmd.c +2
-2@@ -2532,8 +2605,8 @@ start_port(portid_t pid)if (rte_atomic16_cmpset (&(port->port_status),RTE_PORT_HANDLING ,RTE_PORT_STOPPED ) ==0 ) -printf ("Port %d can not be set back " -"to stopped\n" , pi); +printf ("Port %d cannot be set back to stopped\n" , + pi);
Unrelated changes in the patch should be avoided.
Quoted 11 lines
app/test-pmd/testpmd.c +1
-1printf ("Fail to configure port %d tx queues\n" , pi);/* try to reconfigure queues next time */ @@ -2610,16 +2683,16 @@ start_port(portid_t pid) cnt_pi++;/* start port */ - diag =rte_eth_dev_start (pi); + diag =eth_dev_start_mp (pi);if (diag <0 ) { printf("Fail to start port %d: %s\n", pi,
app/test-pmd/testpmd.c start_port(portid_t pid) · line 2695
Quoted 10 lines
app/test-pmd/testpmd.c +3
-3rte_strerror(-diag));/* Fail to setup rx queue, return */ if (rte_atomic16_cmpset (&(port->port_status), -RTE_PORT_HANDLING ,RTE_PORT_STOPPED ) ==0 ) -printf ("Port %d can not be set back to " -"stopped\n" , pi); +RTE_PORT_HANDLING ,RTE_PORT_STOPPED ) ==0 ) +printf ("Port %d cannot be set back to stopped\n" , + pi);
Unrelated changes in the patch should be avoided.
[snip]
Quoted 70 lines
doc/guides/testpmd_app_ug/run_app.rst +62
-0@@ -545,3 +545,85 @@ The command line options are: bit 0 - two hairpin ports loop The default value is 0. Hairpin will use single port mode and implicit Tx flow mode. + + +Testpmd Multi-Process Command-line Options +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The following are the command-line options for testpmd multi-process support: + +* primary process: + +.. code-block:: console + + sudo ./dpdk-testpmd --proc-type=auto -l 0-1 -- -i --rxq=4 --txq=4 \ + --num-procs=2 --proc-id=0 + +* secondary process: + +.. code-block:: console + + sudo ./dpdk-testpmd --proc-type=auto -l 2-3 -- -i --rxq=4 --txq=4 \ + --num-procs=2 --proc-id=1 + +The command line options are: + +* ``--num-procs=N`` + + The number of processes which will be used. + +* ``--proc-id=ID`` + + The ID of the current process (ID < num-procs). ID should be different in + primary process and secondary process, which starts from '0'. + +Calculation rule for queue: +All queues are allocated to different processes based on ``proc_num`` and +``proc_id``. +Calculation rule for the testpmd to allocate queues to each process: + +* start(queue start id) = proc_id * nb_q / num_procs + +* end(queue end id) = start + nb_q / num_procs + +For example, if testpmd is configured to have 4 Tx and Rx queues, +queues 0 and 1 will be used by the primary process and +queues 2 and 3 will be used by the secondary process. + +The number of queues should be a multiple of the number of processes. If not, +redundant queues will exist after queues are allocated to processes. If RSS +is enabled, packet loss occurs when traffic is sent to all processes at the same +time. Some traffic goes to redundant queues and cannot be forwarded. + +All the dev ops is supported in primary process. While secondary process is +not permitted to allocate or release shared memory, so some ops are not supported +as follows: + +- ``dev_configure`` +- ``dev_start`` +- ``dev_stop`` +- ``rx_queue_setup`` +- ``tx_queue_setup`` +- ``rx_queue_release`` +- ``tx_queue_release`` +
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 619
Quoted 10 lines
doc/guides/testpmd_app_ug/run_app.rst +10
-0+So, any command from testpmd which calls those APIs will not be supported in +secondary process, like: + +.. code-block:: console + + port config all rxq|txq|rxd|txd <value> + port config <port_id> rx_offload xxx on/off + port config <port_id> tx_offload xxx on/off + +etc.
I did the formatting cleanup, but I still think that testpmd guide should not dive into such level of details. It should rather highlight multi-process behaviour specifics.
Shouldn't testpmd store state in shared memory to avoid problems when primary is stopped while secondary is running etc.
Some testpmd features rely on reconfigure (i.e. simply change configuration and set flag that reconfigure is required), but configure does nothing and will simply ignore new settings. So, it could look very-very confusing from user point of view.
I'm not sure that it is acceptable to apply the patch in such state and open huge number of bugs in testpmd behaviour when multi-process is used.
I'd even consider to exclude unsupported commands from help etc. However, such level of care about user could be excessive for test tool.
IMHO, it should be no requirement to repeat the primary process command-line configuration in the second process command line (see --rxq=4 --txq=4 above). The information should be obtained from shared state. In theory primary process could even change some settings in interactive mode. I think testpmd should guarantee consistent behaviour even in such conditions. I.e. do not allow to stop ports used by forwarding running in secondary processes. Run-time queues setup and deferred start should be very carefully handled as well.
Quoted 11 lines
doc/guides/testpmd_app_ug/run_app.rst +10
-0+ +Stats is supported, stats will not change when one quits and starts, as they +share the same buffer to store the stats. Flow rules are maintained in process +level: primary and secondary has its own flow list (but one flow list in HW). +The two can see all the queues, so setting the flow rules for the other is OK. +But in the testpmd primary process receiving or transmitting packets from the +queue allocated for secondary process is not permitted, and same for secondary +process. + +Flow API and RSS are supported.
Hi, Andrew ,
Quoted 86 lines · Andrew Rybchenko
Quoted 20 lines · Andrew Rybchenko
From: "Min Hu (Connor)" <[email protected]>
For example the following commands run two testpmd processes:
* the primary process:
./dpdk-testpmd --proc-type=auto -l 0-1 -- -i \ --rxq=4 --txq=4 --num-procs=2 --proc-id=0* the secondary process:
./dpdk-testpmd --proc-type=auto -l 2-3 -- -i \ --rxq=4 --txq=4 --num-procs=2 --proc-id=1Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>Signed-off-by Andrew Rybchenko <[email protected]>Acked-by Xiaoyun Li <[email protected]>Acked-by Ajit Khaparde <[email protected]>Reviewed-by Ferruh Yigit <[email protected]>[snip]
Quoted 62 lines
app/test-pmd/testpmd.c +54
-0@@ -520,6 +520,62 @@ enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS;*/ uint32_t eth_link_speed; +/* +* ID of the current process in multi-process, used to +* configure the queues to be polled. +*/ +int proc_id; + +/* +* Number of processes in multi-process, used to +* configure the queues to be polled. +*/ +unsigned int num_procs =1 ; + +static int +eth_dev_configure_mp (uint16_t port_id,uint16_t nb_rx_q,uint16_t nb_tx_q, +const struct rte_eth_conf *dev_conf) +{ +if (is_proc_primary ()) +return rte_eth_dev_configure (port_id, nb_rx_q, nb_tx_q, + dev_conf); +return 0 ; +} + +static int +eth_dev_start_mp (uint16_t port_id) +{ +if (is_proc_primary ()) +return rte_eth_dev_start (port_id); + +return 0 ; +} + +static int +eth_dev_stop_mp (uint16_t port_id) +{ +if (is_proc_primary ()) +return rte_eth_dev_stop (port_id); + +return 0 ; +} + +static void +mempool_free_mp (struct rte_mempool *mp) +{ +if (is_proc_primary ()) +rte_mempool_free (mp); +} + +static int +eth_dev_set_mtu_mp (uint16_t port_id,uint16_t mtu) +{ +if (is_proc_primary ()) +return rte_eth_dev_set_mtu (port_id, mtu); + +return 0 ;
Quoted 9 lines
app/test-pmd/testpmd.c enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS; · line 578
Quoted 2 lines
app/test-pmd/testpmd.c +2
-0+} +I think above functions should be removed and corresponding checks should be done in caller directly since above functions are used in single place only and just hide what actually happens in the case of secondary process. It is very misleading.
This was done as Ferruh suggested in V9, and this could reduce the complexity for testpmd when added by the multi-process support.
Quoted 41 lines
[snip]
Quoted 12 lines
app/test-pmd/testpmd.c +4
-3@@ -2495,21 +2565,24 @@ start_port(portid_t pid)return -1 ; }/* configure port */ - diag =rte_eth_dev_configure (pi, nb_rxq + nb_hairpinq, - nb_txq + nb_hairpinq, - &(port->dev_conf)); + diag =eth_dev_configure_mp (pi, + nb_rxq + nb_hairpinq, + nb_txq + nb_hairpinq, + &(port->dev_conf));if (diag !=0 ) {app/test-pmd/testpmd.c start_port(portid_t pid) · line 2578
Quoted 10 lines
app/test-pmd/testpmd.c +6
-4-if (rte_atomic16_cmpset (&(port->port_status), -RTE_PORT_HANDLING ,RTE_PORT_STOPPED ) ==0 ) -printf ("Port %d can not be set back " -"to stopped\n" , pi); +if (rte_atomic16_cmpset ( + &(port->port_status), +RTE_PORT_HANDLING , +RTE_PORT_STOPPED ) ==0 ) +printf ("Port %d cannot be set back to stopped\n" , + pi);Unrelated changes in the patch should be avoided since it just makes the review harder.This will be fixed in v16.
Quoted 13 lines
app/test-pmd/testpmd.c +1
-1printf ("Fail to configure port %d\n" , pi);/* try to reconfigure port next time */ port->need_reconfig =1 ;return -1 ; } } -if (port->need_reconfig_queues >0 ) { +if (port->need_reconfig_queues >0 &&is_proc_primary ()) { port->need_reconfig_queues =0 ;/* setup tx queues */ for (qi =0 ; qi < nb_txq; qi++) {@@ -2532,8 +2605,8 @@ start_port(portid_t pid) if (rte_atomic16_cmpset(&(port->port_status),
Quoted 8 lines
app/test-pmd/testpmd.c start_port(portid_t pid) · line 2609
Quoted 6 lines
app/test-pmd/testpmd.c +2
-2RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) -printf ("Port %d can not be set back " -"to stopped\n" , pi); +printf ("Port %d cannot be set back to stopped\n" , + pi);Unrelated changes in the patch should be avoided.
This will be fixed in v16.
Quoted 14 lines
Quoted 14 lines
app/test-pmd/testpmd.c +1
-1printf ("Fail to configure port %d tx queues\n" , pi);/* try to reconfigure queues next time */ @@ -2610,16 +2683,16 @@ start_port(portid_t pid) cnt_pi++;/* start port */ - diag =rte_eth_dev_start (pi); + diag =eth_dev_start_mp (pi);if (diag <0 ) {printf ("Fail to start port %d: %s\n" , pi,rte_strerror (-diag));
Quoted 10 lines
app/test-pmd/testpmd.c start_port(portid_t pid) · line 2695
Quoted 8 lines
app/test-pmd/testpmd.c +3
-3/* Fail to setup rx queue, return */ if (rte_atomic16_cmpset (&(port->port_status), -RTE_PORT_HANDLING ,RTE_PORT_STOPPED ) ==0 ) -printf ("Port %d can not be set back to " -"stopped\n" , pi); +RTE_PORT_HANDLING ,RTE_PORT_STOPPED ) ==0 ) +printf ("Port %d cannot be set back to stopped\n" , + pi);Unrelated changes in the patch should be avoided.
This will be fixed in v16.
Quoted 88 lines
[snip]
Quoted 70 lines
doc/guides/testpmd_app_ug/run_app.rst +62
-0@@ -545,3 +545,85 @@ The command line options are: bit 0 - two hairpin ports loop The default value is 0. Hairpin will use single port mode and implicit Tx flow mode. + + +Testpmd Multi-Process Command-line Options +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The following are the command-line options for testpmd multi-process support: + +* primary process: + +.. code-block:: console + + sudo ./dpdk-testpmd --proc-type=auto -l 0-1 -- -i --rxq=4 --txq=4 \ + --num-procs=2 --proc-id=0 + +* secondary process: + +.. code-block:: console + + sudo ./dpdk-testpmd --proc-type=auto -l 2-3 -- -i --rxq=4 --txq=4 \ + --num-procs=2 --proc-id=1 + +The command line options are: + +* ``--num-procs=N`` + + The number of processes which will be used. + +* ``--proc-id=ID`` + + The ID of the current process (ID < num-procs). ID should be different in + primary process and secondary process, which starts from '0'. + +Calculation rule for queue: +All queues are allocated to different processes based on ``proc_num`` and +``proc_id``. +Calculation rule for the testpmd to allocate queues to each process: + +* start(queue start id) = proc_id * nb_q / num_procs + +* end(queue end id) = start + nb_q / num_procs + +For example, if testpmd is configured to have 4 Tx and Rx queues, +queues 0 and 1 will be used by the primary process and +queues 2 and 3 will be used by the secondary process. + +The number of queues should be a multiple of the number of processes. If not, +redundant queues will exist after queues are allocated to processes. If RSS +is enabled, packet loss occurs when traffic is sent to all processes at the same +time. Some traffic goes to redundant queues and cannot be forwarded. + +All the dev ops is supported in primary process. While secondary process is +not permitted to allocate or release shared memory, so some ops are not supported +as follows: + +- ``dev_configure`` +- ``dev_start`` +- ``dev_stop`` +- ``rx_queue_setup`` +- ``tx_queue_setup`` +- ``rx_queue_release`` +- ``tx_queue_release`` +doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 619
Quoted 10 lines
doc/guides/testpmd_app_ug/run_app.rst +10
-0+So, any command from testpmd which calls those APIs will not be supported in +secondary process, like: + +.. code-block:: console + + port config all rxq|txq|rxd|txd <value> + port config <port_id> rx_offload xxx on/off + port config <port_id> tx_offload xxx on/off + +etc.I did the formatting cleanup, but I still think that testpmd guide should not dive into such level of details. It should rather highlight multi-process behaviour specifics.
Quoted 2 lines
Shouldn't testpmd store state in shared memory to avoid problems when primary is stopped while secondary is running
This could be taken into consideration in future.
Quoted 6 lines
Some testpmd features rely on reconfigure (i.e. simply change configuration and set flag that reconfigure is required), but configure does nothing and will simply ignore new settings. So, it could look very-very confusing from user point of view.
Quoted 7 lines
I'm not sure that it is acceptable to apply the patch in such state and open huge number of bugs in testpmd behaviour when multi-process is used.
I'd even consider to exclude unsupported commands from help etc. However, such level of care about user could be excessive for test tool.
This has been done in doc.
Quoted 6 lines
IMHO, it should be no requirement to repeat the primary process command-line configuration in the second process command line (see --rxq=4 --txq=4 above). The information should be obtained from shared state. In theory primary process could even change some settings in interactive
We think keeping the command line in consistent between primary and secondary is easy to understand for users. While shared memory for keeping in order or communicating could be performed,but this could be done in future patch.
Quoted 5 lines
mode. I think testpmd should guarantee consistent behaviour even in such conditions. I.e. do not allow to stop ports used by forwarding running in secondary processes. Run-time queues setup and deferred start should be very carefully handled as well.
``dev_stop`` is not allowed in secondary, which has described in doc.
Quoted 11 lines
Quoted 11 lines
doc/guides/testpmd_app_ug/run_app.rst +10
-0+ +Stats is supported, stats will not change when one quits and starts, as they +share the same buffer to store the stats. Flow rules are maintained in process +level: primary and secondary has its own flow list (but one flow list in HW). +The two can see all the queues, so setting the flow rules for the other is OK. +But in the testpmd primary process receiving or transmitting packets from the +queue allocated for secondary process is not permitted, and same for secondary +process. + +Flow API and RSS are supported.Thanks for your comment,
This patch supports basic function for multi-process support in testpmd. I think other patches in future could enhance or optimize it, thanks.
Quoted 3 lines
.
Quoted 303 lines · Min Hu (Connor)
Hi, Andrew ,
Quoted 87 lines · Andrew Rybchenko
Quoted 20 lines · Andrew Rybchenko
From: "Min Hu (Connor)" <[email protected]>
For example the following commands run two testpmd processes:
* the primary process:
./dpdk-testpmd --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0* the secondary process:
./dpdk-testpmd --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>Signed-off-by Andrew Rybchenko <[email protected]>Acked-by Xiaoyun Li <[email protected]>Acked-by Ajit Khaparde <[email protected]>Reviewed-by Ferruh Yigit <[email protected]>[snip]
Quoted 63 lines
app/test-pmd/testpmd.c +0
-0@@ -520,6 +520,62 @@ enum rte_eth_rx_mq_mode rx_mq_mode =ETH_MQ_RX_VMDQ_DCB_RSS; */ uint32_t eth_link_speed;+/*
+ * ID of the current process in multi-process, used to
+ * configure the queues to be polled.
+ */app/test-pmd/testpmd.c +10
-0+int proc_id; + +/* +* Number of processes in multi-process, used to +* configure the queues to be polled. +*/ +unsigned int num_procs =1 ; + +static int +eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t
nb_tx_q,
+ const structrte_eth_conf*dev_conf)app/test-pmd/testpmd.c +39
-0+{ + if (is_proc_primary()) + return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q, + dev_conf); + return 0; +} + +static int +eth_dev_start_mp (uint16_t port_id) +{ + if (is_proc_primary()) + return rte_eth_dev_start(port_id); + + return 0; +} + +static int +eth_dev_stop_mp (uint16_t port_id) +{ + if (is_proc_primary()) + return rte_eth_dev_stop(port_id); + + return 0; +} + +static void +mempool_free_mp (struct rte_mempool *mp) +{ + if (is_proc_primary()) + rte_mempool_free(mp); +} + +static int +eth_dev_set_mtu_mp (uint16_t port_id,uint16_t mtu) +{ + if (is_proc_primary()) + return rte_eth_dev_set_mtu(port_id, mtu); + + return 0;Quoted 9 lines
app/test-pmd/testpmd.c
Quoted 2 lines
app/test-pmd/testpmd.c +2
-0+} +I think above functions should be removed and corresponding checks should be done in caller directly since above functions are used in single place only and just hide what actually happens in the case of secondary process. It is very misleading.
This was done as Ferruh suggested in V9, and this could reduce the complexity for testpmd when added by the multi-process support.
Quoted 41 lines
[snip]
Quoted 12 lines
app/test-pmd/testpmd.c +4
-3@@ -2495,21 +2565,24 @@ start_port(portid_t pid) return -1; } /* configure port */ - diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq, - nb_txq + nb_hairpinq, - &(port->dev_conf)); + diag = eth_dev_configure_mp(pi, + nb_rxq + nb_hairpinq, + nb_txq + nb_hairpinq, + &(port->dev_conf)); if (diag != 0) {app/test-pmd/testpmd.c start_port(portid_t pid) · line 2578
Quoted 10 lines
app/test-pmd/testpmd.c +6
-4- if (rte_atomic16_cmpset(&(port->port_status), - RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) - printf("Port %d can not be set back " - "to stopped\n", pi); + if (rte_atomic16_cmpset( + &(port->port_status), + RTE_PORT_HANDLING, + RTE_PORT_STOPPED) == 0) + printf("Port %d cannot be set back to stopped\n", + pi);Unrelated changes in the patch should be avoided since it just makes the review harder.This will be fixed in v16.
Quoted 13 lines
app/test-pmd/testpmd.c +1
-1printf("Fail to configure port %d\n", pi); /* try to reconfigure port next time */ port->need_reconfig = 1; return -1; } } - if (port->need_reconfig_queues > 0) { + if (port->need_reconfig_queues > 0 && is_proc_primary()) { port->need_reconfig_queues = 0; /* setup tx queues */ for (qi = 0; qi < nb_txq; qi++) {@@ -2532,8 +2605,8 @@ start_port(portid_t pid) if (rte_atomic16_cmpset(&(port->port_status),Quoted 8 lines
app/test-pmd/testpmd.c start_port(portid_t pid) · line 2609
Quoted 6 lines
app/test-pmd/testpmd.c +2
-2RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) - printf("Port %d can not be set back " - "to stopped\n", pi); + printf("Port %d cannot be set back to stopped\n", + pi);Unrelated changes in the patch should be avoided.
This will be fixed in v16.
Quoted 14 lines
Quoted 14 lines
app/test-pmd/testpmd.c +1
-1printf("Fail to configure port %d tx queues\n", pi); /* try to reconfigure queues next time */@@ -2610,16 +2683,16 @@ start_port(portid_t pid) cnt_pi++; /* start port */ - diag = rte_eth_dev_start(pi); + diag = eth_dev_start_mp(pi); if (diag < 0) { printf("Fail to start port %d: %s\n", pi, rte_strerror(-diag)); /* Fail to setup rx queue, return */ if (rte_atomic16_cmpset(&(port->port_status),Quoted 8 lines
app/test-pmd/testpmd.c start_port(portid_t pid) · line 2695
Quoted 6 lines
app/test-pmd/testpmd.c +3
-3- RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) - printf("Port %d can not be set back to " - "stopped\n", pi); + RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) + printf("Port %d cannot be set back to stopped\n", + pi);Unrelated changes in the patch should be avoided.
This will be fixed in v16.
Quoted 100 lines
[snip]
Quoted 83 lines
doc/guides/testpmd_app_ug/run_app.rst +0
-0@@ -545,3 +545,85 @@ The command line options are:bit 0 - two hairpin ports loop
The default value is 0. Hairpin will use single port mode and implicit Tx flow mode.doc/guides/testpmd_app_ug/run_app.rst +6
-0+ + +Testpmd Multi-Process Command-line Options +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The following are the command-line options for testpmd multi-processsupport:
doc/guides/testpmd_app_ug/run_app.rst +24
-2+ +* primary process: + +.. code-block:: console + + sudo ./dpdk-testpmd --proc-type=auto -l 0-1 -- -i --rxq=4 --txq=4 \ + --num-procs=2 --proc-id=0 + +* secondary process: + +.. code-block:: console + + sudo ./dpdk-testpmd --proc-type=auto -l 2-3 -- -i --rxq=4 --txq=4 \ + --num-procs=2 --proc-id=1 + +The command line options are: + +* ``--num-procs=N`` + + The number of processes which will be used. + +* ``--proc-id=ID`` + + The ID of the current process (ID < num-procs). ID should bedifferent in
+ primary process and secondary process, which starts from '0'.doc/guides/testpmd_app_ug/run_app.rst +3
-0+ +Calculation rule for queue: +All queues are allocated to different processes based on``proc_num`` and
+``proc_id``.doc/guides/testpmd_app_ug/run_app.rst +11
-0+Calculation rule for the testpmd to allocate queues to each process: + +* start(queue start id) = proc_id * nb_q / num_procs + +* end(queue end id) = start + nb_q / num_procs + +For example, if testpmd is configured to have 4 Tx and Rx queues, +queues 0 and 1 will be used by the primary process and +queues 2 and 3 will be used by the secondary process. + +The number of queues should be a multiple of the number ofprocesses. If not,
+redundant queues will exist after queues are allocated to processes. If RSS
+is enabled, packet loss occurs when traffic is sent to all processes at the samedoc/guides/testpmd_app_ug/run_app.rst +3
-0+time. Some traffic goes to redundant queues and cannot be forwarded. + +All the dev ops is supported in primary process. While secondaryprocess is
+not permitted to allocate or release shared memory, so some ops are not supporteddoc/guides/testpmd_app_ug/run_app.rst +11
-0+as follows: + +- ``dev_configure`` +- ``dev_start`` +- ``dev_stop`` +- ``rx_queue_setup`` +- ``tx_queue_setup`` +- ``rx_queue_release`` +- ``tx_queue_release`` + +So, any command from testpmd which calls those APIs will not besupported in
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 619
Quoted 9 lines
doc/guides/testpmd_app_ug/run_app.rst +9
-0+secondary process, like: + +.. code-block:: console + + port config all rxq|txq|rxd|txd <value> + port config <port_id> rx_offload xxx on/off + port config <port_id> tx_offload xxx on/off + +etc.I did the formatting cleanup, but I still think that testpmd guide should not dive into such level of details. It should rather highlight multi-process behaviour specifics.
Quoted 2 lines
Shouldn't testpmd store state in shared memory to avoid problems when primary is stopped while secondary is running
This could be taken into consideration in future.
Quoted 6 lines
Some testpmd features rely on reconfigure (i.e. simply change configuration and set flag that reconfigure is required), but configure does nothing and will simply ignore new settings. So, it could look very-very confusing from user point of view.
Quoted 7 lines
I'm not sure that it is acceptable to apply the patch in such state and open huge number of bugs in testpmd behaviour when multi-process is used.
I'd even consider to exclude unsupported commands from help etc. However, such level of care about user could be excessive for test tool.
This has been done in doc.
Quoted 6 lines
IMHO, it should be no requirement to repeat the primary process command-line configuration in the second process command line (see --rxq=4 --txq=4 above). The information should be obtained from shared state. In theory primary process could even change some settings in interactive
We think keeping the command line in consistent between primary and secondary is easy to understand for users. While shared memory for keeping in order or communicating could be performed,but this could be done in future patch.
Quoted 6 lines
Quoted 5 lines
mode. I think testpmd should guarantee consistent behaviour even in such conditions. I.e. do not allow to stop ports used by forwarding running in secondary processes. Run-time queues setup and deferred start should be very carefully handled as well.
``dev_stop`` is not allowed in secondary, which has described in doc.
I'm talking about dev_stop in primary while secondary is running.
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
doc/guides/testpmd_app_ug/run_app.rst +2
-0+ +Stats is supported, stats will not change when one quits and starts,as they
+share the same buffer to store the stats. Flow rules are maintained in process
+level: primary and secondary has its own flow list (but one flow list in HW).
+The two can see all the queues, so setting the flow rules for the other is OK.
+But in the testpmd primary process receiving or transmitting packets from the
Quoted 8 lines
Quoted 6 lines
Quoted 6 lines
+queue allocated for secondary process is not permitted, and same for secondary
doc/guides/testpmd_app_ug/run_app.rst +3
-0+process. + +Flow API and RSS are supported.Thanks for your comment,
This patch supports basic function for multi-process support in testpmd. I think other patches in future could enhance or optimize it, thanks.
IMHO, as I state above, current state is insufficient to consider is a start point to be applied.
Hi,Andrew ,
Quoted 334 lines · Andrew Rybchenko
Quoted 303 lines · Min Hu (Connor)
Hi, Andrew ,
Quoted 87 lines · Andrew Rybchenko
Quoted 20 lines · Andrew Rybchenko
From: "Min Hu (Connor)" <[email protected]>
For example the following commands run two testpmd processes:
* the primary process:
./dpdk-testpmd --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0* the secondary process:
./dpdk-testpmd --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>Signed-off-by Andrew Rybchenko <[email protected]>Acked-by Xiaoyun Li <[email protected]>Acked-by Ajit Khaparde <[email protected]>Reviewed-by Ferruh Yigit <[email protected]>[snip]
Quoted 63 lines
app/test-pmd/testpmd.c +53
-0@@ -520,6 +520,62 @@ enum rte_eth_rx_mq_mode rx_mq_mode =ETH_MQ_RX_VMDQ_DCB_RSS; */ uint32_t eth_link_speed; +/* +* ID of the current process in multi-process, used to +* configure the queues to be polled. +*/ +int proc_id; + +/* +* Number of processes in multi-process, used to +* configure the queues to be polled. +*/ +unsigned int num_procs =1 ; + +static int +eth_dev_configure_mp (uint16_t port_id,uint16_t nb_rx_q,uint16_t nb_tx_q, + const struct rte_eth_conf *dev_conf) +{ + if (is_proc_primary()) + return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q, + dev_conf); + return 0; +} + +static int +eth_dev_start_mp (uint16_t port_id) +{ + if (is_proc_primary()) + return rte_eth_dev_start(port_id); + + return 0; +} + +static int +eth_dev_stop_mp (uint16_t port_id) +{ + if (is_proc_primary()) + return rte_eth_dev_stop(port_id); + + return 0; +} + +static void +mempool_free_mp (struct rte_mempool *mp) +{ + if (is_proc_primary()) + rte_mempool_free(mp); +} + +static int +eth_dev_set_mtu_mp (uint16_t port_id,uint16_t mtu) +{ + if (is_proc_primary()) + return rte_eth_dev_set_mtu(port_id, mtu); + + return 0;Quoted 9 lines
app/test-pmd/testpmd.c enum rte_eth_rx_mq_mode rx_mq_mode = · line 579
Quoted 2 lines
app/test-pmd/testpmd.c +2
-0+} +I think above functions should be removed and corresponding checks should be done in caller directly since above functions are used in single place only and just hide what actually happens in the case of secondary process. It is very misleading.
This was done as Ferruh suggested in V9, and this could reduce the complexity for testpmd when added by the multi-process support.
Quoted 41 lines
[snip]
Quoted 12 lines
app/test-pmd/testpmd.c +4
-3@@ -2495,21 +2565,24 @@ start_port(portid_t pid) return -1; } /* configure port */ - diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq, - nb_txq + nb_hairpinq, - &(port->dev_conf)); + diag = eth_dev_configure_mp(pi, + nb_rxq + nb_hairpinq, + nb_txq + nb_hairpinq, + &(port->dev_conf)); if (diag != 0) {app/test-pmd/testpmd.c start_port(portid_t pid) · line 2578
Quoted 10 lines
app/test-pmd/testpmd.c +6
-4- if (rte_atomic16_cmpset(&(port->port_status), - RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) - printf("Port %d can not be set back " - "to stopped\n", pi); + if (rte_atomic16_cmpset( + &(port->port_status), + RTE_PORT_HANDLING, + RTE_PORT_STOPPED) == 0) + printf("Port %d cannot be set back to stopped\n", + pi);Unrelated changes in the patch should be avoided since it just makes the review harder.This will be fixed in v16.
Quoted 13 lines
app/test-pmd/testpmd.c +1
-1printf("Fail to configure port %d\n", pi); /* try to reconfigure port next time */ port->need_reconfig = 1; return -1; } } - if (port->need_reconfig_queues > 0) { + if (port->need_reconfig_queues > 0 && is_proc_primary()) { port->need_reconfig_queues = 0; /* setup tx queues */ for (qi = 0; qi < nb_txq; qi++) {@@ -2532,8 +2605,8 @@ start_port(portid_t pid) if (rte_atomic16_cmpset(&(port->port_status),Quoted 8 lines
app/test-pmd/testpmd.c start_port(portid_t pid) · line 2609
Quoted 6 lines
app/test-pmd/testpmd.c +2
-2RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) - printf("Port %d can not be set back " - "to stopped\n", pi); + printf("Port %d cannot be set back to stopped\n", + pi);Unrelated changes in the patch should be avoided.
This will be fixed in v16.
Quoted 14 lines
Quoted 14 lines
app/test-pmd/testpmd.c +1
-1printf("Fail to configure port %d tx queues\n", pi); /* try to reconfigure queues next time */@@ -2610,16 +2683,16 @@ start_port(portid_t pid) cnt_pi++; /* start port */ - diag = rte_eth_dev_start(pi); + diag = eth_dev_start_mp(pi); if (diag < 0) { printf("Fail to start port %d: %s\n", pi, rte_strerror(-diag)); /* Fail to setup rx queue, return */ if (rte_atomic16_cmpset(&(port->port_status),Quoted 8 lines
app/test-pmd/testpmd.c start_port(portid_t pid) · line 2695
Quoted 6 lines
app/test-pmd/testpmd.c +3
-3- RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) - printf("Port %d can not be set back to " - "stopped\n", pi); + RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) + printf("Port %d cannot be set back to stopped\n", + pi);Unrelated changes in the patch should be avoided.
This will be fixed in v16.
Quoted 100 lines
[snip]
Quoted 83 lines
doc/guides/testpmd_app_ug/run_app.rst +0
-0@@ -545,3 +545,85 @@ The command line options are: bit 0 - two hairpin ports loop The default value is 0. Hairpin will use single port mode andimplicit Tx flow mode.
doc/guides/testpmd_app_ug/run_app.rst +6
-0+ + +Testpmd Multi-Process Command-line Options +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The following are the command-line options for testpmd multi-processsupport:
doc/guides/testpmd_app_ug/run_app.rst +24
-2+ +* primary process: + +.. code-block:: console + + sudo ./dpdk-testpmd --proc-type=auto -l 0-1 -- -i --rxq=4 --txq=4 \ + --num-procs=2 --proc-id=0 + +* secondary process: + +.. code-block:: console + + sudo ./dpdk-testpmd --proc-type=auto -l 2-3 -- -i --rxq=4 --txq=4 \ + --num-procs=2 --proc-id=1 + +The command line options are: + +* ``--num-procs=N`` + + The number of processes which will be used. + +* ``--proc-id=ID`` + + The ID of the current process (ID < num-procs). ID should bedifferent in
+ primary process and secondary process, which starts from '0'.doc/guides/testpmd_app_ug/run_app.rst +3
-0+ +Calculation rule for queue: +All queues are allocated to different processes based on``proc_num`` and
+``proc_id``.doc/guides/testpmd_app_ug/run_app.rst +11
-0+Calculation rule for the testpmd to allocate queues to each process: + +* start(queue start id) = proc_id * nb_q / num_procs + +* end(queue end id) = start + nb_q / num_procs + +For example, if testpmd is configured to have 4 Tx and Rx queues, +queues 0 and 1 will be used by the primary process and +queues 2 and 3 will be used by the secondary process. + +The number of queues should be a multiple of the number ofprocesses. If not,
+redundant queues will exist after queues are allocated to processes. If RSS
+is enabled, packet loss occurs when traffic is sent to all processes at the samedoc/guides/testpmd_app_ug/run_app.rst +3
-0+time. Some traffic goes to redundant queues and cannot be forwarded. + +All the dev ops is supported in primary process. While secondaryprocess is
+not permitted to allocate or release shared memory, so some ops are not supporteddoc/guides/testpmd_app_ug/run_app.rst +11
-0+as follows: + +- ``dev_configure`` +- ``dev_start`` +- ``dev_stop`` +- ``rx_queue_setup`` +- ``tx_queue_setup`` +- ``rx_queue_release`` +- ``tx_queue_release`` + +So, any command from testpmd which calls those APIs will not besupported in
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 619
Quoted 9 lines
doc/guides/testpmd_app_ug/run_app.rst +9
-0+secondary process, like: + +.. code-block:: console + + port config all rxq|txq|rxd|txd <value> + port config <port_id> rx_offload xxx on/off + port config <port_id> tx_offload xxx on/off + +etc.I did the formatting cleanup, but I still think that testpmd guide should not dive into such level of details. It should rather highlight multi-process behaviour specifics.
Quoted 2 lines
Shouldn't testpmd store state in shared memory to avoid problems when primary is stopped while secondary is running
This could be taken into consideration in future.
Quoted 6 lines
Some testpmd features rely on reconfigure (i.e. simply change configuration and set flag that reconfigure is required), but configure does nothing and will simply ignore new settings. So, it could look very-very confusing from user point of view.
Quoted 7 lines
I'm not sure that it is acceptable to apply the patch in such state and open huge number of bugs in testpmd behaviour when multi-process is used.
I'd even consider to exclude unsupported commands from help etc. However, such level of care about user could be excessive for test tool.
This has been done in doc.
Quoted 6 lines
IMHO, it should be no requirement to repeat the primary process command-line configuration in the second process command line (see --rxq=4 --txq=4 above). The information should be obtained from shared state. In theory primary process could even change some settings in interactive
We think keeping the command line in consistent between primary and secondary is easy to understand for users. While shared memory for keeping in order or communicating could be performed,but this could be done in future patch.
Quoted 6 lines
Quoted 5 lines
mode. I think testpmd should guarantee consistent behaviour even in such conditions. I.e. do not allow to stop ports used by forwarding running in secondary processes. Run-time queues setup and deferred start should be very carefully handled as well.
``dev_stop`` is not allowed in secondary, which has described in doc.
I'm talking about
dev_stopin primary while secondary is running.Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
doc/guides/testpmd_app_ug/run_app.rst +2
-0+ +Stats is supported, stats will not change when one quits and starts,as they
+share the same buffer to store the stats. Flow rules are maintained in process
+level: primary and secondary has its own flow list (but one flow list in HW).
+The two can see all the queues, so setting the flow rules for the other is OK.
+But in the testpmd primary process receiving or transmitting packets from theQuoted 8 lines
Quoted 6 lines
Quoted 6 lines
+queue allocated for secondary process is not permitted, and same for secondary
doc/guides/testpmd_app_ug/run_app.rst +3
-0+process. + +Flow API and RSS are supported.Thanks for your comment,
This patch supports basic function for multi-process support in testpmd. I think other patches in future could enhance or optimize it, thanks.
Quoted 2 lines
IMHO, as I state above, current state is insufficient to consider is a start point to be applied.
OK, what is the critical(the most important) as a start point to be applied? I will take action to fix it,as I will the patch could be applied before V21.08.
Quoted 2 lines
.
PATCH v16 1/1 app/testpmd: support multi-process
In reply to Min Hu (Connor)This patch adds multi-process support for testpmd. For example the following commands run two testpmd processes:
* the primary process:
./dpdk-testpmd --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0
* the secondary process:
./dpdk-testpmd --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1
---
V16:
* revert unrelated changes.
* add some restrictions in doc.
v15:
* Fixed release notes.
* Cleanup documentation.
v14:
* Fixed comments by Andrew Rybchenko.
v13:
* Modified the doc syntax.
v12:
* Updated doc info.
v11:
* Fixed some minor syntax.
v10:
* Hid process type checks behind new functions.
* Added comments.
v9:
* Updated release notes and rst doc.
* Deleted deprecated codes.
* move macro and variable.
v8:
* Added warning info about queue numbers and process numbers.
v7:
* Fixed compiling error for unexpected unindent.
v6:
* Add rte flow description for multiple process.
v5:
* Fixed run_app.rst for multiple process description.
* Fix compiling error.
v4:
* Fixed minimum vlaue of Rxq or Txq in doc.
v3:
* Fixed compiling error using gcc10.0.
v2:
* Added document for this patch.
--- app/test-pmd/cmdline.c | 6 ++ app/test-pmd/config.c | 20 ++++++- app/test-pmd/parameters.c | 9 +++ app/test-pmd/testpmd.c | 106 ++++++++++++++++++++++++++++----- app/test-pmd/testpmd.h | 9 +++ doc/guides/rel_notes/release_21_08.rst | 3 + doc/guides/testpmd_app_ug/run_app.rst | 84 ++++++++++++++++++++++++++ 7 files changed, 220 insertions(+), 17 deletions(-)
app/test-pmd/cmdline.c +6 -0
@@ -5459,6 +5459,12 @@ cmd_set_flush_rx_parsed(void *parsed_result, __rte_unused void *data) {struct cmd_set_flush_rx *res = parsed_result; + +if (num_procs >1 && (strcmp (res->mode,"on" ) ==0 )) { +printf ("multi-process doesn't support to flush Rx queues.\n" ); +return ; + } + no_flush_rx = (uint8_t )((strcmp (res->mode,"on" ) ==0 ) ?0 :1 ); }
app/test-pmd/config.c +19 -1
@@ -2976,6 +2976,8 @@ rss_fwd_config_setup(void)queueid_t rxq;queueid_t nb_q;streamid_t sm_id; +int start; +int end; nb_q = nb_rxq;if (nb_q > nb_txq)
@@ -2993,7 +2995,21 @@ rss_fwd_config_setup(void)init_fwd_streams ();setup_fwd_config_of_each_lcore (&cur_fwd_config); - rxp =0 ; rxq =0 ; + +if (proc_id >0 && nb_q % num_procs !=0 ) +printf ("Warning! queue numbers should be multiple of processes, or packet loss will happen.\n" ); + +/** +* In multi-process, All queues are allocated to different +* processes based on num_procs and proc_id. For example: +* if supports 4 queues(nb_q), 2 processes(num_procs), +* the 0~1 queue for primary process. +* the 2~3 queue for secondary process. +*/ + start = proc_id * nb_q / num_procs; + end = start + nb_q / num_procs; + rxp =0 ; + rxq = start;for (sm_id =0 ; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {struct fwd_stream *fs;
@@ -3010,6 +3026,8 @@ rss_fwd_config_setup(void)continue ; rxp =0 ; rxq++; +if (rxq >= end) + rxq = start; } }
app/test-pmd/parameters.c +9 -0
@@ -509,6 +509,9 @@ parse_link_speed(int n)void launch_args_parse (int argc,char ** argv) { +#define PARAM_PROC_ID "proc-id" +#define PARAM_NUM_PROCS "num-procs" +int n, opt;char **argvopt;int opt_idx;
@@ -628,6 +631,8 @@ launch_args_parse(int argc, char** argv) { "rx-mq-mode", 1, 0, 0 }, { "record-core-cycles", 0, 0, 0 }, { "record-burst-stats", 0, 0, 0 }, + { PARAM_NUM_PROCS, 1, 0, 0 }, + { PARAM_PROC_ID, 1, 0, 0 }, { 0, 0, 0, 0 }, };
@@ -1394,6 +1399,10 @@ launch_args_parse(int argc, char** argv) record_core_cycles =1 ;if (!strcmp (lgopts[opt_idx].name,"record-burst-stats" )) record_burst_stats =1 ; +if (!strcmp (lgopts[opt_idx].name,PARAM_NUM_PROCS )) + num_procs =atoi (optarg); +if (!strcmp (lgopts[opt_idx].name,PARAM_PROC_ID )) + proc_id =atoi (optarg);break ;case 'h' :usage (argv[0 ]);
app/test-pmd/testpmd.c +90 -16
179 lines not on this page · raw message
app/test-pmd/testpmd.h +9 -0
@@ -632,6 +632,15 @@ extern enum rte_eth_rx_mq_mode rx_mq_mode;extern struct rte_flow_action_conntrack conntrack_context; +extern int proc_id; +extern unsigned int num_procs; + +static inline bool +is_proc_primary (void ) +{ +return rte_eal_process_type () ==RTE_PROC_PRIMARY ; +} +static inline unsigned int lcore_num (void ) {
doc/guides/rel_notes/release_21_08.rst +3 -0
@@ -90,6 +90,9 @@ New Features usecases. Configuration happens via standard rawdev enq/deq operations. See the :doc:`../rawdevs/cnxk_bphy` rawdev guide for more details on this driver. +* **Added support multi-process for testpmd.** + Added command-line options to specify total number of processes and + current process ID. Each process owns subset of Rx and Tx queues. Removed Items -------------
doc/guides/testpmd_app_ug/run_app.rst +84 -0
87 lines not on this page · raw message
Signature · 1 line
2.7.4
10/07/2021 05:50, Min Hu (Connor):
Quoted 20 lines
This patch adds multi-process support for testpmd. For example the following commands run two testpmd processes:
* the primary process:
./dpdk-testpmd --proc-type=auto -l 0-1 -- -i \ --rxq=4 --txq=4 --num-procs=2 --proc-id=0* the secondary process:
./dpdk-testpmd --proc-type=auto -l 2-3 -- -i \ --rxq=4 --txq=4 --num-procs=2 --proc-id=1Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>Signed-off-by Andrew Rybchenko <[email protected]>Acked-by Xiaoyun Li <[email protected]>Acked-by Ajit Khaparde <[email protected]>Reviewed-by Ferruh Yigit <[email protected]>
Quoted 4 lines
---
V16:
* revert unrelated changes.
* add some restrictions in doc.
I didn't see clear agreement to integrate this feature in DPDK 21.08.
BTW, the testpmd maintainer was not Cc'ed.
Quoted 26 lines · Thomas Monjalon
10/07/2021 05:50, Min Hu (Connor):
Quoted 20 lines
This patch adds multi-process support for testpmd. For example the following commands run two testpmd processes:
* the primary process:
./dpdk-testpmd --proc-type=auto -l 0-1 -- -i \ --rxq=4 --txq=4 --num-procs=2 --proc-id=0* the secondary process:
./dpdk-testpmd --proc-type=auto -l 2-3 -- -i \ --rxq=4 --txq=4 --num-procs=2 --proc-id=1Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>Signed-off-by Andrew Rybchenko <[email protected]>Acked-by Xiaoyun Li <[email protected]>Acked-by Ajit Khaparde <[email protected]>Reviewed-by Ferruh Yigit <[email protected]>Quoted 4 lines
---
V16:
* revert unrelated changes.
* add some restrictions in doc.
Quoted 4 lines
I didn't see clear agreement to integrate this feature in DPDK 21.08.
BTW, the testpmd maintainer was not Cc'ed.
Hi, Thomas,
This patch has two "acked" and one "reviewed-by".
And Xiaoyun Li is the maintainer, I has already Cc'ed to him/her, please check it out, thanks.
Quoted 3 lines
.
26/07/2021 02:26, Min Hu (Connor):
Quoted 27 lines · Thomas Monjalon
Quoted 26 lines
10/07/2021 05:50, Min Hu (Connor):
Quoted 24 lines
This patch adds multi-process support for testpmd. For example the following commands run two testpmd processes:
* the primary process:
./dpdk-testpmd --proc-type=auto -l 0-1 -- -i \ --rxq=4 --txq=4 --num-procs=2 --proc-id=0* the secondary process:
./dpdk-testpmd --proc-type=auto -l 2-3 -- -i \ --rxq=4 --txq=4 --num-procs=2 --proc-id=1Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>Signed-off-by Andrew Rybchenko <[email protected]>Acked-by Xiaoyun Li <[email protected]>Acked-by Ajit Khaparde <[email protected]>Reviewed-by Ferruh Yigit <[email protected]>---
V16:
* revert unrelated changes.
* add some restrictions in doc.
Quoted 8 lines
Quoted 4 lines
I didn't see clear agreement to integrate this feature in DPDK 21.08.
BTW, the testpmd maintainer was not Cc'ed.
Hi, Thomas,
This patch has two "acked" and one "reviewed-by".
And Xiaoyun Li is the maintainer, I has already Cc'ed to him/her, please check it out, thanks.
Andrew's comment from v15:
"
IMHO, as I state above, current state is insufficient to consider is a start point to be applied.
"
Quoted 37 lines · Thomas Monjalon
26/07/2021 02:26, Min Hu (Connor):
Quoted 27 lines · Thomas Monjalon
Quoted 26 lines
10/07/2021 05:50, Min Hu (Connor):
Quoted 24 lines
This patch adds multi-process support for testpmd. For example the following commands run two testpmd processes:
* the primary process:
./dpdk-testpmd --proc-type=auto -l 0-1 -- -i \ --rxq=4 --txq=4 --num-procs=2 --proc-id=0* the secondary process:
./dpdk-testpmd --proc-type=auto -l 2-3 -- -i \ --rxq=4 --txq=4 --num-procs=2 --proc-id=1Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>Signed-off-by Andrew Rybchenko <[email protected]>Acked-by Xiaoyun Li <[email protected]>Acked-by Ajit Khaparde <[email protected]>Reviewed-by Ferruh Yigit <[email protected]>---
V16:
* revert unrelated changes.
* add some restrictions in doc.Quoted 8 lines
Quoted 4 lines
I didn't see clear agreement to integrate this feature in DPDK 21.08.
BTW, the testpmd maintainer was not Cc'ed.
Hi, Thomas,
This patch has two "acked" and one "reviewed-by".
And Xiaoyun Li is the maintainer, I has already Cc'ed to him/her, please check it out, thanks.
Quoted 5 lines
Andrew's comment from v15:
"
IMHO, as I state above, current state is insufficient to consider is a start point to be applied.
"
Hi, Andrew, Thomos,
What is the least thing should be done for this patch to
meet the state sufficient to be applied?
Quoted 4 lines
.
Quoted 40 lines · Min Hu (Connor)
Quoted 37 lines · Thomas Monjalon
26/07/2021 02:26, Min Hu (Connor):
Quoted 35 lines · Thomas Monjalon
Quoted 26 lines
10/07/2021 05:50, Min Hu (Connor):
Quoted 20 lines
This patch adds multi-process support for testpmd. For example the following commands run two testpmd processes:
* the primary process:
./dpdk-testpmd --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0* the secondary process:
./dpdk-testpmd --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>Signed-off-by Andrew Rybchenko <[email protected]>Acked-by Xiaoyun Li <[email protected]>Acked-by Ajit Khaparde <[email protected]>Reviewed-by Ferruh Yigit <[email protected]>Quoted 4 lines
---
V16:
* revert unrelated changes.
* add some restrictions in doc.Quoted 4 lines
I didn't see clear agreement to integrate this feature in DPDK 21.08.
BTW, the testpmd maintainer was not Cc'ed.
Hi, Thomas,
This patch has two "acked" and one "reviewed-by".
And Xiaoyun Li is the maintainer, I has already Cc'ed to him/her, please check it out, thanks.
Quoted 8 lines
Quoted 5 lines
Andrew's comment from v15:
"
IMHO, as I state above, current state is insufficient to consider is a start point to be applied.
"Hi, Andrew, Thomos,
What is the least thing should be done for this patch to
meet the state sufficient to be applied?
Hi, all,
How about this patch, could it be applied?
Quoted 6 lines
Quoted 5 lines
.
.
02/08/2021 03:51, Min Hu (Connor):
Quoted 44 lines · Min Hu (Connor)
Quoted 43 lines · Thomas Monjalon
Quoted 42 lines
26/07/2021 02:26, Min Hu (Connor):
Quoted 27 lines · Thomas Monjalon
Quoted 26 lines
10/07/2021 05:50, Min Hu (Connor):
Quoted 24 lines
This patch adds multi-process support for testpmd. For example the following commands run two testpmd processes:
* the primary process:
./dpdk-testpmd --proc-type=auto -l 0-1 -- -i \ --rxq=4 --txq=4 --num-procs=2 --proc-id=0* the secondary process:
./dpdk-testpmd --proc-type=auto -l 2-3 -- -i \ --rxq=4 --txq=4 --num-procs=2 --proc-id=1Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>Signed-off-by Andrew Rybchenko <[email protected]>Acked-by Xiaoyun Li <[email protected]>Acked-by Ajit Khaparde <[email protected]>Reviewed-by Ferruh Yigit <[email protected]>---
V16:
* revert unrelated changes.
* add some restrictions in doc.Quoted 8 lines
Quoted 4 lines
I didn't see clear agreement to integrate this feature in DPDK 21.08.
BTW, the testpmd maintainer was not Cc'ed.
Hi, Thomas,
This patch has two "acked" and one "reviewed-by".
And Xiaoyun Li is the maintainer, I has already Cc'ed to him/her, please check it out, thanks.Andrew's comment from v15:
"
IMHO, as I state above, current state is insufficient to consider is a start point to be applied.
"
Quoted 5 lines
Quoted 3 lines
Hi, Andrew, Thomos,
What is the least thing should be done for this patch to
meet the state sufficient to be applied?Hi, all,
How about this patch, could it be applied?
I was not confident enough to apply it in 21.08. When others will be back, we'll discuss it again for 21.11.
Singh, Aman Deep
Acked-byQuoted 43 lines · Thomas Monjalon
02/08/2021 03:51, Min Hu (Connor):
Quoted 42 lines · Min Hu (Connor)
Quoted 41 lines · Thomas Monjalon
Quoted 40 lines
26/07/2021 02:26, Min Hu (Connor):
Quoted 26 lines · Thomas Monjalon
Quoted 25 lines
10/07/2021 05:50, Min Hu (Connor):
Quoted 24 lines
This patch adds multi-process support for testpmd. For example the following commands run two testpmd processes:
* the primary process:
./dpdk-testpmd --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0* the secondary process:
./dpdk-testpmd --proc-type=auto -l 2-3 -- -i \ --rxq=4 --txq=4 --num-procs=2 --proc-id=1Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>Signed-off-by Andrew Rybchenko <[email protected]>Acked-by Xiaoyun Li <[email protected]>Acked-by Ajit Khaparde <[email protected]>Reviewed-by Ferruh Yigit <[email protected]>---
V16:
* revert unrelated changes.
* add some restrictions in doc.Quoted 8 lines
Quoted 4 lines
I didn't see clear agreement to integrate this feature in DPDK 21.08.
BTW, the testpmd maintainer was not Cc'ed.
Hi, Thomas,
This patch has two "acked" and one "reviewed-by".
And Xiaoyun Li is the maintainer, I has already Cc'ed to him/her, please check it out, thanks.Andrew's comment from v15:
"
IMHO, as I state above, current state is insufficient to consider is a start point to be applied.
"
Quoted 8 lines
Quoted 5 lines
Quoted 3 lines
Hi, Andrew, Thomos,
What is the least thing should be done for this patch to
meet the state sufficient to be applied?Hi, all,
How about this patch, could it be applied?I was not confident enough to apply it in 21.08. When others will be back, we'll discuss it again for 21.11.
For 21.11 release.
Quoted 48 lines · Aman Deep
Quoted 46 lines · Thomas Monjalon
02/08/2021 03:51, Min Hu (Connor):
Quoted 45 lines · Min Hu (Connor)
Quoted 44 lines · Thomas Monjalon
Quoted 35 lines
26/07/2021 02:26, Min Hu (Connor):
Quoted 26 lines · Thomas Monjalon
Quoted 25 lines
10/07/2021 05:50, Min Hu (Connor):
Quoted 24 lines
This patch adds multi-process support for testpmd. For example the following commands run two testpmd processes:
* the primary process:
./dpdk-testpmd --proc-type=auto -l 0-1 -- -i \ --rxq=4 --txq=4 --num-procs=2 --proc-id=0* the secondary process:
./dpdk-testpmd --proc-type=auto -l 2-3 -- -i \ --rxq=4 --txq=4 --num-procs=2 --proc-id=1Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>Signed-off-by Andrew Rybchenko <[email protected]>Acked-by Xiaoyun Li <[email protected]>Acked-by Ajit Khaparde <[email protected]>Reviewed-by Ferruh Yigit <[email protected]>---
V16:
* revert unrelated changes.
* add some restrictions in doc.Quoted 8 lines
Quoted 4 lines
I didn't see clear agreement to integrate this feature in DPDK 21.08.
BTW, the testpmd maintainer was not Cc'ed.
Hi, Thomas,
This patch has two "acked" and one "reviewed-by".
And Xiaoyun Li is the maintainer, I has already Cc'ed to him/her, please check it out, thanks.Quoted 5 lines
Andrew's comment from v15:
"
IMHO, as I state above, current state is insufficient to consider is a start point to be applied.
"Hi, Andrew, Thomos,
What is the least thing should be done for this patch to
meet the state sufficient to be applied?
Quoted 9 lines
Quoted 5 lines
Quoted 2 lines
Hi, all,
How about this patch, could it be applied?I was not confident enough to apply it in 21.08. When others will be back, we'll discuss it again for 21.11.
For 21.11 release.
Acked-by Aman Deep Singh <[email protected]>
Hi Connor,
Can you please rebase it to latest dpdk (v21.11-rc0) so we can proceed with it?
I am for getting the patch as early as possible in the release cycle, specially before it is getting busy for -rc1.
Thanks,
ferruh
Thanks Ferruh, I will handle it as quickly as possible.
Quoted 70 lines · Ferruh Yigit
Quoted 48 lines · Aman Deep
Quoted 46 lines · Thomas Monjalon
02/08/2021 03:51, Min Hu (Connor):
Quoted 45 lines · Min Hu (Connor)
Quoted 44 lines · Thomas Monjalon
Quoted 35 lines
26/07/2021 02:26, Min Hu (Connor):
Quoted 26 lines · Thomas Monjalon
Quoted 25 lines
10/07/2021 05:50, Min Hu (Connor):
Quoted 24 lines
This patch adds multi-process support for testpmd. For example the following commands run two testpmd processes:
* the primary process:
./dpdk-testpmd --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0* the secondary process:
./dpdk-testpmd --proc-type=auto -l 2-3 -- -i \ --rxq=4 --txq=4 --num-procs=2 --proc-id=1Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>Signed-off-by Andrew Rybchenko <[email protected]>Acked-by Xiaoyun Li <[email protected]>Acked-by Ajit Khaparde <[email protected]>Reviewed-by Ferruh Yigit <[email protected]>---
V16:
* revert unrelated changes.
* add some restrictions in doc.Quoted 8 lines
Quoted 4 lines
I didn't see clear agreement to integrate this feature in DPDK 21.08.
BTW, the testpmd maintainer was not Cc'ed.
Hi, Thomas,
This patch has two "acked" and one "reviewed-by".
And Xiaoyun Li is the maintainer, I has already Cc'ed to him/her, please check it out, thanks.Quoted 5 lines
Andrew's comment from v15:
"
IMHO, as I state above, current state is insufficient to consider is a start point to be applied.
"Hi, Andrew, Thomos,
What is the least thing should be done for this patch to
meet the state sufficient to be applied?Quoted 9 lines
Quoted 5 lines
Quoted 2 lines
Hi, all,
How about this patch, could it be applied?I was not confident enough to apply it in 21.08. When others will be back, we'll discuss it again for 21.11.
For 21.11 release.
Acked-by Aman Deep Singh <[email protected]>Hi Connor,
Can you please rebase it to latest dpdk (v21.11-rc0) so we can proceed with it?
I am for getting the patch as early as possible in the release cycle, specially before it is getting busy for -rc1.
Thanks,
ferruh
.
PATCH v17 1/1 app/testpmd: support multi-process
In reply to Min Hu (Connor)This patch adds multi-process support for testpmd. For example the following commands run two testpmd processes:
* the primary process:
./dpdk-testpmd --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0
* the secondary process:
./dpdk-testpmd --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1
---
v17:
rebase 21.11
V16:
* revert unrelated changes.
* add some restrictions in doc.
v15:
* Fixed release notes.
* Cleanup documentation.
v14:
* Fixed comments by Andrew Rybchenko.
v13:
* Modified the doc syntax.
v12:
* Updated doc info.
v11:
* Fixed some minor syntax.
v10:
* Hid process type checks behind new functions.
* Added comments.
v9:
* Updated release notes and rst doc.
* Deleted deprecated codes.
* move macro and variable.
v8:
* Added warning info about queue numbers and process numbers.
v7:
* Fixed compiling error for unexpected unindent.
v6:
* Add rte flow description for multiple process.
v5:
* Fixed run_app.rst for multiple process description.
* Fix compiling error.
v4:
* Fixed minimum vlaue of Rxq or Txq in doc.
v3:
* Fixed compiling error using gcc10.0.
v2:
* Added document for this patch.
--- app/test-pmd/cmdline.c | 6 +++ app/test-pmd/config.c | 20 ++++++- app/test-pmd/parameters.c | 9 ++++ app/test-pmd/testpmd.c | 95 +++++++++++++++++++++++++++++----- app/test-pmd/testpmd.h | 9 ++++ doc/guides/rel_notes/release_21_11.rst | 4 ++ doc/guides/testpmd_app_ug/run_app.rst | 84 ++++++++++++++++++++++++++++++ 7 files changed, 214 insertions(+), 13 deletions(-)
app/test-pmd/cmdline.c +6 -0
@@ -5491,6 +5491,12 @@ cmd_set_flush_rx_parsed(void *parsed_result, __rte_unused void *data) {struct cmd_set_flush_rx *res = parsed_result; + +if (num_procs >1 && (strcmp (res->mode,"on" ) ==0 )) { +printf ("multi-process doesn't support to flush Rx queues.\n" ); +return ; + } + no_flush_rx = (uint8_t )((strcmp (res->mode,"on" ) ==0 ) ?0 :1 ); }
app/test-pmd/config.c +19 -1
@@ -2998,6 +2998,8 @@ rss_fwd_config_setup(void)queueid_t rxq;queueid_t nb_q;streamid_t sm_id; +int start; +int end; nb_q = nb_rxq;if (nb_q > nb_txq)
@@ -3015,7 +3017,21 @@ rss_fwd_config_setup(void)init_fwd_streams ();setup_fwd_config_of_each_lcore (&cur_fwd_config); - rxp =0 ; rxq =0 ; + +if (proc_id >0 && nb_q % num_procs !=0 ) +printf ("Warning! queue numbers should be multiple of processes, or packet loss will happen.\n" ); + +/** +* In multi-process, All queues are allocated to different +* processes based on num_procs and proc_id. For example: +* if supports 4 queues(nb_q), 2 processes(num_procs), +* the 0~1 queue for primary process. +* the 2~3 queue for secondary process. +*/ + start = proc_id * nb_q / num_procs; + end = start + nb_q / num_procs; + rxp =0 ; + rxq = start;for (sm_id =0 ; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {struct fwd_stream *fs;
@@ -3032,6 +3048,8 @@ rss_fwd_config_setup(void)continue ; rxp =0 ; rxq++; +if (rxq >= end) + rxq = start; } }
app/test-pmd/parameters.c +9 -0
@@ -512,6 +512,9 @@ parse_link_speed(int n)void launch_args_parse (int argc,char ** argv) { +#define PARAM_PROC_ID "proc-id" +#define PARAM_NUM_PROCS "num-procs" +int n, opt;char **argvopt;int opt_idx;
@@ -631,6 +634,8 @@ launch_args_parse(int argc, char** argv) { "rx-mq-mode", 1, 0, 0 }, { "record-core-cycles", 0, 0, 0 }, { "record-burst-stats", 0, 0, 0 }, + { PARAM_NUM_PROCS, 1, 0, 0 }, + { PARAM_PROC_ID, 1, 0, 0 }, { 0, 0, 0, 0 }, };
@@ -1397,6 +1402,10 @@ launch_args_parse(int argc, char** argv) record_core_cycles =1 ;if (!strcmp (lgopts[opt_idx].name,"record-burst-stats" )) record_burst_stats =1 ; +if (!strcmp (lgopts[opt_idx].name,PARAM_NUM_PROCS )) + num_procs =atoi (optarg); +if (!strcmp (lgopts[opt_idx].name,PARAM_PROC_ID )) + proc_id =atoi (optarg);break ;case 'h' :usage (argv[0 ]);
app/test-pmd/testpmd.c +83 -12
167 lines not on this page · raw message
app/test-pmd/testpmd.h +9 -0
@@ -632,6 +632,15 @@ extern enum rte_eth_rx_mq_mode rx_mq_mode;extern struct rte_flow_action_conntrack conntrack_context; +extern int proc_id; +extern unsigned int num_procs; + +static inline bool +is_proc_primary (void ) +{ +return rte_eal_process_type () ==RTE_PROC_PRIMARY ; +} +static inline unsigned int lcore_num (void ) {
doc/guides/rel_notes/release_21_11.rst +4 -0
@@ -51,6 +51,10 @@ New Features * Other libs * Apps, Examples, Tools (if significant) + * **Added support multi-process for testpmd.** + Added command-line options to specify total number of processes and + current process ID. Each process owns subset of Rx and Tx queues. + This section is a comment. Do not overwrite or remove it. Also, make sure to start the actual text at the margin. =======================================================
doc/guides/testpmd_app_ug/run_app.rst +84 -0
87 lines not on this page · raw message
Signature · 1 line
2.7.4
Ferruh Yigit
AppliedQuoted 14 lines · Min Hu (Connor)
This patch adds multi-process support for testpmd. For example the following commands run two testpmd processes:
* the primary process:
./dpdk-testpmd --proc-type=auto -l 0-1 -- -i \ --rxq=4 --txq=4 --num-procs=2 --proc-id=0* the secondary process:
./dpdk-testpmd --proc-type=auto -l 2-3 -- -i \ --rxq=4 --txq=4 --num-procs=2 --proc-id=1
Quoted 7 lines
Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>Signed-off-by Andrew Rybchenko <[email protected]>Acked-by Xiaoyun Li <[email protected]>Acked-by Ajit Khaparde <[email protected]>Reviewed-by Ferruh Yigit <[email protected]>Acked-by Aman Deep Singh <[email protected]>
Applied to dpdk-next-net/main, thanks.
Thanks Connor for the work,
I won't be surprised if we need a few more tweak/fix for the testpmd secondary process support but right now there is no blocker issue, so proceeding with the patch, please continue your support for further required changes.
Quoted 23 lines · Ferruh Yigit
Quoted 14 lines · Min Hu (Connor)
This patch adds multi-process support for testpmd. For example the following commands run two testpmd processes:
* the primary process:
./dpdk-testpmd --proc-type=auto -l 0-1 -- -i \ --rxq=4 --txq=4 --num-procs=2 --proc-id=0* the secondary process:
./dpdk-testpmd --proc-type=auto -l 2-3 -- -i \ --rxq=4 --txq=4 --num-procs=2 --proc-id=1Quoted 7 lines
Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>Signed-off-by Andrew Rybchenko <[email protected]>Acked-by Xiaoyun Li <[email protected]>Acked-by Ajit Khaparde <[email protected]>Reviewed-by Ferruh Yigit <[email protected]>Acked-by Aman Deep Singh <[email protected]>
Quoted 7 lines
Applied to dpdk-next-net/main, thanks.
Thanks Connor for the work,
I won't be surprised if we need a few more tweak/fix for the testpmd secondary process support but right now there is no blocker issue, so proceeding with the patch, please continue your support for further required changes.
Yes, got it, thanks Ferruh.
Quoted 2 lines
.
PATCH v3 1/1 app/testpmd: support multi-process
In reply to Lijun OuFrom: Lijun Ou <[email protected]>
This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0
the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1
--- app/test-pmd/cmdline.c | 12 ++- app/test-pmd/config.c | 9 ++- app/test-pmd/parameters.c | 11 +++ app/test-pmd/testpmd.c | 138 ++++++++++++++++++++++------------ app/test-pmd/testpmd.h | 7 ++ doc/guides/testpmd_app_ug/run_app.rst | 69 +++++++++++++++++ 6 files changed, 196 insertions(+), 50 deletions(-)
app/test-pmd/cmdline.c +10 -2
@@ -71,8 +71,6 @@#include "cmdline_tm.h" #include "bpf_cmd.h" -static struct cmdline *testpmd_cl; -static void cmd_reconfig_device_queue (portid_t id,uint8_t dev,uint8_t queue);/* *** Help command with introduction. *** */
@@ -5351,6 +5349,12 @@ cmd_set_flush_rx_parsed(void *parsed_result, __rte_unused void *data) {struct cmd_set_flush_rx *res = parsed_result; + +if (num_procs >1 && (strcmp (res->mode,"on" ) ==0 )) { +printf ("multi-process doesn't support to flush rx queues.\n" ); +return ; + } + no_flush_rx = (uint8_t )((strcmp (res->mode,"on" ) ==0 ) ?0 :1 ); }
@@ -17227,6 +17231,10 @@ prompt(void)printf ("Cannot set exit function for cmdline\n" );cmdline_interact (testpmd_cl); +if (unlikely (f_quit ==1 )) { +dup2 (testpmd_fd_copy, testpmd_cl->s_in); +close (testpmd_fd_copy); + }if (ret !=0 )cmdline_stdin_exit (testpmd_cl); }
app/test-pmd/config.c +8 -1
@@ -2827,6 +2827,8 @@ rss_fwd_config_setup(void)queueid_t rxq;queueid_t nb_q;streamid_t sm_id; +int start; +int end; nb_q = nb_rxq;if (nb_q > nb_txq)
@@ -2844,7 +2846,10 @@ rss_fwd_config_setup(void)init_fwd_streams ();setup_fwd_config_of_each_lcore (&cur_fwd_config); - rxp =0 ; rxq =0 ; + start = proc_id * nb_q / num_procs; + end = start + nb_q / num_procs; + rxp =0 ; + rxq = start;for (sm_id =0 ; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {struct fwd_stream *fs;
@@ -2861,6 +2866,8 @@ rss_fwd_config_setup(void)continue ; rxp =0 ; rxq++; +if (rxq >= end) + rxq = start; } }
app/test-pmd/parameters.c +11 -0
@@ -45,6 +45,8 @@#include <rte_flow.h> #include "testpmd.h" +#define PARAM_PROC_ID "proc-id" +#define PARAM_NUM_PROCS "num-procs"static void usage (char * progname)
@@ -605,6 +607,8 @@ launch_args_parse(int argc, char** argv) { "rx-mq-mode", 1, 0, 0 }, { "record-core-cycles", 0, 0, 0 }, { "record-burst-stats", 0, 0, 0 }, + { PARAM_NUM_PROCS, 1, 0, 0 }, + { PARAM_PROC_ID, 1, 0, 0 }, { 0, 0, 0, 0 }, };
@@ -1366,6 +1370,13 @@ launch_args_parse(int argc, char** argv) record_core_cycles =1 ;if (!strcmp (lgopts[opt_idx].name,"record-burst-stats" )) record_burst_stats =1 ; + +if (strncmp (lgopts[opt_idx].name, +PARAM_NUM_PROCS ,8 ) ==0 ) + num_procs =atoi (optarg); +if (strncmp (lgopts[opt_idx].name, +PARAM_PROC_ID ,7 ) ==0 ) + proc_id =atoi (optarg);break ;case 'h' :usage (argv[0 ]);
app/test-pmd/testpmd.c +91 -47
253 lines not on this page · raw message
app/test-pmd/testpmd.h +7 -0
@@ -13,6 +13,7 @@#include <rte_gso.h> #include <cmdline.h> #include <sys/queue.h> +#include "cmdline.h" #define RTE_PORT_ALL (~(portid_t)0x0)
@@ -24,6 +25,10 @@#define RTE_PORT_CLOSED (uint16_t)2#define RTE_PORT_HANDLING (uint16_t)3 +uint8_t f_quit; +int testpmd_fd_copy; +struct cmdline *testpmd_cl; +/* * It is used to allocate the memory for hash key. * The hash key size is NIC dependent.
@@ -422,6 +427,8 @@ extern uint64_t noisy_lkup_mem_sz;extern uint64_t noisy_lkup_num_writes;extern uint64_t noisy_lkup_num_reads;extern uint64_t noisy_lkup_num_reads_writes; +extern int proc_id; +extern unsigned int num_procs;extern uint8_t dcb_config;extern uint8_t dcb_test;
doc/guides/testpmd_app_ug/run_app.rst +69 -0
72 lines not on this page · raw message
Signature · 1 line
2.7.4
Hi, ferruh and all,
have you any comments about this patch?
thanks.
Quoted 514 lines · Min Hu (Connor)
From: Lijun Ou <[email protected]>
This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>--- app/test-pmd/cmdline.c | 12 ++- app/test-pmd/config.c | 9 ++- app/test-pmd/parameters.c | 11 +++ app/test-pmd/testpmd.c | 138 ++++++++++++++++++++++------------ app/test-pmd/testpmd.h | 7 ++ doc/guides/testpmd_app_ug/run_app.rst | 69 +++++++++++++++++ 6 files changed, 196 insertions(+), 50 deletions(-)app/test-pmd/cmdline.c +10
-2@@ -71,8 +71,6 @@#include "cmdline_tm.h" #include "bpf_cmd.h" -static struct cmdline *testpmd_cl; -static void cmd_reconfig_device_queue (portid_t id,uint8_t dev,uint8_t queue);/* *** Help command with introduction. *** */ @@ -5351,6 +5349,12 @@ cmd_set_flush_rx_parsed(void *parsed_result, __rte_unused void *data) {struct cmd_set_flush_rx *res = parsed_result; + +if (num_procs >1 && (strcmp (res->mode,"on" ) ==0 )) { +printf ("multi-process doesn't support to flush rx queues.\n" ); +return ; + } + no_flush_rx = (uint8_t )((strcmp (res->mode,"on" ) ==0 ) ?0 :1 ); }@@ -17227,6 +17231,10 @@ prompt(void)printf ("Cannot set exit function for cmdline\n" );cmdline_interact (testpmd_cl); +if (unlikely (f_quit ==1 )) { +dup2 (testpmd_fd_copy, testpmd_cl->s_in); +close (testpmd_fd_copy); + }if (ret !=0 )cmdline_stdin_exit (testpmd_cl); }app/test-pmd/config.c +8
-1@@ -2827,6 +2827,8 @@ rss_fwd_config_setup(void)queueid_t rxq;queueid_t nb_q;streamid_t sm_id; +int start; +int end; nb_q = nb_rxq;if (nb_q > nb_txq)@@ -2844,7 +2846,10 @@ rss_fwd_config_setup(void)init_fwd_streams ();setup_fwd_config_of_each_lcore (&cur_fwd_config); - rxp =0 ; rxq =0 ; + start = proc_id * nb_q / num_procs; + end = start + nb_q / num_procs; + rxp =0 ; + rxq = start;for (sm_id =0 ; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {struct fwd_stream *fs;@@ -2861,6 +2866,8 @@ rss_fwd_config_setup(void)continue ; rxp =0 ; rxq++; +if (rxq >= end) + rxq = start; } }app/test-pmd/parameters.c +11
-0@@ -45,6 +45,8 @@#include <rte_flow.h> #include "testpmd.h" +#define PARAM_PROC_ID "proc-id" +#define PARAM_NUM_PROCS "num-procs"static void usage (char * progname)@@ -605,6 +607,8 @@ launch_args_parse(int argc, char** argv) { "rx-mq-mode", 1, 0, 0 }, { "record-core-cycles", 0, 0, 0 }, { "record-burst-stats", 0, 0, 0 }, + { PARAM_NUM_PROCS, 1, 0, 0 }, + { PARAM_PROC_ID, 1, 0, 0 }, { 0, 0, 0, 0 }, };@@ -1366,6 +1370,13 @@ launch_args_parse(int argc, char** argv) record_core_cycles =1 ;if (!strcmp (lgopts[opt_idx].name,"record-burst-stats" )) record_burst_stats =1 ; + +if (strncmp (lgopts[opt_idx].name, +PARAM_NUM_PROCS ,8 ) ==0 ) + num_procs =atoi (optarg); +if (strncmp (lgopts[opt_idx].name, +PARAM_PROC_ID ,7 ) ==0 ) + proc_id =atoi (optarg);break ;case 'h' :usage (argv[0 ]);app/test-pmd/testpmd.c +91
-47@@ -63,6 +63,8 @@#include "testpmd.h" +int testpmd_fd_copy =500 ;/* the copy of STDIN_FILENO */ +#ifndef MAP_HUGETLB /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ #define HUGE_FLAG (0x40000)@@ -125,6 +127,9 @@ uint8_t port_numa[RTE_MAX_ETHPORTS];*/ uint8_t rxring_numa[RTE_MAX_ETHPORTS ]; +int proc_id; +unsigned int num_procs =1 ; +/* * Store specified sockets on which TX ring to be used by ports * is allocated. @@ -982,16 +987,26 @@ mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,/* wrapper to rte_mempool_create() */ TESTPMD_LOG (INFO ,"preferred mempool ops selected: %s\n" ,rte_mbuf_best_mempool_ops ()); - rte_mp =rte_pktmbuf_pool_create (pool_name, nb_mbuf, - mb_mempool_cache,0 , mbuf_seg_size, socket_id); +if (rte_eal_process_type () ==RTE_PROC_PRIMARY ) + rte_mp =rte_pktmbuf_pool_create (pool_name, + nb_mbuf, mb_mempool_cache,0 , + mbuf_seg_size, socket_id); +else + rte_mp =rte_mempool_lookup (pool_name); +break ; }case MP_ALLOC_ANON : { - rte_mp =rte_mempool_create_empty (pool_name, nb_mbuf, - mb_size, (unsigned int ) mb_mempool_cache, -sizeof (struct rte_pktmbuf_pool_private ), - socket_id, mempool_flags); +if (rte_eal_process_type () ==RTE_PROC_PRIMARY ) + rte_mp =rte_mempool_create_empty (pool_name, + nb_mbuf, mb_size, + (unsigned int )mb_mempool_cache, +sizeof (struct rte_pktmbuf_pool_private ), + socket_id, mempool_flags); +else + rte_mp =rte_mempool_lookup (pool_name); +if (rte_mp ==NULL ) goto err;@@ -1021,9 +1036,13 @@ mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,TESTPMD_LOG (INFO ,"preferred mempool ops selected: %s\n" ,rte_mbuf_best_mempool_ops ()); - rte_mp =rte_pktmbuf_pool_create (pool_name, nb_mbuf, - mb_mempool_cache,0 , mbuf_seg_size, - heap_socket); +if (rte_eal_process_type () ==RTE_PROC_PRIMARY ) + rte_mp =rte_pktmbuf_pool_create (pool_name, + nb_mbuf, mb_mempool_cache,0 , + mbuf_seg_size, heap_socket); +else + rte_mp =rte_mempool_lookup (pool_name); +break ; }case MP_ALLOC_XBUF :@@ -1994,6 +2013,12 @@ flush_fwd_rx_queues(void)uint64_t prev_tsc =0 , diff_tsc, cur_tsc, timer_tsc =0 ;uint64_t timer_period; +if (num_procs >1 ) { +printf ("multi-process not support for flushing fwd rx " +"queues, skip the below lines and return.\n" ); +return ; + } +/* convert to number of cycles */ timer_period =rte_get_timer_hz ();/* 1 second timeout */ @@ -2503,21 +2528,28 @@ start_port(portid_t pid)return -1 ; }/* configure port */ - diag =rte_eth_dev_configure (pi, nb_rxq + nb_hairpinq, +if (rte_eal_process_type () ==RTE_PROC_PRIMARY ) { + diag =rte_eth_dev_configure (pi, + nb_rxq + nb_hairpinq, nb_txq + nb_hairpinq, &(port->dev_conf)); -if (diag !=0 ) { -if (rte_atomic16_cmpset (&(port->port_status), -RTE_PORT_HANDLING ,RTE_PORT_STOPPED ) ==0 ) -printf ("Port %d can not be set back " -"to stopped\n" , pi); -printf ("Fail to configure port %d\n" , pi); -/* try to reconfigure port next time */ - port->need_reconfig =1 ; -return -1 ; +if (diag !=0 ) { +if (rte_atomic16_cmpset ( + &(port->port_status), +RTE_PORT_HANDLING , +RTE_PORT_STOPPED ) ==0 ) +printf ("Port %d can not be set " +"back to stopped\n" , pi); +printf ("Fail to configure port %d\n" , + pi); +/* try to reconfigure port next time */ + port->need_reconfig =1 ; +return -1 ; + } } } -if (port->need_reconfig_queues >0 ) { +if (port->need_reconfig_queues >0 && +rte_eal_process_type () ==RTE_PROC_PRIMARY ) { port->need_reconfig_queues =0 ;/* setup tx queues */ for (qi =0 ; qi < nb_txq; qi++) {@@ -2618,15 +2650,18 @@ start_port(portid_t pid) cnt_pi++;/* start port */ -if (rte_eth_dev_start (pi) <0 ) { -printf ("Fail to start port %d\n" , pi); +if (rte_eal_process_type () ==RTE_PROC_PRIMARY ) { + diag =rte_eth_dev_start (pi); +if (diag <0 ) { +printf ("Fail to start port %d\n" , pi); -/* Fail to setup rx queue, return */ -if (rte_atomic16_cmpset (&(port->port_status), -RTE_PORT_HANDLING ,RTE_PORT_STOPPED ) ==0 ) -printf ("Port %d can not be set back to " +/* Fail to setup rx queue, return */ +if (rte_atomic16_cmpset (&(port->port_status), +RTE_PORT_HANDLING ,RTE_PORT_STOPPED ) ==0 ) +printf ("Port %d can not be set back to " "stopped\n" , pi); -continue ; +continue ; + } } if (rte_atomic16_cmpset(&(port->port_status),@@ -2755,7 +2790,8 @@ stop_port(portid_t pid)if (port->flow_list)port_flow_flush (pi); -if (rte_eth_dev_stop (pi) !=0 ) +if (rte_eal_process_type () ==RTE_PROC_PRIMARY && +rte_eth_dev_stop (pi) !=0 )RTE_LOG (ERR ,EAL ,"rte_eth_dev_stop failed for port %u\n" , pi);@@ -2824,8 +2860,10 @@ close_port(portid_t pid)continue ; } -port_flow_flush (pi); -rte_eth_dev_close (pi); +if (rte_eal_process_type () ==RTE_PROC_PRIMARY ) +port_flow_flush (pi); +if (rte_eal_process_type () ==RTE_PROC_PRIMARY ) +rte_eth_dev_close (pi); }remove_invalid_ports ();@@ -3089,7 +3127,7 @@ pmd_test_exit(void) } }for (i =0 ; i <RTE_DIM (mempools) ; i++) { -if (mempools[i]) +if (rte_eal_process_type () ==RTE_PROC_PRIMARY && mempools[i])rte_mempool_free (mempools[i]); }@@ -3611,6 +3649,10 @@ init_port_dcb_config(portid_t pid,int retval;uint16_t i; +if (num_procs >1 ) { +printf ("The multi-process feature doesn't support dcb.\n" ); +return -ENOTSUP ; + } rte_port = &ports[pid];memset (&port_conf,0 ,sizeof (struct rte_eth_conf ));@@ -3709,13 +3751,6 @@ init_port(void) }static void -force_quit (void ) -{ -pmd_test_exit (); -prompt_exit (); -} - -static void print_stats (void ) {uint8_t i;@@ -3746,12 +3781,16 @@ signal_handler(int signum)if (latencystats_enabled !=0 )rte_latencystats_uninit ();#endif -force_quit ();/* Set flag to indicate the force termination. */ f_quit =1 ; -/* exit with the expected status */ -signal (signum,SIG_DFL ); -kill (getpid (), signum); +if (interactive ==1 ) { +dup2 (testpmd_cl->s_in, testpmd_fd_copy); +close (testpmd_cl->s_in); + }else { +dup2 (0 , testpmd_fd_copy); +close (0 ); + } + } }@@ -3776,10 +3815,6 @@ main(int argc, char** argv)rte_exit (EXIT_FAILURE ,"Cannot init EAL: %s\n" ,rte_strerror (rte_errno)); -if (rte_eal_process_type () ==RTE_PROC_SECONDARY ) -rte_exit (EXIT_FAILURE , -"Secondary process type not supported.\n" ); - ret =register_eth_event_callback ();if (ret !=0 )rte_exit (EXIT_FAILURE ,"Cannot register for ethdev events" );@@ -3875,8 +3910,10 @@ main(int argc, char** argv) } } -if (!no_device_start &&start_port (RTE_PORT_ALL ) !=0 ) +if (!no_device_start &&start_port (RTE_PORT_ALL ) !=0 ) { +pmd_test_exit ();rte_exit (EXIT_FAILURE ,"Start ports failed\n" ); + }/* set all ports to promiscuous mode by default */ RTE_ETH_FOREACH_DEV (port_id) {@@ -3922,6 +3959,8 @@ main(int argc, char** argv) }prompt ();pmd_test_exit (); +if (unlikely (f_quit ==1 )) +prompt_exit (); } else#endif {@@ -3957,6 +3996,11 @@ main(int argc, char** argv)printf ("Press enter to exit\n" ); rc =read (0 , &c,1 );pmd_test_exit (); +if (unlikely (f_quit ==1 )) { +dup2 (testpmd_fd_copy,0 ); +close (testpmd_fd_copy); +prompt_exit (); + }if (rc <0 )return 1 ; }app/test-pmd/testpmd.h +7
-0@@ -13,6 +13,7 @@#include <rte_gso.h> #include <cmdline.h> #include <sys/queue.h> +#include "cmdline.h" #define RTE_PORT_ALL (~(portid_t)0x0)@@ -24,6 +25,10 @@#define RTE_PORT_CLOSED (uint16_t)2#define RTE_PORT_HANDLING (uint16_t)3 +uint8_t f_quit; +int testpmd_fd_copy; +struct cmdline *testpmd_cl; +/* * It is used to allocate the memory for hash key. * The hash key size is NIC dependent. @@ -422,6 +427,8 @@ extern uint64_t noisy_lkup_mem_sz;extern uint64_t noisy_lkup_num_writes;extern uint64_t noisy_lkup_num_reads;extern uint64_t noisy_lkup_num_reads_writes; +extern int proc_id; +extern unsigned int num_procs;extern uint8_t dcb_config;extern uint8_t dcb_test;doc/guides/testpmd_app_ug/run_app.rst +69
-0@@ -536,3 +536,72 @@ The command line options are: bit 1 - two hairpin ports paired bit 0 - two hairpin ports loop The default value is 0. Hairpin will use single port mode and implicit Tx flow mode. + + +Testpmd Support Multi Process Command-line Options +-------------------------------------------------- + +The following are the command-line options for the testpmd applications(support multi process). +They must be separated from the EAL options, shown in the previous section, with a ``--`` separator: + +.. code-block:: console + + primary process: + sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i --rxq=4 --txq=4 --num-procs=2 --proc-id=0 + + secondary process: + sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i --rxq=4 --txq=4 --num-procs=2 --proc-id=1 + +The command line options are: + +* ``-a, --allow`` + + Add a device to the allow list. ``xxx`` means device used which should be the same in primary process + and secondary process. + +* ``--proc-type`` + Specify a given process instance as the primary or secondary DPDK instance. ``auto`` set here is ok. + +* ``-l CORELIST`` + List of cores to run on. the corelist should be different in primary process and secondary process. + +* ``--rxq=N`` + + Set the number of RX queues per port to N, where 1 <= N <= 65535. + The default value is 1. N is the sum of queues used by primary and secondary process. + +* ``--txq=N`` + + Set the number of TX queues per port to N, where 1 <= N <= 65535. + The default value is 1. N is the sum of queues used by primary and secondary process. + +* ``--num-procs=N`` + + The number of processes which will be used. + +* ``--proc-id=id`` + + The id of the current process (id < num-procs). id should be different in primary process and secondary process. + +Calculation rule for queue: +All queues are allocated to different processes based on proc_num and proc_id. +Calculation rule for the Testpmd to allocate queues to each process: +start(queue start id) = proc_id * nb_q / num_procs; +end(queue end id) = start + nb_q / num_procs; + +For example, if supports 4 txq and rxq +the 0~1 for primary process +the 2~3 for secondary process + +Most dev ops is supported in primary and secondary process. While secondary process is not permitted +to allocate or release shared memory, so some ops are not supported as follows: +``dev_start`` +``dev_stop`` +``rx_queue_setup`` +``tx_queue_setup`` +``rx_queue_release`` +``tx_queue_release`` + +RTE_FLOW supported, it applies only on its own process on SW side, but all on HW size. +stats supported, stats will not change when one quit and start, As they share the same buffer to store the stats. +RSS supported, Primary process and secondary process has separate queues to use, RSS will work in their own queues whether primary and secondary process.
PATCH v4 1/1 app/testpmd: support multi-process
In reply to Lijun OuFrom: Lijun Ou <[email protected]>
This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0
the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1
---
v4:
* Fixed minimum vlaue of Rxq or Txq in doc.
v3:
* Fixed compiling error using gcc10.0.
v2:
* Added document for this patch.
--- app/test-pmd/cmdline.c | 12 ++- app/test-pmd/config.c | 9 ++- app/test-pmd/parameters.c | 11 +++ app/test-pmd/testpmd.c | 138 ++++++++++++++++++++++------------ app/test-pmd/testpmd.h | 7 ++ doc/guides/testpmd_app_ug/run_app.rst | 69 +++++++++++++++++ 6 files changed, 196 insertions(+), 50 deletions(-)
app/test-pmd/cmdline.c +10 -2
@@ -71,8 +71,6 @@#include "cmdline_tm.h" #include "bpf_cmd.h" -static struct cmdline *testpmd_cl; -static void cmd_reconfig_device_queue (portid_t id,uint8_t dev,uint8_t queue);/* *** Help command with introduction. *** */
@@ -5351,6 +5349,12 @@ cmd_set_flush_rx_parsed(void *parsed_result, __rte_unused void *data) {struct cmd_set_flush_rx *res = parsed_result; + +if (num_procs >1 && (strcmp (res->mode,"on" ) ==0 )) { +printf ("multi-process doesn't support to flush rx queues.\n" ); +return ; + } + no_flush_rx = (uint8_t )((strcmp (res->mode,"on" ) ==0 ) ?0 :1 ); }
@@ -17227,6 +17231,10 @@ prompt(void)printf ("Cannot set exit function for cmdline\n" );cmdline_interact (testpmd_cl); +if (unlikely (f_quit ==1 )) { +dup2 (testpmd_fd_copy, testpmd_cl->s_in); +close (testpmd_fd_copy); + }if (ret !=0 )cmdline_stdin_exit (testpmd_cl); }
app/test-pmd/config.c +8 -1
@@ -2827,6 +2827,8 @@ rss_fwd_config_setup(void)queueid_t rxq;queueid_t nb_q;streamid_t sm_id; +int start; +int end; nb_q = nb_rxq;if (nb_q > nb_txq)
@@ -2844,7 +2846,10 @@ rss_fwd_config_setup(void)init_fwd_streams ();setup_fwd_config_of_each_lcore (&cur_fwd_config); - rxp =0 ; rxq =0 ; + start = proc_id * nb_q / num_procs; + end = start + nb_q / num_procs; + rxp =0 ; + rxq = start;for (sm_id =0 ; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {struct fwd_stream *fs;
@@ -2861,6 +2866,8 @@ rss_fwd_config_setup(void)continue ; rxp =0 ; rxq++; +if (rxq >= end) + rxq = start; } }
app/test-pmd/parameters.c +11 -0
@@ -45,6 +45,8 @@#include <rte_flow.h> #include "testpmd.h" +#define PARAM_PROC_ID "proc-id" +#define PARAM_NUM_PROCS "num-procs"static void usage (char * progname)
@@ -605,6 +607,8 @@ launch_args_parse(int argc, char** argv) { "rx-mq-mode", 1, 0, 0 }, { "record-core-cycles", 0, 0, 0 }, { "record-burst-stats", 0, 0, 0 }, + { PARAM_NUM_PROCS, 1, 0, 0 }, + { PARAM_PROC_ID, 1, 0, 0 }, { 0, 0, 0, 0 }, };
@@ -1366,6 +1370,13 @@ launch_args_parse(int argc, char** argv) record_core_cycles =1 ;if (!strcmp (lgopts[opt_idx].name,"record-burst-stats" )) record_burst_stats =1 ; + +if (strncmp (lgopts[opt_idx].name, +PARAM_NUM_PROCS ,8 ) ==0 ) + num_procs =atoi (optarg); +if (strncmp (lgopts[opt_idx].name, +PARAM_PROC_ID ,7 ) ==0 ) + proc_id =atoi (optarg);break ;case 'h' :usage (argv[0 ]);
app/test-pmd/testpmd.c +91 -47
253 lines not on this page · raw message
app/test-pmd/testpmd.h +7 -0
@@ -13,6 +13,7 @@#include <rte_gso.h> #include <cmdline.h> #include <sys/queue.h> +#include "cmdline.h" #define RTE_PORT_ALL (~(portid_t)0x0)
@@ -24,6 +25,10 @@#define RTE_PORT_CLOSED (uint16_t)2#define RTE_PORT_HANDLING (uint16_t)3 +uint8_t f_quit; +int testpmd_fd_copy; +struct cmdline *testpmd_cl; +/* * It is used to allocate the memory for hash key. * The hash key size is NIC dependent.
@@ -422,6 +427,8 @@ extern uint64_t noisy_lkup_mem_sz;extern uint64_t noisy_lkup_num_writes;extern uint64_t noisy_lkup_num_reads;extern uint64_t noisy_lkup_num_reads_writes; +extern int proc_id; +extern unsigned int num_procs;extern uint8_t dcb_config;extern uint8_t dcb_test;
doc/guides/testpmd_app_ug/run_app.rst +69 -0
72 lines not on this page · raw message
Signature · 1 line
2.7.4
Quoted 18 lines · Min Hu (Connor)
From: Lijun Ou <[email protected]>
This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>---
v4:
* Fixed minimum vlaue of Rxq or Txq in doc.
Quoted 5 lines
v3:
* Fixed compiling error using gcc10.0.v2:
* Added document for this patch.
+Xiaoyun, testpmd maintainer.
Hi
Quoted 39 lines
-----Original Message-----
From: dev <[email protected]> On Behalf Of Min Hu (Connor)
Sent: Monday, March 22, 2021 15:07
To: [email protected]
Cc: Yigit, Ferruh <[email protected]>; [email protected]
Subject: [dpdk-dev] [PATCH v4] app/testpmd: support multi-processFrom: Lijun Ou <[email protected]>
This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>---
v4:
* Fixed minimum vlaue of Rxq or Txq in doc.v3:
* Fixed compiling error using gcc10.0.v2:
* Added document for this patch.--- app/test-pmd/cmdline.c | 12 ++- app/test-pmd/config.c | 9 ++- app/test-pmd/parameters.c | 11 +++ app/test-pmd/testpmd.c | 138 ++++++++++++++++++++++------------ app/test-pmd/testpmd.h | 7 ++ doc/guides/testpmd_app_ug/run_app.rst | 69 +++++++++++++++++ 6 files changed, 196 insertions(+), 50 deletions(-)
<snip>
app/test-pmd/testpmd.c mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, · line 1048
Quoted 10 lines
app/test-pmd/testpmd.c +7
-0+if (rte_eal_process_type () ==RTE_PROC_PRIMARY ) + rte_mp =rte_pktmbuf_pool_create (pool_name, + nb_mbuf, mb_mempool_cache,0 , + mbuf_seg_size, heap_socket); +else + rte_mp =rte_mempool_lookup (pool_name); +break ; }case MP_ALLOC_XBUF :
What about this one when users use external bufs? Why not addressing secondary process here?
If it works for all cases, you should add a condition at the start of this function, if it's secondary, goto err to check mp and return.
Quoted 7 lines
app/test-pmd/testpmd.c +3
-0@@ -1994,6 +2013,12 @@ flush_fwd_rx_queues(void)uint64_t prev_tsc =0 , diff_tsc, cur_tsc, timer_tsc =0 ;uint64_t timer_period; +if (num_procs >1 ) { +printf ("multi-process not support for flushing fwd rx " +"queues, skip the below lines and return.\n" );
<snip>
app/test-pmd/testpmd.h line 31
Quoted 4 lines
app/test-pmd/testpmd.h +4
-0+uint8_t f_quit; +int testpmd_fd_copy; +struct cmdline *testpmd_cl; +
Please address the compilation failure on patchwork related to these variables (multiple definitions).
Quoted 10 lines
+.. code-block:: console
doc/guides/testpmd_app_ug/run_app.rst +9
-0+ + primary process: + sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i --rxq=4 +--txq=4 --num-procs=2 --proc-id=0 + + secondary process: + sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i --rxq=4 +--txq=4 --num-procs=2 --proc-id=1 +
<snip>
Quoted 6 lines
doc/guides/testpmd_app_ug/run_app.rst +4
-0+* ``--rxq=N`` + + Set the number of RX queues per port to N, where 1 <= N <= 65535. + The default value is 1. N is the sum of queues used by primary and secondaryprocess.
+
Did you upstream wrong patch?
You said you would address the queue number issue Ajit Khaparde mentioned but you didn't in this patch. The number of queues should be a multiple of the number of processes?
Quoted 6 lines
doc/guides/testpmd_app_ug/run_app.rst +4
-0+* ``--txq=N`` + + Set the number of TX queues per port to N, where 1 <= N <= 65535. + The default value is 1. N is the sum of queues used by primary and secondaryprocess.
+
Same as above.
Quoted 1 line
+* ``--num-procs=N``
<snip>
Quoted 9 lines
(quoted diff) +2
-0+Most dev ops is supported in primary and secondary process. While +secondary process is not permitted to allocate or release shared memory, sosome ops are not supported as follows:
+``dev_start``
+``dev_stop``
+``rx_queue_setup``
+``tx_queue_setup``
+``rx_queue_release``
+``tx_queue_release``
What about some config commands?
Such as "clear port stats all". Should this be allowed by secondary?
And like "port config all rxq". If primary hasn't started ports, should the secondary allowed to change traffic related stuff (offloads, rx/txd, rx/txq and so on)?
Quoted 3 lines
(quoted diff) +2
-0+ +RTE_FLOW supported, it applies only on its own process on SW side, but all onHW size.
About rte flow, what do you mean apply only on its own process on SW side?
If I set number-procs=2, rxq=4
Then on secondary process, I set a flow which directs 192.168.0.1 traffic to queue 0. It seems it will directs this kind of traffic to primary process. But I can't see this rule from primary process side. Is this behavior right for multiple process?
Quoted 6 lines
+stats supported, stats will not change when one quit and start, As they share the same buffer to store the stats.
+RSS supported, Primary process and secondary process has separate queues to use, RSS will work in their own queues whether primary and secondary process. --
2.7.4
Quoted 42 lines · Xiaoyun
Hi
Quoted 39 lines
-----Original Message-----
From: dev <[email protected]> On Behalf Of Min Hu (Connor)
Sent: Monday, March 22, 2021 15:07
To: [email protected]
Cc: Yigit, Ferruh <[email protected]>; [email protected]
Subject: [dpdk-dev] [PATCH v4] app/testpmd: support multi-processFrom: Lijun Ou <[email protected]>
This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>---
v4:
* Fixed minimum vlaue of Rxq or Txq in doc.v3:
* Fixed compiling error using gcc10.0.v2:
* Added document for this patch.--- app/test-pmd/cmdline.c | 12 ++- app/test-pmd/config.c | 9 ++- app/test-pmd/parameters.c | 11 +++ app/test-pmd/testpmd.c | 138 ++++++++++++++++++++++------------ app/test-pmd/testpmd.h | 7 ++ doc/guides/testpmd_app_ug/run_app.rst | 69 +++++++++++++++++ 6 files changed, 196 insertions(+), 50 deletions(-)<snip>
Quoted 14 lines
app/test-pmd/testpmd.c mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, · line 1048
Quoted 10 lines
app/test-pmd/testpmd.c +7
-0+if (rte_eal_process_type () ==RTE_PROC_PRIMARY ) + rte_mp =rte_pktmbuf_pool_create (pool_name, + nb_mbuf, mb_mempool_cache,0 , + mbuf_seg_size, heap_socket); +else + rte_mp =rte_mempool_lookup (pool_name); +break ; }case MP_ALLOC_XBUF :What about this one when users use external bufs? Why not addressing secondary process here?
If it works for all cases, you should add a condition at the start of this function, if it's secondary, goto err to check mp and return.
Yes, your are right, I have fixed it in v5, thanks.
Quoted 9 lines
Quoted 7 lines
app/test-pmd/testpmd.c +3
-0@@ -1994,6 +2013,12 @@ flush_fwd_rx_queues(void)uint64_t prev_tsc =0 , diff_tsc, cur_tsc, timer_tsc =0 ;uint64_t timer_period; +if (num_procs >1 ) { +printf ("multi-process not support for flushing fwd rx " +"queues, skip the below lines and return.\n" );<snip>
Quoted 7 lines
app/test-pmd/testpmd.h line 31
Quoted 4 lines
app/test-pmd/testpmd.h +4
-0+uint8_t f_quit; +int testpmd_fd_copy; +struct cmdline *testpmd_cl; +Please address the compilation failure on patchwork related to these variables (multiple definitions).
Done in v5.
Quoted 11 lines
Quoted 10 lines
+.. code-block:: console
doc/guides/testpmd_app_ug/run_app.rst +9
-0+ + primary process: + sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i --rxq=4 +--txq=4 --num-procs=2 --proc-id=0 + + secondary process: + sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i --rxq=4 +--txq=4 --num-procs=2 --proc-id=1 +<snip>
Quoted 11 lines
Quoted 6 lines
doc/guides/testpmd_app_ug/run_app.rst +4
-0+* ``--rxq=N`` + + Set the number of RX queues per port to N, where 1 <= N <= 65535. + The default value is 1. N is the sum of queues used by primary and secondaryprocess.
+Did you upstream wrong patch?
You said you would address the queue number issue Ajit Khaparde mentioned but you didn't in this patch. The number of queues should be a multiple of the number of processes?
Done in v5.
Quoted 10 lines
Quoted 6 lines
doc/guides/testpmd_app_ug/run_app.rst +4
-0+* ``--txq=N`` + + Set the number of TX queues per port to N, where 1 <= N <= 65535. + The default value is 1. N is the sum of queues used by primary and secondaryprocess.
+Same as above.
Quoted 1 line
+* ``--num-procs=N``
<snip>
Quoted 13 lines
Quoted 9 lines
(quoted diff) +2
-0+Most dev ops is supported in primary and secondary process. While +secondary process is not permitted to allocate or release shared memory, sosome ops are not supported as follows:
+``dev_start``
+``dev_stop``
+``rx_queue_setup``
+``tx_queue_setup``
+``rx_queue_release``
+``tx_queue_release``What about some config commands?
Such as "clear port stats all". Should this be allowed by secondary?
I think so, actually, all the queues is visible to primary and secondary. The only thing we do is to separate queues for different process for io (packets) in Rx/Tx. It is of for secondary "clear port stats all".
Quoted 2 lines
And like "port config all rxq". If primary hasn't started ports, should the secondary allowed to change traffic related stuff (offloads, rx/txd, rx/txq and so on)?
Yes, port config all rxq/txq/rxd/txd/offload is not supported in the secondary process. It has been done in v5.
Quoted 9 lines
Quoted 3 lines
(quoted diff) +2
-0+ +RTE_FLOW supported, it applies only on its own process on SW side, but all onHW size.
About rte flow, what do you mean apply only on its own process on SW side?
If I set number-procs=2, rxq=4
Then on secondary process, I set a flow which directs 192.168.0.1 traffic to queue 0. It seems it will directs this kind of traffic to primary process. But I can't see this rule from primary process side. Is this behavior right for multiple process?
According to doc rte_flow.rst, we maintain flow rules in process level:
primary and secondary has its own flow list(but one flow list in HW).
As previously mentioned, the two can see all the queues, so setting the flow rules for the other is OK. Of course, io(receive or transmit packets) in the queue in others is not
permitted.
Quoted 7 lines
Quoted 6 lines
+stats supported, stats will not change when one quit and start, As they share the same buffer to store the stats.
+RSS supported, Primary process and secondary process has separate queues to use, RSS will work in their own queues whether primary and secondary process. --
2.7.4
Quoted 141 lines · Min Hu (Connor)
Quoted 53 lines · Xiaoyun
Hi
Quoted 39 lines
-----Original Message-----
From: dev <[email protected]> On Behalf Of Min Hu (Connor)
Sent: Monday, March 22, 2021 15:07
To: [email protected]
Cc: Yigit, Ferruh <[email protected]>; [email protected]
Subject: [dpdk-dev] [PATCH v4] app/testpmd: support multi-processFrom: Lijun Ou <[email protected]>
This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>---
v4:
* Fixed minimum vlaue of Rxq or Txq in doc.v3:
* Fixed compiling error using gcc10.0.v2:
* Added document for this patch.--- app/test-pmd/cmdline.c | 12 ++- app/test-pmd/config.c | 9 ++- app/test-pmd/parameters.c | 11 +++ app/test-pmd/testpmd.c | 138 ++++++++++++++++++++++------------ app/test-pmd/testpmd.h | 7 ++ doc/guides/testpmd_app_ug/run_app.rst | 69 +++++++++++++++++ 6 files changed, 196 insertions(+), 50 deletions(-)<snip>
app/test-pmd/testpmd.c mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, · line 1048
Quoted 10 lines
app/test-pmd/testpmd.c +7
-0+if (rte_eal_process_type () ==RTE_PROC_PRIMARY ) + rte_mp =rte_pktmbuf_pool_create (pool_name, + nb_mbuf, mb_mempool_cache,0 , + mbuf_seg_size, heap_socket); +else + rte_mp =rte_mempool_lookup (pool_name); +break ; }case MP_ALLOC_XBUF :Quoted 3 lines
What about this one when users use external bufs? Why not addressing secondary process here?
If it works for all cases, you should add a condition at the start of this function, if it's secondary, goto err to check mp and return.Yes, your are right, I have fixed it in v5, thanks.
Quoted 9 lines
Quoted 7 lines
app/test-pmd/testpmd.c +3
-0@@ -1994,6 +2013,12 @@ flush_fwd_rx_queues(void)uint64_t prev_tsc =0 , diff_tsc, cur_tsc, timer_tsc =0 ;uint64_t timer_period; +if (num_procs >1 ) { +printf ("multi-process not support for flushing fwd rx " +"queues, skip the below lines and return.\n" );<snip>
Quoted 7 lines
app/test-pmd/testpmd.h line 31
Quoted 4 lines
app/test-pmd/testpmd.h +4
-0+uint8_t f_quit; +int testpmd_fd_copy; +struct cmdline *testpmd_cl; +Please address the compilation failure on patchwork related to these variables (multiple definitions).
Done in v5.
Quoted 11 lines
Quoted 10 lines
+.. code-block:: console
doc/guides/testpmd_app_ug/run_app.rst +9
-0+ + primary process: + sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i --rxq=4 +--txq=4 --num-procs=2 --proc-id=0 + + secondary process: + sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i --rxq=4 +--txq=4 --num-procs=2 --proc-id=1 +<snip>
Quoted 11 lines
Quoted 6 lines
doc/guides/testpmd_app_ug/run_app.rst +4
-0+* ``--rxq=N`` + + Set the number of RX queues per port to N, where 1 <= N <= 65535. + The default value is 1. N is the sum of queues used by primary and secondaryprocess.
+Did you upstream wrong patch?
You said you would address the queue number issue Ajit Khaparde mentioned but you didn't in this patch. The number of queues should be a multiple of the number of processes?Done in v5.
Quoted 10 lines
Quoted 6 lines
doc/guides/testpmd_app_ug/run_app.rst +4
-0+* ``--txq=N`` + + Set the number of TX queues per port to N, where 1 <= N <= 65535. + The default value is 1. N is the sum of queues used by primary and secondaryprocess.
+Same as above.
Quoted 1 line
+* ``--num-procs=N``
<snip>
Quoted 13 lines
Quoted 9 lines
(quoted diff) +2
-0+Most dev ops is supported in primary and secondary process. While +secondary process is not permitted to allocate or release shared memory, sosome ops are not supported as follows:
+``dev_start``
+``dev_stop``
+``rx_queue_setup``
+``tx_queue_setup``
+``rx_queue_release``
+``tx_queue_release``What about some config commands?
Such as "clear port stats all". Should this be allowed by secondary?I think so, actually, all the queues is visible to primary and secondary. The only thing we do is to separate queues for different process for io (packets) in Rx/Tx. It is of for secondary "clear port stats all".
Quoted 2 lines
And like "port config all rxq". If primary hasn't started ports, should the secondary allowed to change traffic related stuff (offloads, rx/txd, rx/txq and so on)?
Yes, port config all rxq/txq/rxd/txd/offload is not supported in the secondary process. It has been done in v5.
Quoted 9 lines
Quoted 3 lines
(quoted diff) +2
-0+ +RTE_FLOW supported, it applies only on its own process on SW side, but all onHW size.
About rte flow, what do you mean apply only on its own process on SW side?
If I set number-procs=2, rxq=4
Then on secondary process, I set a flow which directs 192.168.0.1 traffic to queue 0. It seems it will directs this kind of traffic to primary process. But I can't see this rule from primary process side. Is this behavior right for multiple process?
Quoted 6 lines
According to doc
rte_flow.rst, we maintain flow rules in process level:
primary and secondary has its own flow list(but one flow list in HW). As previously mentioned, the two can see all the queues, so setting the flow rules for the other is OK.
Of course, io(receive or transmit packets) in the queue in others is not permitted.
Can you add this behavior as well to the testpmd doc. Further isolation of resources and operations between the primary and secondary processes is possible.
But this is a good start. We can add more if needed.
Quoted 7 lines
Quoted 7 lines
Quoted 6 lines
+stats supported, stats will not change when one quit and start, As they share the same buffer to store the stats.
+RSS supported, Primary process and secondary process has separate queues to use, RSS will work in their own queues whether primary and secondary process. --
2.7.4
Quoted 142 lines · Ajit Khaparde
Quoted 141 lines · Min Hu (Connor)
Quoted 53 lines · Xiaoyun
Hi
Quoted 39 lines
-----Original Message-----
From: dev <[email protected]> On Behalf Of Min Hu (Connor)
Sent: Monday, March 22, 2021 15:07
To: [email protected]
Cc: Yigit, Ferruh <[email protected]>; [email protected]
Subject: [dpdk-dev] [PATCH v4] app/testpmd: support multi-processFrom: Lijun Ou <[email protected]>
This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>---
v4:
* Fixed minimum vlaue of Rxq or Txq in doc.v3:
* Fixed compiling error using gcc10.0.v2:
* Added document for this patch.--- app/test-pmd/cmdline.c | 12 ++- app/test-pmd/config.c | 9 ++- app/test-pmd/parameters.c | 11 +++ app/test-pmd/testpmd.c | 138 ++++++++++++++++++++++------------ app/test-pmd/testpmd.h | 7 ++ doc/guides/testpmd_app_ug/run_app.rst | 69 +++++++++++++++++ 6 files changed, 196 insertions(+), 50 deletions(-)<snip>
app/test-pmd/testpmd.c mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, · line 1048
Quoted 10 lines
app/test-pmd/testpmd.c +7
-0+if (rte_eal_process_type () ==RTE_PROC_PRIMARY ) + rte_mp =rte_pktmbuf_pool_create (pool_name, + nb_mbuf, mb_mempool_cache,0 , + mbuf_seg_size, heap_socket); +else + rte_mp =rte_mempool_lookup (pool_name); +break ; }case MP_ALLOC_XBUF :Quoted 3 lines
What about this one when users use external bufs? Why not addressing secondary process here?
If it works for all cases, you should add a condition at the start of this function, if it's secondary, goto err to check mp and return.Yes, your are right, I have fixed it in v5, thanks.
Quoted 9 lines
Quoted 7 lines
app/test-pmd/testpmd.c +3
-0@@ -1994,6 +2013,12 @@ flush_fwd_rx_queues(void)uint64_t prev_tsc =0 , diff_tsc, cur_tsc, timer_tsc =0 ;uint64_t timer_period; +if (num_procs >1 ) { +printf ("multi-process not support for flushing fwd rx " +"queues, skip the below lines and return.\n" );<snip>
Quoted 7 lines
app/test-pmd/testpmd.h line 31
Quoted 4 lines
app/test-pmd/testpmd.h +4
-0+uint8_t f_quit; +int testpmd_fd_copy; +struct cmdline *testpmd_cl; +Please address the compilation failure on patchwork related to these variables (multiple definitions).
Done in v5.
Quoted 11 lines
Quoted 10 lines
+.. code-block:: console
doc/guides/testpmd_app_ug/run_app.rst +9
-0+ + primary process: + sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i --rxq=4 +--txq=4 --num-procs=2 --proc-id=0 + + secondary process: + sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i --rxq=4 +--txq=4 --num-procs=2 --proc-id=1 +<snip>
Quoted 11 lines
Quoted 6 lines
doc/guides/testpmd_app_ug/run_app.rst +4
-0+* ``--rxq=N`` + + Set the number of RX queues per port to N, where 1 <= N <= 65535. + The default value is 1. N is the sum of queues used by primary and secondaryprocess.
+Did you upstream wrong patch?
You said you would address the queue number issue Ajit Khaparde mentioned but you didn't in this patch. The number of queues should be a multiple of the number of processes?Done in v5.
Quoted 10 lines
Quoted 6 lines
doc/guides/testpmd_app_ug/run_app.rst +4
-0+* ``--txq=N`` + + Set the number of TX queues per port to N, where 1 <= N <= 65535. + The default value is 1. N is the sum of queues used by primary and secondaryprocess.
+Same as above.
Quoted 1 line
+* ``--num-procs=N``
<snip>
Quoted 13 lines
Quoted 9 lines
(quoted diff) +2
-0+Most dev ops is supported in primary and secondary process. While +secondary process is not permitted to allocate or release shared memory, sosome ops are not supported as follows:
+``dev_start``
+``dev_stop``
+``rx_queue_setup``
+``tx_queue_setup``
+``rx_queue_release``
+``tx_queue_release``What about some config commands?
Such as "clear port stats all". Should this be allowed by secondary?I think so, actually, all the queues is visible to primary and secondary. The only thing we do is to separate queues for different process for io (packets) in Rx/Tx. It is of for secondary "clear port stats all".
Quoted 2 lines
And like "port config all rxq". If primary hasn't started ports, should the secondary allowed to change traffic related stuff (offloads, rx/txd, rx/txq and so on)?
Yes, port config all rxq/txq/rxd/txd/offload is not supported in the secondary process. It has been done in v5.
Quoted 9 lines
Quoted 3 lines
(quoted diff) +2
-0+ +RTE_FLOW supported, it applies only on its own process on SW side, but all onHW size.
About rte flow, what do you mean apply only on its own process on SW side?
If I set number-procs=2, rxq=4
Then on secondary process, I set a flow which directs 192.168.0.1 traffic to queue 0. It seems it will directs this kind of traffic to primary process. But I can't see this rule from primary process side. Is this behavior right for multiple process?
Quoted 8 lines
Quoted 6 lines
According to doc
rte_flow.rst, we maintain flow rules in process level:
primary and secondary has its own flow list(but one flow list in HW). As previously mentioned, the two can see all the queues, so setting the flow rules for the other is OK.
Of course, io(receive or transmit packets) in the queue in others is not permitted.Can you add this behavior as well to the testpmd doc.
Hi, Ajit, done in v6, thanks.
Quoted 11 lines
Further isolation of resources and operations between the primary and secondary processes is possible.
But this is a good start. We can add more if needed.Quoted 7 lines
Quoted 7 lines
Quoted 6 lines
+stats supported, stats will not change when one quit and start, As they share the same buffer to store the stats.
+RSS supported, Primary process and secondary process has separate queues to use, RSS will work in their own queues whether primary and secondary process. --
2.7.4
PATCH v5 1/1 app/testpmd: support multi-process
In reply to Lijun OuFrom: Lijun Ou <[email protected]>
This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0
the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1
---
v5:
* Fixed run_app.rst for multiple process description.
* Fix compiling error.
v4:
* Fixed minimum vlaue of Rxq or Txq in doc.
v3:
* Fixed compiling error using gcc10.0.
v2:
* Added document for this patch.
--- app/test-pmd/cmdline.c | 12 +++- app/test-pmd/config.c | 9 ++- app/test-pmd/parameters.c | 11 +++ app/test-pmd/testpmd.c | 127 ++++++++++++++++++++++------------ app/test-pmd/testpmd.h | 7 ++ doc/guides/testpmd_app_ug/run_app.rst | 95 +++++++++++++++++++++++++ 6 files changed, 215 insertions(+), 46 deletions(-)
app/test-pmd/cmdline.c +10 -2
@@ -71,8 +71,6 @@#include "cmdline_tm.h" #include "bpf_cmd.h" -static struct cmdline *testpmd_cl; -static void cmd_reconfig_device_queue (portid_t id,uint8_t dev,uint8_t queue);/* *** Help command with introduction. *** */
@@ -5351,6 +5349,12 @@ cmd_set_flush_rx_parsed(void *parsed_result, __rte_unused void *data) {struct cmd_set_flush_rx *res = parsed_result; + +if (num_procs >1 && (strcmp (res->mode,"on" ) ==0 )) { +printf ("multi-process doesn't support to flush rx queues.\n" ); +return ; + } + no_flush_rx = (uint8_t )((strcmp (res->mode,"on" ) ==0 ) ?0 :1 ); }
@@ -17227,6 +17231,10 @@ prompt(void)printf ("Cannot set exit function for cmdline\n" );cmdline_interact (testpmd_cl); +if (unlikely (f_quit ==1 )) { +dup2 (testpmd_fd_copy, testpmd_cl->s_in); +close (testpmd_fd_copy); + }if (ret !=0 )cmdline_stdin_exit (testpmd_cl); }
app/test-pmd/config.c +8 -1
@@ -2827,6 +2827,8 @@ rss_fwd_config_setup(void)queueid_t rxq;queueid_t nb_q;streamid_t sm_id; +int start; +int end; nb_q = nb_rxq;if (nb_q > nb_txq)
@@ -2844,7 +2846,10 @@ rss_fwd_config_setup(void)init_fwd_streams ();setup_fwd_config_of_each_lcore (&cur_fwd_config); - rxp =0 ; rxq =0 ; + start = proc_id * nb_q / num_procs; + end = start + nb_q / num_procs; + rxp =0 ; + rxq = start;for (sm_id =0 ; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {struct fwd_stream *fs;
@@ -2861,6 +2866,8 @@ rss_fwd_config_setup(void)continue ; rxp =0 ; rxq++; +if (rxq >= end) + rxq = start; } }
app/test-pmd/parameters.c +11 -0
@@ -45,6 +45,8 @@#include <rte_flow.h> #include "testpmd.h" +#define PARAM_PROC_ID "proc-id" +#define PARAM_NUM_PROCS "num-procs"static void usage (char * progname)
@@ -644,6 +646,8 @@ launch_args_parse(int argc, char** argv) { "rx-mq-mode", 1, 0, 0 }, { "record-core-cycles", 0, 0, 0 }, { "record-burst-stats", 0, 0, 0 }, + { PARAM_NUM_PROCS, 1, 0, 0 }, + { PARAM_PROC_ID, 1, 0, 0 }, { 0, 0, 0, 0 }, };
@@ -1410,6 +1414,13 @@ launch_args_parse(int argc, char** argv) record_core_cycles =1 ;if (!strcmp (lgopts[opt_idx].name,"record-burst-stats" )) record_burst_stats =1 ; + +if (strncmp (lgopts[opt_idx].name, +PARAM_NUM_PROCS ,8 ) ==0 ) + num_procs =atoi (optarg); +if (strncmp (lgopts[opt_idx].name, +PARAM_PROC_ID ,7 ) ==0 ) + proc_id =atoi (optarg);break ;case 'h' :usage (argv[0 ]);
app/test-pmd/testpmd.c +84 -43
238 lines not on this page · raw message
app/test-pmd/testpmd.h +7 -0
@@ -13,6 +13,7 @@#include <rte_gso.h> #include <cmdline.h> #include <sys/queue.h> +#include "cmdline.h" #define RTE_PORT_ALL (~(portid_t)0x0)
@@ -24,6 +25,10 @@#define RTE_PORT_CLOSED (uint16_t)2#define RTE_PORT_HANDLING (uint16_t)3 +extern uint8_t f_quit; +extern int testpmd_fd_copy; +extern struct cmdline *testpmd_cl; +/* * It is used to allocate the memory for hash key. * The hash key size is NIC dependent.
@@ -423,6 +428,8 @@ extern uint64_t noisy_lkup_mem_sz;extern uint64_t noisy_lkup_num_writes;extern uint64_t noisy_lkup_num_reads;extern uint64_t noisy_lkup_num_reads_writes; +extern int proc_id; +extern unsigned int num_procs;extern uint8_t dcb_config;extern uint8_t dcb_test;
doc/guides/testpmd_app_ug/run_app.rst +95 -0
98 lines not on this page · raw message
Signature · 1 line
2.7.4
PATCH v6 1/1 app/testpmd: support multi-process
In reply to Lijun OuFrom: Lijun Ou <[email protected]>
This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0
the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1
---
v6:
* Add rte flow description for multiple process.
v5:
* Fixed run_app.rst for multiple process description.
* Fix compiling error.
v4:
* Fixed minimum vlaue of Rxq or Txq in doc.
v3:
* Fixed compiling error using gcc10.0.
v2:
* Added document for this patch.
--- app/test-pmd/cmdline.c | 12 +++- app/test-pmd/config.c | 9 ++- app/test-pmd/parameters.c | 11 +++ app/test-pmd/testpmd.c | 127 ++++++++++++++++++++++------------ app/test-pmd/testpmd.h | 7 ++ doc/guides/testpmd_app_ug/run_app.rst | 99 ++++++++++++++++++++++++++ 6 files changed, 219 insertions(+), 46 deletions(-)
app/test-pmd/cmdline.c +10 -2
@@ -71,8 +71,6 @@#include "cmdline_tm.h" #include "bpf_cmd.h" -static struct cmdline *testpmd_cl; -static void cmd_reconfig_device_queue (portid_t id,uint8_t dev,uint8_t queue);/* *** Help command with introduction. *** */
@@ -5351,6 +5349,12 @@ cmd_set_flush_rx_parsed(void *parsed_result, __rte_unused void *data) {struct cmd_set_flush_rx *res = parsed_result; + +if (num_procs >1 && (strcmp (res->mode,"on" ) ==0 )) { +printf ("multi-process doesn't support to flush rx queues.\n" ); +return ; + } + no_flush_rx = (uint8_t )((strcmp (res->mode,"on" ) ==0 ) ?0 :1 ); }
@@ -17227,6 +17231,10 @@ prompt(void)printf ("Cannot set exit function for cmdline\n" );cmdline_interact (testpmd_cl); +if (unlikely (f_quit ==1 )) { +dup2 (testpmd_fd_copy, testpmd_cl->s_in); +close (testpmd_fd_copy); + }if (ret !=0 )cmdline_stdin_exit (testpmd_cl); }
app/test-pmd/config.c +8 -1
@@ -2827,6 +2827,8 @@ rss_fwd_config_setup(void)queueid_t rxq;queueid_t nb_q;streamid_t sm_id; +int start; +int end; nb_q = nb_rxq;if (nb_q > nb_txq)
@@ -2844,7 +2846,10 @@ rss_fwd_config_setup(void)init_fwd_streams ();setup_fwd_config_of_each_lcore (&cur_fwd_config); - rxp =0 ; rxq =0 ; + start = proc_id * nb_q / num_procs; + end = start + nb_q / num_procs; + rxp =0 ; + rxq = start;for (sm_id =0 ; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {struct fwd_stream *fs;
@@ -2861,6 +2866,8 @@ rss_fwd_config_setup(void)continue ; rxp =0 ; rxq++; +if (rxq >= end) + rxq = start; } }
app/test-pmd/parameters.c +11 -0
@@ -45,6 +45,8 @@#include <rte_flow.h> #include "testpmd.h" +#define PARAM_PROC_ID "proc-id" +#define PARAM_NUM_PROCS "num-procs"static void usage (char * progname)
@@ -644,6 +646,8 @@ launch_args_parse(int argc, char** argv) { "rx-mq-mode", 1, 0, 0 }, { "record-core-cycles", 0, 0, 0 }, { "record-burst-stats", 0, 0, 0 }, + { PARAM_NUM_PROCS, 1, 0, 0 }, + { PARAM_PROC_ID, 1, 0, 0 }, { 0, 0, 0, 0 }, };
@@ -1410,6 +1414,13 @@ launch_args_parse(int argc, char** argv) record_core_cycles =1 ;if (!strcmp (lgopts[opt_idx].name,"record-burst-stats" )) record_burst_stats =1 ; + +if (strncmp (lgopts[opt_idx].name, +PARAM_NUM_PROCS ,8 ) ==0 ) + num_procs =atoi (optarg); +if (strncmp (lgopts[opt_idx].name, +PARAM_PROC_ID ,7 ) ==0 ) + proc_id =atoi (optarg);break ;case 'h' :usage (argv[0 ]);
app/test-pmd/testpmd.c +84 -43
238 lines not on this page · raw message
app/test-pmd/testpmd.h +7 -0
@@ -13,6 +13,7 @@#include <rte_gso.h> #include <cmdline.h> #include <sys/queue.h> +#include "cmdline.h" #define RTE_PORT_ALL (~(portid_t)0x0)
@@ -24,6 +25,10 @@#define RTE_PORT_CLOSED (uint16_t)2#define RTE_PORT_HANDLING (uint16_t)3 +extern uint8_t f_quit; +extern int testpmd_fd_copy; +extern struct cmdline *testpmd_cl; +/* * It is used to allocate the memory for hash key. * The hash key size is NIC dependent.
@@ -423,6 +428,8 @@ extern uint64_t noisy_lkup_mem_sz;extern uint64_t noisy_lkup_num_writes;extern uint64_t noisy_lkup_num_reads;extern uint64_t noisy_lkup_num_reads_writes; +extern int proc_id; +extern unsigned int num_procs;extern uint8_t dcb_config;extern uint8_t dcb_test;
doc/guides/testpmd_app_ug/run_app.rst +99 -0
102 lines not on this page · raw message
Signature · 1 line
2.7.4
PATCH v7 1/1 app/testpmd: support multi-process
In reply to Lijun OuFrom: Lijun Ou <[email protected]>
This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0
the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1
---
v7:
* Fixed compiling error for unexpected unindent.
v6:
* Add rte flow description for multiple process.
v5:
* Fixed run_app.rst for multiple process description.
* Fix compiling error.
v4:
* Fixed minimum vlaue of Rxq or Txq in doc.
v3:
* Fixed compiling error using gcc10.0.
v2:
* Added document for this patch.
--- app/test-pmd/cmdline.c | 12 +++- app/test-pmd/config.c | 9 ++- app/test-pmd/parameters.c | 11 +++ app/test-pmd/testpmd.c | 127 ++++++++++++++++++++++------------ app/test-pmd/testpmd.h | 7 ++ doc/guides/testpmd_app_ug/run_app.rst | 99 ++++++++++++++++++++++++++ 6 files changed, 219 insertions(+), 46 deletions(-)
app/test-pmd/cmdline.c +10 -2
@@ -71,8 +71,6 @@#include "cmdline_tm.h" #include "bpf_cmd.h" -static struct cmdline *testpmd_cl; -static void cmd_reconfig_device_queue (portid_t id,uint8_t dev,uint8_t queue);/* *** Help command with introduction. *** */
@@ -5351,6 +5349,12 @@ cmd_set_flush_rx_parsed(void *parsed_result, __rte_unused void *data) {struct cmd_set_flush_rx *res = parsed_result; + +if (num_procs >1 && (strcmp (res->mode,"on" ) ==0 )) { +printf ("multi-process doesn't support to flush rx queues.\n" ); +return ; + } + no_flush_rx = (uint8_t )((strcmp (res->mode,"on" ) ==0 ) ?0 :1 ); }
@@ -17227,6 +17231,10 @@ prompt(void)printf ("Cannot set exit function for cmdline\n" );cmdline_interact (testpmd_cl); +if (unlikely (f_quit ==1 )) { +dup2 (testpmd_fd_copy, testpmd_cl->s_in); +close (testpmd_fd_copy); + }if (ret !=0 )cmdline_stdin_exit (testpmd_cl); }
app/test-pmd/config.c +8 -1
@@ -2827,6 +2827,8 @@ rss_fwd_config_setup(void)queueid_t rxq;queueid_t nb_q;streamid_t sm_id; +int start; +int end; nb_q = nb_rxq;if (nb_q > nb_txq)
@@ -2844,7 +2846,10 @@ rss_fwd_config_setup(void)init_fwd_streams ();setup_fwd_config_of_each_lcore (&cur_fwd_config); - rxp =0 ; rxq =0 ; + start = proc_id * nb_q / num_procs; + end = start + nb_q / num_procs; + rxp =0 ; + rxq = start;for (sm_id =0 ; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {struct fwd_stream *fs;
@@ -2861,6 +2866,8 @@ rss_fwd_config_setup(void)continue ; rxp =0 ; rxq++; +if (rxq >= end) + rxq = start; } }
app/test-pmd/parameters.c +11 -0
@@ -45,6 +45,8 @@#include <rte_flow.h> #include "testpmd.h" +#define PARAM_PROC_ID "proc-id" +#define PARAM_NUM_PROCS "num-procs"static void usage (char * progname)
@@ -644,6 +646,8 @@ launch_args_parse(int argc, char** argv) { "rx-mq-mode", 1, 0, 0 }, { "record-core-cycles", 0, 0, 0 }, { "record-burst-stats", 0, 0, 0 }, + { PARAM_NUM_PROCS, 1, 0, 0 }, + { PARAM_PROC_ID, 1, 0, 0 }, { 0, 0, 0, 0 }, };
@@ -1410,6 +1414,13 @@ launch_args_parse(int argc, char** argv) record_core_cycles =1 ;if (!strcmp (lgopts[opt_idx].name,"record-burst-stats" )) record_burst_stats =1 ; + +if (strncmp (lgopts[opt_idx].name, +PARAM_NUM_PROCS ,8 ) ==0 ) + num_procs =atoi (optarg); +if (strncmp (lgopts[opt_idx].name, +PARAM_PROC_ID ,7 ) ==0 ) + proc_id =atoi (optarg);break ;case 'h' :usage (argv[0 ]);
app/test-pmd/testpmd.c +84 -43
238 lines not on this page · raw message
app/test-pmd/testpmd.h +7 -0
@@ -13,6 +13,7 @@#include <rte_gso.h> #include <cmdline.h> #include <sys/queue.h> +#include "cmdline.h" #define RTE_PORT_ALL (~(portid_t)0x0)
@@ -24,6 +25,10 @@#define RTE_PORT_CLOSED (uint16_t)2#define RTE_PORT_HANDLING (uint16_t)3 +extern uint8_t f_quit; +extern int testpmd_fd_copy; +extern struct cmdline *testpmd_cl; +/* * It is used to allocate the memory for hash key. * The hash key size is NIC dependent.
@@ -423,6 +428,8 @@ extern uint64_t noisy_lkup_mem_sz;extern uint64_t noisy_lkup_num_writes;extern uint64_t noisy_lkup_num_reads;extern uint64_t noisy_lkup_num_reads_writes; +extern int proc_id; +extern unsigned int num_procs;extern uint8_t dcb_config;extern uint8_t dcb_test;
doc/guides/testpmd_app_ug/run_app.rst +99 -0
102 lines not on this page · raw message
Signature · 1 line
2.7.4
Hi
app/test-pmd/cmdline.c
Quoted 55 lines
-----Original Message-----
From: Min Hu (Connor) <[email protected]>
Sent: Friday, March 26, 2021 16:53
To: [email protected]
Cc: Yigit, Ferruh <[email protected]>; [email protected]; Li, Xiaoyun <[email protected]>
Subject: [PATCH v7] app/testpmd: support multi-processFrom: Lijun Ou <[email protected]>
This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>---
v7:
* Fixed compiling error for unexpected unindent.v6:
* Add rte flow description for multiple process.v5:
* Fixedrun_app.rstfor multiple process description.
* Fix compiling error.v4:
* Fixed minimum vlaue of Rxq or Txq in doc.v3:
* Fixed compiling error using gcc10.0.v2:
* Added document for this patch.--- app/test-pmd/cmdline.c | 12 +++- app/test-pmd/config.c | 9 ++- app/test-pmd/parameters.c | 11 +++ app/test-pmd/testpmd.c | 127 ++++++++++++++++++++++------------ app/test-pmd/testpmd.h | 7 ++ doc/guides/testpmd_app_ug/run_app.rst | 99 ++++++++++++++++++++++++++ 6 files changed, 219 insertions(+), 46 deletions(-)app/test-pmd/cmdline.c index renamed from app/test-pmd/cmdline.c +0
-014110eb..287d7a0 100644
app/test-pmd/cmdline.c +0
-0@@ -71,8 +71,6 @@
Why not just restrict queue numbers to be multiple of processes here in rss_fwd_config_setup()? Or at least print a warning to tell users if it's not multiple, packet loss will happen.
Quoted 21 lines
app/test-pmd/cmdline.c +9
-1setup_fwd_config_of_each_lcore (&cur_fwd_config); - rxp =0 ; rxq =0 ; + start = proc_id * nb_q / num_procs; + end = start + nb_q / num_procs; + rxp =0 ; + rxq = start;for (sm_id =0 ; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {struct fwd_stream *fs;@@ -2861,6 +2866,8 @@ rss_fwd_config_setup(void)continue ; rxp =0 ; rxq++; +if (rxq >= end) + rxq = start; } } +* ``--txq=N`` + + Set the number of Tx queues per port to N. N is the sum of queues used by
Quoted 5 lines
primary
+ and secondary process. As primary process and secondary process should
have separate
+ queues, and each should occupy at least one queue.where N should be no
less than two.
Where N should be the multiple of number of processes.
BRs
Xiaoyun
Quoted 2 lines
+
2.7.4
Quoted 52 lines · Xiaoyun
Hi
Quoted 50 lines
-----Original Message-----
From: Min Hu (Connor) <[email protected]>
Sent: Friday, March 26, 2021 16:53
To: [email protected]
Cc: Yigit, Ferruh <[email protected]>; [email protected]; Li, Xiaoyun <[email protected]>
Subject: [PATCH v7] app/testpmd: support multi-processFrom: Lijun Ou <[email protected]>
This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>---
v7:
* Fixed compiling error for unexpected unindent.v6:
* Add rte flow description for multiple process.v5:
* Fixedrun_app.rstfor multiple process description.
* Fix compiling error.v4:
* Fixed minimum vlaue of Rxq or Txq in doc.v3:
* Fixed compiling error using gcc10.0.v2:
* Added document for this patch.--- app/test-pmd/cmdline.c | 12 +++- app/test-pmd/config.c | 9 ++- app/test-pmd/parameters.c | 11 +++ app/test-pmd/testpmd.c | 127 ++++++++++++++++++++++------------ app/test-pmd/testpmd.h | 7 ++ doc/guides/testpmd_app_ug/run_app.rst | 99 ++++++++++++++++++++++++++ 6 files changed, 219 insertions(+), 46 deletions(-)
Quoted 9 lines
app/test-pmd/cmdline.c
Quoted 5 lines
app/test-pmd/cmdline.c index renamed from app/test-pmd/cmdline.c +0
-014110eb..287d7a0 100644
app/test-pmd/cmdline.c +0
-0@@ -71,8 +71,6 @@Why not just restrict queue numbers to be multiple of processes here in
rss_fwd_config_setup()? Or at least print a warning to tell users if it's not multiple, packet loss will happen.
Hi, fixed in v8, thanks.
Quoted 18 lines
Quoted 18 lines
app/test-pmd/cmdline.c +6
-1setup_fwd_config_of_each_lcore (&cur_fwd_config); - rxp =0 ; rxq =0 ; + start = proc_id * nb_q / num_procs; + end = start + nb_q / num_procs; + rxp =0 ; + rxq = start;for (sm_id =0 ; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {struct fwd_stream *fs;@@ -2861,6 +2866,8 @@ rss_fwd_config_setup(void)continue ; rxp =0 ; rxq++; +if (rxq >= end) + rxq = start; } }
Quoted 11 lines
Quoted 8 lines
app/test-pmd/cmdline.c +3
-0+* ``--txq=N`` + + Set the number of Tx queues per port to N. N is the sum of queues used byprimary
+ and secondary process. As primary process and secondary process should
have separate
+ queues, and each should occupy at least one queue.where N should be no
less than two.Where N should be the multiple of number of processes.
Fixed in v8, thanks.
Quoted 5 lines
BRs
XiaoyunQuoted 2 lines
+
2.7.4
PATCH v8 1/1 app/testpmd: support multi-process
In reply to Lijun OuFrom: Lijun Ou <[email protected]>
This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0
the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1
---
v8:
* Added warning info about queue numbers and process numbers.
v7:
* Fixed compiling error for unexpected unindent.
v6:
* Add rte flow description for multiple process.
v5:
* Fixed run_app.rst for multiple process description.
* Fix compiling error.
v4:
* Fixed minimum vlaue of Rxq or Txq in doc.
v3:
* Fixed compiling error using gcc10.0.
v2:
* Added document for this patch.
--- app/test-pmd/cmdline.c | 12 +++- app/test-pmd/config.c | 14 +++- app/test-pmd/parameters.c | 11 +++ app/test-pmd/testpmd.c | 127 ++++++++++++++++++++++------------ app/test-pmd/testpmd.h | 7 ++ doc/guides/testpmd_app_ug/run_app.rst | 101 +++++++++++++++++++++++++++ 6 files changed, 226 insertions(+), 46 deletions(-)
app/test-pmd/cmdline.c +10 -2
@@ -71,8 +71,6 @@#include "cmdline_tm.h" #include "bpf_cmd.h" -static struct cmdline *testpmd_cl; -static void cmd_reconfig_device_queue (portid_t id,uint8_t dev,uint8_t queue);/* *** Help command with introduction. *** */
@@ -5351,6 +5349,12 @@ cmd_set_flush_rx_parsed(void *parsed_result, __rte_unused void *data) {struct cmd_set_flush_rx *res = parsed_result; + +if (num_procs >1 && (strcmp (res->mode,"on" ) ==0 )) { +printf ("multi-process doesn't support to flush rx queues.\n" ); +return ; + } + no_flush_rx = (uint8_t )((strcmp (res->mode,"on" ) ==0 ) ?0 :1 ); }
@@ -17229,6 +17233,10 @@ prompt(void)printf ("Cannot set exit function for cmdline\n" );cmdline_interact (testpmd_cl); +if (unlikely (f_quit ==1 )) { +dup2 (testpmd_fd_copy, testpmd_cl->s_in); +close (testpmd_fd_copy); + }if (ret !=0 )cmdline_stdin_exit (testpmd_cl); }
app/test-pmd/config.c +13 -1
@@ -2827,6 +2827,8 @@ rss_fwd_config_setup(void)queueid_t rxq;queueid_t nb_q;streamid_t sm_id; +int start; +int end; nb_q = nb_rxq;if (nb_q > nb_txq)
@@ -2844,7 +2846,15 @@ rss_fwd_config_setup(void)init_fwd_streams ();setup_fwd_config_of_each_lcore (&cur_fwd_config); - rxp =0 ; rxq =0 ; + +if (proc_id >0 && nb_q % num_procs) +printf ("Warning! queue numbers should be multiple of " +"processes, or packet loss will happen.\n" ); + + start = proc_id * nb_q / num_procs; + end = start + nb_q / num_procs; + rxp =0 ; + rxq = start;for (sm_id =0 ; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {struct fwd_stream *fs;
@@ -2861,6 +2871,8 @@ rss_fwd_config_setup(void)continue ; rxp =0 ; rxq++; +if (rxq >= end) + rxq = start; } }
app/test-pmd/parameters.c +11 -0
@@ -45,6 +45,8 @@#include <rte_flow.h> #include "testpmd.h" +#define PARAM_PROC_ID "proc-id" +#define PARAM_NUM_PROCS "num-procs"static void usage (char * progname)
@@ -644,6 +646,8 @@ launch_args_parse(int argc, char** argv) { "rx-mq-mode", 1, 0, 0 }, { "record-core-cycles", 0, 0, 0 }, { "record-burst-stats", 0, 0, 0 }, + { PARAM_NUM_PROCS, 1, 0, 0 }, + { PARAM_PROC_ID, 1, 0, 0 }, { 0, 0, 0, 0 }, };
@@ -1410,6 +1414,13 @@ launch_args_parse(int argc, char** argv) record_core_cycles =1 ;if (!strcmp (lgopts[opt_idx].name,"record-burst-stats" )) record_burst_stats =1 ; + +if (strncmp (lgopts[opt_idx].name, +PARAM_NUM_PROCS ,8 ) ==0 ) + num_procs =atoi (optarg); +if (strncmp (lgopts[opt_idx].name, +PARAM_PROC_ID ,7 ) ==0 ) + proc_id =atoi (optarg);break ;case 'h' :usage (argv[0 ]);
app/test-pmd/testpmd.c +84 -43
238 lines not on this page · raw message
app/test-pmd/testpmd.h +7 -0
@@ -13,6 +13,7 @@#include <rte_gso.h> #include <cmdline.h> #include <sys/queue.h> +#include "cmdline.h" #define RTE_PORT_ALL (~(portid_t)0x0)
@@ -24,6 +25,10 @@#define RTE_PORT_CLOSED (uint16_t)2#define RTE_PORT_HANDLING (uint16_t)3 +extern uint8_t f_quit; +extern int testpmd_fd_copy; +extern struct cmdline *testpmd_cl; +/* * It is used to allocate the memory for hash key. * The hash key size is NIC dependent.
@@ -423,6 +428,8 @@ extern uint64_t noisy_lkup_mem_sz;extern uint64_t noisy_lkup_num_writes;extern uint64_t noisy_lkup_num_reads;extern uint64_t noisy_lkup_num_reads_writes; +extern int proc_id; +extern unsigned int num_procs;extern uint8_t dcb_config;extern uint8_t dcb_test;
doc/guides/testpmd_app_ug/run_app.rst +101 -0
104 lines not on this page · raw message
Signature · 1 line
2.7.4
Hi
Quoted 53 lines
-----Original Message-----
From: Min Hu (Connor) <[email protected]>
Sent: Tuesday, March 30, 2021 09:48
To: [email protected]
Cc: Yigit, Ferruh <[email protected]>; [email protected]; Li, Xiaoyun <[email protected]>
Subject: [PATCH v8] app/testpmd: support multi-processFrom: Lijun Ou <[email protected]>
This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>---
v8:
* Added warning info about queue numbers and process numbers.v7:
* Fixed compiling error for unexpected unindent.v6:
* Add rte flow description for multiple process.v5:
* Fixedrun_app.rstfor multiple process description.
* Fix compiling error.v4:
* Fixed minimum vlaue of Rxq or Txq in doc.v3:
* Fixed compiling error using gcc10.0.v2:
* Added document for this patch.--- app/test-pmd/cmdline.c | 12 +++- app/test-pmd/config.c | 14 +++- app/test-pmd/parameters.c | 11 +++ app/test-pmd/testpmd.c | 127 ++++++++++++++++++++++------------ app/test-pmd/testpmd.h | 7 ++ doc/guides/testpmd_app_ug/run_app.rst | 101 +++++++++++++++++++++++++++ 6 files changed, 226 insertions(+), 46 deletions(-)
Quoted 1 line
2.7.4
Many commands shouldn't be allowed but now only have a brief guide in doc. This can be done in the future I think. Overall, it looks good to me.
Quoted 57 lines · Xiaoyun
Hi
Quoted 53 lines
-----Original Message-----
From: Min Hu (Connor) <[email protected]>
Sent: Tuesday, March 30, 2021 09:48
To: [email protected]
Cc: Yigit, Ferruh <[email protected]>; [email protected]; Li, Xiaoyun <[email protected]>
Subject: [PATCH v8] app/testpmd: support multi-processFrom: Lijun Ou <[email protected]>
This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>---
v8:
* Added warning info about queue numbers and process numbers.v7:
* Fixed compiling error for unexpected unindent.v6:
* Add rte flow description for multiple process.v5:
* Fixedrun_app.rstfor multiple process description.
* Fix compiling error.v4:
* Fixed minimum vlaue of Rxq or Txq in doc.v3:
* Fixed compiling error using gcc10.0.v2:
* Added document for this patch.--- app/test-pmd/cmdline.c | 12 +++- app/test-pmd/config.c | 14 +++- app/test-pmd/parameters.c | 11 +++ app/test-pmd/testpmd.c | 127 ++++++++++++++++++++++------------ app/test-pmd/testpmd.h | 7 ++ doc/guides/testpmd_app_ug/run_app.rst | 101 +++++++++++++++++++++++++++ 6 files changed, 226 insertions(+), 46 deletions(-)Quoted 1 line
2.7.4
Quoted 3 lines
Many commands shouldn't be allowed but now only have a brief guide in doc. This can be done in the future I think. Overall, it looks good to me.
Thanks xiaoyun.
You are right, Maybe there are more things to do about supporting multiple process for testpmd. We will perfect it continuously in future.
Quoted 2 lines
Acked-by Xiaoyun Li <[email protected]>
Quoted 9 lines · Min Hu (Connor)
From: Lijun Ou <[email protected]>
This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0
Quoted 6 lines
the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>
Some minor nits below. Otherwise looks fine to me.
Quoted 4 lines
---
v8:
* Added warning info about queue numbers and process numbers.
:::snip::::
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 595
Quoted 6 lines
doc/guides/testpmd_app_ug/run_app.rst +6
-0+* ``--rxq=N`` + + Set the number of Rx queues per port to N. N is the sum of queues used by primary + and secondary process. Primary process and secondary process should have separate + queues, and each should occupy at least one queue. Where N should be the multiple + of number of processes.
of the number of processes.
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 602
Quoted 7 lines
doc/guides/testpmd_app_ug/run_app.rst +7
-0+ +* ``--txq=N`` + + Set the number of Tx queues per port to N. N is the sum of queues used by primary + and secondary process. Primary process and secondary process should have separate + queues, and each should occupy at least one queue. Where N should be the multiple + of number of processes.
of the number of processes.
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 607
Quoted 5 lines
doc/guides/testpmd_app_ug/run_app.rst +5
-0+ +* ``--num-procs=N`` + + The number of processes which will be used. +
:::: snip ::::
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 628
Quoted 6 lines
doc/guides/testpmd_app_ug/run_app.rst +6
-0+The number of rings should be a multiple of the number of processes. If not, +redundant queues will exist after queues are allocated to processes. After RSS is +enabled, packet loss occurs when traffic is sent to all processes at the same time. +Some traffic enters redundant queues and cannot be forwarded. + +Most dev ops is supported in primary and secondary process. While secondary process
Most dev ops are supported in the primary and secondary process. While....
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 637
Quoted 9 lines
doc/guides/testpmd_app_ug/run_app.rst +9
-0+is not permitted to allocate or release shared memory, so some ops are not supported +as follows: +``dev_configure`` +``dev_start`` +``dev_stop`` +``rx_queue_setup`` +``tx_queue_setup`` +``rx_queue_release`` +``tx_queue_release``
:::: snip:::
Quoted 10 lines · Ajit Khaparde
Quoted 9 lines · Min Hu (Connor)
From: Lijun Ou <[email protected]>
This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0
Quoted 9 lines
Quoted 6 lines
the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>Some minor nits below. Otherwise looks fine to me.
Acked-by Ajit Khaparde <[email protected]>
Thanks Ajit.
Hi, Ferruh,
Should I send v9 to fix the grammar bugs in doc which Ajit point
out or fix it in future?
Quoted 48 lines
Quoted 4 lines
---
v8:
* Added warning info about queue numbers and process numbers.:::snip::::
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 595
Quoted 6 lines
doc/guides/testpmd_app_ug/run_app.rst +6
-0+* ``--rxq=N`` + + Set the number of Rx queues per port to N. N is the sum of queues used by primary + and secondary process. Primary process and secondary process should have separate + queues, and each should occupy at least one queue. Where N should be the multiple + of number of processes.of the number of processes.
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 602
Quoted 7 lines
doc/guides/testpmd_app_ug/run_app.rst +7
-0+ +* ``--txq=N`` + + Set the number of Tx queues per port to N. N is the sum of queues used by primary + and secondary process. Primary process and secondary process should have separate + queues, and each should occupy at least one queue. Where N should be the multiple + of number of processes.of the number of processes.
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 607
Quoted 5 lines
doc/guides/testpmd_app_ug/run_app.rst +5
-0+ +* ``--num-procs=N`` + + The number of processes which will be used. +:::: snip ::::
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 628
Quoted 6 lines
doc/guides/testpmd_app_ug/run_app.rst +6
-0+The number of rings should be a multiple of the number of processes. If not, +redundant queues will exist after queues are allocated to processes. After RSS is +enabled, packet loss occurs when traffic is sent to all processes at the same time. +Some traffic enters redundant queues and cannot be forwarded. + +Most dev ops is supported in primary and secondary process. While secondary processMost dev ops are supported in the primary and secondary process. While....
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 637
Quoted 9 lines
doc/guides/testpmd_app_ug/run_app.rst +9
-0+is not permitted to allocate or release shared memory, so some ops are not supported +as follows: +``dev_configure`` +``dev_start`` +``dev_stop`` +``rx_queue_setup`` +``tx_queue_setup`` +``rx_queue_release`` +``tx_queue_release``:::: snip:::
Quoted 19 lines · Min Hu (Connor)
Quoted 16 lines · Ajit Khaparde
Quoted 15 lines · Min Hu (Connor)
From: Lijun Ou <[email protected]>
This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>
Quoted 9 lines
Quoted 3 lines
Some minor nits below. Otherwise looks fine to me.
Acked-by Ajit Khaparde <[email protected]>Thanks Ajit.
Hi, Ferruh,
Should I send v9 to fix the grammar bugs in doc which Ajit point
out or fix it in future?
Hi Connor, if they are only outstanding issues, I can fix them while merging.
Quoted 58 lines
Quoted 58 lines
Quoted 4 lines
---
v8:
* Added warning info about queue numbers and process numbers.:::snip::::
Quoted 9 lines
doc/guides/testpmd_app_ug/run_app.rst +3
-0+* ``--rxq=N`` + + Set the number of Rx queues per port to N. N is the sum of queues usedby primary
+ and secondary process. Primary process and secondary process should have separate
+ queues, and each should occupy at least one queue. Where N should be the multiple
+ of number of processes.of the number of processes.
Quoted 10 lines
doc/guides/testpmd_app_ug/run_app.rst +4
-0+ +* ``--txq=N`` + + Set the number of Tx queues per port to N. N is the sum of queues usedby primary
+ and secondary process. Primary process and secondary process should have separate
+ queues, and each should occupy at least one queue. Where N should be the multiple
+ of number of processes.of the number of processes.
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 607
Quoted 5 lines
doc/guides/testpmd_app_ug/run_app.rst +5
-0+ +* ``--num-procs=N`` + + The number of processes which will be used. +:::: snip ::::
Quoted 9 lines
(quoted diff) +2
-0+The number of rings should be a multiple of the number of processes. If not, +redundant queues will exist after queues are allocated to processes. AfterRSS is
+enabled, packet loss occurs when traffic is sent to all processes at the same time.doc/guides/testpmd_app_ug/run_app.rst +3
-0+Some traffic enters redundant queues and cannot be forwarded. + +Most dev ops is supported in primary and secondary process. While secondaryprocess
Most dev ops are supported in the primary and secondary process. While....
Quoted 10 lines
+is not permitted to allocate or release shared memory, so some ops are not supported
+as follows:
+``dev_configure``
+``dev_start``
+``dev_stop``
+``rx_queue_setup``
+``tx_queue_setup``
+``rx_queue_release``
+``tx_queue_release``:::: snip:::
Quoted 23 lines · Ferruh Yigit
Quoted 22 lines · Min Hu (Connor)
Quoted 19 lines · Ajit Khaparde
Quoted 9 lines · Min Hu (Connor)
From: Lijun Ou <[email protected]>
This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0Quoted 6 lines
the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>Some minor nits below. Otherwise looks fine to me.
Acked-by Ajit Khaparde <[email protected]>
Quoted 9 lines
Quoted 6 lines
Thanks Ajit.
Hi, Ferruh,
Should I send v9 to fix the grammar bugs in doc which Ajit point out or fix it in future?Hi Connor, if they are only outstanding issues, I can fix them while merging.
Thanks Ferruh.
Quoted 61 lines
Quoted 47 lines
Quoted 47 lines
Quoted 4 lines
---
v8:
* Added warning info about queue numbers and process numbers.:::snip::::
Quoted 9 lines
doc/guides/testpmd_app_ug/run_app.rst +3
-0+* ``--rxq=N`` + + Set the number of Rx queues per port to N. N is the sum ofqueues used by primary
+ and secondary process. Primary process and secondary process should have separate
+ queues, and each should occupy at least one queue. Where N should be the multiple
+ of number of processes.of the number of processes.
Quoted 10 lines
doc/guides/testpmd_app_ug/run_app.rst +4
-0+ +* ``--txq=N`` + + Set the number of Tx queues per port to N. N is the sum ofqueues used by primary
+ and secondary process. Primary process and secondary process should have separate
+ queues, and each should occupy at least one queue. Where N should be the multiple
+ of number of processes.of the number of processes.
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 607
Quoted 5 lines
doc/guides/testpmd_app_ug/run_app.rst +5
-0+ +* ``--num-procs=N`` + + The number of processes which will be used. +:::: snip ::::
Quoted 10 lines
+The number of rings should be a multiple of the number of processes. If not,
+redundant queues will exist after queues are allocated to processes. After RSS is
+enabled, packet loss occurs when traffic is sent to all processes at the same time.doc/guides/testpmd_app_ug/run_app.rst +3
-0+Some traffic enters redundant queues and cannot be forwarded. + +Most dev ops is supported in primary and secondary process. Whilesecondary process
Most dev ops are supported in the primary and secondary process. While....
Quoted 12 lines
Quoted 12 lines
Quoted 10 lines
+is not permitted to allocate or release shared memory, so some ops are not supported
+as follows:
+``dev_configure``
+``dev_start``
+``dev_stop``
+``rx_queue_setup``
+``tx_queue_setup``
+``rx_queue_release``
+``tx_queue_release``:::: snip:::
.
Hi, Ferry and all,
This patch has been acked:
Are there any other comments about that patch?
Quoted 98 lines · Min Hu (Connor)
Quoted 23 lines · Ferruh Yigit
Quoted 22 lines · Min Hu (Connor)
Quoted 19 lines · Ajit Khaparde
Quoted 9 lines · Min Hu (Connor)
From: Lijun Ou <[email protected]>
This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0Quoted 6 lines
the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>Some minor nits below. Otherwise looks fine to me.
Acked-by Ajit Khaparde <[email protected]>Quoted 9 lines
Quoted 6 lines
Thanks Ajit.
Hi, Ferruh,
Should I send v9 to fix the grammar bugs in doc which Ajit point out or fix it in future?Hi Connor, if they are only outstanding issues, I can fix them while merging.
Thanks Ferruh.
Quoted 60 lines
Quoted 47 lines
Quoted 47 lines
Quoted 4 lines
---
v8:
* Added warning info about queue numbers and process numbers.:::snip::::
Quoted 9 lines
doc/guides/testpmd_app_ug/run_app.rst +3
-0+* ``--rxq=N`` + + Set the number of Rx queues per port to N. N is the sum ofqueues used by primary
+ and secondary process. Primary process and secondary process should have separate
+ queues, and each should occupy at least one queue. Where N should be the multiple
+ of number of processes.of the number of processes.
Quoted 10 lines
doc/guides/testpmd_app_ug/run_app.rst +4
-0+ +* ``--txq=N`` + + Set the number of Tx queues per port to N. N is the sum ofqueues used by primary
+ and secondary process. Primary process and secondary process should have separate
+ queues, and each should occupy at least one queue. Where N should be the multiple
+ of number of processes.of the number of processes.
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 607
Quoted 5 lines
doc/guides/testpmd_app_ug/run_app.rst +5
-0+ +* ``--num-procs=N`` + + The number of processes which will be used. +:::: snip ::::
Quoted 10 lines
+The number of rings should be a multiple of the number of processes. If not,
+redundant queues will exist after queues are allocated to processes. After RSS is
+enabled, packet loss occurs when traffic is sent to all processes at the same time.doc/guides/testpmd_app_ug/run_app.rst +3
-0+Some traffic enters redundant queues and cannot be forwarded. + +Most dev ops is supported in primary and secondary process. Whilesecondary process
Most dev ops are supported in the primary and secondary process. While....
Quoted 12 lines
Quoted 12 lines
Quoted 10 lines
+is not permitted to allocate or release shared memory, so some ops are not supported
+as follows:
+``dev_configure``
+``dev_start``
+``dev_stop``
+``rx_queue_setup``
+``tx_queue_setup``
+``rx_queue_release``
+``tx_queue_release``:::: snip:::
Quoted 1 line
.
.
Quoted 7 lines · Min Hu (Connor)
Hi, Ferry and all,
This patch has been acked:Acked-by Xiaoyun Li <[email protected]>Acked-by Ajit Khaparde <[email protected]>Are there any other comments about that patch?
Hi Connor,
In today's release status meeting I asked if more parties can test this patch, Lets wait a little more for it to be tested.
Quoted 99 lines · Min Hu (Connor)
Quoted 97 lines · Ferruh Yigit
Quoted 23 lines · Min Hu (Connor)
Quoted 22 lines · Ajit Khaparde
Quoted 19 lines · Min Hu (Connor)
Quoted 9 lines
From: Lijun Ou <[email protected]>
This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0Quoted 6 lines
the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>Some minor nits below. Otherwise looks fine to me.
Acked-by Ajit Khaparde <[email protected]>Quoted 9 lines
Quoted 6 lines
Thanks Ajit.
Hi, Ferruh,
Should I send v9 to fix the grammar bugs in doc which Ajit point out or fix it in future?Hi Connor, if they are only outstanding issues, I can fix them while merging.
Thanks Ferruh.
Quoted 59 lines
Quoted 46 lines
Quoted 46 lines
Quoted 4 lines
---
v8:
* Added warning info about queue numbers and process numbers.:::snip::::
Quoted 9 lines
doc/guides/testpmd_app_ug/run_app.rst +3
-0+* ``--rxq=N`` + + Set the number of Rx queues per port to N. N is the sum of queuesused by primary
+ and secondary process. Primary process and secondary process should have separate
+ queues, and each should occupy at least one queue. Where N should be the multiple
+ of number of processes.of the number of processes.
Quoted 10 lines
doc/guides/testpmd_app_ug/run_app.rst +4
-0+ +* ``--txq=N`` + + Set the number of Tx queues per port to N. N is the sum of queuesused by primary
+ and secondary process. Primary process and secondary process should have separate
+ queues, and each should occupy at least one queue. Where N should be the multiple
+ of number of processes.of the number of processes.
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 607
Quoted 5 lines
doc/guides/testpmd_app_ug/run_app.rst +5
-0+ +* ``--num-procs=N`` + + The number of processes which will be used. +:::: snip ::::
Quoted 9 lines
(quoted diff) +2
-0+The number of rings should be a multiple of the number of processes. If not, +redundant queues will exist after queues are allocated to processes.After RSS is
+enabled, packet loss occurs when traffic is sent to all processes at the same time.doc/guides/testpmd_app_ug/run_app.rst +3
-0+Some traffic enters redundant queues and cannot be forwarded. + +Most dev ops is supported in primary and secondary process. Whilesecondary process
Most dev ops are supported in the primary and secondary process. While....
Quoted 12 lines
Quoted 12 lines
Quoted 10 lines
+is not permitted to allocate or release shared memory, so some ops are not supported
+as follows:
+``dev_configure``
+``dev_start``
+``dev_stop``
+``rx_queue_setup``
+``tx_queue_setup``
+``rx_queue_release``
+``tx_queue_release``:::: snip:::
Quoted 1 line
.
.
Quoted 6 lines · Ferruh Yigit
Quoted 5 lines · Min Hu (Connor)
Hi, Ferry and all,
This patch has been acked:Acked-by Xiaoyun Li <[email protected]>Acked-by Ajit Khaparde <[email protected]>
Quoted 8 lines
Quoted 2 lines
Are there any other comments about that patch?
Hi Connor,
In today's release status meeting I asked if more parties can test this patch, Lets wait a little more for it to be tested.
OK, thanks Ferruh.
Quoted 102 lines
Quoted 98 lines · Min Hu (Connor)
Quoted 96 lines · Ferruh Yigit
Quoted 23 lines · Min Hu (Connor)
Quoted 22 lines · Ajit Khaparde
Quoted 19 lines · Min Hu (Connor)
Quoted 9 lines
From: Lijun Ou <[email protected]>
This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0Quoted 6 lines
the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>Some minor nits below. Otherwise looks fine to me.
Acked-by Ajit Khaparde <[email protected]>Quoted 9 lines
Quoted 6 lines
Thanks Ajit.
Hi, Ferruh,
Should I send v9 to fix the grammar bugs in doc which Ajit point out or fix it in future?Hi Connor, if they are only outstanding issues, I can fix them while merging.
Thanks Ferruh.
Quoted 60 lines
Quoted 59 lines
Quoted 59 lines
Quoted 4 lines
---
v8:
* Added warning info about queue numbers and process numbers.:::snip::::
Quoted 9 lines
(quoted diff) +3
-0+* ``--rxq=N`` + + Set the number of Rx queues per port to N. N is the sum ofqueues used by primary
+ and secondary process. Primary process and secondary process should have separate
+ queues, and each should occupy at least one queue. Where N should be the multiple
+ of number of processes.of the number of processes.
Quoted 10 lines
(quoted diff) +4
-0+ +* ``--txq=N`` + + Set the number of Tx queues per port to N. N is the sum ofqueues used by primary
+ and secondary process. Primary process and secondary process should have separate
+ queues, and each should occupy at least one queue. Where N should be the multiple
+ of number of processes.of the number of processes.
Review
Quoted 5 lines
(quoted diff) +5
-0+ +* ``--num-procs=N`` + + The number of processes which will be used. +:::: snip ::::
Quoted 10 lines
+The number of rings should be a multiple of the number of processes. If not,
+redundant queues will exist after queues are allocated to processes. After RSS is
+enabled, packet loss occurs when traffic is sent to all processes at the same time.(quoted diff) +3
-0+Some traffic enters redundant queues and cannot be forwarded. + +Most dev ops is supported in primary and secondary process.While secondary process
Most dev ops are supported in the primary and secondary process. While....
Quoted 10 lines
+is not permitted to allocate or release shared memory, so some ops are not supported
+as follows:
+``dev_configure``
+``dev_start``
+``dev_stop``
+``rx_queue_setup``
+``tx_queue_setup``
+``rx_queue_release``
+``tx_queue_release``:::: snip:::
Quoted 2 lines
Quoted 2 lines
Quoted 1 line
.
.
.
Quoted 8 lines · Min Hu (Connor)
From: Lijun Ou <[email protected]>
This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0
Quoted 4 lines
the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1
Hi Connor,
Please find a few minor comments below, since they are minor issues please feel free to keep the existing acks in next version.
Quoted 24 lines
Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>---
v8:
* Added warning info about queue numbers and process numbers.v7:
* Fixed compiling error for unexpected unindent.v6:
* Add rte flow description for multiple process.v5:
* Fixedrun_app.rstfor multiple process description.
* Fix compiling error.v4:
* Fixed minimum vlaue of Rxq or Txq in doc.v3:
* Fixed compiling error using gcc10.0.v2:
* Added document for this patch.
Quoted 7 lines
--- app/test-pmd/cmdline.c | 12 +++- app/test-pmd/config.c | 14 +++- app/test-pmd/parameters.c | 11 +++ app/test-pmd/testpmd.c | 127 ++++++++++++++++++++++------------ app/test-pmd/testpmd.h | 7 ++ doc/guides/testpmd_app_ug/run_app.rst | 101 +++++++++++++++++++++++++++
Can you please add a release notes update too?
Quoted 29 lines
6 files changed, 226 insertions(+), 46 deletions(-)app/test-pmd/cmdline.c +6
-2@@ -71,8 +71,6 @@#include "cmdline_tm.h" #include "bpf_cmd.h" -static struct cmdline *testpmd_cl; -static void cmd_reconfig_device_queue (portid_t id,uint8_t dev,uint8_t queue);/* *** Help command with introduction. *** */ @@ -5351,6 +5349,12 @@ cmd_set_flush_rx_parsed(void *parsed_result, __rte_unused void *data) {struct cmd_set_flush_rx *res = parsed_result; + +if (num_procs >1 && (strcmp (res->mode,"on" ) ==0 )) { +printf ("multi-process doesn't support to flush rx queues.\n" ); +return ; + } + no_flush_rx = (uint8_t )((strcmp (res->mode,"on" ) ==0 ) ?0 :1 ); }
app/test-pmd/cmdline.c prompt(void) · line 17239
Quoted 7 lines
app/test-pmd/cmdline.c +4
-0@@ -17229,6 +17233,10 @@ prompt(void)printf ("Cannot set exit function for cmdline\n" );cmdline_interact (testpmd_cl); +if (unlikely (f_quit ==1 )) { +dup2 (testpmd_fd_copy, testpmd_cl->s_in); +close (testpmd_fd_copy); + }
Why this is needed? Can you put some comment on the code to explain why this is needed?
Quoted 47 lines
app/test-pmd/cmdline.c +0
-0if (ret !=0 )cmdline_stdin_exit (testpmd_cl); }app/test-pmd/config.c +13
-1@@ -2827,6 +2827,8 @@ rss_fwd_config_setup(void)queueid_t rxq;queueid_t nb_q;streamid_t sm_id; +int start; +int end; nb_q = nb_rxq;if (nb_q > nb_txq)@@ -2844,7 +2846,15 @@ rss_fwd_config_setup(void)init_fwd_streams ();setup_fwd_config_of_each_lcore (&cur_fwd_config); - rxp =0 ; rxq =0 ; + +if (proc_id >0 && nb_q % num_procs) +printf ("Warning! queue numbers should be multiple of " +"processes, or packet loss will happen.\n" ); + + start = proc_id * nb_q / num_procs; + end = start + nb_q / num_procs; + rxp =0 ; + rxq = start;for (sm_id =0 ; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {struct fwd_stream *fs;@@ -2861,6 +2871,8 @@ rss_fwd_config_setup(void)continue ; rxp =0 ; rxq++; +if (rxq >= end) + rxq = start; } }app/test-pmd/parameters.c +0
-0
app/test-pmd/parameters.c line 50
Quoted 6 lines
app/test-pmd/parameters.c +2
-0@@ -45,6 +45,8 @@#include <rte_flow.h> #include "testpmd.h" +#define PARAM_PROC_ID "proc-id" +#define PARAM_NUM_PROCS "num-procs"
Macros just after include is not good, can you please move these just after 'launch_args_parse()' to keep locality.
Quoted 12 lines
app/test-pmd/parameters.c +2
-0static void usage (char * progname)@@ -644,6 +646,8 @@ launch_args_parse(int argc, char** argv) { "rx-mq-mode", 1, 0, 0 }, { "record-core-cycles", 0, 0, 0 }, { "record-burst-stats", 0, 0, 0 }, + { PARAM_NUM_PROCS, 1, 0, 0 }, + { PARAM_PROC_ID, 1, 0, 0 }, { 0, 0, 0, 0 }, };
app/test-pmd/parameters.c launch_args_parse(int argc, char** argv) · line 1419
Quoted 6 lines
app/test-pmd/parameters.c +3
-0@@ -1410,6 +1414,13 @@ launch_args_parse(int argc, char** argv) record_core_cycles =1 ;if (!strcmp (lgopts[opt_idx].name,"record-burst-stats" )) record_burst_stats =1 ; + +if (strncmp (lgopts[opt_idx].name, +PARAM_NUM_PROCS ,8 ) ==0 )
9?
"num-procs"
123456789
Quoted 12 lines
app/test-pmd/parameters.c +4
-0+ num_procs =atoi (optarg); +if (strncmp (lgopts[opt_idx].name, +PARAM_PROC_ID ,7 ) ==0 ) + proc_id =atoi (optarg);break ;case 'h' :usage (argv[0 ]);app/test-pmd/testpmd.c +0
-0
app/test-pmd/testpmd.c line 68
Quoted 6 lines
app/test-pmd/testpmd.c +3
-0@@ -63,6 +63,9 @@#include "testpmd.h" +int testpmd_fd_copy;/* the copy of STDIN_FILENO */ +struct cmdline *testpmd_cl; +
Again the global variables just after include is not good place.
What do you think to keep 'testpmd_cl' in the 'cmdline.c', since the object allocated there and the variable seems match more to that file. 'extern' already added to header, so it can be used here.
'testpmd_fd_copy' can be moved at the bottom of the block that has global variables, and please add more comment on what this variable is for.
app/test-pmd/testpmd.c uint8_t port_numa[RTE_MAX_ETHPORTS]; · line 133
Quoted 10 lines
app/test-pmd/testpmd.c +3
-0#ifndef MAP_HUGETLB /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ #define HUGE_FLAG (0x40000)@@ -125,6 +128,9 @@ uint8_t port_numa[RTE_MAX_ETHPORTS];*/ uint8_t rxring_numa[RTE_MAX_ETHPORTS ]; +int proc_id; +unsigned int num_procs =1 ; +
Similar comments for these global variables.
The testpmd is already complex, it is good to have multi process support for the testpmd but the concern is additional complexity it bring, it may lead very tangled senarious. Please add much more comment on the multi process related additions.
There is a block at the beggining of the this file, that has all global variables with enough comments on them, can we follow same for these new global variables. Group all multiprocess related variables together, appent to the global variables block, and comment all the variables in detail?
Quoted 127 lines
app/test-pmd/testpmd.c +51
-24/* * Store specified sockets on which TX ring to be used by ports * is allocated. @@ -977,6 +983,11 @@ mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, mb_size =sizeof (struct rte_mbuf ) + mbuf_seg_size;mbuf_poolname_build (socket_id, pool_name,sizeof (pool_name), size_idx); +if (rte_eal_process_type () !=RTE_PROC_PRIMARY ) { + rte_mp =rte_mempool_lookup (pool_name); + goto err; + } +TESTPMD_LOG (INFO ,"create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n" , pool_name, nb_mbuf, mbuf_seg_size, socket_id);@@ -1059,9 +1070,14 @@ mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, err:if (rte_mp ==NULL ) { -rte_exit (EXIT_FAILURE , -"Creation of mbuf pool for socket %u failed: %s\n" , - socket_id,rte_strerror (rte_errno)); +if (rte_eal_process_type () ==RTE_PROC_PRIMARY ) +rte_exit (EXIT_FAILURE , +"Creation of mbuf pool for socket %u failed: %s\n" , + socket_id,rte_strerror (rte_errno)); +else +rte_exit (EXIT_FAILURE , +"Get mbuf pool for socket %u failed: %s\n" , + socket_id,rte_strerror (rte_errno)); }else if (verbose_level >0 ) {rte_mempool_dump (stdout, rte_mp); }@@ -2002,6 +2018,12 @@ flush_fwd_rx_queues(void)uint64_t prev_tsc =0 , diff_tsc, cur_tsc, timer_tsc =0 ;uint64_t timer_period; +if (num_procs >1 ) { +printf ("multi-process not support for flushing fwd rx " +"queues, skip the below lines and return.\n" ); +return ; + } +/* convert to number of cycles */ timer_period =rte_get_timer_hz ();/* 1 second timeout */ @@ -2511,21 +2533,28 @@ start_port(portid_t pid)return -1 ; }/* configure port */ - diag =rte_eth_dev_configure (pi, nb_rxq + nb_hairpinq, +if (rte_eal_process_type () ==RTE_PROC_PRIMARY ) { + diag =rte_eth_dev_configure (pi, + nb_rxq + nb_hairpinq, nb_txq + nb_hairpinq, &(port->dev_conf)); -if (diag !=0 ) { -if (rte_atomic16_cmpset (&(port->port_status), -RTE_PORT_HANDLING ,RTE_PORT_STOPPED ) ==0 ) -printf ("Port %d can not be set back " -"to stopped\n" , pi); -printf ("Fail to configure port %d\n" , pi); -/* try to reconfigure port next time */ - port->need_reconfig =1 ; -return -1 ; +if (diag !=0 ) { +if (rte_atomic16_cmpset ( + &(port->port_status), +RTE_PORT_HANDLING , +RTE_PORT_STOPPED ) ==0 ) +printf ("Port %d can not be set " +"back to stopped\n" , pi); +printf ("Fail to configure port %d\n" , + pi); +/* try to reconfigure port next time */ + port->need_reconfig =1 ; +return -1 ; + } } } -if (port->need_reconfig_queues >0 ) { +if (port->need_reconfig_queues >0 && +rte_eal_process_type () ==RTE_PROC_PRIMARY ) { port->need_reconfig_queues =0 ;/* setup tx queues */ for (qi =0 ; qi < nb_txq; qi++) {@@ -2626,17 +2655,20 @@ start_port(portid_t pid) cnt_pi++;/* start port */ - diag =rte_eth_dev_start (pi); -if (diag <0 ) { -printf ("Fail to start port %d: %s\n" , pi, -rte_strerror (-diag)); +if (rte_eal_process_type () ==RTE_PROC_PRIMARY ) { + diag =rte_eth_dev_start (pi); +if (diag <0 ) { +printf ("Fail to start port %d: %s\n" , pi, +rte_strerror (-diag)); -/* Fail to setup rx queue, return */ -if (rte_atomic16_cmpset (&(port->port_status), +/* Fail to setup rx queue, return */ +if (rte_atomic16_cmpset (&(port->port_status),RTE_PORT_HANDLING ,RTE_PORT_STOPPED ) ==0 ) -printf ("Port %d can not be set back to " -"stopped\n" , pi); -continue ; +printf ("Port %d can not be set back to " +"stopped\n" , + pi); +continue ; + } } if (rte_atomic16_cmpset(&(port->port_status),@@ -2765,7 +2797,8 @@ stop_port(portid_t pid)if (port->flow_list)port_flow_flush (pi); -if (rte_eth_dev_stop (pi) !=0 ) +if (rte_eal_process_type () ==RTE_PROC_PRIMARY && +rte_eth_dev_stop (pi) !=0 )RTE_LOG (ERR ,EAL ,"rte_eth_dev_stop failed for port %u\n" , pi);
app/test-pmd/testpmd.c close_port(portid_t pid) · line 2873
Quoted 9 lines
app/test-pmd/testpmd.c +4
-2@@ -2834,8 +2867,10 @@ close_port(portid_t pid)continue ; } -port_flow_flush (pi); -rte_eth_dev_close (pi); +if (rte_eal_process_type () ==RTE_PROC_PRIMARY ) +port_flow_flush (pi); +if (rte_eal_process_type () ==RTE_PROC_PRIMARY ) +rte_eth_dev_close (pi);
Can combine two calls to prevent redundant check.
Quoted 45 lines
app/test-pmd/testpmd.c +5
-10}remove_invalid_ports ();@@ -3099,7 +3134,7 @@ pmd_test_exit(void) } }for (i =0 ; i <RTE_DIM (mempools) ; i++) { -if (mempools[i]) +if (rte_eal_process_type () ==RTE_PROC_PRIMARY && mempools[i])rte_mempool_free (mempools[i]); }@@ -3621,6 +3656,10 @@ init_port_dcb_config(portid_t pid,int retval;uint16_t i; +if (num_procs >1 ) { +printf ("The multi-process feature doesn't support dcb.\n" ); +return -ENOTSUP ; + } rte_port = &ports[pid];memset (&port_conf,0 ,sizeof (struct rte_eth_conf ));@@ -3719,13 +3758,6 @@ init_port(void) }static void -force_quit (void ) -{ -pmd_test_exit (); -prompt_exit (); -} - -static void print_stats (void ) {uint8_t i;@@ -3756,12 +3788,16 @@ signal_handler(int signum)if (latencystats_enabled !=0 )rte_latencystats_uninit ();#endif -force_quit ();/* Set flag to indicate the force termination. */ f_quit =1 ; -/* exit with the expected status */
app/test-pmd/testpmd.c signal_handler(int signum) · line 3800
Quoted 10 lines
app/test-pmd/testpmd.c +8
-2-signal (signum,SIG_DFL ); -kill (getpid (), signum); +if (interactive ==1 ) { +dup2 (testpmd_cl->s_in, testpmd_fd_copy); +close (testpmd_cl->s_in); + }else { +dup2 (0 , testpmd_fd_copy); +close (0 ); + } +
Similarly can you please comment the code why this is needed.
Quoted 27 lines
app/test-pmd/testpmd.c +3
-5} }@@ -3786,10 +3822,6 @@ main(int argc, char** argv)rte_exit (EXIT_FAILURE ,"Cannot init EAL: %s\n" ,rte_strerror (rte_errno)); -if (rte_eal_process_type () ==RTE_PROC_SECONDARY ) -rte_exit (EXIT_FAILURE , -"Secondary process type not supported.\n" ); - ret =register_eth_event_callback ();if (ret !=0 )rte_exit (EXIT_FAILURE ,"Cannot register for ethdev events" );@@ -3885,8 +3917,10 @@ main(int argc, char** argv) } } -if (!no_device_start &&start_port (RTE_PORT_ALL ) !=0 ) +if (!no_device_start &&start_port (RTE_PORT_ALL ) !=0 ) { +pmd_test_exit ();rte_exit (EXIT_FAILURE ,"Start ports failed\n" ); + }/* set all ports to promiscuous mode by default */ RTE_ETH_FOREACH_DEV (port_id) {
app/test-pmd/testpmd.c main(int argc, char** argv) · line 3970
Quoted 5 lines
app/test-pmd/testpmd.c +2
-0@@ -3932,6 +3966,8 @@ main(int argc, char** argv) }prompt ();pmd_test_exit (); +if (unlikely (f_quit ==1 )) +prompt_exit ();
Why additional prompt_exit() is required?
app/test-pmd/testpmd.c main(int argc, char** argv) · line 4010
Quoted 12 lines
app/test-pmd/testpmd.c +5
-0} else#endif {@@ -3967,6 +4003,11 @@ main(int argc, char** argv)printf ("Press enter to exit\n" ); rc =read (0 , &c,1 );pmd_test_exit (); +if (unlikely (f_quit ==1 )) { +dup2 (testpmd_fd_copy,0 ); +close (testpmd_fd_copy); +prompt_exit (); + }
By definition 'f_quit', force quit, is added to support exiting in container environment, since they can't exit with SIGINT/SIGTERM, so main thing with this global variable should be breaking the loop in main function. I wonder why this variable used extensively in this patch, is it used in different purpose here? will quit/Ctrl-C behaviour differ in multi process?
Quoted 16 lines
app/test-pmd/testpmd.c +0
-0if (rc <0 )return 1 ; }app/test-pmd/testpmd.h +1
-0@@ -13,6 +13,7 @@#include <rte_gso.h> #include <cmdline.h> #include <sys/queue.h> +#include "cmdline.h" #define RTE_PORT_ALL (~(portid_t)0x0)
app/test-pmd/testpmd.h line 31
Quoted 7 lines
app/test-pmd/testpmd.h +4
-0@@ -24,6 +25,10 @@#define RTE_PORT_CLOSED (uint16_t)2#define RTE_PORT_HANDLING (uint16_t)3 +extern uint8_t f_quit; +extern int testpmd_fd_copy; +extern struct cmdline *testpmd_cl; +
Again can you please move them in the file, there is already a block in below for extern variables, please place there with sufficient comments.
app/test-pmd/testpmd.h extern uint64_t noisy_lkup_mem_sz; · line 433
Quoted 10 lines
app/test-pmd/testpmd.h +2
-0/* * It is used to allocate the memory for hash key. * The hash key size is NIC dependent. @@ -423,6 +428,8 @@ extern uint64_t noisy_lkup_mem_sz;extern uint64_t noisy_lkup_num_writes;extern uint64_t noisy_lkup_num_reads;extern uint64_t noisy_lkup_num_reads_writes; +extern int proc_id; +extern unsigned int num_procs;
Can you pleaes group multi process related changes together?
Quoted 36 lines
app/test-pmd/testpmd.h +0
-0extern uint8_t dcb_config;extern uint8_t dcb_test;doc/guides/testpmd_app_ug/run_app.rst +26
-0@@ -551,3 +551,104 @@ The command line options are: bit 1 - two hairpin ports paired bit 0 - two hairpin ports loop The default value is 0. Hairpin will use single port mode and implicit Tx flow mode. + + +Testpmd Support Multi Process Command-line Options +-------------------------------------------------- + +The following are the command-line options for the testpmd applications(support +multi process).They must be separated from the EAL options, shown in the previous +section, with a ``--`` separator: + +.. code-block:: console + + primary process: + sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i --rxq=4 --txq=4 \ + --num-procs=2 --proc-id=0 + + secondary process: + sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i --rxq=4 --txq=4 \ + --num-procs=2 --proc-id=1 + +The command line options are: + +* ``-a, --allow`` + + Add a device to the allow list. ``xxx`` means device used which should be the + same in primary process and secondary process. +
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 589
Quoted 10 lines
doc/guides/testpmd_app_ug/run_app.rst +10
-0+* ``--proc-type`` + + Specify a given process instance as the primary or secondary DPDK instance. + ``auto`` set here is OK. + +* ``-l CORELIST`` + + List of cores to run on. the corelist should be different in primary process and + secondary process. +
+1 for documentation, but above are eal parameters, what about not repeting them here but referencing eal documentation, 'doc/guides/linux_gsg/eal_args.include.rst', for it?
Quoted 66 lines
doc/guides/testpmd_app_ug/run_app.rst +65
-0+* ``--rxq=N`` + + Set the number of Rx queues per port to N. N is the sum of queues used by primary + and secondary process. Primary process and secondary process should have separate + queues, and each should occupy at least one queue. Where N should be the multiple + of number of processes. + +* ``--txq=N`` + + Set the number of Tx queues per port to N. N is the sum of queues used by primary + and secondary process. Primary process and secondary process should have separate + queues, and each should occupy at least one queue. Where N should be the multiple + of number of processes. + +* ``--num-procs=N`` + + The number of processes which will be used. + +* ``--proc-id=id`` + + The id of the current process (id < num-procs). id should be different in primary + process and secondary process. + +Calculation rule for queue: +All queues are allocated to different processes based on proc_num and proc_id. +Calculation rule for the Testpmd to allocate queues to each process: +start(queue start id) = proc_id * nb_q / num_procs; +end(queue end id) = start + nb_q / num_procs; + +For example, if supports 4 txq and rxq +the 0~1 for primary process +the 2~3 for secondary process + +The number of rings should be a multiple of the number of processes. If not, +redundant queues will exist after queues are allocated to processes. After RSS is +enabled, packet loss occurs when traffic is sent to all processes at the same time. +Some traffic enters redundant queues and cannot be forwarded. + +Most dev ops is supported in primary and secondary process. While secondary process +is not permitted to allocate or release shared memory, so some ops are not supported +as follows: +``dev_configure`` +``dev_start`` +``dev_stop`` +``rx_queue_setup`` +``tx_queue_setup`` +``rx_queue_release`` +``tx_queue_release`` + +So, any command from testpmd which calls those APIs will not be supported in secondary +process, like: +``port config all rxq|txq|rxd|txd <value>`` +``port config <port_id> rx_offload xxx on/off `` +``port config <port_id> tx_offload xxx on/off`` +etc. + +RTE_FLOW supported, it applies only on its own process on SW side, but all on HW size. +stats supported, stats will not change when one quit and start, As they share the same +buffer to store the stats. Flow rules are maintained in process level: primary and secondary +has its own flow list(but one flow list in HW). The two can see all the queues, so setting +the flow rules for the other is OK. Of course, io(receive or transmit packets) in the queue +from others is not permitted. + +RSS supported, Primary process and secondary process has separate queues to use, RSS +will work in their own queues whether primary and secondary process.
Quoted 9 lines · Ferruh Yigit
Quoted 8 lines · Min Hu (Connor)
From: Lijun Ou <[email protected]>
This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0
Quoted 9 lines
Quoted 4 lines
the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1Hi Connor,
Please find a few minor comments below, since they are minor issues please feel free to keep the existing acks in next version.
Reminder of this patch, waiting for minor updates, it would be good to have it before -rc1, so that the change can be tested thoroughly.
Meanwhile I have concern/question on long term testing of the feature. It is possible to break the multi-process support unintentionally and not recognize it. How can we continually verify the feature?
Are you using DTS?
Does it make sense to add a testcase into dts to test testpmd multi-process support so that it is verified on each release?
Quoted 15 lines · Ferruh Yigit
Quoted 14 lines · Ferruh Yigit
Quoted 12 lines · Min Hu (Connor)
From: Lijun Ou <[email protected]>
This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1
Quoted 7 lines
Quoted 4 lines
Hi Connor,
Please find a few minor comments below, since they are minor issues please feel free to keep the existing acks in next version.
Reminder of this patch, waiting for minor updates, it would be good to have it before -rc1, so that the change can be tested thoroughly. Hi, Ferruh,
v9 has been sent, please check it out, thanks.
Quoted 5 lines
Meanwhile I have concern/question on long term testing of the feature. It is possible to break the multi-process support unintentionally and not recognize it. How can we continually verify the feature?
Are you using DTS?
Does it make sense to add a testcase into dts to test testpmd multi-process support so that it is verified on each release? .
Well, we have our own DTS system which contains testcase of multiple process in testpmd:
1. When we upstream patches to DPDK,the patch should pass all the testcase.
2. Before DPDK version release(in RC1) in every three months, we rebase the new DPDK version, and check all the testcase in our DTS system. If some testcase fails, we will fix it and upstream the patch to DPDK, of course, including multi-process support in testpmd.
PATCH v9 1/1 app/testpmd: support multi-process
In reply to Lijun OuThis patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0
the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1
---
v9:
* Updated release notes and rst doc.
* Deleted deprecated codes.
* move macro and variable.
v8:
* Added warning info about queue numbers and process numbers.
v7:
* Fixed compiling error for unexpected unindent.
v6:
* Add rte flow description for multiple process.
v5:
* Fixed run_app.rst for multiple process description.
* Fix compiling error.
v4:
* Fixed minimum vlaue of Rxq or Txq in doc.
v3:
* Fixed compiling error using gcc10.0.
v2:
* Added document for this patch.
--- app/test-pmd/cmdline.c | 6 ++ app/test-pmd/config.c | 14 ++++- app/test-pmd/parameters.c | 12 ++++ app/test-pmd/testpmd.c | 108 +++++++++++++++++++++++---------- app/test-pmd/testpmd.h | 3 + doc/guides/rel_notes/release_21_05.rst | 1 + doc/guides/testpmd_app_ug/run_app.rst | 86 ++++++++++++++++++++++++++ 7 files changed, 196 insertions(+), 34 deletions(-)
app/test-pmd/cmdline.c +6 -0
@@ -5354,6 +5354,12 @@ cmd_set_flush_rx_parsed(void *parsed_result, __rte_unused void *data) {struct cmd_set_flush_rx *res = parsed_result; + +if (num_procs >1 && (strcmp (res->mode,"on" ) ==0 )) { +printf ("multi-process doesn't support to flush rx queues.\n" ); +return ; + } + no_flush_rx = (uint8_t )((strcmp (res->mode,"on" ) ==0 ) ?0 :1 ); }
app/test-pmd/config.c +13 -1
@@ -2860,6 +2860,8 @@ rss_fwd_config_setup(void)queueid_t rxq;queueid_t nb_q;streamid_t sm_id; +int start; +int end; nb_q = nb_rxq;if (nb_q > nb_txq)
@@ -2877,7 +2879,15 @@ rss_fwd_config_setup(void)init_fwd_streams ();setup_fwd_config_of_each_lcore (&cur_fwd_config); - rxp =0 ; rxq =0 ; + +if (proc_id >0 && nb_q % num_procs) +printf ("Warning! queue numbers should be multiple of " +"processes, or packet loss will happen.\n" ); + + start = proc_id * nb_q / num_procs; + end = start + nb_q / num_procs; + rxp =0 ; + rxq = start;for (sm_id =0 ; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {struct fwd_stream *fs;
@@ -2894,6 +2904,8 @@ rss_fwd_config_setup(void)continue ; rxp =0 ; rxq++; +if (rxq >= end) + rxq = start; } }
app/test-pmd/parameters.c +12 -0
@@ -508,6 +508,9 @@ parse_link_speed(int n)void launch_args_parse (int argc,char ** argv) { +#define PARAM_PROC_ID "proc-id" +#define PARAM_NUM_PROCS "num-procs" +int n, opt;char **argvopt;int opt_idx;
@@ -625,6 +628,8 @@ launch_args_parse(int argc, char** argv) { "rx-mq-mode", 1, 0, 0 }, { "record-core-cycles", 0, 0, 0 }, { "record-burst-stats", 0, 0, 0 }, + { PARAM_NUM_PROCS, 1, 0, 0 }, + { PARAM_PROC_ID, 1, 0, 0 }, { 0, 0, 0, 0 }, };
@@ -1391,6 +1396,13 @@ launch_args_parse(int argc, char** argv) record_core_cycles =1 ;if (!strcmp (lgopts[opt_idx].name,"record-burst-stats" )) record_burst_stats =1 ; + +if (strncmp (lgopts[opt_idx].name, +PARAM_NUM_PROCS ,9 ) ==0 ) + num_procs =atoi (optarg); +if (strncmp (lgopts[opt_idx].name, +PARAM_PROC_ID ,7 ) ==0 ) + proc_id =atoi (optarg);break ;case 'h' :usage (argv[0 ]);
app/test-pmd/testpmd.c +75 -33
193 lines not on this page · raw message
app/test-pmd/testpmd.h +3 -0
@@ -630,6 +630,9 @@ extern struct mplsoudp_decap_conf mplsoudp_decap_conf;extern enum rte_eth_rx_mq_mode rx_mq_mode; +extern int proc_id; +extern unsigned int num_procs; +static inline unsigned int lcore_num (void ) {
doc/guides/rel_notes/release_21_05.rst +1 -0
@@ -185,6 +185,7 @@ New Features ``show port (port_id) rxq (queue_id) desc used count`` * Added command to dump internal representation information of single flow. ``flow dump (port_id) rule (rule_id)`` + * Added support multi-process for testpmd. Removed Items
doc/guides/testpmd_app_ug/run_app.rst +86 -0
89 lines not on this page · raw message
Signature · 1 line
2.7.4
Quoted 10 lines · Min Hu (Connor)
This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1
Hi Connor,
Thanks for the update. +Anatoly as multi-process maintainer, and ethdev maintainers.
Quoted 75 lines
Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>Acked-by Xiaoyun Li <[email protected]>Acked-by Ajit Khaparde <[email protected]>---
v9:
* Updated release notes and rst doc.
* Deleted deprecated codes.
* move macro and variable.v8:
* Added warning info about queue numbers and process numbers.v7:
* Fixed compiling error for unexpected unindent.v6:
* Add rte flow description for multiple process.v5:
* Fixedrun_app.rstfor multiple process description.
* Fix compiling error.v4:
* Fixed minimum vlaue of Rxq or Txq in doc.v3:
* Fixed compiling error using gcc10.0.v2:
* Added document for this patch.--- app/test-pmd/cmdline.c | 6 ++ app/test-pmd/config.c | 14 ++++- app/test-pmd/parameters.c | 12 ++++ app/test-pmd/testpmd.c | 108 +++++++++++++++++++++++---------- app/test-pmd/testpmd.h | 3 + doc/guides/rel_notes/release_21_05.rst | 1 + doc/guides/testpmd_app_ug/run_app.rst | 86 ++++++++++++++++++++++++++ 7 files changed, 196 insertions(+), 34 deletions(-)app/test-pmd/cmdline.c +6
-0@@ -5354,6 +5354,12 @@ cmd_set_flush_rx_parsed(void *parsed_result, __rte_unused void *data) {struct cmd_set_flush_rx *res = parsed_result; + +if (num_procs >1 && (strcmp (res->mode,"on" ) ==0 )) { +printf ("multi-process doesn't support to flush rx queues.\n" ); +return ; + } + no_flush_rx = (uint8_t )((strcmp (res->mode,"on" ) ==0 ) ?0 :1 ); }app/test-pmd/config.c +2
-0@@ -2860,6 +2860,8 @@ rss_fwd_config_setup(void)queueid_t rxq;queueid_t nb_q;streamid_t sm_id; +int start; +int end; nb_q = nb_rxq;if (nb_q > nb_txq)@@ -2877,7 +2879,15 @@ rss_fwd_config_setup(void)init_fwd_streams ();setup_fwd_config_of_each_lcore (&cur_fwd_config);
app/test-pmd/config.c rss_fwd_config_setup(void) · line 2890
Quoted 10 lines
app/test-pmd/config.c +9
-1- rxp =0 ; rxq =0 ; + +if (proc_id >0 && nb_q % num_procs) +printf ("Warning! queue numbers should be multiple of " +"processes, or packet loss will happen.\n" ); + + start = proc_id * nb_q / num_procs; + end = start + nb_q / num_procs; + rxp =0 ; + rxq = start;
Can you put some comment above, on what/why is done, something similar to you already put into the documentation?
Quoted 36 lines
app/test-pmd/config.c +2
-0for (sm_id =0 ; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {struct fwd_stream *fs;@@ -2894,6 +2904,8 @@ rss_fwd_config_setup(void)continue ; rxp =0 ; rxq++; +if (rxq >= end) + rxq = start; } }app/test-pmd/parameters.c +5
-0@@ -508,6 +508,9 @@ parse_link_speed(int n)void launch_args_parse (int argc,char ** argv) { +#define PARAM_PROC_ID "proc-id" +#define PARAM_NUM_PROCS "num-procs" +int n, opt;char **argvopt;int opt_idx;@@ -625,6 +628,8 @@ launch_args_parse(int argc, char** argv) { "rx-mq-mode", 1, 0, 0 }, { "record-core-cycles", 0, 0, 0 }, { "record-burst-stats", 0, 0, 0 }, + { PARAM_NUM_PROCS, 1, 0, 0 }, + { PARAM_PROC_ID, 1, 0, 0 }, { 0, 0, 0, 0 }, };
app/test-pmd/parameters.c launch_args_parse(int argc, char** argv) · line 1399
Quoted 4 lines
app/test-pmd/parameters.c +1
-0@@ -1391,6 +1396,13 @@ launch_args_parse(int argc, char** argv) record_core_cycles =1 ;if (!strcmp (lgopts[opt_idx].name,"record-burst-stats" )) record_burst_stats =1 ; +
To be consistent for rest, this empty line can be removed.
Quoted 14 lines
app/test-pmd/parameters.c +6
-0+if (strncmp (lgopts[opt_idx].name, +PARAM_NUM_PROCS ,9 ) ==0 ) + num_procs =atoi (optarg); +if (strncmp (lgopts[opt_idx].name, +PARAM_PROC_ID ,7 ) ==0 ) + proc_id =atoi (optarg);break ;case 'h' :usage (argv[0 ]);app/test-pmd/testpmd.c +0
-0
app/test-pmd/testpmd.c enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS; · line 523
Quoted 6 lines
app/test-pmd/testpmd.c +3
-0@@ -518,6 +518,16 @@ enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS;*/ uint32_t eth_link_speed; +/* +* Id of the current process. +*/
Can you please add more info, like this being related to the primary/secondary process support and used to configure the queues to be polled.
app/test-pmd/testpmd.c enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS; · line 530
Quoted 7 lines
app/test-pmd/testpmd.c +7
-0+int proc_id; + +/* +* Number of processes. +*/ +unsigned int num_procs =1 ; +
Ditto.
<...>
Quoted 22 lines
app/test-pmd/testpmd.c +6
-10@@ -2511,21 +2537,28 @@ start_port(portid_t pid)return -1 ; }/* configure port */ - diag =rte_eth_dev_configure (pi, nb_rxq + nb_hairpinq, +if (rte_eal_process_type () ==RTE_PROC_PRIMARY ) { + diag =rte_eth_dev_configure (pi, + nb_rxq + nb_hairpinq, nb_txq + nb_hairpinq, &(port->dev_conf)); -if (diag !=0 ) { -if (rte_atomic16_cmpset (&(port->port_status), -RTE_PORT_HANDLING ,RTE_PORT_STOPPED ) ==0 ) -printf ("Port %d can not be set back " -"to stopped\n" , pi); -printf ("Fail to configure port %d\n" , pi); -/* try to reconfigure port next time */ - port->need_reconfig =1 ; -return -1 ; +if (diag !=0 ) { + if (rte_atomic16_cmpset( + &(port->port_status),
app/test-pmd/testpmd.c start_port(portid_t pid) · line 2557
Quoted 10 lines
app/test-pmd/testpmd.c +10
-0+ RTE_PORT_HANDLING, + RTE_PORT_STOPPED) == 0) +printf ("Port %d can not be set " +"back to stopped\n" , pi); +printf ("Fail to configure port %d\n" , + pi); +/* try to reconfigure port next time */ + port->need_reconfig =1 ; +return -1 ; + }
Just an idea,
I am a little worried about the complexity this new process type checks are added to the multiple locations. What do you think hiding process type checks behind new functions, like:eth_dev_configure_mp(...) {
if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
return rte_eth_dev_configure(..)
}
return 0;
}
and replace it with 'rte_eth_dev_configure()':
-diag = rte_eth_dev_configure(...)
+diag = eth_dev_configure_mp(...)
Do you think does this help reducing the complexity added by the multi-process support?
app/test-pmd/testpmd.c start_port(portid_t pid) · line 2561
Quoted 5 lines
app/test-pmd/testpmd.c +2
-1} } -if (port->need_reconfig_queues >0 ) { +if (port->need_reconfig_queues >0 && +rte_eal_process_type () ==RTE_PROC_PRIMARY ) {
According our coding convention, we don't allign with paranthesis but put double tabs.
And what about creating an inline function for primary process check, like is_proc_primary(), that may help.
Quoted 12 lines
app/test-pmd/testpmd.c +1
-4port->need_reconfig_queues =0 ;/* setup tx queues */ for (qi =0 ; qi < nb_txq; qi++) {@@ -2626,17 +2659,20 @@ start_port(portid_t pid) cnt_pi++;/* start port */ - diag =rte_eth_dev_start (pi); -if (diag <0 ) { -printf ("Fail to start port %d: %s\n" , pi, -rte_strerror (-diag)); +if (rte_eal_process_type () ==RTE_PROC_PRIMARY ) {
app/test-pmd/testpmd.c start_port(portid_t pid) · line 2670
Quoted 10 lines
app/test-pmd/testpmd.c +6
-2+ diag =rte_eth_dev_start (pi); +if (diag <0 ) { +printf ("Fail to start port %d: %s\n" , pi, +rte_strerror (-diag)); -/* Fail to setup rx queue, return */ -if (rte_atomic16_cmpset (&(port->port_status), +/* Fail to setup rx queue, return */ +if (rte_atomic16_cmpset (&(port->port_status),RTE_PORT_HANDLING ,RTE_PORT_STOPPED ) ==0 )
Syntax is wrong here after change, but if you go with above idea to hide process type check within functions, you won't need to change these lines at all.
<...>
app/test-pmd/testpmd.c main(int argc, char** argv) · line 3929
Quoted 7 lines
app/test-pmd/testpmd.c +2
-1@@ -3885,8 +3925,10 @@ main(int argc, char** argv) } } -if (!no_device_start &&start_port (RTE_PORT_ALL ) !=0 ) +if (!no_device_start &&start_port (RTE_PORT_ALL ) !=0 ) { +pmd_test_exit ();
Why need to add 'pmd_test_exit()' for the multi-process support, if there is a special case, please add comment for it.
Hi, Ferrruh,
All is fixed in v10, please check it out, thanks.
Quoted 272 lines · Ferruh Yigit
Quoted 10 lines · Min Hu (Connor)
This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1Hi Connor,
Thanks for the update. +Anatoly as multi-process maintainer, and ethdev maintainers.
Quoted 73 lines
Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>Acked-by Xiaoyun Li <[email protected]>Acked-by Ajit Khaparde <[email protected]>---
v9:
* Updated release notes and rst doc.
* Deleted deprecated codes.
* move macro and variable.v8:
* Added warning info about queue numbers and process numbers.v7:
* Fixed compiling error for unexpected unindent.v6:
* Add rte flow description for multiple process.v5:
* Fixedrun_app.rstfor multiple process description.
* Fix compiling error.v4:
* Fixed minimum vlaue of Rxq or Txq in doc.v3:
* Fixed compiling error using gcc10.0.v2:
* Added document for this patch.
---
app/test-pmd/cmdline.c | 6 ++ app/test-pmd/config.c | 14 ++++- app/test-pmd/parameters.c | 12 ++++ app/test-pmd/testpmd.c | 108
+++++++++++++++++++++++----------
app/test-pmd/testpmd.h | 3 +doc/guides/rel_notes/release_21_05.rst| 1 + doc/guides/testpmd_app_ug/run_app.rst | 86 ++++++++++++++++++++++++++7 files changed, 196 insertions(+), 34 deletions(-)app/test-pmd/cmdline.c +6
-0@@ -5354,6 +5354,12 @@ cmd_set_flush_rx_parsed(void *parsed_result, __rte_unused void *data) { struct cmd_set_flush_rx *res = parsed_result; + + if (num_procs > 1 && (strcmp(res->mode, "on") == 0)) { + printf("multi-process doesn't support to flush rx queues.\n"); + return; + } + no_flush_rx = (uint8_t)((strcmp(res->mode, "on") == 0) ? 0 : 1); }app/test-pmd/config.c +2
-0@@ -2860,6 +2860,8 @@ rss_fwd_config_setup(void) queueid_t rxq; queueid_t nb_q; streamid_t sm_id; + int start; + int end; nb_q = nb_rxq; if (nb_q > nb_txq)@@ -2877,7 +2879,15 @@ rss_fwd_config_setup(void) init_fwd_streams(); setup_fwd_config_of_each_lcore(&cur_fwd_config);app/test-pmd/config.c rss_fwd_config_setup(void) · line 2889
Quoted 10 lines
app/test-pmd/config.c +9
-1- rxp = 0; rxq = 0; + + if (proc_id > 0 && nb_q % num_procs) + printf("Warning! queue numbers should be multiple of " + "processes, or packet loss will happen.\n"); + + start = proc_id * nb_q / num_procs; + end = start + nb_q / num_procs; + rxp = 0; + rxq = start;Can you put some comment above, on what/why is done, something similar to you already put into the documentation?
Quoted 33 lines
app/test-pmd/config.c +2
-0for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { struct fwd_stream *fs;@@ -2894,6 +2904,8 @@ rss_fwd_config_setup(void) continue; rxp = 0; rxq++; + if (rxq >= end) + rxq = start; } }app/test-pmd/parameters.c +0
-0@@ -508,6 +508,9 @@ parse_link_speed(int n)void
launch_args_parse(int argc, char** argv){app/test-pmd/parameters.c +3
-0+#define PARAM_PROC_ID "proc-id" +#define PARAM_NUM_PROCS "num-procs" +int n, opt;
char **argvopt;
intopt_idx;app/test-pmd/parameters.c +2
-0@@ -625,6 +628,8 @@ launch_args_parse(int argc, char** argv) { "rx-mq-mode", 1, 0, 0 }, { "record-core-cycles", 0, 0, 0 }, { "record-burst-stats", 0, 0, 0 }, + { PARAM_NUM_PROCS, 1, 0, 0 }, + { PARAM_PROC_ID, 1, 0, 0 }, { 0, 0, 0, 0 }, };app/test-pmd/parameters.c launch_args_parse(int argc, char** argv) · line 1399
Quoted 4 lines
app/test-pmd/parameters.c +1
-0@@ -1391,6 +1396,13 @@ launch_args_parse(int argc, char** argv) record_core_cycles = 1; if (!strcmp(lgopts[opt_idx].name, "record-burst-stats")) record_burst_stats = 1; +To be consistent for rest, this empty line can be removed.
Quoted 14 lines
app/test-pmd/parameters.c +6
-0+ if (strncmp(lgopts[opt_idx].name, + PARAM_NUM_PROCS, 9) == 0) + num_procs = atoi(optarg); + if (strncmp(lgopts[opt_idx].name, + PARAM_PROC_ID, 7) == 0) + proc_id = atoi(optarg); break;case 'h':
usage(argv[0]);app/test-pmd/testpmd.c +0
-0app/test-pmd/testpmd.c enum rte_eth_rx_mq_mode rx_mq_mode = · line 523
Quoted 6 lines
app/test-pmd/testpmd.c +3
-0@@ -518,6 +518,16 @@ enum rte_eth_rx_mq_mode rx_mq_mode =ETH_MQ_RX_VMDQ_DCB_RSS; */ uint32_t eth_link_speed; +/* +* Id of the current process. +*/ Can you please add more info, like this being related to the primary/secondary process support and used to configure the queues to be polled.
app/test-pmd/testpmd.c enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS; · line 530
Quoted 7 lines
app/test-pmd/testpmd.c +7
-0+int proc_id; + +/* +* Number of processes. +*/ +unsigned int num_procs =1 ; +Ditto.
<...>
Quoted 22 lines
app/test-pmd/testpmd.c +6
-10@@ -2511,21 +2537,28 @@ start_port(portid_t pid) return -1; } /* configure port */ - diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq, + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + diag = rte_eth_dev_configure(pi, + nb_rxq + nb_hairpinq, nb_txq + nb_hairpinq, &(port->dev_conf)); - if (diag != 0) { - if (rte_atomic16_cmpset(&(port->port_status), - RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) - printf("Port %d can not be set back " - "to stopped\n", pi); - printf("Fail to configure port %d\n", pi); - /* try to reconfigure port next time */ - port->need_reconfig = 1; - return -1; + if (diag != 0) { + if (rte_atomic16_cmpset( + &(port->port_status),app/test-pmd/testpmd.c start_port(portid_t pid) · line 2557
Quoted 10 lines
app/test-pmd/testpmd.c +10
-0+ RTE_PORT_HANDLING, + RTE_PORT_STOPPED) == 0) + printf("Port %d can not be set " + "back to stopped\n", pi); + printf("Fail to configure port %d\n", + pi); + /* try to reconfigure port next time */ + port->need_reconfig = 1; + return -1; + }Just an idea,
I am a little worried about the complexity this new process type checks are added to the multiple locations. What do you think hiding process type checks behind new functions, like:eth_dev_configure_mp(...){
if (rte_eal_process_type()==RTE_PROC_PRIMARY) {
returnrte_eth_dev_configure(..)
}
return 0;
}and replace it with '
rte_eth_dev_configure()':-diag =
rte_eth_dev_configure(...)
+diag =eth_dev_configure_mp(...)Do you think does this help reducing the complexity added by the multi-process support?
app/test-pmd/testpmd.c start_port(portid_t pid) · line 2561
Quoted 5 lines
app/test-pmd/testpmd.c +2
-1} } - if (port->need_reconfig_queues > 0) { + if (port->need_reconfig_queues > 0 && + rte_eal_process_type() == RTE_PROC_PRIMARY) {According our coding convention, we don't allign with paranthesis but put double tabs.
And what about creating an inline function for primary process check, like
is_proc_primary(), that may help.Quoted 10 lines
app/test-pmd/testpmd.c +0
-4port->need_reconfig_queues = 0; /* setup tx queues */ for (qi = 0; qi < nb_txq; qi++) {@@ -2626,17 +2659,20 @@ start_port(portid_t pid) cnt_pi++; /* start port */ - diag = rte_eth_dev_start(pi); - if (diag < 0) { - printf("Fail to start port %d: %s\n", pi, - rte_strerror(-diag));app/test-pmd/testpmd.c start_port(portid_t pid) · line 2668
Quoted 10 lines
app/test-pmd/testpmd.c +7
-2+ if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + diag = rte_eth_dev_start(pi); + if (diag < 0) { + printf("Fail to start port %d: %s\n", pi, + rte_strerror(-diag)); - /* Fail to setup rx queue, return */ - if (rte_atomic16_cmpset(&(port->port_status), + /* Fail to setup rx queue, return */ + if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)Syntax is wrong here after change, but if you go with above idea to hide process type check within functions, you won't need to change these lines at all.
<...>
app/test-pmd/testpmd.c main(int argc, char** argv) · line 3928
Quoted 6 lines
app/test-pmd/testpmd.c +2
-1@@ -3885,8 +3925,10 @@ main(int argc, char** argv) } } - if (!no_device_start && start_port(RTE_PORT_ALL) != 0) + if (!no_device_start && start_port(RTE_PORT_ALL) != 0) { + pmd_test_exit();Why need to add '
pmd_test_exit()' for the multi-process support, if there is a special case, please add comment for it..
PATCH v10 1/1 app/testpmd: support multi-process
In reply to Lijun OuThis patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0
the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1
---
v10:
* Hid process type checks behind new functions.
* Added comments.
v9:
* Updated release notes and rst doc.
* Deleted deprecated codes.
* move macro and variable.
v8:
* Added warning info about queue numbers and process numbers.
v7:
* Fixed compiling error for unexpected unindent.
v6:
* Add rte flow description for multiple process.
v5:
* Fixed run_app.rst for multiple process description.
* Fix compiling error.
v4:
* Fixed minimum vlaue of Rxq or Txq in doc.
v3:
* Fixed compiling error using gcc10.0.
v2:
* Added document for this patch.
--- app/test-pmd/cmdline.c | 6 ++ app/test-pmd/config.c | 21 +++++- app/test-pmd/parameters.c | 11 +++ app/test-pmd/testpmd.c | 127 ++++++++++++++++++++++++++------- app/test-pmd/testpmd.h | 9 +++ doc/guides/rel_notes/release_21_05.rst | 1 + doc/guides/testpmd_app_ug/run_app.rst | 86 ++++++++++++++++++++++ 7 files changed, 236 insertions(+), 25 deletions(-)
app/test-pmd/cmdline.c +6 -0
@@ -5354,6 +5354,12 @@ cmd_set_flush_rx_parsed(void *parsed_result, __rte_unused void *data) {struct cmd_set_flush_rx *res = parsed_result; + +if (num_procs >1 && (strcmp (res->mode,"on" ) ==0 )) { +printf ("multi-process doesn't support to flush rx queues.\n" ); +return ; + } + no_flush_rx = (uint8_t )((strcmp (res->mode,"on" ) ==0 ) ?0 :1 ); }
app/test-pmd/config.c +20 -1
@@ -2860,6 +2860,8 @@ rss_fwd_config_setup(void)queueid_t rxq;queueid_t nb_q;streamid_t sm_id; +int start; +int end; nb_q = nb_rxq;if (nb_q > nb_txq)
@@ -2877,7 +2879,22 @@ rss_fwd_config_setup(void)init_fwd_streams ();setup_fwd_config_of_each_lcore (&cur_fwd_config); - rxp =0 ; rxq =0 ; + +if (proc_id >0 && nb_q % num_procs) +printf ("Warning! queue numbers should be multiple of " +"processes, or packet loss will happen.\n" ); + +/** +* In multi-process, All queues are allocated to different +* processes based on num_procs and proc_id. For example: +* if supports 4 queues(nb_q), 2 processes(num_procs), +* the 0~1 queue for primary process. +* the 2~3 queue for secondary process. +*/ + start = proc_id * nb_q / num_procs; + end = start + nb_q / num_procs; + rxp =0 ; + rxq = start;for (sm_id =0 ; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {struct fwd_stream *fs;
@@ -2894,6 +2911,8 @@ rss_fwd_config_setup(void)continue ; rxp =0 ; rxq++; +if (rxq >= end) + rxq = start; } }
app/test-pmd/parameters.c +11 -0
@@ -508,6 +508,9 @@ parse_link_speed(int n)void launch_args_parse (int argc,char ** argv) { +#define PARAM_PROC_ID "proc-id" +#define PARAM_NUM_PROCS "num-procs" +int n, opt;char **argvopt;int opt_idx;
@@ -625,6 +628,8 @@ launch_args_parse(int argc, char** argv) { "rx-mq-mode", 1, 0, 0 }, { "record-core-cycles", 0, 0, 0 }, { "record-burst-stats", 0, 0, 0 }, + { PARAM_NUM_PROCS, 1, 0, 0 }, + { PARAM_PROC_ID, 1, 0, 0 }, { 0, 0, 0, 0 }, };
@@ -1391,6 +1396,12 @@ launch_args_parse(int argc, char** argv) record_core_cycles =1 ;if (!strcmp (lgopts[opt_idx].name,"record-burst-stats" )) record_burst_stats =1 ; +if (strncmp (lgopts[opt_idx].name, +PARAM_NUM_PROCS ,9 ) ==0 ) + num_procs =atoi (optarg); +if (strncmp (lgopts[opt_idx].name, +PARAM_PROC_ID ,7 ) ==0 ) + proc_id =atoi (optarg);break ;case 'h' :usage (argv[0 ]);
app/test-pmd/testpmd.c +103 -24
212 lines not on this page · raw message
app/test-pmd/testpmd.h +9 -0
@@ -630,6 +630,15 @@ extern struct mplsoudp_decap_conf mplsoudp_decap_conf;extern enum rte_eth_rx_mq_mode rx_mq_mode; +extern int proc_id; +extern unsigned int num_procs; + +static inline bool +is_proc_primary (void ) +{ +return rte_eal_process_type () ==RTE_PROC_PRIMARY ; +} +static inline unsigned int lcore_num (void ) {
doc/guides/rel_notes/release_21_05.rst +1 -0
@@ -185,6 +185,7 @@ New Features ``show port (port_id) rxq (queue_id) desc used count`` * Added command to dump internal representation information of single flow. ``flow dump (port_id) rule (rule_id)`` + * Added support multi-process for testpmd. Removed Items
doc/guides/testpmd_app_ug/run_app.rst +86 -0
89 lines not on this page · raw message
Signature · 1 line
2.7.4
Quoted 6 lines · Min Hu (Connor)
This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0
Quoted 8 lines
the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>Acked-by Xiaoyun Li <[email protected]>Acked-by Ajit Khaparde <[email protected]>
Hi Connor,
I put some minor syntax comments, when they are fixed,
<...>
app/test-pmd/testpmd.c start_port(portid_t pid) · line 2596
Quoted 12 lines
if (diag != 0) {app/test-pmd/testpmd.c +6
-5-if (rte_atomic16_cmpset (&(port->port_status), -RTE_PORT_HANDLING ,RTE_PORT_STOPPED ) ==0 ) -printf ("Port %d can not be set back " -"to stopped\n" , pi); -printf ("Fail to configure port %d\n" , pi); +if (rte_atomic16_cmpset ( + &(port->port_status), +RTE_PORT_HANDLING , +RTE_PORT_STOPPED ) ==0 ) +printf ("Port %d can not be set " +"back to stopped\n" , pi);
It is OK to have long line for the strings, so it can be like printf("Port %d can not be set back to stopped\n",
pi);
app/test-pmd/testpmd.c start_port(portid_t pid) · line 2598
Quoted 2 lines
app/test-pmd/testpmd.c +2
-0+printf ("Fail to configure port %d\n" , + pi);
No need to update this line.
<...>
app/test-pmd/testpmd.c start_port(portid_t pid) · line 2715
Quoted 8 lines
/* Fail to setup rx queue, return */ if (rte_atomic16_cmpset(&(port->port_status),
app/test-pmd/testpmd.c +3
-2- RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) + RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)printf ("Port %d can not be set back to " -"stopped\n" , pi); +"stopped\n" , + pi);
These are syntax changes, I think can keep these lines as it is.
Thanks Ferruh, fixed in v11, thanks.
Quoted 59 lines · Ferruh Yigit
Quoted 6 lines · Min Hu (Connor)
This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0Quoted 8 lines
the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>Acked-by Xiaoyun Li <[email protected]>Acked-by Ajit Khaparde <[email protected]>Hi Connor,
I put some minor syntax comments, when they are fixed,
Reviewed-by Ferruh Yigit <[email protected]><...>
Quoted 4 lines
if (diag != 0) {
- if (rte_atomic16_cmpset(&(port->port_status), - RTE_PORT_HANDLING, RTE_PORT_STOPPED)== 0) - printf("Port %d can not be set back "Quoted 8 lines
- "to stopped\n", pi); -
printf("Fail to configure port %d\n", pi); + if (rte_atomic16_cmpset(
+ &(port->port_status),
+RTE_PORT_HANDLING,
+RTE_PORT_STOPPED) == 0)
+ printf("Port %d can not be set " + "back to stopped\n", pi);It is OK to have long line for the strings, so it can be like printf("Port %d can not be set back to stopped\n",
pi);Quoted 2 lines
+
printf("Fail to configure port %d\n", + pi);No need to update this line.
<...>
Quoted 8 lines
/* Fail to setup rx queue, return */ if (
rte_atomic16_cmpset(&(port->port_status), - RTE_PORT_HANDLING, RTE_PORT_STOPPED)== 0) +RTE_PORT_HANDLING,RTE_PORT_STOPPED) == 0) printf("Port %d can not be set back to " - "stopped\n", pi);
+ "stopped\n",
+ pi);These are syntax changes, I think can keep these lines as it is.
.
PATCH v11 1/1 app/testpmd: support multi-process
In reply to Lijun OuThis patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0
the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1
---
v11:
* Fixed some minor syntax.
v10:
* Hid process type checks behind new functions.
* Added comments.
v9:
* Updated release notes and rst doc.
* Deleted deprecated codes.
* move macro and variable.
v8:
* Added warning info about queue numbers and process numbers.
v7:
* Fixed compiling error for unexpected unindent.
v6:
* Add rte flow description for multiple process.
v5:
* Fixed run_app.rst for multiple process description.
* Fix compiling error.
v4:
* Fixed minimum vlaue of Rxq or Txq in doc.
v3:
* Fixed compiling error using gcc10.0.
v2:
* Added document for this patch.
--- app/test-pmd/cmdline.c | 6 ++ app/test-pmd/config.c | 21 +++++- app/test-pmd/parameters.c | 11 +++ app/test-pmd/testpmd.c | 129 ++++++++++++++++++++++++++------- app/test-pmd/testpmd.h | 9 +++ doc/guides/rel_notes/release_21_05.rst | 1 + doc/guides/testpmd_app_ug/run_app.rst | 86 ++++++++++++++++++++++ 7 files changed, 236 insertions(+), 27 deletions(-)
app/test-pmd/cmdline.c +6 -0
@@ -5354,6 +5354,12 @@ cmd_set_flush_rx_parsed(void *parsed_result, __rte_unused void *data) {struct cmd_set_flush_rx *res = parsed_result; + +if (num_procs >1 && (strcmp (res->mode,"on" ) ==0 )) { +printf ("multi-process doesn't support to flush rx queues.\n" ); +return ; + } + no_flush_rx = (uint8_t )((strcmp (res->mode,"on" ) ==0 ) ?0 :1 ); }
app/test-pmd/config.c +20 -1
@@ -2859,6 +2859,8 @@ rss_fwd_config_setup(void)queueid_t rxq;queueid_t nb_q;streamid_t sm_id; +int start; +int end; nb_q = nb_rxq;if (nb_q > nb_txq)
@@ -2876,7 +2878,22 @@ rss_fwd_config_setup(void)init_fwd_streams ();setup_fwd_config_of_each_lcore (&cur_fwd_config); - rxp =0 ; rxq =0 ; + +if (proc_id >0 && nb_q % num_procs) +printf ("Warning! queue numbers should be multiple of " +"processes, or packet loss will happen.\n" ); + +/** +* In multi-process, All queues are allocated to different +* processes based on num_procs and proc_id. For example: +* if supports 4 queues(nb_q), 2 processes(num_procs), +* the 0~1 queue for primary process. +* the 2~3 queue for secondary process. +*/ + start = proc_id * nb_q / num_procs; + end = start + nb_q / num_procs; + rxp =0 ; + rxq = start;for (sm_id =0 ; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {struct fwd_stream *fs;
@@ -2893,6 +2910,8 @@ rss_fwd_config_setup(void)continue ; rxp =0 ; rxq++; +if (rxq >= end) + rxq = start; } }
app/test-pmd/parameters.c +11 -0
@@ -508,6 +508,9 @@ parse_link_speed(int n)void launch_args_parse (int argc,char ** argv) { +#define PARAM_PROC_ID "proc-id" +#define PARAM_NUM_PROCS "num-procs" +int n, opt;char **argvopt;int opt_idx;
@@ -625,6 +628,8 @@ launch_args_parse(int argc, char** argv) { "rx-mq-mode", 1, 0, 0 }, { "record-core-cycles", 0, 0, 0 }, { "record-burst-stats", 0, 0, 0 }, + { PARAM_NUM_PROCS, 1, 0, 0 }, + { PARAM_PROC_ID, 1, 0, 0 }, { 0, 0, 0, 0 }, };
@@ -1391,6 +1396,12 @@ launch_args_parse(int argc, char** argv) record_core_cycles =1 ;if (!strcmp (lgopts[opt_idx].name,"record-burst-stats" )) record_burst_stats =1 ; +if (strncmp (lgopts[opt_idx].name, +PARAM_NUM_PROCS ,9 ) ==0 ) + num_procs =atoi (optarg); +if (strncmp (lgopts[opt_idx].name, +PARAM_PROC_ID ,7 ) ==0 ) + proc_id =atoi (optarg);break ;case 'h' :usage (argv[0 ]);
app/test-pmd/testpmd.c +103 -26
220 lines not on this page · raw message
app/test-pmd/testpmd.h +9 -0
@@ -630,6 +630,15 @@ extern struct mplsoudp_decap_conf mplsoudp_decap_conf;extern enum rte_eth_rx_mq_mode rx_mq_mode; +extern int proc_id; +extern unsigned int num_procs; + +static inline bool +is_proc_primary (void ) +{ +return rte_eal_process_type () ==RTE_PROC_PRIMARY ; +} +static inline unsigned int lcore_num (void ) {
doc/guides/rel_notes/release_21_05.rst +1 -0
@@ -192,6 +192,7 @@ New Features ``show port (port_id) rxq (queue_id) desc used count`` * Added command to dump internal representation information of single flow. ``flow dump (port_id) rule (rule_id)`` + * Added support multi-process for testpmd. Removed Items
doc/guides/testpmd_app_ug/run_app.rst +86 -0
89 lines not on this page · raw message
Signature · 1 line
2.7.4
Quoted 10 lines · Min Hu (Connor)
This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1
Hi Connor,
I put some comments on the documentation, can you please check them?
Meanwhile, John, Marko can you please help on the doc review?
Thanks,
ferruh
Quoted 5 lines
Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>Acked-by Xiaoyun Li <[email protected]>Acked-by Ajit Khaparde <[email protected]>Reviewed-by Ferruh Yigit <[email protected]>
<...>
Quoted 8 lines
app/test-pmd/testpmd.c +1
-1@@ -2626,16 +2701,16 @@ start_port(portid_t pid) cnt_pi++;/* start port */ - diag =rte_eth_dev_start (pi); + diag =eth_dev_start_mp (pi);if (diag <0 ) { printf("Fail to start port %d: %s\n", pi,
app/test-pmd/testpmd.c start_port(portid_t pid) · line 2713
Quoted 10 lines
app/test-pmd/testpmd.c +3
-3rte_strerror(-diag));/* Fail to setup rx queue, return */ if (rte_atomic16_cmpset (&(port->port_status), -RTE_PORT_HANDLING ,RTE_PORT_STOPPED ) ==0 ) -printf ("Port %d can not be set back to " -"stopped\n" , pi); +RTE_PORT_HANDLING ,RTE_PORT_STOPPED ) ==0 ) +printf ("Port %d can not be set back to stopped\n" , + pi);
Indentation is wrong.
<...>
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 549
Quoted 10 lines
doc/guides/testpmd_app_ug/run_app.rst +4
-0@@ -543,3 +543,89 @@ The command line options are: bit 1 - two hairpin ports paired bit 0 - two hairpin ports loop The default value is 0. Hairpin will use single port mode and implicit Tx flow mode. + + +Testpmd Support Multi Process Command-line Options +--------------------------------------------------
What do you think making this a sub-section of the "Testpmd Command-line Options"?
And the section name can be "Testpmd Multi-Process Command-line Options"
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 551
Quoted 2 lines
doc/guides/testpmd_app_ug/run_app.rst +2
-0+ +The following are the command-line options for the testpmd applications(support
Space before '('.
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 553
Quoted 2 lines
doc/guides/testpmd_app_ug/run_app.rst +2
-0+multi process).They must be separated from the EAL options, shown in the previous +section, with a ``--`` separator:
If this becomes sub-section of the "Testpmd Command-line Options", above will become duplication, and I think it can be simplified as:
"The following are the command-line options for testpmd multi-process support:"
Quoted 26 lines
doc/guides/testpmd_app_ug/run_app.rst +26
-0+ +.. code-block:: console + + primary process: + sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i --rxq=4 --txq=4 \ + --num-procs=2 --proc-id=0 + + secondary process: + sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i --rxq=4 --txq=4 \ + --num-procs=2 --proc-id=1 + +The command line options are: + +* ``--rxq=N`` + + Set the number of Rx queues per port to N. N is the sum of queues used by primary + and secondary process. Primary process and secondary process should have separate + queues, and each should occupy at least one queue. Where N should be the multiple + of number of processes. + +* ``--txq=N`` + + Set the number of Tx queues per port to N. N is the sum of queues used by primary + and secondary process. Primary process and secondary process should have separate + queues, and each should occupy at least one queue. Where N should be the multiple + of number of processes.
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 589
Quoted 10 lines
doc/guides/testpmd_app_ug/run_app.rst +10
-0+ +* ``--num-procs=N`` + + The number of processes which will be used. + +* ``--proc-id=id`` + + The id of the current process (id < num-procs). id should be different in primary + process and secondary process. +
The famous question, does it start from '0' or '1'?
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 591
Quoted 2 lines
doc/guides/testpmd_app_ug/run_app.rst +2
-0+Calculation rule for queue: +All queues are allocated to different processes based on proc_num and proc_id.
Can highlight proc_num and proc_id, as ``proc_num`` and ``proc_id``.
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 592
Quoted 1 line
doc/guides/testpmd_app_ug/run_app.rst +1
-0+Calculation rule for the Testpmd to allocate queues to each process:
Not sure if testpmd should start with uppercase.
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 595
Quoted 3 lines
doc/guides/testpmd_app_ug/run_app.rst +3
-0+start(queue start id) = proc_id * nb_q / num_procs; +end(queue end id) = start + nb_q / num_procs; +
Can you put above into a code-block.
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 596
Quoted 1 line
doc/guides/testpmd_app_ug/run_app.rst +1
-0+For example, if supports 4 txq and rxq
".., if testpmp supports 4 Tx and Rx queues"
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 605
Quoted 9 lines
doc/guides/testpmd_app_ug/run_app.rst +9
-0+the 0~1 for primary process +the 2~3 for secondary process + +The number of rings should be a multiple of the number of processes. If not, +redundant queues will exist after queues are allocated to processes. After RSS is +enabled, packet loss occurs when traffic is sent to all processes at the same time. +Some traffic enters redundant queues and cannot be forwarded. + +Most dev ops is supported in primary and secondary process. While secondary process
"Most of the device operations are supported in ..."? "While the secondary process ..."?
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 615
Quoted 10 lines
doc/guides/testpmd_app_ug/run_app.rst +10
-0+is not permitted to allocate or release shared memory, so some ops are not supported +as follows: +``dev_configure`` +``dev_start`` +``dev_stop`` +``rx_queue_setup`` +``tx_queue_setup`` +``rx_queue_release`` +``tx_queue_release`` +
This list displayed as single line in the html output, if the intention is to have them listed as been in the source, may need following update:
-as follows:
+as follows::
+
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 621
Quoted 6 lines
doc/guides/testpmd_app_ug/run_app.rst +6
-0+So, any command from testpmd which calls those APIs will not be supported in secondary +process, like: +``port config all rxq|txq|rxd|txd <value>`` +``port config <port_id> rx_offload xxx on/off `` +``port config <port_id> tx_offload xxx on/off`` +etc.
Same here for the list view.
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 623
Quoted 2 lines
doc/guides/testpmd_app_ug/run_app.rst +2
-0+ +RTE_FLOW supported, it applies only on its own process on SW side, but all on HW size.
You may use "Flow API", instead of RTE_FLOW, so it becomes: "Flow API is supported."
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 625
Quoted 2 lines
doc/guides/testpmd_app_ug/run_app.rst +2
-0+stats supported, stats will not change when one quit and start, As they share the same +buffer to store the stats.
The above stats related sentences looks like squeezed within the flow API related description, it may be good to move it.
Quoted 2 lines
Flow rules are maintained in process level: primary and secondary
+has its own flow list(but one flow list in HW). The two can see all the queues, so setting
Space before '('.
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 628
Quoted 2 lines
doc/guides/testpmd_app_ug/run_app.rst +2
-0+the flow rules for the other is OK. Of course, io(receive or transmit packets) in the queue +from others is not permitted.
I understand what you mean, but wording can be improved, can you please try to reword it?
doc/guides/testpmd_app_ug/run_app.rst
Quoted 2 lines
doc/guides/testpmd_app_ug/run_app.rst +2
-0+ +RSS supported, Primary process and secondary process has separate queues to use, RSS
"RSS is supported"? And start 'primary' with lowercase.
Quoted 2 lines
+will work in their own queues whether primary and secondary process.
There is a chance that when RSS is enabled, the packet may end up in other process' queues?
Hi, Ferruh,
Quoted 136 lines · Ferruh Yigit
Quoted 10 lines · Min Hu (Connor)
This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1Hi Connor,
I put some comments on the documentation, can you please check them?
Meanwhile, John, Marko can you please help on the doc review?
Thanks,
ferruhQuoted 5 lines
Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>Acked-by Xiaoyun Li <[email protected]>Acked-by Ajit Khaparde <[email protected]>Reviewed-by Ferruh Yigit <[email protected]><...>
Quoted 6 lines
app/test-pmd/testpmd.c +1
-1@@ -2626,16 +2701,16 @@ start_port(portid_t pid) cnt_pi++; /* start port */ - diag = rte_eth_dev_start(pi); + diag = eth_dev_start_mp(pi); if (diag < 0) {app/test-pmd/testpmd.c start_port(portid_t pid) · line 2711
Quoted 10 lines
app/test-pmd/testpmd.c +3
-3printf("Fail to start port %d: %s\n", pi, rte_strerror(-diag)); /* Fail to setup rx queue, return */ if (rte_atomic16_cmpset(&(port->port_status), - RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) - printf("Port %d can not be set back to " - "stopped\n", pi); + RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) + printf("Port %d can not be set back to stopped\n", + pi);Indentation is wrong.
<...>
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 549
Quoted 10 lines
doc/guides/testpmd_app_ug/run_app.rst +0
-0@@ -543,3 +543,89 @@ The command line options are:bit 1 - two hairpin ports paired
bit 0 - two hairpin ports loop
The default value is 0. Hairpin will use single port mode and implicit Tx flow mode.doc/guides/testpmd_app_ug/run_app.rst +4
-0+ + +Testpmd Support Multi Process Command-line Options +--------------------------------------------------What do you think making this a sub-section of the "Testpmd Command-line Options"?
And the section name can be "Testpmd Multi-Process Command-line Options"
Quoted 3 lines
doc/guides/testpmd_app_ug/run_app.rst +2
-0+ +The following are the command-line options for the testpmdapplications(support
Space before '('.
Quoted 3 lines
+multi process).They must be separated from the EAL options, shown in the previous
+section, with a ``--`` separator:If this becomes sub-section of the "Testpmd Command-line Options", above will become duplication, and I think it can be simplified as:
"The following are the command-line options for testpmd multi-process support:"
Quoted 42 lines
doc/guides/testpmd_app_ug/run_app.rst +16
-2+ +.. code-block:: console + + primary process: + sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i --rxq=4 --txq=4 \ + --num-procs=2 --proc-id=0 + + secondary process: + sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i --rxq=4 --txq=4 \ + --num-procs=2 --proc-id=1 + +The command line options are: + +* ``--rxq=N`` + + Set the number of Rx queues per port to N. N is the sum of queuesused by primary
+ and secondary process. Primary process and secondary process should have separate
+ queues, and each should occupy at least one queue. Where N should be the multiple
+ of number of processes.doc/guides/testpmd_app_ug/run_app.rst +4
-0+ +* ``--txq=N`` + + Set the number of Tx queues per port to N. N is the sum of queuesused by primary
+ and secondary process. Primary process and secondary process should have separate
+ queues, and each should occupy at least one queue. Where N should be the multiple
+ of number of processes.doc/guides/testpmd_app_ug/run_app.rst +8
-0+ +* ``--num-procs=N`` + + The number of processes which will be used. + +* ``--proc-id=id`` + + The id of the current process (id < num-procs). id should beQuoted 3 lines
different in primary
+ process and secondary process.
+The famous question, does it start from '0' or '1'?
Quoted 3 lines
doc/guides/testpmd_app_ug/run_app.rst +2
-0+Calculation rule for queue: +All queues are allocated to different processes based on proc_num and
proc_id.Can highlight
proc_numandproc_id, as ``proc_num`` and ``proc_id``.Quoted 1 line
+Calculation rule for the Testpmd to allocate queues to each process:
Quoted 8 lines
Not sure if testpmd should start with uppercase.
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 595
Quoted 3 lines
doc/guides/testpmd_app_ug/run_app.rst +3
-0+start(queue start id) = proc_id * nb_q / num_procs; +end(queue end id) = start + nb_q / num_procs; +Can you put above into a code-block.
Have already put above into a code-block in previous version.
Quoted 79 lines
Quoted 1 line
+For example, if supports 4 txq and rxq
".., if testpmp supports 4 Tx and Rx queues"
Quoted 12 lines
doc/guides/testpmd_app_ug/run_app.rst +4
-0+the 0~1 for primary process +the 2~3 for secondary process + +The number of rings should be a multiple of the number of processes.If not,
+redundant queues will exist after queues are allocated to processes. After RSS is
+enabled, packet loss occurs when traffic is sent to all processes at the same time.doc/guides/testpmd_app_ug/run_app.rst +3
-0+Some traffic enters redundant queues and cannot be forwarded. + +Most dev ops is supported in primary and secondary process. WhileQuoted 1 line
secondary process
"Most of the device operations are supported in ..."? "While the secondary process ..."?
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 615
Quoted 11 lines
+is not permitted to allocate or release shared memory, so some ops are not supported
doc/guides/testpmd_app_ug/run_app.rst +9
-0+as follows: +``dev_configure`` +``dev_start`` +``dev_stop`` +``rx_queue_setup`` +``tx_queue_setup`` +``rx_queue_release`` +``tx_queue_release`` +This list displayed as single line in the html output, if the intention is to have them listed as been in the source, may need following update:
-as follows:
+as follows::
+doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 621
Quoted 7 lines
+So, any command from testpmd which calls those APIs will not be supported in secondary
doc/guides/testpmd_app_ug/run_app.rst +5
-0+process, like: +``port config all rxq|txq|rxd|txd <value>`` +``port config <port_id> rx_offload xxx on/off `` +``port config <port_id> tx_offload xxx on/off`` +etc.Same here for the list view.
Quoted 3 lines
doc/guides/testpmd_app_ug/run_app.rst +2
-0+ +RTE_FLOW supported, it applies only on its own process on SW side,but all on HW size.
You may use "Flow API", instead of
RTE_FLOW, so it becomes: "Flow API is supported."Quoted 3 lines
+stats supported, stats will not change when one quit and start, As they share the same
+buffer to store the stats.The above stats related sentences looks like squeezed within the flow API related description, it may be good to move it.
Quoted 3 lines
Flow rules are maintained in process level: primary and secondary +has its own flow list(but one flow list in HW). The two can see all the queues, so setting
Space before '('.
Quoted 3 lines
+the flow rules for the other is OK. Of course, io(receive or transmit packets) in the queue
+from others is not permitted.I understand what you mean, but wording can be improved, can you please try to reword it?
Quoted 3 lines
doc/guides/testpmd_app_ug/run_app.rst +2
-0+ +RSS supported, Primary process and secondary process has separatequeues to use, RSS
"RSS is supported"? And start 'primary' with lowercase.
Quoted 4 lines
Quoted 2 lines
+will work in their own queues whether primary and secondary process.
There is a chance that when RSS is enabled, the packet may end up in other process' queues?
When RSS is enabled, packet could end up in other process's queues, but primary and secondary receives packerts from their own queues, and they do not interfere with each other, do not conflict with each other.
Quoted 1 line
.
PATCH v12 1/1 app/testpmd: support multi-process
In reply to Lijun OuThis patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0
the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1
---
v12:
* Updated doc info.
v11:
* Fixed some minor syntax.
v10:
* Hid process type checks behind new functions.
* Added comments.
v9:
* Updated release notes and rst doc.
* Deleted deprecated codes.
* move macro and variable.
v8:
* Added warning info about queue numbers and process numbers.
v7:
* Fixed compiling error for unexpected unindent.
v6:
* Add rte flow description for multiple process.
v5:
* Fixed run_app.rst for multiple process description.
* Fix compiling error.
v4:
* Fixed minimum vlaue of Rxq or Txq in doc.
v3:
* Fixed compiling error using gcc10.0.
v2:
* Added document for this patch.
--- app/test-pmd/cmdline.c | 6 ++ app/test-pmd/config.c | 21 +++++- app/test-pmd/parameters.c | 11 +++ app/test-pmd/testpmd.c | 129 ++++++++++++++++++++++++++------- app/test-pmd/testpmd.h | 9 +++ doc/guides/rel_notes/release_21_05.rst | 1 + doc/guides/testpmd_app_ug/run_app.rst | 84 +++++++++++++++++++++ 7 files changed, 234 insertions(+), 27 deletions(-)
app/test-pmd/cmdline.c +6 -0
@@ -5357,6 +5357,12 @@ cmd_set_flush_rx_parsed(void *parsed_result, __rte_unused void *data) {struct cmd_set_flush_rx *res = parsed_result; + +if (num_procs >1 && (strcmp (res->mode,"on" ) ==0 )) { +printf ("multi-process doesn't support to flush rx queues.\n" ); +return ; + } + no_flush_rx = (uint8_t )((strcmp (res->mode,"on" ) ==0 ) ?0 :1 ); }
app/test-pmd/config.c +20 -1
@@ -2924,6 +2924,8 @@ rss_fwd_config_setup(void)queueid_t rxq;queueid_t nb_q;streamid_t sm_id; +int start; +int end; nb_q = nb_rxq;if (nb_q > nb_txq)
@@ -2941,7 +2943,22 @@ rss_fwd_config_setup(void)init_fwd_streams ();setup_fwd_config_of_each_lcore (&cur_fwd_config); - rxp =0 ; rxq =0 ; + +if (proc_id >0 && nb_q % num_procs) +printf ("Warning! queue numbers should be multiple of " +"processes, or packet loss will happen.\n" ); + +/** +* In multi-process, All queues are allocated to different +* processes based on num_procs and proc_id. For example: +* if supports 4 queues(nb_q), 2 processes(num_procs), +* the 0~1 queue for primary process. +* the 2~3 queue for secondary process. +*/ + start = proc_id * nb_q / num_procs; + end = start + nb_q / num_procs; + rxp =0 ; + rxq = start;for (sm_id =0 ; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {struct fwd_stream *fs;
@@ -2958,6 +2975,8 @@ rss_fwd_config_setup(void)continue ; rxp =0 ; rxq++; +if (rxq >= end) + rxq = start; } }
app/test-pmd/parameters.c +11 -0
@@ -508,6 +508,9 @@ parse_link_speed(int n)void launch_args_parse (int argc,char ** argv) { +#define PARAM_PROC_ID "proc-id" +#define PARAM_NUM_PROCS "num-procs" +int n, opt;char **argvopt;int opt_idx;
@@ -625,6 +628,8 @@ launch_args_parse(int argc, char** argv) { "rx-mq-mode", 1, 0, 0 }, { "record-core-cycles", 0, 0, 0 }, { "record-burst-stats", 0, 0, 0 }, + { PARAM_NUM_PROCS, 1, 0, 0 }, + { PARAM_PROC_ID, 1, 0, 0 }, { 0, 0, 0, 0 }, };
@@ -1391,6 +1396,12 @@ launch_args_parse(int argc, char** argv) record_core_cycles =1 ;if (!strcmp (lgopts[opt_idx].name,"record-burst-stats" )) record_burst_stats =1 ; +if (strncmp (lgopts[opt_idx].name, +PARAM_NUM_PROCS ,9 ) ==0 ) + num_procs =atoi (optarg); +if (strncmp (lgopts[opt_idx].name, +PARAM_PROC_ID ,7 ) ==0 ) + proc_id =atoi (optarg);break ;case 'h' :usage (argv[0 ]);
app/test-pmd/testpmd.c +103 -26
220 lines not on this page · raw message
app/test-pmd/testpmd.h +9 -0
@@ -632,6 +632,15 @@ extern enum rte_eth_rx_mq_mode rx_mq_mode;extern struct rte_flow_action_conntrack conntrack_context; +extern int proc_id; +extern unsigned int num_procs; + +static inline bool +is_proc_primary (void ) +{ +return rte_eal_process_type () ==RTE_PROC_PRIMARY ; +} +static inline unsigned int lcore_num (void ) {
doc/guides/rel_notes/release_21_05.rst +1 -0
@@ -230,6 +230,7 @@ New Features * Added commands to construct conntrack context and relevant indirect action handle creation, update for conntrack action as well as conntrack item matching. + * Added support multi-process for testpmd. * **Updated ipsec-secgw sample application.**
doc/guides/testpmd_app_ug/run_app.rst +84 -0
87 lines not on this page · raw message
Signature · 1 line
2.7.4
PATCH v13 1/1 app/testpmd: support multi-process
In reply to Lijun OuThis patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0
the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1
---
v13:
* Modified the doc syntax.
v12:
* Updated doc info.
v11:
* Fixed some minor syntax.
v10:
* Hid process type checks behind new functions.
* Added comments.
v9:
* Updated release notes and rst doc.
* Deleted deprecated codes.
* move macro and variable.
v8:
* Added warning info about queue numbers and process numbers.
v7:
* Fixed compiling error for unexpected unindent.
v6:
* Add rte flow description for multiple process.
v5:
* Fixed run_app.rst for multiple process description.
* Fix compiling error.
v4:
* Fixed minimum vlaue of Rxq or Txq in doc.
v3:
* Fixed compiling error using gcc10.0.
v2:
* Added document for this patch.
--- app/test-pmd/cmdline.c | 6 ++ app/test-pmd/config.c | 21 +++++- app/test-pmd/parameters.c | 11 +++ app/test-pmd/testpmd.c | 129 ++++++++++++++++++++++++++------- app/test-pmd/testpmd.h | 9 +++ doc/guides/rel_notes/release_21_05.rst | 1 + doc/guides/testpmd_app_ug/run_app.rst | 70 ++++++++++++++++++ 7 files changed, 220 insertions(+), 27 deletions(-)
app/test-pmd/cmdline.c +6 -0
@@ -5450,6 +5450,12 @@ cmd_set_flush_rx_parsed(void *parsed_result, __rte_unused void *data) {struct cmd_set_flush_rx *res = parsed_result; + +if (num_procs >1 && (strcmp (res->mode,"on" ) ==0 )) { +printf ("multi-process doesn't support to flush rx queues.\n" ); +return ; + } + no_flush_rx = (uint8_t )((strcmp (res->mode,"on" ) ==0 ) ?0 :1 ); }
app/test-pmd/config.c +20 -1
@@ -2971,6 +2971,8 @@ rss_fwd_config_setup(void)queueid_t rxq;queueid_t nb_q;streamid_t sm_id; +int start; +int end; nb_q = nb_rxq;if (nb_q > nb_txq)
@@ -2988,7 +2990,22 @@ rss_fwd_config_setup(void)init_fwd_streams ();setup_fwd_config_of_each_lcore (&cur_fwd_config); - rxp =0 ; rxq =0 ; + +if (proc_id >0 && nb_q % num_procs) +printf ("Warning! queue numbers should be multiple of " +"processes, or packet loss will happen.\n" ); + +/** +* In multi-process, All queues are allocated to different +* processes based on num_procs and proc_id. For example: +* if supports 4 queues(nb_q), 2 processes(num_procs), +* the 0~1 queue for primary process. +* the 2~3 queue for secondary process. +*/ + start = proc_id * nb_q / num_procs; + end = start + nb_q / num_procs; + rxp =0 ; + rxq = start;for (sm_id =0 ; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {struct fwd_stream *fs;
@@ -3005,6 +3022,8 @@ rss_fwd_config_setup(void)continue ; rxp =0 ; rxq++; +if (rxq >= end) + rxq = start; } }
app/test-pmd/parameters.c +11 -0
@@ -508,6 +508,9 @@ parse_link_speed(int n)void launch_args_parse (int argc,char ** argv) { +#define PARAM_PROC_ID "proc-id" +#define PARAM_NUM_PROCS "num-procs" +int n, opt;char **argvopt;int opt_idx;
@@ -625,6 +628,8 @@ launch_args_parse(int argc, char** argv) { "rx-mq-mode", 1, 0, 0 }, { "record-core-cycles", 0, 0, 0 }, { "record-burst-stats", 0, 0, 0 }, + { PARAM_NUM_PROCS, 1, 0, 0 }, + { PARAM_PROC_ID, 1, 0, 0 }, { 0, 0, 0, 0 }, };
@@ -1391,6 +1396,12 @@ launch_args_parse(int argc, char** argv) record_core_cycles =1 ;if (!strcmp (lgopts[opt_idx].name,"record-burst-stats" )) record_burst_stats =1 ; +if (strncmp (lgopts[opt_idx].name, +PARAM_NUM_PROCS ,9 ) ==0 ) + num_procs =atoi (optarg); +if (strncmp (lgopts[opt_idx].name, +PARAM_PROC_ID ,7 ) ==0 ) + proc_id =atoi (optarg);break ;case 'h' :usage (argv[0 ]);
app/test-pmd/testpmd.c +103 -26
220 lines not on this page · raw message
app/test-pmd/testpmd.h +9 -0
@@ -632,6 +632,15 @@ extern enum rte_eth_rx_mq_mode rx_mq_mode;extern struct rte_flow_action_conntrack conntrack_context; +extern int proc_id; +extern unsigned int num_procs; + +static inline bool +is_proc_primary (void ) +{ +return rte_eal_process_type () ==RTE_PROC_PRIMARY ; +} +static inline unsigned int lcore_num (void ) {
doc/guides/rel_notes/release_21_05.rst +1 -0
@@ -235,6 +235,7 @@ New Features ``port cleanup (port_id) txq (queue_id) (free_cnt)`` * Added command to show link flow control info. ``show port (port_id) flow_ctrl`` + * Added support multi-process for testpmd. * **Updated ipsec-secgw sample application.**
doc/guides/testpmd_app_ug/run_app.rst +70 -0
73 lines not on this page · raw message
Signature · 1 line
2.7.4
@Thomas, @Ferruh, please, see question below.
Quoted 70 lines · Min Hu (Connor)
This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>Acked-by Xiaoyun Li <[email protected]>Acked-by Ajit Khaparde <[email protected]>Reviewed-by Ferruh Yigit <[email protected]>---
v13:
* Modified the doc syntax.v12:
* Updated doc info.v11:
* Fixed some minor syntax.v10:
* Hid process type checks behind new functions.
* Added comments.v9:
* Updated release notes and rst doc.
* Deleted deprecated codes.
* move macro and variable.v8:
* Added warning info about queue numbers and process numbers.v7:
* Fixed compiling error for unexpected unindent.v6:
* Add rte flow description for multiple process.v5:
* Fixedrun_app.rstfor multiple process description.
* Fix compiling error.v4:
* Fixed minimum vlaue of Rxq or Txq in doc.v3:
* Fixed compiling error using gcc10.0.v2:
* Added document for this patch.--- app/test-pmd/cmdline.c | 6 ++ app/test-pmd/config.c | 21 +++++- app/test-pmd/parameters.c | 11 +++ app/test-pmd/testpmd.c | 129 ++++++++++++++++++++++++++------- app/test-pmd/testpmd.h | 9 +++ doc/guides/rel_notes/release_21_05.rst | 1 + doc/guides/testpmd_app_ug/run_app.rst | 70 ++++++++++++++++++ 7 files changed, 220 insertions(+), 27 deletions(-)app/test-pmd/cmdline.c +0
-0
app/test-pmd/cmdline.c cmd_set_flush_rx_parsed(void *parsed_result, · line 5455
Quoted 6 lines
app/test-pmd/cmdline.c +3
-0@@ -5450,6 +5450,12 @@ cmd_set_flush_rx_parsed(void *parsed_result, __rte_unused void *data) {struct cmd_set_flush_rx *res = parsed_result; + +if (num_procs >1 && (strcmp (res->mode,"on" ) ==0 )) { +printf ("multi-process doesn't support to flush rx queues.\n" );
rx -> Rx
Quoted 20 lines
app/test-pmd/cmdline.c +3
-0+return ; + } + no_flush_rx = (uint8_t )((strcmp (res->mode,"on" ) ==0 ) ?0 :1 ); }app/test-pmd/config.c +2
-0@@ -2971,6 +2971,8 @@ rss_fwd_config_setup(void)queueid_t rxq;queueid_t nb_q;streamid_t sm_id; +int start; +int end; nb_q = nb_rxq;if (nb_q > nb_txq)
app/test-pmd/config.c rss_fwd_config_setup(void) · line 2994
Quoted 6 lines
app/test-pmd/config.c +2
-1@@ -2988,7 +2990,22 @@ rss_fwd_config_setup(void)init_fwd_streams ();setup_fwd_config_of_each_lcore (&cur_fwd_config); - rxp =0 ; rxq =0 ; + +if (proc_id >0 && nb_q % num_procs)
Please, compare result with 0 explicitly.
app/test-pmd/config.c rss_fwd_config_setup(void) · line 2996
Quoted 2 lines
app/test-pmd/config.c +2
-0+printf ("Warning! queue numbers should be multiple of " +"processes, or packet loss will happen.\n" );
Do not split format string across multiple lines.
Frankly speaking I don't undertand why. Why is it impossible to serve 2 queues in the first process and 1 queue in the second process if 3 queues and 2 processes are configured.
I think RSS redirection table can perfectly do it.
Quoted 48 lines
app/test-pmd/config.c +14
-0+ +/** +* In multi-process, All queues are allocated to different +* processes based on num_procs and proc_id. For example: +* if supports 4 queues(nb_q), 2 processes(num_procs), +* the 0~1 queue for primary process. +* the 2~3 queue for secondary process. +*/ + start = proc_id * nb_q / num_procs; + end = start + nb_q / num_procs; + rxp =0 ; + rxq = start;for (sm_id =0 ; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {struct fwd_stream *fs;@@ -3005,6 +3022,8 @@ rss_fwd_config_setup(void)continue ; rxp =0 ; rxq++; +if (rxq >= end) + rxq = start; } }app/test-pmd/parameters.c +5
-0@@ -508,6 +508,9 @@ parse_link_speed(int n)void launch_args_parse (int argc,char ** argv) { +#define PARAM_PROC_ID "proc-id" +#define PARAM_NUM_PROCS "num-procs" +int n, opt;char **argvopt;int opt_idx;@@ -625,6 +628,8 @@ launch_args_parse(int argc, char** argv) { "rx-mq-mode", 1, 0, 0 }, { "record-core-cycles", 0, 0, 0 }, { "record-burst-stats", 0, 0, 0 }, + { PARAM_NUM_PROCS, 1, 0, 0 }, + { PARAM_PROC_ID, 1, 0, 0 }, { 0, 0, 0, 0 }, };
app/test-pmd/parameters.c launch_args_parse(int argc, char** argv) · line 1400
Quoted 5 lines
app/test-pmd/parameters.c +2
-0@@ -1391,6 +1396,12 @@ launch_args_parse(int argc, char** argv) record_core_cycles =1 ;if (!strcmp (lgopts[opt_idx].name,"record-burst-stats" )) record_burst_stats =1 ; +if (strncmp (lgopts[opt_idx].name, +PARAM_NUM_PROCS ,9 ) ==0 )
I strongly dislike 9 here and 7 below. Why is strncmp() used here, but just strcmp() is used for all other options. It makes the code inconsistent.
Quoted 12 lines
app/test-pmd/parameters.c +4
-0+ num_procs =atoi (optarg); +if (strncmp (lgopts[opt_idx].name, +PARAM_PROC_ID ,7 ) ==0 ) + proc_id =atoi (optarg);break ;case 'h' :usage (argv[0 ]);app/test-pmd/testpmd.c +0
-0
app/test-pmd/testpmd.c enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS; · line 522
Quoted 5 lines
app/test-pmd/testpmd.c +2
-0@@ -518,6 +518,62 @@ enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS;*/ uint32_t eth_link_speed; +/* +* Id of the current process in multi-process, used to
Id -> ID in accordance with devtools/words-case.txt
Quoted 33 lines
app/test-pmd/testpmd.c +33
-0+* configure the queues to be polled. +*/ +int proc_id; + +/* +* Number of processes in multi-process, used to +* configure the queues to be polled. +*/ +unsigned int num_procs =1 ; + +static int +eth_dev_configure_mp (uint16_t port_id,uint16_t nb_rx_q,uint16_t nb_tx_q, +const struct rte_eth_conf *dev_conf) +{ +if (is_proc_primary ()) +return rte_eth_dev_configure (port_id, nb_rx_q, nb_tx_q, + dev_conf); +return 0 ; +} + +static int +eth_dev_start_mp (uint16_t port_id) +{ +if (is_proc_primary ()) +return rte_eth_dev_start (port_id); + +return 0 ; +} + +static int +eth_dev_stop_mp (uint16_t port_id) +{ +if (is_proc_primary ())
app/test-pmd/testpmd.c enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS; · line 565
Quoted 10 lines
app/test-pmd/testpmd.c +10
-0+return rte_eth_dev_stop (port_id); + +return 0 ; +} + +static void +mempool_free_mp (struct rte_mempool *mp) +{ +if (is_proc_primary ()) +return rte_mempool_free (mp);
As far as I remember some compilers do not like it for void. Just remove 'return'.
Quoted 15 lines
app/test-pmd/testpmd.c +11
-0+} + +static int +eth_dev_set_mtu_mp (uint16_t port_id,uint16_t mtu) +{ +if (is_proc_primary ()) +return rte_eth_dev_set_mtu (port_id, mtu); + +return 0 ; +} +/* Forward function declarations */ static void setup_attached_port (portid_t pi);static void check_all_ports_link_status (uint32_t port_mask);
app/test-pmd/testpmd.c mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, · line 1038
Quoted 6 lines
app/test-pmd/testpmd.c +3
-0@@ -977,6 +1033,11 @@ mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, mb_size =sizeof (struct rte_mbuf ) + mbuf_seg_size;mbuf_poolname_build (socket_id, pool_name,sizeof (pool_name), size_idx); +if (!is_proc_primary ()) { + rte_mp =rte_mempool_lookup (pool_name); + goto err;
It looks like error path, but it works in the case of success as well. Looks confusing.
Quoted 24 lines
app/test-pmd/testpmd.c +10
-3+ } +TESTPMD_LOG (INFO ,"create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n" , pool_name, nb_mbuf, mbuf_seg_size, socket_id);@@ -1059,9 +1120,14 @@ mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, err:if (rte_mp ==NULL ) { -rte_exit (EXIT_FAILURE , -"Creation of mbuf pool for socket %u failed: %s\n" , - socket_id,rte_strerror (rte_errno)); +if (is_proc_primary ()) +rte_exit (EXIT_FAILURE , +"Creation of mbuf pool for socket %u failed: %s\n" , + socket_id,rte_strerror (rte_errno)); +else +rte_exit (EXIT_FAILURE , +"Get mbuf pool for socket %u failed: %s\n" , + socket_id,rte_strerror (rte_errno)); }else if (verbose_level >0 ) {rte_mempool_dump (stdout, rte_mp); }
app/test-pmd/testpmd.c flush_fwd_rx_queues(void) · line 2072
Quoted 5 lines
app/test-pmd/testpmd.c +2
-0@@ -2002,6 +2068,12 @@ flush_fwd_rx_queues(void)uint64_t prev_tsc =0 , diff_tsc, cur_tsc, timer_tsc =0 ;uint64_t timer_period; +if (num_procs >1 ) { + printf("multi-process not support for flushing fwd rx "
rx -> Rx
also it is better to avoid like split.
Quoted 18 lines
app/test-pmd/testpmd.c +8
-3+ "queues, skip the below lines and return.\n"); +return ; + } +/* convert to number of cycles */ timer_period =rte_get_timer_hz ();/* 1 second timeout */ @@ -2511,21 +2583,24 @@ start_port(portid_t pid)return -1 ; }/* configure port */ - diag =rte_eth_dev_configure (pi, nb_rxq + nb_hairpinq, - nb_txq + nb_hairpinq, - &(port->dev_conf)); + diag =eth_dev_configure_mp (pi, + nb_rxq + nb_hairpinq, + nb_txq + nb_hairpinq, + &(port->dev_conf));
app/test-pmd/testpmd.c start_port(portid_t pid) · line 2595
Quoted 10 lines
app/test-pmd/testpmd.c +5
-4if (diag !=0 ) { -if (rte_atomic16_cmpset (&(port->port_status), -RTE_PORT_HANDLING ,RTE_PORT_STOPPED ) ==0 ) -printf ("Port %d can not be set back " -"to stopped\n" , pi); +if (rte_atomic16_cmpset ( + &(port->port_status), +RTE_PORT_HANDLING , +RTE_PORT_STOPPED ) ==0 ) + printf("Port %d can not be set back to stopped\n",
can not -> cannot (since you touch the line anyway)
Quoted 13 lines
app/test-pmd/testpmd.c +2
-1+ pi);printf ("Fail to configure port %d\n" , pi);/* try to reconfigure port next time */ port->need_reconfig =1 ;return -1 ; } } -if (port->need_reconfig_queues >0 ) { +if (port->need_reconfig_queues >0 &&is_proc_primary ()) { port->need_reconfig_queues =0 ;/* setup tx queues */ for (qi =0 ; qi < nb_txq; qi++) {
app/test-pmd/testpmd.c start_port(portid_t pid) · line 2626
Quoted 6 lines
app/test-pmd/testpmd.c +1
-2@@ -2548,8 +2623,8 @@ start_port(portid_t pid)if (rte_atomic16_cmpset (&(port->port_status),RTE_PORT_HANDLING ,RTE_PORT_STOPPED ) ==0 ) -printf ("Port %d can not be set back " -"to stopped\n" , pi); + printf("Port %d can not be set back to stopped\n",
can not -> cannot
Quoted 11 lines
app/test-pmd/testpmd.c +2
-1+ pi);printf ("Fail to configure port %d tx queues\n" , pi);/* try to reconfigure queues next time */ @@ -2626,16 +2701,16 @@ start_port(portid_t pid) cnt_pi++;/* start port */ - diag =rte_eth_dev_start (pi); + diag =eth_dev_start_mp (pi);if (diag <0 ) {
app/test-pmd/testpmd.c start_port(portid_t pid) · line 2712
Quoted 10 lines
app/test-pmd/testpmd.c +2
-3printf ("Fail to start port %d: %s\n" , pi,rte_strerror (-diag));/* Fail to setup rx queue, return */ if (rte_atomic16_cmpset (&(port->port_status), -RTE_PORT_HANDLING ,RTE_PORT_STOPPED ) ==0 ) -printf ("Port %d can not be set back to " -"stopped\n" , pi); +RTE_PORT_HANDLING ,RTE_PORT_STOPPED ) ==0 ) + printf("Port %d can not be set back to stopped\n",
can not -> cannot
Quoted 91 lines
app/test-pmd/testpmd.c +12
-9+ pi);continue ; }@@ -2765,7 +2840,7 @@ stop_port(portid_t pid)if (port->flow_list)port_flow_flush (pi); -if (rte_eth_dev_stop (pi) !=0 ) +if (eth_dev_stop_mp (pi) !=0 )RTE_LOG (ERR ,EAL ,"rte_eth_dev_stop failed for port %u\n" , pi);@@ -2834,8 +2909,10 @@ close_port(portid_t pid)continue ; } -port_flow_flush (pi); -rte_eth_dev_close (pi); +if (is_proc_primary ()) { +port_flow_flush (pi); +rte_eth_dev_close (pi); + } }remove_invalid_ports ();@@ -3101,7 +3178,7 @@ pmd_test_exit(void) }for (i =0 ; i <RTE_DIM (mempools) ; i++) {if (mempools[i]) -rte_mempool_free (mempools[i]); +mempool_free_mp (mempools[i]); }printf ("\nBye...\n" );@@ -3432,7 +3509,7 @@ update_jumbo_frame_offload(portid_t portid)* if unset do it here */ if ((rx_offloads &DEV_RX_OFFLOAD_JUMBO_FRAME ) ==0 ) { - ret =rte_eth_dev_set_mtu (portid, + ret =eth_dev_set_mtu_mp (portid, port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead);if (ret) printf("Failed to set MTU to %u for port %u\n",@@ -3622,6 +3699,10 @@ init_port_dcb_config(portid_t pid,int retval;uint16_t i; +if (num_procs >1 ) { +printf ("The multi-process feature doesn't support dcb.\n" ); +return -ENOTSUP ; + } rte_port = &ports[pid];memset (&port_conf,0 ,sizeof (struct rte_eth_conf ));@@ -3787,10 +3868,6 @@ main(int argc, char** argv)rte_exit (EXIT_FAILURE ,"Cannot init EAL: %s\n" ,rte_strerror (rte_errno)); -if (rte_eal_process_type () ==RTE_PROC_SECONDARY ) -rte_exit (EXIT_FAILURE , -"Secondary process type not supported.\n" ); - ret =register_eth_event_callback ();if (ret !=0 )rte_exit (EXIT_FAILURE ,"Cannot register for ethdev events" );app/test-pmd/testpmd.h +9
-0@@ -632,6 +632,15 @@ extern enum rte_eth_rx_mq_mode rx_mq_mode;extern struct rte_flow_action_conntrack conntrack_context; +extern int proc_id; +extern unsigned int num_procs; + +static inline bool +is_proc_primary (void ) +{ +return rte_eal_process_type () ==RTE_PROC_PRIMARY ; +} +static inline unsigned int lcore_num (void ) {doc/guides/rel_notes/release_21_05.rst +0
-0
doc/guides/rel_notes/release_21_05.rst New Features · line 238
Quoted 4 lines
doc/guides/rel_notes/release_21_05.rst +1
-0@@ -235,6 +235,7 @@ New Features ``port cleanup (port_id) txq (queue_id) (free_cnt)`` * Added command to show link flow control info. ``show port (port_id) flow_ctrl`` + * Added support multi-process for testpmd.
Please, rebase to 21.08.
Quoted 16 lines
doc/guides/rel_notes/release_21_05.rst +0
-0* **Updated ipsec-secgw sample application.**doc/guides/testpmd_app_ug/run_app.rst +5
-0@@ -543,3 +543,73 @@ The command line options are: bit 1 - two hairpin ports paired bit 0 - two hairpin ports loop The default value is 0. Hairpin will use single port mode and implicit Tx flow mode. + +Testpmd Multi-Process Command-line Options +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The following are the command-line options for testpmd multi-process support:
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 560
Quoted 10 lines
doc/guides/testpmd_app_ug/run_app.rst +10
-0+ +.. code-block:: console + + primary process: + sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i --rxq=4 --txq=4 \ + --num-procs=2 --proc-id=0 + + secondary process: + sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i --rxq=4 --txq=4 \ + --num-procs=2 --proc-id=1
Do we really need --rxq=4 --txq=4 in secodary processes? Can't we get it from already configured device in primary process?
Same question is applicable to --num-procs. May be testpmd can publish in shared memory? If yes, I'm OK to either do it in the patch or defer as a later improvement.
Quoted 26 lines
doc/guides/testpmd_app_ug/run_app.rst +26
-0+ +The command line options are: + +* ``--num-procs=N`` + + The number of processes which will be used. + +* ``--proc-id=id`` + + The id of the current process (id < num-procs). id should be different in primary + process and secondary process, which starts from '0'. + +Calculation rule for queue: +All queues are allocated to different processes based on ``proc_num`` and ``proc_id``. +Calculation rule for the testpmd to allocate queues to each process: +start(queue start id) = proc_id * nb_q / num_procs; +end(queue end id) = start + nb_q / num_procs; + +For example, if testpmd supports 4 Tx and Rx queues +the 0~1 for primary process +the 2~3 for secondary process + +The number of rings should be a multiple of the number of processes. If not, +redundant queues will exist after queues are allocated to processes. After RSS is +enabled, packet loss occurs when traffic is sent to all processes at the same time. +Some traffic enters redundant queues and cannot be forwarded.
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 596
Quoted 10 lines
doc/guides/testpmd_app_ug/run_app.rst +10
-0+ +All the dev ops is supported in primary process. While secondary process is not permitted +to allocate or release shared memory, so some ops are not supported as follows:: +``dev_configure`` +``dev_start`` +``dev_stop`` +``rx_queue_setup`` +``tx_queue_setup`` +``rx_queue_release`` +``tx_queue_release``
@Thomas, @Ferrh, shouldn't it be handled on ethdev level as well if it is really that strict.
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 605
Quoted 9 lines
doc/guides/testpmd_app_ug/run_app.rst +9
-0+ +So, any command from testpmd which calls those APIs will not be supported in secondary +process, like:: +``port config all rxq|txq|rxd|txd <value>`` +``port config <port_id> rx_offload xxx on/off `` +``port config <port_id> tx_offload xxx on/off`` +etc. + +Flow API is supported, it applies only on its own process on SW side, but all on HW side.
Sorry, I don't understand it.
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 615
Quoted 10 lines
doc/guides/testpmd_app_ug/run_app.rst +10
-0+ +Stats is supported, stats will not change when one quit and start, as they share the same +buffer to store the stats. Flow rules are maintained in process level: primary and secondary +has its own flow list (but one flow list in HW). The two can see all the queues, so setting +the flow rules for the other is OK. But in the testpmd primary process receiving or transmitting +packets from the queue allocated for secondary process is not permitted, and same for +secondary process. + +RSS is supported, primary process and secondary process has separate queues to use, RSS +will work in their own queues whether primary or secondary process.
For me it sounds like secondary process has own RSS configuration. If so, it is false. I guess I simply misunderstand above paragraph.
08/06/2021 10:42, Andrew Rybchenko:
Quoted 3 lines · Min Hu (Connor)
Quoted 2 lines
doc/guides/testpmd_app_ug/run_app.rst +0
-0
[...]
Quoted 4 lines
Quoted 4 lines
doc/guides/testpmd_app_ug/run_app.rst +4
-0+All the dev ops is supported in primary process. While secondary process is not permitted +to allocate or release shared memory, so some ops are not supported as follows:: +``dev_configure`` +``dev_start``
Quoted 8 lines
Quoted 5 lines
+``dev_stop``
+``rx_queue_setup``
+``tx_queue_setup``
+``rx_queue_release``
+``tx_queue_release``@Thomas, @Ferrh, shouldn't it be handled on ethdev level as well if it is really that strict.
Yes it should be documented at ethdev level, not testpmd. I think it was kept fuzzy for too long.
Quoted 18 lines · Thomas Monjalon
08/06/2021 10:42, Andrew Rybchenko:
Quoted 3 lines · Min Hu (Connor)
Quoted 2 lines
doc/guides/testpmd_app_ug/run_app.rst +0
-0[...]
Quoted 4 lines
Quoted 4 lines
doc/guides/testpmd_app_ug/run_app.rst +4
-0+All the dev ops is supported in primary process. While secondary process is not permitted +to allocate or release shared memory, so some ops are not supported as follows:: +``dev_configure`` +``dev_start``Quoted 8 lines
Quoted 5 lines
+``dev_stop``
+``rx_queue_setup``
+``tx_queue_setup``
+``rx_queue_release``
+``tx_queue_release``@Thomas, @Ferrh, shouldn't it be handled on ethdev level as well if it is really that strict.
Quoted 2 lines
Yes it should be documented at ethdev level, not testpmd. I think it was kept fuzzy for too long.
To document is good, but I'm talking about more -
add checks in corresponding API functions and
return error.
08/06/2021 12:39, Andrew Rybchenko:
Quoted 19 lines · Thomas Monjalon
Quoted 18 lines
08/06/2021 10:42, Andrew Rybchenko:
Quoted 3 lines · Min Hu (Connor)
Quoted 2 lines
doc/guides/testpmd_app_ug/run_app.rst +0
-0[...]
Quoted 12 lines
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 596
Quoted 9 lines
doc/guides/testpmd_app_ug/run_app.rst +9
-0+All the dev ops is supported in primary process. While secondary process is not permitted +to allocate or release shared memory, so some ops are not supported as follows:: +``dev_configure`` +``dev_start`` +``dev_stop`` +``rx_queue_setup`` +``tx_queue_setup`` +``rx_queue_release`` +``tx_queue_release``@Thomas, @Ferrh, shouldn't it be handled on ethdev level as well if it is really that strict.
Quoted 6 lines
Quoted 2 lines
Yes it should be documented at ethdev level, not testpmd. I think it was kept fuzzy for too long.
To document is good, but I'm talking about more -
add checks in corresponding API functions and
return error.
Yes I am OK with adding checks of the running process.
Quoted 19 lines · Andrew Rybchenko
Quoted 18 lines · Thomas Monjalon
08/06/2021 10:42, Andrew Rybchenko:
Quoted 3 lines · Min Hu (Connor)
Quoted 2 lines
doc/guides/testpmd_app_ug/run_app.rst +0
-0[...]
Quoted 12 lines
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 596
Quoted 9 lines
doc/guides/testpmd_app_ug/run_app.rst +9
-0+All the dev ops is supported in primary process. While secondary process is not permitted +to allocate or release shared memory, so some ops are not supported as follows:: +``dev_configure`` +``dev_start`` +``dev_stop`` +``rx_queue_setup`` +``tx_queue_setup`` +``rx_queue_release`` +``tx_queue_release``@Thomas, @Ferrh, shouldn't it be handled on ethdev level as well if it is really that strict.
Quoted 7 lines
Quoted 2 lines
Yes it should be documented at ethdev level, not testpmd. I think it was kept fuzzy for too long.
To document is good, but I'm talking about more -
add checks in corresponding API functions and
return error.
+Anatoly as multi process maintainer.
There are already some PMDs have these checks in the PMD level.
But current approach is providing more flexibility, if application takes care of the synchronization, is there any reason some of above APIs can't be used by the secondary process?
I am not sure if this flexibility is used or needed, if not needed +1 have the checks in the ethdev layer and remove the PMD ones.
Hi, Andrew,
see replies below, and others without no reply will be fixed in v14.
Quoted 116 lines · Andrew Rybchenko
@Thomas, @Ferruh, please, see question below.
Quoted 70 lines · Min Hu (Connor)
This patch adds multi-process support for testpmd. The test cmd example as follows:
the primary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=0the secondary cmd:
./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i \
--rxq=4 --txq=4 --num-procs=2 --proc-id=1Signed-off-by Min Hu (Connor) <[email protected]>Signed-off-by Lijun Ou <[email protected]>Acked-by Xiaoyun Li <[email protected]>Acked-by Ajit Khaparde <[email protected]>Reviewed-by Ferruh Yigit <[email protected]>---
v13:
* Modified the doc syntax.v12:
* Updated doc info.v11:
* Fixed some minor syntax.v10:
* Hid process type checks behind new functions.
* Added comments.v9:
* Updated release notes and rst doc.
* Deleted deprecated codes.
* move macro and variable.v8:
* Added warning info about queue numbers and process numbers.v7:
* Fixed compiling error for unexpected unindent.v6:
* Add rte flow description for multiple process.v5:
* Fixedrun_app.rstfor multiple process description.
* Fix compiling error.v4:
* Fixed minimum vlaue of Rxq or Txq in doc.v3:
* Fixed compiling error using gcc10.0.v2:
* Added document for this patch.--- app/test-pmd/cmdline.c | 6 ++ app/test-pmd/config.c | 21 +++++- app/test-pmd/parameters.c | 11 +++ app/test-pmd/testpmd.c | 129 ++++++++++++++++++++++++++------- app/test-pmd/testpmd.h | 9 +++ doc/guides/rel_notes/release_21_05.rst | 1 + doc/guides/testpmd_app_ug/run_app.rst | 70 ++++++++++++++++++ 7 files changed, 220 insertions(+), 27 deletions(-)app/test-pmd/cmdline.c +0
-0app/test-pmd/cmdline.c cmd_set_flush_rx_parsed(void *parsed_result, · line 5455
Quoted 6 lines
app/test-pmd/cmdline.c +3
-0@@ -5450,6 +5450,12 @@ cmd_set_flush_rx_parsed(void *parsed_result, __rte_unused void *data) {struct cmd_set_flush_rx *res = parsed_result; + +if (num_procs >1 && (strcmp (res->mode,"on" ) ==0 )) { +printf ("multi-process doesn't support to flush rx queues.\n" );rx -> Rx
Quoted 20 lines
app/test-pmd/cmdline.c +3
-0+return ; + } + no_flush_rx = (uint8_t )((strcmp (res->mode,"on" ) ==0 ) ?0 :1 ); }app/test-pmd/config.c +2
-0@@ -2971,6 +2971,8 @@ rss_fwd_config_setup(void)queueid_t rxq;queueid_t nb_q;streamid_t sm_id; +int start; +int end; nb_q = nb_rxq;if (nb_q > nb_txq)app/test-pmd/config.c rss_fwd_config_setup(void) · line 2994
Quoted 6 lines
app/test-pmd/config.c +2
-1@@ -2988,7 +2990,22 @@ rss_fwd_config_setup(void)init_fwd_streams ();setup_fwd_config_of_each_lcore (&cur_fwd_config); - rxp =0 ; rxq =0 ; + +if (proc_id >0 && nb_q % num_procs)Please, compare result with 0 explicitly.
app/test-pmd/config.c rss_fwd_config_setup(void) · line 2996
Quoted 2 lines
app/test-pmd/config.c +2
-0+printf ("Warning! queue numbers should be multiple of " +"processes, or packet loss will happen.\n" );Do not split format string across multiple lines.
Quoted 5 lines
Frankly speaking I don't undertand why. Why is it impossible to serve 2 queues in the first process and 1 queue in the second process if 3 queues and 2 processes are configured.
I think RSS redirection table can perfectly do it.
Well, currently, my patch is one design implementation. I think this can be done for later improvemnet.
Quoted 389 lines
Quoted 48 lines
app/test-pmd/config.c +14
-0+ +/** +* In multi-process, All queues are allocated to different +* processes based on num_procs and proc_id. For example: +* if supports 4 queues(nb_q), 2 processes(num_procs), +* the 0~1 queue for primary process. +* the 2~3 queue for secondary process. +*/ + start = proc_id * nb_q / num_procs; + end = start + nb_q / num_procs; + rxp =0 ; + rxq = start;for (sm_id =0 ; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {struct fwd_stream *fs;@@ -3005,6 +3022,8 @@ rss_fwd_config_setup(void)continue ; rxp =0 ; rxq++; +if (rxq >= end) + rxq = start; } }app/test-pmd/parameters.c +5
-0@@ -508,6 +508,9 @@ parse_link_speed(int n)void launch_args_parse (int argc,char ** argv) { +#define PARAM_PROC_ID "proc-id" +#define PARAM_NUM_PROCS "num-procs" +int n, opt;char **argvopt;int opt_idx;@@ -625,6 +628,8 @@ launch_args_parse(int argc, char** argv) { "rx-mq-mode", 1, 0, 0 }, { "record-core-cycles", 0, 0, 0 }, { "record-burst-stats", 0, 0, 0 }, + { PARAM_NUM_PROCS, 1, 0, 0 }, + { PARAM_PROC_ID, 1, 0, 0 }, { 0, 0, 0, 0 }, };app/test-pmd/parameters.c launch_args_parse(int argc, char** argv) · line 1400
Quoted 5 lines
app/test-pmd/parameters.c +2
-0@@ -1391,6 +1396,12 @@ launch_args_parse(int argc, char** argv) record_core_cycles =1 ;if (!strcmp (lgopts[opt_idx].name,"record-burst-stats" )) record_burst_stats =1 ; +if (strncmp (lgopts[opt_idx].name, +PARAM_NUM_PROCS ,9 ) ==0 )I strongly dislike 9 here and 7 below. Why is
strncmp()used here, but juststrcmp()is used for all other options. It makes the code inconsistent.Quoted 12 lines
app/test-pmd/parameters.c +4
-0+ num_procs =atoi (optarg); +if (strncmp (lgopts[opt_idx].name, +PARAM_PROC_ID ,7 ) ==0 ) + proc_id =atoi (optarg);break ;case 'h' :usage (argv[0 ]);app/test-pmd/testpmd.c +0
-0app/test-pmd/testpmd.c enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS; · line 522
Quoted 5 lines
app/test-pmd/testpmd.c +2
-0@@ -518,6 +518,62 @@ enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS;*/ uint32_t eth_link_speed; +/* +* Id of the current process in multi-process, used to Id -> ID in accordance with
devtools/words-case.txtQuoted 33 lines
app/test-pmd/testpmd.c +33
-0+* configure the queues to be polled. +*/ +int proc_id; + +/* +* Number of processes in multi-process, used to +* configure the queues to be polled. +*/ +unsigned int num_procs =1 ; + +static int +eth_dev_configure_mp (uint16_t port_id,uint16_t nb_rx_q,uint16_t nb_tx_q, +const struct rte_eth_conf *dev_conf) +{ +if (is_proc_primary ()) +return rte_eth_dev_configure (port_id, nb_rx_q, nb_tx_q, + dev_conf); +return 0 ; +} + +static int +eth_dev_start_mp (uint16_t port_id) +{ +if (is_proc_primary ()) +return rte_eth_dev_start (port_id); + +return 0 ; +} + +static int +eth_dev_stop_mp (uint16_t port_id) +{ +if (is_proc_primary ())app/test-pmd/testpmd.c enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS; · line 565
Quoted 10 lines
app/test-pmd/testpmd.c +10
-0+return rte_eth_dev_stop (port_id); + +return 0 ; +} + +static void +mempool_free_mp (struct rte_mempool *mp) +{ +if (is_proc_primary ()) +return rte_mempool_free (mp);As far as I remember some compilers do not like it for void. Just remove 'return'.
Quoted 15 lines
app/test-pmd/testpmd.c +11
-0+} + +static int +eth_dev_set_mtu_mp (uint16_t port_id,uint16_t mtu) +{ +if (is_proc_primary ()) +return rte_eth_dev_set_mtu (port_id, mtu); + +return 0 ; +} +/* Forward function declarations */ static void setup_attached_port (portid_t pi);static void check_all_ports_link_status (uint32_t port_mask);app/test-pmd/testpmd.c mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, · line 1038
Quoted 6 lines
app/test-pmd/testpmd.c +3
-0@@ -977,6 +1033,11 @@ mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, mb_size =sizeof (struct rte_mbuf ) + mbuf_seg_size;mbuf_poolname_build (socket_id, pool_name,sizeof (pool_name), size_idx); +if (!is_proc_primary ()) { + rte_mp =rte_mempool_lookup (pool_name); + goto err;It looks like error path, but it works in the case of success as well. Looks confusing.
Quoted 24 lines
app/test-pmd/testpmd.c +10
-3+ } +TESTPMD_LOG (INFO ,"create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n" , pool_name, nb_mbuf, mbuf_seg_size, socket_id);@@ -1059,9 +1120,14 @@ mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, err:if (rte_mp ==NULL ) { -rte_exit (EXIT_FAILURE , -"Creation of mbuf pool for socket %u failed: %s\n" , - socket_id,rte_strerror (rte_errno)); +if (is_proc_primary ()) +rte_exit (EXIT_FAILURE , +"Creation of mbuf pool for socket %u failed: %s\n" , + socket_id,rte_strerror (rte_errno)); +else +rte_exit (EXIT_FAILURE , +"Get mbuf pool for socket %u failed: %s\n" , + socket_id,rte_strerror (rte_errno)); }else if (verbose_level >0 ) {rte_mempool_dump (stdout, rte_mp); }app/test-pmd/testpmd.c flush_fwd_rx_queues(void) · line 2072
Quoted 5 lines
app/test-pmd/testpmd.c +2
-0@@ -2002,6 +2068,12 @@ flush_fwd_rx_queues(void)uint64_t prev_tsc =0 , diff_tsc, cur_tsc, timer_tsc =0 ;uint64_t timer_period; +if (num_procs >1 ) { + printf("multi-process not support for flushing fwd rx "rx -> Rx
also it is better to avoid like split.Quoted 18 lines
app/test-pmd/testpmd.c +8
-3+ "queues, skip the below lines and return.\n"); +return ; + } +/* convert to number of cycles */ timer_period =rte_get_timer_hz ();/* 1 second timeout */ @@ -2511,21 +2583,24 @@ start_port(portid_t pid)return -1 ; }/* configure port */ - diag =rte_eth_dev_configure (pi, nb_rxq + nb_hairpinq, - nb_txq + nb_hairpinq, - &(port->dev_conf)); + diag =eth_dev_configure_mp (pi, + nb_rxq + nb_hairpinq, + nb_txq + nb_hairpinq, + &(port->dev_conf));app/test-pmd/testpmd.c start_port(portid_t pid) · line 2595
Quoted 10 lines
app/test-pmd/testpmd.c +5
-4if (diag !=0 ) { -if (rte_atomic16_cmpset (&(port->port_status), -RTE_PORT_HANDLING ,RTE_PORT_STOPPED ) ==0 ) -printf ("Port %d can not be set back " -"to stopped\n" , pi); +if (rte_atomic16_cmpset ( + &(port->port_status), +RTE_PORT_HANDLING , +RTE_PORT_STOPPED ) ==0 ) + printf("Port %d can not be set back to stopped\n",can not -> cannot (since you touch the line anyway)
Quoted 13 lines
app/test-pmd/testpmd.c +2
-1+ pi);printf ("Fail to configure port %d\n" , pi);/* try to reconfigure port next time */ port->need_reconfig =1 ;return -1 ; } } -if (port->need_reconfig_queues >0 ) { +if (port->need_reconfig_queues >0 &&is_proc_primary ()) { port->need_reconfig_queues =0 ;/* setup tx queues */ for (qi =0 ; qi < nb_txq; qi++) {app/test-pmd/testpmd.c start_port(portid_t pid) · line 2626
Quoted 6 lines
app/test-pmd/testpmd.c +1
-2@@ -2548,8 +2623,8 @@ start_port(portid_t pid)if (rte_atomic16_cmpset (&(port->port_status),RTE_PORT_HANDLING ,RTE_PORT_STOPPED ) ==0 ) -printf ("Port %d can not be set back " -"to stopped\n" , pi); + printf("Port %d can not be set back to stopped\n",can not -> cannot
Quoted 11 lines
app/test-pmd/testpmd.c +2
-1+ pi);printf ("Fail to configure port %d tx queues\n" , pi);/* try to reconfigure queues next time */ @@ -2626,16 +2701,16 @@ start_port(portid_t pid) cnt_pi++;/* start port */ - diag =rte_eth_dev_start (pi); + diag =eth_dev_start_mp (pi);if (diag <0 ) {app/test-pmd/testpmd.c start_port(portid_t pid) · line 2712
Quoted 10 lines
app/test-pmd/testpmd.c +2
-3printf ("Fail to start port %d: %s\n" , pi,rte_strerror (-diag));/* Fail to setup rx queue, return */ if (rte_atomic16_cmpset (&(port->port_status), -RTE_PORT_HANDLING ,RTE_PORT_STOPPED ) ==0 ) -printf ("Port %d can not be set back to " -"stopped\n" , pi); +RTE_PORT_HANDLING ,RTE_PORT_STOPPED ) ==0 ) + printf("Port %d can not be set back to stopped\n",can not -> cannot
Quoted 91 lines
app/test-pmd/testpmd.c +12
-9+ pi);continue ; }@@ -2765,7 +2840,7 @@ stop_port(portid_t pid)if (port->flow_list)port_flow_flush (pi); -if (rte_eth_dev_stop (pi) !=0 ) +if (eth_dev_stop_mp (pi) !=0 )RTE_LOG (ERR ,EAL ,"rte_eth_dev_stop failed for port %u\n" , pi);@@ -2834,8 +2909,10 @@ close_port(portid_t pid)continue ; } -port_flow_flush (pi); -rte_eth_dev_close (pi); +if (is_proc_primary ()) { +port_flow_flush (pi); +rte_eth_dev_close (pi); + } }remove_invalid_ports ();@@ -3101,7 +3178,7 @@ pmd_test_exit(void) }for (i =0 ; i <RTE_DIM (mempools) ; i++) {if (mempools[i]) -rte_mempool_free (mempools[i]); +mempool_free_mp (mempools[i]); }printf ("\nBye...\n" );@@ -3432,7 +3509,7 @@ update_jumbo_frame_offload(portid_t portid)* if unset do it here */ if ((rx_offloads &DEV_RX_OFFLOAD_JUMBO_FRAME ) ==0 ) { - ret =rte_eth_dev_set_mtu (portid, + ret =eth_dev_set_mtu_mp (portid, port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead);if (ret) printf("Failed to set MTU to %u for port %u\n",@@ -3622,6 +3699,10 @@ init_port_dcb_config(portid_t pid,int retval;uint16_t i; +if (num_procs >1 ) { +printf ("The multi-process feature doesn't support dcb.\n" ); +return -ENOTSUP ; + } rte_port = &ports[pid];memset (&port_conf,0 ,sizeof (struct rte_eth_conf ));@@ -3787,10 +3868,6 @@ main(int argc, char** argv)rte_exit (EXIT_FAILURE ,"Cannot init EAL: %s\n" ,rte_strerror (rte_errno)); -if (rte_eal_process_type () ==RTE_PROC_SECONDARY ) -rte_exit (EXIT_FAILURE , -"Secondary process type not supported.\n" ); - ret =register_eth_event_callback ();if (ret !=0 )rte_exit (EXIT_FAILURE ,"Cannot register for ethdev events" );app/test-pmd/testpmd.h +9
-0@@ -632,6 +632,15 @@ extern enum rte_eth_rx_mq_mode rx_mq_mode;extern struct rte_flow_action_conntrack conntrack_context; +extern int proc_id; +extern unsigned int num_procs; + +static inline bool +is_proc_primary (void ) +{ +return rte_eal_process_type () ==RTE_PROC_PRIMARY ; +} +static inline unsigned int lcore_num (void ) {doc/guides/rel_notes/release_21_05.rst +0
-0doc/guides/rel_notes/release_21_05.rst New Features · line 238
Quoted 4 lines
doc/guides/rel_notes/release_21_05.rst +1
-0@@ -235,6 +235,7 @@ New Features ``port cleanup (port_id) txq (queue_id) (free_cnt)`` * Added command to show link flow control info. ``show port (port_id) flow_ctrl`` + * Added support multi-process for testpmd.Please, rebase to 21.08.
Quoted 16 lines
doc/guides/rel_notes/release_21_05.rst +0
-0* **Updated ipsec-secgw sample application.**doc/guides/testpmd_app_ug/run_app.rst +5
-0@@ -543,3 +543,73 @@ The command line options are: bit 1 - two hairpin ports paired bit 0 - two hairpin ports loop The default value is 0. Hairpin will use single port mode and implicit Tx flow mode. + +Testpmd Multi-Process Command-line Options +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The following are the command-line options for testpmd multi-process support:doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 560
Quoted 10 lines
doc/guides/testpmd_app_ug/run_app.rst +10
-0+ +.. code-block:: console + + primary process: + sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 0-1 -- -i --rxq=4 --txq=4 \ + --num-procs=2 --proc-id=0 + + secondary process: + sudo ./dpdk-testpmd -a xxx --proc-type=auto -l 2-3 -- -i --rxq=4 --txq=4 \ + --num-procs=2 --proc-id=1Do we really need --rxq=4 --txq=4 in secodary processes? Can't we get it from already configured device in primary process?
Quoted 4 lines
Same question is applicable to --num-procs. May be testpmd can publish in shared memory? If yes, I'm OK to either do it in the patch or defer as a later improvement.
Yes, agree with you, it can be done for later improvement.
Quoted 40 lines
Quoted 26 lines
doc/guides/testpmd_app_ug/run_app.rst +26
-0+ +The command line options are: + +* ``--num-procs=N`` + + The number of processes which will be used. + +* ``--proc-id=id`` + + The id of the current process (id < num-procs). id should be different in primary + process and secondary process, which starts from '0'. + +Calculation rule for queue: +All queues are allocated to different processes based on ``proc_num`` and ``proc_id``. +Calculation rule for the testpmd to allocate queues to each process: +start(queue start id) = proc_id * nb_q / num_procs; +end(queue end id) = start + nb_q / num_procs; + +For example, if testpmd supports 4 Tx and Rx queues +the 0~1 for primary process +the 2~3 for secondary process + +The number of rings should be a multiple of the number of processes. If not, +redundant queues will exist after queues are allocated to processes. After RSS is +enabled, packet loss occurs when traffic is sent to all processes at the same time. +Some traffic enters redundant queues and cannot be forwarded.doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 596
Quoted 10 lines
doc/guides/testpmd_app_ug/run_app.rst +10
-0+ +All the dev ops is supported in primary process. While secondary process is not permitted +to allocate or release shared memory, so some ops are not supported as follows:: +``dev_configure`` +``dev_start`` +``dev_stop`` +``rx_queue_setup`` +``tx_queue_setup`` +``rx_queue_release`` +``tx_queue_release``@Thomas, @Ferrh, shouldn't it be handled on ethdev level as
well if it is really that strict.Yes, API modification may be handled as another patch for later improvement.
Quoted 11 lines
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 605
Quoted 9 lines
doc/guides/testpmd_app_ug/run_app.rst +9
-0+ +So, any command from testpmd which calls those APIs will not be supported in secondary +process, like:: +``port config all rxq|txq|rxd|txd <value>`` +``port config <port_id> rx_offload xxx on/off `` +``port config <port_id> tx_offload xxx on/off`` +etc. + +Flow API is supported, it applies only on its own process on SW side, but all on HW side.Sorry, I don't understand it.
This may be confusing, I will delete the lines.
What I mean is Flow API is supported,that is it.
Quoted 15 lines
doc/guides/testpmd_app_ug/run_app.rst The command line options are: · line 615
Quoted 11 lines
doc/guides/testpmd_app_ug/run_app.rst +10
-0+ +Stats is supported, stats will not change when one quit and start, as they share the same +buffer to store the stats. Flow rules are maintained in process level: primary and secondary +has its own flow list (but one flow list in HW). The two can see all the queues, so setting +the flow rules for the other is OK. But in the testpmd primary process receiving or transmitting +packets from the queue allocated for secondary process is not permitted, and same for +secondary process. + +RSS is supported, primary process and secondary process has separate queues to use, RSS +will work in their own queues whether primary or secondary process.For me it sounds like secondary process has own RSS configuration. If so, it is false. I guess I simply misunderstand above paragraph.
This may be confusing, I will delete the lines.
What I mean is that primary process and secondary process has its own queues, but both the queues are processed by RSS.
Quoted 2 lines
.