app/testpmd: skip stopped queues when forwarding
PATCH 1/1 app/testpmd: skip stopped queues when forwarding
After "port <port_id> rxq|txq <queue_id> stop" the stopped queue was used in forwarding nonetheless, which may cause undefined behavior in the PMD.
Record the configured queue state
and account for it when launching forwarding as follows:
+--------+---------+-----------------+---------------+ |RxQ |TxQ |Configured mode |Launch routine | +--------+---------+-----------------+---------------+ |stopped |stopped |* |- | |stopped |started |txonly |(configured) | |stopped |started |* |- | |started |stopped |* |rxonly | |started |started |* |(configured) | +--------+---------+-----------------+---------------+
Display stopped queues on "show port config rxtx".
--- app/test-pmd/cmdline.c | 8 ++++++++ app/test-pmd/config.c | 6 ++++++ app/test-pmd/testpmd.c | 18 ++++++++++++++++-- app/test-pmd/testpmd.h | 10 ++++++++++ 4 files changed, 40 insertions(+), 2 deletions(-)
app/test-pmd/cmdline.c +8 -0
@@ -2702,6 +2702,14 @@ cmd_config_rxtx_queue_parsed(void *parsed_result,if (ret == -ENOTSUP )fprintf (stderr,"Function not supported in PMD\n" ); +if (ret ==0 ) { +struct rte_port *port; +struct queue_state *states; + + port = &ports[res->portid]; + states = isrx ? port->rxq_state : port->txq_state; + states[res->qid].stopped = !isstart; + } } cmdline_parse_token_string_t cmd_config_rxtx_queue_port =
app/test-pmd/config.c +6 -0
@@ -2817,6 +2817,9 @@ rxtx_config_display(void) rx_conf->share_qid);printf ("\n" ); } +for (qid =0 ; qid < nb_rxq; qid++) +if (ports[pid].rxq_state[qid].stopped) +printf (" RX queue %d is stopped\n" , qid);/* per tx queue config only for first queue to be less verbose */ for (qid =0 ; qid <1 ; qid++) {
@@ -2850,6 +2853,9 @@ rxtx_config_display(void)printf (" TX offloads=0x%" PRIx64" - TX RS bit threshold=%d\n" , offloads_tmp, tx_rs_thresh_tmp); } +for (qid =0 ; qid < nb_txq; qid++) +if (ports[pid].txq_state[qid].stopped) +printf (" TX queue %d is stopped\n" , qid); } }
app/test-pmd/testpmd.c +16 -2
@@ -2152,6 +2152,8 @@ flush_fwd_rx_queues(void)for (rxp =0 ; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {for (rxq =0 ; rxq < nb_rxq; rxq++) { port_id = fwd_ports_ids[rxp]; +if (ports[port_id].rxq_state[rxq].stopped) +continue ;/** * testpmd can stuck in the below do while loop * if rte_eth_rx_burst() always returns nonzero
@@ -2223,8 +2225,20 @@ run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)static int start_pkt_forward_on_core (void *fwd_arg) { -run_pkt_fwd_on_lcore ((struct fwd_lcore *) fwd_arg, - cur_fwd_config.fwd_eng->packet_fwd); +struct fwd_lcore *fc = fwd_arg; +struct fwd_stream *fsm = fwd_streams[fc->stream_idx]; +struct queue_state *rxq = &ports[fsm->rx_port].rxq_state[fsm->rx_queue]; +struct queue_state *txq = &ports[fsm->tx_port].txq_state[fsm->tx_queue]; +struct fwd_engine *fwd_engine = cur_fwd_config.fwd_eng; +packet_fwd_t packet_fwd; + +/* Check if there will ever be any packets to send. */ +if (rxq->stopped && (txq->stopped || fwd_engine != &tx_only_engine)) +return 0 ; +/* Force rxonly mode if RxQ is started, but TxQ is stopped. */ + packet_fwd = !rxq->stopped && txq->stopped ? rx_only_engine.packet_fwd + : fwd_engine->packet_fwd; +run_pkt_fwd_on_lcore (fc, packet_fwd);return 0 ; }
app/test-pmd/testpmd.h +10 -0
@@ -216,6 +216,12 @@ struct xstat_display_info {bool allocated; }; +/** Application state of a queue. */ +struct queue_state { +/** The queue is stopped and should not be used. */ +bool stopped; +}; +/** * The data structure associated with each port. */
@@ -256,6 +262,10 @@ struct rte_port {uint64_t mbuf_dynf;const struct rte_eth_rxtx_callback *tx_set_dynf_cb[RTE_MAX_QUEUES_PER_PORT +1 ];struct xstat_display_info xstats_info; +/** Per-Rx-queue state. */ +struct queue_state rxq_state[RTE_MAX_QUEUES_PER_PORT ]; +/** Per-Tx-queue state. */ +struct queue_state txq_state[RTE_MAX_QUEUES_PER_PORT ]; };/**
Signature · 1 line
2.25.1
Quoted 2 lines
From: Dmitry Kozlyuk <[email protected]>
Sent: Thursday, January 13, 2022 12:21 PM
Hi Aman, Xiaoyun, Yuying,
Any comments on the proposed behavior or the code?
Hi Dmitry,
Thanks for the patch.
Quoted 95 lines · Dmitry Kozlyuk
After "port <
port_id> rxq|txq <queue_id> stop" the stopped queue was used in forwarding nonetheless, which may cause undefined behavior in the PMD.Record the configured queue state
and account for it when launching forwarding as follows:+--------+---------+-----------------+---------------+ |RxQ |TxQ |Configured mode |Launch routine | +--------+---------+-----------------+---------------+ |stopped |stopped |* |- | |stopped |started |txonly |(configured) | |stopped |started |* |- | |started |stopped |* |rxonly | |started |started |* |(configured) | +--------+---------+-----------------+---------------+Display stopped queues on "show port config rxtx".
Signed-off-by Dmitry Kozlyuk <[email protected]>Reviewed-by Raslan Darawsheh <[email protected]>--- app/test-pmd/cmdline.c | 8 ++++++++ app/test-pmd/config.c | 6 ++++++ app/test-pmd/testpmd.c | 18 ++++++++++++++++-- app/test-pmd/testpmd.h | 10 ++++++++++ 4 files changed, 40 insertions(+), 2 deletions(-)app/test-pmd/cmdline.c +8
-0@@ -2702,6 +2702,14 @@ cmd_config_rxtx_queue_parsed(void *parsed_result,if (ret == -ENOTSUP )fprintf (stderr,"Function not supported in PMD\n" ); +if (ret ==0 ) { +struct rte_port *port; +struct queue_state *states; + + port = &ports[res->portid]; + states = isrx ? port->rxq_state : port->txq_state; + states[res->qid].stopped = !isstart; + } } cmdline_parse_token_string_t cmd_config_rxtx_queue_port =app/test-pmd/config.c +6
-0@@ -2817,6 +2817,9 @@ rxtx_config_display(void) rx_conf->share_qid);printf ("\n" ); } +for (qid =0 ; qid < nb_rxq; qid++) +if (ports[pid].rxq_state[qid].stopped) +printf (" RX queue %d is stopped\n" , qid);/* per tx queue config only for first queue to be less verbose */ for (qid =0 ; qid <1 ; qid++) {@@ -2850,6 +2853,9 @@ rxtx_config_display(void)printf (" TX offloads=0x%" PRIx64" - TX RS bit threshold=%d\n" , offloads_tmp, tx_rs_thresh_tmp); } +for (qid =0 ; qid < nb_txq; qid++) +if (ports[pid].txq_state[qid].stopped) +printf (" TX queue %d is stopped\n" , qid); } }app/test-pmd/testpmd.c +5
-2@@ -2152,6 +2152,8 @@ flush_fwd_rx_queues(void)for (rxp =0 ; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {for (rxq =0 ; rxq < nb_rxq; rxq++) { port_id = fwd_ports_ids[rxp]; +if (ports[port_id].rxq_state[rxq].stopped) +continue ;/** * testpmd can stuck in the below do while loop * if rte_eth_rx_burst() always returns nonzero @@ -2223,8 +2225,20 @@ run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)static int start_pkt_forward_on_core (void *fwd_arg) { -run_pkt_fwd_on_lcore ((struct fwd_lcore *) fwd_arg, - cur_fwd_config.fwd_eng->packet_fwd); +struct fwd_lcore *fc = fwd_arg; +struct fwd_stream *fsm = fwd_streams[fc->stream_idx]; +struct queue_state *rxq = &ports[fsm->rx_port].rxq_state[fsm->rx_queue];
app/test-pmd/testpmd.c run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) · line 2240
Quoted 10 lines
app/test-pmd/testpmd.c +10
-0+struct queue_state *txq = &ports[fsm->tx_port].txq_state[fsm->tx_queue]; +struct fwd_engine *fwd_engine = cur_fwd_config.fwd_eng; +packet_fwd_t packet_fwd; + +/* Check if there will ever be any packets to send. */ +if (rxq->stopped && (txq->stopped || fwd_engine != &tx_only_engine)) +return 0 ; +/* Force rxonly mode if RxQ is started, but TxQ is stopped. */ + packet_fwd = !rxq->stopped && txq->stopped ? rx_only_engine.packet_fwd + : fwd_engine->packet_fwd;
Should we have a print here for user info, that mode has been changed or ignored.
Quoted 22 lines
app/test-pmd/testpmd.c +1
-0+run_pkt_fwd_on_lcore (fc, packet_fwd);return 0 ; }app/test-pmd/testpmd.h +6
-0@@ -216,6 +216,12 @@ struct xstat_display_info {bool allocated; }; +/** Application state of a queue. */ +struct queue_state { +/** The queue is stopped and should not be used. */ +bool stopped; +}; +/** * The data structure associated with each port. */
app/test-pmd/testpmd.h struct rte_port { · line 268
Quoted 7 lines
app/test-pmd/testpmd.h +4
-0@@ -256,6 +262,10 @@ struct rte_port {uint64_t mbuf_dynf;const struct rte_eth_rxtx_callback *tx_set_dynf_cb[RTE_MAX_QUEUES_PER_PORT +1 ];struct xstat_display_info xstats_info; +/** Per-Rx-queue state. */ +struct queue_state rxq_state[RTE_MAX_QUEUES_PER_PORT ]; +/** Per-Tx-queue state. */ +struct queue_state txq_state[RTE_MAX_QUEUES_PER_PORT ];
Can we think of adding rxq_state/txq_state as part of existing structures under rte_port->rte_eth_rxconf/rte_eth_txconf. And if it helps, rather than bool can we use u8 with eth_dev defines-
#define RTE_ETH_QUEUE_STATE_STOPPED 0 /**< Queue stopped. */
#define RTE_ETH_QUEUE_STATE_STARTED 1 /**< Queue started. */
Quoted 3 lines
app/test-pmd/testpmd.h +0
-0};/**
Hi Dmitry,
Quoted 124 lines
-----Original Message-----
From: Singh, Aman Deep <[email protected]>
Sent: Thursday, February 3, 2022 9:52 PM
To: Dmitry Kozlyuk <[email protected]>; [email protected]
Cc: Li, Xiaoyun <[email protected]>; Zhang, Yuying <[email protected]>; [email protected]; [email protected]; Raslan Darawsheh <[email protected]>
Subject: Re: [PATCH] app/testpmd: skip stopped queues when forwardingHi Dmitry,
Thanks for the patch.
Quoted 55 lines · Dmitry Kozlyuk
After "port <
port_id> rxq|txq <queue_id> stop"
the stopped queue was used in forwarding nonetheless, which may cause undefined behavior in the PMD.Record the configured queue state
and account for it when launching forwarding as follows:+--------+---------+-----------------+---------------+ |RxQ |TxQ |Configured mode |Launch routine | +--------+---------+-----------------+---------------+ |stopped |stopped |* |- | |stopped |started |txonly |(configured) | |stopped |started |* |- | |started |stopped |* |rxonly | |started |started |* |(configured) | +--------+---------+-----------------+---------------+Display stopped queues on "show port config rxtx".
Signed-off-by Dmitry Kozlyuk <[email protected]>Reviewed-by Raslan Darawsheh <[email protected]>--- app/test-pmd/cmdline.c | 8 ++++++++ app/test-pmd/config.c | 6 ++++++ app/test-pmd/testpmd.c | 18 ++++++++++++++++-- app/test-pmd/testpmd.h | 10 ++++++++++ 4 files changed, 40 insertions(+), 2 deletions(-)app/test-pmd/cmdline.c index renamed from app/test-pmd/cmdline.c +0
-0
e626b1c7d9..8b0920e23d100644app/test-pmd/cmdline.c +8
-0@@ -2702,6 +2702,14 @@ cmd_config_rxtx_queue_parsed(void *parsed_result, if (ret == -ENOTSUP)fprintf (stderr,"Function not supported in PMD\n" ); +if (ret ==0 ) { +struct rte_port *port; +struct queue_state *states; + + port = &ports[res->portid]; + states = isrx ? port->rxq_state : port->txq_state; + states[res->qid].stopped = !isstart; + } }
cmdline_parse_token_string_tcmd_config_rxtx_queue_port= diff --gita/app/test-pmd/config.cb/app/test-pmd/config.cindex1722d6c8f8..7ce9cb483a100644app/test-pmd/config.c rxtx_config_display(void) · line 2824
Quoted 8 lines
app/test-pmd/config.c +3
-0@@ -2817,6 +2817,9 @@ rxtx_config_display(void) rx_conf->share_qid);printf ("\n" ); } +for (qid =0 ; qid < nb_rxq; qid++) +if (ports[pid].rxq_state[qid].stopped) +printf (" RX queue %d is stopped\n" , qid);/* per tx queue config only for first queue to be less verbose */
app/test-pmd/config.c rxtx_config_display(void) · line 2853
Quoted 3 lines
app/test-pmd/config.c +0
-0for (qid =0 ; qid <1 ; qid++) {@@ -2850,6 +2853,9 @@ rxtx_config_display(void) printf(" TX offloads=0x%"PRIx64" - TX RS bitthreshold=%d\n",
Quoted 13 lines
app/test-pmd/config.c +3
-0offloads_tmp, tx_rs_thresh_tmp); } +for (qid =0 ; qid < nb_txq; qid++) +if (ports[pid].txq_state[qid].stopped) +printf (" TX queue %d is stopped\n" , qid); } }app/test-pmd/testpmd.c index renamed from app/test-pmd/testpmd.c +0
-0
6c387bde84..36ff845181100644app/test-pmd/testpmd.c flush_fwd_rx_queues(void) · line 2158
Quoted 7 lines
app/test-pmd/testpmd.c +2
-0@@ -2152,6 +2152,8 @@ flush_fwd_rx_queues(void)for (rxp =0 ; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {for (rxq =0 ; rxq < nb_rxq; rxq++) { port_id = fwd_ports_ids[rxp]; +if (ports[port_id].rxq_state[rxq].stopped) +continue ;/** * testpmd can stuck in the below do while loop
app/test-pmd/testpmd.c flush_fwd_rx_queues(void) · line 2159
Quoted 1 line
app/test-pmd/testpmd.c +0
-0* if rte_eth_rx_burst() always returns nonzero @@ -2223,8
Quoted 1 line
+2225,20 @@ run_pkt_fwd_on_lcore(struct
fwd_lcore*fc,packet_fwd_t
pkt_fwd)Quoted 11 lines
static int start_pkt_forward_on_core(void *fwd_arg) {app/test-pmd/testpmd.c +3
-2-run_pkt_fwd_on_lcore ((struct fwd_lcore *) fwd_arg, - cur_fwd_config.fwd_eng->packet_fwd); +struct fwd_lcore *fc = fwd_arg; +struct fwd_stream *fsm = fwd_streams[fc->stream_idx]; + struct queue_state *rxq = &ports[fsm->rx_port].rxq_state[fsm-
rx_queue];app/test-pmd/testpmd.c +1
-0+ struct queue_state *txq = &ports[fsm->tx_port].txq_state[fsm-
tx_queue];app/test-pmd/testpmd.c run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
Quoted 5 lines
app/test-pmd/testpmd.c +5
-0+struct fwd_engine *fwd_engine = cur_fwd_config.fwd_eng; +packet_fwd_t packet_fwd; + +/* Check if there will ever be any packets to send. */ + if (rxq->stopped && (txq->stopped || fwd_engine !=
&tx_only_engine))
Quoted 1 line
app/test-pmd/testpmd.c run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) · line 2237
Quoted 1 line
app/test-pmd/testpmd.c +1
-0+return 0 ;
Have you considered other fwd_engines such as io_fwd_engine and mac_fwd_engine?
Quoted 6 lines
app/test-pmd/testpmd.c
Quoted 2 lines
app/test-pmd/testpmd.c +2
-0+/* Force rxonly mode if RxQ is started, but TxQ is stopped. */ + packet_fwd = !rxq->stopped && txq->stopped ?rx_only_engine.packet_fwd
app/test-pmd/testpmd.c run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) · line 2240
Quoted 1 line
app/test-pmd/testpmd.c +1
-0+ : fwd_engine->packet_fwd;Should we have a print here for user info, that mode has been changed or ignored.
Why need to force rxonly mode for this situation? BTW, the value of cur_fwd_eng hasn't been updated after you changed forward mode.
Quoted 25 lines
Quoted 22 lines
app/test-pmd/testpmd.c +1
-0+run_pkt_fwd_on_lcore (fc, packet_fwd);return 0 ; }app/test-pmd/testpmd.h index renamed from app/test-pmd/testpmd.h +0
-0
2149ecd93a..2744fa4d76100644app/test-pmd/testpmd.h +6
-0@@ -216,6 +216,12 @@ struct xstat_display_info {bool allocated; }; +/** Application state of a queue. */ +struct queue_state { +/** The queue is stopped and should not be used. */ +bool stopped; +}; +/** * The data structure associated with each port. */ app/test-pmd/testpmd.h struct rte_port { · line 263
Quoted 2 lines
app/test-pmd/testpmd.h +0
-0@@ -256,6 +262,10 @@ struct rte_port {uint64_t mbuf_dynf; const struct rte_eth_rxtx_callback*tx_set_dynf_cb[RTE_MAX_QUEUES_PER_PORT+1];
Quoted 10 lines
app/test-pmd/testpmd.h struct rte_port { · line 268
Quoted 5 lines
app/test-pmd/testpmd.h +4
-0struct xstat_display_info xstats_info; +/** Per-Rx-queue state. */ +struct queue_state rxq_state[RTE_MAX_QUEUES_PER_PORT ]; +/** Per-Tx-queue state. */ +struct queue_state txq_state[RTE_MAX_QUEUES_PER_PORT ];Can we think of adding rxq_state/txq_state as part of existing structures under rte_port->rte_eth_rxconf/rte_eth_txconf.
And if it helps, rather than bool can we use u8 witheth_devdefines- #defineRTE_ETH_QUEUE_STATE_STOPPED0 /**< Queue stopped. */ #defineRTE_ETH_QUEUE_STATE_STARTED1 /**< Queue started. */
The same.
Quoted 3 lines
Quoted 3 lines
app/test-pmd/testpmd.h +0
-0};/**
Hi Aman, Yuying,
You share some concerns, so I'm answering in one thread.
Quoted 2 lines
From: Zhang, Yuying <[email protected]>
Sent: Wednesday, February 9, 2022 12:00 PM
[...]
Quoted 19 lines
Quoted 1 line
Quoted 1 line
+2225,20 @@ run_pkt_fwd_on_lcore(struct
fwd_lcore*fc,
packet_fwd_tQuoted 17 lines
pkt_fwd)Quoted 16 lines
static int start_pkt_forward_on_core(void *fwd_arg) {app/test-pmd/testpmd.c +3
-2-run_pkt_fwd_on_lcore ((struct fwd_lcore *) fwd_arg, - cur_fwd_config.fwd_eng->packet_fwd); +struct fwd_lcore *fc = fwd_arg; +struct fwd_stream *fsm = fwd_streams[fc->stream_idx]; + struct queue_state *rxq = &ports[fsm->rx_port].rxq_state[fsm-
rx_queue];
+ structqueue_state*txq = &ports[fsm->tx_port].txq_state[fsm-tx_queue];app/test-pmd/testpmd.c +5
-0+struct fwd_engine *fwd_engine = cur_fwd_config.fwd_eng; +packet_fwd_t packet_fwd; + +/* Check if there will ever be any packets to send. */ + if (rxq->stopped && (txq->stopped || fwd_engine !=
Quoted 4 lines
Quoted 2 lines
&tx_only_engine))Quoted 1 line
+ return 0;
Have you considered other
fwd_enginessuch asio_fwd_engineandmac_fwd_engine?
The only engine that can send packets without receiving them is "txonly". All other engines call rte_eth_rx_burst(),
which is illegal for a stopped RxQ even if the packets are discarded.
Quoted 7 lines
Quoted 7 lines
Review
Quoted 2 lines
(quoted diff) +2
-0+ /* Force rxonly mode if RxQ is started, but TxQ is stopped. */ + packet_fwd = !rxq->stopped && txq->stopped ?rx_only_engine.packet_fwd
Quoted 1 line
+ : fwd_engine-
packet_fwd;
Should we have a print here for user info, that mode has been changed or ignored.
Good idea.
Quoted 2 lines
Why need to force rxonly mode for this situation? BTW, the value of
cur_fwd_enghasn't been updated after you changed forward mode.
It is logical to preserve as much workload as possible
so that stopping a TxQ does not reduce the Rx from NIC's perspective.
Quoted 30 lines
Quoted 30 lines
Quoted 22 lines
app/test-pmd/testpmd.c +1
-0+run_pkt_fwd_on_lcore (fc, packet_fwd);return 0 ; }app/test-pmd/testpmd.h index renamed from app/test-pmd/testpmd.h +0
-0
2149ecd93a..2744fa4d76100644app/test-pmd/testpmd.h +6
-0@@ -216,6 +216,12 @@ struct xstat_display_info {bool allocated; }; +/** Application state of a queue. */ +struct queue_state { +/** The queue is stopped and should not be used. */ +bool stopped; +}; +/** * The data structure associated with each port. */ app/test-pmd/testpmd.h struct rte_port { · line 263
Quoted 2 lines
app/test-pmd/testpmd.h +0
-0@@ -256,6 +262,10 @@ struct rte_port {uint64_t mbuf_dynf; const struct rte_eth_rxtx_callback*tx_set_dynf_cb[RTE_MAX_QUEUES_PER_PORT+1];
Quoted 5 lines
app/test-pmd/testpmd.h +4
-0struct xstat_display_info xstats_info; +/** Per-Rx-queue state. */ +struct queue_state rxq_state[RTE_MAX_QUEUES_PER_PORT ]; +/** Per-Tx-queue state. */ +struct queue_state txq_state[RTE_MAX_QUEUES_PER_PORT ];
Quoted 8 lines
Quoted 1 line
Can we think of adding rxq_state/txq_state as part of existing
structures
Quoted 2 lines
under rte_port->rte_eth_rxconf/rte_eth_txconf.
And if it helps, rather than bool can we use u8 witheth_devdefines- #define
Quoted 2 lines
RTE_ETH_QUEUE_STATE_STOPPED0 /**< Queue stopped. */ #defineRTE_ETH_QUEUE_STATE_STARTED1 /**< Queue started. */The same.
Will change to constants (even enum maybe?).
Struct rte_eth_{rx,tx}conf cannot be changed, because it's ethdev API, and should not be changed, because it reflects queue configuration, while the new arrays reflect the queue state.
What do you think of the following?
struct port_rxqueue {
struct rte_eth_rxconf conf;
uint8_t state;
};struct port_txqueue {
struct rte_eth_txconf conf;
uint8_t state;
};struct rte_port {
/* ... */
struct port_rxqueue rxq[RTE_MAX_QUEUES_PER_PORT];
struct port_txqueue txq[RTE_MAX_QUEUES_PER_PORT];
};Hi Dmitry
Sorry to be direct, but I don't think this patch makes sense.
I need the code to clarify the problems so I'll use this thread not the one you're answering.
Quoted 59 lines
-----Original Message-----
From: Zhang, Yuying <[email protected]>
Sent: Wednesday, February 9, 2022 09:00
To: Singh, Aman Deep <[email protected]>; Dmitry Kozlyuk <[email protected]>; [email protected]
Cc: Li, Xiaoyun <[email protected]>; [email protected]; [email protected]; Raslan Darawsheh <[email protected]>
Subject: RE: [PATCH] app/testpmd: skip stopped queues when forwardingHi Dmitry,
Quoted 48 lines
-----Original Message-----
From: Singh, Aman Deep <[email protected]>
Sent: Thursday, February 3, 2022 9:52 PM
To: Dmitry Kozlyuk <[email protected]>; [email protected]
Cc: Li, Xiaoyun <[email protected]>; Zhang, Yuying <[email protected]>; [email protected]; [email protected]; Raslan Darawsheh <[email protected]>
Subject: Re: [PATCH] app/testpmd: skip stopped queues when forwardingHi Dmitry,
Thanks for the patch.
Quoted 34 lines · Dmitry Kozlyuk
After "port <
port_id> rxq|txq <queue_id> stop"
the stopped queue was used in forwarding nonetheless, which may cause undefined behavior in the PMD.Record the configured queue state
and account for it when launching forwarding as follows:+--------+---------+-----------------+---------------+ |RxQ |TxQ |Configured mode |Launch routine | +--------+---------+-----------------+---------------+ |stopped |stopped |* |- | |stopped |started |txonly |(configured) | |stopped |started |* |- | |started |stopped |* |rxonly | |started |started |* |(configured) | +--------+---------+-----------------+---------------+Display stopped queues on "show port config rxtx".
Signed-off-by Dmitry Kozlyuk <[email protected]>Reviewed-by Raslan Darawsheh <[email protected]>--- app/test-pmd/cmdline.c | 8 ++++++++ app/test-pmd/config.c | 6 ++++++ app/test-pmd/testpmd.c | 18 ++++++++++++++++-- app/test-pmd/testpmd.h | 10 ++++++++++ 4 files changed, 40 insertions(+), 2 deletions(-)app/test-pmd/cmdline.c index renamed from app/test-pmd/cmdline.c +0
-0
e626b1c7d9..8b0920e23d100644app/test-pmd/cmdline.c +0
-0
<snip>
Quoted 16 lines
Quoted 1 line
Quoted 1 line
+2225,20 @@ run_pkt_fwd_on_lcore(struct
fwd_lcore*fc,
packet_fwd_tQuoted 14 lines
pkt_fwd)app/test-pmd/cmdline.c
Quoted 13 lines
static int start_pkt_forward_on_core(void *fwd_arg) {app/test-pmd/cmdline.c +3
-2-run_pkt_fwd_on_lcore ((struct fwd_lcore *) fwd_arg, - cur_fwd_config.fwd_eng->packet_fwd); +struct fwd_lcore *fc = fwd_arg; +struct fwd_stream *fsm = fwd_streams[fc->stream_idx]; + struct queue_state *rxq = &ports[fsm->rx_port].rxq_state[fsm-
rx_queue];app/test-pmd/cmdline.c +1
-0+ struct queue_state *txq = &ports[fsm->tx_port].txq_state[fsm-
tx_queue];app/test-pmd/cmdline.c +2
-0+struct fwd_engine *fwd_engine = cur_fwd_config.fwd_eng; +packet_fwd_t packet_fwd;
1. For each lcore, there may be many fwd streams which are different queue pair even different port pair.
Just imagine the simplest case, you run iofwd with 4 ports single queue with single forwarding core.
But you're only dealing with the first fwd stream here. Each fc actually includes stream from fc->stream_idx to fc->stream_idx+fc->stream_nb-1.
What are you going to do with this?
If you stop port 0 rxq 0, you shouldn't change the whole lcore behavior to rxonly. Because the pair of port 2 rxq 0-> port 3 txq 0 and port 3 rxq0 -> port 2 txq0 should be normal iofwd. And this is only the basic case.
And I don't think it's that harmful to just let those fwd running even if you stop some rxq. Because their pair won't have pkts to do tx if there's no rx.
2. Even if you're going to narrow down for the case which fc->nb_stream=1.
You keep the value of cur_fwd_config. There's a potential issue here since cur_fwd_config is not really cur_fwd_config anymore.
In stop_packet_forwarding(), It sets fc->stopped=1 and uses cur_fwd_config.fwd_eng->port_fwd_end to stop packeting forward.
Right now, in most of the fwd mode, port_fwd_end=NULL. But txonly and noisy_vnf and icmp_echo are not.
What if someday rxonly engine needs to have a port_fwd_end? Should you call port_fwd_end for your lcore (rxonly) too while others are still cur_fwd_engine?
Also, in this case, shouldn't you init "stopped" (or "state" following other comments) to be 1 for txonly? Because the default value will be 0, following your code, txonly engine will change to rxonly if the txq is stopped.
Also, even if you init "stopped" for txonly, have you considered flow_gen engine? Flow_gen is also kind of a txonly engine (all received pkts will be dropped) to generate multi-flow packets.
Anyway, I still think it's not worth it. And there may be other issues too. The patch looks dangerous to me.
BRs
Xiaoyun
Quoted 54 lines
Quoted 5 lines
app/test-pmd/cmdline.c
Quoted 3 lines
app/test-pmd/cmdline.c +3
-0+ +/* Check if there will ever be any packets to send. */ + if (rxq->stopped && (txq->stopped || fwd_engine !=
&tx_only_engine))app/test-pmd/cmdline.c
Quoted 1 line
app/test-pmd/cmdline.c +1
-0+return 0 ;Have you considered other
fwd_enginessuch asio_fwd_engineandmac_fwd_engine?Quoted 6 lines
app/test-pmd/cmdline.c
Quoted 2 lines
app/test-pmd/cmdline.c +2
-0+/* Force rxonly mode if RxQ is started, but TxQ is stopped. */ + packet_fwd = !rxq->stopped && txq->stopped ?rx_only_engine.packet_fwd
app/test-pmd/cmdline.c
Quoted 1 line
app/test-pmd/cmdline.c +1
-0+ : fwd_engine->packet_fwd;Should we have a print here for user info, that mode has been changed or ignored.
Why need to force rxonly mode for this situation? BTW, the value of
cur_fwd_enghasn't been updated after you changed forward mode.Quoted 24 lines
Quoted 21 lines
app/test-pmd/cmdline.c +1
-0+run_pkt_fwd_on_lcore (fc, packet_fwd);return 0 ; }app/test-pmd/testpmd.h index renamed from app/test-pmd/testpmd.h +0
-0
2149ecd93a..2744fa4d76100644app/test-pmd/testpmd.h +5
-0@@ -216,6 +216,12 @@ struct xstat_display_info {bool allocated; }; +/** Application state of a queue. */ struct queue_state { +/** The queue is stopped and should not be used. */ +bool stopped; +}; +/** * The data structure associated with each port. */ app/test-pmd/testpmd.h struct rte_port { · line 263
Quoted 2 lines
app/test-pmd/testpmd.h +0
-0@@ -256,6 +262,10 @@ struct rte_port {uint64_t mbuf_dynf; const struct rte_eth_rxtx_callback*tx_set_dynf_cb[RTE_MAX_QUEUES_PER_PORT+1];
Quoted 9 lines
app/test-pmd/testpmd.h struct rte_port { · line 268
Quoted 5 lines
app/test-pmd/testpmd.h +4
-0struct xstat_display_info xstats_info; +/** Per-Rx-queue state. */ +struct queue_state rxq_state[RTE_MAX_QUEUES_PER_PORT ]; +/** Per-Tx-queue state. */ +struct queue_state txq_state[RTE_MAX_QUEUES_PER_PORT ];Can we think of adding rxq_state/txq_state as part of existing structures under rte_port->rte_eth_rxconf/rte_eth_txconf. And if it helps, rather than bool can we use u8 with
eth_devdefines- #defineRTE_ETH_QUEUE_STATE_STOPPED0 /**< Queue stopped. */#define
Quoted 1 line
RTE_ETH_QUEUE_STATE_STARTED1 /**< Queue started. */The same.
Quoted 3 lines
Quoted 3 lines
app/test-pmd/testpmd.h +0
-0};/**
- [email protected] (invalid address?), +ethdev maintainers
Quoted 5 lines
From: Li, Xiaoyun <[email protected]>
[...]
Sorry to be direct, but I don't think this patch makes sense. I need the code to clarify the problems so I'll use this thread not the one you're answering.
Thank you for explaining the issues in great detail! I'd like to understand one crucial point below to decide whether testpmd needs any changes at all.
Quoted 4 lines
[...]
And I don't think it's that harmful to just let those fwd running even if you stop some rxq. Because their pair won't have pkts to do tx if there's no rx.
This patch was created with the assumption
that stopped queues may not be used for Rx/Tx.
No-op behavior of rte_eth_rx/tx_burst()
for a stopped queue is not documented.
Yes, at least some PMDs implement it this way.
But is this behavior intended?
It is the application that stops the queue or starts it deferred. Stopping is non-atomic, so polling the queue is not allowed during it. Hence, if the application intends to stop queues, it must either:
a) Know the queue is not polled before stopping it.
Use case: a secondary that was polling the queue has crashed, the primary is doing a recovery to free all mbufs. There is no issue since there is no poller to touch the queue.
b) Tell the poller to skip the queue for the time of stopping it. Use case: deferred queue start or queue reconfiguration. But if the poller is cooperating anyway,
what prevents it from not touching the queue for longer?
The same considerations apply to starting a queue.
No-op behavior is convenient from the application point of view. But it also means that pollers of stopped queues
will go all the way down to PMD Rx/Tx routines, wasting cycles, and some PMDs will do a check for the queue state, even though it may never be needed for a particular application.
Andrew, Ferruh, Thomas, which behavior does ethdev assume (see below)?
Quoted 28 lines
This patch was created with the assumption
that stopped queues may not be used for Rx/Tx.
No-op behavior of rte_eth_rx/tx_burst()
for a stopped queue is not documented.
Yes, at least some PMDs implement it this way.
But is this behavior intended?It is the application that stops the queue or starts it deferred. Stopping is non-atomic, so polling the queue is not allowed during it. Hence, if the application intends to stop queues, it must either:
a) Know the queue is not polled before stopping it.
Use case: a secondary that was polling the queue has crashed, the primary is doing a recovery to free all mbufs. There is no issue since there is no poller to touch the queue.b) Tell the poller to skip the queue for the time of stopping it. Use case: deferred queue start or queue reconfiguration. But if the poller is cooperating anyway,
what prevents it from not touching the queue for longer?The same considerations apply to starting a queue.
No-op behavior is convenient from the application point of view. But it also means that pollers of stopped queues
will go all the way down to PMD Rx/Tx routines, wasting cycles, and some PMDs will do a check for the queue state, even though it may never be needed for a particular application.
21/02/2022 09:58, Dmitry Kozlyuk:
Quoted 1 line
Andrew, Ferruh, Thomas, which behavior does ethdev assume (see below)?
For the whole device stop, this is the documentation:
"
The transmit and receive functions should not be invoked when the device is stopped.
"
There is also this comment on rte_eth_dev_reset:
"
Note: To avoid unexpected behavior, the application should stop calling Tx and Rx functions before calling rte_eth_dev_reset( ). For thread safety, all these controlling functions should be called from the same thread.
"
For queue stop, there is no documented expectation.
There is this comment for queue callback removal:
"
The memory for the callback can be
subsequently freed back by the application by calling rte_free():
- Immediately - if the port is stopped, or the user knows that no callbacks are in flight e.g. if called from the thread doing Rx/Tx on that queue.
- After a short delay - where the delay is sufficient to allow any in-flight callbacks to complete. Alternately, the RCU mechanism can be used to detect when data plane threads have ceased referencing the callback memory.
"
Quoted 4 lines
Quoted 4 lines
This patch was created with the assumption
that stopped queues may not be used for Rx/Tx.
No-op behavior of rte_eth_rx/tx_burst()
for a stopped queue is not documented.
Indeed, it is not documented.
I suggest working on this documentation first.
testpmd could be adjusted later if needed.
Quoted 17 lines
Quoted 17 lines
Yes, at least some PMDs implement it this way.
But is this behavior intended?It is the application that stops the queue or starts it deferred. Stopping is non-atomic, so polling the queue is not allowed during it. Hence, if the application intends to stop queues, it must either:
a) Know the queue is not polled before stopping it.
Use case: a secondary that was polling the queue has crashed, the primary is doing a recovery to free all mbufs. There is no issue since there is no poller to touch the queue.b) Tell the poller to skip the queue for the time of stopping it. Use case: deferred queue start or queue reconfiguration. But if the poller is cooperating anyway,
what prevents it from not touching the queue for longer?
Quoted 7 lines
Quoted 7 lines
The same considerations apply to starting a queue.
No-op behavior is convenient from the application point of view. But it also means that pollers of stopped queues
will go all the way down to PMD Rx/Tx routines, wasting cycles, and some PMDs will do a check for the queue state, even though it may never be needed for a particular application.
Yes that's the question: Do we want
1/ to allow apps to poll stopped queues, adding checks in PMDs, or do we prefer
2/ saving check cycles and expect apps to not poll?
v2 cover app/testpmd: skip stopped queues when forwarding
In reply to Dmitry KozlyukIt was unspecified what happens of a stopped queue is polled. Declare that polling is prohibited and modify testpmd to follow this. Rationale is described i nthe commit log; it follows the discussion:
http://inbox.dpdk.org/dev/BL1PR12MB594551A71273709E1C04A8BAB9309@BL1PR12MB5945.namprd12.prod.outlook.com/
v2:
* Do not try to change forwarding mode partially, disable individual flows that need stopped queues instead.
* Do not store queue status in testpmd,
query it as needed instead.
Dmitry Kozlyuk (2):
ethdev: prohibit polling of a stopped queue
app/testpmd: do not poll stopped queues
app/test-pmd/5tswap.c | 13 ++++++++ app/test-pmd/csumonly.c | 13 ++++++++ app/test-pmd/flowgen.c | 13 ++++++++ app/test-pmd/icmpecho.c | 13 ++++++++ app/test-pmd/ieee1588fwd.c | 13 ++++++++ app/test-pmd/iofwd.c | 13 ++++++++ app/test-pmd/macfwd.c | 13 ++++++++ app/test-pmd/noisy_vnf.c | 13 ++++++++ app/test-pmd/rxonly.c | 13 ++++++++ app/test-pmd/shared_rxq_fwd.c | 13 ++++++++ app/test-pmd/testpmd.c | 57 ++++++++++++++++++++++++++++++++++- app/test-pmd/testpmd.h | 4 +++ app/test-pmd/txonly.c | 13 ++++++++ lib/ethdev/rte_ethdev.h | 2 +- 14 files changed, 204 insertions(+), 2 deletions(-)
Signature · 1 line
2.25.1
PATCH v2 1/2 ethdev: prohibit polling of a stopped queue
Whether it is allowed to call Rx/Tx functions for a stopped queue was undocumented. Some PMDs make this behavior a no-op either by explicitly checking the queue state
or by the way how their routines are implemented or HW works.
No-op behavior may be convenient for application developers. But it also means that pollers of stopped queues
would go all the way down to PMD Rx/Tx routines, wasting cycles. Some PMDs would do a check for the queue state on data path, even though it may never be needed for a particular application. Also, use cases for stopping queues or starting them deferred do not logically require polling stopped queues.
Use case 1: a secondary that was polling the queue has crashed, the primary is doing a recovery to free all mbufs.
By definition the queue to be restarted is not polled.
Use case 2: deferred queue start or queue reconfiguration. The polling thread must be synchronized anyway, because queue start and stop are non-atomic.
Prohibit calling Rx/Tx functions on stopped queues.
--- lib/ethdev/rte_ethdev.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
lib/ethdev/rte_ethdev.h +1 -1
@@ -74,7 +74,7 @@* rte_eth_rx_queue_setup()), it must call rte_eth_dev_stop() first to stop the * device and then do the reconfiguration before calling rte_eth_dev_start() * again. The transmit and receive functions should not be invoked when the -* device is stopped. +* device is stopped or when the queue is stopped (for that queue). * * Please note that some configuration is not stored between calls to * rte_eth_dev_stop()/rte_eth_dev_start(). The following configuration will
Signature · 1 line
2.25.1
PATCH v2 2/2 app/testpmd: do not poll stopped queues
In reply to Dmitry KozlyukCalling Rx/Tx functions on a stopped queue is not supported. Do not run packet forwarding for streams that use stopped queues.
Each stream has a read-only "disabled" field,
so that lcore function can skip such streams.
Forwarding engines can set this field
using a new "stream_init" callback function
by checking relevant queue states.
A helper function is provided to check if a given
Rx queue, Tx queue, or both of them are stopped.
--- app/test-pmd/5tswap.c | 13 ++++++++ app/test-pmd/csumonly.c | 13 ++++++++ app/test-pmd/flowgen.c | 13 ++++++++ app/test-pmd/icmpecho.c | 13 ++++++++ app/test-pmd/ieee1588fwd.c | 13 ++++++++ app/test-pmd/iofwd.c | 13 ++++++++ app/test-pmd/macfwd.c | 13 ++++++++ app/test-pmd/noisy_vnf.c | 13 ++++++++ app/test-pmd/rxonly.c | 13 ++++++++ app/test-pmd/shared_rxq_fwd.c | 13 ++++++++ app/test-pmd/testpmd.c | 57 ++++++++++++++++++++++++++++++++++- app/test-pmd/testpmd.h | 4 +++ app/test-pmd/txonly.c | 13 ++++++++ 13 files changed, 203 insertions(+), 1 deletion(-)
app/test-pmd/5tswap.c +13 -0
@@ -185,9 +185,22 @@ pkt_burst_5tuple_swap(struct fwd_stream *fs)get_end_cycles (fs, start_tsc); } +static int +stream_init_5tuple_swap (struct fwd_stream *fs) +{ +bool rx_stopped, tx_stopped; +int ret; + + ret =fwd_stream_get_stopped_queues (fs, &rx_stopped, &tx_stopped); +if (ret ==0 ) + fs->disabled = rx_stopped || tx_stopped; +return ret; +} +struct fwd_engine five_tuple_swap_fwd_engine = { .fwd_mode_name ="5tswap" , .port_fwd_begin =NULL , .port_fwd_end =NULL , + .stream_init = stream_init_5tuple_swap, .packet_fwd = pkt_burst_5tuple_swap, };
app/test-pmd/csumonly.c +13 -0
@@ -1178,9 +1178,22 @@ pkt_burst_checksum_forward(struct fwd_stream *fs)get_end_cycles (fs, start_tsc); } +static int +stream_init_checksum_forward (struct fwd_stream *fs) +{ +bool rx_stopped, tx_stopped; +int ret; + + ret =fwd_stream_get_stopped_queues (fs, &rx_stopped, &tx_stopped); +if (ret ==0 ) + fs->disabled = rx_stopped || tx_stopped; +return ret; +} +struct fwd_engine csum_fwd_engine = { .fwd_mode_name ="csum" , .port_fwd_begin =NULL , .port_fwd_end =NULL , + .stream_init = stream_init_checksum_forward, .packet_fwd = pkt_burst_checksum_forward, };
app/test-pmd/flowgen.c +13 -0
@@ -207,9 +207,22 @@ flowgen_begin(portid_t pi)return 0 ; } +static int +flowgen_stream_init (struct fwd_stream *fs) +{ +bool rx_stopped, tx_stopped; +int ret; + + ret =fwd_stream_get_stopped_queues (fs, &rx_stopped, &tx_stopped); +if (ret ==0 ) + fs->disabled = rx_stopped || tx_stopped; +return ret; +} +struct fwd_engine flow_gen_engine = { .fwd_mode_name ="flowgen" , .port_fwd_begin = flowgen_begin, .port_fwd_end =NULL , + .stream_init = flowgen_stream_init, .packet_fwd = pkt_burst_flow_gen, };
app/test-pmd/icmpecho.c +13 -0
@@ -512,9 +512,22 @@ reply_to_icmp_echo_rqsts(struct fwd_stream *fs)get_end_cycles (fs, start_tsc); } +static int +icmpecho_stream_init (struct fwd_stream *fs) +{ +bool rx_stopped, tx_stopped; +int ret; + + ret =fwd_stream_get_stopped_queues (fs, &rx_stopped, &tx_stopped); +if (ret ==0 ) + fs->disabled = rx_stopped || tx_stopped; +return ret; +} +struct fwd_engine icmp_echo_engine = { .fwd_mode_name ="icmpecho" , .port_fwd_begin =NULL , .port_fwd_end =NULL , + .stream_init = icmpecho_stream_init, .packet_fwd = reply_to_icmp_echo_rqsts, };
app/test-pmd/ieee1588fwd.c +13 -0
@@ -211,9 +211,22 @@ port_ieee1588_fwd_end(portid_t pi)rte_eth_timesync_disable (pi); } +static int +port_ieee1588_stream_init (struct fwd_stream *fs) +{ +bool rx_stopped, tx_stopped; +int ret; + + ret =fwd_stream_get_stopped_queues (fs, &rx_stopped, &tx_stopped); +if (ret ==0 ) + fs->disabled = rx_stopped || tx_stopped; +return ret; +} +struct fwd_engine ieee1588_fwd_engine = { .fwd_mode_name ="ieee1588" , .port_fwd_begin = port_ieee1588_fwd_begin, .port_fwd_end = port_ieee1588_fwd_end, + .stream_init = port_ieee1588_stream_init, .packet_fwd = ieee1588_packet_fwd, };
app/test-pmd/iofwd.c +13 -0
@@ -88,9 +88,22 @@ pkt_burst_io_forward(struct fwd_stream *fs)get_end_cycles (fs, start_tsc); } +static int +stream_init_forward (struct fwd_stream *fs) +{ +bool rx_stopped, tx_stopped; +int ret; + + ret =fwd_stream_get_stopped_queues (fs, &rx_stopped, &tx_stopped); +if (ret ==0 ) + fs->disabled = rx_stopped || tx_stopped; +return ret; +} +struct fwd_engine io_fwd_engine = { .fwd_mode_name ="io" , .port_fwd_begin =NULL , .port_fwd_end =NULL , + .stream_init = stream_init_forward, .packet_fwd = pkt_burst_io_forward, };
app/test-pmd/macfwd.c +13 -0
@@ -119,9 +119,22 @@ pkt_burst_mac_forward(struct fwd_stream *fs)get_end_cycles (fs, start_tsc); } +static int +stream_init_mac_forward (struct fwd_stream *fs) +{ +bool rx_stopped, tx_stopped; +int ret; + + ret =fwd_stream_get_stopped_queues (fs, &rx_stopped, &tx_stopped); +if (ret ==0 ) + fs->disabled = rx_stopped || tx_stopped; +return ret; +} +struct fwd_engine mac_fwd_engine = { .fwd_mode_name ="mac" , .port_fwd_begin =NULL , .port_fwd_end =NULL , + .stream_init = stream_init_mac_forward, .packet_fwd = pkt_burst_mac_forward, };
app/test-pmd/noisy_vnf.c +13 -0
@@ -277,9 +277,22 @@ noisy_fwd_begin(portid_t pi)return 0 ; } +static int +stream_init_noisy_vnf (struct fwd_stream *fs) +{ +bool rx_stopped, tx_stopped; +int ret; + + ret =fwd_stream_get_stopped_queues (fs, &rx_stopped, &tx_stopped); +if (ret ==0 ) + fs->disabled = rx_stopped || tx_stopped; +return ret; +} +struct fwd_engine noisy_vnf_engine = { .fwd_mode_name ="noisy" , .port_fwd_begin = noisy_fwd_begin, .port_fwd_end = noisy_fwd_end, + .stream_init = stream_init_noisy_vnf, .packet_fwd = pkt_burst_noisy_vnf, };
app/test-pmd/rxonly.c +13 -0
@@ -68,9 +68,22 @@ pkt_burst_receive(struct fwd_stream *fs)get_end_cycles (fs, start_tsc); } +static int +stream_init_receive (struct fwd_stream *fs) +{ +bool rx_stopped; +int ret; + + ret =fwd_stream_get_stopped_queues (fs, &rx_stopped,NULL ); +if (ret ==0 ) + fs->disabled = rx_stopped; +return ret; +} +struct fwd_engine rx_only_engine = { .fwd_mode_name ="rxonly" , .port_fwd_begin =NULL , .port_fwd_end =NULL , + .stream_init = stream_init_receive, .packet_fwd = pkt_burst_receive, };
app/test-pmd/shared_rxq_fwd.c +13 -0
@@ -107,9 +107,22 @@ shared_rxq_fwd(struct fwd_stream *fs)get_end_cycles (fs, start_tsc); } +static int +shared_rxq_stream_init (struct fwd_stream *fs) +{ +bool rx_stopped; +int ret; + + ret =fwd_stream_get_stopped_queues (fs, &rx_stopped,NULL ); +if (ret ==0 ) + fs->disabled = rx_stopped; +return ret; +} +struct fwd_engine shared_rxq_engine = { .fwd_mode_name ="shared_rxq" , .port_fwd_begin =NULL , .port_fwd_end =NULL , + .stream_init = shared_rxq_stream_init, .packet_fwd = shared_rxq_fwd, };
app/test-pmd/testpmd.c +56 -1
87 lines not on this page · raw message
app/test-pmd/testpmd.h +4 -0
@@ -134,6 +134,7 @@ struct fwd_stream {portid_t tx_port;/**< forwarding port of received packets */ queueid_t tx_queue;/**< TX queue to send forwarded packets */ streamid_t peer_addr;/**< index of peer ethernet address of packets */ +bool disabled;/**< the stream is disabled and should not run */ unsigned int retry_enabled;
@@ -323,12 +324,14 @@ struct fwd_lcore {*/ typedef int (*port_fwd_begin_t )(portid_t pi);typedef void (*port_fwd_end_t )(portid_t pi); +typedef int (*stream_init_t )(struct fwd_stream *fs);typedef void (*packet_fwd_t )(struct fwd_stream *fs);struct fwd_engine {const char *fwd_mode_name;/**< Forwarding mode name. */ port_fwd_begin_t port_fwd_begin;/**< NULL if nothing special to do. */ port_fwd_end_t port_fwd_end;/**< NULL if nothing special to do. */ +stream_init_t stream_init;/**< NULL if nothing special to do. */ packet_fwd_t packet_fwd;/**< Mandatory. */ };
@@ -887,6 +890,7 @@ void rxtx_config_display(void);void fwd_config_setup (void );void set_def_fwd_config (void );void reconfig (portid_t new_port_id,unsigned socket_id); +int fwd_stream_get_stopped_queues (struct fwd_stream *fs,bool *rx,bool *tx);int init_fwd_streams (void );void update_fwd_ports (portid_t new_pid);
app/test-pmd/txonly.c +13 -0
@@ -504,9 +504,22 @@ tx_only_begin(portid_t pi)return 0 ; } +static int +tx_only_stream_init (struct fwd_stream *fs) +{ +bool tx_stopped; +int ret; + + ret =fwd_stream_get_stopped_queues (fs,NULL , &tx_stopped); +if (ret ==0 ) + fs->disabled = tx_stopped; +return ret; +} +struct fwd_engine tx_only_engine = { .fwd_mode_name ="txonly" , .port_fwd_begin = tx_only_begin, .port_fwd_end =NULL , + .stream_init = tx_only_stream_init, .packet_fwd = pkt_burst_transmit, };
Signature · 1 line
2.25.1
v3 cover app/testpmd: skip stopped queues when forwarding
In reply to Dmitry KozlyukIt was unspecified what happens of a stopped queue is polled. Declare that polling is prohibited and modify testpmd to follow this. Rationale is described in the commit log; it follows the discussion:
http://inbox.dpdk.org/dev/BL1PR12MB594551A71273709E1C04A8BAB9309@BL1PR12MB5945.namprd12.prod.outlook.com/
v3:
* Revert to storing queue status in testpmd to handle PMDs that don't implement rte_eth_rx/tx_queue_info_get() (Daxue Gao).
* Add missed "macswap" support.
v2:
* Do not try to change forwarding mode partially, disable individual flows that need stopped queues instead.
* Do not store queue status in testpmd,
query it as needed instead.
Dmitry Kozlyuk (2):
app/testpmd: do not poll stopped queues
ethdev: prohibit polling of a stopped queue
app/test-pmd/5tswap.c | 13 +++++ app/test-pmd/cmdline.c | 45 +++++++++------- app/test-pmd/config.c | 8 +-- app/test-pmd/csumonly.c | 13 +++++ app/test-pmd/flowgen.c | 13 +++++ app/test-pmd/icmpecho.c | 13 +++++ app/test-pmd/ieee1588fwd.c | 13 +++++ app/test-pmd/iofwd.c | 13 +++++ app/test-pmd/macfwd.c | 13 +++++ app/test-pmd/macswap.c | 13 +++++ app/test-pmd/noisy_vnf.c | 13 +++++ app/test-pmd/rxonly.c | 8 +++ app/test-pmd/shared_rxq_fwd.c | 8 +++ app/test-pmd/testpmd.c | 96 +++++++++++++++++++++++------------ app/test-pmd/testpmd.h | 19 ++++++- app/test-pmd/txonly.c | 8 +++ lib/ethdev/rte_ethdev.h | 2 +- 17 files changed, 254 insertions(+), 57 deletions(-)
Signature · 1 line
2.25.1
PATCH v3 1/2 app/testpmd: do not poll stopped queues
Calling Rx/Tx functions on a stopped queue is not supported. Do not run packet forwarding for streams that use stopped queues.
Each stream has a read-only "disabled" field, so that lcore function can skip such streams. Forwarding engines can set this field
using a new "stream_init" callback function by checking relevant queue states,
which are stored along with queue configurations (not all PMDs implement rte_eth_rx/tx_queue_info_get() to query the state from there).
--- app/test-pmd/5tswap.c | 13 ++++++ app/test-pmd/cmdline.c | 45 ++++++++++-------- app/test-pmd/config.c | 8 ++-- app/test-pmd/csumonly.c | 13 ++++++ app/test-pmd/flowgen.c | 13 ++++++ app/test-pmd/icmpecho.c | 13 ++++++ app/test-pmd/ieee1588fwd.c | 13 ++++++ app/test-pmd/iofwd.c | 13 ++++++ app/test-pmd/macfwd.c | 13 ++++++ app/test-pmd/macswap.c | 13 ++++++ app/test-pmd/noisy_vnf.c | 13 ++++++ app/test-pmd/rxonly.c | 8 ++++ app/test-pmd/shared_rxq_fwd.c | 8 ++++ app/test-pmd/testpmd.c | 87 ++++++++++++++++++++++------------- app/test-pmd/testpmd.h | 19 +++++++- app/test-pmd/txonly.c | 8 ++++ 16 files changed, 244 insertions(+), 56 deletions(-)
app/test-pmd/5tswap.c +13 -0
@@ -185,9 +185,22 @@ pkt_burst_5tuple_swap(struct fwd_stream *fs)get_end_cycles (fs, start_tsc); } +static void +stream_init_5tuple_swap (struct fwd_stream *fs) +{ +bool rx_stopped, tx_stopped; + + rx_stopped = ports[fs->rx_port].rxq[fs->rx_queue].state == +RTE_ETH_QUEUE_STATE_STOPPED ; + tx_stopped = ports[fs->tx_port].txq[fs->tx_queue].state == +RTE_ETH_QUEUE_STATE_STOPPED ; + fs->disabled = rx_stopped || tx_stopped; +} +struct fwd_engine five_tuple_swap_fwd_engine = { .fwd_mode_name ="5tswap" , .port_fwd_begin =NULL , .port_fwd_end =NULL , + .stream_init = stream_init_5tuple_swap, .packet_fwd = pkt_burst_5tuple_swap, };
app/test-pmd/cmdline.c +27 -18
129 lines not on this page · raw message
app/test-pmd/config.c +4 -4
@@ -3551,8 +3551,8 @@ rxtx_config_display(void) nb_fwd_lcores, nb_fwd_ports);RTE_ETH_FOREACH_DEV (pid) { -struct rte_eth_rxconf *rx_conf = &ports[pid].rx_conf[0 ]; -struct rte_eth_txconf *tx_conf = &ports[pid].tx_conf[0 ]; +struct rte_eth_rxconf *rx_conf = &ports[pid].rxq[0 ].conf; +struct rte_eth_txconf *tx_conf = &ports[pid].txq[0 ].conf;uint16_t *nb_rx_desc = &ports[pid].nb_rx_desc[0 ];uint16_t *nb_tx_desc = &ports[pid].nb_tx_desc[0 ];struct rte_eth_rxq_info rx_qinfo;
@@ -3810,7 +3810,7 @@ fwd_stream_on_other_lcores(uint16_t domain_id, lcoreid_t src_lc, fs = fwd_streams[sm_id]; port = &ports[fs->rx_port]; dev_info = &port->dev_info; - rxq_conf = &port->rx_conf[fs->rx_queue]; + rxq_conf = &port->rxq[fs->rx_queue].conf;if ((dev_info->dev_capa &RTE_ETH_DEV_CAPA_RXQ_SHARE ) ==0 || rxq_conf->share_group ==0 )/* Not shared rxq. */
@@ -3870,7 +3870,7 @@ pkt_fwd_shared_rxq_check(void) fs->lcore = fwd_lcores[lc_id]; port = &ports[fs->rx_port]; dev_info = &port->dev_info; - rxq_conf = &port->rx_conf[fs->rx_queue]; + rxq_conf = &port->rxq[fs->rx_queue].conf;if ((dev_info->dev_capa &RTE_ETH_DEV_CAPA_RXQ_SHARE ) ==0 || rxq_conf->share_group ==0 )/* Not shared rxq. */
app/test-pmd/csumonly.c +13 -0
@@ -1178,9 +1178,22 @@ pkt_burst_checksum_forward(struct fwd_stream *fs)get_end_cycles (fs, start_tsc); } +static void +stream_init_checksum_forward (struct fwd_stream *fs) +{ +bool rx_stopped, tx_stopped; + + rx_stopped = ports[fs->rx_port].rxq[fs->rx_queue].state == +RTE_ETH_QUEUE_STATE_STOPPED ; + tx_stopped = ports[fs->tx_port].txq[fs->tx_queue].state == +RTE_ETH_QUEUE_STATE_STOPPED ; + fs->disabled = rx_stopped || tx_stopped; +} +struct fwd_engine csum_fwd_engine = { .fwd_mode_name ="csum" , .port_fwd_begin =NULL , .port_fwd_end =NULL , + .stream_init = stream_init_checksum_forward, .packet_fwd = pkt_burst_checksum_forward, };
app/test-pmd/flowgen.c +13 -0
@@ -207,9 +207,22 @@ flowgen_begin(portid_t pi)return 0 ; } +static void +flowgen_stream_init (struct fwd_stream *fs) +{ +bool rx_stopped, tx_stopped; + + rx_stopped = ports[fs->rx_port].rxq[fs->rx_queue].state == +RTE_ETH_QUEUE_STATE_STOPPED ; + tx_stopped = ports[fs->tx_port].txq[fs->tx_queue].state == +RTE_ETH_QUEUE_STATE_STOPPED ; + fs->disabled = rx_stopped || tx_stopped; +} +struct fwd_engine flow_gen_engine = { .fwd_mode_name ="flowgen" , .port_fwd_begin = flowgen_begin, .port_fwd_end =NULL , + .stream_init = flowgen_stream_init, .packet_fwd = pkt_burst_flow_gen, };
app/test-pmd/icmpecho.c +13 -0
@@ -512,9 +512,22 @@ reply_to_icmp_echo_rqsts(struct fwd_stream *fs)get_end_cycles (fs, start_tsc); } +static void +icmpecho_stream_init (struct fwd_stream *fs) +{ +bool rx_stopped, tx_stopped; + + rx_stopped = ports[fs->rx_port].rxq[fs->rx_queue].state == +RTE_ETH_QUEUE_STATE_STOPPED ; + tx_stopped = ports[fs->tx_port].txq[fs->tx_queue].state == +RTE_ETH_QUEUE_STATE_STOPPED ; + fs->disabled = rx_stopped || tx_stopped; +} +struct fwd_engine icmp_echo_engine = { .fwd_mode_name ="icmpecho" , .port_fwd_begin =NULL , .port_fwd_end =NULL , + .stream_init = icmpecho_stream_init, .packet_fwd = reply_to_icmp_echo_rqsts, };
app/test-pmd/ieee1588fwd.c +13 -0
@@ -211,9 +211,22 @@ port_ieee1588_fwd_end(portid_t pi)rte_eth_timesync_disable (pi); } +static void +port_ieee1588_stream_init (struct fwd_stream *fs) +{ +bool rx_stopped, tx_stopped; + + rx_stopped = ports[fs->rx_port].rxq[fs->rx_queue].state == +RTE_ETH_QUEUE_STATE_STOPPED ; + tx_stopped = ports[fs->tx_port].txq[fs->tx_queue].state == +RTE_ETH_QUEUE_STATE_STOPPED ; + fs->disabled = rx_stopped || tx_stopped; +} +struct fwd_engine ieee1588_fwd_engine = { .fwd_mode_name ="ieee1588" , .port_fwd_begin = port_ieee1588_fwd_begin, .port_fwd_end = port_ieee1588_fwd_end, + .stream_init = port_ieee1588_stream_init, .packet_fwd = ieee1588_packet_fwd, };
app/test-pmd/iofwd.c +13 -0
@@ -88,9 +88,22 @@ pkt_burst_io_forward(struct fwd_stream *fs)get_end_cycles (fs, start_tsc); } +static void +stream_init_forward (struct fwd_stream *fs) +{ +bool rx_stopped, tx_stopped; + + rx_stopped = ports[fs->rx_port].rxq[fs->rx_queue].state == +RTE_ETH_QUEUE_STATE_STOPPED ; + tx_stopped = ports[fs->tx_port].txq[fs->tx_queue].state == +RTE_ETH_QUEUE_STATE_STOPPED ; + fs->disabled = rx_stopped || tx_stopped; +} +struct fwd_engine io_fwd_engine = { .fwd_mode_name ="io" , .port_fwd_begin =NULL , .port_fwd_end =NULL , + .stream_init = stream_init_forward, .packet_fwd = pkt_burst_io_forward, };
app/test-pmd/macfwd.c +13 -0
@@ -119,9 +119,22 @@ pkt_burst_mac_forward(struct fwd_stream *fs)get_end_cycles (fs, start_tsc); } +static void +stream_init_mac_forward (struct fwd_stream *fs) +{ +bool rx_stopped, tx_stopped; + + rx_stopped = ports[fs->rx_port].rxq[fs->rx_queue].state == +RTE_ETH_QUEUE_STATE_STOPPED ; + tx_stopped = ports[fs->tx_port].txq[fs->tx_queue].state == +RTE_ETH_QUEUE_STATE_STOPPED ; + fs->disabled = rx_stopped || tx_stopped; +} +struct fwd_engine mac_fwd_engine = { .fwd_mode_name ="mac" , .port_fwd_begin =NULL , .port_fwd_end =NULL , + .stream_init = stream_init_mac_forward, .packet_fwd = pkt_burst_mac_forward, };
app/test-pmd/macswap.c +13 -0
@@ -97,9 +97,22 @@ pkt_burst_mac_swap(struct fwd_stream *fs)get_end_cycles (fs, start_tsc); } +static void +stream_init_mac_swap (struct fwd_stream *fs) +{ +bool rx_stopped, tx_stopped; + + rx_stopped = ports[fs->rx_port].rxq[fs->rx_queue].state == +RTE_ETH_QUEUE_STATE_STOPPED ; + tx_stopped = ports[fs->tx_port].txq[fs->tx_queue].state == +RTE_ETH_QUEUE_STATE_STOPPED ; + fs->disabled = rx_stopped || tx_stopped; +} +struct fwd_engine mac_swap_engine = { .fwd_mode_name ="macswap" , .port_fwd_begin =NULL , .port_fwd_end =NULL , + .stream_init = stream_init_mac_swap, .packet_fwd = pkt_burst_mac_swap, };
app/test-pmd/noisy_vnf.c +13 -0
@@ -277,9 +277,22 @@ noisy_fwd_begin(portid_t pi)return 0 ; } +static void +stream_init_noisy_vnf (struct fwd_stream *fs) +{ +bool rx_stopped, tx_stopped; + + rx_stopped = ports[fs->rx_port].rxq[fs->rx_queue].state == +RTE_ETH_QUEUE_STATE_STOPPED ; + tx_stopped = ports[fs->tx_port].txq[fs->tx_queue].state == +RTE_ETH_QUEUE_STATE_STOPPED ; + fs->disabled = rx_stopped || tx_stopped; +} +struct fwd_engine noisy_vnf_engine = { .fwd_mode_name ="noisy" , .port_fwd_begin = noisy_fwd_begin, .port_fwd_end = noisy_fwd_end, + .stream_init = stream_init_noisy_vnf, .packet_fwd = pkt_burst_noisy_vnf, };
app/test-pmd/rxonly.c +8 -0
@@ -68,9 +68,17 @@ pkt_burst_receive(struct fwd_stream *fs)get_end_cycles (fs, start_tsc); } +static void +stream_init_receive (struct fwd_stream *fs) +{ + fs->disabled = ports[fs->rx_port].rxq[fs->rx_queue].state == +RTE_ETH_QUEUE_STATE_STOPPED ; +} +struct fwd_engine rx_only_engine = { .fwd_mode_name ="rxonly" , .port_fwd_begin =NULL , .port_fwd_end =NULL , + .stream_init = stream_init_receive, .packet_fwd = pkt_burst_receive, };
app/test-pmd/shared_rxq_fwd.c +8 -0
@@ -107,9 +107,17 @@ shared_rxq_fwd(struct fwd_stream *fs)get_end_cycles (fs, start_tsc); } +static void +shared_rxq_stream_init (struct fwd_stream *fs) +{ + fs->disabled = ports[fs->rx_port].rxq[fs->rx_queue].state == +RTE_ETH_QUEUE_STATE_STOPPED ; +} +struct fwd_engine shared_rxq_engine = { .fwd_mode_name ="shared_rxq" , .port_fwd_begin =NULL , .port_fwd_end =NULL , + .stream_init = shared_rxq_stream_init, .packet_fwd = shared_rxq_fwd, };
app/test-pmd/testpmd.c +55 -32
231 lines not on this page · raw message
app/test-pmd/testpmd.h +17 -2
@@ -134,6 +134,7 @@ struct fwd_stream {portid_t tx_port;/**< forwarding port of received packets */ queueid_t tx_queue;/**< TX queue to send forwarded packets */ streamid_t peer_addr;/**< index of peer ethernet address of packets */ +bool disabled;/**< the stream is disabled and should not run */ unsigned int retry_enabled;
@@ -238,6 +239,18 @@ struct xstat_display_info {bool allocated; }; +/** RX queue configuration and state. */ +struct port_rxqueue { +struct rte_eth_rxconf conf; +uint8_t state;/**< RTE_ETH_QUEUE_STATE_* value. */ +}; + +/** TX queue configuration and state. */ +struct port_txqueue { +struct rte_eth_txconf conf; +uint8_t state;/**< RTE_ETH_QUEUE_STATE_* value. */ +}; +/** * The data structure associated with each port. */
@@ -260,8 +273,8 @@ struct rte_port {uint8_t dcb_flag;/**< enable dcb */ uint16_t nb_rx_desc[RTE_MAX_QUEUES_PER_PORT +1 ];/**< per queue rx desc number */ uint16_t nb_tx_desc[RTE_MAX_QUEUES_PER_PORT +1 ];/**< per queue tx desc number */ -struct rte_eth_rxconf rx_conf[RTE_MAX_QUEUES_PER_PORT +1 ];/**< per queue rx configuration */ -struct rte_eth_txconf tx_conf[RTE_MAX_QUEUES_PER_PORT +1 ];/**< per queue tx configuration */ +struct port_rxqueue rxq[RTE_MAX_QUEUES_PER_PORT +1 ];/**< per queue rx configuration and state */ +struct port_txqueue txq[RTE_MAX_QUEUES_PER_PORT +1 ];/**< per queue tx configuration and state */ struct rte_ether_addr *mc_addr_pool;/**< pool of multicast addrs */ uint32_t mc_addr_nb;/**< nb. of addr. in mc_addr_pool */ queueid_t queue_nb;/**< nb. of queues for flow rules */
@@ -323,12 +336,14 @@ struct fwd_lcore {*/ typedef int (*port_fwd_begin_t )(portid_t pi);typedef void (*port_fwd_end_t )(portid_t pi); +typedef void (*stream_init_t )(struct fwd_stream *fs);typedef void (*packet_fwd_t )(struct fwd_stream *fs);struct fwd_engine {const char *fwd_mode_name;/**< Forwarding mode name. */ port_fwd_begin_t port_fwd_begin;/**< NULL if nothing special to do. */ port_fwd_end_t port_fwd_end;/**< NULL if nothing special to do. */ +stream_init_t stream_init;/**< NULL if nothing special to do. */ packet_fwd_t packet_fwd;/**< Mandatory. */ };
app/test-pmd/txonly.c +8 -0
@@ -504,9 +504,17 @@ tx_only_begin(portid_t pi)return 0 ; } +static void +tx_only_stream_init (struct fwd_stream *fs) +{ + fs->disabled = ports[fs->tx_port].txq[fs->tx_queue].state == +RTE_ETH_QUEUE_STATE_STOPPED ; +} +struct fwd_engine tx_only_engine = { .fwd_mode_name ="txonly" , .port_fwd_begin = tx_only_begin, .port_fwd_end =NULL , + .stream_init = tx_only_stream_init, .packet_fwd = pkt_burst_transmit, };
Signature · 1 line
2.25.1
RE: [PATCH v3 1/2] app/testpmd: do not poll stopped queues
Quoted 12 lines
Calling Rx/Tx functions on a stopped queue is not supported. Do not run packet forwarding for streams that use stopped queues.
Each stream has a read-only "disabled" field, so that lcore function can skip such streams. Forwarding engines can set this field
using a new "stream_init" callback function by checking relevant queue states,
which are stored along with queue configurations (not all PMDs implement rte_eth_rx/tx_queue_info_get() to query the state from there).
Quoted 6 lines
Fixes 5f4ec54f1d16 ("testpmd: queue start and stop")Signed-off-by Dmitry Kozlyuk <[email protected]>Acked-by Matan Azrad <[email protected]>---
Pinging testpmd maintainers for this bugfix.
The sibling ethdev patch will have to wait until the next release cycle.
Re: [PATCH v3 1/2] app/testpmd: do not poll stopped queues
Quoted 15 lines · On Wed, 9 Mar 2022 10:36:15 +0000 Dmitry Kozlyuk <[email protected]> wrote:
Quoted 15 lines
Calling Rx/Tx functions on a stopped queue is not supported. Do not run packet forwarding for streams that use stopped queues.
Each stream has a read-only "disabled" field, so that lcore function can skip such streams. Forwarding engines can set this field
using a new "stream_init" callback function by checking relevant queue states,
which are stored along with queue configurations (not all PMDs implement rte_eth_rx/tx_queue_info_get() to query the state from there).Fixes 5f4ec54f1d16 ("testpmd: queue start and stop")
Quoted 6 lines
Quoted 3 lines
Signed-off-by Dmitry Kozlyuk <[email protected]>Acked-by Matan Azrad <[email protected]>---
Pinging testpmd maintainers for this bugfix.
The sibling ethdev patch will have to wait until the next release cycle.
So you broke a bunch of drivers now that is merged???
Sound like it needs to be reverted. Certainly not something that should go into stable tree if you expect all drivers to correctly set queue state.
Thomas Monjalon
AppliedRe: [PATCH v3 1/2] app/testpmd: do not poll stopped queues
In reply to Dmitry Kozlyuk07/03/2022 13:53, Dmitry Kozlyuk:
Quoted 12 lines
Calling Rx/Tx functions on a stopped queue is not supported. Do not run packet forwarding for streams that use stopped queues.
Each stream has a read-only "disabled" field, so that lcore function can skip such streams. Forwarding engines can set this field
using a new "stream_init" callback function by checking relevant queue states,
which are stored along with queue configurations (not all PMDs implement rte_eth_rx/tx_queue_info_get() to query the state from there).
Quoted 5 lines
Fixes 5f4ec54f1d16 ("testpmd: queue start and stop")Signed-off-by Dmitry Kozlyuk <[email protected]>Acked-by Matan Azrad <[email protected]>
This patch is waiting for long without any comment.
Applied in next-net, thanks.
RE: [PATCH v3 1/2] app/testpmd: do not poll stopped queues
Hi Kozlyuk,
Here is bugzilla ticket https://bugs.dpdk.org/show_bug.cgi?id=1028, its bad commit id is this patch. Could you please have a look? Thanks so much.
Best regards,
Yu Jiang
Quoted 31 lines
-----Original Message-----
From: Thomas Monjalon <[email protected]>
Sent: Wednesday, May 25, 2022 11:46 PM
To: Dmitry Kozlyuk <[email protected]>
Cc: [email protected]; [email protected]; Matan Azrad <[email protected]>; Li, Xiaoyun <[email protected]>; Singh, Aman Deep <[email protected]>; Zhang, Yuying <[email protected]>
Subject: Re: [PATCH v3 1/2] app/testpmd: do not poll stopped queues07/03/2022 13:53, Dmitry Kozlyuk:
Quoted 11 lines
Calling Rx/Tx functions on a stopped queue is not supported. Do not run packet forwarding for streams that use stopped queues.
Each stream has a read-only "disabled" field, so that lcore function can skip such streams.
Forwarding engines can set this field
using a new "stream_init" callback function by checking relevant queue states, which are stored along with queue configurations (not all PMDs implement rte_eth_rx/tx_queue_info_get() to query the state from there).Quoted 5 lines
Fixes 5f4ec54f1d16 ("testpmd: queue start and stop")Signed-off-by Dmitry Kozlyuk <[email protected]>Acked-by Matan Azrad <[email protected]>This patch is waiting for long without any comment.
Applied in next-net, thanks.
PATCH v3 2/2 ethdev: prohibit polling of a stopped queue
In reply to Dmitry KozlyukWhether it is allowed to call Rx/Tx functions for a stopped queue was undocumented. Some PMDs make this behavior a no-op either by explicitly checking the queue state
or by the way how their routines are implemented or HW works.
No-op behavior may be convenient for application developers. But it also means that pollers of stopped queues
would go all the way down to PMD Rx/Tx routines, wasting cycles. Some PMDs would do a check for the queue state on data path, even though it may never be needed for a particular application. Also, use cases for stopping queues or starting them deferred do not logically require polling stopped queues.
Use case 1: a secondary that was polling the queue has crashed, the primary is doing a recovery to free all mbufs.
By definition the queue to be restarted is not polled.
Use case 2: deferred queue start or queue reconfiguration. The polling thread must be synchronized anyway, because queue start and stop are non-atomic.
Prohibit calling Rx/Tx functions on stopped queues.
--- lib/ethdev/rte_ethdev.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
lib/ethdev/rte_ethdev.h +1 -1
@@ -74,7 +74,7 @@* rte_eth_rx_queue_setup()), it must call rte_eth_dev_stop() first to stop the * device and then do the reconfiguration before calling rte_eth_dev_start() * again. The transmit and receive functions should not be invoked when the -* device is stopped. +* device is stopped or when the queue is stopped (for that queue). * * Please note that some configuration is not stored between calls to * rte_eth_dev_stop()/rte_eth_dev_start(). The following configuration will
Signature · 1 line
2.25.1