net/mlx5: implement vectorized MPRQ burst
PATCH 1/1 net/mlx5: implement vectorized MPRQ burst
MPRQ (Multi-Packet Rx Queue) processes one packet at the time using simple scalar instructions. MPRQ works by posting a single large buffer (consisted of multiple fixed-size strides) in order to receive multiple packets at once on this buffer. A Rx packet is then copied to a user-provided mbuf or PMD attaches the Rx packet to the mbuf by the pointer to an external buffer.
There is an opportunity to speed up the packet receiving by processing 4 packets simultaneously using SIMD (single instruction, multiple data) extensions. Allocate mbufs in batches for every MPRQ buffer and process the packets in the groups of 4 until all the strides are exhausted. Then switch to another MPRQ buffer and repeat the process over again.
The vectorized MPRQ burst routine is engaged automatically in case the mprq_en=1 devarg is specified and the vectorization is not disabled explicitly by providing rx_vec_en=0 devarg. There are two limitations:
- LRO is not supported and scalar MPRQ is selected if it is on.
- CQE compression is disabled in case vectorized MPRQ is engaged.
--- drivers/net/mlx5/linux/mlx5_os.c | 4 + drivers/net/mlx5/mlx5_ethdev.c | 12 +- drivers/net/mlx5/mlx5_rxq.c | 80 +-- drivers/net/mlx5/mlx5_rxtx.c | 30 +- drivers/net/mlx5/mlx5_rxtx.h | 9 +- drivers/net/mlx5/mlx5_rxtx_vec.c | 38 +- drivers/net/mlx5/mlx5_rxtx_vec.h | 21 + drivers/net/mlx5/mlx5_rxtx_vec_altivec.h | 724 +++++++++++++++++++++++ drivers/net/mlx5/mlx5_rxtx_vec_neon.h | 577 ++++++++++++++++++ drivers/net/mlx5/mlx5_rxtx_vec_sse.h | 520 ++++++++++++++++ 10 files changed, 1968 insertions(+), 47 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +4 -0
@@ -568,6 +568,8 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev, cqe_comp =0 ; else cqe_comp =1 ; +if (config.mprq.enabled) + cqe_comp =0 ; config.cqe_comp = cqe_comp;#ifdef HAVE_IBV_MLX5_MOD_CQE_128B_PAD /* Whether device supports 128B Rx CQE padding. */
@@ -973,6 +975,8 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev," setting default value (%u)" , 1 << config.mprq.stride_num_n); } +if (config.mprq.stride_size_n) + config.rx_vec_en = false;if (config.mprq.stride_size_n && (config.mprq.stride_size_n > mprq_max_stride_size_n || config.mprq.stride_size_n < mprq_min_stride_size_n)) {
drivers/net/mlx5/mlx5_ethdev.c +10 -2
@@ -421,7 +421,8 @@ mlx5_dev_supported_ptypes_get(struct rte_eth_dev *dev)if (dev->rx_pkt_burst == mlx5_rx_burst || dev->rx_pkt_burst == mlx5_rx_burst_mprq || - dev->rx_pkt_burst == mlx5_rx_burst_vec) + dev->rx_pkt_burst == mlx5_rx_burst_vec || + dev->rx_pkt_burst == mlx5_rx_burst_mprq_vec)return ptypes;return NULL ; }
@@ -479,12 +480,19 @@ mlx5_select_rx_function(struct rte_eth_dev *dev)eth_rx_burst_t rx_pkt_burst = mlx5_rx_burst;MLX5_ASSERT (dev !=NULL ); -if (mlx5_check_vec_rx_support (dev) >0 ) { +if (mlx5_check_vec_rx_support (dev) >0 && +mlx5_mprq_enabled (dev)) { + rx_pkt_burst = mlx5_rx_burst_mprq_vec; +DRV_LOG (DEBUG ,"port %u selected Multi-Packet Rx vectorized function" , + dev->data->port_id); + }else if (mlx5_check_vec_rx_support (dev) >0 ) { rx_pkt_burst = mlx5_rx_burst_vec;DRV_LOG (DEBUG ,"port %u selected Rx vectorized function" , dev->data->port_id); }else if (mlx5_mprq_enabled (dev)) { rx_pkt_burst = mlx5_rx_burst_mprq; +DRV_LOG (DEBUG ,"port %u selected Multi-Packet Rx function" , + dev->data->port_id); }return rx_pkt_burst; }
drivers/net/mlx5/mlx5_rxq.c +47 -33
173 lines not on this page · raw message
drivers/net/mlx5/mlx5_rxtx.c +27 -3
@@ -614,6 +614,16 @@ mlx5_rx_burst_mode_get(struct rte_eth_dev *dev,snprintf (mode->info,sizeof (mode->info),"%s" ,"Vector AltiVec" );#else return -EINVAL ; +#endif + }else if (pkt_burst == mlx5_rx_burst_mprq_vec) { +#if definedRTE_ARCH_X86_64 +snprintf (mode->info,sizeof (mode->info),"%s" ,"Multi-Packet RQ Vector SSE" ); +#elif definedRTE_ARCH_ARM64 +snprintf (mode->info,sizeof (mode->info),"%s" ,"Multi-Packet RQ Vector Neon" ); +#elif definedRTE_ARCH_PPC_64 +snprintf (mode->info,sizeof (mode->info),"%s" ,"Multi-Packet RQ Vector AltiVec" ); +#else +return -EINVAL ;#endif }else {return -EINVAL ;
@@ -1075,7 +1085,7 @@ mlx5_rx_err_handle(struct mlx5_rxq_data *rxq, uint8_t vec) {const uint16_t cqe_n =1 << rxq->cqe_n;const uint16_t cqe_mask = cqe_n -1 ; -const unsigned int wqe_n =1 << rxq->elts_n; +unsigned int wqe_n =1 << rxq->elts_n;struct mlx5_rxq_ctrl *rxq_ctrl = container_of(rxq, struct mlx5_rxq_ctrl, rxq);union {
@@ -1139,11 +1149,17 @@ mlx5_rx_err_handle(struct mlx5_rxq_data *rxq, uint8_t vec) &sm))return -1 ;if (vec) { -const uint16_t q_mask = wqe_n -1 ; +uint16_t q_mask;uint16_t elt_idx;struct rte_mbuf **elt;int i; -unsigned int n = wqe_n - (rxq->rq_ci - +unsigned int n; + +if (mlx5_rxq_mprq_enabled (&rxq_ctrl->rxq)) + wqe_n *= (1U << + rxq_ctrl->rxq.strd_num_n); + q_mask = wqe_n -1 ; + n = wqe_n - (rxq->rq_ci - rxq->rq_pi);for (i =0 ; i < (int )n; ++i) {
@@ -1982,6 +1998,14 @@ mlx5_rx_burst_vec(void *dpdk_txq __rte_unused,return 0 ; } +__rte_weak uint16_t +mlx5_rx_burst_mprq_vec(void *dpdk_txq __rte_unused, + struct rte_mbuf **pkts __rte_unused, + uint16_t pkts_n __rte_unused) +{ +return 0 ; +} + __rte_weak int mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq __rte_unused) {
drivers/net/mlx5/mlx5_rxtx.h +4 -5
@@ -141,11 +141,8 @@ struct mlx5_rxq_data {uint16_t mprq_max_memcpy_len;/* Maximum size of packet to memcpy. */ volatile void *wqes;volatile struct mlx5_cqe (*cqes)[]; -RTE_STD_C11 - union { -struct rte_mbuf *(*elts)[]; -struct mlx5_mprq_buf *(*mprq_bufs)[]; - }; +struct rte_mbuf *(*elts)[]; +struct mlx5_mprq_buf *(*mprq_bufs)[];struct rte_mempool *mp;struct rte_mempool *mprq_mp;/* Mempool for Multi-Packet RQ. */ struct mlx5_mprq_buf *mprq_repl;/* Stashed mbuf for replenish. */
@@ -518,6 +515,8 @@ int mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq_data);int mlx5_check_vec_rx_support (struct rte_eth_dev *dev);uint16_t mlx5_rx_burst_vec (void *dpdk_txq,struct rte_mbuf **pkts,uint16_t pkts_n); +uint16_t mlx5_rx_burst_mprq_vec (void *dpdk_txq,struct rte_mbuf **pkts, +uint16_t pkts_n);/* mlx5_mr.c */
drivers/net/mlx5/mlx5_rxtx_vec.c +34 -4
@@ -119,6 +119,40 @@ mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)return tn; } +/** +* DPDK callback for MPRQ vectorized RX. +* +* @param dpdk_rxq +* Generic pointer to RX queue structure. +* @param[out] pkts +* Array to store received packets. +* @param pkts_n +* Maximum number of packets in array. +* +* @return +* Number of packets successfully received (<= pkts_n). +*/ +uint16_t +mlx5_rx_burst_mprq_vec (void *dpdk_rxq,struct rte_mbuf **pkts,uint16_t pkts_n) +{ +struct mlx5_rxq_data *rxq = dpdk_rxq; +uint16_t nb_rx =0 ; +uint16_t tn =0 ; +uint64_t err =0 ; +bool no_cq = false; + +do { + nb_rx =rxq_burst_mprq_v (rxq, pkts + tn, pkts_n - tn, + &err, &no_cq); +if (unlikely (err | rxq->err_state)) + nb_rx =rxq_handle_pending_error (rxq, pkts + tn, nb_rx); + tn += nb_rx; +if (unlikely (no_cq)) +break ; + }while (tn != pkts_n); +return tn; +} +/** * Check a RX queue can support vectorized RX. *
@@ -134,8 +168,6 @@ mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq)struct mlx5_rxq_ctrl *ctrl = container_of(rxq, struct mlx5_rxq_ctrl, rxq); -if (mlx5_mprq_enabled (ETH_DEV (ctrl->priv))) -return -ENOTSUP ;if (!ctrl->priv->config.rx_vec_en || rxq->sges_n !=0 )return -ENOTSUP ;if (rxq->lro)
@@ -160,8 +192,6 @@ mlx5_check_vec_rx_support(struct rte_eth_dev *dev)if (!priv->config.rx_vec_en)return -ENOTSUP ; -if (mlx5_mprq_enabled (dev)) -return -ENOTSUP ;/* All the configured queues should support. */ for (i =0 ; i < priv->rxqs_n; ++i) {struct mlx5_rxq_data *rxq = (*priv->rxqs)[i];
drivers/net/mlx5/mlx5_rxtx_vec.h +21 -0
@@ -122,4 +122,25 @@ mlx5_rx_replenish_bulk_mbuf(struct mlx5_rxq_data *rxq, uint16_t n) *rxq->rq_db =rte_cpu_to_be_32 (rxq->rq_ci); } +static inline void +mlx5_rx_replenish_bulk_mprq_mbuf (struct mlx5_rxq_data *rxq, +uint16_t n,uint32_t rq_idx) +{ +const unsigned int strd_n =1 << rxq->strd_num_n; +uint16_t elts_idx = rq_idx * strd_n + + rq_idx *MLX5_VPMD_DESCS_PER_LOOP ; +struct rte_mbuf **elts = &(*rxq->elts)[elts_idx]; +unsigned int i; + + n =RTE_MIN (n, strd_n - rxq->consumed_strd); +if (rte_mempool_get_bulk (rxq->mp, (void *)elts, n) <0 ) { + rxq->stats.rx_nombuf += n; +return ; + } + rxq->rq_repl_thresh =0 ; +/* Prevent overflowing into the next MPRQ mbufs. */ +for (i =0 ; i <MLX5_VPMD_DESCS_PER_LOOP ; ++i) + (*rxq->elts)[elts_idx + strd_n + i] = &rxq->fake_mbuf; +} +#endif /* RTE_PMD_MLX5_RXTX_VEC_H_ */
drivers/net/mlx5/mlx5_rxtx_vec_altivec.h +724 -0
734 lines not on this page · raw message
drivers/net/mlx5/mlx5_rxtx_vec_neon.h +577 -0
587 lines not on this page · raw message
drivers/net/mlx5/mlx5_rxtx_vec_sse.h +520 -0
530 lines not on this page · raw message
Signature · 1 line
2.24.1
v2 cover net/mlx5: add vectorized mprq
The vectorized Rx burst function helps to accelerate the Rx processing by using SIMD (single instruction, multiple data) extensions for the multi-buffer packet processing. Pre-allocating multiple mbufs and filling them in batches of four greatly improves the throughput of the Rx burst routine.
MPRQ (Multi-Packet Rx Queue) lacks the vectorized version currently. It works by posting a single large buffer (consisted of multiple fixed-size strides) in order to receive multiple packets at once on this buffer. A Rx packet is then copied to a user-provided mbuf or PMD attaches the Rx packet to the mbuf by the pointer to an external buffer.
It is proposed to add a vectorized MPRQ Rx routine to speed up the MPRQ buffer handling as well. It would require pre-allocation of multiple mbufs every time we exhaust all the strides from the current MPRQ buffer and switch to a new one. The new mlx5_rx_burst_mprq_vec() routine will take care of this as well as of decision on whether should we copy or attach an external buffer for a packet. The batch processing logic won't be different from the simple vectorized Rx routine.
The new vectorized MPRQ burst function is going to be selected automatically whenever the mprq_en devarg is specified. If SIMD is not available on the platform we fall back to the simple MPRQ Rx burst function. LRO is not supported by the vectorized MPRQ version and fall back to the regular MPRQ will be performed.
Alexander Kozyrev (2):
net/mlx5: refactor vectorized Rx routine
net/mlx5: implement vectorized MPRQ burst
drivers/net/mlx5/mlx5_devx.c | 15 +- drivers/net/mlx5/mlx5_ethdev.c | 20 +- drivers/net/mlx5/mlx5_rxq.c | 96 +++--- drivers/net/mlx5/mlx5_rxtx.c | 237 ++++--------- drivers/net/mlx5/mlx5_rxtx.h | 200 ++++++++++- drivers/net/mlx5/mlx5_rxtx_vec.c | 416 ++++++++++++++++++++++- drivers/net/mlx5/mlx5_rxtx_vec.h | 55 --- drivers/net/mlx5/mlx5_rxtx_vec_altivec.h | 106 ++---- drivers/net/mlx5/mlx5_rxtx_vec_neon.h | 103 ++---- drivers/net/mlx5/mlx5_rxtx_vec_sse.h | 121 ++----- 10 files changed, 813 insertions(+), 556 deletions(-)
Signature · 1 line
2.24.1
PATCH v2 1/2 net/mlx5: refactor vectorized Rx routine
Move the main processing cycle into a separate function:rxq_cq_process_v. Put the regular rxq_burst_v function to a non-arch specific file. Having all SIMD instructions in a single reusable block is a first preparatory step to implement vectorized Rx burst for MPRQ feature.
Pass a pointer to the storage of mbufs directly to the rxq_copy_mbuf_v instead of calculating the pointer inside this function. This is needed for the future vectorized Rx routing which is going to pass a different pointer here.
Calculate the number of packets to replenish inside the mlx5_rx_replenish_bulk_mbuf. Containing this logic in one place allows us to do the same for MPRQ case.
--- drivers/net/mlx5/mlx5_rxtx_vec.c | 104 +++++++++++++++++++ drivers/net/mlx5/mlx5_rxtx_vec.h | 69 ++++++------- drivers/net/mlx5/mlx5_rxtx_vec_altivec.h | 106 ++++---------------- drivers/net/mlx5/mlx5_rxtx_vec_neon.h | 103 ++++--------------- drivers/net/mlx5/mlx5_rxtx_vec_sse.h | 121 +++++------------------ 5 files changed, 204 insertions(+), 299 deletions(-)
drivers/net/mlx5/mlx5_rxtx_vec.c +104 -0
@@ -77,6 +77,110 @@ rxq_handle_pending_error(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts,return n; } +/** +* Receive burst of packets. An errored completion also consumes a mbuf, but the +* packet_type is set to be RTE_PTYPE_ALL_MASK. Marked mbufs should be freed +* before returning to application. +* +* @param rxq +* Pointer to RX queue structure. +* @param[out] pkts +* Array to store received packets. +* @param pkts_n +* Maximum number of packets in array. +* @param[out] err +* Pointer to a flag. Set non-zero value if pkts array has at least one error +* packet to handle. +* @param[out] no_cq +* Pointer to a boolean. Set true if no new CQE seen. +* +* @return +* Number of packets received including errors (<= pkts_n). +*/ +static inline uint16_t +rxq_burst_v (struct mlx5_rxq_data *rxq,struct rte_mbuf **pkts, +uint16_t pkts_n,uint64_t *err,bool *no_cq) +{ +const uint16_t q_n =1 << rxq->cqe_n; +const uint16_t q_mask = q_n -1 ; +const uint16_t e_n =1 << rxq->elts_n; +const uint16_t e_mask = e_n -1 ; +volatile struct mlx5_cqe *cq; +struct rte_mbuf **elts; +uint64_t comp_idx =MLX5_VPMD_DESCS_PER_LOOP ; +uint16_t nocmp_n =0 ; +uint16_t rcvd_pkt =0 ; +unsigned int cq_idx = rxq->cq_ci & q_mask; +unsigned int elts_idx; + +MLX5_ASSERT (rxq->sges_n ==0 ); +MLX5_ASSERT (rxq->cqe_n == rxq->elts_n); + cq = &(*rxq->cqes)[cq_idx]; +rte_prefetch0 (cq); +rte_prefetch0 (cq +1 ); +rte_prefetch0 (cq +2 ); +rte_prefetch0 (cq +3 ); + pkts_n =RTE_MIN (pkts_n,MLX5_VPMD_RX_MAX_BURST ); +mlx5_rx_replenish_bulk_mbuf (rxq); +/* See if there're unreturned mbufs from compressed CQE. */ + rcvd_pkt = rxq->decompressed; +if (rcvd_pkt >0 ) { + rcvd_pkt =RTE_MIN (rcvd_pkt, pkts_n); +rxq_copy_mbuf_v (&(*rxq->elts)[rxq->rq_pi & e_mask], + pkts, rcvd_pkt); + rxq->rq_pi += rcvd_pkt; + rxq->decompressed -= rcvd_pkt; + pkts += rcvd_pkt; + } + elts_idx = rxq->rq_pi & e_mask; + elts = &(*rxq->elts)[elts_idx]; +/* Not to overflow pkts array. */ + pkts_n =RTE_ALIGN_FLOOR (pkts_n - rcvd_pkt,MLX5_VPMD_DESCS_PER_LOOP ); +/* Not to cross queue end. */ + pkts_n =RTE_MIN (pkts_n, q_n - elts_idx); + pkts_n =RTE_MIN (pkts_n, q_n - cq_idx); +if (!pkts_n) { + *no_cq = !rcvd_pkt; +return rcvd_pkt; + } +/* At this point, there shouldn't be any remaining packets. */ +MLX5_ASSERT (rxq->decompressed ==0 ); +/* Process all the CQEs */ + nocmp_n =rxq_cq_process_v (rxq, cq, elts, pkts, pkts_n, err, &comp_idx); +/* If no new CQE seen, return without updating cq_db. */ +if (unlikely (!nocmp_n && comp_idx ==MLX5_VPMD_DESCS_PER_LOOP )) { + *no_cq = true; +return rcvd_pkt; + } +/* Update the consumer indexes for non-compressed CQEs. */ +MLX5_ASSERT (nocmp_n <= pkts_n); + rxq->cq_ci += nocmp_n; + rxq->rq_pi += nocmp_n; + rcvd_pkt += nocmp_n; +/* Decompress the last CQE if compressed. */ +if (comp_idx <MLX5_VPMD_DESCS_PER_LOOP ) { +MLX5_ASSERT (comp_idx == (nocmp_n %MLX5_VPMD_DESCS_PER_LOOP )); + rxq->decompressed =rxq_cq_decompress_v (rxq, &cq[nocmp_n], + &elts[nocmp_n]); + rxq->cq_ci += rxq->decompressed; +/* Return more packets if needed. */ +if (nocmp_n < pkts_n) { +uint16_t n = rxq->decompressed; + + n =RTE_MIN (n, pkts_n - nocmp_n); +rxq_copy_mbuf_v (&(*rxq->elts)[rxq->rq_pi & e_mask], + &pkts[nocmp_n], n); + rxq->rq_pi += n; + rcvd_pkt += n; + rxq->decompressed -= n; + } + } +rte_io_wmb (); + *rxq->cq_db =rte_cpu_to_be_32 (rxq->cq_ci); + *no_cq = !rcvd_pkt; +return rcvd_pkt; +} +/** * DPDK callback for vectorized RX. *
drivers/net/mlx5/mlx5_rxtx_vec.h +35 -34
88 lines not on this page · raw message
drivers/net/mlx5/mlx5_rxtx_vec_altivec.h +20 -86
161 lines not on this page · raw message
drivers/net/mlx5/mlx5_rxtx_vec_neon.h +20 -83
157 lines not on this page · raw message
drivers/net/mlx5/mlx5_rxtx_vec_sse.h +25 -96
188 lines not on this page · raw message
Signature · 1 line
2.24.1
PATCH v2 2/2 net/mlx5: implement vectorized MPRQ burst
In reply to Alexander KozyrevMPRQ (Multi-Packet Rx Queue) processes one packet at a time using simple scalar instructions. MPRQ works by posting a single large buffer (consisted of multiple fixed-size strides) in order to receive multiple packets at once on this buffer. A Rx packet is then copied to a user-provided mbuf or PMD attaches the Rx packet to the mbuf by the pointer to an external buffer.
There is an opportunity to speed up the packet receiving by processing 4 packets simultaneously using SIMD (single instruction, multiple data) extensions. Allocate mbufs in batches for every MPRQ buffer and process the packets in groups of 4 until all the strides are exhausted. Then switch to another MPRQ buffer and repeat the process over again.
The vectorized MPRQ burst routine is engaged automatically in case the mprq_en=1 devarg is specified and the vectorization is not disabled explicitly by providing rx_vec_en=0 devarg. There is a limitation:
LRO is not supported and scalar MPRQ is selected if it is on.
--- drivers/net/mlx5/mlx5_devx.c | 15 +- drivers/net/mlx5/mlx5_ethdev.c | 20 +- drivers/net/mlx5/mlx5_rxq.c | 96 ++++++---- drivers/net/mlx5/mlx5_rxtx.c | 237 ++++++----------------- drivers/net/mlx5/mlx5_rxtx.h | 200 +++++++++++++++++++- drivers/net/mlx5/mlx5_rxtx_vec.c | 312 ++++++++++++++++++++++++++++++- drivers/net/mlx5/mlx5_rxtx_vec.h | 56 ------ 7 files changed, 644 insertions(+), 292 deletions(-)
drivers/net/mlx5/mlx5_devx.c +11 -4
@@ -437,10 +437,17 @@ mlx5_rxq_create_devx_cq_resources(struct rte_eth_dev *dev, uint16_t idx)if (priv->config.cqe_comp && !rxq_data->hw_timestamp && !rxq_data->lro) { cq_attr.cqe_comp_en =1u ; - cq_attr.mini_cqe_res_format = -mlx5_rxq_mprq_enabled (rxq_data) ? -MLX5_CQE_RESP_FORMAT_CSUM_STRIDX : -MLX5_CQE_RESP_FORMAT_HASH ; +/* +* Select CSUM miniCQE format only for non-vectorized MPRQ +* Rx burst, use HASH miniCQE format for everything else. +*/ +if (mlx5_rxq_check_vec_support (rxq_data) <0 && +mlx5_rxq_mprq_enabled (rxq_data)) + cq_attr.mini_cqe_res_format = +MLX5_CQE_RESP_FORMAT_CSUM_STRIDX ; +else + cq_attr.mini_cqe_res_format = +MLX5_CQE_RESP_FORMAT_HASH ;/* * For vectorized Rx, it must not be doubled in order to * make cq_ci and rq_ci aligned.
drivers/net/mlx5/mlx5_ethdev.c +16 -4
@@ -421,7 +421,8 @@ mlx5_dev_supported_ptypes_get(struct rte_eth_dev *dev)if (dev->rx_pkt_burst == mlx5_rx_burst || dev->rx_pkt_burst == mlx5_rx_burst_mprq || - dev->rx_pkt_burst == mlx5_rx_burst_vec) + dev->rx_pkt_burst == mlx5_rx_burst_vec || + dev->rx_pkt_burst == mlx5_rx_burst_mprq_vec)return ptypes;return NULL ; }
@@ -480,11 +481,22 @@ mlx5_select_rx_function(struct rte_eth_dev *dev)MLX5_ASSERT (dev !=NULL );if (mlx5_check_vec_rx_support (dev) >0 ) { - rx_pkt_burst = mlx5_rx_burst_vec; -DRV_LOG (DEBUG ,"port %u selected Rx vectorized function" , - dev->data->port_id); +if (mlx5_mprq_enabled (dev)) { + rx_pkt_burst = mlx5_rx_burst_mprq_vec; +DRV_LOG (DEBUG ,"port %u selected vectorized" +" MPRQ Rx function" , dev->data->port_id); + }else { + rx_pkt_burst = mlx5_rx_burst_vec; +DRV_LOG (DEBUG ,"port %u selected vectorized" +" SPRQ Rx function" , dev->data->port_id); + } }else if (mlx5_mprq_enabled (dev)) { rx_pkt_burst = mlx5_rx_burst_mprq; +DRV_LOG (DEBUG ,"port %u selected MPRQ Rx function" , + dev->data->port_id); + }else { +DRV_LOG (DEBUG ,"port %u selected SPRQ Rx function" , + dev->data->port_id); }return rx_pkt_burst; }
drivers/net/mlx5/mlx5_rxq.c +55 -41
202 lines not on this page · raw message
drivers/net/mlx5/mlx5_rxtx.c +60 -177
358 lines not on this page · raw message
drivers/net/mlx5/mlx5_rxtx.h +195 -5
234 lines not on this page · raw message
drivers/net/mlx5/mlx5_rxtx_vec.c +307 -5
336 lines not on this page · raw message
drivers/net/mlx5/mlx5_rxtx_vec.h +0 -56
66 lines not on this page · raw message
Signature · 1 line
2.24.1
Re: [dpdk-dev] [PATCH v2 0/2] net/mlx5: add vectorized mprq
In reply to Alexander KozyrevHi,
Quoted 51 lines
-----Original Message-----
From: Alexander Kozyrev <[email protected]>
Sent: Wednesday, October 21, 2020 11:30 PM
To: [email protected]
Cc: Raslan Darawsheh <[email protected]>; Matan Azrad <[email protected]>; Slava Ovsiienko <[email protected]>
Subject: [PATCH v2 0/2] net/mlx5: add vectorized mprqThe vectorized Rx burst function helps to accelerate the Rx processing by using SIMD (single instruction, multiple data) extensions for the multi-buffer packet processing. Pre-allocating multiple mbufs and filling them in batches of four greatly improves the throughput of the Rx burst routine.
MPRQ (Multi-Packet Rx Queue) lacks the vectorized version currently. It works by posting a single large buffer (consisted of multiple fixed-size strides) in order to receive multiple packets at once on this buffer. A Rx packet is then copied to a user-provided mbuf or PMD attaches the Rx packet to the mbuf by the pointer to an external buffer.
It is proposed to add a vectorized MPRQ Rx routine to speed up the MPRQ buffer handling as well. It would require pre-allocation of multiple mbufs every time we exhaust all the strides from the current MPRQ buffer and switch to a new one. The new
mlx5_rx_burst_mprq_vec()routine will take care of this as well as of decision on whether should we copy or attach an external buffer for a packet. The batch processing logic won't be different from the simple vectorized Rx routine.The new vectorized MPRQ burst function is going to be selected automatically whenever the
mprq_endevarg is specified. If SIMD is not available on the platform we fall back to the simple MPRQ Rx burst function. LRO is not supported by the vectorized MPRQ version and fall back to the regular MPRQ will be performed.Alexander Kozyrev (2):
net/mlx5: refactor vectorized Rx routine
net/mlx5: implement vectorized MPRQ burstdrivers/net/mlx5/mlx5_devx.c | 15 +- drivers/net/mlx5/mlx5_ethdev.c | 20 +- drivers/net/mlx5/mlx5_rxq.c | 96 +++--- drivers/net/mlx5/mlx5_rxtx.c | 237 ++++--------- drivers/net/mlx5/mlx5_rxtx.h | 200 ++++++++++- drivers/net/mlx5/mlx5_rxtx_vec.c | 416 ++++++++++++++++++++++- drivers/net/mlx5/mlx5_rxtx_vec.h | 55 --- drivers/net/mlx5/mlx5_rxtx_vec_altivec.h | 106 ++---- drivers/net/mlx5/mlx5_rxtx_vec_neon.h | 103 ++---- drivers/net/mlx5/mlx5_rxtx_vec_sse.h | 121 ++----- 10 files changed, 813 insertions(+), 556 deletions(-)
Quoted 2 lines
--
2.24.1
Series applied to next-net-mlx,
Kindest regards,
Raslan Darawsheh