Reduce code duplication across Intel NIC drivers
Subsystems (2)
PATCH 17/21 net/iavf: use common Tx queue mbuf cleanup fn
In reply to Bruce RichardsonAdjust iavf driver to also use the common mbuf freeing functions on Tx queue release/cleanup. The implementation is complicated a little by the need to integrate the additional "has_ctx" parameter for the iavf code, but changes in other drivers are minimal - just a constant "false" parameter.
--- drivers/net/_common_intel/tx.h | 27 +++++++++--------- drivers/net/i40e/i40e_rxtx.c | 6 ++-- drivers/net/iavf/iavf_rxtx.c | 37 ++----------------------- drivers/net/iavf/iavf_rxtx_vec_avx512.c | 24 ++-------------- drivers/net/iavf/iavf_rxtx_vec_common.h | 18 ------------ drivers/net/iavf/iavf_rxtx_vec_sse.c | 9 ++---- drivers/net/ice/ice_dcf_ethdev.c | 4 +-- drivers/net/ice/ice_rxtx.c | 6 ++-- drivers/net/ixgbe/ixgbe_rxtx.c | 6 ++-- 9 files changed, 31 insertions(+), 106 deletions(-)
drivers/net/_common_intel/tx.h +13 -14
@@ -272,23 +272,23 @@ ci_tx_free_bufs_vec(struct ci_tx_queue *txq, ci_desc_done_fn desc_done, bool ctxreturn txq->tx_rs_thresh; } -#define IETH_FREE_BUFS_LOOP (txq, swr, start) do { \ +#define IETH_FREE_BUFS_LOOP (swr, nb_desc, start, end) do { \ uint16_t i = start; \ - if (txq->tx_tail < i) { \ - for (; i < txq->nb_tx_desc; i++) { \ + if (end < i) { \ + for (; i < nb_desc; i++) { \ rte_pktmbuf_free_seg(swr[i].mbuf); \ swr[i].mbuf = NULL; \ } \ i = 0; \ } \ - for (; i < txq->tx_tail; i++) { \ + for (; i < end; i++) { \ rte_pktmbuf_free_seg(swr[i].mbuf); \ swr[i].mbuf = NULL; \ } \ } while (0)static inline void -ci_txq_release_all_mbufs (struct ci_tx_queue *txq) +ci_txq_release_all_mbufs (struct ci_tx_queue *txq,bool use_ctx) {if (unlikely (!txq || !txq->sw_ring))return ;
@@ -307,15 +307,14 @@ ci_txq_release_all_mbufs(struct ci_tx_queue *txq)* vPMD tx will not set sw_ring's mbuf to NULL after free, * so need to free remains more carefully. */ -const uint16_t start = txq->tx_next_dd - txq->tx_rs_thresh +1 ; - -if (txq->vector_sw_ring) { -struct ci_tx_entry_vec *swr = txq->sw_ring_vec; -IETH_FREE_BUFS_LOOP (txq, swr, start); - }else { -struct ci_tx_entry *swr = txq->sw_ring; -IETH_FREE_BUFS_LOOP (txq, swr, start); - } +const uint16_t start = (txq->tx_next_dd - txq->tx_rs_thresh +1 ) >> use_ctx; +const uint16_t nb_desc = txq->nb_tx_desc >> use_ctx; +const uint16_t end = txq->tx_tail >> use_ctx; + +if (txq->vector_sw_ring) +IETH_FREE_BUFS_LOOP (txq->sw_ring_vec, nb_desc, start, end); +else + IETH_FREE_BUFS_LOOP(txq->sw_ring, nb_desc, start, end); }#endif /* _COMMON_INTEL_TX_H_ */
drivers/net/i40e/i40e_rxtx.c +3 -3
@@ -1933,7 +1933,7 @@ i40e_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)return err; } -ci_txq_release_all_mbufs (txq); +ci_txq_release_all_mbufs (txq, false);i40e_reset_tx_queue (txq); dev->data->tx_queue_state[tx_queue_id] =RTE_ETH_QUEUE_STATE_STOPPED ;
@@ -2608,7 +2608,7 @@ i40e_tx_queue_release(void *txq)return ; } -ci_txq_release_all_mbufs (q); +ci_txq_release_all_mbufs (q, false);rte_free (q->sw_ring);rte_memzone_free (q->mz);rte_free (q);
@@ -3071,7 +3071,7 @@ i40e_dev_clear_queues(struct rte_eth_dev *dev)for (i =0 ; i < dev->data->nb_tx_queues; i++) {if (!dev->data->tx_queues[i])continue ; -ci_txq_release_all_mbufs (dev->data->tx_queues[i]); +ci_txq_release_all_mbufs (dev->data->tx_queues[i], false);i40e_reset_tx_queue (dev->data->tx_queues[i]); }
drivers/net/iavf/iavf_rxtx.c +3 -34
@@ -387,24 +387,6 @@ release_rxq_mbufs(struct iavf_rx_queue *rxq) rxq->rx_nb_avail =0 ; } -static inline void -release_txq_mbufs (struct ci_tx_queue *txq) -{ -uint16_t i; - -if (!txq || !txq->sw_ring) { -PMD_DRV_LOG (DEBUG ,"Pointer to rxq or sw_ring is NULL" ); -return ; - } - -for (i =0 ; i < txq->nb_tx_desc; i++) { -if (txq->sw_ring[i].mbuf) { -rte_pktmbuf_free_seg (txq->sw_ring[i].mbuf); - txq->sw_ring[i].mbuf =NULL ; - } - } -} -static const struct iavf_rxq_ops iavf_rxq_release_mbufs_ops[] = { [IAVF_REL_MBUFS_DEFAULT ].release_mbufs = release_rxq_mbufs,
@@ -413,18 +395,6 @@ struct iavf_rxq_ops iavf_rxq_release_mbufs_ops[] = {#endif }; -static const -struct iavf_txq_ops iavf_txq_release_mbufs_ops[] = { - [IAVF_REL_MBUFS_DEFAULT ].release_mbufs = release_txq_mbufs, -#ifdef RTE_ARCH_X86 - [IAVF_REL_MBUFS_SSE_VEC ].release_mbufs = iavf_tx_queue_release_mbufs_sse, -#ifdef CC_AVX512_SUPPORT - [IAVF_REL_MBUFS_AVX512_VEC ].release_mbufs = iavf_tx_queue_release_mbufs_avx512, -#endif -#endif - -}; - static inline void iavf_rxd_to_pkt_fields_by_comms_ovs(__rte_unused struct iavf_rx_queue *rxq, struct rte_mbuf *mb,
@@ -889,7 +859,6 @@ iavf_dev_tx_queue_setup(struct rte_eth_dev *dev, txq->q_set = true; dev->data->tx_queues[queue_idx] = txq; txq->qtx_tail = hw->hw_addr +IAVF_QTX_TAIL1 (queue_idx); - txq->rel_mbufs_type =IAVF_REL_MBUFS_DEFAULT ;if (check_tx_vec_allow (txq) == false) { struct iavf_adapter *ad =
@@ -1068,7 +1037,7 @@ iavf_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id) } txq = dev->data->tx_queues[tx_queue_id]; - iavf_txq_release_mbufs_ops[txq->rel_mbufs_type].release_mbufs (txq); +ci_txq_release_all_mbufs (txq, txq->use_ctx);reset_tx_queue (txq); dev->data->tx_queue_state[tx_queue_id] =RTE_ETH_QUEUE_STATE_STOPPED ;
@@ -1097,7 +1066,7 @@ iavf_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid)if (!q)return ; - iavf_txq_release_mbufs_ops[q->rel_mbufs_type].release_mbufs (q); +ci_txq_release_all_mbufs (q, q->use_ctx);rte_free (q->sw_ring);rte_memzone_free (q->mz);rte_free (q);
@@ -1114,7 +1083,7 @@ iavf_reset_queues(struct rte_eth_dev *dev) txq = dev->data->tx_queues[i];if (!txq)continue ; - iavf_txq_release_mbufs_ops[txq->rel_mbufs_type].release_mbufs (txq); +ci_txq_release_all_mbufs (txq, txq->use_ctx);reset_tx_queue (txq); dev->data->tx_queue_state[i] =RTE_ETH_QUEUE_STATE_STOPPED ; }
drivers/net/iavf/iavf_rxtx_vec_avx512.c +2 -22
@@ -2357,31 +2357,11 @@ iavf_xmit_pkts_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts,return iavf_xmit_pkts_vec_avx512_cmn (tx_queue, tx_pkts, nb_pkts, false); } -void __rte_cold -iavf_tx_queue_release_mbufs_avx512 (struct ci_tx_queue *txq) -{ -unsigned int i; -const uint16_t max_desc = (uint16_t )(txq->nb_tx_desc -1 ); -const uint16_t end_desc = txq->tx_tail >> txq->use_ctx;/* next empty slot */ -const uint16_t wrap_point = txq->nb_tx_desc >> txq->use_ctx;/* end of SW ring */ -struct ci_tx_entry_vec *swr = (void *)txq->sw_ring; - -if (!txq->sw_ring || txq->nb_tx_free == max_desc) -return ; - - i = (txq->tx_next_dd - txq->tx_rs_thresh +1 ) >> txq->use_ctx; -while (i != end_desc) { -rte_pktmbuf_free_seg (swr[i].mbuf); - swr[i].mbuf =NULL ; -if (++i == wrap_point) - i =0 ; - } -} - int __rte_coldiavf_txq_vec_setup_avx512 (struct ci_tx_queue *txq) { - txq->rel_mbufs_type =IAVF_REL_MBUFS_AVX512_VEC ; + txq->vector_tx = true; + txq->vector_sw_ring = true;return 0 ; }
drivers/net/iavf/iavf_rxtx_vec_common.h +0 -18
@@ -60,24 +60,6 @@ _iavf_rx_queue_release_mbufs_vec(struct iavf_rx_queue *rxq)memset (rxq->sw_ring,0 ,sizeof (rxq->sw_ring[0 ]) * rxq->nb_rx_desc); } -static inline void -_iavf_tx_queue_release_mbufs_vec (struct ci_tx_queue *txq) -{ -unsigned i; -const uint16_t max_desc = (uint16_t )(txq->nb_tx_desc -1 ); - -if (!txq->sw_ring || txq->nb_tx_free == max_desc) -return ; - - i = txq->tx_next_dd - txq->tx_rs_thresh +1 ; -while (i != txq->tx_tail) { -rte_pktmbuf_free_seg (txq->sw_ring[i].mbuf); - txq->sw_ring[i].mbuf =NULL ; -if (++i == txq->nb_tx_desc) - i =0 ; - } -} -static inline int iavf_rxq_vec_setup_default (struct iavf_rx_queue *rxq) {
drivers/net/iavf/iavf_rxtx_vec_sse.c +2 -7
@@ -1458,16 +1458,11 @@ iavf_rx_queue_release_mbufs_sse(struct iavf_rx_queue *rxq)_iavf_rx_queue_release_mbufs_vec (rxq); } -void __rte_cold -iavf_tx_queue_release_mbufs_sse (struct ci_tx_queue *txq) -{ -_iavf_tx_queue_release_mbufs_vec (txq); -} - int __rte_coldiavf_txq_vec_setup (struct ci_tx_queue *txq) { - txq->rel_mbufs_type =IAVF_REL_MBUFS_SSE_VEC ; + txq->vector_tx = true; + txq->vector_sw_ring = false;return 0 ; }
drivers/net/ice/ice_dcf_ethdev.c +2 -2
@@ -501,7 +501,7 @@ ice_dcf_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id) } txq = dev->data->tx_queues[tx_queue_id]; -ci_txq_release_all_mbufs (txq); +ci_txq_release_all_mbufs (txq, false);reset_tx_queue (txq); dev->data->tx_queue_state[tx_queue_id] =RTE_ETH_QUEUE_STATE_STOPPED ;
@@ -651,7 +651,7 @@ ice_dcf_stop_queues(struct rte_eth_dev *dev) txq = dev->data->tx_queues[i];if (!txq)continue ; -ci_txq_release_all_mbufs (txq); +ci_txq_release_all_mbufs (txq, false);reset_tx_queue (txq); dev->data->tx_queue_state[i] =RTE_ETH_QUEUE_STATE_STOPPED ; }
drivers/net/ice/ice_rxtx.c +3 -3
@@ -1089,7 +1089,7 @@ ice_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)return -EINVAL ; } -ci_txq_release_all_mbufs (txq); +ci_txq_release_all_mbufs (txq, false);ice_reset_tx_queue (txq); dev->data->tx_queue_state[tx_queue_id] =RTE_ETH_QUEUE_STATE_STOPPED ;
@@ -1152,7 +1152,7 @@ ice_fdir_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)return -EINVAL ; } -ci_txq_release_all_mbufs (txq); +ci_txq_release_all_mbufs (txq, false); txq->qtx_tail =NULL ;return 0 ;
@@ -1531,7 +1531,7 @@ ice_tx_queue_release(void *txq)return ; } -ci_txq_release_all_mbufs (q); +ci_txq_release_all_mbufs (q, false);rte_free (q->sw_ring);rte_memzone_free (q->mz);rte_free (q);
drivers/net/ixgbe/ixgbe_rxtx.c +3 -3
@@ -2457,7 +2457,7 @@ static void __rte_cold ixgbe_tx_queue_release(struct ci_tx_queue *txq) {if (txq !=NULL && txq->ops !=NULL ) { -ci_txq_release_all_mbufs (txq); +ci_txq_release_all_mbufs (txq, false); txq->ops->free_swring (txq);rte_memzone_free (txq->mz);rte_free (txq);
@@ -3364,7 +3364,7 @@ ixgbe_dev_clear_queues(struct rte_eth_dev *dev)struct ci_tx_queue *txq = dev->data->tx_queues[i];if (txq !=NULL ) { -ci_txq_release_all_mbufs (txq); +ci_txq_release_all_mbufs (txq, false); txq->ops->reset (txq); dev->data->tx_queue_state[i] =RTE_ETH_QUEUE_STATE_STOPPED ; }
@@ -5639,7 +5639,7 @@ ixgbe_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id) }if (txq->ops !=NULL ) { -ci_txq_release_all_mbufs (txq); +ci_txq_release_all_mbufs (txq, false); txq->ops->reset (txq); } dev->data->tx_queue_state[tx_queue_id] =RTE_ETH_QUEUE_STATE_STOPPED ;
Signature · 1 line
2.43.0
PATCH 18/21 net/ice: use vector SW ring for all vector paths
In reply to Bruce RichardsonThe AVX-512 code path used a smaller SW ring structure only containing the mbuf pointer, but no other fields. The other fields are only used in the scalar code path, so update all vector driver code paths to use the smaller, faster structure.
--- drivers/net/_common_intel/tx.h | 7 +++++++ drivers/net/ice/ice_rxtx.c | 2 +- drivers/net/ice/ice_rxtx_vec_avx2.c | 12 ++++++------ drivers/net/ice/ice_rxtx_vec_avx512.c | 14 ++------------ drivers/net/ice/ice_rxtx_vec_common.h | 6 ------ drivers/net/ice/ice_rxtx_vec_sse.c | 12 ++++++------ 6 files changed, 22 insertions(+), 31 deletions(-)
drivers/net/_common_intel/tx.h +7 -0
@@ -109,6 +109,13 @@ ci_tx_backlog_entry(struct ci_tx_entry *txep, struct rte_mbuf **tx_pkts, uint16_ txep[i].mbuf = tx_pkts[i]; } +static __rte_always_inline void +ci_tx_backlog_entry_vec (struct ci_tx_entry_vec *txep,struct rte_mbuf **tx_pkts,uint16_t nb_pkts) +{ +for (uint16_t i =0 ; i < nb_pkts; ++i) + txep[i].mbuf = tx_pkts[i]; +} +#define IETH_VPMD_TX_MAX_FREE_BUF 64typedef int (*ci_desc_done_fn )(struct ci_tx_queue *txq,uint16_t idx);
drivers/net/ice/ice_rxtx.c +1 -1
@@ -825,7 +825,7 @@ ice_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)/* record what kind of descriptor cleanup we need on teardown */ txq->vector_tx = ad->tx_vec_allowed; - txq->vector_sw_ring = ad->tx_use_avx512; + txq->vector_sw_ring = txq->vector_tx; dev->data->tx_queue_state[tx_queue_id] =RTE_ETH_QUEUE_STATE_STARTED ;
drivers/net/ice/ice_rxtx_vec_avx2.c +6 -6
@@ -858,7 +858,7 @@ ice_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, {struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile struct ice_tx_desc *txdp; -struct ci_tx_entry *txep; +struct ci_tx_entry_vec *txep;uint16_t n, nb_commit, tx_id;uint64_t flags =ICE_TD_CMD ;uint64_t rs =ICE_TX_DESC_CMD_RS |ICE_TD_CMD ;
@@ -867,7 +867,7 @@ ice_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, nb_pkts =RTE_MIN (nb_pkts, txq->tx_rs_thresh);if (txq->nb_tx_free < txq->tx_free_thresh) -ice_tx_free_bufs_vec (txq); +ci_tx_free_bufs_vec (txq, ice_tx_desc_done, false); nb_commit = nb_pkts = (uint16_t )RTE_MIN (txq->nb_tx_free, nb_pkts);if (unlikely (nb_pkts ==0 ))
@@ -875,13 +875,13 @@ ice_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = txq->tx_tail; txdp = &txq->ice_tx_ring[tx_id]; - txep = &txq->sw_ring[tx_id]; + txep = &txq->sw_ring_vec[tx_id]; txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts); n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) { -ci_tx_backlog_entry (txep, tx_pkts, n); +ci_tx_backlog_entry_vec (txep, tx_pkts, n);ice_vtx (txdp, tx_pkts, n -1 , flags, offload); tx_pkts += (n -1 );
@@ -896,10 +896,10 @@ ice_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts,/* avoid reach the end of ring */ txdp = &txq->ice_tx_ring[tx_id]; - txep = &txq->sw_ring[tx_id]; + txep = &txq->sw_ring_vec[tx_id]; } -ci_tx_backlog_entry (txep, tx_pkts, nb_commit); +ci_tx_backlog_entry_vec (txep, tx_pkts, nb_commit);ice_vtx (txdp, tx_pkts, nb_commit, flags, offload);
drivers/net/ice/ice_rxtx_vec_avx512.c +2 -12
@@ -924,16 +924,6 @@ ice_vtx(volatile struct ice_tx_desc *txdp, struct rte_mbuf **pkt, } } -static __rte_always_inline void -ice_tx_backlog_entry_avx512 (struct ci_tx_entry_vec *txep, -struct rte_mbuf **tx_pkts,uint16_t nb_pkts) -{ -int i; - -for (i =0 ; i < (int )nb_pkts; ++i) - txep[i].mbuf = tx_pkts[i]; -} - static __rte_always_inline uint16_tice_xmit_fixed_burst_vec_avx512 (void *tx_queue,struct rte_mbuf **tx_pkts,uint16_t nb_pkts,bool do_offload)
@@ -964,7 +954,7 @@ ice_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) { -ice_tx_backlog_entry_avx512 (txep, tx_pkts, n); +ci_tx_backlog_entry_vec (txep, tx_pkts, n);ice_vtx (txdp, tx_pkts, n -1 , flags, do_offload); tx_pkts += (n -1 );
@@ -982,7 +972,7 @@ ice_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, txep = (void *)txq->sw_ring; } -ice_tx_backlog_entry_avx512 (txep, tx_pkts, nb_commit); +ci_tx_backlog_entry_vec (txep, tx_pkts, nb_commit);ice_vtx (txdp, tx_pkts, nb_commit, flags, do_offload);
drivers/net/ice/ice_rxtx_vec_common.h +0 -6
@@ -20,12 +20,6 @@ ice_tx_desc_done(struct ci_tx_queue *txq, uint16_t idx)rte_cpu_to_le_64 (ICE_TX_DESC_DTYPE_DESC_DONE ); } -static __rte_always_inline int -ice_tx_free_bufs_vec (struct ci_tx_queue *txq) -{ -return ci_tx_free_bufs (txq, ice_tx_desc_done); -} -static inline void _ice_rx_queue_release_mbufs_vec (struct ice_rx_queue *rxq) {
drivers/net/ice/ice_rxtx_vec_sse.c +6 -6
@@ -699,7 +699,7 @@ ice_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, {struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile struct ice_tx_desc *txdp; -struct ci_tx_entry *txep; +struct ci_tx_entry_vec *txep;uint16_t n, nb_commit, tx_id;uint64_t flags =ICE_TD_CMD ;uint64_t rs =ICE_TX_DESC_CMD_RS |ICE_TD_CMD ;
@@ -709,7 +709,7 @@ ice_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, nb_pkts =RTE_MIN (nb_pkts, txq->tx_rs_thresh);if (txq->nb_tx_free < txq->tx_free_thresh) -ice_tx_free_bufs_vec (txq); +ci_tx_free_bufs_vec (txq, ice_tx_desc_done, false); nb_pkts = (uint16_t )RTE_MIN (txq->nb_tx_free, nb_pkts); nb_commit = nb_pkts;
@@ -718,13 +718,13 @@ ice_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = txq->tx_tail; txdp = &txq->ice_tx_ring[tx_id]; - txep = &txq->sw_ring[tx_id]; + txep = &txq->sw_ring_vec[tx_id]; txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts); n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) { -ci_tx_backlog_entry (txep, tx_pkts, n); +ci_tx_backlog_entry_vec (txep, tx_pkts, n);for (i =0 ; i < n -1 ; ++i, ++tx_pkts, ++txdp)ice_vtx1 (txdp, *tx_pkts, flags);
@@ -738,10 +738,10 @@ ice_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,/* avoid reach the end of ring */ txdp = &txq->ice_tx_ring[tx_id]; - txep = &txq->sw_ring[tx_id]; + txep = &txq->sw_ring_vec[tx_id]; } -ci_tx_backlog_entry (txep, tx_pkts, nb_commit); +ci_tx_backlog_entry_vec (txep, tx_pkts, nb_commit);ice_vtx (txdp, tx_pkts, nb_commit, flags);
Signature · 1 line
2.43.0
PATCH 19/21 net/i40e: use vector SW ring for all vector paths
In reply to Bruce RichardsonThe AVX-512 code path used a smaller SW ring structure only containing the mbuf pointer, but no other fields. The other fields are only used in the scalar code path, so update all vector driver code paths (AVX2, SSE, Neon, Altivec) to use the smaller, faster structure.
--- drivers/net/i40e/i40e_rxtx.c | 8 +++++--- drivers/net/i40e/i40e_rxtx_vec_altivec.c | 12 ++++++------ drivers/net/i40e/i40e_rxtx_vec_avx2.c | 12 ++++++------ drivers/net/i40e/i40e_rxtx_vec_avx512.c | 14 ++------------ drivers/net/i40e/i40e_rxtx_vec_common.h | 6 ------ drivers/net/i40e/i40e_rxtx_vec_neon.c | 12 ++++++------ drivers/net/i40e/i40e_rxtx_vec_sse.c | 12 ++++++------ 7 files changed, 31 insertions(+), 45 deletions(-)
drivers/net/i40e/i40e_rxtx.c +5 -3
@@ -1891,7 +1891,7 @@ i40e_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id) tx_queue_id); txq->vector_tx = ad->tx_vec_allowed; - txq->vector_sw_ring = ad->tx_use_avx512; + txq->vector_sw_ring = txq->vector_tx;/* * tx_queue_id is queue id application refers to, while
@@ -3550,9 +3550,11 @@ i40e_set_tx_function(struct rte_eth_dev *dev) } } +if (rte_vect_get_max_simd_bitwidth () <RTE_VECT_SIMD_128 ) + ad->tx_vec_allowed = false; +if (ad->tx_simple_allowed) { -if (ad->tx_vec_allowed && -rte_vect_get_max_simd_bitwidth () >=RTE_VECT_SIMD_128 ) { +if (ad->tx_vec_allowed) {#ifdef RTE_ARCH_X86 if (ad->tx_use_avx512) {#ifdef CC_AVX512_SUPPORT
drivers/net/i40e/i40e_rxtx_vec_altivec.c +6 -6
@@ -553,14 +553,14 @@ i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, {struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile struct i40e_tx_desc *txdp; -struct ci_tx_entry *txep; +struct ci_tx_entry_vec *txep;uint16_t n, nb_commit, tx_id;uint64_t flags =I40E_TD_CMD ;uint64_t rs =I40E_TX_DESC_CMD_RS |I40E_TD_CMD ;int i;if (txq->nb_tx_free < txq->tx_free_thresh) -i40e_tx_free_bufs (txq); +ci_tx_free_bufs_vec (txq, i40e_tx_desc_done, false); nb_pkts = (uint16_t )RTE_MIN (txq->nb_tx_free, nb_pkts); nb_commit = nb_pkts;
@@ -569,13 +569,13 @@ i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = txq->tx_tail; txdp = &txq->i40e_tx_ring[tx_id]; - txep = &txq->sw_ring[tx_id]; + txep = &txq->sw_ring_vec[tx_id]; txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts); n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) { -ci_tx_backlog_entry (txep, tx_pkts, n); +ci_tx_backlog_entry_vec (txep, tx_pkts, n);for (i =0 ; i < n -1 ; ++i, ++tx_pkts, ++txdp)vtx1 (txdp, *tx_pkts, flags);
@@ -589,10 +589,10 @@ i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,/* avoid reach the end of ring */ txdp = &txq->i40e_tx_ring[tx_id]; - txep = &txq->sw_ring[tx_id]; + txep = &txq->sw_ring_vec[tx_id]; } -ci_tx_backlog_entry (txep, tx_pkts, nb_commit); +ci_tx_backlog_entry_vec (txep, tx_pkts, nb_commit);vtx (txdp, tx_pkts, nb_commit, flags);
drivers/net/i40e/i40e_rxtx_vec_avx2.c +6 -6
@@ -745,13 +745,13 @@ i40e_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, {struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile struct i40e_tx_desc *txdp; -struct ci_tx_entry *txep; +struct ci_tx_entry_vec *txep;uint16_t n, nb_commit, tx_id;uint64_t flags =I40E_TD_CMD ;uint64_t rs =I40E_TX_DESC_CMD_RS |I40E_TD_CMD ;if (txq->nb_tx_free < txq->tx_free_thresh) -i40e_tx_free_bufs (txq); +ci_tx_free_bufs_vec (txq, i40e_tx_desc_done, false); nb_commit = nb_pkts = (uint16_t )RTE_MIN (txq->nb_tx_free, nb_pkts);if (unlikely (nb_pkts ==0 ))
@@ -759,13 +759,13 @@ i40e_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = txq->tx_tail; txdp = &txq->i40e_tx_ring[tx_id]; - txep = &txq->sw_ring[tx_id]; + txep = &txq->sw_ring_vec[tx_id]; txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts); n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) { -ci_tx_backlog_entry (txep, tx_pkts, n); +ci_tx_backlog_entry_vec (txep, tx_pkts, n);vtx (txdp, tx_pkts, n -1 , flags); tx_pkts += (n -1 );
@@ -780,10 +780,10 @@ i40e_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts,/* avoid reach the end of ring */ txdp = &txq->i40e_tx_ring[tx_id]; - txep = &txq->sw_ring[tx_id]; + txep = &txq->sw_ring_vec[tx_id]; } -ci_tx_backlog_entry (txep, tx_pkts, nb_commit); +ci_tx_backlog_entry_vec (txep, tx_pkts, nb_commit);vtx (txdp, tx_pkts, nb_commit, flags);
drivers/net/i40e/i40e_rxtx_vec_avx512.c +2 -12
@@ -807,16 +807,6 @@ vtx(volatile struct i40e_tx_desc *txdp, } } -static __rte_always_inline void -tx_backlog_entry_avx512 (struct ci_tx_entry_vec *txep, -struct rte_mbuf **tx_pkts,uint16_t nb_pkts) -{ -int i; - -for (i =0 ; i < (int )nb_pkts; ++i) - txep[i].mbuf = tx_pkts[i]; -} -static inline uint16_t i40e_xmit_fixed_burst_vec_avx512 (void *tx_queue,struct rte_mbuf **tx_pkts,uint16_t nb_pkts)
@@ -844,7 +834,7 @@ i40e_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) { -tx_backlog_entry_avx512 (txep, tx_pkts, n); +ci_tx_backlog_entry_vec (txep, tx_pkts, n);vtx (txdp, tx_pkts, n -1 , flags); tx_pkts += (n -1 );
@@ -862,7 +852,7 @@ i40e_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, txep = (void *)txq->sw_ring; } -tx_backlog_entry_avx512 (txep, tx_pkts, nb_commit); +ci_tx_backlog_entry_vec (txep, tx_pkts, nb_commit);vtx (txdp, tx_pkts, nb_commit, flags);
drivers/net/i40e/i40e_rxtx_vec_common.h +0 -6
@@ -24,12 +24,6 @@ i40e_tx_desc_done(struct ci_tx_queue *txq, uint16_t idx)rte_cpu_to_le_64 (I40E_TX_DESC_DTYPE_DESC_DONE ); } -static __rte_always_inline int -i40e_tx_free_bufs (struct ci_tx_queue *txq) -{ -return ci_tx_free_bufs (txq, i40e_tx_desc_done); -} -static inline void _i40e_rx_queue_release_mbufs_vec (struct i40e_rx_queue *rxq) {
drivers/net/i40e/i40e_rxtx_vec_neon.c +6 -6
@@ -681,14 +681,14 @@ i40e_xmit_fixed_burst_vec(void *__rte_restrict tx_queue, {struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile struct i40e_tx_desc *txdp; -struct ci_tx_entry *txep; +struct ci_tx_entry_vec *txep;uint16_t n, nb_commit, tx_id;uint64_t flags =I40E_TD_CMD ;uint64_t rs =I40E_TX_DESC_CMD_RS |I40E_TD_CMD ;int i;if (txq->nb_tx_free < txq->tx_free_thresh) -i40e_tx_free_bufs (txq); +ci_tx_free_bufs_vec (txq, i40e_tx_desc_done, false); nb_commit = nb_pkts = (uint16_t )RTE_MIN (txq->nb_tx_free, nb_pkts);if (unlikely (nb_pkts ==0 ))
@@ -696,13 +696,13 @@ i40e_xmit_fixed_burst_vec(void *__rte_restrict tx_queue, tx_id = txq->tx_tail; txdp = &txq->i40e_tx_ring[tx_id]; - txep = &txq->sw_ring[tx_id]; + txep = &txq->sw_ring_vec[tx_id]; txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts); n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) { -ci_tx_backlog_entry (txep, tx_pkts, n); +ci_tx_backlog_entry_vec (txep, tx_pkts, n);for (i =0 ; i < n -1 ; ++i, ++tx_pkts, ++txdp)vtx1 (txdp, *tx_pkts, flags);
@@ -716,10 +716,10 @@ i40e_xmit_fixed_burst_vec(void *__rte_restrict tx_queue,/* avoid reach the end of ring */ txdp = &txq->i40e_tx_ring[tx_id]; - txep = &txq->sw_ring[tx_id]; + txep = &txq->sw_ring_vec[tx_id]; } -ci_tx_backlog_entry (txep, tx_pkts, nb_commit); +ci_tx_backlog_entry_vec (txep, tx_pkts, nb_commit);vtx (txdp, tx_pkts, nb_commit, flags);
drivers/net/i40e/i40e_rxtx_vec_sse.c +6 -6
@@ -700,14 +700,14 @@ i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, {struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile struct i40e_tx_desc *txdp; -struct ci_tx_entry *txep; +struct ci_tx_entry_vec *txep;uint16_t n, nb_commit, tx_id;uint64_t flags =I40E_TD_CMD ;uint64_t rs =I40E_TX_DESC_CMD_RS |I40E_TD_CMD ;int i;if (txq->nb_tx_free < txq->tx_free_thresh) -i40e_tx_free_bufs (txq); +ci_tx_free_bufs_vec (txq, i40e_tx_desc_done, false); nb_commit = nb_pkts = (uint16_t )RTE_MIN (txq->nb_tx_free, nb_pkts);if (unlikely (nb_pkts ==0 ))
@@ -715,13 +715,13 @@ i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = txq->tx_tail; txdp = &txq->i40e_tx_ring[tx_id]; - txep = &txq->sw_ring[tx_id]; + txep = &txq->sw_ring_vec[tx_id]; txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts); n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) { -ci_tx_backlog_entry (txep, tx_pkts, n); +ci_tx_backlog_entry_vec (txep, tx_pkts, n);for (i =0 ; i < n -1 ; ++i, ++tx_pkts, ++txdp)vtx1 (txdp, *tx_pkts, flags);
@@ -735,10 +735,10 @@ i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,/* avoid reach the end of ring */ txdp = &txq->i40e_tx_ring[tx_id]; - txep = &txq->sw_ring[tx_id]; + txep = &txq->sw_ring_vec[tx_id]; } -ci_tx_backlog_entry (txep, tx_pkts, nb_commit); +ci_tx_backlog_entry_vec (txep, tx_pkts, nb_commit);vtx (txdp, tx_pkts, nb_commit, flags);
Signature · 1 line
2.43.0
PATCH 20/21 net/iavf: use vector SW ring for all vector paths
In reply to Bruce RichardsonThe AVX-512 code path used a smaller SW ring structure only containing the mbuf pointer, but no other fields. The other fields are only used in the scalar code path, so update all vector driver code paths (AVX2, SSE) to use the smaller, faster structure.
--- drivers/net/iavf/iavf_rxtx.c | 7 ------- drivers/net/iavf/iavf_rxtx_vec_avx2.c | 12 ++++++------ drivers/net/iavf/iavf_rxtx_vec_avx512.c | 8 -------- drivers/net/iavf/iavf_rxtx_vec_common.h | 6 ------ drivers/net/iavf/iavf_rxtx_vec_sse.c | 14 +++++++------- 5 files changed, 13 insertions(+), 34 deletions(-)
drivers/net/iavf/iavf_rxtx.c +0 -7
@@ -4193,14 +4193,7 @@ iavf_set_tx_function(struct rte_eth_dev *dev) txq = dev->data->tx_queues[i];if (!txq)continue ; -#ifdef CC_AVX512_SUPPORT -if (use_avx512) -iavf_txq_vec_setup_avx512 (txq); -else -iavf_txq_vec_setup (txq); -#else iavf_txq_vec_setup (txq); -#endif }if (no_poll_on_link_down) {
drivers/net/iavf/iavf_rxtx_vec_avx2.c +6 -6
@@ -1736,14 +1736,14 @@ iavf_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, {struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile struct iavf_tx_desc *txdp; -struct ci_tx_entry *txep; +struct ci_tx_entry_vec *txep;uint16_t n, nb_commit, tx_id;/* bit2 is reserved and must be set to 1 according to Spec */ uint64_t flags =IAVF_TX_DESC_CMD_EOP |IAVF_TX_DESC_CMD_ICRC ;uint64_t rs =IAVF_TX_DESC_CMD_RS | flags;if (txq->nb_tx_free < txq->tx_free_thresh) -iavf_tx_free_bufs (txq); +ci_tx_free_bufs_vec (txq, iavf_tx_desc_done, false); nb_pkts = (uint16_t )RTE_MIN (txq->nb_tx_free, nb_pkts);if (unlikely (nb_pkts ==0 ))
@@ -1752,13 +1752,13 @@ iavf_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = txq->tx_tail; txdp = &txq->iavf_tx_ring[tx_id]; - txep = &txq->sw_ring[tx_id]; + txep = &txq->sw_ring_vec[tx_id]; txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts); n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) { -ci_tx_backlog_entry (txep, tx_pkts, n); +ci_tx_backlog_entry_vec (txep, tx_pkts, n);iavf_vtx (txdp, tx_pkts, n -1 , flags, offload); tx_pkts += (n -1 );
@@ -1773,10 +1773,10 @@ iavf_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts,/* avoid reach the end of ring */ txdp = &txq->iavf_tx_ring[tx_id]; - txep = &txq->sw_ring[tx_id]; + txep = &txq->sw_ring_vec[tx_id]; } -ci_tx_backlog_entry (txep, tx_pkts, nb_commit); +ci_tx_backlog_entry_vec (txep, tx_pkts, nb_commit);iavf_vtx (txdp, tx_pkts, nb_commit, flags, offload);
drivers/net/iavf/iavf_rxtx_vec_avx512.c +0 -8
@@ -2357,14 +2357,6 @@ iavf_xmit_pkts_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts,return iavf_xmit_pkts_vec_avx512_cmn (tx_queue, tx_pkts, nb_pkts, false); } -int __rte_cold -iavf_txq_vec_setup_avx512 (struct ci_tx_queue *txq) -{ - txq->vector_tx = true; - txq->vector_sw_ring = true; -return 0 ; -} -uint16_t iavf_xmit_pkts_vec_avx512_offload (void *tx_queue,struct rte_mbuf **tx_pkts,uint16_t nb_pkts)
drivers/net/iavf/iavf_rxtx_vec_common.h +0 -6
@@ -24,12 +24,6 @@ iavf_tx_desc_done(struct ci_tx_queue *txq, uint16_t idx)rte_cpu_to_le_64 (IAVF_TX_DESC_DTYPE_DESC_DONE ); } -static __rte_always_inline int -iavf_tx_free_bufs (struct ci_tx_queue *txq) -{ -return ci_tx_free_bufs (txq, iavf_tx_desc_done); -} -static inline void _iavf_rx_queue_release_mbufs_vec (struct iavf_rx_queue *rxq) {
drivers/net/iavf/iavf_rxtx_vec_sse.c +7 -7
@@ -1368,14 +1368,14 @@ iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, {struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile struct iavf_tx_desc *txdp; -struct ci_tx_entry *txep; +struct ci_tx_entry_vec *txep;uint16_t n, nb_commit, tx_id;uint64_t flags =IAVF_TX_DESC_CMD_EOP |0x04 ;/* bit 2 must be set */ uint64_t rs =IAVF_TX_DESC_CMD_RS | flags;int i;if (txq->nb_tx_free < txq->tx_free_thresh) -iavf_tx_free_bufs (txq); +ci_tx_free_bufs_vec (txq, iavf_tx_desc_done, false); nb_pkts = (uint16_t )RTE_MIN (txq->nb_tx_free, nb_pkts);if (unlikely (nb_pkts ==0 ))
@@ -1384,13 +1384,13 @@ iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = txq->tx_tail; txdp = &txq->iavf_tx_ring[tx_id]; - txep = &txq->sw_ring[tx_id]; + txep = &txq->sw_ring_vec[tx_id]; txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts); n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) { -ci_tx_backlog_entry (txep, tx_pkts, n); +ci_tx_backlog_entry_vec (txep, tx_pkts, n);for (i =0 ; i < n -1 ; ++i, ++tx_pkts, ++txdp)vtx1 (txdp, *tx_pkts, flags);
@@ -1404,10 +1404,10 @@ iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,/* avoid reach the end of ring */ txdp = &txq->iavf_tx_ring[tx_id]; - txep = &txq->sw_ring[tx_id]; + txep = &txq->sw_ring_vec[tx_id]; } -ci_tx_backlog_entry (txep, tx_pkts, nb_commit); +ci_tx_backlog_entry_vec (txep, tx_pkts, nb_commit);iavf_vtx (txdp, tx_pkts, nb_commit, flags);
@@ -1462,7 +1462,7 @@ int __rte_cold iavf_txq_vec_setup(struct ci_tx_queue *txq) { txq->vector_tx = true; - txq->vector_sw_ring = false; + txq->vector_sw_ring = txq->vector_tx;return 0 ; }
Signature · 1 line
2.43.0
PATCH 21/21 net/ixgbe: use common Tx backlog entry fn
In reply to Bruce RichardsonRemove the custom vector Tx backlog entry function and use the standard intel_common one, now that all vector drivers are using the same, smaller ring structure.
--- drivers/net/ixgbe/ixgbe_rxtx_vec_common.h | 10 ---------- drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c | 4 ++-- drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c | 4 ++-- 3 files changed, 4 insertions(+), 14 deletions(-)
drivers/net/ixgbe/ixgbe_rxtx_vec_common.h +0 -10
@@ -68,16 +68,6 @@ ixgbe_tx_free_bufs(struct ci_tx_queue *txq)return txq->tx_rs_thresh; } -static __rte_always_inline void -tx_backlog_entry (struct ci_tx_entry_vec *txep, -struct rte_mbuf **tx_pkts,uint16_t nb_pkts) -{ -int i; - -for (i =0 ; i < (int )nb_pkts; ++i) - txep[i].mbuf = tx_pkts[i]; -} -static inline void _ixgbe_rx_queue_release_mbufs_vec (struct ixgbe_rx_queue *rxq) {
drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c +2 -2
@@ -597,7 +597,7 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) { -tx_backlog_entry (txep, tx_pkts, n); +ci_tx_backlog_entry_vec (txep, tx_pkts, n);for (i =0 ; i < n -1 ; ++i, ++tx_pkts, ++txdp)vtx1 (txdp, *tx_pkts, flags);
@@ -614,7 +614,7 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, txep = &txq->sw_ring_vec[tx_id]; } -tx_backlog_entry (txep, tx_pkts, nb_commit); +ci_tx_backlog_entry_vec (txep, tx_pkts, nb_commit);vtx (txdp, tx_pkts, nb_commit, flags);
drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c +2 -2
@@ -720,7 +720,7 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) { -tx_backlog_entry (txep, tx_pkts, n); +ci_tx_backlog_entry_vec (txep, tx_pkts, n);for (i =0 ; i < n -1 ; ++i, ++tx_pkts, ++txdp)vtx1 (txdp, *tx_pkts, flags);
@@ -737,7 +737,7 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, txep = &txq->sw_ring_vec[tx_id]; } -tx_backlog_entry (txep, tx_pkts, nb_commit); +ci_tx_backlog_entry_vec (txep, tx_pkts, nb_commit);vtx (txdp, tx_pkts, nb_commit, flags);
Signature · 1 line
2.43.0
v2 cover Reduce code duplication across Intel NIC drivers
In reply to Bruce RichardsonThis RFC attempts to reduce the amount of code duplication across a number of Intel NIC drivers, specifically: ixgbe, i40e, iavf, and ice.
The first patch extract a function from the Rx side, otherwise the majority of the changes are on the Tx side, leading to a converged Tx queue structure across the 4 drivers, and a large number of common functions.
v1->v2:
* Fix two additional checkpatch issues that were flagged.
* Added in patch 21, which performs additional cleanup that is possible once all vector drivers use the same mbuf free/release process. [This brings the patchset to having over twice as many lines removed as added (1887 vs 930), and close to having a net removal of 1kloc]
RFC->v1:
* Moved the location of the common code from "common/intel_eth" to "net/_common_intel", and added only ".." to the driver include path so that the paths included "_common_intel" in them, to make it clear it's not driver-local headers.
* Due to change in location, structure/fn prefix changes from "ieth" to "ci" for "common intel".
* Removed the seeming-arbitrary split of vector and non-vector code - since much of the code taken from vector files was scalar code which was used by the vector drivers.
* Split code into separate Rx and Tx files.
* Fixed multiple checkpatch issues (but not all).
* Attempted to improve name standardization, by using "_vec" as a common suffix for all vector-related fns and data. Previously, some names had "vec" in the middle, others had just "_v" suffix or full word "vector" as suffix.
* Other minor changes...
Bruce Richardson (22):
net/_common_intel: add pkt reassembly fn for intel drivers net/_common_intel: provide common Tx entry structures net/_common_intel: add Tx mbuf ring replenish fn drivers/net: align Tx queue struct field names drivers/net: add prefix for driver-specific structs net/_common_intel: merge ice and i40e Tx queue struct net/iavf: use common Tx queue structure net/ixgbe: convert Tx queue context cache field to ptr net/ixgbe: use common Tx queue structure net/_common_intel: pack Tx queue structure net/_common_intel: add post-Tx buffer free function net/_common_intel: add Tx buffer free fn for AVX-512 net/iavf: use common Tx free fn for AVX-512 net/ice: move Tx queue mbuf cleanup fn to common net/i40e: use common Tx queue mbuf cleanup fn net/ixgbe: use common Tx queue mbuf cleanup fn net/iavf: use common Tx queue mbuf cleanup fn net/ice: use vector SW ring for all vector paths net/i40e: use vector SW ring for all vector paths net/iavf: use vector SW ring for all vector paths net/_common_intel: remove unneeded code net/ixgbe: use common Tx backlog entry fn
drivers/net/_common_intel/rx.h | 79 ++++++ drivers/net/_common_intel/tx.h | 249 ++++++++++++++++++ drivers/net/i40e/i40e_ethdev.c | 4 +- drivers/net/i40e/i40e_ethdev.h | 8 +- drivers/net/i40e/i40e_fdir.c | 10 +- .../net/i40e/i40e_recycle_mbufs_vec_common.c | 6 +- drivers/net/i40e/i40e_rxtx.c | 192 +++++--------- drivers/net/i40e/i40e_rxtx.h | 61 +---- drivers/net/i40e/i40e_rxtx_vec_altivec.c | 26 +- drivers/net/i40e/i40e_rxtx_vec_avx2.c | 26 +- drivers/net/i40e/i40e_rxtx_vec_avx512.c | 144 +--------- drivers/net/i40e/i40e_rxtx_vec_common.h | 144 +--------- drivers/net/i40e/i40e_rxtx_vec_neon.c | 26 +- drivers/net/i40e/i40e_rxtx_vec_sse.c | 26 +- drivers/net/i40e/meson.build | 2 +- drivers/net/iavf/iavf.h | 2 +- drivers/net/iavf/iavf_ethdev.c | 4 +- drivers/net/iavf/iavf_rxtx.c | 180 +++++-------- drivers/net/iavf/iavf_rxtx.h | 61 +---- drivers/net/iavf/iavf_rxtx_vec_avx2.c | 47 ++-- drivers/net/iavf/iavf_rxtx_vec_avx512.c | 214 +++------------ drivers/net/iavf/iavf_rxtx_vec_common.h | 160 +---------- drivers/net/iavf/iavf_rxtx_vec_sse.c | 56 ++-- drivers/net/iavf/iavf_vchnl.c | 8 +- drivers/net/iavf/meson.build | 2 +- drivers/net/ice/ice_dcf.c | 4 +- drivers/net/ice/ice_dcf_ethdev.c | 21 +- drivers/net/ice/ice_diagnose.c | 2 +- drivers/net/ice/ice_ethdev.c | 2 +- drivers/net/ice/ice_ethdev.h | 7 +- drivers/net/ice/ice_rxtx.c | 163 +++++------- drivers/net/ice/ice_rxtx.h | 52 +--- drivers/net/ice/ice_rxtx_vec_avx2.c | 26 +- drivers/net/ice/ice_rxtx_vec_avx512.c | 153 +---------- drivers/net/ice/ice_rxtx_vec_common.h | 190 +------------ drivers/net/ice/ice_rxtx_vec_sse.c | 32 +-- drivers/net/ice/meson.build | 2 +- drivers/net/ixgbe/base/ixgbe_osdep.h | 2 +- drivers/net/ixgbe/ixgbe_ethdev.c | 4 +- .../ixgbe/ixgbe_recycle_mbufs_vec_common.c | 6 +- drivers/net/ixgbe/ixgbe_rxtx.c | 139 +++++----- drivers/net/ixgbe/ixgbe_rxtx.h | 73 +---- drivers/net/ixgbe/ixgbe_rxtx_vec_common.h | 128 ++------- drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c | 37 ++- drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c | 37 ++- drivers/net/ixgbe/meson.build | 2 +- 46 files changed, 930 insertions(+), 1889 deletions(-) create mode 100644 drivers/net/_common_intel/rx.h create mode 100644 drivers/net/_common_intel/tx.h
--
2.43.0
PATCH v2 1/22 net/_common_intel: add pkt reassembly fn for intel drivers
The code for reassembling a single, multi-mbuf packet from multiple buffers received from the NIC is duplicated across many drivers. Rather than having multiple copies of this function, we can create an "_common_intel" directory to hold such functions and consolidate multiple functions down to a single one for easier maintenance.
--- drivers/net/_common_intel/rx.h | 79 +++++++++++++++++++++++ drivers/net/i40e/i40e_rxtx_vec_altivec.c | 4 +- drivers/net/i40e/i40e_rxtx_vec_avx2.c | 4 +- drivers/net/i40e/i40e_rxtx_vec_avx512.c | 4 +- drivers/net/i40e/i40e_rxtx_vec_common.h | 64 +----------------- drivers/net/i40e/i40e_rxtx_vec_neon.c | 4 +- drivers/net/i40e/i40e_rxtx_vec_sse.c | 4 +- drivers/net/i40e/meson.build | 2 +- drivers/net/iavf/iavf_rxtx_vec_avx2.c | 8 +-- drivers/net/iavf/iavf_rxtx_vec_avx512.c | 8 +-- drivers/net/iavf/iavf_rxtx_vec_common.h | 65 +------------------ drivers/net/iavf/iavf_rxtx_vec_sse.c | 8 +-- drivers/net/iavf/meson.build | 2 +- drivers/net/ice/ice_rxtx_vec_avx2.c | 4 +- drivers/net/ice/ice_rxtx_vec_avx512.c | 8 +-- drivers/net/ice/ice_rxtx_vec_common.h | 66 +------------------ drivers/net/ice/ice_rxtx_vec_sse.c | 4 +- drivers/net/ice/meson.build | 2 +- drivers/net/ixgbe/ixgbe_rxtx_vec_common.h | 63 +----------------- drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c | 4 +- drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c | 4 +- drivers/net/ixgbe/meson.build | 2 +- 22 files changed, 121 insertions(+), 292 deletions(-) create mode 100644 drivers/net/_common_intel/rx.h
drivers/net/_common_intel/rx.h new file +79 -0
@@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2024 Intel Corporation +*/ + +#ifndef _COMMON_INTEL_RX_H_ +#define _COMMON_INTEL_RX_H_ + +#include <stdint.h> +#include <unistd.h> +#include <rte_mbuf.h> + +#define CI_RX_BURST 32 + +static inline uint16_t +ci_rx_reassemble_packets (struct rte_mbuf **rx_bufs,uint16_t nb_bufs,uint8_t *split_flags, +struct rte_mbuf **pkt_first_seg,struct rte_mbuf **pkt_last_seg, +const uint8_t crc_len) +{ +struct rte_mbuf *pkts[CI_RX_BURST ] = {0 };/*finished pkts*/ +struct rte_mbuf *start = *pkt_first_seg; +struct rte_mbuf *end = *pkt_last_seg; +unsigned int pkt_idx, buf_idx; + +for (buf_idx =0 , pkt_idx =0 ; buf_idx < nb_bufs; buf_idx++) { +if (end) { +/* processing a split packet */ + end->next = rx_bufs[buf_idx]; + rx_bufs[buf_idx]->data_len += crc_len; + + start->nb_segs++; + start->pkt_len += rx_bufs[buf_idx]->data_len; + end = end->next; + +if (!split_flags[buf_idx]) { +/* it's the last packet of the set */ + start->hash = end->hash; + start->vlan_tci = end->vlan_tci; + start->ol_flags = end->ol_flags; +/* we need to strip crc for the whole packet */ + start->pkt_len -= crc_len; +if (end->data_len > crc_len) { + end->data_len -= crc_len; + }else { +/* free up last mbuf */ +struct rte_mbuf *secondlast = start; + + start->nb_segs--; +while (secondlast->next != end) + secondlast = secondlast->next; + secondlast->data_len -= (crc_len - end->data_len); + secondlast->next =NULL ; +rte_pktmbuf_free_seg (end); + } + pkts[pkt_idx++] = start; + start =NULL ; + end =NULL ; + } + }else { +/* not processing a split packet */ +if (!split_flags[buf_idx]) { +/* not a split packet, save and skip */ + pkts[pkt_idx++] = rx_bufs[buf_idx]; +continue ; + } + start = rx_bufs[buf_idx]; + end = start; + rx_bufs[buf_idx]->data_len += crc_len; + rx_bufs[buf_idx]->pkt_len += crc_len; + } + } + +/* save the partial packet for next time */ + *pkt_first_seg = start; + *pkt_last_seg = end; +memcpy (rx_bufs, pkts, pkt_idx * (sizeof (*pkts))); +return pkt_idx; +} + +#endif /* _COMMON_INTEL_RX_H_ */
drivers/net/i40e/i40e_rxtx_vec_altivec.c +2 -2
@@ -494,8 +494,8 @@ i40e_recv_scattered_burst_vec(void *rx_queue, struct rte_mbuf **rx_pkts,if (i == nb_bufs)return nb_bufs; } -return i +reassemble_packets (rxq, &rx_pkts[i], nb_bufs - i, - &split_flags[i]); +return i +ci_rx_reassemble_packets (&rx_pkts[i], nb_bufs - i, &split_flags[i], + &rxq->pkt_first_seg, &rxq->pkt_last_seg, rxq->crc_len); }/**
drivers/net/i40e/i40e_rxtx_vec_avx2.c +2 -2
@@ -657,8 +657,8 @@ i40e_recv_scattered_burst_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,return nb_bufs; rxq->pkt_first_seg = rx_pkts[i]; } -return i +reassemble_packets (rxq, &rx_pkts[i], nb_bufs - i, - &split_flags[i]); +return i +ci_rx_reassemble_packets (&rx_pkts[i], nb_bufs - i, &split_flags[i], + &rxq->pkt_first_seg, &rxq->pkt_last_seg, rxq->crc_len); }/*
drivers/net/i40e/i40e_rxtx_vec_avx512.c +2 -2
@@ -725,8 +725,8 @@ i40e_recv_scattered_burst_vec_avx512(void *rx_queue,return nb_bufs; rxq->pkt_first_seg = rx_pkts[i]; } -return i +reassemble_packets (rxq, &rx_pkts[i], nb_bufs - i, - &split_flags[i]); +return i +ci_rx_reassemble_packets (&rx_pkts[i], nb_bufs - i, &split_flags[i], + &rxq->pkt_first_seg, &rxq->pkt_last_seg, rxq->crc_len); }/**
drivers/net/i40e/i40e_rxtx_vec_common.h +1 -63
76 lines not on this page · raw message
drivers/net/i40e/i40e_rxtx_vec_neon.c +2 -2
@@ -623,8 +623,8 @@ i40e_recv_scattered_burst_vec(void *rx_queue, struct rte_mbuf **rx_pkts,return nb_bufs; rxq->pkt_first_seg = rx_pkts[i]; } -return i +reassemble_packets (rxq, &rx_pkts[i], nb_bufs - i, - &split_flags[i]); +return i +ci_rx_reassemble_packets (&rx_pkts[i], nb_bufs - i, &split_flags[i], + &rxq->pkt_first_seg, &rxq->pkt_last_seg, rxq->crc_len); }/**
drivers/net/i40e/i40e_rxtx_vec_sse.c +2 -2
@@ -641,8 +641,8 @@ i40e_recv_scattered_burst_vec(void *rx_queue, struct rte_mbuf **rx_pkts,return nb_bufs; rxq->pkt_first_seg = rx_pkts[i]; } -return i +reassemble_packets (rxq, &rx_pkts[i], nb_bufs - i, - &split_flags[i]); +return i +ci_rx_reassemble_packets (&rx_pkts[i], nb_bufs - i, &split_flags[i], + &rxq->pkt_first_seg, &rxq->pkt_last_seg, rxq->crc_len); }/**
drivers/net/i40e/meson.build +1 -1
@@ -36,7 +36,7 @@ sources = files( testpmd_sources = files('i40e_testpmd.c') deps += ['hash'] -includes += include_directories('base') +includes += include_directories('base', '..') if arch_subdir == 'x86' sources += files('i40e_rxtx_vec_sse.c')
drivers/net/iavf/iavf_rxtx_vec_avx2.c +4 -4
@@ -1508,8 +1508,8 @@ iavf_recv_scattered_burst_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,return nb_bufs; rxq->pkt_first_seg = rx_pkts[i]; } -return i +reassemble_packets (rxq, &rx_pkts[i], nb_bufs - i, - &split_flags[i]); +return i +ci_rx_reassemble_packets (&rx_pkts[i], nb_bufs - i, &split_flags[i], + &rxq->pkt_first_seg, &rxq->pkt_last_seg, rxq->crc_len); }/**
@@ -1597,8 +1597,8 @@ iavf_recv_scattered_burst_vec_avx2_flex_rxd(void *rx_queue,return nb_bufs; rxq->pkt_first_seg = rx_pkts[i]; } -return i +reassemble_packets (rxq, &rx_pkts[i], nb_bufs - i, - &split_flags[i]); +return i +ci_rx_reassemble_packets (&rx_pkts[i], nb_bufs - i, &split_flags[i], + &rxq->pkt_first_seg, &rxq->pkt_last_seg, rxq->crc_len); }/**
drivers/net/iavf/iavf_rxtx_vec_avx512.c +4 -4
@@ -1685,8 +1685,8 @@ iavf_recv_scattered_burst_vec_avx512(void *rx_queue, struct rte_mbuf **rx_pkts,return nb_bufs; rxq->pkt_first_seg = rx_pkts[i]; } -return i +reassemble_packets (rxq, &rx_pkts[i], nb_bufs - i, - &split_flags[i]); +return i +ci_rx_reassemble_packets (&rx_pkts[i], nb_bufs - i, &split_flags[i], + &rxq->pkt_first_seg, &rxq->pkt_last_seg, rxq->crc_len); }/**
@@ -1761,8 +1761,8 @@ iavf_recv_scattered_burst_vec_avx512_flex_rxd(void *rx_queue,return nb_bufs; rxq->pkt_first_seg = rx_pkts[i]; } -return i +reassemble_packets (rxq, &rx_pkts[i], nb_bufs - i, - &split_flags[i]); +return i +ci_rx_reassemble_packets (&rx_pkts[i], nb_bufs - i, &split_flags[i], + &rxq->pkt_first_seg, &rxq->pkt_last_seg, rxq->crc_len); }/**
drivers/net/iavf/iavf_rxtx_vec_common.h +1 -64
77 lines not on this page · raw message
drivers/net/iavf/iavf_rxtx_vec_sse.c +4 -4
@@ -1238,8 +1238,8 @@ iavf_recv_scattered_burst_vec(void *rx_queue, struct rte_mbuf **rx_pkts,return nb_bufs; rxq->pkt_first_seg = rx_pkts[i]; } -return i +reassemble_packets (rxq, &rx_pkts[i], nb_bufs - i, - &split_flags[i]); +return i +ci_rx_reassemble_packets (&rx_pkts[i], nb_bufs - i, &split_flags[i], + &rxq->pkt_first_seg, &rxq->pkt_last_seg, rxq->crc_len); }/**
@@ -1307,8 +1307,8 @@ iavf_recv_scattered_burst_vec_flex_rxd(void *rx_queue,return nb_bufs; rxq->pkt_first_seg = rx_pkts[i]; } -return i +reassemble_packets (rxq, &rx_pkts[i], nb_bufs - i, - &split_flags[i]); +return i +ci_rx_reassemble_packets (&rx_pkts[i], nb_bufs - i, &split_flags[i], + &rxq->pkt_first_seg, &rxq->pkt_last_seg, rxq->crc_len); }/**
drivers/net/iavf/meson.build +1 -1
@@ -5,7 +5,7 @@ if dpdk_conf.get('RTE_IOVA_IN_MBUF') == 0 subdir_done() endif -includes += include_directories('../../common/iavf') +includes += include_directories('../../common/iavf', '..') testpmd_sources = files('iavf_testpmd.c')
drivers/net/ice/ice_rxtx_vec_avx2.c +2 -2
@@ -726,8 +726,8 @@ ice_recv_scattered_burst_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,return nb_bufs; rxq->pkt_first_seg = rx_pkts[i]; } -return i +ice_rx_reassemble_packets (rxq, &rx_pkts[i], nb_bufs - i, - &split_flags[i]); +return i +ci_rx_reassemble_packets (&rx_pkts[i], nb_bufs - i, &split_flags[i], + &rxq->pkt_first_seg, &rxq->pkt_last_seg, rxq->crc_len); }/**
drivers/net/ice/ice_rxtx_vec_avx512.c +4 -4
@@ -763,8 +763,8 @@ ice_recv_scattered_burst_vec_avx512(void *rx_queue, struct rte_mbuf **rx_pkts,return nb_bufs; rxq->pkt_first_seg = rx_pkts[i]; } -return i +ice_rx_reassemble_packets (rxq, &rx_pkts[i], nb_bufs - i, - &split_flags[i]); +return i +ci_rx_reassemble_packets (&rx_pkts[i], nb_bufs - i, &split_flags[i], + &rxq->pkt_first_seg, &rxq->pkt_last_seg, rxq->crc_len); }/**
@@ -805,8 +805,8 @@ ice_recv_scattered_burst_vec_avx512_offload(void *rx_queue,return nb_bufs; rxq->pkt_first_seg = rx_pkts[i]; } -return i +ice_rx_reassemble_packets (rxq, &rx_pkts[i], nb_bufs - i, - &split_flags[i]); +return i +ci_rx_reassemble_packets (&rx_pkts[i], nb_bufs - i, &split_flags[i], + &rxq->pkt_first_seg, &rxq->pkt_last_seg, rxq->crc_len); }/**
drivers/net/ice/ice_rxtx_vec_common.h +1 -65
78 lines not on this page · raw message
drivers/net/ice/ice_rxtx_vec_sse.c +2 -2
@@ -640,8 +640,8 @@ ice_recv_scattered_burst_vec(void *rx_queue, struct rte_mbuf **rx_pkts,return nb_bufs; rxq->pkt_first_seg = rx_pkts[i]; } -return i +ice_rx_reassemble_packets (rxq, &rx_pkts[i], nb_bufs - i, - &split_flags[i]); +return i +ci_rx_reassemble_packets (&rx_pkts[i], nb_bufs - i, &split_flags[i], + &rxq->pkt_first_seg, &rxq->pkt_last_seg, rxq->crc_len); }/**
drivers/net/ice/meson.build +1 -1
@@ -19,7 +19,7 @@ sources = files( testpmd_sources = files('ice_testpmd.c') deps += ['hash', 'net', 'common_iavf'] -includes += include_directories('base', '../../common/iavf') +includes += include_directories('base', '..') if arch_subdir == 'x86' sources += files('ice_rxtx_vec_sse.c')
drivers/net/ixgbe/ixgbe_rxtx_vec_common.h +1 -62
72 lines not on this page · raw message
drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c +2 -2
@@ -516,8 +516,8 @@ ixgbe_recv_scattered_burst_vec(void *rx_queue, struct rte_mbuf **rx_pkts,return nb_bufs; rxq->pkt_first_seg = rx_pkts[i]; } -return i +reassemble_packets (rxq, &rx_pkts[i], nb_bufs - i, - &split_flags[i]); +return i +ci_rx_reassemble_packets (&rx_pkts[i], nb_bufs - i, &split_flags[i], + &rxq->pkt_first_seg, &rxq->pkt_last_seg, rxq->crc_len); }/**
drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c +2 -2
@@ -639,8 +639,8 @@ ixgbe_recv_scattered_burst_vec(void *rx_queue, struct rte_mbuf **rx_pkts,return nb_bufs; rxq->pkt_first_seg = rx_pkts[i]; } -return i +reassemble_packets (rxq, &rx_pkts[i], nb_bufs - i, - &split_flags[i]); +return i +ci_rx_reassemble_packets (&rx_pkts[i], nb_bufs - i, &split_flags[i], + &rxq->pkt_first_seg, &rxq->pkt_last_seg, rxq->crc_len); }/**
drivers/net/ixgbe/meson.build +1 -1
@@ -35,6 +35,6 @@ elif arch_subdir == 'arm' sources += files('ixgbe_recycle_mbufs_vec_common.c') endif -includes += include_directories('base') +includes += include_directories('base', '..') headers = files('rte_pmd_ixgbe.h')
Signature · 1 line
2.43.0
PATCH v2 2/22 net/_common_intel: provide common Tx entry structures
In reply to Bruce RichardsonThe Tx entry structures, both vector and scalar, are common across Intel drivers, so provide a single definition to be used everywhere.
--- drivers/net/_common_intel/tx.h | 27 +++++++++++++++++++ .../net/i40e/i40e_recycle_mbufs_vec_common.c | 2 +- drivers/net/i40e/i40e_rxtx.c | 18 ++++++------- drivers/net/i40e/i40e_rxtx.h | 14 +++------- drivers/net/i40e/i40e_rxtx_vec_altivec.c | 2 +- drivers/net/i40e/i40e_rxtx_vec_avx2.c | 2 +- drivers/net/i40e/i40e_rxtx_vec_avx512.c | 6 ++--- drivers/net/i40e/i40e_rxtx_vec_common.h | 4 +-- drivers/net/i40e/i40e_rxtx_vec_neon.c | 2 +- drivers/net/i40e/i40e_rxtx_vec_sse.c | 2 +- drivers/net/iavf/iavf_rxtx.c | 12 ++++----- drivers/net/iavf/iavf_rxtx.h | 14 +++------- drivers/net/iavf/iavf_rxtx_vec_avx2.c | 2 +- drivers/net/iavf/iavf_rxtx_vec_avx512.c | 10 +++---- drivers/net/iavf/iavf_rxtx_vec_common.h | 4 +-- drivers/net/iavf/iavf_rxtx_vec_sse.c | 2 +- drivers/net/ice/ice_dcf_ethdev.c | 2 +- drivers/net/ice/ice_rxtx.c | 16 +++++------ drivers/net/ice/ice_rxtx.h | 13 ++------- drivers/net/ice/ice_rxtx_vec_avx2.c | 2 +- drivers/net/ice/ice_rxtx_vec_avx512.c | 6 ++--- drivers/net/ice/ice_rxtx_vec_common.h | 6 ++--- drivers/net/ice/ice_rxtx_vec_sse.c | 2 +- .../ixgbe/ixgbe_recycle_mbufs_vec_common.c | 2 +- drivers/net/ixgbe/ixgbe_rxtx.c | 16 +++++------ drivers/net/ixgbe/ixgbe_rxtx.h | 22 +++------------ drivers/net/ixgbe/ixgbe_rxtx_vec_common.h | 8 +++--- drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c | 2 +- drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c | 2 +- 29 files changed, 105 insertions(+), 117 deletions(-) create mode 100644 drivers/net/_common_intel/tx.h
drivers/net/_common_intel/tx.h new file +27 -0
@@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2024 Intel Corporation +*/ + +#ifndef _COMMON_INTEL_TX_H_ +#define _COMMON_INTEL_TX_H_ + +#include <stdint.h> +#include <rte_mbuf.h> + +/** +* Structure associated with each descriptor of the TX ring of a TX queue. +*/ +struct ci_tx_entry { +struct rte_mbuf *mbuf;/* mbuf associated with TX desc, if any. */ +uint16_t next_id;/* Index of next descriptor in ring. */ +uint16_t last_id;/* Index of last scattered descriptor. */ +}; + +/** +* Structure associated with each descriptor of the TX ring of a TX queue in vector Tx. +*/ +struct ci_tx_entry_vec { +struct rte_mbuf *mbuf;/* mbuf associated with TX desc, if any. */ +}; + +#endif /* _COMMON_INTEL_TX_H_ */
drivers/net/i40e/i40e_recycle_mbufs_vec_common.c +1 -1
@@ -56,7 +56,7 @@ i40e_recycle_tx_mbufs_reuse_vec(void *tx_queue, struct rte_eth_recycle_rxq_info *recycle_rxq_info) {struct i40e_tx_queue *txq = tx_queue; -struct i40e_tx_entry *txep; +struct ci_tx_entry *txep;struct rte_mbuf **rxep;int i, n;uint16_t nb_recycle_mbufs;
drivers/net/i40e/i40e_rxtx.c +9 -9
@@ -378,7 +378,7 @@ i40e_build_ctob(uint32_t td_cmd,static inline int i40e_xmit_cleanup (struct i40e_tx_queue *txq) { -struct i40e_tx_entry *sw_ring = txq->sw_ring; +struct ci_tx_entry *sw_ring = txq->sw_ring;volatile struct i40e_tx_desc *txd = txq->tx_ring;uint16_t last_desc_cleaned = txq->last_desc_cleaned;uint16_t nb_tx_desc = txq->nb_tx_desc;
@@ -1081,8 +1081,8 @@ uint16_t i40e_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) {struct i40e_tx_queue *txq; -struct i40e_tx_entry *sw_ring; -struct i40e_tx_entry *txe, *txn; +struct ci_tx_entry *sw_ring; +struct ci_tx_entry *txe, *txn;volatile struct i40e_tx_desc *txd;volatile struct i40e_tx_desc *txr;struct rte_mbuf *tx_pkt;
@@ -1331,7 +1331,7 @@ i40e_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) static __rte_always_inline inti40e_tx_free_bufs (struct i40e_tx_queue *txq) { -struct i40e_tx_entry *txep; +struct ci_tx_entry *txep;uint16_t tx_rs_thresh = txq->tx_rs_thresh;uint16_t i =0 , j =0 ;struct rte_mbuf *free[RTE_I40E_TX_MAX_FREE_BUF_SZ ];
@@ -1418,7 +1418,7 @@ i40e_tx_fill_hw_ring(struct i40e_tx_queue *txq, uint16_t nb_pkts) {volatile struct i40e_tx_desc *txdp = &(txq->tx_ring[txq->tx_tail]); -struct i40e_tx_entry *txep = &(txq->sw_ring[txq->tx_tail]); +struct ci_tx_entry *txep = &txq->sw_ring[txq->tx_tail];const int N_PER_LOOP =4 ;const int N_PER_LOOP_MASK =N_PER_LOOP -1 ;int mainpart, leftover;
@@ -2555,7 +2555,7 @@ i40e_dev_tx_queue_setup(struct rte_eth_dev *dev,/* Allocate software ring */ txq->sw_ring =rte_zmalloc_socket ("i40e tx sw ring" , -sizeof (struct i40e_tx_entry ) * nb_desc, +sizeof (struct ci_tx_entry ) * nb_desc,RTE_CACHE_LINE_SIZE , socket_id);if (!txq->sw_ring) {
@@ -2723,7 +2723,7 @@ i40e_tx_queue_release_mbufs(struct i40e_tx_queue *txq)*/ #ifdef CC_AVX512_SUPPORT if (dev->tx_pkt_burst == i40e_xmit_pkts_vec_avx512) { -struct i40e_vec_tx_entry *swr = (void *)txq->sw_ring; +struct ci_tx_entry_vec *swr = (void *)txq->sw_ring; i = txq->tx_next_dd - txq->tx_rs_thresh +1 ;if (txq->tx_tail < i) {
@@ -2768,7 +2768,7 @@ static int i40e_tx_done_cleanup_full(struct i40e_tx_queue *txq, uint32_t free_cnt) { -struct i40e_tx_entry *swr_ring = txq->sw_ring; +struct ci_tx_entry *swr_ring = txq->sw_ring;uint16_t i, tx_last, tx_id;uint16_t nb_tx_free_last;uint16_t nb_tx_to_clean;
@@ -2874,7 +2874,7 @@ i40e_tx_done_cleanup(void *txq, uint32_t free_cnt)void i40e_reset_tx_queue (struct i40e_tx_queue *txq) { -struct i40e_tx_entry *txe; +struct ci_tx_entry *txe;uint16_t i, prev, size;if (!txq) {
drivers/net/i40e/i40e_rxtx.h +3 -11
@@ -5,6 +5,8 @@#ifndef _I40E_RXTX_H_#define _I40E_RXTX_H_ +#include <_common_intel/tx.h> +#define RTE_PMD_I40E_RX_MAX_BURST 32#define RTE_PMD_I40E_TX_MAX_BURST 32
@@ -122,16 +124,6 @@ struct i40e_rx_queue {const struct rte_memzone *mz; }; -struct i40e_tx_entry { -struct rte_mbuf *mbuf; -uint16_t next_id; -uint16_t last_id; -}; - -struct i40e_vec_tx_entry { -struct rte_mbuf *mbuf; -}; -/* * Structure associated with each TX queue. */
@@ -139,7 +131,7 @@ struct i40e_tx_queue {uint16_t nb_tx_desc;/**< number of TX descriptors */ uint64_t tx_ring_phys_addr;/**< TX ring DMA address */ volatile struct i40e_tx_desc *tx_ring;/**< TX ring virtual address */ -struct i40e_tx_entry *sw_ring;/**< virtual address of SW ring */ +struct ci_tx_entry *sw_ring;/**< virtual address of SW ring */ uint16_t tx_tail;/**< current value of tail register */ volatile uint8_t *qtx_tail;/**< register address of tail */ uint16_t nb_tx_used;/**< number of TX desc used since RS bit set */
drivers/net/i40e/i40e_rxtx_vec_altivec.c +1 -1
@@ -553,7 +553,7 @@ i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, {struct i40e_tx_queue *txq = (struct i40e_tx_queue *)tx_queue;volatile struct i40e_tx_desc *txdp; -struct i40e_tx_entry *txep; +struct ci_tx_entry *txep;uint16_t n, nb_commit, tx_id;uint64_t flags =I40E_TD_CMD ;uint64_t rs =I40E_TX_DESC_CMD_RS |I40E_TD_CMD ;
drivers/net/i40e/i40e_rxtx_vec_avx2.c +1 -1
@@ -745,7 +745,7 @@ i40e_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, {struct i40e_tx_queue *txq = (struct i40e_tx_queue *)tx_queue;volatile struct i40e_tx_desc *txdp; -struct i40e_tx_entry *txep; +struct ci_tx_entry *txep;uint16_t n, nb_commit, tx_id;uint64_t flags =I40E_TD_CMD ;uint64_t rs =I40E_TX_DESC_CMD_RS |I40E_TD_CMD ;
drivers/net/i40e/i40e_rxtx_vec_avx512.c +3 -3
@@ -757,7 +757,7 @@ i40e_recv_scattered_pkts_vec_avx512(void *rx_queue, static __rte_always_inline inti40e_tx_free_bufs_avx512 (struct i40e_tx_queue *txq) { -struct i40e_vec_tx_entry *txep; +struct ci_tx_entry_vec *txep;uint32_t n;uint32_t i;int nb_free =0 ;
@@ -920,7 +920,7 @@ vtx(volatile struct i40e_tx_desc *txdp, } static __rte_always_inline void -tx_backlog_entry_avx512 (struct i40e_vec_tx_entry *txep, +tx_backlog_entry_avx512 (struct ci_tx_entry_vec *txep,struct rte_mbuf **tx_pkts,uint16_t nb_pkts) {int i;
@@ -935,7 +935,7 @@ i40e_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, {struct i40e_tx_queue *txq = (struct i40e_tx_queue *)tx_queue;volatile struct i40e_tx_desc *txdp; -struct i40e_vec_tx_entry *txep; +struct ci_tx_entry_vec *txep;uint16_t n, nb_commit, tx_id;uint64_t flags =I40E_TD_CMD ;uint64_t rs =I40E_TX_DESC_CMD_RS |I40E_TD_CMD ;
drivers/net/i40e/i40e_rxtx_vec_common.h +2 -2
@@ -19,7 +19,7 @@ static __rte_always_inline inti40e_tx_free_bufs (struct i40e_tx_queue *txq) { -struct i40e_tx_entry *txep; +struct ci_tx_entry *txep;uint32_t n;uint32_t i;int nb_free =0 ;
@@ -85,7 +85,7 @@ i40e_tx_free_bufs(struct i40e_tx_queue *txq) } static __rte_always_inline void -tx_backlog_entry (struct i40e_tx_entry *txep, +tx_backlog_entry (struct ci_tx_entry *txep,struct rte_mbuf **tx_pkts,uint16_t nb_pkts) {int i;
drivers/net/i40e/i40e_rxtx_vec_neon.c +1 -1
@@ -681,7 +681,7 @@ i40e_xmit_fixed_burst_vec(void *__rte_restrict tx_queue, {struct i40e_tx_queue *txq = (struct i40e_tx_queue *)tx_queue;volatile struct i40e_tx_desc *txdp; -struct i40e_tx_entry *txep; +struct ci_tx_entry *txep;uint16_t n, nb_commit, tx_id;uint64_t flags =I40E_TD_CMD ;uint64_t rs =I40E_TX_DESC_CMD_RS |I40E_TD_CMD ;
drivers/net/i40e/i40e_rxtx_vec_sse.c +1 -1
@@ -700,7 +700,7 @@ i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, {struct i40e_tx_queue *txq = (struct i40e_tx_queue *)tx_queue;volatile struct i40e_tx_desc *txdp; -struct i40e_tx_entry *txep; +struct ci_tx_entry *txep;uint16_t n, nb_commit, tx_id;uint64_t flags =I40E_TD_CMD ;uint64_t rs =I40E_TX_DESC_CMD_RS |I40E_TD_CMD ;
drivers/net/iavf/iavf_rxtx.c +6 -6
@@ -284,7 +284,7 @@ reset_rx_queue(struct iavf_rx_queue *rxq)static inline void reset_tx_queue (struct iavf_tx_queue *txq) { -struct iavf_tx_entry *txe; +struct ci_tx_entry *txe;uint32_t i, size;uint16_t prev;
@@ -860,7 +860,7 @@ iavf_dev_tx_queue_setup(struct rte_eth_dev *dev,/* Allocate software ring */ txq->sw_ring =rte_zmalloc_socket ("iavf tx sw ring" , -sizeof (struct iavf_tx_entry ) * nb_desc, +sizeof (struct ci_tx_entry ) * nb_desc,RTE_CACHE_LINE_SIZE , socket_id);if (!txq->sw_ring) {
@@ -2379,7 +2379,7 @@ iavf_recv_pkts_bulk_alloc(void *rx_queue,static inline int iavf_xmit_cleanup (struct iavf_tx_queue *txq) { -struct iavf_tx_entry *sw_ring = txq->sw_ring; +struct ci_tx_entry *sw_ring = txq->sw_ring;uint16_t last_desc_cleaned = txq->last_desc_cleaned;uint16_t nb_tx_desc = txq->nb_tx_desc;uint16_t desc_to_clean_to;
@@ -2797,8 +2797,8 @@ iavf_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) {struct iavf_tx_queue *txq = tx_queue;volatile struct iavf_tx_desc *txr = txq->tx_ring; -struct iavf_tx_entry *txe_ring = txq->sw_ring; -struct iavf_tx_entry *txe, *txn; +struct ci_tx_entry *txe_ring = txq->sw_ring; +struct ci_tx_entry *txe, *txn;struct rte_mbuf *mb, *mb_seg;uint64_t buf_dma_addr;uint16_t desc_idx, desc_idx_last;
@@ -4268,7 +4268,7 @@ static int iavf_tx_done_cleanup_full(struct iavf_tx_queue *txq, uint32_t free_cnt) { -struct iavf_tx_entry *swr_ring = txq->sw_ring; +struct ci_tx_entry *swr_ring = txq->sw_ring;uint16_t tx_last, tx_id;uint16_t nb_tx_free_last;uint16_t nb_tx_to_clean;
drivers/net/iavf/iavf_rxtx.h +3 -11
@@ -5,6 +5,8 @@#ifndef _IAVF_RXTX_H_#define _IAVF_RXTX_H_ +#include <_common_intel/tx.h> +/* In QLEN must be whole number of 32 descriptors. */ #define IAVF_ALIGN_RING_DESC 32#define IAVF_MIN_RING_DESC 64
@@ -271,22 +273,12 @@ struct iavf_rx_queue {uint64_t hw_time_update; }; -struct iavf_tx_entry { -struct rte_mbuf *mbuf; -uint16_t next_id; -uint16_t last_id; -}; - -struct iavf_tx_vec_entry { -struct rte_mbuf *mbuf; -}; -/* Structure associated with each TX queue. */ struct iavf_tx_queue {const struct rte_memzone *mz;/* memzone for Tx ring */ volatile struct iavf_tx_desc *tx_ring;/* Tx ring virtual address */ uint64_t tx_ring_phys_addr;/* Tx ring DMA address */ -struct iavf_tx_entry *sw_ring;/* address array of SW ring */ +struct ci_tx_entry *sw_ring;/* address array of SW ring */ uint16_t nb_tx_desc;/* ring length */ uint16_t tx_tail;/* current value of tail */ volatile uint8_t *qtx_tail;/* register address of tail */
drivers/net/iavf/iavf_rxtx_vec_avx2.c +1 -1
@@ -1736,7 +1736,7 @@ iavf_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, {struct iavf_tx_queue *txq = (struct iavf_tx_queue *)tx_queue;volatile struct iavf_tx_desc *txdp; -struct iavf_tx_entry *txep; +struct ci_tx_entry *txep;uint16_t n, nb_commit, tx_id;/* bit2 is reserved and must be set to 1 according to Spec */ uint64_t flags =IAVF_TX_DESC_CMD_EOP |IAVF_TX_DESC_CMD_ICRC ;
drivers/net/iavf/iavf_rxtx_vec_avx512.c +5 -5
@@ -1847,7 +1847,7 @@ iavf_recv_scattered_pkts_vec_avx512_flex_rxd_offload(void *rx_queue, static __rte_always_inline intiavf_tx_free_bufs_avx512 (struct iavf_tx_queue *txq) { -struct iavf_tx_vec_entry *txep; +struct ci_tx_entry_vec *txep;uint32_t n;uint32_t i;int nb_free =0 ;
@@ -1960,7 +1960,7 @@ iavf_tx_free_bufs_avx512(struct iavf_tx_queue *txq) } static __rte_always_inline void -tx_backlog_entry_avx512 (struct iavf_tx_vec_entry *txep, +tx_backlog_entry_avx512 (struct ci_tx_entry_vec *txep,struct rte_mbuf **tx_pkts,uint16_t nb_pkts) {int i;
@@ -2313,7 +2313,7 @@ iavf_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, {struct iavf_tx_queue *txq = (struct iavf_tx_queue *)tx_queue;volatile struct iavf_tx_desc *txdp; -struct iavf_tx_vec_entry *txep; +struct ci_tx_entry_vec *txep;uint16_t n, nb_commit, tx_id;/* bit2 is reserved and must be set to 1 according to Spec */ uint64_t flags =IAVF_TX_DESC_CMD_EOP |IAVF_TX_DESC_CMD_ICRC ;
@@ -2380,7 +2380,7 @@ iavf_xmit_fixed_burst_vec_avx512_ctx(void *tx_queue, struct rte_mbuf **tx_pkts, {struct iavf_tx_queue *txq = (struct iavf_tx_queue *)tx_queue;volatile struct iavf_tx_desc *txdp; -struct iavf_tx_vec_entry *txep; +struct ci_tx_entry_vec *txep;uint16_t n, nb_commit, nb_mbuf, tx_id;/* bit2 is reserved and must be set to 1 according to Spec */ uint64_t flags =IAVF_TX_DESC_CMD_EOP |IAVF_TX_DESC_CMD_ICRC ;
@@ -2478,7 +2478,7 @@ iavf_tx_queue_release_mbufs_avx512(struct iavf_tx_queue *txq)const uint16_t max_desc = (uint16_t )(txq->nb_tx_desc -1 );const uint16_t end_desc = txq->tx_tail >> txq->use_ctx;/* next empty slot */ const uint16_t wrap_point = txq->nb_tx_desc >> txq->use_ctx;/* end of SW ring */ -struct iavf_tx_vec_entry *swr = (void *)txq->sw_ring; +struct ci_tx_entry_vec *swr = (void *)txq->sw_ring;if (!txq->sw_ring || txq->nb_free == max_desc)return ;
drivers/net/iavf/iavf_rxtx_vec_common.h +2 -2
@@ -19,7 +19,7 @@ static __rte_always_inline intiavf_tx_free_bufs (struct iavf_tx_queue *txq) { -struct iavf_tx_entry *txep; +struct ci_tx_entry *txep;uint32_t n;uint32_t i;int nb_free =0 ;
@@ -74,7 +74,7 @@ iavf_tx_free_bufs(struct iavf_tx_queue *txq) } static __rte_always_inline void -tx_backlog_entry (struct iavf_tx_entry *txep, +tx_backlog_entry (struct ci_tx_entry *txep,struct rte_mbuf **tx_pkts,uint16_t nb_pkts) {int i;
drivers/net/iavf/iavf_rxtx_vec_sse.c +1 -1
@@ -1368,7 +1368,7 @@ iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, {struct iavf_tx_queue *txq = (struct iavf_tx_queue *)tx_queue;volatile struct iavf_tx_desc *txdp; -struct iavf_tx_entry *txep; +struct ci_tx_entry *txep;uint16_t n, nb_commit, tx_id;uint64_t flags =IAVF_TX_DESC_CMD_EOP |0x04 ;/* bit 2 must be set */ uint64_t rs =IAVF_TX_DESC_CMD_RS | flags;
drivers/net/ice/ice_dcf_ethdev.c +1 -1
@@ -389,7 +389,7 @@ reset_rx_queue(struct ice_rx_queue *rxq)static inline void reset_tx_queue (struct ice_tx_queue *txq) { -struct ice_tx_entry *txe; +struct ci_tx_entry *txe;uint32_t i, size;uint16_t prev;
drivers/net/ice/ice_rxtx.c +8 -8
@@ -1028,7 +1028,7 @@ _ice_tx_queue_release_mbufs(struct ice_tx_queue *txq)static void ice_reset_tx_queue (struct ice_tx_queue *txq) { -struct ice_tx_entry *txe; +struct ci_tx_entry *txe;uint16_t i, prev, size;if (!txq) {
@@ -1509,7 +1509,7 @@ ice_tx_queue_setup(struct rte_eth_dev *dev,/* Allocate software ring */ txq->sw_ring =rte_zmalloc_socket (NULL , -sizeof (struct ice_tx_entry ) * nb_desc, +sizeof (struct ci_tx_entry ) * nb_desc,RTE_CACHE_LINE_SIZE , socket_id);if (!txq->sw_ring) {
@@ -2837,7 +2837,7 @@ ice_txd_enable_checksum(uint64_t ol_flags,static inline int ice_xmit_cleanup (struct ice_tx_queue *txq) { -struct ice_tx_entry *sw_ring = txq->sw_ring; +struct ci_tx_entry *sw_ring = txq->sw_ring;volatile struct ice_tx_desc *txd = txq->tx_ring;uint16_t last_desc_cleaned = txq->last_desc_cleaned;uint16_t nb_tx_desc = txq->nb_tx_desc;
@@ -2961,8 +2961,8 @@ ice_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)struct ice_tx_queue *txq;volatile struct ice_tx_desc *tx_ring;volatile struct ice_tx_desc *txd; -struct ice_tx_entry *sw_ring; -struct ice_tx_entry *txe, *txn; +struct ci_tx_entry *sw_ring; +struct ci_tx_entry *txe, *txn;struct rte_mbuf *tx_pkt;struct rte_mbuf *m_seg;uint32_t cd_tunneling_params;
@@ -3184,7 +3184,7 @@ ice_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) static __rte_always_inline intice_tx_free_bufs (struct ice_tx_queue *txq) { -struct ice_tx_entry *txep; +struct ci_tx_entry *txep;uint16_t i; if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &
@@ -3221,7 +3221,7 @@ static int ice_tx_done_cleanup_full(struct ice_tx_queue *txq, uint32_t free_cnt) { -struct ice_tx_entry *swr_ring = txq->sw_ring; +struct ci_tx_entry *swr_ring = txq->sw_ring;uint16_t i, tx_last, tx_id;uint16_t nb_tx_free_last;uint16_t nb_tx_to_clean;
@@ -3361,7 +3361,7 @@ ice_tx_fill_hw_ring(struct ice_tx_queue *txq, struct rte_mbuf **pkts, uint16_t nb_pkts) {volatile struct ice_tx_desc *txdp = &txq->tx_ring[txq->tx_tail]; -struct ice_tx_entry *txep = &txq->sw_ring[txq->tx_tail]; +struct ci_tx_entry *txep = &txq->sw_ring[txq->tx_tail];const int N_PER_LOOP =4 ;const int N_PER_LOOP_MASK =N_PER_LOOP -1 ;int mainpart, leftover;
drivers/net/ice/ice_rxtx.h +2 -11
@@ -5,6 +5,7 @@#ifndef _ICE_RXTX_H_#define _ICE_RXTX_H_ +#include <_common_intel/tx.h> #include "ice_ethdev.h" #define ICE_ALIGN_RING_DESC 32
@@ -144,21 +145,11 @@ struct ice_rx_queue {bool ts_enable;/* if rxq timestamp is enabled */ }; -struct ice_tx_entry { -struct rte_mbuf *mbuf; -uint16_t next_id; -uint16_t last_id; -}; - -struct ice_vec_tx_entry { -struct rte_mbuf *mbuf; -}; -struct ice_tx_queue {uint16_t nb_tx_desc;/* number of TX descriptors */ rte_iova_t tx_ring_dma;/* TX ring DMA address */ volatile struct ice_tx_desc *tx_ring;/* TX ring virtual address */ -struct ice_tx_entry *sw_ring;/* virtual address of SW ring */ +struct ci_tx_entry *sw_ring;/* virtual address of SW ring */ uint16_t tx_tail;/* current value of tail register */ volatile uint8_t *qtx_tail;/* register address of tail */ uint16_t nb_tx_used;/* number of TX desc used since RS bit set */
drivers/net/ice/ice_rxtx_vec_avx2.c +1 -1
@@ -858,7 +858,7 @@ ice_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, {struct ice_tx_queue *txq = (struct ice_tx_queue *)tx_queue;volatile struct ice_tx_desc *txdp; -struct ice_tx_entry *txep; +struct ci_tx_entry *txep;uint16_t n, nb_commit, tx_id;uint64_t flags =ICE_TD_CMD ;uint64_t rs =ICE_TX_DESC_CMD_RS |ICE_TD_CMD ;
drivers/net/ice/ice_rxtx_vec_avx512.c +3 -3
@@ -862,7 +862,7 @@ ice_recv_scattered_pkts_vec_avx512_offload(void *rx_queue, static __rte_always_inline intice_tx_free_bufs_avx512 (struct ice_tx_queue *txq) { -struct ice_vec_tx_entry *txep; +struct ci_tx_entry_vec *txep;uint32_t n;uint32_t i;int nb_free =0 ;
@@ -1040,7 +1040,7 @@ ice_vtx(volatile struct ice_tx_desc *txdp, struct rte_mbuf **pkt, } static __rte_always_inline void -ice_tx_backlog_entry_avx512 (struct ice_vec_tx_entry *txep, +ice_tx_backlog_entry_avx512 (struct ci_tx_entry_vec *txep,struct rte_mbuf **tx_pkts,uint16_t nb_pkts) {int i;
@@ -1055,7 +1055,7 @@ ice_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, {struct ice_tx_queue *txq = (struct ice_tx_queue *)tx_queue;volatile struct ice_tx_desc *txdp; -struct ice_vec_tx_entry *txep; +struct ci_tx_entry_vec *txep;uint16_t n, nb_commit, tx_id;uint64_t flags =ICE_TD_CMD ;uint64_t rs =ICE_TX_DESC_CMD_RS |ICE_TD_CMD ;
drivers/net/ice/ice_rxtx_vec_common.h +3 -3
@@ -15,7 +15,7 @@ static __rte_always_inline intice_tx_free_bufs_vec (struct ice_tx_queue *txq) { -struct ice_tx_entry *txep; +struct ci_tx_entry *txep;uint32_t n;uint32_t i;int nb_free =0 ;
@@ -70,7 +70,7 @@ ice_tx_free_bufs_vec(struct ice_tx_queue *txq) } static __rte_always_inline void -ice_tx_backlog_entry (struct ice_tx_entry *txep, +ice_tx_backlog_entry (struct ci_tx_entry *txep,struct rte_mbuf **tx_pkts,uint16_t nb_pkts) {int i;
@@ -135,7 +135,7 @@ _ice_tx_queue_release_mbufs_vec(struct ice_tx_queue *txq)if (dev->tx_pkt_burst == ice_xmit_pkts_vec_avx512 || dev->tx_pkt_burst == ice_xmit_pkts_vec_avx512_offload) { -struct ice_vec_tx_entry *swr = (void *)txq->sw_ring; +struct ci_tx_entry_vec *swr = (void *)txq->sw_ring;if (txq->tx_tail < i) {for (; i < txq->nb_tx_desc; i++) {
drivers/net/ice/ice_rxtx_vec_sse.c +1 -1
@@ -699,7 +699,7 @@ ice_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, {struct ice_tx_queue *txq = (struct ice_tx_queue *)tx_queue;volatile struct ice_tx_desc *txdp; -struct ice_tx_entry *txep; +struct ci_tx_entry *txep;uint16_t n, nb_commit, tx_id;uint64_t flags =ICE_TD_CMD ;uint64_t rs =ICE_TX_DESC_CMD_RS |ICE_TD_CMD ;
drivers/net/ixgbe/ixgbe_recycle_mbufs_vec_common.c +1 -1
@@ -52,7 +52,7 @@ ixgbe_recycle_tx_mbufs_reuse_vec(void *tx_queue, struct rte_eth_recycle_rxq_info *recycle_rxq_info) {struct ixgbe_tx_queue *txq = tx_queue; -struct ixgbe_tx_entry *txep; +struct ci_tx_entry *txep;struct rte_mbuf **rxep;int i, n;uint32_t status;
drivers/net/ixgbe/ixgbe_rxtx.c +8 -8
@@ -100,7 +100,7 @@ static __rte_always_inline intixgbe_tx_free_bufs (struct ixgbe_tx_queue *txq) { -struct ixgbe_tx_entry *txep; +struct ci_tx_entry *txep;uint32_t status;int i, nb_free =0 ;struct rte_mbuf *m, *free[RTE_IXGBE_TX_MAX_FREE_BUF_SZ ];
@@ -199,7 +199,7 @@ ixgbe_tx_fill_hw_ring(struct ixgbe_tx_queue *txq, struct rte_mbuf **pkts, uint16_t nb_pkts) {volatile union ixgbe_adv_tx_desc *txdp = &(txq->tx_ring[txq->tx_tail]); -struct ixgbe_tx_entry *txep = &(txq->sw_ring[txq->tx_tail]); +struct ci_tx_entry *txep = &txq->sw_ring[txq->tx_tail];const int N_PER_LOOP =4 ;const int N_PER_LOOP_MASK =N_PER_LOOP -1 ;int mainpart, leftover;
@@ -563,7 +563,7 @@ tx_desc_ol_flags_to_cmdtype(uint64_t ol_flags)static inline int ixgbe_xmit_cleanup (struct ixgbe_tx_queue *txq) { -struct ixgbe_tx_entry *sw_ring = txq->sw_ring; +struct ci_tx_entry *sw_ring = txq->sw_ring;volatile union ixgbe_adv_tx_desc *txr = txq->tx_ring;uint16_t last_desc_cleaned = txq->last_desc_cleaned;uint16_t nb_tx_desc = txq->nb_tx_desc;
@@ -624,8 +624,8 @@ ixgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) {struct ixgbe_tx_queue *txq; -struct ixgbe_tx_entry *sw_ring; -struct ixgbe_tx_entry *txe, *txn; +struct ci_tx_entry *sw_ring; +struct ci_tx_entry *txe, *txn;volatile union ixgbe_adv_tx_desc *txr;volatile union ixgbe_adv_tx_desc *txd, *txp;struct rte_mbuf *tx_pkt;
@@ -2352,7 +2352,7 @@ ixgbe_tx_queue_release_mbufs(struct ixgbe_tx_queue *txq)static int ixgbe_tx_done_cleanup_full (struct ixgbe_tx_queue *txq,uint32_t free_cnt) { -struct ixgbe_tx_entry *swr_ring = txq->sw_ring; +struct ci_tx_entry *swr_ring = txq->sw_ring;uint16_t i, tx_last, tx_id;uint16_t nb_tx_free_last;uint16_t nb_tx_to_clean;
@@ -2490,7 +2490,7 @@ static void __rte_cold ixgbe_reset_tx_queue(struct ixgbe_tx_queue *txq) {static const union ixgbe_adv_tx_desc zeroed_desc = {{0 }}; -struct ixgbe_tx_entry *txe = txq->sw_ring; +struct ci_tx_entry *txe = txq->sw_ring;uint16_t prev, i;/* Zero out HW ring memory */
@@ -2795,7 +2795,7 @@ ixgbe_dev_tx_queue_setup(struct rte_eth_dev *dev,/* Allocate software ring */ txq->sw_ring =rte_zmalloc_socket ("txq->sw_ring" , -sizeof (struct ixgbe_tx_entry ) * nb_desc, +sizeof (struct ci_tx_entry ) * nb_desc,RTE_CACHE_LINE_SIZE , socket_id);if (txq->sw_ring ==NULL ) {ixgbe_tx_queue_release (txq);
drivers/net/ixgbe/ixgbe_rxtx.h +4 -18
@@ -5,6 +5,8 @@#ifndef _IXGBE_RXTX_H_#define _IXGBE_RXTX_H_ +#include <_common_intel/tx.h> +/* * Rings setup and release. *
@@ -75,22 +77,6 @@ struct ixgbe_scattered_rx_entry {struct rte_mbuf *fbuf;/**< First segment of the fragmented packet. */ }; -/** -* Structure associated with each descriptor of the TX ring of a TX queue. -*/ -struct ixgbe_tx_entry { -struct rte_mbuf *mbuf;/**< mbuf associated with TX desc, if any. */ -uint16_t next_id;/**< Index of next descriptor in ring. */ -uint16_t last_id;/**< Index of last scattered descriptor. */ -}; - -/** -* Structure associated with each descriptor of the TX ring of a TX queue. -*/ -struct ixgbe_tx_entry_v { -struct rte_mbuf *mbuf;/**< mbuf associated with TX desc, if any. */ -}; -/** * Structure associated with each RX queue. */
@@ -202,8 +188,8 @@ struct ixgbe_tx_queue {volatile union ixgbe_adv_tx_desc *tx_ring;uint64_t tx_ring_phys_addr;/**< TX ring DMA address. */ union { -struct ixgbe_tx_entry *sw_ring;/**< address of SW ring for scalar PMD. */ -struct ixgbe_tx_entry_v *sw_ring_v;/**< address of SW ring for vector PMD */ +struct ci_tx_entry *sw_ring;/**< address of SW ring for scalar PMD. */ +struct ci_tx_entry_vec *sw_ring_v;/**< address of SW ring for vector PMD */ };volatile uint32_t *tdt_reg_addr;/**< Address of TDT register. */ uint16_t nb_tx_desc;/**< number of TX descriptors. */
drivers/net/ixgbe/ixgbe_rxtx_vec_common.h +4 -4
@@ -14,7 +14,7 @@ static __rte_always_inline intixgbe_tx_free_bufs (struct ixgbe_tx_queue *txq) { -struct ixgbe_tx_entry_v *txep; +struct ci_tx_entry_vec *txep;uint32_t status;uint32_t n;uint32_t i;
@@ -69,7 +69,7 @@ ixgbe_tx_free_bufs(struct ixgbe_tx_queue *txq) } static __rte_always_inline void -tx_backlog_entry (struct ixgbe_tx_entry_v *txep, +tx_backlog_entry (struct ci_tx_entry_vec *txep,struct rte_mbuf **tx_pkts,uint16_t nb_pkts) {int i;
@@ -82,7 +82,7 @@ static inline void _ixgbe_tx_queue_release_mbufs_vec(struct ixgbe_tx_queue *txq) {unsigned int i; -struct ixgbe_tx_entry_v *txe; +struct ci_tx_entry_vec *txe;const uint16_t max_desc = (uint16_t )(txq->nb_tx_desc -1 );if (txq->sw_ring ==NULL || txq->nb_tx_free == max_desc)
@@ -149,7 +149,7 @@ static inline void _ixgbe_reset_tx_queue_vec(struct ixgbe_tx_queue *txq) {static const union ixgbe_adv_tx_desc zeroed_desc = { {0 } }; -struct ixgbe_tx_entry_v *txe = txq->sw_ring_v; +struct ci_tx_entry_vec *txe = txq->sw_ring_v;uint16_t i;/* Zero out HW ring memory */
drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c +1 -1
@@ -573,7 +573,7 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, {struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue;volatile union ixgbe_adv_tx_desc *txdp; -struct ixgbe_tx_entry_v *txep; +struct ci_tx_entry_vec *txep;uint16_t n, nb_commit, tx_id;uint64_t flags =DCMD_DTYP_FLAGS ;uint64_t rs =IXGBE_ADVTXD_DCMD_RS |DCMD_DTYP_FLAGS ;
drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c +1 -1
@@ -695,7 +695,7 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, {struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue;volatile union ixgbe_adv_tx_desc *txdp; -struct ixgbe_tx_entry_v *txep; +struct ci_tx_entry_vec *txep;uint16_t n, nb_commit, tx_id;uint64_t flags =DCMD_DTYP_FLAGS ;uint64_t rs =IXGBE_ADVTXD_DCMD_RS |DCMD_DTYP_FLAGS ;
Signature · 1 line
2.43.0
PATCH v2 3/22 net/_common_intel: add Tx mbuf ring replenish fn
In reply to Bruce RichardsonMove the short function used to place mbufs on the SW Tx ring to common code to avoid duplication.
--- drivers/net/_common_intel/tx.h | 7 +++++++ drivers/net/i40e/i40e_rxtx_vec_altivec.c | 4 ++-- drivers/net/i40e/i40e_rxtx_vec_avx2.c | 4 ++-- drivers/net/i40e/i40e_rxtx_vec_common.h | 10 ---------- drivers/net/i40e/i40e_rxtx_vec_neon.c | 4 ++-- drivers/net/i40e/i40e_rxtx_vec_sse.c | 4 ++-- drivers/net/iavf/iavf_rxtx_vec_avx2.c | 4 ++-- drivers/net/iavf/iavf_rxtx_vec_common.h | 10 ---------- drivers/net/iavf/iavf_rxtx_vec_sse.c | 4 ++-- drivers/net/ice/ice_rxtx_vec_avx2.c | 4 ++-- drivers/net/ice/ice_rxtx_vec_common.h | 10 ---------- drivers/net/ice/ice_rxtx_vec_sse.c | 4 ++-- 12 files changed, 23 insertions(+), 46 deletions(-)
drivers/net/_common_intel/tx.h +7 -0
@@ -24,4 +24,11 @@ struct ci_tx_entry_vec {struct rte_mbuf *mbuf;/* mbuf associated with TX desc, if any. */ }; +static __rte_always_inline void +ci_tx_backlog_entry (struct ci_tx_entry *txep,struct rte_mbuf **tx_pkts,uint16_t nb_pkts) +{ +for (uint16_t i =0 ; i < (int )nb_pkts; ++i) + txep[i].mbuf = tx_pkts[i]; +} +#endif /* _COMMON_INTEL_TX_H_ */
drivers/net/i40e/i40e_rxtx_vec_altivec.c +2 -2
@@ -575,7 +575,7 @@ i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) { -tx_backlog_entry (txep, tx_pkts, n); +ci_tx_backlog_entry (txep, tx_pkts, n);for (i =0 ; i < n -1 ; ++i, ++tx_pkts, ++txdp)vtx1 (txdp, *tx_pkts, flags);
@@ -592,7 +592,7 @@ i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, txep = &txq->sw_ring[tx_id]; } -tx_backlog_entry (txep, tx_pkts, nb_commit); +ci_tx_backlog_entry (txep, tx_pkts, nb_commit);vtx (txdp, tx_pkts, nb_commit, flags);
drivers/net/i40e/i40e_rxtx_vec_avx2.c +2 -2
@@ -765,7 +765,7 @@ i40e_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) { -tx_backlog_entry (txep, tx_pkts, n); +ci_tx_backlog_entry (txep, tx_pkts, n);vtx (txdp, tx_pkts, n -1 , flags); tx_pkts += (n -1 );
@@ -783,7 +783,7 @@ i40e_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, txep = &txq->sw_ring[tx_id]; } -tx_backlog_entry (txep, tx_pkts, nb_commit); +ci_tx_backlog_entry (txep, tx_pkts, nb_commit);vtx (txdp, tx_pkts, nb_commit, flags);
drivers/net/i40e/i40e_rxtx_vec_common.h +0 -10
@@ -84,16 +84,6 @@ i40e_tx_free_bufs(struct i40e_tx_queue *txq)return txq->tx_rs_thresh; } -static __rte_always_inline void -tx_backlog_entry (struct ci_tx_entry *txep, -struct rte_mbuf **tx_pkts,uint16_t nb_pkts) -{ -int i; - -for (i =0 ; i < (int )nb_pkts; ++i) - txep[i].mbuf = tx_pkts[i]; -} -static inline void _i40e_rx_queue_release_mbufs_vec (struct i40e_rx_queue *rxq) {
drivers/net/i40e/i40e_rxtx_vec_neon.c +2 -2
@@ -702,7 +702,7 @@ i40e_xmit_fixed_burst_vec(void *__rte_restrict tx_queue, n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) { -tx_backlog_entry (txep, tx_pkts, n); +ci_tx_backlog_entry (txep, tx_pkts, n);for (i =0 ; i < n -1 ; ++i, ++tx_pkts, ++txdp)vtx1 (txdp, *tx_pkts, flags);
@@ -719,7 +719,7 @@ i40e_xmit_fixed_burst_vec(void *__rte_restrict tx_queue, txep = &txq->sw_ring[tx_id]; } -tx_backlog_entry (txep, tx_pkts, nb_commit); +ci_tx_backlog_entry (txep, tx_pkts, nb_commit);vtx (txdp, tx_pkts, nb_commit, flags);
drivers/net/i40e/i40e_rxtx_vec_sse.c +2 -2
@@ -721,7 +721,7 @@ i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) { -tx_backlog_entry (txep, tx_pkts, n); +ci_tx_backlog_entry (txep, tx_pkts, n);for (i =0 ; i < n -1 ; ++i, ++tx_pkts, ++txdp)vtx1 (txdp, *tx_pkts, flags);
@@ -738,7 +738,7 @@ i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, txep = &txq->sw_ring[tx_id]; } -tx_backlog_entry (txep, tx_pkts, nb_commit); +ci_tx_backlog_entry (txep, tx_pkts, nb_commit);vtx (txdp, tx_pkts, nb_commit, flags);
drivers/net/iavf/iavf_rxtx_vec_avx2.c +2 -2
@@ -1757,7 +1757,7 @@ iavf_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) { -tx_backlog_entry (txep, tx_pkts, n); +ci_tx_backlog_entry (txep, tx_pkts, n);iavf_vtx (txdp, tx_pkts, n -1 , flags, offload); tx_pkts += (n -1 );
@@ -1775,7 +1775,7 @@ iavf_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, txep = &txq->sw_ring[tx_id]; } -tx_backlog_entry (txep, tx_pkts, nb_commit); +ci_tx_backlog_entry (txep, tx_pkts, nb_commit);iavf_vtx (txdp, tx_pkts, nb_commit, flags, offload);
drivers/net/iavf/iavf_rxtx_vec_common.h +0 -10
@@ -73,16 +73,6 @@ iavf_tx_free_bufs(struct iavf_tx_queue *txq)return txq->rs_thresh; } -static __rte_always_inline void -tx_backlog_entry (struct ci_tx_entry *txep, -struct rte_mbuf **tx_pkts,uint16_t nb_pkts) -{ -int i; - -for (i =0 ; i < (int )nb_pkts; ++i) - txep[i].mbuf = tx_pkts[i]; -} -static inline void _iavf_rx_queue_release_mbufs_vec (struct iavf_rx_queue *rxq) {
drivers/net/iavf/iavf_rxtx_vec_sse.c +2 -2
@@ -1390,7 +1390,7 @@ iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) { -tx_backlog_entry (txep, tx_pkts, n); +ci_tx_backlog_entry (txep, tx_pkts, n);for (i =0 ; i < n -1 ; ++i, ++tx_pkts, ++txdp)vtx1 (txdp, *tx_pkts, flags);
@@ -1407,7 +1407,7 @@ iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, txep = &txq->sw_ring[tx_id]; } -tx_backlog_entry (txep, tx_pkts, nb_commit); +ci_tx_backlog_entry (txep, tx_pkts, nb_commit);iavf_vtx (txdp, tx_pkts, nb_commit, flags);
drivers/net/ice/ice_rxtx_vec_avx2.c +2 -2
@@ -881,7 +881,7 @@ ice_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) { -ice_tx_backlog_entry (txep, tx_pkts, n); +ci_tx_backlog_entry (txep, tx_pkts, n);ice_vtx (txdp, tx_pkts, n -1 , flags, offload); tx_pkts += (n -1 );
@@ -899,7 +899,7 @@ ice_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, txep = &txq->sw_ring[tx_id]; } -ice_tx_backlog_entry (txep, tx_pkts, nb_commit); +ci_tx_backlog_entry (txep, tx_pkts, nb_commit);ice_vtx (txdp, tx_pkts, nb_commit, flags, offload);
drivers/net/ice/ice_rxtx_vec_common.h +0 -10
@@ -69,16 +69,6 @@ ice_tx_free_bufs_vec(struct ice_tx_queue *txq)return txq->tx_rs_thresh; } -static __rte_always_inline void -ice_tx_backlog_entry (struct ci_tx_entry *txep, -struct rte_mbuf **tx_pkts,uint16_t nb_pkts) -{ -int i; - -for (i =0 ; i < (int )nb_pkts; ++i) - txep[i].mbuf = tx_pkts[i]; -} -static inline void _ice_rx_queue_release_mbufs_vec (struct ice_rx_queue *rxq) {
drivers/net/ice/ice_rxtx_vec_sse.c +2 -2
@@ -724,7 +724,7 @@ ice_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) { -ice_tx_backlog_entry (txep, tx_pkts, n); +ci_tx_backlog_entry (txep, tx_pkts, n);for (i =0 ; i < n -1 ; ++i, ++tx_pkts, ++txdp)ice_vtx1 (txdp, *tx_pkts, flags);
@@ -741,7 +741,7 @@ ice_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, txep = &txq->sw_ring[tx_id]; } -ice_tx_backlog_entry (txep, tx_pkts, nb_commit); +ci_tx_backlog_entry (txep, tx_pkts, nb_commit);ice_vtx (txdp, tx_pkts, nb_commit, flags);
Signature · 1 line
2.43.0
PATCH v2 4/22 drivers/net: align Tx queue struct field names
In reply to Bruce RichardsonAcross the various Intel drivers sometimes different names are given to fields in the Tx queue structure which have the same function. Do some renaming to align things better for future merging.
--- drivers/net/i40e/i40e_rxtx.c | 6 +-- drivers/net/i40e/i40e_rxtx.h | 2 +- drivers/net/iavf/iavf_rxtx.c | 60 ++++++++++++------------- drivers/net/iavf/iavf_rxtx.h | 14 +++--- drivers/net/iavf/iavf_rxtx_vec_avx2.c | 19 ++++---- drivers/net/iavf/iavf_rxtx_vec_avx512.c | 57 +++++++++++------------ drivers/net/iavf/iavf_rxtx_vec_common.h | 24 +++++----- drivers/net/iavf/iavf_rxtx_vec_sse.c | 18 ++++---- drivers/net/iavf/iavf_vchnl.c | 2 +- drivers/net/ixgbe/base/ixgbe_osdep.h | 2 +- drivers/net/ixgbe/ixgbe_rxtx.c | 16 +++---- drivers/net/ixgbe/ixgbe_rxtx.h | 6 +-- drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c | 2 +- drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c | 2 +- 14 files changed, 116 insertions(+), 114 deletions(-)
drivers/net/i40e/i40e_rxtx.c +3 -3
@@ -2549,7 +2549,7 @@ i40e_dev_tx_queue_setup(struct rte_eth_dev *dev, txq->vsi = vsi; txq->tx_deferred_start = tx_conf->tx_deferred_start; - txq->tx_ring_phys_addr = tz->iova; + txq->tx_ring_dma = tz->iova; txq->tx_ring = (struct i40e_tx_desc *)tz->addr;/* Allocate software ring */
@@ -2923,7 +2923,7 @@ i40e_tx_queue_init(struct i40e_tx_queue *txq)/* clear the context structure first */ memset (&tx_ctx,0 ,sizeof (tx_ctx)); tx_ctx.new_context =1 ; - tx_ctx.base = txq->tx_ring_phys_addr /I40E_QUEUE_BASE_ADDR_UNIT ; + tx_ctx.base = txq->tx_ring_dma /I40E_QUEUE_BASE_ADDR_UNIT ; tx_ctx.qlen = txq->nb_tx_desc;#ifdef RTE_LIBRTE_IEEE1588
@@ -3209,7 +3209,7 @@ i40e_fdir_setup_tx_resources(struct i40e_pf *pf) txq->reg_idx = pf->fdir.fdir_vsi->base_queue; txq->vsi = pf->fdir.fdir_vsi; - txq->tx_ring_phys_addr = tz->iova; + txq->tx_ring_dma = tz->iova; txq->tx_ring = (struct i40e_tx_desc *)tz->addr;/*
drivers/net/i40e/i40e_rxtx.h +1 -1
@@ -129,7 +129,7 @@ struct i40e_rx_queue {*/ struct i40e_tx_queue {uint16_t nb_tx_desc;/**< number of TX descriptors */ -uint64_t tx_ring_phys_addr;/**< TX ring DMA address */ +rte_iova_t tx_ring_dma;/**< TX ring DMA address */ volatile struct i40e_tx_desc *tx_ring;/**< TX ring virtual address */ struct ci_tx_entry *sw_ring;/**< virtual address of SW ring */ uint16_t tx_tail;/**< current value of tail register */
drivers/net/iavf/iavf_rxtx.c +30 -30
161 lines not on this page · raw message
drivers/net/iavf/iavf_rxtx.h +7 -7
@@ -277,25 +277,25 @@ struct iavf_rx_queue {struct iavf_tx_queue {const struct rte_memzone *mz;/* memzone for Tx ring */ volatile struct iavf_tx_desc *tx_ring;/* Tx ring virtual address */ -uint64_t tx_ring_phys_addr;/* Tx ring DMA address */ +rte_iova_t tx_ring_dma;/* Tx ring DMA address */ struct ci_tx_entry *sw_ring;/* address array of SW ring */ uint16_t nb_tx_desc;/* ring length */ uint16_t tx_tail;/* current value of tail */ volatile uint8_t *qtx_tail;/* register address of tail */ /* number of used desc since RS bit set */ -uint16_t nb_used; -uint16_t nb_free; +uint16_t nb_tx_used; +uint16_t nb_tx_free;uint16_t last_desc_cleaned;/* last desc have been cleaned*/ -uint16_t free_thresh; -uint16_t rs_thresh; +uint16_t tx_free_thresh; +uint16_t tx_rs_thresh;uint8_t rel_mbufs_type;struct iavf_vsi *vsi;/**< the VSI this queue belongs to */ uint16_t port_id;uint16_t queue_id;uint64_t offloads; -uint16_t next_dd;/* next to set RS, for VPMD */ -uint16_t next_rs;/* next to check DD, for VPMD */ +uint16_t tx_next_dd;/* next to set RS, for VPMD */ +uint16_t tx_next_rs;/* next to check DD, for VPMD */ uint16_t ipsec_crypto_pkt_md_offset;uint64_t mbuf_errors;
drivers/net/iavf/iavf_rxtx_vec_avx2.c +10 -9
@@ -1742,18 +1742,19 @@ iavf_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts,uint64_t flags =IAVF_TX_DESC_CMD_EOP |IAVF_TX_DESC_CMD_ICRC ;uint64_t rs =IAVF_TX_DESC_CMD_RS | flags; -if (txq->nb_free < txq->free_thresh) +if (txq->nb_tx_free < txq->tx_free_thresh)iavf_tx_free_bufs (txq); - nb_commit = nb_pkts = (uint16_t )RTE_MIN (txq->nb_free, nb_pkts); + nb_pkts = (uint16_t )RTE_MIN (txq->nb_tx_free, nb_pkts);if (unlikely (nb_pkts ==0 ))return 0 ; + nb_commit = nb_pkts; tx_id = txq->tx_tail; txdp = &txq->tx_ring[tx_id]; txep = &txq->sw_ring[tx_id]; - txq->nb_free = (uint16_t )(txq->nb_free - nb_pkts); + txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts); n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) {
@@ -1768,7 +1769,7 @@ iavf_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, nb_commit = (uint16_t )(nb_commit - n); tx_id =0 ; - txq->next_rs = (uint16_t )(txq->rs_thresh -1 ); + txq->tx_next_rs = (uint16_t )(txq->tx_rs_thresh -1 );/* avoid reach the end of ring */ txdp = &txq->tx_ring[tx_id];
@@ -1780,12 +1781,12 @@ iavf_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts,iavf_vtx (txdp, tx_pkts, nb_commit, flags, offload); tx_id = (uint16_t )(tx_id + nb_commit); -if (tx_id > txq->next_rs) { - txq->tx_ring[txq->next_rs].cmd_type_offset_bsz |= +if (tx_id > txq->tx_next_rs) { + txq->tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |=rte_cpu_to_le_64 (((uint64_t )IAVF_TX_DESC_CMD_RS ) <<IAVF_TXD_QW1_CMD_SHIFT ); - txq->next_rs = - (uint16_t )(txq->next_rs + txq->rs_thresh); + txq->tx_next_rs = + (uint16_t )(txq->tx_next_rs + txq->tx_rs_thresh); } txq->tx_tail = tx_id;
@@ -1806,7 +1807,7 @@ iavf_xmit_pkts_vec_avx2_common(void *tx_queue, struct rte_mbuf **tx_pkts,uint16_t ret, num;/* cross rs_thresh boundary is not allowed */ - num = (uint16_t )RTE_MIN (nb_pkts, txq->rs_thresh); + num = (uint16_t )RTE_MIN (nb_pkts, txq->tx_rs_thresh); ret =iavf_xmit_fixed_burst_vec_avx2 (tx_queue, &tx_pkts[nb_tx], num, offload); nb_tx += ret;
drivers/net/iavf/iavf_rxtx_vec_avx512.c +29 -28
157 lines not on this page · raw message
drivers/net/iavf/iavf_rxtx_vec_common.h +12 -12
@@ -26,17 +26,17 @@ iavf_tx_free_bufs(struct iavf_tx_queue *txq)struct rte_mbuf *m, *free[IAVF_VPMD_TX_MAX_FREE_BUF ];/* check DD bits on threshold descriptor */ -if ((txq->tx_ring[txq->next_dd].cmd_type_offset_bsz & +if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &rte_cpu_to_le_64 (IAVF_TXD_QW1_DTYPE_MASK )) !=rte_cpu_to_le_64 (IAVF_TX_DESC_DTYPE_DESC_DONE ))return 0 ; - n = txq->rs_thresh; + n = txq->tx_rs_thresh;/* first buffer to free from S/W ring is at index * tx_next_dd - (tx_rs_thresh-1) */ - txep = &txq->sw_ring[txq->next_dd - (n -1 )]; + txep = &txq->sw_ring[txq->tx_next_dd - (n -1 )]; m =rte_pktmbuf_prefree_seg (txep[0 ].mbuf);if (likely (m !=NULL )) { free[0 ] = m;
@@ -65,12 +65,12 @@ iavf_tx_free_bufs(struct iavf_tx_queue *txq) }/* buffers were freed, update counters */ - txq->nb_free = (uint16_t )(txq->nb_free + txq->rs_thresh); - txq->next_dd = (uint16_t )(txq->next_dd + txq->rs_thresh); -if (txq->next_dd >= txq->nb_tx_desc) - txq->next_dd = (uint16_t )(txq->rs_thresh -1 ); + txq->nb_tx_free = (uint16_t )(txq->nb_tx_free + txq->tx_rs_thresh); + txq->tx_next_dd = (uint16_t )(txq->tx_next_dd + txq->tx_rs_thresh); +if (txq->tx_next_dd >= txq->nb_tx_desc) + txq->tx_next_dd = (uint16_t )(txq->tx_rs_thresh -1 ); -return txq->rs_thresh; +return txq->tx_rs_thresh; } static inline void
@@ -109,10 +109,10 @@ _iavf_tx_queue_release_mbufs_vec(struct iavf_tx_queue *txq)unsigned i;const uint16_t max_desc = (uint16_t )(txq->nb_tx_desc -1 ); -if (!txq->sw_ring || txq->nb_free == max_desc) +if (!txq->sw_ring || txq->nb_tx_free == max_desc)return ; - i = txq->next_dd - txq->rs_thresh +1 ; + i = txq->tx_next_dd - txq->tx_rs_thresh +1 ;while (i != txq->tx_tail) {rte_pktmbuf_free_seg (txq->sw_ring[i].mbuf); txq->sw_ring[i].mbuf =NULL ;
@@ -169,8 +169,8 @@ iavf_tx_vec_queue_default(struct iavf_tx_queue *txq)if (!txq)return -1 ; -if (txq->rs_thresh <IAVF_VPMD_TX_MAX_BURST || - txq->rs_thresh >IAVF_VPMD_TX_MAX_FREE_BUF ) +if (txq->tx_rs_thresh <IAVF_VPMD_TX_MAX_BURST || + txq->tx_rs_thresh >IAVF_VPMD_TX_MAX_FREE_BUF )return -1 ;if (txq->offloads &IAVF_TX_NO_VECTOR_FLAGS )
drivers/net/iavf/iavf_rxtx_vec_sse.c +9 -9
@@ -1374,10 +1374,10 @@ iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,uint64_t rs =IAVF_TX_DESC_CMD_RS | flags;int i; -if (txq->nb_free < txq->free_thresh) +if (txq->nb_tx_free < txq->tx_free_thresh)iavf_tx_free_bufs (txq); - nb_pkts = (uint16_t )RTE_MIN (txq->nb_free, nb_pkts); + nb_pkts = (uint16_t )RTE_MIN (txq->nb_tx_free, nb_pkts);if (unlikely (nb_pkts ==0 ))return 0 ; nb_commit = nb_pkts;
@@ -1386,7 +1386,7 @@ iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, txdp = &txq->tx_ring[tx_id]; txep = &txq->sw_ring[tx_id]; - txq->nb_free = (uint16_t )(txq->nb_free - nb_pkts); + txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts); n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) {
@@ -1400,7 +1400,7 @@ iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, nb_commit = (uint16_t )(nb_commit - n); tx_id =0 ; - txq->next_rs = (uint16_t )(txq->rs_thresh -1 ); + txq->tx_next_rs = (uint16_t )(txq->tx_rs_thresh -1 );/* avoid reach the end of ring */ txdp = &txq->tx_ring[tx_id];
@@ -1412,12 +1412,12 @@ iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,iavf_vtx (txdp, tx_pkts, nb_commit, flags); tx_id = (uint16_t )(tx_id + nb_commit); -if (tx_id > txq->next_rs) { - txq->tx_ring[txq->next_rs].cmd_type_offset_bsz |= +if (tx_id > txq->tx_next_rs) { + txq->tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |=rte_cpu_to_le_64 (((uint64_t )IAVF_TX_DESC_CMD_RS ) <<IAVF_TXD_QW1_CMD_SHIFT ); - txq->next_rs = - (uint16_t )(txq->next_rs + txq->rs_thresh); + txq->tx_next_rs = + (uint16_t )(txq->tx_next_rs + txq->tx_rs_thresh); } txq->tx_tail = tx_id;
@@ -1441,7 +1441,7 @@ iavf_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,uint16_t ret, num;/* cross rs_thresh boundary is not allowed */ - num = (uint16_t )RTE_MIN (nb_pkts, txq->rs_thresh); + num = (uint16_t )RTE_MIN (nb_pkts, txq->tx_rs_thresh); ret =iavf_xmit_fixed_burst_vec (tx_queue, &tx_pkts[nb_tx], num); nb_tx += ret; nb_pkts -= ret;
drivers/net/iavf/iavf_vchnl.c +1 -1
@@ -1247,7 +1247,7 @@ iavf_configure_queues(struct iavf_adapter *adapter,/* Virtchnnl configure tx queues by pairs */ if (i < adapter->dev_data->nb_tx_queues) { vc_qp->txq.ring_len = txq[i]->nb_tx_desc; - vc_qp->txq.dma_ring_addr = txq[i]->tx_ring_phys_addr; + vc_qp->txq.dma_ring_addr = txq[i]->tx_ring_dma; } vc_qp->rxq.vsi_id = vf->vsi_res->vsi_id;
drivers/net/ixgbe/base/ixgbe_osdep.h +1 -1
@@ -124,7 +124,7 @@ static inline uint32_t ixgbe_read_addr(volatile void* addr) rte_write32_wc_relaxed((rte_cpu_to_le_32(value)), reg)#define IXGBE_PCI_REG_ADDR (hw, reg) \ - ((volatile uint32_t *)((char *)(hw)->hw_addr + (reg))) + ((volatile void *)((char *)(hw)->hw_addr + (reg)))#define IXGBE_PCI_REG_ARRAY_ADDR (hw, reg, index) \ IXGBE_PCI_REG_ADDR((hw), (reg) + ((index) << 2))
drivers/net/ixgbe/ixgbe_rxtx.c +8 -8
@@ -308,7 +308,7 @@ tx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,/* update tail pointer */ rte_wmb (); -IXGBE_PCI_REG_WC_WRITE_RELAXED (txq->tdt_reg_addr, txq->tx_tail); +IXGBE_PCI_REG_WC_WRITE_RELAXED (txq->qtx_tail, txq->tx_tail);return nb_pkts; }
@@ -946,7 +946,7 @@ ixgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,PMD_TX_LOG (DEBUG ,"port_id=%u queue_id=%u tx_tail=%u nb_tx=%u" , (unsigned ) txq->port_id, (unsigned ) txq->queue_id, (unsigned ) tx_id, (unsigned ) nb_tx); -IXGBE_PCI_REG_WC_WRITE_RELAXED (txq->tdt_reg_addr, tx_id); +IXGBE_PCI_REG_WC_WRITE_RELAXED (txq->qtx_tail, tx_id); txq->tx_tail = tx_id;return nb_tx;
@@ -2786,11 +2786,11 @@ ixgbe_dev_tx_queue_setup(struct rte_eth_dev *dev, hw->mac.type == ixgbe_mac_X550_vf || hw->mac.type == ixgbe_mac_X550EM_x_vf || hw->mac.type == ixgbe_mac_X550EM_a_vf) - txq->tdt_reg_addr = IXGBE_PCI_REG_ADDR(hw, IXGBE_VFTDT(queue_idx)); + txq->qtx_tail = IXGBE_PCI_REG_ADDR(hw, IXGBE_VFTDT(queue_idx)); else - txq->tdt_reg_addr = IXGBE_PCI_REG_ADDR(hw, IXGBE_TDT(txq->reg_idx)); + txq->qtx_tail = IXGBE_PCI_REG_ADDR(hw, IXGBE_TDT(txq->reg_idx)); - txq->tx_ring_phys_addr = tz->iova; + txq->tx_ring_dma = tz->iova; txq->tx_ring = (union ixgbe_adv_tx_desc *) tz->addr;/* Allocate software ring */
@@ -2802,7 +2802,7 @@ ixgbe_dev_tx_queue_setup(struct rte_eth_dev *dev,return -ENOMEM ; }PMD_INIT_LOG (DEBUG ,"sw_ring=%p hw_ring=%p dma_addr=0x%" PRIx64, - txq->sw_ring, txq->tx_ring, txq->tx_ring_phys_addr); + txq->sw_ring, txq->tx_ring, txq->tx_ring_dma);/* set up vector or scalar TX function as appropriate */ ixgbe_set_tx_function (dev, txq);
@@ -5303,7 +5303,7 @@ ixgbe_dev_tx_init(struct rte_eth_dev *dev)for (i =0 ; i < dev->data->nb_tx_queues; i++) { txq = dev->data->tx_queues[i]; - bus_addr = txq->tx_ring_phys_addr; + bus_addr = txq->tx_ring_dma;IXGBE_WRITE_REG (hw,IXGBE_TDBAL (txq->reg_idx), (uint32_t )(bus_addr &0x00000000ffffffffULL )); IXGBE_WRITE_REG(hw, IXGBE_TDBAH(txq->reg_idx),
@@ -5887,7 +5887,7 @@ ixgbevf_dev_tx_init(struct rte_eth_dev *dev)/* Setup the Base and Length of the Tx Descriptor Rings */ for (i =0 ; i < dev->data->nb_tx_queues; i++) { txq = dev->data->tx_queues[i]; - bus_addr = txq->tx_ring_phys_addr; + bus_addr = txq->tx_ring_dma;IXGBE_WRITE_REG (hw,IXGBE_VFTDBAL (i), (uint32_t )(bus_addr &0x00000000ffffffffULL )); IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(i),
drivers/net/ixgbe/ixgbe_rxtx.h +3 -3
@@ -186,12 +186,12 @@ struct ixgbe_advctx_info {struct ixgbe_tx_queue {/** TX ring virtual address. */ volatile union ixgbe_adv_tx_desc *tx_ring; -uint64_t tx_ring_phys_addr;/**< TX ring DMA address. */ +rte_iova_t tx_ring_dma;/**< TX ring DMA address. */ union {struct ci_tx_entry *sw_ring;/**< address of SW ring for scalar PMD. */ struct ci_tx_entry_vec *sw_ring_v;/**< address of SW ring for vector PMD */ }; -volatile uint32_t *tdt_reg_addr;/**< Address of TDT register. */ +volatile uint8_t *qtx_tail;/**< Address of TDT register. */ uint16_t nb_tx_desc;/**< number of TX descriptors. */ uint16_t tx_tail;/**< current value of TDT reg. */ /**< Start freeing TX buffers if there are less free descriptors than
@@ -218,7 +218,7 @@ struct ixgbe_tx_queue {/** Hardware context0 history. */ struct ixgbe_advctx_info ctx_cache[IXGBE_CTX_NUM ];const struct ixgbe_txq_ops *ops;/**< txq ops */ -uint8_t tx_deferred_start;/**< not in global dev start. */ +bool tx_deferred_start;/**< not in global dev start. */ #ifdef RTE_LIB_SECURITY uint8_t using_ipsec;/**< indicates that IPsec TX feature is in use */
drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c +1 -1
@@ -628,7 +628,7 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, txq->tx_tail = tx_id; -IXGBE_PCI_REG_WRITE (txq->tdt_reg_addr, txq->tx_tail); +IXGBE_PCI_REG_WRITE (txq->qtx_tail, txq->tx_tail);return nb_pkts; }
drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c +1 -1
@@ -751,7 +751,7 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, txq->tx_tail = tx_id; -IXGBE_PCI_REG_WC_WRITE (txq->tdt_reg_addr, txq->tx_tail); +IXGBE_PCI_REG_WC_WRITE (txq->qtx_tail, txq->tx_tail);return nb_pkts; }
Signature · 1 line
2.43.0
PATCH v2 5/22 drivers/net: add prefix for driver-specific structs
In reply to Bruce RichardsonIn preparation for merging the Tx structs for multiple drivers into a single struct, rename the driver-specific pointers in each struct to have a prefix on it, to avoid conflicts.
--- drivers/net/i40e/i40e_fdir.c | 6 +-- .../net/i40e/i40e_recycle_mbufs_vec_common.c | 2 +- drivers/net/i40e/i40e_rxtx.c | 30 ++++++------ drivers/net/i40e/i40e_rxtx.h | 4 +- drivers/net/i40e/i40e_rxtx_vec_altivec.c | 6 +-- drivers/net/i40e/i40e_rxtx_vec_avx2.c | 6 +-- drivers/net/i40e/i40e_rxtx_vec_avx512.c | 8 ++-- drivers/net/i40e/i40e_rxtx_vec_common.h | 2 +- drivers/net/i40e/i40e_rxtx_vec_neon.c | 6 +-- drivers/net/i40e/i40e_rxtx_vec_sse.c | 6 +-- drivers/net/iavf/iavf_rxtx.c | 24 +++++----- drivers/net/iavf/iavf_rxtx.h | 4 +- drivers/net/iavf/iavf_rxtx_vec_avx2.c | 6 +-- drivers/net/iavf/iavf_rxtx_vec_avx512.c | 14 +++--- drivers/net/iavf/iavf_rxtx_vec_common.h | 2 +- drivers/net/iavf/iavf_rxtx_vec_sse.c | 6 +-- drivers/net/ice/ice_dcf_ethdev.c | 4 +- drivers/net/ice/ice_rxtx.c | 48 +++++++++---------- drivers/net/ice/ice_rxtx.h | 4 +- drivers/net/ice/ice_rxtx_vec_avx2.c | 6 +-- drivers/net/ice/ice_rxtx_vec_avx512.c | 8 ++-- drivers/net/ice/ice_rxtx_vec_common.h | 4 +- drivers/net/ice/ice_rxtx_vec_sse.c | 6 +-- .../ixgbe/ixgbe_recycle_mbufs_vec_common.c | 2 +- drivers/net/ixgbe/ixgbe_rxtx.c | 22 ++++----- drivers/net/ixgbe/ixgbe_rxtx.h | 2 +- drivers/net/ixgbe/ixgbe_rxtx_vec_common.h | 6 +-- drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c | 6 +-- drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c | 6 +-- 29 files changed, 128 insertions(+), 128 deletions(-)
drivers/net/i40e/i40e_fdir.c +3 -3
@@ -1383,7 +1383,7 @@ i40e_find_available_buffer(struct rte_eth_dev *dev)volatile struct i40e_tx_desc *tmp_txdp; tmp_tail = txq->tx_tail; - tmp_txdp = &txq->tx_ring[tmp_tail +1 ]; + tmp_txdp = &txq->i40e_tx_ring[tmp_tail +1 ]; do { if ((tmp_txdp->cmd_type_offset_bsz &
@@ -1640,7 +1640,7 @@ i40e_flow_fdir_filter_programming(struct i40e_pf *pf,PMD_DRV_LOG (INFO ,"filling filter programming descriptor." ); fdirdp = (volatile struct i40e_filter_program_desc *) - (&txq->tx_ring[txq->tx_tail]); + (&txq->i40e_tx_ring[txq->tx_tail]); fdirdp->qindex_flex_ptype_vsi = rte_cpu_to_le_32((fdir_action->rx_queue <<
@@ -1710,7 +1710,7 @@ i40e_flow_fdir_filter_programming(struct i40e_pf *pf, fdirdp->fd_id =rte_cpu_to_le_32 (filter->soft_id);PMD_DRV_LOG (INFO ,"filling transmit descriptor." ); - txdp = &txq->tx_ring[txq->tx_tail +1 ]; + txdp = &txq->i40e_tx_ring[txq->tx_tail +1 ]; txdp->buffer_addr =rte_cpu_to_le_64 (pf->fdir.dma_addr[txq->tx_tail >>1 ]); td_cmd = I40E_TX_DESC_CMD_EOP |
drivers/net/i40e/i40e_recycle_mbufs_vec_common.c +1 -1
@@ -75,7 +75,7 @@ i40e_recycle_tx_mbufs_reuse_vec(void *tx_queue,return 0 ;/* check DD bits on threshold descriptor */ -if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz & +if ((txq->i40e_tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &rte_cpu_to_le_64 (I40E_TXD_QW1_DTYPE_MASK )) !=rte_cpu_to_le_64 (I40E_TX_DESC_DTYPE_DESC_DONE ))return 0 ;
drivers/net/i40e/i40e_rxtx.c +15 -15
@@ -379,7 +379,7 @@ static inline int i40e_xmit_cleanup(struct i40e_tx_queue *txq) {struct ci_tx_entry *sw_ring = txq->sw_ring; -volatile struct i40e_tx_desc *txd = txq->tx_ring; +volatile struct i40e_tx_desc *txd = txq->i40e_tx_ring;uint16_t last_desc_cleaned = txq->last_desc_cleaned;uint16_t nb_tx_desc = txq->nb_tx_desc;uint16_t desc_to_clean_to;
@@ -1103,7 +1103,7 @@ i40e_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) txq = tx_queue; sw_ring = txq->sw_ring; - txr = txq->tx_ring; + txr = txq->i40e_tx_ring; tx_id = txq->tx_tail; txe = &sw_ring[tx_id];
@@ -1338,7 +1338,7 @@ i40e_tx_free_bufs(struct i40e_tx_queue *txq)const uint16_t k =RTE_ALIGN_FLOOR (tx_rs_thresh,RTE_I40E_TX_MAX_FREE_BUF_SZ );const uint16_t m = tx_rs_thresh %RTE_I40E_TX_MAX_FREE_BUF_SZ ; -if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz & +if ((txq->i40e_tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &rte_cpu_to_le_64 (I40E_TXD_QW1_DTYPE_MASK )) !=rte_cpu_to_le_64 (I40E_TX_DESC_DTYPE_DESC_DONE ))return 0 ;
@@ -1417,7 +1417,7 @@ i40e_tx_fill_hw_ring(struct i40e_tx_queue *txq, struct rte_mbuf **pkts, uint16_t nb_pkts) { -volatile struct i40e_tx_desc *txdp = &(txq->tx_ring[txq->tx_tail]); +volatile struct i40e_tx_desc *txdp = &txq->i40e_tx_ring[txq->tx_tail];struct ci_tx_entry *txep = &txq->sw_ring[txq->tx_tail];const int N_PER_LOOP =4 ;const int N_PER_LOOP_MASK =N_PER_LOOP -1 ;
@@ -1445,7 +1445,7 @@ tx_xmit_pkts(struct i40e_tx_queue *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { -volatile struct i40e_tx_desc *txr = txq->tx_ring; +volatile struct i40e_tx_desc *txr = txq->i40e_tx_ring;uint16_t n =0 ;/**
@@ -1556,7 +1556,7 @@ i40e_xmit_pkts_check(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pktsbool pkt_error = false;const char *reason =NULL ;uint16_t good_pkts = nb_pkts; -struct i40e_adapter *adapter = txq->vsi->adapter; +struct i40e_adapter *adapter = txq->i40e_vsi->adapter;for (idx =0 ; idx < nb_pkts; idx++) { mb = tx_pkts[idx];
@@ -2329,7 +2329,7 @@ i40e_dev_tx_descriptor_status(void *tx_queue, uint16_t offset) desc -= txq->nb_tx_desc; } - status = &txq->tx_ring[desc].cmd_type_offset_bsz; + status = &txq->i40e_tx_ring[desc].cmd_type_offset_bsz; mask =rte_le_to_cpu_64 (I40E_TXD_QW1_DTYPE_MASK ); expect =rte_cpu_to_le_64 (I40E_TX_DESC_DTYPE_DESC_DONE <<I40E_TXD_QW1_DTYPE_SHIFT );
@@ -2527,7 +2527,7 @@ i40e_dev_tx_queue_setup(struct rte_eth_dev *dev,/* Allocate TX hardware ring descriptors. */ ring_size =sizeof (struct i40e_tx_desc ) *I40E_MAX_RING_DESC ; ring_size =RTE_ALIGN (ring_size,I40E_DMA_MEM_ALIGN ); - tz =rte_eth_dma_zone_reserve (dev,"tx_ring" , queue_idx, + tz =rte_eth_dma_zone_reserve (dev,"i40e_tx_ring" , queue_idx, ring_size,I40E_RING_BASE_ALIGN , socket_id);if (!tz) {i40e_tx_queue_release (txq);
@@ -2546,11 +2546,11 @@ i40e_dev_tx_queue_setup(struct rte_eth_dev *dev, txq->reg_idx = reg_idx; txq->port_id = dev->data->port_id; txq->offloads = offloads; - txq->vsi = vsi; + txq->i40e_vsi = vsi; txq->tx_deferred_start = tx_conf->tx_deferred_start; txq->tx_ring_dma = tz->iova; - txq->tx_ring = (struct i40e_tx_desc *)tz->addr; + txq->i40e_tx_ring = (struct i40e_tx_desc *)tz->addr;/* Allocate software ring */ txq->sw_ring =
@@ -2885,11 +2885,11 @@ i40e_reset_tx_queue(struct i40e_tx_queue *txq) txe = txq->sw_ring; size =sizeof (struct i40e_tx_desc ) * txq->nb_tx_desc;for (i =0 ; i < size; i++) - ((volatile char *)txq->tx_ring)[i] =0 ; + ((volatile char *)txq->i40e_tx_ring)[i] =0 ; prev = (uint16_t )(txq->nb_tx_desc -1 );for (i =0 ; i < txq->nb_tx_desc; i++) { -volatile struct i40e_tx_desc *txd = &txq->tx_ring[i]; +volatile struct i40e_tx_desc *txd = &txq->i40e_tx_ring[i]; txd->cmd_type_offset_bsz =rte_cpu_to_le_64 (I40E_TX_DESC_DTYPE_DESC_DONE );
@@ -2914,7 +2914,7 @@ int i40e_tx_queue_init(struct i40e_tx_queue *txq) {enum i40e_status_code err =I40E_SUCCESS ; -struct i40e_vsi *vsi = txq->vsi; +struct i40e_vsi *vsi = txq->i40e_vsi;struct i40e_hw *hw =I40E_VSI_TO_HW (vsi);uint16_t pf_q = txq->reg_idx;struct i40e_hmc_obj_txq tx_ctx;
@@ -3207,10 +3207,10 @@ i40e_fdir_setup_tx_resources(struct i40e_pf *pf) txq->nb_tx_desc =I40E_FDIR_NUM_TX_DESC ; txq->queue_id =I40E_FDIR_QUEUE_ID ; txq->reg_idx = pf->fdir.fdir_vsi->base_queue; - txq->vsi = pf->fdir.fdir_vsi; + txq->i40e_vsi = pf->fdir.fdir_vsi; txq->tx_ring_dma = tz->iova; - txq->tx_ring = (struct i40e_tx_desc *)tz->addr; + txq->i40e_tx_ring = (struct i40e_tx_desc *)tz->addr;/* * don't need to allocate software ring and reset for the fdir
drivers/net/i40e/i40e_rxtx.h +2 -2
@@ -130,7 +130,7 @@ struct i40e_rx_queue {struct i40e_tx_queue {uint16_t nb_tx_desc;/**< number of TX descriptors */ rte_iova_t tx_ring_dma;/**< TX ring DMA address */ -volatile struct i40e_tx_desc *tx_ring;/**< TX ring virtual address */ +volatile struct i40e_tx_desc *i40e_tx_ring;/**< TX ring virtual address */ struct ci_tx_entry *sw_ring;/**< virtual address of SW ring */ uint16_t tx_tail;/**< current value of tail register */ volatile uint8_t *qtx_tail;/**< register address of tail */
@@ -150,7 +150,7 @@ struct i40e_tx_queue {uint16_t port_id;/**< Device port identifier. */ uint16_t queue_id;/**< TX queue index. */ uint16_t reg_idx; -struct i40e_vsi *vsi;/**< the VSI this queue belongs to */ +struct i40e_vsi *i40e_vsi;/**< the VSI this queue belongs to */ uint16_t tx_next_dd;uint16_t tx_next_rs;bool q_set;/**< indicate if tx queue has been configured */
drivers/net/i40e/i40e_rxtx_vec_altivec.c +3 -3
@@ -568,7 +568,7 @@ i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,return 0 ; tx_id = txq->tx_tail; - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->i40e_tx_ring[tx_id]; txep = &txq->sw_ring[tx_id]; txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts);
@@ -588,7 +588,7 @@ i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, txq->tx_next_rs = (uint16_t )(txq->tx_rs_thresh -1 );/* avoid reach the end of ring */ - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->i40e_tx_ring[tx_id]; txep = &txq->sw_ring[tx_id]; }
@@ -598,7 +598,7 @@ i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = (uint16_t )(tx_id + nb_commit);if (tx_id > txq->tx_next_rs) { - txq->tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |= + txq->i40e_tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |=rte_cpu_to_le_64 (((uint64_t )I40E_TX_DESC_CMD_RS ) <<I40E_TXD_QW1_CMD_SHIFT ); txq->tx_next_rs =
drivers/net/i40e/i40e_rxtx_vec_avx2.c +3 -3
@@ -758,7 +758,7 @@ i40e_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts,return 0 ; tx_id = txq->tx_tail; - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->i40e_tx_ring[tx_id]; txep = &txq->sw_ring[tx_id]; txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts);
@@ -779,7 +779,7 @@ i40e_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, txq->tx_next_rs = (uint16_t )(txq->tx_rs_thresh -1 );/* avoid reach the end of ring */ - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->i40e_tx_ring[tx_id]; txep = &txq->sw_ring[tx_id]; }
@@ -789,7 +789,7 @@ i40e_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = (uint16_t )(tx_id + nb_commit);if (tx_id > txq->tx_next_rs) { - txq->tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |= + txq->i40e_tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |=rte_cpu_to_le_64 (((uint64_t )I40E_TX_DESC_CMD_RS ) <<I40E_TXD_QW1_CMD_SHIFT ); txq->tx_next_rs =
drivers/net/i40e/i40e_rxtx_vec_avx512.c +4 -4
@@ -764,7 +764,7 @@ i40e_tx_free_bufs_avx512(struct i40e_tx_queue *txq)struct rte_mbuf *m, *free[RTE_I40E_TX_MAX_FREE_BUF_SZ ];/* check DD bits on threshold descriptor */ -if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz & +if ((txq->i40e_tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &rte_cpu_to_le_64 (I40E_TXD_QW1_DTYPE_MASK )) !=rte_cpu_to_le_64 (I40E_TX_DESC_DTYPE_DESC_DONE ))return 0 ;
@@ -948,7 +948,7 @@ i40e_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts,return 0 ; tx_id = txq->tx_tail; - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->i40e_tx_ring[tx_id]; txep = (void *)txq->sw_ring; txep += tx_id;
@@ -970,7 +970,7 @@ i40e_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, txq->tx_next_rs = (uint16_t )(txq->tx_rs_thresh -1 );/* avoid reach the end of ring */ - txdp = txq->tx_ring; + txdp = txq->i40e_tx_ring; txep = (void *)txq->sw_ring; }
@@ -980,7 +980,7 @@ i40e_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = (uint16_t )(tx_id + nb_commit);if (tx_id > txq->tx_next_rs) { - txq->tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |= + txq->i40e_tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |=rte_cpu_to_le_64 (((uint64_t )I40E_TX_DESC_CMD_RS ) <<I40E_TXD_QW1_CMD_SHIFT ); txq->tx_next_rs =
drivers/net/i40e/i40e_rxtx_vec_common.h +1 -1
@@ -26,7 +26,7 @@ i40e_tx_free_bufs(struct i40e_tx_queue *txq)struct rte_mbuf *m, *free[RTE_I40E_TX_MAX_FREE_BUF_SZ ];/* check DD bits on threshold descriptor */ -if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz & +if ((txq->i40e_tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &rte_cpu_to_le_64 (I40E_TXD_QW1_DTYPE_MASK )) !=rte_cpu_to_le_64 (I40E_TX_DESC_DTYPE_DESC_DONE ))return 0 ;
drivers/net/i40e/i40e_rxtx_vec_neon.c +3 -3
@@ -695,7 +695,7 @@ i40e_xmit_fixed_burst_vec(void *__rte_restrict tx_queue,return 0 ; tx_id = txq->tx_tail; - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->i40e_tx_ring[tx_id]; txep = &txq->sw_ring[tx_id]; txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts);
@@ -715,7 +715,7 @@ i40e_xmit_fixed_burst_vec(void *__rte_restrict tx_queue, txq->tx_next_rs = (uint16_t )(txq->tx_rs_thresh -1 );/* avoid reach the end of ring */ - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->i40e_tx_ring[tx_id]; txep = &txq->sw_ring[tx_id]; }
@@ -725,7 +725,7 @@ i40e_xmit_fixed_burst_vec(void *__rte_restrict tx_queue, tx_id = (uint16_t )(tx_id + nb_commit);if (tx_id > txq->tx_next_rs) { - txq->tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |= + txq->i40e_tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |=rte_cpu_to_le_64 (((uint64_t )I40E_TX_DESC_CMD_RS ) <<I40E_TXD_QW1_CMD_SHIFT ); txq->tx_next_rs =
drivers/net/i40e/i40e_rxtx_vec_sse.c +3 -3
@@ -714,7 +714,7 @@ i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,return 0 ; tx_id = txq->tx_tail; - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->i40e_tx_ring[tx_id]; txep = &txq->sw_ring[tx_id]; txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts);
@@ -734,7 +734,7 @@ i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, txq->tx_next_rs = (uint16_t )(txq->tx_rs_thresh -1 );/* avoid reach the end of ring */ - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->i40e_tx_ring[tx_id]; txep = &txq->sw_ring[tx_id]; }
@@ -744,7 +744,7 @@ i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = (uint16_t )(tx_id + nb_commit);if (tx_id > txq->tx_next_rs) { - txq->tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |= + txq->i40e_tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |=rte_cpu_to_le_64 (((uint64_t )I40E_TX_DESC_CMD_RS ) <<I40E_TXD_QW1_CMD_SHIFT ); txq->tx_next_rs =
drivers/net/iavf/iavf_rxtx.c +12 -12
@@ -296,11 +296,11 @@ reset_tx_queue(struct iavf_tx_queue *txq) txe = txq->sw_ring; size =sizeof (struct iavf_tx_desc ) * txq->nb_tx_desc;for (i =0 ; i < size; i++) - ((volatile char *)txq->tx_ring)[i] =0 ; + ((volatile char *)txq->iavf_tx_ring)[i] =0 ; prev = (uint16_t )(txq->nb_tx_desc -1 );for (i =0 ; i < txq->nb_tx_desc; i++) { - txq->tx_ring[i].cmd_type_offset_bsz = + txq->iavf_tx_ring[i].cmd_type_offset_bsz =rte_cpu_to_le_64 (IAVF_TX_DESC_DTYPE_DESC_DONE ); txe[i].mbuf =NULL ; txe[i].last_id = i;
@@ -851,7 +851,7 @@ iavf_dev_tx_queue_setup(struct rte_eth_dev *dev, txq->port_id = dev->data->port_id; txq->offloads = offloads; txq->tx_deferred_start = tx_conf->tx_deferred_start; - txq->vsi = vsi; + txq->iavf_vsi = vsi;if (iavf_ipsec_crypto_supported (adapter)) txq->ipsec_crypto_pkt_md_offset =
@@ -872,7 +872,7 @@ iavf_dev_tx_queue_setup(struct rte_eth_dev *dev,/* Allocate TX hardware ring descriptors. */ ring_size =sizeof (struct iavf_tx_desc ) *IAVF_MAX_RING_DESC ; ring_size =RTE_ALIGN (ring_size,IAVF_DMA_MEM_ALIGN ); - mz =rte_eth_dma_zone_reserve (dev,"tx_ring" , queue_idx, + mz =rte_eth_dma_zone_reserve (dev,"iavf_tx_ring" , queue_idx, ring_size,IAVF_RING_BASE_ALIGN , socket_id);if (!mz) {
@@ -882,7 +882,7 @@ iavf_dev_tx_queue_setup(struct rte_eth_dev *dev,return -ENOMEM ; } txq->tx_ring_dma = mz->iova; - txq->tx_ring = (struct iavf_tx_desc *)mz->addr; + txq->iavf_tx_ring = (struct iavf_tx_desc *)mz->addr; txq->mz = mz;reset_tx_queue (txq);
@@ -2385,7 +2385,7 @@ iavf_xmit_cleanup(struct iavf_tx_queue *txq)uint16_t desc_to_clean_to;uint16_t nb_tx_to_clean; -volatile struct iavf_tx_desc *txd = txq->tx_ring; +volatile struct iavf_tx_desc *txd = txq->iavf_tx_ring; desc_to_clean_to = (uint16_t )(last_desc_cleaned + txq->tx_rs_thresh);if (desc_to_clean_to >= nb_tx_desc)
@@ -2796,7 +2796,7 @@ uint16_t iavf_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) {struct iavf_tx_queue *txq = tx_queue; -volatile struct iavf_tx_desc *txr = txq->tx_ring; +volatile struct iavf_tx_desc *txr = txq->iavf_tx_ring;struct ci_tx_entry *txe_ring = txq->sw_ring;struct ci_tx_entry *txe, *txn;struct rte_mbuf *mb, *mb_seg;
@@ -3803,10 +3803,10 @@ iavf_xmit_pkts_no_poll(void *tx_queue, struct rte_mbuf **tx_pkts,struct iavf_tx_queue *txq = tx_queue;enum iavf_tx_burst_type tx_burst_type; -if (!txq->vsi || txq->vsi->adapter->no_poll) +if (!txq->iavf_vsi || txq->iavf_vsi->adapter->no_poll)return 0 ; - tx_burst_type = txq->vsi->adapter->tx_burst_type; + tx_burst_type = txq->iavf_vsi->adapter->tx_burst_type;return iavf_tx_pkt_burst_ops[tx_burst_type](tx_queue, tx_pkts, nb_pkts);
@@ -3824,9 +3824,9 @@ iavf_xmit_pkts_check(void *tx_queue, struct rte_mbuf **tx_pkts,const char *reason =NULL ;bool pkt_error = false;struct iavf_tx_queue *txq = tx_queue; -struct iavf_adapter *adapter = txq->vsi->adapter; +struct iavf_adapter *adapter = txq->iavf_vsi->adapter;enum iavf_tx_burst_type tx_burst_type = - txq->vsi->adapter->tx_burst_type; + txq->iavf_vsi->adapter->tx_burst_type;for (idx =0 ; idx < nb_pkts; idx++) { mb = tx_pkts[idx];
@@ -4440,7 +4440,7 @@ iavf_dev_tx_desc_status(void *tx_queue, uint16_t offset) desc -= txq->nb_tx_desc; } - status = &txq->tx_ring[desc].cmd_type_offset_bsz; + status = &txq->iavf_tx_ring[desc].cmd_type_offset_bsz; mask =rte_le_to_cpu_64 (IAVF_TXD_QW1_DTYPE_MASK ); expect =rte_cpu_to_le_64 (IAVF_TX_DESC_DTYPE_DESC_DONE <<IAVF_TXD_QW1_DTYPE_SHIFT );
drivers/net/iavf/iavf_rxtx.h +2 -2
@@ -276,7 +276,7 @@ struct iavf_rx_queue {/* Structure associated with each TX queue. */ struct iavf_tx_queue {const struct rte_memzone *mz;/* memzone for Tx ring */ -volatile struct iavf_tx_desc *tx_ring;/* Tx ring virtual address */ +volatile struct iavf_tx_desc *iavf_tx_ring;/* Tx ring virtual address */ rte_iova_t tx_ring_dma;/* Tx ring DMA address */ struct ci_tx_entry *sw_ring;/* address array of SW ring */ uint16_t nb_tx_desc;/* ring length */
@@ -289,7 +289,7 @@ struct iavf_tx_queue {uint16_t tx_free_thresh;uint16_t tx_rs_thresh;uint8_t rel_mbufs_type; -struct iavf_vsi *vsi;/**< the VSI this queue belongs to */ +struct iavf_vsi *iavf_vsi;/**< the VSI this queue belongs to */ uint16_t port_id;uint16_t queue_id;
drivers/net/iavf/iavf_rxtx_vec_avx2.c +3 -3
@@ -1751,7 +1751,7 @@ iavf_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, nb_commit = nb_pkts; tx_id = txq->tx_tail; - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->iavf_tx_ring[tx_id]; txep = &txq->sw_ring[tx_id]; txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts);
@@ -1772,7 +1772,7 @@ iavf_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, txq->tx_next_rs = (uint16_t )(txq->tx_rs_thresh -1 );/* avoid reach the end of ring */ - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->iavf_tx_ring[tx_id]; txep = &txq->sw_ring[tx_id]; }
@@ -1782,7 +1782,7 @@ iavf_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = (uint16_t )(tx_id + nb_commit);if (tx_id > txq->tx_next_rs) { - txq->tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |= + txq->iavf_tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |=rte_cpu_to_le_64 (((uint64_t )IAVF_TX_DESC_CMD_RS ) <<IAVF_TXD_QW1_CMD_SHIFT ); txq->tx_next_rs =
drivers/net/iavf/iavf_rxtx_vec_avx512.c +7 -7
@@ -1854,7 +1854,7 @@ iavf_tx_free_bufs_avx512(struct iavf_tx_queue *txq)struct rte_mbuf *m, *free[IAVF_VPMD_TX_MAX_FREE_BUF ];/* check DD bits on threshold descriptor */ -if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz & +if ((txq->iavf_tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &rte_cpu_to_le_64 (IAVF_TXD_QW1_DTYPE_MASK )) !=rte_cpu_to_le_64 (IAVF_TX_DESC_DTYPE_DESC_DONE ))return 0 ;
@@ -2328,7 +2328,7 @@ iavf_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, nb_commit = nb_pkts; tx_id = txq->tx_tail; - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->iavf_tx_ring[tx_id]; txep = (void *)txq->sw_ring; txep += tx_id;
@@ -2350,7 +2350,7 @@ iavf_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, txq->tx_next_rs = (uint16_t )(txq->tx_rs_thresh -1 );/* avoid reach the end of ring */ - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->iavf_tx_ring[tx_id]; txep = (void *)txq->sw_ring; txep += tx_id; }
@@ -2361,7 +2361,7 @@ iavf_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = (uint16_t )(tx_id + nb_commit);if (tx_id > txq->tx_next_rs) { - txq->tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |= + txq->iavf_tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |=rte_cpu_to_le_64 (((uint64_t )IAVF_TX_DESC_CMD_RS ) <<IAVF_TXD_QW1_CMD_SHIFT ); txq->tx_next_rs =
@@ -2397,7 +2397,7 @@ iavf_xmit_fixed_burst_vec_avx512_ctx(void *tx_queue, struct rte_mbuf **tx_pkts, nb_pkts = nb_commit >>1 ; tx_id = txq->tx_tail; - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->iavf_tx_ring[tx_id]; txep = (void *)txq->sw_ring; txep += (tx_id >>1 );
@@ -2418,7 +2418,7 @@ iavf_xmit_fixed_burst_vec_avx512_ctx(void *tx_queue, struct rte_mbuf **tx_pkts, txq->tx_next_rs = (uint16_t )(txq->tx_rs_thresh -1 ); tx_id =0 ;/* avoid reach the end of ring */ - txdp = txq->tx_ring; + txdp = txq->iavf_tx_ring; txep = (void *)txq->sw_ring; }
@@ -2429,7 +2429,7 @@ iavf_xmit_fixed_burst_vec_avx512_ctx(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = (uint16_t )(tx_id + nb_commit);if (tx_id > txq->tx_next_rs) { - txq->tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |= + txq->iavf_tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |=rte_cpu_to_le_64 (((uint64_t )IAVF_TX_DESC_CMD_RS ) <<IAVF_TXD_QW1_CMD_SHIFT ); txq->tx_next_rs =
drivers/net/iavf/iavf_rxtx_vec_common.h +1 -1
@@ -26,7 +26,7 @@ iavf_tx_free_bufs(struct iavf_tx_queue *txq)struct rte_mbuf *m, *free[IAVF_VPMD_TX_MAX_FREE_BUF ];/* check DD bits on threshold descriptor */ -if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz & +if ((txq->iavf_tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &rte_cpu_to_le_64 (IAVF_TXD_QW1_DTYPE_MASK )) !=rte_cpu_to_le_64 (IAVF_TX_DESC_DTYPE_DESC_DONE ))return 0 ;
drivers/net/iavf/iavf_rxtx_vec_sse.c +3 -3
@@ -1383,7 +1383,7 @@ iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, nb_commit = nb_pkts; tx_id = txq->tx_tail; - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->iavf_tx_ring[tx_id]; txep = &txq->sw_ring[tx_id]; txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts);
@@ -1403,7 +1403,7 @@ iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, txq->tx_next_rs = (uint16_t )(txq->tx_rs_thresh -1 );/* avoid reach the end of ring */ - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->iavf_tx_ring[tx_id]; txep = &txq->sw_ring[tx_id]; }
@@ -1413,7 +1413,7 @@ iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = (uint16_t )(tx_id + nb_commit);if (tx_id > txq->tx_next_rs) { - txq->tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |= + txq->iavf_tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |=rte_cpu_to_le_64 (((uint64_t )IAVF_TX_DESC_CMD_RS ) <<IAVF_TXD_QW1_CMD_SHIFT ); txq->tx_next_rs =
drivers/net/ice/ice_dcf_ethdev.c +2 -2
@@ -401,11 +401,11 @@ reset_tx_queue(struct ice_tx_queue *txq) txe = txq->sw_ring; size =sizeof (struct ice_tx_desc ) * txq->nb_tx_desc;for (i =0 ; i < size; i++) - ((volatile char *)txq->tx_ring)[i] =0 ; + ((volatile char *)txq->ice_tx_ring)[i] =0 ; prev = (uint16_t )(txq->nb_tx_desc -1 );for (i =0 ; i < txq->nb_tx_desc; i++) { - txq->tx_ring[i].cmd_type_offset_bsz = + txq->ice_tx_ring[i].cmd_type_offset_bsz =rte_cpu_to_le_64 (IAVF_TX_DESC_DTYPE_DESC_DONE ); txe[i].mbuf =NULL ; txe[i].last_id = i;
drivers/net/ice/ice_rxtx.c +24 -24
179 lines not on this page · raw message
drivers/net/ice/ice_rxtx.h +2 -2
@@ -148,7 +148,7 @@ struct ice_rx_queue {struct ice_tx_queue {uint16_t nb_tx_desc;/* number of TX descriptors */ rte_iova_t tx_ring_dma;/* TX ring DMA address */ -volatile struct ice_tx_desc *tx_ring;/* TX ring virtual address */ +volatile struct ice_tx_desc *ice_tx_ring;/* TX ring virtual address */ struct ci_tx_entry *sw_ring;/* virtual address of SW ring */ uint16_t tx_tail;/* current value of tail register */ volatile uint8_t *qtx_tail;/* register address of tail */
@@ -171,7 +171,7 @@ struct ice_tx_queue {uint32_t q_teid;/* TX schedule node id. */ uint16_t reg_idx;uint64_t offloads; -struct ice_vsi *vsi;/* the VSI this queue belongs to */ +struct ice_vsi *ice_vsi;/* the VSI this queue belongs to */ uint16_t tx_next_dd;uint16_t tx_next_rs;uint64_t mbuf_errors;
drivers/net/ice/ice_rxtx_vec_avx2.c +3 -3
@@ -874,7 +874,7 @@ ice_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts,return 0 ; tx_id = txq->tx_tail; - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->ice_tx_ring[tx_id]; txep = &txq->sw_ring[tx_id]; txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts);
@@ -895,7 +895,7 @@ ice_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, txq->tx_next_rs = (uint16_t )(txq->tx_rs_thresh -1 );/* avoid reach the end of ring */ - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->ice_tx_ring[tx_id]; txep = &txq->sw_ring[tx_id]; }
@@ -905,7 +905,7 @@ ice_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = (uint16_t )(tx_id + nb_commit);if (tx_id > txq->tx_next_rs) { - txq->tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |= + txq->ice_tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |=rte_cpu_to_le_64 (((uint64_t )ICE_TX_DESC_CMD_RS ) <<ICE_TXD_QW1_CMD_S ); txq->tx_next_rs =
drivers/net/ice/ice_rxtx_vec_avx512.c +4 -4
@@ -869,7 +869,7 @@ ice_tx_free_bufs_avx512(struct ice_tx_queue *txq)struct rte_mbuf *m, *free[ICE_TX_MAX_FREE_BUF_SZ ];/* check DD bits on threshold descriptor */ -if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz & +if ((txq->ice_tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &rte_cpu_to_le_64 (ICE_TXD_QW1_DTYPE_M )) !=rte_cpu_to_le_64 (ICE_TX_DESC_DTYPE_DESC_DONE ))return 0 ;
@@ -1071,7 +1071,7 @@ ice_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts,return 0 ; tx_id = txq->tx_tail; - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->ice_tx_ring[tx_id]; txep = (void *)txq->sw_ring; txep += tx_id;
@@ -1093,7 +1093,7 @@ ice_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, txq->tx_next_rs = (uint16_t )(txq->tx_rs_thresh -1 );/* avoid reach the end of ring */ - txdp = txq->tx_ring; + txdp = txq->ice_tx_ring; txep = (void *)txq->sw_ring; }
@@ -1103,7 +1103,7 @@ ice_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = (uint16_t )(tx_id + nb_commit);if (tx_id > txq->tx_next_rs) { - txq->tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |= + txq->ice_tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |=rte_cpu_to_le_64 (((uint64_t )ICE_TX_DESC_CMD_RS ) <<ICE_TXD_QW1_CMD_S ); txq->tx_next_rs =
drivers/net/ice/ice_rxtx_vec_common.h +2 -2
@@ -22,7 +22,7 @@ ice_tx_free_bufs_vec(struct ice_tx_queue *txq)struct rte_mbuf *m, *free[ICE_TX_MAX_FREE_BUF_SZ ];/* check DD bits on threshold descriptor */ -if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz & +if ((txq->ice_tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &rte_cpu_to_le_64 (ICE_TXD_QW1_DTYPE_M )) !=rte_cpu_to_le_64 (ICE_TX_DESC_DTYPE_DESC_DONE ))return 0 ;
@@ -121,7 +121,7 @@ _ice_tx_queue_release_mbufs_vec(struct ice_tx_queue *txq) i = txq->tx_next_dd - txq->tx_rs_thresh +1 ;#ifdef __AVX512VL__ -struct rte_eth_dev *dev = &rte_eth_devices[txq->vsi->adapter->pf.dev_data->port_id]; +struct rte_eth_dev *dev = &rte_eth_devices[txq->ice_vsi->adapter->pf.dev_data->port_id];if (dev->tx_pkt_burst == ice_xmit_pkts_vec_avx512 || dev->tx_pkt_burst == ice_xmit_pkts_vec_avx512_offload) {
drivers/net/ice/ice_rxtx_vec_sse.c +3 -3
@@ -717,7 +717,7 @@ ice_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,return 0 ; tx_id = txq->tx_tail; - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->ice_tx_ring[tx_id]; txep = &txq->sw_ring[tx_id]; txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts);
@@ -737,7 +737,7 @@ ice_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, txq->tx_next_rs = (uint16_t )(txq->tx_rs_thresh -1 );/* avoid reach the end of ring */ - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->ice_tx_ring[tx_id]; txep = &txq->sw_ring[tx_id]; }
@@ -747,7 +747,7 @@ ice_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = (uint16_t )(tx_id + nb_commit);if (tx_id > txq->tx_next_rs) { - txq->tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |= + txq->ice_tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |=rte_cpu_to_le_64 (((uint64_t )ICE_TX_DESC_CMD_RS ) <<ICE_TXD_QW1_CMD_S ); txq->tx_next_rs =
drivers/net/ixgbe/ixgbe_recycle_mbufs_vec_common.c +1 -1
@@ -72,7 +72,7 @@ ixgbe_recycle_tx_mbufs_reuse_vec(void *tx_queue,return 0 ;/* check DD bits on threshold descriptor */ - status = txq->tx_ring[txq->tx_next_dd].wb.status; + status = txq->ixgbe_tx_ring[txq->tx_next_dd].wb.status;if (!(status &IXGBE_ADVTXD_STAT_DD ))return 0 ;
drivers/net/ixgbe/ixgbe_rxtx.c +11 -11
@@ -106,7 +106,7 @@ ixgbe_tx_free_bufs(struct ixgbe_tx_queue *txq)struct rte_mbuf *m, *free[RTE_IXGBE_TX_MAX_FREE_BUF_SZ ];/* check DD bit on threshold descriptor */ - status = txq->tx_ring[txq->tx_next_dd].wb.status; + status = txq->ixgbe_tx_ring[txq->tx_next_dd].wb.status;if (!(status &rte_cpu_to_le_32 (IXGBE_ADVTXD_STAT_DD )))return 0 ;
@@ -198,7 +198,7 @@ static inline void ixgbe_tx_fill_hw_ring(struct ixgbe_tx_queue *txq, struct rte_mbuf **pkts, uint16_t nb_pkts) { - volatile union ixgbe_adv_tx_desc *txdp = &(txq->tx_ring[txq->tx_tail]); + volatile union ixgbe_adv_tx_desc *txdp = &txq->ixgbe_tx_ring[txq->tx_tail];struct ci_tx_entry *txep = &txq->sw_ring[txq->tx_tail];const int N_PER_LOOP =4 ;const int N_PER_LOOP_MASK =N_PER_LOOP -1 ;
@@ -232,7 +232,7 @@ tx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) {struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue; -volatile union ixgbe_adv_tx_desc *tx_r = txq->tx_ring; +volatile union ixgbe_adv_tx_desc *tx_r = txq->ixgbe_tx_ring;uint16_t n =0 ;/*
@@ -564,7 +564,7 @@ static inline int ixgbe_xmit_cleanup(struct ixgbe_tx_queue *txq) {struct ci_tx_entry *sw_ring = txq->sw_ring; -volatile union ixgbe_adv_tx_desc *txr = txq->tx_ring; +volatile union ixgbe_adv_tx_desc *txr = txq->ixgbe_tx_ring;uint16_t last_desc_cleaned = txq->last_desc_cleaned;uint16_t nb_tx_desc = txq->nb_tx_desc;uint16_t desc_to_clean_to;
@@ -652,7 +652,7 @@ ixgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, tx_offload.data[1 ] =0 ; txq = tx_queue; sw_ring = txq->sw_ring; - txr = txq->tx_ring; + txr = txq->ixgbe_tx_ring; tx_id = txq->tx_tail; txe = &sw_ring[tx_id]; txp =NULL ;
@@ -2495,13 +2495,13 @@ ixgbe_reset_tx_queue(struct ixgbe_tx_queue *txq)/* Zero out HW ring memory */ for (i =0 ; i < txq->nb_tx_desc; i++) { - txq->tx_ring[i] = zeroed_desc; + txq->ixgbe_tx_ring[i] = zeroed_desc; }/* Initialize SW ring entries */ prev = (uint16_t ) (txq->nb_tx_desc -1 );for (i =0 ; i < txq->nb_tx_desc; i++) { -volatile union ixgbe_adv_tx_desc *txd = &txq->tx_ring[i]; +volatile union ixgbe_adv_tx_desc *txd = &txq->ixgbe_tx_ring[i]; txd->wb.status =rte_cpu_to_le_32 (IXGBE_TXD_STAT_DD ); txe[i].mbuf =NULL ;
@@ -2751,7 +2751,7 @@ ixgbe_dev_tx_queue_setup(struct rte_eth_dev *dev,* handle the maximum ring size is allocated in order to allow for * resizing in later calls to the queue setup function. */ - tz =rte_eth_dma_zone_reserve (dev,"tx_ring" , queue_idx, + tz =rte_eth_dma_zone_reserve (dev,"ixgbe_tx_ring" , queue_idx,sizeof (union ixgbe_adv_tx_desc ) *IXGBE_MAX_RING_DESC ,IXGBE_ALIGN , socket_id);if (tz ==NULL ) {
@@ -2791,7 +2791,7 @@ ixgbe_dev_tx_queue_setup(struct rte_eth_dev *dev, txq->qtx_tail =IXGBE_PCI_REG_ADDR (hw,IXGBE_TDT (txq->reg_idx)); txq->tx_ring_dma = tz->iova; - txq->tx_ring = (union ixgbe_adv_tx_desc *) tz->addr; + txq->ixgbe_tx_ring = (union ixgbe_adv_tx_desc *)tz->addr;/* Allocate software ring */ txq->sw_ring = rte_zmalloc_socket("txq->sw_ring",
@@ -2802,7 +2802,7 @@ ixgbe_dev_tx_queue_setup(struct rte_eth_dev *dev,return -ENOMEM ; }PMD_INIT_LOG (DEBUG ,"sw_ring=%p hw_ring=%p dma_addr=0x%" PRIx64, - txq->sw_ring, txq->tx_ring, txq->tx_ring_dma); + txq->sw_ring, txq->ixgbe_tx_ring, txq->tx_ring_dma);/* set up vector or scalar TX function as appropriate */ ixgbe_set_tx_function (dev, txq);
@@ -3328,7 +3328,7 @@ ixgbe_dev_tx_descriptor_status(void *tx_queue, uint16_t offset) desc -= txq->nb_tx_desc; } - status = &txq->tx_ring[desc].wb.status; + status = &txq->ixgbe_tx_ring[desc].wb.status;if (*status &rte_cpu_to_le_32 (IXGBE_ADVTXD_STAT_DD ))return RTE_ETH_TX_DESC_DONE ;
drivers/net/ixgbe/ixgbe_rxtx.h +1 -1
@@ -185,7 +185,7 @@ struct ixgbe_advctx_info {*/ struct ixgbe_tx_queue {/** TX ring virtual address. */ -volatile union ixgbe_adv_tx_desc *tx_ring; +volatile union ixgbe_adv_tx_desc *ixgbe_tx_ring;rte_iova_t tx_ring_dma;/**< TX ring DMA address. */ union {struct ci_tx_entry *sw_ring;/**< address of SW ring for scalar PMD. */
drivers/net/ixgbe/ixgbe_rxtx_vec_common.h +3 -3
@@ -22,7 +22,7 @@ ixgbe_tx_free_bufs(struct ixgbe_tx_queue *txq)struct rte_mbuf *m, *free[RTE_IXGBE_TX_MAX_FREE_BUF_SZ ];/* check DD bit on threshold descriptor */ - status = txq->tx_ring[txq->tx_next_dd].wb.status; + status = txq->ixgbe_tx_ring[txq->tx_next_dd].wb.status;if (!(status &IXGBE_ADVTXD_STAT_DD ))return 0 ;
@@ -154,11 +154,11 @@ _ixgbe_reset_tx_queue_vec(struct ixgbe_tx_queue *txq)/* Zero out HW ring memory */ for (i =0 ; i < txq->nb_tx_desc; i++) - txq->tx_ring[i] = zeroed_desc; + txq->ixgbe_tx_ring[i] = zeroed_desc;/* Initialize SW ring entries */ for (i =0 ; i < txq->nb_tx_desc; i++) { -volatile union ixgbe_adv_tx_desc *txd = &txq->tx_ring[i]; +volatile union ixgbe_adv_tx_desc *txd = &txq->ixgbe_tx_ring[i]; txd->wb.status =IXGBE_TXD_STAT_DD ; txe[i].mbuf =NULL ;
drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c +3 -3
@@ -590,7 +590,7 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,return 0 ; tx_id = txq->tx_tail; - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->ixgbe_tx_ring[tx_id]; txep = &txq->sw_ring_v[tx_id]; txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts);
@@ -610,7 +610,7 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, txq->tx_next_rs = (uint16_t )(txq->tx_rs_thresh -1 );/* avoid reach the end of ring */ - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->ixgbe_tx_ring[tx_id]; txep = &txq->sw_ring_v[tx_id]; }
@@ -620,7 +620,7 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = (uint16_t )(tx_id + nb_commit);if (tx_id > txq->tx_next_rs) { - txq->tx_ring[txq->tx_next_rs].read.cmd_type_len |= + txq->ixgbe_tx_ring[txq->tx_next_rs].read.cmd_type_len |=rte_cpu_to_le_32 (IXGBE_ADVTXD_DCMD_RS ); txq->tx_next_rs = (uint16_t )(txq->tx_next_rs + txq->tx_rs_thresh);
drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c +3 -3
@@ -712,7 +712,7 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,return 0 ; tx_id = txq->tx_tail; - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->ixgbe_tx_ring[tx_id]; txep = &txq->sw_ring_v[tx_id]; txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts);
@@ -733,7 +733,7 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, txq->tx_next_rs = (uint16_t )(txq->tx_rs_thresh -1 );/* avoid reach the end of ring */ - txdp = &(txq->tx_ring[tx_id]); + txdp = &txq->ixgbe_tx_ring[tx_id]; txep = &txq->sw_ring_v[tx_id]; }
@@ -743,7 +743,7 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = (uint16_t )(tx_id + nb_commit);if (tx_id > txq->tx_next_rs) { - txq->tx_ring[txq->tx_next_rs].read.cmd_type_len |= + txq->ixgbe_tx_ring[txq->tx_next_rs].read.cmd_type_len |=rte_cpu_to_le_32 (IXGBE_ADVTXD_DCMD_RS ); txq->tx_next_rs = (uint16_t )(txq->tx_next_rs + txq->tx_rs_thresh);
Signature · 1 line
2.43.0
PATCH v2 6/22 net/_common_intel: merge ice and i40e Tx queue struct
In reply to Bruce RichardsonThe queue structures of i40e and ice drivers are virtually identical, so merge them into a common struct. This should allow easier function merging in future using that common struct.
--- drivers/net/_common_intel/tx.h | 55 +++++++++++++++++ drivers/net/i40e/i40e_ethdev.c | 4 +- drivers/net/i40e/i40e_ethdev.h | 4 +- drivers/net/i40e/i40e_fdir.c | 4 +- .../net/i40e/i40e_recycle_mbufs_vec_common.c | 2 +- drivers/net/i40e/i40e_rxtx.c | 58 +++++++++--------- drivers/net/i40e/i40e_rxtx.h | 50 ++-------------- drivers/net/i40e/i40e_rxtx_vec_altivec.c | 4 +- drivers/net/i40e/i40e_rxtx_vec_avx2.c | 4 +- drivers/net/i40e/i40e_rxtx_vec_avx512.c | 6 +- drivers/net/i40e/i40e_rxtx_vec_common.h | 2 +- drivers/net/i40e/i40e_rxtx_vec_neon.c | 4 +- drivers/net/i40e/i40e_rxtx_vec_sse.c | 4 +- drivers/net/ice/ice_dcf.c | 4 +- drivers/net/ice/ice_dcf_ethdev.c | 10 ++-- drivers/net/ice/ice_diagnose.c | 2 +- drivers/net/ice/ice_ethdev.c | 2 +- drivers/net/ice/ice_ethdev.h | 4 +- drivers/net/ice/ice_rxtx.c | 60 +++++++++---------- drivers/net/ice/ice_rxtx.h | 41 +------------ drivers/net/ice/ice_rxtx_vec_avx2.c | 4 +- drivers/net/ice/ice_rxtx_vec_avx512.c | 8 +-- drivers/net/ice/ice_rxtx_vec_common.h | 8 +-- drivers/net/ice/ice_rxtx_vec_sse.c | 6 +- 24 files changed, 165 insertions(+), 185 deletions(-)
drivers/net/_common_intel/tx.h +55 -0
@@ -8,6 +8,9 @@#include <stdint.h> #include <rte_mbuf.h> +/* forward declaration of the common intel (ci) queue structure */ +struct ci_tx_queue ; +/** * Structure associated with each descriptor of the TX ring of a TX queue. */
@@ -24,6 +27,58 @@ struct ci_tx_entry_vec {struct rte_mbuf *mbuf;/* mbuf associated with TX desc, if any. */ }; +typedef void (*ice_tx_release_mbufs_t )(struct ci_tx_queue *txq); + +struct ci_tx_queue { +union {/* TX ring virtual address */ +volatile struct ice_tx_desc *ice_tx_ring; +volatile struct i40e_tx_desc *i40e_tx_ring; + }; +volatile uint8_t *qtx_tail;/* register address of tail */ +struct ci_tx_entry *sw_ring;/* virtual address of SW ring */ +rte_iova_t tx_ring_dma;/* TX ring DMA address */ +uint16_t nb_tx_desc;/* number of TX descriptors */ +uint16_t tx_tail;/* current value of tail register */ +uint16_t nb_tx_used;/* number of TX desc used since RS bit set */ +/* index to last TX descriptor to have been cleaned */ +uint16_t last_desc_cleaned; +/* Total number of TX descriptors ready to be allocated. */ +uint16_t nb_tx_free; +/* Start freeing TX buffers if there are less free descriptors than +* this value. +*/ +uint16_t tx_free_thresh; +/* Number of TX descriptors to use before RS bit is set. */ +uint16_t tx_rs_thresh; +uint8_t pthresh;/**< Prefetch threshold register. */ +uint8_t hthresh;/**< Host threshold register. */ +uint8_t wthresh;/**< Write-back threshold reg. */ +uint16_t port_id;/* Device port identifier. */ +uint16_t queue_id;/* TX queue index. */ +uint16_t reg_idx; +uint64_t offloads; +uint16_t tx_next_dd; +uint16_t tx_next_rs; +uint64_t mbuf_errors; +bool tx_deferred_start;/* don't start this queue in dev start */ +bool q_set;/* indicate if tx queue has been configured */ +union {/* the VSI this queue belongs to */ +struct ice_vsi *ice_vsi; +struct i40e_vsi *i40e_vsi; + }; +const struct rte_memzone *mz; + +union { +struct {/* ICE driver specific values */ +ice_tx_release_mbufs_t tx_rel_mbufs; +uint32_t q_teid;/* TX schedule node id. */ + }; +struct {/* I40E driver specific values */ +uint8_t dcb_tc; + }; + }; +}; + static __rte_always_inline voidci_tx_backlog_entry (struct ci_tx_entry *txep,struct rte_mbuf **tx_pkts,uint16_t nb_pkts) {
drivers/net/i40e/i40e_ethdev.c +2 -2
@@ -3685,7 +3685,7 @@ i40e_dev_update_mbuf_stats(struct rte_eth_dev *ethdev, struct i40e_mbuf_stats *mbuf_stats) {uint16_t idx; -struct i40e_tx_queue *txq; +struct ci_tx_queue *txq;for (idx =0 ; idx < ethdev->data->nb_tx_queues; idx++) { txq = ethdev->data->tx_queues[idx];
@@ -6585,7 +6585,7 @@ i40e_dev_tx_init(struct i40e_pf *pf)struct rte_eth_dev_data *data = pf->dev_data;uint16_t i;uint32_t ret =I40E_SUCCESS ; -struct i40e_tx_queue *txq; +struct ci_tx_queue *txq;for (i =0 ; i < data->nb_tx_queues; i++) { txq = data->tx_queues[i];
drivers/net/i40e/i40e_ethdev.h +2 -2
@@ -334,7 +334,7 @@ struct i40e_vsi_list { };struct i40e_rx_queue ; -struct i40e_tx_queue ; +struct ci_tx_queue ;/* Bandwidth limit information */ struct i40e_bw_info {
@@ -738,7 +738,7 @@ TAILQ_HEAD(i40e_fdir_filter_list, i40e_fdir_filter);struct i40e_fdir_info {struct i40e_vsi *fdir_vsi;/* pointer to fdir VSI structure */ uint16_t match_counter_index;/* Statistic counter index used for fdir*/ -struct i40e_tx_queue *txq; +struct ci_tx_queue *txq;struct i40e_rx_queue *rxq;void *prg_pkt[I40E_FDIR_PRG_PKT_CNT ];/* memory for fdir program packet */ uint64_t dma_addr[I40E_FDIR_PRG_PKT_CNT ];/* physic address of packet memory*/
drivers/net/i40e/i40e_fdir.c +2 -2
@@ -1372,7 +1372,7 @@ i40e_find_available_buffer(struct rte_eth_dev *dev) {struct i40e_pf *pf =I40E_DEV_PRIVATE_TO_PF (dev->data->dev_private);struct i40e_fdir_info *fdir_info = &pf->fdir; -struct i40e_tx_queue *txq = pf->fdir.txq; +struct ci_tx_queue *txq = pf->fdir.txq;/* no available buffer * search for more available buffers from the current
@@ -1628,7 +1628,7 @@ i40e_flow_fdir_filter_programming(struct i40e_pf *pf,const struct i40e_fdir_filter_conf *filter, bool add, bool wait_status) { - struct i40e_tx_queue *txq = pf->fdir.txq; + struct ci_tx_queue *txq = pf->fdir.txq;struct i40e_rx_queue *rxq = pf->fdir.rxq;const struct i40e_fdir_action *fdir_action = &filter->action;volatile struct i40e_tx_desc *txdp;
drivers/net/i40e/i40e_recycle_mbufs_vec_common.c +1 -1
@@ -55,7 +55,7 @@ uint16_ti40e_recycle_tx_mbufs_reuse_vec (void *tx_queue, struct rte_eth_recycle_rxq_info *recycle_rxq_info) { -struct i40e_tx_queue *txq = tx_queue; +struct ci_tx_queue *txq = tx_queue;struct ci_tx_entry *txep;struct rte_mbuf **rxep;int i, n;
drivers/net/i40e/i40e_rxtx.c +29 -29
232 lines not on this page · raw message
drivers/net/i40e/i40e_rxtx.h +6 -44
78 lines not on this page · raw message
drivers/net/i40e/i40e_rxtx_vec_altivec.c +2 -2
@@ -551,7 +551,7 @@ uint16_t i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { -struct i40e_tx_queue *txq = (struct i40e_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile struct i40e_tx_desc *txdp;struct ci_tx_entry *txep;uint16_t n, nb_commit, tx_id;
@@ -625,7 +625,7 @@ i40e_rxq_vec_setup(struct i40e_rx_queue *rxq) } int __rte_cold -i40e_txq_vec_setup(struct i40e_tx_queue __rte_unused * txq) +i40e_txq_vec_setup(struct ci_tx_queue __rte_unused * txq) {return 0 ; }
drivers/net/i40e/i40e_rxtx_vec_avx2.c +2 -2
@@ -743,7 +743,7 @@ static inline uint16_t i40e_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { -struct i40e_tx_queue *txq = (struct i40e_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile struct i40e_tx_desc *txdp;struct ci_tx_entry *txep;uint16_t n, nb_commit, tx_id;
@@ -808,7 +808,7 @@ i40e_xmit_pkts_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) {uint16_t nb_tx =0 ; -struct i40e_tx_queue *txq = (struct i40e_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;while (nb_pkts) {uint16_t ret, num;
drivers/net/i40e/i40e_rxtx_vec_avx512.c +3 -3
@@ -755,7 +755,7 @@ i40e_recv_scattered_pkts_vec_avx512(void *rx_queue, } static __rte_always_inline int -i40e_tx_free_bufs_avx512 (struct i40e_tx_queue *txq) +i40e_tx_free_bufs_avx512 (struct ci_tx_queue *txq) {struct ci_tx_entry_vec *txep;uint32_t n;
@@ -933,7 +933,7 @@ static inline uint16_t i40e_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { -struct i40e_tx_queue *txq = (struct i40e_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile struct i40e_tx_desc *txdp;struct ci_tx_entry_vec *txep;uint16_t n, nb_commit, tx_id;
@@ -999,7 +999,7 @@ i40e_xmit_pkts_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) {uint16_t nb_tx =0 ; -struct i40e_tx_queue *txq = (struct i40e_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;while (nb_pkts) {uint16_t ret, num;
drivers/net/i40e/i40e_rxtx_vec_common.h +1 -1
@@ -17,7 +17,7 @@#endif static __rte_always_inline int -i40e_tx_free_bufs (struct i40e_tx_queue *txq) +i40e_tx_free_bufs (struct ci_tx_queue *txq) {struct ci_tx_entry *txep;uint32_t n;
drivers/net/i40e/i40e_rxtx_vec_neon.c +2 -2
@@ -679,7 +679,7 @@ uint16_t i40e_xmit_fixed_burst_vec(void *__rte_restrict tx_queue, struct rte_mbuf **__rte_restrict tx_pkts, uint16_t nb_pkts) { -struct i40e_tx_queue *txq = (struct i40e_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile struct i40e_tx_desc *txdp;struct ci_tx_entry *txep;uint16_t n, nb_commit, tx_id;
@@ -753,7 +753,7 @@ i40e_rxq_vec_setup(struct i40e_rx_queue *rxq) } int __rte_cold -i40e_txq_vec_setup(struct i40e_tx_queue __rte_unused *txq) +i40e_txq_vec_setup(struct ci_tx_queue *txq __rte_unused) {return 0 ; }
drivers/net/i40e/i40e_rxtx_vec_sse.c +2 -2
@@ -698,7 +698,7 @@ uint16_t i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { -struct i40e_tx_queue *txq = (struct i40e_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile struct i40e_tx_desc *txdp;struct ci_tx_entry *txep;uint16_t n, nb_commit, tx_id;
@@ -771,7 +771,7 @@ i40e_rxq_vec_setup(struct i40e_rx_queue *rxq) } int __rte_cold -i40e_txq_vec_setup(struct i40e_tx_queue __rte_unused *txq) +i40e_txq_vec_setup(struct ci_tx_queue *txq __rte_unused) {return 0 ; }
drivers/net/ice/ice_dcf.c +2 -2
@@ -1177,8 +1177,8 @@ ice_dcf_configure_queues(struct ice_dcf_hw *hw) {struct ice_rx_queue **rxq = (struct ice_rx_queue **)hw->eth_dev->data->rx_queues; -struct ice_tx_queue **txq = - (struct ice_tx_queue **)hw->eth_dev->data->tx_queues; +struct ci_tx_queue **txq = + (struct ci_tx_queue **)hw->eth_dev->data->tx_queues;struct virtchnl_vsi_queue_config_info *vc_config;struct virtchnl_queue_pair_info *vc_qp;struct dcf_virtchnl_cmd args;
drivers/net/ice/ice_dcf_ethdev.c +5 -5
@@ -387,7 +387,7 @@ reset_rx_queue(struct ice_rx_queue *rxq) }static inline void -reset_tx_queue (struct ice_tx_queue *txq) +reset_tx_queue (struct ci_tx_queue *txq) {struct ci_tx_entry *txe;uint32_t i, size;
@@ -454,7 +454,7 @@ ice_dcf_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id) {struct ice_dcf_adapter *ad = dev->data->dev_private;struct iavf_hw *hw = &ad->real_hw.avf; -struct ice_tx_queue *txq; +struct ci_tx_queue *txq;int err =0 ;if (tx_queue_id >= dev->data->nb_tx_queues)
@@ -486,7 +486,7 @@ ice_dcf_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id) {struct ice_dcf_adapter *ad = dev->data->dev_private;struct ice_dcf_hw *hw = &ad->real_hw; -struct ice_tx_queue *txq; +struct ci_tx_queue *txq;int err;if (tx_queue_id >= dev->data->nb_tx_queues)
@@ -511,7 +511,7 @@ static int ice_dcf_start_queues(struct rte_eth_dev *dev) {struct ice_rx_queue *rxq; -struct ice_tx_queue *txq; +struct ci_tx_queue *txq;int nb_rxq =0 ;int nb_txq, i;
@@ -638,7 +638,7 @@ ice_dcf_stop_queues(struct rte_eth_dev *dev)struct ice_dcf_adapter *ad = dev->data->dev_private;struct ice_dcf_hw *hw = &ad->real_hw;struct ice_rx_queue *rxq; -struct ice_tx_queue *txq; +struct ci_tx_queue *txq;int ret, i;/* Stop All queues */
drivers/net/ice/ice_diagnose.c +1 -1
@@ -605,7 +605,7 @@ void print_node(const struct rte_eth_dev_data *ethdata, get_elem_type(data->data.elem_type));if (data->data.elem_type ==ICE_AQC_ELEM_TYPE_LEAF ) {for (uint16_t i =0 ; i < ethdata->nb_tx_queues; i++) { -struct ice_tx_queue *q = ethdata->tx_queues[i]; +struct ci_tx_queue *q = ethdata->tx_queues[i];if (q->q_teid == data->node_teid) {fprintf (stream,"\t\t\t\t<tr><td>TXQ</td><td>%u</td></tr>\n" , i);break ;
drivers/net/ice/ice_ethdev.c +1 -1
@@ -6448,7 +6448,7 @@ ice_update_mbuf_stats(struct rte_eth_dev *ethdev, struct ice_mbuf_stats *mbuf_stats) {uint16_t idx; -struct ice_tx_queue *txq; +struct ci_tx_queue *txq;for (idx =0 ; idx < ethdev->data->nb_tx_queues; idx++) { txq = ethdev->data->tx_queues[idx];
drivers/net/ice/ice_ethdev.h +2 -2
@@ -258,7 +258,7 @@ struct ice_vsi_list { };struct ice_rx_queue ; -struct ice_tx_queue ; +struct ci_tx_queue ;/** * Structure that defines a VSI, associated with a adapter.
@@ -408,7 +408,7 @@ struct ice_fdir_counter_pool_container {*/ struct ice_fdir_info {struct ice_vsi *fdir_vsi;/* pointer to fdir VSI structure */ -struct ice_tx_queue *txq; +struct ci_tx_queue *txq;struct ice_rx_queue *rxq;void *prg_pkt;/* memory for fdir program packet */ uint64_t dma_addr;/* physic address of packet memory*/
drivers/net/ice/ice_rxtx.c +30 -30
240 lines not on this page · raw message
drivers/net/ice/ice_rxtx.h +2 -39
65 lines not on this page · raw message
drivers/net/ice/ice_rxtx_vec_avx2.c +2 -2
@@ -856,7 +856,7 @@ static __rte_always_inline uint16_t ice_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts, bool offload) { -struct ice_tx_queue *txq = (struct ice_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile struct ice_tx_desc *txdp;struct ci_tx_entry *txep;uint16_t n, nb_commit, tx_id;
@@ -924,7 +924,7 @@ ice_xmit_pkts_vec_avx2_common(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts, bool offload) { uint16_t nb_tx = 0; -struct ice_tx_queue *txq = (struct ice_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;while (nb_pkts) {uint16_t ret, num;
drivers/net/ice/ice_rxtx_vec_avx512.c +4 -4
@@ -860,7 +860,7 @@ ice_recv_scattered_pkts_vec_avx512_offload(void *rx_queue, } static __rte_always_inline int -ice_tx_free_bufs_avx512 (struct ice_tx_queue *txq) +ice_tx_free_bufs_avx512 (struct ci_tx_queue *txq) {struct ci_tx_entry_vec *txep;uint32_t n;
@@ -1053,7 +1053,7 @@ static __rte_always_inline uint16_t ice_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts, bool do_offload) { -struct ice_tx_queue *txq = (struct ice_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile struct ice_tx_desc *txdp;struct ci_tx_entry_vec *txep;uint16_t n, nb_commit, tx_id;
@@ -1122,7 +1122,7 @@ ice_xmit_pkts_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) {uint16_t nb_tx =0 ; -struct ice_tx_queue *txq = (struct ice_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;while (nb_pkts) {uint16_t ret, num;
@@ -1144,7 +1144,7 @@ ice_xmit_pkts_vec_avx512_offload(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) {uint16_t nb_tx =0 ; -struct ice_tx_queue *txq = (struct ice_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;while (nb_pkts) {uint16_t ret, num;
drivers/net/ice/ice_rxtx_vec_common.h +4 -4
@@ -13,7 +13,7 @@#endif static __rte_always_inline int -ice_tx_free_bufs_vec (struct ice_tx_queue *txq) +ice_tx_free_bufs_vec (struct ci_tx_queue *txq) {struct ci_tx_entry *txep;uint32_t n;
@@ -105,7 +105,7 @@ _ice_rx_queue_release_mbufs_vec(struct ice_rx_queue *rxq) }static inline void -_ice_tx_queue_release_mbufs_vec (struct ice_tx_queue *txq) +_ice_tx_queue_release_mbufs_vec (struct ci_tx_queue *txq) {uint16_t i;
@@ -231,7 +231,7 @@ ice_rx_vec_queue_default(struct ice_rx_queue *rxq) }static inline int -ice_tx_vec_queue_default (struct ice_tx_queue *txq) +ice_tx_vec_queue_default (struct ci_tx_queue *txq) {if (!txq)return -1 ;
@@ -273,7 +273,7 @@ static inline int ice_tx_vec_dev_check_default(struct rte_eth_dev *dev) {int i; -struct ice_tx_queue *txq; +struct ci_tx_queue *txq;int ret =0 ;int result =0 ;
drivers/net/ice/ice_rxtx_vec_sse.c +3 -3
@@ -697,7 +697,7 @@ static uint16_t ice_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { -struct ice_tx_queue *txq = (struct ice_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile struct ice_tx_desc *txdp;struct ci_tx_entry *txep;uint16_t n, nb_commit, tx_id;
@@ -766,7 +766,7 @@ ice_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) {uint16_t nb_tx =0 ; -struct ice_tx_queue *txq = (struct ice_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;while (nb_pkts) {uint16_t ret, num;
@@ -793,7 +793,7 @@ ice_rxq_vec_setup(struct ice_rx_queue *rxq) } int __rte_cold -ice_txq_vec_setup(struct ice_tx_queue __rte_unused *txq) +ice_txq_vec_setup(struct ci_tx_queue *txq __rte_unused) {if (!txq)return -1 ;
Signature · 1 line
2.43.0
PATCH v2 7/22 net/iavf: use common Tx queue structure
In reply to Bruce RichardsonMerge in the few additional fields used by iavf driver and convert it to using the common Tx queue structure also.
--- drivers/net/_common_intel/tx.h | 15 +++++++- drivers/net/iavf/iavf.h | 2 +- drivers/net/iavf/iavf_ethdev.c | 4 +- drivers/net/iavf/iavf_rxtx.c | 42 ++++++++++----------- drivers/net/iavf/iavf_rxtx.h | 49 +++---------------------- drivers/net/iavf/iavf_rxtx_vec_avx2.c | 4 +- drivers/net/iavf/iavf_rxtx_vec_avx512.c | 14 +++---- drivers/net/iavf/iavf_rxtx_vec_common.h | 8 ++-- drivers/net/iavf/iavf_rxtx_vec_sse.c | 8 ++-- drivers/net/iavf/iavf_vchnl.c | 6 +-- 10 files changed, 62 insertions(+), 90 deletions(-)
drivers/net/_common_intel/tx.h +13 -2
@@ -31,8 +31,9 @@ typedef void (*ice_tx_release_mbufs_t)(struct ci_tx_queue *txq);struct ci_tx_queue {union {/* TX ring virtual address */ -volatile struct ice_tx_desc *ice_tx_ring;volatile struct i40e_tx_desc *i40e_tx_ring; +volatile struct iavf_tx_desc *iavf_tx_ring; +volatile struct ice_tx_desc *ice_tx_ring; };volatile uint8_t *qtx_tail;/* register address of tail */ struct ci_tx_entry *sw_ring;/* virtual address of SW ring */
@@ -63,8 +64,9 @@ struct ci_tx_queue {bool tx_deferred_start;/* don't start this queue in dev start */ bool q_set;/* indicate if tx queue has been configured */ union {/* the VSI this queue belongs to */ -struct ice_vsi *ice_vsi;struct i40e_vsi *i40e_vsi; +struct iavf_vsi *iavf_vsi; +struct ice_vsi *ice_vsi; };const struct rte_memzone *mz;
@@ -76,6 +78,15 @@ struct ci_tx_queue {struct {/* I40E driver specific values */ uint8_t dcb_tc; }; +struct {/* iavf driver specific values */ +uint16_t ipsec_crypto_pkt_md_offset; +uint8_t rel_mbufs_type; +#define IAVF_TX_FLAGS_VLAN_TAG_LOC_L2TAG1 BIT(0) +#define IAVF_TX_FLAGS_VLAN_TAG_LOC_L2TAG2 BIT(1) +uint8_t vlan_flag; +uint8_t tc; +bool use_ctx;/* with ctx info, each pkt needs two descriptors */ + }; }; };
drivers/net/iavf/iavf.h +1 -1
@@ -98,7 +98,7 @@struct iavf_adapter ;struct iavf_rx_queue ; -struct iavf_tx_queue ; +struct ci_tx_queue ;struct iavf_ipsec_crypto_stats {
drivers/net/iavf/iavf_ethdev.c +2 -2
@@ -954,7 +954,7 @@ static int iavf_start_queues(struct rte_eth_dev *dev) {struct iavf_rx_queue *rxq; -struct iavf_tx_queue *txq; +struct ci_tx_queue *txq;int i;uint16_t nb_txq, nb_rxq;
@@ -1885,7 +1885,7 @@ iavf_dev_update_mbuf_stats(struct rte_eth_dev *ethdev, struct iavf_mbuf_stats *mbuf_stats) {uint16_t idx; -struct iavf_tx_queue *txq; +struct ci_tx_queue *txq;for (idx =0 ; idx < ethdev->data->nb_tx_queues; idx++) { txq = ethdev->data->tx_queues[idx];
drivers/net/iavf/iavf_rxtx.c +21 -21
168 lines not on this page · raw message
drivers/net/iavf/iavf_rxtx.h +6 -43
84 lines not on this page · raw message
drivers/net/iavf/iavf_rxtx_vec_avx2.c +2 -2
@@ -1734,7 +1734,7 @@ static __rte_always_inline uint16_t iavf_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts, bool offload) { -struct iavf_tx_queue *txq = (struct iavf_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile struct iavf_tx_desc *txdp;struct ci_tx_entry *txep;uint16_t n, nb_commit, tx_id;
@@ -1801,7 +1801,7 @@ iavf_xmit_pkts_vec_avx2_common(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts, bool offload) { uint16_t nb_tx = 0; -struct iavf_tx_queue *txq = (struct iavf_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;while (nb_pkts) {uint16_t ret, num;
drivers/net/iavf/iavf_rxtx_vec_avx512.c +7 -7
@@ -1845,7 +1845,7 @@ iavf_recv_scattered_pkts_vec_avx512_flex_rxd_offload(void *rx_queue, } static __rte_always_inline int -iavf_tx_free_bufs_avx512 (struct iavf_tx_queue *txq) +iavf_tx_free_bufs_avx512 (struct ci_tx_queue *txq) {struct ci_tx_entry_vec *txep;uint32_t n;
@@ -2311,7 +2311,7 @@ static __rte_always_inline uint16_t iavf_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts, bool offload) { -struct iavf_tx_queue *txq = (struct iavf_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile struct iavf_tx_desc *txdp;struct ci_tx_entry_vec *txep;uint16_t n, nb_commit, tx_id;
@@ -2379,7 +2379,7 @@ static __rte_always_inline uint16_t iavf_xmit_fixed_burst_vec_avx512_ctx(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts, bool offload) { -struct iavf_tx_queue *txq = (struct iavf_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile struct iavf_tx_desc *txdp;struct ci_tx_entry_vec *txep;uint16_t n, nb_commit, nb_mbuf, tx_id;
@@ -2447,7 +2447,7 @@ iavf_xmit_pkts_vec_avx512_cmn(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts, bool offload) { uint16_t nb_tx = 0; -struct iavf_tx_queue *txq = (struct iavf_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;while (nb_pkts) {uint16_t ret, num;
@@ -2473,7 +2473,7 @@ iavf_xmit_pkts_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, } void __rte_cold -iavf_tx_queue_release_mbufs_avx512 (struct iavf_tx_queue *txq) +iavf_tx_queue_release_mbufs_avx512 (struct ci_tx_queue *txq) {unsigned int i;const uint16_t max_desc = (uint16_t )(txq->nb_tx_desc -1 );
@@ -2494,7 +2494,7 @@ iavf_tx_queue_release_mbufs_avx512(struct iavf_tx_queue *txq) } int __rte_cold -iavf_txq_vec_setup_avx512 (struct iavf_tx_queue *txq) +iavf_txq_vec_setup_avx512 (struct ci_tx_queue *txq) { txq->rel_mbufs_type =IAVF_REL_MBUFS_AVX512_VEC ;return 0 ;
@@ -2512,7 +2512,7 @@ iavf_xmit_pkts_vec_avx512_ctx_cmn(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts, bool offload) { uint16_t nb_tx = 0; -struct iavf_tx_queue *txq = (struct iavf_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;while (nb_pkts) {uint16_t ret, num;
drivers/net/iavf/iavf_rxtx_vec_common.h +4 -4
@@ -17,7 +17,7 @@#endif static __rte_always_inline int -iavf_tx_free_bufs (struct iavf_tx_queue *txq) +iavf_tx_free_bufs (struct ci_tx_queue *txq) {struct ci_tx_entry *txep;uint32_t n;
@@ -104,7 +104,7 @@ _iavf_rx_queue_release_mbufs_vec(struct iavf_rx_queue *rxq) }static inline void -_iavf_tx_queue_release_mbufs_vec (struct iavf_tx_queue *txq) +_iavf_tx_queue_release_mbufs_vec (struct ci_tx_queue *txq) {unsigned i;const uint16_t max_desc = (uint16_t )(txq->nb_tx_desc -1 );
@@ -164,7 +164,7 @@ iavf_rx_vec_queue_default(struct iavf_rx_queue *rxq) }static inline int -iavf_tx_vec_queue_default (struct iavf_tx_queue *txq) +iavf_tx_vec_queue_default (struct ci_tx_queue *txq) {if (!txq)return -1 ;
@@ -227,7 +227,7 @@ static inline int iavf_tx_vec_dev_check_default(struct rte_eth_dev *dev) {int i; -struct iavf_tx_queue *txq; +struct ci_tx_queue *txq;int ret;int result =0 ;
drivers/net/iavf/iavf_rxtx_vec_sse.c +4 -4
@@ -1366,7 +1366,7 @@ uint16_t iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { -struct iavf_tx_queue *txq = (struct iavf_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile struct iavf_tx_desc *txdp;struct ci_tx_entry *txep;uint16_t n, nb_commit, tx_id;
@@ -1435,7 +1435,7 @@ iavf_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) {uint16_t nb_tx =0 ; -struct iavf_tx_queue *txq = (struct iavf_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;while (nb_pkts) {uint16_t ret, num;
@@ -1459,13 +1459,13 @@ iavf_rx_queue_release_mbufs_sse(struct iavf_rx_queue *rxq) } void __rte_cold -iavf_tx_queue_release_mbufs_sse (struct iavf_tx_queue *txq) +iavf_tx_queue_release_mbufs_sse (struct ci_tx_queue *txq) {_iavf_tx_queue_release_mbufs_vec (txq); } int __rte_cold -iavf_txq_vec_setup (struct iavf_tx_queue *txq) +iavf_txq_vec_setup (struct ci_tx_queue *txq) { txq->rel_mbufs_type =IAVF_REL_MBUFS_SSE_VEC ;return 0 ;
drivers/net/iavf/iavf_vchnl.c +2 -4
@@ -1218,10 +1218,8 @@ int iavf_configure_queues(struct iavf_adapter *adapter, uint16_t num_queue_pairs, uint16_t index) { - struct iavf_rx_queue **rxq = - (struct iavf_rx_queue **)adapter->dev_data->rx_queues; -struct iavf_tx_queue **txq = - (struct iavf_tx_queue **)adapter->dev_data->tx_queues; + struct iavf_rx_queue **rxq = (struct iavf_rx_queue **)adapter->dev_data->rx_queues; +struct ci_tx_queue **txq = (struct ci_tx_queue **)adapter->dev_data->tx_queues;struct iavf_info *vf =IAVF_DEV_PRIVATE_TO_VF (adapter);struct virtchnl_vsi_queue_config_info *vc_config;struct virtchnl_queue_pair_info *vc_qp;
Signature · 1 line
2.43.0
PATCH v2 8/22 net/ixgbe: convert Tx queue context cache field to ptr
In reply to Bruce RichardsonRather than having a two element array of context cache values inside the Tx queue structure, convert it to a pointer to a cache at the end of the structure. This makes future merging of the structure easier as we don't need the "ixgbe_advctx_info" struct defined when defining a combined queue structure.
--- drivers/net/ixgbe/ixgbe_rxtx.c | 7 ++++--- drivers/net/ixgbe/ixgbe_rxtx.h | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-)
drivers/net/ixgbe/ixgbe_rxtx.c +4 -3
@@ -2522,8 +2522,7 @@ ixgbe_reset_tx_queue(struct ixgbe_tx_queue *txq) txq->last_desc_cleaned = (uint16_t )(txq->nb_tx_desc -1 ); txq->nb_tx_free = (uint16_t )(txq->nb_tx_desc -1 ); txq->ctx_curr =0 ; -memset ((void *)&txq->ctx_cache,0 , -IXGBE_CTX_NUM *sizeof (struct ixgbe_advctx_info )); +memset (txq->ctx_cache,0 ,IXGBE_CTX_NUM *sizeof (struct ixgbe_advctx_info )); }static const struct ixgbe_txq_ops def_txq_ops = {
@@ -2741,10 +2740,12 @@ ixgbe_dev_tx_queue_setup(struct rte_eth_dev *dev, }/* First allocate the tx queue data structure */ - txq =rte_zmalloc_socket ("ethdev TX queue" ,sizeof (struct ixgbe_tx_queue ), + txq =rte_zmalloc_socket ("ethdev TX queue" ,sizeof (struct ixgbe_tx_queue ) + +sizeof (struct ixgbe_advctx_info ) *IXGBE_CTX_NUM ,RTE_CACHE_LINE_SIZE , socket_id);if (txq ==NULL )return -ENOMEM ; + txq->ctx_cache =RTE_PTR_ADD (txq,sizeof (struct ixgbe_tx_queue ));/* * Allocate TX ring hardware descriptors. A memzone large enough to
drivers/net/ixgbe/ixgbe_rxtx.h +2 -2
@@ -215,8 +215,8 @@ struct ixgbe_tx_queue {uint8_t wthresh;/**< Write-back threshold reg. */ uint64_t offloads;/**< Tx offload flags of RTE_ETH_TX_OFFLOAD_* */ uint32_t ctx_curr;/**< Hardware context states. */ -/** Hardware context0 history. */ -struct ixgbe_advctx_info ctx_cache[IXGBE_CTX_NUM ]; +/** Hardware context history. */ +struct ixgbe_advctx_info *ctx_cache;const struct ixgbe_txq_ops *ops;/**< txq ops */ bool tx_deferred_start;/**< not in global dev start. */ #ifdef RTE_LIB_SECURITY
Signature · 1 line
2.43.0
PATCH v2 9/22 net/ixgbe: use common Tx queue structure
In reply to Bruce RichardsonMerge in additional fields used by the ixgbe driver and then convert it over to using the common Tx queue structure.
--- drivers/net/_common_intel/tx.h | 14 +++- drivers/net/ixgbe/ixgbe_ethdev.c | 4 +- .../ixgbe/ixgbe_recycle_mbufs_vec_common.c | 2 +- drivers/net/ixgbe/ixgbe_rxtx.c | 64 +++++++++---------- drivers/net/ixgbe/ixgbe_rxtx.h | 56 ++-------------- drivers/net/ixgbe/ixgbe_rxtx_vec_common.h | 26 ++++---- drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c | 14 ++-- drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c | 14 ++-- 8 files changed, 80 insertions(+), 114 deletions(-)
drivers/net/_common_intel/tx.h +13 -1
@@ -34,9 +34,13 @@ struct ci_tx_queue {volatile struct i40e_tx_desc *i40e_tx_ring;volatile struct iavf_tx_desc *iavf_tx_ring;volatile struct ice_tx_desc *ice_tx_ring; +volatile union ixgbe_adv_tx_desc *ixgbe_tx_ring; };volatile uint8_t *qtx_tail;/* register address of tail */ -struct ci_tx_entry *sw_ring;/* virtual address of SW ring */ +union { +struct ci_tx_entry *sw_ring;/* virtual address of SW ring */ +struct ci_tx_entry_vec *sw_ring_vec; + };rte_iova_t tx_ring_dma;/* TX ring DMA address */ uint16_t nb_tx_desc;/* number of TX descriptors */ uint16_t tx_tail;/* current value of tail register */
@@ -87,6 +91,14 @@ struct ci_tx_queue {uint8_t tc;bool use_ctx;/* with ctx info, each pkt needs two descriptors */ }; +struct {/* ixgbe specific values */ +const struct ixgbe_txq_ops *ops; +struct ixgbe_advctx_info *ctx_cache; +uint32_t ctx_curr; +#ifdef RTE_LIB_SECURITY +uint8_t using_ipsec;/**< indicates that IPsec TX feature is in use */ +#endif + }; }; };
drivers/net/ixgbe/ixgbe_ethdev.c +2 -2
@@ -1118,7 +1118,7 @@ eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev, void *init_params __rte_unused)* RX and TX function. */ if (rte_eal_process_type () !=RTE_PROC_PRIMARY ) { -struct ixgbe_tx_queue *txq; +struct ci_tx_queue *txq;/* TX queue function in primary, set by last queue initialized * Tx queue may not initialized by primary process */
@@ -1623,7 +1623,7 @@ eth_ixgbevf_dev_init(struct rte_eth_dev *eth_dev)* RX function */ if (rte_eal_process_type () !=RTE_PROC_PRIMARY ) { -struct ixgbe_tx_queue *txq; +struct ci_tx_queue *txq;/* TX queue function in primary, set by last queue initialized * Tx queue may not initialized by primary process */
drivers/net/ixgbe/ixgbe_recycle_mbufs_vec_common.c +1 -1
@@ -51,7 +51,7 @@ uint16_tixgbe_recycle_tx_mbufs_reuse_vec (void *tx_queue, struct rte_eth_recycle_rxq_info *recycle_rxq_info) { -struct ixgbe_tx_queue *txq = tx_queue; +struct ci_tx_queue *txq = tx_queue;struct ci_tx_entry *txep;struct rte_mbuf **rxep;int i, n;
drivers/net/ixgbe/ixgbe_rxtx.c +32 -32
254 lines not on this page · raw message
drivers/net/ixgbe/ixgbe_rxtx.h +5 -51
75 lines not on this page · raw message
drivers/net/ixgbe/ixgbe_rxtx_vec_common.h +13 -13
@@ -12,7 +12,7 @@#include "ixgbe_rxtx.h" static __rte_always_inline int -ixgbe_tx_free_bufs (struct ixgbe_tx_queue *txq) +ixgbe_tx_free_bufs (struct ci_tx_queue *txq) {struct ci_tx_entry_vec *txep;uint32_t status;
@@ -32,7 +32,7 @@ ixgbe_tx_free_bufs(struct ixgbe_tx_queue *txq)* first buffer to free from S/W ring is at index * tx_next_dd - (tx_rs_thresh-1) */ - txep = &txq->sw_ring_v[txq->tx_next_dd - (n -1 )]; + txep = &txq->sw_ring_vec[txq->tx_next_dd - (n -1 )]; m =rte_pktmbuf_prefree_seg (txep[0 ].mbuf);if (likely (m !=NULL )) { free[0 ] = m;
@@ -79,7 +79,7 @@ tx_backlog_entry(struct ci_tx_entry_vec *txep, }static inline void -_ixgbe_tx_queue_release_mbufs_vec (struct ixgbe_tx_queue *txq) +_ixgbe_tx_queue_release_mbufs_vec (struct ci_tx_queue *txq) {unsigned int i;struct ci_tx_entry_vec *txe;
@@ -92,14 +92,14 @@ _ixgbe_tx_queue_release_mbufs_vec(struct ixgbe_tx_queue *txq)for (i = txq->tx_next_dd - (txq->tx_rs_thresh -1 ); i != txq->tx_tail; i = (i +1 ) % txq->nb_tx_desc) { - txe = &txq->sw_ring_v[i]; + txe = &txq->sw_ring_vec[i];rte_pktmbuf_free_seg (txe->mbuf); } txq->nb_tx_free = max_desc;/* reset tx_entry */ for (i =0 ; i < txq->nb_tx_desc; i++) { - txe = &txq->sw_ring_v[i]; + txe = &txq->sw_ring_vec[i]; txe->mbuf =NULL ; } }
@@ -134,22 +134,22 @@ _ixgbe_rx_queue_release_mbufs_vec(struct ixgbe_rx_queue *rxq) }static inline void -_ixgbe_tx_free_swring_vec (struct ixgbe_tx_queue *txq) +_ixgbe_tx_free_swring_vec (struct ci_tx_queue *txq) {if (txq ==NULL )return ;if (txq->sw_ring !=NULL ) { -rte_free (txq->sw_ring_v -1 ); - txq->sw_ring_v =NULL ; +rte_free (txq->sw_ring_vec -1 ); + txq->sw_ring_vec =NULL ; } }static inline void -_ixgbe_reset_tx_queue_vec (struct ixgbe_tx_queue *txq) +_ixgbe_reset_tx_queue_vec (struct ci_tx_queue *txq) {static const union ixgbe_adv_tx_desc zeroed_desc = { {0 } }; -struct ci_tx_entry_vec *txe = txq->sw_ring_v; +struct ci_tx_entry_vec *txe = txq->sw_ring_vec;uint16_t i;/* Zero out HW ring memory */
@@ -199,14 +199,14 @@ ixgbe_rxq_vec_setup_default(struct ixgbe_rx_queue *rxq) }static inline int -ixgbe_txq_vec_setup_default (struct ixgbe_tx_queue *txq, +ixgbe_txq_vec_setup_default (struct ci_tx_queue *txq,const struct ixgbe_txq_ops *txq_ops) { -if (txq->sw_ring_v ==NULL ) +if (txq->sw_ring_vec ==NULL )return -1 ;/* leave the first one for overflow */ - txq->sw_ring_v = txq->sw_ring_v +1 ; + txq->sw_ring_vec = txq->sw_ring_vec +1 ; txq->ops = txq_ops;return 0 ;
drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c +7 -7
@@ -571,7 +571,7 @@ uint16_t ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { -struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile union ixgbe_adv_tx_desc *txdp;struct ci_tx_entry_vec *txep;uint16_t n, nb_commit, tx_id;
@@ -591,7 +591,7 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = txq->tx_tail; txdp = &txq->ixgbe_tx_ring[tx_id]; - txep = &txq->sw_ring_v[tx_id]; + txep = &txq->sw_ring_vec[tx_id]; txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts);
@@ -611,7 +611,7 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,/* avoid reach the end of ring */ txdp = &txq->ixgbe_tx_ring[tx_id]; - txep = &txq->sw_ring_v[tx_id]; + txep = &txq->sw_ring_vec[tx_id]; }tx_backlog_entry (txep, tx_pkts, nb_commit);
@@ -634,7 +634,7 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, } static void __rte_cold -ixgbe_tx_queue_release_mbufs_vec (struct ixgbe_tx_queue *txq) +ixgbe_tx_queue_release_mbufs_vec (struct ci_tx_queue *txq) {_ixgbe_tx_queue_release_mbufs_vec (txq); }
@@ -646,13 +646,13 @@ ixgbe_rx_queue_release_mbufs_vec(struct ixgbe_rx_queue *rxq) } static void __rte_cold -ixgbe_tx_free_swring (struct ixgbe_tx_queue *txq) +ixgbe_tx_free_swring (struct ci_tx_queue *txq) {_ixgbe_tx_free_swring_vec (txq); } static void __rte_cold -ixgbe_reset_tx_queue (struct ixgbe_tx_queue *txq) +ixgbe_reset_tx_queue (struct ci_tx_queue *txq) {_ixgbe_reset_tx_queue_vec (txq); }
@@ -670,7 +670,7 @@ ixgbe_rxq_vec_setup(struct ixgbe_rx_queue *rxq) } int __rte_cold -ixgbe_txq_vec_setup (struct ixgbe_tx_queue *txq) +ixgbe_txq_vec_setup (struct ci_tx_queue *txq) {return ixgbe_txq_vec_setup_default (txq, &vec_txq_ops); }
drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c +7 -7
@@ -693,7 +693,7 @@ uint16_t ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { -struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile union ixgbe_adv_tx_desc *txdp;struct ci_tx_entry_vec *txep;uint16_t n, nb_commit, tx_id;
@@ -713,7 +713,7 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = txq->tx_tail; txdp = &txq->ixgbe_tx_ring[tx_id]; - txep = &txq->sw_ring_v[tx_id]; + txep = &txq->sw_ring_vec[tx_id]; txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts);
@@ -734,7 +734,7 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,/* avoid reach the end of ring */ txdp = &txq->ixgbe_tx_ring[tx_id]; - txep = &txq->sw_ring_v[tx_id]; + txep = &txq->sw_ring_vec[tx_id]; }tx_backlog_entry (txep, tx_pkts, nb_commit);
@@ -757,7 +757,7 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, } static void __rte_cold -ixgbe_tx_queue_release_mbufs_vec (struct ixgbe_tx_queue *txq) +ixgbe_tx_queue_release_mbufs_vec (struct ci_tx_queue *txq) {_ixgbe_tx_queue_release_mbufs_vec (txq); }
@@ -769,13 +769,13 @@ ixgbe_rx_queue_release_mbufs_vec(struct ixgbe_rx_queue *rxq) } static void __rte_cold -ixgbe_tx_free_swring (struct ixgbe_tx_queue *txq) +ixgbe_tx_free_swring (struct ci_tx_queue *txq) {_ixgbe_tx_free_swring_vec (txq); } static void __rte_cold -ixgbe_reset_tx_queue (struct ixgbe_tx_queue *txq) +ixgbe_reset_tx_queue (struct ci_tx_queue *txq) {_ixgbe_reset_tx_queue_vec (txq); }
@@ -793,7 +793,7 @@ ixgbe_rxq_vec_setup(struct ixgbe_rx_queue *rxq) } int __rte_cold -ixgbe_txq_vec_setup (struct ixgbe_tx_queue *txq) +ixgbe_txq_vec_setup (struct ci_tx_queue *txq) {return ixgbe_txq_vec_setup_default (txq, &vec_txq_ops); }
Signature · 1 line
2.43.0
PATCH v2 10/22 net/_common_intel: pack Tx queue structure
In reply to Bruce RichardsonMove some fields about to better pack the Tx queue structure and make sure all data used by the vector codepaths is on the first cacheline of the structure. Checking with "pahole" on 64-bit build, only one 6-byte hole is left in the structure - on second cacheline - after this patch.
As part of the reordering, move the p/h/wthresh values to the ixgbe-specific part of the union. That is the only driver which actually uses those values. i40e and ice drivers just record the values for later return, so we can drop them from the Tx queue structure for those drivers and just report the defaults in all cases.
--- drivers/net/_common_intel/tx.h | 12 +++++------- drivers/net/i40e/i40e_rxtx.c | 9 +++------ drivers/net/ice/ice_rxtx.c | 9 +++------ 3 files changed, 11 insertions(+), 19 deletions(-)
drivers/net/_common_intel/tx.h +5 -7
@@ -41,7 +41,6 @@ struct ci_tx_queue {struct ci_tx_entry *sw_ring;/* virtual address of SW ring */ struct ci_tx_entry_vec *sw_ring_vec; }; -rte_iova_t tx_ring_dma;/* TX ring DMA address */ uint16_t nb_tx_desc;/* number of TX descriptors */ uint16_t tx_tail;/* current value of tail register */ uint16_t nb_tx_used;/* number of TX desc used since RS bit set */
@@ -55,16 +54,14 @@ struct ci_tx_queue {uint16_t tx_free_thresh;/* Number of TX descriptors to use before RS bit is set. */ uint16_t tx_rs_thresh; -uint8_t pthresh;/**< Prefetch threshold register. */ -uint8_t hthresh;/**< Host threshold register. */ -uint8_t wthresh;/**< Write-back threshold reg. */ uint16_t port_id;/* Device port identifier. */ uint16_t queue_id;/* TX queue index. */ uint16_t reg_idx; -uint64_t offloads;uint16_t tx_next_dd;uint16_t tx_next_rs; +uint64_t offloads;uint64_t mbuf_errors; +rte_iova_t tx_ring_dma;/* TX ring DMA address */ bool tx_deferred_start;/* don't start this queue in dev start */ bool q_set;/* indicate if tx queue has been configured */ union {/* the VSI this queue belongs to */
@@ -95,9 +92,10 @@ struct ci_tx_queue {const struct ixgbe_txq_ops *ops;struct ixgbe_advctx_info *ctx_cache;uint32_t ctx_curr; -#ifdef RTE_LIB_SECURITY +uint8_t pthresh;/**< Prefetch threshold register. */ +uint8_t hthresh;/**< Host threshold register. */ +uint8_t wthresh;/**< Write-back threshold reg. */ uint8_t using_ipsec;/**< indicates that IPsec TX feature is in use */ -#endif }; }; };
drivers/net/i40e/i40e_rxtx.c +3 -6
@@ -2539,9 +2539,6 @@ i40e_dev_tx_queue_setup(struct rte_eth_dev *dev, txq->nb_tx_desc = nb_desc; txq->tx_rs_thresh = tx_rs_thresh; txq->tx_free_thresh = tx_free_thresh; - txq->pthresh = tx_conf->tx_thresh.pthresh; - txq->hthresh = tx_conf->tx_thresh.hthresh; - txq->wthresh = tx_conf->tx_thresh.wthresh; txq->queue_id = queue_idx; txq->reg_idx = reg_idx; txq->port_id = dev->data->port_id;
@@ -3310,9 +3307,9 @@ i40e_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, qinfo->nb_desc = txq->nb_tx_desc; - qinfo->conf.tx_thresh.pthresh = txq->pthresh; - qinfo->conf.tx_thresh.hthresh = txq->hthresh; - qinfo->conf.tx_thresh.wthresh = txq->wthresh; + qinfo->conf.tx_thresh.pthresh =I40E_DEFAULT_TX_PTHRESH ; + qinfo->conf.tx_thresh.hthresh =I40E_DEFAULT_TX_HTHRESH ; + qinfo->conf.tx_thresh.wthresh =I40E_DEFAULT_TX_WTHRESH ; qinfo->conf.tx_free_thresh = txq->tx_free_thresh; qinfo->conf.tx_rs_thresh = txq->tx_rs_thresh;
drivers/net/ice/ice_rxtx.c +3 -6
@@ -1492,9 +1492,6 @@ ice_tx_queue_setup(struct rte_eth_dev *dev, txq->nb_tx_desc = nb_desc; txq->tx_rs_thresh = tx_rs_thresh; txq->tx_free_thresh = tx_free_thresh; - txq->pthresh = tx_conf->tx_thresh.pthresh; - txq->hthresh = tx_conf->tx_thresh.hthresh; - txq->wthresh = tx_conf->tx_thresh.wthresh; txq->queue_id = queue_idx; txq->reg_idx = vsi->base_queue + queue_idx;
@@ -1583,9 +1580,9 @@ ice_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, qinfo->nb_desc = txq->nb_tx_desc; - qinfo->conf.tx_thresh.pthresh = txq->pthresh; - qinfo->conf.tx_thresh.hthresh = txq->hthresh; - qinfo->conf.tx_thresh.wthresh = txq->wthresh; + qinfo->conf.tx_thresh.pthresh =ICE_DEFAULT_TX_PTHRESH ; + qinfo->conf.tx_thresh.hthresh =ICE_DEFAULT_TX_HTHRESH ; + qinfo->conf.tx_thresh.wthresh =ICE_DEFAULT_TX_WTHRESH ; qinfo->conf.tx_free_thresh = txq->tx_free_thresh; qinfo->conf.tx_rs_thresh = txq->tx_rs_thresh;
Signature · 1 line
2.43.0
PATCH v2 11/22 net/_common_intel: add post-Tx buffer free function
In reply to Bruce RichardsonThe actions taken for post-Tx buffer free for the SSE and AVX drivers for i40e, iavf and ice drivers are all common, so centralize those in common/intel_eth driver.
--- drivers/net/_common_intel/tx.h | 71 ++++++++++++++++++++++++ drivers/net/i40e/i40e_rxtx_vec_common.h | 72 ++++--------------------- drivers/net/iavf/iavf_rxtx_vec_common.h | 61 ++++----------------- drivers/net/ice/ice_rxtx_vec_common.h | 61 ++++----------------- 4 files changed, 98 insertions(+), 167 deletions(-)
drivers/net/_common_intel/tx.h +71 -0
@@ -7,6 +7,7 @@#include <stdint.h> #include <rte_mbuf.h> +#include <rte_ethdev.h> /* forward declaration of the common intel (ci) queue structure */ struct ci_tx_queue ;
@@ -107,4 +108,74 @@ ci_tx_backlog_entry(struct ci_tx_entry *txep, struct rte_mbuf **tx_pkts, uint16_ txep[i].mbuf = tx_pkts[i]; } +#define IETH_VPMD_TX_MAX_FREE_BUF 64 + +typedef int (*ci_desc_done_fn )(struct ci_tx_queue *txq,uint16_t idx); + +static __rte_always_inline int +ci_tx_free_bufs (struct ci_tx_queue *txq,ci_desc_done_fn desc_done) +{ +struct ci_tx_entry *txep; +uint32_t n; +uint32_t i; +int nb_free =0 ; +struct rte_mbuf *m, *free[IETH_VPMD_TX_MAX_FREE_BUF ]; + +/* check DD bits on threshold descriptor */ +if (!desc_done (txq, txq->tx_next_dd)) +return 0 ; + + n = txq->tx_rs_thresh; + +/* first buffer to free from S/W ring is at index +* tx_next_dd - (tx_rs_thresh-1) +*/ + txep = &txq->sw_ring[txq->tx_next_dd - (n -1 )]; + +if (txq->offloads &RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE ) { +for (i =0 ; i < n; i++) { + free[i] = txep[i].mbuf; +/* no need to reset txep[i].mbuf in vector path */ + } +rte_mempool_put_bulk (free[0 ]->pool, (void **)free, n); + goto done; + } + + m =rte_pktmbuf_prefree_seg (txep[0 ].mbuf); +if (likely (m !=NULL )) { + free[0 ] = m; + nb_free =1 ; +for (i =1 ; i < n; i++) { + m =rte_pktmbuf_prefree_seg (txep[i].mbuf); +if (likely (m !=NULL )) { +if (likely (m->pool == free[0 ]->pool)) { + free[nb_free++] = m; + }else { +rte_mempool_put_bulk (free[0 ]->pool, + (void *)free, + nb_free); + free[0 ] = m; + nb_free =1 ; + } + } + } +rte_mempool_put_bulk (free[0 ]->pool, (void **)free, nb_free); + }else { +for (i =1 ; i < n; i++) { + m =rte_pktmbuf_prefree_seg (txep[i].mbuf); +if (m !=NULL ) +rte_mempool_put (m->pool, m); + } + } + +done: +/* buffers were freed, update counters */ + txq->nb_tx_free = (uint16_t )(txq->nb_tx_free + txq->tx_rs_thresh); + txq->tx_next_dd = (uint16_t )(txq->tx_next_dd + txq->tx_rs_thresh); +if (txq->tx_next_dd >= txq->nb_tx_desc) + txq->tx_next_dd = (uint16_t )(txq->tx_rs_thresh -1 ); + +return txq->tx_rs_thresh; +} +#endif /* _COMMON_INTEL_TX_H_ */
drivers/net/i40e/i40e_rxtx_vec_common.h +9 -63
81 lines not on this page · raw message
drivers/net/iavf/iavf_rxtx_vec_common.h +9 -52
70 lines not on this page · raw message
drivers/net/ice/ice_rxtx_vec_common.h +9 -52
70 lines not on this page · raw message
Signature · 1 line
2.43.0
PATCH v2 12/22 net/_common_intel: add Tx buffer free fn for AVX-512
In reply to Bruce RichardsonAVX-512 code paths for ice and i40e drivers are common, and differ from the regular post-Tx free function in that the SW ring from which the buffers are freed does not contain anything other than the mbuf pointer. Merge these into a common function in intel_common to reduce duplication.
--- drivers/net/_common_intel/tx.h | 92 +++++++++++++++++++ drivers/net/i40e/i40e_rxtx_vec_avx512.c | 114 +---------------------- drivers/net/ice/ice_rxtx_vec_avx512.c | 117 +----------------------- 3 files changed, 94 insertions(+), 229 deletions(-)
drivers/net/_common_intel/tx.h +92 -0
@@ -178,4 +178,96 @@ ci_tx_free_bufs(struct ci_tx_queue *txq, ci_desc_done_fn desc_done)return txq->tx_rs_thresh; } +static __rte_always_inline int +ci_tx_free_bufs_vec (struct ci_tx_queue *txq,ci_desc_done_fn desc_done) +{ +int nb_free =0 ; +struct rte_mbuf *free[IETH_VPMD_TX_MAX_FREE_BUF ]; +struct rte_mbuf *m; + +/* check DD bits on threshold descriptor */ +if (!desc_done (txq, txq->tx_next_dd)) +return 0 ; + +const uint32_t n = txq->tx_rs_thresh; + +/* first buffer to free from S/W ring is at index +* tx_next_dd - (tx_rs_thresh - 1) +*/ +struct ci_tx_entry_vec *txep = txq->sw_ring_vec; + txep += txq->tx_next_dd - (n -1 ); + +if (txq->offloads &RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE && (n &31 ) ==0 ) { +struct rte_mempool *mp = txep[0 ].mbuf->pool; +void **cache_objs; +struct rte_mempool_cache *cache =rte_mempool_default_cache (mp,rte_lcore_id ()); + +if (!cache || cache->len ==0 ) + goto normal; + + cache_objs = &cache->objs[cache->len]; + +if (n >RTE_MEMPOOL_CACHE_MAX_SIZE ) { +rte_mempool_ops_enqueue_bulk (mp, (void *)txep, n); + goto done; + } + +/* The cache follows the following algorithm +* 1. Add the objects to the cache +* 2. Anything greater than the cache min value (if it +* crosses the cache flush threshold) is flushed to the ring. +*/ +/* Add elements back into the cache */ +uint32_t copied =0 ; +/* n is multiple of 32 */ +while (copied < n) { +memcpy (&cache_objs[copied], &txep[copied],32 *sizeof (void *)); + copied +=32 ; + } + cache->len += n; + +if (cache->len >= cache->flushthresh) { +rte_mempool_ops_enqueue_bulk (mp, &cache->objs[cache->size], + cache->len - cache->size); + cache->len = cache->size; + } + goto done; + } + +normal: + m =rte_pktmbuf_prefree_seg (txep[0 ].mbuf); +if (likely (m)) { + free[0 ] = m; + nb_free =1 ; +for (uint32_t i =1 ; i < n; i++) { + m =rte_pktmbuf_prefree_seg (txep[i].mbuf); +if (likely (m)) { +if (likely (m->pool == free[0 ]->pool)) { + free[nb_free++] = m; + }else { +rte_mempool_put_bulk (free[0 ]->pool, (void *)free, nb_free); + free[0 ] = m; + nb_free =1 ; + } + } + } +rte_mempool_put_bulk (free[0 ]->pool, (void **)free, nb_free); + }else { +for (uint32_t i =1 ; i < n; i++) { + m =rte_pktmbuf_prefree_seg (txep[i].mbuf); +if (m) +rte_mempool_put (m->pool, m); + } + } + +done: +/* buffers were freed, update counters */ + txq->nb_tx_free = (uint16_t )(txq->nb_tx_free + txq->tx_rs_thresh); + txq->tx_next_dd = (uint16_t )(txq->tx_next_dd + txq->tx_rs_thresh); +if (txq->tx_next_dd >= txq->nb_tx_desc) + txq->tx_next_dd = (uint16_t )(txq->tx_rs_thresh -1 ); + +return txq->tx_rs_thresh; +} +#endif /* _COMMON_INTEL_TX_H_ */
drivers/net/i40e/i40e_rxtx_vec_avx512.c +1 -113
126 lines not on this page · raw message
drivers/net/ice/ice_rxtx_vec_avx512.c +1 -116
129 lines not on this page · raw message
Signature · 1 line
2.43.0
PATCH v2 13/22 net/iavf: use common Tx free fn for AVX-512
In reply to Bruce RichardsonSwitch the iavf driver to use the common Tx free function. This requires one additional parameter to that function, since iavf sometimes uses context descriptors which means that we have double the descriptors per SW ring slot.
--- drivers/net/_common_intel/tx.h | 6 +- drivers/net/i40e/i40e_rxtx_vec_avx512.c | 2 +- drivers/net/iavf/iavf_rxtx_vec_avx512.c | 119 +----------------------- drivers/net/ice/ice_rxtx_vec_avx512.c | 2 +- 4 files changed, 7 insertions(+), 122 deletions(-)
drivers/net/_common_intel/tx.h +3 -3
@@ -179,7 +179,7 @@ ci_tx_free_bufs(struct ci_tx_queue *txq, ci_desc_done_fn desc_done) } static __rte_always_inline int -ci_tx_free_bufs_vec (struct ci_tx_queue *txq,ci_desc_done_fn desc_done) +ci_tx_free_bufs_vec (struct ci_tx_queue *txq,ci_desc_done_fn desc_done,bool ctx_descs) {int nb_free =0 ;struct rte_mbuf *free[IETH_VPMD_TX_MAX_FREE_BUF ];
@@ -189,13 +189,13 @@ ci_tx_free_bufs_vec(struct ci_tx_queue *txq, ci_desc_done_fn desc_done)if (!desc_done (txq, txq->tx_next_dd))return 0 ; -const uint32_t n = txq->tx_rs_thresh; +const uint32_t n = txq->tx_rs_thresh >> ctx_descs;/* first buffer to free from S/W ring is at index * tx_next_dd - (tx_rs_thresh - 1) */ struct ci_tx_entry_vec *txep = txq->sw_ring_vec; - txep += txq->tx_next_dd - (n -1 ); + txep += (txq->tx_next_dd >> ctx_descs) - (n -1 );if (txq->offloads &RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE && (n &31 ) ==0 ) {struct rte_mempool *mp = txep[0 ].mbuf->pool;
drivers/net/i40e/i40e_rxtx_vec_avx512.c +1 -1
@@ -829,7 +829,7 @@ i40e_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts,uint64_t rs =I40E_TX_DESC_CMD_RS |I40E_TD_CMD ;if (txq->nb_tx_free < txq->tx_free_thresh) -ci_tx_free_bufs_vec (txq, i40e_tx_desc_done); +ci_tx_free_bufs_vec (txq, i40e_tx_desc_done, false); nb_commit = nb_pkts = (uint16_t )RTE_MIN (txq->nb_tx_free, nb_pkts);if (unlikely (nb_pkts ==0 ))
drivers/net/iavf/iavf_rxtx_vec_avx512.c +2 -117
137 lines not on this page · raw message
drivers/net/ice/ice_rxtx_vec_avx512.c +1 -1
@@ -949,7 +949,7 @@ ice_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, nb_pkts =RTE_MIN (nb_pkts, txq->tx_rs_thresh);if (txq->nb_tx_free < txq->tx_free_thresh) -ci_tx_free_bufs_vec (txq, ice_tx_desc_done); +ci_tx_free_bufs_vec (txq, ice_tx_desc_done, false); nb_commit = nb_pkts = (uint16_t )RTE_MIN (txq->nb_tx_free, nb_pkts);if (unlikely (nb_pkts ==0 ))
Signature · 1 line
2.43.0
PATCH v2 14/22 net/ice: move Tx queue mbuf cleanup fn to common
In reply to Bruce RichardsonThe functions to loop over the Tx queue and clean up all the mbufs on it, e.g. for queue shutdown, is not device specific and so can move into the common_intel headers. Only complication is ensuring that the correct ring format, either minimal vector or full structure, is used. Ice driver currently uses two functions and a function pointer to help with this - though actually one of those functions uses a further check inside it - so we can simplify this down to just one common function, with a flag set in the appropriate place. This avoids checking for AVX-512-specific functions, which were the only function using the smaller struct in this driver.
--- drivers/net/_common_intel/tx.h | 49 ++++++++++++++++++++++++- drivers/net/ice/ice_dcf_ethdev.c | 5 +-- drivers/net/ice/ice_ethdev.h | 3 +- drivers/net/ice/ice_rxtx.c | 33 +++++------------ drivers/net/ice/ice_rxtx_vec_common.h | 51 --------------------------- drivers/net/ice/ice_rxtx_vec_sse.c | 4 --- 6 files changed, 60 insertions(+), 85 deletions(-)
drivers/net/_common_intel/tx.h +48 -1
@@ -65,6 +65,8 @@ struct ci_tx_queue {rte_iova_t tx_ring_dma;/* TX ring DMA address */ bool tx_deferred_start;/* don't start this queue in dev start */ bool q_set;/* indicate if tx queue has been configured */ +bool vector_tx;/* port is using vector TX */ +bool vector_sw_ring;/* port is using vectorized SW ring (ieth_tx_entry_vec) */ union {/* the VSI this queue belongs to */ struct i40e_vsi *i40e_vsi;struct iavf_vsi *iavf_vsi;
@@ -74,7 +76,6 @@ struct ci_tx_queue {union {struct {/* ICE driver specific values */ -ice_tx_release_mbufs_t tx_rel_mbufs;uint32_t q_teid;/* TX schedule node id. */ };struct {/* I40E driver specific values */
@@ -270,4 +271,50 @@ ci_tx_free_bufs_vec(struct ci_tx_queue *txq, ci_desc_done_fn desc_done, bool ctxreturn txq->tx_rs_thresh; } +#define IETH_FREE_BUFS_LOOP (txq, swr, start) do { \ + uint16_t i = start; \ + if (txq->tx_tail < i) { \ + for (; i < txq->nb_tx_desc; i++) { \ + rte_pktmbuf_free_seg(swr[i].mbuf); \ + swr[i].mbuf = NULL; \ + } \ + i = 0; \ + } \ + for (; i < txq->tx_tail; i++) { \ + rte_pktmbuf_free_seg(swr[i].mbuf); \ + swr[i].mbuf = NULL; \ + } \ +} while (0) + +static inline void +ci_txq_release_all_mbufs (struct ci_tx_queue *txq) +{ +if (unlikely (!txq || !txq->sw_ring)) +return ; + +if (!txq->vector_tx) { +for (uint16_t i =0 ; i < txq->nb_tx_desc; i++) { +if (txq->sw_ring[i].mbuf !=NULL ) { +rte_pktmbuf_free_seg (txq->sw_ring[i].mbuf); + txq->sw_ring[i].mbuf =NULL ; + } + } +return ; + } + +/** +* vPMD tx will not set sw_ring's mbuf to NULL after free, +* so need to free remains more carefully. +*/ +const uint16_t start = txq->tx_next_dd - txq->tx_rs_thresh +1 ; + +if (txq->vector_sw_ring) { +struct ci_tx_entry_vec *swr = txq->sw_ring_vec; +IETH_FREE_BUFS_LOOP (txq, swr, start); + }else { +struct ci_tx_entry *swr = txq->sw_ring; +IETH_FREE_BUFS_LOOP (txq, swr, start); + } +} +#endif /* _COMMON_INTEL_TX_H_ */
drivers/net/ice/ice_dcf_ethdev.c +3 -2
@@ -24,6 +24,7 @@#include "ice_generic_flow.h" #include "ice_dcf_ethdev.h" #include "ice_rxtx.h" +#include "_common_intel/tx.h" #define DCF_NUM_MACADDR_MAX 64
@@ -500,7 +501,7 @@ ice_dcf_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id) } txq = dev->data->tx_queues[tx_queue_id]; - txq->tx_rel_mbufs (txq); +ci_txq_release_all_mbufs (txq);reset_tx_queue (txq); dev->data->tx_queue_state[tx_queue_id] =RTE_ETH_QUEUE_STATE_STOPPED ;
@@ -650,7 +651,7 @@ ice_dcf_stop_queues(struct rte_eth_dev *dev) txq = dev->data->tx_queues[i];if (!txq)continue ; - txq->tx_rel_mbufs (txq); +ci_txq_release_all_mbufs (txq);reset_tx_queue (txq); dev->data->tx_queue_state[i] =RTE_ETH_QUEUE_STATE_STOPPED ; }
drivers/net/ice/ice_ethdev.h +1 -2
@@ -621,13 +621,12 @@ struct ice_adapter {/* Set bit if the engine is disabled */ unsigned long disabled_engine_mask;struct ice_parser *psr; -#ifdef RTE_ARCH_X86 +/* used only on X86, zero on other Archs */ bool rx_use_avx2;bool rx_use_avx512;bool tx_use_avx2;bool tx_use_avx512;bool rx_vec_offload_support; -#endif };struct ice_vsi_vlan_pvid_info {
drivers/net/ice/ice_rxtx.c +8 -25
@@ -751,6 +751,7 @@ ice_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)struct ice_aqc_add_tx_qgrp *txq_elem;struct ice_tlan_ctx tx_ctx;int buf_len; +struct ice_adapter *ad =ICE_DEV_PRIVATE_TO_ADAPTER (dev->data->dev_private);PMD_INIT_FUNC_TRACE ();
@@ -822,6 +823,10 @@ ice_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)return -EIO ; } +/* record what kind of descriptor cleanup we need on teardown */ + txq->vector_tx = ad->tx_vec_allowed; + txq->vector_sw_ring = ad->tx_use_avx512; + dev->data->tx_queue_state[tx_queue_id] =RTE_ETH_QUEUE_STATE_STARTED ;rte_free (txq_elem);
@@ -1006,25 +1011,6 @@ ice_fdir_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)return 0 ; } -/* Free all mbufs for descriptors in tx queue */ -static void -_ice_tx_queue_release_mbufs (struct ci_tx_queue *txq) -{ -uint16_t i; - -if (!txq || !txq->sw_ring) { -PMD_DRV_LOG (DEBUG ,"Pointer to txq or sw_ring is NULL" ); -return ; - } - -for (i =0 ; i < txq->nb_tx_desc; i++) { -if (txq->sw_ring[i].mbuf) { -rte_pktmbuf_free_seg (txq->sw_ring[i].mbuf); - txq->sw_ring[i].mbuf =NULL ; - } - } -} -static void ice_reset_tx_queue (struct ci_tx_queue *txq) {
@@ -1103,7 +1089,7 @@ ice_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)return -EINVAL ; } - txq->tx_rel_mbufs (txq); +ci_txq_release_all_mbufs (txq);ice_reset_tx_queue (txq); dev->data->tx_queue_state[tx_queue_id] =RTE_ETH_QUEUE_STATE_STOPPED ;
@@ -1166,7 +1152,7 @@ ice_fdir_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)return -EINVAL ; } - txq->tx_rel_mbufs (txq); +ci_txq_release_all_mbufs (txq); txq->qtx_tail =NULL ;return 0 ;
@@ -1518,7 +1504,6 @@ ice_tx_queue_setup(struct rte_eth_dev *dev,ice_reset_tx_queue (txq); txq->q_set = true; dev->data->tx_queues[queue_idx] = txq; - txq->tx_rel_mbufs = _ice_tx_queue_release_mbufs;ice_set_tx_function_flag (dev, txq);return 0 ;
@@ -1546,8 +1531,7 @@ ice_tx_queue_release(void *txq)return ; } -if (q->tx_rel_mbufs !=NULL ) - q->tx_rel_mbufs (q); +ci_txq_release_all_mbufs (q);rte_free (q->sw_ring);rte_memzone_free (q->mz);rte_free (q);
@@ -2460,7 +2444,6 @@ ice_fdir_setup_tx_resources(struct ice_pf *pf) txq->q_set = true; pf->fdir.txq = txq; - txq->tx_rel_mbufs = _ice_tx_queue_release_mbufs;return ICE_SUCCESS ; }
drivers/net/ice/ice_rxtx_vec_common.h +0 -51
57 lines not on this page · raw message
drivers/net/ice/ice_rxtx_vec_sse.c +0 -4
@@ -795,10 +795,6 @@ ice_rxq_vec_setup(struct ice_rx_queue *rxq) int __rte_cold ice_txq_vec_setup(struct ci_tx_queue *txq __rte_unused) { -if (!txq) -return -1 ; - - txq->tx_rel_mbufs = _ice_tx_queue_release_mbufs_vec;return 0 ; }
Signature · 1 line
2.43.0
PATCH v2 15/22 net/i40e: use common Tx queue mbuf cleanup fn
In reply to Bruce RichardsonUpdate driver to be similar to the "ice" driver and use the common mbuf ring cleanup code on shutdown of a Tx queue.
--- drivers/net/i40e/i40e_ethdev.h | 4 +- drivers/net/i40e/i40e_rxtx.c | 70 ++++------------------------------ drivers/net/i40e/i40e_rxtx.h | 1 - 3 files changed, 9 insertions(+), 66 deletions(-)
drivers/net/i40e/i40e_ethdev.h +2 -2
@@ -1260,12 +1260,12 @@ struct i40e_adapter {/* For RSS reta table update */ uint8_t rss_reta_updated; -#ifdef RTE_ARCH_X86 + +/* used only on x86, zero on other architectures */ bool rx_use_avx2;bool rx_use_avx512;bool tx_use_avx2;bool tx_use_avx512; -#endif };/**
drivers/net/i40e/i40e_rxtx.c +7 -63
106 lines not on this page · raw message
drivers/net/i40e/i40e_rxtx.h +0 -1
@@ -179,7 +179,6 @@ void i40e_dev_clear_queues(struct rte_eth_dev *dev);void i40e_dev_free_queues (struct rte_eth_dev *dev);void i40e_reset_rx_queue (struct i40e_rx_queue *rxq);void i40e_reset_tx_queue (struct ci_tx_queue *txq); -void i40e_tx_queue_release_mbufs (struct ci_tx_queue *txq);int i40e_tx_done_cleanup (void *txq,uint32_t free_cnt);int i40e_alloc_rx_queue_mbufs (struct i40e_rx_queue *rxq);void i40e_rx_queue_release_mbufs (struct i40e_rx_queue *rxq);
Signature · 1 line
2.43.0
PATCH v2 16/22 net/ixgbe: use common Tx queue mbuf cleanup fn
In reply to Bruce RichardsonUpdate driver to use the common cleanup function.
--- drivers/net/ixgbe/ixgbe_rxtx.c | 22 +++--------------- drivers/net/ixgbe/ixgbe_rxtx.h | 1 - drivers/net/ixgbe/ixgbe_rxtx_vec_common.h | 28 ++--------------------- drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c | 7 ------ drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c | 7 ------ 5 files changed, 5 insertions(+), 60 deletions(-)
drivers/net/ixgbe/ixgbe_rxtx.c +3 -19
@@ -2334,21 +2334,6 @@ ixgbe_recv_pkts_lro_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts,* **********************************************************************/ -static void __rte_cold -ixgbe_tx_queue_release_mbufs (struct ci_tx_queue *txq) -{ -unsigned i; - -if (txq->sw_ring !=NULL ) { -for (i =0 ; i < txq->nb_tx_desc; i++) { -if (txq->sw_ring[i].mbuf !=NULL ) { -rte_pktmbuf_free_seg (txq->sw_ring[i].mbuf); - txq->sw_ring[i].mbuf =NULL ; - } - } - } -} -static int ixgbe_tx_done_cleanup_full (struct ci_tx_queue *txq,uint32_t free_cnt) {
@@ -2472,7 +2457,7 @@ static void __rte_cold ixgbe_tx_queue_release(struct ci_tx_queue *txq) {if (txq !=NULL && txq->ops !=NULL ) { - txq->ops->release_mbufs (txq); +ci_txq_release_all_mbufs (txq); txq->ops->free_swring (txq);rte_memzone_free (txq->mz);rte_free (txq);
@@ -2526,7 +2511,6 @@ ixgbe_reset_tx_queue(struct ci_tx_queue *txq) }static const struct ixgbe_txq_ops def_txq_ops = { - .release_mbufs = ixgbe_tx_queue_release_mbufs, .free_swring = ixgbe_tx_free_swring, .reset = ixgbe_reset_tx_queue, };
@@ -3380,7 +3364,7 @@ ixgbe_dev_clear_queues(struct rte_eth_dev *dev)struct ci_tx_queue *txq = dev->data->tx_queues[i];if (txq !=NULL ) { - txq->ops->release_mbufs (txq); +ci_txq_release_all_mbufs (txq); txq->ops->reset (txq); dev->data->tx_queue_state[i] =RTE_ETH_QUEUE_STATE_STOPPED ; }
@@ -5655,7 +5639,7 @@ ixgbe_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id) }if (txq->ops !=NULL ) { - txq->ops->release_mbufs (txq); +ci_txq_release_all_mbufs (txq); txq->ops->reset (txq); } dev->data->tx_queue_state[tx_queue_id] =RTE_ETH_QUEUE_STATE_STOPPED ;
drivers/net/ixgbe/ixgbe_rxtx.h +0 -1
@@ -181,7 +181,6 @@ struct ixgbe_advctx_info { };struct ixgbe_txq_ops { -void (*release_mbufs)(struct ci_tx_queue *txq);void (*free_swring)(struct ci_tx_queue *txq);void (*reset)(struct ci_tx_queue *txq); };
drivers/net/ixgbe/ixgbe_rxtx_vec_common.h +2 -26
@@ -78,32 +78,6 @@ tx_backlog_entry(struct ci_tx_entry_vec *txep, txep[i].mbuf = tx_pkts[i]; } -static inline void -_ixgbe_tx_queue_release_mbufs_vec (struct ci_tx_queue *txq) -{ -unsigned int i; -struct ci_tx_entry_vec *txe; -const uint16_t max_desc = (uint16_t )(txq->nb_tx_desc -1 ); - -if (txq->sw_ring ==NULL || txq->nb_tx_free == max_desc) -return ; - -/* release the used mbufs in sw_ring */ -for (i = txq->tx_next_dd - (txq->tx_rs_thresh -1 ); - i != txq->tx_tail; - i = (i +1 ) % txq->nb_tx_desc) { - txe = &txq->sw_ring_vec[i]; -rte_pktmbuf_free_seg (txe->mbuf); - } - txq->nb_tx_free = max_desc; - -/* reset tx_entry */ -for (i =0 ; i < txq->nb_tx_desc; i++) { - txe = &txq->sw_ring_vec[i]; - txe->mbuf =NULL ; - } -} -static inline void _ixgbe_rx_queue_release_mbufs_vec (struct ixgbe_rx_queue *rxq) {
@@ -208,6 +182,8 @@ ixgbe_txq_vec_setup_default(struct ci_tx_queue *txq,/* leave the first one for overflow */ txq->sw_ring_vec = txq->sw_ring_vec +1 ; txq->ops = txq_ops; + txq->vector_tx =1 ; + txq->vector_sw_ring =1 ;return 0 ; }
drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c +0 -7
@@ -633,12 +633,6 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,return nb_pkts; } -static void __rte_cold -ixgbe_tx_queue_release_mbufs_vec (struct ci_tx_queue *txq) -{ -_ixgbe_tx_queue_release_mbufs_vec (txq); -} - void __rte_coldixgbe_rx_queue_release_mbufs_vec (struct ixgbe_rx_queue *rxq) {
@@ -658,7 +652,6 @@ ixgbe_reset_tx_queue(struct ci_tx_queue *txq) }static const struct ixgbe_txq_ops vec_txq_ops = { - .release_mbufs = ixgbe_tx_queue_release_mbufs_vec, .free_swring = ixgbe_tx_free_swring, .reset = ixgbe_reset_tx_queue, };
drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c +0 -7
@@ -756,12 +756,6 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,return nb_pkts; } -static void __rte_cold -ixgbe_tx_queue_release_mbufs_vec (struct ci_tx_queue *txq) -{ -_ixgbe_tx_queue_release_mbufs_vec (txq); -} - void __rte_coldixgbe_rx_queue_release_mbufs_vec (struct ixgbe_rx_queue *rxq) {
@@ -781,7 +775,6 @@ ixgbe_reset_tx_queue(struct ci_tx_queue *txq) }static const struct ixgbe_txq_ops vec_txq_ops = { - .release_mbufs = ixgbe_tx_queue_release_mbufs_vec, .free_swring = ixgbe_tx_free_swring, .reset = ixgbe_reset_tx_queue, };
Signature · 1 line
2.43.0
PATCH v2 17/22 net/iavf: use common Tx queue mbuf cleanup fn
In reply to Bruce RichardsonAdjust iavf driver to also use the common mbuf freeing functions on Tx queue release/cleanup. The implementation is complicated a little by the need to integrate the additional "has_ctx" parameter for the iavf code, but changes in other drivers are minimal - just a constant "false" parameter.
--- drivers/net/_common_intel/tx.h | 27 +++++++++--------- drivers/net/i40e/i40e_rxtx.c | 6 ++-- drivers/net/iavf/iavf_rxtx.c | 37 ++----------------------- drivers/net/iavf/iavf_rxtx_vec_avx512.c | 24 ++-------------- drivers/net/iavf/iavf_rxtx_vec_common.h | 18 ------------ drivers/net/iavf/iavf_rxtx_vec_sse.c | 9 ++---- drivers/net/ice/ice_dcf_ethdev.c | 4 +-- drivers/net/ice/ice_rxtx.c | 6 ++-- drivers/net/ixgbe/ixgbe_rxtx.c | 6 ++-- 9 files changed, 31 insertions(+), 106 deletions(-)
drivers/net/_common_intel/tx.h +13 -14
@@ -271,23 +271,23 @@ ci_tx_free_bufs_vec(struct ci_tx_queue *txq, ci_desc_done_fn desc_done, bool ctxreturn txq->tx_rs_thresh; } -#define IETH_FREE_BUFS_LOOP (txq, swr, start) do { \ +#define IETH_FREE_BUFS_LOOP (swr, nb_desc, start, end) do { \ uint16_t i = start; \ - if (txq->tx_tail < i) { \ - for (; i < txq->nb_tx_desc; i++) { \ + if (end < i) { \ + for (; i < nb_desc; i++) { \ rte_pktmbuf_free_seg(swr[i].mbuf); \ swr[i].mbuf = NULL; \ } \ i = 0; \ } \ - for (; i < txq->tx_tail; i++) { \ + for (; i < end; i++) { \ rte_pktmbuf_free_seg(swr[i].mbuf); \ swr[i].mbuf = NULL; \ } \ } while (0)static inline void -ci_txq_release_all_mbufs (struct ci_tx_queue *txq) +ci_txq_release_all_mbufs (struct ci_tx_queue *txq,bool use_ctx) {if (unlikely (!txq || !txq->sw_ring))return ;
@@ -306,15 +306,14 @@ ci_txq_release_all_mbufs(struct ci_tx_queue *txq)* vPMD tx will not set sw_ring's mbuf to NULL after free, * so need to free remains more carefully. */ -const uint16_t start = txq->tx_next_dd - txq->tx_rs_thresh +1 ; - -if (txq->vector_sw_ring) { -struct ci_tx_entry_vec *swr = txq->sw_ring_vec; -IETH_FREE_BUFS_LOOP (txq, swr, start); - }else { -struct ci_tx_entry *swr = txq->sw_ring; -IETH_FREE_BUFS_LOOP (txq, swr, start); - } +const uint16_t start = (txq->tx_next_dd - txq->tx_rs_thresh +1 ) >> use_ctx; +const uint16_t nb_desc = txq->nb_tx_desc >> use_ctx; +const uint16_t end = txq->tx_tail >> use_ctx; + +if (txq->vector_sw_ring) +IETH_FREE_BUFS_LOOP (txq->sw_ring_vec, nb_desc, start, end); +else + IETH_FREE_BUFS_LOOP(txq->sw_ring, nb_desc, start, end); }#endif /* _COMMON_INTEL_TX_H_ */
drivers/net/i40e/i40e_rxtx.c +3 -3
@@ -1933,7 +1933,7 @@ i40e_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)return err; } -ci_txq_release_all_mbufs (txq); +ci_txq_release_all_mbufs (txq, false);i40e_reset_tx_queue (txq); dev->data->tx_queue_state[tx_queue_id] =RTE_ETH_QUEUE_STATE_STOPPED ;
@@ -2608,7 +2608,7 @@ i40e_tx_queue_release(void *txq)return ; } -ci_txq_release_all_mbufs (q); +ci_txq_release_all_mbufs (q, false);rte_free (q->sw_ring);rte_memzone_free (q->mz);rte_free (q);
@@ -3071,7 +3071,7 @@ i40e_dev_clear_queues(struct rte_eth_dev *dev)for (i =0 ; i < dev->data->nb_tx_queues; i++) {if (!dev->data->tx_queues[i])continue ; -ci_txq_release_all_mbufs (dev->data->tx_queues[i]); +ci_txq_release_all_mbufs (dev->data->tx_queues[i], false);i40e_reset_tx_queue (dev->data->tx_queues[i]); }
drivers/net/iavf/iavf_rxtx.c +3 -34
@@ -387,24 +387,6 @@ release_rxq_mbufs(struct iavf_rx_queue *rxq) rxq->rx_nb_avail =0 ; } -static inline void -release_txq_mbufs (struct ci_tx_queue *txq) -{ -uint16_t i; - -if (!txq || !txq->sw_ring) { -PMD_DRV_LOG (DEBUG ,"Pointer to rxq or sw_ring is NULL" ); -return ; - } - -for (i =0 ; i < txq->nb_tx_desc; i++) { -if (txq->sw_ring[i].mbuf) { -rte_pktmbuf_free_seg (txq->sw_ring[i].mbuf); - txq->sw_ring[i].mbuf =NULL ; - } - } -} -static const struct iavf_rxq_ops iavf_rxq_release_mbufs_ops[] = { [IAVF_REL_MBUFS_DEFAULT ].release_mbufs = release_rxq_mbufs,
@@ -413,18 +395,6 @@ struct iavf_rxq_ops iavf_rxq_release_mbufs_ops[] = {#endif }; -static const -struct iavf_txq_ops iavf_txq_release_mbufs_ops[] = { - [IAVF_REL_MBUFS_DEFAULT ].release_mbufs = release_txq_mbufs, -#ifdef RTE_ARCH_X86 - [IAVF_REL_MBUFS_SSE_VEC ].release_mbufs = iavf_tx_queue_release_mbufs_sse, -#ifdef CC_AVX512_SUPPORT - [IAVF_REL_MBUFS_AVX512_VEC ].release_mbufs = iavf_tx_queue_release_mbufs_avx512, -#endif -#endif - -}; - static inline void iavf_rxd_to_pkt_fields_by_comms_ovs(__rte_unused struct iavf_rx_queue *rxq, struct rte_mbuf *mb,
@@ -889,7 +859,6 @@ iavf_dev_tx_queue_setup(struct rte_eth_dev *dev, txq->q_set = true; dev->data->tx_queues[queue_idx] = txq; txq->qtx_tail = hw->hw_addr +IAVF_QTX_TAIL1 (queue_idx); - txq->rel_mbufs_type =IAVF_REL_MBUFS_DEFAULT ;if (check_tx_vec_allow (txq) == false) { struct iavf_adapter *ad =
@@ -1068,7 +1037,7 @@ iavf_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id) } txq = dev->data->tx_queues[tx_queue_id]; - iavf_txq_release_mbufs_ops[txq->rel_mbufs_type].release_mbufs (txq); +ci_txq_release_all_mbufs (txq, txq->use_ctx);reset_tx_queue (txq); dev->data->tx_queue_state[tx_queue_id] =RTE_ETH_QUEUE_STATE_STOPPED ;
@@ -1097,7 +1066,7 @@ iavf_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid)if (!q)return ; - iavf_txq_release_mbufs_ops[q->rel_mbufs_type].release_mbufs (q); +ci_txq_release_all_mbufs (q, q->use_ctx);rte_free (q->sw_ring);rte_memzone_free (q->mz);rte_free (q);
@@ -1114,7 +1083,7 @@ iavf_reset_queues(struct rte_eth_dev *dev) txq = dev->data->tx_queues[i];if (!txq)continue ; - iavf_txq_release_mbufs_ops[txq->rel_mbufs_type].release_mbufs (txq); +ci_txq_release_all_mbufs (txq, txq->use_ctx);reset_tx_queue (txq); dev->data->tx_queue_state[i] =RTE_ETH_QUEUE_STATE_STOPPED ; }
drivers/net/iavf/iavf_rxtx_vec_avx512.c +2 -22
@@ -2357,31 +2357,11 @@ iavf_xmit_pkts_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts,return iavf_xmit_pkts_vec_avx512_cmn (tx_queue, tx_pkts, nb_pkts, false); } -void __rte_cold -iavf_tx_queue_release_mbufs_avx512 (struct ci_tx_queue *txq) -{ -unsigned int i; -const uint16_t max_desc = (uint16_t )(txq->nb_tx_desc -1 ); -const uint16_t end_desc = txq->tx_tail >> txq->use_ctx;/* next empty slot */ -const uint16_t wrap_point = txq->nb_tx_desc >> txq->use_ctx;/* end of SW ring */ -struct ci_tx_entry_vec *swr = (void *)txq->sw_ring; - -if (!txq->sw_ring || txq->nb_tx_free == max_desc) -return ; - - i = (txq->tx_next_dd - txq->tx_rs_thresh +1 ) >> txq->use_ctx; -while (i != end_desc) { -rte_pktmbuf_free_seg (swr[i].mbuf); - swr[i].mbuf =NULL ; -if (++i == wrap_point) - i =0 ; - } -} - int __rte_coldiavf_txq_vec_setup_avx512 (struct ci_tx_queue *txq) { - txq->rel_mbufs_type =IAVF_REL_MBUFS_AVX512_VEC ; + txq->vector_tx = true; + txq->vector_sw_ring = true;return 0 ; }
drivers/net/iavf/iavf_rxtx_vec_common.h +0 -18
@@ -60,24 +60,6 @@ _iavf_rx_queue_release_mbufs_vec(struct iavf_rx_queue *rxq)memset (rxq->sw_ring,0 ,sizeof (rxq->sw_ring[0 ]) * rxq->nb_rx_desc); } -static inline void -_iavf_tx_queue_release_mbufs_vec (struct ci_tx_queue *txq) -{ -unsigned i; -const uint16_t max_desc = (uint16_t )(txq->nb_tx_desc -1 ); - -if (!txq->sw_ring || txq->nb_tx_free == max_desc) -return ; - - i = txq->tx_next_dd - txq->tx_rs_thresh +1 ; -while (i != txq->tx_tail) { -rte_pktmbuf_free_seg (txq->sw_ring[i].mbuf); - txq->sw_ring[i].mbuf =NULL ; -if (++i == txq->nb_tx_desc) - i =0 ; - } -} -static inline int iavf_rxq_vec_setup_default (struct iavf_rx_queue *rxq) {
drivers/net/iavf/iavf_rxtx_vec_sse.c +2 -7
@@ -1458,16 +1458,11 @@ iavf_rx_queue_release_mbufs_sse(struct iavf_rx_queue *rxq)_iavf_rx_queue_release_mbufs_vec (rxq); } -void __rte_cold -iavf_tx_queue_release_mbufs_sse (struct ci_tx_queue *txq) -{ -_iavf_tx_queue_release_mbufs_vec (txq); -} - int __rte_coldiavf_txq_vec_setup (struct ci_tx_queue *txq) { - txq->rel_mbufs_type =IAVF_REL_MBUFS_SSE_VEC ; + txq->vector_tx = true; + txq->vector_sw_ring = false;return 0 ; }
drivers/net/ice/ice_dcf_ethdev.c +2 -2
@@ -501,7 +501,7 @@ ice_dcf_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id) } txq = dev->data->tx_queues[tx_queue_id]; -ci_txq_release_all_mbufs (txq); +ci_txq_release_all_mbufs (txq, false);reset_tx_queue (txq); dev->data->tx_queue_state[tx_queue_id] =RTE_ETH_QUEUE_STATE_STOPPED ;
@@ -651,7 +651,7 @@ ice_dcf_stop_queues(struct rte_eth_dev *dev) txq = dev->data->tx_queues[i];if (!txq)continue ; -ci_txq_release_all_mbufs (txq); +ci_txq_release_all_mbufs (txq, false);reset_tx_queue (txq); dev->data->tx_queue_state[i] =RTE_ETH_QUEUE_STATE_STOPPED ; }
drivers/net/ice/ice_rxtx.c +3 -3
@@ -1089,7 +1089,7 @@ ice_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)return -EINVAL ; } -ci_txq_release_all_mbufs (txq); +ci_txq_release_all_mbufs (txq, false);ice_reset_tx_queue (txq); dev->data->tx_queue_state[tx_queue_id] =RTE_ETH_QUEUE_STATE_STOPPED ;
@@ -1152,7 +1152,7 @@ ice_fdir_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)return -EINVAL ; } -ci_txq_release_all_mbufs (txq); +ci_txq_release_all_mbufs (txq, false); txq->qtx_tail =NULL ;return 0 ;
@@ -1531,7 +1531,7 @@ ice_tx_queue_release(void *txq)return ; } -ci_txq_release_all_mbufs (q); +ci_txq_release_all_mbufs (q, false);rte_free (q->sw_ring);rte_memzone_free (q->mz);rte_free (q);
drivers/net/ixgbe/ixgbe_rxtx.c +3 -3
@@ -2457,7 +2457,7 @@ static void __rte_cold ixgbe_tx_queue_release(struct ci_tx_queue *txq) {if (txq !=NULL && txq->ops !=NULL ) { -ci_txq_release_all_mbufs (txq); +ci_txq_release_all_mbufs (txq, false); txq->ops->free_swring (txq);rte_memzone_free (txq->mz);rte_free (txq);
@@ -3364,7 +3364,7 @@ ixgbe_dev_clear_queues(struct rte_eth_dev *dev)struct ci_tx_queue *txq = dev->data->tx_queues[i];if (txq !=NULL ) { -ci_txq_release_all_mbufs (txq); +ci_txq_release_all_mbufs (txq, false); txq->ops->reset (txq); dev->data->tx_queue_state[i] =RTE_ETH_QUEUE_STATE_STOPPED ; }
@@ -5639,7 +5639,7 @@ ixgbe_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id) }if (txq->ops !=NULL ) { -ci_txq_release_all_mbufs (txq); +ci_txq_release_all_mbufs (txq, false); txq->ops->reset (txq); } dev->data->tx_queue_state[tx_queue_id] =RTE_ETH_QUEUE_STATE_STOPPED ;
Signature · 1 line
2.43.0
PATCH v2 18/22 net/ice: use vector SW ring for all vector paths
In reply to Bruce RichardsonThe AVX-512 code path used a smaller SW ring structure only containing the mbuf pointer, but no other fields. The other fields are only used in the scalar code path, so update all vector driver code paths to use the smaller, faster structure.
--- drivers/net/_common_intel/tx.h | 7 +++++++ drivers/net/ice/ice_rxtx.c | 2 +- drivers/net/ice/ice_rxtx_vec_avx2.c | 12 ++++++------ drivers/net/ice/ice_rxtx_vec_avx512.c | 14 ++------------ drivers/net/ice/ice_rxtx_vec_common.h | 6 ------ drivers/net/ice/ice_rxtx_vec_sse.c | 12 ++++++------ 6 files changed, 22 insertions(+), 31 deletions(-)
drivers/net/_common_intel/tx.h +7 -0
@@ -109,6 +109,13 @@ ci_tx_backlog_entry(struct ci_tx_entry *txep, struct rte_mbuf **tx_pkts, uint16_ txep[i].mbuf = tx_pkts[i]; } +static __rte_always_inline void +ci_tx_backlog_entry_vec (struct ci_tx_entry_vec *txep,struct rte_mbuf **tx_pkts,uint16_t nb_pkts) +{ +for (uint16_t i =0 ; i < nb_pkts; ++i) + txep[i].mbuf = tx_pkts[i]; +} +#define IETH_VPMD_TX_MAX_FREE_BUF 64typedef int (*ci_desc_done_fn )(struct ci_tx_queue *txq,uint16_t idx);
drivers/net/ice/ice_rxtx.c +1 -1
@@ -825,7 +825,7 @@ ice_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)/* record what kind of descriptor cleanup we need on teardown */ txq->vector_tx = ad->tx_vec_allowed; - txq->vector_sw_ring = ad->tx_use_avx512; + txq->vector_sw_ring = txq->vector_tx; dev->data->tx_queue_state[tx_queue_id] =RTE_ETH_QUEUE_STATE_STARTED ;
drivers/net/ice/ice_rxtx_vec_avx2.c +6 -6
@@ -858,7 +858,7 @@ ice_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, {struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile struct ice_tx_desc *txdp; -struct ci_tx_entry *txep; +struct ci_tx_entry_vec *txep;uint16_t n, nb_commit, tx_id;uint64_t flags =ICE_TD_CMD ;uint64_t rs =ICE_TX_DESC_CMD_RS |ICE_TD_CMD ;
@@ -867,7 +867,7 @@ ice_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, nb_pkts =RTE_MIN (nb_pkts, txq->tx_rs_thresh);if (txq->nb_tx_free < txq->tx_free_thresh) -ice_tx_free_bufs_vec (txq); +ci_tx_free_bufs_vec (txq, ice_tx_desc_done, false); nb_commit = nb_pkts = (uint16_t )RTE_MIN (txq->nb_tx_free, nb_pkts);if (unlikely (nb_pkts ==0 ))
@@ -875,13 +875,13 @@ ice_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = txq->tx_tail; txdp = &txq->ice_tx_ring[tx_id]; - txep = &txq->sw_ring[tx_id]; + txep = &txq->sw_ring_vec[tx_id]; txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts); n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) { -ci_tx_backlog_entry (txep, tx_pkts, n); +ci_tx_backlog_entry_vec (txep, tx_pkts, n);ice_vtx (txdp, tx_pkts, n -1 , flags, offload); tx_pkts += (n -1 );
@@ -896,10 +896,10 @@ ice_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts,/* avoid reach the end of ring */ txdp = &txq->ice_tx_ring[tx_id]; - txep = &txq->sw_ring[tx_id]; + txep = &txq->sw_ring_vec[tx_id]; } -ci_tx_backlog_entry (txep, tx_pkts, nb_commit); +ci_tx_backlog_entry_vec (txep, tx_pkts, nb_commit);ice_vtx (txdp, tx_pkts, nb_commit, flags, offload);
drivers/net/ice/ice_rxtx_vec_avx512.c +2 -12
@@ -924,16 +924,6 @@ ice_vtx(volatile struct ice_tx_desc *txdp, struct rte_mbuf **pkt, } } -static __rte_always_inline void -ice_tx_backlog_entry_avx512 (struct ci_tx_entry_vec *txep, -struct rte_mbuf **tx_pkts,uint16_t nb_pkts) -{ -int i; - -for (i =0 ; i < (int )nb_pkts; ++i) - txep[i].mbuf = tx_pkts[i]; -} - static __rte_always_inline uint16_tice_xmit_fixed_burst_vec_avx512 (void *tx_queue,struct rte_mbuf **tx_pkts,uint16_t nb_pkts,bool do_offload)
@@ -964,7 +954,7 @@ ice_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) { -ice_tx_backlog_entry_avx512 (txep, tx_pkts, n); +ci_tx_backlog_entry_vec (txep, tx_pkts, n);ice_vtx (txdp, tx_pkts, n -1 , flags, do_offload); tx_pkts += (n -1 );
@@ -982,7 +972,7 @@ ice_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, txep = (void *)txq->sw_ring; } -ice_tx_backlog_entry_avx512 (txep, tx_pkts, nb_commit); +ci_tx_backlog_entry_vec (txep, tx_pkts, nb_commit);ice_vtx (txdp, tx_pkts, nb_commit, flags, do_offload);
drivers/net/ice/ice_rxtx_vec_common.h +0 -6
@@ -20,12 +20,6 @@ ice_tx_desc_done(struct ci_tx_queue *txq, uint16_t idx)rte_cpu_to_le_64 (ICE_TX_DESC_DTYPE_DESC_DONE ); } -static __rte_always_inline int -ice_tx_free_bufs_vec (struct ci_tx_queue *txq) -{ -return ci_tx_free_bufs (txq, ice_tx_desc_done); -} -static inline void _ice_rx_queue_release_mbufs_vec (struct ice_rx_queue *rxq) {
drivers/net/ice/ice_rxtx_vec_sse.c +6 -6
@@ -699,7 +699,7 @@ ice_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, {struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile struct ice_tx_desc *txdp; -struct ci_tx_entry *txep; +struct ci_tx_entry_vec *txep;uint16_t n, nb_commit, tx_id;uint64_t flags =ICE_TD_CMD ;uint64_t rs =ICE_TX_DESC_CMD_RS |ICE_TD_CMD ;
@@ -709,7 +709,7 @@ ice_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, nb_pkts =RTE_MIN (nb_pkts, txq->tx_rs_thresh);if (txq->nb_tx_free < txq->tx_free_thresh) -ice_tx_free_bufs_vec (txq); +ci_tx_free_bufs_vec (txq, ice_tx_desc_done, false); nb_pkts = (uint16_t )RTE_MIN (txq->nb_tx_free, nb_pkts); nb_commit = nb_pkts;
@@ -718,13 +718,13 @@ ice_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = txq->tx_tail; txdp = &txq->ice_tx_ring[tx_id]; - txep = &txq->sw_ring[tx_id]; + txep = &txq->sw_ring_vec[tx_id]; txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts); n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) { -ci_tx_backlog_entry (txep, tx_pkts, n); +ci_tx_backlog_entry_vec (txep, tx_pkts, n);for (i =0 ; i < n -1 ; ++i, ++tx_pkts, ++txdp)ice_vtx1 (txdp, *tx_pkts, flags);
@@ -738,10 +738,10 @@ ice_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,/* avoid reach the end of ring */ txdp = &txq->ice_tx_ring[tx_id]; - txep = &txq->sw_ring[tx_id]; + txep = &txq->sw_ring_vec[tx_id]; } -ci_tx_backlog_entry (txep, tx_pkts, nb_commit); +ci_tx_backlog_entry_vec (txep, tx_pkts, nb_commit);ice_vtx (txdp, tx_pkts, nb_commit, flags);
Signature · 1 line
2.43.0
PATCH v2 19/22 net/i40e: use vector SW ring for all vector paths
In reply to Bruce RichardsonThe AVX-512 code path used a smaller SW ring structure only containing the mbuf pointer, but no other fields. The other fields are only used in the scalar code path, so update all vector driver code paths (AVX2, SSE, Neon, Altivec) to use the smaller, faster structure.
--- drivers/net/i40e/i40e_rxtx.c | 8 +++++--- drivers/net/i40e/i40e_rxtx_vec_altivec.c | 12 ++++++------ drivers/net/i40e/i40e_rxtx_vec_avx2.c | 12 ++++++------ drivers/net/i40e/i40e_rxtx_vec_avx512.c | 14 ++------------ drivers/net/i40e/i40e_rxtx_vec_common.h | 6 ------ drivers/net/i40e/i40e_rxtx_vec_neon.c | 12 ++++++------ drivers/net/i40e/i40e_rxtx_vec_sse.c | 12 ++++++------ 7 files changed, 31 insertions(+), 45 deletions(-)
drivers/net/i40e/i40e_rxtx.c +5 -3
@@ -1891,7 +1891,7 @@ i40e_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id) tx_queue_id); txq->vector_tx = ad->tx_vec_allowed; - txq->vector_sw_ring = ad->tx_use_avx512; + txq->vector_sw_ring = txq->vector_tx;/* * tx_queue_id is queue id application refers to, while
@@ -3550,9 +3550,11 @@ i40e_set_tx_function(struct rte_eth_dev *dev) } } +if (rte_vect_get_max_simd_bitwidth () <RTE_VECT_SIMD_128 ) + ad->tx_vec_allowed = false; +if (ad->tx_simple_allowed) { -if (ad->tx_vec_allowed && -rte_vect_get_max_simd_bitwidth () >=RTE_VECT_SIMD_128 ) { +if (ad->tx_vec_allowed) {#ifdef RTE_ARCH_X86 if (ad->tx_use_avx512) {#ifdef CC_AVX512_SUPPORT
drivers/net/i40e/i40e_rxtx_vec_altivec.c +6 -6
@@ -553,14 +553,14 @@ i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, {struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile struct i40e_tx_desc *txdp; -struct ci_tx_entry *txep; +struct ci_tx_entry_vec *txep;uint16_t n, nb_commit, tx_id;uint64_t flags =I40E_TD_CMD ;uint64_t rs =I40E_TX_DESC_CMD_RS |I40E_TD_CMD ;int i;if (txq->nb_tx_free < txq->tx_free_thresh) -i40e_tx_free_bufs (txq); +ci_tx_free_bufs_vec (txq, i40e_tx_desc_done, false); nb_pkts = (uint16_t )RTE_MIN (txq->nb_tx_free, nb_pkts); nb_commit = nb_pkts;
@@ -569,13 +569,13 @@ i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = txq->tx_tail; txdp = &txq->i40e_tx_ring[tx_id]; - txep = &txq->sw_ring[tx_id]; + txep = &txq->sw_ring_vec[tx_id]; txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts); n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) { -ci_tx_backlog_entry (txep, tx_pkts, n); +ci_tx_backlog_entry_vec (txep, tx_pkts, n);for (i =0 ; i < n -1 ; ++i, ++tx_pkts, ++txdp)vtx1 (txdp, *tx_pkts, flags);
@@ -589,10 +589,10 @@ i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,/* avoid reach the end of ring */ txdp = &txq->i40e_tx_ring[tx_id]; - txep = &txq->sw_ring[tx_id]; + txep = &txq->sw_ring_vec[tx_id]; } -ci_tx_backlog_entry (txep, tx_pkts, nb_commit); +ci_tx_backlog_entry_vec (txep, tx_pkts, nb_commit);vtx (txdp, tx_pkts, nb_commit, flags);
drivers/net/i40e/i40e_rxtx_vec_avx2.c +6 -6
@@ -745,13 +745,13 @@ i40e_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, {struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile struct i40e_tx_desc *txdp; -struct ci_tx_entry *txep; +struct ci_tx_entry_vec *txep;uint16_t n, nb_commit, tx_id;uint64_t flags =I40E_TD_CMD ;uint64_t rs =I40E_TX_DESC_CMD_RS |I40E_TD_CMD ;if (txq->nb_tx_free < txq->tx_free_thresh) -i40e_tx_free_bufs (txq); +ci_tx_free_bufs_vec (txq, i40e_tx_desc_done, false); nb_commit = nb_pkts = (uint16_t )RTE_MIN (txq->nb_tx_free, nb_pkts);if (unlikely (nb_pkts ==0 ))
@@ -759,13 +759,13 @@ i40e_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = txq->tx_tail; txdp = &txq->i40e_tx_ring[tx_id]; - txep = &txq->sw_ring[tx_id]; + txep = &txq->sw_ring_vec[tx_id]; txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts); n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) { -ci_tx_backlog_entry (txep, tx_pkts, n); +ci_tx_backlog_entry_vec (txep, tx_pkts, n);vtx (txdp, tx_pkts, n -1 , flags); tx_pkts += (n -1 );
@@ -780,10 +780,10 @@ i40e_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts,/* avoid reach the end of ring */ txdp = &txq->i40e_tx_ring[tx_id]; - txep = &txq->sw_ring[tx_id]; + txep = &txq->sw_ring_vec[tx_id]; } -ci_tx_backlog_entry (txep, tx_pkts, nb_commit); +ci_tx_backlog_entry_vec (txep, tx_pkts, nb_commit);vtx (txdp, tx_pkts, nb_commit, flags);
drivers/net/i40e/i40e_rxtx_vec_avx512.c +2 -12
@@ -807,16 +807,6 @@ vtx(volatile struct i40e_tx_desc *txdp, } } -static __rte_always_inline void -tx_backlog_entry_avx512 (struct ci_tx_entry_vec *txep, -struct rte_mbuf **tx_pkts,uint16_t nb_pkts) -{ -int i; - -for (i =0 ; i < (int )nb_pkts; ++i) - txep[i].mbuf = tx_pkts[i]; -} -static inline uint16_t i40e_xmit_fixed_burst_vec_avx512 (void *tx_queue,struct rte_mbuf **tx_pkts,uint16_t nb_pkts)
@@ -844,7 +834,7 @@ i40e_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) { -tx_backlog_entry_avx512 (txep, tx_pkts, n); +ci_tx_backlog_entry_vec (txep, tx_pkts, n);vtx (txdp, tx_pkts, n -1 , flags); tx_pkts += (n -1 );
@@ -862,7 +852,7 @@ i40e_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, txep = (void *)txq->sw_ring; } -tx_backlog_entry_avx512 (txep, tx_pkts, nb_commit); +ci_tx_backlog_entry_vec (txep, tx_pkts, nb_commit);vtx (txdp, tx_pkts, nb_commit, flags);
drivers/net/i40e/i40e_rxtx_vec_common.h +0 -6
@@ -24,12 +24,6 @@ i40e_tx_desc_done(struct ci_tx_queue *txq, uint16_t idx)rte_cpu_to_le_64 (I40E_TX_DESC_DTYPE_DESC_DONE ); } -static __rte_always_inline int -i40e_tx_free_bufs (struct ci_tx_queue *txq) -{ -return ci_tx_free_bufs (txq, i40e_tx_desc_done); -} -static inline void _i40e_rx_queue_release_mbufs_vec (struct i40e_rx_queue *rxq) {
drivers/net/i40e/i40e_rxtx_vec_neon.c +6 -6
@@ -681,14 +681,14 @@ i40e_xmit_fixed_burst_vec(void *__rte_restrict tx_queue, {struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile struct i40e_tx_desc *txdp; -struct ci_tx_entry *txep; +struct ci_tx_entry_vec *txep;uint16_t n, nb_commit, tx_id;uint64_t flags =I40E_TD_CMD ;uint64_t rs =I40E_TX_DESC_CMD_RS |I40E_TD_CMD ;int i;if (txq->nb_tx_free < txq->tx_free_thresh) -i40e_tx_free_bufs (txq); +ci_tx_free_bufs_vec (txq, i40e_tx_desc_done, false); nb_commit = nb_pkts = (uint16_t )RTE_MIN (txq->nb_tx_free, nb_pkts);if (unlikely (nb_pkts ==0 ))
@@ -696,13 +696,13 @@ i40e_xmit_fixed_burst_vec(void *__rte_restrict tx_queue, tx_id = txq->tx_tail; txdp = &txq->i40e_tx_ring[tx_id]; - txep = &txq->sw_ring[tx_id]; + txep = &txq->sw_ring_vec[tx_id]; txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts); n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) { -ci_tx_backlog_entry (txep, tx_pkts, n); +ci_tx_backlog_entry_vec (txep, tx_pkts, n);for (i =0 ; i < n -1 ; ++i, ++tx_pkts, ++txdp)vtx1 (txdp, *tx_pkts, flags);
@@ -716,10 +716,10 @@ i40e_xmit_fixed_burst_vec(void *__rte_restrict tx_queue,/* avoid reach the end of ring */ txdp = &txq->i40e_tx_ring[tx_id]; - txep = &txq->sw_ring[tx_id]; + txep = &txq->sw_ring_vec[tx_id]; } -ci_tx_backlog_entry (txep, tx_pkts, nb_commit); +ci_tx_backlog_entry_vec (txep, tx_pkts, nb_commit);vtx (txdp, tx_pkts, nb_commit, flags);
drivers/net/i40e/i40e_rxtx_vec_sse.c +6 -6
@@ -700,14 +700,14 @@ i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, {struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile struct i40e_tx_desc *txdp; -struct ci_tx_entry *txep; +struct ci_tx_entry_vec *txep;uint16_t n, nb_commit, tx_id;uint64_t flags =I40E_TD_CMD ;uint64_t rs =I40E_TX_DESC_CMD_RS |I40E_TD_CMD ;int i;if (txq->nb_tx_free < txq->tx_free_thresh) -i40e_tx_free_bufs (txq); +ci_tx_free_bufs_vec (txq, i40e_tx_desc_done, false); nb_commit = nb_pkts = (uint16_t )RTE_MIN (txq->nb_tx_free, nb_pkts);if (unlikely (nb_pkts ==0 ))
@@ -715,13 +715,13 @@ i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = txq->tx_tail; txdp = &txq->i40e_tx_ring[tx_id]; - txep = &txq->sw_ring[tx_id]; + txep = &txq->sw_ring_vec[tx_id]; txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts); n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) { -ci_tx_backlog_entry (txep, tx_pkts, n); +ci_tx_backlog_entry_vec (txep, tx_pkts, n);for (i =0 ; i < n -1 ; ++i, ++tx_pkts, ++txdp)vtx1 (txdp, *tx_pkts, flags);
@@ -735,10 +735,10 @@ i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,/* avoid reach the end of ring */ txdp = &txq->i40e_tx_ring[tx_id]; - txep = &txq->sw_ring[tx_id]; + txep = &txq->sw_ring_vec[tx_id]; } -ci_tx_backlog_entry (txep, tx_pkts, nb_commit); +ci_tx_backlog_entry_vec (txep, tx_pkts, nb_commit);vtx (txdp, tx_pkts, nb_commit, flags);
Signature · 1 line
2.43.0
PATCH v2 20/22 net/iavf: use vector SW ring for all vector paths
In reply to Bruce RichardsonThe AVX-512 code path used a smaller SW ring structure only containing the mbuf pointer, but no other fields. The other fields are only used in the scalar code path, so update all vector driver code paths (AVX2, SSE) to use the smaller, faster structure.
--- drivers/net/iavf/iavf_rxtx.c | 7 ------- drivers/net/iavf/iavf_rxtx_vec_avx2.c | 12 ++++++------ drivers/net/iavf/iavf_rxtx_vec_avx512.c | 8 -------- drivers/net/iavf/iavf_rxtx_vec_common.h | 6 ------ drivers/net/iavf/iavf_rxtx_vec_sse.c | 14 +++++++------- 5 files changed, 13 insertions(+), 34 deletions(-)
drivers/net/iavf/iavf_rxtx.c +0 -7
@@ -4193,14 +4193,7 @@ iavf_set_tx_function(struct rte_eth_dev *dev) txq = dev->data->tx_queues[i];if (!txq)continue ; -#ifdef CC_AVX512_SUPPORT -if (use_avx512) -iavf_txq_vec_setup_avx512 (txq); -else -iavf_txq_vec_setup (txq); -#else iavf_txq_vec_setup (txq); -#endif }if (no_poll_on_link_down) {
drivers/net/iavf/iavf_rxtx_vec_avx2.c +6 -6
@@ -1736,14 +1736,14 @@ iavf_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, {struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile struct iavf_tx_desc *txdp; -struct ci_tx_entry *txep; +struct ci_tx_entry_vec *txep;uint16_t n, nb_commit, tx_id;/* bit2 is reserved and must be set to 1 according to Spec */ uint64_t flags =IAVF_TX_DESC_CMD_EOP |IAVF_TX_DESC_CMD_ICRC ;uint64_t rs =IAVF_TX_DESC_CMD_RS | flags;if (txq->nb_tx_free < txq->tx_free_thresh) -iavf_tx_free_bufs (txq); +ci_tx_free_bufs_vec (txq, iavf_tx_desc_done, false); nb_pkts = (uint16_t )RTE_MIN (txq->nb_tx_free, nb_pkts);if (unlikely (nb_pkts ==0 ))
@@ -1752,13 +1752,13 @@ iavf_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = txq->tx_tail; txdp = &txq->iavf_tx_ring[tx_id]; - txep = &txq->sw_ring[tx_id]; + txep = &txq->sw_ring_vec[tx_id]; txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts); n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) { -ci_tx_backlog_entry (txep, tx_pkts, n); +ci_tx_backlog_entry_vec (txep, tx_pkts, n);iavf_vtx (txdp, tx_pkts, n -1 , flags, offload); tx_pkts += (n -1 );
@@ -1773,10 +1773,10 @@ iavf_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts,/* avoid reach the end of ring */ txdp = &txq->iavf_tx_ring[tx_id]; - txep = &txq->sw_ring[tx_id]; + txep = &txq->sw_ring_vec[tx_id]; } -ci_tx_backlog_entry (txep, tx_pkts, nb_commit); +ci_tx_backlog_entry_vec (txep, tx_pkts, nb_commit);iavf_vtx (txdp, tx_pkts, nb_commit, flags, offload);
drivers/net/iavf/iavf_rxtx_vec_avx512.c +0 -8
@@ -2357,14 +2357,6 @@ iavf_xmit_pkts_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts,return iavf_xmit_pkts_vec_avx512_cmn (tx_queue, tx_pkts, nb_pkts, false); } -int __rte_cold -iavf_txq_vec_setup_avx512 (struct ci_tx_queue *txq) -{ - txq->vector_tx = true; - txq->vector_sw_ring = true; -return 0 ; -} -uint16_t iavf_xmit_pkts_vec_avx512_offload (void *tx_queue,struct rte_mbuf **tx_pkts,uint16_t nb_pkts)
drivers/net/iavf/iavf_rxtx_vec_common.h +0 -6
@@ -24,12 +24,6 @@ iavf_tx_desc_done(struct ci_tx_queue *txq, uint16_t idx)rte_cpu_to_le_64 (IAVF_TX_DESC_DTYPE_DESC_DONE ); } -static __rte_always_inline int -iavf_tx_free_bufs (struct ci_tx_queue *txq) -{ -return ci_tx_free_bufs (txq, iavf_tx_desc_done); -} -static inline void _iavf_rx_queue_release_mbufs_vec (struct iavf_rx_queue *rxq) {
drivers/net/iavf/iavf_rxtx_vec_sse.c +7 -7
@@ -1368,14 +1368,14 @@ iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, {struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile struct iavf_tx_desc *txdp; -struct ci_tx_entry *txep; +struct ci_tx_entry_vec *txep;uint16_t n, nb_commit, tx_id;uint64_t flags =IAVF_TX_DESC_CMD_EOP |0x04 ;/* bit 2 must be set */ uint64_t rs =IAVF_TX_DESC_CMD_RS | flags;int i;if (txq->nb_tx_free < txq->tx_free_thresh) -iavf_tx_free_bufs (txq); +ci_tx_free_bufs_vec (txq, iavf_tx_desc_done, false); nb_pkts = (uint16_t )RTE_MIN (txq->nb_tx_free, nb_pkts);if (unlikely (nb_pkts ==0 ))
@@ -1384,13 +1384,13 @@ iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = txq->tx_tail; txdp = &txq->iavf_tx_ring[tx_id]; - txep = &txq->sw_ring[tx_id]; + txep = &txq->sw_ring_vec[tx_id]; txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts); n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) { -ci_tx_backlog_entry (txep, tx_pkts, n); +ci_tx_backlog_entry_vec (txep, tx_pkts, n);for (i =0 ; i < n -1 ; ++i, ++tx_pkts, ++txdp)vtx1 (txdp, *tx_pkts, flags);
@@ -1404,10 +1404,10 @@ iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,/* avoid reach the end of ring */ txdp = &txq->iavf_tx_ring[tx_id]; - txep = &txq->sw_ring[tx_id]; + txep = &txq->sw_ring_vec[tx_id]; } -ci_tx_backlog_entry (txep, tx_pkts, nb_commit); +ci_tx_backlog_entry_vec (txep, tx_pkts, nb_commit);iavf_vtx (txdp, tx_pkts, nb_commit, flags);
@@ -1462,7 +1462,7 @@ int __rte_cold iavf_txq_vec_setup(struct ci_tx_queue *txq) { txq->vector_tx = true; - txq->vector_sw_ring = false; + txq->vector_sw_ring = txq->vector_tx;return 0 ; }
Signature · 1 line
2.43.0
PATCH v2 21/22 net/_common_intel: remove unneeded code
In reply to Bruce RichardsonWith all drivers using the common Tx structure updated so that their vector paths all use the simplified Tx mbuf ring format, it's no longer necessary to have a separate flag for the ring format and for use of a vector driver.
Remove the former flag and base all decisions off the vector flag. With that done, we go from having only two paths to consider for releasing all mbufs in the ring, not three. That allows further simpification of the "ci_txq_release_all_mbufs" function.
The separate function to free buffers from the vector driver not using the simplified ring format can similarly be removed as no longer necessary.
--- drivers/net/_common_intel/tx.h | 97 +++-------------------- drivers/net/i40e/i40e_rxtx.c | 1 - drivers/net/iavf/iavf_rxtx_vec_sse.c | 1 - drivers/net/ice/ice_rxtx.c | 1 - drivers/net/ixgbe/ixgbe_rxtx_vec_common.h | 1 - 5 files changed, 10 insertions(+), 91 deletions(-)
drivers/net/_common_intel/tx.h +10 -87
@@ -66,7 +66,6 @@ struct ci_tx_queue {bool tx_deferred_start;/* don't start this queue in dev start */ bool q_set;/* indicate if tx queue has been configured */ bool vector_tx;/* port is using vector TX */ -bool vector_sw_ring;/* port is using vectorized SW ring (ieth_tx_entry_vec) */ union {/* the VSI this queue belongs to */ struct i40e_vsi *i40e_vsi;struct iavf_vsi *iavf_vsi;
@@ -120,72 +119,6 @@ ci_tx_backlog_entry_vec(struct ci_tx_entry_vec *txep, struct rte_mbuf **tx_pkts,typedef int (*ci_desc_done_fn )(struct ci_tx_queue *txq,uint16_t idx); -static __rte_always_inline int -ci_tx_free_bufs (struct ci_tx_queue *txq,ci_desc_done_fn desc_done) -{ -struct ci_tx_entry *txep; -uint32_t n; -uint32_t i; -int nb_free =0 ; -struct rte_mbuf *m, *free[IETH_VPMD_TX_MAX_FREE_BUF ]; - -/* check DD bits on threshold descriptor */ -if (!desc_done (txq, txq->tx_next_dd)) -return 0 ; - - n = txq->tx_rs_thresh; - -/* first buffer to free from S/W ring is at index -* tx_next_dd - (tx_rs_thresh-1) -*/ - txep = &txq->sw_ring[txq->tx_next_dd - (n -1 )]; - -if (txq->offloads &RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE ) { -for (i =0 ; i < n; i++) { - free[i] = txep[i].mbuf; -/* no need to reset txep[i].mbuf in vector path */ - } -rte_mempool_put_bulk (free[0 ]->pool, (void **)free, n); - goto done; - } - - m =rte_pktmbuf_prefree_seg (txep[0 ].mbuf); -if (likely (m !=NULL )) { - free[0 ] = m; - nb_free =1 ; -for (i =1 ; i < n; i++) { - m =rte_pktmbuf_prefree_seg (txep[i].mbuf); -if (likely (m !=NULL )) { -if (likely (m->pool == free[0 ]->pool)) { - free[nb_free++] = m; - }else { -rte_mempool_put_bulk (free[0 ]->pool, - (void *)free, - nb_free); - free[0 ] = m; - nb_free =1 ; - } - } - } -rte_mempool_put_bulk (free[0 ]->pool, (void **)free, nb_free); - }else { -for (i =1 ; i < n; i++) { - m =rte_pktmbuf_prefree_seg (txep[i].mbuf); -if (m !=NULL ) -rte_mempool_put (m->pool, m); - } - } - -done: -/* buffers were freed, update counters */ - txq->nb_tx_free = (uint16_t )(txq->nb_tx_free + txq->tx_rs_thresh); - txq->tx_next_dd = (uint16_t )(txq->tx_next_dd + txq->tx_rs_thresh); -if (txq->tx_next_dd >= txq->nb_tx_desc) - txq->tx_next_dd = (uint16_t )(txq->tx_rs_thresh -1 ); - -return txq->tx_rs_thresh; -} - static __rte_always_inline intci_tx_free_bufs_vec (struct ci_tx_queue *txq,ci_desc_done_fn desc_done,bool ctx_descs) {
@@ -278,21 +211,6 @@ ci_tx_free_bufs_vec(struct ci_tx_queue *txq, ci_desc_done_fn desc_done, bool ctxreturn txq->tx_rs_thresh; } -#define IETH_FREE_BUFS_LOOP (swr, nb_desc, start, end) do { \ - uint16_t i = start; \ - if (end < i) { \ - for (; i < nb_desc; i++) { \ - rte_pktmbuf_free_seg(swr[i].mbuf); \ - swr[i].mbuf = NULL; \ - } \ - i = 0; \ - } \ - for (; i < end; i++) { \ - rte_pktmbuf_free_seg(swr[i].mbuf); \ - swr[i].mbuf = NULL; \ - } \ -} while (0) -static inline void ci_txq_release_all_mbufs (struct ci_tx_queue *txq,bool use_ctx) {
@@ -311,16 +229,21 @@ ci_txq_release_all_mbufs(struct ci_tx_queue *txq, bool use_ctx)/** * vPMD tx will not set sw_ring's mbuf to NULL after free, -* so need to free remains more carefully. +* so determining buffers to free is a little more complex. */ const uint16_t start = (txq->tx_next_dd - txq->tx_rs_thresh +1 ) >> use_ctx;const uint16_t nb_desc = txq->nb_tx_desc >> use_ctx;const uint16_t end = txq->tx_tail >> use_ctx; -if (txq->vector_sw_ring) -IETH_FREE_BUFS_LOOP (txq->sw_ring_vec, nb_desc, start, end); -else - IETH_FREE_BUFS_LOOP(txq->sw_ring, nb_desc, start, end); +uint16_t i = start; +if (end < i) { +for (; i < nb_desc; i++) +rte_pktmbuf_free_seg (txq->sw_ring_vec[i].mbuf); + i =0 ; + } +for (; i < end; i++) +rte_pktmbuf_free_seg (txq->sw_ring_vec[i].mbuf); + memset(txq->sw_ring_vec, 0, sizeof(txq->sw_ring_vec[0]) * nb_desc); }#endif /* _COMMON_INTEL_TX_H_ */
drivers/net/i40e/i40e_rxtx.c +0 -1
@@ -1891,7 +1891,6 @@ i40e_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id) tx_queue_id); txq->vector_tx = ad->tx_vec_allowed; - txq->vector_sw_ring = txq->vector_tx;/* * tx_queue_id is queue id application refers to, while
drivers/net/iavf/iavf_rxtx_vec_sse.c +0 -1
@@ -1462,7 +1462,6 @@ int __rte_cold iavf_txq_vec_setup(struct ci_tx_queue *txq) { txq->vector_tx = true; - txq->vector_sw_ring = txq->vector_tx;return 0 ; }
drivers/net/ice/ice_rxtx.c +0 -1
@@ -825,7 +825,6 @@ ice_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)/* record what kind of descriptor cleanup we need on teardown */ txq->vector_tx = ad->tx_vec_allowed; - txq->vector_sw_ring = txq->vector_tx; dev->data->tx_queue_state[tx_queue_id] =RTE_ETH_QUEUE_STATE_STARTED ;
drivers/net/ixgbe/ixgbe_rxtx_vec_common.h +0 -1
@@ -183,7 +183,6 @@ ixgbe_txq_vec_setup_default(struct ci_tx_queue *txq, txq->sw_ring_vec = txq->sw_ring_vec +1 ; txq->ops = txq_ops; txq->vector_tx =1 ; - txq->vector_sw_ring =1 ;return 0 ; }
Signature · 1 line
2.43.0
PATCH v2 22/22 net/ixgbe: use common Tx backlog entry fn
In reply to Bruce RichardsonRemove the custom vector Tx backlog entry function and use the standard intel_common one, now that all vector drivers are using the same, smaller ring structure.
--- drivers/net/ixgbe/ixgbe_rxtx_vec_common.h | 10 ---------- drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c | 4 ++-- drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c | 4 ++-- 3 files changed, 4 insertions(+), 14 deletions(-)
drivers/net/ixgbe/ixgbe_rxtx_vec_common.h +0 -10
@@ -68,16 +68,6 @@ ixgbe_tx_free_bufs(struct ci_tx_queue *txq)return txq->tx_rs_thresh; } -static __rte_always_inline void -tx_backlog_entry (struct ci_tx_entry_vec *txep, -struct rte_mbuf **tx_pkts,uint16_t nb_pkts) -{ -int i; - -for (i =0 ; i < (int )nb_pkts; ++i) - txep[i].mbuf = tx_pkts[i]; -} -static inline void _ixgbe_rx_queue_release_mbufs_vec (struct ixgbe_rx_queue *rxq) {
drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c +2 -2
@@ -597,7 +597,7 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) { -tx_backlog_entry (txep, tx_pkts, n); +ci_tx_backlog_entry_vec (txep, tx_pkts, n);for (i =0 ; i < n -1 ; ++i, ++tx_pkts, ++txdp)vtx1 (txdp, *tx_pkts, flags);
@@ -614,7 +614,7 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, txep = &txq->sw_ring_vec[tx_id]; } -tx_backlog_entry (txep, tx_pkts, nb_commit); +ci_tx_backlog_entry_vec (txep, tx_pkts, nb_commit);vtx (txdp, tx_pkts, nb_commit, flags);
drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c +2 -2
@@ -720,7 +720,7 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) { -tx_backlog_entry (txep, tx_pkts, n); +ci_tx_backlog_entry_vec (txep, tx_pkts, n);for (i =0 ; i < n -1 ; ++i, ++tx_pkts, ++txdp)vtx1 (txdp, *tx_pkts, flags);
@@ -737,7 +737,7 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, txep = &txq->sw_ring_vec[tx_id]; } -tx_backlog_entry (txep, tx_pkts, nb_commit); +ci_tx_backlog_entry_vec (txep, tx_pkts, nb_commit);vtx (txdp, tx_pkts, nb_commit, flags);
Signature · 1 line
2.43.0
v3 cover Reduce code duplication across Intel NIC drivers
In reply to Bruce RichardsonThis RFC attempts to reduce the amount of code duplication across a number of Intel NIC drivers, specifically: ixgbe, i40e, iavf, and ice.
The first patch extract a function from the Rx side, otherwise the majority of the changes are on the Tx side, leading to a converged Tx queue structure across the 4 drivers, and a large number of common functions.
v2->v3:
* Fix incorrect/unadjusted memset in patch 8, leading to incorrect threshold tracking in ixgbe.
v1->v2:
* Fix two additional checkpatch issues that were flagged.
* Added in patch 21, which performs additional cleanup that is possible once all vector drivers use the same mbuf free/release process. [This brings the patchset to having over twice as many lines removed as added (1887 vs 930), and close to having a net removal of 1kloc]
RFC->v1:
* Moved the location of the common code from "common/intel_eth" to "net/_common_intel", and added only ".." to the driver include path so that the paths included "_common_intel" in them, to make it clear it's not driver-local headers.
* Due to change in location, structure/fn prefix changes from "ieth" to "ci" for "common intel".
* Removed the seeming-arbitrary split of vector and non-vector code - since much of the code taken from vector files was scalar code which was used by the vector drivers.
* Split code into separate Rx and Tx files.
* Fixed multiple checkpatch issues (but not all).
* Attempted to improve name standardization, by using "_vec" as a common suffix for all vector-related fns and data. Previously, some names had "vec" in the middle, others had just "_v" suffix or full word "vector" as suffix.
* Other minor changes...
Bruce Richardson (22):
net/_common_intel: add pkt reassembly fn for intel drivers net/_common_intel: provide common Tx entry structures net/_common_intel: add Tx mbuf ring replenish fn drivers/net: align Tx queue struct field names drivers/net: add prefix for driver-specific structs net/_common_intel: merge ice and i40e Tx queue struct net/iavf: use common Tx queue structure net/ixgbe: convert Tx queue context cache field to ptr net/ixgbe: use common Tx queue structure net/_common_intel: pack Tx queue structure net/_common_intel: add post-Tx buffer free function net/_common_intel: add Tx buffer free fn for AVX-512 net/iavf: use common Tx free fn for AVX-512 net/ice: move Tx queue mbuf cleanup fn to common net/i40e: use common Tx queue mbuf cleanup fn net/ixgbe: use common Tx queue mbuf cleanup fn net/iavf: use common Tx queue mbuf cleanup fn net/ice: use vector SW ring for all vector paths net/i40e: use vector SW ring for all vector paths net/iavf: use vector SW ring for all vector paths net/_common_intel: remove unneeded code net/ixgbe: use common Tx backlog entry fn
drivers/net/_common_intel/rx.h | 79 ++++++ drivers/net/_common_intel/tx.h | 249 ++++++++++++++++++ drivers/net/i40e/i40e_ethdev.c | 4 +- drivers/net/i40e/i40e_ethdev.h | 8 +- drivers/net/i40e/i40e_fdir.c | 10 +- .../net/i40e/i40e_recycle_mbufs_vec_common.c | 6 +- drivers/net/i40e/i40e_rxtx.c | 192 +++++--------- drivers/net/i40e/i40e_rxtx.h | 61 +---- drivers/net/i40e/i40e_rxtx_vec_altivec.c | 26 +- drivers/net/i40e/i40e_rxtx_vec_avx2.c | 26 +- drivers/net/i40e/i40e_rxtx_vec_avx512.c | 144 +--------- drivers/net/i40e/i40e_rxtx_vec_common.h | 144 +--------- drivers/net/i40e/i40e_rxtx_vec_neon.c | 26 +- drivers/net/i40e/i40e_rxtx_vec_sse.c | 26 +- drivers/net/i40e/meson.build | 2 +- drivers/net/iavf/iavf.h | 2 +- drivers/net/iavf/iavf_ethdev.c | 4 +- drivers/net/iavf/iavf_rxtx.c | 180 +++++-------- drivers/net/iavf/iavf_rxtx.h | 61 +---- drivers/net/iavf/iavf_rxtx_vec_avx2.c | 47 ++-- drivers/net/iavf/iavf_rxtx_vec_avx512.c | 214 +++------------ drivers/net/iavf/iavf_rxtx_vec_common.h | 160 +---------- drivers/net/iavf/iavf_rxtx_vec_sse.c | 56 ++-- drivers/net/iavf/iavf_vchnl.c | 8 +- drivers/net/iavf/meson.build | 2 +- drivers/net/ice/ice_dcf.c | 4 +- drivers/net/ice/ice_dcf_ethdev.c | 21 +- drivers/net/ice/ice_diagnose.c | 2 +- drivers/net/ice/ice_ethdev.c | 2 +- drivers/net/ice/ice_ethdev.h | 7 +- drivers/net/ice/ice_rxtx.c | 163 +++++------- drivers/net/ice/ice_rxtx.h | 52 +--- drivers/net/ice/ice_rxtx_vec_avx2.c | 26 +- drivers/net/ice/ice_rxtx_vec_avx512.c | 153 +---------- drivers/net/ice/ice_rxtx_vec_common.h | 190 +------------ drivers/net/ice/ice_rxtx_vec_sse.c | 32 +-- drivers/net/ice/meson.build | 2 +- drivers/net/ixgbe/base/ixgbe_osdep.h | 2 +- drivers/net/ixgbe/ixgbe_ethdev.c | 4 +- .../ixgbe/ixgbe_recycle_mbufs_vec_common.c | 6 +- drivers/net/ixgbe/ixgbe_rxtx.c | 139 +++++----- drivers/net/ixgbe/ixgbe_rxtx.h | 73 +---- drivers/net/ixgbe/ixgbe_rxtx_vec_common.h | 131 ++------- drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c | 37 ++- drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c | 37 ++- drivers/net/ixgbe/meson.build | 2 +- 46 files changed, 931 insertions(+), 1891 deletions(-) create mode 100644 drivers/net/_common_intel/rx.h create mode 100644 drivers/net/_common_intel/tx.h
--
2.43.0
PATCH v3 1/22 net/_common_intel: add pkt reassembly fn for intel drivers
The code for reassembling a single, multi-mbuf packet from multiple buffers received from the NIC is duplicated across many drivers. Rather than having multiple copies of this function, we can create an "_common_intel" directory to hold such functions and consolidate multiple functions down to a single one for easier maintenance.
--- drivers/net/_common_intel/rx.h | 79 +++++++++++++++++++++++ drivers/net/i40e/i40e_rxtx_vec_altivec.c | 4 +- drivers/net/i40e/i40e_rxtx_vec_avx2.c | 4 +- drivers/net/i40e/i40e_rxtx_vec_avx512.c | 4 +- drivers/net/i40e/i40e_rxtx_vec_common.h | 64 +----------------- drivers/net/i40e/i40e_rxtx_vec_neon.c | 4 +- drivers/net/i40e/i40e_rxtx_vec_sse.c | 4 +- drivers/net/i40e/meson.build | 2 +- drivers/net/iavf/iavf_rxtx_vec_avx2.c | 8 +-- drivers/net/iavf/iavf_rxtx_vec_avx512.c | 8 +-- drivers/net/iavf/iavf_rxtx_vec_common.h | 65 +------------------ drivers/net/iavf/iavf_rxtx_vec_sse.c | 8 +-- drivers/net/iavf/meson.build | 2 +- drivers/net/ice/ice_rxtx_vec_avx2.c | 4 +- drivers/net/ice/ice_rxtx_vec_avx512.c | 8 +-- drivers/net/ice/ice_rxtx_vec_common.h | 66 +------------------ drivers/net/ice/ice_rxtx_vec_sse.c | 4 +- drivers/net/ice/meson.build | 2 +- drivers/net/ixgbe/ixgbe_rxtx_vec_common.h | 63 +----------------- drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c | 4 +- drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c | 4 +- drivers/net/ixgbe/meson.build | 2 +- 22 files changed, 121 insertions(+), 292 deletions(-) create mode 100644 drivers/net/_common_intel/rx.h
drivers/net/_common_intel/rx.h new file +79 -0
@@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2024 Intel Corporation +*/ + +#ifndef _COMMON_INTEL_RX_H_ +#define _COMMON_INTEL_RX_H_ + +#include <stdint.h> +#include <unistd.h> +#include <rte_mbuf.h> + +#define CI_RX_BURST 32 + +static inline uint16_t +ci_rx_reassemble_packets (struct rte_mbuf **rx_bufs,uint16_t nb_bufs,uint8_t *split_flags, +struct rte_mbuf **pkt_first_seg,struct rte_mbuf **pkt_last_seg, +const uint8_t crc_len) +{ +struct rte_mbuf *pkts[CI_RX_BURST ] = {0 };/*finished pkts*/ +struct rte_mbuf *start = *pkt_first_seg; +struct rte_mbuf *end = *pkt_last_seg; +unsigned int pkt_idx, buf_idx; + +for (buf_idx =0 , pkt_idx =0 ; buf_idx < nb_bufs; buf_idx++) { +if (end) { +/* processing a split packet */ + end->next = rx_bufs[buf_idx]; + rx_bufs[buf_idx]->data_len += crc_len; + + start->nb_segs++; + start->pkt_len += rx_bufs[buf_idx]->data_len; + end = end->next; + +if (!split_flags[buf_idx]) { +/* it's the last packet of the set */ + start->hash = end->hash; + start->vlan_tci = end->vlan_tci; + start->ol_flags = end->ol_flags; +/* we need to strip crc for the whole packet */ + start->pkt_len -= crc_len; +if (end->data_len > crc_len) { + end->data_len -= crc_len; + }else { +/* free up last mbuf */ +struct rte_mbuf *secondlast = start; + + start->nb_segs--; +while (secondlast->next != end) + secondlast = secondlast->next; + secondlast->data_len -= (crc_len - end->data_len); + secondlast->next =NULL ; +rte_pktmbuf_free_seg (end); + } + pkts[pkt_idx++] = start; + start =NULL ; + end =NULL ; + } + }else { +/* not processing a split packet */ +if (!split_flags[buf_idx]) { +/* not a split packet, save and skip */ + pkts[pkt_idx++] = rx_bufs[buf_idx]; +continue ; + } + start = rx_bufs[buf_idx]; + end = start; + rx_bufs[buf_idx]->data_len += crc_len; + rx_bufs[buf_idx]->pkt_len += crc_len; + } + } + +/* save the partial packet for next time */ + *pkt_first_seg = start; + *pkt_last_seg = end; +memcpy (rx_bufs, pkts, pkt_idx * (sizeof (*pkts))); +return pkt_idx; +} + +#endif /* _COMMON_INTEL_RX_H_ */
drivers/net/i40e/i40e_rxtx_vec_altivec.c +2 -2
@@ -494,8 +494,8 @@ i40e_recv_scattered_burst_vec(void *rx_queue, struct rte_mbuf **rx_pkts,if (i == nb_bufs)return nb_bufs; } -return i +reassemble_packets (rxq, &rx_pkts[i], nb_bufs - i, - &split_flags[i]); +return i +ci_rx_reassemble_packets (&rx_pkts[i], nb_bufs - i, &split_flags[i], + &rxq->pkt_first_seg, &rxq->pkt_last_seg, rxq->crc_len); }/**
drivers/net/i40e/i40e_rxtx_vec_avx2.c +2 -2
@@ -657,8 +657,8 @@ i40e_recv_scattered_burst_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,return nb_bufs; rxq->pkt_first_seg = rx_pkts[i]; } -return i +reassemble_packets (rxq, &rx_pkts[i], nb_bufs - i, - &split_flags[i]); +return i +ci_rx_reassemble_packets (&rx_pkts[i], nb_bufs - i, &split_flags[i], + &rxq->pkt_first_seg, &rxq->pkt_last_seg, rxq->crc_len); }/*
drivers/net/i40e/i40e_rxtx_vec_avx512.c +2 -2
@@ -725,8 +725,8 @@ i40e_recv_scattered_burst_vec_avx512(void *rx_queue,return nb_bufs; rxq->pkt_first_seg = rx_pkts[i]; } -return i +reassemble_packets (rxq, &rx_pkts[i], nb_bufs - i, - &split_flags[i]); +return i +ci_rx_reassemble_packets (&rx_pkts[i], nb_bufs - i, &split_flags[i], + &rxq->pkt_first_seg, &rxq->pkt_last_seg, rxq->crc_len); }/**
drivers/net/i40e/i40e_rxtx_vec_common.h +1 -63
76 lines not on this page · raw message
drivers/net/i40e/i40e_rxtx_vec_neon.c +2 -2
@@ -623,8 +623,8 @@ i40e_recv_scattered_burst_vec(void *rx_queue, struct rte_mbuf **rx_pkts,return nb_bufs; rxq->pkt_first_seg = rx_pkts[i]; } -return i +reassemble_packets (rxq, &rx_pkts[i], nb_bufs - i, - &split_flags[i]); +return i +ci_rx_reassemble_packets (&rx_pkts[i], nb_bufs - i, &split_flags[i], + &rxq->pkt_first_seg, &rxq->pkt_last_seg, rxq->crc_len); }/**
drivers/net/i40e/i40e_rxtx_vec_sse.c +2 -2
@@ -641,8 +641,8 @@ i40e_recv_scattered_burst_vec(void *rx_queue, struct rte_mbuf **rx_pkts,return nb_bufs; rxq->pkt_first_seg = rx_pkts[i]; } -return i +reassemble_packets (rxq, &rx_pkts[i], nb_bufs - i, - &split_flags[i]); +return i +ci_rx_reassemble_packets (&rx_pkts[i], nb_bufs - i, &split_flags[i], + &rxq->pkt_first_seg, &rxq->pkt_last_seg, rxq->crc_len); }/**
drivers/net/i40e/meson.build +1 -1
@@ -36,7 +36,7 @@ sources = files( testpmd_sources = files('i40e_testpmd.c') deps += ['hash'] -includes += include_directories('base') +includes += include_directories('base', '..') if arch_subdir == 'x86' sources += files('i40e_rxtx_vec_sse.c')
drivers/net/iavf/iavf_rxtx_vec_avx2.c +4 -4
@@ -1508,8 +1508,8 @@ iavf_recv_scattered_burst_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,return nb_bufs; rxq->pkt_first_seg = rx_pkts[i]; } -return i +reassemble_packets (rxq, &rx_pkts[i], nb_bufs - i, - &split_flags[i]); +return i +ci_rx_reassemble_packets (&rx_pkts[i], nb_bufs - i, &split_flags[i], + &rxq->pkt_first_seg, &rxq->pkt_last_seg, rxq->crc_len); }/**
@@ -1597,8 +1597,8 @@ iavf_recv_scattered_burst_vec_avx2_flex_rxd(void *rx_queue,return nb_bufs; rxq->pkt_first_seg = rx_pkts[i]; } -return i +reassemble_packets (rxq, &rx_pkts[i], nb_bufs - i, - &split_flags[i]); +return i +ci_rx_reassemble_packets (&rx_pkts[i], nb_bufs - i, &split_flags[i], + &rxq->pkt_first_seg, &rxq->pkt_last_seg, rxq->crc_len); }/**
drivers/net/iavf/iavf_rxtx_vec_avx512.c +4 -4
@@ -1685,8 +1685,8 @@ iavf_recv_scattered_burst_vec_avx512(void *rx_queue, struct rte_mbuf **rx_pkts,return nb_bufs; rxq->pkt_first_seg = rx_pkts[i]; } -return i +reassemble_packets (rxq, &rx_pkts[i], nb_bufs - i, - &split_flags[i]); +return i +ci_rx_reassemble_packets (&rx_pkts[i], nb_bufs - i, &split_flags[i], + &rxq->pkt_first_seg, &rxq->pkt_last_seg, rxq->crc_len); }/**
@@ -1761,8 +1761,8 @@ iavf_recv_scattered_burst_vec_avx512_flex_rxd(void *rx_queue,return nb_bufs; rxq->pkt_first_seg = rx_pkts[i]; } -return i +reassemble_packets (rxq, &rx_pkts[i], nb_bufs - i, - &split_flags[i]); +return i +ci_rx_reassemble_packets (&rx_pkts[i], nb_bufs - i, &split_flags[i], + &rxq->pkt_first_seg, &rxq->pkt_last_seg, rxq->crc_len); }/**
drivers/net/iavf/iavf_rxtx_vec_common.h +1 -64
77 lines not on this page · raw message
drivers/net/iavf/iavf_rxtx_vec_sse.c +4 -4
@@ -1238,8 +1238,8 @@ iavf_recv_scattered_burst_vec(void *rx_queue, struct rte_mbuf **rx_pkts,return nb_bufs; rxq->pkt_first_seg = rx_pkts[i]; } -return i +reassemble_packets (rxq, &rx_pkts[i], nb_bufs - i, - &split_flags[i]); +return i +ci_rx_reassemble_packets (&rx_pkts[i], nb_bufs - i, &split_flags[i], + &rxq->pkt_first_seg, &rxq->pkt_last_seg, rxq->crc_len); }/**
@@ -1307,8 +1307,8 @@ iavf_recv_scattered_burst_vec_flex_rxd(void *rx_queue,return nb_bufs; rxq->pkt_first_seg = rx_pkts[i]; } -return i +reassemble_packets (rxq, &rx_pkts[i], nb_bufs - i, - &split_flags[i]); +return i +ci_rx_reassemble_packets (&rx_pkts[i], nb_bufs - i, &split_flags[i], + &rxq->pkt_first_seg, &rxq->pkt_last_seg, rxq->crc_len); }/**
drivers/net/iavf/meson.build +1 -1
@@ -5,7 +5,7 @@ if dpdk_conf.get('RTE_IOVA_IN_MBUF') == 0 subdir_done() endif -includes += include_directories('../../common/iavf') +includes += include_directories('../../common/iavf', '..') testpmd_sources = files('iavf_testpmd.c')
drivers/net/ice/ice_rxtx_vec_avx2.c +2 -2
@@ -726,8 +726,8 @@ ice_recv_scattered_burst_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,return nb_bufs; rxq->pkt_first_seg = rx_pkts[i]; } -return i +ice_rx_reassemble_packets (rxq, &rx_pkts[i], nb_bufs - i, - &split_flags[i]); +return i +ci_rx_reassemble_packets (&rx_pkts[i], nb_bufs - i, &split_flags[i], + &rxq->pkt_first_seg, &rxq->pkt_last_seg, rxq->crc_len); }/**
drivers/net/ice/ice_rxtx_vec_avx512.c +4 -4
@@ -763,8 +763,8 @@ ice_recv_scattered_burst_vec_avx512(void *rx_queue, struct rte_mbuf **rx_pkts,return nb_bufs; rxq->pkt_first_seg = rx_pkts[i]; } -return i +ice_rx_reassemble_packets (rxq, &rx_pkts[i], nb_bufs - i, - &split_flags[i]); +return i +ci_rx_reassemble_packets (&rx_pkts[i], nb_bufs - i, &split_flags[i], + &rxq->pkt_first_seg, &rxq->pkt_last_seg, rxq->crc_len); }/**
@@ -805,8 +805,8 @@ ice_recv_scattered_burst_vec_avx512_offload(void *rx_queue,return nb_bufs; rxq->pkt_first_seg = rx_pkts[i]; } -return i +ice_rx_reassemble_packets (rxq, &rx_pkts[i], nb_bufs - i, - &split_flags[i]); +return i +ci_rx_reassemble_packets (&rx_pkts[i], nb_bufs - i, &split_flags[i], + &rxq->pkt_first_seg, &rxq->pkt_last_seg, rxq->crc_len); }/**
drivers/net/ice/ice_rxtx_vec_common.h +1 -65
78 lines not on this page · raw message
drivers/net/ice/ice_rxtx_vec_sse.c +2 -2
@@ -640,8 +640,8 @@ ice_recv_scattered_burst_vec(void *rx_queue, struct rte_mbuf **rx_pkts,return nb_bufs; rxq->pkt_first_seg = rx_pkts[i]; } -return i +ice_rx_reassemble_packets (rxq, &rx_pkts[i], nb_bufs - i, - &split_flags[i]); +return i +ci_rx_reassemble_packets (&rx_pkts[i], nb_bufs - i, &split_flags[i], + &rxq->pkt_first_seg, &rxq->pkt_last_seg, rxq->crc_len); }/**
drivers/net/ice/meson.build +1 -1
@@ -19,7 +19,7 @@ sources = files( testpmd_sources = files('ice_testpmd.c') deps += ['hash', 'net', 'common_iavf'] -includes += include_directories('base', '../../common/iavf') +includes += include_directories('base', '..') if arch_subdir == 'x86' sources += files('ice_rxtx_vec_sse.c')
drivers/net/ixgbe/ixgbe_rxtx_vec_common.h +1 -62
72 lines not on this page · raw message
drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c +2 -2
@@ -516,8 +516,8 @@ ixgbe_recv_scattered_burst_vec(void *rx_queue, struct rte_mbuf **rx_pkts,return nb_bufs; rxq->pkt_first_seg = rx_pkts[i]; } -return i +reassemble_packets (rxq, &rx_pkts[i], nb_bufs - i, - &split_flags[i]); +return i +ci_rx_reassemble_packets (&rx_pkts[i], nb_bufs - i, &split_flags[i], + &rxq->pkt_first_seg, &rxq->pkt_last_seg, rxq->crc_len); }/**
drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c +2 -2
@@ -639,8 +639,8 @@ ixgbe_recv_scattered_burst_vec(void *rx_queue, struct rte_mbuf **rx_pkts,return nb_bufs; rxq->pkt_first_seg = rx_pkts[i]; } -return i +reassemble_packets (rxq, &rx_pkts[i], nb_bufs - i, - &split_flags[i]); +return i +ci_rx_reassemble_packets (&rx_pkts[i], nb_bufs - i, &split_flags[i], + &rxq->pkt_first_seg, &rxq->pkt_last_seg, rxq->crc_len); }/**
drivers/net/ixgbe/meson.build +1 -1
@@ -35,6 +35,6 @@ elif arch_subdir == 'arm' sources += files('ixgbe_recycle_mbufs_vec_common.c') endif -includes += include_directories('base') +includes += include_directories('base', '..') headers = files('rte_pmd_ixgbe.h')
Signature · 1 line
2.43.0
PATCH v3 2/22 net/_common_intel: provide common Tx entry structures
In reply to Bruce RichardsonThe Tx entry structures, both vector and scalar, are common across Intel drivers, so provide a single definition to be used everywhere.
--- drivers/net/_common_intel/tx.h | 27 +++++++++++++++++++ .../net/i40e/i40e_recycle_mbufs_vec_common.c | 2 +- drivers/net/i40e/i40e_rxtx.c | 18 ++++++------- drivers/net/i40e/i40e_rxtx.h | 14 +++------- drivers/net/i40e/i40e_rxtx_vec_altivec.c | 2 +- drivers/net/i40e/i40e_rxtx_vec_avx2.c | 2 +- drivers/net/i40e/i40e_rxtx_vec_avx512.c | 6 ++--- drivers/net/i40e/i40e_rxtx_vec_common.h | 4 +-- drivers/net/i40e/i40e_rxtx_vec_neon.c | 2 +- drivers/net/i40e/i40e_rxtx_vec_sse.c | 2 +- drivers/net/iavf/iavf_rxtx.c | 12 ++++----- drivers/net/iavf/iavf_rxtx.h | 14 +++------- drivers/net/iavf/iavf_rxtx_vec_avx2.c | 2 +- drivers/net/iavf/iavf_rxtx_vec_avx512.c | 10 +++---- drivers/net/iavf/iavf_rxtx_vec_common.h | 4 +-- drivers/net/iavf/iavf_rxtx_vec_sse.c | 2 +- drivers/net/ice/ice_dcf_ethdev.c | 2 +- drivers/net/ice/ice_rxtx.c | 16 +++++------ drivers/net/ice/ice_rxtx.h | 13 ++------- drivers/net/ice/ice_rxtx_vec_avx2.c | 2 +- drivers/net/ice/ice_rxtx_vec_avx512.c | 6 ++--- drivers/net/ice/ice_rxtx_vec_common.h | 6 ++--- drivers/net/ice/ice_rxtx_vec_sse.c | 2 +- .../ixgbe/ixgbe_recycle_mbufs_vec_common.c | 2 +- drivers/net/ixgbe/ixgbe_rxtx.c | 16 +++++------ drivers/net/ixgbe/ixgbe_rxtx.h | 22 +++------------ drivers/net/ixgbe/ixgbe_rxtx_vec_common.h | 8 +++--- drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c | 2 +- drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c | 2 +- 29 files changed, 105 insertions(+), 117 deletions(-) create mode 100644 drivers/net/_common_intel/tx.h
drivers/net/_common_intel/tx.h new file +27 -0
@@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2024 Intel Corporation +*/ + +#ifndef _COMMON_INTEL_TX_H_ +#define _COMMON_INTEL_TX_H_ + +#include <stdint.h> +#include <rte_mbuf.h> + +/** +* Structure associated with each descriptor of the TX ring of a TX queue. +*/ +struct ci_tx_entry { +struct rte_mbuf *mbuf;/* mbuf associated with TX desc, if any. */ +uint16_t next_id;/* Index of next descriptor in ring. */ +uint16_t last_id;/* Index of last scattered descriptor. */ +}; + +/** +* Structure associated with each descriptor of the TX ring of a TX queue in vector Tx. +*/ +struct ci_tx_entry_vec { +struct rte_mbuf *mbuf;/* mbuf associated with TX desc, if any. */ +}; + +#endif /* _COMMON_INTEL_TX_H_ */
drivers/net/i40e/i40e_recycle_mbufs_vec_common.c +1 -1
@@ -56,7 +56,7 @@ i40e_recycle_tx_mbufs_reuse_vec(void *tx_queue, struct rte_eth_recycle_rxq_info *recycle_rxq_info) {struct i40e_tx_queue *txq = tx_queue; -struct i40e_tx_entry *txep; +struct ci_tx_entry *txep;struct rte_mbuf **rxep;int i, n;uint16_t nb_recycle_mbufs;
drivers/net/i40e/i40e_rxtx.c +9 -9
@@ -378,7 +378,7 @@ i40e_build_ctob(uint32_t td_cmd,static inline int i40e_xmit_cleanup (struct i40e_tx_queue *txq) { -struct i40e_tx_entry *sw_ring = txq->sw_ring; +struct ci_tx_entry *sw_ring = txq->sw_ring;volatile struct i40e_tx_desc *txd = txq->tx_ring;uint16_t last_desc_cleaned = txq->last_desc_cleaned;uint16_t nb_tx_desc = txq->nb_tx_desc;
@@ -1081,8 +1081,8 @@ uint16_t i40e_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) {struct i40e_tx_queue *txq; -struct i40e_tx_entry *sw_ring; -struct i40e_tx_entry *txe, *txn; +struct ci_tx_entry *sw_ring; +struct ci_tx_entry *txe, *txn;volatile struct i40e_tx_desc *txd;volatile struct i40e_tx_desc *txr;struct rte_mbuf *tx_pkt;
@@ -1331,7 +1331,7 @@ i40e_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) static __rte_always_inline inti40e_tx_free_bufs (struct i40e_tx_queue *txq) { -struct i40e_tx_entry *txep; +struct ci_tx_entry *txep;uint16_t tx_rs_thresh = txq->tx_rs_thresh;uint16_t i =0 , j =0 ;struct rte_mbuf *free[RTE_I40E_TX_MAX_FREE_BUF_SZ ];
@@ -1418,7 +1418,7 @@ i40e_tx_fill_hw_ring(struct i40e_tx_queue *txq, uint16_t nb_pkts) {volatile struct i40e_tx_desc *txdp = &(txq->tx_ring[txq->tx_tail]); -struct i40e_tx_entry *txep = &(txq->sw_ring[txq->tx_tail]); +struct ci_tx_entry *txep = &txq->sw_ring[txq->tx_tail];const int N_PER_LOOP =4 ;const int N_PER_LOOP_MASK =N_PER_LOOP -1 ;int mainpart, leftover;
@@ -2555,7 +2555,7 @@ i40e_dev_tx_queue_setup(struct rte_eth_dev *dev,/* Allocate software ring */ txq->sw_ring =rte_zmalloc_socket ("i40e tx sw ring" , -sizeof (struct i40e_tx_entry ) * nb_desc, +sizeof (struct ci_tx_entry ) * nb_desc,RTE_CACHE_LINE_SIZE , socket_id);if (!txq->sw_ring) {
@@ -2723,7 +2723,7 @@ i40e_tx_queue_release_mbufs(struct i40e_tx_queue *txq)*/ #ifdef CC_AVX512_SUPPORT if (dev->tx_pkt_burst == i40e_xmit_pkts_vec_avx512) { -struct i40e_vec_tx_entry *swr = (void *)txq->sw_ring; +struct ci_tx_entry_vec *swr = (void *)txq->sw_ring; i = txq->tx_next_dd - txq->tx_rs_thresh +1 ;if (txq->tx_tail < i) {
@@ -2768,7 +2768,7 @@ static int i40e_tx_done_cleanup_full(struct i40e_tx_queue *txq, uint32_t free_cnt) { -struct i40e_tx_entry *swr_ring = txq->sw_ring; +struct ci_tx_entry *swr_ring = txq->sw_ring;uint16_t i, tx_last, tx_id;uint16_t nb_tx_free_last;uint16_t nb_tx_to_clean;
@@ -2874,7 +2874,7 @@ i40e_tx_done_cleanup(void *txq, uint32_t free_cnt)void i40e_reset_tx_queue (struct i40e_tx_queue *txq) { -struct i40e_tx_entry *txe; +struct ci_tx_entry *txe;uint16_t i, prev, size;if (!txq) {
drivers/net/i40e/i40e_rxtx.h +3 -11
@@ -5,6 +5,8 @@#ifndef _I40E_RXTX_H_#define _I40E_RXTX_H_ +#include <_common_intel/tx.h> +#define RTE_PMD_I40E_RX_MAX_BURST 32#define RTE_PMD_I40E_TX_MAX_BURST 32
@@ -122,16 +124,6 @@ struct i40e_rx_queue {const struct rte_memzone *mz; }; -struct i40e_tx_entry { -struct rte_mbuf *mbuf; -uint16_t next_id; -uint16_t last_id; -}; - -struct i40e_vec_tx_entry { -struct rte_mbuf *mbuf; -}; -/* * Structure associated with each TX queue. */
@@ -139,7 +131,7 @@ struct i40e_tx_queue {uint16_t nb_tx_desc;/**< number of TX descriptors */ uint64_t tx_ring_phys_addr;/**< TX ring DMA address */ volatile struct i40e_tx_desc *tx_ring;/**< TX ring virtual address */ -struct i40e_tx_entry *sw_ring;/**< virtual address of SW ring */ +struct ci_tx_entry *sw_ring;/**< virtual address of SW ring */ uint16_t tx_tail;/**< current value of tail register */ volatile uint8_t *qtx_tail;/**< register address of tail */ uint16_t nb_tx_used;/**< number of TX desc used since RS bit set */
drivers/net/i40e/i40e_rxtx_vec_altivec.c +1 -1
@@ -553,7 +553,7 @@ i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, {struct i40e_tx_queue *txq = (struct i40e_tx_queue *)tx_queue;volatile struct i40e_tx_desc *txdp; -struct i40e_tx_entry *txep; +struct ci_tx_entry *txep;uint16_t n, nb_commit, tx_id;uint64_t flags =I40E_TD_CMD ;uint64_t rs =I40E_TX_DESC_CMD_RS |I40E_TD_CMD ;
drivers/net/i40e/i40e_rxtx_vec_avx2.c +1 -1
@@ -745,7 +745,7 @@ i40e_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, {struct i40e_tx_queue *txq = (struct i40e_tx_queue *)tx_queue;volatile struct i40e_tx_desc *txdp; -struct i40e_tx_entry *txep; +struct ci_tx_entry *txep;uint16_t n, nb_commit, tx_id;uint64_t flags =I40E_TD_CMD ;uint64_t rs =I40E_TX_DESC_CMD_RS |I40E_TD_CMD ;
drivers/net/i40e/i40e_rxtx_vec_avx512.c +3 -3
@@ -757,7 +757,7 @@ i40e_recv_scattered_pkts_vec_avx512(void *rx_queue, static __rte_always_inline inti40e_tx_free_bufs_avx512 (struct i40e_tx_queue *txq) { -struct i40e_vec_tx_entry *txep; +struct ci_tx_entry_vec *txep;uint32_t n;uint32_t i;int nb_free =0 ;
@@ -920,7 +920,7 @@ vtx(volatile struct i40e_tx_desc *txdp, } static __rte_always_inline void -tx_backlog_entry_avx512 (struct i40e_vec_tx_entry *txep, +tx_backlog_entry_avx512 (struct ci_tx_entry_vec *txep,struct rte_mbuf **tx_pkts,uint16_t nb_pkts) {int i;
@@ -935,7 +935,7 @@ i40e_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, {struct i40e_tx_queue *txq = (struct i40e_tx_queue *)tx_queue;volatile struct i40e_tx_desc *txdp; -struct i40e_vec_tx_entry *txep; +struct ci_tx_entry_vec *txep;uint16_t n, nb_commit, tx_id;uint64_t flags =I40E_TD_CMD ;uint64_t rs =I40E_TX_DESC_CMD_RS |I40E_TD_CMD ;
drivers/net/i40e/i40e_rxtx_vec_common.h +2 -2
@@ -19,7 +19,7 @@ static __rte_always_inline inti40e_tx_free_bufs (struct i40e_tx_queue *txq) { -struct i40e_tx_entry *txep; +struct ci_tx_entry *txep;uint32_t n;uint32_t i;int nb_free =0 ;
@@ -85,7 +85,7 @@ i40e_tx_free_bufs(struct i40e_tx_queue *txq) } static __rte_always_inline void -tx_backlog_entry (struct i40e_tx_entry *txep, +tx_backlog_entry (struct ci_tx_entry *txep,struct rte_mbuf **tx_pkts,uint16_t nb_pkts) {int i;
drivers/net/i40e/i40e_rxtx_vec_neon.c +1 -1
@@ -681,7 +681,7 @@ i40e_xmit_fixed_burst_vec(void *__rte_restrict tx_queue, {struct i40e_tx_queue *txq = (struct i40e_tx_queue *)tx_queue;volatile struct i40e_tx_desc *txdp; -struct i40e_tx_entry *txep; +struct ci_tx_entry *txep;uint16_t n, nb_commit, tx_id;uint64_t flags =I40E_TD_CMD ;uint64_t rs =I40E_TX_DESC_CMD_RS |I40E_TD_CMD ;
drivers/net/i40e/i40e_rxtx_vec_sse.c +1 -1
@@ -700,7 +700,7 @@ i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, {struct i40e_tx_queue *txq = (struct i40e_tx_queue *)tx_queue;volatile struct i40e_tx_desc *txdp; -struct i40e_tx_entry *txep; +struct ci_tx_entry *txep;uint16_t n, nb_commit, tx_id;uint64_t flags =I40E_TD_CMD ;uint64_t rs =I40E_TX_DESC_CMD_RS |I40E_TD_CMD ;
drivers/net/iavf/iavf_rxtx.c +6 -6
@@ -284,7 +284,7 @@ reset_rx_queue(struct iavf_rx_queue *rxq)static inline void reset_tx_queue (struct iavf_tx_queue *txq) { -struct iavf_tx_entry *txe; +struct ci_tx_entry *txe;uint32_t i, size;uint16_t prev;
@@ -860,7 +860,7 @@ iavf_dev_tx_queue_setup(struct rte_eth_dev *dev,/* Allocate software ring */ txq->sw_ring =rte_zmalloc_socket ("iavf tx sw ring" , -sizeof (struct iavf_tx_entry ) * nb_desc, +sizeof (struct ci_tx_entry ) * nb_desc,RTE_CACHE_LINE_SIZE , socket_id);if (!txq->sw_ring) {
@@ -2379,7 +2379,7 @@ iavf_recv_pkts_bulk_alloc(void *rx_queue,static inline int iavf_xmit_cleanup (struct iavf_tx_queue *txq) { -struct iavf_tx_entry *sw_ring = txq->sw_ring; +struct ci_tx_entry *sw_ring = txq->sw_ring;uint16_t last_desc_cleaned = txq->last_desc_cleaned;uint16_t nb_tx_desc = txq->nb_tx_desc;uint16_t desc_to_clean_to;
@@ -2797,8 +2797,8 @@ iavf_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) {struct iavf_tx_queue *txq = tx_queue;volatile struct iavf_tx_desc *txr = txq->tx_ring; -struct iavf_tx_entry *txe_ring = txq->sw_ring; -struct iavf_tx_entry *txe, *txn; +struct ci_tx_entry *txe_ring = txq->sw_ring; +struct ci_tx_entry *txe, *txn;struct rte_mbuf *mb, *mb_seg;uint64_t buf_dma_addr;uint16_t desc_idx, desc_idx_last;
@@ -4268,7 +4268,7 @@ static int iavf_tx_done_cleanup_full(struct iavf_tx_queue *txq, uint32_t free_cnt) { -struct iavf_tx_entry *swr_ring = txq->sw_ring; +struct ci_tx_entry *swr_ring = txq->sw_ring;uint16_t tx_last, tx_id;uint16_t nb_tx_free_last;uint16_t nb_tx_to_clean;
drivers/net/iavf/iavf_rxtx.h +3 -11
@@ -5,6 +5,8 @@#ifndef _IAVF_RXTX_H_#define _IAVF_RXTX_H_ +#include <_common_intel/tx.h> +/* In QLEN must be whole number of 32 descriptors. */ #define IAVF_ALIGN_RING_DESC 32#define IAVF_MIN_RING_DESC 64
@@ -271,22 +273,12 @@ struct iavf_rx_queue {uint64_t hw_time_update; }; -struct iavf_tx_entry { -struct rte_mbuf *mbuf; -uint16_t next_id; -uint16_t last_id; -}; - -struct iavf_tx_vec_entry { -struct rte_mbuf *mbuf; -}; -/* Structure associated with each TX queue. */ struct iavf_tx_queue {const struct rte_memzone *mz;/* memzone for Tx ring */ volatile struct iavf_tx_desc *tx_ring;/* Tx ring virtual address */ uint64_t tx_ring_phys_addr;/* Tx ring DMA address */ -struct iavf_tx_entry *sw_ring;/* address array of SW ring */ +struct ci_tx_entry *sw_ring;/* address array of SW ring */ uint16_t nb_tx_desc;/* ring length */ uint16_t tx_tail;/* current value of tail */ volatile uint8_t *qtx_tail;/* register address of tail */
drivers/net/iavf/iavf_rxtx_vec_avx2.c +1 -1
@@ -1736,7 +1736,7 @@ iavf_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, {struct iavf_tx_queue *txq = (struct iavf_tx_queue *)tx_queue;volatile struct iavf_tx_desc *txdp; -struct iavf_tx_entry *txep; +struct ci_tx_entry *txep;uint16_t n, nb_commit, tx_id;/* bit2 is reserved and must be set to 1 according to Spec */ uint64_t flags =IAVF_TX_DESC_CMD_EOP |IAVF_TX_DESC_CMD_ICRC ;
drivers/net/iavf/iavf_rxtx_vec_avx512.c +5 -5
@@ -1847,7 +1847,7 @@ iavf_recv_scattered_pkts_vec_avx512_flex_rxd_offload(void *rx_queue, static __rte_always_inline intiavf_tx_free_bufs_avx512 (struct iavf_tx_queue *txq) { -struct iavf_tx_vec_entry *txep; +struct ci_tx_entry_vec *txep;uint32_t n;uint32_t i;int nb_free =0 ;
@@ -1960,7 +1960,7 @@ iavf_tx_free_bufs_avx512(struct iavf_tx_queue *txq) } static __rte_always_inline void -tx_backlog_entry_avx512 (struct iavf_tx_vec_entry *txep, +tx_backlog_entry_avx512 (struct ci_tx_entry_vec *txep,struct rte_mbuf **tx_pkts,uint16_t nb_pkts) {int i;
@@ -2313,7 +2313,7 @@ iavf_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, {struct iavf_tx_queue *txq = (struct iavf_tx_queue *)tx_queue;volatile struct iavf_tx_desc *txdp; -struct iavf_tx_vec_entry *txep; +struct ci_tx_entry_vec *txep;uint16_t n, nb_commit, tx_id;/* bit2 is reserved and must be set to 1 according to Spec */ uint64_t flags =IAVF_TX_DESC_CMD_EOP |IAVF_TX_DESC_CMD_ICRC ;
@@ -2380,7 +2380,7 @@ iavf_xmit_fixed_burst_vec_avx512_ctx(void *tx_queue, struct rte_mbuf **tx_pkts, {struct iavf_tx_queue *txq = (struct iavf_tx_queue *)tx_queue;volatile struct iavf_tx_desc *txdp; -struct iavf_tx_vec_entry *txep; +struct ci_tx_entry_vec *txep;uint16_t n, nb_commit, nb_mbuf, tx_id;/* bit2 is reserved and must be set to 1 according to Spec */ uint64_t flags =IAVF_TX_DESC_CMD_EOP |IAVF_TX_DESC_CMD_ICRC ;
@@ -2478,7 +2478,7 @@ iavf_tx_queue_release_mbufs_avx512(struct iavf_tx_queue *txq)const uint16_t max_desc = (uint16_t )(txq->nb_tx_desc -1 );const uint16_t end_desc = txq->tx_tail >> txq->use_ctx;/* next empty slot */ const uint16_t wrap_point = txq->nb_tx_desc >> txq->use_ctx;/* end of SW ring */ -struct iavf_tx_vec_entry *swr = (void *)txq->sw_ring; +struct ci_tx_entry_vec *swr = (void *)txq->sw_ring;if (!txq->sw_ring || txq->nb_free == max_desc)return ;
drivers/net/iavf/iavf_rxtx_vec_common.h +2 -2
@@ -19,7 +19,7 @@ static __rte_always_inline intiavf_tx_free_bufs (struct iavf_tx_queue *txq) { -struct iavf_tx_entry *txep; +struct ci_tx_entry *txep;uint32_t n;uint32_t i;int nb_free =0 ;
@@ -74,7 +74,7 @@ iavf_tx_free_bufs(struct iavf_tx_queue *txq) } static __rte_always_inline void -tx_backlog_entry (struct iavf_tx_entry *txep, +tx_backlog_entry (struct ci_tx_entry *txep,struct rte_mbuf **tx_pkts,uint16_t nb_pkts) {int i;
drivers/net/iavf/iavf_rxtx_vec_sse.c +1 -1
@@ -1368,7 +1368,7 @@ iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, {struct iavf_tx_queue *txq = (struct iavf_tx_queue *)tx_queue;volatile struct iavf_tx_desc *txdp; -struct iavf_tx_entry *txep; +struct ci_tx_entry *txep;uint16_t n, nb_commit, tx_id;uint64_t flags =IAVF_TX_DESC_CMD_EOP |0x04 ;/* bit 2 must be set */ uint64_t rs =IAVF_TX_DESC_CMD_RS | flags;
drivers/net/ice/ice_dcf_ethdev.c +1 -1
@@ -389,7 +389,7 @@ reset_rx_queue(struct ice_rx_queue *rxq)static inline void reset_tx_queue (struct ice_tx_queue *txq) { -struct ice_tx_entry *txe; +struct ci_tx_entry *txe;uint32_t i, size;uint16_t prev;
drivers/net/ice/ice_rxtx.c +8 -8
@@ -1028,7 +1028,7 @@ _ice_tx_queue_release_mbufs(struct ice_tx_queue *txq)static void ice_reset_tx_queue (struct ice_tx_queue *txq) { -struct ice_tx_entry *txe; +struct ci_tx_entry *txe;uint16_t i, prev, size;if (!txq) {
@@ -1509,7 +1509,7 @@ ice_tx_queue_setup(struct rte_eth_dev *dev,/* Allocate software ring */ txq->sw_ring =rte_zmalloc_socket (NULL , -sizeof (struct ice_tx_entry ) * nb_desc, +sizeof (struct ci_tx_entry ) * nb_desc,RTE_CACHE_LINE_SIZE , socket_id);if (!txq->sw_ring) {
@@ -2837,7 +2837,7 @@ ice_txd_enable_checksum(uint64_t ol_flags,static inline int ice_xmit_cleanup (struct ice_tx_queue *txq) { -struct ice_tx_entry *sw_ring = txq->sw_ring; +struct ci_tx_entry *sw_ring = txq->sw_ring;volatile struct ice_tx_desc *txd = txq->tx_ring;uint16_t last_desc_cleaned = txq->last_desc_cleaned;uint16_t nb_tx_desc = txq->nb_tx_desc;
@@ -2961,8 +2961,8 @@ ice_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)struct ice_tx_queue *txq;volatile struct ice_tx_desc *tx_ring;volatile struct ice_tx_desc *txd; -struct ice_tx_entry *sw_ring; -struct ice_tx_entry *txe, *txn; +struct ci_tx_entry *sw_ring; +struct ci_tx_entry *txe, *txn;struct rte_mbuf *tx_pkt;struct rte_mbuf *m_seg;uint32_t cd_tunneling_params;
@@ -3184,7 +3184,7 @@ ice_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) static __rte_always_inline intice_tx_free_bufs (struct ice_tx_queue *txq) { -struct ice_tx_entry *txep; +struct ci_tx_entry *txep;uint16_t i; if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &
@@ -3221,7 +3221,7 @@ static int ice_tx_done_cleanup_full(struct ice_tx_queue *txq, uint32_t free_cnt) { -struct ice_tx_entry *swr_ring = txq->sw_ring; +struct ci_tx_entry *swr_ring = txq->sw_ring;uint16_t i, tx_last, tx_id;uint16_t nb_tx_free_last;uint16_t nb_tx_to_clean;
@@ -3361,7 +3361,7 @@ ice_tx_fill_hw_ring(struct ice_tx_queue *txq, struct rte_mbuf **pkts, uint16_t nb_pkts) {volatile struct ice_tx_desc *txdp = &txq->tx_ring[txq->tx_tail]; -struct ice_tx_entry *txep = &txq->sw_ring[txq->tx_tail]; +struct ci_tx_entry *txep = &txq->sw_ring[txq->tx_tail];const int N_PER_LOOP =4 ;const int N_PER_LOOP_MASK =N_PER_LOOP -1 ;int mainpart, leftover;
drivers/net/ice/ice_rxtx.h +2 -11
@@ -5,6 +5,7 @@#ifndef _ICE_RXTX_H_#define _ICE_RXTX_H_ +#include <_common_intel/tx.h> #include "ice_ethdev.h" #define ICE_ALIGN_RING_DESC 32
@@ -144,21 +145,11 @@ struct ice_rx_queue {bool ts_enable;/* if rxq timestamp is enabled */ }; -struct ice_tx_entry { -struct rte_mbuf *mbuf; -uint16_t next_id; -uint16_t last_id; -}; - -struct ice_vec_tx_entry { -struct rte_mbuf *mbuf; -}; -struct ice_tx_queue {uint16_t nb_tx_desc;/* number of TX descriptors */ rte_iova_t tx_ring_dma;/* TX ring DMA address */ volatile struct ice_tx_desc *tx_ring;/* TX ring virtual address */ -struct ice_tx_entry *sw_ring;/* virtual address of SW ring */ +struct ci_tx_entry *sw_ring;/* virtual address of SW ring */ uint16_t tx_tail;/* current value of tail register */ volatile uint8_t *qtx_tail;/* register address of tail */ uint16_t nb_tx_used;/* number of TX desc used since RS bit set */
drivers/net/ice/ice_rxtx_vec_avx2.c +1 -1
@@ -858,7 +858,7 @@ ice_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, {struct ice_tx_queue *txq = (struct ice_tx_queue *)tx_queue;volatile struct ice_tx_desc *txdp; -struct ice_tx_entry *txep; +struct ci_tx_entry *txep;uint16_t n, nb_commit, tx_id;uint64_t flags =ICE_TD_CMD ;uint64_t rs =ICE_TX_DESC_CMD_RS |ICE_TD_CMD ;
drivers/net/ice/ice_rxtx_vec_avx512.c +3 -3
@@ -862,7 +862,7 @@ ice_recv_scattered_pkts_vec_avx512_offload(void *rx_queue, static __rte_always_inline intice_tx_free_bufs_avx512 (struct ice_tx_queue *txq) { -struct ice_vec_tx_entry *txep; +struct ci_tx_entry_vec *txep;uint32_t n;uint32_t i;int nb_free =0 ;
@@ -1040,7 +1040,7 @@ ice_vtx(volatile struct ice_tx_desc *txdp, struct rte_mbuf **pkt, } static __rte_always_inline void -ice_tx_backlog_entry_avx512 (struct ice_vec_tx_entry *txep, +ice_tx_backlog_entry_avx512 (struct ci_tx_entry_vec *txep,struct rte_mbuf **tx_pkts,uint16_t nb_pkts) {int i;
@@ -1055,7 +1055,7 @@ ice_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, {struct ice_tx_queue *txq = (struct ice_tx_queue *)tx_queue;volatile struct ice_tx_desc *txdp; -struct ice_vec_tx_entry *txep; +struct ci_tx_entry_vec *txep;uint16_t n, nb_commit, tx_id;uint64_t flags =ICE_TD_CMD ;uint64_t rs =ICE_TX_DESC_CMD_RS |ICE_TD_CMD ;
drivers/net/ice/ice_rxtx_vec_common.h +3 -3
@@ -15,7 +15,7 @@ static __rte_always_inline intice_tx_free_bufs_vec (struct ice_tx_queue *txq) { -struct ice_tx_entry *txep; +struct ci_tx_entry *txep;uint32_t n;uint32_t i;int nb_free =0 ;
@@ -70,7 +70,7 @@ ice_tx_free_bufs_vec(struct ice_tx_queue *txq) } static __rte_always_inline void -ice_tx_backlog_entry (struct ice_tx_entry *txep, +ice_tx_backlog_entry (struct ci_tx_entry *txep,struct rte_mbuf **tx_pkts,uint16_t nb_pkts) {int i;
@@ -135,7 +135,7 @@ _ice_tx_queue_release_mbufs_vec(struct ice_tx_queue *txq)if (dev->tx_pkt_burst == ice_xmit_pkts_vec_avx512 || dev->tx_pkt_burst == ice_xmit_pkts_vec_avx512_offload) { -struct ice_vec_tx_entry *swr = (void *)txq->sw_ring; +struct ci_tx_entry_vec *swr = (void *)txq->sw_ring;if (txq->tx_tail < i) {for (; i < txq->nb_tx_desc; i++) {
drivers/net/ice/ice_rxtx_vec_sse.c +1 -1
@@ -699,7 +699,7 @@ ice_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, {struct ice_tx_queue *txq = (struct ice_tx_queue *)tx_queue;volatile struct ice_tx_desc *txdp; -struct ice_tx_entry *txep; +struct ci_tx_entry *txep;uint16_t n, nb_commit, tx_id;uint64_t flags =ICE_TD_CMD ;uint64_t rs =ICE_TX_DESC_CMD_RS |ICE_TD_CMD ;
drivers/net/ixgbe/ixgbe_recycle_mbufs_vec_common.c +1 -1
@@ -52,7 +52,7 @@ ixgbe_recycle_tx_mbufs_reuse_vec(void *tx_queue, struct rte_eth_recycle_rxq_info *recycle_rxq_info) {struct ixgbe_tx_queue *txq = tx_queue; -struct ixgbe_tx_entry *txep; +struct ci_tx_entry *txep;struct rte_mbuf **rxep;int i, n;uint32_t status;
drivers/net/ixgbe/ixgbe_rxtx.c +8 -8
@@ -100,7 +100,7 @@ static __rte_always_inline intixgbe_tx_free_bufs (struct ixgbe_tx_queue *txq) { -struct ixgbe_tx_entry *txep; +struct ci_tx_entry *txep;uint32_t status;int i, nb_free =0 ;struct rte_mbuf *m, *free[RTE_IXGBE_TX_MAX_FREE_BUF_SZ ];
@@ -199,7 +199,7 @@ ixgbe_tx_fill_hw_ring(struct ixgbe_tx_queue *txq, struct rte_mbuf **pkts, uint16_t nb_pkts) {volatile union ixgbe_adv_tx_desc *txdp = &(txq->tx_ring[txq->tx_tail]); -struct ixgbe_tx_entry *txep = &(txq->sw_ring[txq->tx_tail]); +struct ci_tx_entry *txep = &txq->sw_ring[txq->tx_tail];const int N_PER_LOOP =4 ;const int N_PER_LOOP_MASK =N_PER_LOOP -1 ;int mainpart, leftover;
@@ -563,7 +563,7 @@ tx_desc_ol_flags_to_cmdtype(uint64_t ol_flags)static inline int ixgbe_xmit_cleanup (struct ixgbe_tx_queue *txq) { -struct ixgbe_tx_entry *sw_ring = txq->sw_ring; +struct ci_tx_entry *sw_ring = txq->sw_ring;volatile union ixgbe_adv_tx_desc *txr = txq->tx_ring;uint16_t last_desc_cleaned = txq->last_desc_cleaned;uint16_t nb_tx_desc = txq->nb_tx_desc;
@@ -624,8 +624,8 @@ ixgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) {struct ixgbe_tx_queue *txq; -struct ixgbe_tx_entry *sw_ring; -struct ixgbe_tx_entry *txe, *txn; +struct ci_tx_entry *sw_ring; +struct ci_tx_entry *txe, *txn;volatile union ixgbe_adv_tx_desc *txr;volatile union ixgbe_adv_tx_desc *txd, *txp;struct rte_mbuf *tx_pkt;
@@ -2352,7 +2352,7 @@ ixgbe_tx_queue_release_mbufs(struct ixgbe_tx_queue *txq)static int ixgbe_tx_done_cleanup_full (struct ixgbe_tx_queue *txq,uint32_t free_cnt) { -struct ixgbe_tx_entry *swr_ring = txq->sw_ring; +struct ci_tx_entry *swr_ring = txq->sw_ring;uint16_t i, tx_last, tx_id;uint16_t nb_tx_free_last;uint16_t nb_tx_to_clean;
@@ -2490,7 +2490,7 @@ static void __rte_cold ixgbe_reset_tx_queue(struct ixgbe_tx_queue *txq) {static const union ixgbe_adv_tx_desc zeroed_desc = {{0 }}; -struct ixgbe_tx_entry *txe = txq->sw_ring; +struct ci_tx_entry *txe = txq->sw_ring;uint16_t prev, i;/* Zero out HW ring memory */
@@ -2795,7 +2795,7 @@ ixgbe_dev_tx_queue_setup(struct rte_eth_dev *dev,/* Allocate software ring */ txq->sw_ring =rte_zmalloc_socket ("txq->sw_ring" , -sizeof (struct ixgbe_tx_entry ) * nb_desc, +sizeof (struct ci_tx_entry ) * nb_desc,RTE_CACHE_LINE_SIZE , socket_id);if (txq->sw_ring ==NULL ) {ixgbe_tx_queue_release (txq);
drivers/net/ixgbe/ixgbe_rxtx.h +4 -18
@@ -5,6 +5,8 @@#ifndef _IXGBE_RXTX_H_#define _IXGBE_RXTX_H_ +#include <_common_intel/tx.h> +/* * Rings setup and release. *
@@ -75,22 +77,6 @@ struct ixgbe_scattered_rx_entry {struct rte_mbuf *fbuf;/**< First segment of the fragmented packet. */ }; -/** -* Structure associated with each descriptor of the TX ring of a TX queue. -*/ -struct ixgbe_tx_entry { -struct rte_mbuf *mbuf;/**< mbuf associated with TX desc, if any. */ -uint16_t next_id;/**< Index of next descriptor in ring. */ -uint16_t last_id;/**< Index of last scattered descriptor. */ -}; - -/** -* Structure associated with each descriptor of the TX ring of a TX queue. -*/ -struct ixgbe_tx_entry_v { -struct rte_mbuf *mbuf;/**< mbuf associated with TX desc, if any. */ -}; -/** * Structure associated with each RX queue. */
@@ -202,8 +188,8 @@ struct ixgbe_tx_queue {volatile union ixgbe_adv_tx_desc *tx_ring;uint64_t tx_ring_phys_addr;/**< TX ring DMA address. */ union { -struct ixgbe_tx_entry *sw_ring;/**< address of SW ring for scalar PMD. */ -struct ixgbe_tx_entry_v *sw_ring_v;/**< address of SW ring for vector PMD */ +struct ci_tx_entry *sw_ring;/**< address of SW ring for scalar PMD. */ +struct ci_tx_entry_vec *sw_ring_v;/**< address of SW ring for vector PMD */ };volatile uint32_t *tdt_reg_addr;/**< Address of TDT register. */ uint16_t nb_tx_desc;/**< number of TX descriptors. */
drivers/net/ixgbe/ixgbe_rxtx_vec_common.h +4 -4
@@ -14,7 +14,7 @@ static __rte_always_inline intixgbe_tx_free_bufs (struct ixgbe_tx_queue *txq) { -struct ixgbe_tx_entry_v *txep; +struct ci_tx_entry_vec *txep;uint32_t status;uint32_t n;uint32_t i;
@@ -69,7 +69,7 @@ ixgbe_tx_free_bufs(struct ixgbe_tx_queue *txq) } static __rte_always_inline void -tx_backlog_entry (struct ixgbe_tx_entry_v *txep, +tx_backlog_entry (struct ci_tx_entry_vec *txep,struct rte_mbuf **tx_pkts,uint16_t nb_pkts) {int i;
@@ -82,7 +82,7 @@ static inline void _ixgbe_tx_queue_release_mbufs_vec(struct ixgbe_tx_queue *txq) {unsigned int i; -struct ixgbe_tx_entry_v *txe; +struct ci_tx_entry_vec *txe;const uint16_t max_desc = (uint16_t )(txq->nb_tx_desc -1 );if (txq->sw_ring ==NULL || txq->nb_tx_free == max_desc)
@@ -149,7 +149,7 @@ static inline void _ixgbe_reset_tx_queue_vec(struct ixgbe_tx_queue *txq) {static const union ixgbe_adv_tx_desc zeroed_desc = { {0 } }; -struct ixgbe_tx_entry_v *txe = txq->sw_ring_v; +struct ci_tx_entry_vec *txe = txq->sw_ring_v;uint16_t i;/* Zero out HW ring memory */
drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c +1 -1
@@ -573,7 +573,7 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, {struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue;volatile union ixgbe_adv_tx_desc *txdp; -struct ixgbe_tx_entry_v *txep; +struct ci_tx_entry_vec *txep;uint16_t n, nb_commit, tx_id;uint64_t flags =DCMD_DTYP_FLAGS ;uint64_t rs =IXGBE_ADVTXD_DCMD_RS |DCMD_DTYP_FLAGS ;
drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c +1 -1
@@ -695,7 +695,7 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, {struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue;volatile union ixgbe_adv_tx_desc *txdp; -struct ixgbe_tx_entry_v *txep; +struct ci_tx_entry_vec *txep;uint16_t n, nb_commit, tx_id;uint64_t flags =DCMD_DTYP_FLAGS ;uint64_t rs =IXGBE_ADVTXD_DCMD_RS |DCMD_DTYP_FLAGS ;
Signature · 1 line
2.43.0
PATCH v3 3/22 net/_common_intel: add Tx mbuf ring replenish fn
In reply to Bruce RichardsonMove the short function used to place mbufs on the SW Tx ring to common code to avoid duplication.
--- drivers/net/_common_intel/tx.h | 7 +++++++ drivers/net/i40e/i40e_rxtx_vec_altivec.c | 4 ++-- drivers/net/i40e/i40e_rxtx_vec_avx2.c | 4 ++-- drivers/net/i40e/i40e_rxtx_vec_common.h | 10 ---------- drivers/net/i40e/i40e_rxtx_vec_neon.c | 4 ++-- drivers/net/i40e/i40e_rxtx_vec_sse.c | 4 ++-- drivers/net/iavf/iavf_rxtx_vec_avx2.c | 4 ++-- drivers/net/iavf/iavf_rxtx_vec_common.h | 10 ---------- drivers/net/iavf/iavf_rxtx_vec_sse.c | 4 ++-- drivers/net/ice/ice_rxtx_vec_avx2.c | 4 ++-- drivers/net/ice/ice_rxtx_vec_common.h | 10 ---------- drivers/net/ice/ice_rxtx_vec_sse.c | 4 ++-- 12 files changed, 23 insertions(+), 46 deletions(-)
drivers/net/_common_intel/tx.h +7 -0
@@ -24,4 +24,11 @@ struct ci_tx_entry_vec {struct rte_mbuf *mbuf;/* mbuf associated with TX desc, if any. */ }; +static __rte_always_inline void +ci_tx_backlog_entry (struct ci_tx_entry *txep,struct rte_mbuf **tx_pkts,uint16_t nb_pkts) +{ +for (uint16_t i =0 ; i < (int )nb_pkts; ++i) + txep[i].mbuf = tx_pkts[i]; +} +#endif /* _COMMON_INTEL_TX_H_ */
drivers/net/i40e/i40e_rxtx_vec_altivec.c +2 -2
@@ -575,7 +575,7 @@ i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) { -tx_backlog_entry (txep, tx_pkts, n); +ci_tx_backlog_entry (txep, tx_pkts, n);for (i =0 ; i < n -1 ; ++i, ++tx_pkts, ++txdp)vtx1 (txdp, *tx_pkts, flags);
@@ -592,7 +592,7 @@ i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, txep = &txq->sw_ring[tx_id]; } -tx_backlog_entry (txep, tx_pkts, nb_commit); +ci_tx_backlog_entry (txep, tx_pkts, nb_commit);vtx (txdp, tx_pkts, nb_commit, flags);
drivers/net/i40e/i40e_rxtx_vec_avx2.c +2 -2
@@ -765,7 +765,7 @@ i40e_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) { -tx_backlog_entry (txep, tx_pkts, n); +ci_tx_backlog_entry (txep, tx_pkts, n);vtx (txdp, tx_pkts, n -1 , flags); tx_pkts += (n -1 );
@@ -783,7 +783,7 @@ i40e_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, txep = &txq->sw_ring[tx_id]; } -tx_backlog_entry (txep, tx_pkts, nb_commit); +ci_tx_backlog_entry (txep, tx_pkts, nb_commit);vtx (txdp, tx_pkts, nb_commit, flags);
drivers/net/i40e/i40e_rxtx_vec_common.h +0 -10
@@ -84,16 +84,6 @@ i40e_tx_free_bufs(struct i40e_tx_queue *txq)return txq->tx_rs_thresh; } -static __rte_always_inline void -tx_backlog_entry (struct ci_tx_entry *txep, -struct rte_mbuf **tx_pkts,uint16_t nb_pkts) -{ -int i; - -for (i =0 ; i < (int )nb_pkts; ++i) - txep[i].mbuf = tx_pkts[i]; -} -static inline void _i40e_rx_queue_release_mbufs_vec (struct i40e_rx_queue *rxq) {
drivers/net/i40e/i40e_rxtx_vec_neon.c +2 -2
@@ -702,7 +702,7 @@ i40e_xmit_fixed_burst_vec(void *__rte_restrict tx_queue, n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) { -tx_backlog_entry (txep, tx_pkts, n); +ci_tx_backlog_entry (txep, tx_pkts, n);for (i =0 ; i < n -1 ; ++i, ++tx_pkts, ++txdp)vtx1 (txdp, *tx_pkts, flags);
@@ -719,7 +719,7 @@ i40e_xmit_fixed_burst_vec(void *__rte_restrict tx_queue, txep = &txq->sw_ring[tx_id]; } -tx_backlog_entry (txep, tx_pkts, nb_commit); +ci_tx_backlog_entry (txep, tx_pkts, nb_commit);vtx (txdp, tx_pkts, nb_commit, flags);
drivers/net/i40e/i40e_rxtx_vec_sse.c +2 -2
@@ -721,7 +721,7 @@ i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) { -tx_backlog_entry (txep, tx_pkts, n); +ci_tx_backlog_entry (txep, tx_pkts, n);for (i =0 ; i < n -1 ; ++i, ++tx_pkts, ++txdp)vtx1 (txdp, *tx_pkts, flags);
@@ -738,7 +738,7 @@ i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, txep = &txq->sw_ring[tx_id]; } -tx_backlog_entry (txep, tx_pkts, nb_commit); +ci_tx_backlog_entry (txep, tx_pkts, nb_commit);vtx (txdp, tx_pkts, nb_commit, flags);
drivers/net/iavf/iavf_rxtx_vec_avx2.c +2 -2
@@ -1757,7 +1757,7 @@ iavf_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) { -tx_backlog_entry (txep, tx_pkts, n); +ci_tx_backlog_entry (txep, tx_pkts, n);iavf_vtx (txdp, tx_pkts, n -1 , flags, offload); tx_pkts += (n -1 );
@@ -1775,7 +1775,7 @@ iavf_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, txep = &txq->sw_ring[tx_id]; } -tx_backlog_entry (txep, tx_pkts, nb_commit); +ci_tx_backlog_entry (txep, tx_pkts, nb_commit);iavf_vtx (txdp, tx_pkts, nb_commit, flags, offload);
drivers/net/iavf/iavf_rxtx_vec_common.h +0 -10
@@ -73,16 +73,6 @@ iavf_tx_free_bufs(struct iavf_tx_queue *txq)return txq->rs_thresh; } -static __rte_always_inline void -tx_backlog_entry (struct ci_tx_entry *txep, -struct rte_mbuf **tx_pkts,uint16_t nb_pkts) -{ -int i; - -for (i =0 ; i < (int )nb_pkts; ++i) - txep[i].mbuf = tx_pkts[i]; -} -static inline void _iavf_rx_queue_release_mbufs_vec (struct iavf_rx_queue *rxq) {
drivers/net/iavf/iavf_rxtx_vec_sse.c +2 -2
@@ -1390,7 +1390,7 @@ iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) { -tx_backlog_entry (txep, tx_pkts, n); +ci_tx_backlog_entry (txep, tx_pkts, n);for (i =0 ; i < n -1 ; ++i, ++tx_pkts, ++txdp)vtx1 (txdp, *tx_pkts, flags);
@@ -1407,7 +1407,7 @@ iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, txep = &txq->sw_ring[tx_id]; } -tx_backlog_entry (txep, tx_pkts, nb_commit); +ci_tx_backlog_entry (txep, tx_pkts, nb_commit);iavf_vtx (txdp, tx_pkts, nb_commit, flags);
drivers/net/ice/ice_rxtx_vec_avx2.c +2 -2
@@ -881,7 +881,7 @@ ice_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) { -ice_tx_backlog_entry (txep, tx_pkts, n); +ci_tx_backlog_entry (txep, tx_pkts, n);ice_vtx (txdp, tx_pkts, n -1 , flags, offload); tx_pkts += (n -1 );
@@ -899,7 +899,7 @@ ice_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, txep = &txq->sw_ring[tx_id]; } -ice_tx_backlog_entry (txep, tx_pkts, nb_commit); +ci_tx_backlog_entry (txep, tx_pkts, nb_commit);ice_vtx (txdp, tx_pkts, nb_commit, flags, offload);
drivers/net/ice/ice_rxtx_vec_common.h +0 -10
@@ -69,16 +69,6 @@ ice_tx_free_bufs_vec(struct ice_tx_queue *txq)return txq->tx_rs_thresh; } -static __rte_always_inline void -ice_tx_backlog_entry (struct ci_tx_entry *txep, -struct rte_mbuf **tx_pkts,uint16_t nb_pkts) -{ -int i; - -for (i =0 ; i < (int )nb_pkts; ++i) - txep[i].mbuf = tx_pkts[i]; -} -static inline void _ice_rx_queue_release_mbufs_vec (struct ice_rx_queue *rxq) {
drivers/net/ice/ice_rxtx_vec_sse.c +2 -2
@@ -724,7 +724,7 @@ ice_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) { -ice_tx_backlog_entry (txep, tx_pkts, n); +ci_tx_backlog_entry (txep, tx_pkts, n);for (i =0 ; i < n -1 ; ++i, ++tx_pkts, ++txdp)ice_vtx1 (txdp, *tx_pkts, flags);
@@ -741,7 +741,7 @@ ice_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, txep = &txq->sw_ring[tx_id]; } -ice_tx_backlog_entry (txep, tx_pkts, nb_commit); +ci_tx_backlog_entry (txep, tx_pkts, nb_commit);ice_vtx (txdp, tx_pkts, nb_commit, flags);
Signature · 1 line
2.43.0
PATCH v3 4/22 drivers/net: align Tx queue struct field names
In reply to Bruce RichardsonAcross the various Intel drivers sometimes different names are given to fields in the Tx queue structure which have the same function. Do some renaming to align things better for future merging.
--- drivers/net/i40e/i40e_rxtx.c | 6 +-- drivers/net/i40e/i40e_rxtx.h | 2 +- drivers/net/iavf/iavf_rxtx.c | 60 ++++++++++++------------- drivers/net/iavf/iavf_rxtx.h | 14 +++--- drivers/net/iavf/iavf_rxtx_vec_avx2.c | 19 ++++---- drivers/net/iavf/iavf_rxtx_vec_avx512.c | 57 +++++++++++------------ drivers/net/iavf/iavf_rxtx_vec_common.h | 24 +++++----- drivers/net/iavf/iavf_rxtx_vec_sse.c | 18 ++++---- drivers/net/iavf/iavf_vchnl.c | 2 +- drivers/net/ixgbe/base/ixgbe_osdep.h | 2 +- drivers/net/ixgbe/ixgbe_rxtx.c | 16 +++---- drivers/net/ixgbe/ixgbe_rxtx.h | 6 +-- drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c | 2 +- drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c | 2 +- 14 files changed, 116 insertions(+), 114 deletions(-)
drivers/net/i40e/i40e_rxtx.c +3 -3
@@ -2549,7 +2549,7 @@ i40e_dev_tx_queue_setup(struct rte_eth_dev *dev, txq->vsi = vsi; txq->tx_deferred_start = tx_conf->tx_deferred_start; - txq->tx_ring_phys_addr = tz->iova; + txq->tx_ring_dma = tz->iova; txq->tx_ring = (struct i40e_tx_desc *)tz->addr;/* Allocate software ring */
@@ -2923,7 +2923,7 @@ i40e_tx_queue_init(struct i40e_tx_queue *txq)/* clear the context structure first */ memset (&tx_ctx,0 ,sizeof (tx_ctx)); tx_ctx.new_context =1 ; - tx_ctx.base = txq->tx_ring_phys_addr /I40E_QUEUE_BASE_ADDR_UNIT ; + tx_ctx.base = txq->tx_ring_dma /I40E_QUEUE_BASE_ADDR_UNIT ; tx_ctx.qlen = txq->nb_tx_desc;#ifdef RTE_LIBRTE_IEEE1588
@@ -3209,7 +3209,7 @@ i40e_fdir_setup_tx_resources(struct i40e_pf *pf) txq->reg_idx = pf->fdir.fdir_vsi->base_queue; txq->vsi = pf->fdir.fdir_vsi; - txq->tx_ring_phys_addr = tz->iova; + txq->tx_ring_dma = tz->iova; txq->tx_ring = (struct i40e_tx_desc *)tz->addr;/*
drivers/net/i40e/i40e_rxtx.h +1 -1
@@ -129,7 +129,7 @@ struct i40e_rx_queue {*/ struct i40e_tx_queue {uint16_t nb_tx_desc;/**< number of TX descriptors */ -uint64_t tx_ring_phys_addr;/**< TX ring DMA address */ +rte_iova_t tx_ring_dma;/**< TX ring DMA address */ volatile struct i40e_tx_desc *tx_ring;/**< TX ring virtual address */ struct ci_tx_entry *sw_ring;/**< virtual address of SW ring */ uint16_t tx_tail;/**< current value of tail register */
drivers/net/iavf/iavf_rxtx.c +30 -30
161 lines not on this page · raw message
drivers/net/iavf/iavf_rxtx.h +7 -7
@@ -277,25 +277,25 @@ struct iavf_rx_queue {struct iavf_tx_queue {const struct rte_memzone *mz;/* memzone for Tx ring */ volatile struct iavf_tx_desc *tx_ring;/* Tx ring virtual address */ -uint64_t tx_ring_phys_addr;/* Tx ring DMA address */ +rte_iova_t tx_ring_dma;/* Tx ring DMA address */ struct ci_tx_entry *sw_ring;/* address array of SW ring */ uint16_t nb_tx_desc;/* ring length */ uint16_t tx_tail;/* current value of tail */ volatile uint8_t *qtx_tail;/* register address of tail */ /* number of used desc since RS bit set */ -uint16_t nb_used; -uint16_t nb_free; +uint16_t nb_tx_used; +uint16_t nb_tx_free;uint16_t last_desc_cleaned;/* last desc have been cleaned*/ -uint16_t free_thresh; -uint16_t rs_thresh; +uint16_t tx_free_thresh; +uint16_t tx_rs_thresh;uint8_t rel_mbufs_type;struct iavf_vsi *vsi;/**< the VSI this queue belongs to */ uint16_t port_id;uint16_t queue_id;uint64_t offloads; -uint16_t next_dd;/* next to set RS, for VPMD */ -uint16_t next_rs;/* next to check DD, for VPMD */ +uint16_t tx_next_dd;/* next to set RS, for VPMD */ +uint16_t tx_next_rs;/* next to check DD, for VPMD */ uint16_t ipsec_crypto_pkt_md_offset;uint64_t mbuf_errors;
drivers/net/iavf/iavf_rxtx_vec_avx2.c +10 -9
@@ -1742,18 +1742,19 @@ iavf_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts,uint64_t flags =IAVF_TX_DESC_CMD_EOP |IAVF_TX_DESC_CMD_ICRC ;uint64_t rs =IAVF_TX_DESC_CMD_RS | flags; -if (txq->nb_free < txq->free_thresh) +if (txq->nb_tx_free < txq->tx_free_thresh)iavf_tx_free_bufs (txq); - nb_commit = nb_pkts = (uint16_t )RTE_MIN (txq->nb_free, nb_pkts); + nb_pkts = (uint16_t )RTE_MIN (txq->nb_tx_free, nb_pkts);if (unlikely (nb_pkts ==0 ))return 0 ; + nb_commit = nb_pkts; tx_id = txq->tx_tail; txdp = &txq->tx_ring[tx_id]; txep = &txq->sw_ring[tx_id]; - txq->nb_free = (uint16_t )(txq->nb_free - nb_pkts); + txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts); n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) {
@@ -1768,7 +1769,7 @@ iavf_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, nb_commit = (uint16_t )(nb_commit - n); tx_id =0 ; - txq->next_rs = (uint16_t )(txq->rs_thresh -1 ); + txq->tx_next_rs = (uint16_t )(txq->tx_rs_thresh -1 );/* avoid reach the end of ring */ txdp = &txq->tx_ring[tx_id];
@@ -1780,12 +1781,12 @@ iavf_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts,iavf_vtx (txdp, tx_pkts, nb_commit, flags, offload); tx_id = (uint16_t )(tx_id + nb_commit); -if (tx_id > txq->next_rs) { - txq->tx_ring[txq->next_rs].cmd_type_offset_bsz |= +if (tx_id > txq->tx_next_rs) { + txq->tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |=rte_cpu_to_le_64 (((uint64_t )IAVF_TX_DESC_CMD_RS ) <<IAVF_TXD_QW1_CMD_SHIFT ); - txq->next_rs = - (uint16_t )(txq->next_rs + txq->rs_thresh); + txq->tx_next_rs = + (uint16_t )(txq->tx_next_rs + txq->tx_rs_thresh); } txq->tx_tail = tx_id;
@@ -1806,7 +1807,7 @@ iavf_xmit_pkts_vec_avx2_common(void *tx_queue, struct rte_mbuf **tx_pkts,uint16_t ret, num;/* cross rs_thresh boundary is not allowed */ - num = (uint16_t )RTE_MIN (nb_pkts, txq->rs_thresh); + num = (uint16_t )RTE_MIN (nb_pkts, txq->tx_rs_thresh); ret =iavf_xmit_fixed_burst_vec_avx2 (tx_queue, &tx_pkts[nb_tx], num, offload); nb_tx += ret;
drivers/net/iavf/iavf_rxtx_vec_avx512.c +29 -28
157 lines not on this page · raw message
drivers/net/iavf/iavf_rxtx_vec_common.h +12 -12
@@ -26,17 +26,17 @@ iavf_tx_free_bufs(struct iavf_tx_queue *txq)struct rte_mbuf *m, *free[IAVF_VPMD_TX_MAX_FREE_BUF ];/* check DD bits on threshold descriptor */ -if ((txq->tx_ring[txq->next_dd].cmd_type_offset_bsz & +if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &rte_cpu_to_le_64 (IAVF_TXD_QW1_DTYPE_MASK )) !=rte_cpu_to_le_64 (IAVF_TX_DESC_DTYPE_DESC_DONE ))return 0 ; - n = txq->rs_thresh; + n = txq->tx_rs_thresh;/* first buffer to free from S/W ring is at index * tx_next_dd - (tx_rs_thresh-1) */ - txep = &txq->sw_ring[txq->next_dd - (n -1 )]; + txep = &txq->sw_ring[txq->tx_next_dd - (n -1 )]; m =rte_pktmbuf_prefree_seg (txep[0 ].mbuf);if (likely (m !=NULL )) { free[0 ] = m;
@@ -65,12 +65,12 @@ iavf_tx_free_bufs(struct iavf_tx_queue *txq) }/* buffers were freed, update counters */ - txq->nb_free = (uint16_t )(txq->nb_free + txq->rs_thresh); - txq->next_dd = (uint16_t )(txq->next_dd + txq->rs_thresh); -if (txq->next_dd >= txq->nb_tx_desc) - txq->next_dd = (uint16_t )(txq->rs_thresh -1 ); + txq->nb_tx_free = (uint16_t )(txq->nb_tx_free + txq->tx_rs_thresh); + txq->tx_next_dd = (uint16_t )(txq->tx_next_dd + txq->tx_rs_thresh); +if (txq->tx_next_dd >= txq->nb_tx_desc) + txq->tx_next_dd = (uint16_t )(txq->tx_rs_thresh -1 ); -return txq->rs_thresh; +return txq->tx_rs_thresh; } static inline void
@@ -109,10 +109,10 @@ _iavf_tx_queue_release_mbufs_vec(struct iavf_tx_queue *txq)unsigned i;const uint16_t max_desc = (uint16_t )(txq->nb_tx_desc -1 ); -if (!txq->sw_ring || txq->nb_free == max_desc) +if (!txq->sw_ring || txq->nb_tx_free == max_desc)return ; - i = txq->next_dd - txq->rs_thresh +1 ; + i = txq->tx_next_dd - txq->tx_rs_thresh +1 ;while (i != txq->tx_tail) {rte_pktmbuf_free_seg (txq->sw_ring[i].mbuf); txq->sw_ring[i].mbuf =NULL ;
@@ -169,8 +169,8 @@ iavf_tx_vec_queue_default(struct iavf_tx_queue *txq)if (!txq)return -1 ; -if (txq->rs_thresh <IAVF_VPMD_TX_MAX_BURST || - txq->rs_thresh >IAVF_VPMD_TX_MAX_FREE_BUF ) +if (txq->tx_rs_thresh <IAVF_VPMD_TX_MAX_BURST || + txq->tx_rs_thresh >IAVF_VPMD_TX_MAX_FREE_BUF )return -1 ;if (txq->offloads &IAVF_TX_NO_VECTOR_FLAGS )
drivers/net/iavf/iavf_rxtx_vec_sse.c +9 -9
@@ -1374,10 +1374,10 @@ iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,uint64_t rs =IAVF_TX_DESC_CMD_RS | flags;int i; -if (txq->nb_free < txq->free_thresh) +if (txq->nb_tx_free < txq->tx_free_thresh)iavf_tx_free_bufs (txq); - nb_pkts = (uint16_t )RTE_MIN (txq->nb_free, nb_pkts); + nb_pkts = (uint16_t )RTE_MIN (txq->nb_tx_free, nb_pkts);if (unlikely (nb_pkts ==0 ))return 0 ; nb_commit = nb_pkts;
@@ -1386,7 +1386,7 @@ iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, txdp = &txq->tx_ring[tx_id]; txep = &txq->sw_ring[tx_id]; - txq->nb_free = (uint16_t )(txq->nb_free - nb_pkts); + txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts); n = (uint16_t )(txq->nb_tx_desc - tx_id);if (nb_commit >= n) {
@@ -1400,7 +1400,7 @@ iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, nb_commit = (uint16_t )(nb_commit - n); tx_id =0 ; - txq->next_rs = (uint16_t )(txq->rs_thresh -1 ); + txq->tx_next_rs = (uint16_t )(txq->tx_rs_thresh -1 );/* avoid reach the end of ring */ txdp = &txq->tx_ring[tx_id];
@@ -1412,12 +1412,12 @@ iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,iavf_vtx (txdp, tx_pkts, nb_commit, flags); tx_id = (uint16_t )(tx_id + nb_commit); -if (tx_id > txq->next_rs) { - txq->tx_ring[txq->next_rs].cmd_type_offset_bsz |= +if (tx_id > txq->tx_next_rs) { + txq->tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |=rte_cpu_to_le_64 (((uint64_t )IAVF_TX_DESC_CMD_RS ) <<IAVF_TXD_QW1_CMD_SHIFT ); - txq->next_rs = - (uint16_t )(txq->next_rs + txq->rs_thresh); + txq->tx_next_rs = + (uint16_t )(txq->tx_next_rs + txq->tx_rs_thresh); } txq->tx_tail = tx_id;
@@ -1441,7 +1441,7 @@ iavf_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,uint16_t ret, num;/* cross rs_thresh boundary is not allowed */ - num = (uint16_t )RTE_MIN (nb_pkts, txq->rs_thresh); + num = (uint16_t )RTE_MIN (nb_pkts, txq->tx_rs_thresh); ret =iavf_xmit_fixed_burst_vec (tx_queue, &tx_pkts[nb_tx], num); nb_tx += ret; nb_pkts -= ret;
drivers/net/iavf/iavf_vchnl.c +1 -1
@@ -1247,7 +1247,7 @@ iavf_configure_queues(struct iavf_adapter *adapter,/* Virtchnnl configure tx queues by pairs */ if (i < adapter->dev_data->nb_tx_queues) { vc_qp->txq.ring_len = txq[i]->nb_tx_desc; - vc_qp->txq.dma_ring_addr = txq[i]->tx_ring_phys_addr; + vc_qp->txq.dma_ring_addr = txq[i]->tx_ring_dma; } vc_qp->rxq.vsi_id = vf->vsi_res->vsi_id;
drivers/net/ixgbe/base/ixgbe_osdep.h +1 -1
@@ -124,7 +124,7 @@ static inline uint32_t ixgbe_read_addr(volatile void* addr) rte_write32_wc_relaxed((rte_cpu_to_le_32(value)), reg)#define IXGBE_PCI_REG_ADDR (hw, reg) \ - ((volatile uint32_t *)((char *)(hw)->hw_addr + (reg))) + ((volatile void *)((char *)(hw)->hw_addr + (reg)))#define IXGBE_PCI_REG_ARRAY_ADDR (hw, reg, index) \ IXGBE_PCI_REG_ADDR((hw), (reg) + ((index) << 2))
drivers/net/ixgbe/ixgbe_rxtx.c +8 -8
@@ -308,7 +308,7 @@ tx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,/* update tail pointer */ rte_wmb (); -IXGBE_PCI_REG_WC_WRITE_RELAXED (txq->tdt_reg_addr, txq->tx_tail); +IXGBE_PCI_REG_WC_WRITE_RELAXED (txq->qtx_tail, txq->tx_tail);return nb_pkts; }
@@ -946,7 +946,7 @@ ixgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,PMD_TX_LOG (DEBUG ,"port_id=%u queue_id=%u tx_tail=%u nb_tx=%u" , (unsigned ) txq->port_id, (unsigned ) txq->queue_id, (unsigned ) tx_id, (unsigned ) nb_tx); -IXGBE_PCI_REG_WC_WRITE_RELAXED (txq->tdt_reg_addr, tx_id); +IXGBE_PCI_REG_WC_WRITE_RELAXED (txq->qtx_tail, tx_id); txq->tx_tail = tx_id;return nb_tx;
@@ -2786,11 +2786,11 @@ ixgbe_dev_tx_queue_setup(struct rte_eth_dev *dev, hw->mac.type == ixgbe_mac_X550_vf || hw->mac.type == ixgbe_mac_X550EM_x_vf || hw->mac.type == ixgbe_mac_X550EM_a_vf) - txq->tdt_reg_addr = IXGBE_PCI_REG_ADDR(hw, IXGBE_VFTDT(queue_idx)); + txq->qtx_tail = IXGBE_PCI_REG_ADDR(hw, IXGBE_VFTDT(queue_idx)); else - txq->tdt_reg_addr = IXGBE_PCI_REG_ADDR(hw, IXGBE_TDT(txq->reg_idx)); + txq->qtx_tail = IXGBE_PCI_REG_ADDR(hw, IXGBE_TDT(txq->reg_idx)); - txq->tx_ring_phys_addr = tz->iova; + txq->tx_ring_dma = tz->iova; txq->tx_ring = (union ixgbe_adv_tx_desc *) tz->addr;/* Allocate software ring */
@@ -2802,7 +2802,7 @@ ixgbe_dev_tx_queue_setup(struct rte_eth_dev *dev,return -ENOMEM ; }PMD_INIT_LOG (DEBUG ,"sw_ring=%p hw_ring=%p dma_addr=0x%" PRIx64, - txq->sw_ring, txq->tx_ring, txq->tx_ring_phys_addr); + txq->sw_ring, txq->tx_ring, txq->tx_ring_dma);/* set up vector or scalar TX function as appropriate */ ixgbe_set_tx_function (dev, txq);
@@ -5303,7 +5303,7 @@ ixgbe_dev_tx_init(struct rte_eth_dev *dev)for (i =0 ; i < dev->data->nb_tx_queues; i++) { txq = dev->data->tx_queues[i]; - bus_addr = txq->tx_ring_phys_addr; + bus_addr = txq->tx_ring_dma;IXGBE_WRITE_REG (hw,IXGBE_TDBAL (txq->reg_idx), (uint32_t )(bus_addr &0x00000000ffffffffULL )); IXGBE_WRITE_REG(hw, IXGBE_TDBAH(txq->reg_idx),
@@ -5887,7 +5887,7 @@ ixgbevf_dev_tx_init(struct rte_eth_dev *dev)/* Setup the Base and Length of the Tx Descriptor Rings */ for (i =0 ; i < dev->data->nb_tx_queues; i++) { txq = dev->data->tx_queues[i]; - bus_addr = txq->tx_ring_phys_addr; + bus_addr = txq->tx_ring_dma;IXGBE_WRITE_REG (hw,IXGBE_VFTDBAL (i), (uint32_t )(bus_addr &0x00000000ffffffffULL )); IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(i),
drivers/net/ixgbe/ixgbe_rxtx.h +3 -3
@@ -186,12 +186,12 @@ struct ixgbe_advctx_info {struct ixgbe_tx_queue {/** TX ring virtual address. */ volatile union ixgbe_adv_tx_desc *tx_ring; -uint64_t tx_ring_phys_addr;/**< TX ring DMA address. */ +rte_iova_t tx_ring_dma;/**< TX ring DMA address. */ union {struct ci_tx_entry *sw_ring;/**< address of SW ring for scalar PMD. */ struct ci_tx_entry_vec *sw_ring_v;/**< address of SW ring for vector PMD */ }; -volatile uint32_t *tdt_reg_addr;/**< Address of TDT register. */ +volatile uint8_t *qtx_tail;/**< Address of TDT register. */ uint16_t nb_tx_desc;/**< number of TX descriptors. */ uint16_t tx_tail;/**< current value of TDT reg. */ /**< Start freeing TX buffers if there are less free descriptors than
@@ -218,7 +218,7 @@ struct ixgbe_tx_queue {/** Hardware context0 history. */ struct ixgbe_advctx_info ctx_cache[IXGBE_CTX_NUM ];const struct ixgbe_txq_ops *ops;/**< txq ops */ -uint8_t tx_deferred_start;/**< not in global dev start. */ +bool tx_deferred_start;/**< not in global dev start. */ #ifdef RTE_LIB_SECURITY uint8_t using_ipsec;/**< indicates that IPsec TX feature is in use */
drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c +1 -1
@@ -628,7 +628,7 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, txq->tx_tail = tx_id; -IXGBE_PCI_REG_WRITE (txq->tdt_reg_addr, txq->tx_tail); +IXGBE_PCI_REG_WRITE (txq->qtx_tail, txq->tx_tail);return nb_pkts; }
drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c +1 -1
@@ -751,7 +751,7 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, txq->tx_tail = tx_id; -IXGBE_PCI_REG_WC_WRITE (txq->tdt_reg_addr, txq->tx_tail); +IXGBE_PCI_REG_WC_WRITE (txq->qtx_tail, txq->tx_tail);return nb_pkts; }
Signature · 1 line
2.43.0
PATCH v3 5/22 drivers/net: add prefix for driver-specific structs
In reply to Bruce RichardsonIn preparation for merging the Tx structs for multiple drivers into a single struct, rename the driver-specific pointers in each struct to have a prefix on it, to avoid conflicts.
--- drivers/net/i40e/i40e_fdir.c | 6 +-- .../net/i40e/i40e_recycle_mbufs_vec_common.c | 2 +- drivers/net/i40e/i40e_rxtx.c | 30 ++++++------ drivers/net/i40e/i40e_rxtx.h | 4 +- drivers/net/i40e/i40e_rxtx_vec_altivec.c | 6 +-- drivers/net/i40e/i40e_rxtx_vec_avx2.c | 6 +-- drivers/net/i40e/i40e_rxtx_vec_avx512.c | 8 ++-- drivers/net/i40e/i40e_rxtx_vec_common.h | 2 +- drivers/net/i40e/i40e_rxtx_vec_neon.c | 6 +-- drivers/net/i40e/i40e_rxtx_vec_sse.c | 6 +-- drivers/net/iavf/iavf_rxtx.c | 24 +++++----- drivers/net/iavf/iavf_rxtx.h | 4 +- drivers/net/iavf/iavf_rxtx_vec_avx2.c | 6 +-- drivers/net/iavf/iavf_rxtx_vec_avx512.c | 14 +++--- drivers/net/iavf/iavf_rxtx_vec_common.h | 2 +- drivers/net/iavf/iavf_rxtx_vec_sse.c | 6 +-- drivers/net/ice/ice_dcf_ethdev.c | 4 +- drivers/net/ice/ice_rxtx.c | 48 +++++++++---------- drivers/net/ice/ice_rxtx.h | 4 +- drivers/net/ice/ice_rxtx_vec_avx2.c | 6 +-- drivers/net/ice/ice_rxtx_vec_avx512.c | 8 ++-- drivers/net/ice/ice_rxtx_vec_common.h | 4 +- drivers/net/ice/ice_rxtx_vec_sse.c | 6 +-- .../ixgbe/ixgbe_recycle_mbufs_vec_common.c | 2 +- drivers/net/ixgbe/ixgbe_rxtx.c | 22 ++++----- drivers/net/ixgbe/ixgbe_rxtx.h | 2 +- drivers/net/ixgbe/ixgbe_rxtx_vec_common.h | 6 +-- drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c | 6 +-- drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c | 6 +-- 29 files changed, 128 insertions(+), 128 deletions(-)
drivers/net/i40e/i40e_fdir.c +3 -3
@@ -1383,7 +1383,7 @@ i40e_find_available_buffer(struct rte_eth_dev *dev)volatile struct i40e_tx_desc *tmp_txdp; tmp_tail = txq->tx_tail; - tmp_txdp = &txq->tx_ring[tmp_tail +1 ]; + tmp_txdp = &txq->i40e_tx_ring[tmp_tail +1 ]; do { if ((tmp_txdp->cmd_type_offset_bsz &
@@ -1640,7 +1640,7 @@ i40e_flow_fdir_filter_programming(struct i40e_pf *pf,PMD_DRV_LOG (INFO ,"filling filter programming descriptor." ); fdirdp = (volatile struct i40e_filter_program_desc *) - (&txq->tx_ring[txq->tx_tail]); + (&txq->i40e_tx_ring[txq->tx_tail]); fdirdp->qindex_flex_ptype_vsi = rte_cpu_to_le_32((fdir_action->rx_queue <<
@@ -1710,7 +1710,7 @@ i40e_flow_fdir_filter_programming(struct i40e_pf *pf, fdirdp->fd_id =rte_cpu_to_le_32 (filter->soft_id);PMD_DRV_LOG (INFO ,"filling transmit descriptor." ); - txdp = &txq->tx_ring[txq->tx_tail +1 ]; + txdp = &txq->i40e_tx_ring[txq->tx_tail +1 ]; txdp->buffer_addr =rte_cpu_to_le_64 (pf->fdir.dma_addr[txq->tx_tail >>1 ]); td_cmd = I40E_TX_DESC_CMD_EOP |
drivers/net/i40e/i40e_recycle_mbufs_vec_common.c +1 -1
@@ -75,7 +75,7 @@ i40e_recycle_tx_mbufs_reuse_vec(void *tx_queue,return 0 ;/* check DD bits on threshold descriptor */ -if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz & +if ((txq->i40e_tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &rte_cpu_to_le_64 (I40E_TXD_QW1_DTYPE_MASK )) !=rte_cpu_to_le_64 (I40E_TX_DESC_DTYPE_DESC_DONE ))return 0 ;
drivers/net/i40e/i40e_rxtx.c +15 -15
@@ -379,7 +379,7 @@ static inline int i40e_xmit_cleanup(struct i40e_tx_queue *txq) {struct ci_tx_entry *sw_ring = txq->sw_ring; -volatile struct i40e_tx_desc *txd = txq->tx_ring; +volatile struct i40e_tx_desc *txd = txq->i40e_tx_ring;uint16_t last_desc_cleaned = txq->last_desc_cleaned;uint16_t nb_tx_desc = txq->nb_tx_desc;uint16_t desc_to_clean_to;
@@ -1103,7 +1103,7 @@ i40e_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) txq = tx_queue; sw_ring = txq->sw_ring; - txr = txq->tx_ring; + txr = txq->i40e_tx_ring; tx_id = txq->tx_tail; txe = &sw_ring[tx_id];
@@ -1338,7 +1338,7 @@ i40e_tx_free_bufs(struct i40e_tx_queue *txq)const uint16_t k =RTE_ALIGN_FLOOR (tx_rs_thresh,RTE_I40E_TX_MAX_FREE_BUF_SZ );const uint16_t m = tx_rs_thresh %RTE_I40E_TX_MAX_FREE_BUF_SZ ; -if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz & +if ((txq->i40e_tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &rte_cpu_to_le_64 (I40E_TXD_QW1_DTYPE_MASK )) !=rte_cpu_to_le_64 (I40E_TX_DESC_DTYPE_DESC_DONE ))return 0 ;
@@ -1417,7 +1417,7 @@ i40e_tx_fill_hw_ring(struct i40e_tx_queue *txq, struct rte_mbuf **pkts, uint16_t nb_pkts) { -volatile struct i40e_tx_desc *txdp = &(txq->tx_ring[txq->tx_tail]); +volatile struct i40e_tx_desc *txdp = &txq->i40e_tx_ring[txq->tx_tail];struct ci_tx_entry *txep = &txq->sw_ring[txq->tx_tail];const int N_PER_LOOP =4 ;const int N_PER_LOOP_MASK =N_PER_LOOP -1 ;
@@ -1445,7 +1445,7 @@ tx_xmit_pkts(struct i40e_tx_queue *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { -volatile struct i40e_tx_desc *txr = txq->tx_ring; +volatile struct i40e_tx_desc *txr = txq->i40e_tx_ring;uint16_t n =0 ;/**
@@ -1556,7 +1556,7 @@ i40e_xmit_pkts_check(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pktsbool pkt_error = false;const char *reason =NULL ;uint16_t good_pkts = nb_pkts; -struct i40e_adapter *adapter = txq->vsi->adapter; +struct i40e_adapter *adapter = txq->i40e_vsi->adapter;for (idx =0 ; idx < nb_pkts; idx++) { mb = tx_pkts[idx];
@@ -2329,7 +2329,7 @@ i40e_dev_tx_descriptor_status(void *tx_queue, uint16_t offset) desc -= txq->nb_tx_desc; } - status = &txq->tx_ring[desc].cmd_type_offset_bsz; + status = &txq->i40e_tx_ring[desc].cmd_type_offset_bsz; mask =rte_le_to_cpu_64 (I40E_TXD_QW1_DTYPE_MASK ); expect =rte_cpu_to_le_64 (I40E_TX_DESC_DTYPE_DESC_DONE <<I40E_TXD_QW1_DTYPE_SHIFT );
@@ -2527,7 +2527,7 @@ i40e_dev_tx_queue_setup(struct rte_eth_dev *dev,/* Allocate TX hardware ring descriptors. */ ring_size =sizeof (struct i40e_tx_desc ) *I40E_MAX_RING_DESC ; ring_size =RTE_ALIGN (ring_size,I40E_DMA_MEM_ALIGN ); - tz =rte_eth_dma_zone_reserve (dev,"tx_ring" , queue_idx, + tz =rte_eth_dma_zone_reserve (dev,"i40e_tx_ring" , queue_idx, ring_size,I40E_RING_BASE_ALIGN , socket_id);if (!tz) {i40e_tx_queue_release (txq);
@@ -2546,11 +2546,11 @@ i40e_dev_tx_queue_setup(struct rte_eth_dev *dev, txq->reg_idx = reg_idx; txq->port_id = dev->data->port_id; txq->offloads = offloads; - txq->vsi = vsi; + txq->i40e_vsi = vsi; txq->tx_deferred_start = tx_conf->tx_deferred_start; txq->tx_ring_dma = tz->iova; - txq->tx_ring = (struct i40e_tx_desc *)tz->addr; + txq->i40e_tx_ring = (struct i40e_tx_desc *)tz->addr;/* Allocate software ring */ txq->sw_ring =
@@ -2885,11 +2885,11 @@ i40e_reset_tx_queue(struct i40e_tx_queue *txq) txe = txq->sw_ring; size =sizeof (struct i40e_tx_desc ) * txq->nb_tx_desc;for (i =0 ; i < size; i++) - ((volatile char *)txq->tx_ring)[i] =0 ; + ((volatile char *)txq->i40e_tx_ring)[i] =0 ; prev = (uint16_t )(txq->nb_tx_desc -1 );for (i =0 ; i < txq->nb_tx_desc; i++) { -volatile struct i40e_tx_desc *txd = &txq->tx_ring[i]; +volatile struct i40e_tx_desc *txd = &txq->i40e_tx_ring[i]; txd->cmd_type_offset_bsz =rte_cpu_to_le_64 (I40E_TX_DESC_DTYPE_DESC_DONE );
@@ -2914,7 +2914,7 @@ int i40e_tx_queue_init(struct i40e_tx_queue *txq) {enum i40e_status_code err =I40E_SUCCESS ; -struct i40e_vsi *vsi = txq->vsi; +struct i40e_vsi *vsi = txq->i40e_vsi;struct i40e_hw *hw =I40E_VSI_TO_HW (vsi);uint16_t pf_q = txq->reg_idx;struct i40e_hmc_obj_txq tx_ctx;
@@ -3207,10 +3207,10 @@ i40e_fdir_setup_tx_resources(struct i40e_pf *pf) txq->nb_tx_desc =I40E_FDIR_NUM_TX_DESC ; txq->queue_id =I40E_FDIR_QUEUE_ID ; txq->reg_idx = pf->fdir.fdir_vsi->base_queue; - txq->vsi = pf->fdir.fdir_vsi; + txq->i40e_vsi = pf->fdir.fdir_vsi; txq->tx_ring_dma = tz->iova; - txq->tx_ring = (struct i40e_tx_desc *)tz->addr; + txq->i40e_tx_ring = (struct i40e_tx_desc *)tz->addr;/* * don't need to allocate software ring and reset for the fdir
drivers/net/i40e/i40e_rxtx.h +2 -2
@@ -130,7 +130,7 @@ struct i40e_rx_queue {struct i40e_tx_queue {uint16_t nb_tx_desc;/**< number of TX descriptors */ rte_iova_t tx_ring_dma;/**< TX ring DMA address */ -volatile struct i40e_tx_desc *tx_ring;/**< TX ring virtual address */ +volatile struct i40e_tx_desc *i40e_tx_ring;/**< TX ring virtual address */ struct ci_tx_entry *sw_ring;/**< virtual address of SW ring */ uint16_t tx_tail;/**< current value of tail register */ volatile uint8_t *qtx_tail;/**< register address of tail */
@@ -150,7 +150,7 @@ struct i40e_tx_queue {uint16_t port_id;/**< Device port identifier. */ uint16_t queue_id;/**< TX queue index. */ uint16_t reg_idx; -struct i40e_vsi *vsi;/**< the VSI this queue belongs to */ +struct i40e_vsi *i40e_vsi;/**< the VSI this queue belongs to */ uint16_t tx_next_dd;uint16_t tx_next_rs;bool q_set;/**< indicate if tx queue has been configured */
drivers/net/i40e/i40e_rxtx_vec_altivec.c +3 -3
@@ -568,7 +568,7 @@ i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,return 0 ; tx_id = txq->tx_tail; - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->i40e_tx_ring[tx_id]; txep = &txq->sw_ring[tx_id]; txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts);
@@ -588,7 +588,7 @@ i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, txq->tx_next_rs = (uint16_t )(txq->tx_rs_thresh -1 );/* avoid reach the end of ring */ - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->i40e_tx_ring[tx_id]; txep = &txq->sw_ring[tx_id]; }
@@ -598,7 +598,7 @@ i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = (uint16_t )(tx_id + nb_commit);if (tx_id > txq->tx_next_rs) { - txq->tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |= + txq->i40e_tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |=rte_cpu_to_le_64 (((uint64_t )I40E_TX_DESC_CMD_RS ) <<I40E_TXD_QW1_CMD_SHIFT ); txq->tx_next_rs =
drivers/net/i40e/i40e_rxtx_vec_avx2.c +3 -3
@@ -758,7 +758,7 @@ i40e_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts,return 0 ; tx_id = txq->tx_tail; - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->i40e_tx_ring[tx_id]; txep = &txq->sw_ring[tx_id]; txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts);
@@ -779,7 +779,7 @@ i40e_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, txq->tx_next_rs = (uint16_t )(txq->tx_rs_thresh -1 );/* avoid reach the end of ring */ - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->i40e_tx_ring[tx_id]; txep = &txq->sw_ring[tx_id]; }
@@ -789,7 +789,7 @@ i40e_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = (uint16_t )(tx_id + nb_commit);if (tx_id > txq->tx_next_rs) { - txq->tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |= + txq->i40e_tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |=rte_cpu_to_le_64 (((uint64_t )I40E_TX_DESC_CMD_RS ) <<I40E_TXD_QW1_CMD_SHIFT ); txq->tx_next_rs =
drivers/net/i40e/i40e_rxtx_vec_avx512.c +4 -4
@@ -764,7 +764,7 @@ i40e_tx_free_bufs_avx512(struct i40e_tx_queue *txq)struct rte_mbuf *m, *free[RTE_I40E_TX_MAX_FREE_BUF_SZ ];/* check DD bits on threshold descriptor */ -if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz & +if ((txq->i40e_tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &rte_cpu_to_le_64 (I40E_TXD_QW1_DTYPE_MASK )) !=rte_cpu_to_le_64 (I40E_TX_DESC_DTYPE_DESC_DONE ))return 0 ;
@@ -948,7 +948,7 @@ i40e_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts,return 0 ; tx_id = txq->tx_tail; - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->i40e_tx_ring[tx_id]; txep = (void *)txq->sw_ring; txep += tx_id;
@@ -970,7 +970,7 @@ i40e_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, txq->tx_next_rs = (uint16_t )(txq->tx_rs_thresh -1 );/* avoid reach the end of ring */ - txdp = txq->tx_ring; + txdp = txq->i40e_tx_ring; txep = (void *)txq->sw_ring; }
@@ -980,7 +980,7 @@ i40e_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = (uint16_t )(tx_id + nb_commit);if (tx_id > txq->tx_next_rs) { - txq->tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |= + txq->i40e_tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |=rte_cpu_to_le_64 (((uint64_t )I40E_TX_DESC_CMD_RS ) <<I40E_TXD_QW1_CMD_SHIFT ); txq->tx_next_rs =
drivers/net/i40e/i40e_rxtx_vec_common.h +1 -1
@@ -26,7 +26,7 @@ i40e_tx_free_bufs(struct i40e_tx_queue *txq)struct rte_mbuf *m, *free[RTE_I40E_TX_MAX_FREE_BUF_SZ ];/* check DD bits on threshold descriptor */ -if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz & +if ((txq->i40e_tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &rte_cpu_to_le_64 (I40E_TXD_QW1_DTYPE_MASK )) !=rte_cpu_to_le_64 (I40E_TX_DESC_DTYPE_DESC_DONE ))return 0 ;
drivers/net/i40e/i40e_rxtx_vec_neon.c +3 -3
@@ -695,7 +695,7 @@ i40e_xmit_fixed_burst_vec(void *__rte_restrict tx_queue,return 0 ; tx_id = txq->tx_tail; - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->i40e_tx_ring[tx_id]; txep = &txq->sw_ring[tx_id]; txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts);
@@ -715,7 +715,7 @@ i40e_xmit_fixed_burst_vec(void *__rte_restrict tx_queue, txq->tx_next_rs = (uint16_t )(txq->tx_rs_thresh -1 );/* avoid reach the end of ring */ - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->i40e_tx_ring[tx_id]; txep = &txq->sw_ring[tx_id]; }
@@ -725,7 +725,7 @@ i40e_xmit_fixed_burst_vec(void *__rte_restrict tx_queue, tx_id = (uint16_t )(tx_id + nb_commit);if (tx_id > txq->tx_next_rs) { - txq->tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |= + txq->i40e_tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |=rte_cpu_to_le_64 (((uint64_t )I40E_TX_DESC_CMD_RS ) <<I40E_TXD_QW1_CMD_SHIFT ); txq->tx_next_rs =
drivers/net/i40e/i40e_rxtx_vec_sse.c +3 -3
@@ -714,7 +714,7 @@ i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,return 0 ; tx_id = txq->tx_tail; - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->i40e_tx_ring[tx_id]; txep = &txq->sw_ring[tx_id]; txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts);
@@ -734,7 +734,7 @@ i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, txq->tx_next_rs = (uint16_t )(txq->tx_rs_thresh -1 );/* avoid reach the end of ring */ - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->i40e_tx_ring[tx_id]; txep = &txq->sw_ring[tx_id]; }
@@ -744,7 +744,7 @@ i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = (uint16_t )(tx_id + nb_commit);if (tx_id > txq->tx_next_rs) { - txq->tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |= + txq->i40e_tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |=rte_cpu_to_le_64 (((uint64_t )I40E_TX_DESC_CMD_RS ) <<I40E_TXD_QW1_CMD_SHIFT ); txq->tx_next_rs =
drivers/net/iavf/iavf_rxtx.c +12 -12
@@ -296,11 +296,11 @@ reset_tx_queue(struct iavf_tx_queue *txq) txe = txq->sw_ring; size =sizeof (struct iavf_tx_desc ) * txq->nb_tx_desc;for (i =0 ; i < size; i++) - ((volatile char *)txq->tx_ring)[i] =0 ; + ((volatile char *)txq->iavf_tx_ring)[i] =0 ; prev = (uint16_t )(txq->nb_tx_desc -1 );for (i =0 ; i < txq->nb_tx_desc; i++) { - txq->tx_ring[i].cmd_type_offset_bsz = + txq->iavf_tx_ring[i].cmd_type_offset_bsz =rte_cpu_to_le_64 (IAVF_TX_DESC_DTYPE_DESC_DONE ); txe[i].mbuf =NULL ; txe[i].last_id = i;
@@ -851,7 +851,7 @@ iavf_dev_tx_queue_setup(struct rte_eth_dev *dev, txq->port_id = dev->data->port_id; txq->offloads = offloads; txq->tx_deferred_start = tx_conf->tx_deferred_start; - txq->vsi = vsi; + txq->iavf_vsi = vsi;if (iavf_ipsec_crypto_supported (adapter)) txq->ipsec_crypto_pkt_md_offset =
@@ -872,7 +872,7 @@ iavf_dev_tx_queue_setup(struct rte_eth_dev *dev,/* Allocate TX hardware ring descriptors. */ ring_size =sizeof (struct iavf_tx_desc ) *IAVF_MAX_RING_DESC ; ring_size =RTE_ALIGN (ring_size,IAVF_DMA_MEM_ALIGN ); - mz =rte_eth_dma_zone_reserve (dev,"tx_ring" , queue_idx, + mz =rte_eth_dma_zone_reserve (dev,"iavf_tx_ring" , queue_idx, ring_size,IAVF_RING_BASE_ALIGN , socket_id);if (!mz) {
@@ -882,7 +882,7 @@ iavf_dev_tx_queue_setup(struct rte_eth_dev *dev,return -ENOMEM ; } txq->tx_ring_dma = mz->iova; - txq->tx_ring = (struct iavf_tx_desc *)mz->addr; + txq->iavf_tx_ring = (struct iavf_tx_desc *)mz->addr; txq->mz = mz;reset_tx_queue (txq);
@@ -2385,7 +2385,7 @@ iavf_xmit_cleanup(struct iavf_tx_queue *txq)uint16_t desc_to_clean_to;uint16_t nb_tx_to_clean; -volatile struct iavf_tx_desc *txd = txq->tx_ring; +volatile struct iavf_tx_desc *txd = txq->iavf_tx_ring; desc_to_clean_to = (uint16_t )(last_desc_cleaned + txq->tx_rs_thresh);if (desc_to_clean_to >= nb_tx_desc)
@@ -2796,7 +2796,7 @@ uint16_t iavf_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) {struct iavf_tx_queue *txq = tx_queue; -volatile struct iavf_tx_desc *txr = txq->tx_ring; +volatile struct iavf_tx_desc *txr = txq->iavf_tx_ring;struct ci_tx_entry *txe_ring = txq->sw_ring;struct ci_tx_entry *txe, *txn;struct rte_mbuf *mb, *mb_seg;
@@ -3803,10 +3803,10 @@ iavf_xmit_pkts_no_poll(void *tx_queue, struct rte_mbuf **tx_pkts,struct iavf_tx_queue *txq = tx_queue;enum iavf_tx_burst_type tx_burst_type; -if (!txq->vsi || txq->vsi->adapter->no_poll) +if (!txq->iavf_vsi || txq->iavf_vsi->adapter->no_poll)return 0 ; - tx_burst_type = txq->vsi->adapter->tx_burst_type; + tx_burst_type = txq->iavf_vsi->adapter->tx_burst_type;return iavf_tx_pkt_burst_ops[tx_burst_type](tx_queue, tx_pkts, nb_pkts);
@@ -3824,9 +3824,9 @@ iavf_xmit_pkts_check(void *tx_queue, struct rte_mbuf **tx_pkts,const char *reason =NULL ;bool pkt_error = false;struct iavf_tx_queue *txq = tx_queue; -struct iavf_adapter *adapter = txq->vsi->adapter; +struct iavf_adapter *adapter = txq->iavf_vsi->adapter;enum iavf_tx_burst_type tx_burst_type = - txq->vsi->adapter->tx_burst_type; + txq->iavf_vsi->adapter->tx_burst_type;for (idx =0 ; idx < nb_pkts; idx++) { mb = tx_pkts[idx];
@@ -4440,7 +4440,7 @@ iavf_dev_tx_desc_status(void *tx_queue, uint16_t offset) desc -= txq->nb_tx_desc; } - status = &txq->tx_ring[desc].cmd_type_offset_bsz; + status = &txq->iavf_tx_ring[desc].cmd_type_offset_bsz; mask =rte_le_to_cpu_64 (IAVF_TXD_QW1_DTYPE_MASK ); expect =rte_cpu_to_le_64 (IAVF_TX_DESC_DTYPE_DESC_DONE <<IAVF_TXD_QW1_DTYPE_SHIFT );
drivers/net/iavf/iavf_rxtx.h +2 -2
@@ -276,7 +276,7 @@ struct iavf_rx_queue {/* Structure associated with each TX queue. */ struct iavf_tx_queue {const struct rte_memzone *mz;/* memzone for Tx ring */ -volatile struct iavf_tx_desc *tx_ring;/* Tx ring virtual address */ +volatile struct iavf_tx_desc *iavf_tx_ring;/* Tx ring virtual address */ rte_iova_t tx_ring_dma;/* Tx ring DMA address */ struct ci_tx_entry *sw_ring;/* address array of SW ring */ uint16_t nb_tx_desc;/* ring length */
@@ -289,7 +289,7 @@ struct iavf_tx_queue {uint16_t tx_free_thresh;uint16_t tx_rs_thresh;uint8_t rel_mbufs_type; -struct iavf_vsi *vsi;/**< the VSI this queue belongs to */ +struct iavf_vsi *iavf_vsi;/**< the VSI this queue belongs to */ uint16_t port_id;uint16_t queue_id;
drivers/net/iavf/iavf_rxtx_vec_avx2.c +3 -3
@@ -1751,7 +1751,7 @@ iavf_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, nb_commit = nb_pkts; tx_id = txq->tx_tail; - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->iavf_tx_ring[tx_id]; txep = &txq->sw_ring[tx_id]; txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts);
@@ -1772,7 +1772,7 @@ iavf_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, txq->tx_next_rs = (uint16_t )(txq->tx_rs_thresh -1 );/* avoid reach the end of ring */ - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->iavf_tx_ring[tx_id]; txep = &txq->sw_ring[tx_id]; }
@@ -1782,7 +1782,7 @@ iavf_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = (uint16_t )(tx_id + nb_commit);if (tx_id > txq->tx_next_rs) { - txq->tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |= + txq->iavf_tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |=rte_cpu_to_le_64 (((uint64_t )IAVF_TX_DESC_CMD_RS ) <<IAVF_TXD_QW1_CMD_SHIFT ); txq->tx_next_rs =
drivers/net/iavf/iavf_rxtx_vec_avx512.c +7 -7
@@ -1854,7 +1854,7 @@ iavf_tx_free_bufs_avx512(struct iavf_tx_queue *txq)struct rte_mbuf *m, *free[IAVF_VPMD_TX_MAX_FREE_BUF ];/* check DD bits on threshold descriptor */ -if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz & +if ((txq->iavf_tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &rte_cpu_to_le_64 (IAVF_TXD_QW1_DTYPE_MASK )) !=rte_cpu_to_le_64 (IAVF_TX_DESC_DTYPE_DESC_DONE ))return 0 ;
@@ -2328,7 +2328,7 @@ iavf_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, nb_commit = nb_pkts; tx_id = txq->tx_tail; - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->iavf_tx_ring[tx_id]; txep = (void *)txq->sw_ring; txep += tx_id;
@@ -2350,7 +2350,7 @@ iavf_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, txq->tx_next_rs = (uint16_t )(txq->tx_rs_thresh -1 );/* avoid reach the end of ring */ - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->iavf_tx_ring[tx_id]; txep = (void *)txq->sw_ring; txep += tx_id; }
@@ -2361,7 +2361,7 @@ iavf_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = (uint16_t )(tx_id + nb_commit);if (tx_id > txq->tx_next_rs) { - txq->tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |= + txq->iavf_tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |=rte_cpu_to_le_64 (((uint64_t )IAVF_TX_DESC_CMD_RS ) <<IAVF_TXD_QW1_CMD_SHIFT ); txq->tx_next_rs =
@@ -2397,7 +2397,7 @@ iavf_xmit_fixed_burst_vec_avx512_ctx(void *tx_queue, struct rte_mbuf **tx_pkts, nb_pkts = nb_commit >>1 ; tx_id = txq->tx_tail; - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->iavf_tx_ring[tx_id]; txep = (void *)txq->sw_ring; txep += (tx_id >>1 );
@@ -2418,7 +2418,7 @@ iavf_xmit_fixed_burst_vec_avx512_ctx(void *tx_queue, struct rte_mbuf **tx_pkts, txq->tx_next_rs = (uint16_t )(txq->tx_rs_thresh -1 ); tx_id =0 ;/* avoid reach the end of ring */ - txdp = txq->tx_ring; + txdp = txq->iavf_tx_ring; txep = (void *)txq->sw_ring; }
@@ -2429,7 +2429,7 @@ iavf_xmit_fixed_burst_vec_avx512_ctx(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = (uint16_t )(tx_id + nb_commit);if (tx_id > txq->tx_next_rs) { - txq->tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |= + txq->iavf_tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |=rte_cpu_to_le_64 (((uint64_t )IAVF_TX_DESC_CMD_RS ) <<IAVF_TXD_QW1_CMD_SHIFT ); txq->tx_next_rs =
drivers/net/iavf/iavf_rxtx_vec_common.h +1 -1
@@ -26,7 +26,7 @@ iavf_tx_free_bufs(struct iavf_tx_queue *txq)struct rte_mbuf *m, *free[IAVF_VPMD_TX_MAX_FREE_BUF ];/* check DD bits on threshold descriptor */ -if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz & +if ((txq->iavf_tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &rte_cpu_to_le_64 (IAVF_TXD_QW1_DTYPE_MASK )) !=rte_cpu_to_le_64 (IAVF_TX_DESC_DTYPE_DESC_DONE ))return 0 ;
drivers/net/iavf/iavf_rxtx_vec_sse.c +3 -3
@@ -1383,7 +1383,7 @@ iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, nb_commit = nb_pkts; tx_id = txq->tx_tail; - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->iavf_tx_ring[tx_id]; txep = &txq->sw_ring[tx_id]; txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts);
@@ -1403,7 +1403,7 @@ iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, txq->tx_next_rs = (uint16_t )(txq->tx_rs_thresh -1 );/* avoid reach the end of ring */ - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->iavf_tx_ring[tx_id]; txep = &txq->sw_ring[tx_id]; }
@@ -1413,7 +1413,7 @@ iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = (uint16_t )(tx_id + nb_commit);if (tx_id > txq->tx_next_rs) { - txq->tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |= + txq->iavf_tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |=rte_cpu_to_le_64 (((uint64_t )IAVF_TX_DESC_CMD_RS ) <<IAVF_TXD_QW1_CMD_SHIFT ); txq->tx_next_rs =
drivers/net/ice/ice_dcf_ethdev.c +2 -2
@@ -401,11 +401,11 @@ reset_tx_queue(struct ice_tx_queue *txq) txe = txq->sw_ring; size =sizeof (struct ice_tx_desc ) * txq->nb_tx_desc;for (i =0 ; i < size; i++) - ((volatile char *)txq->tx_ring)[i] =0 ; + ((volatile char *)txq->ice_tx_ring)[i] =0 ; prev = (uint16_t )(txq->nb_tx_desc -1 );for (i =0 ; i < txq->nb_tx_desc; i++) { - txq->tx_ring[i].cmd_type_offset_bsz = + txq->ice_tx_ring[i].cmd_type_offset_bsz =rte_cpu_to_le_64 (IAVF_TX_DESC_DTYPE_DESC_DONE ); txe[i].mbuf =NULL ; txe[i].last_id = i;
drivers/net/ice/ice_rxtx.c +24 -24
179 lines not on this page · raw message
drivers/net/ice/ice_rxtx.h +2 -2
@@ -148,7 +148,7 @@ struct ice_rx_queue {struct ice_tx_queue {uint16_t nb_tx_desc;/* number of TX descriptors */ rte_iova_t tx_ring_dma;/* TX ring DMA address */ -volatile struct ice_tx_desc *tx_ring;/* TX ring virtual address */ +volatile struct ice_tx_desc *ice_tx_ring;/* TX ring virtual address */ struct ci_tx_entry *sw_ring;/* virtual address of SW ring */ uint16_t tx_tail;/* current value of tail register */ volatile uint8_t *qtx_tail;/* register address of tail */
@@ -171,7 +171,7 @@ struct ice_tx_queue {uint32_t q_teid;/* TX schedule node id. */ uint16_t reg_idx;uint64_t offloads; -struct ice_vsi *vsi;/* the VSI this queue belongs to */ +struct ice_vsi *ice_vsi;/* the VSI this queue belongs to */ uint16_t tx_next_dd;uint16_t tx_next_rs;uint64_t mbuf_errors;
drivers/net/ice/ice_rxtx_vec_avx2.c +3 -3
@@ -874,7 +874,7 @@ ice_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts,return 0 ; tx_id = txq->tx_tail; - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->ice_tx_ring[tx_id]; txep = &txq->sw_ring[tx_id]; txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts);
@@ -895,7 +895,7 @@ ice_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, txq->tx_next_rs = (uint16_t )(txq->tx_rs_thresh -1 );/* avoid reach the end of ring */ - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->ice_tx_ring[tx_id]; txep = &txq->sw_ring[tx_id]; }
@@ -905,7 +905,7 @@ ice_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = (uint16_t )(tx_id + nb_commit);if (tx_id > txq->tx_next_rs) { - txq->tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |= + txq->ice_tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |=rte_cpu_to_le_64 (((uint64_t )ICE_TX_DESC_CMD_RS ) <<ICE_TXD_QW1_CMD_S ); txq->tx_next_rs =
drivers/net/ice/ice_rxtx_vec_avx512.c +4 -4
@@ -869,7 +869,7 @@ ice_tx_free_bufs_avx512(struct ice_tx_queue *txq)struct rte_mbuf *m, *free[ICE_TX_MAX_FREE_BUF_SZ ];/* check DD bits on threshold descriptor */ -if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz & +if ((txq->ice_tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &rte_cpu_to_le_64 (ICE_TXD_QW1_DTYPE_M )) !=rte_cpu_to_le_64 (ICE_TX_DESC_DTYPE_DESC_DONE ))return 0 ;
@@ -1071,7 +1071,7 @@ ice_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts,return 0 ; tx_id = txq->tx_tail; - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->ice_tx_ring[tx_id]; txep = (void *)txq->sw_ring; txep += tx_id;
@@ -1093,7 +1093,7 @@ ice_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, txq->tx_next_rs = (uint16_t )(txq->tx_rs_thresh -1 );/* avoid reach the end of ring */ - txdp = txq->tx_ring; + txdp = txq->ice_tx_ring; txep = (void *)txq->sw_ring; }
@@ -1103,7 +1103,7 @@ ice_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = (uint16_t )(tx_id + nb_commit);if (tx_id > txq->tx_next_rs) { - txq->tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |= + txq->ice_tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |=rte_cpu_to_le_64 (((uint64_t )ICE_TX_DESC_CMD_RS ) <<ICE_TXD_QW1_CMD_S ); txq->tx_next_rs =
drivers/net/ice/ice_rxtx_vec_common.h +2 -2
@@ -22,7 +22,7 @@ ice_tx_free_bufs_vec(struct ice_tx_queue *txq)struct rte_mbuf *m, *free[ICE_TX_MAX_FREE_BUF_SZ ];/* check DD bits on threshold descriptor */ -if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz & +if ((txq->ice_tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &rte_cpu_to_le_64 (ICE_TXD_QW1_DTYPE_M )) !=rte_cpu_to_le_64 (ICE_TX_DESC_DTYPE_DESC_DONE ))return 0 ;
@@ -121,7 +121,7 @@ _ice_tx_queue_release_mbufs_vec(struct ice_tx_queue *txq) i = txq->tx_next_dd - txq->tx_rs_thresh +1 ;#ifdef __AVX512VL__ -struct rte_eth_dev *dev = &rte_eth_devices[txq->vsi->adapter->pf.dev_data->port_id]; +struct rte_eth_dev *dev = &rte_eth_devices[txq->ice_vsi->adapter->pf.dev_data->port_id];if (dev->tx_pkt_burst == ice_xmit_pkts_vec_avx512 || dev->tx_pkt_burst == ice_xmit_pkts_vec_avx512_offload) {
drivers/net/ice/ice_rxtx_vec_sse.c +3 -3
@@ -717,7 +717,7 @@ ice_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,return 0 ; tx_id = txq->tx_tail; - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->ice_tx_ring[tx_id]; txep = &txq->sw_ring[tx_id]; txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts);
@@ -737,7 +737,7 @@ ice_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, txq->tx_next_rs = (uint16_t )(txq->tx_rs_thresh -1 );/* avoid reach the end of ring */ - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->ice_tx_ring[tx_id]; txep = &txq->sw_ring[tx_id]; }
@@ -747,7 +747,7 @@ ice_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = (uint16_t )(tx_id + nb_commit);if (tx_id > txq->tx_next_rs) { - txq->tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |= + txq->ice_tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |=rte_cpu_to_le_64 (((uint64_t )ICE_TX_DESC_CMD_RS ) <<ICE_TXD_QW1_CMD_S ); txq->tx_next_rs =
drivers/net/ixgbe/ixgbe_recycle_mbufs_vec_common.c +1 -1
@@ -72,7 +72,7 @@ ixgbe_recycle_tx_mbufs_reuse_vec(void *tx_queue,return 0 ;/* check DD bits on threshold descriptor */ - status = txq->tx_ring[txq->tx_next_dd].wb.status; + status = txq->ixgbe_tx_ring[txq->tx_next_dd].wb.status;if (!(status &IXGBE_ADVTXD_STAT_DD ))return 0 ;
drivers/net/ixgbe/ixgbe_rxtx.c +11 -11
@@ -106,7 +106,7 @@ ixgbe_tx_free_bufs(struct ixgbe_tx_queue *txq)struct rte_mbuf *m, *free[RTE_IXGBE_TX_MAX_FREE_BUF_SZ ];/* check DD bit on threshold descriptor */ - status = txq->tx_ring[txq->tx_next_dd].wb.status; + status = txq->ixgbe_tx_ring[txq->tx_next_dd].wb.status;if (!(status &rte_cpu_to_le_32 (IXGBE_ADVTXD_STAT_DD )))return 0 ;
@@ -198,7 +198,7 @@ static inline void ixgbe_tx_fill_hw_ring(struct ixgbe_tx_queue *txq, struct rte_mbuf **pkts, uint16_t nb_pkts) { - volatile union ixgbe_adv_tx_desc *txdp = &(txq->tx_ring[txq->tx_tail]); + volatile union ixgbe_adv_tx_desc *txdp = &txq->ixgbe_tx_ring[txq->tx_tail];struct ci_tx_entry *txep = &txq->sw_ring[txq->tx_tail];const int N_PER_LOOP =4 ;const int N_PER_LOOP_MASK =N_PER_LOOP -1 ;
@@ -232,7 +232,7 @@ tx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) {struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue; -volatile union ixgbe_adv_tx_desc *tx_r = txq->tx_ring; +volatile union ixgbe_adv_tx_desc *tx_r = txq->ixgbe_tx_ring;uint16_t n =0 ;/*
@@ -564,7 +564,7 @@ static inline int ixgbe_xmit_cleanup(struct ixgbe_tx_queue *txq) {struct ci_tx_entry *sw_ring = txq->sw_ring; -volatile union ixgbe_adv_tx_desc *txr = txq->tx_ring; +volatile union ixgbe_adv_tx_desc *txr = txq->ixgbe_tx_ring;uint16_t last_desc_cleaned = txq->last_desc_cleaned;uint16_t nb_tx_desc = txq->nb_tx_desc;uint16_t desc_to_clean_to;
@@ -652,7 +652,7 @@ ixgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, tx_offload.data[1 ] =0 ; txq = tx_queue; sw_ring = txq->sw_ring; - txr = txq->tx_ring; + txr = txq->ixgbe_tx_ring; tx_id = txq->tx_tail; txe = &sw_ring[tx_id]; txp =NULL ;
@@ -2495,13 +2495,13 @@ ixgbe_reset_tx_queue(struct ixgbe_tx_queue *txq)/* Zero out HW ring memory */ for (i =0 ; i < txq->nb_tx_desc; i++) { - txq->tx_ring[i] = zeroed_desc; + txq->ixgbe_tx_ring[i] = zeroed_desc; }/* Initialize SW ring entries */ prev = (uint16_t ) (txq->nb_tx_desc -1 );for (i =0 ; i < txq->nb_tx_desc; i++) { -volatile union ixgbe_adv_tx_desc *txd = &txq->tx_ring[i]; +volatile union ixgbe_adv_tx_desc *txd = &txq->ixgbe_tx_ring[i]; txd->wb.status =rte_cpu_to_le_32 (IXGBE_TXD_STAT_DD ); txe[i].mbuf =NULL ;
@@ -2751,7 +2751,7 @@ ixgbe_dev_tx_queue_setup(struct rte_eth_dev *dev,* handle the maximum ring size is allocated in order to allow for * resizing in later calls to the queue setup function. */ - tz =rte_eth_dma_zone_reserve (dev,"tx_ring" , queue_idx, + tz =rte_eth_dma_zone_reserve (dev,"ixgbe_tx_ring" , queue_idx,sizeof (union ixgbe_adv_tx_desc ) *IXGBE_MAX_RING_DESC ,IXGBE_ALIGN , socket_id);if (tz ==NULL ) {
@@ -2791,7 +2791,7 @@ ixgbe_dev_tx_queue_setup(struct rte_eth_dev *dev, txq->qtx_tail =IXGBE_PCI_REG_ADDR (hw,IXGBE_TDT (txq->reg_idx)); txq->tx_ring_dma = tz->iova; - txq->tx_ring = (union ixgbe_adv_tx_desc *) tz->addr; + txq->ixgbe_tx_ring = (union ixgbe_adv_tx_desc *)tz->addr;/* Allocate software ring */ txq->sw_ring = rte_zmalloc_socket("txq->sw_ring",
@@ -2802,7 +2802,7 @@ ixgbe_dev_tx_queue_setup(struct rte_eth_dev *dev,return -ENOMEM ; }PMD_INIT_LOG (DEBUG ,"sw_ring=%p hw_ring=%p dma_addr=0x%" PRIx64, - txq->sw_ring, txq->tx_ring, txq->tx_ring_dma); + txq->sw_ring, txq->ixgbe_tx_ring, txq->tx_ring_dma);/* set up vector or scalar TX function as appropriate */ ixgbe_set_tx_function (dev, txq);
@@ -3328,7 +3328,7 @@ ixgbe_dev_tx_descriptor_status(void *tx_queue, uint16_t offset) desc -= txq->nb_tx_desc; } - status = &txq->tx_ring[desc].wb.status; + status = &txq->ixgbe_tx_ring[desc].wb.status;if (*status &rte_cpu_to_le_32 (IXGBE_ADVTXD_STAT_DD ))return RTE_ETH_TX_DESC_DONE ;
drivers/net/ixgbe/ixgbe_rxtx.h +1 -1
@@ -185,7 +185,7 @@ struct ixgbe_advctx_info {*/ struct ixgbe_tx_queue {/** TX ring virtual address. */ -volatile union ixgbe_adv_tx_desc *tx_ring; +volatile union ixgbe_adv_tx_desc *ixgbe_tx_ring;rte_iova_t tx_ring_dma;/**< TX ring DMA address. */ union {struct ci_tx_entry *sw_ring;/**< address of SW ring for scalar PMD. */
drivers/net/ixgbe/ixgbe_rxtx_vec_common.h +3 -3
@@ -22,7 +22,7 @@ ixgbe_tx_free_bufs(struct ixgbe_tx_queue *txq)struct rte_mbuf *m, *free[RTE_IXGBE_TX_MAX_FREE_BUF_SZ ];/* check DD bit on threshold descriptor */ - status = txq->tx_ring[txq->tx_next_dd].wb.status; + status = txq->ixgbe_tx_ring[txq->tx_next_dd].wb.status;if (!(status &IXGBE_ADVTXD_STAT_DD ))return 0 ;
@@ -154,11 +154,11 @@ _ixgbe_reset_tx_queue_vec(struct ixgbe_tx_queue *txq)/* Zero out HW ring memory */ for (i =0 ; i < txq->nb_tx_desc; i++) - txq->tx_ring[i] = zeroed_desc; + txq->ixgbe_tx_ring[i] = zeroed_desc;/* Initialize SW ring entries */ for (i =0 ; i < txq->nb_tx_desc; i++) { -volatile union ixgbe_adv_tx_desc *txd = &txq->tx_ring[i]; +volatile union ixgbe_adv_tx_desc *txd = &txq->ixgbe_tx_ring[i]; txd->wb.status =IXGBE_TXD_STAT_DD ; txe[i].mbuf =NULL ;
drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c +3 -3
@@ -590,7 +590,7 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,return 0 ; tx_id = txq->tx_tail; - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->ixgbe_tx_ring[tx_id]; txep = &txq->sw_ring_v[tx_id]; txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts);
@@ -610,7 +610,7 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, txq->tx_next_rs = (uint16_t )(txq->tx_rs_thresh -1 );/* avoid reach the end of ring */ - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->ixgbe_tx_ring[tx_id]; txep = &txq->sw_ring_v[tx_id]; }
@@ -620,7 +620,7 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = (uint16_t )(tx_id + nb_commit);if (tx_id > txq->tx_next_rs) { - txq->tx_ring[txq->tx_next_rs].read.cmd_type_len |= + txq->ixgbe_tx_ring[txq->tx_next_rs].read.cmd_type_len |=rte_cpu_to_le_32 (IXGBE_ADVTXD_DCMD_RS ); txq->tx_next_rs = (uint16_t )(txq->tx_next_rs + txq->tx_rs_thresh);
drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c +3 -3
@@ -712,7 +712,7 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,return 0 ; tx_id = txq->tx_tail; - txdp = &txq->tx_ring[tx_id]; + txdp = &txq->ixgbe_tx_ring[tx_id]; txep = &txq->sw_ring_v[tx_id]; txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts);
@@ -733,7 +733,7 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, txq->tx_next_rs = (uint16_t )(txq->tx_rs_thresh -1 );/* avoid reach the end of ring */ - txdp = &(txq->tx_ring[tx_id]); + txdp = &txq->ixgbe_tx_ring[tx_id]; txep = &txq->sw_ring_v[tx_id]; }
@@ -743,7 +743,7 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = (uint16_t )(tx_id + nb_commit);if (tx_id > txq->tx_next_rs) { - txq->tx_ring[txq->tx_next_rs].read.cmd_type_len |= + txq->ixgbe_tx_ring[txq->tx_next_rs].read.cmd_type_len |=rte_cpu_to_le_32 (IXGBE_ADVTXD_DCMD_RS ); txq->tx_next_rs = (uint16_t )(txq->tx_next_rs + txq->tx_rs_thresh);
Signature · 1 line
2.43.0
PATCH v3 6/22 net/_common_intel: merge ice and i40e Tx queue struct
In reply to Bruce RichardsonThe queue structures of i40e and ice drivers are virtually identical, so merge them into a common struct. This should allow easier function merging in future using that common struct.
--- drivers/net/_common_intel/tx.h | 55 +++++++++++++++++ drivers/net/i40e/i40e_ethdev.c | 4 +- drivers/net/i40e/i40e_ethdev.h | 4 +- drivers/net/i40e/i40e_fdir.c | 4 +- .../net/i40e/i40e_recycle_mbufs_vec_common.c | 2 +- drivers/net/i40e/i40e_rxtx.c | 58 +++++++++--------- drivers/net/i40e/i40e_rxtx.h | 50 ++-------------- drivers/net/i40e/i40e_rxtx_vec_altivec.c | 4 +- drivers/net/i40e/i40e_rxtx_vec_avx2.c | 4 +- drivers/net/i40e/i40e_rxtx_vec_avx512.c | 6 +- drivers/net/i40e/i40e_rxtx_vec_common.h | 2 +- drivers/net/i40e/i40e_rxtx_vec_neon.c | 4 +- drivers/net/i40e/i40e_rxtx_vec_sse.c | 4 +- drivers/net/ice/ice_dcf.c | 4 +- drivers/net/ice/ice_dcf_ethdev.c | 10 ++-- drivers/net/ice/ice_diagnose.c | 2 +- drivers/net/ice/ice_ethdev.c | 2 +- drivers/net/ice/ice_ethdev.h | 4 +- drivers/net/ice/ice_rxtx.c | 60 +++++++++---------- drivers/net/ice/ice_rxtx.h | 41 +------------ drivers/net/ice/ice_rxtx_vec_avx2.c | 4 +- drivers/net/ice/ice_rxtx_vec_avx512.c | 8 +-- drivers/net/ice/ice_rxtx_vec_common.h | 8 +-- drivers/net/ice/ice_rxtx_vec_sse.c | 6 +- 24 files changed, 165 insertions(+), 185 deletions(-)
drivers/net/_common_intel/tx.h +55 -0
@@ -8,6 +8,9 @@#include <stdint.h> #include <rte_mbuf.h> +/* forward declaration of the common intel (ci) queue structure */ +struct ci_tx_queue ; +/** * Structure associated with each descriptor of the TX ring of a TX queue. */
@@ -24,6 +27,58 @@ struct ci_tx_entry_vec {struct rte_mbuf *mbuf;/* mbuf associated with TX desc, if any. */ }; +typedef void (*ice_tx_release_mbufs_t )(struct ci_tx_queue *txq); + +struct ci_tx_queue { +union {/* TX ring virtual address */ +volatile struct ice_tx_desc *ice_tx_ring; +volatile struct i40e_tx_desc *i40e_tx_ring; + }; +volatile uint8_t *qtx_tail;/* register address of tail */ +struct ci_tx_entry *sw_ring;/* virtual address of SW ring */ +rte_iova_t tx_ring_dma;/* TX ring DMA address */ +uint16_t nb_tx_desc;/* number of TX descriptors */ +uint16_t tx_tail;/* current value of tail register */ +uint16_t nb_tx_used;/* number of TX desc used since RS bit set */ +/* index to last TX descriptor to have been cleaned */ +uint16_t last_desc_cleaned; +/* Total number of TX descriptors ready to be allocated. */ +uint16_t nb_tx_free; +/* Start freeing TX buffers if there are less free descriptors than +* this value. +*/ +uint16_t tx_free_thresh; +/* Number of TX descriptors to use before RS bit is set. */ +uint16_t tx_rs_thresh; +uint8_t pthresh;/**< Prefetch threshold register. */ +uint8_t hthresh;/**< Host threshold register. */ +uint8_t wthresh;/**< Write-back threshold reg. */ +uint16_t port_id;/* Device port identifier. */ +uint16_t queue_id;/* TX queue index. */ +uint16_t reg_idx; +uint64_t offloads; +uint16_t tx_next_dd; +uint16_t tx_next_rs; +uint64_t mbuf_errors; +bool tx_deferred_start;/* don't start this queue in dev start */ +bool q_set;/* indicate if tx queue has been configured */ +union {/* the VSI this queue belongs to */ +struct ice_vsi *ice_vsi; +struct i40e_vsi *i40e_vsi; + }; +const struct rte_memzone *mz; + +union { +struct {/* ICE driver specific values */ +ice_tx_release_mbufs_t tx_rel_mbufs; +uint32_t q_teid;/* TX schedule node id. */ + }; +struct {/* I40E driver specific values */ +uint8_t dcb_tc; + }; + }; +}; + static __rte_always_inline voidci_tx_backlog_entry (struct ci_tx_entry *txep,struct rte_mbuf **tx_pkts,uint16_t nb_pkts) {
drivers/net/i40e/i40e_ethdev.c +2 -2
@@ -3685,7 +3685,7 @@ i40e_dev_update_mbuf_stats(struct rte_eth_dev *ethdev, struct i40e_mbuf_stats *mbuf_stats) {uint16_t idx; -struct i40e_tx_queue *txq; +struct ci_tx_queue *txq;for (idx =0 ; idx < ethdev->data->nb_tx_queues; idx++) { txq = ethdev->data->tx_queues[idx];
@@ -6585,7 +6585,7 @@ i40e_dev_tx_init(struct i40e_pf *pf)struct rte_eth_dev_data *data = pf->dev_data;uint16_t i;uint32_t ret =I40E_SUCCESS ; -struct i40e_tx_queue *txq; +struct ci_tx_queue *txq;for (i =0 ; i < data->nb_tx_queues; i++) { txq = data->tx_queues[i];
drivers/net/i40e/i40e_ethdev.h +2 -2
@@ -334,7 +334,7 @@ struct i40e_vsi_list { };struct i40e_rx_queue ; -struct i40e_tx_queue ; +struct ci_tx_queue ;/* Bandwidth limit information */ struct i40e_bw_info {
@@ -738,7 +738,7 @@ TAILQ_HEAD(i40e_fdir_filter_list, i40e_fdir_filter);struct i40e_fdir_info {struct i40e_vsi *fdir_vsi;/* pointer to fdir VSI structure */ uint16_t match_counter_index;/* Statistic counter index used for fdir*/ -struct i40e_tx_queue *txq; +struct ci_tx_queue *txq;struct i40e_rx_queue *rxq;void *prg_pkt[I40E_FDIR_PRG_PKT_CNT ];/* memory for fdir program packet */ uint64_t dma_addr[I40E_FDIR_PRG_PKT_CNT ];/* physic address of packet memory*/
drivers/net/i40e/i40e_fdir.c +2 -2
@@ -1372,7 +1372,7 @@ i40e_find_available_buffer(struct rte_eth_dev *dev) {struct i40e_pf *pf =I40E_DEV_PRIVATE_TO_PF (dev->data->dev_private);struct i40e_fdir_info *fdir_info = &pf->fdir; -struct i40e_tx_queue *txq = pf->fdir.txq; +struct ci_tx_queue *txq = pf->fdir.txq;/* no available buffer * search for more available buffers from the current
@@ -1628,7 +1628,7 @@ i40e_flow_fdir_filter_programming(struct i40e_pf *pf,const struct i40e_fdir_filter_conf *filter, bool add, bool wait_status) { - struct i40e_tx_queue *txq = pf->fdir.txq; + struct ci_tx_queue *txq = pf->fdir.txq;struct i40e_rx_queue *rxq = pf->fdir.rxq;const struct i40e_fdir_action *fdir_action = &filter->action;volatile struct i40e_tx_desc *txdp;
drivers/net/i40e/i40e_recycle_mbufs_vec_common.c +1 -1
@@ -55,7 +55,7 @@ uint16_ti40e_recycle_tx_mbufs_reuse_vec (void *tx_queue, struct rte_eth_recycle_rxq_info *recycle_rxq_info) { -struct i40e_tx_queue *txq = tx_queue; +struct ci_tx_queue *txq = tx_queue;struct ci_tx_entry *txep;struct rte_mbuf **rxep;int i, n;
drivers/net/i40e/i40e_rxtx.c +29 -29
232 lines not on this page · raw message
drivers/net/i40e/i40e_rxtx.h +6 -44
78 lines not on this page · raw message
drivers/net/i40e/i40e_rxtx_vec_altivec.c +2 -2
@@ -551,7 +551,7 @@ uint16_t i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { -struct i40e_tx_queue *txq = (struct i40e_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile struct i40e_tx_desc *txdp;struct ci_tx_entry *txep;uint16_t n, nb_commit, tx_id;
@@ -625,7 +625,7 @@ i40e_rxq_vec_setup(struct i40e_rx_queue *rxq) } int __rte_cold -i40e_txq_vec_setup(struct i40e_tx_queue __rte_unused * txq) +i40e_txq_vec_setup(struct ci_tx_queue __rte_unused * txq) {return 0 ; }
drivers/net/i40e/i40e_rxtx_vec_avx2.c +2 -2
@@ -743,7 +743,7 @@ static inline uint16_t i40e_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { -struct i40e_tx_queue *txq = (struct i40e_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile struct i40e_tx_desc *txdp;struct ci_tx_entry *txep;uint16_t n, nb_commit, tx_id;
@@ -808,7 +808,7 @@ i40e_xmit_pkts_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) {uint16_t nb_tx =0 ; -struct i40e_tx_queue *txq = (struct i40e_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;while (nb_pkts) {uint16_t ret, num;
drivers/net/i40e/i40e_rxtx_vec_avx512.c +3 -3
@@ -755,7 +755,7 @@ i40e_recv_scattered_pkts_vec_avx512(void *rx_queue, } static __rte_always_inline int -i40e_tx_free_bufs_avx512 (struct i40e_tx_queue *txq) +i40e_tx_free_bufs_avx512 (struct ci_tx_queue *txq) {struct ci_tx_entry_vec *txep;uint32_t n;
@@ -933,7 +933,7 @@ static inline uint16_t i40e_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { -struct i40e_tx_queue *txq = (struct i40e_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile struct i40e_tx_desc *txdp;struct ci_tx_entry_vec *txep;uint16_t n, nb_commit, tx_id;
@@ -999,7 +999,7 @@ i40e_xmit_pkts_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) {uint16_t nb_tx =0 ; -struct i40e_tx_queue *txq = (struct i40e_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;while (nb_pkts) {uint16_t ret, num;
drivers/net/i40e/i40e_rxtx_vec_common.h +1 -1
@@ -17,7 +17,7 @@#endif static __rte_always_inline int -i40e_tx_free_bufs (struct i40e_tx_queue *txq) +i40e_tx_free_bufs (struct ci_tx_queue *txq) {struct ci_tx_entry *txep;uint32_t n;
drivers/net/i40e/i40e_rxtx_vec_neon.c +2 -2
@@ -679,7 +679,7 @@ uint16_t i40e_xmit_fixed_burst_vec(void *__rte_restrict tx_queue, struct rte_mbuf **__rte_restrict tx_pkts, uint16_t nb_pkts) { -struct i40e_tx_queue *txq = (struct i40e_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile struct i40e_tx_desc *txdp;struct ci_tx_entry *txep;uint16_t n, nb_commit, tx_id;
@@ -753,7 +753,7 @@ i40e_rxq_vec_setup(struct i40e_rx_queue *rxq) } int __rte_cold -i40e_txq_vec_setup(struct i40e_tx_queue __rte_unused *txq) +i40e_txq_vec_setup(struct ci_tx_queue *txq __rte_unused) {return 0 ; }
drivers/net/i40e/i40e_rxtx_vec_sse.c +2 -2
@@ -698,7 +698,7 @@ uint16_t i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { -struct i40e_tx_queue *txq = (struct i40e_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile struct i40e_tx_desc *txdp;struct ci_tx_entry *txep;uint16_t n, nb_commit, tx_id;
@@ -771,7 +771,7 @@ i40e_rxq_vec_setup(struct i40e_rx_queue *rxq) } int __rte_cold -i40e_txq_vec_setup(struct i40e_tx_queue __rte_unused *txq) +i40e_txq_vec_setup(struct ci_tx_queue *txq __rte_unused) {return 0 ; }
drivers/net/ice/ice_dcf.c +2 -2
@@ -1177,8 +1177,8 @@ ice_dcf_configure_queues(struct ice_dcf_hw *hw) {struct ice_rx_queue **rxq = (struct ice_rx_queue **)hw->eth_dev->data->rx_queues; -struct ice_tx_queue **txq = - (struct ice_tx_queue **)hw->eth_dev->data->tx_queues; +struct ci_tx_queue **txq = + (struct ci_tx_queue **)hw->eth_dev->data->tx_queues;struct virtchnl_vsi_queue_config_info *vc_config;struct virtchnl_queue_pair_info *vc_qp;struct dcf_virtchnl_cmd args;
drivers/net/ice/ice_dcf_ethdev.c +5 -5
@@ -387,7 +387,7 @@ reset_rx_queue(struct ice_rx_queue *rxq) }static inline void -reset_tx_queue (struct ice_tx_queue *txq) +reset_tx_queue (struct ci_tx_queue *txq) {struct ci_tx_entry *txe;uint32_t i, size;
@@ -454,7 +454,7 @@ ice_dcf_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id) {struct ice_dcf_adapter *ad = dev->data->dev_private;struct iavf_hw *hw = &ad->real_hw.avf; -struct ice_tx_queue *txq; +struct ci_tx_queue *txq;int err =0 ;if (tx_queue_id >= dev->data->nb_tx_queues)
@@ -486,7 +486,7 @@ ice_dcf_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id) {struct ice_dcf_adapter *ad = dev->data->dev_private;struct ice_dcf_hw *hw = &ad->real_hw; -struct ice_tx_queue *txq; +struct ci_tx_queue *txq;int err;if (tx_queue_id >= dev->data->nb_tx_queues)
@@ -511,7 +511,7 @@ static int ice_dcf_start_queues(struct rte_eth_dev *dev) {struct ice_rx_queue *rxq; -struct ice_tx_queue *txq; +struct ci_tx_queue *txq;int nb_rxq =0 ;int nb_txq, i;
@@ -638,7 +638,7 @@ ice_dcf_stop_queues(struct rte_eth_dev *dev)struct ice_dcf_adapter *ad = dev->data->dev_private;struct ice_dcf_hw *hw = &ad->real_hw;struct ice_rx_queue *rxq; -struct ice_tx_queue *txq; +struct ci_tx_queue *txq;int ret, i;/* Stop All queues */
drivers/net/ice/ice_diagnose.c +1 -1
@@ -605,7 +605,7 @@ void print_node(const struct rte_eth_dev_data *ethdata, get_elem_type(data->data.elem_type));if (data->data.elem_type ==ICE_AQC_ELEM_TYPE_LEAF ) {for (uint16_t i =0 ; i < ethdata->nb_tx_queues; i++) { -struct ice_tx_queue *q = ethdata->tx_queues[i]; +struct ci_tx_queue *q = ethdata->tx_queues[i];if (q->q_teid == data->node_teid) {fprintf (stream,"\t\t\t\t<tr><td>TXQ</td><td>%u</td></tr>\n" , i);break ;
drivers/net/ice/ice_ethdev.c +1 -1
@@ -6448,7 +6448,7 @@ ice_update_mbuf_stats(struct rte_eth_dev *ethdev, struct ice_mbuf_stats *mbuf_stats) {uint16_t idx; -struct ice_tx_queue *txq; +struct ci_tx_queue *txq;for (idx =0 ; idx < ethdev->data->nb_tx_queues; idx++) { txq = ethdev->data->tx_queues[idx];
drivers/net/ice/ice_ethdev.h +2 -2
@@ -258,7 +258,7 @@ struct ice_vsi_list { };struct ice_rx_queue ; -struct ice_tx_queue ; +struct ci_tx_queue ;/** * Structure that defines a VSI, associated with a adapter.
@@ -408,7 +408,7 @@ struct ice_fdir_counter_pool_container {*/ struct ice_fdir_info {struct ice_vsi *fdir_vsi;/* pointer to fdir VSI structure */ -struct ice_tx_queue *txq; +struct ci_tx_queue *txq;struct ice_rx_queue *rxq;void *prg_pkt;/* memory for fdir program packet */ uint64_t dma_addr;/* physic address of packet memory*/
drivers/net/ice/ice_rxtx.c +30 -30
240 lines not on this page · raw message
drivers/net/ice/ice_rxtx.h +2 -39
65 lines not on this page · raw message
drivers/net/ice/ice_rxtx_vec_avx2.c +2 -2
@@ -856,7 +856,7 @@ static __rte_always_inline uint16_t ice_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts, bool offload) { -struct ice_tx_queue *txq = (struct ice_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile struct ice_tx_desc *txdp;struct ci_tx_entry *txep;uint16_t n, nb_commit, tx_id;
@@ -924,7 +924,7 @@ ice_xmit_pkts_vec_avx2_common(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts, bool offload) { uint16_t nb_tx = 0; -struct ice_tx_queue *txq = (struct ice_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;while (nb_pkts) {uint16_t ret, num;
drivers/net/ice/ice_rxtx_vec_avx512.c +4 -4
@@ -860,7 +860,7 @@ ice_recv_scattered_pkts_vec_avx512_offload(void *rx_queue, } static __rte_always_inline int -ice_tx_free_bufs_avx512 (struct ice_tx_queue *txq) +ice_tx_free_bufs_avx512 (struct ci_tx_queue *txq) {struct ci_tx_entry_vec *txep;uint32_t n;
@@ -1053,7 +1053,7 @@ static __rte_always_inline uint16_t ice_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts, bool do_offload) { -struct ice_tx_queue *txq = (struct ice_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile struct ice_tx_desc *txdp;struct ci_tx_entry_vec *txep;uint16_t n, nb_commit, tx_id;
@@ -1122,7 +1122,7 @@ ice_xmit_pkts_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) {uint16_t nb_tx =0 ; -struct ice_tx_queue *txq = (struct ice_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;while (nb_pkts) {uint16_t ret, num;
@@ -1144,7 +1144,7 @@ ice_xmit_pkts_vec_avx512_offload(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) {uint16_t nb_tx =0 ; -struct ice_tx_queue *txq = (struct ice_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;while (nb_pkts) {uint16_t ret, num;
drivers/net/ice/ice_rxtx_vec_common.h +4 -4
@@ -13,7 +13,7 @@#endif static __rte_always_inline int -ice_tx_free_bufs_vec (struct ice_tx_queue *txq) +ice_tx_free_bufs_vec (struct ci_tx_queue *txq) {struct ci_tx_entry *txep;uint32_t n;
@@ -105,7 +105,7 @@ _ice_rx_queue_release_mbufs_vec(struct ice_rx_queue *rxq) }static inline void -_ice_tx_queue_release_mbufs_vec (struct ice_tx_queue *txq) +_ice_tx_queue_release_mbufs_vec (struct ci_tx_queue *txq) {uint16_t i;
@@ -231,7 +231,7 @@ ice_rx_vec_queue_default(struct ice_rx_queue *rxq) }static inline int -ice_tx_vec_queue_default (struct ice_tx_queue *txq) +ice_tx_vec_queue_default (struct ci_tx_queue *txq) {if (!txq)return -1 ;
@@ -273,7 +273,7 @@ static inline int ice_tx_vec_dev_check_default(struct rte_eth_dev *dev) {int i; -struct ice_tx_queue *txq; +struct ci_tx_queue *txq;int ret =0 ;int result =0 ;
drivers/net/ice/ice_rxtx_vec_sse.c +3 -3
@@ -697,7 +697,7 @@ static uint16_t ice_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { -struct ice_tx_queue *txq = (struct ice_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile struct ice_tx_desc *txdp;struct ci_tx_entry *txep;uint16_t n, nb_commit, tx_id;
@@ -766,7 +766,7 @@ ice_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) {uint16_t nb_tx =0 ; -struct ice_tx_queue *txq = (struct ice_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;while (nb_pkts) {uint16_t ret, num;
@@ -793,7 +793,7 @@ ice_rxq_vec_setup(struct ice_rx_queue *rxq) } int __rte_cold -ice_txq_vec_setup(struct ice_tx_queue __rte_unused *txq) +ice_txq_vec_setup(struct ci_tx_queue *txq __rte_unused) {if (!txq)return -1 ;
Signature · 1 line
2.43.0
PATCH v3 7/22 net/iavf: use common Tx queue structure
In reply to Bruce RichardsonMerge in the few additional fields used by iavf driver and convert it to using the common Tx queue structure also.
--- drivers/net/_common_intel/tx.h | 15 +++++++- drivers/net/iavf/iavf.h | 2 +- drivers/net/iavf/iavf_ethdev.c | 4 +- drivers/net/iavf/iavf_rxtx.c | 42 ++++++++++----------- drivers/net/iavf/iavf_rxtx.h | 49 +++---------------------- drivers/net/iavf/iavf_rxtx_vec_avx2.c | 4 +- drivers/net/iavf/iavf_rxtx_vec_avx512.c | 14 +++---- drivers/net/iavf/iavf_rxtx_vec_common.h | 8 ++-- drivers/net/iavf/iavf_rxtx_vec_sse.c | 8 ++-- drivers/net/iavf/iavf_vchnl.c | 6 +-- 10 files changed, 62 insertions(+), 90 deletions(-)
drivers/net/_common_intel/tx.h +13 -2
@@ -31,8 +31,9 @@ typedef void (*ice_tx_release_mbufs_t)(struct ci_tx_queue *txq);struct ci_tx_queue {union {/* TX ring virtual address */ -volatile struct ice_tx_desc *ice_tx_ring;volatile struct i40e_tx_desc *i40e_tx_ring; +volatile struct iavf_tx_desc *iavf_tx_ring; +volatile struct ice_tx_desc *ice_tx_ring; };volatile uint8_t *qtx_tail;/* register address of tail */ struct ci_tx_entry *sw_ring;/* virtual address of SW ring */
@@ -63,8 +64,9 @@ struct ci_tx_queue {bool tx_deferred_start;/* don't start this queue in dev start */ bool q_set;/* indicate if tx queue has been configured */ union {/* the VSI this queue belongs to */ -struct ice_vsi *ice_vsi;struct i40e_vsi *i40e_vsi; +struct iavf_vsi *iavf_vsi; +struct ice_vsi *ice_vsi; };const struct rte_memzone *mz;
@@ -76,6 +78,15 @@ struct ci_tx_queue {struct {/* I40E driver specific values */ uint8_t dcb_tc; }; +struct {/* iavf driver specific values */ +uint16_t ipsec_crypto_pkt_md_offset; +uint8_t rel_mbufs_type; +#define IAVF_TX_FLAGS_VLAN_TAG_LOC_L2TAG1 BIT(0) +#define IAVF_TX_FLAGS_VLAN_TAG_LOC_L2TAG2 BIT(1) +uint8_t vlan_flag; +uint8_t tc; +bool use_ctx;/* with ctx info, each pkt needs two descriptors */ + }; }; };
drivers/net/iavf/iavf.h +1 -1
@@ -98,7 +98,7 @@struct iavf_adapter ;struct iavf_rx_queue ; -struct iavf_tx_queue ; +struct ci_tx_queue ;struct iavf_ipsec_crypto_stats {
drivers/net/iavf/iavf_ethdev.c +2 -2
@@ -954,7 +954,7 @@ static int iavf_start_queues(struct rte_eth_dev *dev) {struct iavf_rx_queue *rxq; -struct iavf_tx_queue *txq; +struct ci_tx_queue *txq;int i;uint16_t nb_txq, nb_rxq;
@@ -1885,7 +1885,7 @@ iavf_dev_update_mbuf_stats(struct rte_eth_dev *ethdev, struct iavf_mbuf_stats *mbuf_stats) {uint16_t idx; -struct iavf_tx_queue *txq; +struct ci_tx_queue *txq;for (idx =0 ; idx < ethdev->data->nb_tx_queues; idx++) { txq = ethdev->data->tx_queues[idx];
drivers/net/iavf/iavf_rxtx.c +21 -21
168 lines not on this page · raw message
drivers/net/iavf/iavf_rxtx.h +6 -43
84 lines not on this page · raw message
drivers/net/iavf/iavf_rxtx_vec_avx2.c +2 -2
@@ -1734,7 +1734,7 @@ static __rte_always_inline uint16_t iavf_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts, bool offload) { -struct iavf_tx_queue *txq = (struct iavf_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile struct iavf_tx_desc *txdp;struct ci_tx_entry *txep;uint16_t n, nb_commit, tx_id;
@@ -1801,7 +1801,7 @@ iavf_xmit_pkts_vec_avx2_common(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts, bool offload) { uint16_t nb_tx = 0; -struct iavf_tx_queue *txq = (struct iavf_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;while (nb_pkts) {uint16_t ret, num;
drivers/net/iavf/iavf_rxtx_vec_avx512.c +7 -7
@@ -1845,7 +1845,7 @@ iavf_recv_scattered_pkts_vec_avx512_flex_rxd_offload(void *rx_queue, } static __rte_always_inline int -iavf_tx_free_bufs_avx512 (struct iavf_tx_queue *txq) +iavf_tx_free_bufs_avx512 (struct ci_tx_queue *txq) {struct ci_tx_entry_vec *txep;uint32_t n;
@@ -2311,7 +2311,7 @@ static __rte_always_inline uint16_t iavf_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts, bool offload) { -struct iavf_tx_queue *txq = (struct iavf_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile struct iavf_tx_desc *txdp;struct ci_tx_entry_vec *txep;uint16_t n, nb_commit, tx_id;
@@ -2379,7 +2379,7 @@ static __rte_always_inline uint16_t iavf_xmit_fixed_burst_vec_avx512_ctx(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts, bool offload) { -struct iavf_tx_queue *txq = (struct iavf_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile struct iavf_tx_desc *txdp;struct ci_tx_entry_vec *txep;uint16_t n, nb_commit, nb_mbuf, tx_id;
@@ -2447,7 +2447,7 @@ iavf_xmit_pkts_vec_avx512_cmn(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts, bool offload) { uint16_t nb_tx = 0; -struct iavf_tx_queue *txq = (struct iavf_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;while (nb_pkts) {uint16_t ret, num;
@@ -2473,7 +2473,7 @@ iavf_xmit_pkts_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, } void __rte_cold -iavf_tx_queue_release_mbufs_avx512 (struct iavf_tx_queue *txq) +iavf_tx_queue_release_mbufs_avx512 (struct ci_tx_queue *txq) {unsigned int i;const uint16_t max_desc = (uint16_t )(txq->nb_tx_desc -1 );
@@ -2494,7 +2494,7 @@ iavf_tx_queue_release_mbufs_avx512(struct iavf_tx_queue *txq) } int __rte_cold -iavf_txq_vec_setup_avx512 (struct iavf_tx_queue *txq) +iavf_txq_vec_setup_avx512 (struct ci_tx_queue *txq) { txq->rel_mbufs_type =IAVF_REL_MBUFS_AVX512_VEC ;return 0 ;
@@ -2512,7 +2512,7 @@ iavf_xmit_pkts_vec_avx512_ctx_cmn(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts, bool offload) { uint16_t nb_tx = 0; -struct iavf_tx_queue *txq = (struct iavf_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;while (nb_pkts) {uint16_t ret, num;
drivers/net/iavf/iavf_rxtx_vec_common.h +4 -4
@@ -17,7 +17,7 @@#endif static __rte_always_inline int -iavf_tx_free_bufs (struct iavf_tx_queue *txq) +iavf_tx_free_bufs (struct ci_tx_queue *txq) {struct ci_tx_entry *txep;uint32_t n;
@@ -104,7 +104,7 @@ _iavf_rx_queue_release_mbufs_vec(struct iavf_rx_queue *rxq) }static inline void -_iavf_tx_queue_release_mbufs_vec (struct iavf_tx_queue *txq) +_iavf_tx_queue_release_mbufs_vec (struct ci_tx_queue *txq) {unsigned i;const uint16_t max_desc = (uint16_t )(txq->nb_tx_desc -1 );
@@ -164,7 +164,7 @@ iavf_rx_vec_queue_default(struct iavf_rx_queue *rxq) }static inline int -iavf_tx_vec_queue_default (struct iavf_tx_queue *txq) +iavf_tx_vec_queue_default (struct ci_tx_queue *txq) {if (!txq)return -1 ;
@@ -227,7 +227,7 @@ static inline int iavf_tx_vec_dev_check_default(struct rte_eth_dev *dev) {int i; -struct iavf_tx_queue *txq; +struct ci_tx_queue *txq;int ret;int result =0 ;
drivers/net/iavf/iavf_rxtx_vec_sse.c +4 -4
@@ -1366,7 +1366,7 @@ uint16_t iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { -struct iavf_tx_queue *txq = (struct iavf_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile struct iavf_tx_desc *txdp;struct ci_tx_entry *txep;uint16_t n, nb_commit, tx_id;
@@ -1435,7 +1435,7 @@ iavf_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) {uint16_t nb_tx =0 ; -struct iavf_tx_queue *txq = (struct iavf_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;while (nb_pkts) {uint16_t ret, num;
@@ -1459,13 +1459,13 @@ iavf_rx_queue_release_mbufs_sse(struct iavf_rx_queue *rxq) } void __rte_cold -iavf_tx_queue_release_mbufs_sse (struct iavf_tx_queue *txq) +iavf_tx_queue_release_mbufs_sse (struct ci_tx_queue *txq) {_iavf_tx_queue_release_mbufs_vec (txq); } int __rte_cold -iavf_txq_vec_setup (struct iavf_tx_queue *txq) +iavf_txq_vec_setup (struct ci_tx_queue *txq) { txq->rel_mbufs_type =IAVF_REL_MBUFS_SSE_VEC ;return 0 ;
drivers/net/iavf/iavf_vchnl.c +2 -4
@@ -1218,10 +1218,8 @@ int iavf_configure_queues(struct iavf_adapter *adapter, uint16_t num_queue_pairs, uint16_t index) { - struct iavf_rx_queue **rxq = - (struct iavf_rx_queue **)adapter->dev_data->rx_queues; -struct iavf_tx_queue **txq = - (struct iavf_tx_queue **)adapter->dev_data->tx_queues; + struct iavf_rx_queue **rxq = (struct iavf_rx_queue **)adapter->dev_data->rx_queues; +struct ci_tx_queue **txq = (struct ci_tx_queue **)adapter->dev_data->tx_queues;struct iavf_info *vf =IAVF_DEV_PRIVATE_TO_VF (adapter);struct virtchnl_vsi_queue_config_info *vc_config;struct virtchnl_queue_pair_info *vc_qp;
Signature · 1 line
2.43.0
PATCH v3 8/22 net/ixgbe: convert Tx queue context cache field to ptr
In reply to Bruce RichardsonRather than having a two element array of context cache values inside the Tx queue structure, convert it to a pointer to a cache at the end of the structure. This makes future merging of the structure easier as we don't need the "ixgbe_advctx_info" struct defined when defining a combined queue structure.
--- drivers/net/ixgbe/ixgbe_rxtx.c | 7 ++++--- drivers/net/ixgbe/ixgbe_rxtx.h | 4 ++-- drivers/net/ixgbe/ixgbe_rxtx_vec_common.h | 3 +-- 3 files changed, 7 insertions(+), 7 deletions(-)
drivers/net/ixgbe/ixgbe_rxtx.c +4 -3
@@ -2522,8 +2522,7 @@ ixgbe_reset_tx_queue(struct ixgbe_tx_queue *txq) txq->last_desc_cleaned = (uint16_t )(txq->nb_tx_desc -1 ); txq->nb_tx_free = (uint16_t )(txq->nb_tx_desc -1 ); txq->ctx_curr =0 ; -memset ((void *)&txq->ctx_cache,0 , -IXGBE_CTX_NUM *sizeof (struct ixgbe_advctx_info )); +memset (txq->ctx_cache,0 ,IXGBE_CTX_NUM *sizeof (struct ixgbe_advctx_info )); }static const struct ixgbe_txq_ops def_txq_ops = {
@@ -2741,10 +2740,12 @@ ixgbe_dev_tx_queue_setup(struct rte_eth_dev *dev, }/* First allocate the tx queue data structure */ - txq =rte_zmalloc_socket ("ethdev TX queue" ,sizeof (struct ixgbe_tx_queue ), + txq =rte_zmalloc_socket ("ethdev TX queue" ,sizeof (struct ixgbe_tx_queue ) + +sizeof (struct ixgbe_advctx_info ) *IXGBE_CTX_NUM ,RTE_CACHE_LINE_SIZE , socket_id);if (txq ==NULL )return -ENOMEM ; + txq->ctx_cache =RTE_PTR_ADD (txq,sizeof (struct ixgbe_tx_queue ));/* * Allocate TX ring hardware descriptors. A memzone large enough to
drivers/net/ixgbe/ixgbe_rxtx.h +2 -2
@@ -215,8 +215,8 @@ struct ixgbe_tx_queue {uint8_t wthresh;/**< Write-back threshold reg. */ uint64_t offloads;/**< Tx offload flags of RTE_ETH_TX_OFFLOAD_* */ uint32_t ctx_curr;/**< Hardware context states. */ -/** Hardware context0 history. */ -struct ixgbe_advctx_info ctx_cache[IXGBE_CTX_NUM ]; +/** Hardware context history. */ +struct ixgbe_advctx_info *ctx_cache;const struct ixgbe_txq_ops *ops;/**< txq ops */ bool tx_deferred_start;/**< not in global dev start. */ #ifdef RTE_LIB_SECURITY
drivers/net/ixgbe/ixgbe_rxtx_vec_common.h +1 -2
@@ -176,8 +176,7 @@ _ixgbe_reset_tx_queue_vec(struct ixgbe_tx_queue *txq) txq->last_desc_cleaned = (uint16_t )(txq->nb_tx_desc -1 ); txq->nb_tx_free = (uint16_t )(txq->nb_tx_desc -1 ); txq->ctx_curr =0 ; -memset ((void *)&txq->ctx_cache,0 , -IXGBE_CTX_NUM *sizeof (struct ixgbe_advctx_info )); +memset (txq->ctx_cache,0 ,IXGBE_CTX_NUM *sizeof (struct ixgbe_advctx_info )); } static inline int
Signature · 1 line
2.43.0
PATCH v3 9/22 net/ixgbe: use common Tx queue structure
In reply to Bruce RichardsonMerge in additional fields used by the ixgbe driver and then convert it over to using the common Tx queue structure.
--- drivers/net/_common_intel/tx.h | 14 +++- drivers/net/ixgbe/ixgbe_ethdev.c | 4 +- .../ixgbe/ixgbe_recycle_mbufs_vec_common.c | 2 +- drivers/net/ixgbe/ixgbe_rxtx.c | 64 +++++++++---------- drivers/net/ixgbe/ixgbe_rxtx.h | 56 ++-------------- drivers/net/ixgbe/ixgbe_rxtx_vec_common.h | 26 ++++---- drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c | 14 ++-- drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c | 14 ++-- 8 files changed, 80 insertions(+), 114 deletions(-)
drivers/net/_common_intel/tx.h +13 -1
@@ -34,9 +34,13 @@ struct ci_tx_queue {volatile struct i40e_tx_desc *i40e_tx_ring;volatile struct iavf_tx_desc *iavf_tx_ring;volatile struct ice_tx_desc *ice_tx_ring; +volatile union ixgbe_adv_tx_desc *ixgbe_tx_ring; };volatile uint8_t *qtx_tail;/* register address of tail */ -struct ci_tx_entry *sw_ring;/* virtual address of SW ring */ +union { +struct ci_tx_entry *sw_ring;/* virtual address of SW ring */ +struct ci_tx_entry_vec *sw_ring_vec; + };rte_iova_t tx_ring_dma;/* TX ring DMA address */ uint16_t nb_tx_desc;/* number of TX descriptors */ uint16_t tx_tail;/* current value of tail register */
@@ -87,6 +91,14 @@ struct ci_tx_queue {uint8_t tc;bool use_ctx;/* with ctx info, each pkt needs two descriptors */ }; +struct {/* ixgbe specific values */ +const struct ixgbe_txq_ops *ops; +struct ixgbe_advctx_info *ctx_cache; +uint32_t ctx_curr; +#ifdef RTE_LIB_SECURITY +uint8_t using_ipsec;/**< indicates that IPsec TX feature is in use */ +#endif + }; }; };
drivers/net/ixgbe/ixgbe_ethdev.c +2 -2
@@ -1118,7 +1118,7 @@ eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev, void *init_params __rte_unused)* RX and TX function. */ if (rte_eal_process_type () !=RTE_PROC_PRIMARY ) { -struct ixgbe_tx_queue *txq; +struct ci_tx_queue *txq;/* TX queue function in primary, set by last queue initialized * Tx queue may not initialized by primary process */
@@ -1623,7 +1623,7 @@ eth_ixgbevf_dev_init(struct rte_eth_dev *eth_dev)* RX function */ if (rte_eal_process_type () !=RTE_PROC_PRIMARY ) { -struct ixgbe_tx_queue *txq; +struct ci_tx_queue *txq;/* TX queue function in primary, set by last queue initialized * Tx queue may not initialized by primary process */
drivers/net/ixgbe/ixgbe_recycle_mbufs_vec_common.c +1 -1
@@ -51,7 +51,7 @@ uint16_tixgbe_recycle_tx_mbufs_reuse_vec (void *tx_queue, struct rte_eth_recycle_rxq_info *recycle_rxq_info) { -struct ixgbe_tx_queue *txq = tx_queue; +struct ci_tx_queue *txq = tx_queue;struct ci_tx_entry *txep;struct rte_mbuf **rxep;int i, n;
drivers/net/ixgbe/ixgbe_rxtx.c +32 -32
254 lines not on this page · raw message
drivers/net/ixgbe/ixgbe_rxtx.h +5 -51
75 lines not on this page · raw message
drivers/net/ixgbe/ixgbe_rxtx_vec_common.h +13 -13
@@ -12,7 +12,7 @@#include "ixgbe_rxtx.h" static __rte_always_inline int -ixgbe_tx_free_bufs (struct ixgbe_tx_queue *txq) +ixgbe_tx_free_bufs (struct ci_tx_queue *txq) {struct ci_tx_entry_vec *txep;uint32_t status;
@@ -32,7 +32,7 @@ ixgbe_tx_free_bufs(struct ixgbe_tx_queue *txq)* first buffer to free from S/W ring is at index * tx_next_dd - (tx_rs_thresh-1) */ - txep = &txq->sw_ring_v[txq->tx_next_dd - (n -1 )]; + txep = &txq->sw_ring_vec[txq->tx_next_dd - (n -1 )]; m =rte_pktmbuf_prefree_seg (txep[0 ].mbuf);if (likely (m !=NULL )) { free[0 ] = m;
@@ -79,7 +79,7 @@ tx_backlog_entry(struct ci_tx_entry_vec *txep, }static inline void -_ixgbe_tx_queue_release_mbufs_vec (struct ixgbe_tx_queue *txq) +_ixgbe_tx_queue_release_mbufs_vec (struct ci_tx_queue *txq) {unsigned int i;struct ci_tx_entry_vec *txe;
@@ -92,14 +92,14 @@ _ixgbe_tx_queue_release_mbufs_vec(struct ixgbe_tx_queue *txq)for (i = txq->tx_next_dd - (txq->tx_rs_thresh -1 ); i != txq->tx_tail; i = (i +1 ) % txq->nb_tx_desc) { - txe = &txq->sw_ring_v[i]; + txe = &txq->sw_ring_vec[i];rte_pktmbuf_free_seg (txe->mbuf); } txq->nb_tx_free = max_desc;/* reset tx_entry */ for (i =0 ; i < txq->nb_tx_desc; i++) { - txe = &txq->sw_ring_v[i]; + txe = &txq->sw_ring_vec[i]; txe->mbuf =NULL ; } }
@@ -134,22 +134,22 @@ _ixgbe_rx_queue_release_mbufs_vec(struct ixgbe_rx_queue *rxq) }static inline void -_ixgbe_tx_free_swring_vec (struct ixgbe_tx_queue *txq) +_ixgbe_tx_free_swring_vec (struct ci_tx_queue *txq) {if (txq ==NULL )return ;if (txq->sw_ring !=NULL ) { -rte_free (txq->sw_ring_v -1 ); - txq->sw_ring_v =NULL ; +rte_free (txq->sw_ring_vec -1 ); + txq->sw_ring_vec =NULL ; } }static inline void -_ixgbe_reset_tx_queue_vec (struct ixgbe_tx_queue *txq) +_ixgbe_reset_tx_queue_vec (struct ci_tx_queue *txq) {static const union ixgbe_adv_tx_desc zeroed_desc = { {0 } }; -struct ci_tx_entry_vec *txe = txq->sw_ring_v; +struct ci_tx_entry_vec *txe = txq->sw_ring_vec;uint16_t i;/* Zero out HW ring memory */
@@ -198,14 +198,14 @@ ixgbe_rxq_vec_setup_default(struct ixgbe_rx_queue *rxq) }static inline int -ixgbe_txq_vec_setup_default (struct ixgbe_tx_queue *txq, +ixgbe_txq_vec_setup_default (struct ci_tx_queue *txq,const struct ixgbe_txq_ops *txq_ops) { -if (txq->sw_ring_v ==NULL ) +if (txq->sw_ring_vec ==NULL )return -1 ;/* leave the first one for overflow */ - txq->sw_ring_v = txq->sw_ring_v +1 ; + txq->sw_ring_vec = txq->sw_ring_vec +1 ; txq->ops = txq_ops;return 0 ;
drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c +7 -7
@@ -571,7 +571,7 @@ uint16_t ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { -struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile union ixgbe_adv_tx_desc *txdp;struct ci_tx_entry_vec *txep;uint16_t n, nb_commit, tx_id;
@@ -591,7 +591,7 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = txq->tx_tail; txdp = &txq->ixgbe_tx_ring[tx_id]; - txep = &txq->sw_ring_v[tx_id]; + txep = &txq->sw_ring_vec[tx_id]; txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts);
@@ -611,7 +611,7 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,/* avoid reach the end of ring */ txdp = &txq->ixgbe_tx_ring[tx_id]; - txep = &txq->sw_ring_v[tx_id]; + txep = &txq->sw_ring_vec[tx_id]; }tx_backlog_entry (txep, tx_pkts, nb_commit);
@@ -634,7 +634,7 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, } static void __rte_cold -ixgbe_tx_queue_release_mbufs_vec (struct ixgbe_tx_queue *txq) +ixgbe_tx_queue_release_mbufs_vec (struct ci_tx_queue *txq) {_ixgbe_tx_queue_release_mbufs_vec (txq); }
@@ -646,13 +646,13 @@ ixgbe_rx_queue_release_mbufs_vec(struct ixgbe_rx_queue *rxq) } static void __rte_cold -ixgbe_tx_free_swring (struct ixgbe_tx_queue *txq) +ixgbe_tx_free_swring (struct ci_tx_queue *txq) {_ixgbe_tx_free_swring_vec (txq); } static void __rte_cold -ixgbe_reset_tx_queue (struct ixgbe_tx_queue *txq) +ixgbe_reset_tx_queue (struct ci_tx_queue *txq) {_ixgbe_reset_tx_queue_vec (txq); }
@@ -670,7 +670,7 @@ ixgbe_rxq_vec_setup(struct ixgbe_rx_queue *rxq) } int __rte_cold -ixgbe_txq_vec_setup (struct ixgbe_tx_queue *txq) +ixgbe_txq_vec_setup (struct ci_tx_queue *txq) {return ixgbe_txq_vec_setup_default (txq, &vec_txq_ops); }
drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c +7 -7
@@ -693,7 +693,7 @@ uint16_t ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { -struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue; +struct ci_tx_queue *txq = (struct ci_tx_queue *)tx_queue;volatile union ixgbe_adv_tx_desc *txdp;struct ci_tx_entry_vec *txep;uint16_t n, nb_commit, tx_id;
@@ -713,7 +713,7 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = txq->tx_tail; txdp = &txq->ixgbe_tx_ring[tx_id]; - txep = &txq->sw_ring_v[tx_id]; + txep = &txq->sw_ring_vec[tx_id]; txq->nb_tx_free = (uint16_t )(txq->nb_tx_free - nb_pkts);
@@ -734,7 +734,7 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,/* avoid reach the end of ring */ txdp = &txq->ixgbe_tx_ring[tx_id]; - txep = &txq->sw_ring_v[tx_id]; + txep = &txq->sw_ring_vec[tx_id]; }tx_backlog_entry (txep, tx_pkts, nb_commit);
@@ -757,7 +757,7 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, } static void __rte_cold -ixgbe_tx_queue_release_mbufs_vec (struct ixgbe_tx_queue *txq) +ixgbe_tx_queue_release_mbufs_vec (struct ci_tx_queue *txq) {_ixgbe_tx_queue_release_mbufs_vec (txq); }
@@ -769,13 +769,13 @@ ixgbe_rx_queue_release_mbufs_vec(struct ixgbe_rx_queue *rxq) } static void __rte_cold -ixgbe_tx_free_swring (struct ixgbe_tx_queue *txq) +ixgbe_tx_free_swring (struct ci_tx_queue *txq) {_ixgbe_tx_free_swring_vec (txq); } static void __rte_cold -ixgbe_reset_tx_queue (struct ixgbe_tx_queue *txq) +ixgbe_reset_tx_queue (struct ci_tx_queue *txq) {_ixgbe_reset_tx_queue_vec (txq); }
@@ -793,7 +793,7 @@ ixgbe_rxq_vec_setup(struct ixgbe_rx_queue *rxq) } int __rte_cold -ixgbe_txq_vec_setup (struct ixgbe_tx_queue *txq) +ixgbe_txq_vec_setup (struct ci_tx_queue *txq) {return ixgbe_txq_vec_setup_default (txq, &vec_txq_ops); }
Signature · 1 line
2.43.0
PATCH v3 10/22 net/_common_intel: pack Tx queue structure
In reply to Bruce RichardsonMove some fields about to better pack the Tx queue structure and make sure all data used by the vector codepaths is on the first cacheline of the structure. Checking with "pahole" on 64-bit build, only one 6-byte hole is left in the structure - on second cacheline - after this patch.
As part of the reordering, move the p/h/wthresh values to the ixgbe-specific part of the union. That is the only driver which actually uses those values. i40e and ice drivers just record the values for later return, so we can drop them from the Tx queue structure for those drivers and just report the defaults in all cases.
--- drivers/net/_common_intel/tx.h | 12 +++++------- drivers/net/i40e/i40e_rxtx.c | 9 +++------ drivers/net/ice/ice_rxtx.c | 9 +++------ 3 files changed, 11 insertions(+), 19 deletions(-)
drivers/net/_common_intel/tx.h +5 -7
@@ -41,7 +41,6 @@ struct ci_tx_queue {struct ci_tx_entry *sw_ring;/* virtual address of SW ring */ struct ci_tx_entry_vec *sw_ring_vec; }; -rte_iova_t tx_ring_dma;/* TX ring DMA address */ uint16_t nb_tx_desc;/* number of TX descriptors */ uint16_t tx_tail;/* current value of tail register */ uint16_t nb_tx_used;/* number of TX desc used since RS bit set */
@@ -55,16 +54,14 @@ struct ci_tx_queue {uint16_t tx_free_thresh;/* Number of TX descriptors to use before RS bit is set. */ uint16_t tx_rs_thresh; -uint8_t pthresh;/**< Prefetch threshold register. */ -uint8_t hthresh;/**< Host threshold register. */ -uint8_t wthresh;/**< Write-back threshold reg. */ uint16_t port_id;/* Device port identifier. */ uint16_t queue_id;/* TX queue index. */ uint16_t reg_idx; -uint64_t offloads;uint16_t tx_next_dd;uint16_t tx_next_rs; +uint64_t offloads;uint64_t mbuf_errors; +rte_iova_t tx_ring_dma;/* TX ring DMA address */ bool tx_deferred_start;/* don't start this queue in dev start */ bool q_set;/* indicate if tx queue has been configured */ union {/* the VSI this queue belongs to */
@@ -95,9 +92,10 @@ struct ci_tx_queue {const struct ixgbe_txq_ops *ops;struct ixgbe_advctx_info *ctx_cache;uint32_t ctx_curr; -#ifdef RTE_LIB_SECURITY +uint8_t pthresh;/**< Prefetch threshold register. */ +uint8_t hthresh;/**< Host threshold register. */ +uint8_t wthresh;/**< Write-back threshold reg. */ uint8_t using_ipsec;/**< indicates that IPsec TX feature is in use */ -#endif }; }; };
drivers/net/i40e/i40e_rxtx.c +3 -6
@@ -2539,9 +2539,6 @@ i40e_dev_tx_queue_setup(struct rte_eth_dev *dev, txq->nb_tx_desc = nb_desc; txq->tx_rs_thresh = tx_rs_thresh; txq->tx_free_thresh = tx_free_thresh; - txq->pthresh = tx_conf->tx_thresh.pthresh; - txq->hthresh = tx_conf->tx_thresh.hthresh; - txq->wthresh = tx_conf->tx_thresh.wthresh; txq->queue_id = queue_idx; txq->reg_idx = reg_idx; txq->port_id = dev->data->port_id;
@@ -3310,9 +3307,9 @@ i40e_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, qinfo->nb_desc = txq->nb_tx_desc; - qinfo->conf.tx_thresh.pthresh = txq->pthresh; - qinfo->conf.tx_thresh.hthresh = txq->hthresh; - qinfo->conf.tx_thresh.wthresh = txq->wthresh; + qinfo->conf.tx_thresh.pthresh =I40E_DEFAULT_TX_PTHRESH ; + qinfo->conf.tx_thresh.hthresh =I40E_DEFAULT_TX_HTHRESH ; + qinfo->conf.tx_thresh.wthresh =I40E_DEFAULT_TX_WTHRESH ; qinfo->conf.tx_free_thresh = txq->tx_free_thresh; qinfo->conf.tx_rs_thresh = txq->tx_rs_thresh;
drivers/net/ice/ice_rxtx.c +3 -6
@@ -1492,9 +1492,6 @@ ice_tx_queue_setup(struct rte_eth_dev *dev, txq->nb_tx_desc = nb_desc; txq->tx_rs_thresh = tx_rs_thresh; txq->tx_free_thresh = tx_free_thresh; - txq->pthresh = tx_conf->tx_thresh.pthresh; - txq->hthresh = tx_conf->tx_thresh.hthresh; - txq->wthresh = tx_conf->tx_thresh.wthresh; txq->queue_id = queue_idx; txq->reg_idx = vsi->base_queue + queue_idx;
@@ -1583,9 +1580,9 @@ ice_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, qinfo->nb_desc = txq->nb_tx_desc; - qinfo->conf.tx_thresh.pthresh = txq->pthresh; - qinfo->conf.tx_thresh.hthresh = txq->hthresh; - qinfo->conf.tx_thresh.wthresh = txq->wthresh; + qinfo->conf.tx_thresh.pthresh =ICE_DEFAULT_TX_PTHRESH ; + qinfo->conf.tx_thresh.hthresh =ICE_DEFAULT_TX_HTHRESH ; + qinfo->conf.tx_thresh.wthresh =ICE_DEFAULT_TX_WTHRESH ; qinfo->conf.tx_free_thresh = txq->tx_free_thresh; qinfo->conf.tx_rs_thresh = txq->tx_rs_thresh;
Signature · 1 line
2.43.0
PATCH v3 11/22 net/_common_intel: add post-Tx buffer free function
In reply to Bruce RichardsonThe actions taken for post-Tx buffer free for the SSE and AVX drivers for i40e, iavf and ice drivers are all common, so centralize those in common/intel_eth driver.
--- drivers/net/_common_intel/tx.h | 71 ++++++++++++++++++++++++ drivers/net/i40e/i40e_rxtx_vec_common.h | 72 ++++--------------------- drivers/net/iavf/iavf_rxtx_vec_common.h | 61 ++++----------------- drivers/net/ice/ice_rxtx_vec_common.h | 61 ++++----------------- 4 files changed, 98 insertions(+), 167 deletions(-)
drivers/net/_common_intel/tx.h +71 -0
@@ -7,6 +7,7 @@#include <stdint.h> #include <rte_mbuf.h> +#include <rte_ethdev.h> /* forward declaration of the common intel (ci) queue structure */ struct ci_tx_queue ;
@@ -107,4 +108,74 @@ ci_tx_backlog_entry(struct ci_tx_entry *txep, struct rte_mbuf **tx_pkts, uint16_ txep[i].mbuf = tx_pkts[i]; } +#define IETH_VPMD_TX_MAX_FREE_BUF 64 + +typedef int (*ci_desc_done_fn )(struct ci_tx_queue *txq,uint16_t idx); + +static __rte_always_inline int +ci_tx_free_bufs (struct ci_tx_queue *txq,ci_desc_done_fn desc_done) +{ +struct ci_tx_entry *txep; +uint32_t n; +uint32_t i; +int nb_free =0 ; +struct rte_mbuf *m, *free[IETH_VPMD_TX_MAX_FREE_BUF ]; + +/* check DD bits on threshold descriptor */ +if (!desc_done (txq, txq->tx_next_dd)) +return 0 ; + + n = txq->tx_rs_thresh; + +/* first buffer to free from S/W ring is at index +* tx_next_dd - (tx_rs_thresh-1) +*/ + txep = &txq->sw_ring[txq->tx_next_dd - (n -1 )]; + +if (txq->offloads &RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE ) { +for (i =0 ; i < n; i++) { + free[i] = txep[i].mbuf; +/* no need to reset txep[i].mbuf in vector path */ + } +rte_mempool_put_bulk (free[0 ]->pool, (void **)free, n); + goto done; + } + + m =rte_pktmbuf_prefree_seg (txep[0 ].mbuf); +if (likely (m !=NULL )) { + free[0 ] = m; + nb_free =1 ; +for (i =1 ; i < n; i++) { + m =rte_pktmbuf_prefree_seg (txep[i].mbuf); +if (likely (m !=NULL )) { +if (likely (m->pool == free[0 ]->pool)) { + free[nb_free++] = m; + }else { +rte_mempool_put_bulk (free[0 ]->pool, + (void *)free, + nb_free); + free[0 ] = m; + nb_free =1 ; + } + } + } +rte_mempool_put_bulk (free[0 ]->pool, (void **)free, nb_free); + }else { +for (i =1 ; i < n; i++) { + m =rte_pktmbuf_prefree_seg (txep[i].mbuf); +if (m !=NULL ) +rte_mempool_put (m->pool, m); + } + } + +done: +/* buffers were freed, update counters */ + txq->nb_tx_free = (uint16_t )(txq->nb_tx_free + txq->tx_rs_thresh); + txq->tx_next_dd = (uint16_t )(txq->tx_next_dd + txq->tx_rs_thresh); +if (txq->tx_next_dd >= txq->nb_tx_desc) + txq->tx_next_dd = (uint16_t )(txq->tx_rs_thresh -1 ); + +return txq->tx_rs_thresh; +} +#endif /* _COMMON_INTEL_TX_H_ */
drivers/net/i40e/i40e_rxtx_vec_common.h +9 -63
81 lines not on this page · raw message
drivers/net/iavf/iavf_rxtx_vec_common.h +9 -52
70 lines not on this page · raw message
drivers/net/ice/ice_rxtx_vec_common.h +9 -52
70 lines not on this page · raw message
Signature · 1 line
2.43.0
PATCH v3 12/22 net/_common_intel: add Tx buffer free fn for AVX-512
In reply to Bruce RichardsonAVX-512 code paths for ice and i40e drivers are common, and differ from the regular post-Tx free function in that the SW ring from which the buffers are freed does not contain anything other than the mbuf pointer. Merge these into a common function in intel_common to reduce duplication.
--- drivers/net/_common_intel/tx.h | 92 +++++++++++++++++++ drivers/net/i40e/i40e_rxtx_vec_avx512.c | 114 +---------------------- drivers/net/ice/ice_rxtx_vec_avx512.c | 117 +----------------------- 3 files changed, 94 insertions(+), 229 deletions(-)
drivers/net/_common_intel/tx.h +92 -0
@@ -178,4 +178,96 @@ ci_tx_free_bufs(struct ci_tx_queue *txq, ci_desc_done_fn desc_done)return txq->tx_rs_thresh; } +static __rte_always_inline int +ci_tx_free_bufs_vec (struct ci_tx_queue *txq,ci_desc_done_fn desc_done) +{ +int nb_free =0 ; +struct rte_mbuf *free[IETH_VPMD_TX_MAX_FREE_BUF ]; +struct rte_mbuf *m; + +/* check DD bits on threshold descriptor */ +if (!desc_done (txq, txq->tx_next_dd)) +return 0 ; + +const uint32_t n = txq->tx_rs_thresh; + +/* first buffer to free from S/W ring is at index +* tx_next_dd - (tx_rs_thresh - 1) +*/ +struct ci_tx_entry_vec *txep = txq->sw_ring_vec; + txep += txq->tx_next_dd - (n -1 ); + +if (txq->offloads &RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE && (n &31 ) ==0 ) { +struct rte_mempool *mp = txep[0 ].mbuf->pool; +void **cache_objs; +struct rte_mempool_cache *cache =rte_mempool_default_cache (mp,rte_lcore_id ()); + +if (!cache || cache->len ==0 ) + goto normal; + + cache_objs = &cache->objs[cache->len]; + +if (n >RTE_MEMPOOL_CACHE_MAX_SIZE ) { +rte_mempool_ops_enqueue_bulk (mp, (void *)txep, n); + goto done; + } + +/* The cache follows the following algorithm +* 1. Add the objects to the cache +* 2. Anything greater than the cache min value (if it +* crosses the cache flush threshold) is flushed to the ring. +*/ +/* Add elements back into the cache */ +uint32_t copied =0 ; +/* n is multiple of 32 */ +while (copied < n) { +memcpy (&cache_objs[copied], &txep[copied],32 *sizeof (void *)); + copied +=32 ; + } + cache->len += n; + +if (cache->len >= cache->flushthresh) { +rte_mempool_ops_enqueue_bulk (mp, &cache->objs[cache->size], + cache->len - cache->size); + cache->len = cache->size; + } + goto done; + } + +normal: + m =rte_pktmbuf_prefree_seg (txep[0 ].mbuf); +if (likely (m)) { + free[0 ] = m; + nb_free =1 ; +for (uint32_t i =1 ; i < n; i++) { + m =rte_pktmbuf_prefree_seg (txep[i].mbuf); +if (likely (m)) { +if (likely (m->pool == free[0 ]->pool)) { + free[nb_free++] = m; + }else { +rte_mempool_put_bulk (free[0 ]->pool, (void *)free, nb_free); + free[0 ] = m; + nb_free =1 ; + } + } + } +rte_mempool_put_bulk (free[0 ]->pool, (void **)free, nb_free); + }else { +for (uint32_t i =1 ; i < n; i++) { + m =rte_pktmbuf_prefree_seg (txep[i].mbuf); +if (m) +rte_mempool_put (m->pool, m); + } + } + +done: +/* buffers were freed, update counters */ + txq->nb_tx_free = (uint16_t )(txq->nb_tx_free + txq->tx_rs_thresh); + txq->tx_next_dd = (uint16_t )(txq->tx_next_dd + txq->tx_rs_thresh); +if (txq->tx_next_dd >= txq->nb_tx_desc) + txq->tx_next_dd = (uint16_t )(txq->tx_rs_thresh -1 ); + +return txq->tx_rs_thresh; +} +#endif /* _COMMON_INTEL_TX_H_ */
drivers/net/i40e/i40e_rxtx_vec_avx512.c +1 -113
126 lines not on this page · raw message
drivers/net/ice/ice_rxtx_vec_avx512.c +1 -116
129 lines not on this page · raw message
Signature · 1 line
2.43.0
PATCH v3 13/22 net/iavf: use common Tx free fn for AVX-512
In reply to Bruce RichardsonSwitch the iavf driver to use the common Tx free function. This requires one additional parameter to that function, since iavf sometimes uses context descriptors which means that we have double the descriptors per SW ring slot.
--- drivers/net/_common_intel/tx.h | 6 +- drivers/net/i40e/i40e_rxtx_vec_avx512.c | 2 +- drivers/net/iavf/iavf_rxtx_vec_avx512.c | 119 +----------------------- drivers/net/ice/ice_rxtx_vec_avx512.c | 2 +- 4 files changed, 7 insertions(+), 122 deletions(-)
drivers/net/_common_intel/tx.h +3 -3
@@ -179,7 +179,7 @@ ci_tx_free_bufs(struct ci_tx_queue *txq, ci_desc_done_fn desc_done) } static __rte_always_inline int -ci_tx_free_bufs_vec (struct ci_tx_queue *txq,ci_desc_done_fn desc_done) +ci_tx_free_bufs_vec (struct ci_tx_queue *txq,ci_desc_done_fn desc_done,bool ctx_descs) {int nb_free =0 ;struct rte_mbuf *free[IETH_VPMD_TX_MAX_FREE_BUF ];
@@ -189,13 +189,13 @@ ci_tx_free_bufs_vec(struct ci_tx_queue *txq, ci_desc_done_fn desc_done)if (!desc_done (txq, txq->tx_next_dd))return 0 ; -const uint32_t n = txq->tx_rs_thresh; +const uint32_t n = txq->tx_rs_thresh >> ctx_descs;/* first buffer to free from S/W ring is at index * tx_next_dd - (tx_rs_thresh - 1) */ struct ci_tx_entry_vec *txep = txq->sw_ring_vec; - txep += txq->tx_next_dd - (n -1 ); + txep += (txq->tx_next_dd >> ctx_descs) - (n -1 );if (txq->offloads &RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE && (n &31 ) ==0 ) {struct rte_mempool *mp = txep[0 ].mbuf->pool;
drivers/net/i40e/i40e_rxtx_vec_avx512.c +1 -1
@@ -829,7 +829,7 @@ i40e_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts,uint64_t rs =I40E_TX_DESC_CMD_RS |I40E_TD_CMD ;if (txq->nb_tx_free < txq->tx_free_thresh) -ci_tx_free_bufs_vec (txq, i40e_tx_desc_done); +ci_tx_free_bufs_vec (txq, i40e_tx_desc_done, false); nb_commit = nb_pkts = (uint16_t )RTE_MIN (txq->nb_tx_free, nb_pkts);if (unlikely (nb_pkts ==0 ))
drivers/net/iavf/iavf_rxtx_vec_avx512.c +2 -117
137 lines not on this page · raw message
drivers/net/ice/ice_rxtx_vec_avx512.c +1 -1
@@ -949,7 +949,7 @@ ice_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, nb_pkts =RTE_MIN (nb_pkts, txq->tx_rs_thresh);if (txq->nb_tx_free < txq->tx_free_thresh) -ci_tx_free_bufs_vec (txq, ice_tx_desc_done); +ci_tx_free_bufs_vec (txq, ice_tx_desc_done, false); nb_commit = nb_pkts = (uint16_t )RTE_MIN (txq->nb_tx_free, nb_pkts);if (unlikely (nb_pkts ==0 ))
Signature · 1 line
2.43.0