framework for advanced iAVF PMD
cover framework for advanced iAVF PMD
This patchset enable framework for advanced iAVF, includes query DDP pkg info, flexible descriptor support, FDIR mark id and RSS hash support.
Leyi Rong (12):
net/iavf: remove 16B Rx descriptor compile option net/iavf: return error if opcode is mismatched net/iavf: support to query DDP package info net/iavf: flexible Rx descriptor support in normal path net/iavf: flexible Rx descriptor support in AVX path net/iavf: flexible Rx descriptor support in SSE path net/iavf: add flow director enabled switch value net/iavf: support flow mark in normal data path net/iavf: support flow mark in AVX path net/iavf: support flow mark in SSE path net/iavf: add RSS hash parsing in AVX path net/iavf: add RSS hash parsing in SSE path
config/common_base | 1 - drivers/net/iavf/iavf.h | 15 + drivers/net/iavf/iavf_ethdev.c | 15 + drivers/net/iavf/iavf_rxtx.c | 570 ++++++++++++++++- drivers/net/iavf/iavf_rxtx.h | 62 +- drivers/net/iavf/iavf_rxtx_vec_avx2.c | 856 ++++++++++++++++++++++---- drivers/net/iavf/iavf_rxtx_vec_sse.c | 514 ++++++++++++++++ drivers/net/iavf/iavf_vchnl.c | 86 ++- 8 files changed, 1982 insertions(+), 137 deletions(-)
Signature · 1 line
2.17.1
PATCH 1/12 net/iavf: remove 16B Rx descriptor compile option
Remove CONFIG_RTE_LIBRTE_IAVF_16BYTE_RX_DESC as
it's not supported in ice PF host driver.
--- config/common_base | 1 - drivers/net/iavf/iavf_rxtx.c | 2 - drivers/net/iavf/iavf_rxtx.h | 14 +-- drivers/net/iavf/iavf_rxtx_vec_avx2.c | 148 +++++++------------------- 4 files changed, 42 insertions(+), 123 deletions(-)
config/common_base +0 -1
@@ -346,7 +346,6 @@ CONFIG_RTE_LIBRTE_IAVF_DEBUG_TX=n CONFIG_RTE_LIBRTE_IAVF_DEBUG_TX_FREE=n CONFIG_RTE_LIBRTE_IAVF_DEBUG_RX=n CONFIG_RTE_LIBRTE_IAVF_DEBUG_DUMP_DESC=n -CONFIG_RTE_LIBRTE_IAVF_16BYTE_RX_DESC=n # # Compile burst-oriented IPN3KE PMD driver #
drivers/net/iavf/iavf_rxtx.c +0 -2
@@ -230,10 +230,8 @@ alloc_rxq_mbufs(struct iavf_rx_queue *rxq) rxd = &rxq->rx_ring[i]; rxd->read.pkt_addr = dma_addr; rxd->read.hdr_addr =0 ; -#ifndef RTE_LIBRTE_IAVF_16BYTE_RX_DESC rxd->read.rsvd1 =0 ; rxd->read.rsvd2 =0 ; -#endif rxq->sw_ring[i] = mbuf; }
drivers/net/iavf/iavf_rxtx.h +1 -13
@@ -57,12 +57,8 @@#define IAVF_TX_OFFLOAD_NOTSUP_MASK \ (PKT_TX_OFFLOAD_MASK ^ IAVF_TX_OFFLOAD_MASK) -/* HW desc structure, both 16-byte and 32-byte types are supported */ -#ifdef RTE_LIBRTE_IAVF_16BYTE_RX_DESC -#define iavf_rx_desc iavf_16byte_rx_desc -#else +/* HW desc structure, only 32-byte type is supported */ #define iavf_rx_desc iavf_32byte_rx_desc -#endif struct iavf_rxq_ops {void (*release_mbufs)(struct iavf_rx_queue *rxq);
@@ -224,20 +220,12 @@ void iavf_dump_rx_descriptor(struct iavf_rx_queue *rxq, const volatile void *desc, uint16_t rx_id) { -#ifdef RTE_LIBRTE_IAVF_16BYTE_RX_DESC - const volatile union iavf_16byte_rx_desc *rx_desc = desc; - -printf ("Queue %d Rx_desc %d: QW0: 0x%016" PRIx64" QW1: 0x%016" PRIx64"\n" , - rxq->queue_id, rx_id, rx_desc->read.pkt_addr, - rx_desc->read.hdr_addr); -#else const volatile union iavf_32byte_rx_desc *rx_desc = desc;printf ("Queue %d Rx_desc %d: QW0: 0x%016" PRIx64" QW1: 0x%016" PRIx64" QW2: 0x%016" PRIx64" QW3: 0x%016" PRIx64"\n" , rxq->queue_id, rx_id, rx_desc->read.pkt_addr, rx_desc->read.hdr_addr, rx_desc->read.rsvd1, rx_desc->read.rsvd2); -#endif }/* All the descriptors are 16 bytes, so just use one of them
drivers/net/iavf/iavf_rxtx_vec_avx2.c +41 -107
178 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH 2/12 net/iavf: return error if opcode is mismatched
In reply to Leyi RongAdds error return when the opcode of read message is mismatched which is received from adminQ.
--- drivers/net/iavf/iavf_vchnl.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
drivers/net/iavf/iavf_vchnl.c +3 -1
@@ -52,9 +52,11 @@ iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len,PMD_DRV_LOG (DEBUG ,"AQ from pf carries opcode %u, retval %d" , opcode, vf->cmd_retval); -if (opcode != vf->pend_cmd) +if (opcode != vf->pend_cmd) {PMD_DRV_LOG (WARNING ,"command mismatch, expect %u, get %u" , vf->pend_cmd, opcode); +return IAVF_ERR_OPCODE_MISMATCH ; + }return IAVF_SUCCESS ; }
Signature · 1 line
2.17.1
PATCH 3/12 net/iavf: support to query DDP package info
In reply to Leyi RongAdvanced iAVF supports to query DDP package info, includes package version, track id, package name, device serial number and the list of protocols that the PF supports.
--- drivers/net/iavf/iavf.h | 9 ++++++++ drivers/net/iavf/iavf_ethdev.c | 7 ++++++ drivers/net/iavf/iavf_vchnl.c | 41 +++++++++++++++++++++++++++++++++- 3 files changed, 56 insertions(+), 1 deletion(-)
drivers/net/iavf/iavf.h +9 -0
@@ -67,6 +67,13 @@#define IAVF_48_BIT_WIDTH (CHAR_BIT * 6)#define IAVF_48_BIT_MASK RTE_LEN2MASK(IAVF_48_BIT_WIDTH, uint64_t) +/* VF supported comms protocols 64-bits bitmap */ +#define IAVF_COMMS_PROTO_GTP 0x0000000000000001 +#define IAVF_COMMS_PROTO_PPPOE 0x0000000000000002 +#define IAVF_COMMS_PROTO_PFCP 0x0000000000000004 +#define IAVF_COMMS_PROTO_L2TPV3 0x0000000000000008 +#define IAVF_COMMS_PROTO_ESP 0x0000000000000010 +struct iavf_adapter ;struct iavf_rx_queue ;struct iavf_tx_queue ;
@@ -97,6 +104,7 @@ struct iavf_info {struct virtchnl_version_info virtchnl_version;struct virtchnl_vf_resource *vf_res;/* VF resource */ struct virtchnl_vsi_resource *vsi_res;/* LAN VSI */ +struct virtchnl_pkg_info pkg_info;/* package info */ volatile enum virtchnl_ops pend_cmd;/* pending command not finished */ uint32_t cmd_retval;/* return value of the cmd response from PF */
@@ -225,6 +233,7 @@ int iavf_disable_queues(struct iavf_adapter *adapter);int iavf_configure_rss_lut (struct iavf_adapter *adapter);int iavf_configure_rss_key (struct iavf_adapter *adapter);int iavf_configure_queues (struct iavf_adapter *adapter); +int iavf_query_package_info (struct iavf_adapter *adapter);int iavf_config_irq_map (struct iavf_adapter *adapter);void iavf_add_del_all_mac_addr (struct iavf_adapter *adapter,bool add); int iavf_dev_link_update(struct rte_eth_dev *dev,
drivers/net/iavf/iavf_ethdev.c +7 -0
@@ -1236,6 +1236,13 @@ iavf_init_vf(struct rte_eth_dev *dev) goto err_rss; } } + +if (vf->vf_res->vf_cap_flags &VIRTCHNL_VF_OFFLOAD_QUERY_DDP ) { +if (iavf_query_package_info (adapter) !=0 ) { +PMD_INIT_LOG (ERR ,"iavf_query_package_info failed" ); + goto err_rss; + } + }return 0 ; err_rss:rte_free (vf->rss_key);
drivers/net/iavf/iavf_vchnl.c +40 -1
59 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH 4/12 net/iavf: flexible Rx descriptor support in normal path
In reply to Leyi RongSupport flexible Rx descriptor format in normal
path of iAVF PMD.
--- drivers/net/iavf/iavf.h | 2 + drivers/net/iavf/iavf_ethdev.c | 8 + drivers/net/iavf/iavf_rxtx.c | 507 ++++++++++++++++++++++++++++++++- drivers/net/iavf/iavf_rxtx.h | 11 + drivers/net/iavf/iavf_vchnl.c | 43 ++- 5 files changed, 567 insertions(+), 4 deletions(-)
drivers/net/iavf/iavf.h +2 -0
@@ -105,6 +105,7 @@ struct iavf_info {struct virtchnl_vf_resource *vf_res;/* VF resource */ struct virtchnl_vsi_resource *vsi_res;/* LAN VSI */ struct virtchnl_pkg_info pkg_info;/* package info */ +uint64_t supported_rxdid;volatile enum virtchnl_ops pend_cmd;/* pending command not finished */ uint32_t cmd_retval;/* return value of the cmd response from PF */
@@ -233,6 +234,7 @@ int iavf_disable_queues(struct iavf_adapter *adapter);int iavf_configure_rss_lut (struct iavf_adapter *adapter);int iavf_configure_rss_key (struct iavf_adapter *adapter);int iavf_configure_queues (struct iavf_adapter *adapter); +int iavf_get_supported_rxdid (struct iavf_adapter *adapter);int iavf_query_package_info (struct iavf_adapter *adapter);int iavf_config_irq_map (struct iavf_adapter *adapter);void iavf_add_del_all_mac_addr (struct iavf_adapter *adapter,bool add);
drivers/net/iavf/iavf_ethdev.c +8 -0
@@ -1243,6 +1243,14 @@ iavf_init_vf(struct rte_eth_dev *dev) goto err_rss; } } + +if (vf->vf_res->vf_cap_flags &VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC ) { +if (iavf_get_supported_rxdid (adapter) !=0 ) { +PMD_INIT_LOG (ERR ,"failed to do get supported rxdid" ); + goto err_rss; + } + } +return 0 ; err_rss:rte_free (vf->rss_key);
drivers/net/iavf/iavf_rxtx.c +504 -3
579 lines not on this page · raw message
drivers/net/iavf/iavf_rxtx.h +11 -0
@@ -59,6 +59,7 @@/* HW desc structure, only 32-byte type is supported */ #define iavf_rx_desc iavf_32byte_rx_desc +#define iavf_rx_flex_desc iavf_32b_rx_flex_descstruct iavf_rxq_ops {void (*release_mbufs)(struct iavf_rx_queue *rxq);
@@ -83,6 +84,7 @@ struct iavf_rx_queue {struct rte_mbuf *pkt_first_seg;/* first segment of current packet */ struct rte_mbuf *pkt_last_seg;/* last segment of current packet */ struct rte_mbuf fake_mbuf;/* dummy mbuf */ +uint8_t rxdid;/* used for VPMD */ uint16_t rxrearm_nb;/* number of remaining to be re-armed */
@@ -175,9 +177,15 @@ void iavf_dev_tx_queue_release(void *txq);void iavf_stop_queues (struct rte_eth_dev *dev);uint16_t iavf_recv_pkts (void *rx_queue,struct rte_mbuf **rx_pkts,uint16_t nb_pkts); +uint16_t iavf_recv_pkts_flex_rxd (void *rx_queue, +struct rte_mbuf **rx_pkts, +uint16_t nb_pkts);uint16_t iavf_recv_scattered_pkts (void *rx_queue,struct rte_mbuf **rx_pkts,uint16_t nb_pkts); +uint16_t iavf_recv_scattered_pkts_flex_rxd (void *rx_queue, +struct rte_mbuf **rx_pkts, +uint16_t nb_pkts);uint16_t iavf_xmit_pkts (void *tx_queue,struct rte_mbuf **tx_pkts,uint16_t nb_pkts); uint16_t iavf_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
@@ -189,7 +197,10 @@ void iavf_dev_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,void iavf_dev_txq_info_get (struct rte_eth_dev *dev,uint16_t queue_id,struct rte_eth_txq_info *qinfo);uint32_t iavf_dev_rxq_count (struct rte_eth_dev *dev,uint16_t queue_id); +uint32_t iavf_dev_rxq_count_flex_rxd (struct rte_eth_dev *dev, +uint16_t queue_id);int iavf_dev_rx_desc_status (void *rx_queue,uint16_t offset); +int iavf_dev_rx_desc_status_flex_rxd (void *rx_queue,uint16_t offset);int iavf_dev_tx_desc_status (void *tx_queue,uint16_t offset); uint16_t iavf_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
drivers/net/iavf/iavf_vchnl.c +42 -1
67 lines not on this page · raw message
Signature · 1 line
2.17.1
Re: [dpdk-dev] [PATCH 04/12] net/iavf: flexible Rx descriptor support in normal path
One general comment:
Looks like there are exact same code comparing with legacy rx, such as logic to update tail, multi-segments loop. It will be good if all the common code can be wrapped and use for multi recv functions
[...]
+/* Get the number of used descriptors of a rx queue for flexible RXD */ +uint32_t iavf_dev_rxq_count_flex_rxd(struct rte_eth_dev *dev, uint16_t +queue_id) { #define IAVF_RXQ_SCAN_INTERVAL 4
+ volatile union iavf_rx_flex_desc *rxdp;
+ struct iavf_rx_queue *rxq;
+ uint16_t desc = 0;
+
+ rxq = dev->data->rx_queues[queue_id];
+ rxdp = (volatile union iavf_rx_flex_desc *)&rxq->rx_ring[rxq->rx_tail];
+ while ((desc < rxq->nb_rx_desc) &&
+ rte_le_to_cpu_16(rxdp->wb.status_error0) &
+ (1 << IAVF_RX_FLEX_DESC_STATUS0_DD_S)) {
+ /* Check the DD bit of a rx descriptor of each 4 in a group,
+ * to avoid checking too frequently and downgrading performance
+ * too much.
+ */
+ desc += IAVF_RXQ_SCAN_INTERVAL;
+ rxdp += IAVF_RXQ_SCAN_INTERVAL;
+ if (rxq->rx_tail + desc >= rxq->nb_rx_desc)
+ rxdp = (volatile union iavf_rx_flex_desc *)
+ &(rxq->rx_ring[rxq->rx_tail +
+ desc - rxq->nb_rx_desc]);
+ }
+
+ return desc;
+}
No much difference between iavf_dev_rxq_count. Why do we need a new one? DD bit is located in the same place, right?
Can we merge those two functions to one?
+
/* Get the number of used descriptors of a rx queue */ uint32_t iavf_dev_rxq_count(struct rte_eth_dev *dev, uint16_t queue_id) @@ -1795,6 +2264,10 @@ iavf_dev_rxq_count(struct rte_eth_dev *dev, uint16_t queue_id)
rxq = dev->data->rx_queues[queue_id];
rxdp = &rxq->rx_ring[rxq->rx_tail];
+
+ if (rxq->rxdid == IAVF_RXDID_COMMS_OVS_1)
+ return iavf_dev_rxq_count_flex_rxd(dev, queue_id);
+
while ((desc < rxq->nb_rx_desc) &&
((rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len) &
IAVF_RXD_QW1_STATUS_MASK) >> IAVF_RXD_QW1_STATUS_SHIFT) & @@ -1813,6 +2286,31 @@ iavf_dev_rxq_count(struct rte_eth_dev *dev, uint16_t queue_id)
return desc;
}
+int
+iavf_dev_rx_desc_status_flex_rxd(void *rx_queue, uint16_t offset) {
+ volatile union iavf_rx_flex_desc *rxdp;
+ struct iavf_rx_queue *rxq = rx_queue;
+ uint32_t desc;
+
+ if (unlikely(offset >= rxq->nb_rx_desc))
+ return -EINVAL;
+
+ if (offset >= rxq->nb_rx_desc - rxq->nb_rx_hold)
+ return RTE_ETH_RX_DESC_UNAVAIL;
+
+ desc = rxq->rx_tail + offset;
+ if (desc >= rxq->nb_rx_desc)
+ desc -= rxq->nb_rx_desc;
+
+ rxdp = (volatile union iavf_rx_flex_desc *)&rxq->rx_ring[desc];
+ if (rte_le_to_cpu_16(rxdp->wb.status_error0) &
+ (1 << IAVF_RX_FLEX_DESC_STATUS0_DD_S))
+ return RTE_ETH_RX_DESC_DONE;
+
+ return RTE_ETH_RX_DESC_AVAIL;
+}
+
Similar comments as above.
[......]
{ @@ -569,6 +596,20 @@ iavf_configure_queues(struct iavf_adapter *adapter)
vc_qp->rxq.ring_len = rxq[i]->nb_rx_desc;
vc_qp->rxq.dma_ring_addr = rxq[i]->rx_ring_phys_addr;
vc_qp->rxq.databuffer_size = rxq[i]->rx_buf_len;
+
+ if (vf->vf_res->vf_cap_flags &
+ VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC &&
+ vf->supported_rxdid & BIT(IAVF_RXDID_COMMS_OVS_1)) {
+ vc_qp->rxq.rxdid = IAVF_RXDID_COMMS_OVS_1;
+ rxq[i]->rxdid = IAVF_RXDID_COMMS_OVS_1;
Because this function is used to construct virtchnl message for configure queues.
The rxdid in rxq[i] should be set before this function is called. How about to move this line assignment to iavf_dev_rx_queue_setup?
+ PMD_DRV_LOG(NOTICE, "request RXDID == %d in "
+ "Queue[%d]", vc_qp->rxq.rxdid, i);
+ } else {
+ vc_qp->rxq.rxdid = IAVF_RXDID_LEGACY_1;
+ rxq[i]->rxdid = IAVF_RXDID_LEGACY_1;
Same as above.
+ PMD_DRV_LOG(NOTICE, "request RXDID == %d in "
+ "Queue[%d]", vc_qp->rxq.rxdid, i);
+ }
}
}
--
2.17.1
PATCH 5/12 net/iavf: flexible Rx descriptor support in AVX path
In reply to Leyi RongSupport flexible Rx descriptor format in AVX
path of iAVF PMD.
--- drivers/net/iavf/iavf_rxtx.c | 24 +- drivers/net/iavf/iavf_rxtx.h | 6 + drivers/net/iavf/iavf_rxtx_vec_avx2.c | 550 +++++++++++++++++++++++++- 3 files changed, 570 insertions(+), 10 deletions(-)
drivers/net/iavf/iavf_rxtx.c +18 -6
@@ -2115,16 +2115,28 @@ iavf_set_rx_function(struct rte_eth_dev *dev)"Using %sVector Scattered Rx (port %d)." , use_avx2 ?"avx2 " :"" , dev->data->port_id); - dev->rx_pkt_burst = use_avx2 ? - iavf_recv_scattered_pkts_vec_avx2 : - iavf_recv_scattered_pkts_vec; +if (vf->vf_res->vf_cap_flags & +VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC ) + dev->rx_pkt_burst = use_avx2 ? + iavf_recv_scattered_pkts_vec_avx2_flex_rxd : + iavf_recv_scattered_pkts_vec; +else + dev->rx_pkt_burst = use_avx2 ? + iavf_recv_scattered_pkts_vec_avx2 : + iavf_recv_scattered_pkts_vec; }else {PMD_DRV_LOG (DEBUG ,"Using %sVector Rx (port %d)." , use_avx2 ?"avx2 " :"" , dev->data->port_id); - dev->rx_pkt_burst = use_avx2 ? - iavf_recv_pkts_vec_avx2 : - iavf_recv_pkts_vec; +if (vf->vf_res->vf_cap_flags & +VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC ) + dev->rx_pkt_burst = use_avx2 ? + iavf_recv_pkts_vec_avx2_flex_rxd : + iavf_recv_pkts_vec; +else + dev->rx_pkt_burst = use_avx2 ? + iavf_recv_pkts_vec_avx2 : + iavf_recv_pkts_vec; }return ;
drivers/net/iavf/iavf_rxtx.h +6 -0
@@ -212,9 +212,15 @@ uint16_t iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts);uint16_t iavf_recv_pkts_vec_avx2 (void *rx_queue,struct rte_mbuf **rx_pkts,uint16_t nb_pkts); +uint16_t iavf_recv_pkts_vec_avx2_flex_rxd (void *rx_queue, +struct rte_mbuf **rx_pkts, +uint16_t nb_pkts);uint16_t iavf_recv_scattered_pkts_vec_avx2 (void *rx_queue,struct rte_mbuf **rx_pkts,uint16_t nb_pkts); +uint16_t iavf_recv_scattered_pkts_vec_avx2_flex_rxd (void *rx_queue, +struct rte_mbuf **rx_pkts, +uint16_t nb_pkts);uint16_t iavf_xmit_pkts_vec (void *tx_queue,struct rte_mbuf **tx_pkts,uint16_t nb_pkts); uint16_t iavf_xmit_pkts_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts,
drivers/net/iavf/iavf_rxtx_vec_avx2.c +546 -4
585 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH 6/12 net/iavf: flexible Rx descriptor support in SSE path
In reply to Leyi RongSupport flexible Rx descriptor format in SSE
path of iAVF PMD.
--- drivers/net/iavf/iavf_rxtx.c | 4 +- drivers/net/iavf/iavf_rxtx.h | 5 + drivers/net/iavf/iavf_rxtx_vec_sse.c | 414 +++++++++++++++++++++++++++ 3 files changed, 421 insertions(+), 2 deletions(-)
drivers/net/iavf/iavf_rxtx.c +2 -2
@@ -2119,7 +2119,7 @@ iavf_set_rx_function(struct rte_eth_dev *dev) VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) dev->rx_pkt_burst = use_avx2 ? iavf_recv_scattered_pkts_vec_avx2_flex_rxd : - iavf_recv_scattered_pkts_vec; + iavf_recv_scattered_pkts_vec_flex_rxd; else dev->rx_pkt_burst = use_avx2 ? iavf_recv_scattered_pkts_vec_avx2 :
@@ -2132,7 +2132,7 @@ iavf_set_rx_function(struct rte_eth_dev *dev) VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) dev->rx_pkt_burst = use_avx2 ? iavf_recv_pkts_vec_avx2_flex_rxd : - iavf_recv_pkts_vec; + iavf_recv_pkts_vec_flex_rxd; else dev->rx_pkt_burst = use_avx2 ? iavf_recv_pkts_vec_avx2 :
drivers/net/iavf/iavf_rxtx.h +5 -0
@@ -205,9 +205,14 @@ int iavf_dev_tx_desc_status(void *tx_queue, uint16_t offset);uint16_t iavf_recv_pkts_vec (void *rx_queue,struct rte_mbuf **rx_pkts,uint16_t nb_pkts); +uint16_t iavf_recv_pkts_vec_flex_rxd (void *rx_queue,struct rte_mbuf **rx_pkts, +uint16_t nb_pkts);uint16_t iavf_recv_scattered_pkts_vec (void *rx_queue,struct rte_mbuf **rx_pkts,uint16_t nb_pkts); +uint16_t iavf_recv_scattered_pkts_vec_flex_rxd (void *rx_queue, +struct rte_mbuf **rx_pkts, +uint16_t nb_pkts);uint16_t iavf_xmit_fixed_burst_vec (void *tx_queue,struct rte_mbuf **tx_pkts,uint16_t nb_pkts); uint16_t iavf_recv_pkts_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,
drivers/net/iavf/iavf_rxtx_vec_sse.c +414 -0
444 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH 7/12 net/iavf: add flow director enabled switch value
In reply to Leyi RongThe commit adds fdir_enabled flag into iavf_adapter structure to identify if fdir id is active. Rx data path can be benefit if fdir id parsing is not needed, especially in vector path.
--- drivers/net/iavf/iavf.h | 1 + drivers/net/iavf/iavf_rxtx.h | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+)
drivers/net/iavf/iavf.h +1 -0
@@ -142,6 +142,7 @@ struct iavf_adapter {bool tx_vec_allowed;const uint32_t *ptype_tbl;bool stopped; +uint8_t fdir_enabled; };/* IAVF_DEV_PRIVATE_TO */
drivers/net/iavf/iavf_rxtx.h +26 -0
@@ -281,6 +281,32 @@ void iavf_dump_tx_descriptor(const struct iavf_tx_queue *txq, tx_desc->cmd_type_offset_bsz); } +/* Enable/disable flow director Rx processing in data path. */ +static inline +void iavf_fdir_rx_proc_enable (struct iavf_adapter *ad,bool on) +{ +static uint32_t ref_cnt; + +if (on) { +/* enable flow director processing */ +if (ref_cnt++ ==0 ) { + ad->fdir_enabled = on; +PMD_DRV_LOG (DEBUG , +"FDIR processing on RX set to %d" , on); + } + }else { +if (ref_cnt >=1 ) { + ref_cnt--; + +if (ref_cnt ==0 ) { + ad->fdir_enabled = on; +PMD_DRV_LOG (DEBUG , +"FDIR processing on RX set to %d" , on); + } + } + } +} +#ifdef RTE_LIBRTE_IAVF_DEBUG_DUMP_DESC #define IAVF_DUMP_RX_DESC (rxq, desc, rx_id) \ iavf_dump_rx_descriptor(rxq, desc, rx_id)
Signature · 1 line
2.17.1
Re: [dpdk-dev] [PATCH 07/12] net/iavf: add flow director enabled switch value
-----Original Message-----
From: dev [mailto:[email protected]] On Behalf Of Leyi Rong
Sent: Monday, March 16, 2020 3:46 PM
To: Zhang, Qi Z <[email protected]>; Ye, Xiaolong <[email protected]>
Cc: [email protected]; Rong, Leyi <[email protected]>
Subject: [dpdk-dev] [PATCH 07/12] net/iavf: add flow director enabled switch value
The commit adds fdir_enabled flag into iavf_adapter structure to identify if fdir id is active. Rx data path can be benefit if fdir id parsing is not needed, especially in vector path.
--- drivers/net/iavf/iavf.h | 1 + drivers/net/iavf/iavf_rxtx.h | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+)
drivers/net/iavf/iavf.h +1 -0
@@ -142,6 +142,7 @@ struct iavf_adapter {bool tx_vec_allowed;const uint32_t *ptype_tbl;bool stopped; +uint8_t fdir_enabled; };/* IAVF_DEV_PRIVATE_TO */
drivers/net/iavf/iavf_rxtx.h +25 -0
@@ -281,6 +281,32 @@ void iavf_dump_tx_descriptor(const struct iavf_tx_queue *txq, tx_desc->cmd_type_offset_bsz); } +/* Enable/disable flow director Rx processing in data path. */ static +inline void iavf_fdir_rx_proc_enable (struct iavf_adapter *ad,bool on) +{ +static uint32_t ref_cnt; + +if (on) { +/* enable flow director processing */ +if (ref_cnt++ ==0 ) { + ad->fdir_enabled = on; +PMD_DRV_LOG (DEBUG , +"FDIR processing on RX set to %d" , on); + } + }else { +if (ref_cnt >=1 ) { + ref_cnt--; + +if (ref_cnt ==0 ) { + ad->fdir_enabled = on; +PMD_DRV_LOG (DEBUG , +"FDIR processing on RX set to %d" , on); + } + } + } +} +
fdir_enabled is used for fast path. To avoid competition, how about to make it in queue granularity?
#ifdef RTE_LIBRTE_IAVF_DEBUG_DUMP_DESC
#define IAVF_DUMP_RX_DESC(rxq, desc, rx_id) \
iavf_dump_rx_descriptor(rxq, desc, rx_id)
--
2.17.1
PATCH 8/12 net/iavf: support flow mark in normal data path
In reply to Leyi RongSupport Flow Director mark ID parsing in normal path.
--- drivers/net/iavf/iavf.h | 3 +++ drivers/net/iavf/iavf_rxtx.c | 37 ++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+)
drivers/net/iavf/iavf.h +3 -0
@@ -74,6 +74,9 @@#define IAVF_COMMS_PROTO_L2TPV3 0x0000000000000008#define IAVF_COMMS_PROTO_ESP 0x0000000000000010 +#define IAVF_RX_DESC_EXT_STATUS_FLEXBH_MASK 0x03 +#define IAVF_RX_DESC_EXT_STATUS_FLEXBH_FD_ID 0x01 +struct iavf_adapter ;struct iavf_rx_queue ;struct iavf_tx_queue ;
drivers/net/iavf/iavf_rxtx.c +37 -0
@@ -746,6 +746,10 @@ iavf_rxd_to_pkt_flags(uint64_t qword) IAVF_RX_DESC_FLTSTAT_RSS_HASH) == IAVF_RX_DESC_FLTSTAT_RSS_HASH) ? PKT_RX_RSS_HASH : 0; +/* Check if FDIR Match */ + flags |= (qword & (1 <<IAVF_RX_DESC_STATUS_FLM_SHIFT ) ? +PKT_RX_FDIR :0 ); +if (likely ((error_bits &IAVF_RX_ERR_BITS ) ==0 )) { flags |= (PKT_RX_IP_CKSUM_GOOD |PKT_RX_L4_CKSUM_GOOD );return flags;
@@ -766,6 +770,25 @@ iavf_rxd_to_pkt_flags(uint64_t qword)return flags; } +static inline uint64_t +iavf_rxd_build_fdir (volatile union iavf_rx_desc *rxdp,struct rte_mbuf *mb) +{ +uint64_t flags =0 ; +uint16_t flexbh; + + flexbh = (rte_le_to_cpu_32 (rxdp->wb.qword2.ext_status) >> +IAVF_RX_DESC_EXT_STATUS_FLEXBH_SHIFT ) & +IAVF_RX_DESC_EXT_STATUS_FLEXBH_MASK ; + +if (flexbh ==IAVF_RX_DESC_EXT_STATUS_FLEXBH_FD_ID ) { + mb->hash.fdir.hi = +rte_le_to_cpu_32 (rxdp->wb.qword3.hi_dword.fd_id); + flags |=PKT_RX_FDIR_ID ; + } + +return flags; +} +/* Translate the rx flex descriptor status to pkt flags */ static inline void iavf_rxd_to_pkt_fields(struct rte_mbuf *mb,
@@ -782,6 +805,11 @@ iavf_rxd_to_pkt_fields(struct rte_mbuf *mb, mb->hash.rss = rte_le_to_cpu_32(desc->rss_hash); }#endif + +if (desc->flow_id !=0xFFFFFFFF ) { + mb->ol_flags |=PKT_RX_FDIR |PKT_RX_FDIR_ID ; + mb->hash.fdir.hi =rte_le_to_cpu_32 (desc->flow_id); + } }#define IAVF_RX_FLEX_ERR0_BITS \
@@ -917,6 +945,9 @@ iavf_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) rxm->hash.rss =rte_le_to_cpu_32 (rxd.wb.qword0.hi_dword.rss); +if (pkt_flags &PKT_RX_FDIR ) + pkt_flags |=iavf_rxd_build_fdir (&rxd, rxm); + rxm->ol_flags |= pkt_flags; rx_pkts[nb_rx++] = rxm;
@@ -1365,6 +1396,9 @@ iavf_recv_scattered_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, first_seg->hash.rss =rte_le_to_cpu_32 (rxd.wb.qword0.hi_dword.rss); +if (pkt_flags &PKT_RX_FDIR ) + pkt_flags |=iavf_rxd_build_fdir (&rxd, first_seg); + first_seg->ol_flags |= pkt_flags;/* Prefetch data of first segment, if configured to do so. */
@@ -1549,6 +1583,9 @@ iavf_rx_scan_hw_ring(struct iavf_rx_queue *rxq) mb->hash.rss =rte_le_to_cpu_32 ( rxdp[j].wb.qword0.hi_dword.rss); +if (pkt_flags &PKT_RX_FDIR ) + pkt_flags |=iavf_rxd_build_fdir (&rxdp[j], mb); + mb->ol_flags |= pkt_flags; }
Signature · 1 line
2.17.1
PATCH 9/12 net/iavf: support flow mark in AVX path
In reply to Leyi RongSupport Flow Director mark ID parsing from Flex
Rx descriptor in AVX path.
--- drivers/net/iavf/iavf_rxtx_vec_avx2.c | 82 +++++++++++++++++++++++++-- 1 file changed, 77 insertions(+), 5 deletions(-)
drivers/net/iavf/iavf_rxtx_vec_avx2.c +77 -5
@@ -550,6 +550,25 @@ _iavf_recv_raw_pkts_vec_avx2(struct iavf_rx_queue *rxq,return received; } +static inline __m256i +flex_rxd_to_fdir_flags_vec_avx2 (const __m256i fdir_id0_7) +{ +#define FDID_MIS_MAGIC 0xFFFFFFFF +RTE_BUILD_BUG_ON (PKT_RX_FDIR != (1 <<2 )); +RTE_BUILD_BUG_ON (PKT_RX_FDIR_ID != (1 <<13 )); +const __m256i pkt_fdir_bit =_mm256_set1_epi32 (PKT_RX_FDIR | +PKT_RX_FDIR_ID ); +/* desc->flow_id field == 0xFFFFFFFF means fdir mismatch */ +const __m256i fdir_mis_mask =_mm256_set1_epi32 (FDID_MIS_MAGIC ); +__m256i fdir_mask =_mm256_cmpeq_epi32 (fdir_id0_7, + fdir_mis_mask); +/* this XOR op results to bit-reverse the fdir_mask */ + fdir_mask =_mm256_xor_si256 (fdir_mask, fdir_mis_mask); +const __m256i fdir_flags =_mm256_and_si256 (fdir_mask, pkt_fdir_bit); + +return fdir_flags; +} + static inline uint16_t _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq, struct rte_mbuf **rx_pkts,
@@ -612,8 +631,8 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,const __m256i shuf_msk = _mm256_set_epi8 (/* first descriptor */ - 15, 14, - 13, 12, /* octet 12~15, 32 bits rss */ + 0xFF, 0xFF, + 0xFF, 0xFF, /* rss not supported */ 11, 10, /* octet 10~11, 16 bits vlan_macip */ 5, 4, /* octet 4~5, 16 bits data_len */ 0xFF, 0xFF, /* skip hi 16 bits pkt_len, zero out */
@@ -621,8 +640,8 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,0xFF ,0xFF ,/* pkt_type set as unknown */ 0xFF ,0xFF ,/*pkt_type set as unknown */ /* second descriptor */ -15 ,14 , -13 ,12 ,/* octet 12~15, 32 bits rss */ +0xFF ,0xFF , +0xFF ,0xFF ,/* rss not supported */ 11 ,10 ,/* octet 10~11, 16 bits vlan_macip */ 5 ,4 ,/* octet 4~5, 16 bits data_len */ 0xFF, 0xFF, /* skip hi 16 bits pkt_len, zero out */
@@ -864,8 +883,61 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq, rss_vlan_flag_bits);/* merge flags */ -const __m256i mbuf_flags =_mm256_or_si256 (l3_l4_flags, +__m256i mbuf_flags =_mm256_or_si256 (l3_l4_flags, rss_vlan_flags); + +if (rxq->vsi->adapter->fdir_enabled) { +const __m256i fdir_id4_7 = +_mm256_unpackhi_epi32 (raw_desc6_7, raw_desc4_5); + +const __m256i fdir_id0_3 = +_mm256_unpackhi_epi32 (raw_desc2_3, raw_desc0_1); + +const __m256i fdir_id0_7 = +_mm256_unpackhi_epi64 (fdir_id4_7, fdir_id0_3); + +const __m256i fdir_flags = +flex_rxd_to_fdir_flags_vec_avx2 (fdir_id0_7); + +/* merge with fdir_flags */ + mbuf_flags =_mm256_or_si256 (mbuf_flags, fdir_flags); + +/* write to mbuf: have to use scalar store here */ +uint32_t fdir_id_extr; + + fdir_id_extr =_mm256_extract_epi32 (fdir_id0_7,3 ); +if (fdir_id_extr !=FDID_MIS_MAGIC ) + rx_pkts[i +0 ]->hash.fdir.hi = fdir_id_extr; + + fdir_id_extr =_mm256_extract_epi32 (fdir_id0_7,7 ); +if (fdir_id_extr !=FDID_MIS_MAGIC ) + rx_pkts[i +1 ]->hash.fdir.hi = fdir_id_extr; + + fdir_id_extr =_mm256_extract_epi32 (fdir_id0_7,2 ); +if (fdir_id_extr !=FDID_MIS_MAGIC ) + rx_pkts[i +2 ]->hash.fdir.hi = fdir_id_extr; + + fdir_id_extr =_mm256_extract_epi32 (fdir_id0_7,6 ); +if (fdir_id_extr !=FDID_MIS_MAGIC ) + rx_pkts[i +3 ]->hash.fdir.hi = fdir_id_extr; + + fdir_id_extr =_mm256_extract_epi32 (fdir_id0_7,1 ); +if (fdir_id_extr !=FDID_MIS_MAGIC ) + rx_pkts[i +4 ]->hash.fdir.hi = fdir_id_extr; + + fdir_id_extr =_mm256_extract_epi32 (fdir_id0_7,5 ); +if (fdir_id_extr !=FDID_MIS_MAGIC ) + rx_pkts[i +5 ]->hash.fdir.hi = fdir_id_extr; + + fdir_id_extr =_mm256_extract_epi32 (fdir_id0_7,0 ); +if (fdir_id_extr !=FDID_MIS_MAGIC ) + rx_pkts[i +6 ]->hash.fdir.hi = fdir_id_extr; + + fdir_id_extr =_mm256_extract_epi32 (fdir_id0_7,4 ); +if (fdir_id_extr !=FDID_MIS_MAGIC ) + rx_pkts[i +7 ]->hash.fdir.hi = fdir_id_extr; + }/* if() on fdir_enabled */ +/** * At this point, we have the 8 sets of flags in the low 16-bits * of each 32-bit value in vlan0.
Signature · 1 line
2.17.1
PATCH 10/12 net/iavf: support flow mark in SSE path
In reply to Leyi RongSupport Flow Director mark ID parsing from Flex
Rx descriptor in SSE path.
--- drivers/net/iavf/iavf_rxtx_vec_sse.c | 48 +++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-)
drivers/net/iavf/iavf_rxtx_vec_sse.c +47 -1
@@ -189,6 +189,25 @@ desc_to_olflags_v(struct iavf_rx_queue *rxq, __m128i descs[4],_mm_store_si128 ((__m128i *)&rx_pkts[3 ]->rearm_data, rearm3); } +static inline __m128i +flex_rxd_to_fdir_flags_vec (const __m128i fdir_id0_3) +{ +#define FDID_MIS_MAGIC 0xFFFFFFFF +RTE_BUILD_BUG_ON (PKT_RX_FDIR != (1 <<2 )); +RTE_BUILD_BUG_ON (PKT_RX_FDIR_ID != (1 <<13 )); +const __m128i pkt_fdir_bit =_mm_set1_epi32 (PKT_RX_FDIR | +PKT_RX_FDIR_ID ); +/* desc->flow_id field == 0xFFFFFFFF means fdir mismatch */ +const __m128i fdir_mis_mask =_mm_set1_epi32 (FDID_MIS_MAGIC ); +__m128i fdir_mask =_mm_cmpeq_epi32 (fdir_id0_3, + fdir_mis_mask); +/* this XOR op results to bit-reverse the fdir_mask */ + fdir_mask =_mm_xor_si128 (fdir_mask, fdir_mis_mask); +const __m128i fdir_flags =_mm_and_si128 (fdir_mask, pkt_fdir_bit); + +return fdir_flags; +} +static inline void flex_desc_to_olflags_v (struct iavf_rx_queue *rxq,__m128i descs[4 ],struct rte_mbuf **rx_pkts)
@@ -267,6 +286,32 @@ flex_desc_to_olflags_v(struct iavf_rx_queue *rxq, __m128i descs[4],/* merge the flags */ flags =_mm_or_si128 (flags, rss_vlan); +if (rxq->vsi->adapter->fdir_enabled) { +const __m128i fdir_id0_1 = +_mm_unpackhi_epi32 (descs[0 ], descs[1 ]); + +const __m128i fdir_id2_3 = +_mm_unpackhi_epi32 (descs[2 ], descs[3 ]); + +const __m128i fdir_id0_3 = +_mm_unpackhi_epi64 (fdir_id0_1, fdir_id2_3); + +const __m128i fdir_flags = +flex_rxd_to_fdir_flags_vec (fdir_id0_3); + +/* merge with fdir_flags */ + flags =_mm_or_si128 (flags, fdir_flags); + +/* write fdir_id to mbuf if FDIR match */ +uint32_t fdir_id_extr, i; + +for (i =0 ; i <IAVF_VPMD_DESCS_PER_LOOP ; i++) { + fdir_id_extr =_mm_extract_epi32 (fdir_id0_3, i); +if (fdir_id_extr !=FDID_MIS_MAGIC ) + rx_pkts[i]->hash.fdir.hi = fdir_id_extr; + } + }/* if() on fdir_enabled */ +/** * At this point, we have the 4 sets of flags in the low 16-bits * of each 32-bit value in flags.
@@ -604,7 +649,8 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,const __m128i zero =_mm_setzero_si128 ();/* mask to shuffle from desc. to mbuf */ const __m128i shuf_msk = _mm_set_epi8 - (15, 14, 13, 12, /* octet 12~15, 32 bits rss */ + (0xFF, 0xFF, + 0xFF, 0xFF, /* rss not supported */ 11, 10, /* octet 10~11, 16 bits vlan_macip */ 5, 4, /* octet 4~5, 16 bits data_len */ 0xFF, 0xFF, /* skip high 16 bits pkt_len, zero out */
Signature · 1 line
2.17.1
PATCH 11/12 net/iavf: add RSS hash parsing in AVX path
In reply to Leyi RongSupport RSS hash parsing from Flex Rx
descriptor in AVX data path.
--- drivers/net/iavf/iavf_rxtx_vec_avx2.c | 92 ++++++++++++++++++++++++++- 1 file changed, 90 insertions(+), 2 deletions(-)
drivers/net/iavf/iavf_rxtx_vec_avx2.c +90 -2
@@ -632,7 +632,7 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq, _mm256_set_epi8 (/* first descriptor */ 0xFF, 0xFF, - 0xFF, 0xFF, /* rss not supported */ + 0xFF, 0xFF, /* rss hash parsed separately */ 11, 10, /* octet 10~11, 16 bits vlan_macip */ 5, 4, /* octet 4~5, 16 bits data_len */ 0xFF, 0xFF, /* skip hi 16 bits pkt_len, zero out */
@@ -641,7 +641,7 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,0xFF ,0xFF ,/*pkt_type set as unknown */ /* second descriptor */ 0xFF ,0xFF , -0xFF ,0xFF ,/* rss not supported */ +0xFF ,0xFF ,/* rss hash parsed separately */ 11 ,10 ,/* octet 10~11, 16 bits vlan_macip */ 5 ,4 ,/* octet 4~5, 16 bits data_len */ 0xFF, 0xFF, /* skip hi 16 bits pkt_len, zero out */
@@ -938,6 +938,94 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq, rx_pkts[i +7 ]->hash.fdir.hi = fdir_id_extr; }/* if() on fdir_enabled */ +/** +* needs to load 2nd 16B of each desc for RSS hash parsing, +* will cause performance drop to get into this context. +*/ +if (rxq->vsi->adapter->eth_dev->data->dev_conf.rxmode.offloads & +DEV_RX_OFFLOAD_RSS_HASH ) { +/* load bottom half of every 32B desc */ +const __m128i raw_desc_bh7 = +_mm_load_si128 + ((void *)(&rxdp[7 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh6 = +_mm_load_si128 + ((void *)(&rxdp[6 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh5 = +_mm_load_si128 + ((void *)(&rxdp[5 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh4 = +_mm_load_si128 + ((void *)(&rxdp[4 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh3 = +_mm_load_si128 + ((void *)(&rxdp[3 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh2 = +_mm_load_si128 + ((void *)(&rxdp[2 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh1 = +_mm_load_si128 + ((void *)(&rxdp[1 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh0 = +_mm_load_si128 + ((void *)(&rxdp[0 ].wb.status_error1)); + +__m256i raw_desc_bh6_7 = +_mm256_inserti128_si256 + (_mm256_castsi128_si256 (raw_desc_bh6), + raw_desc_bh7,1 ); +__m256i raw_desc_bh4_5 = +_mm256_inserti128_si256 + (_mm256_castsi128_si256 (raw_desc_bh4), + raw_desc_bh5,1 ); +__m256i raw_desc_bh2_3 = +_mm256_inserti128_si256 + (_mm256_castsi128_si256 (raw_desc_bh2), + raw_desc_bh3,1 ); +__m256i raw_desc_bh0_1 = +_mm256_inserti128_si256 + (_mm256_castsi128_si256 (raw_desc_bh0), + raw_desc_bh1,1 ); + +/** +* to shift the 32b RSS hash value to the +* highest 32b of each 128b before mask +*/ +__m256i rss_hash6_7 = +_mm256_slli_epi64 (raw_desc_bh6_7,32 ); +__m256i rss_hash4_5 = +_mm256_slli_epi64 (raw_desc_bh4_5,32 ); +__m256i rss_hash2_3 = +_mm256_slli_epi64 (raw_desc_bh2_3,32 ); +__m256i rss_hash0_1 = +_mm256_slli_epi64 (raw_desc_bh0_1,32 ); + +__m256i rss_hash_msk = +_mm256_set_epi32 (0xFFFFFFFF ,0 ,0 ,0 , +0xFFFFFFFF ,0 ,0 ,0 ); + + rss_hash6_7 =_mm256_and_si256 + (rss_hash6_7, rss_hash_msk); + rss_hash4_5 =_mm256_and_si256 + (rss_hash4_5, rss_hash_msk); + rss_hash2_3 =_mm256_and_si256 + (rss_hash2_3, rss_hash_msk); + rss_hash0_1 =_mm256_and_si256 + (rss_hash0_1, rss_hash_msk); + + mb6_7 =_mm256_or_si256 (mb6_7, rss_hash6_7); + mb4_5 =_mm256_or_si256 (mb4_5, rss_hash4_5); + mb2_3 =_mm256_or_si256 (mb2_3, rss_hash2_3); + mb0_1 =_mm256_or_si256 (mb0_1, rss_hash0_1); + }/* if() on RSS hash parsing */ +/** * At this point, we have the 8 sets of flags in the low 16-bits * of each 32-bit value in vlan0.
Signature · 1 line
2.17.1
PATCH 12/12 net/iavf: add RSS hash parsing in SSE path
In reply to Leyi RongSupport RSS hash parsing from Flex Rx
descriptor in SSE data path.
--- drivers/net/iavf/iavf_rxtx_vec_sse.c | 86 ++++++++++++++++++++++------ 1 file changed, 70 insertions(+), 16 deletions(-)
drivers/net/iavf/iavf_rxtx_vec_sse.c +70 -16
@@ -650,7 +650,7 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,/* mask to shuffle from desc. to mbuf */ const __m128i shuf_msk = _mm_set_epi8 (0xFF, 0xFF, - 0xFF, 0xFF, /* rss not supported */ + 0xFF, 0xFF, /* rss hash parsed separately */ 11, 10, /* octet 10~11, 16 bits vlan_macip */ 5, 4, /* octet 4~5, 16 bits data_len */ 0xFF, 0xFF, /* skip high 16 bits pkt_len, zero out */
@@ -741,7 +741,7 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq, pos += IAVF_VPMD_DESCS_PER_LOOP, rxdp += IAVF_VPMD_DESCS_PER_LOOP) {__m128i descs[IAVF_VPMD_DESCS_PER_LOOP ]; -__m128i pkt_mb1, pkt_mb2, pkt_mb3, pkt_mb4; +__m128i pkt_mb0, pkt_mb1, pkt_mb2, pkt_mb3;__m128i staterr, sterr_tmp1, sterr_tmp2;/* 2 64 bit or 4 32 bit mbuf pointers in one XMM reg. */ __m128i mbp1;
@@ -787,8 +787,12 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,rte_compiler_barrier ();/* D.1 pkt 3,4 convert format from desc to pktmbuf */ - pkt_mb4 =_mm_shuffle_epi8 (descs[3 ], shuf_msk); - pkt_mb3 =_mm_shuffle_epi8 (descs[2 ], shuf_msk); + pkt_mb3 =_mm_shuffle_epi8 (descs[3 ], shuf_msk); + pkt_mb2 =_mm_shuffle_epi8 (descs[2 ], shuf_msk); + +/* D.1 pkt 1,2 convert format from desc to pktmbuf */ + pkt_mb1 =_mm_shuffle_epi8 (descs[1 ], shuf_msk); + pkt_mb0 =_mm_shuffle_epi8 (descs[0 ], shuf_msk);/* C.1 4=>2 filter staterr info only */ sterr_tmp2 =_mm_unpackhi_epi32 (descs[3 ], descs[2 ]);
@@ -798,12 +802,66 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,flex_desc_to_olflags_v (rxq, descs, &rx_pkts[pos]);/* D.2 pkt 3,4 set in_port/nb_seg and remove crc */ - pkt_mb4 =_mm_add_epi16 (pkt_mb4, crc_adjust); pkt_mb3 =_mm_add_epi16 (pkt_mb3, crc_adjust); + pkt_mb2 =_mm_add_epi16 (pkt_mb2, crc_adjust); -/* D.1 pkt 1,2 convert format from desc to pktmbuf */ - pkt_mb2 =_mm_shuffle_epi8 (descs[1 ], shuf_msk); - pkt_mb1 =_mm_shuffle_epi8 (descs[0 ], shuf_msk); +/* D.2 pkt 1,2 set in_port/nb_seg and remove crc */ + pkt_mb1 =_mm_add_epi16 (pkt_mb1, crc_adjust); + pkt_mb0 =_mm_add_epi16 (pkt_mb0, crc_adjust); + +/** +* needs to load 2nd 16B of each desc for RSS hash parsing, +* will cause performance drop to get into this context. +*/ +if (rxq->vsi->adapter->eth_dev->data->dev_conf.rxmode.offloads & +DEV_RX_OFFLOAD_RSS_HASH ) { +/* load bottom half of every 32B desc */ +const __m128i raw_desc_bh3 = +_mm_load_si128 + ((void *)(&rxdp[3 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh2 = +_mm_load_si128 + ((void *)(&rxdp[2 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh1 = +_mm_load_si128 + ((void *)(&rxdp[1 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh0 = +_mm_load_si128 + ((void *)(&rxdp[0 ].wb.status_error1)); + +/** +* to shift the 32b RSS hash value to the +* highest 32b of each 128b before mask +*/ +__m128i rss_hash3 = +_mm_slli_epi64 (raw_desc_bh3,32 ); +__m128i rss_hash2 = +_mm_slli_epi64 (raw_desc_bh2,32 ); +__m128i rss_hash1 = +_mm_slli_epi64 (raw_desc_bh1,32 ); +__m128i rss_hash0 = +_mm_slli_epi64 (raw_desc_bh0,32 ); + +__m128i rss_hash_msk = +_mm_set_epi32 (0xFFFFFFFF ,0 ,0 ,0 ); + + rss_hash3 =_mm_and_si128 + (rss_hash3, rss_hash_msk); + rss_hash2 =_mm_and_si128 + (rss_hash2, rss_hash_msk); + rss_hash1 =_mm_and_si128 + (rss_hash1, rss_hash_msk); + rss_hash0 =_mm_and_si128 + (rss_hash0, rss_hash_msk); + + pkt_mb3 =_mm_or_si128 (pkt_mb3, rss_hash3); + pkt_mb2 =_mm_or_si128 (pkt_mb2, rss_hash2); + pkt_mb1 =_mm_or_si128 (pkt_mb1, rss_hash1); + pkt_mb0 =_mm_or_si128 (pkt_mb0, rss_hash0); + }/* if() on RSS hash parsing */ /* C.2 get 4 pkts staterr value */ staterr =_mm_unpacklo_epi32 (sterr_tmp1, sterr_tmp2);
@@ -811,14 +869,10 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,/* D.3 copy final 3,4 data to rx_pkts */ _mm_storeu_si128 ((void *)&rx_pkts[pos +3 ]->rx_descriptor_fields1, - pkt_mb4); + pkt_mb3);_mm_storeu_si128 ((void *)&rx_pkts[pos +2 ]->rx_descriptor_fields1, - pkt_mb3); - -/* D.2 pkt 1,2 set in_port/nb_seg and remove crc */ - pkt_mb2 =_mm_add_epi16 (pkt_mb2, crc_adjust); - pkt_mb1 =_mm_add_epi16 (pkt_mb1, crc_adjust); + pkt_mb2);/* C* extract and record EOP bit */ if (split_packet) {
@@ -842,9 +896,9 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,/* D.3 copy final 1,2 data to rx_pkts */ _mm_storeu_si128 ((void *)&rx_pkts[pos +1 ]->rx_descriptor_fields1, - pkt_mb2); + pkt_mb1);_mm_storeu_si128 ((void *)&rx_pkts[pos]->rx_descriptor_fields1, - pkt_mb1); + pkt_mb0);flex_desc_to_ptype_v (descs, &rx_pkts[pos], ptype_tbl);/* C.4 calc avaialbe number of desc */ var =__builtin_popcountll (_mm_cvtsi128_si64 (staterr));
Signature · 1 line
2.17.1
v2 cover framework for advanced iAVF PMD
In reply to Leyi RongThis patchset enable framework for advanced iAVF, includes query DDP pkg info, flexible descriptor support, FDIR mark id and RSS hash support.
---
v2:
- Revert macro RTE_LIBRTE_IAVF_16BYTE_RX_DESC deletion as it's defined in AVF spec.
- Typo RTE_LIBRTE_ICE_16BYTE_RX_DESC fix in iavf_rxtx.c.
- Move flex desc definitions into iavf_rxtx.h.
- Up to date to match with the latest version of virtchnl.h.
- Extract a new internal func iavf_update_rx_tail to call.
- Remove iavf_dev_rxq_count_flex_rxd()/iavf_dev_rx_desc_status_flex_rxd(), as the accompanying legacy ones can deal with the flex desc cases.
- Move rxq->rxdid assignment from iavf_configure_queues() to iavf_dev_rx_queue_setup().
- Unfold _mm_extract_epi32(fdir_id0_3, i) to fix build error when using GCC compile option -O0.
Leyi Rong (12):
net/iavf: flexible Rx descriptor definitions net/iavf: return error if opcode is mismatched net/iavf: support to query DDP package info net/iavf: flexible Rx descriptor support in normal path net/iavf: flexible Rx descriptor support in AVX path net/iavf: flexible Rx descriptor support in SSE path net/iavf: add flow director enabled switch value net/iavf: support flow mark in normal data path net/iavf: support flow mark in AVX path net/iavf: support flow mark in SSE path net/iavf: add RSS hash parsing in AVX path net/iavf: add RSS hash parsing in SSE path
drivers/net/iavf/iavf.h | 15 + drivers/net/iavf/iavf_ethdev.c | 15 + drivers/net/iavf/iavf_rxtx.c | 542 ++++++++++++++++++-- drivers/net/iavf/iavf_rxtx.h | 245 +++++++++ drivers/net/iavf/iavf_rxtx_vec_avx2.c | 710 +++++++++++++++++++++++++- drivers/net/iavf/iavf_rxtx_vec_sse.c | 524 +++++++++++++++++++ drivers/net/iavf/iavf_vchnl.c | 85 ++- 7 files changed, 2087 insertions(+), 49 deletions(-)
Signature · 1 line
2.17.1
PATCH v2 1/12 net/iavf: flexible Rx descriptor definitions
Add definitions for flexible Rx descriptor structures and macros.
--- drivers/net/iavf/iavf_rxtx.h | 200 +++++++++++++++++++++++++++++++++++ 1 file changed, 200 insertions(+)
drivers/net/iavf/iavf_rxtx.h +200 -0
@@ -157,6 +157,206 @@ union iavf_tx_offload { }; }; +/* Rx Flex Descriptors +* These descriptors are used instead of the legacy version descriptors +*/ +union iavf_16b_rx_flex_desc { +struct { +__le64 pkt_addr;/* Packet buffer address */ +__le64 hdr_addr;/* Header buffer address */ +/* bit 0 of hdr_addr is DD bit */ + } read; +struct { +/* Qword 0 */ +u8 rxdid;/* descriptor builder profile ID */ +u8 mir_id_umb_cast;/* mirror=[5:0], umb=[7:6] */ +__le16 ptype_flex_flags0;/* ptype=[9:0], ff0=[15:10] */ +__le16 pkt_len;/* [15:14] are reserved */ +__le16 hdr_len_sph_flex_flags1;/* header=[10:0] */ +/* sph=[11:11] */ +/* ff1/ext=[15:12] */ + +/* Qword 1 */ +__le16 status_error0; +__le16 l2tag1; +__le16 flex_meta0; +__le16 flex_meta1; + } wb;/* writeback */ +}; + +union iavf_32b_rx_flex_desc { +struct { +__le64 pkt_addr;/* Packet buffer address */ +__le64 hdr_addr;/* Header buffer address */ +/* bit 0 of hdr_addr is DD bit */ +__le64 rsvd1; +__le64 rsvd2; + } read; +struct { +/* Qword 0 */ +u8 rxdid;/* descriptor builder profile ID */ +u8 mir_id_umb_cast;/* mirror=[5:0], umb=[7:6] */ +__le16 ptype_flex_flags0;/* ptype=[9:0], ff0=[15:10] */ +__le16 pkt_len;/* [15:14] are reserved */ +__le16 hdr_len_sph_flex_flags1;/* header=[10:0] */ +/* sph=[11:11] */ +/* ff1/ext=[15:12] */ + +/* Qword 1 */ +__le16 status_error0; +__le16 l2tag1; +__le16 flex_meta0; +__le16 flex_meta1; + +/* Qword 2 */ +__le16 status_error1; +u8 flex_flags2; +u8 time_stamp_low; +__le16 l2tag2_1st; +__le16 l2tag2_2nd; + +/* Qword 3 */ +__le16 flex_meta2; +__le16 flex_meta3; +union { +struct { +__le16 flex_meta4; +__le16 flex_meta5; + } flex; +__le32 ts_high; + } flex_ts; + } wb;/* writeback */ +}; + +/* Rx Flex Descriptor for Comms Package Profile +* RxDID Profile ID 16-21 +* Flex-field 0: RSS hash lower 16-bits +* Flex-field 1: RSS hash upper 16-bits +* Flex-field 2: Flow ID lower 16-bits +* Flex-field 3: Flow ID upper 16-bits +* Flex-field 4: AUX0 +* Flex-field 5: AUX1 +*/ +struct iavf_32b_rx_flex_desc_comms { +/* Qword 0 */ +u8 rxdid; +u8 mir_id_umb_cast; +__le16 ptype_flexi_flags0; +__le16 pkt_len; +__le16 hdr_len_sph_flex_flags1; + +/* Qword 1 */ +__le16 status_error0; +__le16 l2tag1; +__le32 rss_hash; + +/* Qword 2 */ +__le16 status_error1; +u8 flexi_flags2; +u8 ts_low; +__le16 l2tag2_1st; +__le16 l2tag2_2nd; + +/* Qword 3 */ +__le32 flow_id; +union { +struct { +__le16 aux0; +__le16 aux1; + } flex; +__le32 ts_high; + } flex_ts; +}; + +/* Rx Flex Descriptor for Comms Package Profile +* RxDID Profile ID 22-23 (swap Hash and FlowID) +* Flex-field 0: Flow ID lower 16-bits +* Flex-field 1: Flow ID upper 16-bits +* Flex-field 2: RSS hash lower 16-bits +* Flex-field 3: RSS hash upper 16-bits +* Flex-field 4: AUX0 +* Flex-field 5: AUX1 +*/ +struct iavf_32b_rx_flex_desc_comms_ovs { +/* Qword 0 */ +u8 rxdid; +u8 mir_id_umb_cast; +__le16 ptype_flexi_flags0; +__le16 pkt_len; +__le16 hdr_len_sph_flex_flags1; + +/* Qword 1 */ +__le16 status_error0; +__le16 l2tag1; +__le32 flow_id; + +/* Qword 2 */ +__le16 status_error1; +u8 flexi_flags2; +u8 ts_low; +__le16 l2tag2_1st; +__le16 l2tag2_2nd; + +/* Qword 3 */ +__le32 rss_hash; +union { +struct { +__le16 aux0; +__le16 aux1; + } flex; +__le32 ts_high; + } flex_ts; +}; + +/* Receive Flex Descriptor profile IDs: There are a total +* of 64 profiles where profile IDs 0/1 are for legacy; and +* profiles 2-63 are flex profiles that can be programmed +* with a specific metadata (profile 7 reserved for HW) +*/ +enum iavf_rxdid { +IAVF_RXDID_LEGACY_0 =0 , +IAVF_RXDID_LEGACY_1 =1 , +IAVF_RXDID_FLEX_NIC =2 , +IAVF_RXDID_FLEX_NIC_2 =6 , +IAVF_RXDID_HW =7 , +IAVF_RXDID_COMMS_GENERIC =16 , +IAVF_RXDID_COMMS_AUX_VLAN =17 , +IAVF_RXDID_COMMS_AUX_IPV4 =18 , +IAVF_RXDID_COMMS_AUX_IPV6 =19 , +IAVF_RXDID_COMMS_AUX_IPV6_FLOW =20 , +IAVF_RXDID_COMMS_AUX_TCP =21 , +IAVF_RXDID_COMMS_OVS_1 =22 , +IAVF_RXDID_COMMS_OVS_2 =23 , +IAVF_RXDID_LAST =63 , +}; + +enum iavf_rx_flex_desc_status_error_0_bits { +/* Note: These are predefined bit offsets */ +IAVF_RX_FLEX_DESC_STATUS0_DD_S =0 , +IAVF_RX_FLEX_DESC_STATUS0_EOF_S , +IAVF_RX_FLEX_DESC_STATUS0_HBO_S , +IAVF_RX_FLEX_DESC_STATUS0_L3L4P_S , +IAVF_RX_FLEX_DESC_STATUS0_XSUM_IPE_S , +IAVF_RX_FLEX_DESC_STATUS0_XSUM_L4E_S , +IAVF_RX_FLEX_DESC_STATUS0_XSUM_EIPE_S , +IAVF_RX_FLEX_DESC_STATUS0_XSUM_EUDPE_S , +IAVF_RX_FLEX_DESC_STATUS0_LPBK_S , +IAVF_RX_FLEX_DESC_STATUS0_IPV6EXADD_S , +IAVF_RX_FLEX_DESC_STATUS0_RXE_S , +IAVF_RX_FLEX_DESC_STATUS0_CRCP_S , +IAVF_RX_FLEX_DESC_STATUS0_RSS_VALID_S , +IAVF_RX_FLEX_DESC_STATUS0_L2TAG1P_S , +IAVF_RX_FLEX_DESC_STATUS0_XTRMD0_VALID_S , +IAVF_RX_FLEX_DESC_STATUS0_XTRMD1_VALID_S , +IAVF_RX_FLEX_DESC_STATUS0_LAST /* this entry must be last!!! */ +}; + +/* for iavf_32b_rx_flex_desc.ptype_flex_flags0 member */ +#define IAVF_RX_FLEX_DESC_PTYPE_M (0x3FF)/* 10-bits */ + +/* for iavf_32b_rx_flex_desc.pkt_len member */ +#define IAVF_RX_FLX_DESC_PKT_LEN_M (0x3FFF)/* 14-bits */ + int iavf_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, uint16_t nb_desc,
Signature · 1 line
2.17.1
PATCH v2 2/12 net/iavf: return error if opcode is mismatched
In reply to Leyi RongAdds error return when the opcode of read message is mismatched which is received from adminQ.
--- drivers/net/iavf/iavf_vchnl.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
drivers/net/iavf/iavf_vchnl.c +3 -1
@@ -52,9 +52,11 @@ iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len,PMD_DRV_LOG (DEBUG ,"AQ from pf carries opcode %u, retval %d" , opcode, vf->cmd_retval); -if (opcode != vf->pend_cmd) +if (opcode != vf->pend_cmd) {PMD_DRV_LOG (WARNING ,"command mismatch, expect %u, get %u" , vf->pend_cmd, opcode); +return IAVF_ERR_OPCODE_MISMATCH ; + }return IAVF_SUCCESS ; }
Signature · 1 line
2.17.1
PATCH v2 3/12 net/iavf: support to query DDP package info
In reply to Leyi RongAdvanced iAVF supports to query DDP package info, includes package version, track id, package name, device serial number and the list of protocols that the PF supports.
--- drivers/net/iavf/iavf.h | 9 ++++++++ drivers/net/iavf/iavf_ethdev.c | 7 ++++++ drivers/net/iavf/iavf_vchnl.c | 41 +++++++++++++++++++++++++++++++++- 3 files changed, 56 insertions(+), 1 deletion(-)
drivers/net/iavf/iavf.h +9 -0
@@ -67,6 +67,13 @@#define IAVF_48_BIT_WIDTH (CHAR_BIT * 6)#define IAVF_48_BIT_MASK RTE_LEN2MASK(IAVF_48_BIT_WIDTH, uint64_t) +/* VF supported comms protocols 64-bits bitmap */ +#define IAVF_COMMS_PROTO_GTP 0x0000000000000001 +#define IAVF_COMMS_PROTO_PPPOE 0x0000000000000002 +#define IAVF_COMMS_PROTO_PFCP 0x0000000000000004 +#define IAVF_COMMS_PROTO_L2TPV3 0x0000000000000008 +#define IAVF_COMMS_PROTO_ESP 0x0000000000000010 +struct iavf_adapter ;struct iavf_rx_queue ;struct iavf_tx_queue ;
@@ -97,6 +104,7 @@ struct iavf_info {struct virtchnl_version_info virtchnl_version;struct virtchnl_vf_resource *vf_res;/* VF resource */ struct virtchnl_vsi_resource *vsi_res;/* LAN VSI */ +struct virtchnl_pkg_info pkg_info;/* package info */ volatile enum virtchnl_ops pend_cmd;/* pending command not finished */ uint32_t cmd_retval;/* return value of the cmd response from PF */
@@ -225,6 +233,7 @@ int iavf_disable_queues(struct iavf_adapter *adapter);int iavf_configure_rss_lut (struct iavf_adapter *adapter);int iavf_configure_rss_key (struct iavf_adapter *adapter);int iavf_configure_queues (struct iavf_adapter *adapter); +int iavf_query_package_info (struct iavf_adapter *adapter);int iavf_config_irq_map (struct iavf_adapter *adapter);void iavf_add_del_all_mac_addr (struct iavf_adapter *adapter,bool add); int iavf_dev_link_update(struct rte_eth_dev *dev,
drivers/net/iavf/iavf_ethdev.c +7 -0
@@ -1236,6 +1236,13 @@ iavf_init_vf(struct rte_eth_dev *dev) goto err_rss; } } + +if (vf->vf_res->vf_cap_flags &VIRTCHNL_VF_OFFLOAD_QUERY_DDP ) { +if (iavf_query_package_info (adapter) !=0 ) { +PMD_INIT_LOG (ERR ,"iavf_query_package_info failed" ); + goto err_rss; + } + }return 0 ; err_rss:rte_free (vf->rss_key);
drivers/net/iavf/iavf_vchnl.c +40 -1
59 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH v2 4/12 net/iavf: flexible Rx descriptor support in normal path
In reply to Leyi RongSupport flexible Rx descriptor format in normal
path of iAVF PMD.
--- drivers/net/iavf/iavf.h | 2 + drivers/net/iavf/iavf_ethdev.c | 8 + drivers/net/iavf/iavf_rxtx.c | 481 ++++++++++++++++++++++++++++++--- drivers/net/iavf/iavf_rxtx.h | 8 + drivers/net/iavf/iavf_vchnl.c | 42 ++- 5 files changed, 503 insertions(+), 38 deletions(-)
drivers/net/iavf/iavf.h +2 -0
@@ -105,6 +105,7 @@ struct iavf_info {struct virtchnl_vf_resource *vf_res;/* VF resource */ struct virtchnl_vsi_resource *vsi_res;/* LAN VSI */ struct virtchnl_pkg_info pkg_info;/* package info */ +uint64_t supported_rxdid;volatile enum virtchnl_ops pend_cmd;/* pending command not finished */ uint32_t cmd_retval;/* return value of the cmd response from PF */
@@ -233,6 +234,7 @@ int iavf_disable_queues(struct iavf_adapter *adapter);int iavf_configure_rss_lut (struct iavf_adapter *adapter);int iavf_configure_rss_key (struct iavf_adapter *adapter);int iavf_configure_queues (struct iavf_adapter *adapter); +int iavf_get_supported_rxdid (struct iavf_adapter *adapter);int iavf_query_package_info (struct iavf_adapter *adapter);int iavf_config_irq_map (struct iavf_adapter *adapter);void iavf_add_del_all_mac_addr (struct iavf_adapter *adapter,bool add);
drivers/net/iavf/iavf_ethdev.c +8 -0
@@ -1243,6 +1243,14 @@ iavf_init_vf(struct rte_eth_dev *dev) goto err_rss; } } + +if (vf->vf_res->vf_cap_flags &VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC ) { +if (iavf_get_supported_rxdid (adapter) !=0 ) { +PMD_INIT_LOG (ERR ,"failed to do get supported rxdid" ); + goto err_rss; + } + } +return 0 ; err_rss:rte_free (vf->rss_key);
drivers/net/iavf/iavf_rxtx.c +444 -37
548 lines not on this page · raw message
drivers/net/iavf/iavf_rxtx.h +8 -0
@@ -62,6 +62,7 @@#define iavf_rx_desc iavf_16byte_rx_desc#else #define iavf_rx_desc iavf_32byte_rx_desc +#define iavf_rx_flex_desc iavf_32b_rx_flex_desc#endif struct iavf_rxq_ops {
@@ -87,6 +88,7 @@ struct iavf_rx_queue {struct rte_mbuf *pkt_first_seg;/* first segment of current packet */ struct rte_mbuf *pkt_last_seg;/* last segment of current packet */ struct rte_mbuf fake_mbuf;/* dummy mbuf */ +uint8_t rxdid;/* used for VPMD */ uint16_t rxrearm_nb;/* number of remaining to be re-armed */
@@ -379,9 +381,15 @@ void iavf_dev_tx_queue_release(void *txq);void iavf_stop_queues (struct rte_eth_dev *dev);uint16_t iavf_recv_pkts (void *rx_queue,struct rte_mbuf **rx_pkts,uint16_t nb_pkts); +uint16_t iavf_recv_pkts_flex_rxd (void *rx_queue, +struct rte_mbuf **rx_pkts, +uint16_t nb_pkts);uint16_t iavf_recv_scattered_pkts (void *rx_queue,struct rte_mbuf **rx_pkts,uint16_t nb_pkts); +uint16_t iavf_recv_scattered_pkts_flex_rxd (void *rx_queue, +struct rte_mbuf **rx_pkts, +uint16_t nb_pkts);uint16_t iavf_xmit_pkts (void *tx_queue,struct rte_mbuf **tx_pkts,uint16_t nb_pkts); uint16_t iavf_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
drivers/net/iavf/iavf_vchnl.c +41 -1
66 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH v2 5/12 net/iavf: flexible Rx descriptor support in AVX path
In reply to Leyi RongSupport flexible Rx descriptor format in AVX
path of iAVF PMD.
--- drivers/net/iavf/iavf_rxtx.c | 24 +- drivers/net/iavf/iavf_rxtx.h | 6 + drivers/net/iavf/iavf_rxtx_vec_avx2.c | 550 +++++++++++++++++++++++++- 3 files changed, 570 insertions(+), 10 deletions(-)
drivers/net/iavf/iavf_rxtx.c +18 -6
@@ -2083,16 +2083,28 @@ iavf_set_rx_function(struct rte_eth_dev *dev)"Using %sVector Scattered Rx (port %d)." , use_avx2 ?"avx2 " :"" , dev->data->port_id); - dev->rx_pkt_burst = use_avx2 ? - iavf_recv_scattered_pkts_vec_avx2 : - iavf_recv_scattered_pkts_vec; +if (vf->vf_res->vf_cap_flags & +VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC ) + dev->rx_pkt_burst = use_avx2 ? + iavf_recv_scattered_pkts_vec_avx2_flex_rxd : + iavf_recv_scattered_pkts_vec; +else + dev->rx_pkt_burst = use_avx2 ? + iavf_recv_scattered_pkts_vec_avx2 : + iavf_recv_scattered_pkts_vec; }else {PMD_DRV_LOG (DEBUG ,"Using %sVector Rx (port %d)." , use_avx2 ?"avx2 " :"" , dev->data->port_id); - dev->rx_pkt_burst = use_avx2 ? - iavf_recv_pkts_vec_avx2 : - iavf_recv_pkts_vec; +if (vf->vf_res->vf_cap_flags & +VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC ) + dev->rx_pkt_burst = use_avx2 ? + iavf_recv_pkts_vec_avx2_flex_rxd : + iavf_recv_pkts_vec; +else + dev->rx_pkt_burst = use_avx2 ? + iavf_recv_pkts_vec_avx2 : + iavf_recv_pkts_vec; }return ;
drivers/net/iavf/iavf_rxtx.h +6 -0
@@ -413,9 +413,15 @@ uint16_t iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts);uint16_t iavf_recv_pkts_vec_avx2 (void *rx_queue,struct rte_mbuf **rx_pkts,uint16_t nb_pkts); +uint16_t iavf_recv_pkts_vec_avx2_flex_rxd (void *rx_queue, +struct rte_mbuf **rx_pkts, +uint16_t nb_pkts);uint16_t iavf_recv_scattered_pkts_vec_avx2 (void *rx_queue,struct rte_mbuf **rx_pkts,uint16_t nb_pkts); +uint16_t iavf_recv_scattered_pkts_vec_avx2_flex_rxd (void *rx_queue, +struct rte_mbuf **rx_pkts, +uint16_t nb_pkts);uint16_t iavf_xmit_pkts_vec (void *tx_queue,struct rte_mbuf **tx_pkts,uint16_t nb_pkts); uint16_t iavf_xmit_pkts_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts,
drivers/net/iavf/iavf_rxtx_vec_avx2.c +546 -4
585 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH v2 6/12 net/iavf: flexible Rx descriptor support in SSE path
In reply to Leyi RongSupport flexible Rx descriptor format in SSE
path of iAVF PMD.
--- drivers/net/iavf/iavf_rxtx.c | 4 +- drivers/net/iavf/iavf_rxtx.h | 5 + drivers/net/iavf/iavf_rxtx_vec_sse.c | 414 +++++++++++++++++++++++++++ 3 files changed, 421 insertions(+), 2 deletions(-)
drivers/net/iavf/iavf_rxtx.c +2 -2
@@ -2087,7 +2087,7 @@ iavf_set_rx_function(struct rte_eth_dev *dev) VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) dev->rx_pkt_burst = use_avx2 ? iavf_recv_scattered_pkts_vec_avx2_flex_rxd : - iavf_recv_scattered_pkts_vec; + iavf_recv_scattered_pkts_vec_flex_rxd; else dev->rx_pkt_burst = use_avx2 ? iavf_recv_scattered_pkts_vec_avx2 :
@@ -2100,7 +2100,7 @@ iavf_set_rx_function(struct rte_eth_dev *dev) VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) dev->rx_pkt_burst = use_avx2 ? iavf_recv_pkts_vec_avx2_flex_rxd : - iavf_recv_pkts_vec; + iavf_recv_pkts_vec_flex_rxd; else dev->rx_pkt_burst = use_avx2 ? iavf_recv_pkts_vec_avx2 :
drivers/net/iavf/iavf_rxtx.h +5 -0
@@ -406,9 +406,14 @@ int iavf_dev_tx_desc_status(void *tx_queue, uint16_t offset);uint16_t iavf_recv_pkts_vec (void *rx_queue,struct rte_mbuf **rx_pkts,uint16_t nb_pkts); +uint16_t iavf_recv_pkts_vec_flex_rxd (void *rx_queue,struct rte_mbuf **rx_pkts, +uint16_t nb_pkts);uint16_t iavf_recv_scattered_pkts_vec (void *rx_queue,struct rte_mbuf **rx_pkts,uint16_t nb_pkts); +uint16_t iavf_recv_scattered_pkts_vec_flex_rxd (void *rx_queue, +struct rte_mbuf **rx_pkts, +uint16_t nb_pkts);uint16_t iavf_xmit_fixed_burst_vec (void *tx_queue,struct rte_mbuf **tx_pkts,uint16_t nb_pkts); uint16_t iavf_recv_pkts_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,
drivers/net/iavf/iavf_rxtx_vec_sse.c +414 -0
444 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH v2 7/12 net/iavf: add flow director enabled switch value
In reply to Leyi RongThe commit adds fdir_enabled flag into iavf_adapter structure to identify if fdir id is active. Rx data path can be benefit if fdir id parsing is not needed, especially in vector path.
--- drivers/net/iavf/iavf.h | 1 + drivers/net/iavf/iavf_rxtx.h | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+)
drivers/net/iavf/iavf.h +1 -0
@@ -142,6 +142,7 @@ struct iavf_adapter {bool tx_vec_allowed;const uint32_t *ptype_tbl;bool stopped; +uint8_t fdir_enabled; };/* IAVF_DEV_PRIVATE_TO */
drivers/net/iavf/iavf_rxtx.h +26 -0
@@ -490,6 +490,32 @@ void iavf_dump_tx_descriptor(const struct iavf_tx_queue *txq, tx_desc->cmd_type_offset_bsz); } +/* Enable/disable flow director Rx processing in data path. */ +static inline +void iavf_fdir_rx_proc_enable (struct iavf_adapter *ad,bool on) +{ +static uint32_t ref_cnt; + +if (on) { +/* enable flow director processing */ +if (ref_cnt++ ==0 ) { + ad->fdir_enabled = on; +PMD_DRV_LOG (DEBUG , +"FDIR processing on RX set to %d" , on); + } + }else { +if (ref_cnt >=1 ) { + ref_cnt--; + +if (ref_cnt ==0 ) { + ad->fdir_enabled = on; +PMD_DRV_LOG (DEBUG , +"FDIR processing on RX set to %d" , on); + } + } + } +} +#ifdef RTE_LIBRTE_IAVF_DEBUG_DUMP_DESC #define IAVF_DUMP_RX_DESC (rxq, desc, rx_id) \ iavf_dump_rx_descriptor(rxq, desc, rx_id)
Signature · 1 line
2.17.1
PATCH v2 8/12 net/iavf: support flow mark in normal data path
In reply to Leyi RongSupport Flow Director mark ID parsing in normal path.
--- drivers/net/iavf/iavf.h | 3 +++ drivers/net/iavf/iavf_rxtx.c | 37 ++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+)
drivers/net/iavf/iavf.h +3 -0
@@ -74,6 +74,9 @@#define IAVF_COMMS_PROTO_L2TPV3 0x0000000000000008#define IAVF_COMMS_PROTO_ESP 0x0000000000000010 +#define IAVF_RX_DESC_EXT_STATUS_FLEXBH_MASK 0x03 +#define IAVF_RX_DESC_EXT_STATUS_FLEXBH_FD_ID 0x01 +struct iavf_adapter ;struct iavf_rx_queue ;struct iavf_tx_queue ;
drivers/net/iavf/iavf_rxtx.c +37 -0
@@ -756,6 +756,10 @@ iavf_rxd_to_pkt_flags(uint64_t qword) IAVF_RX_DESC_FLTSTAT_RSS_HASH) == IAVF_RX_DESC_FLTSTAT_RSS_HASH) ? PKT_RX_RSS_HASH : 0; +/* Check if FDIR Match */ + flags |= (qword & (1 <<IAVF_RX_DESC_STATUS_FLM_SHIFT ) ? +PKT_RX_FDIR :0 ); +if (likely ((error_bits &IAVF_RX_ERR_BITS ) ==0 )) { flags |= (PKT_RX_IP_CKSUM_GOOD |PKT_RX_L4_CKSUM_GOOD );return flags;
@@ -776,6 +780,25 @@ iavf_rxd_to_pkt_flags(uint64_t qword)return flags; } +static inline uint64_t +iavf_rxd_build_fdir (volatile union iavf_rx_desc *rxdp,struct rte_mbuf *mb) +{ +uint64_t flags =0 ; +uint16_t flexbh; + + flexbh = (rte_le_to_cpu_32 (rxdp->wb.qword2.ext_status) >> +IAVF_RX_DESC_EXT_STATUS_FLEXBH_SHIFT ) & +IAVF_RX_DESC_EXT_STATUS_FLEXBH_MASK ; + +if (flexbh ==IAVF_RX_DESC_EXT_STATUS_FLEXBH_FD_ID ) { + mb->hash.fdir.hi = +rte_le_to_cpu_32 (rxdp->wb.qword3.hi_dword.fd_id); + flags |=PKT_RX_FDIR_ID ; + } + +return flags; +} +/* Translate the rx flex descriptor status to pkt flags */ static inline void iavf_rxd_to_pkt_fields(struct rte_mbuf *mb,
@@ -792,6 +815,11 @@ iavf_rxd_to_pkt_fields(struct rte_mbuf *mb, mb->hash.rss = rte_le_to_cpu_32(desc->rss_hash); }#endif + +if (desc->flow_id !=0xFFFFFFFF ) { + mb->ol_flags |=PKT_RX_FDIR |PKT_RX_FDIR_ID ; + mb->hash.fdir.hi =rte_le_to_cpu_32 (desc->flow_id); + } }#define IAVF_RX_FLEX_ERR0_BITS \
@@ -953,6 +981,9 @@ iavf_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) rxm->hash.rss =rte_le_to_cpu_32 (rxd.wb.qword0.hi_dword.rss); +if (pkt_flags &PKT_RX_FDIR ) + pkt_flags |=iavf_rxd_build_fdir (&rxd, rxm); + rxm->ol_flags |= pkt_flags; rx_pkts[nb_rx++] = rxm;
@@ -1351,6 +1382,9 @@ iavf_recv_scattered_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, first_seg->hash.rss =rte_le_to_cpu_32 (rxd.wb.qword0.hi_dword.rss); +if (pkt_flags &PKT_RX_FDIR ) + pkt_flags |=iavf_rxd_build_fdir (&rxd, first_seg); + first_seg->ol_flags |= pkt_flags;/* Prefetch data of first segment, if configured to do so. */
@@ -1517,6 +1551,9 @@ iavf_rx_scan_hw_ring(struct iavf_rx_queue *rxq) mb->hash.rss =rte_le_to_cpu_32 ( rxdp[j].wb.qword0.hi_dword.rss); +if (pkt_flags &PKT_RX_FDIR ) + pkt_flags |=iavf_rxd_build_fdir (&rxdp[j], mb); + mb->ol_flags |= pkt_flags; }
Signature · 1 line
2.17.1
PATCH v2 9/12 net/iavf: support flow mark in AVX path
In reply to Leyi RongSupport Flow Director mark ID parsing from Flex
Rx descriptor in AVX path.
--- drivers/net/iavf/iavf_rxtx_vec_avx2.c | 82 +++++++++++++++++++++++++-- 1 file changed, 77 insertions(+), 5 deletions(-)
drivers/net/iavf/iavf_rxtx_vec_avx2.c +77 -5
@@ -616,6 +616,25 @@ _iavf_recv_raw_pkts_vec_avx2(struct iavf_rx_queue *rxq,return received; } +static inline __m256i +flex_rxd_to_fdir_flags_vec_avx2 (const __m256i fdir_id0_7) +{ +#define FDID_MIS_MAGIC 0xFFFFFFFF +RTE_BUILD_BUG_ON (PKT_RX_FDIR != (1 <<2 )); +RTE_BUILD_BUG_ON (PKT_RX_FDIR_ID != (1 <<13 )); +const __m256i pkt_fdir_bit =_mm256_set1_epi32 (PKT_RX_FDIR | +PKT_RX_FDIR_ID ); +/* desc->flow_id field == 0xFFFFFFFF means fdir mismatch */ +const __m256i fdir_mis_mask =_mm256_set1_epi32 (FDID_MIS_MAGIC ); +__m256i fdir_mask =_mm256_cmpeq_epi32 (fdir_id0_7, + fdir_mis_mask); +/* this XOR op results to bit-reverse the fdir_mask */ + fdir_mask =_mm256_xor_si256 (fdir_mask, fdir_mis_mask); +const __m256i fdir_flags =_mm256_and_si256 (fdir_mask, pkt_fdir_bit); + +return fdir_flags; +} + static inline uint16_t _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq, struct rte_mbuf **rx_pkts,
@@ -678,8 +697,8 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,const __m256i shuf_msk = _mm256_set_epi8 (/* first descriptor */ - 15, 14, - 13, 12, /* octet 12~15, 32 bits rss */ + 0xFF, 0xFF, + 0xFF, 0xFF, /* rss not supported */ 11, 10, /* octet 10~11, 16 bits vlan_macip */ 5, 4, /* octet 4~5, 16 bits data_len */ 0xFF, 0xFF, /* skip hi 16 bits pkt_len, zero out */
@@ -687,8 +706,8 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,0xFF ,0xFF ,/* pkt_type set as unknown */ 0xFF ,0xFF ,/*pkt_type set as unknown */ /* second descriptor */ -15 ,14 , -13 ,12 ,/* octet 12~15, 32 bits rss */ +0xFF ,0xFF , +0xFF ,0xFF ,/* rss not supported */ 11 ,10 ,/* octet 10~11, 16 bits vlan_macip */ 5 ,4 ,/* octet 4~5, 16 bits data_len */ 0xFF, 0xFF, /* skip hi 16 bits pkt_len, zero out */
@@ -930,8 +949,61 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq, rss_vlan_flag_bits);/* merge flags */ -const __m256i mbuf_flags =_mm256_or_si256 (l3_l4_flags, +__m256i mbuf_flags =_mm256_or_si256 (l3_l4_flags, rss_vlan_flags); + +if (rxq->vsi->adapter->fdir_enabled) { +const __m256i fdir_id4_7 = +_mm256_unpackhi_epi32 (raw_desc6_7, raw_desc4_5); + +const __m256i fdir_id0_3 = +_mm256_unpackhi_epi32 (raw_desc2_3, raw_desc0_1); + +const __m256i fdir_id0_7 = +_mm256_unpackhi_epi64 (fdir_id4_7, fdir_id0_3); + +const __m256i fdir_flags = +flex_rxd_to_fdir_flags_vec_avx2 (fdir_id0_7); + +/* merge with fdir_flags */ + mbuf_flags =_mm256_or_si256 (mbuf_flags, fdir_flags); + +/* write to mbuf: have to use scalar store here */ +uint32_t fdir_id_extr; + + fdir_id_extr =_mm256_extract_epi32 (fdir_id0_7,3 ); +if (fdir_id_extr !=FDID_MIS_MAGIC ) + rx_pkts[i +0 ]->hash.fdir.hi = fdir_id_extr; + + fdir_id_extr =_mm256_extract_epi32 (fdir_id0_7,7 ); +if (fdir_id_extr !=FDID_MIS_MAGIC ) + rx_pkts[i +1 ]->hash.fdir.hi = fdir_id_extr; + + fdir_id_extr =_mm256_extract_epi32 (fdir_id0_7,2 ); +if (fdir_id_extr !=FDID_MIS_MAGIC ) + rx_pkts[i +2 ]->hash.fdir.hi = fdir_id_extr; + + fdir_id_extr =_mm256_extract_epi32 (fdir_id0_7,6 ); +if (fdir_id_extr !=FDID_MIS_MAGIC ) + rx_pkts[i +3 ]->hash.fdir.hi = fdir_id_extr; + + fdir_id_extr =_mm256_extract_epi32 (fdir_id0_7,1 ); +if (fdir_id_extr !=FDID_MIS_MAGIC ) + rx_pkts[i +4 ]->hash.fdir.hi = fdir_id_extr; + + fdir_id_extr =_mm256_extract_epi32 (fdir_id0_7,5 ); +if (fdir_id_extr !=FDID_MIS_MAGIC ) + rx_pkts[i +5 ]->hash.fdir.hi = fdir_id_extr; + + fdir_id_extr =_mm256_extract_epi32 (fdir_id0_7,0 ); +if (fdir_id_extr !=FDID_MIS_MAGIC ) + rx_pkts[i +6 ]->hash.fdir.hi = fdir_id_extr; + + fdir_id_extr =_mm256_extract_epi32 (fdir_id0_7,4 ); +if (fdir_id_extr !=FDID_MIS_MAGIC ) + rx_pkts[i +7 ]->hash.fdir.hi = fdir_id_extr; + }/* if() on fdir_enabled */ +/** * At this point, we have the 8 sets of flags in the low 16-bits * of each 32-bit value in vlan0.
Signature · 1 line
2.17.1
PATCH v2 10/12 net/iavf: support flow mark in SSE path
In reply to Leyi RongSupport Flow Director mark ID parsing from Flex
Rx descriptor in SSE path.
--- drivers/net/iavf/iavf_rxtx_vec_sse.c | 58 +++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-)
drivers/net/iavf/iavf_rxtx_vec_sse.c +57 -1
@@ -189,6 +189,25 @@ desc_to_olflags_v(struct iavf_rx_queue *rxq, __m128i descs[4],_mm_store_si128 ((__m128i *)&rx_pkts[3 ]->rearm_data, rearm3); } +static inline __m128i +flex_rxd_to_fdir_flags_vec (const __m128i fdir_id0_3) +{ +#define FDID_MIS_MAGIC 0xFFFFFFFF +RTE_BUILD_BUG_ON (PKT_RX_FDIR != (1 <<2 )); +RTE_BUILD_BUG_ON (PKT_RX_FDIR_ID != (1 <<13 )); +const __m128i pkt_fdir_bit =_mm_set1_epi32 (PKT_RX_FDIR | +PKT_RX_FDIR_ID ); +/* desc->flow_id field == 0xFFFFFFFF means fdir mismatch */ +const __m128i fdir_mis_mask =_mm_set1_epi32 (FDID_MIS_MAGIC ); +__m128i fdir_mask =_mm_cmpeq_epi32 (fdir_id0_3, + fdir_mis_mask); +/* this XOR op results to bit-reverse the fdir_mask */ + fdir_mask =_mm_xor_si128 (fdir_mask, fdir_mis_mask); +const __m128i fdir_flags =_mm_and_si128 (fdir_mask, pkt_fdir_bit); + +return fdir_flags; +} +static inline void flex_desc_to_olflags_v (struct iavf_rx_queue *rxq,__m128i descs[4 ],struct rte_mbuf **rx_pkts)
@@ -267,6 +286,42 @@ flex_desc_to_olflags_v(struct iavf_rx_queue *rxq, __m128i descs[4],/* merge the flags */ flags =_mm_or_si128 (flags, rss_vlan); +if (rxq->vsi->adapter->fdir_enabled) { +const __m128i fdir_id0_1 = +_mm_unpackhi_epi32 (descs[0 ], descs[1 ]); + +const __m128i fdir_id2_3 = +_mm_unpackhi_epi32 (descs[2 ], descs[3 ]); + +const __m128i fdir_id0_3 = +_mm_unpackhi_epi64 (fdir_id0_1, fdir_id2_3); + +const __m128i fdir_flags = +flex_rxd_to_fdir_flags_vec (fdir_id0_3); + +/* merge with fdir_flags */ + flags =_mm_or_si128 (flags, fdir_flags); + +/* write fdir_id to mbuf if FDIR match */ +uint32_t fdir_id_extr; + + fdir_id_extr =_mm_extract_epi32 (fdir_id0_3,0 ); +if (fdir_id_extr !=FDID_MIS_MAGIC ) + rx_pkts[0 ]->hash.fdir.hi = fdir_id_extr; + + fdir_id_extr =_mm_extract_epi32 (fdir_id0_3,1 ); +if (fdir_id_extr !=FDID_MIS_MAGIC ) + rx_pkts[1 ]->hash.fdir.hi = fdir_id_extr; + + fdir_id_extr =_mm_extract_epi32 (fdir_id0_3,2 ); +if (fdir_id_extr !=FDID_MIS_MAGIC ) + rx_pkts[2 ]->hash.fdir.hi = fdir_id_extr; + + fdir_id_extr =_mm_extract_epi32 (fdir_id0_3,3 ); +if (fdir_id_extr !=FDID_MIS_MAGIC ) + rx_pkts[3 ]->hash.fdir.hi = fdir_id_extr; + }/* if() on fdir_enabled */ +/** * At this point, we have the 4 sets of flags in the low 16-bits * of each 32-bit value in flags.
@@ -604,7 +659,8 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,const __m128i zero =_mm_setzero_si128 ();/* mask to shuffle from desc. to mbuf */ const __m128i shuf_msk = _mm_set_epi8 - (15, 14, 13, 12, /* octet 12~15, 32 bits rss */ + (0xFF, 0xFF, + 0xFF, 0xFF, /* rss not supported */ 11, 10, /* octet 10~11, 16 bits vlan_macip */ 5, 4, /* octet 4~5, 16 bits data_len */ 0xFF, 0xFF, /* skip high 16 bits pkt_len, zero out */
Signature · 1 line
2.17.1
PATCH v2 11/12 net/iavf: add RSS hash parsing in AVX path
In reply to Leyi RongSupport RSS hash parsing from Flex Rx
descriptor in AVX data path.
--- drivers/net/iavf/iavf_rxtx_vec_avx2.c | 92 ++++++++++++++++++++++++++- 1 file changed, 90 insertions(+), 2 deletions(-)
drivers/net/iavf/iavf_rxtx_vec_avx2.c +90 -2
@@ -698,7 +698,7 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq, _mm256_set_epi8 (/* first descriptor */ 0xFF, 0xFF, - 0xFF, 0xFF, /* rss not supported */ + 0xFF, 0xFF, /* rss hash parsed separately */ 11, 10, /* octet 10~11, 16 bits vlan_macip */ 5, 4, /* octet 4~5, 16 bits data_len */ 0xFF, 0xFF, /* skip hi 16 bits pkt_len, zero out */
@@ -707,7 +707,7 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,0xFF ,0xFF ,/*pkt_type set as unknown */ /* second descriptor */ 0xFF ,0xFF , -0xFF ,0xFF ,/* rss not supported */ +0xFF ,0xFF ,/* rss hash parsed separately */ 11 ,10 ,/* octet 10~11, 16 bits vlan_macip */ 5 ,4 ,/* octet 4~5, 16 bits data_len */ 0xFF, 0xFF, /* skip hi 16 bits pkt_len, zero out */
@@ -1004,6 +1004,94 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq, rx_pkts[i +7 ]->hash.fdir.hi = fdir_id_extr; }/* if() on fdir_enabled */ +/** +* needs to load 2nd 16B of each desc for RSS hash parsing, +* will cause performance drop to get into this context. +*/ +if (rxq->vsi->adapter->eth_dev->data->dev_conf.rxmode.offloads & +DEV_RX_OFFLOAD_RSS_HASH ) { +/* load bottom half of every 32B desc */ +const __m128i raw_desc_bh7 = +_mm_load_si128 + ((void *)(&rxdp[7 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh6 = +_mm_load_si128 + ((void *)(&rxdp[6 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh5 = +_mm_load_si128 + ((void *)(&rxdp[5 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh4 = +_mm_load_si128 + ((void *)(&rxdp[4 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh3 = +_mm_load_si128 + ((void *)(&rxdp[3 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh2 = +_mm_load_si128 + ((void *)(&rxdp[2 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh1 = +_mm_load_si128 + ((void *)(&rxdp[1 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh0 = +_mm_load_si128 + ((void *)(&rxdp[0 ].wb.status_error1)); + +__m256i raw_desc_bh6_7 = +_mm256_inserti128_si256 + (_mm256_castsi128_si256 (raw_desc_bh6), + raw_desc_bh7,1 ); +__m256i raw_desc_bh4_5 = +_mm256_inserti128_si256 + (_mm256_castsi128_si256 (raw_desc_bh4), + raw_desc_bh5,1 ); +__m256i raw_desc_bh2_3 = +_mm256_inserti128_si256 + (_mm256_castsi128_si256 (raw_desc_bh2), + raw_desc_bh3,1 ); +__m256i raw_desc_bh0_1 = +_mm256_inserti128_si256 + (_mm256_castsi128_si256 (raw_desc_bh0), + raw_desc_bh1,1 ); + +/** +* to shift the 32b RSS hash value to the +* highest 32b of each 128b before mask +*/ +__m256i rss_hash6_7 = +_mm256_slli_epi64 (raw_desc_bh6_7,32 ); +__m256i rss_hash4_5 = +_mm256_slli_epi64 (raw_desc_bh4_5,32 ); +__m256i rss_hash2_3 = +_mm256_slli_epi64 (raw_desc_bh2_3,32 ); +__m256i rss_hash0_1 = +_mm256_slli_epi64 (raw_desc_bh0_1,32 ); + +__m256i rss_hash_msk = +_mm256_set_epi32 (0xFFFFFFFF ,0 ,0 ,0 , +0xFFFFFFFF ,0 ,0 ,0 ); + + rss_hash6_7 =_mm256_and_si256 + (rss_hash6_7, rss_hash_msk); + rss_hash4_5 =_mm256_and_si256 + (rss_hash4_5, rss_hash_msk); + rss_hash2_3 =_mm256_and_si256 + (rss_hash2_3, rss_hash_msk); + rss_hash0_1 =_mm256_and_si256 + (rss_hash0_1, rss_hash_msk); + + mb6_7 =_mm256_or_si256 (mb6_7, rss_hash6_7); + mb4_5 =_mm256_or_si256 (mb4_5, rss_hash4_5); + mb2_3 =_mm256_or_si256 (mb2_3, rss_hash2_3); + mb0_1 =_mm256_or_si256 (mb0_1, rss_hash0_1); + }/* if() on RSS hash parsing */ +/** * At this point, we have the 8 sets of flags in the low 16-bits * of each 32-bit value in vlan0.
Signature · 1 line
2.17.1
PATCH v2 12/12 net/iavf: add RSS hash parsing in SSE path
In reply to Leyi RongSupport RSS hash parsing from Flex Rx
descriptor in SSE data path.
--- drivers/net/iavf/iavf_rxtx_vec_sse.c | 86 ++++++++++++++++++++++------ 1 file changed, 70 insertions(+), 16 deletions(-)
drivers/net/iavf/iavf_rxtx_vec_sse.c +70 -16
@@ -660,7 +660,7 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,/* mask to shuffle from desc. to mbuf */ const __m128i shuf_msk = _mm_set_epi8 (0xFF, 0xFF, - 0xFF, 0xFF, /* rss not supported */ + 0xFF, 0xFF, /* rss hash parsed separately */ 11, 10, /* octet 10~11, 16 bits vlan_macip */ 5, 4, /* octet 4~5, 16 bits data_len */ 0xFF, 0xFF, /* skip high 16 bits pkt_len, zero out */
@@ -751,7 +751,7 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq, pos += IAVF_VPMD_DESCS_PER_LOOP, rxdp += IAVF_VPMD_DESCS_PER_LOOP) {__m128i descs[IAVF_VPMD_DESCS_PER_LOOP ]; -__m128i pkt_mb1, pkt_mb2, pkt_mb3, pkt_mb4; +__m128i pkt_mb0, pkt_mb1, pkt_mb2, pkt_mb3;__m128i staterr, sterr_tmp1, sterr_tmp2;/* 2 64 bit or 4 32 bit mbuf pointers in one XMM reg. */ __m128i mbp1;
@@ -797,8 +797,12 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,rte_compiler_barrier ();/* D.1 pkt 3,4 convert format from desc to pktmbuf */ - pkt_mb4 =_mm_shuffle_epi8 (descs[3 ], shuf_msk); - pkt_mb3 =_mm_shuffle_epi8 (descs[2 ], shuf_msk); + pkt_mb3 =_mm_shuffle_epi8 (descs[3 ], shuf_msk); + pkt_mb2 =_mm_shuffle_epi8 (descs[2 ], shuf_msk); + +/* D.1 pkt 1,2 convert format from desc to pktmbuf */ + pkt_mb1 =_mm_shuffle_epi8 (descs[1 ], shuf_msk); + pkt_mb0 =_mm_shuffle_epi8 (descs[0 ], shuf_msk);/* C.1 4=>2 filter staterr info only */ sterr_tmp2 =_mm_unpackhi_epi32 (descs[3 ], descs[2 ]);
@@ -808,12 +812,66 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,flex_desc_to_olflags_v (rxq, descs, &rx_pkts[pos]);/* D.2 pkt 3,4 set in_port/nb_seg and remove crc */ - pkt_mb4 =_mm_add_epi16 (pkt_mb4, crc_adjust); pkt_mb3 =_mm_add_epi16 (pkt_mb3, crc_adjust); + pkt_mb2 =_mm_add_epi16 (pkt_mb2, crc_adjust); -/* D.1 pkt 1,2 convert format from desc to pktmbuf */ - pkt_mb2 =_mm_shuffle_epi8 (descs[1 ], shuf_msk); - pkt_mb1 =_mm_shuffle_epi8 (descs[0 ], shuf_msk); +/* D.2 pkt 1,2 set in_port/nb_seg and remove crc */ + pkt_mb1 =_mm_add_epi16 (pkt_mb1, crc_adjust); + pkt_mb0 =_mm_add_epi16 (pkt_mb0, crc_adjust); + +/** +* needs to load 2nd 16B of each desc for RSS hash parsing, +* will cause performance drop to get into this context. +*/ +if (rxq->vsi->adapter->eth_dev->data->dev_conf.rxmode.offloads & +DEV_RX_OFFLOAD_RSS_HASH ) { +/* load bottom half of every 32B desc */ +const __m128i raw_desc_bh3 = +_mm_load_si128 + ((void *)(&rxdp[3 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh2 = +_mm_load_si128 + ((void *)(&rxdp[2 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh1 = +_mm_load_si128 + ((void *)(&rxdp[1 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh0 = +_mm_load_si128 + ((void *)(&rxdp[0 ].wb.status_error1)); + +/** +* to shift the 32b RSS hash value to the +* highest 32b of each 128b before mask +*/ +__m128i rss_hash3 = +_mm_slli_epi64 (raw_desc_bh3,32 ); +__m128i rss_hash2 = +_mm_slli_epi64 (raw_desc_bh2,32 ); +__m128i rss_hash1 = +_mm_slli_epi64 (raw_desc_bh1,32 ); +__m128i rss_hash0 = +_mm_slli_epi64 (raw_desc_bh0,32 ); + +__m128i rss_hash_msk = +_mm_set_epi32 (0xFFFFFFFF ,0 ,0 ,0 ); + + rss_hash3 =_mm_and_si128 + (rss_hash3, rss_hash_msk); + rss_hash2 =_mm_and_si128 + (rss_hash2, rss_hash_msk); + rss_hash1 =_mm_and_si128 + (rss_hash1, rss_hash_msk); + rss_hash0 =_mm_and_si128 + (rss_hash0, rss_hash_msk); + + pkt_mb3 =_mm_or_si128 (pkt_mb3, rss_hash3); + pkt_mb2 =_mm_or_si128 (pkt_mb2, rss_hash2); + pkt_mb1 =_mm_or_si128 (pkt_mb1, rss_hash1); + pkt_mb0 =_mm_or_si128 (pkt_mb0, rss_hash0); + }/* if() on RSS hash parsing */ /* C.2 get 4 pkts staterr value */ staterr =_mm_unpacklo_epi32 (sterr_tmp1, sterr_tmp2);
@@ -821,14 +879,10 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,/* D.3 copy final 3,4 data to rx_pkts */ _mm_storeu_si128 ((void *)&rx_pkts[pos +3 ]->rx_descriptor_fields1, - pkt_mb4); + pkt_mb3);_mm_storeu_si128 ((void *)&rx_pkts[pos +2 ]->rx_descriptor_fields1, - pkt_mb3); - -/* D.2 pkt 1,2 set in_port/nb_seg and remove crc */ - pkt_mb2 =_mm_add_epi16 (pkt_mb2, crc_adjust); - pkt_mb1 =_mm_add_epi16 (pkt_mb1, crc_adjust); + pkt_mb2);/* C* extract and record EOP bit */ if (split_packet) {
@@ -852,9 +906,9 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,/* D.3 copy final 1,2 data to rx_pkts */ _mm_storeu_si128 ((void *)&rx_pkts[pos +1 ]->rx_descriptor_fields1, - pkt_mb2); + pkt_mb1);_mm_storeu_si128 ((void *)&rx_pkts[pos]->rx_descriptor_fields1, - pkt_mb1); + pkt_mb0);flex_desc_to_ptype_v (descs, &rx_pkts[pos], ptype_tbl);/* C.4 calc avaialbe number of desc */ var =__builtin_popcountll (_mm_cvtsi128_si64 (staterr));
Signature · 1 line
2.17.1
v3 cover framework for advanced iAVF PMD
In reply to Leyi RongThis patchset enable framework for advanced iAVF, includes flexible descriptor support, FDIR mark id and RSS hash support.
---
v3:
- Remove patch to query DDP package info as it's specific to DCF.
- Specify fdir_ref_cnt as per VF value for enable/disable FDIR ID parse.
- Move fdir_enabled flag to per Queue value for cache benefit when access in vector routines.
- Store extracted flow_id value to mbuf without judgement to avoid branch speculation fail, which is benefit to performance.
v2:
- Revert macro RTE_LIBRTE_IAVF_16BYTE_RX_DESC deletion as it's defined in AVF spec.
- Typo RTE_LIBRTE_ICE_16BYTE_RX_DESC fix in iavf_rxtx.c.
- Move flex desc definitions into iavf_rxtx.h.
- Up to date to match with the latest version of virtchnl.h.
- Extract a new internal func iavf_update_rx_tail to call.
- Remove iavf_dev_rxq_count_flex_rxd()/iavf_dev_rx_desc_status_flex_rxd(), as the accompanying legacy ones can deal with the flex desc cases.
- Move rxq->rxdid assignment from iavf_configure_queues() to iavf_dev_rx_queue_setup().
- Unfold _mm_extract_epi32(fdir_id0_3, i) to fix build error when using GCC compile option -O0.
Leyi Rong (11):
net/iavf: flexible Rx descriptor definitions net/iavf: return error if opcode is mismatched net/iavf: flexible Rx descriptor support in normal path net/iavf: flexible Rx descriptor support in AVX path net/iavf: flexible Rx descriptor support in SSE path net/iavf: add flow director enabled switch value net/iavf: support flow mark in normal data path net/iavf: support flow mark in AVX path net/iavf: support flow mark in SSE path net/iavf: add RSS hash parsing in AVX path net/iavf: add RSS hash parsing in SSE path
drivers/net/iavf/iavf.h | 6 + drivers/net/iavf/iavf_ethdev.c | 8 + drivers/net/iavf/iavf_rxtx.c | 540 ++++++++++++++++++-- drivers/net/iavf/iavf_rxtx.h | 248 +++++++++ drivers/net/iavf/iavf_rxtx_vec_avx2.c | 700 +++++++++++++++++++++++++- drivers/net/iavf/iavf_rxtx_vec_sse.c | 518 +++++++++++++++++++ drivers/net/iavf/iavf_vchnl.c | 46 +- 7 files changed, 2017 insertions(+), 49 deletions(-)
Signature · 1 line
2.17.1
PATCH v3 1/11 net/iavf: flexible Rx descriptor definitions
Add definitions for flexible Rx descriptor structures and macros.
--- drivers/net/iavf/iavf_rxtx.h | 200 +++++++++++++++++++++++++++++++++++ 1 file changed, 200 insertions(+)
drivers/net/iavf/iavf_rxtx.h +200 -0
@@ -157,6 +157,206 @@ union iavf_tx_offload { }; }; +/* Rx Flex Descriptors +* These descriptors are used instead of the legacy version descriptors +*/ +union iavf_16b_rx_flex_desc { +struct { +__le64 pkt_addr;/* Packet buffer address */ +__le64 hdr_addr;/* Header buffer address */ +/* bit 0 of hdr_addr is DD bit */ + } read; +struct { +/* Qword 0 */ +u8 rxdid;/* descriptor builder profile ID */ +u8 mir_id_umb_cast;/* mirror=[5:0], umb=[7:6] */ +__le16 ptype_flex_flags0;/* ptype=[9:0], ff0=[15:10] */ +__le16 pkt_len;/* [15:14] are reserved */ +__le16 hdr_len_sph_flex_flags1;/* header=[10:0] */ +/* sph=[11:11] */ +/* ff1/ext=[15:12] */ + +/* Qword 1 */ +__le16 status_error0; +__le16 l2tag1; +__le16 flex_meta0; +__le16 flex_meta1; + } wb;/* writeback */ +}; + +union iavf_32b_rx_flex_desc { +struct { +__le64 pkt_addr;/* Packet buffer address */ +__le64 hdr_addr;/* Header buffer address */ +/* bit 0 of hdr_addr is DD bit */ +__le64 rsvd1; +__le64 rsvd2; + } read; +struct { +/* Qword 0 */ +u8 rxdid;/* descriptor builder profile ID */ +u8 mir_id_umb_cast;/* mirror=[5:0], umb=[7:6] */ +__le16 ptype_flex_flags0;/* ptype=[9:0], ff0=[15:10] */ +__le16 pkt_len;/* [15:14] are reserved */ +__le16 hdr_len_sph_flex_flags1;/* header=[10:0] */ +/* sph=[11:11] */ +/* ff1/ext=[15:12] */ + +/* Qword 1 */ +__le16 status_error0; +__le16 l2tag1; +__le16 flex_meta0; +__le16 flex_meta1; + +/* Qword 2 */ +__le16 status_error1; +u8 flex_flags2; +u8 time_stamp_low; +__le16 l2tag2_1st; +__le16 l2tag2_2nd; + +/* Qword 3 */ +__le16 flex_meta2; +__le16 flex_meta3; +union { +struct { +__le16 flex_meta4; +__le16 flex_meta5; + } flex; +__le32 ts_high; + } flex_ts; + } wb;/* writeback */ +}; + +/* Rx Flex Descriptor for Comms Package Profile +* RxDID Profile ID 16-21 +* Flex-field 0: RSS hash lower 16-bits +* Flex-field 1: RSS hash upper 16-bits +* Flex-field 2: Flow ID lower 16-bits +* Flex-field 3: Flow ID upper 16-bits +* Flex-field 4: AUX0 +* Flex-field 5: AUX1 +*/ +struct iavf_32b_rx_flex_desc_comms { +/* Qword 0 */ +u8 rxdid; +u8 mir_id_umb_cast; +__le16 ptype_flexi_flags0; +__le16 pkt_len; +__le16 hdr_len_sph_flex_flags1; + +/* Qword 1 */ +__le16 status_error0; +__le16 l2tag1; +__le32 rss_hash; + +/* Qword 2 */ +__le16 status_error1; +u8 flexi_flags2; +u8 ts_low; +__le16 l2tag2_1st; +__le16 l2tag2_2nd; + +/* Qword 3 */ +__le32 flow_id; +union { +struct { +__le16 aux0; +__le16 aux1; + } flex; +__le32 ts_high; + } flex_ts; +}; + +/* Rx Flex Descriptor for Comms Package Profile +* RxDID Profile ID 22-23 (swap Hash and FlowID) +* Flex-field 0: Flow ID lower 16-bits +* Flex-field 1: Flow ID upper 16-bits +* Flex-field 2: RSS hash lower 16-bits +* Flex-field 3: RSS hash upper 16-bits +* Flex-field 4: AUX0 +* Flex-field 5: AUX1 +*/ +struct iavf_32b_rx_flex_desc_comms_ovs { +/* Qword 0 */ +u8 rxdid; +u8 mir_id_umb_cast; +__le16 ptype_flexi_flags0; +__le16 pkt_len; +__le16 hdr_len_sph_flex_flags1; + +/* Qword 1 */ +__le16 status_error0; +__le16 l2tag1; +__le32 flow_id; + +/* Qword 2 */ +__le16 status_error1; +u8 flexi_flags2; +u8 ts_low; +__le16 l2tag2_1st; +__le16 l2tag2_2nd; + +/* Qword 3 */ +__le32 rss_hash; +union { +struct { +__le16 aux0; +__le16 aux1; + } flex; +__le32 ts_high; + } flex_ts; +}; + +/* Receive Flex Descriptor profile IDs: There are a total +* of 64 profiles where profile IDs 0/1 are for legacy; and +* profiles 2-63 are flex profiles that can be programmed +* with a specific metadata (profile 7 reserved for HW) +*/ +enum iavf_rxdid { +IAVF_RXDID_LEGACY_0 =0 , +IAVF_RXDID_LEGACY_1 =1 , +IAVF_RXDID_FLEX_NIC =2 , +IAVF_RXDID_FLEX_NIC_2 =6 , +IAVF_RXDID_HW =7 , +IAVF_RXDID_COMMS_GENERIC =16 , +IAVF_RXDID_COMMS_AUX_VLAN =17 , +IAVF_RXDID_COMMS_AUX_IPV4 =18 , +IAVF_RXDID_COMMS_AUX_IPV6 =19 , +IAVF_RXDID_COMMS_AUX_IPV6_FLOW =20 , +IAVF_RXDID_COMMS_AUX_TCP =21 , +IAVF_RXDID_COMMS_OVS_1 =22 , +IAVF_RXDID_COMMS_OVS_2 =23 , +IAVF_RXDID_LAST =63 , +}; + +enum iavf_rx_flex_desc_status_error_0_bits { +/* Note: These are predefined bit offsets */ +IAVF_RX_FLEX_DESC_STATUS0_DD_S =0 , +IAVF_RX_FLEX_DESC_STATUS0_EOF_S , +IAVF_RX_FLEX_DESC_STATUS0_HBO_S , +IAVF_RX_FLEX_DESC_STATUS0_L3L4P_S , +IAVF_RX_FLEX_DESC_STATUS0_XSUM_IPE_S , +IAVF_RX_FLEX_DESC_STATUS0_XSUM_L4E_S , +IAVF_RX_FLEX_DESC_STATUS0_XSUM_EIPE_S , +IAVF_RX_FLEX_DESC_STATUS0_XSUM_EUDPE_S , +IAVF_RX_FLEX_DESC_STATUS0_LPBK_S , +IAVF_RX_FLEX_DESC_STATUS0_IPV6EXADD_S , +IAVF_RX_FLEX_DESC_STATUS0_RXE_S , +IAVF_RX_FLEX_DESC_STATUS0_CRCP_S , +IAVF_RX_FLEX_DESC_STATUS0_RSS_VALID_S , +IAVF_RX_FLEX_DESC_STATUS0_L2TAG1P_S , +IAVF_RX_FLEX_DESC_STATUS0_XTRMD0_VALID_S , +IAVF_RX_FLEX_DESC_STATUS0_XTRMD1_VALID_S , +IAVF_RX_FLEX_DESC_STATUS0_LAST /* this entry must be last!!! */ +}; + +/* for iavf_32b_rx_flex_desc.ptype_flex_flags0 member */ +#define IAVF_RX_FLEX_DESC_PTYPE_M (0x3FF)/* 10-bits */ + +/* for iavf_32b_rx_flex_desc.pkt_len member */ +#define IAVF_RX_FLX_DESC_PKT_LEN_M (0x3FFF)/* 14-bits */ + int iavf_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, uint16_t nb_desc,
Signature · 1 line
2.17.1
PATCH v3 2/11 net/iavf: return error if opcode is mismatched
In reply to Leyi RongAdds error return when the opcode of read message is mismatched which is received from adminQ.
--- drivers/net/iavf/iavf_vchnl.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
drivers/net/iavf/iavf_vchnl.c +3 -1
@@ -52,9 +52,11 @@ iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len,PMD_DRV_LOG (DEBUG ,"AQ from pf carries opcode %u, retval %d" , opcode, vf->cmd_retval); -if (opcode != vf->pend_cmd) +if (opcode != vf->pend_cmd) {PMD_DRV_LOG (WARNING ,"command mismatch, expect %u, get %u" , vf->pend_cmd, opcode); +return IAVF_ERR_OPCODE_MISMATCH ; + }return IAVF_SUCCESS ; }
Signature · 1 line
2.17.1
PATCH v3 3/11 net/iavf: flexible Rx descriptor support in normal path
In reply to Leyi RongSupport flexible Rx descriptor format in normal
path of iAVF PMD.
--- drivers/net/iavf/iavf.h | 2 + drivers/net/iavf/iavf_ethdev.c | 8 + drivers/net/iavf/iavf_rxtx.c | 479 ++++++++++++++++++++++++++++++--- drivers/net/iavf/iavf_rxtx.h | 8 + drivers/net/iavf/iavf_vchnl.c | 42 ++- 5 files changed, 501 insertions(+), 38 deletions(-)
drivers/net/iavf/iavf.h +2 -0
@@ -97,6 +97,7 @@ struct iavf_info {struct virtchnl_version_info virtchnl_version;struct virtchnl_vf_resource *vf_res;/* VF resource */ struct virtchnl_vsi_resource *vsi_res;/* LAN VSI */ +uint64_t supported_rxdid;volatile enum virtchnl_ops pend_cmd;/* pending command not finished */ uint32_t cmd_retval;/* return value of the cmd response from PF */
@@ -225,6 +226,7 @@ int iavf_disable_queues(struct iavf_adapter *adapter);int iavf_configure_rss_lut (struct iavf_adapter *adapter);int iavf_configure_rss_key (struct iavf_adapter *adapter);int iavf_configure_queues (struct iavf_adapter *adapter); +int iavf_get_supported_rxdid (struct iavf_adapter *adapter);int iavf_config_irq_map (struct iavf_adapter *adapter);void iavf_add_del_all_mac_addr (struct iavf_adapter *adapter,bool add); int iavf_dev_link_update(struct rte_eth_dev *dev,
drivers/net/iavf/iavf_ethdev.c +8 -0
@@ -1236,6 +1236,14 @@ iavf_init_vf(struct rte_eth_dev *dev) goto err_rss; } } + +if (vf->vf_res->vf_cap_flags &VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC ) { +if (iavf_get_supported_rxdid (adapter) !=0 ) { +PMD_INIT_LOG (ERR ,"failed to do get supported rxdid" ); + goto err_rss; + } + } +return 0 ; err_rss:rte_free (vf->rss_key);
drivers/net/iavf/iavf_rxtx.c +442 -37
546 lines not on this page · raw message
drivers/net/iavf/iavf_rxtx.h +8 -0
@@ -62,6 +62,7 @@#define iavf_rx_desc iavf_16byte_rx_desc#else #define iavf_rx_desc iavf_32byte_rx_desc +#define iavf_rx_flex_desc iavf_32b_rx_flex_desc#endif struct iavf_rxq_ops {
@@ -87,6 +88,7 @@ struct iavf_rx_queue {struct rte_mbuf *pkt_first_seg;/* first segment of current packet */ struct rte_mbuf *pkt_last_seg;/* last segment of current packet */ struct rte_mbuf fake_mbuf;/* dummy mbuf */ +uint8_t rxdid;/* used for VPMD */ uint16_t rxrearm_nb;/* number of remaining to be re-armed */
@@ -379,9 +381,15 @@ void iavf_dev_tx_queue_release(void *txq);void iavf_stop_queues (struct rte_eth_dev *dev);uint16_t iavf_recv_pkts (void *rx_queue,struct rte_mbuf **rx_pkts,uint16_t nb_pkts); +uint16_t iavf_recv_pkts_flex_rxd (void *rx_queue, +struct rte_mbuf **rx_pkts, +uint16_t nb_pkts);uint16_t iavf_recv_scattered_pkts (void *rx_queue,struct rte_mbuf **rx_pkts,uint16_t nb_pkts); +uint16_t iavf_recv_scattered_pkts_flex_rxd (void *rx_queue, +struct rte_mbuf **rx_pkts, +uint16_t nb_pkts);uint16_t iavf_xmit_pkts (void *tx_queue,struct rte_mbuf **tx_pkts,uint16_t nb_pkts); uint16_t iavf_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
drivers/net/iavf/iavf_vchnl.c +41 -1
66 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH v3 4/11 net/iavf: flexible Rx descriptor support in AVX path
In reply to Leyi RongSupport flexible Rx descriptor format in AVX
path of iAVF PMD.
--- drivers/net/iavf/iavf_rxtx.c | 24 +- drivers/net/iavf/iavf_rxtx.h | 6 + drivers/net/iavf/iavf_rxtx_vec_avx2.c | 550 +++++++++++++++++++++++++- 3 files changed, 570 insertions(+), 10 deletions(-)
drivers/net/iavf/iavf_rxtx.c +18 -6
@@ -2081,16 +2081,28 @@ iavf_set_rx_function(struct rte_eth_dev *dev)"Using %sVector Scattered Rx (port %d)." , use_avx2 ?"avx2 " :"" , dev->data->port_id); - dev->rx_pkt_burst = use_avx2 ? - iavf_recv_scattered_pkts_vec_avx2 : - iavf_recv_scattered_pkts_vec; +if (vf->vf_res->vf_cap_flags & +VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC ) + dev->rx_pkt_burst = use_avx2 ? + iavf_recv_scattered_pkts_vec_avx2_flex_rxd : + iavf_recv_scattered_pkts_vec; +else + dev->rx_pkt_burst = use_avx2 ? + iavf_recv_scattered_pkts_vec_avx2 : + iavf_recv_scattered_pkts_vec; }else {PMD_DRV_LOG (DEBUG ,"Using %sVector Rx (port %d)." , use_avx2 ?"avx2 " :"" , dev->data->port_id); - dev->rx_pkt_burst = use_avx2 ? - iavf_recv_pkts_vec_avx2 : - iavf_recv_pkts_vec; +if (vf->vf_res->vf_cap_flags & +VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC ) + dev->rx_pkt_burst = use_avx2 ? + iavf_recv_pkts_vec_avx2_flex_rxd : + iavf_recv_pkts_vec; +else + dev->rx_pkt_burst = use_avx2 ? + iavf_recv_pkts_vec_avx2 : + iavf_recv_pkts_vec; }return ;
drivers/net/iavf/iavf_rxtx.h +6 -0
@@ -413,9 +413,15 @@ uint16_t iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts);uint16_t iavf_recv_pkts_vec_avx2 (void *rx_queue,struct rte_mbuf **rx_pkts,uint16_t nb_pkts); +uint16_t iavf_recv_pkts_vec_avx2_flex_rxd (void *rx_queue, +struct rte_mbuf **rx_pkts, +uint16_t nb_pkts);uint16_t iavf_recv_scattered_pkts_vec_avx2 (void *rx_queue,struct rte_mbuf **rx_pkts,uint16_t nb_pkts); +uint16_t iavf_recv_scattered_pkts_vec_avx2_flex_rxd (void *rx_queue, +struct rte_mbuf **rx_pkts, +uint16_t nb_pkts);uint16_t iavf_xmit_pkts_vec (void *tx_queue,struct rte_mbuf **tx_pkts,uint16_t nb_pkts); uint16_t iavf_xmit_pkts_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts,
drivers/net/iavf/iavf_rxtx_vec_avx2.c +546 -4
585 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH v3 5/11 net/iavf: flexible Rx descriptor support in SSE path
In reply to Leyi RongSupport flexible Rx descriptor format in SSE
path of iAVF PMD.
--- drivers/net/iavf/iavf_rxtx.c | 4 +- drivers/net/iavf/iavf_rxtx.h | 5 + drivers/net/iavf/iavf_rxtx_vec_sse.c | 414 +++++++++++++++++++++++++++ 3 files changed, 421 insertions(+), 2 deletions(-)
drivers/net/iavf/iavf_rxtx.c +2 -2
@@ -2085,7 +2085,7 @@ iavf_set_rx_function(struct rte_eth_dev *dev) VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) dev->rx_pkt_burst = use_avx2 ? iavf_recv_scattered_pkts_vec_avx2_flex_rxd : - iavf_recv_scattered_pkts_vec; + iavf_recv_scattered_pkts_vec_flex_rxd; else dev->rx_pkt_burst = use_avx2 ? iavf_recv_scattered_pkts_vec_avx2 :
@@ -2098,7 +2098,7 @@ iavf_set_rx_function(struct rte_eth_dev *dev) VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) dev->rx_pkt_burst = use_avx2 ? iavf_recv_pkts_vec_avx2_flex_rxd : - iavf_recv_pkts_vec; + iavf_recv_pkts_vec_flex_rxd; else dev->rx_pkt_burst = use_avx2 ? iavf_recv_pkts_vec_avx2 :
drivers/net/iavf/iavf_rxtx.h +5 -0
@@ -406,9 +406,14 @@ int iavf_dev_tx_desc_status(void *tx_queue, uint16_t offset);uint16_t iavf_recv_pkts_vec (void *rx_queue,struct rte_mbuf **rx_pkts,uint16_t nb_pkts); +uint16_t iavf_recv_pkts_vec_flex_rxd (void *rx_queue,struct rte_mbuf **rx_pkts, +uint16_t nb_pkts);uint16_t iavf_recv_scattered_pkts_vec (void *rx_queue,struct rte_mbuf **rx_pkts,uint16_t nb_pkts); +uint16_t iavf_recv_scattered_pkts_vec_flex_rxd (void *rx_queue, +struct rte_mbuf **rx_pkts, +uint16_t nb_pkts);uint16_t iavf_xmit_fixed_burst_vec (void *tx_queue,struct rte_mbuf **tx_pkts,uint16_t nb_pkts); uint16_t iavf_recv_pkts_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,
drivers/net/iavf/iavf_rxtx_vec_sse.c +414 -0
444 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH v3 6/11 net/iavf: add flow director enabled switch value
In reply to Leyi RongThe commit adds fdir_enabled flag into iavf_rx_queue structure to identify if fdir id is active. Rx data path can be benefit if fdir id parsing is not needed, especially in vector path.
--- drivers/net/iavf/iavf.h | 1 + drivers/net/iavf/iavf_rxtx.h | 29 +++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+)
drivers/net/iavf/iavf.h +1 -0
@@ -134,6 +134,7 @@ struct iavf_adapter {bool tx_vec_allowed;const uint32_t *ptype_tbl;bool stopped; +uint16_t fdir_ref_cnt; };/* IAVF_DEV_PRIVATE_TO */
drivers/net/iavf/iavf_rxtx.h +29 -0
@@ -103,6 +103,7 @@ struct iavf_rx_queue {uint16_t port_id;/* device port ID */ uint8_t crc_len;/* 0 if CRC stripped, 4 otherwise */ +uint8_t fdir_enabled;/* 0 if FDIR disabled, 1 when enabled */ uint16_t queue_id;/* Rx queue index */ uint16_t rx_buf_len;/* The packet buffer size */ uint16_t rx_hdr_len;/* The header buffer size */
@@ -490,6 +491,34 @@ void iavf_dump_tx_descriptor(const struct iavf_tx_queue *txq, tx_desc->cmd_type_offset_bsz); } +#define FDIR_PROC_ENABLE_PER_QUEUE (ad, on) do { \ + for (int i = 0; i < ad->eth_dev->data->nb_rx_queues; i++) { \ + struct iavf_rx_queue *rxq = ad->eth_dev->data->rx_queues[i]; \ + if (!rxq) \ + continue; \ + rxq->fdir_enabled = on; \ + } \ + PMD_DRV_LOG(DEBUG, "FDIR processing on RX set to %d", on); \ +} while (0) + +/* Enable/disable flow director Rx processing in data path. */ +static inline +void iavf_fdir_rx_proc_enable (struct iavf_adapter *ad,bool on) +{ +if (on) { +/* enable flow director processing */ +if (ad->fdir_ref_cnt++ ==0 ) +FDIR_PROC_ENABLE_PER_QUEUE (ad, on); + }else { +if (ad->fdir_ref_cnt >=1 ) { + ad->fdir_ref_cnt--; + +if (ad->fdir_ref_cnt ==0 ) +FDIR_PROC_ENABLE_PER_QUEUE (ad, on); + } + } +} +#ifdef RTE_LIBRTE_IAVF_DEBUG_DUMP_DESC #define IAVF_DUMP_RX_DESC (rxq, desc, rx_id) \ iavf_dump_rx_descriptor(rxq, desc, rx_id)
Signature · 1 line
2.17.1
PATCH v3 7/11 net/iavf: support flow mark in normal data path
In reply to Leyi RongSupport Flow Director mark ID parsing in normal path.
--- drivers/net/iavf/iavf.h | 3 +++ drivers/net/iavf/iavf_rxtx.c | 37 ++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+)
drivers/net/iavf/iavf.h +3 -0
@@ -67,6 +67,9 @@#define IAVF_48_BIT_WIDTH (CHAR_BIT * 6)#define IAVF_48_BIT_MASK RTE_LEN2MASK(IAVF_48_BIT_WIDTH, uint64_t) +#define IAVF_RX_DESC_EXT_STATUS_FLEXBH_MASK 0x03 +#define IAVF_RX_DESC_EXT_STATUS_FLEXBH_FD_ID 0x01 +struct iavf_adapter ;struct iavf_rx_queue ;struct iavf_tx_queue ;
drivers/net/iavf/iavf_rxtx.c +37 -0
@@ -756,6 +756,10 @@ iavf_rxd_to_pkt_flags(uint64_t qword) IAVF_RX_DESC_FLTSTAT_RSS_HASH) == IAVF_RX_DESC_FLTSTAT_RSS_HASH) ? PKT_RX_RSS_HASH : 0; +/* Check if FDIR Match */ + flags |= (qword & (1 <<IAVF_RX_DESC_STATUS_FLM_SHIFT ) ? +PKT_RX_FDIR :0 ); +if (likely ((error_bits &IAVF_RX_ERR_BITS ) ==0 )) { flags |= (PKT_RX_IP_CKSUM_GOOD |PKT_RX_L4_CKSUM_GOOD );return flags;
@@ -776,6 +780,25 @@ iavf_rxd_to_pkt_flags(uint64_t qword)return flags; } +static inline uint64_t +iavf_rxd_build_fdir (volatile union iavf_rx_desc *rxdp,struct rte_mbuf *mb) +{ +uint64_t flags =0 ; +uint16_t flexbh; + + flexbh = (rte_le_to_cpu_32 (rxdp->wb.qword2.ext_status) >> +IAVF_RX_DESC_EXT_STATUS_FLEXBH_SHIFT ) & +IAVF_RX_DESC_EXT_STATUS_FLEXBH_MASK ; + +if (flexbh ==IAVF_RX_DESC_EXT_STATUS_FLEXBH_FD_ID ) { + mb->hash.fdir.hi = +rte_le_to_cpu_32 (rxdp->wb.qword3.hi_dword.fd_id); + flags |=PKT_RX_FDIR_ID ; + } + +return flags; +} +/* Translate the rx flex descriptor status to pkt flags */ static inline void iavf_rxd_to_pkt_fields(struct rte_mbuf *mb,
@@ -792,6 +815,11 @@ iavf_rxd_to_pkt_fields(struct rte_mbuf *mb, mb->hash.rss = rte_le_to_cpu_32(desc->rss_hash); }#endif + +if (desc->flow_id !=0xFFFFFFFF ) { + mb->ol_flags |=PKT_RX_FDIR |PKT_RX_FDIR_ID ; + mb->hash.fdir.hi =rte_le_to_cpu_32 (desc->flow_id); + } }#define IAVF_RX_FLEX_ERR0_BITS \
@@ -951,6 +979,9 @@ iavf_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) rxm->hash.rss =rte_le_to_cpu_32 (rxd.wb.qword0.hi_dword.rss); +if (pkt_flags &PKT_RX_FDIR ) + pkt_flags |=iavf_rxd_build_fdir (&rxd, rxm); + rxm->ol_flags |= pkt_flags; rx_pkts[nb_rx++] = rxm;
@@ -1349,6 +1380,9 @@ iavf_recv_scattered_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, first_seg->hash.rss =rte_le_to_cpu_32 (rxd.wb.qword0.hi_dword.rss); +if (pkt_flags &PKT_RX_FDIR ) + pkt_flags |=iavf_rxd_build_fdir (&rxd, first_seg); + first_seg->ol_flags |= pkt_flags;/* Prefetch data of first segment, if configured to do so. */
@@ -1515,6 +1549,9 @@ iavf_rx_scan_hw_ring(struct iavf_rx_queue *rxq) mb->hash.rss =rte_le_to_cpu_32 ( rxdp[j].wb.qword0.hi_dword.rss); +if (pkt_flags &PKT_RX_FDIR ) + pkt_flags |=iavf_rxd_build_fdir (&rxdp[j], mb); + mb->ol_flags |= pkt_flags; }
Signature · 1 line
2.17.1
PATCH v3 8/11 net/iavf: support flow mark in AVX path
In reply to Leyi RongSupport Flow Director mark ID parsing from Flex
Rx descriptor in AVX path.
--- drivers/net/iavf/iavf_rxtx_vec_avx2.c | 72 +++++++++++++++++++++++++-- 1 file changed, 67 insertions(+), 5 deletions(-)
drivers/net/iavf/iavf_rxtx_vec_avx2.c +67 -5
@@ -616,6 +616,25 @@ _iavf_recv_raw_pkts_vec_avx2(struct iavf_rx_queue *rxq,return received; } +static inline __m256i +flex_rxd_to_fdir_flags_vec_avx2 (const __m256i fdir_id0_7) +{ +#define FDID_MIS_MAGIC 0xFFFFFFFF +RTE_BUILD_BUG_ON (PKT_RX_FDIR != (1 <<2 )); +RTE_BUILD_BUG_ON (PKT_RX_FDIR_ID != (1 <<13 )); +const __m256i pkt_fdir_bit =_mm256_set1_epi32 (PKT_RX_FDIR | +PKT_RX_FDIR_ID ); +/* desc->flow_id field == 0xFFFFFFFF means fdir mismatch */ +const __m256i fdir_mis_mask =_mm256_set1_epi32 (FDID_MIS_MAGIC ); +__m256i fdir_mask =_mm256_cmpeq_epi32 (fdir_id0_7, + fdir_mis_mask); +/* this XOR op results to bit-reverse the fdir_mask */ + fdir_mask =_mm256_xor_si256 (fdir_mask, fdir_mis_mask); +const __m256i fdir_flags =_mm256_and_si256 (fdir_mask, pkt_fdir_bit); + +return fdir_flags; +} + static inline uint16_t _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq, struct rte_mbuf **rx_pkts,
@@ -678,8 +697,8 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,const __m256i shuf_msk = _mm256_set_epi8 (/* first descriptor */ - 15, 14, - 13, 12, /* octet 12~15, 32 bits rss */ + 0xFF, 0xFF, + 0xFF, 0xFF, /* rss not supported */ 11, 10, /* octet 10~11, 16 bits vlan_macip */ 5, 4, /* octet 4~5, 16 bits data_len */ 0xFF, 0xFF, /* skip hi 16 bits pkt_len, zero out */
@@ -687,8 +706,8 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,0xFF ,0xFF ,/* pkt_type set as unknown */ 0xFF ,0xFF ,/*pkt_type set as unknown */ /* second descriptor */ -15 ,14 , -13 ,12 ,/* octet 12~15, 32 bits rss */ +0xFF ,0xFF , +0xFF ,0xFF ,/* rss not supported */ 11 ,10 ,/* octet 10~11, 16 bits vlan_macip */ 5 ,4 ,/* octet 4~5, 16 bits data_len */ 0xFF, 0xFF, /* skip hi 16 bits pkt_len, zero out */
@@ -930,8 +949,51 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq, rss_vlan_flag_bits);/* merge flags */ -const __m256i mbuf_flags =_mm256_or_si256 (l3_l4_flags, +__m256i mbuf_flags =_mm256_or_si256 (l3_l4_flags, rss_vlan_flags); + +if (rxq->fdir_enabled) { +const __m256i fdir_id4_7 = +_mm256_unpackhi_epi32 (raw_desc6_7, raw_desc4_5); + +const __m256i fdir_id0_3 = +_mm256_unpackhi_epi32 (raw_desc2_3, raw_desc0_1); + +const __m256i fdir_id0_7 = +_mm256_unpackhi_epi64 (fdir_id4_7, fdir_id0_3); + +const __m256i fdir_flags = +flex_rxd_to_fdir_flags_vec_avx2 (fdir_id0_7); + +/* merge with fdir_flags */ + mbuf_flags =_mm256_or_si256 (mbuf_flags, fdir_flags); + +/* write to mbuf: have to use scalar store here */ + rx_pkts[i +0 ]->hash.fdir.hi = +_mm256_extract_epi32 (fdir_id0_7,3 ); + + rx_pkts[i +1 ]->hash.fdir.hi = +_mm256_extract_epi32 (fdir_id0_7,7 ); + + rx_pkts[i +2 ]->hash.fdir.hi = +_mm256_extract_epi32 (fdir_id0_7,2 ); + + rx_pkts[i +3 ]->hash.fdir.hi = +_mm256_extract_epi32 (fdir_id0_7,6 ); + + rx_pkts[i +4 ]->hash.fdir.hi = +_mm256_extract_epi32 (fdir_id0_7,1 ); + + rx_pkts[i +5 ]->hash.fdir.hi = +_mm256_extract_epi32 (fdir_id0_7,5 ); + + rx_pkts[i +6 ]->hash.fdir.hi = +_mm256_extract_epi32 (fdir_id0_7,0 ); + + rx_pkts[i +7 ]->hash.fdir.hi = +_mm256_extract_epi32 (fdir_id0_7,4 ); + }/* if() on fdir_enabled */ +/** * At this point, we have the 8 sets of flags in the low 16-bits * of each 32-bit value in vlan0.
Signature · 1 line
2.17.1
PATCH v3 9/11 net/iavf: support flow mark in SSE path
In reply to Leyi RongSupport Flow Director mark ID parsing from Flex
Rx descriptor in SSE path.
--- drivers/net/iavf/iavf_rxtx_vec_sse.c | 52 +++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-)
drivers/net/iavf/iavf_rxtx_vec_sse.c +51 -1
@@ -189,6 +189,25 @@ desc_to_olflags_v(struct iavf_rx_queue *rxq, __m128i descs[4],_mm_store_si128 ((__m128i *)&rx_pkts[3 ]->rearm_data, rearm3); } +static inline __m128i +flex_rxd_to_fdir_flags_vec (const __m128i fdir_id0_3) +{ +#define FDID_MIS_MAGIC 0xFFFFFFFF +RTE_BUILD_BUG_ON (PKT_RX_FDIR != (1 <<2 )); +RTE_BUILD_BUG_ON (PKT_RX_FDIR_ID != (1 <<13 )); +const __m128i pkt_fdir_bit =_mm_set1_epi32 (PKT_RX_FDIR | +PKT_RX_FDIR_ID ); +/* desc->flow_id field == 0xFFFFFFFF means fdir mismatch */ +const __m128i fdir_mis_mask =_mm_set1_epi32 (FDID_MIS_MAGIC ); +__m128i fdir_mask =_mm_cmpeq_epi32 (fdir_id0_3, + fdir_mis_mask); +/* this XOR op results to bit-reverse the fdir_mask */ + fdir_mask =_mm_xor_si128 (fdir_mask, fdir_mis_mask); +const __m128i fdir_flags =_mm_and_si128 (fdir_mask, pkt_fdir_bit); + +return fdir_flags; +} +static inline void flex_desc_to_olflags_v (struct iavf_rx_queue *rxq,__m128i descs[4 ],struct rte_mbuf **rx_pkts)
@@ -267,6 +286,36 @@ flex_desc_to_olflags_v(struct iavf_rx_queue *rxq, __m128i descs[4],/* merge the flags */ flags =_mm_or_si128 (flags, rss_vlan); +if (rxq->fdir_enabled) { +const __m128i fdir_id0_1 = +_mm_unpackhi_epi32 (descs[0 ], descs[1 ]); + +const __m128i fdir_id2_3 = +_mm_unpackhi_epi32 (descs[2 ], descs[3 ]); + +const __m128i fdir_id0_3 = +_mm_unpackhi_epi64 (fdir_id0_1, fdir_id2_3); + +const __m128i fdir_flags = +flex_rxd_to_fdir_flags_vec (fdir_id0_3); + +/* merge with fdir_flags */ + flags =_mm_or_si128 (flags, fdir_flags); + +/* write fdir_id to mbuf */ + rx_pkts[0 ]->hash.fdir.hi = +_mm_extract_epi32 (fdir_id0_3,0 ); + + rx_pkts[1 ]->hash.fdir.hi = +_mm_extract_epi32 (fdir_id0_3,1 ); + + rx_pkts[2 ]->hash.fdir.hi = +_mm_extract_epi32 (fdir_id0_3,2 ); + + rx_pkts[3 ]->hash.fdir.hi = +_mm_extract_epi32 (fdir_id0_3,3 ); + }/* if() on fdir_enabled */ +/** * At this point, we have the 4 sets of flags in the low 16-bits * of each 32-bit value in flags.
@@ -604,7 +653,8 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,const __m128i zero =_mm_setzero_si128 ();/* mask to shuffle from desc. to mbuf */ const __m128i shuf_msk = _mm_set_epi8 - (15, 14, 13, 12, /* octet 12~15, 32 bits rss */ + (0xFF, 0xFF, + 0xFF, 0xFF, /* rss not supported */ 11, 10, /* octet 10~11, 16 bits vlan_macip */ 5, 4, /* octet 4~5, 16 bits data_len */ 0xFF, 0xFF, /* skip high 16 bits pkt_len, zero out */
Signature · 1 line
2.17.1
PATCH v3 10/11 net/iavf: add RSS hash parsing in AVX path
In reply to Leyi RongSupport RSS hash parsing from Flex Rx
descriptor in AVX data path.
--- drivers/net/iavf/iavf_rxtx_vec_avx2.c | 92 ++++++++++++++++++++++++++- 1 file changed, 90 insertions(+), 2 deletions(-)
drivers/net/iavf/iavf_rxtx_vec_avx2.c +90 -2
@@ -698,7 +698,7 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq, _mm256_set_epi8 (/* first descriptor */ 0xFF, 0xFF, - 0xFF, 0xFF, /* rss not supported */ + 0xFF, 0xFF, /* rss hash parsed separately */ 11, 10, /* octet 10~11, 16 bits vlan_macip */ 5, 4, /* octet 4~5, 16 bits data_len */ 0xFF, 0xFF, /* skip hi 16 bits pkt_len, zero out */
@@ -707,7 +707,7 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,0xFF ,0xFF ,/*pkt_type set as unknown */ /* second descriptor */ 0xFF ,0xFF , -0xFF ,0xFF ,/* rss not supported */ +0xFF ,0xFF ,/* rss hash parsed separately */ 11 ,10 ,/* octet 10~11, 16 bits vlan_macip */ 5 ,4 ,/* octet 4~5, 16 bits data_len */ 0xFF, 0xFF, /* skip hi 16 bits pkt_len, zero out */
@@ -994,6 +994,94 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,_mm256_extract_epi32 (fdir_id0_7,4 ); }/* if() on fdir_enabled */ +/** +* needs to load 2nd 16B of each desc for RSS hash parsing, +* will cause performance drop to get into this context. +*/ +if (rxq->vsi->adapter->eth_dev->data->dev_conf.rxmode.offloads & +DEV_RX_OFFLOAD_RSS_HASH ) { +/* load bottom half of every 32B desc */ +const __m128i raw_desc_bh7 = +_mm_load_si128 + ((void *)(&rxdp[7 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh6 = +_mm_load_si128 + ((void *)(&rxdp[6 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh5 = +_mm_load_si128 + ((void *)(&rxdp[5 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh4 = +_mm_load_si128 + ((void *)(&rxdp[4 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh3 = +_mm_load_si128 + ((void *)(&rxdp[3 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh2 = +_mm_load_si128 + ((void *)(&rxdp[2 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh1 = +_mm_load_si128 + ((void *)(&rxdp[1 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh0 = +_mm_load_si128 + ((void *)(&rxdp[0 ].wb.status_error1)); + +__m256i raw_desc_bh6_7 = +_mm256_inserti128_si256 + (_mm256_castsi128_si256 (raw_desc_bh6), + raw_desc_bh7,1 ); +__m256i raw_desc_bh4_5 = +_mm256_inserti128_si256 + (_mm256_castsi128_si256 (raw_desc_bh4), + raw_desc_bh5,1 ); +__m256i raw_desc_bh2_3 = +_mm256_inserti128_si256 + (_mm256_castsi128_si256 (raw_desc_bh2), + raw_desc_bh3,1 ); +__m256i raw_desc_bh0_1 = +_mm256_inserti128_si256 + (_mm256_castsi128_si256 (raw_desc_bh0), + raw_desc_bh1,1 ); + +/** +* to shift the 32b RSS hash value to the +* highest 32b of each 128b before mask +*/ +__m256i rss_hash6_7 = +_mm256_slli_epi64 (raw_desc_bh6_7,32 ); +__m256i rss_hash4_5 = +_mm256_slli_epi64 (raw_desc_bh4_5,32 ); +__m256i rss_hash2_3 = +_mm256_slli_epi64 (raw_desc_bh2_3,32 ); +__m256i rss_hash0_1 = +_mm256_slli_epi64 (raw_desc_bh0_1,32 ); + +__m256i rss_hash_msk = +_mm256_set_epi32 (0xFFFFFFFF ,0 ,0 ,0 , +0xFFFFFFFF ,0 ,0 ,0 ); + + rss_hash6_7 =_mm256_and_si256 + (rss_hash6_7, rss_hash_msk); + rss_hash4_5 =_mm256_and_si256 + (rss_hash4_5, rss_hash_msk); + rss_hash2_3 =_mm256_and_si256 + (rss_hash2_3, rss_hash_msk); + rss_hash0_1 =_mm256_and_si256 + (rss_hash0_1, rss_hash_msk); + + mb6_7 =_mm256_or_si256 (mb6_7, rss_hash6_7); + mb4_5 =_mm256_or_si256 (mb4_5, rss_hash4_5); + mb2_3 =_mm256_or_si256 (mb2_3, rss_hash2_3); + mb0_1 =_mm256_or_si256 (mb0_1, rss_hash0_1); + }/* if() on RSS hash parsing */ +/** * At this point, we have the 8 sets of flags in the low 16-bits * of each 32-bit value in vlan0.
Signature · 1 line
2.17.1
PATCH v3 11/11 net/iavf: add RSS hash parsing in SSE path
In reply to Leyi RongSupport RSS hash parsing from Flex Rx
descriptor in SSE data path.
--- drivers/net/iavf/iavf_rxtx_vec_sse.c | 86 ++++++++++++++++++++++------ 1 file changed, 70 insertions(+), 16 deletions(-)
drivers/net/iavf/iavf_rxtx_vec_sse.c +70 -16
@@ -654,7 +654,7 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,/* mask to shuffle from desc. to mbuf */ const __m128i shuf_msk = _mm_set_epi8 (0xFF, 0xFF, - 0xFF, 0xFF, /* rss not supported */ + 0xFF, 0xFF, /* rss hash parsed separately */ 11, 10, /* octet 10~11, 16 bits vlan_macip */ 5, 4, /* octet 4~5, 16 bits data_len */ 0xFF, 0xFF, /* skip high 16 bits pkt_len, zero out */
@@ -745,7 +745,7 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq, pos += IAVF_VPMD_DESCS_PER_LOOP, rxdp += IAVF_VPMD_DESCS_PER_LOOP) {__m128i descs[IAVF_VPMD_DESCS_PER_LOOP ]; -__m128i pkt_mb1, pkt_mb2, pkt_mb3, pkt_mb4; +__m128i pkt_mb0, pkt_mb1, pkt_mb2, pkt_mb3;__m128i staterr, sterr_tmp1, sterr_tmp2;/* 2 64 bit or 4 32 bit mbuf pointers in one XMM reg. */ __m128i mbp1;
@@ -791,8 +791,12 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,rte_compiler_barrier ();/* D.1 pkt 3,4 convert format from desc to pktmbuf */ - pkt_mb4 =_mm_shuffle_epi8 (descs[3 ], shuf_msk); - pkt_mb3 =_mm_shuffle_epi8 (descs[2 ], shuf_msk); + pkt_mb3 =_mm_shuffle_epi8 (descs[3 ], shuf_msk); + pkt_mb2 =_mm_shuffle_epi8 (descs[2 ], shuf_msk); + +/* D.1 pkt 1,2 convert format from desc to pktmbuf */ + pkt_mb1 =_mm_shuffle_epi8 (descs[1 ], shuf_msk); + pkt_mb0 =_mm_shuffle_epi8 (descs[0 ], shuf_msk);/* C.1 4=>2 filter staterr info only */ sterr_tmp2 =_mm_unpackhi_epi32 (descs[3 ], descs[2 ]);
@@ -802,12 +806,66 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,flex_desc_to_olflags_v (rxq, descs, &rx_pkts[pos]);/* D.2 pkt 3,4 set in_port/nb_seg and remove crc */ - pkt_mb4 =_mm_add_epi16 (pkt_mb4, crc_adjust); pkt_mb3 =_mm_add_epi16 (pkt_mb3, crc_adjust); + pkt_mb2 =_mm_add_epi16 (pkt_mb2, crc_adjust); -/* D.1 pkt 1,2 convert format from desc to pktmbuf */ - pkt_mb2 =_mm_shuffle_epi8 (descs[1 ], shuf_msk); - pkt_mb1 =_mm_shuffle_epi8 (descs[0 ], shuf_msk); +/* D.2 pkt 1,2 set in_port/nb_seg and remove crc */ + pkt_mb1 =_mm_add_epi16 (pkt_mb1, crc_adjust); + pkt_mb0 =_mm_add_epi16 (pkt_mb0, crc_adjust); + +/** +* needs to load 2nd 16B of each desc for RSS hash parsing, +* will cause performance drop to get into this context. +*/ +if (rxq->vsi->adapter->eth_dev->data->dev_conf.rxmode.offloads & +DEV_RX_OFFLOAD_RSS_HASH ) { +/* load bottom half of every 32B desc */ +const __m128i raw_desc_bh3 = +_mm_load_si128 + ((void *)(&rxdp[3 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh2 = +_mm_load_si128 + ((void *)(&rxdp[2 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh1 = +_mm_load_si128 + ((void *)(&rxdp[1 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh0 = +_mm_load_si128 + ((void *)(&rxdp[0 ].wb.status_error1)); + +/** +* to shift the 32b RSS hash value to the +* highest 32b of each 128b before mask +*/ +__m128i rss_hash3 = +_mm_slli_epi64 (raw_desc_bh3,32 ); +__m128i rss_hash2 = +_mm_slli_epi64 (raw_desc_bh2,32 ); +__m128i rss_hash1 = +_mm_slli_epi64 (raw_desc_bh1,32 ); +__m128i rss_hash0 = +_mm_slli_epi64 (raw_desc_bh0,32 ); + +__m128i rss_hash_msk = +_mm_set_epi32 (0xFFFFFFFF ,0 ,0 ,0 ); + + rss_hash3 =_mm_and_si128 + (rss_hash3, rss_hash_msk); + rss_hash2 =_mm_and_si128 + (rss_hash2, rss_hash_msk); + rss_hash1 =_mm_and_si128 + (rss_hash1, rss_hash_msk); + rss_hash0 =_mm_and_si128 + (rss_hash0, rss_hash_msk); + + pkt_mb3 =_mm_or_si128 (pkt_mb3, rss_hash3); + pkt_mb2 =_mm_or_si128 (pkt_mb2, rss_hash2); + pkt_mb1 =_mm_or_si128 (pkt_mb1, rss_hash1); + pkt_mb0 =_mm_or_si128 (pkt_mb0, rss_hash0); + }/* if() on RSS hash parsing */ /* C.2 get 4 pkts staterr value */ staterr =_mm_unpacklo_epi32 (sterr_tmp1, sterr_tmp2);
@@ -815,14 +873,10 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,/* D.3 copy final 3,4 data to rx_pkts */ _mm_storeu_si128 ((void *)&rx_pkts[pos +3 ]->rx_descriptor_fields1, - pkt_mb4); + pkt_mb3);_mm_storeu_si128 ((void *)&rx_pkts[pos +2 ]->rx_descriptor_fields1, - pkt_mb3); - -/* D.2 pkt 1,2 set in_port/nb_seg and remove crc */ - pkt_mb2 =_mm_add_epi16 (pkt_mb2, crc_adjust); - pkt_mb1 =_mm_add_epi16 (pkt_mb1, crc_adjust); + pkt_mb2);/* C* extract and record EOP bit */ if (split_packet) {
@@ -846,9 +900,9 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,/* D.3 copy final 1,2 data to rx_pkts */ _mm_storeu_si128 ((void *)&rx_pkts[pos +1 ]->rx_descriptor_fields1, - pkt_mb2); + pkt_mb1);_mm_storeu_si128 ((void *)&rx_pkts[pos]->rx_descriptor_fields1, - pkt_mb1); + pkt_mb0);flex_desc_to_ptype_v (descs, &rx_pkts[pos], ptype_tbl);/* C.4 calc avaialbe number of desc */ var =__builtin_popcountll (_mm_cvtsi128_si64 (staterr));
Signature · 1 line
2.17.1
v4 cover framework for advanced iAVF PMD
In reply to Leyi RongThis patchset enable framework for advanced iAVF, includes flexible descriptor support, FDIR mark id and RSS hash support.
---
v4:
- Avoid initial declaration in 'for' loop to backward-compatible with older gcc versions.
v3:
- Remove patch to query DDP package info as it's specific to DCF.
- Specify fdir_ref_cnt as per VF value for enable/disable FDIR ID parse.
- Move fdir_enabled flag to per Queue value for cache benefit when access in vector routines.
- Store extracted flow_id value to mbuf without judgement to avoid branch speculation fail, which is benefit to performance.
v2:
- Revert macro RTE_LIBRTE_IAVF_16BYTE_RX_DESC deletion as it's defined in AVF spec.
- Typo RTE_LIBRTE_ICE_16BYTE_RX_DESC fix in iavf_rxtx.c.
- Move flex desc definitions into iavf_rxtx.h.
- Up to date to match with the latest version of virtchnl.h.
- Extract a new internal func iavf_update_rx_tail to call.
- Remove iavf_dev_rxq_count_flex_rxd()/iavf_dev_rx_desc_status_flex_rxd(), as the accompanying legacy ones can deal with the flex desc cases.
- Move rxq->rxdid assignment from iavf_configure_queues() to iavf_dev_rx_queue_setup().
- Unfold _mm_extract_epi32(fdir_id0_3, i) to fix build error when using GCC compile option -O0.
Leyi Rong (11):
net/iavf: flexible Rx descriptor definitions net/iavf: return error if opcode is mismatched net/iavf: flexible Rx descriptor support in normal path net/iavf: flexible Rx descriptor support in AVX path net/iavf: flexible Rx descriptor support in SSE path net/iavf: add flow director enabled switch value net/iavf: support flow mark in normal data path net/iavf: support flow mark in AVX path net/iavf: support flow mark in SSE path net/iavf: add RSS hash parsing in AVX path net/iavf: add RSS hash parsing in SSE path
drivers/net/iavf/iavf.h | 6 + drivers/net/iavf/iavf_ethdev.c | 8 + drivers/net/iavf/iavf_rxtx.c | 540 ++++++++++++++++++-- drivers/net/iavf/iavf_rxtx.h | 249 +++++++++ drivers/net/iavf/iavf_rxtx_vec_avx2.c | 700 +++++++++++++++++++++++++- drivers/net/iavf/iavf_rxtx_vec_sse.c | 518 +++++++++++++++++++ drivers/net/iavf/iavf_vchnl.c | 46 +- 7 files changed, 2018 insertions(+), 49 deletions(-)
Signature · 1 line
2.17.1
PATCH v4 1/11 net/iavf: flexible Rx descriptor definitions
Add definitions for flexible Rx descriptor structures and macros.
--- drivers/net/iavf/iavf_rxtx.h | 200 +++++++++++++++++++++++++++++++++++ 1 file changed, 200 insertions(+)
drivers/net/iavf/iavf_rxtx.h +200 -0
@@ -157,6 +157,206 @@ union iavf_tx_offload { }; }; +/* Rx Flex Descriptors +* These descriptors are used instead of the legacy version descriptors +*/ +union iavf_16b_rx_flex_desc { +struct { +__le64 pkt_addr;/* Packet buffer address */ +__le64 hdr_addr;/* Header buffer address */ +/* bit 0 of hdr_addr is DD bit */ + } read; +struct { +/* Qword 0 */ +u8 rxdid;/* descriptor builder profile ID */ +u8 mir_id_umb_cast;/* mirror=[5:0], umb=[7:6] */ +__le16 ptype_flex_flags0;/* ptype=[9:0], ff0=[15:10] */ +__le16 pkt_len;/* [15:14] are reserved */ +__le16 hdr_len_sph_flex_flags1;/* header=[10:0] */ +/* sph=[11:11] */ +/* ff1/ext=[15:12] */ + +/* Qword 1 */ +__le16 status_error0; +__le16 l2tag1; +__le16 flex_meta0; +__le16 flex_meta1; + } wb;/* writeback */ +}; + +union iavf_32b_rx_flex_desc { +struct { +__le64 pkt_addr;/* Packet buffer address */ +__le64 hdr_addr;/* Header buffer address */ +/* bit 0 of hdr_addr is DD bit */ +__le64 rsvd1; +__le64 rsvd2; + } read; +struct { +/* Qword 0 */ +u8 rxdid;/* descriptor builder profile ID */ +u8 mir_id_umb_cast;/* mirror=[5:0], umb=[7:6] */ +__le16 ptype_flex_flags0;/* ptype=[9:0], ff0=[15:10] */ +__le16 pkt_len;/* [15:14] are reserved */ +__le16 hdr_len_sph_flex_flags1;/* header=[10:0] */ +/* sph=[11:11] */ +/* ff1/ext=[15:12] */ + +/* Qword 1 */ +__le16 status_error0; +__le16 l2tag1; +__le16 flex_meta0; +__le16 flex_meta1; + +/* Qword 2 */ +__le16 status_error1; +u8 flex_flags2; +u8 time_stamp_low; +__le16 l2tag2_1st; +__le16 l2tag2_2nd; + +/* Qword 3 */ +__le16 flex_meta2; +__le16 flex_meta3; +union { +struct { +__le16 flex_meta4; +__le16 flex_meta5; + } flex; +__le32 ts_high; + } flex_ts; + } wb;/* writeback */ +}; + +/* Rx Flex Descriptor for Comms Package Profile +* RxDID Profile ID 16-21 +* Flex-field 0: RSS hash lower 16-bits +* Flex-field 1: RSS hash upper 16-bits +* Flex-field 2: Flow ID lower 16-bits +* Flex-field 3: Flow ID upper 16-bits +* Flex-field 4: AUX0 +* Flex-field 5: AUX1 +*/ +struct iavf_32b_rx_flex_desc_comms { +/* Qword 0 */ +u8 rxdid; +u8 mir_id_umb_cast; +__le16 ptype_flexi_flags0; +__le16 pkt_len; +__le16 hdr_len_sph_flex_flags1; + +/* Qword 1 */ +__le16 status_error0; +__le16 l2tag1; +__le32 rss_hash; + +/* Qword 2 */ +__le16 status_error1; +u8 flexi_flags2; +u8 ts_low; +__le16 l2tag2_1st; +__le16 l2tag2_2nd; + +/* Qword 3 */ +__le32 flow_id; +union { +struct { +__le16 aux0; +__le16 aux1; + } flex; +__le32 ts_high; + } flex_ts; +}; + +/* Rx Flex Descriptor for Comms Package Profile +* RxDID Profile ID 22-23 (swap Hash and FlowID) +* Flex-field 0: Flow ID lower 16-bits +* Flex-field 1: Flow ID upper 16-bits +* Flex-field 2: RSS hash lower 16-bits +* Flex-field 3: RSS hash upper 16-bits +* Flex-field 4: AUX0 +* Flex-field 5: AUX1 +*/ +struct iavf_32b_rx_flex_desc_comms_ovs { +/* Qword 0 */ +u8 rxdid; +u8 mir_id_umb_cast; +__le16 ptype_flexi_flags0; +__le16 pkt_len; +__le16 hdr_len_sph_flex_flags1; + +/* Qword 1 */ +__le16 status_error0; +__le16 l2tag1; +__le32 flow_id; + +/* Qword 2 */ +__le16 status_error1; +u8 flexi_flags2; +u8 ts_low; +__le16 l2tag2_1st; +__le16 l2tag2_2nd; + +/* Qword 3 */ +__le32 rss_hash; +union { +struct { +__le16 aux0; +__le16 aux1; + } flex; +__le32 ts_high; + } flex_ts; +}; + +/* Receive Flex Descriptor profile IDs: There are a total +* of 64 profiles where profile IDs 0/1 are for legacy; and +* profiles 2-63 are flex profiles that can be programmed +* with a specific metadata (profile 7 reserved for HW) +*/ +enum iavf_rxdid { +IAVF_RXDID_LEGACY_0 =0 , +IAVF_RXDID_LEGACY_1 =1 , +IAVF_RXDID_FLEX_NIC =2 , +IAVF_RXDID_FLEX_NIC_2 =6 , +IAVF_RXDID_HW =7 , +IAVF_RXDID_COMMS_GENERIC =16 , +IAVF_RXDID_COMMS_AUX_VLAN =17 , +IAVF_RXDID_COMMS_AUX_IPV4 =18 , +IAVF_RXDID_COMMS_AUX_IPV6 =19 , +IAVF_RXDID_COMMS_AUX_IPV6_FLOW =20 , +IAVF_RXDID_COMMS_AUX_TCP =21 , +IAVF_RXDID_COMMS_OVS_1 =22 , +IAVF_RXDID_COMMS_OVS_2 =23 , +IAVF_RXDID_LAST =63 , +}; + +enum iavf_rx_flex_desc_status_error_0_bits { +/* Note: These are predefined bit offsets */ +IAVF_RX_FLEX_DESC_STATUS0_DD_S =0 , +IAVF_RX_FLEX_DESC_STATUS0_EOF_S , +IAVF_RX_FLEX_DESC_STATUS0_HBO_S , +IAVF_RX_FLEX_DESC_STATUS0_L3L4P_S , +IAVF_RX_FLEX_DESC_STATUS0_XSUM_IPE_S , +IAVF_RX_FLEX_DESC_STATUS0_XSUM_L4E_S , +IAVF_RX_FLEX_DESC_STATUS0_XSUM_EIPE_S , +IAVF_RX_FLEX_DESC_STATUS0_XSUM_EUDPE_S , +IAVF_RX_FLEX_DESC_STATUS0_LPBK_S , +IAVF_RX_FLEX_DESC_STATUS0_IPV6EXADD_S , +IAVF_RX_FLEX_DESC_STATUS0_RXE_S , +IAVF_RX_FLEX_DESC_STATUS0_CRCP_S , +IAVF_RX_FLEX_DESC_STATUS0_RSS_VALID_S , +IAVF_RX_FLEX_DESC_STATUS0_L2TAG1P_S , +IAVF_RX_FLEX_DESC_STATUS0_XTRMD0_VALID_S , +IAVF_RX_FLEX_DESC_STATUS0_XTRMD1_VALID_S , +IAVF_RX_FLEX_DESC_STATUS0_LAST /* this entry must be last!!! */ +}; + +/* for iavf_32b_rx_flex_desc.ptype_flex_flags0 member */ +#define IAVF_RX_FLEX_DESC_PTYPE_M (0x3FF)/* 10-bits */ + +/* for iavf_32b_rx_flex_desc.pkt_len member */ +#define IAVF_RX_FLX_DESC_PKT_LEN_M (0x3FFF)/* 14-bits */ + int iavf_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, uint16_t nb_desc,
Signature · 1 line
2.17.1
PATCH v4 2/11 net/iavf: return error if opcode is mismatched
In reply to Leyi RongAdds error return when the opcode of read message is mismatched which is received from adminQ.
--- drivers/net/iavf/iavf_vchnl.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
drivers/net/iavf/iavf_vchnl.c +3 -1
@@ -52,9 +52,11 @@ iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len,PMD_DRV_LOG (DEBUG ,"AQ from pf carries opcode %u, retval %d" , opcode, vf->cmd_retval); -if (opcode != vf->pend_cmd) +if (opcode != vf->pend_cmd) {PMD_DRV_LOG (WARNING ,"command mismatch, expect %u, get %u" , vf->pend_cmd, opcode); +return IAVF_ERR_OPCODE_MISMATCH ; + }return IAVF_SUCCESS ; }
Signature · 1 line
2.17.1
PATCH v4 3/11 net/iavf: flexible Rx descriptor support in normal path
In reply to Leyi RongSupport flexible Rx descriptor format in normal
path of iAVF PMD.
--- drivers/net/iavf/iavf.h | 2 + drivers/net/iavf/iavf_ethdev.c | 8 + drivers/net/iavf/iavf_rxtx.c | 479 ++++++++++++++++++++++++++++++--- drivers/net/iavf/iavf_rxtx.h | 8 + drivers/net/iavf/iavf_vchnl.c | 42 ++- 5 files changed, 501 insertions(+), 38 deletions(-)
drivers/net/iavf/iavf.h +2 -0
@@ -103,6 +103,7 @@ struct iavf_info {struct virtchnl_version_info virtchnl_version;struct virtchnl_vf_resource *vf_res;/* VF resource */ struct virtchnl_vsi_resource *vsi_res;/* LAN VSI */ +uint64_t supported_rxdid;volatile enum virtchnl_ops pend_cmd;/* pending command not finished */ uint32_t cmd_retval;/* return value of the cmd response from PF */
@@ -235,6 +236,7 @@ int iavf_disable_queues(struct iavf_adapter *adapter);int iavf_configure_rss_lut (struct iavf_adapter *adapter);int iavf_configure_rss_key (struct iavf_adapter *adapter);int iavf_configure_queues (struct iavf_adapter *adapter); +int iavf_get_supported_rxdid (struct iavf_adapter *adapter);int iavf_config_irq_map (struct iavf_adapter *adapter);void iavf_add_del_all_mac_addr (struct iavf_adapter *adapter,bool add); int iavf_dev_link_update(struct rte_eth_dev *dev,
drivers/net/iavf/iavf_ethdev.c +8 -0
@@ -1245,6 +1245,14 @@ iavf_init_vf(struct rte_eth_dev *dev) goto err_rss; } } + +if (vf->vf_res->vf_cap_flags &VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC ) { +if (iavf_get_supported_rxdid (adapter) !=0 ) { +PMD_INIT_LOG (ERR ,"failed to do get supported rxdid" ); + goto err_rss; + } + } +return 0 ; err_rss:rte_free (vf->rss_key);
drivers/net/iavf/iavf_rxtx.c +442 -37
546 lines not on this page · raw message
drivers/net/iavf/iavf_rxtx.h +8 -0
@@ -62,6 +62,7 @@#define iavf_rx_desc iavf_16byte_rx_desc#else #define iavf_rx_desc iavf_32byte_rx_desc +#define iavf_rx_flex_desc iavf_32b_rx_flex_desc#endif struct iavf_rxq_ops {
@@ -87,6 +88,7 @@ struct iavf_rx_queue {struct rte_mbuf *pkt_first_seg;/* first segment of current packet */ struct rte_mbuf *pkt_last_seg;/* last segment of current packet */ struct rte_mbuf fake_mbuf;/* dummy mbuf */ +uint8_t rxdid;/* used for VPMD */ uint16_t rxrearm_nb;/* number of remaining to be re-armed */
@@ -379,9 +381,15 @@ void iavf_dev_tx_queue_release(void *txq);void iavf_stop_queues (struct rte_eth_dev *dev);uint16_t iavf_recv_pkts (void *rx_queue,struct rte_mbuf **rx_pkts,uint16_t nb_pkts); +uint16_t iavf_recv_pkts_flex_rxd (void *rx_queue, +struct rte_mbuf **rx_pkts, +uint16_t nb_pkts);uint16_t iavf_recv_scattered_pkts (void *rx_queue,struct rte_mbuf **rx_pkts,uint16_t nb_pkts); +uint16_t iavf_recv_scattered_pkts_flex_rxd (void *rx_queue, +struct rte_mbuf **rx_pkts, +uint16_t nb_pkts);uint16_t iavf_xmit_pkts (void *tx_queue,struct rte_mbuf **tx_pkts,uint16_t nb_pkts); uint16_t iavf_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
drivers/net/iavf/iavf_vchnl.c +41 -1
66 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH v4 4/11 net/iavf: flexible Rx descriptor support in AVX path
In reply to Leyi RongSupport flexible Rx descriptor format in AVX
path of iAVF PMD.
--- drivers/net/iavf/iavf_rxtx.c | 24 +- drivers/net/iavf/iavf_rxtx.h | 6 + drivers/net/iavf/iavf_rxtx_vec_avx2.c | 550 +++++++++++++++++++++++++- 3 files changed, 570 insertions(+), 10 deletions(-)
drivers/net/iavf/iavf_rxtx.c +18 -6
@@ -2081,16 +2081,28 @@ iavf_set_rx_function(struct rte_eth_dev *dev)"Using %sVector Scattered Rx (port %d)." , use_avx2 ?"avx2 " :"" , dev->data->port_id); - dev->rx_pkt_burst = use_avx2 ? - iavf_recv_scattered_pkts_vec_avx2 : - iavf_recv_scattered_pkts_vec; +if (vf->vf_res->vf_cap_flags & +VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC ) + dev->rx_pkt_burst = use_avx2 ? + iavf_recv_scattered_pkts_vec_avx2_flex_rxd : + iavf_recv_scattered_pkts_vec; +else + dev->rx_pkt_burst = use_avx2 ? + iavf_recv_scattered_pkts_vec_avx2 : + iavf_recv_scattered_pkts_vec; }else {PMD_DRV_LOG (DEBUG ,"Using %sVector Rx (port %d)." , use_avx2 ?"avx2 " :"" , dev->data->port_id); - dev->rx_pkt_burst = use_avx2 ? - iavf_recv_pkts_vec_avx2 : - iavf_recv_pkts_vec; +if (vf->vf_res->vf_cap_flags & +VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC ) + dev->rx_pkt_burst = use_avx2 ? + iavf_recv_pkts_vec_avx2_flex_rxd : + iavf_recv_pkts_vec; +else + dev->rx_pkt_burst = use_avx2 ? + iavf_recv_pkts_vec_avx2 : + iavf_recv_pkts_vec; }return ;
drivers/net/iavf/iavf_rxtx.h +6 -0
@@ -413,9 +413,15 @@ uint16_t iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts);uint16_t iavf_recv_pkts_vec_avx2 (void *rx_queue,struct rte_mbuf **rx_pkts,uint16_t nb_pkts); +uint16_t iavf_recv_pkts_vec_avx2_flex_rxd (void *rx_queue, +struct rte_mbuf **rx_pkts, +uint16_t nb_pkts);uint16_t iavf_recv_scattered_pkts_vec_avx2 (void *rx_queue,struct rte_mbuf **rx_pkts,uint16_t nb_pkts); +uint16_t iavf_recv_scattered_pkts_vec_avx2_flex_rxd (void *rx_queue, +struct rte_mbuf **rx_pkts, +uint16_t nb_pkts);uint16_t iavf_xmit_pkts_vec (void *tx_queue,struct rte_mbuf **tx_pkts,uint16_t nb_pkts); uint16_t iavf_xmit_pkts_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts,
drivers/net/iavf/iavf_rxtx_vec_avx2.c +546 -4
585 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH v4 5/11 net/iavf: flexible Rx descriptor support in SSE path
In reply to Leyi RongSupport flexible Rx descriptor format in SSE
path of iAVF PMD.
--- drivers/net/iavf/iavf_rxtx.c | 4 +- drivers/net/iavf/iavf_rxtx.h | 5 + drivers/net/iavf/iavf_rxtx_vec_sse.c | 414 +++++++++++++++++++++++++++ 3 files changed, 421 insertions(+), 2 deletions(-)
drivers/net/iavf/iavf_rxtx.c +2 -2
@@ -2085,7 +2085,7 @@ iavf_set_rx_function(struct rte_eth_dev *dev) VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) dev->rx_pkt_burst = use_avx2 ? iavf_recv_scattered_pkts_vec_avx2_flex_rxd : - iavf_recv_scattered_pkts_vec; + iavf_recv_scattered_pkts_vec_flex_rxd; else dev->rx_pkt_burst = use_avx2 ? iavf_recv_scattered_pkts_vec_avx2 :
@@ -2098,7 +2098,7 @@ iavf_set_rx_function(struct rte_eth_dev *dev) VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) dev->rx_pkt_burst = use_avx2 ? iavf_recv_pkts_vec_avx2_flex_rxd : - iavf_recv_pkts_vec; + iavf_recv_pkts_vec_flex_rxd; else dev->rx_pkt_burst = use_avx2 ? iavf_recv_pkts_vec_avx2 :
drivers/net/iavf/iavf_rxtx.h +5 -0
@@ -406,9 +406,14 @@ int iavf_dev_tx_desc_status(void *tx_queue, uint16_t offset);uint16_t iavf_recv_pkts_vec (void *rx_queue,struct rte_mbuf **rx_pkts,uint16_t nb_pkts); +uint16_t iavf_recv_pkts_vec_flex_rxd (void *rx_queue,struct rte_mbuf **rx_pkts, +uint16_t nb_pkts);uint16_t iavf_recv_scattered_pkts_vec (void *rx_queue,struct rte_mbuf **rx_pkts,uint16_t nb_pkts); +uint16_t iavf_recv_scattered_pkts_vec_flex_rxd (void *rx_queue, +struct rte_mbuf **rx_pkts, +uint16_t nb_pkts);uint16_t iavf_xmit_fixed_burst_vec (void *tx_queue,struct rte_mbuf **tx_pkts,uint16_t nb_pkts); uint16_t iavf_recv_pkts_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,
drivers/net/iavf/iavf_rxtx_vec_sse.c +414 -0
444 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH v4 6/11 net/iavf: add flow director enabled switch value
In reply to Leyi RongThe commit adds fdir_enabled flag into iavf_rx_queue structure to identify if fdir id is active. Rx data path can be benefit if fdir id parsing is not needed, especially in vector path.
--- drivers/net/iavf/iavf.h | 1 + drivers/net/iavf/iavf_rxtx.h | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+)
drivers/net/iavf/iavf.h +1 -0
@@ -144,6 +144,7 @@ struct iavf_adapter {bool tx_vec_allowed;const uint32_t *ptype_tbl;bool stopped; +uint16_t fdir_ref_cnt; };/* IAVF_DEV_PRIVATE_TO */
drivers/net/iavf/iavf_rxtx.h +30 -0
@@ -103,6 +103,7 @@ struct iavf_rx_queue {uint16_t port_id;/* device port ID */ uint8_t crc_len;/* 0 if CRC stripped, 4 otherwise */ +uint8_t fdir_enabled;/* 0 if FDIR disabled, 1 when enabled */ uint16_t queue_id;/* Rx queue index */ uint16_t rx_buf_len;/* The packet buffer size */ uint16_t rx_hdr_len;/* The header buffer size */
@@ -490,6 +491,35 @@ void iavf_dump_tx_descriptor(const struct iavf_tx_queue *txq, tx_desc->cmd_type_offset_bsz); } +#define FDIR_PROC_ENABLE_PER_QUEUE (ad, on) do { \ + int i; \ + for (i = 0; i < (ad)->eth_dev->data->nb_rx_queues; i++) { \ + struct iavf_rx_queue *rxq = (ad)->eth_dev->data->rx_queues[i]; \ + if (!rxq) \ + continue; \ + rxq->fdir_enabled = on; \ + } \ + PMD_DRV_LOG(DEBUG, "FDIR processing on RX set to %d", on); \ +} while (0) + +/* Enable/disable flow director Rx processing in data path. */ +static inline +void iavf_fdir_rx_proc_enable (struct iavf_adapter *ad,bool on) +{ +if (on) { +/* enable flow director processing */ +if (ad->fdir_ref_cnt++ ==0 ) +FDIR_PROC_ENABLE_PER_QUEUE (ad, on); + }else { +if (ad->fdir_ref_cnt >=1 ) { + ad->fdir_ref_cnt--; + +if (ad->fdir_ref_cnt ==0 ) +FDIR_PROC_ENABLE_PER_QUEUE (ad, on); + } + } +} +#ifdef RTE_LIBRTE_IAVF_DEBUG_DUMP_DESC #define IAVF_DUMP_RX_DESC (rxq, desc, rx_id) \ iavf_dump_rx_descriptor(rxq, desc, rx_id)
Signature · 1 line
2.17.1
PATCH v4 7/11 net/iavf: support flow mark in normal data path
In reply to Leyi RongSupport Flow Director mark ID parsing in normal path.
--- drivers/net/iavf/iavf.h | 3 +++ drivers/net/iavf/iavf_rxtx.c | 37 ++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+)
drivers/net/iavf/iavf.h +3 -0
@@ -67,6 +67,9 @@#define IAVF_48_BIT_WIDTH (CHAR_BIT * 6)#define IAVF_48_BIT_MASK RTE_LEN2MASK(IAVF_48_BIT_WIDTH, uint64_t) +#define IAVF_RX_DESC_EXT_STATUS_FLEXBH_MASK 0x03 +#define IAVF_RX_DESC_EXT_STATUS_FLEXBH_FD_ID 0x01 +struct iavf_adapter ;struct iavf_rx_queue ;struct iavf_tx_queue ;
drivers/net/iavf/iavf_rxtx.c +37 -0
@@ -756,6 +756,10 @@ iavf_rxd_to_pkt_flags(uint64_t qword) IAVF_RX_DESC_FLTSTAT_RSS_HASH) == IAVF_RX_DESC_FLTSTAT_RSS_HASH) ? PKT_RX_RSS_HASH : 0; +/* Check if FDIR Match */ + flags |= (qword & (1 <<IAVF_RX_DESC_STATUS_FLM_SHIFT ) ? +PKT_RX_FDIR :0 ); +if (likely ((error_bits &IAVF_RX_ERR_BITS ) ==0 )) { flags |= (PKT_RX_IP_CKSUM_GOOD |PKT_RX_L4_CKSUM_GOOD );return flags;
@@ -776,6 +780,25 @@ iavf_rxd_to_pkt_flags(uint64_t qword)return flags; } +static inline uint64_t +iavf_rxd_build_fdir (volatile union iavf_rx_desc *rxdp,struct rte_mbuf *mb) +{ +uint64_t flags =0 ; +uint16_t flexbh; + + flexbh = (rte_le_to_cpu_32 (rxdp->wb.qword2.ext_status) >> +IAVF_RX_DESC_EXT_STATUS_FLEXBH_SHIFT ) & +IAVF_RX_DESC_EXT_STATUS_FLEXBH_MASK ; + +if (flexbh ==IAVF_RX_DESC_EXT_STATUS_FLEXBH_FD_ID ) { + mb->hash.fdir.hi = +rte_le_to_cpu_32 (rxdp->wb.qword3.hi_dword.fd_id); + flags |=PKT_RX_FDIR_ID ; + } + +return flags; +} +/* Translate the rx flex descriptor status to pkt flags */ static inline void iavf_rxd_to_pkt_fields(struct rte_mbuf *mb,
@@ -792,6 +815,11 @@ iavf_rxd_to_pkt_fields(struct rte_mbuf *mb, mb->hash.rss = rte_le_to_cpu_32(desc->rss_hash); }#endif + +if (desc->flow_id !=0xFFFFFFFF ) { + mb->ol_flags |=PKT_RX_FDIR |PKT_RX_FDIR_ID ; + mb->hash.fdir.hi =rte_le_to_cpu_32 (desc->flow_id); + } }#define IAVF_RX_FLEX_ERR0_BITS \
@@ -951,6 +979,9 @@ iavf_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) rxm->hash.rss =rte_le_to_cpu_32 (rxd.wb.qword0.hi_dword.rss); +if (pkt_flags &PKT_RX_FDIR ) + pkt_flags |=iavf_rxd_build_fdir (&rxd, rxm); + rxm->ol_flags |= pkt_flags; rx_pkts[nb_rx++] = rxm;
@@ -1349,6 +1380,9 @@ iavf_recv_scattered_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, first_seg->hash.rss =rte_le_to_cpu_32 (rxd.wb.qword0.hi_dword.rss); +if (pkt_flags &PKT_RX_FDIR ) + pkt_flags |=iavf_rxd_build_fdir (&rxd, first_seg); + first_seg->ol_flags |= pkt_flags;/* Prefetch data of first segment, if configured to do so. */
@@ -1515,6 +1549,9 @@ iavf_rx_scan_hw_ring(struct iavf_rx_queue *rxq) mb->hash.rss =rte_le_to_cpu_32 ( rxdp[j].wb.qword0.hi_dword.rss); +if (pkt_flags &PKT_RX_FDIR ) + pkt_flags |=iavf_rxd_build_fdir (&rxdp[j], mb); + mb->ol_flags |= pkt_flags; }
Signature · 1 line
2.17.1
PATCH v4 8/11 net/iavf: support flow mark in AVX path
In reply to Leyi RongSupport Flow Director mark ID parsing from Flex
Rx descriptor in AVX path.
--- drivers/net/iavf/iavf_rxtx_vec_avx2.c | 72 +++++++++++++++++++++++++-- 1 file changed, 67 insertions(+), 5 deletions(-)
drivers/net/iavf/iavf_rxtx_vec_avx2.c +67 -5
@@ -616,6 +616,25 @@ _iavf_recv_raw_pkts_vec_avx2(struct iavf_rx_queue *rxq,return received; } +static inline __m256i +flex_rxd_to_fdir_flags_vec_avx2 (const __m256i fdir_id0_7) +{ +#define FDID_MIS_MAGIC 0xFFFFFFFF +RTE_BUILD_BUG_ON (PKT_RX_FDIR != (1 <<2 )); +RTE_BUILD_BUG_ON (PKT_RX_FDIR_ID != (1 <<13 )); +const __m256i pkt_fdir_bit =_mm256_set1_epi32 (PKT_RX_FDIR | +PKT_RX_FDIR_ID ); +/* desc->flow_id field == 0xFFFFFFFF means fdir mismatch */ +const __m256i fdir_mis_mask =_mm256_set1_epi32 (FDID_MIS_MAGIC ); +__m256i fdir_mask =_mm256_cmpeq_epi32 (fdir_id0_7, + fdir_mis_mask); +/* this XOR op results to bit-reverse the fdir_mask */ + fdir_mask =_mm256_xor_si256 (fdir_mask, fdir_mis_mask); +const __m256i fdir_flags =_mm256_and_si256 (fdir_mask, pkt_fdir_bit); + +return fdir_flags; +} + static inline uint16_t _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq, struct rte_mbuf **rx_pkts,
@@ -678,8 +697,8 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,const __m256i shuf_msk = _mm256_set_epi8 (/* first descriptor */ - 15, 14, - 13, 12, /* octet 12~15, 32 bits rss */ + 0xFF, 0xFF, + 0xFF, 0xFF, /* rss not supported */ 11, 10, /* octet 10~11, 16 bits vlan_macip */ 5, 4, /* octet 4~5, 16 bits data_len */ 0xFF, 0xFF, /* skip hi 16 bits pkt_len, zero out */
@@ -687,8 +706,8 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,0xFF ,0xFF ,/* pkt_type set as unknown */ 0xFF ,0xFF ,/*pkt_type set as unknown */ /* second descriptor */ -15 ,14 , -13 ,12 ,/* octet 12~15, 32 bits rss */ +0xFF ,0xFF , +0xFF ,0xFF ,/* rss not supported */ 11 ,10 ,/* octet 10~11, 16 bits vlan_macip */ 5 ,4 ,/* octet 4~5, 16 bits data_len */ 0xFF, 0xFF, /* skip hi 16 bits pkt_len, zero out */
@@ -930,8 +949,51 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq, rss_vlan_flag_bits);/* merge flags */ -const __m256i mbuf_flags =_mm256_or_si256 (l3_l4_flags, +__m256i mbuf_flags =_mm256_or_si256 (l3_l4_flags, rss_vlan_flags); + +if (rxq->fdir_enabled) { +const __m256i fdir_id4_7 = +_mm256_unpackhi_epi32 (raw_desc6_7, raw_desc4_5); + +const __m256i fdir_id0_3 = +_mm256_unpackhi_epi32 (raw_desc2_3, raw_desc0_1); + +const __m256i fdir_id0_7 = +_mm256_unpackhi_epi64 (fdir_id4_7, fdir_id0_3); + +const __m256i fdir_flags = +flex_rxd_to_fdir_flags_vec_avx2 (fdir_id0_7); + +/* merge with fdir_flags */ + mbuf_flags =_mm256_or_si256 (mbuf_flags, fdir_flags); + +/* write to mbuf: have to use scalar store here */ + rx_pkts[i +0 ]->hash.fdir.hi = +_mm256_extract_epi32 (fdir_id0_7,3 ); + + rx_pkts[i +1 ]->hash.fdir.hi = +_mm256_extract_epi32 (fdir_id0_7,7 ); + + rx_pkts[i +2 ]->hash.fdir.hi = +_mm256_extract_epi32 (fdir_id0_7,2 ); + + rx_pkts[i +3 ]->hash.fdir.hi = +_mm256_extract_epi32 (fdir_id0_7,6 ); + + rx_pkts[i +4 ]->hash.fdir.hi = +_mm256_extract_epi32 (fdir_id0_7,1 ); + + rx_pkts[i +5 ]->hash.fdir.hi = +_mm256_extract_epi32 (fdir_id0_7,5 ); + + rx_pkts[i +6 ]->hash.fdir.hi = +_mm256_extract_epi32 (fdir_id0_7,0 ); + + rx_pkts[i +7 ]->hash.fdir.hi = +_mm256_extract_epi32 (fdir_id0_7,4 ); + }/* if() on fdir_enabled */ +/** * At this point, we have the 8 sets of flags in the low 16-bits * of each 32-bit value in vlan0.
Signature · 1 line
2.17.1
PATCH v4 9/11 net/iavf: support flow mark in SSE path
In reply to Leyi RongSupport Flow Director mark ID parsing from Flex
Rx descriptor in SSE path.
--- drivers/net/iavf/iavf_rxtx_vec_sse.c | 52 +++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-)
drivers/net/iavf/iavf_rxtx_vec_sse.c +51 -1
@@ -189,6 +189,25 @@ desc_to_olflags_v(struct iavf_rx_queue *rxq, __m128i descs[4],_mm_store_si128 ((__m128i *)&rx_pkts[3 ]->rearm_data, rearm3); } +static inline __m128i +flex_rxd_to_fdir_flags_vec (const __m128i fdir_id0_3) +{ +#define FDID_MIS_MAGIC 0xFFFFFFFF +RTE_BUILD_BUG_ON (PKT_RX_FDIR != (1 <<2 )); +RTE_BUILD_BUG_ON (PKT_RX_FDIR_ID != (1 <<13 )); +const __m128i pkt_fdir_bit =_mm_set1_epi32 (PKT_RX_FDIR | +PKT_RX_FDIR_ID ); +/* desc->flow_id field == 0xFFFFFFFF means fdir mismatch */ +const __m128i fdir_mis_mask =_mm_set1_epi32 (FDID_MIS_MAGIC ); +__m128i fdir_mask =_mm_cmpeq_epi32 (fdir_id0_3, + fdir_mis_mask); +/* this XOR op results to bit-reverse the fdir_mask */ + fdir_mask =_mm_xor_si128 (fdir_mask, fdir_mis_mask); +const __m128i fdir_flags =_mm_and_si128 (fdir_mask, pkt_fdir_bit); + +return fdir_flags; +} +static inline void flex_desc_to_olflags_v (struct iavf_rx_queue *rxq,__m128i descs[4 ],struct rte_mbuf **rx_pkts)
@@ -267,6 +286,36 @@ flex_desc_to_olflags_v(struct iavf_rx_queue *rxq, __m128i descs[4],/* merge the flags */ flags =_mm_or_si128 (flags, rss_vlan); +if (rxq->fdir_enabled) { +const __m128i fdir_id0_1 = +_mm_unpackhi_epi32 (descs[0 ], descs[1 ]); + +const __m128i fdir_id2_3 = +_mm_unpackhi_epi32 (descs[2 ], descs[3 ]); + +const __m128i fdir_id0_3 = +_mm_unpackhi_epi64 (fdir_id0_1, fdir_id2_3); + +const __m128i fdir_flags = +flex_rxd_to_fdir_flags_vec (fdir_id0_3); + +/* merge with fdir_flags */ + flags =_mm_or_si128 (flags, fdir_flags); + +/* write fdir_id to mbuf */ + rx_pkts[0 ]->hash.fdir.hi = +_mm_extract_epi32 (fdir_id0_3,0 ); + + rx_pkts[1 ]->hash.fdir.hi = +_mm_extract_epi32 (fdir_id0_3,1 ); + + rx_pkts[2 ]->hash.fdir.hi = +_mm_extract_epi32 (fdir_id0_3,2 ); + + rx_pkts[3 ]->hash.fdir.hi = +_mm_extract_epi32 (fdir_id0_3,3 ); + }/* if() on fdir_enabled */ +/** * At this point, we have the 4 sets of flags in the low 16-bits * of each 32-bit value in flags.
@@ -604,7 +653,8 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,const __m128i zero =_mm_setzero_si128 ();/* mask to shuffle from desc. to mbuf */ const __m128i shuf_msk = _mm_set_epi8 - (15, 14, 13, 12, /* octet 12~15, 32 bits rss */ + (0xFF, 0xFF, + 0xFF, 0xFF, /* rss not supported */ 11, 10, /* octet 10~11, 16 bits vlan_macip */ 5, 4, /* octet 4~5, 16 bits data_len */ 0xFF, 0xFF, /* skip high 16 bits pkt_len, zero out */
Signature · 1 line
2.17.1
PATCH v4 10/11 net/iavf: add RSS hash parsing in AVX path
In reply to Leyi RongSupport RSS hash parsing from Flex Rx
descriptor in AVX data path.
--- drivers/net/iavf/iavf_rxtx_vec_avx2.c | 92 ++++++++++++++++++++++++++- 1 file changed, 90 insertions(+), 2 deletions(-)
drivers/net/iavf/iavf_rxtx_vec_avx2.c +90 -2
@@ -698,7 +698,7 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq, _mm256_set_epi8 (/* first descriptor */ 0xFF, 0xFF, - 0xFF, 0xFF, /* rss not supported */ + 0xFF, 0xFF, /* rss hash parsed separately */ 11, 10, /* octet 10~11, 16 bits vlan_macip */ 5, 4, /* octet 4~5, 16 bits data_len */ 0xFF, 0xFF, /* skip hi 16 bits pkt_len, zero out */
@@ -707,7 +707,7 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,0xFF ,0xFF ,/*pkt_type set as unknown */ /* second descriptor */ 0xFF ,0xFF , -0xFF ,0xFF ,/* rss not supported */ +0xFF ,0xFF ,/* rss hash parsed separately */ 11 ,10 ,/* octet 10~11, 16 bits vlan_macip */ 5 ,4 ,/* octet 4~5, 16 bits data_len */ 0xFF, 0xFF, /* skip hi 16 bits pkt_len, zero out */
@@ -994,6 +994,94 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,_mm256_extract_epi32 (fdir_id0_7,4 ); }/* if() on fdir_enabled */ +/** +* needs to load 2nd 16B of each desc for RSS hash parsing, +* will cause performance drop to get into this context. +*/ +if (rxq->vsi->adapter->eth_dev->data->dev_conf.rxmode.offloads & +DEV_RX_OFFLOAD_RSS_HASH ) { +/* load bottom half of every 32B desc */ +const __m128i raw_desc_bh7 = +_mm_load_si128 + ((void *)(&rxdp[7 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh6 = +_mm_load_si128 + ((void *)(&rxdp[6 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh5 = +_mm_load_si128 + ((void *)(&rxdp[5 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh4 = +_mm_load_si128 + ((void *)(&rxdp[4 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh3 = +_mm_load_si128 + ((void *)(&rxdp[3 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh2 = +_mm_load_si128 + ((void *)(&rxdp[2 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh1 = +_mm_load_si128 + ((void *)(&rxdp[1 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh0 = +_mm_load_si128 + ((void *)(&rxdp[0 ].wb.status_error1)); + +__m256i raw_desc_bh6_7 = +_mm256_inserti128_si256 + (_mm256_castsi128_si256 (raw_desc_bh6), + raw_desc_bh7,1 ); +__m256i raw_desc_bh4_5 = +_mm256_inserti128_si256 + (_mm256_castsi128_si256 (raw_desc_bh4), + raw_desc_bh5,1 ); +__m256i raw_desc_bh2_3 = +_mm256_inserti128_si256 + (_mm256_castsi128_si256 (raw_desc_bh2), + raw_desc_bh3,1 ); +__m256i raw_desc_bh0_1 = +_mm256_inserti128_si256 + (_mm256_castsi128_si256 (raw_desc_bh0), + raw_desc_bh1,1 ); + +/** +* to shift the 32b RSS hash value to the +* highest 32b of each 128b before mask +*/ +__m256i rss_hash6_7 = +_mm256_slli_epi64 (raw_desc_bh6_7,32 ); +__m256i rss_hash4_5 = +_mm256_slli_epi64 (raw_desc_bh4_5,32 ); +__m256i rss_hash2_3 = +_mm256_slli_epi64 (raw_desc_bh2_3,32 ); +__m256i rss_hash0_1 = +_mm256_slli_epi64 (raw_desc_bh0_1,32 ); + +__m256i rss_hash_msk = +_mm256_set_epi32 (0xFFFFFFFF ,0 ,0 ,0 , +0xFFFFFFFF ,0 ,0 ,0 ); + + rss_hash6_7 =_mm256_and_si256 + (rss_hash6_7, rss_hash_msk); + rss_hash4_5 =_mm256_and_si256 + (rss_hash4_5, rss_hash_msk); + rss_hash2_3 =_mm256_and_si256 + (rss_hash2_3, rss_hash_msk); + rss_hash0_1 =_mm256_and_si256 + (rss_hash0_1, rss_hash_msk); + + mb6_7 =_mm256_or_si256 (mb6_7, rss_hash6_7); + mb4_5 =_mm256_or_si256 (mb4_5, rss_hash4_5); + mb2_3 =_mm256_or_si256 (mb2_3, rss_hash2_3); + mb0_1 =_mm256_or_si256 (mb0_1, rss_hash0_1); + }/* if() on RSS hash parsing */ +/** * At this point, we have the 8 sets of flags in the low 16-bits * of each 32-bit value in vlan0.
Signature · 1 line
2.17.1
PATCH v4 11/11 net/iavf: add RSS hash parsing in SSE path
In reply to Leyi RongSupport RSS hash parsing from Flex Rx
descriptor in SSE data path.
--- drivers/net/iavf/iavf_rxtx_vec_sse.c | 86 ++++++++++++++++++++++------ 1 file changed, 70 insertions(+), 16 deletions(-)
drivers/net/iavf/iavf_rxtx_vec_sse.c +70 -16
@@ -654,7 +654,7 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,/* mask to shuffle from desc. to mbuf */ const __m128i shuf_msk = _mm_set_epi8 (0xFF, 0xFF, - 0xFF, 0xFF, /* rss not supported */ + 0xFF, 0xFF, /* rss hash parsed separately */ 11, 10, /* octet 10~11, 16 bits vlan_macip */ 5, 4, /* octet 4~5, 16 bits data_len */ 0xFF, 0xFF, /* skip high 16 bits pkt_len, zero out */
@@ -745,7 +745,7 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq, pos += IAVF_VPMD_DESCS_PER_LOOP, rxdp += IAVF_VPMD_DESCS_PER_LOOP) {__m128i descs[IAVF_VPMD_DESCS_PER_LOOP ]; -__m128i pkt_mb1, pkt_mb2, pkt_mb3, pkt_mb4; +__m128i pkt_mb0, pkt_mb1, pkt_mb2, pkt_mb3;__m128i staterr, sterr_tmp1, sterr_tmp2;/* 2 64 bit or 4 32 bit mbuf pointers in one XMM reg. */ __m128i mbp1;
@@ -791,8 +791,12 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,rte_compiler_barrier ();/* D.1 pkt 3,4 convert format from desc to pktmbuf */ - pkt_mb4 =_mm_shuffle_epi8 (descs[3 ], shuf_msk); - pkt_mb3 =_mm_shuffle_epi8 (descs[2 ], shuf_msk); + pkt_mb3 =_mm_shuffle_epi8 (descs[3 ], shuf_msk); + pkt_mb2 =_mm_shuffle_epi8 (descs[2 ], shuf_msk); + +/* D.1 pkt 1,2 convert format from desc to pktmbuf */ + pkt_mb1 =_mm_shuffle_epi8 (descs[1 ], shuf_msk); + pkt_mb0 =_mm_shuffle_epi8 (descs[0 ], shuf_msk);/* C.1 4=>2 filter staterr info only */ sterr_tmp2 =_mm_unpackhi_epi32 (descs[3 ], descs[2 ]);
@@ -802,12 +806,66 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,flex_desc_to_olflags_v (rxq, descs, &rx_pkts[pos]);/* D.2 pkt 3,4 set in_port/nb_seg and remove crc */ - pkt_mb4 =_mm_add_epi16 (pkt_mb4, crc_adjust); pkt_mb3 =_mm_add_epi16 (pkt_mb3, crc_adjust); + pkt_mb2 =_mm_add_epi16 (pkt_mb2, crc_adjust); -/* D.1 pkt 1,2 convert format from desc to pktmbuf */ - pkt_mb2 =_mm_shuffle_epi8 (descs[1 ], shuf_msk); - pkt_mb1 =_mm_shuffle_epi8 (descs[0 ], shuf_msk); +/* D.2 pkt 1,2 set in_port/nb_seg and remove crc */ + pkt_mb1 =_mm_add_epi16 (pkt_mb1, crc_adjust); + pkt_mb0 =_mm_add_epi16 (pkt_mb0, crc_adjust); + +/** +* needs to load 2nd 16B of each desc for RSS hash parsing, +* will cause performance drop to get into this context. +*/ +if (rxq->vsi->adapter->eth_dev->data->dev_conf.rxmode.offloads & +DEV_RX_OFFLOAD_RSS_HASH ) { +/* load bottom half of every 32B desc */ +const __m128i raw_desc_bh3 = +_mm_load_si128 + ((void *)(&rxdp[3 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh2 = +_mm_load_si128 + ((void *)(&rxdp[2 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh1 = +_mm_load_si128 + ((void *)(&rxdp[1 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh0 = +_mm_load_si128 + ((void *)(&rxdp[0 ].wb.status_error1)); + +/** +* to shift the 32b RSS hash value to the +* highest 32b of each 128b before mask +*/ +__m128i rss_hash3 = +_mm_slli_epi64 (raw_desc_bh3,32 ); +__m128i rss_hash2 = +_mm_slli_epi64 (raw_desc_bh2,32 ); +__m128i rss_hash1 = +_mm_slli_epi64 (raw_desc_bh1,32 ); +__m128i rss_hash0 = +_mm_slli_epi64 (raw_desc_bh0,32 ); + +__m128i rss_hash_msk = +_mm_set_epi32 (0xFFFFFFFF ,0 ,0 ,0 ); + + rss_hash3 =_mm_and_si128 + (rss_hash3, rss_hash_msk); + rss_hash2 =_mm_and_si128 + (rss_hash2, rss_hash_msk); + rss_hash1 =_mm_and_si128 + (rss_hash1, rss_hash_msk); + rss_hash0 =_mm_and_si128 + (rss_hash0, rss_hash_msk); + + pkt_mb3 =_mm_or_si128 (pkt_mb3, rss_hash3); + pkt_mb2 =_mm_or_si128 (pkt_mb2, rss_hash2); + pkt_mb1 =_mm_or_si128 (pkt_mb1, rss_hash1); + pkt_mb0 =_mm_or_si128 (pkt_mb0, rss_hash0); + }/* if() on RSS hash parsing */ /* C.2 get 4 pkts staterr value */ staterr =_mm_unpacklo_epi32 (sterr_tmp1, sterr_tmp2);
@@ -815,14 +873,10 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,/* D.3 copy final 3,4 data to rx_pkts */ _mm_storeu_si128 ((void *)&rx_pkts[pos +3 ]->rx_descriptor_fields1, - pkt_mb4); + pkt_mb3);_mm_storeu_si128 ((void *)&rx_pkts[pos +2 ]->rx_descriptor_fields1, - pkt_mb3); - -/* D.2 pkt 1,2 set in_port/nb_seg and remove crc */ - pkt_mb2 =_mm_add_epi16 (pkt_mb2, crc_adjust); - pkt_mb1 =_mm_add_epi16 (pkt_mb1, crc_adjust); + pkt_mb2);/* C* extract and record EOP bit */ if (split_packet) {
@@ -846,9 +900,9 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,/* D.3 copy final 1,2 data to rx_pkts */ _mm_storeu_si128 ((void *)&rx_pkts[pos +1 ]->rx_descriptor_fields1, - pkt_mb2); + pkt_mb1);_mm_storeu_si128 ((void *)&rx_pkts[pos]->rx_descriptor_fields1, - pkt_mb1); + pkt_mb0);flex_desc_to_ptype_v (descs, &rx_pkts[pos], ptype_tbl);/* C.4 calc avaialbe number of desc */ var =__builtin_popcountll (_mm_cvtsi128_si64 (staterr));
Signature · 1 line
2.17.1
v5 cover framework for advanced iAVF PMD
In reply to Leyi RongThis patchset enable framework for advanced iAVF, includes flexible descriptor support, FDIR mark id and RSS hash support.
---
v5:
- Set rxdid = 0 in iavf_configure_queues() when it's supported and RTE_LIBRTE_IAVF_16BYTE_RX_DESC is y. So the error can be captured explicitly as kernel PF does not support it yet.
- Fix some build errors when RTE_LIBRTE_IAVF_16BYTE_RX_DESC is y.
v4:
- Avoid initial declaration in 'for' loop to backward-compatible with older gcc versions.
v3:
- Remove patch to query DDP package info as it's specific to DCF.
- Specify fdir_ref_cnt as per VF value for enable/disable FDIR ID parse.
- Move fdir_enabled flag to per Queue value for cache benefit when access in vector routines.
- Store extracted flow_id value to mbuf without judgement to avoid branch speculation fail, which is benefit to performance.
v2:
- Revert macro RTE_LIBRTE_IAVF_16BYTE_RX_DESC deletion as it's defined in AVF spec.
- Typo RTE_LIBRTE_ICE_16BYTE_RX_DESC fix in iavf_rxtx.c.
- Move flex desc definitions into iavf_rxtx.h.
- Up to date to match with the latest version of virtchnl.h.
- Extract a new internal func iavf_update_rx_tail to call.
- Remove iavf_dev_rxq_count_flex_rxd()/iavf_dev_rx_desc_status_flex_rxd(), as the accompanying legacy ones can deal with the flex desc cases.
- Move rxq->rxdid assignment from iavf_configure_queues() to iavf_dev_rx_queue_setup().
- Unfold _mm_extract_epi32(fdir_id0_3, i) to fix build error when using GCC compile option -O0.
Leyi Rong (11):
net/iavf: flexible Rx descriptor definitions net/iavf: return error if opcode is mismatched net/iavf: flexible Rx descriptor support in normal path net/iavf: flexible Rx descriptor support in AVX path net/iavf: flexible Rx descriptor support in SSE path net/iavf: add flow director enabled switch value net/iavf: support flow mark in normal data path net/iavf: support flow mark in AVX path net/iavf: support flow mark in SSE path net/iavf: add RSS hash parsing in AVX path net/iavf: add RSS hash parsing in SSE path
drivers/net/iavf/iavf.h | 6 + drivers/net/iavf/iavf_ethdev.c | 8 + drivers/net/iavf/iavf_rxtx.c | 546 ++++++++++++++++++-- drivers/net/iavf/iavf_rxtx.h | 250 +++++++++ drivers/net/iavf/iavf_rxtx_vec_avx2.c | 702 +++++++++++++++++++++++++- drivers/net/iavf/iavf_rxtx_vec_sse.c | 520 +++++++++++++++++++ drivers/net/iavf/iavf_vchnl.c | 59 ++- 7 files changed, 2042 insertions(+), 49 deletions(-)
Signature · 1 line
2.17.1
PATCH v5 1/11 net/iavf: flexible Rx descriptor definitions
Add definitions for flexible Rx descriptor structures and macros.
--- drivers/net/iavf/iavf_rxtx.h | 200 +++++++++++++++++++++++++++++++++++ 1 file changed, 200 insertions(+)
drivers/net/iavf/iavf_rxtx.h +200 -0
@@ -157,6 +157,206 @@ union iavf_tx_offload { }; }; +/* Rx Flex Descriptors +* These descriptors are used instead of the legacy version descriptors +*/ +union iavf_16b_rx_flex_desc { +struct { +__le64 pkt_addr;/* Packet buffer address */ +__le64 hdr_addr;/* Header buffer address */ +/* bit 0 of hdr_addr is DD bit */ + } read; +struct { +/* Qword 0 */ +u8 rxdid;/* descriptor builder profile ID */ +u8 mir_id_umb_cast;/* mirror=[5:0], umb=[7:6] */ +__le16 ptype_flex_flags0;/* ptype=[9:0], ff0=[15:10] */ +__le16 pkt_len;/* [15:14] are reserved */ +__le16 hdr_len_sph_flex_flags1;/* header=[10:0] */ +/* sph=[11:11] */ +/* ff1/ext=[15:12] */ + +/* Qword 1 */ +__le16 status_error0; +__le16 l2tag1; +__le16 flex_meta0; +__le16 flex_meta1; + } wb;/* writeback */ +}; + +union iavf_32b_rx_flex_desc { +struct { +__le64 pkt_addr;/* Packet buffer address */ +__le64 hdr_addr;/* Header buffer address */ +/* bit 0 of hdr_addr is DD bit */ +__le64 rsvd1; +__le64 rsvd2; + } read; +struct { +/* Qword 0 */ +u8 rxdid;/* descriptor builder profile ID */ +u8 mir_id_umb_cast;/* mirror=[5:0], umb=[7:6] */ +__le16 ptype_flex_flags0;/* ptype=[9:0], ff0=[15:10] */ +__le16 pkt_len;/* [15:14] are reserved */ +__le16 hdr_len_sph_flex_flags1;/* header=[10:0] */ +/* sph=[11:11] */ +/* ff1/ext=[15:12] */ + +/* Qword 1 */ +__le16 status_error0; +__le16 l2tag1; +__le16 flex_meta0; +__le16 flex_meta1; + +/* Qword 2 */ +__le16 status_error1; +u8 flex_flags2; +u8 time_stamp_low; +__le16 l2tag2_1st; +__le16 l2tag2_2nd; + +/* Qword 3 */ +__le16 flex_meta2; +__le16 flex_meta3; +union { +struct { +__le16 flex_meta4; +__le16 flex_meta5; + } flex; +__le32 ts_high; + } flex_ts; + } wb;/* writeback */ +}; + +/* Rx Flex Descriptor +* RxDID Profile ID 16-21 +* Flex-field 0: RSS hash lower 16-bits +* Flex-field 1: RSS hash upper 16-bits +* Flex-field 2: Flow ID lower 16-bits +* Flex-field 3: Flow ID upper 16-bits +* Flex-field 4: AUX0 +* Flex-field 5: AUX1 +*/ +struct iavf_32b_rx_flex_desc_comms { +/* Qword 0 */ +u8 rxdid; +u8 mir_id_umb_cast; +__le16 ptype_flexi_flags0; +__le16 pkt_len; +__le16 hdr_len_sph_flex_flags1; + +/* Qword 1 */ +__le16 status_error0; +__le16 l2tag1; +__le32 rss_hash; + +/* Qword 2 */ +__le16 status_error1; +u8 flexi_flags2; +u8 ts_low; +__le16 l2tag2_1st; +__le16 l2tag2_2nd; + +/* Qword 3 */ +__le32 flow_id; +union { +struct { +__le16 aux0; +__le16 aux1; + } flex; +__le32 ts_high; + } flex_ts; +}; + +/* Rx Flex Descriptor +* RxDID Profile ID 22-23 (swap Hash and FlowID) +* Flex-field 0: Flow ID lower 16-bits +* Flex-field 1: Flow ID upper 16-bits +* Flex-field 2: RSS hash lower 16-bits +* Flex-field 3: RSS hash upper 16-bits +* Flex-field 4: AUX0 +* Flex-field 5: AUX1 +*/ +struct iavf_32b_rx_flex_desc_comms_ovs { +/* Qword 0 */ +u8 rxdid; +u8 mir_id_umb_cast; +__le16 ptype_flexi_flags0; +__le16 pkt_len; +__le16 hdr_len_sph_flex_flags1; + +/* Qword 1 */ +__le16 status_error0; +__le16 l2tag1; +__le32 flow_id; + +/* Qword 2 */ +__le16 status_error1; +u8 flexi_flags2; +u8 ts_low; +__le16 l2tag2_1st; +__le16 l2tag2_2nd; + +/* Qword 3 */ +__le32 rss_hash; +union { +struct { +__le16 aux0; +__le16 aux1; + } flex; +__le32 ts_high; + } flex_ts; +}; + +/* Receive Flex Descriptor profile IDs: There are a total +* of 64 profiles where profile IDs 0/1 are for legacy; and +* profiles 2-63 are flex profiles that can be programmed +* with a specific metadata (profile 7 reserved for HW) +*/ +enum iavf_rxdid { +IAVF_RXDID_LEGACY_0 =0 , +IAVF_RXDID_LEGACY_1 =1 , +IAVF_RXDID_FLEX_NIC =2 , +IAVF_RXDID_FLEX_NIC_2 =6 , +IAVF_RXDID_HW =7 , +IAVF_RXDID_COMMS_GENERIC =16 , +IAVF_RXDID_COMMS_AUX_VLAN =17 , +IAVF_RXDID_COMMS_AUX_IPV4 =18 , +IAVF_RXDID_COMMS_AUX_IPV6 =19 , +IAVF_RXDID_COMMS_AUX_IPV6_FLOW =20 , +IAVF_RXDID_COMMS_AUX_TCP =21 , +IAVF_RXDID_COMMS_OVS_1 =22 , +IAVF_RXDID_COMMS_OVS_2 =23 , +IAVF_RXDID_LAST =63 , +}; + +enum iavf_rx_flex_desc_status_error_0_bits { +/* Note: These are predefined bit offsets */ +IAVF_RX_FLEX_DESC_STATUS0_DD_S =0 , +IAVF_RX_FLEX_DESC_STATUS0_EOF_S , +IAVF_RX_FLEX_DESC_STATUS0_HBO_S , +IAVF_RX_FLEX_DESC_STATUS0_L3L4P_S , +IAVF_RX_FLEX_DESC_STATUS0_XSUM_IPE_S , +IAVF_RX_FLEX_DESC_STATUS0_XSUM_L4E_S , +IAVF_RX_FLEX_DESC_STATUS0_XSUM_EIPE_S , +IAVF_RX_FLEX_DESC_STATUS0_XSUM_EUDPE_S , +IAVF_RX_FLEX_DESC_STATUS0_LPBK_S , +IAVF_RX_FLEX_DESC_STATUS0_IPV6EXADD_S , +IAVF_RX_FLEX_DESC_STATUS0_RXE_S , +IAVF_RX_FLEX_DESC_STATUS0_CRCP_S , +IAVF_RX_FLEX_DESC_STATUS0_RSS_VALID_S , +IAVF_RX_FLEX_DESC_STATUS0_L2TAG1P_S , +IAVF_RX_FLEX_DESC_STATUS0_XTRMD0_VALID_S , +IAVF_RX_FLEX_DESC_STATUS0_XTRMD1_VALID_S , +IAVF_RX_FLEX_DESC_STATUS0_LAST /* this entry must be last!!! */ +}; + +/* for iavf_32b_rx_flex_desc.ptype_flex_flags0 member */ +#define IAVF_RX_FLEX_DESC_PTYPE_M (0x3FF)/* 10-bits */ + +/* for iavf_32b_rx_flex_desc.pkt_len member */ +#define IAVF_RX_FLX_DESC_PKT_LEN_M (0x3FFF)/* 14-bits */ + int iavf_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, uint16_t nb_desc,
Signature · 1 line
2.17.1
PATCH v5 2/11 net/iavf: return error if opcode is mismatched
In reply to Leyi RongAdds error return when the opcode of read message is mismatched which is received from adminQ.
--- drivers/net/iavf/iavf_vchnl.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
drivers/net/iavf/iavf_vchnl.c +3 -1
@@ -52,9 +52,11 @@ iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len,PMD_DRV_LOG (DEBUG ,"AQ from pf carries opcode %u, retval %d" , opcode, vf->cmd_retval); -if (opcode != vf->pend_cmd) +if (opcode != vf->pend_cmd) {PMD_DRV_LOG (WARNING ,"command mismatch, expect %u, get %u" , vf->pend_cmd, opcode); +return IAVF_ERR_OPCODE_MISMATCH ; + }return IAVF_SUCCESS ; }
Signature · 1 line
2.17.1
PATCH v5 3/11 net/iavf: flexible Rx descriptor support in normal path
In reply to Leyi RongSupport flexible Rx descriptor format in normal
path of iAVF PMD.
--- drivers/net/iavf/iavf.h | 2 + drivers/net/iavf/iavf_ethdev.c | 8 + drivers/net/iavf/iavf_rxtx.c | 485 ++++++++++++++++++++++++++++++--- drivers/net/iavf/iavf_rxtx.h | 9 + drivers/net/iavf/iavf_vchnl.c | 55 +++- 5 files changed, 521 insertions(+), 38 deletions(-)
drivers/net/iavf/iavf.h +2 -0
@@ -103,6 +103,7 @@ struct iavf_info {struct virtchnl_version_info virtchnl_version;struct virtchnl_vf_resource *vf_res;/* VF resource */ struct virtchnl_vsi_resource *vsi_res;/* LAN VSI */ +uint64_t supported_rxdid;volatile enum virtchnl_ops pend_cmd;/* pending command not finished */ uint32_t cmd_retval;/* return value of the cmd response from PF */
@@ -235,6 +236,7 @@ int iavf_disable_queues(struct iavf_adapter *adapter);int iavf_configure_rss_lut (struct iavf_adapter *adapter);int iavf_configure_rss_key (struct iavf_adapter *adapter);int iavf_configure_queues (struct iavf_adapter *adapter); +int iavf_get_supported_rxdid (struct iavf_adapter *adapter);int iavf_config_irq_map (struct iavf_adapter *adapter);void iavf_add_del_all_mac_addr (struct iavf_adapter *adapter,bool add); int iavf_dev_link_update(struct rte_eth_dev *dev,
drivers/net/iavf/iavf_ethdev.c +8 -0
@@ -1245,6 +1245,14 @@ iavf_init_vf(struct rte_eth_dev *dev) goto err_rss; } } + +if (vf->vf_res->vf_cap_flags &VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC ) { +if (iavf_get_supported_rxdid (adapter) !=0 ) { +PMD_INIT_LOG (ERR ,"failed to do get supported rxdid" ); + goto err_rss; + } + } +return 0 ; err_rss:rte_free (vf->rss_key);
drivers/net/iavf/iavf_rxtx.c +448 -37
552 lines not on this page · raw message
drivers/net/iavf/iavf_rxtx.h +9 -0
@@ -60,8 +60,10 @@/* HW desc structure, both 16-byte and 32-byte types are supported */ #ifdef RTE_LIBRTE_IAVF_16BYTE_RX_DESC #define iavf_rx_desc iavf_16byte_rx_desc +#define iavf_rx_flex_desc iavf_16b_rx_flex_desc#else #define iavf_rx_desc iavf_32byte_rx_desc +#define iavf_rx_flex_desc iavf_32b_rx_flex_desc#endif struct iavf_rxq_ops {
@@ -87,6 +89,7 @@ struct iavf_rx_queue {struct rte_mbuf *pkt_first_seg;/* first segment of current packet */ struct rte_mbuf *pkt_last_seg;/* last segment of current packet */ struct rte_mbuf fake_mbuf;/* dummy mbuf */ +uint8_t rxdid;/* used for VPMD */ uint16_t rxrearm_nb;/* number of remaining to be re-armed */
@@ -379,9 +382,15 @@ void iavf_dev_tx_queue_release(void *txq);void iavf_stop_queues (struct rte_eth_dev *dev);uint16_t iavf_recv_pkts (void *rx_queue,struct rte_mbuf **rx_pkts,uint16_t nb_pkts); +uint16_t iavf_recv_pkts_flex_rxd (void *rx_queue, +struct rte_mbuf **rx_pkts, +uint16_t nb_pkts);uint16_t iavf_recv_scattered_pkts (void *rx_queue,struct rte_mbuf **rx_pkts,uint16_t nb_pkts); +uint16_t iavf_recv_scattered_pkts_flex_rxd (void *rx_queue, +struct rte_mbuf **rx_pkts, +uint16_t nb_pkts);uint16_t iavf_xmit_pkts (void *tx_queue,struct rte_mbuf **tx_pkts,uint16_t nb_pkts); uint16_t iavf_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
drivers/net/iavf/iavf_vchnl.c +54 -1
79 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH v5 4/11 net/iavf: flexible Rx descriptor support in AVX path
In reply to Leyi RongSupport flexible Rx descriptor format in AVX
path of iAVF PMD.
--- drivers/net/iavf/iavf_rxtx.c | 24 +- drivers/net/iavf/iavf_rxtx.h | 6 + drivers/net/iavf/iavf_rxtx_vec_avx2.c | 550 +++++++++++++++++++++++++- 3 files changed, 570 insertions(+), 10 deletions(-)
drivers/net/iavf/iavf_rxtx.c +18 -6
@@ -2087,16 +2087,28 @@ iavf_set_rx_function(struct rte_eth_dev *dev)"Using %sVector Scattered Rx (port %d)." , use_avx2 ?"avx2 " :"" , dev->data->port_id); - dev->rx_pkt_burst = use_avx2 ? - iavf_recv_scattered_pkts_vec_avx2 : - iavf_recv_scattered_pkts_vec; +if (vf->vf_res->vf_cap_flags & +VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC ) + dev->rx_pkt_burst = use_avx2 ? + iavf_recv_scattered_pkts_vec_avx2_flex_rxd : + iavf_recv_scattered_pkts_vec; +else + dev->rx_pkt_burst = use_avx2 ? + iavf_recv_scattered_pkts_vec_avx2 : + iavf_recv_scattered_pkts_vec; }else {PMD_DRV_LOG (DEBUG ,"Using %sVector Rx (port %d)." , use_avx2 ?"avx2 " :"" , dev->data->port_id); - dev->rx_pkt_burst = use_avx2 ? - iavf_recv_pkts_vec_avx2 : - iavf_recv_pkts_vec; +if (vf->vf_res->vf_cap_flags & +VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC ) + dev->rx_pkt_burst = use_avx2 ? + iavf_recv_pkts_vec_avx2_flex_rxd : + iavf_recv_pkts_vec; +else + dev->rx_pkt_burst = use_avx2 ? + iavf_recv_pkts_vec_avx2 : + iavf_recv_pkts_vec; }return ;
drivers/net/iavf/iavf_rxtx.h +6 -0
@@ -414,9 +414,15 @@ uint16_t iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts);uint16_t iavf_recv_pkts_vec_avx2 (void *rx_queue,struct rte_mbuf **rx_pkts,uint16_t nb_pkts); +uint16_t iavf_recv_pkts_vec_avx2_flex_rxd (void *rx_queue, +struct rte_mbuf **rx_pkts, +uint16_t nb_pkts);uint16_t iavf_recv_scattered_pkts_vec_avx2 (void *rx_queue,struct rte_mbuf **rx_pkts,uint16_t nb_pkts); +uint16_t iavf_recv_scattered_pkts_vec_avx2_flex_rxd (void *rx_queue, +struct rte_mbuf **rx_pkts, +uint16_t nb_pkts);uint16_t iavf_xmit_pkts_vec (void *tx_queue,struct rte_mbuf **tx_pkts,uint16_t nb_pkts); uint16_t iavf_xmit_pkts_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts,
drivers/net/iavf/iavf_rxtx_vec_avx2.c +546 -4
585 lines not on this page · raw message
Signature · 1 line
2.17.1
Re: [dpdk-dev] [PATCH v5 04/11] net/iavf: flexible Rx descriptor support in AVX path
Quoted 4 lines · Leyi Rong
Support flexible Rx descriptor format in AVX
path of iAVF PMD.Signed-off-by Leyi Rong <[email protected]>
<...>
Quoted 9 lines
drivers/net/iavf/iavf_rxtx_vec_avx2.c +1
-1@@ -11,14 +11,16 @@#endif static inline void -iavf_rxq_rearm (struct iavf_rx_queue *rxq) +iavf_rxq_rearm (struct iavf_rx_queue *rxq,volatile union iavf_rx_desc *rxdp) {
drivers/net/iavf/iavf_rxtx_vec_avx2.c line 23
Quoted 10 lines
drivers/net/iavf/iavf_rxtx_vec_avx2.c +4
-2int i;uint16_t rx_id; -volatile union iavf_rx_desc *rxdp;struct rte_mbuf **rxp = &rxq->sw_ring[rxq->rxrearm_start]; - rxdp = rxq->rx_ring + rxq->rxrearm_start; +if (rxq->rxdid ==IAVF_RXDID_COMMS_OVS_1 ) { +volatile union iavf_rx_flex_desc *rxdp = + (union iavf_rx_flex_desc *)rxdp; + }
Hi Leyi,
This code is causing a build error in icc [1] but what this code exactly does? It is creating a local version of the variable 'rxdp', which is confusing, but that variable's scope is only if block, so it looks like this code really doesn't do anything at all, am I missing something?
[1]
.../drivers/net/iavf/iavf_rxtx_vec_avx2.c(22): warning #592: variable "rxdp" is used before its value is set
(union iavf_rx_flex_desc *)rxdp;
PATCH v5 5/11 net/iavf: flexible Rx descriptor support in SSE path
In reply to Leyi RongSupport flexible Rx descriptor format in SSE
path of iAVF PMD.
--- drivers/net/iavf/iavf_rxtx.c | 4 +- drivers/net/iavf/iavf_rxtx.h | 5 + drivers/net/iavf/iavf_rxtx_vec_sse.c | 414 +++++++++++++++++++++++++++ 3 files changed, 421 insertions(+), 2 deletions(-)
drivers/net/iavf/iavf_rxtx.c +2 -2
@@ -2091,7 +2091,7 @@ iavf_set_rx_function(struct rte_eth_dev *dev) VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) dev->rx_pkt_burst = use_avx2 ? iavf_recv_scattered_pkts_vec_avx2_flex_rxd : - iavf_recv_scattered_pkts_vec; + iavf_recv_scattered_pkts_vec_flex_rxd; else dev->rx_pkt_burst = use_avx2 ? iavf_recv_scattered_pkts_vec_avx2 :
@@ -2104,7 +2104,7 @@ iavf_set_rx_function(struct rte_eth_dev *dev) VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) dev->rx_pkt_burst = use_avx2 ? iavf_recv_pkts_vec_avx2_flex_rxd : - iavf_recv_pkts_vec; + iavf_recv_pkts_vec_flex_rxd; else dev->rx_pkt_burst = use_avx2 ? iavf_recv_pkts_vec_avx2 :
drivers/net/iavf/iavf_rxtx.h +5 -0
@@ -407,9 +407,14 @@ int iavf_dev_tx_desc_status(void *tx_queue, uint16_t offset);uint16_t iavf_recv_pkts_vec (void *rx_queue,struct rte_mbuf **rx_pkts,uint16_t nb_pkts); +uint16_t iavf_recv_pkts_vec_flex_rxd (void *rx_queue,struct rte_mbuf **rx_pkts, +uint16_t nb_pkts);uint16_t iavf_recv_scattered_pkts_vec (void *rx_queue,struct rte_mbuf **rx_pkts,uint16_t nb_pkts); +uint16_t iavf_recv_scattered_pkts_vec_flex_rxd (void *rx_queue, +struct rte_mbuf **rx_pkts, +uint16_t nb_pkts);uint16_t iavf_xmit_fixed_burst_vec (void *tx_queue,struct rte_mbuf **tx_pkts,uint16_t nb_pkts); uint16_t iavf_recv_pkts_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,
drivers/net/iavf/iavf_rxtx_vec_sse.c +414 -0
444 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH v5 6/11 net/iavf: add flow director enabled switch value
In reply to Leyi RongThe commit adds fdir_enabled flag into iavf_rx_queue structure to identify if fdir id is active. Rx data path can be benefit if fdir id parsing is not needed, especially in vector path.
--- drivers/net/iavf/iavf.h | 1 + drivers/net/iavf/iavf_rxtx.h | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+)
drivers/net/iavf/iavf.h +1 -0
@@ -144,6 +144,7 @@ struct iavf_adapter {bool tx_vec_allowed;const uint32_t *ptype_tbl;bool stopped; +uint16_t fdir_ref_cnt; };/* IAVF_DEV_PRIVATE_TO */
drivers/net/iavf/iavf_rxtx.h +30 -0
@@ -104,6 +104,7 @@ struct iavf_rx_queue {uint16_t port_id;/* device port ID */ uint8_t crc_len;/* 0 if CRC stripped, 4 otherwise */ +uint8_t fdir_enabled;/* 0 if FDIR disabled, 1 when enabled */ uint16_t queue_id;/* Rx queue index */ uint16_t rx_buf_len;/* The packet buffer size */ uint16_t rx_hdr_len;/* The header buffer size */
@@ -491,6 +492,35 @@ void iavf_dump_tx_descriptor(const struct iavf_tx_queue *txq, tx_desc->cmd_type_offset_bsz); } +#define FDIR_PROC_ENABLE_PER_QUEUE (ad, on) do { \ + int i; \ + for (i = 0; i < (ad)->eth_dev->data->nb_rx_queues; i++) { \ + struct iavf_rx_queue *rxq = (ad)->eth_dev->data->rx_queues[i]; \ + if (!rxq) \ + continue; \ + rxq->fdir_enabled = on; \ + } \ + PMD_DRV_LOG(DEBUG, "FDIR processing on RX set to %d", on); \ +} while (0) + +/* Enable/disable flow director Rx processing in data path. */ +static inline +void iavf_fdir_rx_proc_enable (struct iavf_adapter *ad,bool on) +{ +if (on) { +/* enable flow director processing */ +if (ad->fdir_ref_cnt++ ==0 ) +FDIR_PROC_ENABLE_PER_QUEUE (ad, on); + }else { +if (ad->fdir_ref_cnt >=1 ) { + ad->fdir_ref_cnt--; + +if (ad->fdir_ref_cnt ==0 ) +FDIR_PROC_ENABLE_PER_QUEUE (ad, on); + } + } +} +#ifdef RTE_LIBRTE_IAVF_DEBUG_DUMP_DESC #define IAVF_DUMP_RX_DESC (rxq, desc, rx_id) \ iavf_dump_rx_descriptor(rxq, desc, rx_id)
Signature · 1 line
2.17.1
PATCH v5 7/11 net/iavf: support flow mark in normal data path
In reply to Leyi RongSupport Flow Director mark ID parsing in normal path.
--- drivers/net/iavf/iavf.h | 3 +++ drivers/net/iavf/iavf_rxtx.c | 51 +++++++++++++++++++++++++++++++----- 2 files changed, 47 insertions(+), 7 deletions(-)
drivers/net/iavf/iavf.h +3 -0
@@ -67,6 +67,9 @@#define IAVF_48_BIT_WIDTH (CHAR_BIT * 6)#define IAVF_48_BIT_MASK RTE_LEN2MASK(IAVF_48_BIT_WIDTH, uint64_t) +#define IAVF_RX_DESC_EXT_STATUS_FLEXBH_MASK 0x03 +#define IAVF_RX_DESC_EXT_STATUS_FLEXBH_FD_ID 0x01 +struct iavf_adapter ;struct iavf_rx_queue ;struct iavf_tx_queue ;
drivers/net/iavf/iavf_rxtx.c +44 -7
117 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH v5 8/11 net/iavf: support flow mark in AVX path
In reply to Leyi RongSupport Flow Director mark ID parsing from Flex
Rx descriptor in AVX path.
--- drivers/net/iavf/iavf_rxtx_vec_avx2.c | 72 +++++++++++++++++++++++++-- 1 file changed, 67 insertions(+), 5 deletions(-)
drivers/net/iavf/iavf_rxtx_vec_avx2.c +67 -5
@@ -616,6 +616,25 @@ _iavf_recv_raw_pkts_vec_avx2(struct iavf_rx_queue *rxq,return received; } +static inline __m256i +flex_rxd_to_fdir_flags_vec_avx2 (const __m256i fdir_id0_7) +{ +#define FDID_MIS_MAGIC 0xFFFFFFFF +RTE_BUILD_BUG_ON (PKT_RX_FDIR != (1 <<2 )); +RTE_BUILD_BUG_ON (PKT_RX_FDIR_ID != (1 <<13 )); +const __m256i pkt_fdir_bit =_mm256_set1_epi32 (PKT_RX_FDIR | +PKT_RX_FDIR_ID ); +/* desc->flow_id field == 0xFFFFFFFF means fdir mismatch */ +const __m256i fdir_mis_mask =_mm256_set1_epi32 (FDID_MIS_MAGIC ); +__m256i fdir_mask =_mm256_cmpeq_epi32 (fdir_id0_7, + fdir_mis_mask); +/* this XOR op results to bit-reverse the fdir_mask */ + fdir_mask =_mm256_xor_si256 (fdir_mask, fdir_mis_mask); +const __m256i fdir_flags =_mm256_and_si256 (fdir_mask, pkt_fdir_bit); + +return fdir_flags; +} + static inline uint16_t _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq, struct rte_mbuf **rx_pkts,
@@ -678,8 +697,8 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,const __m256i shuf_msk = _mm256_set_epi8 (/* first descriptor */ - 15, 14, - 13, 12, /* octet 12~15, 32 bits rss */ + 0xFF, 0xFF, + 0xFF, 0xFF, /* rss not supported */ 11, 10, /* octet 10~11, 16 bits vlan_macip */ 5, 4, /* octet 4~5, 16 bits data_len */ 0xFF, 0xFF, /* skip hi 16 bits pkt_len, zero out */
@@ -687,8 +706,8 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,0xFF ,0xFF ,/* pkt_type set as unknown */ 0xFF ,0xFF ,/*pkt_type set as unknown */ /* second descriptor */ -15 ,14 , -13 ,12 ,/* octet 12~15, 32 bits rss */ +0xFF ,0xFF , +0xFF ,0xFF ,/* rss not supported */ 11 ,10 ,/* octet 10~11, 16 bits vlan_macip */ 5 ,4 ,/* octet 4~5, 16 bits data_len */ 0xFF, 0xFF, /* skip hi 16 bits pkt_len, zero out */
@@ -930,8 +949,51 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq, rss_vlan_flag_bits);/* merge flags */ -const __m256i mbuf_flags =_mm256_or_si256 (l3_l4_flags, +__m256i mbuf_flags =_mm256_or_si256 (l3_l4_flags, rss_vlan_flags); + +if (rxq->fdir_enabled) { +const __m256i fdir_id4_7 = +_mm256_unpackhi_epi32 (raw_desc6_7, raw_desc4_5); + +const __m256i fdir_id0_3 = +_mm256_unpackhi_epi32 (raw_desc2_3, raw_desc0_1); + +const __m256i fdir_id0_7 = +_mm256_unpackhi_epi64 (fdir_id4_7, fdir_id0_3); + +const __m256i fdir_flags = +flex_rxd_to_fdir_flags_vec_avx2 (fdir_id0_7); + +/* merge with fdir_flags */ + mbuf_flags =_mm256_or_si256 (mbuf_flags, fdir_flags); + +/* write to mbuf: have to use scalar store here */ + rx_pkts[i +0 ]->hash.fdir.hi = +_mm256_extract_epi32 (fdir_id0_7,3 ); + + rx_pkts[i +1 ]->hash.fdir.hi = +_mm256_extract_epi32 (fdir_id0_7,7 ); + + rx_pkts[i +2 ]->hash.fdir.hi = +_mm256_extract_epi32 (fdir_id0_7,2 ); + + rx_pkts[i +3 ]->hash.fdir.hi = +_mm256_extract_epi32 (fdir_id0_7,6 ); + + rx_pkts[i +4 ]->hash.fdir.hi = +_mm256_extract_epi32 (fdir_id0_7,1 ); + + rx_pkts[i +5 ]->hash.fdir.hi = +_mm256_extract_epi32 (fdir_id0_7,5 ); + + rx_pkts[i +6 ]->hash.fdir.hi = +_mm256_extract_epi32 (fdir_id0_7,0 ); + + rx_pkts[i +7 ]->hash.fdir.hi = +_mm256_extract_epi32 (fdir_id0_7,4 ); + }/* if() on fdir_enabled */ +/** * At this point, we have the 8 sets of flags in the low 16-bits * of each 32-bit value in vlan0.
Signature · 1 line
2.17.1
PATCH v5 9/11 net/iavf: support flow mark in SSE path
In reply to Leyi RongSupport Flow Director mark ID parsing from Flex
Rx descriptor in SSE path.
--- drivers/net/iavf/iavf_rxtx_vec_sse.c | 52 +++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-)
drivers/net/iavf/iavf_rxtx_vec_sse.c +51 -1
@@ -189,6 +189,25 @@ desc_to_olflags_v(struct iavf_rx_queue *rxq, __m128i descs[4],_mm_store_si128 ((__m128i *)&rx_pkts[3 ]->rearm_data, rearm3); } +static inline __m128i +flex_rxd_to_fdir_flags_vec (const __m128i fdir_id0_3) +{ +#define FDID_MIS_MAGIC 0xFFFFFFFF +RTE_BUILD_BUG_ON (PKT_RX_FDIR != (1 <<2 )); +RTE_BUILD_BUG_ON (PKT_RX_FDIR_ID != (1 <<13 )); +const __m128i pkt_fdir_bit =_mm_set1_epi32 (PKT_RX_FDIR | +PKT_RX_FDIR_ID ); +/* desc->flow_id field == 0xFFFFFFFF means fdir mismatch */ +const __m128i fdir_mis_mask =_mm_set1_epi32 (FDID_MIS_MAGIC ); +__m128i fdir_mask =_mm_cmpeq_epi32 (fdir_id0_3, + fdir_mis_mask); +/* this XOR op results to bit-reverse the fdir_mask */ + fdir_mask =_mm_xor_si128 (fdir_mask, fdir_mis_mask); +const __m128i fdir_flags =_mm_and_si128 (fdir_mask, pkt_fdir_bit); + +return fdir_flags; +} +static inline void flex_desc_to_olflags_v (struct iavf_rx_queue *rxq,__m128i descs[4 ],struct rte_mbuf **rx_pkts)
@@ -267,6 +286,36 @@ flex_desc_to_olflags_v(struct iavf_rx_queue *rxq, __m128i descs[4],/* merge the flags */ flags =_mm_or_si128 (flags, rss_vlan); +if (rxq->fdir_enabled) { +const __m128i fdir_id0_1 = +_mm_unpackhi_epi32 (descs[0 ], descs[1 ]); + +const __m128i fdir_id2_3 = +_mm_unpackhi_epi32 (descs[2 ], descs[3 ]); + +const __m128i fdir_id0_3 = +_mm_unpackhi_epi64 (fdir_id0_1, fdir_id2_3); + +const __m128i fdir_flags = +flex_rxd_to_fdir_flags_vec (fdir_id0_3); + +/* merge with fdir_flags */ + flags =_mm_or_si128 (flags, fdir_flags); + +/* write fdir_id to mbuf */ + rx_pkts[0 ]->hash.fdir.hi = +_mm_extract_epi32 (fdir_id0_3,0 ); + + rx_pkts[1 ]->hash.fdir.hi = +_mm_extract_epi32 (fdir_id0_3,1 ); + + rx_pkts[2 ]->hash.fdir.hi = +_mm_extract_epi32 (fdir_id0_3,2 ); + + rx_pkts[3 ]->hash.fdir.hi = +_mm_extract_epi32 (fdir_id0_3,3 ); + }/* if() on fdir_enabled */ +/** * At this point, we have the 4 sets of flags in the low 16-bits * of each 32-bit value in flags.
@@ -604,7 +653,8 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,const __m128i zero =_mm_setzero_si128 ();/* mask to shuffle from desc. to mbuf */ const __m128i shuf_msk = _mm_set_epi8 - (15, 14, 13, 12, /* octet 12~15, 32 bits rss */ + (0xFF, 0xFF, + 0xFF, 0xFF, /* rss not supported */ 11, 10, /* octet 10~11, 16 bits vlan_macip */ 5, 4, /* octet 4~5, 16 bits data_len */ 0xFF, 0xFF, /* skip high 16 bits pkt_len, zero out */
Signature · 1 line
2.17.1
PATCH v5 10/11 net/iavf: add RSS hash parsing in AVX path
In reply to Leyi RongSupport RSS hash parsing from Flex Rx
descriptor in AVX data path.
--- drivers/net/iavf/iavf_rxtx_vec_avx2.c | 94 ++++++++++++++++++++++++++- 1 file changed, 92 insertions(+), 2 deletions(-)
drivers/net/iavf/iavf_rxtx_vec_avx2.c +92 -2
@@ -698,7 +698,7 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq, _mm256_set_epi8 (/* first descriptor */ 0xFF, 0xFF, - 0xFF, 0xFF, /* rss not supported */ + 0xFF, 0xFF, /* rss hash parsed separately */ 11, 10, /* octet 10~11, 16 bits vlan_macip */ 5, 4, /* octet 4~5, 16 bits data_len */ 0xFF, 0xFF, /* skip hi 16 bits pkt_len, zero out */
@@ -707,7 +707,7 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,0xFF ,0xFF ,/*pkt_type set as unknown */ /* second descriptor */ 0xFF ,0xFF , -0xFF ,0xFF ,/* rss not supported */ +0xFF ,0xFF ,/* rss hash parsed separately */ 11 ,10 ,/* octet 10~11, 16 bits vlan_macip */ 5 ,4 ,/* octet 4~5, 16 bits data_len */ 0xFF, 0xFF, /* skip hi 16 bits pkt_len, zero out */
@@ -994,6 +994,96 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,_mm256_extract_epi32 (fdir_id0_7,4 ); }/* if() on fdir_enabled */ +#ifndef RTE_LIBRTE_IAVF_16BYTE_RX_DESC +/** +* needs to load 2nd 16B of each desc for RSS hash parsing, +* will cause performance drop to get into this context. +*/ +if (rxq->vsi->adapter->eth_dev->data->dev_conf.rxmode.offloads & +DEV_RX_OFFLOAD_RSS_HASH ) { +/* load bottom half of every 32B desc */ +const __m128i raw_desc_bh7 = +_mm_load_si128 + ((void *)(&rxdp[7 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh6 = +_mm_load_si128 + ((void *)(&rxdp[6 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh5 = +_mm_load_si128 + ((void *)(&rxdp[5 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh4 = +_mm_load_si128 + ((void *)(&rxdp[4 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh3 = +_mm_load_si128 + ((void *)(&rxdp[3 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh2 = +_mm_load_si128 + ((void *)(&rxdp[2 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh1 = +_mm_load_si128 + ((void *)(&rxdp[1 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh0 = +_mm_load_si128 + ((void *)(&rxdp[0 ].wb.status_error1)); + +__m256i raw_desc_bh6_7 = +_mm256_inserti128_si256 + (_mm256_castsi128_si256 (raw_desc_bh6), + raw_desc_bh7,1 ); +__m256i raw_desc_bh4_5 = +_mm256_inserti128_si256 + (_mm256_castsi128_si256 (raw_desc_bh4), + raw_desc_bh5,1 ); +__m256i raw_desc_bh2_3 = +_mm256_inserti128_si256 + (_mm256_castsi128_si256 (raw_desc_bh2), + raw_desc_bh3,1 ); +__m256i raw_desc_bh0_1 = +_mm256_inserti128_si256 + (_mm256_castsi128_si256 (raw_desc_bh0), + raw_desc_bh1,1 ); + +/** +* to shift the 32b RSS hash value to the +* highest 32b of each 128b before mask +*/ +__m256i rss_hash6_7 = +_mm256_slli_epi64 (raw_desc_bh6_7,32 ); +__m256i rss_hash4_5 = +_mm256_slli_epi64 (raw_desc_bh4_5,32 ); +__m256i rss_hash2_3 = +_mm256_slli_epi64 (raw_desc_bh2_3,32 ); +__m256i rss_hash0_1 = +_mm256_slli_epi64 (raw_desc_bh0_1,32 ); + +__m256i rss_hash_msk = +_mm256_set_epi32 (0xFFFFFFFF ,0 ,0 ,0 , +0xFFFFFFFF ,0 ,0 ,0 ); + + rss_hash6_7 =_mm256_and_si256 + (rss_hash6_7, rss_hash_msk); + rss_hash4_5 =_mm256_and_si256 + (rss_hash4_5, rss_hash_msk); + rss_hash2_3 =_mm256_and_si256 + (rss_hash2_3, rss_hash_msk); + rss_hash0_1 =_mm256_and_si256 + (rss_hash0_1, rss_hash_msk); + + mb6_7 =_mm256_or_si256 (mb6_7, rss_hash6_7); + mb4_5 =_mm256_or_si256 (mb4_5, rss_hash4_5); + mb2_3 =_mm256_or_si256 (mb2_3, rss_hash2_3); + mb0_1 =_mm256_or_si256 (mb0_1, rss_hash0_1); + }/* if() on RSS hash parsing */ +#endif +/** * At this point, we have the 8 sets of flags in the low 16-bits * of each 32-bit value in vlan0.
Signature · 1 line
2.17.1
PATCH v5 11/11 net/iavf: add RSS hash parsing in SSE path
In reply to Leyi RongSupport RSS hash parsing from Flex Rx
descriptor in SSE data path.
--- drivers/net/iavf/iavf_rxtx_vec_sse.c | 88 +++++++++++++++++++++++----- 1 file changed, 72 insertions(+), 16 deletions(-)
drivers/net/iavf/iavf_rxtx_vec_sse.c +72 -16
@@ -654,7 +654,7 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,/* mask to shuffle from desc. to mbuf */ const __m128i shuf_msk = _mm_set_epi8 (0xFF, 0xFF, - 0xFF, 0xFF, /* rss not supported */ + 0xFF, 0xFF, /* rss hash parsed separately */ 11, 10, /* octet 10~11, 16 bits vlan_macip */ 5, 4, /* octet 4~5, 16 bits data_len */ 0xFF, 0xFF, /* skip high 16 bits pkt_len, zero out */
@@ -745,7 +745,7 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq, pos += IAVF_VPMD_DESCS_PER_LOOP, rxdp += IAVF_VPMD_DESCS_PER_LOOP) {__m128i descs[IAVF_VPMD_DESCS_PER_LOOP ]; -__m128i pkt_mb1, pkt_mb2, pkt_mb3, pkt_mb4; +__m128i pkt_mb0, pkt_mb1, pkt_mb2, pkt_mb3;__m128i staterr, sterr_tmp1, sterr_tmp2;/* 2 64 bit or 4 32 bit mbuf pointers in one XMM reg. */ __m128i mbp1;
@@ -791,8 +791,12 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,rte_compiler_barrier ();/* D.1 pkt 3,4 convert format from desc to pktmbuf */ - pkt_mb4 =_mm_shuffle_epi8 (descs[3 ], shuf_msk); - pkt_mb3 =_mm_shuffle_epi8 (descs[2 ], shuf_msk); + pkt_mb3 =_mm_shuffle_epi8 (descs[3 ], shuf_msk); + pkt_mb2 =_mm_shuffle_epi8 (descs[2 ], shuf_msk); + +/* D.1 pkt 1,2 convert format from desc to pktmbuf */ + pkt_mb1 =_mm_shuffle_epi8 (descs[1 ], shuf_msk); + pkt_mb0 =_mm_shuffle_epi8 (descs[0 ], shuf_msk);/* C.1 4=>2 filter staterr info only */ sterr_tmp2 =_mm_unpackhi_epi32 (descs[3 ], descs[2 ]);
@@ -802,12 +806,68 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,flex_desc_to_olflags_v (rxq, descs, &rx_pkts[pos]);/* D.2 pkt 3,4 set in_port/nb_seg and remove crc */ - pkt_mb4 =_mm_add_epi16 (pkt_mb4, crc_adjust); pkt_mb3 =_mm_add_epi16 (pkt_mb3, crc_adjust); + pkt_mb2 =_mm_add_epi16 (pkt_mb2, crc_adjust); -/* D.1 pkt 1,2 convert format from desc to pktmbuf */ - pkt_mb2 =_mm_shuffle_epi8 (descs[1 ], shuf_msk); - pkt_mb1 =_mm_shuffle_epi8 (descs[0 ], shuf_msk); +/* D.2 pkt 1,2 set in_port/nb_seg and remove crc */ + pkt_mb1 =_mm_add_epi16 (pkt_mb1, crc_adjust); + pkt_mb0 =_mm_add_epi16 (pkt_mb0, crc_adjust); + +#ifndef RTE_LIBRTE_IAVF_16BYTE_RX_DESC +/** +* needs to load 2nd 16B of each desc for RSS hash parsing, +* will cause performance drop to get into this context. +*/ +if (rxq->vsi->adapter->eth_dev->data->dev_conf.rxmode.offloads & +DEV_RX_OFFLOAD_RSS_HASH ) { +/* load bottom half of every 32B desc */ +const __m128i raw_desc_bh3 = +_mm_load_si128 + ((void *)(&rxdp[3 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh2 = +_mm_load_si128 + ((void *)(&rxdp[2 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh1 = +_mm_load_si128 + ((void *)(&rxdp[1 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh0 = +_mm_load_si128 + ((void *)(&rxdp[0 ].wb.status_error1)); + +/** +* to shift the 32b RSS hash value to the +* highest 32b of each 128b before mask +*/ +__m128i rss_hash3 = +_mm_slli_epi64 (raw_desc_bh3,32 ); +__m128i rss_hash2 = +_mm_slli_epi64 (raw_desc_bh2,32 ); +__m128i rss_hash1 = +_mm_slli_epi64 (raw_desc_bh1,32 ); +__m128i rss_hash0 = +_mm_slli_epi64 (raw_desc_bh0,32 ); + +__m128i rss_hash_msk = +_mm_set_epi32 (0xFFFFFFFF ,0 ,0 ,0 ); + + rss_hash3 =_mm_and_si128 + (rss_hash3, rss_hash_msk); + rss_hash2 =_mm_and_si128 + (rss_hash2, rss_hash_msk); + rss_hash1 =_mm_and_si128 + (rss_hash1, rss_hash_msk); + rss_hash0 =_mm_and_si128 + (rss_hash0, rss_hash_msk); + + pkt_mb3 =_mm_or_si128 (pkt_mb3, rss_hash3); + pkt_mb2 =_mm_or_si128 (pkt_mb2, rss_hash2); + pkt_mb1 =_mm_or_si128 (pkt_mb1, rss_hash1); + pkt_mb0 =_mm_or_si128 (pkt_mb0, rss_hash0); + }/* if() on RSS hash parsing */ +#endif /* C.2 get 4 pkts staterr value */ staterr =_mm_unpacklo_epi32 (sterr_tmp1, sterr_tmp2);
@@ -815,14 +875,10 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,/* D.3 copy final 3,4 data to rx_pkts */ _mm_storeu_si128 ((void *)&rx_pkts[pos +3 ]->rx_descriptor_fields1, - pkt_mb4); + pkt_mb3);_mm_storeu_si128 ((void *)&rx_pkts[pos +2 ]->rx_descriptor_fields1, - pkt_mb3); - -/* D.2 pkt 1,2 set in_port/nb_seg and remove crc */ - pkt_mb2 =_mm_add_epi16 (pkt_mb2, crc_adjust); - pkt_mb1 =_mm_add_epi16 (pkt_mb1, crc_adjust); + pkt_mb2);/* C* extract and record EOP bit */ if (split_packet) {
@@ -846,9 +902,9 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,/* D.3 copy final 1,2 data to rx_pkts */ _mm_storeu_si128 ((void *)&rx_pkts[pos +1 ]->rx_descriptor_fields1, - pkt_mb2); + pkt_mb1);_mm_storeu_si128 ((void *)&rx_pkts[pos]->rx_descriptor_fields1, - pkt_mb1); + pkt_mb0);flex_desc_to_ptype_v (descs, &rx_pkts[pos], ptype_tbl);/* C.4 calc avaialbe number of desc */ var =__builtin_popcountll (_mm_cvtsi128_si64 (staterr));
Signature · 1 line
2.17.1
Quoted 71 lines
-----Original Message-----
From: Rong, Leyi <[email protected]>
Sent: Thursday, April 16, 2020 4:10 PM
To: Wu, Jingjing <[email protected]>; Zhang, Qi Z <[email protected]>; Xing, Beilei <[email protected]>; Ye, Xiaolong <[email protected]>
Cc: [email protected]; Rong, Leyi <[email protected]>
Subject: [PATCH v5 00/11] framework for advanced iAVF PMDThis patchset enable framework for advanced iAVF, includes flexible descriptor support, FDIR mark id and RSS hash support.
---
v5:
- Set rxdid = 0 iniavf_configure_queues()when it's supported andRTE_LIBRTE_IAVF_16BYTE_RX_DESCis y. So the error can be captured explicitly as kernel PF does not support it yet.
- Fix some build errors whenRTE_LIBRTE_IAVF_16BYTE_RX_DESCis y.v4:
- Avoid initial declaration in 'for' loop to backward-compatible with older gcc versions.v3:
- Remove patch to query DDP package info as it's specific to DCF.
- Specifyfdir_ref_cntas per VF value for enable/disable FDIR ID parse.
- Movefdir_enabledflag to per Queue value for cache benefit when access in vector routines.
- Store extractedflow_idvalue to mbuf without judgement to avoid branch speculation fail, which is benefit to performance.v2:
- Revert macroRTE_LIBRTE_IAVF_16BYTE_RX_DESCdeletion as it's defined in AVF spec.
- TypoRTE_LIBRTE_ICE_16BYTE_RX_DESCfix iniavf_rxtx.c.
- Move flex desc definitions intoiavf_rxtx.h.
- Up to date to match with the latest version ofvirtchnl.h.
- Extract a new internal funciavf_update_rx_tailto call.
- Remove iavf_dev_rxq_count_flex_rxd()/iavf_dev_rx_desc_status_flex_rxd(), as the accompanying legacy ones can deal with the flex desc cases.
- Moverxq->rxdidassignment fromiavf_configure_queues()toiavf_dev_rx_queue_setup().
- Unfold_mm_extract_epi32(fdir_id0_3, i)to fix build error when using GCC compile option -O0.Leyi Rong (11):
net/iavf: flexible Rx descriptor definitions net/iavf: return error if opcode is mismatched net/iavf: flexible Rx descriptor support in normal path net/iavf: flexible Rx descriptor support in AVX path net/iavf: flexible Rx descriptor support in SSE path net/iavf: add flow director enabled switch value net/iavf: support flow mark in normal data path net/iavf: support flow mark in AVX path net/iavf: support flow mark in SSE path net/iavf: add RSS hash parsing in AVX path net/iavf: add RSS hash parsing in SSE pathdrivers/net/iavf/iavf.h | 6 + drivers/net/iavf/iavf_ethdev.c | 8 + drivers/net/iavf/iavf_rxtx.c | 546 ++++++++++++++++++-- drivers/net/iavf/iavf_rxtx.h | 250 +++++++++ drivers/net/iavf/iavf_rxtx_vec_avx2.c | 702 +++++++++++++++++++++++++- drivers/net/iavf/iavf_rxtx_vec_sse.c | 520 +++++++++++++++++++ drivers/net/iavf/iavf_vchnl.c | 59 ++- 7 files changed, 2042 insertions(+), 49 deletions(-)
Quoted 2 lines
--
2.17.1
Ye Xiaolong
AppliedQuoted 71 lines · Qi Z
Quoted 71 lines
-----Original Message-----
From: Rong, Leyi <[email protected]>
Sent: Thursday, April 16, 2020 4:10 PM
To: Wu, Jingjing <[email protected]>; Zhang, Qi Z <[email protected]>; Xing, Beilei <[email protected]>; Ye, Xiaolong <[email protected]>
Cc: [email protected]; Rong, Leyi <[email protected]>
Subject: [PATCH v5 00/11] framework for advanced iAVF PMDThis patchset enable framework for advanced iAVF, includes flexible descriptor support, FDIR mark id and RSS hash support.
---
v5:
- Set rxdid = 0 iniavf_configure_queues()when it's supported andRTE_LIBRTE_IAVF_16BYTE_RX_DESCis y. So the error can be captured explicitly as kernel PF does not support it yet.
- Fix some build errors whenRTE_LIBRTE_IAVF_16BYTE_RX_DESCis y.v4:
- Avoid initial declaration in 'for' loop to backward-compatible with older gcc versions.v3:
- Remove patch to query DDP package info as it's specific to DCF.
- Specifyfdir_ref_cntas per VF value for enable/disable FDIR ID parse.
- Movefdir_enabledflag to per Queue value for cache benefit when access in vector routines.
- Store extractedflow_idvalue to mbuf without judgement to avoid branch speculation fail, which is benefit to performance.v2:
- Revert macroRTE_LIBRTE_IAVF_16BYTE_RX_DESCdeletion as it's defined in AVF spec.
- TypoRTE_LIBRTE_ICE_16BYTE_RX_DESCfix iniavf_rxtx.c.
- Move flex desc definitions intoiavf_rxtx.h.
- Up to date to match with the latest version ofvirtchnl.h.
- Extract a new internal funciavf_update_rx_tailto call.
- Remove iavf_dev_rxq_count_flex_rxd()/iavf_dev_rx_desc_status_flex_rxd(), as the accompanying legacy ones can deal with the flex desc cases.
- Moverxq->rxdidassignment fromiavf_configure_queues()toiavf_dev_rx_queue_setup().
- Unfold_mm_extract_epi32(fdir_id0_3, i)to fix build error when using GCC compile option -O0.Leyi Rong (11):
net/iavf: flexible Rx descriptor definitions net/iavf: return error if opcode is mismatched net/iavf: flexible Rx descriptor support in normal path net/iavf: flexible Rx descriptor support in AVX path net/iavf: flexible Rx descriptor support in SSE path net/iavf: add flow director enabled switch value net/iavf: support flow mark in normal data path net/iavf: support flow mark in AVX path net/iavf: support flow mark in SSE path net/iavf: add RSS hash parsing in AVX path net/iavf: add RSS hash parsing in SSE pathdrivers/net/iavf/iavf.h | 6 + drivers/net/iavf/iavf_ethdev.c | 8 + drivers/net/iavf/iavf_rxtx.c | 546 ++++++++++++++++++-- drivers/net/iavf/iavf_rxtx.h | 250 +++++++++ drivers/net/iavf/iavf_rxtx_vec_avx2.c | 702 +++++++++++++++++++++++++- drivers/net/iavf/iavf_rxtx_vec_sse.c | 520 +++++++++++++++++++ drivers/net/iavf/iavf_vchnl.c | 59 ++- 7 files changed, 2042 insertions(+), 49 deletions(-)
Quoted 6 lines
Quoted 2 lines
--
2.17.1Reviewed-by Qi Zhang <[email protected]>
Applied to dpdk-next-net-intel, Thanks.
v6 cover framework for advanced iAVF PMD
In reply to Leyi RongThis patchset enable framework for advanced iAVF, includes flexible descriptor support, FDIR mark id and RSS hash support.
---
v6:
- Remove some redundant code which may cause ICC build error.
v5:
- Set rxdid = 0 in iavf_configure_queues() when it's supported and RTE_LIBRTE_IAVF_16BYTE_RX_DESC is y. So the error can be captured explicitly as kernel PF does not support it yet.
- Fix some build errors when RTE_LIBRTE_IAVF_16BYTE_RX_DESC is y.
v4:
- Avoid initial declaration in 'for' loop to backward-compatible with older gcc versions.
v3:
- Remove patch to query DDP package info as it's specific to DCF.
- Specify fdir_ref_cnt as per VF value for enable/disable FDIR ID parse.
- Move fdir_enabled flag to per Queue value for cache benefit when access in vector routines.
- Store extracted flow_id value to mbuf without judgement to avoid branch speculation fail, which is benefit to performance.
v2:
- Revert macro RTE_LIBRTE_IAVF_16BYTE_RX_DESC deletion as it's defined in AVF spec.
- Typo RTE_LIBRTE_ICE_16BYTE_RX_DESC fix in iavf_rxtx.c.
- Move flex desc definitions into iavf_rxtx.h.
- Up to date to match with the latest version of virtchnl.h.
- Extract a new internal func iavf_update_rx_tail to call.
- Remove iavf_dev_rxq_count_flex_rxd()/iavf_dev_rx_desc_status_flex_rxd(), as the accompanying legacy ones can deal with the flex desc cases.
- Move rxq->rxdid assignment from iavf_configure_queues() to iavf_dev_rx_queue_setup().
- Unfold _mm_extract_epi32(fdir_id0_3, i) to fix build error when using GCC compile option -O0.
Leyi Rong (11):
net/iavf: flexible Rx descriptor definitions net/iavf: return error if opcode is mismatched net/iavf: flexible Rx descriptor support in normal path net/iavf: flexible Rx descriptor support in AVX path net/iavf: flexible Rx descriptor support in SSE path net/iavf: add flow director enabled switch value net/iavf: support flow mark in normal data path net/iavf: support flow mark in AVX path net/iavf: support flow mark in SSE path net/iavf: add RSS hash parsing in AVX path net/iavf: add RSS hash parsing in SSE path
drivers/net/iavf/iavf.h | 6 + drivers/net/iavf/iavf_ethdev.c | 8 + drivers/net/iavf/iavf_rxtx.c | 546 ++++++++++++++++++-- drivers/net/iavf/iavf_rxtx.h | 250 ++++++++++ drivers/net/iavf/iavf_rxtx_vec_avx2.c | 691 ++++++++++++++++++++++++++ drivers/net/iavf/iavf_rxtx_vec_sse.c | 520 +++++++++++++++++++ drivers/net/iavf/iavf_vchnl.c | 59 ++- 7 files changed, 2035 insertions(+), 45 deletions(-)
Signature · 1 line
2.17.1
PATCH v6 1/11 net/iavf: flexible Rx descriptor definitions
Add definitions for flexible Rx descriptor structures and macros.
--- drivers/net/iavf/iavf_rxtx.h | 200 +++++++++++++++++++++++++++++++++++ 1 file changed, 200 insertions(+)
drivers/net/iavf/iavf_rxtx.h +200 -0
@@ -157,6 +157,206 @@ union iavf_tx_offload { }; }; +/* Rx Flex Descriptors +* These descriptors are used instead of the legacy version descriptors +*/ +union iavf_16b_rx_flex_desc { +struct { +__le64 pkt_addr;/* Packet buffer address */ +__le64 hdr_addr;/* Header buffer address */ +/* bit 0 of hdr_addr is DD bit */ + } read; +struct { +/* Qword 0 */ +u8 rxdid;/* descriptor builder profile ID */ +u8 mir_id_umb_cast;/* mirror=[5:0], umb=[7:6] */ +__le16 ptype_flex_flags0;/* ptype=[9:0], ff0=[15:10] */ +__le16 pkt_len;/* [15:14] are reserved */ +__le16 hdr_len_sph_flex_flags1;/* header=[10:0] */ +/* sph=[11:11] */ +/* ff1/ext=[15:12] */ + +/* Qword 1 */ +__le16 status_error0; +__le16 l2tag1; +__le16 flex_meta0; +__le16 flex_meta1; + } wb;/* writeback */ +}; + +union iavf_32b_rx_flex_desc { +struct { +__le64 pkt_addr;/* Packet buffer address */ +__le64 hdr_addr;/* Header buffer address */ +/* bit 0 of hdr_addr is DD bit */ +__le64 rsvd1; +__le64 rsvd2; + } read; +struct { +/* Qword 0 */ +u8 rxdid;/* descriptor builder profile ID */ +u8 mir_id_umb_cast;/* mirror=[5:0], umb=[7:6] */ +__le16 ptype_flex_flags0;/* ptype=[9:0], ff0=[15:10] */ +__le16 pkt_len;/* [15:14] are reserved */ +__le16 hdr_len_sph_flex_flags1;/* header=[10:0] */ +/* sph=[11:11] */ +/* ff1/ext=[15:12] */ + +/* Qword 1 */ +__le16 status_error0; +__le16 l2tag1; +__le16 flex_meta0; +__le16 flex_meta1; + +/* Qword 2 */ +__le16 status_error1; +u8 flex_flags2; +u8 time_stamp_low; +__le16 l2tag2_1st; +__le16 l2tag2_2nd; + +/* Qword 3 */ +__le16 flex_meta2; +__le16 flex_meta3; +union { +struct { +__le16 flex_meta4; +__le16 flex_meta5; + } flex; +__le32 ts_high; + } flex_ts; + } wb;/* writeback */ +}; + +/* Rx Flex Descriptor +* RxDID Profile ID 16-21 +* Flex-field 0: RSS hash lower 16-bits +* Flex-field 1: RSS hash upper 16-bits +* Flex-field 2: Flow ID lower 16-bits +* Flex-field 3: Flow ID upper 16-bits +* Flex-field 4: AUX0 +* Flex-field 5: AUX1 +*/ +struct iavf_32b_rx_flex_desc_comms { +/* Qword 0 */ +u8 rxdid; +u8 mir_id_umb_cast; +__le16 ptype_flexi_flags0; +__le16 pkt_len; +__le16 hdr_len_sph_flex_flags1; + +/* Qword 1 */ +__le16 status_error0; +__le16 l2tag1; +__le32 rss_hash; + +/* Qword 2 */ +__le16 status_error1; +u8 flexi_flags2; +u8 ts_low; +__le16 l2tag2_1st; +__le16 l2tag2_2nd; + +/* Qword 3 */ +__le32 flow_id; +union { +struct { +__le16 aux0; +__le16 aux1; + } flex; +__le32 ts_high; + } flex_ts; +}; + +/* Rx Flex Descriptor +* RxDID Profile ID 22-23 (swap Hash and FlowID) +* Flex-field 0: Flow ID lower 16-bits +* Flex-field 1: Flow ID upper 16-bits +* Flex-field 2: RSS hash lower 16-bits +* Flex-field 3: RSS hash upper 16-bits +* Flex-field 4: AUX0 +* Flex-field 5: AUX1 +*/ +struct iavf_32b_rx_flex_desc_comms_ovs { +/* Qword 0 */ +u8 rxdid; +u8 mir_id_umb_cast; +__le16 ptype_flexi_flags0; +__le16 pkt_len; +__le16 hdr_len_sph_flex_flags1; + +/* Qword 1 */ +__le16 status_error0; +__le16 l2tag1; +__le32 flow_id; + +/* Qword 2 */ +__le16 status_error1; +u8 flexi_flags2; +u8 ts_low; +__le16 l2tag2_1st; +__le16 l2tag2_2nd; + +/* Qword 3 */ +__le32 rss_hash; +union { +struct { +__le16 aux0; +__le16 aux1; + } flex; +__le32 ts_high; + } flex_ts; +}; + +/* Receive Flex Descriptor profile IDs: There are a total +* of 64 profiles where profile IDs 0/1 are for legacy; and +* profiles 2-63 are flex profiles that can be programmed +* with a specific metadata (profile 7 reserved for HW) +*/ +enum iavf_rxdid { +IAVF_RXDID_LEGACY_0 =0 , +IAVF_RXDID_LEGACY_1 =1 , +IAVF_RXDID_FLEX_NIC =2 , +IAVF_RXDID_FLEX_NIC_2 =6 , +IAVF_RXDID_HW =7 , +IAVF_RXDID_COMMS_GENERIC =16 , +IAVF_RXDID_COMMS_AUX_VLAN =17 , +IAVF_RXDID_COMMS_AUX_IPV4 =18 , +IAVF_RXDID_COMMS_AUX_IPV6 =19 , +IAVF_RXDID_COMMS_AUX_IPV6_FLOW =20 , +IAVF_RXDID_COMMS_AUX_TCP =21 , +IAVF_RXDID_COMMS_OVS_1 =22 , +IAVF_RXDID_COMMS_OVS_2 =23 , +IAVF_RXDID_LAST =63 , +}; + +enum iavf_rx_flex_desc_status_error_0_bits { +/* Note: These are predefined bit offsets */ +IAVF_RX_FLEX_DESC_STATUS0_DD_S =0 , +IAVF_RX_FLEX_DESC_STATUS0_EOF_S , +IAVF_RX_FLEX_DESC_STATUS0_HBO_S , +IAVF_RX_FLEX_DESC_STATUS0_L3L4P_S , +IAVF_RX_FLEX_DESC_STATUS0_XSUM_IPE_S , +IAVF_RX_FLEX_DESC_STATUS0_XSUM_L4E_S , +IAVF_RX_FLEX_DESC_STATUS0_XSUM_EIPE_S , +IAVF_RX_FLEX_DESC_STATUS0_XSUM_EUDPE_S , +IAVF_RX_FLEX_DESC_STATUS0_LPBK_S , +IAVF_RX_FLEX_DESC_STATUS0_IPV6EXADD_S , +IAVF_RX_FLEX_DESC_STATUS0_RXE_S , +IAVF_RX_FLEX_DESC_STATUS0_CRCP_S , +IAVF_RX_FLEX_DESC_STATUS0_RSS_VALID_S , +IAVF_RX_FLEX_DESC_STATUS0_L2TAG1P_S , +IAVF_RX_FLEX_DESC_STATUS0_XTRMD0_VALID_S , +IAVF_RX_FLEX_DESC_STATUS0_XTRMD1_VALID_S , +IAVF_RX_FLEX_DESC_STATUS0_LAST /* this entry must be last!!! */ +}; + +/* for iavf_32b_rx_flex_desc.ptype_flex_flags0 member */ +#define IAVF_RX_FLEX_DESC_PTYPE_M (0x3FF)/* 10-bits */ + +/* for iavf_32b_rx_flex_desc.pkt_len member */ +#define IAVF_RX_FLX_DESC_PKT_LEN_M (0x3FFF)/* 14-bits */ + int iavf_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, uint16_t nb_desc,
Signature · 1 line
2.17.1
PATCH v6 2/11 net/iavf: return error if opcode is mismatched
In reply to Leyi RongAdds error return when the opcode of read message is mismatched which is received from adminQ.
--- drivers/net/iavf/iavf_vchnl.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
drivers/net/iavf/iavf_vchnl.c +3 -1
@@ -52,9 +52,11 @@ iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len,PMD_DRV_LOG (DEBUG ,"AQ from pf carries opcode %u, retval %d" , opcode, vf->cmd_retval); -if (opcode != vf->pend_cmd) +if (opcode != vf->pend_cmd) {PMD_DRV_LOG (WARNING ,"command mismatch, expect %u, get %u" , vf->pend_cmd, opcode); +return IAVF_ERR_OPCODE_MISMATCH ; + }return IAVF_SUCCESS ; }
Signature · 1 line
2.17.1
PATCH v6 3/11 net/iavf: flexible Rx descriptor support in normal path
In reply to Leyi RongSupport flexible Rx descriptor format in normal
path of iAVF PMD.
--- drivers/net/iavf/iavf.h | 2 + drivers/net/iavf/iavf_ethdev.c | 8 + drivers/net/iavf/iavf_rxtx.c | 485 ++++++++++++++++++++++++++++++--- drivers/net/iavf/iavf_rxtx.h | 9 + drivers/net/iavf/iavf_vchnl.c | 55 +++- 5 files changed, 521 insertions(+), 38 deletions(-)
drivers/net/iavf/iavf.h +2 -0
@@ -103,6 +103,7 @@ struct iavf_info {struct virtchnl_version_info virtchnl_version;struct virtchnl_vf_resource *vf_res;/* VF resource */ struct virtchnl_vsi_resource *vsi_res;/* LAN VSI */ +uint64_t supported_rxdid;volatile enum virtchnl_ops pend_cmd;/* pending command not finished */ uint32_t cmd_retval;/* return value of the cmd response from PF */
@@ -235,6 +236,7 @@ int iavf_disable_queues(struct iavf_adapter *adapter);int iavf_configure_rss_lut (struct iavf_adapter *adapter);int iavf_configure_rss_key (struct iavf_adapter *adapter);int iavf_configure_queues (struct iavf_adapter *adapter); +int iavf_get_supported_rxdid (struct iavf_adapter *adapter);int iavf_config_irq_map (struct iavf_adapter *adapter);void iavf_add_del_all_mac_addr (struct iavf_adapter *adapter,bool add); int iavf_dev_link_update(struct rte_eth_dev *dev,
drivers/net/iavf/iavf_ethdev.c +8 -0
@@ -1245,6 +1245,14 @@ iavf_init_vf(struct rte_eth_dev *dev) goto err_rss; } } + +if (vf->vf_res->vf_cap_flags &VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC ) { +if (iavf_get_supported_rxdid (adapter) !=0 ) { +PMD_INIT_LOG (ERR ,"failed to do get supported rxdid" ); + goto err_rss; + } + } +return 0 ; err_rss:rte_free (vf->rss_key);
drivers/net/iavf/iavf_rxtx.c +448 -37
552 lines not on this page · raw message
drivers/net/iavf/iavf_rxtx.h +9 -0
@@ -60,8 +60,10 @@/* HW desc structure, both 16-byte and 32-byte types are supported */ #ifdef RTE_LIBRTE_IAVF_16BYTE_RX_DESC #define iavf_rx_desc iavf_16byte_rx_desc +#define iavf_rx_flex_desc iavf_16b_rx_flex_desc#else #define iavf_rx_desc iavf_32byte_rx_desc +#define iavf_rx_flex_desc iavf_32b_rx_flex_desc#endif struct iavf_rxq_ops {
@@ -87,6 +89,7 @@ struct iavf_rx_queue {struct rte_mbuf *pkt_first_seg;/* first segment of current packet */ struct rte_mbuf *pkt_last_seg;/* last segment of current packet */ struct rte_mbuf fake_mbuf;/* dummy mbuf */ +uint8_t rxdid;/* used for VPMD */ uint16_t rxrearm_nb;/* number of remaining to be re-armed */
@@ -379,9 +382,15 @@ void iavf_dev_tx_queue_release(void *txq);void iavf_stop_queues (struct rte_eth_dev *dev);uint16_t iavf_recv_pkts (void *rx_queue,struct rte_mbuf **rx_pkts,uint16_t nb_pkts); +uint16_t iavf_recv_pkts_flex_rxd (void *rx_queue, +struct rte_mbuf **rx_pkts, +uint16_t nb_pkts);uint16_t iavf_recv_scattered_pkts (void *rx_queue,struct rte_mbuf **rx_pkts,uint16_t nb_pkts); +uint16_t iavf_recv_scattered_pkts_flex_rxd (void *rx_queue, +struct rte_mbuf **rx_pkts, +uint16_t nb_pkts);uint16_t iavf_xmit_pkts (void *tx_queue,struct rte_mbuf **tx_pkts,uint16_t nb_pkts); uint16_t iavf_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
drivers/net/iavf/iavf_vchnl.c +54 -1
79 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH v6 4/11 net/iavf: flexible Rx descriptor support in AVX path
In reply to Leyi RongSupport flexible Rx descriptor format in AVX
path of iAVF PMD.
--- drivers/net/iavf/iavf_rxtx.c | 24 +- drivers/net/iavf/iavf_rxtx.h | 6 + drivers/net/iavf/iavf_rxtx_vec_avx2.c | 539 ++++++++++++++++++++++++++ 3 files changed, 563 insertions(+), 6 deletions(-)
drivers/net/iavf/iavf_rxtx.c +18 -6
@@ -2087,16 +2087,28 @@ iavf_set_rx_function(struct rte_eth_dev *dev)"Using %sVector Scattered Rx (port %d)." , use_avx2 ?"avx2 " :"" , dev->data->port_id); - dev->rx_pkt_burst = use_avx2 ? - iavf_recv_scattered_pkts_vec_avx2 : - iavf_recv_scattered_pkts_vec; +if (vf->vf_res->vf_cap_flags & +VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC ) + dev->rx_pkt_burst = use_avx2 ? + iavf_recv_scattered_pkts_vec_avx2_flex_rxd : + iavf_recv_scattered_pkts_vec; +else + dev->rx_pkt_burst = use_avx2 ? + iavf_recv_scattered_pkts_vec_avx2 : + iavf_recv_scattered_pkts_vec; }else {PMD_DRV_LOG (DEBUG ,"Using %sVector Rx (port %d)." , use_avx2 ?"avx2 " :"" , dev->data->port_id); - dev->rx_pkt_burst = use_avx2 ? - iavf_recv_pkts_vec_avx2 : - iavf_recv_pkts_vec; +if (vf->vf_res->vf_cap_flags & +VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC ) + dev->rx_pkt_burst = use_avx2 ? + iavf_recv_pkts_vec_avx2_flex_rxd : + iavf_recv_pkts_vec; +else + dev->rx_pkt_burst = use_avx2 ? + iavf_recv_pkts_vec_avx2 : + iavf_recv_pkts_vec; }return ;
drivers/net/iavf/iavf_rxtx.h +6 -0
@@ -414,9 +414,15 @@ uint16_t iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts);uint16_t iavf_recv_pkts_vec_avx2 (void *rx_queue,struct rte_mbuf **rx_pkts,uint16_t nb_pkts); +uint16_t iavf_recv_pkts_vec_avx2_flex_rxd (void *rx_queue, +struct rte_mbuf **rx_pkts, +uint16_t nb_pkts);uint16_t iavf_recv_scattered_pkts_vec_avx2 (void *rx_queue,struct rte_mbuf **rx_pkts,uint16_t nb_pkts); +uint16_t iavf_recv_scattered_pkts_vec_avx2_flex_rxd (void *rx_queue, +struct rte_mbuf **rx_pkts, +uint16_t nb_pkts);uint16_t iavf_xmit_pkts_vec (void *tx_queue,struct rte_mbuf **tx_pkts,uint16_t nb_pkts); uint16_t iavf_xmit_pkts_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts,
drivers/net/iavf/iavf_rxtx_vec_avx2.c +539 -0
557 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH v6 5/11 net/iavf: flexible Rx descriptor support in SSE path
In reply to Leyi RongSupport flexible Rx descriptor format in SSE
path of iAVF PMD.
--- drivers/net/iavf/iavf_rxtx.c | 4 +- drivers/net/iavf/iavf_rxtx.h | 5 + drivers/net/iavf/iavf_rxtx_vec_sse.c | 414 +++++++++++++++++++++++++++ 3 files changed, 421 insertions(+), 2 deletions(-)
drivers/net/iavf/iavf_rxtx.c +2 -2
@@ -2091,7 +2091,7 @@ iavf_set_rx_function(struct rte_eth_dev *dev) VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) dev->rx_pkt_burst = use_avx2 ? iavf_recv_scattered_pkts_vec_avx2_flex_rxd : - iavf_recv_scattered_pkts_vec; + iavf_recv_scattered_pkts_vec_flex_rxd; else dev->rx_pkt_burst = use_avx2 ? iavf_recv_scattered_pkts_vec_avx2 :
@@ -2104,7 +2104,7 @@ iavf_set_rx_function(struct rte_eth_dev *dev) VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) dev->rx_pkt_burst = use_avx2 ? iavf_recv_pkts_vec_avx2_flex_rxd : - iavf_recv_pkts_vec; + iavf_recv_pkts_vec_flex_rxd; else dev->rx_pkt_burst = use_avx2 ? iavf_recv_pkts_vec_avx2 :
drivers/net/iavf/iavf_rxtx.h +5 -0
@@ -407,9 +407,14 @@ int iavf_dev_tx_desc_status(void *tx_queue, uint16_t offset);uint16_t iavf_recv_pkts_vec (void *rx_queue,struct rte_mbuf **rx_pkts,uint16_t nb_pkts); +uint16_t iavf_recv_pkts_vec_flex_rxd (void *rx_queue,struct rte_mbuf **rx_pkts, +uint16_t nb_pkts);uint16_t iavf_recv_scattered_pkts_vec (void *rx_queue,struct rte_mbuf **rx_pkts,uint16_t nb_pkts); +uint16_t iavf_recv_scattered_pkts_vec_flex_rxd (void *rx_queue, +struct rte_mbuf **rx_pkts, +uint16_t nb_pkts);uint16_t iavf_xmit_fixed_burst_vec (void *tx_queue,struct rte_mbuf **tx_pkts,uint16_t nb_pkts); uint16_t iavf_recv_pkts_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,
drivers/net/iavf/iavf_rxtx_vec_sse.c +414 -0
444 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH v6 6/11 net/iavf: add flow director enabled switch value
In reply to Leyi RongThe commit adds fdir_enabled flag into iavf_rx_queue structure to identify if fdir id is active. Rx data path can be benefit if fdir id parsing is not needed, especially in vector path.
--- drivers/net/iavf/iavf.h | 1 + drivers/net/iavf/iavf_rxtx.h | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+)
drivers/net/iavf/iavf.h +1 -0
@@ -144,6 +144,7 @@ struct iavf_adapter {bool tx_vec_allowed;const uint32_t *ptype_tbl;bool stopped; +uint16_t fdir_ref_cnt; };/* IAVF_DEV_PRIVATE_TO */
drivers/net/iavf/iavf_rxtx.h +30 -0
@@ -104,6 +104,7 @@ struct iavf_rx_queue {uint16_t port_id;/* device port ID */ uint8_t crc_len;/* 0 if CRC stripped, 4 otherwise */ +uint8_t fdir_enabled;/* 0 if FDIR disabled, 1 when enabled */ uint16_t queue_id;/* Rx queue index */ uint16_t rx_buf_len;/* The packet buffer size */ uint16_t rx_hdr_len;/* The header buffer size */
@@ -491,6 +492,35 @@ void iavf_dump_tx_descriptor(const struct iavf_tx_queue *txq, tx_desc->cmd_type_offset_bsz); } +#define FDIR_PROC_ENABLE_PER_QUEUE (ad, on) do { \ + int i; \ + for (i = 0; i < (ad)->eth_dev->data->nb_rx_queues; i++) { \ + struct iavf_rx_queue *rxq = (ad)->eth_dev->data->rx_queues[i]; \ + if (!rxq) \ + continue; \ + rxq->fdir_enabled = on; \ + } \ + PMD_DRV_LOG(DEBUG, "FDIR processing on RX set to %d", on); \ +} while (0) + +/* Enable/disable flow director Rx processing in data path. */ +static inline +void iavf_fdir_rx_proc_enable (struct iavf_adapter *ad,bool on) +{ +if (on) { +/* enable flow director processing */ +if (ad->fdir_ref_cnt++ ==0 ) +FDIR_PROC_ENABLE_PER_QUEUE (ad, on); + }else { +if (ad->fdir_ref_cnt >=1 ) { + ad->fdir_ref_cnt--; + +if (ad->fdir_ref_cnt ==0 ) +FDIR_PROC_ENABLE_PER_QUEUE (ad, on); + } + } +} +#ifdef RTE_LIBRTE_IAVF_DEBUG_DUMP_DESC #define IAVF_DUMP_RX_DESC (rxq, desc, rx_id) \ iavf_dump_rx_descriptor(rxq, desc, rx_id)
Signature · 1 line
2.17.1
PATCH v6 7/11 net/iavf: support flow mark in normal data path
In reply to Leyi RongSupport Flow Director mark ID parsing in normal path.
--- drivers/net/iavf/iavf.h | 3 +++ drivers/net/iavf/iavf_rxtx.c | 51 +++++++++++++++++++++++++++++++----- 2 files changed, 47 insertions(+), 7 deletions(-)
drivers/net/iavf/iavf.h +3 -0
@@ -67,6 +67,9 @@#define IAVF_48_BIT_WIDTH (CHAR_BIT * 6)#define IAVF_48_BIT_MASK RTE_LEN2MASK(IAVF_48_BIT_WIDTH, uint64_t) +#define IAVF_RX_DESC_EXT_STATUS_FLEXBH_MASK 0x03 +#define IAVF_RX_DESC_EXT_STATUS_FLEXBH_FD_ID 0x01 +struct iavf_adapter ;struct iavf_rx_queue ;struct iavf_tx_queue ;
drivers/net/iavf/iavf_rxtx.c +44 -7
117 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH v6 8/11 net/iavf: support flow mark in AVX path
In reply to Leyi RongSupport Flow Director mark ID parsing from Flex
Rx descriptor in AVX path.
--- drivers/net/iavf/iavf_rxtx_vec_avx2.c | 72 +++++++++++++++++++++++++-- 1 file changed, 67 insertions(+), 5 deletions(-)
drivers/net/iavf/iavf_rxtx_vec_avx2.c +67 -5
@@ -614,6 +614,25 @@ _iavf_recv_raw_pkts_vec_avx2(struct iavf_rx_queue *rxq,return received; } +static inline __m256i +flex_rxd_to_fdir_flags_vec_avx2 (const __m256i fdir_id0_7) +{ +#define FDID_MIS_MAGIC 0xFFFFFFFF +RTE_BUILD_BUG_ON (PKT_RX_FDIR != (1 <<2 )); +RTE_BUILD_BUG_ON (PKT_RX_FDIR_ID != (1 <<13 )); +const __m256i pkt_fdir_bit =_mm256_set1_epi32 (PKT_RX_FDIR | +PKT_RX_FDIR_ID ); +/* desc->flow_id field == 0xFFFFFFFF means fdir mismatch */ +const __m256i fdir_mis_mask =_mm256_set1_epi32 (FDID_MIS_MAGIC ); +__m256i fdir_mask =_mm256_cmpeq_epi32 (fdir_id0_7, + fdir_mis_mask); +/* this XOR op results to bit-reverse the fdir_mask */ + fdir_mask =_mm256_xor_si256 (fdir_mask, fdir_mis_mask); +const __m256i fdir_flags =_mm256_and_si256 (fdir_mask, pkt_fdir_bit); + +return fdir_flags; +} + static inline uint16_t _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq, struct rte_mbuf **rx_pkts,
@@ -675,8 +694,8 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,const __m256i shuf_msk = _mm256_set_epi8 (/* first descriptor */ - 15, 14, - 13, 12, /* octet 12~15, 32 bits rss */ + 0xFF, 0xFF, + 0xFF, 0xFF, /* rss not supported */ 11, 10, /* octet 10~11, 16 bits vlan_macip */ 5, 4, /* octet 4~5, 16 bits data_len */ 0xFF, 0xFF, /* skip hi 16 bits pkt_len, zero out */
@@ -684,8 +703,8 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,0xFF ,0xFF ,/* pkt_type set as unknown */ 0xFF ,0xFF ,/*pkt_type set as unknown */ /* second descriptor */ -15 ,14 , -13 ,12 ,/* octet 12~15, 32 bits rss */ +0xFF ,0xFF , +0xFF ,0xFF ,/* rss not supported */ 11 ,10 ,/* octet 10~11, 16 bits vlan_macip */ 5 ,4 ,/* octet 4~5, 16 bits data_len */ 0xFF, 0xFF, /* skip hi 16 bits pkt_len, zero out */
@@ -927,8 +946,51 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq, rss_vlan_flag_bits);/* merge flags */ -const __m256i mbuf_flags =_mm256_or_si256 (l3_l4_flags, +__m256i mbuf_flags =_mm256_or_si256 (l3_l4_flags, rss_vlan_flags); + +if (rxq->fdir_enabled) { +const __m256i fdir_id4_7 = +_mm256_unpackhi_epi32 (raw_desc6_7, raw_desc4_5); + +const __m256i fdir_id0_3 = +_mm256_unpackhi_epi32 (raw_desc2_3, raw_desc0_1); + +const __m256i fdir_id0_7 = +_mm256_unpackhi_epi64 (fdir_id4_7, fdir_id0_3); + +const __m256i fdir_flags = +flex_rxd_to_fdir_flags_vec_avx2 (fdir_id0_7); + +/* merge with fdir_flags */ + mbuf_flags =_mm256_or_si256 (mbuf_flags, fdir_flags); + +/* write to mbuf: have to use scalar store here */ + rx_pkts[i +0 ]->hash.fdir.hi = +_mm256_extract_epi32 (fdir_id0_7,3 ); + + rx_pkts[i +1 ]->hash.fdir.hi = +_mm256_extract_epi32 (fdir_id0_7,7 ); + + rx_pkts[i +2 ]->hash.fdir.hi = +_mm256_extract_epi32 (fdir_id0_7,2 ); + + rx_pkts[i +3 ]->hash.fdir.hi = +_mm256_extract_epi32 (fdir_id0_7,6 ); + + rx_pkts[i +4 ]->hash.fdir.hi = +_mm256_extract_epi32 (fdir_id0_7,1 ); + + rx_pkts[i +5 ]->hash.fdir.hi = +_mm256_extract_epi32 (fdir_id0_7,5 ); + + rx_pkts[i +6 ]->hash.fdir.hi = +_mm256_extract_epi32 (fdir_id0_7,0 ); + + rx_pkts[i +7 ]->hash.fdir.hi = +_mm256_extract_epi32 (fdir_id0_7,4 ); + }/* if() on fdir_enabled */ +/** * At this point, we have the 8 sets of flags in the low 16-bits * of each 32-bit value in vlan0.
Signature · 1 line
2.17.1
PATCH v6 9/11 net/iavf: support flow mark in SSE path
In reply to Leyi RongSupport Flow Director mark ID parsing from Flex
Rx descriptor in SSE path.
--- drivers/net/iavf/iavf_rxtx_vec_sse.c | 52 +++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-)
drivers/net/iavf/iavf_rxtx_vec_sse.c +51 -1
@@ -189,6 +189,25 @@ desc_to_olflags_v(struct iavf_rx_queue *rxq, __m128i descs[4],_mm_store_si128 ((__m128i *)&rx_pkts[3 ]->rearm_data, rearm3); } +static inline __m128i +flex_rxd_to_fdir_flags_vec (const __m128i fdir_id0_3) +{ +#define FDID_MIS_MAGIC 0xFFFFFFFF +RTE_BUILD_BUG_ON (PKT_RX_FDIR != (1 <<2 )); +RTE_BUILD_BUG_ON (PKT_RX_FDIR_ID != (1 <<13 )); +const __m128i pkt_fdir_bit =_mm_set1_epi32 (PKT_RX_FDIR | +PKT_RX_FDIR_ID ); +/* desc->flow_id field == 0xFFFFFFFF means fdir mismatch */ +const __m128i fdir_mis_mask =_mm_set1_epi32 (FDID_MIS_MAGIC ); +__m128i fdir_mask =_mm_cmpeq_epi32 (fdir_id0_3, + fdir_mis_mask); +/* this XOR op results to bit-reverse the fdir_mask */ + fdir_mask =_mm_xor_si128 (fdir_mask, fdir_mis_mask); +const __m128i fdir_flags =_mm_and_si128 (fdir_mask, pkt_fdir_bit); + +return fdir_flags; +} +static inline void flex_desc_to_olflags_v (struct iavf_rx_queue *rxq,__m128i descs[4 ],struct rte_mbuf **rx_pkts)
@@ -267,6 +286,36 @@ flex_desc_to_olflags_v(struct iavf_rx_queue *rxq, __m128i descs[4],/* merge the flags */ flags =_mm_or_si128 (flags, rss_vlan); +if (rxq->fdir_enabled) { +const __m128i fdir_id0_1 = +_mm_unpackhi_epi32 (descs[0 ], descs[1 ]); + +const __m128i fdir_id2_3 = +_mm_unpackhi_epi32 (descs[2 ], descs[3 ]); + +const __m128i fdir_id0_3 = +_mm_unpackhi_epi64 (fdir_id0_1, fdir_id2_3); + +const __m128i fdir_flags = +flex_rxd_to_fdir_flags_vec (fdir_id0_3); + +/* merge with fdir_flags */ + flags =_mm_or_si128 (flags, fdir_flags); + +/* write fdir_id to mbuf */ + rx_pkts[0 ]->hash.fdir.hi = +_mm_extract_epi32 (fdir_id0_3,0 ); + + rx_pkts[1 ]->hash.fdir.hi = +_mm_extract_epi32 (fdir_id0_3,1 ); + + rx_pkts[2 ]->hash.fdir.hi = +_mm_extract_epi32 (fdir_id0_3,2 ); + + rx_pkts[3 ]->hash.fdir.hi = +_mm_extract_epi32 (fdir_id0_3,3 ); + }/* if() on fdir_enabled */ +/** * At this point, we have the 4 sets of flags in the low 16-bits * of each 32-bit value in flags.
@@ -604,7 +653,8 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,const __m128i zero =_mm_setzero_si128 ();/* mask to shuffle from desc. to mbuf */ const __m128i shuf_msk = _mm_set_epi8 - (15, 14, 13, 12, /* octet 12~15, 32 bits rss */ + (0xFF, 0xFF, + 0xFF, 0xFF, /* rss not supported */ 11, 10, /* octet 10~11, 16 bits vlan_macip */ 5, 4, /* octet 4~5, 16 bits data_len */ 0xFF, 0xFF, /* skip high 16 bits pkt_len, zero out */
Signature · 1 line
2.17.1
PATCH v6 10/11 net/iavf: add RSS hash parsing in AVX path
In reply to Leyi RongSupport RSS hash parsing from Flex Rx
descriptor in AVX data path.
--- drivers/net/iavf/iavf_rxtx_vec_avx2.c | 94 ++++++++++++++++++++++++++- 1 file changed, 92 insertions(+), 2 deletions(-)
drivers/net/iavf/iavf_rxtx_vec_avx2.c +92 -2
@@ -695,7 +695,7 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq, _mm256_set_epi8 (/* first descriptor */ 0xFF, 0xFF, - 0xFF, 0xFF, /* rss not supported */ + 0xFF, 0xFF, /* rss hash parsed separately */ 11, 10, /* octet 10~11, 16 bits vlan_macip */ 5, 4, /* octet 4~5, 16 bits data_len */ 0xFF, 0xFF, /* skip hi 16 bits pkt_len, zero out */
@@ -704,7 +704,7 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,0xFF ,0xFF ,/*pkt_type set as unknown */ /* second descriptor */ 0xFF ,0xFF , -0xFF ,0xFF ,/* rss not supported */ +0xFF ,0xFF ,/* rss hash parsed separately */ 11 ,10 ,/* octet 10~11, 16 bits vlan_macip */ 5 ,4 ,/* octet 4~5, 16 bits data_len */ 0xFF, 0xFF, /* skip hi 16 bits pkt_len, zero out */
@@ -991,6 +991,96 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,_mm256_extract_epi32 (fdir_id0_7,4 ); }/* if() on fdir_enabled */ +#ifndef RTE_LIBRTE_IAVF_16BYTE_RX_DESC +/** +* needs to load 2nd 16B of each desc for RSS hash parsing, +* will cause performance drop to get into this context. +*/ +if (rxq->vsi->adapter->eth_dev->data->dev_conf.rxmode.offloads & +DEV_RX_OFFLOAD_RSS_HASH ) { +/* load bottom half of every 32B desc */ +const __m128i raw_desc_bh7 = +_mm_load_si128 + ((void *)(&rxdp[7 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh6 = +_mm_load_si128 + ((void *)(&rxdp[6 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh5 = +_mm_load_si128 + ((void *)(&rxdp[5 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh4 = +_mm_load_si128 + ((void *)(&rxdp[4 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh3 = +_mm_load_si128 + ((void *)(&rxdp[3 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh2 = +_mm_load_si128 + ((void *)(&rxdp[2 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh1 = +_mm_load_si128 + ((void *)(&rxdp[1 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh0 = +_mm_load_si128 + ((void *)(&rxdp[0 ].wb.status_error1)); + +__m256i raw_desc_bh6_7 = +_mm256_inserti128_si256 + (_mm256_castsi128_si256 (raw_desc_bh6), + raw_desc_bh7,1 ); +__m256i raw_desc_bh4_5 = +_mm256_inserti128_si256 + (_mm256_castsi128_si256 (raw_desc_bh4), + raw_desc_bh5,1 ); +__m256i raw_desc_bh2_3 = +_mm256_inserti128_si256 + (_mm256_castsi128_si256 (raw_desc_bh2), + raw_desc_bh3,1 ); +__m256i raw_desc_bh0_1 = +_mm256_inserti128_si256 + (_mm256_castsi128_si256 (raw_desc_bh0), + raw_desc_bh1,1 ); + +/** +* to shift the 32b RSS hash value to the +* highest 32b of each 128b before mask +*/ +__m256i rss_hash6_7 = +_mm256_slli_epi64 (raw_desc_bh6_7,32 ); +__m256i rss_hash4_5 = +_mm256_slli_epi64 (raw_desc_bh4_5,32 ); +__m256i rss_hash2_3 = +_mm256_slli_epi64 (raw_desc_bh2_3,32 ); +__m256i rss_hash0_1 = +_mm256_slli_epi64 (raw_desc_bh0_1,32 ); + +__m256i rss_hash_msk = +_mm256_set_epi32 (0xFFFFFFFF ,0 ,0 ,0 , +0xFFFFFFFF ,0 ,0 ,0 ); + + rss_hash6_7 =_mm256_and_si256 + (rss_hash6_7, rss_hash_msk); + rss_hash4_5 =_mm256_and_si256 + (rss_hash4_5, rss_hash_msk); + rss_hash2_3 =_mm256_and_si256 + (rss_hash2_3, rss_hash_msk); + rss_hash0_1 =_mm256_and_si256 + (rss_hash0_1, rss_hash_msk); + + mb6_7 =_mm256_or_si256 (mb6_7, rss_hash6_7); + mb4_5 =_mm256_or_si256 (mb4_5, rss_hash4_5); + mb2_3 =_mm256_or_si256 (mb2_3, rss_hash2_3); + mb0_1 =_mm256_or_si256 (mb0_1, rss_hash0_1); + }/* if() on RSS hash parsing */ +#endif +/** * At this point, we have the 8 sets of flags in the low 16-bits * of each 32-bit value in vlan0.
Signature · 1 line
2.17.1
PATCH v6 11/11 net/iavf: add RSS hash parsing in SSE path
In reply to Leyi RongSupport RSS hash parsing from Flex Rx
descriptor in SSE data path.
--- drivers/net/iavf/iavf_rxtx_vec_sse.c | 88 +++++++++++++++++++++++----- 1 file changed, 72 insertions(+), 16 deletions(-)
drivers/net/iavf/iavf_rxtx_vec_sse.c +72 -16
@@ -654,7 +654,7 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,/* mask to shuffle from desc. to mbuf */ const __m128i shuf_msk = _mm_set_epi8 (0xFF, 0xFF, - 0xFF, 0xFF, /* rss not supported */ + 0xFF, 0xFF, /* rss hash parsed separately */ 11, 10, /* octet 10~11, 16 bits vlan_macip */ 5, 4, /* octet 4~5, 16 bits data_len */ 0xFF, 0xFF, /* skip high 16 bits pkt_len, zero out */
@@ -745,7 +745,7 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq, pos += IAVF_VPMD_DESCS_PER_LOOP, rxdp += IAVF_VPMD_DESCS_PER_LOOP) {__m128i descs[IAVF_VPMD_DESCS_PER_LOOP ]; -__m128i pkt_mb1, pkt_mb2, pkt_mb3, pkt_mb4; +__m128i pkt_mb0, pkt_mb1, pkt_mb2, pkt_mb3;__m128i staterr, sterr_tmp1, sterr_tmp2;/* 2 64 bit or 4 32 bit mbuf pointers in one XMM reg. */ __m128i mbp1;
@@ -791,8 +791,12 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,rte_compiler_barrier ();/* D.1 pkt 3,4 convert format from desc to pktmbuf */ - pkt_mb4 =_mm_shuffle_epi8 (descs[3 ], shuf_msk); - pkt_mb3 =_mm_shuffle_epi8 (descs[2 ], shuf_msk); + pkt_mb3 =_mm_shuffle_epi8 (descs[3 ], shuf_msk); + pkt_mb2 =_mm_shuffle_epi8 (descs[2 ], shuf_msk); + +/* D.1 pkt 1,2 convert format from desc to pktmbuf */ + pkt_mb1 =_mm_shuffle_epi8 (descs[1 ], shuf_msk); + pkt_mb0 =_mm_shuffle_epi8 (descs[0 ], shuf_msk);/* C.1 4=>2 filter staterr info only */ sterr_tmp2 =_mm_unpackhi_epi32 (descs[3 ], descs[2 ]);
@@ -802,12 +806,68 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,flex_desc_to_olflags_v (rxq, descs, &rx_pkts[pos]);/* D.2 pkt 3,4 set in_port/nb_seg and remove crc */ - pkt_mb4 =_mm_add_epi16 (pkt_mb4, crc_adjust); pkt_mb3 =_mm_add_epi16 (pkt_mb3, crc_adjust); + pkt_mb2 =_mm_add_epi16 (pkt_mb2, crc_adjust); -/* D.1 pkt 1,2 convert format from desc to pktmbuf */ - pkt_mb2 =_mm_shuffle_epi8 (descs[1 ], shuf_msk); - pkt_mb1 =_mm_shuffle_epi8 (descs[0 ], shuf_msk); +/* D.2 pkt 1,2 set in_port/nb_seg and remove crc */ + pkt_mb1 =_mm_add_epi16 (pkt_mb1, crc_adjust); + pkt_mb0 =_mm_add_epi16 (pkt_mb0, crc_adjust); + +#ifndef RTE_LIBRTE_IAVF_16BYTE_RX_DESC +/** +* needs to load 2nd 16B of each desc for RSS hash parsing, +* will cause performance drop to get into this context. +*/ +if (rxq->vsi->adapter->eth_dev->data->dev_conf.rxmode.offloads & +DEV_RX_OFFLOAD_RSS_HASH ) { +/* load bottom half of every 32B desc */ +const __m128i raw_desc_bh3 = +_mm_load_si128 + ((void *)(&rxdp[3 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh2 = +_mm_load_si128 + ((void *)(&rxdp[2 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh1 = +_mm_load_si128 + ((void *)(&rxdp[1 ].wb.status_error1)); +rte_compiler_barrier (); +const __m128i raw_desc_bh0 = +_mm_load_si128 + ((void *)(&rxdp[0 ].wb.status_error1)); + +/** +* to shift the 32b RSS hash value to the +* highest 32b of each 128b before mask +*/ +__m128i rss_hash3 = +_mm_slli_epi64 (raw_desc_bh3,32 ); +__m128i rss_hash2 = +_mm_slli_epi64 (raw_desc_bh2,32 ); +__m128i rss_hash1 = +_mm_slli_epi64 (raw_desc_bh1,32 ); +__m128i rss_hash0 = +_mm_slli_epi64 (raw_desc_bh0,32 ); + +__m128i rss_hash_msk = +_mm_set_epi32 (0xFFFFFFFF ,0 ,0 ,0 ); + + rss_hash3 =_mm_and_si128 + (rss_hash3, rss_hash_msk); + rss_hash2 =_mm_and_si128 + (rss_hash2, rss_hash_msk); + rss_hash1 =_mm_and_si128 + (rss_hash1, rss_hash_msk); + rss_hash0 =_mm_and_si128 + (rss_hash0, rss_hash_msk); + + pkt_mb3 =_mm_or_si128 (pkt_mb3, rss_hash3); + pkt_mb2 =_mm_or_si128 (pkt_mb2, rss_hash2); + pkt_mb1 =_mm_or_si128 (pkt_mb1, rss_hash1); + pkt_mb0 =_mm_or_si128 (pkt_mb0, rss_hash0); + }/* if() on RSS hash parsing */ +#endif /* C.2 get 4 pkts staterr value */ staterr =_mm_unpacklo_epi32 (sterr_tmp1, sterr_tmp2);
@@ -815,14 +875,10 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,/* D.3 copy final 3,4 data to rx_pkts */ _mm_storeu_si128 ((void *)&rx_pkts[pos +3 ]->rx_descriptor_fields1, - pkt_mb4); + pkt_mb3);_mm_storeu_si128 ((void *)&rx_pkts[pos +2 ]->rx_descriptor_fields1, - pkt_mb3); - -/* D.2 pkt 1,2 set in_port/nb_seg and remove crc */ - pkt_mb2 =_mm_add_epi16 (pkt_mb2, crc_adjust); - pkt_mb1 =_mm_add_epi16 (pkt_mb1, crc_adjust); + pkt_mb2);/* C* extract and record EOP bit */ if (split_packet) {
@@ -846,9 +902,9 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,/* D.3 copy final 1,2 data to rx_pkts */ _mm_storeu_si128 ((void *)&rx_pkts[pos +1 ]->rx_descriptor_fields1, - pkt_mb2); + pkt_mb1);_mm_storeu_si128 ((void *)&rx_pkts[pos]->rx_descriptor_fields1, - pkt_mb1); + pkt_mb0);flex_desc_to_ptype_v (descs, &rx_pkts[pos], ptype_tbl);/* C.4 calc avaialbe number of desc */ var =__builtin_popcountll (_mm_cvtsi128_si64 (staterr));
Signature · 1 line
2.17.1
Ye Xiaolong
AppliedQuoted 53 lines · Leyi Rong
This patchset enable framework for advanced iAVF, includes flexible descriptor support, FDIR mark id and RSS hash support.
---
v6:
- Remove some redundant code which may cause ICC build error.v5:
- Set rxdid = 0 iniavf_configure_queues()when it's supported andRTE_LIBRTE_IAVF_16BYTE_RX_DESCis y. So the error can be captured explicitly as kernel PF does not support it yet.
- Fix some build errors whenRTE_LIBRTE_IAVF_16BYTE_RX_DESCis y.v4:
- Avoid initial declaration in 'for' loop to backward-compatible with older gcc versions.v3:
- Remove patch to query DDP package info as it's specific to DCF.
- Specifyfdir_ref_cntas per VF value for enable/disable FDIR ID parse.
- Movefdir_enabledflag to per Queue value for cache benefit when access in vector routines.
- Store extractedflow_idvalue to mbuf without judgement to avoid branch speculation fail, which is benefit to performance.v2:
- Revert macroRTE_LIBRTE_IAVF_16BYTE_RX_DESCdeletion as it's defined in AVF spec.
- TypoRTE_LIBRTE_ICE_16BYTE_RX_DESCfix iniavf_rxtx.c.
- Move flex desc definitions intoiavf_rxtx.h.
- Up to date to match with the latest version ofvirtchnl.h.
- Extract a new internal funciavf_update_rx_tailto call.
- Remove iavf_dev_rxq_count_flex_rxd()/iavf_dev_rx_desc_status_flex_rxd(), as the accompanying legacy ones can deal with the flex desc cases.
- Moverxq->rxdidassignment fromiavf_configure_queues()toiavf_dev_rx_queue_setup().
- Unfold_mm_extract_epi32(fdir_id0_3, i)to fix build error when using GCC compile option -O0.Leyi Rong (11):
net/iavf: flexible Rx descriptor definitions net/iavf: return error if opcode is mismatched net/iavf: flexible Rx descriptor support in normal path net/iavf: flexible Rx descriptor support in AVX path net/iavf: flexible Rx descriptor support in SSE path net/iavf: add flow director enabled switch value net/iavf: support flow mark in normal data path net/iavf: support flow mark in AVX path net/iavf: support flow mark in SSE path net/iavf: add RSS hash parsing in AVX path net/iavf: add RSS hash parsing in SSE path
Quoted 12 lines
drivers/net/iavf/iavf.h | 6 + drivers/net/iavf/iavf_ethdev.c | 8 + drivers/net/iavf/iavf_rxtx.c | 546 ++++++++++++++++++-- drivers/net/iavf/iavf_rxtx.h | 250 ++++++++++ drivers/net/iavf/iavf_rxtx_vec_avx2.c | 691 ++++++++++++++++++++++++++ drivers/net/iavf/iavf_rxtx_vec_sse.c | 520 +++++++++++++++++++ drivers/net/iavf/iavf_vchnl.c | 59 ++- 7 files changed, 2035 insertions(+), 45 deletions(-)Signature · 1 line
2.17.1
Applied to dpdk-next-net-intel, Thanks.