Skip to contentSkip to conversation

framework for advanced iAVF PMD

PATCH Superseded Leyi Rong 5 participants 80 messages Mar 16, 2020 – Apr 21, 2020 Network drivers

lore
Share & tools
lore

Leyi Rong

author
#0 lore raw

cover framework for advanced iAVF PMD

This patchset enable framework for advanced iAVF, includes query DDP pkg info, flexible descriptor support, FDIR mark id and RSS hash support.

Leyi Rong (12):
net/iavf: remove 16B Rx descriptor compile option net/iavf: return error if opcode is mismatched net/iavf: support to query DDP package info net/iavf: flexible Rx descriptor support in normal path net/iavf: flexible Rx descriptor support in AVX path net/iavf: flexible Rx descriptor support in SSE path net/iavf: add flow director enabled switch value net/iavf: support flow mark in normal data path net/iavf: support flow mark in AVX path net/iavf: support flow mark in SSE path net/iavf: add RSS hash parsing in AVX path net/iavf: add RSS hash parsing in SSE path

 config/common_base                    |   1 -
 drivers/net/iavf/iavf.h               |  15 +
 drivers/net/iavf/iavf_ethdev.c        |  15 +
 drivers/net/iavf/iavf_rxtx.c          | 570 ++++++++++++++++-
 drivers/net/iavf/iavf_rxtx.h          |  62 +-
 drivers/net/iavf/iavf_rxtx_vec_avx2.c | 856 ++++++++++++++++++++++----
 drivers/net/iavf/iavf_rxtx_vec_sse.c  | 514 ++++++++++++++++
 drivers/net/iavf/iavf_vchnl.c         |  86 ++-
 8 files changed, 1982 insertions(+), 137 deletions(-)
Signature · 1 line
2.17.1
author
#1 lore raw

PATCH 1/12 net/iavf: remove 16B Rx descriptor compile option

Remove CONFIG_RTE_LIBRTE_IAVF_16BYTE_RX_DESC as
it's not supported in ice PF host driver.

---
 config/common_base                    |   1 -
 drivers/net/iavf/iavf_rxtx.c          |   2 -
 drivers/net/iavf/iavf_rxtx.h          |  14 +--
 drivers/net/iavf/iavf_rxtx_vec_avx2.c | 148 +++++++-------------------
 4 files changed, 42 insertions(+), 123 deletions(-)
config/common_base +0 -1
index c31175f9d..eea53cb35 100644
@@ -346,7 +346,6 @@ CONFIG_RTE_LIBRTE_IAVF_DEBUG_TX=n
 CONFIG_RTE_LIBRTE_IAVF_DEBUG_TX_FREE=n
 CONFIG_RTE_LIBRTE_IAVF_DEBUG_RX=n
 CONFIG_RTE_LIBRTE_IAVF_DEBUG_DUMP_DESC=n
-CONFIG_RTE_LIBRTE_IAVF_16BYTE_RX_DESC=n
 #
 # Compile burst-oriented IPN3KE PMD driver
 #
drivers/net/iavf/iavf_rxtx.c +0 -2
index 9eccb7c41..fbb18a713 100644
@@ -230,10 +230,8 @@ alloc_rxq_mbufs(struct iavf_rx_queue *rxq)
 		rxd = &rxq->rx_ring[i];
 		rxd->read.pkt_addr = dma_addr;
 		rxd->read.hdr_addr = 0;
-#ifndef RTE_LIBRTE_IAVF_16BYTE_RX_DESC
 		rxd->read.rsvd1 = 0;
 		rxd->read.rsvd2 = 0;
-#endif
 
 		rxq->sw_ring[i] = mbuf;
 	}
drivers/net/iavf/iavf_rxtx.h +1 -13
index 09b5bd99e..416433504 100644
@@ -57,12 +57,8 @@
 #define IAVF_TX_OFFLOAD_NOTSUP_MASK \
 		(PKT_TX_OFFLOAD_MASK ^ IAVF_TX_OFFLOAD_MASK)
 
-/* HW desc structure, both 16-byte and 32-byte types are supported */
-#ifdef RTE_LIBRTE_IAVF_16BYTE_RX_DESC
-#define iavf_rx_desc iavf_16byte_rx_desc
-#else
+/* HW desc structure, only 32-byte type is supported */
 #define iavf_rx_desc iavf_32byte_rx_desc
-#endif
 
 struct iavf_rxq_ops {
 	void (*release_mbufs)(struct iavf_rx_queue *rxq);
@@ -224,20 +220,12 @@ void iavf_dump_rx_descriptor(struct iavf_rx_queue *rxq,
 			    const volatile void *desc,
 			    uint16_t rx_id)
 {
-#ifdef RTE_LIBRTE_IAVF_16BYTE_RX_DESC
-	const volatile union iavf_16byte_rx_desc *rx_desc = desc;
-
-	printf("Queue %d Rx_desc %d: QW0: 0x%016"PRIx64" QW1: 0x%016"PRIx64"\n",
-	       rxq->queue_id, rx_id, rx_desc->read.pkt_addr,
-	       rx_desc->read.hdr_addr);
-#else
 	const volatile union iavf_32byte_rx_desc *rx_desc = desc;
 
 	printf("Queue %d Rx_desc %d: QW0: 0x%016"PRIx64" QW1: 0x%016"PRIx64
 	       " QW2: 0x%016"PRIx64" QW3: 0x%016"PRIx64"\n", rxq->queue_id,
 	       rx_id, rx_desc->read.pkt_addr, rx_desc->read.hdr_addr,
 	       rx_desc->read.rsvd1, rx_desc->read.rsvd2);
-#endif
 }
 
 /* All the descriptors are 16 bytes, so just use one of them
drivers/net/iavf/iavf_rxtx_vec_avx2.c +41 -107

178 lines not on this page · raw message

Signature · 1 line
2.17.1
author
#2 lore raw

PATCH 2/12 net/iavf: return error if opcode is mismatched

In reply to Leyi Rong

Adds error return when the opcode of read message is mismatched which is received from adminQ.

Fixes 22b123a36d07 ("net/avf: initialize PMD")
---
 drivers/net/iavf/iavf_vchnl.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
drivers/net/iavf/iavf_vchnl.c +3 -1
index fa4da3a6d..b7fb05d32 100644
@@ -52,9 +52,11 @@ iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len,
 	PMD_DRV_LOG(DEBUG, "AQ from pf carries opcode %u, retval %d",
 		    opcode, vf->cmd_retval);
 
-	if (opcode != vf->pend_cmd)
+	if (opcode != vf->pend_cmd) {
 		PMD_DRV_LOG(WARNING, "command mismatch, expect %u, get %u",
 			    vf->pend_cmd, opcode);
+		return IAVF_ERR_OPCODE_MISMATCH;
+	}
 
 	return IAVF_SUCCESS;
 }
Signature · 1 line
2.17.1
author
#3 lore raw

PATCH 3/12 net/iavf: support to query DDP package info

In reply to Leyi Rong

Advanced iAVF supports to query DDP package info, includes package version, track id, package name, device serial number and the list of protocols that the PF supports.

---
 drivers/net/iavf/iavf.h        |  9 ++++++++
 drivers/net/iavf/iavf_ethdev.c |  7 ++++++
 drivers/net/iavf/iavf_vchnl.c  | 41 +++++++++++++++++++++++++++++++++-
 3 files changed, 56 insertions(+), 1 deletion(-)
drivers/net/iavf/iavf.h +9 -0
index 526040c6e..39c6eeec9 100644
@@ -67,6 +67,13 @@
 #define IAVF_48_BIT_WIDTH (CHAR_BIT * 6)
 #define IAVF_48_BIT_MASK  RTE_LEN2MASK(IAVF_48_BIT_WIDTH, uint64_t)
 
+/* VF supported comms protocols 64-bits bitmap */
+#define IAVF_COMMS_PROTO_GTP            0x0000000000000001
+#define IAVF_COMMS_PROTO_PPPOE          0x0000000000000002
+#define IAVF_COMMS_PROTO_PFCP           0x0000000000000004
+#define IAVF_COMMS_PROTO_L2TPV3         0x0000000000000008
+#define IAVF_COMMS_PROTO_ESP            0x0000000000000010
+
 struct iavf_adapter;
 struct iavf_rx_queue;
 struct iavf_tx_queue;
@@ -97,6 +104,7 @@ struct iavf_info {
 	struct virtchnl_version_info virtchnl_version;
 	struct virtchnl_vf_resource *vf_res; /* VF resource */
 	struct virtchnl_vsi_resource *vsi_res; /* LAN VSI */
+	struct virtchnl_pkg_info pkg_info; /* package info */
 
 	volatile enum virtchnl_ops pend_cmd; /* pending command not finished */
 	uint32_t cmd_retval; /* return value of the cmd response from PF */
@@ -225,6 +233,7 @@ int iavf_disable_queues(struct iavf_adapter *adapter);
 int iavf_configure_rss_lut(struct iavf_adapter *adapter);
 int iavf_configure_rss_key(struct iavf_adapter *adapter);
 int iavf_configure_queues(struct iavf_adapter *adapter);
+int iavf_query_package_info(struct iavf_adapter *adapter);
 int iavf_config_irq_map(struct iavf_adapter *adapter);
 void iavf_add_del_all_mac_addr(struct iavf_adapter *adapter, bool add);
 int iavf_dev_link_update(struct rte_eth_dev *dev,
drivers/net/iavf/iavf_ethdev.c +7 -0
index ee9f82249..9c8f789da 100644
@@ -1236,6 +1236,13 @@ iavf_init_vf(struct rte_eth_dev *dev)
 			goto err_rss;
 		}
 	}
+
+	if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_QUERY_DDP) {
+		if (iavf_query_package_info(adapter) != 0) {
+			PMD_INIT_LOG(ERR, "iavf_query_package_info failed");
+			goto err_rss;
+		}
+	}
 	return 0;
 err_rss:
 	rte_free(vf->rss_key);
drivers/net/iavf/iavf_vchnl.c +40 -1

59 lines not on this page · raw message

Signature · 1 line
2.17.1
author
#4 lore raw

PATCH 4/12 net/iavf: flexible Rx descriptor support in normal path

In reply to Leyi Rong

Support flexible Rx descriptor format in normal
path of iAVF PMD.

---
 drivers/net/iavf/iavf.h        |   2 +
 drivers/net/iavf/iavf_ethdev.c |   8 +
 drivers/net/iavf/iavf_rxtx.c   | 507 ++++++++++++++++++++++++++++++++-
 drivers/net/iavf/iavf_rxtx.h   |  11 +
 drivers/net/iavf/iavf_vchnl.c  |  43 ++-
 5 files changed, 567 insertions(+), 4 deletions(-)
drivers/net/iavf/iavf.h +2 -0
index 39c6eeec9..4fe15237a 100644
@@ -105,6 +105,7 @@ struct iavf_info {
 	struct virtchnl_vf_resource *vf_res; /* VF resource */
 	struct virtchnl_vsi_resource *vsi_res; /* LAN VSI */
 	struct virtchnl_pkg_info pkg_info; /* package info */
+	uint64_t supported_rxdid;
 
 	volatile enum virtchnl_ops pend_cmd; /* pending command not finished */
 	uint32_t cmd_retval; /* return value of the cmd response from PF */
@@ -233,6 +234,7 @@ int iavf_disable_queues(struct iavf_adapter *adapter);
 int iavf_configure_rss_lut(struct iavf_adapter *adapter);
 int iavf_configure_rss_key(struct iavf_adapter *adapter);
 int iavf_configure_queues(struct iavf_adapter *adapter);
+int iavf_get_supported_rxdid(struct iavf_adapter *adapter);
 int iavf_query_package_info(struct iavf_adapter *adapter);
 int iavf_config_irq_map(struct iavf_adapter *adapter);
 void iavf_add_del_all_mac_addr(struct iavf_adapter *adapter, bool add);
drivers/net/iavf/iavf_ethdev.c +8 -0
index 9c8f789da..f7b2562c1 100644
@@ -1243,6 +1243,14 @@ iavf_init_vf(struct rte_eth_dev *dev)
 			goto err_rss;
 		}
 	}
+
+	if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) {
+		if (iavf_get_supported_rxdid(adapter) != 0) {
+			PMD_INIT_LOG(ERR, "failed to do get supported rxdid");
+			goto err_rss;
+		}
+	}
+
 	return 0;
 err_rss:
 	rte_free(vf->rss_key);
drivers/net/iavf/iavf_rxtx.c +504 -3

579 lines not on this page · raw message

drivers/net/iavf/iavf_rxtx.h +11 -0
index 416433504..ee306d400 100644
@@ -59,6 +59,7 @@
 
 /* HW desc structure, only 32-byte type is supported */
 #define iavf_rx_desc iavf_32byte_rx_desc
+#define iavf_rx_flex_desc iavf_32b_rx_flex_desc
 
 struct iavf_rxq_ops {
 	void (*release_mbufs)(struct iavf_rx_queue *rxq);
@@ -83,6 +84,7 @@ struct iavf_rx_queue {
 	struct rte_mbuf *pkt_first_seg; /* first segment of current packet */
 	struct rte_mbuf *pkt_last_seg;  /* last segment of current packet */
 	struct rte_mbuf fake_mbuf;      /* dummy mbuf */
+	uint8_t rxdid;
 
 	/* used for VPMD */
 	uint16_t rxrearm_nb;       /* number of remaining to be re-armed */
@@ -175,9 +177,15 @@ void iavf_dev_tx_queue_release(void *txq);
 void iavf_stop_queues(struct rte_eth_dev *dev);
 uint16_t iavf_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 		       uint16_t nb_pkts);
+uint16_t iavf_recv_pkts_flex_rxd(void *rx_queue,
+				 struct rte_mbuf **rx_pkts,
+				 uint16_t nb_pkts);
 uint16_t iavf_recv_scattered_pkts(void *rx_queue,
 				 struct rte_mbuf **rx_pkts,
 				 uint16_t nb_pkts);
+uint16_t iavf_recv_scattered_pkts_flex_rxd(void *rx_queue,
+					   struct rte_mbuf **rx_pkts,
+					   uint16_t nb_pkts);
 uint16_t iavf_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 		       uint16_t nb_pkts);
 uint16_t iavf_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
@@ -189,7 +197,10 @@ void iavf_dev_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
 void iavf_dev_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
 			  struct rte_eth_txq_info *qinfo);
 uint32_t iavf_dev_rxq_count(struct rte_eth_dev *dev, uint16_t queue_id);
+uint32_t iavf_dev_rxq_count_flex_rxd(struct rte_eth_dev *dev,
+				     uint16_t queue_id);
 int iavf_dev_rx_desc_status(void *rx_queue, uint16_t offset);
+int iavf_dev_rx_desc_status_flex_rxd(void *rx_queue, uint16_t offset);
 int iavf_dev_tx_desc_status(void *tx_queue, uint16_t offset);
 
 uint16_t iavf_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
drivers/net/iavf/iavf_vchnl.c +42 -1

67 lines not on this page · raw message

Signature · 1 line
2.17.1
#5 lore raw

Re: [dpdk-dev] [PATCH 04/12] net/iavf: flexible Rx descriptor support in normal path

One general comment:
Looks like there are exact same code comparing with legacy rx, such as logic to update tail, multi-segments loop. It will be good if all the common code can be wrapped and use for multi recv functions

[...]

+/* Get the number of used descriptors of a rx queue for flexible RXD */ +uint32_t iavf_dev_rxq_count_flex_rxd(struct rte_eth_dev *dev, uint16_t +queue_id) { #define IAVF_RXQ_SCAN_INTERVAL 4
+ volatile union iavf_rx_flex_desc *rxdp;
+ struct iavf_rx_queue *rxq;
+ uint16_t desc = 0;
+
+ rxq = dev->data->rx_queues[queue_id];
+ rxdp = (volatile union iavf_rx_flex_desc *)&rxq->rx_ring[rxq->rx_tail];
+ while ((desc < rxq->nb_rx_desc) &&
+ rte_le_to_cpu_16(rxdp->wb.status_error0) &
+ (1 << IAVF_RX_FLEX_DESC_STATUS0_DD_S)) {
+ /* Check the DD bit of a rx descriptor of each 4 in a group,
+ * to avoid checking too frequently and downgrading performance
+ * too much.
+ */
+ desc += IAVF_RXQ_SCAN_INTERVAL;
+ rxdp += IAVF_RXQ_SCAN_INTERVAL;
+ if (rxq->rx_tail + desc >= rxq->nb_rx_desc)
+ rxdp = (volatile union iavf_rx_flex_desc *)
+ &(rxq->rx_ring[rxq->rx_tail +
+ desc - rxq->nb_rx_desc]);
+ }
+
+ return desc;
+}

No much difference between iavf_dev_rxq_count. Why do we need a new one? DD bit is located in the same place, right?
Can we merge those two functions to one?
+
/* Get the number of used descriptors of a rx queue */ uint32_t iavf_dev_rxq_count(struct rte_eth_dev *dev, uint16_t queue_id) @@ -1795,6 +2264,10 @@ iavf_dev_rxq_count(struct rte_eth_dev *dev, uint16_t queue_id)

rxq = dev->data->rx_queues[queue_id];
rxdp = &rxq->rx_ring[rxq->rx_tail];
+
+ if (rxq->rxdid == IAVF_RXDID_COMMS_OVS_1)
+ return iavf_dev_rxq_count_flex_rxd(dev, queue_id);
+
while ((desc < rxq->nb_rx_desc) &&
((rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len) &
IAVF_RXD_QW1_STATUS_MASK) >> IAVF_RXD_QW1_STATUS_SHIFT) & @@ -1813,6 +2286,31 @@ iavf_dev_rxq_count(struct rte_eth_dev *dev, uint16_t queue_id)
return desc;
}

+int
+iavf_dev_rx_desc_status_flex_rxd(void *rx_queue, uint16_t offset) {
+ volatile union iavf_rx_flex_desc *rxdp;
+ struct iavf_rx_queue *rxq = rx_queue;
+ uint32_t desc;
+
+ if (unlikely(offset >= rxq->nb_rx_desc))
+ return -EINVAL;
+
+ if (offset >= rxq->nb_rx_desc - rxq->nb_rx_hold)
+ return RTE_ETH_RX_DESC_UNAVAIL;
+
+ desc = rxq->rx_tail + offset;
+ if (desc >= rxq->nb_rx_desc)
+ desc -= rxq->nb_rx_desc;
+
+ rxdp = (volatile union iavf_rx_flex_desc *)&rxq->rx_ring[desc];
+ if (rte_le_to_cpu_16(rxdp->wb.status_error0) &
+ (1 << IAVF_RX_FLEX_DESC_STATUS0_DD_S))
+ return RTE_ETH_RX_DESC_DONE;
+
+ return RTE_ETH_RX_DESC_AVAIL;
+}
+

Similar comments as above.

[......]

{ @@ -569,6 +596,20 @@ iavf_configure_queues(struct iavf_adapter *adapter)
vc_qp->rxq.ring_len = rxq[i]->nb_rx_desc;
vc_qp->rxq.dma_ring_addr = rxq[i]->rx_ring_phys_addr;
vc_qp->rxq.databuffer_size = rxq[i]->rx_buf_len;
+
+ if (vf->vf_res->vf_cap_flags &
+ VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC &&
+ vf->supported_rxdid & BIT(IAVF_RXDID_COMMS_OVS_1)) {
+ vc_qp->rxq.rxdid = IAVF_RXDID_COMMS_OVS_1;
+ rxq[i]->rxdid = IAVF_RXDID_COMMS_OVS_1;
Because this function is used to construct virtchnl message for configure queues.
The rxdid in rxq[i] should be set before this function is called. How about to move this line assignment to iavf_dev_rx_queue_setup?
+ PMD_DRV_LOG(NOTICE, "request RXDID == %d in "
+ "Queue[%d]", vc_qp->rxq.rxdid, i);
+ } else {
+ vc_qp->rxq.rxdid = IAVF_RXDID_LEGACY_1;
+ rxq[i]->rxdid = IAVF_RXDID_LEGACY_1;
Same as above.

+ PMD_DRV_LOG(NOTICE, "request RXDID == %d in "
+ "Queue[%d]", vc_qp->rxq.rxdid, i);
+ }
}
}

--
2.17.1

author
#6 lore raw

PATCH 5/12 net/iavf: flexible Rx descriptor support in AVX path

In reply to Leyi Rong

Support flexible Rx descriptor format in AVX
path of iAVF PMD.

---
 drivers/net/iavf/iavf_rxtx.c          |  24 +-
 drivers/net/iavf/iavf_rxtx.h          |   6 +
 drivers/net/iavf/iavf_rxtx_vec_avx2.c | 550 +++++++++++++++++++++++++-
 3 files changed, 570 insertions(+), 10 deletions(-)
drivers/net/iavf/iavf_rxtx.c +18 -6
index b9b35bdbb..57fffece9 100644
@@ -2115,16 +2115,28 @@ iavf_set_rx_function(struct rte_eth_dev *dev)
 				    "Using %sVector Scattered Rx (port %d).",
 				    use_avx2 ? "avx2 " : "",
 				    dev->data->port_id);
-			dev->rx_pkt_burst = use_avx2 ?
-					    iavf_recv_scattered_pkts_vec_avx2 :
-					    iavf_recv_scattered_pkts_vec;
+			if (vf->vf_res->vf_cap_flags &
+				VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC)
+				dev->rx_pkt_burst = use_avx2 ?
+					iavf_recv_scattered_pkts_vec_avx2_flex_rxd :
+					iavf_recv_scattered_pkts_vec;
+			else
+				dev->rx_pkt_burst = use_avx2 ?
+					iavf_recv_scattered_pkts_vec_avx2 :
+					iavf_recv_scattered_pkts_vec;
 		} else {
 			PMD_DRV_LOG(DEBUG, "Using %sVector Rx (port %d).",
 				    use_avx2 ? "avx2 " : "",
 				    dev->data->port_id);
-			dev->rx_pkt_burst = use_avx2 ?
-					    iavf_recv_pkts_vec_avx2 :
-					    iavf_recv_pkts_vec;
+			if (vf->vf_res->vf_cap_flags &
+				VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC)
+				dev->rx_pkt_burst = use_avx2 ?
+					iavf_recv_pkts_vec_avx2_flex_rxd :
+					iavf_recv_pkts_vec;
+			else
+				dev->rx_pkt_burst = use_avx2 ?
+					iavf_recv_pkts_vec_avx2 :
+					iavf_recv_pkts_vec;
 		}
 
 		return;
drivers/net/iavf/iavf_rxtx.h +6 -0
index ee306d400..de13fd516 100644
@@ -212,9 +212,15 @@ uint16_t iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
 				  uint16_t nb_pkts);
 uint16_t iavf_recv_pkts_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,
 				 uint16_t nb_pkts);
+uint16_t iavf_recv_pkts_vec_avx2_flex_rxd(void *rx_queue,
+					  struct rte_mbuf **rx_pkts,
+					  uint16_t nb_pkts);
 uint16_t iavf_recv_scattered_pkts_vec_avx2(void *rx_queue,
 					   struct rte_mbuf **rx_pkts,
 					   uint16_t nb_pkts);
+uint16_t iavf_recv_scattered_pkts_vec_avx2_flex_rxd(void *rx_queue,
+						    struct rte_mbuf **rx_pkts,
+						    uint16_t nb_pkts);
 uint16_t iavf_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
 			    uint16_t nb_pkts);
 uint16_t iavf_xmit_pkts_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts,
drivers/net/iavf/iavf_rxtx_vec_avx2.c +546 -4

585 lines not on this page · raw message

Signature · 1 line
2.17.1
author
#7 lore raw

PATCH 6/12 net/iavf: flexible Rx descriptor support in SSE path

In reply to Leyi Rong

Support flexible Rx descriptor format in SSE
path of iAVF PMD.

---
 drivers/net/iavf/iavf_rxtx.c         |   4 +-
 drivers/net/iavf/iavf_rxtx.h         |   5 +
 drivers/net/iavf/iavf_rxtx_vec_sse.c | 414 +++++++++++++++++++++++++++
 3 files changed, 421 insertions(+), 2 deletions(-)
drivers/net/iavf/iavf_rxtx.c +2 -2
index 57fffece9..5d484d5e9 100644
@@ -2119,7 +2119,7 @@ iavf_set_rx_function(struct rte_eth_dev *dev)
 				VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC)
 				dev->rx_pkt_burst = use_avx2 ?
 					iavf_recv_scattered_pkts_vec_avx2_flex_rxd :
-					iavf_recv_scattered_pkts_vec;
+					iavf_recv_scattered_pkts_vec_flex_rxd;
 			else
 				dev->rx_pkt_burst = use_avx2 ?
 					iavf_recv_scattered_pkts_vec_avx2 :
@@ -2132,7 +2132,7 @@ iavf_set_rx_function(struct rte_eth_dev *dev)
 				VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC)
 				dev->rx_pkt_burst = use_avx2 ?
 					iavf_recv_pkts_vec_avx2_flex_rxd :
-					iavf_recv_pkts_vec;
+					iavf_recv_pkts_vec_flex_rxd;
 			else
 				dev->rx_pkt_burst = use_avx2 ?
 					iavf_recv_pkts_vec_avx2 :
drivers/net/iavf/iavf_rxtx.h +5 -0
index de13fd516..c85207dae 100644
@@ -205,9 +205,14 @@ int iavf_dev_tx_desc_status(void *tx_queue, uint16_t offset);
 
 uint16_t iavf_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
 			   uint16_t nb_pkts);
+uint16_t iavf_recv_pkts_vec_flex_rxd(void *rx_queue, struct rte_mbuf **rx_pkts,
+				     uint16_t nb_pkts);
 uint16_t iavf_recv_scattered_pkts_vec(void *rx_queue,
 				     struct rte_mbuf **rx_pkts,
 				     uint16_t nb_pkts);
+uint16_t iavf_recv_scattered_pkts_vec_flex_rxd(void *rx_queue,
+					       struct rte_mbuf **rx_pkts,
+					       uint16_t nb_pkts);
 uint16_t iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
 				  uint16_t nb_pkts);
 uint16_t iavf_recv_pkts_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,
drivers/net/iavf/iavf_rxtx_vec_sse.c +414 -0

444 lines not on this page · raw message

Signature · 1 line
2.17.1
author
#8 lore raw

PATCH 7/12 net/iavf: add flow director enabled switch value

In reply to Leyi Rong

The commit adds fdir_enabled flag into iavf_adapter structure to identify if fdir id is active. Rx data path can be benefit if fdir id parsing is not needed, especially in vector path.

---
 drivers/net/iavf/iavf.h      |  1 +
 drivers/net/iavf/iavf_rxtx.h | 26 ++++++++++++++++++++++++++
 2 files changed, 27 insertions(+)
drivers/net/iavf/iavf.h +1 -0
index 4fe15237a..1918a67f1 100644
@@ -142,6 +142,7 @@ struct iavf_adapter {
 	bool tx_vec_allowed;
 	const uint32_t *ptype_tbl;
 	bool stopped;
+	uint8_t fdir_enabled;
 };
 
 /* IAVF_DEV_PRIVATE_TO */
drivers/net/iavf/iavf_rxtx.h +26 -0
index c85207dae..5548d1adb 100644
@@ -281,6 +281,32 @@ void iavf_dump_tx_descriptor(const struct iavf_tx_queue *txq,
 	       tx_desc->cmd_type_offset_bsz);
 }
 
+/* Enable/disable flow director Rx processing in data path. */
+static inline
+void iavf_fdir_rx_proc_enable(struct iavf_adapter *ad, bool on)
+{
+	static uint32_t ref_cnt;
+
+	if (on) {
+		/* enable flow director processing */
+		if (ref_cnt++ == 0) {
+			ad->fdir_enabled = on;
+			PMD_DRV_LOG(DEBUG,
+				    "FDIR processing on RX set to %d", on);
+		}
+	} else {
+		if (ref_cnt >= 1) {
+			ref_cnt--;
+
+			if (ref_cnt == 0) {
+				ad->fdir_enabled = on;
+				PMD_DRV_LOG(DEBUG,
+					    "FDIR processing on RX set to %d", on);
+			}
+		}
+	}
+}
+
 #ifdef RTE_LIBRTE_IAVF_DEBUG_DUMP_DESC
 #define IAVF_DUMP_RX_DESC(rxq, desc, rx_id) \
 	iavf_dump_rx_descriptor(rxq, desc, rx_id)
Signature · 1 line
2.17.1
#9 lore raw

Re: [dpdk-dev] [PATCH 07/12] net/iavf: add flow director enabled switch value

-----Original Message-----
From: dev [mailto:[email protected]] On Behalf Of Leyi Rong
Sent: Monday, March 16, 2020 3:46 PM
To: Zhang, Qi Z <[email protected]>; Ye, Xiaolong <[email protected]>
Cc: [email protected]; Rong, Leyi <[email protected]>
Subject: [dpdk-dev] [PATCH 07/12] net/iavf: add flow director enabled switch value

The commit adds fdir_enabled flag into iavf_adapter structure to identify if fdir id is active. Rx data path can be benefit if fdir id parsing is not needed, especially in vector path.

---
 drivers/net/iavf/iavf.h      |  1 +
 drivers/net/iavf/iavf_rxtx.h | 26 ++++++++++++++++++++++++++
 2 files changed, 27 insertions(+)
drivers/net/iavf/iavf.h +1 -0
@@ -142,6 +142,7 @@ struct iavf_adapter {
 	bool tx_vec_allowed;
 	const uint32_t *ptype_tbl;
 	bool stopped;
+	uint8_t fdir_enabled;
 };
 
 /* IAVF_DEV_PRIVATE_TO */
drivers/net/iavf/iavf_rxtx.h +25 -0
@@ -281,6 +281,32 @@ void iavf_dump_tx_descriptor(const struct iavf_tx_queue *txq,
 	       tx_desc->cmd_type_offset_bsz);
 }
 
+/* Enable/disable flow director Rx processing in data path. */ static 
+inline void iavf_fdir_rx_proc_enable(struct iavf_adapter *ad, bool on) 
+{
+	static uint32_t ref_cnt;
+
+	if (on) {
+		/* enable flow director processing */
+		if (ref_cnt++ == 0) {
+			ad->fdir_enabled = on;
+			PMD_DRV_LOG(DEBUG,
+				    "FDIR processing on RX set to %d", on);
+		}
+	} else {
+		if (ref_cnt >= 1) {
+			ref_cnt--;
+
+			if (ref_cnt == 0) {
+				ad->fdir_enabled = on;
+				PMD_DRV_LOG(DEBUG,
+					    "FDIR processing on RX set to %d", on);
+			}
+		}
+	}
+}
+
 

fdir_enabled is used for fast path. To avoid competition, how about to make it in queue granularity?

#ifdef RTE_LIBRTE_IAVF_DEBUG_DUMP_DESC
#define IAVF_DUMP_RX_DESC(rxq, desc, rx_id) \
iavf_dump_rx_descriptor(rxq, desc, rx_id)
--
2.17.1

author
#10 lore raw

PATCH 8/12 net/iavf: support flow mark in normal data path

In reply to Leyi Rong

Support Flow Director mark ID parsing in normal path.

---
 drivers/net/iavf/iavf.h      |  3 +++
 drivers/net/iavf/iavf_rxtx.c | 37 ++++++++++++++++++++++++++++++++++++
 2 files changed, 40 insertions(+)
drivers/net/iavf/iavf.h +3 -0
index 1918a67f1..533856f21 100644
@@ -74,6 +74,9 @@
 #define IAVF_COMMS_PROTO_L2TPV3         0x0000000000000008
 #define IAVF_COMMS_PROTO_ESP            0x0000000000000010
 
+#define IAVF_RX_DESC_EXT_STATUS_FLEXBH_MASK  0x03
+#define IAVF_RX_DESC_EXT_STATUS_FLEXBH_FD_ID 0x01
+
 struct iavf_adapter;
 struct iavf_rx_queue;
 struct iavf_tx_queue;
drivers/net/iavf/iavf_rxtx.c +37 -0
index 5d484d5e9..d941249e3 100644
@@ -746,6 +746,10 @@ iavf_rxd_to_pkt_flags(uint64_t qword)
 					IAVF_RX_DESC_FLTSTAT_RSS_HASH) ==
 			IAVF_RX_DESC_FLTSTAT_RSS_HASH) ? PKT_RX_RSS_HASH : 0;
 
+	/* Check if FDIR Match */
+	flags |= (qword & (1 << IAVF_RX_DESC_STATUS_FLM_SHIFT) ?
+				PKT_RX_FDIR : 0);
+
 	if (likely((error_bits & IAVF_RX_ERR_BITS) == 0)) {
 		flags |= (PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD);
 		return flags;
@@ -766,6 +770,25 @@ iavf_rxd_to_pkt_flags(uint64_t qword)
 	return flags;
 }
 
+static inline uint64_t
+iavf_rxd_build_fdir(volatile union iavf_rx_desc *rxdp, struct rte_mbuf *mb)
+{
+	uint64_t flags = 0;
+	uint16_t flexbh;
+
+	flexbh = (rte_le_to_cpu_32(rxdp->wb.qword2.ext_status) >>
+		IAVF_RX_DESC_EXT_STATUS_FLEXBH_SHIFT) &
+		IAVF_RX_DESC_EXT_STATUS_FLEXBH_MASK;
+
+	if (flexbh == IAVF_RX_DESC_EXT_STATUS_FLEXBH_FD_ID) {
+		mb->hash.fdir.hi =
+			rte_le_to_cpu_32(rxdp->wb.qword3.hi_dword.fd_id);
+		flags |= PKT_RX_FDIR_ID;
+	}
+
+	return flags;
+}
+
 /* Translate the rx flex descriptor status to pkt flags */
 static inline void
 iavf_rxd_to_pkt_fields(struct rte_mbuf *mb,
@@ -782,6 +805,11 @@ iavf_rxd_to_pkt_fields(struct rte_mbuf *mb,
 		mb->hash.rss = rte_le_to_cpu_32(desc->rss_hash);
 	}
 #endif
+
+	if (desc->flow_id != 0xFFFFFFFF) {
+		mb->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID;
+		mb->hash.fdir.hi = rte_le_to_cpu_32(desc->flow_id);
+	}
 }
 
 #define IAVF_RX_FLEX_ERR0_BITS	\
@@ -917,6 +945,9 @@ iavf_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 			rxm->hash.rss =
 				rte_le_to_cpu_32(rxd.wb.qword0.hi_dword.rss);
 
+		if (pkt_flags & PKT_RX_FDIR)
+			pkt_flags |= iavf_rxd_build_fdir(&rxd, rxm);
+
 		rxm->ol_flags |= pkt_flags;
 
 		rx_pkts[nb_rx++] = rxm;
@@ -1365,6 +1396,9 @@ iavf_recv_scattered_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 			first_seg->hash.rss =
 				rte_le_to_cpu_32(rxd.wb.qword0.hi_dword.rss);
 
+		if (pkt_flags & PKT_RX_FDIR)
+			pkt_flags |= iavf_rxd_build_fdir(&rxd, first_seg);
+
 		first_seg->ol_flags |= pkt_flags;
 
 		/* Prefetch data of first segment, if configured to do so. */
@@ -1549,6 +1583,9 @@ iavf_rx_scan_hw_ring(struct iavf_rx_queue *rxq)
 				mb->hash.rss = rte_le_to_cpu_32(
 					rxdp[j].wb.qword0.hi_dword.rss);
 
+			if (pkt_flags & PKT_RX_FDIR)
+				pkt_flags |= iavf_rxd_build_fdir(&rxdp[j], mb);
+
 			mb->ol_flags |= pkt_flags;
 		}
 
Signature · 1 line
2.17.1
author
#11 lore raw

PATCH 9/12 net/iavf: support flow mark in AVX path

In reply to Leyi Rong

Support Flow Director mark ID parsing from Flex
Rx descriptor in AVX path.

---
 drivers/net/iavf/iavf_rxtx_vec_avx2.c | 82 +++++++++++++++++++++++++--
 1 file changed, 77 insertions(+), 5 deletions(-)
drivers/net/iavf/iavf_rxtx_vec_avx2.c +77 -5
index 74f672c7e..9bbf75632 100644
@@ -550,6 +550,25 @@ _iavf_recv_raw_pkts_vec_avx2(struct iavf_rx_queue *rxq,
 	return received;
 }
 
+static inline __m256i
+flex_rxd_to_fdir_flags_vec_avx2(const __m256i fdir_id0_7)
+{
+#define FDID_MIS_MAGIC 0xFFFFFFFF
+	RTE_BUILD_BUG_ON(PKT_RX_FDIR != (1 << 2));
+	RTE_BUILD_BUG_ON(PKT_RX_FDIR_ID != (1 << 13));
+	const __m256i pkt_fdir_bit = _mm256_set1_epi32(PKT_RX_FDIR |
+			PKT_RX_FDIR_ID);
+	/* desc->flow_id field == 0xFFFFFFFF means fdir mismatch */
+	const __m256i fdir_mis_mask = _mm256_set1_epi32(FDID_MIS_MAGIC);
+	__m256i fdir_mask = _mm256_cmpeq_epi32(fdir_id0_7,
+			fdir_mis_mask);
+	/* this XOR op results to bit-reverse the fdir_mask */
+	fdir_mask = _mm256_xor_si256(fdir_mask, fdir_mis_mask);
+	const __m256i fdir_flags = _mm256_and_si256(fdir_mask, pkt_fdir_bit);
+
+	return fdir_flags;
+}
+
 static inline uint16_t
 _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 				      struct rte_mbuf **rx_pkts,
@@ -612,8 +631,8 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 	const __m256i shuf_msk =
 		_mm256_set_epi8
 			(/* first descriptor */
-			 15, 14,
-			 13, 12,	/* octet 12~15, 32 bits rss */
+			 0xFF, 0xFF,
+			 0xFF, 0xFF,    /* rss not supported */
 			 11, 10,	/* octet 10~11, 16 bits vlan_macip */
 			 5, 4,		/* octet 4~5, 16 bits data_len */
 			 0xFF, 0xFF,	/* skip hi 16 bits pkt_len, zero out */
@@ -621,8 +640,8 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 			 0xFF, 0xFF,	/* pkt_type set as unknown */
 			 0xFF, 0xFF,	/*pkt_type set as unknown */
 			 /* second descriptor */
-			 15, 14,
-			 13, 12,	/* octet 12~15, 32 bits rss */
+			 0xFF, 0xFF,
+			 0xFF, 0xFF,    /* rss not supported */
 			 11, 10,	/* octet 10~11, 16 bits vlan_macip */
 			 5, 4,		/* octet 4~5, 16 bits data_len */
 			 0xFF, 0xFF,	/* skip hi 16 bits pkt_len, zero out */
@@ -864,8 +883,61 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 					    rss_vlan_flag_bits);
 
 		/* merge flags */
-		const __m256i mbuf_flags = _mm256_or_si256(l3_l4_flags,
+		__m256i mbuf_flags = _mm256_or_si256(l3_l4_flags,
 				rss_vlan_flags);
+
+		if (rxq->vsi->adapter->fdir_enabled) {
+			const __m256i fdir_id4_7 =
+				_mm256_unpackhi_epi32(raw_desc6_7, raw_desc4_5);
+
+			const __m256i fdir_id0_3 =
+				_mm256_unpackhi_epi32(raw_desc2_3, raw_desc0_1);
+
+			const __m256i fdir_id0_7 =
+				_mm256_unpackhi_epi64(fdir_id4_7, fdir_id0_3);
+
+			const __m256i fdir_flags =
+				flex_rxd_to_fdir_flags_vec_avx2(fdir_id0_7);
+
+			/* merge with fdir_flags */
+			mbuf_flags = _mm256_or_si256(mbuf_flags, fdir_flags);
+
+			/* write to mbuf: have to use scalar store here */
+			uint32_t fdir_id_extr;
+
+			fdir_id_extr = _mm256_extract_epi32(fdir_id0_7, 3);
+			if (fdir_id_extr != FDID_MIS_MAGIC)
+				rx_pkts[i + 0]->hash.fdir.hi = fdir_id_extr;
+
+			fdir_id_extr = _mm256_extract_epi32(fdir_id0_7, 7);
+			if (fdir_id_extr != FDID_MIS_MAGIC)
+				rx_pkts[i + 1]->hash.fdir.hi = fdir_id_extr;
+
+			fdir_id_extr = _mm256_extract_epi32(fdir_id0_7, 2);
+			if (fdir_id_extr != FDID_MIS_MAGIC)
+				rx_pkts[i + 2]->hash.fdir.hi = fdir_id_extr;
+
+			fdir_id_extr = _mm256_extract_epi32(fdir_id0_7, 6);
+			if (fdir_id_extr != FDID_MIS_MAGIC)
+				rx_pkts[i + 3]->hash.fdir.hi = fdir_id_extr;
+
+			fdir_id_extr = _mm256_extract_epi32(fdir_id0_7, 1);
+			if (fdir_id_extr != FDID_MIS_MAGIC)
+				rx_pkts[i + 4]->hash.fdir.hi = fdir_id_extr;
+
+			fdir_id_extr = _mm256_extract_epi32(fdir_id0_7, 5);
+			if (fdir_id_extr != FDID_MIS_MAGIC)
+				rx_pkts[i + 5]->hash.fdir.hi = fdir_id_extr;
+
+			fdir_id_extr = _mm256_extract_epi32(fdir_id0_7, 0);
+			if (fdir_id_extr != FDID_MIS_MAGIC)
+				rx_pkts[i + 6]->hash.fdir.hi = fdir_id_extr;
+
+			fdir_id_extr = _mm256_extract_epi32(fdir_id0_7, 4);
+			if (fdir_id_extr != FDID_MIS_MAGIC)
+				rx_pkts[i + 7]->hash.fdir.hi = fdir_id_extr;
+		} /* if() on fdir_enabled */
+
 		/**
 		 * At this point, we have the 8 sets of flags in the low 16-bits
 		 * of each 32-bit value in vlan0.
Signature · 1 line
2.17.1
author
#12 lore raw

PATCH 10/12 net/iavf: support flow mark in SSE path

In reply to Leyi Rong

Support Flow Director mark ID parsing from Flex
Rx descriptor in SSE path.

---
 drivers/net/iavf/iavf_rxtx_vec_sse.c | 48 +++++++++++++++++++++++++++-
 1 file changed, 47 insertions(+), 1 deletion(-)
drivers/net/iavf/iavf_rxtx_vec_sse.c +47 -1
index 9c1f2a445..eb784938f 100644
@@ -189,6 +189,25 @@ desc_to_olflags_v(struct iavf_rx_queue *rxq, __m128i descs[4],
 	_mm_store_si128((__m128i *)&rx_pkts[3]->rearm_data, rearm3);
 }
 
+static inline __m128i
+flex_rxd_to_fdir_flags_vec(const __m128i fdir_id0_3)
+{
+#define FDID_MIS_MAGIC 0xFFFFFFFF
+	RTE_BUILD_BUG_ON(PKT_RX_FDIR != (1 << 2));
+	RTE_BUILD_BUG_ON(PKT_RX_FDIR_ID != (1 << 13));
+	const __m128i pkt_fdir_bit = _mm_set1_epi32(PKT_RX_FDIR |
+			PKT_RX_FDIR_ID);
+	/* desc->flow_id field == 0xFFFFFFFF means fdir mismatch */
+	const __m128i fdir_mis_mask = _mm_set1_epi32(FDID_MIS_MAGIC);
+	__m128i fdir_mask = _mm_cmpeq_epi32(fdir_id0_3,
+			fdir_mis_mask);
+	/* this XOR op results to bit-reverse the fdir_mask */
+	fdir_mask = _mm_xor_si128(fdir_mask, fdir_mis_mask);
+	const __m128i fdir_flags = _mm_and_si128(fdir_mask, pkt_fdir_bit);
+
+	return fdir_flags;
+}
+
 static inline void
 flex_desc_to_olflags_v(struct iavf_rx_queue *rxq, __m128i descs[4],
 		       struct rte_mbuf **rx_pkts)
@@ -267,6 +286,32 @@ flex_desc_to_olflags_v(struct iavf_rx_queue *rxq, __m128i descs[4],
 	/* merge the flags */
 	flags = _mm_or_si128(flags, rss_vlan);
 
+	if (rxq->vsi->adapter->fdir_enabled) {
+		const __m128i fdir_id0_1 =
+			_mm_unpackhi_epi32(descs[0], descs[1]);
+
+		const __m128i fdir_id2_3 =
+			_mm_unpackhi_epi32(descs[2], descs[3]);
+
+		const __m128i fdir_id0_3 =
+			_mm_unpackhi_epi64(fdir_id0_1, fdir_id2_3);
+
+		const __m128i fdir_flags =
+			flex_rxd_to_fdir_flags_vec(fdir_id0_3);
+
+		/* merge with fdir_flags */
+		flags = _mm_or_si128(flags, fdir_flags);
+
+		/* write fdir_id to mbuf if FDIR match */
+		uint32_t fdir_id_extr, i;
+
+		for (i = 0; i < IAVF_VPMD_DESCS_PER_LOOP; i++) {
+			fdir_id_extr = _mm_extract_epi32(fdir_id0_3, i);
+			if (fdir_id_extr != FDID_MIS_MAGIC)
+				rx_pkts[i]->hash.fdir.hi = fdir_id_extr;
+		}
+	} /* if() on fdir_enabled */
+
 	/**
 	 * At this point, we have the 4 sets of flags in the low 16-bits
 	 * of each 32-bit value in flags.
@@ -604,7 +649,8 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 	const __m128i zero = _mm_setzero_si128();
 	/* mask to shuffle from desc. to mbuf */
 	const __m128i shuf_msk = _mm_set_epi8
-			(15, 14, 13, 12,  /* octet 12~15, 32 bits rss */
+			(0xFF, 0xFF,
+			 0xFF, 0xFF,  /* rss not supported */
 			 11, 10,      /* octet 10~11, 16 bits vlan_macip */
 			 5, 4,        /* octet 4~5, 16 bits data_len */
 			 0xFF, 0xFF,  /* skip high 16 bits pkt_len, zero out */
Signature · 1 line
2.17.1
author
#13 lore raw

PATCH 11/12 net/iavf: add RSS hash parsing in AVX path

In reply to Leyi Rong

Support RSS hash parsing from Flex Rx
descriptor in AVX data path.

---
 drivers/net/iavf/iavf_rxtx_vec_avx2.c | 92 ++++++++++++++++++++++++++-
 1 file changed, 90 insertions(+), 2 deletions(-)
drivers/net/iavf/iavf_rxtx_vec_avx2.c +90 -2
index 9bbf75632..efecdd714 100644
@@ -632,7 +632,7 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 		_mm256_set_epi8
 			(/* first descriptor */
 			 0xFF, 0xFF,
-			 0xFF, 0xFF,    /* rss not supported */
+			 0xFF, 0xFF,    /* rss hash parsed separately */
 			 11, 10,	/* octet 10~11, 16 bits vlan_macip */
 			 5, 4,		/* octet 4~5, 16 bits data_len */
 			 0xFF, 0xFF,	/* skip hi 16 bits pkt_len, zero out */
@@ -641,7 +641,7 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 			 0xFF, 0xFF,	/*pkt_type set as unknown */
 			 /* second descriptor */
 			 0xFF, 0xFF,
-			 0xFF, 0xFF,    /* rss not supported */
+			 0xFF, 0xFF,    /* rss hash parsed separately */
 			 11, 10,	/* octet 10~11, 16 bits vlan_macip */
 			 5, 4,		/* octet 4~5, 16 bits data_len */
 			 0xFF, 0xFF,	/* skip hi 16 bits pkt_len, zero out */
@@ -938,6 +938,94 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 				rx_pkts[i + 7]->hash.fdir.hi = fdir_id_extr;
 		} /* if() on fdir_enabled */
 
+		/**
+		 * needs to load 2nd 16B of each desc for RSS hash parsing,
+		 * will cause performance drop to get into this context.
+		 */
+		if (rxq->vsi->adapter->eth_dev->data->dev_conf.rxmode.offloads &
+				DEV_RX_OFFLOAD_RSS_HASH) {
+			/* load bottom half of every 32B desc */
+			const __m128i raw_desc_bh7 =
+				_mm_load_si128
+					((void *)(&rxdp[7].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh6 =
+				_mm_load_si128
+					((void *)(&rxdp[6].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh5 =
+				_mm_load_si128
+					((void *)(&rxdp[5].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh4 =
+				_mm_load_si128
+					((void *)(&rxdp[4].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh3 =
+				_mm_load_si128
+					((void *)(&rxdp[3].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh2 =
+				_mm_load_si128
+					((void *)(&rxdp[2].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh1 =
+				_mm_load_si128
+					((void *)(&rxdp[1].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh0 =
+				_mm_load_si128
+					((void *)(&rxdp[0].wb.status_error1));
+
+			__m256i raw_desc_bh6_7 =
+				_mm256_inserti128_si256
+					(_mm256_castsi128_si256(raw_desc_bh6),
+					raw_desc_bh7, 1);
+			__m256i raw_desc_bh4_5 =
+				_mm256_inserti128_si256
+					(_mm256_castsi128_si256(raw_desc_bh4),
+					raw_desc_bh5, 1);
+			__m256i raw_desc_bh2_3 =
+				_mm256_inserti128_si256
+					(_mm256_castsi128_si256(raw_desc_bh2),
+					raw_desc_bh3, 1);
+			__m256i raw_desc_bh0_1 =
+				_mm256_inserti128_si256
+					(_mm256_castsi128_si256(raw_desc_bh0),
+					raw_desc_bh1, 1);
+
+			/**
+			 * to shift the 32b RSS hash value to the
+			 * highest 32b of each 128b before mask
+			 */
+			__m256i rss_hash6_7 =
+				_mm256_slli_epi64(raw_desc_bh6_7, 32);
+			__m256i rss_hash4_5 =
+				_mm256_slli_epi64(raw_desc_bh4_5, 32);
+			__m256i rss_hash2_3 =
+				_mm256_slli_epi64(raw_desc_bh2_3, 32);
+			__m256i rss_hash0_1 =
+				_mm256_slli_epi64(raw_desc_bh0_1, 32);
+
+			__m256i rss_hash_msk =
+				_mm256_set_epi32(0xFFFFFFFF, 0, 0, 0,
+						 0xFFFFFFFF, 0, 0, 0);
+
+			rss_hash6_7 = _mm256_and_si256
+					(rss_hash6_7, rss_hash_msk);
+			rss_hash4_5 = _mm256_and_si256
+					(rss_hash4_5, rss_hash_msk);
+			rss_hash2_3 = _mm256_and_si256
+					(rss_hash2_3, rss_hash_msk);
+			rss_hash0_1 = _mm256_and_si256
+					(rss_hash0_1, rss_hash_msk);
+
+			mb6_7 = _mm256_or_si256(mb6_7, rss_hash6_7);
+			mb4_5 = _mm256_or_si256(mb4_5, rss_hash4_5);
+			mb2_3 = _mm256_or_si256(mb2_3, rss_hash2_3);
+			mb0_1 = _mm256_or_si256(mb0_1, rss_hash0_1);
+		} /* if() on RSS hash parsing */
+
 		/**
 		 * At this point, we have the 8 sets of flags in the low 16-bits
 		 * of each 32-bit value in vlan0.
Signature · 1 line
2.17.1
author
#14 lore raw

PATCH 12/12 net/iavf: add RSS hash parsing in SSE path

In reply to Leyi Rong

Support RSS hash parsing from Flex Rx
descriptor in SSE data path.

---
 drivers/net/iavf/iavf_rxtx_vec_sse.c | 86 ++++++++++++++++++++++------
 1 file changed, 70 insertions(+), 16 deletions(-)
drivers/net/iavf/iavf_rxtx_vec_sse.c +70 -16
index eb784938f..a615f22e2 100644
@@ -650,7 +650,7 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 	/* mask to shuffle from desc. to mbuf */
 	const __m128i shuf_msk = _mm_set_epi8
 			(0xFF, 0xFF,
-			 0xFF, 0xFF,  /* rss not supported */
+			 0xFF, 0xFF,  /* rss hash parsed separately */
 			 11, 10,      /* octet 10~11, 16 bits vlan_macip */
 			 5, 4,        /* octet 4~5, 16 bits data_len */
 			 0xFF, 0xFF,  /* skip high 16 bits pkt_len, zero out */
@@ -741,7 +741,7 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 	     pos += IAVF_VPMD_DESCS_PER_LOOP,
 	     rxdp += IAVF_VPMD_DESCS_PER_LOOP) {
 		__m128i descs[IAVF_VPMD_DESCS_PER_LOOP];
-		__m128i pkt_mb1, pkt_mb2, pkt_mb3, pkt_mb4;
+		__m128i pkt_mb0, pkt_mb1, pkt_mb2, pkt_mb3;
 		__m128i staterr, sterr_tmp1, sterr_tmp2;
 		/* 2 64 bit or 4 32 bit mbuf pointers in one XMM reg. */
 		__m128i mbp1;
@@ -787,8 +787,12 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 		rte_compiler_barrier();
 
 		/* D.1 pkt 3,4 convert format from desc to pktmbuf */
-		pkt_mb4 = _mm_shuffle_epi8(descs[3], shuf_msk);
-		pkt_mb3 = _mm_shuffle_epi8(descs[2], shuf_msk);
+		pkt_mb3 = _mm_shuffle_epi8(descs[3], shuf_msk);
+		pkt_mb2 = _mm_shuffle_epi8(descs[2], shuf_msk);
+
+		/* D.1 pkt 1,2 convert format from desc to pktmbuf */
+		pkt_mb1 = _mm_shuffle_epi8(descs[1], shuf_msk);
+		pkt_mb0 = _mm_shuffle_epi8(descs[0], shuf_msk);
 
 		/* C.1 4=>2 filter staterr info only */
 		sterr_tmp2 = _mm_unpackhi_epi32(descs[3], descs[2]);
@@ -798,12 +802,66 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 		flex_desc_to_olflags_v(rxq, descs, &rx_pkts[pos]);
 
 		/* D.2 pkt 3,4 set in_port/nb_seg and remove crc */
-		pkt_mb4 = _mm_add_epi16(pkt_mb4, crc_adjust);
 		pkt_mb3 = _mm_add_epi16(pkt_mb3, crc_adjust);
+		pkt_mb2 = _mm_add_epi16(pkt_mb2, crc_adjust);
 
-		/* D.1 pkt 1,2 convert format from desc to pktmbuf */
-		pkt_mb2 = _mm_shuffle_epi8(descs[1], shuf_msk);
-		pkt_mb1 = _mm_shuffle_epi8(descs[0], shuf_msk);
+		/* D.2 pkt 1,2 set in_port/nb_seg and remove crc */
+		pkt_mb1 = _mm_add_epi16(pkt_mb1, crc_adjust);
+		pkt_mb0 = _mm_add_epi16(pkt_mb0, crc_adjust);
+
+		/**
+		 * needs to load 2nd 16B of each desc for RSS hash parsing,
+		 * will cause performance drop to get into this context.
+		 */
+		if (rxq->vsi->adapter->eth_dev->data->dev_conf.rxmode.offloads &
+				DEV_RX_OFFLOAD_RSS_HASH) {
+			/* load bottom half of every 32B desc */
+			const __m128i raw_desc_bh3 =
+				_mm_load_si128
+					((void *)(&rxdp[3].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh2 =
+				_mm_load_si128
+					((void *)(&rxdp[2].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh1 =
+				_mm_load_si128
+					((void *)(&rxdp[1].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh0 =
+				_mm_load_si128
+					((void *)(&rxdp[0].wb.status_error1));
+
+			/**
+			 * to shift the 32b RSS hash value to the
+			 * highest 32b of each 128b before mask
+			 */
+			__m128i rss_hash3 =
+				_mm_slli_epi64(raw_desc_bh3, 32);
+			__m128i rss_hash2 =
+				_mm_slli_epi64(raw_desc_bh2, 32);
+			__m128i rss_hash1 =
+				_mm_slli_epi64(raw_desc_bh1, 32);
+			__m128i rss_hash0 =
+				_mm_slli_epi64(raw_desc_bh0, 32);
+
+			__m128i rss_hash_msk =
+				_mm_set_epi32(0xFFFFFFFF, 0, 0, 0);
+
+			rss_hash3 = _mm_and_si128
+					(rss_hash3, rss_hash_msk);
+			rss_hash2 = _mm_and_si128
+					(rss_hash2, rss_hash_msk);
+			rss_hash1 = _mm_and_si128
+					(rss_hash1, rss_hash_msk);
+			rss_hash0 = _mm_and_si128
+					(rss_hash0, rss_hash_msk);
+
+			pkt_mb3 = _mm_or_si128(pkt_mb3, rss_hash3);
+			pkt_mb2 = _mm_or_si128(pkt_mb2, rss_hash2);
+			pkt_mb1 = _mm_or_si128(pkt_mb1, rss_hash1);
+			pkt_mb0 = _mm_or_si128(pkt_mb0, rss_hash0);
+		} /* if() on RSS hash parsing */
 
 		/* C.2 get 4 pkts staterr value  */
 		staterr = _mm_unpacklo_epi32(sterr_tmp1, sterr_tmp2);
@@ -811,14 +869,10 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 		/* D.3 copy final 3,4 data to rx_pkts */
 		_mm_storeu_si128
 			((void *)&rx_pkts[pos + 3]->rx_descriptor_fields1,
-			 pkt_mb4);
+			 pkt_mb3);
 		_mm_storeu_si128
 			((void *)&rx_pkts[pos + 2]->rx_descriptor_fields1,
-			 pkt_mb3);
-
-		/* D.2 pkt 1,2 set in_port/nb_seg and remove crc */
-		pkt_mb2 = _mm_add_epi16(pkt_mb2, crc_adjust);
-		pkt_mb1 = _mm_add_epi16(pkt_mb1, crc_adjust);
+			 pkt_mb2);
 
 		/* C* extract and record EOP bit */
 		if (split_packet) {
@@ -842,9 +896,9 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 		/* D.3 copy final 1,2 data to rx_pkts */
 		_mm_storeu_si128
 			((void *)&rx_pkts[pos + 1]->rx_descriptor_fields1,
-			 pkt_mb2);
+			 pkt_mb1);
 		_mm_storeu_si128((void *)&rx_pkts[pos]->rx_descriptor_fields1,
-				 pkt_mb1);
+				 pkt_mb0);
 		flex_desc_to_ptype_v(descs, &rx_pkts[pos], ptype_tbl);
 		/* C.4 calc avaialbe number of desc */
 		var = __builtin_popcountll(_mm_cvtsi128_si64(staterr));
Signature · 1 line
2.17.1
author
#15 lore raw

v2 cover framework for advanced iAVF PMD

In reply to Leyi Rong

This patchset enable framework for advanced iAVF, includes query DDP pkg info, flexible descriptor support, FDIR mark id and RSS hash support.

---
v2:
- Revert macro RTE_LIBRTE_IAVF_16BYTE_RX_DESC deletion as it's defined in AVF spec.
- Typo RTE_LIBRTE_ICE_16BYTE_RX_DESC fix in iavf_rxtx.c.
- Move flex desc definitions into iavf_rxtx.h.
- Up to date to match with the latest version of virtchnl.h.
- Extract a new internal func iavf_update_rx_tail to call.
- Remove iavf_dev_rxq_count_flex_rxd()/iavf_dev_rx_desc_status_flex_rxd(), as the accompanying legacy ones can deal with the flex desc cases.
- Move rxq->rxdid assignment from iavf_configure_queues() to iavf_dev_rx_queue_setup().
- Unfold _mm_extract_epi32(fdir_id0_3, i) to fix build error when using GCC compile option -O0.

Leyi Rong (12):
net/iavf: flexible Rx descriptor definitions net/iavf: return error if opcode is mismatched net/iavf: support to query DDP package info net/iavf: flexible Rx descriptor support in normal path net/iavf: flexible Rx descriptor support in AVX path net/iavf: flexible Rx descriptor support in SSE path net/iavf: add flow director enabled switch value net/iavf: support flow mark in normal data path net/iavf: support flow mark in AVX path net/iavf: support flow mark in SSE path net/iavf: add RSS hash parsing in AVX path net/iavf: add RSS hash parsing in SSE path

 drivers/net/iavf/iavf.h               |  15 +
 drivers/net/iavf/iavf_ethdev.c        |  15 +
 drivers/net/iavf/iavf_rxtx.c          | 542 ++++++++++++++++++--
 drivers/net/iavf/iavf_rxtx.h          | 245 +++++++++
 drivers/net/iavf/iavf_rxtx_vec_avx2.c | 710 +++++++++++++++++++++++++-
 drivers/net/iavf/iavf_rxtx_vec_sse.c  | 524 +++++++++++++++++++
 drivers/net/iavf/iavf_vchnl.c         |  85 ++-
 7 files changed, 2087 insertions(+), 49 deletions(-)
Signature · 1 line
2.17.1
author
#16 lore raw

PATCH v2 1/12 net/iavf: flexible Rx descriptor definitions

Add definitions for flexible Rx descriptor structures and macros.

---
 drivers/net/iavf/iavf_rxtx.h | 200 +++++++++++++++++++++++++++++++++++
 1 file changed, 200 insertions(+)
drivers/net/iavf/iavf_rxtx.h +200 -0
index 09b5bd99e..5e309631e 100644
@@ -157,6 +157,206 @@ union iavf_tx_offload {
 	};
 };
 
+/* Rx Flex Descriptors
+ * These descriptors are used instead of the legacy version descriptors
+ */
+union iavf_16b_rx_flex_desc {
+	struct {
+		__le64 pkt_addr; /* Packet buffer address */
+		__le64 hdr_addr; /* Header buffer address */
+				 /* bit 0 of hdr_addr is DD bit */
+	} read;
+	struct {
+		/* Qword 0 */
+		u8 rxdid; /* descriptor builder profile ID */
+		u8 mir_id_umb_cast; /* mirror=[5:0], umb=[7:6] */
+		__le16 ptype_flex_flags0; /* ptype=[9:0], ff0=[15:10] */
+		__le16 pkt_len; /* [15:14] are reserved */
+		__le16 hdr_len_sph_flex_flags1; /* header=[10:0] */
+						/* sph=[11:11] */
+						/* ff1/ext=[15:12] */
+
+		/* Qword 1 */
+		__le16 status_error0;
+		__le16 l2tag1;
+		__le16 flex_meta0;
+		__le16 flex_meta1;
+	} wb; /* writeback */
+};
+
+union iavf_32b_rx_flex_desc {
+	struct {
+		__le64 pkt_addr; /* Packet buffer address */
+		__le64 hdr_addr; /* Header buffer address */
+				 /* bit 0 of hdr_addr is DD bit */
+		__le64 rsvd1;
+		__le64 rsvd2;
+	} read;
+	struct {
+		/* Qword 0 */
+		u8 rxdid; /* descriptor builder profile ID */
+		u8 mir_id_umb_cast; /* mirror=[5:0], umb=[7:6] */
+		__le16 ptype_flex_flags0; /* ptype=[9:0], ff0=[15:10] */
+		__le16 pkt_len; /* [15:14] are reserved */
+		__le16 hdr_len_sph_flex_flags1; /* header=[10:0] */
+						/* sph=[11:11] */
+						/* ff1/ext=[15:12] */
+
+		/* Qword 1 */
+		__le16 status_error0;
+		__le16 l2tag1;
+		__le16 flex_meta0;
+		__le16 flex_meta1;
+
+		/* Qword 2 */
+		__le16 status_error1;
+		u8 flex_flags2;
+		u8 time_stamp_low;
+		__le16 l2tag2_1st;
+		__le16 l2tag2_2nd;
+
+		/* Qword 3 */
+		__le16 flex_meta2;
+		__le16 flex_meta3;
+		union {
+			struct {
+				__le16 flex_meta4;
+				__le16 flex_meta5;
+			} flex;
+			__le32 ts_high;
+		} flex_ts;
+	} wb; /* writeback */
+};
+
+/* Rx Flex Descriptor for Comms Package Profile
+ * RxDID Profile ID 16-21
+ * Flex-field 0: RSS hash lower 16-bits
+ * Flex-field 1: RSS hash upper 16-bits
+ * Flex-field 2: Flow ID lower 16-bits
+ * Flex-field 3: Flow ID upper 16-bits
+ * Flex-field 4: AUX0
+ * Flex-field 5: AUX1
+ */
+struct iavf_32b_rx_flex_desc_comms {
+	/* Qword 0 */
+	u8 rxdid;
+	u8 mir_id_umb_cast;
+	__le16 ptype_flexi_flags0;
+	__le16 pkt_len;
+	__le16 hdr_len_sph_flex_flags1;
+
+	/* Qword 1 */
+	__le16 status_error0;
+	__le16 l2tag1;
+	__le32 rss_hash;
+
+	/* Qword 2 */
+	__le16 status_error1;
+	u8 flexi_flags2;
+	u8 ts_low;
+	__le16 l2tag2_1st;
+	__le16 l2tag2_2nd;
+
+	/* Qword 3 */
+	__le32 flow_id;
+	union {
+		struct {
+			__le16 aux0;
+			__le16 aux1;
+		} flex;
+		__le32 ts_high;
+	} flex_ts;
+};
+
+/* Rx Flex Descriptor for Comms Package Profile
+ * RxDID Profile ID 22-23 (swap Hash and FlowID)
+ * Flex-field 0: Flow ID lower 16-bits
+ * Flex-field 1: Flow ID upper 16-bits
+ * Flex-field 2: RSS hash lower 16-bits
+ * Flex-field 3: RSS hash upper 16-bits
+ * Flex-field 4: AUX0
+ * Flex-field 5: AUX1
+ */
+struct iavf_32b_rx_flex_desc_comms_ovs {
+	/* Qword 0 */
+	u8 rxdid;
+	u8 mir_id_umb_cast;
+	__le16 ptype_flexi_flags0;
+	__le16 pkt_len;
+	__le16 hdr_len_sph_flex_flags1;
+
+	/* Qword 1 */
+	__le16 status_error0;
+	__le16 l2tag1;
+	__le32 flow_id;
+
+	/* Qword 2 */
+	__le16 status_error1;
+	u8 flexi_flags2;
+	u8 ts_low;
+	__le16 l2tag2_1st;
+	__le16 l2tag2_2nd;
+
+	/* Qword 3 */
+	__le32 rss_hash;
+	union {
+		struct {
+			__le16 aux0;
+			__le16 aux1;
+		} flex;
+		__le32 ts_high;
+	} flex_ts;
+};
+
+/* Receive Flex Descriptor profile IDs: There are a total
+ * of 64 profiles where profile IDs 0/1 are for legacy; and
+ * profiles 2-63 are flex profiles that can be programmed
+ * with a specific metadata (profile 7 reserved for HW)
+ */
+enum iavf_rxdid {
+	IAVF_RXDID_LEGACY_0		= 0,
+	IAVF_RXDID_LEGACY_1		= 1,
+	IAVF_RXDID_FLEX_NIC		= 2,
+	IAVF_RXDID_FLEX_NIC_2		= 6,
+	IAVF_RXDID_HW			= 7,
+	IAVF_RXDID_COMMS_GENERIC	= 16,
+	IAVF_RXDID_COMMS_AUX_VLAN	= 17,
+	IAVF_RXDID_COMMS_AUX_IPV4	= 18,
+	IAVF_RXDID_COMMS_AUX_IPV6	= 19,
+	IAVF_RXDID_COMMS_AUX_IPV6_FLOW	= 20,
+	IAVF_RXDID_COMMS_AUX_TCP	= 21,
+	IAVF_RXDID_COMMS_OVS_1		= 22,
+	IAVF_RXDID_COMMS_OVS_2		= 23,
+	IAVF_RXDID_LAST			= 63,
+};
+
+enum iavf_rx_flex_desc_status_error_0_bits {
+	/* Note: These are predefined bit offsets */
+	IAVF_RX_FLEX_DESC_STATUS0_DD_S = 0,
+	IAVF_RX_FLEX_DESC_STATUS0_EOF_S,
+	IAVF_RX_FLEX_DESC_STATUS0_HBO_S,
+	IAVF_RX_FLEX_DESC_STATUS0_L3L4P_S,
+	IAVF_RX_FLEX_DESC_STATUS0_XSUM_IPE_S,
+	IAVF_RX_FLEX_DESC_STATUS0_XSUM_L4E_S,
+	IAVF_RX_FLEX_DESC_STATUS0_XSUM_EIPE_S,
+	IAVF_RX_FLEX_DESC_STATUS0_XSUM_EUDPE_S,
+	IAVF_RX_FLEX_DESC_STATUS0_LPBK_S,
+	IAVF_RX_FLEX_DESC_STATUS0_IPV6EXADD_S,
+	IAVF_RX_FLEX_DESC_STATUS0_RXE_S,
+	IAVF_RX_FLEX_DESC_STATUS0_CRCP_S,
+	IAVF_RX_FLEX_DESC_STATUS0_RSS_VALID_S,
+	IAVF_RX_FLEX_DESC_STATUS0_L2TAG1P_S,
+	IAVF_RX_FLEX_DESC_STATUS0_XTRMD0_VALID_S,
+	IAVF_RX_FLEX_DESC_STATUS0_XTRMD1_VALID_S,
+	IAVF_RX_FLEX_DESC_STATUS0_LAST /* this entry must be last!!! */
+};
+
+/* for iavf_32b_rx_flex_desc.ptype_flex_flags0 member */
+#define IAVF_RX_FLEX_DESC_PTYPE_M	(0x3FF) /* 10-bits */
+
+/* for iavf_32b_rx_flex_desc.pkt_len member */
+#define IAVF_RX_FLX_DESC_PKT_LEN_M	(0x3FFF) /* 14-bits */
+
 int iavf_dev_rx_queue_setup(struct rte_eth_dev *dev,
 			   uint16_t queue_idx,
 			   uint16_t nb_desc,
Signature · 1 line
2.17.1
author
#17 lore raw

PATCH v2 2/12 net/iavf: return error if opcode is mismatched

In reply to Leyi Rong

Adds error return when the opcode of read message is mismatched which is received from adminQ.

---
 drivers/net/iavf/iavf_vchnl.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
drivers/net/iavf/iavf_vchnl.c +3 -1
index fa4da3a6d..b7fb05d32 100644
@@ -52,9 +52,11 @@ iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len,
 	PMD_DRV_LOG(DEBUG, "AQ from pf carries opcode %u, retval %d",
 		    opcode, vf->cmd_retval);
 
-	if (opcode != vf->pend_cmd)
+	if (opcode != vf->pend_cmd) {
 		PMD_DRV_LOG(WARNING, "command mismatch, expect %u, get %u",
 			    vf->pend_cmd, opcode);
+		return IAVF_ERR_OPCODE_MISMATCH;
+	}
 
 	return IAVF_SUCCESS;
 }
Signature · 1 line
2.17.1
author
#18 lore raw

PATCH v2 3/12 net/iavf: support to query DDP package info

In reply to Leyi Rong

Advanced iAVF supports to query DDP package info, includes package version, track id, package name, device serial number and the list of protocols that the PF supports.

---
 drivers/net/iavf/iavf.h        |  9 ++++++++
 drivers/net/iavf/iavf_ethdev.c |  7 ++++++
 drivers/net/iavf/iavf_vchnl.c  | 41 +++++++++++++++++++++++++++++++++-
 3 files changed, 56 insertions(+), 1 deletion(-)
drivers/net/iavf/iavf.h +9 -0
index 526040c6e..39c6eeec9 100644
@@ -67,6 +67,13 @@
 #define IAVF_48_BIT_WIDTH (CHAR_BIT * 6)
 #define IAVF_48_BIT_MASK  RTE_LEN2MASK(IAVF_48_BIT_WIDTH, uint64_t)
 
+/* VF supported comms protocols 64-bits bitmap */
+#define IAVF_COMMS_PROTO_GTP            0x0000000000000001
+#define IAVF_COMMS_PROTO_PPPOE          0x0000000000000002
+#define IAVF_COMMS_PROTO_PFCP           0x0000000000000004
+#define IAVF_COMMS_PROTO_L2TPV3         0x0000000000000008
+#define IAVF_COMMS_PROTO_ESP            0x0000000000000010
+
 struct iavf_adapter;
 struct iavf_rx_queue;
 struct iavf_tx_queue;
@@ -97,6 +104,7 @@ struct iavf_info {
 	struct virtchnl_version_info virtchnl_version;
 	struct virtchnl_vf_resource *vf_res; /* VF resource */
 	struct virtchnl_vsi_resource *vsi_res; /* LAN VSI */
+	struct virtchnl_pkg_info pkg_info; /* package info */
 
 	volatile enum virtchnl_ops pend_cmd; /* pending command not finished */
 	uint32_t cmd_retval; /* return value of the cmd response from PF */
@@ -225,6 +233,7 @@ int iavf_disable_queues(struct iavf_adapter *adapter);
 int iavf_configure_rss_lut(struct iavf_adapter *adapter);
 int iavf_configure_rss_key(struct iavf_adapter *adapter);
 int iavf_configure_queues(struct iavf_adapter *adapter);
+int iavf_query_package_info(struct iavf_adapter *adapter);
 int iavf_config_irq_map(struct iavf_adapter *adapter);
 void iavf_add_del_all_mac_addr(struct iavf_adapter *adapter, bool add);
 int iavf_dev_link_update(struct rte_eth_dev *dev,
drivers/net/iavf/iavf_ethdev.c +7 -0
index 7a8bec9c9..f90e96fd3 100644
@@ -1236,6 +1236,13 @@ iavf_init_vf(struct rte_eth_dev *dev)
 			goto err_rss;
 		}
 	}
+
+	if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_QUERY_DDP) {
+		if (iavf_query_package_info(adapter) != 0) {
+			PMD_INIT_LOG(ERR, "iavf_query_package_info failed");
+			goto err_rss;
+		}
+	}
 	return 0;
 err_rss:
 	rte_free(vf->rss_key);
drivers/net/iavf/iavf_vchnl.c +40 -1

59 lines not on this page · raw message

Signature · 1 line
2.17.1
author
#19 lore raw

PATCH v2 4/12 net/iavf: flexible Rx descriptor support in normal path

In reply to Leyi Rong

Support flexible Rx descriptor format in normal
path of iAVF PMD.

---
 drivers/net/iavf/iavf.h        |   2 +
 drivers/net/iavf/iavf_ethdev.c |   8 +
 drivers/net/iavf/iavf_rxtx.c   | 481 ++++++++++++++++++++++++++++++---
 drivers/net/iavf/iavf_rxtx.h   |   8 +
 drivers/net/iavf/iavf_vchnl.c  |  42 ++-
 5 files changed, 503 insertions(+), 38 deletions(-)
drivers/net/iavf/iavf.h +2 -0
index 39c6eeec9..4fe15237a 100644
@@ -105,6 +105,7 @@ struct iavf_info {
 	struct virtchnl_vf_resource *vf_res; /* VF resource */
 	struct virtchnl_vsi_resource *vsi_res; /* LAN VSI */
 	struct virtchnl_pkg_info pkg_info; /* package info */
+	uint64_t supported_rxdid;
 
 	volatile enum virtchnl_ops pend_cmd; /* pending command not finished */
 	uint32_t cmd_retval; /* return value of the cmd response from PF */
@@ -233,6 +234,7 @@ int iavf_disable_queues(struct iavf_adapter *adapter);
 int iavf_configure_rss_lut(struct iavf_adapter *adapter);
 int iavf_configure_rss_key(struct iavf_adapter *adapter);
 int iavf_configure_queues(struct iavf_adapter *adapter);
+int iavf_get_supported_rxdid(struct iavf_adapter *adapter);
 int iavf_query_package_info(struct iavf_adapter *adapter);
 int iavf_config_irq_map(struct iavf_adapter *adapter);
 void iavf_add_del_all_mac_addr(struct iavf_adapter *adapter, bool add);
drivers/net/iavf/iavf_ethdev.c +8 -0
index f90e96fd3..4f916b413 100644
@@ -1243,6 +1243,14 @@ iavf_init_vf(struct rte_eth_dev *dev)
 			goto err_rss;
 		}
 	}
+
+	if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) {
+		if (iavf_get_supported_rxdid(adapter) != 0) {
+			PMD_INIT_LOG(ERR, "failed to do get supported rxdid");
+			goto err_rss;
+		}
+	}
+
 	return 0;
 err_rss:
 	rte_free(vf->rss_key);
drivers/net/iavf/iavf_rxtx.c +444 -37

548 lines not on this page · raw message

drivers/net/iavf/iavf_rxtx.h +8 -0
index 5e309631e..f33d1df41 100644
@@ -62,6 +62,7 @@
 #define iavf_rx_desc iavf_16byte_rx_desc
 #else
 #define iavf_rx_desc iavf_32byte_rx_desc
+#define iavf_rx_flex_desc iavf_32b_rx_flex_desc
 #endif
 
 struct iavf_rxq_ops {
@@ -87,6 +88,7 @@ struct iavf_rx_queue {
 	struct rte_mbuf *pkt_first_seg; /* first segment of current packet */
 	struct rte_mbuf *pkt_last_seg;  /* last segment of current packet */
 	struct rte_mbuf fake_mbuf;      /* dummy mbuf */
+	uint8_t rxdid;
 
 	/* used for VPMD */
 	uint16_t rxrearm_nb;       /* number of remaining to be re-armed */
@@ -379,9 +381,15 @@ void iavf_dev_tx_queue_release(void *txq);
 void iavf_stop_queues(struct rte_eth_dev *dev);
 uint16_t iavf_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 		       uint16_t nb_pkts);
+uint16_t iavf_recv_pkts_flex_rxd(void *rx_queue,
+				 struct rte_mbuf **rx_pkts,
+				 uint16_t nb_pkts);
 uint16_t iavf_recv_scattered_pkts(void *rx_queue,
 				 struct rte_mbuf **rx_pkts,
 				 uint16_t nb_pkts);
+uint16_t iavf_recv_scattered_pkts_flex_rxd(void *rx_queue,
+					   struct rte_mbuf **rx_pkts,
+					   uint16_t nb_pkts);
 uint16_t iavf_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 		       uint16_t nb_pkts);
 uint16_t iavf_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
drivers/net/iavf/iavf_vchnl.c +41 -1

66 lines not on this page · raw message

Signature · 1 line
2.17.1
author
#20 lore raw

PATCH v2 5/12 net/iavf: flexible Rx descriptor support in AVX path

In reply to Leyi Rong

Support flexible Rx descriptor format in AVX
path of iAVF PMD.

---
 drivers/net/iavf/iavf_rxtx.c          |  24 +-
 drivers/net/iavf/iavf_rxtx.h          |   6 +
 drivers/net/iavf/iavf_rxtx_vec_avx2.c | 550 +++++++++++++++++++++++++-
 3 files changed, 570 insertions(+), 10 deletions(-)
drivers/net/iavf/iavf_rxtx.c +18 -6
index a66002a25..4fa8d2358 100644
@@ -2083,16 +2083,28 @@ iavf_set_rx_function(struct rte_eth_dev *dev)
 				    "Using %sVector Scattered Rx (port %d).",
 				    use_avx2 ? "avx2 " : "",
 				    dev->data->port_id);
-			dev->rx_pkt_burst = use_avx2 ?
-					    iavf_recv_scattered_pkts_vec_avx2 :
-					    iavf_recv_scattered_pkts_vec;
+			if (vf->vf_res->vf_cap_flags &
+				VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC)
+				dev->rx_pkt_burst = use_avx2 ?
+					iavf_recv_scattered_pkts_vec_avx2_flex_rxd :
+					iavf_recv_scattered_pkts_vec;
+			else
+				dev->rx_pkt_burst = use_avx2 ?
+					iavf_recv_scattered_pkts_vec_avx2 :
+					iavf_recv_scattered_pkts_vec;
 		} else {
 			PMD_DRV_LOG(DEBUG, "Using %sVector Rx (port %d).",
 				    use_avx2 ? "avx2 " : "",
 				    dev->data->port_id);
-			dev->rx_pkt_burst = use_avx2 ?
-					    iavf_recv_pkts_vec_avx2 :
-					    iavf_recv_pkts_vec;
+			if (vf->vf_res->vf_cap_flags &
+				VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC)
+				dev->rx_pkt_burst = use_avx2 ?
+					iavf_recv_pkts_vec_avx2_flex_rxd :
+					iavf_recv_pkts_vec;
+			else
+				dev->rx_pkt_burst = use_avx2 ?
+					iavf_recv_pkts_vec_avx2 :
+					iavf_recv_pkts_vec;
 		}
 
 		return;
drivers/net/iavf/iavf_rxtx.h +6 -0
index f33d1df41..8e1db2588 100644
@@ -413,9 +413,15 @@ uint16_t iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
 				  uint16_t nb_pkts);
 uint16_t iavf_recv_pkts_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,
 				 uint16_t nb_pkts);
+uint16_t iavf_recv_pkts_vec_avx2_flex_rxd(void *rx_queue,
+					  struct rte_mbuf **rx_pkts,
+					  uint16_t nb_pkts);
 uint16_t iavf_recv_scattered_pkts_vec_avx2(void *rx_queue,
 					   struct rte_mbuf **rx_pkts,
 					   uint16_t nb_pkts);
+uint16_t iavf_recv_scattered_pkts_vec_avx2_flex_rxd(void *rx_queue,
+						    struct rte_mbuf **rx_pkts,
+						    uint16_t nb_pkts);
 uint16_t iavf_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
 			    uint16_t nb_pkts);
 uint16_t iavf_xmit_pkts_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts,
drivers/net/iavf/iavf_rxtx_vec_avx2.c +546 -4

585 lines not on this page · raw message

Signature · 1 line
2.17.1
author
#21 lore raw

PATCH v2 6/12 net/iavf: flexible Rx descriptor support in SSE path

In reply to Leyi Rong

Support flexible Rx descriptor format in SSE
path of iAVF PMD.

---
 drivers/net/iavf/iavf_rxtx.c         |   4 +-
 drivers/net/iavf/iavf_rxtx.h         |   5 +
 drivers/net/iavf/iavf_rxtx_vec_sse.c | 414 +++++++++++++++++++++++++++
 3 files changed, 421 insertions(+), 2 deletions(-)
drivers/net/iavf/iavf_rxtx.c +2 -2
index 4fa8d2358..d7471e600 100644
@@ -2087,7 +2087,7 @@ iavf_set_rx_function(struct rte_eth_dev *dev)
 				VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC)
 				dev->rx_pkt_burst = use_avx2 ?
 					iavf_recv_scattered_pkts_vec_avx2_flex_rxd :
-					iavf_recv_scattered_pkts_vec;
+					iavf_recv_scattered_pkts_vec_flex_rxd;
 			else
 				dev->rx_pkt_burst = use_avx2 ?
 					iavf_recv_scattered_pkts_vec_avx2 :
@@ -2100,7 +2100,7 @@ iavf_set_rx_function(struct rte_eth_dev *dev)
 				VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC)
 				dev->rx_pkt_burst = use_avx2 ?
 					iavf_recv_pkts_vec_avx2_flex_rxd :
-					iavf_recv_pkts_vec;
+					iavf_recv_pkts_vec_flex_rxd;
 			else
 				dev->rx_pkt_burst = use_avx2 ?
 					iavf_recv_pkts_vec_avx2 :
drivers/net/iavf/iavf_rxtx.h +5 -0
index 8e1db2588..290dd68c1 100644
@@ -406,9 +406,14 @@ int iavf_dev_tx_desc_status(void *tx_queue, uint16_t offset);
 
 uint16_t iavf_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
 			   uint16_t nb_pkts);
+uint16_t iavf_recv_pkts_vec_flex_rxd(void *rx_queue, struct rte_mbuf **rx_pkts,
+				     uint16_t nb_pkts);
 uint16_t iavf_recv_scattered_pkts_vec(void *rx_queue,
 				     struct rte_mbuf **rx_pkts,
 				     uint16_t nb_pkts);
+uint16_t iavf_recv_scattered_pkts_vec_flex_rxd(void *rx_queue,
+					       struct rte_mbuf **rx_pkts,
+					       uint16_t nb_pkts);
 uint16_t iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
 				  uint16_t nb_pkts);
 uint16_t iavf_recv_pkts_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,
drivers/net/iavf/iavf_rxtx_vec_sse.c +414 -0

444 lines not on this page · raw message

Signature · 1 line
2.17.1
author
#22 lore raw

PATCH v2 7/12 net/iavf: add flow director enabled switch value

In reply to Leyi Rong

The commit adds fdir_enabled flag into iavf_adapter structure to identify if fdir id is active. Rx data path can be benefit if fdir id parsing is not needed, especially in vector path.

---
 drivers/net/iavf/iavf.h      |  1 +
 drivers/net/iavf/iavf_rxtx.h | 26 ++++++++++++++++++++++++++
 2 files changed, 27 insertions(+)
drivers/net/iavf/iavf.h +1 -0
index 4fe15237a..1918a67f1 100644
@@ -142,6 +142,7 @@ struct iavf_adapter {
 	bool tx_vec_allowed;
 	const uint32_t *ptype_tbl;
 	bool stopped;
+	uint8_t fdir_enabled;
 };
 
 /* IAVF_DEV_PRIVATE_TO */
drivers/net/iavf/iavf_rxtx.h +26 -0
index 290dd68c1..b4f4c5131 100644
@@ -490,6 +490,32 @@ void iavf_dump_tx_descriptor(const struct iavf_tx_queue *txq,
 	       tx_desc->cmd_type_offset_bsz);
 }
 
+/* Enable/disable flow director Rx processing in data path. */
+static inline
+void iavf_fdir_rx_proc_enable(struct iavf_adapter *ad, bool on)
+{
+	static uint32_t ref_cnt;
+
+	if (on) {
+		/* enable flow director processing */
+		if (ref_cnt++ == 0) {
+			ad->fdir_enabled = on;
+			PMD_DRV_LOG(DEBUG,
+				    "FDIR processing on RX set to %d", on);
+		}
+	} else {
+		if (ref_cnt >= 1) {
+			ref_cnt--;
+
+			if (ref_cnt == 0) {
+				ad->fdir_enabled = on;
+				PMD_DRV_LOG(DEBUG,
+					    "FDIR processing on RX set to %d", on);
+			}
+		}
+	}
+}
+
 #ifdef RTE_LIBRTE_IAVF_DEBUG_DUMP_DESC
 #define IAVF_DUMP_RX_DESC(rxq, desc, rx_id) \
 	iavf_dump_rx_descriptor(rxq, desc, rx_id)
Signature · 1 line
2.17.1
author
#23 lore raw

PATCH v2 8/12 net/iavf: support flow mark in normal data path

In reply to Leyi Rong

Support Flow Director mark ID parsing in normal path.

---
 drivers/net/iavf/iavf.h      |  3 +++
 drivers/net/iavf/iavf_rxtx.c | 37 ++++++++++++++++++++++++++++++++++++
 2 files changed, 40 insertions(+)
drivers/net/iavf/iavf.h +3 -0
index 1918a67f1..533856f21 100644
@@ -74,6 +74,9 @@
 #define IAVF_COMMS_PROTO_L2TPV3         0x0000000000000008
 #define IAVF_COMMS_PROTO_ESP            0x0000000000000010
 
+#define IAVF_RX_DESC_EXT_STATUS_FLEXBH_MASK  0x03
+#define IAVF_RX_DESC_EXT_STATUS_FLEXBH_FD_ID 0x01
+
 struct iavf_adapter;
 struct iavf_rx_queue;
 struct iavf_tx_queue;
drivers/net/iavf/iavf_rxtx.c +37 -0
index d7471e600..848d28333 100644
@@ -756,6 +756,10 @@ iavf_rxd_to_pkt_flags(uint64_t qword)
 					IAVF_RX_DESC_FLTSTAT_RSS_HASH) ==
 			IAVF_RX_DESC_FLTSTAT_RSS_HASH) ? PKT_RX_RSS_HASH : 0;
 
+	/* Check if FDIR Match */
+	flags |= (qword & (1 << IAVF_RX_DESC_STATUS_FLM_SHIFT) ?
+				PKT_RX_FDIR : 0);
+
 	if (likely((error_bits & IAVF_RX_ERR_BITS) == 0)) {
 		flags |= (PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD);
 		return flags;
@@ -776,6 +780,25 @@ iavf_rxd_to_pkt_flags(uint64_t qword)
 	return flags;
 }
 
+static inline uint64_t
+iavf_rxd_build_fdir(volatile union iavf_rx_desc *rxdp, struct rte_mbuf *mb)
+{
+	uint64_t flags = 0;
+	uint16_t flexbh;
+
+	flexbh = (rte_le_to_cpu_32(rxdp->wb.qword2.ext_status) >>
+		IAVF_RX_DESC_EXT_STATUS_FLEXBH_SHIFT) &
+		IAVF_RX_DESC_EXT_STATUS_FLEXBH_MASK;
+
+	if (flexbh == IAVF_RX_DESC_EXT_STATUS_FLEXBH_FD_ID) {
+		mb->hash.fdir.hi =
+			rte_le_to_cpu_32(rxdp->wb.qword3.hi_dword.fd_id);
+		flags |= PKT_RX_FDIR_ID;
+	}
+
+	return flags;
+}
+
 /* Translate the rx flex descriptor status to pkt flags */
 static inline void
 iavf_rxd_to_pkt_fields(struct rte_mbuf *mb,
@@ -792,6 +815,11 @@ iavf_rxd_to_pkt_fields(struct rte_mbuf *mb,
 		mb->hash.rss = rte_le_to_cpu_32(desc->rss_hash);
 	}
 #endif
+
+	if (desc->flow_id != 0xFFFFFFFF) {
+		mb->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID;
+		mb->hash.fdir.hi = rte_le_to_cpu_32(desc->flow_id);
+	}
 }
 
 #define IAVF_RX_FLEX_ERR0_BITS	\
@@ -953,6 +981,9 @@ iavf_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 			rxm->hash.rss =
 				rte_le_to_cpu_32(rxd.wb.qword0.hi_dword.rss);
 
+		if (pkt_flags & PKT_RX_FDIR)
+			pkt_flags |= iavf_rxd_build_fdir(&rxd, rxm);
+
 		rxm->ol_flags |= pkt_flags;
 
 		rx_pkts[nb_rx++] = rxm;
@@ -1351,6 +1382,9 @@ iavf_recv_scattered_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 			first_seg->hash.rss =
 				rte_le_to_cpu_32(rxd.wb.qword0.hi_dword.rss);
 
+		if (pkt_flags & PKT_RX_FDIR)
+			pkt_flags |= iavf_rxd_build_fdir(&rxd, first_seg);
+
 		first_seg->ol_flags |= pkt_flags;
 
 		/* Prefetch data of first segment, if configured to do so. */
@@ -1517,6 +1551,9 @@ iavf_rx_scan_hw_ring(struct iavf_rx_queue *rxq)
 				mb->hash.rss = rte_le_to_cpu_32(
 					rxdp[j].wb.qword0.hi_dword.rss);
 
+			if (pkt_flags & PKT_RX_FDIR)
+				pkt_flags |= iavf_rxd_build_fdir(&rxdp[j], mb);
+
 			mb->ol_flags |= pkt_flags;
 		}
 
Signature · 1 line
2.17.1
author
#24 lore raw

PATCH v2 9/12 net/iavf: support flow mark in AVX path

In reply to Leyi Rong

Support Flow Director mark ID parsing from Flex
Rx descriptor in AVX path.

---
 drivers/net/iavf/iavf_rxtx_vec_avx2.c | 82 +++++++++++++++++++++++++--
 1 file changed, 77 insertions(+), 5 deletions(-)
drivers/net/iavf/iavf_rxtx_vec_avx2.c +77 -5
index b23188fd3..340bf5556 100644
@@ -616,6 +616,25 @@ _iavf_recv_raw_pkts_vec_avx2(struct iavf_rx_queue *rxq,
 	return received;
 }
 
+static inline __m256i
+flex_rxd_to_fdir_flags_vec_avx2(const __m256i fdir_id0_7)
+{
+#define FDID_MIS_MAGIC 0xFFFFFFFF
+	RTE_BUILD_BUG_ON(PKT_RX_FDIR != (1 << 2));
+	RTE_BUILD_BUG_ON(PKT_RX_FDIR_ID != (1 << 13));
+	const __m256i pkt_fdir_bit = _mm256_set1_epi32(PKT_RX_FDIR |
+			PKT_RX_FDIR_ID);
+	/* desc->flow_id field == 0xFFFFFFFF means fdir mismatch */
+	const __m256i fdir_mis_mask = _mm256_set1_epi32(FDID_MIS_MAGIC);
+	__m256i fdir_mask = _mm256_cmpeq_epi32(fdir_id0_7,
+			fdir_mis_mask);
+	/* this XOR op results to bit-reverse the fdir_mask */
+	fdir_mask = _mm256_xor_si256(fdir_mask, fdir_mis_mask);
+	const __m256i fdir_flags = _mm256_and_si256(fdir_mask, pkt_fdir_bit);
+
+	return fdir_flags;
+}
+
 static inline uint16_t
 _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 				      struct rte_mbuf **rx_pkts,
@@ -678,8 +697,8 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 	const __m256i shuf_msk =
 		_mm256_set_epi8
 			(/* first descriptor */
-			 15, 14,
-			 13, 12,	/* octet 12~15, 32 bits rss */
+			 0xFF, 0xFF,
+			 0xFF, 0xFF,    /* rss not supported */
 			 11, 10,	/* octet 10~11, 16 bits vlan_macip */
 			 5, 4,		/* octet 4~5, 16 bits data_len */
 			 0xFF, 0xFF,	/* skip hi 16 bits pkt_len, zero out */
@@ -687,8 +706,8 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 			 0xFF, 0xFF,	/* pkt_type set as unknown */
 			 0xFF, 0xFF,	/*pkt_type set as unknown */
 			 /* second descriptor */
-			 15, 14,
-			 13, 12,	/* octet 12~15, 32 bits rss */
+			 0xFF, 0xFF,
+			 0xFF, 0xFF,    /* rss not supported */
 			 11, 10,	/* octet 10~11, 16 bits vlan_macip */
 			 5, 4,		/* octet 4~5, 16 bits data_len */
 			 0xFF, 0xFF,	/* skip hi 16 bits pkt_len, zero out */
@@ -930,8 +949,61 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 					    rss_vlan_flag_bits);
 
 		/* merge flags */
-		const __m256i mbuf_flags = _mm256_or_si256(l3_l4_flags,
+		__m256i mbuf_flags = _mm256_or_si256(l3_l4_flags,
 				rss_vlan_flags);
+
+		if (rxq->vsi->adapter->fdir_enabled) {
+			const __m256i fdir_id4_7 =
+				_mm256_unpackhi_epi32(raw_desc6_7, raw_desc4_5);
+
+			const __m256i fdir_id0_3 =
+				_mm256_unpackhi_epi32(raw_desc2_3, raw_desc0_1);
+
+			const __m256i fdir_id0_7 =
+				_mm256_unpackhi_epi64(fdir_id4_7, fdir_id0_3);
+
+			const __m256i fdir_flags =
+				flex_rxd_to_fdir_flags_vec_avx2(fdir_id0_7);
+
+			/* merge with fdir_flags */
+			mbuf_flags = _mm256_or_si256(mbuf_flags, fdir_flags);
+
+			/* write to mbuf: have to use scalar store here */
+			uint32_t fdir_id_extr;
+
+			fdir_id_extr = _mm256_extract_epi32(fdir_id0_7, 3);
+			if (fdir_id_extr != FDID_MIS_MAGIC)
+				rx_pkts[i + 0]->hash.fdir.hi = fdir_id_extr;
+
+			fdir_id_extr = _mm256_extract_epi32(fdir_id0_7, 7);
+			if (fdir_id_extr != FDID_MIS_MAGIC)
+				rx_pkts[i + 1]->hash.fdir.hi = fdir_id_extr;
+
+			fdir_id_extr = _mm256_extract_epi32(fdir_id0_7, 2);
+			if (fdir_id_extr != FDID_MIS_MAGIC)
+				rx_pkts[i + 2]->hash.fdir.hi = fdir_id_extr;
+
+			fdir_id_extr = _mm256_extract_epi32(fdir_id0_7, 6);
+			if (fdir_id_extr != FDID_MIS_MAGIC)
+				rx_pkts[i + 3]->hash.fdir.hi = fdir_id_extr;
+
+			fdir_id_extr = _mm256_extract_epi32(fdir_id0_7, 1);
+			if (fdir_id_extr != FDID_MIS_MAGIC)
+				rx_pkts[i + 4]->hash.fdir.hi = fdir_id_extr;
+
+			fdir_id_extr = _mm256_extract_epi32(fdir_id0_7, 5);
+			if (fdir_id_extr != FDID_MIS_MAGIC)
+				rx_pkts[i + 5]->hash.fdir.hi = fdir_id_extr;
+
+			fdir_id_extr = _mm256_extract_epi32(fdir_id0_7, 0);
+			if (fdir_id_extr != FDID_MIS_MAGIC)
+				rx_pkts[i + 6]->hash.fdir.hi = fdir_id_extr;
+
+			fdir_id_extr = _mm256_extract_epi32(fdir_id0_7, 4);
+			if (fdir_id_extr != FDID_MIS_MAGIC)
+				rx_pkts[i + 7]->hash.fdir.hi = fdir_id_extr;
+		} /* if() on fdir_enabled */
+
 		/**
 		 * At this point, we have the 8 sets of flags in the low 16-bits
 		 * of each 32-bit value in vlan0.
Signature · 1 line
2.17.1
author
#25 lore raw

PATCH v2 10/12 net/iavf: support flow mark in SSE path

In reply to Leyi Rong

Support Flow Director mark ID parsing from Flex
Rx descriptor in SSE path.

---
 drivers/net/iavf/iavf_rxtx_vec_sse.c | 58 +++++++++++++++++++++++++++-
 1 file changed, 57 insertions(+), 1 deletion(-)
drivers/net/iavf/iavf_rxtx_vec_sse.c +57 -1
index 9c1f2a445..801a81620 100644
@@ -189,6 +189,25 @@ desc_to_olflags_v(struct iavf_rx_queue *rxq, __m128i descs[4],
 	_mm_store_si128((__m128i *)&rx_pkts[3]->rearm_data, rearm3);
 }
 
+static inline __m128i
+flex_rxd_to_fdir_flags_vec(const __m128i fdir_id0_3)
+{
+#define FDID_MIS_MAGIC 0xFFFFFFFF
+	RTE_BUILD_BUG_ON(PKT_RX_FDIR != (1 << 2));
+	RTE_BUILD_BUG_ON(PKT_RX_FDIR_ID != (1 << 13));
+	const __m128i pkt_fdir_bit = _mm_set1_epi32(PKT_RX_FDIR |
+			PKT_RX_FDIR_ID);
+	/* desc->flow_id field == 0xFFFFFFFF means fdir mismatch */
+	const __m128i fdir_mis_mask = _mm_set1_epi32(FDID_MIS_MAGIC);
+	__m128i fdir_mask = _mm_cmpeq_epi32(fdir_id0_3,
+			fdir_mis_mask);
+	/* this XOR op results to bit-reverse the fdir_mask */
+	fdir_mask = _mm_xor_si128(fdir_mask, fdir_mis_mask);
+	const __m128i fdir_flags = _mm_and_si128(fdir_mask, pkt_fdir_bit);
+
+	return fdir_flags;
+}
+
 static inline void
 flex_desc_to_olflags_v(struct iavf_rx_queue *rxq, __m128i descs[4],
 		       struct rte_mbuf **rx_pkts)
@@ -267,6 +286,42 @@ flex_desc_to_olflags_v(struct iavf_rx_queue *rxq, __m128i descs[4],
 	/* merge the flags */
 	flags = _mm_or_si128(flags, rss_vlan);
 
+	if (rxq->vsi->adapter->fdir_enabled) {
+		const __m128i fdir_id0_1 =
+			_mm_unpackhi_epi32(descs[0], descs[1]);
+
+		const __m128i fdir_id2_3 =
+			_mm_unpackhi_epi32(descs[2], descs[3]);
+
+		const __m128i fdir_id0_3 =
+			_mm_unpackhi_epi64(fdir_id0_1, fdir_id2_3);
+
+		const __m128i fdir_flags =
+			flex_rxd_to_fdir_flags_vec(fdir_id0_3);
+
+		/* merge with fdir_flags */
+		flags = _mm_or_si128(flags, fdir_flags);
+
+		/* write fdir_id to mbuf if FDIR match */
+		uint32_t fdir_id_extr;
+
+		fdir_id_extr = _mm_extract_epi32(fdir_id0_3, 0);
+		if (fdir_id_extr != FDID_MIS_MAGIC)
+			rx_pkts[0]->hash.fdir.hi = fdir_id_extr;
+
+		fdir_id_extr = _mm_extract_epi32(fdir_id0_3, 1);
+		if (fdir_id_extr != FDID_MIS_MAGIC)
+			rx_pkts[1]->hash.fdir.hi = fdir_id_extr;
+
+		fdir_id_extr = _mm_extract_epi32(fdir_id0_3, 2);
+		if (fdir_id_extr != FDID_MIS_MAGIC)
+			rx_pkts[2]->hash.fdir.hi = fdir_id_extr;
+
+		fdir_id_extr = _mm_extract_epi32(fdir_id0_3, 3);
+		if (fdir_id_extr != FDID_MIS_MAGIC)
+			rx_pkts[3]->hash.fdir.hi = fdir_id_extr;
+	} /* if() on fdir_enabled */
+
 	/**
 	 * At this point, we have the 4 sets of flags in the low 16-bits
 	 * of each 32-bit value in flags.
@@ -604,7 +659,8 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 	const __m128i zero = _mm_setzero_si128();
 	/* mask to shuffle from desc. to mbuf */
 	const __m128i shuf_msk = _mm_set_epi8
-			(15, 14, 13, 12,  /* octet 12~15, 32 bits rss */
+			(0xFF, 0xFF,
+			 0xFF, 0xFF,  /* rss not supported */
 			 11, 10,      /* octet 10~11, 16 bits vlan_macip */
 			 5, 4,        /* octet 4~5, 16 bits data_len */
 			 0xFF, 0xFF,  /* skip high 16 bits pkt_len, zero out */
Signature · 1 line
2.17.1
author
#26 lore raw

PATCH v2 11/12 net/iavf: add RSS hash parsing in AVX path

In reply to Leyi Rong

Support RSS hash parsing from Flex Rx
descriptor in AVX data path.

---
 drivers/net/iavf/iavf_rxtx_vec_avx2.c | 92 ++++++++++++++++++++++++++-
 1 file changed, 90 insertions(+), 2 deletions(-)
drivers/net/iavf/iavf_rxtx_vec_avx2.c +90 -2
index 340bf5556..b2b0406da 100644
@@ -698,7 +698,7 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 		_mm256_set_epi8
 			(/* first descriptor */
 			 0xFF, 0xFF,
-			 0xFF, 0xFF,    /* rss not supported */
+			 0xFF, 0xFF,    /* rss hash parsed separately */
 			 11, 10,	/* octet 10~11, 16 bits vlan_macip */
 			 5, 4,		/* octet 4~5, 16 bits data_len */
 			 0xFF, 0xFF,	/* skip hi 16 bits pkt_len, zero out */
@@ -707,7 +707,7 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 			 0xFF, 0xFF,	/*pkt_type set as unknown */
 			 /* second descriptor */
 			 0xFF, 0xFF,
-			 0xFF, 0xFF,    /* rss not supported */
+			 0xFF, 0xFF,    /* rss hash parsed separately */
 			 11, 10,	/* octet 10~11, 16 bits vlan_macip */
 			 5, 4,		/* octet 4~5, 16 bits data_len */
 			 0xFF, 0xFF,	/* skip hi 16 bits pkt_len, zero out */
@@ -1004,6 +1004,94 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 				rx_pkts[i + 7]->hash.fdir.hi = fdir_id_extr;
 		} /* if() on fdir_enabled */
 
+		/**
+		 * needs to load 2nd 16B of each desc for RSS hash parsing,
+		 * will cause performance drop to get into this context.
+		 */
+		if (rxq->vsi->adapter->eth_dev->data->dev_conf.rxmode.offloads &
+				DEV_RX_OFFLOAD_RSS_HASH) {
+			/* load bottom half of every 32B desc */
+			const __m128i raw_desc_bh7 =
+				_mm_load_si128
+					((void *)(&rxdp[7].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh6 =
+				_mm_load_si128
+					((void *)(&rxdp[6].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh5 =
+				_mm_load_si128
+					((void *)(&rxdp[5].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh4 =
+				_mm_load_si128
+					((void *)(&rxdp[4].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh3 =
+				_mm_load_si128
+					((void *)(&rxdp[3].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh2 =
+				_mm_load_si128
+					((void *)(&rxdp[2].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh1 =
+				_mm_load_si128
+					((void *)(&rxdp[1].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh0 =
+				_mm_load_si128
+					((void *)(&rxdp[0].wb.status_error1));
+
+			__m256i raw_desc_bh6_7 =
+				_mm256_inserti128_si256
+					(_mm256_castsi128_si256(raw_desc_bh6),
+					raw_desc_bh7, 1);
+			__m256i raw_desc_bh4_5 =
+				_mm256_inserti128_si256
+					(_mm256_castsi128_si256(raw_desc_bh4),
+					raw_desc_bh5, 1);
+			__m256i raw_desc_bh2_3 =
+				_mm256_inserti128_si256
+					(_mm256_castsi128_si256(raw_desc_bh2),
+					raw_desc_bh3, 1);
+			__m256i raw_desc_bh0_1 =
+				_mm256_inserti128_si256
+					(_mm256_castsi128_si256(raw_desc_bh0),
+					raw_desc_bh1, 1);
+
+			/**
+			 * to shift the 32b RSS hash value to the
+			 * highest 32b of each 128b before mask
+			 */
+			__m256i rss_hash6_7 =
+				_mm256_slli_epi64(raw_desc_bh6_7, 32);
+			__m256i rss_hash4_5 =
+				_mm256_slli_epi64(raw_desc_bh4_5, 32);
+			__m256i rss_hash2_3 =
+				_mm256_slli_epi64(raw_desc_bh2_3, 32);
+			__m256i rss_hash0_1 =
+				_mm256_slli_epi64(raw_desc_bh0_1, 32);
+
+			__m256i rss_hash_msk =
+				_mm256_set_epi32(0xFFFFFFFF, 0, 0, 0,
+						 0xFFFFFFFF, 0, 0, 0);
+
+			rss_hash6_7 = _mm256_and_si256
+					(rss_hash6_7, rss_hash_msk);
+			rss_hash4_5 = _mm256_and_si256
+					(rss_hash4_5, rss_hash_msk);
+			rss_hash2_3 = _mm256_and_si256
+					(rss_hash2_3, rss_hash_msk);
+			rss_hash0_1 = _mm256_and_si256
+					(rss_hash0_1, rss_hash_msk);
+
+			mb6_7 = _mm256_or_si256(mb6_7, rss_hash6_7);
+			mb4_5 = _mm256_or_si256(mb4_5, rss_hash4_5);
+			mb2_3 = _mm256_or_si256(mb2_3, rss_hash2_3);
+			mb0_1 = _mm256_or_si256(mb0_1, rss_hash0_1);
+		} /* if() on RSS hash parsing */
+
 		/**
 		 * At this point, we have the 8 sets of flags in the low 16-bits
 		 * of each 32-bit value in vlan0.
Signature · 1 line
2.17.1
author
#27 lore raw

PATCH v2 12/12 net/iavf: add RSS hash parsing in SSE path

In reply to Leyi Rong

Support RSS hash parsing from Flex Rx
descriptor in SSE data path.

---
 drivers/net/iavf/iavf_rxtx_vec_sse.c | 86 ++++++++++++++++++++++------
 1 file changed, 70 insertions(+), 16 deletions(-)
drivers/net/iavf/iavf_rxtx_vec_sse.c +70 -16
index 801a81620..5ae640743 100644
@@ -660,7 +660,7 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 	/* mask to shuffle from desc. to mbuf */
 	const __m128i shuf_msk = _mm_set_epi8
 			(0xFF, 0xFF,
-			 0xFF, 0xFF,  /* rss not supported */
+			 0xFF, 0xFF,  /* rss hash parsed separately */
 			 11, 10,      /* octet 10~11, 16 bits vlan_macip */
 			 5, 4,        /* octet 4~5, 16 bits data_len */
 			 0xFF, 0xFF,  /* skip high 16 bits pkt_len, zero out */
@@ -751,7 +751,7 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 	     pos += IAVF_VPMD_DESCS_PER_LOOP,
 	     rxdp += IAVF_VPMD_DESCS_PER_LOOP) {
 		__m128i descs[IAVF_VPMD_DESCS_PER_LOOP];
-		__m128i pkt_mb1, pkt_mb2, pkt_mb3, pkt_mb4;
+		__m128i pkt_mb0, pkt_mb1, pkt_mb2, pkt_mb3;
 		__m128i staterr, sterr_tmp1, sterr_tmp2;
 		/* 2 64 bit or 4 32 bit mbuf pointers in one XMM reg. */
 		__m128i mbp1;
@@ -797,8 +797,12 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 		rte_compiler_barrier();
 
 		/* D.1 pkt 3,4 convert format from desc to pktmbuf */
-		pkt_mb4 = _mm_shuffle_epi8(descs[3], shuf_msk);
-		pkt_mb3 = _mm_shuffle_epi8(descs[2], shuf_msk);
+		pkt_mb3 = _mm_shuffle_epi8(descs[3], shuf_msk);
+		pkt_mb2 = _mm_shuffle_epi8(descs[2], shuf_msk);
+
+		/* D.1 pkt 1,2 convert format from desc to pktmbuf */
+		pkt_mb1 = _mm_shuffle_epi8(descs[1], shuf_msk);
+		pkt_mb0 = _mm_shuffle_epi8(descs[0], shuf_msk);
 
 		/* C.1 4=>2 filter staterr info only */
 		sterr_tmp2 = _mm_unpackhi_epi32(descs[3], descs[2]);
@@ -808,12 +812,66 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 		flex_desc_to_olflags_v(rxq, descs, &rx_pkts[pos]);
 
 		/* D.2 pkt 3,4 set in_port/nb_seg and remove crc */
-		pkt_mb4 = _mm_add_epi16(pkt_mb4, crc_adjust);
 		pkt_mb3 = _mm_add_epi16(pkt_mb3, crc_adjust);
+		pkt_mb2 = _mm_add_epi16(pkt_mb2, crc_adjust);
 
-		/* D.1 pkt 1,2 convert format from desc to pktmbuf */
-		pkt_mb2 = _mm_shuffle_epi8(descs[1], shuf_msk);
-		pkt_mb1 = _mm_shuffle_epi8(descs[0], shuf_msk);
+		/* D.2 pkt 1,2 set in_port/nb_seg and remove crc */
+		pkt_mb1 = _mm_add_epi16(pkt_mb1, crc_adjust);
+		pkt_mb0 = _mm_add_epi16(pkt_mb0, crc_adjust);
+
+		/**
+		 * needs to load 2nd 16B of each desc for RSS hash parsing,
+		 * will cause performance drop to get into this context.
+		 */
+		if (rxq->vsi->adapter->eth_dev->data->dev_conf.rxmode.offloads &
+				DEV_RX_OFFLOAD_RSS_HASH) {
+			/* load bottom half of every 32B desc */
+			const __m128i raw_desc_bh3 =
+				_mm_load_si128
+					((void *)(&rxdp[3].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh2 =
+				_mm_load_si128
+					((void *)(&rxdp[2].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh1 =
+				_mm_load_si128
+					((void *)(&rxdp[1].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh0 =
+				_mm_load_si128
+					((void *)(&rxdp[0].wb.status_error1));
+
+			/**
+			 * to shift the 32b RSS hash value to the
+			 * highest 32b of each 128b before mask
+			 */
+			__m128i rss_hash3 =
+				_mm_slli_epi64(raw_desc_bh3, 32);
+			__m128i rss_hash2 =
+				_mm_slli_epi64(raw_desc_bh2, 32);
+			__m128i rss_hash1 =
+				_mm_slli_epi64(raw_desc_bh1, 32);
+			__m128i rss_hash0 =
+				_mm_slli_epi64(raw_desc_bh0, 32);
+
+			__m128i rss_hash_msk =
+				_mm_set_epi32(0xFFFFFFFF, 0, 0, 0);
+
+			rss_hash3 = _mm_and_si128
+					(rss_hash3, rss_hash_msk);
+			rss_hash2 = _mm_and_si128
+					(rss_hash2, rss_hash_msk);
+			rss_hash1 = _mm_and_si128
+					(rss_hash1, rss_hash_msk);
+			rss_hash0 = _mm_and_si128
+					(rss_hash0, rss_hash_msk);
+
+			pkt_mb3 = _mm_or_si128(pkt_mb3, rss_hash3);
+			pkt_mb2 = _mm_or_si128(pkt_mb2, rss_hash2);
+			pkt_mb1 = _mm_or_si128(pkt_mb1, rss_hash1);
+			pkt_mb0 = _mm_or_si128(pkt_mb0, rss_hash0);
+		} /* if() on RSS hash parsing */
 
 		/* C.2 get 4 pkts staterr value  */
 		staterr = _mm_unpacklo_epi32(sterr_tmp1, sterr_tmp2);
@@ -821,14 +879,10 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 		/* D.3 copy final 3,4 data to rx_pkts */
 		_mm_storeu_si128
 			((void *)&rx_pkts[pos + 3]->rx_descriptor_fields1,
-			 pkt_mb4);
+			 pkt_mb3);
 		_mm_storeu_si128
 			((void *)&rx_pkts[pos + 2]->rx_descriptor_fields1,
-			 pkt_mb3);
-
-		/* D.2 pkt 1,2 set in_port/nb_seg and remove crc */
-		pkt_mb2 = _mm_add_epi16(pkt_mb2, crc_adjust);
-		pkt_mb1 = _mm_add_epi16(pkt_mb1, crc_adjust);
+			 pkt_mb2);
 
 		/* C* extract and record EOP bit */
 		if (split_packet) {
@@ -852,9 +906,9 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 		/* D.3 copy final 1,2 data to rx_pkts */
 		_mm_storeu_si128
 			((void *)&rx_pkts[pos + 1]->rx_descriptor_fields1,
-			 pkt_mb2);
+			 pkt_mb1);
 		_mm_storeu_si128((void *)&rx_pkts[pos]->rx_descriptor_fields1,
-				 pkt_mb1);
+				 pkt_mb0);
 		flex_desc_to_ptype_v(descs, &rx_pkts[pos], ptype_tbl);
 		/* C.4 calc avaialbe number of desc */
 		var = __builtin_popcountll(_mm_cvtsi128_si64(staterr));
Signature · 1 line
2.17.1
author
#28 lore raw

v3 cover framework for advanced iAVF PMD

In reply to Leyi Rong

This patchset enable framework for advanced iAVF, includes flexible descriptor support, FDIR mark id and RSS hash support.

---
v3:
- Remove patch to query DDP package info as it's specific to DCF.
- Specify fdir_ref_cnt as per VF value for enable/disable FDIR ID parse.
- Move fdir_enabled flag to per Queue value for cache benefit when access in vector routines.
- Store extracted flow_id value to mbuf without judgement to avoid branch speculation fail, which is benefit to performance.

v2:
- Revert macro RTE_LIBRTE_IAVF_16BYTE_RX_DESC deletion as it's defined in AVF spec.
- Typo RTE_LIBRTE_ICE_16BYTE_RX_DESC fix in iavf_rxtx.c.
- Move flex desc definitions into iavf_rxtx.h.
- Up to date to match with the latest version of virtchnl.h.
- Extract a new internal func iavf_update_rx_tail to call.
- Remove iavf_dev_rxq_count_flex_rxd()/iavf_dev_rx_desc_status_flex_rxd(), as the accompanying legacy ones can deal with the flex desc cases.
- Move rxq->rxdid assignment from iavf_configure_queues() to iavf_dev_rx_queue_setup().
- Unfold _mm_extract_epi32(fdir_id0_3, i) to fix build error when using GCC compile option -O0.

Leyi Rong (11):
net/iavf: flexible Rx descriptor definitions net/iavf: return error if opcode is mismatched net/iavf: flexible Rx descriptor support in normal path net/iavf: flexible Rx descriptor support in AVX path net/iavf: flexible Rx descriptor support in SSE path net/iavf: add flow director enabled switch value net/iavf: support flow mark in normal data path net/iavf: support flow mark in AVX path net/iavf: support flow mark in SSE path net/iavf: add RSS hash parsing in AVX path net/iavf: add RSS hash parsing in SSE path

 drivers/net/iavf/iavf.h               |   6 +
 drivers/net/iavf/iavf_ethdev.c        |   8 +
 drivers/net/iavf/iavf_rxtx.c          | 540 ++++++++++++++++++--
 drivers/net/iavf/iavf_rxtx.h          | 248 +++++++++
 drivers/net/iavf/iavf_rxtx_vec_avx2.c | 700 +++++++++++++++++++++++++-
 drivers/net/iavf/iavf_rxtx_vec_sse.c  | 518 +++++++++++++++++++
 drivers/net/iavf/iavf_vchnl.c         |  46 +-
 7 files changed, 2017 insertions(+), 49 deletions(-)
Signature · 1 line
2.17.1
author
#29 lore raw

PATCH v3 1/11 net/iavf: flexible Rx descriptor definitions

Add definitions for flexible Rx descriptor structures and macros.

---
 drivers/net/iavf/iavf_rxtx.h | 200 +++++++++++++++++++++++++++++++++++
 1 file changed, 200 insertions(+)
drivers/net/iavf/iavf_rxtx.h +200 -0
index 09b5bd99e..5e309631e 100644
@@ -157,6 +157,206 @@ union iavf_tx_offload {
 	};
 };
 
+/* Rx Flex Descriptors
+ * These descriptors are used instead of the legacy version descriptors
+ */
+union iavf_16b_rx_flex_desc {
+	struct {
+		__le64 pkt_addr; /* Packet buffer address */
+		__le64 hdr_addr; /* Header buffer address */
+				 /* bit 0 of hdr_addr is DD bit */
+	} read;
+	struct {
+		/* Qword 0 */
+		u8 rxdid; /* descriptor builder profile ID */
+		u8 mir_id_umb_cast; /* mirror=[5:0], umb=[7:6] */
+		__le16 ptype_flex_flags0; /* ptype=[9:0], ff0=[15:10] */
+		__le16 pkt_len; /* [15:14] are reserved */
+		__le16 hdr_len_sph_flex_flags1; /* header=[10:0] */
+						/* sph=[11:11] */
+						/* ff1/ext=[15:12] */
+
+		/* Qword 1 */
+		__le16 status_error0;
+		__le16 l2tag1;
+		__le16 flex_meta0;
+		__le16 flex_meta1;
+	} wb; /* writeback */
+};
+
+union iavf_32b_rx_flex_desc {
+	struct {
+		__le64 pkt_addr; /* Packet buffer address */
+		__le64 hdr_addr; /* Header buffer address */
+				 /* bit 0 of hdr_addr is DD bit */
+		__le64 rsvd1;
+		__le64 rsvd2;
+	} read;
+	struct {
+		/* Qword 0 */
+		u8 rxdid; /* descriptor builder profile ID */
+		u8 mir_id_umb_cast; /* mirror=[5:0], umb=[7:6] */
+		__le16 ptype_flex_flags0; /* ptype=[9:0], ff0=[15:10] */
+		__le16 pkt_len; /* [15:14] are reserved */
+		__le16 hdr_len_sph_flex_flags1; /* header=[10:0] */
+						/* sph=[11:11] */
+						/* ff1/ext=[15:12] */
+
+		/* Qword 1 */
+		__le16 status_error0;
+		__le16 l2tag1;
+		__le16 flex_meta0;
+		__le16 flex_meta1;
+
+		/* Qword 2 */
+		__le16 status_error1;
+		u8 flex_flags2;
+		u8 time_stamp_low;
+		__le16 l2tag2_1st;
+		__le16 l2tag2_2nd;
+
+		/* Qword 3 */
+		__le16 flex_meta2;
+		__le16 flex_meta3;
+		union {
+			struct {
+				__le16 flex_meta4;
+				__le16 flex_meta5;
+			} flex;
+			__le32 ts_high;
+		} flex_ts;
+	} wb; /* writeback */
+};
+
+/* Rx Flex Descriptor for Comms Package Profile
+ * RxDID Profile ID 16-21
+ * Flex-field 0: RSS hash lower 16-bits
+ * Flex-field 1: RSS hash upper 16-bits
+ * Flex-field 2: Flow ID lower 16-bits
+ * Flex-field 3: Flow ID upper 16-bits
+ * Flex-field 4: AUX0
+ * Flex-field 5: AUX1
+ */
+struct iavf_32b_rx_flex_desc_comms {
+	/* Qword 0 */
+	u8 rxdid;
+	u8 mir_id_umb_cast;
+	__le16 ptype_flexi_flags0;
+	__le16 pkt_len;
+	__le16 hdr_len_sph_flex_flags1;
+
+	/* Qword 1 */
+	__le16 status_error0;
+	__le16 l2tag1;
+	__le32 rss_hash;
+
+	/* Qword 2 */
+	__le16 status_error1;
+	u8 flexi_flags2;
+	u8 ts_low;
+	__le16 l2tag2_1st;
+	__le16 l2tag2_2nd;
+
+	/* Qword 3 */
+	__le32 flow_id;
+	union {
+		struct {
+			__le16 aux0;
+			__le16 aux1;
+		} flex;
+		__le32 ts_high;
+	} flex_ts;
+};
+
+/* Rx Flex Descriptor for Comms Package Profile
+ * RxDID Profile ID 22-23 (swap Hash and FlowID)
+ * Flex-field 0: Flow ID lower 16-bits
+ * Flex-field 1: Flow ID upper 16-bits
+ * Flex-field 2: RSS hash lower 16-bits
+ * Flex-field 3: RSS hash upper 16-bits
+ * Flex-field 4: AUX0
+ * Flex-field 5: AUX1
+ */
+struct iavf_32b_rx_flex_desc_comms_ovs {
+	/* Qword 0 */
+	u8 rxdid;
+	u8 mir_id_umb_cast;
+	__le16 ptype_flexi_flags0;
+	__le16 pkt_len;
+	__le16 hdr_len_sph_flex_flags1;
+
+	/* Qword 1 */
+	__le16 status_error0;
+	__le16 l2tag1;
+	__le32 flow_id;
+
+	/* Qword 2 */
+	__le16 status_error1;
+	u8 flexi_flags2;
+	u8 ts_low;
+	__le16 l2tag2_1st;
+	__le16 l2tag2_2nd;
+
+	/* Qword 3 */
+	__le32 rss_hash;
+	union {
+		struct {
+			__le16 aux0;
+			__le16 aux1;
+		} flex;
+		__le32 ts_high;
+	} flex_ts;
+};
+
+/* Receive Flex Descriptor profile IDs: There are a total
+ * of 64 profiles where profile IDs 0/1 are for legacy; and
+ * profiles 2-63 are flex profiles that can be programmed
+ * with a specific metadata (profile 7 reserved for HW)
+ */
+enum iavf_rxdid {
+	IAVF_RXDID_LEGACY_0		= 0,
+	IAVF_RXDID_LEGACY_1		= 1,
+	IAVF_RXDID_FLEX_NIC		= 2,
+	IAVF_RXDID_FLEX_NIC_2		= 6,
+	IAVF_RXDID_HW			= 7,
+	IAVF_RXDID_COMMS_GENERIC	= 16,
+	IAVF_RXDID_COMMS_AUX_VLAN	= 17,
+	IAVF_RXDID_COMMS_AUX_IPV4	= 18,
+	IAVF_RXDID_COMMS_AUX_IPV6	= 19,
+	IAVF_RXDID_COMMS_AUX_IPV6_FLOW	= 20,
+	IAVF_RXDID_COMMS_AUX_TCP	= 21,
+	IAVF_RXDID_COMMS_OVS_1		= 22,
+	IAVF_RXDID_COMMS_OVS_2		= 23,
+	IAVF_RXDID_LAST			= 63,
+};
+
+enum iavf_rx_flex_desc_status_error_0_bits {
+	/* Note: These are predefined bit offsets */
+	IAVF_RX_FLEX_DESC_STATUS0_DD_S = 0,
+	IAVF_RX_FLEX_DESC_STATUS0_EOF_S,
+	IAVF_RX_FLEX_DESC_STATUS0_HBO_S,
+	IAVF_RX_FLEX_DESC_STATUS0_L3L4P_S,
+	IAVF_RX_FLEX_DESC_STATUS0_XSUM_IPE_S,
+	IAVF_RX_FLEX_DESC_STATUS0_XSUM_L4E_S,
+	IAVF_RX_FLEX_DESC_STATUS0_XSUM_EIPE_S,
+	IAVF_RX_FLEX_DESC_STATUS0_XSUM_EUDPE_S,
+	IAVF_RX_FLEX_DESC_STATUS0_LPBK_S,
+	IAVF_RX_FLEX_DESC_STATUS0_IPV6EXADD_S,
+	IAVF_RX_FLEX_DESC_STATUS0_RXE_S,
+	IAVF_RX_FLEX_DESC_STATUS0_CRCP_S,
+	IAVF_RX_FLEX_DESC_STATUS0_RSS_VALID_S,
+	IAVF_RX_FLEX_DESC_STATUS0_L2TAG1P_S,
+	IAVF_RX_FLEX_DESC_STATUS0_XTRMD0_VALID_S,
+	IAVF_RX_FLEX_DESC_STATUS0_XTRMD1_VALID_S,
+	IAVF_RX_FLEX_DESC_STATUS0_LAST /* this entry must be last!!! */
+};
+
+/* for iavf_32b_rx_flex_desc.ptype_flex_flags0 member */
+#define IAVF_RX_FLEX_DESC_PTYPE_M	(0x3FF) /* 10-bits */
+
+/* for iavf_32b_rx_flex_desc.pkt_len member */
+#define IAVF_RX_FLX_DESC_PKT_LEN_M	(0x3FFF) /* 14-bits */
+
 int iavf_dev_rx_queue_setup(struct rte_eth_dev *dev,
 			   uint16_t queue_idx,
 			   uint16_t nb_desc,
Signature · 1 line
2.17.1
author
#30 lore raw

PATCH v3 2/11 net/iavf: return error if opcode is mismatched

In reply to Leyi Rong

Adds error return when the opcode of read message is mismatched which is received from adminQ.

---
 drivers/net/iavf/iavf_vchnl.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
drivers/net/iavf/iavf_vchnl.c +3 -1
index fa4da3a6d..b7fb05d32 100644
@@ -52,9 +52,11 @@ iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len,
 	PMD_DRV_LOG(DEBUG, "AQ from pf carries opcode %u, retval %d",
 		    opcode, vf->cmd_retval);
 
-	if (opcode != vf->pend_cmd)
+	if (opcode != vf->pend_cmd) {
 		PMD_DRV_LOG(WARNING, "command mismatch, expect %u, get %u",
 			    vf->pend_cmd, opcode);
+		return IAVF_ERR_OPCODE_MISMATCH;
+	}
 
 	return IAVF_SUCCESS;
 }
Signature · 1 line
2.17.1
author
#31 lore raw

PATCH v3 3/11 net/iavf: flexible Rx descriptor support in normal path

In reply to Leyi Rong

Support flexible Rx descriptor format in normal
path of iAVF PMD.

---
 drivers/net/iavf/iavf.h        |   2 +
 drivers/net/iavf/iavf_ethdev.c |   8 +
 drivers/net/iavf/iavf_rxtx.c   | 479 ++++++++++++++++++++++++++++++---
 drivers/net/iavf/iavf_rxtx.h   |   8 +
 drivers/net/iavf/iavf_vchnl.c  |  42 ++-
 5 files changed, 501 insertions(+), 38 deletions(-)
drivers/net/iavf/iavf.h +2 -0
index 526040c6e..67d625053 100644
@@ -97,6 +97,7 @@ struct iavf_info {
 	struct virtchnl_version_info virtchnl_version;
 	struct virtchnl_vf_resource *vf_res; /* VF resource */
 	struct virtchnl_vsi_resource *vsi_res; /* LAN VSI */
+	uint64_t supported_rxdid;
 
 	volatile enum virtchnl_ops pend_cmd; /* pending command not finished */
 	uint32_t cmd_retval; /* return value of the cmd response from PF */
@@ -225,6 +226,7 @@ int iavf_disable_queues(struct iavf_adapter *adapter);
 int iavf_configure_rss_lut(struct iavf_adapter *adapter);
 int iavf_configure_rss_key(struct iavf_adapter *adapter);
 int iavf_configure_queues(struct iavf_adapter *adapter);
+int iavf_get_supported_rxdid(struct iavf_adapter *adapter);
 int iavf_config_irq_map(struct iavf_adapter *adapter);
 void iavf_add_del_all_mac_addr(struct iavf_adapter *adapter, bool add);
 int iavf_dev_link_update(struct rte_eth_dev *dev,
drivers/net/iavf/iavf_ethdev.c +8 -0
index 7a8bec9c9..32df64121 100644
@@ -1236,6 +1236,14 @@ iavf_init_vf(struct rte_eth_dev *dev)
 			goto err_rss;
 		}
 	}
+
+	if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) {
+		if (iavf_get_supported_rxdid(adapter) != 0) {
+			PMD_INIT_LOG(ERR, "failed to do get supported rxdid");
+			goto err_rss;
+		}
+	}
+
 	return 0;
 err_rss:
 	rte_free(vf->rss_key);
drivers/net/iavf/iavf_rxtx.c +442 -37

546 lines not on this page · raw message

drivers/net/iavf/iavf_rxtx.h +8 -0
index 5e309631e..f33d1df41 100644
@@ -62,6 +62,7 @@
 #define iavf_rx_desc iavf_16byte_rx_desc
 #else
 #define iavf_rx_desc iavf_32byte_rx_desc
+#define iavf_rx_flex_desc iavf_32b_rx_flex_desc
 #endif
 
 struct iavf_rxq_ops {
@@ -87,6 +88,7 @@ struct iavf_rx_queue {
 	struct rte_mbuf *pkt_first_seg; /* first segment of current packet */
 	struct rte_mbuf *pkt_last_seg;  /* last segment of current packet */
 	struct rte_mbuf fake_mbuf;      /* dummy mbuf */
+	uint8_t rxdid;
 
 	/* used for VPMD */
 	uint16_t rxrearm_nb;       /* number of remaining to be re-armed */
@@ -379,9 +381,15 @@ void iavf_dev_tx_queue_release(void *txq);
 void iavf_stop_queues(struct rte_eth_dev *dev);
 uint16_t iavf_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 		       uint16_t nb_pkts);
+uint16_t iavf_recv_pkts_flex_rxd(void *rx_queue,
+				 struct rte_mbuf **rx_pkts,
+				 uint16_t nb_pkts);
 uint16_t iavf_recv_scattered_pkts(void *rx_queue,
 				 struct rte_mbuf **rx_pkts,
 				 uint16_t nb_pkts);
+uint16_t iavf_recv_scattered_pkts_flex_rxd(void *rx_queue,
+					   struct rte_mbuf **rx_pkts,
+					   uint16_t nb_pkts);
 uint16_t iavf_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 		       uint16_t nb_pkts);
 uint16_t iavf_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
drivers/net/iavf/iavf_vchnl.c +41 -1

66 lines not on this page · raw message

Signature · 1 line
2.17.1
author
#32 lore raw

PATCH v3 4/11 net/iavf: flexible Rx descriptor support in AVX path

In reply to Leyi Rong

Support flexible Rx descriptor format in AVX
path of iAVF PMD.

---
 drivers/net/iavf/iavf_rxtx.c          |  24 +-
 drivers/net/iavf/iavf_rxtx.h          |   6 +
 drivers/net/iavf/iavf_rxtx_vec_avx2.c | 550 +++++++++++++++++++++++++-
 3 files changed, 570 insertions(+), 10 deletions(-)
drivers/net/iavf/iavf_rxtx.c +18 -6
index 67297dcb7..34c41d104 100644
@@ -2081,16 +2081,28 @@ iavf_set_rx_function(struct rte_eth_dev *dev)
 				    "Using %sVector Scattered Rx (port %d).",
 				    use_avx2 ? "avx2 " : "",
 				    dev->data->port_id);
-			dev->rx_pkt_burst = use_avx2 ?
-					    iavf_recv_scattered_pkts_vec_avx2 :
-					    iavf_recv_scattered_pkts_vec;
+			if (vf->vf_res->vf_cap_flags &
+				VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC)
+				dev->rx_pkt_burst = use_avx2 ?
+					iavf_recv_scattered_pkts_vec_avx2_flex_rxd :
+					iavf_recv_scattered_pkts_vec;
+			else
+				dev->rx_pkt_burst = use_avx2 ?
+					iavf_recv_scattered_pkts_vec_avx2 :
+					iavf_recv_scattered_pkts_vec;
 		} else {
 			PMD_DRV_LOG(DEBUG, "Using %sVector Rx (port %d).",
 				    use_avx2 ? "avx2 " : "",
 				    dev->data->port_id);
-			dev->rx_pkt_burst = use_avx2 ?
-					    iavf_recv_pkts_vec_avx2 :
-					    iavf_recv_pkts_vec;
+			if (vf->vf_res->vf_cap_flags &
+				VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC)
+				dev->rx_pkt_burst = use_avx2 ?
+					iavf_recv_pkts_vec_avx2_flex_rxd :
+					iavf_recv_pkts_vec;
+			else
+				dev->rx_pkt_burst = use_avx2 ?
+					iavf_recv_pkts_vec_avx2 :
+					iavf_recv_pkts_vec;
 		}
 
 		return;
drivers/net/iavf/iavf_rxtx.h +6 -0
index f33d1df41..8e1db2588 100644
@@ -413,9 +413,15 @@ uint16_t iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
 				  uint16_t nb_pkts);
 uint16_t iavf_recv_pkts_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,
 				 uint16_t nb_pkts);
+uint16_t iavf_recv_pkts_vec_avx2_flex_rxd(void *rx_queue,
+					  struct rte_mbuf **rx_pkts,
+					  uint16_t nb_pkts);
 uint16_t iavf_recv_scattered_pkts_vec_avx2(void *rx_queue,
 					   struct rte_mbuf **rx_pkts,
 					   uint16_t nb_pkts);
+uint16_t iavf_recv_scattered_pkts_vec_avx2_flex_rxd(void *rx_queue,
+						    struct rte_mbuf **rx_pkts,
+						    uint16_t nb_pkts);
 uint16_t iavf_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
 			    uint16_t nb_pkts);
 uint16_t iavf_xmit_pkts_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts,
drivers/net/iavf/iavf_rxtx_vec_avx2.c +546 -4

585 lines not on this page · raw message

Signature · 1 line
2.17.1
author
#33 lore raw

PATCH v3 5/11 net/iavf: flexible Rx descriptor support in SSE path

In reply to Leyi Rong

Support flexible Rx descriptor format in SSE
path of iAVF PMD.

---
 drivers/net/iavf/iavf_rxtx.c         |   4 +-
 drivers/net/iavf/iavf_rxtx.h         |   5 +
 drivers/net/iavf/iavf_rxtx_vec_sse.c | 414 +++++++++++++++++++++++++++
 3 files changed, 421 insertions(+), 2 deletions(-)
drivers/net/iavf/iavf_rxtx.c +2 -2
index 34c41d104..09ce5e3f3 100644
@@ -2085,7 +2085,7 @@ iavf_set_rx_function(struct rte_eth_dev *dev)
 				VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC)
 				dev->rx_pkt_burst = use_avx2 ?
 					iavf_recv_scattered_pkts_vec_avx2_flex_rxd :
-					iavf_recv_scattered_pkts_vec;
+					iavf_recv_scattered_pkts_vec_flex_rxd;
 			else
 				dev->rx_pkt_burst = use_avx2 ?
 					iavf_recv_scattered_pkts_vec_avx2 :
@@ -2098,7 +2098,7 @@ iavf_set_rx_function(struct rte_eth_dev *dev)
 				VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC)
 				dev->rx_pkt_burst = use_avx2 ?
 					iavf_recv_pkts_vec_avx2_flex_rxd :
-					iavf_recv_pkts_vec;
+					iavf_recv_pkts_vec_flex_rxd;
 			else
 				dev->rx_pkt_burst = use_avx2 ?
 					iavf_recv_pkts_vec_avx2 :
drivers/net/iavf/iavf_rxtx.h +5 -0
index 8e1db2588..290dd68c1 100644
@@ -406,9 +406,14 @@ int iavf_dev_tx_desc_status(void *tx_queue, uint16_t offset);
 
 uint16_t iavf_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
 			   uint16_t nb_pkts);
+uint16_t iavf_recv_pkts_vec_flex_rxd(void *rx_queue, struct rte_mbuf **rx_pkts,
+				     uint16_t nb_pkts);
 uint16_t iavf_recv_scattered_pkts_vec(void *rx_queue,
 				     struct rte_mbuf **rx_pkts,
 				     uint16_t nb_pkts);
+uint16_t iavf_recv_scattered_pkts_vec_flex_rxd(void *rx_queue,
+					       struct rte_mbuf **rx_pkts,
+					       uint16_t nb_pkts);
 uint16_t iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
 				  uint16_t nb_pkts);
 uint16_t iavf_recv_pkts_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,
drivers/net/iavf/iavf_rxtx_vec_sse.c +414 -0

444 lines not on this page · raw message

Signature · 1 line
2.17.1
author
#34 lore raw

PATCH v3 6/11 net/iavf: add flow director enabled switch value

In reply to Leyi Rong

The commit adds fdir_enabled flag into iavf_rx_queue structure to identify if fdir id is active. Rx data path can be benefit if fdir id parsing is not needed, especially in vector path.

---
 drivers/net/iavf/iavf.h      |  1 +
 drivers/net/iavf/iavf_rxtx.h | 29 +++++++++++++++++++++++++++++
 2 files changed, 30 insertions(+)
drivers/net/iavf/iavf.h +1 -0
index 67d625053..0cd0117c2 100644
@@ -134,6 +134,7 @@ struct iavf_adapter {
 	bool tx_vec_allowed;
 	const uint32_t *ptype_tbl;
 	bool stopped;
+	uint16_t fdir_ref_cnt;
 };
 
 /* IAVF_DEV_PRIVATE_TO */
drivers/net/iavf/iavf_rxtx.h +29 -0
index 290dd68c1..46ffbb738 100644
@@ -103,6 +103,7 @@ struct iavf_rx_queue {
 
 	uint16_t port_id;        /* device port ID */
 	uint8_t crc_len;        /* 0 if CRC stripped, 4 otherwise */
+	uint8_t fdir_enabled;   /* 0 if FDIR disabled, 1 when enabled */
 	uint16_t queue_id;      /* Rx queue index */
 	uint16_t rx_buf_len;    /* The packet buffer size */
 	uint16_t rx_hdr_len;    /* The header buffer size */
@@ -490,6 +491,34 @@ void iavf_dump_tx_descriptor(const struct iavf_tx_queue *txq,
 	       tx_desc->cmd_type_offset_bsz);
 }
 
+#define FDIR_PROC_ENABLE_PER_QUEUE(ad, on) do { \
+	for (int i = 0; i < ad->eth_dev->data->nb_rx_queues; i++) { \
+		struct iavf_rx_queue *rxq = ad->eth_dev->data->rx_queues[i]; \
+		if (!rxq) \
+			continue; \
+		rxq->fdir_enabled = on; \
+	} \
+	PMD_DRV_LOG(DEBUG, "FDIR processing on RX set to %d", on); \
+} while (0)
+
+/* Enable/disable flow director Rx processing in data path. */
+static inline
+void iavf_fdir_rx_proc_enable(struct iavf_adapter *ad, bool on)
+{
+	if (on) {
+		/* enable flow director processing */
+		if (ad->fdir_ref_cnt++ == 0)
+			FDIR_PROC_ENABLE_PER_QUEUE(ad, on);
+	} else {
+		if (ad->fdir_ref_cnt >= 1) {
+			ad->fdir_ref_cnt--;
+
+			if (ad->fdir_ref_cnt == 0)
+				FDIR_PROC_ENABLE_PER_QUEUE(ad, on);
+		}
+	}
+}
+
 #ifdef RTE_LIBRTE_IAVF_DEBUG_DUMP_DESC
 #define IAVF_DUMP_RX_DESC(rxq, desc, rx_id) \
 	iavf_dump_rx_descriptor(rxq, desc, rx_id)
Signature · 1 line
2.17.1
author
#35 lore raw

PATCH v3 7/11 net/iavf: support flow mark in normal data path

In reply to Leyi Rong

Support Flow Director mark ID parsing in normal path.

---
 drivers/net/iavf/iavf.h      |  3 +++
 drivers/net/iavf/iavf_rxtx.c | 37 ++++++++++++++++++++++++++++++++++++
 2 files changed, 40 insertions(+)
drivers/net/iavf/iavf.h +3 -0
index 0cd0117c2..b63efd4e8 100644
@@ -67,6 +67,9 @@
 #define IAVF_48_BIT_WIDTH (CHAR_BIT * 6)
 #define IAVF_48_BIT_MASK  RTE_LEN2MASK(IAVF_48_BIT_WIDTH, uint64_t)
 
+#define IAVF_RX_DESC_EXT_STATUS_FLEXBH_MASK  0x03
+#define IAVF_RX_DESC_EXT_STATUS_FLEXBH_FD_ID 0x01
+
 struct iavf_adapter;
 struct iavf_rx_queue;
 struct iavf_tx_queue;
drivers/net/iavf/iavf_rxtx.c +37 -0
index 09ce5e3f3..725dd9e45 100644
@@ -756,6 +756,10 @@ iavf_rxd_to_pkt_flags(uint64_t qword)
 					IAVF_RX_DESC_FLTSTAT_RSS_HASH) ==
 			IAVF_RX_DESC_FLTSTAT_RSS_HASH) ? PKT_RX_RSS_HASH : 0;
 
+	/* Check if FDIR Match */
+	flags |= (qword & (1 << IAVF_RX_DESC_STATUS_FLM_SHIFT) ?
+				PKT_RX_FDIR : 0);
+
 	if (likely((error_bits & IAVF_RX_ERR_BITS) == 0)) {
 		flags |= (PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD);
 		return flags;
@@ -776,6 +780,25 @@ iavf_rxd_to_pkt_flags(uint64_t qword)
 	return flags;
 }
 
+static inline uint64_t
+iavf_rxd_build_fdir(volatile union iavf_rx_desc *rxdp, struct rte_mbuf *mb)
+{
+	uint64_t flags = 0;
+	uint16_t flexbh;
+
+	flexbh = (rte_le_to_cpu_32(rxdp->wb.qword2.ext_status) >>
+		IAVF_RX_DESC_EXT_STATUS_FLEXBH_SHIFT) &
+		IAVF_RX_DESC_EXT_STATUS_FLEXBH_MASK;
+
+	if (flexbh == IAVF_RX_DESC_EXT_STATUS_FLEXBH_FD_ID) {
+		mb->hash.fdir.hi =
+			rte_le_to_cpu_32(rxdp->wb.qword3.hi_dword.fd_id);
+		flags |= PKT_RX_FDIR_ID;
+	}
+
+	return flags;
+}
+
 /* Translate the rx flex descriptor status to pkt flags */
 static inline void
 iavf_rxd_to_pkt_fields(struct rte_mbuf *mb,
@@ -792,6 +815,11 @@ iavf_rxd_to_pkt_fields(struct rte_mbuf *mb,
 		mb->hash.rss = rte_le_to_cpu_32(desc->rss_hash);
 	}
 #endif
+
+	if (desc->flow_id != 0xFFFFFFFF) {
+		mb->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID;
+		mb->hash.fdir.hi = rte_le_to_cpu_32(desc->flow_id);
+	}
 }
 
 #define IAVF_RX_FLEX_ERR0_BITS	\
@@ -951,6 +979,9 @@ iavf_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 			rxm->hash.rss =
 				rte_le_to_cpu_32(rxd.wb.qword0.hi_dword.rss);
 
+		if (pkt_flags & PKT_RX_FDIR)
+			pkt_flags |= iavf_rxd_build_fdir(&rxd, rxm);
+
 		rxm->ol_flags |= pkt_flags;
 
 		rx_pkts[nb_rx++] = rxm;
@@ -1349,6 +1380,9 @@ iavf_recv_scattered_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 			first_seg->hash.rss =
 				rte_le_to_cpu_32(rxd.wb.qword0.hi_dword.rss);
 
+		if (pkt_flags & PKT_RX_FDIR)
+			pkt_flags |= iavf_rxd_build_fdir(&rxd, first_seg);
+
 		first_seg->ol_flags |= pkt_flags;
 
 		/* Prefetch data of first segment, if configured to do so. */
@@ -1515,6 +1549,9 @@ iavf_rx_scan_hw_ring(struct iavf_rx_queue *rxq)
 				mb->hash.rss = rte_le_to_cpu_32(
 					rxdp[j].wb.qword0.hi_dword.rss);
 
+			if (pkt_flags & PKT_RX_FDIR)
+				pkt_flags |= iavf_rxd_build_fdir(&rxdp[j], mb);
+
 			mb->ol_flags |= pkt_flags;
 		}
 
Signature · 1 line
2.17.1
author
#36 lore raw

PATCH v3 8/11 net/iavf: support flow mark in AVX path

In reply to Leyi Rong

Support Flow Director mark ID parsing from Flex
Rx descriptor in AVX path.

---
 drivers/net/iavf/iavf_rxtx_vec_avx2.c | 72 +++++++++++++++++++++++++--
 1 file changed, 67 insertions(+), 5 deletions(-)
drivers/net/iavf/iavf_rxtx_vec_avx2.c +67 -5
index b23188fd3..3bf5833fa 100644
@@ -616,6 +616,25 @@ _iavf_recv_raw_pkts_vec_avx2(struct iavf_rx_queue *rxq,
 	return received;
 }
 
+static inline __m256i
+flex_rxd_to_fdir_flags_vec_avx2(const __m256i fdir_id0_7)
+{
+#define FDID_MIS_MAGIC 0xFFFFFFFF
+	RTE_BUILD_BUG_ON(PKT_RX_FDIR != (1 << 2));
+	RTE_BUILD_BUG_ON(PKT_RX_FDIR_ID != (1 << 13));
+	const __m256i pkt_fdir_bit = _mm256_set1_epi32(PKT_RX_FDIR |
+			PKT_RX_FDIR_ID);
+	/* desc->flow_id field == 0xFFFFFFFF means fdir mismatch */
+	const __m256i fdir_mis_mask = _mm256_set1_epi32(FDID_MIS_MAGIC);
+	__m256i fdir_mask = _mm256_cmpeq_epi32(fdir_id0_7,
+			fdir_mis_mask);
+	/* this XOR op results to bit-reverse the fdir_mask */
+	fdir_mask = _mm256_xor_si256(fdir_mask, fdir_mis_mask);
+	const __m256i fdir_flags = _mm256_and_si256(fdir_mask, pkt_fdir_bit);
+
+	return fdir_flags;
+}
+
 static inline uint16_t
 _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 				      struct rte_mbuf **rx_pkts,
@@ -678,8 +697,8 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 	const __m256i shuf_msk =
 		_mm256_set_epi8
 			(/* first descriptor */
-			 15, 14,
-			 13, 12,	/* octet 12~15, 32 bits rss */
+			 0xFF, 0xFF,
+			 0xFF, 0xFF,    /* rss not supported */
 			 11, 10,	/* octet 10~11, 16 bits vlan_macip */
 			 5, 4,		/* octet 4~5, 16 bits data_len */
 			 0xFF, 0xFF,	/* skip hi 16 bits pkt_len, zero out */
@@ -687,8 +706,8 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 			 0xFF, 0xFF,	/* pkt_type set as unknown */
 			 0xFF, 0xFF,	/*pkt_type set as unknown */
 			 /* second descriptor */
-			 15, 14,
-			 13, 12,	/* octet 12~15, 32 bits rss */
+			 0xFF, 0xFF,
+			 0xFF, 0xFF,    /* rss not supported */
 			 11, 10,	/* octet 10~11, 16 bits vlan_macip */
 			 5, 4,		/* octet 4~5, 16 bits data_len */
 			 0xFF, 0xFF,	/* skip hi 16 bits pkt_len, zero out */
@@ -930,8 +949,51 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 					    rss_vlan_flag_bits);
 
 		/* merge flags */
-		const __m256i mbuf_flags = _mm256_or_si256(l3_l4_flags,
+		__m256i mbuf_flags = _mm256_or_si256(l3_l4_flags,
 				rss_vlan_flags);
+
+		if (rxq->fdir_enabled) {
+			const __m256i fdir_id4_7 =
+				_mm256_unpackhi_epi32(raw_desc6_7, raw_desc4_5);
+
+			const __m256i fdir_id0_3 =
+				_mm256_unpackhi_epi32(raw_desc2_3, raw_desc0_1);
+
+			const __m256i fdir_id0_7 =
+				_mm256_unpackhi_epi64(fdir_id4_7, fdir_id0_3);
+
+			const __m256i fdir_flags =
+				flex_rxd_to_fdir_flags_vec_avx2(fdir_id0_7);
+
+			/* merge with fdir_flags */
+			mbuf_flags = _mm256_or_si256(mbuf_flags, fdir_flags);
+
+			/* write to mbuf: have to use scalar store here */
+			rx_pkts[i + 0]->hash.fdir.hi =
+				_mm256_extract_epi32(fdir_id0_7, 3);
+
+			rx_pkts[i + 1]->hash.fdir.hi =
+				_mm256_extract_epi32(fdir_id0_7, 7);
+
+			rx_pkts[i + 2]->hash.fdir.hi =
+				_mm256_extract_epi32(fdir_id0_7, 2);
+
+			rx_pkts[i + 3]->hash.fdir.hi =
+				_mm256_extract_epi32(fdir_id0_7, 6);
+
+			rx_pkts[i + 4]->hash.fdir.hi =
+				_mm256_extract_epi32(fdir_id0_7, 1);
+
+			rx_pkts[i + 5]->hash.fdir.hi =
+				_mm256_extract_epi32(fdir_id0_7, 5);
+
+			rx_pkts[i + 6]->hash.fdir.hi =
+				_mm256_extract_epi32(fdir_id0_7, 0);
+
+			rx_pkts[i + 7]->hash.fdir.hi =
+				_mm256_extract_epi32(fdir_id0_7, 4);
+		} /* if() on fdir_enabled */
+
 		/**
 		 * At this point, we have the 8 sets of flags in the low 16-bits
 		 * of each 32-bit value in vlan0.
Signature · 1 line
2.17.1
author
#37 lore raw

PATCH v3 9/11 net/iavf: support flow mark in SSE path

In reply to Leyi Rong

Support Flow Director mark ID parsing from Flex
Rx descriptor in SSE path.

---
 drivers/net/iavf/iavf_rxtx_vec_sse.c | 52 +++++++++++++++++++++++++++-
 1 file changed, 51 insertions(+), 1 deletion(-)
drivers/net/iavf/iavf_rxtx_vec_sse.c +51 -1
index 9c1f2a445..d7d840853 100644
@@ -189,6 +189,25 @@ desc_to_olflags_v(struct iavf_rx_queue *rxq, __m128i descs[4],
 	_mm_store_si128((__m128i *)&rx_pkts[3]->rearm_data, rearm3);
 }
 
+static inline __m128i
+flex_rxd_to_fdir_flags_vec(const __m128i fdir_id0_3)
+{
+#define FDID_MIS_MAGIC 0xFFFFFFFF
+	RTE_BUILD_BUG_ON(PKT_RX_FDIR != (1 << 2));
+	RTE_BUILD_BUG_ON(PKT_RX_FDIR_ID != (1 << 13));
+	const __m128i pkt_fdir_bit = _mm_set1_epi32(PKT_RX_FDIR |
+			PKT_RX_FDIR_ID);
+	/* desc->flow_id field == 0xFFFFFFFF means fdir mismatch */
+	const __m128i fdir_mis_mask = _mm_set1_epi32(FDID_MIS_MAGIC);
+	__m128i fdir_mask = _mm_cmpeq_epi32(fdir_id0_3,
+			fdir_mis_mask);
+	/* this XOR op results to bit-reverse the fdir_mask */
+	fdir_mask = _mm_xor_si128(fdir_mask, fdir_mis_mask);
+	const __m128i fdir_flags = _mm_and_si128(fdir_mask, pkt_fdir_bit);
+
+	return fdir_flags;
+}
+
 static inline void
 flex_desc_to_olflags_v(struct iavf_rx_queue *rxq, __m128i descs[4],
 		       struct rte_mbuf **rx_pkts)
@@ -267,6 +286,36 @@ flex_desc_to_olflags_v(struct iavf_rx_queue *rxq, __m128i descs[4],
 	/* merge the flags */
 	flags = _mm_or_si128(flags, rss_vlan);
 
+	if (rxq->fdir_enabled) {
+		const __m128i fdir_id0_1 =
+			_mm_unpackhi_epi32(descs[0], descs[1]);
+
+		const __m128i fdir_id2_3 =
+			_mm_unpackhi_epi32(descs[2], descs[3]);
+
+		const __m128i fdir_id0_3 =
+			_mm_unpackhi_epi64(fdir_id0_1, fdir_id2_3);
+
+		const __m128i fdir_flags =
+			flex_rxd_to_fdir_flags_vec(fdir_id0_3);
+
+		/* merge with fdir_flags */
+		flags = _mm_or_si128(flags, fdir_flags);
+
+		/* write fdir_id to mbuf */
+		rx_pkts[0]->hash.fdir.hi =
+			_mm_extract_epi32(fdir_id0_3, 0);
+
+		rx_pkts[1]->hash.fdir.hi =
+			_mm_extract_epi32(fdir_id0_3, 1);
+
+		rx_pkts[2]->hash.fdir.hi =
+			_mm_extract_epi32(fdir_id0_3, 2);
+
+		rx_pkts[3]->hash.fdir.hi =
+			_mm_extract_epi32(fdir_id0_3, 3);
+	} /* if() on fdir_enabled */
+
 	/**
 	 * At this point, we have the 4 sets of flags in the low 16-bits
 	 * of each 32-bit value in flags.
@@ -604,7 +653,8 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 	const __m128i zero = _mm_setzero_si128();
 	/* mask to shuffle from desc. to mbuf */
 	const __m128i shuf_msk = _mm_set_epi8
-			(15, 14, 13, 12,  /* octet 12~15, 32 bits rss */
+			(0xFF, 0xFF,
+			 0xFF, 0xFF,  /* rss not supported */
 			 11, 10,      /* octet 10~11, 16 bits vlan_macip */
 			 5, 4,        /* octet 4~5, 16 bits data_len */
 			 0xFF, 0xFF,  /* skip high 16 bits pkt_len, zero out */
Signature · 1 line
2.17.1
author
#38 lore raw

PATCH v3 10/11 net/iavf: add RSS hash parsing in AVX path

In reply to Leyi Rong

Support RSS hash parsing from Flex Rx
descriptor in AVX data path.

---
 drivers/net/iavf/iavf_rxtx_vec_avx2.c | 92 ++++++++++++++++++++++++++-
 1 file changed, 90 insertions(+), 2 deletions(-)
drivers/net/iavf/iavf_rxtx_vec_avx2.c +90 -2
index 3bf5833fa..22f1b7887 100644
@@ -698,7 +698,7 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 		_mm256_set_epi8
 			(/* first descriptor */
 			 0xFF, 0xFF,
-			 0xFF, 0xFF,    /* rss not supported */
+			 0xFF, 0xFF,    /* rss hash parsed separately */
 			 11, 10,	/* octet 10~11, 16 bits vlan_macip */
 			 5, 4,		/* octet 4~5, 16 bits data_len */
 			 0xFF, 0xFF,	/* skip hi 16 bits pkt_len, zero out */
@@ -707,7 +707,7 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 			 0xFF, 0xFF,	/*pkt_type set as unknown */
 			 /* second descriptor */
 			 0xFF, 0xFF,
-			 0xFF, 0xFF,    /* rss not supported */
+			 0xFF, 0xFF,    /* rss hash parsed separately */
 			 11, 10,	/* octet 10~11, 16 bits vlan_macip */
 			 5, 4,		/* octet 4~5, 16 bits data_len */
 			 0xFF, 0xFF,	/* skip hi 16 bits pkt_len, zero out */
@@ -994,6 +994,94 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 				_mm256_extract_epi32(fdir_id0_7, 4);
 		} /* if() on fdir_enabled */
 
+		/**
+		 * needs to load 2nd 16B of each desc for RSS hash parsing,
+		 * will cause performance drop to get into this context.
+		 */
+		if (rxq->vsi->adapter->eth_dev->data->dev_conf.rxmode.offloads &
+				DEV_RX_OFFLOAD_RSS_HASH) {
+			/* load bottom half of every 32B desc */
+			const __m128i raw_desc_bh7 =
+				_mm_load_si128
+					((void *)(&rxdp[7].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh6 =
+				_mm_load_si128
+					((void *)(&rxdp[6].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh5 =
+				_mm_load_si128
+					((void *)(&rxdp[5].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh4 =
+				_mm_load_si128
+					((void *)(&rxdp[4].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh3 =
+				_mm_load_si128
+					((void *)(&rxdp[3].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh2 =
+				_mm_load_si128
+					((void *)(&rxdp[2].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh1 =
+				_mm_load_si128
+					((void *)(&rxdp[1].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh0 =
+				_mm_load_si128
+					((void *)(&rxdp[0].wb.status_error1));
+
+			__m256i raw_desc_bh6_7 =
+				_mm256_inserti128_si256
+					(_mm256_castsi128_si256(raw_desc_bh6),
+					raw_desc_bh7, 1);
+			__m256i raw_desc_bh4_5 =
+				_mm256_inserti128_si256
+					(_mm256_castsi128_si256(raw_desc_bh4),
+					raw_desc_bh5, 1);
+			__m256i raw_desc_bh2_3 =
+				_mm256_inserti128_si256
+					(_mm256_castsi128_si256(raw_desc_bh2),
+					raw_desc_bh3, 1);
+			__m256i raw_desc_bh0_1 =
+				_mm256_inserti128_si256
+					(_mm256_castsi128_si256(raw_desc_bh0),
+					raw_desc_bh1, 1);
+
+			/**
+			 * to shift the 32b RSS hash value to the
+			 * highest 32b of each 128b before mask
+			 */
+			__m256i rss_hash6_7 =
+				_mm256_slli_epi64(raw_desc_bh6_7, 32);
+			__m256i rss_hash4_5 =
+				_mm256_slli_epi64(raw_desc_bh4_5, 32);
+			__m256i rss_hash2_3 =
+				_mm256_slli_epi64(raw_desc_bh2_3, 32);
+			__m256i rss_hash0_1 =
+				_mm256_slli_epi64(raw_desc_bh0_1, 32);
+
+			__m256i rss_hash_msk =
+				_mm256_set_epi32(0xFFFFFFFF, 0, 0, 0,
+						 0xFFFFFFFF, 0, 0, 0);
+
+			rss_hash6_7 = _mm256_and_si256
+					(rss_hash6_7, rss_hash_msk);
+			rss_hash4_5 = _mm256_and_si256
+					(rss_hash4_5, rss_hash_msk);
+			rss_hash2_3 = _mm256_and_si256
+					(rss_hash2_3, rss_hash_msk);
+			rss_hash0_1 = _mm256_and_si256
+					(rss_hash0_1, rss_hash_msk);
+
+			mb6_7 = _mm256_or_si256(mb6_7, rss_hash6_7);
+			mb4_5 = _mm256_or_si256(mb4_5, rss_hash4_5);
+			mb2_3 = _mm256_or_si256(mb2_3, rss_hash2_3);
+			mb0_1 = _mm256_or_si256(mb0_1, rss_hash0_1);
+		} /* if() on RSS hash parsing */
+
 		/**
 		 * At this point, we have the 8 sets of flags in the low 16-bits
 		 * of each 32-bit value in vlan0.
Signature · 1 line
2.17.1
author
#39 lore raw

PATCH v3 11/11 net/iavf: add RSS hash parsing in SSE path

In reply to Leyi Rong

Support RSS hash parsing from Flex Rx
descriptor in SSE data path.

---
 drivers/net/iavf/iavf_rxtx_vec_sse.c | 86 ++++++++++++++++++++++------
 1 file changed, 70 insertions(+), 16 deletions(-)
drivers/net/iavf/iavf_rxtx_vec_sse.c +70 -16
index d7d840853..85b50a7f1 100644
@@ -654,7 +654,7 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 	/* mask to shuffle from desc. to mbuf */
 	const __m128i shuf_msk = _mm_set_epi8
 			(0xFF, 0xFF,
-			 0xFF, 0xFF,  /* rss not supported */
+			 0xFF, 0xFF,  /* rss hash parsed separately */
 			 11, 10,      /* octet 10~11, 16 bits vlan_macip */
 			 5, 4,        /* octet 4~5, 16 bits data_len */
 			 0xFF, 0xFF,  /* skip high 16 bits pkt_len, zero out */
@@ -745,7 +745,7 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 	     pos += IAVF_VPMD_DESCS_PER_LOOP,
 	     rxdp += IAVF_VPMD_DESCS_PER_LOOP) {
 		__m128i descs[IAVF_VPMD_DESCS_PER_LOOP];
-		__m128i pkt_mb1, pkt_mb2, pkt_mb3, pkt_mb4;
+		__m128i pkt_mb0, pkt_mb1, pkt_mb2, pkt_mb3;
 		__m128i staterr, sterr_tmp1, sterr_tmp2;
 		/* 2 64 bit or 4 32 bit mbuf pointers in one XMM reg. */
 		__m128i mbp1;
@@ -791,8 +791,12 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 		rte_compiler_barrier();
 
 		/* D.1 pkt 3,4 convert format from desc to pktmbuf */
-		pkt_mb4 = _mm_shuffle_epi8(descs[3], shuf_msk);
-		pkt_mb3 = _mm_shuffle_epi8(descs[2], shuf_msk);
+		pkt_mb3 = _mm_shuffle_epi8(descs[3], shuf_msk);
+		pkt_mb2 = _mm_shuffle_epi8(descs[2], shuf_msk);
+
+		/* D.1 pkt 1,2 convert format from desc to pktmbuf */
+		pkt_mb1 = _mm_shuffle_epi8(descs[1], shuf_msk);
+		pkt_mb0 = _mm_shuffle_epi8(descs[0], shuf_msk);
 
 		/* C.1 4=>2 filter staterr info only */
 		sterr_tmp2 = _mm_unpackhi_epi32(descs[3], descs[2]);
@@ -802,12 +806,66 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 		flex_desc_to_olflags_v(rxq, descs, &rx_pkts[pos]);
 
 		/* D.2 pkt 3,4 set in_port/nb_seg and remove crc */
-		pkt_mb4 = _mm_add_epi16(pkt_mb4, crc_adjust);
 		pkt_mb3 = _mm_add_epi16(pkt_mb3, crc_adjust);
+		pkt_mb2 = _mm_add_epi16(pkt_mb2, crc_adjust);
 
-		/* D.1 pkt 1,2 convert format from desc to pktmbuf */
-		pkt_mb2 = _mm_shuffle_epi8(descs[1], shuf_msk);
-		pkt_mb1 = _mm_shuffle_epi8(descs[0], shuf_msk);
+		/* D.2 pkt 1,2 set in_port/nb_seg and remove crc */
+		pkt_mb1 = _mm_add_epi16(pkt_mb1, crc_adjust);
+		pkt_mb0 = _mm_add_epi16(pkt_mb0, crc_adjust);
+
+		/**
+		 * needs to load 2nd 16B of each desc for RSS hash parsing,
+		 * will cause performance drop to get into this context.
+		 */
+		if (rxq->vsi->adapter->eth_dev->data->dev_conf.rxmode.offloads &
+				DEV_RX_OFFLOAD_RSS_HASH) {
+			/* load bottom half of every 32B desc */
+			const __m128i raw_desc_bh3 =
+				_mm_load_si128
+					((void *)(&rxdp[3].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh2 =
+				_mm_load_si128
+					((void *)(&rxdp[2].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh1 =
+				_mm_load_si128
+					((void *)(&rxdp[1].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh0 =
+				_mm_load_si128
+					((void *)(&rxdp[0].wb.status_error1));
+
+			/**
+			 * to shift the 32b RSS hash value to the
+			 * highest 32b of each 128b before mask
+			 */
+			__m128i rss_hash3 =
+				_mm_slli_epi64(raw_desc_bh3, 32);
+			__m128i rss_hash2 =
+				_mm_slli_epi64(raw_desc_bh2, 32);
+			__m128i rss_hash1 =
+				_mm_slli_epi64(raw_desc_bh1, 32);
+			__m128i rss_hash0 =
+				_mm_slli_epi64(raw_desc_bh0, 32);
+
+			__m128i rss_hash_msk =
+				_mm_set_epi32(0xFFFFFFFF, 0, 0, 0);
+
+			rss_hash3 = _mm_and_si128
+					(rss_hash3, rss_hash_msk);
+			rss_hash2 = _mm_and_si128
+					(rss_hash2, rss_hash_msk);
+			rss_hash1 = _mm_and_si128
+					(rss_hash1, rss_hash_msk);
+			rss_hash0 = _mm_and_si128
+					(rss_hash0, rss_hash_msk);
+
+			pkt_mb3 = _mm_or_si128(pkt_mb3, rss_hash3);
+			pkt_mb2 = _mm_or_si128(pkt_mb2, rss_hash2);
+			pkt_mb1 = _mm_or_si128(pkt_mb1, rss_hash1);
+			pkt_mb0 = _mm_or_si128(pkt_mb0, rss_hash0);
+		} /* if() on RSS hash parsing */
 
 		/* C.2 get 4 pkts staterr value  */
 		staterr = _mm_unpacklo_epi32(sterr_tmp1, sterr_tmp2);
@@ -815,14 +873,10 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 		/* D.3 copy final 3,4 data to rx_pkts */
 		_mm_storeu_si128
 			((void *)&rx_pkts[pos + 3]->rx_descriptor_fields1,
-			 pkt_mb4);
+			 pkt_mb3);
 		_mm_storeu_si128
 			((void *)&rx_pkts[pos + 2]->rx_descriptor_fields1,
-			 pkt_mb3);
-
-		/* D.2 pkt 1,2 set in_port/nb_seg and remove crc */
-		pkt_mb2 = _mm_add_epi16(pkt_mb2, crc_adjust);
-		pkt_mb1 = _mm_add_epi16(pkt_mb1, crc_adjust);
+			 pkt_mb2);
 
 		/* C* extract and record EOP bit */
 		if (split_packet) {
@@ -846,9 +900,9 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 		/* D.3 copy final 1,2 data to rx_pkts */
 		_mm_storeu_si128
 			((void *)&rx_pkts[pos + 1]->rx_descriptor_fields1,
-			 pkt_mb2);
+			 pkt_mb1);
 		_mm_storeu_si128((void *)&rx_pkts[pos]->rx_descriptor_fields1,
-				 pkt_mb1);
+				 pkt_mb0);
 		flex_desc_to_ptype_v(descs, &rx_pkts[pos], ptype_tbl);
 		/* C.4 calc avaialbe number of desc */
 		var = __builtin_popcountll(_mm_cvtsi128_si64(staterr));
Signature · 1 line
2.17.1
author
#40 lore raw

v4 cover framework for advanced iAVF PMD

In reply to Leyi Rong

This patchset enable framework for advanced iAVF, includes flexible descriptor support, FDIR mark id and RSS hash support.

---
v4:
- Avoid initial declaration in 'for' loop to backward-compatible with older gcc versions.

v3:
- Remove patch to query DDP package info as it's specific to DCF.
- Specify fdir_ref_cnt as per VF value for enable/disable FDIR ID parse.
- Move fdir_enabled flag to per Queue value for cache benefit when access in vector routines.
- Store extracted flow_id value to mbuf without judgement to avoid branch speculation fail, which is benefit to performance.

v2:
- Revert macro RTE_LIBRTE_IAVF_16BYTE_RX_DESC deletion as it's defined in AVF spec.
- Typo RTE_LIBRTE_ICE_16BYTE_RX_DESC fix in iavf_rxtx.c.
- Move flex desc definitions into iavf_rxtx.h.
- Up to date to match with the latest version of virtchnl.h.
- Extract a new internal func iavf_update_rx_tail to call.
- Remove iavf_dev_rxq_count_flex_rxd()/iavf_dev_rx_desc_status_flex_rxd(), as the accompanying legacy ones can deal with the flex desc cases.
- Move rxq->rxdid assignment from iavf_configure_queues() to iavf_dev_rx_queue_setup().
- Unfold _mm_extract_epi32(fdir_id0_3, i) to fix build error when using GCC compile option -O0.

Leyi Rong (11):
net/iavf: flexible Rx descriptor definitions net/iavf: return error if opcode is mismatched net/iavf: flexible Rx descriptor support in normal path net/iavf: flexible Rx descriptor support in AVX path net/iavf: flexible Rx descriptor support in SSE path net/iavf: add flow director enabled switch value net/iavf: support flow mark in normal data path net/iavf: support flow mark in AVX path net/iavf: support flow mark in SSE path net/iavf: add RSS hash parsing in AVX path net/iavf: add RSS hash parsing in SSE path

 drivers/net/iavf/iavf.h               |   6 +
 drivers/net/iavf/iavf_ethdev.c        |   8 +
 drivers/net/iavf/iavf_rxtx.c          | 540 ++++++++++++++++++--
 drivers/net/iavf/iavf_rxtx.h          | 249 +++++++++
 drivers/net/iavf/iavf_rxtx_vec_avx2.c | 700 +++++++++++++++++++++++++-
 drivers/net/iavf/iavf_rxtx_vec_sse.c  | 518 +++++++++++++++++++
 drivers/net/iavf/iavf_vchnl.c         |  46 +-
 7 files changed, 2018 insertions(+), 49 deletions(-)
Signature · 1 line
2.17.1
author
#41 lore raw

PATCH v4 1/11 net/iavf: flexible Rx descriptor definitions

Add definitions for flexible Rx descriptor structures and macros.

---
 drivers/net/iavf/iavf_rxtx.h | 200 +++++++++++++++++++++++++++++++++++
 1 file changed, 200 insertions(+)
drivers/net/iavf/iavf_rxtx.h +200 -0
index 09b5bd99e..5e309631e 100644
@@ -157,6 +157,206 @@ union iavf_tx_offload {
 	};
 };
 
+/* Rx Flex Descriptors
+ * These descriptors are used instead of the legacy version descriptors
+ */
+union iavf_16b_rx_flex_desc {
+	struct {
+		__le64 pkt_addr; /* Packet buffer address */
+		__le64 hdr_addr; /* Header buffer address */
+				 /* bit 0 of hdr_addr is DD bit */
+	} read;
+	struct {
+		/* Qword 0 */
+		u8 rxdid; /* descriptor builder profile ID */
+		u8 mir_id_umb_cast; /* mirror=[5:0], umb=[7:6] */
+		__le16 ptype_flex_flags0; /* ptype=[9:0], ff0=[15:10] */
+		__le16 pkt_len; /* [15:14] are reserved */
+		__le16 hdr_len_sph_flex_flags1; /* header=[10:0] */
+						/* sph=[11:11] */
+						/* ff1/ext=[15:12] */
+
+		/* Qword 1 */
+		__le16 status_error0;
+		__le16 l2tag1;
+		__le16 flex_meta0;
+		__le16 flex_meta1;
+	} wb; /* writeback */
+};
+
+union iavf_32b_rx_flex_desc {
+	struct {
+		__le64 pkt_addr; /* Packet buffer address */
+		__le64 hdr_addr; /* Header buffer address */
+				 /* bit 0 of hdr_addr is DD bit */
+		__le64 rsvd1;
+		__le64 rsvd2;
+	} read;
+	struct {
+		/* Qword 0 */
+		u8 rxdid; /* descriptor builder profile ID */
+		u8 mir_id_umb_cast; /* mirror=[5:0], umb=[7:6] */
+		__le16 ptype_flex_flags0; /* ptype=[9:0], ff0=[15:10] */
+		__le16 pkt_len; /* [15:14] are reserved */
+		__le16 hdr_len_sph_flex_flags1; /* header=[10:0] */
+						/* sph=[11:11] */
+						/* ff1/ext=[15:12] */
+
+		/* Qword 1 */
+		__le16 status_error0;
+		__le16 l2tag1;
+		__le16 flex_meta0;
+		__le16 flex_meta1;
+
+		/* Qword 2 */
+		__le16 status_error1;
+		u8 flex_flags2;
+		u8 time_stamp_low;
+		__le16 l2tag2_1st;
+		__le16 l2tag2_2nd;
+
+		/* Qword 3 */
+		__le16 flex_meta2;
+		__le16 flex_meta3;
+		union {
+			struct {
+				__le16 flex_meta4;
+				__le16 flex_meta5;
+			} flex;
+			__le32 ts_high;
+		} flex_ts;
+	} wb; /* writeback */
+};
+
+/* Rx Flex Descriptor for Comms Package Profile
+ * RxDID Profile ID 16-21
+ * Flex-field 0: RSS hash lower 16-bits
+ * Flex-field 1: RSS hash upper 16-bits
+ * Flex-field 2: Flow ID lower 16-bits
+ * Flex-field 3: Flow ID upper 16-bits
+ * Flex-field 4: AUX0
+ * Flex-field 5: AUX1
+ */
+struct iavf_32b_rx_flex_desc_comms {
+	/* Qword 0 */
+	u8 rxdid;
+	u8 mir_id_umb_cast;
+	__le16 ptype_flexi_flags0;
+	__le16 pkt_len;
+	__le16 hdr_len_sph_flex_flags1;
+
+	/* Qword 1 */
+	__le16 status_error0;
+	__le16 l2tag1;
+	__le32 rss_hash;
+
+	/* Qword 2 */
+	__le16 status_error1;
+	u8 flexi_flags2;
+	u8 ts_low;
+	__le16 l2tag2_1st;
+	__le16 l2tag2_2nd;
+
+	/* Qword 3 */
+	__le32 flow_id;
+	union {
+		struct {
+			__le16 aux0;
+			__le16 aux1;
+		} flex;
+		__le32 ts_high;
+	} flex_ts;
+};
+
+/* Rx Flex Descriptor for Comms Package Profile
+ * RxDID Profile ID 22-23 (swap Hash and FlowID)
+ * Flex-field 0: Flow ID lower 16-bits
+ * Flex-field 1: Flow ID upper 16-bits
+ * Flex-field 2: RSS hash lower 16-bits
+ * Flex-field 3: RSS hash upper 16-bits
+ * Flex-field 4: AUX0
+ * Flex-field 5: AUX1
+ */
+struct iavf_32b_rx_flex_desc_comms_ovs {
+	/* Qword 0 */
+	u8 rxdid;
+	u8 mir_id_umb_cast;
+	__le16 ptype_flexi_flags0;
+	__le16 pkt_len;
+	__le16 hdr_len_sph_flex_flags1;
+
+	/* Qword 1 */
+	__le16 status_error0;
+	__le16 l2tag1;
+	__le32 flow_id;
+
+	/* Qword 2 */
+	__le16 status_error1;
+	u8 flexi_flags2;
+	u8 ts_low;
+	__le16 l2tag2_1st;
+	__le16 l2tag2_2nd;
+
+	/* Qword 3 */
+	__le32 rss_hash;
+	union {
+		struct {
+			__le16 aux0;
+			__le16 aux1;
+		} flex;
+		__le32 ts_high;
+	} flex_ts;
+};
+
+/* Receive Flex Descriptor profile IDs: There are a total
+ * of 64 profiles where profile IDs 0/1 are for legacy; and
+ * profiles 2-63 are flex profiles that can be programmed
+ * with a specific metadata (profile 7 reserved for HW)
+ */
+enum iavf_rxdid {
+	IAVF_RXDID_LEGACY_0		= 0,
+	IAVF_RXDID_LEGACY_1		= 1,
+	IAVF_RXDID_FLEX_NIC		= 2,
+	IAVF_RXDID_FLEX_NIC_2		= 6,
+	IAVF_RXDID_HW			= 7,
+	IAVF_RXDID_COMMS_GENERIC	= 16,
+	IAVF_RXDID_COMMS_AUX_VLAN	= 17,
+	IAVF_RXDID_COMMS_AUX_IPV4	= 18,
+	IAVF_RXDID_COMMS_AUX_IPV6	= 19,
+	IAVF_RXDID_COMMS_AUX_IPV6_FLOW	= 20,
+	IAVF_RXDID_COMMS_AUX_TCP	= 21,
+	IAVF_RXDID_COMMS_OVS_1		= 22,
+	IAVF_RXDID_COMMS_OVS_2		= 23,
+	IAVF_RXDID_LAST			= 63,
+};
+
+enum iavf_rx_flex_desc_status_error_0_bits {
+	/* Note: These are predefined bit offsets */
+	IAVF_RX_FLEX_DESC_STATUS0_DD_S = 0,
+	IAVF_RX_FLEX_DESC_STATUS0_EOF_S,
+	IAVF_RX_FLEX_DESC_STATUS0_HBO_S,
+	IAVF_RX_FLEX_DESC_STATUS0_L3L4P_S,
+	IAVF_RX_FLEX_DESC_STATUS0_XSUM_IPE_S,
+	IAVF_RX_FLEX_DESC_STATUS0_XSUM_L4E_S,
+	IAVF_RX_FLEX_DESC_STATUS0_XSUM_EIPE_S,
+	IAVF_RX_FLEX_DESC_STATUS0_XSUM_EUDPE_S,
+	IAVF_RX_FLEX_DESC_STATUS0_LPBK_S,
+	IAVF_RX_FLEX_DESC_STATUS0_IPV6EXADD_S,
+	IAVF_RX_FLEX_DESC_STATUS0_RXE_S,
+	IAVF_RX_FLEX_DESC_STATUS0_CRCP_S,
+	IAVF_RX_FLEX_DESC_STATUS0_RSS_VALID_S,
+	IAVF_RX_FLEX_DESC_STATUS0_L2TAG1P_S,
+	IAVF_RX_FLEX_DESC_STATUS0_XTRMD0_VALID_S,
+	IAVF_RX_FLEX_DESC_STATUS0_XTRMD1_VALID_S,
+	IAVF_RX_FLEX_DESC_STATUS0_LAST /* this entry must be last!!! */
+};
+
+/* for iavf_32b_rx_flex_desc.ptype_flex_flags0 member */
+#define IAVF_RX_FLEX_DESC_PTYPE_M	(0x3FF) /* 10-bits */
+
+/* for iavf_32b_rx_flex_desc.pkt_len member */
+#define IAVF_RX_FLX_DESC_PKT_LEN_M	(0x3FFF) /* 14-bits */
+
 int iavf_dev_rx_queue_setup(struct rte_eth_dev *dev,
 			   uint16_t queue_idx,
 			   uint16_t nb_desc,
Signature · 1 line
2.17.1
author
#42 lore raw

PATCH v4 2/11 net/iavf: return error if opcode is mismatched

In reply to Leyi Rong

Adds error return when the opcode of read message is mismatched which is received from adminQ.

---
 drivers/net/iavf/iavf_vchnl.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
drivers/net/iavf/iavf_vchnl.c +3 -1
index fa4da3a6d..b7fb05d32 100644
@@ -52,9 +52,11 @@ iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len,
 	PMD_DRV_LOG(DEBUG, "AQ from pf carries opcode %u, retval %d",
 		    opcode, vf->cmd_retval);
 
-	if (opcode != vf->pend_cmd)
+	if (opcode != vf->pend_cmd) {
 		PMD_DRV_LOG(WARNING, "command mismatch, expect %u, get %u",
 			    vf->pend_cmd, opcode);
+		return IAVF_ERR_OPCODE_MISMATCH;
+	}
 
 	return IAVF_SUCCESS;
 }
Signature · 1 line
2.17.1
author
#43 lore raw

PATCH v4 3/11 net/iavf: flexible Rx descriptor support in normal path

In reply to Leyi Rong

Support flexible Rx descriptor format in normal
path of iAVF PMD.

---
 drivers/net/iavf/iavf.h        |   2 +
 drivers/net/iavf/iavf_ethdev.c |   8 +
 drivers/net/iavf/iavf_rxtx.c   | 479 ++++++++++++++++++++++++++++++---
 drivers/net/iavf/iavf_rxtx.h   |   8 +
 drivers/net/iavf/iavf_vchnl.c  |  42 ++-
 5 files changed, 501 insertions(+), 38 deletions(-)
drivers/net/iavf/iavf.h +2 -0
index 368ddf604..fdb31b929 100644
@@ -103,6 +103,7 @@ struct iavf_info {
 	struct virtchnl_version_info virtchnl_version;
 	struct virtchnl_vf_resource *vf_res; /* VF resource */
 	struct virtchnl_vsi_resource *vsi_res; /* LAN VSI */
+	uint64_t supported_rxdid;
 
 	volatile enum virtchnl_ops pend_cmd; /* pending command not finished */
 	uint32_t cmd_retval; /* return value of the cmd response from PF */
@@ -235,6 +236,7 @@ int iavf_disable_queues(struct iavf_adapter *adapter);
 int iavf_configure_rss_lut(struct iavf_adapter *adapter);
 int iavf_configure_rss_key(struct iavf_adapter *adapter);
 int iavf_configure_queues(struct iavf_adapter *adapter);
+int iavf_get_supported_rxdid(struct iavf_adapter *adapter);
 int iavf_config_irq_map(struct iavf_adapter *adapter);
 void iavf_add_del_all_mac_addr(struct iavf_adapter *adapter, bool add);
 int iavf_dev_link_update(struct rte_eth_dev *dev,
drivers/net/iavf/iavf_ethdev.c +8 -0
index 9dadee3d6..4280a92bb 100644
@@ -1245,6 +1245,14 @@ iavf_init_vf(struct rte_eth_dev *dev)
 			goto err_rss;
 		}
 	}
+
+	if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) {
+		if (iavf_get_supported_rxdid(adapter) != 0) {
+			PMD_INIT_LOG(ERR, "failed to do get supported rxdid");
+			goto err_rss;
+		}
+	}
+
 	return 0;
 err_rss:
 	rte_free(vf->rss_key);
drivers/net/iavf/iavf_rxtx.c +442 -37

546 lines not on this page · raw message

drivers/net/iavf/iavf_rxtx.h +8 -0
index 5e309631e..f33d1df41 100644
@@ -62,6 +62,7 @@
 #define iavf_rx_desc iavf_16byte_rx_desc
 #else
 #define iavf_rx_desc iavf_32byte_rx_desc
+#define iavf_rx_flex_desc iavf_32b_rx_flex_desc
 #endif
 
 struct iavf_rxq_ops {
@@ -87,6 +88,7 @@ struct iavf_rx_queue {
 	struct rte_mbuf *pkt_first_seg; /* first segment of current packet */
 	struct rte_mbuf *pkt_last_seg;  /* last segment of current packet */
 	struct rte_mbuf fake_mbuf;      /* dummy mbuf */
+	uint8_t rxdid;
 
 	/* used for VPMD */
 	uint16_t rxrearm_nb;       /* number of remaining to be re-armed */
@@ -379,9 +381,15 @@ void iavf_dev_tx_queue_release(void *txq);
 void iavf_stop_queues(struct rte_eth_dev *dev);
 uint16_t iavf_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 		       uint16_t nb_pkts);
+uint16_t iavf_recv_pkts_flex_rxd(void *rx_queue,
+				 struct rte_mbuf **rx_pkts,
+				 uint16_t nb_pkts);
 uint16_t iavf_recv_scattered_pkts(void *rx_queue,
 				 struct rte_mbuf **rx_pkts,
 				 uint16_t nb_pkts);
+uint16_t iavf_recv_scattered_pkts_flex_rxd(void *rx_queue,
+					   struct rte_mbuf **rx_pkts,
+					   uint16_t nb_pkts);
 uint16_t iavf_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 		       uint16_t nb_pkts);
 uint16_t iavf_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
drivers/net/iavf/iavf_vchnl.c +41 -1

66 lines not on this page · raw message

Signature · 1 line
2.17.1
author
#44 lore raw

PATCH v4 4/11 net/iavf: flexible Rx descriptor support in AVX path

In reply to Leyi Rong

Support flexible Rx descriptor format in AVX
path of iAVF PMD.

---
 drivers/net/iavf/iavf_rxtx.c          |  24 +-
 drivers/net/iavf/iavf_rxtx.h          |   6 +
 drivers/net/iavf/iavf_rxtx_vec_avx2.c | 550 +++++++++++++++++++++++++-
 3 files changed, 570 insertions(+), 10 deletions(-)
drivers/net/iavf/iavf_rxtx.c +18 -6
index 67297dcb7..34c41d104 100644
@@ -2081,16 +2081,28 @@ iavf_set_rx_function(struct rte_eth_dev *dev)
 				    "Using %sVector Scattered Rx (port %d).",
 				    use_avx2 ? "avx2 " : "",
 				    dev->data->port_id);
-			dev->rx_pkt_burst = use_avx2 ?
-					    iavf_recv_scattered_pkts_vec_avx2 :
-					    iavf_recv_scattered_pkts_vec;
+			if (vf->vf_res->vf_cap_flags &
+				VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC)
+				dev->rx_pkt_burst = use_avx2 ?
+					iavf_recv_scattered_pkts_vec_avx2_flex_rxd :
+					iavf_recv_scattered_pkts_vec;
+			else
+				dev->rx_pkt_burst = use_avx2 ?
+					iavf_recv_scattered_pkts_vec_avx2 :
+					iavf_recv_scattered_pkts_vec;
 		} else {
 			PMD_DRV_LOG(DEBUG, "Using %sVector Rx (port %d).",
 				    use_avx2 ? "avx2 " : "",
 				    dev->data->port_id);
-			dev->rx_pkt_burst = use_avx2 ?
-					    iavf_recv_pkts_vec_avx2 :
-					    iavf_recv_pkts_vec;
+			if (vf->vf_res->vf_cap_flags &
+				VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC)
+				dev->rx_pkt_burst = use_avx2 ?
+					iavf_recv_pkts_vec_avx2_flex_rxd :
+					iavf_recv_pkts_vec;
+			else
+				dev->rx_pkt_burst = use_avx2 ?
+					iavf_recv_pkts_vec_avx2 :
+					iavf_recv_pkts_vec;
 		}
 
 		return;
drivers/net/iavf/iavf_rxtx.h +6 -0
index f33d1df41..8e1db2588 100644
@@ -413,9 +413,15 @@ uint16_t iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
 				  uint16_t nb_pkts);
 uint16_t iavf_recv_pkts_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,
 				 uint16_t nb_pkts);
+uint16_t iavf_recv_pkts_vec_avx2_flex_rxd(void *rx_queue,
+					  struct rte_mbuf **rx_pkts,
+					  uint16_t nb_pkts);
 uint16_t iavf_recv_scattered_pkts_vec_avx2(void *rx_queue,
 					   struct rte_mbuf **rx_pkts,
 					   uint16_t nb_pkts);
+uint16_t iavf_recv_scattered_pkts_vec_avx2_flex_rxd(void *rx_queue,
+						    struct rte_mbuf **rx_pkts,
+						    uint16_t nb_pkts);
 uint16_t iavf_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
 			    uint16_t nb_pkts);
 uint16_t iavf_xmit_pkts_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts,
drivers/net/iavf/iavf_rxtx_vec_avx2.c +546 -4

585 lines not on this page · raw message

Signature · 1 line
2.17.1
author
#45 lore raw

PATCH v4 5/11 net/iavf: flexible Rx descriptor support in SSE path

In reply to Leyi Rong

Support flexible Rx descriptor format in SSE
path of iAVF PMD.

---
 drivers/net/iavf/iavf_rxtx.c         |   4 +-
 drivers/net/iavf/iavf_rxtx.h         |   5 +
 drivers/net/iavf/iavf_rxtx_vec_sse.c | 414 +++++++++++++++++++++++++++
 3 files changed, 421 insertions(+), 2 deletions(-)
drivers/net/iavf/iavf_rxtx.c +2 -2
index 34c41d104..09ce5e3f3 100644
@@ -2085,7 +2085,7 @@ iavf_set_rx_function(struct rte_eth_dev *dev)
 				VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC)
 				dev->rx_pkt_burst = use_avx2 ?
 					iavf_recv_scattered_pkts_vec_avx2_flex_rxd :
-					iavf_recv_scattered_pkts_vec;
+					iavf_recv_scattered_pkts_vec_flex_rxd;
 			else
 				dev->rx_pkt_burst = use_avx2 ?
 					iavf_recv_scattered_pkts_vec_avx2 :
@@ -2098,7 +2098,7 @@ iavf_set_rx_function(struct rte_eth_dev *dev)
 				VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC)
 				dev->rx_pkt_burst = use_avx2 ?
 					iavf_recv_pkts_vec_avx2_flex_rxd :
-					iavf_recv_pkts_vec;
+					iavf_recv_pkts_vec_flex_rxd;
 			else
 				dev->rx_pkt_burst = use_avx2 ?
 					iavf_recv_pkts_vec_avx2 :
drivers/net/iavf/iavf_rxtx.h +5 -0
index 8e1db2588..290dd68c1 100644
@@ -406,9 +406,14 @@ int iavf_dev_tx_desc_status(void *tx_queue, uint16_t offset);
 
 uint16_t iavf_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
 			   uint16_t nb_pkts);
+uint16_t iavf_recv_pkts_vec_flex_rxd(void *rx_queue, struct rte_mbuf **rx_pkts,
+				     uint16_t nb_pkts);
 uint16_t iavf_recv_scattered_pkts_vec(void *rx_queue,
 				     struct rte_mbuf **rx_pkts,
 				     uint16_t nb_pkts);
+uint16_t iavf_recv_scattered_pkts_vec_flex_rxd(void *rx_queue,
+					       struct rte_mbuf **rx_pkts,
+					       uint16_t nb_pkts);
 uint16_t iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
 				  uint16_t nb_pkts);
 uint16_t iavf_recv_pkts_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,
drivers/net/iavf/iavf_rxtx_vec_sse.c +414 -0

444 lines not on this page · raw message

Signature · 1 line
2.17.1
author
#46 lore raw

PATCH v4 6/11 net/iavf: add flow director enabled switch value

In reply to Leyi Rong

The commit adds fdir_enabled flag into iavf_rx_queue structure to identify if fdir id is active. Rx data path can be benefit if fdir id parsing is not needed, especially in vector path.

---
 drivers/net/iavf/iavf.h      |  1 +
 drivers/net/iavf/iavf_rxtx.h | 30 ++++++++++++++++++++++++++++++
 2 files changed, 31 insertions(+)
drivers/net/iavf/iavf.h +1 -0
index fdb31b929..17ceff734 100644
@@ -144,6 +144,7 @@ struct iavf_adapter {
 	bool tx_vec_allowed;
 	const uint32_t *ptype_tbl;
 	bool stopped;
+	uint16_t fdir_ref_cnt;
 };
 
 /* IAVF_DEV_PRIVATE_TO */
drivers/net/iavf/iavf_rxtx.h +30 -0
index 290dd68c1..dd5544ee2 100644
@@ -103,6 +103,7 @@ struct iavf_rx_queue {
 
 	uint16_t port_id;        /* device port ID */
 	uint8_t crc_len;        /* 0 if CRC stripped, 4 otherwise */
+	uint8_t fdir_enabled;   /* 0 if FDIR disabled, 1 when enabled */
 	uint16_t queue_id;      /* Rx queue index */
 	uint16_t rx_buf_len;    /* The packet buffer size */
 	uint16_t rx_hdr_len;    /* The header buffer size */
@@ -490,6 +491,35 @@ void iavf_dump_tx_descriptor(const struct iavf_tx_queue *txq,
 	       tx_desc->cmd_type_offset_bsz);
 }
 
+#define FDIR_PROC_ENABLE_PER_QUEUE(ad, on) do { \
+	int i; \
+	for (i = 0; i < (ad)->eth_dev->data->nb_rx_queues; i++) { \
+		struct iavf_rx_queue *rxq = (ad)->eth_dev->data->rx_queues[i]; \
+		if (!rxq) \
+			continue; \
+		rxq->fdir_enabled = on; \
+	} \
+	PMD_DRV_LOG(DEBUG, "FDIR processing on RX set to %d", on); \
+} while (0)
+
+/* Enable/disable flow director Rx processing in data path. */
+static inline
+void iavf_fdir_rx_proc_enable(struct iavf_adapter *ad, bool on)
+{
+	if (on) {
+		/* enable flow director processing */
+		if (ad->fdir_ref_cnt++ == 0)
+			FDIR_PROC_ENABLE_PER_QUEUE(ad, on);
+	} else {
+		if (ad->fdir_ref_cnt >= 1) {
+			ad->fdir_ref_cnt--;
+
+			if (ad->fdir_ref_cnt == 0)
+				FDIR_PROC_ENABLE_PER_QUEUE(ad, on);
+		}
+	}
+}
+
 #ifdef RTE_LIBRTE_IAVF_DEBUG_DUMP_DESC
 #define IAVF_DUMP_RX_DESC(rxq, desc, rx_id) \
 	iavf_dump_rx_descriptor(rxq, desc, rx_id)
Signature · 1 line
2.17.1
author
#47 lore raw

PATCH v4 7/11 net/iavf: support flow mark in normal data path

In reply to Leyi Rong

Support Flow Director mark ID parsing in normal path.

---
 drivers/net/iavf/iavf.h      |  3 +++
 drivers/net/iavf/iavf_rxtx.c | 37 ++++++++++++++++++++++++++++++++++++
 2 files changed, 40 insertions(+)
drivers/net/iavf/iavf.h +3 -0
index 17ceff734..78bdaff20 100644
@@ -67,6 +67,9 @@
 #define IAVF_48_BIT_WIDTH (CHAR_BIT * 6)
 #define IAVF_48_BIT_MASK  RTE_LEN2MASK(IAVF_48_BIT_WIDTH, uint64_t)
 
+#define IAVF_RX_DESC_EXT_STATUS_FLEXBH_MASK  0x03
+#define IAVF_RX_DESC_EXT_STATUS_FLEXBH_FD_ID 0x01
+
 struct iavf_adapter;
 struct iavf_rx_queue;
 struct iavf_tx_queue;
drivers/net/iavf/iavf_rxtx.c +37 -0
index 09ce5e3f3..725dd9e45 100644
@@ -756,6 +756,10 @@ iavf_rxd_to_pkt_flags(uint64_t qword)
 					IAVF_RX_DESC_FLTSTAT_RSS_HASH) ==
 			IAVF_RX_DESC_FLTSTAT_RSS_HASH) ? PKT_RX_RSS_HASH : 0;
 
+	/* Check if FDIR Match */
+	flags |= (qword & (1 << IAVF_RX_DESC_STATUS_FLM_SHIFT) ?
+				PKT_RX_FDIR : 0);
+
 	if (likely((error_bits & IAVF_RX_ERR_BITS) == 0)) {
 		flags |= (PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD);
 		return flags;
@@ -776,6 +780,25 @@ iavf_rxd_to_pkt_flags(uint64_t qword)
 	return flags;
 }
 
+static inline uint64_t
+iavf_rxd_build_fdir(volatile union iavf_rx_desc *rxdp, struct rte_mbuf *mb)
+{
+	uint64_t flags = 0;
+	uint16_t flexbh;
+
+	flexbh = (rte_le_to_cpu_32(rxdp->wb.qword2.ext_status) >>
+		IAVF_RX_DESC_EXT_STATUS_FLEXBH_SHIFT) &
+		IAVF_RX_DESC_EXT_STATUS_FLEXBH_MASK;
+
+	if (flexbh == IAVF_RX_DESC_EXT_STATUS_FLEXBH_FD_ID) {
+		mb->hash.fdir.hi =
+			rte_le_to_cpu_32(rxdp->wb.qword3.hi_dword.fd_id);
+		flags |= PKT_RX_FDIR_ID;
+	}
+
+	return flags;
+}
+
 /* Translate the rx flex descriptor status to pkt flags */
 static inline void
 iavf_rxd_to_pkt_fields(struct rte_mbuf *mb,
@@ -792,6 +815,11 @@ iavf_rxd_to_pkt_fields(struct rte_mbuf *mb,
 		mb->hash.rss = rte_le_to_cpu_32(desc->rss_hash);
 	}
 #endif
+
+	if (desc->flow_id != 0xFFFFFFFF) {
+		mb->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID;
+		mb->hash.fdir.hi = rte_le_to_cpu_32(desc->flow_id);
+	}
 }
 
 #define IAVF_RX_FLEX_ERR0_BITS	\
@@ -951,6 +979,9 @@ iavf_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 			rxm->hash.rss =
 				rte_le_to_cpu_32(rxd.wb.qword0.hi_dword.rss);
 
+		if (pkt_flags & PKT_RX_FDIR)
+			pkt_flags |= iavf_rxd_build_fdir(&rxd, rxm);
+
 		rxm->ol_flags |= pkt_flags;
 
 		rx_pkts[nb_rx++] = rxm;
@@ -1349,6 +1380,9 @@ iavf_recv_scattered_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 			first_seg->hash.rss =
 				rte_le_to_cpu_32(rxd.wb.qword0.hi_dword.rss);
 
+		if (pkt_flags & PKT_RX_FDIR)
+			pkt_flags |= iavf_rxd_build_fdir(&rxd, first_seg);
+
 		first_seg->ol_flags |= pkt_flags;
 
 		/* Prefetch data of first segment, if configured to do so. */
@@ -1515,6 +1549,9 @@ iavf_rx_scan_hw_ring(struct iavf_rx_queue *rxq)
 				mb->hash.rss = rte_le_to_cpu_32(
 					rxdp[j].wb.qword0.hi_dword.rss);
 
+			if (pkt_flags & PKT_RX_FDIR)
+				pkt_flags |= iavf_rxd_build_fdir(&rxdp[j], mb);
+
 			mb->ol_flags |= pkt_flags;
 		}
 
Signature · 1 line
2.17.1
author
#48 lore raw

PATCH v4 8/11 net/iavf: support flow mark in AVX path

In reply to Leyi Rong

Support Flow Director mark ID parsing from Flex
Rx descriptor in AVX path.

---
 drivers/net/iavf/iavf_rxtx_vec_avx2.c | 72 +++++++++++++++++++++++++--
 1 file changed, 67 insertions(+), 5 deletions(-)
drivers/net/iavf/iavf_rxtx_vec_avx2.c +67 -5
index b23188fd3..3bf5833fa 100644
@@ -616,6 +616,25 @@ _iavf_recv_raw_pkts_vec_avx2(struct iavf_rx_queue *rxq,
 	return received;
 }
 
+static inline __m256i
+flex_rxd_to_fdir_flags_vec_avx2(const __m256i fdir_id0_7)
+{
+#define FDID_MIS_MAGIC 0xFFFFFFFF
+	RTE_BUILD_BUG_ON(PKT_RX_FDIR != (1 << 2));
+	RTE_BUILD_BUG_ON(PKT_RX_FDIR_ID != (1 << 13));
+	const __m256i pkt_fdir_bit = _mm256_set1_epi32(PKT_RX_FDIR |
+			PKT_RX_FDIR_ID);
+	/* desc->flow_id field == 0xFFFFFFFF means fdir mismatch */
+	const __m256i fdir_mis_mask = _mm256_set1_epi32(FDID_MIS_MAGIC);
+	__m256i fdir_mask = _mm256_cmpeq_epi32(fdir_id0_7,
+			fdir_mis_mask);
+	/* this XOR op results to bit-reverse the fdir_mask */
+	fdir_mask = _mm256_xor_si256(fdir_mask, fdir_mis_mask);
+	const __m256i fdir_flags = _mm256_and_si256(fdir_mask, pkt_fdir_bit);
+
+	return fdir_flags;
+}
+
 static inline uint16_t
 _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 				      struct rte_mbuf **rx_pkts,
@@ -678,8 +697,8 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 	const __m256i shuf_msk =
 		_mm256_set_epi8
 			(/* first descriptor */
-			 15, 14,
-			 13, 12,	/* octet 12~15, 32 bits rss */
+			 0xFF, 0xFF,
+			 0xFF, 0xFF,    /* rss not supported */
 			 11, 10,	/* octet 10~11, 16 bits vlan_macip */
 			 5, 4,		/* octet 4~5, 16 bits data_len */
 			 0xFF, 0xFF,	/* skip hi 16 bits pkt_len, zero out */
@@ -687,8 +706,8 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 			 0xFF, 0xFF,	/* pkt_type set as unknown */
 			 0xFF, 0xFF,	/*pkt_type set as unknown */
 			 /* second descriptor */
-			 15, 14,
-			 13, 12,	/* octet 12~15, 32 bits rss */
+			 0xFF, 0xFF,
+			 0xFF, 0xFF,    /* rss not supported */
 			 11, 10,	/* octet 10~11, 16 bits vlan_macip */
 			 5, 4,		/* octet 4~5, 16 bits data_len */
 			 0xFF, 0xFF,	/* skip hi 16 bits pkt_len, zero out */
@@ -930,8 +949,51 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 					    rss_vlan_flag_bits);
 
 		/* merge flags */
-		const __m256i mbuf_flags = _mm256_or_si256(l3_l4_flags,
+		__m256i mbuf_flags = _mm256_or_si256(l3_l4_flags,
 				rss_vlan_flags);
+
+		if (rxq->fdir_enabled) {
+			const __m256i fdir_id4_7 =
+				_mm256_unpackhi_epi32(raw_desc6_7, raw_desc4_5);
+
+			const __m256i fdir_id0_3 =
+				_mm256_unpackhi_epi32(raw_desc2_3, raw_desc0_1);
+
+			const __m256i fdir_id0_7 =
+				_mm256_unpackhi_epi64(fdir_id4_7, fdir_id0_3);
+
+			const __m256i fdir_flags =
+				flex_rxd_to_fdir_flags_vec_avx2(fdir_id0_7);
+
+			/* merge with fdir_flags */
+			mbuf_flags = _mm256_or_si256(mbuf_flags, fdir_flags);
+
+			/* write to mbuf: have to use scalar store here */
+			rx_pkts[i + 0]->hash.fdir.hi =
+				_mm256_extract_epi32(fdir_id0_7, 3);
+
+			rx_pkts[i + 1]->hash.fdir.hi =
+				_mm256_extract_epi32(fdir_id0_7, 7);
+
+			rx_pkts[i + 2]->hash.fdir.hi =
+				_mm256_extract_epi32(fdir_id0_7, 2);
+
+			rx_pkts[i + 3]->hash.fdir.hi =
+				_mm256_extract_epi32(fdir_id0_7, 6);
+
+			rx_pkts[i + 4]->hash.fdir.hi =
+				_mm256_extract_epi32(fdir_id0_7, 1);
+
+			rx_pkts[i + 5]->hash.fdir.hi =
+				_mm256_extract_epi32(fdir_id0_7, 5);
+
+			rx_pkts[i + 6]->hash.fdir.hi =
+				_mm256_extract_epi32(fdir_id0_7, 0);
+
+			rx_pkts[i + 7]->hash.fdir.hi =
+				_mm256_extract_epi32(fdir_id0_7, 4);
+		} /* if() on fdir_enabled */
+
 		/**
 		 * At this point, we have the 8 sets of flags in the low 16-bits
 		 * of each 32-bit value in vlan0.
Signature · 1 line
2.17.1
author
#49 lore raw

PATCH v4 9/11 net/iavf: support flow mark in SSE path

In reply to Leyi Rong

Support Flow Director mark ID parsing from Flex
Rx descriptor in SSE path.

---
 drivers/net/iavf/iavf_rxtx_vec_sse.c | 52 +++++++++++++++++++++++++++-
 1 file changed, 51 insertions(+), 1 deletion(-)
drivers/net/iavf/iavf_rxtx_vec_sse.c +51 -1
index 9c1f2a445..d7d840853 100644
@@ -189,6 +189,25 @@ desc_to_olflags_v(struct iavf_rx_queue *rxq, __m128i descs[4],
 	_mm_store_si128((__m128i *)&rx_pkts[3]->rearm_data, rearm3);
 }
 
+static inline __m128i
+flex_rxd_to_fdir_flags_vec(const __m128i fdir_id0_3)
+{
+#define FDID_MIS_MAGIC 0xFFFFFFFF
+	RTE_BUILD_BUG_ON(PKT_RX_FDIR != (1 << 2));
+	RTE_BUILD_BUG_ON(PKT_RX_FDIR_ID != (1 << 13));
+	const __m128i pkt_fdir_bit = _mm_set1_epi32(PKT_RX_FDIR |
+			PKT_RX_FDIR_ID);
+	/* desc->flow_id field == 0xFFFFFFFF means fdir mismatch */
+	const __m128i fdir_mis_mask = _mm_set1_epi32(FDID_MIS_MAGIC);
+	__m128i fdir_mask = _mm_cmpeq_epi32(fdir_id0_3,
+			fdir_mis_mask);
+	/* this XOR op results to bit-reverse the fdir_mask */
+	fdir_mask = _mm_xor_si128(fdir_mask, fdir_mis_mask);
+	const __m128i fdir_flags = _mm_and_si128(fdir_mask, pkt_fdir_bit);
+
+	return fdir_flags;
+}
+
 static inline void
 flex_desc_to_olflags_v(struct iavf_rx_queue *rxq, __m128i descs[4],
 		       struct rte_mbuf **rx_pkts)
@@ -267,6 +286,36 @@ flex_desc_to_olflags_v(struct iavf_rx_queue *rxq, __m128i descs[4],
 	/* merge the flags */
 	flags = _mm_or_si128(flags, rss_vlan);
 
+	if (rxq->fdir_enabled) {
+		const __m128i fdir_id0_1 =
+			_mm_unpackhi_epi32(descs[0], descs[1]);
+
+		const __m128i fdir_id2_3 =
+			_mm_unpackhi_epi32(descs[2], descs[3]);
+
+		const __m128i fdir_id0_3 =
+			_mm_unpackhi_epi64(fdir_id0_1, fdir_id2_3);
+
+		const __m128i fdir_flags =
+			flex_rxd_to_fdir_flags_vec(fdir_id0_3);
+
+		/* merge with fdir_flags */
+		flags = _mm_or_si128(flags, fdir_flags);
+
+		/* write fdir_id to mbuf */
+		rx_pkts[0]->hash.fdir.hi =
+			_mm_extract_epi32(fdir_id0_3, 0);
+
+		rx_pkts[1]->hash.fdir.hi =
+			_mm_extract_epi32(fdir_id0_3, 1);
+
+		rx_pkts[2]->hash.fdir.hi =
+			_mm_extract_epi32(fdir_id0_3, 2);
+
+		rx_pkts[3]->hash.fdir.hi =
+			_mm_extract_epi32(fdir_id0_3, 3);
+	} /* if() on fdir_enabled */
+
 	/**
 	 * At this point, we have the 4 sets of flags in the low 16-bits
 	 * of each 32-bit value in flags.
@@ -604,7 +653,8 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 	const __m128i zero = _mm_setzero_si128();
 	/* mask to shuffle from desc. to mbuf */
 	const __m128i shuf_msk = _mm_set_epi8
-			(15, 14, 13, 12,  /* octet 12~15, 32 bits rss */
+			(0xFF, 0xFF,
+			 0xFF, 0xFF,  /* rss not supported */
 			 11, 10,      /* octet 10~11, 16 bits vlan_macip */
 			 5, 4,        /* octet 4~5, 16 bits data_len */
 			 0xFF, 0xFF,  /* skip high 16 bits pkt_len, zero out */
Signature · 1 line
2.17.1
author
#50 lore raw

PATCH v4 10/11 net/iavf: add RSS hash parsing in AVX path

In reply to Leyi Rong

Support RSS hash parsing from Flex Rx
descriptor in AVX data path.

---
 drivers/net/iavf/iavf_rxtx_vec_avx2.c | 92 ++++++++++++++++++++++++++-
 1 file changed, 90 insertions(+), 2 deletions(-)
drivers/net/iavf/iavf_rxtx_vec_avx2.c +90 -2
index 3bf5833fa..22f1b7887 100644
@@ -698,7 +698,7 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 		_mm256_set_epi8
 			(/* first descriptor */
 			 0xFF, 0xFF,
-			 0xFF, 0xFF,    /* rss not supported */
+			 0xFF, 0xFF,    /* rss hash parsed separately */
 			 11, 10,	/* octet 10~11, 16 bits vlan_macip */
 			 5, 4,		/* octet 4~5, 16 bits data_len */
 			 0xFF, 0xFF,	/* skip hi 16 bits pkt_len, zero out */
@@ -707,7 +707,7 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 			 0xFF, 0xFF,	/*pkt_type set as unknown */
 			 /* second descriptor */
 			 0xFF, 0xFF,
-			 0xFF, 0xFF,    /* rss not supported */
+			 0xFF, 0xFF,    /* rss hash parsed separately */
 			 11, 10,	/* octet 10~11, 16 bits vlan_macip */
 			 5, 4,		/* octet 4~5, 16 bits data_len */
 			 0xFF, 0xFF,	/* skip hi 16 bits pkt_len, zero out */
@@ -994,6 +994,94 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 				_mm256_extract_epi32(fdir_id0_7, 4);
 		} /* if() on fdir_enabled */
 
+		/**
+		 * needs to load 2nd 16B of each desc for RSS hash parsing,
+		 * will cause performance drop to get into this context.
+		 */
+		if (rxq->vsi->adapter->eth_dev->data->dev_conf.rxmode.offloads &
+				DEV_RX_OFFLOAD_RSS_HASH) {
+			/* load bottom half of every 32B desc */
+			const __m128i raw_desc_bh7 =
+				_mm_load_si128
+					((void *)(&rxdp[7].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh6 =
+				_mm_load_si128
+					((void *)(&rxdp[6].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh5 =
+				_mm_load_si128
+					((void *)(&rxdp[5].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh4 =
+				_mm_load_si128
+					((void *)(&rxdp[4].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh3 =
+				_mm_load_si128
+					((void *)(&rxdp[3].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh2 =
+				_mm_load_si128
+					((void *)(&rxdp[2].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh1 =
+				_mm_load_si128
+					((void *)(&rxdp[1].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh0 =
+				_mm_load_si128
+					((void *)(&rxdp[0].wb.status_error1));
+
+			__m256i raw_desc_bh6_7 =
+				_mm256_inserti128_si256
+					(_mm256_castsi128_si256(raw_desc_bh6),
+					raw_desc_bh7, 1);
+			__m256i raw_desc_bh4_5 =
+				_mm256_inserti128_si256
+					(_mm256_castsi128_si256(raw_desc_bh4),
+					raw_desc_bh5, 1);
+			__m256i raw_desc_bh2_3 =
+				_mm256_inserti128_si256
+					(_mm256_castsi128_si256(raw_desc_bh2),
+					raw_desc_bh3, 1);
+			__m256i raw_desc_bh0_1 =
+				_mm256_inserti128_si256
+					(_mm256_castsi128_si256(raw_desc_bh0),
+					raw_desc_bh1, 1);
+
+			/**
+			 * to shift the 32b RSS hash value to the
+			 * highest 32b of each 128b before mask
+			 */
+			__m256i rss_hash6_7 =
+				_mm256_slli_epi64(raw_desc_bh6_7, 32);
+			__m256i rss_hash4_5 =
+				_mm256_slli_epi64(raw_desc_bh4_5, 32);
+			__m256i rss_hash2_3 =
+				_mm256_slli_epi64(raw_desc_bh2_3, 32);
+			__m256i rss_hash0_1 =
+				_mm256_slli_epi64(raw_desc_bh0_1, 32);
+
+			__m256i rss_hash_msk =
+				_mm256_set_epi32(0xFFFFFFFF, 0, 0, 0,
+						 0xFFFFFFFF, 0, 0, 0);
+
+			rss_hash6_7 = _mm256_and_si256
+					(rss_hash6_7, rss_hash_msk);
+			rss_hash4_5 = _mm256_and_si256
+					(rss_hash4_5, rss_hash_msk);
+			rss_hash2_3 = _mm256_and_si256
+					(rss_hash2_3, rss_hash_msk);
+			rss_hash0_1 = _mm256_and_si256
+					(rss_hash0_1, rss_hash_msk);
+
+			mb6_7 = _mm256_or_si256(mb6_7, rss_hash6_7);
+			mb4_5 = _mm256_or_si256(mb4_5, rss_hash4_5);
+			mb2_3 = _mm256_or_si256(mb2_3, rss_hash2_3);
+			mb0_1 = _mm256_or_si256(mb0_1, rss_hash0_1);
+		} /* if() on RSS hash parsing */
+
 		/**
 		 * At this point, we have the 8 sets of flags in the low 16-bits
 		 * of each 32-bit value in vlan0.
Signature · 1 line
2.17.1
author
#51 lore raw

PATCH v4 11/11 net/iavf: add RSS hash parsing in SSE path

In reply to Leyi Rong

Support RSS hash parsing from Flex Rx
descriptor in SSE data path.

---
 drivers/net/iavf/iavf_rxtx_vec_sse.c | 86 ++++++++++++++++++++++------
 1 file changed, 70 insertions(+), 16 deletions(-)
drivers/net/iavf/iavf_rxtx_vec_sse.c +70 -16
index d7d840853..85b50a7f1 100644
@@ -654,7 +654,7 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 	/* mask to shuffle from desc. to mbuf */
 	const __m128i shuf_msk = _mm_set_epi8
 			(0xFF, 0xFF,
-			 0xFF, 0xFF,  /* rss not supported */
+			 0xFF, 0xFF,  /* rss hash parsed separately */
 			 11, 10,      /* octet 10~11, 16 bits vlan_macip */
 			 5, 4,        /* octet 4~5, 16 bits data_len */
 			 0xFF, 0xFF,  /* skip high 16 bits pkt_len, zero out */
@@ -745,7 +745,7 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 	     pos += IAVF_VPMD_DESCS_PER_LOOP,
 	     rxdp += IAVF_VPMD_DESCS_PER_LOOP) {
 		__m128i descs[IAVF_VPMD_DESCS_PER_LOOP];
-		__m128i pkt_mb1, pkt_mb2, pkt_mb3, pkt_mb4;
+		__m128i pkt_mb0, pkt_mb1, pkt_mb2, pkt_mb3;
 		__m128i staterr, sterr_tmp1, sterr_tmp2;
 		/* 2 64 bit or 4 32 bit mbuf pointers in one XMM reg. */
 		__m128i mbp1;
@@ -791,8 +791,12 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 		rte_compiler_barrier();
 
 		/* D.1 pkt 3,4 convert format from desc to pktmbuf */
-		pkt_mb4 = _mm_shuffle_epi8(descs[3], shuf_msk);
-		pkt_mb3 = _mm_shuffle_epi8(descs[2], shuf_msk);
+		pkt_mb3 = _mm_shuffle_epi8(descs[3], shuf_msk);
+		pkt_mb2 = _mm_shuffle_epi8(descs[2], shuf_msk);
+
+		/* D.1 pkt 1,2 convert format from desc to pktmbuf */
+		pkt_mb1 = _mm_shuffle_epi8(descs[1], shuf_msk);
+		pkt_mb0 = _mm_shuffle_epi8(descs[0], shuf_msk);
 
 		/* C.1 4=>2 filter staterr info only */
 		sterr_tmp2 = _mm_unpackhi_epi32(descs[3], descs[2]);
@@ -802,12 +806,66 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 		flex_desc_to_olflags_v(rxq, descs, &rx_pkts[pos]);
 
 		/* D.2 pkt 3,4 set in_port/nb_seg and remove crc */
-		pkt_mb4 = _mm_add_epi16(pkt_mb4, crc_adjust);
 		pkt_mb3 = _mm_add_epi16(pkt_mb3, crc_adjust);
+		pkt_mb2 = _mm_add_epi16(pkt_mb2, crc_adjust);
 
-		/* D.1 pkt 1,2 convert format from desc to pktmbuf */
-		pkt_mb2 = _mm_shuffle_epi8(descs[1], shuf_msk);
-		pkt_mb1 = _mm_shuffle_epi8(descs[0], shuf_msk);
+		/* D.2 pkt 1,2 set in_port/nb_seg and remove crc */
+		pkt_mb1 = _mm_add_epi16(pkt_mb1, crc_adjust);
+		pkt_mb0 = _mm_add_epi16(pkt_mb0, crc_adjust);
+
+		/**
+		 * needs to load 2nd 16B of each desc for RSS hash parsing,
+		 * will cause performance drop to get into this context.
+		 */
+		if (rxq->vsi->adapter->eth_dev->data->dev_conf.rxmode.offloads &
+				DEV_RX_OFFLOAD_RSS_HASH) {
+			/* load bottom half of every 32B desc */
+			const __m128i raw_desc_bh3 =
+				_mm_load_si128
+					((void *)(&rxdp[3].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh2 =
+				_mm_load_si128
+					((void *)(&rxdp[2].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh1 =
+				_mm_load_si128
+					((void *)(&rxdp[1].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh0 =
+				_mm_load_si128
+					((void *)(&rxdp[0].wb.status_error1));
+
+			/**
+			 * to shift the 32b RSS hash value to the
+			 * highest 32b of each 128b before mask
+			 */
+			__m128i rss_hash3 =
+				_mm_slli_epi64(raw_desc_bh3, 32);
+			__m128i rss_hash2 =
+				_mm_slli_epi64(raw_desc_bh2, 32);
+			__m128i rss_hash1 =
+				_mm_slli_epi64(raw_desc_bh1, 32);
+			__m128i rss_hash0 =
+				_mm_slli_epi64(raw_desc_bh0, 32);
+
+			__m128i rss_hash_msk =
+				_mm_set_epi32(0xFFFFFFFF, 0, 0, 0);
+
+			rss_hash3 = _mm_and_si128
+					(rss_hash3, rss_hash_msk);
+			rss_hash2 = _mm_and_si128
+					(rss_hash2, rss_hash_msk);
+			rss_hash1 = _mm_and_si128
+					(rss_hash1, rss_hash_msk);
+			rss_hash0 = _mm_and_si128
+					(rss_hash0, rss_hash_msk);
+
+			pkt_mb3 = _mm_or_si128(pkt_mb3, rss_hash3);
+			pkt_mb2 = _mm_or_si128(pkt_mb2, rss_hash2);
+			pkt_mb1 = _mm_or_si128(pkt_mb1, rss_hash1);
+			pkt_mb0 = _mm_or_si128(pkt_mb0, rss_hash0);
+		} /* if() on RSS hash parsing */
 
 		/* C.2 get 4 pkts staterr value  */
 		staterr = _mm_unpacklo_epi32(sterr_tmp1, sterr_tmp2);
@@ -815,14 +873,10 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 		/* D.3 copy final 3,4 data to rx_pkts */
 		_mm_storeu_si128
 			((void *)&rx_pkts[pos + 3]->rx_descriptor_fields1,
-			 pkt_mb4);
+			 pkt_mb3);
 		_mm_storeu_si128
 			((void *)&rx_pkts[pos + 2]->rx_descriptor_fields1,
-			 pkt_mb3);
-
-		/* D.2 pkt 1,2 set in_port/nb_seg and remove crc */
-		pkt_mb2 = _mm_add_epi16(pkt_mb2, crc_adjust);
-		pkt_mb1 = _mm_add_epi16(pkt_mb1, crc_adjust);
+			 pkt_mb2);
 
 		/* C* extract and record EOP bit */
 		if (split_packet) {
@@ -846,9 +900,9 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 		/* D.3 copy final 1,2 data to rx_pkts */
 		_mm_storeu_si128
 			((void *)&rx_pkts[pos + 1]->rx_descriptor_fields1,
-			 pkt_mb2);
+			 pkt_mb1);
 		_mm_storeu_si128((void *)&rx_pkts[pos]->rx_descriptor_fields1,
-				 pkt_mb1);
+				 pkt_mb0);
 		flex_desc_to_ptype_v(descs, &rx_pkts[pos], ptype_tbl);
 		/* C.4 calc avaialbe number of desc */
 		var = __builtin_popcountll(_mm_cvtsi128_si64(staterr));
Signature · 1 line
2.17.1
author
#52 lore raw

v5 cover framework for advanced iAVF PMD

In reply to Leyi Rong

This patchset enable framework for advanced iAVF, includes flexible descriptor support, FDIR mark id and RSS hash support.

---
v5:
- Set rxdid = 0 in iavf_configure_queues() when it's supported and RTE_LIBRTE_IAVF_16BYTE_RX_DESC is y. So the error can be captured explicitly as kernel PF does not support it yet.
- Fix some build errors when RTE_LIBRTE_IAVF_16BYTE_RX_DESC is y.

v4:
- Avoid initial declaration in 'for' loop to backward-compatible with older gcc versions.

v3:
- Remove patch to query DDP package info as it's specific to DCF.
- Specify fdir_ref_cnt as per VF value for enable/disable FDIR ID parse.
- Move fdir_enabled flag to per Queue value for cache benefit when access in vector routines.
- Store extracted flow_id value to mbuf without judgement to avoid branch speculation fail, which is benefit to performance.

v2:
- Revert macro RTE_LIBRTE_IAVF_16BYTE_RX_DESC deletion as it's defined in AVF spec.
- Typo RTE_LIBRTE_ICE_16BYTE_RX_DESC fix in iavf_rxtx.c.
- Move flex desc definitions into iavf_rxtx.h.
- Up to date to match with the latest version of virtchnl.h.
- Extract a new internal func iavf_update_rx_tail to call.
- Remove iavf_dev_rxq_count_flex_rxd()/iavf_dev_rx_desc_status_flex_rxd(), as the accompanying legacy ones can deal with the flex desc cases.
- Move rxq->rxdid assignment from iavf_configure_queues() to iavf_dev_rx_queue_setup().
- Unfold _mm_extract_epi32(fdir_id0_3, i) to fix build error when using GCC compile option -O0.

Leyi Rong (11):
net/iavf: flexible Rx descriptor definitions net/iavf: return error if opcode is mismatched net/iavf: flexible Rx descriptor support in normal path net/iavf: flexible Rx descriptor support in AVX path net/iavf: flexible Rx descriptor support in SSE path net/iavf: add flow director enabled switch value net/iavf: support flow mark in normal data path net/iavf: support flow mark in AVX path net/iavf: support flow mark in SSE path net/iavf: add RSS hash parsing in AVX path net/iavf: add RSS hash parsing in SSE path

 drivers/net/iavf/iavf.h               |   6 +
 drivers/net/iavf/iavf_ethdev.c        |   8 +
 drivers/net/iavf/iavf_rxtx.c          | 546 ++++++++++++++++++--
 drivers/net/iavf/iavf_rxtx.h          | 250 +++++++++
 drivers/net/iavf/iavf_rxtx_vec_avx2.c | 702 +++++++++++++++++++++++++-
 drivers/net/iavf/iavf_rxtx_vec_sse.c  | 520 +++++++++++++++++++
 drivers/net/iavf/iavf_vchnl.c         |  59 ++-
 7 files changed, 2042 insertions(+), 49 deletions(-)
Signature · 1 line
2.17.1
author
#53 lore raw

PATCH v5 1/11 net/iavf: flexible Rx descriptor definitions

Add definitions for flexible Rx descriptor structures and macros.

---
 drivers/net/iavf/iavf_rxtx.h | 200 +++++++++++++++++++++++++++++++++++
 1 file changed, 200 insertions(+)
drivers/net/iavf/iavf_rxtx.h +200 -0
index 09b5bd99e..d04233bf8 100644
@@ -157,6 +157,206 @@ union iavf_tx_offload {
 	};
 };
 
+/* Rx Flex Descriptors
+ * These descriptors are used instead of the legacy version descriptors
+ */
+union iavf_16b_rx_flex_desc {
+	struct {
+		__le64 pkt_addr; /* Packet buffer address */
+		__le64 hdr_addr; /* Header buffer address */
+				 /* bit 0 of hdr_addr is DD bit */
+	} read;
+	struct {
+		/* Qword 0 */
+		u8 rxdid; /* descriptor builder profile ID */
+		u8 mir_id_umb_cast; /* mirror=[5:0], umb=[7:6] */
+		__le16 ptype_flex_flags0; /* ptype=[9:0], ff0=[15:10] */
+		__le16 pkt_len; /* [15:14] are reserved */
+		__le16 hdr_len_sph_flex_flags1; /* header=[10:0] */
+						/* sph=[11:11] */
+						/* ff1/ext=[15:12] */
+
+		/* Qword 1 */
+		__le16 status_error0;
+		__le16 l2tag1;
+		__le16 flex_meta0;
+		__le16 flex_meta1;
+	} wb; /* writeback */
+};
+
+union iavf_32b_rx_flex_desc {
+	struct {
+		__le64 pkt_addr; /* Packet buffer address */
+		__le64 hdr_addr; /* Header buffer address */
+				 /* bit 0 of hdr_addr is DD bit */
+		__le64 rsvd1;
+		__le64 rsvd2;
+	} read;
+	struct {
+		/* Qword 0 */
+		u8 rxdid; /* descriptor builder profile ID */
+		u8 mir_id_umb_cast; /* mirror=[5:0], umb=[7:6] */
+		__le16 ptype_flex_flags0; /* ptype=[9:0], ff0=[15:10] */
+		__le16 pkt_len; /* [15:14] are reserved */
+		__le16 hdr_len_sph_flex_flags1; /* header=[10:0] */
+						/* sph=[11:11] */
+						/* ff1/ext=[15:12] */
+
+		/* Qword 1 */
+		__le16 status_error0;
+		__le16 l2tag1;
+		__le16 flex_meta0;
+		__le16 flex_meta1;
+
+		/* Qword 2 */
+		__le16 status_error1;
+		u8 flex_flags2;
+		u8 time_stamp_low;
+		__le16 l2tag2_1st;
+		__le16 l2tag2_2nd;
+
+		/* Qword 3 */
+		__le16 flex_meta2;
+		__le16 flex_meta3;
+		union {
+			struct {
+				__le16 flex_meta4;
+				__le16 flex_meta5;
+			} flex;
+			__le32 ts_high;
+		} flex_ts;
+	} wb; /* writeback */
+};
+
+/* Rx Flex Descriptor
+ * RxDID Profile ID 16-21
+ * Flex-field 0: RSS hash lower 16-bits
+ * Flex-field 1: RSS hash upper 16-bits
+ * Flex-field 2: Flow ID lower 16-bits
+ * Flex-field 3: Flow ID upper 16-bits
+ * Flex-field 4: AUX0
+ * Flex-field 5: AUX1
+ */
+struct iavf_32b_rx_flex_desc_comms {
+	/* Qword 0 */
+	u8 rxdid;
+	u8 mir_id_umb_cast;
+	__le16 ptype_flexi_flags0;
+	__le16 pkt_len;
+	__le16 hdr_len_sph_flex_flags1;
+
+	/* Qword 1 */
+	__le16 status_error0;
+	__le16 l2tag1;
+	__le32 rss_hash;
+
+	/* Qword 2 */
+	__le16 status_error1;
+	u8 flexi_flags2;
+	u8 ts_low;
+	__le16 l2tag2_1st;
+	__le16 l2tag2_2nd;
+
+	/* Qword 3 */
+	__le32 flow_id;
+	union {
+		struct {
+			__le16 aux0;
+			__le16 aux1;
+		} flex;
+		__le32 ts_high;
+	} flex_ts;
+};
+
+/* Rx Flex Descriptor
+ * RxDID Profile ID 22-23 (swap Hash and FlowID)
+ * Flex-field 0: Flow ID lower 16-bits
+ * Flex-field 1: Flow ID upper 16-bits
+ * Flex-field 2: RSS hash lower 16-bits
+ * Flex-field 3: RSS hash upper 16-bits
+ * Flex-field 4: AUX0
+ * Flex-field 5: AUX1
+ */
+struct iavf_32b_rx_flex_desc_comms_ovs {
+	/* Qword 0 */
+	u8 rxdid;
+	u8 mir_id_umb_cast;
+	__le16 ptype_flexi_flags0;
+	__le16 pkt_len;
+	__le16 hdr_len_sph_flex_flags1;
+
+	/* Qword 1 */
+	__le16 status_error0;
+	__le16 l2tag1;
+	__le32 flow_id;
+
+	/* Qword 2 */
+	__le16 status_error1;
+	u8 flexi_flags2;
+	u8 ts_low;
+	__le16 l2tag2_1st;
+	__le16 l2tag2_2nd;
+
+	/* Qword 3 */
+	__le32 rss_hash;
+	union {
+		struct {
+			__le16 aux0;
+			__le16 aux1;
+		} flex;
+		__le32 ts_high;
+	} flex_ts;
+};
+
+/* Receive Flex Descriptor profile IDs: There are a total
+ * of 64 profiles where profile IDs 0/1 are for legacy; and
+ * profiles 2-63 are flex profiles that can be programmed
+ * with a specific metadata (profile 7 reserved for HW)
+ */
+enum iavf_rxdid {
+	IAVF_RXDID_LEGACY_0		= 0,
+	IAVF_RXDID_LEGACY_1		= 1,
+	IAVF_RXDID_FLEX_NIC		= 2,
+	IAVF_RXDID_FLEX_NIC_2		= 6,
+	IAVF_RXDID_HW			= 7,
+	IAVF_RXDID_COMMS_GENERIC	= 16,
+	IAVF_RXDID_COMMS_AUX_VLAN	= 17,
+	IAVF_RXDID_COMMS_AUX_IPV4	= 18,
+	IAVF_RXDID_COMMS_AUX_IPV6	= 19,
+	IAVF_RXDID_COMMS_AUX_IPV6_FLOW	= 20,
+	IAVF_RXDID_COMMS_AUX_TCP	= 21,
+	IAVF_RXDID_COMMS_OVS_1		= 22,
+	IAVF_RXDID_COMMS_OVS_2		= 23,
+	IAVF_RXDID_LAST			= 63,
+};
+
+enum iavf_rx_flex_desc_status_error_0_bits {
+	/* Note: These are predefined bit offsets */
+	IAVF_RX_FLEX_DESC_STATUS0_DD_S = 0,
+	IAVF_RX_FLEX_DESC_STATUS0_EOF_S,
+	IAVF_RX_FLEX_DESC_STATUS0_HBO_S,
+	IAVF_RX_FLEX_DESC_STATUS0_L3L4P_S,
+	IAVF_RX_FLEX_DESC_STATUS0_XSUM_IPE_S,
+	IAVF_RX_FLEX_DESC_STATUS0_XSUM_L4E_S,
+	IAVF_RX_FLEX_DESC_STATUS0_XSUM_EIPE_S,
+	IAVF_RX_FLEX_DESC_STATUS0_XSUM_EUDPE_S,
+	IAVF_RX_FLEX_DESC_STATUS0_LPBK_S,
+	IAVF_RX_FLEX_DESC_STATUS0_IPV6EXADD_S,
+	IAVF_RX_FLEX_DESC_STATUS0_RXE_S,
+	IAVF_RX_FLEX_DESC_STATUS0_CRCP_S,
+	IAVF_RX_FLEX_DESC_STATUS0_RSS_VALID_S,
+	IAVF_RX_FLEX_DESC_STATUS0_L2TAG1P_S,
+	IAVF_RX_FLEX_DESC_STATUS0_XTRMD0_VALID_S,
+	IAVF_RX_FLEX_DESC_STATUS0_XTRMD1_VALID_S,
+	IAVF_RX_FLEX_DESC_STATUS0_LAST /* this entry must be last!!! */
+};
+
+/* for iavf_32b_rx_flex_desc.ptype_flex_flags0 member */
+#define IAVF_RX_FLEX_DESC_PTYPE_M	(0x3FF) /* 10-bits */
+
+/* for iavf_32b_rx_flex_desc.pkt_len member */
+#define IAVF_RX_FLX_DESC_PKT_LEN_M	(0x3FFF) /* 14-bits */
+
 int iavf_dev_rx_queue_setup(struct rte_eth_dev *dev,
 			   uint16_t queue_idx,
 			   uint16_t nb_desc,
Signature · 1 line
2.17.1
author
#54 lore raw

PATCH v5 2/11 net/iavf: return error if opcode is mismatched

In reply to Leyi Rong

Adds error return when the opcode of read message is mismatched which is received from adminQ.

---
 drivers/net/iavf/iavf_vchnl.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
drivers/net/iavf/iavf_vchnl.c +3 -1
index fa4da3a6d..b7fb05d32 100644
@@ -52,9 +52,11 @@ iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len,
 	PMD_DRV_LOG(DEBUG, "AQ from pf carries opcode %u, retval %d",
 		    opcode, vf->cmd_retval);
 
-	if (opcode != vf->pend_cmd)
+	if (opcode != vf->pend_cmd) {
 		PMD_DRV_LOG(WARNING, "command mismatch, expect %u, get %u",
 			    vf->pend_cmd, opcode);
+		return IAVF_ERR_OPCODE_MISMATCH;
+	}
 
 	return IAVF_SUCCESS;
 }
Signature · 1 line
2.17.1
author
#55 lore raw

PATCH v5 3/11 net/iavf: flexible Rx descriptor support in normal path

In reply to Leyi Rong

Support flexible Rx descriptor format in normal
path of iAVF PMD.

---
 drivers/net/iavf/iavf.h        |   2 +
 drivers/net/iavf/iavf_ethdev.c |   8 +
 drivers/net/iavf/iavf_rxtx.c   | 485 ++++++++++++++++++++++++++++++---
 drivers/net/iavf/iavf_rxtx.h   |   9 +
 drivers/net/iavf/iavf_vchnl.c  |  55 +++-
 5 files changed, 521 insertions(+), 38 deletions(-)
drivers/net/iavf/iavf.h +2 -0
index 368ddf604..fdb31b929 100644
@@ -103,6 +103,7 @@ struct iavf_info {
 	struct virtchnl_version_info virtchnl_version;
 	struct virtchnl_vf_resource *vf_res; /* VF resource */
 	struct virtchnl_vsi_resource *vsi_res; /* LAN VSI */
+	uint64_t supported_rxdid;
 
 	volatile enum virtchnl_ops pend_cmd; /* pending command not finished */
 	uint32_t cmd_retval; /* return value of the cmd response from PF */
@@ -235,6 +236,7 @@ int iavf_disable_queues(struct iavf_adapter *adapter);
 int iavf_configure_rss_lut(struct iavf_adapter *adapter);
 int iavf_configure_rss_key(struct iavf_adapter *adapter);
 int iavf_configure_queues(struct iavf_adapter *adapter);
+int iavf_get_supported_rxdid(struct iavf_adapter *adapter);
 int iavf_config_irq_map(struct iavf_adapter *adapter);
 void iavf_add_del_all_mac_addr(struct iavf_adapter *adapter, bool add);
 int iavf_dev_link_update(struct rte_eth_dev *dev,
drivers/net/iavf/iavf_ethdev.c +8 -0
index 9dadee3d6..4280a92bb 100644
@@ -1245,6 +1245,14 @@ iavf_init_vf(struct rte_eth_dev *dev)
 			goto err_rss;
 		}
 	}
+
+	if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) {
+		if (iavf_get_supported_rxdid(adapter) != 0) {
+			PMD_INIT_LOG(ERR, "failed to do get supported rxdid");
+			goto err_rss;
+		}
+	}
+
 	return 0;
 err_rss:
 	rte_free(vf->rss_key);
drivers/net/iavf/iavf_rxtx.c +448 -37

552 lines not on this page · raw message

drivers/net/iavf/iavf_rxtx.h +9 -0
index d04233bf8..8246e797f 100644
@@ -60,8 +60,10 @@
 /* HW desc structure, both 16-byte and 32-byte types are supported */
 #ifdef RTE_LIBRTE_IAVF_16BYTE_RX_DESC
 #define iavf_rx_desc iavf_16byte_rx_desc
+#define iavf_rx_flex_desc iavf_16b_rx_flex_desc
 #else
 #define iavf_rx_desc iavf_32byte_rx_desc
+#define iavf_rx_flex_desc iavf_32b_rx_flex_desc
 #endif
 
 struct iavf_rxq_ops {
@@ -87,6 +89,7 @@ struct iavf_rx_queue {
 	struct rte_mbuf *pkt_first_seg; /* first segment of current packet */
 	struct rte_mbuf *pkt_last_seg;  /* last segment of current packet */
 	struct rte_mbuf fake_mbuf;      /* dummy mbuf */
+	uint8_t rxdid;
 
 	/* used for VPMD */
 	uint16_t rxrearm_nb;       /* number of remaining to be re-armed */
@@ -379,9 +382,15 @@ void iavf_dev_tx_queue_release(void *txq);
 void iavf_stop_queues(struct rte_eth_dev *dev);
 uint16_t iavf_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 		       uint16_t nb_pkts);
+uint16_t iavf_recv_pkts_flex_rxd(void *rx_queue,
+				 struct rte_mbuf **rx_pkts,
+				 uint16_t nb_pkts);
 uint16_t iavf_recv_scattered_pkts(void *rx_queue,
 				 struct rte_mbuf **rx_pkts,
 				 uint16_t nb_pkts);
+uint16_t iavf_recv_scattered_pkts_flex_rxd(void *rx_queue,
+					   struct rte_mbuf **rx_pkts,
+					   uint16_t nb_pkts);
 uint16_t iavf_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 		       uint16_t nb_pkts);
 uint16_t iavf_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
drivers/net/iavf/iavf_vchnl.c +54 -1

79 lines not on this page · raw message

Signature · 1 line
2.17.1
author
#56 lore raw

PATCH v5 4/11 net/iavf: flexible Rx descriptor support in AVX path

In reply to Leyi Rong

Support flexible Rx descriptor format in AVX
path of iAVF PMD.

---
 drivers/net/iavf/iavf_rxtx.c          |  24 +-
 drivers/net/iavf/iavf_rxtx.h          |   6 +
 drivers/net/iavf/iavf_rxtx_vec_avx2.c | 550 +++++++++++++++++++++++++-
 3 files changed, 570 insertions(+), 10 deletions(-)
drivers/net/iavf/iavf_rxtx.c +18 -6
index 2eac1de2b..8a1292823 100644
@@ -2087,16 +2087,28 @@ iavf_set_rx_function(struct rte_eth_dev *dev)
 				    "Using %sVector Scattered Rx (port %d).",
 				    use_avx2 ? "avx2 " : "",
 				    dev->data->port_id);
-			dev->rx_pkt_burst = use_avx2 ?
-					    iavf_recv_scattered_pkts_vec_avx2 :
-					    iavf_recv_scattered_pkts_vec;
+			if (vf->vf_res->vf_cap_flags &
+				VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC)
+				dev->rx_pkt_burst = use_avx2 ?
+					iavf_recv_scattered_pkts_vec_avx2_flex_rxd :
+					iavf_recv_scattered_pkts_vec;
+			else
+				dev->rx_pkt_burst = use_avx2 ?
+					iavf_recv_scattered_pkts_vec_avx2 :
+					iavf_recv_scattered_pkts_vec;
 		} else {
 			PMD_DRV_LOG(DEBUG, "Using %sVector Rx (port %d).",
 				    use_avx2 ? "avx2 " : "",
 				    dev->data->port_id);
-			dev->rx_pkt_burst = use_avx2 ?
-					    iavf_recv_pkts_vec_avx2 :
-					    iavf_recv_pkts_vec;
+			if (vf->vf_res->vf_cap_flags &
+				VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC)
+				dev->rx_pkt_burst = use_avx2 ?
+					iavf_recv_pkts_vec_avx2_flex_rxd :
+					iavf_recv_pkts_vec;
+			else
+				dev->rx_pkt_burst = use_avx2 ?
+					iavf_recv_pkts_vec_avx2 :
+					iavf_recv_pkts_vec;
 		}
 
 		return;
drivers/net/iavf/iavf_rxtx.h +6 -0
index 8246e797f..84ec39132 100644
@@ -414,9 +414,15 @@ uint16_t iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
 				  uint16_t nb_pkts);
 uint16_t iavf_recv_pkts_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,
 				 uint16_t nb_pkts);
+uint16_t iavf_recv_pkts_vec_avx2_flex_rxd(void *rx_queue,
+					  struct rte_mbuf **rx_pkts,
+					  uint16_t nb_pkts);
 uint16_t iavf_recv_scattered_pkts_vec_avx2(void *rx_queue,
 					   struct rte_mbuf **rx_pkts,
 					   uint16_t nb_pkts);
+uint16_t iavf_recv_scattered_pkts_vec_avx2_flex_rxd(void *rx_queue,
+						    struct rte_mbuf **rx_pkts,
+						    uint16_t nb_pkts);
 uint16_t iavf_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
 			    uint16_t nb_pkts);
 uint16_t iavf_xmit_pkts_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts,
drivers/net/iavf/iavf_rxtx_vec_avx2.c +546 -4

585 lines not on this page · raw message

Signature · 1 line
2.17.1
#57 lore raw

Re: [dpdk-dev] [PATCH v5 04/11] net/iavf: flexible Rx descriptor support in AVX path

Quoted 4 lines · Leyi Rong

Support flexible Rx descriptor format in AVX
path of iAVF PMD.

<...>

Quoted 9 lines
drivers/net/iavf/iavf_rxtx_vec_avx2.c +1 -1
@@ -11,14 +11,16 @@
 #endif
 
 static inline void
-iavf_rxq_rearm(struct iavf_rx_queue *rxq)
+iavf_rxq_rearm(struct iavf_rx_queue *rxq, volatile union iavf_rx_desc *rxdp)
 {

drivers/net/iavf/iavf_rxtx_vec_avx2.c line 23

Quoted 10 lines
drivers/net/iavf/iavf_rxtx_vec_avx2.c +4 -2
 	int i;
 	uint16_t rx_id;
-	volatile union iavf_rx_desc *rxdp;
 	struct rte_mbuf **rxp = &rxq->sw_ring[rxq->rxrearm_start];
 
-	rxdp = rxq->rx_ring + rxq->rxrearm_start;
+	if (rxq->rxdid == IAVF_RXDID_COMMS_OVS_1) {
+		volatile union iavf_rx_flex_desc *rxdp =
+			(union iavf_rx_flex_desc *)rxdp;
+	}

Hi Leyi,

This code is causing a build error in icc [1] but what this code exactly does? It is creating a local version of the variable 'rxdp', which is confusing, but that variable's scope is only if block, so it looks like this code really doesn't do anything at all, am I missing something?

[1]
.../drivers/net/iavf/iavf_rxtx_vec_avx2.c(22): warning #592: variable "rxdp" is used before its value is set

    (union iavf_rx_flex_desc *)rxdp;
author
#58 lore raw

PATCH v5 5/11 net/iavf: flexible Rx descriptor support in SSE path

In reply to Leyi Rong

Support flexible Rx descriptor format in SSE
path of iAVF PMD.

---
 drivers/net/iavf/iavf_rxtx.c         |   4 +-
 drivers/net/iavf/iavf_rxtx.h         |   5 +
 drivers/net/iavf/iavf_rxtx_vec_sse.c | 414 +++++++++++++++++++++++++++
 3 files changed, 421 insertions(+), 2 deletions(-)
drivers/net/iavf/iavf_rxtx.c +2 -2
index 8a1292823..92bf4cf8b 100644
@@ -2091,7 +2091,7 @@ iavf_set_rx_function(struct rte_eth_dev *dev)
 				VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC)
 				dev->rx_pkt_burst = use_avx2 ?
 					iavf_recv_scattered_pkts_vec_avx2_flex_rxd :
-					iavf_recv_scattered_pkts_vec;
+					iavf_recv_scattered_pkts_vec_flex_rxd;
 			else
 				dev->rx_pkt_burst = use_avx2 ?
 					iavf_recv_scattered_pkts_vec_avx2 :
@@ -2104,7 +2104,7 @@ iavf_set_rx_function(struct rte_eth_dev *dev)
 				VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC)
 				dev->rx_pkt_burst = use_avx2 ?
 					iavf_recv_pkts_vec_avx2_flex_rxd :
-					iavf_recv_pkts_vec;
+					iavf_recv_pkts_vec_flex_rxd;
 			else
 				dev->rx_pkt_burst = use_avx2 ?
 					iavf_recv_pkts_vec_avx2 :
drivers/net/iavf/iavf_rxtx.h +5 -0
index 84ec39132..9bc857312 100644
@@ -407,9 +407,14 @@ int iavf_dev_tx_desc_status(void *tx_queue, uint16_t offset);
 
 uint16_t iavf_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
 			   uint16_t nb_pkts);
+uint16_t iavf_recv_pkts_vec_flex_rxd(void *rx_queue, struct rte_mbuf **rx_pkts,
+				     uint16_t nb_pkts);
 uint16_t iavf_recv_scattered_pkts_vec(void *rx_queue,
 				     struct rte_mbuf **rx_pkts,
 				     uint16_t nb_pkts);
+uint16_t iavf_recv_scattered_pkts_vec_flex_rxd(void *rx_queue,
+					       struct rte_mbuf **rx_pkts,
+					       uint16_t nb_pkts);
 uint16_t iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
 				  uint16_t nb_pkts);
 uint16_t iavf_recv_pkts_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,
drivers/net/iavf/iavf_rxtx_vec_sse.c +414 -0

444 lines not on this page · raw message

Signature · 1 line
2.17.1
author
#59 lore raw

PATCH v5 6/11 net/iavf: add flow director enabled switch value

In reply to Leyi Rong

The commit adds fdir_enabled flag into iavf_rx_queue structure to identify if fdir id is active. Rx data path can be benefit if fdir id parsing is not needed, especially in vector path.

---
 drivers/net/iavf/iavf.h      |  1 +
 drivers/net/iavf/iavf_rxtx.h | 30 ++++++++++++++++++++++++++++++
 2 files changed, 31 insertions(+)
drivers/net/iavf/iavf.h +1 -0
index fdb31b929..17ceff734 100644
@@ -144,6 +144,7 @@ struct iavf_adapter {
 	bool tx_vec_allowed;
 	const uint32_t *ptype_tbl;
 	bool stopped;
+	uint16_t fdir_ref_cnt;
 };
 
 /* IAVF_DEV_PRIVATE_TO */
drivers/net/iavf/iavf_rxtx.h +30 -0
index 9bc857312..73968847f 100644
@@ -104,6 +104,7 @@ struct iavf_rx_queue {
 
 	uint16_t port_id;        /* device port ID */
 	uint8_t crc_len;        /* 0 if CRC stripped, 4 otherwise */
+	uint8_t fdir_enabled;   /* 0 if FDIR disabled, 1 when enabled */
 	uint16_t queue_id;      /* Rx queue index */
 	uint16_t rx_buf_len;    /* The packet buffer size */
 	uint16_t rx_hdr_len;    /* The header buffer size */
@@ -491,6 +492,35 @@ void iavf_dump_tx_descriptor(const struct iavf_tx_queue *txq,
 	       tx_desc->cmd_type_offset_bsz);
 }
 
+#define FDIR_PROC_ENABLE_PER_QUEUE(ad, on) do { \
+	int i; \
+	for (i = 0; i < (ad)->eth_dev->data->nb_rx_queues; i++) { \
+		struct iavf_rx_queue *rxq = (ad)->eth_dev->data->rx_queues[i]; \
+		if (!rxq) \
+			continue; \
+		rxq->fdir_enabled = on; \
+	} \
+	PMD_DRV_LOG(DEBUG, "FDIR processing on RX set to %d", on); \
+} while (0)
+
+/* Enable/disable flow director Rx processing in data path. */
+static inline
+void iavf_fdir_rx_proc_enable(struct iavf_adapter *ad, bool on)
+{
+	if (on) {
+		/* enable flow director processing */
+		if (ad->fdir_ref_cnt++ == 0)
+			FDIR_PROC_ENABLE_PER_QUEUE(ad, on);
+	} else {
+		if (ad->fdir_ref_cnt >= 1) {
+			ad->fdir_ref_cnt--;
+
+			if (ad->fdir_ref_cnt == 0)
+				FDIR_PROC_ENABLE_PER_QUEUE(ad, on);
+		}
+	}
+}
+
 #ifdef RTE_LIBRTE_IAVF_DEBUG_DUMP_DESC
 #define IAVF_DUMP_RX_DESC(rxq, desc, rx_id) \
 	iavf_dump_rx_descriptor(rxq, desc, rx_id)
Signature · 1 line
2.17.1
author
#60 lore raw

PATCH v5 7/11 net/iavf: support flow mark in normal data path

In reply to Leyi Rong

Support Flow Director mark ID parsing in normal path.

---
 drivers/net/iavf/iavf.h      |  3 +++
 drivers/net/iavf/iavf_rxtx.c | 51 +++++++++++++++++++++++++++++++-----
 2 files changed, 47 insertions(+), 7 deletions(-)
drivers/net/iavf/iavf.h +3 -0
index 17ceff734..78bdaff20 100644
@@ -67,6 +67,9 @@
 #define IAVF_48_BIT_WIDTH (CHAR_BIT * 6)
 #define IAVF_48_BIT_MASK  RTE_LEN2MASK(IAVF_48_BIT_WIDTH, uint64_t)
 
+#define IAVF_RX_DESC_EXT_STATUS_FLEXBH_MASK  0x03
+#define IAVF_RX_DESC_EXT_STATUS_FLEXBH_FD_ID 0x01
+
 struct iavf_adapter;
 struct iavf_rx_queue;
 struct iavf_tx_queue;
drivers/net/iavf/iavf_rxtx.c +44 -7

117 lines not on this page · raw message

Signature · 1 line
2.17.1
author
#61 lore raw

PATCH v5 8/11 net/iavf: support flow mark in AVX path

In reply to Leyi Rong

Support Flow Director mark ID parsing from Flex
Rx descriptor in AVX path.

---
 drivers/net/iavf/iavf_rxtx_vec_avx2.c | 72 +++++++++++++++++++++++++--
 1 file changed, 67 insertions(+), 5 deletions(-)
drivers/net/iavf/iavf_rxtx_vec_avx2.c +67 -5
index b23188fd3..3bf5833fa 100644
@@ -616,6 +616,25 @@ _iavf_recv_raw_pkts_vec_avx2(struct iavf_rx_queue *rxq,
 	return received;
 }
 
+static inline __m256i
+flex_rxd_to_fdir_flags_vec_avx2(const __m256i fdir_id0_7)
+{
+#define FDID_MIS_MAGIC 0xFFFFFFFF
+	RTE_BUILD_BUG_ON(PKT_RX_FDIR != (1 << 2));
+	RTE_BUILD_BUG_ON(PKT_RX_FDIR_ID != (1 << 13));
+	const __m256i pkt_fdir_bit = _mm256_set1_epi32(PKT_RX_FDIR |
+			PKT_RX_FDIR_ID);
+	/* desc->flow_id field == 0xFFFFFFFF means fdir mismatch */
+	const __m256i fdir_mis_mask = _mm256_set1_epi32(FDID_MIS_MAGIC);
+	__m256i fdir_mask = _mm256_cmpeq_epi32(fdir_id0_7,
+			fdir_mis_mask);
+	/* this XOR op results to bit-reverse the fdir_mask */
+	fdir_mask = _mm256_xor_si256(fdir_mask, fdir_mis_mask);
+	const __m256i fdir_flags = _mm256_and_si256(fdir_mask, pkt_fdir_bit);
+
+	return fdir_flags;
+}
+
 static inline uint16_t
 _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 				      struct rte_mbuf **rx_pkts,
@@ -678,8 +697,8 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 	const __m256i shuf_msk =
 		_mm256_set_epi8
 			(/* first descriptor */
-			 15, 14,
-			 13, 12,	/* octet 12~15, 32 bits rss */
+			 0xFF, 0xFF,
+			 0xFF, 0xFF,    /* rss not supported */
 			 11, 10,	/* octet 10~11, 16 bits vlan_macip */
 			 5, 4,		/* octet 4~5, 16 bits data_len */
 			 0xFF, 0xFF,	/* skip hi 16 bits pkt_len, zero out */
@@ -687,8 +706,8 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 			 0xFF, 0xFF,	/* pkt_type set as unknown */
 			 0xFF, 0xFF,	/*pkt_type set as unknown */
 			 /* second descriptor */
-			 15, 14,
-			 13, 12,	/* octet 12~15, 32 bits rss */
+			 0xFF, 0xFF,
+			 0xFF, 0xFF,    /* rss not supported */
 			 11, 10,	/* octet 10~11, 16 bits vlan_macip */
 			 5, 4,		/* octet 4~5, 16 bits data_len */
 			 0xFF, 0xFF,	/* skip hi 16 bits pkt_len, zero out */
@@ -930,8 +949,51 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 					    rss_vlan_flag_bits);
 
 		/* merge flags */
-		const __m256i mbuf_flags = _mm256_or_si256(l3_l4_flags,
+		__m256i mbuf_flags = _mm256_or_si256(l3_l4_flags,
 				rss_vlan_flags);
+
+		if (rxq->fdir_enabled) {
+			const __m256i fdir_id4_7 =
+				_mm256_unpackhi_epi32(raw_desc6_7, raw_desc4_5);
+
+			const __m256i fdir_id0_3 =
+				_mm256_unpackhi_epi32(raw_desc2_3, raw_desc0_1);
+
+			const __m256i fdir_id0_7 =
+				_mm256_unpackhi_epi64(fdir_id4_7, fdir_id0_3);
+
+			const __m256i fdir_flags =
+				flex_rxd_to_fdir_flags_vec_avx2(fdir_id0_7);
+
+			/* merge with fdir_flags */
+			mbuf_flags = _mm256_or_si256(mbuf_flags, fdir_flags);
+
+			/* write to mbuf: have to use scalar store here */
+			rx_pkts[i + 0]->hash.fdir.hi =
+				_mm256_extract_epi32(fdir_id0_7, 3);
+
+			rx_pkts[i + 1]->hash.fdir.hi =
+				_mm256_extract_epi32(fdir_id0_7, 7);
+
+			rx_pkts[i + 2]->hash.fdir.hi =
+				_mm256_extract_epi32(fdir_id0_7, 2);
+
+			rx_pkts[i + 3]->hash.fdir.hi =
+				_mm256_extract_epi32(fdir_id0_7, 6);
+
+			rx_pkts[i + 4]->hash.fdir.hi =
+				_mm256_extract_epi32(fdir_id0_7, 1);
+
+			rx_pkts[i + 5]->hash.fdir.hi =
+				_mm256_extract_epi32(fdir_id0_7, 5);
+
+			rx_pkts[i + 6]->hash.fdir.hi =
+				_mm256_extract_epi32(fdir_id0_7, 0);
+
+			rx_pkts[i + 7]->hash.fdir.hi =
+				_mm256_extract_epi32(fdir_id0_7, 4);
+		} /* if() on fdir_enabled */
+
 		/**
 		 * At this point, we have the 8 sets of flags in the low 16-bits
 		 * of each 32-bit value in vlan0.
Signature · 1 line
2.17.1
author
#62 lore raw

PATCH v5 9/11 net/iavf: support flow mark in SSE path

In reply to Leyi Rong

Support Flow Director mark ID parsing from Flex
Rx descriptor in SSE path.

---
 drivers/net/iavf/iavf_rxtx_vec_sse.c | 52 +++++++++++++++++++++++++++-
 1 file changed, 51 insertions(+), 1 deletion(-)
drivers/net/iavf/iavf_rxtx_vec_sse.c +51 -1
index 9c1f2a445..d7d840853 100644
@@ -189,6 +189,25 @@ desc_to_olflags_v(struct iavf_rx_queue *rxq, __m128i descs[4],
 	_mm_store_si128((__m128i *)&rx_pkts[3]->rearm_data, rearm3);
 }
 
+static inline __m128i
+flex_rxd_to_fdir_flags_vec(const __m128i fdir_id0_3)
+{
+#define FDID_MIS_MAGIC 0xFFFFFFFF
+	RTE_BUILD_BUG_ON(PKT_RX_FDIR != (1 << 2));
+	RTE_BUILD_BUG_ON(PKT_RX_FDIR_ID != (1 << 13));
+	const __m128i pkt_fdir_bit = _mm_set1_epi32(PKT_RX_FDIR |
+			PKT_RX_FDIR_ID);
+	/* desc->flow_id field == 0xFFFFFFFF means fdir mismatch */
+	const __m128i fdir_mis_mask = _mm_set1_epi32(FDID_MIS_MAGIC);
+	__m128i fdir_mask = _mm_cmpeq_epi32(fdir_id0_3,
+			fdir_mis_mask);
+	/* this XOR op results to bit-reverse the fdir_mask */
+	fdir_mask = _mm_xor_si128(fdir_mask, fdir_mis_mask);
+	const __m128i fdir_flags = _mm_and_si128(fdir_mask, pkt_fdir_bit);
+
+	return fdir_flags;
+}
+
 static inline void
 flex_desc_to_olflags_v(struct iavf_rx_queue *rxq, __m128i descs[4],
 		       struct rte_mbuf **rx_pkts)
@@ -267,6 +286,36 @@ flex_desc_to_olflags_v(struct iavf_rx_queue *rxq, __m128i descs[4],
 	/* merge the flags */
 	flags = _mm_or_si128(flags, rss_vlan);
 
+	if (rxq->fdir_enabled) {
+		const __m128i fdir_id0_1 =
+			_mm_unpackhi_epi32(descs[0], descs[1]);
+
+		const __m128i fdir_id2_3 =
+			_mm_unpackhi_epi32(descs[2], descs[3]);
+
+		const __m128i fdir_id0_3 =
+			_mm_unpackhi_epi64(fdir_id0_1, fdir_id2_3);
+
+		const __m128i fdir_flags =
+			flex_rxd_to_fdir_flags_vec(fdir_id0_3);
+
+		/* merge with fdir_flags */
+		flags = _mm_or_si128(flags, fdir_flags);
+
+		/* write fdir_id to mbuf */
+		rx_pkts[0]->hash.fdir.hi =
+			_mm_extract_epi32(fdir_id0_3, 0);
+
+		rx_pkts[1]->hash.fdir.hi =
+			_mm_extract_epi32(fdir_id0_3, 1);
+
+		rx_pkts[2]->hash.fdir.hi =
+			_mm_extract_epi32(fdir_id0_3, 2);
+
+		rx_pkts[3]->hash.fdir.hi =
+			_mm_extract_epi32(fdir_id0_3, 3);
+	} /* if() on fdir_enabled */
+
 	/**
 	 * At this point, we have the 4 sets of flags in the low 16-bits
 	 * of each 32-bit value in flags.
@@ -604,7 +653,8 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 	const __m128i zero = _mm_setzero_si128();
 	/* mask to shuffle from desc. to mbuf */
 	const __m128i shuf_msk = _mm_set_epi8
-			(15, 14, 13, 12,  /* octet 12~15, 32 bits rss */
+			(0xFF, 0xFF,
+			 0xFF, 0xFF,  /* rss not supported */
 			 11, 10,      /* octet 10~11, 16 bits vlan_macip */
 			 5, 4,        /* octet 4~5, 16 bits data_len */
 			 0xFF, 0xFF,  /* skip high 16 bits pkt_len, zero out */
Signature · 1 line
2.17.1
author
#63 lore raw

PATCH v5 10/11 net/iavf: add RSS hash parsing in AVX path

In reply to Leyi Rong

Support RSS hash parsing from Flex Rx
descriptor in AVX data path.

---
 drivers/net/iavf/iavf_rxtx_vec_avx2.c | 94 ++++++++++++++++++++++++++-
 1 file changed, 92 insertions(+), 2 deletions(-)
drivers/net/iavf/iavf_rxtx_vec_avx2.c +92 -2
index 3bf5833fa..c725d9d66 100644
@@ -698,7 +698,7 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 		_mm256_set_epi8
 			(/* first descriptor */
 			 0xFF, 0xFF,
-			 0xFF, 0xFF,    /* rss not supported */
+			 0xFF, 0xFF,    /* rss hash parsed separately */
 			 11, 10,	/* octet 10~11, 16 bits vlan_macip */
 			 5, 4,		/* octet 4~5, 16 bits data_len */
 			 0xFF, 0xFF,	/* skip hi 16 bits pkt_len, zero out */
@@ -707,7 +707,7 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 			 0xFF, 0xFF,	/*pkt_type set as unknown */
 			 /* second descriptor */
 			 0xFF, 0xFF,
-			 0xFF, 0xFF,    /* rss not supported */
+			 0xFF, 0xFF,    /* rss hash parsed separately */
 			 11, 10,	/* octet 10~11, 16 bits vlan_macip */
 			 5, 4,		/* octet 4~5, 16 bits data_len */
 			 0xFF, 0xFF,	/* skip hi 16 bits pkt_len, zero out */
@@ -994,6 +994,96 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 				_mm256_extract_epi32(fdir_id0_7, 4);
 		} /* if() on fdir_enabled */
 
+#ifndef RTE_LIBRTE_IAVF_16BYTE_RX_DESC
+		/**
+		 * needs to load 2nd 16B of each desc for RSS hash parsing,
+		 * will cause performance drop to get into this context.
+		 */
+		if (rxq->vsi->adapter->eth_dev->data->dev_conf.rxmode.offloads &
+				DEV_RX_OFFLOAD_RSS_HASH) {
+			/* load bottom half of every 32B desc */
+			const __m128i raw_desc_bh7 =
+				_mm_load_si128
+					((void *)(&rxdp[7].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh6 =
+				_mm_load_si128
+					((void *)(&rxdp[6].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh5 =
+				_mm_load_si128
+					((void *)(&rxdp[5].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh4 =
+				_mm_load_si128
+					((void *)(&rxdp[4].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh3 =
+				_mm_load_si128
+					((void *)(&rxdp[3].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh2 =
+				_mm_load_si128
+					((void *)(&rxdp[2].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh1 =
+				_mm_load_si128
+					((void *)(&rxdp[1].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh0 =
+				_mm_load_si128
+					((void *)(&rxdp[0].wb.status_error1));
+
+			__m256i raw_desc_bh6_7 =
+				_mm256_inserti128_si256
+					(_mm256_castsi128_si256(raw_desc_bh6),
+					raw_desc_bh7, 1);
+			__m256i raw_desc_bh4_5 =
+				_mm256_inserti128_si256
+					(_mm256_castsi128_si256(raw_desc_bh4),
+					raw_desc_bh5, 1);
+			__m256i raw_desc_bh2_3 =
+				_mm256_inserti128_si256
+					(_mm256_castsi128_si256(raw_desc_bh2),
+					raw_desc_bh3, 1);
+			__m256i raw_desc_bh0_1 =
+				_mm256_inserti128_si256
+					(_mm256_castsi128_si256(raw_desc_bh0),
+					raw_desc_bh1, 1);
+
+			/**
+			 * to shift the 32b RSS hash value to the
+			 * highest 32b of each 128b before mask
+			 */
+			__m256i rss_hash6_7 =
+				_mm256_slli_epi64(raw_desc_bh6_7, 32);
+			__m256i rss_hash4_5 =
+				_mm256_slli_epi64(raw_desc_bh4_5, 32);
+			__m256i rss_hash2_3 =
+				_mm256_slli_epi64(raw_desc_bh2_3, 32);
+			__m256i rss_hash0_1 =
+				_mm256_slli_epi64(raw_desc_bh0_1, 32);
+
+			__m256i rss_hash_msk =
+				_mm256_set_epi32(0xFFFFFFFF, 0, 0, 0,
+						 0xFFFFFFFF, 0, 0, 0);
+
+			rss_hash6_7 = _mm256_and_si256
+					(rss_hash6_7, rss_hash_msk);
+			rss_hash4_5 = _mm256_and_si256
+					(rss_hash4_5, rss_hash_msk);
+			rss_hash2_3 = _mm256_and_si256
+					(rss_hash2_3, rss_hash_msk);
+			rss_hash0_1 = _mm256_and_si256
+					(rss_hash0_1, rss_hash_msk);
+
+			mb6_7 = _mm256_or_si256(mb6_7, rss_hash6_7);
+			mb4_5 = _mm256_or_si256(mb4_5, rss_hash4_5);
+			mb2_3 = _mm256_or_si256(mb2_3, rss_hash2_3);
+			mb0_1 = _mm256_or_si256(mb0_1, rss_hash0_1);
+		} /* if() on RSS hash parsing */
+#endif
+
 		/**
 		 * At this point, we have the 8 sets of flags in the low 16-bits
 		 * of each 32-bit value in vlan0.
Signature · 1 line
2.17.1
author
#64 lore raw

PATCH v5 11/11 net/iavf: add RSS hash parsing in SSE path

In reply to Leyi Rong

Support RSS hash parsing from Flex Rx
descriptor in SSE data path.

---
 drivers/net/iavf/iavf_rxtx_vec_sse.c | 88 +++++++++++++++++++++++-----
 1 file changed, 72 insertions(+), 16 deletions(-)
drivers/net/iavf/iavf_rxtx_vec_sse.c +72 -16
index d7d840853..2f115aa94 100644
@@ -654,7 +654,7 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 	/* mask to shuffle from desc. to mbuf */
 	const __m128i shuf_msk = _mm_set_epi8
 			(0xFF, 0xFF,
-			 0xFF, 0xFF,  /* rss not supported */
+			 0xFF, 0xFF,  /* rss hash parsed separately */
 			 11, 10,      /* octet 10~11, 16 bits vlan_macip */
 			 5, 4,        /* octet 4~5, 16 bits data_len */
 			 0xFF, 0xFF,  /* skip high 16 bits pkt_len, zero out */
@@ -745,7 +745,7 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 	     pos += IAVF_VPMD_DESCS_PER_LOOP,
 	     rxdp += IAVF_VPMD_DESCS_PER_LOOP) {
 		__m128i descs[IAVF_VPMD_DESCS_PER_LOOP];
-		__m128i pkt_mb1, pkt_mb2, pkt_mb3, pkt_mb4;
+		__m128i pkt_mb0, pkt_mb1, pkt_mb2, pkt_mb3;
 		__m128i staterr, sterr_tmp1, sterr_tmp2;
 		/* 2 64 bit or 4 32 bit mbuf pointers in one XMM reg. */
 		__m128i mbp1;
@@ -791,8 +791,12 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 		rte_compiler_barrier();
 
 		/* D.1 pkt 3,4 convert format from desc to pktmbuf */
-		pkt_mb4 = _mm_shuffle_epi8(descs[3], shuf_msk);
-		pkt_mb3 = _mm_shuffle_epi8(descs[2], shuf_msk);
+		pkt_mb3 = _mm_shuffle_epi8(descs[3], shuf_msk);
+		pkt_mb2 = _mm_shuffle_epi8(descs[2], shuf_msk);
+
+		/* D.1 pkt 1,2 convert format from desc to pktmbuf */
+		pkt_mb1 = _mm_shuffle_epi8(descs[1], shuf_msk);
+		pkt_mb0 = _mm_shuffle_epi8(descs[0], shuf_msk);
 
 		/* C.1 4=>2 filter staterr info only */
 		sterr_tmp2 = _mm_unpackhi_epi32(descs[3], descs[2]);
@@ -802,12 +806,68 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 		flex_desc_to_olflags_v(rxq, descs, &rx_pkts[pos]);
 
 		/* D.2 pkt 3,4 set in_port/nb_seg and remove crc */
-		pkt_mb4 = _mm_add_epi16(pkt_mb4, crc_adjust);
 		pkt_mb3 = _mm_add_epi16(pkt_mb3, crc_adjust);
+		pkt_mb2 = _mm_add_epi16(pkt_mb2, crc_adjust);
 
-		/* D.1 pkt 1,2 convert format from desc to pktmbuf */
-		pkt_mb2 = _mm_shuffle_epi8(descs[1], shuf_msk);
-		pkt_mb1 = _mm_shuffle_epi8(descs[0], shuf_msk);
+		/* D.2 pkt 1,2 set in_port/nb_seg and remove crc */
+		pkt_mb1 = _mm_add_epi16(pkt_mb1, crc_adjust);
+		pkt_mb0 = _mm_add_epi16(pkt_mb0, crc_adjust);
+
+#ifndef RTE_LIBRTE_IAVF_16BYTE_RX_DESC
+		/**
+		 * needs to load 2nd 16B of each desc for RSS hash parsing,
+		 * will cause performance drop to get into this context.
+		 */
+		if (rxq->vsi->adapter->eth_dev->data->dev_conf.rxmode.offloads &
+				DEV_RX_OFFLOAD_RSS_HASH) {
+			/* load bottom half of every 32B desc */
+			const __m128i raw_desc_bh3 =
+				_mm_load_si128
+					((void *)(&rxdp[3].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh2 =
+				_mm_load_si128
+					((void *)(&rxdp[2].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh1 =
+				_mm_load_si128
+					((void *)(&rxdp[1].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh0 =
+				_mm_load_si128
+					((void *)(&rxdp[0].wb.status_error1));
+
+			/**
+			 * to shift the 32b RSS hash value to the
+			 * highest 32b of each 128b before mask
+			 */
+			__m128i rss_hash3 =
+				_mm_slli_epi64(raw_desc_bh3, 32);
+			__m128i rss_hash2 =
+				_mm_slli_epi64(raw_desc_bh2, 32);
+			__m128i rss_hash1 =
+				_mm_slli_epi64(raw_desc_bh1, 32);
+			__m128i rss_hash0 =
+				_mm_slli_epi64(raw_desc_bh0, 32);
+
+			__m128i rss_hash_msk =
+				_mm_set_epi32(0xFFFFFFFF, 0, 0, 0);
+
+			rss_hash3 = _mm_and_si128
+					(rss_hash3, rss_hash_msk);
+			rss_hash2 = _mm_and_si128
+					(rss_hash2, rss_hash_msk);
+			rss_hash1 = _mm_and_si128
+					(rss_hash1, rss_hash_msk);
+			rss_hash0 = _mm_and_si128
+					(rss_hash0, rss_hash_msk);
+
+			pkt_mb3 = _mm_or_si128(pkt_mb3, rss_hash3);
+			pkt_mb2 = _mm_or_si128(pkt_mb2, rss_hash2);
+			pkt_mb1 = _mm_or_si128(pkt_mb1, rss_hash1);
+			pkt_mb0 = _mm_or_si128(pkt_mb0, rss_hash0);
+		} /* if() on RSS hash parsing */
+#endif
 
 		/* C.2 get 4 pkts staterr value  */
 		staterr = _mm_unpacklo_epi32(sterr_tmp1, sterr_tmp2);
@@ -815,14 +875,10 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 		/* D.3 copy final 3,4 data to rx_pkts */
 		_mm_storeu_si128
 			((void *)&rx_pkts[pos + 3]->rx_descriptor_fields1,
-			 pkt_mb4);
+			 pkt_mb3);
 		_mm_storeu_si128
 			((void *)&rx_pkts[pos + 2]->rx_descriptor_fields1,
-			 pkt_mb3);
-
-		/* D.2 pkt 1,2 set in_port/nb_seg and remove crc */
-		pkt_mb2 = _mm_add_epi16(pkt_mb2, crc_adjust);
-		pkt_mb1 = _mm_add_epi16(pkt_mb1, crc_adjust);
+			 pkt_mb2);
 
 		/* C* extract and record EOP bit */
 		if (split_packet) {
@@ -846,9 +902,9 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 		/* D.3 copy final 1,2 data to rx_pkts */
 		_mm_storeu_si128
 			((void *)&rx_pkts[pos + 1]->rx_descriptor_fields1,
-			 pkt_mb2);
+			 pkt_mb1);
 		_mm_storeu_si128((void *)&rx_pkts[pos]->rx_descriptor_fields1,
-				 pkt_mb1);
+				 pkt_mb0);
 		flex_desc_to_ptype_v(descs, &rx_pkts[pos], ptype_tbl);
 		/* C.4 calc avaialbe number of desc */
 		var = __builtin_popcountll(_mm_cvtsi128_si64(staterr));
Signature · 1 line
2.17.1
Reviewed-by on the v5 series
#65 lore raw
In reply to Leyi Rong
Quoted 71 lines

-----Original Message-----
From: Rong, Leyi <[email protected]>
Sent: Thursday, April 16, 2020 4:10 PM
To: Wu, Jingjing <[email protected]>; Zhang, Qi Z <[email protected]>; Xing, Beilei <[email protected]>; Ye, Xiaolong <[email protected]>
Cc: [email protected]; Rong, Leyi <[email protected]>
Subject: [PATCH v5 00/11] framework for advanced iAVF PMD

This patchset enable framework for advanced iAVF, includes flexible descriptor support, FDIR mark id and RSS hash support.

---
v5:
- Set rxdid = 0 in iavf_configure_queues() when it's supported and RTE_LIBRTE_IAVF_16BYTE_RX_DESC is y. So the error can be captured explicitly as kernel PF does not support it yet.
- Fix some build errors when RTE_LIBRTE_IAVF_16BYTE_RX_DESC is y.

v4:
- Avoid initial declaration in 'for' loop to backward-compatible with older gcc versions.

v3:
- Remove patch to query DDP package info as it's specific to DCF.
- Specify fdir_ref_cnt as per VF value for enable/disable FDIR ID parse.
- Move fdir_enabled flag to per Queue value for cache benefit when access in vector routines.
- Store extracted flow_id value to mbuf without judgement to avoid branch speculation fail, which is benefit to performance.

v2:
- Revert macro RTE_LIBRTE_IAVF_16BYTE_RX_DESC deletion as it's defined in AVF spec.
- Typo RTE_LIBRTE_ICE_16BYTE_RX_DESC fix in iavf_rxtx.c.
- Move flex desc definitions into iavf_rxtx.h.
- Up to date to match with the latest version of virtchnl.h.
- Extract a new internal func iavf_update_rx_tail to call.
- Remove iavf_dev_rxq_count_flex_rxd()/iavf_dev_rx_desc_status_flex_rxd(), as the accompanying legacy ones can deal with the flex desc cases.
- Move rxq->rxdid assignment from iavf_configure_queues() to iavf_dev_rx_queue_setup().
- Unfold _mm_extract_epi32(fdir_id0_3, i) to fix build error when using GCC compile option -O0.

Leyi Rong (11):
net/iavf: flexible Rx descriptor definitions net/iavf: return error if opcode is mismatched net/iavf: flexible Rx descriptor support in normal path net/iavf: flexible Rx descriptor support in AVX path net/iavf: flexible Rx descriptor support in SSE path net/iavf: add flow director enabled switch value net/iavf: support flow mark in normal data path net/iavf: support flow mark in AVX path net/iavf: support flow mark in SSE path net/iavf: add RSS hash parsing in AVX path net/iavf: add RSS hash parsing in SSE path

 drivers/net/iavf/iavf.h               |   6 +
 drivers/net/iavf/iavf_ethdev.c        |   8 +
 drivers/net/iavf/iavf_rxtx.c          | 546 ++++++++++++++++++--
 drivers/net/iavf/iavf_rxtx.h          | 250 +++++++++
 drivers/net/iavf/iavf_rxtx_vec_avx2.c | 702
+++++++++++++++++++++++++-  drivers/net/iavf/iavf_rxtx_vec_sse.c  |
520 +++++++++++++++++++
 drivers/net/iavf/iavf_vchnl.c         |  59 ++-
 7 files changed, 2042 insertions(+), 49 deletions(-)
Quoted 2 lines

--
2.17.1

Applied
#66 lore raw
Quoted 71 lines · Qi Z
Quoted 71 lines

-----Original Message-----
From: Rong, Leyi <[email protected]>
Sent: Thursday, April 16, 2020 4:10 PM
To: Wu, Jingjing <[email protected]>; Zhang, Qi Z <[email protected]>; Xing, Beilei <[email protected]>; Ye, Xiaolong <[email protected]>
Cc: [email protected]; Rong, Leyi <[email protected]>
Subject: [PATCH v5 00/11] framework for advanced iAVF PMD

This patchset enable framework for advanced iAVF, includes flexible descriptor support, FDIR mark id and RSS hash support.

---
v5:
- Set rxdid = 0 in iavf_configure_queues() when it's supported and RTE_LIBRTE_IAVF_16BYTE_RX_DESC is y. So the error can be captured explicitly as kernel PF does not support it yet.
- Fix some build errors when RTE_LIBRTE_IAVF_16BYTE_RX_DESC is y.

v4:
- Avoid initial declaration in 'for' loop to backward-compatible with older gcc versions.

v3:
- Remove patch to query DDP package info as it's specific to DCF.
- Specify fdir_ref_cnt as per VF value for enable/disable FDIR ID parse.
- Move fdir_enabled flag to per Queue value for cache benefit when access in vector routines.
- Store extracted flow_id value to mbuf without judgement to avoid branch speculation fail, which is benefit to performance.

v2:
- Revert macro RTE_LIBRTE_IAVF_16BYTE_RX_DESC deletion as it's defined in AVF spec.
- Typo RTE_LIBRTE_ICE_16BYTE_RX_DESC fix in iavf_rxtx.c.
- Move flex desc definitions into iavf_rxtx.h.
- Up to date to match with the latest version of virtchnl.h.
- Extract a new internal func iavf_update_rx_tail to call.
- Remove iavf_dev_rxq_count_flex_rxd()/iavf_dev_rx_desc_status_flex_rxd(), as the accompanying legacy ones can deal with the flex desc cases.
- Move rxq->rxdid assignment from iavf_configure_queues() to iavf_dev_rx_queue_setup().
- Unfold _mm_extract_epi32(fdir_id0_3, i) to fix build error when using GCC compile option -O0.

Leyi Rong (11):
net/iavf: flexible Rx descriptor definitions net/iavf: return error if opcode is mismatched net/iavf: flexible Rx descriptor support in normal path net/iavf: flexible Rx descriptor support in AVX path net/iavf: flexible Rx descriptor support in SSE path net/iavf: add flow director enabled switch value net/iavf: support flow mark in normal data path net/iavf: support flow mark in AVX path net/iavf: support flow mark in SSE path net/iavf: add RSS hash parsing in AVX path net/iavf: add RSS hash parsing in SSE path

 drivers/net/iavf/iavf.h               |   6 +
 drivers/net/iavf/iavf_ethdev.c        |   8 +
 drivers/net/iavf/iavf_rxtx.c          | 546 ++++++++++++++++++--
 drivers/net/iavf/iavf_rxtx.h          | 250 +++++++++
 drivers/net/iavf/iavf_rxtx_vec_avx2.c | 702
+++++++++++++++++++++++++-  drivers/net/iavf/iavf_rxtx_vec_sse.c  |
520 +++++++++++++++++++
 drivers/net/iavf/iavf_vchnl.c         |  59 ++-
 7 files changed, 2042 insertions(+), 49 deletions(-)
Quoted 6 lines
Quoted 2 lines

--
2.17.1

Applied to dpdk-next-net-intel, Thanks.

author
#67 lore raw

v6 cover framework for advanced iAVF PMD

In reply to Leyi Rong

This patchset enable framework for advanced iAVF, includes flexible descriptor support, FDIR mark id and RSS hash support.

---
v6:
- Remove some redundant code which may cause ICC build error.

v5:
- Set rxdid = 0 in iavf_configure_queues() when it's supported and RTE_LIBRTE_IAVF_16BYTE_RX_DESC is y. So the error can be captured explicitly as kernel PF does not support it yet.
- Fix some build errors when RTE_LIBRTE_IAVF_16BYTE_RX_DESC is y.

v4:
- Avoid initial declaration in 'for' loop to backward-compatible with older gcc versions.

v3:
- Remove patch to query DDP package info as it's specific to DCF.
- Specify fdir_ref_cnt as per VF value for enable/disable FDIR ID parse.
- Move fdir_enabled flag to per Queue value for cache benefit when access in vector routines.
- Store extracted flow_id value to mbuf without judgement to avoid branch speculation fail, which is benefit to performance.

v2:
- Revert macro RTE_LIBRTE_IAVF_16BYTE_RX_DESC deletion as it's defined in AVF spec.
- Typo RTE_LIBRTE_ICE_16BYTE_RX_DESC fix in iavf_rxtx.c.
- Move flex desc definitions into iavf_rxtx.h.
- Up to date to match with the latest version of virtchnl.h.
- Extract a new internal func iavf_update_rx_tail to call.
- Remove iavf_dev_rxq_count_flex_rxd()/iavf_dev_rx_desc_status_flex_rxd(), as the accompanying legacy ones can deal with the flex desc cases.
- Move rxq->rxdid assignment from iavf_configure_queues() to iavf_dev_rx_queue_setup().
- Unfold _mm_extract_epi32(fdir_id0_3, i) to fix build error when using GCC compile option -O0.

Leyi Rong (11):
net/iavf: flexible Rx descriptor definitions net/iavf: return error if opcode is mismatched net/iavf: flexible Rx descriptor support in normal path net/iavf: flexible Rx descriptor support in AVX path net/iavf: flexible Rx descriptor support in SSE path net/iavf: add flow director enabled switch value net/iavf: support flow mark in normal data path net/iavf: support flow mark in AVX path net/iavf: support flow mark in SSE path net/iavf: add RSS hash parsing in AVX path net/iavf: add RSS hash parsing in SSE path

 drivers/net/iavf/iavf.h               |   6 +
 drivers/net/iavf/iavf_ethdev.c        |   8 +
 drivers/net/iavf/iavf_rxtx.c          | 546 ++++++++++++++++++--
 drivers/net/iavf/iavf_rxtx.h          | 250 ++++++++++
 drivers/net/iavf/iavf_rxtx_vec_avx2.c | 691 ++++++++++++++++++++++++++
 drivers/net/iavf/iavf_rxtx_vec_sse.c  | 520 +++++++++++++++++++
 drivers/net/iavf/iavf_vchnl.c         |  59 ++-
 7 files changed, 2035 insertions(+), 45 deletions(-)
Signature · 1 line
2.17.1
author
#68 lore raw

PATCH v6 1/11 net/iavf: flexible Rx descriptor definitions

Add definitions for flexible Rx descriptor structures and macros.

---
 drivers/net/iavf/iavf_rxtx.h | 200 +++++++++++++++++++++++++++++++++++
 1 file changed, 200 insertions(+)
drivers/net/iavf/iavf_rxtx.h +200 -0
index 09b5bd99e..d04233bf8 100644
@@ -157,6 +157,206 @@ union iavf_tx_offload {
 	};
 };
 
+/* Rx Flex Descriptors
+ * These descriptors are used instead of the legacy version descriptors
+ */
+union iavf_16b_rx_flex_desc {
+	struct {
+		__le64 pkt_addr; /* Packet buffer address */
+		__le64 hdr_addr; /* Header buffer address */
+				 /* bit 0 of hdr_addr is DD bit */
+	} read;
+	struct {
+		/* Qword 0 */
+		u8 rxdid; /* descriptor builder profile ID */
+		u8 mir_id_umb_cast; /* mirror=[5:0], umb=[7:6] */
+		__le16 ptype_flex_flags0; /* ptype=[9:0], ff0=[15:10] */
+		__le16 pkt_len; /* [15:14] are reserved */
+		__le16 hdr_len_sph_flex_flags1; /* header=[10:0] */
+						/* sph=[11:11] */
+						/* ff1/ext=[15:12] */
+
+		/* Qword 1 */
+		__le16 status_error0;
+		__le16 l2tag1;
+		__le16 flex_meta0;
+		__le16 flex_meta1;
+	} wb; /* writeback */
+};
+
+union iavf_32b_rx_flex_desc {
+	struct {
+		__le64 pkt_addr; /* Packet buffer address */
+		__le64 hdr_addr; /* Header buffer address */
+				 /* bit 0 of hdr_addr is DD bit */
+		__le64 rsvd1;
+		__le64 rsvd2;
+	} read;
+	struct {
+		/* Qword 0 */
+		u8 rxdid; /* descriptor builder profile ID */
+		u8 mir_id_umb_cast; /* mirror=[5:0], umb=[7:6] */
+		__le16 ptype_flex_flags0; /* ptype=[9:0], ff0=[15:10] */
+		__le16 pkt_len; /* [15:14] are reserved */
+		__le16 hdr_len_sph_flex_flags1; /* header=[10:0] */
+						/* sph=[11:11] */
+						/* ff1/ext=[15:12] */
+
+		/* Qword 1 */
+		__le16 status_error0;
+		__le16 l2tag1;
+		__le16 flex_meta0;
+		__le16 flex_meta1;
+
+		/* Qword 2 */
+		__le16 status_error1;
+		u8 flex_flags2;
+		u8 time_stamp_low;
+		__le16 l2tag2_1st;
+		__le16 l2tag2_2nd;
+
+		/* Qword 3 */
+		__le16 flex_meta2;
+		__le16 flex_meta3;
+		union {
+			struct {
+				__le16 flex_meta4;
+				__le16 flex_meta5;
+			} flex;
+			__le32 ts_high;
+		} flex_ts;
+	} wb; /* writeback */
+};
+
+/* Rx Flex Descriptor
+ * RxDID Profile ID 16-21
+ * Flex-field 0: RSS hash lower 16-bits
+ * Flex-field 1: RSS hash upper 16-bits
+ * Flex-field 2: Flow ID lower 16-bits
+ * Flex-field 3: Flow ID upper 16-bits
+ * Flex-field 4: AUX0
+ * Flex-field 5: AUX1
+ */
+struct iavf_32b_rx_flex_desc_comms {
+	/* Qword 0 */
+	u8 rxdid;
+	u8 mir_id_umb_cast;
+	__le16 ptype_flexi_flags0;
+	__le16 pkt_len;
+	__le16 hdr_len_sph_flex_flags1;
+
+	/* Qword 1 */
+	__le16 status_error0;
+	__le16 l2tag1;
+	__le32 rss_hash;
+
+	/* Qword 2 */
+	__le16 status_error1;
+	u8 flexi_flags2;
+	u8 ts_low;
+	__le16 l2tag2_1st;
+	__le16 l2tag2_2nd;
+
+	/* Qword 3 */
+	__le32 flow_id;
+	union {
+		struct {
+			__le16 aux0;
+			__le16 aux1;
+		} flex;
+		__le32 ts_high;
+	} flex_ts;
+};
+
+/* Rx Flex Descriptor
+ * RxDID Profile ID 22-23 (swap Hash and FlowID)
+ * Flex-field 0: Flow ID lower 16-bits
+ * Flex-field 1: Flow ID upper 16-bits
+ * Flex-field 2: RSS hash lower 16-bits
+ * Flex-field 3: RSS hash upper 16-bits
+ * Flex-field 4: AUX0
+ * Flex-field 5: AUX1
+ */
+struct iavf_32b_rx_flex_desc_comms_ovs {
+	/* Qword 0 */
+	u8 rxdid;
+	u8 mir_id_umb_cast;
+	__le16 ptype_flexi_flags0;
+	__le16 pkt_len;
+	__le16 hdr_len_sph_flex_flags1;
+
+	/* Qword 1 */
+	__le16 status_error0;
+	__le16 l2tag1;
+	__le32 flow_id;
+
+	/* Qword 2 */
+	__le16 status_error1;
+	u8 flexi_flags2;
+	u8 ts_low;
+	__le16 l2tag2_1st;
+	__le16 l2tag2_2nd;
+
+	/* Qword 3 */
+	__le32 rss_hash;
+	union {
+		struct {
+			__le16 aux0;
+			__le16 aux1;
+		} flex;
+		__le32 ts_high;
+	} flex_ts;
+};
+
+/* Receive Flex Descriptor profile IDs: There are a total
+ * of 64 profiles where profile IDs 0/1 are for legacy; and
+ * profiles 2-63 are flex profiles that can be programmed
+ * with a specific metadata (profile 7 reserved for HW)
+ */
+enum iavf_rxdid {
+	IAVF_RXDID_LEGACY_0		= 0,
+	IAVF_RXDID_LEGACY_1		= 1,
+	IAVF_RXDID_FLEX_NIC		= 2,
+	IAVF_RXDID_FLEX_NIC_2		= 6,
+	IAVF_RXDID_HW			= 7,
+	IAVF_RXDID_COMMS_GENERIC	= 16,
+	IAVF_RXDID_COMMS_AUX_VLAN	= 17,
+	IAVF_RXDID_COMMS_AUX_IPV4	= 18,
+	IAVF_RXDID_COMMS_AUX_IPV6	= 19,
+	IAVF_RXDID_COMMS_AUX_IPV6_FLOW	= 20,
+	IAVF_RXDID_COMMS_AUX_TCP	= 21,
+	IAVF_RXDID_COMMS_OVS_1		= 22,
+	IAVF_RXDID_COMMS_OVS_2		= 23,
+	IAVF_RXDID_LAST			= 63,
+};
+
+enum iavf_rx_flex_desc_status_error_0_bits {
+	/* Note: These are predefined bit offsets */
+	IAVF_RX_FLEX_DESC_STATUS0_DD_S = 0,
+	IAVF_RX_FLEX_DESC_STATUS0_EOF_S,
+	IAVF_RX_FLEX_DESC_STATUS0_HBO_S,
+	IAVF_RX_FLEX_DESC_STATUS0_L3L4P_S,
+	IAVF_RX_FLEX_DESC_STATUS0_XSUM_IPE_S,
+	IAVF_RX_FLEX_DESC_STATUS0_XSUM_L4E_S,
+	IAVF_RX_FLEX_DESC_STATUS0_XSUM_EIPE_S,
+	IAVF_RX_FLEX_DESC_STATUS0_XSUM_EUDPE_S,
+	IAVF_RX_FLEX_DESC_STATUS0_LPBK_S,
+	IAVF_RX_FLEX_DESC_STATUS0_IPV6EXADD_S,
+	IAVF_RX_FLEX_DESC_STATUS0_RXE_S,
+	IAVF_RX_FLEX_DESC_STATUS0_CRCP_S,
+	IAVF_RX_FLEX_DESC_STATUS0_RSS_VALID_S,
+	IAVF_RX_FLEX_DESC_STATUS0_L2TAG1P_S,
+	IAVF_RX_FLEX_DESC_STATUS0_XTRMD0_VALID_S,
+	IAVF_RX_FLEX_DESC_STATUS0_XTRMD1_VALID_S,
+	IAVF_RX_FLEX_DESC_STATUS0_LAST /* this entry must be last!!! */
+};
+
+/* for iavf_32b_rx_flex_desc.ptype_flex_flags0 member */
+#define IAVF_RX_FLEX_DESC_PTYPE_M	(0x3FF) /* 10-bits */
+
+/* for iavf_32b_rx_flex_desc.pkt_len member */
+#define IAVF_RX_FLX_DESC_PKT_LEN_M	(0x3FFF) /* 14-bits */
+
 int iavf_dev_rx_queue_setup(struct rte_eth_dev *dev,
 			   uint16_t queue_idx,
 			   uint16_t nb_desc,
Signature · 1 line
2.17.1
author
#69 lore raw

PATCH v6 2/11 net/iavf: return error if opcode is mismatched

In reply to Leyi Rong

Adds error return when the opcode of read message is mismatched which is received from adminQ.

---
 drivers/net/iavf/iavf_vchnl.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
drivers/net/iavf/iavf_vchnl.c +3 -1
index fa4da3a6d..b7fb05d32 100644
@@ -52,9 +52,11 @@ iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len,
 	PMD_DRV_LOG(DEBUG, "AQ from pf carries opcode %u, retval %d",
 		    opcode, vf->cmd_retval);
 
-	if (opcode != vf->pend_cmd)
+	if (opcode != vf->pend_cmd) {
 		PMD_DRV_LOG(WARNING, "command mismatch, expect %u, get %u",
 			    vf->pend_cmd, opcode);
+		return IAVF_ERR_OPCODE_MISMATCH;
+	}
 
 	return IAVF_SUCCESS;
 }
Signature · 1 line
2.17.1
author
#70 lore raw

PATCH v6 3/11 net/iavf: flexible Rx descriptor support in normal path

In reply to Leyi Rong

Support flexible Rx descriptor format in normal
path of iAVF PMD.

---
 drivers/net/iavf/iavf.h        |   2 +
 drivers/net/iavf/iavf_ethdev.c |   8 +
 drivers/net/iavf/iavf_rxtx.c   | 485 ++++++++++++++++++++++++++++++---
 drivers/net/iavf/iavf_rxtx.h   |   9 +
 drivers/net/iavf/iavf_vchnl.c  |  55 +++-
 5 files changed, 521 insertions(+), 38 deletions(-)
drivers/net/iavf/iavf.h +2 -0
index 368ddf604..fdb31b929 100644
@@ -103,6 +103,7 @@ struct iavf_info {
 	struct virtchnl_version_info virtchnl_version;
 	struct virtchnl_vf_resource *vf_res; /* VF resource */
 	struct virtchnl_vsi_resource *vsi_res; /* LAN VSI */
+	uint64_t supported_rxdid;
 
 	volatile enum virtchnl_ops pend_cmd; /* pending command not finished */
 	uint32_t cmd_retval; /* return value of the cmd response from PF */
@@ -235,6 +236,7 @@ int iavf_disable_queues(struct iavf_adapter *adapter);
 int iavf_configure_rss_lut(struct iavf_adapter *adapter);
 int iavf_configure_rss_key(struct iavf_adapter *adapter);
 int iavf_configure_queues(struct iavf_adapter *adapter);
+int iavf_get_supported_rxdid(struct iavf_adapter *adapter);
 int iavf_config_irq_map(struct iavf_adapter *adapter);
 void iavf_add_del_all_mac_addr(struct iavf_adapter *adapter, bool add);
 int iavf_dev_link_update(struct rte_eth_dev *dev,
drivers/net/iavf/iavf_ethdev.c +8 -0
index 9dadee3d6..4280a92bb 100644
@@ -1245,6 +1245,14 @@ iavf_init_vf(struct rte_eth_dev *dev)
 			goto err_rss;
 		}
 	}
+
+	if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) {
+		if (iavf_get_supported_rxdid(adapter) != 0) {
+			PMD_INIT_LOG(ERR, "failed to do get supported rxdid");
+			goto err_rss;
+		}
+	}
+
 	return 0;
 err_rss:
 	rte_free(vf->rss_key);
drivers/net/iavf/iavf_rxtx.c +448 -37

552 lines not on this page · raw message

drivers/net/iavf/iavf_rxtx.h +9 -0
index d04233bf8..8246e797f 100644
@@ -60,8 +60,10 @@
 /* HW desc structure, both 16-byte and 32-byte types are supported */
 #ifdef RTE_LIBRTE_IAVF_16BYTE_RX_DESC
 #define iavf_rx_desc iavf_16byte_rx_desc
+#define iavf_rx_flex_desc iavf_16b_rx_flex_desc
 #else
 #define iavf_rx_desc iavf_32byte_rx_desc
+#define iavf_rx_flex_desc iavf_32b_rx_flex_desc
 #endif
 
 struct iavf_rxq_ops {
@@ -87,6 +89,7 @@ struct iavf_rx_queue {
 	struct rte_mbuf *pkt_first_seg; /* first segment of current packet */
 	struct rte_mbuf *pkt_last_seg;  /* last segment of current packet */
 	struct rte_mbuf fake_mbuf;      /* dummy mbuf */
+	uint8_t rxdid;
 
 	/* used for VPMD */
 	uint16_t rxrearm_nb;       /* number of remaining to be re-armed */
@@ -379,9 +382,15 @@ void iavf_dev_tx_queue_release(void *txq);
 void iavf_stop_queues(struct rte_eth_dev *dev);
 uint16_t iavf_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 		       uint16_t nb_pkts);
+uint16_t iavf_recv_pkts_flex_rxd(void *rx_queue,
+				 struct rte_mbuf **rx_pkts,
+				 uint16_t nb_pkts);
 uint16_t iavf_recv_scattered_pkts(void *rx_queue,
 				 struct rte_mbuf **rx_pkts,
 				 uint16_t nb_pkts);
+uint16_t iavf_recv_scattered_pkts_flex_rxd(void *rx_queue,
+					   struct rte_mbuf **rx_pkts,
+					   uint16_t nb_pkts);
 uint16_t iavf_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 		       uint16_t nb_pkts);
 uint16_t iavf_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
drivers/net/iavf/iavf_vchnl.c +54 -1

79 lines not on this page · raw message

Signature · 1 line
2.17.1
author
#71 lore raw

PATCH v6 4/11 net/iavf: flexible Rx descriptor support in AVX path

In reply to Leyi Rong

Support flexible Rx descriptor format in AVX
path of iAVF PMD.

---
 drivers/net/iavf/iavf_rxtx.c          |  24 +-
 drivers/net/iavf/iavf_rxtx.h          |   6 +
 drivers/net/iavf/iavf_rxtx_vec_avx2.c | 539 ++++++++++++++++++++++++++
 3 files changed, 563 insertions(+), 6 deletions(-)
drivers/net/iavf/iavf_rxtx.c +18 -6
index 2eac1de2b..8a1292823 100644
@@ -2087,16 +2087,28 @@ iavf_set_rx_function(struct rte_eth_dev *dev)
 				    "Using %sVector Scattered Rx (port %d).",
 				    use_avx2 ? "avx2 " : "",
 				    dev->data->port_id);
-			dev->rx_pkt_burst = use_avx2 ?
-					    iavf_recv_scattered_pkts_vec_avx2 :
-					    iavf_recv_scattered_pkts_vec;
+			if (vf->vf_res->vf_cap_flags &
+				VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC)
+				dev->rx_pkt_burst = use_avx2 ?
+					iavf_recv_scattered_pkts_vec_avx2_flex_rxd :
+					iavf_recv_scattered_pkts_vec;
+			else
+				dev->rx_pkt_burst = use_avx2 ?
+					iavf_recv_scattered_pkts_vec_avx2 :
+					iavf_recv_scattered_pkts_vec;
 		} else {
 			PMD_DRV_LOG(DEBUG, "Using %sVector Rx (port %d).",
 				    use_avx2 ? "avx2 " : "",
 				    dev->data->port_id);
-			dev->rx_pkt_burst = use_avx2 ?
-					    iavf_recv_pkts_vec_avx2 :
-					    iavf_recv_pkts_vec;
+			if (vf->vf_res->vf_cap_flags &
+				VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC)
+				dev->rx_pkt_burst = use_avx2 ?
+					iavf_recv_pkts_vec_avx2_flex_rxd :
+					iavf_recv_pkts_vec;
+			else
+				dev->rx_pkt_burst = use_avx2 ?
+					iavf_recv_pkts_vec_avx2 :
+					iavf_recv_pkts_vec;
 		}
 
 		return;
drivers/net/iavf/iavf_rxtx.h +6 -0
index 8246e797f..84ec39132 100644
@@ -414,9 +414,15 @@ uint16_t iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
 				  uint16_t nb_pkts);
 uint16_t iavf_recv_pkts_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,
 				 uint16_t nb_pkts);
+uint16_t iavf_recv_pkts_vec_avx2_flex_rxd(void *rx_queue,
+					  struct rte_mbuf **rx_pkts,
+					  uint16_t nb_pkts);
 uint16_t iavf_recv_scattered_pkts_vec_avx2(void *rx_queue,
 					   struct rte_mbuf **rx_pkts,
 					   uint16_t nb_pkts);
+uint16_t iavf_recv_scattered_pkts_vec_avx2_flex_rxd(void *rx_queue,
+						    struct rte_mbuf **rx_pkts,
+						    uint16_t nb_pkts);
 uint16_t iavf_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
 			    uint16_t nb_pkts);
 uint16_t iavf_xmit_pkts_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts,
drivers/net/iavf/iavf_rxtx_vec_avx2.c +539 -0

557 lines not on this page · raw message

Signature · 1 line
2.17.1
author
#72 lore raw

PATCH v6 5/11 net/iavf: flexible Rx descriptor support in SSE path

In reply to Leyi Rong

Support flexible Rx descriptor format in SSE
path of iAVF PMD.

---
 drivers/net/iavf/iavf_rxtx.c         |   4 +-
 drivers/net/iavf/iavf_rxtx.h         |   5 +
 drivers/net/iavf/iavf_rxtx_vec_sse.c | 414 +++++++++++++++++++++++++++
 3 files changed, 421 insertions(+), 2 deletions(-)
drivers/net/iavf/iavf_rxtx.c +2 -2
index 8a1292823..92bf4cf8b 100644
@@ -2091,7 +2091,7 @@ iavf_set_rx_function(struct rte_eth_dev *dev)
 				VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC)
 				dev->rx_pkt_burst = use_avx2 ?
 					iavf_recv_scattered_pkts_vec_avx2_flex_rxd :
-					iavf_recv_scattered_pkts_vec;
+					iavf_recv_scattered_pkts_vec_flex_rxd;
 			else
 				dev->rx_pkt_burst = use_avx2 ?
 					iavf_recv_scattered_pkts_vec_avx2 :
@@ -2104,7 +2104,7 @@ iavf_set_rx_function(struct rte_eth_dev *dev)
 				VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC)
 				dev->rx_pkt_burst = use_avx2 ?
 					iavf_recv_pkts_vec_avx2_flex_rxd :
-					iavf_recv_pkts_vec;
+					iavf_recv_pkts_vec_flex_rxd;
 			else
 				dev->rx_pkt_burst = use_avx2 ?
 					iavf_recv_pkts_vec_avx2 :
drivers/net/iavf/iavf_rxtx.h +5 -0
index 84ec39132..9bc857312 100644
@@ -407,9 +407,14 @@ int iavf_dev_tx_desc_status(void *tx_queue, uint16_t offset);
 
 uint16_t iavf_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
 			   uint16_t nb_pkts);
+uint16_t iavf_recv_pkts_vec_flex_rxd(void *rx_queue, struct rte_mbuf **rx_pkts,
+				     uint16_t nb_pkts);
 uint16_t iavf_recv_scattered_pkts_vec(void *rx_queue,
 				     struct rte_mbuf **rx_pkts,
 				     uint16_t nb_pkts);
+uint16_t iavf_recv_scattered_pkts_vec_flex_rxd(void *rx_queue,
+					       struct rte_mbuf **rx_pkts,
+					       uint16_t nb_pkts);
 uint16_t iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
 				  uint16_t nb_pkts);
 uint16_t iavf_recv_pkts_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,
drivers/net/iavf/iavf_rxtx_vec_sse.c +414 -0

444 lines not on this page · raw message

Signature · 1 line
2.17.1
author
#73 lore raw

PATCH v6 6/11 net/iavf: add flow director enabled switch value

In reply to Leyi Rong

The commit adds fdir_enabled flag into iavf_rx_queue structure to identify if fdir id is active. Rx data path can be benefit if fdir id parsing is not needed, especially in vector path.

---
 drivers/net/iavf/iavf.h      |  1 +
 drivers/net/iavf/iavf_rxtx.h | 30 ++++++++++++++++++++++++++++++
 2 files changed, 31 insertions(+)
drivers/net/iavf/iavf.h +1 -0
index fdb31b929..17ceff734 100644
@@ -144,6 +144,7 @@ struct iavf_adapter {
 	bool tx_vec_allowed;
 	const uint32_t *ptype_tbl;
 	bool stopped;
+	uint16_t fdir_ref_cnt;
 };
 
 /* IAVF_DEV_PRIVATE_TO */
drivers/net/iavf/iavf_rxtx.h +30 -0
index 9bc857312..73968847f 100644
@@ -104,6 +104,7 @@ struct iavf_rx_queue {
 
 	uint16_t port_id;        /* device port ID */
 	uint8_t crc_len;        /* 0 if CRC stripped, 4 otherwise */
+	uint8_t fdir_enabled;   /* 0 if FDIR disabled, 1 when enabled */
 	uint16_t queue_id;      /* Rx queue index */
 	uint16_t rx_buf_len;    /* The packet buffer size */
 	uint16_t rx_hdr_len;    /* The header buffer size */
@@ -491,6 +492,35 @@ void iavf_dump_tx_descriptor(const struct iavf_tx_queue *txq,
 	       tx_desc->cmd_type_offset_bsz);
 }
 
+#define FDIR_PROC_ENABLE_PER_QUEUE(ad, on) do { \
+	int i; \
+	for (i = 0; i < (ad)->eth_dev->data->nb_rx_queues; i++) { \
+		struct iavf_rx_queue *rxq = (ad)->eth_dev->data->rx_queues[i]; \
+		if (!rxq) \
+			continue; \
+		rxq->fdir_enabled = on; \
+	} \
+	PMD_DRV_LOG(DEBUG, "FDIR processing on RX set to %d", on); \
+} while (0)
+
+/* Enable/disable flow director Rx processing in data path. */
+static inline
+void iavf_fdir_rx_proc_enable(struct iavf_adapter *ad, bool on)
+{
+	if (on) {
+		/* enable flow director processing */
+		if (ad->fdir_ref_cnt++ == 0)
+			FDIR_PROC_ENABLE_PER_QUEUE(ad, on);
+	} else {
+		if (ad->fdir_ref_cnt >= 1) {
+			ad->fdir_ref_cnt--;
+
+			if (ad->fdir_ref_cnt == 0)
+				FDIR_PROC_ENABLE_PER_QUEUE(ad, on);
+		}
+	}
+}
+
 #ifdef RTE_LIBRTE_IAVF_DEBUG_DUMP_DESC
 #define IAVF_DUMP_RX_DESC(rxq, desc, rx_id) \
 	iavf_dump_rx_descriptor(rxq, desc, rx_id)
Signature · 1 line
2.17.1
author
#74 lore raw

PATCH v6 7/11 net/iavf: support flow mark in normal data path

In reply to Leyi Rong

Support Flow Director mark ID parsing in normal path.

---
 drivers/net/iavf/iavf.h      |  3 +++
 drivers/net/iavf/iavf_rxtx.c | 51 +++++++++++++++++++++++++++++++-----
 2 files changed, 47 insertions(+), 7 deletions(-)
drivers/net/iavf/iavf.h +3 -0
index 17ceff734..78bdaff20 100644
@@ -67,6 +67,9 @@
 #define IAVF_48_BIT_WIDTH (CHAR_BIT * 6)
 #define IAVF_48_BIT_MASK  RTE_LEN2MASK(IAVF_48_BIT_WIDTH, uint64_t)
 
+#define IAVF_RX_DESC_EXT_STATUS_FLEXBH_MASK  0x03
+#define IAVF_RX_DESC_EXT_STATUS_FLEXBH_FD_ID 0x01
+
 struct iavf_adapter;
 struct iavf_rx_queue;
 struct iavf_tx_queue;
drivers/net/iavf/iavf_rxtx.c +44 -7

117 lines not on this page · raw message

Signature · 1 line
2.17.1
author
#75 lore raw

PATCH v6 8/11 net/iavf: support flow mark in AVX path

In reply to Leyi Rong

Support Flow Director mark ID parsing from Flex
Rx descriptor in AVX path.

---
 drivers/net/iavf/iavf_rxtx_vec_avx2.c | 72 +++++++++++++++++++++++++--
 1 file changed, 67 insertions(+), 5 deletions(-)
drivers/net/iavf/iavf_rxtx_vec_avx2.c +67 -5
index 04603e7d4..9f467d4fc 100644
@@ -614,6 +614,25 @@ _iavf_recv_raw_pkts_vec_avx2(struct iavf_rx_queue *rxq,
 	return received;
 }
 
+static inline __m256i
+flex_rxd_to_fdir_flags_vec_avx2(const __m256i fdir_id0_7)
+{
+#define FDID_MIS_MAGIC 0xFFFFFFFF
+	RTE_BUILD_BUG_ON(PKT_RX_FDIR != (1 << 2));
+	RTE_BUILD_BUG_ON(PKT_RX_FDIR_ID != (1 << 13));
+	const __m256i pkt_fdir_bit = _mm256_set1_epi32(PKT_RX_FDIR |
+			PKT_RX_FDIR_ID);
+	/* desc->flow_id field == 0xFFFFFFFF means fdir mismatch */
+	const __m256i fdir_mis_mask = _mm256_set1_epi32(FDID_MIS_MAGIC);
+	__m256i fdir_mask = _mm256_cmpeq_epi32(fdir_id0_7,
+			fdir_mis_mask);
+	/* this XOR op results to bit-reverse the fdir_mask */
+	fdir_mask = _mm256_xor_si256(fdir_mask, fdir_mis_mask);
+	const __m256i fdir_flags = _mm256_and_si256(fdir_mask, pkt_fdir_bit);
+
+	return fdir_flags;
+}
+
 static inline uint16_t
 _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 				      struct rte_mbuf **rx_pkts,
@@ -675,8 +694,8 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 	const __m256i shuf_msk =
 		_mm256_set_epi8
 			(/* first descriptor */
-			 15, 14,
-			 13, 12,	/* octet 12~15, 32 bits rss */
+			 0xFF, 0xFF,
+			 0xFF, 0xFF,    /* rss not supported */
 			 11, 10,	/* octet 10~11, 16 bits vlan_macip */
 			 5, 4,		/* octet 4~5, 16 bits data_len */
 			 0xFF, 0xFF,	/* skip hi 16 bits pkt_len, zero out */
@@ -684,8 +703,8 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 			 0xFF, 0xFF,	/* pkt_type set as unknown */
 			 0xFF, 0xFF,	/*pkt_type set as unknown */
 			 /* second descriptor */
-			 15, 14,
-			 13, 12,	/* octet 12~15, 32 bits rss */
+			 0xFF, 0xFF,
+			 0xFF, 0xFF,    /* rss not supported */
 			 11, 10,	/* octet 10~11, 16 bits vlan_macip */
 			 5, 4,		/* octet 4~5, 16 bits data_len */
 			 0xFF, 0xFF,	/* skip hi 16 bits pkt_len, zero out */
@@ -927,8 +946,51 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 					    rss_vlan_flag_bits);
 
 		/* merge flags */
-		const __m256i mbuf_flags = _mm256_or_si256(l3_l4_flags,
+		__m256i mbuf_flags = _mm256_or_si256(l3_l4_flags,
 				rss_vlan_flags);
+
+		if (rxq->fdir_enabled) {
+			const __m256i fdir_id4_7 =
+				_mm256_unpackhi_epi32(raw_desc6_7, raw_desc4_5);
+
+			const __m256i fdir_id0_3 =
+				_mm256_unpackhi_epi32(raw_desc2_3, raw_desc0_1);
+
+			const __m256i fdir_id0_7 =
+				_mm256_unpackhi_epi64(fdir_id4_7, fdir_id0_3);
+
+			const __m256i fdir_flags =
+				flex_rxd_to_fdir_flags_vec_avx2(fdir_id0_7);
+
+			/* merge with fdir_flags */
+			mbuf_flags = _mm256_or_si256(mbuf_flags, fdir_flags);
+
+			/* write to mbuf: have to use scalar store here */
+			rx_pkts[i + 0]->hash.fdir.hi =
+				_mm256_extract_epi32(fdir_id0_7, 3);
+
+			rx_pkts[i + 1]->hash.fdir.hi =
+				_mm256_extract_epi32(fdir_id0_7, 7);
+
+			rx_pkts[i + 2]->hash.fdir.hi =
+				_mm256_extract_epi32(fdir_id0_7, 2);
+
+			rx_pkts[i + 3]->hash.fdir.hi =
+				_mm256_extract_epi32(fdir_id0_7, 6);
+
+			rx_pkts[i + 4]->hash.fdir.hi =
+				_mm256_extract_epi32(fdir_id0_7, 1);
+
+			rx_pkts[i + 5]->hash.fdir.hi =
+				_mm256_extract_epi32(fdir_id0_7, 5);
+
+			rx_pkts[i + 6]->hash.fdir.hi =
+				_mm256_extract_epi32(fdir_id0_7, 0);
+
+			rx_pkts[i + 7]->hash.fdir.hi =
+				_mm256_extract_epi32(fdir_id0_7, 4);
+		} /* if() on fdir_enabled */
+
 		/**
 		 * At this point, we have the 8 sets of flags in the low 16-bits
 		 * of each 32-bit value in vlan0.
Signature · 1 line
2.17.1
author
#76 lore raw

PATCH v6 9/11 net/iavf: support flow mark in SSE path

In reply to Leyi Rong

Support Flow Director mark ID parsing from Flex
Rx descriptor in SSE path.

---
 drivers/net/iavf/iavf_rxtx_vec_sse.c | 52 +++++++++++++++++++++++++++-
 1 file changed, 51 insertions(+), 1 deletion(-)
drivers/net/iavf/iavf_rxtx_vec_sse.c +51 -1
index 9c1f2a445..d7d840853 100644
@@ -189,6 +189,25 @@ desc_to_olflags_v(struct iavf_rx_queue *rxq, __m128i descs[4],
 	_mm_store_si128((__m128i *)&rx_pkts[3]->rearm_data, rearm3);
 }
 
+static inline __m128i
+flex_rxd_to_fdir_flags_vec(const __m128i fdir_id0_3)
+{
+#define FDID_MIS_MAGIC 0xFFFFFFFF
+	RTE_BUILD_BUG_ON(PKT_RX_FDIR != (1 << 2));
+	RTE_BUILD_BUG_ON(PKT_RX_FDIR_ID != (1 << 13));
+	const __m128i pkt_fdir_bit = _mm_set1_epi32(PKT_RX_FDIR |
+			PKT_RX_FDIR_ID);
+	/* desc->flow_id field == 0xFFFFFFFF means fdir mismatch */
+	const __m128i fdir_mis_mask = _mm_set1_epi32(FDID_MIS_MAGIC);
+	__m128i fdir_mask = _mm_cmpeq_epi32(fdir_id0_3,
+			fdir_mis_mask);
+	/* this XOR op results to bit-reverse the fdir_mask */
+	fdir_mask = _mm_xor_si128(fdir_mask, fdir_mis_mask);
+	const __m128i fdir_flags = _mm_and_si128(fdir_mask, pkt_fdir_bit);
+
+	return fdir_flags;
+}
+
 static inline void
 flex_desc_to_olflags_v(struct iavf_rx_queue *rxq, __m128i descs[4],
 		       struct rte_mbuf **rx_pkts)
@@ -267,6 +286,36 @@ flex_desc_to_olflags_v(struct iavf_rx_queue *rxq, __m128i descs[4],
 	/* merge the flags */
 	flags = _mm_or_si128(flags, rss_vlan);
 
+	if (rxq->fdir_enabled) {
+		const __m128i fdir_id0_1 =
+			_mm_unpackhi_epi32(descs[0], descs[1]);
+
+		const __m128i fdir_id2_3 =
+			_mm_unpackhi_epi32(descs[2], descs[3]);
+
+		const __m128i fdir_id0_3 =
+			_mm_unpackhi_epi64(fdir_id0_1, fdir_id2_3);
+
+		const __m128i fdir_flags =
+			flex_rxd_to_fdir_flags_vec(fdir_id0_3);
+
+		/* merge with fdir_flags */
+		flags = _mm_or_si128(flags, fdir_flags);
+
+		/* write fdir_id to mbuf */
+		rx_pkts[0]->hash.fdir.hi =
+			_mm_extract_epi32(fdir_id0_3, 0);
+
+		rx_pkts[1]->hash.fdir.hi =
+			_mm_extract_epi32(fdir_id0_3, 1);
+
+		rx_pkts[2]->hash.fdir.hi =
+			_mm_extract_epi32(fdir_id0_3, 2);
+
+		rx_pkts[3]->hash.fdir.hi =
+			_mm_extract_epi32(fdir_id0_3, 3);
+	} /* if() on fdir_enabled */
+
 	/**
 	 * At this point, we have the 4 sets of flags in the low 16-bits
 	 * of each 32-bit value in flags.
@@ -604,7 +653,8 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 	const __m128i zero = _mm_setzero_si128();
 	/* mask to shuffle from desc. to mbuf */
 	const __m128i shuf_msk = _mm_set_epi8
-			(15, 14, 13, 12,  /* octet 12~15, 32 bits rss */
+			(0xFF, 0xFF,
+			 0xFF, 0xFF,  /* rss not supported */
 			 11, 10,      /* octet 10~11, 16 bits vlan_macip */
 			 5, 4,        /* octet 4~5, 16 bits data_len */
 			 0xFF, 0xFF,  /* skip high 16 bits pkt_len, zero out */
Signature · 1 line
2.17.1
author
#77 lore raw

PATCH v6 10/11 net/iavf: add RSS hash parsing in AVX path

In reply to Leyi Rong

Support RSS hash parsing from Flex Rx
descriptor in AVX data path.

---
 drivers/net/iavf/iavf_rxtx_vec_avx2.c | 94 ++++++++++++++++++++++++++-
 1 file changed, 92 insertions(+), 2 deletions(-)
drivers/net/iavf/iavf_rxtx_vec_avx2.c +92 -2
index 9f467d4fc..e5e0fd309 100644
@@ -695,7 +695,7 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 		_mm256_set_epi8
 			(/* first descriptor */
 			 0xFF, 0xFF,
-			 0xFF, 0xFF,    /* rss not supported */
+			 0xFF, 0xFF,    /* rss hash parsed separately */
 			 11, 10,	/* octet 10~11, 16 bits vlan_macip */
 			 5, 4,		/* octet 4~5, 16 bits data_len */
 			 0xFF, 0xFF,	/* skip hi 16 bits pkt_len, zero out */
@@ -704,7 +704,7 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 			 0xFF, 0xFF,	/*pkt_type set as unknown */
 			 /* second descriptor */
 			 0xFF, 0xFF,
-			 0xFF, 0xFF,    /* rss not supported */
+			 0xFF, 0xFF,    /* rss hash parsed separately */
 			 11, 10,	/* octet 10~11, 16 bits vlan_macip */
 			 5, 4,		/* octet 4~5, 16 bits data_len */
 			 0xFF, 0xFF,	/* skip hi 16 bits pkt_len, zero out */
@@ -991,6 +991,96 @@ _iavf_recv_raw_pkts_vec_avx2_flex_rxd(struct iavf_rx_queue *rxq,
 				_mm256_extract_epi32(fdir_id0_7, 4);
 		} /* if() on fdir_enabled */
 
+#ifndef RTE_LIBRTE_IAVF_16BYTE_RX_DESC
+		/**
+		 * needs to load 2nd 16B of each desc for RSS hash parsing,
+		 * will cause performance drop to get into this context.
+		 */
+		if (rxq->vsi->adapter->eth_dev->data->dev_conf.rxmode.offloads &
+				DEV_RX_OFFLOAD_RSS_HASH) {
+			/* load bottom half of every 32B desc */
+			const __m128i raw_desc_bh7 =
+				_mm_load_si128
+					((void *)(&rxdp[7].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh6 =
+				_mm_load_si128
+					((void *)(&rxdp[6].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh5 =
+				_mm_load_si128
+					((void *)(&rxdp[5].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh4 =
+				_mm_load_si128
+					((void *)(&rxdp[4].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh3 =
+				_mm_load_si128
+					((void *)(&rxdp[3].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh2 =
+				_mm_load_si128
+					((void *)(&rxdp[2].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh1 =
+				_mm_load_si128
+					((void *)(&rxdp[1].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh0 =
+				_mm_load_si128
+					((void *)(&rxdp[0].wb.status_error1));
+
+			__m256i raw_desc_bh6_7 =
+				_mm256_inserti128_si256
+					(_mm256_castsi128_si256(raw_desc_bh6),
+					raw_desc_bh7, 1);
+			__m256i raw_desc_bh4_5 =
+				_mm256_inserti128_si256
+					(_mm256_castsi128_si256(raw_desc_bh4),
+					raw_desc_bh5, 1);
+			__m256i raw_desc_bh2_3 =
+				_mm256_inserti128_si256
+					(_mm256_castsi128_si256(raw_desc_bh2),
+					raw_desc_bh3, 1);
+			__m256i raw_desc_bh0_1 =
+				_mm256_inserti128_si256
+					(_mm256_castsi128_si256(raw_desc_bh0),
+					raw_desc_bh1, 1);
+
+			/**
+			 * to shift the 32b RSS hash value to the
+			 * highest 32b of each 128b before mask
+			 */
+			__m256i rss_hash6_7 =
+				_mm256_slli_epi64(raw_desc_bh6_7, 32);
+			__m256i rss_hash4_5 =
+				_mm256_slli_epi64(raw_desc_bh4_5, 32);
+			__m256i rss_hash2_3 =
+				_mm256_slli_epi64(raw_desc_bh2_3, 32);
+			__m256i rss_hash0_1 =
+				_mm256_slli_epi64(raw_desc_bh0_1, 32);
+
+			__m256i rss_hash_msk =
+				_mm256_set_epi32(0xFFFFFFFF, 0, 0, 0,
+						 0xFFFFFFFF, 0, 0, 0);
+
+			rss_hash6_7 = _mm256_and_si256
+					(rss_hash6_7, rss_hash_msk);
+			rss_hash4_5 = _mm256_and_si256
+					(rss_hash4_5, rss_hash_msk);
+			rss_hash2_3 = _mm256_and_si256
+					(rss_hash2_3, rss_hash_msk);
+			rss_hash0_1 = _mm256_and_si256
+					(rss_hash0_1, rss_hash_msk);
+
+			mb6_7 = _mm256_or_si256(mb6_7, rss_hash6_7);
+			mb4_5 = _mm256_or_si256(mb4_5, rss_hash4_5);
+			mb2_3 = _mm256_or_si256(mb2_3, rss_hash2_3);
+			mb0_1 = _mm256_or_si256(mb0_1, rss_hash0_1);
+		} /* if() on RSS hash parsing */
+#endif
+
 		/**
 		 * At this point, we have the 8 sets of flags in the low 16-bits
 		 * of each 32-bit value in vlan0.
Signature · 1 line
2.17.1
author
#78 lore raw

PATCH v6 11/11 net/iavf: add RSS hash parsing in SSE path

In reply to Leyi Rong

Support RSS hash parsing from Flex Rx
descriptor in SSE data path.

---
 drivers/net/iavf/iavf_rxtx_vec_sse.c | 88 +++++++++++++++++++++++-----
 1 file changed, 72 insertions(+), 16 deletions(-)
drivers/net/iavf/iavf_rxtx_vec_sse.c +72 -16
index d7d840853..2f115aa94 100644
@@ -654,7 +654,7 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 	/* mask to shuffle from desc. to mbuf */
 	const __m128i shuf_msk = _mm_set_epi8
 			(0xFF, 0xFF,
-			 0xFF, 0xFF,  /* rss not supported */
+			 0xFF, 0xFF,  /* rss hash parsed separately */
 			 11, 10,      /* octet 10~11, 16 bits vlan_macip */
 			 5, 4,        /* octet 4~5, 16 bits data_len */
 			 0xFF, 0xFF,  /* skip high 16 bits pkt_len, zero out */
@@ -745,7 +745,7 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 	     pos += IAVF_VPMD_DESCS_PER_LOOP,
 	     rxdp += IAVF_VPMD_DESCS_PER_LOOP) {
 		__m128i descs[IAVF_VPMD_DESCS_PER_LOOP];
-		__m128i pkt_mb1, pkt_mb2, pkt_mb3, pkt_mb4;
+		__m128i pkt_mb0, pkt_mb1, pkt_mb2, pkt_mb3;
 		__m128i staterr, sterr_tmp1, sterr_tmp2;
 		/* 2 64 bit or 4 32 bit mbuf pointers in one XMM reg. */
 		__m128i mbp1;
@@ -791,8 +791,12 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 		rte_compiler_barrier();
 
 		/* D.1 pkt 3,4 convert format from desc to pktmbuf */
-		pkt_mb4 = _mm_shuffle_epi8(descs[3], shuf_msk);
-		pkt_mb3 = _mm_shuffle_epi8(descs[2], shuf_msk);
+		pkt_mb3 = _mm_shuffle_epi8(descs[3], shuf_msk);
+		pkt_mb2 = _mm_shuffle_epi8(descs[2], shuf_msk);
+
+		/* D.1 pkt 1,2 convert format from desc to pktmbuf */
+		pkt_mb1 = _mm_shuffle_epi8(descs[1], shuf_msk);
+		pkt_mb0 = _mm_shuffle_epi8(descs[0], shuf_msk);
 
 		/* C.1 4=>2 filter staterr info only */
 		sterr_tmp2 = _mm_unpackhi_epi32(descs[3], descs[2]);
@@ -802,12 +806,68 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 		flex_desc_to_olflags_v(rxq, descs, &rx_pkts[pos]);
 
 		/* D.2 pkt 3,4 set in_port/nb_seg and remove crc */
-		pkt_mb4 = _mm_add_epi16(pkt_mb4, crc_adjust);
 		pkt_mb3 = _mm_add_epi16(pkt_mb3, crc_adjust);
+		pkt_mb2 = _mm_add_epi16(pkt_mb2, crc_adjust);
 
-		/* D.1 pkt 1,2 convert format from desc to pktmbuf */
-		pkt_mb2 = _mm_shuffle_epi8(descs[1], shuf_msk);
-		pkt_mb1 = _mm_shuffle_epi8(descs[0], shuf_msk);
+		/* D.2 pkt 1,2 set in_port/nb_seg and remove crc */
+		pkt_mb1 = _mm_add_epi16(pkt_mb1, crc_adjust);
+		pkt_mb0 = _mm_add_epi16(pkt_mb0, crc_adjust);
+
+#ifndef RTE_LIBRTE_IAVF_16BYTE_RX_DESC
+		/**
+		 * needs to load 2nd 16B of each desc for RSS hash parsing,
+		 * will cause performance drop to get into this context.
+		 */
+		if (rxq->vsi->adapter->eth_dev->data->dev_conf.rxmode.offloads &
+				DEV_RX_OFFLOAD_RSS_HASH) {
+			/* load bottom half of every 32B desc */
+			const __m128i raw_desc_bh3 =
+				_mm_load_si128
+					((void *)(&rxdp[3].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh2 =
+				_mm_load_si128
+					((void *)(&rxdp[2].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh1 =
+				_mm_load_si128
+					((void *)(&rxdp[1].wb.status_error1));
+			rte_compiler_barrier();
+			const __m128i raw_desc_bh0 =
+				_mm_load_si128
+					((void *)(&rxdp[0].wb.status_error1));
+
+			/**
+			 * to shift the 32b RSS hash value to the
+			 * highest 32b of each 128b before mask
+			 */
+			__m128i rss_hash3 =
+				_mm_slli_epi64(raw_desc_bh3, 32);
+			__m128i rss_hash2 =
+				_mm_slli_epi64(raw_desc_bh2, 32);
+			__m128i rss_hash1 =
+				_mm_slli_epi64(raw_desc_bh1, 32);
+			__m128i rss_hash0 =
+				_mm_slli_epi64(raw_desc_bh0, 32);
+
+			__m128i rss_hash_msk =
+				_mm_set_epi32(0xFFFFFFFF, 0, 0, 0);
+
+			rss_hash3 = _mm_and_si128
+					(rss_hash3, rss_hash_msk);
+			rss_hash2 = _mm_and_si128
+					(rss_hash2, rss_hash_msk);
+			rss_hash1 = _mm_and_si128
+					(rss_hash1, rss_hash_msk);
+			rss_hash0 = _mm_and_si128
+					(rss_hash0, rss_hash_msk);
+
+			pkt_mb3 = _mm_or_si128(pkt_mb3, rss_hash3);
+			pkt_mb2 = _mm_or_si128(pkt_mb2, rss_hash2);
+			pkt_mb1 = _mm_or_si128(pkt_mb1, rss_hash1);
+			pkt_mb0 = _mm_or_si128(pkt_mb0, rss_hash0);
+		} /* if() on RSS hash parsing */
+#endif
 
 		/* C.2 get 4 pkts staterr value  */
 		staterr = _mm_unpacklo_epi32(sterr_tmp1, sterr_tmp2);
@@ -815,14 +875,10 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 		/* D.3 copy final 3,4 data to rx_pkts */
 		_mm_storeu_si128
 			((void *)&rx_pkts[pos + 3]->rx_descriptor_fields1,
-			 pkt_mb4);
+			 pkt_mb3);
 		_mm_storeu_si128
 			((void *)&rx_pkts[pos + 2]->rx_descriptor_fields1,
-			 pkt_mb3);
-
-		/* D.2 pkt 1,2 set in_port/nb_seg and remove crc */
-		pkt_mb2 = _mm_add_epi16(pkt_mb2, crc_adjust);
-		pkt_mb1 = _mm_add_epi16(pkt_mb1, crc_adjust);
+			 pkt_mb2);
 
 		/* C* extract and record EOP bit */
 		if (split_packet) {
@@ -846,9 +902,9 @@ _recv_raw_pkts_vec_flex_rxd(struct iavf_rx_queue *rxq,
 		/* D.3 copy final 1,2 data to rx_pkts */
 		_mm_storeu_si128
 			((void *)&rx_pkts[pos + 1]->rx_descriptor_fields1,
-			 pkt_mb2);
+			 pkt_mb1);
 		_mm_storeu_si128((void *)&rx_pkts[pos]->rx_descriptor_fields1,
-				 pkt_mb1);
+				 pkt_mb0);
 		flex_desc_to_ptype_v(descs, &rx_pkts[pos], ptype_tbl);
 		/* C.4 calc avaialbe number of desc */
 		var = __builtin_popcountll(_mm_cvtsi128_si64(staterr));
Signature · 1 line
2.17.1
Applied
#79 lore raw
In reply to Leyi Rong
Quoted 53 lines · Leyi Rong

This patchset enable framework for advanced iAVF, includes flexible descriptor support, FDIR mark id and RSS hash support.

---
v6:
- Remove some redundant code which may cause ICC build error.

v5:
- Set rxdid = 0 in iavf_configure_queues() when it's supported and RTE_LIBRTE_IAVF_16BYTE_RX_DESC is y. So the error can be captured explicitly as kernel PF does not support it yet.
- Fix some build errors when RTE_LIBRTE_IAVF_16BYTE_RX_DESC is y.

v4:
- Avoid initial declaration in 'for' loop to backward-compatible with older gcc versions.

v3:
- Remove patch to query DDP package info as it's specific to DCF.
- Specify fdir_ref_cnt as per VF value for enable/disable FDIR ID parse.
- Move fdir_enabled flag to per Queue value for cache benefit when access in vector routines.
- Store extracted flow_id value to mbuf without judgement to avoid branch speculation fail, which is benefit to performance.

v2:
- Revert macro RTE_LIBRTE_IAVF_16BYTE_RX_DESC deletion as it's defined in AVF spec.
- Typo RTE_LIBRTE_ICE_16BYTE_RX_DESC fix in iavf_rxtx.c.
- Move flex desc definitions into iavf_rxtx.h.
- Up to date to match with the latest version of virtchnl.h.
- Extract a new internal func iavf_update_rx_tail to call.
- Remove iavf_dev_rxq_count_flex_rxd()/iavf_dev_rx_desc_status_flex_rxd(), as the accompanying legacy ones can deal with the flex desc cases.
- Move rxq->rxdid assignment from iavf_configure_queues() to iavf_dev_rx_queue_setup().
- Unfold _mm_extract_epi32(fdir_id0_3, i) to fix build error when using GCC compile option -O0.

Leyi Rong (11):
net/iavf: flexible Rx descriptor definitions net/iavf: return error if opcode is mismatched net/iavf: flexible Rx descriptor support in normal path net/iavf: flexible Rx descriptor support in AVX path net/iavf: flexible Rx descriptor support in SSE path net/iavf: add flow director enabled switch value net/iavf: support flow mark in normal data path net/iavf: support flow mark in AVX path net/iavf: support flow mark in SSE path net/iavf: add RSS hash parsing in AVX path net/iavf: add RSS hash parsing in SSE path

Quoted 12 lines
drivers/net/iavf/iavf.h               |   6 +
drivers/net/iavf/iavf_ethdev.c        |   8 +
drivers/net/iavf/iavf_rxtx.c          | 546 ++++++++++++++++++--
drivers/net/iavf/iavf_rxtx.h          | 250 ++++++++++
drivers/net/iavf/iavf_rxtx_vec_avx2.c | 691 ++++++++++++++++++++++++++
drivers/net/iavf/iavf_rxtx_vec_sse.c  | 520 +++++++++++++++++++
drivers/net/iavf/iavf_vchnl.c         |  59 ++-
7 files changed, 2035 insertions(+), 45 deletions(-)
Signature · 1 line
2.17.1

Applied to dpdk-next-net-intel, Thanks.