Skip to contentSkip to conversation

net/i40e: improve free mbuf for Tx

PATCH Superseded Feifei Wang 2 participants 5 messages Jun 30, 2021 Network drivers

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

v2 cover net/i40e: improve free mbuf for Tx

For i40e Tx path, use bulk free of the buffers when mbuf fast free mode is enabled. This can efficiently improve the performance.

v2:
1. fix bug when tx_rs_thres > RTE_I40E_TX_MAX_FREE_BUF_SZ (Beilei)

Feifei Wang (2):
net/i40e: improve performance for scalar Tx
net/i40e: improve performance for vector Tx

 drivers/net/i40e/i40e_rxtx.c            | 26 +++++++++++++++++++++----
 drivers/net/i40e/i40e_rxtx_vec_common.h | 11 +++++++++++
 2 files changed, 33 insertions(+), 4 deletions(-)
Signature · 1 line
2.25.1
#1 lore raw

PATCH v2 1/2 net/i40e: improve performance for scalar Tx

For i40e scalar Tx path, if implement FAST_FREE_MBUF mode, it means per-queue all mbufs come from the same mempool and have refcnt = 1.

Thus we can use bulk free of the buffers when mbuf fast free mode is enabled.

Following are the test results with this patch:

MRR L3FWD Test:
two ports & bi-directional flows & one core RX API: i40e_recv_pkts_bulk_alloc
TX API: i40e_xmit_pkts_simple ring_descs_size = 1024; Ring_I40E_TX_MAX_FREE_SZ = 64; tx_rs_thresh = I40E_DEFAULT_TX_RSBIT_THRESH = 32; tx_free_thresh = I40E_DEFAULT_TX_FREE_THRESH = 32;

For scalar path in arm platform with default 'tx_rs_thresh':
In n1sdp, performance is improved by 7.9%;
In thunderx2, performance is improved by 7.6%.

For scalar path in x86 platform with default 'tx_rs_thresh':
performance is improved by 4.7%.

---
 drivers/net/i40e/i40e_rxtx.c | 26 ++++++++++++++++++++++----
 1 file changed, 22 insertions(+), 4 deletions(-)
drivers/net/i40e/i40e_rxtx.c +22 -4
index 6c58decece..8c72391cde 100644
@@ -1294,7 +1294,11 @@ static __rte_always_inline int
 i40e_tx_free_bufs(struct i40e_tx_queue *txq)
 {
 	struct i40e_tx_entry *txep;
-	uint16_t i;
+	int n = txq->tx_rs_thresh;
+	uint16_t i = 0, j = 0;
+	struct rte_mbuf *free[RTE_I40E_TX_MAX_FREE_BUF_SZ];
+	const int32_t k = RTE_ALIGN_FLOOR(n, RTE_I40E_TX_MAX_FREE_BUF_SZ);
+	const int32_t m = n % RTE_I40E_TX_MAX_FREE_BUF_SZ;
 
 	if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &
 			rte_cpu_to_le_64(I40E_TXD_QW1_DTYPE_MASK)) !=
@@ -1307,9 +1311,23 @@ i40e_tx_free_bufs(struct i40e_tx_queue *txq)
 		rte_prefetch0((txep + i)->mbuf);
 
 	if (txq->offloads & DEV_TX_OFFLOAD_MBUF_FAST_FREE) {
-		for (i = 0; i < txq->tx_rs_thresh; ++i, ++txep) {
-			rte_mempool_put(txep->mbuf->pool, txep->mbuf);
-			txep->mbuf = NULL;
+		if (k) {
+			for (j = 0; j != k; j += RTE_I40E_TX_MAX_FREE_BUF_SZ) {
+				for (i = 0; i < RTE_I40E_TX_MAX_FREE_BUF_SZ; ++i, ++txep) {
+					free[i] = txep->mbuf;
+					txep->mbuf = NULL;
+				}
+				rte_mempool_put_bulk(free[0]->pool, (void **)free,
+						RTE_I40E_TX_MAX_FREE_BUF_SZ);
+			}
+		}
+
+		if (m) {
+			for (i = 0; i < m; ++i, ++txep) {
+				free[i] = txep->mbuf;
+				txep->mbuf = NULL;
+			}
+			rte_mempool_put_bulk(free[0]->pool, (void **)free, m);
 		}
 	} else {
 		for (i = 0; i < txq->tx_rs_thresh; ++i, ++txep) {
Signature · 1 line
2.25.1
#2 lore raw

Re: [dpdk-dev] [PATCH v2 1/2] net/i40e: improve performance for scalar Tx

Quoted 44 lines

-----Original Message-----
From: Feifei Wang <[email protected]>
Sent: Wednesday, June 30, 2021 10:04 AM
To: Xing, Beilei <[email protected]>
Cc: [email protected]; [email protected]; Feifei Wang <[email protected]>; Ruifeng Wang <[email protected]>
Subject: [PATCH v2 1/2] net/i40e: improve performance for scalar Tx

For i40e scalar Tx path, if implement FAST_FREE_MBUF mode, it means per- queue all mbufs come from the same mempool and have refcnt = 1.

Thus we can use bulk free of the buffers when mbuf fast free mode is enabled.

Following are the test results with this patch:

MRR L3FWD Test:
two ports & bi-directional flows & one core RX API:
i40e_recv_pkts_bulk_alloc TX API: i40e_xmit_pkts_simple ring_descs_size = 1024; Ring_I40E_TX_MAX_FREE_SZ = 64; tx_rs_thresh = I40E_DEFAULT_TX_RSBIT_THRESH = 32; tx_free_thresh = I40E_DEFAULT_TX_FREE_THRESH = 32;

For scalar path in arm platform with default 'tx_rs_thresh':
In n1sdp, performance is improved by 7.9%; In thunderx2, performance is improved by 7.6%.

For scalar path in x86 platform with default 'tx_rs_thresh':
performance is improved by 4.7%.

---
 drivers/net/i40e/i40e_rxtx.c | 26 ++++++++++++++++++++++----
 1 file changed, 22 insertions(+), 4 deletions(-)
drivers/net/i40e/i40e_rxtx.c index renamed from drivers/net/i40e/i40e_rxtx.c +0 -0

6c58decece..8c72391cde 100644

drivers/net/i40e/i40e_rxtx.c static __rte_always_inline int · line 1296

Quoted 4 lines
drivers/net/i40e/i40e_rxtx.c +1 -1
@@ -1294,7 +1294,11 @@ static __rte_always_inline int
 i40e_tx_free_bufs(struct i40e_tx_queue *txq)  {
 	struct i40e_tx_entry *txep;
-	uint16_t i;
+	int n = txq->tx_rs_thresh;

Thanks for the patch, just little comment, can we use 'tx_rs_thresh' to replace 'n' to make it more readable?

Quoted 42 lines
drivers/net/i40e/i40e_rxtx.c +22 -3
+	uint16_t i = 0, j = 0;
+	struct rte_mbuf *free[RTE_I40E_TX_MAX_FREE_BUF_SZ];
+	const int32_t k = RTE_ALIGN_FLOOR(n,
 RTE_I40E_TX_MAX_FREE_BUF_SZ);
+	const int32_t m = n % RTE_I40E_TX_MAX_FREE_BUF_SZ;
 
 	if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &
 			rte_cpu_to_le_64(I40E_TXD_QW1_DTYPE_MASK)) !=
@@ -1307,9 +1311,23 @@ i40e_tx_free_bufs(struct i40e_tx_queue *txq)
 		rte_prefetch0((txep + i)->mbuf);
 
 	if (txq->offloads & DEV_TX_OFFLOAD_MBUF_FAST_FREE) {
-		for (i = 0; i < txq->tx_rs_thresh; ++i, ++txep) {
-			rte_mempool_put(txep->mbuf->pool, txep->mbuf);
-			txep->mbuf = NULL;
+		if (k) {
+			for (j = 0; j != k; j += RTE_I40E_TX_MAX_FREE_BUF_SZ)
 {
+				for (i = 0; i < RTE_I40E_TX_MAX_FREE_BUF_SZ;
++i, ++txep) {
+					free[i] = txep->mbuf;
+					txep->mbuf = NULL;
+				}
+				rte_mempool_put_bulk(free[0]->pool, (void
 **)free,
+
 	RTE_I40E_TX_MAX_FREE_BUF_SZ);
+			}
+		}
+
+		if (m) {
+			for (i = 0; i < m; ++i, ++txep) {
+				free[i] = txep->mbuf;
+				txep->mbuf = NULL;
+			}
+			rte_mempool_put_bulk(free[0]->pool, (void **)free,

m);

drivers/net/i40e/i40e_rxtx.c +0 -1
 		}
 	} else {
 		for (i = 0; i < txq->tx_rs_thresh; ++i, ++txep) {
--

2.25.1

#3 lore raw

[dpdk-dev] 回复: [PATCH v2 1/2] net/i40e: improve performance for scalar Tx

Quoted 48 lines

-----邮件原件-----
发件人: Xing, Beilei <[email protected]>
发送时间: 2021年6月30日 11:43
收件人: Feifei Wang <[email protected]>
抄送: [email protected]; nd <[email protected]>; Ruifeng Wang <[email protected]>
主题: RE: [PATCH v2 1/2] net/i40e: improve performance for scalar Tx

Quoted 37 lines

-----Original Message-----
From: Feifei Wang <[email protected]>
Sent: Wednesday, June 30, 2021 10:04 AM
To: Xing, Beilei <[email protected]>
Cc: [email protected]; [email protected]; Feifei Wang <[email protected]>; Ruifeng Wang <[email protected]>
Subject: [PATCH v2 1/2] net/i40e: improve performance for scalar Tx

For i40e scalar Tx path, if implement FAST_FREE_MBUF mode, it means per- queue all mbufs come from the same mempool and have refcnt = 1.

Thus we can use bulk free of the buffers when mbuf fast free mode is enabled.

Following are the test results with this patch:

MRR L3FWD Test:
two ports & bi-directional flows & one core RX API:
i40e_recv_pkts_bulk_alloc TX API: i40e_xmit_pkts_simple ring_descs_size = 1024; Ring_I40E_TX_MAX_FREE_SZ = 64; tx_rs_thresh = I40E_DEFAULT_TX_RSBIT_THRESH = 32; tx_free_thresh = I40E_DEFAULT_TX_FREE_THRESH = 32;

For scalar path in arm platform with default 'tx_rs_thresh':
In n1sdp, performance is improved by 7.9%; In thunderx2, performance is improved by 7.6%.

For scalar path in x86 platform with default 'tx_rs_thresh':
performance is improved by 4.7%.

---
 drivers/net/i40e/i40e_rxtx.c | 26 ++++++++++++++++++++++----
 1 file changed, 22 insertions(+), 4 deletions(-)
Quoted 12 lines

drivers/net/i40e/i40e_rxtx.c static __rte_always_inline int · line 1296

Quoted 9 lines
(quoted diff) +0 -0

b/drivers/net/i40e/i40e_rxtx.c index 6c58decece..8c72391cde 100644

drivers/net/i40e/i40e_rxtx.c +1 -1
@@ -1294,7 +1294,11 @@ static __rte_always_inline int
 i40e_tx_free_bufs(struct i40e_tx_queue *txq)  {
 	struct i40e_tx_entry *txep;
-	uint16_t i;
+	int n = txq->tx_rs_thresh;

Thanks for the patch, just little comment, can we use 'tx_rs_thresh' to replace 'n' to make it more readable?

Good comments for this, I will update it, thanks.

Quoted 47 lines

drivers/net/i40e/i40e_rxtx.c static __rte_always_inline int

Quoted 9 lines
drivers/net/i40e/i40e_rxtx.c +4 -0
+	uint16_t i = 0, j = 0;
+	struct rte_mbuf *free[RTE_I40E_TX_MAX_FREE_BUF_SZ];
+	const int32_t k = RTE_ALIGN_FLOOR(n,
 RTE_I40E_TX_MAX_FREE_BUF_SZ);
+	const int32_t m = n % RTE_I40E_TX_MAX_FREE_BUF_SZ;
 
 	if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &
 
	rte_cpu_to_le_64(I40E_TXD_QW1_DTYPE_MASK)) != @@ -1307,9

+1311,23

drivers/net/i40e/i40e_rxtx.c i40e_tx_free_bufs(struct i40e_tx_queue *txq)

Quoted 9 lines

@@ i40e_tx_free_bufs(struct i40e_tx_queue *txq)

drivers/net/i40e/i40e_rxtx.c +2 -3
 		rte_prefetch0((txep + i)->mbuf);
 
 	if (txq->offloads & DEV_TX_OFFLOAD_MBUF_FAST_FREE) {
-		for (i = 0; i < txq->tx_rs_thresh; ++i, ++txep) {
-			rte_mempool_put(txep->mbuf->pool, txep->mbuf);
-			txep->mbuf = NULL;
+		if (k) {
+			for (j = 0; j != k; j +=

RTE_I40E_TX_MAX_FREE_BUF_SZ)

drivers/net/i40e/i40e_rxtx.c

Quoted 2 lines

{

drivers/net/i40e/i40e_rxtx.c +1 -0
+				for (i = 0; i <

RTE_I40E_TX_MAX_FREE_BUF_SZ;

Quoted 23 lines

++i, ++txep) {

drivers/net/i40e/i40e_rxtx.c +4 -0
+					free[i] = txep->mbuf;
+					txep->mbuf = NULL;
+				}
+				rte_mempool_put_bulk(free[0]->pool, (void

**)free,
+

	RTE_I40E_TX_MAX_FREE_BUF_SZ);
drivers/net/i40e/i40e_rxtx.c +9 -0
+			}
+		}
+
+		if (m) {
+			for (i = 0; i < m; ++i, ++txep) {
+				free[i] = txep->mbuf;
+				txep->mbuf = NULL;
+			}
+			rte_mempool_put_bulk(free[0]->pool, (void **)free,

m);

drivers/net/i40e/i40e_rxtx.c +0 -1
 		}
 	} else {
 		for (i = 0; i < txq->tx_rs_thresh; ++i, ++txep) {
--

2.25.1

#4 lore raw

PATCH v2 2/2 net/i40e: improve performance for vector Tx

In reply to Feifei Wang

For i40e vector Tx path, if tx_offload is set as FAST_FREE_MBUF mode, no mbuf fast free operations are executed. To fix this, add mbuf fast free mode for vector Tx path.

Furthermore, for i40e vector Tx path, if implement FAST_FREE_MBUF mode, it means per-queue all mbufs come from the same mempool and have refcnt = 1. Thus we can use bulk free of the buffers when mbuf fast free mode is enabled.

For vector path in arm platform:
In n1sdp, performance is improved by 18.4%;
In thunderx2, performance is improved by 23%.

For vector path in x86 platform:
No performance changes.

---
 drivers/net/i40e/i40e_rxtx_vec_common.h | 11 +++++++++++
 1 file changed, 11 insertions(+)
drivers/net/i40e/i40e_rxtx_vec_common.h +11 -0
index 16fcf0aec6..f52ed98d62 100644
@@ -99,6 +99,16 @@ i40e_tx_free_bufs(struct i40e_tx_queue *txq)
 	  * tx_next_dd - (tx_rs_thresh-1)
 	  */
 	txep = &txq->sw_ring[txq->tx_next_dd - (n - 1)];
+
+	if (txq->offloads & DEV_TX_OFFLOAD_MBUF_FAST_FREE) {
+		for (i = 0; i < n; i++) {
+			free[i] = txep[i].mbuf;
+			txep[i].mbuf = NULL;
+		}
+		rte_mempool_put_bulk(free[0]->pool, (void **)free, n);
+		goto done;
+	}
+
 	m = rte_pktmbuf_prefree_seg(txep[0].mbuf);
 	if (likely(m != NULL)) {
 		free[0] = m;
@@ -126,6 +136,7 @@ i40e_tx_free_bufs(struct i40e_tx_queue *txq)
 		}
 	}
 
+done:
 	/* buffers were freed, update counters */
 	txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh);
 	txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh);
Signature · 1 line
2.25.1