Skip to contentSkip to conversation

net: mana: Add support for coalesced RX packets

PATCH In review Haiyang Zhang 4 participants 23 messages Jan 6 – Feb 22

Subsystems (5)
lore
Share & tools
lore
#0 lore raw

v2 cover net: mana: Add support for coalesced RX packets

From: Haiyang Zhang <[email protected]>

Our NIC can have up to 4 RX packets on 1 CQE. To support this feature, update the RX code path, and ethtool handler. Also add counters for it.

Haiyang Zhang (2):
net: mana: Add support for coalesced RX packets on CQE
net: mana: Add ethtool counters for RX CQEs in coalesced type

 drivers/net/ethernet/microsoft/mana/mana_en.c | 49 +++++++++----
 .../ethernet/microsoft/mana/mana_ethtool.c    | 72 +++++++++++++++++--
 include/net/mana/mana.h                       | 12 ++--
 3 files changed, 112 insertions(+), 21 deletions(-)
Signature · 1 line
2.34.1
#1 lore raw

PATCH v2 1/2 net: mana: Add support for coalesced RX packets on CQE

From: Haiyang Zhang <[email protected]>

Our NIC can have up to 4 RX packets on 1 CQE. To support this feature, check and process the type CQE_RX_COALESCED_4. The default setting is disabled, to avoid possible regression on latency.

And add ethtool handler to switch this feature. To turn it on, run:
ethtool -C <nic> rx-frames 4
To turn it off:
ethtool -C <nic> rx-frames 1

---
V2:
Updated extack msg, as recommended by Jakub Kicinski, and Simon Horman.

---
 drivers/net/ethernet/microsoft/mana/mana_en.c | 32 ++++++-----
 .../ethernet/microsoft/mana/mana_ethtool.c    | 55 +++++++++++++++++++
 include/net/mana/mana.h                       |  2 +
 3 files changed, 74 insertions(+), 15 deletions(-)
drivers/net/ethernet/microsoft/mana/mana_en.c +17 -15
index 1ad154f9db1a..a46a1adf83bc 100644
@@ -1330,7 +1330,7 @@ static int mana_cfg_vport_steering(struct mana_port_context *apc,
 	req->update_hashkey = update_key;
 	req->update_indir_tab = update_tab;
 	req->default_rxobj = apc->default_rxobj;
-	req->cqe_coalescing_enable = 0;
+	req->cqe_coalescing_enable = apc->cqe_coalescing_enable;
 
 	if (update_key)
 		memcpy(&req->hashkey, apc->hashkey, MANA_HASH_KEY_SIZE);
@@ -1864,11 +1864,12 @@ static struct sk_buff *mana_build_skb(struct mana_rxq *rxq, void *buf_va,
 }
 
 static void mana_rx_skb(void *buf_va, bool from_pool,
-			struct mana_rxcomp_oob *cqe, struct mana_rxq *rxq)
+			struct mana_rxcomp_oob *cqe, struct mana_rxq *rxq,
+			int i)
 {
 	struct mana_stats_rx *rx_stats = &rxq->stats;
 	struct net_device *ndev = rxq->ndev;
-	uint pkt_len = cqe->ppi[0].pkt_len;
+	uint pkt_len = cqe->ppi[i].pkt_len;
 	u16 rxq_idx = rxq->rxq_idx;
 	struct napi_struct *napi;
 	struct xdp_buff xdp = {};
@@ -1912,7 +1913,7 @@ static void mana_rx_skb(void *buf_va, bool from_pool,
 	}
 
 	if (cqe->rx_hashtype != 0 && (ndev->features & NETIF_F_RXHASH)) {
-		hash_value = cqe->ppi[0].pkt_hash;
+		hash_value = cqe->ppi[i].pkt_hash;
 
 		if (cqe->rx_hashtype & MANA_HASH_L4)
 			skb_set_hash(skb, hash_value, PKT_HASH_TYPE_L4);
@@ -2047,9 +2048,11 @@ static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq,
 	struct mana_recv_buf_oob *rxbuf_oob;
 	struct mana_port_context *apc;
 	struct device *dev = gc->dev;
+	bool coalesced = false;
 	void *old_buf = NULL;
 	u32 curr, pktlen;
 	bool old_fp;
+	int i = 0;
 
 	apc = netdev_priv(ndev);
 
@@ -2064,9 +2067,8 @@ static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq,
 		goto drop;
 
 	case CQE_RX_COALESCED_4:
-		netdev_err(ndev, "RX coalescing is unsupported\n");
-		apc->eth_stats.rx_coalesced_err++;
-		return;
+		coalesced = true;
+		break;
 
 	case CQE_RX_OBJECT_FENCE:
 		complete(&rxq->fence_event);
@@ -2079,14 +2081,10 @@ static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq,
 		return;
 	}
 
-	pktlen = oob->ppi[0].pkt_len;
-
-	if (pktlen == 0) {
-		/* data packets should never have packetlength of zero */
-		netdev_err(ndev, "RX pkt len=0, rq=%u, cq=%u, rxobj=0x%llx\n",
-			   rxq->gdma_id, cq->gdma_id, rxq->rxobj);
+nextpkt:
+	pktlen = oob->ppi[i].pkt_len;
+	if (pktlen == 0)
 		return;
-	}
 
 	curr = rxq->buf_index;
 	rxbuf_oob = &rxq->rx_oobs[curr];
@@ -2097,12 +2095,15 @@ static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq,
 	/* Unsuccessful refill will have old_buf == NULL.
 	 * In this case, mana_rx_skb() will drop the packet.
 	 */
-	mana_rx_skb(old_buf, old_fp, oob, rxq);
+	mana_rx_skb(old_buf, old_fp, oob, rxq, i);
 
 drop:
 	mana_move_wq_tail(rxq->gdma_rq, rxbuf_oob->wqe_inf.wqe_size_in_bu);
 
 	mana_post_pkt_rxq(rxq);
+
+	if (coalesced && (++i < MANA_RXCOMP_OOB_NUM_PPI))
+		goto nextpkt;
 }
 
 static void mana_poll_rx_cq(struct mana_cq *cq)
@@ -3276,6 +3277,7 @@ static int mana_probe_port(struct mana_context *ac, int port_idx,
 	apc->port_handle = INVALID_MANA_HANDLE;
 	apc->pf_filter_handle = INVALID_MANA_HANDLE;
 	apc->port_idx = port_idx;
+	apc->cqe_coalescing_enable = 0;
 
 	mutex_init(&apc->vport_mutex);
 	apc->vport_use_count = 0;
drivers/net/ethernet/microsoft/mana/mana_ethtool.c +55 -0

73 lines not on this page · raw message

include/net/mana/mana.h +2 -0
index d7e089c6b694..51d26ebeff6c 100644
@@ -556,6 +556,8 @@ struct mana_port_context {
 	bool port_is_up;
 	bool port_st_save; /* Saved port state */
 
+	u8 cqe_coalescing_enable;
+
 	struct mana_ethtool_stats eth_stats;
 
 	struct mana_ethtool_phy_stats phy_stats;
Signature · 1 line
2.34.1
Reviewed-by on v2 1/2
#2 lore raw

RE: [PATCH V2,net-next, 1/2] net: mana: Add support for coalesced RX packets on CQE

Quoted 9 lines

Subject: [PATCH V2,net-next, 1/2] net: mana: Add support for coalesced RX packets on CQE

From: Haiyang Zhang <[email protected]>

Our NIC can have up to 4 RX packets on 1 CQE. To support this feature, check and process the type CQE_RX_COALESCED_4. The default setting is disabled, to avoid possible regression on latency.

Quoted 6 lines

And add ethtool handler to switch this feature. To turn it on, run:
ethtool -C <nic> rx-frames 4
To turn it off:
ethtool -C <nic> rx-frames 1

Quoted 219 lines

---
V2:
Updated extack msg, as recommended by Jakub Kicinski, and Simon Horman.

---
 drivers/net/ethernet/microsoft/mana/mana_en.c | 32 ++++++-----
 .../ethernet/microsoft/mana/mana_ethtool.c    | 55 +++++++++++++++++++
 include/net/mana/mana.h                       |  2 +
 3 files changed, 74 insertions(+), 15 deletions(-)
drivers/net/ethernet/microsoft/mana/mana_en.c +1 -1
index 1ad154f9db1a..a46a1adf83bc 100644
@@ -1330,7 +1330,7 @@ static int mana_cfg_vport_steering(struct
 mana_port_context *apc,
 	req->update_hashkey = update_key;
 	req->update_indir_tab = update_tab;
 	req->default_rxobj = apc->default_rxobj;
-	req->cqe_coalescing_enable = 0;
+	req->cqe_coalescing_enable = apc->cqe_coalescing_enable;
 
 	if (update_key)

memcpy(&req->hashkey, apc->hashkey,
MANA_HASH_KEY_SIZE); @@ -1864,11 +1864,12 @@ static struct sk_buff *mana_build_skb(struct mana_rxq *rxq, void *buf_va, }

static void mana_rx_skb(void *buf_va, bool from_pool,

drivers/net/ethernet/microsoft/mana/mana_en.c +4 -3
-			struct mana_rxcomp_oob *cqe, struct mana_rxq *rxq)
+			struct mana_rxcomp_oob *cqe, struct mana_rxq *rxq,
+			int i)
 {
 	struct mana_stats_rx *rx_stats = &rxq->stats;
 	struct net_device *ndev = rxq->ndev;
-	uint pkt_len = cqe->ppi[0].pkt_len;
+	uint pkt_len = cqe->ppi[i].pkt_len;
 	u16 rxq_idx = rxq->rxq_idx;
 	struct napi_struct *napi;
 	struct xdp_buff xdp = {};
@@ -1912,7 +1913,7 @@ static void mana_rx_skb(void *buf_va, bool
 from_pool,
 	}
 
 	if (cqe->rx_hashtype != 0 && (ndev->features & NETIF_F_RXHASH)) {
-		hash_value = cqe->ppi[0].pkt_hash;
+		hash_value = cqe->ppi[i].pkt_hash;
 
 		if (cqe->rx_hashtype & MANA_HASH_L4)
drivers/net/ethernet/microsoft/mana/mana_en.c +4 -3
 			skb_set_hash(skb, hash_value, PKT_HASH_TYPE_L4);
@@ -2047,9 +2048,11 @@ static void mana_process_rx_cqe(struct
 mana_rxq *rxq, struct mana_cq *cq,
 	struct mana_recv_buf_oob *rxbuf_oob;
 	struct mana_port_context *apc;
 	struct device *dev = gc->dev;
+	bool coalesced = false;
 	void *old_buf = NULL;
 	u32 curr, pktlen;
 	bool old_fp;
+	int i = 0;
 
 	apc = netdev_priv(ndev);
@@ -2064,9 +2067,8 @@ static void mana_process_rx_cqe(struct mana_rxq
 *rxq, struct mana_cq *cq,
 		goto drop;
 
 	case CQE_RX_COALESCED_4:
-		netdev_err(ndev, "RX coalescing is unsupported\n");
-		apc->eth_stats.rx_coalesced_err++;
-		return;
+		coalesced = true;
+		break;
 
 	case CQE_RX_OBJECT_FENCE:
drivers/net/ethernet/microsoft/mana/mana_en.c +3 -7
 		complete(&rxq->fence_event);
@@ -2079,14 +2081,10 @@ static void mana_process_rx_cqe(struct
 mana_rxq *rxq, struct mana_cq *cq,
 		return;
 	}
 
-	pktlen = oob->ppi[0].pkt_len;
-
-	if (pktlen == 0) {
-		/* data packets should never have packetlength of zero */
-		netdev_err(ndev, "RX pkt len=0, rq=%u, cq=%u,
 rxobj=0x%llx\n",
-			   rxq->gdma_id, cq->gdma_id, rxq->rxobj);
+nextpkt:
+	pktlen = oob->ppi[i].pkt_len;
+	if (pktlen == 0)
 		return;
-	}
 
drivers/net/ethernet/microsoft/mana/mana_en.c +4 -1
 	curr = rxq->buf_index;
 	rxbuf_oob = &rxq->rx_oobs[curr];
@@ -2097,12 +2095,15 @@ static void mana_process_rx_cqe(struct
 mana_rxq *rxq, struct mana_cq *cq,
 	/* Unsuccessful refill will have old_buf == NULL.
 	 * In this case, mana_rx_skb() will drop the packet.
 	 */
-	mana_rx_skb(old_buf, old_fp, oob, rxq);
+	mana_rx_skb(old_buf, old_fp, oob, rxq, i);
 
 drop:
 	mana_move_wq_tail(rxq->gdma_rq, rxbuf_oob-
 >wqe_inf.wqe_size_in_bu);
 
 	mana_post_pkt_rxq(rxq);
+
+	if (coalesced && (++i < MANA_RXCOMP_OOB_NUM_PPI))
+		goto nextpkt;
 }

static void mana_poll_rx_cq(struct mana_cq *cq) @@ -3276,6 +3277,7 @@
static int mana_probe_port(struct mana_context *ac, int port_idx,

drivers/net/ethernet/microsoft/mana/mana_en.c +1 -0
 	apc->port_handle = INVALID_MANA_HANDLE;
 	apc->pf_filter_handle = INVALID_MANA_HANDLE;
 	apc->port_idx = port_idx;
+	apc->cqe_coalescing_enable = 0;
 
 	mutex_init(&apc->vport_mutex);
 	apc->vport_use_count = 0;
drivers/net/ethernet/microsoft/mana/mana_ethtool.c +52 -0
index 0e2f4343ac67..b2b9bfb50396 100644
@@ -397,6 +397,58 @@ static void mana_get_channels(struct net_device
 *ndev,
 	channel->combined_count = apc->num_queues;  }
 
+static int mana_get_coalesce(struct net_device *ndev,
+			     struct ethtool_coalesce *ec,
+			     struct kernel_ethtool_coalesce *kernel_coal,
+			     struct netlink_ext_ack *extack) {
+	struct mana_port_context *apc = netdev_priv(ndev);
+
+	ec->rx_max_coalesced_frames =
+		apc->cqe_coalescing_enable ?
 MANA_RXCOMP_OOB_NUM_PPI : 1;
+
+	return 0;
+}
+
+static int mana_set_coalesce(struct net_device *ndev,
+			     struct ethtool_coalesce *ec,
+			     struct kernel_ethtool_coalesce *kernel_coal,
+			     struct netlink_ext_ack *extack) {
+	struct mana_port_context *apc = netdev_priv(ndev);
+	u8 saved_cqe_coalescing_enable;
+	int err;
+
+	if (ec->rx_max_coalesced_frames != 1 &&
+	    ec->rx_max_coalesced_frames != MANA_RXCOMP_OOB_NUM_PPI)
 {
+		NL_SET_ERR_MSG_FMT(extack,
+				   "rx-frames must be 1 or %u, got %u",
+				   MANA_RXCOMP_OOB_NUM_PPI,
+				   ec->rx_max_coalesced_frames);
+		return -EINVAL;
+	}
+
+	saved_cqe_coalescing_enable = apc->cqe_coalescing_enable;
+	apc->cqe_coalescing_enable =
+		ec->rx_max_coalesced_frames ==
 MANA_RXCOMP_OOB_NUM_PPI;
+
+	if (!apc->port_is_up)
+		return 0;
+
+	err = mana_config_rss(apc, TRI_STATE_TRUE, false, false);
+
+	if (err) {
+		netdev_err(ndev, "Set rx-frames to %u failed:%d\n",
+			   ec->rx_max_coalesced_frames, err);
+		NL_SET_ERR_MSG_FMT(extack, "Set rx-frames to %u failed",
+				   ec->rx_max_coalesced_frames);
+
+		apc->cqe_coalescing_enable = saved_cqe_coalescing_enable;
+	}
+
+	return err;
+}
+
 static int mana_set_channels(struct net_device *ndev,
 			     struct ethtool_channels *channels)  { @@ -517,6
+569,7 @@ static int mana_get_link_ksettings(struct net_device *ndev,  }
 
 const struct ethtool_ops mana_ethtool_ops = {
+	.supported_coalesce_params =

ETHTOOL_COALESCE_RX_MAX_FRAMES,

drivers/net/ethernet/microsoft/mana/mana_ethtool.c +2 -0
 	.get_ethtool_stats	= mana_get_ethtool_stats,
 	.get_sset_count		= mana_get_sset_count,
 	.get_strings		= mana_get_strings,
@@ -527,6 +580,8 @@ const struct ethtool_ops mana_ethtool_ops = {
 	.set_rxfh		= mana_set_rxfh,
 	.get_channels		= mana_get_channels,
 	.set_channels		= mana_set_channels,
+	.get_coalesce		= mana_get_coalesce,
+	.set_coalesce		= mana_set_coalesce,
 	.get_ringparam          = mana_get_ringparam,
 	.set_ringparam          = mana_set_ringparam,
 	.get_link_ksettings	= mana_get_link_ksettings,
include/net/mana/mana.h index renamed from include/net/mana/mana.h +0 -0

d7e089c6b694..51d26ebeff6c 100644

include/net/mana/mana.h +2 -1
@@ -556,6 +556,8 @@ struct mana_port_context {
 	bool port_is_up;
 	bool port_st_save; /* Saved port state */
 
+	u8 cqe_coalescing_enable;
+
 	struct mana_ethtool_stats eth_stats;
 
 	struct mana_ethtool_phy_stats phy_stats;
--

2.34.1

maintainer
#3 lore raw

Re: [PATCH V2,net-next, 1/2] net: mana: Add support for coalesced RX packets on CQE

In reply to Haiyang Zhang
Quoted 10 lines · On Tue, 6 Jan 2026 12:46:46 -0800 Haiyang Zhang wrote:

From: Haiyang Zhang <[email protected]>

Our NIC can have up to 4 RX packets on 1 CQE. To support this feature, check and process the type CQE_RX_COALESCED_4. The default setting is disabled, to avoid possible regression on latency.

And add ethtool handler to switch this feature. To turn it on, run:
ethtool -C <nic> rx-frames 4
To turn it off:
ethtool -C <nic> rx-frames 1

Exposing just rx frame count, and only two values is quite unusual. Please explain in more detail the coalescing logic of the device.

Quoted 22 lines
drivers/net/ethernet/microsoft/mana/mana_en.c +3 -7
@@ -2079,14 +2081,10 @@ static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq,
 		return;
 	}
 
-	pktlen = oob->ppi[0].pkt_len;
-
-	if (pktlen == 0) {
-		/* data packets should never have packetlength of zero */
-		netdev_err(ndev, "RX pkt len=0, rq=%u, cq=%u, rxobj=0x%llx\n",
-			   rxq->gdma_id, cq->gdma_id, rxq->rxobj);
+nextpkt:
+	pktlen = oob->ppi[i].pkt_len;
+	if (pktlen == 0)
 		return;
-	}
 
 	curr = rxq->buf_index;
 	rxbuf_oob = &rxq->rx_oobs[curr];
@@ -2097,12 +2095,15 @@ static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq,
 	/* Unsuccessful refill will have old_buf == NULL.
 	 * In this case, mana_rx_skb() will drop the packet.
 	 */

drivers/net/ethernet/microsoft/mana/mana_en.c static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq, · line 2106

Quoted 10 lines
drivers/net/ethernet/microsoft/mana/mana_en.c +4 -1
-	mana_rx_skb(old_buf, old_fp, oob, rxq);
+	mana_rx_skb(old_buf, old_fp, oob, rxq, i);
 
 drop:
 	mana_move_wq_tail(rxq->gdma_rq, rxbuf_oob->wqe_inf.wqe_size_in_bu);
 
 	mana_post_pkt_rxq(rxq);
+
+	if (coalesced && (++i < MANA_RXCOMP_OOB_NUM_PPI))
+		goto nextpkt;

Please code this up as a loop. Using gotos for control flow other than to jump to error handling epilogues is a poor coding practice (see the kernel coding style).

Quoted 17 lines
drivers/net/ethernet/microsoft/mana/mana_ethtool.c +17 -0
+static int mana_set_coalesce(struct net_device *ndev,
+			     struct ethtool_coalesce *ec,
+			     struct kernel_ethtool_coalesce *kernel_coal,
+			     struct netlink_ext_ack *extack)
+{
+	struct mana_port_context *apc = netdev_priv(ndev);
+	u8 saved_cqe_coalescing_enable;
+	int err;
+
+	if (ec->rx_max_coalesced_frames != 1 &&
+	    ec->rx_max_coalesced_frames != MANA_RXCOMP_OOB_NUM_PPI) {
+		NL_SET_ERR_MSG_FMT(extack,
+				   "rx-frames must be 1 or %u, got %u",
+				   MANA_RXCOMP_OOB_NUM_PPI,
+				   ec->rx_max_coalesced_frames);
+		return -EINVAL;
+	}

drivers/net/ethernet/microsoft/mana/mana_ethtool.c static void mana_get_channels(struct net_device *ndev, · line 439

Quoted 10 lines
drivers/net/ethernet/microsoft/mana/mana_ethtool.c +10 -0
+
+	saved_cqe_coalescing_enable = apc->cqe_coalescing_enable;
+	apc->cqe_coalescing_enable =
+		ec->rx_max_coalesced_frames == MANA_RXCOMP_OOB_NUM_PPI;
+
+	if (!apc->port_is_up)
+		return 0;
+
+	err = mana_config_rss(apc, TRI_STATE_TRUE, false, false);
+

unnecessary empty line

drivers/net/ethernet/microsoft/mana/mana_ethtool.c static void mana_get_channels(struct net_device *ndev, · line 444

Quoted 5 lines
drivers/net/ethernet/microsoft/mana/mana_ethtool.c +5 -0
+	if (err) {
+		netdev_err(ndev, "Set rx-frames to %u failed:%d\n",
+			   ec->rx_max_coalesced_frames, err);
+		NL_SET_ERR_MSG_FMT(extack, "Set rx-frames to %u failed",
+				   ec->rx_max_coalesced_frames);

These messages are both pointless. If HW communication has failed presumably there will already be an error in the logs. The extack gives the user no information they wouldn't already have.

Quoted 5 lines
drivers/net/ethernet/microsoft/mana/mana_ethtool.c +5 -0
+		apc->cqe_coalescing_enable = saved_cqe_coalescing_enable;
+	}
+
+	return err;
+}
maintainer
#4 lore raw

RE: [EXTERNAL] Re: [PATCH V2,net-next, 1/2] net: mana: Add support for coalesced RX packets on CQE

Quoted 30 lines

-----Original Message-----
From: Jakub Kicinski <[email protected]>
Sent: Friday, January 9, 2026 8:56 PM
To: Haiyang Zhang <[email protected]>
Cc: [email protected]; [email protected]; KY Srinivasan <[email protected]>; Haiyang Zhang <[email protected]>; Wei Liu <[email protected]>; Dexuan Cui <[email protected]>; Long Li <[email protected]>; Andrew Lunn <[email protected]>; David S. Miller <[email protected]>; Eric Dumazet <[email protected]>; Paolo Abeni <[email protected]>; Konstantin Taranov <[email protected]>; Simon Horman <[email protected]>; Erni Sri Satya Vennela <[email protected]>; Shradha Gupta <[email protected]>; Saurabh Sengar <[email protected]>; Aditya Garg <[email protected]>; Dipayaan Roy <[email protected]>; Shiraz Saleem <[email protected]>; [email protected]; linux- [email protected]; Paul Rosswurm <[email protected]>
Subject: [EXTERNAL] Re: [PATCH V2,net-next, 1/2] net: mana: Add support for coalesced RX packets on CQE

Quoted 6 lines · On Tue, 6 Jan 2026 12:46:46 -0800 Haiyang Zhang wrote:

From: Haiyang Zhang <[email protected]>

Our NIC can have up to 4 RX packets on 1 CQE. To support this feature, check and process the type CQE_RX_COALESCED_4. The default setting is disabled, to avoid possible regression on latency.

Quoted 7 lines
Quoted 4 lines

And add ethtool handler to switch this feature. To turn it on, run:
ethtool -C <nic> rx-frames 4
To turn it off:
ethtool -C <nic> rx-frames 1

Exposing just rx frame count, and only two values is quite unusual. Please explain in more detail the coalescing logic of the device.

Our NIC device only supports coalescing on RX. And when it's disabled each RX CQE indicates 1 RX packet; when enabled each RX CQE indicates up to 4 packets.

Quoted 31 lines

Review static void mana_process_rx_cqe(struct mana_rxq

Quoted 2 lines
(quoted diff) +0 -0
@@ -2079,14 +2081,10 @@ static void mana_process_rx_cqe(struct mana_rxq

*rxq, struct mana_cq *cq,

drivers/net/ethernet/microsoft/mana/mana_en.c static void mana_process_rx_cqe(struct mana_rxq

Quoted 18 lines
drivers/net/ethernet/microsoft/mana/mana_en.c +3 -7
 		return;
 	}
 
-	pktlen = oob->ppi[0].pkt_len;
-
-	if (pktlen == 0) {
-		/* data packets should never have packetlength of zero */
-		netdev_err(ndev, "RX pkt len=0, rq=%u, cq=%u, rxobj=0x%llx\n",
-			   rxq->gdma_id, cq->gdma_id, rxq->rxobj);
+nextpkt:
+	pktlen = oob->ppi[i].pkt_len;
+	if (pktlen == 0)
 		return;
-	}
 
 	curr = rxq->buf_index;
 	rxbuf_oob = &rxq->rx_oobs[curr];
@@ -2097,12 +2095,15 @@ static void mana_process_rx_cqe(struct mana_rxq

*rxq, struct mana_cq *cq,

Quoted 9 lines
drivers/net/ethernet/microsoft/mana/mana_en.c +1 -1
 	/* Unsuccessful refill will have old_buf == NULL.
 	 * In this case, mana_rx_skb() will drop the packet.
 	 */
-	mana_rx_skb(old_buf, old_fp, oob, rxq);
+	mana_rx_skb(old_buf, old_fp, oob, rxq, i);
 
 drop:
 	mana_move_wq_tail(rxq->gdma_rq, rxbuf_oob->wqe_inf.wqe_size_in_bu);
 
Quoted 8 lines

drivers/net/ethernet/microsoft/mana/mana_en.c static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq, · line 2106

Quoted 4 lines
drivers/net/ethernet/microsoft/mana/mana_en.c +3 -0
 	mana_post_pkt_rxq(rxq);
+
+	if (coalesced && (++i < MANA_RXCOMP_OOB_NUM_PPI))
+		goto nextpkt;

Please code this up as a loop. Using gotos for control flow other than to jump to error handling epilogues is a poor coding practice (see the kernel coding style).

Will do.

Quoted 22 lines
Quoted 22 lines
drivers/net/ethernet/microsoft/mana/mana_ethtool.c +21 -0
+static int mana_set_coalesce(struct net_device *ndev,
+			     struct ethtool_coalesce *ec,
+			     struct kernel_ethtool_coalesce *kernel_coal,
+			     struct netlink_ext_ack *extack)
+{
+	struct mana_port_context *apc = netdev_priv(ndev);
+	u8 saved_cqe_coalescing_enable;
+	int err;
+
+	if (ec->rx_max_coalesced_frames != 1 &&
+	    ec->rx_max_coalesced_frames != MANA_RXCOMP_OOB_NUM_PPI) {
+		NL_SET_ERR_MSG_FMT(extack,
+				   "rx-frames must be 1 or %u, got %u",
+				   MANA_RXCOMP_OOB_NUM_PPI,
+				   ec->rx_max_coalesced_frames);
+		return -EINVAL;
+	}
+
+	saved_cqe_coalescing_enable = apc->cqe_coalescing_enable;
+	apc->cqe_coalescing_enable =
+		ec->rx_max_coalesced_frames == MANA_RXCOMP_OOB_NUM_PPI;
Quoted 8 lines

drivers/net/ethernet/microsoft/mana/mana_ethtool.c static void mana_get_channels(struct net_device *ndev, · line 439

Quoted 6 lines
drivers/net/ethernet/microsoft/mana/mana_ethtool.c +6 -0
+
+	if (!apc->port_is_up)
+		return 0;
+
+	err = mana_config_rss(apc, TRI_STATE_TRUE, false, false);
+

unnecessary empty line

Will rm.

Quoted 10 lines

drivers/net/ethernet/microsoft/mana/mana_ethtool.c static void mana_get_channels(struct net_device *ndev, · line 444

Quoted 6 lines
drivers/net/ethernet/microsoft/mana/mana_ethtool.c +5 -0
+	if (err) {
+		netdev_err(ndev, "Set rx-frames to %u failed:%d\n",
+			   ec->rx_max_coalesced_frames, err);
+		NL_SET_ERR_MSG_FMT(extack, "Set rx-frames to %u failed",
+				   ec->rx_max_coalesced_frames);

These messages are both pointless. If HW communication has failed presumably there will already be an error in the logs. The extack gives the user no information they wouldn't already have.

Will rm.

Thanks,
- Haiyang

maintainer
#5 lore raw

Re: [EXTERNAL] Re: [PATCH V2,net-next, 1/2] net: mana: Add support for coalesced RX packets on CQE

Quoted 9 lines · On Mon, 12 Jan 2026 21:01:59 +0000 Haiyang Zhang wrote:
Quoted 9 lines
Quoted 8 lines

Our NIC can have up to 4 RX packets on 1 CQE. To support this feature, check and process the type CQE_RX_COALESCED_4. The default setting is disabled, to avoid possible regression on latency.

And add ethtool handler to switch this feature. To turn it on, run:
ethtool -C <nic> rx-frames 4
To turn it off:
ethtool -C <nic> rx-frames 1

Quoted 4 lines
Quoted 2 lines

Exposing just rx frame count, and only two values is quite unusual. Please explain in more detail the coalescing logic of the device.

Our NIC device only supports coalescing on RX. And when it's disabled each RX CQE indicates 1 RX packet; when enabled each RX CQE indicates up to 4 packets.

I get that. What is the logic for combining 4 packets into a single completion? How does it work? Your commit message mentions "regression on latency" - what is the bound on that regression?

maintainer
#6 lore raw

RE: [EXTERNAL] Re: [PATCH V2,net-next, 1/2] net: mana: Add support for coalesced RX packets on CQE

Quoted 37 lines

-----Original Message-----
From: Jakub Kicinski <[email protected]>
Sent: Monday, January 12, 2026 8:22 PM
To: Haiyang Zhang <[email protected]>
Cc: Haiyang Zhang <[email protected]>; linux- [email protected]; [email protected]; KY Srinivasan <[email protected]>; Wei Liu <[email protected]>; Dexuan Cui <[email protected]>; Long Li <[email protected]>; Andrew Lunn <[email protected]>; David S. Miller <[email protected]>; Eric Dumazet <[email protected]>; Paolo Abeni <[email protected]>; Konstantin Taranov <[email protected]>; Simon Horman <[email protected]>; Erni Sri Satya Vennela <[email protected]>; Shradha Gupta <[email protected]>; Saurabh Sengar <[email protected]>; Aditya Garg <[email protected]>; Dipayaan Roy <[email protected]>; Shiraz Saleem <[email protected]>; [email protected]; linux- [email protected]; Paul Rosswurm <[email protected]>
Subject: Re: [EXTERNAL] Re: [PATCH V2,net-next, 1/2] net: mana: Add support for coalesced RX packets on CQE

Quoted 1 line · On Mon, 12 Jan 2026 21:01:59 +0000 Haiyang Zhang wrote:
Quoted 1 line
Quoted 1 line

Our NIC can have up to 4 RX packets on 1 CQE. To support this

feature,

Quoted 1 line
Quoted 1 line
Quoted 1 line

check and process the type CQE_RX_COALESCED_4. The default setting

is

Quoted 9 lines
Quoted 9 lines
Quoted 6 lines

disabled, to avoid possible regression on latency.

And add ethtool handler to switch this feature. To turn it on, run:
ethtool -C <nic> rx-frames 4
To turn it off:
ethtool -C <nic> rx-frames 1

Exposing just rx frame count, and only two values is quite unusual. Please explain in more detail the coalescing logic of the device.

Quoted 8 lines
Quoted 1 line

Our NIC device only supports coalescing on RX. And when it's disabled

each

Quoted 1 line

RX CQE indicates 1 RX packet; when enabled each RX CQE indicates up to 4

packets.

I get that. What is the logic for combining 4 packets into a single completion? How does it work? Your commit message mentions "regression on latency" - what is the bound on that regression?

When we received CQE type CQE_RX_COALESCED_4, it's a coalesced CQE. And in the CQE OOB, there is an array with 4 PPI elements, with each pkt's length:
oob->ppi[i].pkt_len.

So we read the related WQE and the DMA buffers for the RX pkt payloads, up to 4. But, if the coalesced pkts <4, the pkt_len will be 0 after the last pkt, so we know when to stop reading the WQEs.

Thanks,
- Haiyang

maintainer
#7 lore raw

RE: [EXTERNAL] Re: [PATCH V2,net-next, 1/2] net: mana: Add support for coalesced RX packets on CQE

Quoted 77 lines

-----Original Message-----
From: Haiyang Zhang <[email protected]>
Sent: Tuesday, January 13, 2026 10:09 AM
To: Jakub Kicinski <[email protected]>
Cc: Haiyang Zhang <[email protected]>; linux- [email protected]; [email protected]; KY Srinivasan <[email protected]>; Wei Liu <[email protected]>; Dexuan Cui <[email protected]>; Long Li <[email protected]>; Andrew Lunn <[email protected]>; David S. Miller <[email protected]>; Eric Dumazet <[email protected]>; Paolo Abeni <[email protected]>; Konstantin Taranov <[email protected]>; Simon Horman <[email protected]>; Erni Sri Satya Vennela <[email protected]>; Shradha Gupta <[email protected]>; Saurabh Sengar <[email protected]>; Aditya Garg <[email protected]>; Dipayaan Roy <[email protected]>; Shiraz Saleem <[email protected]>; [email protected]; linux- [email protected]; Paul Rosswurm <[email protected]>
Subject: RE: [EXTERNAL] Re: [PATCH V2,net-next, 1/2] net: mana: Add support for coalesced RX packets on CQE

Quoted 10 lines

-----Original Message-----
From: Jakub Kicinski <[email protected]>
Sent: Monday, January 12, 2026 8:22 PM
To: Haiyang Zhang <[email protected]>
Cc: Haiyang Zhang <[email protected]>; linux- [email protected]; [email protected]; KY Srinivasan <[email protected]>; Wei Liu <[email protected]>; Dexuan Cui <[email protected]>; Long Li <[email protected]>; Andrew Lunn <[email protected]>; David S. Miller <[email protected]>; Eric Dumazet <[email protected]>; Paolo Abeni <[email protected]>;

Konstantin

Quoted 11 lines

Taranov <[email protected]>; Simon Horman <[email protected]>; Erni Sri Satya Vennela <[email protected]>; Shradha Gupta <[email protected]>; Saurabh Sengar <[email protected]>; Aditya Garg <[email protected]>; Dipayaan Roy <[email protected]>; Shiraz Saleem <[email protected]>; [email protected]; linux- [email protected]; Paul Rosswurm <[email protected]>
Subject: Re: [EXTERNAL] Re: [PATCH V2,net-next, 1/2] net: mana: Add support for coalesced RX packets on CQE

Quoted 8 lines
Quoted 1 line · On Mon, 12 Jan 2026 21:01:59 +0000 Haiyang Zhang wrote:
Quoted 1 line
Quoted 1 line

Our NIC can have up to 4 RX packets on 1 CQE. To support this

feature,

Quoted 1 line
Quoted 1 line
Quoted 1 line

check and process the type CQE_RX_COALESCED_4. The default setting

is

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

disabled, to avoid possible regression on latency.

And add ethtool handler to switch this feature. To turn it on,

run:

Quoted 9 lines
Quoted 7 lines
Quoted 6 lines
Quoted 3 lines

ethtool -C <nic> rx-frames 4
To turn it off:
ethtool -C <nic> rx-frames 1

Exposing just rx frame count, and only two values is quite unusual. Please explain in more detail the coalescing logic of the device.

Our NIC device only supports coalescing on RX. And when it's disabled

each

Quoted 1 line

RX CQE indicates 1 RX packet; when enabled each RX CQE indicates up to

4

Quoted 5 lines

packets.

I get that. What is the logic for combining 4 packets into a single completion? How does it work? Your commit message mentions "regression on latency" - what is the bound on that regression?

When we received CQE type CQE_RX_COALESCED_4, it's a coalesced CQE. And in the CQE
OOB, there is an array with 4 PPI elements, with each pkt's length:
oob->ppi[i].pkt_len.

Quoted 5 lines

So we read the related WQE and the DMA buffers for the RX pkt payloads, up to 4.
But, if the coalesced pkts <4, the pkt_len will be 0 after the last pkt, so we
know when to stop reading the WQEs.

And, the coalescing can add up to 2 microseconds into one-way latency.

Thanks,
- Haiyang

maintainer
#8 lore raw

Re: [EXTERNAL] Re: [PATCH V2,net-next, 1/2] net: mana: Add support for coalesced RX packets on CQE

Quoted 8 lines · On Tue, 13 Jan 2026 15:13:24 +0000 Haiyang Zhang wrote:
Quoted 8 lines
Quoted 3 lines

I get that. What is the logic for combining 4 packets into a single completion? How does it work? Your commit message mentions "regression on latency" - what is the bound on that regression?

When we received CQE type CQE_RX_COALESCED_4, it's a coalesced CQE. And in the CQE OOB, there is an array with 4 PPI elements, with each pkt's length:
oob->ppi[i].pkt_len.

Quoted 6 lines
Quoted 4 lines

So we read the related WQE and the DMA buffers for the RX pkt payloads, up to 4.
But, if the coalesced pkts <4, the pkt_len will be 0 after the last pkt, so we know when to stop reading the WQEs.

And, the coalescing can add up to 2 microseconds into one-way latency.

I am asking you how the _device_ (hypervisor?) decides when to coalesce and when to send a partial CQE (<4 packets in 4 pkt CQE). You are using the coalescing uAPI, so I'm trying to make sure this is the correct API. CQE configuration can also be done via ringparam.

maintainer
#9 lore raw

RE: [EXTERNAL] Re: [PATCH V2,net-next, 1/2] net: mana: Add support for coalesced RX packets on CQE

Quoted 40 lines

-----Original Message-----
From: Jakub Kicinski <[email protected]>
Sent: Tuesday, January 13, 2026 8:10 PM
To: Haiyang Zhang <[email protected]>
Cc: Haiyang Zhang <[email protected]>; linux- [email protected]; [email protected]; KY Srinivasan <[email protected]>; Wei Liu <[email protected]>; Dexuan Cui <[email protected]>; Long Li <[email protected]>; Andrew Lunn <[email protected]>; David S. Miller <[email protected]>; Eric Dumazet <[email protected]>; Paolo Abeni <[email protected]>; Konstantin Taranov <[email protected]>; Simon Horman <[email protected]>; Erni Sri Satya Vennela <[email protected]>; Shradha Gupta <[email protected]>; Saurabh Sengar <[email protected]>; Aditya Garg <[email protected]>; Dipayaan Roy <[email protected]>; Shiraz Saleem <[email protected]>; [email protected]; linux- [email protected]; Paul Rosswurm <[email protected]>
Subject: Re: [EXTERNAL] Re: [PATCH V2,net-next, 1/2] net: mana: Add support for coalesced RX packets on CQE

Quoted 2 lines · On Tue, 13 Jan 2026 15:13:24 +0000 Haiyang Zhang wrote:
Quoted 2 lines
Quoted 2 lines

I get that. What is the logic for combining 4 packets into a single completion? How does it work? Your commit message mentions

"regression

Quoted 3 lines
Quoted 3 lines
Quoted 1 line

on latency" - what is the bound on that regression?

When we received CQE type CQE_RX_COALESCED_4, it's a coalesced CQE.

And in

Quoted 1 line
Quoted 1 line

the CQE OOB, there is an array with 4 PPI elements, with each pkt's

length:

Quoted 3 lines
Quoted 3 lines

oob->ppi[i].pkt_len.

So we read the related WQE and the DMA buffers for the RX pkt

payloads, up

Quoted 2 lines
Quoted 2 lines

to 4.
But, if the coalesced pkts <4, the pkt_len will be 0 after the last

pkt,

Quoted 8 lines
Quoted 3 lines
Quoted 1 line

so we know when to stop reading the WQEs.

And, the coalescing can add up to 2 microseconds into one-way latency.

I am asking you how the _device_ (hypervisor?) decides when to coalesce and when to send a partial CQE (<4 packets in 4 pkt CQE). You are using the coalescing uAPI, so I'm trying to make sure this is the correct API. CQE configuration can also be done via ringparam.

When coalescing is enabled, the device waits for packets which can have the CQE coalesced with previous packet(s). That coalescing process is finished (and a CQE written to the appropriate CQ) when the CQE is filled with 4 pkts, or time expired, or other device specific logic is satisfied.

Thanks,
- Haiyang

maintainer
#10 lore raw

Re: [EXTERNAL] Re: [PATCH V2,net-next, 1/2] net: mana: Add support for coalesced RX packets on CQE

Quoted 7 lines · On Wed, 14 Jan 2026 18:27:50 +0000 Haiyang Zhang wrote:
Quoted 6 lines
Quoted 1 line

And, the coalescing can add up to 2 microseconds into one-way latency.

I am asking you how the _device_ (hypervisor?) decides when to coalesce and when to send a partial CQE (<4 packets in 4 pkt CQE). You are using the coalescing uAPI, so I'm trying to make sure this is the correct API. CQE configuration can also be done via ringparam.

Quoted 5 lines

When coalescing is enabled, the device waits for packets which can have the CQE coalesced with previous packet(s). That coalescing process is finished (and a CQE written to the appropriate CQ) when the CQE is filled with 4 pkts, or time expired, or other device specific logic is satisfied.

See, what I'm afraid is happening here is that you are enabling completion coalescing (how long the device keeps the CQE pending). Which is _not_ what rx_max_coalesced_frames controls for most NICs. For most NICs rx_max_coalesced_frames controls IRQ generation logic.

The NIC first buffers up CQEs for typically single digit usecs, and then once CQE timer exipred and writeback happened it starts an IRQ coalescing timer. Once the IRQ coalescing timer expires IRQ is triggered, which schedules NAPI. (broad strokes, obviously many differences and optimizations exist)

Is my guess correct? Are you controlling CQE coalescing>

Can you control the timeout instead of the frame count?

maintainer
#11 lore raw

RE: [EXTERNAL] Re: [PATCH V2,net-next, 1/2] net: mana: Add support for coalesced RX packets on CQE

Quoted 52 lines

-----Original Message-----
From: Jakub Kicinski <[email protected]>
Sent: Wednesday, January 14, 2026 9:55 PM
To: Haiyang Zhang <[email protected]>
Cc: Haiyang Zhang <[email protected]>; linux- [email protected]; [email protected]; KY Srinivasan <[email protected]>; Wei Liu <[email protected]>; Dexuan Cui <[email protected]>; Long Li <[email protected]>; Andrew Lunn <[email protected]>; David S. Miller <[email protected]>; Eric Dumazet <[email protected]>; Paolo Abeni <[email protected]>; Konstantin Taranov <[email protected]>; Simon Horman <[email protected]>; Erni Sri Satya Vennela <[email protected]>; Shradha Gupta <[email protected]>; Saurabh Sengar <[email protected]>; Aditya Garg <[email protected]>; Dipayaan Roy <[email protected]>; Shiraz Saleem <[email protected]>; [email protected]; linux- [email protected]; Paul Rosswurm <[email protected]>
Subject: Re: [EXTERNAL] Re: [PATCH V2,net-next, 1/2] net: mana: Add support for coalesced RX packets on CQE

Quoted 1 line · On Wed, 14 Jan 2026 18:27:50 +0000 Haiyang Zhang wrote:
Quoted 1 line
Quoted 1 line

And, the coalescing can add up to 2 microseconds into one-way

latency.

Quoted 2 lines
Quoted 2 lines

I am asking you how the _device_ (hypervisor?) decides when to

coalesce

Quoted 1 line
Quoted 1 line

and when to send a partial CQE (<4 packets in 4 pkt CQE). You are

using

Quoted 1 line
Quoted 1 line

the coalescing uAPI, so I'm trying to make sure this is the correct

API.

Quoted 7 lines
Quoted 1 line

CQE configuration can also be done via ringparam.

When coalescing is enabled, the device waits for packets which can have the CQE coalesced with previous packet(s). That coalescing process is finished (and a CQE written to the appropriate CQ) when the CQE is filled with 4 pkts, or time expired, or other device specific logic is satisfied.

See, what I'm afraid is happening here is that you are enabling completion coalescing (how long the device keeps the CQE pending). Which is _not_ what rx_max_coalesced_frames controls for most NICs. For most NICs rx_max_coalesced_frames controls IRQ generation logic.

The NIC first buffers up CQEs for typically single digit usecs, and then once CQE timer exipred and writeback happened it starts an IRQ coalescing timer. Once the IRQ coalescing timer expires IRQ is triggered, which schedules NAPI. (broad strokes, obviously many differences and optimizations exist)

Quoted 3 lines

Is my guess correct? Are you controlling CQE coalescing>

Can you control the timeout instead of the frame count?

Our NIC's timeout value cannot be controlled by driver. Also, the timeout may be changed in future NIC HW.

So, I use the ethtool/rx-frames, which is either 1 or 4 on our NIC, to switch the CQE coalescing feature on/off.

Thanks,
- Haiyang

maintainer
#12 lore raw

Re: [EXTERNAL] Re: [PATCH V2,net-next, 1/2] net: mana: Add support for coalesced RX packets on CQE

Quoted 19 lines · On Thu, 15 Jan 2026 19:57:44 +0000 Haiyang Zhang wrote:
Quoted 19 lines
Quoted 5 lines

When coalescing is enabled, the device waits for packets which can have the CQE coalesced with previous packet(s). That coalescing process is finished (and a CQE written to the appropriate CQ) when the CQE is filled with 4 pkts, or time expired, or other device specific logic is satisfied.

See, what I'm afraid is happening here is that you are enabling completion coalescing (how long the device keeps the CQE pending). Which is _not_ what rx_max_coalesced_frames controls for most NICs. For most NICs rx_max_coalesced_frames controls IRQ generation logic.

The NIC first buffers up CQEs for typically single digit usecs, and then once CQE timer exipred and writeback happened it starts an IRQ coalescing timer. Once the IRQ coalescing timer expires IRQ is triggered, which schedules NAPI. (broad strokes, obviously many differences and optimizations exist)

Is my guess correct? Are you controlling CQE coalescing>

Quoted 7 lines
Quoted 1 line

Can you control the timeout instead of the frame count?

Our NIC's timeout value cannot be controlled by driver. Also, the timeout may be changed in future NIC HW.

So, I use the ethtool/rx-frames, which is either 1 or 4 on our NIC, to switch the CQE coalescing feature on/off.

I feel like this is not the first time I'm having a conversation with you where you are not answering my direct questions, not just one sliver. IDK why you're doing this, but being able to participate in an email exchange is a bare minimum for participating upstream. Please consider this a warning.

If I interpret your reply correctly you are indeed coalescing writeback. You need to add a new param to the uAPI. Please add both size and timeout. Expose the timeout as read only if your device doesn't support controlling it per queue.

maintainer
#13 lore raw

RE: [EXTERNAL] Re: [PATCH V2,net-next, 1/2] net: mana: Add support for coalesced RX packets on CQE

Quoted 51 lines

-----Original Message-----
From: Jakub Kicinski <[email protected]>
Sent: Thursday, January 15, 2026 9:15 PM
To: Haiyang Zhang <[email protected]>
Cc: Haiyang Zhang <[email protected]>; linux- [email protected]; [email protected]; KY Srinivasan <[email protected]>; Wei Liu <[email protected]>; Dexuan Cui <[email protected]>; Long Li <[email protected]>; Andrew Lunn <[email protected]>; David S. Miller <[email protected]>; Eric Dumazet <[email protected]>; Paolo Abeni <[email protected]>; Konstantin Taranov <[email protected]>; Simon Horman <[email protected]>; Erni Sri Satya Vennela <[email protected]>; Shradha Gupta <[email protected]>; Saurabh Sengar <[email protected]>; Aditya Garg <[email protected]>; Dipayaan Roy <[email protected]>; Shiraz Saleem <[email protected]>; [email protected]; linux- [email protected]; Paul Rosswurm <[email protected]>
Subject: Re: [EXTERNAL] Re: [PATCH V2,net-next, 1/2] net: mana: Add support for coalesced RX packets on CQE

Quoted 2 lines · On Thu, 15 Jan 2026 19:57:44 +0000 Haiyang Zhang wrote:
Quoted 2 lines
Quoted 2 lines

When coalescing is enabled, the device waits for packets which can have the CQE coalesced with previous packet(s). That coalescing

process

Quoted 1 line
Quoted 1 line
Quoted 1 line

is finished (and a CQE written to the appropriate CQ) when the CQE

is

Quoted 1 line
Quoted 1 line
Quoted 1 line

filled with 4 pkts, or time expired, or other device specific logic

is

Quoted 20 lines
Quoted 13 lines
Quoted 1 line

satisfied.

See, what I'm afraid is happening here is that you are enabling completion coalescing (how long the device keeps the CQE pending). Which is _not_ what rx_max_coalesced_frames controls for most NICs. For most NICs rx_max_coalesced_frames controls IRQ generation logic.

The NIC first buffers up CQEs for typically single digit usecs, and then once CQE timer exipred and writeback happened it starts an IRQ coalescing timer. Once the IRQ coalescing timer expires IRQ is triggered, which schedules NAPI. (broad strokes, obviously many differences and optimizations exist)

Quoted 3 lines

Is my guess correct? Are you controlling CQE coalescing>

Can you control the timeout instead of the frame count?

Our NIC's timeout value cannot be controlled by driver. Also, the timeout may be changed in future NIC HW.

Quoted 8 lines
Quoted 2 lines

So, I use the ethtool/rx-frames, which is either 1 or 4 on our NIC, to switch the CQE coalescing feature on/off.

I feel like this is not the first time I'm having a conversation with you where you are not answering my direct questions, not just one sliver. IDK why you're doing this, but being able to participate in an email exchange is a bare minimum for participating upstream. Please consider this a warning.

Sure, let me try to reply again -- does this (see below) answer all your questions? And, feel free to ask any further questions, we are willing to collaborate with you and other upstream people at any time :)

Quoted 6 lines

The NIC first buffers up CQEs for typically single digit usecs, and then once CQE timer exipred and writeback happened it starts an IRQ coalescing timer. Once the IRQ coalescing timer expires IRQ is triggered, which schedules NAPI. (broad strokes, obviously many differences and optimizations exist)
Is my guess correct? Are you controlling CQE coalescing?

Yes, it's correct. And we are controlling "CQE coalescing".

Quoted 2 lines

If I interpret your reply correctly you are indeed coalescing writeback.

Yes, we are coalescing CQE writeback.

Quoted 1 line

You need to add a new param to the uAPI.

Since this feature is not common to other NICs, can we use an ethtool private flag instead?
When the flag is set, the CQE coalescing will be enabled and put up to 4 pkts in a CQE.

Quoted 3 lines

Please add both size and
timeout. Expose the timeout as read only if your device doesn't support controlling it per queue.

Does the "size" mean the max pks per CQE (1 or 4)?
The timeout value is not even exposed to driver, and subject to change in the future. Also the HW mechanism is proprietary... So, can we not "expose" the timeout value in "ethtool -c" outputs, because it's not available at driver level?

Thanks,
- Haiyang

maintainer
#14 lore raw

Re: [EXTERNAL] Re: [PATCH V2,net-next, 1/2] net: mana: Add support for coalesced RX packets on CQE

Quoted 4 lines · On Fri, 16 Jan 2026 16:44:33 +0000 Haiyang Zhang wrote:
Quoted 1 line

You need to add a new param to the uAPI.

Since this feature is not common to other NICs, can we use an ethtool private flag instead?

It's extremely common. Descriptor writeback at the granularity of one packet would kill PCIe performance. We just don't have uAPI so NICs either don't expose the knob or "reuse" another coalescing param.

Quoted 8 lines

When the flag is set, the CQE coalescing will be enabled and put up to 4 pkts in a CQE.

Quoted 3 lines

Please add both size and
timeout. Expose the timeout as read only if your device doesn't support controlling it per queue.

Does the "size" mean the max pks per CQE (1 or 4)?

The definition of "size" is always a little funny when it comes to coalescing and ringparam. In Tx does one frame mean one wire frame or one TSO superframe? I wouldn't worry about the exact meaning of size too much. Important thing is that user knows what making this param smaller or larger will do.

Quoted 4 lines

The timeout value is not even exposed to driver, and subject to change in the future. Also the HW mechanism is proprietary... So, can we not "expose" the timeout value in "ethtool -c" outputs, because it's not available at driver level?

Add it to the FW API and have FW send the current value to the driver? You were concerned (in the commit msg) that there's a latency cost, which is fair but I think for 99% of users 2usec is absolutely not detectable (it takes longer for the CPU to wake). So I think it'd be very valuable to the user to understand the order of magnitude of latency we're talking about here.

maintainer
#15 lore raw

RE: [EXTERNAL] Re: [PATCH V2,net-next, 1/2] net: mana: Add support for coalesced RX packets on CQE

Quoted 24 lines

-----Original Message-----
From: Jakub Kicinski <[email protected]>
Sent: Saturday, January 17, 2026 11:59 AM
To: Haiyang Zhang <[email protected]>
Cc: Haiyang Zhang <[email protected]>; linux- [email protected]; [email protected]; KY Srinivasan <[email protected]>; Wei Liu <[email protected]>; Dexuan Cui <[email protected]>; Long Li <[email protected]>; Andrew Lunn <[email protected]>; David S. Miller <[email protected]>; Eric Dumazet <[email protected]>; Paolo Abeni <[email protected]>; Konstantin Taranov <[email protected]>; Simon Horman <[email protected]>; Erni Sri Satya Vennela <[email protected]>; Shradha Gupta <[email protected]>; Saurabh Sengar <[email protected]>; Aditya Garg <[email protected]>; Dipayaan Roy <[email protected]>; Shiraz Saleem <[email protected]>; [email protected]; linux- [email protected]; Paul Rosswurm <[email protected]>
Subject: Re: [EXTERNAL] Re: [PATCH V2,net-next, 1/2] net: mana: Add support for coalesced RX packets on CQE

On Fri, 16 Jan 2026 16:44:33 +0000 Haiyang Zhang wrote:

Quoted 8 lines
Quoted 4 lines
Quoted 1 line

You need to add a new param to the uAPI.

Since this feature is not common to other NICs, can we use an ethtool private flag instead?

It's extremely common. Descriptor writeback at the granularity of one packet would kill PCIe performance. We just don't have uAPI so NICs either don't expose the knob or "reuse" another coalescing param.

I see. So how about adding a new param like below to "ethtool -C"? ethtool -C|--coalesce devname [rx-cqe-coalesce on|off]

Quoted 8 lines
Quoted 5 lines

When the flag is set, the CQE coalescing will be enabled and put up to 4 pkts in a CQE.

Quoted 2 lines

Please add both size and
timeout. Expose the timeout as read only if your device doesn't

support

Quoted 2 lines
Quoted 1 line

controlling it per queue.

Quoted 7 lines
Quoted 1 line

Does the "size" mean the max pks per CQE (1 or 4)?

The definition of "size" is always a little funny when it comes to coalescing and ringparam. In Tx does one frame mean one wire frame or one TSO superframe? I wouldn't worry about the exact meaning of size too much. Important thing is that user knows what making this param smaller or larger will do.

In "ethtool -c" output, add a new value like this?
rx-cqe-frames: (1 or 4 frames/CQE for this NIC)

Quoted 6 lines
Quoted 4 lines

The timeout value is not even exposed to driver, and subject to change in the future. Also the HW mechanism is proprietary... So, can we not "expose" the timeout value in "ethtool -c" outputs, because it's not available at driver level?

Add it to the FW API and have FW send the current value to the driver?

I don't know where is the timeout value in the HW / FW layers. Adding new info to the HW/FW API needs other team's approval, and their work, which will need a complex process and a long time.

Quoted 5 lines

You were concerned (in the commit msg) that there's a latency cost, which is fair but I think for 99% of users 2usec is absolutely not detectable (it takes longer for the CPU to wake). So I think it'd be very valuable to the user to understand the order of magnitude of latency we're talking about here.

For now, may I document the 2us in the patch description? And add a new item to the "ethtool -c" output, like "rx-cqe-usecs", label is as "n/a" for now, while we work out with other teams on the time value API at HW/FW layers? So, this CQE coalescing feature support won't be blocked by this "2usec" info API for a long time?

Thanks,
- Haiyang

maintainer
#16 lore raw

Re: [EXTERNAL] Re: [PATCH V2,net-next, 1/2] net: mana: Add support for coalesced RX packets on CQE

Quoted 9 lines · On Sat, 17 Jan 2026 18:01:18 +0000 Haiyang Zhang wrote:
Quoted 6 lines
Quoted 2 lines

Since this feature is not common to other NICs, can we use an ethtool private flag instead?

It's extremely common. Descriptor writeback at the granularity of one packet would kill PCIe performance. We just don't have uAPI so NICs either don't expose the knob or "reuse" another coalescing param.

I see. So how about adding a new param like below to "ethtool -C"? ethtool -C|--coalesce devname [rx-cqe-coalesce on|off]

I don't think we need on / off, just the params.
If someone needs on / off setting - the size to 1 is basically off.

Quoted 7 lines
Quoted 3 lines
Quoted 3 lines

When the flag is set, the CQE coalescing will be enabled and put up to 4 pkts in a CQE. support
Does the "size" mean the max pks per CQE (1 or 4)?

[...]

In "ethtool -c" output, add a new value like this?
rx-cqe-frames: (1 or 4 frames/CQE for this NIC)

SG

Quoted 17 lines
Quoted 6 lines
Quoted 4 lines

The timeout value is not even exposed to driver, and subject to change in the future. Also the HW mechanism is proprietary... So, can we not "expose" the timeout value in "ethtool -c" outputs, because it's not available at driver level?

Add it to the FW API and have FW send the current value to the driver?

I don't know where is the timeout value in the HW / FW layers. Adding new info to the HW/FW API needs other team's approval, and their work, which will need a complex process and a long time.

Quoted 5 lines

You were concerned (in the commit msg) that there's a latency cost, which is fair but I think for 99% of users 2usec is absolutely not detectable (it takes longer for the CPU to wake). So I think it'd be very valuable to the user to understand the order of magnitude of latency we're talking about here.

Quoted 5 lines

For now, may I document the 2us in the patch description? And add a new item to the "ethtool -c" output, like "rx-cqe-usecs", label is as "n/a" for now, while we work out with other teams on the time value API at HW/FW layers? So, this CQE coalescing feature support won't be blocked by this "2usec" info API for a long time?

Please do it right. We are in no rush upstream. It can't be that hard to add a single API to the FW within a single organization..

maintainer
#17 lore raw

Re: [EXTERNAL] Re: [PATCH V2,net-next, 1/2] net: mana: Add support for coalesced RX packets on CQE

Quoted 31 lines

-----Original Message-----
From: Jakub Kicinski <[email protected]>
Sent: Saturday, January 17, 2026 5:49 PM
To: Haiyang Zhang <[email protected]>
Cc: Haiyang Zhang <[email protected]>; linux- [email protected]; [email protected]; KY Srinivasan <[email protected]>; Wei Liu <[email protected]>; Dexuan Cui <[email protected]>; Long Li <[email protected]>; Andrew Lunn <[email protected]>; David S. Miller <[email protected]>; Eric Dumazet <[email protected]>; Paolo Abeni <[email protected]>; Konstantin Taranov <[email protected]>; Simon Horman <[email protected]>; Erni Sri Satya Vennela <[email protected]>; Shradha Gupta <[email protected]>; Saurabh Sengar <[email protected]>; Aditya Garg <[email protected]>; Dipayaan Roy <[email protected]>; Shiraz Saleem <[email protected]>; [email protected]; linux- [email protected]; Paul Rosswurm <[email protected]>
Subject: Re: [EXTERNAL] Re: [PATCH V2,net-next, 1/2] net: mana: Add support for coalesced RX packets on CQE

Quoted 7 lines · On Sat, 17 Jan 2026 18:01:18 +0000 Haiyang Zhang wrote:
Quoted 6 lines
Quoted 2 lines

Since this feature is not common to other NICs, can we use an ethtool private flag instead?

It's extremely common. Descriptor writeback at the granularity of one packet would kill PCIe performance. We just don't have uAPI so NICs either don't expose the knob or "reuse" another coalescing param.

Quoted 5 lines
Quoted 2 lines

I see. So how about adding a new param like below to "ethtool -C"? ethtool -C|--coalesce devname [rx-cqe-coalesce on|off]

I don't think we need on / off, just the params.
If someone needs on / off setting - the size to 1 is basically off.

Ok --
I will add a numerical param "rx-cqe-frames" to "ethtool -C":
ethtool -C|--coalesce devname [rx-cqe-frames N]
//Accepts 1 or 4 frames/CQE for this NIC

Quoted 10 lines
Quoted 8 lines
Quoted 4 lines
Quoted 4 lines

When the flag is set, the CQE coalescing will be enabled and put up to 4 pkts in a CQE. support
Does the "size" mean the max pks per CQE (1 or 4)?

[...]

In "ethtool -c" output, add a new value like this?
rx-cqe-frames: (1 or 4 frames/CQE for this NIC)

SG

Thanks.

Quoted 19 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line

The timeout value is not even exposed to driver, and subject to

change

Quoted 1 line
Quoted 1 line
Quoted 1 line

in the future. Also the HW mechanism is proprietary... So, can we

not

Quoted 15 lines
Quoted 4 lines
Quoted 2 lines

"expose" the timeout value in "ethtool -c" outputs, because it's not available at driver level?

Add it to the FW API and have FW send the current value to the driver?

I don't know where is the timeout value in the HW / FW layers. Adding new info to the HW/FW API needs other team's approval, and their work, which will need a complex process and a long time.

Quoted 5 lines

You were concerned (in the commit msg) that there's a latency cost, which is fair but I think for 99% of users 2usec is absolutely not detectable (it takes longer for the CPU to wake). So I think it'd be very valuable to the user to understand the order of magnitude of latency we're talking about here.

Quoted 8 lines
Quoted 5 lines

For now, may I document the 2us in the patch description? And add a new item to the "ethtool -c" output, like "rx-cqe-usecs", label is as "n/a" for now, while we work out with other teams on the time value API at HW/FW layers? So, this CQE coalescing feature support won't be blocked by this "2usec" info API for a long time?

Please do it right. We are in no rush upstream. It can't be that hard to add a single API to the FW within a single organization..

I will discuss this with our HW/FW teams.

Thanks,
- Haiyang

maintainer
#18 lore raw

RE: [EXTERNAL] Re: [PATCH V2,net-next, 1/2] net: mana: Add support for coalesced RX packets on CQE

In reply to Jakub Kicinski
Quoted 66 lines

-----Original Message-----
From: Jakub Kicinski <[email protected]>
Sent: Saturday, January 17, 2026 5:49 PM
To: Haiyang Zhang <[email protected]>
Cc: Haiyang Zhang <[email protected]>; linux- [email protected]; [email protected]; KY Srinivasan <[email protected]>; Wei Liu <[email protected]>; Dexuan Cui <[email protected]>; Long Li <[email protected]>; Andrew Lunn <[email protected]>; David S. Miller <[email protected]>; Eric Dumazet <[email protected]>; Paolo Abeni <[email protected]>; Konstantin Taranov <[email protected]>; Simon Horman <[email protected]>; Erni Sri Satya Vennela <[email protected]>; Shradha Gupta <[email protected]>; Saurabh Sengar <[email protected]>; Aditya Garg <[email protected]>; Dipayaan Roy <[email protected]>; Shiraz Saleem <[email protected]>; [email protected]; linux- [email protected]; Paul Rosswurm <[email protected]>
Subject: Re: [EXTERNAL] Re: [PATCH V2,net-next, 1/2] net: mana: Add support for coalesced RX packets on CQE

Quoted 9 lines · On Sat, 17 Jan 2026 18:01:18 +0000 Haiyang Zhang wrote:
Quoted 6 lines
Quoted 2 lines

Since this feature is not common to other NICs, can we use an ethtool private flag instead?

It's extremely common. Descriptor writeback at the granularity of one packet would kill PCIe performance. We just don't have uAPI so NICs either don't expose the knob or "reuse" another coalescing param.

I see. So how about adding a new param like below to "ethtool -C"? ethtool -C|--coalesce devname [rx-cqe-coalesce on|off]

I don't think we need on / off, just the params.
If someone needs on / off setting - the size to 1 is basically off.

Quoted 7 lines
Quoted 3 lines
Quoted 3 lines

When the flag is set, the CQE coalescing will be enabled and put up to 4 pkts in a CQE. support
Does the "size" mean the max pks per CQE (1 or 4)?

[...]

In "ethtool -c" output, add a new value like this?
rx-cqe-frames: (1 or 4 frames/CQE for this NIC)

SG

Quoted 1 line
Quoted 1 line
Quoted 1 line

The timeout value is not even exposed to driver, and subject to

change

Quoted 1 line
Quoted 1 line
Quoted 1 line

in the future. Also the HW mechanism is proprietary... So, can we

not

Quoted 15 lines
Quoted 4 lines
Quoted 2 lines

"expose" the timeout value in "ethtool -c" outputs, because it's not available at driver level?

Add it to the FW API and have FW send the current value to the driver?

I don't know where is the timeout value in the HW / FW layers. Adding new info to the HW/FW API needs other team's approval, and their work, which will need a complex process and a long time.

Quoted 5 lines

You were concerned (in the commit msg) that there's a latency cost, which is fair but I think for 99% of users 2usec is absolutely not detectable (it takes longer for the CPU to wake). So I think it'd be very valuable to the user to understand the order of magnitude of latency we're talking about here.

Quoted 8 lines
Quoted 5 lines

For now, may I document the 2us in the patch description? And add a new item to the "ethtool -c" output, like "rx-cqe-usecs", label is as "n/a" for now, while we work out with other teams on the time value API at HW/FW layers? So, this CQE coalescing feature support won't be blocked by this "2usec" info API for a long time?

Please do it right. We are in no rush upstream. It can't be that hard to add a single API to the FW within a single organization..

I have sent out a patch to add two parameters for ethtool:
COALESCE_RX_CQE_FRAMES/NSECS

I will send out ethtool user cmd patch, and driver patches later, after the new parameters are added to kernel.

Thanks,
- Haiyang

#19 lore raw

PATCH v2 2/2 net: mana: Add ethtool counters for RX CQEs in coalesced type

In reply to Haiyang Zhang

From: Haiyang Zhang <[email protected]>

For RX CQEs with type CQE_RX_COALESCED_4, to measure the coalescing efficiency, add counters to count how many contains 2, 3, 4 packets respectively.
Also, add a counter for the error case of first packet with length == 0.

---
 drivers/net/ethernet/microsoft/mana/mana_en.c | 25 +++++++++++++++++--
 .../ethernet/microsoft/mana/mana_ethtool.c    | 17 ++++++++++---
 include/net/mana/mana.h                       | 10 +++++---
 3 files changed, 42 insertions(+), 10 deletions(-)
drivers/net/ethernet/microsoft/mana/mana_en.c +23 -2
index a46a1adf83bc..78824567d80b 100644
@@ -2083,8 +2083,22 @@ static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq,
 
 nextpkt:
 	pktlen = oob->ppi[i].pkt_len;
-	if (pktlen == 0)
+	if (pktlen == 0) {
+		/* Collect coalesced CQE count based on packets processed.
+		 * Coalesced CQEs have at least 2 packets, so index is i - 2.
+		 */
+		if (i > 1) {
+			u64_stats_update_begin(&rxq->stats.syncp);
+			rxq->stats.coalesced_cqe[i - 2]++;
+			u64_stats_update_end(&rxq->stats.syncp);
+		} else if (i == 0) {
+			/* Error case stat */
+			u64_stats_update_begin(&rxq->stats.syncp);
+			rxq->stats.pkt_len0_err++;
+			u64_stats_update_end(&rxq->stats.syncp);
+		}
 		return;
+	}
 
 	curr = rxq->buf_index;
 	rxbuf_oob = &rxq->rx_oobs[curr];
@@ -2102,8 +2116,15 @@ static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq,
 
 	mana_post_pkt_rxq(rxq);
 
-	if (coalesced && (++i < MANA_RXCOMP_OOB_NUM_PPI))
+	if (!coalesced)
+		return;
+
+	if (++i < MANA_RXCOMP_OOB_NUM_PPI)
 		goto nextpkt;
+
+	u64_stats_update_begin(&rxq->stats.syncp);
+	rxq->stats.coalesced_cqe[MANA_RXCOMP_OOB_NUM_PPI - 2]++;
+	u64_stats_update_end(&rxq->stats.syncp);
 }
 
 static void mana_poll_rx_cq(struct mana_cq *cq)
drivers/net/ethernet/microsoft/mana/mana_ethtool.c +13 -4
index b2b9bfb50396..635796bfdaf1 100644
@@ -20,8 +20,6 @@ static const struct mana_stats_desc mana_eth_stats[] = {
 					tx_cqe_unknown_type)},
 	{"tx_linear_pkt_cnt", offsetof(struct mana_ethtool_stats,
 				       tx_linear_pkt_cnt)},
-	{"rx_coalesced_err", offsetof(struct mana_ethtool_stats,
-					rx_coalesced_err)},
 	{"rx_cqe_unknown_type", offsetof(struct mana_ethtool_stats,
 					rx_cqe_unknown_type)},
 };
@@ -151,7 +149,7 @@ static void mana_get_strings(struct net_device *ndev, u32 stringset, u8 *data)
 {
 	struct mana_port_context *apc = netdev_priv(ndev);
 	unsigned int num_queues = apc->num_queues;
-	int i;
+	int i, j;
 
 	if (stringset != ETH_SS_STATS)
 		return;
@@ -170,6 +168,9 @@ static void mana_get_strings(struct net_device *ndev, u32 stringset, u8 *data)
 		ethtool_sprintf(&data, "rx_%d_xdp_drop", i);
 		ethtool_sprintf(&data, "rx_%d_xdp_tx", i);
 		ethtool_sprintf(&data, "rx_%d_xdp_redirect", i);
+		ethtool_sprintf(&data, "rx_%d_pkt_len0_err", i);
+		for (j = 0; j < MANA_RXCOMP_OOB_NUM_PPI - 1; j++)
+			ethtool_sprintf(&data, "rx_%d_coalesced_cqe_%d", i, j + 2);
 	}
 
 	for (i = 0; i < num_queues; i++) {
@@ -203,6 +204,8 @@ static void mana_get_ethtool_stats(struct net_device *ndev,
 	u64 xdp_xmit;
 	u64 xdp_drop;
 	u64 xdp_tx;
+	u64 pkt_len0_err;
+	u64 coalesced_cqe[MANA_RXCOMP_OOB_NUM_PPI - 1];
 	u64 tso_packets;
 	u64 tso_bytes;
 	u64 tso_inner_packets;
@@ -211,7 +214,7 @@ static void mana_get_ethtool_stats(struct net_device *ndev,
 	u64 short_pkt_fmt;
 	u64 csum_partial;
 	u64 mana_map_err;
-	int q, i = 0;
+	int q, i = 0, j;
 
 	if (!apc->port_is_up)
 		return;
@@ -241,6 +244,9 @@ static void mana_get_ethtool_stats(struct net_device *ndev,
 			xdp_drop = rx_stats->xdp_drop;
 			xdp_tx = rx_stats->xdp_tx;
 			xdp_redirect = rx_stats->xdp_redirect;
+			pkt_len0_err = rx_stats->pkt_len0_err;
+			for (j = 0; j < MANA_RXCOMP_OOB_NUM_PPI - 1; j++)
+				coalesced_cqe[j] = rx_stats->coalesced_cqe[j];
 		} while (u64_stats_fetch_retry(&rx_stats->syncp, start));
 
 		data[i++] = packets;
@@ -248,6 +254,9 @@ static void mana_get_ethtool_stats(struct net_device *ndev,
 		data[i++] = xdp_drop;
 		data[i++] = xdp_tx;
 		data[i++] = xdp_redirect;
+		data[i++] = pkt_len0_err;
+		for (j = 0; j < MANA_RXCOMP_OOB_NUM_PPI - 1; j++)
+			data[i++] = coalesced_cqe[j];
 	}
 
 	for (q = 0; q < num_queues; q++) {
include/net/mana/mana.h +6 -4
index 51d26ebeff6c..f8dd19860103 100644
@@ -61,8 +61,11 @@ enum TRI_STATE {
 
 #define MAX_PORTS_IN_MANA_DEV 256
 
+/* Maximum number of packets per coalesced CQE */
+#define MANA_RXCOMP_OOB_NUM_PPI 4
+
 /* Update this count whenever the respective structures are changed */
-#define MANA_STATS_RX_COUNT 5
+#define MANA_STATS_RX_COUNT (6 + MANA_RXCOMP_OOB_NUM_PPI - 1)
 #define MANA_STATS_TX_COUNT 11
 
 #define MANA_RX_FRAG_ALIGNMENT 64
@@ -73,6 +76,8 @@ struct mana_stats_rx {
 	u64 xdp_drop;
 	u64 xdp_tx;
 	u64 xdp_redirect;
+	u64 pkt_len0_err;
+	u64 coalesced_cqe[MANA_RXCOMP_OOB_NUM_PPI - 1];
 	struct u64_stats_sync syncp;
 };
 
@@ -227,8 +232,6 @@ struct mana_rxcomp_perpkt_info {
 	u32 pkt_hash;
 }; /* HW DATA */
 
-#define MANA_RXCOMP_OOB_NUM_PPI 4
-
 /* Receive completion OOB */
 struct mana_rxcomp_oob {
 	struct mana_cqe_header cqe_hdr;
@@ -378,7 +381,6 @@ struct mana_ethtool_stats {
 	u64 tx_cqe_err;
 	u64 tx_cqe_unknown_type;
 	u64 tx_linear_pkt_cnt;
-	u64 rx_coalesced_err;
 	u64 rx_cqe_unknown_type;
 };
 
Signature · 1 line
2.34.1
Reviewed-by on v2 2/2
#20 lore raw

RE: [PATCH V2,net-next, 2/2] net: mana: Add ethtool counters for RX CQEs in coalesced type

Quoted 11 lines

Subject: [PATCH V2,net-next, 2/2] net: mana: Add ethtool counters for RX CQEs in coalesced type

From: Haiyang Zhang <[email protected]>

For RX CQEs with type CQE_RX_COALESCED_4, to measure the coalescing efficiency, add counters to count how many contains 2, 3, 4 packets respectively.
Also, add a counter for the error case of first packet with length == 0.

Quoted 179 lines
---
 drivers/net/ethernet/microsoft/mana/mana_en.c | 25
+++++++++++++++++--
 .../ethernet/microsoft/mana/mana_ethtool.c    | 17 ++++++++++---
 include/net/mana/mana.h                       | 10 +++++---
 3 files changed, 42 insertions(+), 10 deletions(-)
drivers/net/ethernet/microsoft/mana/mana_en.c +15 -1
index a46a1adf83bc..78824567d80b 100644
@@ -2083,8 +2083,22 @@ static void mana_process_rx_cqe(struct
 mana_rxq *rxq, struct mana_cq *cq,
 
 nextpkt:
 	pktlen = oob->ppi[i].pkt_len;
-	if (pktlen == 0)
+	if (pktlen == 0) {
+		/* Collect coalesced CQE count based on packets processed.
+		 * Coalesced CQEs have at least 2 packets, so index is i - 2.
+		 */
+		if (i > 1) {
+			u64_stats_update_begin(&rxq->stats.syncp);
+			rxq->stats.coalesced_cqe[i - 2]++;
+			u64_stats_update_end(&rxq->stats.syncp);
+		} else if (i == 0) {
+			/* Error case stat */
+			u64_stats_update_begin(&rxq->stats.syncp);
+			rxq->stats.pkt_len0_err++;
+			u64_stats_update_end(&rxq->stats.syncp);
+		}
 		return;
+	}
 
 	curr = rxq->buf_index;
drivers/net/ethernet/microsoft/mana/mana_en.c +8 -1
 	rxbuf_oob = &rxq->rx_oobs[curr];
@@ -2102,8 +2116,15 @@ static void mana_process_rx_cqe(struct
 mana_rxq *rxq, struct mana_cq *cq,
 
 	mana_post_pkt_rxq(rxq);
 
-	if (coalesced && (++i < MANA_RXCOMP_OOB_NUM_PPI))
+	if (!coalesced)
+		return;
+
+	if (++i < MANA_RXCOMP_OOB_NUM_PPI)
 		goto nextpkt;
+
+	u64_stats_update_begin(&rxq->stats.syncp);
+	rxq->stats.coalesced_cqe[MANA_RXCOMP_OOB_NUM_PPI - 2]++;
+	u64_stats_update_end(&rxq->stats.syncp);
 }
 

static void mana_poll_rx_cq(struct mana_cq *cq) diff --git
a/drivers/net/ethernet/microsoft/mana/mana_ethtool.c b/drivers/net/ethernet/microsoft/mana/mana_ethtool.c index b2b9bfb50396..635796bfdaf1 100644

drivers/net/ethernet/microsoft/mana/mana_ethtool.c +0 -2
@@ -20,8 +20,6 @@ static const struct mana_stats_desc mana_eth_stats[] =
 {
 					tx_cqe_unknown_type)},
 	{"tx_linear_pkt_cnt", offsetof(struct mana_ethtool_stats,
 				       tx_linear_pkt_cnt)},
-	{"rx_coalesced_err", offsetof(struct mana_ethtool_stats,
-					rx_coalesced_err)},
 	{"rx_cqe_unknown_type", offsetof(struct mana_ethtool_stats,
 					rx_cqe_unknown_type)},

};

drivers/net/ethernet/microsoft/mana/mana_ethtool.c +4 -1
@@ -151,7 +149,7 @@ static void mana_get_strings(struct net_device *ndev,
 u32 stringset, u8 *data)  {
 	struct mana_port_context *apc = netdev_priv(ndev);
 	unsigned int num_queues = apc->num_queues;
-	int i;
+	int i, j;
 
 	if (stringset != ETH_SS_STATS)
 		return;
@@ -170,6 +168,9 @@ static void mana_get_strings(struct net_device *ndev,
 u32 stringset, u8 *data)
 		ethtool_sprintf(&data, "rx_%d_xdp_drop", i);
 		ethtool_sprintf(&data, "rx_%d_xdp_tx", i);
 		ethtool_sprintf(&data, "rx_%d_xdp_redirect", i);
+		ethtool_sprintf(&data, "rx_%d_pkt_len0_err", i);
+		for (j = 0; j < MANA_RXCOMP_OOB_NUM_PPI - 1; j++)
+			ethtool_sprintf(&data, "rx_%d_coalesced_cqe_%d", i,
 j + 2);
 	}
drivers/net/ethernet/microsoft/mana/mana_ethtool.c +2 -0
 	for (i = 0; i < num_queues; i++) {
@@ -203,6 +204,8 @@ static void mana_get_ethtool_stats(struct net_device
 *ndev,
 	u64 xdp_xmit;
 	u64 xdp_drop;
 	u64 xdp_tx;
+	u64 pkt_len0_err;
+	u64 coalesced_cqe[MANA_RXCOMP_OOB_NUM_PPI - 1];
 	u64 tso_packets;
 	u64 tso_bytes;
drivers/net/ethernet/microsoft/mana/mana_ethtool.c +1 -1
 	u64 tso_inner_packets;
@@ -211,7 +214,7 @@ static void mana_get_ethtool_stats(struct net_device
 *ndev,
 	u64 short_pkt_fmt;
 	u64 csum_partial;
 	u64 mana_map_err;
-	int q, i = 0;
+	int q, i = 0, j;
 
 	if (!apc->port_is_up)
drivers/net/ethernet/microsoft/mana/mana_ethtool.c +3 -0
 		return;
@@ -241,6 +244,9 @@ static void mana_get_ethtool_stats(struct net_device
 *ndev,
 			xdp_drop = rx_stats->xdp_drop;
 			xdp_tx = rx_stats->xdp_tx;
 			xdp_redirect = rx_stats->xdp_redirect;
+			pkt_len0_err = rx_stats->pkt_len0_err;
+			for (j = 0; j < MANA_RXCOMP_OOB_NUM_PPI - 1;
 j++)
+				coalesced_cqe[j] = rx_stats->coalesced_cqe[j];
 		} while (u64_stats_fetch_retry(&rx_stats->syncp, start));
drivers/net/ethernet/microsoft/mana/mana_ethtool.c +3 -0
 		data[i++] = packets;
@@ -248,6 +254,9 @@ static void mana_get_ethtool_stats(struct net_device
 *ndev,
 		data[i++] = xdp_drop;
 		data[i++] = xdp_tx;
 		data[i++] = xdp_redirect;
+		data[i++] = pkt_len0_err;
+		for (j = 0; j < MANA_RXCOMP_OOB_NUM_PPI - 1; j++)
+			data[i++] = coalesced_cqe[j];
 	}
 
drivers/net/ethernet/microsoft/mana/mana_ethtool.c +0 -0
 	for (q = 0; q < num_queues; q++) {
include/net/mana/mana.h index renamed from include/net/mana/mana.h +0 -0

51d26ebeff6c..f8dd19860103 100644

include/net/mana/mana.h +6 -4
@@ -61,8 +61,11 @@ enum TRI_STATE {
 
 #define MAX_PORTS_IN_MANA_DEV 256
 
+/* Maximum number of packets per coalesced CQE */ #define
+MANA_RXCOMP_OOB_NUM_PPI 4
+
 /* Update this count whenever the respective structures are changed */ -
 #define MANA_STATS_RX_COUNT 5
+#define MANA_STATS_RX_COUNT (6 + MANA_RXCOMP_OOB_NUM_PPI - 1)
 #define MANA_STATS_TX_COUNT 11
 
 #define MANA_RX_FRAG_ALIGNMENT 64
@@ -73,6 +76,8 @@ struct mana_stats_rx {
 	u64 xdp_drop;
 	u64 xdp_tx;
 	u64 xdp_redirect;
+	u64 pkt_len0_err;
+	u64 coalesced_cqe[MANA_RXCOMP_OOB_NUM_PPI - 1];
 	struct u64_stats_sync syncp;
 };
 
@@ -227,8 +232,6 @@ struct mana_rxcomp_perpkt_info {
 	u32 pkt_hash;
 }; /* HW DATA */
 
-#define MANA_RXCOMP_OOB_NUM_PPI 4
-
 /* Receive completion OOB */
 struct mana_rxcomp_oob {
 	struct mana_cqe_header cqe_hdr;
@@ -378,7 +381,6 @@ struct mana_ethtool_stats {
 	u64 tx_cqe_err;
 	u64 tx_cqe_unknown_type;
 	u64 tx_linear_pkt_cnt;
-	u64 rx_coalesced_err;
 	u64 rx_cqe_unknown_type;
 };
 
--

2.34.1

maintainer
#21 lore raw

Re: [PATCH V2,net-next, 2/2] net: mana: Add ethtool counters for RX CQEs in coalesced type

In reply to Haiyang Zhang
Quoted 10 lines · On Tue, 6 Jan 2026 12:46:47 -0800 Haiyang Zhang wrote:
include/net/mana/mana.h +0 -2
@@ -227,8 +232,6 @@ struct mana_rxcomp_perpkt_info {
 	u32 pkt_hash;
 }; /* HW DATA */
 
-#define MANA_RXCOMP_OOB_NUM_PPI 4
-
 /* Receive completion OOB */
 struct mana_rxcomp_oob {
 	struct mana_cqe_header cqe_hdr;

include/net/mana/mana.h struct mana_ethtool_stats { · line 385

Quoted 6 lines
include/net/mana/mana.h +0 -1
@@ -378,7 +381,6 @@ struct mana_ethtool_stats {
 	u64 tx_cqe_err;
 	u64 tx_cqe_unknown_type;
 	u64 tx_linear_pkt_cnt;
-	u64 rx_coalesced_err;
 	u64 rx_cqe_unknown_type;
 };

This should be deleted in the previous patch already

maintainer
#22 lore raw

RE: [EXTERNAL] Re: [PATCH V2,net-next, 2/2] net: mana: Add ethtool counters for RX CQEs in coalesced type

Quoted 28 lines

-----Original Message-----
From: Jakub Kicinski <[email protected]>
Sent: Friday, January 9, 2026 8:56 PM
To: Haiyang Zhang <[email protected]>
Cc: [email protected]; [email protected]; KY Srinivasan <[email protected]>; Haiyang Zhang <[email protected]>; Wei Liu <[email protected]>; Dexuan Cui <[email protected]>; Long Li <[email protected]>; Andrew Lunn <[email protected]>; David S. Miller <[email protected]>; Eric Dumazet <[email protected]>; Paolo Abeni <[email protected]>; Konstantin Taranov <[email protected]>; Simon Horman <[email protected]>; Erni Sri Satya Vennela <[email protected]>; Shradha Gupta <[email protected]>; Saurabh Sengar <[email protected]>; Aditya Garg <[email protected]>; Dipayaan Roy <[email protected]>; Shiraz Saleem <[email protected]>; [email protected]; linux- [email protected]; Paul Rosswurm <[email protected]>
Subject: [EXTERNAL] Re: [PATCH V2,net-next, 2/2] net: mana: Add ethtool counters for RX CQEs in coalesced type

Quoted 4 lines · On Tue, 6 Jan 2026 12:46:47 -0800 Haiyang Zhang wrote:
include/net/mana/mana.h +0 -0
@@ -227,8 +232,6 @@ struct mana_rxcomp_perpkt_info {
 	u32 pkt_hash;
 }; /* HW DATA */
 
Quoted 14 lines

include/net/mana/mana.h struct mana_ethtool_stats { · line 385

Quoted 12 lines
include/net/mana/mana.h +0 -3
-#define MANA_RXCOMP_OOB_NUM_PPI 4
-
 /* Receive completion OOB */
 struct mana_rxcomp_oob {
 	struct mana_cqe_header cqe_hdr;
@@ -378,7 +381,6 @@ struct mana_ethtool_stats {
 	u64 tx_cqe_err;
 	u64 tx_cqe_unknown_type;
 	u64 tx_linear_pkt_cnt;
-	u64 rx_coalesced_err;
 	u64 rx_cqe_unknown_type;
 };

This should be deleted in the previous patch already

Will do.

- Haiyang