Skip to contentSkip to conversation

net: mana: Add Interrupt Moderation support

PATCH Superseded Haiyang Zhang 4 participants 4 messages Jun 4 – Jun 11

Subsystems (5)

v2 of 5 Superseded Latest is v5 →

lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 net: mana: Add Interrupt Moderation support

From: Haiyang Zhang <[email protected]>

Add Static and Dynamic Interrupt Moderation (DIM) support for Rx and Tx.
Update queue creation procedure with new data struct with the related settings.
Add functions to collect stat for DIM, and workers to update DIM data and settings.
Update ethtool handler to get/set the moderation settings from a user. By default, adaptive-rx/tx (DIM) are enabled.

---
v2:
Updated with comments from Jedrzej.

---
 drivers/net/ethernet/microsoft/Kconfig        |   1 +
 .../net/ethernet/microsoft/mana/gdma_main.c   |  29 ++++
 drivers/net/ethernet/microsoft/mana/mana_en.c | 105 +++++++++++++-
 .../ethernet/microsoft/mana/mana_ethtool.c    | 129 +++++++++++++++++-
 include/net/mana/gdma.h                       |  24 +++-
 include/net/mana/mana.h                       |  45 ++++++
 6 files changed, 325 insertions(+), 8 deletions(-)
drivers/net/ethernet/microsoft/Kconfig +1 -0
index 3f36ee6a8ece..e9be18c92ca5 100644
@@ -21,6 +21,7 @@ config MICROSOFT_MANA
 	depends on X86_64 || (ARM64 && !CPU_BIG_ENDIAN)
 	depends on PCI_HYPERV
 	select AUXILIARY_BUS
+	select DIMLIB
 	select PAGE_POOL
 	select NET_SHAPER
 	help
drivers/net/ethernet/microsoft/mana/gdma_main.c +29 -0
index 712a0881d720..9d7d39f7786a 100644
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /* Copyright (c) 2021, Microsoft Corporation. */
 
+#include <linux/bitfield.h>
 #include <linux/debugfs.h>
 #include <linux/module.h>
 #include <linux/pci.h>
@@ -405,6 +406,7 @@ static int mana_gd_disable_queue(struct gdma_queue *queue)
 #define DOORBELL_OFFSET_RQ	0x400
 #define DOORBELL_OFFSET_CQ	0x800
 #define DOORBELL_OFFSET_EQ	0xFF8
+#define DOORBELL_OFFSET_DIM	0x820
 
 static void mana_gd_ring_doorbell(struct gdma_context *gc, u32 db_index,
 				  enum gdma_queue_type q_type, u32 qid,
@@ -445,6 +447,16 @@ static void mana_gd_ring_doorbell(struct gdma_context *gc, u32 db_index,
 		addr += DOORBELL_OFFSET_SQ;
 		break;
 
+	case GDMA_DIM:
+		e.dim.id = qid;
+		e.dim.mod_usec = FIELD_GET(MANA_INTR_MODR_USEC_MAX, tail_ptr);
+		e.dim.mod_usec_vld = !!(tail_ptr & MANA_INTR_MODR_USEC_VLD);
+		e.dim.mod_comps = FIELD_GET(MANA_INTR_MODR_COMP_MASK, tail_ptr);
+		e.dim.mod_comps_vld = num_req;
+
+		addr += DOORBELL_OFFSET_DIM;
+		break;
+
 	default:
 		WARN_ON(1);
 		return;
@@ -479,6 +491,23 @@ void mana_gd_ring_cq(struct gdma_queue *cq, u8 arm_bit)
 }
 EXPORT_SYMBOL_NS(mana_gd_ring_cq, "NET_MANA");
 
+void mana_gd_ring_dim(struct gdma_queue *cq, u32 mod_usec, bool mod_usec_vld,
+		      u32 mod_comps, bool mod_comps_vld)
+{
+	struct gdma_context *gc = cq->gdma_dev->gdma_context;
+	u32 dim_val;
+
+	/* Convert the DIM values to doorbell parameters */
+	dim_val = FIELD_PREP(MANA_INTR_MODR_USEC_MAX, mod_usec) |
+		  FIELD_PREP(MANA_INTR_MODR_COMP_MASK, mod_comps);
+	if (mod_usec_vld)
+		dim_val |= MANA_INTR_MODR_USEC_VLD;
+
+	mana_gd_ring_doorbell(gc, cq->gdma_dev->doorbell, GDMA_DIM, cq->id,
+			      dim_val, mod_comps_vld);
+}
+EXPORT_SYMBOL_NS(mana_gd_ring_dim, "NET_MANA");
+
 #define MANA_SERVICE_PERIOD 10
 
 static void mana_serv_rescan(struct pci_dev *pdev)
drivers/net/ethernet/microsoft/mana/mana_en.c +104 -1

177 lines not on this page · raw message

drivers/net/ethernet/microsoft/mana/mana_ethtool.c +123 -6

162 lines not on this page · raw message

include/net/mana/gdma.h +23 -1
index 70d62bc32837..86d58b0150c7 100644
@@ -47,6 +47,7 @@ enum gdma_queue_type {
 	GDMA_RQ,
 	GDMA_CQ,
 	GDMA_EQ,
+	GDMA_DIM,
 };
 
 enum gdma_work_request_flags {
@@ -126,6 +127,17 @@ union gdma_doorbell_entry {
 		u64 tail_ptr	: 31;
 		u64 arm		: 1;
 	} eq;
+
+	struct {
+		u64 id           : 24;
+		u64 reserved     : 8;
+		u64 mod_usec     : 10;
+		u64 reserve1     : 5;
+		u64 mod_usec_vld : 1;
+		u64 mod_comps    : 8;
+		u64 reserve2     : 7;
+		u64 mod_comps_vld: 1;
+	} dim;
 }; /* HW DATA */
 
 struct gdma_msg_hdr {
@@ -484,6 +496,9 @@ void mana_gd_ring_cq(struct gdma_queue *cq, u8 arm_bit);
 
 int mana_schedule_serv_work(struct gdma_context *gc, enum gdma_eqe_type type);
 
+void mana_gd_ring_dim(struct gdma_queue *cq, u32 mod_usec, bool mod_usec_vld,
+		      u32 mod_comps, bool mod_comps_vld);
+
 struct gdma_wqe {
 	u32 reserved	:24;
 	u32 last_vbytes	:8;
@@ -629,6 +644,9 @@ enum {
 /* Driver supports self recovery on Hardware Channel timeouts */
 #define GDMA_DRV_CAP_FLAG_1_HWC_TIMEOUT_RECOVERY BIT(25)
 
+/* Driver supports dynamic interrupt moderation - DIM */
+#define GDMA_DRV_CAP_FLAG_1_DYN_INTERRUPT_MODERATION BIT(28)
+
 #define GDMA_DRV_CAP_FLAGS1 \
 	(GDMA_DRV_CAP_FLAG_1_EQ_SHARING_MULTI_VPORT | \
 	 GDMA_DRV_CAP_FLAG_1_NAPI_WKDONE_FIX | \
@@ -643,7 +661,8 @@ enum {
 	 GDMA_DRV_CAP_FLAG_1_SKB_LINEARIZE | \
 	 GDMA_DRV_CAP_FLAG_1_PROBE_RECOVERY | \
 	 GDMA_DRV_CAP_FLAG_1_HANDLE_STALL_SQ_RECOVERY | \
-	 GDMA_DRV_CAP_FLAG_1_HWC_TIMEOUT_RECOVERY)
+	 GDMA_DRV_CAP_FLAG_1_HWC_TIMEOUT_RECOVERY | \
+	 GDMA_DRV_CAP_FLAG_1_DYN_INTERRUPT_MODERATION)
 
 #define GDMA_DRV_CAP_FLAGS2 0
 
@@ -679,6 +698,9 @@ struct gdma_verify_ver_req {
 	u8 os_ver_str4[128];
 }; /* HW DATA */
 
+/* HW supports dynamic interrupt moderation - DIM */
+#define GDMA_PF_CAP_FLAG_1_DYN_INTERRUPT_MODERATION BIT(15)
+
 struct gdma_verify_ver_resp {
 	struct gdma_resp_hdr hdr;
 	u64 gdma_protocol_ver;
include/net/mana/mana.h +45 -0

87 lines not on this page · raw message

Signature · 1 line
2.34.1
#1 lore raw
Thank you for your contribution! Sashiko AI review found 3 potential issue(s) to consider: - [High] Use-after-free and workqueue corruption due to a race between NAPI polling and interface teardown.…

Thank you for your contribution! Sashiko AI review found 3 potential issue(s) to consider:
- [High] Use-after-free and workqueue corruption due to a race between NAPI polling and interface teardown.
- [High] Unconditional use of GDMA_MESSAGE_V3 breaks backward compatibility with older firmware.
- [Medium] Static interrupt moderation is silently disabled if only one parameter is configured. --

commit ec13708996b121633f56552fac3f0ceba004bf71
Author: Haiyang Zhang <[email protected]>

net: mana: Add Interrupt Moderation support

This commit adds Static and Dynamic Interrupt Moderation (DIM) support for Rx and Tx. It updates the queue creation procedure with new data structures and settings, adds statistics collection functions for DIM, and introduces workers to update DIM settings. It also hooks into ethtool to allow configuration.

drivers/net/ethernet/microsoft/mana/mana_en.c int mana_create_wq_obj(struct mana_port_context *apc, · line 1557

Quoted 12 lines
drivers/net/ethernet/microsoft/mana/mana_en.c +3 -0
index db14357d3732e..b1e0c444f4144 100644
@@ -1551,6 +1551,9 @@ int mana_create_wq_obj(struct mana_port_context *apc,
 
 	mana_gd_init_req_hdr(&req.hdr, MANA_CREATE_WQ_OBJ,
 			     sizeof(req), sizeof(resp));
+
+	req.hdr.req.msg_version = GDMA_MESSAGE_V3;
+	req.hdr.resp.msg_version = GDMA_MESSAGE_V2;
 	req.vport = vport;

[Severity: High]
Will this break initialization on older firmware? We unconditionally set msg_version to GDMA_MESSAGE_V3 here without checking if the hardware actually supports it.

If an older Azure host without the GDMA_PF_CAP_FLAG_1_DYN_INTERRUPT_MODERATION capability rejects the V3 request, could this cause queue creation to fail and result in a complete loss of network connectivity?

[ ... ]

drivers/net/ethernet/microsoft/mana/mana_en.c static int mana_cq_handler(void *context, struct gdma_queue *gdma_queue) · line 2353

Quoted 12 lines
drivers/net/ethernet/microsoft/mana/mana_en.c +7 -1
@@ -2274,7 +2344,13 @@ static int mana_cq_handler(void *context, struct gdma_queue *gdma_queue)
 	if (w < cq->budget) {
 		mana_gd_ring_cq(gdma_queue, SET_ARM_BIT);
 		cq->work_done_since_doorbell = 0;
-		napi_complete_done(&cq->napi, w);
+
+		if (napi_complete_done(&cq->napi, w)) {
+			if (cq->type == MANA_CQ_TYPE_RX)
+				mana_update_rx_dim(cq);
+			else
+				mana_update_tx_dim(cq);
+		}

[Severity: High]
Is there a race condition here during teardown? If napi_complete_done() clears the scheduling state, a concurrent teardown thread in mana_destroy_rxq() can return from napi_disable_locked() and proceed to call cancel_work_sync().

Because mana_update_rx_dim() hasn't been called yet to schedule the work, cancel_work_sync() would return immediately and the queue memory would be freed. When the NAPI thread resumes and calls mana_update_rx_dim(), could this lead to a use-after-free or corruption of the workqueue linked list by scheduling work on freed memory?

[ ... ]

drivers/net/ethernet/microsoft/mana/mana_en.c static struct mana_rxq *mana_create_rxq(struct mana_port_context *apc, · line 2875

Quoted 8 lines
drivers/net/ethernet/microsoft/mana/mana_en.c +4 -0
@@ -2783,6 +2869,10 @@ static struct mana_rxq *mana_create_rxq(struct mana_port_context *apc,
 	cq_spec.queue_size = cq->gdma_cq->queue_size;
 	cq_spec.modr_ctx_id = 0;
 	cq_spec.attached_eq = cq->gdma_cq->cq.parent->id;
+	cq_spec.req_cq_moderation = apc->rx_dim_enabled ||
+		(apc->intr_modr_rx_usec && apc->intr_modr_rx_comp);
+	cq_spec.cq_moderation_usec = apc->intr_modr_rx_usec;
+	cq_spec.cq_moderation_comp = apc->intr_modr_rx_comp;

[Severity: Medium]
Does this logic unintentionally ignore user configurations where only one parameter is non-zero?

In mana_set_coalesce(), ethtool allows setting rx-usecs without setting rx-frames. If a user sets only time-based moderation (e.g., rx-usecs 50, rx-frames 0), the condition here evaluates to false and silently disables static moderation without returning an error via ethtool.

Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=1
maintainer
#2 lore raw
In reply to Haiyang Zhang

drivers/net/ethernet/microsoft/mana/mana_en.c int mana_create_wq_obj(struct mana_port_context *apc, · line 1556

Quoted 11 lines · Haiyang Zhang
drivers/net/ethernet/microsoft/mana/mana_en.c +3 -0
index db14357d3732..b1e0c444f414 100644
@@ -1551,6 +1551,9 @@ int mana_create_wq_obj(struct mana_port_context *apc,
 
 	mana_gd_init_req_hdr(&req.hdr, MANA_CREATE_WQ_OBJ,
 			     sizeof(req), sizeof(resp));
+
+	req.hdr.req.msg_version = GDMA_MESSAGE_V3;
+	req.hdr.resp.msg_version = GDMA_MESSAGE_V2;

Sashiko noted the above cold break initialization on older firmware:

https://sashiko.dev/#/patchset/20260604234211.2056341-1-haiyangz%40linux.microsoft.com

[...]

drivers/net/ethernet/microsoft/mana/mana_en.c static void mana_poll_rx_cq(struct mana_cq *cq) · line 2305

Quoted 5 lines
drivers/net/ethernet/microsoft/mana/mana_en.c +5 -0
+static void mana_update_rx_dim(struct mana_cq *cq)
+{
+	struct mana_port_context *apc = netdev_priv(cq->rxq->ndev);
+	struct mana_rxq *rxq = cq->rxq;
+	struct dim_sample dim_sample = {};

Minor nit: please fix the variable declaration order above. Other occurrences below.

[...]

Quoted 74 lines
drivers/net/ethernet/microsoft/mana/mana_en.c +60 -4
@@ -440,17 +474,94 @@ static int mana_set_coalesce(struct net_device *ndev,
 		return -EINVAL;
 	}
 
-	saved_cqe_coalescing_enable = apc->cqe_coalescing_enable;
+	if (ec->rx_coalesce_usecs > MANA_INTR_MODR_USEC_MAX ||
+	    ec->tx_coalesce_usecs > MANA_INTR_MODR_USEC_MAX) {
+		NL_SET_ERR_MSG_FMT(extack,
+				   "coalesce usecs must be <= %lu",
+				   MANA_INTR_MODR_USEC_MAX);
+		return -EINVAL;
+	}
+
+	if (ec->rx_max_coalesced_frames > MANA_INTR_MODR_COMP_MAX ||
+	    ec->tx_max_coalesced_frames > MANA_INTR_MODR_COMP_MAX) {
+		NL_SET_ERR_MSG_FMT(extack,
+				   "coalesce frames must be <= %lu",
+				   MANA_INTR_MODR_COMP_MAX);
+		return -EINVAL;
+	}
+
+	if (ec->rx_coalesce_usecs != apc->intr_modr_rx_usec ||
+	    ec->rx_max_coalesced_frames != apc->intr_modr_rx_comp ||
+	    ec->tx_coalesce_usecs != apc->intr_modr_tx_usec ||
+	    ec->tx_max_coalesced_frames != apc->intr_modr_tx_comp)
+		modr_changed = true;
+
+	saved.intr_modr_rx_usec = apc->intr_modr_rx_usec;
+	saved.intr_modr_rx_comp = apc->intr_modr_rx_comp;
+	saved.intr_modr_tx_usec = apc->intr_modr_tx_usec;
+	saved.intr_modr_tx_comp = apc->intr_modr_tx_comp;
+
+	apc->intr_modr_rx_usec = ec->rx_coalesce_usecs;
+	apc->intr_modr_rx_comp = ec->rx_max_coalesced_frames;
+	apc->intr_modr_tx_usec = ec->tx_coalesce_usecs;
+	apc->intr_modr_tx_comp = ec->tx_max_coalesced_frames;
+
+	if (!!ec->use_adaptive_rx_coalesce != apc->rx_dim_enabled ||
+	    !!ec->use_adaptive_tx_coalesce != apc->tx_dim_enabled)
+		dim_changed = true;
+
+	saved.rx_dim_enabled = apc->rx_dim_enabled;
+	saved.tx_dim_enabled = apc->tx_dim_enabled;
+	apc->rx_dim_enabled = !!ec->use_adaptive_rx_coalesce;
+	apc->tx_dim_enabled = !!ec->use_adaptive_tx_coalesce;
+
+	saved.cqe_coalescing_enable = apc->cqe_coalescing_enable;
 	apc->cqe_coalescing_enable =
 		kernel_coal->rx_cqe_frames == MANA_RXCOMP_OOB_NUM_PPI;
 
 	if (!apc->port_is_up)
 		return 0;
 
-	err = mana_config_rss(apc, TRI_STATE_TRUE, false, false);
-	if (err)
-		apc->cqe_coalescing_enable = saved_cqe_coalescing_enable;
+	if (apc->cqe_coalescing_enable != saved.cqe_coalescing_enable &&
+	    !modr_changed && !dim_changed) {
+		/* If only CQE coalescing setting is changed, we can just update
+		 * RSS configuration.
+		 */
+		err = mana_config_rss(apc, TRI_STATE_TRUE, false, false);
+		if (err) {
+			netdev_err(ndev, "Change CQE coalescing failed: %d\n",
+				   err);
+			apc->cqe_coalescing_enable =
+				saved.cqe_coalescing_enable;
+			return err;
+		}
+		return 0;
+	}
+
+	if (modr_changed || dim_changed) {
+		err = mana_detach(ndev, false);

drivers/net/ethernet/microsoft/mana/mana_en.c static int mana_set_coalesce(struct net_device *ndev, · line 552

Quoted 10 lines
drivers/net/ethernet/microsoft/mana/mana_en.c +10 -0
+		if (err) {
+			netdev_err(ndev, "mana_detach failed: %d\n", err);
+			goto restore_modr;
+		}
+
+		err = mana_attach(ndev);
+		if (err) {
+			netdev_err(ndev, "mana_attach failed: %d\n", err);
+			goto restore_modr;
+		}

You should try hard to avoid this sequence: if mana_attach fails, mana_set_coalesce() will leave the NIC unexpectedly down.

/P

maintainer
#3 lore raw
Quoted 32 lines

-----Original Message-----
From: Paolo Abeni <[email protected]>
Sent: Tuesday, June 9, 2026 9:49 AM
To: Haiyang Zhang <[email protected]>; linux- [email protected]; [email protected]; KY Srinivasan <[email protected]>; Haiyang Zhang <[email protected]>; Wei Liu <[email protected]>; Dexuan Cui <[email protected]>; Long Li <[email protected]>; Andrew Lunn <[email protected]>; David S. Miller <[email protected]>; Eric Dumazet <[email protected]>; Jakub Kicinski <[email protected]>; Konstantin Taranov <[email protected]>; Simon Horman <[email protected]>; Shradha Gupta <[email protected]>; Erni Sri Satya Vennela <[email protected]>; Dipayaan Roy <[email protected]>; Aditya Garg <[email protected]>; Kees Cook <[email protected]>; Breno Leitao <[email protected]>; [email protected]; linux- [email protected]
Cc: Paul Rosswurm <[email protected]>
Subject: [EXTERNAL] Re: [PATCH net-next v2] net: mana: Add Interrupt Moderation support

Review

Quoted 1 line · Haiyang Zhang
(quoted diff) +0 -0

b/drivers/net/ethernet/microsoft/mana/mana_en.c

drivers/net/ethernet/microsoft/mana/mana_en.c int mana_create_wq_obj(struct mana_port_context

Quoted 4 lines

index db14357d3732..b1e0c444f414 100644

drivers/net/ethernet/microsoft/mana/mana_en.c +0 -0
@@ -1551,6 +1551,9 @@ int mana_create_wq_obj(struct mana_port_context

*apc,

Quoted 1 line
Quoted 7 lines

drivers/net/ethernet/microsoft/mana/mana_en.c int mana_create_wq_obj(struct mana_port_context *apc, · line 1556

Quoted 5 lines
drivers/net/ethernet/microsoft/mana/mana_en.c +3 -0
 	mana_gd_init_req_hdr(&req.hdr, MANA_CREATE_WQ_OBJ,
 			     sizeof(req), sizeof(resp));
+
+	req.hdr.req.msg_version = GDMA_MESSAGE_V3;
+	req.hdr.resp.msg_version = GDMA_MESSAGE_V2;

Sashiko noted the above cold break initialization on older firmware:

Our firmware is forward compatible with newer message versions, so the old firmware still properly handles this message, just the new feature fields are ignored, and queue creation will be successful. And if the DIM capability bit is zero from FW, driver will keep the DIM feature to be off and unchangeable.

Quoted 9 lines

https://nam06.safelinks.protection.outlook.com/?url=https%3A%2F%2Fsashiko.
dev%2F%23%2Fpatchset%2F20260604234211.2056341-1- haiyangz%2540linux.microsoft.com&data=05%7C02%7Chaiyangz%40microsoft.com%7 C9cc8d2c7aa7f472ff8e308dec62def04%7C72f988bf86f141af91ab2d7cd011db47%7C1%7 C0%7C639166097783522606%7CUnknown%7CTWFpbGZsb3d8eyJFbXB0eU1hcGkiOnRydWUsIl YiOiIwLjAuMDAwMCIsIlAiOiJXaW4zMiIsIkFOIjoiTWFpbCIsIldUIjoyfQ%3D%3D%7C0%7C% 7C%7C&sdata=TB7q6EhtR6HJ02Q4f767sXUCmYZyGr3wH1Sz3vLPWfA%3D&reserved=0

[...]

Quoted 8 lines

drivers/net/ethernet/microsoft/mana/mana_en.c static void mana_poll_rx_cq(struct mana_cq *cq) · line 2305

Quoted 5 lines
drivers/net/ethernet/microsoft/mana/mana_en.c +5 -0
+static void mana_update_rx_dim(struct mana_cq *cq)
+{
+	struct mana_port_context *apc = netdev_priv(cq->rxq->ndev);
+	struct mana_rxq *rxq = cq->rxq;
+	struct dim_sample dim_sample = {};

Minor nit: please fix the variable declaration order above. Other occurrences below.

Done in the new version.

Quoted 83 lines

[...]

drivers/net/ethernet/microsoft/mana/mana_en.c static int mana_set_coalesce(struct net_device

Quoted 1 line
drivers/net/ethernet/microsoft/mana/mana_en.c +0 -0
@@ -440,17 +474,94 @@ static int mana_set_coalesce(struct net_device

*ndev,

Quoted 48 lines
drivers/net/ethernet/microsoft/mana/mana_en.c +42 -1
 		return -EINVAL;
 	}
 
-	saved_cqe_coalescing_enable = apc->cqe_coalescing_enable;
+	if (ec->rx_coalesce_usecs > MANA_INTR_MODR_USEC_MAX ||
+	    ec->tx_coalesce_usecs > MANA_INTR_MODR_USEC_MAX) {
+		NL_SET_ERR_MSG_FMT(extack,
+				   "coalesce usecs must be <= %lu",
+				   MANA_INTR_MODR_USEC_MAX);
+		return -EINVAL;
+	}
+
+	if (ec->rx_max_coalesced_frames > MANA_INTR_MODR_COMP_MAX ||
+	    ec->tx_max_coalesced_frames > MANA_INTR_MODR_COMP_MAX) {
+		NL_SET_ERR_MSG_FMT(extack,
+				   "coalesce frames must be <= %lu",
+				   MANA_INTR_MODR_COMP_MAX);
+		return -EINVAL;
+	}
+
+	if (ec->rx_coalesce_usecs != apc->intr_modr_rx_usec ||
+	    ec->rx_max_coalesced_frames != apc->intr_modr_rx_comp ||
+	    ec->tx_coalesce_usecs != apc->intr_modr_tx_usec ||
+	    ec->tx_max_coalesced_frames != apc->intr_modr_tx_comp)
+		modr_changed = true;
+
+	saved.intr_modr_rx_usec = apc->intr_modr_rx_usec;
+	saved.intr_modr_rx_comp = apc->intr_modr_rx_comp;
+	saved.intr_modr_tx_usec = apc->intr_modr_tx_usec;
+	saved.intr_modr_tx_comp = apc->intr_modr_tx_comp;
+
+	apc->intr_modr_rx_usec = ec->rx_coalesce_usecs;
+	apc->intr_modr_rx_comp = ec->rx_max_coalesced_frames;
+	apc->intr_modr_tx_usec = ec->tx_coalesce_usecs;
+	apc->intr_modr_tx_comp = ec->tx_max_coalesced_frames;
+
+	if (!!ec->use_adaptive_rx_coalesce != apc->rx_dim_enabled ||
+	    !!ec->use_adaptive_tx_coalesce != apc->tx_dim_enabled)
+		dim_changed = true;
+
+	saved.rx_dim_enabled = apc->rx_dim_enabled;
+	saved.tx_dim_enabled = apc->tx_dim_enabled;
+	apc->rx_dim_enabled = !!ec->use_adaptive_rx_coalesce;
+	apc->tx_dim_enabled = !!ec->use_adaptive_tx_coalesce;
+
+	saved.cqe_coalescing_enable = apc->cqe_coalescing_enable;
 	apc->cqe_coalescing_enable =
 		kernel_coal->rx_cqe_frames == MANA_RXCOMP_OOB_NUM_PPI;

drivers/net/ethernet/microsoft/mana/mana_en.c static int mana_set_coalesce(struct net_device · line 527

Quoted 10 lines
drivers/net/ethernet/microsoft/mana/mana_en.c +3 -3
 
 	if (!apc->port_is_up)
 		return 0;
 
-	err = mana_config_rss(apc, TRI_STATE_TRUE, false, false);
-	if (err)
-		apc->cqe_coalescing_enable = saved_cqe_coalescing_enable;
+	if (apc->cqe_coalescing_enable != saved.cqe_coalescing_enable &&
+	    !modr_changed && !dim_changed) {
+		/* If only CQE coalescing setting is changed, we can just

update

Quoted 20 lines
drivers/net/ethernet/microsoft/mana/mana_en.c +20 -0
+		 * RSS configuration.
+		 */
+		err = mana_config_rss(apc, TRI_STATE_TRUE, false, false);
+		if (err) {
+			netdev_err(ndev, "Change CQE coalescing failed: %d\n",
+				   err);
+			apc->cqe_coalescing_enable =
+				saved.cqe_coalescing_enable;
+			return err;
+		}
+		return 0;
+	}
+
+	if (modr_changed || dim_changed) {
+		err = mana_detach(ndev, false);
+		if (err) {
+			netdev_err(ndev, "mana_detach failed: %d\n", err);
+			goto restore_modr;
+		}
+
Quoted 8 lines

drivers/net/ethernet/microsoft/mana/mana_en.c static int mana_set_coalesce(struct net_device · line 552

Quoted 5 lines
drivers/net/ethernet/microsoft/mana/mana_en.c +5 -0
+		err = mana_attach(ndev);
+		if (err) {
+			netdev_err(ndev, "mana_attach failed: %d\n", err);
+			goto restore_modr;
+		}

You should try hard to avoid this sequence: if mana_attach fails, mana_set_coalesce() will leave the NIC unexpectedly down.

I have updated the patch to use doorbell for this setting change without re-attach, and will submit the new version soon.

Thanks,
- Haiyang