Skip to contentSkip to conversation

net: mana: Fix race of mana_hwc_post_rx_wqe and new hwc response

PATCH Applied Haiyang Zhang 4 participants 5 messages Aug 21, 2024 – Aug 23, 2024

Subsystems (7)
lore
Share & tools
lore

Haiyang Zhang

author maintainer
#0 lore raw

PATCH 1/1 net: mana: Fix race of mana_hwc_post_rx_wqe and new hwc response

The mana_hwc_rx_event_handler() / mana_hwc_handle_resp() calls complete(&ctx->comp_event) before posting the wqe back. It's possible that other callers, like mana_create_txq(), start the next round of mana_hwc_send_request() before the posting of wqe. And if the HW is fast enough to respond, it can hit no_wqe error on the HW channel, then the response message is lost. The mana driver may fail to create queues and open, because of waiting for the HW response and timed out.
Sample dmesg:

[  528.610840] mana 39d4:00:02.0: HWC: Request timed out!
[  528.614452] mana 39d4:00:02.0: Failed to send mana message: -110, 0x0
[  528.618326] mana 39d4:00:02.0 enP14804s2: Failed to create WQ object: -110

To fix it, move posting of rx wqe before complete(&ctx->comp_event).

Fixes ca9c54d2d6a5 ("net: mana: Add a driver for Microsoft Azure Network Adapter (MANA)")
---
 .../net/ethernet/microsoft/mana/hw_channel.c  | 62 ++++++++++---------
 1 file changed, 34 insertions(+), 28 deletions(-)
drivers/net/ethernet/microsoft/mana/hw_channel.c +34 -28
index cafded2f9382..a00f915c5188 100644
@@ -52,9 +52,33 @@ static int mana_hwc_verify_resp_msg(const struct hwc_caller_ctx *caller_ctx,
 	return 0;
 }
 
+static int mana_hwc_post_rx_wqe(const struct hwc_wq *hwc_rxq,
+				struct hwc_work_request *req)
+{
+	struct device *dev = hwc_rxq->hwc->dev;
+	struct gdma_sge *sge;
+	int err;
+
+	sge = &req->sge;
+	sge->address = (u64)req->buf_sge_addr;
+	sge->mem_key = hwc_rxq->msg_buf->gpa_mkey;
+	sge->size = req->buf_len;
+
+	memset(&req->wqe_req, 0, sizeof(struct gdma_wqe_request));
+	req->wqe_req.sgl = sge;
+	req->wqe_req.num_sge = 1;
+	req->wqe_req.client_data_unit = 0;
+
+	err = mana_gd_post_and_ring(hwc_rxq->gdma_wq, &req->wqe_req, NULL);
+	if (err)
+		dev_err(dev, "Failed to post WQE on HWC RQ: %d\n", err);
+	return err;
+}
+
 static void mana_hwc_handle_resp(struct hw_channel_context *hwc, u32 resp_len,
-				 const struct gdma_resp_hdr *resp_msg)
+				 struct hwc_work_request *rx_req)
 {
+	const struct gdma_resp_hdr *resp_msg = rx_req->buf_va;
 	struct hwc_caller_ctx *ctx;
 	int err;
 
@@ -62,6 +86,7 @@ static void mana_hwc_handle_resp(struct hw_channel_context *hwc, u32 resp_len,
 		      hwc->inflight_msg_res.map)) {
 		dev_err(hwc->dev, "hwc_rx: invalid msg_id = %u\n",
 			resp_msg->response.hwc_msg_id);
+		mana_hwc_post_rx_wqe(hwc->rxq, rx_req);
 		return;
 	}
 
@@ -75,30 +100,13 @@ static void mana_hwc_handle_resp(struct hw_channel_context *hwc, u32 resp_len,
 	memcpy(ctx->output_buf, resp_msg, resp_len);
 out:
 	ctx->error = err;
-	complete(&ctx->comp_event);
-}
-
-static int mana_hwc_post_rx_wqe(const struct hwc_wq *hwc_rxq,
-				struct hwc_work_request *req)
-{
-	struct device *dev = hwc_rxq->hwc->dev;
-	struct gdma_sge *sge;
-	int err;
-
-	sge = &req->sge;
-	sge->address = (u64)req->buf_sge_addr;
-	sge->mem_key = hwc_rxq->msg_buf->gpa_mkey;
-	sge->size = req->buf_len;
 
-	memset(&req->wqe_req, 0, sizeof(struct gdma_wqe_request));
-	req->wqe_req.sgl = sge;
-	req->wqe_req.num_sge = 1;
-	req->wqe_req.client_data_unit = 0;
+	/* Must post rx wqe before complete(), otherwise the next rx may
+	 * hit no_wqe error.
+	 */
+	mana_hwc_post_rx_wqe(hwc->rxq, rx_req);
 
-	err = mana_gd_post_and_ring(hwc_rxq->gdma_wq, &req->wqe_req, NULL);
-	if (err)
-		dev_err(dev, "Failed to post WQE on HWC RQ: %d\n", err);
-	return err;
+	complete(&ctx->comp_event);
 }
 
 static void mana_hwc_init_event_handler(void *ctx, struct gdma_queue *q_self,
@@ -235,14 +243,12 @@ static void mana_hwc_rx_event_handler(void *ctx, u32 gdma_rxq_id,
 		return;
 	}
 
-	mana_hwc_handle_resp(hwc, rx_oob->tx_oob_data_size, resp);
+	mana_hwc_handle_resp(hwc, rx_oob->tx_oob_data_size, rx_req);
 
-	/* Do no longer use 'resp', because the buffer is posted to the HW
-	 * in the below mana_hwc_post_rx_wqe().
+	/* Can no longer use 'resp', because the buffer is posted to the HW
+	 * in mana_hwc_handle_resp() above.
 	 */
 	resp = NULL;
-
-	mana_hwc_post_rx_wqe(hwc_rxq, rx_req);
 }
 
 static void mana_hwc_tx_event_handler(void *ctx, u32 gdma_txq_id,
Signature · 1 line
2.34.1
Reviewed-by on v1 1/1
#1 lore raw
Quoted 15 lines

Subject: [PATCH net] net: mana: Fix race of mana_hwc_post_rx_wqe and new hwc response

The mana_hwc_rx_event_handler() / mana_hwc_handle_resp() calls complete(&ctx->comp_event) before posting the wqe back. It's possible that other callers, like mana_create_txq(), start the next round of mana_hwc_send_request() before the posting of wqe.
And if the HW is fast enough to respond, it can hit no_wqe error on the HW channel, then the response message is lost. The mana driver may fail to create queues and open, because of waiting for the HW response and timed out. Sample dmesg:

[  528.610840] mana 39d4:00:02.0: HWC: Request timed out!
[  528.614452] mana 39d4:00:02.0: Failed to send mana message: -110, 0x0
[  528.618326] mana 39d4:00:02.0 enP14804s2: Failed to create WQ object: -110
Quoted 6 lines

To fix it, move posting of rx wqe before complete(&ctx->comp_event).

Fixes ca9c54d2d6a5 ("net: mana: Add a driver for Microsoft Azure Network

Adapter (MANA)")

Quoted 115 lines
---
 .../net/ethernet/microsoft/mana/hw_channel.c  | 62 ++++++++++---------
 1 file changed, 34 insertions(+), 28 deletions(-)
drivers/net/ethernet/microsoft/mana/hw_channel.c +25 -1
index cafded2f9382..a00f915c5188 100644
@@ -52,9 +52,33 @@ static int mana_hwc_verify_resp_msg(const struct
 hwc_caller_ctx *caller_ctx,
 	return 0;
 }
 
+static int mana_hwc_post_rx_wqe(const struct hwc_wq *hwc_rxq,
+				struct hwc_work_request *req)
+{
+	struct device *dev = hwc_rxq->hwc->dev;
+	struct gdma_sge *sge;
+	int err;
+
+	sge = &req->sge;
+	sge->address = (u64)req->buf_sge_addr;
+	sge->mem_key = hwc_rxq->msg_buf->gpa_mkey;
+	sge->size = req->buf_len;
+
+	memset(&req->wqe_req, 0, sizeof(struct gdma_wqe_request));
+	req->wqe_req.sgl = sge;
+	req->wqe_req.num_sge = 1;
+	req->wqe_req.client_data_unit = 0;
+
+	err = mana_gd_post_and_ring(hwc_rxq->gdma_wq, &req->wqe_req,
 NULL);
+	if (err)
+		dev_err(dev, "Failed to post WQE on HWC RQ: %d\n", err);
+	return err;
+}
+
 static void mana_hwc_handle_resp(struct hw_channel_context *hwc, u32
 resp_len,
-				 const struct gdma_resp_hdr *resp_msg)
+				 struct hwc_work_request *rx_req)
 {
+	const struct gdma_resp_hdr *resp_msg = rx_req->buf_va;
drivers/net/ethernet/microsoft/mana/hw_channel.c +6 -22
 	struct hwc_caller_ctx *ctx;
 	int err;
@@ -62,6 +86,7 @@ static void mana_hwc_handle_resp(struct
 hw_channel_context *hwc, u32 resp_len,
 		      hwc->inflight_msg_res.map)) {
 		dev_err(hwc->dev, "hwc_rx: invalid msg_id = %u\n",
 			resp_msg->response.hwc_msg_id);
+		mana_hwc_post_rx_wqe(hwc->rxq, rx_req);
 		return;
 	}
@@ -75,30 +100,13 @@ static void mana_hwc_handle_resp(struct
 hw_channel_context *hwc, u32 resp_len,
 	memcpy(ctx->output_buf, resp_msg, resp_len);
 out:
 	ctx->error = err;
-	complete(&ctx->comp_event);
-}
-
-static int mana_hwc_post_rx_wqe(const struct hwc_wq *hwc_rxq,
-				struct hwc_work_request *req)
-{
-	struct device *dev = hwc_rxq->hwc->dev;
-	struct gdma_sge *sge;
-	int err;
-
-	sge = &req->sge;
-	sge->address = (u64)req->buf_sge_addr;
-	sge->mem_key = hwc_rxq->msg_buf->gpa_mkey;
-	sge->size = req->buf_len;
 
-	memset(&req->wqe_req, 0, sizeof(struct gdma_wqe_request));
-	req->wqe_req.sgl = sge;
-	req->wqe_req.num_sge = 1;
-	req->wqe_req.client_data_unit = 0;
+	/* Must post rx wqe before complete(), otherwise the next rx may
+	 * hit no_wqe error.
+	 */
+	mana_hwc_post_rx_wqe(hwc->rxq, rx_req);
 
-	err = mana_gd_post_and_ring(hwc_rxq->gdma_wq, &req->wqe_req,
 NULL);
-	if (err)
-		dev_err(dev, "Failed to post WQE on HWC RQ: %d\n", err);
-	return err;
+	complete(&ctx->comp_event);
 }

static void mana_hwc_init_event_handler(void *ctx, struct gdma_queue *q_self,

drivers/net/ethernet/microsoft/mana/hw_channel.c +3 -5
@@ -235,14 +243,12 @@ static void mana_hwc_rx_event_handler(void *ctx,
 u32 gdma_rxq_id,
 		return;
 	}
 
-	mana_hwc_handle_resp(hwc, rx_oob->tx_oob_data_size, resp);
+	mana_hwc_handle_resp(hwc, rx_oob->tx_oob_data_size, rx_req);
 
-	/* Do no longer use 'resp', because the buffer is posted to the HW
-	 * in the below mana_hwc_post_rx_wqe().
+	/* Can no longer use 'resp', because the buffer is posted to the HW
+	 * in mana_hwc_handle_resp() above.
 	 */
 	resp = NULL;
-
-	mana_hwc_post_rx_wqe(hwc_rxq, rx_req);
 }
 

static void mana_hwc_tx_event_handler(void *ctx, u32 gdma_txq_id,
--
2.34.1

#2 lore raw
In reply to Haiyang Zhang

Le 21/08/2024 à 22:42, Haiyang Zhang a écrit :

Quoted 36 lines

The mana_hwc_rx_event_handler() / mana_hwc_handle_resp() calls complete(&ctx->comp_event) before posting the wqe back. It's possible that other callers, like mana_create_txq(), start the next round of mana_hwc_send_request() before the posting of wqe. And if the HW is fast enough to respond, it can hit no_wqe error on the HW channel, then the response message is lost. The mana driver may fail to create queues and open, because of waiting for the HW response and timed out.
Sample dmesg:

[  528.610840] mana 39d4:00:02.0: HWC: Request timed out!
[  528.614452] mana 39d4:00:02.0: Failed to send mana message: -110, 0x0
[  528.618326] mana 39d4:00:02.0 enP14804s2: Failed to create WQ object: -110

To fix it, move posting of rx wqe before complete(&ctx->comp_event).

Fixes ca9c54d2d6a5 ("net: mana: Add a driver for Microsoft Azure Network Adapter (MANA)")
---
  .../net/ethernet/microsoft/mana/hw_channel.c  | 62 ++++++++++---------
  1 file changed, 34 insertions(+), 28 deletions(-)
drivers/net/ethernet/microsoft/mana/hw_channel.c +6 -0
index cafded2f9382..a00f915c5188 100644
@@ -52,9 +52,33 @@ static int mana_hwc_verify_resp_msg(const struct hwc_caller_ctx *caller_ctx,
  	return 0;
  }
 
+static int mana_hwc_post_rx_wqe(const struct hwc_wq *hwc_rxq,
+				struct hwc_work_request *req)
+{
+	struct device *dev = hwc_rxq->hwc->dev;
+	struct gdma_sge *sge;
+	int err;

drivers/net/ethernet/microsoft/mana/hw_channel.c static int mana_hwc_verify_resp_msg(const struct hwc_caller_ctx *caller_ctx, · line 70

Quoted 10 lines
drivers/net/ethernet/microsoft/mana/hw_channel.c +10 -0
+
+	sge = &req->sge;
+	sge->address = (u64)req->buf_sge_addr;
+	sge->mem_key = hwc_rxq->msg_buf->gpa_mkey;
+	sge->size = req->buf_len;
+
+	memset(&req->wqe_req, 0, sizeof(struct gdma_wqe_request));
+	req->wqe_req.sgl = sge;
+	req->wqe_req.num_sge = 1;
+	req->wqe_req.client_data_unit = 0;

Hi,

unrelated to your patch, but this initialization is useless, it is already memset(0)'ed a few lines above. So why client_data_unit and not some other fields?

drivers/net/ethernet/microsoft/mana/hw_channel.c static int mana_hwc_verify_resp_msg(const struct hwc_caller_ctx *caller_ctx, · line 76

Quoted 6 lines
drivers/net/ethernet/microsoft/mana/hw_channel.c +6 -0
+
+	err = mana_gd_post_and_ring(hwc_rxq->gdma_wq, &req->wqe_req, NULL);
+	if (err)
+		dev_err(dev, "Failed to post WQE on HWC RQ: %d\n", err);
+	return err;
+}

...

Just my 2c.

CJ

author maintainer
#3 lore raw
Quoted 75 lines

-----Original Message-----
From: Christophe JAILLET <[email protected]>
Sent: Thursday, August 22, 2024 1:34 AM
To: Haiyang Zhang <[email protected]>
Cc: [email protected]; [email protected]; [email protected]; [email protected]; Dexuan Cui <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; KY Srinivasan <[email protected]>; [email protected]; [email protected]; linux- [email protected]; [email protected]; Long Li <[email protected]>; [email protected]; [email protected]; [email protected]; Paul Rosswurm <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: Re: [PATCH net] net: mana: Fix race of mana_hwc_post_rx_wqe and new hwc response

Le 21/08/2024 à 22:42, Haiyang Zhang a écrit :

Quoted 9 lines

The mana_hwc_rx_event_handler() / mana_hwc_handle_resp() calls complete(&ctx->comp_event) before posting the wqe back. It's possible that other callers, like mana_create_txq(), start the next round of mana_hwc_send_request() before the posting of wqe. And if the HW is fast enough to respond, it can hit no_wqe error on the HW channel, then the response message is lost. The mana driver may fail to create queues and open, because of waiting for the HW response and timed out.
Sample dmesg:

Quoted 3 lines
[  528.610840] mana 39d4:00:02.0: HWC: Request timed out!
[  528.614452] mana 39d4:00:02.0: Failed to send mana message: -110, 0x0
[  528.618326] mana 39d4:00:02.0 enP14804s2: Failed to create WQ object:

-110

Quoted 5 lines

To fix it, move posting of rx wqe before complete(&ctx->comp_event).

Fixes ca9c54d2d6a5 ("net: mana: Add a driver for Microsoft Azure

Network Adapter (MANA)")

Quoted 1 line
Signed-off-by Haiyang Zhang <haiyangz-

[email protected]>

Review

Quoted 5 lines
---
  .../net/ethernet/microsoft/mana/hw_channel.c  | 62 ++++++++++---------
  1 file changed, 34 insertions(+), 28 deletions(-)
(quoted diff) +0 -0

b/drivers/net/ethernet/microsoft/mana/hw_channel.c

drivers/net/ethernet/microsoft/mana/hw_channel.c static int mana_hwc_verify_resp_msg(const struct

Quoted 4 lines

index cafded2f9382..a00f915c5188 100644

drivers/net/ethernet/microsoft/mana/hw_channel.c +0 -0
@@ -52,9 +52,33 @@ static int mana_hwc_verify_resp_msg(const struct

hwc_caller_ctx *caller_ctx,

Quoted 9 lines
drivers/net/ethernet/microsoft/mana/hw_channel.c +6 -0
  	return 0;
  }
 
+static int mana_hwc_post_rx_wqe(const struct hwc_wq *hwc_rxq,
+				struct hwc_work_request *req)
+{
+	struct device *dev = hwc_rxq->hwc->dev;
+	struct gdma_sge *sge;
+	int err;

drivers/net/ethernet/microsoft/mana/hw_channel.c static int mana_hwc_verify_resp_msg(const struct hwc_caller_ctx *caller_ctx, · line 70

Quoted 10 lines
drivers/net/ethernet/microsoft/mana/hw_channel.c +10 -0
+
+	sge = &req->sge;
+	sge->address = (u64)req->buf_sge_addr;
+	sge->mem_key = hwc_rxq->msg_buf->gpa_mkey;
+	sge->size = req->buf_len;
+
+	memset(&req->wqe_req, 0, sizeof(struct gdma_wqe_request));
+	req->wqe_req.sgl = sge;
+	req->wqe_req.num_sge = 1;
+	req->wqe_req.client_data_unit = 0;

Hi,

Quoted 3 lines

unrelated to your patch, but this initialization is useless, it is already memset(0)'ed a few lines above.
So why client_data_unit and not some other fields?

Agreed. This patch just moves the function for the bug fix. We will do code cleanups in other patches.

Thanks,
- Haiyang

#4 lore raw
In reply to Haiyang Zhang
This patch was applied to netdev/net.git (main) by David S. Miller <[email protected]>:

Hello:

This patch was applied to netdev/net.git (main)
by David S. Miller <[email protected]>:

Quoted 9 lines · On Wed, 21 Aug 2024 13:42:29 -0700 you wrote:

The mana_hwc_rx_event_handler() / mana_hwc_handle_resp() calls complete(&ctx->comp_event) before posting the wqe back. It's possible that other callers, like mana_create_txq(), start the next round of mana_hwc_send_request() before the posting of wqe. And if the HW is fast enough to respond, it can hit no_wqe error on the HW channel, then the response message is lost. The mana driver may fail to create queues and open, because of waiting for the HW response and timed out.
Sample dmesg:

Quoted 5 lines
[  528.610840] mana 39d4:00:02.0: HWC: Request timed out!
[  528.614452] mana 39d4:00:02.0: Failed to send mana message: -110, 0x0
[  528.618326] mana 39d4:00:02.0 enP14804s2: Failed to create WQ object: -110

[...]

Here is the summary with links:
- [net] net: mana: Fix race of mana_hwc_post_rx_wqe and new hwc response

    https://git.kernel.org/netdev/net/c/8af174ea863c

You are awesome, thank you!

Signature · 2 lines
Deet-doot-dot, I am a bot.
https://korg.docs.kernel.org/patchwork/pwbot.html