Skip to contentSkip to conversation

nvmet-tcp: fix a segmentation fault during io parsing error

PATCH Superseded [email protected] 2 participants 4 messages Mar 29, 2021 – Mar 30, 2021

Subsystems (2)

v3 of 4 Superseded Latest is v4 →

lore
Share & tools
lore
#0 lore raw

PATCH v3 1/1 nvmet-tcp: fix a segmentation fault during io parsing error

From: Elad Grupi <[email protected]>

In case there is an io that contains inline data and it goes to parsing error flow, command response will free command and iov before clearing the data on the socket buffer.
This will delay the command response until receive flow is completed.

Fixes 872d26a391da ("nvmet-tcp: add NVMe over TCP target driver")
---
 drivers/nvme/target/tcp.c | 35 ++++++++++++++++++++++++++++-------
 1 file changed, 28 insertions(+), 7 deletions(-)
drivers/nvme/target/tcp.c +28 -7
index 70cc507d1565..41102fc09595 100644
@@ -525,11 +525,34 @@ static void nvmet_tcp_queue_response(struct nvmet_req *req)
 	struct nvmet_tcp_cmd *cmd =
 		container_of(req, struct nvmet_tcp_cmd, req);
 	struct nvmet_tcp_queue	*queue = cmd->queue;
+	struct nvme_sgl_desc *sgl;
+	u32 len;
+
+	if (unlikely(cmd == queue->cmd)) {
+		sgl = &cmd->req.cmd->common.dptr.sgl;
+		len = le32_to_cpu(sgl->length);
+
+		/*
+		 * Wait for inline data before processing the response.
+		 * Avoid using helpers, this might happen before
+		 * nvmet_req_init is completed.
+		 */
+		if (len && cmd->rcv_state == NVMET_TCP_RECV_PDU)
+			return;
+	}
 
 	llist_add(&cmd->lentry, &queue->resp_list);
 	queue_work_on(queue_cpu(queue), nvmet_tcp_wq, &cmd->queue->io_work);
 }
 
+static void nvmet_tcp_execute_request(struct nvmet_tcp_cmd *cmd)
+{
+	if (unlikely(cmd->flags & NVMET_TCP_F_INIT_FAILED))
+		nvmet_tcp_queue_response(&cmd->req);
+	else
+		cmd->req.execute(&cmd->req);
+}
+
 static int nvmet_try_send_data_pdu(struct nvmet_tcp_cmd *cmd)
 {
 	u8 hdgst = nvmet_tcp_hdgst_len(cmd->queue);
@@ -1103,10 +1126,8 @@ static int nvmet_tcp_try_recv_data(struct nvmet_tcp_queue *queue)
 		return 0;
 	}
 
-	if (!(cmd->flags & NVMET_TCP_F_INIT_FAILED) &&
-	    cmd->rbytes_done == cmd->req.transfer_len) {
-		cmd->req.execute(&cmd->req);
-	}
+	if (cmd->rbytes_done == cmd->req.transfer_len)
+		nvmet_tcp_execute_request(cmd);
 
 	nvmet_prepare_receive_pdu(queue);
 	return 0;
@@ -1143,9 +1164,9 @@ static int nvmet_tcp_try_recv_ddgst(struct nvmet_tcp_queue *queue)
 		goto out;
 	}
 
-	if (!(cmd->flags & NVMET_TCP_F_INIT_FAILED) &&
-	    cmd->rbytes_done == cmd->req.transfer_len)
-		cmd->req.execute(&cmd->req);
+	if (cmd->rbytes_done == cmd->req.transfer_len)
+		nvmet_tcp_execute_request(cmd);
+
 	ret = 0;
 out:
 	nvmet_prepare_receive_pdu(queue);
Signature · 7 lines
2.18.2


_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
#1 lore raw
Quoted 12 lines · Elad Grupi
drivers/nvme/target/tcp.c +4 -0
index 70cc507d1565..41102fc09595 100644
@@ -525,11 +525,34 @@ static void nvmet_tcp_queue_response(struct nvmet_req *req)
 	struct nvmet_tcp_cmd *cmd =
 		container_of(req, struct nvmet_tcp_cmd, req);
 	struct nvmet_tcp_queue	*queue = cmd->queue;
+	struct nvme_sgl_desc *sgl;
+	u32 len;
+
+	if (unlikely(cmd == queue->cmd)) {

drivers/nvme/target/tcp.c static void nvmet_tcp_queue_response(struct nvmet_req *req) · line 541

Quoted 10 lines
drivers/nvme/target/tcp.c +10 -0
+		sgl = &cmd->req.cmd->common.dptr.sgl;
+		len = le32_to_cpu(sgl->length);
+
+		/*
+		 * Wait for inline data before processing the response.
+		 * Avoid using helpers, this might happen before
+		 * nvmet_req_init is completed.
+		 */
+		if (len && cmd->rcv_state == NVMET_TCP_RECV_PDU)
+			return;

Is it queue->rcv_state ?
I tried this patch, the identify command could get here. And nvme connect could hang. We need to figure out a way to tell if it needs abort queue the request. Or maybe we could use the v2 version.

Thanks,
Hou

drivers/nvme/target/tcp.c static void nvmet_tcp_queue_response(struct nvmet_req *req) · line 546

Quoted 5 lines
drivers/nvme/target/tcp.c +1 -0
+	}
 
 	llist_add(&cmd->lentry, &queue->resp_list);
 	queue_work_on(queue_cpu(queue), nvmet_tcp_wq, &cmd->queue->io_work);
 }

_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme

#2 lore raw
Quoted 20 lines · Hou Pu
Quoted 19 lines · Elad Grupi
drivers/nvme/target/tcp.c +11 -0
index 70cc507d1565..41102fc09595 100644
@@ -525,11 +525,34 @@ static void nvmet_tcp_queue_response(struct nvmet_req *req)
 	struct nvmet_tcp_cmd *cmd =
 		container_of(req, struct nvmet_tcp_cmd, req);
 	struct nvmet_tcp_queue	*queue = cmd->queue;
+	struct nvme_sgl_desc *sgl;
+	u32 len;
+
+	if (unlikely(cmd == queue->cmd)) {
+		sgl = &cmd->req.cmd->common.dptr.sgl;
+		len = le32_to_cpu(sgl->length);
+
+		/*
+		 * Wait for inline data before processing the response.
+		 * Avoid using helpers, this might happen before
+		 * nvmet_req_init is completed.
Quoted 8 lines

drivers/nvme/target/tcp.c static void nvmet_tcp_queue_response(struct nvmet_req *req) · line 541

Quoted 3 lines
drivers/nvme/target/tcp.c +3 -0
+		 */
+		if (len && cmd->rcv_state == NVMET_TCP_RECV_PDU)
+			return;

Is it queue->rcv_state ?
I tried this patch, the identify command could get here. And nvme connect could hang. We need to figure out a way to tell if it needs abort queue the request. Or maybe we could use the v2 version.

Adding nvme_is_write() would solve the problem. Also as we skip queue queue->io_work, we should return 0 instead -EAGAIN like below to consume the inline data in nvmet_tcp_try_recv_one(). Or the io_work might not have a chance to run.

drivers/nvme/target/tcp.c +3 -2
index a10a3bd59..f3d117771 100644
@@ -537,7 +537,8 @@ static void nvmet_tcp_queue_response(struct nvmet_req *req)
                 * Avoid using helpers, this might happen before
                 * nvmet_req_init is completed.
                 */
-               if (len && cmd->rcv_state == NVMET_TCP_RECV_PDU)
+               if (len && queue->rcv_state == NVMET_TCP_RECV_PDU &&
+                   nvme_is_write(cmd->req.cmd))
                        return;
        }
 
@@ -984,7 +985,7 @@ static int nvmet_tcp_done_recv_pdu(struct nvmet_tcp_queue *queue)
                        le32_to_cpu(req->cmd->common.dptr.sgl.length));
 
                nvmet_tcp_handle_req_failure(queue, queue->cmd, req);
-               return -EAGAIN;
+               return 0;
        }
 
        ret = nvmet_tcp_map_data(queue->cmd);

Thanks,
Hou

_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme

#3 lore raw

Right. There is a race in patch v3.

Thanks

-----Original Message-----
From: Hou Pu <[email protected]>
Sent: Tuesday, 30 March 2021 8:49
To: [email protected]
Cc: Grupi, Elad; [email protected]; [email protected]
Subject: Re: [PATCH v3] nvmet-tcp: fix a segmentation fault during io parsing error

[EXTERNAL EMAIL]

Quoted 20 lines · Hou Pu
Quoted 19 lines · Elad Grupi
drivers/nvme/target/tcp.c +11 -0
index 70cc507d1565..41102fc09595 100644
@@ -525,11 +525,34 @@ static void nvmet_tcp_queue_response(struct nvmet_req *req)
 	struct nvmet_tcp_cmd *cmd =
 		container_of(req, struct nvmet_tcp_cmd, req);
 	struct nvmet_tcp_queue	*queue = cmd->queue;
+	struct nvme_sgl_desc *sgl;
+	u32 len;
+
+	if (unlikely(cmd == queue->cmd)) {
+		sgl = &cmd->req.cmd->common.dptr.sgl;
+		len = le32_to_cpu(sgl->length);
+
+		/*
+		 * Wait for inline data before processing the response.
+		 * Avoid using helpers, this might happen before
+		 * nvmet_req_init is completed.
Quoted 8 lines

drivers/nvme/target/tcp.c static void nvmet_tcp_queue_response(struct nvmet_req *req) · line 541

Quoted 3 lines
drivers/nvme/target/tcp.c +3 -0
+		 */
+		if (len && cmd->rcv_state == NVMET_TCP_RECV_PDU)
+			return;

Is it queue->rcv_state ?
I tried this patch, the identify command could get here. And nvme connect could hang. We need to figure out a way to tell if it needs abort queue the request. Or maybe we could use the v2 version.

Adding nvme_is_write() would solve the problem.
Also as we skip queue queue->io_work, we should return
0 instead -EAGAIN like below to consume the inline data in nvmet_tcp_try_recv_one(). Or the io_work might not have a chance to run.

drivers/nvme/target/tcp.c +3 -2
@@ -537,7 +537,8 @@ static void nvmet_tcp_queue_response(struct nvmet_req *req)
                 * Avoid using helpers, this might happen before
                 * nvmet_req_init is completed.
                 */
-               if (len && cmd->rcv_state == NVMET_TCP_RECV_PDU)
+               if (len && queue->rcv_state == NVMET_TCP_RECV_PDU &&
+                   nvme_is_write(cmd->req.cmd))
                        return;
        }
 
@@ -984,7 +985,7 @@ static int nvmet_tcp_done_recv_pdu(struct nvmet_tcp_queue *queue)
                        le32_to_cpu(req->cmd->common.dptr.sgl.length));
 
                nvmet_tcp_handle_req_failure(queue, queue->cmd, req);
-               return -EAGAIN;
+               return 0;
        }
 
        ret = nvmet_tcp_map_data(queue->cmd);

Thanks,
Hou
_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme