Skip to contentSkip to conversation

nvme-tcp: protection from misbehaving link partners

PATCH New Mikhail Skorzhinskii 2 participants 4 messages Nov 14, 2018 – Nov 21, 2018

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

cover nvme-tcp: protection from misbehaving link partners

This patch sequence are about to fix a few minor kernel crashes and memory corruptions that was spot on a few stress test runs with misbehvaing link partners.

patches 1-2 prevents these awkward crashes and corruptions

Mikhail Skorzhinskii (2):
nvmet-tcp: add protection for already completed requests on h2c path
nvme-tcp: add protection against target sending more then maxr2t

 drivers/nvme/host/tcp.c   | 19 ++++++++++++++++++-
 drivers/nvme/target/tcp.c | 19 +++++++++++++++++--
 include/linux/nvme-tcp.h  |  3 +++
 3 files changed, 38 insertions(+), 3 deletions(-)
Signature · 1 line
2.16.4
#1 lore raw

PATCH 1/2 nvmet-tcp: add protection for already completed requests on h2c path

Handling H2C requests pushing completion to block layer in case of errors. But sometimes these request have no representation in a block layer, since they are already completed or was never allocated.

This protection causing immediate fatal error but no response to initiator since there are no API for sending transport error responses yet.

Signed-off-by Mikhail Skorzhinskii <mskorzhinskiy at solarflare.com>
---
 drivers/nvme/target/tcp.c | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)
drivers/nvme/target/tcp.c +17 -2
index fbfda1da52c7..02159713a5a1 100644
@@ -46,6 +46,7 @@ struct nvmet_tcp_send_ctx {
 
 enum {
 	NVMET_TCP_F_INIT_FAILED = (1 << 0),
+	NVMET_TCP_F_PENDING_DATA = (1 << 1),
 };
 
 struct nvmet_tcp_cmd {
@@ -398,6 +399,8 @@ static void nvmet_setup_r2t_pdu(struct nvmet_tcp_cmd *cmd)
 	struct nvmet_tcp_queue *queue = cmd->queue;
 	u8 hdgst = nvmet_tcp_hdgst_len(cmd->queue);
 
+	cmd->flags |= NVMET_TCP_F_PENDING_DATA;
+
 	cmd->snd.offset = 0;
 	cmd->snd.state = NVMET_TCP_SEND_R2T;
 
@@ -835,6 +838,12 @@ static int nvmet_tcp_handle_h2c_data_pdu(struct nvmet_tcp_queue *queue)
 
 	cmd = &queue->cmds[data->ttag];
 
+	if (!(cmd->flags & NVMET_TCP_F_PENDING_DATA)) {
+		pr_err("queue %d unexpected h2c with ttag %u\n",
+			queue->idx, data->ttag);
+		return -EPROTO;
+	}
+
 	if (le32_to_cpu(data->data_offset) != cmd->rbytes_done) {
 		pr_err("queue %d ttag %u unexpected data offset %u (expected %u)\n",
 			queue->idx, data->ttag, le32_to_cpu(data->data_offset),
@@ -1021,6 +1030,12 @@ static void nvmet_tcp_prep_recv_ddgst(struct nvmet_tcp_cmd *cmd)
 	queue->rcv.state = NVMET_TCP_RECV_DDGST;
 }
 
+static void nvmet_tcp_req_execute(struct nvmet_tcp_cmd *cmd)
+{
+	cmd->flags &= ~NVMET_TCP_F_PENDING_DATA;
+	nvmet_req_execute(&cmd->req);
+}
+
 static int nvmet_tcp_try_recv_data(struct nvmet_tcp_queue *queue)
 {
 	struct nvmet_tcp_cmd  *cmd = queue->rcv.cmd;
@@ -1044,7 +1059,7 @@ static int nvmet_tcp_try_recv_data(struct nvmet_tcp_queue *queue)
 			nvmet_tcp_prep_recv_ddgst(cmd);
 			return 0;
 		} else {
-			nvmet_req_execute(&cmd->req);
+			nvmet_tcp_req_execute(cmd);
 		}
 	}
 
@@ -1085,7 +1100,7 @@ static int nvmet_tcp_try_recv_ddgst(struct nvmet_tcp_queue *queue)
 
 	if (!(cmd->flags & NVMET_TCP_F_INIT_FAILED) &&
 	    cmd->rbytes_done == cmd->req.transfer_len)
-		nvmet_req_execute(&cmd->req);
+		nvmet_tcp_req_execute(cmd);
 	ret = 0;
 out:
 	nvmet_prepare_receive_pdu(queue);
Signature · 1 line
2.16.4
#2 lore raw

PATCH 2/2 nvme-tcp: add protection against target sending more then maxr2t

In reply to Mikhail Skorzhinskii

This is a simple protection against misbehaving targets to protect the initiator from R2T sent on a wrong time.

Signed-off-by Mikhail Skorzhinskii <mskorzhinskiy at solarflare.com>
---
 drivers/nvme/host/tcp.c  | 19 ++++++++++++++++++-
 include/linux/nvme-tcp.h |  3 +++
 2 files changed, 21 insertions(+), 1 deletion(-)
drivers/nvme/host/tcp.c +18 -1
index e28b9dd8c448..834243fa8359 100644
@@ -32,6 +32,7 @@ struct nvme_tcp_send_ctx {
 	size_t			offset;
 	size_t			data_sent;
 	enum nvme_tcp_send_state state;
+	int                     r2t_cnt;
 };
 
 struct nvme_tcp_recv_ctx {
@@ -566,12 +567,21 @@ static int nvme_tcp_handle_r2t(struct nvme_tcp_queue *queue,
 	}
 	req = blk_mq_rq_to_pdu(rq);
 
+	if (req->snd.r2t_cnt > NVME_TCP_MAX_R2T) {
+		dev_err(queue->ctrl->ctrl.device,
+			"queue %d tag %#x sent %d r2t while only %d is supported!\n",
+			nvme_tcp_queue_id(queue), pdu->command_id,
+			req->snd.r2t_cnt + 1, NVME_TCP_MAX_R2T);
+		return -ENOENT;
+	}
+
 	ret = nvme_tcp_setup_h2c_data_pdu(req, pdu);
 	if (unlikely(ret))
 		return ret;
 
 	req->snd.state = NVME_TCP_SEND_H2C_PDU;
 	req->snd.offset = 0;
+	req->snd.r2t_cnt++;
 
 	nvme_tcp_queue_request(req);
 
@@ -849,6 +859,11 @@ static void nvme_tcp_state_change(struct sock *sk)
 
 static inline void nvme_tcp_done_send_req(struct nvme_tcp_queue *queue)
 {
+	struct nvme_tcp_request *request = queue->request;
+
+	if (request->snd.state == NVME_TCP_SEND_H2C_PDU)
+		request->snd.r2t_cnt--;
+
 	queue->request = NULL;
 }
 
@@ -1176,7 +1191,7 @@ static int nvme_tcp_init_connection(struct nvme_tcp_queue *queue)
 	icreq->hdr.pdo = 0;
 	icreq->hdr.plen = cpu_to_le32(icreq->hdr.hlen);
 	icreq->pfv = cpu_to_le16(NVME_TCP_PFV_1_0);
-	icreq->maxr2t = 0; /* single inflight r2t supported */
+	icreq->maxr2t = NVME_TCP_MAX_R2T;
 	icreq->hpda = 0; /* no alignment constraint */
 	if (queue->hdr_digest)
 		icreq->digest |= NVME_TCP_HDR_DIGEST_ENABLE;
@@ -1981,6 +1996,7 @@ static void nvme_tcp_submit_async_event(struct nvme_ctrl *arg)
 	cmd->common.flags |= NVME_CMD_SGL_METABUF;
 	nvme_tcp_set_sg_null(cmd);
 
+	ctrl->async_req.snd.r2t_cnt = 0;
 	ctrl->async_req.snd.state = NVME_TCP_SEND_CMD_PDU;
 	ctrl->async_req.snd.offset = 0;
 	ctrl->async_req.snd.curr_bio = NULL;
@@ -2053,6 +2069,7 @@ static blk_status_t nvme_tcp_setup_cmd_pdu(struct nvme_ns *ns,
 		return ret;
 
 	req->snd.state = NVME_TCP_SEND_CMD_PDU;
+	req->snd.r2t_cnt = 0;
 	req->snd.offset = 0;
 	req->snd.data_sent = 0;
 	req->pdu_len = 0;
include/linux/nvme-tcp.h +3 -0
index 685780d1ed04..f659a87299ae 100644
@@ -13,6 +13,9 @@
 #define NVME_TCP_ADMIN_CCSZ	SZ_8K
 #define NVME_TCP_DIGEST_LENGTH	4
 
+/* single inflight r2t supported */
+#define NVME_TCP_MAX_R2T        cpu_to_le16(0)
+
 enum nvme_tcp_pfv {
 	NVME_TCP_PFV_1_0 = 0x0,
 };
Signature · 1 line
2.16.4
maintainer
#3 lore raw

cover nvme-tcp: protection from misbehaving link partners

In reply to Mikhail Skorzhinskii

Mikhail,

Quoted 5 lines

This patch sequence are about to fix a few minor kernel crashes and memory corruptions that was spot on a few stress test runs with misbehvaing link partners.

patches 1-2 prevents these awkward crashes and corruptions

Given that this is on the negative testing and effects the state machine I prefer that we get the initial version included first and then get this incremental fix in following rc cycles.