nvme-tcp: fixup I/O stall on congested sockets
v4 cover nvme-tcp: fixup I/O stall on congested sockets
Hi all,
I have been chasing keep-alive timeouts with TLS enabled in the last few weeks (months, even ...). On larger setups (eg with 32 queues) the connection never got established properly as I've been hitting keep-alive timeouts before the last queue got connected.
Turns out that occasionally we simply do not send the keep-alive request; it's been added to the request list but the io_work workqueue function is never restarted as it bails out after nvme_tcp_try_recv() returns -EAGAIN. During debugging I also found that we're quite lazy with the list handling of requests, so I've added a patche to ensure that all list elements are properly terminated.
As usual, comments and reviews are welcome.
Changes to v3:
- Drop first patch as it already had been applied
- Include reviews from Sagi
- Check for sk_sock_is_writeable() to avoid requeing io_work when the socket is blocked
Changes to v2:
- Removed AEN patches again
Changes to the original submission:
- Include reviews from Chris Leech
- Add patch to requeue namespace scan
- Add patch to re-read ANA log page
Hannes Reinecke (2):
nvme-tcp: sanitize request list handling
nvme-tcp: fix I/O stalls on congested sockets
drivers/nvme/host/tcp.c | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-)
Signature · 1 line
2.35.3
PATCH v4 1/2 nvme-tcp: sanitize request list handling
Validate the request in nvme_tcp_handle_r2t() to ensure it's not part of any list, otherwise a malicious R2T PDU might inject a loop in request list processing.
--- drivers/nvme/host/tcp.c | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-)
drivers/nvme/host/tcp.c +17 -2
@@ -452,7 +452,8 @@ nvme_tcp_fetch_request(struct nvme_tcp_queue *queue)return NULL ; } -list_del (&req->entry); +list_del_init (&req->entry); +init_llist_node (&req->lentry);return req; }
@@ -560,6 +561,8 @@ static int nvme_tcp_init_request(struct blk_mq_tag_set *set, req->queue = queue;nvme_req (rq)->ctrl = &ctrl->ctrl;nvme_req (rq)->cmd = &pdu->cmd; +init_llist_node (&req->lentry); +INIT_LIST_HEAD (&req->entry);return 0 ; }
@@ -764,6 +767,15 @@ static int nvme_tcp_handle_r2t(struct nvme_tcp_queue *queue,return -EPROTO ; } +if (queue->request == req || +llist_on_list (&req->lentry) || + !list_empty (&req->entry)) { +dev_err (queue->ctrl->ctrl.device, +"req %d unexpected r2t while processing request\n" , + rq->tag); +return -EPROTO ; + } + req->pdu_len =0 ; req->h2cdata_left = r2t_length; req->h2cdata_offset = r2t_offset;
@@ -772,7 +784,8 @@ static int nvme_tcp_handle_r2t(struct nvme_tcp_queue *queue,nvme_tcp_setup_h2c_data_pdu (req);llist_add (&req->lentry, &queue->req_list); -queue_work_on (queue->io_cpu, nvme_tcp_wq, &queue->io_work); +if (list_empty (&queue->send_list)) +queue_work_on (queue->io_cpu, nvme_tcp_wq, &queue->io_work);return 0 ; }
@@ -2611,6 +2624,8 @@ static void nvme_tcp_submit_async_event(struct nvme_ctrl *arg) ctrl->async_req.offset =0 ; ctrl->async_req.curr_bio =NULL ; ctrl->async_req.data_len =0 ; +init_llist_node (&ctrl->async_req.lentry); +INIT_LIST_HEAD (&ctrl->async_req.entry);nvme_tcp_queue_request (&ctrl->async_req, true); }
Signature · 1 line
2.35.3
Sagi Grimberg
maintainerRe: [PATCH 1/2] nvme-tcp: sanitize request list handling
Quoted 33 lines · Hannes Reinecke
Validate the request in
nvme_tcp_handle_r2t()to ensure it's not part of any list, otherwise a malicious R2T PDU might inject a loop in request list processing.Signed-off-by Hannes Reinecke <[email protected]>--- drivers/nvme/host/tcp.c | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-)drivers/nvme/host/tcp.c +4
-1@@ -452,7 +452,8 @@ nvme_tcp_fetch_request(struct nvme_tcp_queue *queue)return NULL ; } -list_del (&req->entry); +list_del_init (&req->entry); +init_llist_node (&req->lentry);return req; }@@ -560,6 +561,8 @@ static int nvme_tcp_init_request(struct blk_mq_tag_set *set, req->queue = queue;nvme_req (rq)->ctrl = &ctrl->ctrl;nvme_req (rq)->cmd = &pdu->cmd; +init_llist_node (&req->lentry); +INIT_LIST_HEAD (&req->entry);return 0 ; }
drivers/nvme/host/tcp.c static int nvme_tcp_handle_r2t(struct nvme_tcp_queue *queue, · line 770
Quoted 4 lines
drivers/nvme/host/tcp.c +1
-0@@ -764,6 +767,15 @@ static int nvme_tcp_handle_r2t(struct nvme_tcp_queue *queue,return -EPROTO ; } + if (queue->request == req ||
This is wrong, please remove this condition. queue->request can still reference the request when handling r2t.
PATCH v4 2/2 nvme-tcp: fix I/O stalls on congested sockets
In reply to Hannes ReineckeWhen the socket is busy processing nvme_tcp_try_recv() might return -EAGAIN, but this doesn't automatically imply that the sending side is blocked, too.
So check if there are pending requests once nvme_tcp_try_recv() returns -EAGAIN and continue with the sending loop to avoid I/O stalls.
--- drivers/nvme/host/tcp.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-)
drivers/nvme/host/tcp.c +6 -1
@@ -1363,7 +1363,7 @@ static int nvme_tcp_try_recv(struct nvme_tcp_queue *queue) queue->nr_cqe =0 ; consumed = sock->ops->read_sock (sk, &rd_desc, nvme_tcp_recv_skb);release_sock (sk); -return consumed; +return consumed == -EAGAIN ?0 : consumed; }static void nvme_tcp_io_work (struct work_struct *w)
@@ -1391,6 +1391,11 @@ static void nvme_tcp_io_work(struct work_struct *w)else if (unlikely (result <0 ))return ; +/* did we get some space after spending time in recv? */ +if (nvme_tcp_queue_has_pending (queue) && +sk_stream_is_writeable (queue->sock->sk)) + pending = true; +if (!pending || !queue->rd_enabled)return ;
Signature · 1 line
2.35.3
Sagi Grimberg
maintainerRe: [PATCH 2/2] nvme-tcp: fix I/O stalls on congested sockets
Quoted 28 lines · Hannes Reinecke
When the socket is busy processing
nvme_tcp_try_recv()might return-EAGAIN, but this doesn't automatically imply that the sending side is blocked, too.
So check if there are pending requests oncenvme_tcp_try_recv()returns-EAGAINand continue with the sending loop to avoid I/O stalls.Acked-by Chris Leech <[email protected]>Signed-off-by Hannes Reinecke <[email protected]>--- drivers/nvme/host/tcp.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-)drivers/nvme/host/tcp.c +1
-1@@ -1363,7 +1363,7 @@ static int nvme_tcp_try_recv(struct nvme_tcp_queue *queue) queue->nr_cqe =0 ; consumed = sock->ops->read_sock (sk, &rd_desc, nvme_tcp_recv_skb);release_sock (sk); -return consumed; +return consumed == -EAGAIN ?0 : consumed; }static void nvme_tcp_io_work (struct work_struct *w)@@ -1391,6 +1391,11 @@ static void nvme_tcp_io_work(struct work_struct *w)else if (unlikely (result <0 ))
drivers/nvme/host/tcp.c static void nvme_tcp_io_work(struct work_struct *w) · line 1401
Quoted 10 lines
drivers/nvme/host/tcp.c +5
-0return ; +/* did we get some space after spending time in recv? */ +if (nvme_tcp_queue_has_pending (queue) && +sk_stream_is_writeable (queue->sock->sk)) + pending = true; +if (!pending || !queue->rd_enabled)return ;
Looks fine,
Need Kamaljit to verify it.