Skip to contentSkip to conversation

nvme-rdma: avoid repeated request completion for concurrent nvme_rdma_timeout

PATCH Superseded Chao Leng 2 participants 5 messages Jan 6, 2021 – Jan 14, 2021 NVMe

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore

Chao Leng

author
#0 lore raw

PATCH 1/1 nvme-rdma: avoid repeated request completion for concurrent nvme_rdma_timeout

A crash happens when inject completing request long time(nearly 30s). Each name space has a request queue, when inject completing request long time, multi request queues may has time out requests at the same time, nvme_rdma_timeout will execute concurrently. Multi requests in different request queues may be queued in the same rdma queue, multi nvme_rdma_timeout may call nvme_rdma_stop_queue at the same time. The first nvme_rdma_timeout will clear NVME_RDMA_Q_LIVE and continue stopping the rdma queue(drain qp), but the others check NVME_RDMA_Q_LIVE is already cleared, and then directly complete the requests, but the rdma queue may be not stopped and the request may be already completed in qp and wait treated, the request will be repeated completed. Add a multex lock to serialize nvme_rdma_stop_queue.

---
 drivers/nvme/host/rdma.c | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)
drivers/nvme/host/rdma.c +11 -4
index df9f6f4549f1..d5c66e3eeb4a 100644
@@ -97,6 +97,7 @@ struct nvme_rdma_queue {
 	struct completion	cm_done;
 	bool			pi_support;
 	int			cq_size;
+	struct mutex		queue_lock;
 };
 
 struct nvme_rdma_ctrl {
@@ -579,6 +580,7 @@ static int nvme_rdma_alloc_queue(struct nvme_rdma_ctrl *ctrl,
 	int ret;
 
 	queue = &ctrl->queues[idx];
+	mutex_init(&queue->queue_lock);
 	queue->ctrl = ctrl;
 	if (idx && ctrl->ctrl.max_integrity_segments)
 		queue->pi_support = true;
@@ -598,7 +600,8 @@ static int nvme_rdma_alloc_queue(struct nvme_rdma_ctrl *ctrl,
 	if (IS_ERR(queue->cm_id)) {
 		dev_info(ctrl->ctrl.device,
 			"failed to create CM ID: %ld\n", PTR_ERR(queue->cm_id));
-		return PTR_ERR(queue->cm_id);
+		ret = PTR_ERR(queue->cm_id);
+		goto out_destroy_mutex;
 	}
 
 	if (ctrl->ctrl.opts->mask & NVMF_OPT_HOST_TRADDR)
@@ -628,6 +631,8 @@ static int nvme_rdma_alloc_queue(struct nvme_rdma_ctrl *ctrl,
 out_destroy_cm_id:
 	rdma_destroy_id(queue->cm_id);
 	nvme_rdma_destroy_queue_ib(queue);
+out_destroy_mutex:
+	mutex_destroy(&queue->queue_lock);
 	return ret;
 }
 
@@ -639,9 +644,10 @@ static void __nvme_rdma_stop_queue(struct nvme_rdma_queue *queue)
 
 static void nvme_rdma_stop_queue(struct nvme_rdma_queue *queue)
 {
-	if (!test_and_clear_bit(NVME_RDMA_Q_LIVE, &queue->flags))
-		return;
-	__nvme_rdma_stop_queue(queue);
+	mutex_lock(&queue->queue_lock);
+	if (test_and_clear_bit(NVME_RDMA_Q_LIVE, &queue->flags))
+		__nvme_rdma_stop_queue(queue);
+	mutex_unlock(&queue->queue_lock);
 }
 
 static void nvme_rdma_free_queue(struct nvme_rdma_queue *queue)
@@ -649,6 +655,7 @@ static void nvme_rdma_free_queue(struct nvme_rdma_queue *queue)
 	if (!test_and_clear_bit(NVME_RDMA_Q_ALLOCATED, &queue->flags))
 		return;
 
+	mutex_destroy(&queue->queue_lock);
 	nvme_rdma_destroy_queue_ib(queue);
 	rdma_destroy_id(queue->cm_id);
 }
Signature · 7 lines
2.16.4


_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
maintainer
#1 lore raw
Quoted 4 lines · Chao Leng

A crash happens when inject completing request long time(nearly 30s). Each name space has a request queue, when inject completing request long time, multi request queues may has time out requests at the same time, nvme_rdma_timeout will execute concurrently. Multi requests in different

Quoted 8 lines

request queues may be queued in the same rdma queue, multi nvme_rdma_timeout may call nvme_rdma_stop_queue at the same time. The first nvme_rdma_timeout will clear NVME_RDMA_Q_LIVE and continue stopping the rdma queue(drain qp), but the others check NVME_RDMA_Q_LIVE is already cleared, and then directly complete the requests, but the rdma queue may be not stopped and the request may be already completed in qp and wait treated, the request will be repeated completed. Add a multex lock to serialize nvme_rdma_stop_queue.

This looks reasonable to me,

Mind sending one for nvme-tcp as well?

Quoted 61 lines
---
  drivers/nvme/host/rdma.c | 15 +++++++++++----
  1 file changed, 11 insertions(+), 4 deletions(-)
drivers/nvme/host/rdma.c +10 -4
index df9f6f4549f1..d5c66e3eeb4a 100644
@@ -97,6 +97,7 @@ struct nvme_rdma_queue {
  	struct completion	cm_done;
  	bool			pi_support;
  	int			cq_size;
+	struct mutex		queue_lock;
  };
 
  struct nvme_rdma_ctrl {
@@ -579,6 +580,7 @@ static int nvme_rdma_alloc_queue(struct nvme_rdma_ctrl *ctrl,
  	int ret;
 
  	queue = &ctrl->queues[idx];
+	mutex_init(&queue->queue_lock);
  	queue->ctrl = ctrl;
  	if (idx && ctrl->ctrl.max_integrity_segments)
  		queue->pi_support = true;
@@ -598,7 +600,8 @@ static int nvme_rdma_alloc_queue(struct nvme_rdma_ctrl *ctrl,
  	if (IS_ERR(queue->cm_id)) {
  		dev_info(ctrl->ctrl.device,
  			"failed to create CM ID: %ld\n", PTR_ERR(queue->cm_id));
-		return PTR_ERR(queue->cm_id);
+		ret = PTR_ERR(queue->cm_id);
+		goto out_destroy_mutex;
  	}
 
  	if (ctrl->ctrl.opts->mask & NVMF_OPT_HOST_TRADDR)
@@ -628,6 +631,8 @@ static int nvme_rdma_alloc_queue(struct nvme_rdma_ctrl *ctrl,
  out_destroy_cm_id:
  	rdma_destroy_id(queue->cm_id);
  	nvme_rdma_destroy_queue_ib(queue);
+out_destroy_mutex:
+	mutex_destroy(&queue->queue_lock);
  	return ret;
  }
 
@@ -639,9 +644,10 @@ static void __nvme_rdma_stop_queue(struct nvme_rdma_queue *queue)
 
  static void nvme_rdma_stop_queue(struct nvme_rdma_queue *queue)
  {
-	if (!test_and_clear_bit(NVME_RDMA_Q_LIVE, &queue->flags))
-		return;
-	__nvme_rdma_stop_queue(queue);
+	mutex_lock(&queue->queue_lock);
+	if (test_and_clear_bit(NVME_RDMA_Q_LIVE, &queue->flags))
+		__nvme_rdma_stop_queue(queue);
+	mutex_unlock(&queue->queue_lock);
  }
 
  static void nvme_rdma_free_queue(struct nvme_rdma_queue *queue)

drivers/nvme/host/rdma.c static void nvme_rdma_free_queue(struct nvme_rdma_queue *queue) · line 661

Quoted 7 lines
drivers/nvme/host/rdma.c +1 -0
@@ -649,6 +655,7 @@ static void nvme_rdma_free_queue(struct nvme_rdma_queue *queue)
  	if (!test_and_clear_bit(NVME_RDMA_Q_ALLOCATED, &queue->flags))
  		return;
 
+	mutex_destroy(&queue->queue_lock);
  	nvme_rdma_destroy_queue_ib(queue);
  	rdma_destroy_id(queue->cm_id);
  }

_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme

maintainer
#2 lore raw
Quoted 15 lines · Chao Leng
Quoted 4 lines

A crash happens when inject completing request long time(nearly 30s). Each name space has a request queue, when inject completing request long time, multi request queues may has time out requests at the same time, nvme_rdma_timeout will execute concurrently. Multi requests in different

Quoted 8 lines

request queues may be queued in the same rdma queue, multi nvme_rdma_timeout may call nvme_rdma_stop_queue at the same time. The first nvme_rdma_timeout will clear NVME_RDMA_Q_LIVE and continue stopping the rdma queue(drain qp), but the others check NVME_RDMA_Q_LIVE is already cleared, and then directly complete the requests, but the rdma queue may be not stopped and the request may be already completed in qp and wait treated, the request will be repeated completed. Add a multex lock to serialize nvme_rdma_stop_queue.

Quoted 3 lines

This looks reasonable to me,

Mind sending one for nvme-tcp as well?

BTW, I'm assuming you mean use-after-free or double completion when you are referring to repeated completions? I think it would be easier to understand if you just say that completing request before the qp is fully drained may lead to a use-after-free condition.

_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme

author
#3 lore raw
Quoted 15 lines · Sagi Grimberg
Quoted 15 lines · Chao Leng
Quoted 12 lines

A crash happens when inject completing request long time(nearly 30s). Each name space has a request queue, when inject completing request long time, multi request queues may has time out requests at the same time, nvme_rdma_timeout will execute concurrently. Multi requests in different request queues may be queued in the same rdma queue, multi nvme_rdma_timeout may call nvme_rdma_stop_queue at the same time. The first nvme_rdma_timeout will clear NVME_RDMA_Q_LIVE and continue stopping the rdma queue(drain qp), but the others check NVME_RDMA_Q_LIVE is already cleared, and then directly complete the requests, but the rdma queue may be not stopped and the request may be already completed in qp and wait treated, the request will be repeated completed. Add a multex lock to serialize nvme_rdma_stop_queue.

Quoted 8 lines
Quoted 3 lines

This looks reasonable to me,

Mind sending one for nvme-tcp as well?

BTW, I'm assuming you mean use-after-free or double completion when you are referring to repeated completions? I think it would be easier to understand if you just say that completing request before the qp is fully drained may lead to a use-after-free condition.

ok, thanks for your suggestion.

Quoted 1 line

.

_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme

author
#4 lore raw
In reply to Sagi Grimberg
Quoted 15 lines · Sagi Grimberg
Quoted 4 lines · Chao Leng

A crash happens when inject completing request long time(nearly 30s). Each name space has a request queue, when inject completing request long time, multi request queues may has time out requests at the same time, nvme_rdma_timeout will execute concurrently. Multi requests in different

Quoted 8 lines

request queues may be queued in the same rdma queue, multi nvme_rdma_timeout may call nvme_rdma_stop_queue at the same time. The first nvme_rdma_timeout will clear NVME_RDMA_Q_LIVE and continue stopping the rdma queue(drain qp), but the others check NVME_RDMA_Q_LIVE is already cleared, and then directly complete the requests, but the rdma queue may be not stopped and the request may be already completed in qp and wait treated, the request will be repeated completed. Add a multex lock to serialize nvme_rdma_stop_queue.

Quoted 3 lines

This looks reasonable to me,

Mind sending one for nvme-tcp as well?

ok, I will send patch v2 latter.

Quoted 55 lines
Quoted 55 lines

---
  drivers/nvme/host/rdma.c | 15 +++++++++++----

  1 file changed, 11 insertions(+), 4 deletions(-)
drivers/nvme/host/rdma.c +4 -1
index df9f6f4549f1..d5c66e3eeb4a 100644
@@ -97,6 +97,7 @@ struct nvme_rdma_queue {
       struct completion    cm_done;
       bool            pi_support;
       int            cq_size;
+    struct mutex        queue_lock;
   };
   struct nvme_rdma_ctrl {
@@ -579,6 +580,7 @@ static int nvme_rdma_alloc_queue(struct nvme_rdma_ctrl *ctrl,
       int ret;
       queue = &ctrl->queues[idx];
+    mutex_init(&queue->queue_lock);
       queue->ctrl = ctrl;
       if (idx && ctrl->ctrl.max_integrity_segments)
           queue->pi_support = true;
@@ -598,7 +600,8 @@ static int nvme_rdma_alloc_queue(struct nvme_rdma_ctrl *ctrl,
       if (IS_ERR(queue->cm_id)) {
           dev_info(ctrl->ctrl.device,
               "failed to create CM ID: %ld\n", PTR_ERR(queue->cm_id));
-        return PTR_ERR(queue->cm_id);
+        ret = PTR_ERR(queue->cm_id);
+        goto out_destroy_mutex;
       }
       if (ctrl->ctrl.opts->mask & NVMF_OPT_HOST_TRADDR)
@@ -628,6 +631,8 @@ static int nvme_rdma_alloc_queue(struct nvme_rdma_ctrl *ctrl,

  out_destroy_cm_id:
      rdma_destroy_id(queue->cm_id);
      nvme_rdma_destroy_queue_ib(queue);
+out_destroy_mutex:
+    mutex_destroy(&queue->queue_lock);
      return ret;
  }

drivers/nvme/host/rdma.c +4 -3
@@ -639,9 +644,10 @@ static void __nvme_rdma_stop_queue(struct nvme_rdma_queue *queue)
   static void nvme_rdma_stop_queue(struct nvme_rdma_queue *queue)
   {
-    if (!test_and_clear_bit(NVME_RDMA_Q_LIVE, &queue->flags))
-        return;
-    __nvme_rdma_stop_queue(queue);
+    mutex_lock(&queue->queue_lock);
+    if (test_and_clear_bit(NVME_RDMA_Q_LIVE, &queue->flags))
+        __nvme_rdma_stop_queue(queue);
+    mutex_unlock(&queue->queue_lock);
   }
   static void nvme_rdma_free_queue(struct nvme_rdma_queue *queue)
@@ -649,6 +655,7 @@ static void nvme_rdma_free_queue(struct nvme_rdma_queue *queue)
Quoted 8 lines

drivers/nvme/host/rdma.c static void nvme_rdma_free_queue(struct nvme_rdma_queue *queue)

Quoted 7 lines
drivers/nvme/host/rdma.c +1 -0
       if (!test_and_clear_bit(NVME_RDMA_Q_ALLOCATED, &queue->flags))
           return;
+    mutex_destroy(&queue->queue_lock);
       nvme_rdma_destroy_queue_ib(queue);
       rdma_destroy_id(queue->cm_id);
   }
 

.

_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme