Skip to contentSkip to conversation

nvme-multipath: call del_gendisk() in nvme_mpath_check_last_path()

PATCH Superseded Hannes Reinecke 2 participants 4 messages May 31, 2021 – Jun 7, 2021 NVMe

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 nvme-multipath: call del_gendisk() in nvme_mpath_check_last_path()

We cannot call del_gendisk() during the final nvme_put_ns_head(), as this might be called from blkdev_put(). Doing so will deadlock on bd_mutex():

[ 1108.529767]  __mutex_lock.constprop.0+0x2a4/0x470
[ 1108.534472]  ? __kernfs_remove.part.0+0x174/0x1f0
[ 1108.539178]  ? kernfs_remove_by_name_ns+0x5c/0x90
[ 1108.543885]  del_gendisk+0x99/0x230
[ 1108.547378]  nvme_mpath_remove_disk+0x97/0xb0 [nvme_core]
[ 1108.552787]  nvme_put_ns_head+0x2a/0xb0 [nvme_core]
[ 1108.557664]  __blkdev_put+0x115/0x160
[ 1108.561339]  blkdev_put+0x4c/0x130
[ 1108.564745]  blkdev_close+0x22/0x30
[ 1108.568238]  __fput+0x94/0x240

So call 'del_gendisk()' when checking if the last path has been removed to avoid this deadlock.

---
 drivers/nvme/host/multipath.c | 13 +++++++++++--
 drivers/nvme/host/nvme.h      |  9 +--------
 2 files changed, 12 insertions(+), 10 deletions(-)
drivers/nvme/host/multipath.c +11 -2
index 127a17b4c13d..22e2febf86bc 100644
@@ -776,14 +776,23 @@ void nvme_mpath_add_disk(struct nvme_ns *ns, struct nvme_id_ns *id)
 #endif
 }
 
-void nvme_mpath_remove_disk(struct nvme_ns_head *head)
+void nvme_mpath_check_last_path(struct nvme_ns *ns)
 {
+	struct nvme_ns_head *head = ns->head;
+
 	if (!head->disk)
 		return;
-	if (head->disk->flags & GENHD_FL_UP) {
+
+	if (list_empty(&head->list) && head->disk->flags & GENHD_FL_UP) {
 		nvme_cdev_del(&head->cdev, &head->cdev_device);
 		del_gendisk(head->disk);
 	}
+}
+
+void nvme_mpath_remove_disk(struct nvme_ns_head *head)
+{
+	if (!head->disk)
+		return;
 	blk_set_queue_dying(head->disk->queue);
 	/* make sure all pending bios are cleaned up */
 	kblockd_schedule_work(&head->requeue_work);
drivers/nvme/host/nvme.h +1 -8
index 1f397ecba16c..8a497a420ea2 100644
@@ -716,14 +716,7 @@ void nvme_mpath_uninit(struct nvme_ctrl *ctrl);
 void nvme_mpath_stop(struct nvme_ctrl *ctrl);
 bool nvme_mpath_clear_current_path(struct nvme_ns *ns);
 void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl);
-
-static inline void nvme_mpath_check_last_path(struct nvme_ns *ns)
-{
-	struct nvme_ns_head *head = ns->head;
-
-	if (head->disk && list_empty(&head->list))
-		kblockd_schedule_work(&head->requeue_work);
-}
+void nvme_mpath_check_last_path(struct nvme_ns *ns);
 
 static inline void nvme_trace_bio_complete(struct request *req)
 {
Signature · 7 lines
2.29.2


_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
#1 lore raw

drivers/nvme/host/multipath.c void nvme_mpath_add_disk(struct nvme_ns *ns, struct nvme_id_ns *id) · line 786

Quoted 9 lines
drivers/nvme/host/multipath.c +5 -1
+void nvme_mpath_check_last_path(struct nvme_ns *ns)
 {
+	struct nvme_ns_head *head = ns->head;
+
 	if (!head->disk)
 		return;
-	if (head->disk->flags & GENHD_FL_UP) {
+
+	if (list_empty(&head->list) && head->disk->flags & GENHD_FL_UP) {

Please add the braces around the "&".

drivers/nvme/host/multipath.c void nvme_mpath_add_disk(struct nvme_ns *ns, struct nvme_id_ns *id) · line 790

Quoted 4 lines
drivers/nvme/host/multipath.c +1 -0
 		nvme_cdev_del(&head->cdev, &head->cdev_device);
 		del_gendisk(head->disk);
 	}
+}

Also it seems like we can just pass the head here, no need for ns.

drivers/nvme/host/nvme.h void nvme_mpath_uninit(struct nvme_ctrl *ctrl); · line 725

Quoted 2 lines
drivers/nvme/host/nvme.h +0 -2
-	if (head->disk && list_empty(&head->list))
-		kblockd_schedule_work(&head->requeue_work);

Did this get lost in the new version?

_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme

#3 lore raw
In reply to Christoph Hellwig
Quoted 12 lines · Christoph Hellwig

drivers/nvme/host/multipath.c void nvme_mpath_add_disk(struct nvme_ns *ns, struct nvme_id_ns *id) · line 786

Quoted 9 lines
drivers/nvme/host/multipath.c +5 -1
+void nvme_mpath_check_last_path(struct nvme_ns *ns)
 {
+	struct nvme_ns_head *head = ns->head;
+
 	if (!head->disk)
 		return;
-	if (head->disk->flags & GENHD_FL_UP) {
+
+	if (list_empty(&head->list) && head->disk->flags & GENHD_FL_UP) {

Please add the braces around the "&".

Ok.

Quoted 7 lines

drivers/nvme/host/multipath.c void nvme_mpath_add_disk(struct nvme_ns *ns, struct nvme_id_ns *id) · line 790

Quoted 4 lines
drivers/nvme/host/multipath.c +1 -0
 		nvme_cdev_del(&head->cdev, &head->cdev_device);
 		del_gendisk(head->disk);
 	}
+}

Also it seems like we can just pass the head here, no need for ns.

Ok.

Quoted 5 lines

drivers/nvme/host/nvme.h void nvme_mpath_uninit(struct nvme_ctrl *ctrl); · line 725

Quoted 2 lines
drivers/nvme/host/nvme.h +0 -2
-	if (head->disk && list_empty(&head->list))
-		kblockd_schedule_work(&head->requeue_work);

Did this get lost in the new version?

I had removed it as it'll be called from nvme_mpath_remove_disk() anyway, but seeing that the refcount might induce a different lifetime I'll be re-adding it.

Cheers,

Hannes

Signature · 9 lines
Dr. Hannes Reinecke		        Kernel Storage Architect
[email protected]			               +49 911 74053 688
SUSE Software Solutions Germany GmbH, 90409 Nürnberg
GF: F. Imendörffer, HRB 36809 (AG Nürnberg)

_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme