Skip to contentSkip to conversation

fix error unwinding in nvme_map_data

PATCH In review Christoph Hellwig 3 participants 8 messages Jan 20, 2021 NVMe

lore
Share & tools
lore
author maintainer
#1 lore raw

PATCH 1/2 nvme-pci: refactor nvme_unmap_data

Split out three helpers from nvme_unmap_data that will allow finer grained unwinding from nvme_map_data.

---
 drivers/nvme/host/pci.c | 77 ++++++++++++++++++++++++++---------------
 1 file changed, 49 insertions(+), 28 deletions(-)
drivers/nvme/host/pci.c +49 -28
index 25456d02eddb8c..e29ece9e4d4b8e 100644
@@ -543,50 +543,71 @@ static inline bool nvme_pci_use_sgls(struct nvme_dev *dev, struct request *req)
 	return true;
 }
 
-static void nvme_unmap_data(struct nvme_dev *dev, struct request *req)
+static void nvme_free_prps(struct nvme_dev *dev, struct request *req)
 {
-	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 	const int last_prp = NVME_CTRL_PAGE_SIZE / sizeof(__le64) - 1;
-	dma_addr_t dma_addr = iod->first_dma, next_dma_addr;
+	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
+	dma_addr_t dma_addr = iod->first_dma;
 	int i;
 
-	if (iod->dma_len) {
-		dma_unmap_page(dev->dev, dma_addr, iod->dma_len,
-			       rq_dma_dir(req));
-		return;
+	for (i = 0; i < iod->npages; i++) {
+		__le64 *prp_list = nvme_pci_iod_list(req)[i];
+		dma_addr_t next_dma_addr = le64_to_cpu(prp_list[last_prp]);
+
+		dma_pool_free(dev->prp_page_pool, prp_list, dma_addr);
+		dma_addr = next_dma_addr;
 	}
 
-	WARN_ON_ONCE(!iod->nents);
+}
 
-	if (is_pci_p2pdma_page(sg_page(iod->sg)))
-		pci_p2pdma_unmap_sg(dev->dev, iod->sg, iod->nents,
-				    rq_dma_dir(req));
-	else
-		dma_unmap_sg(dev->dev, iod->sg, iod->nents, rq_dma_dir(req));
+static void nvme_free_sgls(struct nvme_dev *dev, struct request *req)
+{
+	const int last_sg = SGES_PER_PAGE - 1;
+	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
+	dma_addr_t dma_addr = iod->first_dma;
+	int i;
 
+	for (i = 0; i < iod->npages; i++) {
+		struct nvme_sgl_desc *sg_list = nvme_pci_iod_list(req)[i];
+		dma_addr_t next_dma_addr = le64_to_cpu((sg_list[last_sg]).addr);
 
-	if (iod->npages == 0)
-		dma_pool_free(dev->prp_small_pool, nvme_pci_iod_list(req)[0],
-			dma_addr);
+		dma_pool_free(dev->prp_page_pool, sg_list, dma_addr);
+		dma_addr = next_dma_addr;
+	}
 
-	for (i = 0; i < iod->npages; i++) {
-		void *addr = nvme_pci_iod_list(req)[i];
+}
 
-		if (iod->use_sgl) {
-			struct nvme_sgl_desc *sg_list = addr;
+static void nvme_unmap_sg(struct nvme_dev *dev, struct request *req)
+{
+	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 
-			next_dma_addr =
-			    le64_to_cpu((sg_list[SGES_PER_PAGE - 1]).addr);
-		} else {
-			__le64 *prp_list = addr;
+	if (is_pci_p2pdma_page(sg_page(iod->sg)))
+		pci_p2pdma_unmap_sg(dev->dev, iod->sg, iod->nents,
+				    rq_dma_dir(req));
+	else
+		dma_unmap_sg(dev->dev, iod->sg, iod->nents, rq_dma_dir(req));
+}
 
-			next_dma_addr = le64_to_cpu(prp_list[last_prp]);
-		}
+static void nvme_unmap_data(struct nvme_dev *dev, struct request *req)
+{
+	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 
-		dma_pool_free(dev->prp_page_pool, addr, dma_addr);
-		dma_addr = next_dma_addr;
+	if (iod->dma_len) {
+		dma_unmap_page(dev->dev, iod->first_dma, iod->dma_len,
+			       rq_dma_dir(req));
+		return;
 	}
 
+	WARN_ON_ONCE(!iod->nents);
+
+	nvme_unmap_sg(dev, req);
+	if (iod->npages == 0)
+		dma_pool_free(dev->prp_small_pool, nvme_pci_iod_list(req)[0],
+			      iod->first_dma);
+	else if (iod->use_sgl)
+		nvme_free_sgls(dev, req);
+	else
+		nvme_free_prps(dev, req);
 	mempool_free(iod->sg, dev->iod_mempool);
 }
 
Signature · 7 lines
2.29.2


_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Reviewed-by on v1 1/2
#2 lore raw

Re: [PATCH 1/2] nvme-pci: refactor nvme_unmap_data

Quoted 102 lines · Christoph Hellwig

Split out three helpers from nvme_unmap_data that will allow finer grained unwinding from nvme_map_data.

---
 drivers/nvme/host/pci.c | 77 ++++++++++++++++++++++++++---------------
 1 file changed, 49 insertions(+), 28 deletions(-)
drivers/nvme/host/pci.c +41 -28
index 25456d02eddb8c..e29ece9e4d4b8e 100644
@@ -543,50 +543,71 @@ static inline bool nvme_pci_use_sgls(struct nvme_dev *dev, struct request *req)
        return true;
 }
 
-static void nvme_unmap_data(struct nvme_dev *dev, struct request *req)
+static void nvme_free_prps(struct nvme_dev *dev, struct request *req)
 {
-       struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
        const int last_prp = NVME_CTRL_PAGE_SIZE / sizeof(__le64) - 1;
-       dma_addr_t dma_addr = iod->first_dma, next_dma_addr;
+       struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
+       dma_addr_t dma_addr = iod->first_dma;
        int i;
 
-       if (iod->dma_len) {
-               dma_unmap_page(dev->dev, dma_addr, iod->dma_len,
-                              rq_dma_dir(req));
-               return;
+       for (i = 0; i < iod->npages; i++) {
+               __le64 *prp_list = nvme_pci_iod_list(req)[i];
+               dma_addr_t next_dma_addr = le64_to_cpu(prp_list[last_prp]);
+
+               dma_pool_free(dev->prp_page_pool, prp_list, dma_addr);
+               dma_addr = next_dma_addr;
        }
 
-       WARN_ON_ONCE(!iod->nents);
+}
 
-       if (is_pci_p2pdma_page(sg_page(iod->sg)))
-               pci_p2pdma_unmap_sg(dev->dev, iod->sg, iod->nents,
-                                   rq_dma_dir(req));
-       else
-               dma_unmap_sg(dev->dev, iod->sg, iod->nents, rq_dma_dir(req));
+static void nvme_free_sgls(struct nvme_dev *dev, struct request *req)
+{
+       const int last_sg = SGES_PER_PAGE - 1;
+       struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
+       dma_addr_t dma_addr = iod->first_dma;
+       int i;
 
+       for (i = 0; i < iod->npages; i++) {
+               struct nvme_sgl_desc *sg_list = nvme_pci_iod_list(req)[i];
+               dma_addr_t next_dma_addr = le64_to_cpu((sg_list[last_sg]).addr);
 
-       if (iod->npages == 0)
-               dma_pool_free(dev->prp_small_pool, nvme_pci_iod_list(req)[0],
-                       dma_addr);
+               dma_pool_free(dev->prp_page_pool, sg_list, dma_addr);
+               dma_addr = next_dma_addr;
+       }
 
-       for (i = 0; i < iod->npages; i++) {
-               void *addr = nvme_pci_iod_list(req)[i];
+}
 
-               if (iod->use_sgl) {
-                       struct nvme_sgl_desc *sg_list = addr;
+static void nvme_unmap_sg(struct nvme_dev *dev, struct request *req)
+{
+       struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 
-                       next_dma_addr =
-                           le64_to_cpu((sg_list[SGES_PER_PAGE - 1]).addr);
-               } else {
-                       __le64 *prp_list = addr;
+       if (is_pci_p2pdma_page(sg_page(iod->sg)))
+               pci_p2pdma_unmap_sg(dev->dev, iod->sg, iod->nents,
+                                   rq_dma_dir(req));
+       else
+               dma_unmap_sg(dev->dev, iod->sg, iod->nents, rq_dma_dir(req));
+}
 
-                       next_dma_addr = le64_to_cpu(prp_list[last_prp]);
-               }
+static void nvme_unmap_data(struct nvme_dev *dev, struct request *req)
+{
+       struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 
-               dma_pool_free(dev->prp_page_pool, addr, dma_addr);
-               dma_addr = next_dma_addr;
+       if (iod->dma_len) {
+               dma_unmap_page(dev->dev, iod->first_dma, iod->dma_len,
+                              rq_dma_dir(req));
+               return;
        }
 
+       WARN_ON_ONCE(!iod->nents);
+

drivers/nvme/host/pci.c static inline bool nvme_pci_use_sgls(struct nvme_dev *dev, struct request *req) · line 612

Quoted 10 lines
drivers/nvme/host/pci.c +8 -0
+       nvme_unmap_sg(dev, req);
+       if (iod->npages == 0)
+               dma_pool_free(dev->prp_small_pool, nvme_pci_iod_list(req)[0],
+                             iod->first_dma);
+       else if (iod->use_sgl)
+               nvme_free_sgls(dev, req);
+       else
+               nvme_free_prps(dev, req);
        mempool_free(iod->sg, dev->iod_mempool);
 }
Quoted 4 lines
drivers/nvme/host/pci.c +0 -1
--

2.29.2

_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme

author maintainer
#3 lore raw

PATCH 2/2 nvme-pci: fix error unwind in nvme_map_data

In reply to Christoph Hellwig

Properly unwind step by step using refactored helpers from nvme_unmap_data to avoid a potential double dma_unmap on a mapping failure.

---
 drivers/nvme/host/pci.c | 28 ++++++++++++++++++----------
 1 file changed, 18 insertions(+), 10 deletions(-)
drivers/nvme/host/pci.c +18 -10
index e29ece9e4d4b8e..4e93d1b52df202 100644
@@ -683,7 +683,7 @@ static blk_status_t nvme_pci_setup_prps(struct nvme_dev *dev,
 			__le64 *old_prp_list = prp_list;
 			prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);
 			if (!prp_list)
-				return BLK_STS_RESOURCE;
+				goto free_prps;
 			list[iod->npages++] = prp_list;
 			prp_list[0] = old_prp_list[i - 1];
 			old_prp_list[i - 1] = cpu_to_le64(prp_dma);
@@ -703,14 +703,14 @@ static blk_status_t nvme_pci_setup_prps(struct nvme_dev *dev,
 		dma_addr = sg_dma_address(sg);
 		dma_len = sg_dma_len(sg);
 	}
-
 done:
 	cmnd->dptr.prp1 = cpu_to_le64(sg_dma_address(iod->sg));
 	cmnd->dptr.prp2 = cpu_to_le64(iod->first_dma);
-
 	return BLK_STS_OK;
-
- bad_sgl:
+free_prps:
+	nvme_free_prps(dev, req);
+	return BLK_STS_RESOURCE;
+bad_sgl:
 	WARN(DO_ONCE(nvme_print_sgl, iod->sg, iod->nents),
 			"Invalid SGL for payload:%d nents:%d\n",
 			blk_rq_payload_bytes(req), iod->nents);
@@ -782,7 +782,7 @@ static blk_status_t nvme_pci_setup_sgls(struct nvme_dev *dev,
 
 			sg_list = dma_pool_alloc(pool, GFP_ATOMIC, &sgl_dma);
 			if (!sg_list)
-				return BLK_STS_RESOURCE;
+				goto free_sgls;
 
 			i = 0;
 			nvme_pci_iod_list(req)[iod->npages++] = sg_list;
@@ -795,6 +795,9 @@ static blk_status_t nvme_pci_setup_sgls(struct nvme_dev *dev,
 	} while (--entries > 0);
 
 	return BLK_STS_OK;
+free_sgls:
+	nvme_free_sgls(dev, req);
+	return BLK_STS_RESOURCE;
 }
 
 static blk_status_t nvme_setup_prp_simple(struct nvme_dev *dev,
@@ -863,7 +866,7 @@ static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req,
 	sg_init_table(iod->sg, blk_rq_nr_phys_segments(req));
 	iod->nents = blk_rq_map_sg(req->q, req, iod->sg);
 	if (!iod->nents)
-		goto out;
+		goto out_free_sg;
 
 	if (is_pci_p2pdma_page(sg_page(iod->sg)))
 		nr_mapped = pci_p2pdma_map_sg_attrs(dev->dev, iod->sg,
@@ -872,16 +875,21 @@ static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req,
 		nr_mapped = dma_map_sg_attrs(dev->dev, iod->sg, iod->nents,
 					     rq_dma_dir(req), DMA_ATTR_NO_WARN);
 	if (!nr_mapped)
-		goto out;
+		goto out_free_sg;
 
 	iod->use_sgl = nvme_pci_use_sgls(dev, req);
 	if (iod->use_sgl)
 		ret = nvme_pci_setup_sgls(dev, req, &cmnd->rw, nr_mapped);
 	else
 		ret = nvme_pci_setup_prps(dev, req, &cmnd->rw);
-out:
 	if (ret != BLK_STS_OK)
-		nvme_unmap_data(dev, req);
+		goto out_dma_unmap;
+	return BLK_STS_OK;
+
+out_dma_unmap:
+	nvme_unmap_sg(dev, req);
+out_free_sg:
+	mempool_free(iod->sg, dev->iod_mempool);
 	return ret;
 }
 
Signature · 7 lines
2.29.2


_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Reviewed-by on v1 2/2
#4 lore raw

Re: [PATCH 2/2] nvme-pci: fix error unwind in nvme_map_data

Quoted 6 lines · Christoph Hellwig

Properly unwind step by step using refactored helpers from nvme_unmap_data to avoid a potential double dma_unmap on a mapping failure.

Should these patches go to stable?

Quoted 73 lines
---
 drivers/nvme/host/pci.c | 28 ++++++++++++++++++----------
 1 file changed, 18 insertions(+), 10 deletions(-)
drivers/nvme/host/pci.c +11 -8
index e29ece9e4d4b8e..4e93d1b52df202 100644
@@ -683,7 +683,7 @@ static blk_status_t nvme_pci_setup_prps(struct nvme_dev *dev,
                        __le64 *old_prp_list = prp_list;
                        prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);
                        if (!prp_list)
-                               return BLK_STS_RESOURCE;
+                               goto free_prps;
                        list[iod->npages++] = prp_list;
                        prp_list[0] = old_prp_list[i - 1];
                        old_prp_list[i - 1] = cpu_to_le64(prp_dma);
@@ -703,14 +703,14 @@ static blk_status_t nvme_pci_setup_prps(struct nvme_dev *dev,
                dma_addr = sg_dma_address(sg);
                dma_len = sg_dma_len(sg);
        }
-
 done:
        cmnd->dptr.prp1 = cpu_to_le64(sg_dma_address(iod->sg));
        cmnd->dptr.prp2 = cpu_to_le64(iod->first_dma);
-
        return BLK_STS_OK;
-
- bad_sgl:
+free_prps:
+       nvme_free_prps(dev, req);
+       return BLK_STS_RESOURCE;
+bad_sgl:
        WARN(DO_ONCE(nvme_print_sgl, iod->sg, iod->nents),
                        "Invalid SGL for payload:%d nents:%d\n",
                        blk_rq_payload_bytes(req), iod->nents);
@@ -782,7 +782,7 @@ static blk_status_t nvme_pci_setup_sgls(struct nvme_dev *dev,
 
                        sg_list = dma_pool_alloc(pool, GFP_ATOMIC, &sgl_dma);
                        if (!sg_list)
-                               return BLK_STS_RESOURCE;
+                               goto free_sgls;
 
                        i = 0;
                        nvme_pci_iod_list(req)[iod->npages++] = sg_list;
@@ -795,6 +795,9 @@ static blk_status_t nvme_pci_setup_sgls(struct nvme_dev *dev,
        } while (--entries > 0);
 
        return BLK_STS_OK;
+free_sgls:
+       nvme_free_sgls(dev, req);
+       return BLK_STS_RESOURCE;
 }
 
 static blk_status_t nvme_setup_prp_simple(struct nvme_dev *dev,
@@ -863,7 +866,7 @@ static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req,
        sg_init_table(iod->sg, blk_rq_nr_phys_segments(req));
        iod->nents = blk_rq_map_sg(req->q, req, iod->sg);
        if (!iod->nents)
-               goto out;
+               goto out_free_sg;
 
        if (is_pci_p2pdma_page(sg_page(iod->sg)))
                nr_mapped = pci_p2pdma_map_sg_attrs(dev->dev, iod->sg,
@@ -872,16 +875,21 @@ static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req,
                nr_mapped = dma_map_sg_attrs(dev->dev, iod->sg, iod->nents,
                                             rq_dma_dir(req), DMA_ATTR_NO_WARN);
        if (!nr_mapped)
-               goto out;
+               goto out_free_sg;
 
        iod->use_sgl = nvme_pci_use_sgls(dev, req);
        if (iod->use_sgl)

drivers/nvme/host/pci.c static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req, · line 889

Quoted 10 lines
drivers/nvme/host/pci.c +4 -2
                ret = nvme_pci_setup_sgls(dev, req, &cmnd->rw, nr_mapped);
        else
                ret = nvme_pci_setup_prps(dev, req, &cmnd->rw);
-out:
        if (ret != BLK_STS_OK)
-               nvme_unmap_data(dev, req);
+               goto out_dma_unmap;
+       return BLK_STS_OK;
+
+out_dma_unmap:

nit: Naming this label out_sg_unmap or out_unmap_sg might be a little more clear, since the line below doesn't have the word dma anywhere.

drivers/nvme/host/pci.c static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req, · line 894

Quoted 5 lines
drivers/nvme/host/pci.c +3 -0
+       nvme_unmap_sg(dev, req);
+out_free_sg:
+       mempool_free(iod->sg, dev->iod_mempool);
        return ret;
 }
Quoted 4 lines
drivers/nvme/host/pci.c +0 -1
--

2.29.2

_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme

author maintainer
#5 lore raw

Re: [PATCH 2/2] nvme-pci: fix error unwind in nvme_map_data

Quoted 9 lines · Marc Orr
Quoted 6 lines · Christoph Hellwig

Properly unwind step by step using refactored helpers from nvme_unmap_data to avoid a potential double dma_unmap on a mapping failure.

Should these patches go to stable?

Yes. I've added the proper fixes tag:

Fixes 7fe07d14f71f ("nvme-pci: merge nvme_free_iod into nvme_unmap_data")

_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme

maintainer Reviewed-by
#6 lore raw

Re: [PATCH 2/2] nvme-pci: fix error unwind in nvme_map_data

Quoted 6 lines · Christoph Hellwig
Quoted 5 lines · Marc Orr
Quoted 4 lines · Christoph Hellwig

Properly unwind step by step using refactored helpers from nvme_unmap_data to avoid a potential double dma_unmap on a mapping failure.

Quoted 8 lines
Quoted 4 lines
Quoted 2 lines

Should these patches go to stable?

Yes. I've added the proper fixes tag:

Fixes 7fe07d14f71f ("nvme-pci: merge nvme_free_iod into nvme_unmap_data")

Patches look good.

Just fyi, the 2/2 is the only one with the "Fixes" tag, but it depends on 1/2 to compile. It applies cleanly to the recent stables, though, so stable bot might have some trouble with it.

_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme

author maintainer
#7 lore raw

Re: [PATCH 2/2] nvme-pci: fix error unwind in nvme_map_data

Quoted 16 lines · Keith Busch
Quoted 6 lines · Christoph Hellwig
Quoted 5 lines · Marc Orr
Quoted 4 lines · Christoph Hellwig

Properly unwind step by step using refactored helpers from nvme_unmap_data to avoid a potential double dma_unmap on a mapping failure.

Quoted 8 lines
Quoted 4 lines
Quoted 2 lines

Should these patches go to stable?

Yes. I've added the proper fixes tag:

Fixes 7fe07d14f71f ("nvme-pci: merge nvme_free_iod into nvme_unmap_data")
Quoted 7 lines

Patches look good.

Just fyi, the 2/2 is the only one with the "Fixes" tag, but it depends on 1/2 to compile. It applies cleanly to the recent stables, though, so stable bot might have some trouble with it.

patch 2 obviously depends on 1, the bot has been pretty good at figuring that out.

_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme