Skip to contentSkip to conversation

nvme: fix dcache invalidation range in identify command

PATCH Superseded Prashant Kamble 2 participants 5 messages May 19 – Jun 3

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 nvme: fix dcache invalidation range in identify command

When the identify buffer crosses a page boundary, PRP2 is used and dma_addr is advanced to the second page:

    dma_addr += (page_size - offset);

The subsequent invalidate_dcache_range() calls then use the modified dma_addr instead of the original buffer start address.

As a result, the beginning of the identify buffer is not invalidated and the invalidation range extends past the end of the buffer.

Fix this by preserving the original DMA buffer address for cache invalidation.

---
 drivers/nvme/nvme.c | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)
drivers/nvme/nvme.c +5 -4
index 4f9473367d3..298010de536 100644
@@ -456,6 +456,7 @@ int nvme_identify(struct nvme_dev *dev, unsigned nsid,
 	u32 page_size = dev->page_size;
 	int offset = dma_addr & (page_size - 1);
 	int length = sizeof(struct nvme_id_ctrl);
+	dma_addr_t orig_dma_addr = dma_addr;
 	int ret;
 
 	memset(&c, 0, sizeof(c));
@@ -473,13 +474,13 @@ int nvme_identify(struct nvme_dev *dev, unsigned nsid,
 
 	c.identify.cns = cpu_to_le32(cns);
 
-	invalidate_dcache_range(dma_addr,
-				dma_addr + sizeof(struct nvme_id_ctrl));
+	invalidate_dcache_range(orig_dma_addr,
+				orig_dma_addr + sizeof(struct nvme_id_ctrl));
 
 	ret = nvme_submit_admin_cmd(dev, &c, NULL);
 	if (!ret)
-		invalidate_dcache_range(dma_addr,
-					dma_addr + sizeof(struct nvme_id_ctrl));
+		invalidate_dcache_range(orig_dma_addr,
+					orig_dma_addr + sizeof(struct nvme_id_ctrl));
 
 	return ret;
 }
Signature · 1 line
2.43.0
#1 lore raw
Quoted 37 lines · Prashant Kamble

When the identify buffer crosses a page boundary, PRP2 is used and dma_addr is advanced to the second page:

     dma_addr += (page_size - offset);

The subsequent invalidate_dcache_range() calls then use the modified dma_addr instead of the original buffer start address.

As a result, the beginning of the identify buffer is not invalidated and the invalidation range extends past the end of the buffer.

Fix this by preserving the original DMA buffer address for cache invalidation.

---
  drivers/nvme/nvme.c | 9 +++++----
  1 file changed, 5 insertions(+), 4 deletions(-)
drivers/nvme/nvme.c +1 -1
index 4f9473367d3..298010de536 100644
@@ -456,6 +456,7 @@ int nvme_identify(struct nvme_dev *dev, unsigned nsid,
  	u32 page_size = dev->page_size;
  	int offset = dma_addr & (page_size - 1);
  	int length = sizeof(struct nvme_id_ctrl);
+	dma_addr_t orig_dma_addr = dma_addr;
  	int ret;
 
  	memset(&c, 0, sizeof(c));
@@ -473,13 +474,13 @@ int nvme_identify(struct nvme_dev *dev, unsigned nsid,
 
  	c.identify.cns = cpu_to_le32(cns);
 
-	invalidate_dcache_range(dma_addr,

drivers/nvme/nvme.c int nvme_identify(struct nvme_dev *dev, unsigned nsid, · line 483

Quoted 10 lines
drivers/nvme/nvme.c +4 -3
-				dma_addr + sizeof(struct nvme_id_ctrl));
+	invalidate_dcache_range(orig_dma_addr,
+				orig_dma_addr + sizeof(struct nvme_id_ctrl));
 
  	ret = nvme_submit_admin_cmd(dev, &c, NULL);
  	if (!ret)
-		invalidate_dcache_range(dma_addr,
-					dma_addr + sizeof(struct nvme_id_ctrl));
+		invalidate_dcache_range(orig_dma_addr,
+					orig_dma_addr + sizeof(struct nvme_id_ctrl));

The fix is good, but I think this is not enough, the page_size used in this function is the NVMe page_size, not the host CPU cacheline size, so the dma_addr could not be aligned on a proper page boundary and the invalidate op could be a no-op.

The proper way to do that is like I added into dwc3:
static inline void dwc3_invalidate_cache(uintptr_t addr, int length)

{
	uintptr_t start_addr = (uintptr_t)addr & ~(CACHELINE_SIZE - 1);
	uintptr_t end_addr = ALIGN((uintptr_t)addr + length, CACHELINE_SIZE);

	invalidate_dcache_range((unsigned long)start_addr, (unsigned long)end_addr);
}

Neil

Quoted 3 lines
drivers/nvme/nvme.c +0 -0
  	return ret;
  }
#2 lore raw

PATCH v2 1/1 nvme: fix dcache invalidation range in identify command

In reply to Prashant Kamble

When the identify buffer crosses a page boundary, PRP2 is used and dma_addr is advanced to the second page:

    dma_addr += (page_size - offset);

The subsequent invalidate_dcache_range() calls then use the modified dma_addr instead of the original buffer start address.

As a result, the beginning of the identify buffer is not invalidated and the invalidation range extends past the end of the buffer.

Fix this by preserving the original DMA buffer address for cache invalidation.

---
v2:
- Align invalidate range to cacheline boundaries
- Add local helper function to calculate the invalidate range

---
 drivers/nvme/nvme.c | 26 ++++++++++++++++++++++----
 1 file changed, 22 insertions(+), 4 deletions(-)
drivers/nvme/nvme.c +22 -4
index 0631b190b97..da7dad9f52b 100644
@@ -27,6 +27,23 @@
 #define IO_TIMEOUT		30
 #define MAX_PRP_POOL		512
 
+/**
+ * nvme_invalidate_cache_aligned() - invalidate cache with proper alignment
+ *
+ * Aligns cache invalidation to cacheline boundaries to ensure correct
+ * behavior even when the DMA buffer is not aligned to page boundaries.
+ *
+ * @addr:	The start address of the buffer
+ * @length:	The length of the buffer in bytes
+ */
+static inline void nvme_invalidate_cache_aligned(uintptr_t addr, int length)
+{
+	uintptr_t start_addr = addr & ~(ARCH_DMA_MINALIGN - 1);
+	uintptr_t end_addr = ALIGN(addr + length, ARCH_DMA_MINALIGN);
+
+	invalidate_dcache_range(start_addr, end_addr);
+}
+
 static int nvme_wait_csts(struct nvme_dev *dev, u32 mask, u32 val)
 {
 	int timeout;
@@ -456,6 +473,7 @@ int nvme_identify(struct nvme_dev *dev, unsigned nsid,
 	u32 page_size = dev->page_size;
 	int offset = dma_addr & (page_size - 1);
 	int length = sizeof(struct nvme_id_ctrl);
+	dma_addr_t orig_dma_addr = dma_addr;
 	int ret;
 
 	memset(&c, 0, sizeof(c));
@@ -473,13 +491,13 @@ int nvme_identify(struct nvme_dev *dev, unsigned nsid,
 
 	c.identify.cns = cpu_to_le32(cns);
 
-	invalidate_dcache_range(dma_addr,
-				dma_addr + sizeof(struct nvme_id_ctrl));
+	nvme_invalidate_cache_aligned((uintptr_t)orig_dma_addr,
+				      sizeof(struct nvme_id_ctrl));
 
 	ret = nvme_submit_admin_cmd(dev, &c, NULL);
 	if (!ret)
-		invalidate_dcache_range(dma_addr,
-					dma_addr + sizeof(struct nvme_id_ctrl));
+		nvme_invalidate_cache_aligned((uintptr_t)orig_dma_addr,
+					      sizeof(struct nvme_id_ctrl));
 
 	return ret;
 }
Signature · 1 line
2.43.0
Reviewed-by on v2 1/1
#3 lore raw
Quoted 70 lines · Prashant Kamble

When the identify buffer crosses a page boundary, PRP2 is used and dma_addr is advanced to the second page:

     dma_addr += (page_size - offset);

The subsequent invalidate_dcache_range() calls then use the modified dma_addr instead of the original buffer start address.

As a result, the beginning of the identify buffer is not invalidated and the invalidation range extends past the end of the buffer.

Fix this by preserving the original DMA buffer address for cache invalidation.

---
v2:
- Align invalidate range to cacheline boundaries
- Add local helper function to calculate the invalidate range

---
  drivers/nvme/nvme.c | 26 ++++++++++++++++++++++----
  1 file changed, 22 insertions(+), 4 deletions(-)
drivers/nvme/nvme.c +20 -2
index 0631b190b97..da7dad9f52b 100644
@@ -27,6 +27,23 @@
  #define IO_TIMEOUT		30
  #define MAX_PRP_POOL		512
 
+/**
+ * nvme_invalidate_cache_aligned() - invalidate cache with proper alignment
+ *
+ * Aligns cache invalidation to cacheline boundaries to ensure correct
+ * behavior even when the DMA buffer is not aligned to page boundaries.
+ *
+ * @addr:	The start address of the buffer
+ * @length:	The length of the buffer in bytes
+ */
+static inline void nvme_invalidate_cache_aligned(uintptr_t addr, int length)
+{
+	uintptr_t start_addr = addr & ~(ARCH_DMA_MINALIGN - 1);
+	uintptr_t end_addr = ALIGN(addr + length, ARCH_DMA_MINALIGN);
+
+	invalidate_dcache_range(start_addr, end_addr);
+}
+
  static int nvme_wait_csts(struct nvme_dev *dev, u32 mask, u32 val)
  {
  	int timeout;
@@ -456,6 +473,7 @@ int nvme_identify(struct nvme_dev *dev, unsigned nsid,
  	u32 page_size = dev->page_size;
  	int offset = dma_addr & (page_size - 1);
  	int length = sizeof(struct nvme_id_ctrl);
+	dma_addr_t orig_dma_addr = dma_addr;
  	int ret;
 
  	memset(&c, 0, sizeof(c));
@@ -473,13 +491,13 @@ int nvme_identify(struct nvme_dev *dev, unsigned nsid,
 
  	c.identify.cns = cpu_to_le32(cns);
 
-	invalidate_dcache_range(dma_addr,
-				dma_addr + sizeof(struct nvme_id_ctrl));
+	nvme_invalidate_cache_aligned((uintptr_t)orig_dma_addr,
+				      sizeof(struct nvme_id_ctrl));

drivers/nvme/nvme.c int nvme_identify(struct nvme_dev *dev, unsigned nsid, · line 503

Quoted 10 lines
drivers/nvme/nvme.c +2 -2
 
  	ret = nvme_submit_admin_cmd(dev, &c, NULL);
  	if (!ret)
-		invalidate_dcache_range(dma_addr,
-					dma_addr + sizeof(struct nvme_id_ctrl));
+		nvme_invalidate_cache_aligned((uintptr_t)orig_dma_addr,
+					      sizeof(struct nvme_id_ctrl));
 
  	return ret;
  }

Thanks,
Neil

#4 lore raw
In reply to Prashant Kamble

Hi,

Quoted 9 lines · Prashant Kamble

When the identify buffer crosses a page boundary, PRP2 is used and dma_addr is advanced to the second page:

    dma_addr += (page_size - offset);

The subsequent invalidate_dcache_range() calls then use the modified dma_addr instead of the original buffer start address.

[...]

Thanks, Applied to https://source.denx.de/u-boot/custodians/u-boot-ufs (u-boot-nvme-fixes)

[1/1] nvme: fix dcache invalidation range in identify command

      https://source.denx.de/u-boot/custodians/u-boot-ufs/-/commit/29c40bb2a13ed1c294497044e2299cc0a511ce32
Signature · 1 line
Neil