nvme: fix dcache invalidation range in identify command
PATCH 1/1 nvme: fix dcache invalidation range in identify command
When the identify buffer crosses a page boundary, PRP2 is used and dma_addr is advanced to the second page:
dma_addr += (page_size - offset);
The subsequent invalidate_dcache_range() calls then use the modified dma_addr instead of the original buffer start address.
As a result, the beginning of the identify buffer is not invalidated and the invalidation range extends past the end of the buffer.
Fix this by preserving the original DMA buffer address for cache invalidation.
--- drivers/nvme/nvme.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-)
drivers/nvme/nvme.c +5 -4
@@ -456,6 +456,7 @@ int nvme_identify(struct nvme_dev *dev, unsigned nsid,u32 page_size = dev->page_size;int offset = dma_addr & (page_size -1 );int length =sizeof (struct nvme_id_ctrl ); +dma_addr_t orig_dma_addr = dma_addr;int ret;memset (&c,0 ,sizeof (c));
@@ -473,13 +474,13 @@ int nvme_identify(struct nvme_dev *dev, unsigned nsid, c.identify.cns =cpu_to_le32 (cns); -invalidate_dcache_range (dma_addr, - dma_addr +sizeof (struct nvme_id_ctrl )); +invalidate_dcache_range (orig_dma_addr, + orig_dma_addr +sizeof (struct nvme_id_ctrl )); ret =nvme_submit_admin_cmd (dev, &c,NULL );if (!ret) -invalidate_dcache_range (dma_addr, - dma_addr +sizeof (struct nvme_id_ctrl )); +invalidate_dcache_range (orig_dma_addr, + orig_dma_addr +sizeof (struct nvme_id_ctrl ));return ret; }
Signature · 1 line
2.43.0
Quoted 37 lines · Prashant Kamble
When the identify buffer crosses a page boundary, PRP2 is used and
dma_addris advanced to the second page:dma_addr += (page_size - offset);The subsequent
invalidate_dcache_range()calls then use the modifieddma_addrinstead of the original buffer start address.As a result, the beginning of the identify buffer is not invalidated and the invalidation range extends past the end of the buffer.
Fix this by preserving the original DMA buffer address for cache invalidation.
Signed-off-by Prashant Kamble <[email protected]>--- drivers/nvme/nvme.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-)drivers/nvme/nvme.c +1
-1@@ -456,6 +456,7 @@ int nvme_identify(struct nvme_dev *dev, unsigned nsid,u32 page_size = dev->page_size;int offset = dma_addr & (page_size -1 );int length =sizeof (struct nvme_id_ctrl ); +dma_addr_t orig_dma_addr = dma_addr;int ret;memset (&c,0 ,sizeof (c));@@ -473,13 +474,13 @@ int nvme_identify(struct nvme_dev *dev, unsigned nsid, c.identify.cns =cpu_to_le32 (cns); - invalidate_dcache_range(dma_addr,
drivers/nvme/nvme.c int nvme_identify(struct nvme_dev *dev, unsigned nsid, · line 483
Quoted 10 lines
drivers/nvme/nvme.c +4
-3- dma_addr + sizeof(struct nvme_id_ctrl)); +invalidate_dcache_range (orig_dma_addr, + orig_dma_addr +sizeof (struct nvme_id_ctrl )); ret =nvme_submit_admin_cmd (dev, &c,NULL );if (!ret) -invalidate_dcache_range (dma_addr, - dma_addr +sizeof (struct nvme_id_ctrl )); +invalidate_dcache_range (orig_dma_addr, + orig_dma_addr +sizeof (struct nvme_id_ctrl ));
The fix is good, but I think this is not enough, the page_size used in this function is the NVMe page_size, not the host CPU cacheline size, so the dma_addr could not be aligned on a proper page boundary and the invalidate op could be a no-op.
The proper way to do that is like I added into dwc3:
static inline void dwc3_invalidate_cache(uintptr_t addr, int length)
{
uintptr_t start_addr = (uintptr_t)addr & ~(CACHELINE_SIZE - 1);
uintptr_t end_addr = ALIGN((uintptr_t)addr + length, CACHELINE_SIZE);
invalidate_dcache_range((unsigned long)start_addr, (unsigned long)end_addr);
}Neil
Quoted 3 lines
drivers/nvme/nvme.c +0
-0return ret; }
PATCH v2 1/1 nvme: fix dcache invalidation range in identify command
In reply to Prashant KambleWhen the identify buffer crosses a page boundary, PRP2 is used and dma_addr is advanced to the second page:
dma_addr += (page_size - offset);
The subsequent invalidate_dcache_range() calls then use the modified dma_addr instead of the original buffer start address.
As a result, the beginning of the identify buffer is not invalidated and the invalidation range extends past the end of the buffer.
Fix this by preserving the original DMA buffer address for cache invalidation.
---
v2:
- Align invalidate range to cacheline boundaries
- Add local helper function to calculate the invalidate range
--- drivers/nvme/nvme.c | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-)
drivers/nvme/nvme.c +22 -4
@@ -27,6 +27,23 @@#define IO_TIMEOUT 30#define MAX_PRP_POOL 512 +/** +* nvme_invalidate_cache_aligned() - invalidate cache with proper alignment +* +* Aligns cache invalidation to cacheline boundaries to ensure correct +* behavior even when the DMA buffer is not aligned to page boundaries. +* +* @addr: The start address of the buffer +* @length: The length of the buffer in bytes +*/ +static inline void nvme_invalidate_cache_aligned (uintptr_t addr,int length) +{ +uintptr_t start_addr = addr & ~(ARCH_DMA_MINALIGN -1 ); +uintptr_t end_addr =ALIGN (addr + length,ARCH_DMA_MINALIGN ); + +invalidate_dcache_range (start_addr, end_addr); +} +static int nvme_wait_csts (struct nvme_dev *dev,u32 mask,u32 val) {int timeout;
@@ -456,6 +473,7 @@ int nvme_identify(struct nvme_dev *dev, unsigned nsid,u32 page_size = dev->page_size;int offset = dma_addr & (page_size -1 );int length =sizeof (struct nvme_id_ctrl ); +dma_addr_t orig_dma_addr = dma_addr;int ret;memset (&c,0 ,sizeof (c));
@@ -473,13 +491,13 @@ int nvme_identify(struct nvme_dev *dev, unsigned nsid, c.identify.cns =cpu_to_le32 (cns); -invalidate_dcache_range (dma_addr, - dma_addr +sizeof (struct nvme_id_ctrl )); +nvme_invalidate_cache_aligned ((uintptr_t )orig_dma_addr, +sizeof (struct nvme_id_ctrl )); ret =nvme_submit_admin_cmd (dev, &c,NULL );if (!ret) -invalidate_dcache_range (dma_addr, - dma_addr +sizeof (struct nvme_id_ctrl )); +nvme_invalidate_cache_aligned ((uintptr_t )orig_dma_addr, +sizeof (struct nvme_id_ctrl ));return ret; }
Signature · 1 line
2.43.0
Quoted 70 lines · Prashant Kamble
When the identify buffer crosses a page boundary, PRP2 is used and
dma_addris advanced to the second page:dma_addr += (page_size - offset);The subsequent
invalidate_dcache_range()calls then use the modifieddma_addrinstead of the original buffer start address.As a result, the beginning of the identify buffer is not invalidated and the invalidation range extends past the end of the buffer.
Fix this by preserving the original DMA buffer address for cache invalidation.
Suggested-by Neil Armstrong <[email protected]>Signed-off-by Prashant Kamble <[email protected]>---
v2:
- Align invalidate range to cacheline boundaries
- Add local helper function to calculate the invalidate range--- drivers/nvme/nvme.c | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-)drivers/nvme/nvme.c +20
-2@@ -27,6 +27,23 @@#define IO_TIMEOUT 30#define MAX_PRP_POOL 512 +/** +* nvme_invalidate_cache_aligned() - invalidate cache with proper alignment +* +* Aligns cache invalidation to cacheline boundaries to ensure correct +* behavior even when the DMA buffer is not aligned to page boundaries. +* +* @addr: The start address of the buffer +* @length: The length of the buffer in bytes +*/ +static inline void nvme_invalidate_cache_aligned (uintptr_t addr,int length) +{ +uintptr_t start_addr = addr & ~(ARCH_DMA_MINALIGN -1 ); +uintptr_t end_addr =ALIGN (addr + length,ARCH_DMA_MINALIGN ); + +invalidate_dcache_range (start_addr, end_addr); +} +static int nvme_wait_csts (struct nvme_dev *dev,u32 mask,u32 val) {int timeout;@@ -456,6 +473,7 @@ int nvme_identify(struct nvme_dev *dev, unsigned nsid,u32 page_size = dev->page_size;int offset = dma_addr & (page_size -1 );int length =sizeof (struct nvme_id_ctrl ); +dma_addr_t orig_dma_addr = dma_addr;int ret;memset (&c,0 ,sizeof (c));@@ -473,13 +491,13 @@ int nvme_identify(struct nvme_dev *dev, unsigned nsid, c.identify.cns =cpu_to_le32 (cns); -invalidate_dcache_range (dma_addr, - dma_addr +sizeof (struct nvme_id_ctrl )); +nvme_invalidate_cache_aligned ((uintptr_t )orig_dma_addr, +sizeof (struct nvme_id_ctrl ));
drivers/nvme/nvme.c int nvme_identify(struct nvme_dev *dev, unsigned nsid, · line 503
Quoted 10 lines
drivers/nvme/nvme.c +2
-2ret =nvme_submit_admin_cmd (dev, &c,NULL );if (!ret) -invalidate_dcache_range (dma_addr, - dma_addr +sizeof (struct nvme_id_ctrl )); +nvme_invalidate_cache_aligned ((uintptr_t )orig_dma_addr, +sizeof (struct nvme_id_ctrl ));return ret; }
Thanks,
Neil
Hi,
Quoted 9 lines · Prashant Kamble
When the identify buffer crosses a page boundary, PRP2 is used and
dma_addris advanced to the second page:dma_addr += (page_size - offset);The subsequent
invalidate_dcache_range()calls then use the modifieddma_addrinstead of the original buffer start address.[...]
Thanks, Applied to https://source.denx.de/u-boot/custodians/u-boot-ufs (u-boot-nvme-fixes)
[1/1] nvme: fix dcache invalidation range in identify command
https://source.denx.de/u-boot/custodians/u-boot-ufs/-/commit/29c40bb2a13ed1c294497044e2299cc0a511ce32
Signature · 1 line
Neil