nvme-pci: Fix page size checks
PATCH 1/3 nvme-pci: Fix page size checks
From: Keith Busch <[email protected]>
The size allocated out of the dma pool is at most NVME_CTRL_PAGE_SIZE, which may be smaller than the PAGE_SIZE.
--- drivers/nvme/host/pci.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)
drivers/nvme/host/pci.c +4 -4
@@ -36,7 +36,7 @@#define SQ_SIZE (q) ((q)->q_depth << (q)->sqes)#define CQ_SIZE (q) ((q)->q_depth * sizeof(struct nvme_completion)) -#define SGES_PER_PAGE (PAGE_SIZE / sizeof(struct nvme_sgl_desc)) +#define SGES_PER_PAGE (NVME_CTRL_PAGE_SIZE / sizeof(struct nvme_sgl_desc))/* * These can be higher, but we need to ensure that any command doesn't
@@ -382,7 +382,7 @@ static int nvme_pci_npages_prp(void) {unsigned max_bytes = (NVME_MAX_KB_SZ *1024 ) +NVME_CTRL_PAGE_SIZE ;unsigned nprps =DIV_ROUND_UP (max_bytes,NVME_CTRL_PAGE_SIZE ); -return DIV_ROUND_UP (8 * nprps,PAGE_SIZE -8 ); +return DIV_ROUND_UP (8 * nprps,NVME_CTRL_PAGE_SIZE -8 ); }/*
@@ -392,7 +392,7 @@ static int nvme_pci_npages_prp(void)static int nvme_pci_npages_sgl (void ) {return DIV_ROUND_UP (NVME_MAX_SEGS *sizeof (struct nvme_sgl_desc ), -PAGE_SIZE ); +NVME_CTRL_PAGE_SIZE ); } static int nvme_admin_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
@@ -708,7 +708,7 @@ static void nvme_pci_sgl_set_seg(struct nvme_sgl_desc *sge, sge->length =cpu_to_le32 (entries *sizeof (*sge)); sge->type =NVME_SGL_FMT_LAST_SEG_DESC <<4 ; }else { - sge->length =cpu_to_le32 (PAGE_SIZE ); + sge->length =cpu_to_le32 (NVME_CTRL_PAGE_SIZE ); sge->type =NVME_SGL_FMT_SEG_DESC <<4 ; } }
Signature · 1 line
2.30.2
PATCH 2/3 nvme-pci: Remove SGL chaining
From: Keith Busch <[email protected]>
The max segments this driver can see is 127, well below the 256 threshold needed to chain an nvme sgl segment. Remove all the useless checks and dead code.
--- drivers/nvme/host/pci.c | 47 +++++------------------------------------ 1 file changed, 5 insertions(+), 42 deletions(-)
drivers/nvme/host/pci.c +5 -42
@@ -547,22 +547,6 @@ static void nvme_free_prps(struct nvme_dev *dev, struct request *req) } } -static void nvme_free_sgls (struct nvme_dev *dev,struct request *req) -{ -const int last_sg =SGES_PER_PAGE -1 ; -struct nvme_iod *iod =blk_mq_rq_to_pdu (req); -dma_addr_t dma_addr = iod->first_dma; -int i; - -for (i =0 ; i < iod->nr_allocations; i++) { -struct nvme_sgl_desc *sg_list =nvme_pci_iod_list (req)[i]; -dma_addr_t next_dma_addr =le64_to_cpu ((sg_list[last_sg]).addr); - -dma_pool_free (dev->prp_page_pool, sg_list, dma_addr); - dma_addr = next_dma_addr; - } -} -static void nvme_unmap_data (struct nvme_dev *dev,struct request *req) {struct nvme_iod *iod =blk_mq_rq_to_pdu (req);
@@ -581,7 +565,8 @@ static void nvme_unmap_data(struct nvme_dev *dev, struct request *req)dma_pool_free (dev->prp_small_pool,nvme_pci_iod_list (req)[0 ], iod->first_dma); else if (iod->use_sgl) - nvme_free_sgls(dev, req); + dma_pool_free(dev->prp_page_pool, nvme_pci_iod_list(req)[0], + iod->first_dma); elsenvme_free_prps (dev ,req );mempool_free (iod->sgt.sgl, dev->iod_mempool);
@@ -704,13 +689,8 @@ static void nvme_pci_sgl_set_seg(struct nvme_sgl_desc *sge, dma_addr_t dma_addr, int entries) { sge->addr =cpu_to_le64 (dma_addr); -if (entries <SGES_PER_PAGE ) { - sge->length =cpu_to_le32 (entries *sizeof (*sge)); - sge->type =NVME_SGL_FMT_LAST_SEG_DESC <<4 ; - }else { - sge->length =cpu_to_le32 (NVME_CTRL_PAGE_SIZE ); - sge->type =NVME_SGL_FMT_SEG_DESC <<4 ; - } + sge->length =cpu_to_le32 (entries *sizeof (*sge)); + sge->type =NVME_SGL_FMT_LAST_SEG_DESC <<4 ; } static blk_status_t nvme_pci_setup_sgls(struct nvme_dev *dev,
@@ -750,30 +730,12 @@ static blk_status_t nvme_pci_setup_sgls(struct nvme_dev *dev, iod->first_dma = sgl_dma;nvme_pci_sgl_set_seg (&cmd->dptr.sgl, sgl_dma, entries); -do { -if (i ==SGES_PER_PAGE ) { -struct nvme_sgl_desc *old_sg_desc = sg_list; -struct nvme_sgl_desc *link = &old_sg_desc[i -1 ]; - - sg_list =dma_pool_alloc (pool,GFP_ATOMIC , &sgl_dma); -if (!sg_list) - goto free_sgls; - - i =0 ; -nvme_pci_iod_list (req)[iod->nr_allocations++] = sg_list; - sg_list[i++] = *link; -nvme_pci_sgl_set_seg (link, sgl_dma, entries); - } -nvme_pci_sgl_set_data (&sg_list[i++], sg); sg =sg_next (sg); }while (--entries >0 );return BLK_STS_OK ; -free_sgls: -nvme_free_sgls (dev, req); -return BLK_STS_RESOURCE ; } static blk_status_t nvme_setup_prp_simple(struct nvme_dev *dev,
@@ -3523,6 +3485,7 @@ static int __init nvme_init(void)BUILD_BUG_ON (IRQ_AFFINITY_MAX_SETS <2 );BUILD_BUG_ON (DIV_ROUND_UP (nvme_pci_npages_prp (),NVME_CTRL_PAGE_SIZE ) >S8_MAX ); +BUILD_BUG_ON (NVME_MAX_SEGS >SGES_PER_PAGE );return pci_register_driver (&nvme_driver); }
Signature · 1 line
2.30.2
Sagi Grimberg
maintainerRe: [PATCH 2/3] nvme-pci: Remove SGL chaining
Quoted 5 lines · Keith Busch
From: Keith Busch <[email protected]>
The max segments this driver can see is 127, well below the 256 threshold needed to chain an nvme sgl segment. Remove all the useless checks and dead code.
What about nvme_pci_npages_sgl() ? is it still necessary?
PATCH 3/3 nvme-pci: use mapped entries for sgl decision
In reply to Keith BuschFrom: Keith Busch <[email protected]>
The driver uses the dma entries for setting up its command's SGL/PRP lists. The dma mapping might have fewer entries than the virtual segments, so check the dma mapped count to determine which nvme data layout method is more optimal.
--- drivers/nvme/host/pci.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
drivers/nvme/host/pci.c +3 -3
@@ -514,10 +514,10 @@ static void **nvme_pci_iod_list(struct request *req)return (void **)(iod->sgt.sgl +blk_rq_nr_phys_segments (req)); } -static inline bool nvme_pci_use_sgls (struct nvme_dev *dev,struct request *req) +static inline bool nvme_pci_use_sgls (struct nvme_dev *dev,struct request *req, +int nseg) {struct nvme_queue *nvmeq = req->mq_hctx->driver_data; -int nseg =blk_rq_nr_phys_segments (req);unsigned int avg_seg_size; avg_seg_size =DIV_ROUND_UP (blk_rq_payload_bytes (req), nseg);
@@ -817,7 +817,7 @@ static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req, goto out_free_sg; } - iod->use_sgl =nvme_pci_use_sgls (dev, req); + iod->use_sgl =nvme_pci_use_sgls (dev, req, iod->sgt.nents);if (iod->use_sgl) ret =nvme_pci_setup_sgls (dev, req, &cmnd->rw);else
Signature · 1 line
2.30.2
Re: [PATCH 3/3] nvme-pci: use mapped entries for sgl decision
Christoph Hellwig
maintainerI've applied just this one to nvme-6.2. The others looks sensible, but not critical enough for 6.2.