[PATCHv3 0/4]
v3 cover [PATCHv3 0/4]
From: Keith Busch <[email protected]>
The series adjusts the struct nvme_iod fields for optimal sizing.
Changes from v3: rebased to upstream nvme driver with sgtable changes; this makes the previous series' patch 4/5 unnecessary, so it was dropped.
Keith Busch (4):
nvme-pci: remove nvme_queue from nvme_iod
nvme-pci: iod's 'aborted' is a bool
nvme-pci: iod npages fits in s8
nvme-pci: move iod dma_len fill gaps
drivers/nvme/host/pci.c | 64 ++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 32 deletions(-)
Signature · 1 line
2.30.2
PATCH v3 1/4 nvme-pci: remove nvme_queue from nvme_iod
From: Keith Busch <[email protected]>
We can get the nvme_queue from the req just as easily, so remove the duplicate path to the same structure to save some space.
--- drivers/nvme/host/pci.c | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-)
drivers/nvme/host/pci.c +13 -15
@@ -226,7 +226,6 @@ struct nvme_queue {struct nvme_iod {struct nvme_request req;struct nvme_command cmd; -struct nvme_queue *nvmeq;bool use_sgl;int aborted;int npages;/* In the PRP list. 0 means small pool in use */
@@ -430,11 +429,6 @@ static int nvme_pci_init_request(struct blk_mq_tag_set *set, {struct nvme_dev *dev = set->driver_data;struct nvme_iod *iod =blk_mq_rq_to_pdu (req); -int queue_idx = (set == &dev->tagset) ? hctx_idx +1 :0 ; -struct nvme_queue *nvmeq = &dev->queues[queue_idx]; - -BUG_ON (!nvmeq); - iod->nvmeq = nvmeq;nvme_req (req)->ctrl = &dev->ctrl;nvme_req (req)->cmd = &iod->cmd;
@@ -526,7 +520,7 @@ static void **nvme_pci_iod_list(struct request *req)static inline bool nvme_pci_use_sgls (struct nvme_dev *dev,struct request *req) { -struct nvme_iod *iod =blk_mq_rq_to_pdu (req); +struct nvme_queue *nvmeq = req->mq_hctx->driver_data;int nseg =blk_rq_nr_phys_segments (req);unsigned int avg_seg_size;
@@ -534,7 +528,7 @@ static inline bool nvme_pci_use_sgls(struct nvme_dev *dev, struct request *req)if (!nvme_ctrl_sgl_supported (&dev->ctrl))return false; -if (!iod->nvmeq->qid) +if (!nvmeq->qid)return false;if (!sgl_threshold || avg_seg_size < sgl_threshold)return false;
@@ -831,6 +825,7 @@ static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req,int rc;if (blk_rq_nr_phys_segments (req) ==1 ) { +struct nvme_queue *nvmeq = req->mq_hctx->driver_data;struct bio_vec bv =req_bvec (req);if (!is_pci_p2pdma_page (bv.bv_page)) {
@@ -838,7 +833,7 @@ static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req,return nvme_setup_prp_simple (dev, req, &cmnd->rw, &bv); -if (iod->nvmeq->qid && sgl_threshold && +if (nvmeq->qid && sgl_threshold &&nvme_ctrl_sgl_supported (&dev->ctrl))return nvme_setup_sgl_simple (dev, req, &cmnd->rw, &bv);
@@ -1017,12 +1012,16 @@ static void nvme_queue_rqs(struct request **rqlist) static __always_inline void nvme_pci_unmap_rq(struct request *req) { -struct nvme_iod *iod =blk_mq_rq_to_pdu (req); -struct nvme_dev *dev = iod->nvmeq->dev; +struct nvme_queue *nvmeq = req->mq_hctx->driver_data; +struct nvme_dev *dev = nvmeq->dev; + +if (blk_integrity_rq (req)) { +struct nvme_iod *iod =blk_mq_rq_to_pdu (req); -if (blk_integrity_rq (req))dma_unmap_page (dev->dev, iod->meta_dma,rq_integrity_vec (req)->bv_len,rq_data_dir (req)); + } +if (blk_rq_nr_phys_segments (req))nvme_unmap_data (dev, req); }
@@ -1270,8 +1269,7 @@ static int adapter_delete_sq(struct nvme_dev *dev, u16 sqid)static void abort_endio (struct request *req,blk_status_t error) { -struct nvme_iod *iod =blk_mq_rq_to_pdu (req); -struct nvme_queue *nvmeq = iod->nvmeq; +struct nvme_queue *nvmeq = req->mq_hctx->driver_data;dev_warn (nvmeq->dev->ctrl.device,"Abort status: 0x%x" ,nvme_req (req)->status);
@@ -1333,7 +1331,7 @@ static void nvme_warn_reset(struct nvme_dev *dev, u32 csts)static enum blk_eh_timer_return nvme_timeout (struct request *req) {struct nvme_iod *iod =blk_mq_rq_to_pdu (req); -struct nvme_queue *nvmeq = iod->nvmeq; +struct nvme_queue *nvmeq = req->mq_hctx->driver_data;struct nvme_dev *dev = nvmeq->dev;struct request *abort_req;struct nvme_command cmd = { };
Signature · 1 line
2.30.2
PATCH v3 2/4 nvme-pci: iod's 'aborted' is a bool
In reply to Keith BuschFrom: Keith Busch <[email protected]>
It's only true or false, so make this a bool to reflect that and save some space in nvme_iod.
--- drivers/nvme/host/pci.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
drivers/nvme/host/pci.c +3 -3
@@ -227,7 +227,7 @@ struct nvme_iod {struct nvme_request req;struct nvme_command cmd;bool use_sgl; -int aborted; +bool aborted;int npages;/* In the PRP list. 0 means small pool in use */ dma_addr_t first_dma;unsigned int dma_len;/* length of single DMA segment mapping */
@@ -891,7 +891,7 @@ static blk_status_t nvme_prep_rq(struct nvme_dev *dev, struct request *req)struct nvme_iod *iod =blk_mq_rq_to_pdu (req);blk_status_t ret; - iod->aborted =0 ; + iod->aborted = false; iod->npages =-1 ; iod->sgt.nents =0 ;
@@ -1412,7 +1412,7 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req)atomic_inc (&dev->ctrl.abort_limit);return BLK_EH_RESET_TIMER ; } - iod->aborted =1 ; + iod->aborted = true; cmd.abort.opcode = nvme_admin_abort_cmd; cmd.abort.cid =nvme_cid (req);
Signature · 1 line
2.30.2
PATCH v3 3/4 nvme-pci: iod npages fits in s8
In reply to Keith BuschFrom: Keith Busch <[email protected]>
The largest allowed transfer is 4MB, which can use at most 1025 PRPs. Each PRP is 8 bytes, so the maximum number of 4k nvme pages needed for the iod_list is 3, which fits in an 's8' type.
While modifying this field, change the name to "nr_allocations" to better represent that this is referring to the number of units allocated from a dma_pool.
Also introduce a BUILD_BUG_ON to ensure we never accidently increase the largest transfer limit beyond 127 chained prp lists.
--- drivers/nvme/host/pci.c | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-)
drivers/nvme/host/pci.c +15 -13
@@ -228,7 +228,7 @@ struct nvme_iod {struct nvme_command cmd;bool use_sgl;bool aborted; -int npages;/* In the PRP list. 0 means small pool in use */ +s8 nr_allocations;/* PRP list pool allocations. 0 means small pool in use */ dma_addr_t first_dma;unsigned int dma_len;/* length of single DMA segment mapping */ dma_addr_t meta_dma;
@@ -542,7 +542,7 @@ static void nvme_free_prps(struct nvme_dev *dev, struct request *req)dma_addr_t dma_addr = iod->first_dma;int i; -for (i =0 ; i < iod->npages; i++) { +for (i =0 ; i < iod->nr_allocations; i++) {__le64 *prp_list =nvme_pci_iod_list (req)[i];dma_addr_t next_dma_addr =le64_to_cpu (prp_list[last_prp]);
@@ -558,7 +558,7 @@ static void nvme_free_sgls(struct nvme_dev *dev, struct request *req)dma_addr_t dma_addr = iod->first_dma;int i; -for (i =0 ; i < iod->npages; i++) { +for (i =0 ; i < iod->nr_allocations; i++) {struct nvme_sgl_desc *sg_list =nvme_pci_iod_list (req)[i];dma_addr_t next_dma_addr =le64_to_cpu ((sg_list[last_sg]).addr);
@@ -581,7 +581,7 @@ static void nvme_unmap_data(struct nvme_dev *dev, struct request *req)dma_unmap_sgtable (dev->dev, &iod->sgt,rq_dma_dir (req),0 ); -if (iod->npages ==0 ) +if (iod->nr_allocations ==0 )dma_pool_free (dev->prp_small_pool,nvme_pci_iod_list (req)[0 ], iod->first_dma);else if (iod->use_sgl)
@@ -643,15 +643,15 @@ static blk_status_t nvme_pci_setup_prps(struct nvme_dev *dev, nprps =DIV_ROUND_UP (length,NVME_CTRL_PAGE_SIZE );if (nprps <= (256 /8 )) { pool = dev->prp_small_pool; - iod->npages =0 ; + iod->nr_allocations =0 ; }else { pool = dev->prp_page_pool; - iod->npages =1 ; + iod->nr_allocations =1 ; } prp_list =dma_pool_alloc (pool,GFP_ATOMIC , &prp_dma);if (!prp_list) { - iod->npages =-1 ; + iod->nr_allocations =-1 ;return BLK_STS_RESOURCE ; } list[0 ] = prp_list;
@@ -663,7 +663,7 @@ static blk_status_t nvme_pci_setup_prps(struct nvme_dev *dev, prp_list =dma_pool_alloc (pool,GFP_ATOMIC , &prp_dma);if (!prp_list) goto free_prps; - list[iod->npages++] = prp_list; + list[iod->nr_allocations++] = prp_list; prp_list[0 ] = old_prp_list[i -1 ]; old_prp_list[i -1 ] =cpu_to_le64 (prp_dma); i =1 ;
@@ -738,15 +738,15 @@ static blk_status_t nvme_pci_setup_sgls(struct nvme_dev *dev,if (entries <= (256 /sizeof (struct nvme_sgl_desc ))) { pool = dev->prp_small_pool; - iod->npages =0 ; + iod->nr_allocations =0 ; }else { pool = dev->prp_page_pool; - iod->npages =1 ; + iod->nr_allocations =1 ; } sg_list =dma_pool_alloc (pool,GFP_ATOMIC , &sgl_dma);if (!sg_list) { - iod->npages =-1 ; + iod->nr_allocations =-1 ;return BLK_STS_RESOURCE ; }
@@ -765,7 +765,7 @@ static blk_status_t nvme_pci_setup_sgls(struct nvme_dev *dev, goto free_sgls; i =0 ; -nvme_pci_iod_list (req)[iod->npages++] = sg_list; +nvme_pci_iod_list (req)[iod->nr_allocations++] = sg_list; sg_list[i++] = *link;nvme_pci_sgl_set_seg (link, sgl_dma, entries); }
@@ -892,7 +892,7 @@ static blk_status_t nvme_prep_rq(struct nvme_dev *dev, struct request *req)blk_status_t ret; iod->aborted = false; - iod->npages =-1 ; + iod->nr_allocations =-1 ; iod->sgt.nents =0 ; ret =nvme_setup_cmd (req->q->queuedata, req);
@@ -3559,6 +3559,8 @@ static int __init nvme_init(void)BUILD_BUG_ON (sizeof (struct nvme_create_sq ) !=64 );BUILD_BUG_ON (sizeof (struct nvme_delete_queue ) !=64 );BUILD_BUG_ON (IRQ_AFFINITY_MAX_SETS <2 ); +BUILD_BUG_ON (DIV_ROUND_UP (nvme_pci_npages_prp (),NVME_CTRL_PAGE_SIZE ) > +S8_MAX );return pci_register_driver (&nvme_driver); }
Signature · 1 line
2.30.2
PATCH v3 4/4 nvme-pci: move iod dma_len fill gaps
In reply to Keith BuschFrom: Keith Busch <[email protected]>
The 32-bit field, dma_len, packs better in the iod struct above the dma_addr_t on 64-bit systems.
--- drivers/nvme/host/pci.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/nvme/host/pci.c +1 -1
@@ -229,8 +229,8 @@ struct nvme_iod {bool use_sgl;bool aborted;s8 nr_allocations;/* PRP list pool allocations. 0 means small pool in use */ -dma_addr_t first_dma;unsigned int dma_len;/* length of single DMA segment mapping */ +dma_addr_t first_dma;dma_addr_t meta_dma;struct sg_table sgt; };
Signature · 1 line
2.30.2
Thanks,
applied to nvme-6.1.