nvme-pci: iod optimisations
v2 cover nvme-pci: iod optimisations
From: Keith Busch <[email protected]>
The series adjusts the struct nvme_iod fields for optimal sizing. The end result on 64-bit arch removes 24 bytes, from 152 down to 128. We typically allocate many thousands of these, and the cache aligned size is a nice bonus.
Keith Busch (5):
nvme-pci: remove nvme_queue from nvme_iod
nvme-pci: iod's 'aborted' is a bool
nvme-pci: iod npages fits in s8
nvme-pci: iod nents fits in u8
nvme-pci: move iod dma_len fill gaps
drivers/nvme/host/pci.c | 67 +++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 33 deletions(-)
Signature · 1 line
2.30.2
PATCH v2 1/5 nvme-pci: remove nvme_queue from nvme_iod
From: Keith Busch <[email protected]>
We can get the nvme_queue from the req just as easily, so remove the duplicate path to the same structure to save some space.
--- drivers/nvme/host/pci.c | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-)
drivers/nvme/host/pci.c +13 -15
@@ -226,7 +226,6 @@ struct nvme_queue {struct nvme_iod {struct nvme_request req;struct nvme_command cmd; -struct nvme_queue *nvmeq;bool use_sgl;int aborted;int npages;/* In the PRP list. 0 means small pool in use */
@@ -431,11 +430,6 @@ static int nvme_pci_init_request(struct blk_mq_tag_set *set, {struct nvme_dev *dev = set->driver_data;struct nvme_iod *iod =blk_mq_rq_to_pdu (req); -int queue_idx = (set == &dev->tagset) ? hctx_idx +1 :0 ; -struct nvme_queue *nvmeq = &dev->queues[queue_idx]; - -BUG_ON (!nvmeq); - iod->nvmeq = nvmeq;nvme_req (req)->ctrl = &dev->ctrl;nvme_req (req)->cmd = &iod->cmd;
@@ -529,7 +523,7 @@ static void **nvme_pci_iod_list(struct request *req)static inline bool nvme_pci_use_sgls (struct nvme_dev *dev,struct request *req) { -struct nvme_iod *iod =blk_mq_rq_to_pdu (req); +struct nvme_queue *nvmeq = req->mq_hctx->driver_data;int nseg =blk_rq_nr_phys_segments (req);unsigned int avg_seg_size;
@@ -537,7 +531,7 @@ static inline bool nvme_pci_use_sgls(struct nvme_dev *dev, struct request *req)if (!nvme_ctrl_sgl_supported (&dev->ctrl))return false; -if (!iod->nvmeq->qid) +if (!nvmeq->qid)return false;if (!sgl_threshold || avg_seg_size < sgl_threshold)return false;
@@ -843,6 +837,7 @@ static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req,int nr_mapped;if (blk_rq_nr_phys_segments (req) ==1 ) { +struct nvme_queue *nvmeq = req->mq_hctx->driver_data;struct bio_vec bv =req_bvec (req);if (!is_pci_p2pdma_page (bv.bv_page)) {
@@ -850,7 +845,7 @@ static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req,return nvme_setup_prp_simple (dev, req, &cmnd->rw, &bv); -if (iod->nvmeq->qid && sgl_threshold && +if (nvmeq->qid && sgl_threshold &&nvme_ctrl_sgl_supported (&dev->ctrl))return nvme_setup_sgl_simple (dev, req, &cmnd->rw, &bv);
@@ -1030,12 +1025,16 @@ static void nvme_queue_rqs(struct request **rqlist) static __always_inline void nvme_pci_unmap_rq(struct request *req) { -struct nvme_iod *iod =blk_mq_rq_to_pdu (req); -struct nvme_dev *dev = iod->nvmeq->dev; +struct nvme_queue *nvmeq = req->mq_hctx->driver_data; +struct nvme_dev *dev = nvmeq->dev; + +if (blk_integrity_rq (req)) { +struct nvme_iod *iod =blk_mq_rq_to_pdu (req); -if (blk_integrity_rq (req))dma_unmap_page (dev->dev, iod->meta_dma,rq_integrity_vec (req)->bv_len,rq_data_dir (req)); + } +if (blk_rq_nr_phys_segments (req))nvme_unmap_data (dev, req); }
@@ -1283,8 +1282,7 @@ static int adapter_delete_sq(struct nvme_dev *dev, u16 sqid)static void abort_endio (struct request *req,blk_status_t error) { -struct nvme_iod *iod =blk_mq_rq_to_pdu (req); -struct nvme_queue *nvmeq = iod->nvmeq; +struct nvme_queue *nvmeq = req->mq_hctx->driver_data;dev_warn (nvmeq->dev->ctrl.device,"Abort status: 0x%x" ,nvme_req (req)->status);
@@ -1346,7 +1344,7 @@ static void nvme_warn_reset(struct nvme_dev *dev, u32 csts)static enum blk_eh_timer_return nvme_timeout (struct request *req) {struct nvme_iod *iod =blk_mq_rq_to_pdu (req); -struct nvme_queue *nvmeq = iod->nvmeq; +struct nvme_queue *nvmeq = req->mq_hctx->driver_data;struct nvme_dev *dev = nvmeq->dev;struct request *abort_req;struct nvme_command cmd = { };
Signature · 1 line
2.30.2
PATCH v2 2/5 nvme-pci: iod's 'aborted' is a bool
In reply to Keith BuschFrom: Keith Busch <[email protected]>
It's only true or false, so make this a bool to reflect that and save some space in nvme_iod.
--- drivers/nvme/host/pci.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
drivers/nvme/host/pci.c +3 -3
@@ -227,7 +227,7 @@ struct nvme_iod {struct nvme_request req;struct nvme_command cmd;bool use_sgl; -int aborted; +bool aborted;int npages;/* In the PRP list. 0 means small pool in use */ int nents;/* Used in scatterlist */ dma_addr_t first_dma;
@@ -904,7 +904,7 @@ static blk_status_t nvme_prep_rq(struct nvme_dev *dev, struct request *req)struct nvme_iod *iod =blk_mq_rq_to_pdu (req);blk_status_t ret; - iod->aborted =0 ; + iod->aborted = false; iod->npages =-1 ; iod->nents =0 ;
@@ -1425,7 +1425,7 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req)atomic_inc (&dev->ctrl.abort_limit);return BLK_EH_RESET_TIMER ; } - iod->aborted =1 ; + iod->aborted = true; cmd.abort.opcode = nvme_admin_abort_cmd; cmd.abort.cid =nvme_cid (req);
Signature · 1 line
2.30.2
PATCH v2 3/5 nvme-pci: iod npages fits in s8
In reply to Keith BuschFrom: Keith Busch <[email protected]>
The largest allowed transfer is 4MB, which can use at most 1025 PRPs. Each PRP is 8 bytes, so the maximum number of 4k nvme pages needed for the iod_list is 3, which fits in an 's8' type.
While modifying this field, change the name to "nr_allocations" to better represent that this is referring to the number of units allocated from a dma_pool.
Also introduce a BUILD_BUG_ON to ensure we never accidently increase the largest transfer limit beyond 127 chained prp lists.
---v1->v2:
Renamed field from npages to nr_allocations (Christoph)
Added BUILD_BUG_ON
drivers/nvme/host/pci.c | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-)
drivers/nvme/host/pci.c +15 -13
@@ -228,7 +228,7 @@ struct nvme_iod {struct nvme_command cmd;bool use_sgl;bool aborted; -int npages;/* In the PRP list. 0 means small pool in use */ +s8 nr_allocations;/* PRP list pool allocations. 0 means small pool in use */ int nents;/* Used in scatterlist */ dma_addr_t first_dma;unsigned int dma_len;/* length of single DMA segment mapping */
@@ -545,7 +545,7 @@ static void nvme_free_prps(struct nvme_dev *dev, struct request *req)dma_addr_t dma_addr = iod->first_dma;int i; -for (i =0 ; i < iod->npages; i++) { +for (i =0 ; i < iod->nr_allocations; i++) {__le64 *prp_list =nvme_pci_iod_list (req)[i];dma_addr_t next_dma_addr =le64_to_cpu (prp_list[last_prp]);
@@ -561,7 +561,7 @@ static void nvme_free_sgls(struct nvme_dev *dev, struct request *req)dma_addr_t dma_addr = iod->first_dma;int i; -for (i =0 ; i < iod->npages; i++) { +for (i =0 ; i < iod->nr_allocations; i++) {struct nvme_sgl_desc *sg_list =nvme_pci_iod_list (req)[i];dma_addr_t next_dma_addr =le64_to_cpu ((sg_list[last_sg]).addr);
@@ -594,7 +594,7 @@ static void nvme_unmap_data(struct nvme_dev *dev, struct request *req)WARN_ON_ONCE (!iod->nents);nvme_unmap_sg (dev, req); -if (iod->npages ==0 ) +if (iod->nr_allocations ==0 )dma_pool_free (dev->prp_small_pool,nvme_pci_iod_list (req)[0 ], iod->first_dma);else if (iod->use_sgl)
@@ -656,15 +656,15 @@ static blk_status_t nvme_pci_setup_prps(struct nvme_dev *dev, nprps =DIV_ROUND_UP (length,NVME_CTRL_PAGE_SIZE );if (nprps <= (256 /8 )) { pool = dev->prp_small_pool; - iod->npages =0 ; + iod->nr_allocations =0 ; }else { pool = dev->prp_page_pool; - iod->npages =1 ; + iod->nr_allocations =1 ; } prp_list =dma_pool_alloc (pool,GFP_ATOMIC , &prp_dma);if (!prp_list) { - iod->npages =-1 ; + iod->nr_allocations =-1 ;return BLK_STS_RESOURCE ; } list[0 ] = prp_list;
@@ -676,7 +676,7 @@ static blk_status_t nvme_pci_setup_prps(struct nvme_dev *dev, prp_list =dma_pool_alloc (pool,GFP_ATOMIC , &prp_dma);if (!prp_list) goto free_prps; - list[iod->npages++] = prp_list; + list[iod->nr_allocations++] = prp_list; prp_list[0 ] = old_prp_list[i -1 ]; old_prp_list[i -1 ] =cpu_to_le64 (prp_dma); i =1 ;
@@ -750,15 +750,15 @@ static blk_status_t nvme_pci_setup_sgls(struct nvme_dev *dev,if (entries <= (256 /sizeof (struct nvme_sgl_desc ))) { pool = dev->prp_small_pool; - iod->npages =0 ; + iod->nr_allocations =0 ; }else { pool = dev->prp_page_pool; - iod->npages =1 ; + iod->nr_allocations =1 ; } sg_list =dma_pool_alloc (pool,GFP_ATOMIC , &sgl_dma);if (!sg_list) { - iod->npages =-1 ; + iod->nr_allocations =-1 ;return BLK_STS_RESOURCE ; }
@@ -777,7 +777,7 @@ static blk_status_t nvme_pci_setup_sgls(struct nvme_dev *dev, goto free_sgls; i =0 ; -nvme_pci_iod_list (req)[iod->npages++] = sg_list; +nvme_pci_iod_list (req)[iod->nr_allocations++] = sg_list; sg_list[i++] = *link;nvme_pci_sgl_set_seg (link, sgl_dma, entries); }
@@ -905,7 +905,7 @@ static blk_status_t nvme_prep_rq(struct nvme_dev *dev, struct request *req)blk_status_t ret; iod->aborted = false; - iod->npages =-1 ; + iod->nr_allocations =-1 ; iod->nents =0 ; ret =nvme_setup_cmd (req->q->queuedata, req);
@@ -3562,6 +3562,8 @@ static int __init nvme_init(void)BUILD_BUG_ON (sizeof (struct nvme_create_sq ) !=64 );BUILD_BUG_ON (sizeof (struct nvme_delete_queue ) !=64 );BUILD_BUG_ON (IRQ_AFFINITY_MAX_SETS <2 ); +BUILD_BUG_ON (DIV_ROUND_UP (nvme_pci_npages_prp (),NVME_CTRL_PAGE_SIZE ) > +S8_MAX );return pci_register_driver (&nvme_driver); }
Signature · 1 line
2.30.2
PATCH v2 4/5 nvme-pci: iod nents fits in u8
In reply to Keith BuschFrom: Keith Busch <[email protected]>
The maximum number of 'nents' is 127, set by the queue limit, and is only >= 0, so it fits in an 'u8' type. Set to that type to save some space in this highly allocated struct, and ensure we don't accidently let nents exceed this type in the future.
---v1->v2:
Added BUILD_BUG_ON (Christoph)
Made this a u8 instead of s8
drivers/nvme/host/pci.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
drivers/nvme/host/pci.c +2 -1
@@ -229,7 +229,7 @@ struct nvme_iod {bool use_sgl;bool aborted;s8 nr_allocations;/* PRP list pool allocations. 0 means small pool in use */ -int nents;/* Used in scatterlist */ +u8 nents;/* Used in scatterlist */ dma_addr_t first_dma;unsigned int dma_len;/* length of single DMA segment mapping */ dma_addr_t meta_dma;
@@ -3562,6 +3562,7 @@ static int __init nvme_init(void)BUILD_BUG_ON (sizeof (struct nvme_create_sq ) !=64 );BUILD_BUG_ON (sizeof (struct nvme_delete_queue ) !=64 );BUILD_BUG_ON (IRQ_AFFINITY_MAX_SETS <2 ); +BUILD_BUG_ON (NVME_MAX_SEGS >U8_MAX );BUILD_BUG_ON (DIV_ROUND_UP (nvme_pci_npages_prp (),NVME_CTRL_PAGE_SIZE ) >S8_MAX );
Signature · 1 line
2.30.2
PATCH v2 5/5 nvme-pci: move iod dma_len fill gaps
In reply to Keith BuschFrom: Keith Busch <[email protected]>
The 32-bit field, dma_len, packs better in the iod struct above the dma_addr_t on 64-bit systems.
--- drivers/nvme/host/pci.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/nvme/host/pci.c +1 -1
@@ -230,8 +230,8 @@ struct nvme_iod {bool aborted;s8 nr_allocations;/* PRP list pool allocations. 0 means small pool in use */ u8 nents;/* Used in scatterlist */ -dma_addr_t first_dma;unsigned int dma_len;/* length of single DMA segment mapping */ +dma_addr_t first_dma;dma_addr_t meta_dma;struct scatterlist *sg; };
Signature · 1 line
2.30.2
Quoted 6 lines · Keith Busch
From: Keith Busch <[email protected]>
The series adjusts the struct
nvme_iodfields for optimal sizing. The end result on 64-bit arch removes 24 bytes, from 152 down to 128. We typically allocate many thousands of these, and the cache aligned size is a nice bonus.
But would you like to reduce 8 more bytes by killing meta_dma field? This one:
drivers/nvme/host/pci.c +0 -1
@@ -232,7 +232,6 @@ struct nvme_iod {u8 nents;/* Used in scatterlist */ unsigned int dma_len;/* length of single DMA segment mapping */ dma_addr_t first_dma; -dma_addr_t meta_dma;struct scatterlist *sg;
};
@@ -889,13 +888,13 @@ static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req, static blk_status_t nvme_map_metadata(struct nvme_dev *dev, struct request *req,
struct nvme_command *cmnd)
{
- struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
+ dma_addr_t meta_dma;
- iod->meta_dma = dma_map_bvec(dev->dev, rq_integrity_vec(req),
+ meta_dma = dma_map_bvec(dev->dev, rq_integrity_vec(req),
rq_dma_dir(req), 0);
- if (dma_mapping_error(dev->dev, iod->meta_dma))
+ if (dma_mapping_error(dev->dev, meta_dma))
return BLK_STS_IOERR;
- cmnd->rw.metadata = cpu_to_le64(iod->meta_dma);
+ cmnd->rw.metadata = cpu_to_le64(meta_dma);
return BLK_STS_OK;
}@@ -1030,8 +1029,9 @@ static __always_inline void nvme_pci_unmap_rq(struct request *req)
if (blk_integrity_rq(req)) {
struct nvme_iod *iod = blk_mq_rq_to_pdu(req);+ dma_addr_t meta_dma = le64_to_cpu(iod->cmd.rw.metadata);
- dma_unmap_page(dev->dev, iod->meta_dma,
+ dma_unmap_page(dev->dev, meta_dma,
rq_integrity_vec(req)->bv_len, rq_data_dir(req));
}Keith Busch
maintainerQuoted 8 lines · Kanchan Joshi
Quoted 6 lines · Keith Busch
From: Keith Busch <[email protected]>
The series adjusts the struct
nvme_iodfields for optimal sizing. The end result on 64-bit arch removes 24 bytes, from 152 down to 128. We typically allocate many thousands of these, and the cache aligned size is a nice bonus.
Quoted 4 lines
Reviewed-by Kanchan Joshi <[email protected]>But would you like to reduce 8 more bytes by killing
meta_dmafield? This one:
Hm, yes, the iod's command is a local copy, so it is safe to do that. I suppose we could use the same idea to remove 'first_dma' as well: it's a copy of either dptr.prp2 or dptr.sgl.addr.
Quoted 14 lines · Keith Busch
From: Keith Busch <[email protected]>
The series adjusts the struct
nvme_iodfields for optimal sizing. The end result on 64-bit arch removes 24 bytes, from 152 down to 128. We typically allocate many thousands of these, and the cache aligned size is a nice bonus.Keith Busch (5):
nvme-pci: removenvme_queuefromnvme_iod
nvme-pci: iod's 'aborted' is a bool
nvme-pci: iod npages fits in s8
nvme-pci: iod nents fits in u8
nvme-pci: move ioddma_lenfill gaps
Quoted 3 lines
drivers/nvme/host/pci.c | 67 +++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 33 deletions(-)
-ck