Skip to contentSkip to conversation

nvme-pci: iod optimisations

PATCH Superseded Keith Busch 4 participants 9 messages Jul 29, 2022 – Aug 2, 2022 NVMe

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

v2 cover nvme-pci: iod optimisations

From: Keith Busch <[email protected]>

The series adjusts the struct nvme_iod fields for optimal sizing. The end result on 64-bit arch removes 24 bytes, from 152 down to 128. We typically allocate many thousands of these, and the cache aligned size is a nice bonus.

Keith Busch (5):
nvme-pci: remove nvme_queue from nvme_iod
nvme-pci: iod's 'aborted' is a bool
nvme-pci: iod npages fits in s8
nvme-pci: iod nents fits in u8
nvme-pci: move iod dma_len fill gaps

 drivers/nvme/host/pci.c | 67 +++++++++++++++++++++--------------------
 1 file changed, 34 insertions(+), 33 deletions(-)
Signature · 1 line
2.30.2
#1 lore raw

PATCH v2 1/5 nvme-pci: remove nvme_queue from nvme_iod

From: Keith Busch <[email protected]>

We can get the nvme_queue from the req just as easily, so remove the duplicate path to the same structure to save some space.

---
 drivers/nvme/host/pci.c | 28 +++++++++++++---------------
 1 file changed, 13 insertions(+), 15 deletions(-)
drivers/nvme/host/pci.c +13 -15
index 644664098ae7..138d408f7306 100644
@@ -226,7 +226,6 @@ struct nvme_queue {
 struct nvme_iod {
 	struct nvme_request req;
 	struct nvme_command cmd;
-	struct nvme_queue *nvmeq;
 	bool use_sgl;
 	int aborted;
 	int npages;		/* In the PRP list. 0 means small pool in use */
@@ -431,11 +430,6 @@ static int nvme_pci_init_request(struct blk_mq_tag_set *set,
 {
 	struct nvme_dev *dev = set->driver_data;
 	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
-	int queue_idx = (set == &dev->tagset) ? hctx_idx + 1 : 0;
-	struct nvme_queue *nvmeq = &dev->queues[queue_idx];
-
-	BUG_ON(!nvmeq);
-	iod->nvmeq = nvmeq;
 
 	nvme_req(req)->ctrl = &dev->ctrl;
 	nvme_req(req)->cmd = &iod->cmd;
@@ -529,7 +523,7 @@ static void **nvme_pci_iod_list(struct request *req)
 
 static inline bool nvme_pci_use_sgls(struct nvme_dev *dev, struct request *req)
 {
-	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
+	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
 	int nseg = blk_rq_nr_phys_segments(req);
 	unsigned int avg_seg_size;
 
@@ -537,7 +531,7 @@ static inline bool nvme_pci_use_sgls(struct nvme_dev *dev, struct request *req)
 
 	if (!nvme_ctrl_sgl_supported(&dev->ctrl))
 		return false;
-	if (!iod->nvmeq->qid)
+	if (!nvmeq->qid)
 		return false;
 	if (!sgl_threshold || avg_seg_size < sgl_threshold)
 		return false;
@@ -843,6 +837,7 @@ static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req,
 	int nr_mapped;
 
 	if (blk_rq_nr_phys_segments(req) == 1) {
+		struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
 		struct bio_vec bv = req_bvec(req);
 
 		if (!is_pci_p2pdma_page(bv.bv_page)) {
@@ -850,7 +845,7 @@ static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req,
 				return nvme_setup_prp_simple(dev, req,
 							     &cmnd->rw, &bv);
 
-			if (iod->nvmeq->qid && sgl_threshold &&
+			if (nvmeq->qid && sgl_threshold &&
 			    nvme_ctrl_sgl_supported(&dev->ctrl))
 				return nvme_setup_sgl_simple(dev, req,
 							     &cmnd->rw, &bv);
@@ -1030,12 +1025,16 @@ static void nvme_queue_rqs(struct request **rqlist)
 
 static __always_inline void nvme_pci_unmap_rq(struct request *req)
 {
-	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
-	struct nvme_dev *dev = iod->nvmeq->dev;
+	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
+	struct nvme_dev *dev = nvmeq->dev;
+
+	if (blk_integrity_rq(req)) {
+	        struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 
-	if (blk_integrity_rq(req))
 		dma_unmap_page(dev->dev, iod->meta_dma,
 			       rq_integrity_vec(req)->bv_len, rq_data_dir(req));
+	}
+
 	if (blk_rq_nr_phys_segments(req))
 		nvme_unmap_data(dev, req);
 }
@@ -1283,8 +1282,7 @@ static int adapter_delete_sq(struct nvme_dev *dev, u16 sqid)
 
 static void abort_endio(struct request *req, blk_status_t error)
 {
-	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
-	struct nvme_queue *nvmeq = iod->nvmeq;
+	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
 
 	dev_warn(nvmeq->dev->ctrl.device,
 		 "Abort status: 0x%x", nvme_req(req)->status);
@@ -1346,7 +1344,7 @@ static void nvme_warn_reset(struct nvme_dev *dev, u32 csts)
 static enum blk_eh_timer_return nvme_timeout(struct request *req)
 {
 	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
-	struct nvme_queue *nvmeq = iod->nvmeq;
+	struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
 	struct nvme_dev *dev = nvmeq->dev;
 	struct request *abort_req;
 	struct nvme_command cmd = { };
Signature · 1 line
2.30.2
#2 lore raw

PATCH v2 2/5 nvme-pci: iod's 'aborted' is a bool

In reply to Keith Busch

From: Keith Busch <[email protected]>

It's only true or false, so make this a bool to reflect that and save some space in nvme_iod.

---
 drivers/nvme/host/pci.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
drivers/nvme/host/pci.c +3 -3
index 138d408f7306..a8ae043e33de 100644
@@ -227,7 +227,7 @@ struct nvme_iod {
 	struct nvme_request req;
 	struct nvme_command cmd;
 	bool use_sgl;
-	int aborted;
+	bool aborted;
 	int npages;		/* In the PRP list. 0 means small pool in use */
 	int nents;		/* Used in scatterlist */
 	dma_addr_t first_dma;
@@ -904,7 +904,7 @@ static blk_status_t nvme_prep_rq(struct nvme_dev *dev, struct request *req)
 	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 	blk_status_t ret;
 
-	iod->aborted = 0;
+	iod->aborted = false;
 	iod->npages = -1;
 	iod->nents = 0;
 
@@ -1425,7 +1425,7 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req)
 		atomic_inc(&dev->ctrl.abort_limit);
 		return BLK_EH_RESET_TIMER;
 	}
-	iod->aborted = 1;
+	iod->aborted = true;
 
 	cmd.abort.opcode = nvme_admin_abort_cmd;
 	cmd.abort.cid = nvme_cid(req);
Signature · 1 line
2.30.2
#3 lore raw

PATCH v2 3/5 nvme-pci: iod npages fits in s8

In reply to Keith Busch

From: Keith Busch <[email protected]>

The largest allowed transfer is 4MB, which can use at most 1025 PRPs. Each PRP is 8 bytes, so the maximum number of 4k nvme pages needed for the iod_list is 3, which fits in an 's8' type.

While modifying this field, change the name to "nr_allocations" to better represent that this is referring to the number of units allocated from a dma_pool.

Also introduce a BUILD_BUG_ON to ensure we never accidently increase the largest transfer limit beyond 127 chained prp lists.

---
v1->v2:
Renamed field from npages to nr_allocations (Christoph)

Added BUILD_BUG_ON

 drivers/nvme/host/pci.c | 28 +++++++++++++++-------------
 1 file changed, 15 insertions(+), 13 deletions(-)
drivers/nvme/host/pci.c +15 -13
index a8ae043e33de..e5b8e1c277b2 100644
@@ -228,7 +228,7 @@ struct nvme_iod {
 	struct nvme_command cmd;
 	bool use_sgl;
 	bool aborted;
-	int npages;		/* In the PRP list. 0 means small pool in use */
+	s8 nr_allocations;	/* PRP list pool allocations. 0 means small pool in use */
 	int nents;		/* Used in scatterlist */
 	dma_addr_t first_dma;
 	unsigned int dma_len;	/* length of single DMA segment mapping */
@@ -545,7 +545,7 @@ static void nvme_free_prps(struct nvme_dev *dev, struct request *req)
 	dma_addr_t dma_addr = iod->first_dma;
 	int i;
 
-	for (i = 0; i < iod->npages; i++) {
+	for (i = 0; i < iod->nr_allocations; i++) {
 		__le64 *prp_list = nvme_pci_iod_list(req)[i];
 		dma_addr_t next_dma_addr = le64_to_cpu(prp_list[last_prp]);
 
@@ -561,7 +561,7 @@ static void nvme_free_sgls(struct nvme_dev *dev, struct request *req)
 	dma_addr_t dma_addr = iod->first_dma;
 	int i;
 
-	for (i = 0; i < iod->npages; i++) {
+	for (i = 0; i < iod->nr_allocations; i++) {
 		struct nvme_sgl_desc *sg_list = nvme_pci_iod_list(req)[i];
 		dma_addr_t next_dma_addr = le64_to_cpu((sg_list[last_sg]).addr);
 
@@ -594,7 +594,7 @@ static void nvme_unmap_data(struct nvme_dev *dev, struct request *req)
 	WARN_ON_ONCE(!iod->nents);
 
 	nvme_unmap_sg(dev, req);
-	if (iod->npages == 0)
+	if (iod->nr_allocations == 0)
 		dma_pool_free(dev->prp_small_pool, nvme_pci_iod_list(req)[0],
 			      iod->first_dma);
 	else if (iod->use_sgl)
@@ -656,15 +656,15 @@ static blk_status_t nvme_pci_setup_prps(struct nvme_dev *dev,
 	nprps = DIV_ROUND_UP(length, NVME_CTRL_PAGE_SIZE);
 	if (nprps <= (256 / 8)) {
 		pool = dev->prp_small_pool;
-		iod->npages = 0;
+		iod->nr_allocations = 0;
 	} else {
 		pool = dev->prp_page_pool;
-		iod->npages = 1;
+		iod->nr_allocations = 1;
 	}
 
 	prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);
 	if (!prp_list) {
-		iod->npages = -1;
+		iod->nr_allocations = -1;
 		return BLK_STS_RESOURCE;
 	}
 	list[0] = prp_list;
@@ -676,7 +676,7 @@ static blk_status_t nvme_pci_setup_prps(struct nvme_dev *dev,
 			prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);
 			if (!prp_list)
 				goto free_prps;
-			list[iod->npages++] = prp_list;
+			list[iod->nr_allocations++] = prp_list;
 			prp_list[0] = old_prp_list[i - 1];
 			old_prp_list[i - 1] = cpu_to_le64(prp_dma);
 			i = 1;
@@ -750,15 +750,15 @@ static blk_status_t nvme_pci_setup_sgls(struct nvme_dev *dev,
 
 	if (entries <= (256 / sizeof(struct nvme_sgl_desc))) {
 		pool = dev->prp_small_pool;
-		iod->npages = 0;
+		iod->nr_allocations = 0;
 	} else {
 		pool = dev->prp_page_pool;
-		iod->npages = 1;
+		iod->nr_allocations = 1;
 	}
 
 	sg_list = dma_pool_alloc(pool, GFP_ATOMIC, &sgl_dma);
 	if (!sg_list) {
-		iod->npages = -1;
+		iod->nr_allocations = -1;
 		return BLK_STS_RESOURCE;
 	}
 
@@ -777,7 +777,7 @@ static blk_status_t nvme_pci_setup_sgls(struct nvme_dev *dev,
 				goto free_sgls;
 
 			i = 0;
-			nvme_pci_iod_list(req)[iod->npages++] = sg_list;
+			nvme_pci_iod_list(req)[iod->nr_allocations++] = sg_list;
 			sg_list[i++] = *link;
 			nvme_pci_sgl_set_seg(link, sgl_dma, entries);
 		}
@@ -905,7 +905,7 @@ static blk_status_t nvme_prep_rq(struct nvme_dev *dev, struct request *req)
 	blk_status_t ret;
 
 	iod->aborted = false;
-	iod->npages = -1;
+	iod->nr_allocations = -1;
 	iod->nents = 0;
 
 	ret = nvme_setup_cmd(req->q->queuedata, req);
@@ -3562,6 +3562,8 @@ static int __init nvme_init(void)
 	BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64);
 	BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64);
 	BUILD_BUG_ON(IRQ_AFFINITY_MAX_SETS < 2);
+	BUILD_BUG_ON(DIV_ROUND_UP(nvme_pci_npages_prp(), NVME_CTRL_PAGE_SIZE) >
+		     S8_MAX);
 
 	return pci_register_driver(&nvme_driver);
 }
Signature · 1 line
2.30.2
#4 lore raw

PATCH v2 4/5 nvme-pci: iod nents fits in u8

In reply to Keith Busch

From: Keith Busch <[email protected]>

The maximum number of 'nents' is 127, set by the queue limit, and is only >= 0, so it fits in an 'u8' type. Set to that type to save some space in this highly allocated struct, and ensure we don't accidently let nents exceed this type in the future.

---
v1->v2:
Added BUILD_BUG_ON (Christoph)

Made this a u8 instead of s8

 drivers/nvme/host/pci.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
drivers/nvme/host/pci.c +2 -1
index e5b8e1c277b2..6d491a27ed73 100644
@@ -229,7 +229,7 @@ struct nvme_iod {
 	bool use_sgl;
 	bool aborted;
 	s8 nr_allocations;	/* PRP list pool allocations. 0 means small pool in use */
-	int nents;		/* Used in scatterlist */
+	u8 nents;		/* Used in scatterlist */
 	dma_addr_t first_dma;
 	unsigned int dma_len;	/* length of single DMA segment mapping */
 	dma_addr_t meta_dma;
@@ -3562,6 +3562,7 @@ static int __init nvme_init(void)
 	BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64);
 	BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64);
 	BUILD_BUG_ON(IRQ_AFFINITY_MAX_SETS < 2);
+	BUILD_BUG_ON(NVME_MAX_SEGS > U8_MAX);
 	BUILD_BUG_ON(DIV_ROUND_UP(nvme_pci_npages_prp(), NVME_CTRL_PAGE_SIZE) >
 		     S8_MAX);
 
Signature · 1 line
2.30.2
#5 lore raw

PATCH v2 5/5 nvme-pci: move iod dma_len fill gaps

In reply to Keith Busch

From: Keith Busch <[email protected]>

The 32-bit field, dma_len, packs better in the iod struct above the dma_addr_t on 64-bit systems.

---
 drivers/nvme/host/pci.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/nvme/host/pci.c +1 -1
index 6d491a27ed73..06b7ec0d5558 100644
@@ -230,8 +230,8 @@ struct nvme_iod {
 	bool aborted;
 	s8 nr_allocations;	/* PRP list pool allocations. 0 means small pool in use */
 	u8 nents;		/* Used in scatterlist */
-	dma_addr_t first_dma;
 	unsigned int dma_len;	/* length of single DMA segment mapping */
+	dma_addr_t first_dma;
 	dma_addr_t meta_dma;
 	struct scatterlist *sg;
 };
Signature · 1 line
2.30.2
Reviewed-by on the v2 series
#6 lore raw
In reply to Keith Busch
Quoted 6 lines · Keith Busch

From: Keith Busch <[email protected]>

The series adjusts the struct nvme_iod fields for optimal sizing. The end result on 64-bit arch removes 24 bytes, from 152 down to 128. We typically allocate many thousands of these, and the cache aligned size is a nice bonus.

But would you like to reduce 8 more bytes by killing meta_dma field? This one:

drivers/nvme/host/pci.c +0 -1
index 06b7ec0d5558..119192363b31 100644
@@ -232,7 +232,6 @@ struct nvme_iod {
       u8 nents;               /* Used in scatterlist */
       unsigned int dma_len;   /* length of single DMA segment mapping */
       dma_addr_t first_dma;
-       dma_addr_t meta_dma;
       struct scatterlist *sg;

};

@@ -889,13 +888,13 @@ static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req, static blk_status_t nvme_map_metadata(struct nvme_dev *dev, struct request *req,
struct nvme_command *cmnd)
{
- struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
+ dma_addr_t meta_dma;

- iod->meta_dma = dma_map_bvec(dev->dev, rq_integrity_vec(req),
+ meta_dma = dma_map_bvec(dev->dev, rq_integrity_vec(req),

                       rq_dma_dir(req), 0);
-       if (dma_mapping_error(dev->dev, iod->meta_dma))
+       if (dma_mapping_error(dev->dev, meta_dma))
               return BLK_STS_IOERR;
-       cmnd->rw.metadata = cpu_to_le64(iod->meta_dma);
+       cmnd->rw.metadata = cpu_to_le64(meta_dma);
       return BLK_STS_OK;
}

@@ -1030,8 +1029,9 @@ static __always_inline void nvme_pci_unmap_rq(struct request *req)

       if (blk_integrity_rq(req)) {
               struct nvme_iod *iod = blk_mq_rq_to_pdu(req);

+ dma_addr_t meta_dma = le64_to_cpu(iod->cmd.rw.metadata);

- dma_unmap_page(dev->dev, iod->meta_dma,
+ dma_unmap_page(dev->dev, meta_dma,

                              rq_integrity_vec(req)->bv_len, rq_data_dir(req));
       }
maintainer
#7 lore raw
Quoted 8 lines · Kanchan Joshi
Quoted 6 lines · Keith Busch

From: Keith Busch <[email protected]>

The series adjusts the struct nvme_iod fields for optimal sizing. The end result on 64-bit arch removes 24 bytes, from 152 down to 128. We typically allocate many thousands of these, and the cache aligned size is a nice bonus.

Quoted 4 lines

But would you like to reduce 8 more bytes by killing meta_dma field? This one:

Hm, yes, the iod's command is a local copy, so it is safe to do that. I suppose we could use the same idea to remove 'first_dma' as well: it's a copy of either dptr.prp2 or dptr.sgl.addr.

Reviewed-by on the v2 series
#8 lore raw
In reply to Keith Busch
Quoted 14 lines · Keith Busch

From: Keith Busch <[email protected]>

The series adjusts the struct nvme_iod fields for optimal sizing. The end result on 64-bit arch removes 24 bytes, from 152 down to 128. We typically allocate many thousands of these, and the cache aligned size is a nice bonus.

Keith Busch (5):
nvme-pci: remove nvme_queue from nvme_iod
nvme-pci: iod's 'aborted' is a bool
nvme-pci: iod npages fits in s8
nvme-pci: iod nents fits in u8
nvme-pci: move iod dma_len fill gaps

Quoted 3 lines
  drivers/nvme/host/pci.c | 67 +++++++++++++++++++++--------------------
  1 file changed, 34 insertions(+), 33 deletions(-)

-ck