nvme-pci: always use blk_map_iter for metadata
PATCH 1/1 nvme-pci: always use blk_map_iter for metadata
From: Keith Busch <[email protected]>
The dma_map_bvec helper doesn't work for p2p data. Rather than special case it, just use the same mapping logic so that the driver doesn't need to consider memory types.
--- drivers/nvme/host/pci.c | 89 ++++++++++++++--------------------------- 1 file changed, 31 insertions(+), 58 deletions(-)
drivers/nvme/host/pci.c +31 -58
@@ -259,9 +259,6 @@ enum nvme_iod_flags {/* single segment dma mapping */ IOD_SINGLE_SEGMENT = 1U << 2, - -/* Metadata using non-coalesced MPTR */ - IOD_SINGLE_META_SEGMENT = 1U << 5, };struct nvme_dma_vec {
@@ -596,17 +593,6 @@ enum nvme_use_sgl { SGL_FORCED, }; -static inline bool nvme_pci_metadata_use_sgls (struct request *req) -{ -struct nvme_queue *nvmeq = req->mq_hctx->driver_data; -struct nvme_dev *dev = nvmeq->dev; - -if (!nvme_ctrl_meta_sgl_supported (&dev->ctrl)) -return false; -return req->nr_integrity_segments >1 || -nvme_req (req)->flags &NVME_REQ_USERCMD ; -} -static inline enum nvme_use_sgl nvme_pci_use_sgls (struct nvme_dev *dev,struct request *req) {
@@ -724,13 +710,6 @@ static void nvme_unmap_metadata(struct request *req)struct device *dma_dev = nvmeq->dev->dev;struct nvme_sgl_desc *sge = iod->meta_descriptor; -if (iod->flags &IOD_SINGLE_META_SEGMENT ) { -dma_unmap_page (dma_dev, iod->meta_dma, -rq_integrity_vec (req).bv_len, -rq_dma_dir (req)); -return ; - } -if (!blk_rq_integrity_dma_unmap (req, dma_dev, &iod->meta_dma_state, iod->meta_total_len)) {if (nvme_pci_cmd_use_meta_sgl (&iod->cmd))
@@ -1042,21 +1021,27 @@ static blk_status_t nvme_map_data(struct request *req)return nvme_pci_setup_data_prp (req, &iter); } -static blk_status_t nvme_pci_setup_meta_sgls (struct request *req) +static blk_status_t nvme_pci_setup_meta_mptr (struct nvme_iod *iod, +struct blk_dma_iter *iter) +{ + iod->cmd.common.metadata =cpu_to_le64 (iter->addr); + iod->meta_total_len = iter->len; + iod->meta_dma = iter->addr; + iod->meta_descriptor =NULL ; +return BLK_STS_OK ; +} + +static blk_status_t nvme_pci_setup_meta_sgls (struct request *req, +struct blk_dma_iter *iter) {struct nvme_queue *nvmeq = req->mq_hctx->driver_data;unsigned int entries = req->nr_integrity_segments;struct nvme_iod *iod =blk_mq_rq_to_pdu (req);struct nvme_dev *dev = nvmeq->dev;struct nvme_sgl_desc *sg_list; -struct blk_dma_iter iter;dma_addr_t sgl_dma;int i =0 ; -if (!blk_rq_integrity_dma_map_iter_start (req, dev->dev, - &iod->meta_dma_state, &iter)) -return iter.status; -if (blk_rq_dma_map_coalesce (&iod->meta_dma_state)) entries =1 ;
@@ -1073,13 +1058,8 @@ static blk_status_t nvme_pci_setup_meta_sgls(struct request *req)* mechanism to catch any misunderstandings between the application and * device. */ -if (entries ==1 && !(nvme_req (req)->flags &NVME_REQ_USERCMD )) { - iod->cmd.common.metadata =cpu_to_le64 (iter.addr); - iod->meta_total_len = iter.len; - iod->meta_dma = iter.addr; - iod->meta_descriptor =NULL ; -return BLK_STS_OK ; - } +if (entries ==1 && !(nvme_req (req)->flags &NVME_REQ_USERCMD )) +return nvme_pci_setup_meta_mptr (iod, iter); sg_list =dma_pool_alloc (nvmeq->descriptor_pools.small,GFP_ATOMIC , &sgl_dma);
@@ -1091,45 +1071,38 @@ static blk_status_t nvme_pci_setup_meta_sgls(struct request *req) iod->cmd.common.flags =NVME_CMD_SGL_METASEG ; iod->cmd.common.metadata =cpu_to_le64 (sgl_dma);if (entries ==1 ) { - iod->meta_total_len = iter.len; -nvme_pci_sgl_set_data (sg_list, &iter); + iod->meta_total_len = iter->len; +nvme_pci_sgl_set_data (sg_list, iter);return BLK_STS_OK ; } sgl_dma +=sizeof (*sg_list);do { -nvme_pci_sgl_set_data (&sg_list[++i], &iter); - iod->meta_total_len += iter.len; - }while (blk_rq_integrity_dma_map_iter_next (req, dev->dev, &iter)); +nvme_pci_sgl_set_data (&sg_list[++i], iter); + iod->meta_total_len += iter->len; + }while (blk_rq_integrity_dma_map_iter_next (req, dev->dev, iter));nvme_pci_sgl_set_seg (sg_list, sgl_dma, i); -if (unlikely (iter.status)) +if (unlikely (iter->status))nvme_unmap_metadata (req); -return iter.status; +return iter->status; } -static blk_status_t nvme_pci_setup_meta_mptr (struct request *req) +static blk_status_t nvme_map_metadata (struct request *req) { -struct nvme_iod *iod =blk_mq_rq_to_pdu (req);struct nvme_queue *nvmeq = req->mq_hctx->driver_data; -struct bio_vec bv =rq_integrity_vec (req); +struct nvme_iod *iod =blk_mq_rq_to_pdu (req); +struct nvme_dev *dev = nvmeq->dev; +struct blk_dma_iter iter; - iod->meta_dma =dma_map_bvec (nvmeq->dev->dev, &bv,rq_dma_dir (req),0 ); -if (dma_mapping_error (nvmeq->dev->dev, iod->meta_dma)) -return BLK_STS_IOERR ; - iod->cmd.common.metadata =cpu_to_le64 (iod->meta_dma); - iod->flags |=IOD_SINGLE_META_SEGMENT ; -return BLK_STS_OK ; -} +if (blk_rq_integrity_dma_map_iter_start (req, dev->dev, + &iod->meta_dma_state, &iter)) +return iter.status; -static blk_status_t nvme_map_metadata (struct request *req) -{ -struct nvme_iod *iod =blk_mq_rq_to_pdu (req); +if (!nvme_ctrl_meta_sgl_supported (&dev->ctrl)) +return nvme_pci_setup_meta_mptr (iod, &iter); -if ((iod->cmd.common.flags &NVME_CMD_SGL_METABUF ) && -nvme_pci_metadata_use_sgls (req)) -return nvme_pci_setup_meta_sgls (req); -return nvme_pci_setup_meta_mptr (req); +return nvme_pci_setup_meta_sgls (req, &iter); }static blk_status_t nvme_prep_rq (struct request *req)
Signature · 1 line
2.47.3
Quoted 8 lines · Keith Busch
From: Keith Busch<[email protected]>
The
dma_map_bvechelper doesn't work for p2p data. Rather than special case it, just use the same mapping logic so that the driver doesn't need to consider memory types.Reported-by Leon Romanovsky<[email protected]>Signed-off-by Keith Busch<[email protected]>
Looks good.
-ck
Christoph Hellwig
maintainerQuoted 5 lines · Keith Busch
From: Keith Busch <[email protected]>
The
dma_map_bvechelper doesn't work for p2p data. Rather than special case it, just use the same mapping logic so that the driver doesn't need to consider memory types.
We already consider the memory types for the data path, so treating the metadasta path where p2p is even more unlikely sounds like the wrong tradeoff. If we have a single segment we just need a single is_pci_p2pdma_page check to skip direct mapping path. Something like this untested patch:
drivers/nvme/host/pci.c +9 -5
@@ -1042,7 +1042,7 @@ static blk_status_t nvme_map_data(struct request *req)return nvme_pci_setup_data_prp (req, &iter); } -static blk_status_t nvme_pci_setup_meta_sgls (struct request *req) +static blk_status_t nvme_pci_setup_meta_iter (struct request *req) {struct nvme_queue *nvmeq = req->mq_hctx->driver_data;unsigned int entries = req->nr_integrity_segments;
@@ -1073,7 +1073,9 @@ static blk_status_t nvme_pci_setup_meta_sgls(struct request *req)* mechanism to catch any misunderstandings between the application and * device. */ -if (entries ==1 && !(nvme_req (req)->flags &NVME_REQ_USERCMD )) { +if (entries ==1 && + !(nvme_req (req)->flags &NVME_REQ_USERCMD ) && +nvme_ctrl_meta_sgl_supported (&dev->ctrl)) { iod->cmd.common.metadata =cpu_to_le64 (iter.addr); iod->meta_total_len = iter.len; iod->meta_dma = iter.addr;
@@ -1125,10 +1127,12 @@ static blk_status_t nvme_pci_setup_meta_mptr(struct request *req)static blk_status_t nvme_map_metadata (struct request *req) {struct nvme_iod *iod =blk_mq_rq_to_pdu (req); +struct bio_vec bv =req_bvec (req); -if ((iod->cmd.common.flags &NVME_CMD_SGL_METABUF ) && -nvme_pci_metadata_use_sgls (req)) -return nvme_pci_setup_meta_sgls (req); +if (((iod->cmd.common.flags &NVME_CMD_SGL_METABUF ) && +nvme_pci_metadata_use_sgls (req)) || +is_pci_p2pdma_page (bv.bv_page)) +return nvme_pci_setup_meta_iter (req);return nvme_pci_setup_meta_mptr (req); }
Keith Busch
maintainerQuoted 10 lines · Christoph Hellwig
Quoted 5 lines · Keith Busch
From: Keith Busch <[email protected]>
The
dma_map_bvechelper doesn't work for p2p data. Rather than special case it, just use the same mapping logic so that the driver doesn't need to consider memory types.We already consider the memory types for the data path, so treating the metadasta path where p2p is even more unlikely sounds like the wrong tradeoff.
Maybe the data path should only use blk_dma_iter too. Is dma_map_bvec that much faster for the single vector case? I'm going to test both on real hardware and see if there's a difference in CPU utilization or latency.
Quoted 3 lines
If we have a single segment we just need a single
is_pci_p2pdma_pagecheck to skip direct mapping path. Something like this untested patch:
The logic you used isn't accurate, but I know what you mean. I'll spin a more minimal v2 patch.
Quoted 8 lines · Keith Busch
Quoted 7 lines · Christoph Hellwig
Quoted 5 lines · Keith Busch
From: Keith Busch <[email protected]>
The
dma_map_bvechelper doesn't work for p2p data. Rather than special case it, just use the same mapping logic so that the driver doesn't need to consider memory types.
Quoted 8 lines
Quoted 3 lines
We already consider the memory types for the data path, so treating the metadasta path where p2p is even more unlikely sounds like the wrong tradeoff.
Maybe the data path should only use
blk_dma_itertoo. Isdma_map_bvecthat much faster for the single vector case? I'm going to test both on real hardware and see if there's a difference in CPU utilization or latency.
We tried that path (remove special case for single segment), but for very performance oriented case: direct mode without IOMMU, no p2p and extremely powerful RAID (~94M IOPS), Jens and Kanchan saw performance drop. On consumer grade HW (~2M IOPS), the performance change was not noticed.
Thanks
Christoph Hellwig
maintainerQuoted 10 lines · Leon Romanovsky
Quoted 4 lines
Maybe the data path should only use
blk_dma_itertoo. Isdma_map_bvecthat much faster for the single vector case? I'm going to test both on real hardware and see if there's a difference in CPU utilization or latency.We tried that path (remove special case for single segment), but for very performance oriented case: direct mode without IOMMU, no p2p and extremely powerful RAID (~94M IOPS), Jens and Kanchan saw performance drop. On consumer grade HW (~2M IOPS), the performance change was not noticed.
Yes. Sorry, I wanted to reply something like this to Keith earlier but managed to drop the ball.
Keith Busch
maintainerQuoted 6 lines · Christoph Hellwig
Quoted 5 lines · Leon Romanovsky
Quoted 4 lines
Maybe the data path should only use
blk_dma_itertoo. Isdma_map_bvecthat much faster for the single vector case? I'm going to test both on real hardware and see if there's a difference in CPU utilization or latency.
Quoted 8 lines
Quoted 5 lines
We tried that path (remove special case for single segment), but for very performance oriented case: direct mode without IOMMU, no p2p and extremely powerful RAID (~94M IOPS), Jens and Kanchan saw performance drop. On consumer grade HW (~2M IOPS), the performance change was not noticed.
Yes. Sorry, I wanted to reply something like this to Keith earlier but managed to drop the ball.
Yeah, no problem. I was reaching the same conclusion anyway and moved on.