blk dma iter for integrity metadata
Subsystems (2)
v6 cover blk dma iter for integrity metadata
From: Keith Busch <[email protected]>
Previous version:
https://lore.kernel.org/linux-block/[email protected]/
Changes since v5, addressing review feedback from Christoph:
- Keep the phys_vec, create a different iterator for the lower layer separated from the phys_vec
- Commit log changes describing the bvec array being iterated.
- Rename the blk_map_iter initialiation function; take a pointer instead of returning a copy; make it inline
- Rename the bvec pointer being iterated to "bvecs"
- Have bio-integrity legacy scatter-gather iteration subscribe to the new dma api. This required relocating some inline functions to header files.
- Using 'bio_integrity()' to avoid needing "#ifdef CONFIG_" usage
- Kernel doc for integrity dma mapping APIs
- nvme comment explaining when we do or don't use MPTR format
- Various nvme code cleanups
Keith Busch (8):
blk-mq-dma: create blk_map_iter type
blk-mq-dma: provide the bio_vec array being iterated
blk-mq-dma: require unmap caller provide p2p map type
blk-mq: remove REQ_P2PDMA flag
blk-mq-dma: move common dma start code to a helper
blk-mq-dma: add support for mapping integrity metadata
nvme-pci: create common sgl unmapping helper
nvme-pci: convert metadata mapping to dma iter
block/bio.c | 2 +- block/blk-integrity.c | 43 +++--- block/blk-mq-dma.c | 251 +++++++++++++++++++++++----------- block/blk-mq.h | 26 ++++ drivers/nvme/host/pci.c | 213 +++++++++++++++++------------ include/linux/blk-integrity.h | 17 +++ include/linux/blk-mq-dma.h | 14 +- include/linux/blk_types.h | 2 - 8 files changed, 364 insertions(+), 204 deletions(-)
Signature · 1 line
2.47.3
PATCH v6 1/8 blk-mq-dma: create blk_map_iter type
From: Keith Busch <[email protected]>
The req_iterator happens to have similar fields to what the dma iterator needs, but we're not necessarily iterating a request's bi_io_vec. Create a new type specific to the blk-dma api that can be amended for new future uses.
--- block/blk-mq-dma.c | 4 ++-- include/linux/blk-mq-dma.h | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-)
block/blk-mq-dma.c +2 -2
@@ -10,7 +10,7 @@ struct phys_vec {u32 len; }; -static bool blk_map_iter_next (struct request *req,struct req_iterator *iter, +static bool blk_map_iter_next (struct request *req,struct blk_map_iter *iter,struct phys_vec *vec) {unsigned int max_size;
@@ -246,7 +246,7 @@ blk_next_sg(struct scatterlist **sg, struct scatterlist *sglist)int __blk_rq_map_sg (struct request *rq,struct scatterlist *sglist,struct scatterlist **last_sg) { -struct req_iterator iter = { +struct blk_map_iter iter = { .bio = rq->bio, };struct phys_vec vec;
include/linux/blk-mq-dma.h +6 -1
@@ -5,6 +5,11 @@#include <linux/blk-mq.h> #include <linux/pci-p2pdma.h> +struct blk_map_iter { +struct bvec_iter iter; +struct bio *bio; +}; +struct blk_dma_iter {/* Output address range for this iteration */ dma_addr_t addr;
@@ -14,7 +19,7 @@ struct blk_dma_iter {blk_status_t status;/* Internal to blk_rq_dma_map_iter_* */ -struct req_iterator iter; +struct blk_map_iter iter;struct pci_p2pdma_map_state p2pdma; };
Signature · 1 line
2.47.3
Re: [PATCHv6 1/8] blk-mq-dma: create blk_map_iter type
Looks good:
PATCH v6 2/8 blk-mq-dma: provide the bio_vec array being iterated
In reply to Keith BuschFrom: Keith Busch <[email protected]>
This will make it easier to add different sources of the bvec array, like for upcoming integrity support, rather than assume to use the bio's bi_io_vec. It also makes iterating "special" payloads more in common with iterating normal payloads.
--- block/blk-mq-dma.c | 56 ++++++++++++++++++++++---------------- include/linux/blk-mq-dma.h | 1 + 2 files changed, 34 insertions(+), 23 deletions(-)
block/blk-mq-dma.c +33 -23
@@ -16,23 +16,14 @@ static bool blk_map_iter_next(struct request *req, struct blk_map_iter *iter,unsigned int max_size;struct bio_vec bv; -if (req->rq_flags &RQF_SPECIAL_PAYLOAD ) { -if (!iter->bio) -return false; - vec->paddr =bvec_phys (&req->special_vec); - vec->len = req->special_vec.bv_len; - iter->bio =NULL ; -return true; - } -if (!iter->iter.bi_size)return false; - bv =mp_bvec_iter_bvec (iter->bio->bi_io_vec, iter->iter); + bv =mp_bvec_iter_bvec (iter->bvecs, iter->iter); vec->paddr =bvec_phys (&bv); max_size =get_max_segment_size (&req->q->limits, vec->paddr,UINT_MAX ); bv.bv_len =min (bv.bv_len, max_size); -bio_advance_iter_single (iter->bio, &iter->iter, bv.bv_len); +bvec_iter_advance_single (iter->bvecs, &iter->iter, bv.bv_len);/* * If we are entirely done with this bi_io_vec entry, check if the next
@@ -43,19 +34,20 @@ static bool blk_map_iter_next(struct request *req, struct blk_map_iter *iter,struct bio_vec next;if (!iter->iter.bi_size) { -if (!iter->bio->bi_next) +if (!iter->bio || !iter->bio->bi_next)break ; iter->bio = iter->bio->bi_next; iter->iter = iter->bio->bi_iter; + iter->bvecs = iter->bio->bi_io_vec; } - next =mp_bvec_iter_bvec (iter->bio->bi_io_vec, iter->iter); + next =mp_bvec_iter_bvec (iter->bvecs, iter->iter);if (bv.bv_len + next.bv_len > max_size || !biovec_phys_mergeable (req->q, &bv, &next))break ; bv.bv_len += next.bv_len; -bio_advance_iter_single (iter->bio, &iter->iter, next.bv_len); +bvec_iter_advance_single (iter->bvecs, &iter->iter, next.bv_len); } vec->len = bv.bv_len;
@@ -125,6 +117,30 @@ static bool blk_rq_dma_map_iova(struct request *req, struct device *dma_dev,return true; } +static inline void blk_rq_map_iter_init (struct request *rq, +struct blk_map_iter *iter) +{ +struct bio *bio = rq->bio; + +if (rq->rq_flags &RQF_SPECIAL_PAYLOAD ) { + *iter = (struct blk_map_iter ) { + .bvecs = &rq->special_vec, + .iter = { + .bi_size = rq->special_vec.bv_len, + } + }; + }else if (bio) { + *iter = (struct blk_map_iter ) { + .bio = bio, + .bvecs = bio->bi_io_vec, + .iter = bio->bi_iter, + }; + }else { +/* the internal flush request may not have bio attached */ + *iter = (struct blk_map_iter ) {}; + } +} +/** * blk_rq_dma_map_iter_start - map the first DMA segment for a request * @req: request to map
@@ -153,8 +169,7 @@ bool blk_rq_dma_map_iter_start(struct request *req, struct device *dma_dev,unsigned int total_len =blk_rq_payload_bytes (req);struct phys_vec vec; - iter->iter.bio = req->bio; - iter->iter.iter = req->bio->bi_iter; +blk_rq_map_iter_init (req, &iter->iter);memset (&iter->p2pdma,0 ,sizeof (iter->p2pdma)); iter->status =BLK_STS_OK ;
@@ -246,16 +261,11 @@ blk_next_sg(struct scatterlist **sg, struct scatterlist *sglist)int __blk_rq_map_sg (struct request *rq,struct scatterlist *sglist,struct scatterlist **last_sg) { -struct blk_map_iter iter = { - .bio = rq->bio, - }; +struct blk_map_iter iter;struct phys_vec vec;int nsegs =0 ; -/* the internal flush request may not have bio attached */ -if (iter.bio) - iter.iter = iter.bio->bi_iter; - +blk_rq_map_iter_init (rq, &iter);while (blk_map_iter_next (rq, &iter, &vec)) { *last_sg =blk_next_sg (last_sg, sglist); sg_set_page(*last_sg, phys_to_page(vec.paddr), vec.len,
include/linux/blk-mq-dma.h +1 -0
@@ -8,6 +8,7 @@struct blk_map_iter {struct bvec_iter iter;struct bio *bio; +struct bio_vec *bvecs; };struct blk_dma_iter {
Signature · 1 line
2.47.3
Re: [PATCHv6 2/8] blk-mq-dma: provide the bio_vec array being iterated
Looks good:
PATCH v6 3/8 blk-mq-dma: require unmap caller provide p2p map type
In reply to Keith BuschFrom: Keith Busch <[email protected]>
In preparing for integrity dma mappings, we can't rely on the request flag because data and metadata may have different mapping types.
--- drivers/nvme/host/pci.c | 9 ++++++++- include/linux/blk-mq-dma.h | 5 +++-- 2 files changed, 11 insertions(+), 3 deletions(-)
drivers/nvme/host/pci.c +8 -1
@@ -261,6 +261,9 @@ enum nvme_iod_flags {/* single segment dma mapping */ IOD_SINGLE_SEGMENT =1U <<2 , + +/* DMA mapped with PCI_P2PDMA_MAP_BUS_ADDR */ + IOD_P2P_BUS_ADDR = 1U << 3, }; struct nvme_dma_vec {
@@ -725,7 +728,8 @@ static void nvme_unmap_data(struct request *req)return ; } -if (!blk_rq_dma_unmap (req, dma_dev, &iod->dma_state, iod->total_len)) { +if (!blk_rq_dma_unmap (req, dma_dev, &iod->dma_state, iod->total_len, + iod->flags &IOD_P2P_BUS_ADDR )) {if (nvme_pci_cmd_use_sgl (&iod->cmd))nvme_free_sgls (req);else
@@ -1000,6 +1004,9 @@ static blk_status_t nvme_map_data(struct request *req)if (!blk_rq_dma_map_iter_start (req, dev->dev, &iod->dma_state, &iter))return iter.status; +if (iter.p2pdma.map ==PCI_P2PDMA_MAP_BUS_ADDR ) + iod->flags |=IOD_P2P_BUS_ADDR ; +if (use_sgl ==SGL_FORCED || (use_sgl ==SGL_SUPPORTED && (sgl_threshold &&nvme_pci_avg_seg_size (req) >= sgl_threshold)))
include/linux/blk-mq-dma.h +3 -2
@@ -47,14 +47,15 @@ static inline bool blk_rq_dma_map_coalesce(struct dma_iova_state *state)* @dma_dev: device to unmap from * @state: DMA IOVA state * @mapped_len: number of bytes to unmap +* @is_p2p: true if mapped with PCI_P2PDMA_MAP_BUS_ADDR * * Returns %false if the callers need to manually unmap every DMA segment * mapped using @iter or %true if no work is left to be done. */ static inline bool blk_rq_dma_unmap (struct request *req,struct device *dma_dev, -struct dma_iova_state *state,size_t mapped_len) +struct dma_iova_state *state,size_t mapped_len,bool is_p2p) { -if (req->cmd_flags &REQ_P2PDMA ) +if (is_p2p)return true;if (dma_use_iova (state)) {
Signature · 1 line
2.47.3
PATCH v6 4/8 blk-mq: remove REQ_P2PDMA flag
In reply to Keith BuschFrom: Keith Busch <[email protected]>
It's not serving any particular purpose. pci_p2pdma_state() already has all the appropriate checks, so the config and flag checks are not guarding anything.
--- block/bio.c | 2 +- block/blk-mq-dma.c | 30 ++++++++++++++---------------- include/linux/blk_types.h | 2 -- 3 files changed, 15 insertions(+), 19 deletions(-)
block/bio.c +1 -1
@@ -981,7 +981,7 @@ void __bio_add_page(struct bio *bio, struct page *page,WARN_ON_ONCE (bio_full (bio, len));if (is_pci_p2pdma_page (page)) - bio->bi_opf |=REQ_P2PDMA |REQ_NOMERGE ; + bio->bi_opf |=REQ_NOMERGE ;bvec_set_page (&bio->bi_io_vec[bio->bi_vcnt], page, len, off); bio->bi_iter.bi_size += len;
block/blk-mq-dma.c +14 -16
@@ -180,22 +180,20 @@ bool blk_rq_dma_map_iter_start(struct request *req, struct device *dma_dev,if (!blk_map_iter_next (req, &iter->iter, &vec))return false; -if (IS_ENABLED (CONFIG_PCI_P2PDMA ) && (req->cmd_flags &REQ_P2PDMA )) { -switch (pci_p2pdma_state (&iter->p2pdma, dma_dev, -phys_to_page (vec.paddr))) { -case PCI_P2PDMA_MAP_BUS_ADDR : -return blk_dma_map_bus (iter, &vec); -case PCI_P2PDMA_MAP_THRU_HOST_BRIDGE : -/* -* P2P transfers through the host bridge are treated the -* same as non-P2P transfers below and during unmap. -*/ - req->cmd_flags &= ~REQ_P2PDMA ; -break ; -default : - iter->status =BLK_STS_INVAL ; -return false; - } +switch (pci_p2pdma_state (&iter->p2pdma, dma_dev, +phys_to_page (vec.paddr))) { +case PCI_P2PDMA_MAP_BUS_ADDR : +return blk_dma_map_bus (iter, &vec); +case PCI_P2PDMA_MAP_THRU_HOST_BRIDGE : +/* +* P2P transfers through the host bridge are treated the +* same as non-P2P transfers below and during unmap. +*/ +case PCI_P2PDMA_MAP_NONE : +break ; +default : + iter->status =BLK_STS_INVAL ; +return false; } if (blk_can_dma_map_iova(req, dma_dev) &&
include/linux/blk_types.h +0 -2
@@ -386,7 +386,6 @@ enum req_flag_bits { __REQ_DRV,/* for driver use */ __REQ_FS_PRIVATE,/* for file system (submitter) use */ __REQ_ATOMIC, /* for atomic write operations */ - __REQ_P2PDMA, /* contains P2P DMA pages *//* * Command specific flags, keep last: */
@@ -419,7 +418,6 @@ enum req_flag_bits {#define REQ_DRV (__force blk_opf_t)(1ULL << __REQ_DRV)#define REQ_FS_PRIVATE (__force blk_opf_t)(1ULL << __REQ_FS_PRIVATE)#define REQ_ATOMIC (__force blk_opf_t)(1ULL << __REQ_ATOMIC) -#define REQ_P2PDMA (__force blk_opf_t)(1ULL << __REQ_P2PDMA)#define REQ_NOUNMAP (__force blk_opf_t)(1ULL << __REQ_NOUNMAP)
Signature · 1 line
2.47.3
PATCH v6 5/8 blk-mq-dma: move common dma start code to a helper
In reply to Keith BuschFrom: Keith Busch <[email protected]>
In preparing for dma mapping integrity metadata, move the common dma setup to a helper.
--- block/blk-mq-dma.c | 59 ++++++++++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 26 deletions(-)
block/blk-mq-dma.c +33 -26
@@ -141,35 +141,12 @@ static inline void blk_rq_map_iter_init(struct request *rq, } } -/** -* blk_rq_dma_map_iter_start - map the first DMA segment for a request -* @req: request to map -* @dma_dev: device to map to -* @state: DMA IOVA state -* @iter: block layer DMA iterator -* -* Start DMA mapping @req to @dma_dev. @state and @iter are provided by the -* caller and don't need to be initialized. @state needs to be stored for use -* at unmap time, @iter is only needed at map time. -* -* Returns %false if there is no segment to map, including due to an error, or -* %true ft it did map a segment. -* -* If a segment was mapped, the DMA address for it is returned in @iter.addr and -* the length in @iter.len. If no segment was mapped the status code is -* returned in @iter.status. -* -* The caller can call blk_rq_dma_map_coalesce() to check if further segments -* need to be mapped after this, or go straight to blk_rq_dma_map_iter_next() -* to try to map the following segments. -*/ -bool blk_rq_dma_map_iter_start (struct request *req,struct device *dma_dev, -struct dma_iova_state *state,struct blk_dma_iter *iter) +static bool blk_dma_map_iter_start (struct request *req,struct device *dma_dev, +struct dma_iova_state *state,struct blk_dma_iter *iter, +unsigned int total_len) { -unsigned int total_len =blk_rq_payload_bytes (req);struct phys_vec vec; -blk_rq_map_iter_init (req, &iter->iter);memset (&iter->p2pdma,0 ,sizeof (iter->p2pdma)); iter->status =BLK_STS_OK ;
@@ -201,6 +178,36 @@ bool blk_rq_dma_map_iter_start(struct request *req, struct device *dma_dev,return blk_rq_dma_map_iova (req, dma_dev, state, iter, &vec);return blk_dma_map_direct (req, dma_dev, iter, &vec); } + +/** +* blk_rq_dma_map_iter_start - map the first DMA segment for a request +* @req: request to map +* @dma_dev: device to map to +* @state: DMA IOVA state +* @iter: block layer DMA iterator +* +* Start DMA mapping @req to @dma_dev. @state and @iter are provided by the +* caller and don't need to be initialized. @state needs to be stored for use +* at unmap time, @iter is only needed at map time. +* +* Returns %false if there is no segment to map, including due to an error, or +* %true ft it did map a segment. +* +* If a segment was mapped, the DMA address for it is returned in @iter.addr and +* the length in @iter.len. If no segment was mapped the status code is +* returned in @iter.status. +* +* The caller can call blk_rq_dma_map_coalesce() to check if further segments +* need to be mapped after this, or go straight to blk_rq_dma_map_iter_next() +* to try to map the following segments. +*/ +bool blk_rq_dma_map_iter_start (struct request *req,struct device *dma_dev, +struct dma_iova_state *state,struct blk_dma_iter *iter) +{ +blk_rq_map_iter_init (req, &iter->iter); +return blk_dma_map_iter_start (req, dma_dev, state, iter, +blk_rq_payload_bytes (req)); +}EXPORT_SYMBOL_GPL (blk_rq_dma_map_iter_start);/**
Signature · 1 line
2.47.3
PATCH v6 6/8 blk-mq-dma: add support for mapping integrity metadata
In reply to Keith BuschFrom: Keith Busch <[email protected]>
Provide integrity metadata helpers equivalent to the data payload helpers for iterating a request for dma setup.
--- block/blk-integrity.c | 43 +++++------- block/blk-mq-dma.c | 120 +++++++++++++++++++++++++++------- block/blk-mq.h | 26 ++++++++ include/linux/blk-integrity.h | 17 +++++ include/linux/blk-mq-dma.h | 1 + 5 files changed, 157 insertions(+), 50 deletions(-)
block/blk-integrity.c +16 -27
@@ -16,6 +16,7 @@#include <linux/t10-pi.h> #include "blk.h" +#include "blk-mq.h" /** * blk_rq_count_integrity_sg - Count number of integrity scatterlist elements
@@ -134,37 +135,25 @@ int blk_get_meta_cap(struct block_device *bdev, unsigned int cmd,*/ int blk_rq_map_integrity_sg (struct request *rq,struct scatterlist *sglist) { -struct bio_vec iv, ivprv = {NULL };struct request_queue *q = rq->q;struct scatterlist *sg =NULL ;struct bio *bio = rq->bio;unsigned int segments =0 ; -struct bvec_iter iter; -int prev =0 ; - -bio_for_each_integrity_vec (iv, bio, iter) { -if (prev) { -if (!biovec_phys_mergeable (q, &ivprv, &iv)) - goto new_segment; -if (sg->length + iv.bv_len >queue_max_segment_size (q)) - goto new_segment; - - sg->length += iv.bv_len; - }else { -new_segment: -if (!sg) - sg = sglist; -else { -sg_unmark_end (sg); - sg =sg_next (sg); - } - -sg_set_page (sg, iv.bv_page, iv.bv_len, iv.bv_offset); - segments++; - } - - prev =1 ; - ivprv = iv; +struct blk_map_iter iter; +struct phys_vec vec; + + iter = (struct blk_map_iter ) { + .bio = bio, + .iter =bio_integrity (bio)->bip_iter, + .bvecs =bio_integrity (bio)->bip_vec, + .is_integrity = true, + }; + +while (blk_map_iter_next (rq, &iter, &vec)) { + sg =blk_next_sg (&sg, sglist); +sg_set_page (sg,phys_to_page (vec.paddr), vec.len, +offset_in_page (vec.paddr)); + segments++; }if (sg)
block/blk-mq-dma.c +97 -23
139 lines not on this page · raw message
block/blk-mq.h +26 -0
@@ -449,4 +449,30 @@ static inline bool blk_mq_can_poll(struct request_queue *q) q->tag_set->map[HCTX_TYPE_POLL].nr_queues; } +struct phys_vec { +phys_addr_t paddr; +u32 len; +}; + +struct blk_map_iter ; +bool blk_map_iter_next (struct request *req,struct blk_map_iter *iter, +struct phys_vec *vec); + +static inline struct scatterlist * +blk_next_sg (struct scatterlist **sg,struct scatterlist *sglist) +{ +if (!*sg) +return sglist; + +/* +* If the driver previously mapped a shorter list, we could see a +* termination bit prematurely unless it fully inits the sg table +* on each mapping. We KNOW that there must be more entries here +* or the driver would be buggy, so force clear the termination bit +* to avoid doing a full sg_init_table() in drivers for each command. +*/ +sg_unmark_end (*sg); +return sg_next (*sg); +} +#endif
include/linux/blk-integrity.h +17 -0
@@ -4,6 +4,7 @@#include <linux/blk-mq.h> #include <linux/bio-integrity.h> +#include <linux/blk-mq-dma.h> struct request ;
@@ -31,6 +32,11 @@ int blk_rq_integrity_map_user(struct request *rq, void __user *ubuf, ssize_t bytes);int blk_get_meta_cap (struct block_device *bdev,unsigned int cmd,struct logical_block_metadata_cap __user *argp); +bool blk_rq_integrity_dma_map_iter_start (struct request *req, +struct device *dma_dev,struct dma_iova_state *state, +struct blk_dma_iter *iter); +bool blk_rq_integrity_dma_map_iter_next (struct request *req, +struct device *dma_dev,struct blk_dma_iter *iter);static inline bool blk_integrity_queue_supports_integrity (struct request_queue *q)
@@ -115,6 +121,17 @@ static inline int blk_rq_integrity_map_user(struct request *rq, {return -EINVAL ; } +static inline bool blk_rq_integrity_dma_map_iter_start (struct request *req, +struct device *dma_dev,struct dma_iova_state *state, +struct blk_dma_iter *iter) +{ +return false; +} +static inline bool blk_rq_integrity_dma_map_iter_next (struct request *req, +struct device *dma_dev,struct blk_dma_iter *iter) +{ +return false; +}static inline struct blk_integrity *bdev_get_integrity (struct block_device *b) {return NULL ;
include/linux/blk-mq-dma.h +1 -0
@@ -9,6 +9,7 @@ struct blk_map_iter {struct bvec_iter iter;struct bio *bio;struct bio_vec *bvecs; +bool is_integrity; };struct blk_dma_iter {
Signature · 1 line
2.47.3
kernel test robot
build botRe: [PATCHv6 6/8] blk-mq-dma: add support for mapping integrity metadata
kernel test robot noticed the following build warnings:
Hi Keith,
kernel test robot noticed the following build warnings:
[auto build test WARNING on axboe-block/for-next]
[also build test WARNING on linus/master v6.17-rc1 next-20250812] [cannot apply to linux-nvme/for-next]
[If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Keith-Busch/blk-mq-dma-create-blk_map_iter-type/20250812-222130 base: https://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux-block.git for-next patch link: https://lore.kernel.org/r/20250812135210.4172178-7-kbusch%40meta.com patch subject: [PATCHv6 6/8] blk-mq-dma: add support for mapping integrity metadata config: csky-randconfig-001-20250813 (https://download.01.org/0day-ci/archive/20250813/[email protected]/config) compiler: csky-linux-gcc (GCC) 14.3.0 reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20250813/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All warnings (new ones prefixed by >>):
>> block/blk-mq-dma.c:10:13: warning: '__blk_map_iter_next' defined but not used [-Wunused-function]
10 | static bool __blk_map_iter_next(struct blk_map_iter *iter)
| ^~~~~~~~~~~~~~~~~~~
vim +/__blk_map_iter_next +10 block/blk-mq-dma.c
9
> 10 static bool __blk_map_iter_next(struct blk_map_iter *iter)
11 {
12 if (iter->iter.bi_size)
13 return true;
14 if (!iter->bio || !iter->bio->bi_next)
15 return false;
16
17 iter->bio = iter->bio->bi_next;
18 if (iter->is_integrity) {
19 iter->iter = bio_integrity(iter->bio)->bip_iter;
20 iter->bvecs = bio_integrity(iter->bio)->bip_vec;
21 } else {
22 iter->iter = iter->bio->bi_iter;
23 iter->bvecs = iter->bio->bi_io_vec;
24 }
25 return true;
26 }
27 Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki
Christoph Hellwig
maintainerRe: [PATCHv6 6/8] blk-mq-dma: add support for mapping integrity metadata
In reply to Keith BuschQuoted 4 lines · Keith Busch
From: Keith Busch <[email protected]>
Provide integrity metadata helpers equivalent to the data payload helpers for iterating a request for dma setup.
This actually does two things:
1) convert the existing SG helpers
2) add the new helpers
Which probably should be split into a separate patches, and all of them could use a more detailed commit log.
block/blk-integrity.c int blk_get_meta_cap(struct block_device *bdev, unsigned int cmd, · line 150
Quoted 9 lines
block/blk-integrity.c +9
-0+struct blk_map_iter iter; +struct phys_vec vec; + + iter = (struct blk_map_iter ) { + .bio = bio, + .iter =bio_integrity (bio)->bip_iter, + .bvecs =bio_integrity (bio)->bip_vec, + .is_integrity = true, + };
Just initialize iter at declaration time:
Quoted 14 lines
block/blk-mq-dma.c +13
-0+struct blk_map_iter iter = { + .bio = bio, + .iter =bio_integrity (bio)->bip_iter, + .bvecs =bio_integrity (bio)->bip_vec, + .is_integrity = true, + }; +static bool __blk_map_iter_next (struct blk_map_iter *iter) +{ +if (iter->iter.bi_size) +return true; +if (!iter->bio || !iter->bio->bi_next) +return false; +
block/blk-mq-dma.c line 26
Quoted 10 lines
block/blk-mq-dma.c +10
-0+ iter->bio = iter->bio->bi_next; +if (iter->is_integrity) { + iter->iter =bio_integrity (iter->bio)->bip_iter; + iter->bvecs =bio_integrity (iter->bio)->bip_vec; + }else { + iter->iter = iter->bio->bi_iter; + iter->bvecs = iter->bio->bi_io_vec; + } +return true; +}
This seems unused, even with all patches applied.
Quoted 6 lines
block/blk-mq-dma.c +0
-6-static inline struct scatterlist * -blk_next_sg (struct scatterlist **sg,struct scatterlist *sglist) -{ -if (!*sg) -return sglist; -
block/blk-mq-dma.c bool blk_rq_dma_map_iter_next(struct request *req, struct device *dma_dev, · line 260
Quoted 10 lines
block/blk-mq-dma.c +0
-10-/* -* If the driver previously mapped a shorter list, we could see a -* termination bit prematurely unless it fully inits the sg table -* on each mapping. We KNOW that there must be more entries here -* or the driver would be buggy, so force clear the termination bit -* to avoid doing a full sg_init_table() in drivers for each command. -*/ -sg_unmark_end (*sg); -return sg_next (*sg); -}
Maybe just add the metadata mapping helpers into this file instead of moving stuff around?
block/blk-mq.h static inline bool blk_mq_can_poll(struct request_queue *q) · line 455
Quoted 10 lines
block/blk-mq.h +4
-0@@ -449,4 +449,30 @@ static inline bool blk_mq_can_poll(struct request_queue *q) q->tag_set->map[HCTX_TYPE_POLL ].nr_queues; } +struct phys_vec { +phys_addr_t paddr; +u32 len; +};
Leon has a patch to move this to linux/types.h, which if we have to move it is probably a better place.
Otherwise this looks good.
PATCH v6 7/8 nvme-pci: create common sgl unmapping helper
In reply to Keith BuschFrom: Keith Busch <[email protected]>
This can be reused by metadata sgls once that starts using the blk-mq dma api.
--- drivers/nvme/host/pci.c | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-)
drivers/nvme/host/pci.c +22 -14
@@ -693,25 +693,33 @@ static void nvme_free_prps(struct request *req)mempool_free (iod->dma_vecs, nvmeq->dev->dmavec_mempool); } -static void nvme_free_sgls (struct request *req) + +static void __nvme_free_sgls (struct request *req,struct nvme_sgl_desc *sge, +struct nvme_sgl_desc *sg_list) { -struct nvme_iod *iod =blk_mq_rq_to_pdu (req);struct nvme_queue *nvmeq = req->mq_hctx->driver_data; -struct device *dma_dev = nvmeq->dev->dev; -dma_addr_t sqe_dma_addr =le64_to_cpu (iod->cmd.common.dptr.sgl.addr); -unsigned int sqe_dma_len =le32_to_cpu (iod->cmd.common.dptr.sgl.length); -struct nvme_sgl_desc *sg_list = iod->descriptors[0 ];enum dma_data_direction dir =rq_dma_dir (req); +unsigned int len =le32_to_cpu (sge->length); +struct device *dma_dev = nvmeq->dev->dev; +unsigned int i; -if (iod->nr_descriptors) { -unsigned int nr_entries = sqe_dma_len /sizeof (*sg_list), i; - -for (i =0 ; i < nr_entries; i++) -dma_unmap_page (dma_dev,le64_to_cpu (sg_list[i].addr), -le32_to_cpu (sg_list[i].length), dir); - }else { -dma_unmap_page (dma_dev, sqe_dma_addr, sqe_dma_len, dir); +if (sge->type == (NVME_SGL_FMT_DATA_DESC <<4 )) { +dma_unmap_page (dma_dev,le64_to_cpu (sge->addr), len, dir); +return ; } + +for (i =0 ; i < len /sizeof (*sg_list); i++) +dma_unmap_page (dma_dev,le64_to_cpu (sg_list[i].addr), +le32_to_cpu (sg_list[i].length), dir); +} + +static void nvme_free_sgls (struct request *req) +{ +struct nvme_iod *iod =blk_mq_rq_to_pdu (req); +struct nvme_sgl_desc *sg_list = iod->descriptors[0 ]; +struct nvme_sgl_desc *sge = &iod->cmd.common.dptr.sgl; + +__nvme_free_sgls (req, sge, sg_list); }static void nvme_unmap_data (struct request *req)
Signature · 1 line
2.47.3
Re: [PATCHv6 7/8] nvme-pci: create common sgl unmapping helper
drivers/nvme/host/pci.c static void nvme_free_prps(struct request *req) · line 722
Quoted 7 lines · Keith Busch
drivers/nvme/host/pci.c +7
-0+static void nvme_free_sgls (struct request *req) +{ +struct nvme_iod *iod =blk_mq_rq_to_pdu (req); +struct nvme_sgl_desc *sg_list = iod->descriptors[0 ]; +struct nvme_sgl_desc *sge = &iod->cmd.common.dptr.sgl; + +__nvme_free_sgls (req, sge, sg_list);
No real need for the local variables for sg_list and sge.
And I'd just kill this function and rename __nvme_free_sgls to nvme_free_sgls. But if you think that's just noise right now I'd be happy to do a clean up pass later.
PATCH v6 8/8 nvme-pci: convert metadata mapping to dma iter
In reply to Keith BuschFrom: Keith Busch <[email protected]>
Aligns data and metadata to the similar dma mapping scheme and removes one more user of the scatter-gather dma mapping.
--- drivers/nvme/host/pci.c | 170 ++++++++++++++++++++++------------------ 1 file changed, 94 insertions(+), 76 deletions(-)
drivers/nvme/host/pci.c +94 -76
@@ -172,9 +172,7 @@ struct nvme_dev {u32 last_ps;bool hmb;struct sg_table *hmb_sgt; -mempool_t *dmavec_mempool; -mempool_t *iod_meta_mempool;/* shadow doorbell buffer support: */ __le32 *dbbuf_dbs;
@@ -264,6 +262,12 @@ enum nvme_iod_flags {/* DMA mapped with PCI_P2PDMA_MAP_BUS_ADDR */ IOD_P2P_BUS_ADDR =1U <<3 , + +/* Metadata DMA mapped with PCI_P2PDMA_MAP_BUS_ADDR */ +IOD_META_P2P_BUS_ADDR =1U <<4 , + +/* Metadata using non-coalesced MPTR */ + IOD_SINGLE_META_SEGMENT = 1U << 5, }; struct nvme_dma_vec {
@@ -287,7 +291,8 @@ struct nvme_iod {unsigned int nr_dma_vecs;dma_addr_t meta_dma; -struct sg_table meta_sgt; +unsigned int meta_total_len; +struct dma_iova_state meta_dma_state;struct nvme_sgl_desc *meta_descriptor; };
@@ -644,6 +649,11 @@ static inline struct dma_pool *nvme_dma_pool(struct nvme_queue *nvmeq,return nvmeq->descriptor_pools.large; } +static inline bool nvme_pci_cmd_use_meta_sgl (struct nvme_command *cmd) +{ +return (cmd->common.flags &NVME_CMD_SGL_ALL ) ==NVME_CMD_SGL_METASEG ; +} +static inline bool nvme_pci_cmd_use_sgl (struct nvme_command *cmd) { return cmd->common.flags &
@@ -713,6 +723,43 @@ static void __nvme_free_sgls(struct request *req, struct nvme_sgl_desc *sge, le32_to_cpu(sg_list[i].length), dir); } +static void nvme_free_meta_sgls (struct request *req) +{ +struct nvme_iod *iod =blk_mq_rq_to_pdu (req); +struct nvme_sgl_desc *sge = iod->meta_descriptor; + +__nvme_free_sgls (req, sge, &sge[1 ]); +} + +static void nvme_unmap_metadata (struct request *req) +{ +struct nvme_queue *nvmeq = req->mq_hctx->driver_data; +enum dma_data_direction dir =rq_dma_dir (req); +struct nvme_iod *iod =blk_mq_rq_to_pdu (req); +struct device *dma_dev = nvmeq->dev->dev; + +if (iod->flags &IOD_SINGLE_META_SEGMENT ) { +dma_unmap_page (dma_dev, iod->meta_dma, +rq_integrity_vec (req).bv_len, +rq_dma_dir (req)); +return ; + } + +if (!blk_rq_dma_unmap (req, dma_dev, &iod->meta_dma_state, + iod->meta_total_len, + iod->flags &IOD_META_P2P_BUS_ADDR )) { +if (nvme_pci_cmd_use_meta_sgl (&iod->cmd)) +nvme_free_meta_sgls (req); +else +dma_unmap_page (dma_dev, iod->meta_dma, + iod->meta_total_len, dir); + } + +if (iod->meta_descriptor) +dma_pool_free (nvmeq->descriptor_pools.small, + iod->meta_descriptor, iod->meta_dma); +} +static void nvme_free_sgls (struct request *req) {struct nvme_iod *iod =blk_mq_rq_to_pdu (req);
@@ -1022,70 +1069,72 @@ static blk_status_t nvme_map_data(struct request *req)return nvme_pci_setup_data_prp (req, &iter); } -static void nvme_pci_sgl_set_data_sg (struct nvme_sgl_desc *sge, -struct scatterlist *sg) -{ - sge->addr =cpu_to_le64 (sg_dma_address (sg)); - sge->length =cpu_to_le32 (sg_dma_len (sg)); - sge->type =NVME_SGL_FMT_DATA_DESC <<4 ; -} -static blk_status_t nvme_pci_setup_meta_sgls (struct request *req) {struct nvme_queue *nvmeq = req->mq_hctx->driver_data; -struct nvme_dev *dev = nvmeq->dev; +unsigned int entries = req->nr_integrity_segments;struct nvme_iod *iod =blk_mq_rq_to_pdu (req); +struct nvme_dev *dev = nvmeq->dev;struct nvme_sgl_desc *sg_list; -struct scatterlist *sgl, *sg; -unsigned int entries; +struct blk_dma_iter iter;dma_addr_t sgl_dma; -int rc, i; +int i =0 ; - iod->meta_sgt.sgl =mempool_alloc (dev->iod_meta_mempool,GFP_ATOMIC ); -if (!iod->meta_sgt.sgl) -return BLK_STS_RESOURCE ; +if (!blk_rq_integrity_dma_map_iter_start (req, dev->dev, + &iod->meta_dma_state, &iter)) +return iter.status; -sg_init_table (iod->meta_sgt.sgl, req->nr_integrity_segments); - iod->meta_sgt.orig_nents =blk_rq_map_integrity_sg (req, - iod->meta_sgt.sgl); -if (!iod->meta_sgt.orig_nents) - goto out_free_sg; +if (iter.p2pdma.map ==PCI_P2PDMA_MAP_BUS_ADDR ) + iod->flags |=IOD_META_P2P_BUS_ADDR ; +else if (blk_rq_dma_map_coalesce (&iod->meta_dma_state)) + entries =1 ; - rc =dma_map_sgtable (dev->dev, &iod->meta_sgt,rq_dma_dir (req), -DMA_ATTR_NO_WARN ); -if (rc) - goto out_free_sg; +/* +* The NVMe MPTR descriptor has an implicit length that the host and +* device must agree on to avoid data/memory corruption. We trust the +* kernel allocated correctly based on the format's parameters, so use +* the more efficient MPTR to avoid extra dma pool allocations for the +* SGL indirection. +* +* But for user commands, we don't necessarily know what they do, so +* the driver can't validate the metadata buffer size. The SGL +* descriptor provides an explicit length, so we're relying on that +* mechanism to catch any misunderstandings between the application and +* device. +*/ +if (entries ==1 && !(nvme_req (req)->flags &NVME_REQ_USERCMD )) { + iod->cmd.common.metadata =cpu_to_le64 (iter.addr); + iod->meta_total_len = iter.len; + iod->meta_dma = iter.addr; + iod->meta_descriptor =NULL ; +return BLK_STS_OK ; + } sg_list =dma_pool_alloc (nvmeq->descriptor_pools.small,GFP_ATOMIC , &sgl_dma);if (!sg_list) - goto out_unmap_sg; +return BLK_STS_RESOURCE ; - entries = iod->meta_sgt.nents; iod->meta_descriptor = sg_list; iod->meta_dma = sgl_dma; - iod->cmd.common.flags =NVME_CMD_SGL_METASEG ; iod->cmd.common.metadata =cpu_to_le64 (sgl_dma); - - sgl = iod->meta_sgt.sgl;if (entries ==1 ) { -nvme_pci_sgl_set_data_sg (sg_list, sgl); + iod->meta_total_len = iter.len; +nvme_pci_sgl_set_data (sg_list, &iter);return BLK_STS_OK ; } sgl_dma +=sizeof (*sg_list); -nvme_pci_sgl_set_seg (sg_list, sgl_dma, entries); -for_each_sg (sgl, sg, entries, i) -nvme_pci_sgl_set_data_sg (&sg_list[i +1 ], sg); - -return BLK_STS_OK ; +do { +nvme_pci_sgl_set_data (&sg_list[++i], &iter); + iod->meta_total_len += iter.len; + }while (blk_rq_integrity_dma_map_iter_next (req, dev->dev, &iter)); -out_unmap_sg: -dma_unmap_sgtable (dev->dev, &iod->meta_sgt,rq_dma_dir (req),0 ); -out_free_sg: -mempool_free (iod->meta_sgt.sgl, dev->iod_meta_mempool); -return BLK_STS_RESOURCE ; +nvme_pci_sgl_set_seg (sg_list, sgl_dma, i); +if (unlikely (iter.status)) +nvme_unmap_metadata (req); +return iter.status; }static blk_status_t nvme_pci_setup_meta_mptr (struct request *req)
@@ -1098,6 +1147,7 @@ static blk_status_t nvme_pci_setup_meta_mptr(struct request *req)if (dma_mapping_error (nvmeq->dev->dev, iod->meta_dma))return BLK_STS_IOERR ; iod->cmd.common.metadata =cpu_to_le64 (iod->meta_dma); + iod->flags |=IOD_SINGLE_META_SEGMENT ;return BLK_STS_OK ; }
@@ -1119,7 +1169,7 @@ static blk_status_t nvme_prep_rq(struct request *req) iod->flags =0 ; iod->nr_descriptors =0 ; iod->total_len =0 ; - iod->meta_sgt.nents =0 ; + iod->meta_total_len =0 ; ret =nvme_setup_cmd (req->q->queuedata, req);if (ret)
@@ -1230,25 +1280,6 @@ static void nvme_queue_rqs(struct rq_list *rqlist) *rqlist = requeue_list; } -static __always_inline void nvme_unmap_metadata(struct request *req) -{ -struct nvme_iod *iod =blk_mq_rq_to_pdu (req); -struct nvme_queue *nvmeq = req->mq_hctx->driver_data; -struct nvme_dev *dev = nvmeq->dev; - -if (!iod->meta_sgt.nents) { -dma_unmap_page (dev->dev, iod->meta_dma, -rq_integrity_vec (req).bv_len, -rq_dma_dir (req)); -return ; - } - -dma_pool_free (nvmeq->descriptor_pools.small, iod->meta_descriptor, - iod->meta_dma); -dma_unmap_sgtable (dev->dev, &iod->meta_sgt,rq_dma_dir (req),0 ); -mempool_free (iod->meta_sgt.sgl, dev->iod_meta_mempool); -} - static __always_inline void nvme_pci_unmap_rq(struct request *req) {if (blk_integrity_rq (req))
@@ -3054,7 +3085,6 @@ static int nvme_disable_prepare_reset(struct nvme_dev *dev, bool shutdown)static int nvme_pci_alloc_iod_mempool (struct nvme_dev *dev) { -size_t meta_size =sizeof (struct scatterlist ) * (NVME_MAX_META_SEGS +1 );size_t alloc_size =sizeof (struct nvme_dma_vec ) *NVME_MAX_SEGS ; dev->dmavec_mempool = mempool_create_node(1,
@@ -3063,17 +3093,7 @@ static int nvme_pci_alloc_iod_mempool(struct nvme_dev *dev) dev_to_node(dev->dev));if (!dev->dmavec_mempool)return -ENOMEM ; - - dev->iod_meta_mempool =mempool_create_node (1 , - mempool_kmalloc, mempool_kfree, - (void *)meta_size,GFP_KERNEL , -dev_to_node (dev->dev)); -if (!dev->iod_meta_mempool) - goto free;return 0 ; -free: -mempool_destroy (dev->dmavec_mempool); -return -ENOMEM ; }static void nvme_free_tagset (struct nvme_dev *dev)
@@ -3523,7 +3543,6 @@ static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)nvme_free_queues (dev,0 ); out_release_iod_mempool:mempool_destroy (dev->dmavec_mempool); -mempool_destroy (dev->iod_meta_mempool); out_dev_unmap:nvme_dev_unmap (dev); out_uninit_ctrl:
@@ -3587,7 +3606,6 @@ static void nvme_remove(struct pci_dev *pdev)nvme_dbbuf_dma_free (dev);nvme_free_queues (dev,0 );mempool_destroy (dev->dmavec_mempool); -mempool_destroy (dev->iod_meta_mempool);nvme_release_descriptor_pools (dev);nvme_dev_unmap (dev);nvme_uninit_ctrl (&dev->ctrl);
Signature · 1 line
2.47.3
Re: [PATCHv6 8/8] nvme-pci: convert metadata mapping to dma iter
Looks good: