more integrity cleanups v2
Subsystems (4)
Christoph Hellwig
maintainerHi Jens and Martin,
this series has more cleanups to the block layer integrity code. It splits the bio integrity APIs into their own header as they are only used by very few source files, cleans up their stubs a little bit, and then in the last patch change when the bio_integrity_payload is freed when it is not owned by the block layer. This avoids having to know the submitter in the core code and will simplify adding other consuer of the API like file systems or the io_uring non-passthrough PI support.
v2 has been rebased to the block for-next branch as there are conflicting changes in 6.10-rc but not in the for-6.11/block branch.
Diffstat:
block/bio-integrity.c | 87 ++++++++--------------- block/bio.c | 2 block/blk-map.c | 3 block/blk.h | 14 +++ block/bounce.c | 2 drivers/md/dm.c | 1 drivers/nvme/host/ioctl.c | 16 +--- drivers/scsi/sd.c | 3 include/linux/bio-integrity.h | 153 +++++++++++++++++++++++++++++++++++++++++ include/linux/bio.h | 156 ------------------------------------------ include/linux/blk-integrity.h | 1 11 files changed, 211 insertions(+), 227 deletions(-)
Christoph Hellwig
maintainerPATCH 1/5 block: split integrity support out of bio.h
Split struct bio_integrity_payload and the related prototypes out of bio.h into a separate bio-integrity.h header so that it is only pulled in by the few places that need it.
--- block/bio.c | 2 +- block/blk.h | 1 + block/bounce.c | 2 +- drivers/md/dm.c | 1 + drivers/nvme/host/ioctl.c | 1 + drivers/scsi/sd.c | 3 +- include/linux/bio-integrity.h | 153 +++++++++++++++++++++++++++++++++ include/linux/bio.h | 156 ---------------------------------- include/linux/blk-integrity.h | 1 + 9 files changed, 161 insertions(+), 159 deletions(-) create mode 100644 include/linux/bio-integrity.h
block/bio.c +1 -1
@@ -4,7 +4,7 @@*/ #include <linux/mm.h> #include <linux/swap.h> -#include <linux/bio.h> +#include <linux/bio-integrity.h> #include <linux/blkdev.h> #include <linux/uio.h> #include <linux/iocontext.h>
block/blk.h +1 -0
@@ -2,6 +2,7 @@#ifndef BLK_INTERNAL_H #define BLK_INTERNAL_H +#include <linux/bio-integrity.h> #include <linux/blk-crypto.h> #include <linux/memblock.h> /* for max_pfn/max_low_pfn */ #include <linux/sched/sysctl.h>
block/bounce.c +1 -1
@@ -10,7 +10,7 @@#include <linux/export.h> #include <linux/swap.h> #include <linux/gfp.h> -#include <linux/bio.h> +#include <linux/bio-integrity.h> #include <linux/pagemap.h> #include <linux/mempool.h> #include <linux/blkdev.h>
drivers/md/dm.c +1 -0
@@ -11,6 +11,7 @@#include "dm-uevent.h" #include "dm-ima.h" +#include <linux/bio-integrity.h> #include <linux/init.h> #include <linux/module.h> #include <linux/mutex.h>
drivers/nvme/host/ioctl.c +1 -0
@@ -3,6 +3,7 @@* Copyright (c) 2011-2014, Intel Corporation. * Copyright (c) 2017-2021 Christoph Hellwig. */ +#include <linux/bio-integrity.h> #include <linux/ptrace.h> /* for force_successful_syscall_return */ #include <linux/nvme_ioctl.h> #include <linux/io_uring/cmd.h>
drivers/scsi/sd.c +2 -1
@@ -33,11 +33,12 @@* than the level indicated above to trigger output. */ +#include <linux/bio-integrity.h> #include <linux/module.h> #include <linux/fs.h> #include <linux/kernel.h> #include <linux/mm.h> -#include <linux/bio.h> +#include <linux/bio-integrity.h> #include <linux/hdreg.h> #include <linux/errno.h> #include <linux/idr.h>
include/linux/bio-integrity.h new file +153 -0
153 lines not on this page · raw message
include/linux/bio.h +0 -156
168 lines not on this page · raw message
include/linux/blk-integrity.h +1 -0
@@ -3,6 +3,7 @@#define _LINUX_BLK_INTEGRITY_H#include <linux/blk-mq.h> +#include <linux/bio-integrity.h> struct request ;
Signature · 1 line
2.43.0
Re: [PATCH 1/5] block: split integrity support out of bio.h
Re: [PATCH 1/5] block: split integrity support out of bio.h
In reply to Christoph HellwigChristoph Hellwig
maintainerPATCH 2/5 block: also return bio_integrity_payload * from stubs
In reply to Christoph Hellwigstruct bio_integrity_payload is defined unconditionally. No need to return void * from bio_integrity() and bio_integrity_alloc().
--- include/linux/bio-integrity.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
include/linux/bio-integrity.h +3 -3
@@ -85,7 +85,7 @@ void bio_integrity_init(void);#else /* CONFIG_BLK_DEV_INTEGRITY */ -static inline void *bio_integrity (struct bio *bio) +static inline struct bio_integrity_payload *bio_integrity (struct bio *bio) {return NULL ; }
@@ -138,8 +138,8 @@ static inline bool bio_integrity_flagged(struct bio *bio, enum bip_flags flag)return false; } -static inline void *bio_integrity_alloc (struct bio *bio,gfp_t gfp, -unsigned int nr) +static inline struct bio_integrity_payload * +bio_integrity_alloc (struct bio *bio,gfp_t gfp,unsigned int nr) {return ERR_PTR (-EINVAL ); }
Signature · 1 line
2.43.0
Re: [PATCH 2/5] block: also return bio_integrity_payload * from stubs
Re: [PATCH 2/5] block: also return bio_integrity_payload * from stubs
In reply to Christoph HellwigChristoph Hellwig
maintainerPATCH 3/5 block: call bio_integrity_unmap_free_user from blk_rq_unmap_user
In reply to Christoph Hellwigblk_rq_unmap_user always maps user space pass-through request. If such a request has integrity data attached it must come from a user mapping as well. Call bio_integrity_unmap_free_user from blk_rq_unmap_user and remove the nvme_unmap_bio wrapper in the nvme driver.
--- block/bio-integrity.c | 1 - block/blk-map.c | 3 +++ drivers/nvme/host/ioctl.c | 15 ++++----------- 3 files changed, 7 insertions(+), 12 deletions(-)
block/bio-integrity.c +0 -1
@@ -174,7 +174,6 @@ void bio_integrity_unmap_free_user(struct bio *bio) bio->bi_integrity =NULL ; bio->bi_opf &= ~REQ_INTEGRITY ; } -EXPORT_SYMBOL (bio_integrity_unmap_free_user);/** * bio_integrity_add_page - Attach integrity metadata
block/blk-map.c +3 -0
@@ -757,6 +757,9 @@ int blk_rq_unmap_user(struct bio *bio)bio_release_pages (bio,bio_data_dir (bio) ==READ ); } +if (bio_integrity (bio)) +bio_integrity_unmap_free_user (bio); + next_bio = bio; bio = bio->bi_next;blk_mq_map_bio_put (next_bio);
drivers/nvme/host/ioctl.c +4 -11
@@ -112,13 +112,6 @@ static struct request *nvme_alloc_user_request(struct request_queue *q,return req; } -static void nvme_unmap_bio (struct bio *bio) -{ -if (bio_integrity (bio)) -bio_integrity_unmap_free_user (bio); -blk_rq_unmap_user (bio); -} -static int nvme_map_user_request (struct request *req,u64 ubuffer,unsigned bufflen,void __user *meta_buffer,unsigned meta_len,u32 meta_seed,struct io_uring_cmd *ioucmd,unsigned int flags)
@@ -165,7 +158,7 @@ static int nvme_map_user_request(struct request *req, u64 ubuffer, out_unmap:if (bio) -nvme_unmap_bio (bio); +blk_rq_unmap_user (bio); out:blk_mq_free_request (req);return ret;
@@ -203,7 +196,7 @@ static int nvme_submit_user_cmd(struct request_queue *q,if (result) *result =le64_to_cpu (nvme_req (req)->result.u64);if (bio) -nvme_unmap_bio (bio); +blk_rq_unmap_user (bio);blk_mq_free_request (req);if (effects)
@@ -414,7 +407,7 @@ static void nvme_uring_task_cb(struct io_uring_cmd *ioucmd,struct nvme_uring_cmd_pdu *pdu =nvme_uring_cmd_pdu (ioucmd);if (pdu->bio) -nvme_unmap_bio (pdu->bio); +blk_rq_unmap_user (pdu->bio);io_uring_cmd_done (ioucmd, pdu->status, pdu->result, issue_flags); }
@@ -440,7 +433,7 @@ static enum rq_end_io_ret nvme_uring_cmd_end_io(struct request *req,*/ if (blk_rq_is_poll (req)) {if (pdu->bio) -nvme_unmap_bio (pdu->bio); +blk_rq_unmap_user (pdu->bio);io_uring_cmd_iopoll_done (ioucmd, pdu->result, pdu->status); }else {io_uring_cmd_do_in_task_lazy (ioucmd, nvme_uring_task_cb);
Signature · 1 line
2.43.0
Re: [PATCH 3/5] block: call bio_integrity_unmap_free_user from blk_rq_unmap_user
Quoted 1 line · Christoph Hellwig
blk_rq_unmap_useralways maps user space pass-through request
always maps -> always unmaps?
Otherwise, looks good.
Re: [PATCH 3/5] block: call bio_integrity_unmap_free_user from blk_rq_unmap_user
In reply to Christoph HellwigApart from the typo that Kanchan mentioned, Looks good to me.
--
Anuj Gupta
Christoph Hellwig
maintainerPATCH 4/5 block: don't free submitter owned integrity payload on I/O completion
In reply to Christoph HellwigCurrently __bio_integrity_endio frees the integrity payload unless it is explicitly marked as user-mapped. This means in-kernel callers that allocate their own integrity payload never get to see it on I/O completion. The current two users don't need it as they just pre-mapped PI tuples received over the network, but this limits uses of integrity data lot.
Change bio_integrity_endio to call __bio_integrity_endio for block layer generated integrity data only, and leave freeing of submitter allocated integrity data to bio_uninit which also gets called from the final bio_put. This requires that unmapping user mapped or copied integrity data is now always done by the caller, and the special BIP_INTEGRITY_USER flag can go away.
--- block/bio-integrity.c | 57 ++++++++++++++--------------------- block/blk.h | 13 ++++++-- include/linux/bio-integrity.h | 3 +- 3 files changed, 34 insertions(+), 39 deletions(-)
block/bio-integrity.c +22 -35
@@ -22,9 +22,17 @@ void blk_flush_integrity(void)flush_workqueue (kintegrityd_wq); } -static void __bio_integrity_free (struct bio_set *bs, -struct bio_integrity_payload *bip) +/** +* bio_integrity_free - Free bio integrity payload +* @bio: bio containing bip to be freed +* +* Description: Free the integrity portion of a bio. +*/ +void bio_integrity_free (struct bio *bio) { +struct bio_integrity_payload *bip =bio_integrity (bio); +struct bio_set *bs = bio->bi_pool; +if (bs &&mempool_initialized (&bs->bio_integrity_pool)) {if (bip->bip_vec) bvec_free(&bs->bvec_integrity_pool, bip->bip_vec,
@@ -33,6 +41,8 @@ static void __bio_integrity_free(struct bio_set *bs, }else {kfree (bip); } + bio->bi_integrity =NULL ; + bio->bi_opf &= ~REQ_INTEGRITY ; }/**
@@ -86,7 +96,10 @@ struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio,return bip; err: -__bio_integrity_free (bs, bip); +if (bs &&mempool_initialized (&bs->bio_integrity_pool)) +mempool_free (bip, &bs->bio_integrity_pool); +else +kfree (bip);return ERR_PTR (-ENOMEM ); }EXPORT_SYMBOL (bio_integrity_alloc);
@@ -132,28 +145,6 @@ static void bio_integrity_unmap_user(struct bio_integrity_payload *bip)bio_integrity_unpin_bvec (bip->bip_vec, bip->bip_max_vcnt, dirty); } -/** -* bio_integrity_free - Free bio integrity payload -* @bio: bio containing bip to be freed -* -* Description: Used to free the integrity portion of a bio. Usually -* called from bio_free(). -*/ -void bio_integrity_free (struct bio *bio) -{ -struct bio_integrity_payload *bip =bio_integrity (bio); -struct bio_set *bs = bio->bi_pool; - -if (bip->bip_flags &BIP_INTEGRITY_USER ) -return ; -if (bip->bip_flags &BIP_BLOCK_INTEGRITY ) -kfree (bvec_virt (bip->bip_vec)); - -__bio_integrity_free (bs, bip); - bio->bi_integrity =NULL ; - bio->bi_opf &= ~REQ_INTEGRITY ; -} -/** * bio_integrity_unmap_free_user - Unmap and free bio user integrity payload * @bio: bio containing bip to be unmapped and freed
@@ -165,14 +156,9 @@ void bio_integrity_free(struct bio *bio)void bio_integrity_unmap_free_user (struct bio *bio) {struct bio_integrity_payload *bip =bio_integrity (bio); -struct bio_set *bs = bio->bi_pool; -if (WARN_ON_ONCE (!(bip->bip_flags &BIP_INTEGRITY_USER ))) -return ;bio_integrity_unmap_user (bip); -__bio_integrity_free (bs, bip); - bio->bi_integrity =NULL ; - bio->bi_opf &= ~REQ_INTEGRITY ; +bio_integrity_free (bio); }/**
@@ -273,7 +259,7 @@ static int bio_integrity_copy_user(struct bio *bio, struct bio_vec *bvec, goto free_bip; } - bip->bip_flags |=BIP_INTEGRITY_USER |BIP_COPY_USER ; + bip->bip_flags |=BIP_COPY_USER ; bip->bip_iter.bi_sector = seed; bip->bip_vcnt = nr_vecs;return 0 ;
@@ -294,7 +280,6 @@ static int bio_integrity_init_user(struct bio *bio, struct bio_vec *bvec,return PTR_ERR (bip);memcpy (bip->bip_vec, bvec, nr_vecs *sizeof (*bvec)); - bip->bip_flags |=BIP_INTEGRITY_USER ; bip->bip_iter.bi_sector = seed; bip->bip_iter.bi_size = len; bip->bip_vcnt = nr_vecs;
@@ -502,6 +487,8 @@ static void bio_integrity_verify_fn(struct work_struct *work)struct bio *bio = bip->bip_bio;blk_integrity_verify (bio); + +kfree (bvec_virt (bip->bip_vec));bio_integrity_free (bio);bio_endio (bio); }
@@ -522,13 +509,13 @@ bool __bio_integrity_endio(struct bio *bio)struct blk_integrity *bi =blk_get_integrity (bio->bi_bdev->bd_disk);struct bio_integrity_payload *bip =bio_integrity (bio); -if (bio_op (bio) ==REQ_OP_READ && !bio->bi_status && - (bip->bip_flags &BIP_BLOCK_INTEGRITY ) && bi->csum_type) { +if (bio_op (bio) ==REQ_OP_READ && !bio->bi_status && bi->csum_type) {INIT_WORK (&bip->bip_work, bio_integrity_verify_fn);queue_work (kintegrityd_wq, &bip->bip_work);return false; } +kfree (bvec_virt (bip->bip_vec));bio_integrity_free (bio);return true; }
block/blk.h +11 -2
@@ -202,11 +202,20 @@ static inline unsigned int blk_queue_get_max_sectors(struct request *rq)#ifdef CONFIG_BLK_DEV_INTEGRITY void blk_flush_integrity (void ); -bool __bio_integrity_endio (struct bio *);void bio_integrity_free (struct bio *bio); + +/* +* Integrity payloads can either be owned by the submitter, in which case +* bio_uninit will free them, or owned and generated by the block layer, +* in which case we'll verify them here (for reads) and free them before +* the bio is handed back to the submitted. +*/ +bool __bio_integrity_endio (struct bio *bio);static inline bool bio_integrity_endio (struct bio *bio) { -if (bio_integrity (bio)) +struct bio_integrity_payload *bip =bio_integrity (bio); + +if (bip && (bip->bip_flags &BIP_BLOCK_INTEGRITY ))return __bio_integrity_endio (bio);return true; }
include/linux/bio-integrity.h +1 -2
@@ -10,8 +10,7 @@ enum bip_flags {BIP_CTRL_NOCHECK =1 <<2 ,/* disable HBA integrity checking */ BIP_DISK_NOCHECK =1 <<3 ,/* disable disk integrity checking */ BIP_IP_CHECKSUM =1 <<4 ,/* IP checksum */ -BIP_INTEGRITY_USER =1 <<5 ,/* Integrity payload is user address */ - BIP_COPY_USER = 1 << 6, /* Kernel bounce buffer in use */ + BIP_COPY_USER = 1 << 5, /* Kernel bounce buffer in use */ }; struct bio_integrity_payload {
Signature · 1 line
2.43.0
Re: [PATCH 4/5] block: don't free submitter owned integrity payload on I/O completion
Quoted 4 lines · Christoph Hellwig
block/blk.h +4
-0+/* +* Integrity payloads can either be owned by the submitter, in which case +* bio_uninit will free them, or owned and generated by the block layer, +* in which case we'll verify them here (for reads) and free them before
block/blk.h static inline unsigned int blk_queue_get_max_sectors(struct request *rq) · line 219
Quoted 10 lines
block/blk.h +6
-1+* the bio is handed back to the submitted. +*/ +bool __bio_integrity_endio (struct bio *bio);static inline bool bio_integrity_endio (struct bio *bio) { -if (bio_integrity (bio)) +struct bio_integrity_payload *bip =bio_integrity (bio); + +if (bip && (bip->bip_flags &BIP_BLOCK_INTEGRITY ))return __bio_integrity_endio (bio);
The patch will cause regression for nvme-passthrough. For that completion order is:
(a) bio_endio()
(b) req->end_io
(c) blk_rq_unmap_user.
And current code ensures that integrity is freed explicitly only after
(a) and (b).
With the patch, integrity will get freed during (a) itself.
There are two places in bio_endio() that can free the integrity. It first calls bio_integrity_endio() - which is handled fine above. But it also calls bio_uninit() - which will free the integrity. We don't want that to happen before passthrough gets the chance to unpin/copy-back.
Christoph Hellwig
maintainerRe: [PATCH 4/5] block: don't free submitter owned integrity payload on I/O completion
Quoted 9 lines · Kanchan Joshi
The patch will cause regression for nvme-passthrough. For that completion order is:
(a)bio_endio()
(b)req->end_io
(c)blk_rq_unmap_user.And current code ensures that integrity is freed explicitly only after
(a) and (b).
With the patch, integrity will get freed during (a) itself.
It is supposed to be freed from (c), specifically from blk_mq_map_bio_put.
Quoted 5 lines
There are two places in
bio_endio()that can free the integrity. It first callsbio_integrity_endio()- which is handled fine above. But it also callsbio_uninit()- which will free the integrity. We don't want that to happen before passthrough gets the chance to unpin/copy-back.
But yes, that messed it up. I'm kinda curious why it didn't trip up during my testing of the passthrough metadata code. That bio_uninit in bio_endio is quite bogus and I'm a bit suprised it hasn't caught more errors - the reason why bio_uninit exists is specifically to deal with those on-stack or embedded into bigger structure bios.
Re: [PATCH 4/5] block: don't free submitter owned integrity payload on I/O completion
Quoted 15 lines · Christoph Hellwig
Quoted 9 lines · Kanchan Joshi
The patch will cause regression for nvme-passthrough. For that completion order is:
(a)bio_endio()
(b)req->end_io
(c)blk_rq_unmap_user.And current code ensures that integrity is freed explicitly only after
(a) and (b).
With the patch, integrity will get freed during (a) itself.It is supposed to be freed from (c), specifically from
blk_mq_map_bio_put.Quoted 1 line
Quoted 7 lines
Quoted 4 lines
There are two places in
bio_endio()that can free the integrity. It first callsbio_integrity_endio()- which is handled fine above. But it also callsbio_uninit()- which will free the integrity. We don't want that to happen before passthrough gets the chance to unpin/copy-back.But yes, that messed it up. I'm kinda curious why it didn't trip up during my testing of the passthrough metadata code.
Yeah, it didn't trigger any errors in my tests either. It's just that integrity vecs
were not unpinned during completion.
Christoph Hellwig
maintainerPATCH 5/5 block: don't free the integrity payload in bio_integrity_unmap_free_user
In reply to Christoph HellwigNow that the integrity payload is always freed in bio_uninit, don't bother freeing it a little earlier in bio_integrity_unmap_free_user. With that the separate bio_integrity_unmap_free_user can go away by just passing the bio to bio_integrity_unmap_user.
--- block/bio-integrity.c | 31 +++++++++++-------------------- block/blk-map.c | 2 +- include/linux/bio-integrity.h | 4 ++-- 3 files changed, 14 insertions(+), 23 deletions(-)
block/bio-integrity.c +11 -20
@@ -131,34 +131,25 @@ static void bio_integrity_uncopy_user(struct bio_integrity_payload *bip)bio_integrity_unpin_bvec (copy, nr_vecs, true); } -static void bio_integrity_unmap_user (struct bio_integrity_payload *bip) +/** +* bio_integrity_unmap_user - Unmap user integrity payload +* @bio: bio containing bip to be unmapped +* +* Unmap the user mapped integrity portion of a bio. +*/ +void bio_integrity_unmap_user (struct bio *bio) { -bool dirty =bio_data_dir (bip->bip_bio) ==READ ; +struct bio_integrity_payload *bip =bio_integrity (bio);if (bip->bip_flags &BIP_COPY_USER ) { -if (dirty) +if (bio_data_dir (bio) ==READ )bio_integrity_uncopy_user (bip);kfree (bvec_virt (bip->bip_vec));return ; } -bio_integrity_unpin_bvec (bip->bip_vec, bip->bip_max_vcnt, dirty); -} - -/** -* bio_integrity_unmap_free_user - Unmap and free bio user integrity payload -* @bio: bio containing bip to be unmapped and freed -* -* Description: Used to unmap and free the user mapped integrity portion of a -* bio. Submitter attaching the user integrity buffer is responsible for -* unmapping and freeing it during completion. -*/ -void bio_integrity_unmap_free_user (struct bio *bio) -{ -struct bio_integrity_payload *bip =bio_integrity (bio); - -bio_integrity_unmap_user (bip); -bio_integrity_free (bio); +bio_integrity_unpin_bvec (bip->bip_vec, bip->bip_max_vcnt, +bio_data_dir (bio) ==READ ); }/**
block/blk-map.c +1 -1
@@ -758,7 +758,7 @@ int blk_rq_unmap_user(struct bio *bio) }if (bio_integrity (bio)) -bio_integrity_unmap_free_user (bio); +bio_integrity_unmap_user (bio); next_bio = bio; bio = bio->bi_next;
include/linux/bio-integrity.h +2 -2
@@ -73,7 +73,7 @@ struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio, gfp_t gfp,int bio_integrity_add_page (struct bio *bio,struct page *page,unsigned int len,unsigned int offset);int bio_integrity_map_user (struct bio *bio,void __user *ubuf,ssize_t len,u32 seed); -void bio_integrity_unmap_free_user (struct bio *bio); +void bio_integrity_unmap_user (struct bio *bio);bool bio_integrity_prep (struct bio *bio);void bio_integrity_advance (struct bio *bio,unsigned int bytes_done);void bio_integrity_trim (struct bio *bio);
@@ -104,7 +104,7 @@ static inline int bio_integrity_map_user(struct bio *bio, void __user *ubuf,return -EINVAL ; } -static inline void bio_integrity_unmap_free_user (struct bio *bio) +static inline void bio_integrity_unmap_user (struct bio *bio) { }
Signature · 1 line
2.43.0