Fix gaps detection in bio merges
Subsystems (2)
cover Fix gaps detection in bio merges
This series makes gaps go away for real if a driver asked to not see them (nvme and soon iser).
patch 1 addresses front merges where gaps weren't checked at all.
patches 2,3 adds gap detection for integrity payloads.
patch 4 makes blk_rq_map_user_iov turn to bounce buffer in case of an iovec with gaps.
I've been testing this for a couple of days now and I'll run some more tests on this, but since it was very easy to trigger these cases before I'm pretty sure we are covered.
Once this lands in, I will be able to get rid of the bounce buffer logic in the iser driver as it will rely on the block layer to get it right.
Jens Axboe (1):
block: Check for gaps on front and back merges
Sagi Grimberg (3):
block: Refuse request/bio merges with gaps in the integrity payload
block: Refuse adding appending a gaped integrity page to a bio
block: Copy a user iovec if it includes gaps
block/bio-integrity.c | 5 +++++ block/blk-integrity.c | 3 +++ block/blk-map.c | 26 ++++++++++++++++++++++++-- block/blk-merge.c | 25 ++++++++++++------------- include/linux/blkdev.h | 40 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 84 insertions(+), 15 deletions(-)
Signature · 1 line
1.8.4.3
PATCH 1/4 block: Check for gaps on front and back merges
From: Jens Axboe <[email protected]>
We are checking for gaps to previous bio_vec, which can only detect back merges gaps. Moreover, at the point where we check for a gap, we don't know if we will attempt a back or a front merge. Thus, check for gap to prev in a back merge attempt and check for a gap to next in a front merge attempt.
--- block/blk-merge.c | 19 ++++++------------- include/linux/blkdev.h | 20 ++++++++++++++++++++ 2 files changed, 26 insertions(+), 13 deletions(-)
block/blk-merge.c +6 -13
@@ -312,6 +312,8 @@ no_merge:int ll_back_merge_fn (struct request_queue *q,struct request *req,struct bio *bio) { +if (req_gap_back_merge (req, bio)) +return 0 ;if (blk_rq_sectors (req) +bio_sectors (bio) >blk_rq_get_max_sectors (req)) { req->cmd_flags |=REQ_NOMERGE ;
@@ -330,6 +332,9 @@ int ll_back_merge_fn(struct request_queue *q, struct request *req,int ll_front_merge_fn (struct request_queue *q,struct request *req,struct bio *bio) { + +if (req_gap_front_merge (req, bio)) +return 0 ;if (blk_rq_sectors (req) +bio_sectors (bio) >blk_rq_get_max_sectors (req)) { req->cmd_flags |=REQ_NOMERGE ;
@@ -356,14 +361,6 @@ static bool req_no_special_merge(struct request *req)return !q->mq_ops && req->special; } -static int req_gap_to_prev (struct request *req,struct bio *next) -{ -struct bio *prev = req->biotail; - -return bvec_gap_to_prev (req->q, &prev->bi_io_vec[prev->bi_vcnt -1 ], - next->bi_io_vec[0 ].bv_offset); -} -static int ll_merge_requests_fn (struct request_queue *q,struct request *req,struct request *next) {
@@ -378,7 +375,7 @@ static int ll_merge_requests_fn(struct request_queue *q, struct request *req,if (req_no_special_merge (req) ||req_no_special_merge (next))return 0 ; -if (req_gap_to_prev (req, next->bio)) +if (req_gap_back_merge (req, next->bio))return 0 ;/*
@@ -586,10 +583,6 @@ bool blk_rq_merge_ok(struct request *rq, struct bio *bio) !blk_write_same_mergeable(rq->bio, bio))return false; -/* Only check gaps if the bio carries data */ -if (bio_has_data (bio) &&req_gap_to_prev (rq, bio)) -return false; -return true; }
include/linux/blkdev.h +20 -0
@@ -1373,6 +1373,26 @@ static inline bool bvec_gap_to_prev(struct request_queue *q, ((bprv->bv_offset + bprv->bv_len) &queue_virt_boundary (q)); } +static inline bool bio_will_gap (struct request_queue *q,struct bio *prev, +struct bio *next) +{ +if (!bio_has_data (prev)) +return false; + +return bvec_gap_to_prev (q, &prev->bi_io_vec[prev->bi_vcnt -1 ], + next->bi_io_vec[0 ].bv_offset); +} + +static inline bool req_gap_back_merge (struct request *req,struct bio *bio) +{ +return bio_will_gap (req->q, req->biotail, bio); +} + +static inline bool req_gap_front_merge (struct request *req,struct bio *bio) +{ +return bio_will_gap (req->q, bio, req->bio); +} +struct work_struct ;int kblockd_schedule_work (struct work_struct *work);int kblockd_schedule_delayed_work (struct delayed_work *dwork,unsigned long delay);
Signature · 1 line
1.8.4.3
PATCH 2/4 block: Refuse request/bio merges with gaps in the integrity payload
In reply to Sagi GrimbergIf a driver sets the block queue virtual boundary mask, it means that it cannot handle gaps so we must not allow those in the integrity payload as well.
--- block/blk-integrity.c | 3 +++ block/blk-merge.c | 6 ++++++ include/linux/blkdev.h | 20 ++++++++++++++++++++ 3 files changed, 29 insertions(+)
block/blk-integrity.c +3 -0
@@ -204,6 +204,9 @@ bool blk_integrity_merge_rq(struct request_queue *q, struct request *req, q->limits.max_integrity_segments)return false; +if (integrity_req_gap_back_merge (req, next->bio)) +return false; +return true; }EXPORT_SYMBOL (blk_integrity_merge_rq);
block/blk-merge.c +6 -0
@@ -314,6 +314,9 @@ int ll_back_merge_fn(struct request_queue *q, struct request *req, {if (req_gap_back_merge (req, bio))return 0 ; +if (blk_integrity_rq (req) && +integrity_req_gap_back_merge (req, bio)) +return 0 ;if (blk_rq_sectors (req) +bio_sectors (bio) >blk_rq_get_max_sectors (req)) { req->cmd_flags |=REQ_NOMERGE ;
@@ -335,6 +338,9 @@ int ll_front_merge_fn(struct request_queue *q, struct request *req,if (req_gap_front_merge (req, bio))return 0 ; +if (blk_integrity_rq (req) && +integrity_req_gap_front_merge (req, bio)) +return 0 ;if (blk_rq_sectors (req) +bio_sectors (bio) >blk_rq_get_max_sectors (req)) { req->cmd_flags |=REQ_NOMERGE ;
include/linux/blkdev.h +20 -0
@@ -1393,6 +1393,26 @@ static inline bool req_gap_front_merge(struct request *req, struct bio *bio)return bio_will_gap (req->q, bio, req->bio); } +static inline bool integrity_req_gap_back_merge (struct request *req, +struct bio *next) +{ +struct bio_integrity_payload *bip =bio_integrity (req->bio); +struct bio_integrity_payload *bip_next =bio_integrity (next); + +return bvec_gap_to_prev (req->q, &bip->bip_vec[bip->bip_vcnt -1 ], + bip_next->bip_vec[0 ].bv_offset); +} + +static inline bool integrity_req_gap_front_merge (struct request *req, +struct bio *bio) +{ +struct bio_integrity_payload *bip =bio_integrity (bio); +struct bio_integrity_payload *bip_next =bio_integrity (req->bio); + +return bvec_gap_to_prev (req->q, &bip->bip_vec[bip->bip_vcnt -1 ], + bip_next->bip_vec[0 ].bv_offset); +} +struct work_struct ;int kblockd_schedule_work (struct work_struct *work);int kblockd_schedule_delayed_work (struct delayed_work *dwork,unsigned long delay);
Signature · 1 line
1.8.4.3
PATCH 3/4 block: Refuse adding appending a gaped integrity page to a bio
In reply to Sagi GrimbergThis is only theoretical at the moment given that the only subsystems that generate integrity payloads are the block layer itself and the scsi target (which generate well aligned integrity payloads). But when we will expose integrity meta-data to user-space, we'll need to refuse appending a page with a gap (if the queue virtual boundary is set).
--- block/bio-integrity.c | 5 +++++ 1 file changed, 5 insertions(+)
block/bio-integrity.c +5 -0
@@ -140,6 +140,11 @@ int bio_integrity_add_page(struct bio *bio, struct page *page, iv = bip->bip_vec + bip->bip_vcnt; +if (bip->bip_vcnt && +bvec_gap_to_prev (bdev_get_queue (bio->bi_bdev), + &bip->bip_vec[bip->bip_vcnt -1 ], offset)) +return 0 ; + iv->bv_page = page; iv->bv_len = len; iv->bv_offset = offset;
Signature · 1 line
1.8.4.3
PATCH 3/4 block: Refuse adding appending a gaped integrity page to a bio
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
"Sagi" == Sagi Grimberg <sagig at mellanox.com> writes:
Sagi> This is only theoretical at the moment given that the only Sagi> subsystems that generate integrity payloads are the block layer Sagi> itself and the scsi target (which generate well aligned integrity Sagi> payloads). But when we will expose integrity meta-data to Sagi> user-space, we'll need to refuse appending a page with a gap (if Sagi> the queue virtual boundary is set).
Code looks fine but my concerns outlined in:
http://lists.infradead.org/pipermail/linux-nvme/2015-July/002063.html
still stand.
Signature · 1 line
Martin K. Petersen Oracle Linux Engineering
PATCH 3/4 block: Refuse adding appending a gaped integrity page to a bio
Quoted 9 lines · Martin K. Petersen
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
"Sagi" == Sagi Grimberg <sagig at mellanox.com> writes:
Sagi> This is only theoretical at the moment given that the only Sagi> subsystems that generate integrity payloads are the block layer Sagi> itself and the scsi target (which generate well aligned integrity Sagi> payloads). But when we will expose integrity meta-data to Sagi> user-space, we'll need to refuse appending a page with a gap (if Sagi> the queue virtual boundary is set).
Hi Martin,
Quoted 3 lines
Code looks fine but my concerns outlined in:
http://lists.infradead.org/pipermail/linux-nvme/2015-July/002063.html
So given that this is a bug fix (was completely not handled before) I think we should make an incremental progress towards getting it right. Are you fine with that?
Now, given that drivers which set virt_boundary won't be able to accept 8-byte sg elements for DIX, would you prefer it being bounced rather than refusing the merge? This would require some additional logic given that we won't want to bounce on every merge attempt but only when the request is kicked?
Sagi.
PATCH 3/4 block: Refuse adding appending a gaped integrity page to a bio
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
"Sagi" == Sagi Grimberg <sagig at dev.mellanox.co.il> writes:
Sagi,
Sagi> So given that this is a bug fix (was completely not handled Sagi> before) I think we should make an incremental progress towards Sagi> getting it right. Are you fine with that?
Yeah, I don't have an objection to your changes.
Sagi> Now, given that drivers which set virt_boundary won't be able to Sagi> accept 8-byte sg elements for DIX, would you prefer it being Sagi> bounced rather than refusing the merge? This would require some Sagi> additional logic given that we won't want to bounce on every merge Sagi> attempt but only when the request is kicked?
For some workloads I think bouncing is inevitable by virtue of the PI payload being teeny tiny by design. I don't mind that happening in block as opposed to the driver. This would also help things like NVMe.
Signature · 1 line
Martin K. Petersen Oracle Linux Engineering
PATCH 3/4 block: Refuse adding appending a gaped integrity page to a bio
Quoted 10 lines · Martin K. Petersen
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
"Sagi" == Sagi Grimberg <sagig at dev.mellanox.co.il> writes:
Sagi,
Sagi> So given that this is a bug fix (was completely not handled Sagi> before) I think we should make an incremental progress towards Sagi> getting it right. Are you fine with that?
Yeah, I don't have an objection to your changes.
Quoted 8 lines
Sagi> Now, given that drivers which set
virt_boundarywon't be able to Sagi> accept 8-byte sg elements for DIX, would you prefer it being Sagi> bounced rather than refusing the merge? This would require some Sagi> additional logic given that we won't want to bounce on every merge Sagi> attempt but only when the request is kicked?For some workloads I think bouncing is inevitable by virtue of the PI payload being teeny tiny by design.
I see. I guess we can bounce DIX data, but is it always "right" to bounce? I don't know, as you say, it is workload dependent.
PATCH 3/4 block: Refuse adding appending a gaped integrity page to a bio
Quoted 4 lines
Quoted 4 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
"Sagi" == Sagi Grimberg <sagig at dev.mellanox.co.il> writes:
For some workloads I think bouncing is inevitable by virtue of the PI payload being teeny tiny by design.
Sagi> I see. I guess we can bounce DIX data, but is it always "right" to Sagi> bounce? I don't know, as you say, it is workload dependent.
The right place to make that call, I guess, is in bio_integrity_add_page() based on whether we exceed any of the constraints set by the LLD.
Signature · 1 line
Martin K. Petersen Oracle Linux Engineering
PATCH 3/4 block: Refuse adding appending a gaped integrity page to a bio
Quoted 11 lines · Martin K. Petersen
Quoted 4 lines
Quoted 4 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
"Sagi" == Sagi Grimberg <sagig at dev.mellanox.co.il> writes:
For some workloads I think bouncing is inevitable by virtue of the PI payload being teeny tiny by design.
Sagi> I see. I guess we can bounce DIX data, but is it always "right" to Sagi> bounce? I don't know, as you say, it is workload dependent.
The right place to make that call, I guess, is in
bio_integrity_add_page()based on whether we exceed any of the constraints set by the LLD.
In case the consumer passed a gappy integrity payload then we can bounce but AFAICT, merge flows do not include a call site to bio_integrity_add_page.
It would be easy enough to bounce gappy IOs and refuse gappy merges, would that be sufficient?
Sagi.
PATCH 4/4 block: Copy a user iovec if it includes gaps
In reply to Sagi GrimbergFor drivers that don't support gaps in the SG lists handed to them we must bounce (copy the user buffers) and pass a bio that does not include gaps. This doesn't matter for any current user, but will help to allow iser which can't handle gaps to use the block virtual boundary instead of using driver-local bounce buffering when handling SG_IO commands.
--- block/blk-map.c | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-)
block/blk-map.c +24 -2
@@ -9,6 +9,24 @@#include "blk.h" +static bool iovec_gap_to_prv (struct request_queue *q, +struct iovec *prv,struct iovec *cur) +{ +unsigned long prev_end; + +if (!queue_virt_boundary (q)) +return false; + +if (prv->iov_base ==NULL && prv->iov_len ==0 ) +/* prv is not set - don't check */ +return false; + + prev_end = (unsigned long )(prv->iov_base + prv->iov_len); + +return (((unsigned long )cur->iov_base &queue_virt_boundary (q)) || + prev_end &queue_virt_boundary (q)); +} +int blk_rq_append_bio (struct request_queue *q,struct request *rq,struct bio *bio) {
@@ -67,7 +85,7 @@ int blk_rq_map_user_iov(struct request_queue *q, struct request *rq,struct bio *bio;int unaligned =0 ;struct iov_iter i; -struct iovec iov; +struct iovec iov, prv = {.iov_base =NULL , .iov_len =0 };if (!iter || !iter->count)return -EINVAL ;
@@ -81,8 +99,12 @@ int blk_rq_map_user_iov(struct request_queue *q, struct request *rq,/* * Keep going so we check length of all segments */ -if (uaddr &queue_dma_alignment (q)) +if ((uaddr &queue_dma_alignment (q)) || +iovec_gap_to_prv (q, &prv, &iov)) unaligned =1 ; + + prv.iov_base = iov.iov_base; + prv.iov_len = iov.iov_len; }if (unaligned || (q->dma_pad_mask & iter->count) || map_data)
Signature · 1 line
1.8.4.3