Skip to contentSkip to conversation

Fix gaps detection in bio merges

PATCH New Sagi Grimberg 3 participants 11 messages Sep 3, 2015 – Sep 13, 2015

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

cover Fix gaps detection in bio merges

This series makes gaps go away for real if a driver asked to not see them (nvme and soon iser).

patch 1 addresses front merges where gaps weren't checked at all.

patches 2,3 adds gap detection for integrity payloads.

patch 4 makes blk_rq_map_user_iov turn to bounce buffer in case of an iovec with gaps.

I've been testing this for a couple of days now and I'll run some more tests on this, but since it was very easy to trigger these cases before I'm pretty sure we are covered.

Once this lands in, I will be able to get rid of the bounce buffer logic in the iser driver as it will rely on the block layer to get it right.

Jens Axboe (1):
block: Check for gaps on front and back merges

Sagi Grimberg (3):
block: Refuse request/bio merges with gaps in the integrity payload
block: Refuse adding appending a gaped integrity page to a bio
block: Copy a user iovec if it includes gaps

 block/bio-integrity.c  |  5 +++++
 block/blk-integrity.c  |  3 +++
 block/blk-map.c        | 26 ++++++++++++++++++++++++--
 block/blk-merge.c      | 25 ++++++++++++-------------
 include/linux/blkdev.h | 40 ++++++++++++++++++++++++++++++++++++++++
 5 files changed, 84 insertions(+), 15 deletions(-)
Signature · 1 line
1.8.4.3
#1 lore raw

PATCH 1/4 block: Check for gaps on front and back merges

From: Jens Axboe <[email protected]>

We are checking for gaps to previous bio_vec, which can only detect back merges gaps. Moreover, at the point where we check for a gap, we don't know if we will attempt a back or a front merge. Thus, check for gap to prev in a back merge attempt and check for a gap to next in a front merge attempt.

Signed-off-by Jens Axboe <axboe at fb.com>
[sagig: Minor rename change]
Signed-off-by Sagi Grimberg <sagig at mellanox.com>
---
 block/blk-merge.c      | 19 ++++++-------------
 include/linux/blkdev.h | 20 ++++++++++++++++++++
 2 files changed, 26 insertions(+), 13 deletions(-)
block/blk-merge.c +6 -13
index 09bf5cb116e2..8fbe79829d79 100644
@@ -312,6 +312,8 @@ no_merge:
 int ll_back_merge_fn(struct request_queue *q, struct request *req,
 		     struct bio *bio)
 {
+	if (req_gap_back_merge(req, bio))
+		return 0;
 	if (blk_rq_sectors(req) + bio_sectors(bio) >
 	    blk_rq_get_max_sectors(req)) {
 		req->cmd_flags |= REQ_NOMERGE;
@@ -330,6 +332,9 @@ int ll_back_merge_fn(struct request_queue *q, struct request *req,
 int ll_front_merge_fn(struct request_queue *q, struct request *req,
 		      struct bio *bio)
 {
+
+	if (req_gap_front_merge(req, bio))
+		return 0;
 	if (blk_rq_sectors(req) + bio_sectors(bio) >
 	    blk_rq_get_max_sectors(req)) {
 		req->cmd_flags |= REQ_NOMERGE;
@@ -356,14 +361,6 @@ static bool req_no_special_merge(struct request *req)
 	return !q->mq_ops && req->special;
 }
 
-static int req_gap_to_prev(struct request *req, struct bio *next)
-{
-	struct bio *prev = req->biotail;
-
-	return bvec_gap_to_prev(req->q, &prev->bi_io_vec[prev->bi_vcnt - 1],
-			next->bi_io_vec[0].bv_offset);
-}
-
 static int ll_merge_requests_fn(struct request_queue *q, struct request *req,
 				struct request *next)
 {
@@ -378,7 +375,7 @@ static int ll_merge_requests_fn(struct request_queue *q, struct request *req,
 	if (req_no_special_merge(req) || req_no_special_merge(next))
 		return 0;
 
-	if (req_gap_to_prev(req, next->bio))
+	if (req_gap_back_merge(req, next->bio))
 		return 0;
 
 	/*
@@ -586,10 +583,6 @@ bool blk_rq_merge_ok(struct request *rq, struct bio *bio)
 	    !blk_write_same_mergeable(rq->bio, bio))
 		return false;
 
-	/* Only check gaps if the bio carries data */
-	if (bio_has_data(bio) && req_gap_to_prev(rq, bio))
-		return false;
-
 	return true;
 }
 
include/linux/blkdev.h +20 -0
index 68adec9da25f..a65b94480422 100644
@@ -1373,6 +1373,26 @@ static inline bool bvec_gap_to_prev(struct request_queue *q,
 		((bprv->bv_offset + bprv->bv_len) & queue_virt_boundary(q));
 }
 
+static inline bool bio_will_gap(struct request_queue *q, struct bio *prev,
+			 struct bio *next)
+{
+	if (!bio_has_data(prev))
+		return false;
+
+	return bvec_gap_to_prev(q, &prev->bi_io_vec[prev->bi_vcnt - 1],
+				next->bi_io_vec[0].bv_offset);
+}
+
+static inline bool req_gap_back_merge(struct request *req, struct bio *bio)
+{
+	return bio_will_gap(req->q, req->biotail, bio);
+}
+
+static inline bool req_gap_front_merge(struct request *req, struct bio *bio)
+{
+	return bio_will_gap(req->q, bio, req->bio);
+}
+
 struct work_struct;
 int kblockd_schedule_work(struct work_struct *work);
 int kblockd_schedule_delayed_work(struct delayed_work *dwork, unsigned long delay);
Signature · 1 line
1.8.4.3
#2 lore raw

PATCH 2/4 block: Refuse request/bio merges with gaps in the integrity payload

In reply to Sagi Grimberg

If a driver sets the block queue virtual boundary mask, it means that it cannot handle gaps so we must not allow those in the integrity payload as well.

Signed-off-by Sagi Grimberg <sagig at mellanox.com>
---
 block/blk-integrity.c  |  3 +++
 block/blk-merge.c      |  6 ++++++
 include/linux/blkdev.h | 20 ++++++++++++++++++++
 3 files changed, 29 insertions(+)
block/blk-integrity.c +3 -0
index f548b64be092..75f29cf70188 100644
@@ -204,6 +204,9 @@ bool blk_integrity_merge_rq(struct request_queue *q, struct request *req,
 	    q->limits.max_integrity_segments)
 		return false;
 
+	if (integrity_req_gap_back_merge(req, next->bio))
+		return false;
+
 	return true;
 }
 EXPORT_SYMBOL(blk_integrity_merge_rq);
block/blk-merge.c +6 -0
index 8fbe79829d79..b4be61f4b65e 100644
@@ -314,6 +314,9 @@ int ll_back_merge_fn(struct request_queue *q, struct request *req,
 {
 	if (req_gap_back_merge(req, bio))
 		return 0;
+	if (blk_integrity_rq(req) &&
+	    integrity_req_gap_back_merge(req, bio))
+		return 0;
 	if (blk_rq_sectors(req) + bio_sectors(bio) >
 	    blk_rq_get_max_sectors(req)) {
 		req->cmd_flags |= REQ_NOMERGE;
@@ -335,6 +338,9 @@ int ll_front_merge_fn(struct request_queue *q, struct request *req,
 
 	if (req_gap_front_merge(req, bio))
 		return 0;
+	if (blk_integrity_rq(req) &&
+	    integrity_req_gap_front_merge(req, bio))
+		return 0;
 	if (blk_rq_sectors(req) + bio_sectors(bio) >
 	    blk_rq_get_max_sectors(req)) {
 		req->cmd_flags |= REQ_NOMERGE;
include/linux/blkdev.h +20 -0
index a65b94480422..dead6088af1c 100644
@@ -1393,6 +1393,26 @@ static inline bool req_gap_front_merge(struct request *req, struct bio *bio)
 	return bio_will_gap(req->q, bio, req->bio);
 }
 
+static inline bool integrity_req_gap_back_merge(struct request *req,
+						struct bio *next)
+{
+	struct bio_integrity_payload *bip = bio_integrity(req->bio);
+	struct bio_integrity_payload *bip_next = bio_integrity(next);
+
+	return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1],
+				bip_next->bip_vec[0].bv_offset);
+}
+
+static inline bool integrity_req_gap_front_merge(struct request *req,
+						 struct bio *bio)
+{
+	struct bio_integrity_payload *bip = bio_integrity(bio);
+	struct bio_integrity_payload *bip_next = bio_integrity(req->bio);
+
+	return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1],
+				bip_next->bip_vec[0].bv_offset);
+}
+
 struct work_struct;
 int kblockd_schedule_work(struct work_struct *work);
 int kblockd_schedule_delayed_work(struct delayed_work *dwork, unsigned long delay);
Signature · 1 line
1.8.4.3
#3 lore raw

PATCH 3/4 block: Refuse adding appending a gaped integrity page to a bio

In reply to Sagi Grimberg

This is only theoretical at the moment given that the only subsystems that generate integrity payloads are the block layer itself and the scsi target (which generate well aligned integrity payloads). But when we will expose integrity meta-data to user-space, we'll need to refuse appending a page with a gap (if the queue virtual boundary is set).

Signed-off-by Sagi Grimberg <sagig at mellanox.com>
---
 block/bio-integrity.c | 5 +++++
 1 file changed, 5 insertions(+)
block/bio-integrity.c +5 -0
index 719b7152aed1..4e5a43465ddb 100644
@@ -140,6 +140,11 @@ int bio_integrity_add_page(struct bio *bio, struct page *page,
 
 	iv = bip->bip_vec + bip->bip_vcnt;
 
+	if (bip->bip_vcnt &&
+	    bvec_gap_to_prev(bdev_get_queue(bio->bi_bdev),
+			     &bip->bip_vec[bip->bip_vcnt - 1], offset))
+		return 0;
+
 	iv->bv_page = page;
 	iv->bv_len = len;
 	iv->bv_offset = offset;
Signature · 1 line
1.8.4.3
#4 lore raw

PATCH 3/4 block: Refuse adding appending a gaped integrity page to a bio

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

"Sagi" == Sagi Grimberg <sagig at mellanox.com> writes:

Sagi> This is only theoretical at the moment given that the only Sagi> subsystems that generate integrity payloads are the block layer Sagi> itself and the scsi target (which generate well aligned integrity Sagi> payloads). But when we will expose integrity meta-data to Sagi> user-space, we'll need to refuse appending a page with a gap (if Sagi> the queue virtual boundary is set).

Code looks fine but my concerns outlined in:

http://lists.infradead.org/pipermail/linux-nvme/2015-July/002063.html

still stand.

Signature · 1 line
Martin K. Petersen	Oracle Linux Engineering
#5 lore raw

PATCH 3/4 block: Refuse adding appending a gaped integrity page to a bio

Quoted 9 lines · Martin K. Petersen
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

"Sagi" == Sagi Grimberg <sagig at mellanox.com> writes:

Sagi> This is only theoretical at the moment given that the only Sagi> subsystems that generate integrity payloads are the block layer Sagi> itself and the scsi target (which generate well aligned integrity Sagi> payloads). But when we will expose integrity meta-data to Sagi> user-space, we'll need to refuse appending a page with a gap (if Sagi> the queue virtual boundary is set).

Hi Martin,

Quoted 3 lines

Code looks fine but my concerns outlined in:

http://lists.infradead.org/pipermail/linux-nvme/2015-July/002063.html

So given that this is a bug fix (was completely not handled before) I think we should make an incremental progress towards getting it right. Are you fine with that?

Now, given that drivers which set virt_boundary won't be able to accept 8-byte sg elements for DIX, would you prefer it being bounced rather than refusing the merge? This would require some additional logic given that we won't want to bounce on every merge attempt but only when the request is kicked?

Sagi.

#6 lore raw

PATCH 3/4 block: Refuse adding appending a gaped integrity page to a bio

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

"Sagi" == Sagi Grimberg <sagig at dev.mellanox.co.il> writes:

Sagi,

Sagi> So given that this is a bug fix (was completely not handled Sagi> before) I think we should make an incremental progress towards Sagi> getting it right. Are you fine with that?

Yeah, I don't have an objection to your changes.

Sagi> Now, given that drivers which set virt_boundary won't be able to Sagi> accept 8-byte sg elements for DIX, would you prefer it being Sagi> bounced rather than refusing the merge? This would require some Sagi> additional logic given that we won't want to bounce on every merge Sagi> attempt but only when the request is kicked?

For some workloads I think bouncing is inevitable by virtue of the PI payload being teeny tiny by design. I don't mind that happening in block as opposed to the driver. This would also help things like NVMe.

Signature · 1 line
Martin K. Petersen	Oracle Linux Engineering
#7 lore raw

PATCH 3/4 block: Refuse adding appending a gaped integrity page to a bio

Quoted 10 lines · Martin K. Petersen
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

"Sagi" == Sagi Grimberg <sagig at dev.mellanox.co.il> writes:

Sagi,

Sagi> So given that this is a bug fix (was completely not handled Sagi> before) I think we should make an incremental progress towards Sagi> getting it right. Are you fine with that?

Yeah, I don't have an objection to your changes.

Quoted 8 lines

Sagi> Now, given that drivers which set virt_boundary won't be able to Sagi> accept 8-byte sg elements for DIX, would you prefer it being Sagi> bounced rather than refusing the merge? This would require some Sagi> additional logic given that we won't want to bounce on every merge Sagi> attempt but only when the request is kicked?

For some workloads I think bouncing is inevitable by virtue of the PI payload being teeny tiny by design.

I see. I guess we can bounce DIX data, but is it always "right" to bounce? I don't know, as you say, it is workload dependent.

#8 lore raw

PATCH 3/4 block: Refuse adding appending a gaped integrity page to a bio

Quoted 4 lines
Quoted 4 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line

"Sagi" == Sagi Grimberg <sagig at dev.mellanox.co.il> writes:

For some workloads I think bouncing is inevitable by virtue of the PI payload being teeny tiny by design.

Sagi> I see. I guess we can bounce DIX data, but is it always "right" to Sagi> bounce? I don't know, as you say, it is workload dependent.

The right place to make that call, I guess, is in bio_integrity_add_page() based on whether we exceed any of the constraints set by the LLD.

Signature · 1 line
Martin K. Petersen	Oracle Linux Engineering
#9 lore raw

PATCH 3/4 block: Refuse adding appending a gaped integrity page to a bio

Quoted 11 lines · Martin K. Petersen
Quoted 4 lines
Quoted 4 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line

"Sagi" == Sagi Grimberg <sagig at dev.mellanox.co.il> writes:

For some workloads I think bouncing is inevitable by virtue of the PI payload being teeny tiny by design.

Sagi> I see. I guess we can bounce DIX data, but is it always "right" to Sagi> bounce? I don't know, as you say, it is workload dependent.

The right place to make that call, I guess, is in bio_integrity_add_page() based on whether we exceed any of the constraints set by the LLD.

In case the consumer passed a gappy integrity payload then we can bounce but AFAICT, merge flows do not include a call site to bio_integrity_add_page.

It would be easy enough to bounce gappy IOs and refuse gappy merges, would that be sufficient?

Sagi.

#10 lore raw

PATCH 4/4 block: Copy a user iovec if it includes gaps

In reply to Sagi Grimberg

For drivers that don't support gaps in the SG lists handed to them we must bounce (copy the user buffers) and pass a bio that does not include gaps. This doesn't matter for any current user, but will help to allow iser which can't handle gaps to use the block virtual boundary instead of using driver-local bounce buffering when handling SG_IO commands.

Signed-off-by Christoph Hellwig <hch at lst.de>
Signed-off-by Sagi Grimberg <sagig at mellanox.com>
---
 block/blk-map.c | 26 ++++++++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)
block/blk-map.c +24 -2
index da310a105429..ac581a674b9b 100644
@@ -9,6 +9,24 @@
 
 #include "blk.h"
 
+static bool iovec_gap_to_prv(struct request_queue *q,
+			     struct iovec *prv, struct iovec *cur)
+{
+	unsigned long prev_end;
+
+	if (!queue_virt_boundary(q))
+		return false;
+
+	if (prv->iov_base == NULL && prv->iov_len == 0)
+		/* prv is not set - don't check */
+		return false;
+
+	prev_end = (unsigned long)(prv->iov_base + prv->iov_len);
+
+	return (((unsigned long)cur->iov_base & queue_virt_boundary(q)) ||
+		prev_end & queue_virt_boundary(q));
+}
+
 int blk_rq_append_bio(struct request_queue *q, struct request *rq,
 		      struct bio *bio)
 {
@@ -67,7 +85,7 @@ int blk_rq_map_user_iov(struct request_queue *q, struct request *rq,
 	struct bio *bio;
 	int unaligned = 0;
 	struct iov_iter i;
-	struct iovec iov;
+	struct iovec iov, prv = {.iov_base = NULL, .iov_len = 0};
 
 	if (!iter || !iter->count)
 		return -EINVAL;
@@ -81,8 +99,12 @@ int blk_rq_map_user_iov(struct request_queue *q, struct request *rq,
 		/*
 		 * Keep going so we check length of all segments
 		 */
-		if (uaddr & queue_dma_alignment(q))
+		if ((uaddr & queue_dma_alignment(q)) ||
+		    iovec_gap_to_prv(q, &prv, &iov))
 			unaligned = 1;
+
+		prv.iov_base = iov.iov_base;
+		prv.iov_len = iov.iov_len;
 	}
 
 	if (unaligned || (q->dma_pad_mask & iter->count) || map_data)
Signature · 1 line
1.8.4.3