Skip to contentSkip to conversation

blk-mq-sched: fix put_rq_private() lock inconsistency

PATCH In review Jens Axboe 3 participants 6 messages Feb 2, 2017 – Feb 7, 2017 Block layer

lore
Share & tools
lore

Jens Axboe

author
#0 lore raw

cover blk-mq-sched: fix put_rq_private() lock inconsistency

I tested the patch I sent to Paolo yesterday, and it seems to work fine. I broke it up into two pieces, so the functional change is restricted to patch #2.

Basically this fixes the case where we can invoke the blk-mq-sched put request functions in an inconsistent state. Most of the time we invoke them without any locks held, but for the case where we get a successful request-to-request merge on the back of a bio-to-request merge, we can invoke it with whatever lock the scheduler held when it called blk_mq_sched_try_merge().

Signature · 1 line
Jens Axboe
author
#1 lore raw

PATCH 1/2 blk-merge: return the merged request

When we attempt to merge request-to-request, we return a 0/1 if we ended up merging or not. Change that to return the pointer to the request that we freed. We will use this to move the freeing of that request out of the merge logic, so that callers can drop locks before freeing the request.

There should be no functional changes in this patch.

---
 block/blk-merge.c | 31 ++++++++++++++++---------------
 block/blk.h       |  4 ++--
 2 files changed, 18 insertions(+), 17 deletions(-)
block/blk-merge.c +16 -15
index 6aa43dec5af4..3826fc32b72c 100644
@@ -659,31 +659,32 @@ static void blk_account_io_merge(struct request *req)
 }
 
 /*
- * Has to be called with the request spinlock acquired
+ * For non-mq, this has to be called with the request spinlock acquired.
+ * For mq with scheduling, the appropriate queue wide lock should be held.
  */
-static int attempt_merge(struct request_queue *q, struct request *req,
-			  struct request *next)
+static struct request *attempt_merge(struct request_queue *q,
+				     struct request *req, struct request *next)
 {
 	if (!rq_mergeable(req) || !rq_mergeable(next))
-		return 0;
+		return NULL;
 
 	if (req_op(req) != req_op(next))
-		return 0;
+		return NULL;
 
 	/*
 	 * not contiguous
 	 */
 	if (blk_rq_pos(req) + blk_rq_sectors(req) != blk_rq_pos(next))
-		return 0;
+		return NULL;
 
 	if (rq_data_dir(req) != rq_data_dir(next)
 	    || req->rq_disk != next->rq_disk
 	    || req_no_special_merge(next))
-		return 0;
+		return NULL;
 
 	if (req_op(req) == REQ_OP_WRITE_SAME &&
 	    !blk_write_same_mergeable(req->bio, next->bio))
-		return 0;
+		return NULL;
 
 	/*
 	 * If we are allowed to merge, then append bio list
@@ -692,7 +693,7 @@ static int attempt_merge(struct request_queue *q, struct request *req,
 	 * counts here.
 	 */
 	if (!ll_merge_requests_fn(q, req, next))
-		return 0;
+		return NULL;
 
 	/*
 	 * If failfast settings disagree or any of the two is already
@@ -735,27 +736,27 @@ static int attempt_merge(struct request_queue *q, struct request *req,
 	/* owner-ship of bio passed from next to req */
 	next->bio = NULL;
 	__blk_put_request(q, next);
-	return 1;
+	return next;
 }
 
-int attempt_back_merge(struct request_queue *q, struct request *rq)
+struct request *attempt_back_merge(struct request_queue *q, struct request *rq)
 {
 	struct request *next = elv_latter_request(q, rq);
 
 	if (next)
 		return attempt_merge(q, rq, next);
 
-	return 0;
+	return NULL;
 }
 
-int attempt_front_merge(struct request_queue *q, struct request *rq)
+struct request *attempt_front_merge(struct request_queue *q, struct request *rq)
 {
 	struct request *prev = elv_former_request(q, rq);
 
 	if (prev)
 		return attempt_merge(q, prev, rq);
 
-	return 0;
+	return NULL;
 }
 
 int blk_attempt_req_merge(struct request_queue *q, struct request *rq,
@@ -767,7 +768,7 @@ int blk_attempt_req_merge(struct request_queue *q, struct request *rq,
 		if (!e->type->ops.sq.elevator_allow_rq_merge_fn(q, rq, next))
 			return 0;
 
-	return attempt_merge(q, rq, next);
+	return attempt_merge(q, rq, next) != NULL;
 }
 
 bool blk_rq_merge_ok(struct request *rq, struct bio *bio)
block/blk.h +2 -2
index c1bd4bf9e645..918cea38d51e 100644
@@ -204,8 +204,8 @@ int ll_back_merge_fn(struct request_queue *q, struct request *req,
 		     struct bio *bio);
 int ll_front_merge_fn(struct request_queue *q, struct request *req, 
 		      struct bio *bio);
-int attempt_back_merge(struct request_queue *q, struct request *rq);
-int attempt_front_merge(struct request_queue *q, struct request *rq);
+struct request *attempt_back_merge(struct request_queue *q, struct request *rq);
+struct request *attempt_front_merge(struct request_queue *q, struct request *rq);
 int blk_attempt_req_merge(struct request_queue *q, struct request *rq,
 				struct request *next);
 void blk_recalc_rq_segments(struct request *rq);
Signature · 1 line
2.7.4
Reviewed-by on v1 1/2
#2 lore raw

Re: [PATCH 1/2] blk-merge: return the merged request

Quoted 7 lines · Jens Axboe

When we attempt to merge request-to-request, we return a 0/1 if we ended up merging or not. Change that to return the pointer to the request that we freed. We will use this to move the freeing of that request out of the merge logic, so that callers can drop locks before freeing the request.

There should be no functional changes in this patch.

Quoted 5 lines
---
 block/blk-merge.c | 31 ++++++++++++++++---------------
 block/blk.h       |  4 ++--
 2 files changed, 18 insertions(+), 17 deletions(-)
author
#3 lore raw

PATCH 2/2 block: free merged request in the caller

In reply to Jens Axboe

If we end up doing a request-to-request merge when we have completed a bio-to-request merge, we free the request from deep down in that path. For blk-mq-sched, the merge path has to hold the appropriate lock, but we don't need it for freeing the request. And in fact holding the lock is problematic, since we are now calling the mq sched put_rq_private() hook with the lock held. Other call paths do not hold this lock.

Fix this inconsistency by ensuring that the caller frees a merged request. Then we can do it outside of the lock, making it both more efficient and fixing the blk-mq-sched problem of invoking parts of the scheduler with an unknown lock state.

---
 block/blk-core.c     | 12 +++++++++---
 block/blk-merge.c    | 15 ++++++++++++---
 block/blk-mq-sched.c |  9 ++++++---
 block/blk-mq-sched.h |  3 ++-
 block/mq-deadline.c  |  8 ++++++--
 5 files changed, 35 insertions(+), 12 deletions(-)
block/blk-core.c +9 -3
index a5726e01f839..00c90f8cd682 100644
@@ -1591,7 +1591,7 @@ static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio)
 {
 	struct blk_plug *plug;
 	int el_ret, where = ELEVATOR_INSERT_SORT;
-	struct request *req;
+	struct request *req, *free;
 	unsigned int request_count = 0;
 	unsigned int wb_acct;
 
@@ -1632,15 +1632,21 @@ static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio)
 	if (el_ret == ELEVATOR_BACK_MERGE) {
 		if (bio_attempt_back_merge(q, req, bio)) {
 			elv_bio_merged(q, req, bio);
-			if (!attempt_back_merge(q, req))
+			free = attempt_back_merge(q, req);
+			if (!free)
 				elv_merged_request(q, req, el_ret);
+			else
+				__blk_put_request(q, free);
 			goto out_unlock;
 		}
 	} else if (el_ret == ELEVATOR_FRONT_MERGE) {
 		if (bio_attempt_front_merge(q, req, bio)) {
 			elv_bio_merged(q, req, bio);
-			if (!attempt_front_merge(q, req))
+			free = attempt_front_merge(q, req);
+			if (!free)
 				elv_merged_request(q, req, el_ret);
+			else
+				__blk_put_request(q, free);
 			goto out_unlock;
 		}
 	}
block/blk-merge.c +12 -3
index 3826fc32b72c..a373416dbc9a 100644
@@ -733,9 +733,11 @@ static struct request *attempt_merge(struct request_queue *q,
 	if (blk_rq_cpu_valid(next))
 		req->cpu = next->cpu;
 
-	/* owner-ship of bio passed from next to req */
+	/*
+	 * ownership of bio passed from next to req, return 'next' for
+	 * the caller to free
+	 */
 	next->bio = NULL;
-	__blk_put_request(q, next);
 	return next;
 }
 
@@ -763,12 +765,19 @@ int blk_attempt_req_merge(struct request_queue *q, struct request *rq,
 			  struct request *next)
 {
 	struct elevator_queue *e = q->elevator;
+	struct request *free;
 
 	if (!e->uses_mq && e->type->ops.sq.elevator_allow_rq_merge_fn)
 		if (!e->type->ops.sq.elevator_allow_rq_merge_fn(q, rq, next))
 			return 0;
 
-	return attempt_merge(q, rq, next) != NULL;
+	free = attempt_merge(q, rq, next);
+	if (free) {
+		__blk_put_request(q, free);
+		return 1;
+	}
+
+	return 0;
 }
 
 bool blk_rq_merge_ok(struct request *rq, struct bio *bio)
block/blk-mq-sched.c +6 -3
index 114814ec3d49..d93b56d53c4e 100644
@@ -234,7 +234,8 @@ void blk_mq_sched_move_to_dispatch(struct blk_mq_hw_ctx *hctx,
 }
 EXPORT_SYMBOL_GPL(blk_mq_sched_move_to_dispatch);
 
-bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio)
+bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio,
+			    struct request **merged_request)
 {
 	struct request *rq;
 	int ret;
@@ -244,7 +245,8 @@ bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio)
 		if (!blk_mq_sched_allow_merge(q, rq, bio))
 			return false;
 		if (bio_attempt_back_merge(q, rq, bio)) {
-			if (!attempt_back_merge(q, rq))
+			*merged_request = attempt_back_merge(q, rq);
+			if (!*merged_request)
 				elv_merged_request(q, rq, ret);
 			return true;
 		}
@@ -252,7 +254,8 @@ bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio)
 		if (!blk_mq_sched_allow_merge(q, rq, bio))
 			return false;
 		if (bio_attempt_front_merge(q, rq, bio)) {
-			if (!attempt_front_merge(q, rq))
+			*merged_request = attempt_front_merge(q, rq);
+			if (!*merged_request)
 				elv_merged_request(q, rq, ret);
 			return true;
 		}
block/blk-mq-sched.h +2 -1
index 9478aaeb48c5..3643686a54b8 100644
@@ -16,7 +16,8 @@ void blk_mq_sched_put_request(struct request *rq);
 
 void blk_mq_sched_request_inserted(struct request *rq);
 bool blk_mq_sched_bypass_insert(struct blk_mq_hw_ctx *hctx, struct request *rq);
-bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio);
+bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio,
+				struct request **merged_request);
 bool __blk_mq_sched_bio_merge(struct request_queue *q, struct bio *bio);
 bool blk_mq_sched_try_insert_merge(struct request_queue *q, struct request *rq);
 void blk_mq_sched_restart_queues(struct blk_mq_hw_ctx *hctx);
block/mq-deadline.c +6 -2
index 49583536698c..682fa64f55ff 100644
@@ -371,12 +371,16 @@ static bool dd_bio_merge(struct blk_mq_hw_ctx *hctx, struct bio *bio)
 {
 	struct request_queue *q = hctx->queue;
 	struct deadline_data *dd = q->elevator->elevator_data;
-	int ret;
+	struct request *free = NULL;
+	bool ret;
 
 	spin_lock(&dd->lock);
-	ret = blk_mq_sched_try_merge(q, bio);
+	ret = blk_mq_sched_try_merge(q, bio, &free);
 	spin_unlock(&dd->lock);
 
+	if (free)
+		blk_mq_free_request(free);
+
 	return ret;
 }
 
Signature · 1 line
2.7.4
Reviewed-by on v1 2/2
#4 lore raw

Re: [PATCH 2/2] block: free merged request in the caller

Quoted 8 lines · Jens Axboe

If we end up doing a request-to-request merge when we have completed a bio-to-request merge, we free the request from deep down in that path. For blk-mq-sched, the merge path has to hold the appropriate lock, but we don't need it for freeing the request. And in fact holding the lock is problematic, since we are now calling the mq sched put_rq_private() hook with the lock held. Other call paths do not hold this lock.

Quoted 6 lines

Fix this inconsistency by ensuring that the caller frees a merged request. Then we can do it outside of the lock, making it both more efficient and fixing the blk-mq-sched problem of invoking parts of the scheduler with an unknown lock state.

Quoted 8 lines
---
 block/blk-core.c     | 12 +++++++++---
 block/blk-merge.c    | 15 ++++++++++++---
 block/blk-mq-sched.c |  9 ++++++---
 block/blk-mq-sched.h |  3 ++-
 block/mq-deadline.c  |  8 ++++++--
 5 files changed, 35 insertions(+), 12 deletions(-)
#5 lore raw

Re: [PATCH 2/2] block: free merged request in the caller

In reply to Jens Axboe
Quoted 2 lines

Il giorno 02 feb 2017, alle ore 17:06, Jens Axboe <[email protected]> ha =

scritto:

block/blk-core.c static blk_qc_t blk_queue_bio(struct =

Quoted 29 lines

=20
If we end up doing a request-to-request merge when we have completed a bio-to-request merge, we free the request from deep down in that path. For blk-mq-sched, the merge path has to hold the appropriate lock, but we don't need it for freeing the request. And in fact holding the lock is problematic, since we are now calling the mq sched put_rq_private() hook with the lock held. Other call paths do not hold this lock.
=20
Fix this inconsistency by ensuring that the caller frees a merged request. Then we can do it outside of the lock, making it both more efficient and fixing the blk-mq-sched problem of invoking parts of the scheduler with an unknown lock state.
=20

---

block/blk-core.c     | 12 +++++++++---
block/blk-merge.c    | 15 ++++++++++++---
block/blk-mq-sched.c |  9 ++++++---
block/blk-mq-sched.h |  3 ++-
block/mq-deadline.c  |  8 ++++++--
5 files changed, 35 insertions(+), 12 deletions(-)

=20

block/blk-core.c +0 -0
index a5726e01f839..00c90f8cd682 100644
@@ -1591,7 +1591,7 @@ static blk_qc_t blk_queue_bio(struct =

request_queue *q, struct bio *bio)

block/blk-core.c static blk_qc_t blk_queue_bio(struct =

Quoted 9 lines
block/blk-core.c +1 -1
 {
 	struct blk_plug *plug;
 	int el_ret, where =3D ELEVATOR_INSERT_SORT;
-	struct request *req;
+	struct request *req, *free;
 	unsigned int request_count =3D 0;
 	unsigned int wb_acct;

=20

block/blk-core.c +0 -0
@@ -1632,15 +1632,21 @@ static blk_qc_t blk_queue_bio(struct =

request_queue *q, struct bio *bio)

block/blk-merge.c static struct request *attempt_merge(struct =

Quoted 28 lines
block/blk-core.c +8 -2
 	if (el_ret =3D=3D ELEVATOR_BACK_MERGE) {
 		if (bio_attempt_back_merge(q, req, bio)) {
 			elv_bio_merged(q, req, bio);
-			if (!attempt_back_merge(q, req))
+			free =3D attempt_back_merge(q, req);
+			if (!free)
 				elv_merged_request(q, req, el_ret);
+			else
+				__blk_put_request(q, free);
 			goto out_unlock;
 		}
 	} else if (el_ret =3D=3D ELEVATOR_FRONT_MERGE) {
 		if (bio_attempt_front_merge(q, req, bio)) {
 			elv_bio_merged(q, req, bio);
-			if (!attempt_front_merge(q, req))
+			free =3D attempt_front_merge(q, req);
+			if (!free)
 				elv_merged_request(q, req, el_ret);
+			else
+				__blk_put_request(q, free);
 			goto out_unlock;
 		}
 	}
block/blk-merge.c +0 -0
index 3826fc32b72c..a373416dbc9a 100644
@@ -733,9 +733,11 @@ static struct request *attempt_merge(struct =

request_queue *q,

block/blk-merge.c int blk_attempt_req_merge(struct request_queue =

Quoted 14 lines
block/blk-merge.c +0 -0
 	if (blk_rq_cpu_valid(next))
 		req->cpu =3D next->cpu;

=20

block/blk-merge.c +4 -1
-	/* owner-ship of bio passed from next to req */
+	/*
+	 * ownership of bio passed from next to req, return 'next' for
+	 * the caller to free
+	 */
	next->bio =3D NULL;
block/blk-merge.c +0 -1
-	__blk_put_request(q, next);
	return next;
}

=20

block/blk-merge.c +0 -0
@@ -763,12 +765,19 @@ int blk_attempt_req_merge(struct request_queue =

*q, struct request *rq,

Quoted 7 lines
block/blk-merge.c +1 -0
 			  struct request *next)
 {
 	struct elevator_queue *e =3D q->elevator;
+	struct request *free;

=20

	if (!e->uses_mq && e->type->ops.sq.elevator_allow_rq_merge_fn)
		if (!e->type->ops.sq.elevator_allow_rq_merge_fn(q, rq, =

next))

Quoted 10 lines
			return 0;

=20

block/blk-merge.c +7 -1
-	return attempt_merge(q, rq, next) !=3D NULL;
+	free =3D attempt_merge(q, rq, next);
+	if (free) {
+		__blk_put_request(q, free);
+		return 1;
+	}
+
+	return 0;
Quoted 2 lines

}
=20

Hi Jens,
blk_attempt_req_merge is reached also from blk_mq_sched_try_insert_merge, which is invoked in mq-deadline. According to the current usage of the scheduler lock in mq-deadline, that lock should be taken before invoking blk_mq_sched_try_insert_merge too, as I pointed out in a previous reply of mine. But, if it is taken, then we have again the double-lock-state problem in put_rq_private.

Unfortunately, from your general explanation on how to best protect shared data structures, I did not understand (sorry) whether the scheduler lock must or must not be taken around blk_mq_sched_try_insert_merge in the current implementation of mq-deadline. Could you patiently provide a thickheaded-proof, yes/no answer, if possible?

Thanks,
Paolo

block/blk-mq-sched.c void blk_mq_sched_move_to_dispatch(struct =

Quoted 6 lines

bool blk_rq_merge_ok(struct request *rq, struct bio *bio)

block/blk-mq-sched.c +0 -0
index 114814ec3d49..d93b56d53c4e 100644
@@ -234,7 +234,8 @@ void blk_mq_sched_move_to_dispatch(struct =

blk_mq_hw_ctx *hctx,

block/blk-mq-sched.c bool blk_mq_sched_try_merge(struct request_queue =

Quoted 10 lines
block/blk-mq-sched.c +0 -0
 }
 EXPORT_SYMBOL_GPL(blk_mq_sched_move_to_dispatch);

=20

block/blk-mq-sched.c +2 -1
-bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio)
+bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio,
+			    struct request **merged_request)
{
	struct request *rq;
	int ret;
block/blk-mq-sched.c +0 -0
@@ -244,7 +245,8 @@ bool blk_mq_sched_try_merge(struct request_queue =

*q, struct bio *bio)

block/blk-mq-sched.c bool blk_mq_sched_try_merge(struct request_queue =

Quoted 10 lines
block/blk-mq-sched.c +2 -1
 		if (!blk_mq_sched_allow_merge(q, rq, bio))
 			return false;
 		if (bio_attempt_back_merge(q, rq, bio)) {
-			if (!attempt_back_merge(q, rq))
+			*merged_request =3D attempt_back_merge(q, rq);
+			if (!*merged_request)
 				elv_merged_request(q, rq, ret);
 			return true;
 		}
@@ -252,7 +254,8 @@ bool blk_mq_sched_try_merge(struct request_queue =

*q, struct bio *bio)

Quoted 14 lines
block/blk-mq-sched.c +2 -1
 		if (!blk_mq_sched_allow_merge(q, rq, bio))
 			return false;
 		if (bio_attempt_front_merge(q, rq, bio)) {
-			if (!attempt_front_merge(q, rq))
+			*merged_request =3D attempt_front_merge(q, rq);
+			if (!*merged_request)
 				elv_merged_request(q, rq, ret);
 			return true;
 		}
block/blk-mq-sched.h +0 -0
index 9478aaeb48c5..3643686a54b8 100644
@@ -16,7 +16,8 @@ void blk_mq_sched_put_request(struct request *rq);
Quoted 3 lines

=20
void blk_mq_sched_request_inserted(struct request *rq); bool blk_mq_sched_bypass_insert(struct blk_mq_hw_ctx *hctx, struct =

request *rq);

Quoted 1 line

-bool blk_mq_sched_try_merge(struct request_queue *q, struct bio =

*bio);

Quoted 3 lines
block/blk-mq-sched.h +2 -0
+bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio,
+				struct request **merged_request);

bool __blk_mq_sched_bio_merge(struct request_queue *q, struct bio =

*bio);

Quoted 1 line

bool blk_mq_sched_try_insert_merge(struct request_queue *q, struct =

request *rq);

block/mq-deadline.c static bool dd_bio_merge(struct blk_mq_hw_ctx =

Quoted 6 lines

void blk_mq_sched_restart_queues(struct blk_mq_hw_ctx *hctx);

block/mq-deadline.c +0 -0
index 49583536698c..682fa64f55ff 100644
@@ -371,12 +371,16 @@ static bool dd_bio_merge(struct blk_mq_hw_ctx =

*hctx, struct bio *bio)

Quoted 21 lines
block/mq-deadline.c +2 -1
 {
 	struct request_queue *q =3D hctx->queue;
 	struct deadline_data *dd =3D q->elevator->elevator_data;
-	int ret;
+	struct request *free =3D NULL;
+	bool ret;

=20

	spin_lock(&dd->lock);
block/mq-deadline.c +1 -1
-	ret =3D blk_mq_sched_try_merge(q, bio);
+	ret =3D blk_mq_sched_try_merge(q, bio, &free);
	spin_unlock(&dd->lock);

=20

block/mq-deadline.c +3 -0
+	if (free)
+		blk_mq_free_request(free);
+
	return ret;
}

=20
--=20
2.7.4
=20