Skip to contentSkip to conversation

blk-mq: avoid double ->queue_rq() because of early timeout

PATCH Superseded Ming Lei 2 participants 4 messages Oct 26, 2022

Subsystems (2)

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore

Ming Lei

author
#0 lore raw

PATCH v2 1/1 blk-mq: avoid double ->queue_rq() because of early timeout

From: David Jeffery <[email protected]>

David Jeffery found one double ->queue_rq() issue, so far it can be triggered in VM use case because of long vmexit latency or preempt latency of vCPU pthread or long page fault in vCPU pthread, then block IO req could be timed out before queuing the request to hardware but after calling blk_mq_start_request() during ->queue_rq(), then timeout handler may handle it by requeue, then double ->queue_rq() is caused, and kernel panic.

So far, it is driver's responsibility to cover the race between timeout and completion, so it seems supposed to be solved in driver in theory, given driver has enough knowledge.

But it is really one common problem, lots of driver could have similar issue, and could be hard to fix all affected drivers, even it isn't easy for driver to handle the race. So David suggests this patch by draining in-progress ->queue_rq() for solving this issue.

Cc Stefan Hajnoczi <[email protected]>
Cc Keith Busch <[email protected]>
Cc Bart Van Assche <[email protected]>

---
V2:
- follow Jens's suggestion to run sync rcu only if there is timeout
- rename 'now' as 'start_timeout'

 block/blk-mq.c | 53 ++++++++++++++++++++++++++++++++++++++------------
 1 file changed, 41 insertions(+), 12 deletions(-)
block/blk-mq.c +41 -12
index 33292c01875d..431e71af0e06 100644
@@ -1523,7 +1523,14 @@ static void blk_mq_rq_timed_out(struct request *req)
 	blk_add_timer(req);
 }
 
-static bool blk_mq_req_expired(struct request *rq, unsigned long *next)
+struct blk_expired_data {
+	bool check_only;
+	bool has_timedout_rq;
+	unsigned long next;
+	unsigned long timeout_start;
+};
+
+static bool blk_mq_req_expired(struct request *rq, struct blk_expired_data *expired)
 {
 	unsigned long deadline;
 
@@ -1533,13 +1540,13 @@ static bool blk_mq_req_expired(struct request *rq, unsigned long *next)
 		return false;
 
 	deadline = READ_ONCE(rq->deadline);
-	if (time_after_eq(jiffies, deadline))
+	if (time_after_eq(expired->timeout_start, deadline))
 		return true;
 
-	if (*next == 0)
-		*next = deadline;
-	else if (time_after(*next, deadline))
-		*next = deadline;
+	if (expired->next == 0)
+		expired->next = deadline;
+	else if (time_after(expired->next, deadline))
+		expired->next = deadline;
 	return false;
 }
 
@@ -1555,7 +1562,7 @@ void blk_mq_put_rq_ref(struct request *rq)
 
 static bool blk_mq_check_expired(struct request *rq, void *priv)
 {
-	unsigned long *next = priv;
+	struct blk_expired_data *expired = priv;
 
 	/*
 	 * blk_mq_queue_tag_busy_iter() has locked the request, so it cannot
@@ -1564,8 +1571,13 @@ static bool blk_mq_check_expired(struct request *rq, void *priv)
 	 * it was completed and reallocated as a new request after returning
 	 * from blk_mq_check_expired().
 	 */
-	if (blk_mq_req_expired(rq, next))
+	if (blk_mq_req_expired(rq, expired)) {
+		if (expired->check_only) {
+			expired->has_timedout_rq = true;
+			return false;
+		}
 		blk_mq_rq_timed_out(rq);
+	}
 	return true;
 }
 
@@ -1573,7 +1585,10 @@ static void blk_mq_timeout_work(struct work_struct *work)
 {
 	struct request_queue *q =
 		container_of(work, struct request_queue, timeout_work);
-	unsigned long next = 0;
+	struct blk_expired_data expired = {
+		.check_only = true,
+		.timeout_start = jiffies,
+	};
 	struct blk_mq_hw_ctx *hctx;
 	unsigned long i;
 
@@ -1593,10 +1608,24 @@ static void blk_mq_timeout_work(struct work_struct *work)
 	if (!percpu_ref_tryget(&q->q_usage_counter))
 		return;
 
-	blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &next);
+	/* check if there is any timed-out request */
+	blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &expired);
+	if (expired.has_timedout_rq) {
+		/*
+		 * Before walking tags, we must ensure any submit started
+		 * before the current time has finished. Since the submit
+		 * uses srcu or rcu, wait for a synchronization point to
+		 * ensure all running submits have finished
+		 */
+		blk_mq_wait_quiesce_done(q);
+
+		expired.check_only = false;
+		expired.next = 0;
+		blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &expired);
+	}
 
-	if (next != 0) {
-		mod_timer(&q->timeout, next);
+	if (expired.next != 0) {
+		mod_timer(&q->timeout, expired.next);
 	} else {
 		/*
 		 * Request timeouts are handled as a forward rolling timer. If
Signature · 1 line
2.31.1
#1 lore raw
Quoted 36 lines · Ming Lei
block/blk-mq.c +14 -3
@@ -1564,8 +1571,13 @@ static bool blk_mq_check_expired(struct request *rq, void *priv)
 	 * it was completed and reallocated as a new request after returning
 	 * from blk_mq_check_expired().
 	 */
-	if (blk_mq_req_expired(rq, next))
+	if (blk_mq_req_expired(rq, expired)) {
+		if (expired->check_only) {
+			expired->has_timedout_rq = true;
+			return false;
+		}
 		blk_mq_rq_timed_out(rq);
+	}
 	return true;
 }
 
@@ -1573,7 +1585,10 @@ static void blk_mq_timeout_work(struct work_struct *work)
 {
 	struct request_queue *q =
 		container_of(work, struct request_queue, timeout_work);
-	unsigned long next = 0;
+	struct blk_expired_data expired = {
+		.check_only = true,
+		.timeout_start = jiffies,
+	};
 	struct blk_mq_hw_ctx *hctx;
 	unsigned long i;
 
@@ -1593,10 +1608,24 @@ static void blk_mq_timeout_work(struct work_struct *work)
 	if (!percpu_ref_tryget(&q->q_usage_counter))
 		return;
 
-	blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &next);
+	/* check if there is any timed-out request */
+	blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &expired);
+	if (expired.has_timedout_rq) {
+		/*

block/blk-mq.c static void blk_mq_timeout_work(struct work_struct *work) · line 1624

Quoted 10 lines
block/blk-mq.c +10 -0
+		 * Before walking tags, we must ensure any submit started
+		 * before the current time has finished. Since the submit
+		 * uses srcu or rcu, wait for a synchronization point to
+		 * ensure all running submits have finished
+		 */
+		blk_mq_wait_quiesce_done(q);
+
+		expired.check_only = false;
+		expired.next = 0;
+		blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &expired);

I think it would be easier to follow with separate callbacks instead of special casing for 'check_only'. One callback for checking timeouts, and a different one for handling them?

author
#2 lore raw
Quoted 43 lines · Keith Busch
Quoted 42 lines · Ming Lei
block/blk-mq.c +20 -3
@@ -1564,8 +1571,13 @@ static bool blk_mq_check_expired(struct request *rq, void *priv)
 	 * it was completed and reallocated as a new request after returning
 	 * from blk_mq_check_expired().
 	 */
-	if (blk_mq_req_expired(rq, next))
+	if (blk_mq_req_expired(rq, expired)) {
+		if (expired->check_only) {
+			expired->has_timedout_rq = true;
+			return false;
+		}
 		blk_mq_rq_timed_out(rq);
+	}
 	return true;
 }
 
@@ -1573,7 +1585,10 @@ static void blk_mq_timeout_work(struct work_struct *work)
 {
 	struct request_queue *q =
 		container_of(work, struct request_queue, timeout_work);
-	unsigned long next = 0;
+	struct blk_expired_data expired = {
+		.check_only = true,
+		.timeout_start = jiffies,
+	};
 	struct blk_mq_hw_ctx *hctx;
 	unsigned long i;
 
@@ -1593,10 +1608,24 @@ static void blk_mq_timeout_work(struct work_struct *work)
 	if (!percpu_ref_tryget(&q->q_usage_counter))
 		return;
 
-	blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &next);
+	/* check if there is any timed-out request */
+	blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &expired);
+	if (expired.has_timedout_rq) {
+		/*
+		 * Before walking tags, we must ensure any submit started
+		 * before the current time has finished. Since the submit
+		 * uses srcu or rcu, wait for a synchronization point to
+		 * ensure all running submits have finished
+		 */
+		blk_mq_wait_quiesce_done(q);
Quoted 8 lines

block/blk-mq.c static void blk_mq_timeout_work(struct work_struct *work) · line 1624

Quoted 4 lines
block/blk-mq.c +4 -0
+
+		expired.check_only = false;
+		expired.next = 0;
+		blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &expired);

I think it would be easier to follow with separate callbacks instead of special casing for 'check_only'. One callback for checking timeouts, and a different one for handling them?

Both two are basically same, with two callbacks, just .check_only is saved, nothing else, meantime with one extra similar callback added.

If you or anyone think it is one big deal, I can switch to two callback version.

Thanks,
Ming

_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

#3 lore raw
Quoted 2 lines · Ming Lei

Both two are basically same, with two callbacks, just .check_only is saved, nothing else, meantime with one extra similar callback added.

They don't seem to share much at all. The original callback was 4 lines of code, and the special case adds 5 new lines that returns before half of the function's pre-existing code.