block: Reorder the request allocation code in blk_mq_submit_bio()
PATCH 1/3 block: Reorder the request allocation code in blk_mq_submit_bio()
From: Bart Van Assche <[email protected]>
[ Upstream commit 44e41381591dc5b4ea67a9f170b4ec85c817586e ]
Help the CPU branch predictor in case of a cache hit by handling the cache hit scenario first.
Stable-dep-of: dc278e9bf2b9 ("blk-mq: pop cached request if it is usable")
--- block/blk-mq.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
block/blk-mq.c +3 -3
@@ -3121,12 +3121,12 @@ void blk_mq_submit_bio(struct bio *bio) } new_request: -if (!rq) { +if (rq) { +blk_mq_use_cached_rq (rq, plug, bio); + }else { rq =blk_mq_get_new_requests (q, plug, bio, nr_segs);if (unlikely (!rq)) goto queue_exit; - }else { -blk_mq_use_cached_rq (rq, plug, bio); }trace_block_getrq (bio);
base-commit: 25c09b42358e73e1476e517b296edb6344f2e4bd
Signature · 1 line
2.50.1
PATCH 2/3 blk-mq: pop cached request if it is usable
From: Keith Busch <[email protected]>
[ Upstream commit dc278e9bf2b9513a763353e6b9cc21e0f532954e ]
When submitting a bio to blk-mq, if the task should sleep after peeking a cached request, but before it pops it, the plug flushes and calls blk_mq_free_plug_rqs, freeing the cached_rqs. This creates a use-after-free bug. Fix this by popping the cached request before any possible blocking calls if it is suitable for use.
Popping this request first holds a queue reference, so avoid any serialization races with queue freezes and can safely proceed with dispatching that request to the driver. This potentially increases a timing window from when a driver wants to freeze its queue to when requests stop being dispatched. That scenario is off the fast path though, and drivers need to appropriately handle requests during a freeze request anyway.
The downside is the popped element needs to be individually freed when we performed a bio plug merge. The cached request would have had to be freed later anyway, but this patch does it inline with building the plug list instead of after flushing it.
--- block/blk-mq.c | 34 +++++++++------------------------- 1 file changed, 9 insertions(+), 25 deletions(-)
block/blk-mq.c +9 -25
@@ -2989,7 +2989,7 @@ static struct request *blk_mq_get_new_requests(struct request_queue *q,/* * Check if there is a suitable cached request and return it. */ -static struct request *blk_mq_peek_cached_request (struct blk_plug *plug, +static struct request *blk_mq_get_cached_request (struct blk_plug *plug,struct request_queue *q,blk_opf_t opf) {enum hctx_type type =blk_mq_get_hctx_type (opf);
@@ -3005,27 +3005,10 @@ static struct request *blk_mq_peek_cached_request(struct blk_plug *plug,return NULL ;if (op_is_flush (rq->cmd_flags) !=op_is_flush (opf))return NULL ; +rq_list_pop (&plug->cached_rqs);return rq; } -static void blk_mq_use_cached_rq (struct request *rq,struct blk_plug *plug, -struct bio *bio) -{ -if (rq_list_pop (&plug->cached_rqs) != rq) -WARN_ON_ONCE (1 ); - -/* -* If any qos ->throttle() end up blocking, we will have flushed the -* plug and hence killed the cached_rq list as well. Pop this entry -* before we throttle. -*/ -rq_qos_throttle (rq->q, bio); - -blk_mq_rq_time_init (rq,0 ); - rq->cmd_flags = bio->bi_opf; -INIT_LIST_HEAD (&rq->queuelist); -} -static bool bio_unaligned (const struct bio *bio,struct request_queue *q) {unsigned int bs_mask =queue_logical_block_size (q) -1 ;
@@ -3063,7 +3046,7 @@ void blk_mq_submit_bio(struct bio *bio)/* * If the plug has a cached request for this queue, try to use it. */ - rq =blk_mq_peek_cached_request (plug, q, bio->bi_opf); + rq =blk_mq_get_cached_request (plug, q, bio->bi_opf);/* * A BIO that was released from a zone write plug has already been
@@ -3122,7 +3105,10 @@ void blk_mq_submit_bio(struct bio *bio) new_request:if (rq) { -blk_mq_use_cached_rq (rq, plug, bio); +rq_qos_throttle (rq->q, bio); +blk_mq_rq_time_init (rq,blk_time_get_ns ()); + rq->cmd_flags = bio->bi_opf; +INIT_LIST_HEAD (&rq->queuelist); }else { rq =blk_mq_get_new_requests (q, plug, bio, nr_segs);if (unlikely (!rq))
@@ -3165,12 +3151,10 @@ void blk_mq_submit_bio(struct bio *bio)return ; queue_exit: -/* -* Don't drop the queue reference if we were trying to use a cached -* request and thus didn't acquire one. -*/ if (!rq)blk_queue_exit (q); +else +blk_mq_free_request (rq); }#ifdef CONFIG_BLK_MQ_STACKING
Signature · 1 line
2.50.1
PATCH 3/3 blk-mq: reinsert cached request to the list
In reply to Simon LieboldFrom: Keith Busch <[email protected]>
[ Upstream commit b051bb6bf0a231117036aa607cadf55be8e63910 ]
A previous commit removed an optimization out of caution for a scenario that turns out not to be real: all the "queue_exit" goto's are safe to reinsert the request into the cached_rq's plug list as they are either from a non-blocking path, or a successful merge that already holds the queue reference. This optimization is most needed for small sequential workloads that successfully merge into larger requests.
--- block/blk-mq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
block/blk-mq.c +1 -1
@@ -3154,7 +3154,7 @@ void blk_mq_submit_bio(struct bio *bio)if (!rq)blk_queue_exit (q);else -blk_mq_free_request (rq); +rq_list_add_head (&plug->cached_rqs, rq); }#ifdef CONFIG_BLK_MQ_STACKING
Signature · 1 line
2.50.1
Quoted 6 lines
From: Bart Van Assche <[email protected]>
[ Upstream commit
44e41381591dc5b4ea67a9f170b4ec85c817586e]Help the CPU branch predictor in case of a cache hit by handling the cache hit scenario first.
Queued the series for 6.12, thanks.
Signature · 2 lines
Thanks, Sasha