block: avoid hctx spinlock for plug with multiple queues
v2 cover block: avoid hctx spinlock for plug with multiple queues
blk_mq_flush_plug_list() has a fast path if all requests in the plug are destined for the same request_queue. It calls ->queue_rqs() with the whole batch of requests, falling back on ->queue_rq() for any requests not handled by ->queue_rqs(). However, if the requests are destined for multiple queues, blk_mq_flush_plug_list() has a slow path that calls blk_mq_dispatch_list() repeatedly to filter the requests by ctx/hctx. Each queue's requests are inserted into the hctx's dispatch list under a spinlock, then __blk_mq_sched_dispatch_requests() takes them out of the dispatch list (taking the spinlock again), and finally blk_mq_dispatch_rq_list() calls ->queue_rq() on each request.
Acquiring the hctx spinlock twice and calling ->queue_rq() instead of ->queue_rqs() makes the slow path significantly more expensive. Thus, batching more requests into a single plug (e.g. io_uring_enter syscall) can counterintuitively hurt performance by causing the plug to span multiple queues. We have observed 2-3% of CPU time spent acquiring the hctx spinlock alone on workloads issuing requests to multiple NVMe devices in the same io_uring SQE batches.
Add a medium path in blk_mq_flush_plug_list() for plugs that don't have elevators or come from a schedule, but do span multiple queues. Filter the requests by queue and call ->queue_rqs()/->queue_rq() on the list of requests destined to each request_queue.
With this change, we no longer see any CPU time spent in _raw_spin_lock from blk_mq_flush_plug_list and throughput increases accordingly.
Caleb Sander Mateos (3):
block: take rq_list instead of plug in dispatch functions
block: factor out blk_mq_dispatch_queue_requests() helper
block: avoid hctx spinlock for plug with multiple queues
block/blk-mq.c | 110 +++++++++++++++++++++++++++++++------------- block/mq-deadline.c | 2 +- 2 files changed, 79 insertions(+), 33 deletions(-)
v2:
- Leave unmatched requests in plug list instead of building a new list
- Add Reviewed-by tags
Signature · 1 line
2.45.2
PATCH v2 1/3 block: take rq_list instead of plug in dispatch functions
blk_mq_plug_issue_direct(), __blk_mq_flush_plug_list(), and blk_mq_dispatch_plug_list() take a struct blk_plug * but only use its mq_list. Pass the struct rq_list * instead in preparation for calling them with other lists of requests.
Drop "plug" from the function names as they are no longer plug-specific.
--- block/blk-mq.c | 26 ++++++++++++-------------- block/mq-deadline.c | 2 +- 2 files changed, 13 insertions(+), 15 deletions(-)
block/blk-mq.c +12 -14
@@ -2788,19 +2788,19 @@ static blk_status_t blk_mq_request_issue_directly(struct request *rq, bool last)if (!blk_mq_get_budget_and_tag (rq))return BLK_STS_RESOURCE ;return __blk_mq_issue_directly (hctx, rq, last); } -static void blk_mq_plug_issue_direct (struct blk_plug *plug) +static void blk_mq_issue_direct (struct rq_list *rqs) {struct blk_mq_hw_ctx *hctx =NULL ;struct request *rq;int queued =0 ;blk_status_t ret =BLK_STS_OK ; -while ((rq =rq_list_pop (&plug->mq_list))) { -bool last =rq_list_empty (&plug->mq_list); +while ((rq =rq_list_pop (rqs))) { +bool last =rq_list_empty (rqs);if (hctx != rq->mq_hctx) {if (hctx) {blk_mq_commit_rqs (hctx, queued, false); queued =0 ;
@@ -2827,29 +2827,28 @@ static void blk_mq_plug_issue_direct(struct blk_plug *plug) out:if (ret !=BLK_STS_OK )blk_mq_commit_rqs (hctx, queued, false); } -static void __blk_mq_flush_plug_list (struct request_queue *q, -struct blk_plug *plug) +static void __blk_mq_flush_list (struct request_queue *q,struct rq_list *rqs) {if (blk_queue_quiesced (q))return ; - q->mq_ops->queue_rqs (&plug->mq_list); + q->mq_ops->queue_rqs (rqs); } -static void blk_mq_dispatch_plug_list (struct blk_plug *plug,bool from_sched) +static void blk_mq_dispatch_list (struct rq_list *rqs,bool from_sched) {struct blk_mq_hw_ctx *this_hctx =NULL ;struct blk_mq_ctx *this_ctx =NULL ;struct rq_list requeue_list = {};unsigned int depth =0 ;bool is_passthrough = false;LIST_HEAD (list); do { - struct request *rq = rq_list_pop(&plug->mq_list); + struct request *rq = rq_list_pop(rqs); if (!this_hctx) { this_hctx = rq->mq_hctx; this_ctx = rq->mq_ctx; is_passthrough = blk_rq_is_passthrough(rq);
@@ -2858,13 +2857,13 @@ static void blk_mq_dispatch_plug_list(struct blk_plug *plug, bool from_sched)rq_list_add_tail (&requeue_list, rq);continue ; }list_add_tail (&rq->queuelist, &list); depth++; - }while (!rq_list_empty (&plug->mq_list)); + }while (!rq_list_empty (rqs)); - plug->mq_list = requeue_list; + *rqs = requeue_list;trace_block_unplug (this_hctx->queue, depth, !from_sched);percpu_ref_get (&this_hctx->queue->q_usage_counter);/* passthrough requests should never be issued to the I/O scheduler */ if (is_passthrough) {
@@ -2912,23 +2911,22 @@ void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)* already know at this point that all requests belong to the * same queue, caller must ensure that's the case. */ if (q->mq_ops->queue_rqs) {blk_mq_run_dispatch_ops (q, -__blk_mq_flush_plug_list (q, plug)); +__blk_mq_flush_list (q, &plug->mq_list));if (rq_list_empty (&plug->mq_list))return ; } -blk_mq_run_dispatch_ops (q, -blk_mq_plug_issue_direct (plug)); +blk_mq_run_dispatch_ops (q,blk_mq_issue_direct (&plug->mq_list));if (rq_list_empty (&plug->mq_list))return ; }do { -blk_mq_dispatch_plug_list (plug, from_schedule); +blk_mq_dispatch_list (&plug->mq_list, from_schedule); }while (!rq_list_empty (&plug->mq_list)); }static void blk_mq_try_issue_list_directly (struct blk_mq_hw_ctx *hctx,struct list_head *list)
block/mq-deadline.c +1 -1
@@ -713,11 +713,11 @@ static void dd_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,list_add_tail (&rq->queuelist, &per_prio->fifo_list[data_dir]); } }/* -* Called from blk_mq_insert_request() or blk_mq_dispatch_plug_list(). +* Called from blk_mq_insert_request() or blk_mq_dispatch_list(). */ static void dd_insert_requests (struct blk_mq_hw_ctx *hctx,struct list_head *list,blk_insert_t flags) {
Signature · 1 line
2.45.2
PATCH v2 2/3 block: factor out blk_mq_dispatch_queue_requests() helper
In reply to Caleb Sander MateosFactor out the logic from blk_mq_flush_plug_list() that calls ->queue_rqs() with a fallback to ->queue_rq() into a helper function blk_mq_dispatch_queue_requests(). This is in preparation for using this code with other lists of requests.
--- block/blk-mq.c | 43 ++++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-)
block/blk-mq.c +22 -21
@@ -2834,10 +2834,31 @@ static void __blk_mq_flush_list(struct request_queue *q, struct rq_list *rqs)if (blk_queue_quiesced (q))return ; q->mq_ops->queue_rqs (rqs); } +static void blk_mq_dispatch_queue_requests (struct rq_list *rqs,unsigned depth) +{ +struct request_queue *q =rq_list_peek (rqs)->q; + +trace_block_unplug (q, depth, true); + +/* +* Peek first request and see if we have a ->queue_rqs() hook. +* If we do, we can dispatch the whole list in one go. +* We already know at this point that all requests belong to the +* same queue, caller must ensure that's the case. +*/ +if (q->mq_ops->queue_rqs) { +blk_mq_run_dispatch_ops (q,__blk_mq_flush_list (q, rqs)); +if (rq_list_empty (rqs)) +return ; + } + +blk_mq_run_dispatch_ops (q,blk_mq_issue_direct (rqs)); +} +static void blk_mq_dispatch_list (struct rq_list *rqs,bool from_sched) {struct blk_mq_hw_ctx *this_hctx =NULL ;struct blk_mq_ctx *this_ctx =NULL ;struct rq_list requeue_list = {};
@@ -2881,11 +2902,10 @@ static void blk_mq_dispatch_list(struct rq_list *rqs, bool from_sched)percpu_ref_put (&this_hctx->queue->q_usage_counter); }void blk_mq_flush_plug_list (struct blk_plug *plug,bool from_schedule) { -struct request *rq;unsigned int depth;/* * We may have been called recursively midway through handling * plug->mq_list via a schedule() in the driver's queue_rq() callback.
@@ -2897,30 +2917,11 @@ void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)return ; depth = plug->rq_count; plug->rq_count =0 ;if (!plug->multiple_queues && !plug->has_elevator && !from_schedule) { -struct request_queue *q; - - rq =rq_list_peek (&plug->mq_list); - q = rq->q; -trace_block_unplug (q, depth, true); - -/* -* Peek first request and see if we have a ->queue_rqs() hook. -* If we do, we can dispatch the whole plug list in one go. We -* already know at this point that all requests belong to the -* same queue, caller must ensure that's the case. -*/ -if (q->mq_ops->queue_rqs) { -blk_mq_run_dispatch_ops (q, -__blk_mq_flush_list (q, &plug->mq_list)); -if (rq_list_empty (&plug->mq_list)) -return ; - } - -blk_mq_run_dispatch_ops (q,blk_mq_issue_direct (&plug->mq_list)); +blk_mq_dispatch_queue_requests (&plug->mq_list, depth);if (rq_list_empty (&plug->mq_list))return ; } do {
Signature · 1 line
2.45.2
PATCH v2 3/3 block: avoid hctx spinlock for plug with multiple queues
In reply to Caleb Sander Mateosblk_mq_flush_plug_list() has a fast path if all requests in the plug are destined for the same request_queue. It calls ->queue_rqs() with the whole batch of requests, falling back on ->queue_rq() for any requests not handled by ->queue_rqs(). However, if the requests are destined for multiple queues, blk_mq_flush_plug_list() has a slow path that calls blk_mq_dispatch_list() repeatedly to filter the requests by ctx/hctx. Each queue's requests are inserted into the hctx's dispatch list under a spinlock, then __blk_mq_sched_dispatch_requests() takes them out of the dispatch list (taking the spinlock again), and finally blk_mq_dispatch_rq_list() calls ->queue_rq() on each request.
Acquiring the hctx spinlock twice and calling ->queue_rq() instead of ->queue_rqs() makes the slow path significantly more expensive. Thus, batching more requests into a single plug (e.g. io_uring_enter syscall) can counterintuitively hurt performance by causing the plug to span multiple queues. We have observed 2-3% of CPU time spent acquiring the hctx spinlock alone on workloads issuing requests to multiple NVMe devices in the same io_uring SQE batches.
Add a medium path in blk_mq_flush_plug_list() for plugs that don't have elevators or come from a schedule, but do span multiple queues. Filter the requests by queue and call ->queue_rqs()/->queue_rq() on the list of requests destined to each request_queue.
With this change, we no longer see any CPU time spent in _raw_spin_lock from blk_mq_flush_plug_list and throughput increases accordingly.
--- block/blk-mq.c | 49 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-)
block/blk-mq.c +48 -1
@@ -2834,10 +2834,39 @@ static void __blk_mq_flush_list(struct request_queue *q, struct rq_list *rqs)if (blk_queue_quiesced (q))return ; q->mq_ops->queue_rqs (rqs); } +static unsigned blk_mq_extract_queue_requests (struct rq_list *rqs, +struct rq_list *queue_rqs) +{ +struct request *rq =rq_list_pop (rqs); +struct request_queue *this_q = rq->q; +struct request **prev = &rqs->head; +struct rq_list matched_rqs = {}; +struct request *last =NULL ; +unsigned depth =1 ; + +rq_list_add_tail (&matched_rqs, rq); +while ((rq = *prev)) { +if (rq->q == this_q) { +/* move rq from rqs to matched_rqs */ + *prev = rq->rq_next; +rq_list_add_tail (&matched_rqs, rq); + depth++; + }else { +/* leave rq in rqs */ + prev = &rq->rq_next; + last = rq; + } + } + + rqs->tail = last; + *queue_rqs = matched_rqs; +return depth; +} +static void blk_mq_dispatch_queue_requests (struct rq_list *rqs,unsigned depth) {struct request_queue *q =rq_list_peek (rqs)->q;trace_block_unplug (q, depth, true);
@@ -2900,10 +2929,23 @@ static void blk_mq_dispatch_list(struct rq_list *rqs, bool from_sched)blk_mq_insert_requests (this_hctx, this_ctx, &list, from_sched); }percpu_ref_put (&this_hctx->queue->q_usage_counter); } +static void blk_mq_dispatch_multiple_queue_requests (struct rq_list *rqs) +{ +do { +struct rq_list queue_rqs; +unsigned depth; + + depth =blk_mq_extract_queue_requests (rqs, &queue_rqs); +blk_mq_dispatch_queue_requests (&queue_rqs, depth); +while (!rq_list_empty (&queue_rqs)) +blk_mq_dispatch_list (&queue_rqs, false); + }while (!rq_list_empty (rqs)); +} +void blk_mq_flush_plug_list (struct blk_plug *plug,bool from_schedule) {unsigned int depth;/*
@@ -2916,11 +2958,16 @@ void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)if (plug->rq_count ==0 )return ; depth = plug->rq_count; plug->rq_count =0 ; -if (!plug->multiple_queues && !plug->has_elevator && !from_schedule) { +if (!plug->has_elevator && !from_schedule) { +if (plug->multiple_queues) { +blk_mq_dispatch_multiple_queue_requests (&plug->mq_list); +return ; + } +blk_mq_dispatch_queue_requests (&plug->mq_list, depth);if (rq_list_empty (&plug->mq_list))return ; }
Signature · 1 line
2.45.2
Looks good:
Hi Jens,
Christoph has reviewed this series. Would you mind queueing it up for 6.16?
Thanks,
Caleb
Quoted 44 lines · Caleb Sander Mateos
blk_mq_flush_plug_list()has a fast path if all requests in the plug are destined for the samerequest_queue. It calls ->queue_rqs() with the whole batch of requests, falling back on ->queue_rq() for any requests not handled by ->queue_rqs(). However, if the requests are destined for multiple queues,blk_mq_flush_plug_list()has a slow path that callsblk_mq_dispatch_list()repeatedly to filter the requests by ctx/hctx. Each queue's requests are inserted into the hctx's dispatch list under a spinlock, then__blk_mq_sched_dispatch_requests()takes them out of the dispatch list (taking the spinlock again), and finallyblk_mq_dispatch_rq_list()calls ->queue_rq() on each request.Acquiring the hctx spinlock twice and calling ->queue_rq() instead of ->queue_rqs() makes the slow path significantly more expensive. Thus, batching more requests into a single plug (e.g.
io_uring_entersyscall) can counterintuitively hurt performance by causing the plug to span multiple queues. We have observed 2-3% of CPU time spent acquiring the hctx spinlock alone on workloads issuing requests to multiple NVMe devices in the sameio_uringSQE batches.Add a medium path in
blk_mq_flush_plug_list()for plugs that don't have elevators or come from a schedule, but do span multiple queues. Filter the requests by queue and call ->queue_rqs()/->queue_rq() on the list of requests destined to eachrequest_queue.With this change, we no longer see any CPU time spent in
_raw_spin_lockfromblk_mq_flush_plug_listand throughput increases accordingly.Caleb Sander Mateos (3):
block: takerq_listinstead of plug in dispatch functions
block: factor outblk_mq_dispatch_queue_requests()helper
block: avoid hctx spinlock for plug with multiple queuesblock/blk-mq.c | 110 +++++++++++++++++++++++++++++++------------- block/mq-deadline.c | 2 +- 2 files changed, 79 insertions(+), 33 deletions(-)v2:
- Leave unmatched requests in plug list instead of building a new list
- Add Reviewed-by tags--
2.45.2
Quoted 11 lines · Caleb Sander Mateos
blk_mq_flush_plug_list()has a fast path if all requests in the plug are destined for the samerequest_queue. It calls ->queue_rqs() with the whole batch of requests, falling back on ->queue_rq() for any requests not handled by ->queue_rqs(). However, if the requests are destined for multiple queues,blk_mq_flush_plug_list()has a slow path that callsblk_mq_dispatch_list()repeatedly to filter the requests by ctx/hctx. Each queue's requests are inserted into the hctx's dispatch list under a spinlock, then__blk_mq_sched_dispatch_requests()takes them out of the dispatch list (taking the spinlock again), and finallyblk_mq_dispatch_rq_list()calls ->queue_rq() on each request.
Quoted 1 line
[...]
Applied, thanks!
[1/3] block: take rq_list instead of plug in dispatch functions
commit: 0aeb7ebfc7e3d4bef3542aadd33505452d2f9b82
[2/3] block: factor out blk_mq_dispatch_queue_requests() helper
commit: a5728a1d1ef2d927c67e73fda94946c2c15106df
[3/3] block: avoid hctx spinlock for plug with multiple queues
commit: 9712c57ec1117d6b5fe1cc8ad420049171140b26Best regards,
Signature · 1 line
Jens Axboe