blk-mq: support batching dispatch from scheduler
Subsystems (2)
v4 cover blk-mq: support batching dispatch from scheduler
Hi Jens,
More and more drivers want to get batching requests queued from block layer, such as mmc[1], and tcp based storage drivers[2]. Also current in-tree users have virtio-scsi, virtio-blk and nvme.
For none, we already support batching dispatch.
But for io scheduler, every time we just take one request from scheduler and pass the single request to blk_mq_dispatch_rq_list(). This way makes batching dispatch not possible when io scheduler is applied. One reason is that we don't want to hurt sequential IO performance, becasue IO merge chance is reduced if more requests are dequeued from scheduler queue.
Tries to start the support by dequeuing more requests from scheduler if budget is enough and device isn't busy.
Simple fio test over virtio-scsi shows IO can get improved by 5~10%.
Baolin has tested previous versions and found performance on MMC can be improved.
Patches can be found from the following tree too:
https://github.com/ming1/linux/commits/v5.7-rc-blk-mq-batching-submission
Patch 1 ~ 4 are improvement and cleanup, which can't applied without supporting batching dispatch.
Patch 5 ~ 6 starts to support batching dispatch from scheduler.
[1] https://lore.kernel.org/linux-block/20200512075501.GF1531898@T590/#r
[2] https://lore.kernel.org/linux-block/[email protected]/
V4:
- fix releasing budgets and avoids IO hang(5/6)
- dispatch more batches if the device can accept more(6/6)
- verified by running more tests
V3:
- add reviewed-by tag
- fix one typo
- fix one budget leak issue in case that .queue_rq returned *_RESOURCE in 5/6
V2:
- remove 'got_budget' from blk_mq_dispatch_rq_list
- drop patch for getting driver tag & handling partial dispatch
Ming Lei (6):
blk-mq: pass request queue into get/put budget callback
blk-mq: pass hctx to blk_mq_dispatch_rq_list
blk-mq: move getting driver tag and budget into one helper
blk-mq: remove dead check from blk_mq_dispatch_rq_list
blk-mq: pass obtained budget count to blk_mq_dispatch_rq_list
blk-mq: support batching dispatch in case of io scheduler
block/blk-mq-sched.c | 116 +++++++++++++++++++++++++++++++++++----- block/blk-mq.c | 115 +++++++++++++++++++++++++-------------- block/blk-mq.h | 15 +++--- drivers/scsi/scsi_lib.c | 8 ++- include/linux/blk-mq.h | 4 +- 5 files changed, 187 insertions(+), 71 deletions(-)
Signature · 1 line
2.25.2
PATCH v4 1/6 blk-mq: pass request queue into get/put budget callback
blk-mq budget is abstract from scsi's device queue depth, and it is always per-request-queue instead of hctx.
It can be quite absurd to get a budget from one hctx, then dequeue a request from scheduler queue, and this request may not belong to this hctx, at least for bfq and deadline.
So fix the mess and always pass request queue to get/put budget callback.
--- block/blk-mq-sched.c | 8 ++++---- block/blk-mq.c | 8 ++++---- block/blk-mq.h | 12 ++++-------- drivers/scsi/scsi_lib.c | 8 +++----- include/linux/blk-mq.h | 4 ++-- 5 files changed, 17 insertions(+), 23 deletions(-)
block/blk-mq-sched.c +4 -4
@@ -108,12 +108,12 @@ static int blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx)break ; } -if (!blk_mq_get_dispatch_budget (hctx)) +if (!blk_mq_get_dispatch_budget (q))break ; rq = e->type->ops.dispatch_request (hctx);if (!rq) { -blk_mq_put_dispatch_budget (hctx); +blk_mq_put_dispatch_budget (q);/* * We're releasing without dispatching. Holding the * budget could have blocked any "hctx"s with the
@@ -173,12 +173,12 @@ static int blk_mq_do_dispatch_ctx(struct blk_mq_hw_ctx *hctx)if (!sbitmap_any_bit_set (&hctx->ctx_map))break ; -if (!blk_mq_get_dispatch_budget (hctx)) +if (!blk_mq_get_dispatch_budget (q))break ; rq =blk_mq_dequeue_from_ctx (hctx, ctx);if (!rq) { -blk_mq_put_dispatch_budget (hctx); +blk_mq_put_dispatch_budget (q);/* * We're releasing without dispatching. Holding the * budget could have blocked any "hctx"s with the
block/blk-mq.c +4 -4
@@ -1262,7 +1262,7 @@ bool blk_mq_dispatch_rq_list(struct request_queue *q, struct list_head *list, rq = list_first_entry(list, struct request, queuelist); hctx = rq->mq_hctx; -if (!got_budget && !blk_mq_get_dispatch_budget (hctx)) { +if (!got_budget && !blk_mq_get_dispatch_budget (q)) {blk_mq_put_driver_tag (rq); no_budget_avail = true;break ;
@@ -1277,7 +1277,7 @@ bool blk_mq_dispatch_rq_list(struct request_queue *q, struct list_head *list,* we'll re-run it below. */ if (!blk_mq_mark_tag_wait (hctx, rq)) { -blk_mq_put_dispatch_budget (hctx); +blk_mq_put_dispatch_budget (q);/* * For non-shared tags, the RESTART check * will suffice.
@@ -1925,11 +1925,11 @@ static blk_status_t __blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,if (q->elevator && !bypass_insert) goto insert; -if (!blk_mq_get_dispatch_budget (hctx)) +if (!blk_mq_get_dispatch_budget (q)) goto insert;if (!blk_mq_get_driver_tag (rq)) { -blk_mq_put_dispatch_budget (hctx); +blk_mq_put_dispatch_budget (q); goto insert; }
block/blk-mq.h +4 -8
@@ -180,20 +180,16 @@ unsigned int blk_mq_in_flight(struct request_queue *q, struct hd_struct *part);void blk_mq_in_flight_rw (struct request_queue *q,struct hd_struct *part,unsigned int inflight[2 ]); -static inline void blk_mq_put_dispatch_budget (struct blk_mq_hw_ctx *hctx) +static inline void blk_mq_put_dispatch_budget (struct request_queue *q) { -struct request_queue *q = hctx->queue; -if (q->mq_ops->put_budget) - q->mq_ops->put_budget (hctx); + q->mq_ops->put_budget (q); } -static inline bool blk_mq_get_dispatch_budget (struct blk_mq_hw_ctx *hctx) +static inline bool blk_mq_get_dispatch_budget (struct request_queue *q) { -struct request_queue *q = hctx->queue; -if (q->mq_ops->get_budget) -return q->mq_ops->get_budget (hctx); +return q->mq_ops->get_budget (q);return true; }
drivers/scsi/scsi_lib.c +3 -5
@@ -1624,17 +1624,15 @@ static void scsi_mq_done(struct scsi_cmnd *cmd)clear_bit (SCMD_STATE_COMPLETE , &cmd->state); } -static void scsi_mq_put_budget (struct blk_mq_hw_ctx *hctx) +static void scsi_mq_put_budget (struct request_queue *q) { -struct request_queue *q = hctx->queue;struct scsi_device *sdev = q->queuedata;atomic_dec (&sdev->device_busy); } -static bool scsi_mq_get_budget (struct blk_mq_hw_ctx *hctx) +static bool scsi_mq_get_budget (struct request_queue *q) { -struct request_queue *q = hctx->queue;struct scsi_device *sdev = q->queuedata;return scsi_dev_queue_ready (q, sdev);
@@ -1701,7 +1699,7 @@ static blk_status_t scsi_queue_rq(struct blk_mq_hw_ctx *hctx,if (scsi_target (sdev)->can_queue >0 )atomic_dec (&scsi_target (sdev)->target_busy); out_put_budget: -scsi_mq_put_budget (hctx); +scsi_mq_put_budget (q);switch (ret) {case BLK_STS_OK :break ;
include/linux/blk-mq.h +2 -2
@@ -270,8 +270,8 @@ struct blk_mq_queue_data {typedef blk_status_t (queue_rq_fn )(struct blk_mq_hw_ctx *,const struct blk_mq_queue_data *);typedef void (commit_rqs_fn )(struct blk_mq_hw_ctx *); -typedef bool (get_budget_fn )(struct blk_mq_hw_ctx *); -typedef void (put_budget_fn )(struct blk_mq_hw_ctx *); +typedef bool (get_budget_fn )(struct request_queue *); +typedef void (put_budget_fn )(struct request_queue *);typedef enum blk_eh_timer_return (timeout_fn )(struct request *,bool );typedef int (init_hctx_fn )(struct blk_mq_hw_ctx *,void *,unsigned int );typedef void (exit_hctx_fn )(struct blk_mq_hw_ctx *,unsigned int );
Signature · 1 line
2.25.2
PATCH v4 2/6 blk-mq: pass hctx to blk_mq_dispatch_rq_list
In reply to Ming LeiAll requests in the 'list' of blk_mq_dispatch_rq_list belong to same hctx, so it is better to pass hctx instead of request queue, because blk-mq's dispatch target is hctx instead of request queue.
--- block/blk-mq-sched.c | 14 ++++++-------- block/blk-mq.c | 6 +++--- block/blk-mq.h | 2 +- 3 files changed, 10 insertions(+), 12 deletions(-)
block/blk-mq-sched.c +6 -8
@@ -96,10 +96,9 @@ static int blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx)struct elevator_queue *e = q->elevator;LIST_HEAD (rq_list);int ret =0 ; +struct request *rq; do { - struct request *rq; - if (e->type->ops.has_work && !e->type->ops.has_work(hctx)) break;
@@ -131,7 +130,7 @@ static int blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx)* in blk_mq_dispatch_rq_list(). */ list_add (&rq->queuelist, &rq_list); - }while (blk_mq_dispatch_rq_list (q, &rq_list, true)); + }while (blk_mq_dispatch_rq_list (rq->mq_hctx, &rq_list, true));return ret; }
@@ -161,10 +160,9 @@ static int blk_mq_do_dispatch_ctx(struct blk_mq_hw_ctx *hctx)LIST_HEAD (rq_list);struct blk_mq_ctx *ctx =READ_ONCE (hctx->dispatch_from);int ret =0 ; +struct request *rq; do { - struct request *rq; - if (!list_empty_careful(&hctx->dispatch)) { ret = -EAGAIN; break;
@@ -200,7 +198,7 @@ static int blk_mq_do_dispatch_ctx(struct blk_mq_hw_ctx *hctx)/* round robin for fair dispatch */ ctx =blk_mq_next_ctx (hctx, rq->mq_ctx); - }while (blk_mq_dispatch_rq_list (q, &rq_list, true)); + }while (blk_mq_dispatch_rq_list (rq->mq_hctx, &rq_list, true));WRITE_ONCE (hctx->dispatch_from, ctx);return ret;
@@ -240,7 +238,7 @@ static int __blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx)*/ if (!list_empty (&rq_list)) {blk_mq_sched_mark_restart_hctx (hctx); -if (blk_mq_dispatch_rq_list (q, &rq_list, false)) { +if (blk_mq_dispatch_rq_list (hctx, &rq_list, false)) {if (has_sched_dispatch) ret =blk_mq_do_dispatch_sched (hctx);else
@@ -253,7 +251,7 @@ static int __blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx) ret =blk_mq_do_dispatch_ctx (hctx); }else {blk_mq_flush_busy_ctxs (hctx, &rq_list); -blk_mq_dispatch_rq_list (q, &rq_list, false); +blk_mq_dispatch_rq_list (hctx, &rq_list, false); }return ret;
block/blk-mq.c +3 -3
@@ -1236,10 +1236,10 @@ static void blk_mq_handle_zone_resource(struct request *rq,/* * Returns true if we did some work AND can potentially do more. */ -bool blk_mq_dispatch_rq_list (struct request_queue *q,struct list_head *list, +bool blk_mq_dispatch_rq_list (struct blk_mq_hw_ctx *hctx,struct list_head *list,bool got_budget) { -struct blk_mq_hw_ctx *hctx; +struct request_queue *q = hctx->queue;struct request *rq, *nxt;bool no_tag = false;int errors, queued;
@@ -1261,7 +1261,7 @@ bool blk_mq_dispatch_rq_list(struct request_queue *q, struct list_head *list, rq = list_first_entry(list, struct request, queuelist); - hctx = rq->mq_hctx; +WARN_ON_ONCE (hctx != rq->mq_hctx);if (!got_budget && !blk_mq_get_dispatch_budget (q)) {blk_mq_put_driver_tag (rq); no_budget_avail = true;
block/blk-mq.h +1 -1
@@ -40,7 +40,7 @@ struct blk_mq_ctx {void blk_mq_exit_queue (struct request_queue *q);int blk_mq_update_nr_requests (struct request_queue *q,unsigned int nr);void blk_mq_wake_waiters (struct request_queue *q); -bool blk_mq_dispatch_rq_list (struct request_queue *,struct list_head *,bool ); +bool blk_mq_dispatch_rq_list (struct blk_mq_hw_ctx *hctx,struct list_head *,bool );void blk_mq_add_to_requeue_list (struct request *rq,bool at_head,bool kick_requeue_list);void blk_mq_flush_busy_ctxs (struct blk_mq_hw_ctx *hctx,struct list_head *list);
Signature · 1 line
2.25.2
Re: [PATCH V4 2/6] blk-mq: pass hctx to blk_mq_dispatch_rq_list
Quoted 35 lines · Ming Lei
All requests in the 'list' of
blk_mq_dispatch_rq_listbelong to same hctx, so it is better to pass hctx instead of request queue, because blk-mq's dispatch target is hctx instead of request queue.Cc Sagi Grimberg <[email protected]>Cc Baolin Wang <[email protected]>Cc Christoph Hellwig <[email protected]>Reviewed-by Christoph Hellwig <[email protected]>Reviewed-by Sagi Grimberg <[email protected]>Reviewed-by Johannes Thumshirn <[email protected]>Tested-by Baolin Wang <[email protected]>Signed-off-by Ming Lei <[email protected]>--- block/blk-mq-sched.c | 14 ++++++-------- block/blk-mq.c | 6 +++--- block/blk-mq.h | 2 +- 3 files changed, 10 insertions(+), 12 deletions(-)block/blk-mq-sched.c +1
-2@@ -96,10 +96,9 @@ static int blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx)struct elevator_queue *e = q->elevator;LIST_HEAD (rq_list);int ret =0 ; +struct request *rq; do { - struct request *rq; - if (e->type->ops.has_work && !e->type->ops.has_work(hctx)) break;
block/blk-mq-sched.c static int blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx) · line 133
Quoted 5 lines
block/blk-mq-sched.c +1
-1@@ -131,7 +130,7 @@ static int blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx)* in blk_mq_dispatch_rq_list(). */ list_add (&rq->queuelist, &rq_list); - }while (blk_mq_dispatch_rq_list (q, &rq_list, true)); + }while (blk_mq_dispatch_rq_list (rq->mq_hctx, &rq_list, true));
Why not use the hctx argument passed to the function instead of rq->mq_hctx ?
Quoted 16 lines
block/blk-mq-sched.c +1
-2return ret; }@@ -161,10 +160,9 @@ static int blk_mq_do_dispatch_ctx(struct blk_mq_hw_ctx *hctx)LIST_HEAD (rq_list);struct blk_mq_ctx *ctx =READ_ONCE (hctx->dispatch_from);int ret =0 ; +struct request *rq; do { - struct request *rq; - if (!list_empty_careful(&hctx->dispatch)) { ret = -EAGAIN; break;
block/blk-mq-sched.c static int blk_mq_do_dispatch_ctx(struct blk_mq_hw_ctx *hctx) · line 201
Quoted 5 lines
block/blk-mq-sched.c +1
-1@@ -200,7 +198,7 @@ static int blk_mq_do_dispatch_ctx(struct blk_mq_hw_ctx *hctx)/* round robin for fair dispatch */ ctx =blk_mq_next_ctx (hctx, rq->mq_ctx); - }while (blk_mq_dispatch_rq_list (q, &rq_list, true)); + }while (blk_mq_dispatch_rq_list (rq->mq_hctx, &rq_list, true));
Same here.
Quoted 61 lines
block/blk-mq-sched.c +2
-2WRITE_ONCE (hctx->dispatch_from, ctx);return ret;@@ -240,7 +238,7 @@ static int __blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx)*/ if (!list_empty (&rq_list)) {blk_mq_sched_mark_restart_hctx (hctx); -if (blk_mq_dispatch_rq_list (q, &rq_list, false)) { +if (blk_mq_dispatch_rq_list (hctx, &rq_list, false)) {if (has_sched_dispatch) ret =blk_mq_do_dispatch_sched (hctx);else @@ -253,7 +251,7 @@ static int __blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx) ret =blk_mq_do_dispatch_ctx (hctx); }else {blk_mq_flush_busy_ctxs (hctx, &rq_list); -blk_mq_dispatch_rq_list (q, &rq_list, false); +blk_mq_dispatch_rq_list (hctx, &rq_list, false); }return ret;block/blk-mq.c +3
-3@@ -1236,10 +1236,10 @@ static void blk_mq_handle_zone_resource(struct request *rq,/* * Returns true if we did some work AND can potentially do more. */ -bool blk_mq_dispatch_rq_list (struct request_queue *q,struct list_head *list, +bool blk_mq_dispatch_rq_list (struct blk_mq_hw_ctx *hctx,struct list_head *list,bool got_budget) { -struct blk_mq_hw_ctx *hctx; +struct request_queue *q = hctx->queue;struct request *rq, *nxt;bool no_tag = false;int errors, queued;@@ -1261,7 +1261,7 @@ bool blk_mq_dispatch_rq_list(struct request_queue *q, struct list_head *list, rq = list_first_entry(list, struct request, queuelist); - hctx = rq->mq_hctx; +WARN_ON_ONCE (hctx != rq->mq_hctx);if (!got_budget && !blk_mq_get_dispatch_budget (q)) {blk_mq_put_driver_tag (rq); no_budget_avail = true;block/blk-mq.h +1
-1@@ -40,7 +40,7 @@ struct blk_mq_ctx {void blk_mq_exit_queue (struct request_queue *q);int blk_mq_update_nr_requests (struct request_queue *q,unsigned int nr);void blk_mq_wake_waiters (struct request_queue *q); -bool blk_mq_dispatch_rq_list (struct request_queue *,struct list_head *,bool ); +bool blk_mq_dispatch_rq_list (struct blk_mq_hw_ctx *hctx,struct list_head *,bool );void blk_mq_add_to_requeue_list (struct request *rq,bool at_head,bool kick_requeue_list);void blk_mq_flush_busy_ctxs (struct blk_mq_hw_ctx *hctx,struct list_head *list);
Signature · 2 lines
Damien Le Moal Western Digital Research
Re: [PATCH V4 2/6] blk-mq: pass hctx to blk_mq_dispatch_rq_list
Quoted 35 lines · Damien Le Moal
Quoted 34 lines · Ming Lei
All requests in the 'list' of
blk_mq_dispatch_rq_listbelong to same hctx, so it is better to pass hctx instead of request queue, because blk-mq's dispatch target is hctx instead of request queue.Cc Sagi Grimberg <[email protected]>Cc Baolin Wang <[email protected]>Cc Christoph Hellwig <[email protected]>Reviewed-by Christoph Hellwig <[email protected]>Reviewed-by Sagi Grimberg <[email protected]>Reviewed-by Johannes Thumshirn <[email protected]>Tested-by Baolin Wang <[email protected]>Signed-off-by Ming Lei <[email protected]>--- block/blk-mq-sched.c | 14 ++++++-------- block/blk-mq.c | 6 +++--- block/blk-mq.h | 2 +- 3 files changed, 10 insertions(+), 12 deletions(-)block/blk-mq-sched.c +1
-2@@ -96,10 +96,9 @@ static int blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx)struct elevator_queue *e = q->elevator;LIST_HEAD (rq_list);int ret =0 ; +struct request *rq; do { - struct request *rq; - if (e->type->ops.has_work && !e->type->ops.has_work(hctx)) break;
Quoted 8 lines
block/blk-mq-sched.c static int blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx) · line 133
Quoted 6 lines
block/blk-mq-sched.c +1
-1@@ -131,7 +130,7 @@ static int blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx)* in blk_mq_dispatch_rq_list(). */ list_add (&rq->queuelist, &rq_list); - }while (blk_mq_dispatch_rq_list (q, &rq_list, true)); + }while (blk_mq_dispatch_rq_list (rq->mq_hctx, &rq_list, true));Why not use the hctx argument passed to the function instead of
rq->mq_hctx?
e->type->ops.dispatch_request(hctx) may return one request which's .mq_hctx isn't same with the 'hctx' argument, so far bfq and deadline may do that.
Thanks,
Ming
Re: [PATCH V4 2/6] blk-mq: pass hctx to blk_mq_dispatch_rq_list
Quoted 45 lines · Ming Lei
Quoted 35 lines · Damien Le Moal
Quoted 34 lines · Ming Lei
All requests in the 'list' of
blk_mq_dispatch_rq_listbelong to same hctx, so it is better to pass hctx instead of request queue, because blk-mq's dispatch target is hctx instead of request queue.Cc Sagi Grimberg <[email protected]>Cc Baolin Wang <[email protected]>Cc Christoph Hellwig <[email protected]>Reviewed-by Christoph Hellwig <[email protected]>Reviewed-by Sagi Grimberg <[email protected]>Reviewed-by Johannes Thumshirn <[email protected]>Tested-by Baolin Wang <[email protected]>Signed-off-by Ming Lei <[email protected]>--- block/blk-mq-sched.c | 14 ++++++-------- block/blk-mq.c | 6 +++--- block/blk-mq.h | 2 +- 3 files changed, 10 insertions(+), 12 deletions(-)block/blk-mq-sched.c +1
-2@@ -96,10 +96,9 @@ static int blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx)struct elevator_queue *e = q->elevator;LIST_HEAD (rq_list);int ret =0 ; +struct request *rq; do { - struct request *rq; - if (e->type->ops.has_work && !e->type->ops.has_work(hctx)) break;Quoted 8 lines
block/blk-mq-sched.c static int blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx) · line 133
Quoted 6 lines
block/blk-mq-sched.c +1
-1@@ -131,7 +130,7 @@ static int blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx)* in blk_mq_dispatch_rq_list(). */ list_add (&rq->queuelist, &rq_list); - }while (blk_mq_dispatch_rq_list (q, &rq_list, true)); + }while (blk_mq_dispatch_rq_list (rq->mq_hctx, &rq_list, true));Why not use the hctx argument passed to the function instead of
rq->mq_hctx?
Quoted 3 lines
e->type->ops.dispatch_request(hctx)may return one request which's .mq_hctx isn't same with the 'hctx' argument, so far bfq and deadline may do that.
Ah, OK. But then all requests in rq_list may have different hctx. So is it wise to pass hctx as an argument to blk_mq_dispatch_rq_list() ? The loop in that function will still need to look at each rq hctx (hctx = rq->mq_hctx) for the budget. So the hctx argument may not be needed at all, no ? Am I missing something ?
Quoted 6 lines
Thanks,
Ming
Signature · 2 lines
Damien Le Moal Western Digital Research
Re: [PATCH V4 2/6] blk-mq: pass hctx to blk_mq_dispatch_rq_list
Quoted 46 lines · Damien Le Moal
Quoted 45 lines · Ming Lei
Quoted 43 lines · Damien Le Moal
Quoted 35 lines · Ming Lei
All requests in the 'list' of
blk_mq_dispatch_rq_listbelong to same hctx, so it is better to pass hctx instead of request queue, because blk-mq's dispatch target is hctx instead of request queue.Cc Sagi Grimberg <[email protected]>Cc Baolin Wang <[email protected]>Cc Christoph Hellwig <[email protected]>Reviewed-by Christoph Hellwig <[email protected]>Reviewed-by Sagi Grimberg <[email protected]>Reviewed-by Johannes Thumshirn <[email protected]>Tested-by Baolin Wang <[email protected]>Signed-off-by Ming Lei <[email protected]>--- block/blk-mq-sched.c | 14 ++++++-------- block/blk-mq.c | 6 +++--- block/blk-mq.h | 2 +- 3 files changed, 10 insertions(+), 12 deletions(-)block/blk-mq-sched.c +1
-2@@ -96,10 +96,9 @@ static int blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx)struct elevator_queue *e = q->elevator;LIST_HEAD (rq_list);int ret =0 ; +struct request *rq; do { - struct request *rq; - if (e->type->ops.has_work && !e->type->ops.has_work(hctx)) break;block/blk-mq-sched.c static int blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx) · line 133
Quoted 5 lines
block/blk-mq-sched.c +1
-1@@ -131,7 +130,7 @@ static int blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx)* in blk_mq_dispatch_rq_list(). */ list_add (&rq->queuelist, &rq_list); - }while (blk_mq_dispatch_rq_list (q, &rq_list, true)); + }while (blk_mq_dispatch_rq_list (rq->mq_hctx, &rq_list, true));Why not use the hctx argument passed to the function instead of
rq->mq_hctx?
Quoted 6 lines
Quoted 3 lines
e->type->ops.dispatch_request(hctx)may return one request which's .mq_hctx isn't same with the 'hctx' argument, so far bfq and deadline may do that.Ah, OK. But then all requests in
rq_listmay have different hctx. So is it wise to pass hctx as an argument toblk_mq_dispatch_rq_list()? The loop in that
&rq_list is one single request list.
Quoted 2 lines
function will still need to look at each rq hctx (hctx =
rq->mq_hctx) for the budget. So the hctx argument may not be needed at all, no ? Am I missing something ?
The final patch will add batching dispatch support, and more requests in same hctx will be added to this list, at that time, the hctx argument becomes reasonable.
Thanks,
Ming
PATCH v4 3/6 blk-mq: move getting driver tag and budget into one helper
In reply to Ming LeiMove code for getting driver tag and budget into one helper, so blk_mq_dispatch_rq_list gets a bit simplified, and easier to read.
Meantime move updating of 'no_tag' and 'no_budget_available' into the branch for handling partial dispatch because that is exactly consumer of the two local variables.
Also rename the parameter of 'got_budget' as 'ask_budget'.
No functional change.
--- block/blk-mq.c | 75 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 49 insertions(+), 26 deletions(-)
block/blk-mq.c +49 -26
@@ -1233,18 +1233,51 @@ static void blk_mq_handle_zone_resource(struct request *rq,__blk_mq_requeue_request (rq); } +enum prep_dispatch { +PREP_DISPATCH_OK , +PREP_DISPATCH_NO_TAG , +PREP_DISPATCH_NO_BUDGET , +}; + +static enum prep_dispatch blk_mq_prep_dispatch_rq (struct request *rq, +bool ask_budget) +{ +struct blk_mq_hw_ctx *hctx = rq->mq_hctx; + +if (ask_budget && !blk_mq_get_dispatch_budget (rq->q)) { +blk_mq_put_driver_tag (rq); +return PREP_DISPATCH_NO_BUDGET ; + } + +if (!blk_mq_get_driver_tag (rq)) { +/* +* The initial allocation attempt failed, so we need to +* rerun the hardware queue when a tag is freed. The +* waitqueue takes care of that. If the queue is run +* before we add this entry back on the dispatch list, +* we'll re-run it below. +*/ +if (!blk_mq_mark_tag_wait (hctx, rq)) { +/* budget is always obtained before getting tag */ +blk_mq_put_dispatch_budget (rq->q); +return PREP_DISPATCH_NO_TAG ; + } + } + +return PREP_DISPATCH_OK ; +} +/* * Returns true if we did some work AND can potentially do more. */ bool blk_mq_dispatch_rq_list (struct blk_mq_hw_ctx *hctx,struct list_head *list,bool got_budget) { +enum prep_dispatch prep;struct request_queue *q = hctx->queue;struct request *rq, *nxt; -bool no_tag = false;int errors, queued;blk_status_t ret =BLK_STS_OK ; -bool no_budget_avail = false;LIST_HEAD (zone_list);if (list_empty (list))
@@ -1262,31 +1295,9 @@ bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list, rq = list_first_entry(list, struct request, queuelist);WARN_ON_ONCE (hctx != rq->mq_hctx); -if (!got_budget && !blk_mq_get_dispatch_budget (q)) { -blk_mq_put_driver_tag (rq); - no_budget_avail = true; + prep =blk_mq_prep_dispatch_rq (rq, !got_budget); +if (prep !=PREP_DISPATCH_OK )break ; - } - -if (!blk_mq_get_driver_tag (rq)) { -/* -* The initial allocation attempt failed, so we need to -* rerun the hardware queue when a tag is freed. The -* waitqueue takes care of that. If the queue is run -* before we add this entry back on the dispatch list, -* we'll re-run it below. -*/ -if (!blk_mq_mark_tag_wait (hctx, rq)) { -blk_mq_put_dispatch_budget (q); -/* -* For non-shared tags, the RESTART check -* will suffice. -*/ -if (hctx->flags &BLK_MQ_F_TAG_SHARED ) - no_tag = true; -break ; - } - }list_del_init (&rq->queuelist);
@@ -1339,6 +1350,18 @@ bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list,*/ if (!list_empty (list)) {bool needs_restart; +bool no_tag = false; +bool no_budget_avail = false; + +/* +* For non-shared tags, the RESTART check +* will suffice. +*/ +if (prep ==PREP_DISPATCH_NO_TAG && + (hctx->flags &BLK_MQ_F_TAG_SHARED )) + no_tag = true; +if (prep ==PREP_DISPATCH_NO_BUDGET ) + no_budget_avail = true;/* * If we didn't flush the entire list, we could have told
Signature · 1 line
2.25.2
Re: [PATCH V4 3/6] blk-mq: move getting driver tag and budget into one helper
Quoted 29 lines · Ming Lei
Move code for getting driver tag and budget into one helper, so
blk_mq_dispatch_rq_listgets a bit simplified, and easier to read.Meantime move updating of '
no_tag' and 'no_budget_available' into the branch for handling partial dispatch because that is exactly consumer of the two local variables.Also rename the parameter of '
got_budget' as 'ask_budget'.No functional change.
Cc Sagi Grimberg <[email protected]>Cc Baolin Wang <[email protected]>Cc Christoph Hellwig <[email protected]>Reviewed-by Sagi Grimberg <[email protected]>Reviewed-by Christoph Hellwig <[email protected]>Reviewed-by Johannes Thumshirn <[email protected]>Tested-by Baolin Wang <[email protected]>Signed-off-by Ming Lei <[email protected]>--- block/blk-mq.c | 75 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 49 insertions(+), 26 deletions(-)block/blk-mq.c +0
-0@@ -1233,18 +1233,51 @@ static void blk_mq_handle_zone_resource(struct request *rq,__blk_mq_requeue_request (rq);
block/blk-mq.c static void blk_mq_handle_zone_resource(struct request *rq, · line 1243
Quoted 10 lines
block/blk-mq.c +8
-0} +enum prep_dispatch { +PREP_DISPATCH_OK , +PREP_DISPATCH_NO_TAG , +PREP_DISPATCH_NO_BUDGET , +}; + +static enum prep_dispatch blk_mq_prep_dispatch_rq (struct request *rq, +bool ask_budget)
"need_budget" may be a better argument name instead of ask_budget. More reflective of the fact that this is a boolean.
Quoted 8 lines
block/blk-mq.c +8
-0+{ +struct blk_mq_hw_ctx *hctx = rq->mq_hctx; + +if (ask_budget && !blk_mq_get_dispatch_budget (rq->q)) { +blk_mq_put_driver_tag (rq); +return PREP_DISPATCH_NO_BUDGET ; + } +
block/blk-mq.c static void blk_mq_handle_zone_resource(struct request *rq, · line 1261
Quoted 10 lines
block/blk-mq.c +10
-0+if (!blk_mq_get_driver_tag (rq)) { +/* +* The initial allocation attempt failed, so we need to +* rerun the hardware queue when a tag is freed. The +* waitqueue takes care of that. If the queue is run +* before we add this entry back on the dispatch list, +* we'll re-run it below. +*/ +if (!blk_mq_mark_tag_wait (hctx, rq)) { +/* budget is always obtained before getting tag */
This is clear since it was down a few lines above. Maybe drop this comment ?
Quoted 62 lines
block/blk-mq.c +11
-26+blk_mq_put_dispatch_budget (rq->q); +return PREP_DISPATCH_NO_TAG ; + } + } + +return PREP_DISPATCH_OK ; +} +/* * Returns true if we did some work AND can potentially do more. */ bool blk_mq_dispatch_rq_list (struct blk_mq_hw_ctx *hctx,struct list_head *list,bool got_budget) { +enum prep_dispatch prep;struct request_queue *q = hctx->queue;struct request *rq, *nxt; -bool no_tag = false;int errors, queued;blk_status_t ret =BLK_STS_OK ; -bool no_budget_avail = false;LIST_HEAD (zone_list);if (list_empty (list))@@ -1262,31 +1295,9 @@ bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list, rq = list_first_entry(list, struct request, queuelist);WARN_ON_ONCE (hctx != rq->mq_hctx); -if (!got_budget && !blk_mq_get_dispatch_budget (q)) { -blk_mq_put_driver_tag (rq); - no_budget_avail = true; + prep =blk_mq_prep_dispatch_rq (rq, !got_budget); +if (prep !=PREP_DISPATCH_OK )break ; - } - -if (!blk_mq_get_driver_tag (rq)) { -/* -* The initial allocation attempt failed, so we need to -* rerun the hardware queue when a tag is freed. The -* waitqueue takes care of that. If the queue is run -* before we add this entry back on the dispatch list, -* we'll re-run it below. -*/ -if (!blk_mq_mark_tag_wait (hctx, rq)) { -blk_mq_put_dispatch_budget (q); -/* -* For non-shared tags, the RESTART check -* will suffice. -*/ -if (hctx->flags &BLK_MQ_F_TAG_SHARED ) - no_tag = true; -break ; - } - }list_del_init (&rq->queuelist);@@ -1339,6 +1350,18 @@ bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list,*/ if (!list_empty (list)) {bool needs_restart;
block/blk-mq.c bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list, · line 1362
Quoted 10 lines
block/blk-mq.c +10
-0+bool no_tag = false; +bool no_budget_avail = false; + +/* +* For non-shared tags, the RESTART check +* will suffice. +*/ +if (prep ==PREP_DISPATCH_NO_TAG && + (hctx->flags &BLK_MQ_F_TAG_SHARED )) + no_tag = true;
You could drop the variable declaration initialization and simply have:
no_tag = prep == PREP_DISPATCH_NO_TAG && (hctx->flags & BLK_MQ_F_TAG_SHARED)
block/blk-mq.c bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list, · line 1364
Quoted 2 lines
block/blk-mq.c +2
-0+if (prep ==PREP_DISPATCH_NO_BUDGET ) + no_budget_avail = true;
Same here:
no_budget = prep == PREP_DISPATCH_NO_BUDGET;
Also to be symmetric with "no_tag", "no_budget" may be a better variable name ?
Quoted 4 lines
block/blk-mq.c +0
-0/* * If we didn't flush the entire list, we could have told
Signature · 2 lines
Damien Le Moal Western Digital Research
PATCH v4 4/6 blk-mq: remove dead check from blk_mq_dispatch_rq_list
In reply to Ming LeiWhen BLK_STS_RESOURCE or BLK_STS_DEV_RESOURCE is returned from .queue_rq, the 'list' variable always holds this rq which isn't queued to LLD successfully.
So blk_mq_dispatch_rq_list() always returns false from the branch of '!list_empty(list)'.
No functional change.
--- block/blk-mq.c | 7 ------- 1 file changed, 7 deletions(-)
block/blk-mq.c +0 -7
@@ -1413,13 +1413,6 @@ bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list, }else blk_mq_update_dispatch_busy (hctx, false); -/* -* If the host/device is unable to accept more work, inform the -* caller of that. -*/ -if (ret ==BLK_STS_RESOURCE || ret ==BLK_STS_DEV_RESOURCE ) -return false; -return (queued + errors) !=0 ; }
Signature · 1 line
2.25.2
PATCH v4 5/6 blk-mq: pass obtained budget count to blk_mq_dispatch_rq_list
In reply to Ming LeiPass obtained budget count to blk_mq_dispatch_rq_list(), and prepare for supporting fully batching submission.
With the obtained budget count, it is easier to put extra budgets in case of .queue_rq failure.
Meantime remove the old 'got_budget' parameter.
--- block/blk-mq-sched.c | 8 ++++---- block/blk-mq.c | 27 +++++++++++++++++++++++---- block/blk-mq.h | 3 ++- 3 files changed, 29 insertions(+), 9 deletions(-)
block/blk-mq-sched.c +4 -4
@@ -130,7 +130,7 @@ static int blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx)* in blk_mq_dispatch_rq_list(). */ list_add (&rq->queuelist, &rq_list); - }while (blk_mq_dispatch_rq_list (rq->mq_hctx, &rq_list, true)); + }while (blk_mq_dispatch_rq_list (rq->mq_hctx, &rq_list,1 ));return ret; }
@@ -198,7 +198,7 @@ static int blk_mq_do_dispatch_ctx(struct blk_mq_hw_ctx *hctx)/* round robin for fair dispatch */ ctx =blk_mq_next_ctx (hctx, rq->mq_ctx); - }while (blk_mq_dispatch_rq_list (rq->mq_hctx, &rq_list, true)); + }while (blk_mq_dispatch_rq_list (rq->mq_hctx, &rq_list,1 ));WRITE_ONCE (hctx->dispatch_from, ctx);return ret;
@@ -238,7 +238,7 @@ static int __blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx)*/ if (!list_empty (&rq_list)) {blk_mq_sched_mark_restart_hctx (hctx); -if (blk_mq_dispatch_rq_list (hctx, &rq_list, false)) { +if (blk_mq_dispatch_rq_list (hctx, &rq_list,0 )) {if (has_sched_dispatch) ret =blk_mq_do_dispatch_sched (hctx);else
@@ -251,7 +251,7 @@ static int __blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx) ret =blk_mq_do_dispatch_ctx (hctx); }else {blk_mq_flush_busy_ctxs (hctx, &rq_list); -blk_mq_dispatch_rq_list (hctx, &rq_list, false); +blk_mq_dispatch_rq_list (hctx, &rq_list,0 ); }return ret;
block/blk-mq.c +23 -4
@@ -1259,7 +1259,8 @@ static enum prep_dispatch blk_mq_prep_dispatch_rq(struct request *rq,*/ if (!blk_mq_mark_tag_wait (hctx, rq)) {/* budget is always obtained before getting tag */ -blk_mq_put_dispatch_budget (rq->q); +if (ask_budget) +blk_mq_put_dispatch_budget (rq->q);return PREP_DISPATCH_NO_TAG ; } }
@@ -1267,11 +1268,21 @@ static enum prep_dispatch blk_mq_prep_dispatch_rq(struct request *rq,return PREP_DISPATCH_OK ; } +static void blk_mq_release_budgets (struct request_queue *q, +unsigned int nr_budgets) +{ +int i =0 ; + +/* release got budgets */ +while (i++ < nr_budgets) +blk_mq_put_dispatch_budget (q); +} +/* * Returns true if we did some work AND can potentially do more. */ bool blk_mq_dispatch_rq_list (struct blk_mq_hw_ctx *hctx,struct list_head *list, -bool got_budget) +unsigned int nr_budgets) {enum prep_dispatch prep;struct request_queue *q = hctx->queue;
@@ -1283,7 +1294,7 @@ bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list,if (list_empty (list))return false; -WARN_ON (!list_is_singular (list) && got_budget); +WARN_ON (!list_is_singular (list) && nr_budgets);/* * Now process all the entries, sending them to the driver.
@@ -1295,7 +1306,7 @@ bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list, rq = list_first_entry(list, struct request, queuelist);WARN_ON_ONCE (hctx != rq->mq_hctx); - prep =blk_mq_prep_dispatch_rq (rq, !got_budget); + prep =blk_mq_prep_dispatch_rq (rq, !nr_budgets);if (prep !=PREP_DISPATCH_OK )break ;
@@ -1314,6 +1325,12 @@ bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list, bd.last = !blk_mq_get_driver_tag (nxt); } +/* +* once the request is queued to lld, no need to cover the +* budget any more +*/ +if (nr_budgets) + nr_budgets--; ret = q->mq_ops->queue_rq (hctx, &bd);if (ret ==BLK_STS_RESOURCE || ret ==BLK_STS_DEV_RESOURCE ) {blk_mq_handle_dev_resource (rq, list);
@@ -1353,6 +1370,8 @@ bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list,bool no_tag = false;bool no_budget_avail = false; +blk_mq_release_budgets (q, nr_budgets); +/* * For non-shared tags, the RESTART check * will suffice.
block/blk-mq.h +2 -1
@@ -40,7 +40,8 @@ struct blk_mq_ctx {void blk_mq_exit_queue (struct request_queue *q);int blk_mq_update_nr_requests (struct request_queue *q,unsigned int nr);void blk_mq_wake_waiters (struct request_queue *q); -bool blk_mq_dispatch_rq_list (struct blk_mq_hw_ctx *hctx,struct list_head *,bool ); +bool blk_mq_dispatch_rq_list (struct blk_mq_hw_ctx *hctx,struct list_head *, +unsigned int );void blk_mq_add_to_requeue_list (struct request *rq,bool at_head,bool kick_requeue_list);void blk_mq_flush_busy_ctxs (struct blk_mq_hw_ctx *hctx,struct list_head *list);
Signature · 1 line
2.25.2
Re: [PATCH V4 5/6] blk-mq: pass obtained budget count to blk_mq_dispatch_rq_list
Quoted 74 lines · Ming Lei
Pass obtained budget count to
blk_mq_dispatch_rq_list(), and prepare for supporting fully batching submission.With the obtained budget count, it is easier to put extra budgets in case of .queue_rq failure.
Meantime remove the old '
got_budget' parameter.Cc Sagi Grimberg <[email protected]>Cc Baolin Wang <[email protected]>Cc Christoph Hellwig <[email protected]>Tested-by Baolin Wang <[email protected]>Signed-off-by Ming Lei <[email protected]>--- block/blk-mq-sched.c | 8 ++++---- block/blk-mq.c | 27 +++++++++++++++++++++++---- block/blk-mq.h | 3 ++- 3 files changed, 29 insertions(+), 9 deletions(-)block/blk-mq-sched.c +4
-4@@ -130,7 +130,7 @@ static int blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx)* in blk_mq_dispatch_rq_list(). */ list_add (&rq->queuelist, &rq_list); - }while (blk_mq_dispatch_rq_list (rq->mq_hctx, &rq_list, true)); + }while (blk_mq_dispatch_rq_list (rq->mq_hctx, &rq_list,1 ));return ret; }@@ -198,7 +198,7 @@ static int blk_mq_do_dispatch_ctx(struct blk_mq_hw_ctx *hctx)/* round robin for fair dispatch */ ctx =blk_mq_next_ctx (hctx, rq->mq_ctx); - }while (blk_mq_dispatch_rq_list (rq->mq_hctx, &rq_list, true)); + }while (blk_mq_dispatch_rq_list (rq->mq_hctx, &rq_list,1 ));WRITE_ONCE (hctx->dispatch_from, ctx);return ret;@@ -238,7 +238,7 @@ static int __blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx)*/ if (!list_empty (&rq_list)) {blk_mq_sched_mark_restart_hctx (hctx); -if (blk_mq_dispatch_rq_list (hctx, &rq_list, false)) { +if (blk_mq_dispatch_rq_list (hctx, &rq_list,0 )) {if (has_sched_dispatch) ret =blk_mq_do_dispatch_sched (hctx);else @@ -251,7 +251,7 @@ static int __blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx) ret =blk_mq_do_dispatch_ctx (hctx); }else {blk_mq_flush_busy_ctxs (hctx, &rq_list); -blk_mq_dispatch_rq_list (hctx, &rq_list, false); +blk_mq_dispatch_rq_list (hctx, &rq_list,0 ); }return ret;block/blk-mq.c +2
-1@@ -1259,7 +1259,8 @@ static enum prep_dispatch blk_mq_prep_dispatch_rq(struct request *rq,*/ if (!blk_mq_mark_tag_wait (hctx, rq)) {/* budget is always obtained before getting tag */ -blk_mq_put_dispatch_budget (rq->q); +if (ask_budget) +blk_mq_put_dispatch_budget (rq->q);return PREP_DISPATCH_NO_TAG ; } }
block/blk-mq.c static enum prep_dispatch blk_mq_prep_dispatch_rq(struct request *rq, · line 1277
Quoted 10 lines
block/blk-mq.c +7
-0@@ -1267,11 +1268,21 @@ static enum prep_dispatch blk_mq_prep_dispatch_rq(struct request *rq,return PREP_DISPATCH_OK ; } +static void blk_mq_release_budgets (struct request_queue *q, +unsigned int nr_budgets) +{ +int i =0 ; + +/* release got budgets */ +while (i++ < nr_budgets)
A for loop would be simpler I think...
Quoted 67 lines
block/blk-mq.c +14
-3+blk_mq_put_dispatch_budget (q); +} +/* * Returns true if we did some work AND can potentially do more. */ bool blk_mq_dispatch_rq_list (struct blk_mq_hw_ctx *hctx,struct list_head *list, -bool got_budget) +unsigned int nr_budgets) {enum prep_dispatch prep;struct request_queue *q = hctx->queue;@@ -1283,7 +1294,7 @@ bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list,if (list_empty (list))return false; -WARN_ON (!list_is_singular (list) && got_budget); +WARN_ON (!list_is_singular (list) && nr_budgets);/* * Now process all the entries, sending them to the driver. @@ -1295,7 +1306,7 @@ bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list, rq = list_first_entry(list, struct request, queuelist);WARN_ON_ONCE (hctx != rq->mq_hctx); - prep =blk_mq_prep_dispatch_rq (rq, !got_budget); + prep =blk_mq_prep_dispatch_rq (rq, !nr_budgets);if (prep !=PREP_DISPATCH_OK )break ;@@ -1314,6 +1325,12 @@ bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list, bd.last = !blk_mq_get_driver_tag (nxt); } +/* +* once the request is queued to lld, no need to cover the +* budget any more +*/ +if (nr_budgets) + nr_budgets--; ret = q->mq_ops->queue_rq (hctx, &bd);if (ret ==BLK_STS_RESOURCE || ret ==BLK_STS_DEV_RESOURCE ) {blk_mq_handle_dev_resource (rq, list);@@ -1353,6 +1370,8 @@ bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list,bool no_tag = false;bool no_budget_avail = false; +blk_mq_release_budgets (q, nr_budgets); +/* * For non-shared tags, the RESTART check * will suffice. block/blk-mq.h +2
-1@@ -40,7 +40,8 @@ struct blk_mq_ctx {void blk_mq_exit_queue (struct request_queue *q);int blk_mq_update_nr_requests (struct request_queue *q,unsigned int nr);void blk_mq_wake_waiters (struct request_queue *q); -bool blk_mq_dispatch_rq_list (struct blk_mq_hw_ctx *hctx,struct list_head *,bool ); +bool blk_mq_dispatch_rq_list (struct blk_mq_hw_ctx *hctx,struct list_head *, +unsigned int );void blk_mq_add_to_requeue_list (struct request *rq,bool at_head,bool kick_requeue_list);void blk_mq_flush_busy_ctxs (struct blk_mq_hw_ctx *hctx,struct list_head *list);
Signature · 2 lines
Damien Le Moal Western Digital Research
PATCH v4 6/6 blk-mq: support batching dispatch in case of io scheduler
In reply to Ming LeiMore and more drivers want to get batching requests queued from block layer, such as mmc, and tcp based storage drivers. Also current in-tree users have virtio-scsi, virtio-blk and nvme.
For none, we already support batching dispatch.
But for io scheduler, every time we just take one request from scheduler and pass the single request to blk_mq_dispatch_rq_list(). This way makes batching dispatch not possible when io scheduler is applied. One reason is that we don't want to hurt sequential IO performance, becasue IO merge chance is reduced if more requests are dequeued from scheduler queue.
Try to support batching dispatch for io scheduler by starting with the following simple approach:
1) still make sure we can get budget before dequeueing request
2) use hctx->dispatch_busy to evaluate if queue is busy, if it is busy we fackback to non-batching dispatch, otherwise dequeue as many as possible requests from scheduler, and pass them to blk_mq_dispatch_rq_list().
Wrt. 2), we use similar policy for none, and turns out that SCSI SSD performance got improved much.
In future, maybe we can develop more intelligent algorithem for batching dispatch.
Baolin has tested this patch and found that MMC performance is improved[3].
[1] https://lore.kernel.org/linux-block/20200512075501.GF1531898@T590/#r
[2] https://lore.kernel.org/linux-block/[email protected]/
[3] https://lore.kernel.org/linux-block/CADBw62o9eTQDJ9RvNgEqSpXmg6Xcq=2TxH0Hfxhp29uF2W=TXA@mail.gmail.com/
--- block/blk-mq-sched.c | 96 ++++++++++++++++++++++++++++++++++++++++++-- block/blk-mq.c | 2 - 2 files changed, 93 insertions(+), 5 deletions(-)
block/blk-mq-sched.c +93 -3
@@ -7,6 +7,7 @@#include <linux/kernel.h> #include <linux/module.h> #include <linux/blk-mq.h> +#include <linux/list_sort.h> #include <trace/events/block.h>
@@ -80,6 +81,74 @@ void blk_mq_sched_restart(struct blk_mq_hw_ctx *hctx)blk_mq_run_hw_queue (hctx, true); } +/* +* We know bfq and deadline apply single scheduler queue instead of multi +* queue. However, the two are often used on single queue devices, also +* the current @hctx should affect the real device status most of times +* because of locality principle. +* +* So use current hctx->dispatch_busy directly for figuring out batching +* dispatch count. +*/ +static unsigned int blk_mq_sched_get_batching_nr (struct blk_mq_hw_ctx *hctx) +{ +if (hctx->dispatch_busy) +return 1 ; +return hctx->queue->nr_requests; +} + +static int sched_rq_cmp (void *priv,struct list_head *a,struct list_head *b) +{ + struct request *rqa = container_of(a, struct request, queuelist); + struct request *rqb = container_of(b, struct request, queuelist); + +return rqa->mq_hctx > rqb->mq_hctx; +} + +static inline bool blk_mq_do_dispatch_rq_lists (struct blk_mq_hw_ctx *hctx, +struct list_head *lists,bool multi_hctxs,unsigned count) +{ +bool ret; + +if (!count) +return false; + +if (likely (!multi_hctxs)) +return blk_mq_dispatch_rq_list (hctx, lists, count); + +/* +* Requests from different hctx may be dequeued from some scheduler, +* such as bfq and deadline. +* +* Sort the requests in the list according to their hctx, dispatch +* batching requests from same hctx +*/ +list_sort (NULL , lists, sched_rq_cmp); + + ret = false; +while (!list_empty (lists)) { +LIST_HEAD (list); +struct request *new, *rq =list_first_entry (lists, + struct request, queuelist); +unsigned cnt =0 ; + +list_for_each_entry (new, lists, queuelist) { +if (new->mq_hctx != rq->mq_hctx) +break ; + cnt++; + } + +if (new->mq_hctx == rq->mq_hctx) +list_splice_tail_init (lists, &list); +else +list_cut_before (&list, lists, &new->queuelist); + + ret =blk_mq_dispatch_rq_list (rq->mq_hctx, &list, cnt); + } + +return ret; +} +#define BLK_MQ_BUDGET_DELAY 3/* ms units */ /*
@@ -97,7 +166,15 @@ static int blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx)LIST_HEAD (rq_list);int ret =0 ;struct request *rq; - +int cnt; +unsigned int max_dispatch; +bool multi_hctxs, run_queue; + + again: +/* prepare one batch for dispatch */ + cnt =0 ; + max_dispatch =blk_mq_sched_get_batching_nr (hctx); + multi_hctxs = run_queue = false; do { if (e->type->ops.has_work && !e->type->ops.has_work(hctx)) break;
@@ -120,7 +197,7 @@ static int blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx)* no guarantee anyone will kick the queue. Kick it * ourselves. */ -blk_mq_delay_run_hw_queues (q,BLK_MQ_BUDGET_DELAY ); + run_queue = true;break ; }
@@ -130,7 +207,20 @@ static int blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx)* in blk_mq_dispatch_rq_list(). */ list_add (&rq->queuelist, &rq_list); - }while (blk_mq_dispatch_rq_list (rq->mq_hctx, &rq_list,1 )); + cnt++; + +if (rq->mq_hctx != hctx && !multi_hctxs) + multi_hctxs = true; + }while (cnt < max_dispatch); + +/* dispatch more if we may do more */ +if (blk_mq_do_dispatch_rq_lists (hctx, &rq_list, multi_hctxs, cnt) && + !ret) + goto again; + +/* in-flight request's completion can rerun queue */ +if (!cnt && run_queue) +blk_mq_delay_run_hw_queues (q,BLK_MQ_BUDGET_DELAY );return ret; }
block/blk-mq.c +0 -2
@@ -1294,8 +1294,6 @@ bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list,if (list_empty (list))return false; -WARN_ON (!list_is_singular (list) && nr_budgets); -/* * Now process all the entries, sending them to the driver. */
Signature · 1 line
2.25.2