cleanup request insertation parameters v2
Subsystems (2)
Hi Jens,
in context of his latest series Bart commented that it's too hard to find all spots that do a head insertation into the blk-mq dispatch queues. This series collapses various far too deep callchains, drop two of the three bools and then replaced the final once with a greppable constant.
This will create some rebased work for Bart of top of the other comments he got, but I think this will allow us to sort out some of the request order issues much better while also making the code a lot more readable.
Changes since v1:
- add back a blk_mq_run_hw_queue in blk_insert_flush that got lost
- use a __bitwise type for the insert flags
- sort out header hell a bit
- various typo fixes
Diffstat:
b/block/bfq-iosched.c | 17 +- b/block/blk-flush.c | 15 -- b/block/blk-mq-cpumap.c | 1 b/block/blk-mq-debugfs.c | 2 b/block/blk-mq-pci.c | 1 b/block/blk-mq-sched.c | 112 ------------------ b/block/blk-mq-sched.h | 7 - b/block/blk-mq-sysfs.c | 2 b/block/blk-mq-tag.c | 2 b/block/blk-mq-virtio.c | 1 b/block/blk-mq.c | 285 ++++++++++++++++++++++++++++------------------- b/block/blk-mq.h | 74 ++++++++++-- b/block/blk-pm.c | 2 b/block/blk-stat.c | 1 b/block/blk-sysfs.c | 1 b/block/elevator.h | 4 b/block/kyber-iosched.c | 7 - b/block/mq-deadline.c | 13 -- block/blk-mq-tag.h | 73 ------------ 19 files changed, 265 insertions(+), 355 deletions(-)
PATCH 1/18 blk-mq: don't plug for head insertions in blk_execute_rq_nowait
Plugs never insert at head, so don't plug for head insertions.
--- block/blk-mq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
block/blk-mq.c +1 -1
@@ -1299,7 +1299,7 @@ void blk_execute_rq_nowait(struct request *rq, bool at_head)* device, directly accessing the plug instead of using blk_mq_plug() * should not have any consequences. */ -if (current->plug) +if (current->plug && !at_head)blk_add_rq_to_plug (current->plug, rq);else blk_mq_sched_insert_request (rq, at_head, true, false);
Signature · 1 line
2.39.2
Re: [PATCH 01/18] blk-mq: don't plug for head insertions in blk_execute_rq_nowait
Quoted 5 lines · Christoph Hellwig
Plugs never insert at head, so don't plug for head insertions.
Fixes 1c2d2fff6dc0 ("block: wire-up support for passthrough plugging")Signed-off-by Christoph Hellwig <[email protected]>Reviewed-by Bart Van Assche <[email protected]>
Quoted 17 lines
--- block/blk-mq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)block/blk-mq.c +1
-1@@ -1299,7 +1299,7 @@ void blk_execute_rq_nowait(struct request *rq, bool at_head)* device, directly accessing the plug instead of using blk_mq_plug() * should not have any consequences. */ -if (current->plug) +if (current->plug && !at_head)blk_add_rq_to_plug (current->plug, rq);else blk_mq_sched_insert_request (rq, at_head, true, false);
PATCH 2/18 blk-mq: remove blk-mq-tag.h
In reply to Christoph Hellwigblk-mq-tag.h is always included by blk-mq.h, and causes recursive inclusion hell with further changes. Just merge it into blk-mq.h instead.
--- block/bfq-iosched.c | 1 - block/blk-flush.c | 1 - block/blk-mq-debugfs.c | 1 - block/blk-mq-sched.c | 1 - block/blk-mq-sched.h | 1 - block/blk-mq-sysfs.c | 1 - block/blk-mq-tag.c | 1 - block/blk-mq-tag.h | 73 ------------------------------------------ block/blk-mq.c | 1 - block/blk-mq.h | 61 ++++++++++++++++++++++++++++++++++- block/blk-pm.c | 1 - block/kyber-iosched.c | 1 - block/mq-deadline.c | 1 - 13 files changed, 60 insertions(+), 85 deletions(-) delete mode 100644 block/blk-mq-tag.h
block/bfq-iosched.c +0 -1
@@ -129,7 +129,6 @@#include "elevator.h" #include "blk.h" #include "blk-mq.h" -#include "blk-mq-tag.h" #include "blk-mq-sched.h" #include "bfq-iosched.h" #include "blk-wbt.h"
block/blk-flush.c +0 -1
@@ -73,7 +73,6 @@#include "blk.h" #include "blk-mq.h" -#include "blk-mq-tag.h" #include "blk-mq-sched.h" /* PREFLUSH/FUA sequences */
block/blk-mq-debugfs.c +0 -1
@@ -12,7 +12,6 @@#include "blk-mq.h" #include "blk-mq-debugfs.h" #include "blk-mq-sched.h" -#include "blk-mq-tag.h" #include "blk-rq-qos.h" static int queue_poll_stat_show (void *data,struct seq_file *m)
block/blk-mq-sched.c +0 -1
@@ -15,7 +15,6 @@#include "blk-mq.h" #include "blk-mq-debugfs.h" #include "blk-mq-sched.h" -#include "blk-mq-tag.h" #include "blk-wbt.h" /*
block/blk-mq-sched.h +0 -1
@@ -4,7 +4,6 @@#include "elevator.h" #include "blk-mq.h" -#include "blk-mq-tag.h" #define MAX_SCHED_RQ (16 * BLKDEV_DEFAULT_RQ)
block/blk-mq-sysfs.c +0 -1
@@ -13,7 +13,6 @@#include <linux/blk-mq.h> #include "blk.h" #include "blk-mq.h" -#include "blk-mq-tag.h" static void blk_mq_sysfs_release (struct kobject *kobj) {
block/blk-mq-tag.c +0 -1
@@ -14,7 +14,6 @@#include "blk.h" #include "blk-mq.h" #include "blk-mq-sched.h" -#include "blk-mq-tag.h" /* * Recalculate wakeup batch when tag is shared by hctx.
block/blk-mq-tag.h deleted +0 -73
73 lines not on this page · raw message
block/blk-mq.c +0 -1
@@ -37,7 +37,6 @@#include "blk.h" #include "blk-mq.h" #include "blk-mq-debugfs.h" -#include "blk-mq-tag.h" #include "blk-pm.h" #include "blk-stat.h" #include "blk-mq-sched.h"
block/blk-mq.h +60 -1
79 lines not on this page · raw message
block/blk-pm.c +0 -1
@@ -5,7 +5,6 @@#include <linux/blkdev.h> #include <linux/pm_runtime.h> #include "blk-mq.h" -#include "blk-mq-tag.h" /** * blk_pm_runtime_init - Block layer runtime PM initialization routine
block/kyber-iosched.c +0 -1
@@ -19,7 +19,6 @@#include "blk-mq.h" #include "blk-mq-debugfs.h" #include "blk-mq-sched.h" -#include "blk-mq-tag.h" #define CREATE_TRACE_POINTS #include <trace/events/kyber.h>
block/mq-deadline.c +0 -1
@@ -23,7 +23,6 @@#include "blk.h" #include "blk-mq.h" #include "blk-mq-debugfs.h" -#include "blk-mq-tag.h" #include "blk-mq-sched.h" /*
Signature · 1 line
2.39.2
Re: [PATCH 02/18] blk-mq: remove blk-mq-tag.h
Quoted 5 lines · Christoph Hellwig
blk-mq-tag.his always included byblk-mq.h, and causes recursive inclusion hell with further changes. Just merge it intoblk-mq.hinstead.Signed-off-by Christoph Hellwig <[email protected]>
Looks good to me.
PATCH 3/18 blk-mq: include <linux/blk-mq.h> in block/blk-mq.h
In reply to Christoph Hellwigblock/blk-mq.h needs various definitions from <linux/blk-mq.h>, include it there instead of relying on the source files to include both.
--- block/blk-flush.c | 1 - block/blk-mq-cpumap.c | 1 - block/blk-mq-debugfs.c | 1 - block/blk-mq-pci.c | 1 - block/blk-mq-sched.c | 1 - block/blk-mq-sysfs.c | 1 - block/blk-mq-tag.c | 1 - block/blk-mq-virtio.c | 1 - block/blk-mq.c | 1 - block/blk-mq.h | 1 + block/blk-pm.c | 1 - block/blk-stat.c | 1 - block/blk-sysfs.c | 1 - block/kyber-iosched.c | 1 - block/mq-deadline.c | 1 - 15 files changed, 1 insertion(+), 14 deletions(-)
block/blk-flush.c +0 -1
@@ -68,7 +68,6 @@#include <linux/bio.h> #include <linux/blkdev.h> #include <linux/gfp.h> -#include <linux/blk-mq.h> #include <linux/part_stat.h> #include "blk.h"
block/blk-mq-cpumap.c +0 -1
@@ -12,7 +12,6 @@#include <linux/cpu.h> #include <linux/group_cpus.h> -#include <linux/blk-mq.h> #include "blk.h" #include "blk-mq.h"
block/blk-mq-debugfs.c +0 -1
@@ -7,7 +7,6 @@#include <linux/blkdev.h> #include <linux/debugfs.h> -#include <linux/blk-mq.h> #include "blk.h" #include "blk-mq.h" #include "blk-mq-debugfs.h"
block/blk-mq-pci.c +0 -1
@@ -4,7 +4,6 @@*/ #include <linux/kobject.h> #include <linux/blkdev.h> -#include <linux/blk-mq.h> #include <linux/blk-mq-pci.h> #include <linux/pci.h> #include <linux/module.h>
block/blk-mq-sched.c +0 -1
@@ -6,7 +6,6 @@*/ #include <linux/kernel.h> #include <linux/module.h> -#include <linux/blk-mq.h> #include <linux/list_sort.h> #include <trace/events/block.h>
block/blk-mq-sysfs.c +0 -1
@@ -10,7 +10,6 @@#include <linux/workqueue.h> #include <linux/smp.h> -#include <linux/blk-mq.h> #include "blk.h" #include "blk-mq.h"
block/blk-mq-tag.c +0 -1
@@ -9,7 +9,6 @@#include <linux/kernel.h> #include <linux/module.h> -#include <linux/blk-mq.h> #include <linux/delay.h> #include "blk.h" #include "blk-mq.h"
block/blk-mq-virtio.c +0 -1
@@ -3,7 +3,6 @@* Copyright (c) 2016 Christoph Hellwig. */ #include <linux/device.h> -#include <linux/blk-mq.h> #include <linux/blk-mq-virtio.h> #include <linux/virtio_config.h> #include <linux/module.h>
block/blk-mq.c +0 -1
@@ -32,7 +32,6 @@#include <trace/events/block.h> -#include <linux/blk-mq.h> #include <linux/t10-pi.h> #include "blk.h" #include "blk-mq.h"
block/blk-mq.h +1 -0
@@ -2,6 +2,7 @@#ifndef INT_BLK_MQ_H #define INT_BLK_MQ_H +#include <linux/blk-mq.h> #include "blk-stat.h" struct blk_mq_tag_set ;
block/blk-pm.c +0 -1
@@ -1,6 +1,5 @@// SPDX-License-Identifier: GPL-2.0 -#include <linux/blk-mq.h> #include <linux/blk-pm.h> #include <linux/blkdev.h> #include <linux/pm_runtime.h>
block/blk-stat.c +0 -1
@@ -6,7 +6,6 @@*/ #include <linux/kernel.h> #include <linux/rculist.h> -#include <linux/blk-mq.h> #include "blk-stat.h" #include "blk-mq.h"
block/blk-sysfs.c +0 -1
@@ -9,7 +9,6 @@#include <linux/blkdev.h> #include <linux/backing-dev.h> #include <linux/blktrace_api.h> -#include <linux/blk-mq.h> #include <linux/debugfs.h> #include "blk.h"
block/kyber-iosched.c +0 -1
@@ -8,7 +8,6 @@#include <linux/kernel.h> #include <linux/blkdev.h> -#include <linux/blk-mq.h> #include <linux/module.h> #include <linux/sbitmap.h>
block/mq-deadline.c +0 -1
@@ -8,7 +8,6 @@#include <linux/kernel.h> #include <linux/fs.h> #include <linux/blkdev.h> -#include <linux/blk-mq.h> #include <linux/bio.h> #include <linux/module.h> #include <linux/slab.h>
Signature · 1 line
2.39.2
Re: [PATCH 03/18] blk-mq: include <linux/blk-mq.h> in block/blk-mq.h
Quoted 5 lines · Christoph Hellwig
block/blk-mq.hneeds various definitions from <linux/blk-mq.h>, include it there instead of relying on the source files to include both.Signed-off-by Christoph Hellwig <[email protected]>
Looks good to me.
PATCH 4/18 blk-mq: move more logic into blk_mq_insert_requests
In reply to Christoph HellwigMove all logic related to the direct insert into blk_mq_insert_requests to clean the code flow up a bit, and to allow marking blk_mq_try_issue_list_directly static.
--- block/blk-mq-sched.c | 17 ++--------------- block/blk-mq.c | 20 ++++++++++++++++++-- block/blk-mq.h | 4 +--- 3 files changed, 21 insertions(+), 20 deletions(-)
block/blk-mq-sched.c +2 -15
@@ -472,23 +472,10 @@ void blk_mq_sched_insert_requests(struct blk_mq_hw_ctx *hctx, e = hctx->queue->elevator;if (e) { e->type->ops.insert_requests (hctx, list, false); +blk_mq_run_hw_queue (hctx, run_queue_async); }else { -/* -* try to issue requests directly if the hw queue isn't -* busy in case of 'none' scheduler, and this way may save -* us one extra enqueue & dequeue to sw queue. -*/ -if (!hctx->dispatch_busy && !run_queue_async) { -blk_mq_run_dispatch_ops (hctx->queue, -blk_mq_try_issue_list_directly (hctx, list)); -if (list_empty (list)) - goto out; - } -blk_mq_insert_requests (hctx, ctx, list); +blk_mq_insert_requests (hctx, ctx, list, run_queue_async); } - -blk_mq_run_hw_queue (hctx, run_queue_async); - out:percpu_ref_put (&q->q_usage_counter); }
block/blk-mq.c +18 -2
@@ -44,6 +44,9 @@ static DEFINE_PER_CPU(struct llist_head, blk_cpu_done); +static void blk_mq_try_issue_list_directly (struct blk_mq_hw_ctx *hctx, +struct list_head *list); +static inline struct blk_mq_hw_ctx *blk_qc_to_hctx (struct request_queue *q,blk_qc_t qc) {
@@ -2495,12 +2498,23 @@ void blk_mq_request_bypass_insert(struct request *rq, bool at_head, }void blk_mq_insert_requests (struct blk_mq_hw_ctx *hctx,struct blk_mq_ctx *ctx, -struct list_head *list) +struct list_head *list,bool run_queue_async) {struct request *rq;enum hctx_type type = hctx->type; +/* +* Try to issue requests directly if the hw queue isn't busy to save an +* extra enqueue & dequeue to the sw queue. +*/ +if (!hctx->dispatch_busy && !run_queue_async) { +blk_mq_run_dispatch_ops (hctx->queue, +blk_mq_try_issue_list_directly (hctx, list)); +if (list_empty (list)) + goto out; + } +/* * preemption doesn't flush plug list, so it's possible ctx->cpu is * offline now
@@ -2514,6 +2528,8 @@ void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,list_splice_tail_init (list, &ctx->rq_lists[type]);blk_mq_hctx_mark_pending (hctx, ctx);spin_unlock (&ctx->lock); +out: +blk_mq_run_hw_queue (hctx, run_queue_async); } static void blk_mq_bio_to_request(struct request *rq, struct bio *bio,
@@ -2755,7 +2771,7 @@ void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule) }while (!rq_list_empty (plug->mq_list)); } -void blk_mq_try_issue_list_directly (struct blk_mq_hw_ctx *hctx, +static void blk_mq_try_issue_list_directly (struct blk_mq_hw_ctx *hctx,struct list_head *list) {int queued =0 ;
block/blk-mq.h +1 -3
@@ -70,9 +70,7 @@ void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,void blk_mq_request_bypass_insert (struct request *rq,bool at_head,bool run_queue);void blk_mq_insert_requests (struct blk_mq_hw_ctx *hctx,struct blk_mq_ctx *ctx, -struct list_head *list); -void blk_mq_try_issue_list_directly (struct blk_mq_hw_ctx *hctx, -struct list_head *list); +struct list_head *list,bool run_queue_async);/* * CPU -> queue mappings
Signature · 1 line
2.39.2
Re: [PATCH 04/18] blk-mq: move more logic into blk_mq_insert_requests
Quoted 3 lines · Christoph Hellwig
Move all logic related to the direct insert into
blk_mq_insert_requeststo clean the code flow up a bit, and to allow markingblk_mq_try_issue_list_directlystatic.
Nit: maybe mention that blk_mq_insert_requests() will now call blk_mq_run_hw_queue(), just to be clear (even though that is implied by the "move all logic" statement).
Quoted 3 lines
Signed-off-by Christoph Hellwig <[email protected]>Reviewed-by Bart Van Assche <[email protected]>
Looks good to me.
PATCH 5/18 blk-mq: fold blk_mq_sched_insert_requests into blk_mq_dispatch_plug_list
In reply to Christoph Hellwigblk_mq_dispatch_plug_list is the only caller of blk_mq_sched_insert_requests, and it makes sense to just fold it there as blk_mq_sched_insert_requests isn't specific to I/O schedulers despite the name.
--- block/blk-mq-sched.c | 24 ------------------------ block/blk-mq-sched.h | 3 --- block/blk-mq.c | 17 +++++++++++++---- block/blk-mq.h | 2 -- block/mq-deadline.c | 2 +- 5 files changed, 14 insertions(+), 34 deletions(-)
block/blk-mq-sched.c +0 -24
@@ -455,30 +455,6 @@ void blk_mq_sched_insert_request(struct request *rq, bool at_head,blk_mq_run_hw_queue (hctx, async); } -void blk_mq_sched_insert_requests (struct blk_mq_hw_ctx *hctx, -struct blk_mq_ctx *ctx, -struct list_head *list,bool run_queue_async) -{ -struct elevator_queue *e; -struct request_queue *q = hctx->queue; - -/* -* blk_mq_sched_insert_requests() is called from flush plug -* context only, and hold one usage counter to prevent queue -* from being released. -*/ -percpu_ref_get (&q->q_usage_counter); - - e = hctx->queue->elevator; -if (e) { - e->type->ops.insert_requests (hctx, list, false); -blk_mq_run_hw_queue (hctx, run_queue_async); - }else { -blk_mq_insert_requests (hctx, ctx, list, run_queue_async); - } -percpu_ref_put (&q->q_usage_counter); -} -static int blk_mq_sched_alloc_map_and_rqs (struct request_queue *q,struct blk_mq_hw_ctx *hctx,unsigned int hctx_idx)
block/blk-mq-sched.h +0 -3
@@ -18,9 +18,6 @@ void __blk_mq_sched_restart(struct blk_mq_hw_ctx *hctx);void blk_mq_sched_insert_request (struct request *rq,bool at_head,bool run_queue,bool async); -void blk_mq_sched_insert_requests (struct blk_mq_hw_ctx *hctx, -struct blk_mq_ctx *ctx, -struct list_head *list,bool run_queue_async);void blk_mq_sched_dispatch_requests (struct blk_mq_hw_ctx *hctx);
block/blk-mq.c +13 -4
@@ -2497,9 +2497,9 @@ void blk_mq_request_bypass_insert(struct request *rq, bool at_head,blk_mq_run_hw_queue (hctx, false); } -void blk_mq_insert_requests (struct blk_mq_hw_ctx *hctx,struct blk_mq_ctx *ctx, -struct list_head *list,bool run_queue_async) - +static void blk_mq_insert_requests (struct blk_mq_hw_ctx *hctx, +struct blk_mq_ctx *ctx,struct list_head *list, +bool run_queue_async) {struct request *rq;enum hctx_type type = hctx->type;
@@ -2725,7 +2725,16 @@ static void blk_mq_dispatch_plug_list(struct blk_plug *plug, bool from_sched) plug->mq_list = requeue_list;trace_block_unplug (this_hctx->queue, depth, !from_sched); -blk_mq_sched_insert_requests (this_hctx, this_ctx, &list, from_sched); + +percpu_ref_get (&this_hctx->queue->q_usage_counter); +if (this_hctx->queue->elevator) { + this_hctx->queue->elevator->type->ops.insert_requests (this_hctx, + &list, false); +blk_mq_run_hw_queue (this_hctx, from_sched); + }else { +blk_mq_insert_requests (this_hctx, this_ctx, &list, from_sched); + } +percpu_ref_put (&this_hctx->queue->q_usage_counter); }void blk_mq_flush_plug_list (struct blk_plug *plug,bool from_schedule)
block/blk-mq.h +0 -2
@@ -69,8 +69,6 @@ void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, bool at_head);void blk_mq_request_bypass_insert (struct request *rq,bool at_head,bool run_queue); -void blk_mq_insert_requests (struct blk_mq_hw_ctx *hctx,struct blk_mq_ctx *ctx, -struct list_head *list,bool run_queue_async);/* * CPU -> queue mappings
block/mq-deadline.c +1 -1
@@ -820,7 +820,7 @@ static void dd_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, }/* -* Called from blk_mq_sched_insert_request() or blk_mq_sched_insert_requests(). +* Called from blk_mq_sched_insert_request() or blk_mq_dispatch_plug_list(). */ static void dd_insert_requests (struct blk_mq_hw_ctx *hctx,struct list_head *list,bool at_head)
Signature · 1 line
2.39.2
Re: [PATCH 05/18] blk-mq: fold blk_mq_sched_insert_requests into blk_mq_dispatch_plug_list
Quoted 7 lines · Christoph Hellwig
blk_mq_dispatch_plug_listis the only caller ofblk_mq_sched_insert_requests, and it makes sense to just fold it there asblk_mq_sched_insert_requestsisn't specific to I/O schedulers despite the name.Signed-off-by Christoph Hellwig <[email protected]>Reviewed-by Bart Van Assche <[email protected]>
Looks good to me.
PATCH 6/18 blk-mq: move blk_mq_sched_insert_request to blk-mq.c
In reply to Christoph Hellwigblk_mq_sched_insert_request is the main request insert helper and not directly I/O scheduler related. Move blk_mq_sched_insert_request to blk-mq.c, rename it to blk_mq_insert_request and mark it static.
--- block/blk-mq-sched.c | 73 ------------------------------------- block/blk-mq-sched.h | 3 -- block/blk-mq.c | 87 +++++++++++++++++++++++++++++++++++++++++--- block/mq-deadline.c | 2 +- 4 files changed, 82 insertions(+), 83 deletions(-)
block/blk-mq-sched.c +0 -73
@@ -382,79 +382,6 @@ bool blk_mq_sched_try_insert_merge(struct request_queue *q, struct request *rq, }EXPORT_SYMBOL_GPL (blk_mq_sched_try_insert_merge); -static bool blk_mq_sched_bypass_insert (struct blk_mq_hw_ctx *hctx, -struct request *rq) -{ -/* -* dispatch flush and passthrough rq directly -* -* passthrough request has to be added to hctx->dispatch directly. -* For some reason, device may be in one situation which can't -* handle FS request, so STS_RESOURCE is always returned and the -* FS request will be added to hctx->dispatch. However passthrough -* request may be required at that time for fixing the problem. If -* passthrough request is added to scheduler queue, there isn't any -* chance to dispatch it given we prioritize requests in hctx->dispatch. -*/ -if ((rq->rq_flags &RQF_FLUSH_SEQ ) ||blk_rq_is_passthrough (rq)) -return true; - -return false; -} - -void blk_mq_sched_insert_request (struct request *rq,bool at_head, -bool run_queue,bool async) -{ -struct request_queue *q = rq->q; -struct elevator_queue *e = q->elevator; -struct blk_mq_ctx *ctx = rq->mq_ctx; -struct blk_mq_hw_ctx *hctx = rq->mq_hctx; - -WARN_ON (e && (rq->tag !=BLK_MQ_NO_TAG )); - -if (blk_mq_sched_bypass_insert (hctx, rq)) { -/* -* Firstly normal IO request is inserted to scheduler queue or -* sw queue, meantime we add flush request to dispatch queue( -* hctx->dispatch) directly and there is at most one in-flight -* flush request for each hw queue, so it doesn't matter to add -* flush request to tail or front of the dispatch queue. -* -* Secondly in case of NCQ, flush request belongs to non-NCQ -* command, and queueing it will fail when there is any -* in-flight normal IO request(NCQ command). When adding flush -* rq to the front of hctx->dispatch, it is easier to introduce -* extra time to flush rq's latency because of S_SCHED_RESTART -* compared with adding to the tail of dispatch queue, then -* chance of flush merge is increased, and less flush requests -* will be issued to controller. It is observed that ~10% time -* is saved in blktests block/004 on disk attached to AHCI/NCQ -* drive when adding flush rq to the front of hctx->dispatch. -* -* Simply queue flush rq to the front of hctx->dispatch so that -* intensive flush workloads can benefit in case of NCQ HW. -*/ - at_head = (rq->rq_flags &RQF_FLUSH_SEQ ) ? true : at_head; -blk_mq_request_bypass_insert (rq, at_head, false); - goto run; - } - -if (e) { -LIST_HEAD (list); - -list_add (&rq->queuelist, &list); - e->type->ops.insert_requests (hctx, &list, at_head); - }else { -spin_lock (&ctx->lock); -__blk_mq_insert_request (hctx, rq, at_head); -spin_unlock (&ctx->lock); - } - -run: -if (run_queue) -blk_mq_run_hw_queue (hctx, async); -} -static int blk_mq_sched_alloc_map_and_rqs (struct request_queue *q,struct blk_mq_hw_ctx *hctx,unsigned int hctx_idx)
block/blk-mq-sched.h +0 -3
@@ -16,9 +16,6 @@ bool blk_mq_sched_try_insert_merge(struct request_queue *q, struct request *rq,void blk_mq_sched_mark_restart_hctx (struct blk_mq_hw_ctx *hctx);void __blk_mq_sched_restart (struct blk_mq_hw_ctx *hctx); -void blk_mq_sched_insert_request (struct request *rq,bool at_head, -bool run_queue,bool async); -void blk_mq_sched_dispatch_requests (struct blk_mq_hw_ctx *hctx);int blk_mq_init_sched (struct request_queue *q,struct elevator_type *e);
block/blk-mq.c +81 -6
134 lines not on this page · raw message
block/mq-deadline.c +1 -1
@@ -820,7 +820,7 @@ static void dd_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, }/* -* Called from blk_mq_sched_insert_request() or blk_mq_dispatch_plug_list(). +* Called from blk_mq_insert_request() or blk_mq_dispatch_plug_list(). */ static void dd_insert_requests (struct blk_mq_hw_ctx *hctx,struct list_head *list,bool at_head)
Signature · 1 line
2.39.2
Re: [PATCH 06/18] blk-mq: move blk_mq_sched_insert_request to blk-mq.c
Quoted 6 lines · Christoph Hellwig
blk_mq_sched_insert_requestis the main request insert helper and not directly I/O scheduler related. Moveblk_mq_sched_insert_requesttoblk-mq.c, rename it toblk_mq_insert_requestand mark it static.Signed-off-by Christoph Hellwig <[email protected]>Reviewed-by Bart Van Assche <[email protected]>
[...]
block/blk-mq-sched.c bool blk_mq_sched_try_insert_merge(struct request_queue *q, struct request *rq, · line 415
Quoted 11 lines
block/blk-mq-sched.c +0
-11-void blk_mq_sched_insert_request (struct request *rq,bool at_head, -bool run_queue,bool async) -{ -struct request_queue *q = rq->q; -struct elevator_queue *e = q->elevator; -struct blk_mq_ctx *ctx = rq->mq_ctx; -struct blk_mq_hw_ctx *hctx = rq->mq_hctx; - -WARN_ON (e && (rq->tag !=BLK_MQ_NO_TAG )); - -if (blk_mq_sched_bypass_insert (hctx, rq)) {
Nit: given the super confusing name that blk_mq_sched_bypass_insert() has, replacing the above if with:
if ((rq->rq_flags & RQF_FLUSH_SEQ) || blk_rq_is_passthrough(rq))
with the comment above it would make things even more readable I think.
Otherwise looks good.
Re: [PATCH 06/18] blk-mq: move blk_mq_sched_insert_request to blk-mq.c
Quoted 25 lines · Damien Le Moal
Quoted 6 lines · Christoph Hellwig
blk_mq_sched_insert_requestis the main request insert helper and not directly I/O scheduler related. Moveblk_mq_sched_insert_requesttoblk-mq.c, rename it toblk_mq_insert_requestand mark it static.Signed-off-by Christoph Hellwig <[email protected]>Reviewed-by Bart Van Assche <[email protected]>[...]
block/blk-mq-sched.c bool blk_mq_sched_try_insert_merge(struct request_queue *q, struct request *rq, · line 415
Quoted 11 lines
block/blk-mq-sched.c +0
-11-void blk_mq_sched_insert_request (struct request *rq,bool at_head, -bool run_queue,bool async) -{ -struct request_queue *q = rq->q; -struct elevator_queue *e = q->elevator; -struct blk_mq_ctx *ctx = rq->mq_ctx; -struct blk_mq_hw_ctx *hctx = rq->mq_hctx; - -WARN_ON (e && (rq->tag !=BLK_MQ_NO_TAG )); - -if (blk_mq_sched_bypass_insert (hctx, rq)) {Nit: given the super confusing name that
blk_mq_sched_bypass_insert()has, replacing the above if with:
Quoted 3 lines
if ((rq->rq_flags & RQF_FLUSH_SEQ) || blk_rq_is_passthrough(rq))with the comment above it would make things even more readable I think.
Ignore. Just saw you reworked this in patch 10.
Quoted 6 lines
Otherwise looks good.
Reviewed-by Damien Le Moal <[email protected]>
PATCH 7/18 blk-mq: fold __blk_mq_insert_request into blk_mq_insert_request
In reply to Christoph HellwigThere is no good point in keeping the __blk_mq_insert_request around for two function calls and a singler caller.
--- block/blk-mq.c | 14 ++------------ block/blk-mq.h | 2 -- 2 files changed, 2 insertions(+), 14 deletions(-)
block/blk-mq.c +2 -12
@@ -2463,17 +2463,6 @@ static inline void __blk_mq_insert_req_list(struct blk_mq_hw_ctx *hctx,list_add_tail (&rq->queuelist, &ctx->rq_lists[type]); } -void __blk_mq_insert_request (struct blk_mq_hw_ctx *hctx,struct request *rq, -bool at_head) -{ -struct blk_mq_ctx *ctx = rq->mq_ctx; - -lockdep_assert_held (&ctx->lock); - -__blk_mq_insert_req_list (hctx, rq, at_head); -blk_mq_hctx_mark_pending (hctx, ctx); -} -/** * blk_mq_request_bypass_insert - Insert a request at dispatch list. * @rq: Pointer to request to be inserted.
@@ -2598,7 +2587,8 @@ static void blk_mq_insert_request(struct request *rq, bool at_head, e->type->ops.insert_requests (hctx, &list, at_head); }else {spin_lock (&ctx->lock); -__blk_mq_insert_request (hctx, rq, at_head); +__blk_mq_insert_req_list (hctx, rq, at_head); +blk_mq_hctx_mark_pending (hctx, ctx);spin_unlock (&ctx->lock); }
block/blk-mq.h +0 -2
@@ -65,8 +65,6 @@ void blk_mq_free_map_and_rqs(struct blk_mq_tag_set *set,/* * Internal helpers for request insertion into sw queues */ -void __blk_mq_insert_request (struct blk_mq_hw_ctx *hctx,struct request *rq, -bool at_head);void blk_mq_request_bypass_insert (struct request *rq,bool at_head,bool run_queue);
Signature · 1 line
2.39.2
Re: [PATCH 07/18] blk-mq: fold __blk_mq_insert_request into blk_mq_insert_request
Quoted 5 lines · Christoph Hellwig
There is no good point in keeping the
__blk_mq_insert_requestaround for two function calls and a singler caller.Signed-off-by Christoph Hellwig <[email protected]>Reviewed-by Bart Van Assche <[email protected]>
Looks good.
PATCH 8/18 blk-mq: fold __blk_mq_insert_req_list into blk_mq_insert_request
In reply to Christoph HellwigRemove this very small helper and fold it into the only caller.
--- block/blk-mq.c | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-)
block/blk-mq.c +7 -18
@@ -2446,23 +2446,6 @@ static void blk_mq_run_work_fn(struct work_struct *work)__blk_mq_run_hw_queue (hctx); } -static inline void __blk_mq_insert_req_list (struct blk_mq_hw_ctx *hctx, -struct request *rq, -bool at_head) -{ -struct blk_mq_ctx *ctx = rq->mq_ctx; -enum hctx_type type = hctx->type; - -lockdep_assert_held (&ctx->lock); - -trace_block_rq_insert (rq); - -if (at_head) -list_add (&rq->queuelist, &ctx->rq_lists[type]); -else -list_add_tail (&rq->queuelist, &ctx->rq_lists[type]); -} -/** * blk_mq_request_bypass_insert - Insert a request at dispatch list. * @rq: Pointer to request to be inserted.
@@ -2586,8 +2569,14 @@ static void blk_mq_insert_request(struct request *rq, bool at_head,list_add (&rq->queuelist, &list); e->type->ops.insert_requests (hctx, &list, at_head); }else { +trace_block_rq_insert (rq); +spin_lock (&ctx->lock); -__blk_mq_insert_req_list (hctx, rq, at_head); +if (at_head) +list_add (&rq->queuelist, &ctx->rq_lists[hctx->type]); +else +list_add_tail (&rq->queuelist, + &ctx->rq_lists[hctx->type]);blk_mq_hctx_mark_pending (hctx, ctx);spin_unlock (&ctx->lock); }
Signature · 1 line
2.39.2
Re: [PATCH 08/18] blk-mq: fold __blk_mq_insert_req_list into blk_mq_insert_request
Quoted 37 lines · Christoph Hellwig
Remove this very small helper and fold it into the only caller.
Signed-off-by Christoph Hellwig <[email protected]>Reviewed-by Bart Van Assche <[email protected]>--- block/blk-mq.c | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-)block/blk-mq.c +0
-17@@ -2446,23 +2446,6 @@ static void blk_mq_run_work_fn(struct work_struct *work)__blk_mq_run_hw_queue (hctx); } -static inline void __blk_mq_insert_req_list (struct blk_mq_hw_ctx *hctx, -struct request *rq, -bool at_head) -{ -struct blk_mq_ctx *ctx = rq->mq_ctx; -enum hctx_type type = hctx->type; - -lockdep_assert_held (&ctx->lock); - -trace_block_rq_insert (rq); - -if (at_head) -list_add (&rq->queuelist, &ctx->rq_lists[type]); -else -list_add_tail (&rq->queuelist, &ctx->rq_lists[type]); -} -/** * blk_mq_request_bypass_insert - Insert a request at dispatch list. * @rq: Pointer to request to be inserted.
block/blk-mq.c static void blk_mq_insert_request(struct request *rq, bool at_head, · line 2572
Quoted 4 lines
block/blk-mq.c +1
-0@@ -2586,8 +2569,14 @@ static void blk_mq_insert_request(struct request *rq, bool at_head,list_add (&rq->queuelist, &list); e->type->ops.insert_requests (hctx, &list, at_head); }else { +trace_block_rq_insert (rq);
Shouldn't we keep the trace call under ctx->lock to preserve precise tracing ?
Quoted 11 lines
block/blk-mq.c +6
-1+spin_lock (&ctx->lock); -__blk_mq_insert_req_list (hctx, rq, at_head); +if (at_head) +list_add (&rq->queuelist, &ctx->rq_lists[hctx->type]); +else +list_add_tail (&rq->queuelist, + &ctx->rq_lists[hctx->type]);blk_mq_hctx_mark_pending (hctx, ctx);spin_unlock (&ctx->lock); }
Re: [PATCH 08/18] blk-mq: fold __blk_mq_insert_req_list into blk_mq_insert_request
Quoted 4 lines · Damien Le Moal
block/blk-mq.c static void blk_mq_insert_request(struct request *rq, bool at_head, · line 2572
Quoted 2 lines
block/blk-mq.c +1
-0}else { +trace_block_rq_insert (rq);Shouldn't we keep the trace call under
ctx->lockto preserve precise tracing ?
ctx->lock doesn't synchronize any of the in the request that is traced here.
Re: [PATCH 08/18] blk-mq: fold __blk_mq_insert_req_list into blk_mq_insert_request
Quoted 8 lines · Christoph Hellwig
Quoted 4 lines · Damien Le Moal
block/blk-mq.c static void blk_mq_insert_request(struct request *rq, bool at_head, · line 2572
Quoted 2 lines
block/blk-mq.c +1
-0}else { +trace_block_rq_insert (rq);Shouldn't we keep the trace call under
ctx->lockto preserve precise tracing ?
ctx->lockdoesn't synchronize any of the in the request that is traced here.
I am not worried about the values shown by the trace entries, but rather the order of the inserts: with the trace call outside the lock, the trace may end up showing an incorrect insertion order ?
Re: [PATCH 08/18] blk-mq: fold __blk_mq_insert_req_list into blk_mq_insert_request
Quoted 7 lines · Damien Le Moal
Quoted 6 lines · Christoph Hellwig
Quoted 4 lines · Damien Le Moal
block/blk-mq.c static void blk_mq_insert_request(struct request *rq, bool at_head, · line 2572
Quoted 2 lines
block/blk-mq.c +1
-0}else { +trace_block_rq_insert (rq);Shouldn't we keep the trace call under
ctx->lockto preserve precise tracing ?
Quoted 6 lines
Quoted 2 lines
ctx->lockdoesn't synchronize any of the in the request that is traced here.I am not worried about the values shown by the trace entries, but rather the order of the inserts: with the trace call outside the lock, the trace may end up showing an incorrect insertion order ?
Maybe. I can respin the series and move it back under the lock.
Re: [PATCH 08/18] blk-mq: fold __blk_mq_insert_req_list into blk_mq_insert_request
In reply to Damien Le MoalQuoted 3 lines · Damien Le Moal
I am not worried about the values shown by the trace entries, but rather the order of the inserts: with the trace call outside the lock, the trace may end up showing an incorrect insertion order ?
... turns out none of the other calls to trace_block_rq_insert is under ctx->lock either. The I/O scheduler ones are under their own per-request_queue locks, so maybe that counts as ordering, but blk_mq_insert_requests doesn't lock at all.
Damien Le Moal
Reviewed-byRe: [PATCH 08/18] blk-mq: fold __blk_mq_insert_req_list into blk_mq_insert_request
Quoted 9 lines · Christoph Hellwig
Quoted 3 lines · Damien Le Moal
I am not worried about the values shown by the trace entries, but rather the order of the inserts: with the trace call outside the lock, the trace may end up showing an incorrect insertion order ?
... turns out none of the other calls to
trace_block_rq_insertis underctx->lockeither. The I/O scheduler ones are under their own per-request_queue locks, so maybe that counts as ordering, butblk_mq_insert_requestsdoesn't lock at all.
OK. And since nobody ever complained (that I know of), I guess it is fine then. Feel free to add:
PATCH 9/18 blk-mq: remove blk_flush_queue_rq
In reply to Christoph HellwigJust call blk_mq_add_to_requeue_list directly from the two callers.
--- block/blk-flush.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-)
block/blk-flush.c +2 -7
@@ -136,11 +136,6 @@ static void blk_flush_restore_request(struct request *rq) rq->end_io = rq->flush.saved_end_io; } -static void blk_flush_queue_rq (struct request *rq,bool add_front) -{ -blk_mq_add_to_requeue_list (rq, add_front, true); -} -static void blk_account_io_flush (struct request *rq) {struct block_device *part = rq->q->disk->part0;
@@ -193,7 +188,7 @@ static void blk_flush_complete_seq(struct request *rq,case REQ_FSEQ_DATA :list_move_tail (&rq->flush.list, &fq->flush_data_in_flight); -blk_flush_queue_rq (rq, true); +blk_mq_add_to_requeue_list (rq, true, true);break ;case REQ_FSEQ_DONE :
@@ -350,7 +345,7 @@ static void blk_kick_flush(struct request_queue *q, struct blk_flush_queue *fq,smp_wmb ();req_ref_set (flush_rq,1 ); -blk_flush_queue_rq (flush_rq, false); +blk_mq_add_to_requeue_list (flush_rq, false, true); } static enum rq_end_io_ret mq_flush_data_end_io(struct request *rq,
Signature · 1 line
2.39.2
Re: [PATCH 09/18] blk-mq: remove blk_flush_queue_rq
Quoted 4 lines · Christoph Hellwig
Just call
blk_mq_add_to_requeue_listdirectly from the two callers.Signed-off-by Christoph Hellwig <[email protected]>Reviewed-by Bart Van Assche <[email protected]>
Looks good.
PATCH 10/18 blk-mq: refactor passthrough vs flush handling in blk_mq_insert_request
In reply to Christoph HellwigWhile both passthrough and flush requests call directly into blk_mq_request_bypass_insert, the parameters aren't the same. Split the handling into two separate conditionals and turn the whole function into an if/elif/elif/else flow instead of the gotos.
--- block/blk-mq.c | 50 ++++++++++++++++++-------------------------------- 1 file changed, 18 insertions(+), 32 deletions(-)
block/blk-mq.c +18 -32
@@ -2506,37 +2506,26 @@ static void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx,blk_mq_run_hw_queue (hctx, run_queue_async); } -static bool blk_mq_sched_bypass_insert (struct blk_mq_hw_ctx *hctx, -struct request *rq) -{ -/* -* dispatch flush and passthrough rq directly -* -* passthrough request has to be added to hctx->dispatch directly. -* For some reason, device may be in one situation which can't -* handle FS request, so STS_RESOURCE is always returned and the -* FS request will be added to hctx->dispatch. However passthrough -* request may be required at that time for fixing the problem. If -* passthrough request is added to scheduler queue, there isn't any -* chance to dispatch it given we prioritize requests in hctx->dispatch. -*/ -if ((rq->rq_flags &RQF_FLUSH_SEQ ) ||blk_rq_is_passthrough (rq)) -return true; - -return false; -} -static void blk_mq_insert_request (struct request *rq,bool at_head,bool run_queue,bool async) {struct request_queue *q = rq->q; -struct elevator_queue *e = q->elevator;struct blk_mq_ctx *ctx = rq->mq_ctx;struct blk_mq_hw_ctx *hctx = rq->mq_hctx; -WARN_ON (e && (rq->tag !=BLK_MQ_NO_TAG )); - -if (blk_mq_sched_bypass_insert (hctx, rq)) { +if (blk_rq_is_passthrough (rq)) { +/* +* Passthrough request have to be added to hctx->dispatch +* directly. The device may be in a situation where it can't +* handle FS request, and always returns BLK_STS_RESOURCE for +* them, which gets them added to hctx->dispatch. +* +* If a passthrough request is required to unblock the queues, +* and it is added to the scheduler queue, there is no chance to +* dispatch it given we prioritize requests in hctx->dispatch. +*/ +blk_mq_request_bypass_insert (rq, at_head, false); + }else if (rq->rq_flags &RQF_FLUSH_SEQ ) {/* * Firstly normal IO request is inserted to scheduler queue or * sw queue, meantime we add flush request to dispatch queue(
@@ -2558,16 +2547,14 @@ static void blk_mq_insert_request(struct request *rq, bool at_head,* Simply queue flush rq to the front of hctx->dispatch so that * intensive flush workloads can benefit in case of NCQ HW. */ - at_head = (rq->rq_flags &RQF_FLUSH_SEQ ) ? true : at_head; -blk_mq_request_bypass_insert (rq, at_head, false); - goto run; - } - -if (e) { +blk_mq_request_bypass_insert (rq, true, false); + }else if (q->elevator) {LIST_HEAD (list); +WARN_ON_ONCE (rq->tag !=BLK_MQ_NO_TAG ); +list_add (&rq->queuelist, &list); - e->type->ops.insert_requests (hctx, &list, at_head); + q->elevator->type->ops.insert_requests (hctx, &list, at_head); }else {trace_block_rq_insert (rq);
@@ -2581,7 +2568,6 @@ static void blk_mq_insert_request(struct request *rq, bool at_head,spin_unlock (&ctx->lock); } -run:if (run_queue)blk_mq_run_hw_queue (hctx, async); }
Signature · 1 line
2.39.2
Re: [PATCH 10/18] blk-mq: refactor passthrough vs flush handling in blk_mq_insert_request
Quoted 7 lines · Christoph Hellwig
While both passthrough and flush requests call directly into
blk_mq_request_bypass_insert, the parameters aren't the same. Split the handling into two separate conditionals and turn the whole function into an if/elif/elif/else flow instead of the gotos.Signed-off-by Christoph Hellwig <[email protected]>Reviewed-by Bart Van Assche <[email protected]>
Looks good.
PATCH 11/18 blk-mq: refactor the DONTPREP/SOFTBARRIER andling in blk_mq_requeue_work
In reply to Christoph HellwigSplit the RQF_DONTPREP and RQF_SOFTBARRIER in separate branches to make the code more readable.
--- block/blk-mq.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-)
block/blk-mq.c +7 -7
@@ -1427,20 +1427,20 @@ static void blk_mq_requeue_work(struct work_struct *work)spin_unlock_irq (&q->requeue_lock);list_for_each_entry_safe (rq, next, &rq_list, queuelist) { -if (!(rq->rq_flags & (RQF_SOFTBARRIER |RQF_DONTPREP ))) -continue ; - - rq->rq_flags &= ~RQF_SOFTBARRIER ; -list_del_init (&rq->queuelist);/* * If RQF_DONTPREP, rq has contained some driver specific * data, so insert it to hctx dispatch list to avoid any * merge. */ -if (rq->rq_flags &RQF_DONTPREP ) +if (rq->rq_flags &RQF_DONTPREP ) { + rq->rq_flags &= ~RQF_SOFTBARRIER ; +list_del_init (&rq->queuelist);blk_mq_request_bypass_insert (rq, false, false); -else + }else if (rq->rq_flags &RQF_SOFTBARRIER ) { + rq->rq_flags &= ~RQF_SOFTBARRIER ; +list_del_init (&rq->queuelist);blk_mq_insert_request (rq, true, false, false); + } }while (!list_empty (&rq_list)) {
Signature · 1 line
2.39.2
Re: [PATCH 11/18] blk-mq: refactor the DONTPREP/SOFTBARRIER andling in blk_mq_requeue_work
Quoted 14 lines · Christoph Hellwig
Split the
RQF_DONTPREPandRQF_SOFTBARRIERin separate branches to make the code more readable.Signed-off-by Christoph Hellwig <[email protected]>Reviewed-by Bart Van Assche <[email protected]>--- block/blk-mq.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-)block/blk-mq.c +0
-0
block/blk-mq.c static void blk_mq_requeue_work(struct work_struct *work) · line 1431
Quoted 10 lines
block/blk-mq.c +0
-5@@ -1427,20 +1427,20 @@ static void blk_mq_requeue_work(struct work_struct *work)spin_unlock_irq (&q->requeue_lock);list_for_each_entry_safe (rq, next, &rq_list, queuelist) { -if (!(rq->rq_flags & (RQF_SOFTBARRIER |RQF_DONTPREP ))) -continue ; - - rq->rq_flags &= ~RQF_SOFTBARRIER ; -list_del_init (&rq->queuelist);/* * If RQF_DONTPREP, rq has contained some driver specific
Nit: while at it, you could fix the bad english here:
rq has contained som... -> rq has some...
Otherwise looks good.
PATCH 12/18 blk-mq: factor out a blk_mq_get_budget_and_tag helper
In reply to Christoph HellwigFactor out a helper from __blk_mq_try_issue_directly in preparation of folding that function into its two callers.
--- block/blk-mq.c | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-)
block/blk-mq.c +16 -10
@@ -2623,13 +2623,27 @@ static blk_status_t __blk_mq_issue_directly(struct blk_mq_hw_ctx *hctx,return ret; } +static bool blk_mq_get_budget_and_tag (struct request *rq) +{ +int budget_token; + + budget_token =blk_mq_get_dispatch_budget (rq->q); +if (budget_token <0 ) +return false; +blk_mq_set_rq_budget_token (rq, budget_token); +if (!blk_mq_get_driver_tag (rq)) { +blk_mq_put_dispatch_budget (rq->q, budget_token); +return false; + } +return true; +} +static blk_status_t __blk_mq_try_issue_directly (struct blk_mq_hw_ctx *hctx,struct request *rq,bool bypass_insert,bool last) {struct request_queue *q = rq->q;bool run_queue = true; -int budget_token;/* * RCU or SRCU read lock is needed before checking quiesced flag.
@@ -2647,16 +2661,8 @@ static blk_status_t __blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,if ((rq->rq_flags &RQF_ELV ) && !bypass_insert) goto insert; - budget_token =blk_mq_get_dispatch_budget (q); -if (budget_token <0 ) - goto insert; - -blk_mq_set_rq_budget_token (rq, budget_token); - -if (!blk_mq_get_driver_tag (rq)) { -blk_mq_put_dispatch_budget (q, budget_token); +if (!blk_mq_get_budget_and_tag (rq)) goto insert; - }return __blk_mq_issue_directly (hctx, rq, last); insert:
Signature · 1 line
2.39.2
Re: [PATCH 12/18] blk-mq: factor out a blk_mq_get_budget_and_tag helper
Quoted 5 lines · Christoph Hellwig
Factor out a helper from
__blk_mq_try_issue_directlyin preparation of folding that function into its two callers.Signed-off-by Christoph Hellwig <[email protected]>Reviewed-by Bart Van Assche <[email protected]>
Looks good.
PATCH 13/18 blk-mq: fold __blk_mq_try_issue_directly into its two callers
In reply to Christoph HellwigDue the wildly different behavior based on the bypass_insert argument, not a whole lot of code in __blk_mq_try_issue_directly is actually shared between blk_mq_try_issue_directly and blk_mq_request_issue_directly.
Remove __blk_mq_try_issue_directly and fold the code into the two callers instead.
--- block/blk-mq.c | 72 ++++++++++++++++++++++---------------------------- 1 file changed, 31 insertions(+), 41 deletions(-)
block/blk-mq.c +31 -41
@@ -2638,42 +2638,6 @@ static bool blk_mq_get_budget_and_tag(struct request *rq)return true; } -static blk_status_t __blk_mq_try_issue_directly (struct blk_mq_hw_ctx *hctx, -struct request *rq, -bool bypass_insert,bool last) -{ -struct request_queue *q = rq->q; -bool run_queue = true; - -/* -* RCU or SRCU read lock is needed before checking quiesced flag. -* -* When queue is stopped or quiesced, ignore 'bypass_insert' from -* blk_mq_request_issue_directly(), and return BLK_STS_OK to caller, -* and avoid driver to try to dispatch again. -*/ -if (blk_mq_hctx_stopped (hctx) ||blk_queue_quiesced (q)) { - run_queue = false; - bypass_insert = false; - goto insert; - } - -if ((rq->rq_flags &RQF_ELV ) && !bypass_insert) - goto insert; - -if (!blk_mq_get_budget_and_tag (rq)) - goto insert; - -return __blk_mq_issue_directly (hctx, rq, last); -insert: -if (bypass_insert) -return BLK_STS_RESOURCE ; - -blk_mq_insert_request (rq, false, run_queue, false); - -return BLK_STS_OK ; -} -/** * blk_mq_try_issue_directly - Try to send a request directly to device driver. * @hctx: Pointer of the associated hardware queue.
@@ -2687,18 +2651,44 @@ static blk_status_t __blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,static void blk_mq_try_issue_directly (struct blk_mq_hw_ctx *hctx,struct request *rq) { -blk_status_t ret = -__blk_mq_try_issue_directly (hctx, rq, false, true); +blk_status_t ret; + +if (blk_mq_hctx_stopped (hctx) ||blk_queue_quiesced (rq->q)) { +blk_mq_insert_request (rq, false, false, false); +return ; + } + +if ((rq->rq_flags &RQF_ELV ) || !blk_mq_get_budget_and_tag (rq)) { +blk_mq_insert_request (rq, false, true, false); +return ; + } -if (ret ==BLK_STS_RESOURCE || ret ==BLK_STS_DEV_RESOURCE ) + ret =__blk_mq_issue_directly (hctx, rq, true); +switch (ret) { +case BLK_STS_OK : +break ; +case BLK_STS_RESOURCE : +case BLK_STS_DEV_RESOURCE :blk_mq_request_bypass_insert (rq, false, true); -else if (ret !=BLK_STS_OK ) +break ; +default :blk_mq_end_request (rq, ret); +break ; + } }static blk_status_t blk_mq_request_issue_directly (struct request *rq,bool last) { -return __blk_mq_try_issue_directly (rq->mq_hctx, rq, true, last); +struct blk_mq_hw_ctx *hctx = rq->mq_hctx; + +if (blk_mq_hctx_stopped (hctx) ||blk_queue_quiesced (rq->q)) { +blk_mq_insert_request (rq, false, false, false); +return BLK_STS_OK ; + } + +if (!blk_mq_get_budget_and_tag (rq)) +return BLK_STS_RESOURCE ; +return __blk_mq_issue_directly (hctx, rq, last); }static void blk_mq_plug_issue_direct (struct blk_plug *plug)
Signature · 1 line
2.39.2
Re: [PATCH 13/18] blk-mq: fold __blk_mq_try_issue_directly into its two callers
Quoted 1 line · Christoph Hellwig
Due the wildly different behavior based on the
bypass_insertargument,
s/Due/Due to
Quoted 8 lines
not a whole lot of code in
__blk_mq_try_issue_directlyis actually shared betweenblk_mq_try_issue_directlyandblk_mq_request_issue_directly.Remove
__blk_mq_try_issue_directlyand fold the code into the two callers instead.Signed-off-by Christoph Hellwig <[email protected]>Reviewed-by Bart Van Assche <[email protected]>
Otherwise looks good to me.
PATCH 14/18 blk-mq: don't run the hw_queue from blk_mq_insert_request
In reply to Christoph Hellwigblk_mq_insert_request takes two bool parameters to control how to run the queue at the end of the function. Move the blk_mq_run_hw_queue call to the callers that want it instead.
--- block/blk-mq.c | 56 ++++++++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 24 deletions(-)
block/blk-mq.c +32 -24
@@ -44,8 +44,7 @@ static DEFINE_PER_CPU(struct llist_head, blk_cpu_done); -static void blk_mq_insert_request (struct request *rq,bool at_head, -bool run_queue,bool async); +static void blk_mq_insert_request (struct request *rq,bool at_head);static void blk_mq_try_issue_list_directly (struct blk_mq_hw_ctx *hctx,struct list_head *list);
@@ -1292,6 +1291,8 @@ static void blk_add_rq_to_plug(struct blk_plug *plug, struct request *rq)*/ void blk_execute_rq_nowait (struct request *rq,bool at_head) { +struct blk_mq_hw_ctx *hctx = rq->mq_hctx; +WARN_ON (irqs_disabled ());WARN_ON (!blk_rq_is_passthrough (rq));
@@ -1302,10 +1303,13 @@ void blk_execute_rq_nowait(struct request *rq, bool at_head)* device, directly accessing the plug instead of using blk_mq_plug() * should not have any consequences. */ -if (current->plug && !at_head) +if (current->plug && !at_head) {blk_add_rq_to_plug (current->plug, rq); -else -blk_mq_insert_request (rq, at_head, true, false); +return ; + } + +blk_mq_insert_request (rq, at_head); +blk_mq_run_hw_queue (hctx, false); }EXPORT_SYMBOL_GPL (blk_execute_rq_nowait);
@@ -1355,6 +1359,7 @@ static void blk_rq_poll_completion(struct request *rq, struct completion *wait)*/ blk_status_t blk_execute_rq (struct request *rq,bool at_head) { +struct blk_mq_hw_ctx *hctx = rq->mq_hctx;struct blk_rq_wait wait = { .done =COMPLETION_INITIALIZER_ONSTACK (wait.done), };
@@ -1366,7 +1371,8 @@ blk_status_t blk_execute_rq(struct request *rq, bool at_head) rq->end_io = blk_end_sync_rq;blk_account_io_start (rq); -blk_mq_insert_request (rq, at_head, true, false); +blk_mq_insert_request (rq, at_head); +blk_mq_run_hw_queue (hctx, false);if (blk_rq_is_poll (rq)) {blk_rq_poll_completion (rq, &wait.done);
@@ -1439,14 +1445,14 @@ static void blk_mq_requeue_work(struct work_struct *work) }else if (rq->rq_flags &RQF_SOFTBARRIER ) { rq->rq_flags &= ~RQF_SOFTBARRIER ;list_del_init (&rq->queuelist); -blk_mq_insert_request (rq, true, false, false); +blk_mq_insert_request (rq, true); } }while (!list_empty (&rq_list)) { rq = list_entry(rq_list.next, struct request, queuelist);list_del_init (&rq->queuelist); -blk_mq_insert_request (rq, false, false, false); +blk_mq_insert_request (rq, false); }blk_mq_run_hw_queues (q, false);
@@ -2506,8 +2512,7 @@ static void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx,blk_mq_run_hw_queue (hctx, run_queue_async); } -static void blk_mq_insert_request (struct request *rq,bool at_head, -bool run_queue,bool async) +static void blk_mq_insert_request (struct request *rq,bool at_head) {struct request_queue *q = rq->q;struct blk_mq_ctx *ctx = rq->mq_ctx;
@@ -2567,9 +2572,6 @@ static void blk_mq_insert_request(struct request *rq, bool at_head,blk_mq_hctx_mark_pending (hctx, ctx);spin_unlock (&ctx->lock); } - -if (run_queue) -blk_mq_run_hw_queue (hctx, async); } static void blk_mq_bio_to_request(struct request *rq, struct bio *bio,
@@ -2654,12 +2656,13 @@ static void blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,blk_status_t ret;if (blk_mq_hctx_stopped (hctx) ||blk_queue_quiesced (rq->q)) { -blk_mq_insert_request (rq, false, false, false); +blk_mq_insert_request (rq, false);return ; }if ((rq->rq_flags &RQF_ELV ) || !blk_mq_get_budget_and_tag (rq)) { -blk_mq_insert_request (rq, false, true, false); +blk_mq_insert_request (rq, false); +blk_mq_run_hw_queue (hctx, false);return ; }
@@ -2682,7 +2685,7 @@ static blk_status_t blk_mq_request_issue_directly(struct request *rq, bool last)struct blk_mq_hw_ctx *hctx = rq->mq_hctx;if (blk_mq_hctx_stopped (hctx) ||blk_queue_quiesced (rq->q)) { -blk_mq_insert_request (rq, false, false, false); +blk_mq_insert_request (rq, false);return BLK_STS_OK ; }
@@ -2962,6 +2965,7 @@ void blk_mq_submit_bio(struct bio *bio)struct request_queue *q =bdev_get_queue (bio->bi_bdev);struct blk_plug *plug =blk_mq_plug (bio);const int is_sync =op_is_sync (bio->bi_opf); +struct blk_mq_hw_ctx *hctx;struct request *rq;unsigned int nr_segs =1 ;blk_status_t ret;
@@ -3006,15 +3010,19 @@ void blk_mq_submit_bio(struct bio *bio)return ; } -if (plug) +if (plug) {blk_add_rq_to_plug (plug, rq); -else if ((rq->rq_flags &RQF_ELV ) || - (rq->mq_hctx->dispatch_busy && - (q->nr_hw_queues ==1 || !is_sync))) -blk_mq_insert_request (rq, false, true, true); -else -blk_mq_run_dispatch_ops (rq->q, -blk_mq_try_issue_directly (rq->mq_hctx, rq)); +return ; + } + + hctx = rq->mq_hctx; +if ((rq->rq_flags &RQF_ELV ) || + (hctx->dispatch_busy && (q->nr_hw_queues ==1 || !is_sync))) { +blk_mq_insert_request (rq, false); +blk_mq_run_hw_queue (hctx, true); + }else { +blk_mq_run_dispatch_ops (q,blk_mq_try_issue_directly (hctx, rq)); + } }#ifdef CONFIG_BLK_MQ_STACKING
Signature · 1 line
2.39.2
Re: [PATCH 14/18] blk-mq: don't run the hw_queue from blk_mq_insert_request
Quoted 6 lines · Christoph Hellwig
blk_mq_insert_requesttakes two bool parameters to control how to run the queue at the end of the function. Move theblk_mq_run_hw_queuecall to the callers that want it instead.Signed-off-by Christoph Hellwig <[email protected]>Reviewed-by Bart Van Assche <[email protected]>
This is nice !
PATCH 15/18 blk-mq: don't run the hw_queue from blk_mq_request_bypass_insert
In reply to Christoph Hellwigblk_mq_request_bypass_insert takes a bool parameter to control how to run the queue at the end of the function. Move the blk_mq_run_hw_queue call to the callers that want it instead.
--- block/blk-flush.c | 4 +++- block/blk-mq.c | 24 +++++++++++------------- block/blk-mq.h | 3 +-- 3 files changed, 15 insertions(+), 16 deletions(-)
block/blk-flush.c +3 -1
@@ -389,6 +389,7 @@ void blk_insert_flush(struct request *rq)unsigned long fflags = q->queue_flags;/* may change, cache */ unsigned int policy =blk_flush_policy (fflags, rq);struct blk_flush_queue *fq =blk_get_flush_queue (q, rq->mq_ctx); +struct blk_mq_hw_ctx *hctx = rq->mq_hctx;/* * @policy now records what operations need to be done. Adjust
@@ -425,7 +426,8 @@ void blk_insert_flush(struct request *rq)*/ if ((policy &REQ_FSEQ_DATA ) && !(policy & (REQ_FSEQ_PREFLUSH |REQ_FSEQ_POSTFLUSH ))) { -blk_mq_request_bypass_insert (rq, false, true); +blk_mq_request_bypass_insert (rq, false); +blk_mq_run_hw_queue (hctx, false);return ; }
block/blk-mq.c +11 -13
@@ -1441,7 +1441,7 @@ static void blk_mq_requeue_work(struct work_struct *work)if (rq->rq_flags &RQF_DONTPREP ) { rq->rq_flags &= ~RQF_SOFTBARRIER ;list_del_init (&rq->queuelist); -blk_mq_request_bypass_insert (rq, false, false); +blk_mq_request_bypass_insert (rq, false); }else if (rq->rq_flags &RQF_SOFTBARRIER ) { rq->rq_flags &= ~RQF_SOFTBARRIER ;list_del_init (&rq->queuelist);
@@ -2456,13 +2456,11 @@ static void blk_mq_run_work_fn(struct work_struct *work)* blk_mq_request_bypass_insert - Insert a request at dispatch list. * @rq: Pointer to request to be inserted. * @at_head: true if the request should be inserted at the head of the list. -* @run_queue: If we should run the hardware queue after inserting the request. * * Should only be used carefully, when the caller knows we want to * bypass a potential IO scheduler on the target device. */ -void blk_mq_request_bypass_insert (struct request *rq,bool at_head, -bool run_queue) +void blk_mq_request_bypass_insert (struct request *rq,bool at_head) {struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
@@ -2472,9 +2470,6 @@ void blk_mq_request_bypass_insert(struct request *rq, bool at_head,else list_add_tail (&rq->queuelist, &hctx->dispatch);spin_unlock (&hctx->lock); - -if (run_queue) -blk_mq_run_hw_queue (hctx, false); } static void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx,
@@ -2529,7 +2524,7 @@ static void blk_mq_insert_request(struct request *rq, bool at_head)* and it is added to the scheduler queue, there is no chance to * dispatch it given we prioritize requests in hctx->dispatch. */ -blk_mq_request_bypass_insert (rq, at_head, false); +blk_mq_request_bypass_insert (rq, at_head); }else if (rq->rq_flags &RQF_FLUSH_SEQ ) {/* * Firstly normal IO request is inserted to scheduler queue or
@@ -2552,7 +2547,7 @@ static void blk_mq_insert_request(struct request *rq, bool at_head)* Simply queue flush rq to the front of hctx->dispatch so that * intensive flush workloads can benefit in case of NCQ HW. */ -blk_mq_request_bypass_insert (rq, true, false); +blk_mq_request_bypass_insert (rq, true); }else if (q->elevator) {LIST_HEAD (list);
@@ -2672,7 +2667,8 @@ static void blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,break ;case BLK_STS_RESOURCE :case BLK_STS_DEV_RESOURCE : -blk_mq_request_bypass_insert (rq, false, true); +blk_mq_request_bypass_insert (rq, false); +blk_mq_run_hw_queue (hctx, false);break ;default :blk_mq_end_request (rq, ret);
@@ -2719,7 +2715,8 @@ static void blk_mq_plug_issue_direct(struct blk_plug *plug)break ;case BLK_STS_RESOURCE :case BLK_STS_DEV_RESOURCE : -blk_mq_request_bypass_insert (rq, false, true); +blk_mq_request_bypass_insert (rq, false); +blk_mq_run_hw_queue (hctx, false); goto out;default :blk_mq_end_request (rq, ret);
@@ -2837,8 +2834,9 @@ static void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx,break ;case BLK_STS_RESOURCE :case BLK_STS_DEV_RESOURCE : -blk_mq_request_bypass_insert (rq, false, -list_empty (list)); +blk_mq_request_bypass_insert (rq, false); +if (list_empty (list)) +blk_mq_run_hw_queue (hctx, false); goto out;default :blk_mq_end_request (rq, ret);
block/blk-mq.h +1 -2
@@ -65,8 +65,7 @@ void blk_mq_free_map_and_rqs(struct blk_mq_tag_set *set,/* * Internal helpers for request insertion into sw queues */ -void blk_mq_request_bypass_insert (struct request *rq,bool at_head, -bool run_queue); +void blk_mq_request_bypass_insert (struct request *rq,bool at_head);/* * CPU -> queue mappings
Signature · 1 line
2.39.2
Re: [PATCH 15/18] blk-mq: don't run the hw_queue from blk_mq_request_bypass_insert
Quoted 5 lines · Christoph Hellwig
blk_mq_request_bypass_inserttakes a bool parameter to control how to run the queue at the end of the function. Move theblk_mq_run_hw_queuecall to the callers that want it instead.Signed-off-by Christoph Hellwig <[email protected]>
PATCH 16/18 blk-mq: pass a flags argument to blk_mq_insert_request
In reply to Christoph HellwigReplace the at_head bool with a flags argument that so far only contains a single BLK_MQ_INSERT_AT_HEAD value. This makes it much easier to grep for head insertions into the blk-mq dispatch queues.
--- block/blk-mq.c | 19 ++++++++++--------- block/blk-mq.h | 3 +++ 2 files changed, 13 insertions(+), 9 deletions(-)
block/blk-mq.c +10 -9
@@ -44,7 +44,7 @@ static DEFINE_PER_CPU(struct llist_head, blk_cpu_done); -static void blk_mq_insert_request (struct request *rq,bool at_head); +static void blk_mq_insert_request (struct request *rq,blk_insert_t flags);static void blk_mq_try_issue_list_directly (struct blk_mq_hw_ctx *hctx,struct list_head *list);
@@ -1308,7 +1308,7 @@ void blk_execute_rq_nowait(struct request *rq, bool at_head)return ; } -blk_mq_insert_request (rq, at_head); +blk_mq_insert_request (rq, at_head ?BLK_MQ_INSERT_AT_HEAD :0 );blk_mq_run_hw_queue (hctx, false); }EXPORT_SYMBOL_GPL (blk_execute_rq_nowait);
@@ -1371,7 +1371,7 @@ blk_status_t blk_execute_rq(struct request *rq, bool at_head) rq->end_io = blk_end_sync_rq;blk_account_io_start (rq); -blk_mq_insert_request (rq, at_head); +blk_mq_insert_request (rq, at_head ?BLK_MQ_INSERT_AT_HEAD :0 );blk_mq_run_hw_queue (hctx, false);if (blk_rq_is_poll (rq)) {
@@ -1445,14 +1445,14 @@ static void blk_mq_requeue_work(struct work_struct *work) }else if (rq->rq_flags &RQF_SOFTBARRIER ) { rq->rq_flags &= ~RQF_SOFTBARRIER ;list_del_init (&rq->queuelist); -blk_mq_insert_request (rq, true); +blk_mq_insert_request (rq,BLK_MQ_INSERT_AT_HEAD ); } }while (!list_empty (&rq_list)) { rq = list_entry(rq_list.next, struct request, queuelist);list_del_init (&rq->queuelist); -blk_mq_insert_request (rq, false); +blk_mq_insert_request (rq,0 ); }blk_mq_run_hw_queues (q, false);
@@ -2507,7 +2507,7 @@ static void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx,blk_mq_run_hw_queue (hctx, run_queue_async); } -static void blk_mq_insert_request (struct request *rq,bool at_head) +static void blk_mq_insert_request (struct request *rq,blk_insert_t flags) {struct request_queue *q = rq->q;struct blk_mq_ctx *ctx = rq->mq_ctx;
@@ -2524,7 +2524,7 @@ static void blk_mq_insert_request(struct request *rq, bool at_head)* and it is added to the scheduler queue, there is no chance to * dispatch it given we prioritize requests in hctx->dispatch. */ -blk_mq_request_bypass_insert (rq, at_head); +blk_mq_request_bypass_insert (rq, flags &BLK_MQ_INSERT_AT_HEAD ); }else if (rq->rq_flags &RQF_FLUSH_SEQ ) {/* * Firstly normal IO request is inserted to scheduler queue or
@@ -2554,12 +2554,13 @@ static void blk_mq_insert_request(struct request *rq, bool at_head)WARN_ON_ONCE (rq->tag !=BLK_MQ_NO_TAG );list_add (&rq->queuelist, &list); - q->elevator->type->ops.insert_requests (hctx, &list, at_head); + q->elevator->type->ops.insert_requests (hctx, &list, + flags &BLK_MQ_INSERT_AT_HEAD ); }else {trace_block_rq_insert (rq);spin_lock (&ctx->lock); -if (at_head) +if (flags &BLK_MQ_INSERT_AT_HEAD )list_add (&rq->queuelist, &ctx->rq_lists[hctx->type]);else list_add_tail(&rq->queuelist,
block/blk-mq.h +3 -0
@@ -36,6 +36,9 @@ enum { BLK_MQ_TAG_MAX = BLK_MQ_NO_TAG - 1, }; +typedef unsigned int __bitwise blk_insert_t ; +#define BLK_MQ_INSERT_AT_HEAD ((__force blk_insert_t)0x01) +void blk_mq_submit_bio (struct bio *bio);int blk_mq_poll (struct request_queue *q,blk_qc_t cookie,struct io_comp_batch *iob,unsigned int flags);
Signature · 1 line
2.39.2
Re: [PATCH 16/18] blk-mq: pass a flags argument to blk_mq_insert_request
Quoted 5 lines · Christoph Hellwig
Replace the
at_headbool with a flags argument that so far only contains a singleBLK_MQ_INSERT_AT_HEADvalue. This makes it much easier to grep for head insertions into the blk-mq dispatch queues.Signed-off-by Christoph Hellwig <[email protected]>
PATCH 17/18 blk-mq: pass a flags argument to blk_mq_request_bypass_insert
In reply to Christoph HellwigReplace the boolean at_head argument with the same flags that are already passed to blk_mq_insert_request.
--- block/blk-flush.c | 2 +- block/blk-mq.c | 18 +++++++++--------- block/blk-mq.h | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-)
block/blk-flush.c +1 -1
@@ -426,7 +426,7 @@ void blk_insert_flush(struct request *rq)*/ if ((policy &REQ_FSEQ_DATA ) && !(policy & (REQ_FSEQ_PREFLUSH |REQ_FSEQ_POSTFLUSH ))) { -blk_mq_request_bypass_insert (rq, false); +blk_mq_request_bypass_insert (rq,0 );blk_mq_run_hw_queue (hctx, false);return ; }
block/blk-mq.c +9 -9
@@ -1441,7 +1441,7 @@ static void blk_mq_requeue_work(struct work_struct *work)if (rq->rq_flags &RQF_DONTPREP ) { rq->rq_flags &= ~RQF_SOFTBARRIER ;list_del_init (&rq->queuelist); -blk_mq_request_bypass_insert (rq, false); +blk_mq_request_bypass_insert (rq,0 ); }else if (rq->rq_flags &RQF_SOFTBARRIER ) { rq->rq_flags &= ~RQF_SOFTBARRIER ;list_del_init (&rq->queuelist);
@@ -2455,17 +2455,17 @@ static void blk_mq_run_work_fn(struct work_struct *work)/** * blk_mq_request_bypass_insert - Insert a request at dispatch list. * @rq: Pointer to request to be inserted. -* @at_head: true if the request should be inserted at the head of the list. +* @flags: BLK_MQ_INSERT_* * * Should only be used carefully, when the caller knows we want to * bypass a potential IO scheduler on the target device. */ -void blk_mq_request_bypass_insert (struct request *rq,bool at_head) +void blk_mq_request_bypass_insert (struct request *rq,blk_insert_t flags) {struct blk_mq_hw_ctx *hctx = rq->mq_hctx;spin_lock (&hctx->lock); -if (at_head) +if (flags &BLK_MQ_INSERT_AT_HEAD )list_add (&rq->queuelist, &hctx->dispatch);else list_add_tail (&rq->queuelist, &hctx->dispatch);
@@ -2524,7 +2524,7 @@ static void blk_mq_insert_request(struct request *rq, blk_insert_t flags)* and it is added to the scheduler queue, there is no chance to * dispatch it given we prioritize requests in hctx->dispatch. */ -blk_mq_request_bypass_insert (rq, flags &BLK_MQ_INSERT_AT_HEAD ); +blk_mq_request_bypass_insert (rq, flags); }else if (rq->rq_flags &RQF_FLUSH_SEQ ) {/* * Firstly normal IO request is inserted to scheduler queue or
@@ -2547,7 +2547,7 @@ static void blk_mq_insert_request(struct request *rq, blk_insert_t flags)* Simply queue flush rq to the front of hctx->dispatch so that * intensive flush workloads can benefit in case of NCQ HW. */ -blk_mq_request_bypass_insert (rq, true); +blk_mq_request_bypass_insert (rq,BLK_MQ_INSERT_AT_HEAD ); }else if (q->elevator) {LIST_HEAD (list);
@@ -2668,7 +2668,7 @@ static void blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,break ;case BLK_STS_RESOURCE :case BLK_STS_DEV_RESOURCE : -blk_mq_request_bypass_insert (rq, false); +blk_mq_request_bypass_insert (rq,0 );blk_mq_run_hw_queue (hctx, false);break ;default :
@@ -2716,7 +2716,7 @@ static void blk_mq_plug_issue_direct(struct blk_plug *plug)break ;case BLK_STS_RESOURCE :case BLK_STS_DEV_RESOURCE : -blk_mq_request_bypass_insert (rq, false); +blk_mq_request_bypass_insert (rq,0 );blk_mq_run_hw_queue (hctx, false); goto out;default :
@@ -2835,7 +2835,7 @@ static void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx,break ;case BLK_STS_RESOURCE :case BLK_STS_DEV_RESOURCE : -blk_mq_request_bypass_insert (rq, false); +blk_mq_request_bypass_insert (rq,0 );if (list_empty (list))blk_mq_run_hw_queue (hctx, false); goto out;
block/blk-mq.h +1 -1
@@ -68,7 +68,7 @@ void blk_mq_free_map_and_rqs(struct blk_mq_tag_set *set,/* * Internal helpers for request insertion into sw queues */ -void blk_mq_request_bypass_insert (struct request *rq,bool at_head); +void blk_mq_request_bypass_insert (struct request *rq,blk_insert_t flags);/* * CPU -> queue mappings
Signature · 1 line
2.39.2
Re: [PATCH 17/18] blk-mq: pass a flags argument to blk_mq_request_bypass_insert
Quoted 5 lines · Christoph Hellwig
Replace the boolean
at_headargument with the same flags that are already passed toblk_mq_insert_request.Signed-off-by Christoph Hellwig <[email protected]>Reviewed-by Bart Van Assche <[email protected]>
Nice cleanup.
PATCH 18/18 blk-mq: pass the flags argument to elevator_type->insert_requests
In reply to Christoph HellwigInstead of passing a bool at_head, pass down the full flags from the blk_mq_insert_request interface.
--- block/bfq-iosched.c | 16 ++++++++-------- block/blk-mq.c | 5 ++--- block/elevator.h | 4 +++- block/kyber-iosched.c | 5 +++-- block/mq-deadline.c | 9 +++++---- 5 files changed, 21 insertions(+), 18 deletions(-)
block/bfq-iosched.c +8 -8
@@ -6231,7 +6231,7 @@ static inline void bfq_update_insert_stats(struct request_queue *q,static struct bfq_queue *bfq_init_rq (struct request *rq);static void bfq_insert_request (struct blk_mq_hw_ctx *hctx,struct request *rq, -bool at_head) +blk_insert_t flags) {struct request_queue *q = hctx->queue;struct bfq_data *bfqd = q->elevator->elevator_data;
@@ -6254,11 +6254,10 @@ static void bfq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,trace_block_rq_insert (rq); -if (!bfqq || at_head) { -if (at_head) -list_add (&rq->queuelist, &bfqd->dispatch); -else -list_add_tail (&rq->queuelist, &bfqd->dispatch); +if (flags &BLK_MQ_INSERT_AT_HEAD ) { +list_add (&rq->queuelist, &bfqd->dispatch); + }else if (!bfqq) { +list_add_tail (&rq->queuelist, &bfqd->dispatch); }else { idle_timer_disabled =__bfq_insert_request (bfqd, rq);/*
@@ -6288,14 +6287,15 @@ static void bfq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, }static void bfq_insert_requests (struct blk_mq_hw_ctx *hctx, -struct list_head *list,bool at_head) +struct list_head *list, +blk_insert_t flags) {while (!list_empty (list)) {struct request *rq; rq = list_first_entry(list, struct request, queuelist);list_del_init (&rq->queuelist); -bfq_insert_request (hctx, rq, at_head); +bfq_insert_request (hctx, rq, flags); } }
block/blk-mq.c +2 -3
@@ -2554,8 +2554,7 @@ static void blk_mq_insert_request(struct request *rq, blk_insert_t flags)WARN_ON_ONCE (rq->tag !=BLK_MQ_NO_TAG );list_add (&rq->queuelist, &list); - q->elevator->type->ops.insert_requests (hctx, &list, - flags &BLK_MQ_INSERT_AT_HEAD ); + q->elevator->type->ops.insert_requests (hctx, &list, flags); }else {trace_block_rq_insert (rq);
@@ -2766,7 +2765,7 @@ static void blk_mq_dispatch_plug_list(struct blk_plug *plug, bool from_sched)percpu_ref_get (&this_hctx->queue->q_usage_counter);if (this_hctx->queue->elevator) { this_hctx->queue->elevator->type->ops.insert_requests (this_hctx, - &list, false); + &list,0 );blk_mq_run_hw_queue (this_hctx, from_sched); }else {blk_mq_insert_requests (this_hctx, this_ctx, &list, from_sched);
block/elevator.h +3 -1
@@ -4,6 +4,7 @@#include <linux/percpu.h> #include <linux/hashtable.h> +#include "blk-mq.h" struct io_cq ;struct elevator_type ;
@@ -37,7 +38,8 @@ struct elevator_mq_ops {void (*limit_depth)(blk_opf_t ,struct blk_mq_alloc_data *);void (*prepare_request)(struct request *);void (*finish_request)(struct request *); -void (*insert_requests)(struct blk_mq_hw_ctx *,struct list_head *,bool ); +void (*insert_requests)(struct blk_mq_hw_ctx *hctx,struct list_head *list, +blk_insert_t flags);struct request *(*dispatch_request)(struct blk_mq_hw_ctx *);bool (*has_work)(struct blk_mq_hw_ctx *);void (*completed_request)(struct request *,u64 );
block/kyber-iosched.c +3 -2
@@ -588,7 +588,8 @@ static void kyber_prepare_request(struct request *rq) }static void kyber_insert_requests (struct blk_mq_hw_ctx *hctx, -struct list_head *rq_list,bool at_head) +struct list_head *rq_list, +blk_insert_t flags) {struct kyber_hctx_data *khd = hctx->sched_data;struct request *rq, *next;
@@ -600,7 +601,7 @@ static void kyber_insert_requests(struct blk_mq_hw_ctx *hctx,spin_lock (&kcq->lock);trace_block_rq_insert (rq); -if (at_head) +if (flags &BLK_MQ_INSERT_AT_HEAD )list_move (&rq->queuelist, head);else list_move_tail (&rq->queuelist, head);
block/mq-deadline.c +5 -4
@@ -766,7 +766,7 @@ static bool dd_bio_merge(struct request_queue *q, struct bio *bio,* add rq to rbtree and fifo */ static void dd_insert_request (struct blk_mq_hw_ctx *hctx,struct request *rq, -bool at_head) +blk_insert_t flags) {struct request_queue *q = hctx->queue;struct deadline_data *dd = q->elevator->elevator_data;
@@ -799,7 +799,7 @@ static void dd_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,trace_block_rq_insert (rq); -if (at_head) { +if (flags &BLK_MQ_INSERT_AT_HEAD ) {list_add (&rq->queuelist, &per_prio->dispatch); rq->fifo_time = jiffies; }else {
@@ -823,7 +823,8 @@ static void dd_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,* Called from blk_mq_insert_request() or blk_mq_dispatch_plug_list(). */ static void dd_insert_requests (struct blk_mq_hw_ctx *hctx, -struct list_head *list,bool at_head) +struct list_head *list, +blk_insert_t flags) {struct request_queue *q = hctx->queue;struct deadline_data *dd = q->elevator->elevator_data;
@@ -834,7 +835,7 @@ static void dd_insert_requests(struct blk_mq_hw_ctx *hctx, rq = list_first_entry(list, struct request, queuelist);list_del_init (&rq->queuelist); -dd_insert_request (hctx, rq, at_head); +dd_insert_request (hctx, rq, flags); }spin_unlock (&dd->lock); }
Signature · 1 line
2.39.2
Re: [PATCH 18/18] blk-mq: pass the flags argument to elevator_type->insert_requests
Quoted 5 lines · Christoph Hellwig
Instead of passing a bool
at_head, pass down the full flags from theblk_mq_insert_requestinterface.Signed-off-by Christoph Hellwig <[email protected]>Reviewed-by Bart Van Assche <[email protected]>
Looks good.