blk-mq: driver tag related cleanup
v2 cover blk-mq: driver tag related cleanup
Hi Jens,
The 1st & 2nd patch moves get/put driver tag helpers into blk-mq.c, and the 3rd patch centralise related handling into blk_mq_get_driver_tag, so both flush & blk-mq code get simplified.
V2:
- add reviewed-by tag
- don't put blk_mq_tag_busy() into blk_mq_get_driver_tag
- use BLK_MQ_NO_TAG in blk-flush.c
Ming Lei (3):
blk-mq: move blk_mq_get_driver_tag into blk-mq.c
blk-mq: move blk_mq_put_driver_tag() into blk-mq.c
blk-mq: centralise related handling into blk_mq_get_driver_tag
block/blk-flush.c | 17 +++++------- block/blk-mq-tag.c | 58 ---------------------------------------- block/blk-mq-tag.h | 41 +++++++++++++++++----------- block/blk-mq.c | 66 +++++++++++++++++++++++++++++++++++++++++----- block/blk-mq.h | 20 -------------- block/blk.h | 5 ---- 6 files changed, 91 insertions(+), 116 deletions(-)
Signature · 1 line
2.25.2
PATCH v2 1/3 blk-mq: move blk_mq_get_driver_tag into blk-mq.c
blk_mq_get_driver_tag() is only used by blk-mq.c and is supposed to stay in blk-mq.c, so move it and preparing for cleanup code of get/put driver tag.
Meantime hctx_may_queue() is moved to header file and it is fine since it is defined as inline always.
No functional change.
--- block/blk-mq-tag.c | 58 ---------------------------------------------- block/blk-mq-tag.h | 39 ++++++++++++++++++++++++------- block/blk-mq.c | 34 +++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 66 deletions(-)
block/blk-mq-tag.c +0 -58
@@ -56,37 +56,6 @@ void __blk_mq_tag_idle(struct blk_mq_hw_ctx *hctx)blk_mq_tag_wakeup_all (tags, false); } -/* -* For shared tag users, we track the number of currently active users -* and attempt to provide a fair share of the tag depth for each of them. -*/ -static inline bool hctx_may_queue (struct blk_mq_hw_ctx *hctx, -struct sbitmap_queue *bt) -{ -unsigned int depth, users; - -if (!hctx || !(hctx->flags &BLK_MQ_F_TAG_SHARED )) -return true; -if (!test_bit (BLK_MQ_S_TAG_ACTIVE , &hctx->state)) -return true; - -/* -* Don't try dividing an ant -*/ -if (bt->sb.depth ==1 ) -return true; - - users =atomic_read (&hctx->tags->active_queues); -if (!users) -return true; - -/* -* Allow at least some tags -*/ - depth =max ((bt->sb.depth + users -1 ) / users,4U ); -return atomic_read (&hctx->nr_active) < depth; -} -static int __blk_mq_get_tag (struct blk_mq_alloc_data *data,struct sbitmap_queue *bt) {
@@ -191,33 +160,6 @@ unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data)return tag + tag_offset; } -bool __blk_mq_get_driver_tag (struct request *rq) -{ -struct sbitmap_queue *bt = &rq->mq_hctx->tags->bitmap_tags; -unsigned int tag_offset = rq->mq_hctx->tags->nr_reserved_tags; -bool shared =blk_mq_tag_busy (rq->mq_hctx); -int tag; - -if (blk_mq_tag_is_reserved (rq->mq_hctx->sched_tags, rq->internal_tag)) { - bt = &rq->mq_hctx->tags->breserved_tags; - tag_offset =0 ; - } - -if (!hctx_may_queue (rq->mq_hctx, bt)) -return false; - tag =__sbitmap_queue_get (bt); -if (tag ==BLK_MQ_NO_TAG ) -return false; - - rq->tag = tag + tag_offset; -if (shared) { - rq->rq_flags |=RQF_MQ_INFLIGHT ; -atomic_inc (&rq->mq_hctx->nr_active); - } - rq->mq_hctx->tags->rqs[rq->tag] = rq; -return true; -} -void blk_mq_put_tag (struct blk_mq_tags *tags,struct blk_mq_ctx *ctx,unsigned int tag) {
block/blk-mq-tag.h +31 -8
@@ -51,14 +51,6 @@ enum { BLK_MQ_TAG_MAX = BLK_MQ_NO_TAG - 1, }; -bool __blk_mq_get_driver_tag (struct request *rq); -static inline bool blk_mq_get_driver_tag (struct request *rq) -{ -if (rq->tag !=BLK_MQ_NO_TAG ) -return true; -return __blk_mq_get_driver_tag (rq); -} -extern bool __blk_mq_tag_busy (struct blk_mq_hw_ctx *);extern void __blk_mq_tag_idle (struct blk_mq_hw_ctx *);
@@ -78,6 +70,37 @@ static inline void blk_mq_tag_idle(struct blk_mq_hw_ctx *hctx)__blk_mq_tag_idle (hctx); } +/* +* For shared tag users, we track the number of currently active users +* and attempt to provide a fair share of the tag depth for each of them. +*/ +static inline bool hctx_may_queue (struct blk_mq_hw_ctx *hctx, +struct sbitmap_queue *bt) +{ +unsigned int depth, users; + +if (!hctx || !(hctx->flags &BLK_MQ_F_TAG_SHARED )) +return true; +if (!test_bit (BLK_MQ_S_TAG_ACTIVE , &hctx->state)) +return true; + +/* +* Don't try dividing an ant +*/ +if (bt->sb.depth ==1 ) +return true; + + users =atomic_read (&hctx->tags->active_queues); +if (!users) +return true; + +/* +* Allow at least some tags +*/ + depth =max ((bt->sb.depth + users -1 ) / users,4U ); +return atomic_read (&hctx->nr_active) < depth; +} +/* * This helper should only be used for flush request to share tag * with the request cloned from, and both the two requests can't be
block/blk-mq.c +34 -0
@@ -1107,6 +1107,40 @@ static inline unsigned int queued_to_index(unsigned int queued)return min (BLK_MQ_MAX_DISPATCH_ORDER -1 ,ilog2 (queued) +1 ); } +static bool __blk_mq_get_driver_tag (struct request *rq) +{ +struct sbitmap_queue *bt = &rq->mq_hctx->tags->bitmap_tags; +unsigned int tag_offset = rq->mq_hctx->tags->nr_reserved_tags; +bool shared =blk_mq_tag_busy (rq->mq_hctx); +int tag; + +if (blk_mq_tag_is_reserved (rq->mq_hctx->sched_tags, rq->internal_tag)) { + bt = &rq->mq_hctx->tags->breserved_tags; + tag_offset =0 ; + } + +if (!hctx_may_queue (rq->mq_hctx, bt)) +return false; + tag =__sbitmap_queue_get (bt); +if (tag ==BLK_MQ_NO_TAG ) +return false; + + rq->tag = tag + tag_offset; +if (shared) { + rq->rq_flags |=RQF_MQ_INFLIGHT ; +atomic_inc (&rq->mq_hctx->nr_active); + } + rq->mq_hctx->tags->rqs[rq->tag] = rq; +return true; +} + +static bool blk_mq_get_driver_tag (struct request *rq) +{ +if (rq->tag !=BLK_MQ_NO_TAG ) +return true; +return __blk_mq_get_driver_tag (rq); +} +static int blk_mq_dispatch_wake (wait_queue_entry_t *wait,unsigned mode,int flags,void *key) {
Signature · 1 line
2.25.2
PATCH v2 2/3 blk-mq: move blk_mq_put_driver_tag() into blk-mq.c
In reply to Ming LeiIt is used by blk-mq.c only, so move it to the source file.
--- block/blk-mq.c | 20 ++++++++++++++++++++ block/blk-mq.h | 20 -------------------- 2 files changed, 20 insertions(+), 20 deletions(-)
block/blk-mq.c +20 -0
@@ -670,6 +670,26 @@ static inline bool blk_mq_complete_need_ipi(struct request *rq)return cpu_online (rq->mq_ctx->cpu); } +static void __blk_mq_put_driver_tag (struct blk_mq_hw_ctx *hctx, +struct request *rq) +{ +blk_mq_put_tag (hctx->tags, rq->mq_ctx, rq->tag); + rq->tag =BLK_MQ_NO_TAG ; + +if (rq->rq_flags &RQF_MQ_INFLIGHT ) { + rq->rq_flags &= ~RQF_MQ_INFLIGHT ; +atomic_dec (&hctx->nr_active); + } +} + +static inline void blk_mq_put_driver_tag (struct request *rq) +{ +if (rq->tag ==BLK_MQ_NO_TAG || rq->internal_tag ==BLK_MQ_NO_TAG ) +return ; + +__blk_mq_put_driver_tag (rq->mq_hctx, rq); +} +bool blk_mq_complete_request_remote (struct request *rq) {WRITE_ONCE (rq->state,MQ_RQ_COMPLETE );
block/blk-mq.h +0 -20
@@ -196,26 +196,6 @@ static inline bool blk_mq_get_dispatch_budget(struct blk_mq_hw_ctx *hctx)return true; } -static inline void __blk_mq_put_driver_tag (struct blk_mq_hw_ctx *hctx, -struct request *rq) -{ -blk_mq_put_tag (hctx->tags, rq->mq_ctx, rq->tag); - rq->tag =BLK_MQ_NO_TAG ; - -if (rq->rq_flags &RQF_MQ_INFLIGHT ) { - rq->rq_flags &= ~RQF_MQ_INFLIGHT ; -atomic_dec (&hctx->nr_active); - } -} - -static inline void blk_mq_put_driver_tag (struct request *rq) -{ -if (rq->tag ==BLK_MQ_NO_TAG || rq->internal_tag ==BLK_MQ_NO_TAG ) -return ; - -__blk_mq_put_driver_tag (rq->mq_hctx, rq); -} -static inline void blk_mq_clear_mq_map (struct blk_mq_queue_map *qmap) {int cpu;
Signature · 1 line
2.25.2
PATCH v2 3/3 blk-mq: centralise related handling into blk_mq_get_driver_tag
In reply to Ming LeiMove .nr_active update and request assignment into blk_mq_get_driver_tag(), all are good to do during getting driver tag.
Meantime blk-flush related code is simplified and flush request needn't to update the request table manually any more.
--- block/blk-flush.c | 17 ++++++----------- block/blk-mq-tag.h | 12 ------------ block/blk-mq.c | 30 ++++++++++++++---------------- block/blk.h | 5 ----- 4 files changed, 20 insertions(+), 44 deletions(-)
block/blk-flush.c +6 -11
@@ -236,12 +236,10 @@ static void flush_end_io(struct request *flush_rq, blk_status_t error) error = fq->rq_status; hctx = flush_rq->mq_hctx; -if (!q->elevator) { -blk_mq_tag_set_rq (hctx, flush_rq->tag, fq->orig_rq); - flush_rq->tag =-1 ; - }else { - flush_rq->internal_tag =-1 ; - } +if (!q->elevator) + flush_rq->tag =BLK_MQ_NO_TAG ; +else + flush_rq->internal_tag =BLK_MQ_NO_TAG ; running = &fq->flush_queue[fq->flush_running_idx];BUG_ON (fq->flush_pending_idx == fq->flush_running_idx);
@@ -315,13 +313,10 @@ static void blk_kick_flush(struct request_queue *q, struct blk_flush_queue *fq, flush_rq->mq_ctx = first_rq->mq_ctx; flush_rq->mq_hctx = first_rq->mq_hctx; -if (!q->elevator) { - fq->orig_rq = first_rq; +if (!q->elevator) flush_rq->tag = first_rq->tag; -blk_mq_tag_set_rq (flush_rq->mq_hctx, first_rq->tag, flush_rq); - }else { +else flush_rq->internal_tag = first_rq->internal_tag; - } flush_rq->cmd_flags =REQ_OP_FLUSH |REQ_PREFLUSH ; flush_rq->cmd_flags |= (flags &REQ_DRV ) | (flags &REQ_FAILFAST_MASK );
block/blk-mq-tag.h +0 -12
@@ -101,18 +101,6 @@ static inline bool hctx_may_queue(struct blk_mq_hw_ctx *hctx,return atomic_read (&hctx->nr_active) < depth; } -/* -* This helper should only be used for flush request to share tag -* with the request cloned from, and both the two requests can't be -* in flight at the same time. The caller has to make sure the tag -* can't be freed. -*/ -static inline void blk_mq_tag_set_rq (struct blk_mq_hw_ctx *hctx, -unsigned int tag,struct request *rq) -{ - hctx->tags->rqs[tag] = rq; -} -static inline bool blk_mq_tag_is_reserved (struct blk_mq_tags *tags,unsigned int tag) {
block/blk-mq.c +14 -16
@@ -277,26 +277,20 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data, {struct blk_mq_tags *tags =blk_mq_tags_from_data (data);struct request *rq = tags->static_rqs[tag]; -req_flags_t rq_flags =0 ;if (data->flags &BLK_MQ_REQ_INTERNAL ) { rq->tag =BLK_MQ_NO_TAG ; rq->internal_tag = tag; }else { -if (data->hctx->flags &BLK_MQ_F_TAG_SHARED ) { - rq_flags =RQF_MQ_INFLIGHT ; -atomic_inc (&data->hctx->nr_active); - } rq->tag = tag; rq->internal_tag =BLK_MQ_NO_TAG ; - data->hctx->tags->rqs[rq->tag] = rq; }/* csd/requeue_work/fifo_time is initialized before use */ rq->q = data->q; rq->mq_ctx = data->ctx; rq->mq_hctx = data->hctx; - rq->rq_flags = rq_flags; + rq->rq_flags =0 ; rq->cmd_flags = data->cmd_flags;if (data->flags &BLK_MQ_REQ_PREEMPT ) rq->rq_flags |=RQF_PREEMPT ;
@@ -1131,9 +1125,10 @@ static bool __blk_mq_get_driver_tag(struct request *rq) {struct sbitmap_queue *bt = &rq->mq_hctx->tags->bitmap_tags;unsigned int tag_offset = rq->mq_hctx->tags->nr_reserved_tags; -bool shared =blk_mq_tag_busy (rq->mq_hctx);int tag; +blk_mq_tag_busy (rq->mq_hctx); +if (blk_mq_tag_is_reserved (rq->mq_hctx->sched_tags, rq->internal_tag)) { bt = &rq->mq_hctx->tags->breserved_tags; tag_offset =0 ;
@@ -1146,19 +1141,22 @@ static bool __blk_mq_get_driver_tag(struct request *rq)return false; rq->tag = tag + tag_offset; -if (shared) { - rq->rq_flags |=RQF_MQ_INFLIGHT ; -atomic_inc (&rq->mq_hctx->nr_active); - } - rq->mq_hctx->tags->rqs[rq->tag] = rq;return true; }static bool blk_mq_get_driver_tag (struct request *rq) { -if (rq->tag !=BLK_MQ_NO_TAG ) -return true; -return __blk_mq_get_driver_tag (rq); +struct blk_mq_hw_ctx *hctx = rq->mq_hctx; + +if (rq->tag ==BLK_MQ_NO_TAG && !__blk_mq_get_driver_tag (rq)) +return false; + +if (hctx->flags &BLK_MQ_F_TAG_SHARED ) { + rq->rq_flags |=RQF_MQ_INFLIGHT ; +atomic_inc (&hctx->nr_active); + } + hctx->tags->rqs[rq->tag] = rq; +return true; } static int blk_mq_dispatch_wake(wait_queue_entry_t *wait, unsigned mode,
block/blk.h +0 -5
@@ -25,11 +25,6 @@ struct blk_flush_queue {struct list_head flush_data_in_flight;struct request *flush_rq; -/* -* flush_rq shares tag with this rq, both can't be active -* at the same time -*/ -struct request *orig_rq;struct lock_class_key key;spinlock_t mq_flush_lock; };
Signature · 1 line
2.25.2
Re: [PATCH V2 3/3] blk-mq: centralise related handling into blk_mq_get_driver_tag
Quoted 5 lines · Ming Lei
Move .nr_active update and request assignment into
blk_mq_get_driver_tag(), all are good to do during getting driver tag.Meantime blk-flush related code is simplified and flush request needn't to update the request table manually any more.
As mentioned last time: BLK_MQ_REQ_INTERNAL is gone in the latest block tree, so this won't apply.