blk-mq: fix schedule from atomic context
cover blk-mq: fix schedule from atomic context
Hi Jens,
These two patches fix the problem of schedule from atomic context which is introduced in bitmap based allocation.
Thanks,
--
Ming Lei
PATCH 1/2 blk-mq: move blk_mq_get_ctx/blk_mq_put_ctx to mq private header
The blk-mq tag code need these helpers.
--- block/blk-mq.c | 22 ---------------------- block/blk-mq.h | 22 ++++++++++++++++++++++ 2 files changed, 22 insertions(+), 22 deletions(-)
block/blk-mq.c +0 -22
@@ -33,28 +33,6 @@ static LIST_HEAD(all_q_list);static void __blk_mq_run_hw_queue (struct blk_mq_hw_ctx *hctx); -static struct blk_mq_ctx *__blk_mq_get_ctx (struct request_queue *q, -unsigned int cpu) -{ -return per_cpu_ptr (q->queue_ctx, cpu); -} - -/* -* This assumes per-cpu software queueing queues. They could be per-node -* as well, for instance. For now this is hardcoded as-is. Note that we don't -* care about preemption, since we know the ctx's are persistent. This does -* mean that we can't rely on ctx always matching the currently running CPU. -*/ -static struct blk_mq_ctx *blk_mq_get_ctx (struct request_queue *q) -{ -return __blk_mq_get_ctx (q,get_cpu ()); -} - -static void blk_mq_put_ctx (struct blk_mq_ctx *ctx) -{ -put_cpu (); -} -/* * Check if any of the ctx's have pending work in this hardware queue */
block/blk-mq.h +22 -0
@@ -69,4 +69,26 @@ struct blk_align_bitmap { unsigned long depth; }____cacheline_aligned_in_smp ; +static inline struct blk_mq_ctx *__blk_mq_get_ctx (struct request_queue *q, +unsigned int cpu) +{ +return per_cpu_ptr (q->queue_ctx, cpu); +} + +/* +* This assumes per-cpu software queueing queues. They could be per-node +* as well, for instance. For now this is hardcoded as-is. Note that we don't +* care about preemption, since we know the ctx's are persistent. This does +* mean that we can't rely on ctx always matching the currently running CPU. +*/ +static inline struct blk_mq_ctx *blk_mq_get_ctx (struct request_queue *q) +{ +return __blk_mq_get_ctx (q,get_cpu ()); +} + +static inline void blk_mq_put_ctx (struct blk_mq_ctx *ctx) +{ +put_cpu (); +} +#endif
Signature · 1 line
1.7.9.5
PATCH 2/2 blk-mq: fix schedule from atomic context
In reply to Ming Leiblk_mq_put_ctx() has to be called before io_schedule() in bt_get().
This patch fixes the problem by taking similar approach from percpu_ida allocation for the situation.
--- block/blk-mq-tag.c | 48 +++++++++++++++++++++++++++++++----------------- block/blk-mq-tag.h | 2 +- block/blk-mq.c | 36 +++++++++++++++++++++++------------- block/blk-mq.h | 23 +++++++++++++++++++++++ 4 files changed, 78 insertions(+), 31 deletions(-)
block/blk-mq-tag.c +31 -17
@@ -221,8 +221,10 @@ static struct bt_wait_state *bt_wait_ptr(struct blk_mq_bitmap_tags *bt,return bs; } -static int bt_get (struct blk_mq_bitmap_tags *bt,struct blk_mq_hw_ctx *hctx, -unsigned int *last_tag,gfp_t gfp) +static int bt_get (struct blk_mq_alloc_data *data, +struct blk_mq_bitmap_tags *bt, +struct blk_mq_hw_ctx *hctx, +unsigned int *last_tag) {struct bt_wait_state *bs;DEFINE_WAIT (wait);
@@ -232,7 +234,7 @@ static int bt_get(struct blk_mq_bitmap_tags *bt, struct blk_mq_hw_ctx *hctx,if (tag !=-1 )return tag; -if (!(gfp & __GFP_WAIT)) +if (!(data->gfp & __GFP_WAIT))return -1 ; bs =bt_wait_ptr (bt, hctx);
@@ -249,50 +251,62 @@ static int bt_get(struct blk_mq_bitmap_tags *bt, struct blk_mq_hw_ctx *hctx,if (was_empty)atomic_set (&bs->wait_cnt, bt->wake_cnt); +blk_mq_put_ctx (data->ctx); +io_schedule (); + + data->ctx =blk_mq_get_ctx (data->q); + data->hctx = data->q->mq_ops->map_queue (data->q, + data->ctx->cpu); +if (data->reserved) { + bt = &data->hctx->tags->breserved_tags; + }else { + last_tag = &data->ctx->last_tag; + hctx = data->hctx; + bt = &hctx->tags->bitmap_tags; + } +finish_wait (&bs->wait, &wait); + bs =bt_wait_ptr (bt, hctx); }while (1 );finish_wait (&bs->wait, &wait);return tag; } -static unsigned int __blk_mq_get_tag (struct blk_mq_tags *tags, -struct blk_mq_hw_ctx *hctx, -unsigned int *last_tag,gfp_t gfp) +static unsigned int __blk_mq_get_tag (struct blk_mq_alloc_data *data) {int tag; - tag =bt_get (&tags->bitmap_tags, hctx, last_tag, gfp); + tag =bt_get (data, &data->hctx->tags->bitmap_tags, data->hctx, + &data->ctx->last_tag);if (tag >=0 ) -return tag + tags->nr_reserved_tags; +return tag + data->hctx->tags->nr_reserved_tags;return BLK_MQ_TAG_FAIL ; } -static unsigned int __blk_mq_get_reserved_tag (struct blk_mq_tags *tags, -gfp_t gfp) +static unsigned int __blk_mq_get_reserved_tag (struct blk_mq_alloc_data *data) {int tag, zero =0 ; -if (unlikely (!tags->nr_reserved_tags)) { +if (unlikely (!data->hctx->tags->nr_reserved_tags)) {WARN_ON_ONCE (1 );return BLK_MQ_TAG_FAIL ; } - tag =bt_get (&tags->breserved_tags,NULL , &zero, gfp); + tag =bt_get (data, &data->hctx->tags->breserved_tags,NULL , &zero);if (tag <0 )return BLK_MQ_TAG_FAIL ;return tag; } -unsigned int blk_mq_get_tag (struct blk_mq_hw_ctx *hctx,unsigned int *last_tag, -gfp_t gfp,bool reserved) +unsigned int blk_mq_get_tag (struct blk_mq_alloc_data *data) { -if (!reserved) -return __blk_mq_get_tag (hctx->tags, hctx, last_tag, gfp); +if (!data->reserved) +return __blk_mq_get_tag (data); -return __blk_mq_get_reserved_tag (hctx->tags, gfp); +return __blk_mq_get_reserved_tag (data); }static struct bt_wait_state *bt_wake_ptr (struct blk_mq_bitmap_tags *bt)
block/blk-mq-tag.h +1 -1
@@ -48,7 +48,7 @@ struct blk_mq_tags {extern struct blk_mq_tags *blk_mq_init_tags (unsigned int nr_tags,unsigned int reserved_tags,int node);extern void blk_mq_free_tags (struct blk_mq_tags *tags); -extern unsigned int blk_mq_get_tag (struct blk_mq_hw_ctx *hctx,unsigned int *last_tag,gfp_t gfp,bool reserved); +extern unsigned int blk_mq_get_tag (struct blk_mq_alloc_data *data);extern void blk_mq_put_tag (struct blk_mq_hw_ctx *hctx,unsigned int tag,unsigned int *last_tag);extern bool blk_mq_has_free_tags (struct blk_mq_tags *tags);extern ssize_t blk_mq_tag_sysfs_show (struct blk_mq_tags *tags,char *page);
block/blk-mq.c +23 -13
@@ -210,24 +210,23 @@ static void blk_mq_rq_ctx_init(struct request_queue *q, struct blk_mq_ctx *ctx, }static struct request * -__blk_mq_alloc_request (struct request_queue *q,struct blk_mq_hw_ctx *hctx, -struct blk_mq_ctx *ctx,int rw,gfp_t gfp,bool reserved) +__blk_mq_alloc_request (struct blk_mq_alloc_data *data,int rw) {struct request *rq;unsigned int tag; - tag =blk_mq_get_tag (hctx, &ctx->last_tag, gfp, reserved); + tag =blk_mq_get_tag (data);if (tag !=BLK_MQ_TAG_FAIL ) { - rq = hctx->tags->rqs[tag]; + rq = data->hctx->tags->rqs[tag]; rq->cmd_flags =0 ; -if (blk_mq_tag_busy (hctx)) { +if (blk_mq_tag_busy (data->hctx)) { rq->cmd_flags =REQ_MQ_INFLIGHT ; -atomic_inc (&hctx->nr_active); +atomic_inc (&data->hctx->nr_active); } rq->tag = tag; -blk_mq_rq_ctx_init (q, ctx, rq, rw); +blk_mq_rq_ctx_init (data->q, data->ctx, rq, rw);return rq; }
@@ -240,22 +239,27 @@ struct request *blk_mq_alloc_request(struct request_queue *q, int rw, gfp_t gfp,struct blk_mq_ctx *ctx;struct blk_mq_hw_ctx *hctx;struct request *rq; +struct blk_mq_alloc_data alloc_data;if (blk_mq_queue_enter (q))return NULL ; ctx =blk_mq_get_ctx (q); hctx = q->mq_ops->map_queue (q, ctx->cpu); +blk_mq_set_alloc_data (&alloc_data, q, gfp & ~__GFP_WAIT, + reserved, ctx, hctx); - rq =__blk_mq_alloc_request (q, hctx, ctx, rw, gfp & ~__GFP_WAIT, - reserved); + rq =__blk_mq_alloc_request (&alloc_data, rw);if (!rq && (gfp & __GFP_WAIT)) {__blk_mq_run_hw_queue (hctx);blk_mq_put_ctx (ctx); ctx =blk_mq_get_ctx (q); hctx = q->mq_ops->map_queue (q, ctx->cpu); - rq =__blk_mq_alloc_request (q, hctx, ctx, rw, gfp, reserved); +blk_mq_set_alloc_data (&alloc_data, q, gfp, reserved, ctx, + hctx); + rq =__blk_mq_alloc_request (&alloc_data, rw); + ctx = alloc_data.ctx; }blk_mq_put_ctx (ctx);return rq;
@@ -1136,6 +1140,7 @@ static struct request *blk_mq_map_request(struct request_queue *q,struct blk_mq_ctx *ctx;struct request *rq;int rw =bio_data_dir (bio); +struct blk_mq_alloc_data alloc_data;if (unlikely (blk_mq_queue_enter (q))) {bio_endio (bio, -EIO );
@@ -1149,7 +1154,9 @@ static struct request *blk_mq_map_request(struct request_queue *q, rw |=REQ_SYNC ;trace_block_getrq (q, bio, rw); - rq =__blk_mq_alloc_request (q, hctx, ctx, rw,GFP_ATOMIC , false); +blk_mq_set_alloc_data (&alloc_data, q,GFP_ATOMIC , false, ctx, + hctx); + rq =__blk_mq_alloc_request (&alloc_data, rw);if (unlikely (!rq)) {__blk_mq_run_hw_queue (hctx);blk_mq_put_ctx (ctx);
@@ -1157,8 +1164,11 @@ static struct request *blk_mq_map_request(struct request_queue *q, ctx =blk_mq_get_ctx (q); hctx = q->mq_ops->map_queue (q, ctx->cpu); - rq =__blk_mq_alloc_request (q, hctx, ctx, rw, - __GFP_WAIT|GFP_ATOMIC , false); +blk_mq_set_alloc_data (&alloc_data, q, + __GFP_WAIT|GFP_ATOMIC , false, ctx, hctx); + rq =__blk_mq_alloc_request (&alloc_data, rw); + ctx = alloc_data.ctx; + hctx = alloc_data.hctx; } hctx->queued++;
block/blk-mq.h +23 -0
@@ -91,4 +91,27 @@ static inline void blk_mq_put_ctx(struct blk_mq_ctx *ctx) put_cpu(); } +struct blk_mq_alloc_data { +/* input parameter */ +struct request_queue *q; +gfp_t gfp; +bool reserved; + +/* input & output parameter */ +struct blk_mq_ctx *ctx; +struct blk_mq_hw_ctx *hctx; +}; + +static inline void blk_mq_set_alloc_data (struct blk_mq_alloc_data *data, +struct request_queue *q,gfp_t gfp,bool reserved, +struct blk_mq_ctx *ctx, +struct blk_mq_hw_ctx *hctx) +{ + data->q = q; + data->gfp = gfp; + data->reserved = reserved; + data->ctx = ctx; + data->hctx = hctx; +} +#endif
Signature · 1 line
1.7.9.5
Jens Axboe
maintainerQuoted 5 lines · Ming Lei
blk_mq_put_ctx()has to be called beforeio_schedule()inbt_get().This patch fixes the problem by taking similar approach from
percpu_idaallocation for the situation.
Thanks, I've applied these to for post-merge integration. I do think we can clean up further around 'data' for bt_get(), for instance, and possibly in other places too.
Signature · 1 line
Jens Axboe