blk-mq: refactor request allocation
This series streamlines the request allocation path.
PATCH 1/5 blk-mq: merge blk_mq_alloc_reserved_request into blk_mq_alloc_request
Instead of having two almost identical copies of the same code just let the callers pass in the reserved flag directly.
--- block/blk-core.c | 2 +- block/blk-mq.c | 20 +++----------------- drivers/block/mtip32xx/mtip32xx.c | 2 +- include/linux/blk-mq.h | 4 ++-- 4 files changed, 7 insertions(+), 21 deletions(-)
block/blk-core.c +1 -1
@@ -1173,7 +1173,7 @@ static struct request *blk_old_get_request(struct request_queue *q, int rw,struct request *blk_get_request (struct request_queue *q,int rw,gfp_t gfp_mask) {if (q->mq_ops) -return blk_mq_alloc_request (q, rw, gfp_mask); +return blk_mq_alloc_request (q, rw, gfp_mask, false);else return blk_old_get_request (q, rw, gfp_mask); }
block/blk-mq.c +3 -17
@@ -294,35 +294,21 @@ static struct request *blk_mq_alloc_request_pinned(struct request_queue *q,return rq; } -struct request *blk_mq_alloc_request (struct request_queue *q,int rw,gfp_t gfp) +struct request *blk_mq_alloc_request (struct request_queue *q,int rw,gfp_t gfp, +bool reserved) {struct request *rq;if (blk_mq_queue_enter (q))return NULL ; - rq =blk_mq_alloc_request_pinned (q, rw, gfp, false); + rq =blk_mq_alloc_request_pinned (q, rw, gfp, reserved);if (rq)blk_mq_put_ctx (rq->mq_ctx);return rq; }EXPORT_SYMBOL (blk_mq_alloc_request); -struct request *blk_mq_alloc_reserved_request (struct request_queue *q,int rw, -gfp_t gfp) -{ -struct request *rq; - -if (blk_mq_queue_enter (q)) -return NULL ; - - rq =blk_mq_alloc_request_pinned (q, rw, gfp, true); -if (rq) -blk_mq_put_ctx (rq->mq_ctx); -return rq; -} -EXPORT_SYMBOL (blk_mq_alloc_reserved_request); -static void __blk_mq_free_request (struct blk_mq_hw_ctx *hctx,struct blk_mq_ctx *ctx,struct request *rq) {
drivers/block/mtip32xx/mtip32xx.c +1 -1
@@ -178,7 +178,7 @@ static struct mtip_cmd *mtip_get_int_command(struct driver_data *dd) {struct request *rq; - rq =blk_mq_alloc_reserved_request (dd->queue,0 , __GFP_WAIT); + rq =blk_mq_alloc_request (dd->queue,0 , __GFP_WAIT, true);return blk_mq_rq_to_pdu (rq); }
include/linux/blk-mq.h +2 -2
@@ -160,8 +160,8 @@ void blk_mq_insert_request(struct request *, bool, bool, bool);void blk_mq_run_queues (struct request_queue *q,bool async);void blk_mq_free_request (struct request *rq);bool blk_mq_can_queue (struct blk_mq_hw_ctx *); -struct request *blk_mq_alloc_request (struct request_queue *q,int rw,gfp_t gfp); -struct request *blk_mq_alloc_reserved_request (struct request_queue *q,int rw,gfp_t gfp); +struct request *blk_mq_alloc_request (struct request_queue *q,int rw, +gfp_t gfp,bool reserved);struct request *blk_mq_tag_to_rq (struct blk_mq_tags *tags,unsigned int tag);struct blk_mq_hw_ctx *blk_mq_map_queue (struct request_queue *,const int ctx_index);
Signature · 1 line
1.7.10.4
PATCH 2/5 blk-mq: initialize request in __blk_mq_alloc_request
In reply to Christoph HellwigBoth callers if __blk_mq_alloc_request want to initialize the request, so lift it into the common path.
--- block/blk-mq.c | 62 +++++++++++++++++++++++++++----------------------------- 1 file changed, 30 insertions(+), 32 deletions(-)
block/blk-mq.c +30 -32
@@ -92,30 +92,6 @@ static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx,clear_bit (CTX_TO_BIT (hctx, ctx), &bm->word); } -static struct request *__blk_mq_alloc_request (struct blk_mq_hw_ctx *hctx, -struct blk_mq_ctx *ctx, -gfp_t gfp,bool reserved) -{ -struct request *rq; -unsigned int tag; - - tag =blk_mq_get_tag (hctx, &ctx->last_tag, gfp, reserved); -if (tag !=BLK_MQ_TAG_FAIL ) { - rq = hctx->tags->rqs[tag]; - - rq->cmd_flags =0 ; -if (blk_mq_tag_busy (hctx)) { - rq->cmd_flags =REQ_MQ_INFLIGHT ; -atomic_inc (&hctx->nr_active); - } - - rq->tag = tag; -return rq; - } - -return NULL ; -} -static int blk_mq_queue_enter (struct request_queue *q) {int ret;
@@ -263,6 +239,32 @@ static void blk_mq_rq_ctx_init(struct request_queue *q, struct blk_mq_ctx *ctx, ctx->rq_dispatched[rw_is_sync (rw_flags)]++; } +static struct request * +__blk_mq_alloc_request (struct request_queue *q,struct blk_mq_hw_ctx *hctx, +struct blk_mq_ctx *ctx,int rw,gfp_t gfp,bool reserved) +{ +struct request *rq; +unsigned int tag; + + tag =blk_mq_get_tag (hctx, &ctx->last_tag, gfp, reserved); +if (tag !=BLK_MQ_TAG_FAIL ) { + rq = hctx->tags->rqs[tag]; + + rq->cmd_flags =0 ; +if (blk_mq_tag_busy (hctx)) { + rq->cmd_flags =REQ_MQ_INFLIGHT ; +atomic_inc (&hctx->nr_active); + } + + rq->tag = tag; +blk_mq_rq_ctx_init (q, ctx, rq, rw); +return rq; + } + +return NULL ; +} + +static struct request *blk_mq_alloc_request_pinned (struct request_queue *q,int rw,gfp_t gfp,bool reserved)
@@ -273,12 +275,10 @@ static struct request *blk_mq_alloc_request_pinned(struct request_queue *q,struct blk_mq_ctx *ctx =blk_mq_get_ctx (q);struct blk_mq_hw_ctx *hctx = q->mq_ops->map_queue (q, ctx->cpu); - rq =__blk_mq_alloc_request (hctx, ctx, gfp & ~__GFP_WAIT, + rq =__blk_mq_alloc_request (q, hctx, ctx, rw, gfp & ~__GFP_WAIT, reserved); -if (rq) { -blk_mq_rq_ctx_init (q, ctx, rq, rw); +if (rq)break ; - }if (gfp & __GFP_WAIT) {__blk_mq_run_hw_queue (hctx);
@@ -1114,10 +1114,8 @@ static struct request *blk_mq_map_request(struct request_queue *q, rw |=REQ_SYNC ;trace_block_getrq (q, bio, rw); - rq =__blk_mq_alloc_request (hctx, ctx,GFP_ATOMIC , false); -if (likely (rq)) -blk_mq_rq_ctx_init (q, ctx, rq, rw); -else { + rq =__blk_mq_alloc_request (q, hctx, ctx, rw,GFP_ATOMIC , false); +if (unlikely (!rq)) {blk_mq_put_ctx (ctx);trace_block_sleeprq (q, bio, rw); rq = blk_mq_alloc_request_pinned(q, rw, __GFP_WAIT|GFP_ATOMIC,
Signature · 1 line
1.7.10.4
PATCH 3/5 blk-mq: remove blk_mq_wait_for_tags
In reply to Christoph HellwigThe current logic for blocking tag allocation is rather confusing, as we first allocated and then free again a tag in blk_mq_wait_for_tags, just to attempt a non-blocking allocation and then repeat if someone else managed to grab the tag before us.
Instead change blk_mq_alloc_request_pinned to simply do a blocking tag allocation itself and use the request we get back from it.
--- block/blk-mq-tag.c | 8 -------- block/blk-mq-tag.h | 1 - block/blk-mq.c | 13 ++++++------- 3 files changed, 6 insertions(+), 16 deletions(-)
block/blk-mq-tag.c +0 -8
@@ -7,14 +7,6 @@#include "blk-mq.h" #include "blk-mq-tag.h" -void blk_mq_wait_for_tags (struct blk_mq_hw_ctx *hctx,bool reserved) -{ -int tag, zero =0 ; - - tag =blk_mq_get_tag (hctx, &zero, __GFP_WAIT, reserved); -blk_mq_put_tag (hctx, tag, &zero); -} -static bool bt_has_free_tags (struct blk_mq_bitmap_tags *bt) {int i;
block/blk-mq-tag.h +0 -1
@@ -49,7 +49,6 @@ extern struct blk_mq_tags *blk_mq_init_tags(unsigned int nr_tags, unsigned int rextern void blk_mq_free_tags (struct blk_mq_tags *tags);extern unsigned int blk_mq_get_tag (struct blk_mq_hw_ctx *hctx,unsigned int *last_tag,gfp_t gfp,bool reserved); -extern void blk_mq_wait_for_tags (struct blk_mq_hw_ctx *hctx,bool reserved);extern void blk_mq_put_tag (struct blk_mq_hw_ctx *hctx,unsigned int tag,unsigned int *last_tag);extern bool blk_mq_has_free_tags (struct blk_mq_tags *tags);extern ssize_t blk_mq_tag_sysfs_show (struct blk_mq_tags *tags,char *page);
block/blk-mq.c +6 -7
@@ -264,31 +264,30 @@ __blk_mq_alloc_request(struct request_queue *q, struct blk_mq_hw_ctx *hctx,return NULL ; } -static struct request *blk_mq_alloc_request_pinned (struct request_queue *q,int rw,gfp_t gfp,bool reserved) { +bool gfp_mask = gfp & ~__GFP_WAIT;struct request *rq;do {struct blk_mq_ctx *ctx =blk_mq_get_ctx (q);struct blk_mq_hw_ctx *hctx = q->mq_ops->map_queue (q, ctx->cpu); - rq =__blk_mq_alloc_request (q, hctx, ctx, rw, gfp & ~__GFP_WAIT, + rq =__blk_mq_alloc_request (q, hctx, ctx, rw, gfp_mask, reserved);if (rq)break ; -if (gfp & __GFP_WAIT) { -__blk_mq_run_hw_queue (hctx); -blk_mq_put_ctx (ctx); - }else { +if (!(gfp & __GFP_WAIT)) {blk_mq_put_ctx (ctx);break ; } -blk_mq_wait_for_tags (hctx, reserved); +__blk_mq_run_hw_queue (hctx); +blk_mq_put_ctx (ctx); + gfp_mask = gfp; }while (1 );return rq;
Signature · 1 line
1.7.10.4
PATCH 4/5 blk-mq: do not use blk_mq_alloc_request_pinned in blk_mq_map_request
In reply to Christoph HellwigWe already do a non-blocking allocation in blk_mq_map_request, no need to repeat it. Just call __blk_mq_alloc_request to wait directly.
--- block/blk-mq.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-)
block/blk-mq.c +5 -3
@@ -1115,12 +1115,14 @@ static struct request *blk_mq_map_request(struct request_queue *q,trace_block_getrq (q, bio, rw); rq =__blk_mq_alloc_request (q, hctx, ctx, rw,GFP_ATOMIC , false);if (unlikely (!rq)) { +__blk_mq_run_hw_queue (hctx);blk_mq_put_ctx (ctx);trace_block_sleeprq (q, bio, rw); - rq =blk_mq_alloc_request_pinned (q, rw, __GFP_WAIT|GFP_ATOMIC , - false); - ctx = rq->mq_ctx; + + ctx =blk_mq_get_ctx (q); hctx = q->mq_ops->map_queue (q, ctx->cpu); + rq =__blk_mq_alloc_request (q, hctx, ctx, rw, + __GFP_WAIT|GFP_ATOMIC , false); } hctx->queued++;
Signature · 1 line
1.7.10.4
PATCH 5/5 blk-mq: remove blk_mq_alloc_request_pinned
In reply to Christoph HellwigWe now only have one caller left and can open code it there in a cleaner way.
--- block/blk-mq.c | 48 ++++++++++++++++-------------------------------- 1 file changed, 16 insertions(+), 32 deletions(-)
block/blk-mq.c +16 -32
@@ -264,46 +264,30 @@ __blk_mq_alloc_request(struct request_queue *q, struct blk_mq_hw_ctx *hctx,return NULL ; } -static struct request *blk_mq_alloc_request_pinned (struct request_queue *q, -int rw,gfp_t gfp, -bool reserved) -{ -bool gfp_mask = gfp & ~__GFP_WAIT; -struct request *rq; - -do { -struct blk_mq_ctx *ctx =blk_mq_get_ctx (q); -struct blk_mq_hw_ctx *hctx = q->mq_ops->map_queue (q, ctx->cpu); - - rq =__blk_mq_alloc_request (q, hctx, ctx, rw, gfp_mask, - reserved); -if (rq) -break ; - -if (!(gfp & __GFP_WAIT)) { -blk_mq_put_ctx (ctx); -break ; - } - -__blk_mq_run_hw_queue (hctx); -blk_mq_put_ctx (ctx); - gfp_mask = gfp; - }while (1 ); - -return rq; -} -struct request *blk_mq_alloc_request (struct request_queue *q,int rw,gfp_t gfp,bool reserved) { +struct blk_mq_ctx *ctx; +struct blk_mq_hw_ctx *hctx;struct request *rq;if (blk_mq_queue_enter (q))return NULL ; - rq =blk_mq_alloc_request_pinned (q, rw, gfp, reserved); -if (rq) -blk_mq_put_ctx (rq->mq_ctx); + ctx =blk_mq_get_ctx (q); + hctx = q->mq_ops->map_queue (q, ctx->cpu); + + rq =__blk_mq_alloc_request (q, hctx, ctx, rw, gfp & ~__GFP_WAIT, + reserved); +if (!rq && (gfp & __GFP_WAIT)) { +__blk_mq_run_hw_queue (hctx); +blk_mq_put_ctx (ctx); + + ctx =blk_mq_get_ctx (q); + hctx = q->mq_ops->map_queue (q, ctx->cpu); + rq =__blk_mq_alloc_request (q, hctx, ctx, rw, gfp, reserved); + } +blk_mq_put_ctx (ctx);return rq; }EXPORT_SYMBOL (blk_mq_alloc_request);
Signature · 1 line
1.7.10.4
Quoted 2 lines · Christoph Hellwig
This series streamlines the request allocation path.
Series looks innocuous enough to me, but it's about a 1.5% performance drop here with an actual device. These tests are very stable, anything over ~0.1% is definitely outside of noise. I repeated and rebooted a few times and tested both, it's persistent. No smoking guns in the profile.
Quoted 8 lines · Jens Axboe
Quoted 2 lines · Christoph Hellwig
This series streamlines the request allocation path.
Series looks innocuous enough to me, but it's about a 1.5% performance drop here with an actual device. These tests are very stable, anything over ~0.1% is definitely outside of noise. I repeated and rebooted a few times and tested both, it's persistent. No smoking guns in the profile.
Can you do a bisect to narrow it down to one of the patches?
Quoted 11 lines · Christoph Hellwig
Quoted 8 lines · Jens Axboe
Quoted 2 lines · Christoph Hellwig
This series streamlines the request allocation path.
Series looks innocuous enough to me, but it's about a 1.5% performance drop here with an actual device. These tests are very stable, anything over ~0.1% is definitely outside of noise. I repeated and rebooted a few times and tested both, it's persistent. No smoking guns in the profile.
Can you do a bisect to narrow it down to one of the patches?
Sure, I'll give it a whirl.
Signature · 1 line
Jens Axboe
Quoted 11 lines · Christoph Hellwig
Quoted 8 lines · Jens Axboe
Quoted 2 lines · Christoph Hellwig
This series streamlines the request allocation path.
Series looks innocuous enough to me, but it's about a 1.5% performance drop here with an actual device. These tests are very stable, anything over ~0.1% is definitely outside of noise. I repeated and rebooted a few times and tested both, it's persistent. No smoking guns in the profile.
Can you do a bisect to narrow it down to one of the patches?
Did bisect and even one-by-one, and now it doesn't show up of course. The nvme branch was rebased on top of a new base since yesterday, so it could just be layout weirdness from the previous one.
So I think we can call it fine, I'll apply it.
Signature · 1 line
Jens Axboe