blk-mq: fix start_time_ns and alloc_time_ns for pre-allocated rq
v4 cover blk-mq: fix start_time_ns and alloc_time_ns for pre-allocated rq
From: Chengming Zhou <[email protected]>
Hello,
This patchset fix start_time_ns and alloc_time_ns for pre-allocated rq.
patch 1 is preparation that we move data->rq_flags initialization code outside blk_mq_rq_ctx_init() to its caller, since we will need it to tell if we need to get start_time_ns.
patch 2 is the fix patch that we set alloc_time_ns and start_time_ns to now when the pre-allocated rq is actually used and we skip setting during pre-allocation.
v4:
- Combine the last two patches into one patch for clearer review.
- Use blk_mq_alloc_data to pass start_time_ns instead of passing down yet another parameter. Thanks Christoph Hellwig.
- [v3] https://lore.kernel.org/all/[email protected]/
v3:
- Skip setting the alloc_time_ns and start_time_ns during pre-allocation, which is clearer, as suggested by Tejun.
- [v2] https://lore.kernel.org/all/[email protected]/
v2:
- Let blk_mq_rq_ctx_init() receive start_time_ns for batched time setting.
- Set alloc_time_ns and start_time_ns when the pre-allocated rq is actually used, as suggested by Tejun.
- [v1] https://lore.kernel.org/all/[email protected]/
Chengming Zhou (2):
blk-mq: move data->rq_flags initialization code
blk-mq: fix start_time_ns and alloc_time_ns for pre-allocated rq
block/blk-mq.c | 87 ++++++++++++++++++++++++------------------ block/blk-mq.h | 1 + include/linux/blk-mq.h | 6 +-- 3 files changed, 53 insertions(+), 41 deletions(-)
Signature · 1 line
2.39.2
PATCH v4 1/2 blk-mq: move data->rq_flags initialization code
From: Chengming Zhou <[email protected]>
The blk_mq_rq_ctx_init() has some data->rq_flags initialization:
``` if (data->flags & BLK_MQ_REQ_PM) data->rq_flags |= RQF_PM; if (blk_queue_io_stat(q)) data->rq_flags |= RQF_IO_STAT; rq->rq_flags = data->rq_flags; ```
We will need this data->rq_flags to tell if we need start_time_ns, so we need to put these initialization in the callers of blk_mq_rq_ctx_init().
Now we basically have two callers, the 1st is general __blk_mq_alloc_requests(), the 2nd is the special blk_mq_alloc_request_hctx(). So change the 2nd caller to reuse the 1st __blk_mq_alloc_requests() to avoid code duplication.
This is safe because blk_mq_alloc_request_hctx() always has BLK_MQ_REQ_NOWAIT flag, it won't go into the normal retry path when doesn't have free tag. But it indeed make the general __blk_mq_alloc_requests() more complex.
--- block/blk-mq.c | 46 ++++++++++++++++++---------------------------- 1 file changed, 18 insertions(+), 28 deletions(-)
block/blk-mq.c +18 -28
@@ -349,11 +349,6 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data, rq->mq_ctx = ctx; rq->mq_hctx = hctx; rq->cmd_flags = data->cmd_flags; - -if (data->flags &BLK_MQ_REQ_PM ) - data->rq_flags |=RQF_PM ; -if (blk_queue_io_stat (q)) - data->rq_flags |=RQF_IO_STAT ; rq->rq_flags = data->rq_flags;if (data->rq_flags &RQF_SCHED_TAGS ) {
@@ -447,6 +442,15 @@ static struct request *__blk_mq_alloc_requests(struct blk_mq_alloc_data *data)if (data->cmd_flags &REQ_NOWAIT ) data->flags |=BLK_MQ_REQ_NOWAIT ; +if (data->flags &BLK_MQ_REQ_RESERVED ) + data->rq_flags |=RQF_RESV ; + +if (data->flags &BLK_MQ_REQ_PM ) + data->rq_flags |=RQF_PM ; + +if (blk_queue_io_stat (q)) + data->rq_flags |=RQF_IO_STAT ; +if (q->elevator) {/* * All requests use scheduler tags when an I/O scheduler is
@@ -471,14 +475,15 @@ static struct request *__blk_mq_alloc_requests(struct blk_mq_alloc_data *data) } retry: - data->ctx =blk_mq_get_ctx (q); - data->hctx =blk_mq_map_queue (q, data->cmd_flags, data->ctx); +/* See blk_mq_alloc_request_hctx() for details */ +if (!data->ctx) { + data->ctx =blk_mq_get_ctx (q); + data->hctx =blk_mq_map_queue (q, data->cmd_flags, data->ctx); + } +if (!(data->rq_flags &RQF_SCHED_TAGS ))blk_mq_tag_busy (data->hctx); -if (data->flags &BLK_MQ_REQ_RESERVED ) - data->rq_flags |=RQF_RESV ; -/* * Try batched alloc if we want more than 1 tag. */
@@ -505,6 +510,7 @@ static struct request *__blk_mq_alloc_requests(struct blk_mq_alloc_data *data)* is going away. */ msleep (3 ); + data->ctx =NULL ; goto retry; }
@@ -613,16 +619,10 @@ struct request *blk_mq_alloc_request_hctx(struct request_queue *q, .cmd_flags = opf, .nr_tags = 1, }; -u64 alloc_time_ns =0 ;struct request *rq;unsigned int cpu; -unsigned int tag;int ret; -/* alloc_time includes depth and tag waits */ -if (blk_queue_rq_alloc_time (q)) - alloc_time_ns =ktime_get_ns (); -/* * If the tag allocator sleeps we could get an allocation for a * different hardware context. No need to complicate the low level
@@ -653,20 +653,10 @@ struct request *blk_mq_alloc_request_hctx(struct request_queue *q, goto out_queue_exit; data.ctx =__blk_mq_get_ctx (q, cpu); -if (q->elevator) - data.rq_flags |=RQF_SCHED_TAGS ; -else -blk_mq_tag_busy (data.hctx); - -if (flags &BLK_MQ_REQ_RESERVED ) - data.rq_flags |=RQF_RESV ; - ret = -EWOULDBLOCK ; - tag =blk_mq_get_tag (&data); -if (tag ==BLK_MQ_NO_TAG ) + rq =__blk_mq_alloc_requests (&data); +if (!rq) goto out_queue_exit; - rq =blk_mq_rq_ctx_init (&data,blk_mq_tags_from_data (&data), tag, - alloc_time_ns); rq->__data_len =0 ; rq->__sector = (sector_t) -1 ; rq->bio = rq->biotail =NULL ;
Signature · 1 line
2.39.2
PATCH v4 2/2 blk-mq: fix start_time_ns and alloc_time_ns for pre-allocated rq
In reply to [email protected]From: Chengming Zhou <[email protected]>
The iocost rely on rq start_time_ns and alloc_time_ns to tell saturation state of the block device. Most of the time request is allocated after rq_qos_throttle() and its alloc_time_ns or start_time_ns won't be affected.
But for plug batched allocation introduced by the commit 47c122e35d7e ("block: pre-allocate requests if plug is started and is a batch"), we can rq_qos_throttle() after the allocation of the request. This is what the blk_mq_get_cached_request() does.
In this case, the cached request alloc_time_ns or start_time_ns is much ahead if blocked in any qos ->throttle().
This patch fix it by setting alloc_time_ns and start_time_ns to now when the pre-allocated rq is actually used. And we skip setting the alloc_time_ns and start_time_ns during pre-allocation, so just pass 0 in __blk_mq_alloc_requests_batch().
--- block/blk-mq.c | 41 +++++++++++++++++++++++++++++++---------- block/blk-mq.h | 1 + include/linux/blk-mq.h | 6 +++--- 3 files changed, 35 insertions(+), 13 deletions(-)
block/blk-mq.c +31 -10
@@ -337,6 +337,24 @@ void blk_rq_init(struct request_queue *q, struct request *rq) }EXPORT_SYMBOL (blk_rq_init); +/* Set alloc and start time when pre-allocated rq is actually used */ +static inline void blk_mq_rq_time_init (struct request *rq) +{ +if (blk_mq_need_time_stamp (rq->rq_flags)) { +u64 now =ktime_get_ns (); + +#ifdef CONFIG_BLK_RQ_ALLOC_TIME +/* +* The alloc time is only used by iocost for now, +* only possible when blk_mq_need_time_stamp(). +*/ +if (blk_queue_rq_alloc_time (rq->q)) + rq->alloc_time_ns = now; +#endif + rq->start_time_ns = now; + } +} +static struct request *blk_mq_rq_ctx_init (struct blk_mq_alloc_data *data,struct blk_mq_tags *tags,unsigned int tag,u64 alloc_time_ns) {
@@ -360,14 +378,11 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data, } rq->timeout =0 ; -if (blk_mq_need_time_stamp (rq)) - rq->start_time_ns =ktime_get_ns (); -else - rq->start_time_ns =0 ; rq->part =NULL ;#ifdef CONFIG_BLK_RQ_ALLOC_TIME rq->alloc_time_ns = alloc_time_ns;#endif + rq->start_time_ns = data->start_time_ns; rq->io_start_time_ns =0 ; rq->stats_sectors =0 ; rq->nr_phys_segments =0 ;
@@ -397,8 +412,7 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data, }static inline struct request * -__blk_mq_alloc_requests_batch (struct blk_mq_alloc_data *data, -u64 alloc_time_ns) +__blk_mq_alloc_requests_batch (struct blk_mq_alloc_data *data) {unsigned int tag, tag_offset;struct blk_mq_tags *tags;
@@ -417,7 +431,7 @@ __blk_mq_alloc_requests_batch(struct blk_mq_alloc_data *data, tag = tag_offset + i;prefetch (tags->static_rqs[tag]); tag_mask &= ~(1UL << i); - rq =blk_mq_rq_ctx_init (data, tags, tag, alloc_time_ns); + rq =blk_mq_rq_ctx_init (data, tags, tag,0 );rq_list_add (data->cached_rq, rq); nr++; }
@@ -488,9 +502,11 @@ static struct request *__blk_mq_alloc_requests(struct blk_mq_alloc_data *data)* Try batched alloc if we want more than 1 tag. */ if (data->nr_tags >1 ) { - rq =__blk_mq_alloc_requests_batch (data, alloc_time_ns); -if (rq) + rq =__blk_mq_alloc_requests_batch (data); +if (rq) { +blk_mq_rq_time_init (rq);return rq; + } data->nr_tags =1 ; }
@@ -514,6 +530,9 @@ static struct request *__blk_mq_alloc_requests(struct blk_mq_alloc_data *data) goto retry; } +if (blk_mq_need_time_stamp (data->rq_flags)) + data->start_time_ns =ktime_get_ns (); +return blk_mq_rq_ctx_init (data,blk_mq_tags_from_data (data), tag, alloc_time_ns); }
@@ -570,6 +589,7 @@ static struct request *blk_mq_alloc_cached_request(struct request_queue *q,return NULL ; plug->cached_rq =rq_list_next (rq); +blk_mq_rq_time_init (rq); } rq->cmd_flags = opf;
@@ -1004,7 +1024,7 @@ static inline void __blk_mq_end_request_acct(struct request *rq, u64 now)inline void __blk_mq_end_request (struct request *rq,blk_status_t error) { -if (blk_mq_need_time_stamp (rq)) +if (blk_mq_need_time_stamp (rq->rq_flags))__blk_mq_end_request_acct (rq,ktime_get_ns ());if (rq->end_io) {
@@ -2891,6 +2911,7 @@ static inline struct request *blk_mq_get_cached_request(struct request_queue *q, plug->cached_rq =rq_list_next (rq);rq_qos_throttle (q, *bio); +blk_mq_rq_time_init (rq); rq->cmd_flags = (*bio)->bi_opf;INIT_LIST_HEAD (&rq->queuelist);return rq;
block/blk-mq.h +1 -0
@@ -150,6 +150,7 @@ struct blk_mq_alloc_data {unsigned int shallow_depth;blk_opf_t cmd_flags;req_flags_t rq_flags; +u64 start_time_ns;/* allocate multiple requests/tags in one go */ unsigned int nr_tags;
include/linux/blk-mq.h +3 -3
@@ -834,9 +834,9 @@ void blk_mq_end_request_batch(struct io_comp_batch *ib);* Only need start/end time stamping if we have iostat or * blk stats enabled, or using an IO scheduler. */ -static inline bool blk_mq_need_time_stamp (struct request *rq) +static inline bool blk_mq_need_time_stamp (req_flags_t rq_flags) { -return (rq->rq_flags & (RQF_IO_STAT |RQF_STATS |RQF_USE_SCHED )); +return (rq_flags & (RQF_IO_STAT |RQF_STATS |RQF_USE_SCHED )); }static inline bool blk_mq_is_reserved_rq (struct request *rq)
@@ -860,7 +860,7 @@ static inline bool blk_mq_add_to_batch(struct request *req, iob->complete = complete; else if (iob->complete != complete) return false; - iob->need_ts |=blk_mq_need_time_stamp (req); + iob->need_ts |=blk_mq_need_time_stamp (req->rq_flags);rq_list_add (&iob->req_list, req);return true; }
Signature · 1 line
2.39.2