blk-mq: avoid excessive boot delays with large lun counts
PATCH 1/1 blk-mq: avoid excessive boot delays with large lun counts
Hi,
Zhangqing Luo reported long boot times on a system with thousands of LUNs when scsi-mq was enabled. He narrowed the problem down to blk_mq_add_queue_tag_set, where every queue is frozen in order to set the BLK_MQ_F_TAG_SHARED flag. Each added device will freeze all queues added before it in sequence, which involves waiting for an RCU grace period for each one. We don't need to do this. After the second queue is added, only new queues need to be initialized with the shared tag. We can do that by percolating the flag up to the blk_mq_tag_set, since new hctxs already inherit the tag_set's flags. That's what the below patch does.
The re-check in blk_mq_add_queue_tag_set is done because we could have two queues racing to be the second queue added for the tag set. In such a case, it's possible that not all of the hctxs within the loser will be initialized. Because this initialization is done sequentially, we can simply check for the BLK_MQ_F_TAG_SHARED flag in the first hctx.
This problem was introduced by commit 0d2602ca30e41 (blk-mq: improve support for shared tags maps).
block/blk-mq.c +38 -16
@@ -1860,27 +1860,26 @@ static void blk_mq_map_swqueue(struct request_queue *q, } } -static void blk_mq_update_tag_set_depth (struct blk_mq_tag_set *set) +static void queue_set_hctx_shared (struct request_queue *q,bool shared) {struct blk_mq_hw_ctx *hctx; -struct request_queue *q; -bool shared;int i; -if (set->tag_list.next == set->tag_list.prev) - shared = false; -else - shared = true; +queue_for_each_hw_ctx (q, hctx, i) { +if (shared) + hctx->flags |=BLK_MQ_F_TAG_SHARED ; +else + hctx->flags &= ~BLK_MQ_F_TAG_SHARED ; + } +} + +static void blk_mq_update_tag_set_depth (struct blk_mq_tag_set *set,bool shared) +{ +struct request_queue *q;list_for_each_entry (q, &set->tag_list, tag_set_list) {blk_mq_freeze_queue (q); - -queue_for_each_hw_ctx (q, hctx, i) { -if (shared) - hctx->flags |=BLK_MQ_F_TAG_SHARED ; -else - hctx->flags &= ~BLK_MQ_F_TAG_SHARED ; - } +queue_set_hctx_shared (q, shared);blk_mq_unfreeze_queue (q); } }
@@ -1891,7 +1890,13 @@ static void blk_mq_del_queue_tag_set(struct request_queue *q)mutex_lock (&set->tag_list_lock);list_del_init (&q->tag_set_list); -blk_mq_update_tag_set_depth (set); + +if (set->tag_list.next == set->tag_list.prev) { +/* just transitioned to unshared */ + set->flags &= ~BLK_MQ_F_TAG_SHARED ; +/* update existing queue */ +blk_mq_update_tag_set_depth (set, false); + }mutex_unlock (&set->tag_list_lock); }
@@ -1902,7 +1907,24 @@ static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,mutex_lock (&set->tag_list_lock);list_add_tail (&q->tag_set_list, &set->tag_list); -blk_mq_update_tag_set_depth (set); + +if (set->tag_list.next != set->tag_list.prev) { +/* +* Only update the tag set state if the state has +* actually changed. +*/ +if (!(set->flags &BLK_MQ_F_TAG_SHARED )) { +/* just transitioned to shared tags */ + set->flags |=BLK_MQ_F_TAG_SHARED ; +blk_mq_update_tag_set_depth (set, true); + }else { +/* ensure we didn't race with another addition */ +struct blk_mq_hw_ctx *hctx =queue_first_hw_ctx (q); +if ((hctx->flags &BLK_MQ_F_TAG_SHARED ) != +BLK_MQ_F_TAG_SHARED ) +queue_set_hctx_shared (q, true); + } + }mutex_unlock (&set->tag_list_lock); }
include/linux/blk-mq.h +3 -0
@@ -254,6 +254,9 @@ static inline void *blk_mq_rq_to_pdu(struct request *rq)for ((i) =0 ; (i) < (hctx)->nr_ctx && \ ({ ctx = (hctx)->ctxs[(i)]; 1; }); (i)++) +#define queue_first_hw_ctx (q) \ + (q)->queue_hw_ctx[0] +#define blk_ctx_sum (q, sum) \ ({ \ struct blk_mq_ctx *__x; \
Hi,
Does anyone have time to look into the patch? I've tested, it works
/Thanks.
Quoted 126 lines · Jeff Moyer
Hi,
Zhangqing Luo reported long boot times on a system with thousands of LUNs when scsi-mq was enabled. He narrowed the problem down to
blk_mq_add_queue_tag_set, where every queue is frozen in order to set theBLK_MQ_F_TAG_SHAREDflag. Each added device will freeze all queues added before it in sequence, which involves waiting for an RCU grace period for each one. We don't need to do this. After the second queue is added, only new queues need to be initialized with the shared tag. We can do that by percolating the flag up to theblk_mq_tag_set, since new hctxs already inherit the tag_set's flags. That's what the below patch does.The re-check in
blk_mq_add_queue_tag_setis done because we could have two queues racing to be the second queue added for the tag set. In such a case, it's possible that not all of the hctxs within the loser will be initialized. Because this initialization is done sequentially, we can simply check for theBLK_MQ_F_TAG_SHAREDflag in the first hctx.This problem was introduced by commit
0d2602ca30e41(blk-mq: improve support for shared tags maps).Reported-and-tested-by Jason Luo <[email protected]>Signed-off-by Jeff Moyer <[email protected]>block/blk-mq.c +38
-16@@ -1860,27 +1860,26 @@ static void blk_mq_map_swqueue(struct request_queue *q, } } -static void blk_mq_update_tag_set_depth (struct blk_mq_tag_set *set) +static void queue_set_hctx_shared (struct request_queue *q,bool shared) {struct blk_mq_hw_ctx *hctx; -struct request_queue *q; -bool shared;int i; -if (set->tag_list.next == set->tag_list.prev) - shared = false; -else - shared = true; +queue_for_each_hw_ctx (q, hctx, i) { +if (shared) + hctx->flags |=BLK_MQ_F_TAG_SHARED ; +else + hctx->flags &= ~BLK_MQ_F_TAG_SHARED ; + } +} + +static void blk_mq_update_tag_set_depth (struct blk_mq_tag_set *set,bool shared) +{ +struct request_queue *q;list_for_each_entry (q, &set->tag_list, tag_set_list) {blk_mq_freeze_queue (q); - -queue_for_each_hw_ctx (q, hctx, i) { -if (shared) - hctx->flags |=BLK_MQ_F_TAG_SHARED ; -else - hctx->flags &= ~BLK_MQ_F_TAG_SHARED ; - } +queue_set_hctx_shared (q, shared);blk_mq_unfreeze_queue (q); } }@@ -1891,7 +1890,13 @@ static void blk_mq_del_queue_tag_set(struct request_queue *q)mutex_lock (&set->tag_list_lock);list_del_init (&q->tag_set_list); -blk_mq_update_tag_set_depth (set); + +if (set->tag_list.next == set->tag_list.prev) { +/* just transitioned to unshared */ + set->flags &= ~BLK_MQ_F_TAG_SHARED ; +/* update existing queue */ +blk_mq_update_tag_set_depth (set, false); + }mutex_unlock (&set->tag_list_lock); }@@ -1902,7 +1907,24 @@ static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,mutex_lock (&set->tag_list_lock);list_add_tail (&q->tag_set_list, &set->tag_list); -blk_mq_update_tag_set_depth (set); + +if (set->tag_list.next != set->tag_list.prev) { +/* +* Only update the tag set state if the state has +* actually changed. +*/ +if (!(set->flags &BLK_MQ_F_TAG_SHARED )) { +/* just transitioned to shared tags */ + set->flags |=BLK_MQ_F_TAG_SHARED ; +blk_mq_update_tag_set_depth (set, true); + }else { +/* ensure we didn't race with another addition */ +struct blk_mq_hw_ctx *hctx =queue_first_hw_ctx (q); +if ((hctx->flags &BLK_MQ_F_TAG_SHARED ) != +BLK_MQ_F_TAG_SHARED ) +queue_set_hctx_shared (q, true); + } + }mutex_unlock (&set->tag_list_lock); }include/linux/blk-mq.h +3
-0@@ -254,6 +254,9 @@ static inline void *blk_mq_rq_to_pdu(struct request *rq)for ((i) =0 ; (i) < (hctx)->nr_ctx && \ ({ ctx = (hctx)->ctxs[(i)]; 1; }); (i)++) +#define queue_first_hw_ctx (q) \ + (q)->queue_hw_ctx[0] +#define blk_ctx_sum (q, sum) \ ({ \ struct blk_mq_ctx *__x; \
Quoted 12 lines · Jeff Moyer
Hi,
Zhangqing Luo reported long boot times on a system with thousands of LUNs when scsi-mq was enabled. He narrowed the problem down to
blk_mq_add_queue_tag_set, where every queue is frozen in order to set theBLK_MQ_F_TAG_SHAREDflag. Each added device will freeze all queues added before it in sequence, which involves waiting for an RCU grace period for each one. We don't need to do this. After the second queue is added, only new queues need to be initialized with the shared tag. We can do that by percolating the flag up to theblk_mq_tag_set, since new hctxs already inherit the tag_set's flags. That's what the below patch does.
Looks the idea is correct.
Quoted 3 lines
The re-check in
blk_mq_add_queue_tag_setis done because we could have two queues racing to be the second queue added for the tag set. In such
That isn't possible because of set->tag_list_lock.
Quoted 81 lines
a case, it's possible that not all of the hctxs within the loser will be initialized. Because this initialization is done sequentially, we can simply check for the
BLK_MQ_F_TAG_SHAREDflag in the first hctx.This problem was introduced by commit
0d2602ca30e41(blk-mq: improve support for shared tags maps).Reported-and-tested-by Jason Luo <[email protected]>Signed-off-by Jeff Moyer <[email protected]>block/blk-mq.c +26
-16@@ -1860,27 +1860,26 @@ static void blk_mq_map_swqueue(struct request_queue *q, } } -static void blk_mq_update_tag_set_depth (struct blk_mq_tag_set *set) +static void queue_set_hctx_shared (struct request_queue *q,bool shared) {struct blk_mq_hw_ctx *hctx; -struct request_queue *q; -bool shared;int i; -if (set->tag_list.next == set->tag_list.prev) - shared = false; -else - shared = true; +queue_for_each_hw_ctx (q, hctx, i) { +if (shared) + hctx->flags |=BLK_MQ_F_TAG_SHARED ; +else + hctx->flags &= ~BLK_MQ_F_TAG_SHARED ; + } +} + +static void blk_mq_update_tag_set_depth (struct blk_mq_tag_set *set,bool shared) +{ +struct request_queue *q;list_for_each_entry (q, &set->tag_list, tag_set_list) {blk_mq_freeze_queue (q); - -queue_for_each_hw_ctx (q, hctx, i) { -if (shared) - hctx->flags |=BLK_MQ_F_TAG_SHARED ; -else - hctx->flags &= ~BLK_MQ_F_TAG_SHARED ; - } +queue_set_hctx_shared (q, shared);blk_mq_unfreeze_queue (q); } }@@ -1891,7 +1890,13 @@ static void blk_mq_del_queue_tag_set(struct request_queue *q)mutex_lock (&set->tag_list_lock);list_del_init (&q->tag_set_list); -blk_mq_update_tag_set_depth (set); + +if (set->tag_list.next == set->tag_list.prev) { +/* just transitioned to unshared */ + set->flags &= ~BLK_MQ_F_TAG_SHARED ; +/* update existing queue */ +blk_mq_update_tag_set_depth (set, false); + }mutex_unlock (&set->tag_list_lock); }@@ -1902,7 +1907,24 @@ static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,mutex_lock (&set->tag_list_lock);list_add_tail (&q->tag_set_list, &set->tag_list); -blk_mq_update_tag_set_depth (set); + +if (set->tag_list.next != set->tag_list.prev) { +/* +* Only update the tag set state if the state has +* actually changed. +*/
block/blk-mq.c static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set, · line 1925
Quoted 10 lines
block/blk-mq.c +10
-0+if (!(set->flags &BLK_MQ_F_TAG_SHARED )) { +/* just transitioned to shared tags */ + set->flags |=BLK_MQ_F_TAG_SHARED ; +blk_mq_update_tag_set_depth (set, true); + }else { +/* ensure we didn't race with another addition */ +struct blk_mq_hw_ctx *hctx =queue_first_hw_ctx (q); +if ((hctx->flags &BLK_MQ_F_TAG_SHARED ) != +BLK_MQ_F_TAG_SHARED ) +queue_set_hctx_shared (q, true);
The above check isn't necessary because the queue can't be up now, so it is OK to just call 'queue_set_hctx_shared(q, true)' directly.
Quoted 19 lines
block/blk-mq.c +2
-0+ } + }mutex_unlock (&set->tag_list_lock); }include/linux/blk-mq.h +3
-0@@ -254,6 +254,9 @@ static inline void *blk_mq_rq_to_pdu(struct request *rq)for ((i) =0 ; (i) < (hctx)->nr_ctx && \ ({ ctx = (hctx)->ctxs[(i)]; 1; }); (i)++) +#define queue_first_hw_ctx (q) \ + (q)->queue_hw_ctx[0] +#define blk_ctx_sum (q, sum) \ ({ \ struct blk_mq_ctx *__x; \
Signature · 1 line
Ming Lei
Hi Ming,
Thanks for taking a look.
Quoted 14 lines · Ming Lei
Quoted 12 lines · Jeff Moyer
Hi,
Zhangqing Luo reported long boot times on a system with thousands of LUNs when scsi-mq was enabled. He narrowed the problem down to
blk_mq_add_queue_tag_set, where every queue is frozen in order to set theBLK_MQ_F_TAG_SHAREDflag. Each added device will freeze all queues added before it in sequence, which involves waiting for an RCU grace period for each one. We don't need to do this. After the second queue is added, only new queues need to be initialized with the shared tag. We can do that by percolating the flag up to theblk_mq_tag_set, since new hctxs already inherit the tag_set's flags. That's what the below patch does.
Quoted 7 lines
Looks the idea is correct.
Quoted 3 lines
The re-check in
blk_mq_add_queue_tag_setis done because we could have two queues racing to be the second queue added for the tag set. In suchThat isn't possible because of
set->tag_list_lock.
Yes, it is possible. blk_mq_init_allocated_queue calls blk_mq_init_hw_queues without the set->tag_list_lock. Consider a single queue is present for the tag_set, and thus the BLK_MQ_F_TAG_SHARED flag is not set. Then, two more async scan events are running on separate CPUs. Each will go through this loop in blk_mq_init_hw_queues:
queue_for_each_hw_ctx(q, hctx, i) {
if (blk_mq_init_hctx(q, set, hctx, i))
break;
}and blk_mq_init_hctx copies the flags:
hctx->flags = set->flags;
At this point, neither queue's hctxs have the shared flag set. Next, both will race to get the tag_list_lock for the tag_set inside of blk_mq_add_queue_tag_set. Only one will win and mark the initial queue's hctx's as shared (as well as its own). Then, when the second queue gets the lock, it will find that the shared tag is already set, and assume that it doesn't have to do anything. But, because its hctx->flags were copied before the BLK_MQ_F_TAG_SHARED flag was set, it now will operate as if it's the only queue in the tag set.
So, yes, there is a race, and that extra check is necessary. Hopefully I've explained it well enough. If not, let me know and I'll try again.
Cheers,
Jeff
Quoted 115 lines
Quoted 82 lines
a case, it's possible that not all of the hctxs within the loser will be initialized. Because this initialization is done sequentially, we can simply check for the
BLK_MQ_F_TAG_SHAREDflag in the first hctx.This problem was introduced by commit
0d2602ca30e41(blk-mq: improve support for shared tags maps).Reported-and-tested-by Jason Luo <[email protected]>Signed-off-by Jeff Moyer <[email protected]>block/blk-mq.c +26
-16@@ -1860,27 +1860,26 @@ static void blk_mq_map_swqueue(struct request_queue *q, } } -static void blk_mq_update_tag_set_depth (struct blk_mq_tag_set *set) +static void queue_set_hctx_shared (struct request_queue *q,bool shared) {struct blk_mq_hw_ctx *hctx; -struct request_queue *q; -bool shared;int i; -if (set->tag_list.next == set->tag_list.prev) - shared = false; -else - shared = true; +queue_for_each_hw_ctx (q, hctx, i) { +if (shared) + hctx->flags |=BLK_MQ_F_TAG_SHARED ; +else + hctx->flags &= ~BLK_MQ_F_TAG_SHARED ; + } +} + +static void blk_mq_update_tag_set_depth (struct blk_mq_tag_set *set,bool shared) +{ +struct request_queue *q;list_for_each_entry (q, &set->tag_list, tag_set_list) {blk_mq_freeze_queue (q); - -queue_for_each_hw_ctx (q, hctx, i) { -if (shared) - hctx->flags |=BLK_MQ_F_TAG_SHARED ; -else - hctx->flags &= ~BLK_MQ_F_TAG_SHARED ; - } +queue_set_hctx_shared (q, shared);blk_mq_unfreeze_queue (q); } }@@ -1891,7 +1890,13 @@ static void blk_mq_del_queue_tag_set(struct request_queue *q)mutex_lock (&set->tag_list_lock);list_del_init (&q->tag_set_list); -blk_mq_update_tag_set_depth (set); + +if (set->tag_list.next == set->tag_list.prev) { +/* just transitioned to unshared */ + set->flags &= ~BLK_MQ_F_TAG_SHARED ; +/* update existing queue */ +blk_mq_update_tag_set_depth (set, false); + }mutex_unlock (&set->tag_list_lock); }@@ -1902,7 +1907,24 @@ static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,mutex_lock (&set->tag_list_lock);list_add_tail (&q->tag_set_list, &set->tag_list); -blk_mq_update_tag_set_depth (set); + +if (set->tag_list.next != set->tag_list.prev) { +/* +* Only update the tag set state if the state has +* actually changed. +*/ block/blk-mq.c static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set, · line 1925
Quoted 10 lines
block/blk-mq.c +10
-0+if (!(set->flags &BLK_MQ_F_TAG_SHARED )) { +/* just transitioned to shared tags */ + set->flags |=BLK_MQ_F_TAG_SHARED ; +blk_mq_update_tag_set_depth (set, true); + }else { +/* ensure we didn't race with another addition */ +struct blk_mq_hw_ctx *hctx =queue_first_hw_ctx (q); +if ((hctx->flags &BLK_MQ_F_TAG_SHARED ) != +BLK_MQ_F_TAG_SHARED ) +queue_set_hctx_shared (q, true);The above check isn't necessary because the queue can't be up now, so it is OK to just call '
queue_set_hctx_shared(q, true)' directly.Quoted 19 lines
block/blk-mq.c +2
-0+ } + }mutex_unlock (&set->tag_list_lock); }include/linux/blk-mq.h +3
-0@@ -254,6 +254,9 @@ static inline void *blk_mq_rq_to_pdu(struct request *rq)for ((i) =0 ; (i) < (hctx)->nr_ctx && \ ({ ctx = (hctx)->ctxs[(i)]; 1; }); (i)++) +#define queue_first_hw_ctx (q) \ + (q)->queue_hw_ctx[0] +#define blk_ctx_sum (q, sum) \ ({ \ struct blk_mq_ctx *__x; \
Quoted 34 lines · Jeff Moyer
Hi Ming,
Thanks for taking a look.
Quoted 14 lines · Ming Lei
Quoted 12 lines · Jeff Moyer
Hi,
Zhangqing Luo reported long boot times on a system with thousands of LUNs when scsi-mq was enabled. He narrowed the problem down to
blk_mq_add_queue_tag_set, where every queue is frozen in order to set theBLK_MQ_F_TAG_SHAREDflag. Each added device will freeze all queues added before it in sequence, which involves waiting for an RCU grace period for each one. We don't need to do this. After the second queue is added, only new queues need to be initialized with the shared tag. We can do that by percolating the flag up to theblk_mq_tag_set, since new hctxs already inherit the tag_set's flags. That's what the below patch does.Quoted 7 lines
Looks the idea is correct.
Quoted 3 lines
The re-check in
blk_mq_add_queue_tag_setis done because we could have two queues racing to be the second queue added for the tag set. In suchThat isn't possible because of
set->tag_list_lock.Yes, it is possible.
blk_mq_init_allocated_queuecallsblk_mq_init_hw_queueswithout theset->tag_list_lock. Consider a single queue is present for thetag_set, and thus theBLK_MQ_F_TAG_SHAREDflag is not set. Then, two more async scan events are running on separate CPUs. Each will go through this loop inblk_mq_init_hw_queues:
Quoted 7 lines
queue_for_each_hw_ctx(q, hctx, i) { if (blk_mq_init_hctx(q, set, hctx, i)) break; }and
blk_mq_init_hctxcopies the flags:hctx->flags = set->flags;
Looks we should have cleared the TAG_SHARED flag during blk_mq_init_hctx() and just let blk_mq_update_tag_set_depth() deal with that, then the race can be avoided.
Quoted 7 lines
At this point, neither queue's hctxs have the shared flag set. Next, both will race to get the
tag_list_lockfor thetag_setinside ofblk_mq_add_queue_tag_set. Only one will win and mark the initial queue's hctx's as shared (as well as its own). Then, when the second queue gets the lock, it will find that the shared tag is already set, and assume that it doesn't have to do anything. But, because its
As I suggested, we can set it always in case that TAG_SHARED is set in set->flags because we know the queue isn't ready yet at that time.
Quoted 2 lines
hctx->flagswere copied before theBLK_MQ_F_TAG_SHAREDflag was set, it now will operate as if it's the only queue in the tag set.
The shared flag should be set here because the flag has been set in set->flags, then no such issue, is there?
Quoted 122 lines
So, yes, there is a race, and that extra check is necessary. Hopefully I've explained it well enough. If not, let me know and I'll try again.
Cheers,
JeffQuoted 115 lines
Quoted 82 lines
a case, it's possible that not all of the hctxs within the loser will be initialized. Because this initialization is done sequentially, we can simply check for the
BLK_MQ_F_TAG_SHAREDflag in the first hctx.This problem was introduced by commit
0d2602ca30e41(blk-mq: improve support for shared tags maps).Reported-and-tested-by Jason Luo <[email protected]>Signed-off-by Jeff Moyer <[email protected]>block/blk-mq.c +26
-16@@ -1860,27 +1860,26 @@ static void blk_mq_map_swqueue(struct request_queue *q, } } -static void blk_mq_update_tag_set_depth (struct blk_mq_tag_set *set) +static void queue_set_hctx_shared (struct request_queue *q,bool shared) {struct blk_mq_hw_ctx *hctx; -struct request_queue *q; -bool shared;int i; -if (set->tag_list.next == set->tag_list.prev) - shared = false; -else - shared = true; +queue_for_each_hw_ctx (q, hctx, i) { +if (shared) + hctx->flags |=BLK_MQ_F_TAG_SHARED ; +else + hctx->flags &= ~BLK_MQ_F_TAG_SHARED ; + } +} + +static void blk_mq_update_tag_set_depth (struct blk_mq_tag_set *set,bool shared) +{ +struct request_queue *q;list_for_each_entry (q, &set->tag_list, tag_set_list) {blk_mq_freeze_queue (q); - -queue_for_each_hw_ctx (q, hctx, i) { -if (shared) - hctx->flags |=BLK_MQ_F_TAG_SHARED ; -else - hctx->flags &= ~BLK_MQ_F_TAG_SHARED ; - } +queue_set_hctx_shared (q, shared);blk_mq_unfreeze_queue (q); } }@@ -1891,7 +1890,13 @@ static void blk_mq_del_queue_tag_set(struct request_queue *q)mutex_lock (&set->tag_list_lock);list_del_init (&q->tag_set_list); -blk_mq_update_tag_set_depth (set); + +if (set->tag_list.next == set->tag_list.prev) { +/* just transitioned to unshared */ + set->flags &= ~BLK_MQ_F_TAG_SHARED ; +/* update existing queue */ +blk_mq_update_tag_set_depth (set, false); + }mutex_unlock (&set->tag_list_lock); }@@ -1902,7 +1907,24 @@ static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,mutex_lock (&set->tag_list_lock);list_add_tail (&q->tag_set_list, &set->tag_list); -blk_mq_update_tag_set_depth (set); + +if (set->tag_list.next != set->tag_list.prev) { +/* +* Only update the tag set state if the state has +* actually changed. +*/ block/blk-mq.c static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set, · line 1925
Quoted 10 lines
block/blk-mq.c +10
-0+if (!(set->flags &BLK_MQ_F_TAG_SHARED )) { +/* just transitioned to shared tags */ + set->flags |=BLK_MQ_F_TAG_SHARED ; +blk_mq_update_tag_set_depth (set, true); + }else { +/* ensure we didn't race with another addition */ +struct blk_mq_hw_ctx *hctx =queue_first_hw_ctx (q); +if ((hctx->flags &BLK_MQ_F_TAG_SHARED ) != +BLK_MQ_F_TAG_SHARED ) +queue_set_hctx_shared (q, true);The above check isn't necessary because the queue can't be up now, so it is OK to just call '
queue_set_hctx_shared(q, true)' directly.Quoted 19 lines
block/blk-mq.c +2
-0+ } + }mutex_unlock (&set->tag_list_lock); }include/linux/blk-mq.h +3
-0@@ -254,6 +254,9 @@ static inline void *blk_mq_rq_to_pdu(struct request *rq)for ((i) =0 ; (i) < (hctx)->nr_ctx && \ ({ ctx = (hctx)->ctxs[(i)]; 1; }); (i)++) +#define queue_first_hw_ctx (q) \ + (q)->queue_hw_ctx[0] +#define blk_ctx_sum (q, sum) \ ({ \ struct blk_mq_ctx *__x; \
Signature · 1 line
Ming Lei
Quoted 3 lines · Ming Lei
Looks we should have cleared the
TAG_SHAREDflag duringblk_mq_init_hctx()and just letblk_mq_update_tag_set_depth()deal with that, then the race can be avoided.
The whole point of the patch set is to propagate the flag up to the tag set so that we can avoid iterating all hctxs in all queues.
Quoted 10 lines
Quoted 6 lines
At this point, neither queue's hctxs have the shared flag set. Next, both will race to get the
tag_list_lockfor thetag_setinside ofblk_mq_add_queue_tag_set. Only one will win and mark the initial queue's hctx's as shared (as well as its own). Then, when the second queue gets the lock, it will find that the shared tag is already set, and assume that it doesn't have to do anything. But, because itsAs I suggested, we can set it always in case that
TAG_SHAREDis set inset->flagsbecause we know the queue isn't ready yet at that time.
I see. You are suggesting that I just get rid of the conditional. We could do that, but you will get the exact same result as what I posted. I'm not sure why you would prefer that over the explicit check. With the patch I posted, we can avoid walking the list of hctxs a second time.
Anyway, here's a patch that I think implements your suggestion. I prefer the original, but this should achieve the same exact result. Let me know if I've misunderstood.
Cheers,
Jeff
block/blk-mq.c +35 -16
@@ -1860,27 +1860,26 @@ static void blk_mq_map_swqueue(struct request_queue *q, } } -static void blk_mq_update_tag_set_depth (struct blk_mq_tag_set *set) +static void queue_set_hctx_shared (struct request_queue *q,bool shared) {struct blk_mq_hw_ctx *hctx; -struct request_queue *q; -bool shared;int i; -if (set->tag_list.next == set->tag_list.prev) - shared = false; -else - shared = true; +queue_for_each_hw_ctx (q, hctx, i) { +if (shared) + hctx->flags |=BLK_MQ_F_TAG_SHARED ; +else + hctx->flags &= ~BLK_MQ_F_TAG_SHARED ; + } +} + +static void blk_mq_update_tag_set_depth (struct blk_mq_tag_set *set,bool shared) +{ +struct request_queue *q;list_for_each_entry (q, &set->tag_list, tag_set_list) {blk_mq_freeze_queue (q); - -queue_for_each_hw_ctx (q, hctx, i) { -if (shared) - hctx->flags |=BLK_MQ_F_TAG_SHARED ; -else - hctx->flags &= ~BLK_MQ_F_TAG_SHARED ; - } +queue_set_hctx_shared (q, shared);blk_mq_unfreeze_queue (q); } }
@@ -1891,7 +1890,13 @@ static void blk_mq_del_queue_tag_set(struct request_queue *q)mutex_lock (&set->tag_list_lock);list_del_init (&q->tag_set_list); -blk_mq_update_tag_set_depth (set); + +if (set->tag_list.next == set->tag_list.prev) { +/* just transitioned to unshared */ + set->flags &= ~BLK_MQ_F_TAG_SHARED ; +/* update existing queue */ +blk_mq_update_tag_set_depth (set, false); + }mutex_unlock (&set->tag_list_lock); }
@@ -1902,7 +1907,21 @@ static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,mutex_lock (&set->tag_list_lock);list_add_tail (&q->tag_set_list, &set->tag_list); -blk_mq_update_tag_set_depth (set); + +if (set->tag_list.next != set->tag_list.prev) { +/* +* Only update the tag set state if the state has +* actually changed. +*/ +if (!(set->flags &BLK_MQ_F_TAG_SHARED )) { +/* just transitioned to shared tags */ + set->flags |=BLK_MQ_F_TAG_SHARED ; +blk_mq_update_tag_set_depth (set, true); + }else { +/* ensure we didn't race with another addition */ +queue_set_hctx_shared (q, true); + } + }mutex_unlock (&set->tag_list_lock); }
Quoted 8 lines · Jeff Moyer
Quoted 3 lines · Ming Lei
Looks we should have cleared the
TAG_SHAREDflag duringblk_mq_init_hctx()and just letblk_mq_update_tag_set_depth()deal with that, then the race can be avoided.The whole point of the patch set is to propagate the flag up to the tag set so that we can avoid iterating all hctxs in all queues.
Yes, the idea is correct, and the difference is that I don't see there isn't any race, for which your original patch tryed to address.
Quoted 18 lines
Quoted 11 lines
Quoted 7 lines
At this point, neither queue's hctxs have the shared flag set. Next, both will race to get the
tag_list_lockfor thetag_setinside ofblk_mq_add_queue_tag_set. Only one will win and mark the initial queue's hctx's as shared (as well as its own). Then, when the second queue gets the lock, it will find that the shared tag is already set, and assume that it doesn't have to do anything. But, because itsAs I suggested, we can set it always in case that
TAG_SHAREDis set inset->flagsbecause we know the queue isn't ready yet at that time.I see. You are suggesting that I just get rid of the conditional. We could do that, but you will get the exact same result as what I posted. I'm not sure why you would prefer that over the explicit check. With the patch I posted, we can avoid walking the list of hctxs a second time.
Quoted 3 lines
Anyway, here's a patch that I think implements your suggestion. I prefer the original, but this should achieve the same exact result. Let me know if I've misunderstood.
Yes, this one is better, and the big difference is that there isn't so called race, and the new one is more clean.
block/blk-mq.c static void blk_mq_map_swqueue(struct request_queue *q, · line 1861
Quoted 11 lines
Cheers,
Jeffblock/blk-mq.c +0
-0@@ -1860,27 +1860,26 @@ static void blk_mq_map_swqueue(struct request_queue *q, } }
The following change should be put into blk_mq_init_hctx() too.
hctx->flags = set->flags & ~BLK_MQ_F_TAG_SHARED;
and the idea is that we always figure out the shared flag in following blk_mq_update_tag_set_depth().
Quoted 60 lines
block/blk-mq.c +22
-16-static void blk_mq_update_tag_set_depth (struct blk_mq_tag_set *set) +static void queue_set_hctx_shared (struct request_queue *q,bool shared) {struct blk_mq_hw_ctx *hctx; -struct request_queue *q; -bool shared;int i; -if (set->tag_list.next == set->tag_list.prev) - shared = false; -else - shared = true; +queue_for_each_hw_ctx (q, hctx, i) { +if (shared) + hctx->flags |=BLK_MQ_F_TAG_SHARED ; +else + hctx->flags &= ~BLK_MQ_F_TAG_SHARED ; + } +} + +static void blk_mq_update_tag_set_depth (struct blk_mq_tag_set *set,bool shared) +{ +struct request_queue *q;list_for_each_entry (q, &set->tag_list, tag_set_list) {blk_mq_freeze_queue (q); - -queue_for_each_hw_ctx (q, hctx, i) { -if (shared) - hctx->flags |=BLK_MQ_F_TAG_SHARED ; -else - hctx->flags &= ~BLK_MQ_F_TAG_SHARED ; - } +queue_set_hctx_shared (q, shared);blk_mq_unfreeze_queue (q); } }@@ -1891,7 +1890,13 @@ static void blk_mq_del_queue_tag_set(struct request_queue *q)mutex_lock (&set->tag_list_lock);list_del_init (&q->tag_set_list); -blk_mq_update_tag_set_depth (set); + +if (set->tag_list.next == set->tag_list.prev) { +/* just transitioned to unshared */ + set->flags &= ~BLK_MQ_F_TAG_SHARED ; +/* update existing queue */ +blk_mq_update_tag_set_depth (set, false); + }mutex_unlock (&set->tag_list_lock); }@@ -1902,7 +1907,21 @@ static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,mutex_lock (&set->tag_list_lock);list_add_tail (&q->tag_set_list, &set->tag_list); -blk_mq_update_tag_set_depth (set); + +if (set->tag_list.next != set->tag_list.prev) {
block/blk-mq.c static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set, · line 1921
Quoted 10 lines
block/blk-mq.c +10
-0+/* +* Only update the tag set state if the state has +* actually changed. +*/ +if (!(set->flags &BLK_MQ_F_TAG_SHARED )) { +/* just transitioned to shared tags */ + set->flags |=BLK_MQ_F_TAG_SHARED ; +blk_mq_update_tag_set_depth (set, true); + }else { +/* ensure we didn't race with another addition */
The above comment should be removed, and I don't see there is any race now.
block/blk-mq.c static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set, · line 1927
Quoted 6 lines
block/blk-mq.c +3
-0+queue_set_hctx_shared (q, true); + } + }mutex_unlock (&set->tag_list_lock); }
This one is a bit more clean than original one, and the trick of queue_first_hw_ctx() can be avoided.
Signature · 1 line
Ming Lei
Quoted 6 lines · Ming Lei
The following change should be put into
blk_mq_init_hctx()too.hctx->flags = set->flags & ~BLK_MQ_F_TAG_SHARED;and the idea is that we always figure out the shared flag in following
blk_mq_update_tag_set_depth().
OK, this is the part I was missing from your earlier mails. That makes sense, I'll update the patch and repost.
Thanks, Ming!
-Jeff