Reduce the amount of memory required for request queues
v2 cover Reduce the amount of memory required for request queues
Hi Jens,
This patch series reduces the amount of memory required for request queues and also includes a patche that is the result of reviewing code related to the modified code. Please consider these patches for kernel version v5.5.
Thanks,
Bart.
Changes compared to v1:
- Dropped the blk_poll() patch from this series.
Bart Van Assche (3):
block: Remove the synchronize_rcu() call from
__blk_mq_update_nr_hw_queues()
block: Reduce the amount of memory required per request queue
block: Reduce the amount of memory used for tag sets
block/blk-mq.c | 75 +++++++++++++++++++++++++++++++------------------- 1 file changed, 47 insertions(+), 28 deletions(-)
Signature · 1 line
2.24.0.rc0.303.g954a862665-goog
PATCH v2 1/3 block: Remove the synchronize_rcu() call from __blk_mq_update_nr_hw_queues()
Since the blk_mq_{,un}freeze_queue() calls in __blk_mq_update_nr_hw_queues() already serialize __blk_mq_update_nr_hw_queues() against blk_mq_queue_tag_busy_iter(), the synchronize_rcu() call in __blk_mq_update_nr_hw_queues() is not necessary. Hence remove it.
Note: the synchronize_rcu() call in __blk_mq_update_nr_hw_queues() was introduced by commit f5bbbbe4d635 ("blk-mq: sync the update nr_hw_queues with blk_mq_queue_tag_busy_iter"). Commit 530ca2c9bd69 ("blk-mq: Allow blocking queue tag iter callbacks") removed the rcu_read_{,un}lock() calls that correspond to the synchronize_rcu() call in __blk_mq_update_nr_hw_queues().
--- block/blk-mq.c | 4 ---- 1 file changed, 4 deletions(-)
block/blk-mq.c +0 -4
@@ -3242,10 +3242,6 @@ static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set,list_for_each_entry (q, &set->tag_list, tag_set_list)blk_mq_freeze_queue (q); -/* -* Sync with blk_mq_queue_tag_busy_iter. -*/ -synchronize_rcu ();/* * Switch IO scheduler to 'none', cleaning up the data associated * with the previous scheduler. We will switch back once we are done
Signature · 1 line
2.24.0.rc0.303.g954a862665-goog
PATCH v2 2/3 block: Reduce the amount of memory required per request queue
In reply to Bart Van AsscheInstead of always allocating at least nr_cpu_ids hardware queues per request queue, reallocate q->queue_hw_ctx if it has to grow. This patch improves behavior that was introduced by commit 868f2f0b7206 ("blk-mq: dynamic h/w context count").
--- block/blk-mq.c | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-)
block/blk-mq.c +17 -7
@@ -2761,6 +2761,23 @@ static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set,int i, j, end;struct blk_mq_hw_ctx **hctxs = q->queue_hw_ctx; +if (q->nr_hw_queues < set->nr_hw_queues) { +struct blk_mq_hw_ctx **new_hctxs; + + new_hctxs =kcalloc_node (set->nr_hw_queues, +sizeof (*new_hctxs),GFP_KERNEL , + set->numa_node); +if (!new_hctxs) +return ; +if (hctxs) +memcpy (new_hctxs, hctxs, q->nr_hw_queues * +sizeof (*hctxs)); + q->queue_hw_ctx = new_hctxs; + q->nr_hw_queues = set->nr_hw_queues; +kfree (hctxs); + hctxs = new_hctxs; + } +/* protect against switching io scheduler */ mutex_lock (&q->sysfs_lock);for (i =0 ; i < set->nr_hw_queues; i++) {
@@ -2848,12 +2865,6 @@ struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,/* init q->mq_kobj and sw queues' kobjects */ blk_mq_sysfs_init (q); - q->queue_hw_ctx =kcalloc_node (nr_hw_queues (set), -sizeof (*(q->queue_hw_ctx)),GFP_KERNEL , - set->numa_node); -if (!q->queue_hw_ctx) - goto err_sys_init; -INIT_LIST_HEAD (&q->unused_hctx_list);spin_lock_init (&q->unused_hctx_lock);
@@ -2901,7 +2912,6 @@ struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set, err_hctxs:kfree (q->queue_hw_ctx); q->nr_hw_queues =0 ; -err_sys_init:blk_mq_sysfs_deinit (q); err_poll:blk_stat_free_callback (q->poll_cb);
Signature · 1 line
2.24.0.rc0.303.g954a862665-goog
PATCH v2 3/3 block: Reduce the amount of memory used for tag sets
In reply to Bart Van AsscheInstead of allocating an array of size nr_cpu_ids for set->tags, allocate an array of size set->nr_hw_queues. This patch improves behavior that was introduced by commit 868f2f0b7206 ("blk-mq: dynamic h/w context count").
Reallocating tag sets from inside __blk_mq_update_nr_hw_queues() is safe because:
- All request queues that share the tag sets are frozen before the tag sets are reallocated.
- blk_mq_queue_tag_busy_iter() holds q->q_usage_counter while active and hence is serialized against __blk_mq_update_nr_hw_queues().
--- block/blk-mq.c | 47 ++++++++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 17 deletions(-)
block/blk-mq.c +30 -17
@@ -2833,19 +2833,6 @@ static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set,mutex_unlock (&q->sysfs_lock); } -/* -* Maximum number of hardware queues we support. For single sets, we'll never -* have more than the CPUs (software queues). For multiple sets, the tag_set -* user may have set ->nr_hw_queues larger. -*/ -static unsigned int nr_hw_queues (struct blk_mq_tag_set *set) -{ -if (set->nr_maps ==1 ) -return nr_cpu_ids; - -return max (set->nr_hw_queues, nr_cpu_ids); -} -struct request_queue *blk_mq_init_allocated_queue (struct blk_mq_tag_set *set,struct request_queue *q,bool elevator_init)
@@ -3012,6 +2999,29 @@ static int blk_mq_update_queue_map(struct blk_mq_tag_set *set) } } +static int blk_mq_realloc_tag_set_tags (struct blk_mq_tag_set *set, +int cur_nr_hw_queues,int new_nr_hw_queues) +{ +struct blk_mq_tags **new_tags; + +if (cur_nr_hw_queues >= new_nr_hw_queues) +return 0 ; + + new_tags =kcalloc_node (new_nr_hw_queues,sizeof (struct blk_mq_tags *), +GFP_KERNEL , set->numa_node); +if (!new_tags) +return -ENOMEM ; + +if (set->tags) +memcpy (new_tags, set->tags, cur_nr_hw_queues * +sizeof (*set->tags)); +kfree (set->tags); + set->tags = new_tags; + set->nr_hw_queues = new_nr_hw_queues; + +return 0 ; +} +/* * Alloc a tag set to be associated with one or more request queues. * May fail with EINVAL for various error conditions. May adjust the
@@ -3065,9 +3075,7 @@ int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set)if (set->nr_maps ==1 && set->nr_hw_queues > nr_cpu_ids) set->nr_hw_queues = nr_cpu_ids; - set->tags =kcalloc_node (nr_hw_queues (set),sizeof (struct blk_mq_tags *), -GFP_KERNEL , set->numa_node); -if (!set->tags) +if (blk_mq_realloc_tag_set_tags (set,0 , set->nr_hw_queues) <0 )return -ENOMEM ; ret = -ENOMEM ;
@@ -3108,7 +3116,7 @@ void blk_mq_free_tag_set(struct blk_mq_tag_set *set) {int i, j; -for (i =0 ; i <nr_hw_queues (set); i++) +for (i =0 ; i < set->nr_hw_queues; i++)blk_mq_free_map_and_requests (set, i);for (j =0 ; j < set->nr_maps; j++) {
@@ -3266,6 +3274,10 @@ static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set,blk_mq_sysfs_unregister (q); } +if (blk_mq_realloc_tag_set_tags (set, set->nr_hw_queues, nr_hw_queues) < +0 ) + goto reregister; + prev_nr_hw_queues = set->nr_hw_queues; set->nr_hw_queues = nr_hw_queues;blk_mq_update_queue_map (set);
@@ -3282,6 +3294,7 @@ static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set,blk_mq_map_swqueue (q); } +reregister:list_for_each_entry (q, &set->tag_list, tag_set_list) {blk_mq_sysfs_register (q);blk_mq_debugfs_register_hctxs (q);
Signature · 1 line
2.24.0.rc0.303.g954a862665-goog
Quoted 5 lines · Bart Van Assche
Hi Jens,
This patch series reduces the amount of memory required for request queues and also includes a patche that is the result of reviewing code related to the modified code. Please consider these patches for kernel version v5.5.
Applied, thanks Bart.
Signature · 1 line
Jens Axboe