mm/vmalloc: Use dedicated unbound workqueue for vmap purge/drain
Subsystems (2)
Uladzislau Rezki (Sony)
maintainerPATCH 1/1 mm/vmalloc: Use dedicated unbound workqueue for vmap purge/drain
The drain_vmap_area_work() function can take >10ms to complete when there are many accumulated vmap areas in a system with a high CPU count, causing workqueue watchdog warnings when run via schedule_work():
[ 2069.796205] workqueue: drain_vmap_area_work hogged CPU for >10000us 4 times, consider switching to WQ_UNBOUND [ 2192.823225] workqueue: drain_vmap_area_work hogged CPU for >10000us 5 times, consider switching to WQ_UNBOUND
Switch to a dedicated WQ_UNBOUND workqueue to allow the scheduler to run this background task on any available CPU, improving responsiveness. Use WQ_MEM_RECLAIM to ensure forward progress under memory pressure.
Also simplify purge helper scheduling by removing cpumask-based iteration in favour to iterating directly over vmap nodes with pending work.
--- mm/vmalloc.c | 63 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 39 insertions(+), 24 deletions(-)
mm/vmalloc.c +39 -24
@@ -949,6 +949,7 @@ static struct vmap_node {struct list_head purge_list;struct work_struct purge_work;unsigned long nr_purged; +bool work_queued; } single;/*
@@ -1067,6 +1068,7 @@ static void reclaim_and_purge_vmap_areas(void);static BLOCKING_NOTIFIER_HEAD (vmap_notify_list);static void drain_vmap_area_work (struct work_struct *work); static DECLARE_WORK(drain_vmap_work, drain_vmap_area_work); +static struct workqueue_struct *drain_vmap_wq; static __cacheline_aligned_in_smp atomic_long_t nr_vmalloc_pages; static __cacheline_aligned_in_smp atomic_long_t vmap_lazy_nr;
@@ -2335,6 +2337,19 @@ static void purge_vmap_node(struct work_struct *work)reclaim_list_global (&local_list); } +static bool +schedule_drain_vmap_work (struct work_struct *work) +{ +struct workqueue_struct *wq =READ_ONCE (drain_vmap_wq); + +if (wq) { +queue_work (wq, work); +return true; + } + +return false; +} +/* * Purges all lazily-freed vmap areas. */
@@ -2342,19 +2357,12 @@ static bool __purge_vmap_area_lazy(unsigned long start, unsigned long end, bool full_pool_decay) {unsigned long nr_purged_areas =0 ; +unsigned int nr_purge_nodes =0 ;unsigned int nr_purge_helpers; -static cpumask_t purge_nodes; -unsigned int nr_purge_nodes;struct vmap_node *vn; -int i;lockdep_assert_held (&vmap_purge_lock); -/* -* Use cpumask to mark which node has to be processed. -*/ - purge_nodes =CPU_MASK_NONE ; -for_each_vmap_node (vn) {INIT_LIST_HEAD (&vn->purge_list); vn->skip_populate = full_pool_decay;
@@ -2374,10 +2382,9 @@ static bool __purge_vmap_area_lazy(unsigned long start, unsigned long end, end =max (end,list_last_entry (&vn->purge_list, struct vmap_area, list)->va_end); -cpumask_set_cpu (node_to_id (vn), &purge_nodes); + nr_purge_nodes++; } - nr_purge_nodes =cpumask_weight (&purge_nodes);if (nr_purge_nodes >0 ) {flush_tlb_kernel_range (start, end);
@@ -2385,29 +2392,25 @@ static bool __purge_vmap_area_lazy(unsigned long start, unsigned long end, nr_purge_helpers =atomic_long_read (&vmap_lazy_nr) /lazy_max_pages (); nr_purge_helpers =clamp (nr_purge_helpers,1U , nr_purge_nodes) -1 ; -for_each_cpu (i, &purge_nodes) { - vn = &vmap_nodes[i]; +for_each_vmap_node (vn) { + vn->work_queued = false; + +if (list_empty (&vn->purge_list)) +continue ;if (nr_purge_helpers >0 ) {INIT_WORK (&vn->purge_work, purge_vmap_node); - -if (cpumask_test_cpu (i, cpu_online_mask)) -schedule_work_on (i, &vn->purge_work); -else -schedule_work (&vn->purge_work); - + vn->work_queued =schedule_drain_vmap_work (&vn->purge_work); nr_purge_helpers--; }else { - vn->purge_work.func =NULL ;purge_vmap_node (&vn->purge_work); nr_purged_areas += vn->nr_purged; } } -for_each_cpu (i, &purge_nodes) { - vn = &vmap_nodes[i]; - -if (vn->purge_work.func) { +/* Wait for completion if queued any. */ +for_each_vmap_node (vn) { +if (vn->work_queued) {flush_work (&vn->purge_work); nr_purged_areas += vn->nr_purged; }
@@ -2471,7 +2474,7 @@ static void free_vmap_area_noflush(struct vmap_area *va)/* After this point, we may free va at any time */ if (unlikely (nr_lazy > nr_lazy_max)) -schedule_work (&drain_vmap_work); +schedule_drain_vmap_work (&drain_vmap_work); }/*
@@ -5483,3 +5486,15 @@ void __init vmalloc_init(void) vmap_node_shrinker->scan_objects = vmap_node_shrink_scan;shrinker_register (vmap_node_shrinker); } + +static int __init vmalloc_init_workqueue (void ) +{ +struct workqueue_struct *wq; + + wq =alloc_workqueue ("vmap_drain" ,WQ_UNBOUND |WQ_MEM_RECLAIM ,0 ); +WARN_ON (wq ==NULL ); +WRITE_ONCE (drain_vmap_wq, wq); + +return 0 ; +} +early_initcall (vmalloc_init_workqueue);
Signature · 1 line
2.47.3
Uladzislau Rezki
maintainerQuoted 163 lines · Uladzislau Rezki (Sony)
The
drain_vmap_area_work()function can take >10ms to complete when there are many accumulated vmap areas in a system with a high CPU count, causing workqueue watchdog warnings when run viaschedule_work():[ 2069.796205] workqueue: drain_vmap_area_work hogged CPU for >10000us 4 times, consider switching to WQ_UNBOUND [ 2192.823225] workqueue: drain_vmap_area_work hogged CPU for >10000us 5 times, consider switching to WQ_UNBOUNDSwitch to a dedicated
WQ_UNBOUNDworkqueue to allow the scheduler to run this background task on any available CPU, improving responsiveness. UseWQ_MEM_RECLAIMto ensure forward progress under memory pressure.Also simplify purge helper scheduling by removing cpumask-based iteration in favour to iterating directly over vmap nodes with pending work.
Cc lirongqing <[email protected]>Signed-off-by Uladzislau Rezki (Sony) <[email protected]>--- mm/vmalloc.c | 63 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 39 insertions(+), 24 deletions(-)mm/vmalloc.c +39
-24@@ -949,6 +949,7 @@ static struct vmap_node {struct list_head purge_list;struct work_struct purge_work;unsigned long nr_purged; +bool work_queued; } single;/* @@ -1067,6 +1068,7 @@ static void reclaim_and_purge_vmap_areas(void);static BLOCKING_NOTIFIER_HEAD (vmap_notify_list);static void drain_vmap_area_work (struct work_struct *work); static DECLARE_WORK(drain_vmap_work, drain_vmap_area_work); +static struct workqueue_struct *drain_vmap_wq; static __cacheline_aligned_in_smp atomic_long_t nr_vmalloc_pages; static __cacheline_aligned_in_smp atomic_long_t vmap_lazy_nr;@@ -2335,6 +2337,19 @@ static void purge_vmap_node(struct work_struct *work)reclaim_list_global (&local_list); } +static bool +schedule_drain_vmap_work (struct work_struct *work) +{ +struct workqueue_struct *wq =READ_ONCE (drain_vmap_wq); + +if (wq) { +queue_work (wq, work); +return true; + } + +return false; +} +/* * Purges all lazily-freed vmap areas. */ @@ -2342,19 +2357,12 @@ static bool __purge_vmap_area_lazy(unsigned long start, unsigned long end, bool full_pool_decay) {unsigned long nr_purged_areas =0 ; +unsigned int nr_purge_nodes =0 ;unsigned int nr_purge_helpers; -static cpumask_t purge_nodes; -unsigned int nr_purge_nodes;struct vmap_node *vn; -int i;lockdep_assert_held (&vmap_purge_lock); -/* -* Use cpumask to mark which node has to be processed. -*/ - purge_nodes =CPU_MASK_NONE ; -for_each_vmap_node (vn) {INIT_LIST_HEAD (&vn->purge_list); vn->skip_populate = full_pool_decay;@@ -2374,10 +2382,9 @@ static bool __purge_vmap_area_lazy(unsigned long start, unsigned long end, end =max (end,list_last_entry (&vn->purge_list, struct vmap_area, list)->va_end); -cpumask_set_cpu (node_to_id (vn), &purge_nodes); + nr_purge_nodes++; } - nr_purge_nodes =cpumask_weight (&purge_nodes);if (nr_purge_nodes >0 ) {flush_tlb_kernel_range (start, end);@@ -2385,29 +2392,25 @@ static bool __purge_vmap_area_lazy(unsigned long start, unsigned long end, nr_purge_helpers =atomic_long_read (&vmap_lazy_nr) /lazy_max_pages (); nr_purge_helpers =clamp (nr_purge_helpers,1U , nr_purge_nodes) -1 ; -for_each_cpu (i, &purge_nodes) { - vn = &vmap_nodes[i]; +for_each_vmap_node (vn) { + vn->work_queued = false; + +if (list_empty (&vn->purge_list)) +continue ;if (nr_purge_helpers >0 ) {INIT_WORK (&vn->purge_work, purge_vmap_node); - -if (cpumask_test_cpu (i, cpu_online_mask)) -schedule_work_on (i, &vn->purge_work); -else -schedule_work (&vn->purge_work); - + vn->work_queued =schedule_drain_vmap_work (&vn->purge_work); nr_purge_helpers--; }else { - vn->purge_work.func =NULL ;purge_vmap_node (&vn->purge_work); nr_purged_areas += vn->nr_purged; } } -for_each_cpu (i, &purge_nodes) { - vn = &vmap_nodes[i]; - -if (vn->purge_work.func) { +/* Wait for completion if queued any. */ +for_each_vmap_node (vn) { +if (vn->work_queued) {flush_work (&vn->purge_work); nr_purged_areas += vn->nr_purged; }@@ -2471,7 +2474,7 @@ static void free_vmap_area_noflush(struct vmap_area *va)/* After this point, we may free va at any time */ if (unlikely (nr_lazy > nr_lazy_max)) -schedule_work (&drain_vmap_work); +schedule_drain_vmap_work (&drain_vmap_work); }/* @@ -5483,3 +5486,15 @@ void __init vmalloc_init(void) vmap_node_shrinker->scan_objects = vmap_node_shrink_scan;shrinker_register (vmap_node_shrinker); } + +static int __init vmalloc_init_workqueue (void ) +{ +struct workqueue_struct *wq; + + wq =alloc_workqueue ("vmap_drain" ,WQ_UNBOUND |WQ_MEM_RECLAIM ,0 ); +WARN_ON (wq ==NULL ); +WRITE_ONCE (drain_vmap_wq, wq); + +return 0 ; +} +early_initcall (vmalloc_init_workqueue);Signature · 1 line
2.47.3
I will send v2 to prevent progress lose during boot.
--
Uladzislau Rezki