Skip to contentSkip to conversation

mm/vmalloc: Use dedicated unbound workqueue for vmap purge/drain

PATCH Superseded Uladzislau Rezki (Sony) 3 participants 7 messages Mar 30 – Mar 31

Subsystems (2)

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore

Uladzislau Rezki (Sony)

author maintainer
#0 lore raw

PATCH v2 1/1 mm/vmalloc: Use dedicated unbound workqueue for vmap purge/drain

The drain_vmap_area_work() function can take >10ms to complete when there are many accumulated vmap areas in a system with a high CPU count, causing workqueue watchdog warnings when run via schedule_work():

[ 2069.796205] workqueue: drain_vmap_area_work hogged CPU for >10000us 4 times, consider switching to WQ_UNBOUND
[ 2192.823225] workqueue: drain_vmap_area_work hogged CPU for >10000us 5 times, consider switching to WQ_UNBOUND

Switch to a dedicated WQ_UNBOUND workqueue to allow the scheduler to run this background task on any available CPU, improving responsiveness. Use WQ_MEM_RECLAIM to ensure forward progress under memory pressure.

If queuing work to the dedicated workqueue is not possible(during early boot), fall back to processing locally to avoid losing progress.

Also simplify purge helper scheduling by removing cpumask-based iteration in favour to iterating directly over vmap nodes with pending work.

---
 mm/vmalloc.c | 74 +++++++++++++++++++++++++++++++++-------------------
 1 file changed, 47 insertions(+), 27 deletions(-)
mm/vmalloc.c +47 -27
index 61caa55a4402..6bc2523bf75b 100644
@@ -949,6 +949,7 @@ static struct vmap_node {
 	struct list_head purge_list;
 	struct work_struct purge_work;
 	unsigned long nr_purged;
+	bool work_queued;
 } single;
 
 /*
@@ -1067,6 +1068,7 @@ static void reclaim_and_purge_vmap_areas(void);
 static BLOCKING_NOTIFIER_HEAD(vmap_notify_list);
 static void drain_vmap_area_work(struct work_struct *work);
 static DECLARE_WORK(drain_vmap_work, drain_vmap_area_work);
+static struct workqueue_struct *drain_vmap_wq;
 
 static __cacheline_aligned_in_smp atomic_long_t nr_vmalloc_pages;
 static __cacheline_aligned_in_smp atomic_long_t vmap_lazy_nr;
@@ -2335,6 +2337,19 @@ static void purge_vmap_node(struct work_struct *work)
 	reclaim_list_global(&local_list);
 }
 
+static bool
+schedule_drain_vmap_work(struct work_struct *work)
+{
+	struct workqueue_struct *wq = READ_ONCE(drain_vmap_wq);
+
+	if (wq) {
+		queue_work(wq, work);
+		return true;
+	}
+
+	return false;
+}
+
 /*
  * Purges all lazily-freed vmap areas.
  */
@@ -2342,19 +2357,12 @@ static bool __purge_vmap_area_lazy(unsigned long start, unsigned long end,
 		bool full_pool_decay)
 {
 	unsigned long nr_purged_areas = 0;
+	unsigned int nr_purge_nodes = 0;
 	unsigned int nr_purge_helpers;
-	static cpumask_t purge_nodes;
-	unsigned int nr_purge_nodes;
 	struct vmap_node *vn;
-	int i;
 
 	lockdep_assert_held(&vmap_purge_lock);
 
-	/*
-	 * Use cpumask to mark which node has to be processed.
-	 */
-	purge_nodes = CPU_MASK_NONE;
-
 	for_each_vmap_node(vn) {
 		INIT_LIST_HEAD(&vn->purge_list);
 		vn->skip_populate = full_pool_decay;
@@ -2374,10 +2382,9 @@ static bool __purge_vmap_area_lazy(unsigned long start, unsigned long end,
 		end = max(end, list_last_entry(&vn->purge_list,
 			struct vmap_area, list)->va_end);
 
-		cpumask_set_cpu(node_to_id(vn), &purge_nodes);
+		nr_purge_nodes++;
 	}
 
-	nr_purge_nodes = cpumask_weight(&purge_nodes);
 	if (nr_purge_nodes > 0) {
 		flush_tlb_kernel_range(start, end);
 
@@ -2385,29 +2392,30 @@ static bool __purge_vmap_area_lazy(unsigned long start, unsigned long end,
 		nr_purge_helpers = atomic_long_read(&vmap_lazy_nr) / lazy_max_pages();
 		nr_purge_helpers = clamp(nr_purge_helpers, 1U, nr_purge_nodes) - 1;
 
-		for_each_cpu(i, &purge_nodes) {
-			vn = &vmap_nodes[i];
+		for_each_vmap_node(vn) {
+			vn->work_queued = false;
+
+			if (list_empty(&vn->purge_list))
+				continue;
 
 			if (nr_purge_helpers > 0) {
 				INIT_WORK(&vn->purge_work, purge_vmap_node);
+				vn->work_queued = schedule_drain_vmap_work(&vn->purge_work);
 
-				if (cpumask_test_cpu(i, cpu_online_mask))
-					schedule_work_on(i, &vn->purge_work);
-				else
-					schedule_work(&vn->purge_work);
-
-				nr_purge_helpers--;
-			} else {
-				vn->purge_work.func = NULL;
-				purge_vmap_node(&vn->purge_work);
-				nr_purged_areas += vn->nr_purged;
+				if (vn->work_queued) {
+					nr_purge_helpers--;
+					continue;
+				}
 			}
-		}
 
-		for_each_cpu(i, &purge_nodes) {
-			vn = &vmap_nodes[i];
+			/* Sync path. Process locally. */
+			purge_vmap_node(&vn->purge_work);
+			nr_purged_areas += vn->nr_purged;
+		}
 
-			if (vn->purge_work.func) {
+		/* Wait for completion if queued any. */
+		for_each_vmap_node(vn) {
+			if (vn->work_queued) {
 				flush_work(&vn->purge_work);
 				nr_purged_areas += vn->nr_purged;
 			}
@@ -2471,7 +2479,7 @@ static void free_vmap_area_noflush(struct vmap_area *va)
 
 	/* After this point, we may free va at any time */
 	if (unlikely(nr_lazy > nr_lazy_max))
-		schedule_work(&drain_vmap_work);
+		schedule_drain_vmap_work(&drain_vmap_work);
 }
 
 /*
@@ -5483,3 +5491,15 @@ void __init vmalloc_init(void)
 	vmap_node_shrinker->scan_objects = vmap_node_shrink_scan;
 	shrinker_register(vmap_node_shrinker);
 }
+
+static int __init vmalloc_init_workqueue(void)
+{
+	struct workqueue_struct *wq;
+
+	wq = alloc_workqueue("vmap_drain", WQ_UNBOUND | WQ_MEM_RECLAIM, 0);
+	WARN_ON(wq == NULL);
+	WRITE_ONCE(drain_vmap_wq, wq);
+
+	return 0;
+}
+early_initcall(vmalloc_init_workqueue);
Signature · 1 line
2.47.3
maintainer
#1 lore raw
Quoted 12 lines · On Mon, 30 Mar 2026 19:58:24 +0200 "Uladzislau Rezki (Sony)"…

The drain_vmap_area_work() function can take >10ms to complete when there are many accumulated vmap areas in a system with a high CPU count, causing workqueue watchdog warnings when run via schedule_work():

[ 2069.796205] workqueue: drain_vmap_area_work hogged CPU for >10000us 4 times, consider switching to WQ_UNBOUND
[ 2192.823225] workqueue: drain_vmap_area_work hogged CPU for >10000us 5 times, consider switching to WQ_UNBOUND

Switch to a dedicated WQ_UNBOUND workqueue to allow the scheduler to run this background task on any available CPU, improving responsiveness. Use WQ_MEM_RECLAIM to ensure forward progress under memory pressure.

Quoted 6 lines

If queuing work to the dedicated workqueue is not possible(during early boot), fall back to processing locally to avoid losing progress.

Also simplify purge helper scheduling by removing cpumask-based iteration in favour to iterating directly over vmap nodes with pending work.

Thanks, both.

Quoted 3 lines

We don't want to be scaring our users with kernel warnings. Do you think a Fixes: or cc:stable are justified?

And I wonder if that workqueue warning should be WARN_ON_ONCE. That would mean that other, later call sites wouldn't get the report, but we'll still get to hear about those callsites from someone else.

author maintainer
#2 lore raw
Quoted 21 lines · Andrew Morton
Quoted 12 lines · On Mon, 30 Mar 2026 19:58:24 +0200 "Uladzislau Rezki (Sony)"…

The drain_vmap_area_work() function can take >10ms to complete when there are many accumulated vmap areas in a system with a high CPU count, causing workqueue watchdog warnings when run via schedule_work():

[ 2069.796205] workqueue: drain_vmap_area_work hogged CPU for >10000us 4 times, consider switching to WQ_UNBOUND
[ 2192.823225] workqueue: drain_vmap_area_work hogged CPU for >10000us 5 times, consider switching to WQ_UNBOUND

Switch to a dedicated WQ_UNBOUND workqueue to allow the scheduler to run this background task on any available CPU, improving responsiveness. Use WQ_MEM_RECLAIM to ensure forward progress under memory pressure.

Quoted 6 lines

If queuing work to the dedicated workqueue is not possible(during early boot), fall back to processing locally to avoid losing progress.

Also simplify purge helper scheduling by removing cpumask-based iteration in favour to iterating directly over vmap nodes with pending work.

Quoted 9 lines

Thanks, both.

Quoted 3 lines

We don't want to be scaring our users with kernel warnings. Do you think a Fixes: or cc:stable are justified?

Probably we can CC stable.

Quoted 4 lines

And I wonder if that workqueue warning should be WARN_ON_ONCE. That would mean that other, later call sites wouldn't get the report, but we'll still get to hear about those callsites from someone else.

I can switch to ONCE version. Below code:

static int __init vmalloc_init_workqueue(void)

{
	struct workqueue_struct *wq;

	wq = alloc_workqueue("vmap_drain", WQ_UNBOUND | WQ_MEM_RECLAIM, 0);
	WARN_ON(wq == NULL);
	WRITE_ONCE(drain_vmap_wq, wq);

	return 0;
}

early_initcall(vmalloc_init_workqueue);

implies to be called/initialized only once.

--
Uladzislau Rezki

maintainer
#3 lore raw
In reply to Uladzislau Rezki (Sony)
Quoted 12 lines · On Mon, 30 Mar 2026 19:58:24 +0200 "Uladzislau Rezki (Sony)"…

The drain_vmap_area_work() function can take >10ms to complete when there are many accumulated vmap areas in a system with a high CPU count, causing workqueue watchdog warnings when run via schedule_work():

[ 2069.796205] workqueue: drain_vmap_area_work hogged CPU for >10000us 4 times, consider switching to WQ_UNBOUND
[ 2192.823225] workqueue: drain_vmap_area_work hogged CPU for >10000us 5 times, consider switching to WQ_UNBOUND

Switch to a dedicated WQ_UNBOUND workqueue to allow the scheduler to run this background task on any available CPU, improving responsiveness. Use WQ_MEM_RECLAIM to ensure forward progress under memory pressure.

Quoted 6 lines

If queuing work to the dedicated workqueue is not possible(during early boot), fall back to processing locally to avoid losing progress.

Also simplify purge helper scheduling by removing cpumask-based iteration in favour to iterating directly over vmap nodes with pending work.

Thanks. AI review flagged a couple of possible issues. Do they look real to you?

	https://sashiko.dev/#/patchset/[email protected]
author maintainer
#4 lore raw
Quoted 17 lines · Andrew Morton
Quoted 15 lines · On Mon, 30 Mar 2026 19:58:24 +0200 "Uladzislau Rezki (Sony)"…

The drain_vmap_area_work() function can take >10ms to complete when there are many accumulated vmap areas in a system with a high CPU count, causing workqueue watchdog warnings when run via schedule_work():

[ 2069.796205] workqueue: drain_vmap_area_work hogged CPU for >10000us 4 times, consider switching to WQ_UNBOUND
[ 2192.823225] workqueue: drain_vmap_area_work hogged CPU for >10000us 5 times, consider switching to WQ_UNBOUND

Switch to a dedicated WQ_UNBOUND workqueue to allow the scheduler to run this background task on any available CPU, improving responsiveness. Use WQ_MEM_RECLAIM to ensure forward progress under memory pressure.

If queuing work to the dedicated workqueue is not possible(during early boot), fall back to processing locally to avoid losing progress.

Quoted 8 lines
Quoted 3 lines

Also simplify purge helper scheduling by removing cpumask-based iteration in favour to iterating directly over vmap nodes with pending work.

Thanks. AI review flagged a couple of possible issues. Do they look real to you?

	https://sashiko.dev/#/patchset/[email protected]

I think the problem about itself locking if running by rescue thread is a valid concern. I will address this. I think the easiest is to use two UNBOUND queues one for master/main thread and second for helpers which reclaim if there are too many objects so the help is needed.

I will work on v3.

Thank you for review!

--
Uladzislau Rezki

author maintainer
#5 lore raw
Quoted 31 lines · Uladzislau Rezki
Quoted 17 lines · Andrew Morton
Quoted 15 lines · On Mon, 30 Mar 2026 19:58:24 +0200 "Uladzislau Rezki (Sony)"…

The drain_vmap_area_work() function can take >10ms to complete when there are many accumulated vmap areas in a system with a high CPU count, causing workqueue watchdog warnings when run via schedule_work():

[ 2069.796205] workqueue: drain_vmap_area_work hogged CPU for >10000us 4 times, consider switching to WQ_UNBOUND
[ 2192.823225] workqueue: drain_vmap_area_work hogged CPU for >10000us 5 times, consider switching to WQ_UNBOUND

Switch to a dedicated WQ_UNBOUND workqueue to allow the scheduler to run this background task on any available CPU, improving responsiveness. Use WQ_MEM_RECLAIM to ensure forward progress under memory pressure.

If queuing work to the dedicated workqueue is not possible(during early boot), fall back to processing locally to avoid losing progress.

Quoted 8 lines
Quoted 3 lines

Also simplify purge helper scheduling by removing cpumask-based iteration in favour to iterating directly over vmap nodes with pending work.

Thanks. AI review flagged a couple of possible issues. Do they look real to you?

	https://sashiko.dev/#/patchset/[email protected]

I think the problem about itself locking if running by rescue thread is a valid concern. I will address this. I think the easiest is to use two UNBOUND queues one for master/main thread and second for helpers which reclaim if there are too many objects so the help is needed.

Quoted 7 lines

I will work on v3.

Thank you for review!

--
Uladzislau Rezki

I will fix the AI concern by maintaining two queues. One is parent second one is for child helpers. That way both will not block each other and both have a rescue context to move progress forward:

mm/vmalloc.c +16 -9
index 6bc2523bf75b..2c1ed76cffe8 100644
@@ -1068,6 +1068,7 @@ static void reclaim_and_purge_vmap_areas(void);
 static BLOCKING_NOTIFIER_HEAD(vmap_notify_list);
 static void drain_vmap_area_work(struct work_struct *work);
 static DECLARE_WORK(drain_vmap_work, drain_vmap_area_work);
+static struct workqueue_struct *drain_vmap_helpers_wq;
 static struct workqueue_struct *drain_vmap_wq;
 
 static __cacheline_aligned_in_smp atomic_long_t nr_vmalloc_pages;
@@ -2338,10 +2339,9 @@ static void purge_vmap_node(struct work_struct *work)
 }
 
 static bool
-schedule_drain_vmap_work(struct work_struct *work)
+schedule_drain_vmap_work(struct workqueue_struct *wq,
+		struct work_struct *work)
 {
-	struct workqueue_struct *wq = READ_ONCE(drain_vmap_wq);
-
 	if (wq) {
 		queue_work(wq, work);
 		return true;
@@ -2400,7 +2400,8 @@ static bool __purge_vmap_area_lazy(unsigned long start, unsigned long end,
 
 			if (nr_purge_helpers > 0) {
 				INIT_WORK(&vn->purge_work, purge_vmap_node);
-				vn->work_queued = schedule_drain_vmap_work(&vn->purge_work);
+				vn->work_queued = schedule_drain_vmap_work(
+					READ_ONCE(drain_vmap_helpers_wq), &vn->purge_work);
 
 				if (vn->work_queued) {
 					nr_purge_helpers--;
@@ -2479,7 +2480,8 @@ static void free_vmap_area_noflush(struct vmap_area *va)
 
 	/* After this point, we may free va at any time */
 	if (unlikely(nr_lazy > nr_lazy_max))
-		schedule_drain_vmap_work(&drain_vmap_work);
+		schedule_drain_vmap_work(READ_ONCE(drain_vmap_wq),
+			&drain_vmap_work);
 }
 
 /*
@@ -5494,11 +5496,16 @@ void __init vmalloc_init(void)
 
 static int __init vmalloc_init_workqueue(void)
 {
-	struct workqueue_struct *wq;
+	struct workqueue_struct *drain_wq, *helpers_wq;
+	unsigned int flags = WQ_UNBOUND | WQ_MEM_RECLAIM;
+
+	drain_wq = alloc_workqueue("vmap_drain", flags, 0);
+	WARN_ON_ONCE(drain_wq == NULL);
+	WRITE_ONCE(drain_vmap_wq, drain_wq);
 
-	wq = alloc_workqueue("vmap_drain", WQ_UNBOUND | WQ_MEM_RECLAIM, 0);
-	WARN_ON(wq == NULL);
-	WRITE_ONCE(drain_vmap_wq, wq);
+	helpers_wq = alloc_workqueue("vmap_drain_helpers", flags, 0);
+	WARN_ON_ONCE(helpers_wq == NULL);
+	WRITE_ONCE(drain_vmap_helpers_wq, helpers_wq);
 
 	return 0;
 }

if no complains, i will send out v3 soon.

--
Uladzislau Rezki

syzbot Tested-by [email protected] on v2 1/1
#6 lore raw
In reply to Uladzislau Rezki (Sony)
syzbot ci has tested the following series

syzbot ci has tested the following series

[v2] mm/vmalloc: Use dedicated unbound workqueue for vmap purge/drain
https://lore.kernel.org/all/[email protected]
* [PATCH v2] mm/vmalloc: Use dedicated unbound workqueue for vmap purge/drain

and found the following issue:
possible deadlock in touch_wq_lockdep_map

Full report is available here:
https://ci.syzbot.org/series/4a8f638e-3a3f-4346-8189-2b5a0eb9ceaf

***

possible deadlock in touch_wq_lockdep_map

tree:      mm-new
URL:       https://kernel.googlesource.com/pub/scm/linux/kernel/git/akpm/mm.git
base:      af42d6d3650b95295a3f08fc35189998bc26c2e1
arch:      amd64
compiler:  Debian clang version 21.1.8 (++20251221033036+2078da43e25a-1~exp1~20251221153213.50), Debian LLD 21.1.8
config:    https://ci.syzbot.org/builds/d94b1f61-6a69-4415-b385-b3c99018e8c2/config
syz repro: https://ci.syzbot.org/findings/01e4d436-e924-4ce6-8902-78c6000bf34e/syz_repro

============================================
WARNING: possible recursive locking detected
syzkaller #0 Not tainted
--------------------------------------------
kworker/u9:1/33 is trying to acquire lock:
ffff88810168e948 ((wq_completion)vmap_drain){+.+.}-{0:0}, at: touch_wq_lockdep_map+0xb5/0x180 kernel/workqueue.c:3991

but task is already holding lock:
ffff88810168e948 ((wq_completion)vmap_drain){+.+.}-{0:0}, at: process_one_work kernel/workqueue.c:3251 [inline] ffff88810168e948 ((wq_completion)vmap_drain){+.+.}-{0:0}, at: process_scheduled_works+0xa52/0x18c0 kernel/workqueue.c:3359

other info that might help us debug this:

 Possible unsafe locking scenario:
       CPU0
       ----

lock((wq_completion)vmap_drain);
lock((wq_completion)vmap_drain);

*** DEADLOCK ***

May be due to missing lock nesting notation

4 locks held by kworker/u9:1/33:
#0: ffff88810168e948 ((wq_completion)vmap_drain){+.+.}-{0:0}, at: process_one_work kernel/workqueue.c:3251 [inline] #0: ffff88810168e948 ((wq_completion)vmap_drain){+.+.}-{0:0}, at: process_scheduled_works+0xa52/0x18c0 kernel/workqueue.c:3359 #1: ffffc90000a97c40 (drain_vmap_work){+.+.}-{0:0}, at: process_one_work kernel/workqueue.c:3252 [inline] #1: ffffc90000a97c40 (drain_vmap_work){+.+.}-{0:0}, at: process_scheduled_works+0xa8d/0x18c0 kernel/workqueue.c:3359 #2: ffffffff8e87ed68 (vmap_purge_lock){+.+.}-{4:4}, at: drain_vmap_area_work+0x17/0x40 mm/vmalloc.c:2443 #3: ffffffff8e75e5e0 (rcu_read_lock){....}-{1:3}, at: rcu_lock_acquire include/linux/rcupdate.h:312 [inline] #3: ffffffff8e75e5e0 (rcu_read_lock){....}-{1:3}, at: rcu_read_lock include/linux/rcupdate.h:850 [inline] #3: ffffffff8e75e5e0 (rcu_read_lock){....}-{1:3}, at: start_flush_work kernel/workqueue.c:4234 [inline] #3: ffffffff8e75e5e0 (rcu_read_lock){....}-{1:3}, at: __flush_work+0x100/0xc50 kernel/workqueue.c:4292

stack backtrace:
CPU: 1 UID: 0 PID: 33 Comm: kworker/u9:1 Not tainted syzkaller #0 PREEMPT(full) 
Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 1.16.2-debian-1.16.2-1 04/01/2014
Workqueue: vmap_drain drain_vmap_area_work
Call Trace:
 <TASK>
 dump_stack_lvl+0xe8/0x150 lib/dump_stack.c:120
 print_deadlock_bug+0x279/0x290 kernel/locking/lockdep.c:3041
 check_deadlock kernel/locking/lockdep.c:3093 [inline]
 validate_chain kernel/locking/lockdep.c:3895 [inline]
 __lock_acquire+0x253f/0x2cf0 kernel/locking/lockdep.c:5237
 lock_acquire+0xf0/0x2e0 kernel/locking/lockdep.c:5868
 touch_wq_lockdep_map+0xcb/0x180 kernel/workqueue.c:3991
 start_flush_work kernel/workqueue.c:4272 [inline]
 __flush_work+0x87c/0xc50 kernel/workqueue.c:4292
 __purge_vmap_area_lazy+0x7db/0xab0 mm/vmalloc.c:2419
 drain_vmap_area_work+0x27/0x40 mm/vmalloc.c:2444
 process_one_work kernel/workqueue.c:3276 [inline]
 process_scheduled_works+0xb6e/0x18c0 kernel/workqueue.c:3359
 worker_thread+0xa53/0xfc0 kernel/workqueue.c:3440
 kthread+0x388/0x470 kernel/kthread.c:436
 ret_from_fork+0x51e/0xb90 arch/x86/kernel/process.c:158
 ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:245
 </TASK>

***

If these findings have caused you to resend the series or submit a separate fix, please add the following tag to your commit message:

---
This report is generated by a bot. It may contain errors. syzbot ci engineers can be reached at [email protected].

To test a patch for this bug, please reply with #syz test (should be on a separate line).

The patch should be attached to the email.
Note: arguments like custom git repos and branches are not supported.