Skip to contentSkip to conversation

re-enable remote per-cpu-pages draining

PATCH In review Marcelo Tosatti 2 participants 11 messages Jun 16, 2020 – Jun 17, 2020

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

cover re-enable remote per-cpu-pages draining

Remote draining of pages was removed from 5.6-rt.

Unfortunately its necessary for use-cases which have a busy spinning SCHED_FIFO thread on isolated CPU:

[ 7475.821066] INFO: task ld:274531 blocked for more than 600 seconds.
[ 7475.822157]       Not tainted 4.18.0-208.rt5.20.el8.x86_64 #1
[ 7475.823094] echo 0  /proc/sys/kernel/hung_task_timeout_secs disables this message.
[ 7475.824392] ld              D    0 274531 274530 0x00084080
[ 7475.825307] Call Trace:
[ 7475.825761]  __schedule+0x342/0x850
[ 7475.826377]  schedule+0x39/0xd0
[ 7475.826923]  schedule_timeout+0x20e/0x410
[ 7475.827610]  ? __schedule+0x34a/0x850
[ 7475.828247]  ? ___preempt_schedule+0x16/0x18
[ 7475.828953]  wait_for_completion+0x85/0xe0
[ 7475.829653]  flush_work+0x11a/0x1c0
[ 7475.830313]  ? flush_workqueue_prep_pwqs+0x130/0x130
[ 7475.831148]  drain_all_pages+0x140/0x190
[ 7475.831803]  __alloc_pages_slowpath+0x3f8/0xe20
[ 7475.832571]  ? mem_cgroup_commit_charge+0xcb/0x510
[ 7475.833371]  __alloc_pages_nodemask+0x1ca/0x2b0
[ 7475.834134]  pagecache_get_page+0xb5/0x2d0
[ 7475.834814]  ? account_page_dirtied+0x11a/0x220
[ 7475.835579]  grab_cache_page_write_begin+0x1f/0x40
[ 7475.836379]  iomap_write_begin.constprop.44+0x1c1/0x370
[ 7475.837241]  ? iomap_write_end+0x91/0x290
[ 7475.837911]  iomap_write_actor+0x92/0x170

See individual patches for details.

#1 lore raw

PATCH 1/2 rt: add local_lock_on and local_lock_irqsave_on to locallock.h

Re-add local_lock_on type functions to locallock.h (from the 5.4-RT tree).

To be used by remote page draining.

The original commit message is:

Introduce locallock. For !RT this maps to preempt_disable()/ local_irq_disable() so there is not much that changes. For RT this will map to a spinlock. This makes preemption possible and locked "ressource" gets the lockdep anotation it wouldn't have otherwise. The locks are recursive for owner == current. Also, all locks user migrate_disable() which ensures that the task is not migrated to another CPU while the lock is held and the owner is preempted.

From: Thomas Gleixner <[email protected]>

---
 include/linux/locallock.h |   25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)
include/linux/locallock.h +25 -0
Index: linux-rt-devel/include/linux/locallock.h
===================================================================
@@ -51,6 +51,9 @@ static inline void __local_lock(struct l
 #define local_lock(lvar)					\
 	do { __local_lock(&get_local_var(lvar)); } while (0)
 
+#define local_lock_on(lvar, cpu)				\
+	do { __local_lock(&per_cpu(lvar, cpu)); } while (0)
+
 static inline int __local_trylock(struct local_irq_lock *lv)
 {
 	if (lv->owner != current && spin_trylock(&lv->lock)) {
@@ -92,6 +95,9 @@ static inline void __local_unlock(struct
 		put_local_var(lvar);				\
 	} while (0)
 
+#define local_unlock_on(lvar, cpu)                       \
+	do { __local_unlock(&per_cpu(lvar, cpu)); } while (0)
+
 static inline void __local_lock_irq(struct local_irq_lock *lv)
 {
 	spin_lock_irqsave(&lv->lock, lv->flags);
@@ -104,6 +110,9 @@ static inline void __local_lock_irq(stru
 #define local_lock_irq(lvar)						\
 	do { __local_lock_irq(&get_local_var(lvar)); } while (0)
 
+#define local_lock_irq_on(lvar, cpu)					\
+	do { __local_lock_irq(&per_cpu(lvar, cpu)); } while (0)
+
 static inline void __local_unlock_irq(struct local_irq_lock *lv)
 {
 	LL_WARN(!lv->nestcnt);
@@ -119,6 +128,11 @@ static inline void __local_unlock_irq(st
 		put_local_var(lvar);					\
 	} while (0)
 
+#define local_unlock_irq_on(lvar, cpu)					\
+	do {								\
+		__local_unlock_irq(&per_cpu(lvar, cpu));		\
+	} while (0)
+
 static inline int __local_lock_irqsave(struct local_irq_lock *lv)
 {
 	if (lv->owner != current) {
@@ -137,6 +151,12 @@ static inline int __local_lock_irqsave(s
 		_flags = __this_cpu_read(lvar.flags);			\
 	} while (0)
 
+#define local_lock_irqsave_on(lvar, _flags, cpu)			\
+	do {								\
+		__local_lock_irqsave(&per_cpu(lvar, cpu));		\
+		_flags = per_cpu(lvar, cpu).flags;			\
+	} while (0)
+
 static inline int __local_unlock_irqrestore(struct local_irq_lock *lv,
 					    unsigned long flags)
 {
@@ -156,6 +176,11 @@ static inline int __local_unlock_irqrest
 			put_local_var(lvar);				\
 	} while (0)
 
+#define local_unlock_irqrestore_on(lvar, flags, cpu)			\
+	do {								\
+		__local_unlock_irqrestore(&per_cpu(lvar, cpu), flags);	\
+	} while (0)
+
 #define local_spin_trylock_irq(lvar, lock)				\
 	({								\
 		int __locked;						\
#2 lore raw

PATCH 2/2 mm: page_alloc: drain pages remotely

In reply to Marcelo Tosatti

Remote draining of pages was removed from 5.6-rt.

Unfortunately its necessary for use-cases which have a busy spinning SCHED_FIFO thread on isolated CPU:

[ 7475.821066] INFO: task ld:274531 blocked for more than 600 seconds.
[ 7475.822157]       Not tainted 4.18.0-208.rt5.20.el8.x86_64 #1
[ 7475.823094] echo 0  /proc/sys/kernel/hung_task_timeout_secs disables this message.
[ 7475.824392] ld              D    0 274531 274530 0x00084080
[ 7475.825307] Call Trace:
[ 7475.825761]  __schedule+0x342/0x850
[ 7475.826377]  schedule+0x39/0xd0
[ 7475.826923]  schedule_timeout+0x20e/0x410
[ 7475.827610]  ? __schedule+0x34a/0x850
[ 7475.828247]  ? ___preempt_schedule+0x16/0x18
[ 7475.828953]  wait_for_completion+0x85/0xe0
[ 7475.829653]  flush_work+0x11a/0x1c0
[ 7475.830313]  ? flush_workqueue_prep_pwqs+0x130/0x130
[ 7475.831148]  drain_all_pages+0x140/0x190
[ 7475.831803]  __alloc_pages_slowpath+0x3f8/0xe20
[ 7475.832571]  ? mem_cgroup_commit_charge+0xcb/0x510
[ 7475.833371]  __alloc_pages_nodemask+0x1ca/0x2b0
[ 7475.834134]  pagecache_get_page+0xb5/0x2d0
[ 7475.834814]  ? account_page_dirtied+0x11a/0x220
[ 7475.835579]  grab_cache_page_write_begin+0x1f/0x40
[ 7475.836379]  iomap_write_begin.constprop.44+0x1c1/0x370
[ 7475.837241]  ? iomap_write_end+0x91/0x290
[ 7475.837911]  iomap_write_actor+0x92/0x170

...

So enable remote draining again.

The original commit message is:

mm: page_alloc: rt-friendly per-cpu pages

rt-friendly per-cpu pages: convert the irqs-off per-cpu locking method into a preemptible, explicit-per-cpu-locks method.

Contains fixes from:
Peter Zijlstra <[email protected]> Thomas Gleixner <[email protected]>

From: Ingo Molnar <[email protected]>

---
 mm/page_alloc.c |   21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)
mm/page_alloc.c +19 -2
Index: linux-rt-devel/mm/page_alloc.c
===================================================================
@@ -360,6 +360,16 @@ EXPORT_SYMBOL(nr_online_nodes);
 
 static DEFINE_LOCAL_IRQ_LOCK(pa_lock);
 
+#ifdef CONFIG_PREEMPT_RT
+# define cpu_lock_irqsave(cpu, flags)          \
+	local_lock_irqsave_on(pa_lock, flags, cpu)
+# define cpu_unlock_irqrestore(cpu, flags)     \
+	local_unlock_irqrestore_on(pa_lock, flags, cpu)
+#else
+# define cpu_lock_irqsave(cpu, flags)		local_irq_save(flags)
+# define cpu_unlock_irqrestore(cpu, flags)	local_irq_restore(flags)
+#endif
+
 int page_group_by_mobility_disabled __read_mostly;
 
 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
@@ -2852,7 +2862,7 @@ static void drain_pages_zone(unsigned in
 	LIST_HEAD(dst);
 	int count;
 
-	local_lock_irqsave(pa_lock, flags);
+	cpu_lock_irqsave(cpu, flags);
 	pset = per_cpu_ptr(zone->pageset, cpu);
 
 	pcp = &pset->pcp;
@@ -2860,7 +2870,7 @@ static void drain_pages_zone(unsigned in
 	if (count)
 		isolate_pcp_pages(count, pcp, &dst);
 
-	local_unlock_irqrestore(pa_lock, flags);
+	cpu_unlock_irqrestore(cpu, flags);
 
 	if (count)
 		free_pcppages_bulk(zone, &dst, false);
@@ -2898,6 +2908,7 @@ void drain_local_pages(struct zone *zone
 		drain_pages(cpu);
 }
 
+#ifndef CONFIG_PREEMPT_RT
 static void drain_local_pages_wq(struct work_struct *work)
 {
 	struct pcpu_drain *drain;
@@ -2915,6 +2926,7 @@ static void drain_local_pages_wq(struct
 	drain_local_pages(drain->zone);
 	migrate_enable();
 }
+#endif
 
 /*
  * Spill all the per-cpu pages from all CPUs back into the buddy allocator.
@@ -2982,6 +2994,7 @@ void drain_all_pages(struct zone *zone)
 			cpumask_clear_cpu(cpu, &cpus_with_pcps);
 	}
 
+#ifndef CONFIG_PREEMPT_RT
 	for_each_cpu(cpu, &cpus_with_pcps) {
 		struct pcpu_drain *drain = per_cpu_ptr(&pcpu_drain, cpu);
 
@@ -2991,6 +3004,10 @@ void drain_all_pages(struct zone *zone)
 	}
 	for_each_cpu(cpu, &cpus_with_pcps)
 		flush_work(&per_cpu_ptr(&pcpu_drain, cpu)->work);
+#else
+	for_each_cpu(cpu, &cpus_with_pcps)
+		drain_pages(cpu);
+#endif
 
 	mutex_unlock(&pcpu_drain_mutex);
 }
#3 lore raw

Re: [patch 2/2] mm: page_alloc: drain pages remotely

Quoted 28 lines · Marcelo Tosatti

Remote draining of pages was removed from 5.6-rt.

Unfortunately its necessary for use-cases which have a busy spinning SCHED_FIFO thread on isolated CPU:

[ 7475.821066] INFO: task ld:274531 blocked for more than 600 seconds.
[ 7475.822157]       Not tainted 4.18.0-208.rt5.20.el8.x86_64 #1
[ 7475.823094] echo 0  /proc/sys/kernel/hung_task_timeout_secs disables this message.
[ 7475.824392] ld              D    0 274531 274530 0x00084080
[ 7475.825307] Call Trace:
[ 7475.825761]  __schedule+0x342/0x850
[ 7475.826377]  schedule+0x39/0xd0
[ 7475.826923]  schedule_timeout+0x20e/0x410
[ 7475.827610]  ? __schedule+0x34a/0x850
[ 7475.828247]  ? ___preempt_schedule+0x16/0x18
[ 7475.828953]  wait_for_completion+0x85/0xe0
[ 7475.829653]  flush_work+0x11a/0x1c0
[ 7475.830313]  ? flush_workqueue_prep_pwqs+0x130/0x130
[ 7475.831148]  drain_all_pages+0x140/0x190
[ 7475.831803]  __alloc_pages_slowpath+0x3f8/0xe20
[ 7475.832571]  ? mem_cgroup_commit_charge+0xcb/0x510
[ 7475.833371]  __alloc_pages_nodemask+0x1ca/0x2b0
[ 7475.834134]  pagecache_get_page+0xb5/0x2d0
[ 7475.834814]  ? account_page_dirtied+0x11a/0x220
[ 7475.835579]  grab_cache_page_write_begin+0x1f/0x40
[ 7475.836379]  iomap_write_begin.constprop.44+0x1c1/0x370
[ 7475.837241]  ? iomap_write_end+0x91/0x290
[ 7475.837911]  iomap_write_actor+0x92/0x170
Quoted 3 lines

...

So enable remote draining again.

Is upstream affected by this? And if not, why not?

Quoted 7 lines

Index: linux-rt-devel/mm/page_alloc.c
===================================================================

mm/page_alloc.c +0 -0
@@ -360,6 +360,16 @@ EXPORT_SYMBOL(nr_online_nodes);
 
 static DEFINE_LOCAL_IRQ_LOCK(pa_lock);

mm/page_alloc.c EXPORT_SYMBOL(nr_online_nodes); · line 371

Quoted 10 lines
mm/page_alloc.c +9 -0
 
+#ifdef CONFIG_PREEMPT_RT
+# define cpu_lock_irqsave(cpu, flags)          \
+	local_lock_irqsave_on(pa_lock, flags, cpu)
+# define cpu_unlock_irqrestore(cpu, flags)     \
+	local_unlock_irqrestore_on(pa_lock, flags, cpu)
+#else
+# define cpu_lock_irqsave(cpu, flags)		local_irq_save(flags)
+# define cpu_unlock_irqrestore(cpu, flags)	local_irq_restore(flags)
+#endif

This is going to be tough. I removed the cross-CPU local-locks from RT because it does something different for !RT. Furthermore we have local_locks in upstream as of v5.8-rc1, see commit
91710728d1725 ("locking: Introduce local_lock()")

so whatever happens here should have upstream blessing or I will be forced to drop the patch again while moving forward.

Before this, I looked for cases where remote drain is useful / needed and didn't find one. I talked to Frederick and for the NO_HZ_FULL people it is not a problem because they don't go to kernel and so they never got anything on their per-CPU list.

We had this
https://lore.kernel.org/linux-mm/[email protected]/

Sebastian

#4 lore raw

Re: [patch 2/2] mm: page_alloc: drain pages remotely

Quoted 58 lines · Sebastian Andrzej Siewior
Quoted 28 lines · Marcelo Tosatti

Remote draining of pages was removed from 5.6-rt.

Unfortunately its necessary for use-cases which have a busy spinning SCHED_FIFO thread on isolated CPU:

[ 7475.821066] INFO: task ld:274531 blocked for more than 600 seconds.
[ 7475.822157]       Not tainted 4.18.0-208.rt5.20.el8.x86_64 #1
[ 7475.823094] echo 0  /proc/sys/kernel/hung_task_timeout_secs disables this message.
[ 7475.824392] ld              D    0 274531 274530 0x00084080
[ 7475.825307] Call Trace:
[ 7475.825761]  __schedule+0x342/0x850
[ 7475.826377]  schedule+0x39/0xd0
[ 7475.826923]  schedule_timeout+0x20e/0x410
[ 7475.827610]  ? __schedule+0x34a/0x850
[ 7475.828247]  ? ___preempt_schedule+0x16/0x18
[ 7475.828953]  wait_for_completion+0x85/0xe0
[ 7475.829653]  flush_work+0x11a/0x1c0
[ 7475.830313]  ? flush_workqueue_prep_pwqs+0x130/0x130
[ 7475.831148]  drain_all_pages+0x140/0x190
[ 7475.831803]  __alloc_pages_slowpath+0x3f8/0xe20
[ 7475.832571]  ? mem_cgroup_commit_charge+0xcb/0x510
[ 7475.833371]  __alloc_pages_nodemask+0x1ca/0x2b0
[ 7475.834134]  pagecache_get_page+0xb5/0x2d0
[ 7475.834814]  ? account_page_dirtied+0x11a/0x220
[ 7475.835579]  grab_cache_page_write_begin+0x1f/0x40
[ 7475.836379]  iomap_write_begin.constprop.44+0x1c1/0x370
[ 7475.837241]  ? iomap_write_end+0x91/0x290
[ 7475.837911]  iomap_write_actor+0x92/0x170
Quoted 3 lines

...

So enable remote draining again.

Is upstream affected by this? And if not, why not?

Quoted 7 lines

Index: linux-rt-devel/mm/page_alloc.c
===================================================================

mm/page_alloc.c +0 -0
@@ -360,6 +360,16 @@ EXPORT_SYMBOL(nr_online_nodes);
 
 static DEFINE_LOCAL_IRQ_LOCK(pa_lock);

mm/page_alloc.c EXPORT_SYMBOL(nr_online_nodes); · line 371

Quoted 10 lines
mm/page_alloc.c +9 -0
 
+#ifdef CONFIG_PREEMPT_RT
+# define cpu_lock_irqsave(cpu, flags)          \
+	local_lock_irqsave_on(pa_lock, flags, cpu)
+# define cpu_unlock_irqrestore(cpu, flags)     \
+	local_unlock_irqrestore_on(pa_lock, flags, cpu)
+#else
+# define cpu_lock_irqsave(cpu, flags)		local_irq_save(flags)
+# define cpu_unlock_irqrestore(cpu, flags)	local_irq_restore(flags)
+#endif

This is going to be tough. I removed the cross-CPU local-locks from RT because it does something different for !RT. Furthermore we have local_locks in upstream as of v5.8-rc1, see commit
91710728d1725 ("locking: Introduce local_lock()")

Quoted 2 lines

so whatever happens here should have upstream blessing or I will be forced to drop the patch again while moving forward.

Understood.

Quoted 2 lines

Before this, I looked for cases where remote drain is useful / needed and didn't find one.

Just pointed out one.

Quoted 3 lines

I talked to Frederick and for the NO_HZ_FULL people
it is not a problem because they don't go to kernel and so they never got anything on their per-CPU list.

People are using NOHZ_FULL CPUs to run both SCHED_FIFO realtime workloads and normal workloads. Moreover, even with syscall-less applications:

1) Setup application (malloc buffers, etc).
2) Set SCHED_FIFO priority.
3) sched_setaffinity() to NOHZ_FULL CPU.

Per-CPU buffers will be large and must be shrunk.

Quoted 2 lines

We had this
https://lore.kernel.org/linux-mm/[email protected]/

Will reply to that thread. Do you want to refresh/resend that patchset or should I?

#5 lore raw

Re: [patch 2/2] mm: page_alloc: drain pages remotely

Quoted 5 lines · Marcelo Tosatti
Quoted 2 lines

We had this
https://lore.kernel.org/linux-mm/[email protected]/

Will reply to that thread. Do you want to refresh/resend that patchset or should I?

I don't plan to refresh/resend it because what we have now with local-locks in v5.8-rc1 works for RT.
I dropped that from RT in v5.4.26-rt17 so the previous release should have the last version of this. Please be aware that Joe Korty reported that this patch somehow breaks a workload for him. The report never made it to the list but he confirmed in
https://lkml.kernel.org/r/[email protected]

that is problem is gone with the removal of the patch.

Sebastian

#6 lore raw

Re: [patch 2/2] mm: page_alloc: drain pages remotely

Quoted 9 lines · Sebastian Andrzej Siewior
Quoted 5 lines · Marcelo Tosatti
Quoted 2 lines

We had this
https://lore.kernel.org/linux-mm/[email protected]/

Will reply to that thread. Do you want to refresh/resend that patchset or should I?

I don't plan to refresh/resend it because what we have now with local-locks in v5.8-rc1 works for RT.

If your application busy-spins with DPDK (such as FlexRAN does):

https://www.slideshare.net/LuminaNetworks/osn-bay-area-feb-2019-meetup-intel-5g-flexran-solution-131647692

Then v5.8-rc1 does not work, right? (remote pagevec locking + remote draining would be necessary).

Quoted 5 lines

I dropped that from RT in v5.4.26-rt17 so the previous release should have the last version of this. Please be aware that Joe Korty reported that this patch somehow breaks a workload for him. The report never made it to the list but he confirmed in
https://lkml.kernel.org/r/[email protected]

OK, will see if i can reproduce it and assist in the pagevec locking upstreaming effort.

Thanks!

Quoted 4 lines

that is problem is gone with the removal of the patch.

Sebastian

#7 lore raw

Re: [patch 2/2] mm: page_alloc: drain pages remotely

Quoted 3 lines · Marcelo Tosatti

If your application busy-spins with DPDK (such as FlexRAN does):

https://www.slideshare.net/LuminaNetworks/osn-bay-area-feb-2019-meetup-intel-5g-flexran-solution-131647692

I see.

Quoted 2 lines

Then v5.8-rc1 does not work, right? (remote pagevec locking + remote draining would be necessary).

No, it is what we have in latest rt-devel: A per-CPU spinlock replacement for preempt_disable() on CONFIG_PREEMPT_RT. We use the schedule_work_on() as mainline does.

Sebastian

#8 lore raw

Re: [patch 2/2] mm: page_alloc: drain pages remotely

In reply to Sebastian Andrzej Siewior
Quoted 31 lines · Sebastian Andrzej Siewior
Quoted 30 lines · Marcelo Tosatti

Remote draining of pages was removed from 5.6-rt.

Unfortunately its necessary for use-cases which have a busy spinning SCHED_FIFO thread on isolated CPU:

[ 7475.821066] INFO: task ld:274531 blocked for more than 600 seconds.
[ 7475.822157]       Not tainted 4.18.0-208.rt5.20.el8.x86_64 #1
[ 7475.823094] echo 0  /proc/sys/kernel/hung_task_timeout_secs disables this message.
[ 7475.824392] ld              D    0 274531 274530 0x00084080
[ 7475.825307] Call Trace:
[ 7475.825761]  __schedule+0x342/0x850
[ 7475.826377]  schedule+0x39/0xd0
[ 7475.826923]  schedule_timeout+0x20e/0x410
[ 7475.827610]  ? __schedule+0x34a/0x850
[ 7475.828247]  ? ___preempt_schedule+0x16/0x18
[ 7475.828953]  wait_for_completion+0x85/0xe0
[ 7475.829653]  flush_work+0x11a/0x1c0
[ 7475.830313]  ? flush_workqueue_prep_pwqs+0x130/0x130
[ 7475.831148]  drain_all_pages+0x140/0x190
[ 7475.831803]  __alloc_pages_slowpath+0x3f8/0xe20
[ 7475.832571]  ? mem_cgroup_commit_charge+0xcb/0x510
[ 7475.833371]  __alloc_pages_nodemask+0x1ca/0x2b0
[ 7475.834134]  pagecache_get_page+0xb5/0x2d0
[ 7475.834814]  ? account_page_dirtied+0x11a/0x220
[ 7475.835579]  grab_cache_page_write_begin+0x1f/0x40
[ 7475.836379]  iomap_write_begin.constprop.44+0x1c1/0x370
[ 7475.837241]  ? iomap_write_end+0x91/0x290
[ 7475.837911]  iomap_write_actor+0x92/0x170

...

Quoted 3 lines
Quoted 1 line

So enable remote draining again.

Is upstream affected by this? And if not, why not?

It is affected. The workqueue thread for a given CPU is SCHED_OTHER, which can't interrupt a SCHED_FIFO task which does not sleep.

#9 lore raw

Re: [patch 2/2] mm: page_alloc: drain pages remotely

Quoted 2 lines · Marcelo Tosatti

It is affected. The workqueue thread for a given CPU is SCHED_OTHER, which can't interrupt a SCHED_FIFO task which does not sleep.

Okay. So it is a problem which exists independently of -RT. As I pointed out to Juri, vmstat_update() should have the very same problem, correct?

Sebastian

#10 lore raw

Re: [patch 2/2] mm: page_alloc: drain pages remotely

Quoted 7 lines · Sebastian Andrzej Siewior
Quoted 2 lines · Marcelo Tosatti

It is affected. The workqueue thread for a given CPU is SCHED_OTHER, which can't interrupt a SCHED_FIFO task which does not sleep.

Okay. So it is a problem which exists independently of -RT. As I pointed out to Juri, vmstat_update() should have the very same problem, correct?

Yes.