mm: perform lru_add_drain_all() remotely
PATCH v2 1/1 mm: perform lru_add_drain_all() remotely
lru_add_drain_all() works by scheduling lru_add_drain_cpu() to run on all CPUs that have non-empty LRU pagevecs and then waiting for the scheduled work to complete. However, workqueue threads may never have the chance to run on a CPU that's running a SCHED_FIFO task. This causes lru_add_drain_all() to block forever.
This commit solves this problem by changing lru_add_drain_all() to drain the LRU pagevecs of remote CPUs. This is done by grabbing swapvec_lock and calling lru_add_drain_cpu().
PS: This is based on an idea and initial implementation by
Rik van Riel.
Signed-off-by Rik van Riel <[email protected]>
Signed-off-by Luiz Capitulino <[email protected]>
---
Sebastian,
It's not clear to me from our last discussion what's the best plan for this patch. I'm sending v2 with the changes you suggested.
Changelog
v2
- Simple renames
local_lock_other_cpu() -> local_lock_on() local_lock_irqsave_other_cpu() -> local_lock_irq_on() (unlock follows same renaiming)
include/linux/locallock.h | 31 +++++++++++++++++++++++++++++++ mm/swap.c | 35 +++++++++++++++++++++++++---------- 2 files changed, 56 insertions(+), 10 deletions(-)
include/linux/locallock.h +31 -0
@@ -104,6 +104,17 @@ static inline void __local_unlock(struct local_irq_lock *lv)put_local_var (lvar); \ } while (0) +#define local_lock_on (lvar, cpu) \ + do { \ + __local_lock(&per_cpu(lvar, cpu)); \ + } while (0) + +#define local_unlock_on (lvar, cpu) \ + do { \ + __local_unlock(&per_cpu(lvar, cpu)); \ + } while (0) + +static inline void __local_lock_irq (struct local_irq_lock *lv) {spin_lock_irqsave (&lv->lock, lv->flags);
@@ -163,6 +174,22 @@ static inline int __local_lock_irqsave(struct local_irq_lock *lv) _flags =per_cpu (lvar, cpu).flags; \ } while (0) +#define local_lock_irq_on (lvar, _flags, cpu) \ + do { \ + if (cpu == smp_processor_id()) \ + local_lock_irqsave(lvar, _flags); \ + else \ + local_lock_on(lvar, cpu); \ + } while (0) + +#define local_unlock_irq_on (lvar, _flags, cpu) \ + do { \ + if (cpu == smp_processor_id()) \ + local_unlock_irqrestore(lvar, _flags); \ + else \ + local_unlock_on(lvar, cpu); \ + } while (0) +static inline int __local_unlock_irqrestore (struct local_irq_lock *lv,unsigned long flags) {
@@ -250,6 +277,10 @@ static inline void local_irq_lock_init(int lvar) { }#define local_unlock_irq (lvar) local_irq_enable()#define local_lock_irqsave (lvar, flags) local_irq_save(flags)#define local_unlock_irqrestore (lvar, flags) local_irq_restore(flags) +#define local_lock_irq_on (lvar, flags, cpu) \ + local_irq_save(flags) +#define local_unlock_irq_on (lvar, flags, cpu) \ + local_irq_restore(flags)#define local_spin_trylock_irq (lvar, lock) spin_trylock_irq(lock)#define local_spin_lock_irq (lvar, lock) spin_lock_irq(lock)
mm/swap.c +23 -4
@@ -821,9 +821,9 @@ void lru_add_drain_cpu(int cpu)unsigned long flags;/* No harm done if a racing interrupt already did this */ -local_lock_irqsave (rotate_lock, flags); +local_lock_irq_on (rotate_lock, flags, cpu);pagevec_move_tail (pvec); -local_unlock_irqrestore (rotate_lock, flags); +local_unlock_irq_on (rotate_lock, flags, cpu); } pvec = &per_cpu (lru_deactivate_file_pvecs, cpu);
@@ -866,12 +866,32 @@ void lru_add_drain(void)local_unlock_cpu (swapvec_lock); } +static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work); + +#ifdef CONFIG_PREEMPT_RT_BASE +static inline void remote_lru_add_drain (int cpu,struct cpumask *has_work) +{ +local_lock_on (swapvec_lock, cpu); +lru_add_drain_cpu (cpu); +local_unlock_on (swapvec_lock, cpu); +} +#else static void lru_add_drain_per_cpu (struct work_struct *dummy) {lru_add_drain (); } -static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work); +static inline void remote_lru_add_drain (int cpu,struct cpumask *has_work) +{ +struct work_struct *work = &per_cpu (lru_add_drain_work, cpu); + +INIT_WORK (work, lru_add_drain_per_cpu); +schedule_work_on (cpu, work); +cpumask_set_cpu (cpu, has_work); + +} +#endif +void lru_add_drain_all (void ) {
@@ -884,16 +904,11 @@ void lru_add_drain_all(void)cpumask_clear (&has_work);for_each_online_cpu (cpu) { -struct work_struct *work = &per_cpu (lru_add_drain_work, cpu);
Quoted 5 lines · * Luiz Capitulino | 2016-05-24 15:15:51 [-0400]:
Sebastian,
It's not clear to me from our last discussion what's the best plan for this patch. I'm sending v2 with the changes you suggested.
The patch at the bottom is what I intend to take into the next v4.6-RT. local_lock_irqsave_on() was already part of the first v4.6 release. I don't like the part where local_lock_irqsave_on() is a simple local_irq_safe() on !RT but that is another story.
include/linux/locallock.h +6 -0
@@ -66,6 +66,9 @@ static inline void __local_lock(struct local_irq_lock *lv)#define local_lock (lvar) \ do { __local_lock(&get_local_var(lvar)); } while (0) +#define local_lock_on (lvar, cpu) \ + do { __local_lock(&per_cpu(lvar, cpu)); } while (0) +static inline int __local_trylock (struct local_irq_lock *lv) {if (lv->owner != current &&spin_trylock_local (&lv->lock)) {
@@ -104,6 +107,9 @@ static inline void __local_unlock(struct local_irq_lock *lv)put_local_var (lvar); \ } while (0) +#define local_unlock_on (lvar, cpu) \ + do { __local_unlock(&per_cpu(lvar, cpu)); } while (0) +static inline void __local_lock_irq (struct local_irq_lock *lv) {spin_lock_irqsave (&lv->lock, lv->flags);
mm/swap.c +22 -3
@@ -596,9 +596,9 @@ void lru_add_drain_cpu(int cpu)unsigned long flags;/* No harm done if a racing interrupt already did this */ -local_lock_irqsave (rotate_lock, flags); +local_lock_irqsave_on (rotate_lock, flags, cpu);pagevec_move_tail (pvec); -local_unlock_irqrestore (rotate_lock, flags); +local_unlock_irqrestore_on (rotate_lock, flags, cpu); } pvec = &per_cpu (lru_deactivate_file_pvecs, cpu);
@@ -666,12 +666,32 @@ void lru_add_drain(void)local_unlock_cpu (swapvec_lock); } + +#ifdef CONFIG_PREEMPT_RT_BASE +static inline void remote_lru_add_drain (int cpu,struct cpumask *has_work) +{ +local_lock_on (swapvec_lock, cpu); +lru_add_drain_cpu (cpu); +local_unlock_on (swapvec_lock, cpu); +} + +#else +static void lru_add_drain_per_cpu (struct work_struct *dummy) {lru_add_drain (); } static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work); +static inline void remote_lru_add_drain (int cpu,struct cpumask *has_work) +{ +struct work_struct *work = &per_cpu (lru_add_drain_work, cpu); + +INIT_WORK (work, lru_add_drain_per_cpu); +schedule_work_on (cpu, work); +cpumask_set_cpu (cpu, has_work); +} +#endif void lru_add_drain_all (void ) {
@@ -684,21 +704,18 @@ void lru_add_drain_all(void)cpumask_clear (&has_work);for_each_online_cpu (cpu) { -struct work_struct *work = &per_cpu (lru_add_drain_work, cpu);
Quoted 4 lines · On Fri, 27 May 2016 15:10:24 +0200 Sebastian Andrzej Siewior…
Quoted 2 lines · * Luiz Capitulino | 2016-05-24 15:15:51 [-0400]:
Sebastian,
Quoted 8 lines
Quoted 3 lines
It's not clear to me from our last discussion what's the best plan for this patch. I'm sending v2 with the changes you suggested.
The patch at the bottom is what I intend to take into the next v4.6-RT.
local_lock_irqsave_on()was already part of the first v4.6 release. I don't like the part wherelocal_lock_irqsave_on()is a simplelocal_irq_safe()on !RT but that is another story.
Looks good to me.
Quoted 100 lines
include/linux/locallock.h +6
-0@@ -66,6 +66,9 @@ static inline void __local_lock(struct local_irq_lock *lv)#define local_lock (lvar) \ do { __local_lock(&get_local_var(lvar)); } while (0) +#define local_lock_on (lvar, cpu) \ + do { __local_lock(&per_cpu(lvar, cpu)); } while (0) +static inline int __local_trylock (struct local_irq_lock *lv) {if (lv->owner != current &&spin_trylock_local (&lv->lock)) {@@ -104,6 +107,9 @@ static inline void __local_unlock(struct local_irq_lock *lv)put_local_var (lvar); \ } while (0) +#define local_unlock_on (lvar, cpu) \ + do { __local_unlock(&per_cpu(lvar, cpu)); } while (0) +static inline void __local_lock_irq (struct local_irq_lock *lv) {spin_lock_irqsave (&lv->lock, lv->flags);mm/swap.c +26
-9@@ -596,9 +596,9 @@ void lru_add_drain_cpu(int cpu)unsigned long flags;/* No harm done if a racing interrupt already did this */ -local_lock_irqsave (rotate_lock, flags); +local_lock_irqsave_on (rotate_lock, flags, cpu);pagevec_move_tail (pvec); -local_unlock_irqrestore (rotate_lock, flags); +local_unlock_irqrestore_on (rotate_lock, flags, cpu); } pvec = &per_cpu (lru_deactivate_file_pvecs, cpu);@@ -666,12 +666,32 @@ void lru_add_drain(void)local_unlock_cpu (swapvec_lock); } + +#ifdef CONFIG_PREEMPT_RT_BASE +static inline void remote_lru_add_drain (int cpu,struct cpumask *has_work) +{ +local_lock_on (swapvec_lock, cpu); +lru_add_drain_cpu (cpu); +local_unlock_on (swapvec_lock, cpu); +} + +#else +static void lru_add_drain_per_cpu (struct work_struct *dummy) {lru_add_drain (); } static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work); +static inline void remote_lru_add_drain (int cpu,struct cpumask *has_work) +{ +struct work_struct *work = &per_cpu (lru_add_drain_work, cpu); + +INIT_WORK (work, lru_add_drain_per_cpu); +schedule_work_on (cpu, work); +cpumask_set_cpu (cpu, has_work); +} +#endif void lru_add_drain_all (void ) {@@ -684,21 +704,18 @@ void lru_add_drain_all(void)cpumask_clear (&has_work);for_each_online_cpu (cpu) { -struct work_struct *work = &per_cpu (lru_add_drain_work, cpu); -if (pagevec_count (&per_cpu (lru_add_pvec, cpu)) ||pagevec_count (&per_cpu (lru_rotate_pvecs, cpu)) ||pagevec_count (&per_cpu (lru_deactivate_file_pvecs, cpu)) ||pagevec_count (&per_cpu (lru_deactivate_pvecs, cpu)) || -need_activate_page_drain (cpu)) { -INIT_WORK (work, lru_add_drain_per_cpu); -schedule_work_on (cpu, work); -cpumask_set_cpu (cpu, &has_work); - } +need_activate_page_drain (cpu)) +remote_lru_add_drain (cpu, &has_work); } +#ifndef CONFIG_PREEMPT_RT_BASE for_each_cpu (cpu, &has_work)flush_work (&per_cpu (lru_add_drain_work, cpu)); +#endif put_online_cpus ();mutex_unlock (&lock);