Skip to contentSkip to conversation

mm: perform lru_add_drain_all() remotely

RFC PATCH Superseded Luiz Capitulino 2 participants 3 messages Apr 28, 2016 – Apr 29, 2016 Memory management

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

RFC PATCH 1/1 mm: perform lru_add_drain_all() remotely

lru_add_drain_all() works by scheduling lru_add_drain_cpu() to run on all CPUs that have non-empty LRU pagevecs and then waiting for the scheduled work to complete. However, workqueue threads may never have the chance to run on a CPU that's running a SCHED_FIFO task. This causes lru_add_drain_all() to block forever.

This commit solves this problem by changing lru_add_drain_all() to drain the LRU pagevecs of remote CPUs. This is done by grabbing swapvec_lock and calling lru_add_drain_cpu().

PS: This is based on an idea and initial implementation by
Rik van Riel.

---
 include/linux/locallock.h | 27 +++++++++++++++++++++++++++
 mm/swap.c                 | 35 +++++++++++++++++++++++++----------
 2 files changed, 52 insertions(+), 10 deletions(-)
include/linux/locallock.h +27 -0
index 6fe5928..2de478b 100644
@@ -104,6 +104,17 @@ static inline void __local_unlock(struct local_irq_lock *lv)
 		put_local_var(lvar);				\
 	} while (0)
 
+#define local_lock_other_cpu(lvar, cpu)                         \
+	do {                                                    \
+		__local_lock(&per_cpu(lvar, cpu));              \
+	} while (0)
+
+#define local_unlock_other_cpu(lvar, cpu)                       \
+	do {                                                    \
+		__local_unlock(&per_cpu(lvar, cpu));            \
+	} while (0)
+
+
 static inline void __local_lock_irq(struct local_irq_lock *lv)
 {
 	spin_lock_irqsave(&lv->lock, lv->flags);
@@ -163,6 +174,22 @@ static inline int __local_lock_irqsave(struct local_irq_lock *lv)
 		_flags = per_cpu(lvar, cpu).flags;			\
 	} while (0)
 
+#define local_lock_irqsave_other_cpu(lvar, _flags, cpu)			\
+	do {								\
+		if (cpu == smp_processor_id())				\
+			local_lock_irqsave(lvar, _flags);		\
+		else							\
+			local_lock_other_cpu(lvar, cpu);		\
+	} while (0)
+
+#define local_unlock_irqrestore_other_cpu(lvar, _flags, cpu)	        \
+	do {								\
+		if (cpu == smp_processor_id())				\
+			local_unlock_irqrestore(lvar, _flags);		\
+		else							\
+			local_unlock_other_cpu(lvar, cpu);		\
+	} while (0)
+
 static inline int __local_unlock_irqrestore(struct local_irq_lock *lv,
 					    unsigned long flags)
 {
mm/swap.c +23 -4
index ca194ae..9dc6956 100644
@@ -821,9 +821,9 @@ void lru_add_drain_cpu(int cpu)
 		unsigned long flags;
 
 		/* No harm done if a racing interrupt already did this */
-		local_lock_irqsave(rotate_lock, flags);
+		local_lock_irqsave_other_cpu(rotate_lock, flags, cpu);
 		pagevec_move_tail(pvec);
-		local_unlock_irqrestore(rotate_lock, flags);
+		local_unlock_irqrestore_other_cpu(rotate_lock, flags, cpu);
 	}
 
 	pvec = &per_cpu(lru_deactivate_file_pvecs, cpu);
@@ -866,12 +866,32 @@ void lru_add_drain(void)
 	local_unlock_cpu(swapvec_lock);
 }
 
+static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work);
+
+#ifdef CONFIG_PREEMPT_RT_BASE
+static inline void remote_lru_add_drain(int cpu, struct cpumask *has_work)
+{
+	local_lock_other_cpu(swapvec_lock, cpu);
+	lru_add_drain_cpu(cpu);
+	local_unlock_other_cpu(swapvec_lock, cpu);
+}
+#else
 static void lru_add_drain_per_cpu(struct work_struct *dummy)
 {
 	lru_add_drain();
 }
 
-static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work);
+static inline void remote_lru_add_drain(int cpu, struct cpumask *has_work)
+{
+		struct work_struct *work = &per_cpu(lru_add_drain_work, cpu);
+
+		INIT_WORK(work, lru_add_drain_per_cpu);
+		schedule_work_on(cpu, work);
+		cpumask_set_cpu(cpu, &has_work);
+
+}
+#endif
+
 
 void lru_add_drain_all(void)
 {
@@ -884,16 +904,11 @@ void lru_add_drain_all(void)
 	cpumask_clear(&has_work);
 
 	for_each_online_cpu(cpu) {
-		struct work_struct *work = &per_cpu(lru_add_drain_work, cpu);
#1 lore raw
Quoted 6 lines · Luiz Capitulino

lru_add_drain_all() works by scheduling lru_add_drain_cpu() to run on all CPUs that have non-empty LRU pagevecs and then waiting for the scheduled work to complete. However, workqueue threads may never have the chance to run on a CPU that's running a SCHED_FIFO task. This causes lru_add_drain_all() to block forever.

Quoted 6 lines

This commit solves this problem by changing lru_add_drain_all() to drain the LRU pagevecs of remote CPUs. This is done by grabbing swapvec_lock and calling lru_add_drain_cpu().

PS: This is based on an idea and initial implementation by Rik van Riel.

I wrote maybe half the code in this patch. It should probably have my signed-off-by line too :)

Anyway, the patch looks fine to me and seems to work.

Quoted 147 lines

---
 include/linux/locallock.h | 27 +++++++++++++++++++++++++++ mm/swap.c                 | 35 +++++++++++++++++++++++++----------

 2 files changed, 52 insertions(+), 10 deletions(-)
include/linux/locallock.h +0 -0
index 6fe5928..2de478b 100644
@@ -104,6 +104,17 @@ static inline void __local_unlock(struct
 local_irq_lock *lv)
  		put_local_var(lvar);				
 \
  	} while (0)

include/linux/locallock.h +11 -0
+#define local_lock_other_cpu(lvar, cpu)                         \
+	do {                                                    \
+		__local_lock(&per_cpu(lvar, cpu));              \
+	} while (0)
+
+#define local_unlock_other_cpu(lvar, cpu)                       \
+	do {                                                    \
+		__local_unlock(&per_cpu(lvar, cpu));            \
+	} while (0)
+
+

 static inline void __local_lock_irq(struct local_irq_lock *lv) {
  spin_lock_irqsave(&lv->lock, lv->flags);

include/linux/locallock.h +0 -0
@@ -163,6 +174,22 @@ static inline int __local_lock_irqsave(struct
 local_irq_lock *lv)
  		_flags = per_cpu(lvar, cpu).flags;			
 \
  	} while (0)


+#define local_lock_irqsave_other_cpu(lvar, _flags, cpu)

	\
include/linux/locallock.h +1 -0
+	do {								

\

include/linux/locallock.h +1 -0
+		if (cpu == smp_processor_id())			
	\
include/linux/locallock.h +1 -0
+			local_lock_irqsave(lvar, _flags);		

\

include/linux/locallock.h +1 -0
+		else							

\

include/linux/locallock.h +1 -0
+			local_lock_other_cpu(lvar, cpu);		

\

include/linux/locallock.h +3 -0
+	} while (0)
+
+#define local_unlock_irqrestore_other_cpu(lvar, _flags, cpu)	

        \

include/linux/locallock.h +1 -0
+	do {								

\

include/linux/locallock.h +1 -0
+		if (cpu == smp_processor_id())			
	\
include/linux/locallock.h +1 -0
+			local_unlock_irqrestore(lvar, _flags);	
	\
include/linux/locallock.h +1 -0
+		else							

\

include/linux/locallock.h +1 -0
+			local_unlock_other_cpu(lvar, cpu);		

\

include/linux/locallock.h +2 -0
+	} while (0)
+

 static inline int __local_unlock_irqrestore(struct local_irq_lock *lv,
      unsigned long flags) {

mm/swap.c +0 -0
index ca194ae..9dc6956 100644
@@ -821,9 +821,9 @@ void lru_add_drain_cpu(int cpu)
  		unsigned long flags;


  /* No harm done if a racing interrupt already did this */

mm/swap.c +1 -1
-		local_lock_irqsave(rotate_lock, flags);
+		local_lock_irqsave_other_cpu(rotate_lock, flags,

cpu);
  pagevec_move_tail(pvec);

mm/swap.c +1 -1
-		local_unlock_irqrestore(rotate_lock, flags);
+		local_unlock_irqrestore_other_cpu(rotate_lock,

flags, cpu);
  }

  pvec = &per_cpu(lru_deactivate_file_pvecs, cpu);

mm/swap.c +0 -0
@@ -866,12 +866,32 @@ void lru_add_drain(void)
  	local_unlock_cpu(swapvec_lock);
  }

mm/swap.c +4 -0
+static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work);
+
+#ifdef CONFIG_PREEMPT_RT_BASE
+static inline void remote_lru_add_drain(int cpu, struct cpumask

*has_work)

mm/swap.c +6 -0
+{
+	local_lock_other_cpu(swapvec_lock, cpu);
+	lru_add_drain_cpu(cpu);
+	local_unlock_other_cpu(swapvec_lock, cpu);
+}
+#else

 static void lru_add_drain_per_cpu(struct work_struct *dummy) {
  lru_add_drain();
 }

mm/swap.c +1 -1
-static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work);
+static inline void remote_lru_add_drain(int cpu, struct cpumask

*has_work)

mm/swap.c +2 -0
+{
+		struct work_struct *work =

&per_cpu(lru_add_drain_work, cpu);

mm/swap.c +8 -0
+
+		INIT_WORK(work, lru_add_drain_per_cpu);
+		schedule_work_on(cpu, work);
+		cpumask_set_cpu(cpu, &has_work);
+
+}
+#endif
+


 void lru_add_drain_all(void)
 {

mm/swap.c +0 -0
@@ -884,16 +904,11 @@ void lru_add_drain_all(void)
  	cpumask_clear(&has_work);


  for_each_online_cpu(cpu) {

mm/swap.c +0 -1
-		struct work_struct *work =

&per_cpu(lru_add_drain_work, cpu);
-
  if (pagevec_count(&per_cpu(lru_add_pvec, cpu)) || pagevec_count(&per_cpu(lru_rotate_pvecs, cpu)) ||
      pagevec_count(&per_cpu(lru_deactivate_file_pvecs , cpu)) ||

mm/swap.c +2 -5
-		    need_activate_page_drain(cpu)) {
-			INIT_WORK(work, lru_add_drain_per_cpu);
-			schedule_work_on(cpu, work);
-			cpumask_set_cpu(cpu, &has_work);
-		}
+		    need_activate_page_drain(cpu))
+				remote_lru_add_drain(cpu,

&has_work);
  }

  for_each_cpu(cpu, &has_work)

Signature · 1 line
All Rights Reversed.
#2 lore raw
Quoted 7 lines · On Thu, 28 Apr 2016 18:47:16 -0400 Rik van Riel <[email protected]> wrote:
Quoted 6 lines · Luiz Capitulino

lru_add_drain_all() works by scheduling lru_add_drain_cpu() to run on all CPUs that have non-empty LRU pagevecs and then waiting for the scheduled work to complete. However, workqueue threads may never have the chance to run on a CPU that's running a SCHED_FIFO task. This causes lru_add_drain_all() to block forever.

Quoted 8 lines
Quoted 6 lines

This commit solves this problem by changing lru_add_drain_all() to drain the LRU pagevecs of remote CPUs. This is done by grabbing swapvec_lock and calling lru_add_drain_cpu().

PS: This is based on an idea and initial implementation by Rik van Riel.

I wrote maybe half the code in this patch.

And it was the hard part, sorry for not adding your signed-off-by.

Quoted 145 lines

It should
probably have my signed-off-by line too :)

Anyway, the patch looks fine to me and seems to work.

Quoted 139 lines

---
 include/linux/locallock.h | 27 +++++++++++++++++++++++++++ mm/swap.c                 | 35 +++++++++++++++++++++++++----------

 2 files changed, 52 insertions(+), 10 deletions(-)
include/linux/locallock.h +0 -0
index 6fe5928..2de478b 100644
@@ -104,6 +104,17 @@ static inline void __local_unlock(struct
 local_irq_lock *lv)
  		put_local_var(lvar);				
 \
  	} while (0)

include/linux/locallock.h +11 -0
+#define local_lock_other_cpu(lvar, cpu)                         \
+	do {                                                    \
+		__local_lock(&per_cpu(lvar, cpu));              \
+	} while (0)
+
+#define local_unlock_other_cpu(lvar, cpu)                       \
+	do {                                                    \
+		__local_unlock(&per_cpu(lvar, cpu));            \
+	} while (0)
+
+

 static inline void __local_lock_irq(struct local_irq_lock *lv) {
  spin_lock_irqsave(&lv->lock, lv->flags);

include/linux/locallock.h +0 -0
@@ -163,6 +174,22 @@ static inline int __local_lock_irqsave(struct
 local_irq_lock *lv)
  		_flags = per_cpu(lvar, cpu).flags;			
 \
  	} while (0)


+#define local_lock_irqsave_other_cpu(lvar, _flags, cpu)

	\
include/linux/locallock.h +1 -0
+	do {								

\

include/linux/locallock.h +1 -0
+		if (cpu == smp_processor_id())			
	\
include/linux/locallock.h +1 -0
+			local_lock_irqsave(lvar, _flags);		

\

include/linux/locallock.h +1 -0
+		else							

\

include/linux/locallock.h +1 -0
+			local_lock_other_cpu(lvar, cpu);		

\

include/linux/locallock.h +3 -0
+	} while (0)
+
+#define local_unlock_irqrestore_other_cpu(lvar, _flags, cpu)	

        \

include/linux/locallock.h +1 -0
+	do {								

\

include/linux/locallock.h +1 -0
+		if (cpu == smp_processor_id())			
	\
include/linux/locallock.h +1 -0
+			local_unlock_irqrestore(lvar, _flags);	
	\
include/linux/locallock.h +1 -0
+		else							

\

include/linux/locallock.h +1 -0
+			local_unlock_other_cpu(lvar, cpu);		

\

include/linux/locallock.h +2 -0
+	} while (0)
+

 static inline int __local_unlock_irqrestore(struct local_irq_lock *lv,
      unsigned long flags) {

mm/swap.c +0 -0
index ca194ae..9dc6956 100644
@@ -821,9 +821,9 @@ void lru_add_drain_cpu(int cpu)
  		unsigned long flags;


  /* No harm done if a racing interrupt already did this */

mm/swap.c +1 -1
-		local_lock_irqsave(rotate_lock, flags);
+		local_lock_irqsave_other_cpu(rotate_lock, flags,

cpu);
  pagevec_move_tail(pvec);

mm/swap.c +1 -1
-		local_unlock_irqrestore(rotate_lock, flags);
+		local_unlock_irqrestore_other_cpu(rotate_lock,

flags, cpu);
  }

  pvec = &per_cpu(lru_deactivate_file_pvecs, cpu);

mm/swap.c +0 -0
@@ -866,12 +866,32 @@ void lru_add_drain(void)
  	local_unlock_cpu(swapvec_lock);
  }

mm/swap.c +4 -0
+static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work);
+
+#ifdef CONFIG_PREEMPT_RT_BASE
+static inline void remote_lru_add_drain(int cpu, struct cpumask

*has_work)

mm/swap.c +6 -0
+{
+	local_lock_other_cpu(swapvec_lock, cpu);
+	lru_add_drain_cpu(cpu);
+	local_unlock_other_cpu(swapvec_lock, cpu);
+}
+#else

 static void lru_add_drain_per_cpu(struct work_struct *dummy) {
  lru_add_drain();
 }

mm/swap.c +1 -1
-static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work);
+static inline void remote_lru_add_drain(int cpu, struct cpumask

*has_work)

mm/swap.c +2 -0
+{
+		struct work_struct *work =

&per_cpu(lru_add_drain_work, cpu);

mm/swap.c +8 -0
+
+		INIT_WORK(work, lru_add_drain_per_cpu);
+		schedule_work_on(cpu, work);
+		cpumask_set_cpu(cpu, &has_work);
+
+}
+#endif
+


 void lru_add_drain_all(void)
 {

mm/swap.c +0 -0
@@ -884,16 +904,11 @@ void lru_add_drain_all(void)
  	cpumask_clear(&has_work);


  for_each_online_cpu(cpu) {

mm/swap.c +0 -1
-		struct work_struct *work =

&per_cpu(lru_add_drain_work, cpu);
-
  if (pagevec_count(&per_cpu(lru_add_pvec, cpu)) || pagevec_count(&per_cpu(lru_rotate_pvecs, cpu)) ||
      pagevec_count(&per_cpu(lru_deactivate_file_pvecs , cpu)) ||

mm/swap.c +0 -3
-		    need_activate_page_drain(cpu)) {
-			INIT_WORK(work, lru_add_drain_per_cpu);
-			schedule_work_on(cpu, work);
Quoted 8 lines
Quoted 8 lines
mm/swap.c +2 -2
-			cpumask_set_cpu(cpu, &has_work);
-		}
+		    need_activate_page_drain(cpu))
+				remote_lru_add_drain(cpu,

&has_work);
  }

  for_each_cpu(cpu, &has_work)

--
To unsubscribe from this list: send the line "unsubscribe linux-rt-users" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html