mm/munlock: protect the per-CPU pagevec by a local_lock_t

PATCH New Andrew Morton 1 participant 1 message Apr 1, 2022

Subsystems (6)

Memory management MM core Page allocator Memory mapping +2 more subsystems

lore

Share & tools

lore

Andrew Morton

author maintainer

2022-04-01 18:21 UTC #0 lore raw

PATCH 8/16 mm/munlock: protect the per-CPU pagevec by a local_lock_t

From: Sebastian Andrzej Siewior <[email protected]>
Subject: mm/munlock: protect the per-CPU pagevec by a local_lock_t

The access to mlock_pvec is protected by disabling preemption via get_cpu_var() or implicit by having preemption disabled by the caller (in mlock_page_drain() case). This breaks on PREEMPT_RT since folio_lruvec_lock_irq() acquires a sleeping lock in this section.

Create struct mlock_pvec which consits of the local_lock_t and the pagevec. Acquire the local_lock() before accessing the per-CPU pagevec. Replace mlock_page_drain() with a _local() version which is invoked on the local CPU and acquires the local_lock_t and a _remote() version which uses the pagevec from a remote CPU which offline.

Link https://lkml.kernel.org/r/[email protected] open here

Signed-off-by Sebastian Andrzej Siewior <[email protected]>

Acked-by Hugh Dickins <[email protected]>

Cc Vlastimil Babka <[email protected]>

Cc Matthew Wilcox <[email protected]>

Cc Thomas Gleixner <[email protected]>

Signed-off-by Andrew Morton <[email protected]>

---

 mm/internal.h   |    6 ++++--
 mm/migrate.c    |    2 +-
 mm/mlock.c      |   46 ++++++++++++++++++++++++++++++++++++----------
 mm/page_alloc.c |    1 +
 mm/rmap.c       |    4 ++--
 mm/swap.c       |    4 +++-
 6 files changed, 47 insertions(+), 16 deletions(-)

mm/internal.h renamed from mm/internal.h~mm-munlock-protect-the-per-cpu-pagevec-by-a-local_lock_t +4 -2

@@ -456,7 +456,8 @@ static inline void munlock_vma_page(stru
 }
 void mlock_new_page(struct page *page);
 bool need_mlock_page_drain(int cpu);
-void mlock_page_drain(int cpu);
+void mlock_page_drain_local(void);
+void mlock_page_drain_remote(int cpu);
 
 extern pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma);

@@ -539,7 +540,8 @@ static inline void munlock_vma_page(stru
 			struct vm_area_struct *vma, bool compound) { }
 static inline void mlock_new_page(struct page *page) { }
 static inline bool need_mlock_page_drain(int cpu) { return false; }
-static inline void mlock_page_drain(int cpu) { }
+static inline void mlock_page_drain_local(void) { }
+static inline void mlock_page_drain_remote(int cpu) { }
 static inline void vunmap_range_noflush(unsigned long start, unsigned long end)
 {
 }

mm/migrate.c renamed from mm/migrate.c~mm-munlock-protect-the-per-cpu-pagevec-by-a-local_lock_t +1 -1

@@ -246,7 +246,7 @@ static bool remove_migration_pte(struct
 			set_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte);
 		}
 		if (vma->vm_flags & VM_LOCKED)
-			mlock_page_drain(smp_processor_id());
+			mlock_page_drain_local();
 
 		trace_remove_migration_pte(pvmw.address, pte_val(pte),
 					   compound_order(new));

mm/mlock.c renamed from mm/mlock.c~mm-munlock-protect-the-per-cpu-pagevec-by-a-local_lock_t +36 ~~-10~~

106 lines not on this page · raw message

mm/page_alloc.c renamed from mm/page_alloc.c~mm-munlock-protect-the-per-cpu-pagevec-by-a-local_lock_t +1 -0

@@ -8367,6 +8367,7 @@ static int page_alloc_cpu_dead(unsigned
 	struct zone *zone;
 
 	lru_add_drain_cpu(cpu);
+	mlock_page_drain_remote(cpu);
 	drain_pages(cpu);
 
 	/*

mm/rmap.c renamed from mm/rmap.c~mm-munlock-protect-the-per-cpu-pagevec-by-a-local_lock_t +2 -2

@@ -1683,7 +1683,7 @@ discard:
 		 */
 		page_remove_rmap(subpage, vma, folio_test_hugetlb(folio));
 		if (vma->vm_flags & VM_LOCKED)
-			mlock_page_drain(smp_processor_id());
+			mlock_page_drain_local();
 		folio_put(folio);
 	}

@@ -1961,7 +1961,7 @@ static bool try_to_migrate_one(struct fo
 		 */
 		page_remove_rmap(subpage, vma, folio_test_hugetlb(folio));
 		if (vma->vm_flags & VM_LOCKED)
-			mlock_page_drain(smp_processor_id());
+			mlock_page_drain_local();
 		folio_put(folio);
 	}

mm/swap.c renamed from mm/swap.c~mm-munlock-protect-the-per-cpu-pagevec-by-a-local_lock_t +3 -1

@@ -624,7 +624,6 @@ void lru_add_drain_cpu(int cpu)
 		pagevec_lru_move_fn(pvec, lru_lazyfree_fn);
 
 	activate_page_drain(cpu);
-	mlock_page_drain(cpu);
 }
 
 /**

@@ -706,6 +705,7 @@ void lru_add_drain(void)
 	local_lock(&lru_pvecs.lock);
 	lru_add_drain_cpu(smp_processor_id());
 	local_unlock(&lru_pvecs.lock);
+	mlock_page_drain_local();
 }
 
 /*

@@ -720,6 +720,7 @@ static void lru_add_and_bh_lrus_drain(vo
 	lru_add_drain_cpu(smp_processor_id());
 	local_unlock(&lru_pvecs.lock);
 	invalidate_bh_lrus_cpu();
+	mlock_page_drain_local();
 }
 
 void lru_add_drain_cpu_zone(struct zone *zone)

@@ -728,6 +729,7 @@ void lru_add_drain_cpu_zone(struct zone
 	lru_add_drain_cpu(smp_processor_id());
 	drain_local_pages(zone);
 	local_unlock(&lru_pvecs.lock);
+	mlock_page_drain_local();
 }
 
 #ifdef CONFIG_SMP