Skip to contentSkip to conversation

mm-make-vma-cache-slab_typesafe_by_rcu.patch removed from -mm tree

merged mm-stable Andrew Morton 1 participant 1 message Mar 17, 2025

lore
Share & tools
lore

Andrew Morton

author tree notice Applied
#0 lore raw

The quilt patch titled
Subject: mm: make vma cache SLAB_TYPESAFE_BY_RCU
has been removed from the -mm tree. Its filename was
mm-make-vma-cache-slab_typesafe_by_rcu.patch

This patch was dropped because it was merged into the mm-stable branch of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm

------------------------------------------------------
From: Suren Baghdasaryan <[email protected]>
Subject: mm: make vma cache SLAB_TYPESAFE_BY_RCU
Date: Thu, 13 Feb 2025 14:46:54 -0800

To enable SLAB_TYPESAFE_BY_RCU for vma cache we need to ensure that object reuse before RCU grace period is over will be detected by lock_vma_under_rcu().

Current checks are sufficient as long as vma is detached before it is freed. The only place this is not currently happening is in exit_mmap(). Add the missing vma_mark_detached() in exit_mmap().

Another issue which might trick lock_vma_under_rcu() during vma reuse is vm_area_dup(), which copies the entire content of the vma into a new one, overriding new vma's vm_refcnt and temporarily making it appear as attached. This might trick a racing lock_vma_under_rcu() to operate on a reused vma if it found the vma before it got reused. To prevent this situation, we should ensure that vm_refcnt stays at detached state (0) when it is copied and advances to attached state only after it is added into the vma tree. Introduce vm_area_init_from() which preserves new vma's vm_refcnt and use it in vm_area_dup(). Since all vmas are in detached state with no current readers when they are freed,

lock_vma_under_rcu() will not be able to take vm_refcnt after vma got detached even if vma is reused. vma_mark_attached() in modified to include a release fence to ensure all stores to the vma happen before vm_refcnt gets initialized.

Finally, make vm_area_cachep SLAB_TYPESAFE_BY_RCU. This will facilitate vm_area_struct reuse and will minimize the number of call_rcu() calls.

[[email protected]: remove atomic_set_release() usage in tools/]
Cc Christian Brauner <[email protected]>
Cc David Hildenbrand <[email protected]>
Cc David Howells <[email protected]>
Cc Davidlohr Bueso <[email protected]>
Cc Hugh Dickins <[email protected]>
Cc Jann Horn <[email protected]>
Cc Johannes Weiner <[email protected]>
Cc Jonathan Corbet <[email protected]>
Cc Klara Modin <[email protected]>
Cc Liam R. Howlett <[email protected]>
Cc Lokesh Gidra <[email protected]>
Cc Lorenzo Stoakes <[email protected]>
Cc Mateusz Guzik <[email protected]>
Cc Matthew Wilcox <[email protected]>
Cc Mel Gorman <[email protected]>
Cc Michal Hocko <[email protected]>
Cc Minchan Kim <[email protected]>
Cc Oleg Nesterov <[email protected]>
Cc Pasha Tatashin <[email protected]>
Cc "Paul E . McKenney" <[email protected]>
Cc Peter Xu <[email protected]>
Cc Peter Zijlstra (Intel) <[email protected]>
Cc Shakeel Butt <[email protected]>
Cc Sourav Panda <[email protected]>
Cc Wei Yang <[email protected]>
Cc Will Deacon <[email protected]>
Cc Heiko Carstens <[email protected]>
Cc Stephen Rothwell <[email protected]>

---

 include/linux/mm.h               |    4 -
 include/linux/mm_types.h         |   13 +++--
 include/linux/slab.h             |    6 --
 kernel/fork.c                    |   73 ++++++++++++++++++-----------
 mm/mmap.c                        |    3 -
 mm/vma.c                         |   11 +---
 mm/vma.h                         |    2 
 tools/include/linux/refcount.h   |    5 +
 tools/testing/vma/vma_internal.h |    9 ---
 9 files changed, 70 insertions(+), 56 deletions(-)
include/linux/mm.h renamed from include/linux/mm.h~mm-make-vma-cache-slab_typesafe_by_rcu +1 -3
@@ -258,8 +258,6 @@ void setup_initial_init_mm(void *start_c
 struct vm_area_struct *vm_area_alloc(struct mm_struct *);
 struct vm_area_struct *vm_area_dup(struct vm_area_struct *);
 void vm_area_free(struct vm_area_struct *);
-/* Use only if VMA has no other users */
-void __vm_area_free(struct vm_area_struct *vma);
 
 #ifndef CONFIG_MMU
 extern struct rb_root nommu_region_tree;
@@ -890,7 +888,7 @@ static inline void vma_mark_attached(str
 {
 	vma_assert_write_locked(vma);
 	vma_assert_detached(vma);
-	refcount_set(&vma->vm_refcnt, 1);
+	refcount_set_release(&vma->vm_refcnt, 1);
 }
 
 void vma_mark_detached(struct vm_area_struct *vma);
include/linux/mm_types.h renamed from include/linux/mm_types.h~mm-make-vma-cache-slab_typesafe_by_rcu +10 -3
@@ -575,6 +575,12 @@ static inline void *folio_get_private(st
 typedef unsigned long vm_flags_t;
 
 /*
+ * freeptr_t represents a SLUB freelist pointer, which might be encoded
+ * and not dereferenceable if CONFIG_SLAB_FREELIST_HARDENED is enabled.
+ */
+typedef struct { unsigned long v; } freeptr_t;
+
+/*
  * A region containing a mapping of a non-memory backed file under NOMMU
  * conditions.  These are held in a global tree and are pinned by the VMAs that
  * map parts of them.
@@ -677,6 +683,9 @@ struct vma_numab_state {
  *
  * Only explicitly marked struct members may be accessed by RCU readers before
  * getting a stable reference.
+ *
+ * WARNING: when adding new members, please update vm_area_init_from() to copy
+ * them during vm_area_struct content duplication.
  */
 struct vm_area_struct {
 	/* The first cache line has the info for VMA tree walking. */
@@ -687,9 +696,7 @@ struct vm_area_struct {
 			unsigned long vm_start;
 			unsigned long vm_end;
 		};
-#ifdef CONFIG_PER_VMA_LOCK
-		struct rcu_head vm_rcu;	/* Used for deferred freeing. */
-#endif
+		freeptr_t vm_freeptr; /* Pointer used by SLAB_TYPESAFE_BY_RCU */
 	};
 
 	/*
include/linux/slab.h renamed from include/linux/slab.h~mm-make-vma-cache-slab_typesafe_by_rcu +0 -6
@@ -244,12 +244,6 @@ enum _slab_flag_bits {
 #endif
 
 /*
- * freeptr_t represents a SLUB freelist pointer, which might be encoded
- * and not dereferenceable if CONFIG_SLAB_FREELIST_HARDENED is enabled.
- */
-typedef struct { unsigned long v; } freeptr_t;
-
-/*
  * ZERO_SIZE_PTR will be returned for zero sized kmalloc requests.
  *
  * Dereferencing ZERO_SIZE_PTR will lead to a distinct access fault.
kernel/fork.c renamed from kernel/fork.c~mm-make-vma-cache-slab_typesafe_by_rcu +46 -27

109 lines not on this page · raw message

mm/mmap.c renamed from mm/mmap.c~mm-make-vma-cache-slab_typesafe_by_rcu +2 -1
@@ -1305,7 +1305,8 @@ void exit_mmap(struct mm_struct *mm)
 	do {
 		if (vma->vm_flags & VM_ACCOUNT)
 			nr_accounted += vma_pages(vma);
-		remove_vma(vma, /* unreachable = */ true);
+		vma_mark_detached(vma);
+		remove_vma(vma);
 		count++;
 		cond_resched();
 		vma = vma_next(&vmi);
mm/vma.c renamed from mm/vma.c~mm-make-vma-cache-slab_typesafe_by_rcu +3 -8
@@ -420,19 +420,14 @@ static bool can_vma_merge_right(struct v
 /*
  * Close a vm structure and free it.
  */
-void remove_vma(struct vm_area_struct *vma, bool unreachable)
+void remove_vma(struct vm_area_struct *vma)
 {
 	might_sleep();
 	vma_close(vma);
 	if (vma->vm_file)
 		fput(vma->vm_file);
 	mpol_put(vma_policy(vma));
-	if (unreachable) {
-		vma_mark_detached(vma);
-		__vm_area_free(vma);
-	} else {
-		vm_area_free(vma);
-	}
+	vm_area_free(vma);
 }
 
 /*
@@ -1218,7 +1213,7 @@ static void vms_complete_munmap_vmas(str
 	/* Remove and clean up vmas */
 	mas_set(mas_detach, 0);
 	mas_for_each(mas_detach, vma, ULONG_MAX)
-		remove_vma(vma, /* unreachable = */ false);
+		remove_vma(vma);
 
 	vm_unacct_memory(vms->nr_accounted);
 	validate_mm(mm);
mm/vma.h renamed from mm/vma.h~mm-make-vma-cache-slab_typesafe_by_rcu +1 -1
@@ -218,7 +218,7 @@ int do_vmi_munmap(struct vma_iterator *v
 		  unsigned long start, size_t len, struct list_head *uf,
 		  bool unlock);
 
-void remove_vma(struct vm_area_struct *vma, bool unreachable);
+void remove_vma(struct vm_area_struct *vma);
 
 void unmap_region(struct ma_state *mas, struct vm_area_struct *vma,
 		struct vm_area_struct *prev, struct vm_area_struct *next);
tools/include/linux/refcount.h renamed from tools/include/linux/refcount.h~mm-make-vma-cache-slab_typesafe_by_rcu +5 -0
@@ -60,6 +60,11 @@ static inline void refcount_set(refcount
 	atomic_set(&r->refs, n);
 }
 
+static inline void refcount_set_release(refcount_t *r, unsigned int n)
+{
+	atomic_set(&r->refs, n);
+}
+
 static inline unsigned int refcount_read(const refcount_t *r)
 {
 	return atomic_read(&r->refs);
tools/testing/vma/vma_internal.h renamed from tools/testing/vma/vma_internal.h~mm-make-vma-cache-slab_typesafe_by_rcu +2 -7
@@ -476,7 +476,7 @@ static inline void vma_mark_attached(str
 {
 	vma_assert_write_locked(vma);
 	vma_assert_detached(vma);
-	refcount_set(&vma->vm_refcnt, 1);
+	refcount_set_release(&vma->vm_refcnt, 1);
 }
 
 static inline void vma_mark_detached(struct vm_area_struct *vma)
@@ -696,14 +696,9 @@ static inline void mpol_put(struct mempo
 {
 }
 
-static inline void __vm_area_free(struct vm_area_struct *vma)
-{
-	free(vma);
-}
-
 static inline void vm_area_free(struct vm_area_struct *vma)
 {
-	__vm_area_free(vma);
+	free(vma);
 }
 
 static inline void lru_add_drain(void)

_

Patches currently in -mm which might be from [email protected] are