mm-make-vma-cache-slab_typesafe_by_rcu.patch removed from -mm tree
The quilt patch titled
Subject: mm: make vma cache SLAB_TYPESAFE_BY_RCU
has been removed from the -mm tree. Its filename was
mm-make-vma-cache-slab_typesafe_by_rcu.patch
This patch was dropped because it was merged into the mm-stable branch of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
------------------------------------------------------
From: Suren Baghdasaryan <[email protected]>
Subject: mm: make vma cache SLAB_TYPESAFE_BY_RCU
Date: Thu, 13 Feb 2025 14:46:54 -0800
To enable SLAB_TYPESAFE_BY_RCU for vma cache we need to ensure that object reuse before RCU grace period is over will be detected by lock_vma_under_rcu().
Current checks are sufficient as long as vma is detached before it is freed. The only place this is not currently happening is in exit_mmap(). Add the missing vma_mark_detached() in exit_mmap().
Another issue which might trick lock_vma_under_rcu() during vma reuse is vm_area_dup(), which copies the entire content of the vma into a new one, overriding new vma's vm_refcnt and temporarily making it appear as attached. This might trick a racing lock_vma_under_rcu() to operate on a reused vma if it found the vma before it got reused. To prevent this situation, we should ensure that vm_refcnt stays at detached state (0) when it is copied and advances to attached state only after it is added into the vma tree. Introduce vm_area_init_from() which preserves new vma's vm_refcnt and use it in vm_area_dup(). Since all vmas are in detached state with no current readers when they are freed,
lock_vma_under_rcu() will not be able to take vm_refcnt after vma got detached even if vma is reused. vma_mark_attached() in modified to include a release fence to ensure all stores to the vma happen before vm_refcnt gets initialized.
Finally, make vm_area_cachep SLAB_TYPESAFE_BY_RCU. This will facilitate vm_area_struct reuse and will minimize the number of call_rcu() calls.
---
include/linux/mm.h | 4 - include/linux/mm_types.h | 13 +++-- include/linux/slab.h | 6 -- kernel/fork.c | 73 ++++++++++++++++++----------- mm/mmap.c | 3 - mm/vma.c | 11 +--- mm/vma.h | 2 tools/include/linux/refcount.h | 5 + tools/testing/vma/vma_internal.h | 9 --- 9 files changed, 70 insertions(+), 56 deletions(-)
include/linux/mm.h renamed from include/linux/mm.h~mm-make-vma-cache-slab_typesafe_by_rcu +1 -3
@@ -258,8 +258,6 @@ void setup_initial_init_mm(void *start_cstruct vm_area_struct *vm_area_alloc (struct mm_struct *);struct vm_area_struct *vm_area_dup (struct vm_area_struct *);void vm_area_free (struct vm_area_struct *); -/* Use only if VMA has no other users */ -void __vm_area_free (struct vm_area_struct *vma);#ifndef CONFIG_MMU extern struct rb_root nommu_region_tree;
@@ -890,7 +888,7 @@ static inline void vma_mark_attached(str {vma_assert_write_locked (vma);vma_assert_detached (vma); -refcount_set (&vma->vm_refcnt,1 ); +refcount_set_release (&vma->vm_refcnt,1 ); }void vma_mark_detached (struct vm_area_struct *vma);
include/linux/mm_types.h renamed from include/linux/mm_types.h~mm-make-vma-cache-slab_typesafe_by_rcu +10 -3
@@ -575,6 +575,12 @@ static inline void *folio_get_private(sttypedef unsigned long vm_flags_t ;/* +* freeptr_t represents a SLUB freelist pointer, which might be encoded +* and not dereferenceable if CONFIG_SLAB_FREELIST_HARDENED is enabled. +*/ +typedef struct {unsigned long v; }freeptr_t ; + +/* * A region containing a mapping of a non-memory backed file under NOMMU * conditions. These are held in a global tree and are pinned by the VMAs that * map parts of them.
@@ -677,6 +683,9 @@ struct vma_numab_state {* * Only explicitly marked struct members may be accessed by RCU readers before * getting a stable reference. +* +* WARNING: when adding new members, please update vm_area_init_from() to copy +* them during vm_area_struct content duplication. */ struct vm_area_struct {/* The first cache line has the info for VMA tree walking. */
@@ -687,9 +696,7 @@ struct vm_area_struct {unsigned long vm_start;unsigned long vm_end; }; -#ifdef CONFIG_PER_VMA_LOCK -struct rcu_head vm_rcu;/* Used for deferred freeing. */ -#endif +freeptr_t vm_freeptr;/* Pointer used by SLAB_TYPESAFE_BY_RCU */ };/*
include/linux/slab.h renamed from include/linux/slab.h~mm-make-vma-cache-slab_typesafe_by_rcu +0 -6
@@ -244,12 +244,6 @@ enum _slab_flag_bits {#endif /* -* freeptr_t represents a SLUB freelist pointer, which might be encoded -* and not dereferenceable if CONFIG_SLAB_FREELIST_HARDENED is enabled. -*/ -typedef struct {unsigned long v; }freeptr_t ; - -/* * ZERO_SIZE_PTR will be returned for zero sized kmalloc requests. * * Dereferencing ZERO_SIZE_PTR will lead to a distinct access fault.
kernel/fork.c renamed from kernel/fork.c~mm-make-vma-cache-slab_typesafe_by_rcu +46 -27
109 lines not on this page · raw message
mm/mmap.c renamed from mm/mmap.c~mm-make-vma-cache-slab_typesafe_by_rcu +2 -1
@@ -1305,7 +1305,8 @@ void exit_mmap(struct mm_struct *mm) do { if (vma->vm_flags & VM_ACCOUNT) nr_accounted += vma_pages(vma); - remove_vma(vma, /* unreachable = */ true); + vma_mark_detached(vma); + remove_vma(vma); count++; cond_resched(); vma = vma_next(&vmi);
mm/vma.c renamed from mm/vma.c~mm-make-vma-cache-slab_typesafe_by_rcu +3 -8
@@ -420,19 +420,14 @@ static bool can_vma_merge_right(struct v/* * Close a vm structure and free it. */ -void remove_vma (struct vm_area_struct *vma,bool unreachable) +void remove_vma (struct vm_area_struct *vma) {might_sleep ();vma_close (vma);if (vma->vm_file)fput (vma->vm_file);mpol_put (vma_policy (vma)); -if (unreachable) { -vma_mark_detached (vma); -__vm_area_free (vma); - }else { -vm_area_free (vma); - } +vm_area_free (vma); }/*
@@ -1218,7 +1213,7 @@ static void vms_complete_munmap_vmas(str/* Remove and clean up vmas */ mas_set (mas_detach,0 );mas_for_each (mas_detach, vma,ULONG_MAX ) -remove_vma (vma,/* unreachable = */ false); +remove_vma (vma);vm_unacct_memory (vms->nr_accounted);validate_mm (mm);
mm/vma.h renamed from mm/vma.h~mm-make-vma-cache-slab_typesafe_by_rcu +1 -1
@@ -218,7 +218,7 @@ int do_vmi_munmap(struct vma_iterator *v unsigned long start, size_t len, struct list_head *uf, bool unlock); -void remove_vma (struct vm_area_struct *vma,bool unreachable); +void remove_vma (struct vm_area_struct *vma);void unmap_region (struct ma_state *mas,struct vm_area_struct *vma,struct vm_area_struct *prev,struct vm_area_struct *next);
tools/include/linux/refcount.h renamed from tools/include/linux/refcount.h~mm-make-vma-cache-slab_typesafe_by_rcu +5 -0
@@ -60,6 +60,11 @@ static inline void refcount_set(refcountatomic_set (&r->refs, n); } +static inline void refcount_set_release (refcount_t *r,unsigned int n) +{ +atomic_set (&r->refs, n); +} +static inline unsigned int refcount_read (const refcount_t *r) {return atomic_read (&r->refs);
tools/testing/vma/vma_internal.h renamed from tools/testing/vma/vma_internal.h~mm-make-vma-cache-slab_typesafe_by_rcu +2 -7
@@ -476,7 +476,7 @@ static inline void vma_mark_attached(str {vma_assert_write_locked (vma);vma_assert_detached (vma); -refcount_set (&vma->vm_refcnt,1 ); +refcount_set_release (&vma->vm_refcnt,1 ); }static inline void vma_mark_detached (struct vm_area_struct *vma)
@@ -696,14 +696,9 @@ static inline void mpol_put(struct mempo { } -static inline void __vm_area_free (struct vm_area_struct *vma) -{ -free (vma); -} -static inline void vm_area_free (struct vm_area_struct *vma) { -__vm_area_free (vma); +free (vma); }static inline void lru_add_drain (void )
_
Patches currently in -mm which might be from [email protected] are