mm: convert mm_lock_seq to a proper seqcount
Subsystems (6)
Suren Baghdasaryan
maintainerPATCH 1/2 mm: convert mm_lock_seq to a proper seqcount
Convert mm_lock_seq to be seqcount_t and change all mmap_write_lock variants to increment it, in-line with the usual seqcount usage pattern. This lets us check whether the mmap_lock is write-locked by checking mm_lock_seq.sequence counter (odd=locked, even=unlocked). This will be used when implementing mmap_lock speculation functions.
As a result vm_lock_seq is also change to be unsigned to match the type of mm_lock_seq.sequence.
---
Applies over mm-unstable
This conversion was discussed at [1] and these patches will likely be incorporated into the next version od Andrii's patcset.
The issue of the seqcount_t.sequence being an unsigned rather than unsigned long will be addressed separately in collaoration with Jann Horn.
[1] https://lore.kernel.org/all/[email protected]/
include/linux/mm.h | 12 +++---- include/linux/mm_types.h | 7 ++-- include/linux/mmap_lock.h | 58 +++++++++++++++++++++----------- kernel/fork.c | 5 +-- mm/init-mm.c | 2 +- tools/testing/vma/vma.c | 4 +-- tools/testing/vma/vma_internal.h | 4 +-- 7 files changed, 56 insertions(+), 36 deletions(-)
include/linux/mm.h +6 -6
@@ -698,7 +698,7 @@ static inline bool vma_start_read(struct vm_area_struct *vma)* we don't rely on for anything - the mm_lock_seq read against which we * need ordering is below. */ -if (READ_ONCE (vma->vm_lock_seq) ==READ_ONCE (vma->vm_mm->mm_lock_seq)) +if (READ_ONCE (vma->vm_lock_seq) ==READ_ONCE (vma->vm_mm->mm_lock_seq.sequence))return false;if (unlikely (down_read_trylock (&vma->vm_lock->lock) ==0 ))
@@ -715,7 +715,7 @@ static inline bool vma_start_read(struct vm_area_struct *vma)* after it has been unlocked. * This pairs with RELEASE semantics in vma_end_write_all(). */ -if (unlikely (vma->vm_lock_seq ==smp_load_acquire (&vma->vm_mm->mm_lock_seq))) { +if (unlikely (vma->vm_lock_seq ==raw_read_seqcount (&vma->vm_mm->mm_lock_seq))) {up_read (&vma->vm_lock->lock);return false; }
@@ -730,7 +730,7 @@ static inline void vma_end_read(struct vm_area_struct *vma) }/* WARNING! Can only be used if mmap_lock is expected to be write-locked */ -static bool __is_vma_write_locked (struct vm_area_struct *vma,int *mm_lock_seq) +static bool __is_vma_write_locked (struct vm_area_struct *vma,unsigned int *mm_lock_seq) {mmap_assert_write_locked (vma->vm_mm);
@@ -738,7 +738,7 @@ static bool __is_vma_write_locked(struct vm_area_struct *vma, int *mm_lock_seq)* current task is holding mmap_write_lock, both vma->vm_lock_seq and * mm->mm_lock_seq can't be concurrently modified. */ - *mm_lock_seq = vma->vm_mm->mm_lock_seq; + *mm_lock_seq = vma->vm_mm->mm_lock_seq.sequence;return (vma->vm_lock_seq == *mm_lock_seq); }
@@ -749,7 +749,7 @@ static bool __is_vma_write_locked(struct vm_area_struct *vma, int *mm_lock_seq)*/ static inline void vma_start_write (struct vm_area_struct *vma) { -int mm_lock_seq; +unsigned int mm_lock_seq;if (__is_vma_write_locked (vma, &mm_lock_seq))return ;
@@ -767,7 +767,7 @@ static inline void vma_start_write(struct vm_area_struct *vma)static inline void vma_assert_write_locked (struct vm_area_struct *vma) { -int mm_lock_seq; +unsigned int mm_lock_seq;VM_BUG_ON_VMA (!__is_vma_write_locked (vma, &mm_lock_seq), vma); }
include/linux/mm_types.h +5 -2
@@ -715,7 +715,7 @@ struct vm_area_struct {* counter reuse can only lead to occasional unnecessary use of the * slowpath. */ -int vm_lock_seq; +unsigned int vm_lock_seq;/* Unstable RCU readers are allowed to read this. */ struct vma_lock *vm_lock;#endif
@@ -887,6 +887,9 @@ struct mm_struct {* Roughly speaking, incrementing the sequence number is * equivalent to releasing locks on VMAs; reading the sequence * number can be part of taking a read lock on a VMA. +* Incremented every time mmap_lock is write-locked/unlocked. +* Initialized to 0, therefore odd values indicate mmap_lock +* is write-locked and even values that it's released. * * Can be modified under write mmap_lock using RELEASE * semantics.
@@ -895,7 +898,7 @@ struct mm_struct {* Can be read with ACQUIRE semantics if not holding write * mmap_lock. */ -int mm_lock_seq; +seqcount_t mm_lock_seq;#endif
include/linux/mmap_lock.h +39 -19
94 lines not on this page · raw message
kernel/fork.c +1 -4
@@ -447,7 +447,7 @@ static bool vma_lock_alloc(struct vm_area_struct *vma)return false;init_rwsem (&vma->vm_lock->lock); - vma->vm_lock_seq =-1 ; + vma->vm_lock_seq =UINT_MAX ;return true; }
@@ -1260,9 +1260,6 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,seqcount_init (&mm->write_protect_seq);mmap_init_lock (mm);INIT_LIST_HEAD (&mm->mmlist); -#ifdef CONFIG_PER_VMA_LOCK - mm->mm_lock_seq =0 ; -#endif mm_pgtables_bytes_init (mm); mm->map_count =0 ; mm->locked_vm =0 ;
mm/init-mm.c +1 -1
@@ -40,7 +40,7 @@ struct mm_struct init_mm = { .arg_lock = __SPIN_LOCK_UNLOCKED(init_mm.arg_lock), .mmlist =LIST_HEAD_INIT (init_mm.mmlist), #ifdef CONFIG_PER_VMA_LOCK - .mm_lock_seq = 0, + .mm_lock_seq = SEQCNT_ZERO(init_mm.mm_lock_seq), #endif .user_ns = &init_user_ns, .cpu_bitmap = CPU_BITS_NONE,
tools/testing/vma/vma.c +2 -2
@@ -89,7 +89,7 @@ static struct vm_area_struct *alloc_and_link_vma(struct mm_struct *mm,* begun. Linking to the tree will have caused this to be incremented, * which means we will get a false positive otherwise. */ - vma->vm_lock_seq =-1 ; + vma->vm_lock_seq =UINT_MAX ;return vma; }
@@ -214,7 +214,7 @@ static bool vma_write_started(struct vm_area_struct *vma)int seq = vma->vm_lock_seq;/* We reset after each check. */ - vma->vm_lock_seq =-1 ; + vma->vm_lock_seq =UINT_MAX ;/* The vma_start_write() stub simply increments this value. */ return seq >-1 ;
tools/testing/vma/vma_internal.h +2 -2
@@ -241,7 +241,7 @@ struct vm_area_struct {* counter reuse can only lead to occasional unnecessary use of the * slowpath. */ -int vm_lock_seq; +unsigned int vm_lock_seq;struct vma_lock *vm_lock;#endif
@@ -416,7 +416,7 @@ static inline bool vma_lock_alloc(struct vm_area_struct *vma)return false;init_rwsem (&vma->vm_lock->lock); - vma->vm_lock_seq =-1 ; + vma->vm_lock_seq =UINT_MAX ;return true; }
base-commit: 9c111059234a949a4d3442a413ade19cc65ab927
Signature · 1 line
2.47.0.163.g1226f6d8fa-goog
Suren Baghdasaryan
maintainerPATCH 2/2 mm: introduce mmap_lock_speculation_{begin|end}
Add helper functions to speculatively perform operations without read-locking mmap_lock, expecting that mmap_lock will not be write-locked and mm is not modified from under us.
--- include/linux/mmap_lock.h | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-)
include/linux/mmap_lock.h +27 -2
@@ -71,6 +71,7 @@ static inline void mmap_assert_write_locked(const struct mm_struct *mm) }#ifdef CONFIG_PER_VMA_LOCK +static inline void mm_lock_seqcount_init (struct mm_struct *mm) {seqcount_init (&mm->mm_lock_seq);
@@ -86,11 +87,35 @@ static inline void mm_lock_seqcount_end(struct mm_struct *mm)do_raw_write_seqcount_end (&mm->mm_lock_seq); } -#else +static inline bool mmap_lock_speculation_begin (struct mm_struct *mm,unsigned int *seq) +{ + *seq =raw_read_seqcount (&mm->mm_lock_seq); +/* Allow speculation if mmap_lock is not write-locked */ +return (*seq &1 ) ==0 ; +} + +static inline bool mmap_lock_speculation_end (struct mm_struct *mm,unsigned int seq) +{ +return !do_read_seqcount_retry (&mm->mm_lock_seq, seq); +} + +#else /* CONFIG_PER_VMA_LOCK */ +static inline void mm_lock_seqcount_init (struct mm_struct *mm) {}static inline void mm_lock_seqcount_begin (struct mm_struct *mm) {}static inline void mm_lock_seqcount_end (struct mm_struct *mm) {} -#endif + +static inline bool mmap_lock_speculation_begin (struct mm_struct *mm,unsigned int *seq) +{ +return false; +} + +static inline bool mmap_lock_speculation_end (struct mm_struct *mm,unsigned int seq) +{ +return false; +} + +#endif /* CONFIG_PER_VMA_LOCK */ static inline void mmap_init_lock (struct mm_struct *mm) {
Signature · 1 line
2.47.0.163.g1226f6d8fa-goog
Quoted 9 lines · Suren Baghdasaryan
Convert
mm_lock_seqto beseqcount_tand change allmmap_write_lockvariants to increment it, in-line with the usual seqcount usage pattern. This lets us check whether themmap_lockis write-locked by checking mm_lock_seq.sequence counter (odd=locked, even=unlocked). This will be used when implementingmmap_lockspeculation functions.
As a resultvm_lock_seqis also change to be unsigned to match the type of mm_lock_seq.sequence.
Quoted 7 lines
Suggested-by Peter Zijlstra <[email protected]>Signed-off-by Suren Baghdasaryan <[email protected]>---
Applies over mm-unstableThis conversion was discussed at [1] and these patches will likely be incorporated into the next version od Andrii's patcset.
I got a notification that this patch set was applied to mm-unstable by Andrew. But I was wondering if Andrew and Peter would agree to move the patches into tip's perf/core branch, given this is a dependency of my pending uprobe series ([0]) and, as far as I'm aware, there is no urgent need for this API in mm tree(s). If we route this through mm-unstable, we either need a stable tag from mm tree for Peter to merge into perf/core (a bit of a hassle for both Andrew and Peter, I'm sure), or I'd have to wait for 5-6 weeks until after next merge window closes, which would be a huge bummer, given I'd been at this for a while already with basically done patches, and would prefer to get my changes sooner.
So, I'd very much prefer to just route these changes through perf/core, if mm folks don't oppose this? In fact, I'll go ahead and will send my patches with Suren's patches included with the assumption that we can reroute all this. Thanks for understanding!
P.S. And yeah, Suren's patches apply cleanly to perf/core just as well, I checked.
[0] https://lore.kernel.org/linux-trace-kernel/[email protected]/
Quoted 290 lines
The issue of the seqcount_t.sequence being an unsigned rather than unsigned long will be addressed separately in collaoration with Jann Horn.
[1] https://lore.kernel.org/all/[email protected]/
include/linux/mm.h | 12 +++---- include/linux/mm_types.h | 7 ++-- include/linux/mmap_lock.h | 58 +++++++++++++++++++++----------- kernel/fork.c | 5 +-- mm/init-mm.c | 2 +- tools/testing/vma/vma.c | 4 +-- tools/testing/vma/vma_internal.h | 4 +-- 7 files changed, 56 insertions(+), 36 deletions(-)include/linux/mm.h +6
-6@@ -698,7 +698,7 @@ static inline bool vma_start_read(struct vm_area_struct *vma)* we don't rely on for anything - the mm_lock_seq read against which we * need ordering is below. */ -if (READ_ONCE (vma->vm_lock_seq) ==READ_ONCE (vma->vm_mm->mm_lock_seq)) +if (READ_ONCE (vma->vm_lock_seq) ==READ_ONCE (vma->vm_mm->mm_lock_seq.sequence))return false;if (unlikely (down_read_trylock (&vma->vm_lock->lock) ==0 ))@@ -715,7 +715,7 @@ static inline bool vma_start_read(struct vm_area_struct *vma)* after it has been unlocked. * This pairs with RELEASE semantics in vma_end_write_all(). */ -if (unlikely (vma->vm_lock_seq ==smp_load_acquire (&vma->vm_mm->mm_lock_seq))) { +if (unlikely (vma->vm_lock_seq ==raw_read_seqcount (&vma->vm_mm->mm_lock_seq))) {up_read (&vma->vm_lock->lock);return false; }@@ -730,7 +730,7 @@ static inline void vma_end_read(struct vm_area_struct *vma) }/* WARNING! Can only be used if mmap_lock is expected to be write-locked */ -static bool __is_vma_write_locked (struct vm_area_struct *vma,int *mm_lock_seq) +static bool __is_vma_write_locked (struct vm_area_struct *vma,unsigned int *mm_lock_seq) {mmap_assert_write_locked (vma->vm_mm);@@ -738,7 +738,7 @@ static bool __is_vma_write_locked(struct vm_area_struct *vma, int *mm_lock_seq)* current task is holding mmap_write_lock, both vma->vm_lock_seq and * mm->mm_lock_seq can't be concurrently modified. */ - *mm_lock_seq = vma->vm_mm->mm_lock_seq; + *mm_lock_seq = vma->vm_mm->mm_lock_seq.sequence;return (vma->vm_lock_seq == *mm_lock_seq); }@@ -749,7 +749,7 @@ static bool __is_vma_write_locked(struct vm_area_struct *vma, int *mm_lock_seq)*/ static inline void vma_start_write (struct vm_area_struct *vma) { -int mm_lock_seq; +unsigned int mm_lock_seq;if (__is_vma_write_locked (vma, &mm_lock_seq))return ;@@ -767,7 +767,7 @@ static inline void vma_start_write(struct vm_area_struct *vma)static inline void vma_assert_write_locked (struct vm_area_struct *vma) { -int mm_lock_seq; +unsigned int mm_lock_seq;VM_BUG_ON_VMA (!__is_vma_write_locked (vma, &mm_lock_seq), vma); }include/linux/mm_types.h +5
-2@@ -715,7 +715,7 @@ struct vm_area_struct {* counter reuse can only lead to occasional unnecessary use of the * slowpath. */ -int vm_lock_seq; +unsigned int vm_lock_seq;/* Unstable RCU readers are allowed to read this. */ struct vma_lock *vm_lock;#endif @@ -887,6 +887,9 @@ struct mm_struct {* Roughly speaking, incrementing the sequence number is * equivalent to releasing locks on VMAs; reading the sequence * number can be part of taking a read lock on a VMA. +* Incremented every time mmap_lock is write-locked/unlocked. +* Initialized to 0, therefore odd values indicate mmap_lock +* is write-locked and even values that it's released. * * Can be modified under write mmap_lock using RELEASE * semantics. @@ -895,7 +898,7 @@ struct mm_struct {* Can be read with ACQUIRE semantics if not holding write * mmap_lock. */ -int mm_lock_seq; +seqcount_t mm_lock_seq;#endif include/linux/mmap_lock.h +39
-19@@ -71,39 +71,38 @@ static inline void mmap_assert_write_locked(const struct mm_struct *mm) }#ifdef CONFIG_PER_VMA_LOCK -/* -* Drop all currently-held per-VMA locks. -* This is called from the mmap_lock implementation directly before releasing -* a write-locked mmap_lock (or downgrading it to read-locked). -* This should normally NOT be called manually from other places. -* If you want to call this manually anyway, keep in mind that this will release -* *all* VMA write locks, including ones from further up the stack. -*/ -static inline void vma_end_write_all (struct mm_struct *mm) +static inline void mm_lock_seqcount_init (struct mm_struct *mm) { -mmap_assert_write_locked (mm); -/* -* Nobody can concurrently modify mm->mm_lock_seq due to exclusive -* mmap_lock being held. -* We need RELEASE semantics here to ensure that preceding stores into -* the VMA take effect before we unlock it with this store. -* Pairs with ACQUIRE semantics in vma_start_read(). -*/ -smp_store_release (&mm->mm_lock_seq, mm->mm_lock_seq +1 ); +seqcount_init (&mm->mm_lock_seq); +} + +static inline void mm_lock_seqcount_begin (struct mm_struct *mm) +{ +do_raw_write_seqcount_begin (&mm->mm_lock_seq); +} + +static inline void mm_lock_seqcount_end (struct mm_struct *mm) +{ +do_raw_write_seqcount_end (&mm->mm_lock_seq); } +#else -static inline void vma_end_write_all (struct mm_struct *mm) {} +static inline void mm_lock_seqcount_init (struct mm_struct *mm) {} +static inline void mm_lock_seqcount_begin (struct mm_struct *mm) {} +static inline void mm_lock_seqcount_end (struct mm_struct *mm) {}#endif static inline void mmap_init_lock (struct mm_struct *mm) {init_rwsem (&mm->mmap_lock); +mm_lock_seqcount_init (mm); }static inline void mmap_write_lock (struct mm_struct *mm) {__mmap_lock_trace_start_locking (mm, true);down_write (&mm->mmap_lock); +mm_lock_seqcount_begin (mm);__mmap_lock_trace_acquire_returned (mm, true, true); }@@ -111,6 +110,7 @@ static inline void mmap_write_lock_nested(struct mm_struct *mm, int subclass) {__mmap_lock_trace_start_locking (mm, true);down_write_nested (&mm->mmap_lock, subclass); +mm_lock_seqcount_begin (mm);__mmap_lock_trace_acquire_returned (mm, true, true); }@@ -120,10 +120,30 @@ static inline int mmap_write_lock_killable(struct mm_struct *mm)__mmap_lock_trace_start_locking (mm, true); ret =down_write_killable (&mm->mmap_lock); +if (!ret) +mm_lock_seqcount_begin (mm);__mmap_lock_trace_acquire_returned (mm, true, ret ==0 );return ret; } +/* +* Drop all currently-held per-VMA locks. +* This is called from the mmap_lock implementation directly before releasing +* a write-locked mmap_lock (or downgrading it to read-locked). +* This should normally NOT be called manually from other places. +* If you want to call this manually anyway, keep in mind that this will release +* *all* VMA write locks, including ones from further up the stack. +*/ +static inline void vma_end_write_all (struct mm_struct *mm) +{ +mmap_assert_write_locked (mm); +/* +* Nobody can concurrently modify mm->mm_lock_seq due to exclusive +* mmap_lock being held. +*/ +mm_lock_seqcount_end (mm); +} +static inline void mmap_write_unlock (struct mm_struct *mm) {__mmap_lock_trace_released (mm, true);kernel/fork.c +1
-4@@ -447,7 +447,7 @@ static bool vma_lock_alloc(struct vm_area_struct *vma)return false;init_rwsem (&vma->vm_lock->lock); - vma->vm_lock_seq =-1 ; + vma->vm_lock_seq =UINT_MAX ;return true; }@@ -1260,9 +1260,6 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,seqcount_init (&mm->write_protect_seq);mmap_init_lock (mm);INIT_LIST_HEAD (&mm->mmlist); -#ifdef CONFIG_PER_VMA_LOCK - mm->mm_lock_seq =0 ; -#endif mm_pgtables_bytes_init (mm); mm->map_count =0 ; mm->locked_vm =0 ;mm/init-mm.c +1
-1@@ -40,7 +40,7 @@ struct mm_struct init_mm = { .arg_lock = __SPIN_LOCK_UNLOCKED(init_mm.arg_lock), .mmlist =LIST_HEAD_INIT (init_mm.mmlist), #ifdef CONFIG_PER_VMA_LOCK - .mm_lock_seq = 0, + .mm_lock_seq = SEQCNT_ZERO(init_mm.mm_lock_seq), #endif .user_ns = &init_user_ns, .cpu_bitmap = CPU_BITS_NONE,tools/testing/vma/vma.c +2
-2@@ -89,7 +89,7 @@ static struct vm_area_struct *alloc_and_link_vma(struct mm_struct *mm,* begun. Linking to the tree will have caused this to be incremented, * which means we will get a false positive otherwise. */ - vma->vm_lock_seq =-1 ; + vma->vm_lock_seq =UINT_MAX ;return vma; }@@ -214,7 +214,7 @@ static bool vma_write_started(struct vm_area_struct *vma)int seq = vma->vm_lock_seq;/* We reset after each check. */ - vma->vm_lock_seq =-1 ; + vma->vm_lock_seq =UINT_MAX ;/* The vma_start_write() stub simply increments this value. */ return seq >-1 ;tools/testing/vma/vma_internal.h +2
-2@@ -241,7 +241,7 @@ struct vm_area_struct {* counter reuse can only lead to occasional unnecessary use of the * slowpath. */ -int vm_lock_seq; +unsigned int vm_lock_seq;struct vma_lock *vm_lock;#endif @@ -416,7 +416,7 @@ static inline bool vma_lock_alloc(struct vm_area_struct *vma)return false;init_rwsem (&vma->vm_lock->lock); - vma->vm_lock_seq =-1 ; + vma->vm_lock_seq =UINT_MAX ;return true; }base-commit:
9c111059234a949a4d3442a413ade19cc65ab927--
2.47.0.163.g1226f6d8fa-goog
Quoted 31 lines · Andrii Nakryiko
Quoted 9 lines · Suren Baghdasaryan
Convert
mm_lock_seqto beseqcount_tand change allmmap_write_lockvariants to increment it, in-line with the usual seqcount usage pattern. This lets us check whether themmap_lockis write-locked by checking mm_lock_seq.sequence counter (odd=locked, even=unlocked). This will be used when implementingmmap_lockspeculation functions.
As a resultvm_lock_seqis also change to be unsigned to match the type of mm_lock_seq.sequence.Quoted 7 lines
Suggested-by Peter Zijlstra <[email protected]>Signed-off-by Suren Baghdasaryan <[email protected]>---
Applies over mm-unstableThis conversion was discussed at [1] and these patches will likely be incorporated into the next version od Andrii's patcset.
I got a notification that this patch set was applied to mm-unstable by Andrew. But I was wondering if Andrew and Peter would agree to move the patches into tip's perf/core branch, given this is a dependency of my pending uprobe series ([0]) and, as far as I'm aware, there is no urgent need for this API in mm tree(s). If we route this through mm-unstable, we either need a stable tag from mm tree for Peter to merge into perf/core (a bit of a hassle for both Andrew and Peter, I'm sure), or I'd have to wait for 5-6 weeks until after next merge window closes, which would be a huge bummer, given I'd been at this for a while already with basically done patches, and would prefer to get my changes sooner.
Quoted 7 lines
So, I'd very much prefer to just route these changes through perf/core, if mm folks don't oppose this? In fact, I'll go ahead and will send my patches with Suren's patches included with the assumption that we can reroute all this. Thanks for understanding!
P.S. And yeah, Suren's patches apply cleanly to perf/core just as well, I checked.
Sent all of that as v4, see [2]
[2] https://lore.kernel.org/linux-trace-kernel/[email protected]/
Quoted 293 lines
[0] https://lore.kernel.org/linux-trace-kernel/[email protected]/
Quoted 290 lines
The issue of the seqcount_t.sequence being an unsigned rather than unsigned long will be addressed separately in collaoration with Jann Horn.
[1] https://lore.kernel.org/all/[email protected]/
include/linux/mm.h | 12 +++---- include/linux/mm_types.h | 7 ++-- include/linux/mmap_lock.h | 58 +++++++++++++++++++++----------- kernel/fork.c | 5 +-- mm/init-mm.c | 2 +- tools/testing/vma/vma.c | 4 +-- tools/testing/vma/vma_internal.h | 4 +-- 7 files changed, 56 insertions(+), 36 deletions(-)include/linux/mm.h +6
-6@@ -698,7 +698,7 @@ static inline bool vma_start_read(struct vm_area_struct *vma)* we don't rely on for anything - the mm_lock_seq read against which we * need ordering is below. */ -if (READ_ONCE (vma->vm_lock_seq) ==READ_ONCE (vma->vm_mm->mm_lock_seq)) +if (READ_ONCE (vma->vm_lock_seq) ==READ_ONCE (vma->vm_mm->mm_lock_seq.sequence))return false;if (unlikely (down_read_trylock (&vma->vm_lock->lock) ==0 ))@@ -715,7 +715,7 @@ static inline bool vma_start_read(struct vm_area_struct *vma)* after it has been unlocked. * This pairs with RELEASE semantics in vma_end_write_all(). */ -if (unlikely (vma->vm_lock_seq ==smp_load_acquire (&vma->vm_mm->mm_lock_seq))) { +if (unlikely (vma->vm_lock_seq ==raw_read_seqcount (&vma->vm_mm->mm_lock_seq))) {up_read (&vma->vm_lock->lock);return false; }@@ -730,7 +730,7 @@ static inline void vma_end_read(struct vm_area_struct *vma) }/* WARNING! Can only be used if mmap_lock is expected to be write-locked */ -static bool __is_vma_write_locked (struct vm_area_struct *vma,int *mm_lock_seq) +static bool __is_vma_write_locked (struct vm_area_struct *vma,unsigned int *mm_lock_seq) {mmap_assert_write_locked (vma->vm_mm);@@ -738,7 +738,7 @@ static bool __is_vma_write_locked(struct vm_area_struct *vma, int *mm_lock_seq)* current task is holding mmap_write_lock, both vma->vm_lock_seq and * mm->mm_lock_seq can't be concurrently modified. */ - *mm_lock_seq = vma->vm_mm->mm_lock_seq; + *mm_lock_seq = vma->vm_mm->mm_lock_seq.sequence;return (vma->vm_lock_seq == *mm_lock_seq); }@@ -749,7 +749,7 @@ static bool __is_vma_write_locked(struct vm_area_struct *vma, int *mm_lock_seq)*/ static inline void vma_start_write (struct vm_area_struct *vma) { -int mm_lock_seq; +unsigned int mm_lock_seq;if (__is_vma_write_locked (vma, &mm_lock_seq))return ;@@ -767,7 +767,7 @@ static inline void vma_start_write(struct vm_area_struct *vma)static inline void vma_assert_write_locked (struct vm_area_struct *vma) { -int mm_lock_seq; +unsigned int mm_lock_seq;VM_BUG_ON_VMA (!__is_vma_write_locked (vma, &mm_lock_seq), vma); }include/linux/mm_types.h +5
-2@@ -715,7 +715,7 @@ struct vm_area_struct {* counter reuse can only lead to occasional unnecessary use of the * slowpath. */ -int vm_lock_seq; +unsigned int vm_lock_seq;/* Unstable RCU readers are allowed to read this. */ struct vma_lock *vm_lock;#endif @@ -887,6 +887,9 @@ struct mm_struct {* Roughly speaking, incrementing the sequence number is * equivalent to releasing locks on VMAs; reading the sequence * number can be part of taking a read lock on a VMA. +* Incremented every time mmap_lock is write-locked/unlocked. +* Initialized to 0, therefore odd values indicate mmap_lock +* is write-locked and even values that it's released. * * Can be modified under write mmap_lock using RELEASE * semantics. @@ -895,7 +898,7 @@ struct mm_struct {* Can be read with ACQUIRE semantics if not holding write * mmap_lock. */ -int mm_lock_seq; +seqcount_t mm_lock_seq;#endif include/linux/mmap_lock.h +39
-19@@ -71,39 +71,38 @@ static inline void mmap_assert_write_locked(const struct mm_struct *mm) }#ifdef CONFIG_PER_VMA_LOCK -/* -* Drop all currently-held per-VMA locks. -* This is called from the mmap_lock implementation directly before releasing -* a write-locked mmap_lock (or downgrading it to read-locked). -* This should normally NOT be called manually from other places. -* If you want to call this manually anyway, keep in mind that this will release -* *all* VMA write locks, including ones from further up the stack. -*/ -static inline void vma_end_write_all (struct mm_struct *mm) +static inline void mm_lock_seqcount_init (struct mm_struct *mm) { -mmap_assert_write_locked (mm); -/* -* Nobody can concurrently modify mm->mm_lock_seq due to exclusive -* mmap_lock being held. -* We need RELEASE semantics here to ensure that preceding stores into -* the VMA take effect before we unlock it with this store. -* Pairs with ACQUIRE semantics in vma_start_read(). -*/ -smp_store_release (&mm->mm_lock_seq, mm->mm_lock_seq +1 ); +seqcount_init (&mm->mm_lock_seq); +} + +static inline void mm_lock_seqcount_begin (struct mm_struct *mm) +{ +do_raw_write_seqcount_begin (&mm->mm_lock_seq); +} + +static inline void mm_lock_seqcount_end (struct mm_struct *mm) +{ +do_raw_write_seqcount_end (&mm->mm_lock_seq); } +#else -static inline void vma_end_write_all (struct mm_struct *mm) {} +static inline void mm_lock_seqcount_init (struct mm_struct *mm) {} +static inline void mm_lock_seqcount_begin (struct mm_struct *mm) {} +static inline void mm_lock_seqcount_end (struct mm_struct *mm) {}#endif static inline void mmap_init_lock (struct mm_struct *mm) {init_rwsem (&mm->mmap_lock); +mm_lock_seqcount_init (mm); }static inline void mmap_write_lock (struct mm_struct *mm) {__mmap_lock_trace_start_locking (mm, true);down_write (&mm->mmap_lock); +mm_lock_seqcount_begin (mm);__mmap_lock_trace_acquire_returned (mm, true, true); }@@ -111,6 +110,7 @@ static inline void mmap_write_lock_nested(struct mm_struct *mm, int subclass) {__mmap_lock_trace_start_locking (mm, true);down_write_nested (&mm->mmap_lock, subclass); +mm_lock_seqcount_begin (mm);__mmap_lock_trace_acquire_returned (mm, true, true); }@@ -120,10 +120,30 @@ static inline int mmap_write_lock_killable(struct mm_struct *mm)__mmap_lock_trace_start_locking (mm, true); ret =down_write_killable (&mm->mmap_lock); +if (!ret) +mm_lock_seqcount_begin (mm);__mmap_lock_trace_acquire_returned (mm, true, ret ==0 );return ret; } +/* +* Drop all currently-held per-VMA locks. +* This is called from the mmap_lock implementation directly before releasing +* a write-locked mmap_lock (or downgrading it to read-locked). +* This should normally NOT be called manually from other places. +* If you want to call this manually anyway, keep in mind that this will release +* *all* VMA write locks, including ones from further up the stack. +*/ +static inline void vma_end_write_all (struct mm_struct *mm) +{ +mmap_assert_write_locked (mm); +/* +* Nobody can concurrently modify mm->mm_lock_seq due to exclusive +* mmap_lock being held. +*/ +mm_lock_seqcount_end (mm); +} +static inline void mmap_write_unlock (struct mm_struct *mm) {__mmap_lock_trace_released (mm, true);kernel/fork.c +1
-4@@ -447,7 +447,7 @@ static bool vma_lock_alloc(struct vm_area_struct *vma)return false;init_rwsem (&vma->vm_lock->lock); - vma->vm_lock_seq =-1 ; + vma->vm_lock_seq =UINT_MAX ;return true; }@@ -1260,9 +1260,6 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,seqcount_init (&mm->write_protect_seq);mmap_init_lock (mm);INIT_LIST_HEAD (&mm->mmlist); -#ifdef CONFIG_PER_VMA_LOCK - mm->mm_lock_seq =0 ; -#endif mm_pgtables_bytes_init (mm); mm->map_count =0 ; mm->locked_vm =0 ;mm/init-mm.c +1
-1@@ -40,7 +40,7 @@ struct mm_struct init_mm = { .arg_lock = __SPIN_LOCK_UNLOCKED(init_mm.arg_lock), .mmlist =LIST_HEAD_INIT (init_mm.mmlist), #ifdef CONFIG_PER_VMA_LOCK - .mm_lock_seq = 0, + .mm_lock_seq = SEQCNT_ZERO(init_mm.mm_lock_seq), #endif .user_ns = &init_user_ns, .cpu_bitmap = CPU_BITS_NONE,tools/testing/vma/vma.c +2
-2@@ -89,7 +89,7 @@ static struct vm_area_struct *alloc_and_link_vma(struct mm_struct *mm,* begun. Linking to the tree will have caused this to be incremented, * which means we will get a false positive otherwise. */ - vma->vm_lock_seq =-1 ; + vma->vm_lock_seq =UINT_MAX ;return vma; }@@ -214,7 +214,7 @@ static bool vma_write_started(struct vm_area_struct *vma)int seq = vma->vm_lock_seq;/* We reset after each check. */ - vma->vm_lock_seq =-1 ; + vma->vm_lock_seq =UINT_MAX ;/* The vma_start_write() stub simply increments this value. */ return seq >-1 ;tools/testing/vma/vma_internal.h +2
-2@@ -241,7 +241,7 @@ struct vm_area_struct {* counter reuse can only lead to occasional unnecessary use of the * slowpath. */ -int vm_lock_seq; +unsigned int vm_lock_seq;struct vma_lock *vm_lock;#endif @@ -416,7 +416,7 @@ static inline bool vma_lock_alloc(struct vm_area_struct *vma)return false;init_rwsem (&vma->vm_lock->lock); - vma->vm_lock_seq =-1 ; + vma->vm_lock_seq =UINT_MAX ;return true; }base-commit:
9c111059234a949a4d3442a413ade19cc65ab927--
2.47.0.163.g1226f6d8fa-goog
Andrew Morton
maintainerQuoted 5 lines · On Sun, 27 Oct 2024 17:57:34 -0700 Andrii Nakryiko <[email protected]>…
I got a notification that this patch set was applied to mm-unstable by Andrew. But I was wondering if Andrew and Peter would agree to move the patches into tip's perf/core branch, given this is a dependency of my pending uprobe series ([0]) and, as far as I'm aware, there is no urgent need for this API in mm tree(s).
OK by me.
Quoted 10 lines · Andrew Morton
Quoted 5 lines · On Sun, 27 Oct 2024 17:57:34 -0700 Andrii Nakryiko <[email protected]>…
I got a notification that this patch set was applied to mm-unstable by Andrew. But I was wondering if Andrew and Peter would agree to move the patches into tip's perf/core branch, given this is a dependency of my pending uprobe series ([0]) and, as far as I'm aware, there is no urgent need for this API in mm tree(s).
OK by me.
Great, thank you! Then I'll hope Peter will just apply all 4 patches into his tree.