seqlock: add raw_seqcount_try_begin
Subsystems (7)
Suren Baghdasaryan
maintainerPATCH v3 1/3 seqlock: add raw_seqcount_try_begin
Add raw_seqcount_try_begin() to opens a read critical section of the given seqcount_t if the counter is even. This enables eliding the critical section entirely if the counter is odd, instead of doing the speculation knowing it will fail.
---
Applies over Linus' ToT
Changes since v2 [1]
- Added SOB, per David Hildenbrand
- Added SOB, per Liam Howlett
[1] https://lore.kernel.org/all/[email protected]/
include/linux/seqlock.h | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+)
include/linux/seqlock.h +22 -0
@@ -318,6 +318,28 @@ SEQCOUNT_LOCKNAME(mutex, struct mutex, true, mutex) __seq; \ }) +/** +* raw_seqcount_try_begin() - begin a seqcount_t read critical section +* w/o lockdep and w/o counter stabilization +* @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants +* +* Similar to raw_seqcount_begin(), except it enables eliding the critical +* section entirely if odd, instead of doing the speculation knowing it will +* fail. +* +* Useful when counter stabilization is more or less equivalent to taking +* the lock and there is a slowpath that does that. +* +* If true, start will be set to the (even) sequence count read. +* +* Return: true when a read critical section is started. +*/ +#define raw_seqcount_try_begin (s, start) \ +({ \ + start = raw_read_seqcount(s); \ + !(start & 1); \ +}) +/** * raw_seqcount_begin() - begin a seqcount_t read critical section w/o * lockdep and w/o counter stabilization
base-commit: 28eb75e178d389d325f1666e422bc13bbbb9804c
Signature · 1 line
2.47.0.371.ga323438b13-goog
Suren Baghdasaryan
maintainerPATCH v3 2/3 mm: convert mm_lock_seq to a proper seqcount
Convert mm_lock_seq to be seqcount_t and change all mmap_write_lock variants to increment it, in-line with the usual seqcount usage pattern. This lets us check whether the mmap_lock is write-locked by checking mm_lock_seq.sequence counter (odd=locked, even=unlocked). This will be used when implementing mmap_lock speculation functions.
As a result vm_lock_seq is also change to be unsigned to match the type of mm_lock_seq.sequence.
---
Changes since v2 [1]
- Moved ASSERT_EXCLUSIVE_WRITER() into mm_lock_seqcount_end to fix !CONFIG_PER_VMA_LOCK, per kernel test bot
- Added SOB, per Liam Howlett
[1] https://lore.kernel.org/all/[email protected]/
include/linux/mm.h | 12 +++---- include/linux/mm_types.h | 7 ++-- include/linux/mmap_lock.h | 55 +++++++++++++++++++++----------- kernel/fork.c | 5 +-- mm/init-mm.c | 2 +- tools/testing/vma/vma.c | 4 +-- tools/testing/vma/vma_internal.h | 4 +-- 7 files changed, 53 insertions(+), 36 deletions(-)
include/linux/mm.h +6 -6
@@ -710,7 +710,7 @@ static inline bool vma_start_read(struct vm_area_struct *vma)* we don't rely on for anything - the mm_lock_seq read against which we * need ordering is below. */ -if (READ_ONCE (vma->vm_lock_seq) ==READ_ONCE (vma->vm_mm->mm_lock_seq)) +if (READ_ONCE (vma->vm_lock_seq) ==READ_ONCE (vma->vm_mm->mm_lock_seq.sequence))return false;if (unlikely (down_read_trylock (&vma->vm_lock->lock) ==0 ))
@@ -727,7 +727,7 @@ static inline bool vma_start_read(struct vm_area_struct *vma)* after it has been unlocked. * This pairs with RELEASE semantics in vma_end_write_all(). */ -if (unlikely (vma->vm_lock_seq ==smp_load_acquire (&vma->vm_mm->mm_lock_seq))) { +if (unlikely (vma->vm_lock_seq ==raw_read_seqcount (&vma->vm_mm->mm_lock_seq))) {up_read (&vma->vm_lock->lock);return false; }
@@ -742,7 +742,7 @@ static inline void vma_end_read(struct vm_area_struct *vma) }/* WARNING! Can only be used if mmap_lock is expected to be write-locked */ -static bool __is_vma_write_locked (struct vm_area_struct *vma,int *mm_lock_seq) +static bool __is_vma_write_locked (struct vm_area_struct *vma,unsigned int *mm_lock_seq) {mmap_assert_write_locked (vma->vm_mm);
@@ -750,7 +750,7 @@ static bool __is_vma_write_locked(struct vm_area_struct *vma, int *mm_lock_seq)* current task is holding mmap_write_lock, both vma->vm_lock_seq and * mm->mm_lock_seq can't be concurrently modified. */ - *mm_lock_seq = vma->vm_mm->mm_lock_seq; + *mm_lock_seq = vma->vm_mm->mm_lock_seq.sequence;return (vma->vm_lock_seq == *mm_lock_seq); }
@@ -761,7 +761,7 @@ static bool __is_vma_write_locked(struct vm_area_struct *vma, int *mm_lock_seq)*/ static inline void vma_start_write (struct vm_area_struct *vma) { -int mm_lock_seq; +unsigned int mm_lock_seq;if (__is_vma_write_locked (vma, &mm_lock_seq))return ;
@@ -779,7 +779,7 @@ static inline void vma_start_write(struct vm_area_struct *vma)static inline void vma_assert_write_locked (struct vm_area_struct *vma) { -int mm_lock_seq; +unsigned int mm_lock_seq;VM_BUG_ON_VMA (!__is_vma_write_locked (vma, &mm_lock_seq), vma); }
include/linux/mm_types.h +5 -2
@@ -697,7 +697,7 @@ struct vm_area_struct {* counter reuse can only lead to occasional unnecessary use of the * slowpath. */ -int vm_lock_seq; +unsigned int vm_lock_seq;/* Unstable RCU readers are allowed to read this. */ struct vma_lock *vm_lock;#endif
@@ -891,6 +891,9 @@ struct mm_struct {* Roughly speaking, incrementing the sequence number is * equivalent to releasing locks on VMAs; reading the sequence * number can be part of taking a read lock on a VMA. +* Incremented every time mmap_lock is write-locked/unlocked. +* Initialized to 0, therefore odd values indicate mmap_lock +* is write-locked and even values that it's released. * * Can be modified under write mmap_lock using RELEASE * semantics.
@@ -899,7 +902,7 @@ struct mm_struct {* Can be read with ACQUIRE semantics if not holding write * mmap_lock. */ -int mm_lock_seq; +seqcount_t mm_lock_seq;#endif
include/linux/mmap_lock.h +36 -19
91 lines not on this page · raw message
kernel/fork.c +1 -4
@@ -449,7 +449,7 @@ static bool vma_lock_alloc(struct vm_area_struct *vma)return false;init_rwsem (&vma->vm_lock->lock); - vma->vm_lock_seq =-1 ; + vma->vm_lock_seq =UINT_MAX ;return true; }
@@ -1262,9 +1262,6 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,seqcount_init (&mm->write_protect_seq);mmap_init_lock (mm);INIT_LIST_HEAD (&mm->mmlist); -#ifdef CONFIG_PER_VMA_LOCK - mm->mm_lock_seq =0 ; -#endif mm_pgtables_bytes_init (mm); mm->map_count =0 ; mm->locked_vm =0 ;
mm/init-mm.c +1 -1
@@ -40,7 +40,7 @@ struct mm_struct init_mm = { .arg_lock = __SPIN_LOCK_UNLOCKED(init_mm.arg_lock), .mmlist =LIST_HEAD_INIT (init_mm.mmlist), #ifdef CONFIG_PER_VMA_LOCK - .mm_lock_seq = 0, + .mm_lock_seq = SEQCNT_ZERO(init_mm.mm_lock_seq), #endif .user_ns = &init_user_ns, .cpu_bitmap = CPU_BITS_NONE,
tools/testing/vma/vma.c +2 -2
@@ -87,7 +87,7 @@ static struct vm_area_struct *alloc_and_link_vma(struct mm_struct *mm,* begun. Linking to the tree will have caused this to be incremented, * which means we will get a false positive otherwise. */ - vma->vm_lock_seq =-1 ; + vma->vm_lock_seq =UINT_MAX ;return vma; }
@@ -212,7 +212,7 @@ static bool vma_write_started(struct vm_area_struct *vma)int seq = vma->vm_lock_seq;/* We reset after each check. */ - vma->vm_lock_seq =-1 ; + vma->vm_lock_seq =UINT_MAX ;/* The vma_start_write() stub simply increments this value. */ return seq >-1 ;
tools/testing/vma/vma_internal.h +2 -2
@@ -231,7 +231,7 @@ struct vm_area_struct {* counter reuse can only lead to occasional unnecessary use of the * slowpath. */ -int vm_lock_seq; +unsigned int vm_lock_seq;struct vma_lock *vm_lock;#endif
@@ -406,7 +406,7 @@ static inline bool vma_lock_alloc(struct vm_area_struct *vma)return false;init_rwsem (&vma->vm_lock->lock); - vma->vm_lock_seq =-1 ; + vma->vm_lock_seq =UINT_MAX ;return true; }
Signature · 1 line
2.47.0.371.ga323438b13-goog
[tip: perf/core] mm: convert mm_lock_seq to a proper seqcount
The following commit has been merged into the perf/core branch of tip:
The following commit has been merged into the perf/core branch of tip:
Commit-ID: eb449bd96954b1c1e491d19066cfd2a010f0aa47 Gitweb: https://git.kernel.org/tip/eb449bd96954b1c1e491d19066cfd2a010f0aa47 Author: Suren Baghdasaryan <[email protected]> AuthorDate: Fri, 22 Nov 2024 09:44:15 -08:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 02 Dec 2024 12:01:38 +01:00
mm: convert mm_lock_seq to a proper seqcount
Convert mm_lock_seq to be seqcount_t and change all mmap_write_lock variants to increment it, in-line with the usual seqcount usage pattern. This lets us check whether the mmap_lock is write-locked by checking mm_lock_seq.sequence counter (odd=locked, even=unlocked). This will be used when implementing mmap_lock speculation functions.
As a result vm_lock_seq is also change to be unsigned to match the type of mm_lock_seq.sequence.
--- include/linux/mm.h | 12 +++---- include/linux/mm_types.h | 7 ++-- include/linux/mmap_lock.h | 55 ++++++++++++++++++++----------- kernel/fork.c | 5 +--- mm/init-mm.c | 2 +- tools/testing/vma/vma.c | 4 +- tools/testing/vma/vma_internal.h | 4 +- 7 files changed, 53 insertions(+), 36 deletions(-)
include/linux/mm.h +6 -6
@@ -710,7 +710,7 @@ static inline bool vma_start_read(struct vm_area_struct *vma)* we don't rely on for anything - the mm_lock_seq read against which we * need ordering is below. */ -if (READ_ONCE (vma->vm_lock_seq) ==READ_ONCE (vma->vm_mm->mm_lock_seq)) +if (READ_ONCE (vma->vm_lock_seq) ==READ_ONCE (vma->vm_mm->mm_lock_seq.sequence))return false;if (unlikely (down_read_trylock (&vma->vm_lock->lock) ==0 ))
@@ -727,7 +727,7 @@ static inline bool vma_start_read(struct vm_area_struct *vma)* after it has been unlocked. * This pairs with RELEASE semantics in vma_end_write_all(). */ -if (unlikely (vma->vm_lock_seq ==smp_load_acquire (&vma->vm_mm->mm_lock_seq))) { +if (unlikely (vma->vm_lock_seq ==raw_read_seqcount (&vma->vm_mm->mm_lock_seq))) {up_read (&vma->vm_lock->lock);return false; }
@@ -742,7 +742,7 @@ static inline void vma_end_read(struct vm_area_struct *vma) }/* WARNING! Can only be used if mmap_lock is expected to be write-locked */ -static bool __is_vma_write_locked (struct vm_area_struct *vma,int *mm_lock_seq) +static bool __is_vma_write_locked (struct vm_area_struct *vma,unsigned int *mm_lock_seq) {mmap_assert_write_locked (vma->vm_mm);
@@ -750,7 +750,7 @@ static bool __is_vma_write_locked(struct vm_area_struct *vma, int *mm_lock_seq)* current task is holding mmap_write_lock, both vma->vm_lock_seq and * mm->mm_lock_seq can't be concurrently modified. */ - *mm_lock_seq = vma->vm_mm->mm_lock_seq; + *mm_lock_seq = vma->vm_mm->mm_lock_seq.sequence;return (vma->vm_lock_seq == *mm_lock_seq); }
@@ -761,7 +761,7 @@ static bool __is_vma_write_locked(struct vm_area_struct *vma, int *mm_lock_seq)*/ static inline void vma_start_write (struct vm_area_struct *vma) { -int mm_lock_seq; +unsigned int mm_lock_seq;if (__is_vma_write_locked (vma, &mm_lock_seq))return ;
@@ -779,7 +779,7 @@ static inline void vma_start_write(struct vm_area_struct *vma)static inline void vma_assert_write_locked (struct vm_area_struct *vma) { -int mm_lock_seq; +unsigned int mm_lock_seq;VM_BUG_ON_VMA (!__is_vma_write_locked (vma, &mm_lock_seq), vma); }
include/linux/mm_types.h +5 -2
@@ -697,7 +697,7 @@ struct vm_area_struct {* counter reuse can only lead to occasional unnecessary use of the * slowpath. */ -int vm_lock_seq; +unsigned int vm_lock_seq;/* Unstable RCU readers are allowed to read this. */ struct vma_lock *vm_lock;#endif
@@ -891,6 +891,9 @@ struct mm_struct {* Roughly speaking, incrementing the sequence number is * equivalent to releasing locks on VMAs; reading the sequence * number can be part of taking a read lock on a VMA. +* Incremented every time mmap_lock is write-locked/unlocked. +* Initialized to 0, therefore odd values indicate mmap_lock +* is write-locked and even values that it's released. * * Can be modified under write mmap_lock using RELEASE * semantics.
@@ -899,7 +902,7 @@ struct mm_struct {* Can be read with ACQUIRE semantics if not holding write * mmap_lock. */ -int mm_lock_seq; +seqcount_t mm_lock_seq;#endif
include/linux/mmap_lock.h +36 -19
91 lines not on this page · raw message
kernel/fork.c +1 -4
@@ -448,7 +448,7 @@ static bool vma_lock_alloc(struct vm_area_struct *vma)return false;init_rwsem (&vma->vm_lock->lock); - vma->vm_lock_seq =-1 ; + vma->vm_lock_seq =UINT_MAX ;return true; }
@@ -1267,9 +1267,6 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,seqcount_init (&mm->write_protect_seq);mmap_init_lock (mm);INIT_LIST_HEAD (&mm->mmlist); -#ifdef CONFIG_PER_VMA_LOCK - mm->mm_lock_seq =0 ; -#endif mm_pgtables_bytes_init (mm); mm->map_count =0 ; mm->locked_vm =0 ;
mm/init-mm.c +1 -1
@@ -40,7 +40,7 @@ struct mm_struct init_mm = { .arg_lock = __SPIN_LOCK_UNLOCKED(init_mm.arg_lock), .mmlist =LIST_HEAD_INIT (init_mm.mmlist), #ifdef CONFIG_PER_VMA_LOCK - .mm_lock_seq = 0, + .mm_lock_seq = SEQCNT_ZERO(init_mm.mm_lock_seq), #endif .user_ns = &init_user_ns, .cpu_bitmap = CPU_BITS_NONE,
tools/testing/vma/vma.c +2 -2
@@ -89,7 +89,7 @@ static struct vm_area_struct *alloc_and_link_vma(struct mm_struct *mm,* begun. Linking to the tree will have caused this to be incremented, * which means we will get a false positive otherwise. */ - vma->vm_lock_seq =-1 ; + vma->vm_lock_seq =UINT_MAX ;return vma; }
@@ -214,7 +214,7 @@ static bool vma_write_started(struct vm_area_struct *vma)int seq = vma->vm_lock_seq;/* We reset after each check. */ - vma->vm_lock_seq =-1 ; + vma->vm_lock_seq =UINT_MAX ;/* The vma_start_write() stub simply increments this value. */ return seq >-1 ;
tools/testing/vma/vma_internal.h +2 -2
@@ -241,7 +241,7 @@ struct vm_area_struct {* counter reuse can only lead to occasional unnecessary use of the * slowpath. */ -int vm_lock_seq; +unsigned int vm_lock_seq;struct vma_lock *vm_lock;#endif
@@ -416,7 +416,7 @@ static inline bool vma_lock_alloc(struct vm_area_struct *vma)return false;init_rwsem (&vma->vm_lock->lock); - vma->vm_lock_seq =-1 ; + vma->vm_lock_seq =UINT_MAX ;return true; }
Suren Baghdasaryan
maintainerPATCH v3 3/3 mm: introduce mmap_lock_speculate_{try_begin|retry}
In reply to Suren BaghdasaryanAdd helper functions to speculatively perform operations without read-locking mmap_lock, expecting that mmap_lock will not be write-locked and mm is not modified from under us.
---
Changes since v2 [1]
- Added SOB, per Liam Howlett
[1] https://lore.kernel.org/all/[email protected]/
include/linux/mmap_lock.h | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-)
include/linux/mmap_lock.h +31 -2
@@ -71,6 +71,7 @@ static inline void mmap_assert_write_locked(const struct mm_struct *mm) }#ifdef CONFIG_PER_VMA_LOCK +static inline void mm_lock_seqcount_init (struct mm_struct *mm) {seqcount_init (&mm->mm_lock_seq);
@@ -87,11 +88,39 @@ static inline void mm_lock_seqcount_end(struct mm_struct *mm)do_raw_write_seqcount_end (&mm->mm_lock_seq); } -#else +static inline bool mmap_lock_speculate_try_begin (struct mm_struct *mm,unsigned int *seq) +{ +/* +* Since mmap_lock is a sleeping lock, and waiting for it to become +* unlocked is more or less equivalent with taking it ourselves, don't +* bother with the speculative path if mmap_lock is already write-locked +* and take the slow path, which takes the lock. +*/ +return raw_seqcount_try_begin (&mm->mm_lock_seq, *seq); +} + +static inline bool mmap_lock_speculate_retry (struct mm_struct *mm,unsigned int seq) +{ +return do_read_seqcount_retry (&mm->mm_lock_seq, seq); +} + +#else /* CONFIG_PER_VMA_LOCK */ +static inline void mm_lock_seqcount_init (struct mm_struct *mm) {}static inline void mm_lock_seqcount_begin (struct mm_struct *mm) {}static inline void mm_lock_seqcount_end (struct mm_struct *mm) {} -#endif + +static inline bool mmap_lock_speculate_try_begin (struct mm_struct *mm,unsigned int *seq) +{ +return false; +} + +static inline bool mmap_lock_speculate_retry (struct mm_struct *mm,unsigned int seq) +{ +return true; +} + +#endif /* CONFIG_PER_VMA_LOCK */ static inline void mmap_init_lock (struct mm_struct *mm) {
Signature · 1 line
2.47.0.371.ga323438b13-goog
Re: [PATCH v3 3/3] mm: introduce mmap_lock_speculate_{try_begin|retry}
Quoted 37 lines · Suren Baghdasaryan
Add helper functions to speculatively perform operations without read-locking
mmap_lock, expecting thatmmap_lockwill not be write-locked and mm is not modified from under us.Suggested-by Peter Zijlstra <[email protected]>Signed-off-by Suren Baghdasaryan <[email protected]>Reviewed-by Liam R. Howlett <[email protected]>---
Changes since v2 [1]
- Added SOB, per Liam Howlett[1] https://lore.kernel.org/all/[email protected]/
include/linux/mmap_lock.h| 33 +++++++++++++++++++++++++++++++--1 file changed, 31 insertions(+), 2 deletions(-)include/linux/mmap_lock.h +5
-1@@ -71,6 +71,7 @@ static inline void mmap_assert_write_locked(const struct mm_struct *mm) }#ifdef CONFIG_PER_VMA_LOCK +static inline void mm_lock_seqcount_init (struct mm_struct *mm) {seqcount_init (&mm->mm_lock_seq);@@ -87,11 +88,39 @@ static inline void mm_lock_seqcount_end(struct mm_struct *mm)do_raw_write_seqcount_end (&mm->mm_lock_seq); } -#else +static inline bool mmap_lock_speculate_try_begin (struct mm_struct *mm,unsigned int *seq) +{ +/* +* Since mmap_lock is a sleeping lock, and waiting for it to become
include/linux/mmap_lock.h static inline void mm_lock_seqcount_end(struct mm_struct *mm) · line 104
Quoted 10 lines
include/linux/mmap_lock.h +10
-0+* unlocked is more or less equivalent with taking it ourselves, don't +* bother with the speculative path if mmap_lock is already write-locked +* and take the slow path, which takes the lock. +*/ +return raw_seqcount_try_begin (&mm->mm_lock_seq, *seq); +} + +static inline bool mmap_lock_speculate_retry (struct mm_struct *mm,unsigned int seq) +{ +return do_read_seqcount_retry (&mm->mm_lock_seq, seq);
Just curious why we don't use read_seqcount_retry().
Looks this is the only user outside seqlock.h.
Quoted 25 lines
include/linux/mmap_lock.h +16
-1+} + +#else /* CONFIG_PER_VMA_LOCK */ +static inline void mm_lock_seqcount_init (struct mm_struct *mm) {}static inline void mm_lock_seqcount_begin (struct mm_struct *mm) {}static inline void mm_lock_seqcount_end (struct mm_struct *mm) {} -#endif + +static inline bool mmap_lock_speculate_try_begin (struct mm_struct *mm,unsigned int *seq) +{ +return false; +} + +static inline bool mmap_lock_speculate_retry (struct mm_struct *mm,unsigned int seq) +{ +return true; +} + +#endif /* CONFIG_PER_VMA_LOCK */ static inline void mmap_init_lock (struct mm_struct *mm) {Signature · 1 line
2.47.0.371.ga323438b13-goog
Signature · 2 lines
Wei Yang Help you, Help me
Suren Baghdasaryan
maintainerRe: [PATCH v3 3/3] mm: introduce mmap_lock_speculate_{try_begin|retry}
Quoted 52 lines · Wei Yang
Quoted 37 lines · Suren Baghdasaryan
Add helper functions to speculatively perform operations without read-locking
mmap_lock, expecting thatmmap_lockwill not be write-locked and mm is not modified from under us.Suggested-by Peter Zijlstra <[email protected]>Signed-off-by Suren Baghdasaryan <[email protected]>Reviewed-by Liam R. Howlett <[email protected]>---
Changes since v2 [1]
- Added SOB, per Liam Howlett[1] https://lore.kernel.org/all/[email protected]/
include/linux/mmap_lock.h| 33 +++++++++++++++++++++++++++++++--1 file changed, 31 insertions(+), 2 deletions(-)include/linux/mmap_lock.h +5
-1@@ -71,6 +71,7 @@ static inline void mmap_assert_write_locked(const struct mm_struct *mm) }#ifdef CONFIG_PER_VMA_LOCK +static inline void mm_lock_seqcount_init (struct mm_struct *mm) {seqcount_init (&mm->mm_lock_seq);@@ -87,11 +88,39 @@ static inline void mm_lock_seqcount_end(struct mm_struct *mm)do_raw_write_seqcount_end (&mm->mm_lock_seq); } -#else +static inline bool mmap_lock_speculate_try_begin (struct mm_struct *mm,unsigned int *seq) +{ +/* +* Since mmap_lock is a sleeping lock, and waiting for it to become include/linux/mmap_lock.h static inline void mm_lock_seqcount_end(struct mm_struct *mm) · line 104
Quoted 10 lines
include/linux/mmap_lock.h +10
-0+* unlocked is more or less equivalent with taking it ourselves, don't +* bother with the speculative path if mmap_lock is already write-locked +* and take the slow path, which takes the lock. +*/ +return raw_seqcount_try_begin (&mm->mm_lock_seq, *seq); +} + +static inline bool mmap_lock_speculate_retry (struct mm_struct *mm,unsigned int seq) +{ +return do_read_seqcount_retry (&mm->mm_lock_seq, seq);Just curious why we don't use
read_seqcount_retry().
Quoted 1 line
Looks this is the only user outside
seqlock.h.
Ah, good eye! read_seqcount_retry() would be better.
Peter, do you want me to post a new patchset or you can patch it when picking it up?
Quoted 30 lines
Quoted 25 lines
include/linux/mmap_lock.h +16
-2+} + +#else /* CONFIG_PER_VMA_LOCK */ +static inline void mm_lock_seqcount_init (struct mm_struct *mm) {}static inline void mm_lock_seqcount_begin (struct mm_struct *mm) {}static inline void mm_lock_seqcount_end (struct mm_struct *mm) {} -#endif + +static inline bool mmap_lock_speculate_try_begin (struct mm_struct *mm,unsigned int *seq) +{ +return false; +} + +static inline bool mmap_lock_speculate_retry (struct mm_struct *mm,unsigned int seq) +{ +return true; +} + +#endif /* CONFIG_PER_VMA_LOCK */ static inline void mmap_init_lock (struct mm_struct *mm) { --Quoted 1 line
2.47.0.371.ga323438b13-goog
--
Wei Yang
Help you, Help me
Peter Zijlstra
maintainerRe: [PATCH v3 3/3] mm: introduce mmap_lock_speculate_{try_begin|retry}
Quoted 6 lines · Suren Baghdasaryan
Quoted 6 lines
include/linux/mmap_lock.h static inline void mm_lock_seqcount_end(struct mm_struct *mm) · line 104
Quoted 3 lines
include/linux/mmap_lock.h +3
-0+static inline bool mmap_lock_speculate_retry (struct mm_struct *mm,unsigned int seq) +{ +return do_read_seqcount_retry (&mm->mm_lock_seq, seq);Just curious why we don't use
read_seqcount_retry().
Quoted 6 lines
Quoted 1 line
Looks this is the only user outside
seqlock.h.Ah, good eye!
read_seqcount_retry()would be better.Peter, do you want me to post a new patchset or you can patch it when picking it up?
Fixed up my local copy, thanks!
Andrew Morton
maintainerRe: [PATCH v3 3/3] mm: introduce mmap_lock_speculate_{try_begin|retry}
Quoted 8 lines · On Mon, 25 Nov 2024 17:53:57 +0100 Peter Zijlstra <[email protected]> wrote:
Quoted 6 lines · Suren Baghdasaryan
Quoted 6 lines
include/linux/mmap_lock.h static inline void mm_lock_seqcount_end(struct mm_struct *mm) · line 104
Quoted 3 lines
include/linux/mmap_lock.h +3
-0+static inline bool mmap_lock_speculate_retry (struct mm_struct *mm,unsigned int seq) +{ +return do_read_seqcount_retry (&mm->mm_lock_seq, seq);Just curious why we don't use
read_seqcount_retry().
Quoted 8 lines
Quoted 6 lines
Quoted 1 line
Looks this is the only user outside
seqlock.h.Ah, good eye!
read_seqcount_retry()would be better.Peter, do you want me to post a new patchset or you can patch it when picking it up?
Fixed up my local copy, thanks!
I take this to mean that you'll be merging these. I don't see this series in -next and I'm sitting on the v1 series, which I shall now drop.
[tip: perf/core] mm: introduce mmap_lock_speculate_{try_begin|retry}
In reply to Suren BaghdasaryanThe following commit has been merged into the perf/core branch of tip:
The following commit has been merged into the perf/core branch of tip:
Commit-ID: 03a001b156d2da186a5618de242750d06bf81e2d Gitweb: https://git.kernel.org/tip/03a001b156d2da186a5618de242750d06bf81e2d Author: Suren Baghdasaryan <[email protected]> AuthorDate: Fri, 22 Nov 2024 09:44:16 -08:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 02 Dec 2024 12:01:38 +01:00
mm: introduce mmap_lock_speculate_{try_begin|retry}
Add helper functions to speculatively perform operations without read-locking mmap_lock, expecting that mmap_lock will not be write-locked and mm is not modified from under us.
--- include/linux/mmap_lock.h | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-)
include/linux/mmap_lock.h +31 -2
@@ -71,6 +71,7 @@ static inline void mmap_assert_write_locked(const struct mm_struct *mm) }#ifdef CONFIG_PER_VMA_LOCK +static inline void mm_lock_seqcount_init (struct mm_struct *mm) {seqcount_init (&mm->mm_lock_seq);
@@ -87,11 +88,39 @@ static inline void mm_lock_seqcount_end(struct mm_struct *mm)do_raw_write_seqcount_end (&mm->mm_lock_seq); } -#else +static inline bool mmap_lock_speculate_try_begin (struct mm_struct *mm,unsigned int *seq) +{ +/* +* Since mmap_lock is a sleeping lock, and waiting for it to become +* unlocked is more or less equivalent with taking it ourselves, don't +* bother with the speculative path if mmap_lock is already write-locked +* and take the slow path, which takes the lock. +*/ +return raw_seqcount_try_begin (&mm->mm_lock_seq, *seq); +} + +static inline bool mmap_lock_speculate_retry (struct mm_struct *mm,unsigned int seq) +{ +return read_seqcount_retry (&mm->mm_lock_seq, seq); +} + +#else /* CONFIG_PER_VMA_LOCK */ +static inline void mm_lock_seqcount_init (struct mm_struct *mm) {}static inline void mm_lock_seqcount_begin (struct mm_struct *mm) {}static inline void mm_lock_seqcount_end (struct mm_struct *mm) {} -#endif + +static inline bool mmap_lock_speculate_try_begin (struct mm_struct *mm,unsigned int *seq) +{ +return false; +} + +static inline bool mmap_lock_speculate_retry (struct mm_struct *mm,unsigned int seq) +{ +return true; +} + +#endif /* CONFIG_PER_VMA_LOCK */ static inline void mmap_init_lock (struct mm_struct *mm) {
The following commit has been merged into the perf/core branch of tip:
The following commit has been merged into the perf/core branch of tip:
Commit-ID: 96450ead16527cbef559b5bd046182e731228f95 Gitweb: https://git.kernel.org/tip/96450ead16527cbef559b5bd046182e731228f95 Author: Suren Baghdasaryan <[email protected]> AuthorDate: Fri, 22 Nov 2024 09:44:14 -08:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 02 Dec 2024 12:01:37 +01:00
seqlock: add raw_seqcount_try_begin
Add raw_seqcount_try_begin() to opens a read critical section of the given seqcount_t if the counter is even. This enables eliding the critical section entirely if the counter is odd, instead of doing the speculation knowing it will fail.
--- include/linux/seqlock.h | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+)
include/linux/seqlock.h +22 -0
@@ -319,6 +319,28 @@ SEQCOUNT_LOCKNAME(mutex, struct mutex, true, mutex) })/** +* raw_seqcount_try_begin() - begin a seqcount_t read critical section +* w/o lockdep and w/o counter stabilization +* @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants +* +* Similar to raw_seqcount_begin(), except it enables eliding the critical +* section entirely if odd, instead of doing the speculation knowing it will +* fail. +* +* Useful when counter stabilization is more or less equivalent to taking +* the lock and there is a slowpath that does that. +* +* If true, start will be set to the (even) sequence count read. +* +* Return: true when a read critical section is started. +*/ +#define raw_seqcount_try_begin (s, start) \ +({ \ + start = raw_read_seqcount(s); \ + !(start & 1); \ +}) + +/** * raw_seqcount_begin() - begin a seqcount_t read critical section w/o * lockdep and w/o counter stabilization * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants