seqlock: add raw_seqcount_try_begin
Subsystems (7)
Suren Baghdasaryan
maintainerPATCH v2 1/3 seqlock: add raw_seqcount_try_begin
Add raw_seqcount_try_begin() to opens a read critical section of the given seqcount_t if the counter is even. This enables eliding the critical section entirely if the counter is odd, instead of doing the speculation knowing it will fail.
---
Applies over Linus' ToT
include/linux/seqlock.h | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+)
include/linux/seqlock.h +22 -0
@@ -318,6 +318,28 @@ SEQCOUNT_LOCKNAME(mutex, struct mutex, true, mutex) __seq; \ }) +/** +* raw_seqcount_try_begin() - begin a seqcount_t read critical section +* w/o lockdep and w/o counter stabilization +* @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants +* +* Similar to raw_seqcount_begin(), except it enables eliding the critical +* section entirely if odd, instead of doing the speculation knowing it will +* fail. +* +* Useful when counter stabilization is more or less equivalent to taking +* the lock and there is a slowpath that does that. +* +* If true, start will be set to the (even) sequence count read. +* +* Return: true when a read critical section is started. +*/ +#define raw_seqcount_try_begin (s, start) \ +({ \ + start = raw_read_seqcount(s); \ + !(start & 1); \ +}) +/** * raw_seqcount_begin() - begin a seqcount_t read critical section w/o * lockdep and w/o counter stabilization
base-commit: 43fb83c17ba2d63dfb798f0be7453ed55ca3f9c2
Signature · 1 line
2.47.0.338.g60cca15819-goog
Suren Baghdasaryan
maintainerPATCH v2 2/3 mm: convert mm_lock_seq to a proper seqcount
Convert mm_lock_seq to be seqcount_t and change all mmap_write_lock variants to increment it, in-line with the usual seqcount usage pattern. This lets us check whether the mmap_lock is write-locked by checking mm_lock_seq.sequence counter (odd=locked, even=unlocked). This will be used when implementing mmap_lock speculation functions.
As a result vm_lock_seq is also change to be unsigned to match the type of mm_lock_seq.sequence.
---
Changes since v1 [1]
- Added ASSERT_EXCLUSIVE_WRITER() instead of a comment in vma_end_write_all, per Peter Zijlstra
[1] https://lore.kernel.org/all/[email protected]/
include/linux/mm.h | 12 +++---- include/linux/mm_types.h | 7 ++-- include/linux/mmap_lock.h | 55 +++++++++++++++++++++----------- kernel/fork.c | 5 +-- mm/init-mm.c | 2 +- tools/testing/vma/vma.c | 4 +-- tools/testing/vma/vma_internal.h | 4 +-- 7 files changed, 53 insertions(+), 36 deletions(-)
include/linux/mm.h +6 -6
@@ -710,7 +710,7 @@ static inline bool vma_start_read(struct vm_area_struct *vma)* we don't rely on for anything - the mm_lock_seq read against which we * need ordering is below. */ -if (READ_ONCE (vma->vm_lock_seq) ==READ_ONCE (vma->vm_mm->mm_lock_seq)) +if (READ_ONCE (vma->vm_lock_seq) ==READ_ONCE (vma->vm_mm->mm_lock_seq.sequence))return false;if (unlikely (down_read_trylock (&vma->vm_lock->lock) ==0 ))
@@ -727,7 +727,7 @@ static inline bool vma_start_read(struct vm_area_struct *vma)* after it has been unlocked. * This pairs with RELEASE semantics in vma_end_write_all(). */ -if (unlikely (vma->vm_lock_seq ==smp_load_acquire (&vma->vm_mm->mm_lock_seq))) { +if (unlikely (vma->vm_lock_seq ==raw_read_seqcount (&vma->vm_mm->mm_lock_seq))) {up_read (&vma->vm_lock->lock);return false; }
@@ -742,7 +742,7 @@ static inline void vma_end_read(struct vm_area_struct *vma) }/* WARNING! Can only be used if mmap_lock is expected to be write-locked */ -static bool __is_vma_write_locked (struct vm_area_struct *vma,int *mm_lock_seq) +static bool __is_vma_write_locked (struct vm_area_struct *vma,unsigned int *mm_lock_seq) {mmap_assert_write_locked (vma->vm_mm);
@@ -750,7 +750,7 @@ static bool __is_vma_write_locked(struct vm_area_struct *vma, int *mm_lock_seq)* current task is holding mmap_write_lock, both vma->vm_lock_seq and * mm->mm_lock_seq can't be concurrently modified. */ - *mm_lock_seq = vma->vm_mm->mm_lock_seq; + *mm_lock_seq = vma->vm_mm->mm_lock_seq.sequence;return (vma->vm_lock_seq == *mm_lock_seq); }
@@ -761,7 +761,7 @@ static bool __is_vma_write_locked(struct vm_area_struct *vma, int *mm_lock_seq)*/ static inline void vma_start_write (struct vm_area_struct *vma) { -int mm_lock_seq; +unsigned int mm_lock_seq;if (__is_vma_write_locked (vma, &mm_lock_seq))return ;
@@ -779,7 +779,7 @@ static inline void vma_start_write(struct vm_area_struct *vma)static inline void vma_assert_write_locked (struct vm_area_struct *vma) { -int mm_lock_seq; +unsigned int mm_lock_seq;VM_BUG_ON_VMA (!__is_vma_write_locked (vma, &mm_lock_seq), vma); }
include/linux/mm_types.h +5 -2
@@ -715,7 +715,7 @@ struct vm_area_struct {* counter reuse can only lead to occasional unnecessary use of the * slowpath. */ -int vm_lock_seq; +unsigned int vm_lock_seq;/* Unstable RCU readers are allowed to read this. */ struct vma_lock *vm_lock;#endif
@@ -909,6 +909,9 @@ struct mm_struct {* Roughly speaking, incrementing the sequence number is * equivalent to releasing locks on VMAs; reading the sequence * number can be part of taking a read lock on a VMA. +* Incremented every time mmap_lock is write-locked/unlocked. +* Initialized to 0, therefore odd values indicate mmap_lock +* is write-locked and even values that it's released. * * Can be modified under write mmap_lock using RELEASE * semantics.
@@ -917,7 +920,7 @@ struct mm_struct {* Can be read with ACQUIRE semantics if not holding write * mmap_lock. */ -int mm_lock_seq; +seqcount_t mm_lock_seq;#endif
include/linux/mmap_lock.h +36 -19
91 lines not on this page · raw message
kernel/fork.c +1 -4
@@ -449,7 +449,7 @@ static bool vma_lock_alloc(struct vm_area_struct *vma)return false;init_rwsem (&vma->vm_lock->lock); - vma->vm_lock_seq =-1 ; + vma->vm_lock_seq =UINT_MAX ;return true; }
@@ -1262,9 +1262,6 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,seqcount_init (&mm->write_protect_seq);mmap_init_lock (mm);INIT_LIST_HEAD (&mm->mmlist); -#ifdef CONFIG_PER_VMA_LOCK - mm->mm_lock_seq =0 ; -#endif mm_pgtables_bytes_init (mm); mm->map_count =0 ; mm->locked_vm =0 ;
mm/init-mm.c +1 -1
@@ -40,7 +40,7 @@ struct mm_struct init_mm = { .arg_lock = __SPIN_LOCK_UNLOCKED(init_mm.arg_lock), .mmlist =LIST_HEAD_INIT (init_mm.mmlist), #ifdef CONFIG_PER_VMA_LOCK - .mm_lock_seq = 0, + .mm_lock_seq = SEQCNT_ZERO(init_mm.mm_lock_seq), #endif .user_ns = &init_user_ns, .cpu_bitmap = CPU_BITS_NONE,
tools/testing/vma/vma.c +2 -2
@@ -87,7 +87,7 @@ static struct vm_area_struct *alloc_and_link_vma(struct mm_struct *mm,* begun. Linking to the tree will have caused this to be incremented, * which means we will get a false positive otherwise. */ - vma->vm_lock_seq =-1 ; + vma->vm_lock_seq =UINT_MAX ;return vma; }
@@ -212,7 +212,7 @@ static bool vma_write_started(struct vm_area_struct *vma)int seq = vma->vm_lock_seq;/* We reset after each check. */ - vma->vm_lock_seq =-1 ; + vma->vm_lock_seq =UINT_MAX ;/* The vma_start_write() stub simply increments this value. */ return seq >-1 ;
tools/testing/vma/vma_internal.h +2 -2
@@ -231,7 +231,7 @@ struct vm_area_struct {* counter reuse can only lead to occasional unnecessary use of the * slowpath. */ -int vm_lock_seq; +unsigned int vm_lock_seq;struct vma_lock *vm_lock;#endif
@@ -406,7 +406,7 @@ static inline bool vma_lock_alloc(struct vm_area_struct *vma)return false;init_rwsem (&vma->vm_lock->lock); - vma->vm_lock_seq =-1 ; + vma->vm_lock_seq =UINT_MAX ;return true; }
Signature · 1 line
2.47.0.338.g60cca15819-goog
Re: [PATCH v2 2/3] mm: convert mm_lock_seq to a proper seqcount
Quoted 10 lines · Suren Baghdasaryan
Convert
mm_lock_seqto beseqcount_tand change allmmap_write_lockvariants to increment it, in-line with the usual seqcount usage pattern. This lets us check whether themmap_lockis write-locked by checking mm_lock_seq.sequence counter (odd=locked, even=unlocked). This will be used when implementingmmap_lockspeculation functions.
As a resultvm_lock_seqis also change to be unsigned to match the type of mm_lock_seq.sequence.Suggested-by Peter Zijlstra <[email protected]>Signed-off-by Suren Baghdasaryan <[email protected]>
Quoted 287 lines
---
Changes since v1 [1]
- AddedASSERT_EXCLUSIVE_WRITER()instead of a comment invma_end_write_all, per Peter Zijlstra[1] https://lore.kernel.org/all/[email protected]/
include/linux/mm.h | 12 +++---- include/linux/mm_types.h | 7 ++-- include/linux/mmap_lock.h | 55 +++++++++++++++++++++----------- kernel/fork.c | 5 +-- mm/init-mm.c | 2 +- tools/testing/vma/vma.c | 4 +-- tools/testing/vma/vma_internal.h | 4 +-- 7 files changed, 53 insertions(+), 36 deletions(-)include/linux/mm.h +6
-6@@ -710,7 +710,7 @@ static inline bool vma_start_read(struct vm_area_struct *vma)* we don't rely on for anything - the mm_lock_seq read against which we * need ordering is below. */ -if (READ_ONCE (vma->vm_lock_seq) ==READ_ONCE (vma->vm_mm->mm_lock_seq)) +if (READ_ONCE (vma->vm_lock_seq) ==READ_ONCE (vma->vm_mm->mm_lock_seq.sequence))return false;if (unlikely (down_read_trylock (&vma->vm_lock->lock) ==0 ))@@ -727,7 +727,7 @@ static inline bool vma_start_read(struct vm_area_struct *vma)* after it has been unlocked. * This pairs with RELEASE semantics in vma_end_write_all(). */ -if (unlikely (vma->vm_lock_seq ==smp_load_acquire (&vma->vm_mm->mm_lock_seq))) { +if (unlikely (vma->vm_lock_seq ==raw_read_seqcount (&vma->vm_mm->mm_lock_seq))) {up_read (&vma->vm_lock->lock);return false; }@@ -742,7 +742,7 @@ static inline void vma_end_read(struct vm_area_struct *vma) }/* WARNING! Can only be used if mmap_lock is expected to be write-locked */ -static bool __is_vma_write_locked (struct vm_area_struct *vma,int *mm_lock_seq) +static bool __is_vma_write_locked (struct vm_area_struct *vma,unsigned int *mm_lock_seq) {mmap_assert_write_locked (vma->vm_mm);@@ -750,7 +750,7 @@ static bool __is_vma_write_locked(struct vm_area_struct *vma, int *mm_lock_seq)* current task is holding mmap_write_lock, both vma->vm_lock_seq and * mm->mm_lock_seq can't be concurrently modified. */ - *mm_lock_seq = vma->vm_mm->mm_lock_seq; + *mm_lock_seq = vma->vm_mm->mm_lock_seq.sequence;return (vma->vm_lock_seq == *mm_lock_seq); }@@ -761,7 +761,7 @@ static bool __is_vma_write_locked(struct vm_area_struct *vma, int *mm_lock_seq)*/ static inline void vma_start_write (struct vm_area_struct *vma) { -int mm_lock_seq; +unsigned int mm_lock_seq;if (__is_vma_write_locked (vma, &mm_lock_seq))return ;@@ -779,7 +779,7 @@ static inline void vma_start_write(struct vm_area_struct *vma)static inline void vma_assert_write_locked (struct vm_area_struct *vma) { -int mm_lock_seq; +unsigned int mm_lock_seq;VM_BUG_ON_VMA (!__is_vma_write_locked (vma, &mm_lock_seq), vma); }include/linux/mm_types.h +5
-2@@ -715,7 +715,7 @@ struct vm_area_struct {* counter reuse can only lead to occasional unnecessary use of the * slowpath. */ -int vm_lock_seq; +unsigned int vm_lock_seq;/* Unstable RCU readers are allowed to read this. */ struct vma_lock *vm_lock;#endif @@ -909,6 +909,9 @@ struct mm_struct {* Roughly speaking, incrementing the sequence number is * equivalent to releasing locks on VMAs; reading the sequence * number can be part of taking a read lock on a VMA. +* Incremented every time mmap_lock is write-locked/unlocked. +* Initialized to 0, therefore odd values indicate mmap_lock +* is write-locked and even values that it's released. * * Can be modified under write mmap_lock using RELEASE * semantics. @@ -917,7 +920,7 @@ struct mm_struct {* Can be read with ACQUIRE semantics if not holding write * mmap_lock. */ -int mm_lock_seq; +seqcount_t mm_lock_seq;#endif include/linux/mmap_lock.h +36
-19@@ -71,39 +71,38 @@ static inline void mmap_assert_write_locked(const struct mm_struct *mm) }#ifdef CONFIG_PER_VMA_LOCK -/* -* Drop all currently-held per-VMA locks. -* This is called from the mmap_lock implementation directly before releasing -* a write-locked mmap_lock (or downgrading it to read-locked). -* This should normally NOT be called manually from other places. -* If you want to call this manually anyway, keep in mind that this will release -* *all* VMA write locks, including ones from further up the stack. -*/ -static inline void vma_end_write_all (struct mm_struct *mm) +static inline void mm_lock_seqcount_init (struct mm_struct *mm) { -mmap_assert_write_locked (mm); -/* -* Nobody can concurrently modify mm->mm_lock_seq due to exclusive -* mmap_lock being held. -* We need RELEASE semantics here to ensure that preceding stores into -* the VMA take effect before we unlock it with this store. -* Pairs with ACQUIRE semantics in vma_start_read(). -*/ -smp_store_release (&mm->mm_lock_seq, mm->mm_lock_seq +1 ); +seqcount_init (&mm->mm_lock_seq); +} + +static inline void mm_lock_seqcount_begin (struct mm_struct *mm) +{ +do_raw_write_seqcount_begin (&mm->mm_lock_seq); +} + +static inline void mm_lock_seqcount_end (struct mm_struct *mm) +{ +do_raw_write_seqcount_end (&mm->mm_lock_seq); } +#else -static inline void vma_end_write_all (struct mm_struct *mm) {} +static inline void mm_lock_seqcount_init (struct mm_struct *mm) {} +static inline void mm_lock_seqcount_begin (struct mm_struct *mm) {} +static inline void mm_lock_seqcount_end (struct mm_struct *mm) {}#endif static inline void mmap_init_lock (struct mm_struct *mm) {init_rwsem (&mm->mmap_lock); +mm_lock_seqcount_init (mm); }static inline void mmap_write_lock (struct mm_struct *mm) {__mmap_lock_trace_start_locking (mm, true);down_write (&mm->mmap_lock); +mm_lock_seqcount_begin (mm);__mmap_lock_trace_acquire_returned (mm, true, true); }@@ -111,6 +110,7 @@ static inline void mmap_write_lock_nested(struct mm_struct *mm, int subclass) {__mmap_lock_trace_start_locking (mm, true);down_write_nested (&mm->mmap_lock, subclass); +mm_lock_seqcount_begin (mm);__mmap_lock_trace_acquire_returned (mm, true, true); }@@ -120,10 +120,27 @@ static inline int mmap_write_lock_killable(struct mm_struct *mm)__mmap_lock_trace_start_locking (mm, true); ret =down_write_killable (&mm->mmap_lock); +if (!ret) +mm_lock_seqcount_begin (mm);__mmap_lock_trace_acquire_returned (mm, true, ret ==0 );return ret; } +/* +* Drop all currently-held per-VMA locks. +* This is called from the mmap_lock implementation directly before releasing +* a write-locked mmap_lock (or downgrading it to read-locked). +* This should normally NOT be called manually from other places. +* If you want to call this manually anyway, keep in mind that this will release +* *all* VMA write locks, including ones from further up the stack. +*/ +static inline void vma_end_write_all (struct mm_struct *mm) +{ +mmap_assert_write_locked (mm); +ASSERT_EXCLUSIVE_WRITER (mm->mm_lock_seq); +mm_lock_seqcount_end (mm); +} +static inline void mmap_write_unlock (struct mm_struct *mm) {__mmap_lock_trace_released (mm, true);kernel/fork.c +1
-4@@ -449,7 +449,7 @@ static bool vma_lock_alloc(struct vm_area_struct *vma)return false;init_rwsem (&vma->vm_lock->lock); - vma->vm_lock_seq =-1 ; + vma->vm_lock_seq =UINT_MAX ;return true; }@@ -1262,9 +1262,6 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,seqcount_init (&mm->write_protect_seq);mmap_init_lock (mm);INIT_LIST_HEAD (&mm->mmlist); -#ifdef CONFIG_PER_VMA_LOCK - mm->mm_lock_seq =0 ; -#endif mm_pgtables_bytes_init (mm); mm->map_count =0 ; mm->locked_vm =0 ;mm/init-mm.c +1
-1@@ -40,7 +40,7 @@ struct mm_struct init_mm = { .arg_lock = __SPIN_LOCK_UNLOCKED(init_mm.arg_lock), .mmlist =LIST_HEAD_INIT (init_mm.mmlist), #ifdef CONFIG_PER_VMA_LOCK - .mm_lock_seq = 0, + .mm_lock_seq = SEQCNT_ZERO(init_mm.mm_lock_seq), #endif .user_ns = &init_user_ns, .cpu_bitmap = CPU_BITS_NONE,tools/testing/vma/vma.c +2
-2@@ -87,7 +87,7 @@ static struct vm_area_struct *alloc_and_link_vma(struct mm_struct *mm,* begun. Linking to the tree will have caused this to be incremented, * which means we will get a false positive otherwise. */ - vma->vm_lock_seq =-1 ; + vma->vm_lock_seq =UINT_MAX ;return vma; }@@ -212,7 +212,7 @@ static bool vma_write_started(struct vm_area_struct *vma)int seq = vma->vm_lock_seq;/* We reset after each check. */ - vma->vm_lock_seq =-1 ; + vma->vm_lock_seq =UINT_MAX ;/* The vma_start_write() stub simply increments this value. */ return seq >-1 ;tools/testing/vma/vma_internal.h +2
-2@@ -231,7 +231,7 @@ struct vm_area_struct {* counter reuse can only lead to occasional unnecessary use of the * slowpath. */ -int vm_lock_seq; +unsigned int vm_lock_seq;struct vma_lock *vm_lock;#endif @@ -406,7 +406,7 @@ static inline bool vma_lock_alloc(struct vm_area_struct *vma)return false;init_rwsem (&vma->vm_lock->lock); - vma->vm_lock_seq =-1 ; + vma->vm_lock_seq =UINT_MAX ;return true; }Signature · 1 line
2.47.0.338.g60cca15819-goog
kernel test robot
build botRe: [PATCH v2 2/3] mm: convert mm_lock_seq to a proper seqcount
In reply to Suren Baghdasaryankernel test robot noticed the following build errors:
Hi Suren,
kernel test robot noticed the following build errors:
[auto build test ERROR on 43fb83c17ba2d63dfb798f0be7453ed55ca3f9c2]
url: https://github.com/intel-lab-lkp/linux/commits/Suren-Baghdasaryan/mm-convert-mm_lock_seq-to-a-proper-seqcount/20241122-003050 base: 43fb83c17ba2d63dfb798f0be7453ed55ca3f9c2 patch link: https://lore.kernel.org/r/20241121162826.987947-2-surenb%40google.com patch subject: [PATCH v2 2/3] mm: convert mm_lock_seq to a proper seqcount config: sh-allmodconfig (https://download.01.org/0day-ci/archive/20241123/[email protected]/config) compiler: sh4-linux-gcc (GCC) 14.2.0 reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20241123/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All errors (new ones prefixed by >>):
In file included from include/asm-generic/rwonce.h:27,
from ./arch/sh/include/generated/asm/rwonce.h:1, from include/linux/compiler.h:317, from include/linux/export.h:5,
from include/linux/linkage.h:7,
from arch/sh/include/asm/bug.h:5, from include/linux/bug.h:5,
from include/linux/mmdebug.h:5,
from include/linux/mm.h:6,
from arch/sh/kernel/asm-offsets.c:14:
include/linux/mmap_lock.h: In function 'vma_end_write_all':
>> include/linux/mmap_lock.h:140:35: error: 'struct mm_struct' has no member named 'mm_lock_seq'
140 | ASSERT_EXCLUSIVE_WRITER(mm->mm_lock_seq);
| ^~
include/linux/kcsan-checks.h:371:32: note: in definition of macro 'ASSERT_EXCLUSIVE_WRITER'
371 | __kcsan_check_access(&(var), sizeof(var), KCSAN_ACCESS_ASSERT)
| ^~~
>> include/linux/mmap_lock.h:140:35: error: 'struct mm_struct' has no member named 'mm_lock_seq'
140 | ASSERT_EXCLUSIVE_WRITER(mm->mm_lock_seq);
| ^~ include/linux/kcsan-checks.h:371:45: note: in definition of macro 'ASSERT_EXCLUSIVE_WRITER'
371 | __kcsan_check_access(&(var), sizeof(var), KCSAN_ACCESS_ASSERT)
| ^~~
make[3]: *** [scripts/Makefile.build:102: arch/sh/kernel/asm-offsets.s] Error 1 make[3]: Target 'prepare' not remade because of errors.
make[2]: *** [Makefile:1203: prepare0] Error 2
make[2]: Target 'prepare' not remade because of errors.
make[1]: *** [Makefile:224: __sub-make] Error 2
make[1]: Target 'prepare' not remade because of errors.
make: *** [Makefile:224: __sub-make] Error 2
make: Target 'prepare' not remade because of errors.
vim +140 include/linux/mmap_lock.h
128
129 /*
130 * Drop all currently-held per-VMA locks.
131 * This is called from the mmap_lock implementation directly before releasing
132 * a write-locked mmap_lock (or downgrading it to read-locked).
133 * This should normally NOT be called manually from other places.
134 * If you want to call this manually anyway, keep in mind that this will release
135 * *all* VMA write locks, including ones from further up the stack.
136 */
137 static inline void vma_end_write_all(struct mm_struct *mm)
138 {
139 mmap_assert_write_locked(mm);
> 140 ASSERT_EXCLUSIVE_WRITER(mm->mm_lock_seq);
141 mm_lock_seqcount_end(mm);
142 }
143
Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki
Suren Baghdasaryan
maintainerRe: [PATCH v2 2/3] mm: convert mm_lock_seq to a proper seqcount
Quoted 33 lines · kernel test robot
Hi Suren,
kernel test robot noticed the following build errors:
[auto build test ERROR on
43fb83c17ba2d63dfb798f0be7453ed55ca3f9c2]url: https://github.com/intel-lab-lkp/linux/commits/Suren-Baghdasaryan/mm-convert-mm_lock_seq-to-a-proper-seqcount/20241122-003050 base: 43fb83c17ba2d63dfb798f0be7453ed55ca3f9c2 patch link: https://lore.kernel.org/r/20241121162826.987947-2-surenb%40google.com patch subject: [PATCH v2 2/3] mm: convert mm_lock_seq to a proper seqcount config: sh-allmodconfig (https://download.01.org/0day-ci/archive/20241123/[email protected]/config) compiler: sh4-linux-gcc (GCC) 14.2.0 reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20241123/[email protected]/reproduce)If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/All errors (new ones prefixed by >>):
In file included from
include/asm-generic/rwonce.h:27,
from./arch/sh/include/generated/asm/rwonce.h:1, frominclude/linux/compiler.h:317, frominclude/linux/export.h:5,
frominclude/linux/linkage.h:7,
fromarch/sh/include/asm/bug.h:5, frominclude/linux/bug.h:5,
frominclude/linux/mmdebug.h:5,
frominclude/linux/mm.h:6,
fromarch/sh/kernel/asm-offsets.c:14:
include/linux/mmap_lock.h: In function 'vma_end_write_all':
Quoted 3 lines
Quoted 1 line
Quoted 1 line
include/linux/mmap_lock.h:140:35: error: 'structmm_struct' has no member named 'mm_lock_seq'140 | ASSERT_EXCLUSIVE_WRITER(mm->mm_lock_seq); | ^~
Oh, this is due to CONFIG_PER_VMA_LOCK=n. Trivial fix is to move ASSERT_EXCLUSIVE_WRITER() into appropriate mm_lock_seqcount_end(). Will re-post with the fix shortly.
Quoted 42 lines
include/linux/kcsan-checks.h:371:32: note: in definition of macro 'ASSERT_EXCLUSIVE_WRITER'
371 |__kcsan_check_access(&(var), sizeof(var), KCSAN_ACCESS_ASSERT)
| ^~~Quoted 1 line
Quoted 1 line
include/linux/mmap_lock.h:140:35: error: 'structmm_struct' has no member named 'mm_lock_seq'140 | ASSERT_EXCLUSIVE_WRITER(mm->mm_lock_seq); | ^~
include/linux/kcsan-checks.h:371:45: note: in definition of macro 'ASSERT_EXCLUSIVE_WRITER'
371 |__kcsan_check_access(&(var), sizeof(var), KCSAN_ACCESS_ASSERT)
| ^~~
make[3]: *** [scripts/Makefile.build:102:arch/sh/kernel/asm-offsets.s] Error 1 make[3]: Target 'prepare' not remade because of errors.
make[2]: *** [Makefile:1203: prepare0] Error 2
make[2]: Target 'prepare' not remade because of errors.
make[1]: *** [Makefile:224: __sub-make] Error 2
make[1]: Target 'prepare' not remade because of errors.
make: *** [Makefile:224: __sub-make] Error 2
make: Target 'prepare' not remade because of errors.vim +140
include/linux/mmap_lock.h128
129 /*130 * Drop all currently-held per-VMA locks. 131 * This is called from the mmap_lock implementation directly before releasing 132 * a write-locked mmap_lock (or downgrading it to read-locked). 133 * This should normally NOT be called manually from other places. 134 * If you want to call this manually anyway, keep in mind that this will release 135 * *all* VMA write locks, including ones from further up the stack. 136 */ 137 static inline void vma_end_write_all(struct mm_struct *mm) 138 { 139 mmap_assert_write_locked(mm);Quoted 1 line
140
ASSERT_EXCLUSIVE_WRITER(mm->mm_lock_seq);141 mm_lock_seqcount_end(mm); 142 } 143--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
Suren Baghdasaryan
maintainerRe: [PATCH v2 2/3] mm: convert mm_lock_seq to a proper seqcount
Quoted 35 lines · Suren Baghdasaryan
Quoted 33 lines · kernel test robot
Hi Suren,
kernel test robot noticed the following build errors:
[auto build test ERROR on
43fb83c17ba2d63dfb798f0be7453ed55ca3f9c2]url: https://github.com/intel-lab-lkp/linux/commits/Suren-Baghdasaryan/mm-convert-mm_lock_seq-to-a-proper-seqcount/20241122-003050 base: 43fb83c17ba2d63dfb798f0be7453ed55ca3f9c2 patch link: https://lore.kernel.org/r/20241121162826.987947-2-surenb%40google.com patch subject: [PATCH v2 2/3] mm: convert mm_lock_seq to a proper seqcount config: sh-allmodconfig (https://download.01.org/0day-ci/archive/20241123/[email protected]/config) compiler: sh4-linux-gcc (GCC) 14.2.0 reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20241123/[email protected]/reproduce)If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/All errors (new ones prefixed by >>):
In file included from
include/asm-generic/rwonce.h:27,
from./arch/sh/include/generated/asm/rwonce.h:1, frominclude/linux/compiler.h:317, frominclude/linux/export.h:5,
frominclude/linux/linkage.h:7,
fromarch/sh/include/asm/bug.h:5, frominclude/linux/bug.h:5,
frominclude/linux/mmdebug.h:5,
frominclude/linux/mm.h:6,
fromarch/sh/kernel/asm-offsets.c:14:
include/linux/mmap_lock.h: In function 'vma_end_write_all':
Quoted 7 lines
Quoted 3 lines
Quoted 1 line
Quoted 1 line
include/linux/mmap_lock.h:140:35: error: 'structmm_struct' has no member named 'mm_lock_seq'140 | ASSERT_EXCLUSIVE_WRITER(mm->mm_lock_seq); | ^~Oh, this is due to
CONFIG_PER_VMA_LOCK=n. Trivial fix is to moveASSERT_EXCLUSIVE_WRITER()into appropriatemm_lock_seqcount_end(). Will re-post with the fix shortly.
Fixed in v3 here:
https://lore.kernel.org/all/[email protected]/
Quoted 44 lines
Quoted 44 lines
include/linux/kcsan-checks.h:371:32: note: in definition of macro 'ASSERT_EXCLUSIVE_WRITER'
371 |__kcsan_check_access(&(var), sizeof(var), KCSAN_ACCESS_ASSERT)
| ^~~Quoted 1 line
Quoted 1 line
include/linux/mmap_lock.h:140:35: error: 'structmm_struct' has no member named 'mm_lock_seq'140 | ASSERT_EXCLUSIVE_WRITER(mm->mm_lock_seq); | ^~
include/linux/kcsan-checks.h:371:45: note: in definition of macro 'ASSERT_EXCLUSIVE_WRITER'
371 |__kcsan_check_access(&(var), sizeof(var), KCSAN_ACCESS_ASSERT)
| ^~~
make[3]: *** [scripts/Makefile.build:102:arch/sh/kernel/asm-offsets.s] Error 1 make[3]: Target 'prepare' not remade because of errors.
make[2]: *** [Makefile:1203: prepare0] Error 2
make[2]: Target 'prepare' not remade because of errors.
make[1]: *** [Makefile:224: __sub-make] Error 2
make[1]: Target 'prepare' not remade because of errors.
make: *** [Makefile:224: __sub-make] Error 2
make: Target 'prepare' not remade because of errors.vim +140
include/linux/mmap_lock.h128
129 /*130 * Drop all currently-held per-VMA locks. 131 * This is called from the mmap_lock implementation directly before releasing 132 * a write-locked mmap_lock (or downgrading it to read-locked). 133 * This should normally NOT be called manually from other places. 134 * If you want to call this manually anyway, keep in mind that this will release 135 * *all* VMA write locks, including ones from further up the stack. 136 */ 137 static inline void vma_end_write_all(struct mm_struct *mm) 138 { 139 mmap_assert_write_locked(mm);Quoted 1 line
140
ASSERT_EXCLUSIVE_WRITER(mm->mm_lock_seq);141 mm_lock_seqcount_end(mm); 142 } 143--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
kernel test robot
build botRe: [PATCH v2 2/3] mm: convert mm_lock_seq to a proper seqcount
In reply to Suren Baghdasaryankernel test robot noticed the following build errors:
Hi Suren,
kernel test robot noticed the following build errors:
[auto build test ERROR on 43fb83c17ba2d63dfb798f0be7453ed55ca3f9c2]
url: https://github.com/intel-lab-lkp/linux/commits/Suren-Baghdasaryan/mm-convert-mm_lock_seq-to-a-proper-seqcount/20241122-003050 base: 43fb83c17ba2d63dfb798f0be7453ed55ca3f9c2 patch link: https://lore.kernel.org/r/20241121162826.987947-2-surenb%40google.com patch subject: [PATCH v2 2/3] mm: convert mm_lock_seq to a proper seqcount config: hexagon-allmodconfig (https://download.01.org/0day-ci/archive/20241123/[email protected]/config) compiler: clang version 20.0.0git (https://github.com/llvm/llvm-project 592c0fe55f6d9a811028b5f3507be91458ab2713) reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20241123/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All errors (new ones prefixed by >>):
In file included from drivers/char/tpm/tpm-chip.c:24:
In file included from include/linux/tpm_eventlog.h:6:
In file included from include/linux/tpm.h:24:
In file included from include/linux/highmem.h:10:
In file included from include/linux/mm.h:16:
>> include/linux/mmap_lock.h:140:30: error: no member named 'mm_lock_seq' in 'struct mm_struct'
140 | ASSERT_EXCLUSIVE_WRITER(mm->mm_lock_seq);
| ~~ ^
include/linux/kcsan-checks.h:371:25: note: expanded from macro 'ASSERT_EXCLUSIVE_WRITER'
371 | __kcsan_check_access(&(var), sizeof(var), KCSAN_ACCESS_ASSERT)
| ^~~
In file included from drivers/char/tpm/tpm-chip.c:24:
In file included from include/linux/tpm_eventlog.h:6:
In file included from include/linux/tpm.h:24:
In file included from include/linux/highmem.h:10:
In file included from include/linux/mm.h:16:
>> include/linux/mmap_lock.h:140:30: error: no member named 'mm_lock_seq' in 'struct mm_struct'
140 | ASSERT_EXCLUSIVE_WRITER(mm->mm_lock_seq);
| ~~ ^
include/linux/kcsan-checks.h:371:38: note: expanded from macro 'ASSERT_EXCLUSIVE_WRITER'
371 | __kcsan_check_access(&(var), sizeof(var), KCSAN_ACCESS_ASSERT)
| ^~~
In file included from drivers/char/tpm/tpm-chip.c:24:
In file included from include/linux/tpm_eventlog.h:6:
In file included from include/linux/tpm.h:24:
In file included from include/linux/highmem.h:10:
In file included from include/linux/mm.h:2225:
include/linux/vmstat.h:518:36: warning: arithmetic between different enumeration types ('enum node_stat_item' and 'enum lru_list') [-Wenum-enum-conversion]
518 | return node_stat_name(NR_LRU_BASE + lru) + 3; // skip "nr_"
| ~~~~~~~~~~~ ^ ~~~
In file included from drivers/char/tpm/tpm-chip.c:24:
include/linux/tpm_eventlog.h:167:6: warning: variable 'mapping_size' set but not used [-Wunused-but-set-variable]
167 | int mapping_size;
| ^ 2 warnings and 2 errors generated.
--
In file included from drivers/char/tpm/tpm-buf.c:8:
In file included from include/linux/tpm.h:24:
In file included from include/linux/highmem.h:10:
In file included from include/linux/mm.h:16:
>> include/linux/mmap_lock.h:140:30: error: no member named 'mm_lock_seq' in 'struct mm_struct'
140 | ASSERT_EXCLUSIVE_WRITER(mm->mm_lock_seq);
| ~~ ^
include/linux/kcsan-checks.h:371:25: note: expanded from macro 'ASSERT_EXCLUSIVE_WRITER'
371 | __kcsan_check_access(&(var), sizeof(var), KCSAN_ACCESS_ASSERT)
| ^~~
In file included from drivers/char/tpm/tpm-buf.c:8:
In file included from include/linux/tpm.h:24:
In file included from include/linux/highmem.h:10:
In file included from include/linux/mm.h:16:
>> include/linux/mmap_lock.h:140:30: error: no member named 'mm_lock_seq' in 'struct mm_struct'
140 | ASSERT_EXCLUSIVE_WRITER(mm->mm_lock_seq);
| ~~ ^
include/linux/kcsan-checks.h:371:38: note: expanded from macro 'ASSERT_EXCLUSIVE_WRITER'
371 | __kcsan_check_access(&(var), sizeof(var), KCSAN_ACCESS_ASSERT)
| ^~~
In file included from drivers/char/tpm/tpm-buf.c:8:
In file included from include/linux/tpm.h:24:
In file included from include/linux/highmem.h:10:
In file included from include/linux/mm.h:2225:
include/linux/vmstat.h:518:36: warning: arithmetic between different enumeration types ('enum node_stat_item' and 'enum lru_list') [-Wenum-enum-conversion]
518 | return node_stat_name(NR_LRU_BASE + lru) + 3; // skip "nr_"
| ~~~~~~~~~~~ ^ ~~~ 1 warning and 2 errors generated.
--
In file included from drivers/char/tpm/tpm_tis_spi_main.c:32:
In file included from include/linux/spi/spi.h:17:
In file included from include/linux/scatterlist.h:8:
In file included from include/linux/mm.h:16:
>> include/linux/mmap_lock.h:140:30: error: no member named 'mm_lock_seq' in 'struct mm_struct'
140 | ASSERT_EXCLUSIVE_WRITER(mm->mm_lock_seq);
| ~~ ^
include/linux/kcsan-checks.h:371:25: note: expanded from macro 'ASSERT_EXCLUSIVE_WRITER'
371 | __kcsan_check_access(&(var), sizeof(var), KCSAN_ACCESS_ASSERT)
| ^~~
In file included from drivers/char/tpm/tpm_tis_spi_main.c:32:
In file included from include/linux/spi/spi.h:17:
In file included from include/linux/scatterlist.h:8:
In file included from include/linux/mm.h:16:
>> include/linux/mmap_lock.h:140:30: error: no member named 'mm_lock_seq' in 'struct mm_struct'
140 | ASSERT_EXCLUSIVE_WRITER(mm->mm_lock_seq);
| ~~ ^
include/linux/kcsan-checks.h:371:38: note: expanded from macro 'ASSERT_EXCLUSIVE_WRITER'
371 | __kcsan_check_access(&(var), sizeof(var), KCSAN_ACCESS_ASSERT)
| ^~~
In file included from drivers/char/tpm/tpm_tis_spi_main.c:32:
In file included from include/linux/spi/spi.h:17:
In file included from include/linux/scatterlist.h:8:
In file included from include/linux/mm.h:1132:
In file included from include/linux/huge_mm.h:8:
In file included from include/linux/fs.h:33:
In file included from include/linux/percpu-rwsem.h:7:
In file included from include/linux/rcuwait.h:6:
In file included from include/linux/sched/signal.h:6:
include/linux/signal.h:98:11: warning: array index 3 is past the end of the array (that has type 'unsigned long[2]') [-Warray-bounds]
98 | return (set->sig[3] | set->sig[2] |
| ^ ~
include/uapi/asm-generic/signal.h:62:2: note: array 'sig' declared here
62 | unsigned long sig[_NSIG_WORDS];
| ^
In file included from drivers/char/tpm/tpm_tis_spi_main.c:32:
In file included from include/linux/spi/spi.h:17:
In file included from include/linux/scatterlist.h:8:
In file included from include/linux/mm.h:1132:
In file included from include/linux/huge_mm.h:8:
In file included from include/linux/fs.h:33:
In file included from include/linux/percpu-rwsem.h:7:
In file included from include/linux/rcuwait.h:6:
In file included from include/linux/sched/signal.h:6:
include/linux/signal.h:98:25: warning: array index 2 is past the end of the array (that has type 'unsigned long[2]') [-Warray-bounds]
98 | return (set->sig[3] | set->sig[2] |
| ^ ~
include/uapi/asm-generic/signal.h:62:2: note: array 'sig' declared here
62 | unsigned long sig[_NSIG_WORDS];
| ^
In file included from drivers/char/tpm/tpm_tis_spi_main.c:32:
In file included from include/linux/spi/spi.h:17:
In file included from include/linux/scatterlist.h:8:
In file included from include/linux/mm.h:1132:
In file included from include/linux/huge_mm.h:8:
In file included from include/linux/fs.h:33:
In file included from include/linux/percpu-rwsem.h:7:
In file included from include/linux/rcuwait.h:6:
In file included from include/linux/sched/signal.h:6:
include/linux/signal.h:114:11: warning: array index 3 is past the end of the array (that has type 'const unsigned long[2]') [-Warray-bounds]
114 | return (set1->sig[3] == set2->sig[3]) &&
| ^ ~
include/uapi/asm-generic/signal.h:62:2: note: array 'sig' declared here
62 | unsigned long sig[_NSIG_WORDS];
| ^
In file included from drivers/char/tpm/tpm_tis_spi_main.c:32:
In file included from include/linux/spi/spi.h:17:
In file included from include/linux/scatterlist.h:8:
In file included from include/linux/mm.h:1132:
In file included from include/linux/huge_mm.h:8:
In file included from include/linux/fs.h:33:
In file included from include/linux/percpu-rwsem.h:7:
In file included from include/linux/rcuwait.h:6:
In file included from include/linux/sched/signal.h:6:
include/linux/signal.h:114:27: warning: array index 3 is past the end of the array (that has type 'const unsigned long[2]') [-Warray-bounds]
114 | return (set1->sig[3] == set2->sig[3]) &&
| ^ ~
include/uapi/asm-generic/signal.h:62:2: note: array 'sig' declared here
62 | unsigned long sig[_NSIG_WORDS];
| ^
In file included from drivers/char/tpm/tpm_tis_spi_main.c:32:
In file included from include/linux/spi/spi.h:17:
In file included from include/linux/scatterlist.h:8:
In file included from include/linux/mm.h:1132:
In file included from include/linux/huge_mm.h:8:
In file included from include/linux/fs.h:33:
In file included from include/linux/percpu-rwsem.h:7:
In file included from include/linux/rcuwait.h:6:
In file included from include/linux/sched/signal.h:6:
include/linux/signal.h:115:5: warning: array index 2 is past the end of the array (that has type 'const unsigned long[2]') [-Warray-bounds]
115 | (set1->sig[2] == set2->sig[2]) &&
| ^ ~
include/uapi/asm-generic/signal.h:62:2: note: array 'sig' declared here
62 | unsigned long sig[_NSIG_WORDS];
| ^
In file included from drivers/char/tpm/tpm_tis_spi_main.c:32:
In file included from include/linux/spi/spi.h:17:
In file included from include/linux/scatterlist.h:8:
In file included from include/linux/mm.h:1132:
In file included from include/linux/huge_mm.h:8:
In file included from include/linux/fs.h:33:
In file included from include/linux/percpu-rwsem.h:7:
In file included from include/linux/rcuwait.h:6:
In file included from include/linux/sched/signal.h:6:
include/linux/signal.h:115:21: warning: array index 2 is past the end of the array (that has type 'const unsigned long[2]') [-Warray-bounds]
115 | (set1->sig[2] == set2->sig[2]) &&
| ^ ~
include/uapi/asm-generic/signal.h:62:2: note: array 'sig' declared here
62 | unsigned long sig[_NSIG_WORDS];
| ^ In file included from drivers/char/tpm/tpm_tis_spi_main.c:32:
In file included from include/linux/spi/spi.h:17:
In file included from include/linux/scatterlist.h:8:
In file included from include/linux/mm.h:1132:
In file included from include/linux/huge_mm.h:8:
vim +140 include/linux/mmap_lock.h
128
129 /*
130 * Drop all currently-held per-VMA locks.
131 * This is called from the mmap_lock implementation directly before releasing
132 * a write-locked mmap_lock (or downgrading it to read-locked).
133 * This should normally NOT be called manually from other places.
134 * If you want to call this manually anyway, keep in mind that this will release
135 * *all* VMA write locks, including ones from further up the stack.
136 */
137 static inline void vma_end_write_all(struct mm_struct *mm)
138 {
139 mmap_assert_write_locked(mm);
> 140 ASSERT_EXCLUSIVE_WRITER(mm->mm_lock_seq);
141 mm_lock_seqcount_end(mm);
142 }
143
Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki
Suren Baghdasaryan
maintainerPATCH v2 3/3 mm: introduce mmap_lock_speculate_{try_begin|retry}
In reply to Suren BaghdasaryanAdd helper functions to speculatively perform operations without read-locking mmap_lock, expecting that mmap_lock will not be write-locked and mm is not modified from under us.
---
Changes since v1 [1]
- Changed to use new raw_seqcount_try_begin() API, per Peter Zijlstra
- Renamed mmap_lock_speculation_{begin|end} into mmap_lock_speculate_{try_begin|retry}, per Peter Zijlstra
Note: the return value of mmap_lock_speculate_retry() is opposive to what it was in mmap_lock_speculation_end(). true now means speculation failed.
[1] https://lore.kernel.org/all/[email protected]/
include/linux/mmap_lock.h | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-)
include/linux/mmap_lock.h +31 -2
@@ -71,6 +71,7 @@ static inline void mmap_assert_write_locked(const struct mm_struct *mm) }#ifdef CONFIG_PER_VMA_LOCK +static inline void mm_lock_seqcount_init (struct mm_struct *mm) {seqcount_init (&mm->mm_lock_seq);
@@ -86,11 +87,39 @@ static inline void mm_lock_seqcount_end(struct mm_struct *mm)do_raw_write_seqcount_end (&mm->mm_lock_seq); } -#else +static inline bool mmap_lock_speculate_try_begin (struct mm_struct *mm,unsigned int *seq) +{ +/* +* Since mmap_lock is a sleeping lock, and waiting for it to become +* unlocked is more or less equivalent with taking it ourselves, don't +* bother with the speculative path if mmap_lock is already write-locked +* and take the slow path, which takes the lock. +*/ +return raw_seqcount_try_begin (&mm->mm_lock_seq, *seq); +} + +static inline bool mmap_lock_speculate_retry (struct mm_struct *mm,unsigned int seq) +{ +return do_read_seqcount_retry (&mm->mm_lock_seq, seq); +} + +#else /* CONFIG_PER_VMA_LOCK */ +static inline void mm_lock_seqcount_init (struct mm_struct *mm) {}static inline void mm_lock_seqcount_begin (struct mm_struct *mm) {}static inline void mm_lock_seqcount_end (struct mm_struct *mm) {} -#endif + +static inline bool mmap_lock_speculate_try_begin (struct mm_struct *mm,unsigned int *seq) +{ +return false; +} + +static inline bool mmap_lock_speculate_retry (struct mm_struct *mm,unsigned int seq) +{ +return true; +} + +#endif /* CONFIG_PER_VMA_LOCK */ static inline void mmap_init_lock (struct mm_struct *mm) {
Signature · 1 line
2.47.0.338.g60cca15819-goog
Peter Zijlstra
maintainerRe: [PATCH v2 3/3] mm: introduce mmap_lock_speculate_{try_begin|retry}
Quoted 6 lines · Suren Baghdasaryan
Add helper functions to speculatively perform operations without read-locking
mmap_lock, expecting thatmmap_lockwill not be write-locked and mm is not modified from under us.Suggested-by Peter Zijlstra <[email protected]>Signed-off-by Suren Baghdasaryan <[email protected]>
Thanks for these, you're okay with me taking these through tip/perf/core for the next cycle along with Andrii's uprobe patch?
Suren Baghdasaryan
maintainerRe: [PATCH v2 3/3] mm: introduce mmap_lock_speculate_{try_begin|retry}
Quoted 11 lines · Peter Zijlstra
Quoted 6 lines · Suren Baghdasaryan
Add helper functions to speculatively perform operations without read-locking
mmap_lock, expecting thatmmap_lockwill not be write-locked and mm is not modified from under us.Suggested-by Peter Zijlstra <[email protected]>Signed-off-by Suren Baghdasaryan <[email protected]>Thanks for these, you're okay with me taking these through tip/perf/core for the next cycle along with Andrii's uprobe patch?
Yes, I'm fine with that plan. Thank you!
Re: [PATCH v2 3/3] mm: introduce mmap_lock_speculate_{try_begin|retry}
In reply to Suren BaghdasaryanQuoted 6 lines · Suren Baghdasaryan
Add helper functions to speculatively perform operations without read-locking
mmap_lock, expecting thatmmap_lockwill not be write-locked and mm is not modified from under us.Suggested-by Peter Zijlstra <[email protected]>Signed-off-by Suren Baghdasaryan <[email protected]>
Quoted 72 lines
---
Changes since v1 [1]
- Changed to use newraw_seqcount_try_begin()API, per Peter Zijlstra
- Renamed mmap_lock_speculation_{begin|end} into mmap_lock_speculate_{try_begin|retry}, per Peter ZijlstraNote: the return value of
mmap_lock_speculate_retry()is opposive to what it was inmmap_lock_speculation_end(). true now means speculation failed.[1] https://lore.kernel.org/all/[email protected]/
include/linux/mmap_lock.h | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-)include/linux/mmap_lock.h +31
-2@@ -71,6 +71,7 @@ static inline void mmap_assert_write_locked(const struct mm_struct *mm) }#ifdef CONFIG_PER_VMA_LOCK +static inline void mm_lock_seqcount_init (struct mm_struct *mm) {seqcount_init (&mm->mm_lock_seq);@@ -86,11 +87,39 @@ static inline void mm_lock_seqcount_end(struct mm_struct *mm)do_raw_write_seqcount_end (&mm->mm_lock_seq); } -#else +static inline bool mmap_lock_speculate_try_begin (struct mm_struct *mm,unsigned int *seq) +{ +/* +* Since mmap_lock is a sleeping lock, and waiting for it to become +* unlocked is more or less equivalent with taking it ourselves, don't +* bother with the speculative path if mmap_lock is already write-locked +* and take the slow path, which takes the lock. +*/ +return raw_seqcount_try_begin (&mm->mm_lock_seq, *seq); +} + +static inline bool mmap_lock_speculate_retry (struct mm_struct *mm,unsigned int seq) +{ +return do_read_seqcount_retry (&mm->mm_lock_seq, seq); +} + +#else /* CONFIG_PER_VMA_LOCK */ +static inline void mm_lock_seqcount_init (struct mm_struct *mm) {}static inline void mm_lock_seqcount_begin (struct mm_struct *mm) {}static inline void mm_lock_seqcount_end (struct mm_struct *mm) {} -#endif + +static inline bool mmap_lock_speculate_try_begin (struct mm_struct *mm,unsigned int *seq) +{ +return false; +} + +static inline bool mmap_lock_speculate_retry (struct mm_struct *mm,unsigned int seq) +{ +return true; +} + +#endif /* CONFIG_PER_VMA_LOCK */ static inline void mmap_init_lock (struct mm_struct *mm) {Signature · 1 line
2.47.0.338.g60cca15819-goog
Quoted 32 lines · Suren Baghdasaryan
Add
raw_seqcount_try_begin()to opens a read critical section of the givenseqcount_tif the counter is even. This enables eliding the critical section entirely if the counter is odd, instead of doing the speculation knowing it will fail.Suggested-by Peter Zijlstra <[email protected]>Signed-off-by Suren Baghdasaryan <[email protected]>---
Applies over Linus' ToTinclude/linux/seqlock.h | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+)include/linux/seqlock.h +11
-0@@ -318,6 +318,28 @@ SEQCOUNT_LOCKNAME(mutex, struct mutex, true, mutex) __seq; \ }) +/** +* raw_seqcount_try_begin() - begin a seqcount_t read critical section +* w/o lockdep and w/o counter stabilization +* @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants +* +* Similar to raw_seqcount_begin(), except it enables eliding the critical +* section entirely if odd, instead of doing the speculation knowing it will +* fail. +* +* Useful when counter stabilization is more or less equivalent to taking +* the lock and there is a slowpath that does that.
include/linux/seqlock.h SEQCOUNT_LOCKNAME(mutex, struct mutex, true, mutex) · line 341
Quoted 10 lines
include/linux/seqlock.h +10
-0+* +* If true, start will be set to the (even) sequence count read. +* +* Return: true when a read critical section is started. +*/ +#define raw_seqcount_try_begin (s, start) \ +({ \ + start = raw_read_seqcount(s); \ + !(start & 1); \ +})
In gup_fast(), we simply do
seq = raw_read_seqcount(¤t->mm->write_protect_seq); if (seq & 1)
return 0;
Should we be using that there as well?
if (!raw_seqcount_try_begin(¤t->mm->write_protect_seqs, seq))
return 0;
Signature · 3 lines
Cheers, David / dhildenb
Peter Zijlstra
maintainerQuoted 10 lines · David Hildenbrand
In
gup_fast(), we simply doseq =
raw_read_seqcount(¤t->mm->write_protect_seq); if (seq & 1)return 0;Should we be using that there as well?
if (!raw_seqcount_try_begin(¤t->mm->write_protect_seqs, seq))
return 0;
Might as well. A quick grep doesn't find me another instance of this pattern, but does find me something 'funny' in net/netfilter/x_tables.c. Let's pretend I didn't see that for now ... *sigh*
Want me to stick a patch like this on, or do you want to do that later, when the dust has settled?
David Hildenbrand
Reviewed-byQuoted 8 lines · Peter Zijlstra
Quoted 6 lines · David Hildenbrand
In
gup_fast(), we simply doseq =
raw_read_seqcount(¤t->mm->write_protect_seq); if (seq & 1)return 0;
Quoted 8 lines
Quoted 4 lines
Should we be using that there as well?
if (!raw_seqcount_try_begin(¤t->mm->write_protect_seqs, seq))
return 0;Might as well. A quick grep doesn't find me another instance of this pattern, but does find me something 'funny' in
net/netfilter/x_tables.c. Let's pretend I didn't see that for now ... *sigh*
:)
I'm also not 100% sure about barriers in gup_fast() around the raw_seqcount .... and I'm pretending I didn't see any of that as well ...
Quoted 3 lines
Want me to stick a patch like this on, or do you want to do that later, when the dust has settled?
Feel free to add a patch to just change that as well, it's an easy change.
For this patch
Signature · 3 lines
Cheers, David / dhildenb
Quoted 7 lines · Suren Baghdasaryan
Add
raw_seqcount_try_begin()to opens a read critical section of the givenseqcount_tif the counter is even. This enables eliding the critical section entirely if the counter is odd, instead of doing the speculation knowing it will fail.Suggested-by Peter Zijlstra <[email protected]>Signed-off-by Suren Baghdasaryan <[email protected]>
Quoted 44 lines
---
Applies over Linus' ToTinclude/linux/seqlock.h | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+)include/linux/seqlock.h +22
-0@@ -318,6 +318,28 @@ SEQCOUNT_LOCKNAME(mutex, struct mutex, true, mutex) __seq; \ }) +/** +* raw_seqcount_try_begin() - begin a seqcount_t read critical section +* w/o lockdep and w/o counter stabilization +* @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants +* +* Similar to raw_seqcount_begin(), except it enables eliding the critical +* section entirely if odd, instead of doing the speculation knowing it will +* fail. +* +* Useful when counter stabilization is more or less equivalent to taking +* the lock and there is a slowpath that does that. +* +* If true, start will be set to the (even) sequence count read. +* +* Return: true when a read critical section is started. +*/ +#define raw_seqcount_try_begin (s, start) \ +({ \ + start = raw_read_seqcount(s); \ + !(start & 1); \ +}) +/** * raw_seqcount_begin() - begin a seqcount_t read critical section w/o * lockdep and w/o counter stabilization base-commit:
43fb83c17ba2d63dfb798f0be7453ed55ca3f9c2Signature · 1 line
2.47.0.338.g60cca15819-goog