use vma locks for proc/pid/{smaps|numa_maps} reads
Subsystems (5)
v2 cover use vma locks for proc/pid/{smaps|numa_maps} reads
Use per-vma locks when reading /proc/pid/smaps and /proc/pid/numa_maps similar to /proc/pid/maps to reduce contention on central mmap_lock. One major difference between maps and smaps/numa_maps reading is that the latter executes page table walk which can't be done under RCU due to a possibility of sleeping. Therefore we drop RCU read lock before this walk while keeping the VMA locked. After the walk we retake RCU read lock, reset VMA iterator and proceed with the next VMA.
The last two patches extend /proc/pid/maps test to cover /proc/pid/smaps reading during concurrent address space modification.
Changes since v1[1]
- moved drop_rcu earlied in smap_gather_stats to avoid sleeping under RCU lock in shmem_swap_usage(), per Sashiko
- skip page walks for gate VMA in show_numa_map(), per Sashiko
- introduced parse_vma_line() and copy_line() helper functions to ensure input string passed to sscanf() is always NUL-terminated, per Sashiko
- used FIXTURE_VARIANT to run both maps and smaps tests in a single test run, per Liam R. Howlett
Applies over mm-unstable.
[1] https://lore.kernel.org/all/[email protected]/
Suren Baghdasaryan (3):
fs/proc/task_mmu: read proc/pid/{smaps|numa_maps} under per-vma lock selftests/proc: ensure the test is performed at the right page
boundary
selftests/proc: add /proc/pid/smaps tearing tests
fs/proc/task_mmu.c | 195 +++++++++--- tools/testing/selftests/proc/proc-maps-race.c | 293 ++++++++++++++---- 2 files changed, 387 insertions(+), 101 deletions(-)
base-commit: 761e9fad336afb6fe2cd488c7bd522e2783064fc
Signature · 1 line
2.54.0.545.g6539524ca2-goog
PATCH v2 1/3 fs/proc/task_mmu: read proc/pid/{smaps|numa_maps} under per-vma lock
proc/pid/{smaps|numa_maps} can be read using the combination of RCU and VMA read locks, similar to proc/pid/maps. RCU is required to safely traverse the VMA tree and VMA lock stabilizes the VMA being processed and the pagetable walk.
--- fs/proc/task_mmu.c | 195 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 156 insertions(+), 39 deletions(-)
fs/proc/task_mmu.c +156 -39
@@ -132,6 +132,22 @@ static void release_task_mempolicy(struct proc_maps_private *priv)#ifdef CONFIG_PER_VMA_LOCK +static inline int lock_ctx_mm (struct proc_maps_locking_ctx *lock_ctx) +{ +int ret =mmap_read_lock_killable (lock_ctx->mm); + +if (!ret) + lock_ctx->mmap_locked = true; + +return ret; +} + +static inline void unlock_ctx_mm (struct proc_maps_locking_ctx *lock_ctx) +{ +mmap_read_unlock (lock_ctx->mm); + lock_ctx->mmap_locked = false; +} +static void reset_lock_ctx (struct proc_maps_locking_ctx *lock_ctx) { lock_ctx->locked_vma =NULL ;
@@ -146,25 +162,11 @@ static void unlock_ctx_vma(struct proc_maps_locking_ctx *lock_ctx) } } -static const struct seq_operations proc_pid_maps_op; -static inline bool lock_vma_range (struct seq_file *m,struct proc_maps_locking_ctx *lock_ctx) { -/* -* smaps and numa_maps perform page table walk, therefore require -* mmap_lock but maps can be read with locking just the vma and -* walking the vma tree under rcu read protection. -*/ -if (m->op != &proc_pid_maps_op) { -if (mmap_read_lock_killable (lock_ctx->mm)) -return false; - - lock_ctx->mmap_locked = true; - }else { -rcu_read_lock (); -reset_lock_ctx (lock_ctx); - } +rcu_read_lock (); +reset_lock_ctx (lock_ctx);return true; }
@@ -172,7 +174,7 @@ static inline bool lock_vma_range(struct seq_file *m,static inline void unlock_vma_range (struct proc_maps_locking_ctx *lock_ctx) {if (lock_ctx->mmap_locked) { -mmap_read_unlock (lock_ctx->mm); +unlock_ctx_mm (lock_ctx); }else {unlock_ctx_vma (lock_ctx);rcu_read_unlock ();
@@ -213,17 +215,45 @@ static inline bool fallback_to_mmap_lock(struct proc_maps_private *priv,return true; } +static inline void drop_rcu (struct proc_maps_private *priv) +{ +if (priv->lock_ctx.mmap_locked) +return ; + +rcu_read_unlock (); +} + +static inline void reacquire_rcu (struct proc_maps_private *priv) +{ +if (priv->lock_ctx.mmap_locked) +return ; + +rcu_read_lock (); +/* Reinitialize the iterator. */ +vma_iter_set (&priv->iter, priv->lock_ctx.locked_vma->vm_end); +} +#else /* CONFIG_PER_VMA_LOCK */ +static inline int lock_ctx_mm (struct proc_maps_locking_ctx *lock_ctx) +{ +return mmap_read_lock_killable (lock_ctx->mm); +} + +static inline void unlock_ctx_mm (struct proc_maps_locking_ctx *lock_ctx) +{ +mmap_read_unlock (lock_ctx->mm); +} +static inline bool lock_vma_range (struct seq_file *m,struct proc_maps_locking_ctx *lock_ctx) { -return mmap_read_lock_killable (lock_ctx->mm) ==0 ; +return lock_ctx_mm (lock_ctx) ==0 ; }static inline void unlock_vma_range (struct proc_maps_locking_ctx *lock_ctx) { -mmap_read_unlock (lock_ctx->mm); +unlock_ctx_mm (lock_ctx); } static struct vm_area_struct *get_next_vma(struct proc_maps_private *priv,
@@ -238,6 +268,9 @@ static inline bool fallback_to_mmap_lock(struct proc_maps_private *priv, return false; } +static inline void drop_rcu (struct proc_maps_private *priv) {} +static inline void reacquire_rcu (struct proc_maps_private *priv) {} +#endif /* CONFIG_PER_VMA_LOCK */ static struct vm_area_struct *proc_get_vma (struct seq_file *m,loff_t *ppos)
@@ -538,12 +571,10 @@ static int query_vma_setup(struct proc_maps_locking_ctx *lock_ctx)static void query_vma_teardown (struct proc_maps_locking_ctx *lock_ctx) { -if (lock_ctx->mmap_locked) { -mmap_read_unlock (lock_ctx->mm); - lock_ctx->mmap_locked = false; - }else { +if (lock_ctx->mmap_locked) +unlock_ctx_mm (lock_ctx); +else unlock_ctx_vma (lock_ctx); - } } static struct vm_area_struct *query_vma_find_by_addr(struct proc_maps_locking_ctx *lock_ctx,
@@ -1280,21 +1311,75 @@ static const struct mm_walk_ops smaps_shmem_walk_ops = { .walk_lock = PGWALK_RDLOCK, }; +#ifdef CONFIG_PER_VMA_LOCK + +static const struct mm_walk_ops smaps_walk_vma_lock_ops = { + .pmd_entry = smaps_pte_range, + .hugetlb_entry = smaps_hugetlb_range, + .walk_lock =PGWALK_VMA_RDLOCK_VERIFY , +}; + +static const struct mm_walk_ops smaps_shmem_walk_vma_lock_ops = { + .pmd_entry = smaps_pte_range, + .hugetlb_entry = smaps_hugetlb_range, + .pte_hole = smaps_pte_hole, + .walk_lock =PGWALK_VMA_RDLOCK_VERIFY , +}; + +static inline const struct mm_walk_ops * +get_smaps_walk_ops (struct proc_maps_private *priv) +{ +if (priv->lock_ctx.mmap_locked) +return &smaps_walk_ops; +return &smaps_walk_vma_lock_ops; +} + +static inline const struct mm_walk_ops * +get_smaps_shmem_walk_ops (struct proc_maps_private *priv) +{ +if (priv->lock_ctx.mmap_locked) +return &smaps_shmem_walk_ops; +return &smaps_shmem_walk_vma_lock_ops; +} + +#else /* CONFIG_PER_VMA_LOCK */ + +static inline const struct mm_walk_ops * +get_smaps_walk_ops (struct proc_maps_private *priv) +{ +return &smaps_walk_ops; +} + +static inline const struct mm_walk_ops * +get_smaps_shmem_walk_ops (struct proc_maps_private *priv) +{ +return &smaps_shmem_walk_ops; +} + +#endif /* CONFIG_PER_VMA_LOCK */ +/* * Gather mem stats from @vma with the indicated beginning * address @start, and keep them in @mss. * * Use vm_start of @vma as the beginning address if @start is 0. */ -static void smap_gather_stats (struct vm_area_struct *vma, -struct mem_size_stats *mss,unsigned long start) +static void smap_gather_stats (struct proc_maps_private *priv, +struct vm_area_struct *vma, +struct mem_size_stats *mss,unsigned long start) { -const struct mm_walk_ops *ops = &smaps_walk_ops; +const struct mm_walk_ops *ops =get_smaps_walk_ops (priv);/* Invalid start */ if (start >= vma->vm_end)return ; +if (vma ==get_gate_vma (priv->lock_ctx.mm)) +return ; + +/* Might sleep. Drop RCU read lock but keep the VMA locked. */ +drop_rcu (priv); +if (vma->vm_file &&shmem_mapping (vma->vm_file->f_mapping)) {/* * For shared or readonly shmem mappings we know that all
@@ -1312,15 +1397,16 @@ static void smap_gather_stats(struct vm_area_struct *vma, !(vma->vm_flags & VM_WRITE))) { mss->swap += shmem_swapped; }else { - ops = &smaps_shmem_walk_ops; + ops =get_smaps_shmem_walk_ops (priv); } } -/* mmap_lock is held in m_start */ if (!start)walk_page_vma (vma, ops, mss);else walk_page_range (vma->vm_mm, start, vma->vm_end, ops, mss); + +reacquire_rcu (priv); }#define SEQ_PUT_DEC (str, val) \
@@ -1369,10 +1455,11 @@ static void __show_smap(struct seq_file *m, const struct mem_size_stats *mss,static int show_smap (struct seq_file *m,void *v) { +struct proc_maps_private *priv = m->private;struct vm_area_struct *vma = v;struct mem_size_stats mss = {}; -smap_gather_stats (vma, &mss,0 ); +smap_gather_stats (priv, vma, &mss,0 );show_map_vma (m, vma);
@@ -1413,7 +1500,7 @@ static int show_smaps_rollup(struct seq_file *m, void *v) goto out_put_task; } - ret =mmap_read_lock_killable (mm); + ret =lock_ctx_mm (&priv->lock_ctx);if (ret) goto out_put_mm;
@@ -1425,7 +1512,7 @@ static int show_smaps_rollup(struct seq_file *m, void *v) vma_start = vma->vm_start; do { - smap_gather_stats(vma, &mss, 0); + smap_gather_stats(priv, vma, &mss, 0); last_vma_end = vma->vm_end; /*
@@ -1434,8 +1521,8 @@ static int show_smaps_rollup(struct seq_file *m, void *v)*/ if (mmap_lock_is_contended (mm)) {vma_iter_invalidate (&vmi); -mmap_read_unlock (mm); - ret =mmap_read_lock_killable (mm); +unlock_ctx_mm (&priv->lock_ctx); + ret =lock_ctx_mm (&priv->lock_ctx);if (ret) {release_task_mempolicy (priv); goto out_put_mm;
@@ -1484,14 +1571,14 @@ static int show_smaps_rollup(struct seq_file *m, void *v)/* Case 1 and 2 above */ if (vma->vm_start >= last_vma_end) { -smap_gather_stats (vma, &mss,0 ); +smap_gather_stats (priv, vma, &mss,0 ); last_vma_end = vma->vm_end;continue ; }/* Case 4 above */ if (vma->vm_end > last_vma_end) { -smap_gather_stats (vma, &mss, last_vma_end); +smap_gather_stats (priv, vma, &mss, last_vma_end); last_vma_end = vma->vm_end; } }
@@ -1505,7 +1592,7 @@ static int show_smaps_rollup(struct seq_file *m, void *v)__show_smap (m, &mss, true);release_task_mempolicy (priv); -mmap_read_unlock (mm); +unlock_ctx_mm (&priv->lock_ctx); out_put_mm:mmput (mm);
@@ -3291,6 +3378,31 @@ static const struct mm_walk_ops show_numa_ops = { .walk_lock = PGWALK_RDLOCK, }; +#ifdef CONFIG_PER_VMA_LOCK +static const struct mm_walk_ops show_numa_vma_lock_ops = { + .hugetlb_entry = gather_hugetlb_stats, + .pmd_entry = gather_pte_stats, + .walk_lock =PGWALK_VMA_RDLOCK_VERIFY , +}; + +static inline const struct mm_walk_ops * +get_show_numa_ops (struct proc_maps_private *priv) +{ +if (priv->lock_ctx.mmap_locked) +return &show_numa_ops; +return &show_numa_vma_lock_ops; +} + +#else /* CONFIG_PER_VMA_LOCK */ + +static inline const struct mm_walk_ops * +get_show_numa_ops (struct proc_maps_private *priv) +{ +return &show_numa_ops; +} + +#endif /* CONFIG_PER_VMA_LOCK */ +/* * Display pages allocated per node and memory policy via /proc. */
@@ -3335,8 +3447,13 @@ static int show_numa_map(struct seq_file *m, void *v)if (is_vm_hugetlb_page (vma))seq_puts (m," huge" ); -/* mmap_lock is held by m_start */ -walk_page_vma (vma, &show_numa_ops, md); +/* Skip walking pages if gate VMA */ +if (vma !=get_gate_vma (proc_priv->lock_ctx.mm)) { +/* Might sleep. Drop RCU read lock but keep the VMA locked. */ +drop_rcu (proc_priv); +walk_page_vma (vma,get_show_numa_ops (proc_priv), md); +reacquire_rcu (proc_priv); + }if (!md->pages) goto out;
Signature · 1 line
2.54.0.545.g6539524ca2-goog
Re: [PATCH v2 1/3] fs/proc/task_mmu: read proc/pid/{smaps|numa_maps} under per-vma lock
Quoted 7 lines · Suren Baghdasaryan
proc/pid/{smaps|numa_maps} can be read using the combination of RCU and VMA read locks, similar to proc/pid/maps. RCU is required to safely traverse the VMA tree and VMA lock stabilizes the VMA being processed and the pagetable walk.
Signed-off-by Suren Baghdasaryan <[email protected]>Reviewed-by Liam R. Howlett <[email protected]>
The logic LGTM, some nitty things below but in general:
fs/proc/task_mmu.c static void release_task_mempolicy(struct proc_maps_private *priv) · line 135
Quoted 13 lines
--- fs/proc/task_mmu.c | 195 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 156 insertions(+), 39 deletions(-)fs/proc/task_mmu.c +1
-0@@ -132,6 +132,22 @@ static void release_task_mempolicy(struct proc_maps_private *priv)#ifdef CONFIG_PER_VMA_LOCK +static inline int lock_ctx_mm (struct proc_maps_locking_ctx *lock_ctx)
NIT: But these inlines are unnecessary in a C file. At the kernel optimisation level the compiler already decides whether to inline or not.
However in the kernel we redefine this so we lose function tracing and also we don't get warnings about unused functions [0], so this is actually a bit harmful
- you can end up with dead code and not know abou tit.
Same goes for all the other static inline's in task_mmu.c.
[0]: https://elixir.bootlin.com/linux/v7.0.10/source/include/linux/compiler_types.h#L235
Quoted 65 lines
fs/proc/task_mmu.c +24
-17+{ +int ret =mmap_read_lock_killable (lock_ctx->mm); + +if (!ret) + lock_ctx->mmap_locked = true; + +return ret; +} + +static inline void unlock_ctx_mm (struct proc_maps_locking_ctx *lock_ctx) +{ +mmap_read_unlock (lock_ctx->mm); + lock_ctx->mmap_locked = false; +} +static void reset_lock_ctx (struct proc_maps_locking_ctx *lock_ctx) { lock_ctx->locked_vma =NULL ;@@ -146,25 +162,11 @@ static void unlock_ctx_vma(struct proc_maps_locking_ctx *lock_ctx) } } -static const struct seq_operations proc_pid_maps_op; -static inline bool lock_vma_range (struct seq_file *m,struct proc_maps_locking_ctx *lock_ctx) { -/* -* smaps and numa_maps perform page table walk, therefore require -* mmap_lock but maps can be read with locking just the vma and -* walking the vma tree under rcu read protection. -*/ -if (m->op != &proc_pid_maps_op) { -if (mmap_read_lock_killable (lock_ctx->mm)) -return false; - - lock_ctx->mmap_locked = true; - }else { -rcu_read_lock (); -reset_lock_ctx (lock_ctx); - } +rcu_read_lock (); +reset_lock_ctx (lock_ctx);return true; }@@ -172,7 +174,7 @@ static inline bool lock_vma_range(struct seq_file *m,static inline void unlock_vma_range (struct proc_maps_locking_ctx *lock_ctx) {if (lock_ctx->mmap_locked) { -mmap_read_unlock (lock_ctx->mm); +unlock_ctx_mm (lock_ctx); }else {unlock_ctx_vma (lock_ctx);rcu_read_unlock ();@@ -213,17 +215,45 @@ static inline bool fallback_to_mmap_lock(struct proc_maps_private *priv,return true; } +static inline void drop_rcu (struct proc_maps_private *priv) +{ +if (priv->lock_ctx.mmap_locked) +return ; + +rcu_read_unlock ();
fs/proc/task_mmu.c static inline bool fallback_to_mmap_lock(struct proc_maps_private *priv, · line 233
Quoted 10 lines
fs/proc/task_mmu.c +10
-0+} + +static inline void reacquire_rcu (struct proc_maps_private *priv) +{ +if (priv->lock_ctx.mmap_locked) +return ; + +rcu_read_lock (); +/* Reinitialize the iterator. */ +vma_iter_set (&priv->iter, priv->lock_ctx.locked_vma->vm_end);
OK good that we reset this :)
fs/proc/task_mmu.c static inline bool fallback_to_mmap_lock(struct proc_maps_private *priv, · line 237
Quoted 4 lines
fs/proc/task_mmu.c +2
-0+} +#else /* CONFIG_PER_VMA_LOCK */
We need to pull out the patches from Dave's series and remove this option and default VMA locks on by default :)
Quoted 135 lines
fs/proc/task_mmu.c +76
-11+static inline int lock_ctx_mm (struct proc_maps_locking_ctx *lock_ctx) +{ +return mmap_read_lock_killable (lock_ctx->mm); +} + +static inline void unlock_ctx_mm (struct proc_maps_locking_ctx *lock_ctx) +{ +mmap_read_unlock (lock_ctx->mm); +} +static inline bool lock_vma_range (struct seq_file *m,struct proc_maps_locking_ctx *lock_ctx) { -return mmap_read_lock_killable (lock_ctx->mm) ==0 ; +return lock_ctx_mm (lock_ctx) ==0 ; }static inline void unlock_vma_range (struct proc_maps_locking_ctx *lock_ctx) { -mmap_read_unlock (lock_ctx->mm); +unlock_ctx_mm (lock_ctx); } static struct vm_area_struct *get_next_vma(struct proc_maps_private *priv,@@ -238,6 +268,9 @@ static inline bool fallback_to_mmap_lock(struct proc_maps_private *priv, return false; } +static inline void drop_rcu (struct proc_maps_private *priv) {} +static inline void reacquire_rcu (struct proc_maps_private *priv) {} +#endif /* CONFIG_PER_VMA_LOCK */ static struct vm_area_struct *proc_get_vma (struct seq_file *m,loff_t *ppos)@@ -538,12 +571,10 @@ static int query_vma_setup(struct proc_maps_locking_ctx *lock_ctx)static void query_vma_teardown (struct proc_maps_locking_ctx *lock_ctx) { -if (lock_ctx->mmap_locked) { -mmap_read_unlock (lock_ctx->mm); - lock_ctx->mmap_locked = false; - }else { +if (lock_ctx->mmap_locked) +unlock_ctx_mm (lock_ctx); +else unlock_ctx_vma (lock_ctx); - } } static struct vm_area_struct *query_vma_find_by_addr(struct proc_maps_locking_ctx *lock_ctx,@@ -1280,21 +1311,75 @@ static const struct mm_walk_ops smaps_shmem_walk_ops = { .walk_lock = PGWALK_RDLOCK, }; +#ifdef CONFIG_PER_VMA_LOCK + +static const struct mm_walk_ops smaps_walk_vma_lock_ops = { + .pmd_entry = smaps_pte_range, + .hugetlb_entry = smaps_hugetlb_range, + .walk_lock =PGWALK_VMA_RDLOCK_VERIFY , +}; + +static const struct mm_walk_ops smaps_shmem_walk_vma_lock_ops = { + .pmd_entry = smaps_pte_range, + .hugetlb_entry = smaps_hugetlb_range, + .pte_hole = smaps_pte_hole, + .walk_lock =PGWALK_VMA_RDLOCK_VERIFY , +}; + +static inline const struct mm_walk_ops * +get_smaps_walk_ops (struct proc_maps_private *priv) +{ +if (priv->lock_ctx.mmap_locked) +return &smaps_walk_ops; +return &smaps_walk_vma_lock_ops; +} + +static inline const struct mm_walk_ops * +get_smaps_shmem_walk_ops (struct proc_maps_private *priv) +{ +if (priv->lock_ctx.mmap_locked) +return &smaps_shmem_walk_ops; +return &smaps_shmem_walk_vma_lock_ops; +} + +#else /* CONFIG_PER_VMA_LOCK */ + +static inline const struct mm_walk_ops * +get_smaps_walk_ops (struct proc_maps_private *priv) +{ +return &smaps_walk_ops; +} + +static inline const struct mm_walk_ops * +get_smaps_shmem_walk_ops (struct proc_maps_private *priv) +{ +return &smaps_shmem_walk_ops; +} + +#endif /* CONFIG_PER_VMA_LOCK */ +/* * Gather mem stats from @vma with the indicated beginning * address @start, and keep them in @mss. * * Use vm_start of @vma as the beginning address if @start is 0. */ -static void smap_gather_stats (struct vm_area_struct *vma, -struct mem_size_stats *mss,unsigned long start) +static void smap_gather_stats (struct proc_maps_private *priv, +struct vm_area_struct *vma, +struct mem_size_stats *mss,unsigned long start) { -const struct mm_walk_ops *ops = &smaps_walk_ops; +const struct mm_walk_ops *ops =get_smaps_walk_ops (priv);/* Invalid start */ if (start >= vma->vm_end)return ; +if (vma ==get_gate_vma (priv->lock_ctx.mm)) +return ; + +/* Might sleep. Drop RCU read lock but keep the VMA locked. */ +drop_rcu (priv); +if (vma->vm_file &&shmem_mapping (vma->vm_file->f_mapping)) {/* * For shared or readonly shmem mappings we know that all @@ -1312,15 +1397,16 @@ static void smap_gather_stats(struct vm_area_struct *vma, !(vma->vm_flags & VM_WRITE))) { mss->swap += shmem_swapped; } else { - ops = &smaps_shmem_walk_ops; + ops =get_smaps_shmem_walk_ops (priv);
fs/proc/task_mmu.c static void smap_gather_stats(struct vm_area_struct *vma, · line 1409
Quoted 10 lines
fs/proc/task_mmu.c +2
-1} } -/* mmap_lock is held in m_start */ if (!start)walk_page_vma (vma, ops, mss);else walk_page_range (vma->vm_mm, start, vma->vm_end, ops, mss); + +reacquire_rcu (priv);
OK I was going to say that this RCU lock dance is a bit iffy, but I see that we're sharing code such that this is necessary.
Quoted 122 lines
fs/proc/task_mmu.c +41
-11}#define SEQ_PUT_DEC (str, val) \@@ -1369,10 +1455,11 @@ static void __show_smap(struct seq_file *m, const struct mem_size_stats *mss,static int show_smap (struct seq_file *m,void *v) { +struct proc_maps_private *priv = m->private;struct vm_area_struct *vma = v;struct mem_size_stats mss = {}; -smap_gather_stats (vma, &mss,0 ); +smap_gather_stats (priv, vma, &mss,0 );show_map_vma (m, vma);@@ -1413,7 +1500,7 @@ static int show_smaps_rollup(struct seq_file *m, void *v) goto out_put_task; } - ret =mmap_read_lock_killable (mm); + ret =lock_ctx_mm (&priv->lock_ctx);if (ret) goto out_put_mm;@@ -1425,7 +1512,7 @@ static int show_smaps_rollup(struct seq_file *m, void *v) vma_start = vma->vm_start; do { - smap_gather_stats(vma, &mss, 0); + smap_gather_stats(priv, vma, &mss, 0); last_vma_end = vma->vm_end; /*@@ -1434,8 +1521,8 @@ static int show_smaps_rollup(struct seq_file *m, void *v)*/ if (mmap_lock_is_contended (mm)) {vma_iter_invalidate (&vmi); -mmap_read_unlock (mm); - ret =mmap_read_lock_killable (mm); +unlock_ctx_mm (&priv->lock_ctx); + ret =lock_ctx_mm (&priv->lock_ctx);if (ret) {release_task_mempolicy (priv); goto out_put_mm;@@ -1484,14 +1571,14 @@ static int show_smaps_rollup(struct seq_file *m, void *v)/* Case 1 and 2 above */ if (vma->vm_start >= last_vma_end) { -smap_gather_stats (vma, &mss,0 ); +smap_gather_stats (priv, vma, &mss,0 ); last_vma_end = vma->vm_end;continue ; }/* Case 4 above */ if (vma->vm_end > last_vma_end) { -smap_gather_stats (vma, &mss, last_vma_end); +smap_gather_stats (priv, vma, &mss, last_vma_end); last_vma_end = vma->vm_end; } }@@ -1505,7 +1592,7 @@ static int show_smaps_rollup(struct seq_file *m, void *v)__show_smap (m, &mss, true);release_task_mempolicy (priv); -mmap_read_unlock (mm); +unlock_ctx_mm (&priv->lock_ctx); out_put_mm:mmput (mm);@@ -3291,6 +3378,31 @@ static const struct mm_walk_ops show_numa_ops = { .walk_lock = PGWALK_RDLOCK, }; +#ifdef CONFIG_PER_VMA_LOCK +static const struct mm_walk_ops show_numa_vma_lock_ops = { + .hugetlb_entry = gather_hugetlb_stats, + .pmd_entry = gather_pte_stats, + .walk_lock =PGWALK_VMA_RDLOCK_VERIFY , +}; + +static inline const struct mm_walk_ops * +get_show_numa_ops (struct proc_maps_private *priv) +{ +if (priv->lock_ctx.mmap_locked) +return &show_numa_ops; +return &show_numa_vma_lock_ops; +} + +#else /* CONFIG_PER_VMA_LOCK */ + +static inline const struct mm_walk_ops * +get_show_numa_ops (struct proc_maps_private *priv) +{ +return &show_numa_ops; +} + +#endif /* CONFIG_PER_VMA_LOCK */ +/* * Display pages allocated per node and memory policy via /proc. */ @@ -3335,8 +3447,13 @@ static int show_numa_map(struct seq_file *m, void *v)if (is_vm_hugetlb_page (vma))seq_puts (m," huge" ); -/* mmap_lock is held by m_start */ -walk_page_vma (vma, &show_numa_ops, md); +/* Skip walking pages if gate VMA */ +if (vma !=get_gate_vma (proc_priv->lock_ctx.mm)) { +/* Might sleep. Drop RCU read lock but keep the VMA locked. */ +drop_rcu (proc_priv); +walk_page_vma (vma,get_show_numa_ops (proc_priv), md); +reacquire_rcu (proc_priv); + }if (!md->pages) goto out; --2.54.0.545.g6539524ca2-goog
Re: [PATCH v2 1/3] fs/proc/task_mmu: read proc/pid/{smaps|numa_maps} under per-vma lock
Quoted 7 lines · Lorenzo Stoakes
Quoted 5 lines · Suren Baghdasaryan
proc/pid/{smaps|numa_maps} can be read using the combination of RCU and VMA read locks, similar to proc/pid/maps. RCU is required to safely traverse the VMA tree and VMA lock stabilizes the VMA being processed and the pagetable walk.
Quoted 6 lines
Quoted 2 lines
Signed-off-by Suren Baghdasaryan <[email protected]>Reviewed-by Liam R. Howlett <[email protected]>The logic LGTM, some nitty things below but in general:
Reviewed-by Lorenzo Stoakes <[email protected]>
Thanks!
Quoted 18 lines
Quoted 10 lines
--- fs/proc/task_mmu.c | 195 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 156 insertions(+), 39 deletions(-)fs/proc/task_mmu.c +0
-0fs/proc/task_mmu.c static void release_task_mempolicy(struct proc_maps_private *priv) · line 135
Quoted 4 lines
fs/proc/task_mmu.c +1
-0@@ -132,6 +132,22 @@ static void release_task_mempolicy(struct proc_maps_private *priv)#ifdef CONFIG_PER_VMA_LOCK +static inline int lock_ctx_mm (struct proc_maps_locking_ctx *lock_ctx)NIT: But these inlines are unnecessary in a C file. At the kernel optimisation level the compiler already decides whether to inline or not.
Quoted 5 lines
However in the kernel we redefine this so we lose function tracing and also we don't get warnings about unused functions [0], so this is actually a bit harmful
- you can end up with dead code and not know abou tit.Same goes for all the other static inline's in
task_mmu.c.
I see. There are a bunch of these small inline functions. Once the dust settles, I can clean them up. Dave's series and my folloup patch to change smaps_rollup will be changing this code. Maybe after all that is done?
Quoted 81 lines
[0]: https://elixir.bootlin.com/linux/v7.0.10/source/include/linux/compiler_types.h#L235
Quoted 65 lines
fs/proc/task_mmu.c +24
-17+{ +int ret =mmap_read_lock_killable (lock_ctx->mm); + +if (!ret) + lock_ctx->mmap_locked = true; + +return ret; +} + +static inline void unlock_ctx_mm (struct proc_maps_locking_ctx *lock_ctx) +{ +mmap_read_unlock (lock_ctx->mm); + lock_ctx->mmap_locked = false; +} +static void reset_lock_ctx (struct proc_maps_locking_ctx *lock_ctx) { lock_ctx->locked_vma =NULL ;@@ -146,25 +162,11 @@ static void unlock_ctx_vma(struct proc_maps_locking_ctx *lock_ctx) } } -static const struct seq_operations proc_pid_maps_op; -static inline bool lock_vma_range (struct seq_file *m,struct proc_maps_locking_ctx *lock_ctx) { -/* -* smaps and numa_maps perform page table walk, therefore require -* mmap_lock but maps can be read with locking just the vma and -* walking the vma tree under rcu read protection. -*/ -if (m->op != &proc_pid_maps_op) { -if (mmap_read_lock_killable (lock_ctx->mm)) -return false; - - lock_ctx->mmap_locked = true; - }else { -rcu_read_lock (); -reset_lock_ctx (lock_ctx); - } +rcu_read_lock (); +reset_lock_ctx (lock_ctx);return true; }@@ -172,7 +174,7 @@ static inline bool lock_vma_range(struct seq_file *m,static inline void unlock_vma_range (struct proc_maps_locking_ctx *lock_ctx) {if (lock_ctx->mmap_locked) { -mmap_read_unlock (lock_ctx->mm); +unlock_ctx_mm (lock_ctx); }else {unlock_ctx_vma (lock_ctx);rcu_read_unlock ();@@ -213,17 +215,45 @@ static inline bool fallback_to_mmap_lock(struct proc_maps_private *priv,return true; } +static inline void drop_rcu (struct proc_maps_private *priv) +{ +if (priv->lock_ctx.mmap_locked) +return ; + +rcu_read_unlock ();fs/proc/task_mmu.c static inline bool fallback_to_mmap_lock(struct proc_maps_private *priv, · line 233
Quoted 10 lines
fs/proc/task_mmu.c +10
-0+} + +static inline void reacquire_rcu (struct proc_maps_private *priv) +{ +if (priv->lock_ctx.mmap_locked) +return ; + +rcu_read_lock (); +/* Reinitialize the iterator. */ +vma_iter_set (&priv->iter, priv->lock_ctx.locked_vma->vm_end);OK good that we reset this :)
Quoted 7 lines
fs/proc/task_mmu.c static inline bool fallback_to_mmap_lock(struct proc_maps_private *priv, · line 237
Quoted 4 lines
fs/proc/task_mmu.c +2
-0+} +#else /* CONFIG_PER_VMA_LOCK */ We need to pull out the patches from Dave's series and remove this option and default VMA locks on by default :)
I didn't see any followup after Dave posted his v1. I agree it would make things much simpler but I don't want to delay these changes too much.
I'll ask Dave if he plans to continue his cleanup. If not, I'll pick it up.
Quoted 272 lines
Quoted 136 lines
fs/proc/task_mmu.c +76
-11+static inline int lock_ctx_mm (struct proc_maps_locking_ctx *lock_ctx) +{ +return mmap_read_lock_killable (lock_ctx->mm); +} + +static inline void unlock_ctx_mm (struct proc_maps_locking_ctx *lock_ctx) +{ +mmap_read_unlock (lock_ctx->mm); +} +static inline bool lock_vma_range (struct seq_file *m,struct proc_maps_locking_ctx *lock_ctx) { -return mmap_read_lock_killable (lock_ctx->mm) ==0 ; +return lock_ctx_mm (lock_ctx) ==0 ; }static inline void unlock_vma_range (struct proc_maps_locking_ctx *lock_ctx) { -mmap_read_unlock (lock_ctx->mm); +unlock_ctx_mm (lock_ctx); } static struct vm_area_struct *get_next_vma(struct proc_maps_private *priv,@@ -238,6 +268,9 @@ static inline bool fallback_to_mmap_lock(struct proc_maps_private *priv, return false; } +static inline void drop_rcu (struct proc_maps_private *priv) {} +static inline void reacquire_rcu (struct proc_maps_private *priv) {} +#endif /* CONFIG_PER_VMA_LOCK */ static struct vm_area_struct *proc_get_vma (struct seq_file *m,loff_t *ppos)@@ -538,12 +571,10 @@ static int query_vma_setup(struct proc_maps_locking_ctx *lock_ctx)static void query_vma_teardown (struct proc_maps_locking_ctx *lock_ctx) { -if (lock_ctx->mmap_locked) { -mmap_read_unlock (lock_ctx->mm); - lock_ctx->mmap_locked = false; - }else { +if (lock_ctx->mmap_locked) +unlock_ctx_mm (lock_ctx); +else unlock_ctx_vma (lock_ctx); - } } static struct vm_area_struct *query_vma_find_by_addr(struct proc_maps_locking_ctx *lock_ctx,@@ -1280,21 +1311,75 @@ static const struct mm_walk_ops smaps_shmem_walk_ops = { .walk_lock = PGWALK_RDLOCK, }; +#ifdef CONFIG_PER_VMA_LOCK + +static const struct mm_walk_ops smaps_walk_vma_lock_ops = { + .pmd_entry = smaps_pte_range, + .hugetlb_entry = smaps_hugetlb_range, + .walk_lock =PGWALK_VMA_RDLOCK_VERIFY , +}; + +static const struct mm_walk_ops smaps_shmem_walk_vma_lock_ops = { + .pmd_entry = smaps_pte_range, + .hugetlb_entry = smaps_hugetlb_range, + .pte_hole = smaps_pte_hole, + .walk_lock =PGWALK_VMA_RDLOCK_VERIFY , +}; + +static inline const struct mm_walk_ops * +get_smaps_walk_ops (struct proc_maps_private *priv) +{ +if (priv->lock_ctx.mmap_locked) +return &smaps_walk_ops; +return &smaps_walk_vma_lock_ops; +} + +static inline const struct mm_walk_ops * +get_smaps_shmem_walk_ops (struct proc_maps_private *priv) +{ +if (priv->lock_ctx.mmap_locked) +return &smaps_shmem_walk_ops; +return &smaps_shmem_walk_vma_lock_ops; +} + +#else /* CONFIG_PER_VMA_LOCK */ + +static inline const struct mm_walk_ops * +get_smaps_walk_ops (struct proc_maps_private *priv) +{ +return &smaps_walk_ops; +} + +static inline const struct mm_walk_ops * +get_smaps_shmem_walk_ops (struct proc_maps_private *priv) +{ +return &smaps_shmem_walk_ops; +} + +#endif /* CONFIG_PER_VMA_LOCK */ +/* * Gather mem stats from @vma with the indicated beginning * address @start, and keep them in @mss. * * Use vm_start of @vma as the beginning address if @start is 0. */ -static void smap_gather_stats (struct vm_area_struct *vma, -struct mem_size_stats *mss,unsigned long start) +static void smap_gather_stats (struct proc_maps_private *priv, +struct vm_area_struct *vma, +struct mem_size_stats *mss,unsigned long start) { -const struct mm_walk_ops *ops = &smaps_walk_ops; +const struct mm_walk_ops *ops =get_smaps_walk_ops (priv);/* Invalid start */ if (start >= vma->vm_end)return ; +if (vma ==get_gate_vma (priv->lock_ctx.mm)) +return ; + +/* Might sleep. Drop RCU read lock but keep the VMA locked. */ +drop_rcu (priv); +if (vma->vm_file &&shmem_mapping (vma->vm_file->f_mapping)) {/* * For shared or readonly shmem mappings we know that all @@ -1312,15 +1397,16 @@ static void smap_gather_stats(struct vm_area_struct *vma, !(vma->vm_flags & VM_WRITE))) { mss->swap += shmem_swapped; } else { - ops = &smaps_shmem_walk_ops; + ops =get_smaps_shmem_walk_ops (priv);fs/proc/task_mmu.c static void smap_gather_stats(struct vm_area_struct *vma, · line 1409
Quoted 10 lines
fs/proc/task_mmu.c +2
-1} } -/* mmap_lock is held in m_start */ if (!start)walk_page_vma (vma, ops, mss);else walk_page_range (vma->vm_mm, start, vma->vm_end, ops, mss); + +reacquire_rcu (priv);OK I was going to say that this RCU lock dance is a bit iffy, but I see that we're sharing code such that this is necessary.
Quoted 122 lines
fs/proc/task_mmu.c +41
-11}#define SEQ_PUT_DEC (str, val) \@@ -1369,10 +1455,11 @@ static void __show_smap(struct seq_file *m, const struct mem_size_stats *mss,static int show_smap (struct seq_file *m,void *v) { +struct proc_maps_private *priv = m->private;struct vm_area_struct *vma = v;struct mem_size_stats mss = {}; -smap_gather_stats (vma, &mss,0 ); +smap_gather_stats (priv, vma, &mss,0 );show_map_vma (m, vma);@@ -1413,7 +1500,7 @@ static int show_smaps_rollup(struct seq_file *m, void *v) goto out_put_task; } - ret =mmap_read_lock_killable (mm); + ret =lock_ctx_mm (&priv->lock_ctx);if (ret) goto out_put_mm;@@ -1425,7 +1512,7 @@ static int show_smaps_rollup(struct seq_file *m, void *v) vma_start = vma->vm_start; do { - smap_gather_stats(vma, &mss, 0); + smap_gather_stats(priv, vma, &mss, 0); last_vma_end = vma->vm_end; /*@@ -1434,8 +1521,8 @@ static int show_smaps_rollup(struct seq_file *m, void *v)*/ if (mmap_lock_is_contended (mm)) {vma_iter_invalidate (&vmi); -mmap_read_unlock (mm); - ret =mmap_read_lock_killable (mm); +unlock_ctx_mm (&priv->lock_ctx); + ret =lock_ctx_mm (&priv->lock_ctx);if (ret) {release_task_mempolicy (priv); goto out_put_mm;@@ -1484,14 +1571,14 @@ static int show_smaps_rollup(struct seq_file *m, void *v)/* Case 1 and 2 above */ if (vma->vm_start >= last_vma_end) { -smap_gather_stats (vma, &mss,0 ); +smap_gather_stats (priv, vma, &mss,0 ); last_vma_end = vma->vm_end;continue ; }/* Case 4 above */ if (vma->vm_end > last_vma_end) { -smap_gather_stats (vma, &mss, last_vma_end); +smap_gather_stats (priv, vma, &mss, last_vma_end); last_vma_end = vma->vm_end; } }@@ -1505,7 +1592,7 @@ static int show_smaps_rollup(struct seq_file *m, void *v)__show_smap (m, &mss, true);release_task_mempolicy (priv); -mmap_read_unlock (mm); +unlock_ctx_mm (&priv->lock_ctx); out_put_mm:mmput (mm);@@ -3291,6 +3378,31 @@ static const struct mm_walk_ops show_numa_ops = { .walk_lock = PGWALK_RDLOCK, }; +#ifdef CONFIG_PER_VMA_LOCK +static const struct mm_walk_ops show_numa_vma_lock_ops = { + .hugetlb_entry = gather_hugetlb_stats, + .pmd_entry = gather_pte_stats, + .walk_lock =PGWALK_VMA_RDLOCK_VERIFY , +}; + +static inline const struct mm_walk_ops * +get_show_numa_ops (struct proc_maps_private *priv) +{ +if (priv->lock_ctx.mmap_locked) +return &show_numa_ops; +return &show_numa_vma_lock_ops; +} + +#else /* CONFIG_PER_VMA_LOCK */ + +static inline const struct mm_walk_ops * +get_show_numa_ops (struct proc_maps_private *priv) +{ +return &show_numa_ops; +} + +#endif /* CONFIG_PER_VMA_LOCK */ +/* * Display pages allocated per node and memory policy via /proc. */ @@ -3335,8 +3447,13 @@ static int show_numa_map(struct seq_file *m, void *v)if (is_vm_hugetlb_page (vma))seq_puts (m," huge" ); -/* mmap_lock is held by m_start */ -walk_page_vma (vma, &show_numa_ops, md); +/* Skip walking pages if gate VMA */ +if (vma !=get_gate_vma (proc_priv->lock_ctx.mm)) { +/* Might sleep. Drop RCU read lock but keep the VMA locked. */ +drop_rcu (proc_priv); +walk_page_vma (vma,get_show_numa_ops (proc_priv), md); +reacquire_rcu (proc_priv); + }if (!md->pages) goto out; --2.54.0.545.g6539524ca2-goog
Lorenzo Stoakes
maintainerRe: [PATCH v2 1/3] fs/proc/task_mmu: read proc/pid/{smaps|numa_maps} under per-vma lock
Quoted 39 lines · Suren Baghdasaryan
Quoted 7 lines · Lorenzo Stoakes
Quoted 5 lines · Suren Baghdasaryan
proc/pid/{smaps|numa_maps} can be read using the combination of RCU and VMA read locks, similar to proc/pid/maps. RCU is required to safely traverse the VMA tree and VMA lock stabilizes the VMA being processed and the pagetable walk.
Quoted 6 lines
Quoted 2 lines
Signed-off-by Suren Baghdasaryan <[email protected]>Reviewed-by Liam R. Howlett <[email protected]>The logic LGTM, some nitty things below but in general:
Reviewed-by Lorenzo Stoakes <[email protected]>Thanks!
Quoted 22 lines
Quoted 10 lines
--- fs/proc/task_mmu.c | 195 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 156 insertions(+), 39 deletions(-)fs/proc/task_mmu.c +0
-0fs/proc/task_mmu.c static void release_task_mempolicy(struct proc_maps_private *priv) · line 135
Quoted 4 lines
fs/proc/task_mmu.c +1
-0@@ -132,6 +132,22 @@ static void release_task_mempolicy(struct proc_maps_private *priv)#ifdef CONFIG_PER_VMA_LOCK +static inline int lock_ctx_mm (struct proc_maps_locking_ctx *lock_ctx)NIT: But these inlines are unnecessary in a C file. At the kernel optimisation level the compiler already decides whether to inline or not.
However in the kernel we redefine this so we lose function tracing and also we don't get warnings about unused functions [0], so this is actually a bit harmful
- you can end up with dead code and not know abou tit.
Quoted 6 lines
Quoted 1 line
Same goes for all the other static inline's in
task_mmu.c.I see. There are a bunch of these small inline functions. Once the dust settles, I can clean them up. Dave's series and my folloup patch to change
smaps_rollupwill be changing this code. Maybe after all that is done?
Yeah that's fine this obviously isn't urgent! :) tiny nits really.
Quoted 90 lines
Quoted 82 lines
[0]: https://elixir.bootlin.com/linux/v7.0.10/source/include/linux/compiler_types.h#L235
Quoted 65 lines
fs/proc/task_mmu.c +24
-17+{ +int ret =mmap_read_lock_killable (lock_ctx->mm); + +if (!ret) + lock_ctx->mmap_locked = true; + +return ret; +} + +static inline void unlock_ctx_mm (struct proc_maps_locking_ctx *lock_ctx) +{ +mmap_read_unlock (lock_ctx->mm); + lock_ctx->mmap_locked = false; +} +static void reset_lock_ctx (struct proc_maps_locking_ctx *lock_ctx) { lock_ctx->locked_vma =NULL ;@@ -146,25 +162,11 @@ static void unlock_ctx_vma(struct proc_maps_locking_ctx *lock_ctx) } } -static const struct seq_operations proc_pid_maps_op; -static inline bool lock_vma_range (struct seq_file *m,struct proc_maps_locking_ctx *lock_ctx) { -/* -* smaps and numa_maps perform page table walk, therefore require -* mmap_lock but maps can be read with locking just the vma and -* walking the vma tree under rcu read protection. -*/ -if (m->op != &proc_pid_maps_op) { -if (mmap_read_lock_killable (lock_ctx->mm)) -return false; - - lock_ctx->mmap_locked = true; - }else { -rcu_read_lock (); -reset_lock_ctx (lock_ctx); - } +rcu_read_lock (); +reset_lock_ctx (lock_ctx);return true; }@@ -172,7 +174,7 @@ static inline bool lock_vma_range(struct seq_file *m,static inline void unlock_vma_range (struct proc_maps_locking_ctx *lock_ctx) {if (lock_ctx->mmap_locked) { -mmap_read_unlock (lock_ctx->mm); +unlock_ctx_mm (lock_ctx); }else {unlock_ctx_vma (lock_ctx);rcu_read_unlock ();@@ -213,17 +215,45 @@ static inline bool fallback_to_mmap_lock(struct proc_maps_private *priv,return true; } +static inline void drop_rcu (struct proc_maps_private *priv) +{ +if (priv->lock_ctx.mmap_locked) +return ; + +rcu_read_unlock ();fs/proc/task_mmu.c static inline bool fallback_to_mmap_lock(struct proc_maps_private *priv, · line 233
Quoted 10 lines
fs/proc/task_mmu.c +10
-0+} + +static inline void reacquire_rcu (struct proc_maps_private *priv) +{ +if (priv->lock_ctx.mmap_locked) +return ; + +rcu_read_lock (); +/* Reinitialize the iterator. */ +vma_iter_set (&priv->iter, priv->lock_ctx.locked_vma->vm_end);OK good that we reset this :)
Quoted 7 lines
fs/proc/task_mmu.c static inline bool fallback_to_mmap_lock(struct proc_maps_private *priv, · line 237
Quoted 4 lines
fs/proc/task_mmu.c +2
-0+} +#else /* CONFIG_PER_VMA_LOCK */ We need to pull out the patches from Dave's series and remove this option and default VMA locks on by default :)
Quoted 4 lines
I didn't see any followup after Dave posted his v1. I agree it would make things much simpler but I don't want to delay these changes too much.
I'll ask Dave if he plans to continue his cleanup. If not, I'll pick it up.
Great thanks!
Quoted 274 lines
Quoted 274 lines
Quoted 138 lines
fs/proc/task_mmu.c +76
-11+static inline int lock_ctx_mm (struct proc_maps_locking_ctx *lock_ctx) +{ +return mmap_read_lock_killable (lock_ctx->mm); +} + +static inline void unlock_ctx_mm (struct proc_maps_locking_ctx *lock_ctx) +{ +mmap_read_unlock (lock_ctx->mm); +} +static inline bool lock_vma_range (struct seq_file *m,struct proc_maps_locking_ctx *lock_ctx) { -return mmap_read_lock_killable (lock_ctx->mm) ==0 ; +return lock_ctx_mm (lock_ctx) ==0 ; }static inline void unlock_vma_range (struct proc_maps_locking_ctx *lock_ctx) { -mmap_read_unlock (lock_ctx->mm); +unlock_ctx_mm (lock_ctx); } static struct vm_area_struct *get_next_vma(struct proc_maps_private *priv,@@ -238,6 +268,9 @@ static inline bool fallback_to_mmap_lock(struct proc_maps_private *priv, return false; } +static inline void drop_rcu (struct proc_maps_private *priv) {} +static inline void reacquire_rcu (struct proc_maps_private *priv) {} +#endif /* CONFIG_PER_VMA_LOCK */ static struct vm_area_struct *proc_get_vma (struct seq_file *m,loff_t *ppos)@@ -538,12 +571,10 @@ static int query_vma_setup(struct proc_maps_locking_ctx *lock_ctx)static void query_vma_teardown (struct proc_maps_locking_ctx *lock_ctx) { -if (lock_ctx->mmap_locked) { -mmap_read_unlock (lock_ctx->mm); - lock_ctx->mmap_locked = false; - }else { +if (lock_ctx->mmap_locked) +unlock_ctx_mm (lock_ctx); +else unlock_ctx_vma (lock_ctx); - } } static struct vm_area_struct *query_vma_find_by_addr(struct proc_maps_locking_ctx *lock_ctx,@@ -1280,21 +1311,75 @@ static const struct mm_walk_ops smaps_shmem_walk_ops = { .walk_lock = PGWALK_RDLOCK, }; +#ifdef CONFIG_PER_VMA_LOCK + +static const struct mm_walk_ops smaps_walk_vma_lock_ops = { + .pmd_entry = smaps_pte_range, + .hugetlb_entry = smaps_hugetlb_range, + .walk_lock =PGWALK_VMA_RDLOCK_VERIFY , +}; + +static const struct mm_walk_ops smaps_shmem_walk_vma_lock_ops = { + .pmd_entry = smaps_pte_range, + .hugetlb_entry = smaps_hugetlb_range, + .pte_hole = smaps_pte_hole, + .walk_lock =PGWALK_VMA_RDLOCK_VERIFY , +}; + +static inline const struct mm_walk_ops * +get_smaps_walk_ops (struct proc_maps_private *priv) +{ +if (priv->lock_ctx.mmap_locked) +return &smaps_walk_ops; +return &smaps_walk_vma_lock_ops; +} + +static inline const struct mm_walk_ops * +get_smaps_shmem_walk_ops (struct proc_maps_private *priv) +{ +if (priv->lock_ctx.mmap_locked) +return &smaps_shmem_walk_ops; +return &smaps_shmem_walk_vma_lock_ops; +} + +#else /* CONFIG_PER_VMA_LOCK */ + +static inline const struct mm_walk_ops * +get_smaps_walk_ops (struct proc_maps_private *priv) +{ +return &smaps_walk_ops; +} + +static inline const struct mm_walk_ops * +get_smaps_shmem_walk_ops (struct proc_maps_private *priv) +{ +return &smaps_shmem_walk_ops; +} + +#endif /* CONFIG_PER_VMA_LOCK */ +/* * Gather mem stats from @vma with the indicated beginning * address @start, and keep them in @mss. * * Use vm_start of @vma as the beginning address if @start is 0. */ -static void smap_gather_stats (struct vm_area_struct *vma, -struct mem_size_stats *mss,unsigned long start) +static void smap_gather_stats (struct proc_maps_private *priv, +struct vm_area_struct *vma, +struct mem_size_stats *mss,unsigned long start) { -const struct mm_walk_ops *ops = &smaps_walk_ops; +const struct mm_walk_ops *ops =get_smaps_walk_ops (priv);/* Invalid start */ if (start >= vma->vm_end)return ; +if (vma ==get_gate_vma (priv->lock_ctx.mm)) +return ; + +/* Might sleep. Drop RCU read lock but keep the VMA locked. */ +drop_rcu (priv); +if (vma->vm_file &&shmem_mapping (vma->vm_file->f_mapping)) {/* * For shared or readonly shmem mappings we know that all @@ -1312,15 +1397,16 @@ static void smap_gather_stats(struct vm_area_struct *vma, !(vma->vm_flags & VM_WRITE))) { mss->swap += shmem_swapped; } else { - ops = &smaps_shmem_walk_ops; + ops =get_smaps_shmem_walk_ops (priv);fs/proc/task_mmu.c static void smap_gather_stats(struct vm_area_struct *vma, · line 1409
Quoted 10 lines
fs/proc/task_mmu.c +2
-1} } -/* mmap_lock is held in m_start */ if (!start)walk_page_vma (vma, ops, mss);else walk_page_range (vma->vm_mm, start, vma->vm_end, ops, mss); + +reacquire_rcu (priv);OK I was going to say that this RCU lock dance is a bit iffy, but I see that we're sharing code such that this is necessary.
Quoted 122 lines
fs/proc/task_mmu.c +41
-11}#define SEQ_PUT_DEC (str, val) \@@ -1369,10 +1455,11 @@ static void __show_smap(struct seq_file *m, const struct mem_size_stats *mss,static int show_smap (struct seq_file *m,void *v) { +struct proc_maps_private *priv = m->private;struct vm_area_struct *vma = v;struct mem_size_stats mss = {}; -smap_gather_stats (vma, &mss,0 ); +smap_gather_stats (priv, vma, &mss,0 );show_map_vma (m, vma);@@ -1413,7 +1500,7 @@ static int show_smaps_rollup(struct seq_file *m, void *v) goto out_put_task; } - ret =mmap_read_lock_killable (mm); + ret =lock_ctx_mm (&priv->lock_ctx);if (ret) goto out_put_mm;@@ -1425,7 +1512,7 @@ static int show_smaps_rollup(struct seq_file *m, void *v) vma_start = vma->vm_start; do { - smap_gather_stats(vma, &mss, 0); + smap_gather_stats(priv, vma, &mss, 0); last_vma_end = vma->vm_end; /*@@ -1434,8 +1521,8 @@ static int show_smaps_rollup(struct seq_file *m, void *v)*/ if (mmap_lock_is_contended (mm)) {vma_iter_invalidate (&vmi); -mmap_read_unlock (mm); - ret =mmap_read_lock_killable (mm); +unlock_ctx_mm (&priv->lock_ctx); + ret =lock_ctx_mm (&priv->lock_ctx);if (ret) {release_task_mempolicy (priv); goto out_put_mm;@@ -1484,14 +1571,14 @@ static int show_smaps_rollup(struct seq_file *m, void *v)/* Case 1 and 2 above */ if (vma->vm_start >= last_vma_end) { -smap_gather_stats (vma, &mss,0 ); +smap_gather_stats (priv, vma, &mss,0 ); last_vma_end = vma->vm_end;continue ; }/* Case 4 above */ if (vma->vm_end > last_vma_end) { -smap_gather_stats (vma, &mss, last_vma_end); +smap_gather_stats (priv, vma, &mss, last_vma_end); last_vma_end = vma->vm_end; } }@@ -1505,7 +1592,7 @@ static int show_smaps_rollup(struct seq_file *m, void *v)__show_smap (m, &mss, true);release_task_mempolicy (priv); -mmap_read_unlock (mm); +unlock_ctx_mm (&priv->lock_ctx); out_put_mm:mmput (mm);@@ -3291,6 +3378,31 @@ static const struct mm_walk_ops show_numa_ops = { .walk_lock = PGWALK_RDLOCK, }; +#ifdef CONFIG_PER_VMA_LOCK +static const struct mm_walk_ops show_numa_vma_lock_ops = { + .hugetlb_entry = gather_hugetlb_stats, + .pmd_entry = gather_pte_stats, + .walk_lock =PGWALK_VMA_RDLOCK_VERIFY , +}; + +static inline const struct mm_walk_ops * +get_show_numa_ops (struct proc_maps_private *priv) +{ +if (priv->lock_ctx.mmap_locked) +return &show_numa_ops; +return &show_numa_vma_lock_ops; +} + +#else /* CONFIG_PER_VMA_LOCK */ + +static inline const struct mm_walk_ops * +get_show_numa_ops (struct proc_maps_private *priv) +{ +return &show_numa_ops; +} + +#endif /* CONFIG_PER_VMA_LOCK */ +/* * Display pages allocated per node and memory policy via /proc. */ @@ -3335,8 +3447,13 @@ static int show_numa_map(struct seq_file *m, void *v)if (is_vm_hugetlb_page (vma))seq_puts (m," huge" ); -/* mmap_lock is held by m_start */ -walk_page_vma (vma, &show_numa_ops, md); +/* Skip walking pages if gate VMA */ +if (vma !=get_gate_vma (proc_priv->lock_ctx.mm)) { +/* Might sleep. Drop RCU read lock but keep the VMA locked. */ +drop_rcu (proc_priv); +walk_page_vma (vma,get_show_numa_ops (proc_priv), md); +reacquire_rcu (proc_priv); + }if (!md->pages) goto out; --2.54.0.545.g6539524ca2-goog
Re: [PATCH v2 1/3] fs/proc/task_mmu: read proc/pid/{smaps|numa_maps} under per-vma lock
Quoted 140 lines · Lorenzo Stoakes
Quoted 39 lines · Suren Baghdasaryan
Quoted 7 lines · Lorenzo Stoakes
Quoted 5 lines · Suren Baghdasaryan
proc/pid/{smaps|numa_maps} can be read using the combination of RCU and VMA read locks, similar to proc/pid/maps. RCU is required to safely traverse the VMA tree and VMA lock stabilizes the VMA being processed and the pagetable walk.
Quoted 6 lines
Quoted 2 lines
Signed-off-by Suren Baghdasaryan <[email protected]>Reviewed-by Liam R. Howlett <[email protected]>The logic LGTM, some nitty things below but in general:
Reviewed-by Lorenzo Stoakes <[email protected]>Thanks!
Quoted 22 lines
Quoted 10 lines
--- fs/proc/task_mmu.c | 195 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 156 insertions(+), 39 deletions(-)fs/proc/task_mmu.c +0
-0fs/proc/task_mmu.c static void release_task_mempolicy(struct proc_maps_private *priv) · line 135
Quoted 4 lines
fs/proc/task_mmu.c +1
-0@@ -132,6 +132,22 @@ static void release_task_mempolicy(struct proc_maps_private *priv)#ifdef CONFIG_PER_VMA_LOCK +static inline int lock_ctx_mm (struct proc_maps_locking_ctx *lock_ctx)NIT: But these inlines are unnecessary in a C file. At the kernel optimisation level the compiler already decides whether to inline or not.
However in the kernel we redefine this so we lose function tracing and also we don't get warnings about unused functions [0], so this is actually a bit harmful
- you can end up with dead code and not know abou tit.Quoted 6 lines
Quoted 1 line
Same goes for all the other static inline's in
task_mmu.c.I see. There are a bunch of these small inline functions. Once the dust settles, I can clean them up. Dave's series and my folloup patch to change
smaps_rollupwill be changing this code. Maybe after all that is done?Yeah that's fine this obviously isn't urgent! :) tiny nits really.
Quoted 90 lines
Quoted 89 lines
[0]: https://elixir.bootlin.com/linux/v7.0.10/source/include/linux/compiler_types.h#L235
Quoted 65 lines
fs/proc/task_mmu.c +24
-17+{ +int ret =mmap_read_lock_killable (lock_ctx->mm); + +if (!ret) + lock_ctx->mmap_locked = true; + +return ret; +} + +static inline void unlock_ctx_mm (struct proc_maps_locking_ctx *lock_ctx) +{ +mmap_read_unlock (lock_ctx->mm); + lock_ctx->mmap_locked = false; +} +static void reset_lock_ctx (struct proc_maps_locking_ctx *lock_ctx) { lock_ctx->locked_vma =NULL ;@@ -146,25 +162,11 @@ static void unlock_ctx_vma(struct proc_maps_locking_ctx *lock_ctx) } } -static const struct seq_operations proc_pid_maps_op; -static inline bool lock_vma_range (struct seq_file *m,struct proc_maps_locking_ctx *lock_ctx) { -/* -* smaps and numa_maps perform page table walk, therefore require -* mmap_lock but maps can be read with locking just the vma and -* walking the vma tree under rcu read protection. -*/ -if (m->op != &proc_pid_maps_op) { -if (mmap_read_lock_killable (lock_ctx->mm)) -return false; - - lock_ctx->mmap_locked = true; - }else { -rcu_read_lock (); -reset_lock_ctx (lock_ctx); - } +rcu_read_lock (); +reset_lock_ctx (lock_ctx);return true; }@@ -172,7 +174,7 @@ static inline bool lock_vma_range(struct seq_file *m,static inline void unlock_vma_range (struct proc_maps_locking_ctx *lock_ctx) {if (lock_ctx->mmap_locked) { -mmap_read_unlock (lock_ctx->mm); +unlock_ctx_mm (lock_ctx); }else {unlock_ctx_vma (lock_ctx);rcu_read_unlock ();@@ -213,17 +215,45 @@ static inline bool fallback_to_mmap_lock(struct proc_maps_private *priv,return true; } +static inline void drop_rcu (struct proc_maps_private *priv) +{ +if (priv->lock_ctx.mmap_locked) +return ; + +rcu_read_unlock ();fs/proc/task_mmu.c static inline bool fallback_to_mmap_lock(struct proc_maps_private *priv, · line 233
Quoted 10 lines
fs/proc/task_mmu.c +10
-0+} + +static inline void reacquire_rcu (struct proc_maps_private *priv) +{ +if (priv->lock_ctx.mmap_locked) +return ; + +rcu_read_lock (); +/* Reinitialize the iterator. */ +vma_iter_set (&priv->iter, priv->lock_ctx.locked_vma->vm_end);OK good that we reset this :)
fs/proc/task_mmu.c static inline bool fallback_to_mmap_lock(struct proc_maps_private *priv, · line 237
Quoted 4 lines
fs/proc/task_mmu.c +2
-0+} +#else /* CONFIG_PER_VMA_LOCK */ We need to pull out the patches from Dave's series and remove this option and default VMA locks on by default :)
Quoted 6 lines
Quoted 4 lines
I didn't see any followup after Dave posted his v1. I agree it would make things much simpler but I don't want to delay these changes too much.
I'll ask Dave if he plans to continue his cleanup. If not, I'll pick it up.Great thanks!
I checked with Dave and he will post the next version in a week or so.
Quoted 275 lines
Quoted 275 lines
Quoted 275 lines
Quoted 139 lines
fs/proc/task_mmu.c +76
-11+static inline int lock_ctx_mm (struct proc_maps_locking_ctx *lock_ctx) +{ +return mmap_read_lock_killable (lock_ctx->mm); +} + +static inline void unlock_ctx_mm (struct proc_maps_locking_ctx *lock_ctx) +{ +mmap_read_unlock (lock_ctx->mm); +} +static inline bool lock_vma_range (struct seq_file *m,struct proc_maps_locking_ctx *lock_ctx) { -return mmap_read_lock_killable (lock_ctx->mm) ==0 ; +return lock_ctx_mm (lock_ctx) ==0 ; }static inline void unlock_vma_range (struct proc_maps_locking_ctx *lock_ctx) { -mmap_read_unlock (lock_ctx->mm); +unlock_ctx_mm (lock_ctx); } static struct vm_area_struct *get_next_vma(struct proc_maps_private *priv,@@ -238,6 +268,9 @@ static inline bool fallback_to_mmap_lock(struct proc_maps_private *priv, return false; } +static inline void drop_rcu (struct proc_maps_private *priv) {} +static inline void reacquire_rcu (struct proc_maps_private *priv) {} +#endif /* CONFIG_PER_VMA_LOCK */ static struct vm_area_struct *proc_get_vma (struct seq_file *m,loff_t *ppos)@@ -538,12 +571,10 @@ static int query_vma_setup(struct proc_maps_locking_ctx *lock_ctx)static void query_vma_teardown (struct proc_maps_locking_ctx *lock_ctx) { -if (lock_ctx->mmap_locked) { -mmap_read_unlock (lock_ctx->mm); - lock_ctx->mmap_locked = false; - }else { +if (lock_ctx->mmap_locked) +unlock_ctx_mm (lock_ctx); +else unlock_ctx_vma (lock_ctx); - } } static struct vm_area_struct *query_vma_find_by_addr(struct proc_maps_locking_ctx *lock_ctx,@@ -1280,21 +1311,75 @@ static const struct mm_walk_ops smaps_shmem_walk_ops = { .walk_lock = PGWALK_RDLOCK, }; +#ifdef CONFIG_PER_VMA_LOCK + +static const struct mm_walk_ops smaps_walk_vma_lock_ops = { + .pmd_entry = smaps_pte_range, + .hugetlb_entry = smaps_hugetlb_range, + .walk_lock =PGWALK_VMA_RDLOCK_VERIFY , +}; + +static const struct mm_walk_ops smaps_shmem_walk_vma_lock_ops = { + .pmd_entry = smaps_pte_range, + .hugetlb_entry = smaps_hugetlb_range, + .pte_hole = smaps_pte_hole, + .walk_lock =PGWALK_VMA_RDLOCK_VERIFY , +}; + +static inline const struct mm_walk_ops * +get_smaps_walk_ops (struct proc_maps_private *priv) +{ +if (priv->lock_ctx.mmap_locked) +return &smaps_walk_ops; +return &smaps_walk_vma_lock_ops; +} + +static inline const struct mm_walk_ops * +get_smaps_shmem_walk_ops (struct proc_maps_private *priv) +{ +if (priv->lock_ctx.mmap_locked) +return &smaps_shmem_walk_ops; +return &smaps_shmem_walk_vma_lock_ops; +} + +#else /* CONFIG_PER_VMA_LOCK */ + +static inline const struct mm_walk_ops * +get_smaps_walk_ops (struct proc_maps_private *priv) +{ +return &smaps_walk_ops; +} + +static inline const struct mm_walk_ops * +get_smaps_shmem_walk_ops (struct proc_maps_private *priv) +{ +return &smaps_shmem_walk_ops; +} + +#endif /* CONFIG_PER_VMA_LOCK */ +/* * Gather mem stats from @vma with the indicated beginning * address @start, and keep them in @mss. * * Use vm_start of @vma as the beginning address if @start is 0. */ -static void smap_gather_stats (struct vm_area_struct *vma, -struct mem_size_stats *mss,unsigned long start) +static void smap_gather_stats (struct proc_maps_private *priv, +struct vm_area_struct *vma, +struct mem_size_stats *mss,unsigned long start) { -const struct mm_walk_ops *ops = &smaps_walk_ops; +const struct mm_walk_ops *ops =get_smaps_walk_ops (priv);/* Invalid start */ if (start >= vma->vm_end)return ; +if (vma ==get_gate_vma (priv->lock_ctx.mm)) +return ; + +/* Might sleep. Drop RCU read lock but keep the VMA locked. */ +drop_rcu (priv); +if (vma->vm_file &&shmem_mapping (vma->vm_file->f_mapping)) {/* * For shared or readonly shmem mappings we know that all @@ -1312,15 +1397,16 @@ static void smap_gather_stats(struct vm_area_struct *vma, !(vma->vm_flags & VM_WRITE))) { mss->swap += shmem_swapped; } else { - ops = &smaps_shmem_walk_ops; + ops =get_smaps_shmem_walk_ops (priv);fs/proc/task_mmu.c static void smap_gather_stats(struct vm_area_struct *vma, · line 1409
Quoted 10 lines
fs/proc/task_mmu.c +2
-1} } -/* mmap_lock is held in m_start */ if (!start)walk_page_vma (vma, ops, mss);else walk_page_range (vma->vm_mm, start, vma->vm_end, ops, mss); + +reacquire_rcu (priv);OK I was going to say that this RCU lock dance is a bit iffy, but I see that we're sharing code such that this is necessary.
Quoted 122 lines
fs/proc/task_mmu.c +41
-11}#define SEQ_PUT_DEC (str, val) \@@ -1369,10 +1455,11 @@ static void __show_smap(struct seq_file *m, const struct mem_size_stats *mss,static int show_smap (struct seq_file *m,void *v) { +struct proc_maps_private *priv = m->private;struct vm_area_struct *vma = v;struct mem_size_stats mss = {}; -smap_gather_stats (vma, &mss,0 ); +smap_gather_stats (priv, vma, &mss,0 );show_map_vma (m, vma);@@ -1413,7 +1500,7 @@ static int show_smaps_rollup(struct seq_file *m, void *v) goto out_put_task; } - ret =mmap_read_lock_killable (mm); + ret =lock_ctx_mm (&priv->lock_ctx);if (ret) goto out_put_mm;@@ -1425,7 +1512,7 @@ static int show_smaps_rollup(struct seq_file *m, void *v) vma_start = vma->vm_start; do { - smap_gather_stats(vma, &mss, 0); + smap_gather_stats(priv, vma, &mss, 0); last_vma_end = vma->vm_end; /*@@ -1434,8 +1521,8 @@ static int show_smaps_rollup(struct seq_file *m, void *v)*/ if (mmap_lock_is_contended (mm)) {vma_iter_invalidate (&vmi); -mmap_read_unlock (mm); - ret =mmap_read_lock_killable (mm); +unlock_ctx_mm (&priv->lock_ctx); + ret =lock_ctx_mm (&priv->lock_ctx);if (ret) {release_task_mempolicy (priv); goto out_put_mm;@@ -1484,14 +1571,14 @@ static int show_smaps_rollup(struct seq_file *m, void *v)/* Case 1 and 2 above */ if (vma->vm_start >= last_vma_end) { -smap_gather_stats (vma, &mss,0 ); +smap_gather_stats (priv, vma, &mss,0 ); last_vma_end = vma->vm_end;continue ; }/* Case 4 above */ if (vma->vm_end > last_vma_end) { -smap_gather_stats (vma, &mss, last_vma_end); +smap_gather_stats (priv, vma, &mss, last_vma_end); last_vma_end = vma->vm_end; } }@@ -1505,7 +1592,7 @@ static int show_smaps_rollup(struct seq_file *m, void *v)__show_smap (m, &mss, true);release_task_mempolicy (priv); -mmap_read_unlock (mm); +unlock_ctx_mm (&priv->lock_ctx); out_put_mm:mmput (mm);@@ -3291,6 +3378,31 @@ static const struct mm_walk_ops show_numa_ops = { .walk_lock = PGWALK_RDLOCK, }; +#ifdef CONFIG_PER_VMA_LOCK +static const struct mm_walk_ops show_numa_vma_lock_ops = { + .hugetlb_entry = gather_hugetlb_stats, + .pmd_entry = gather_pte_stats, + .walk_lock =PGWALK_VMA_RDLOCK_VERIFY , +}; + +static inline const struct mm_walk_ops * +get_show_numa_ops (struct proc_maps_private *priv) +{ +if (priv->lock_ctx.mmap_locked) +return &show_numa_ops; +return &show_numa_vma_lock_ops; +} + +#else /* CONFIG_PER_VMA_LOCK */ + +static inline const struct mm_walk_ops * +get_show_numa_ops (struct proc_maps_private *priv) +{ +return &show_numa_ops; +} + +#endif /* CONFIG_PER_VMA_LOCK */ +/* * Display pages allocated per node and memory policy via /proc. */ @@ -3335,8 +3447,13 @@ static int show_numa_map(struct seq_file *m, void *v)if (is_vm_hugetlb_page (vma))seq_puts (m," huge" ); -/* mmap_lock is held by m_start */ -walk_page_vma (vma, &show_numa_ops, md); +/* Skip walking pages if gate VMA */ +if (vma !=get_gate_vma (proc_priv->lock_ctx.mm)) { +/* Might sleep. Drop RCU read lock but keep the VMA locked. */ +drop_rcu (proc_priv); +walk_page_vma (vma,get_show_numa_ops (proc_priv), md); +reacquire_rcu (proc_priv); + }if (!md->pages) goto out; --2.54.0.545.g6539524ca2-goog
Lorenzo Stoakes
maintainerRe: [PATCH v2 1/3] fs/proc/task_mmu: read proc/pid/{smaps|numa_maps} under per-vma lock
Quoted 13 lines · Suren Baghdasaryan
Quoted 9 lines
Quoted 7 lines
Quoted 2 lines
We need to pull out the patches from Dave's series and remove this option and default VMA locks on by default :)
I didn't see any followup after Dave posted his v1. I agree it would make things much simpler but I don't want to delay these changes too much.
I'll ask Dave if he plans to continue his cleanup. If not, I'll pick it up.Great thanks!
I checked with Dave and he will post the next version in a week or so.
Great thanks! :)
Cheers, Lorenzo
PATCH v2 2/3 selftests/proc: ensure the test is performed at the right page boundary
In reply to Suren BaghdasaryanWhen running tearing tests we need to ensure the pages we use include VMAs that were mapped by the child process for this test. Currently we always use the first two pages, checking VMAs at their boundaries and this works, however once we add tests for /proc/pid/smaps, the first two pages might not contain the VMAs that child modifies.
Locate the page that contains the first VMA mapped by the child and use that and the next page for the test.
--- tools/testing/selftests/proc/proc-maps-race.c | 119 +++++++++++++++--- 1 file changed, 100 insertions(+), 19 deletions(-)
tools/testing/selftests/proc/proc-maps-race.c +100 -19
@@ -39,6 +39,13 @@#include <sys/types.h> #include <sys/wait.h> +#define min (a, b) \ + ({ \ + typeof(a) _a = (a); \ + typeof(b) _b = (b); \ + _a < _b ? _a : _b; \ + }) +/* /proc/pid/maps parsing routines */ struct page_content {char *data;
@@ -77,6 +84,7 @@ FIXTURE(proc_maps_race)struct line_content first_line;unsigned long duration_sec;int shared_mem_size; +int skip_pages;int page_size;int vma_count;bool verbose;
@@ -105,38 +113,102 @@ struct vma_modifier_info {void *child_mapped_addr[]; }; - -static bool read_two_pages (FIXTURE_DATA (proc_maps_race) *self) +static bool read_page (FIXTURE_DATA (proc_maps_race) *self, +struct page_content *page) {ssize_t bytes_read; -if (lseek (self->maps_fd,0 ,SEEK_SET ) <0 ) + bytes_read =read (self->maps_fd, page->data, self->page_size); +if (bytes_read <=0 )return false; - bytes_read =read (self->maps_fd, self->page1.data, self->page_size); -if (bytes_read <=0 ) +/* Make sure data always ends with a newline character. */ +if (page->data[bytes_read -1 ] !='\n' )return false; - self->page1.size = bytes_read; + page->size = bytes_read; - bytes_read =read (self->maps_fd, self->page2.data, self->page_size); -if (bytes_read <=0 ) +return true; +} + +static bool parse_vma_line (char *line_start,char *line_end, +unsigned long *start,unsigned long *end) +{ +bool found; + + *line_end ='\0' ;/* stop sscanf at the EOL */ + found = (sscanf (line_start,"%lx-%lx" , start, end) ==2 ); + *line_end ='\n' ; + +return found; +} + +static int locate_containing_page (FIXTURE_DATA (proc_maps_race) *self, +unsigned long addr,unsigned long size) +{ +unsigned long start, end; +int page =0 ; + +if (lseek (self->maps_fd,0 ,SEEK_SET ) <0 ) +return -1 ; + +while (true) { +char *curr_pos; +char *end_pos; + +if (!read_page (self, &self->page1)) +return -1 ; + + curr_pos = self->page1.data; + end_pos = self->page1.data + self->page1.size; +while (curr_pos < end_pos) { +char *line_end; + + line_end =strchr (curr_pos,'\n' ); +if (!line_end) +break ; + +if (parse_vma_line (curr_pos, line_end, &start, &end) && + start == addr && end == addr + size) +return page; + + curr_pos = line_end +1 ; + } + page++; + } + +return 0 ; +} + +static bool read_two_pages (FIXTURE_DATA (proc_maps_race) *self) +{ +if (lseek (self->maps_fd,0 ,SEEK_SET ) <0 )return false; - self->page2.size = bytes_read; +for (int i =0 ; i < self->skip_pages; i++) +if (!read_page (self, &self->page1)) +return false; -return true; +return read_page (self, &self->page1) &&read_page (self, &self->page2); } -static void copy_first_line (struct page_content *page,char *first_line) +static void copy_line (const char *line_start,const char *line_end, +char *buf,size_t buf_size) { -char *pos =strchr (page->data,'\n' ); +size_t len =min (line_end - line_start, buf_size -1 ); -strncpy (first_line, page->data, pos - page->data); - first_line[pos - page->data] ='\0' ; +strncpy (buf, line_start, len); + buf[len] ='\0' ; } -static void copy_last_line (struct page_content *page,char *last_line) +static void copy_first_line (struct page_content *page,char *first_line, +size_t line_size) +{ +copy_line (page->data,strchr (page->data,'\n' ), first_line, line_size); +} + +static void copy_last_line (struct page_content *page,char *last_line, +size_t line_size) {/* Get the last line in the first page */ const char *end = page->data + page->size -1 ;
@@ -146,8 +218,8 @@ static void copy_last_line(struct page_content *page, char *last_line)/* search previous newline */ while (pos[-1 ] !='\n' ) pos--; -strncpy (last_line, pos, end - pos); - last_line[end - pos] ='\0' ; + +copy_line (pos, end, last_line, line_size); }/* Read the last line of the first page and the first line of the second page */
@@ -158,8 +230,8 @@ static bool read_boundary_lines(FIXTURE_DATA(proc_maps_race) *self,if (!read_two_pages (self))return false; -copy_last_line (&self->page1, last_line->text); -copy_first_line (&self->page2, first_line->text); +copy_last_line (&self->page1, last_line->text,LINE_MAX_SIZE ); +copy_first_line (&self->page2, first_line->text,LINE_MAX_SIZE );return sscanf (last_line->text,"%lx-%lx" , &last_line->start_addr, &last_line->end_addr) == 2 &&
@@ -418,6 +490,8 @@ FIXTURE_SETUP(proc_maps_race)struct vma_modifier_info *mod_info;pthread_mutexattr_t mutex_attr;pthread_condattr_t cond_attr; +unsigned long first_map_addr; +unsigned long last_map_addr;unsigned long duration_sec;char fname[32 ];
@@ -502,6 +576,13 @@ FIXTURE_SETUP(proc_maps_race) self->page2.data =malloc (self->page_size);ASSERT_NE (self->page2.data,NULL ); + first_map_addr = (unsigned long )mod_info->child_mapped_addr[0 ]; + last_map_addr = (unsigned long )mod_info->child_mapped_addr[mod_info->vma_count -1 ]; + + self->skip_pages =locate_containing_page (self, +min (first_map_addr, last_map_addr), + self->page_size *3 ); +ASSERT_NE (self->skip_pages,-1 );ASSERT_TRUE (read_boundary_lines (self, &self->last_line, &self->first_line));/*
Signature · 1 line
2.54.0.545.g6539524ca2-goog
PATCH v2 3/3 selftests/proc: add /proc/pid/smaps tearing tests
In reply to Suren BaghdasaryanAdd tearing tests for /proc/pid/smaps file. New tests reuse the same logic as with maps file but skipping all the data except for the VMA addresses, which are the only part relevant for the tearing tests. Skip PROCMAP_QUERY parts of the tests because smaps does not implement that ioctl.
--- tools/testing/selftests/proc/proc-maps-race.c | 178 +++++++++++++----- 1 file changed, 133 insertions(+), 45 deletions(-)
tools/testing/selftests/proc/proc-maps-race.c +133 -45
@@ -17,8 +17,8 @@*/ /* * Fork a child that concurrently modifies address space while the main -* process is reading /proc/$PID/maps and verifying the results. Address -* space modifications include: +* process is reading /proc/$PID/maps and /proc/$PID/smaps, verifying the +* results. Address space modifications include: * VMA splitting and merging * */
@@ -73,6 +73,11 @@ enum test_state { TEST_DONE, }; +enum maps_file { +MAPS , +SMAPS , +}; +struct vma_modifier_info ;FIXTURE (proc_maps_race)
@@ -83,6 +88,7 @@ FIXTURE(proc_maps_race)struct line_content last_line;struct line_content first_line;unsigned long duration_sec; +enum maps_file maps_file;int shared_mem_size;int skip_pages;int page_size;
@@ -92,6 +98,19 @@ FIXTURE(proc_maps_race)pid_t pid; }; +FIXTURE_VARIANT (proc_maps_race) +{ +const enum maps_file maps_file; +}; + +FIXTURE_VARIANT_ADD(proc_maps_race, maps) { + .maps_file =MAPS , +}; + +FIXTURE_VARIANT_ADD(proc_maps_race, smaps) { + .maps_file = SMAPS, +}; +typedef bool (*vma_modifier_op )(FIXTURE_DATA (proc_maps_race) *self); typedef bool (*vma_mod_result_check_op)(struct line_content *mod_last_line, struct line_content *mod_first_line,
@@ -222,6 +241,57 @@ static void copy_last_line(struct page_content *page, char *last_line,copy_line (pos, end, last_line, line_size); } +static bool copy_first_entry (struct page_content *page,char *first_line, +size_t line_size) +{ +char *start_pos = page->data; + +while (start_pos < page->data + page->size) { +unsigned long start_addr; +unsigned long end_addr; +char *end_pos; + + end_pos =strchr (start_pos,'\n' ); +if (!end_pos) +break ; + +if (parse_vma_line (start_pos, end_pos, &start_addr, &end_addr)) { +copy_line (start_pos, end_pos, first_line, line_size); +return true; + } + + start_pos = end_pos +1 ; + } + +return false; +} + +static bool copy_last_entry (struct page_content *page,char *last_line, +size_t line_size) +{ +char *end_pos = page->data + page->size -1 ; +char *start_pos; + +while (end_pos > page->data) { +unsigned long start_addr; +unsigned long end_addr; + +/* skip last newline */ + start_pos = end_pos -1 ; +/* search previous newline */ +while (start_pos > page->data && start_pos[-1 ] !='\n' ) + start_pos--; +if (parse_vma_line (start_pos, end_pos, &start_addr, &end_addr)) { +copy_line (start_pos, end_pos, last_line, line_size); +return true; + } + + end_pos = start_pos -1 ; + } + +return false; +} +/* Read the last line of the first page and the first line of the second page */ static bool read_boundary_lines(FIXTURE_DATA(proc_maps_race) *self, struct line_content *last_line,
@@ -230,8 +300,16 @@ static bool read_boundary_lines(FIXTURE_DATA(proc_maps_race) *self,if (!read_two_pages (self))return false; -copy_last_line (&self->page1, last_line->text,LINE_MAX_SIZE ); -copy_first_line (&self->page2, first_line->text,LINE_MAX_SIZE ); +if (self->maps_file ==MAPS ) { +copy_last_line (&self->page1, last_line->text,LINE_MAX_SIZE ); +copy_first_line (&self->page2, first_line->text,LINE_MAX_SIZE ); + }else if (self->maps_file ==SMAPS ) { +if (!copy_last_entry (&self->page1, last_line->text,LINE_MAX_SIZE ) || + !copy_first_entry (&self->page2, first_line->text,LINE_MAX_SIZE )) +return false; + }else { +return false; + }return sscanf (last_line->text,"%lx-%lx" , &last_line->start_addr, &last_line->end_addr) == 2 &&
@@ -497,6 +575,7 @@ FIXTURE_SETUP(proc_maps_race) self->page_size = (unsigned long )sysconf (_SC_PAGESIZE); self->verbose = verbose && !strncmp (verbose,"1" ,1 ); + self->maps_file = variant->maps_file; duration_sec = duration ?atol (duration) :0 ; self->duration_sec = duration_sec ? duration_sec :5UL ;
@@ -563,7 +642,16 @@ FIXTURE_SETUP(proc_maps_race)exit (0 ); } -sprintf (fname,"/proc/%d/maps" , self->pid); +switch (self->maps_file) { +case MAPS : +sprintf (fname,"/proc/%d/maps" , self->pid); +break ; +case SMAPS : +sprintf (fname,"/proc/%d/smaps" , self->pid); +break ; +default : +ksft_exit_fail (); + } self->maps_fd =open (fname,O_RDONLY );ASSERT_NE (self->maps_fd,-1 );
@@ -608,7 +696,6 @@ FIXTURE_SETUP(proc_maps_race)ASSERT_TRUE (mod_info->addr && mod_info->next_addr);signal_state (mod_info,PARENT_READY ); - }FIXTURE_TEARDOWN (proc_maps_race)
@@ -698,20 +785,20 @@ TEST_F(proc_maps_race, test_maps_tearing_from_split) last_line_changed =strcmp (new_last_line.text, self->last_line.text) !=0 ; first_line_changed =strcmp (new_first_line.text, self->first_line.text) !=0 ;ASSERT_EQ (last_line_changed, first_line_changed); - -/* Check if PROCMAP_QUERY ioclt() finds the right VMA */ -ASSERT_TRUE (query_addr_at (self->maps_fd, mod_info->addr + self->page_size, - &vma_start, &vma_end)); -/* -* The vma at the split address can be either the same as -* original one (if read before the split) or the same as the -* first line in the second page (if read after the split). -*/ -ASSERT_TRUE ((vma_start == self->last_line.start_addr && - vma_end == self->last_line.end_addr) || - (vma_start == split_first_line.start_addr && - vma_end == split_first_line.end_addr)); - +if (self->maps_file ==MAPS ) { +/* Check if PROCMAP_QUERY ioclt() finds the right VMA */ +ASSERT_TRUE (query_addr_at (self->maps_fd, mod_info->addr + self->page_size, + &vma_start, &vma_end)); +/* +* The vma at the split address can be either the same as +* original one (if read before the split) or the same as the +* first line in the second page (if read after the split). +*/ +ASSERT_TRUE ((vma_start == self->last_line.start_addr && + vma_end == self->last_line.end_addr) || + (vma_start == split_first_line.start_addr && + vma_end == split_first_line.end_addr)); + }clock_gettime (CLOCK_MONOTONIC_COARSE , &end_ts);end_test_iteration (&end_ts, self->verbose); }while (end_ts.tv_sec - start_ts.tv_sec < self->duration_sec);
@@ -781,17 +868,18 @@ TEST_F(proc_maps_race, test_maps_tearing_from_resize)strcmp (new_first_line.text, restored_first_line.text), "Expand result invalid", self)); } - -/* Check if PROCMAP_QUERY ioclt() finds the right VMA */ -ASSERT_TRUE (query_addr_at (self->maps_fd, mod_info->addr, &vma_start, &vma_end)); -/* -* The vma should stay at the same address and have either the -* original size of 3 pages or 1 page if read after shrinking. -*/ -ASSERT_TRUE (vma_start == self->last_line.start_addr && - (vma_end - vma_start == self->page_size *3 || - vma_end - vma_start == self->page_size)); - +if (self->maps_file ==MAPS ) { +/* Check if PROCMAP_QUERY ioclt() finds the right VMA */ +ASSERT_TRUE (query_addr_at (self->maps_fd, mod_info->addr, + &vma_start, &vma_end)); +/* +* The vma should stay at the same address and have either the +* original size of 3 pages or 1 page if read after shrinking. +*/ +ASSERT_TRUE (vma_start == self->last_line.start_addr && + (vma_end - vma_start == self->page_size *3 || + vma_end - vma_start == self->page_size)); + }clock_gettime (CLOCK_MONOTONIC_COARSE , &end_ts);end_test_iteration (&end_ts, self->verbose); }while (end_ts.tv_sec - start_ts.tv_sec < self->duration_sec);
@@ -861,20 +949,20 @@ TEST_F(proc_maps_race, test_maps_tearing_from_remap)strcmp (new_first_line.text, restored_first_line.text), "Remap restore result invalid", self)); } - -/* Check if PROCMAP_QUERY ioclt() finds the right VMA */ -ASSERT_TRUE (query_addr_at (self->maps_fd, mod_info->addr + self->page_size, - &vma_start, &vma_end)); -/* -* The vma should either stay at the same address and have the -* original size of 3 pages or we should find the remapped vma -* at the remap destination address with size of 1 page. -*/ -ASSERT_TRUE ((vma_start == self->last_line.start_addr && - vma_end - vma_start == self->page_size *3 ) || - (vma_start == self->last_line.start_addr + self->page_size && - vma_end - vma_start == self->page_size)); - +if (self->maps_file ==MAPS ) { +/* Check if PROCMAP_QUERY ioclt() finds the right VMA */ +ASSERT_TRUE (query_addr_at (self->maps_fd, mod_info->addr + self->page_size, + &vma_start, &vma_end)); +/* +* The vma should either stay at the same address and have the +* original size of 3 pages or we should find the remapped vma +* at the remap destination address with size of 1 page. +*/ +ASSERT_TRUE ((vma_start == self->last_line.start_addr && + vma_end - vma_start == self->page_size *3 ) || + (vma_start == self->last_line.start_addr + self->page_size && + vma_end - vma_start == self->page_size)); + }clock_gettime (CLOCK_MONOTONIC_COARSE , &end_ts);end_test_iteration (&end_ts, self->verbose); }while (end_ts.tv_sec - start_ts.tv_sec < self->duration_sec);
Signature · 1 line
2.54.0.545.g6539524ca2-goog
Andrew Morton
maintainerQuoted 2 lines · On Sat, 25 Apr 2026 23:27:15 -0700 Suren Baghdasaryan <[email protected]> wrote:
Use per-vma locks when reading
/proc/pid/smapsand/proc/pid/numa_mapssimilar to/proc/pid/mapsto reduce contention on centralmmap_lock.
Was the benefit measured/measurable?
Sashiko doesn't like your strchr():
https://sashiko.dev/#/patchset/[email protected]
Quoted 10 lines · Andrew Morton
Quoted 2 lines · On Sat, 25 Apr 2026 23:27:15 -0700 Suren Baghdasaryan <[email protected]> wrote:
Use per-vma locks when reading
/proc/pid/smapsand/proc/pid/numa_mapssimilar to/proc/pid/mapsto reduce contention on centralmmap_lock.Was the benefit measured/measurable?
Sashiko doesn't like your
strchr():https://sashiko.dev/#/patchset/[email protected]
Sorry, missed your comment earlier.
I think strchr() is fine here and Sashiko did notice that read_page() always makes sure the buffer ends with a newline character, IOW *end_pos == '\n'. So, once we found this last newline and moved past it, we terminate the loop. I guess Sashiko is being overprotective...
Quoted 7 lines · Suren Baghdasaryan
Use per-vma locks when reading
/proc/pid/smapsand/proc/pid/numa_mapssimilar to/proc/pid/mapsto reduce contention on centralmmap_lock. One major difference between maps and smaps/numa_maps reading is that the latter executes page table walk which can't be done under RCU due to a possibility of sleeping. Therefore we drop RCU read lock before this walk while keeping the VMA locked. After the walk we retake RCU read lock, reset VMA iterator and proceed with the next VMA.
With many small VMAs, is that overhead noticable?
Signature · 3 lines
Cheers, David
Quoted 11 lines · David Hildenbrand (Arm)
Quoted 7 lines · Suren Baghdasaryan
Use per-vma locks when reading
/proc/pid/smapsand/proc/pid/numa_mapssimilar to/proc/pid/mapsto reduce contention on centralmmap_lock. One major difference between maps and smaps/numa_maps reading is that the latter executes page table walk which can't be done under RCU due to a possibility of sleeping. Therefore we drop RCU read lock before this walk while keeping the VMA locked. After the walk we retake RCU read lock, reset VMA iterator and proceed with the next VMA.With many small VMAs, is that overhead noticable?
It might be but the point of this patchset (and the previous one that made a similar change for /proc/pid/maps) is to reduce mmap_lock contention, not to speed up the read operation, which is not a performance critical part. The original problem that Paul McKenney described and which kicked these series of changes is that a low-priority monitoring process reading /proc/pid/{maps|smaps|...} can block a high-priority updates by holding the mmap_lock. You can see details about this problem and the numbers Paul obtained with the previous change in here:
https://lore.kernel.org/all/[email protected]/
Quoted 5 lines
--
Cheers,David
Quoted 12 lines · Suren Baghdasaryan
Quoted 10 lines · David Hildenbrand (Arm)
Quoted 7 lines · Suren Baghdasaryan
Use per-vma locks when reading
/proc/pid/smapsand/proc/pid/numa_mapssimilar to/proc/pid/mapsto reduce contention on centralmmap_lock. One major difference between maps and smaps/numa_maps reading is that the latter executes page table walk which can't be done under RCU due to a possibility of sleeping. Therefore we drop RCU read lock before this walk while keeping the VMA locked. After the walk we retake RCU read lock, reset VMA iterator and proceed with the next VMA.
Quoted 6 lines
Quoted 1 line
With many small VMAs, is that overhead noticable?
It might be but the point of this patchset (and the previous one that made a similar change for
/proc/pid/maps) is to reducemmap_lockcontention, not to speed up the read operation, which is not a performance critical part.
Well, this interface has been around .. forever, so if there is a noticeable change in performance it should be called out.
Quoted 7 lines
The original problem that Paul McKenney
described and which kicked these series of changes is that a low-priority monitoring process reading /proc/pid/{maps|smaps|...} can block a high-priority updates by holding themmap_lock. You can see details about this problem and the numbers Paul obtained with the previous change in here:
https://lore.kernel.org/all/[email protected]/
Yes, I know. A problem that has been around ... forever as well :)
Signature · 3 lines
Cheers, David
Quoted 16 lines · David Hildenbrand (Arm)
Quoted 14 lines · Suren Baghdasaryan
Quoted 11 lines · David Hildenbrand (Arm)
Quoted 7 lines · Suren Baghdasaryan
Use per-vma locks when reading
/proc/pid/smapsand/proc/pid/numa_mapssimilar to/proc/pid/mapsto reduce contention on centralmmap_lock. One major difference between maps and smaps/numa_maps reading is that the latter executes page table walk which can't be done under RCU due to a possibility of sleeping. Therefore we drop RCU read lock before this walk while keeping the VMA locked. After the walk we retake RCU read lock, reset VMA iterator and proceed with the next VMA.With many small VMAs, is that overhead noticable?
Quoted 7 lines
Quoted 4 lines
It might be but the point of this patchset (and the previous one that made a similar change for
/proc/pid/maps) is to reducemmap_lockcontention, not to speed up the read operation, which is not a performance critical part.Well, this interface has been around .. forever, so if there is a noticeable change in performance it should be called out.
Sorry, I missed your reply. I'll see if I can adopt Paul's test for /proc/pid/maps [1] for benchmarking smaps but I would expect similar results as was reported in [2].
[1] https://github.com/paulmckrcu/proc-mmap_sem-test
[2] https://lore.kernel.org/all/[email protected]/
Quoted 15 lines
Quoted 8 lines
The original problem that Paul McKenney
described and which kicked these series of changes is that a low-priority monitoring process reading /proc/pid/{maps|smaps|...} can block a high-priority updates by holding themmap_lock. You can see details about this problem and the numbers Paul obtained with the previous change in here:
https://lore.kernel.org/all/[email protected]/Yes, I know. A problem that has been around ... forever as well :)
--
Cheers,David
Andrew Morton
maintainerQuoted 11 lines · On Thu, 21 May 2026 08:16:01 -0700 Suren Baghdasaryan <[email protected]> wrote:
Quoted 7 lines
Quoted 4 lines
It might be but the point of this patchset (and the previous one that made a similar change for
/proc/pid/maps) is to reducemmap_lockcontention, not to speed up the read operation, which is not a performance critical part.Well, this interface has been around .. forever, so if there is a noticeable change in performance it should be called out.
Sorry, I missed your reply. I'll see if I can adopt Paul's test for
/proc/pid/maps[1] for benchmarking smaps but I would expect similar results as was reported in [2].
How's it coming along ;)
Quoted 2 lines
[1] https://github.com/paulmckrcu/proc-mmap_sem-test
[2] https://lore.kernel.org/all/[email protected]/
I've moved this series to the tail of mm-unstable to permit more time.
Lorenzo Stoakes
maintainerQuoted 14 lines · Andrew Morton
Quoted 11 lines · On Thu, 21 May 2026 08:16:01 -0700 Suren Baghdasaryan <[email protected]> wrote:
Quoted 7 lines
Quoted 4 lines
It might be but the point of this patchset (and the previous one that made a similar change for
/proc/pid/maps) is to reducemmap_lockcontention, not to speed up the read operation, which is not a performance critical part.Well, this interface has been around .. forever, so if there is a noticeable change in performance it should be called out.
Sorry, I missed your reply. I'll see if I can adopt Paul's test for
/proc/pid/maps[1] for benchmarking smaps but I would expect similar results as was reported in [2].
Quoted 6 lines
How's it coming along ;)
Quoted 2 lines
[1] https://github.com/paulmckrcu/proc-mmap_sem-test
[2] https://lore.kernel.org/all/[email protected]/I've moved this series to the tail of mm-unstable to permit more time.
Well I'm not sure it's _vital_ to get stats for this, it's pretty much an extension of existing VMA lock work in /proc/$pid/maps -> smaps, and the logic is sound.
There's no reason to believe there will be anything other than a reduction in lock contention here at least under whichever workloads happen to hammer smaps, but doesn't feel like there's a downside!
Cheers, Lorenzo
Quoted 23 lines · Lorenzo Stoakes
Quoted 14 lines · Andrew Morton
Quoted 11 lines · On Thu, 21 May 2026 08:16:01 -0700 Suren Baghdasaryan <[email protected]> wrote:
Quoted 7 lines
Quoted 4 lines
It might be but the point of this patchset (and the previous one that made a similar change for
/proc/pid/maps) is to reducemmap_lockcontention, not to speed up the read operation, which is not a performance critical part.Well, this interface has been around .. forever, so if there is a noticeable change in performance it should be called out.
Sorry, I missed your reply. I'll see if I can adopt Paul's test for
/proc/pid/maps[1] for benchmarking smaps but I would expect similar results as was reported in [2].Quoted 6 lines
How's it coming along ;)
Quoted 2 lines
[1] https://github.com/paulmckrcu/proc-mmap_sem-test
[2] https://lore.kernel.org/all/[email protected]/I've moved this series to the tail of mm-unstable to permit more time.
Quoted 7 lines
Well I'm not sure it's _vital_ to get stats for this, it's pretty much an extension of existing VMA lock work in
/proc/$pid/maps-> smaps, and the logic is sound.There's no reason to believe there will be anything other than a reduction in lock contention here at least under whichever workloads happen to hammer smaps, but doesn't feel like there's a downside!
Just finished running Paul's test. I ran a smaller number of iterations (20 instead of 100) because it takes lots of time but results are quite convincing.
In the baseline, all runs had maximum latencies exceeding 9.7 milliseconds. In contrast, the patched version has maximum latencies in the range of 1.8-3.2 milliseconds.
Median performance of the baseline is at 1.135 milliseconds and patch series at 7 microseconds.
Baseline
./run-proc-vs-map.sh --nsamples 20 --rawdata -- --busyduration 2
1.135 1.078 9.737
1.135 1.086 9.834
1.135 1.090 9.890
1.135 1.091 9.892
1.135 1.111 9.894
1.135 1.119 9.896
1.135 1.120 9.918
1.135 1.123 9.927
1.135 1.130 9.937
1.135 1.132 9.940
1.135 1.138 9.942
1.135 1.140 9.944
1.135 1.141 9.952
1.135 1.148 9.952
1.135 1.153 9.958
1.135 1.153 9.967
1.135 1.167 9.974
1.135 1.168 9.975
1.135 1.173 9.982
1.135 1.199 10.014Patched
./run-proc-vs-map.sh --nsamples 20 --rawdata -- --busyduration 2
0.007 0.006 1.862
0.007 0.006 1.937
0.007 0.006 1.947
0.007 0.007 1.956
0.007 0.007 1.956
0.007 0.007 1.998
0.007 0.007 2.020
0.007 0.007 2.027
0.007 0.007 2.085
0.007 0.007 2.100
0.007 0.007 2.115
0.007 0.007 2.233
0.007 0.007 2.304
0.007 0.007 2.325
0.007 0.007 2.327
0.007 0.007 2.427
0.007 0.007 2.489
0.007 0.007 2.500
0.007 0.007 3.123
0.007 0.007 3.151Quoted 2 lines
Cheers, Lorenzo