Clean up i386-PAE
Subsystems (10)
Peter Zijlstra
maintainercover Clean up i386-PAE
Hi!
While reviewing some other patches [1], Willy pointed out that lockless page-table walkers should probably be using the same magic for PMD as we do for PTE on things like i386-PAE.
These patches are the result of that and apply on top of that earlier series.
Peter Zijlstra
maintainerRFC PATCH 1/9 mm: Update ptep_get_lockless()s comment
Improve the comment and add a lockdep assertion because nobody reads comments.
--- include/linux/pgtable.h | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-)
include/linux/pgtable.h +8 -9
@@ -260,15 +260,12 @@ static inline pte_t ptep_get(pte_t *ptep#ifdef CONFIG_GUP_GET_PTE_LOW_HIGH /* -* WARNING: only to be used in the get_user_pages_fast() implementation. -* -* With get_user_pages_fast(), we walk down the pagetables without taking any -* locks. For this we would like to load the pointers atomically, but sometimes -* that is not possible (e.g. without expensive cmpxchg8b on x86_32 PAE). What -* we do have is the guarantee that a PTE will only either go from not present -* to present, or present to not present or both -- it will not switch to a -* completely different present page without a TLB flush in between; something -* that we are blocking by holding interrupts off. +* For walking the pagetables without holding any locks. Some architectures +* (eg x86-32 PAE) cannot load the entries atomically without using expensive +* instructions. We are guaranteed that a PTE will only either go from not +* present to present, or present to not present -- it will not switch to a +* completely different present page without a TLB flush inbetween; which we +* are blocking by holding interrupts off. * * Setting ptes from not present to present goes: *
@@ -294,6 +291,8 @@ static inline pte_t ptep_get_lockless(pt {pte_t pte; +lockdep_assert_irqs_disabled (); + do { pte.pte_low = ptep->pte_low; smp_rmb();
Peter Zijlstra
maintainerRFC PATCH 2/9 x86/mm/pae: Make pmd_t similar to pte_t
In reply to Peter ZijlstraInstead of mucking about with at least 2 different ways of fudging it, do the same thing we do for pte_t.
--- arch/x86/include/asm/pgtable-3level.h | 42 +++++++++------------------- arch/x86/include/asm/pgtable-3level_types.h | 7 ++++ arch/x86/include/asm/pgtable_64_types.h | 1 arch/x86/include/asm/pgtable_types.h | 4 -- 4 files changed, 23 insertions(+), 31 deletions(-)
arch/x86/include/asm/pgtable-3level.h +14 -28
@@ -87,7 +87,7 @@ static inline pmd_t pmd_read_atomic(pmd_ ret |= ((pmdval_t)*(tmp +1 )) <<32 ; } -return (pmd_t ) { ret }; +return (pmd_t ) { .pmd = ret }; }static inline void native_set_pte_atomic (pte_t *ptep,pte_t pte)
@@ -121,12 +121,11 @@ static inline void native_pte_clear(stru ptep->pte_high =0 ; } -static inline void native_pmd_clear (pmd_t *pmd) +static inline void native_pmd_clear (pmd_t *pmdp) { -u32 *tmp = (u32 *)pmd; - *tmp =0 ; + pmdp->pmd_low =0 ;smp_wmb (); - *(tmp +1 ) =0 ; + pmdp->pmd_high =0 ; }static inline void native_pud_clear (pud_t *pudp)
@@ -162,25 +161,17 @@ static inline pte_t native_ptep_get_and_#define native_ptep_get_and_clear (xp) native_local_ptep_get_and_clear(xp)#endif -union split_pmd { -struct { -u32 pmd_low; -u32 pmd_high; - }; -pmd_t pmd; -}; -#ifdef CONFIG_SMP static inline pmd_t native_pmdp_get_and_clear (pmd_t *pmdp) { -union split_pmd res, *orig = (union split_pmd *)pmdp; +pmd_t res;/* xchg acts as a barrier before setting of the high bits */ - res.pmd_low =xchg (&orig->pmd_low,0 ); - res.pmd_high = orig->pmd_high; - orig->pmd_high =0 ; + res.pmd_low =xchg (&pmdp->pmd_low,0 ); + res.pmd_high =READ_ONCE (pmdp->pmd_high); +WRITE_ONCE (pmdp->pmd_high,0 ); -return res.pmd; +return res; }#else #define native_pmdp_get_and_clear (xp) native_local_pmdp_get_and_clear(xp)
@@ -199,17 +190,12 @@ static inline pmd_t pmdp_establish(struc* anybody. */ if (!(pmd_val (pmd) & _PAGE_PRESENT)) { -union split_pmd old, new, *ptr; - - ptr = (union split_pmd *)pmdp; - - new.pmd = pmd; -/* xchg acts as a barrier before setting of the high bits */ - old.pmd_low =xchg (&ptr->pmd_low, new.pmd_low); - old.pmd_high = ptr->pmd_high; - ptr->pmd_high = new.pmd_high; -return old.pmd; + old.pmd_low =xchg (&pmdp->pmd_low, pmd.pmd_low); + old.pmd_high =READ_ONCE (pmdp->pmd_high); +WRITE_ONCE (pmdp->pmd_high, pmd.pmd_high); + +return old; } do {
arch/x86/include/asm/pgtable-3level_types.h +7 -0
@@ -18,6 +18,13 @@ typedef union { };pteval_t pte; } pte_t; + +typedef union { +struct { +unsigned long pmd_low, pmd_high; + }; +pmdval_t pmd; +}pmd_t ;#endif /* !__ASSEMBLY__ */ #define SHARED_KERNEL_PMD (!static_cpu_has(X86_FEATURE_PTI))
arch/x86/include/asm/pgtable_64_types.h +1 -0
@@ -19,6 +19,7 @@ typedef unsigned long pgdval_t;typedef unsigned long pgprotval_t ;typedef struct {pteval_t pte; }pte_t ; +typedef struct {pmdval_t pmd; }pmd_t ;#ifdef CONFIG_X86_5LEVEL extern unsigned int __pgtable_l5_enabled;
arch/x86/include/asm/pgtable_types.h +1 -3
@@ -381,11 +381,9 @@ static inline pudval_t native_pud_val(pu#endif #if CONFIG_PGTABLE_LEVELS >2 -typedef struct {pmdval_t pmd; }pmd_t ; -static inline pmd_t native_make_pmd (pmdval_t val) { -return (pmd_t ) { val }; +return (pmd_t ) { .pmd = val }; }static inline pmdval_t native_pmd_val (pmd_t pmd)
Peter Zijlstra
maintainerRFC PATCH 3/9 sh/mm: Make pmd_t similar to pte_t
In reply to Peter ZijlstraJust like 64bit pte_t, have a low/high split in pmd_t.
--- arch/sh/include/asm/pgtable-3level.h | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-)
arch/sh/include/asm/pgtable-3level.h +8 -2
@@ -28,9 +28,15 @@#define pmd_ERROR (e) \ printk("%s:%d: bad pmd %016llx.\n", __FILE__, __LINE__, pmd_val(e)) -typedef struct {unsigned long long pmd; }pmd_t ; +typedef struct { +struct { +unsigned long pmd_low; +unsigned long pmd_high; + }; +unsigned long long pmd; +}pmd_t ;#define pmd_val (x) ((x).pmd) -#define __pmd (x) ((pmd_t) { (x) } ) +#define __pmd (x) ((pmd_t) { .pmd = (x) } )static inline unsigned long pud_page_vaddr (pud_t pud) {
RE: [RFC][PATCH 3/9] sh/mm: Make pmd_t similar to pte_t
From: Peter Zijlstra
Quoted 13 lines
Sent: 30 November 2020 11:27
Just like 64bit
pte_t, have a low/high split inpmd_t.Signed-off-by Peter Zijlstra (Intel) <[email protected]>--- arch/sh/include/asm/pgtable-3level.h | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-)arch/sh/include/asm/pgtable-3level.h +0
-0@@ -28,9 +28,15 @@#define pmd_ERROR (e) \
arch/sh/include/asm/pgtable-3level.h line 37
Quoted 10 lines
arch/sh/include/asm/pgtable-3level.h +7
-1printk("%s:%d: bad pmd %016llx.\n", __FILE__, __LINE__, pmd_val(e)) -typedef struct {unsigned long long pmd; }pmd_t ; +typedef struct { +struct { +unsigned long pmd_low; +unsigned long pmd_high; + }; +unsigned long long pmd; +}pmd_t ;
Would it be better to use u32 and u64?
David
-
Registered Address Lakeside, Bramley Road, Mount Farm, Milton Keynes, MK1 1PT, UK Registration No: 1397386 (Wales)
Peter Zijlstra
maintainerRe: [RFC][PATCH 3/9] sh/mm: Make pmd_t similar to pte_t
Quoted 16 lines · David Laight
From: Peter Zijlstra
Quoted 15 lines
Sent: 30 November 2020 11:27
Just like 64bit
pte_t, have a low/high split inpmd_t.Signed-off-by Peter Zijlstra (Intel) <[email protected]>--- arch/sh/include/asm/pgtable-3level.h | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-)arch/sh/include/asm/pgtable-3level.h +0
-0@@ -28,9 +28,15 @@#define pmd_ERROR (e) \ printk("%s:%d: bad pmd %016llx.\n", __FILE__, __LINE__, pmd_val(e))
Quoted 10 lines
arch/sh/include/asm/pgtable-3level.h line 37
Quoted 8 lines
arch/sh/include/asm/pgtable-3level.h +7
-1-typedef struct {unsigned long long pmd; }pmd_t ; +typedef struct { +struct { +unsigned long pmd_low; +unsigned long pmd_high; + }; +unsigned long long pmd; +}pmd_t ;Would it be better to use u32 and u64?
That would be inconsistent with the rest of SH. If you want to go clean up SH, have at, but that's not what this series is for.
Peter Zijlstra
maintainerRFC PATCH 4/9 mm: Fix pmd_read_atomic()
In reply to Peter ZijlstraAFAICT there's no reason to do anything different than what we do for PTEs. Make it so (also affects SH).
--- arch/x86/include/asm/pgtable-3level.h | 56 ---------------------------------- include/linux/pgtable.h | 49 +++++++++++++++++++++++------ 2 files changed, 39 insertions(+), 66 deletions(-)
arch/x86/include/asm/pgtable-3level.h +0 -56
@@ -34,62 +34,6 @@ static inline void native_set_pte(pte_t ptep->pte_low = pte.pte_low; } -#define pmd_read_atomic pmd_read_atomic -/* -* pte_offset_map_lock() on 32-bit PAE kernels was reading the pmd_t with -* a "*pmdp" dereference done by GCC. Problem is, in certain places -* where pte_offset_map_lock() is called, concurrent page faults are -* allowed, if the mmap_lock is hold for reading. An example is mincore -* vs page faults vs MADV_DONTNEED. On the page fault side -* pmd_populate() rightfully does a set_64bit(), but if we're reading the -* pmd_t with a "*pmdp" on the mincore side, a SMP race can happen -* because GCC will not read the 64-bit value of the pmd atomically. -* -* To fix this all places running pte_offset_map_lock() while holding the -* mmap_lock in read mode, shall read the pmdp pointer using this -* function to know if the pmd is null or not, and in turn to know if -* they can run pte_offset_map_lock() or pmd_trans_huge() or other pmd -* operations. -* -* Without THP if the mmap_lock is held for reading, the pmd can only -* transition from null to not null while pmd_read_atomic() runs. So -* we can always return atomic pmd values with this function. -* -* With THP if the mmap_lock is held for reading, the pmd can become -* trans_huge or none or point to a pte (and in turn become "stable") -* at any time under pmd_read_atomic(). We could read it truly -* atomically here with an atomic64_read() for the THP enabled case (and -* it would be a whole lot simpler), but to avoid using cmpxchg8b we -* only return an atomic pmdval if the low part of the pmdval is later -* found to be stable (i.e. pointing to a pte). We are also returning a -* 'none' (zero) pmdval if the low part of the pmd is zero. -* -* In some cases the high and low part of the pmdval returned may not be -* consistent if THP is enabled (the low part may point to previously -* mapped hugepage, while the high part may point to a more recently -* mapped hugepage), but pmd_none_or_trans_huge_or_clear_bad() only -* needs the low part of the pmd to be read atomically to decide if the -* pmd is unstable or not, with the only exception when the low part -* of the pmd is zero, in which case we return a 'none' pmd. -*/ -static inline pmd_t pmd_read_atomic (pmd_t *pmdp) -{ -pmdval_t ret; -u32 *tmp = (u32 *)pmdp; - - ret = (pmdval_t) (*tmp); -if (ret) { -/* -* If the low part is null, we must not read the high part -* or we can end up with a partial pmd. -*/ -smp_rmb (); - ret |= ((pmdval_t)*(tmp +1 )) <<32 ; - } - -return (pmd_t ) { .pmd = ret }; -} -static inline void native_set_pte_atomic (pte_t *ptep,pte_t pte) {set_64bit ((unsigned long long *)(ptep),native_pte_val (pte));
include/linux/pgtable.h +39 -10
77 lines not on this page · raw message
Peter Zijlstra
maintainerRFC PATCH 5/9 mm: Rename pmd_read_atomic()
In reply to Peter ZijlstraThere's no point in having the identical routines for PTE/PMD have different names.
--- include/linux/pgtable.h | 7 +------ mm/hmm.c | 2 +- mm/mapping_dirty_helpers.c | 2 +- mm/mprotect.c | 2 +- mm/userfaultfd.c | 2 +- 5 files changed, 5 insertions(+), 10 deletions(-)
include/linux/pgtable.h +1 -6
@@ -1244,11 +1244,6 @@ static inline int pud_trans_unstable(pud#endif } -static inline pmd_t pmd_read_atomic (pmd_t *pmdp) -{ -return pmdp_get_lockless (pmdp); -} -#ifndef arch_needs_pgtable_deposit#define arch_needs_pgtable_deposit () (false)#endif
@@ -1275,7 +1270,7 @@ static inline pmd_t pmd_read_atomic(pmd_*/ static inline int pmd_none_or_trans_huge_or_clear_bad (pmd_t *pmd) { -pmd_t pmdval =pmd_read_atomic (pmd); +pmd_t pmdval =pmdp_get_lockless (pmd);/* * The barrier will stabilize the pmdval in a register or on * the stack so that it will stop changing under the code.
mm/hmm.c +1 -1
@@ -356,7 +356,7 @@ static int hmm_vma_walk_pmd(pmd_t *pmdp,* huge or device mapping one and compute corresponding pfn * values. */ - pmd =pmd_read_atomic (pmdp); + pmd =pmdp_get_lockless (pmdp);barrier ();if (!pmd_devmap (pmd) && !pmd_trans_huge (pmd)) goto again;
mm/mapping_dirty_helpers.c +1 -1
@@ -123,7 +123,7 @@ static int clean_record_pte(pte_t *pte,static int wp_clean_pmd_entry (pmd_t *pmd,unsigned long addr,unsigned long end,struct mm_walk *walk) { -pmd_t pmdval =pmd_read_atomic (pmd); +pmd_t pmdval =pmdp_get_lockless (pmd);if (!pmd_trans_unstable (&pmdval))return 0 ;
mm/mprotect.c +1 -1
@@ -190,7 +190,7 @@ static unsigned long change_pte_range(st*/ static inline int pmd_none_or_clear_bad_unless_trans_huge (pmd_t *pmd) { -pmd_t pmdval =pmd_read_atomic (pmd); +pmd_t pmdval =pmdp_get_lockless (pmd);/* See pmd_none_or_trans_huge_or_clear_bad for info on barrier */ #ifdef CONFIG_TRANSPARENT_HUGEPAGE
mm/userfaultfd.c +1 -1
@@ -553,7 +553,7 @@ static __always_inline ssize_t __mcopy_abreak ; } - dst_pmdval =pmd_read_atomic (dst_pmd); + dst_pmdval =pmdp_get_lockless (dst_pmd);/* * If the dst_pmd is mapped as THP don't * override it and just be strict.
Re: [RFC][PATCH 5/9] mm: Rename pmd_read_atomic()
Quoted 35 lines · Peter Zijlstra
There's no point in having the identical routines for PTE/PMD have different names.
Signed-off-by Peter Zijlstra (Intel) <[email protected]>include/linux/pgtable.h | 7 +------ mm/hmm.c | 2 +- mm/mapping_dirty_helpers.c | 2 +- mm/mprotect.c | 2 +- mm/userfaultfd.c | 2 +- 5 files changed, 5 insertions(+), 10 deletions(-)+++
b/include/linux/pgtable.hinclude/linux/pgtable.h +2
-6@@ -1244,11 +1244,6 @@ static inline int pud_trans_unstable(pud#endif } -static inline pmd_t pmd_read_atomic (pmd_t *pmdp) -{ -return pmdp_get_lockless (pmdp); -} -#ifndef arch_needs_pgtable_deposit#define arch_needs_pgtable_deposit () (false)#endif @@ -1275,7 +1270,7 @@ static inline pmd_t pmd_read_atomic(pmd_*/ static inline int pmd_none_or_trans_huge_or_clear_bad (pmd_t *pmd) { -pmd_t pmdval =pmd_read_atomic (pmd); +pmd_t pmdval =pmdp_get_lockless (pmd);/* * The barrier will stabilize the pmdval in a register or on * the stack so that it will stop changing under the code. +++ b/mm/hmm.c
include/linux/pgtable.h static int hmm_vma_walk_pmd(pmd_t *pmdp, · line 362
Quoted 8 lines
include/linux/pgtable.h +1
-1@@ -356,7 +356,7 @@ static int hmm_vma_walk_pmd(pmd_t *pmdp,* huge or device mapping one and compute corresponding pfn * values. */ - pmd =pmd_read_atomic (pmdp); + pmd =pmdp_get_lockless (pmdp);barrier ();if (!pmd_devmap (pmd) && !pmd_trans_huge (pmd)) goto again;
The pagewalk API doesn't call the functions with interrupts disabled, doesn't this mean we hit this assertion?
+#if CONFIG_PGTABLE_LEVELS > 2
+static inline pmd_t pmdp_get_lockless(pmd_t *pmdp) +{
+ pmd_t pmd;
+
+ lockdep_assert_irqs_disabled();
+
It is only holding the read side of the mmap_sem here
Jason
Peter Zijlstra
maintainerRe: [RFC][PATCH 5/9] mm: Rename pmd_read_atomic()
Quoted 56 lines · Jason Gunthorpe
Quoted 35 lines · Peter Zijlstra
There's no point in having the identical routines for PTE/PMD have different names.
Signed-off-by Peter Zijlstra (Intel) <[email protected]>include/linux/pgtable.h | 7 +------ mm/hmm.c | 2 +- mm/mapping_dirty_helpers.c | 2 +- mm/mprotect.c | 2 +- mm/userfaultfd.c | 2 +- 5 files changed, 5 insertions(+), 10 deletions(-)+++
b/include/linux/pgtable.hinclude/linux/pgtable.h +2
-6@@ -1244,11 +1244,6 @@ static inline int pud_trans_unstable(pud#endif } -static inline pmd_t pmd_read_atomic (pmd_t *pmdp) -{ -return pmdp_get_lockless (pmdp); -} -#ifndef arch_needs_pgtable_deposit#define arch_needs_pgtable_deposit () (false)#endif @@ -1275,7 +1270,7 @@ static inline pmd_t pmd_read_atomic(pmd_*/ static inline int pmd_none_or_trans_huge_or_clear_bad (pmd_t *pmd) { -pmd_t pmdval =pmd_read_atomic (pmd); +pmd_t pmdval =pmdp_get_lockless (pmd);/* * The barrier will stabilize the pmdval in a register or on * the stack so that it will stop changing under the code. +++ b/mm/hmm.c include/linux/pgtable.h static int hmm_vma_walk_pmd(pmd_t *pmdp, · line 362
Quoted 8 lines
include/linux/pgtable.h +1
-1@@ -356,7 +356,7 @@ static int hmm_vma_walk_pmd(pmd_t *pmdp,* huge or device mapping one and compute corresponding pfn * values. */ - pmd =pmd_read_atomic (pmdp); + pmd =pmdp_get_lockless (pmdp);barrier ();if (!pmd_devmap (pmd) && !pmd_trans_huge (pmd)) goto again;The pagewalk API doesn't call the functions with interrupts disabled, doesn't this mean we hit this assertion?
(quoted diff) +7
-0+#if CONFIG_PGTABLE_LEVELS > 2 +static inline pmd_t pmdp_get_lockless(pmd_t *pmdp) +{ + pmd_t pmd; + + lockdep_assert_irqs_disabled(); +
Quoted 1 line
It is only holding the read side of the
mmap_semhere
Hurmph, good point. I'll see what I can do about that.
Peter Zijlstra
maintainerRFC PATCH 6/9 mm/gup: Fix the lockless walkers
In reply to Peter ZijlstraOn architectures where the PTE/PMD is larger than the native word size (i386-PAE for example), READ_ONCE() can do the wrong thing. Use pmdp_get_lockless() just like we use ptep_get_lockless().
--- kernel/events/core.c | 2 +- mm/gup.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-)
kernel/events/core.c +1 -1
@@ -7041,7 +7041,7 @@ static u64 perf_get_pgtable_size(structreturn pud_leaf_size (pud); pmdp =pmd_offset_lockless (pudp, pud, addr); - pmd =READ_ONCE (*pmdp); + pmd =pmdp_get_lockless (pmdp);if (!pmd_present (pmd))return 0 ;
mm/gup.c +1 -1
@@ -2467,7 +2467,7 @@ static int gup_pmd_range(pud_t *pudp, pu pmdp =pmd_offset_lockless (pudp, pud, addr); do { - pmd_t pmd = READ_ONCE(*pmdp); + pmd_t pmd = pmdp_get_lockless(pmdp); next = pmd_addr_end(addr, end); if (!pmd_present(pmd))
Peter Zijlstra
maintainerRFC PATCH 7/9 x86/mm/pae: Dont (ab)use atomic64
In reply to Peter ZijlstraPAE implies CX8, write readable code.
--- arch/x86/include/asm/pgtable-3level.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-)
arch/x86/include/asm/pgtable-3level.h +5 -5
@@ -2,8 +2,6 @@#ifndef _ASM_X86_PGTABLE_3LEVEL_H#define _ASM_X86_PGTABLE_3LEVEL_H -#include <asm/atomic64_32.h> -/* * Intel Physical Address Extension (PAE) Mode - three-level page * tables on PPro+ CPUs.
@@ -96,11 +94,13 @@ static inline void pud_clear(pud_t *pudp#ifdef CONFIG_SMP static inline pte_t native_ptep_get_and_clear (pte_t *ptep) { -pte_t res; +pte_t old; - res.pte = (pteval_t )arch_atomic64_xchg ((atomic64_t *)ptep,0 ); +do { + old = *ptep; + }while (cmpxchg64 (&ptep->pte, old.pte,0ULL ) != old.pte); -return res; +return old; }#else #define native_ptep_get_and_clear (xp) native_local_ptep_get_and_clear(xp)
Peter Zijlstra
maintainerRFC PATCH 8/9 x86/mm/pae: Use WRITE_ONCE()
In reply to Peter Zijlstra--- arch/x86/include/asm/pgtable-3level.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-)
arch/x86/include/asm/pgtable-3level.h +5 -5
@@ -27,9 +27,9 @@*/ static inline void native_set_pte (pte_t *ptep,pte_t pte) { - ptep->pte_high = pte.pte_high; +WRITE_ONCE (ptep->pte_high, pte.pte_high);smp_wmb (); - ptep->pte_low = pte.pte_low; +WRITE_ONCE (ptep->pte_low, pte.pte_low); }static inline void native_set_pte_atomic (pte_t *ptep,pte_t pte)
@@ -58,14 +58,14 @@ static inline void native_set_pud(pud_tstatic inline void native_pte_clear (struct mm_struct *mm,unsigned long addr,pte_t *ptep) { - ptep->pte_low =0 ; +WRITE_ONCE (ptep->pte_low,0 );smp_wmb (); - ptep->pte_high =0 ; +WRITE_ONCE (ptep->pte_high,0 ); }static inline void native_pmd_clear (pmd_t *pmdp) { - pmdp->pmd_low =0 ; +WRITE_ONCE (pmdp->pmd_low,0 );smp_wmb (); pmdp->pmd_high =0 ; }
Peter Zijlstra
maintainerRFC PATCH 9/9 x86/mm/pae: Be consistent with pXXp_get_and_clear()
In reply to Peter ZijlstraGiven that ptep_get_and_clear() uses cmpxchg8b, and that should be by far the most common case, there's no point in having an optimized variant for pmd/pud.
--- arch/x86/include/asm/pgtable-3level.h | 34 ++++++++++------------------------ 1 file changed, 10 insertions(+), 24 deletions(-)
arch/x86/include/asm/pgtable-3level.h +10 -24
@@ -108,14 +108,13 @@ static inline pte_t native_ptep_get_and_#ifdef CONFIG_SMP static inline pmd_t native_pmdp_get_and_clear (pmd_t *pmdp) { -pmd_t res; +pmd_t old; -/* xchg acts as a barrier before setting of the high bits */ - res.pmd_low =xchg (&pmdp->pmd_low,0 ); - res.pmd_high =READ_ONCE (pmdp->pmd_high); -WRITE_ONCE (pmdp->pmd_high,0 ); +do { + old = *pmdp; + }while (cmpxchg64 (&pmdp->pmd, old.pmd,0ULL ) != old.pmd); -return res; +return old; }#else #define native_pmdp_get_and_clear (xp) native_local_pmdp_get_and_clear(xp)
@@ -151,28 +150,15 @@ static inline pmd_t pmdp_establish(struc#endif #ifdef CONFIG_SMP -union split_pud { -struct { -u32 pud_low; -u32 pud_high; - }; -pud_t pud; -}; -static inline pud_t native_pudp_get_and_clear (pud_t *pudp) { -union split_pud res, *orig = (union split_pud *)pudp; +pud_t old; -#ifdef CONFIG_PAGE_TABLE_ISOLATION -pti_set_user_pgtbl (&pudp->p4d.pgd,__pgd (0 )); -#endif - -/* xchg acts as a barrier before setting of the high bits */ - res.pud_low =xchg (&orig->pud_low,0 ); - res.pud_high = orig->pud_high; - orig->pud_high =0 ; +do { + old = *pudp; + }while (cmpxchg64 (&pudp->pud, old.pud,0ULL ) != old.pud); -return res.pud; +return old; }#else #define native_pudp_get_and_clear (xp) native_local_pudp_get_and_clear(xp)