powerpc/mm/radix: Update pte update sequence for pte clear case
PATCH 1/3 powerpc/mm/radix: Update pte update sequence for pte clear case
In the kernel we do follow the below sequence in different code paths. pte = ptep_get_clear(ptep)
....set_pte_at(ptep, pte)
We do that for mremap, autonuma protection update and softdirty clearing. This implies our optimization to skip a tlb flush when clearing a pte update is not valid, because for DD1 system that followup set_pte_at will be done witout doing the required tlbflush. Fix that by always doing the dd1 style pte update irrespective of new_pte value. In a later patch we will optimize the application exit case.
--- arch/powerpc/include/asm/book3s/64/radix.h | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-)
arch/powerpc/include/asm/book3s/64/radix.h +3 -9
@@ -144,16 +144,10 @@ static inline unsigned long radix__pte_update(struct mm_struct *mm,* new value of pte */ new_pte = (old_pte | set) & ~clr; -/* -* If we are trying to clear the pte, we can skip -* the below sequence and batch the tlb flush. The -* tlb flush batching is done by mmu gather code -*/ -if (new_pte) { - asmvolatile ("ptesync" : : :"memory" ); -radix__flush_tlb_pte_p9_dd1 (old_pte, mm, addr); + asmvolatile ("ptesync" : : :"memory" ); +radix__flush_tlb_pte_p9_dd1 (old_pte, mm, addr); +if (new_pte)__radix_pte_update (ptep,0 , new_pte); - } }else old_pte =__radix_pte_update (ptep, clr, set); asmvolatile ("ptesync" : : :"memory" );
Signature · 1 line
2.7.4
PATCH 2/3 powerpc/mm/radix: Use ptep_get_and_clear_full when clearing pte for full mm
This helps us to do some optimization for application exit case, where we can skip the DD1 style pte update sequence.
--- arch/powerpc/include/asm/book3s/64/pgtable.h | 17 +++++++++++++++++ arch/powerpc/include/asm/book3s/64/radix.h | 23 ++++++++++++++++++++++- 2 files changed, 39 insertions(+), 1 deletion(-)
arch/powerpc/include/asm/book3s/64/pgtable.h +17 -0
@@ -373,6 +373,23 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm,return __pte (old); } +#define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL +static inline pte_t ptep_get_and_clear_full (struct mm_struct *mm, +unsigned long addr, +pte_t *ptep,int full) +{ +if (full &&radix_enabled ()) { +/* +* Let's skip the DD1 style pte update here. We know that +* this is a full mm pte clear and hence can be sure there is +* no parallel set_pte. +*/ +return radix__ptep_get_and_clear_full (mm, addr, ptep, full); + } +return ptep_get_and_clear (mm, addr, ptep); +} + +static inline void pte_clear (struct mm_struct *mm,unsigned long addr,pte_t * ptep) {
arch/powerpc/include/asm/book3s/64/radix.h +22 -1
@@ -139,7 +139,7 @@ static inline unsigned long radix__pte_update(struct mm_struct *mm,unsigned long new_pte; - old_pte =__radix_pte_update (ptep, ~0 ,0 ); + old_pte =__radix_pte_update (ptep, ~0ul ,0 );/* * new value of pte */
@@ -157,6 +157,27 @@ static inline unsigned long radix__pte_update(struct mm_struct *mm,return old_pte; } +static inline pte_t radix__ptep_get_and_clear_full (struct mm_struct *mm, +unsigned long addr, +pte_t *ptep,int full) +{ +unsigned long old_pte; + +if (full) { +/* +* If we are trying to clear the pte, we can skip +* the DD1 pte update sequence and batch the tlb flush. The +* tlb flush batching is done by mmu gather code. We +* still keep the cmp_xchg update to make sure we get +* correct R/C bit which might be updated via Nest MMU. +*/ + old_pte =__radix_pte_update (ptep, ~0ul ,0 ); + }else + old_pte =radix__pte_update (mm, addr, ptep, ~0ul ,0 ,0 ); + +return __pte (old_pte); +} +/* * Set the dirty and/or accessed bits atomically in a linux PTE, this * function doesn't need to invalidate tlb.
Signature · 1 line
2.7.4
Re: [PATCH 2/3] powerpc/mm/radix: Use ptep_get_and_clear_full when clearing pte for full mm
Quoted 1 line · Aneesh Kumar K.V
This helps us to do some optimization for application exit case, where we=
can
Quoted 3 lines
skip the DD1 style pte update sequence.
=20Signed-off-by Aneesh Kumar K.V <[email protected]>
Quoted 3 lines
---
=C2=A0arch/powerpc/include/asm/book3s/64/pgtable.h | 17 +++++++++++++++++ =C2=A0arch/powerpc/include/asm/book3s/64/radix.h=C2=A0=C2=A0=C2=A0| 23 ++=
++++++++++++++++++++-
arch/powerpc/include/asm/book3s/64/pgtable.h static inline pte_t ptep_get_and_clear(struct mm_str=
Quoted 8 lines
=C2=A02 files changed, 39 insertions(+), 1 deletion(-) =20
arch/powerpc/include/asm/book3s/64/pgtable.h +0
-0@@ -373,6 +373,23 @@ static inline pte_t ptep_get_and_clear(struct mm_str=
uct
Quoted 21 lines
arch/powerpc/include/asm/book3s/64/pgtable.h +0
-0*mm, =C2=A0 return __pte(old); =C2=A0}=C2=A0
arch/powerpc/include/asm/book3s/64/pgtable.h +17
-0+#define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL +static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, + =C2=A0=C2=A0=C2=A0=C2=A0unsigned long addr, + =C2=A0=C2=A0=C2=A0=C2=A0pte_t *ptep, int full) +{ + if (full && radix_enabled()) { + /* + =C2=A0* Let's skip the DD1 style pte update here. We know that + =C2=A0* this is a full mm pte clear and hence can be sure there is + =C2=A0* no parallel set_pte. + =C2=A0*/ + return radix__ptep_get_and_clear_full(mm, addr, ptep, full); + } +return ptep_get_and_clear (mm, addr, ptep); +} + +
Quoted 1 line
=C2=A0static inline void pte_clear(struct
mm_struct*mm, unsigned long ad=
dr,
Quoted 43 lines
=C2=A0 =C2=A0=C2=A0=C2=A0=C2=A0=C2=A0pte_t * ptep)
=C2=A0{arch/powerpc/include/asm/book3s/64/radix.h +0
-0@@ -139,7 +139,7 @@ static inline unsigned long radix__pte_update(struct mm_struct *mm,=C2=A0
=C2=A0 unsigned longnew_pte;
=C2=A0arch/powerpc/include/asm/book3s/64/radix.h +1
-1- old_pte =3D __radix_pte_update(ptep, ~0, 0); + old_pte =3D __radix_pte_update(ptep, ~0ul, 0);=C2=A0 /*
=C2=A0 =C2=A0* new value of pte
=C2=A0 =C2=A0*/arch/powerpc/include/asm/book3s/64/radix.h +0
-0@@ -157,6 +157,27 @@ static inline unsigned long radix__pte_update(structmm_struct *mm, =C2=A0 return old_pte; =C2=A0}=C2=A0
arch/powerpc/include/asm/book3s/64/radix.h +21
-0+static inline pte_t radix__ptep_get_and_clear_full(struct mm_struct *mm, + =C2=A0=C2=A0=C2=A0unsigned long addr, + =C2=A0=C2=A0=C2=A0pte_t *ptep, int full) +{ + unsigned long old_pte; + +if (full) { +/* +=C2=A0* If we are trying to clear the pte, we can skip +=C2=A0* the DD1 pte update sequence and batch the tlb flush. The +=C2=A0* tlb flush batching is done by mmu gather code. We +=C2=A0* still keep the cmp_xchg update to make sure we get +=C2=A0* correct R/C bit which might be updated via Nest MMU. +=C2=A0*/ + old_pte =3D __radix_pte_update(ptep, ~0ul, 0); + }else + old_pte =3D radix__pte_update(mm, addr, ptep, ~0ul, 0, 0); + +return __pte (old_pte); +} +
Quoted 2 lines
=C2=A0/*
=C2=A0 * Set the dirty and/or accessed bits atomically in a linux PTE, th=
is
Quoted 1 line
=C2=A0 * function doesn't need to invalidate tlb.
PATCH 3/3 powerpc/mm/radix: Skip ptesync in pte update helpers
In reply to Aneesh Kumar K.VWe do them at the start of tlb flush, and we are sure a pte update will be followed by a tlbflush. Hence we can skip the ptesync in pte update helpers.
--- arch/powerpc/include/asm/book3s/64/radix.h | 3 --- 1 file changed, 3 deletions(-)
arch/powerpc/include/asm/book3s/64/radix.h +0 -3
@@ -144,13 +144,11 @@ static inline unsigned long radix__pte_update(struct mm_struct *mm,* new value of pte */ new_pte = (old_pte | set) & ~clr; - asmvolatile ("ptesync" : : :"memory" );radix__flush_tlb_pte_p9_dd1 (old_pte, mm, addr);if (new_pte)__radix_pte_update (ptep,0 , new_pte); }else old_pte =__radix_pte_update (ptep, clr, set); - asmvolatile ("ptesync" : : :"memory" );if (!huge)assert_pte_locked (mm, addr);
@@ -195,7 +193,6 @@ static inline void radix__ptep_set_access_flags(struct mm_struct *mm,unsigned long old_pte, new_pte; old_pte =__radix_pte_update (ptep, ~0 ,0 ); - asmvolatile ("ptesync" : : :"memory" );/* * new value of pte */
Signature · 1 line
2.7.4
Re: [PATCH 3/3] powerpc/mm/radix: Skip ptesync in pte update helpers
Quoted 1 line · Aneesh Kumar K.V
We do them at the start of tlb flush, and we are sure a pte update will b=
e
Quoted 1 line
followed by a tlbflush. Hence we can skip the ptesync in pte update helpe=
rs.
Quoted 2 lines
=20
Signed-off-by Aneesh Kumar K.V <[email protected]>
arch/powerpc/include/asm/book3s/64/radix.h static inline unsigned long radix__pte_update(struc=
Quoted 10 lines
---
=C2=A0arch/powerpc/include/asm/book3s/64/radix.h | 3 --- =C2=A01 file changed, 3 deletions(-)
=20arch/powerpc/include/asm/book3s/64/radix.h +0
-0@@ -144,13 +144,11 @@ static inline unsigned long radix__pte_update(struc=
t
arch/powerpc/include/asm/book3s/64/radix.h static inline void radix__ptep_set_access_flags(struc=
Quoted 15 lines
arch/powerpc/include/asm/book3s/64/radix.h +0
-0mm_struct *mm,=C2=A0 =C2=A0* new value of pte
=C2=A0 =C2=A0*/
=C2=A0new_pte=3D (old_pte| set) & ~clr;arch/powerpc/include/asm/book3s/64/radix.h +0
-1- asmvolatile ("ptesync" : : :"memory" );=C2=A0
radix__flush_tlb_pte_p9_dd1(old_pte, mm, addr);
=C2=A0 if (new_pte)
=C2=A0__radix_pte_update(ptep, 0, new_pte);
=C2=A0 } else
=C2=A0old_pte=3D__radix_pte_update(ptep, clr, set);arch/powerpc/include/asm/book3s/64/radix.h +0
-1- asmvolatile ("ptesync" : : :"memory" );=C2=A0 if (!huge)
=C2=A0assert_pte_locked(mm, addr);
=C2=A0arch/powerpc/include/asm/book3s/64/radix.h +0
-0@@ -195,7 +193,6 @@ static inline void radix__ptep_set_access_flags(struc=
t
Quoted 8 lines
arch/powerpc/include/asm/book3s/64/radix.h +0
-0mm_struct *mm, =C2=A0 unsigned long old_pte, new_pte;=C2=A0
=C2=A0old_pte=3D__radix_pte_update(ptep, ~0, 0);arch/powerpc/include/asm/book3s/64/radix.h +0
-1- asmvolatile ("ptesync" : : :"memory" );=C2=A0 /*
=C2=A0 =C2=A0* new value of pte
=C2=A0 =C2=A0*/
Quoted 11 lines · Aneesh Kumar K.V
In the kernel we do follow the below sequence in different code paths. pte =
ptep_get_clear(ptep)
....set_pte_at(ptep, pte)We do that for mremap, autonuma protection update and softdirty clearing. This implies our optimization to skip a tlb flush when clearing a pte update is not valid, because for DD1 system that followup
set_pte_atwill be done witout doing the required tlbflush. Fix that by always doing the dd1 style pte update irrespective ofnew_ptevalue. In a later patch we will optimize the application exit case.
What about my change to set_pte_at() ? We seem to be overwriting valid PTEs, shouldn't we deal with that ?
Cheers,
Ben.
Quoted 30 lines
Signed-off-by Benjamin Herrenschmidt <[email protected]>Quoted 1 line
Signed-off-by Aneesh Kumar K.V <[email protected]>---
arch/powerpc/include/asm/book3s/64/radix.h | 12 +++---------1 file changed, 3 insertions(+), 9 deletions(-)arch/powerpc/include/asm/book3s/64/radix.h +0
-0@@ -144,16 +144,10 @@ static inline unsigned long radix__pte_update(struct mm_struct *mm,Quoted 19 lines
* new value of pte
*/
new_pte= (old_pte| set) & ~clr;arch/powerpc/include/asm/book3s/64/radix.h +3
-8-/* -* If we are trying to clear the pte, we can skip -* the below sequence and batch the tlb flush. The -* tlb flush batching is done by mmu gather code -*/ -if (new_pte) { - asmvolatile ("ptesync" : : :"memory" ); -radix__flush_tlb_pte_p9_dd1 (old_pte, mm, addr); + asmvolatile ("ptesync" : : :"memory" ); +radix__flush_tlb_pte_p9_dd1 (old_pte, mm, addr); +if (new_pte)
__radix_pte_update(ptep, 0, new_pte);arch/powerpc/include/asm/book3s/64/radix.h +0
-1- }} else
old_pte=__radix_pte_update(ptep, clr, set); asmvolatile("ptesync" : : : "memory");
Quoted 19 lines · Benjamin Herrenschmidt
Quoted 6 lines · Aneesh Kumar K.V
In the kernel we do follow the below sequence in different code paths.
pte =ptep_get_clear(ptep)
....set_pte_at(ptep, pte)Quoted 11 lines
We do that for mremap, autonuma protection update and softdirty clearing. This
implies our optimization to skip a tlb flush when clearing a pte update is
not valid, because for DD1 system that followupset_pte_atwill be done witout
doing the required tlbflush. Fix that by always doing the dd1 style pte update
irrespective ofnew_ptevalue. In a later patch we will optimize the application
exit case.
Quoted 3 lines
What about my change to
set_pte_at()? We seem to be overwriting valid PTEs,
shouldn't we deal with that ?
So the HW guys confirmed that the TLB will never cache a valid entry that has all permissions clear. That leaves the THP write problem though.
Cheers,
Ben.
Quoted 39 lines
Cheers,
Ben.Quoted 36 lines
Signed-off-by Benjamin Herrenschmidt <[email protected]>Quoted 1 line
Signed-off-by Aneesh Kumar K.V <[email protected]>---
arch/powerpc/include/asm/book3s/64/radix.h | 12 +++---------1 file changed, 3 insertions(+), 9 deletions(-)arch/powerpc/include/asm/book3s/64/radix.h +0
-0@@ -144,16 +144,10 @@ static inline unsigned long radix__pte_update(struct mm_struct *mm,Quoted 22 lines
* new value of pte
*/
new_pte= (old_pte| set) & ~clr;(quoted diff) +0
-3- /* - * If we are trying to clear the pte, we can - * the below sequence and batch the tlb flush.The
arch/powerpc/include/asm/book3s/64/radix.h +0
-5-* tlb flush batching is done by mmu gather code -*/ -if (new_pte) { - asmvolatile ("ptesync" : : :"memory" ); - radix__flush_tlb_pte_p9_dd1(old_pte, mm,addr);
arch/powerpc/include/asm/book3s/64/radix.h +3
-0+ asmvolatile ("ptesync" : : :"memory" ); +radix__flush_tlb_pte_p9_dd1 (old_pte, mm, addr); +if (new_pte)
__radix_pte_update(ptep, 0, new_pte);arch/powerpc/include/asm/book3s/64/radix.h +0
-1- }} else
old_pte=__radix_pte_update(ptep, clr, set); asmvolatile("ptesync" : : : "memory");
Quoted 20 lines · Benjamin Herrenschmidt
Quoted 19 lines · Benjamin Herrenschmidt
Quoted 17 lines · Aneesh Kumar K.V
In the kernel we do follow the below sequence in different code paths.
pte =ptep_get_clear(ptep)
....set_pte_at(ptep, pte)We do that for mremap, autonuma protection update and softdirty clearing. This
implies our optimization to skip a tlb flush when clearing a pte update is
not valid, because for DD1 system that followupset_pte_atwill be done witout
doing the required tlbflush. Fix that by always doing the dd1 style pte update
irrespective ofnew_ptevalue. In a later patch we will optimize the application
exit case.
Quoted 7 lines
Quoted 3 lines
What about my change to
set_pte_at()? We seem to be overwriting valid PTEs,
shouldn't we deal with that ?So the HW guys confirmed that the TLB will never cache a valid entry that has all permissions clear. That leaves the THP write problem though.
Which is fixed by the autonuma related changes posted at
https://lkml.kernel.org/r/[email protected]
Right now i am running a kernel compile in loop with parallel perf bench numa mem run to make sure we got most of the details correct. (this is on p8)
-aneesh
Quoted 6 lines · Aneesh Kumar K.V
In the kernel we do follow the below sequence in different code paths. pte =3D
ptep_get_clear(ptep)
....set_pte_at(ptep, pte)
=20
We do that for mremap, autonuma protection update and softdirty clearing.=
This
Quoted 1 line
implies our optimization to skip a tlb flush when clearing a pte update i=
s
Quoted 1 line
not valid, because for DD1 system that followup
set_pte_atwill be done w=
itout
Quoted 1 line
doing the required tlbflush. Fix that by always doing the dd1 style pte u=
pdate
Quoted 6 lines
irrespective of
new_ptevalue. In a later patch we will optimize the application
exit case.
=20Signed-off-by Benjamin Herrenschmidt <[email protected]>Signed-off-by Aneesh Kumar K.V <[email protected]>
arch/powerpc/include/asm/book3s/64/radix.h static inline unsigned long radix__pte_update(struc=
Quoted 10 lines
---
=C2=A0arch/powerpc/include/asm/book3s/64/radix.h | 12 +++--------- =C2=A01 file changed, 3 insertions(+), 9 deletions(-) =20arch/powerpc/include/asm/book3s/64/radix.h +0
-0@@ -144,16 +144,10 @@ static inline unsigned long radix__pte_update(struc=
t
Quoted 20 lines
arch/powerpc/include/asm/book3s/64/radix.h +0
-0mm_struct *mm,=C2=A0 =C2=A0* new value of pte
=C2=A0 =C2=A0*/
=C2=A0new_pte=3D (old_pte| set) & ~clr;arch/powerpc/include/asm/book3s/64/radix.h +3
-8-/* -=C2=A0* If we are trying to clear the pte, we can skip -=C2=A0* the below sequence and batch the tlb flush. The -=C2=A0* tlb flush batching is done by mmu gather code -=C2=A0*/ -if (new_pte) { - asmvolatile ("ptesync" : : :"memory" ); -radix__flush_tlb_pte_p9_dd1 (old_pte, mm, addr); + asmvolatile ("ptesync" : : :"memory" ); +radix__flush_tlb_pte_p9_dd1 (old_pte, mm, addr); +if (new_pte)=C2=A0
__radix_pte_update(ptep, 0, new_pte);arch/powerpc/include/asm/book3s/64/radix.h +0
-1- }=C2=A0 } else
=C2=A0old_pte=3D__radix_pte_update(ptep, clr, set);
=C2=A0 asmvolatile("ptesync" : : : "memory");
Quoted 12 lines · Aneesh Kumar K.V
In the kernel we do follow the below sequence in different code paths. pte =
ptep_get_clear(ptep)
....set_pte_at(ptep, pte)We do that for mremap, autonuma protection update and softdirty clearing. This implies our optimization to skip a tlb flush when clearing a pte update is not valid, because for DD1 system that followup
set_pte_atwill be done witout doing the required tlbflush. Fix that by always doing the dd1 style pte update irrespective ofnew_ptevalue. In a later patch we will optimize the application exit case.
Quoted 3 lines
Signed-off-by Benjamin Herrenschmidt <[email protected]>Signed-off-by Aneesh Kumar K.V <[email protected]>Tested-by Michael Neuling <[email protected]>
Series applied to powerpc next, thanks.
https://git.kernel.org/powerpc/c/ca94573b9c69d224e50e1084a27767
cheers