Skip to contentSkip to conversation

powerpc/mm/radix: Update pte update sequence for pte clear case

PATCH In review Aneesh Kumar K.V 4 participants 10 messages Feb 9, 2017 – Feb 16, 2017 PowerPC

lore
Share & tools
lore
#0 lore raw

PATCH 1/3 powerpc/mm/radix: Update pte update sequence for pte clear case

In the kernel we do follow the below sequence in different code paths. pte = ptep_get_clear(ptep)
....
set_pte_at(ptep, pte)

We do that for mremap, autonuma protection update and softdirty clearing. This implies our optimization to skip a tlb flush when clearing a pte update is not valid, because for DD1 system that followup set_pte_at will be done witout doing the required tlbflush. Fix that by always doing the dd1 style pte update irrespective of new_pte value. In a later patch we will optimize the application exit case.

---
 arch/powerpc/include/asm/book3s/64/radix.h | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)
arch/powerpc/include/asm/book3s/64/radix.h +3 -9
index b4d1302387a3..70a3cdcdbe47 100644
@@ -144,16 +144,10 @@ static inline unsigned long radix__pte_update(struct mm_struct *mm,
 		 * new value of pte
 		 */
 		new_pte = (old_pte | set) & ~clr;
-		/*
-		 * If we are trying to clear the pte, we can skip
-		 * the below sequence and batch the tlb flush. The
-		 * tlb flush batching is done by mmu gather code
-		 */
-		if (new_pte) {
-			asm volatile("ptesync" : : : "memory");
-			radix__flush_tlb_pte_p9_dd1(old_pte, mm, addr);
+		asm volatile("ptesync" : : : "memory");
+		radix__flush_tlb_pte_p9_dd1(old_pte, mm, addr);
+		if (new_pte)
 			__radix_pte_update(ptep, 0, new_pte);
-		}
 	} else
 		old_pte = __radix_pte_update(ptep, clr, set);
 	asm volatile("ptesync" : : : "memory");
Signature · 1 line
2.7.4
#1 lore raw

PATCH 2/3 powerpc/mm/radix: Use ptep_get_and_clear_full when clearing pte for full mm

This helps us to do some optimization for application exit case, where we can skip the DD1 style pte update sequence.

---
 arch/powerpc/include/asm/book3s/64/pgtable.h | 17 +++++++++++++++++
 arch/powerpc/include/asm/book3s/64/radix.h   | 23 ++++++++++++++++++++++-
 2 files changed, 39 insertions(+), 1 deletion(-)
arch/powerpc/include/asm/book3s/64/pgtable.h +17 -0
index 6f15bde94da2..e91ada786d48 100644
@@ -373,6 +373,23 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
 	return __pte(old);
 }
 
+#define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL
+static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
+					    unsigned long addr,
+					    pte_t *ptep, int full)
+{
+	if (full && radix_enabled()) {
+		/*
+		 * Let's skip the DD1 style pte update here. We know that
+		 * this is a full mm pte clear and hence can be sure there is
+		 * no parallel set_pte.
+		 */
+		return radix__ptep_get_and_clear_full(mm, addr, ptep, full);
+	}
+	return ptep_get_and_clear(mm, addr, ptep);
+}
+
+
 static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
 			     pte_t * ptep)
 {
arch/powerpc/include/asm/book3s/64/radix.h +22 -1
index 70a3cdcdbe47..fcf822d6c204 100644
@@ -139,7 +139,7 @@ static inline unsigned long radix__pte_update(struct mm_struct *mm,
 
 		unsigned long new_pte;
 
-		old_pte = __radix_pte_update(ptep, ~0, 0);
+		old_pte = __radix_pte_update(ptep, ~0ul, 0);
 		/*
 		 * new value of pte
 		 */
@@ -157,6 +157,27 @@ static inline unsigned long radix__pte_update(struct mm_struct *mm,
 	return old_pte;
 }
 
+static inline pte_t radix__ptep_get_and_clear_full(struct mm_struct *mm,
+						   unsigned long addr,
+						   pte_t *ptep, int full)
+{
+	unsigned long old_pte;
+
+	if (full) {
+		/*
+		 * If we are trying to clear the pte, we can skip
+		 * the DD1 pte update sequence and batch the tlb flush. The
+		 * tlb flush batching is done by mmu gather code. We
+		 * still keep the cmp_xchg update to make sure we get
+		 * correct R/C bit which might be updated via Nest MMU.
+		 */
+		old_pte = __radix_pte_update(ptep, ~0ul, 0);
+	} else
+		old_pte = radix__pte_update(mm, addr, ptep, ~0ul, 0, 0);
+
+	return __pte(old_pte);
+}
+
 /*
  * Set the dirty and/or accessed bits atomically in a linux PTE, this
  * function doesn't need to invalidate tlb.
Signature · 1 line
2.7.4
Tested-by on v1 2/3
#2 lore raw

Re: [PATCH 2/3] powerpc/mm/radix: Use ptep_get_and_clear_full when clearing pte for full mm

Quoted 1 line · Aneesh Kumar K.V

This helps us to do some optimization for application exit case, where we=

can

Quoted 3 lines

skip the DD1 style pte update sequence.
=20

Quoted 3 lines

---
=C2=A0arch/powerpc/include/asm/book3s/64/pgtable.h | 17 +++++++++++++++++ =C2=A0arch/powerpc/include/asm/book3s/64/radix.h=C2=A0=C2=A0=C2=A0| 23 ++=

++++++++++++++++++++-

arch/powerpc/include/asm/book3s/64/pgtable.h static inline pte_t ptep_get_and_clear(struct mm_str=

Quoted 8 lines

=C2=A02 files changed, 39 insertions(+), 1 deletion(-) =20

arch/powerpc/include/asm/book3s/64/pgtable.h +0 -0
index 6f15bde94da2..e91ada786d48 100644
@@ -373,6 +373,23 @@ static inline pte_t ptep_get_and_clear(struct mm_str=

uct

Quoted 21 lines
arch/powerpc/include/asm/book3s/64/pgtable.h +0 -0
 *mm,
 =C2=A0	return __pte(old);
 =C2=A0}

=C2=A0

arch/powerpc/include/asm/book3s/64/pgtable.h +17 -0
+#define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL
+static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
+					=C2=A0=C2=A0=C2=A0=C2=A0unsigned long addr,
+					=C2=A0=C2=A0=C2=A0=C2=A0pte_t *ptep, int full)
+{
+	if (full && radix_enabled()) {
+		/*
+		=C2=A0* Let's skip the DD1 style pte update here. We know that
+		=C2=A0* this is a full mm pte clear and hence can be sure there is
+		=C2=A0* no parallel set_pte.
+		=C2=A0*/
+		return radix__ptep_get_and_clear_full(mm, addr, ptep, full);
+	}
+	return ptep_get_and_clear(mm, addr, ptep);
+}
+
+
Quoted 1 line

=C2=A0static inline void pte_clear(struct mm_struct *mm, unsigned long ad=

dr,

Quoted 43 lines

=C2=A0 =C2=A0=C2=A0=C2=A0=C2=A0=C2=A0pte_t * ptep)
=C2=A0{

arch/powerpc/include/asm/book3s/64/radix.h +0 -0
index 70a3cdcdbe47..fcf822d6c204 100644
@@ -139,7 +139,7 @@ static inline unsigned long radix__pte_update(struct
 mm_struct *mm,

=C2=A0
=C2=A0 unsigned long new_pte;
=C2=A0

arch/powerpc/include/asm/book3s/64/radix.h +1 -1
-		old_pte =3D __radix_pte_update(ptep, ~0, 0);
+		old_pte =3D __radix_pte_update(ptep, ~0ul, 0);

=C2=A0 /*
=C2=A0 =C2=A0* new value of pte
=C2=A0 =C2=A0*/

arch/powerpc/include/asm/book3s/64/radix.h +0 -0
@@ -157,6 +157,27 @@ static inline unsigned long radix__pte_update(struct
 mm_struct *mm,
 =C2=A0	return old_pte;
 =C2=A0}

=C2=A0

arch/powerpc/include/asm/book3s/64/radix.h +21 -0
+static inline pte_t radix__ptep_get_and_clear_full(struct mm_struct *mm,
+						=C2=A0=C2=A0=C2=A0unsigned long addr,
+						=C2=A0=C2=A0=C2=A0pte_t *ptep, int full)
+{
+	unsigned long old_pte;
+
+	if (full) {
+		/*
+		=C2=A0* If we are trying to clear the pte, we can skip
+		=C2=A0* the DD1 pte update sequence and batch the tlb flush. The
+		=C2=A0* tlb flush batching is done by mmu gather code. We
+		=C2=A0* still keep the cmp_xchg update to make sure we get
+		=C2=A0* correct R/C bit which might be updated via Nest MMU.
+		=C2=A0*/
+		old_pte =3D __radix_pte_update(ptep, ~0ul, 0);
+	} else
+		old_pte =3D radix__pte_update(mm, addr, ptep, ~0ul, 0, 0);
+
+	return __pte(old_pte);
+}
+
Quoted 2 lines

=C2=A0/*
=C2=A0 * Set the dirty and/or accessed bits atomically in a linux PTE, th=

is

Quoted 1 line

=C2=A0 * function doesn't need to invalidate tlb.

#3 lore raw

PATCH 3/3 powerpc/mm/radix: Skip ptesync in pte update helpers

In reply to Aneesh Kumar K.V

We do them at the start of tlb flush, and we are sure a pte update will be followed by a tlbflush. Hence we can skip the ptesync in pte update helpers.

---
 arch/powerpc/include/asm/book3s/64/radix.h | 3 ---
 1 file changed, 3 deletions(-)
arch/powerpc/include/asm/book3s/64/radix.h +0 -3
index fcf822d6c204..77e590c77299 100644
@@ -144,13 +144,11 @@ static inline unsigned long radix__pte_update(struct mm_struct *mm,
 		 * new value of pte
 		 */
 		new_pte = (old_pte | set) & ~clr;
-		asm volatile("ptesync" : : : "memory");
 		radix__flush_tlb_pte_p9_dd1(old_pte, mm, addr);
 		if (new_pte)
 			__radix_pte_update(ptep, 0, new_pte);
 	} else
 		old_pte = __radix_pte_update(ptep, clr, set);
-	asm volatile("ptesync" : : : "memory");
 	if (!huge)
 		assert_pte_locked(mm, addr);
 
@@ -195,7 +193,6 @@ static inline void radix__ptep_set_access_flags(struct mm_struct *mm,
 		unsigned long old_pte, new_pte;
 
 		old_pte = __radix_pte_update(ptep, ~0, 0);
-		asm volatile("ptesync" : : : "memory");
 		/*
 		 * new value of pte
 		 */
Signature · 1 line
2.7.4
Tested-by on v1 3/3
#4 lore raw

Re: [PATCH 3/3] powerpc/mm/radix: Skip ptesync in pte update helpers

Quoted 1 line · Aneesh Kumar K.V

We do them at the start of tlb flush, and we are sure a pte update will b=

e

Quoted 1 line

followed by a tlbflush. Hence we can skip the ptesync in pte update helpe=

rs.

Quoted 2 lines

=20

arch/powerpc/include/asm/book3s/64/radix.h static inline unsigned long radix__pte_update(struc=

Quoted 10 lines

---
=C2=A0arch/powerpc/include/asm/book3s/64/radix.h | 3 --- =C2=A01 file changed, 3 deletions(-)
=20

arch/powerpc/include/asm/book3s/64/radix.h +0 -0
index fcf822d6c204..77e590c77299 100644
@@ -144,13 +144,11 @@ static inline unsigned long radix__pte_update(struc=

t

arch/powerpc/include/asm/book3s/64/radix.h static inline void radix__ptep_set_access_flags(struc=

Quoted 15 lines
arch/powerpc/include/asm/book3s/64/radix.h +0 -0
 mm_struct *mm,

=C2=A0 =C2=A0* new value of pte
=C2=A0 =C2=A0*/
=C2=A0 new_pte =3D (old_pte | set) & ~clr;

arch/powerpc/include/asm/book3s/64/radix.h +0 -1
-		asm volatile("ptesync" : : : "memory");

=C2=A0 radix__flush_tlb_pte_p9_dd1(old_pte, mm, addr);
=C2=A0 if (new_pte)
=C2=A0 __radix_pte_update(ptep, 0, new_pte);
=C2=A0 } else
=C2=A0 old_pte =3D __radix_pte_update(ptep, clr, set);

arch/powerpc/include/asm/book3s/64/radix.h +0 -1
-	asm volatile("ptesync" : : : "memory");

=C2=A0 if (!huge)
=C2=A0 assert_pte_locked(mm, addr);
=C2=A0

arch/powerpc/include/asm/book3s/64/radix.h +0 -0
@@ -195,7 +193,6 @@ static inline void radix__ptep_set_access_flags(struc=

t

Quoted 8 lines
arch/powerpc/include/asm/book3s/64/radix.h +0 -0
 mm_struct *mm,
 =C2=A0		unsigned long old_pte, new_pte;

=C2=A0
=C2=A0 old_pte =3D __radix_pte_update(ptep, ~0, 0);

arch/powerpc/include/asm/book3s/64/radix.h +0 -1
-		asm volatile("ptesync" : : : "memory");

=C2=A0 /*
=C2=A0 =C2=A0* new value of pte
=C2=A0 =C2=A0*/

#5 lore raw
In reply to Aneesh Kumar K.V
Quoted 11 lines · Aneesh Kumar K.V

In the kernel we do follow the below sequence in different code paths. pte = ptep_get_clear(ptep)
....
set_pte_at(ptep, pte)

We do that for mremap, autonuma protection update and softdirty clearing. This implies our optimization to skip a tlb flush when clearing a pte update is not valid, because for DD1 system that followup set_pte_at will be done witout doing the required tlbflush. Fix that by always doing the dd1 style pte update irrespective of new_pte value. In a later patch we will optimize the application exit case.

What about my change to set_pte_at() ? We seem to be overwriting valid PTEs, shouldn't we deal with that ?

Cheers,
Ben.

Quoted 30 lines
Quoted 1 line

---
 arch/powerpc/include/asm/book3s/64/radix.h | 12 +++---------

 1 file changed, 3 insertions(+), 9 deletions(-)
arch/powerpc/include/asm/book3s/64/radix.h +0 -0
index b4d1302387a3..70a3cdcdbe47 100644
@@ -144,16 +144,10 @@ static inline unsigned long radix__pte_update(struct mm_struct *mm,
Quoted 19 lines

   * new value of pte
   */
  new_pte = (old_pte | set) & ~clr;

arch/powerpc/include/asm/book3s/64/radix.h +3 -8
-		/*
-		 * If we are trying to clear the pte, we can skip
-		 * the below sequence and batch the tlb flush. The
-		 * tlb flush batching is done by mmu gather code
-		 */
-		if (new_pte) {
-			asm volatile("ptesync" : : : "memory");
-			radix__flush_tlb_pte_p9_dd1(old_pte, mm, addr);
+		asm volatile("ptesync" : : : "memory");
+		radix__flush_tlb_pte_p9_dd1(old_pte, mm, addr);
+		if (new_pte)

  __radix_pte_update(ptep, 0, new_pte);

arch/powerpc/include/asm/book3s/64/radix.h +0 -1
-		}

  } else
  old_pte = __radix_pte_update(ptep, clr, set); asm volatile("ptesync" : : : "memory");

#6 lore raw
Quoted 19 lines · Benjamin Herrenschmidt
Quoted 6 lines · Aneesh Kumar K.V

In the kernel we do follow the below sequence in different code paths.
pte = ptep_get_clear(ptep)
....
set_pte_at(ptep, pte)

Quoted 11 lines

We do that for mremap, autonuma protection update and softdirty clearing. This
implies our optimization to skip a tlb flush when clearing a pte update is
not valid, because for DD1 system that followup set_pte_at will be done witout
doing the required tlbflush. Fix that by always doing the dd1 style pte update
irrespective of new_pte value. In a later patch we will optimize the application
exit case.

Quoted 3 lines

What about my change to set_pte_at() ? We seem to be overwriting valid PTEs,
shouldn't we deal with that ?

So the HW guys confirmed that the TLB will never cache a valid entry that has all permissions clear. That leaves the THP write problem though.

Cheers,
Ben.

Quoted 39 lines

Cheers,
Ben.

Quoted 36 lines
Quoted 1 line

---
 arch/powerpc/include/asm/book3s/64/radix.h | 12 +++---------

 1 file changed, 3 insertions(+), 9 deletions(-)
arch/powerpc/include/asm/book3s/64/radix.h +0 -0
index b4d1302387a3..70a3cdcdbe47 100644
@@ -144,16 +144,10 @@ static inline unsigned long
 radix__pte_update(struct mm_struct *mm,
Quoted 22 lines

   * new value of pte
   */
  new_pte = (old_pte | set) & ~clr;

(quoted diff) +0 -3
-		/*
-		 * If we are trying to clear the pte, we can
skip
-		 * the below sequence and batch the tlb flush.

The

arch/powerpc/include/asm/book3s/64/radix.h +0 -5
-		 * tlb flush batching is done by mmu gather code
-		 */
-		if (new_pte) {
-			asm volatile("ptesync" : : : "memory");
-			radix__flush_tlb_pte_p9_dd1(old_pte, mm,

addr);

arch/powerpc/include/asm/book3s/64/radix.h +3 -0
+		asm volatile("ptesync" : : : "memory");
+		radix__flush_tlb_pte_p9_dd1(old_pte, mm, addr);
+		if (new_pte)

  __radix_pte_update(ptep, 0, new_pte);

arch/powerpc/include/asm/book3s/64/radix.h +0 -1
-		}

  } else
  old_pte = __radix_pte_update(ptep, clr, set); asm volatile("ptesync" : : : "memory");

#7 lore raw
Quoted 20 lines · Benjamin Herrenschmidt
Quoted 19 lines · Benjamin Herrenschmidt
Quoted 17 lines · Aneesh Kumar K.V

In the kernel we do follow the below sequence in different code paths.
pte = ptep_get_clear(ptep)
....
set_pte_at(ptep, pte)

We do that for mremap, autonuma protection update and softdirty clearing. This
implies our optimization to skip a tlb flush when clearing a pte update is
not valid, because for DD1 system that followup set_pte_at will be done witout
doing the required tlbflush. Fix that by always doing the dd1 style pte update
irrespective of new_pte value. In a later patch we will optimize the application
exit case.

Quoted 7 lines
Quoted 3 lines

What about my change to set_pte_at() ? We seem to be overwriting valid PTEs,
shouldn't we deal with that ?

So the HW guys confirmed that the TLB will never cache a valid entry that has all permissions clear. That leaves the THP write problem though.

Which is fixed by the autonuma related changes posted at
https://lkml.kernel.org/r/[email protected]

Right now i am running a kernel compile in loop with parallel perf bench numa mem run to make sure we got most of the details correct. (this is on p8)

-aneesh

Tested-by on v1 1/3
#8 lore raw
In reply to Aneesh Kumar K.V
Quoted 6 lines · Aneesh Kumar K.V

In the kernel we do follow the below sequence in different code paths. pte =3D ptep_get_clear(ptep)
....
set_pte_at(ptep, pte)
=20
We do that for mremap, autonuma protection update and softdirty clearing.=

This

Quoted 1 line

implies our optimization to skip a tlb flush when clearing a pte update i=

s

Quoted 1 line

not valid, because for DD1 system that followup set_pte_at will be done w=

itout

Quoted 1 line

doing the required tlbflush. Fix that by always doing the dd1 style pte u=

pdate

Quoted 6 lines

irrespective of new_pte value. In a later patch we will optimize the application
exit case.
=20

arch/powerpc/include/asm/book3s/64/radix.h static inline unsigned long radix__pte_update(struc=

Quoted 10 lines

---
=C2=A0arch/powerpc/include/asm/book3s/64/radix.h | 12 +++--------- =C2=A01 file changed, 3 insertions(+), 9 deletions(-) =20

arch/powerpc/include/asm/book3s/64/radix.h +0 -0
index b4d1302387a3..70a3cdcdbe47 100644
@@ -144,16 +144,10 @@ static inline unsigned long radix__pte_update(struc=

t

Quoted 20 lines
arch/powerpc/include/asm/book3s/64/radix.h +0 -0
 mm_struct *mm,

=C2=A0 =C2=A0* new value of pte
=C2=A0 =C2=A0*/
=C2=A0 new_pte =3D (old_pte | set) & ~clr;

arch/powerpc/include/asm/book3s/64/radix.h +3 -8
-		/*
-		=C2=A0* If we are trying to clear the pte, we can skip
-		=C2=A0* the below sequence and batch the tlb flush. The
-		=C2=A0* tlb flush batching is done by mmu gather code
-		=C2=A0*/
-		if (new_pte) {
-			asm volatile("ptesync" : : : "memory");
-			radix__flush_tlb_pte_p9_dd1(old_pte, mm, addr);
+		asm volatile("ptesync" : : : "memory");
+		radix__flush_tlb_pte_p9_dd1(old_pte, mm, addr);
+		if (new_pte)

=C2=A0 __radix_pte_update(ptep, 0, new_pte);

arch/powerpc/include/asm/book3s/64/radix.h +0 -1
-		}

=C2=A0 } else
=C2=A0 old_pte =3D __radix_pte_update(ptep, clr, set);
=C2=A0 asm volatile("ptesync" : : : "memory");

#9 lore raw
In reply to Aneesh Kumar K.V
Quoted 12 lines · Aneesh Kumar K.V

In the kernel we do follow the below sequence in different code paths. pte = ptep_get_clear(ptep)
....
set_pte_at(ptep, pte)

We do that for mremap, autonuma protection update and softdirty clearing. This implies our optimization to skip a tlb flush when clearing a pte update is not valid, because for DD1 system that followup set_pte_at will be done witout doing the required tlbflush. Fix that by always doing the dd1 style pte update irrespective of new_pte value. In a later patch we will optimize the application exit case.

Quoted 3 lines

Series applied to powerpc next, thanks.

https://git.kernel.org/powerpc/c/ca94573b9c69d224e50e1084a27767

cheers