mm: tlb: unmap scalability

RFC PATCH In review Aaro Koskinen 3 participants 13 messages Mar 9, 2009 – Mar 16, 2009

Subsystems (15)

Memory management Arm (32-bit) x86 PowerPC +11 more subsystems

lore

Share & tools

lore

Aaro Koskinen

author

2009-03-09 12:56 UTC #0 lore raw

cover mm: tlb: unmap scalability

Hello,

Here's a patch proposal to make unmap to scale linearly on architectures that implement tlb_start_vma() and tlb_end_vma(), by adding range parameters. See <http://marc.info/?l=linux-kernel&m=123610437815468&w=2> for the current problem.

The first patch only adds the new parameters. The second one changes the ARM architecture to use those parameters. A similar change should be of course made also for other architectures implementing those routines.

The patch was made for 2.6.29-rc7.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Aaro Koskinen

author

2009-03-09 13:00 UTC #1 lore raw

RFC PATCH 1/2 mm: tlb: Add range to tlb_start_vma() and tlb_end_vma()

Pass the range to be teared down with tlb_start_vma() and tlb_end_vma(). This allows architectures doing per-VMA handling to flush only the needed range instead of the full VMA region.

This patch changes the interface only, no changes in functionality.

Signed-off-by Aaro Koskinen <[email protected]>

---
 arch/alpha/include/asm/tlb.h    |    4 ++--
 arch/arm/include/asm/tlb.h      |    6 ++++--
 arch/avr32/include/asm/tlb.h    |    4 ++--
 arch/blackfin/include/asm/tlb.h |    4 ++--
 arch/cris/include/asm/tlb.h     |    4 ++--
 arch/ia64/include/asm/tlb.h     |    8 ++++----
 arch/m68k/include/asm/tlb.h     |    4 ++--
 arch/mips/include/asm/tlb.h     |    4 ++--
 arch/parisc/include/asm/tlb.h   |    4 ++--
 arch/powerpc/include/asm/tlb.h  |    4 ++--
 arch/s390/include/asm/tlb.h     |    4 ++--
 arch/sh/include/asm/tlb.h       |    4 ++--
 arch/sparc/include/asm/tlb_32.h |    4 ++--
 arch/sparc/include/asm/tlb_64.h |    4 ++--
 arch/um/include/asm/tlb.h       |    4 ++--
 arch/x86/include/asm/tlb.h      |    4 ++--
 arch/xtensa/include/asm/tlb.h   |    8 ++++----
 include/asm-frv/tlb.h           |    4 ++--
 include/asm-m32r/tlb.h          |    4 ++--
 include/asm-mn10300/tlb.h       |    4 ++--
 mm/memory.c                     |   10 +++++++---
 21 files changed, 53 insertions(+), 47 deletions(-)

arch/alpha/include/asm/tlb.h +2 -2

index c136365..26991bc 100644
@@ -1,8 +1,8 @@
 #ifndef _ALPHA_TLB_H
 #define _ALPHA_TLB_H
 
-#define tlb_start_vma(tlb, vma)			do { } while (0)
-#define tlb_end_vma(tlb, vma)			do { } while (0)
+#define tlb_start_vma(tlb, vma, range_start, range_end)	do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end)	do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, pte, addr)	do { } while (0)
 
 #define tlb_flush(tlb)				flush_tlb_mm((tlb)->mm)

arch/arm/include/asm/tlb.h +4 -2

index 857f1df..d10c9c3 100644
@@ -71,14 +71,16 @@ tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end)
  * the vmas are adjusted to only cover the region to be torn down.
  */
 static inline void
-tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
+tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma,
+	      unsigned long range_start, unsigned long range_end)
 {
 	if (!tlb->fullmm)
 		flush_cache_range(vma, vma->vm_start, vma->vm_end);
 }
 
 static inline void
-tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
+tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma,
+	    unsigned long range_start, unsigned long range_end)
 {
 	if (!tlb->fullmm)
 		flush_tlb_range(vma, vma->vm_start, vma->vm_end);

arch/avr32/include/asm/tlb.h +2 -2

index 5c55f9c..41381e9 100644
@@ -8,10 +8,10 @@
 #ifndef __ASM_AVR32_TLB_H
 #define __ASM_AVR32_TLB_H
 
-#define tlb_start_vma(tlb, vma) \
+#define tlb_start_vma(tlb, vma, range_start, range_end) \
 	flush_cache_range(vma, vma->vm_start, vma->vm_end)
 
-#define tlb_end_vma(tlb, vma) \
+#define tlb_end_vma(tlb, vma, range_start, range_end) \
 	flush_tlb_range(vma, vma->vm_start, vma->vm_end)
 
 #define __tlb_remove_tlb_entry(tlb, pte, address) do { } while(0)

arch/blackfin/include/asm/tlb.h +2 -2

index 89a12ee..cf7eb67 100644
@@ -1,8 +1,8 @@
 #ifndef _BLACKFIN_TLB_H
 #define _BLACKFIN_TLB_H
 
-#define tlb_start_vma(tlb, vma)	do { } while (0)
-#define tlb_end_vma(tlb, vma)	do { } while (0)
+#define tlb_start_vma(tlb, vma, range_start, range_end)	do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end)	do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address)	do { } while (0)
 
 /*

arch/cris/include/asm/tlb.h +2 -2

index 77384ea..87e9879 100644
@@ -9,8 +9,8 @@
  * cris doesn't need any special per-pte or
  * per-vma handling..
  */
-#define tlb_start_vma(tlb, vma) do { } while (0)
-#define tlb_end_vma(tlb, vma) do { } while (0)
+#define tlb_start_vma(tlb, vma, range_start, range_end) do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end) do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
 
 #define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)

arch/ia64/include/asm/tlb.h +4 -4

index 20d8a39..b1c7bbf 100644
@@ -25,14 +25,14 @@
  *	tlb <- tlb_gather_mmu(mm, full_mm_flush);	// start unmap for address space MM
  *	{
  *	  for each vma that needs a shootdown do {
- *	    tlb_start_vma(tlb, vma);
+ *	    tlb_start_vma(tlb, vma, range_start, range_end);
  *	      for each page-table-entry PTE that needs to be removed do {
  *		tlb_remove_tlb_entry(tlb, pte, address);
  *		if (pte refers to a normal page) {
  *		  tlb_remove_page(tlb, page);
  *		}
  *	      }
- *	    tlb_end_vma(tlb, vma);
+ *	    tlb_end_vma(tlb, vma, range_start, range_end);
  *	  }
  *	}
  *	tlb_finish_mmu(tlb, start, end);	// finish unmap for address space MM

@@ -227,8 +227,8 @@ __tlb_remove_tlb_entry (struct mmu_gather *tlb, pte_t *ptep, unsigned long addre
 
 #define tlb_migrate_finish(mm)	platform_tlb_migrate_finish(mm)
 
-#define tlb_start_vma(tlb, vma)			do { } while (0)
-#define tlb_end_vma(tlb, vma)			do { } while (0)
+#define tlb_start_vma(tlb, vma, range_start, range_end)	do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end)	do { } while (0)
 
 #define tlb_remove_tlb_entry(tlb, ptep, addr)		\
 do {							\

arch/m68k/include/asm/tlb.h +2 -2

index 1785cff..0363248 100644
@@ -5,8 +5,8 @@
  * m68k doesn't need any special per-pte or
  * per-vma handling..
  */
-#define tlb_start_vma(tlb, vma)	do { } while (0)
-#define tlb_end_vma(tlb, vma)	do { } while (0)
+#define tlb_start_vma(tlb, vma, range_start, range_end)	do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end)	do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address)	do { } while (0)
 
 /*

arch/mips/include/asm/tlb.h +2 -2

index 80d9dfc..8491179 100644
@@ -5,12 +5,12 @@
  * MIPS doesn't need any special per-pte or per-vma handling, except
  * we need to flush cache for area to be unmapped.
  */
-#define tlb_start_vma(tlb, vma) 				\
+#define tlb_start_vma(tlb, vma, range_start, range_end) 	\
 	do {							\
 		if (!tlb->fullmm)				\
 			flush_cache_range(vma, vma->vm_start, vma->vm_end); \
 	}  while (0)
-#define tlb_end_vma(tlb, vma) do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end) do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
 
 /*

arch/parisc/include/asm/tlb.h +2 -2

index 383b1db..37c40e5 100644
@@ -6,12 +6,12 @@ do {	if ((tlb)->fullmm)		\
 		flush_tlb_mm((tlb)->mm);\
 } while (0)
 
-#define tlb_start_vma(tlb, vma) \
+#define tlb_start_vma(tlb, vma, range_start, range_end) \
 do {	if (!(tlb)->fullmm)	\
 		flush_cache_range(vma, vma->vm_start, vma->vm_end); \
 } while (0)
 
-#define tlb_end_vma(tlb, vma)	\
+#define tlb_end_vma(tlb, vma, range_start, range_end)	\
 do {	if (!(tlb)->fullmm)	\
 		flush_tlb_range(vma, vma->vm_start, vma->vm_end); \
 } while (0)

arch/powerpc/include/asm/tlb.h +2 -2

index e20ff75..d7ab142 100644
@@ -27,8 +27,8 @@
 
 struct mmu_gather;
 
-#define tlb_start_vma(tlb, vma)	do { } while (0)
-#define tlb_end_vma(tlb, vma)	do { } while (0)
+#define tlb_start_vma(tlb, vma, range_start, range_end)	do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end)	do { } while (0)
 
 #if !defined(CONFIG_PPC_STD_MMU)

arch/s390/include/asm/tlb.h +2 -2

index 3d8a96d..718d16f 100644
@@ -148,8 +148,8 @@ static inline void pud_free_tlb(struct mmu_gather *tlb, pud_t *pud)
 #endif
 }
 
-#define tlb_start_vma(tlb, vma)			do { } while (0)
-#define tlb_end_vma(tlb, vma)			do { } while (0)
+#define tlb_start_vma(tlb, vma, range_start, range_end)	do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end)	do { } while (0)
 #define tlb_remove_tlb_entry(tlb, ptep, addr)	do { } while (0)
 #define tlb_migrate_finish(mm)			do { } while (0)

arch/sh/include/asm/tlb.h +2 -2

index 88ff1ae..84ad1f9 100644
@@ -7,10 +7,10 @@
 
 #ifndef __ASSEMBLY__
 
-#define tlb_start_vma(tlb, vma) \
+#define tlb_start_vma(tlb, vma, range_start, range_end) \
 	flush_cache_range(vma, vma->vm_start, vma->vm_end)
 
-#define tlb_end_vma(tlb, vma)	\
+#define tlb_end_vma(tlb, vma, range_start, range_end)	\
 	flush_tlb_range(vma, vma->vm_start, vma->vm_end)
 
 #define __tlb_remove_tlb_entry(tlb, pte, address)	do { } while (0)

arch/sparc/include/asm/tlb_32.h +2 -2

index 6d02d1c..8161627 100644
@@ -1,12 +1,12 @@
 #ifndef _SPARC_TLB_H
 #define _SPARC_TLB_H
 
-#define tlb_start_vma(tlb, vma) \
+#define tlb_start_vma(tlb, vma, range_start, range_end) \
 do {								\
 	flush_cache_range(vma, vma->vm_start, vma->vm_end);	\
 } while (0)
 
-#define tlb_end_vma(tlb, vma) \
+#define tlb_end_vma(tlb, vma, range_start, range_end) \
 do {								\
 	flush_tlb_range(vma, vma->vm_start, vma->vm_end);	\
 } while (0)

arch/sparc/include/asm/tlb_64.h +2 -2

index ec81cde..e5d121e 100644
@@ -105,7 +105,7 @@ static inline void tlb_remove_page(struct mmu_gather *mp, struct page *page)
 #define pud_free_tlb(tlb,pudp) __pud_free_tlb(tlb,pudp)
 
 #define tlb_migrate_finish(mm)	do { } while (0)
-#define tlb_start_vma(tlb, vma) do { } while (0)
-#define tlb_end_vma(tlb, vma)	do { } while (0)
+#define tlb_start_vma(tlb, vma, range_start, range_end) do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end)	do { } while (0)
 
 #endif /* _SPARC64_TLB_H */

arch/um/include/asm/tlb.h +2 -2

index 5240fa1..a2eafcc 100644
@@ -7,8 +7,8 @@
 #include <asm/pgalloc.h>
 #include <asm/tlbflush.h>
 
-#define tlb_start_vma(tlb, vma) do { } while (0)
-#define tlb_end_vma(tlb, vma) do { } while (0)
+#define tlb_start_vma(tlb, vma, range_start, range_end) do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end) do { } while (0)
 #define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
 
 /* struct mmu_gather is an opaque type used by the mm code for passing around

arch/x86/include/asm/tlb.h +2 -2

index 829215f..7421c06 100644
@@ -1,8 +1,8 @@
 #ifndef _ASM_X86_TLB_H
 #define _ASM_X86_TLB_H
 
-#define tlb_start_vma(tlb, vma) do { } while (0)
-#define tlb_end_vma(tlb, vma) do { } while (0)
+#define tlb_start_vma(tlb, vma, range_start, range_end) do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end) do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
 #define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)

arch/xtensa/include/asm/tlb.h +4 -4

index 31c220f..8f99a8e 100644
@@ -18,18 +18,18 @@
 
 /* Note, read http://lkml.org/lkml/2004/1/15/6 */
 
-# define tlb_start_vma(tlb,vma)			do { } while (0)
-# define tlb_end_vma(tlb,vma)			do { } while (0)
+# define tlb_start_vma(tlb, vma, range_start, range_end) do { } while (0)
+# define tlb_end_vma(tlb, vma, range_start, range_end) do { } while (0)
 
 #else
 
-# define tlb_start_vma(tlb, vma)					      \
+# define tlb_start_vma(tlb, vma, range_start, range_end)		      \
 	do {								      \
 		if (!tlb->fullmm)					      \
 			flush_cache_range(vma, vma->vm_start, vma->vm_end);   \
 	} while(0)
 
-# define tlb_end_vma(tlb, vma)						      \
+# define tlb_end_vma(tlb, vma, range_start, range_end)			      \
 	do {								      \
 		if (!tlb->fullmm)					      \
 			flush_tlb_range(vma, vma->vm_start, vma->vm_end);     \

include/asm-frv/tlb.h +2 -2

index cd458eb..8553784 100644
@@ -12,8 +12,8 @@ extern void check_pgt_cache(void);
 /*
  * we don't need any special per-pte or per-vma handling...
  */
-#define tlb_start_vma(tlb, vma)				do { } while (0)
-#define tlb_end_vma(tlb, vma)				do { } while (0)
+#define tlb_start_vma(tlb, vma, range_start, range_end)	do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end)	do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address)	do { } while (0)
 
 /*

include/asm-m32r/tlb.h +2 -2

index c7ebd8d..3f4c08d 100644
@@ -5,8 +5,8 @@
  * x86 doesn't need any special per-pte or
  * per-vma handling..
  */
-#define tlb_start_vma(tlb, vma) do { } while (0)
-#define tlb_end_vma(tlb, vma) do { } while (0)
+#define tlb_start_vma(tlb, vma, range_start, range_end) do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end) do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, pte, address) do { } while (0)
 
 /*

include/asm-mn10300/tlb.h +2 -2

index 65d232b..89acf74 100644
@@ -19,8 +19,8 @@ extern void check_pgt_cache(void);
 /*
  * we don't need any special per-pte or per-vma handling...
  */
-#define tlb_start_vma(tlb, vma)				do { } while (0)
-#define tlb_end_vma(tlb, vma)				do { } while (0)
+#define tlb_start_vma(tlb, vma, range_start, range_end)	do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end)	do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address)	do { } while (0)
 
 /*

mm/memory.c +7 -3

index baa999e..44996b6 100644
@@ -896,17 +896,21 @@ static inline unsigned long zap_pud_range(struct mmu_gather *tlb,
 
 static unsigned long unmap_page_range(struct mmu_gather *tlb,
 				struct vm_area_struct *vma,
-				unsigned long addr, unsigned long end,
+				unsigned long range_start, unsigned long end,
 				long *zap_work, struct zap_details *details)
 {
 	pgd_t *pgd;
 	unsigned long next;
+	unsigned long addr = range_start;
+	unsigned long range_end;
 
 	if (details && !details->check_mapping && !details->nonlinear_vma)
 		details = NULL;
 
 	BUG_ON(addr >= end);
-	tlb_start_vma(tlb, vma);
+	BUG_ON(*zap_work <= 0);
+	range_end = addr + min(end - addr, (unsigned long)*zap_work);
+	tlb_start_vma(tlb, vma, range_start, range_end);
 	pgd = pgd_offset(vma->vm_mm, addr);
 	do {
 		next = pgd_addr_end(addr, end);

@@ -917,7 +921,7 @@ static unsigned long unmap_page_range(struct mmu_gather *tlb,
 		next = zap_pud_range(tlb, vma, pgd, addr, next,
 						zap_work, details);
 	} while (pgd++, addr = next, (addr != end && *zap_work > 0));
-	tlb_end_vma(tlb, vma);
+	tlb_end_vma(tlb, vma, range_start, range_end);
 
 	return addr;
 }

Signature · 7 lines

1.5.4.3

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Aaro Koskinen

author

2009-03-09 13:00 UTC #2 lore raw

RFC PATCH 2/2 ARM: tlb: Use range in tlb_start_vma() and tlb_end_vma()

Flush only the pages that were unmapped.

Signed-off-by Aaro Koskinen <[email protected]>

---
 arch/arm/include/asm/tlb.h |    6 +++---
 1 files changed, 3 insertions(+), 3 deletions(-)

arch/arm/include/asm/tlb.h +3 -3

index d10c9c3..a034b6d 100644
@@ -68,14 +68,14 @@ tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end)
 /*
  * In the case of tlb vma handling, we can optimise these away in the
  * case where we're doing a full MM flush.  When we're doing a munmap,
- * the vmas are adjusted to only cover the region to be torn down.
+ * the range is adjusted to only cover the region to be torn down.
  */
 static inline void
 tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma,
 	      unsigned long range_start, unsigned long range_end)
 {
 	if (!tlb->fullmm)
-		flush_cache_range(vma, vma->vm_start, vma->vm_end);
+		flush_cache_range(vma, range_start, range_end);
 }
 
 static inline void

@@ -83,7 +83,7 @@ tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma,
 	    unsigned long range_start, unsigned long range_end)
 {
 	if (!tlb->fullmm)
-		flush_tlb_range(vma, vma->vm_start, vma->vm_end);
+		flush_tlb_range(vma, range_start, range_end);
 }
 
 #define tlb_remove_page(tlb,page)	free_page_and_swap_cache(page)

Signature · 7 lines

1.5.4.3

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Hugh Dickins

2009-03-09 14:16 UTC #3 lore raw

Re: [RFC PATCH 1/2] mm: tlb: Add range to tlb_start_vma() and tlb_end_vma()

In reply to Aaro Koskinen

Quoted 30 lines · Aaro Koskinen

Pass the range to be teared down with tlb_start_vma() and tlb_end_vma(). This allows architectures doing per-VMA handling to flush only the needed range instead of the full VMA region.

This patch changes the interface only, no changes in functionality.

Signed-off-by Aaro Koskinen <[email protected]>

---
 arch/alpha/include/asm/tlb.h    |    4 ++--
 arch/arm/include/asm/tlb.h      |    6 ++++--
 arch/avr32/include/asm/tlb.h    |    4 ++--
 arch/blackfin/include/asm/tlb.h |    4 ++--
 arch/cris/include/asm/tlb.h     |    4 ++--
 arch/ia64/include/asm/tlb.h     |    8 ++++----
 arch/m68k/include/asm/tlb.h     |    4 ++--
 arch/mips/include/asm/tlb.h     |    4 ++--
 arch/parisc/include/asm/tlb.h   |    4 ++--
 arch/powerpc/include/asm/tlb.h  |    4 ++--
 arch/s390/include/asm/tlb.h     |    4 ++--
 arch/sh/include/asm/tlb.h       |    4 ++--
 arch/sparc/include/asm/tlb_32.h |    4 ++--
 arch/sparc/include/asm/tlb_64.h |    4 ++--
 arch/um/include/asm/tlb.h       |    4 ++--
 arch/x86/include/asm/tlb.h      |    4 ++--
 arch/xtensa/include/asm/tlb.h   |    8 ++++----
 include/asm-frv/tlb.h           |    4 ++--
 include/asm-m32r/tlb.h          |    4 ++--
 include/asm-mn10300/tlb.h       |    4 ++--
 mm/memory.c                     |   10 +++++++---
 21 files changed, 53 insertions(+), 47 deletions(-)

...

Quoted 28 lines

mm/memory.c +6 -2

@@ -896,17 +896,21 @@ static inline unsigned long zap_pud_range(struct mmu_gather *tlb,
 
 static unsigned long unmap_page_range(struct mmu_gather *tlb,
 				struct vm_area_struct *vma,
-				unsigned long addr, unsigned long end,
+				unsigned long range_start, unsigned long end,
 				long *zap_work, struct zap_details *details)
 {
 	pgd_t *pgd;
 	unsigned long next;
+	unsigned long addr = range_start;
+	unsigned long range_end;
 
 	if (details && !details->check_mapping && !details->nonlinear_vma)
 		details = NULL;
 
 	BUG_ON(addr >= end);
-	tlb_start_vma(tlb, vma);
+	BUG_ON(*zap_work <= 0);
+	range_end = addr + min(end - addr, (unsigned long)*zap_work);
+	tlb_start_vma(tlb, vma, range_start, range_end);
 	pgd = pgd_offset(vma->vm_mm, addr);
 	do {
 		next = pgd_addr_end(addr, end);

mm/memory.c static unsigned long unmap_page_range(struct mmu_gather *tlb, · line 927

Quoted 8 lines

mm/memory.c +1 -1

@@ -917,7 +921,7 @@ static unsigned long unmap_page_range(struct mmu_gather *tlb,
 		next = zap_pud_range(tlb, vma, pgd, addr, next,
 						zap_work, details);
 	} while (pgd++, addr = next, (addr != end && *zap_work > 0));
-	tlb_end_vma(tlb, vma);
+	tlb_end_vma(tlb, vma, range_start, range_end);
 
 	return addr;
 }

Sorry, I don't like this second-guessing of zap_work at all (okay, we all hate zap_work, and would love to rework the tlb mmu_gather stuff to be preemptible, but the file truncation case has so far discouraged us).

Take a look at the levels below, in particular zap_pte_range(), and you'll see that zap_work is just an approximate cap upon the amount of work being done while zapping, and is decremented by wildly different amounts if a pte (or swap entry) is there or not.

So the range_end you calculate will usually be misleadingly different from the actual end of the range.

I don't see that you need to change the interface and other arches at all. What prevents ARM from noting the first and last addresses freed in its struct mmu_gather when tlb_remove_tlb_entry() is called (see arch/um/include/asm/tlb.h for an example of that), then using that in its tlb_end_vma() TLB flushing?

Admittedly you won't know the end for cache flusing in tlb_start_vma(), but you haven't mentioned that one as a problem, and I expect you can devise (ARM-specific) optimizations to avoid repetition there too.

Hugh

Aaro Koskinen

author

2009-03-09 17:00 UTC #4 lore raw

Re: [RFC PATCH 1/2] mm: tlb: Add range to tlb_start_vma() and tlb_end_vma()

Hello,

Quoted 4 lines · Hugh Dickins

Quoted 3 lines · Aaro Koskinen
Pass the range to be teared down with tlb_start_vma() and tlb_end_vma(). This allows architectures doing per-VMA handling to flush only the needed range instead of the full VMA region.

[...]

Quoted 44 lines

Quoted 23 lines
static unsigned long unmap_page_range(struct mmu_gather *tlb,
(quoted diff) +6 -2
 				struct vm_area_struct *vma,
-				unsigned long addr, unsigned long end,
+				unsigned long range_start, unsigned long end,
 				long *zap_work, struct zap_details *details)
 {
 	pgd_t *pgd;
 	unsigned long next;
+	unsigned long addr = range_start;
+	unsigned long range_end;
 
 	if (details && !details->check_mapping && !details->nonlinear_vma)
 		details = NULL;
 
 	BUG_ON(addr >= end);
-	tlb_start_vma(tlb, vma);
+	BUG_ON(*zap_work <= 0);
+	range_end = addr + min(end - addr, (unsigned long)*zap_work);
+	tlb_start_vma(tlb, vma, range_start, range_end);
 	pgd = pgd_offset(vma->vm_mm, addr);
 	do {
 		next = pgd_addr_end(addr, end);
Review static unsigned long unmap_page_range(struct mmu_gather *tlb, · line 927
Quoted 8 lines
(quoted diff) +1 -1
@@ -917,7 +921,7 @@ static unsigned long unmap_page_range(struct mmu_gather *tlb,
 		next = zap_pud_range(tlb, vma, pgd, addr, next,
 						zap_work, details);
 	} while (pgd++, addr = next, (addr != end && *zap_work > 0));
-	tlb_end_vma(tlb, vma);
+	tlb_end_vma(tlb, vma, range_start, range_end);
 
 	return addr;
 }
Sorry, I don't like this second-guessing of zap_work at all (okay, we all hate zap_work, and would love to rework the tlb mmu_gather stuff to be preemptible, but the file truncation case has so far discouraged us).
Take a look at the levels below, in particular zap_pte_range(), and you'll see that zap_work is just an approximate cap upon the amount of work being done while zapping, and is decremented by wildly different amounts if a pte (or swap entry) is there or not.
So the range_end you calculate will usually be misleadingly different from the actual end of the range.

Quoted 5 lines · but that is probably irrelevant because of what you

I don't see that you need to change the interface and other arches at all. What prevents ARM from noting the first and last addresses freed in its struct mmu_gather when tlb_remove_tlb_entry() is called (see arch/um/include/asm/tlb.h for an example of that), then using that in its tlb_end_vma() TLB flushing?

This would probably work, thanks for pointing it out. I should have taken a better look of the full API, not just what was implemented in ARM.

So, there's a new ARM-only patch draft below based on this idea, adding also linux-arm-kernel again.

Quoted 3 lines

Admittedly you won't know the end for cache flusing in tlb_start_vma(), but you haven't mentioned that one as a problem, and I expect you can devise (ARM-specific) optimizations to avoid repetition there too.

Yes, the execution time of tlb_start_vma() does not depend on the range size, so that is a lesser problem.

Thanks,

---

From: Aaro Koskinen <[email protected]>
Subject: [RFC PATCH] [ARM] Flush only the needed range when unmapping VMA

Signed-off-by Aaro Koskinen <[email protected]>

---
 arch/arm/include/asm/tlb.h |   25 ++++++++++++++++++++++---
 1 files changed, 22 insertions(+), 3 deletions(-)

arch/arm/include/asm/tlb.h +22 -3

index 857f1df..2729fb9 100644
@@ -36,6 +36,8 @@
 struct mmu_gather {
 	struct mm_struct	*mm;
 	unsigned int		fullmm;
+	unsigned long		range_start;
+	unsigned long		range_end;
 };
 
 DECLARE_PER_CPU(struct mmu_gather, mmu_gathers);

@@ -47,6 +49,8 @@ tlb_gather_mmu(struct mm_struct *mm, unsigned int full_mm_flush)
 
 	tlb->mm = mm;
 	tlb->fullmm = full_mm_flush;
+	tlb->range_start = TASK_SIZE;
+	tlb->range_end = 0;
 
 	return tlb;
 }

@@ -63,7 +67,19 @@ tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end)
 	put_cpu_var(mmu_gathers);
 }
 
-#define tlb_remove_tlb_entry(tlb,ptep,address)	do { } while (0)
+/*
+ * Memorize the range for the TLB flush.
+ */
+static inline void
+tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, unsigned long addr)
+{
+	if (!tlb->fullmm) {
+		if (addr < tlb->range_start)
+			tlb->range_start = addr;
+		if (addr + PAGE_SIZE > tlb->range_end)
+			tlb->range_end = addr + PAGE_SIZE;
+	}
+}
 
 /*
  * In the case of tlb vma handling, we can optimise these away in the

@@ -80,8 +96,11 @@ tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
 static inline void
 tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
 {
-	if (!tlb->fullmm)
-		flush_tlb_range(vma, vma->vm_start, vma->vm_end);
+	if (!tlb->fullmm && tlb->range_end > 0) {
+		flush_tlb_range(vma, tlb->range_start, tlb->range_end);
+		tlb->range_start = TASK_SIZE;
+		tlb->range_end = 0;
+	}
 }
 
 #define tlb_remove_page(tlb,page)	free_page_and_swap_cache(page)

Signature · 7 lines

1.5.4.3

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Hugh Dickins

2009-03-09 18:20 UTC #5 lore raw

Re: [RFC PATCH 1/2] mm: tlb: Add range to tlb_start_vma() and tlb_end_vma()

Quoted 8 lines · Aaro Koskinen

Quoted 5 lines · Hugh Dickins
I don't see that you need to change the interface and other arches at all. What prevents ARM from noting the first and last addresses freed in its struct mmu_gather when tlb_remove_tlb_entry() is called (see arch/um/include/asm/tlb.h for an example of that), then using that in its tlb_end_vma() TLB flushing?

Quoted 5 lines

This would probably work, thanks for pointing it out. I should have taken a better look of the full API, not just what was implemented in ARM.
So, there's a new ARM-only patch draft below based on this idea, adding also linux-arm-kernel again.

This one is much better, thank you. I would think it more natural to do the initialization of range_start and range_end in your tlb_start_vma() - to complement tlb_end_vma() where you deal with the final result - rather than in two places you have sited it; but that's somewhat a matter of taste, your patch should work as is.

Hugh

Quoted 70 lines

---

From: Aaro Koskinen <[email protected]>
Subject: [RFC PATCH] [ARM] Flush only the needed range when unmapping VMA

Signed-off-by Aaro Koskinen <[email protected]>

---
 arch/arm/include/asm/tlb.h |   25 ++++++++++++++++++++++---
 1 files changed, 22 insertions(+), 3 deletions(-)

arch/arm/include/asm/tlb.h +4 -0

index 857f1df..2729fb9 100644
@@ -36,6 +36,8 @@
 struct mmu_gather {
 	struct mm_struct	*mm;
 	unsigned int		fullmm;
+	unsigned long		range_start;
+	unsigned long		range_end;
 };
 
 DECLARE_PER_CPU(struct mmu_gather, mmu_gathers);

@@ -47,6 +49,8 @@ tlb_gather_mmu(struct mm_struct *mm, unsigned int
 full_mm_flush)
 
 	tlb->mm = mm;
 	tlb->fullmm = full_mm_flush;
+	tlb->range_start = TASK_SIZE;
+	tlb->range_end = 0;
 
 	return tlb;

}

arch/arm/include/asm/tlb.h +13 -1

@@ -63,7 +67,19 @@ tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start,
 unsigned long end)
 	put_cpu_var(mmu_gathers);
 }
 
-#define tlb_remove_tlb_entry(tlb,ptep,address)	do { } while (0)
+/*
+ * Memorize the range for the TLB flush.
+ */
+static inline void
+tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, unsigned long addr)
+{
+	if (!tlb->fullmm) {
+		if (addr < tlb->range_start)
+			tlb->range_start = addr;
+		if (addr + PAGE_SIZE > tlb->range_end)
+			tlb->range_end = addr + PAGE_SIZE;
+	}
+}
 
 /*

* In the case of tlb vma handling, we can optimise these away in the

arch/arm/include/asm/tlb.h +5 -2

@@ -80,8 +96,11 @@ tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct
 *vma)
 static inline void
 tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
 {
-	if (!tlb->fullmm)
-		flush_tlb_range(vma, vma->vm_start, vma->vm_end);
+	if (!tlb->fullmm && tlb->range_end > 0) {
+		flush_tlb_range(vma, tlb->range_start, tlb->range_end);
+		tlb->range_start = TASK_SIZE;
+		tlb->range_end = 0;
+	}
 }

Quoted 3 lines

#define tlb_remove_page(tlb,page) free_page_and_swap_cache(page)
Signature · 1 line
1.5.4.3

Aaro Koskinen

author

2009-03-10 13:02 UTC #6 lore raw

PATCH 1/1 Flush only the needed range when unmapping a VMA

In reply to Aaro Koskinen

When unmapping N pages (e.g. shared memory) the amount of TLB flushes done can be (N*PAGE_SIZE/ZAP_BLOCK_SIZE)*N although it should be N at maximum. With PREEMPT kernel ZAP_BLOCK_SIZE is 8 pages, so there is a noticeable performance penalty when unmapping a large VMA and the system is spending its time in flush_tlb_range().

The problem is that tlb_end_vma() is always flushing the full VMA range. The subrange that needs to be flushed can be calculated by tlb_remove_tlb_entry(). This approach was suggested by Hugh Dickins, and is also used by other arches.

Signed-off-by Aaro Koskinen <[email protected]>

Cc Hugh Dickins <[email protected]>

Cc [email protected]

---

For earlier discussion, see:
http://marc.info/?t=123609820900002&r=1&w=2
http://marc.info/?t=123660375800003&r=1&w=2

 arch/arm/include/asm/tlb.h |   25 +++++++++++++++++++++----
 1 files changed, 21 insertions(+), 4 deletions(-)

arch/arm/include/asm/tlb.h +21 -4

index 857f1df..321c83e 100644
@@ -36,6 +36,8 @@
 struct mmu_gather {
 	struct mm_struct	*mm;
 	unsigned int		fullmm;
+	unsigned long		range_start;
+	unsigned long		range_end;
 };
 
 DECLARE_PER_CPU(struct mmu_gather, mmu_gathers);

@@ -63,7 +65,19 @@ tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end)
 	put_cpu_var(mmu_gathers);
 }
 
-#define tlb_remove_tlb_entry(tlb,ptep,address)	do { } while (0)
+/*
+ * Memorize the range for the TLB flush.
+ */
+static inline void
+tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, unsigned long addr)
+{
+	if (!tlb->fullmm) {
+		if (addr < tlb->range_start)
+			tlb->range_start = addr;
+		if (addr + PAGE_SIZE > tlb->range_end)
+			tlb->range_end = addr + PAGE_SIZE;
+	}
+}
 
 /*
  * In the case of tlb vma handling, we can optimise these away in the

@@ -73,15 +87,18 @@ tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end)
 static inline void
 tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
 {
-	if (!tlb->fullmm)
+	if (!tlb->fullmm) {
 		flush_cache_range(vma, vma->vm_start, vma->vm_end);
+		tlb->range_start = TASK_SIZE;
+		tlb->range_end = 0;
+	}
 }
 
 static inline void
 tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
 {
-	if (!tlb->fullmm)
-		flush_tlb_range(vma, vma->vm_start, vma->vm_end);
+	if (!tlb->fullmm && tlb->range_end > 0)
+		flush_tlb_range(vma, tlb->range_start, tlb->range_end);
 }
 
 #define tlb_remove_page(tlb,page)	free_page_and_swap_cache(page)

Signature · 7 lines

1.5.4.3

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Hugh Dickins

Acked-by on v1 1/1

2009-03-10 14:50 UTC #7 lore raw

Re: [PATCH] [ARM] Flush only the needed range when unmapping a VMA

Quoted 11 lines · Aaro Koskinen

When unmapping N pages (e.g. shared memory) the amount of TLB flushes done can be (N*PAGE_SIZE/ZAP_BLOCK_SIZE)*N although it should be N at maximum. With PREEMPT kernel ZAP_BLOCK_SIZE is 8 pages, so there is a noticeable performance penalty when unmapping a large VMA and the system is spending its time in flush_tlb_range().
The problem is that tlb_end_vma() is always flushing the full VMA range. The subrange that needs to be flushed can be calculated by tlb_remove_tlb_entry(). This approach was suggested by Hugh Dickins, and is also used by other arches.

Quoted 4 lines

Signed-off-by Aaro Koskinen <[email protected]>
Cc Hugh Dickins <[email protected]>
Cc [email protected]
---

Looks good to me:

Acked-by Hugh Dickins <[email protected]>

Quoted 67 lines

For earlier discussion, see:
http://marc.info/?t=123609820900002&r=1&w=2
http://marc.info/?t=123660375800003&r=1&w=2

 arch/arm/include/asm/tlb.h |   25 +++++++++++++++++++++----
 1 files changed, 21 insertions(+), 4 deletions(-)

arch/arm/include/asm/tlb.h +21 -4

index 857f1df..321c83e 100644
@@ -36,6 +36,8 @@
 struct mmu_gather {
 	struct mm_struct	*mm;
 	unsigned int		fullmm;
+	unsigned long		range_start;
+	unsigned long		range_end;
 };
 
 DECLARE_PER_CPU(struct mmu_gather, mmu_gathers);

@@ -63,7 +65,19 @@ tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end)
 	put_cpu_var(mmu_gathers);
 }
 
-#define tlb_remove_tlb_entry(tlb,ptep,address)	do { } while (0)
+/*
+ * Memorize the range for the TLB flush.
+ */
+static inline void
+tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, unsigned long addr)
+{
+	if (!tlb->fullmm) {
+		if (addr < tlb->range_start)
+			tlb->range_start = addr;
+		if (addr + PAGE_SIZE > tlb->range_end)
+			tlb->range_end = addr + PAGE_SIZE;
+	}
+}
 
 /*
  * In the case of tlb vma handling, we can optimise these away in the

@@ -73,15 +87,18 @@ tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end)
 static inline void
 tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
 {
-	if (!tlb->fullmm)
+	if (!tlb->fullmm) {
 		flush_cache_range(vma, vma->vm_start, vma->vm_end);
+		tlb->range_start = TASK_SIZE;
+		tlb->range_end = 0;
+	}
 }
 
 static inline void
 tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
 {
-	if (!tlb->fullmm)
-		flush_tlb_range(vma, vma->vm_start, vma->vm_end);
+	if (!tlb->fullmm && tlb->range_end > 0)
+		flush_tlb_range(vma, tlb->range_start, tlb->range_end);
 }
 
 #define tlb_remove_page(tlb,page)	free_page_and_swap_cache(page)

Signature · 1 line

1.5.4.3

Russell King - ARM Linux

2009-03-12 21:30 UTC #8 lore raw

Re: [PATCH] [ARM] Flush only the needed range when unmapping a VMA

In reply to Aaro Koskinen

Quoted 5 lines · Aaro Koskinen

When unmapping N pages (e.g. shared memory) the amount of TLB flushes done can be (N*PAGE_SIZE/ZAP_BLOCK_SIZE)*N although it should be N at maximum. With PREEMPT kernel ZAP_BLOCK_SIZE is 8 pages, so there is a noticeable performance penalty when unmapping a large VMA and the system is spending its time in flush_tlb_range().

It would be nice to have some figures for the speedup gained by this optimisation - is there any chance you could provide a comparison?

Aaro Koskinen

author

2009-03-13 11:37 UTC #9 lore raw

Re: [PATCH] [ARM] Flush only the needed range when unmapping a VMA

Hello,

Quoted 9 lines · ext Russell King - ARM Linux

Quoted 5 lines · Aaro Koskinen
When unmapping N pages (e.g. shared memory) the amount of TLB flushes done can be (N*PAGE_SIZE/ZAP_BLOCK_SIZE)*N although it should be N at maximum. With PREEMPT kernel ZAP_BLOCK_SIZE is 8 pages, so there is a noticeable performance penalty when unmapping a large VMA and the system is spending its time in flush_tlb_range().
It would be nice to have some figures for the speedup gained by this optimisation - is there any chance you could provide a comparison?

Here's a test on an OMAP3 board, 2.6.28 with linux-omap fixes and PREEMPT enabled. Without the patch:

~ # for PAGES in 1000 2000 3000 4000 5000 6000 7000 8000; do time ./shmtst $PAGES; done shm segment size 4096000 bytes

real    0m 0.12s
user    0m 0.02s
sys     0m 0.10s
shm segment size 8192000 bytes
real    0m 0.36s
user    0m 0.00s
sys     0m 0.35s
shm segment size 12288000 bytes
real    0m 0.71s
user    0m 0.03s
sys     0m 0.67s
shm segment size 16384000 bytes
real    0m 1.17s
user    0m 0.07s
sys     0m 1.10s
shm segment size 20480000 bytes
real    0m 1.75s
user    0m 0.03s
sys     0m 1.71s
shm segment size 24576000 bytes
real    0m 2.44s
user    0m 0.03s
sys     0m 2.39s
shm segment size 28672000 bytes
real    0m 3.24s
user    0m 0.10s
sys     0m 3.14s
shm segment size 32768000 bytes
real    0m 4.16s
user    0m 0.11s
sys     0m 4.04s

With the patch:

~ # for PAGES in 1000 2000 3000 4000 5000 6000 7000 8000; do time ./shmtst $PAGES; done shm segment size 4096000 bytes

real    0m 0.07s
user    0m 0.01s
sys     0m 0.05s
shm segment size 8192000 bytes
real    0m 0.13s
user    0m 0.02s
sys     0m 0.10s
shm segment size 12288000 bytes
real    0m 0.20s
user    0m 0.00s
sys     0m 0.19s
shm segment size 16384000 bytes
real    0m 0.27s
user    0m 0.04s
sys     0m 0.22s
shm segment size 20480000 bytes
real    0m 0.33s
user    0m 0.02s
sys     0m 0.31s
shm segment size 24576000 bytes
real    0m 0.40s
user    0m 0.03s
sys     0m 0.36s
shm segment size 28672000 bytes
real    0m 0.47s
user    0m 0.03s
sys     0m 0.42s
shm segment size 32768000 bytes
real    0m 0.53s
user    0m 0.09s
sys     0m 0.43s

The test program:

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include <sys/ipc.h>
#include <sys/shm.h>

volatile char dummy;

int main (int argc, char *argv[])

{
        void *addr;
        int i;
        int shmid;
        int pages = 0;

        if (argc == 2)
                pages = atoi(argv[1]);

        if (pages < 0) {
                fprintf(stderr, "usage: %s <pages>\n", argv[0]);
                return 0;
        }

        printf("shm segment size %d bytes\n", pages*4096);

        shmid = shmget(IPC_PRIVATE, pages*4096, IPC_CREAT | 0777);
        addr = shmat(shmid, NULL, 0);
        memset(addr, 0xBA, pages*4096);
        shmdt(addr);

        addr = shmat(shmid, NULL, 0);
        for (i = 0; i < pages*4096; i += 4096)
                dummy += *((char *)addr + i);
        shmdt(addr);

        addr = shmat(shmid, NULL, 0);
        shmctl(shmid, IPC_RMID, 0);
        shmdt(addr);

        return 0;
}

Russell King - ARM Linux

2009-03-15 10:28 UTC #10 lore raw

Re: [PATCH] [ARM] Flush only the needed range when unmapping a VMA

Quoted 8 lines · Aaro Koskinen

shm segment size 4096000 bytes

real    0m 0.12s
user    0m 0.02s
sys     0m 0.10s
shm segment size 8192000 bytes
real    0m 0.36s
user    0m 0.00s
sys     0m 0.35s

...

Quoted 8 lines

shm segment size 4096000 bytes

real    0m 0.07s
user    0m 0.01s
sys     0m 0.05s
shm segment size 8192000 bytes
real    0m 0.13s
user    0m 0.02s
sys     0m 0.10s

Yes, that's quite a speedup. Can you provide some words in the patch description which indicate that we have about a 3x speed increase for 8M mappings, which only gets better for larger mappings?

Given that this patch is now only touching ARM, is there any need to indicate that it was CC'd to Hugh when it gets committed?

Aaro Koskinen

author

2009-03-16 12:59 UTC #11 lore raw

Re: [PATCH] [ARM] Flush only the needed range when unmapping a VMA

Quoted 3 lines · ext Russell King - ARM Linux

Yes, that's quite a speedup. Can you provide some words in the patch description which indicate that we have about a 3x speed increase for 8M mappings, which only gets better for larger mappings?

Ok, I will resend the patch with updated description.

Quoted 2 lines

Given that this patch is now only touching ARM, is there any need to indicate that it was CC'd to Hugh when it gets committed?

Probably no...

Aaro Koskinen

author

2009-03-16 13:30 UTC #12 lore raw

PATCH v2 1/1 Flush only the needed range when unmapping a VMA

The speed increase is roughly 3x for 8M mappings and for larger mappings even more.

Signed-off-by Aaro Koskinen <[email protected]>

---

v2: Patch description updated, no other changes.

 arch/arm/include/asm/tlb.h |   25 +++++++++++++++++++++----
 1 files changed, 21 insertions(+), 4 deletions(-)

arch/arm/include/asm/tlb.h +21 -4

index 857f1df..321c83e 100644
@@ -36,6 +36,8 @@
 struct mmu_gather {
 	struct mm_struct	*mm;
 	unsigned int		fullmm;
+	unsigned long		range_start;
+	unsigned long		range_end;
 };
 
 DECLARE_PER_CPU(struct mmu_gather, mmu_gathers);

@@ -63,7 +65,19 @@ tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end)
 	put_cpu_var(mmu_gathers);
 }
 
-#define tlb_remove_tlb_entry(tlb,ptep,address)	do { } while (0)
+/*
+ * Memorize the range for the TLB flush.
+ */
+static inline void
+tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, unsigned long addr)
+{
+	if (!tlb->fullmm) {
+		if (addr < tlb->range_start)
+			tlb->range_start = addr;
+		if (addr + PAGE_SIZE > tlb->range_end)
+			tlb->range_end = addr + PAGE_SIZE;
+	}
+}
 
 /*
  * In the case of tlb vma handling, we can optimise these away in the

@@ -73,15 +87,18 @@ tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end)
 static inline void
 tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
 {
-	if (!tlb->fullmm)
+	if (!tlb->fullmm) {
 		flush_cache_range(vma, vma->vm_start, vma->vm_end);
+		tlb->range_start = TASK_SIZE;
+		tlb->range_end = 0;
+	}
 }
 
 static inline void
 tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
 {
-	if (!tlb->fullmm)
-		flush_tlb_range(vma, vma->vm_start, vma->vm_end);
+	if (!tlb->fullmm && tlb->range_end > 0)
+		flush_tlb_range(vma, tlb->range_start, tlb->range_end);
 }
 
 #define tlb_remove_page(tlb,page)	free_page_and_swap_cache(page)

Signature · 7 lines

1.5.4.3

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>