: MM: Make Paget Tables Relocatable--Conditional TLB Flush
Subsystems (10)
RFC PATCH 1/2 : MM: Make Paget Tables Relocatable--Conditional TLB Flush
These patches make page tables relocatable for numa, memory defragmentation, and memory hotplug. The need to rewalk the page tables before making any changes causes a 3.5% performance degredation in the lmbench page miss micro benchmark. Please check the linux-mm list archives for more benchmark information.
---
These two patches are not completely seperated. There are one or two lines in each that really belong in the other, but I didn't feel it was worth the effort to make the patches stand alone.
The last version of these patches did not generate any discussion on the linux-mm list, so I'm opening them up for more discusion.
This patch is against generic 2.6.23 and has only been fully implemented and tested on x86_64. Other architectures require a method to reload the page tables. Implementations should be easy, but I have no way to test them.
arch/alpha/kernel/smp.c +2 -0
@@ -850,6 +850,8 @@ flush_tlb_mm(struct mm_struct *mm) {preempt_disable (); +clear_bit (MMF_NNED_FLUSH , mm->flags); +if (mm == current->active_mm) {flush_tlb_current (mm);if (atomic_read (&mm->mm_users) <=1 ) {
arch/arm/kernel/smp.c +2 -0
@@ -713,6 +713,8 @@ void flush_tlb_mm(struct mm_struct *mm) {cpumask_t mask = mm->cpu_vm_mask; +clear_bit (MMF_NNED_FLUSH , mm->flags); +on_each_cpu_mask (ipi_flush_tlb_mm, mm,1 ,1 , mask); }
arch/avr32/mm/tlb.c +2 -0
@@ -249,6 +249,8 @@ void flush_tlb_kernel_range(unsigned lonvoid flush_tlb_mm (struct mm_struct *mm) { +clear_bit (MMF_NNED_FLUSH , mm->flags); +/* Invalidate all TLB entries of this process by getting a new ASID */ if (mm->context !=NO_CONTEXT ) {unsigned long flags;
arch/cris/arch-v10/mm/tlb.c +2 -0
@@ -69,6 +69,8 @@ flush_tlb_mm(struct mm_struct *mm)D (printk ("tlb: flush mm context %d (%p)\n" , page_id, mm)); +clear_bit (MMF_NNED_FLUSH , mm->flags); +if (page_id ==NO_CONTEXT )return ;
arch/cris/arch-v32/kernel/smp.c +1 -0
@@ -237,6 +237,7 @@ void flush_tlb_all(void)void flush_tlb_mm (struct mm_struct *mm) { +clear_bit (MMF_NNED_FLUSH , mm->flags);__flush_tlb_mm (mm);flush_tlb_common (mm,FLUSH_ALL ,0 );/* No more mappings in other CPUs */
arch/i386/kernel/smp.c +2 -0
@@ -410,6 +410,8 @@ void flush_tlb_mm (struct mm_struct * mm {cpumask_t cpu_mask; +clear_bit (MMF_NNED_FLUSH , mm->flags); +preempt_disable (); cpu_mask = mm->cpu_vm_mask;cpu_clear (smp_processor_id (), cpu_mask);
arch/i386/mach-voyager/voyager_smp.c +2 -0
@@ -924,6 +924,8 @@ flush_tlb_mm (struct mm_struct * mm) {unsigned long cpu_mask; +clear_bit (MMF_NNED_FLUSH , mm->flags); +preempt_disable (); cpu_mask =cpus_addr (mm->cpu_vm_mask)[0 ] & ~(1 <<smp_processor_id ());
arch/ia64/kernel/smp.c +2 -0
@@ -325,6 +325,8 @@ smp_flush_tlb_all (void)void smp_flush_tlb_mm (struct mm_struct *mm) { +clear_bit (MMF_NNED_FLUSH , mm->flags); +preempt_disable ();/* this happens for the common case of a single-threaded fork(): */ if (likely (mm == current->active_mm &&atomic_read (&mm->mm_users) ==1 ))
arch/m32r/kernel/smp.c +2 -0
@@ -280,6 +280,8 @@ void smp_flush_tlb_mm(struct mm_struct *unsigned long *mmc;unsigned long flags; +clear_bit (MMF_NNED_FLUSH , mm->flags); +preempt_disable (); cpu_id =smp_processor_id (); mmc = &mm->context[cpu_id];
arch/mips/kernel/smp.c +2 -0
@@ -387,6 +387,8 @@ static inline void smp_on_each_tlb(voidvoid flush_tlb_mm (struct mm_struct *mm) { +clear_bit (MMF_NNED_FLUSH , mm->flags); +preempt_disable ();if ((atomic_read (&mm->mm_users) !=1 ) || (current->mm != mm)) {
arch/powerpc/mm/tlb_32.c +2 -0
@@ -144,6 +144,8 @@ void flush_tlb_mm(struct mm_struct *mm) {struct vm_area_struct *mp; +clear_bit (MMF_NNED_FLUSH , mm->flags); +if (Hash ==0 ) {_tlbia ();return ;
arch/ppc/mm/tlb.c +2 -0
@@ -144,6 +144,8 @@ void flush_tlb_mm(struct mm_struct *mm) {struct vm_area_struct *mp; +clear_bit (MMF_NNED_FLUSH , mm->flags); +if (Hash ==0 ) {_tlbia ();return ;
arch/sh64/mm/fault.c +2 -0
@@ -517,6 +517,8 @@ void flush_tlb_mm(struct mm_struct *mm) ++calls_to_flush_tlb_mm;#endif +clear_bit (MMF_NNED_FLUSH , mm->flags); +if (mm->context ==NO_CONTEXT )return ;
arch/sparc/kernel/smp.c +2 -0
@@ -163,6 +163,8 @@ void smp_flush_cache_mm(struct mm_structvoid smp_flush_tlb_mm (struct mm_struct *mm) { +clear_bit (MMF_NNED_FLUSH , mm->flags); +if (mm->context !=NO_CONTEXT ) {cpumask_t cpu_mask = mm->cpu_vm_mask;cpu_clear (smp_processor_id (), cpu_mask);
arch/sparc64/kernel/smp.c +2 -0
@@ -1112,6 +1112,8 @@ void smp_flush_tlb_mm(struct mm_struct *u32 ctx =CTX_HWBITS (mm->context);int cpu =get_cpu (); +clear_bit (MMF_NNED_FLUSH , mm->flags); +if (atomic_read (&mm->mm_users) ==1 ) { mm->cpu_vm_mask =cpumask_of_cpu (cpu); goto local_flush_and_out;
arch/um/kernel/tlb.c +1 -0
@@ -402,6 +402,7 @@ void flush_tlb_range(struct vm_area_struvoid flush_tlb_mm (struct mm_struct *mm) { +clear_bit (MMF_NNED_FLUSH , mm->flags);CHOOSE_MODE_PROC (flush_tlb_mm_tt, flush_tlb_mm_skas, mm); }
arch/xtensa/mm/tlb.c +2 -0
@@ -63,6 +63,8 @@ void flush_tlb_all (void)void flush_tlb_mm (struct mm_struct *mm) { +clear_bit (MMF_NNED_FLUSH , mm->flags); +if (mm == current->active_mm) {int flags;local_save_flags (flags);
include/asm-alpha/tlbflush.h +1 -1
@@ -153,5 +153,5 @@ extern void flush_tlb_range(struct vm_ar#endif /* CONFIG_SMP */ #define flush_tlb_kernel_range (start, end) flush_tlb_all() - +#include <asm-generic/tlbflush.h> #endif /* _ALPHA_TLBFLUSH_H */
include/asm-arm/tlbflush.h +1 -0
@@ -471,5 +471,6 @@ extern void update_mmu_cache(struct vm_a#endif #endif /* CONFIG_MMU */ +#include <asm-generic/tlbflush.h> #endif
include/asm-avr32/tlbflush.h +1 -0
@@ -36,5 +36,6 @@ static inline void flush_tlb_pgtables(st }extern void flush_tlb_kernel_range (unsigned long start,unsigned long end); +#include <asm-generic/tlbflush.h> #endif /* __ASM_AVR32_TLBFLUSH_H */
include/asm-blackfin/tlbflush.h +1 -0
@@ -59,4 +59,5 @@ static inline void flush_tlb_pgtables(st BUG(); } +#include <asm-generic/tlbflush.h> #endif
include/asm-cris/tlbflush.h +1 -0
@@ -51,5 +51,6 @@ static inline void flush_tlb(void) }#define flush_tlb_kernel_range (start, end) flush_tlb_all() +#include <asm-generic/tlbflush.h> #endif /* _CRIS_TLBFLUSH_H */
include/asm-frv/tlbflush.h +1 -0
@@ -71,6 +71,7 @@ do { \#define flush_tlb_kernel_range (start, end) BUG()#endif +#include <asm-generic/tlbflush.h> #endif /* _ASM_TLBFLUSH_H */
include/asm-generic/tlbflush.h +103 -0
103 lines not on this page · raw message
include/asm-h8300/tlbflush.h +2 -0
@@ -58,4 +58,6 @@ static inline void flush_tlb_pgtables(st BUG(); } +#include <asm-generic/tlbflush.h> +#endif /* _H8300_TLBFLUSH_H */
include/asm-i386/tlbflush.h +2 -0
@@ -172,4 +172,6 @@ static inline void flush_tlb_pgtables(st /* i386 does not keep any page table caches in TLB */ } +#include <asm-generic/tlbflush.h> +#endif /* _I386_TLBFLUSH_H */
include/asm-ia64/tlbflush.h +1 -0
@@ -106,5 +106,6 @@ void smp_local_flush_tlb(void);#endif #define flush_tlb_kernel_range (start, end) flush_tlb_all()/* XXX fix me */ +#include <asm-generic/tlbflush.h> #endif /* _ASM_IA64_TLBFLUSH_H */
include/asm-m32r/tlbflush.h +1 -0
@@ -96,5 +96,6 @@ static __inline__ void __flush_tlb_all(v#define flush_tlb_pgtables (mm, start, end) do { } while (0)extern void update_mmu_cache (struct vm_area_struct *,unsigned long ,pte_t ); +#include <asm-generic/tlbflush.h> #endif /* _ASM_M32R_TLBFLUSH_H */
include/asm-m68k/tlbflush.h +1 -0
@@ -225,5 +225,6 @@ static inline void flush_tlb_pgtables(st }#endif +#include <asm-generic/tlbflush.h> #endif /* _M68K_TLBFLUSH_H */
include/asm-m68knommu/tlbflush.h +2 -0
@@ -58,4 +58,6 @@ static inline void flush_tlb_pgtables(st BUG(); } +#include <asm-generic/tlbflush.h> +#endif /* _M68KNOMMU_TLBFLUSH_H */
include/asm-mips/tlbflush.h +1 -0
@@ -50,5 +50,6 @@ static inline void flush_tlb_pgtables(st {/* Nothing to do on MIPS. */ } +#include <asm-generic/tlbflush.h> #endif /* __ASM_TLBFLUSH_H */
include/asm-parisc/tlbflush.h +1 -0
@@ -80,5 +80,6 @@ void __flush_tlb_range(unsigned long sid#define flush_tlb_range (vma,start,end) __flush_tlb_range((vma)->vm_mm->context,start,end)#define flush_tlb_kernel_range (start, end) __flush_tlb_range(0,start,end) +#include <asm-generic/tlbflush.h> #endif
include/asm-powerpc/tlbflush.h +2 -0
@@ -183,5 +183,7 @@ static inline void flush_tlb_pgtables(st { } +#include <asm-generic/tlbflush.h> +#endif /*__KERNEL__ */ #endif /* _ASM_POWERPC_TLBFLUSH_H */
include/asm-s390/tlbflush.h +2 -0
@@ -158,4 +158,6 @@ static inline void flush_tlb_pgtables(st /* S/390 does not keep any page table caches in TLB */ } +#include <asm-generic/tlbflush.h> +#endif /* _S390_TLBFLUSH_H */
include/asm-sh/tlbflush.h +3 -0
@@ -52,4 +52,7 @@ static inline void flush_tlb_pgtables(st {/* Nothing to do */ } + +#include <asm-generic/tlbflush.h> +#endif /* __ASM_SH_TLBFLUSH_H */
include/asm-sh64/tlbflush.h +2 -0
@@ -27,5 +27,7 @@ static inline void flush_tlb_pgtables(stextern void flush_tlb_kernel_range (unsigned long start,unsigned long end); +#include <asm-generic/tlbflush.h> +#endif /* __ASM_SH64_TLBFLUSH_H */
include/asm-sparc/tlbflush.h +2 -0
@@ -63,4 +63,6 @@ static inline void flush_tlb_kernel_rang flush_tlb_all(); } +#include <asm-generic/tlbflush.h> +#endif /* _SPARC_TLBFLUSH_H */
include/asm-sparc64/tlbflush.h +2 -0
@@ -48,4 +48,6 @@ static inline void flush_tlb_pgtables(st */ } +#include <asm-generic/tlbflush.h> +#endif /* _SPARC64_TLBFLUSH_H */
include/asm-um/tlbflush.h +2 -0
@@ -47,4 +47,6 @@ static inline void flush_tlb_pgtables(st { } +#include <asm-generic/tlbflush.h> +#endif
include/asm-v850/tlbflush.h +2 -0
@@ -67,4 +67,6 @@ static inline void flush_tlb_pgtables(st BUG (); } +#include <asm-generic/tlbflush.h> +#endif /* __V850_TLBFLUSH_H__ */
include/asm-xtensa/tlbflush.h +2 -0
@@ -197,6 +197,8 @@ static inline unsigned long read_itlb_trreturn tmp; } +#include <asm-generic/tlbflush.h> +#endif /* __ASSEMBLY__ */ #endif /* __KERNEL__ */ #endif /* _XTENSA_TLBFLUSH_H */
include/linux/sched.h +4 -0
@@ -366,6 +366,10 @@ extern int get_dumpable(struct mm_struct#define MMF_DUMP_FILTER_DEFAULT \ ((1 << MMF_DUMP_ANON_PRIVATE) | (1 << MMF_DUMP_ANON_SHARED)) +/* Misc MM flags. */ +#define MMF_NEED_FLUSH 6 +#define MMF_NEED_RELOAD 7/* Only meaningful on some archs. */ +struct mm_struct {struct vm_area_struct * mmap;/* list of VMAs */ struct rb_root mm_rb;
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Hello.
This is a nitpick, but all of archtectures code except generic use MMF_NNED_FLUSH at clear_bit()...
^
Please fix misspell.
Bye.
Quoted 13 lines
diff -uprwNbB -X 2.6.23/Documentation/dontdiff
2.6.23/arch/alpha/kernel/smp.c2.6.23a/arch/alpha/kernel/smp.carch/alpha/kernel/smp.c +2
-0@@ -850,6 +850,8 @@ flush_tlb_mm(struct mm_struct *mm) {preempt_disable (); +clear_bit (MMF_NNED_FLUSH , mm->flags); +if (mm == current->active_mm) {flush_tlb_current (mm);if (atomic_read (&mm->mm_users) <=1 ) {
Signature · 8 lines
Yasunori Goto -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 2 lines · Yasunori Goto
This is a nitpick, but all of archtectures code except generic use
MMF_NNED_FLUSHatclear_bit()...
I'd say that's a bit more than a nit. Thanks for noticing that. Please hang on, I think I've figured out how to reduce the 3.5% page fault overhead to almost nothing.
Ross
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>