Skip to contentSkip to conversation

to support hookable flush_tlb* functions

PATCH In review Thomas Schlichter 4 participants 4 messages Jan 24, 2003 – Jan 26, 2003 Memory management

lore
Share & tools
lore

Thomas Schlichter

author
#0 lore raw

PATCH 1/1 to support hookable flush_tlb* functions

Hello,

with this mail I send a patch that allows kernel modules to hook into the different flush_tlb* functions defined in <asm/tlbflush.h> or <asm/pgtable.h> in order to synchronize devices TLBs.

This is necessary for devices that provide their own TLB and cannot participate on the CPU busses shootdown protocol. With this patch it is possible to assure TLB consistency.

Currently this extension could possibly be used by high performance interconnects like QsNet from Quadrics (www.quadrics.com), and in the near future by even more high performance, low latency NICs that will implement direct user space DMA transfers to not pinned user pages. TLBs are there a mandatory requirement.

Currently I am writing my diploma thesis about the development of such a device where I need this patch, and as it looks good to me I want to provide it to the public community so it can be reviewed and even more tested. (the i386 parts are tested work fine for me)

The patch consists of two parts, one generic part and for each supported architecture an other part that depends on the generic one.

Attached to this mail is only the generic part and the architecture dependend part for i386 compatible machines just not to waste everyones bandwidth... But if requested I can send you patches for the other architectures, too.

The i386 patch also includes some cleanups by renaming __flush_tlb_* to local_flush_tlb_*.

I hope some time this patches will make it into the kernel sources. (perhaps even into 2.6.x ?)

Sincerely yours

Thomas Schlichter

P.S.: To test this patch I've also written a module that counts the different flush_tlb* calls and shows them in /proc/tlbstat. If requested I could send you this, too.

include/linux/tlbhook.h +122 -0
diff -urP linux-2.5.59/include/linux/tlbhook.h linux-2.5.59_patched/include/linux/tlbhook.h
@@ -0,0 +1,122 @@
+#ifndef _LINUX_TLBHOOK_H
+#define _LINUX_TLBHOOK_H
+
+#include <linux/mm.h>
+
+typedef struct tlb_hook_struct {
+	void (*flush_tlb)( void );
+	void (*flush_tlb_all)( void );
+	void (*flush_tlb_mm)( struct mm_struct *mm );
+	void (*flush_tlb_page)( struct vm_area_struct *vma,
+				unsigned long addr );
+	void (*flush_tlb_range)( struct vm_area_struct *vma,
+				unsigned long start, unsigned long end );
+	void (*flush_tlb_kernel_range)( unsigned long start,
+				unsigned long end );
+	void (*flush_tlb_pgtables)( struct mm_struct *mm,
+				unsigned long start, unsigned long end );
+
+	struct tlb_hook_struct *next;
+	struct tlb_hook_struct *last;
+} tlb_hook_t;
+
+extern tlb_hook_t *tlb_hook_root;
+
+extern int register_tlb_hook( tlb_hook_t *hook );
+extern int unregister_tlb_hook( tlb_hook_t *hook );
+
+static inline void flush_tlb_hook( void )
+{
+	tlb_hook_t *hook = tlb_hook_root;
+
+	while( hook )
+	{
+		if( hook->flush_tlb )
+			hook->flush_tlb( );
+		hook = hook->next;
+	}
+}
+
+
+static inline void flush_tlb_all_hook( void )
+{
+	tlb_hook_t *hook = tlb_hook_root;
+
+	while( hook )
+	{
+		if( hook->flush_tlb_all )
+			hook->flush_tlb_all( );
+		hook = hook->next;
+	}
+}
+
+
+static inline void flush_tlb_mm_hook( struct mm_struct *mm )
+{
+	tlb_hook_t *hook = tlb_hook_root;
+
+	while( hook )
+	{
+		if( hook->flush_tlb_mm )
+			hook->flush_tlb_mm( mm );
+		hook = hook->next;
+	}
+}
+
+
+static inline void flush_tlb_page_hook( struct vm_area_struct *vma,
+				unsigned long addr )
+{
+	tlb_hook_t *hook = tlb_hook_root;
+
+	while( hook )
+	{
+		if( hook->flush_tlb_page )
+			hook->flush_tlb_page( vma, addr );
+		hook = hook->next;
+	}
+}
+
+
+static inline void flush_tlb_range_hook( struct vm_area_struct *vma,
+				unsigned long start, unsigned long end )
+{
+	tlb_hook_t *hook = tlb_hook_root;
+
+	while( hook )
+	{
+		if( hook->flush_tlb_range )
+			hook->flush_tlb_range( vma, start, end );
+		hook = hook->next;
+	}
+}
+
+
+static inline void flush_tlb_kernel_range_hook( unsigned long start,
+				unsigned long end )
+{
+	tlb_hook_t *hook = tlb_hook_root;
+
+	while( hook)
+	{
+		if( hook->flush_tlb_kernel_range )
+			hook->flush_tlb_kernel_range( start, end );
+		hook = hook->next;
+	}
+}
+
+
+static inline void flush_tlb_pgtables_hook( struct mm_struct *mm,
+				unsigned long start, unsigned long end )
+{
+	tlb_hook_t *hook = tlb_hook_root;
+
+	while( hook )
+	{
+		if( hook->flush_tlb_pgtables )
+			hook->flush_tlb_pgtables( mm, start, end );
+		hook = hook->next;
+	}
+}
+
+#endif /* _LINUX_TLBHOOK_H */
mm/Makefile +3 -2
diff -urP linux-2.5.59/mm/Makefile linux-2.5.59_patched/mm/Makefile
@@ -2,7 +2,8 @@
 # Makefile for the linux memory manager.
 #
 
-export-objs := shmem.o filemap.o mempool.o page_alloc.o page-writeback.o
+export-objs 		:= shmem.o filemap.o mempool.o page_alloc.o page-writeback.o \
+			   tlbhook.o
 
 mmu-y			:= nommu.o
 mmu-$(CONFIG_MMU)	:= fremap.o highmem.o madvise.o memory.o mincore.o \

@@ -11,6 +12,6 @@
 
 obj-y			:= bootmem.o filemap.o mempool.o oom_kill.o \
 			   page_alloc.o page-writeback.o pdflush.o readahead.o \
-			   slab.o swap.o truncate.o vcache.o vmscan.o $(mmu-y)
+			   slab.o swap.o tlbhook.o truncate.o vcache.o vmscan.o $(mmu-y)
 
 obj-$(CONFIG_SWAP)	+= page_io.o swap_state.o swapfile.o
mm/tlbhook.c +70 -0

70 lines not on this page · raw message

arch/i386/kernel/smp.c +7 -7
diff -urP linux-2.5.59/arch/i386/kernel/smp.c linux-2.5.59_patched/arch/i386/kernel/smp.c
@@ -327,7 +327,7 @@
 			if (flush_va == FLUSH_ALL)
 				local_flush_tlb();
 			else
-				__flush_tlb_one(flush_va);
+				local_flush_tlb_one(flush_va);
 		} else
 			leave_mm(cpu);
 	}

@@ -382,7 +382,7 @@
 	spin_unlock(&tlbstate_lock);
 }
 	
-void flush_tlb_current_task(void)
+void smp_flush_tlb(void)
 {
 	struct mm_struct *mm = current->mm;
 	unsigned long cpu_mask;

@@ -396,7 +396,7 @@
 	preempt_enable();
 }
 
-void flush_tlb_mm (struct mm_struct * mm)
+void smp_flush_tlb_mm (struct mm_struct * mm)
 {
 	unsigned long cpu_mask;
 

@@ -415,7 +415,7 @@
 	preempt_enable();
 }
 
-void flush_tlb_page(struct vm_area_struct * vma, unsigned long va)
+void smp_flush_tlb_page(struct vm_area_struct * vma, unsigned long va)
 {
 	struct mm_struct *mm = vma->vm_mm;
 	unsigned long cpu_mask;

@@ -425,7 +425,7 @@
 
 	if (current->active_mm == mm) {
 		if(current->mm)
-			__flush_tlb_one(va);
+			local_flush_tlb_one(va);
 		 else
 		 	leave_mm(smp_processor_id());
 	}

@@ -440,7 +440,7 @@
 {
 	unsigned long cpu = smp_processor_id();
 
-	__flush_tlb_all();
+	local_flush_tlb_all();
 	if (cpu_tlbstate[cpu].state == TLBSTATE_LAZY)
 		leave_mm(cpu);
 }

@@ -450,7 +450,7 @@
 	do_flush_tlb_all_local();
 }
 
-void flush_tlb_all(void)
+void smp_flush_tlb_all(void)
 {
 	smp_call_function (flush_tlb_all_ipi,0,1,1);
arch/i386/mach-voyager/voyager_smp.c +7 -7
diff -urP linux-2.5.59/arch/i386/mach-voyager/voyager_smp.c linux-2.5.59_patched/arch/i386/mach-voyager/voyager_smp.c
@@ -892,7 +892,7 @@
 			if (flush_va == FLUSH_ALL)
 				local_flush_tlb();
 			else
-				__flush_tlb_one(flush_va);
+				local_flush_tlb_one(flush_va);
 		} else
 			leave_mm(cpu);
 	}

@@ -948,7 +948,7 @@
 }
 
 void
-flush_tlb_current_task(void)
+smp_flush_tlb(void)
 {
 	struct mm_struct *mm = current->mm;
 	unsigned long cpu_mask;

@@ -965,7 +965,7 @@
 
 
 void
-flush_tlb_mm (struct mm_struct * mm)
+smp_flush_tlb_mm (struct mm_struct * mm)
 {
 	unsigned long cpu_mask;
 

@@ -985,7 +985,7 @@
 	preempt_enable();
 }
 
-void flush_tlb_page(struct vm_area_struct * vma, unsigned long va)
+void smp_flush_tlb_page(struct vm_area_struct * vma, unsigned long va)
 {
 	struct mm_struct *mm = vma->vm_mm;
 	unsigned long cpu_mask;

@@ -995,7 +995,7 @@
 	cpu_mask = mm->cpu_vm_mask & ~(1 << smp_processor_id());
 	if (current->active_mm == mm) {
 		if(current->mm)
-			__flush_tlb_one(va);
+			local_flush_tlb_one(va);
 		 else
 		 	leave_mm(smp_processor_id());
 	}

@@ -1217,7 +1217,7 @@
 {
 	unsigned long cpu = smp_processor_id();
 
-	__flush_tlb_all();
+	local_flush_tlb_all();
 	if (cpu_tlbstate[cpu].state == TLBSTATE_LAZY)
 		leave_mm(cpu);
 }

@@ -1231,7 +1231,7 @@
 
 /* flush the TLB of every active CPU in the system */
 void
-flush_tlb_all(void)
+smp_flush_tlb_all(void)
 {
 	smp_call_function (flush_tlb_all_function, 0, 1, 1);
arch/i386/mm/highmem.c +2 -2
diff -urP linux-2.5.59/arch/i386/mm/highmem.c linux-2.5.59_patched/arch/i386/mm/highmem.c
@@ -42,7 +42,7 @@
 		BUG();
 #endif
 	set_pte(kmap_pte-idx, mk_pte(page, kmap_prot));
-	__flush_tlb_one(vaddr);
+	local_flush_tlb_one(vaddr);
 
 	return (void*) vaddr;
 }

@@ -66,7 +66,7 @@
 	 * this pte without first remap it
 	 */
 	pte_clear(kmap_pte-idx);
-	__flush_tlb_one(vaddr);
+	local_flush_tlb_one(vaddr);
 #endif
 
 	dec_preempt_count();
arch/i386/mm/init.c +1 -1
diff -urP linux-2.5.59/arch/i386/mm/init.c linux-2.5.59_patched/arch/i386/mm/init.c
@@ -369,7 +369,7 @@
 	if (cpu_has_pae)
 		set_in_cr4(X86_CR4_PAE);
 #endif
-	__flush_tlb_all();
+	local_flush_tlb_all();
 
 	kmap_init();
 	zone_sizes_init();
arch/i386/mm/pageattr.c +1 -1
diff -urP linux-2.5.59/arch/i386/mm/pageattr.c linux-2.5.59_patched/arch/i386/mm/pageattr.c
@@ -53,7 +53,7 @@
 	/* Flush all to work around Errata in early athlons regarding 
 	 * large page flushing. 
 	 */
-	__flush_tlb_all(); 	
+	local_flush_tlb_all(); 	
 }
 
 static void set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte)
arch/i386/mm/pgtable.c +2 -2
diff -urP linux-2.5.59/arch/i386/mm/pgtable.c linux-2.5.59_patched/arch/i386/mm/pgtable.c
@@ -81,7 +81,7 @@
 	 * It's enough to flush this one mapping.
 	 * (PGE mappings get flushed as well)
 	 */
-	__flush_tlb_one(vaddr);
+	local_flush_tlb_one(vaddr);
 }
 
 /*

@@ -114,7 +114,7 @@
 	 * It's enough to flush this one mapping.
 	 * (PGE mappings get flushed as well)
 	 */
-	__flush_tlb_one(vaddr);
+	local_flush_tlb_one(vaddr);
 }
 
 void __set_fixmap (enum fixed_addresses idx, unsigned long phys, pgprot_t flags)
include/asm-i386/tlbflush.h +70 -24

164 lines not on this page · raw message

#1 lore raw
Quoted 6 lines · Thomas Schlichter

Hello,

with this mail I send a patch that allows kernel modules to hook into the different flush_tlb* functions defined in <asm/tlbflush.h> or <asm/pgtable.h> in order to synchronize devices TLBs.

Looks sensible enough.

A few coding-style nits:

	+typedef struct tlb_hook_struct {
	+...
	+} tlb_hook_t;

typedefs are unpopular. Please just use

	struct tlb_hook {
		...
	};

+static inline void flush_tlb_hook( void )

extraneous whitespace - Linus style is flush_tlb_hook(void)

+ while( hook )

while (hook)

+ {
+ if( hook->flush_tlb )

		if (hook->flush_tlb)

+ hook->flush_tlb( );

			hook->flush_tlb();

etc...

The unregister_hook implementation is racy - the hook could be in use on another CPU. That's OK - we have the RCU infrastructure which will allow hooks to be torn down safely. And nice people who can help others who are using that code.

Quoted 2 lines

The i386 patch also includes some cleanups by renaming __flush_tlb_* to local_flush_tlb_*.

That makes plenty of sense.

Quoted 2 lines

I hope some time this patches will make it into the kernel sources. (perhaps even into 2.6.x ?)

Well the big questions are: where are the drivers for these devices? When can we expect to see significant demand for these devices? Will there be significant demand across the lifetime of the 2.6 kernel?

BTW, when you say "low latency NICs that will implement direct user space DMA transfers to not pinned user pages", what do you mean by "not pinned"?

#3 lore raw
In reply to Andrew Morton

Hi,

Quoted 27 lines · Andrew Morton

A few coding-style nits:

	+typedef struct tlb_hook_struct {
	+...
	+} tlb_hook_t;

typedefs are unpopular. Please just use

	struct tlb_hook {
		...
	};

+static inline void flush_tlb_hook( void )

extraneous whitespace - Linus style is flush_tlb_hook(void)

(quoted diff) +1 -0
+	while( hook )

while (hook)

(quoted diff) +2 -0
+	{
+		if( hook->flush_tlb )

		if (hook->flush_tlb)
(quoted diff) +1 -0
+			hook->flush_tlb( );
Quoted 3 lines
			hook->flush_tlb();

etc...

Thanks for these nits, they should be fixed in the new attached version of the patches...

Quoted 4 lines

The unregister_hook implementation is racy - the hook could be in use on another CPU. That's OK - we have the RCU infrastructure which will allow hooks to be torn down safely. And nice people who can help others who are using that code.

I do not know what you ment with the RCU infrastructure, but the unregister-race should be fixed in the new patches, I hope...

Quoted 4 lines
Quoted 2 lines

The i386 patch also includes some cleanups by renaming __flush_tlb_* to local_flush_tlb_*.

That makes plenty of sense.

Fine... ;-)

Quoted 6 lines
Quoted 2 lines

I hope some time this patches will make it into the kernel sources. (perhaps even into 2.6.x ?)

Well the big questions are: where are the drivers for these devices? When can we expect to see significant demand for these devices? Will there be significant demand across the lifetime of the 2.6 kernel?

Well, I cannot say this for sure as we just do universitary research, but I really think this may be interesting even for companies which already create their own kernel patches for existing products...

Quoted 2 lines

BTW, when you say "low latency NICs that will implement direct user space DMA transfers to not pinned user pages", what do you mean by "not pinned"?

Well, with "not pinned" I mean the users memory pages need not to be pinned before the transfer begins...

If the pages are not present they have to be swapped in by the kernel and be pinned for the transfer, else the NIC may pin the page itself and start the transfer.

The address translation may be performed directly from the network device or via an interrupt in the kernel. The translation is then stored in the devices TLB which may be flushed when the page is unpinned (then this patch is not neccessary) or just if the translation is invalidated from the kernel (then this patch is neccessary).

The second approach has the advantage that the TLB hit ratio can be improved without having many user-pages pinned, and so removed from the memory pool, for a long time...

Regards
Thomas Schlichter

include/linux/tlbhook.h +151 -0
diff -urP linux-2.5.59/include/linux/tlbhook.h linux-2.5.59_patched/include/linux/tlbhook.h
@@ -0,0 +1,151 @@
+#ifndef _LINUX_TLBHOOK_H
+#define _LINUX_TLBHOOK_H
+
+#include <linux/mm.h>
+
+struct tlb_hook_struct {
+	void (*flush_tlb)(void);
+	void (*flush_tlb_all)(void);
+	void (*flush_tlb_mm)(struct mm_struct *mm);
+	void (*flush_tlb_page)(struct vm_area_struct *vma,
+				unsigned long addr);
+	void (*flush_tlb_range)(struct vm_area_struct *vma,
+				unsigned long start, unsigned long end);
+	void (*flush_tlb_kernel_range)(unsigned long start,
+				unsigned long end);
+	void (*flush_tlb_pgtables)(struct mm_struct *mm,
+				unsigned long start, unsigned long end);
+
+	struct tlb_hook_struct *next;
+	struct tlb_hook_struct *last;
+        volatile int used;
+};
+
+extern struct tlb_hook_struct *tlb_hook_root;
+
+extern int register_tlb_hook(struct tlb_hook_struct *hook);
+extern int unregister_tlb_hook(struct tlb_hook_struct *hook);
+
+static inline void flush_tlb_hook(void)
+{
+	struct tlb_hook_struct *hook = tlb_hook_root;
+
+	while(hook)
+	{
+		if(hook->flush_tlb)
+		{
+			hook->used = 1;
+			hook->flush_tlb();
+			hook->used = 0;
+		}
+		hook = hook->next;
+	}
+}
+
+
+static inline void flush_tlb_all_hook(void)
+{
+	struct tlb_hook_struct *hook = tlb_hook_root;
+
+	while(hook)
+	{
+		if(hook->flush_tlb_all)
+		{
+			hook->used = 1;
+			hook->flush_tlb_all();
+			hook->used = 0;
+		}
+		hook = hook->next;
+	}
+}
+
+
+static inline void flush_tlb_mm_hook(struct mm_struct *mm)
+{
+	struct tlb_hook_struct *hook = tlb_hook_root;
+
+	while(hook)
+	{
+		if(hook->flush_tlb_mm)
+		{
+			hook->used = 1;
+			hook->flush_tlb_mm(mm);
+			hook->used = 0;
+		}
+		hook = hook->next;
+	}
+}
+
+
+static inline void flush_tlb_page_hook(struct vm_area_struct *vma,
+				unsigned long addr)
+{
+	struct tlb_hook_struct *hook = tlb_hook_root;
+
+	while(hook)
+	{
+		if(hook->flush_tlb_page)
+		{
+			hook->used = 1;
+			hook->flush_tlb_page(vma, addr);
+			hook->used = 0;
+		}
+		hook = hook->next;
+	}
+}
+
+
+static inline void flush_tlb_range_hook(struct vm_area_struct *vma,
+				unsigned long start, unsigned long end)
+{
+	struct tlb_hook_struct *hook = tlb_hook_root;
+
+	while(hook)
+	{
+		if(hook->flush_tlb_range)
+		{
+			hook->used = 1;
+			hook->flush_tlb_range(vma, start, end);
+			hook->used = 0;
+		}
+		hook = hook->next;
+	}
+}
+
+
+static inline void flush_tlb_kernel_range_hook(unsigned long start,
+				unsigned long end)
+{
+	struct tlb_hook_struct *hook = tlb_hook_root;
+
+	while(hook)
+	{
+		if(hook->flush_tlb_kernel_range)
+		{
+			hook->used = 1;
+			hook->flush_tlb_kernel_range(start, end);
+			hook->used = 0;
+		}
+		hook = hook->next;
+	}
+}
+
+
+static inline void flush_tlb_pgtables_hook(struct mm_struct *mm,
+				unsigned long start, unsigned long end)
+{
+	struct tlb_hook_struct *hook = tlb_hook_root;
+
+	while(hook)
+	{
+		if(hook->flush_tlb_pgtables)
+		{
+			hook->used = 1;
+			hook->flush_tlb_pgtables(mm, start, end);
+			hook->used = 0;
+		}
+		hook = hook->next;
+	}
+}
+
+#endif /* _LINUX_TLBHOOK_H */
mm/Makefile +3 -2
diff -urP linux-2.5.59/mm/Makefile linux-2.5.59_patched/mm/Makefile
@@ -2,7 +2,8 @@
 # Makefile for the linux memory manager.
 #
 
-export-objs := shmem.o filemap.o mempool.o page_alloc.o page-writeback.o
+export-objs 		:= shmem.o filemap.o mempool.o page_alloc.o page-writeback.o \
+			   tlbhook.o
 
 mmu-y			:= nommu.o
 mmu-$(CONFIG_MMU)	:= fremap.o highmem.o madvise.o memory.o mincore.o \

@@ -11,6 +12,6 @@
 
 obj-y			:= bootmem.o filemap.o mempool.o oom_kill.o \
 			   page_alloc.o page-writeback.o pdflush.o readahead.o \
-			   slab.o swap.o truncate.o vcache.o vmscan.o $(mmu-y)
+			   slab.o swap.o tlbhook.o truncate.o vcache.o vmscan.o $(mmu-y)
 
 obj-$(CONFIG_SWAP)	+= page_io.o swap_state.o swapfile.o
mm/tlbhook.c +68 -0

68 lines not on this page · raw message

arch/i386/kernel/smp.c +7 -7
diff -urP linux-2.5.59/arch/i386/kernel/smp.c linux-2.5.59_patched/arch/i386/kernel/smp.c
@@ -327,7 +327,7 @@
 			if (flush_va == FLUSH_ALL)
 				local_flush_tlb();
 			else
-				__flush_tlb_one(flush_va);
+				local_flush_tlb_one(flush_va);
 		} else
 			leave_mm(cpu);
 	}

@@ -382,7 +382,7 @@
 	spin_unlock(&tlbstate_lock);
 }
 	
-void flush_tlb_current_task(void)
+void smp_flush_tlb(void)
 {
 	struct mm_struct *mm = current->mm;
 	unsigned long cpu_mask;

@@ -396,7 +396,7 @@
 	preempt_enable();
 }
 
-void flush_tlb_mm (struct mm_struct * mm)
+void smp_flush_tlb_mm(struct mm_struct * mm)
 {
 	unsigned long cpu_mask;
 

@@ -415,7 +415,7 @@
 	preempt_enable();
 }
 
-void flush_tlb_page(struct vm_area_struct * vma, unsigned long va)
+void smp_flush_tlb_page(struct vm_area_struct * vma, unsigned long va)
 {
 	struct mm_struct *mm = vma->vm_mm;
 	unsigned long cpu_mask;

@@ -425,7 +425,7 @@
 
 	if (current->active_mm == mm) {
 		if(current->mm)
-			__flush_tlb_one(va);
+			local_flush_tlb_one(va);
 		 else
 		 	leave_mm(smp_processor_id());
 	}

@@ -440,7 +440,7 @@
 {
 	unsigned long cpu = smp_processor_id();
 
-	__flush_tlb_all();
+	local_flush_tlb_all();
 	if (cpu_tlbstate[cpu].state == TLBSTATE_LAZY)
 		leave_mm(cpu);
 }

@@ -450,7 +450,7 @@
 	do_flush_tlb_all_local();
 }
 
-void flush_tlb_all(void)
+void smp_flush_tlb_all(void)
 {
 	smp_call_function (flush_tlb_all_ipi,0,1,1);
arch/i386/mach-voyager/voyager_smp.c +7 -7
diff -urP linux-2.5.59/arch/i386/mach-voyager/voyager_smp.c linux-2.5.59_patched/arch/i386/mach-voyager/voyager_smp.c
@@ -892,7 +892,7 @@
 			if (flush_va == FLUSH_ALL)
 				local_flush_tlb();
 			else
-				__flush_tlb_one(flush_va);
+				local_flush_tlb_one(flush_va);
 		} else
 			leave_mm(cpu);
 	}

@@ -948,7 +948,7 @@
 }
 
 void
-flush_tlb_current_task(void)
+smp_flush_tlb(void)
 {
 	struct mm_struct *mm = current->mm;
 	unsigned long cpu_mask;

@@ -965,7 +965,7 @@
 
 
 void
-flush_tlb_mm (struct mm_struct * mm)
+smp_flush_tlb_mm(struct mm_struct * mm)
 {
 	unsigned long cpu_mask;
 

@@ -985,7 +985,7 @@
 	preempt_enable();
 }
 
-void flush_tlb_page(struct vm_area_struct * vma, unsigned long va)
+void smp_flush_tlb_page(struct vm_area_struct * vma, unsigned long va)
 {
 	struct mm_struct *mm = vma->vm_mm;
 	unsigned long cpu_mask;

@@ -995,7 +995,7 @@
 	cpu_mask = mm->cpu_vm_mask & ~(1 << smp_processor_id());
 	if (current->active_mm == mm) {
 		if(current->mm)
-			__flush_tlb_one(va);
+			local_flush_tlb_one(va);
 		 else
 		 	leave_mm(smp_processor_id());
 	}

@@ -1217,7 +1217,7 @@
 {
 	unsigned long cpu = smp_processor_id();
 
-	__flush_tlb_all();
+	local_flush_tlb_all();
 	if (cpu_tlbstate[cpu].state == TLBSTATE_LAZY)
 		leave_mm(cpu);
 }

@@ -1231,7 +1231,7 @@
 
 /* flush the TLB of every active CPU in the system */
 void
-flush_tlb_all(void)
+smp_flush_tlb_all(void)
 {
 	smp_call_function (flush_tlb_all_function, 0, 1, 1);
arch/i386/mm/highmem.c +2 -2
diff -urP linux-2.5.59/arch/i386/mm/highmem.c linux-2.5.59_patched/arch/i386/mm/highmem.c
@@ -42,7 +42,7 @@
 		BUG();
 #endif
 	set_pte(kmap_pte-idx, mk_pte(page, kmap_prot));
-	__flush_tlb_one(vaddr);
+	local_flush_tlb_one(vaddr);
 
 	return (void*) vaddr;
 }

@@ -66,7 +66,7 @@
 	 * this pte without first remap it
 	 */
 	pte_clear(kmap_pte-idx);
-	__flush_tlb_one(vaddr);
+	local_flush_tlb_one(vaddr);
 #endif
 
 	dec_preempt_count();
arch/i386/mm/init.c +1 -1
diff -urP linux-2.5.59/arch/i386/mm/init.c linux-2.5.59_patched/arch/i386/mm/init.c
@@ -369,7 +369,7 @@
 	if (cpu_has_pae)
 		set_in_cr4(X86_CR4_PAE);
 #endif
-	__flush_tlb_all();
+	local_flush_tlb_all();
 
 	kmap_init();
 	zone_sizes_init();
arch/i386/mm/pageattr.c +1 -1
diff -urP linux-2.5.59/arch/i386/mm/pageattr.c linux-2.5.59_patched/arch/i386/mm/pageattr.c
@@ -53,7 +53,7 @@
 	/* Flush all to work around Errata in early athlons regarding 
 	 * large page flushing. 
 	 */
-	__flush_tlb_all(); 	
+	local_flush_tlb_all(); 	
 }
 
 static void set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte)
arch/i386/mm/pgtable.c +2 -2
diff -urP linux-2.5.59/arch/i386/mm/pgtable.c linux-2.5.59_patched/arch/i386/mm/pgtable.c
@@ -81,7 +81,7 @@
 	 * It's enough to flush this one mapping.
 	 * (PGE mappings get flushed as well)
 	 */
-	__flush_tlb_one(vaddr);
+	local_flush_tlb_one(vaddr);
 }
 
 /*

@@ -114,7 +114,7 @@
 	 * It's enough to flush this one mapping.
 	 * (PGE mappings get flushed as well)
 	 */
-	__flush_tlb_one(vaddr);
+	local_flush_tlb_one(vaddr);
 }
 
 void __set_fixmap (enum fixed_addresses idx, unsigned long phys, pgprot_t flags)
include/asm-i386/tlbflush.h +70 -24

164 lines not on this page · raw message