Backport of Ingo/Arjan highpte to 2.4.18 (+O1 scheduler)

PATCH Martin J. Bligh 2 participants 9 messages Mar 21, 2002 – Mar 31, 2002

Subsystems (14)

Memory management VFS USB MM core +10 more subsystems

lore

Share & tools

lore

Martin J. Bligh

author

2002-03-21 23:12 UTC #0 lore raw

The smaller two patches they published against 2.5.5 work fine, but the main one needed a little work - this seems to work for me, but I've only really touch tested it ... just thought this might save someone some time .... there didn't seem to be anything too complex in the merge, but that doesn't necessarily mean I got it right ;-)

arch/i386/config.in +17 -4

diff -urN linux-2.4.18-prepte/arch/i386/config.in linux-2.4.18-highpte/arch/i386/config.in
@@ -171,16 +171,27 @@
 tristate '/dev/cpu/*/cpuid - CPU information support' CONFIG_X86_CPUID
 
 choice 'High Memory Support' \
-	"off    CONFIG_NOHIGHMEM \
-	 4GB    CONFIG_HIGHMEM4G \
-	 64GB   CONFIG_HIGHMEM64G" off
+	"off           CONFIG_NOHIGHMEM \
+	 4GB           CONFIG_HIGHMEM4G \
+	 4GB-highpte   CONFIG_HIGHMEM4G_HIGHPTE \
+	 64GB          CONFIG_HIGHMEM64G \
+	 64GB-highpte  CONFIG_HIGHMEM64G_HIGHPTE" off
 if [ "$CONFIG_HIGHMEM4G" = "y" ]; then
    define_bool CONFIG_HIGHMEM y
 fi
+if [ "$CONFIG_HIGHMEM4G_HIGHPTE" = "y" ]; then
+   define_bool CONFIG_HIGHMEM y
+   define_bool CONFIG_HIGHPTE y
+fi
 if [ "$CONFIG_HIGHMEM64G" = "y" ]; then
    define_bool CONFIG_HIGHMEM y
    define_bool CONFIG_X86_PAE y
 fi
+if [ "$CONFIG_HIGHMEM64G_HIGHPTE" = "y" ]; then
+   define_bool CONFIG_HIGHMEM y
+   define_bool CONFIG_HIGHPTE y
+   define_bool CONFIG_X86_PAE y
+fi
 
 bool 'Math emulation' CONFIG_MATH_EMULATION
 bool 'MTRR (Memory Type Range Register) support' CONFIG_MTRR

@@ -416,11 +427,13 @@
 
 bool 'Kernel debugging' CONFIG_DEBUG_KERNEL
 if [ "$CONFIG_DEBUG_KERNEL" != "n" ]; then
-   bool '  Debug high memory support' CONFIG_DEBUG_HIGHMEM
    bool '  Debug memory allocations' CONFIG_DEBUG_SLAB
    bool '  Memory mapped I/O debugging' CONFIG_DEBUG_IOVIRT
    bool '  Magic SysRq key' CONFIG_MAGIC_SYSRQ
    bool '  Spinlock debugging' CONFIG_DEBUG_SPINLOCK
+   if [ "$CONFIG_HIGHMEM" = "y" ]; then
+      bool '  Highmem debugging' CONFIG_DEBUG_HIGHMEM
+   fi
    bool '  Verbose BUG() reporting (adds 70K)' CONFIG_DEBUG_BUGVERBOSE
 fi

arch/i386/kernel/process.c +0 -1

diff -urN linux-2.4.18-prepte/arch/i386/kernel/process.c linux-2.4.18-highpte/arch/i386/kernel/process.c
@@ -132,7 +132,6 @@
 		if (!current->need_resched)
 			idle();
 		schedule();
-		check_pgt_cache();
 	}
 }

arch/i386/kernel/smpboot.c +0 -4

diff -urN linux-2.4.18-prepte/arch/i386/kernel/smpboot.c linux-2.4.18-highpte/arch/i386/kernel/smpboot.c
@@ -144,10 +144,6 @@
 	struct cpuinfo_x86 *c = cpu_data + id;
 
 	*c = boot_cpu_data;
-	c->pte_quick = 0;
-	c->pmd_quick = 0;
-	c->pgd_quick = 0;
-	c->pgtable_cache_sz = 0;
 	identify_cpu(c);
 	/*
 	 * Mask B, Pentium, but not Pentium MMX

arch/i386/kernel/traps.c +1 -1

diff -urN linux-2.4.18-prepte/arch/i386/kernel/traps.c linux-2.4.18-highpte/arch/i386/kernel/traps.c
@@ -734,7 +734,7 @@
 	page = (unsigned long) vmalloc(PAGE_SIZE);
 	pgd = pgd_offset(&init_mm, page);
 	pmd = pmd_offset(pgd, page);
-	pte = pte_offset(pmd, page);
+	pte = pte_offset_kernel(pmd, page);
 	__free_page(pte_page(*pte));
 	*pte = mk_pte_phys(__pa(&idt_table), PAGE_KERNEL_RO);
 	/*

arch/i386/kernel/vm86.c +5 -2

diff -urN linux-2.4.18-prepte/arch/i386/kernel/vm86.c linux-2.4.18-highpte/arch/i386/kernel/vm86.c
@@ -94,7 +94,7 @@
 {
 	pgd_t *pgd;
 	pmd_t *pmd;
-	pte_t *pte;
+	pte_t *pte, *mapped;
 	int i;
 
 	pgd = pgd_offset(tsk->mm, 0xA0000);

@@ -113,12 +113,15 @@
 		pmd_clear(pmd);
 		return;
 	}
-	pte = pte_offset(pmd, 0xA0000);
+	preempt_disable();
+	pte = mapped = pte_offset_map(pmd, 0xA0000);
 	for (i = 0; i < 32; i++) {
 		if (pte_present(*pte))
 			set_pte(pte, pte_wrprotect(*pte));
 		pte++;
 	}
+	pte_unmap(mapped);
+	preempt_enable();
 	flush_tlb();
 }

arch/i386/mm/fault.c +9 -1

diff -urN linux-2.4.18-prepte/arch/i386/mm/fault.c linux-2.4.18-highpte/arch/i386/mm/fault.c
@@ -324,12 +324,20 @@
 	asm("movl %%cr3,%0":"=r" (page));
 	page = ((unsigned long *) __va(page))[address >> 22];
 	printk(KERN_ALERT "*pde = %08lx\n", page);
+	/*
+	 * We must not directly access the pte in the highpte
+	 * case, the page table might be allocated in highmem.
+	 * And lets rather not kmap-atomic the pte, just in case
+	 * it's allocated already.
+	 */
+#ifndef CONFIG_HIGHPTE
 	if (page & 1) {
 		page &= PAGE_MASK;
 		address &= 0x003ff000;
 		page = ((unsigned long *) __va(page))[address >> PAGE_SHIFT];
 		printk(KERN_ALERT "*pte = %08lx\n", page);
 	}
+#endif
 	die("Oops", regs, error_code);
 	bust_spinlocks(0);
 	do_exit(SIGKILL);

@@ -399,7 +407,7 @@
 			goto no_context;
 		set_pmd(pmd, *pmd_k);
 
-		pte_k = pte_offset(pmd_k, address);
+		pte_k = pte_offset_kernel(pmd_k, address);
 		if (!pte_present(*pte_k))
 			goto no_context;
 		return;

arch/i386/mm/init.c +6 ~~-29~~

diff -urN linux-2.4.18-prepte/arch/i386/mm/init.c linux-2.4.18-highpte/arch/i386/mm/init.c
@@ -43,28 +43,6 @@
 static unsigned long totalram_pages;
 static unsigned long totalhigh_pages;
 
-int do_check_pgt_cache(int low, int high)
-{
-	int freed = 0;
-	if(pgtable_cache_size > high) {
-		do {
-			if (pgd_quicklist) {
-				free_pgd_slow(get_pgd_fast());
-				freed++;
-			}
-			if (pmd_quicklist) {
-				pmd_free_slow(pmd_alloc_one_fast(NULL, 0));
-				freed++;
-			}
-			if (pte_quicklist) {
-				pte_free_slow(pte_alloc_one_fast(NULL, 0));
-				freed++;
-			}
-		} while(pgtable_cache_size > low);
-	}
-	return freed;
-}
-
 /*
  * NOTE: pagetable_init alloc all the fixmap pagetables contiguous on the
  * physical space so we can cache the place of the first one and move

@@ -76,7 +54,7 @@
 pgprot_t kmap_prot;
 
 #define kmap_get_fixmap_pte(vaddr)					\
-	pte_offset(pmd_offset(pgd_offset_k(vaddr), (vaddr)), (vaddr))
+	pte_offset_kernel(pmd_offset(pgd_offset_k(vaddr), (vaddr)), (vaddr))
 
 void __init kmap_init(void)
 {

@@ -116,7 +94,6 @@
 	printk("%d reserved pages\n",reserved);
 	printk("%d pages shared\n",shared);
 	printk("%d pages swap cached\n",cached);
-	printk("%ld pages in page table cache\n",pgtable_cache_size);
 	show_buffers();
 }

@@ -143,7 +120,7 @@
 		printk("PAE BUG #01!\n");
 		return;
 	}
-	pte = pte_offset(pmd, vaddr);
+	pte = pte_offset_kernel(pmd, vaddr);
 	if (pte_val(*pte))
 		pte_ERROR(*pte);
 	pgprot_val(prot) = pgprot_val(PAGE_KERNEL) | pgprot_val(flags);

@@ -196,7 +173,7 @@
 			if (pmd_none(*pmd)) {
 				pte = (pte_t *) alloc_bootmem_low_pages(PAGE_SIZE);
 				set_pmd(pmd, __pmd(_KERNPG_TABLE + __pa(pte)));
-				if (pte != pte_offset(pmd, 0))
+				if (pte != pte_offset_kernel(pmd, 0))
 					BUG();
 			}
 			vaddr += PMD_SIZE;

@@ -267,7 +244,7 @@
 				*pte = mk_pte_phys(__pa(vaddr), PAGE_KERNEL);
 			}
 			set_pmd(pmd, __pmd(_KERNPG_TABLE + __pa(pte_base)));
-			if (pte_base != pte_offset(pmd, 0))
+			if (pte_base != pte_offset_kernel(pmd, 0))
 				BUG();
 
 		}

@@ -289,7 +266,7 @@
 
 	pgd = swapper_pg_dir + __pgd_offset(vaddr);
 	pmd = pmd_offset(pgd, vaddr);
-	pte = pte_offset(pmd, vaddr);
+	pte = pte_offset_kernel(pmd, vaddr);
 	pkmap_page_table = pte;
 #endif

@@ -398,7 +375,7 @@
 
 	pgd = swapper_pg_dir + __pgd_offset(vaddr);
 	pmd = pmd_offset(pgd, vaddr);
-	pte = pte_offset(pmd, vaddr);
+	pte = pte_offset_kernel(pmd, vaddr);
 	old_pte = *pte;
 	*pte = mk_pte_phys(0, PAGE_READONLY);
 	local_flush_tlb();

arch/i386/mm/ioremap.c +1 -1

diff -urN linux-2.4.18-prepte/arch/i386/mm/ioremap.c linux-2.4.18-highpte/arch/i386/mm/ioremap.c
@@ -49,7 +49,7 @@
 	if (address >= end)
 		BUG();
 	do {
-		pte_t * pte = pte_alloc(&init_mm, pmd, address);
+		pte_t * pte = pte_alloc_kernel(&init_mm, pmd, address);
 		if (!pte)
 			return -ENOMEM;
 		remap_area_pte(pte, address, end - address, address + phys_addr, flags);

drivers/char/drm/drm_proc.h +4 -1

diff -urN linux-2.4.18-prepte/drivers/char/drm/drm_proc.h linux-2.4.18-highpte/drivers/char/drm/drm_proc.h
@@ -449,7 +449,8 @@
 		for (i = vma->vm_start; i < vma->vm_end; i += PAGE_SIZE) {
 			pgd = pgd_offset(vma->vm_mm, i);
 			pmd = pmd_offset(pgd, i);
-			pte = pte_offset(pmd, i);
+			preempt_disable();
+			pte = pte_offset_map(pmd, i);
 			if (pte_present(*pte)) {
 				address = __pa(pte_page(*pte))
 					+ (i & (PAGE_SIZE-1));

@@ -465,6 +466,8 @@
 			} else {
 				DRM_PROC_PRINT("      0x%08lx\n", i);
 			}
+			pte_unmap(pte);
+			preempt_enable();
 		}
 #endif
 	}

drivers/char/drm/drm_scatter.h +8 -2

diff -urN linux-2.4.18-prepte/drivers/char/drm/drm_scatter.h linux-2.4.18-highpte/drivers/char/drm/drm_scatter.h
@@ -143,9 +143,15 @@
 		if ( !pmd_present( *pmd ) )
 			goto failed;
 
-		pte = pte_offset( pmd, i );
-		if ( !pte_present( *pte ) )
+		preempt_disable();
+		pte = pte_offset_map( pmd, i );
+		if ( !pte_present( *pte ) ) {
+			pte_unmap(pte);
+			preempt_enable();
 			goto failed;
+		}
+		pte_unmap(pte);
+		preempt_enable();
 
 		entry->pagelist[j] = pte_page( *pte );

drivers/char/drm/drm_vm.h +9 -2

diff -urN linux-2.4.18-prepte/drivers/char/drm/drm_vm.h linux-2.4.18-highpte/drivers/char/drm/drm_vm.h
@@ -169,8 +169,15 @@
 	if( !pgd_present( *pgd ) ) return NOPAGE_OOM;
 	pmd = pmd_offset( pgd, i );
 	if( !pmd_present( *pmd ) ) return NOPAGE_OOM;
-	pte = pte_offset( pmd, i );
-	if( !pte_present( *pte ) ) return NOPAGE_OOM;
+	preempt_disable();
+	pte = pte_offset_map( pmd, i );
+	if( !pte_present( *pte ) ) {
+		pte_unmap(pte);
+		preempt_enable();
+		return NOPAGE_OOM;
+	}
+	pte_unmap(pte);
+	preempt_enable();
 
 	page = pte_page(*pte);
 	get_page(page);

drivers/sgi/char/graphics.c +5 -2

diff -urN linux-2.4.18-prepte/drivers/sgi/char/graphics.c linux-2.4.18-highpte/drivers/sgi/char/graphics.c
@@ -221,6 +221,7 @@
 	int board = GRAPHICS_CARD (vma->vm_dentry->d_inode->i_rdev);
 
 	unsigned long virt_add, phys_add;
+	struct page * page;
 
 #ifdef DEBUG
 	printk ("Got a page fault for board %d address=%lx guser=%lx\n", board,

@@ -247,8 +248,10 @@
 
 	pgd = pgd_offset(current->mm, address);
 	pmd = pmd_offset(pgd, address);
-	pte = pte_offset(pmd, address);
-	return pte_page(*pte);
+	pte = pte_kmap_offset(pmd, address);
+	page = pte_page(*pte);
+	pte_kunmap(pte);
+	return page;
 }
 
 /*

drivers/usb/stv680.c +4 -1

diff -urN linux-2.4.18-prepte/drivers/usb/stv680.c linux-2.4.18-highpte/drivers/usb/stv680.c
@@ -133,8 +133,11 @@
 	if (!pgd_none (*pgd)) {
 		pmd = pmd_offset (pgd, adr);
 		if (!pmd_none (*pmd)) {
-			ptep = pte_offset (pmd, adr);
+			preempt_disable();
+			ptep = pte_offset_map (pmd, adr);
 			pte = *ptep;
+			pte_unmap(pte);
+			preempt_enable();
 			if (pte_present (pte)) {
 				ret = (unsigned long) page_address (pte_page (pte));
 				ret |= (adr & (PAGE_SIZE - 1));

drivers/usb/vicam.c +4 -1

diff -urN linux-2.4.18-prepte/drivers/usb/vicam.c linux-2.4.18-highpte/drivers/usb/vicam.c
@@ -115,8 +115,11 @@
 	if (!pgd_none(*pgd)) {
 		pmd = pmd_offset(pgd, adr);
 		if (!pmd_none(*pmd)) {
-			ptep = pte_offset(pmd, adr);
+			preempt_disable();
+			ptep = pte_offset_map(pmd, adr);
 			pte = *ptep;
+			pte_unmap(pte);
+			preempt_enable();
 			if(pte_present(pte)) {
 				ret  = (unsigned long) page_address(pte_page(pte));
 				ret |= (adr & (PAGE_SIZE - 1));

fs/exec.c +5 -2

diff -urN linux-2.4.18-prepte/fs/exec.c linux-2.4.18-highpte/fs/exec.c
@@ -270,15 +270,18 @@
 	pmd = pmd_alloc(tsk->mm, pgd, address);
 	if (!pmd)
 		goto out;
-	pte = pte_alloc(tsk->mm, pmd, address);
+	pte = pte_alloc_map(tsk->mm, pmd, address);
 	if (!pte)
 		goto out;
-	if (!pte_none(*pte))
+	if (!pte_none(*pte)) {
+		pte_unmap(pte);
 		goto out;
+	}
 	lru_cache_add(page);
 	flush_dcache_page(page);
 	flush_page_to_ram(page);
 	set_pte(pte, pte_mkdirty(pte_mkwrite(mk_pte(page, PAGE_COPY))));
+	pte_unmap(pte);
 	tsk->mm->rss++;
 	spin_unlock(&tsk->mm->page_table_lock);

fs/proc/array.c +10 -8

diff -urN linux-2.4.18-prepte/fs/proc/array.c linux-2.4.18-highpte/fs/proc/array.c
@@ -392,11 +392,10 @@
 	return res;
 }
 		
-static inline void statm_pte_range(pmd_t * pmd, unsigned long address, unsigned long size,
-	int * pages, int * shared, int * dirty, int * total)
+static inline void statm_pte_range(pmd_t * pmd, unsigned long address, unsigned long size, int * pages, int * shared, int * dirty, int * total)
 {
-	pte_t * pte;
-	unsigned long end;
+	unsigned long end, pmd_end;
+	pte_t *pte;
 
 	if (pmd_none(*pmd))
 		return;

@@ -405,11 +404,12 @@
 		pmd_clear(pmd);
 		return;
 	}
-	pte = pte_offset(pmd, address);
-	address &= ~PMD_MASK;
+	preempt_disable();
+	pte = pte_offset_map(pmd, address);
 	end = address + size;
-	if (end > PMD_SIZE)
-		end = PMD_SIZE;
+	pmd_end = (address + PMD_SIZE) & PMD_MASK;
+	if (end > pmd_end)
+		end = pmd_end;
 	do {
 		pte_t page = *pte;
 		struct page *ptpage;

@@ -430,6 +430,8 @@
 		if (page_count(pte_page(page)) > 1)
 			++*shared;
 	} while (address < end);
+	pte_unmap(pte - 1);
+	preempt_enable();
 }
 
 static inline void statm_pmd_range(pgd_t * pgd, unsigned long address, unsigned long size,

include/asm-i386/highmem.h +3 -9

diff -urN linux-2.4.18-prepte/include/asm-i386/highmem.h linux-2.4.18-highpte/include/asm-i386/highmem.h
@@ -26,12 +26,6 @@
 #include <asm/kmap_types.h>
 #include <asm/pgtable.h>
 
-#ifdef CONFIG_DEBUG_HIGHMEM
-#define HIGHMEM_DEBUG 1
-#else
-#define HIGHMEM_DEBUG 0
-#endif
-
 /* declarations for highmem.c */
 extern unsigned long highstart_pfn, highend_pfn;

@@ -93,7 +87,7 @@
 
 	idx = type + KM_TYPE_NR*smp_processor_id();
 	vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
-#if HIGHMEM_DEBUG
+#if CONFIG_DEBUG_HIGHMEM
 	if (!pte_none(*(kmap_pte-idx)))
 		BUG();
 #endif

@@ -105,8 +99,8 @@
 
 static inline void kunmap_atomic(void *kvaddr, enum km_type type)
 {
-#if HIGHMEM_DEBUG
-	unsigned long vaddr = (unsigned long) kvaddr;
+#if CONFIG_DEBUG_HIGHMEM
+	unsigned long vaddr = (unsigned long) kvaddr & PAGE_MASK;
 	enum fixed_addresses idx = type + KM_TYPE_NR*smp_processor_id();
 
 	if (vaddr < FIXADDR_START) // FIXME

include/asm-i386/kmap_types.h +19 -6

diff -urN linux-2.4.18-prepte/include/asm-i386/kmap_types.h linux-2.4.18-highpte/include/asm-i386/kmap_types.h
@@ -1,13 +1,26 @@
 #ifndef _ASM_KMAP_TYPES_H
 #define _ASM_KMAP_TYPES_H
 
+#include <linux/config.h>
+
+#if CONFIG_DEBUG_HIGHMEM
+# define D(n) __KM_FENCE_##n ,
+#else
+# define D(n)
+#endif
+
 enum km_type {
-	KM_BOUNCE_READ,
-	KM_SKB_DATA,
-	KM_SKB_DATA_SOFTIRQ,
-	KM_USER0,
-	KM_USER1,
-	KM_TYPE_NR
+D(0)  KM_BOUNCE_READ,
+D(1)  KM_SKB_DATA,
+D(2)  KM_SKB_DATA_SOFTIRQ,
+D(3)  KM_USER0,
+D(4)  KM_USER1,
+D(5)  KM_BIO_IRQ,
+D(6)  KM_PTE0,
+D(7)  KM_PTE1,
+D(8)  KM_TYPE_NR
 };
+
+#undef D
 
 #endif

include/asm-i386/pgalloc.h +50 ~~-63~~

165 lines not on this page · raw message

include/asm-i386/pgtable.h +12 -3

diff -urN linux-2.4.18-prepte/include/asm-i386/pgtable.h linux-2.4.18-highpte/include/asm-i386/pgtable.h
@@ -315,9 +315,12 @@
 
 #define page_pte(page) page_pte_prot(page, __pgprot(0))
 
-#define pmd_page(pmd) \
+#define pmd_page_kernel(pmd) \
 ((unsigned long) __va(pmd_val(pmd) & PAGE_MASK))
 
+#define pmd_page(pmd) \
+	(mem_map + (pmd_val(pmd) >> PAGE_SHIFT))
+
 /* to find an entry in a page-table-directory. */
 #define pgd_index(address) ((address >> PGDIR_SHIFT) & (PTRS_PER_PGD-1))

@@ -334,8 +337,14 @@
 /* Find an entry in the third-level page table.. */
 #define __pte_offset(address) \
 		((address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1))
-#define pte_offset(dir, address) ((pte_t *) pmd_page(*(dir)) + \
-			__pte_offset(address))
+#define pte_offset_kernel(dir, address) \
+	((pte_t *) pmd_page_kernel(*(dir)) +  __pte_offset(address))
+#define pte_offset_map(dir, address) \
+	((pte_t *)kmap_atomic(pmd_page(*(dir)),KM_PTE0) + __pte_offset(address))
+#define pte_offset_map2(dir, address) \
+	((pte_t *)kmap_atomic(pmd_page(*(dir)),KM_PTE1) + __pte_offset(address))
+#define pte_unmap(pte) kunmap_atomic(pte, KM_PTE0)
+#define pte_unmap2(pte) kunmap_atomic(pte, KM_PTE1)
 
 /*
  * The i386 doesn't have any external MMU info: the kernel page

include/asm-i386/processor.h +0 -4

diff -urN linux-2.4.18-prepte/include/asm-i386/processor.h linux-2.4.18-highpte/include/asm-i386/processor.h
@@ -49,10 +49,6 @@
 	int	f00f_bug;
 	int	coma_bug;
 	unsigned long loops_per_jiffy;
-	unsigned long *pgd_quick;
-	unsigned long *pmd_quick;
-	unsigned long *pte_quick;
-	unsigned long pgtable_cache_sz;
 } __attribute__((__aligned__(SMP_CACHE_BYTES)));
 
 #define X86_VENDOR_INTEL 0

include/linux/highmem.h +9 -8

diff -urN linux-2.4.18-prepte/include/linux/highmem.h linux-2.4.18-highpte/include/linux/highmem.h
@@ -2,7 +2,6 @@
 #define _LINUX_HIGHMEM_H
 
 #include <linux/config.h>
-#include <asm/pgalloc.h>
 
 #ifdef CONFIG_HIGHMEM

@@ -14,7 +13,7 @@
 unsigned int nr_free_highpages(void);
 
 extern struct buffer_head * create_bounce(int rw, struct buffer_head * bh_orig);
-
+extern void check_highmem_ptes(void);
 
 static inline char *bh_kmap(struct buffer_head *bh)
 {

@@ -52,8 +51,9 @@
 
 static inline void clear_highpage(struct page *page)
 {
-	clear_page(kmap(page));
-	kunmap(page);
+	void *kaddr = kmap_atomic(page, KM_USER0);
+	clear_page(kaddr);
+	kunmap_atomic(kaddr, KM_USER0);
 }
 
 /*

@@ -61,15 +61,16 @@
  */
 static inline void memclear_highpage_flush(struct page *page, unsigned int offset, unsigned int size)
 {
-	char *kaddr;
+	void *kaddr;
 
 	if (offset + size > PAGE_SIZE)
 		BUG();
-	kaddr = kmap(page);
-	memset(kaddr + offset, 0, size);
+
+	kaddr = kmap_atomic(page, KM_USER0);
+	memset((char *)kaddr + offset, 0, size);
 	flush_dcache_page(page);
 	flush_page_to_ram(page);
-	kunmap(page);
+	kunmap_atomic(kaddr, KM_USER0);
 }
 
 static inline void copy_user_highpage(struct page *to, struct page *from, unsigned long vaddr)

include/linux/mm.h +3 -5

diff -urN linux-2.4.18-prepte/include/linux/mm.h linux-2.4.18-highpte/include/linux/mm.h
@@ -411,7 +411,8 @@
 
 extern int vmtruncate(struct inode * inode, loff_t offset);
 extern pmd_t *FASTCALL(__pmd_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address));
-extern pte_t *FASTCALL(pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address));
+extern pte_t *FASTCALL(pte_alloc_kernel(struct mm_struct *mm, pmd_t *pmd, unsigned long address));
+extern pte_t *FASTCALL(pte_alloc_map(struct mm_struct *mm, pmd_t *pmd, unsigned long address));
 extern int handle_mm_fault(struct mm_struct *mm,struct vm_area_struct *vma, unsigned long address, int write_access);
 extern int make_pages_present(unsigned long addr, unsigned long end);
 extern int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write);

@@ -427,7 +428,7 @@
 
 /*
  * On a two-level page table, this ends up being trivial. Thus the
- * inlining and the symmetry break with pte_alloc() that does all
+ * inlining and the symmetry break with pte_alloc_map() that does all
  * of this out-of-line.
  */
 static inline pmd_t *pmd_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)

@@ -436,9 +437,6 @@
 		return __pmd_alloc(mm, pgd, address);
 	return pmd_offset(pgd, address);
 }
-
-extern int pgt_cache_water[2];
-extern int check_pgt_cache(void);
 
 extern void free_area_init(unsigned long * zones_size);
 extern void free_area_init_node(int nid, pg_data_t *pgdat, struct page *pmap,

kernel/sched.c +5 -0

diff -urN linux-2.4.18-prepte/kernel/sched.c linux-2.4.18-highpte/kernel/sched.c
@@ -20,6 +20,7 @@
 #include <linux/interrupt.h>
 #include <asm/mmu_context.h>
 #include <linux/kernel_stat.h>
+#include <linux/highmem.h>
 
 /*
  * Priority of a process goes from 0 to 139. The 0-99

@@ -751,6 +752,10 @@
 
 	if (unlikely(in_interrupt()))
 		BUG();
+#if CONFIG_DEBUG_HIGHMEM
+	check_highmem_ptes();
+#endif
+
 	release_kernel_lock(prev, smp_processor_id());
 	prev->sleep_timestamp = jiffies;
 	spin_lock_irq(&rq->lock);

kernel/sysctl.c +0 -4

diff -urN linux-2.4.18-prepte/kernel/sysctl.c linux-2.4.18-highpte/kernel/sysctl.c
@@ -96,8 +96,6 @@
 extern int acct_parm[];
 #endif
 
-extern int pgt_cache_water[];
-
 static int parse_table(int *, int, void *, size_t *, void *, size_t,
 		       ctl_table *, void **);
 static int proc_doutsstring(ctl_table *table, int write, struct file *filp,

@@ -267,8 +265,6 @@
 	 sizeof(sysctl_overcommit_memory), 0644, NULL, &proc_dointvec},
 	{VM_PAGERDAEMON, "kswapd",
 	 &pager_daemon, sizeof(pager_daemon_t), 0644, NULL, &proc_dointvec},
-	{VM_PGT_CACHE, "pagetable_cache", 
-	 &pgt_cache_water, 2*sizeof(int), 0644, NULL, &proc_dointvec},
 	{VM_PAGE_CLUSTER, "page-cluster", 
 	 &page_cluster, sizeof(int), 0644, NULL, &proc_dointvec},
 	{VM_MIN_READAHEAD, "min-readahead",

mm/filemap.c +17 ~~-22~~

diff -urN linux-2.4.18-prepte/mm/filemap.c linux-2.4.18-highpte/mm/filemap.c
@@ -1989,7 +1989,7 @@
 /* Called with mm->page_table_lock held to protect against other
  * threads/the swapper from ripping pte's out from under us.
  */
-static inline int filemap_sync_pte(pte_t * ptep, struct vm_area_struct *vma,
+static inline int filemap_sync_pte(pte_t *ptep, pmd_t *pmdp, struct vm_area_struct *vma,
 	unsigned long address, unsigned int flags)
 {
 	pte_t pte = *ptep;

@@ -2005,11 +2005,10 @@
 }
 
 static inline int filemap_sync_pte_range(pmd_t * pmd,
-	unsigned long address, unsigned long size, 
-	struct vm_area_struct *vma, unsigned long offset, unsigned int flags)
+	unsigned long address, unsigned long end, 
+	struct vm_area_struct *vma, unsigned int flags)
 {
-	pte_t * pte;
-	unsigned long end;
+	pte_t *pte;
 	int error;
 
 	if (pmd_none(*pmd))

@@ -2019,27 +2018,26 @@
 		pmd_clear(pmd);
 		return 0;
 	}
-	pte = pte_offset(pmd, address);
-	offset += address & PMD_MASK;
-	address &= ~PMD_MASK;
-	end = address + size;
-	if (end > PMD_SIZE)
-		end = PMD_SIZE;
+	pte = pte_offset_map(pmd, address);
+	if ((address & PMD_MASK) != (end & PMD_MASK))
+		end = (address & PMD_MASK) + PMD_SIZE;
 	error = 0;
 	do {
-		error |= filemap_sync_pte(pte, vma, address + offset, flags);
+		error |= filemap_sync_pte(pte, pmd, vma, address, flags);
 		address += PAGE_SIZE;
 		pte++;
 	} while (address && (address < end));
+
+	pte_unmap(pte - 1);
+
 	return error;
 }
 
 static inline int filemap_sync_pmd_range(pgd_t * pgd,
-	unsigned long address, unsigned long size, 
+	unsigned long address, unsigned long end, 
 	struct vm_area_struct *vma, unsigned int flags)
 {
 	pmd_t * pmd;
-	unsigned long offset, end;
 	int error;
 
 	if (pgd_none(*pgd))

@@ -2050,14 +2048,11 @@
 		return 0;
 	}
 	pmd = pmd_offset(pgd, address);
-	offset = address & PGDIR_MASK;
-	address &= ~PGDIR_MASK;
-	end = address + size;
-	if (end > PGDIR_SIZE)
-		end = PGDIR_SIZE;
+	if ((address & PGDIR_MASK) != (end & PGDIR_MASK))
+		end = (address & PGDIR_MASK) + PGDIR_SIZE;
 	error = 0;
 	do {
-		error |= filemap_sync_pte_range(pmd, address, end - address, vma, offset, flags);
+		error |= filemap_sync_pte_range(pmd, address, end, vma, flags);
 		address = (address + PMD_SIZE) & PMD_MASK;
 		pmd++;
 	} while (address && (address < end));

@@ -2077,11 +2072,11 @@
 	spin_lock(&vma->vm_mm->page_table_lock);
 
 	dir = pgd_offset(vma->vm_mm, address);
-	flush_cache_range(vma->vm_mm, end - size, end);
+	flush_cache_range(vma, address, end);
 	if (address >= end)
 		BUG();
 	do {
-		error |= filemap_sync_pmd_range(dir, address, end - address, vma, flags);
+		error |= filemap_sync_pmd_range(dir, address, end, vma, flags);
 		address = (address + PGDIR_SIZE) & PGDIR_MASK;
 		dir++;
 	} while (address && (address < end));

mm/highmem.c +15 -0

diff -urN linux-2.4.18-prepte/mm/highmem.c linux-2.4.18-highpte/mm/highmem.c
@@ -21,6 +21,7 @@
 #include <linux/highmem.h>
 #include <linux/swap.h>
 #include <linux/slab.h>
+#include <asm/pgalloc.h>
 
 /*
  * Virtual_count is not a pure "count".

@@ -446,3 +447,17 @@
 	return bh;
 }
 
+#if CONFIG_DEBUG_HIGHMEM
+void check_highmem_ptes(void)
+{
+	int idx, type;
+ 
+	for (type = 0; type < KM_TYPE_NR; type++) {
+		idx = type + KM_TYPE_NR*smp_processor_id();
+ 		if (!pte_none(*(kmap_pte-idx))) {
+ 			printk("scheduling with KM_TYPE %d held!\n", type);
+ 			BUG();
+ 		}
+ 	}
+}
+#endif

mm/memory.c +120 ~~-89~~

461 lines not on this page · raw message

mm/mprotect.c +3 -1

diff -urN linux-2.4.18-prepte/mm/mprotect.c linux-2.4.18-highpte/mm/mprotect.c
@@ -11,6 +11,7 @@
 #include <asm/uaccess.h>
 #include <asm/pgalloc.h>
 #include <asm/pgtable.h>
+#include <linux/highmem.h>
 
 static inline void change_pte_range(pmd_t * pmd, unsigned long address,
 	unsigned long size, pgprot_t newprot)

@@ -25,7 +26,7 @@
 		pmd_clear(pmd);
 		return;
 	}
-	pte = pte_offset(pmd, address);
+	pte = pte_offset_map(pmd, address);
 	address &= ~PMD_MASK;
 	end = address + size;
 	if (end > PMD_SIZE)

@@ -44,6 +45,7 @@
 		address += PAGE_SIZE;
 		pte++;
 	} while (address && (address < end));
+	pte_unmap(pte - 1);
 }
 
 static inline void change_pmd_range(pgd_t * pgd, unsigned long address,

mm/mremap.c +15 -9

diff -urN linux-2.4.18-prepte/mm/mremap.c linux-2.4.18-highpte/mm/mremap.c
@@ -15,7 +15,7 @@
 
 extern int vm_enough_memory(long pages);
 
-static inline pte_t *get_one_pte(struct mm_struct *mm, unsigned long addr)
+static inline pte_t *get_one_pte_map2(struct mm_struct *mm, unsigned long addr)
 {
 	pgd_t * pgd;
 	pmd_t * pmd;

@@ -39,21 +39,23 @@
 		goto end;
 	}
 
-	pte = pte_offset(pmd, addr);
-	if (pte_none(*pte))
+	pte = pte_offset_map2(pmd, addr);
+	if (pte_none(*pte)) {
+		pte_unmap2(pte);
 		pte = NULL;
+	}
 end:
 	return pte;
 }
 
-static inline pte_t *alloc_one_pte(struct mm_struct *mm, unsigned long addr)
+static inline pte_t *alloc_one_pte_map(struct mm_struct *mm, unsigned long addr)
 {
 	pmd_t * pmd;
 	pte_t * pte = NULL;
 
 	pmd = pmd_alloc(mm, pgd_offset(mm, addr), addr);
 	if (pmd)
-		pte = pte_alloc(mm, pmd, addr);
+		pte = pte_alloc_map(mm, pmd, addr);
 	return pte;
 }

@@ -77,12 +79,16 @@
 static int move_one_page(struct mm_struct *mm, unsigned long old_addr, unsigned long new_addr)
 {
 	int error = 0;
-	pte_t * src;
+	pte_t *src, *dst;
 
 	spin_lock(&mm->page_table_lock);
-	src = get_one_pte(mm, old_addr);
-	if (src)
-		error = copy_one_pte(mm, src, alloc_one_pte(mm, new_addr));
+	src = get_one_pte_map2(mm, old_addr);
+	if (src) {
+		dst = alloc_one_pte_map(mm, new_addr);
+		error = copy_one_pte(mm, src, dst);
+		pte_unmap2(src);
+		pte_unmap(dst);
+	}
 	spin_unlock(&mm->page_table_lock);
 	return error;
 }

mm/swapfile.c +2 -1

diff -urN linux-2.4.18-prepte/mm/swapfile.c linux-2.4.18-highpte/mm/swapfile.c
@@ -393,7 +393,7 @@
 		pmd_clear(dir);
 		return;
 	}
-	pte = pte_offset(dir, address);
+	pte = pte_offset_map(dir, address);
 	offset += address & PMD_MASK;
 	address &= ~PMD_MASK;
 	end = address + size;

@@ -404,6 +404,7 @@
 		address += PAGE_SIZE;
 		pte++;
 	} while (address && (address < end));
+	pte_unmap(pte - 1);
 }
 
 /* mmlist_lock and vma->vm_mm->page_table_lock are held */

mm/vmalloc.c +2 -2

diff -urN linux-2.4.18-prepte/mm/vmalloc.c linux-2.4.18-highpte/mm/vmalloc.c
@@ -31,7 +31,7 @@
 		pmd_clear(pmd);
 		return;
 	}
-	pte = pte_offset(pmd, address);
+	pte = pte_offset_kernel(pmd, address);
 	address &= ~PMD_MASK;
 	end = address + size;
 	if (end > PMD_SIZE)

@@ -126,7 +126,7 @@
 	if (end > PGDIR_SIZE)
 		end = PGDIR_SIZE;
 	do {
-		pte_t * pte = pte_alloc(&init_mm, pmd, address);
+		pte_t * pte = pte_alloc_kernel(&init_mm, pmd, address);
 		if (!pte)
 			return -ENOMEM;
 		if (alloc_area_pte(pte, address, end - address, gfp_mask, prot))

mm/vmscan.c +3 -1

diff -urN linux-2.4.18-prepte/mm/vmscan.c linux-2.4.18-highpte/mm/vmscan.c
@@ -166,7 +166,7 @@
 		return count;
 	}
 	
-	pte = pte_offset(dir, address);
+	pte = pte_offset_map(dir, address);
 	
 	pmd_end = (address + PMD_SIZE) & PMD_MASK;
 	if (end > pmd_end)

@@ -180,6 +180,7 @@
 				count -= try_to_swap_out(mm, vma, address, pte, page, classzone);
 				if (!count) {
 					address += PAGE_SIZE;
+					pte++;
 					break;
 				}
 			}

@@ -187,6 +188,7 @@
 		address += PAGE_SIZE;
 		pte++;
 	} while (address && (address < end));
+	pte_unmap(pte - 1);
 	mm->swap_address = address;
 	return count;
 }

Martin J. Bligh

author

2002-03-21 23:21 UTC #1 lore raw

Quoted 5 lines

The smaller two patches they published against 2.5.5 work fine, but the main one needed a little work - this seems to work for me, but I've only really touch tested it ... just thought this might save someone some time .... there didn't seem to be anything too complex in the merge, but that doesn't necessarily mean I got it right ;-)

Sigh .... old version ... forgot to rediff. Working version below - sorry.

arch/i386/config.in +17 -4

diff -urN linux-2.4.18-prepte/arch/i386/config.in linux-2.4.18-highpte/arch/i386/config.in
@@ -171,16 +171,27 @@
 tristate '/dev/cpu/*/cpuid - CPU information support' CONFIG_X86_CPUID
 
 choice 'High Memory Support' \
-	"off    CONFIG_NOHIGHMEM \
-	 4GB    CONFIG_HIGHMEM4G \
-	 64GB   CONFIG_HIGHMEM64G" off
+	"off           CONFIG_NOHIGHMEM \
+	 4GB           CONFIG_HIGHMEM4G \
+	 4GB-highpte   CONFIG_HIGHMEM4G_HIGHPTE \
+	 64GB          CONFIG_HIGHMEM64G \
+	 64GB-highpte  CONFIG_HIGHMEM64G_HIGHPTE" off
 if [ "$CONFIG_HIGHMEM4G" = "y" ]; then
    define_bool CONFIG_HIGHMEM y
 fi
+if [ "$CONFIG_HIGHMEM4G_HIGHPTE" = "y" ]; then
+   define_bool CONFIG_HIGHMEM y
+   define_bool CONFIG_HIGHPTE y
+fi
 if [ "$CONFIG_HIGHMEM64G" = "y" ]; then
    define_bool CONFIG_HIGHMEM y
    define_bool CONFIG_X86_PAE y
 fi
+if [ "$CONFIG_HIGHMEM64G_HIGHPTE" = "y" ]; then
+   define_bool CONFIG_HIGHMEM y
+   define_bool CONFIG_HIGHPTE y
+   define_bool CONFIG_X86_PAE y
+fi
 
 bool 'Math emulation' CONFIG_MATH_EMULATION
 bool 'MTRR (Memory Type Range Register) support' CONFIG_MTRR

@@ -416,11 +427,13 @@
 
 bool 'Kernel debugging' CONFIG_DEBUG_KERNEL
 if [ "$CONFIG_DEBUG_KERNEL" != "n" ]; then
-   bool '  Debug high memory support' CONFIG_DEBUG_HIGHMEM
    bool '  Debug memory allocations' CONFIG_DEBUG_SLAB
    bool '  Memory mapped I/O debugging' CONFIG_DEBUG_IOVIRT
    bool '  Magic SysRq key' CONFIG_MAGIC_SYSRQ
    bool '  Spinlock debugging' CONFIG_DEBUG_SPINLOCK
+   if [ "$CONFIG_HIGHMEM" = "y" ]; then
+      bool '  Highmem debugging' CONFIG_DEBUG_HIGHMEM
+   fi
    bool '  Verbose BUG() reporting (adds 70K)' CONFIG_DEBUG_BUGVERBOSE
 fi

arch/i386/kernel/process.c +0 -1

diff -urN linux-2.4.18-prepte/arch/i386/kernel/process.c linux-2.4.18-highpte/arch/i386/kernel/process.c
@@ -132,7 +132,6 @@
 		if (!current->need_resched)
 			idle();
 		schedule();
-		check_pgt_cache();
 	}
 }

arch/i386/kernel/smpboot.c +0 -4

diff -urN linux-2.4.18-prepte/arch/i386/kernel/smpboot.c linux-2.4.18-highpte/arch/i386/kernel/smpboot.c
@@ -144,10 +144,6 @@
 	struct cpuinfo_x86 *c = cpu_data + id;
 
 	*c = boot_cpu_data;
-	c->pte_quick = 0;
-	c->pmd_quick = 0;
-	c->pgd_quick = 0;
-	c->pgtable_cache_sz = 0;
 	identify_cpu(c);
 	/*
 	 * Mask B, Pentium, but not Pentium MMX

arch/i386/kernel/traps.c +1 -1

diff -urN linux-2.4.18-prepte/arch/i386/kernel/traps.c linux-2.4.18-highpte/arch/i386/kernel/traps.c
@@ -734,7 +734,7 @@
 	page = (unsigned long) vmalloc(PAGE_SIZE);
 	pgd = pgd_offset(&init_mm, page);
 	pmd = pmd_offset(pgd, page);
-	pte = pte_offset(pmd, page);
+	pte = pte_offset_kernel(pmd, page);
 	__free_page(pte_page(*pte));
 	*pte = mk_pte_phys(__pa(&idt_table), PAGE_KERNEL_RO);
 	/*

arch/i386/kernel/vm86.c +3 -2

diff -urN linux-2.4.18-prepte/arch/i386/kernel/vm86.c linux-2.4.18-highpte/arch/i386/kernel/vm86.c
@@ -94,7 +94,7 @@
 {
 	pgd_t *pgd;
 	pmd_t *pmd;
-	pte_t *pte;
+	pte_t *pte, *mapped;
 	int i;
 
 	pgd = pgd_offset(tsk->mm, 0xA0000);

@@ -113,12 +113,13 @@
 		pmd_clear(pmd);
 		return;
 	}
-	pte = pte_offset(pmd, 0xA0000);
+	pte = mapped = pte_offset_map(pmd, 0xA0000);
 	for (i = 0; i < 32; i++) {
 		if (pte_present(*pte))
 			set_pte(pte, pte_wrprotect(*pte));
 		pte++;
 	}
+	pte_unmap(mapped);
 	flush_tlb();
 }

arch/i386/mm/fault.c +9 -1

diff -urN linux-2.4.18-prepte/arch/i386/mm/fault.c linux-2.4.18-highpte/arch/i386/mm/fault.c
@@ -324,12 +324,20 @@
 	asm("movl %%cr3,%0":"=r" (page));
 	page = ((unsigned long *) __va(page))[address >> 22];
 	printk(KERN_ALERT "*pde = %08lx\n", page);
+	/*
+	 * We must not directly access the pte in the highpte
+	 * case, the page table might be allocated in highmem.
+	 * And lets rather not kmap-atomic the pte, just in case
+	 * it's allocated already.
+	 */
+#ifndef CONFIG_HIGHPTE
 	if (page & 1) {
 		page &= PAGE_MASK;
 		address &= 0x003ff000;
 		page = ((unsigned long *) __va(page))[address >> PAGE_SHIFT];
 		printk(KERN_ALERT "*pte = %08lx\n", page);
 	}
+#endif
 	die("Oops", regs, error_code);
 	bust_spinlocks(0);
 	do_exit(SIGKILL);

@@ -399,7 +407,7 @@
 			goto no_context;
 		set_pmd(pmd, *pmd_k);
 
-		pte_k = pte_offset(pmd_k, address);
+		pte_k = pte_offset_kernel(pmd_k, address);
 		if (!pte_present(*pte_k))
 			goto no_context;
 		return;

arch/i386/mm/init.c +6 ~~-29~~

diff -urN linux-2.4.18-prepte/arch/i386/mm/init.c linux-2.4.18-highpte/arch/i386/mm/init.c
@@ -43,28 +43,6 @@
 static unsigned long totalram_pages;
 static unsigned long totalhigh_pages;
 
-int do_check_pgt_cache(int low, int high)
-{
-	int freed = 0;
-	if(pgtable_cache_size > high) {
-		do {
-			if (pgd_quicklist) {
-				free_pgd_slow(get_pgd_fast());
-				freed++;
-			}
-			if (pmd_quicklist) {
-				pmd_free_slow(pmd_alloc_one_fast(NULL, 0));
-				freed++;
-			}
-			if (pte_quicklist) {
-				pte_free_slow(pte_alloc_one_fast(NULL, 0));
-				freed++;
-			}
-		} while(pgtable_cache_size > low);
-	}
-	return freed;
-}
-
 /*
  * NOTE: pagetable_init alloc all the fixmap pagetables contiguous on the
  * physical space so we can cache the place of the first one and move

@@ -76,7 +54,7 @@
 pgprot_t kmap_prot;
 
 #define kmap_get_fixmap_pte(vaddr)					\
-	pte_offset(pmd_offset(pgd_offset_k(vaddr), (vaddr)), (vaddr))
+	pte_offset_kernel(pmd_offset(pgd_offset_k(vaddr), (vaddr)), (vaddr))
 
 void __init kmap_init(void)
 {

@@ -116,7 +94,6 @@
 	printk("%d reserved pages\n",reserved);
 	printk("%d pages shared\n",shared);
 	printk("%d pages swap cached\n",cached);
-	printk("%ld pages in page table cache\n",pgtable_cache_size);
 	show_buffers();
 }

@@ -143,7 +120,7 @@
 		printk("PAE BUG #01!\n");
 		return;
 	}
-	pte = pte_offset(pmd, vaddr);
+	pte = pte_offset_kernel(pmd, vaddr);
 	if (pte_val(*pte))
 		pte_ERROR(*pte);
 	pgprot_val(prot) = pgprot_val(PAGE_KERNEL) | pgprot_val(flags);

@@ -196,7 +173,7 @@
 			if (pmd_none(*pmd)) {
 				pte = (pte_t *) alloc_bootmem_low_pages(PAGE_SIZE);
 				set_pmd(pmd, __pmd(_KERNPG_TABLE + __pa(pte)));
-				if (pte != pte_offset(pmd, 0))
+				if (pte != pte_offset_kernel(pmd, 0))
 					BUG();
 			}
 			vaddr += PMD_SIZE;

@@ -267,7 +244,7 @@
 				*pte = mk_pte_phys(__pa(vaddr), PAGE_KERNEL);
 			}
 			set_pmd(pmd, __pmd(_KERNPG_TABLE + __pa(pte_base)));
-			if (pte_base != pte_offset(pmd, 0))
+			if (pte_base != pte_offset_kernel(pmd, 0))
 				BUG();
 
 		}

@@ -289,7 +266,7 @@
 
 	pgd = swapper_pg_dir + __pgd_offset(vaddr);
 	pmd = pmd_offset(pgd, vaddr);
-	pte = pte_offset(pmd, vaddr);
+	pte = pte_offset_kernel(pmd, vaddr);
 	pkmap_page_table = pte;
 #endif

@@ -398,7 +375,7 @@
 
 	pgd = swapper_pg_dir + __pgd_offset(vaddr);
 	pmd = pmd_offset(pgd, vaddr);
-	pte = pte_offset(pmd, vaddr);
+	pte = pte_offset_kernel(pmd, vaddr);
 	old_pte = *pte;
 	*pte = mk_pte_phys(0, PAGE_READONLY);
 	local_flush_tlb();

arch/i386/mm/ioremap.c +1 -1

diff -urN linux-2.4.18-prepte/arch/i386/mm/ioremap.c linux-2.4.18-highpte/arch/i386/mm/ioremap.c
@@ -49,7 +49,7 @@
 	if (address >= end)
 		BUG();
 	do {
-		pte_t * pte = pte_alloc(&init_mm, pmd, address);
+		pte_t * pte = pte_alloc_kernel(&init_mm, pmd, address);
 		if (!pte)
 			return -ENOMEM;
 		remap_area_pte(pte, address, end - address, address + phys_addr, flags);

drivers/char/drm/drm_proc.h +2 -1

diff -urN linux-2.4.18-prepte/drivers/char/drm/drm_proc.h linux-2.4.18-highpte/drivers/char/drm/drm_proc.h
@@ -449,7 +449,7 @@
 		for (i = vma->vm_start; i < vma->vm_end; i += PAGE_SIZE) {
 			pgd = pgd_offset(vma->vm_mm, i);
 			pmd = pmd_offset(pgd, i);
-			pte = pte_offset(pmd, i);
+			pte = pte_offset_map(pmd, i);
 			if (pte_present(*pte)) {
 				address = __pa(pte_page(*pte))
 					+ (i & (PAGE_SIZE-1));

@@ -465,6 +465,7 @@
 			} else {
 				DRM_PROC_PRINT("      0x%08lx\n", i);
 			}
+			pte_unmap(pte);
 		}
 #endif
 	}

drivers/char/drm/drm_scatter.h +5 -2

diff -urN linux-2.4.18-prepte/drivers/char/drm/drm_scatter.h linux-2.4.18-highpte/drivers/char/drm/drm_scatter.h
@@ -143,9 +143,12 @@
 		if ( !pmd_present( *pmd ) )
 			goto failed;
 
-		pte = pte_offset( pmd, i );
-		if ( !pte_present( *pte ) )
+		pte = pte_offset_map( pmd, i );
+		if ( !pte_present( *pte ) ) {
+			pte_unmap(pte);
 			goto failed;
+		}
+		pte_unmap(pte);
 
 		entry->pagelist[j] = pte_page( *pte );

drivers/char/drm/drm_vm.h +6 -2

diff -urN linux-2.4.18-prepte/drivers/char/drm/drm_vm.h linux-2.4.18-highpte/drivers/char/drm/drm_vm.h
@@ -169,8 +169,12 @@
 	if( !pgd_present( *pgd ) ) return NOPAGE_OOM;
 	pmd = pmd_offset( pgd, i );
 	if( !pmd_present( *pmd ) ) return NOPAGE_OOM;
-	pte = pte_offset( pmd, i );
-	if( !pte_present( *pte ) ) return NOPAGE_OOM;
+	pte = pte_offset_map( pmd, i );
+	if( !pte_present( *pte ) ) {
+		pte_unmap(pte);
+		return NOPAGE_OOM;
+	}
+	pte_unmap(pte);
 
 	page = pte_page(*pte);
 	get_page(page);

drivers/sgi/char/graphics.c +5 -2

diff -urN linux-2.4.18-prepte/drivers/sgi/char/graphics.c linux-2.4.18-highpte/drivers/sgi/char/graphics.c
@@ -221,6 +221,7 @@
 	int board = GRAPHICS_CARD (vma->vm_dentry->d_inode->i_rdev);
 
 	unsigned long virt_add, phys_add;
+	struct page * page;
 
 #ifdef DEBUG
 	printk ("Got a page fault for board %d address=%lx guser=%lx\n", board,

@@ -247,8 +248,10 @@
 
 	pgd = pgd_offset(current->mm, address);
 	pmd = pmd_offset(pgd, address);
-	pte = pte_offset(pmd, address);
-	return pte_page(*pte);
+	pte = pte_kmap_offset(pmd, address);
+	page = pte_page(*pte);
+	pte_kunmap(pte);
+	return page;
 }
 
 /*

drivers/usb/stv680.c +2 -1

diff -urN linux-2.4.18-prepte/drivers/usb/stv680.c linux-2.4.18-highpte/drivers/usb/stv680.c
@@ -133,8 +133,9 @@
 	if (!pgd_none (*pgd)) {
 		pmd = pmd_offset (pgd, adr);
 		if (!pmd_none (*pmd)) {
-			ptep = pte_offset (pmd, adr);
+			ptep = pte_offset_map (pmd, adr);
 			pte = *ptep;
+			pte_unmap(pte);
 			if (pte_present (pte)) {
 				ret = (unsigned long) page_address (pte_page (pte));
 				ret |= (adr & (PAGE_SIZE - 1));

drivers/usb/vicam.c +2 -1

diff -urN linux-2.4.18-prepte/drivers/usb/vicam.c linux-2.4.18-highpte/drivers/usb/vicam.c
@@ -115,8 +115,9 @@
 	if (!pgd_none(*pgd)) {
 		pmd = pmd_offset(pgd, adr);
 		if (!pmd_none(*pmd)) {
-			ptep = pte_offset(pmd, adr);
+			ptep = pte_offset_map(pmd, adr);
 			pte = *ptep;
+			pte_unmap(pte);
 			if(pte_present(pte)) {
 				ret  = (unsigned long) page_address(pte_page(pte));
 				ret |= (adr & (PAGE_SIZE - 1));

fs/exec.c +5 -2

diff -urN linux-2.4.18-prepte/fs/exec.c linux-2.4.18-highpte/fs/exec.c
@@ -270,15 +270,18 @@
 	pmd = pmd_alloc(tsk->mm, pgd, address);
 	if (!pmd)
 		goto out;
-	pte = pte_alloc(tsk->mm, pmd, address);
+	pte = pte_alloc_map(tsk->mm, pmd, address);
 	if (!pte)
 		goto out;
-	if (!pte_none(*pte))
+	if (!pte_none(*pte)) {
+		pte_unmap(pte);
 		goto out;
+	}
 	lru_cache_add(page);
 	flush_dcache_page(page);
 	flush_page_to_ram(page);
 	set_pte(pte, pte_mkdirty(pte_mkwrite(mk_pte(page, PAGE_COPY))));
+	pte_unmap(pte);
 	tsk->mm->rss++;
 	spin_unlock(&tsk->mm->page_table_lock);

fs/proc/array.c +8 -8

diff -urN linux-2.4.18-prepte/fs/proc/array.c linux-2.4.18-highpte/fs/proc/array.c
@@ -392,11 +392,10 @@
 	return res;
 }
 		
-static inline void statm_pte_range(pmd_t * pmd, unsigned long address, unsigned long size,
-	int * pages, int * shared, int * dirty, int * total)
+static inline void statm_pte_range(pmd_t * pmd, unsigned long address, unsigned long size, int * pages, int * shared, int * dirty, int * total)
 {
-	pte_t * pte;
-	unsigned long end;
+	unsigned long end, pmd_end;
+	pte_t *pte;
 
 	if (pmd_none(*pmd))
 		return;

@@ -405,11 +404,11 @@
 		pmd_clear(pmd);
 		return;
 	}
-	pte = pte_offset(pmd, address);
-	address &= ~PMD_MASK;
+	pte = pte_offset_map(pmd, address);
 	end = address + size;
-	if (end > PMD_SIZE)
-		end = PMD_SIZE;
+	pmd_end = (address + PMD_SIZE) & PMD_MASK;
+	if (end > pmd_end)
+		end = pmd_end;
 	do {
 		pte_t page = *pte;
 		struct page *ptpage;

@@ -430,6 +429,7 @@
 		if (page_count(pte_page(page)) > 1)
 			++*shared;
 	} while (address < end);
+	pte_unmap(pte - 1);
 }
 
 static inline void statm_pmd_range(pgd_t * pgd, unsigned long address, unsigned long size,

include/asm-i386/highmem.h +3 -9

diff -urN linux-2.4.18-prepte/include/asm-i386/highmem.h linux-2.4.18-highpte/include/asm-i386/highmem.h
@@ -26,12 +26,6 @@
 #include <asm/kmap_types.h>
 #include <asm/pgtable.h>
 
-#ifdef CONFIG_DEBUG_HIGHMEM
-#define HIGHMEM_DEBUG 1
-#else
-#define HIGHMEM_DEBUG 0
-#endif
-
 /* declarations for highmem.c */
 extern unsigned long highstart_pfn, highend_pfn;

@@ -93,7 +87,7 @@
 
 	idx = type + KM_TYPE_NR*smp_processor_id();
 	vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
-#if HIGHMEM_DEBUG
+#if CONFIG_DEBUG_HIGHMEM
 	if (!pte_none(*(kmap_pte-idx)))
 		BUG();
 #endif

@@ -105,8 +99,8 @@
 
 static inline void kunmap_atomic(void *kvaddr, enum km_type type)
 {
-#if HIGHMEM_DEBUG
-	unsigned long vaddr = (unsigned long) kvaddr;
+#if CONFIG_DEBUG_HIGHMEM
+	unsigned long vaddr = (unsigned long) kvaddr & PAGE_MASK;
 	enum fixed_addresses idx = type + KM_TYPE_NR*smp_processor_id();
 
 	if (vaddr < FIXADDR_START) // FIXME

include/asm-i386/kmap_types.h +19 -6

diff -urN linux-2.4.18-prepte/include/asm-i386/kmap_types.h linux-2.4.18-highpte/include/asm-i386/kmap_types.h
@@ -1,13 +1,26 @@
 #ifndef _ASM_KMAP_TYPES_H
 #define _ASM_KMAP_TYPES_H
 
+#include <linux/config.h>
+
+#if CONFIG_DEBUG_HIGHMEM
+# define D(n) __KM_FENCE_##n ,
+#else
+# define D(n)
+#endif
+
 enum km_type {
-	KM_BOUNCE_READ,
-	KM_SKB_DATA,
-	KM_SKB_DATA_SOFTIRQ,
-	KM_USER0,
-	KM_USER1,
-	KM_TYPE_NR
+D(0)  KM_BOUNCE_READ,
+D(1)  KM_SKB_DATA,
+D(2)  KM_SKB_DATA_SOFTIRQ,
+D(3)  KM_USER0,
+D(4)  KM_USER1,
+D(5)  KM_BIO_IRQ,
+D(6)  KM_PTE0,
+D(7)  KM_PTE1,
+D(8)  KM_TYPE_NR
 };
+
+#undef D
 
 #endif

include/asm-i386/pgalloc.h +50 ~~-63~~

165 lines not on this page · raw message

include/asm-i386/pgtable.h +12 -3

diff -urN linux-2.4.18-prepte/include/asm-i386/pgtable.h linux-2.4.18-highpte/include/asm-i386/pgtable.h
@@ -315,9 +315,12 @@
 
 #define page_pte(page) page_pte_prot(page, __pgprot(0))
 
-#define pmd_page(pmd) \
+#define pmd_page_kernel(pmd) \
 ((unsigned long) __va(pmd_val(pmd) & PAGE_MASK))
 
+#define pmd_page(pmd) \
+	(mem_map + (pmd_val(pmd) >> PAGE_SHIFT))
+
 /* to find an entry in a page-table-directory. */
 #define pgd_index(address) ((address >> PGDIR_SHIFT) & (PTRS_PER_PGD-1))

@@ -334,8 +337,14 @@
 /* Find an entry in the third-level page table.. */
 #define __pte_offset(address) \
 		((address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1))
-#define pte_offset(dir, address) ((pte_t *) pmd_page(*(dir)) + \
-			__pte_offset(address))
+#define pte_offset_kernel(dir, address) \
+	((pte_t *) pmd_page_kernel(*(dir)) +  __pte_offset(address))
+#define pte_offset_map(dir, address) \
+	((pte_t *)kmap_atomic(pmd_page(*(dir)),KM_PTE0) + __pte_offset(address))
+#define pte_offset_map2(dir, address) \
+	((pte_t *)kmap_atomic(pmd_page(*(dir)),KM_PTE1) + __pte_offset(address))
+#define pte_unmap(pte) kunmap_atomic(pte, KM_PTE0)
+#define pte_unmap2(pte) kunmap_atomic(pte, KM_PTE1)
 
 /*
  * The i386 doesn't have any external MMU info: the kernel page

include/asm-i386/processor.h +0 -4

diff -urN linux-2.4.18-prepte/include/asm-i386/processor.h linux-2.4.18-highpte/include/asm-i386/processor.h
@@ -49,10 +49,6 @@
 	int	f00f_bug;
 	int	coma_bug;
 	unsigned long loops_per_jiffy;
-	unsigned long *pgd_quick;
-	unsigned long *pmd_quick;
-	unsigned long *pte_quick;
-	unsigned long pgtable_cache_sz;
 } __attribute__((__aligned__(SMP_CACHE_BYTES)));
 
 #define X86_VENDOR_INTEL 0

include/linux/highmem.h +9 -8

diff -urN linux-2.4.18-prepte/include/linux/highmem.h linux-2.4.18-highpte/include/linux/highmem.h
@@ -2,7 +2,6 @@
 #define _LINUX_HIGHMEM_H
 
 #include <linux/config.h>
-#include <asm/pgalloc.h>
 
 #ifdef CONFIG_HIGHMEM

@@ -14,7 +13,7 @@
 unsigned int nr_free_highpages(void);
 
 extern struct buffer_head * create_bounce(int rw, struct buffer_head * bh_orig);
-
+extern void check_highmem_ptes(void);
 
 static inline char *bh_kmap(struct buffer_head *bh)
 {

@@ -52,8 +51,9 @@
 
 static inline void clear_highpage(struct page *page)
 {
-	clear_page(kmap(page));
-	kunmap(page);
+	void *kaddr = kmap_atomic(page, KM_USER0);
+	clear_page(kaddr);
+	kunmap_atomic(kaddr, KM_USER0);
 }
 
 /*

@@ -61,15 +61,16 @@
  */
 static inline void memclear_highpage_flush(struct page *page, unsigned int offset, unsigned int size)
 {
-	char *kaddr;
+	void *kaddr;
 
 	if (offset + size > PAGE_SIZE)
 		BUG();
-	kaddr = kmap(page);
-	memset(kaddr + offset, 0, size);
+
+	kaddr = kmap_atomic(page, KM_USER0);
+	memset((char *)kaddr + offset, 0, size);
 	flush_dcache_page(page);
 	flush_page_to_ram(page);
-	kunmap(page);
+	kunmap_atomic(kaddr, KM_USER0);
 }
 
 static inline void copy_user_highpage(struct page *to, struct page *from, unsigned long vaddr)

include/linux/mm.h +3 -5

diff -urN linux-2.4.18-prepte/include/linux/mm.h linux-2.4.18-highpte/include/linux/mm.h
@@ -411,7 +411,8 @@
 
 extern int vmtruncate(struct inode * inode, loff_t offset);
 extern pmd_t *FASTCALL(__pmd_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address));
-extern pte_t *FASTCALL(pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address));
+extern pte_t *FASTCALL(pte_alloc_kernel(struct mm_struct *mm, pmd_t *pmd, unsigned long address));
+extern pte_t *FASTCALL(pte_alloc_map(struct mm_struct *mm, pmd_t *pmd, unsigned long address));
 extern int handle_mm_fault(struct mm_struct *mm,struct vm_area_struct *vma, unsigned long address, int write_access);
 extern int make_pages_present(unsigned long addr, unsigned long end);
 extern int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write);

@@ -427,7 +428,7 @@
 
 /*
  * On a two-level page table, this ends up being trivial. Thus the
- * inlining and the symmetry break with pte_alloc() that does all
+ * inlining and the symmetry break with pte_alloc_map() that does all
  * of this out-of-line.
  */
 static inline pmd_t *pmd_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)

@@ -436,9 +437,6 @@
 		return __pmd_alloc(mm, pgd, address);
 	return pmd_offset(pgd, address);
 }
-
-extern int pgt_cache_water[2];
-extern int check_pgt_cache(void);
 
 extern void free_area_init(unsigned long * zones_size);
 extern void free_area_init_node(int nid, pg_data_t *pgdat, struct page *pmap,

kernel/sched.c +5 -0

diff -urN linux-2.4.18-prepte/kernel/sched.c linux-2.4.18-highpte/kernel/sched.c
@@ -20,6 +20,7 @@
 #include <linux/interrupt.h>
 #include <asm/mmu_context.h>
 #include <linux/kernel_stat.h>
+#include <linux/highmem.h>
 
 /*
  * Priority of a process goes from 0 to 139. The 0-99

@@ -751,6 +752,10 @@
 
 	if (unlikely(in_interrupt()))
 		BUG();
+#if CONFIG_DEBUG_HIGHMEM
+	check_highmem_ptes();
+#endif
+
 	release_kernel_lock(prev, smp_processor_id());
 	prev->sleep_timestamp = jiffies;
 	spin_lock_irq(&rq->lock);

kernel/sysctl.c +0 -4

diff -urN linux-2.4.18-prepte/kernel/sysctl.c linux-2.4.18-highpte/kernel/sysctl.c
@@ -96,8 +96,6 @@
 extern int acct_parm[];
 #endif
 
-extern int pgt_cache_water[];
-
 static int parse_table(int *, int, void *, size_t *, void *, size_t,
 		       ctl_table *, void **);
 static int proc_doutsstring(ctl_table *table, int write, struct file *filp,

@@ -267,8 +265,6 @@
 	 sizeof(sysctl_overcommit_memory), 0644, NULL, &proc_dointvec},
 	{VM_PAGERDAEMON, "kswapd",
 	 &pager_daemon, sizeof(pager_daemon_t), 0644, NULL, &proc_dointvec},
-	{VM_PGT_CACHE, "pagetable_cache", 
-	 &pgt_cache_water, 2*sizeof(int), 0644, NULL, &proc_dointvec},
 	{VM_PAGE_CLUSTER, "page-cluster", 
 	 &page_cluster, sizeof(int), 0644, NULL, &proc_dointvec},
 	{VM_MIN_READAHEAD, "min-readahead",

mm/filemap.c +17 ~~-22~~

diff -urN linux-2.4.18-prepte/mm/filemap.c linux-2.4.18-highpte/mm/filemap.c
@@ -1989,7 +1989,7 @@
 /* Called with mm->page_table_lock held to protect against other
  * threads/the swapper from ripping pte's out from under us.
  */
-static inline int filemap_sync_pte(pte_t * ptep, struct vm_area_struct *vma,
+static inline int filemap_sync_pte(pte_t *ptep, pmd_t *pmdp, struct vm_area_struct *vma,
 	unsigned long address, unsigned int flags)
 {
 	pte_t pte = *ptep;

@@ -2005,11 +2005,10 @@
 }
 
 static inline int filemap_sync_pte_range(pmd_t * pmd,
-	unsigned long address, unsigned long size, 
-	struct vm_area_struct *vma, unsigned long offset, unsigned int flags)
+	unsigned long address, unsigned long end, 
+	struct vm_area_struct *vma, unsigned int flags)
 {
-	pte_t * pte;
-	unsigned long end;
+	pte_t *pte;
 	int error;
 
 	if (pmd_none(*pmd))

@@ -2019,27 +2018,26 @@
 		pmd_clear(pmd);
 		return 0;
 	}
-	pte = pte_offset(pmd, address);
-	offset += address & PMD_MASK;
-	address &= ~PMD_MASK;
-	end = address + size;
-	if (end > PMD_SIZE)
-		end = PMD_SIZE;
+	pte = pte_offset_map(pmd, address);
+	if ((address & PMD_MASK) != (end & PMD_MASK))
+		end = (address & PMD_MASK) + PMD_SIZE;
 	error = 0;
 	do {
-		error |= filemap_sync_pte(pte, vma, address + offset, flags);
+		error |= filemap_sync_pte(pte, pmd, vma, address, flags);
 		address += PAGE_SIZE;
 		pte++;
 	} while (address && (address < end));
+
+	pte_unmap(pte - 1);
+
 	return error;
 }
 
 static inline int filemap_sync_pmd_range(pgd_t * pgd,
-	unsigned long address, unsigned long size, 
+	unsigned long address, unsigned long end, 
 	struct vm_area_struct *vma, unsigned int flags)
 {
 	pmd_t * pmd;
-	unsigned long offset, end;
 	int error;
 
 	if (pgd_none(*pgd))

@@ -2050,14 +2048,11 @@
 		return 0;
 	}
 	pmd = pmd_offset(pgd, address);
-	offset = address & PGDIR_MASK;
-	address &= ~PGDIR_MASK;
-	end = address + size;
-	if (end > PGDIR_SIZE)
-		end = PGDIR_SIZE;
+	if ((address & PGDIR_MASK) != (end & PGDIR_MASK))
+		end = (address & PGDIR_MASK) + PGDIR_SIZE;
 	error = 0;
 	do {
-		error |= filemap_sync_pte_range(pmd, address, end - address, vma, offset, flags);
+		error |= filemap_sync_pte_range(pmd, address, end, vma, flags);
 		address = (address + PMD_SIZE) & PMD_MASK;
 		pmd++;
 	} while (address && (address < end));

@@ -2077,11 +2072,11 @@
 	spin_lock(&vma->vm_mm->page_table_lock);
 
 	dir = pgd_offset(vma->vm_mm, address);
-	flush_cache_range(vma->vm_mm, end - size, end);
+	flush_cache_range(vma, address, end);
 	if (address >= end)
 		BUG();
 	do {
-		error |= filemap_sync_pmd_range(dir, address, end - address, vma, flags);
+		error |= filemap_sync_pmd_range(dir, address, end, vma, flags);
 		address = (address + PGDIR_SIZE) & PGDIR_MASK;
 		dir++;
 	} while (address && (address < end));

mm/highmem.c +15 -0

diff -urN linux-2.4.18-prepte/mm/highmem.c linux-2.4.18-highpte/mm/highmem.c
@@ -21,6 +21,7 @@
 #include <linux/highmem.h>
 #include <linux/swap.h>
 #include <linux/slab.h>
+#include <asm/pgalloc.h>
 
 /*
  * Virtual_count is not a pure "count".

@@ -446,3 +447,17 @@
 	return bh;
 }
 
+#if CONFIG_DEBUG_HIGHMEM
+void check_highmem_ptes(void)
+{
+	int idx, type;
+ 
+	for (type = 0; type < KM_TYPE_NR; type++) {
+		idx = type + KM_TYPE_NR*smp_processor_id();
+ 		if (!pte_none(*(kmap_pte-idx))) {
+ 			printk("scheduling with KM_TYPE %d held!\n", type);
+ 			BUG();
+ 		}
+ 	}
+}
+#endif

mm/memory.c +117 ~~-91~~

473 lines not on this page · raw message

mm/mprotect.c +3 -1

diff -urN linux-2.4.18-prepte/mm/mprotect.c linux-2.4.18-highpte/mm/mprotect.c
@@ -11,6 +11,7 @@
 #include <asm/uaccess.h>
 #include <asm/pgalloc.h>
 #include <asm/pgtable.h>
+#include <linux/highmem.h>
 
 static inline void change_pte_range(pmd_t * pmd, unsigned long address,
 	unsigned long size, pgprot_t newprot)

@@ -25,7 +26,7 @@
 		pmd_clear(pmd);
 		return;
 	}
-	pte = pte_offset(pmd, address);
+	pte = pte_offset_map(pmd, address);
 	address &= ~PMD_MASK;
 	end = address + size;
 	if (end > PMD_SIZE)

@@ -44,6 +45,7 @@
 		address += PAGE_SIZE;
 		pte++;
 	} while (address && (address < end));
+	pte_unmap(pte - 1);
 }
 
 static inline void change_pmd_range(pgd_t * pgd, unsigned long address,

mm/mremap.c +15 -9

diff -urN linux-2.4.18-prepte/mm/mremap.c linux-2.4.18-highpte/mm/mremap.c
@@ -15,7 +15,7 @@
 
 extern int vm_enough_memory(long pages);
 
-static inline pte_t *get_one_pte(struct mm_struct *mm, unsigned long addr)
+static inline pte_t *get_one_pte_map2(struct mm_struct *mm, unsigned long addr)
 {
 	pgd_t * pgd;
 	pmd_t * pmd;

@@ -39,21 +39,23 @@
 		goto end;
 	}
 
-	pte = pte_offset(pmd, addr);
-	if (pte_none(*pte))
+	pte = pte_offset_map2(pmd, addr);
+	if (pte_none(*pte)) {
+		pte_unmap2(pte);
 		pte = NULL;
+	}
 end:
 	return pte;
 }
 
-static inline pte_t *alloc_one_pte(struct mm_struct *mm, unsigned long addr)
+static inline pte_t *alloc_one_pte_map(struct mm_struct *mm, unsigned long addr)
 {
 	pmd_t * pmd;
 	pte_t * pte = NULL;
 
 	pmd = pmd_alloc(mm, pgd_offset(mm, addr), addr);
 	if (pmd)
-		pte = pte_alloc(mm, pmd, addr);
+		pte = pte_alloc_map(mm, pmd, addr);
 	return pte;
 }

@@ -77,12 +79,16 @@
 static int move_one_page(struct mm_struct *mm, unsigned long old_addr, unsigned long new_addr)
 {
 	int error = 0;
-	pte_t * src;
+	pte_t *src, *dst;
 
 	spin_lock(&mm->page_table_lock);
-	src = get_one_pte(mm, old_addr);
-	if (src)
-		error = copy_one_pte(mm, src, alloc_one_pte(mm, new_addr));
+	src = get_one_pte_map2(mm, old_addr);
+	if (src) {
+		dst = alloc_one_pte_map(mm, new_addr);
+		error = copy_one_pte(mm, src, dst);
+		pte_unmap2(src);
+		pte_unmap(dst);
+	}
 	spin_unlock(&mm->page_table_lock);
 	return error;
 }

mm/swapfile.c +2 -1

diff -urN linux-2.4.18-prepte/mm/swapfile.c linux-2.4.18-highpte/mm/swapfile.c
@@ -393,7 +393,7 @@
 		pmd_clear(dir);
 		return;
 	}
-	pte = pte_offset(dir, address);
+	pte = pte_offset_map(dir, address);
 	offset += address & PMD_MASK;
 	address &= ~PMD_MASK;
 	end = address + size;

@@ -404,6 +404,7 @@
 		address += PAGE_SIZE;
 		pte++;
 	} while (address && (address < end));
+	pte_unmap(pte - 1);
 }
 
 /* mmlist_lock and vma->vm_mm->page_table_lock are held */

mm/vmalloc.c +2 -2

diff -urN linux-2.4.18-prepte/mm/vmalloc.c linux-2.4.18-highpte/mm/vmalloc.c
@@ -31,7 +31,7 @@
 		pmd_clear(pmd);
 		return;
 	}
-	pte = pte_offset(pmd, address);
+	pte = pte_offset_kernel(pmd, address);
 	address &= ~PMD_MASK;
 	end = address + size;
 	if (end > PMD_SIZE)

@@ -126,7 +126,7 @@
 	if (end > PGDIR_SIZE)
 		end = PGDIR_SIZE;
 	do {
-		pte_t * pte = pte_alloc(&init_mm, pmd, address);
+		pte_t * pte = pte_alloc_kernel(&init_mm, pmd, address);
 		if (!pte)
 			return -ENOMEM;
 		if (alloc_area_pte(pte, address, end - address, gfp_mask, prot))

mm/vmscan.c +3 -1

diff -urN linux-2.4.18-prepte/mm/vmscan.c linux-2.4.18-highpte/mm/vmscan.c
@@ -166,7 +166,7 @@
 		return count;
 	}
 	
-	pte = pte_offset(dir, address);
+	pte = pte_offset_map(dir, address);
 	
 	pmd_end = (address + PMD_SIZE) & PMD_MASK;
 	if (end > pmd_end)

@@ -180,6 +180,7 @@
 				count -= try_to_swap_out(mm, vma, address, pte, page, classzone);
 				if (!count) {
 					address += PAGE_SIZE;
+					pte++;
 					break;
 				}
 			}

@@ -187,6 +188,7 @@
 		address += PAGE_SIZE;
 		pte++;
 	} while (address && (address < end));
+	pte_unmap(pte - 1);
 	mm->swap_address = address;
 	return count;
 }

Andrea Arcangeli

2002-03-26 17:09 UTC #2 lore raw

In reply to Martin J. Bligh

Quoted 51 lines · Martin J. Bligh

diff -urN linux-2.4.18-prepte/mm/mremap.c linux-2.4.18-highpte/mm/mremap.c

mm/mremap.c +8 -9

@@ -15,7 +15,7 @@
 
 extern int vm_enough_memory(long pages);
 
-static inline pte_t *get_one_pte(struct mm_struct *mm, unsigned long addr)
+static inline pte_t *get_one_pte_map2(struct mm_struct *mm, unsigned long addr)
 {
 	pgd_t * pgd;
 	pmd_t * pmd;

@@ -39,21 +39,23 @@
 		goto end;
 	}
 
-	pte = pte_offset(pmd, addr);
-	if (pte_none(*pte))
+	pte = pte_offset_map2(pmd, addr);
+	if (pte_none(*pte)) {
+		pte_unmap2(pte);
 		pte = NULL;
+	}
 end:
 	return pte;
 }
 
-static inline pte_t *alloc_one_pte(struct mm_struct *mm, unsigned long addr)
+static inline pte_t *alloc_one_pte_map(struct mm_struct *mm, unsigned long addr)
 {
 	pmd_t * pmd;
 	pte_t * pte = NULL;
 
 	pmd = pmd_alloc(mm, pgd_offset(mm, addr), addr);
 	if (pmd)
-		pte = pte_alloc(mm, pmd, addr);
+		pte = pte_alloc_map(mm, pmd, addr);
 	return pte;
 }

@@ -77,12 +79,16 @@
 static int move_one_page(struct mm_struct *mm, unsigned long old_addr, unsigned long new_addr)
 {
 	int error = 0;
-	pte_t * src;
+	pte_t *src, *dst;
 
 	spin_lock(&mm->page_table_lock);
-	src = get_one_pte(mm, old_addr);
-	if (src)
-		error = copy_one_pte(mm, src, alloc_one_pte(mm, new_addr));

mm/mremap.c line 94

Quoted 10 lines

mm/mremap.c +7 -0

+	src = get_one_pte_map2(mm, old_addr);
+	if (src) {
+		dst = alloc_one_pte_map(mm, new_addr);
+		error = copy_one_pte(mm, src, dst);
+		pte_unmap2(src);
+		pte_unmap(dst);
+	}
 	spin_unlock(&mm->page_table_lock);
 	return error;
 }

while removing the persistent kmaps in pte-highmem from the fast paths, I had a look how you solved this mremap thing without the persistent kmaps. First, your backport is clearly broken because it will oops in copy_one_pte if the alloc_one_pte_map fails.

2.5.7 gets such bit right (checking for dst == null within copy_one_pte), but it's broken too and it will be unstable with high-pte enabled. Either it's a coincidence or Ingo copied the changes from my pte-highmem but he apparently forgot that his pte_offset_map isn't persistent. The code works fine in pte-highmem, and it will break on 2.5.7 due the atomic kmaps. The reason is that it's doing a pte_alloc within a pte_offset_map_nested, so the pte_offset_map_nested can be corrupted if pte_alloc sleeps while allocating the memory for the pagetable.

Now, I'm not nearly backporting Ingo's code to 2.4, in particular the drop of the quicklist without an affine allocator is a performance optimization removal that doesn't make sense in 2.4, and as well the whole pte_offset_kernel thing is flawed in general IMHO (in particular in 2.5 where all drivers are using proper api to get physical pages of vmalloc space). mremap isn't an extremely fast path like the anon page faults/execve etc.., so in pte-highmem I'm going to left the persistent kmap in get_one_pte, to left the code simple and to avoid an overwork with preallocation of memory or ugly #ifdefs around. That's one of the purposes of the persistent kmaps of course, to keep the code clean and efficient (think UP here). And in 2.5, once we'll switch all the pte_offset_map to use the KM_USER_PAGETABLE0 entry in the user-fixmap area, those mremap mappings will be automatically persistent and the current 2.5 bug will go away automatically that way. Then also all the memory.c changes that I will have to backport to make your workload scale (sensless in particular with a 64bit point of view) in Ingo's patch can be backed out from 2.5 (with the exclusion of the pte_unmap* of course :).

BTW, another place where I'm not going to write ugly code in 2.4 is /proc/, while it could be atomic in mainline, I've a scheduling point there to provide lower latency, and I don't want to clobber it with stuff like:

	if (current->need_resched) {
		pte_kunmap(pte);
		schedule();
		pte = pte_offset(...)
	}

just to make ps faster.

The persistent kmaps scales less on a 16-way NUMA-Q (not even a problem for normal servers), so I'm just replacing the absolutely required fast paths to make the kernel compile faster, but in all the slow paths where I'd need to write ugly code, I'm definitely keeping the persistent kmaps.

With the user-fixmap that you proposed some day ago, this whole thing will be solved cleanly in 2.5. 2.4 pte-highmem will remain an hybrid.

Then there's also the naming issues of the pte_offset calls, for now I'm keeping them called:

pte_offset_atomic/pte_kunmap (atomic) pte_offset/pte_kunmap (persistent) pte_offset_atomic2/pte_kunmap_atomic2 (atomic 2nd entry)

the original names in pte-highmem, because of this hybrid thing and to remeber it works differently than the atomic-and-persistent user-fixmap that we'll soon have in 2.5 with pte_offset_map.

(I could have pte_kunmap to take care of pte_offset_atomic2 too, but doing explicit is faster and it happens only in a few places anyways, if you prefer to have a pte_kunmap able to unmap pte_offset_atomic2 too let me know)

I'm not considering to drop pte-highmem from 2.4 and to only support the user-fixmaps in 2.5 because it is a showstopper bugfix for lots of important users that definitely cannot wait 2.6. I'm also not considering backporting the user-fixmaps because that would be a quite invasive change messing also with the alignment of the user stack (I know it could stay into kernel space, but right after the user stack it will be more optimized and cleaner/simpler, so I prefer to put the few virtual pages there).

Andrea

Martin J. Bligh

author

2002-03-26 18:03 UTC #3 lore raw

Quoted 3 lines

First, your backport is clearly broken because it will oops in copy_one_pte if the alloc_one_pte_map fails.

That doesn't suprise me ... I did a quick backport without staring at the code too much, just so I could get some testing number to see what the difference in performance would be. Arjan is doing a proper backport to 2.4, and he obviously knows this patch far better than I, so hopefully he'll address this ;-)

Thanks for pointing this out.

Quoted 1 line

....

The bulk of the rest of this will take me a while to think about ;-)

Thanks,

PS. The backport of the 2.5 highpte stuff works fine for me in limited touch-testing, but I don't have it playing with the discontigmem stuff yet, so I can't give you any numbers at the moment ...

Martin J. Bligh

author

2002-03-29 22:50 UTC #4 lore raw

In reply to Andrea Arcangeli

Quoted 8 lines

I'm not considering to drop pte-highmem from 2.4 and to only support the user-fixmaps in 2.5 because it is a showstopper bugfix for lots of important users that definitely cannot wait 2.6. I'm also not considering backporting the user-fixmaps because that would be a quite invasive change messing also with the alignment of the user stack (I know it could stay into kernel space, but right after the user stack it will be more optimized and cleaner/simpler, so I prefer to put the few virtual pages there).

Can you explain the problem with the aligment of the user stack? I can't see what the problem is here .... and we need to start thinking about how to fix it if you've seen a problem that we haven't ....

Thanks,

Andrea Arcangeli

2002-03-26 18:18 UTC #5 lore raw

In reply to Martin J. Bligh

Quoted 8 lines · Martin J. Bligh

Quoted 3 lines
First, your backport is clearly broken because it will oops in copy_one_pte if the alloc_one_pte_map fails.
That doesn't suprise me ... I did a quick backport without staring at the code too much, just so I could get some testing number to see what the difference in performance would be. Arjan is doing a proper backport to 2.4, and he obviously knows this patch far

I need persistent kmaps for the pagetables, and also the quicklists, so I'm not excited to integrate a 2.4 backport of an halfway solution that isn't optimal for 2.5 either (plus at the moment it's buggy and incidentally the right fix is to add the persistent kmaps in 2.5 too, go figure).

Andrea

Martin J. Bligh

author

2002-03-28 22:49 UTC #6 lore raw

Testing of Ingo/Arjan highpte on 16-way NUMA-Q

Adding your highpte patches to the NUMA-Q seems to make no performance impact whatsoever (just doing kernel compile). The profiles are so similar, it almost looks like it's not doing anything at all ;-) I find this a little strange, as all my ZONE_NORMAL is on node 0, so I'd expected this to have some sort of impact (either positive or negative ;-)).

Is there an easy way to test whether this was working or not?

Thanks,

PS. yes I did remember to turn on the 4G-highpte option ;-)

Andrea Arcangeli

2002-03-30 19:44 UTC #7 lore raw

In reply to Martin J. Bligh

Quoted 9 lines · Martin J. Bligh

Quoted 8 lines
I'm not considering to drop pte-highmem from 2.4 and to only support the user-fixmaps in 2.5 because it is a showstopper bugfix for lots of important users that definitely cannot wait 2.6. I'm also not considering backporting the user-fixmaps because that would be a quite invasive change messing also with the alignment of the user stack (I know it could stay into kernel space, but right after the user stack it will be more optimized and cleaner/simpler, so I prefer to put the few virtual pages there).

Quoted 3 lines

Can you explain the problem with the aligment of the user stack? I can't see what the problem is here .... and we need to start thinking about how to fix it if you've seen a problem that we haven't ....

the user stack never started on a non power of two, so far, so some buggy app may do assumptions on that, and bugs could trigger than. No app should break in theory. If somebody is doing a map fixed near the stack using a and-mask (instead of a minus) on %esp to calculate the offset of the map-fixed, that could break if we eat a copule of virtual pages of stack. I triggered something related when I was using a bit larger heap-stack-gap, I know there are apps putting mappings very near the stack (the one I've in mind isn't even open source so I don't know how it chooses the address), but most probably correctly none of them makes assumption on the alignment of the top of the stack, so it's should be safe, but you know you never know :). Also given the complexity of the patch, and also given it's definitely not necessary in 2.4 just to get the scalability bit I recommend it for 2.5 only (my new scalable pte-highmem using atomics in the fast paths and persistent in the slow paths where 2.5 is also currently broken is just working just fine on top of pre4). I was ready to release it today (just running for one day of stress testing), but pre5 came out so I'll have to spend some more hour to slowly resync and rediff.

Andrea

Andrea Arcangeli

2002-03-31 14:51 UTC #8 lore raw

In reply to Martin J. Bligh

Quoted 3 lines · Andrea Arcangeli

just fine on top of pre4). I was ready to release it today (just running for one day of stress testing), but pre5 came out so I'll have to spend some more hour to slowly resync and rediff.

Ok, after a few hours of a very pleasant resyncing work and re-testing (it was great to see lots of good stuff integrated from Marcelo) it's out right now. If you could retest the NUMA-Q support on top of 2.4.19pre5aa1 that would be helpful to know if the presistent kmap is a problem in mremap (the place where it's currently buggy in 2.5). Thanks!

Andrea