allow non-PMD-aligned PAGE_OFFSET
Subsystems (2)
PATCH 1/1 allow non-PMD-aligned PAGE_OFFSET
Some crazy person with PAE thinks that they need more user virtual space, This patch allows PAGE_OFFSET to be in much more arbitrary places, with PAE on. I described the first bit of the patch here:
http://marc.theaimsgroup.com/?l=linux-kernel&m=104197008817507&w=2
This patch also allows us to simulate the conditions that occur when we have large amounts of physical RAM (like having pte chains and mem_map consuting all of ZONE_NORMAL) with much lower actual amounts of RAM.
This part adds the userspace page tables component, so that we can _actually_ run userspace programs. Thanks to Bill Irwin for showing me how to do the pgd_alloc() in a relatively sane way.
The second patch I've attached will take you from the normal 3:1 split, to a 7:1 split. I've also booted on a machine with PAE on, 4GB of physical RAM, and a 15:1 split (PAGE_OFFSET == 0xF0000000).
This is from the 15:1 split:
3615MB HIGHMEM available.
128MB LOWMEM available.
MemTotal: 3790824 kB MemFree: 3758920 kB Buffers: 1472 kB Cached: 7636 kB SwapCached: 0 kB Active: 6576 kB Inactive: 4380 kB HighTotal: 3702756 kB HighFree: 3692160 kB LowTotal: 88068 kB LowFree: 66760 kB SwapTotal: 0 kB SwapFree: 0 kB Dirty: 116 kB Writeback: 0 kB Mapped: 4804 kB Slab: 5092 kB Committed_AS: 2776 kB PageTables: 1868 kB ReverseMaps: 3091
Signature · 2 lines
Dave Hansen [email protected]
arch/i386/mm/init.c +62 -28
@@ -121,6 +121,24 @@ } } + +/* +* Abstract out using large pages when mapping KVA, or the SMP identity +* mapping +*/ +void pmd_map_pfn_range (pmd_t * pmd_entry,unsigned long pfn,unsigned long max_pfn) +{ +int pte_ofs; +/* Map with big pages if possible, otherwise create normal page tables. */ +if (cpu_has_pse) { +set_pmd (pmd_entry,pfn_pmd (pfn,PAGE_KERNEL_LARGE )); + pfn +=PTRS_PER_PTE ; + }else { +pte_t * pte =one_page_table_init (pmd_entry); +for (pte_ofs =0 ; pte_ofs <PTRS_PER_PTE && pfn < max_pfn; pte++, pfn++, pte_ofs++) +set_pte (pte,pfn_pte (pfn,PAGE_KERNEL )); + } +}/* * This maps the physical memory to kernel virtual address space, a total * of max_low_pfn pages, by creating page tables starting from address
@@ -131,8 +149,7 @@unsigned long pfn;pgd_t *pgd;pmd_t *pmd; -pte_t *pte; -int pgd_ofs, pmd_ofs, pte_ofs; +int pgd_ofs, pmd_ofs; pgd_ofs =__pgd_offset (PAGE_OFFSET ); pgd = pgd_base + pgd_ofs;
@@ -142,19 +159,47 @@ pmd =one_md_table_init (pgd);if (pfn >= max_low_pfn)continue ; -for (pmd_ofs =0 ; pmd_ofs <PTRS_PER_PMD && pfn < max_low_pfn; pmd++, pmd_ofs++) { -/* Map with big pages if possible, otherwise create normal page tables. */ -if (cpu_has_pse) { -set_pmd (pmd,pfn_pmd (pfn,PAGE_KERNEL_LARGE )); - pfn +=PTRS_PER_PTE ; - }else { - pte =one_page_table_init (pmd); - -for (pte_ofs =0 ; pte_ofs <PTRS_PER_PTE && pfn < max_low_pfn; pte++, pfn++, pte_ofs++) -set_pte (pte,pfn_pte (pfn,PAGE_KERNEL )); - } + +/* beware of starting KVA in the middle of a pmd. */ +if ( pgd_ofs ==__pgd_offset (PAGE_OFFSET ) ) { + pmd_ofs =__pmd_offset (PAGE_OFFSET ); + pmd = &pmd[pmd_ofs]; + }else + pmd_ofs =0 ; + +for (; pmd_ofs <PTRS_PER_PMD && pfn < max_low_pfn; pmd++, pmd_ofs++) { +pmd_map_pfn_range (pmd, pfn, max_low_pfn); + pfn +=PTRS_PER_PTE ; } - } + } +} + + +/* +* Add low memory identity-mappings - SMP needs it when +* starting up on an AP from real-mode. In the non-PAE +* case we already have these mappings through head.S. +* All user-space mappings are explicitly cleared after +* SMP startup in zap_low_mappings(). +*/ +static void __init low_physical_mapping_init (pgd_t *pgd_base) +{ +#if CONFIG_X86_PAE +unsigned long pfn =0 ; +int pmd_ofs =0 ; +pmd_t *pmd =one_md_table_init (pgd_base); + +if (!cpu_has_pse) { +printk ("PAE enabled, but no support for PSE (large pages)!\n" ); +printk ("this is likely to waste some RAM." ); + } + +for (; pmd_ofs <PTRS_PER_PMD && pfn <= max_low_pfn; pmd++, pmd_ofs++) { +pmd_map_pfn_range (pmd, pfn, max_low_pfn); + pfn +=PTRS_PER_PTE ; + } +#endif + }static inline int page_kills_ppro (unsigned long pagenr)
@@ -217,7 +262,7 @@ pgd = swapper_pg_dir +__pgd_offset (vaddr); pmd =pmd_offset (pgd, vaddr); pte =pte_offset_kernel (pmd, vaddr); - pkmap_page_table = pte; + pkmap_page_table = pte; }void __init one_highpage_init (struct page *page,int pfn,int bad_ppro)
@@ -282,6 +327,7 @@ }kernel_physical_mapping_init (pgd_base); +low_physical_mapping_init (pgd_base);remap_numa_kva ();/*
@@ -290,19 +336,7 @@*/ vaddr =__fix_to_virt (__end_of_fixed_addresses -1 ) &PMD_MASK ;page_table_range_init (vaddr,0 , pgd_base); -permanent_kmaps_init (pgd_base); - -#if CONFIG_X86_PAE -/* -* Add low memory identity-mappings - SMP needs it when -* starting up on an AP from real-mode. In the non-PAE -* case we already have these mappings through head.S. -* All user-space mappings are explicitly cleared after -* SMP startup. -*/ - pgd_base[0 ] = pgd_base[USER_PTRS_PER_PGD ]; -#endif }void zap_low_mappings (void )
@@ -314,7 +348,7 @@* Note that "pgd_clear()" doesn't do it for * us, because pgd_clear() is a no-op on i386. */ - for (i = 0; i < USER_PTRS_PER_PGD; i++) + for (i = 0; i < __USER_PTRS_PER_PGD; i++)#if CONFIG_X86_PAE set_pgd (swapper_pg_dir+i,__pgd (1 +__pa (empty_zero_page)));#else
arch/i386/mm/pgtable.c +14 -3
@@ -181,9 +181,20 @@clear_page (pmd);set_pgd (pgd + i,__pgd (1 +__pa (pmd))); } -memcpy (pgd +USER_PTRS_PER_PGD , - swapper_pg_dir +USER_PTRS_PER_PGD , - (PTRS_PER_PGD -USER_PTRS_PER_PGD ) *sizeof (pgd_t)); +if (USER_PTRS_PER_PGD <PTRS_PER_PGD ) +memcpy (pgd +USER_PTRS_PER_PGD , + swapper_pg_dir +USER_PTRS_PER_PGD , + (PTRS_PER_PGD -USER_PTRS_PER_PGD ) *sizeof (pgd_t)); +if (PARTIAL_PGD ) { +pgd_t *kpgd, *upgd; +pmd_t *kpmd, *upmd; + + kpgd =pgd_offset_k (TASK_SIZE ); + upgd = pgd +__pgd_offset (TASK_SIZE ); + kpmd =pmd_offset (kpgd,TASK_SIZE ); + upmd =pmd_offset (upgd,TASK_SIZE ); +memcpy (upmd, kpmd, (PTRS_PER_PMD -PARTIAL_PMD )*sizeof (pmd_t)); + } }return pgd; out_oom:
include/asm-alpha/pgtable.h +1 -0
@@ -39,6 +39,7 @@#define PTRS_PER_PMD (1UL << (PAGE_SHIFT-3))#define PTRS_PER_PGD (1UL << (PAGE_SHIFT-3))#define USER_PTRS_PER_PGD (TASK_SIZE / PGDIR_SIZE) +#define USER_PTRS_PER_PMD (x) (PTRS_PER_PMD)#define FIRST_USER_PGD_NR 0/* Number of pointers that fit on a page: this will go away. */
include/asm-arm/pgtable.h +1 -0
@@ -45,6 +45,7 @@#define FIRST_USER_PGD_NR 1#define USER_PTRS_PER_PGD ((TASK_SIZE/PGDIR_SIZE) - FIRST_USER_PGD_NR) +#define USER_PTRS_PER_PMD (x) (PTRS_PER_PMD)/* * The table below defines the page protection levels that we insert into our
include/asm-cris/pgtable.h +1 -0
@@ -203,6 +203,7 @@*/ #define USER_PTRS_PER_PGD (TASK_SIZE/PGDIR_SIZE) +#define USER_PTRS_PER_PMD (x) (PTRS_PER_PMD)#define FIRST_USER_PGD_NR 0/*
include/asm-i386/pgtable.h +8 -1
@@ -65,7 +65,14 @@#define PGDIR_SIZE (1UL << PGDIR_SHIFT)#define PGDIR_MASK (~(PGDIR_SIZE-1)) -#define USER_PTRS_PER_PGD (TASK_SIZE/PGDIR_SIZE) +#define __USER_PTRS_PER_PGD (TASK_SIZE/PGDIR_SIZE) +#define PARTIAL_PGD (TASK_SIZE > __USER_PTRS_PER_PGD*PGDIR_SIZE ? 1 : 0) +#define PARTIAL_PMD ((TASK_SIZE % PGDIR_SIZE)/PMD_SIZE) +#define USER_PTRS_PER_PGD (PARTIAL_PGD + __USER_PTRS_PER_PGD) +#define USER_PTRS_PER_PMD (x) ((PARTIAL_PGD && ((x)==USER_PTRS_PER_PGD-1)) ? \ + (PTRS_PER_PMD-PARTIAL_PMD) : \ + PTRS_PER_PMD) +#define FIRST_USER_PGD_NR 0#define USER_PGD_PTRS (PAGE_OFFSET >> PGDIR_SHIFT)
include/asm-ia64/pgtable.h +1 -0
@@ -87,6 +87,7 @@#define PGDIR_MASK (~(PGDIR_SIZE-1))#define PTRS_PER_PGD (__IA64_UL(1) << (PAGE_SHIFT-3))#define USER_PTRS_PER_PGD (5*PTRS_PER_PGD/8)/* regions 0-4 are user regions */ +#define USER_PTRS_PER_PMD (x) (PTRS_PER_PMD)#define FIRST_USER_PGD_NR 0/*
include/asm-m68k/pgtable.h +1 -0
@@ -58,6 +58,7 @@#define PTRS_PER_PGD 128#endif #define USER_PTRS_PER_PGD (TASK_SIZE/PGDIR_SIZE) +#define USER_PTRS_PER_PMD (x) (PTRS_PER_PMD)#define FIRST_USER_PGD_NR 0/* Virtual address region for use by kernel_map() */
include/asm-mips/pgtable.h +1 -0
@@ -98,6 +98,7 @@#define PTRS_PER_PMD 1#define PTRS_PER_PGD 1024#define USER_PTRS_PER_PGD (TASK_SIZE/PGDIR_SIZE) +#define USER_PTRS_PER_PMD (x) (PTRS_PER_PMD)#define FIRST_USER_PGD_NR 0#define VMALLOC_START KSEG2
include/asm-mips64/pgtable.h +1 -0
@@ -127,6 +127,7 @@#define PTRS_PER_PMD 1024#define PTRS_PER_PTE 512#define USER_PTRS_PER_PGD (TASK_SIZE/PGDIR_SIZE) +#define USER_PTRS_PER_PMD (x) (PTRS_PER_PMD)#define FIRST_USER_PGD_NR 0#define KPTBL_PAGE_ORDER 1
include/asm-parisc/pgtable.h +1 -0
@@ -81,6 +81,7 @@#define PGDIR_MASK (~(PGDIR_SIZE-1))#define PTRS_PER_PGD (1UL << (PAGE_SHIFT - PT_NLEVELS))#define USER_PTRS_PER_PGD PTRS_PER_PGD +#define USER_PTRS_PER_PMD (x) (PTRS_PER_PMD)/* Definitions for 2nd level */ #define pgtable_cache_init () do { } while (0)
include/asm-ppc/pgtable.h +1 -0
@@ -83,6 +83,7 @@#define PTRS_PER_PMD 1#define PTRS_PER_PGD 1024#define USER_PTRS_PER_PGD (TASK_SIZE / PGDIR_SIZE) +#define USER_PTRS_PER_PMD (x) (PTRS_PER_PMD)#define FIRST_USER_PGD_NR 0#define USER_PGD_PTRS (PAGE_OFFSET >> PGDIR_SHIFT)
include/asm-ppc64/pgtable.h +1 -0
@@ -36,6 +36,7 @@#define PTRS_PER_PGD (1 << PGD_INDEX_SIZE)#define USER_PTRS_PER_PGD (1024) +#define USER_PTRS_PER_PMD (x) (PTRS_PER_PMD)#define FIRST_USER_PGD_NR 0#define EADDR_SIZE (PTE_INDEX_SIZE + PMD_INDEX_SIZE + \
include/asm-s390/pgtable.h +1 -0
@@ -74,6 +74,7 @@* pgd entries used up by user/kernel: */ #define USER_PTRS_PER_PGD 512 +#define USER_PTRS_PER_PMD (x) (PTRS_PER_PMD)#define USER_PGD_PTRS 512#define KERNEL_PGD_PTRS 512#define FIRST_USER_PGD_NR 0
include/asm-s390x/pgtable.h +1 -0
@@ -68,6 +68,7 @@* pgd entries used up by user/kernel: */ #define USER_PTRS_PER_PGD 2048 +#define USER_PTRS_PER_PMD (x) (PTRS_PER_PMD)#define USER_PGD_PTRS 2048#define KERNEL_PGD_PTRS 2048#define FIRST_USER_PGD_NR 0
include/asm-sh/pgtable.h +1 -0
@@ -106,6 +106,7 @@#define PGDIR_MASK (~(PGDIR_SIZE-1))#define USER_PTRS_PER_PGD (TASK_SIZE/PGDIR_SIZE) +#define USER_PTRS_PER_PMD (x) (PTRS_PER_PMD)#define FIRST_USER_PGD_NR 0#define PTE_PHYS_MASK 0x1ffff000
include/asm-sparc/pgtable.h +1 -0
@@ -109,6 +109,7 @@#define PTRS_PER_PMD BTFIXUP_SIMM13(ptrs_per_pmd)#define PTRS_PER_PGD BTFIXUP_SIMM13(ptrs_per_pgd)#define USER_PTRS_PER_PGD BTFIXUP_SIMM13(user_ptrs_per_pgd) +#define USER_PTRS_PER_PMD (x) (PTRS_PER_PMD)#define FIRST_USER_PGD_NR 0#define PAGE_NONE __pgprot(BTFIXUP_INT(page_none))
include/asm-sparc64/pgtable.h +1 -0
@@ -93,6 +93,7 @@/* Kernel has a separate 44bit address space. */ #define USER_PTRS_PER_PGD ((const int)(test_thread_flag(TIF_32BIT)) ? \ (1) : (PTRS_PER_PGD)) +#define USER_PTRS_PER_PMD (x) (PTRS_PER_PMD)#define FIRST_USER_PGD_NR 0#define pte_ERROR (e) __builtin_trap()
include/asm-um/pgtable.h +1 -0
@@ -37,6 +37,7 @@#define PTRS_PER_PMD 1#define PTRS_PER_PGD 1024#define USER_PTRS_PER_PGD (TASK_SIZE/PGDIR_SIZE) +#define USER_PTRS_PER_PMD (x) (PTRS_PER_PMD)#define FIRST_USER_PGD_NR 0#define pte_ERROR (e) \
include/asm-x86_64/pgtable.h +1 -0
@@ -109,6 +109,7 @@#define PGDIR_MASK (~(PGDIR_SIZE-1))#define USER_PTRS_PER_PGD (TASK_SIZE/PGDIR_SIZE) +#define USER_PTRS_PER_PMD (x) (PTRS_PER_PMD)#define FIRST_USER_PGD_NR 0#define USER_PGD_PTRS (PAGE_OFFSET >> PGDIR_SHIFT)
mm/memory.c +7 -6
@@ -99,10 +99,11 @@pte_free_tlb (tlb, page); } -static inline void free_one_pgd (struct mmu_gather *tlb,pgd_t * dir) +static inline void free_one_pgd (struct mmu_gather *tlb,pgd_t *pgd,unsigned long pgdi) {int j;pmd_t * pmd; +pgd_t *dir = pgd + pgdi;if (pgd_none (*dir))return ;
@@ -113,7 +114,7 @@ } pmd =pmd_offset (dir,0 );pgd_clear (dir); -for (j =0 ; j <PTRS_PER_PMD ; j++) +for (j =0 ; j <USER_PTRS_PER_PMD (pgdi) ; j++)free_one_pmd (tlb, pmd+j);pmd_free_tlb (tlb, pmd); }
@@ -127,11 +128,11 @@void clear_page_tables (struct mmu_gather *tlb,unsigned long first,int nr) {pgd_t * page_dir = tlb->mm->pgd; - - page_dir += first; +int index = first; +do { -free_one_pgd (tlb, page_dir); - page_dir++; +free_one_pgd (tlb, page_dir, index); + index++; }while (--nr); }
arch/i386/vmlinux.lds.S +1 -1
@@ -7,7 +7,7 @@ jiffies = jiffies_64; SECTIONS { - . = 0xC0000000 + 0x100000; + . = 0xE0000000 + 0x100000; /* read-only */ _text = .; /* Text and read-only data */ .text : {
include/asm-i386/page.h +1 -1
@@ -89,7 +89,7 @@* and CONFIG_HIGHMEM64G options in the kernel configuration. */ -#define __PAGE_OFFSET (0xC0000000) +#define __PAGE_OFFSET (0xE0000000)/* * This much address space is reserved for vmalloc() and iomap()