s390 page tables on steroids ..
Subsystems (15)
cover s390 page tables on steroids ..
Greetings,
this patch series of six patches contains three mm related changes for the s390 architecture:
i) 1K/2K page tables. With this patch set the cheating with the pmd_t on s390 stops. So far a pmd contained 2/4 pointers instead of just one which would be correct from an architectural stand point. The Trouble with this is that it requires two common code changes to make sub-page page tables possible. This features is an important requirement for the kvm support on s390.
ii) Support for 4 levels of page tables. The address space limit for
64 bit processes is now 2^53. That should be enough for anyone ?
iii) Support for different number of page table levels. The limit
of 2^53 is nice but it slows down the tlb lookup that now has to walk 4 instead of 3 levels. Patch #5 make the number of page table levels dependent on the highest address a process is using. If an mmap is done that raises the limit to the next level, the page table gets another level. A downgrade is only done at process start, so that 31 bit processes get a two level page table. A normal 64 bit process starts with three levels.
The first three patches in the series contain the common code changes that are needed to get all of this done. I did my best to find all the place in the different architectures that need to be updated after the common code changed. Please let me know if you find a place I missed.
The patches are against Linus's git tree.
Signature · 4 lines
blue skies, Martin. "Reality continues to ruin my life." - Calvin.
PATCH 1/6 add mm argument to pte/pmd/pud/pgd_free.
From: Benjamin Herrenschmidt <[email protected]>
From: Martin Schwidefsky <[email protected]>
The pgd/pud/pmd/pte page table allocation functions get a mm_struct pointer as first argument. The free functions do not get the mm_struct argument. This is 1) asymmetrical and 2) to do mm related page table allocations the mm argument is needed on the free function as well.
Signed-off-by Benjamin Herrenschmidt <[email protected]>
Signed-off-by Martin Schwidefsky <[email protected]>
---
arch/arm/kernel/smp.c | 2 +- arch/arm/mm/ioremap.c | 2 +- arch/arm/mm/pgd.c | 8 ++++---- arch/frv/mm/pgalloc.c | 2 +- arch/powerpc/mm/pgtable_32.c | 6 +++--- arch/ppc/mm/pgtable.c | 6 +++--- arch/um/kernel/mem.c | 2 +- arch/um/kernel/skas/mmu.c | 8 ++++---- arch/x86/mm/pgtable_32.c | 2 +- include/asm-alpha/pgalloc.h | 8 ++++---- include/asm-alpha/tlb.h | 4 ++-- include/asm-arm/pgalloc.h | 10 +++++----- include/asm-arm/tlb.h | 4 ++-- include/asm-avr32/pgalloc.h | 6 +++--- include/asm-cris/pgalloc.h | 6 +++--- include/asm-frv/pgalloc.h | 8 ++++---- include/asm-frv/pgtable.h | 2 +- include/asm-generic/4level-fixup.h | 2 +- include/asm-generic/pgtable-nopmd.h | 2 +- include/asm-generic/pgtable-nopud.h | 2 +- include/asm-ia64/pgalloc.h | 16 ++++++++-------- include/asm-m32r/pgalloc.h | 10 +++++----- include/asm-m68k/motorola_pgalloc.h | 10 +++++----- include/asm-m68k/sun3_pgalloc.h | 8 ++++---- include/asm-mips/pgalloc.h | 12 ++++++------ include/asm-parisc/pgalloc.h | 10 +++++----- include/asm-parisc/tlb.h | 4 ++-- include/asm-powerpc/pgalloc-32.h | 10 +++++----- include/asm-powerpc/pgalloc-64.h | 10 +++++----- include/asm-ppc/pgalloc.h | 10 +++++----- include/asm-s390/pgalloc.h | 14 +++++++------- include/asm-s390/tlb.h | 8 ++++---- include/asm-sh/pgalloc.h | 8 ++++---- include/asm-sh64/pgalloc.h | 12 ++++++------ include/asm-sparc/pgalloc.h | 12 ++++++------ include/asm-sparc64/pgalloc.h | 8 ++++---- include/asm-sparc64/tlb.h | 4 ++-- include/asm-um/pgalloc.h | 8 ++++---- include/asm-x86/pgalloc_32.h | 8 ++++---- include/asm-x86/pgalloc_64.h | 10 +++++----- include/asm-xtensa/pgalloc.h | 6 +++--- include/asm-xtensa/tlb.h | 2 +- kernel/fork.c | 2 +- mm/memory.c | 10 +++++----- 44 files changed, 152 insertions(+), 152 deletions(-)
arch/arm/kernel/smp.c +1 -1
@@ -150,7 +150,7 @@ int __cpuinit __cpu_up(unsigned int cpu) secondary_data.pgdir =0 ; *pmd_offset (pgd,PHYS_OFFSET ) =__pmd (0 ); -pgd_free (pgd); +pgd_free (&init_mm, pgd);if (ret) {printk (KERN_CRIT "CPU%u: processor failed to boot\n" , cpu);
arch/arm/mm/ioremap.c +1 -1
@@ -162,7 +162,7 @@ static void unmap_area_sections(unsigned* Free the page table, if there was one. */ if ((pmd_val (pmd) &PMD_TYPE_MASK ) ==PMD_TYPE_TABLE ) -pte_free_kernel (pmd_page_vaddr (pmd)); +pte_free_kernel (&init_mm,pmd_page_vaddr (pmd)); } addr +=PGDIR_SIZE ;
arch/arm/mm/pgd.c +4 -4
@@ -65,14 +65,14 @@ pgd_t *get_pgd_slow(struct mm_struct *mmreturn new_pgd; no_pte: -pmd_free (new_pmd); +pmd_free (mm, new_pmd); no_pmd:free_pages ((unsigned long )new_pgd,2 ); no_pgd:return NULL ; } -void free_pgd_slow (pgd_t *pgd) +void free_pgd_slow (struct mm_struct *mm,pgd_t *pgd) {pmd_t *pmd;struct page *pte;
@@ -94,8 +94,8 @@ void free_pgd_slow(pgd_t *pgd)pmd_clear (pmd);dec_zone_page_state (virt_to_page ((unsigned long *)pgd),NR_PAGETABLE );pte_lock_deinit (pte); -pte_free (pte); -pmd_free (pmd); +pte_free (mm, pte); +pmd_free (mm, pmd); free:free_pages ((unsigned long ) pgd,2 ); }
arch/frv/mm/pgalloc.c +1 -1
@@ -140,7 +140,7 @@ pgd_t *pgd_alloc(struct mm_struct *mm)return pgd; } -void pgd_free (pgd_t *pgd) +void pgd_free (struct mm_struct *mm,pgd_t *pgd) {/* in the non-PAE case, clear_page_tables() clears user pgd entries */ quicklist_free (0 , pgd_dtor, pgd);
arch/powerpc/mm/pgtable_32.c +3 -3
@@ -86,7 +86,7 @@ pgd_t *pgd_alloc(struct mm_struct *mm)return ret; } -void pgd_free (pgd_t *pgd) +void pgd_free (struct mm_struct *mm,pgd_t *pgd) {free_pages ((unsigned long )pgd,PGDIR_ORDER ); }
@@ -123,7 +123,7 @@ struct page *pte_alloc_one(struct mm_strreturn ptepage; } -void pte_free_kernel (pte_t *pte) +void pte_free_kernel (struct mm_struct *mm,pte_t *pte) {#ifdef CONFIG_SMP hash_page_sync ();
@@ -131,7 +131,7 @@ void pte_free_kernel(pte_t *pte)free_page ((unsigned long )pte); } -void pte_free (struct page *ptepage) +void pte_free (struct mm_struct *mm,struct page *ptepage) {#ifdef CONFIG_SMP hash_page_sync ();
arch/ppc/mm/pgtable.c +3 -3
@@ -87,7 +87,7 @@ pgd_t *pgd_alloc(struct mm_struct *mm)return ret; } -void pgd_free (pgd_t *pgd) +void pgd_free (struct mm_struct *mm,pgd_t *pgd) {free_pages ((unsigned long )pgd,PGDIR_ORDER ); }
@@ -124,7 +124,7 @@ struct page *pte_alloc_one(struct mm_strreturn ptepage; } -void pte_free_kernel (pte_t *pte) +void pte_free_kernel (struct mm_struct *mm,pte_t *pte) {#ifdef CONFIG_SMP hash_page_sync ();
@@ -132,7 +132,7 @@ void pte_free_kernel(pte_t *pte)free_page ((unsigned long )pte); } -void pte_free (struct page *ptepage) +void pte_free (struct mm_struct *mm,struct page *ptepage) {#ifdef CONFIG_SMP hash_page_sync ();
arch/um/kernel/mem.c +1 -1
@@ -348,7 +348,7 @@ pgd_t *pgd_alloc(struct mm_struct *mm)return pgd; } -void pgd_free (pgd_t *pgd) +void pgd_free (struct mm_struct *mm,pgd_t *pgd) {free_page ((unsigned long ) pgd); }
arch/um/kernel/skas/mmu.c +4 -4
@@ -58,9 +58,9 @@ static int init_stub_pte(struct mm_strucreturn 0 ; out_pmd: -pud_free (pud); +pud_free (mm, pud); out_pte: -pmd_free (pmd); +pmd_free (mm, pmd); out:return -ENOMEM ; }
@@ -144,10 +144,10 @@ void destroy_context(struct mm_struct *mif (!proc_mm || !ptrace_faultinfo) {free_page (mmu->id.stack);pte_lock_deinit (virt_to_page (mmu->last_page_table)); -pte_free_kernel ((pte_t *) mmu->last_page_table); +pte_free_kernel (mm, (pte_t *) mmu->last_page_table);dec_zone_page_state (virt_to_page (mmu->last_page_table),NR_PAGETABLE );#ifdef CONFIG_3_LEVEL_PGTABLES -pmd_free ((pmd_t *) mmu->last_pmd); +pmd_free (mm, (pmd_t *) mmu->last_pmd);#endif }
arch/x86/mm/pgtable_32.c +1 -1
@@ -352,7 +352,7 @@ out_oom:return NULL ; } -void pgd_free (pgd_t *pgd) +void pgd_free (struct mm_struct *mm,pgd_t *pgd) {int i;
include/asm-alpha/pgalloc.h +4 -4
@@ -31,7 +31,7 @@ pgd_populate(struct mm_struct *mm, pgd_textern pgd_t *pgd_alloc (struct mm_struct *mm);static inline void -pgd_free (pgd_t *pgd) +pgd_free (struct mm_struct *mm,pgd_t *pgd) {free_page ((unsigned long )pgd); }
@@ -44,7 +44,7 @@ pmd_alloc_one(struct mm_struct *mm, unsi }static inline void -pmd_free (pmd_t *pmd) +pmd_free (struct mm_struct *mm,pmd_t *pmd) {free_page ((unsigned long )pmd); }
@@ -52,7 +52,7 @@ pmd_free(pmd_t *pmd)extern pte_t *pte_alloc_one_kernel (struct mm_struct *mm,unsigned long addr);static inline void -pte_free_kernel (pte_t *pte) +pte_free_kernel (struct mm_struct *mm,pte_t *pte) {free_page ((unsigned long )pte); }
@@ -67,7 +67,7 @@ pte_alloc_one(struct mm_struct *mm, unsi }static inline void -pte_free (struct page *page) +pte_free (struct mm_struct *mm,struct page *page) {__free_page (page); }
include/asm-alpha/tlb.h +2 -2
@@ -9,7 +9,7 @@#include <asm-generic/tlb.h> -#define __pte_free_tlb (tlb,pte) pte_free(pte) -#define __pmd_free_tlb (tlb,pmd) pmd_free(pmd) +#define __pte_free_tlb (tlb,pte) pte_free((tlb)->mm, pte) +#define __pmd_free_tlb (tlb,pmd) pmd_free((tlb)->mm, pmd)#endif
include/asm-arm/pgalloc.h +5 -5
@@ -27,14 +27,14 @@* Since we have only two-level page tables, these are trivial */ #define pmd_alloc_one (mm,addr) ({ BUG(); ((pmd_t *)2); }) -#define pmd_free (pmd) do { } while (0) +#define pmd_free (mm,pmd) do { } while (0)#define pgd_populate (mm,pmd,pte) BUG()extern pgd_t *get_pgd_slow (struct mm_struct *mm); -extern void free_pgd_slow (pgd_t *pgd); +extern void free_pgd_slow (struct mm_struct *mm,pgd_t *pgd);#define pgd_alloc (mm) get_pgd_slow(mm) -#define pgd_free (pgd) free_pgd_slow(pgd) +#define pgd_free (mm, pgd) free_pgd_slow(mm, pgd)/* * Allocate one PTE table.
@@ -83,7 +83,7 @@ pte_alloc_one(struct mm_struct *mm, unsi/* * Free one PTE table. */ -static inline void pte_free_kernel (pte_t *pte) +static inline void pte_free_kernel (struct mm_struct *mm,pte_t *pte) {if (pte) { pte -=PTRS_PER_PTE ;
@@ -91,7 +91,7 @@ static inline void pte_free_kernel(pte_t } } -static inline void pte_free (struct page *pte) +static inline void pte_free (struct mm_struct *mm,struct page *pte) {__free_page (pte); }
include/asm-arm/tlb.h +2 -2
@@ -85,8 +85,8 @@ tlb_end_vma(struct mmu_gather *tlb, stru }#define tlb_remove_page (tlb,page) free_page_and_swap_cache(page) -#define pte_free_tlb (tlb,ptep) pte_free(ptep) -#define pmd_free_tlb (tlb,pmdp) pmd_free(pmdp) +#define pte_free_tlb (tlb,ptep) pte_free((tlb)->mm, ptep) +#define pmd_free_tlb (tlb,pmdp) pmd_free((tlb)->mm, pmdp)#define tlb_migrate_finish (mm) do { } while (0)
include/asm-avr32/pgalloc.h +3 -3
@@ -30,7 +30,7 @@ static __inline__ pgd_t *pgd_alloc(strucreturn kcalloc (USER_PTRS_PER_PGD ,sizeof (pgd_t),GFP_KERNEL ); } -static inline void pgd_free (pgd_t *pgd) +static inline void pgd_free (struct mm_struct *mm,pgd_t *pgd) {kfree (pgd); }
@@ -55,12 +55,12 @@ static inline struct page *pte_alloc_onereturn pte; } -static inline void pte_free_kernel (pte_t *pte) +static inline void pte_free_kernel (struct mm_struct *mm,pte_t *pte) {free_page ((unsigned long )pte); } -static inline void pte_free (struct page *pte) +static inline void pte_free (struct mm_struct *mm,struct page *pte) {__free_page (pte); }
include/asm-cris/pgalloc.h +3 -3
@@ -16,7 +16,7 @@ static inline pgd_t *pgd_alloc (struct mreturn (pgd_t *)get_zeroed_page (GFP_KERNEL ); } -static inline void pgd_free (pgd_t *pgd) +static inline void pgd_free (struct mm_struct *mm,pgd_t *pgd) {free_page ((unsigned long )pgd); }
@@ -34,12 +34,12 @@ static inline struct page *pte_alloc_onereturn pte; } -static inline void pte_free_kernel (pte_t *pte) +static inline void pte_free_kernel (struct mm_struct *mm,pte_t *pte) {free_page ((unsigned long )pte); } -static inline void pte_free (struct page *pte) +static inline void pte_free (struct mm_struct *mm,struct page *pte) {__free_page (pte); }
include/asm-frv/pgalloc.h +4 -4
@@ -31,18 +31,18 @@ do { \*/ extern pgd_t *pgd_alloc (struct mm_struct *); -extern void pgd_free (pgd_t *); +extern void pgd_free (struct mm_struct *mm,pgd_t *);extern pte_t *pte_alloc_one_kernel (struct mm_struct *,unsigned long );extern struct page *pte_alloc_one (struct mm_struct *,unsigned long ); -static inline void pte_free_kernel (pte_t *pte) +static inline void pte_free_kernel (struct mm_struct *mm,pte_t *pte) {free_page ((unsigned long )pte); } -static inline void pte_free (struct page *pte) +static inline void pte_free (struct mm_struct *mm,struct page *pte) {__free_page (pte); }
@@ -55,7 +55,7 @@ static inline void pte_free(struct page* (In the PAE case we free the pmds as part of the pgd.) */ #define pmd_alloc_one (mm, addr) ({ BUG(); ((pmd_t *) 2); }) -#define pmd_free (x) do { } while (0) +#define pmd_free (mm, x) do { } while (0)#define __pmd_free_tlb (tlb,x) do { } while (0)#endif /* CONFIG_MMU */
include/asm-frv/pgtable.h +1 -1
@@ -226,7 +226,7 @@ static inline pud_t *pud_offset(pgd_t *p* inside the pgd, so has no extra memory associated with it. */ #define pud_alloc_one (mm, address) NULL -#define pud_free (x) do { } while (0) +#define pud_free (mm, x) do { } while (0)#define __pud_free_tlb (tlb, x) do { } while (0)/*
include/asm-generic/4level-fixup.h +1 -1
@@ -28,7 +28,7 @@#undef pud_free_tlb#define pud_free_tlb (tlb, x) do { } while (0) -#define pud_free (x) do { } while (0) +#define pud_free (mm, x) do { } while (0)#define __pud_free_tlb (tlb, x) do { } while (0)#undef pud_addr_end
include/asm-generic/pgtable-nopmd.h +1 -1
@@ -54,7 +54,7 @@ static inline pmd_t * pmd_offset(pud_t ** inside the pud, so has no extra memory associated with it. */ #define pmd_alloc_one (mm, address) NULL -#define pmd_free (x) do { } while (0) +#define pmd_free (mm, x) do { } while (0)#define __pmd_free_tlb (tlb, x) do { } while (0)#undef pmd_addr_end
include/asm-generic/pgtable-nopud.h +1 -1
@@ -51,7 +51,7 @@ static inline pud_t * pud_offset(pgd_t ** inside the pgd, so has no extra memory associated with it. */ #define pud_alloc_one (mm, address) NULL -#define pud_free (x) do { } while (0) +#define pud_free (mm, x) do { } while (0)#define __pud_free_tlb (tlb, x) do { } while (0)#undef pud_addr_end
include/asm-ia64/pgalloc.h +8 -8
@@ -27,7 +27,7 @@ static inline pgd_t *pgd_alloc(struct mmreturn quicklist_alloc (0 ,GFP_KERNEL ,NULL ); } -static inline void pgd_free (pgd_t * pgd) +static inline void pgd_free (struct mm_struct *mm,pgd_t * pgd) {quicklist_free (0 ,NULL , pgd); }
@@ -44,11 +44,11 @@ static inline pud_t *pud_alloc_one(strucreturn quicklist_alloc (0 ,GFP_KERNEL ,NULL ); } -static inline void pud_free (pud_t * pud) +static inline void pud_free (struct mm_struct *mm,pud_t * pud) {quicklist_free (0 ,NULL , pud); } -#define __pud_free_tlb (tlb, pud) pud_free(pud) +#define __pud_free_tlb (tlb, pud) pud_free((tlb)->mm, pud)#endif /* CONFIG_PGTABLE_4 */ static inline void
@@ -62,12 +62,12 @@ static inline pmd_t *pmd_alloc_one(strucreturn quicklist_alloc (0 ,GFP_KERNEL ,NULL ); } -static inline void pmd_free (pmd_t * pmd) +static inline void pmd_free (struct mm_struct *mm,pmd_t * pmd) {quicklist_free (0 ,NULL , pmd); } -#define __pmd_free_tlb (tlb, pmd) pmd_free(pmd) +#define __pmd_free_tlb (tlb, pmd) pmd_free((tlb)->mm, pmd)static inline void pmd_populate (struct mm_struct *mm,pmd_t * pmd_entry,struct page *pte)
@@ -94,12 +94,12 @@ static inline pte_t *pte_alloc_one_kernereturn quicklist_alloc (0 ,GFP_KERNEL ,NULL ); } -static inline void pte_free (struct page *pte) +static inline void pte_free (struct mm_struct *mm,struct page *pte) {quicklist_free_page (0 ,NULL , pte); } -static inline void pte_free_kernel (pte_t * pte) +static inline void pte_free_kernel (struct mm_struct *mm,pte_t * pte) {quicklist_free (0 ,NULL , pte); }
@@ -109,6 +109,6 @@ static inline void check_pgt_cache(void) quicklist_trim(0, NULL, 25, 16); } -#define __pte_free_tlb (tlb, pte) pte_free(pte) +#define __pte_free_tlb (tlb, pte) pte_free((tlb)->mm, pte)#endif /* _ASM_IA64_PGALLOC_H */
include/asm-m32r/pgalloc.h +5 -5
@@ -24,7 +24,7 @@ static __inline__ pgd_t *pgd_alloc(strucreturn pgd; } -static __inline__void pgd_free (pgd_t *pgd) +static __inline__void pgd_free (struct mm_struct *mm,pgd_t *pgd) {free_page ((unsigned long )pgd); }
@@ -46,17 +46,17 @@ static __inline__ struct page *pte_allocreturn pte; } -static __inline__void pte_free_kernel (pte_t *pte) +static __inline__void pte_free_kernel (struct mm_struct *mm,pte_t *pte) {free_page ((unsigned long )pte); } -static __inline__void pte_free (struct page *pte) +static __inline__void pte_free (struct mm_struct *mm,struct page *pte) {__free_page (pte); } -#define __pte_free_tlb (tlb, pte) pte_free((pte)) +#define __pte_free_tlb (tlb, pte) pte_free((tlb)->mm, (pte))/* * allocating and freeing a pmd is trivial: the 1-entry pmd is
@@ -65,7 +65,7 @@ static __inline__ void pte_free(struct p*/ #define pmd_alloc_one (mm, addr) ({ BUG(); ((pmd_t *)2); }) -#define pmd_free (x) do { } while (0) +#define pmd_free (mm, x) do { } while (0)#define __pmd_free_tlb (tlb, x) do { } while (0)#define pgd_populate (mm, pmd, pte) BUG()
include/asm-m68k/motorola_pgalloc.h +5 -5
@@ -22,7 +22,7 @@ static inline pte_t *pte_alloc_one_kernereturn pte; } -static inline void pte_free_kernel (pte_t *pte) +static inline void pte_free_kernel (struct mm_struct *mm,pte_t *pte) {cache_page (pte);free_page ((unsigned long ) pte);
@@ -47,7 +47,7 @@ static inline struct page *pte_alloc_onereturn page; } -static inline void pte_free (struct page *page) +static inline void pte_free (struct mm_struct *mm,struct page *page) {cache_page (kmap (page));kunmap (page);
@@ -67,7 +67,7 @@ static inline pmd_t *pmd_alloc_one(strucreturn get_pointer_table (); } -static inline int pmd_free (pmd_t *pmd) +static inline int pmd_free (struct mm_struct *mm,pmd_t *pmd) {return free_pointer_table (pmd); }
@@ -78,9 +78,9 @@ static inline int __pmd_free_tlb(struct } -static inline void pgd_free (pgd_t *pgd) +static inline void pgd_free (struct mm_struct *mm,pgd_t *pgd) { -pmd_free ((pmd_t *)pgd); +pmd_free (mm, (pmd_t *)pgd); }static inline pgd_t *pgd_alloc (struct mm_struct *mm)
include/asm-m68k/sun3_pgalloc.h +4 -4
@@ -21,12 +21,12 @@ extern const char bad_pmd_string[];#define pmd_alloc_one (mm,address) ({ BUG(); ((pmd_t *)2); }) -static inline void pte_free_kernel (pte_t * pte) +static inline void pte_free_kernel (struct mm_struct *mm,pte_t * pte) {free_page ((unsigned long ) pte); } -static inline void pte_free (struct page *page) +static inline void pte_free (struct mm_struct *mm,struct page *page) {__free_page (page); }
@@ -72,10 +72,10 @@ static inline void pmd_populate(struct m* allocating and freeing a pmd is trivial: the 1-entry pmd is * inside the pgd, so has no extra memory associated with it. */ -#define pmd_free (x) do { } while (0) +#define pmd_free (mm, x) do { } while (0)#define __pmd_free_tlb (tlb, x) do { } while (0) -static inline void pgd_free (pgd_t * pgd) +static inline void pgd_free (struct mm_struct *mm,pgd_t * pgd) {free_page ((unsigned long ) pgd); }
include/asm-mips/pgalloc.h +6 -6
@@ -58,7 +58,7 @@ static inline pgd_t *pgd_alloc(struct mmreturn ret; } -static inline void pgd_free (pgd_t *pgd) +static inline void pgd_free (struct mm_struct *mm,pgd_t *pgd) {free_pages ((unsigned long )pgd,PGD_ORDER ); }
@@ -85,12 +85,12 @@ static inline struct page *pte_alloc_onereturn pte; } -static inline void pte_free_kernel (pte_t *pte) +static inline void pte_free_kernel (struct mm_struct *mm,pte_t *pte) {free_pages ((unsigned long )pte,PTE_ORDER ); } -static inline void pte_free (struct page *pte) +static inline void pte_free (struct mm_struct *mm,struct page *pte) {__free_pages (pte,PTE_ORDER ); }
@@ -103,7 +103,7 @@ static inline void pte_free(struct page* allocating and freeing a pmd is trivial: the 1-entry pmd is * inside the pgd, so has no extra memory associated with it. */ -#define pmd_free (x) do { } while (0) +#define pmd_free (mm, x) do { } while (0)#define __pmd_free_tlb (tlb, x) do { } while (0)#endif
@@ -120,12 +120,12 @@ static inline pmd_t *pmd_alloc_one(struc return pmd; } -static inline void pmd_free (pmd_t *pmd) +static inline void pmd_free (struct mm_struct *mm,pmd_t *pmd) {free_pages ((unsigned long )pmd,PMD_ORDER ); } -#define __pmd_free_tlb (tlb, x) pmd_free(x) +#define __pmd_free_tlb (tlb, x) pmd_free((tlb)->mm, x)#endif
include/asm-parisc/pgalloc.h +5 -5
@@ -43,7 +43,7 @@ static inline pgd_t *pgd_alloc(struct mmreturn actual_pgd; } -static inline void pgd_free (pgd_t *pgd) +static inline void pgd_free (struct mm_struct *mm,pgd_t *pgd) {#ifdef CONFIG_64BIT pgd -=PTRS_PER_PGD ;
@@ -70,7 +70,7 @@ static inline pmd_t *pmd_alloc_one(strucreturn pmd; } -static inline void pmd_free (pmd_t *pmd) +static inline void pmd_free (struct mm_struct *mm,pmd_t *pmd) {#ifdef CONFIG_64BIT if (pmd_flag (*pmd) & PxD_FLAG_ATTACHED)
@@ -91,7 +91,7 @@ static inline void pmd_free(pmd_t *pmd)*/ #define pmd_alloc_one (mm, addr) ({ BUG(); ((pmd_t *)2); }) -#define pmd_free (x) do { } while (0) +#define pmd_free (mm, x) do { } while (0)#define pgd_populate (mm, pmd, pte) BUG()#endif
@@ -130,12 +130,12 @@ pte_alloc_one_kernel(struct mm_struct *mreturn pte; } -static inline void pte_free_kernel (pte_t *pte) +static inline void pte_free_kernel (struct mm_struct *mm,pte_t *pte) {free_page ((unsigned long )pte); } -#define pte_free (page) pte_free_kernel(page_address(page)) +#define pte_free (mm, page) pte_free_kernel(page_address(page))#define check_pgt_cache () do { } while (0)
include/asm-parisc/tlb.h +2 -2
@@ -21,7 +21,7 @@ do { if (!(tlb)->fullmm) \#include <asm-generic/tlb.h> -#define __pmd_free_tlb (tlb, pmd) pmd_free(pmd) -#define __pte_free_tlb (tlb, pte) pte_free(pte) +#define __pmd_free_tlb (tlb, pmd) pmd_free((tlb)->mm, pmd) +#define __pte_free_tlb (tlb, pte) pte_free((tlb)->mm, pte)#endif
include/asm-powerpc/pgalloc-32.h +5 -5
@@ -6,14 +6,14 @@extern void __bad_pte (pmd_t *pmd);extern pgd_t *pgd_alloc (struct mm_struct *mm); -extern void pgd_free (pgd_t *pgd); +extern void pgd_free (struct mm_struct *mm,pgd_t *pgd);/* * We don't have any real pmd's, and this code never triggers because * the pgd will always be present.. */ /* #define pmd_alloc_one(mm,address) ({ BUG(); ((pmd_t *)2); }) */ -#define pmd_free (x) do { } while (0) +#define pmd_free (mm, x) do { } while (0)#define __pmd_free_tlb (tlb,x) do { } while (0)/* #define pgd_populate(mm, pmd, pte) BUG() */
@@ -31,10 +31,10 @@ extern void pgd_free(pgd_t *pgd);extern pte_t *pte_alloc_one_kernel (struct mm_struct *mm,unsigned long addr);extern struct page *pte_alloc_one (struct mm_struct *mm,unsigned long addr); -extern void pte_free_kernel (pte_t *pte); -extern void pte_free (struct page *pte); +extern void pte_free_kernel (struct mm_struct *mm,pte_t *pte); +extern void pte_free (struct mm_struct *mm,struct page *pte); -#define __pte_free_tlb (tlb, pte) pte_free((pte)) +#define __pte_free_tlb (tlb, pte) pte_free((tlb)->mm, (pte))#define check_pgt_cache () do { } while (0)
include/asm-powerpc/pgalloc-64.h +5 -5
@@ -25,7 +25,7 @@ static inline pgd_t *pgd_alloc(struct mmreturn kmem_cache_alloc (pgtable_cache[PGD_CACHE_NUM ],GFP_KERNEL ); } -static inline void pgd_free (pgd_t *pgd) +static inline void pgd_free (struct mm_struct *mm,pgd_t *pgd) {kmem_cache_free (pgtable_cache[PGD_CACHE_NUM ], pgd); }
@@ -40,7 +40,7 @@ static inline pud_t *pud_alloc_one(struc GFP_KERNEL|__GFP_REPEAT); } -static inline void pud_free (pud_t *pud) +static inline void pud_free (struct mm_struct *mm,pud_t *pud) {kmem_cache_free (pgtable_cache[PUD_CACHE_NUM ], pud); }
@@ -76,7 +76,7 @@ static inline pmd_t *pmd_alloc_one(struc GFP_KERNEL|__GFP_REPEAT); } -static inline void pmd_free (pmd_t *pmd) +static inline void pmd_free (struct mm_struct *mm,pmd_t *pmd) {kmem_cache_free (pgtable_cache[PMD_CACHE_NUM ], pmd); }
@@ -94,12 +94,12 @@ static inline struct page *pte_alloc_onereturn pte ?virt_to_page (pte) :NULL ; } -static inline void pte_free_kernel (pte_t *pte) +static inline void pte_free_kernel (struct mm_struct *mm,pte_t *pte) {free_page ((unsigned long )pte); } -static inline void pte_free (struct page *ptepage) +static inline void pte_free (struct mm_struct *mm,struct page *ptepage) {__free_page (ptepage); }
include/asm-ppc/pgalloc.h +5 -5
@@ -7,14 +7,14 @@extern void __bad_pte (pmd_t *pmd);extern pgd_t *pgd_alloc (struct mm_struct *mm); -extern void pgd_free (pgd_t *pgd); +extern void pgd_free (struct mm_struct *mm,pgd_t *pgd);/* * We don't have any real pmd's, and this code never triggers because * the pgd will always be present.. */ #define pmd_alloc_one (mm,address) ({ BUG(); ((pmd_t *)2); }) -#define pmd_free (x) do { } while (0) +#define pmd_free (mm, x) do { } while (0)#define __pmd_free_tlb (tlb,x) do { } while (0)#define pgd_populate (mm, pmd, pte) BUG()
@@ -32,10 +32,10 @@ extern void pgd_free(pgd_t *pgd);extern pte_t *pte_alloc_one_kernel (struct mm_struct *mm,unsigned long addr);extern struct page *pte_alloc_one (struct mm_struct *mm,unsigned long addr); -extern void pte_free_kernel (pte_t *pte); -extern void pte_free (struct page *pte); +extern void pte_free_kernel (struct mm_struct *mm,pte_t *pte); +extern void pte_free (struct mm_struct *mm,struct page *pte); -#define __pte_free_tlb (tlb, pte) pte_free((pte)) +#define __pte_free_tlb (tlb, pte) pte_free((tlb)->mm, (pte))#define check_pgt_cache () do { } while (0)
include/asm-s390/pgalloc.h +7 -7
@@ -57,10 +57,10 @@ static inline unsigned long pgd_entry_ty }#define pud_alloc_one (mm,address) ({ BUG(); ((pud_t *)2); }) -#define pud_free (x) do { } while (0) +#define pud_free (mm, x) do { } while (0)#define pmd_alloc_one (mm,address) ({ BUG(); ((pmd_t *)2); }) -#define pmd_free (x) do { } while (0) +#define pmd_free (mm, x) do { } while (0)#define pgd_populate (mm, pgd, pud) BUG()#define pgd_populate_kernel (mm, pgd, pud) BUG()
@@ -76,7 +76,7 @@ static inline unsigned long pgd_entry_ty }#define pud_alloc_one (mm,address) ({ BUG(); ((pud_t *)2); }) -#define pud_free (x) do { } while (0) +#define pud_free (mm, x) do { } while (0)static inline pmd_t *pmd_alloc_one (struct mm_struct *mm,unsigned long vmaddr) {
@@ -85,7 +85,7 @@ static inline pmd_t *pmd_alloc_one(struccrst_table_init (crst, _SEGMENT_ENTRY_EMPTY);return (pmd_t *) crst; } -#define pmd_free (pmd) crst_table_free((unsigned long *) pmd) +#define pmd_free (mm, pmd) crst_table_free((unsigned long *) pmd)#define pgd_populate (mm, pgd, pud) BUG()#define pgd_populate_kernel (mm, pgd, pud) BUG()
@@ -115,7 +115,7 @@ static inline pgd_t *pgd_alloc(struct mmcrst_table_init (crst,pgd_entry_type (mm));return (pgd_t *) crst; } -#define pgd_free (pgd) crst_table_free((unsigned long *) pgd) +#define pgd_free (mm, pgd) crst_table_free((unsigned long *) pgd)static inline void pmd_populate_kernel (struct mm_struct *mm,pmd_t *pmd,pte_t *pte)
@@ -151,9 +151,9 @@ pmd_populate(struct mm_struct *mm, pmd_t#define pte_alloc_one (mm, vmaddr) \ virt_to_page(page_table_alloc(s390_noexec)) -#define pte_free_kernel (pte) \ +#define pte_free_kernel (mm, pte) \ page_table_free((unsigned long *) pte) -#define pte_free (pte) \ +#define pte_free (mm, pte) \ page_table_free((unsigned long *) page_to_phys((struct page *) pte))#endif /* _S390_PGALLOC_H */
include/asm-s390/tlb.h +4 -4
@@ -65,9 +65,9 @@ static inline void tlb_flush_mmu(structif (!tlb->fullmm && (tlb->nr_ptes >0 || tlb->nr_pmds <TLB_NR_PTRS ))__tlb_flush_mm (tlb->mm);while (tlb->nr_ptes >0 ) -pte_free (tlb->array[--tlb->nr_ptes]); +pte_free (tlb->mm, tlb->array[--tlb->nr_ptes]);while (tlb->nr_pmds <TLB_NR_PTRS ) -pmd_free ((pmd_t *) tlb->array[tlb->nr_pmds++]); +pmd_free (tlb->mm, (pmd_t *) tlb->array[tlb->nr_pmds++]); } static inline void tlb_finish_mmu(struct mmu_gather *tlb,
@@ -102,7 +102,7 @@ static inline void pte_free_tlb(struct mif (tlb->nr_ptes >= tlb->nr_pmds)tlb_flush_mmu (tlb,0 ,0 ); }else -pte_free (page); +pte_free (tlb->mm, page); }/*
@@ -117,7 +117,7 @@ static inline void pmd_free_tlb(struct mif (tlb->nr_ptes >= tlb->nr_pmds) tlb_flush_mmu(tlb, 0, 0); }else -pmd_free (pmd); +pmd_free (tlb->mm, pmd);#endif }
include/asm-sh/pgalloc.h +4 -4
@@ -36,7 +36,7 @@ static inline pgd_t *pgd_alloc(struct mmreturn quicklist_alloc (QUICK_PGD ,GFP_KERNEL | __GFP_REPEAT, pgd_ctor); } -static inline void pgd_free (pgd_t *pgd) +static inline void pgd_free (struct mm_struct *mm,pgd_t *pgd) {quicklist_free (QUICK_PGD ,NULL , pgd); }
@@ -54,12 +54,12 @@ static inline struct page *pte_alloc_onereturn pg ?virt_to_page (pg) :NULL ; } -static inline void pte_free_kernel (pte_t *pte) +static inline void pte_free_kernel (struct mm_struct *mm,pte_t *pte) {quicklist_free (QUICK_PT ,NULL , pte); } -static inline void pte_free (struct page *pte) +static inline void pte_free (struct mm_struct *mm,struct page *pte) {quicklist_free_page (QUICK_PT ,NULL , pte); }
@@ -71,7 +71,7 @@ static inline void pte_free(struct page* inside the pgd, so has no extra memory associated with it. */ -#define pmd_free (x) do { } while (0) +#define pmd_free (mm, x) do { } while (0)#define __pmd_free_tlb (tlb,x) do { } while (0)static inline void check_pgt_cache (void )
include/asm-sh64/pgalloc.h +6 -6
@@ -46,7 +46,7 @@ static inline pgd_t *pgd_alloc(struct mmreturn quicklist_alloc (0 ,GFP_KERNEL ,NULL ); } -static inline void pgd_free (pgd_t *pgd) +static inline void pgd_free (struct mm_struct *mm,pgd_t *pgd) {quicklist_free (0 ,NULL , pgd); }
@@ -58,12 +58,12 @@ static inline struct page *pte_alloc_onereturn pg ?virt_to_page (pg) :NULL ; } -static inline void pte_free_kernel (pte_t *pte) +static inline void pte_free_kernel (struct mm_struct *mm,pte_t *pte) {quicklist_free (0 ,NULL , pte); } -static inline void pte_free (struct page *pte) +static inline void pte_free (struct mm_struct *mm,struct page *pte) {quicklist_free_page (0 ,NULL , pte); }
@@ -84,7 +84,7 @@ static inline pte_t *pte_alloc_one_kerne#if defined(CONFIG_SH64_PGTABLE_2_LEVEL )#define pmd_alloc_one (mm, addr) ({ BUG(); ((pmd_t *)2); }) -#define pmd_free (x) do { } while (0) +#define pmd_free (mm, x) do { } while (0)#define pgd_populate (mm, pmd, pte) BUG()#define __pte_free_tlb (tlb,pte) tlb_remove_page((tlb),(pte))#define __pmd_free_tlb (tlb,pmd) do { } while (0)
@@ -96,13 +96,13 @@ static inline pmd_t *pmd_alloc_one(struc return quicklist_alloc(0, GFP_KERNEL, NULL); } -static inline void pmd_free (pmd_t *pmd) +static inline void pmd_free (struct mm_struct *mm,pmd_t *pmd) {quicklist_free (0 ,NULL , pmd); }#define pgd_populate (mm, pgd, pmd) pgd_set(pgd, pmd) -#define __pmd_free_tlb (tlb,pmd) pmd_free(pmd) +#define __pmd_free_tlb (tlb,pmd) pmd_free((tlb)->mm, pmd)#else #error "No defined page table size"
include/asm-sparc/pgalloc.h +6 -6
@@ -32,7 +32,7 @@ BTFIXUPDEF_CALL(pgd_t *, get_pgd_fast, v BTFIXUPDEF_CALL(void, free_pgd_fast, pgd_t *) #define free_pgd_fast(pgd) BTFIXUP_CALL(free_pgd_fast)(pgd) -#define pgd_free(pgd) free_pgd_fast(pgd) +#define pgd_free(mm,pgd) free_pgd_fast(pgd)#define pgd_alloc (mm) get_pgd_fast() BTFIXUPDEF_CALL(void, pgd_set, pgd_t *, pmd_t *)
@@ -45,8 +45,8 @@ BTFIXUPDEF_CALL(pmd_t *, pmd_alloc_one, BTFIXUPDEF_CALL(void, free_pmd_fast, pmd_t *) #define free_pmd_fast(pmd) BTFIXUP_CALL(free_pmd_fast)(pmd) -#define pmd_free(pmd) free_pmd_fast(pmd) -#define __pmd_free_tlb (tlb, pmd) pmd_free(pmd) +#define pmd_free(mm, pmd) free_pmd_fast(pmd) +#define __pmd_free_tlb (tlb, pmd) pmd_free((tlb)->mm, pmd) BTFIXUPDEF_CALL(void, pmd_populate, pmd_t *, struct page *) #define pmd_populate(MM, PMD, PTE) BTFIXUP_CALL(pmd_populate)(PMD, PTE)
@@ -59,10 +59,10 @@ BTFIXUPDEF_CALL(pte_t *, pte_alloc_one_k#define pte_alloc_one_kernel (mm, addr) BTFIXUP_CALL(pte_alloc_one_kernel)(mm, addr) BTFIXUPDEF_CALL(void, free_pte_fast, pte_t *) -#define pte_free_kernel(pte) BTFIXUP_CALL(free_pte_fast)(pte) +#define pte_free_kernel(mm,pte) BTFIXUP_CALL(free_pte_fast)(pte) BTFIXUPDEF_CALL(void, pte_free, struct page *) -#define pte_free(pte) BTFIXUP_CALL(pte_free)(pte) -#define __pte_free_tlb(tlb, pte) pte_free(pte) +#define pte_free(mm,pte) BTFIXUP_CALL(pte_free)(pte) +#define __pte_free_tlb(tlb, pte) pte_free((tlb)->mm, pte)#endif /* _SPARC_PGALLOC_H */
include/asm-sparc64/pgalloc.h +4 -4
@@ -20,7 +20,7 @@ static inline pgd_t *pgd_alloc(struct mmreturn quicklist_alloc (0 ,GFP_KERNEL ,NULL ); } -static inline void pgd_free (pgd_t *pgd) +static inline void pgd_free (struct mm_struct *mm,pgd_t *pgd) {quicklist_free (0 ,NULL , pgd); }
@@ -32,7 +32,7 @@ static inline pmd_t *pmd_alloc_one(strucreturn quicklist_alloc (0 ,GFP_KERNEL ,NULL ); } -static inline void pmd_free (pmd_t *pmd) +static inline void pmd_free (struct mm_struct *mm,pmd_t *pmd) {quicklist_free (0 ,NULL , pmd); }
@@ -50,12 +50,12 @@ static inline struct page *pte_alloc_onereturn pg ?virt_to_page (pg) :NULL ; } -static inline void pte_free_kernel (pte_t *pte) +static inline void pte_free_kernel (struct mm_struct *mm,pte_t *pte) {quicklist_free (0 ,NULL , pte); } -static inline void pte_free (struct page *ptepage) +static inline void pte_free (struct mm_struct *mm,struct page *ptepage) {quicklist_free_page (0 ,NULL , ptepage); }
include/asm-sparc64/tlb.h +2 -2
@@ -100,8 +100,8 @@ static inline void tlb_remove_page(struc }#define tlb_remove_tlb_entry (mp,ptep,addr) do { } while (0) -#define pte_free_tlb (mp,ptepage) pte_free(ptepage) -#define pmd_free_tlb (mp,pmdp) pmd_free(pmdp) +#define pte_free_tlb (mp,ptepage) pte_free((mp)->mm, ptepage) +#define pmd_free_tlb (mp,pmdp) pmd_free((mp)->mm, pmdp)#define pud_free_tlb (tlb,pudp) __pud_free_tlb(tlb,pudp)#define tlb_migrate_finish (mm) do { } while (0)
include/asm-um/pgalloc.h +4 -4
@@ -23,17 +23,17 @@* Allocate and free page tables. */ extern pgd_t *pgd_alloc (struct mm_struct *); -extern void pgd_free (pgd_t *pgd); +extern void pgd_free (struct mm_struct *mm,pgd_t *pgd);extern pte_t *pte_alloc_one_kernel (struct mm_struct *,unsigned long );extern struct page *pte_alloc_one (struct mm_struct *,unsigned long ); -static inline void pte_free_kernel (pte_t *pte) +static inline void pte_free_kernel (struct mm_struct *mm,pte_t *pte) {free_page ((unsigned long ) pte); } -static inline void pte_free (struct page *pte) +static inline void pte_free (struct mm_struct *mm,struct page *pte) {__free_page (pte); }
@@ -42,7 +42,7 @@ static inline void pte_free(struct page#ifdef CONFIG_3_LEVEL_PGTABLES -static inline void pmd_free (pmd_t *pmd) +static inline void pmd_free (struct mm_struct *mm,pmd_t *pmd) {free_page ((unsigned long )pmd); }
include/asm-x86/pgalloc_32.h +4 -4
@@ -33,17 +33,17 @@ do { \* Allocate and free page tables. */ extern pgd_t *pgd_alloc (struct mm_struct *); -extern void pgd_free (pgd_t *pgd); +extern void pgd_free (struct mm_struct *mm,pgd_t *pgd);extern pte_t *pte_alloc_one_kernel (struct mm_struct *,unsigned long );extern struct page *pte_alloc_one (struct mm_struct *,unsigned long ); -static inline void pte_free_kernel (pte_t *pte) +static inline void pte_free_kernel (struct mm_struct *mm,pte_t *pte) {free_page ((unsigned long )pte); } -static inline void pte_free (struct page *pte) +static inline void pte_free (struct mm_struct *mm,struct page *pte) {__free_page (pte); }
@@ -60,7 +60,7 @@ do { \* In the PAE case we free the pmds as part of the pgd. */ #define pmd_alloc_one (mm, addr) ({ BUG(); ((pmd_t *)2); }) -#define pmd_free (x) do { } while (0) +#define pmd_free (mm, x) do { } while (0)#define __pmd_free_tlb (tlb,x) do { } while (0)#define pud_populate (mm, pmd, pte) BUG()#endif
include/asm-x86/pgalloc_64.h +5 -5
@@ -17,7 +17,7 @@ static inline void pmd_populate(struct mset_pmd (pmd,__pmd (_PAGE_TABLE | (page_to_pfn (pte) <<PAGE_SHIFT ))); } -static inline void pmd_free (pmd_t *pmd) +static inline void pmd_free (struct mm_struct *mm,pmd_t *pmd) {BUG_ON ((unsigned long )pmd & (PAGE_SIZE -1 ));free_page ((unsigned long )pmd);
@@ -33,7 +33,7 @@ static inline pud_t *pud_alloc_one(strucreturn (pud_t *)get_zeroed_page (GFP_KERNEL |__GFP_REPEAT); } -static inline void pud_free (pud_t *pud) +static inline void pud_free (struct mm_struct *mm,pud_t *pud) {BUG_ON ((unsigned long )pud & (PAGE_SIZE -1 ));free_page ((unsigned long )pud);
@@ -77,7 +77,7 @@ static inline pgd_t *pgd_alloc(struct mmreturn pgd; } -static inline void pgd_free (pgd_t *pgd) +static inline void pgd_free (struct mm_struct *mm,pgd_t *pgd) {BUG_ON ((unsigned long )pgd & (PAGE_SIZE -1 ));pgd_list_del (pgd);
@@ -100,13 +100,13 @@ static inline struct page *pte_alloc_one/* Should really implement gc for free page table pages. This could be done with a reference count in struct page. */ -static inline void pte_free_kernel (pte_t *pte) +static inline void pte_free_kernel (struct mm_struct *mm,pte_t *pte) {BUG_ON ((unsigned long )pte & (PAGE_SIZE -1 ));free_page ((unsigned long )pte); } -static inline void pte_free (struct page *pte) +static inline void pte_free (struct mm_struct *mm,struct page *pte) {__free_page (pte); }
include/asm-xtensa/pgalloc.h +3 -3
@@ -31,7 +31,7 @@ pgd_alloc(struct mm_struct *mm)return (pgd_t *)__get_free_pages (GFP_KERNEL | __GFP_ZERO,PGD_ORDER ); } -static inline void pgd_free (pgd_t *pgd) +static inline void pgd_free (struct mm_struct *mm,pgd_t *pgd) {free_page ((unsigned long )pgd); }
@@ -52,12 +52,12 @@ static inline struct page *pte_alloc_onereturn virt_to_page (pte_alloc_one_kernel (mm, addr)); } -static inline void pte_free_kernel (pte_t *pte) +static inline void pte_free_kernel (struct mm_struct *mm,pte_t *pte) {kmem_cache_free (pgtable_cache, pte); } -static inline void pte_free (struct page *page) +static inline void pte_free (struct mm_struct *mm,struct page *page) {kmem_cache_free (pgtable_cache,page_address (page)); }
include/asm-xtensa/tlb.h +1 -1
@@ -42,6 +42,6 @@#include <asm-generic/tlb.h> -#define __pte_free_tlb (tlb,pte) pte_free(pte) +#define __pte_free_tlb (tlb,pte) pte_free((tlb)->mm, pte)#endif /* _XTENSA_TLB_H */
kernel/fork.c +1 -1
@@ -324,7 +324,7 @@ static inline int mm_alloc_pgd(struct mmstatic inline void mm_free_pgd (struct mm_struct * mm) { -pgd_free (mm->pgd); +pgd_free (mm, mm->pgd); }#else #define dup_mmap (mm, oldmm) (0)
mm/memory.c +5 -5
@@ -305,7 +305,7 @@ int __pte_alloc(struct mm_struct *mm, pmspin_lock (&mm->page_table_lock);if (pmd_present (*pmd)) {/* Another has populated it */ pte_lock_deinit (new); -pte_free (new); +pte_free (mm, new); }else { mm->nr_ptes++;inc_zone_page_state (new,NR_PAGETABLE );
@@ -323,7 +323,7 @@ int __pte_alloc_kernel(pmd_t *pmd, unsigspin_lock (&init_mm.page_table_lock);if (pmd_present (*pmd))/* Another has populated it */ -pte_free_kernel (new); +pte_free_kernel (&init_mm, new);else pmd_populate_kernel (&init_mm, pmd, new);spin_unlock (&init_mm.page_table_lock);
@@ -2557,7 +2557,7 @@ int __pud_alloc(struct mm_struct *mm, pgspin_lock (&mm->page_table_lock);if (pgd_present (*pgd))/* Another has populated it */ -pud_free (new); +pud_free (mm, new);else pgd_populate (mm, pgd, new);spin_unlock (&mm->page_table_lock);
@@ -2579,12 +2579,12 @@ int __pmd_alloc(struct mm_struct *mm, puspin_lock (&mm->page_table_lock);#ifndef __ARCH_HAS_4LEVEL_HACKif (pud_present (*pud))/* Another has populated it */ -pmd_free (new); +pmd_free (mm, new);else pud_populate (mm, pud, new);#else if (pgd_present (*pud))/* Another has populated it */ -pmd_free (new); +pmd_free (mm, new);else pgd_populate (mm, pud, new);#endif /* __ARCH_HAS_4LEVEL_HACK */
Signature · 4 lines
blue skies, Martin. "Reality continues to ruin my life." - Calvin.
PATCH 2/6 CONFIG_HIGHPTE vs. sub-page page tables.
In reply to Martin SchwidefskyFrom: Martin Schwidefsky <[email protected]>
Background: I've implemented 1K/2K page tables for s390. These sub-page page tables are required to properly support the s390 virtualization instruction with KVM. The SIE instruction requires that the page tables have 256 page table entries (pte) followed by 256 page status table entries (pgste). The pgstes are only required if the process is using the SIE instruction. The pgstes are updated by the hardware and by the hypervisor for a number of reasons, one of them is dirty and reference bit tracking. To avoid wasting memory the standard pte table allocation should return 1K/2K (31/64 bit) and 2K/4K if the process is using SIE.
Problem: Page size on s390 is 4K, page table size is 1K or 2K. That means the s390 version for pte_alloc_one cannot return a pointer to a struct page. Trouble is that with the CONFIG_HIGHPTE feature on x86 pte_alloc_one cannot return a pointer to a pte either, since that would require more than 32 bit for the return value of pte_alloc_one (and the pte * would not be accessible since its not kmapped).
Solution: The only solution I found to this dilemma is a new typedef:
a pgtable_t. For s390 pgtable_t will be a (pte *) - to be introduced with a later patch. For everybody else it will be a (struct page *). The additional problem with the initialization of the ptl lock and the NR_PAGETABLE accounting is solved with a constructor pgtable_page_ctor and a destructor pgtable_page_dtor. The page table allocation and free functions need to call these two whenever a page table page is allocated or freed. pmd_populate will get a pgtable_t instead of a struct page pointer. To get the pgtable_t back from a pmd entry that has been installed with pmd_populate a new function pmd_pgtable is added. It replaces the pmd_page call in free_pte_range and apply_to_pte_range.
Signed-off-by Martin Schwidefsky <[email protected]>
---
arch/frv/mm/pgalloc.c | 8 +++++--- arch/powerpc/mm/pgtable_32.c | 16 +++++++++------- arch/ppc/mm/pgtable.c | 9 ++++++--- arch/s390/mm/pgtable.c | 2 ++ arch/sparc/mm/srmmu.c | 10 +++++++--- arch/sparc/mm/sun4c.c | 14 ++++++++++---- arch/um/kernel/mem.c | 4 +++- arch/x86/mm/pgtable_32.c | 4 +++- include/asm-alpha/page.h | 2 ++ include/asm-alpha/pgalloc.h | 22 ++++++++++++++-------- include/asm-arm/page.h | 2 ++ include/asm-arm/pgalloc.h | 9 ++++++--- include/asm-avr32/page.h | 1 + include/asm-avr32/pgalloc.h | 16 ++++++++++++---- include/asm-cris/page.h | 1 + include/asm-cris/pgalloc.h | 14 ++++++++++---- include/asm-frv/page.h | 1 + include/asm-frv/pgalloc.h | 12 +++++++++--- include/asm-ia64/page.h | 2 ++ include/asm-ia64/pgalloc.h | 20 ++++++++++++++------ include/asm-m32r/page.h | 1 + include/asm-m32r/pgalloc.h | 10 ++++++---- include/asm-m68k/motorola_pgalloc.h | 14 ++++++++------ include/asm-m68k/page.h | 1 + include/asm-m68k/sun3_pgalloc.h | 17 ++++++++++++----- include/asm-mips/page.h | 1 + include/asm-mips/pgalloc.h | 5 +++-- include/asm-parisc/page.h | 1 + include/asm-parisc/pgalloc.h | 11 +++++++++-- include/asm-powerpc/page.h | 2 ++ include/asm-powerpc/pgalloc-32.h | 6 ++++-- include/asm-powerpc/pgalloc-64.h | 26 +++++++++++++++++++------- include/asm-ppc/pgalloc.h | 6 ++++-- include/asm-s390/page.h | 2 ++ include/asm-s390/pgalloc.h | 3 ++- include/asm-s390/tlb.h | 2 +- include/asm-sh/page.h | 2 ++ include/asm-sh/pgalloc.h | 27 ++++++++++++++++++++------- include/asm-sh64/page.h | 2 ++ include/asm-sh64/pgalloc.h | 27 ++++++++++++++++++++------- include/asm-sparc/page.h | 2 ++ include/asm-sparc/pgalloc.h | 5 +++-- include/asm-sparc64/page.h | 2 ++ include/asm-sparc64/pgalloc.h | 19 ++++++++++++++----- include/asm-um/page.h | 2 ++ include/asm-um/pgalloc.h | 12 +++++++++--- include/asm-x86/page_32.h | 2 ++ include/asm-x86/page_64.h | 2 ++ include/asm-x86/pgalloc_32.h | 7 +++++-- include/asm-x86/pgalloc_64.h | 22 +++++++++++++++++----- include/asm-xtensa/page.h | 1 + include/asm-xtensa/pgalloc.h | 17 ++++++++++++----- include/linux/mm.h | 14 +++++++++++++- mm/memory.c | 32 +++++++++++++++----------------- mm/vmalloc.c | 2 +- 55 files changed, 339 insertions(+), 137 deletions(-)
arch/frv/mm/pgalloc.c +5 -3
@@ -28,7 +28,7 @@ pte_t *pte_alloc_one_kernel(struct mm_streturn pte; } -struct page *pte_alloc_one (struct mm_struct *mm,unsigned long address) +pgtable_t pte_alloc_one (struct mm_struct *mm,unsigned long address) {struct page *page;
@@ -37,9 +37,11 @@ struct page *pte_alloc_one(struct mm_str#else page =alloc_pages (GFP_KERNEL |__GFP_REPEAT,0 );#endif -if (page) +if (page) {clear_highpage (page); -flush_dcache_page (page); +pgtable_page_ctor (page); +flush_dcache_page (page); + }return page; }
arch/powerpc/mm/pgtable_32.c +9 -7
@@ -107,20 +107,21 @@ __init_refok pte_t *pte_alloc_one_kernelreturn pte; } -struct page *pte_alloc_one (struct mm_struct *mm,unsigned long address) +pgtable_t pte_alloc_one (struct mm_struct *mm,unsigned long address) {struct page *ptepage;#ifdef CONFIG_HIGHPTE -gfp_t flags =GFP_KERNEL | __GFP_HIGHMEM | __GFP_REPEAT; +gfp_t flags =GFP_KERNEL | __GFP_HIGHMEM | __GFP_REPEAT | __GFP_ZERO;#else -gfp_t flags =GFP_KERNEL | __GFP_REPEAT; +gfp_t flags =GFP_KERNEL | __GFP_REPEAT | __GFP_ZERO;#endif ptepage =alloc_pages (flags,0 ); -if (ptepage) -clear_highpage (ptepage); -return ptepage; +if (!ptepage) +return NULL ; +pgtable_page_ctor (ptepage); +return page_address (ptepage); }void pte_free_kernel (struct mm_struct *mm,pte_t *pte)
@@ -131,11 +132,12 @@ void pte_free_kernel(struct mm_struct *mfree_page ((unsigned long )pte); } -void pte_free (struct mm_struct *mm,struct page *ptepage) +void pte_free (struct mm_struct *mm,pgtable_t ptepage) {#ifdef CONFIG_SMP hash_page_sync ();#endif +pgtable_page_dtor (ptepage);__free_page (ptepage); }
arch/ppc/mm/pgtable.c +6 -3
@@ -108,7 +108,7 @@ __init_refok pte_t *pte_alloc_one_kernelreturn pte; } -struct page *pte_alloc_one (struct mm_struct *mm,unsigned long address) +pgtable_t pte_alloc_one (struct mm_struct *mm,unsigned long address) {struct page *ptepage;
@@ -119,8 +119,10 @@ struct page *pte_alloc_one(struct mm_str#endif ptepage =alloc_pages (flags,0 ); -if (ptepage) +if (ptepage) {clear_highpage (ptepage); +pgtable_page_ctor (ptepage); + }return ptepage; }
@@ -132,11 +134,12 @@ void pte_free_kernel(struct mm_struct *mfree_page ((unsigned long )pte); } -void pte_free (struct mm_struct *mm,struct page *ptepage) +void pte_free (struct mm_struct *mm,pgtable_t ptepage) {#ifdef CONFIG_SMP hash_page_sync ();#endif +pgtable_page_dtor (ptepage);__free_page (ptepage); }
arch/s390/mm/pgtable.c +2 -0
@@ -78,6 +78,7 @@ unsigned long *page_table_alloc(int noexclear_table (table, _PAGE_TYPE_EMPTY,PAGE_SIZE ); page->index = (addr_t ) table; } +pgtable_page_ctor (page); table = (unsigned long *)page_to_phys (page);clear_table (table, _PAGE_TYPE_EMPTY,PAGE_SIZE );return table;
@@ -87,6 +88,7 @@ void page_table_free(unsigned long *tabl {unsigned long *shadow =get_shadow_pte (table); +pgtable_page_dtor (virt_to_page (table));if (shadow)free_page ((unsigned long ) shadow);free_page ((unsigned long ) table);
arch/sparc/mm/srmmu.c +7 -3
@@ -490,14 +490,17 @@ srmmu_pte_alloc_one_kernel(struct mm_strreturn (pte_t *)srmmu_get_nocache (PTE_SIZE ,PTE_SIZE ); } -static struct page * +static pgtable_t srmmu_pte_alloc_one (struct mm_struct *mm,unsigned long address) {unsigned long pte; +struct page *page;if ((pte = (unsigned long )srmmu_pte_alloc_one_kernel (mm, address)) ==0 )return NULL ; -return pfn_to_page (__nocache_pa (pte) >>PAGE_SHIFT ); + page =pfn_to_page (__nocache_pa (pte) >>PAGE_SHIFT ); +pgtable_page_ctor (page); +return page; }static void srmmu_free_pte_fast (pte_t *pte)
@@ -505,10 +508,11 @@ static void srmmu_free_pte_fast(pte_t *psrmmu_free_nocache ((unsigned long )pte,PTE_SIZE ); } -static void srmmu_pte_free (struct page *pte) +static void srmmu_pte_free (pgtable_t pte) {unsigned long p; +pgtable_page_dtor (pte); p = (unsigned long )page_address (pte);/* Cached address (for test) */ if (p ==0 )BUG ();
arch/sparc/mm/sun4c.c +10 -4
@@ -1948,12 +1948,17 @@ static pte_t *sun4c_pte_alloc_one_kernelreturn pte; } -static struct page *sun4c_pte_alloc_one (struct mm_struct *mm,unsigned long address) +static pgtable_t sun4c_pte_alloc_one (struct mm_struct *mm,unsigned long address) { -pte_t *pte =sun4c_pte_alloc_one_kernel (mm, address); +pte_t *pte; +struct page *page; + + pte =sun4c_pte_alloc_one_kernel (mm, address);if (pte ==NULL )return NULL ; -return virt_to_page (pte); + page =virt_to_page (pte); +pgtable_page_ctor (page); +return page; }static __inline__void sun4c_free_pte_fast (pte_t *pte)
@@ -1963,8 +1968,9 @@ static __inline__ void sun4c_free_pte_fa pgtable_cache_size++; } -static void sun4c_pte_free (struct page *pte) +static void sun4c_pte_free (pgtable_t pte) { +pgtable_page_dtor (pte);sun4c_free_pte_fast (page_address (pte)); }
arch/um/kernel/mem.c +3 -1
@@ -361,10 +361,12 @@ pte_t *pte_alloc_one_kernel(struct mm_streturn pte; } -struct page *pte_alloc_one (struct mm_struct *mm,unsigned long address) +pgtable_t pte_alloc_one (struct mm_struct *mm,unsigned long address) {struct page *pte; pte =alloc_page (GFP_KERNEL |__GFP_REPEAT|__GFP_ZERO); +if (pte) +pgtable_page_ctor (pte);return pte; }
arch/x86/mm/pgtable_32.c +3 -1
@@ -183,7 +183,7 @@ pte_t *pte_alloc_one_kernel(struct mm_streturn (pte_t *)__get_free_page (GFP_KERNEL |__GFP_REPEAT|__GFP_ZERO); } -struct page *pte_alloc_one (struct mm_struct *mm,unsigned long address) +pgtable_t pte_alloc_one (struct mm_struct *mm,unsigned long address) {struct page *pte;
@@ -192,6 +192,8 @@ struct page *pte_alloc_one(struct mm_str#else pte =alloc_pages (GFP_KERNEL |__GFP_REPEAT|__GFP_ZERO,0 );#endif +if (pte) +pgtable_page_ctor (pte);return pte; }
include/asm-alpha/page.h +2 -0
@@ -64,6 +64,8 @@ typedef unsigned long pgprot_t;#endif /* STRICT_MM_TYPECHECKS */ +typedef struct page *pgtable_t ; +#ifdef USE_48_BIT_KSEG #define PAGE_OFFSET 0xffff800000000000UL#else
include/asm-alpha/pgalloc.h +14 -8
@@ -11,10 +11,11 @@*/ static inline void -pmd_populate (struct mm_struct *mm,pmd_t *pmd,struct page *pte) +pmd_populate (struct mm_struct *mm,pmd_t *pmd,pgtable_t pte) {pmd_set (pmd, (pte_t *)(page_to_pa (pte) +PAGE_OFFSET )); } +#define pmd_pgtable (pmd) pmd_page(pmd)static inline void pmd_populate_kernel (struct mm_struct *mm,pmd_t *pmd,pte_t *pte)
@@ -57,18 +58,23 @@ pte_free_kernel(struct mm_struct *mm, ptfree_page ((unsigned long )pte); } -static inline struct page * -pte_alloc_one (struct mm_struct *mm,unsigned long addr) +static inline pgtable_t +pte_alloc_one (struct mm_struct *mm,unsigned long address) { -pte_t *pte =pte_alloc_one_kernel (mm, addr); -if (pte) -return virt_to_page (pte); -return NULL ; +pte_t *pte =pte_alloc_one_kernel (mm, address); +struct page *page; + +if (!pte) +return NULL ; + page =virt_to_page (pte); +pgtable_page_ctor (page); +return page; }static inline void -pte_free (struct mm_struct *mm,struct page *page) +pte_free (struct mm_struct *mm,pgtable_t page) { +pgtable_page_dtor (page);__free_page (page); }
include/asm-arm/page.h +2 -0
@@ -174,6 +174,8 @@ typedef unsigned long pgprot_t;#endif /* STRICT_MM_TYPECHECKS */ +typedef struct page *pgtable_t ; +#endif /* CONFIG_MMU */ #include <asm/memory.h>
include/asm-arm/pgalloc.h +6 -3
@@ -66,7 +66,7 @@ pte_alloc_one_kernel(struct mm_struct *mreturn pte; } -static inline struct page * +static inline pgtable_t pte_alloc_one (struct mm_struct *mm,unsigned long addr) {struct page *pte;
@@ -75,6 +75,7 @@ pte_alloc_one(struct mm_struct *mm, unsiif (pte) {void *page =page_address (pte);clean_dcache_area (page,sizeof (pte_t) *PTRS_PER_PTE ); +pgtable_page_ctor (pte); }return pte;
@@ -91,8 +92,9 @@ static inline void pte_free_kernel(struc } } -static inline void pte_free (struct mm_struct *mm,struct page *pte) +static inline void pte_free (struct mm_struct *mm,pgtable_t pte) { +pgtable_page_dtor (pte);__free_page (pte); }
@@ -123,10 +125,11 @@ pmd_populate_kernel(struct mm_struct *mm }static inline void -pmd_populate (struct mm_struct *mm,pmd_t *pmdp,struct page *ptep) +pmd_populate (struct mm_struct *mm,pmd_t *pmdp,pgtable_t ptep) {__pmd_populate (pmdp,page_to_pfn (ptep) <<PAGE_SHIFT | _PAGE_USER_TABLE); } +#define pmd_pgtable (pmd) pmd_page(pmd)#endif /* CONFIG_MMU */
include/asm-avr32/page.h +1 -0
@@ -36,6 +36,7 @@ extern void copy_page(void *to, void *frtypedef struct {unsigned long pte; }pte_t ;typedef struct {unsigned long pgd; }pgd_t ;typedef struct {unsigned long pgprot; }pgprot_t ; +typedef struct page *pgtable_t ;#define pte_val (x) ((x).pte)#define pgd_val (x) ((x).pgd)
include/asm-avr32/pgalloc.h +12 -4
@@ -17,10 +17,11 @@ set_pmd(pmd, __pmd(_PAGE_TABLE + __pa(pte)))static __inline__void pmd_populate (struct mm_struct *mm,pmd_t *pmd, -struct page *pte) +pgtable_t pte) {set_pmd (pmd,__pmd (_PAGE_TABLE +page_to_phys (pte))); } +#define pmd_pgtable (pmd) pmd_page(pmd)/* * Allocate and free page tables
@@ -51,7 +52,9 @@ static inline struct page *pte_alloc_onestruct page *pte; pte =alloc_page (GFP_KERNEL | __GFP_REPEAT | __GFP_ZERO); - +if (!page) +return NULL ; +pgtable_page_ctor (page);return pte; }
@@ -60,12 +63,17 @@ static inline void pte_free_kernel(strucfree_page ((unsigned long )pte); } -static inline void pte_free (struct mm_struct *mm,struct page *pte) +static inline void pte_free (struct mm_struct *mm,pgtable_t pte) { +pgtable_page_dtor (pte);__free_page (pte); } -#define __pte_free_tlb (tlb,pte) tlb_remove_page((tlb),(pte)) +#define __pte_free_tlb (tlb,pte) \ +do { \ + pgtable_page_dtor(pte); \ + tlb_remove_page((tlb), pte); \ +} while (0)#define check_pgt_cache () do { } while(0)
include/asm-cris/page.h +1 -0
@@ -31,6 +31,7 @@typedef struct {unsigned long pte; }pte_t ;typedef struct {unsigned long pgd; }pgd_t ;typedef struct {unsigned long pgprot; }pgprot_t ; +typedef struct page *pgtable_t ;#endif #define pte_val (x) ((x).pte)
include/asm-cris/pgalloc.h +10 -4
@@ -6,6 +6,7 @@#define pmd_populate_kernel (mm, pmd, pte) pmd_set(pmd, pte)#define pmd_populate (mm, pmd, pte) pmd_set(pmd, page_address(pte)) +#define pmd_pgtable (pmd) pmd_page(pmd)/* * Allocate and free page tables.
@@ -27,10 +28,11 @@ static inline pte_t *pte_alloc_one_kernereturn pte; } -static inline struct page *pte_alloc_one (struct mm_struct *mm,unsigned long address) +static inline pgtable_t pte_alloc_one (struct mm_struct *mm,unsigned long address) {struct page *pte; pte =alloc_pages (GFP_KERNEL |__GFP_REPEAT|__GFP_ZERO,0 ); +pgtable_page_ctor (pte);return pte; }
@@ -39,13 +41,17 @@ static inline void pte_free_kernel(strucfree_page ((unsigned long )pte); } -static inline void pte_free (struct mm_struct *mm,struct page *pte) +static inline void pte_free (struct mm_struct *mm,pgtable_t pte) { +pgtable_page_dtor (pte);__free_page (pte); } - -#define __pte_free_tlb (tlb,pte) tlb_remove_page((tlb),(pte)) +#define __pte_free_tlb (tlb,pte) \ +do { \ + pgtable_page_dtor(pte); \ + tlb_remove_page((tlb), pte); \ +} while (0)#define check_pgt_cache () do { } while (0)
include/asm-frv/page.h +1 -0
@@ -27,6 +27,7 @@ typedef struct { unsigned long ste[64];}typedef struct {pmd_t pue[1 ]; }pud_t ;typedef struct {pud_t pge[1 ]; }pgd_t ;typedef struct {unsigned long pgprot; }pgprot_t ; +typedef struct page *pgtable_t ;#define pte_val (x) ((x).pte)#define pmd_val (x) ((x).ste[0])
include/asm-frv/pgalloc.h +9 -3
@@ -25,6 +25,7 @@do { \__set_pmd ((PMD ),page_to_pfn (PAGE ) <<PAGE_SHIFT | _PAGE_TABLE); \ } while(0) +#define pmd_pgtable (pmd) pmd_page(pmd)/* * Allocate and free page tables.
@@ -35,19 +36,24 @@ extern void pgd_free(struct mm_struct *mextern pte_t *pte_alloc_one_kernel (struct mm_struct *,unsigned long ); -extern struct page *pte_alloc_one (struct mm_struct *,unsigned long ); +extern pgtable_t pte_alloc_one (struct mm_struct *,unsigned long );static inline void pte_free_kernel (struct mm_struct *mm,pte_t *pte) {free_page ((unsigned long )pte); } -static inline void pte_free (struct mm_struct *mm,struct page *pte) +static inline void pte_free (struct mm_struct *mm,pgtable_t pte) { +pgtable_page_dtor (pte);__free_page (pte); } -#define __pte_free_tlb (tlb,pte) tlb_remove_page((tlb),(pte)) +#define __pte_free_tlb (tlb,pte) \ +do { \ + pgtable_page_dtor(pte); \ + tlb_remove_page((tlb),(pte)); \ +} while (0)/* * allocating and freeing a pmd is trivial: the 1-entry pmd is
include/asm-ia64/page.h +2 -0
@@ -187,6 +187,7 @@ get_order (unsigned long size)#endif typedef struct {unsigned long pgd; }pgd_t ;typedef struct {unsigned long pgprot; }pgprot_t ; +typedef struct page *pgtable_t ;# define pte_val (x) ((x).pte)# define pmd_val (x) ((x).pmd)
@@ -208,6 +209,7 @@ get_order (unsigned long size)typedef unsigned long pmd_t ;typedef unsigned long pgd_t ;typedef unsigned long pgprot_t ; +typedef struct page *pgtable_t ;# endif # define pte_val (x) (x)
include/asm-ia64/pgalloc.h +14 -6
@@ -70,10 +70,11 @@ static inline void pmd_free(struct mm_st#define __pmd_free_tlb (tlb, pmd) pmd_free((tlb)->mm, pmd)static inline void -pmd_populate (struct mm_struct *mm,pmd_t * pmd_entry,struct page *pte) +pmd_populate (struct mm_struct *mm,pmd_t * pmd_entry,pgtable_t pte) {pmd_val (*pmd_entry) =page_to_phys (pte); } +#define pmd_pgtable (pmd) pmd_page(pmd)static inline void pmd_populate_kernel (struct mm_struct *mm,pmd_t * pmd_entry,pte_t * pte)
@@ -81,11 +82,17 @@ pmd_populate_kernel(struct mm_struct *mmpmd_val (*pmd_entry) =__pa (pte); } -static inline struct page *pte_alloc_one (struct mm_struct *mm, -unsigned long addr) +static inline pgtable_t pte_alloc_one (struct mm_struct *mm,unsigned long addr) { -void *pg =quicklist_alloc (0 ,GFP_KERNEL ,NULL ); -return pg ?virt_to_page (pg) :NULL ; +struct page *page; +void *pg; + + pg =quicklist_alloc (0 ,GFP_KERNEL ,NULL ); +if (!pg) +return NULL ; + page =virt_to_page (pg); +pgtable_page_ctor (page); +return page; } static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
@@ -94,8 +101,9 @@ static inline pte_t *pte_alloc_one_kernereturn quicklist_alloc (0 ,GFP_KERNEL ,NULL ); } -static inline void pte_free (struct mm_struct *mm,struct page *pte) +static inline void pte_free (struct mm_struct *mm,pgtable_t pte) { +pgtable_page_dtor (pte);quicklist_free_page (0 ,NULL , pte); }
include/asm-m32r/page.h +1 -0
@@ -29,6 +29,7 @@ typedef struct { unsigned long pgd; } pg#define PTE_MASK PAGE_MASKtypedef struct {unsigned long pgprot; }pgprot_t ; +typedef struct page *pgtable_t ;#define pmd_val (x) ((x).pmd)#define pgd_val (x) ((x).pgd)
include/asm-m32r/pgalloc.h +6 -4
@@ -9,10 +9,11 @@ set_pmd(pmd, __pmd(_PAGE_TABLE + __pa(pte)))static __inline__void pmd_populate (struct mm_struct *mm,pmd_t *pmd, -struct page *pte) +pgtable_t pte) {set_pmd (pmd,__pmd (_PAGE_TABLE +page_to_phys (pte))); } +#define pmd_pgtable (pmd) pmd_page(pmd)/* * Allocate and free page tables.
@@ -37,12 +38,12 @@ static __inline__ pte_t *pte_alloc_one_kreturn pte; } -static __inline__struct page *pte_alloc_one (struct mm_struct *mm, +static __inline__pgtable_t pte_alloc_one (struct mm_struct *mm,unsigned long address) {struct page *pte =alloc_page (GFP_KERNEL |__GFP_ZERO); - +pgtable_page_ctor (pte);return pte; }
@@ -51,8 +52,9 @@ static __inline__ void pte_free_kernel(sfree_page ((unsigned long )pte); } -static __inline__void pte_free (struct mm_struct *mm,struct page *pte) +static __inline__void pte_free (struct mm_struct *mm,pgtable_t pte) { +pgtable_page_dtor (pte);__free_page (pte); }
include/asm-m68k/motorola_pgalloc.h +8 -6
@@ -7,7 +7,6 @@extern pmd_t *get_pointer_table (void );extern int free_pointer_table (pmd_t *); -static inline pte_t *pte_alloc_one_kernel (struct mm_struct *mm,unsigned long address) {pte_t *pte;
@@ -28,7 +27,7 @@ static inline void pte_free_kernel(strucfree_page ((unsigned long ) pte); } -static inline struct page *pte_alloc_one (struct mm_struct *mm,unsigned long address) +static inline pgtable_t pte_alloc_one (struct mm_struct *mm,unsigned long address) {struct page *page =alloc_pages (GFP_KERNEL |__GFP_REPEAT|__GFP_ZERO,0 );pte_t *pte;
@@ -43,19 +42,21 @@ static inline struct page *pte_alloc_onenocache_page (pte); }kunmap (pte); - +pgtable_page_ctor (page);return page; } -static inline void pte_free (struct mm_struct *mm,struct page *page) +static inline void pte_free (struct mm_struct *mm,pgtable_t page) { +pgtable_page_dtor (page);cache_page (kmap (page));kunmap (page);__free_page (page); } -static inline void __pte_free_tlb (struct mmu_gather *tlb,struct page *page) +static inline void __pte_free_tlb (struct mmu_gather *tlb,pgtable_t page) { +pgtable_page_dtor (page);cache_page (kmap (page));kunmap (page);__free_page (page);
@@ -94,10 +95,11 @@ static inline void pmd_populate_kernel(spmd_set (pmd, pte); } -static inline void pmd_populate (struct mm_struct *mm,pmd_t *pmd,struct page *page) +static inline void pmd_populate (struct mm_struct *mm,pmd_t *pmd,pgtable_t page) {pmd_set (pmd,page_address (page)); } +#define pmd_pgtable (pmd) pmd_page(pmd)static inline void pgd_populate (struct mm_struct *mm,pgd_t *pgd,pmd_t *pmd) {
include/asm-m68k/page.h +1 -0
@@ -94,6 +94,7 @@ typedef struct { unsigned long pte; } pttypedef struct {unsigned long pmd[16 ]; }pmd_t ;typedef struct {unsigned long pgd; }pgd_t ;typedef struct {unsigned long pgprot; }pgprot_t ; +typedef struct page *pgtable_t ;#define pte_val (x) ((x).pte)#define pmd_val (x) ((&x)->pmd[0])
include/asm-m68k/sun3_pgalloc.h +12 -5
@@ -26,12 +26,17 @@ static inline void pte_free_kernel(strucfree_page ((unsigned long ) pte); } -static inline void pte_free (struct mm_struct *mm,struct page *page) +static inline void pte_free (struct mm_struct *mm,pgtable_t page) { +pgtable_page_dtor (page);__free_page (page); } -#define __pte_free_tlb (tlb,pte) tlb_remove_page((tlb),(pte)) +#define __pte_free_tlb (tlb,pte) \ +do { \ + pgtable_page_dtor(pte); \ + tlb_remove_page((tlb), pte); \ +} while (0)static inline pte_t *pte_alloc_one_kernel (struct mm_struct *mm,unsigned long address)
@@ -45,8 +50,8 @@ static inline pte_t *pte_alloc_one_kernereturn (pte_t *) (page); } -static inline struct page *pte_alloc_one (struct mm_struct *mm, -unsigned long address) +static inline pgtable_t pte_alloc_one (struct mm_struct *mm, +unsigned long address) {struct page *page =alloc_pages (GFP_KERNEL |__GFP_REPEAT,0 );
@@ -54,6 +59,7 @@ static inline struct page *pte_alloc_onereturn NULL ;clear_highpage (page); +pgtable_page_ctor (page);return page; }
@@ -63,10 +69,11 @@ static inline void pmd_populate_kernel(spmd_val (*pmd) =__pa ((unsigned long )pte); } -static inline void pmd_populate (struct mm_struct *mm,pmd_t *pmd,struct page *page) +static inline void pmd_populate (struct mm_struct *mm,pmd_t *pmd,pgtable_t page) {pmd_val (*pmd) =__pa ((unsigned long )page_address (page)); } +#define pmd_pgtable (pmd) pmd_page(pmd)/* * allocating and freeing a pmd is trivial: the 1-entry pmd is
include/asm-mips/page.h +1 -0
@@ -93,6 +93,7 @@ typedef struct { unsigned long pte; } pt#define pte_val (x) ((x).pte)#define __pte (x) ((pte_t) { (x) } )#endif +typedef struct page *pgtable_t ;/* * For 3-level pagetables we defines these ourselves, for 2-level the
include/asm-mips/pgalloc.h +3 -2
@@ -20,10 +20,11 @@ static inline void pmd_populate_kernel(s }static inline void pmd_populate (struct mm_struct *mm,pmd_t *pmd, -struct page *pte) +pgtable_t pte) {set_pmd (pmd,__pmd ((unsigned long )page_address (pte))); } +#define pmd_pgtable (pmd) pmd_page(pmd)/* * Initialize a new pmd table with invalid pointers.
@@ -90,7 +91,7 @@ static inline void pte_free_kernel(strucfree_pages ((unsigned long )pte,PTE_ORDER ); } -static inline void pte_free (struct mm_struct *mm,struct page *pte) +static inline void pte_free (struct mm_struct *mm,pgtable_t pte) {__free_pages (pte,PTE_ORDER ); }
include/asm-parisc/page.h +1 -0
@@ -93,6 +93,7 @@ typedef unsigned long pgprot_t;#endif /* STRICT_MM_TYPECHECKS */ +typedef struct page *pgtable_t ; typedef struct __physmem_range { unsigned long start_pfn;
include/asm-parisc/pgalloc.h +9 -2
@@ -115,11 +115,14 @@ pmd_populate_kernel(struct mm_struct *mm#define pmd_populate (mm, pmd, pte_page) \ pmd_populate_kernel(mm, pmd, page_address(pte_page)) +#define pmd_pgtable (pmd) pmd_page(pmd) -static inline struct page * +static inline pgtable_t pte_alloc_one (struct mm_struct *mm,unsigned long address) {struct page *page =alloc_page (GFP_KERNEL |__GFP_REPEAT|__GFP_ZERO); +if (page) +pgtable_page_ctor (page);return page; }
@@ -135,7 +138,11 @@ static inline void pte_free_kernel(strucfree_page ((unsigned long )pte); } -#define pte_free (mm, page) pte_free_kernel(page_address(page)) +static inline void pte_free_kernel (struct mm_struct *mm,struct page *pte) +{ +pgtable_page_dtor (pte); + pte_free_kernel(page_address((pte)); +}#define check_pgt_cache () do { } while (0)
include/asm-powerpc/page.h +2 -0
@@ -191,6 +191,8 @@ extern int page_is_ram(unsigned long pfnstruct vm_area_struct ; +typedef struct page *pgtable_t ; +#include <asm-generic/memory_model.h> #endif /* __ASSEMBLY__ */
include/asm-powerpc/pgalloc-32.h +4 -2
@@ -22,17 +22,19 @@ extern void pgd_free(struct mm_struct *m (pmd_val(*(pmd)) = __pa(pte) | _PMD_PRESENT)#define pmd_populate (mm, pmd, pte) \ (pmd_val(*(pmd)) = (page_to_pfn(pte) << PAGE_SHIFT) | _PMD_PRESENT) +#define pmd_pgtable (pmd) pmd_page(pmd)#else #define pmd_populate_kernel (mm, pmd, pte) \ (pmd_val(*(pmd)) = (unsigned long)pte | _PMD_PRESENT)#define pmd_populate (mm, pmd, pte) \ (pmd_val(*(pmd)) = (unsigned long)lowmem_page_address(pte) | _PMD_PRESENT) +#define pmd_pgtable (pmd) pmd_page(pmd)#endif extern pte_t *pte_alloc_one_kernel (struct mm_struct *mm,unsigned long addr); -extern struct page *pte_alloc_one (struct mm_struct *mm,unsigned long addr); +extern pgtable_t pte_alloc_one (struct mm_struct *mm,unsigned long addr);extern void pte_free_kernel (struct mm_struct *mm,pte_t *pte); -extern void pte_free (struct mm_struct *mm,struct page *pte); +extern void pte_free (struct mm_struct *mm,pgtable_t pte);#define __pte_free_tlb (tlb, pte) pte_free((tlb)->mm, (pte))
include/asm-powerpc/pgalloc-64.h +19 -7
@@ -53,6 +53,7 @@ static inline void pud_populate(struct m#define pmd_populate (mm, pmd, pte_page) \ pmd_populate_kernel(mm, pmd, page_address(pte_page))#define pmd_populate_kernel (mm, pmd, pte) pmd_set(pmd, (unsigned long)(pte)) +#define pmd_pgtable (pmd) pmd_page(pmd)#else /* CONFIG_PPC_64K_PAGES */
@@ -87,11 +88,18 @@ static inline pte_t *pte_alloc_one_kernereturn (pte_t *)__get_free_page (GFP_KERNEL | __GFP_REPEAT | __GFP_ZERO); } -static inline struct page *pte_alloc_one (struct mm_struct *mm, -unsigned long address) +static inline pgtable_t pte_alloc_one (struct mm_struct *mm, +unsigned long address) { -pte_t *pte =pte_alloc_one_kernel (mm, address); -return pte ?virt_to_page (pte) :NULL ; +struct page *page; +pte_t *pte; + + pte =pte_alloc_one_kernel (mm, address); +if (!pte) +return NULL ; + page =virt_to_page (pte); +pgtable_page_ctor (page); +return page; }static inline void pte_free_kernel (struct mm_struct *mm,pte_t *pte)
@@ -99,8 +107,9 @@ static inline void pte_free_kernel(strucfree_page ((unsigned long )pte); } -static inline void pte_free (struct mm_struct *mm,struct page *ptepage) +static inline void pte_free (struct mm_struct *mm,pgtable_t ptepage) { +pgtable_page_dtor (ptepage);__free_page (ptepage); }
@@ -131,9 +140,12 @@ static inline void pgtable_free(pgtable_extern void pgtable_free_tlb (struct mmu_gather *tlb,pgtable_free_t pgf); -#define __pte_free_tlb (tlb, ptepage) \ +#define __pte_free_tlb (tlb,ptepage) \ +do { \ + pgtable_page_dtor(ptepage); \ pgtable_free_tlb(tlb, pgtable_free_cache(page_address(ptepage), \ - PTE_NONCACHE_NUM, PTE_TABLE_SIZE-1)) + PTE_NONCACHE_NUM, PTE_TABLE_SIZE-1)); \ +} while (0)#define __pmd_free_tlb (tlb, pmd) \ pgtable_free_tlb(tlb, pgtable_free_cache(pmd, \ PMD_CACHE_NUM, PMD_TABLE_SIZE-1))
include/asm-ppc/pgalloc.h +4 -2
@@ -23,17 +23,19 @@ extern void pgd_free(struct mm_struct *m (pmd_val(*(pmd)) = __pa(pte) | _PMD_PRESENT)#define pmd_populate (mm, pmd, pte) \ (pmd_val(*(pmd)) = (page_to_pfn(pte) << PAGE_SHIFT) | _PMD_PRESENT) +#define pmd_pgtable (pmd) pmd_page(pmd)#else #define pmd_populate_kernel (mm, pmd, pte) \ (pmd_val(*(pmd)) = (unsigned long)pte | _PMD_PRESENT)#define pmd_populate (mm, pmd, pte) \ (pmd_val(*(pmd)) = (unsigned long)lowmem_page_address(pte) | _PMD_PRESENT) +#define pmd_pgtable (pmd) pmd_page(pmd)#endif extern pte_t *pte_alloc_one_kernel (struct mm_struct *mm,unsigned long addr); -extern struct page *pte_alloc_one (struct mm_struct *mm,unsigned long addr); +extern pgtable_t pte_alloc_one (struct mm_struct *mm,unsigned long addr);extern void pte_free_kernel (struct mm_struct *mm,pte_t *pte); -extern void pte_free (struct mm_struct *mm,struct page *pte); +extern void pte_free (struct mm_struct *mm,pgtable_t pte);#define __pte_free_tlb (tlb, pte) pte_free((tlb)->mm, (pte))
include/asm-s390/page.h +2 -0
@@ -110,6 +110,8 @@ typedef struct { unsigned long pgd; } pg#endif /* __s390x__ */ +typedef struct page *pgtable_t ; +#define __pte (x) ((pte_t) { (x) } )#define __pmd (x) ((pmd_t) { (x) } )#define __pgd (x) ((pgd_t) { (x) } )
include/asm-s390/pgalloc.h +2 -1
@@ -132,7 +132,7 @@ pmd_populate_kernel(struct mm_struct *mm }static inline void -pmd_populate (struct mm_struct *mm,pmd_t *pmd,struct page *page) +pmd_populate (struct mm_struct *mm,pmd_t *pmd,pgtable_t page) {pte_t *pte = (pte_t *)page_to_phys (page);pmd_t *shadow_pmd =get_shadow_table (pmd);
@@ -142,6 +142,7 @@ pmd_populate(struct mm_struct *mm, pmd_tif (shadow_pmd && shadow_pte)pmd_populate_kernel (mm, shadow_pmd, shadow_pte); } +#define pmd_pgtable (pmd) pmd_page(pmd)/* * page table entry allocation/free routines.
include/asm-s390/tlb.h +1 -1
@@ -95,7 +95,7 @@ static inline void tlb_remove_page(struc* pte_free_tlb frees a pte table and clears the CRSTE for the * page table from the tlb. */ -static inline void pte_free_tlb (struct mmu_gather *tlb,struct page *page) +static inline void pte_free_tlb (struct mmu_gather *tlb,pgtable_t page) {if (!tlb->fullmm) { tlb->array[tlb->nr_ptes++] = page;
include/asm-sh/page.h +2 -0
@@ -107,6 +107,8 @@ typedef struct { unsigned long pgd; } pg#define __pgd (x) ((pgd_t) { (x) } )#define __pgprot (x) ((pgprot_t) { (x) } ) +typedef struct page *pgtable_t ; +#endif /* !__ASSEMBLY__ */ /* to align the pointer to the (next) page boundary */
include/asm-sh/pgalloc.h +20 -7
@@ -14,10 +14,11 @@ static inline void pmd_populate_kernel(s }static inline void pmd_populate (struct mm_struct *mm,pmd_t *pmd, -struct page *pte) +pgtable_t pte) {set_pmd (pmd,__pmd ((unsigned long )page_address (pte))); } +#define pmd_pgtable (pmd) pmd_page(pmd)static inline void pgd_ctor (void *x) {
@@ -47,11 +48,18 @@ static inline pte_t *pte_alloc_one_kernereturn quicklist_alloc (QUICK_PT ,GFP_KERNEL | __GFP_REPEAT,NULL ); } -static inline struct page *pte_alloc_one (struct mm_struct *mm, -unsigned long address) +static inline pgtable_t pte_alloc_one (struct mm_struct *mm, +unsigned long address) { -void *pg =quicklist_alloc (QUICK_PT ,GFP_KERNEL | __GFP_REPEAT,NULL ); -return pg ?virt_to_page (pg) :NULL ; +struct page *page; +void *pg; + + pg =quicklist_alloc (QUICK_PT ,GFP_KERNEL | __GFP_REPEAT,NULL ); +if (!pg) +return NULL ; + page =virt_to_page (pg); +pgtable_page_ctor (page); +return page; }static inline void pte_free_kernel (struct mm_struct *mm,pte_t *pte)
@@ -59,12 +67,17 @@ static inline void pte_free_kernel(strucquicklist_free (QUICK_PT ,NULL , pte); } -static inline void pte_free (struct mm_struct *mm,struct page *pte) +static inline void pte_free (struct mm_struct *mm,pgtable_t pte) { +pgtable_page_dtor (pte);quicklist_free_page (QUICK_PT ,NULL , pte); } -#define __pte_free_tlb (tlb,pte) tlb_remove_page((tlb),(pte)) +#define __pte_free_tlb (tlb,pte) \ +do { \ + pgtable_page_dtor(pte); \ + tlb_remove_page((tlb), (pte)); \ +} while (0)/* * allocating and freeing a pmd is trivial: the 1-entry pmd is
include/asm-sh64/page.h +2 -0
@@ -83,6 +83,8 @@ typedef struct { unsigned long pgprot; }#define __pgd (x) ((pgd_t) { (x) } )#define __pgprot (x) ((pgprot_t) { (x) } ) +typedef struct page *pgtable_t ; +#endif /* !__ASSEMBLY__ */ /* to align the pointer to the (next) page boundary */
include/asm-sh64/pgalloc.h +20 -7
@@ -51,11 +51,18 @@ static inline void pgd_free(struct mm_stquicklist_free (0 ,NULL , pgd); } -static inline struct page *pte_alloc_one (struct mm_struct *mm, -unsigned long address) +static inline pgtable_t pte_alloc_one (struct mm_struct *mm, +unsigned long address) { -void *pg =quicklist_alloc (0 ,GFP_KERNEL ,NULL ); -return pg ?virt_to_page (pg) :NULL ; +struct page *page; +void *pg; + + pg =quicklist_alloc (0 ,GFP_KERNEL ,NULL ); +if (!pg) +return NULL ; + page =virt_to_page (pg); +pgtable_page_ctor (page); +return page; }static inline void pte_free_kernel (struct mm_struct *mm,pte_t *pte)
@@ -63,8 +70,9 @@ static inline void pte_free_kernel(strucquicklist_free (0 ,NULL , pte); } -static inline void pte_free (struct mm_struct *mm,struct page *pte) +static inline void pte_free (struct mm_struct *mm,pgtable_t pte) { +pgtable_page_dtor (pte);quicklist_free_page (0 ,NULL , pte); }
@@ -74,7 +82,11 @@ static inline pte_t *pte_alloc_one_kernereturn quicklist_alloc (0 ,GFP_KERNEL ,NULL ); } -#define __pte_free_tlb (tlb,pte) tlb_remove_page((tlb),(pte)) +#define __pte_free_tlb (tlb,pte) \ +do { \ + pgtable_page_dtor(pte); \ + tlb_remove_page((tlb),(pte)); \ +} while (0)/* * allocating and freeing a pmd is trivial: the 1-entry pmd is
@@ -112,10 +124,11 @@ static inline void pmd_free(struct mm_st set_pmd(pmd, __pmd(_PAGE_TABLE + (unsigned long) (pte)))static inline void pmd_populate (struct mm_struct *mm,pmd_t *pmd, -struct page *pte) +pgtable_t pte) {set_pmd (pmd,__pmd (_PAGE_TABLE + (unsigned long )page_address (pte))); } +#define pmd_pgtable (pmd) pmd_page(pmd)static inline void check_pgt_cache (void ) {
include/asm-sparc/page.h +2 -0
@@ -125,6 +125,8 @@ typedef unsigned long iopgprot_t;#endif +typedef struct page *pgtable_t ; +extern unsigned long sparc_unmapped_base;BTFIXUPDEF_SETHI (sparc_unmapped_base)
include/asm-sparc/pgalloc.h +3 -2
@@ -50,10 +50,11 @@ BTFIXUPDEF_CALL(void, free_pmd_fast, pmd BTFIXUPDEF_CALL(void, pmd_populate, pmd_t *, struct page *) #define pmd_populate(MM, PMD, PTE) BTFIXUP_CALL(pmd_populate)(PMD, PTE) +#define pmd_pgtable(pmd) pmd_page(pmd) BTFIXUPDEF_CALL(void, pmd_set, pmd_t *, pte_t *) #define pmd_populate_kernel(MM, PMD, PTE) BTFIXUP_CALL(pmd_set)(PMD, PTE) -BTFIXUPDEF_CALL(struct page *, pte_alloc_one, struct mm_struct *, unsigned long) +BTFIXUPDEF_CALL(pgtable_t , pte_alloc_one, struct mm_struct *, unsigned long) #define pte_alloc_one(mm, address) BTFIXUP_CALL(pte_alloc_one)(mm, address) BTFIXUPDEF_CALL(pte_t *, pte_alloc_one_kernel, struct mm_struct *, unsigned long) #define pte_alloc_one_kernel(mm, addr) BTFIXUP_CALL(pte_alloc_one_kernel)(mm, addr)
@@ -61,7 +62,7 @@ BTFIXUPDEF_CALL(pte_t *, pte_alloc_one_k BTFIXUPDEF_CALL(void, free_pte_fast, pte_t *) #define pte_free_kernel(mm,pte) BTFIXUP_CALL(free_pte_fast)(pte) -BTFIXUPDEF_CALL(void, pte_free, struct page *) +BTFIXUPDEF_CALL (void, pte_free, pgtable_t ) #define pte_free(mm,pte) BTFIXUP_CALL(pte_free)(pte)#define __pte_free_tlb (tlb, pte) pte_free((tlb)->mm, pte)
include/asm-sparc64/page.h +2 -0
@@ -106,6 +106,8 @@ typedef unsigned long pgprot_t;#endif /* (STRICT_MM_TYPECHECKS) */ +typedef struct page *pgtable_t ; +#define TASK_UNMAPPED_BASE (test_thread_flag(TIF_32BIT) ? \ (_AC(0x0000000070000000,UL)) : \ (_AC(0xfffff80000000000,UL) + (1UL << 32UL)))
include/asm-sparc64/pgalloc.h +14 -5
@@ -43,11 +43,18 @@ static inline pte_t *pte_alloc_one_kernereturn quicklist_alloc (0 ,GFP_KERNEL ,NULL ); } -static inline struct page *pte_alloc_one (struct mm_struct *mm, -unsigned long address) +static inline pgtable_t pte_alloc_one (struct mm_struct *mm, +unsigned long address) { -void *pg =quicklist_alloc (0 ,GFP_KERNEL ,NULL ); -return pg ?virt_to_page (pg) :NULL ; +struct page *page; +void *pg; + + pg =quicklist_alloc (0 ,GFP_KERNEL ,NULL ); +if (!pg) +return NULL ; + page =virt_to_page (pg); +pgtable_page_ctor (page); +return page; }static inline void pte_free_kernel (struct mm_struct *mm,pte_t *pte)
@@ -55,8 +62,9 @@ static inline void pte_free_kernel(strucquicklist_free (0 ,NULL , pte); } -static inline void pte_free (struct mm_struct *mm,struct page *ptepage) +static inline void pte_free (struct mm_struct *mm,pgtable_t ptepage) { +pgtable_page_dtor (ptepage);quicklist_free_page (0 ,NULL , ptepage); }
@@ -64,6 +72,7 @@ static inline void pte_free(struct mm_st#define pmd_populate_kernel (MM ,PMD ,PTE ) pmd_set(PMD, PTE)#define pmd_populate (MM ,PMD ,PTE_PAGE ) \ pmd_populate_kernel(MM,PMD,page_address(PTE_PAGE)) +#define pmd_pgtable (pmd) pmd_page(pmd)static inline void check_pgt_cache (void ) {
include/asm-um/page.h +2 -0
@@ -79,6 +79,8 @@ typedef unsigned long phys_t;typedef struct {unsigned long pgprot; }pgprot_t ; +typedef struct page *pgtable_t ; +#define pgd_val (x) ((x).pgd)#define pgprot_val (x) ((x).pgprot)
include/asm-um/pgalloc.h +9 -3
@@ -18,6 +18,7 @@set_pmd (pmd,__pmd (_PAGE_TABLE + \ ((unsigned long long )page_to_pfn (pte) << \ (unsigned long long) PAGE_SHIFT))) +#define pmd_pgtable (pmd) pmd_page(pmd)/* * Allocate and free page tables.
@@ -26,19 +27,24 @@ extern pgd_t *pgd_alloc(struct mm_structextern void pgd_free (struct mm_struct *mm,pgd_t *pgd);extern pte_t *pte_alloc_one_kernel (struct mm_struct *,unsigned long ); -extern struct page *pte_alloc_one (struct mm_struct *,unsigned long ); +extern pgtable_t pte_alloc_one (struct mm_struct *,unsigned long );static inline void pte_free_kernel (struct mm_struct *mm,pte_t *pte) {free_page ((unsigned long ) pte); } -static inline void pte_free (struct mm_struct *mm,struct page *pte) +static inline void pte_free (struct mm_struct *mm,pgtable_t pte) { +pgtable_page_dtor (pte);__free_page (pte); } -#define __pte_free_tlb (tlb,pte) tlb_remove_page((tlb),(pte)) +#define __pte_free_tlb (tlb,pte) \ +do { \ + pgtable_page_dtor(pte); \ + tlb_remove_page((tlb),(pte)); \ +} while (0)#ifdef CONFIG_3_LEVEL_PGTABLES
include/asm-x86/page_32.h +2 -0
@@ -116,6 +116,8 @@ static inline pte_t native_make_pte(unsi#include <asm-generic/pgtable-nopmd.h> #endif /* CONFIG_X86_PAE */ +typedef struct page *pgtable_t ; +#define PTE_MASK PAGE_MASK#ifdef CONFIG_HUGETLB_PAGE
include/asm-x86/page_64.h +2 -0
@@ -62,6 +62,8 @@ typedef struct { unsigned long pgd; } pgtypedef struct {unsigned long pgprot; }pgprot_t ; +typedef struct page *pgtable_t ; +extern unsigned long phys_base;#define pte_val (x) ((x).pte)
include/asm-x86/pgalloc_32.h +5 -2
@@ -28,6 +28,7 @@ do { \ ((unsigned long long )page_to_pfn (pte) << \ (unsigned long long) PAGE_SHIFT))); \ } while (0) +#define pmd_pgtable (pmd) pmd_page(pmd)/* * Allocate and free page tables.
@@ -36,21 +37,23 @@ extern pgd_t *pgd_alloc(struct mm_structextern void pgd_free (struct mm_struct *mm,pgd_t *pgd);extern pte_t *pte_alloc_one_kernel (struct mm_struct *,unsigned long ); -extern struct page *pte_alloc_one (struct mm_struct *,unsigned long ); +extern pgtable_t pte_alloc_one (struct mm_struct *,unsigned long );static inline void pte_free_kernel (struct mm_struct *mm,pte_t *pte) {free_page ((unsigned long )pte); } -static inline void pte_free (struct mm_struct *mm,struct page *pte) +static inline void pte_free (struct mm_struct *mm,pgtable_t pte) { +pgtable_page_dtor (pte);__free_page (pte); }#define __pte_free_tlb (tlb,pte) \ do { \ + pgtable_page_dtor(pte); \ paravirt_release_pt(page_to_pfn(pte)); \ tlb_remove_page((tlb),(pte)); \ } while (0)
include/asm-x86/pgalloc_64.h +17 -5
@@ -12,6 +12,8 @@#define pgd_populate (mm, pgd, pud) \ set_pgd(pgd, __pgd(_PAGE_TABLE | __pa(pud))) +#define pmd_pgtable (pmd) pmd_page(pmd) +static inline void pmd_populate (struct mm_struct *mm,pmd_t *pmd,struct page *pte) {set_pmd (pmd,__pmd (_PAGE_TABLE | (page_to_pfn (pte) <<PAGE_SHIFT )));
@@ -89,12 +91,17 @@ static inline pte_t *pte_alloc_one_kernereturn (pte_t *)get_zeroed_page (GFP_KERNEL |__GFP_REPEAT); } -static inline struct page *pte_alloc_one (struct mm_struct *mm,unsigned long address) +static inline pgtable_t pte_alloc_one (struct mm_struct *mm,unsigned long address) { -void *p = (void *)get_zeroed_page (GFP_KERNEL |__GFP_REPEAT); +struct page *page; +void *p; + + p = (void *)get_zeroed_page (GFP_KERNEL |__GFP_REPEAT);if (!p)return NULL ; -return virt_to_page (p); + page =virt_to_page (p); +pgtable_page_ctor (page); +return page; }/* Should really implement gc for free page table pages. This could be
@@ -106,12 +113,17 @@ static inline void pte_free_kernel(strucfree_page ((unsigned long )pte); } -static inline void pte_free (struct mm_struct *mm,struct page *pte) +static inline void pte_free (struct mm_struct *mm,pgtable_t pte) { +pgtable_page_dtor (pte);__free_page (pte); } -#define __pte_free_tlb (tlb,pte) tlb_remove_page((tlb),(pte)) +#define __pte_free_tlb (tlb,pte) \ +do { \ + pgtable_page_dtor((pte)); \ + tlb_remove_page((tlb), (pte)); \ +} while (0)#define __pmd_free_tlb (tlb,x) tlb_remove_page((tlb),virt_to_page(x))#define __pud_free_tlb (tlb,x) tlb_remove_page((tlb),virt_to_page(x))
include/asm-xtensa/page.h +1 -0
@@ -100,6 +100,7 @@typedef struct {unsigned long pte; }pte_t ;/* page table entry */ typedef struct {unsigned long pgd; }pgd_t ;/* PGD table entry */ typedef struct {unsigned long pgprot; }pgprot_t ; +typedef struct page *pgtable_t ;#define pte_val (x) ((x).pte)#define pgd_val (x) ((x).pgd)
include/asm-xtensa/pgalloc.h +12 -5
@@ -24,6 +24,7 @@ (pmd_val(*(pmdp)) = ((unsigned long)ptep))#define pmd_populate (mm, pmdp, page) \ (pmd_val(*(pmdp)) = ((unsigned long)page_to_virt(page))) +#define pmd_pgtable (pmd) pmd_page(pmd)static inline pgd_t *pgd_alloc (struct mm_struct *mm)
@@ -46,10 +47,14 @@ static inline pte_t *pte_alloc_one_kernereturn kmem_cache_alloc (pgtable_cache,GFP_KERNEL |__GFP_REPEAT); } -static inline struct page *pte_alloc_one (struct mm_struct *mm, -unsigned long addr) +static inline pte_token_t pte_alloc_one (struct mm_struct *mm, +unsigned long addr) { -return virt_to_page (pte_alloc_one_kernel (mm, addr)); +struct page *page; + + page =virt_to_page (pte_alloc_one_kernel (mm, addr)); +pgtable_page_ctor (page); +return page; }static inline void pte_free_kernel (struct mm_struct *mm,pte_t *pte)
@@ -57,10 +62,12 @@ static inline void pte_free_kernel(struckmem_cache_free (pgtable_cache, pte); } -static inline void pte_free (struct mm_struct *mm,struct page *page) +static inline void pte_free (struct mm_struct *mm,pgtable_t pte) { -kmem_cache_free (pgtable_cache,page_address (page)); +pgtable_page_dtor (pte); +kmem_cache_free (pgtable_cache,page_address (pte)); } +#define pmd_pgtable (pmd) pmd_page(pmd)#endif /* __KERNEL__ */ #endif /* _XTENSA_PGALLOC_H */
include/linux/mm.h +13 -1
@@ -843,6 +843,18 @@ static inline pmd_t *pmd_alloc(struct mm#define pte_lockptr (mm, pmd) ({(void)(pmd); &(mm)->page_table_lock;})#endif /* NR_CPUS < CONFIG_SPLIT_PTLOCK_CPUS */ +static inline void pgtable_page_ctor (struct page *page) +{ +pte_lock_init (page); +inc_zone_page_state (page,NR_PAGETABLE ); +} + +static inline void pgtable_page_dtor (struct page *page) +{ +pte_lock_deinit (page); +dec_zone_page_state (page,NR_PAGETABLE ); +} +#define pte_offset_map_lock (mm, pmd, address, ptlp) \ ({ \ spinlock_t *__ptl = pte_lockptr(mm, pmd); \
@@ -1087,7 +1099,7 @@ struct page *follow_page(struct vm_area_#define FOLL_GET 0x04/* do get_page on page */ #define FOLL_ANON 0x08/* give ZERO_PAGE if no pgtable */ -typedef int (*pte_fn_t )(pte_t *pte,struct page *pmd_page,unsigned long addr, +typedef int (*pte_fn_t )(pte_t *pte,pgtable_t token,unsigned long addr,void *data);extern int apply_to_page_range (struct mm_struct *mm,unsigned long address,unsigned long size,pte_fn_t fn,void *data);
mm/memory.c +15 -17
@@ -122,11 +122,9 @@ void pmd_clear_bad(pmd_t *pmd)*/ static void free_pte_range (struct mmu_gather *tlb,pmd_t *pmd) { -struct page *page =pmd_page (*pmd); +pgtable_t token =pmd_pgtable (*pmd);pmd_clear (pmd); -pte_lock_deinit (page); -pte_free_tlb (tlb, page); -dec_zone_page_state (page,NR_PAGETABLE ); +pte_free_tlb (tlb, token); tlb->mm->nr_ptes--; }
@@ -297,21 +295,19 @@ void free_pgtables(struct mmu_gather **tint __pte_alloc (struct mm_struct *mm,pmd_t *pmd,unsigned long address) { -struct page *new =pte_alloc_one (mm, address); +pgtable_t new =pte_alloc_one (mm, address);if (!new)return -ENOMEM ; -pte_lock_init (new);spin_lock (&mm->page_table_lock); -if (pmd_present (*pmd)) {/* Another has populated it */ -pte_lock_deinit (new); -pte_free (mm, new); - }else { +if (!pmd_present (*pmd)) {/* Has another populated it ? */ mm->nr_ptes++; -inc_zone_page_state (new,NR_PAGETABLE );pmd_populate (mm, pmd, new); + new =NULL ; }spin_unlock (&mm->page_table_lock); +if (new) +pte_free (mm, new);return 0 ; }
@@ -322,11 +318,13 @@ int __pte_alloc_kernel(pmd_t *pmd, unsigreturn -ENOMEM ;spin_lock (&init_mm.page_table_lock); -if (pmd_present (*pmd))/* Another has populated it */ -pte_free_kernel (&init_mm, new); -else +if (!pmd_present (*pmd)) {/* Has another populated it ? */ pmd_populate_kernel (&init_mm, pmd, new); + new =NULL ; + }spin_unlock (&init_mm.page_table_lock); +if (new) +pte_free_kernel (&init_mm, new);return 0 ; }
@@ -1368,7 +1366,7 @@ static int apply_to_pte_range(struct mm_ {pte_t *pte;int err; -struct page *pmd_page; +pgtable_t token;spinlock_t *uninitialized_var (ptl ); pte = (mm == &init_mm) ?
@@ -1379,10 +1377,10 @@ static int apply_to_pte_range(struct mm_BUG_ON (pmd_huge (*pmd)); - pmd_page =pmd_page (*pmd); + token =pmd_pgtable (*pmd);do { - err =fn (pte, pmd_page, addr, data); + err =fn (pte, token, addr, data);if (err)break ; }while (pte++, addr +=PAGE_SIZE , addr != end);
mm/vmalloc.c +1 -1
@@ -772,7 +772,7 @@ void __attribute__((weak)) vmalloc_sync } -static int f (pte_t *pte,struct page *pmd_page,unsigned long addr,void *data) +static int f (pte_t *pte,pgtable_t table,unsigned long addr,void *data) {/* apply_to_page_range() does all the hard work. */ return 0 ;
Signature · 4 lines
blue skies, Martin. "Reality continues to ruin my life." - Calvin.
Re: [patch 2/6] CONFIG_HIGHPTE vs. sub-page page tables.
Quoted 12 lines
Solution: The only solution I found to this dilemma is a new typedef:
apgtable_t. For s390pgtable_twill be a (pte *) - to be introduced with a later patch. For everybody else it will be a (struct page *). The additional problem with the initialization of the ptl lock and theNR_PAGETABLEaccounting is solved with a constructorpgtable_page_ctorand a destructorpgtable_page_dtor. The page table allocation and free functions need to call these two whenever a page table page is allocated or freed.pmd_populatewill get apgtable_tinstead of a struct page pointer. To get thepgtable_tback from a pmd entry that has been installed withpmd_populatea new functionpmd_pgtableis added. It replaces thepmd_pagecall infree_pte_rangeandapply_to_pte_range.
Quoted 2 lines
Signed-off-by Martin Schwidefsky <[email protected]>---
Interesting. That means I don't need to have a PTE page to be a struct page anymore ? I can have good use for that on powerpc as well...
Cheers,
Ben.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [patch 2/6] CONFIG_HIGHPTE vs. sub-page page tables.
Quoted 12 lines · Benjamin Herrenschmidt
Quoted 11 lines
Solution: The only solution I found to this dilemma is a new typedef:
apgtable_t. For s390pgtable_twill be a (pte *) - to be introduced with a later patch. For everybody else it will be a (struct page *). The additional problem with the initialization of the ptl lock and theNR_PAGETABLEaccounting is solved with a constructorpgtable_page_ctorand a destructorpgtable_page_dtor. The page table allocation and free functions need to call these two whenever a page table page is allocated or freed.pmd_populatewill get apgtable_tinstead of a struct page pointer. To get thepgtable_tback from a pmd entry that has been installed withpmd_populatea new functionpmd_pgtableis added. It replaces thepmd_pagecall infree_pte_rangeandapply_to_pte_range.
Quoted 2 lines
Interesting. That means I don't need to have a PTE page to be a struct page anymore ? I can have good use for that on powerpc as well...
That would be good news. I'm curious, can you elaborate on what the use case is?
Signature · 11 lines
blue skies, Martin. "Reality continues to ruin my life." - Calvin. -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [patch 2/6] CONFIG_HIGHPTE vs. sub-page page tables.
Quoted 13 lines · Martin Schwidefsky
Quoted 12 lines · Benjamin Herrenschmidt
Quoted 11 lines
Solution: The only solution I found to this dilemma is a new typedef:
apgtable_t. For s390pgtable_twill be a (pte *) - to be introduced with a later patch. For everybody else it will be a (struct page *). The additional problem with the initialization of the ptl lock and theNR_PAGETABLEaccounting is solved with a constructorpgtable_page_ctorand a destructorpgtable_page_dtor. The page table allocation and free functions need to call these two whenever a page table page is allocated or freed.pmd_populatewill get apgtable_tinstead of a struct page pointer. To get thepgtable_tback from a pmd entry that has been installed withpmd_populatea new functionpmd_pgtableis added. It replaces thepmd_pagecall infree_pte_rangeandapply_to_pte_range.
Quoted 5 lines
Quoted 2 lines
Interesting. That means I don't need to have a PTE page to be a struct page anymore ? I can have good use for that on powerpc as well...
That would be good news. I'm curious, can you elaborate on what the use case is?
When using 64K pages, we use 32K of PTEs and 32K of "extension". The extension thing is used when using HW 4K pages, to keep track of the subpages. On setups where that isn't needed, we can save memory by allocating half pages...
Ben.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [patch 2/6] CONFIG_HIGHPTE vs. sub-page page tables.
Quoted 10 lines · Benjamin Herrenschmidt
Quoted 5 lines
Quoted 2 lines
Interesting. That means I don't need to have a PTE page to be a struct page anymore ? I can have good use for that on powerpc as well...
That would be good news. I'm curious, can you elaborate on what the use case is?
When using 64K pages, we use 32K of PTEs and 32K of "extension". The extension thing is used when using HW 4K pages, to keep track of the subpages. On setups where that isn't needed, we can save memory by allocating half pages...
Ahh, that is exactly the same reason as for s390. Page tables with a size that is sub-page.
Signature · 4 lines
blue skies, Martin. "Reality continues to ruin my life." - Calvin.
Re: [patch 2/6] CONFIG_HIGHPTE vs. sub-page page tables.
In reply to Benjamin HerrenschmidtQuoted 18 lines
arch/powerpc/mm/pgtable_32.c +3
-3@@ -107,20 +107,21 @@ __init_refok pte_t *pte_alloc_one_kernelreturn pte; } -struct page *pte_alloc_one (struct mm_struct *mm,unsigned long address) +pgtable_t pte_alloc_one (struct mm_struct *mm,unsigned long address) {struct page *ptepage;#ifdef CONFIG_HIGHPTE -gfp_t flags =GFP_KERNEL | __GFP_HIGHMEM | __GFP_REPEAT; +gfp_t flags =GFP_KERNEL | __GFP_HIGHMEM | __GFP_REPEAT | __GFP_ZERO;#else -gfp_t flags =GFP_KERNEL | __GFP_REPEAT; +gfp_t flags =GFP_KERNEL | __GFP_REPEAT | __GFP_ZERO;#endif ptepage =alloc_pages (flags,0 );
arch/powerpc/mm/pgtable_32.c __init_refok pte_t *pte_alloc_one_kernel · line 127
Quoted 10 lines
arch/powerpc/mm/pgtable_32.c +4
-3-if (ptepage) -clear_highpage (ptepage); -return ptepage; +if (!ptepage) +return NULL ; +pgtable_page_ctor (ptepage); +return page_address (ptepage); }void pte_free_kernel (struct mm_struct *mm,pte_t *pte)
Hmpf, where is my brown paper bag? The pte_alloc_one function for 32 bit powerpc should better return a struct page pointer .. fix below.
Signature · 19 lines
blue skies, Martin. "Reality continues to ruin my life." - Calvin. --- diff -urpN linux-2.6/arch/powerpc/mm/pgtable_32.c linux-2.6-patched/arch/powerpc/mm/pgtable_32.c --- linux-2.6/arch/powerpc/mm/pgtable_32.c 2007-10-26 09:38:30.000000000 +0200 +++ linux-2.6-patched/arch/powerpc/mm/pgtable_32.c 2007-10-26 10:01:10.000000000 +0200 @@ -121,7 +121,7 @@ pgtable_t pte_alloc_one(struct mm_struct if (!ptepage) return NULL; pgtable_page_ctor(ptepage); - return page_address(ptepage); + return ptepage; } void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
PATCH 3/6 arch_update_pgd call
In reply to Martin SchwidefskyFrom: Martin Schwidefsky <[email protected]>
In order to change the layout of the page tables after an mmap has crossed the adress space limit of the current page table layout a architecture hook in get_unmapped_area is needed. The arguments are the address of the new mapping and the length of it.
Signed-off-by Martin Schwidefsky <[email protected]>
---
mm/mmap.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-)
mm/mmap.c +5 -1
@@ -36,6 +36,10 @@#define arch_mmap_check (addr, len, flags) (0)#endif +#ifndef arch_update_pgd +#define arch_update_pgd (addr, len) (addr) +#endif +static void unmap_region (struct mm_struct *mm,struct vm_area_struct *vma,struct vm_area_struct *prev,unsigned long start,unsigned long end);
@@ -1420,7 +1424,7 @@ get_unmapped_area(struct file *file, unsif (addr & ~PAGE_MASK )return -EINVAL ; -return addr; +return arch_update_pgd (addr, len); }EXPORT_SYMBOL (get_unmapped_area);
Signature · 4 lines
blue skies, Martin. "Reality continues to ruin my life." - Calvin.
Re: [patch 3/6] arch_update_pgd call
Quoted 9 lines · Martin Schwidefsky
plain text document attachment (003-mm-update-pgd.diff)
From: Martin Schwidefsky <[email protected]>In order to change the layout of the page tables after an mmap has crossed the adress space limit of the current page table layout a architecture hook in
get_unmapped_areais needed. The arguments are the address of the new mapping and the length of it.Signed-off-by Martin Schwidefsky <[email protected]>
I'm not at all fan of the hook there and it's name...
Any reason why you can't do that in your arch gua ?
If not, then why can't you call it something nicer, like arch_rebalance_pgtables() ?
Ben.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [patch 3/6] arch_update_pgd call
Quoted 1 line · Benjamin Herrenschmidt
I'm not at all fan of the hook there and it's name...
And before somebody jumps on that one.... s/it's/its
Ben.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [patch 3/6] arch_update_pgd call
In reply to Benjamin HerrenschmidtQuoted 9 lines · Benjamin Herrenschmidt
Quoted 8 lines · Martin Schwidefsky
plain text document attachment (003-mm-update-pgd.diff)
From: Martin Schwidefsky <[email protected]>In order to change the layout of the page tables after an mmap has crossed the adress space limit of the current page table layout a architecture hook in
get_unmapped_areais needed. The arguments are the address of the new mapping and the length of it.
Quoted 8 lines
Quoted 1 line
Signed-off-by Martin Schwidefsky <[email protected]>I'm not at all fan of the hook there and it's name...
Any reason why you can't do that in your arch gua ?
If not, then why can't you call it something nicer, like
arch_rebalance_pgtables()?
The name can be changed in no time. I've tried to use one of the existing arch calls like arch_mmap_check or arch_get_unmapped_area but it didn't work out. I really need the final address to make the call to extend the page tables.
Signature · 4 lines
blue skies, Martin. "Reality continues to ruin my life." - Calvin.
Re: [patch 3/6] arch_update_pgd call
Quoted 16 lines · Martin Schwidefsky
Quoted 15 lines · Benjamin Herrenschmidt
Quoted 9 lines · Martin Schwidefsky
plain text document attachment (003-mm-update-pgd.diff)
From: Martin Schwidefsky <[email protected]>In order to change the layout of the page tables after an mmap has crossed the adress space limit of the current page table layout a architecture hook in
get_unmapped_areais needed. The arguments are the address of the new mapping and the length of it.Signed-off-by Martin Schwidefsky <[email protected]>I'm not at all fan of the hook there and it's name...
Any reason why you can't do that in your arch gua ?
Quoted 7 lines
Quoted 2 lines
If not, then why can't you call it something nicer, like
arch_rebalance_pgtables()?The name can be changed in no time. I've tried to use one of the existing arch calls like
arch_mmap_checkorarch_get_unmapped_areabut it didn't work out. I really need the final address to make the call to extend the page tables.
You arch get_unmapped_area() has it...
Ben.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [patch 3/6] arch_update_pgd call
Quoted 7 lines · Benjamin Herrenschmidt
Quoted 7 lines
Quoted 6 lines
I'm not at all fan of the hook there and it's name...
Any reason why you can't do that in your arch gua ?
If not, then why can't you call it something nicer, like
arch_rebalance_pgtables()?
Quoted 6 lines
Quoted 4 lines
The name can be changed in no time. I've tried to use one of the existing arch calls like
arch_mmap_checkorarch_get_unmapped_areabut it didn't work out. I really need the final address to make the call to extend the page tables.You arch
get_unmapped_area()has it...
Hmm, I got worried about the file->f_op->get_unmapped_area indirection. At least the get_unmapped_area_mem function in drivers/char/mem.c does not call the standard get_unmapped_area function again like ipc/shm.c does. As I see it is not guaranteed that the architecture version of arch_get_unmapped_area gets control. Want to play safe there.
Signature · 4 lines
blue skies, Martin. "Reality continues to ruin my life." - Calvin.
Re: [patch 3/6] arch_update_pgd call
Quoted 13 lines · Martin Schwidefsky
Quoted 12 lines · Benjamin Herrenschmidt
Quoted 11 lines
Quoted 6 lines
I'm not at all fan of the hook there and it's name...
Any reason why you can't do that in your arch gua ?
If not, then why can't you call it something nicer, like
arch_rebalance_pgtables()?The name can be changed in no time. I've tried to use one of the existing arch calls like
arch_mmap_checkorarch_get_unmapped_areabut it didn't work out. I really need the final address to make the call to extend the page tables.
Quoted 7 lines
Quoted 1 line
You arch
get_unmapped_area()has it...Hmm, I got worried about the
file->f_op->get_unmapped_areaindirection. At least theget_unmapped_area_memfunction indrivers/char/mem.cdoes not call the standardget_unmapped_areafunction again likeipc/shm.cdoes. As I see it is not guaranteed that the architecture version ofarch_get_unmapped_areagets control. Want to play safe there.
Hrm... last I looked, the only "other" gua was hugetlb which is arch too, but I may have missed something.
Ben.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH 4/6 s390: 1K/2K page table pages.
In reply to Martin SchwidefskyFrom: Martin Schwidefsky <[email protected]>
This patch implements 1K/2K page table pages for s390.
Signed-off-by Martin Schwidefsky <[email protected]>
---
arch/s390/mm/pgtable.c | 100 ++++++++++++++++++++++++++++++--------- arch/s390/mm/vmem.c | 10 +++ include/asm-s390/elf.h | 12 ++++ include/asm-s390/mmu.h | 7 +- include/asm-s390/mmu_context.h | 16 +++--- include/asm-s390/page.h | 37 +------------- include/asm-s390/pgalloc.h | 79 ++++++++++++++---------------- include/asm-s390/pgtable.h | 105 ++++++++++++++--------------------------- include/asm-s390/tlb.h | 6 +- include/asm-s390/tlbflush.h | 6 -- 10 files changed, 197 insertions(+), 181 deletions(-)
arch/s390/mm/pgtable.c +79 -21
@@ -26,8 +26,12 @@#ifndef CONFIG_64BIT #define ALLOC_ORDER 1 +#define TABLES_PER_PAGE 4 +#define SECOND_HALVES 10#else #define ALLOC_ORDER 2 +#define TABLES_PER_PAGE 2 +#define SECOND_HALVES 2#endif unsigned long *crst_table_alloc (struct mm_struct *mm,int noexec)
@@ -45,13 +49,20 @@ unsigned long *crst_table_alloc(struct m } page->index =page_to_phys (shadow); } +spin_lock (&mm->page_table_lock); +list_add (&page->lru, &mm->context.crst_list); +spin_unlock (&mm->page_table_lock);return (unsigned long *)page_to_phys (page); } -void crst_table_free (unsigned long *table) +void crst_table_free (struct mm_struct *mm,unsigned long *table) {unsigned long *shadow =get_shadow_table (table); +struct page *page =virt_to_page (table); +spin_lock (&mm->page_table_lock); +list_del (&page->lru); +spin_unlock (&mm->page_table_lock);if (shadow)free_pages ((unsigned long ) shadow,ALLOC_ORDER );free_pages ((unsigned long ) table,ALLOC_ORDER );
@@ -60,37 +71,84 @@ void crst_table_free(unsigned long *tabl/* * page table entry allocation/free routines. */ -unsigned long *page_table_alloc (int noexec) +unsigned long *page_table_alloc (struct mm_struct *mm) { -struct page *page =alloc_page (GFP_KERNEL ); +struct page *page;unsigned long *table; +unsigned long bits; -if (!page) -return NULL ; - page->index =0 ; -if (noexec) { -struct page *shadow =alloc_page (GFP_KERNEL ); -if (!shadow) { -__free_page (page); + bits = mm->context.noexec ?3UL :1UL ; +spin_lock (&mm->page_table_lock); + page =NULL ; +if (!list_empty (&mm->context.pgtable_list)) { + page =list_first_entry (&mm->context.pgtable_list, + struct page, lru); +if (page->flags == ((1UL <<TABLES_PER_PAGE ) -1 )) + page =NULL ; + } +if (!page) { +spin_unlock (&mm->page_table_lock); + page =alloc_page (GFP_KERNEL |__GFP_REPEAT); +if (!page)return NULL ; - } - table = (unsigned long *)page_to_phys (shadow); +pgtable_page_ctor (page); + page->flags =0 ; + table = (unsigned long *)page_to_phys (page);clear_table (table, _PAGE_TYPE_EMPTY,PAGE_SIZE ); - page->index = (addr_t ) table; +spin_lock (&mm->page_table_lock); +list_add (&page->lru, &mm->context.pgtable_list); } -pgtable_page_ctor (page); table = (unsigned long *)page_to_phys (page); -clear_table (table, _PAGE_TYPE_EMPTY,PAGE_SIZE ); +while (page->flags & bits) { + table +=256 ; + bits <<=1 ; + } + page->flags |= bits; +if (page->flags == ((1UL <<TABLES_PER_PAGE ) -1 )) +list_move_tail (&page->lru, &mm->context.pgtable_list); +spin_unlock (&mm->page_table_lock);return table; } -void page_table_free (unsigned long *table) +void page_table_free (struct mm_struct *mm,unsigned long *table) { -unsigned long *shadow =get_shadow_pte (table); +struct page *page; +unsigned long bits; -pgtable_page_dtor (virt_to_page (table)); -if (shadow) -free_page ((unsigned long ) shadow); -free_page ((unsigned long ) table); + bits = mm->context.noexec ?3UL :1UL ; + bits <<= (__pa (table) & (PAGE_SIZE -1 )) /256 /sizeof (unsigned long ); + page =pfn_to_page (__pa (table) >>PAGE_SHIFT ); +spin_lock (&mm->page_table_lock); + page->flags ^= bits; +if (page->flags) { +/* Page now has some free pgtable fragments. */ +list_move (&page->lru, &mm->context.pgtable_list); + page =NULL ; + }else +/* All fragments of the 4K page have been freed. */ +list_del (&page->lru); +spin_unlock (&mm->page_table_lock); +if (page) { +pgtable_page_dtor (page); +__free_page (page); + } +} +void disable_noexec (struct mm_struct *mm,struct task_struct *tsk) +{ +struct page *page; + +spin_lock (&mm->page_table_lock); +/* Free shadow region and segment tables. */ +list_for_each_entry (page, &mm->context.crst_list, lru) +if (page->index) { +free_pages ((unsigned long ) page->index,ALLOC_ORDER ); + page->index =0 ; + } +/* "Free" second halves of page tables. */ +list_for_each_entry (page, &mm->context.pgtable_list, lru) + page->flags &= ~SECOND_HALVES ; +spin_unlock (&mm->page_table_lock); + mm->context.noexec =0 ; +update_mm (mm, tsk); }
arch/s390/mm/vmem.c +10 -0
@@ -75,6 +75,13 @@ static void __init_refok *vmem_alloc_pag#define vmem_pud_alloc () ({ BUG(); ((pud_t *) NULL); }) +static inline void *vmem_alloc_slab (size_t size) +{ +if (slab_is_available ()) +return (void *)kmalloc (size,GFP_KERNEL ); +return alloc_bootmem (size); +} +static inline pmd_t *vmem_pmd_alloc (void ) {pmd_t *pmd =NULL ;
@@ -363,6 +370,9 @@ void __init vmem_map_init(void)unsigned long map_size;int i; +INIT_LIST_HEAD (&init_mm.context.crst_list); +INIT_LIST_HEAD (&init_mm.context.pgtable_list); + init_mm.context.noexec =0 ; map_size =ALIGN (max_low_pfn,MAX_ORDER_NR_PAGES ) *sizeof (struct page ); vmalloc_end =PFN_ALIGN (VMALLOC_END_INIT ) -PFN_ALIGN (map_size); vmem_map = (struct page *) vmalloc_end;
include/asm-s390/elf.h +12 -0
@@ -116,6 +116,7 @@ typedef s390_regs elf_gregset_t;#ifdef __KERNEL__#include <linux/sched.h> /* for task_struct */ #include <asm/system.h> /* for save_access_regs */ +#include <asm/mmu_context.h> /* * This is used to ensure we don't load something for the wrong architecture.
@@ -214,6 +215,17 @@ do { \clear_thread_flag (TIF_31BIT ); \ } while (0)#endif /* __s390x__ */ +/* +* An executable for which elf_read_implies_exec() returns TRUE will +* have the READ_IMPLIES_EXEC personality flag set automatically. +*/ +#define elf_read_implies_exec (ex, executable_stack) \ +({ \ + if (current->mm->context.noexec && \ + executable_stack != EXSTACK_DISABLE_X) \ + disable_noexec(current->mm, current); \ + current->mm->context.noexec == 0; \ +})#endif #endif
include/asm-s390/mmu_context.h +9 -7
@@ -10,13 +10,15 @@#define __S390_MMU_CONTEXT_H#include <asm/pgalloc.h> +#include <asm/uaccess.h> #include <asm-generic/mm_hooks.h> -/* -* get a new mmu context.. S390 don't know about contexts. -*/ -#define init_new_context (tsk,mm) 0 - +static inline int init_new_context (struct task_struct *tsk, +struct mm_struct *mm) +{ + mm->context.noexec = s390_noexec; +return (0 ); +}#define destroy_context (mm) do { } while (0)#ifndef __s390x__
@@ -38,8 +40,8 @@ static inline void update_mm(struct mm_s S390_lowcore.user_asce = asce_bits |__pa (pgd);if (switch_amode) {/* Load primary space page table origin. */ -pgd_t *shadow_pgd =get_shadow_table (pgd) ? : pgd; - S390_lowcore.user_exec_asce = asce_bits |__pa (shadow_pgd); + pgd = mm->context.noexec ?get_shadow_table (pgd) : pgd; + S390_lowcore.user_exec_asce = asce_bits |__pa (pgd); asmvolatile (LCTL_OPCODE " 1,1,%0\n" : :"m" (S390_lowcore.user_exec_asce) ); }else
include/asm-s390/mmu.h +5 -2
@@ -1,7 +1,10 @@#ifndef __MMU_H#define __MMU_H -/* Default "unsigned long" context */ -typedef unsigned long mm_context_t ; +typedef struct { +struct list_head crst_list; +struct list_head pgtable_list; +int noexec; +}mm_context_t ;#endif
include/asm-s390/page.h +5 -32
@@ -75,43 +75,17 @@ static inline void copy_page(void *to, vtypedef struct {unsigned long pgprot; }pgprot_t ;typedef struct {unsigned long pte; }pte_t ; - -#define pte_val (x) ((x).pte) -#define pgprot_val (x) ((x).pgprot) - -#ifndef __s390x__ -typedef struct {unsigned long pmd; }pmd_t ;typedef struct {unsigned long pud; }pud_t ; -typedef struct { -unsigned long pgd0; -unsigned long pgd1; -unsigned long pgd2; -unsigned long pgd3; - }pgd_t ; - -#define pmd_val (x) ((x).pmd) -#define pud_val (x) ((x).pud) -#define pgd_val (x) ((x).pgd0) - -#else /* __s390x__ */ - -typedef struct { -unsigned long pmd0; -unsigned long pmd1; - }pmd_t ; -typedef struct {unsigned long pud; }pud_t ;typedef struct {unsigned long pgd; }pgd_t ; +typedef pte_t *pgtable_t ; -#define pmd_val (x) ((x).pmd0) -#define pmd_val1 (x) ((x).pmd1) +#define pgprot_val (x) ((x).pgprot) +#define pte_val (x) ((x).pte) +#define pmd_val (x) ((x).pmd)#define pud_val (x) ((x).pud)#define pgd_val (x) ((x).pgd) -#endif /* __s390x__ */ - -typedef struct page *pgtable_t ; -#define __pte (x) ((pte_t) { (x) } )#define __pmd (x) ((pmd_t) { (x) } )#define __pgd (x) ((pgd_t) { (x) } )
@@ -168,8 +142,7 @@ static inline int pfn_valid(unsigned lon#define page_to_phys (page) (page_to_pfn(page) << PAGE_SHIFT)#define virt_addr_valid (kaddr) pfn_valid(__pa(kaddr) >> PAGE_SHIFT) -#define VM_DATA_DEFAULT_FLAGS (VM_READ | VM_WRITE | VM_EXEC | \ - VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC) +#define VM_DATA_DEFAULT_FLAGS (VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE )#include <asm-generic/memory_model.h> #include <asm-generic/page.h>
include/asm-s390/pgalloc.h +36 -43
121 lines not on this page · raw message
include/asm-s390/pgtable.h +36 -69
203 lines not on this page · raw message
include/asm-s390/tlbflush.h +2 -4
@@ -61,10 +61,8 @@ static inline void __tlb_flush_mm(struct* only ran on the local cpu. */ if (MACHINE_HAS_IDTE ) { -pgd_t *shadow_pgd =get_shadow_table (mm->pgd); - -if (shadow_pgd) -__tlb_flush_idte (shadow_pgd); +if (mm->context.noexec) +__tlb_flush_idte (get_shadow_table (mm->pgd));__tlb_flush_idte (mm->pgd);return ; }
include/asm-s390/tlb.h +3 -3
@@ -95,14 +95,14 @@ static inline void tlb_remove_page(struc* pte_free_tlb frees a pte table and clears the CRSTE for the * page table from the tlb. */ -static inline void pte_free_tlb (struct mmu_gather *tlb,pgtable_t page) +static inline void pte_free_tlb (struct mmu_gather *tlb,pgtable_t pte) {if (!tlb->fullmm) { - tlb->array[tlb->nr_ptes++] = page; + tlb->array[tlb->nr_ptes++] = pte;if (tlb->nr_ptes >= tlb->nr_pmds)tlb_flush_mmu (tlb,0 ,0 ); }else -pte_free (tlb->mm, page); +pte_free (tlb->mm, pte); }/*
Signature · 4 lines
blue skies, Martin. "Reality continues to ruin my life." - Calvin.
PATCH 5/6 s390: Add four level page tables for CONFIG_64BIT=y.
In reply to Martin SchwidefskyFrom: Martin Schwidefsky <[email protected]>
Signed-off-by Martin Schwidefsky <[email protected]>
---
arch/s390/mm/init.c | 4 +- arch/s390/mm/vmem.c | 16 ++++++++++- include/asm-s390/a.out.h | 9 +++++- include/asm-s390/elf.h | 9 ------ include/asm-s390/mmu_context.h | 2 - include/asm-s390/pgalloc.h | 29 +++++++++++++++++--- include/asm-s390/pgtable.h | 57 ++++++++++++++++++++++++++++++++--------- include/asm-s390/processor.h | 11 ++----- include/asm-s390/tlb.h | 33 +++++++++++++++++------ 9 files changed, 122 insertions(+), 48 deletions(-)
arch/s390/mm/init.c +2 -2
@@ -112,8 +112,8 @@ void __init paging_init(void) init_mm.pgd = swapper_pg_dir; S390_lowcore.kernel_asce =__pa (init_mm.pgd) &PAGE_MASK ;#ifdef CONFIG_64BIT - S390_lowcore.kernel_asce |= _ASCE_TYPE_REGION3 | _ASCE_TABLE_LENGTH; - pgd_type = _REGION3_ENTRY_EMPTY; + S390_lowcore.kernel_asce |= _ASCE_TYPE_REGION2 | _ASCE_TABLE_LENGTH; + pgd_type = _REGION2_ENTRY_EMPTY;#else S390_lowcore.kernel_asce |= _ASCE_TABLE_LENGTH; pgd_type = _SEGMENT_ENTRY_EMPTY;
arch/s390/mm/vmem.c +14 -2
@@ -73,8 +73,6 @@ static void __init_refok *vmem_alloc_pagreturn alloc_bootmem_pages ((1 << order) *PAGE_SIZE ); } -#define vmem_pud_alloc () ({ BUG(); ((pud_t *) NULL); }) -static inline void *vmem_alloc_slab (size_t size) {if (slab_is_available ())
@@ -82,6 +80,20 @@ static inline void *vmem_alloc_slab(sizereturn alloc_bootmem (size); } +static inline pud_t *vmem_pud_alloc (void ) +{ +pud_t *pud =NULL ; + +#ifdef CONFIG_64BIT + pud =vmem_alloc_pages (2 ); +if (!pud) +return NULL ; +pud_val (*pud) = _REGION3_ENTRY_EMPTY; +memcpy (pud +1 , pud, (PTRS_PER_PUD -1 )*sizeof (pud_t)); +#endif +return pud; +} +static inline pmd_t *vmem_pmd_alloc (void ) {pmd_t *pmd =NULL ;
include/asm-s390/a.out.h +7 -2
@@ -31,8 +31,13 @@ struct exec#ifdef __KERNEL__ -#define STACK_TOP TASK_SIZE -#define STACK_TOP_MAX DEFAULT_TASK_SIZE +#ifndef __s390x__ +#define STACK_TOP (1UL << 31) +#else /* __s390x__ */ +#define STACK_TOP (1UL << (test_thread_flag(TIF_31BIT) ? 31:53)) +#endif /* __s390x__ */ + +#define STACK_TOP_MAX STACK_TOP#endif
include/asm-s390/elf.h +1 -8
@@ -139,14 +139,7 @@ typedef s390_regs elf_gregset_t;use of this is to invoke "./ld.so someprog" to test out a new version of the loader. We need to make sure that it is out of the way of the program that it will "exec", and that there is sufficient room for the brk. */ - -#ifndef __s390x__ -#define ELF_ET_DYN_BASE ((TASK_SIZE & 0x80000000) \ - ? TASK_SIZE / 3 * 2 \ - : 2 * TASK_SIZE / 3) -#else /* __s390x__ */ -#define ELF_ET_DYN_BASE (TASK_SIZE / 3 * 2) -#endif /* __s390x__ */ +#define ELF_ET_DYN_BASE (TASK_SIZE / 3 * 2)/* Wow, the "main" arch needs arch dependent functions too.. :) */
include/asm-s390/mmu_context.h +1 -1
@@ -35,7 +35,7 @@ static inline void update_mm(struct mm_s/* Calculate asce bits from the first pgd table entry. */ asce_bits = _ASCE_TABLE_LENGTH | _ASCE_USER_BITS;#ifdef CONFIG_64BIT - asce_bits |= _ASCE_TYPE_REGION3; + asce_bits |= _ASCE_TYPE_REGION2;#endif S390_lowcore.user_asce = asce_bits |__pa (pgd);if (switch_amode) {
include/asm-s390/pgalloc.h +24 -5
@@ -73,11 +73,17 @@ static inline unsigned long pgd_entry_tystatic inline unsigned long pgd_entry_type (struct mm_struct *mm) { -return _REGION3_ENTRY_EMPTY; +return _REGION2_ENTRY_EMPTY; } -#define pud_alloc_one (mm,address) ({ BUG(); ((pud_t *)2); }) -#define pud_free (mm, x) do { } while (0) +static inline pud_t *pud_alloc_one (struct mm_struct *mm,unsigned long address) +{ +unsigned long *table =crst_table_alloc (mm, mm->context.noexec); +if (table) +crst_table_init (table, _REGION3_ENTRY_EMPTY); +return (pud_t *) table; +} +#define pud_free (mm, pud) crst_table_free(mm, (unsigned long *) pud)static inline pmd_t *pmd_alloc_one (struct mm_struct *mm,unsigned long vmaddr) {
@@ -88,8 +94,21 @@ static inline pmd_t *pmd_alloc_one(struc }#define pmd_free (mm, pmd) crst_table_free(mm, (unsigned long *) pmd) -#define pgd_populate (mm, pgd, pud) BUG() -#define pgd_populate_kernel (mm, pgd, pud) BUG() +static inline void pgd_populate_kernel (struct mm_struct *mm, +pgd_t *pgd,pud_t *pud) +{ +pgd_val (*pgd) = _REGION2_ENTRY |__pa (pud); +} + +static inline void pgd_populate (struct mm_struct *mm,pgd_t *pgd,pud_t *pud) +{ +pgd_t *shadow_pgd =get_shadow_table (pgd); +pud_t *shadow_pud =get_shadow_table (pud); + +if (shadow_pgd && shadow_pud) +pgd_populate_kernel (mm, shadow_pgd, shadow_pud); +pgd_populate_kernel (mm, pgd, pud); +}static inline void pud_populate_kernel (struct mm_struct *mm,pud_t *pud,pmd_t *pmd)
include/asm-s390/pgtable.h +45 -12
103 lines not on this page · raw message
include/asm-s390/processor.h +4 -7
@@ -67,16 +67,13 @@ extern struct task_struct *last_task_use*/ #ifndef __s390x__ -# define TASK_SIZE (0x80000000UL) -# define TASK_UNMAPPED_BASE (TASK_SIZE / 2) -# define DEFAULT_TASK_SIZE (0x80000000UL) +#define TASK_SIZE (1UL << 31) +#define TASK_UNMAPPED_BASE (1UL << 30)#else /* __s390x__ */ -# define TASK_SIZE (test_thread_flag(TIF_31BIT) ? \ - (0x80000000UL) : (0x40000000000UL)) -# define TASK_UNMAPPED_BASE (TASK_SIZE / 2) -# define DEFAULT_TASK_SIZE (0x40000000000UL) +#define TASK_SIZE (1UL << (test_thread_flag(TIF_31BIT) ? 31:53)) +#define TASK_UNMAPPED_BASE (1UL << (test_thread_flag(TIF_31BIT) ? 30:41))#endif /* __s390x__ */
include/asm-s390/tlb.h +24 -9
@@ -38,7 +38,7 @@ struct mmu_gather {struct mm_struct *mm;unsigned int fullmm;unsigned int nr_ptes; -unsigned int nr_pmds; +unsigned int nr_pxds;void *array[TLB_NR_PTRS ]; };
@@ -53,7 +53,7 @@ static inline struct mmu_gather *tlb_gat tlb->fullmm = full_mm_flush || (num_online_cpus () ==1 ) || (atomic_read (&mm->mm_users) <=1 && mm == current->active_mm); tlb->nr_ptes =0 ; - tlb->nr_pmds =TLB_NR_PTRS ; + tlb->nr_pxds =TLB_NR_PTRS ;if (tlb->fullmm)__tlb_flush_mm (mm);return tlb;
@@ -62,12 +62,13 @@ static inline struct mmu_gather *tlb_gatstatic inline void tlb_flush_mmu (struct mmu_gather *tlb,unsigned long start,unsigned long end) { -if (!tlb->fullmm && (tlb->nr_ptes >0 || tlb->nr_pmds <TLB_NR_PTRS )) +if (!tlb->fullmm && (tlb->nr_ptes >0 || tlb->nr_pxds <TLB_NR_PTRS ))__tlb_flush_mm (tlb->mm);while (tlb->nr_ptes >0 )pte_free (tlb->mm, tlb->array[--tlb->nr_ptes]); -while (tlb->nr_pmds <TLB_NR_PTRS ) -pmd_free (tlb->mm, (pmd_t *) tlb->array[tlb->nr_pmds++]); +while (tlb->nr_pxds <TLB_NR_PTRS ) +/* pgd_free frees the pointer as region or segment table */ +pgd_free (tlb->mm, tlb->array[tlb->nr_pxds++]); } static inline void tlb_finish_mmu(struct mmu_gather *tlb,
@@ -99,7 +100,7 @@ static inline void pte_free_tlb(struct m {if (!tlb->fullmm) { tlb->array[tlb->nr_ptes++] = pte; -if (tlb->nr_ptes >= tlb->nr_pmds) +if (tlb->nr_ptes >= tlb->nr_pxds)tlb_flush_mmu (tlb,0 ,0 ); }else pte_free (tlb->mm, pte);
@@ -113,15 +114,29 @@ static inline void pmd_free_tlb(struct m {#ifdef __s390x__if (!tlb->fullmm) { - tlb->array[--tlb->nr_pmds] = (struct page *) pmd; -if (tlb->nr_ptes >= tlb->nr_pmds) + tlb->array[--tlb->nr_pxds] = pmd; +if (tlb->nr_ptes >= tlb->nr_pxds)tlb_flush_mmu (tlb,0 ,0 ); }else pmd_free (tlb->mm, pmd);#endif } -#define pud_free_tlb (tlb, pud) do { } while (0) +/* +* pud_free_tlb frees a pud table and clears the CRSTE for the +* region third table entry from the tlb. +*/ +static inline void pud_free_tlb (struct mmu_gather *tlb,pud_t *pud) +{ +#ifdef __s390x__ +if (!tlb->fullmm) { + tlb->array[--tlb->nr_pxds] = pud; +if (tlb->nr_ptes >= tlb->nr_pxds) +tlb_flush_mmu (tlb,0 ,0 ); + }else +pud_free (tlb->mm, pud); +#endif +}#define tlb_start_vma (tlb, vma) do { } while (0)#define tlb_end_vma (tlb, vma) do { } while (0)
Signature · 4 lines
blue skies, Martin. "Reality continues to ruin my life." - Calvin.
PATCH 6/6 s390: dynamic page tables.
In reply to Martin SchwidefskyFrom: Martin Schwidefsky <[email protected]>
Add support for different number of page table levels dependent on the highest address used for a process. This will cause a 31 bit process to use a two level page table instead of the four level page table that is the default after the pud has been introduced. Likewise a normal 64 bit process will use three levels instead of four. Only if a process runs out of the 4 tera bytes which can be addressed with a three level page table the fourth level is dynamically added. Then the process can use up to 8 peta byte.
To upgrade a page table to the next level the arch_update_pgd hook in get_unmapped_area is used.
Signed-off-by Martin Schwidefsky <[email protected]>
---
arch/s390/kernel/traps.c | 3 +- arch/s390/mm/fault.c | 40 ++++++++++++++++++++++++++++ arch/s390/mm/init.c | 5 ++- arch/s390/mm/mmap.c | 4 ++ arch/s390/mm/pgtable.c | 57 +++++++++++++++++++++++++++++++++++++++++ include/asm-s390/a.out.h | 2 - include/asm-s390/elf.h | 2 - include/asm-s390/mmu.h | 1 include/asm-s390/mmu_context.h | 10 +++++-- include/asm-s390/pgalloc.h | 47 +++++++++++++++++++++++++-------- include/asm-s390/pgtable.h | 24 +++++++++++++---- include/asm-s390/tlb.h | 10 +++++++ 12 files changed, 181 insertions(+), 24 deletions(-)
arch/s390/kernel/traps.c +2 -1
@@ -58,6 +58,7 @@ int sysctl_userprocess_debug = 0;extern pgm_check_handler_t do_protection_exception;extern pgm_check_handler_t do_dat_exception;extern pgm_check_handler_t do_monitor_call; +extern pgm_check_handler_t do_asce_exception;#define stack_pointer ({ void **sp; asm("la %0,0(15)" : "=&d" (sp)); sp; })
@@ -712,7 +713,7 @@ void __init trap_init(void) pgm_check_table[0x12 ] = &translation_exception; pgm_check_table[0x13 ] = &special_op_exception;#ifdef CONFIG_64BIT - pgm_check_table[0x38 ] = &do_dat_exception; + pgm_check_table[0x38 ] = &do_asce_exception; pgm_check_table[0x39 ] = &do_dat_exception; pgm_check_table[0x3A ] = &do_dat_exception; pgm_check_table[0x3B ] = &do_dat_exception;
arch/s390/mm/fault.c +40 -0
@@ -32,6 +32,7 @@#include <asm/system.h> #include <asm/pgtable.h> #include <asm/s390_ext.h> +#include <asm/mmu_context.h> #ifndef CONFIG_64BIT #define __FAIL_ADDR_MASK 0x7ffff000
@@ -444,6 +445,45 @@ void __kprobes do_dat_exception(struct pdo_exception (regs, error_code &0xff ,0 ); } +#ifdef CONFIG_64BIT +void __kprobes do_asce_exception (struct pt_regs *regs,unsigned long error_code) +{ +struct mm_struct *mm; +struct vm_area_struct *vma; +unsigned long address; +int space; + + mm = current->mm; + address = S390_lowcore.trans_exc_code & __FAIL_ADDR_MASK; + space =check_space (current); + +if (unlikely (space ==0 ||in_atomic () || !mm)) + goto no_context; + +local_irq_enable (); + +down_read (&mm->mmap_sem); + vma =find_vma (mm, address); +up_read (&mm->mmap_sem); + +if (vma) { +update_mm (mm, current); +return ; + } + +/* User mode accesses just cause a SIGSEGV */ +if (regs->psw.mask &PSW_MASK_PSTATE ) { + current->thread.prot_addr = address; + current->thread.trap_no = error_code; +do_sigsegv (regs, error_code,SEGV_MAPERR , address); +return ; + } + +no_context: +do_no_context (regs, error_code, address); +} +#endif +#ifdef CONFIG_PFAULT /* * 'pfault' pseudo page faults routines.
arch/s390/mm/init.c +3 -2
@@ -112,8 +112,9 @@ void __init paging_init(void) init_mm.pgd = swapper_pg_dir; S390_lowcore.kernel_asce =__pa (init_mm.pgd) &PAGE_MASK ;#ifdef CONFIG_64BIT - S390_lowcore.kernel_asce |= _ASCE_TYPE_REGION2 | _ASCE_TABLE_LENGTH; - pgd_type = _REGION2_ENTRY_EMPTY; +/* A three level page table (4TB) is enough for the kernel space. */ + S390_lowcore.kernel_asce |= _ASCE_TYPE_REGION3 | _ASCE_TABLE_LENGTH; + pgd_type = _REGION3_ENTRY_EMPTY;#else S390_lowcore.kernel_asce |= _ASCE_TABLE_LENGTH; pgd_type = _SEGMENT_ENTRY_EMPTY;
arch/s390/mm/mmap.c +4 -0
@@ -27,6 +27,7 @@#include <linux/personality.h> #include <linux/mm.h> #include <linux/module.h> +#include <asm/pgalloc.h> /* * Top of mmap area (just below the process stack).
@@ -81,6 +82,9 @@ void arch_pick_mmap_layout(struct mm_str mm->get_unmapped_area = arch_get_unmapped_area_topdown; mm->unmap_area = arch_unmap_area_topdown; } +#ifdef CONFIG_64BIT +pgd_downgrade (mm); +#endif }EXPORT_SYMBOL_GPL (arch_pick_mmap_layout);
arch/s390/mm/pgtable.c +57 -0
69 lines not on this page · raw message
include/asm-s390/a.out.h +1 -1
@@ -34,7 +34,7 @@ struct exec#ifndef __s390x__#define STACK_TOP (1UL << 31)#else /* __s390x__ */ -#define STACK_TOP (1UL << (test_thread_flag(TIF_31BIT) ? 31:53)) +#define STACK_TOP (1UL << (test_thread_flag(TIF_31BIT) ? 31:42))#endif /* __s390x__ */ #define STACK_TOP_MAX STACK_TOP
include/asm-s390/elf.h +1 -1
@@ -139,7 +139,7 @@ typedef s390_regs elf_gregset_t;use of this is to invoke "./ld.so someprog" to test out a new version of the loader. We need to make sure that it is out of the way of the program that it will "exec", and that there is sufficient room for the brk. */ -#define ELF_ET_DYN_BASE (TASK_SIZE / 3 * 2) +#define ELF_ET_DYN_BASE (STACK_TOP / 3 * 2)/* Wow, the "main" arch needs arch dependent functions too.. :) */
include/asm-s390/mmu_context.h +7 -3
@@ -9,6 +9,7 @@#ifndef __S390_MMU_CONTEXT_H#define __S390_MMU_CONTEXT_H +#include <asm/a.out.h> #include <asm/pgalloc.h> #include <asm/uaccess.h> #include <asm-generic/mm_hooks.h>
@@ -17,6 +18,9 @@ static inline int init_new_context(struc struct mm_struct *mm) { mm->context.noexec = s390_noexec; + mm->context.asce_limit = + current->mm ? current->mm->context.asce_limit :STACK_TOP ; +crst_table_init ((unsigned long *) mm->pgd,pgd_entry_type (mm));return (0 ); }#define destroy_context (mm) do { } while (0)
@@ -35,9 +39,9 @@ static inline void update_mm(struct mm_s/* Calculate asce bits from the first pgd table entry. */ asce_bits = _ASCE_TABLE_LENGTH | _ASCE_USER_BITS;#ifdef CONFIG_64BIT - asce_bits |= _ASCE_TYPE_REGION2; + asce_bits |= (pgd_val (*pgd) & _ASCE_TYPE_MASK);#endif - S390_lowcore.user_asce = asce_bits |__pa (pgd); + S390_lowcore.user_asce = asce_bits |__pa (mm->pgd);if (switch_amode) {/* Load primary space page table origin. */ pgd = mm->context.noexec ?get_shadow_table (pgd) : pgd;
@@ -48,6 +52,7 @@ static inline void update_mm(struct mm_s/* Load home space page table origin. */ asmvolatile (LCTL_OPCODE " 13,13,%0" : :"m" (S390_lowcore.user_asce) ); +set_fs (current->thread.mm_segment); } static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
@@ -66,7 +71,6 @@ static inline void activate_mm(struct mm struct mm_struct *next) {switch_mm (prev, next, current); -set_fs (current->thread.mm_segment); }#endif /* __S390_MMU_CONTEXT_H */
include/asm-s390/mmu.h +1 -0
@@ -4,6 +4,7 @@typedef struct {struct list_head crst_list;struct list_head pgtable_list; +unsigned long asce_limit;int noexec; }mm_context_t ;
include/asm-s390/pgalloc.h +36 -11
71 lines not on this page · raw message
include/asm-s390/pgtable.h +19 -5
@@ -428,11 +428,15 @@ static inline int pud_bad(pud_t pud) {static inline int pgd_present (pgd_t pgd) { +if ((pgd_val (pgd) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R2) +return 1 ;return pgd_val (pgd) & _REGION_ENTRY_ORIGIN; }static inline int pgd_none (pgd_t pgd) { +if ((pgd_val (pgd) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R2) +return 0 ;return pgd_val (pgd) & _REGION_ENTRY_INV; }
@@ -446,11 +450,15 @@ static inline int pgd_bad(pgd_t pgd)static inline int pud_present (pud_t pud) { +if ((pud_val (pud) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R3) +return 1 ;return pud_val (pud) & _REGION_ENTRY_ORIGIN; }static inline int pud_none (pud_t pud) { +if ((pud_val (pud) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R3) +return 0 ;return pud_val (pud) & _REGION_ENTRY_INV; }
@@ -542,7 +550,8 @@ static inline int pte_young(pte_t pte)static inline void pgd_clear_kernel (pgd_t * pgd) { -pgd_val (*pgd) = _REGION2_ENTRY_EMPTY; +if ((pgd_val (*pgd) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R2) +pgd_val (*pgd) = _REGION2_ENTRY_EMPTY; }static inline void pgd_clear (pgd_t * pgd)
@@ -556,10 +565,11 @@ static inline void pgd_clear(pgd_t * pgdstatic inline void pud_clear_kernel (pud_t *pud) { -pud_val (*pud) = _REGION3_ENTRY_EMPTY; +if ((pud_val (*pud) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) +pud_val (*pud) = _REGION3_ENTRY_EMPTY; } -static inline void pud_clear (pud_t * pud) +static inline void pud_clear (pud_t *pud) {pud_t *shadow =get_shadow_table (pud);
@@ -848,13 +858,17 @@ static inline pte_t mk_pte(struct page *static inline pud_t *pud_offset (pgd_t *pgd,unsigned long address) { -pud_t *pud = (pud_t *)pgd_deref (*pgd); +pud_t *pud = (pud_t *) pgd; +if ((pgd_val (*pgd) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R2) + pud = (pud_t *)pgd_deref (*pgd);return pud +pud_index (address); }static inline pmd_t *pmd_offset (pud_t *pud,unsigned long address) { -pmd_t *pmd = (pmd_t *)pud_deref (*pud); +pmd_t *pmd = (pmd_t *) pud; +if ((pud_val (*pud) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) + pmd = (pmd_t *)pud_deref (*pud);return pmd +pmd_index (address); }
include/asm-s390/tlb.h +10 -0
@@ -109,10 +109,15 @@ static inline void pte_free_tlb(struct m/* * pmd_free_tlb frees a pmd table and clears the CRSTE for the * segment table entry from the tlb. +* If the mm uses a two level page table the single pmd is freed +* as the pgd. pmd_free_tlb checks the asce_limit against 2GB +* to avoid the double free of the pmd in this case. */ static inline void pmd_free_tlb (struct mmu_gather *tlb,pmd_t *pmd) {#ifdef __s390x__ +if (tlb->mm->context.asce_limit <= (1UL <<31 )) +return ;if (!tlb->fullmm) { tlb->array[--tlb->nr_pxds] = pmd;if (tlb->nr_ptes >= tlb->nr_pxds)
@@ -125,10 +130,15 @@ static inline void pmd_free_tlb(struct m/* * pud_free_tlb frees a pud table and clears the CRSTE for the * region third table entry from the tlb. +* If the mm uses a three level page table the single pud is freed +* as the pgd. pud_free_tlb checks the asce_limit against 4TB +* to avoid the double free of the pud in this case. */ static inline void pud_free_tlb (struct mmu_gather *tlb,pud_t *pud) {#ifdef __s390x__ +if (tlb->mm->context.asce_limit <= (1UL <<42 )) +return ;if (!tlb->fullmm) { tlb->array[--tlb->nr_pxds] = pud;if (tlb->nr_ptes >= tlb->nr_pxds)
Signature · 4 lines
blue skies, Martin. "Reality continues to ruin my life." - Calvin.