powerpc: add support for folded p4d page tables
Subsystems (3)
PATCH 1/1 powerpc: add support for folded p4d page tables
From: Mike Rapoport <[email protected]>
Implement primitives necessary for the 4th level folding, add walks of p4d level where appropriate and replace 5level-fixup.h with pgtable-nop4d.h.
--- arch/powerpc/include/asm/book3s/32/pgtable.h | 1 - arch/powerpc/include/asm/book3s/64/hash.h | 4 +- arch/powerpc/include/asm/book3s/64/pgalloc.h | 4 +- arch/powerpc/include/asm/book3s/64/pgtable.h | 58 ++++++++++-------- arch/powerpc/include/asm/book3s/64/radix.h | 6 +- arch/powerpc/include/asm/nohash/32/pgtable.h | 1 - arch/powerpc/include/asm/nohash/64/pgalloc.h | 2 +- .../include/asm/nohash/64/pgtable-4k.h | 32 +++++----- arch/powerpc/include/asm/nohash/64/pgtable.h | 6 +- arch/powerpc/include/asm/pgtable.h | 8 +++ arch/powerpc/kvm/book3s_64_mmu_radix.c | 59 ++++++++++++++++--- arch/powerpc/lib/code-patching.c | 7 ++- arch/powerpc/mm/book3s32/mmu.c | 2 +- arch/powerpc/mm/book3s32/tlb.c | 4 +- arch/powerpc/mm/book3s64/hash_pgtable.c | 4 +- arch/powerpc/mm/book3s64/radix_pgtable.c | 19 ++++-- arch/powerpc/mm/book3s64/subpage_prot.c | 6 +- arch/powerpc/mm/hugetlbpage.c | 28 +++++---- arch/powerpc/mm/kasan/kasan_init_32.c | 8 +-- arch/powerpc/mm/mem.c | 4 +- arch/powerpc/mm/nohash/40x.c | 4 +- arch/powerpc/mm/nohash/book3e_pgtable.c | 15 +++-- arch/powerpc/mm/pgtable.c | 25 +++++++- arch/powerpc/mm/pgtable_32.c | 28 +++++---- arch/powerpc/mm/pgtable_64.c | 10 ++-- arch/powerpc/mm/ptdump/hashpagetable.c | 20 ++++++- arch/powerpc/mm/ptdump/ptdump.c | 22 ++++++- arch/powerpc/xmon/xmon.c | 17 +++++- 28 files changed, 284 insertions(+), 120 deletions(-)
arch/powerpc/include/asm/book3s/32/pgtable.h +0 -1
@@ -2,7 +2,6 @@#ifndef _ASM_POWERPC_BOOK3S_32_PGTABLE_H#define _ASM_POWERPC_BOOK3S_32_PGTABLE_H -#define __ARCH_USE_5LEVEL_HACK#include <asm-generic/pgtable-nopmd.h> #include <asm/book3s/32/hash.h>
arch/powerpc/include/asm/book3s/64/hash.h +2 -2
@@ -134,9 +134,9 @@ static inline int get_region_id(unsigned long ea)#define hash__pmd_bad (pmd) (pmd_val(pmd) & H_PMD_BAD_BITS)#define hash__pud_bad (pud) (pud_val(pud) & H_PUD_BAD_BITS) -static inline int hash__pgd_bad (pgd_t pgd) +static inline int hash__p4d_bad (p4d_t p4d) { -return (pgd_val (pgd) ==0 ); +return (p4d_val (p4d) ==0 ); }#ifdef CONFIG_STRICT_KERNEL_RWX extern void hash__mark_rodata_ro (void );
arch/powerpc/include/asm/book3s/64/pgalloc.h +2 -2
@@ -87,9 +87,9 @@ static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)kmem_cache_free (PGT_CACHE (PGD_INDEX_SIZE ), pgd); } -static inline void pgd_populate (struct mm_struct *mm,pgd_t *pgd,pud_t *pud) +static inline void p4d_populate (struct mm_struct *mm,p4d_t *pgd,pud_t *pud) { - *pgd =__pgd (__pgtable_ptr_val (pud) |PGD_VAL_BITS ); + *pgd =__p4d (__pgtable_ptr_val (pud) |PGD_VAL_BITS ); }static inline pud_t *pud_alloc_one (struct mm_struct *mm,unsigned long addr)
arch/powerpc/include/asm/book3s/64/pgtable.h +35 -23
117 lines not on this page · raw message
arch/powerpc/include/asm/book3s/64/radix.h +3 -3
@@ -30,7 +30,7 @@/* Don't have anything in the reserved bits and leaf bits */ #define RADIX_PMD_BAD_BITS 0x60000000000000e0UL#define RADIX_PUD_BAD_BITS 0x60000000000000e0UL -#define RADIX_PGD_BAD_BITS 0x60000000000000e0UL +#define RADIX_P4D_BAD_BITS 0x60000000000000e0UL#define RADIX_PMD_SHIFT (PAGE_SHIFT + RADIX_PTE_INDEX_SIZE)#define RADIX_PUD_SHIFT (RADIX_PMD_SHIFT + RADIX_PMD_INDEX_SIZE)
@@ -227,9 +227,9 @@ static inline int radix__pud_bad(pud_t pud) } -static inline int radix__pgd_bad (pgd_t pgd) +static inline int radix__p4d_bad (p4d_t p4d) { -return !!(pgd_val (pgd) &RADIX_PGD_BAD_BITS ); +return !!(p4d_val (p4d) &RADIX_P4D_BAD_BITS ); }#ifdef CONFIG_TRANSPARENT_HUGEPAGE
arch/powerpc/include/asm/nohash/32/pgtable.h +0 -1
@@ -2,7 +2,6 @@#ifndef _ASM_POWERPC_NOHASH_32_PGTABLE_H#define _ASM_POWERPC_NOHASH_32_PGTABLE_H -#define __ARCH_USE_5LEVEL_HACK#include <asm-generic/pgtable-nopmd.h> #ifndef __ASSEMBLY__
arch/powerpc/include/asm/nohash/64/pgalloc.h +1 -1
@@ -15,7 +15,7 @@ struct vmemmap_backing { };extern struct vmemmap_backing *vmemmap_list; -#define pgd_populate (MM ,PGD ,PUD ) pgd_set(PGD, (unsigned long)PUD) +#define p4d_populate (MM ,P4D ,PUD ) p4d_set(P4D, (unsigned long)PUD)static inline pud_t *pud_alloc_one (struct mm_struct *mm,unsigned long addr) {
arch/powerpc/include/asm/nohash/64/pgtable-4k.h +16 -16
@@ -2,7 +2,7 @@#ifndef _ASM_POWERPC_NOHASH_64_PGTABLE_4K_H#define _ASM_POWERPC_NOHASH_64_PGTABLE_4K_H -#include <asm-generic/5level-fixup.h> +#include <asm-generic/pgtable-nop4d.h> /* * Entries per page directory level. The PTE level must use a 64b record
@@ -45,41 +45,41 @@#define PMD_MASKED_BITS 0/* Bits to mask out from a PUD to get to the PMD page */ #define PUD_MASKED_BITS 0 -/* Bits to mask out from a PGD to get to the PUD page */ -#define PGD_MASKED_BITS 0 +/* Bits to mask out from a P4D to get to the PUD page */ +#define P4D_MASKED_BITS 0/* * 4-level page tables related bits */ -#define pgd_none (pgd) (!pgd_val(pgd)) -#define pgd_bad (pgd) (pgd_val(pgd) == 0) -#define pgd_present (pgd) (pgd_val(pgd) != 0) -#define pgd_page_vaddr (pgd) (pgd_val(pgd) & ~PGD_MASKED_BITS) +#define p4d_none (p4d) (!p4d_val(p4d)) +#define p4d_bad (p4d) (p4d_val(p4d) == 0) +#define p4d_present (p4d) (p4d_val(p4d) != 0) +#define p4d_page_vaddr (p4d) (p4d_val(p4d) & ~P4D_MASKED_BITS)#ifndef __ASSEMBLY__ -static inline void pgd_clear (pgd_t *pgdp) +static inline void p4d_clear (p4d_t *p4dp) { - *pgdp =__pgd (0 ); + *p4dp =__p4d (0 ); } -static inline pte_t pgd_pte (pgd_t pgd) +static inline pte_t p4d_pte (p4d_t p4d) { -return __pte (pgd_val (pgd)); +return __pte (p4d_val (p4d)); } -static inline pgd_t pte_pgd (pte_t pte) +static inline p4d_t pte_p4d (pte_t pte) { -return __pgd (pte_val (pte)); +return __p4d (pte_val (pte)); } -extern struct page *pgd_page (pgd_t pgd); +extern struct page *p4d_page (p4d_t p4d);#endif /* !__ASSEMBLY__ */ -#define pud_offset (pgdp, addr) \ - (((pud_t *) pgd_page_vaddr(*(pgdp))) + \ +#define pud_offset (p4dp, addr) \ + (((pud_t *) p4d_page_vaddr(*(p4dp))) + \ (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1)))#define pud_ERROR (e) \
arch/powerpc/include/asm/nohash/64/pgtable.h +3 -3
@@ -175,11 +175,11 @@ static inline pud_t pte_pud(pte_t pte)return __pud (pte_val (pte)); }#define pud_write (pud) pte_write(pud_pte(pud)) -#define pgd_write (pgd) pte_write(pgd_pte(pgd)) +#define p4d_write (pgd) pte_write(p4d_pte(p4d)) -static inline void pgd_set (pgd_t *pgdp,unsigned long val) +static inline void p4d_set (p4d_t *p4dp,unsigned long val) { - *pgdp =__pgd (val); + *p4dp =__p4d (val); }/*
arch/powerpc/include/asm/pgtable.h +8 -0
@@ -145,6 +145,14 @@ static inline bool pud_is_leaf(pud_t pud) }#endif +#ifndef p4d_is_leaf +#define p4d_is_leaf p4d_is_leaf +static inline bool p4d_is_leaf (p4d_t p4d) +{ +return false; +} +#endif +#ifndef pgd_is_leaf#define pgd_is_leaf pgd_is_leafstatic inline bool pgd_is_leaf (pgd_t pgd)
arch/powerpc/kvm/book3s_64_mmu_radix.c +51 -8
108 lines not on this page · raw message
arch/powerpc/lib/code-patching.c +6 -1
@@ -107,13 +107,18 @@ static inline int unmap_patch_area(unsigned long addr)pte_t *ptep;pmd_t *pmdp;pud_t *pudp; +p4d_t *p4dp;pgd_t *pgdp; pgdp =pgd_offset_k (addr);if (unlikely (!pgdp))return -EINVAL ; - pudp =pud_offset (pgdp, addr); + p4dp =p4d_offset (pgdp, addr); +if (unlikely (!p4dp)) +return -EINVAL ; + + pudp =pud_offset (p4dp, addr);if (unlikely (!pudp))return -EINVAL ;
arch/powerpc/mm/book3s32/mmu.c +1 -1
@@ -312,7 +312,7 @@ void hash_preload(struct mm_struct *mm, unsigned long ea)if (!Hash)return ; - pmd =pmd_offset (pud_offset (pgd_offset (mm, ea), ea), ea); + pmd =pmd_offset (pud_offset (p4d_offset (pgd_offset (mm, ea), ea), ea), ea);if (!pmd_none (*pmd))add_hash_page (mm->context.id, ea,pmd_val (*pmd)); }
arch/powerpc/mm/book3s32/tlb.c +2 -2
@@ -87,7 +87,7 @@ static void flush_range(struct mm_struct *mm, unsigned long start,if (start >= end)return ; end = (end -1 ) | ~PAGE_MASK ; - pmd =pmd_offset (pud_offset (pgd_offset (mm, start), start), start); + pmd =pmd_offset (pud_offset (p4d_offset (pgd_offset (mm, start), start), start), start);for (;;) { pmd_end = ((start +PGDIR_SIZE ) &PGDIR_MASK ) -1 ;if (pmd_end > end)
@@ -145,7 +145,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)return ; } mm = (vmaddr <TASK_SIZE )? vma->vm_mm: &init_mm; - pmd =pmd_offset (pud_offset (pgd_offset (mm, vmaddr), vmaddr), vmaddr); + pmd =pmd_offset (pud_offset (p4d_offset (pgd_offset (mm, vmaddr), vmaddr), vmaddr), vmaddr);if (!pmd_none (*pmd))flush_hash_pages (mm->context.id, vmaddr,pmd_val (*pmd),1 ); }
arch/powerpc/mm/book3s64/hash_pgtable.c +3 -1
@@ -148,6 +148,7 @@ void hash__vmemmap_remove_mapping(unsigned long start,int hash__map_kernel_page (unsigned long ea,unsigned long pa,pgprot_t prot) {pgd_t *pgdp; +p4d_t *p4dp;pud_t *pudp;pmd_t *pmdp;pte_t *ptep;
@@ -155,7 +156,8 @@ int hash__map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot)BUILD_BUG_ON (TASK_SIZE_USER64 >H_PGTABLE_RANGE );if (slab_is_available ()) { pgdp =pgd_offset_k (ea); - pudp =pud_alloc (&init_mm, pgdp, ea); + p4dp =p4d_offset (pgdp, ea); + pudp =pud_alloc (&init_mm, p4dp, ea);if (!pudp)return -ENOMEM ; pmdp =pmd_alloc (&init_mm, pudp, ea);
arch/powerpc/mm/book3s64/radix_pgtable.c +15 -4
@@ -64,17 +64,24 @@ static int early_map_kernel_page(unsigned long ea, unsigned long pa, {unsigned long pfn = pa >>PAGE_SHIFT ;pgd_t *pgdp; +p4d_t *p4dp;pud_t *pudp;pmd_t *pmdp;pte_t *ptep; pgdp =pgd_offset_k (ea);if (pgd_none (*pgdp)) { + p4dp =early_alloc_pgtable (PGD_TABLE_SIZE , nid, + region_start, region_end); +pgd_populate (&init_mm, pgdp, p4dp); + } + p4dp =p4d_offset (pgdp, ea); +if (p4d_none (*p4dp)) { pudp =early_alloc_pgtable (PUD_TABLE_SIZE , nid, region_start, region_end); -pgd_populate (&init_mm, pgdp, pudp); +p4d_populate (&init_mm, p4dp, pudp); } - pudp =pud_offset (pgdp, ea); + pudp =pud_offset (p4dp, ea);if (map_page_size ==PUD_SIZE ) { ptep = (pte_t *)pudp; goto set_the_pte;
@@ -114,6 +121,7 @@ static int __map_kernel_page(unsigned long ea, unsigned long pa, {unsigned long pfn = pa >>PAGE_SHIFT ;pgd_t *pgdp; +p4d_t *p4dp;pud_t *pudp;pmd_t *pmdp;pte_t *ptep;
@@ -136,7 +144,8 @@ static int __map_kernel_page(unsigned long ea, unsigned long pa,* boot. */ pgdp =pgd_offset_k (ea); - pudp =pud_alloc (&init_mm, pgdp, ea); + p4dp =p4d_offset (pgdp, ea); + pudp =pud_alloc (&init_mm, p4dp, ea);if (!pudp)return -ENOMEM ;if (map_page_size ==PUD_SIZE ) {
@@ -173,6 +182,7 @@ void radix__change_memory_range(unsigned long start, unsigned long end, {unsigned long idx;pgd_t *pgdp; +p4d_t *p4dp;pud_t *pudp;pmd_t *pmdp;pte_t *ptep;
@@ -185,7 +195,8 @@ void radix__change_memory_range(unsigned long start, unsigned long end,for (idx = start; idx < end; idx +=PAGE_SIZE ) { pgdp =pgd_offset_k (idx); - pudp =pud_alloc (&init_mm, pgdp, idx); + p4dp =p4d_offset (pgdp, idx); + pudp =pud_alloc (&init_mm, p4dp, idx);if (!pudp)continue ;if (pud_is_leaf (*pudp)) {
arch/powerpc/mm/book3s64/subpage_prot.c +5 -1
@@ -54,6 +54,7 @@ static void hpte_flush_range(struct mm_struct *mm, unsigned long addr, int npages) {pgd_t *pgd; +p4d_t *p4d;pud_t *pud;pmd_t *pmd;pte_t *pte;
@@ -62,7 +63,10 @@ static void hpte_flush_range(struct mm_struct *mm, unsigned long addr, pgd =pgd_offset (mm, addr);if (pgd_none (*pgd))return ; - pud =pud_offset (pgd, addr); + p4d =p4d_offset (pgd, addr); +if (p4d_none (*p4d)) +return ; + pud =pud_offset (p4d, addr);if (pud_none (*pud))return ; pmd =pmd_offset (pud, addr);
arch/powerpc/mm/hugetlbpage.c +16 -12
@@ -112,6 +112,7 @@ static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp,pte_t *huge_pte_alloc (struct mm_struct *mm,unsigned long addr,unsigned long sz) {pgd_t *pg; +p4d_t *p4;pud_t *pu;pmd_t *pm;hugepd_t *hpdp =NULL ;
@@ -121,20 +122,21 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz addr &= ~(sz-1 ); pg =pgd_offset (mm, addr); + p4 =p4d_offset (pg, addr);#ifdef CONFIG_PPC_BOOK3S_64 if (pshift ==PGDIR_SHIFT )/* 16GB huge page */ -return (pte_t *) pg; +return (pte_t *) p4;else if (pshift >PUD_SHIFT ) {/* * We need to use hugepd table */ ptl = &mm->page_table_lock; - hpdp = (hugepd_t *)pg; + hpdp = (hugepd_t *)p4; }else { pdshift =PUD_SHIFT ; - pu =pud_alloc (mm, pg, addr); + pu =pud_alloc (mm, p4, addr);if (!pu)return NULL ;if (pshift ==PUD_SHIFT )
@@ -159,10 +161,10 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz#else if (pshift >=PGDIR_SHIFT ) { ptl = &mm->page_table_lock; - hpdp = (hugepd_t *)pg; + hpdp = (hugepd_t *)p4; }else { pdshift =PUD_SHIFT ; - pu =pud_alloc (mm, pg, addr); + pu =pud_alloc (mm, p4, addr);if (!pu)return NULL ;if (pshift >=PUD_SHIFT ) {
@@ -384,7 +386,7 @@ static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud,mm_dec_nr_pmds (tlb->mm); } -static void hugetlb_free_pud_range (struct mmu_gather *tlb,pgd_t *pgd, +static void hugetlb_free_pud_range (struct mmu_gather *tlb,p4d_t *p4d,unsigned long addr,unsigned long end,unsigned long floor,unsigned long ceiling) {
@@ -394,7 +396,7 @@ static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd, start = addr; do { - pud = pud_offset(pgd, addr); + pud = pud_offset(p4d, addr); next = pud_addr_end(addr, end); if (!is_hugepd(__hugepd(pud_val(*pud)))) { if (pud_none_or_clear_bad(pud))
@@ -429,8 +431,8 @@ static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,if (end -1 > ceiling -1 )return ; - pud =pud_offset (pgd, start); -pgd_clear (pgd); + pud =pud_offset (p4d, start); +p4d_clear (p4d);pud_free_tlb (tlb, pud, start);mm_dec_nr_puds (tlb->mm); }
@@ -443,6 +445,7 @@ void hugetlb_free_pgd_range(struct mmu_gather *tlb, unsigned long floor, unsigned long ceiling) {pgd_t *pgd; +p4d_t *p4d;unsigned long next;/*
@@ -465,10 +468,11 @@ void hugetlb_free_pgd_range(struct mmu_gather *tlb, do { next = pgd_addr_end(addr, end); pgd = pgd_offset(tlb->mm, addr); + p4d = p4d_offset(pgd, addr); if (!is_hugepd(__hugepd(pgd_val(*pgd)))) { - if (pgd_none_or_clear_bad(pgd)) + if (p4d_none_or_clear_bad(p4d)) continue; - hugetlb_free_pud_range(tlb, pgd, addr, next, floor, ceiling); + hugetlb_free_pud_range(tlb, p4d, addr, next, floor, ceiling); } else { unsigned long more; /*
@@ -481,7 +485,7 @@ void hugetlb_free_pgd_range(struct mmu_gather *tlb,if (more > next) next = more; -free_hugepd_range (tlb, (hugepd_t *)pgd,PGDIR_SHIFT , +free_hugepd_range (tlb, (hugepd_t *)p4d,PGDIR_SHIFT , addr, next, floor, ceiling); } }while (addr = next, addr != end);
arch/powerpc/mm/kasan/kasan_init_32.c +4 -4
@@ -36,7 +36,7 @@ static int __ref kasan_init_shadow_page_tables(unsigned long k_start, unsigned lunsigned long k_cur, k_next;pgprot_t prot =slab_is_available () ?kasan_prot_ro () :PAGE_KERNEL ; - pmd =pmd_offset (pud_offset (pgd_offset_k (k_start), k_start), k_start); + pmd =pmd_offset (pud_offset (p4d_offset (pgd_offset_k (k_start), k_start), k_start), k_start);for (k_cur = k_start; k_cur != k_end; k_cur = k_next, pmd++) {pte_t *new;
@@ -94,7 +94,7 @@ static int __ref kasan_init_region(void *start, size_t size) block =memblock_alloc (k_end - k_start,PAGE_SIZE );for (k_cur = k_start &PAGE_MASK ; k_cur < k_end; k_cur +=PAGE_SIZE ) { -pmd_t *pmd =pmd_offset (pud_offset (pgd_offset_k (k_cur), k_cur), k_cur); +pmd_t *pmd =pmd_offset (pud_offset (p4d_offset (pgd_offset_k (k_cur), k_cur), k_cur), k_cur);void *va = block ? block + k_cur - k_start :kasan_get_one_page ();pte_t pte =pfn_pte (PHYS_PFN (__pa (va)),PAGE_KERNEL );
@@ -118,7 +118,7 @@ static void __init kasan_remap_early_shadow_ro(void)kasan_populate_pte (kasan_early_shadow_pte, prot);for (k_cur = k_start &PAGE_MASK ; k_cur < k_end; k_cur +=PAGE_SIZE ) { -pmd_t *pmd =pmd_offset (pud_offset (pgd_offset_k (k_cur), k_cur), k_cur); +pmd_t *pmd =pmd_offset (pud_offset (p4d_offset (pgd_offset_k (k_cur), k_cur), k_cur), k_cur);pte_t *ptep =pte_offset_kernel (pmd, k_cur);if ((pte_val (*ptep) &PTE_RPN_MASK ) != pa)
@@ -205,7 +205,7 @@ void __init kasan_early_init(void)unsigned long addr =KASAN_SHADOW_START ;unsigned long end =KASAN_SHADOW_END ;unsigned long next; -pmd_t *pmd =pmd_offset (pud_offset (pgd_offset_k (addr), addr), addr); +pmd_t *pmd =pmd_offset (pud_offset (p4d_offset (pgd_offset_k (addr), addr), addr), addr);BUILD_BUG_ON (KASAN_SHADOW_START & ~PGDIR_MASK );
arch/powerpc/mm/mem.c +2 -2
@@ -68,8 +68,8 @@ EXPORT_SYMBOL(kmap_prot);static inline pte_t *virt_to_kpte (unsigned long vaddr) { -return pte_offset_kernel (pmd_offset (pud_offset (pgd_offset_k (vaddr), - vaddr), vaddr), vaddr); +return pte_offset_kernel (pmd_offset (pud_offset (p4d_offset (pgd_offset_k (vaddr), + vaddr), vaddr), vaddr), vaddr); }#endif
arch/powerpc/mm/nohash/40x.c +2 -2
@@ -104,7 +104,7 @@ unsigned long __init mmu_mapin_ram(unsigned long base, unsigned long top)pmd_t *pmdp;unsigned long val = p | _PMD_SIZE_16M | _PAGE_EXEC | _PAGE_HWWRITE; - pmdp =pmd_offset (pud_offset (pgd_offset_k (v), v), v); + pmdp =pmd_offset (pud_offset (p4d_offset (pgd_offset_k (v), v), v), v); *pmdp++ =__pmd (val); *pmdp++ =__pmd (val); *pmdp++ =__pmd (val);
@@ -119,7 +119,7 @@ unsigned long __init mmu_mapin_ram(unsigned long base, unsigned long top)pmd_t *pmdp;unsigned long val = p | _PMD_SIZE_4M | _PAGE_EXEC | _PAGE_HWWRITE; - pmdp =pmd_offset (pud_offset (pgd_offset_k (v), v), v); + pmdp =pmd_offset (pud_offset (p4d_offset (pgd_offset_k (v), v), v), v); *pmdp =__pmd (val); v +=LARGE_PAGE_SIZE_4M ;
arch/powerpc/mm/nohash/book3e_pgtable.c +11 -4
@@ -73,6 +73,7 @@ static void __init *early_alloc_pgtable(unsigned long size)int __ref map_kernel_page (unsigned long ea,unsigned long pa,pgprot_t prot) {pgd_t *pgdp; +p4d_t *p4dp;pud_t *pudp;pmd_t *pmdp;pte_t *ptep;
@@ -80,7 +81,10 @@ int __ref map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot)BUILD_BUG_ON (TASK_SIZE_USER64 >PGTABLE_RANGE );if (slab_is_available ()) { pgdp =pgd_offset_k (ea); - pudp =pud_alloc (&init_mm, pgdp, ea); + p4dp =p4d_alloc (&init_mm, pgdp, ea); +if (!p4dp) +return -ENOMEM ; + pudp =pud_alloc (&init_mm, p4dp, ea);if (!pudp)return -ENOMEM ; pmdp =pmd_alloc (&init_mm, pudp, ea);
@@ -91,13 +95,16 @@ int __ref map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot)return -ENOMEM ; }else { pgdp =pgd_offset_k (ea); -#ifndef __PAGETABLE_PUD_FOLDEDif (pgd_none (*pgdp)) { pudp =early_alloc_pgtable (PUD_TABLE_SIZE );pgd_populate (&init_mm, pgdp, pudp); } -#endif /* !__PAGETABLE_PUD_FOLDED */ - pudp =pud_offset (pgdp, ea); + p4dp =p4d_offset (pgdp, ea); +if (p4d_none (*p4dp)) { + pmdp =early_alloc_pgtable (PMD_TABLE_SIZE ); +p4d_populate (&init_mm, p4dp, pmdp); + } + pudp =pud_offset (p4dp, ea);if (pud_none (*pudp)) { pmdp =early_alloc_pgtable (PMD_TABLE_SIZE );pud_populate (&init_mm, pudp, pmdp);
arch/powerpc/mm/pgtable.c +23 -2
@@ -265,6 +265,7 @@ int huge_ptep_set_access_flags(struct vm_area_struct *vma,void assert_pte_locked (struct mm_struct *mm,unsigned long addr) {pgd_t *pgd; +p4d_t *p4d;pud_t *pud;pmd_t *pmd;
@@ -272,7 +273,9 @@ void assert_pte_locked(struct mm_struct *mm, unsigned long addr)return ; pgd = mm->pgd +pgd_index (addr);BUG_ON (pgd_none (*pgd)); - pud =pud_offset (pgd, addr); + p4d =p4d_offset (pgd, addr); +BUG_ON (p4d_none (*p4d)); + pud =pud_offset (p4d, addr);BUG_ON (pud_none (*pud)); pmd =pmd_offset (pud, addr);/*
@@ -313,6 +316,7 @@ pte_t *__find_linux_pte(pgd_t *pgdir, unsigned long ea, bool *is_thp, unsigned *hpage_shift) { pgd_t pgd, *pgdp; +p4d_t p4d, *p4dp;pud_t pud, *pudp;pmd_t pmd, *pmdp;pte_t *ret_pte;
@@ -346,13 +350,30 @@ pte_t *__find_linux_pte(pgd_t *pgdir, unsigned long ea, goto out_huge; } + pdshift =P4D_SHIFT ; + p4dp =p4d_offset (&pgd, ea); + p4d =READ_ONCE (*p4dp); + +if (p4d_none (p4d)) +return NULL ; + +if (p4d_is_leaf (p4d)) { + ret_pte = (pte_t *)p4dp; + goto out; + } + +if (is_hugepd (__hugepd (p4d_val (p4d)))) { + hpdp = (hugepd_t *)&p4d; + goto out_huge; + } +/* * Even if we end up with an unmap, the pgtable will not * be freed, because we do an rcu free and here we are * irq disabled */ pdshift =PUD_SHIFT ; - pudp =pud_offset (&pgd, ea); + pudp =pud_offset (&p4d, ea); pud =READ_ONCE (*pudp);if (pud_none (pud))
arch/powerpc/mm/pgtable_32.c +16 -12
@@ -63,7 +63,7 @@ int __ref map_kernel_page(unsigned long va, phys_addr_t pa, pgprot_t prot)int err = -ENOMEM ;/* Use upper 10 bits of VA to index the first level map */ - pd =pmd_offset (pud_offset (pgd_offset_k (va), va), va); + pd =pmd_offset (pud_offset (p4d_offset (pgd_offset_k (va), va), va), va);/* Use middle 10 bits of VA to index the second-level map */ if (likely (slab_is_available ())) pg =pte_alloc_kernel (pd, va);
@@ -130,6 +130,7 @@ static int get_pteptr(struct mm_struct *mm, unsigned long addr, pte_t **ptep, pmd_t **pmdp) { pgd_t *pgd; +p4d_t *p4d;pud_t *pud;pmd_t *pmd;pte_t *pte;
@@ -137,17 +138,20 @@ get_pteptr(struct mm_struct *mm, unsigned long addr, pte_t **ptep, pmd_t **pmdp) pgd =pgd_offset (mm, addr &PAGE_MASK );if (pgd) { - pud =pud_offset (pgd, addr &PAGE_MASK ); -if (pud &&pud_present (*pud)) { - pmd =pmd_offset (pud, addr &PAGE_MASK ); -if (pmd_present (*pmd)) { - pte =pte_offset_map (pmd, addr &PAGE_MASK ); -if (pte) { - retval =1 ; - *ptep = pte; -if (pmdp) - *pmdp = pmd; -/* XXX caller needs to do pte_unmap, yuck */ + p4d =p4d_offset (pgd, addr &PAGE_MASK ); +if (p4d &&p4d_present (*p4d)) { + pud =pud_offset (p4d, addr &PAGE_MASK ); +if (pud &&pud_present (*pud)) { + pmd =pmd_offset (pud, addr &PAGE_MASK ); +if (pmd_present (*pmd)) { + pte =pte_offset_map (pmd, addr &PAGE_MASK ); +if (pte) { + retval =1 ; + *ptep = pte; +if (pmdp) + *pmdp = pmd; +/* XXX caller needs to do pte_unmap, yuck */ + } } } }
arch/powerpc/mm/pgtable_64.c +5 -5
@@ -101,13 +101,13 @@ EXPORT_SYMBOL(__pte_frag_size_shift);#ifndef __PAGETABLE_PUD_FOLDED/* 4 level page table */ -struct page *pgd_page (pgd_t pgd) +struct page *p4d_page (p4d_t p4d) { -if (pgd_is_leaf (pgd)) { -VM_WARN_ON (!pgd_huge (pgd)); -return pte_page (pgd_pte (pgd)); +if (p4d_is_leaf (p4d)) { +VM_WARN_ON (!p4d_huge (p4d)); +return pte_page (p4d_pte (p4d)); } -return virt_to_page (pgd_page_vaddr (pgd)); +return virt_to_page (p4d_page_vaddr (p4d)); }#endif
arch/powerpc/mm/ptdump/hashpagetable.c +17 -3
@@ -417,9 +417,9 @@ static void walk_pmd(struct pg_state *st, pud_t *pud, unsigned long start) } } -static void walk_pud (struct pg_state *st,pgd_t *pgd,unsigned long start) +static void walk_pud (struct pg_state *st,p4d_t *p4d,unsigned long start) { -pud_t *pud =pud_offset (pgd,0 ); +pud_t *pud =pud_offset (p4d,0 );unsigned long addr;unsigned int i;
@@ -431,6 +431,20 @@ static void walk_pud(struct pg_state *st, pgd_t *pgd, unsigned long start) } } +static void walk_p4d (struct pg_state *st,pgd_t *pgd,unsigned long start) +{ +p4d_t *p4d =p4d_offset (pgd,0 ); +unsigned long addr; +unsigned int i; + +for (i =0 ; i <PTRS_PER_P4D ; i++, p4d++) { + addr = start + i *P4D_SIZE ; +if (!p4d_none (*p4d)) +/* p4d exists */ +walk_pud (st, p4d, addr); + } +} +static void walk_pagetables (struct pg_state *st) {pgd_t *pgd =pgd_offset_k (0UL );
@@ -445,7 +459,7 @@ static void walk_pagetables(struct pg_state *st) addr =KERN_VIRT_START + i *PGDIR_SIZE ;if (!pgd_none (*pgd))/* pgd exists */ -walk_pud (st, pgd, addr); +walk_p4d (st, pgd, addr); } }
arch/powerpc/mm/ptdump/ptdump.c +19 -3
@@ -273,9 +273,9 @@ static void walk_pmd(struct pg_state *st, pud_t *pud, unsigned long start) } } -static void walk_pud (struct pg_state *st,pgd_t *pgd,unsigned long start) +static void walk_pud (struct pg_state *st,p4d_t *p4d,unsigned long start) { -pud_t *pud =pud_offset (pgd,0 ); +pud_t *pud =pud_offset (p4d,0 );unsigned long addr;unsigned int i;
@@ -289,6 +289,22 @@ static void walk_pud(struct pg_state *st, pgd_t *pgd, unsigned long start) } } +static void walk_p4d (struct pg_state *st,pgd_t *pgd,unsigned long start) +{ +p4d_t *p4d =p4d_offset (pgd,0 ); +unsigned long addr; +unsigned int i; + +for (i =0 ; i <PTRS_PER_P4D ; i++, p4d++) { + addr = start + i *P4D_SIZE ; +if (!p4d_none (*p4d) && !p4d_is_leaf (*p4d)) +/* p4d exists */ +walk_pud (st, p4d, addr); +else +note_page (st, addr,2 ,p4d_val (*p4d)); + } +} +static void walk_pagetables (struct pg_state *st) {unsigned int i;
@@ -302,7 +318,7 @@ static void walk_pagetables(struct pg_state *st)for (i =pgd_index (addr); i <PTRS_PER_PGD ; i++, pgd++, addr +=PGDIR_SIZE ) {if (!pgd_none (*pgd) && !pgd_is_leaf (*pgd))/* pgd exists */ -walk_pud (st, pgd, addr); +walk_p4d (st, pgd, addr);else note_page (st, addr,1 ,pgd_val (*pgd)); }
arch/powerpc/xmon/xmon.c +16 -1
@@ -3128,6 +3128,7 @@ static void show_pte(unsigned long addr)struct task_struct *tsk =NULL ;struct mm_struct *mm;pgd_t *pgdp, *pgdir; +p4d_t *p4dp;pud_t *pudp;pmd_t *pmdp;pte_t *ptep;
@@ -3172,7 +3173,21 @@ static void show_pte(unsigned long addr) }printf ("pgdp @ 0x%px = 0x%016lx\n" , pgdp,pgd_val (*pgdp)); - pudp =pud_offset (pgdp, addr); + p4dp =p4d_offset (pgdp, addr); + +if (p4d_none (*p4dp)) { +printf ("No valid P4D\n" ); +return ; + } + +if (p4d_is_leaf (*p4dp)) { +format_pte (p4dp,p4d_val (*p4dp)); +return ; + } + +printf ("p4dp @ 0x%px = 0x%016lx\n" , p4dp,p4d_val (*p4dp)); + + pudp =pud_offset (p4dp, addr);if (pud_none (*pudp)) {printf ("No valid PUD\n" );
Signature · 1 line
2.24.0
Le 09/12/2019 à 16:09, Mike Rapoport a écrit :
Quoted 6 lines
From: Mike Rapoport <[email protected]>
Implement primitives necessary for the 4th level folding, add walks of p4d level where appropriate and replace
5level-fixup.hwithpgtable-nop4d.h.Signed-off-by Mike Rapoport <[email protected]>
Tested on 8xx and 83xx, no problem observed (ie book3s/32 and nohash/32)
Christophe
Quoted 1,149 lines
--- arch/powerpc/include/asm/book3s/32/pgtable.h | 1 - arch/powerpc/include/asm/book3s/64/hash.h | 4 +- arch/powerpc/include/asm/book3s/64/pgalloc.h | 4 +- arch/powerpc/include/asm/book3s/64/pgtable.h | 58 ++++++++++-------- arch/powerpc/include/asm/book3s/64/radix.h | 6 +- arch/powerpc/include/asm/nohash/32/pgtable.h | 1 - arch/powerpc/include/asm/nohash/64/pgalloc.h | 2 +- .../include/asm/nohash/64/pgtable-4k.h | 32 +++++----- arch/powerpc/include/asm/nohash/64/pgtable.h | 6 +- arch/powerpc/include/asm/pgtable.h | 8 +++ arch/powerpc/kvm/book3s_64_mmu_radix.c | 59 ++++++++++++++++--- arch/powerpc/lib/code-patching.c | 7 ++- arch/powerpc/mm/book3s32/mmu.c | 2 +- arch/powerpc/mm/book3s32/tlb.c | 4 +- arch/powerpc/mm/book3s64/hash_pgtable.c | 4 +- arch/powerpc/mm/book3s64/radix_pgtable.c | 19 ++++-- arch/powerpc/mm/book3s64/subpage_prot.c | 6 +- arch/powerpc/mm/hugetlbpage.c | 28 +++++---- arch/powerpc/mm/kasan/kasan_init_32.c | 8 +-- arch/powerpc/mm/mem.c | 4 +- arch/powerpc/mm/nohash/40x.c | 4 +- arch/powerpc/mm/nohash/book3e_pgtable.c | 15 +++-- arch/powerpc/mm/pgtable.c | 25 +++++++- arch/powerpc/mm/pgtable_32.c | 28 +++++---- arch/powerpc/mm/pgtable_64.c | 10 ++-- arch/powerpc/mm/ptdump/hashpagetable.c | 20 ++++++- arch/powerpc/mm/ptdump/ptdump.c | 22 ++++++- arch/powerpc/xmon/xmon.c | 17 +++++- 28 files changed, 284 insertions(+), 120 deletions(-)arch/powerpc/include/asm/book3s/32/pgtable.h +0
-1@@ -2,7 +2,6 @@#ifndef _ASM_POWERPC_BOOK3S_32_PGTABLE_H#define _ASM_POWERPC_BOOK3S_32_PGTABLE_H -#define __ARCH_USE_5LEVEL_HACK#include <asm-generic/pgtable-nopmd.h> #include <asm/book3s/32/hash.h> arch/powerpc/include/asm/book3s/64/hash.h +2
-2@@ -134,9 +134,9 @@ static inline int get_region_id(unsigned long ea)#define hash__pmd_bad (pmd) (pmd_val(pmd) & H_PMD_BAD_BITS)#define hash__pud_bad (pud) (pud_val(pud) & H_PUD_BAD_BITS) -static inline int hash__pgd_bad (pgd_t pgd) +static inline int hash__p4d_bad (p4d_t p4d) { -return (pgd_val (pgd) ==0 ); +return (p4d_val (p4d) ==0 ); }#ifdef CONFIG_STRICT_KERNEL_RWX extern void hash__mark_rodata_ro (void );arch/powerpc/include/asm/book3s/64/pgalloc.h +2
-2@@ -87,9 +87,9 @@ static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)kmem_cache_free (PGT_CACHE (PGD_INDEX_SIZE ), pgd); } -static inline void pgd_populate (struct mm_struct *mm,pgd_t *pgd,pud_t *pud) +static inline void p4d_populate (struct mm_struct *mm,p4d_t *pgd,pud_t *pud) { - *pgd =__pgd (__pgtable_ptr_val (pud) |PGD_VAL_BITS ); + *pgd =__p4d (__pgtable_ptr_val (pud) |PGD_VAL_BITS ); }static inline pud_t *pud_alloc_one (struct mm_struct *mm,unsigned long addr)arch/powerpc/include/asm/book3s/64/pgtable.h +35
-23@@ -2,7 +2,7 @@#ifndef _ASM_POWERPC_BOOK3S_64_PGTABLE_H_#define _ASM_POWERPC_BOOK3S_64_PGTABLE_H_ -#include <asm-generic/5level-fixup.h> +#include <asm-generic/pgtable-nop4d.h> #ifndef __ASSEMBLY__#include <linux/mmdebug.h> @@ -251,7 +251,7 @@ extern unsigned long __pmd_frag_size_shift;/* Bits to mask out from a PUD to get to the PMD page */ #define PUD_MASKED_BITS 0xc0000000000000ffUL/* Bits to mask out from a PGD to get to the PUD page */ -#define PGD_MASKED_BITS 0xc0000000000000ffUL +#define P4D_MASKED_BITS 0xc0000000000000ffUL/* * Used as an indicator for rcu callback functions @@ -949,54 +949,60 @@ static inline bool pud_access_permitted(pud_t pud, bool write)return pte_access_permitted (pud_pte (pud), write); } -#define pgd_write (pgd) pte_write(pgd_pte(pgd)) +#define __p4d_raw (x) ((p4d_t) { __pgd_raw(x) }) +static inline __be64 p4d_raw (p4d_t x) +{ +return pgd_raw (x.pgd); +} + +#define p4d_write (p4d) pte_write(p4d_pte(p4d)) -static inline void pgd_clear (pgd_t *pgdp) +static inline void p4d_clear (p4d_t *p4dp) { - *pgdp =__pgd (0 ); + *p4dp =__p4d (0 ); } -static inline int pgd_none (pgd_t pgd) +static inline int p4d_none (p4d_t p4d) { -return !pgd_raw (pgd); +return !p4d_raw (p4d); } -static inline int pgd_present (pgd_t pgd) +static inline int p4d_present (p4d_t p4d) { -return !!(pgd_raw (pgd) &cpu_to_be64 (_PAGE_PRESENT)); +return !!(p4d_raw (p4d) &cpu_to_be64 (_PAGE_PRESENT)); } -static inline pte_t pgd_pte (pgd_t pgd) +static inline pte_t p4d_pte (p4d_t p4d) { -return __pte_raw (pgd_raw (pgd)); +return __pte_raw (p4d_raw (p4d)); } -static inline pgd_t pte_pgd (pte_t pte) +static inline p4d_t pte_p4d (pte_t pte) { -return __pgd_raw (pte_raw (pte)); +return __p4d_raw (pte_raw (pte)); } -static inline int pgd_bad (pgd_t pgd) +static inline int p4d_bad (p4d_t p4d) {if (radix_enabled ()) -return radix__pgd_bad (pgd); -return hash__pgd_bad (pgd); +return radix__p4d_bad (p4d); +return hash__p4d_bad (p4d); } -#define pgd_access_permitted pgd_access_permitted -static inline bool pgd_access_permitted (pgd_t pgd,bool write) +#define p4d_access_permitted p4d_access_permitted +static inline bool p4d_access_permitted (p4d_t p4d,bool write) { -return pte_access_permitted (pgd_pte (pgd), write); +return pte_access_permitted (p4d_pte (p4d), write); } -extern struct page *pgd_page (pgd_t pgd); +extern struct page *p4d_page (p4d_t p4d);/* Pointers in the page table tree are physical addresses */ #define __pgtable_ptr_val (ptr) __pa(ptr)#define pmd_page_vaddr (pmd) __va(pmd_val(pmd) & ~PMD_MASKED_BITS)#define pud_page_vaddr (pud) __va(pud_val(pud) & ~PUD_MASKED_BITS) -#define pgd_page_vaddr (pgd) __va(pgd_val(pgd) & ~PGD_MASKED_BITS) +#define p4d_page_vaddr (p4d) __va(p4d_val(p4d) & ~P4D_MASKED_BITS)#define pgd_index (address) (((address) >> (PGDIR_SHIFT)) & (PTRS_PER_PGD - 1))#define pud_index (address) (((address) >> (PUD_SHIFT)) & (PTRS_PER_PUD - 1))@@ -1010,8 +1016,8 @@ extern struct page *pgd_page(pgd_t pgd);#define pgd_offset (mm, address) ((mm)->pgd + pgd_index(address)) -#define pud_offset (pgdp, addr) \ - (((pud_t *) pgd_page_vaddr(*(pgdp))) + pud_index(addr)) +#define pud_offset (p4dp, addr) \ + (((pud_t *) p4d_page_vaddr(*(p4dp))) + pud_index(addr))#define pmd_offset (pudp,addr) \ (((pmd_t *) pud_page_vaddr(*(pudp))) + pmd_index(addr))#define pte_offset_kernel (dir,addr) \@@ -1366,6 +1372,12 @@ static inline bool pud_is_leaf(pud_t pud)return !!(pud_raw (pud) &cpu_to_be64 (_PAGE_PTE)); } +#define p4d_is_leaf p4d_is_leaf +static inline bool p4d_is_leaf (p4d_t p4d) +{ +return !!(p4d_raw (p4d) &cpu_to_be64 (_PAGE_PTE)); +} +#define pgd_is_leaf pgd_is_leafstatic inline bool pgd_is_leaf (pgd_t pgd) {arch/powerpc/include/asm/book3s/64/radix.h +3
-3@@ -30,7 +30,7 @@/* Don't have anything in the reserved bits and leaf bits */ #define RADIX_PMD_BAD_BITS 0x60000000000000e0UL#define RADIX_PUD_BAD_BITS 0x60000000000000e0UL -#define RADIX_PGD_BAD_BITS 0x60000000000000e0UL +#define RADIX_P4D_BAD_BITS 0x60000000000000e0UL#define RADIX_PMD_SHIFT (PAGE_SHIFT + RADIX_PTE_INDEX_SIZE)#define RADIX_PUD_SHIFT (RADIX_PMD_SHIFT + RADIX_PMD_INDEX_SIZE)@@ -227,9 +227,9 @@ static inline int radix__pud_bad(pud_t pud) } -static inline int radix__pgd_bad (pgd_t pgd) +static inline int radix__p4d_bad (p4d_t p4d) { -return !!(pgd_val (pgd) &RADIX_PGD_BAD_BITS ); +return !!(p4d_val (p4d) &RADIX_P4D_BAD_BITS ); }#ifdef CONFIG_TRANSPARENT_HUGEPAGE arch/powerpc/include/asm/nohash/32/pgtable.h +0
-1@@ -2,7 +2,6 @@#ifndef _ASM_POWERPC_NOHASH_32_PGTABLE_H#define _ASM_POWERPC_NOHASH_32_PGTABLE_H -#define __ARCH_USE_5LEVEL_HACK#include <asm-generic/pgtable-nopmd.h> #ifndef __ASSEMBLY__arch/powerpc/include/asm/nohash/64/pgalloc.h +1
-1@@ -15,7 +15,7 @@ struct vmemmap_backing { };extern struct vmemmap_backing *vmemmap_list; -#define pgd_populate (MM ,PGD ,PUD ) pgd_set(PGD, (unsigned long)PUD) +#define p4d_populate (MM ,P4D ,PUD ) p4d_set(P4D, (unsigned long)PUD)static inline pud_t *pud_alloc_one (struct mm_struct *mm,unsigned long addr) {arch/powerpc/include/asm/nohash/64/pgtable-4k.h +16
-16@@ -2,7 +2,7 @@#ifndef _ASM_POWERPC_NOHASH_64_PGTABLE_4K_H#define _ASM_POWERPC_NOHASH_64_PGTABLE_4K_H -#include <asm-generic/5level-fixup.h> +#include <asm-generic/pgtable-nop4d.h> /* * Entries per page directory level. The PTE level must use a 64b record @@ -45,41 +45,41 @@#define PMD_MASKED_BITS 0/* Bits to mask out from a PUD to get to the PMD page */ #define PUD_MASKED_BITS 0 -/* Bits to mask out from a PGD to get to the PUD page */ -#define PGD_MASKED_BITS 0 +/* Bits to mask out from a P4D to get to the PUD page */ +#define P4D_MASKED_BITS 0/* * 4-level page tables related bits */ -#define pgd_none (pgd) (!pgd_val(pgd)) -#define pgd_bad (pgd) (pgd_val(pgd) == 0) -#define pgd_present (pgd) (pgd_val(pgd) != 0) -#define pgd_page_vaddr (pgd) (pgd_val(pgd) & ~PGD_MASKED_BITS) +#define p4d_none (p4d) (!p4d_val(p4d)) +#define p4d_bad (p4d) (p4d_val(p4d) == 0) +#define p4d_present (p4d) (p4d_val(p4d) != 0) +#define p4d_page_vaddr (p4d) (p4d_val(p4d) & ~P4D_MASKED_BITS)#ifndef __ASSEMBLY__ -static inline void pgd_clear (pgd_t *pgdp) +static inline void p4d_clear (p4d_t *p4dp) { - *pgdp =__pgd (0 ); + *p4dp =__p4d (0 ); } -static inline pte_t pgd_pte (pgd_t pgd) +static inline pte_t p4d_pte (p4d_t p4d) { -return __pte (pgd_val (pgd)); +return __pte (p4d_val (p4d)); } -static inline pgd_t pte_pgd (pte_t pte) +static inline p4d_t pte_p4d (pte_t pte) { -return __pgd (pte_val (pte)); +return __p4d (pte_val (pte)); } -extern struct page *pgd_page (pgd_t pgd); +extern struct page *p4d_page (p4d_t p4d);#endif /* !__ASSEMBLY__ */ -#define pud_offset (pgdp, addr) \ - (((pud_t *) pgd_page_vaddr(*(pgdp))) + \ +#define pud_offset (p4dp, addr) \ + (((pud_t *) p4d_page_vaddr(*(p4dp))) + \ (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1)))#define pud_ERROR (e) \arch/powerpc/include/asm/nohash/64/pgtable.h +3
-3@@ -175,11 +175,11 @@ static inline pud_t pte_pud(pte_t pte)return __pud (pte_val (pte)); }#define pud_write (pud) pte_write(pud_pte(pud)) -#define pgd_write (pgd) pte_write(pgd_pte(pgd)) +#define p4d_write (pgd) pte_write(p4d_pte(p4d)) -static inline void pgd_set (pgd_t *pgdp,unsigned long val) +static inline void p4d_set (p4d_t *p4dp,unsigned long val) { - *pgdp =__pgd (val); + *p4dp =__p4d (val); }/* arch/powerpc/include/asm/pgtable.h +8
-0@@ -145,6 +145,14 @@ static inline bool pud_is_leaf(pud_t pud) }#endif +#ifndef p4d_is_leaf +#define p4d_is_leaf p4d_is_leaf +static inline bool p4d_is_leaf (p4d_t p4d) +{ +return false; +} +#endif +#ifndef pgd_is_leaf#define pgd_is_leaf pgd_is_leafstatic inline bool pgd_is_leaf (pgd_t pgd)arch/powerpc/kvm/book3s_64_mmu_radix.c +51
-8@@ -496,17 +496,39 @@ static void kvmppc_unmap_free_pud(struct kvm *kvm, pud_t *pud,pud_free (kvm->mm, pud); } +static void kvmppc_unmap_free_p4d (struct kvm *kvm,p4d_t *p4d, +unsigned int lpid) +{ +unsigned long iu; +p4d_t *p = p4d; + +for (iu =0 ; iu <PTRS_PER_P4D ; ++iu, ++p) { +if (!p4d_present (*p)) +continue ; +if (p4d_is_leaf (*p)) { +p4d_clear (p); + }else { +pud_t *pud; + + pud =pud_offset (p,0 ); +kvmppc_unmap_free_pud (kvm, pud, lpid); +p4d_clear (p); + } + } +p4d_free (kvm->mm, p4d); +} +void kvmppc_free_pgtable_radix (struct kvm *kvm,pgd_t *pgd,unsigned int lpid) {unsigned long ig;for (ig =0 ; ig <PTRS_PER_PGD ; ++ig, ++pgd) { -pud_t *pud; +p4d_t *p4d;if (!pgd_present (*pgd))continue ; - pud =pud_offset (pgd,0 ); -kvmppc_unmap_free_pud (kvm, pud, lpid); + p4d =p4d_offset (pgd,0 ); +kvmppc_unmap_free_p4d (kvm, p4d, lpid);pgd_clear (pgd); } }@@ -568,6 +590,7 @@ int kvmppc_create_pte(struct kvm *kvm, pgd_t *pgtable, pte_t pte, unsigned long *rmapp, struct rmap_nested **n_rmap) { pgd_t *pgd; +p4d_t *p4d, *new_p4d =NULL ;pud_t *pud, *new_pud =NULL ;pmd_t *pmd, *new_pmd =NULL ;pte_t *ptep, *new_ptep =NULL ;@@ -575,9 +598,15 @@ int kvmppc_create_pte(struct kvm *kvm, pgd_t *pgtable, pte_t pte,/* Traverse the guest's 2nd-level tree, allocate new levels needed */ pgd = pgtable +pgd_index (gpa); - pud =NULL ; + p4d =NULL ;if (pgd_present (*pgd)) - pud =pud_offset (pgd, gpa); + p4d =p4d_offset (pgd, gpa); +else + new_p4d =p4d_alloc_one (kvm->mm, gpa); + + pud =NULL ; +if (p4d_present (*p4d)) + pud =pud_offset (p4d, gpa);else new_pud =pud_alloc_one (kvm->mm, gpa);@@ -599,12 +628,18 @@ int kvmppc_create_pte(struct kvm *kvm, pgd_t *pgtable, pte_t pte,/* Now traverse again under the lock and change the tree */ ret = -ENOMEM ;if (pgd_none (*pgd)) { +if (!new_p4d) + goto out_unlock; +pgd_populate (kvm->mm, pgd, new_p4d); + new_p4d =NULL ; + } +if (p4d_none (*p4d)) {if (!new_pud) goto out_unlock; -pgd_populate (kvm->mm, pgd, new_pud); +p4d_populate (kvm->mm, p4d, new_pud); new_pud =NULL ; } - pud =pud_offset (pgd, gpa); + pud =pud_offset (p4d, gpa);if (pud_is_leaf (*pud)) {unsigned long hgpa = gpa &PUD_MASK ;@@ -1222,6 +1257,7 @@ static ssize_t debugfs_radix_read(struct file *file, char __user *buf,pgd_t *pgt;struct kvm_nested_guest *nested;pgd_t pgd, *pgdp; +p4d_t p4d, *p4dp;pud_t pud, *pudp;pmd_t pmd, *pmdp;pte_t *ptep;@@ -1300,7 +1336,14 @@ static ssize_t debugfs_radix_read(struct file *file, char __user *buf,continue ; } - pudp =pud_offset (&pgd, gpa); + p4dp =p4d_offset (&pgd, gpa); + p4d =READ_ONCE (*p4dp); +if (!(p4d_val (p4d) & _PAGE_PRESENT)) { + gpa = (gpa &P4D_MASK ) +P4D_SIZE ; +continue ; + } + + pudp =pud_offset (&p4d, gpa); pud =READ_ONCE (*pudp);if (!(pud_val (pud) & _PAGE_PRESENT)) { gpa = (gpa &PUD_MASK ) +PUD_SIZE ;arch/powerpc/lib/code-patching.c +6
-1@@ -107,13 +107,18 @@ static inline int unmap_patch_area(unsigned long addr)pte_t *ptep;pmd_t *pmdp;pud_t *pudp; +p4d_t *p4dp;pgd_t *pgdp; pgdp =pgd_offset_k (addr);if (unlikely (!pgdp))return -EINVAL ; - pudp =pud_offset (pgdp, addr); + p4dp =p4d_offset (pgdp, addr); +if (unlikely (!p4dp)) +return -EINVAL ; + + pudp =pud_offset (p4dp, addr);if (unlikely (!pudp))return -EINVAL ;arch/powerpc/mm/book3s32/mmu.c +1
-1@@ -312,7 +312,7 @@ void hash_preload(struct mm_struct *mm, unsigned long ea)if (!Hash)return ; - pmd =pmd_offset (pud_offset (pgd_offset (mm, ea), ea), ea); + pmd =pmd_offset (pud_offset (p4d_offset (pgd_offset (mm, ea), ea), ea), ea);if (!pmd_none (*pmd))add_hash_page (mm->context.id, ea,pmd_val (*pmd)); }arch/powerpc/mm/book3s32/tlb.c +2
-2@@ -87,7 +87,7 @@ static void flush_range(struct mm_struct *mm, unsigned long start,if (start >= end)return ; end = (end -1 ) | ~PAGE_MASK ; - pmd =pmd_offset (pud_offset (pgd_offset (mm, start), start), start); + pmd =pmd_offset (pud_offset (p4d_offset (pgd_offset (mm, start), start), start), start);for (;;) { pmd_end = ((start +PGDIR_SIZE ) &PGDIR_MASK ) -1 ;if (pmd_end > end)@@ -145,7 +145,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)return ; } mm = (vmaddr <TASK_SIZE )? vma->vm_mm: &init_mm; - pmd =pmd_offset (pud_offset (pgd_offset (mm, vmaddr), vmaddr), vmaddr); + pmd =pmd_offset (pud_offset (p4d_offset (pgd_offset (mm, vmaddr), vmaddr), vmaddr), vmaddr);if (!pmd_none (*pmd))flush_hash_pages (mm->context.id, vmaddr,pmd_val (*pmd),1 ); }arch/powerpc/mm/book3s64/hash_pgtable.c +3
-1@@ -148,6 +148,7 @@ void hash__vmemmap_remove_mapping(unsigned long start,int hash__map_kernel_page (unsigned long ea,unsigned long pa,pgprot_t prot) {pgd_t *pgdp; +p4d_t *p4dp;pud_t *pudp;pmd_t *pmdp;pte_t *ptep;@@ -155,7 +156,8 @@ int hash__map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot)BUILD_BUG_ON (TASK_SIZE_USER64 >H_PGTABLE_RANGE );if (slab_is_available ()) { pgdp =pgd_offset_k (ea); - pudp =pud_alloc (&init_mm, pgdp, ea); + p4dp =p4d_offset (pgdp, ea); + pudp =pud_alloc (&init_mm, p4dp, ea);if (!pudp)return -ENOMEM ; pmdp =pmd_alloc (&init_mm, pudp, ea);arch/powerpc/mm/book3s64/radix_pgtable.c +15
-4@@ -64,17 +64,24 @@ static int early_map_kernel_page(unsigned long ea, unsigned long pa, {unsigned long pfn = pa >>PAGE_SHIFT ;pgd_t *pgdp; +p4d_t *p4dp;pud_t *pudp;pmd_t *pmdp;pte_t *ptep; pgdp =pgd_offset_k (ea);if (pgd_none (*pgdp)) { + p4dp =early_alloc_pgtable (PGD_TABLE_SIZE , nid, + region_start, region_end); +pgd_populate (&init_mm, pgdp, p4dp); + } + p4dp =p4d_offset (pgdp, ea); +if (p4d_none (*p4dp)) { pudp =early_alloc_pgtable (PUD_TABLE_SIZE , nid, region_start, region_end); -pgd_populate (&init_mm, pgdp, pudp); +p4d_populate (&init_mm, p4dp, pudp); } - pudp =pud_offset (pgdp, ea); + pudp =pud_offset (p4dp, ea);if (map_page_size ==PUD_SIZE ) { ptep = (pte_t *)pudp; goto set_the_pte;@@ -114,6 +121,7 @@ static int __map_kernel_page(unsigned long ea, unsigned long pa, {unsigned long pfn = pa >>PAGE_SHIFT ;pgd_t *pgdp; +p4d_t *p4dp;pud_t *pudp;pmd_t *pmdp;pte_t *ptep;@@ -136,7 +144,8 @@ static int __map_kernel_page(unsigned long ea, unsigned long pa,* boot. */ pgdp =pgd_offset_k (ea); - pudp =pud_alloc (&init_mm, pgdp, ea); + p4dp =p4d_offset (pgdp, ea); + pudp =pud_alloc (&init_mm, p4dp, ea);if (!pudp)return -ENOMEM ;if (map_page_size ==PUD_SIZE ) {@@ -173,6 +182,7 @@ void radix__change_memory_range(unsigned long start, unsigned long end, {unsigned long idx;pgd_t *pgdp; +p4d_t *p4dp;pud_t *pudp;pmd_t *pmdp;pte_t *ptep;@@ -185,7 +195,8 @@ void radix__change_memory_range(unsigned long start, unsigned long end,for (idx = start; idx < end; idx +=PAGE_SIZE ) { pgdp =pgd_offset_k (idx); - pudp =pud_alloc (&init_mm, pgdp, idx); + p4dp =p4d_offset (pgdp, idx); + pudp =pud_alloc (&init_mm, p4dp, idx);if (!pudp)continue ;if (pud_is_leaf (*pudp)) {arch/powerpc/mm/book3s64/subpage_prot.c +5
-1@@ -54,6 +54,7 @@ static void hpte_flush_range(struct mm_struct *mm, unsigned long addr, int npages) {pgd_t *pgd; +p4d_t *p4d;pud_t *pud;pmd_t *pmd;pte_t *pte;@@ -62,7 +63,10 @@ static void hpte_flush_range(struct mm_struct *mm, unsigned long addr, pgd =pgd_offset (mm, addr);if (pgd_none (*pgd))return ; - pud =pud_offset (pgd, addr); + p4d =p4d_offset (pgd, addr); +if (p4d_none (*p4d)) +return ; + pud =pud_offset (p4d, addr);if (pud_none (*pud))return ; pmd =pmd_offset (pud, addr);arch/powerpc/mm/hugetlbpage.c +16
-12@@ -112,6 +112,7 @@ static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp,pte_t *huge_pte_alloc (struct mm_struct *mm,unsigned long addr,unsigned long sz) {pgd_t *pg; +p4d_t *p4;pud_t *pu;pmd_t *pm;hugepd_t *hpdp =NULL ;@@ -121,20 +122,21 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz addr &= ~(sz-1 ); pg =pgd_offset (mm, addr); + p4 =p4d_offset (pg, addr);#ifdef CONFIG_PPC_BOOK3S_64 if (pshift ==PGDIR_SHIFT )/* 16GB huge page */ -return (pte_t *) pg; +return (pte_t *) p4;else if (pshift >PUD_SHIFT ) {/* * We need to use hugepd table */ ptl = &mm->page_table_lock; - hpdp = (hugepd_t *)pg; + hpdp = (hugepd_t *)p4; }else { pdshift =PUD_SHIFT ; - pu =pud_alloc (mm, pg, addr); + pu =pud_alloc (mm, p4, addr);if (!pu)return NULL ;if (pshift ==PUD_SHIFT )@@ -159,10 +161,10 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz#else if (pshift >=PGDIR_SHIFT ) { ptl = &mm->page_table_lock; - hpdp = (hugepd_t *)pg; + hpdp = (hugepd_t *)p4; }else { pdshift =PUD_SHIFT ; - pu =pud_alloc (mm, pg, addr); + pu =pud_alloc (mm, p4, addr);if (!pu)return NULL ;if (pshift >=PUD_SHIFT ) {@@ -384,7 +386,7 @@ static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud,mm_dec_nr_pmds (tlb->mm); } -static void hugetlb_free_pud_range (struct mmu_gather *tlb,pgd_t *pgd, +static void hugetlb_free_pud_range (struct mmu_gather *tlb,p4d_t *p4d,unsigned long addr,unsigned long end,unsigned long floor,unsigned long ceiling) {@@ -394,7 +396,7 @@ static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd, start = addr; do { - pud = pud_offset(pgd, addr); + pud = pud_offset(p4d, addr); next = pud_addr_end(addr, end); if (!is_hugepd(__hugepd(pud_val(*pud)))) { if (pud_none_or_clear_bad(pud))@@ -429,8 +431,8 @@ static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,if (end -1 > ceiling -1 )return ; - pud =pud_offset (pgd, start); -pgd_clear (pgd); + pud =pud_offset (p4d, start); +p4d_clear (p4d);pud_free_tlb (tlb, pud, start);mm_dec_nr_puds (tlb->mm); }@@ -443,6 +445,7 @@ void hugetlb_free_pgd_range(struct mmu_gather *tlb, unsigned long floor, unsigned long ceiling) {pgd_t *pgd; +p4d_t *p4d;unsigned long next;/* @@ -465,10 +468,11 @@ void hugetlb_free_pgd_range(struct mmu_gather *tlb, do { next = pgd_addr_end(addr, end); pgd = pgd_offset(tlb->mm, addr); + p4d = p4d_offset(pgd, addr); if (!is_hugepd(__hugepd(pgd_val(*pgd)))) { - if (pgd_none_or_clear_bad(pgd)) + if (p4d_none_or_clear_bad(p4d)) continue; - hugetlb_free_pud_range(tlb, pgd, addr, next, floor, ceiling); + hugetlb_free_pud_range(tlb, p4d, addr, next, floor, ceiling); } else { unsigned long more; /*@@ -481,7 +485,7 @@ void hugetlb_free_pgd_range(struct mmu_gather *tlb,if (more > next) next = more; -free_hugepd_range (tlb, (hugepd_t *)pgd,PGDIR_SHIFT , +free_hugepd_range (tlb, (hugepd_t *)p4d,PGDIR_SHIFT , addr, next, floor, ceiling); } }while (addr = next, addr != end);arch/powerpc/mm/kasan/kasan_init_32.c +4
-4@@ -36,7 +36,7 @@ static int __ref kasan_init_shadow_page_tables(unsigned long k_start, unsigned lunsigned long k_cur, k_next;pgprot_t prot =slab_is_available () ?kasan_prot_ro () :PAGE_KERNEL ; - pmd =pmd_offset (pud_offset (pgd_offset_k (k_start), k_start), k_start); + pmd =pmd_offset (pud_offset (p4d_offset (pgd_offset_k (k_start), k_start), k_start), k_start);for (k_cur = k_start; k_cur != k_end; k_cur = k_next, pmd++) {pte_t *new;@@ -94,7 +94,7 @@ static int __ref kasan_init_region(void *start, size_t size) block =memblock_alloc (k_end - k_start,PAGE_SIZE );for (k_cur = k_start &PAGE_MASK ; k_cur < k_end; k_cur +=PAGE_SIZE ) { -pmd_t *pmd =pmd_offset (pud_offset (pgd_offset_k (k_cur), k_cur), k_cur); +pmd_t *pmd =pmd_offset (pud_offset (p4d_offset (pgd_offset_k (k_cur), k_cur), k_cur), k_cur);void *va = block ? block + k_cur - k_start :kasan_get_one_page ();pte_t pte =pfn_pte (PHYS_PFN (__pa (va)),PAGE_KERNEL );@@ -118,7 +118,7 @@ static void __init kasan_remap_early_shadow_ro(void)kasan_populate_pte (kasan_early_shadow_pte, prot);for (k_cur = k_start &PAGE_MASK ; k_cur < k_end; k_cur +=PAGE_SIZE ) { -pmd_t *pmd =pmd_offset (pud_offset (pgd_offset_k (k_cur), k_cur), k_cur); +pmd_t *pmd =pmd_offset (pud_offset (p4d_offset (pgd_offset_k (k_cur), k_cur), k_cur), k_cur);pte_t *ptep =pte_offset_kernel (pmd, k_cur);if ((pte_val (*ptep) &PTE_RPN_MASK ) != pa)@@ -205,7 +205,7 @@ void __init kasan_early_init(void)unsigned long addr =KASAN_SHADOW_START ;unsigned long end =KASAN_SHADOW_END ;unsigned long next; -pmd_t *pmd =pmd_offset (pud_offset (pgd_offset_k (addr), addr), addr); +pmd_t *pmd =pmd_offset (pud_offset (p4d_offset (pgd_offset_k (addr), addr), addr), addr);BUILD_BUG_ON (KASAN_SHADOW_START & ~PGDIR_MASK );arch/powerpc/mm/mem.c +2
-2@@ -68,8 +68,8 @@ EXPORT_SYMBOL(kmap_prot);static inline pte_t *virt_to_kpte (unsigned long vaddr) { -return pte_offset_kernel (pmd_offset (pud_offset (pgd_offset_k (vaddr), - vaddr), vaddr), vaddr); +return pte_offset_kernel (pmd_offset (pud_offset (p4d_offset (pgd_offset_k (vaddr), + vaddr), vaddr), vaddr), vaddr); }#endif arch/powerpc/mm/nohash/40x.c +2
-2@@ -104,7 +104,7 @@ unsigned long __init mmu_mapin_ram(unsigned long base, unsigned long top)pmd_t *pmdp;unsigned long val = p | _PMD_SIZE_16M | _PAGE_EXEC | _PAGE_HWWRITE; - pmdp =pmd_offset (pud_offset (pgd_offset_k (v), v), v); + pmdp =pmd_offset (pud_offset (p4d_offset (pgd_offset_k (v), v), v), v); *pmdp++ =__pmd (val); *pmdp++ =__pmd (val); *pmdp++ =__pmd (val);@@ -119,7 +119,7 @@ unsigned long __init mmu_mapin_ram(unsigned long base, unsigned long top)pmd_t *pmdp;unsigned long val = p | _PMD_SIZE_4M | _PAGE_EXEC | _PAGE_HWWRITE; - pmdp =pmd_offset (pud_offset (pgd_offset_k (v), v), v); + pmdp =pmd_offset (pud_offset (p4d_offset (pgd_offset_k (v), v), v), v); *pmdp =__pmd (val); v +=LARGE_PAGE_SIZE_4M ;arch/powerpc/mm/nohash/book3e_pgtable.c +11
-4@@ -73,6 +73,7 @@ static void __init *early_alloc_pgtable(unsigned long size)int __ref map_kernel_page (unsigned long ea,unsigned long pa,pgprot_t prot) {pgd_t *pgdp; +p4d_t *p4dp;pud_t *pudp;pmd_t *pmdp;pte_t *ptep;@@ -80,7 +81,10 @@ int __ref map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot)BUILD_BUG_ON (TASK_SIZE_USER64 >PGTABLE_RANGE );if (slab_is_available ()) { pgdp =pgd_offset_k (ea); - pudp =pud_alloc (&init_mm, pgdp, ea); + p4dp =p4d_alloc (&init_mm, pgdp, ea); +if (!p4dp) +return -ENOMEM ; + pudp =pud_alloc (&init_mm, p4dp, ea);if (!pudp)return -ENOMEM ; pmdp =pmd_alloc (&init_mm, pudp, ea);@@ -91,13 +95,16 @@ int __ref map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot)return -ENOMEM ; }else { pgdp =pgd_offset_k (ea); -#ifndef __PAGETABLE_PUD_FOLDEDif (pgd_none (*pgdp)) { pudp =early_alloc_pgtable (PUD_TABLE_SIZE );pgd_populate (&init_mm, pgdp, pudp); } -#endif /* !__PAGETABLE_PUD_FOLDED */ - pudp =pud_offset (pgdp, ea); + p4dp =p4d_offset (pgdp, ea); +if (p4d_none (*p4dp)) { + pmdp =early_alloc_pgtable (PMD_TABLE_SIZE ); +p4d_populate (&init_mm, p4dp, pmdp); + } + pudp =pud_offset (p4dp, ea);if (pud_none (*pudp)) { pmdp =early_alloc_pgtable (PMD_TABLE_SIZE );pud_populate (&init_mm, pudp, pmdp);arch/powerpc/mm/pgtable.c +23
-2@@ -265,6 +265,7 @@ int huge_ptep_set_access_flags(struct vm_area_struct *vma,void assert_pte_locked (struct mm_struct *mm,unsigned long addr) {pgd_t *pgd; +p4d_t *p4d;pud_t *pud;pmd_t *pmd;@@ -272,7 +273,9 @@ void assert_pte_locked(struct mm_struct *mm, unsigned long addr)return ; pgd = mm->pgd +pgd_index (addr);BUG_ON (pgd_none (*pgd)); - pud =pud_offset (pgd, addr); + p4d =p4d_offset (pgd, addr); +BUG_ON (p4d_none (*p4d)); + pud =pud_offset (p4d, addr);BUG_ON (pud_none (*pud)); pmd =pmd_offset (pud, addr);/* @@ -313,6 +316,7 @@ pte_t *__find_linux_pte(pgd_t *pgdir, unsigned long ea, bool *is_thp, unsigned *hpage_shift) { pgd_t pgd, *pgdp; +p4d_t p4d, *p4dp;pud_t pud, *pudp;pmd_t pmd, *pmdp;pte_t *ret_pte;@@ -346,13 +350,30 @@ pte_t *__find_linux_pte(pgd_t *pgdir, unsigned long ea, goto out_huge; } + pdshift =P4D_SHIFT ; + p4dp =p4d_offset (&pgd, ea); + p4d =READ_ONCE (*p4dp); + +if (p4d_none (p4d)) +return NULL ; + +if (p4d_is_leaf (p4d)) { + ret_pte = (pte_t *)p4dp; + goto out; + } + +if (is_hugepd (__hugepd (p4d_val (p4d)))) { + hpdp = (hugepd_t *)&p4d; + goto out_huge; + } +/* * Even if we end up with an unmap, the pgtable will not * be freed, because we do an rcu free and here we are * irq disabled */ pdshift =PUD_SHIFT ; - pudp =pud_offset (&pgd, ea); + pudp =pud_offset (&p4d, ea); pud =READ_ONCE (*pudp);if (pud_none (pud))arch/powerpc/mm/pgtable_32.c +16
-12@@ -63,7 +63,7 @@ int __ref map_kernel_page(unsigned long va, phys_addr_t pa, pgprot_t prot)int err = -ENOMEM ;/* Use upper 10 bits of VA to index the first level map */ - pd =pmd_offset (pud_offset (pgd_offset_k (va), va), va); + pd =pmd_offset (pud_offset (p4d_offset (pgd_offset_k (va), va), va), va);/* Use middle 10 bits of VA to index the second-level map */ if (likely (slab_is_available ())) pg =pte_alloc_kernel (pd, va);@@ -130,6 +130,7 @@ static int get_pteptr(struct mm_struct *mm, unsigned long addr, pte_t **ptep, pmd_t **pmdp) { pgd_t *pgd; +p4d_t *p4d;pud_t *pud;pmd_t *pmd;pte_t *pte;@@ -137,17 +138,20 @@ get_pteptr(struct mm_struct *mm, unsigned long addr, pte_t **ptep, pmd_t **pmdp) pgd =pgd_offset (mm, addr &PAGE_MASK );if (pgd) { - pud =pud_offset (pgd, addr &PAGE_MASK ); -if (pud &&pud_present (*pud)) { - pmd =pmd_offset (pud, addr &PAGE_MASK ); -if (pmd_present (*pmd)) { - pte =pte_offset_map (pmd, addr &PAGE_MASK ); -if (pte) { - retval =1 ; - *ptep = pte; -if (pmdp) - *pmdp = pmd; -/* XXX caller needs to do pte_unmap, yuck */ + p4d =p4d_offset (pgd, addr &PAGE_MASK ); +if (p4d &&p4d_present (*p4d)) { + pud =pud_offset (p4d, addr &PAGE_MASK ); +if (pud &&pud_present (*pud)) { + pmd =pmd_offset (pud, addr &PAGE_MASK ); +if (pmd_present (*pmd)) { + pte =pte_offset_map (pmd, addr &PAGE_MASK ); +if (pte) { + retval =1 ; + *ptep = pte; +if (pmdp) + *pmdp = pmd; +/* XXX caller needs to do pte_unmap, yuck */ + } } } }arch/powerpc/mm/pgtable_64.c +5
-5@@ -101,13 +101,13 @@ EXPORT_SYMBOL(__pte_frag_size_shift);#ifndef __PAGETABLE_PUD_FOLDED/* 4 level page table */ -struct page *pgd_page (pgd_t pgd) +struct page *p4d_page (p4d_t p4d) { -if (pgd_is_leaf (pgd)) { -VM_WARN_ON (!pgd_huge (pgd)); -return pte_page (pgd_pte (pgd)); +if (p4d_is_leaf (p4d)) { +VM_WARN_ON (!p4d_huge (p4d)); +return pte_page (p4d_pte (p4d)); } -return virt_to_page (pgd_page_vaddr (pgd)); +return virt_to_page (p4d_page_vaddr (p4d)); }#endif arch/powerpc/mm/ptdump/hashpagetable.c +17
-3@@ -417,9 +417,9 @@ static void walk_pmd(struct pg_state *st, pud_t *pud, unsigned long start) } } -static void walk_pud (struct pg_state *st,pgd_t *pgd,unsigned long start) +static void walk_pud (struct pg_state *st,p4d_t *p4d,unsigned long start) { -pud_t *pud =pud_offset (pgd,0 ); +pud_t *pud =pud_offset (p4d,0 );unsigned long addr;unsigned int i;@@ -431,6 +431,20 @@ static void walk_pud(struct pg_state *st, pgd_t *pgd, unsigned long start) } } +static void walk_p4d (struct pg_state *st,pgd_t *pgd,unsigned long start) +{ +p4d_t *p4d =p4d_offset (pgd,0 ); +unsigned long addr; +unsigned int i; + +for (i =0 ; i <PTRS_PER_P4D ; i++, p4d++) { + addr = start + i *P4D_SIZE ; +if (!p4d_none (*p4d)) +/* p4d exists */ +walk_pud (st, p4d, addr); + } +} +static void walk_pagetables (struct pg_state *st) {pgd_t *pgd =pgd_offset_k (0UL );@@ -445,7 +459,7 @@ static void walk_pagetables(struct pg_state *st) addr =KERN_VIRT_START + i *PGDIR_SIZE ;if (!pgd_none (*pgd))/* pgd exists */ -walk_pud (st, pgd, addr); +walk_p4d (st, pgd, addr); } }arch/powerpc/mm/ptdump/ptdump.c +19
-3@@ -273,9 +273,9 @@ static void walk_pmd(struct pg_state *st, pud_t *pud, unsigned long start) } } -static void walk_pud (struct pg_state *st,pgd_t *pgd,unsigned long start) +static void walk_pud (struct pg_state *st,p4d_t *p4d,unsigned long start) { -pud_t *pud =pud_offset (pgd,0 ); +pud_t *pud =pud_offset (p4d,0 );unsigned long addr;unsigned int i;@@ -289,6 +289,22 @@ static void walk_pud(struct pg_state *st, pgd_t *pgd, unsigned long start) } } +static void walk_p4d (struct pg_state *st,pgd_t *pgd,unsigned long start) +{ +p4d_t *p4d =p4d_offset (pgd,0 ); +unsigned long addr; +unsigned int i; + +for (i =0 ; i <PTRS_PER_P4D ; i++, p4d++) { + addr = start + i *P4D_SIZE ; +if (!p4d_none (*p4d) && !p4d_is_leaf (*p4d)) +/* p4d exists */ +walk_pud (st, p4d, addr); +else +note_page (st, addr,2 ,p4d_val (*p4d)); + } +} +static void walk_pagetables (struct pg_state *st) {unsigned int i;@@ -302,7 +318,7 @@ static void walk_pagetables(struct pg_state *st)for (i =pgd_index (addr); i <PTRS_PER_PGD ; i++, pgd++, addr +=PGDIR_SIZE ) {if (!pgd_none (*pgd) && !pgd_is_leaf (*pgd))/* pgd exists */ -walk_pud (st, pgd, addr); +walk_p4d (st, pgd, addr);else note_page (st, addr,1 ,pgd_val (*pgd)); }arch/powerpc/xmon/xmon.c +16
-1@@ -3128,6 +3128,7 @@ static void show_pte(unsigned long addr)struct task_struct *tsk =NULL ;struct mm_struct *mm;pgd_t *pgdp, *pgdir; +p4d_t *p4dp;pud_t *pudp;pmd_t *pmdp;pte_t *ptep;@@ -3172,7 +3173,21 @@ static void show_pte(unsigned long addr) }printf ("pgdp @ 0x%px = 0x%016lx\n" , pgdp,pgd_val (*pgdp)); - pudp =pud_offset (pgdp, addr); + p4dp =p4d_offset (pgdp, addr); + +if (p4d_none (*p4dp)) { +printf ("No valid P4D\n" ); +return ; + } + +if (p4d_is_leaf (*p4dp)) { +format_pte (p4dp,p4d_val (*p4dp)); +return ; + } + +printf ("p4dp @ 0x%px = 0x%016lx\n" , p4dp,p4d_val (*p4dp)); + + pudp =pud_offset (p4dp, addr);if (pud_none (*pudp)) {printf ("No valid PUD\n" );
Quoted 8 lines · Christophe Leroy
Le 09/12/2019 à 16:09, Mike Rapoport a écrit :
Quoted 5 lines
From: Mike Rapoport <[email protected]>
Implement primitives necessary for the 4th level folding, add walks of p4d level where appropriate and replace
5level-fixup.hwithpgtable-nop4d.h.
Quoted 7 lines
Quoted 1 line
Signed-off-by Mike Rapoport <[email protected]>Tested on 8xx and 83xx, no problem observed (ie book3s/32 and nohash/32)
Tested-by Christophe Leroy <[email protected]>Christophe
Did you add anything else to the next 1100 lines that you didn't bother to snip? I can't be arsed to scroll through them all looking.
Any updates on this?
Quoted 1,157 lines · Mike Rapoport
From: Mike Rapoport <[email protected]>
Implement primitives necessary for the 4th level folding, add walks of p4d level where appropriate and replace
5level-fixup.hwithpgtable-nop4d.h.Signed-off-by Mike Rapoport <[email protected]>--- arch/powerpc/include/asm/book3s/32/pgtable.h | 1 - arch/powerpc/include/asm/book3s/64/hash.h | 4 +- arch/powerpc/include/asm/book3s/64/pgalloc.h | 4 +- arch/powerpc/include/asm/book3s/64/pgtable.h | 58 ++++++++++-------- arch/powerpc/include/asm/book3s/64/radix.h | 6 +- arch/powerpc/include/asm/nohash/32/pgtable.h | 1 - arch/powerpc/include/asm/nohash/64/pgalloc.h | 2 +- .../include/asm/nohash/64/pgtable-4k.h | 32 +++++----- arch/powerpc/include/asm/nohash/64/pgtable.h | 6 +- arch/powerpc/include/asm/pgtable.h | 8 +++ arch/powerpc/kvm/book3s_64_mmu_radix.c | 59 ++++++++++++++++--- arch/powerpc/lib/code-patching.c | 7 ++- arch/powerpc/mm/book3s32/mmu.c | 2 +- arch/powerpc/mm/book3s32/tlb.c | 4 +- arch/powerpc/mm/book3s64/hash_pgtable.c | 4 +- arch/powerpc/mm/book3s64/radix_pgtable.c | 19 ++++-- arch/powerpc/mm/book3s64/subpage_prot.c | 6 +- arch/powerpc/mm/hugetlbpage.c | 28 +++++---- arch/powerpc/mm/kasan/kasan_init_32.c | 8 +-- arch/powerpc/mm/mem.c | 4 +- arch/powerpc/mm/nohash/40x.c | 4 +- arch/powerpc/mm/nohash/book3e_pgtable.c | 15 +++-- arch/powerpc/mm/pgtable.c | 25 +++++++- arch/powerpc/mm/pgtable_32.c | 28 +++++---- arch/powerpc/mm/pgtable_64.c | 10 ++-- arch/powerpc/mm/ptdump/hashpagetable.c | 20 ++++++- arch/powerpc/mm/ptdump/ptdump.c | 22 ++++++- arch/powerpc/xmon/xmon.c | 17 +++++- 28 files changed, 284 insertions(+), 120 deletions(-)arch/powerpc/include/asm/book3s/32/pgtable.h +0
-1@@ -2,7 +2,6 @@#ifndef _ASM_POWERPC_BOOK3S_32_PGTABLE_H#define _ASM_POWERPC_BOOK3S_32_PGTABLE_H -#define __ARCH_USE_5LEVEL_HACK#include <asm-generic/pgtable-nopmd.h> #include <asm/book3s/32/hash.h> arch/powerpc/include/asm/book3s/64/hash.h +2
-2@@ -134,9 +134,9 @@ static inline int get_region_id(unsigned long ea)#define hash__pmd_bad (pmd) (pmd_val(pmd) & H_PMD_BAD_BITS)#define hash__pud_bad (pud) (pud_val(pud) & H_PUD_BAD_BITS) -static inline int hash__pgd_bad (pgd_t pgd) +static inline int hash__p4d_bad (p4d_t p4d) { -return (pgd_val (pgd) ==0 ); +return (p4d_val (p4d) ==0 ); }#ifdef CONFIG_STRICT_KERNEL_RWX extern void hash__mark_rodata_ro (void );arch/powerpc/include/asm/book3s/64/pgalloc.h +2
-2@@ -87,9 +87,9 @@ static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)kmem_cache_free (PGT_CACHE (PGD_INDEX_SIZE ), pgd); } -static inline void pgd_populate (struct mm_struct *mm,pgd_t *pgd,pud_t *pud) +static inline void p4d_populate (struct mm_struct *mm,p4d_t *pgd,pud_t *pud) { - *pgd =__pgd (__pgtable_ptr_val (pud) |PGD_VAL_BITS ); + *pgd =__p4d (__pgtable_ptr_val (pud) |PGD_VAL_BITS ); }static inline pud_t *pud_alloc_one (struct mm_struct *mm,unsigned long addr)arch/powerpc/include/asm/book3s/64/pgtable.h +35
-23@@ -2,7 +2,7 @@#ifndef _ASM_POWERPC_BOOK3S_64_PGTABLE_H_#define _ASM_POWERPC_BOOK3S_64_PGTABLE_H_ -#include <asm-generic/5level-fixup.h> +#include <asm-generic/pgtable-nop4d.h> #ifndef __ASSEMBLY__#include <linux/mmdebug.h> @@ -251,7 +251,7 @@ extern unsigned long __pmd_frag_size_shift;/* Bits to mask out from a PUD to get to the PMD page */ #define PUD_MASKED_BITS 0xc0000000000000ffUL/* Bits to mask out from a PGD to get to the PUD page */ -#define PGD_MASKED_BITS 0xc0000000000000ffUL +#define P4D_MASKED_BITS 0xc0000000000000ffUL/* * Used as an indicator for rcu callback functions @@ -949,54 +949,60 @@ static inline bool pud_access_permitted(pud_t pud, bool write)return pte_access_permitted (pud_pte (pud), write); } -#define pgd_write (pgd) pte_write(pgd_pte(pgd)) +#define __p4d_raw (x) ((p4d_t) { __pgd_raw(x) }) +static inline __be64 p4d_raw (p4d_t x) +{ +return pgd_raw (x.pgd); +} + +#define p4d_write (p4d) pte_write(p4d_pte(p4d)) -static inline void pgd_clear (pgd_t *pgdp) +static inline void p4d_clear (p4d_t *p4dp) { - *pgdp =__pgd (0 ); + *p4dp =__p4d (0 ); } -static inline int pgd_none (pgd_t pgd) +static inline int p4d_none (p4d_t p4d) { -return !pgd_raw (pgd); +return !p4d_raw (p4d); } -static inline int pgd_present (pgd_t pgd) +static inline int p4d_present (p4d_t p4d) { -return !!(pgd_raw (pgd) &cpu_to_be64 (_PAGE_PRESENT)); +return !!(p4d_raw (p4d) &cpu_to_be64 (_PAGE_PRESENT)); } -static inline pte_t pgd_pte (pgd_t pgd) +static inline pte_t p4d_pte (p4d_t p4d) { -return __pte_raw (pgd_raw (pgd)); +return __pte_raw (p4d_raw (p4d)); } -static inline pgd_t pte_pgd (pte_t pte) +static inline p4d_t pte_p4d (pte_t pte) { -return __pgd_raw (pte_raw (pte)); +return __p4d_raw (pte_raw (pte)); } -static inline int pgd_bad (pgd_t pgd) +static inline int p4d_bad (p4d_t p4d) {if (radix_enabled ()) -return radix__pgd_bad (pgd); -return hash__pgd_bad (pgd); +return radix__p4d_bad (p4d); +return hash__p4d_bad (p4d); } -#define pgd_access_permitted pgd_access_permitted -static inline bool pgd_access_permitted (pgd_t pgd,bool write) +#define p4d_access_permitted p4d_access_permitted +static inline bool p4d_access_permitted (p4d_t p4d,bool write) { -return pte_access_permitted (pgd_pte (pgd), write); +return pte_access_permitted (p4d_pte (p4d), write); } -extern struct page *pgd_page (pgd_t pgd); +extern struct page *p4d_page (p4d_t p4d);/* Pointers in the page table tree are physical addresses */ #define __pgtable_ptr_val (ptr) __pa(ptr)#define pmd_page_vaddr (pmd) __va(pmd_val(pmd) & ~PMD_MASKED_BITS)#define pud_page_vaddr (pud) __va(pud_val(pud) & ~PUD_MASKED_BITS) -#define pgd_page_vaddr (pgd) __va(pgd_val(pgd) & ~PGD_MASKED_BITS) +#define p4d_page_vaddr (p4d) __va(p4d_val(p4d) & ~P4D_MASKED_BITS)#define pgd_index (address) (((address) >> (PGDIR_SHIFT)) & (PTRS_PER_PGD - 1))#define pud_index (address) (((address) >> (PUD_SHIFT)) & (PTRS_PER_PUD - 1))@@ -1010,8 +1016,8 @@ extern struct page *pgd_page(pgd_t pgd);#define pgd_offset (mm, address) ((mm)->pgd + pgd_index(address)) -#define pud_offset (pgdp, addr) \ - (((pud_t *) pgd_page_vaddr(*(pgdp))) + pud_index(addr)) +#define pud_offset (p4dp, addr) \ + (((pud_t *) p4d_page_vaddr(*(p4dp))) + pud_index(addr))#define pmd_offset (pudp,addr) \ (((pmd_t *) pud_page_vaddr(*(pudp))) + pmd_index(addr))#define pte_offset_kernel (dir,addr) \@@ -1366,6 +1372,12 @@ static inline bool pud_is_leaf(pud_t pud)return !!(pud_raw (pud) &cpu_to_be64 (_PAGE_PTE)); } +#define p4d_is_leaf p4d_is_leaf +static inline bool p4d_is_leaf (p4d_t p4d) +{ +return !!(p4d_raw (p4d) &cpu_to_be64 (_PAGE_PTE)); +} +#define pgd_is_leaf pgd_is_leafstatic inline bool pgd_is_leaf (pgd_t pgd) {arch/powerpc/include/asm/book3s/64/radix.h +3
-3@@ -30,7 +30,7 @@/* Don't have anything in the reserved bits and leaf bits */ #define RADIX_PMD_BAD_BITS 0x60000000000000e0UL#define RADIX_PUD_BAD_BITS 0x60000000000000e0UL -#define RADIX_PGD_BAD_BITS 0x60000000000000e0UL +#define RADIX_P4D_BAD_BITS 0x60000000000000e0UL#define RADIX_PMD_SHIFT (PAGE_SHIFT + RADIX_PTE_INDEX_SIZE)#define RADIX_PUD_SHIFT (RADIX_PMD_SHIFT + RADIX_PMD_INDEX_SIZE)@@ -227,9 +227,9 @@ static inline int radix__pud_bad(pud_t pud) } -static inline int radix__pgd_bad (pgd_t pgd) +static inline int radix__p4d_bad (p4d_t p4d) { -return !!(pgd_val (pgd) &RADIX_PGD_BAD_BITS ); +return !!(p4d_val (p4d) &RADIX_P4D_BAD_BITS ); }#ifdef CONFIG_TRANSPARENT_HUGEPAGE arch/powerpc/include/asm/nohash/32/pgtable.h +0
-1@@ -2,7 +2,6 @@#ifndef _ASM_POWERPC_NOHASH_32_PGTABLE_H#define _ASM_POWERPC_NOHASH_32_PGTABLE_H -#define __ARCH_USE_5LEVEL_HACK#include <asm-generic/pgtable-nopmd.h> #ifndef __ASSEMBLY__arch/powerpc/include/asm/nohash/64/pgalloc.h +1
-1@@ -15,7 +15,7 @@ struct vmemmap_backing { };extern struct vmemmap_backing *vmemmap_list; -#define pgd_populate (MM ,PGD ,PUD ) pgd_set(PGD, (unsigned long)PUD) +#define p4d_populate (MM ,P4D ,PUD ) p4d_set(P4D, (unsigned long)PUD)static inline pud_t *pud_alloc_one (struct mm_struct *mm,unsigned long addr) {arch/powerpc/include/asm/nohash/64/pgtable-4k.h +16
-16@@ -2,7 +2,7 @@#ifndef _ASM_POWERPC_NOHASH_64_PGTABLE_4K_H#define _ASM_POWERPC_NOHASH_64_PGTABLE_4K_H -#include <asm-generic/5level-fixup.h> +#include <asm-generic/pgtable-nop4d.h> /* * Entries per page directory level. The PTE level must use a 64b record @@ -45,41 +45,41 @@#define PMD_MASKED_BITS 0/* Bits to mask out from a PUD to get to the PMD page */ #define PUD_MASKED_BITS 0 -/* Bits to mask out from a PGD to get to the PUD page */ -#define PGD_MASKED_BITS 0 +/* Bits to mask out from a P4D to get to the PUD page */ +#define P4D_MASKED_BITS 0/* * 4-level page tables related bits */ -#define pgd_none (pgd) (!pgd_val(pgd)) -#define pgd_bad (pgd) (pgd_val(pgd) == 0) -#define pgd_present (pgd) (pgd_val(pgd) != 0) -#define pgd_page_vaddr (pgd) (pgd_val(pgd) & ~PGD_MASKED_BITS) +#define p4d_none (p4d) (!p4d_val(p4d)) +#define p4d_bad (p4d) (p4d_val(p4d) == 0) +#define p4d_present (p4d) (p4d_val(p4d) != 0) +#define p4d_page_vaddr (p4d) (p4d_val(p4d) & ~P4D_MASKED_BITS)#ifndef __ASSEMBLY__ -static inline void pgd_clear (pgd_t *pgdp) +static inline void p4d_clear (p4d_t *p4dp) { - *pgdp =__pgd (0 ); + *p4dp =__p4d (0 ); } -static inline pte_t pgd_pte (pgd_t pgd) +static inline pte_t p4d_pte (p4d_t p4d) { -return __pte (pgd_val (pgd)); +return __pte (p4d_val (p4d)); } -static inline pgd_t pte_pgd (pte_t pte) +static inline p4d_t pte_p4d (pte_t pte) { -return __pgd (pte_val (pte)); +return __p4d (pte_val (pte)); } -extern struct page *pgd_page (pgd_t pgd); +extern struct page *p4d_page (p4d_t p4d);#endif /* !__ASSEMBLY__ */ -#define pud_offset (pgdp, addr) \ - (((pud_t *) pgd_page_vaddr(*(pgdp))) + \ +#define pud_offset (p4dp, addr) \ + (((pud_t *) p4d_page_vaddr(*(p4dp))) + \ (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1)))#define pud_ERROR (e) \arch/powerpc/include/asm/nohash/64/pgtable.h +3
-3@@ -175,11 +175,11 @@ static inline pud_t pte_pud(pte_t pte)return __pud (pte_val (pte)); }#define pud_write (pud) pte_write(pud_pte(pud)) -#define pgd_write (pgd) pte_write(pgd_pte(pgd)) +#define p4d_write (pgd) pte_write(p4d_pte(p4d)) -static inline void pgd_set (pgd_t *pgdp,unsigned long val) +static inline void p4d_set (p4d_t *p4dp,unsigned long val) { - *pgdp =__pgd (val); + *p4dp =__p4d (val); }/* arch/powerpc/include/asm/pgtable.h +8
-0@@ -145,6 +145,14 @@ static inline bool pud_is_leaf(pud_t pud) }#endif +#ifndef p4d_is_leaf +#define p4d_is_leaf p4d_is_leaf +static inline bool p4d_is_leaf (p4d_t p4d) +{ +return false; +} +#endif +#ifndef pgd_is_leaf#define pgd_is_leaf pgd_is_leafstatic inline bool pgd_is_leaf (pgd_t pgd)arch/powerpc/kvm/book3s_64_mmu_radix.c +51
-8@@ -496,17 +496,39 @@ static void kvmppc_unmap_free_pud(struct kvm *kvm, pud_t *pud,pud_free (kvm->mm, pud); } +static void kvmppc_unmap_free_p4d (struct kvm *kvm,p4d_t *p4d, +unsigned int lpid) +{ +unsigned long iu; +p4d_t *p = p4d; + +for (iu =0 ; iu <PTRS_PER_P4D ; ++iu, ++p) { +if (!p4d_present (*p)) +continue ; +if (p4d_is_leaf (*p)) { +p4d_clear (p); + }else { +pud_t *pud; + + pud =pud_offset (p,0 ); +kvmppc_unmap_free_pud (kvm, pud, lpid); +p4d_clear (p); + } + } +p4d_free (kvm->mm, p4d); +} +void kvmppc_free_pgtable_radix (struct kvm *kvm,pgd_t *pgd,unsigned int lpid) {unsigned long ig;for (ig =0 ; ig <PTRS_PER_PGD ; ++ig, ++pgd) { -pud_t *pud; +p4d_t *p4d;if (!pgd_present (*pgd))continue ; - pud =pud_offset (pgd,0 ); -kvmppc_unmap_free_pud (kvm, pud, lpid); + p4d =p4d_offset (pgd,0 ); +kvmppc_unmap_free_p4d (kvm, p4d, lpid);pgd_clear (pgd); } }@@ -568,6 +590,7 @@ int kvmppc_create_pte(struct kvm *kvm, pgd_t *pgtable, pte_t pte, unsigned long *rmapp, struct rmap_nested **n_rmap) { pgd_t *pgd; +p4d_t *p4d, *new_p4d =NULL ;pud_t *pud, *new_pud =NULL ;pmd_t *pmd, *new_pmd =NULL ;pte_t *ptep, *new_ptep =NULL ;@@ -575,9 +598,15 @@ int kvmppc_create_pte(struct kvm *kvm, pgd_t *pgtable, pte_t pte,/* Traverse the guest's 2nd-level tree, allocate new levels needed */ pgd = pgtable +pgd_index (gpa); - pud =NULL ; + p4d =NULL ;if (pgd_present (*pgd)) - pud =pud_offset (pgd, gpa); + p4d =p4d_offset (pgd, gpa); +else + new_p4d =p4d_alloc_one (kvm->mm, gpa); + + pud =NULL ; +if (p4d_present (*p4d)) + pud =pud_offset (p4d, gpa);else new_pud =pud_alloc_one (kvm->mm, gpa);@@ -599,12 +628,18 @@ int kvmppc_create_pte(struct kvm *kvm, pgd_t *pgtable, pte_t pte,/* Now traverse again under the lock and change the tree */ ret = -ENOMEM ;if (pgd_none (*pgd)) { +if (!new_p4d) + goto out_unlock; +pgd_populate (kvm->mm, pgd, new_p4d); + new_p4d =NULL ; + } +if (p4d_none (*p4d)) {if (!new_pud) goto out_unlock; -pgd_populate (kvm->mm, pgd, new_pud); +p4d_populate (kvm->mm, p4d, new_pud); new_pud =NULL ; } - pud =pud_offset (pgd, gpa); + pud =pud_offset (p4d, gpa);if (pud_is_leaf (*pud)) {unsigned long hgpa = gpa &PUD_MASK ;@@ -1222,6 +1257,7 @@ static ssize_t debugfs_radix_read(struct file *file, char __user *buf,pgd_t *pgt;struct kvm_nested_guest *nested;pgd_t pgd, *pgdp; +p4d_t p4d, *p4dp;pud_t pud, *pudp;pmd_t pmd, *pmdp;pte_t *ptep;@@ -1300,7 +1336,14 @@ static ssize_t debugfs_radix_read(struct file *file, char __user *buf,continue ; } - pudp =pud_offset (&pgd, gpa); + p4dp =p4d_offset (&pgd, gpa); + p4d =READ_ONCE (*p4dp); +if (!(p4d_val (p4d) & _PAGE_PRESENT)) { + gpa = (gpa &P4D_MASK ) +P4D_SIZE ; +continue ; + } + + pudp =pud_offset (&p4d, gpa); pud =READ_ONCE (*pudp);if (!(pud_val (pud) & _PAGE_PRESENT)) { gpa = (gpa &PUD_MASK ) +PUD_SIZE ;arch/powerpc/lib/code-patching.c +6
-1@@ -107,13 +107,18 @@ static inline int unmap_patch_area(unsigned long addr)pte_t *ptep;pmd_t *pmdp;pud_t *pudp; +p4d_t *p4dp;pgd_t *pgdp; pgdp =pgd_offset_k (addr);if (unlikely (!pgdp))return -EINVAL ; - pudp =pud_offset (pgdp, addr); + p4dp =p4d_offset (pgdp, addr); +if (unlikely (!p4dp)) +return -EINVAL ; + + pudp =pud_offset (p4dp, addr);if (unlikely (!pudp))return -EINVAL ;arch/powerpc/mm/book3s32/mmu.c +1
-1@@ -312,7 +312,7 @@ void hash_preload(struct mm_struct *mm, unsigned long ea)if (!Hash)return ; - pmd =pmd_offset (pud_offset (pgd_offset (mm, ea), ea), ea); + pmd =pmd_offset (pud_offset (p4d_offset (pgd_offset (mm, ea), ea), ea), ea);if (!pmd_none (*pmd))add_hash_page (mm->context.id, ea,pmd_val (*pmd)); }arch/powerpc/mm/book3s32/tlb.c +2
-2@@ -87,7 +87,7 @@ static void flush_range(struct mm_struct *mm, unsigned long start,if (start >= end)return ; end = (end -1 ) | ~PAGE_MASK ; - pmd =pmd_offset (pud_offset (pgd_offset (mm, start), start), start); + pmd =pmd_offset (pud_offset (p4d_offset (pgd_offset (mm, start), start), start), start);for (;;) { pmd_end = ((start +PGDIR_SIZE ) &PGDIR_MASK ) -1 ;if (pmd_end > end)@@ -145,7 +145,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)return ; } mm = (vmaddr <TASK_SIZE )? vma->vm_mm: &init_mm; - pmd =pmd_offset (pud_offset (pgd_offset (mm, vmaddr), vmaddr), vmaddr); + pmd =pmd_offset (pud_offset (p4d_offset (pgd_offset (mm, vmaddr), vmaddr), vmaddr), vmaddr);if (!pmd_none (*pmd))flush_hash_pages (mm->context.id, vmaddr,pmd_val (*pmd),1 ); }arch/powerpc/mm/book3s64/hash_pgtable.c +3
-1@@ -148,6 +148,7 @@ void hash__vmemmap_remove_mapping(unsigned long start,int hash__map_kernel_page (unsigned long ea,unsigned long pa,pgprot_t prot) {pgd_t *pgdp; +p4d_t *p4dp;pud_t *pudp;pmd_t *pmdp;pte_t *ptep;@@ -155,7 +156,8 @@ int hash__map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot)BUILD_BUG_ON (TASK_SIZE_USER64 >H_PGTABLE_RANGE );if (slab_is_available ()) { pgdp =pgd_offset_k (ea); - pudp =pud_alloc (&init_mm, pgdp, ea); + p4dp =p4d_offset (pgdp, ea); + pudp =pud_alloc (&init_mm, p4dp, ea);if (!pudp)return -ENOMEM ; pmdp =pmd_alloc (&init_mm, pudp, ea);arch/powerpc/mm/book3s64/radix_pgtable.c +15
-4@@ -64,17 +64,24 @@ static int early_map_kernel_page(unsigned long ea, unsigned long pa, {unsigned long pfn = pa >>PAGE_SHIFT ;pgd_t *pgdp; +p4d_t *p4dp;pud_t *pudp;pmd_t *pmdp;pte_t *ptep; pgdp =pgd_offset_k (ea);if (pgd_none (*pgdp)) { + p4dp =early_alloc_pgtable (PGD_TABLE_SIZE , nid, + region_start, region_end); +pgd_populate (&init_mm, pgdp, p4dp); + } + p4dp =p4d_offset (pgdp, ea); +if (p4d_none (*p4dp)) { pudp =early_alloc_pgtable (PUD_TABLE_SIZE , nid, region_start, region_end); -pgd_populate (&init_mm, pgdp, pudp); +p4d_populate (&init_mm, p4dp, pudp); } - pudp =pud_offset (pgdp, ea); + pudp =pud_offset (p4dp, ea);if (map_page_size ==PUD_SIZE ) { ptep = (pte_t *)pudp; goto set_the_pte;@@ -114,6 +121,7 @@ static int __map_kernel_page(unsigned long ea, unsigned long pa, {unsigned long pfn = pa >>PAGE_SHIFT ;pgd_t *pgdp; +p4d_t *p4dp;pud_t *pudp;pmd_t *pmdp;pte_t *ptep;@@ -136,7 +144,8 @@ static int __map_kernel_page(unsigned long ea, unsigned long pa,* boot. */ pgdp =pgd_offset_k (ea); - pudp =pud_alloc (&init_mm, pgdp, ea); + p4dp =p4d_offset (pgdp, ea); + pudp =pud_alloc (&init_mm, p4dp, ea);if (!pudp)return -ENOMEM ;if (map_page_size ==PUD_SIZE ) {@@ -173,6 +182,7 @@ void radix__change_memory_range(unsigned long start, unsigned long end, {unsigned long idx;pgd_t *pgdp; +p4d_t *p4dp;pud_t *pudp;pmd_t *pmdp;pte_t *ptep;@@ -185,7 +195,8 @@ void radix__change_memory_range(unsigned long start, unsigned long end,for (idx = start; idx < end; idx +=PAGE_SIZE ) { pgdp =pgd_offset_k (idx); - pudp =pud_alloc (&init_mm, pgdp, idx); + p4dp =p4d_offset (pgdp, idx); + pudp =pud_alloc (&init_mm, p4dp, idx);if (!pudp)continue ;if (pud_is_leaf (*pudp)) {arch/powerpc/mm/book3s64/subpage_prot.c +5
-1@@ -54,6 +54,7 @@ static void hpte_flush_range(struct mm_struct *mm, unsigned long addr, int npages) {pgd_t *pgd; +p4d_t *p4d;pud_t *pud;pmd_t *pmd;pte_t *pte;@@ -62,7 +63,10 @@ static void hpte_flush_range(struct mm_struct *mm, unsigned long addr, pgd =pgd_offset (mm, addr);if (pgd_none (*pgd))return ; - pud =pud_offset (pgd, addr); + p4d =p4d_offset (pgd, addr); +if (p4d_none (*p4d)) +return ; + pud =pud_offset (p4d, addr);if (pud_none (*pud))return ; pmd =pmd_offset (pud, addr);arch/powerpc/mm/hugetlbpage.c +16
-12@@ -112,6 +112,7 @@ static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp,pte_t *huge_pte_alloc (struct mm_struct *mm,unsigned long addr,unsigned long sz) {pgd_t *pg; +p4d_t *p4;pud_t *pu;pmd_t *pm;hugepd_t *hpdp =NULL ;@@ -121,20 +122,21 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz addr &= ~(sz-1 ); pg =pgd_offset (mm, addr); + p4 =p4d_offset (pg, addr);#ifdef CONFIG_PPC_BOOK3S_64 if (pshift ==PGDIR_SHIFT )/* 16GB huge page */ -return (pte_t *) pg; +return (pte_t *) p4;else if (pshift >PUD_SHIFT ) {/* * We need to use hugepd table */ ptl = &mm->page_table_lock; - hpdp = (hugepd_t *)pg; + hpdp = (hugepd_t *)p4; }else { pdshift =PUD_SHIFT ; - pu =pud_alloc (mm, pg, addr); + pu =pud_alloc (mm, p4, addr);if (!pu)return NULL ;if (pshift ==PUD_SHIFT )@@ -159,10 +161,10 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz#else if (pshift >=PGDIR_SHIFT ) { ptl = &mm->page_table_lock; - hpdp = (hugepd_t *)pg; + hpdp = (hugepd_t *)p4; }else { pdshift =PUD_SHIFT ; - pu =pud_alloc (mm, pg, addr); + pu =pud_alloc (mm, p4, addr);if (!pu)return NULL ;if (pshift >=PUD_SHIFT ) {@@ -384,7 +386,7 @@ static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud,mm_dec_nr_pmds (tlb->mm); } -static void hugetlb_free_pud_range (struct mmu_gather *tlb,pgd_t *pgd, +static void hugetlb_free_pud_range (struct mmu_gather *tlb,p4d_t *p4d,unsigned long addr,unsigned long end,unsigned long floor,unsigned long ceiling) {@@ -394,7 +396,7 @@ static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd, start = addr; do { - pud = pud_offset(pgd, addr); + pud = pud_offset(p4d, addr); next = pud_addr_end(addr, end); if (!is_hugepd(__hugepd(pud_val(*pud)))) { if (pud_none_or_clear_bad(pud))@@ -429,8 +431,8 @@ static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,if (end -1 > ceiling -1 )return ; - pud =pud_offset (pgd, start); -pgd_clear (pgd); + pud =pud_offset (p4d, start); +p4d_clear (p4d);pud_free_tlb (tlb, pud, start);mm_dec_nr_puds (tlb->mm); }@@ -443,6 +445,7 @@ void hugetlb_free_pgd_range(struct mmu_gather *tlb, unsigned long floor, unsigned long ceiling) {pgd_t *pgd; +p4d_t *p4d;unsigned long next;/* @@ -465,10 +468,11 @@ void hugetlb_free_pgd_range(struct mmu_gather *tlb, do { next = pgd_addr_end(addr, end); pgd = pgd_offset(tlb->mm, addr); + p4d = p4d_offset(pgd, addr); if (!is_hugepd(__hugepd(pgd_val(*pgd)))) { - if (pgd_none_or_clear_bad(pgd)) + if (p4d_none_or_clear_bad(p4d)) continue; - hugetlb_free_pud_range(tlb, pgd, addr, next, floor, ceiling); + hugetlb_free_pud_range(tlb, p4d, addr, next, floor, ceiling); } else { unsigned long more; /*@@ -481,7 +485,7 @@ void hugetlb_free_pgd_range(struct mmu_gather *tlb,if (more > next) next = more; -free_hugepd_range (tlb, (hugepd_t *)pgd,PGDIR_SHIFT , +free_hugepd_range (tlb, (hugepd_t *)p4d,PGDIR_SHIFT , addr, next, floor, ceiling); } }while (addr = next, addr != end);arch/powerpc/mm/kasan/kasan_init_32.c +4
-4@@ -36,7 +36,7 @@ static int __ref kasan_init_shadow_page_tables(unsigned long k_start, unsigned lunsigned long k_cur, k_next;pgprot_t prot =slab_is_available () ?kasan_prot_ro () :PAGE_KERNEL ; - pmd =pmd_offset (pud_offset (pgd_offset_k (k_start), k_start), k_start); + pmd =pmd_offset (pud_offset (p4d_offset (pgd_offset_k (k_start), k_start), k_start), k_start);for (k_cur = k_start; k_cur != k_end; k_cur = k_next, pmd++) {pte_t *new;@@ -94,7 +94,7 @@ static int __ref kasan_init_region(void *start, size_t size) block =memblock_alloc (k_end - k_start,PAGE_SIZE );for (k_cur = k_start &PAGE_MASK ; k_cur < k_end; k_cur +=PAGE_SIZE ) { -pmd_t *pmd =pmd_offset (pud_offset (pgd_offset_k (k_cur), k_cur), k_cur); +pmd_t *pmd =pmd_offset (pud_offset (p4d_offset (pgd_offset_k (k_cur), k_cur), k_cur), k_cur);void *va = block ? block + k_cur - k_start :kasan_get_one_page ();pte_t pte =pfn_pte (PHYS_PFN (__pa (va)),PAGE_KERNEL );@@ -118,7 +118,7 @@ static void __init kasan_remap_early_shadow_ro(void)kasan_populate_pte (kasan_early_shadow_pte, prot);for (k_cur = k_start &PAGE_MASK ; k_cur < k_end; k_cur +=PAGE_SIZE ) { -pmd_t *pmd =pmd_offset (pud_offset (pgd_offset_k (k_cur), k_cur), k_cur); +pmd_t *pmd =pmd_offset (pud_offset (p4d_offset (pgd_offset_k (k_cur), k_cur), k_cur), k_cur);pte_t *ptep =pte_offset_kernel (pmd, k_cur);if ((pte_val (*ptep) &PTE_RPN_MASK ) != pa)@@ -205,7 +205,7 @@ void __init kasan_early_init(void)unsigned long addr =KASAN_SHADOW_START ;unsigned long end =KASAN_SHADOW_END ;unsigned long next; -pmd_t *pmd =pmd_offset (pud_offset (pgd_offset_k (addr), addr), addr); +pmd_t *pmd =pmd_offset (pud_offset (p4d_offset (pgd_offset_k (addr), addr), addr), addr);BUILD_BUG_ON (KASAN_SHADOW_START & ~PGDIR_MASK );arch/powerpc/mm/mem.c +2
-2@@ -68,8 +68,8 @@ EXPORT_SYMBOL(kmap_prot);static inline pte_t *virt_to_kpte (unsigned long vaddr) { -return pte_offset_kernel (pmd_offset (pud_offset (pgd_offset_k (vaddr), - vaddr), vaddr), vaddr); +return pte_offset_kernel (pmd_offset (pud_offset (p4d_offset (pgd_offset_k (vaddr), + vaddr), vaddr), vaddr), vaddr); }#endif arch/powerpc/mm/nohash/40x.c +2
-2@@ -104,7 +104,7 @@ unsigned long __init mmu_mapin_ram(unsigned long base, unsigned long top)pmd_t *pmdp;unsigned long val = p | _PMD_SIZE_16M | _PAGE_EXEC | _PAGE_HWWRITE; - pmdp =pmd_offset (pud_offset (pgd_offset_k (v), v), v); + pmdp =pmd_offset (pud_offset (p4d_offset (pgd_offset_k (v), v), v), v); *pmdp++ =__pmd (val); *pmdp++ =__pmd (val); *pmdp++ =__pmd (val);@@ -119,7 +119,7 @@ unsigned long __init mmu_mapin_ram(unsigned long base, unsigned long top)pmd_t *pmdp;unsigned long val = p | _PMD_SIZE_4M | _PAGE_EXEC | _PAGE_HWWRITE; - pmdp =pmd_offset (pud_offset (pgd_offset_k (v), v), v); + pmdp =pmd_offset (pud_offset (p4d_offset (pgd_offset_k (v), v), v), v); *pmdp =__pmd (val); v +=LARGE_PAGE_SIZE_4M ;arch/powerpc/mm/nohash/book3e_pgtable.c +11
-4@@ -73,6 +73,7 @@ static void __init *early_alloc_pgtable(unsigned long size)int __ref map_kernel_page (unsigned long ea,unsigned long pa,pgprot_t prot) {pgd_t *pgdp; +p4d_t *p4dp;pud_t *pudp;pmd_t *pmdp;pte_t *ptep;@@ -80,7 +81,10 @@ int __ref map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot)BUILD_BUG_ON (TASK_SIZE_USER64 >PGTABLE_RANGE );if (slab_is_available ()) { pgdp =pgd_offset_k (ea); - pudp =pud_alloc (&init_mm, pgdp, ea); + p4dp =p4d_alloc (&init_mm, pgdp, ea); +if (!p4dp) +return -ENOMEM ; + pudp =pud_alloc (&init_mm, p4dp, ea);if (!pudp)return -ENOMEM ; pmdp =pmd_alloc (&init_mm, pudp, ea);@@ -91,13 +95,16 @@ int __ref map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot)return -ENOMEM ; }else { pgdp =pgd_offset_k (ea); -#ifndef __PAGETABLE_PUD_FOLDEDif (pgd_none (*pgdp)) { pudp =early_alloc_pgtable (PUD_TABLE_SIZE );pgd_populate (&init_mm, pgdp, pudp); } -#endif /* !__PAGETABLE_PUD_FOLDED */ - pudp =pud_offset (pgdp, ea); + p4dp =p4d_offset (pgdp, ea); +if (p4d_none (*p4dp)) { + pmdp =early_alloc_pgtable (PMD_TABLE_SIZE ); +p4d_populate (&init_mm, p4dp, pmdp); + } + pudp =pud_offset (p4dp, ea);if (pud_none (*pudp)) { pmdp =early_alloc_pgtable (PMD_TABLE_SIZE );pud_populate (&init_mm, pudp, pmdp);arch/powerpc/mm/pgtable.c +23
-2@@ -265,6 +265,7 @@ int huge_ptep_set_access_flags(struct vm_area_struct *vma,void assert_pte_locked (struct mm_struct *mm,unsigned long addr) {pgd_t *pgd; +p4d_t *p4d;pud_t *pud;pmd_t *pmd;@@ -272,7 +273,9 @@ void assert_pte_locked(struct mm_struct *mm, unsigned long addr)return ; pgd = mm->pgd +pgd_index (addr);BUG_ON (pgd_none (*pgd)); - pud =pud_offset (pgd, addr); + p4d =p4d_offset (pgd, addr); +BUG_ON (p4d_none (*p4d)); + pud =pud_offset (p4d, addr);BUG_ON (pud_none (*pud)); pmd =pmd_offset (pud, addr);/* @@ -313,6 +316,7 @@ pte_t *__find_linux_pte(pgd_t *pgdir, unsigned long ea, bool *is_thp, unsigned *hpage_shift) { pgd_t pgd, *pgdp; +p4d_t p4d, *p4dp;pud_t pud, *pudp;pmd_t pmd, *pmdp;pte_t *ret_pte;@@ -346,13 +350,30 @@ pte_t *__find_linux_pte(pgd_t *pgdir, unsigned long ea, goto out_huge; } + pdshift =P4D_SHIFT ; + p4dp =p4d_offset (&pgd, ea); + p4d =READ_ONCE (*p4dp); + +if (p4d_none (p4d)) +return NULL ; + +if (p4d_is_leaf (p4d)) { + ret_pte = (pte_t *)p4dp; + goto out; + } + +if (is_hugepd (__hugepd (p4d_val (p4d)))) { + hpdp = (hugepd_t *)&p4d; + goto out_huge; + } +/* * Even if we end up with an unmap, the pgtable will not * be freed, because we do an rcu free and here we are * irq disabled */ pdshift =PUD_SHIFT ; - pudp =pud_offset (&pgd, ea); + pudp =pud_offset (&p4d, ea); pud =READ_ONCE (*pudp);if (pud_none (pud))arch/powerpc/mm/pgtable_32.c +16
-12@@ -63,7 +63,7 @@ int __ref map_kernel_page(unsigned long va, phys_addr_t pa, pgprot_t prot)int err = -ENOMEM ;/* Use upper 10 bits of VA to index the first level map */ - pd =pmd_offset (pud_offset (pgd_offset_k (va), va), va); + pd =pmd_offset (pud_offset (p4d_offset (pgd_offset_k (va), va), va), va);/* Use middle 10 bits of VA to index the second-level map */ if (likely (slab_is_available ())) pg =pte_alloc_kernel (pd, va);@@ -130,6 +130,7 @@ static int get_pteptr(struct mm_struct *mm, unsigned long addr, pte_t **ptep, pmd_t **pmdp) { pgd_t *pgd; +p4d_t *p4d;pud_t *pud;pmd_t *pmd;pte_t *pte;@@ -137,17 +138,20 @@ get_pteptr(struct mm_struct *mm, unsigned long addr, pte_t **ptep, pmd_t **pmdp) pgd =pgd_offset (mm, addr &PAGE_MASK );if (pgd) { - pud =pud_offset (pgd, addr &PAGE_MASK ); -if (pud &&pud_present (*pud)) { - pmd =pmd_offset (pud, addr &PAGE_MASK ); -if (pmd_present (*pmd)) { - pte =pte_offset_map (pmd, addr &PAGE_MASK ); -if (pte) { - retval =1 ; - *ptep = pte; -if (pmdp) - *pmdp = pmd; -/* XXX caller needs to do pte_unmap, yuck */ + p4d =p4d_offset (pgd, addr &PAGE_MASK ); +if (p4d &&p4d_present (*p4d)) { + pud =pud_offset (p4d, addr &PAGE_MASK ); +if (pud &&pud_present (*pud)) { + pmd =pmd_offset (pud, addr &PAGE_MASK ); +if (pmd_present (*pmd)) { + pte =pte_offset_map (pmd, addr &PAGE_MASK ); +if (pte) { + retval =1 ; + *ptep = pte; +if (pmdp) + *pmdp = pmd; +/* XXX caller needs to do pte_unmap, yuck */ + } } } }arch/powerpc/mm/pgtable_64.c +5
-5@@ -101,13 +101,13 @@ EXPORT_SYMBOL(__pte_frag_size_shift);#ifndef __PAGETABLE_PUD_FOLDED/* 4 level page table */ -struct page *pgd_page (pgd_t pgd) +struct page *p4d_page (p4d_t p4d) { -if (pgd_is_leaf (pgd)) { -VM_WARN_ON (!pgd_huge (pgd)); -return pte_page (pgd_pte (pgd)); +if (p4d_is_leaf (p4d)) { +VM_WARN_ON (!p4d_huge (p4d)); +return pte_page (p4d_pte (p4d)); } -return virt_to_page (pgd_page_vaddr (pgd)); +return virt_to_page (p4d_page_vaddr (p4d)); }#endif arch/powerpc/mm/ptdump/hashpagetable.c +17
-3@@ -417,9 +417,9 @@ static void walk_pmd(struct pg_state *st, pud_t *pud, unsigned long start) } } -static void walk_pud (struct pg_state *st,pgd_t *pgd,unsigned long start) +static void walk_pud (struct pg_state *st,p4d_t *p4d,unsigned long start) { -pud_t *pud =pud_offset (pgd,0 ); +pud_t *pud =pud_offset (p4d,0 );unsigned long addr;unsigned int i;@@ -431,6 +431,20 @@ static void walk_pud(struct pg_state *st, pgd_t *pgd, unsigned long start) } } +static void walk_p4d (struct pg_state *st,pgd_t *pgd,unsigned long start) +{ +p4d_t *p4d =p4d_offset (pgd,0 ); +unsigned long addr; +unsigned int i; + +for (i =0 ; i <PTRS_PER_P4D ; i++, p4d++) { + addr = start + i *P4D_SIZE ; +if (!p4d_none (*p4d)) +/* p4d exists */ +walk_pud (st, p4d, addr); + } +} +static void walk_pagetables (struct pg_state *st) {pgd_t *pgd =pgd_offset_k (0UL );@@ -445,7 +459,7 @@ static void walk_pagetables(struct pg_state *st) addr =KERN_VIRT_START + i *PGDIR_SIZE ;if (!pgd_none (*pgd))/* pgd exists */ -walk_pud (st, pgd, addr); +walk_p4d (st, pgd, addr); } }arch/powerpc/mm/ptdump/ptdump.c +19
-3@@ -273,9 +273,9 @@ static void walk_pmd(struct pg_state *st, pud_t *pud, unsigned long start) } } -static void walk_pud (struct pg_state *st,pgd_t *pgd,unsigned long start) +static void walk_pud (struct pg_state *st,p4d_t *p4d,unsigned long start) { -pud_t *pud =pud_offset (pgd,0 ); +pud_t *pud =pud_offset (p4d,0 );unsigned long addr;unsigned int i;@@ -289,6 +289,22 @@ static void walk_pud(struct pg_state *st, pgd_t *pgd, unsigned long start) } } +static void walk_p4d (struct pg_state *st,pgd_t *pgd,unsigned long start) +{ +p4d_t *p4d =p4d_offset (pgd,0 ); +unsigned long addr; +unsigned int i; + +for (i =0 ; i <PTRS_PER_P4D ; i++, p4d++) { + addr = start + i *P4D_SIZE ; +if (!p4d_none (*p4d) && !p4d_is_leaf (*p4d)) +/* p4d exists */ +walk_pud (st, p4d, addr); +else +note_page (st, addr,2 ,p4d_val (*p4d)); + } +} +static void walk_pagetables (struct pg_state *st) {unsigned int i;@@ -302,7 +318,7 @@ static void walk_pagetables(struct pg_state *st)for (i =pgd_index (addr); i <PTRS_PER_PGD ; i++, pgd++, addr +=PGDIR_SIZE ) {if (!pgd_none (*pgd) && !pgd_is_leaf (*pgd))/* pgd exists */ -walk_pud (st, pgd, addr); +walk_p4d (st, pgd, addr);else note_page (st, addr,1 ,pgd_val (*pgd)); }arch/powerpc/xmon/xmon.c +16
-1@@ -3128,6 +3128,7 @@ static void show_pte(unsigned long addr)struct task_struct *tsk =NULL ;struct mm_struct *mm;pgd_t *pgdp, *pgdir; +p4d_t *p4dp;pud_t *pudp;pmd_t *pmdp;pte_t *ptep;@@ -3172,7 +3173,21 @@ static void show_pte(unsigned long addr) }printf ("pgdp @ 0x%px = 0x%016lx\n" , pgdp,pgd_val (*pgdp)); - pudp =pud_offset (pgdp, addr); + p4dp =p4d_offset (pgdp, addr); + +if (p4d_none (*p4dp)) { +printf ("No valid P4D\n" ); +return ; + } + +if (p4d_is_leaf (*p4dp)) { +format_pte (p4dp,p4d_val (*p4dp)); +return ; + } + +printf ("p4dp @ 0x%px = 0x%016lx\n" , p4dp,p4d_val (*p4dp)); + + pudp =pud_offset (p4dp, addr);if (pud_none (*pudp)) {printf ("No valid PUD\n" );Signature · 1 line
2.24.0
Signature · 2 lines
Sincerely yours, Mike.
Mike Rapoport <[email protected]> a écrit :
Quoted 1 line
Any updates on this?
Checkpatch reported several points, see https://patchwork.ozlabs.org/patch/1206344/
Christophe
Quoted 1,223 lines · Mike Rapoport
Quoted 1,215 lines
From: Mike Rapoport <[email protected]>
Implement primitives necessary for the 4th level folding, add walks of p4d level where appropriate and replace
5level-fixup.hwithpgtable-nop4d.h.Signed-off-by Mike Rapoport <[email protected]>--- arch/powerpc/include/asm/book3s/32/pgtable.h | 1 - arch/powerpc/include/asm/book3s/64/hash.h | 4 +- arch/powerpc/include/asm/book3s/64/pgalloc.h | 4 +- arch/powerpc/include/asm/book3s/64/pgtable.h | 58 ++++++++++-------- arch/powerpc/include/asm/book3s/64/radix.h | 6 +- arch/powerpc/include/asm/nohash/32/pgtable.h | 1 - arch/powerpc/include/asm/nohash/64/pgalloc.h | 2 +- .../include/asm/nohash/64/pgtable-4k.h | 32 +++++----- arch/powerpc/include/asm/nohash/64/pgtable.h | 6 +- arch/powerpc/include/asm/pgtable.h | 8 +++ arch/powerpc/kvm/book3s_64_mmu_radix.c | 59 ++++++++++++++++--- arch/powerpc/lib/code-patching.c | 7 ++- arch/powerpc/mm/book3s32/mmu.c | 2 +- arch/powerpc/mm/book3s32/tlb.c | 4 +- arch/powerpc/mm/book3s64/hash_pgtable.c | 4 +- arch/powerpc/mm/book3s64/radix_pgtable.c | 19 ++++-- arch/powerpc/mm/book3s64/subpage_prot.c | 6 +- arch/powerpc/mm/hugetlbpage.c | 28 +++++---- arch/powerpc/mm/kasan/kasan_init_32.c | 8 +-- arch/powerpc/mm/mem.c | 4 +- arch/powerpc/mm/nohash/40x.c | 4 +- arch/powerpc/mm/nohash/book3e_pgtable.c | 15 +++-- arch/powerpc/mm/pgtable.c | 25 +++++++- arch/powerpc/mm/pgtable_32.c | 28 +++++---- arch/powerpc/mm/pgtable_64.c | 10 ++-- arch/powerpc/mm/ptdump/hashpagetable.c | 20 ++++++- arch/powerpc/mm/ptdump/ptdump.c | 22 ++++++- arch/powerpc/xmon/xmon.c | 17 +++++- 28 files changed, 284 insertions(+), 120 deletions(-)arch/powerpc/include/asm/book3s/32/pgtable.h +0
-1@@ -2,7 +2,6 @@#ifndef _ASM_POWERPC_BOOK3S_32_PGTABLE_H#define _ASM_POWERPC_BOOK3S_32_PGTABLE_H -#define __ARCH_USE_5LEVEL_HACK#include <asm-generic/pgtable-nopmd.h> #include <asm/book3s/32/hash.h> arch/powerpc/include/asm/book3s/64/hash.h +2
-2@@ -134,9 +134,9 @@ static inline int get_region_id(unsigned long ea)#define hash__pmd_bad (pmd) (pmd_val(pmd) & H_PMD_BAD_BITS)#define hash__pud_bad (pud) (pud_val(pud) & H_PUD_BAD_BITS) -static inline int hash__pgd_bad (pgd_t pgd) +static inline int hash__p4d_bad (p4d_t p4d) { -return (pgd_val (pgd) ==0 ); +return (p4d_val (p4d) ==0 ); }#ifdef CONFIG_STRICT_KERNEL_RWX extern void hash__mark_rodata_ro (void );arch/powerpc/include/asm/book3s/64/pgalloc.h +2
-2@@ -87,9 +87,9 @@ static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)kmem_cache_free (PGT_CACHE (PGD_INDEX_SIZE ),pgd ); } -static inline void pgd_populate (struct mm_struct *mm,pgd_t *pgd, pud_t *pud) +static inline void p4d_populate (struct mm_struct *mm,p4d_t *pgd,pud_t *pud) { - *pgd =__pgd (__pgtable_ptr_val (pud) |PGD_VAL_BITS ); + *pgd =__p4d (__pgtable_ptr_val (pud) |PGD_VAL_BITS );}
static inline
pud_t*pud_alloc_one(structmm_struct*mm, unsigned
long addr)arch/powerpc/include/asm/book3s/64/pgtable.h +27
-21@@ -2,7 +2,7 @@#ifndef _ASM_POWERPC_BOOK3S_64_PGTABLE_H_#define _ASM_POWERPC_BOOK3S_64_PGTABLE_H_ -#include <asm-generic/5level-fixup.h> +#include <asm-generic/pgtable-nop4d.h> #ifndef __ASSEMBLY__#include <linux/mmdebug.h> @@ -251,7 +251,7 @@ extern unsigned long __pmd_frag_size_shift;/* Bits to mask out from a PUD to get to the PMD page */ #define PUD_MASKED_BITS 0xc0000000000000ffUL/* Bits to mask out from a PGD to get to the PUD page */ -#define PGD_MASKED_BITS 0xc0000000000000ffUL +#define P4D_MASKED_BITS 0xc0000000000000ffUL/* * Used as an indicator for rcu callback functions @@ -949,54 +949,60 @@ static inline bool pud_access_permitted(pud_t pud, bool write)return pte_access_permitted (pud_pte (pud), write); } -#define pgd_write (pgd) pte_write(pgd_pte(pgd)) +#define __p4d_raw (x) ((p4d_t) { __pgd_raw(x) }) +static inline __be64 p4d_raw (p4d_t x) +{ +return pgd_raw (x.pgd); +} + +#define p4d_write (p4d) pte_write(p4d_pte(p4d)) -static inline void pgd_clear (pgd_t *pgdp) +static inline void p4d_clear (p4d_t *p4dp) { - *pgdp =__pgd (0 ); + *p4dp =__p4d (0 ); } -static inline int pgd_none (pgd_t pgd) +static inline int p4d_none (p4d_t p4d) { -return !pgd_raw (pgd); +return !p4d_raw (p4d); } -static inline int pgd_present (pgd_t pgd) +static inline int p4d_present (p4d_t p4d) { -return !!(pgd_raw (pgd) &cpu_to_be64 (_PAGE_PRESENT)); +return !!(p4d_raw (p4d) &cpu_to_be64 (_PAGE_PRESENT)); } -static inline pte_t pgd_pte (pgd_t pgd) +static inline pte_t p4d_pte (p4d_t p4d) { -return __pte_raw (pgd_raw (pgd)); +return __pte_raw (p4d_raw (p4d)); } -static inline pgd_t pte_pgd (pte_t pte) +static inline p4d_t pte_p4d (pte_t pte) { -return __pgd_raw (pte_raw (pte)); +return __p4d_raw (pte_raw (pte)); } -static inline int pgd_bad (pgd_t pgd) +static inline int p4d_bad (p4d_t p4d) {if (radix_enabled ()) -return radix__pgd_bad (pgd); -return hash__pgd_bad (pgd); +return radix__p4d_bad (p4d); +return hash__p4d_bad (p4d); } -#define pgd_access_permitted pgd_access_permitted -static inline bool pgd_access_permitted (pgd_t pgd,bool write) +#define p4d_access_permitted p4d_access_permitted +static inline bool p4d_access_permitted (p4d_t p4d,bool write) { -return pte_access_permitted (pgd_pte (pgd), write); +return pte_access_permitted (p4d_pte (p4d), write); } -extern struct page *pgd_page (pgd_t pgd); +extern struct page *p4d_page (p4d_t p4d);/* Pointers in the page table tree are physical addresses */ #define __pgtable_ptr_val (ptr) __pa(ptr)#define pmd_page_vaddr (pmd) __va(pmd_val(pmd) & ~PMD_MASKED_BITS)#define pud_page_vaddr (pud) __va(pud_val(pud) & ~PUD_MASKED_BITS) -#define pgd_page_vaddr (pgd) __va(pgd_val(pgd) & ~PGD_MASKED_BITS) +#define p4d_page_vaddr (p4d) __va(p4d_val(p4d) & ~P4D_MASKED_BITS)#define pgd_index (address) (((address) >> (PGDIR_SHIFT)) &(
PTRS_PER_PGD- 1))
#definepud_index(address)(((address) >> (PUD_SHIFT)) &
(PTRS_PER_PUD- 1))arch/powerpc/include/asm/book3s/64/pgtable.h +8
-2@@ -1010,8 +1016,8 @@ extern struct page *pgd_page(pgd_t pgd);#define pgd_offset (mm, address) ((mm)->pgd + pgd_index(address)) -#define pud_offset (pgdp, addr) \ - (((pud_t *) pgd_page_vaddr(*(pgdp))) + pud_index(addr)) +#define pud_offset (p4dp, addr) \ + (((pud_t *) p4d_page_vaddr(*(p4dp))) + pud_index(addr))#define pmd_offset (pudp,addr) \ (((pmd_t *) pud_page_vaddr(*(pudp))) + pmd_index(addr))#define pte_offset_kernel (dir,addr) \@@ -1366,6 +1372,12 @@ static inline bool pud_is_leaf(pud_t pud)return !!(pud_raw (pud) &cpu_to_be64 (_PAGE_PTE)); } +#define p4d_is_leaf p4d_is_leaf +static inline bool p4d_is_leaf (p4d_t p4d) +{ +return !!(p4d_raw (p4d) &cpu_to_be64 (_PAGE_PTE)); +} +#define pgd_is_leaf pgd_is_leafstatic inline bool pgd_is_leaf (pgd_t pgd) {arch/powerpc/include/asm/book3s/64/radix.h +3
-3@@ -30,7 +30,7 @@/* Don't have anything in the reserved bits and leaf bits */ #define RADIX_PMD_BAD_BITS 0x60000000000000e0UL#define RADIX_PUD_BAD_BITS 0x60000000000000e0UL -#define RADIX_PGD_BAD_BITS 0x60000000000000e0UL +#define RADIX_P4D_BAD_BITS 0x60000000000000e0UL#define RADIX_PMD_SHIFT (PAGE_SHIFT + RADIX_PTE_INDEX_SIZE)#define RADIX_PUD_SHIFT (RADIX_PMD_SHIFT + RADIX_PMD_INDEX_SIZE)@@ -227,9 +227,9 @@ static inline int radix__pud_bad(pud_t pud) } -static inline int radix__pgd_bad (pgd_t pgd) +static inline int radix__p4d_bad (p4d_t p4d) { -return !!(pgd_val (pgd) &RADIX_PGD_BAD_BITS ); +return !!(p4d_val (p4d) &RADIX_P4D_BAD_BITS ); }#ifdef CONFIG_TRANSPARENT_HUGEPAGE arch/powerpc/include/asm/nohash/32/pgtable.h +0
-1@@ -2,7 +2,6 @@#ifndef _ASM_POWERPC_NOHASH_32_PGTABLE_H#define _ASM_POWERPC_NOHASH_32_PGTABLE_H -#define __ARCH_USE_5LEVEL_HACK#include <asm-generic/pgtable-nopmd.h> #ifndef __ASSEMBLY__arch/powerpc/include/asm/nohash/64/pgalloc.h +1
-1@@ -15,7 +15,7 @@ struct vmemmap_backing { };extern struct vmemmap_backing *vmemmap_list; -#define pgd_populate (MM ,PGD ,PUD ) pgd_set(PGD, (unsigned long)PUD) +#define p4d_populate (MM ,P4D ,PUD ) p4d_set(P4D, (unsigned long)PUD)static inline pud_t *pud_alloc_one (struct mm_struct *mm,unsigned long addr){
arch/powerpc/include/asm/nohash/64/pgtable-4k.h +16
-16@@ -2,7 +2,7 @@#ifndef _ASM_POWERPC_NOHASH_64_PGTABLE_4K_H#define _ASM_POWERPC_NOHASH_64_PGTABLE_4K_H -#include <asm-generic/5level-fixup.h> +#include <asm-generic/pgtable-nop4d.h> /* * Entries per page directory level. The PTE level must use a 64b record @@ -45,41 +45,41 @@#define PMD_MASKED_BITS 0/* Bits to mask out from a PUD to get to the PMD page */ #define PUD_MASKED_BITS 0 -/* Bits to mask out from a PGD to get to the PUD page */ -#define PGD_MASKED_BITS 0 +/* Bits to mask out from a P4D to get to the PUD page */ +#define P4D_MASKED_BITS 0/* * 4-level page tables related bits */ -#define pgd_none (pgd) (!pgd_val(pgd)) -#define pgd_bad (pgd) (pgd_val(pgd) == 0) -#define pgd_present (pgd) (pgd_val(pgd) != 0) -#define pgd_page_vaddr (pgd) (pgd_val(pgd) & ~PGD_MASKED_BITS) +#define p4d_none (p4d) (!p4d_val(p4d)) +#define p4d_bad (p4d) (p4d_val(p4d) == 0) +#define p4d_present (p4d) (p4d_val(p4d) != 0) +#define p4d_page_vaddr (p4d) (p4d_val(p4d) & ~P4D_MASKED_BITS)#ifndef __ASSEMBLY__ -static inline void pgd_clear (pgd_t *pgdp) +static inline void p4d_clear (p4d_t *p4dp) { - *pgdp =__pgd (0 ); + *p4dp =__p4d (0 ); } -static inline pte_t pgd_pte (pgd_t pgd) +static inline pte_t p4d_pte (p4d_t p4d) { -return __pte (pgd_val (pgd)); +return __pte (p4d_val (p4d)); } -static inline pgd_t pte_pgd (pte_t pte) +static inline p4d_t pte_p4d (pte_t pte) { -return __pgd (pte_val (pte)); +return __p4d (pte_val (pte)); } -extern struct page *pgd_page (pgd_t pgd); +extern struct page *p4d_page (p4d_t p4d);#endif /* !__ASSEMBLY__ */ -#define pud_offset (pgdp, addr) \ - (((pud_t *) pgd_page_vaddr(*(pgdp))) + \ +#define pud_offset (p4dp, addr) \ + (((pud_t *) p4d_page_vaddr(*(p4dp))) + \ (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1)))#define pud_ERROR (e) \arch/powerpc/include/asm/nohash/64/pgtable.h +3
-3@@ -175,11 +175,11 @@ static inline pud_t pte_pud(pte_t pte)return __pud (pte_val (pte)); }#define pud_write (pud) pte_write(pud_pte(pud)) -#define pgd_write (pgd) pte_write(pgd_pte(pgd)) +#define p4d_write (pgd) pte_write(p4d_pte(p4d)) -static inline void pgd_set (pgd_t *pgdp,unsigned long val) +static inline void p4d_set (p4d_t *p4dp,unsigned long val) { - *pgdp =__pgd (val); + *p4dp =__p4d (val); }/* arch/powerpc/include/asm/pgtable.h +8
-0@@ -145,6 +145,14 @@ static inline bool pud_is_leaf(pud_t pud) }#endif +#ifndef p4d_is_leaf +#define p4d_is_leaf p4d_is_leaf +static inline bool p4d_is_leaf (p4d_t p4d) +{ +return false; +} +#endif +#ifndef pgd_is_leaf#define pgd_is_leaf pgd_is_leafstatic inline bool pgd_is_leaf (pgd_t pgd)arch/powerpc/kvm/book3s_64_mmu_radix.c +25
-3@@ -496,17 +496,39 @@ static void kvmppc_unmap_free_pud(struct kvm *kvm, pud_t *pud,pud_free (kvm->mm, pud); } +static void kvmppc_unmap_free_p4d (struct kvm *kvm,p4d_t *p4d, +unsigned int lpid) +{ +unsigned long iu; +p4d_t *p = p4d; + +for (iu =0 ; iu <PTRS_PER_P4D ; ++iu, ++p) { +if (!p4d_present (*p)) +continue ; +if (p4d_is_leaf (*p)) { +p4d_clear (p); + }else { +pud_t *pud; + + pud =pud_offset (p,0 ); +kvmppc_unmap_free_pud (kvm, pud, lpid); +p4d_clear (p); + } + } +p4d_free (kvm->mm, p4d); +} +void kvmppc_free_pgtable_radix (struct kvm *kvm,pgd_t *pgd,unsigned int lpid) {unsigned long ig;for (ig =0 ; ig <PTRS_PER_PGD ; ++ig, ++pgd) { -pud_t *pud; +p4d_t *p4d;if (!pgd_present (*pgd))continue ; - pud =pud_offset (pgd,0 ); -kvmppc_unmap_free_pud (kvm, pud, lpid); + p4d =p4d_offset (pgd,0 ); +kvmppc_unmap_free_p4d (kvm, p4d, lpid);pgd_clear (pgd);arch/powerpc/kvm/book3s_64_mmu_radix.c +1
-0} }@@ -568,6 +590,7 @@ int kvmppc_create_pte(struct kvm *kvm, pgd_t *pgtable, pte_t pte, unsigned long *rmapp, struct rmap_nested **n_rmap) {pgd_t *pgd; +p4d_t *p4d, *new_p4d =NULL ;pud_t *pud, *new_pud =NULL ;pmd_t *pmd, *new_pmd =NULL ;arch/powerpc/kvm/book3s_64_mmu_radix.c +17
-4pte_t *ptep, *new_ptep =NULL ;@@ -575,9 +598,15 @@ int kvmppc_create_pte(struct kvm *kvm, pgd_t *pgtable, pte_t pte,/* Traverse the guest's 2nd-level tree, allocate new levels needed */ pgd = pgtable +pgd_index (gpa); - pud =NULL ; + p4d =NULL ;if (pgd_present (*pgd)) - pud =pud_offset (pgd, gpa); + p4d =p4d_offset (pgd, gpa); +else + new_p4d =p4d_alloc_one (kvm->mm, gpa); + + pud =NULL ; +if (p4d_present (*p4d)) + pud =pud_offset (p4d, gpa);else new_pud =pud_alloc_one (kvm->mm, gpa);@@ -599,12 +628,18 @@ int kvmppc_create_pte(struct kvm *kvm, pgd_t *pgtable, pte_t pte,/* Now traverse again under the lock and change the tree */ ret = -ENOMEM ;if (pgd_none (*pgd)) { +if (!new_p4d) + goto out_unlock; +pgd_populate (kvm->mm, pgd, new_p4d); + new_p4d =NULL ; + } +if (p4d_none (*p4d)) {if (!new_pud) goto out_unlock; -pgd_populate (kvm->mm, pgd, new_pud); +p4d_populate (kvm->mm, p4d, new_pud); new_pud =NULL ; } - pud =pud_offset (pgd, gpa); + pud =pud_offset (p4d, gpa);if (pud_is_leaf (*pud)) {unsigned long hgpa = gpa &PUD_MASK ;@@ -1222,6 +1257,7 @@ static ssize_t debugfs_radix_read(struct file *file, char__user *buf, pgd_t *pgt;struct kvm_nested_guest *nested;pgd_t pgd, *pgdp; +p4d_t p4d, *p4dp;pud_t pud, *pudp;pmd_t pmd, *pmdp;arch/powerpc/kvm/book3s_64_mmu_radix.c +8
-1pte_t *ptep;@@ -1300,7 +1336,14 @@ static ssize_t debugfs_radix_read(struct file *file, char__user *buf, continue; } - pudp =pud_offset (&pgd, gpa); + p4dp =p4d_offset (&pgd, gpa); + p4d =READ_ONCE (*p4dp); +if (!(p4d_val (p4d) & _PAGE_PRESENT)) { + gpa = (gpa &P4D_MASK ) +P4D_SIZE ; +continue ; + } + + pudp =pud_offset (&p4d, gpa); pud =READ_ONCE (*pudp);if (!(pud_val (pud) & _PAGE_PRESENT)) {arch/powerpc/kvm/book3s_64_mmu_radix.c +0
-0gpa = (gpa &PUD_MASK ) +PUD_SIZE ;arch/powerpc/lib/code-patching.c +6
-1@@ -107,13 +107,18 @@ static inline int unmap_patch_area(unsigned long addr)pte_t *ptep;pmd_t *pmdp;pud_t *pudp; +p4d_t *p4dp;pgd_t *pgdp; pgdp =pgd_offset_k (addr);if (unlikely (!pgdp))return -EINVAL ; - pudp =pud_offset (pgdp, addr); + p4dp =p4d_offset (pgdp, addr); +if (unlikely (!p4dp)) +return -EINVAL ; + + pudp =pud_offset (p4dp, addr);if (unlikely (!pudp))return -EINVAL ;arch/powerpc/mm/book3s32/mmu.c +1
-1@@ -312,7 +312,7 @@ void hash_preload(struct mm_struct *mm, unsigned long ea)if (!Hash)return ; - pmd =pmd_offset (pud_offset (pgd_offset (mm, ea), ea), ea); + pmd =pmd_offset (pud_offset (p4d_offset (pgd_offset (mm, ea), ea), ea), ea);if (!pmd_none (*pmd))add_hash_page (mm->context.id, ea,pmd_val (*pmd));}
arch/powerpc/mm/book3s32/tlb.c +1
-1@@ -87,7 +87,7 @@ static void flush_range(struct mm_struct *mm,unsigned long start, if (start >= end) return; end = (end - 1) | ~PAGE_MASK; - pmd =pmd_offset (pud_offset (pgd_offset (mm, start), start), start); + pmd =pmd_offset (pud_offset (p4d_offset (pgd_offset (mm, start), start), start), start);for (;;) {arch/powerpc/mm/book3s32/tlb.c +1
-1pmd_end = ((start +PGDIR_SIZE ) &PGDIR_MASK ) -1 ;if (pmd_end > end)@@ -145,7 +145,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr) return; } mm = (vmaddr <TASK_SIZE )? vma->vm_mm: &init_mm; - pmd =pmd_offset (pud_offset (pgd_offset (mm, vmaddr), vmaddr), vmaddr); + pmd =pmd_offset (pud_offset (p4d_offset (pgd_offset (mm, vmaddr), vmaddr), vmaddr), vmaddr);if (!pmd_none (*pmd))arch/powerpc/mm/book3s32/tlb.c +0
-0flush_hash_pages (mm->context.id, vmaddr,pmd_val (*pmd),1 ); }arch/powerpc/mm/book3s64/hash_pgtable.c +1
-0@@ -148,6 +148,7 @@ void hash__vmemmap_remove_mapping(unsigned long start,int hash__map_kernel_page (unsigned long ea,unsigned long pa,pgprot_t prot) {pgd_t *pgdp; +p4d_t *p4dp;pud_t *pudp;pmd_t *pmdp;arch/powerpc/mm/book3s64/hash_pgtable.c +2
-1pte_t *ptep;@@ -155,7 +156,8 @@ int hash__map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot) BUILD_BUG_ON(TASK_SIZE_USER64 > H_PGTABLE_RANGE);if (slab_is_available ()) { pgdp =pgd_offset_k (ea); - pudp =pud_alloc (&init_mm, pgdp, ea); + p4dp =p4d_offset (pgdp, ea); + pudp =pud_alloc (&init_mm, p4dp, ea);if (!pudp)return -ENOMEM ;arch/powerpc/mm/book3s64/hash_pgtable.c +0
-0pmdp =pmd_alloc (&init_mm, pudp, ea);arch/powerpc/mm/book3s64/radix_pgtable.c +9
-2@@ -64,17 +64,24 @@ static int early_map_kernel_page(unsigned long ea, unsigned long pa, {unsigned long pfn = pa >>PAGE_SHIFT ;pgd_t *pgdp; +p4d_t *p4dp;pud_t *pudp;pmd_t *pmdp;pte_t *ptep; pgdp =pgd_offset_k (ea);if (pgd_none (*pgdp)) { + p4dp =early_alloc_pgtable (PGD_TABLE_SIZE , nid, + region_start, region_end); +pgd_populate (&init_mm, pgdp, p4dp); + } + p4dp =p4d_offset (pgdp, ea); +if (p4d_none (*p4dp)) { pudp =early_alloc_pgtable (PUD_TABLE_SIZE , nid, region_start, region_end); -pgd_populate (&init_mm, pgdp, pudp); +p4d_populate (&init_mm, p4dp, pudp); } - pudp =pud_offset (pgdp, ea); + pudp =pud_offset (p4dp, ea);if (map_page_size ==PUD_SIZE ) { ptep = (pte_t *)pudp;arch/powerpc/mm/book3s64/radix_pgtable.c +1
-0goto set_the_pte;@@ -114,6 +121,7 @@ static int __map_kernel_page(unsigned long ea, unsigned long pa, {unsigned long pfn = pa >>PAGE_SHIFT ;pgd_t *pgdp; +p4d_t *p4dp;pud_t *pudp;pmd_t *pmdp;arch/powerpc/mm/book3s64/radix_pgtable.c +2
-1pte_t *ptep;@@ -136,7 +144,8 @@ static int __map_kernel_page(unsigned long ea,unsigned long pa, * boot. */ pgdp =pgd_offset_k (ea); - pudp =pud_alloc (&init_mm, pgdp, ea); + p4dp =p4d_offset (pgdp, ea); + pudp =pud_alloc (&init_mm, p4dp, ea);if (!pudp)return -ENOMEM ;arch/powerpc/mm/book3s64/radix_pgtable.c +1
-0if (map_page_size ==PUD_SIZE ) {@@ -173,6 +182,7 @@ void radix__change_memory_range(unsigned long start, unsigned long end, {unsigned long idx;pgd_t *pgdp; +p4d_t *p4dp;pud_t *pudp;pmd_t *pmdp;arch/powerpc/mm/book3s64/radix_pgtable.c +2
-1pte_t *ptep;@@ -185,7 +195,8 @@ void radix__change_memory_range(unsigned long start, unsigned long end, for (idx = start; idx < end; idx += PAGE_SIZE) { pgdp =pgd_offset_k (idx); - pudp =pud_alloc (&init_mm, pgdp, idx); + p4dp =p4d_offset (pgdp, idx); + pudp =pud_alloc (&init_mm, p4dp, idx);if (!pudp)continue ;arch/powerpc/mm/book3s64/radix_pgtable.c +0
-0if (pud_is_leaf (*pudp)) {arch/powerpc/mm/book3s64/subpage_prot.c +1
-0@@ -54,6 +54,7 @@ static void hpte_flush_range(struct mm_struct *mm, unsigned long addr, int npages) {pgd_t *pgd; +p4d_t *p4d;pud_t *pud;pmd_t *pmd;arch/powerpc/mm/book3s64/subpage_prot.c +4
-1pte_t *pte;@@ -62,7 +63,10 @@ static void hpte_flush_range(struct mm_struct *mm, unsigned long addr, pgd =pgd_offset (mm, addr);if (pgd_none (*pgd))return ; - pud =pud_offset (pgd, addr); + p4d =p4d_offset (pgd, addr); +if (p4d_none (*p4d)) +return ; + pud =pud_offset (p4d, addr);if (pud_none (*pud))return ;arch/powerpc/mm/book3s64/subpage_prot.c +0
-0pmd =pmd_offset (pud, addr);arch/powerpc/mm/hugetlbpage.c +1
-0@@ -112,6 +112,7 @@ static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp, pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz) {pgd_t *pg; +p4d_t *p4;pud_t *pu;arch/powerpc/mm/hugetlbpage.c +0
-0pmd_t *pm;hugepd_t *hpdp =NULL ;@@ -121,20 +122,21 @@ pte_t *huge_pte_alloc(struct mm_struct *mm,unsigned long addr, unsigned long sz
arch/powerpc/mm/hugetlbpage.c +4
-3addr &= ~(sz-1 ); pg =pgd_offset (mm, addr); + p4 =p4d_offset (pg, addr);#ifdef CONFIG_PPC_BOOK3S_64 if (pshift ==PGDIR_SHIFT )/* 16GB huge page */ -return (pte_t *) pg; +return (pte_t *) p4;else if (pshift >PUD_SHIFT ) {/* * We need to use hugepd table */ ptl = &mm->page_table_lock; - hpdp = (hugepd_t *)pg; + hpdp = (hugepd_t *)p4; }else { pdshift =PUD_SHIFT ; - pu =pud_alloc (mm, pg, addr); + pu =pud_alloc (mm, p4, addr);if (!pu)return NULL ;if (pshift ==PUD_SHIFT )@@ -159,10 +161,10 @@ pte_t *huge_pte_alloc(struct mm_struct *mm,unsigned long addr, unsigned long sz
#elsearch/powerpc/mm/hugetlbpage.c +3
-3if (pshift >=PGDIR_SHIFT ) { ptl = &mm->page_table_lock; - hpdp = (hugepd_t *)pg; + hpdp = (hugepd_t *)p4; }else { pdshift =PUD_SHIFT ; - pu =pud_alloc (mm, pg, addr); + pu =pud_alloc (mm, p4, addr);if (!pu)return NULL ;if (pshift >=PUD_SHIFT ) {@@ -384,7 +386,7 @@ static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud,mm_dec_nr_pmds (tlb->mm); } -static void hugetlb_free_pud_range (struct mmu_gather *tlb,pgd_t *pgd, +static void hugetlb_free_pud_range (struct mmu_gather *tlb,p4d_t *p4d,unsigned long addr,unsigned long end,unsigned long floor,unsigned long ceiling){
arch/powerpc/mm/hugetlbpage.c +1
-1@@ -394,7 +396,7 @@ static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd, start = addr; do { - pud = pud_offset(pgd, addr); + pud = pud_offset(p4d, addr); next = pud_addr_end(addr, end); if (!is_hugepd(__hugepd(pud_val(*pud)))) {arch/powerpc/mm/hugetlbpage.c +2
-2if (pud_none_or_clear_bad (pud))@@ -429,8 +431,8 @@ static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd, if (end - 1 > ceiling - 1) return; - pud =pud_offset (pgd, start); -pgd_clear (pgd); + pud =pud_offset (p4d, start); +p4d_clear (p4d);pud_free_tlb (tlb, pud, start);mm_dec_nr_puds (tlb->mm);}
arch/powerpc/mm/hugetlbpage.c +5
-3@@ -443,6 +445,7 @@ void hugetlb_free_pgd_range(struct mmu_gather *tlb, unsigned long floor, unsigned long ceiling) {pgd_t *pgd; +p4d_t *p4d;unsigned long next;/* @@ -465,10 +468,11 @@ void hugetlb_free_pgd_range(struct mmu_gather *tlb, do { next = pgd_addr_end(addr, end); pgd = pgd_offset(tlb->mm, addr); + p4d = p4d_offset(pgd, addr); if (!is_hugepd(__hugepd(pgd_val(*pgd)))) { - if (pgd_none_or_clear_bad(pgd)) + if (p4d_none_or_clear_bad(p4d)) continue; - hugetlb_free_pud_range(tlb, pgd, addr, next, floor, ceiling); + hugetlb_free_pud_range(tlb, p4d, addr, next, floor, ceiling); } else { unsigned long more; /*@@ -481,7 +485,7 @@ void hugetlb_free_pgd_range(struct mmu_gather *tlb,if (more > next) next = more; -free_hugepd_range (tlb, (hugepd_t *)pgd,PGDIR_SHIFT , +free_hugepd_range (tlb, (hugepd_t *)p4d,PGDIR_SHIFT , addr, next, floor, ceiling); } }while (addr = next, addr != end);arch/powerpc/mm/kasan/kasan_init_32.c +0
-0@@ -36,7 +36,7 @@ static int __refkasan_init_shadow_page_tables(unsigned long
k_start, unsigned larch/powerpc/mm/kasan/kasan_init_32.c +1
-1unsigned long k_cur, k_next;pgprot_t prot =slab_is_available () ?kasan_prot_ro () :PAGE_KERNEL ; - pmd =pmd_offset (pud_offset (pgd_offset_k (k_start), k_start), k_start); + pmd = pmd_offset(pud_offset(p4d_offset(pgd_offset_k(k_start),
k_start),k_start),k_start);arch/powerpc/mm/kasan/kasan_init_32.c +1
-1for (k_cur = k_start; k_cur != k_end; k_cur = k_next, pmd++) {pte_t *new;@@ -94,7 +94,7 @@ static int __ref kasan_init_region(void *start, size_t size) block =memblock_alloc (k_end - k_start,PAGE_SIZE );for (k_cur = k_start &PAGE_MASK ; k_cur < k_end; k_cur +=PAGE_SIZE ) { -pmd_t *pmd =pmd_offset (pud_offset (pgd_offset_k (k_cur), k_cur), k_cur); +pmd_t *pmd =pmd_offset (pud_offset (p4d_offset (pgd_offset_k (k_cur), k_cur), k_cur), k_cur);void *va = block ? block + k_cur - k_start :kasan_get_one_page ();arch/powerpc/mm/kasan/kasan_init_32.c +1
-1pte_t pte =pfn_pte (PHYS_PFN (__pa (va)),PAGE_KERNEL );@@ -118,7 +118,7 @@ static void __init kasan_remap_early_shadow_ro(void)kasan_populate_pte (kasan_early_shadow_pte, prot);for (k_cur = k_start &PAGE_MASK ; k_cur < k_end; k_cur +=PAGE_SIZE ) { -pmd_t *pmd =pmd_offset (pud_offset (pgd_offset_k (k_cur), k_cur), k_cur); +pmd_t *pmd =pmd_offset (pud_offset (p4d_offset (pgd_offset_k (k_cur), k_cur), k_cur), k_cur);pte_t *ptep =pte_offset_kernel (pmd, k_cur);arch/powerpc/mm/kasan/kasan_init_32.c +1
-1if ((pte_val (*ptep) &PTE_RPN_MASK ) != pa)@@ -205,7 +205,7 @@ void __init kasan_early_init(void)unsigned long addr =KASAN_SHADOW_START ;unsigned long end =KASAN_SHADOW_END ;unsigned long next; -pmd_t *pmd =pmd_offset (pud_offset (pgd_offset_k (addr), addr), addr); +pmd_t *pmd =pmd_offset (pud_offset (p4d_offset (pgd_offset_k (addr), addr), addr), addr);BUILD_BUG_ON (KASAN_SHADOW_START & ~PGDIR_MASK );arch/powerpc/mm/mem.c +2
-2@@ -68,8 +68,8 @@ EXPORT_SYMBOL(kmap_prot);static inline pte_t *virt_to_kpte (unsigned long vaddr) { -return pte_offset_kernel (pmd_offset (pud_offset (pgd_offset_k (vaddr), - vaddr), vaddr), vaddr); +return pte_offset_kernel (pmd_offset (pud_offset (p4d_offset (pgd_offset_k (vaddr), + vaddr), vaddr), vaddr), vaddr); }#endif arch/powerpc/mm/nohash/40x.c +1
-1@@ -104,7 +104,7 @@ unsigned long __init mmu_mapin_ram(unsigned long base, unsigned long top) pmd_t *pmdp;unsigned long val = p | _PMD_SIZE_16M | _PAGE_EXEC | _PAGE_HWWRITE; - pmdp =pmd_offset (pud_offset (pgd_offset_k (v), v), v); + pmdp =pmd_offset (pud_offset (p4d_offset (pgd_offset_k (v), v), v), v); *pmdp++ =__pmd (val); *pmdp++ =__pmd (val);arch/powerpc/mm/nohash/40x.c +1
-1*pmdp++ =__pmd (val);@@ -119,7 +119,7 @@ unsigned long __init mmu_mapin_ram(unsigned long base, unsigned long top) pmd_t *pmdp;unsigned long val = p | _PMD_SIZE_4M | _PAGE_EXEC | _PAGE_HWWRITE; - pmdp =pmd_offset (pud_offset (pgd_offset_k (v), v), v); + pmdp =pmd_offset (pud_offset (p4d_offset (pgd_offset_k (v), v), v), v); *pmdp =__pmd (val);arch/powerpc/mm/nohash/40x.c +0
-0v +=LARGE_PAGE_SIZE_4M ;arch/powerpc/mm/nohash/book3e_pgtable.c +1
-0@@ -73,6 +73,7 @@ static void __init *early_alloc_pgtable(unsigned long size)int __ref map_kernel_page (unsigned long ea,unsigned long pa,pgprot_t prot) {pgd_t *pgdp; +p4d_t *p4dp;pud_t *pudp;arch/powerpc/mm/nohash/book3e_pgtable.c +4
-1pmd_t *pmdp;pte_t *ptep;@@ -80,7 +81,10 @@ int __ref map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot) BUILD_BUG_ON(TASK_SIZE_USER64 > PGTABLE_RANGE);if (slab_is_available ()) { pgdp =pgd_offset_k (ea); - pudp =pud_alloc (&init_mm, pgdp, ea); + p4dp =p4d_alloc (&init_mm, pgdp, ea); +if (!p4dp) +return -ENOMEM ; + pudp =pud_alloc (&init_mm, p4dp, ea);if (!pudp)return -ENOMEM ;arch/powerpc/mm/nohash/book3e_pgtable.c +6
-3pmdp =pmd_alloc (&init_mm, pudp, ea);@@ -91,13 +95,16 @@ int __ref map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot) return -ENOMEM; }else { pgdp =pgd_offset_k (ea); -#ifndef __PAGETABLE_PUD_FOLDEDif (pgd_none (*pgdp)) { pudp =early_alloc_pgtable (PUD_TABLE_SIZE );pgd_populate (&init_mm, pgdp, pudp); } -#endif /* !__PAGETABLE_PUD_FOLDED */ - pudp =pud_offset (pgdp, ea); + p4dp =p4d_offset (pgdp, ea); +if (p4d_none (*p4dp)) { + pmdp =early_alloc_pgtable (PMD_TABLE_SIZE ); +p4d_populate (&init_mm, p4dp, pmdp); + } + pudp =pud_offset (p4dp, ea);if (pud_none (*pudp)) { pmdp =early_alloc_pgtable (PMD_TABLE_SIZE );arch/powerpc/mm/nohash/book3e_pgtable.c +0
-0pud_populate (&init_mm, pudp, pmdp);arch/powerpc/mm/pgtable.c +4
-1@@ -265,6 +265,7 @@ int huge_ptep_set_access_flags(structvm_area_struct *vma, void assert_pte_locked(struct mm_struct *mm, unsigned long addr) { pgd_t *pgd; +p4d_t *p4d;pud_t *pud;pmd_t *pmd;@@ -272,7 +273,9 @@ void assert_pte_locked(struct mm_struct *mm, unsigned long addr) return; pgd = mm->pgd +pgd_index (addr);BUG_ON (pgd_none (*pgd)); - pud =pud_offset (pgd, addr); + p4d =p4d_offset (pgd, addr); +BUG_ON (p4d_none (*p4d)); + pud =pud_offset (p4d, addr);BUG_ON (pud_none (*pud)); pmd =pmd_offset (pud, addr);arch/powerpc/mm/pgtable.c +19
-1/* @@ -313,6 +316,7 @@ pte_t *__find_linux_pte(pgd_t *pgdir, unsigned long ea, bool *is_thp, unsigned *hpage_shift) { pgd_t pgd, *pgdp; +p4d_t p4d, *p4dp;pud_t pud, *pudp;pmd_t pmd, *pmdp;pte_t *ret_pte;@@ -346,13 +350,30 @@ pte_t *__find_linux_pte(pgd_t *pgdir,unsigned long ea, goto out_huge; } + pdshift =P4D_SHIFT ; + p4dp =p4d_offset (&pgd, ea); + p4d =READ_ONCE (*p4dp); + +if (p4d_none (p4d)) +return NULL ; + +if (p4d_is_leaf (p4d)) { + ret_pte = (pte_t *)p4dp; + goto out; + } + +if (is_hugepd (__hugepd (p4d_val (p4d)))) { + hpdp = (hugepd_t *)&p4d; + goto out_huge; + } +/* * Even if we end up with an unmap, the pgtable will not * be freed, because we do an rcu free and here we are * irq disabled */ pdshift =PUD_SHIFT ; - pudp =pud_offset (&pgd, ea); + pudp =pud_offset (&p4d, ea); pud =READ_ONCE (*pudp);arch/powerpc/mm/pgtable.c +0
-0if (pud_none (pud))arch/powerpc/mm/pgtable_32.c +1
-1@@ -63,7 +63,7 @@ int __ref map_kernel_page(unsigned long va, phys_addr_t pa, pgprot_t prot) int err = -ENOMEM;/* Use upper 10 bits of VA to index the first level map */ - pd =pmd_offset (pud_offset (pgd_offset_k (va), va), va); + pd =pmd_offset (pud_offset (p4d_offset (pgd_offset_k (va), va), va), va);/* Use middle 10 bits of VA to index the second-level map */ if (likely (slab_is_available ()))arch/powerpc/mm/pgtable_32.c +1
-0pg =pte_alloc_kernel (pd, va);@@ -130,6 +130,7 @@ static int get_pteptr(struct mm_struct *mm, unsigned long addr, pte_t **ptep, pmd_t **pmdp) { pgd_t *pgd; +p4d_t *p4d;pud_t *pud;pmd_t *pmd;pte_t *pte;arch/powerpc/mm/pgtable_32.c +14
-11@@ -137,17 +138,20 @@ get_pteptr(struct mm_struct *mm, unsigned long addr, pte_t **ptep, pmd_t **pmdp) pgd =pgd_offset (mm, addr &PAGE_MASK );if (pgd) { - pud =pud_offset (pgd, addr &PAGE_MASK ); -if (pud &&pud_present (*pud)) { - pmd =pmd_offset (pud, addr &PAGE_MASK ); -if (pmd_present (*pmd)) { - pte =pte_offset_map (pmd, addr &PAGE_MASK ); -if (pte) { - retval =1 ; - *ptep = pte; -if (pmdp) - *pmdp = pmd; -/* XXX caller needs to do pte_unmap, yuck */ + p4d =p4d_offset (pgd, addr &PAGE_MASK ); +if (p4d &&p4d_present (*p4d)) { + pud =pud_offset (p4d, addr &PAGE_MASK ); +if (pud &&pud_present (*pud)) { + pmd =pmd_offset (pud, addr &PAGE_MASK ); +if (pmd_present (*pmd)) { + pte =pte_offset_map (pmd, addr &PAGE_MASK ); +if (pte) { + retval =1 ; + *ptep = pte; +if (pmdp) + *pmdp = pmd; +/* XXX caller needs to do pte_unmap, yuck */ + } } }arch/powerpc/mm/pgtable_32.c +0
-0}arch/powerpc/mm/pgtable_64.c +5
-5@@ -101,13 +101,13 @@ EXPORT_SYMBOL(__pte_frag_size_shift);#ifndef __PAGETABLE_PUD_FOLDED/* 4 level page table */ -struct page *pgd_page (pgd_t pgd) +struct page *p4d_page (p4d_t p4d) { -if (pgd_is_leaf (pgd)) { -VM_WARN_ON (!pgd_huge (pgd)); -return pte_page (pgd_pte (pgd)); +if (p4d_is_leaf (p4d)) { +VM_WARN_ON (!p4d_huge (p4d)); +return pte_page (p4d_pte (p4d)); } -return virt_to_page (pgd_page_vaddr (pgd)); +return virt_to_page (p4d_page_vaddr (p4d)); }#endif arch/powerpc/mm/ptdump/hashpagetable.c +16
-2@@ -417,9 +417,9 @@ static void walk_pmd(struct pg_state *st, pud_t *pud, unsigned long start) } } -static void walk_pud (struct pg_state *st,pgd_t *pgd,unsigned long start) +static void walk_pud (struct pg_state *st,p4d_t *p4d,unsigned long start) { -pud_t *pud =pud_offset (pgd,0 ); +pud_t *pud =pud_offset (p4d,0 );unsigned long addr;unsigned int i;@@ -431,6 +431,20 @@ static void walk_pud(struct pg_state *st, pgd_t *pgd, unsigned long start) } } +static void walk_p4d (struct pg_state *st,pgd_t *pgd,unsigned long start) +{ +p4d_t *p4d =p4d_offset (pgd,0 ); +unsigned long addr; +unsigned int i; + +for (i =0 ; i <PTRS_PER_P4D ; i++, p4d++) { + addr = start + i *P4D_SIZE ; +if (!p4d_none (*p4d)) +/* p4d exists */ +walk_pud (st, p4d, addr); + } +} +static void walk_pagetables (struct pg_state *st) {arch/powerpc/mm/ptdump/hashpagetable.c +1
-1pgd_t *pgd =pgd_offset_k (0UL );@@ -445,7 +459,7 @@ static void walk_pagetables(struct pg_state *st) addr =KERN_VIRT_START + i *PGDIR_SIZE ;if (!pgd_none (*pgd))/* pgd exists */ -walk_pud (st, pgd, addr); +walk_p4d (st, pgd, addr); } }arch/powerpc/mm/ptdump/ptdump.c +18
-2@@ -273,9 +273,9 @@ static void walk_pmd(struct pg_state *st, pud_t *pud, unsigned long start) } } -static void walk_pud (struct pg_state *st,pgd_t *pgd,unsigned long start) +static void walk_pud (struct pg_state *st,p4d_t *p4d,unsigned long start) { -pud_t *pud =pud_offset (pgd,0 ); +pud_t *pud =pud_offset (p4d,0 );unsigned long addr;unsigned int i;@@ -289,6 +289,22 @@ static void walk_pud(struct pg_state *st, pgd_t *pgd, unsigned long start) } } +static void walk_p4d (struct pg_state *st,pgd_t *pgd,unsigned long start) +{ +p4d_t *p4d =p4d_offset (pgd,0 ); +unsigned long addr; +unsigned int i; + +for (i =0 ; i <PTRS_PER_P4D ; i++, p4d++) { + addr = start + i *P4D_SIZE ; +if (!p4d_none (*p4d) && !p4d_is_leaf (*p4d)) +/* p4d exists */ +walk_pud (st, p4d, addr); +else +note_page (st, addr,2 ,p4d_val (*p4d)); + } +} +static void walk_pagetables (struct pg_state *st) {arch/powerpc/mm/ptdump/ptdump.c +1
-1unsigned int i;@@ -302,7 +318,7 @@ static void walk_pagetables(struct pg_state *st)for (i =pgd_index (addr); i <PTRS_PER_PGD ; i++, pgd++, addr +=PGDIR_SIZE ) {if (!pgd_none (*pgd) && !pgd_is_leaf (*pgd))/* pgd exists */ -walk_pud (st, pgd, addr); +walk_p4d (st, pgd, addr);else note_page (st, addr,1 ,pgd_val (*pgd));arch/powerpc/mm/ptdump/ptdump.c +0
-0}arch/powerpc/xmon/xmon.c +16
-2@@ -3128,6 +3128,7 @@ static void show_pte(unsigned long addr)struct task_struct *tsk =NULL ;struct mm_struct *mm;pgd_t *pgdp, *pgdir; +p4d_t *p4dp;pud_t *pudp;pmd_t *pmdp;pte_t *ptep;@@ -3172,7 +3173,21 @@ static void show_pte(unsigned long addr) }printf ("pgdp @ 0x%px = 0x%016lx\n" , pgdp,pgd_val (*pgdp)); - pudp =pud_offset (pgdp, addr); + p4dp =p4d_offset (pgdp, addr); + +if (p4d_none (*p4dp)) { +printf ("No valid P4D\n" ); +return ; + } + +if (p4d_is_leaf (*p4dp)) { +format_pte (p4dp,p4d_val (*p4dp)); +return ; + } + +printf ("p4dp @ 0x%px = 0x%016lx\n" , p4dp,p4d_val (*p4dp)); + + pudp =pud_offset (p4dp, addr);if (pud_none (*pudp)) {printf ("No valid PUD\n" ); --Quoted 2 lines
2.24.0
--
Sincerely yours,
Mike.
Quoted 6 lines · Christophe Leroy
Mike Rapoport <[email protected]> a écrit :
Quoted 1 line
Any updates on this?
Checkpatch reported several points, see
https://patchwork.ozlabs.org/patch/1206344/
Well, for the most part checkpatch is unhappy because I've tried to keep the changes consistent with the old code. And, there are some lines over 90 characters that do no seem worth breaking.
Quoted 2 lines
Christophe
Michael Ellerman
reviewerQuoted 1 line · Mike Rapoport
Any updates on this?
It's very ... big, and kind of intrusive.
It's not an improvement as far as the powerpc code's readability is concerned. I assume the plan is that the 5-level hack can eventually be removed and so this conversion is a prerequisite for that?
cheers
Quoted 37 lines · Mike Rapoport
Quoted 36 lines
From: Mike Rapoport <[email protected]>
Implement primitives necessary for the 4th level folding, add walks of p4d level where appropriate and replace
5level-fixup.hwithpgtable-nop4d.h.Signed-off-by Mike Rapoport <[email protected]>--- arch/powerpc/include/asm/book3s/32/pgtable.h | 1 - arch/powerpc/include/asm/book3s/64/hash.h | 4 +- arch/powerpc/include/asm/book3s/64/pgalloc.h | 4 +- arch/powerpc/include/asm/book3s/64/pgtable.h | 58 ++++++++++-------- arch/powerpc/include/asm/book3s/64/radix.h | 6 +- arch/powerpc/include/asm/nohash/32/pgtable.h | 1 - arch/powerpc/include/asm/nohash/64/pgalloc.h | 2 +- .../include/asm/nohash/64/pgtable-4k.h | 32 +++++----- arch/powerpc/include/asm/nohash/64/pgtable.h | 6 +- arch/powerpc/include/asm/pgtable.h | 8 +++ arch/powerpc/kvm/book3s_64_mmu_radix.c | 59 ++++++++++++++++--- arch/powerpc/lib/code-patching.c | 7 ++- arch/powerpc/mm/book3s32/mmu.c | 2 +- arch/powerpc/mm/book3s32/tlb.c | 4 +- arch/powerpc/mm/book3s64/hash_pgtable.c | 4 +- arch/powerpc/mm/book3s64/radix_pgtable.c | 19 ++++-- arch/powerpc/mm/book3s64/subpage_prot.c | 6 +- arch/powerpc/mm/hugetlbpage.c | 28 +++++---- arch/powerpc/mm/kasan/kasan_init_32.c | 8 +-- arch/powerpc/mm/mem.c | 4 +- arch/powerpc/mm/nohash/40x.c | 4 +- arch/powerpc/mm/nohash/book3e_pgtable.c | 15 +++-- arch/powerpc/mm/pgtable.c | 25 +++++++- arch/powerpc/mm/pgtable_32.c | 28 +++++---- arch/powerpc/mm/pgtable_64.c | 10 ++-- arch/powerpc/mm/ptdump/hashpagetable.c | 20 ++++++- arch/powerpc/mm/ptdump/ptdump.c | 22 ++++++- arch/powerpc/xmon/xmon.c | 17 +++++- 28 files changed, 284 insertions(+), 120 deletions(-)
Quoted 4 lines · Michael Ellerman
Quoted 1 line · Mike Rapoport
Any updates on this?
It's very ... big, and kind of intrusive.
I've tried to split it to several smaller ones, but I couldn't find a way to do it without breaking bisectability.
As for the intrusive, most of the changes are mechanical
Quoted 3 lines
It's not an improvement as far as the powerpc code's readability is concerned. I assume the plan is that the 5-level hack can eventually be removed and so this conversion is a prerequisite for that?
Yep, the idea is to remove asm-generic/pgtable-nop4d-hack.h and asm-generic/5level-fixup.h.
And, there is a small benefit for powerpc as kernel size is reduced a bit, e.g. for pseries_defconfig:
$ size old/vmlinux new/vmlinux text data bss dec hex filename 13702955 5961022 1517520 21181497 1433439 old/vmlinux 13702507 5961006 1517520 21181033 1433269 new/vmlinux
Quoted 40 lines
cheers
Quoted 37 lines · Mike Rapoport
Quoted 36 lines
From: Mike Rapoport <[email protected]>
Implement primitives necessary for the 4th level folding, add walks of p4d level where appropriate and replace
5level-fixup.hwithpgtable-nop4d.h.Signed-off-by Mike Rapoport <[email protected]>--- arch/powerpc/include/asm/book3s/32/pgtable.h | 1 - arch/powerpc/include/asm/book3s/64/hash.h | 4 +- arch/powerpc/include/asm/book3s/64/pgalloc.h | 4 +- arch/powerpc/include/asm/book3s/64/pgtable.h | 58 ++++++++++-------- arch/powerpc/include/asm/book3s/64/radix.h | 6 +- arch/powerpc/include/asm/nohash/32/pgtable.h | 1 - arch/powerpc/include/asm/nohash/64/pgalloc.h | 2 +- .../include/asm/nohash/64/pgtable-4k.h | 32 +++++----- arch/powerpc/include/asm/nohash/64/pgtable.h | 6 +- arch/powerpc/include/asm/pgtable.h | 8 +++ arch/powerpc/kvm/book3s_64_mmu_radix.c | 59 ++++++++++++++++--- arch/powerpc/lib/code-patching.c | 7 ++- arch/powerpc/mm/book3s32/mmu.c | 2 +- arch/powerpc/mm/book3s32/tlb.c | 4 +- arch/powerpc/mm/book3s64/hash_pgtable.c | 4 +- arch/powerpc/mm/book3s64/radix_pgtable.c | 19 ++++-- arch/powerpc/mm/book3s64/subpage_prot.c | 6 +- arch/powerpc/mm/hugetlbpage.c | 28 +++++---- arch/powerpc/mm/kasan/kasan_init_32.c | 8 +-- arch/powerpc/mm/mem.c | 4 +- arch/powerpc/mm/nohash/40x.c | 4 +- arch/powerpc/mm/nohash/book3e_pgtable.c | 15 +++-- arch/powerpc/mm/pgtable.c | 25 +++++++- arch/powerpc/mm/pgtable_32.c | 28 +++++---- arch/powerpc/mm/pgtable_64.c | 10 ++-- arch/powerpc/mm/ptdump/hashpagetable.c | 20 ++++++- arch/powerpc/mm/ptdump/ptdump.c | 22 ++++++- arch/powerpc/xmon/xmon.c | 17 +++++- 28 files changed, 284 insertions(+), 120 deletions(-)
Signature · 2 lines
Sincerely yours, Mike.
Michael Ellerman
reviewerQuoted 8 lines · Mike Rapoport
Quoted 4 lines · Michael Ellerman
Quoted 1 line · Mike Rapoport
Any updates on this?
It's very ... big, and kind of intrusive.
I've tried to split it to several smaller ones, but I couldn't find a way to do it without breaking bisectability.
Yeah I didn't necessarily mean splitting it, it's just a lot of churn.
It seems to break qemu mac99 booting pmac32 defconfig, haven't had time to look any further:
Loading compiled-in X.509 certificates
rtc-generic rtc-generic: setting system clock to 2020-01-07T02:42:59 UTC (1578364979)
BUG: Unable to handle kernel unaligned access at 0xf10af004
Faulting instruction address: 0xc01cc6c4
Vector: 600 (Alignment) at [ef0b5de0]
pc: c01cc6c4: f_dupfd+0x6c/0xb8
lr: c01cc698: f_dupfd+0x40/0xb8
sp: ef0b5e98
msr: 9032
dar: f10af004
dsisr: 40000140
current = 0xef0b0000
pid = 1, comm = swapper
Linux version 5.5.0-rc2+ (michael@alpine1-p1) (gcc version 9.2.1 20191127 (Ubuntu 9.2.1-20ubuntu3)) #8 Tue Jan 7 13:38:04 AEDT 2020
enter ? for help
[ef0b5eb8] c000550c console_on_rootfs+0x44/0x90
[ef0b5ed8] c090d7c0 kernel_init_freeable+0x1a4/0x24c
[ef0b5f18] c0005770 kernel_init+0x18/0x108
[ef0b5f38] c0017274 ret_from_kernel_thread+0x14/0x1c
FAIL! Booting BE pmac32cheers
Le 05/01/2020 à 08:02, Mike Rapoport a écrit :
Quoted 12 lines · Christophe Leroy
Quoted 6 lines
Mike Rapoport <[email protected]> a écrit :
Quoted 1 line
Any updates on this?
Checkpatch reported several points, see
https://patchwork.ozlabs.org/patch/1206344/Well, for the most part checkpatch is unhappy because I've tried to keep the changes consistent with the old code. And, there are some lines over 90 characters that do no seem worth breaking.
Yes I see, many places we have stuff like:
pmd = pmd_offset(pud_offset(pgd_offset(mm, start), start), start);
I sent a patch to refactor that. This should help fixing that long line issue and reducing the number of lines impacted by your patch.
Christophe